diff options
Diffstat (limited to 'arch/sparc64')
-rw-r--r-- | arch/sparc64/kernel/Makefile | 2 | ||||
-rw-r--r-- | arch/sparc64/kernel/binfmt_aout32.c | 13 | ||||
-rw-r--r-- | arch/sparc64/kernel/dtlb_backend.S | 170 | ||||
-rw-r--r-- | arch/sparc64/kernel/dtlb_base.S | 109 | ||||
-rw-r--r-- | arch/sparc64/kernel/dtlb_miss.S | 39 | ||||
-rw-r--r-- | arch/sparc64/kernel/etrap.S | 2 | ||||
-rw-r--r-- | arch/sparc64/kernel/head.S | 46 | ||||
-rw-r--r-- | arch/sparc64/kernel/itlb_base.S | 79 | ||||
-rw-r--r-- | arch/sparc64/kernel/itlb_miss.S | 39 | ||||
-rw-r--r-- | arch/sparc64/kernel/ktlb.S | 263 | ||||
-rw-r--r-- | arch/sparc64/kernel/process.c | 25 | ||||
-rw-r--r-- | arch/sparc64/kernel/rtrap.S | 6 | ||||
-rw-r--r-- | arch/sparc64/kernel/smp.c | 12 | ||||
-rw-r--r-- | arch/sparc64/kernel/trampoline.S | 33 | ||||
-rw-r--r-- | arch/sparc64/kernel/tsb.S | 169 | ||||
-rw-r--r-- | arch/sparc64/kernel/ttable.S | 6 | ||||
-rw-r--r-- | arch/sparc64/kernel/vmlinux.lds.S | 3 | ||||
-rw-r--r-- | arch/sparc64/kernel/winfixup.S | 8 | ||||
-rw-r--r-- | arch/sparc64/mm/Makefile | 2 | ||||
-rw-r--r-- | arch/sparc64/mm/init.c | 91 | ||||
-rw-r--r-- | arch/sparc64/mm/tlb.c | 61 | ||||
-rw-r--r-- | arch/sparc64/mm/tsb.c | 84 | ||||
-rw-r--r-- | arch/sparc64/mm/ultra.S | 58 |
23 files changed, 489 insertions, 831 deletions
diff --git a/arch/sparc64/kernel/Makefile b/arch/sparc64/kernel/Makefile index 83d67eb18895..a482a9ffe5bc 100644 --- a/arch/sparc64/kernel/Makefile +++ b/arch/sparc64/kernel/Makefile | |||
@@ -38,5 +38,5 @@ else | |||
38 | CMODEL_CFLAG := -m64 -mcmodel=medlow | 38 | CMODEL_CFLAG := -m64 -mcmodel=medlow |
39 | endif | 39 | endif |
40 | 40 | ||
41 | head.o: head.S ttable.S itlb_base.S dtlb_base.S dtlb_backend.S dtlb_prot.S \ | 41 | head.o: head.S ttable.S itlb_miss.S dtlb_miss.S ktlb.S tsb.S \ |
42 | etrap.S rtrap.S winfixup.S entry.S | 42 | etrap.S rtrap.S winfixup.S entry.S |
diff --git a/arch/sparc64/kernel/binfmt_aout32.c b/arch/sparc64/kernel/binfmt_aout32.c index 202a80c24b6f..a57d7f2b6f13 100644 --- a/arch/sparc64/kernel/binfmt_aout32.c +++ b/arch/sparc64/kernel/binfmt_aout32.c | |||
@@ -31,6 +31,7 @@ | |||
31 | #include <asm/system.h> | 31 | #include <asm/system.h> |
32 | #include <asm/uaccess.h> | 32 | #include <asm/uaccess.h> |
33 | #include <asm/pgalloc.h> | 33 | #include <asm/pgalloc.h> |
34 | #include <asm/mmu_context.h> | ||
34 | 35 | ||
35 | static int load_aout32_binary(struct linux_binprm *, struct pt_regs * regs); | 36 | static int load_aout32_binary(struct linux_binprm *, struct pt_regs * regs); |
36 | static int load_aout32_library(struct file*); | 37 | static int load_aout32_library(struct file*); |
@@ -329,15 +330,9 @@ beyond_if: | |||
329 | 330 | ||
330 | current->mm->start_stack = | 331 | current->mm->start_stack = |
331 | (unsigned long) create_aout32_tables((char __user *)bprm->p, bprm); | 332 | (unsigned long) create_aout32_tables((char __user *)bprm->p, bprm); |
332 | if (!(orig_thr_flags & _TIF_32BIT)) { | 333 | tsb_context_switch(__pa(current->mm->pgd), |
333 | unsigned long pgd_cache = get_pgd_cache(current->mm->pgd); | 334 | current->mm->context.sparc64_tsb); |
334 | 335 | ||
335 | __asm__ __volatile__("stxa\t%0, [%1] %2\n\t" | ||
336 | "membar #Sync" | ||
337 | : /* no outputs */ | ||
338 | : "r" (pgd_cache), | ||
339 | "r" (TSB_REG), "i" (ASI_DMMU)); | ||
340 | } | ||
341 | start_thread32(regs, ex.a_entry, current->mm->start_stack); | 336 | start_thread32(regs, ex.a_entry, current->mm->start_stack); |
342 | if (current->ptrace & PT_PTRACED) | 337 | if (current->ptrace & PT_PTRACED) |
343 | send_sig(SIGTRAP, current, 0); | 338 | send_sig(SIGTRAP, current, 0); |
diff --git a/arch/sparc64/kernel/dtlb_backend.S b/arch/sparc64/kernel/dtlb_backend.S deleted file mode 100644 index acc889a7f9c1..000000000000 --- a/arch/sparc64/kernel/dtlb_backend.S +++ /dev/null | |||
@@ -1,170 +0,0 @@ | |||
1 | /* $Id: dtlb_backend.S,v 1.16 2001/10/09 04:02:11 davem Exp $ | ||
2 | * dtlb_backend.S: Back end to DTLB miss replacement strategy. | ||
3 | * This is included directly into the trap table. | ||
4 | * | ||
5 | * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com) | ||
6 | * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz) | ||
7 | */ | ||
8 | |||
9 | #include <asm/pgtable.h> | ||
10 | #include <asm/mmu.h> | ||
11 | |||
12 | #define VALID_SZ_BITS (_PAGE_VALID | _PAGE_SZBITS) | ||
13 | |||
14 | #define VPTE_BITS (_PAGE_CP | _PAGE_CV | _PAGE_P ) | ||
15 | #define VPTE_SHIFT (PAGE_SHIFT - 3) | ||
16 | |||
17 | /* Ways we can get here: | ||
18 | * | ||
19 | * 1) Nucleus loads and stores to/from PA-->VA direct mappings at tl>1. | ||
20 | * 2) Nucleus loads and stores to/from user/kernel window save areas. | ||
21 | * 3) VPTE misses from dtlb_base and itlb_base. | ||
22 | * | ||
23 | * We need to extract out the PMD and PGDIR indexes from the | ||
24 | * linear virtual page table access address. The PTE index | ||
25 | * is at the bottom, but we are not concerned with it. Bits | ||
26 | * 0 to 2 are clear since each PTE is 8 bytes in size. Each | ||
27 | * PMD and PGDIR entry are 4 bytes in size. Thus, this | ||
28 | * address looks something like: | ||
29 | * | ||
30 | * |---------------------------------------------------------------| | ||
31 | * | ... | PGDIR index | PMD index | PTE index | | | ||
32 | * |---------------------------------------------------------------| | ||
33 | * 63 F E D C B A 3 2 0 <- bit nr | ||
34 | * | ||
35 | * The variable bits above are defined as: | ||
36 | * A --> 3 + (PAGE_SHIFT - log2(8)) | ||
37 | * --> 3 + (PAGE_SHIFT - 3) - 1 | ||
38 | * (ie. this is "bit 3" + PAGE_SIZE - size of PTE entry in bits - 1) | ||
39 | * B --> A + 1 | ||
40 | * C --> B + (PAGE_SHIFT - log2(4)) | ||
41 | * --> B + (PAGE_SHIFT - 2) - 1 | ||
42 | * (ie. this is "bit B" + PAGE_SIZE - size of PMD entry in bits - 1) | ||
43 | * D --> C + 1 | ||
44 | * E --> D + (PAGE_SHIFT - log2(4)) | ||
45 | * --> D + (PAGE_SHIFT - 2) - 1 | ||
46 | * (ie. this is "bit D" + PAGE_SIZE - size of PGDIR entry in bits - 1) | ||
47 | * F --> E + 1 | ||
48 | * | ||
49 | * (Note how "B" always evalutes to PAGE_SHIFT, all the other constants | ||
50 | * cancel out.) | ||
51 | * | ||
52 | * For 8K PAGE_SIZE (thus, PAGE_SHIFT of 13) the bit numbers are: | ||
53 | * A --> 12 | ||
54 | * B --> 13 | ||
55 | * C --> 23 | ||
56 | * D --> 24 | ||
57 | * E --> 34 | ||
58 | * F --> 35 | ||
59 | * | ||
60 | * For 64K PAGE_SIZE (thus, PAGE_SHIFT of 16) the bit numbers are: | ||
61 | * A --> 15 | ||
62 | * B --> 16 | ||
63 | * C --> 29 | ||
64 | * D --> 30 | ||
65 | * E --> 43 | ||
66 | * F --> 44 | ||
67 | * | ||
68 | * Because bits both above and below each PGDIR and PMD index need to | ||
69 | * be masked out, and the index can be as long as 14 bits (when using a | ||
70 | * 64K PAGE_SIZE, and thus a PAGE_SHIFT of 16), we need 3 instructions | ||
71 | * to extract each index out. | ||
72 | * | ||
73 | * Shifts do not pair very well on UltraSPARC-I, II, IIi, and IIe, so | ||
74 | * we try to avoid using them for the entire operation. We could setup | ||
75 | * a mask anywhere from bit 31 down to bit 10 using the sethi instruction. | ||
76 | * | ||
77 | * We need a mask covering bits B --> C and one covering D --> E. | ||
78 | * For 8K PAGE_SIZE these masks are 0x00ffe000 and 0x7ff000000. | ||
79 | * For 64K PAGE_SIZE these masks are 0x3fff0000 and 0xfffc0000000. | ||
80 | * The second in each set cannot be loaded with a single sethi | ||
81 | * instruction, because the upper bits are past bit 32. We would | ||
82 | * need to use a sethi + a shift. | ||
83 | * | ||
84 | * For the time being, we use 2 shifts and a simple "and" mask. | ||
85 | * We shift left to clear the bits above the index, we shift down | ||
86 | * to clear the bits below the index (sans the log2(4 or 8) bits) | ||
87 | * and a mask to clear the log2(4 or 8) bits. We need therefore | ||
88 | * define 4 shift counts, all of which are relative to PAGE_SHIFT. | ||
89 | * | ||
90 | * Although unsupportable for other reasons, this does mean that | ||
91 | * 512K and 4MB page sizes would be generaally supported by the | ||
92 | * kernel. (ELF binaries would break with > 64K PAGE_SIZE since | ||
93 | * the sections are only aligned that strongly). | ||
94 | * | ||
95 | * The operations performed for extraction are thus: | ||
96 | * | ||
97 | * ((X << FOO_SHIFT_LEFT) >> FOO_SHIFT_RIGHT) & ~0x3 | ||
98 | * | ||
99 | */ | ||
100 | |||
101 | #define A (3 + (PAGE_SHIFT - 3) - 1) | ||
102 | #define B (A + 1) | ||
103 | #define C (B + (PAGE_SHIFT - 2) - 1) | ||
104 | #define D (C + 1) | ||
105 | #define E (D + (PAGE_SHIFT - 2) - 1) | ||
106 | #define F (E + 1) | ||
107 | |||
108 | #define PMD_SHIFT_LEFT (64 - D) | ||
109 | #define PMD_SHIFT_RIGHT (64 - (D - B) - 2) | ||
110 | #define PGDIR_SHIFT_LEFT (64 - F) | ||
111 | #define PGDIR_SHIFT_RIGHT (64 - (F - D) - 2) | ||
112 | #define LOW_MASK_BITS 0x3 | ||
113 | |||
114 | /* TLB1 ** ICACHE line 1: tl1 DTLB and quick VPTE miss */ | ||
115 | ldxa [%g1 + %g1] ASI_DMMU, %g4 ! Get TAG_ACCESS | ||
116 | add %g3, %g3, %g5 ! Compute VPTE base | ||
117 | cmp %g4, %g5 ! VPTE miss? | ||
118 | bgeu,pt %xcc, 1f ! Continue here | ||
119 | andcc %g4, TAG_CONTEXT_BITS, %g5 ! tl0 miss Nucleus test | ||
120 | ba,a,pt %xcc, from_tl1_trap ! Fall to tl0 miss | ||
121 | 1: sllx %g6, VPTE_SHIFT, %g4 ! Position TAG_ACCESS | ||
122 | or %g4, %g5, %g4 ! Prepare TAG_ACCESS | ||
123 | |||
124 | /* TLB1 ** ICACHE line 2: Quick VPTE miss */ | ||
125 | mov TSB_REG, %g1 ! Grab TSB reg | ||
126 | ldxa [%g1] ASI_DMMU, %g5 ! Doing PGD caching? | ||
127 | sllx %g6, PMD_SHIFT_LEFT, %g1 ! Position PMD offset | ||
128 | be,pn %xcc, sparc64_vpte_nucleus ! Is it from Nucleus? | ||
129 | srlx %g1, PMD_SHIFT_RIGHT, %g1 ! Mask PMD offset bits | ||
130 | brnz,pt %g5, sparc64_vpte_continue ! Yep, go like smoke | ||
131 | andn %g1, LOW_MASK_BITS, %g1 ! Final PMD mask | ||
132 | sllx %g6, PGDIR_SHIFT_LEFT, %g5 ! Position PGD offset | ||
133 | |||
134 | /* TLB1 ** ICACHE line 3: Quick VPTE miss */ | ||
135 | srlx %g5, PGDIR_SHIFT_RIGHT, %g5 ! Mask PGD offset bits | ||
136 | andn %g5, LOW_MASK_BITS, %g5 ! Final PGD mask | ||
137 | lduwa [%g7 + %g5] ASI_PHYS_USE_EC, %g5! Load PGD | ||
138 | brz,pn %g5, vpte_noent ! Valid? | ||
139 | sparc64_kpte_continue: | ||
140 | sllx %g5, 11, %g5 ! Shift into place | ||
141 | sparc64_vpte_continue: | ||
142 | lduwa [%g5 + %g1] ASI_PHYS_USE_EC, %g5! Load PMD | ||
143 | sllx %g5, 11, %g5 ! Shift into place | ||
144 | brz,pn %g5, vpte_noent ! Valid? | ||
145 | |||
146 | /* TLB1 ** ICACHE line 4: Quick VPTE miss */ | ||
147 | mov (VALID_SZ_BITS >> 61), %g1 ! upper vpte into %g1 | ||
148 | sllx %g1, 61, %g1 ! finish calc | ||
149 | or %g5, VPTE_BITS, %g5 ! Prepare VPTE data | ||
150 | or %g5, %g1, %g5 ! ... | ||
151 | mov TLB_SFSR, %g1 ! Restore %g1 value | ||
152 | stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Load VPTE into TLB | ||
153 | stxa %g4, [%g1 + %g1] ASI_DMMU ! Restore previous TAG_ACCESS | ||
154 | retry ! Load PTE once again | ||
155 | |||
156 | #undef VALID_SZ_BITS | ||
157 | #undef VPTE_SHIFT | ||
158 | #undef VPTE_BITS | ||
159 | #undef A | ||
160 | #undef B | ||
161 | #undef C | ||
162 | #undef D | ||
163 | #undef E | ||
164 | #undef F | ||
165 | #undef PMD_SHIFT_LEFT | ||
166 | #undef PMD_SHIFT_RIGHT | ||
167 | #undef PGDIR_SHIFT_LEFT | ||
168 | #undef PGDIR_SHIFT_RIGHT | ||
169 | #undef LOW_MASK_BITS | ||
170 | |||
diff --git a/arch/sparc64/kernel/dtlb_base.S b/arch/sparc64/kernel/dtlb_base.S deleted file mode 100644 index 6528786840c0..000000000000 --- a/arch/sparc64/kernel/dtlb_base.S +++ /dev/null | |||
@@ -1,109 +0,0 @@ | |||
1 | /* $Id: dtlb_base.S,v 1.17 2001/10/11 22:33:52 davem Exp $ | ||
2 | * dtlb_base.S: Front end to DTLB miss replacement strategy. | ||
3 | * This is included directly into the trap table. | ||
4 | * | ||
5 | * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com) | ||
6 | * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz) | ||
7 | */ | ||
8 | |||
9 | #include <asm/pgtable.h> | ||
10 | #include <asm/mmu.h> | ||
11 | |||
12 | /* %g1 TLB_SFSR (%g1 + %g1 == TLB_TAG_ACCESS) | ||
13 | * %g2 (KERN_HIGHBITS | KERN_LOWBITS) | ||
14 | * %g3 VPTE base (0xfffffffe00000000) Spitfire/Blackbird (44-bit VA space) | ||
15 | * (0xffe0000000000000) Cheetah (64-bit VA space) | ||
16 | * %g7 __pa(current->mm->pgd) | ||
17 | * | ||
18 | * The VPTE base value is completely magic, but note that | ||
19 | * few places in the kernel other than these TLB miss | ||
20 | * handlers know anything about the VPTE mechanism or | ||
21 | * how it works (see VPTE_SIZE, TASK_SIZE and PTRS_PER_PGD). | ||
22 | * Consider the 44-bit VADDR Ultra-I/II case as an example: | ||
23 | * | ||
24 | * VA[0 : (1<<43)] produce VPTE index [%g3 : 0] | ||
25 | * VA[0 : -(1<<43)] produce VPTE index [%g3-(1<<(43-PAGE_SHIFT+3)) : %g3] | ||
26 | * | ||
27 | * For Cheetah's 64-bit VADDR space this is: | ||
28 | * | ||
29 | * VA[0 : (1<<63)] produce VPTE index [%g3 : 0] | ||
30 | * VA[0 : -(1<<63)] produce VPTE index [%g3-(1<<(63-PAGE_SHIFT+3)) : %g3] | ||
31 | * | ||
32 | * If you're paying attention you'll notice that this means half of | ||
33 | * the VPTE table is above %g3 and half is below, low VA addresses | ||
34 | * map progressively upwards from %g3, and high VA addresses map | ||
35 | * progressively upwards towards %g3. This trick was needed to make | ||
36 | * the same 8 instruction handler work both for Spitfire/Blackbird's | ||
37 | * peculiar VA space hole configuration and the full 64-bit VA space | ||
38 | * one of Cheetah at the same time. | ||
39 | */ | ||
40 | |||
41 | /* Ways we can get here: | ||
42 | * | ||
43 | * 1) Nucleus loads and stores to/from PA-->VA direct mappings. | ||
44 | * 2) Nucleus loads and stores to/from vmalloc() areas. | ||
45 | * 3) User loads and stores. | ||
46 | * 4) User space accesses by nucleus at tl0 | ||
47 | */ | ||
48 | |||
49 | #if PAGE_SHIFT == 13 | ||
50 | /* | ||
51 | * To compute vpte offset, we need to do ((addr >> 13) << 3), | ||
52 | * which can be optimized to (addr >> 10) if bits 10/11/12 can | ||
53 | * be guaranteed to be 0 ... mmu_context.h does guarantee this | ||
54 | * by only using 10 bits in the hwcontext value. | ||
55 | */ | ||
56 | #define CREATE_VPTE_OFFSET1(r1, r2) nop | ||
57 | #define CREATE_VPTE_OFFSET2(r1, r2) \ | ||
58 | srax r1, 10, r2 | ||
59 | #else | ||
60 | #define CREATE_VPTE_OFFSET1(r1, r2) \ | ||
61 | srax r1, PAGE_SHIFT, r2 | ||
62 | #define CREATE_VPTE_OFFSET2(r1, r2) \ | ||
63 | sllx r2, 3, r2 | ||
64 | #endif | ||
65 | |||
66 | /* DTLB ** ICACHE line 1: Quick user TLB misses */ | ||
67 | mov TLB_SFSR, %g1 | ||
68 | ldxa [%g1 + %g1] ASI_DMMU, %g4 ! Get TAG_ACCESS | ||
69 | andcc %g4, TAG_CONTEXT_BITS, %g0 ! From Nucleus? | ||
70 | from_tl1_trap: | ||
71 | rdpr %tl, %g5 ! For TL==3 test | ||
72 | CREATE_VPTE_OFFSET1(%g4, %g6) ! Create VPTE offset | ||
73 | be,pn %xcc, kvmap ! Yep, special processing | ||
74 | CREATE_VPTE_OFFSET2(%g4, %g6) ! Create VPTE offset | ||
75 | cmp %g5, 4 ! Last trap level? | ||
76 | |||
77 | /* DTLB ** ICACHE line 2: User finish + quick kernel TLB misses */ | ||
78 | be,pn %xcc, longpath ! Yep, cannot risk VPTE miss | ||
79 | nop ! delay slot | ||
80 | ldxa [%g3 + %g6] ASI_S, %g5 ! Load VPTE | ||
81 | 1: brgez,pn %g5, longpath ! Invalid, branch out | ||
82 | nop ! Delay-slot | ||
83 | 9: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB | ||
84 | retry ! Trap return | ||
85 | nop | ||
86 | |||
87 | /* DTLB ** ICACHE line 3: winfixups+real_faults */ | ||
88 | longpath: | ||
89 | rdpr %pstate, %g5 ! Move into alternate globals | ||
90 | wrpr %g5, PSTATE_AG|PSTATE_MG, %pstate | ||
91 | rdpr %tl, %g4 ! See where we came from. | ||
92 | cmp %g4, 1 ! Is etrap/rtrap window fault? | ||
93 | mov TLB_TAG_ACCESS, %g4 ! Prepare for fault processing | ||
94 | ldxa [%g4] ASI_DMMU, %g5 ! Load faulting VA page | ||
95 | be,pt %xcc, sparc64_realfault_common ! Jump to normal fault handling | ||
96 | mov FAULT_CODE_DTLB, %g4 ! It was read from DTLB | ||
97 | |||
98 | /* DTLB ** ICACHE line 4: Unused... */ | ||
99 | ba,a,pt %xcc, winfix_trampoline ! Call window fixup code | ||
100 | nop | ||
101 | nop | ||
102 | nop | ||
103 | nop | ||
104 | nop | ||
105 | nop | ||
106 | nop | ||
107 | |||
108 | #undef CREATE_VPTE_OFFSET1 | ||
109 | #undef CREATE_VPTE_OFFSET2 | ||
diff --git a/arch/sparc64/kernel/dtlb_miss.S b/arch/sparc64/kernel/dtlb_miss.S new file mode 100644 index 000000000000..d0f1565cb564 --- /dev/null +++ b/arch/sparc64/kernel/dtlb_miss.S | |||
@@ -0,0 +1,39 @@ | |||
1 | /* DTLB ** ICACHE line 1: Context 0 check and TSB load */ | ||
2 | ldxa [%g0] ASI_DMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer | ||
3 | ldxa [%g0] ASI_DMMU, %g6 ! Get TAG TARGET | ||
4 | srlx %g6, 48, %g5 ! Get context | ||
5 | brz,pn %g5, kvmap_dtlb ! Context 0 processing | ||
6 | nop ! Delay slot (fill me) | ||
7 | ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB entry | ||
8 | nop ! Push branch to next I$ line | ||
9 | cmp %g4, %g6 ! Compare TAG | ||
10 | |||
11 | /* DTLB ** ICACHE line 2: TSB compare and TLB load */ | ||
12 | bne,pn %xcc, tsb_miss_dtlb ! Miss | ||
13 | mov FAULT_CODE_DTLB, %g3 | ||
14 | stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Load TLB | ||
15 | retry ! Trap done | ||
16 | nop | ||
17 | nop | ||
18 | nop | ||
19 | nop | ||
20 | |||
21 | /* DTLB ** ICACHE line 3: */ | ||
22 | nop | ||
23 | nop | ||
24 | nop | ||
25 | nop | ||
26 | nop | ||
27 | nop | ||
28 | nop | ||
29 | nop | ||
30 | |||
31 | /* DTLB ** ICACHE line 4: */ | ||
32 | nop | ||
33 | nop | ||
34 | nop | ||
35 | nop | ||
36 | nop | ||
37 | nop | ||
38 | nop | ||
39 | nop | ||
diff --git a/arch/sparc64/kernel/etrap.S b/arch/sparc64/kernel/etrap.S index 0d8eba21111b..567dbb765c34 100644 --- a/arch/sparc64/kernel/etrap.S +++ b/arch/sparc64/kernel/etrap.S | |||
@@ -99,6 +99,7 @@ etrap_irq: | |||
99 | wrpr %g0, ETRAP_PSTATE2, %pstate | 99 | wrpr %g0, ETRAP_PSTATE2, %pstate |
100 | mov %l6, %g6 | 100 | mov %l6, %g6 |
101 | #ifdef CONFIG_SMP | 101 | #ifdef CONFIG_SMP |
102 | #error IMMU TSB usage must be fixed | ||
102 | mov TSB_REG, %g3 | 103 | mov TSB_REG, %g3 |
103 | ldxa [%g3] ASI_IMMU, %g5 | 104 | ldxa [%g3] ASI_IMMU, %g5 |
104 | #endif | 105 | #endif |
@@ -248,6 +249,7 @@ scetrap: rdpr %pil, %g2 | |||
248 | mov %l6, %g6 | 249 | mov %l6, %g6 |
249 | stx %i7, [%sp + PTREGS_OFF + PT_V9_I7] | 250 | stx %i7, [%sp + PTREGS_OFF + PT_V9_I7] |
250 | #ifdef CONFIG_SMP | 251 | #ifdef CONFIG_SMP |
252 | #error IMMU TSB usage must be fixed | ||
251 | mov TSB_REG, %g3 | 253 | mov TSB_REG, %g3 |
252 | ldxa [%g3] ASI_IMMU, %g5 | 254 | ldxa [%g3] ASI_IMMU, %g5 |
253 | #endif | 255 | #endif |
diff --git a/arch/sparc64/kernel/head.S b/arch/sparc64/kernel/head.S index b49dcd4504b0..d00e20693be1 100644 --- a/arch/sparc64/kernel/head.S +++ b/arch/sparc64/kernel/head.S | |||
@@ -429,17 +429,6 @@ setup_trap_table: | |||
429 | * | 429 | * |
430 | * %g6 --> current_thread_info() | 430 | * %g6 --> current_thread_info() |
431 | * | 431 | * |
432 | * MMU Globals (PSTATE_MG): | ||
433 | * | ||
434 | * %g1 --> TLB_SFSR | ||
435 | * %g2 --> ((_PAGE_VALID | _PAGE_SZ4MB | | ||
436 | * _PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W) | ||
437 | * ^ 0xfffff80000000000) | ||
438 | * (this %g2 value is used for computing the PAGE_OFFSET kernel | ||
439 | * TLB entries quickly, the virtual address of the fault XOR'd | ||
440 | * with this %g2 value is the PTE to load into the TLB) | ||
441 | * %g3 --> VPTE_BASE_CHEETAH or VPTE_BASE_SPITFIRE | ||
442 | * | ||
443 | * Interrupt Globals (PSTATE_IG, setup by init_irqwork_curcpu()): | 432 | * Interrupt Globals (PSTATE_IG, setup by init_irqwork_curcpu()): |
444 | * | 433 | * |
445 | * %g6 --> __irq_work[smp_processor_id()] | 434 | * %g6 --> __irq_work[smp_processor_id()] |
@@ -450,40 +439,6 @@ setup_trap_table: | |||
450 | wrpr %o1, PSTATE_AG, %pstate | 439 | wrpr %o1, PSTATE_AG, %pstate |
451 | mov %o2, %g6 | 440 | mov %o2, %g6 |
452 | 441 | ||
453 | #define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000) | ||
454 | #define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W) | ||
455 | wrpr %o1, PSTATE_MG, %pstate | ||
456 | mov TSB_REG, %g1 | ||
457 | stxa %g0, [%g1] ASI_DMMU | ||
458 | membar #Sync | ||
459 | stxa %g0, [%g1] ASI_IMMU | ||
460 | membar #Sync | ||
461 | mov TLB_SFSR, %g1 | ||
462 | sethi %uhi(KERN_HIGHBITS), %g2 | ||
463 | or %g2, %ulo(KERN_HIGHBITS), %g2 | ||
464 | sllx %g2, 32, %g2 | ||
465 | or %g2, KERN_LOWBITS, %g2 | ||
466 | |||
467 | BRANCH_IF_ANY_CHEETAH(g3,g7,8f) | ||
468 | ba,pt %xcc, 9f | ||
469 | nop | ||
470 | |||
471 | 8: | ||
472 | sethi %uhi(VPTE_BASE_CHEETAH), %g3 | ||
473 | or %g3, %ulo(VPTE_BASE_CHEETAH), %g3 | ||
474 | ba,pt %xcc, 2f | ||
475 | sllx %g3, 32, %g3 | ||
476 | |||
477 | 9: | ||
478 | sethi %uhi(VPTE_BASE_SPITFIRE), %g3 | ||
479 | or %g3, %ulo(VPTE_BASE_SPITFIRE), %g3 | ||
480 | sllx %g3, 32, %g3 | ||
481 | |||
482 | 2: | ||
483 | clr %g7 | ||
484 | #undef KERN_HIGHBITS | ||
485 | #undef KERN_LOWBITS | ||
486 | |||
487 | /* Kill PROM timer */ | 442 | /* Kill PROM timer */ |
488 | sethi %hi(0x80000000), %o2 | 443 | sethi %hi(0x80000000), %o2 |
489 | sllx %o2, 32, %o2 | 444 | sllx %o2, 32, %o2 |
@@ -538,6 +493,7 @@ sparc64_boot_end: | |||
538 | 493 | ||
539 | #include "systbls.S" | 494 | #include "systbls.S" |
540 | #include "ktlb.S" | 495 | #include "ktlb.S" |
496 | #include "tsb.S" | ||
541 | #include "etrap.S" | 497 | #include "etrap.S" |
542 | #include "rtrap.S" | 498 | #include "rtrap.S" |
543 | #include "winfixup.S" | 499 | #include "winfixup.S" |
diff --git a/arch/sparc64/kernel/itlb_base.S b/arch/sparc64/kernel/itlb_base.S deleted file mode 100644 index 4951ff8f6877..000000000000 --- a/arch/sparc64/kernel/itlb_base.S +++ /dev/null | |||
@@ -1,79 +0,0 @@ | |||
1 | /* $Id: itlb_base.S,v 1.12 2002/02/09 19:49:30 davem Exp $ | ||
2 | * itlb_base.S: Front end to ITLB miss replacement strategy. | ||
3 | * This is included directly into the trap table. | ||
4 | * | ||
5 | * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com) | ||
6 | * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz) | ||
7 | */ | ||
8 | |||
9 | #if PAGE_SHIFT == 13 | ||
10 | /* | ||
11 | * To compute vpte offset, we need to do ((addr >> 13) << 3), | ||
12 | * which can be optimized to (addr >> 10) if bits 10/11/12 can | ||
13 | * be guaranteed to be 0 ... mmu_context.h does guarantee this | ||
14 | * by only using 10 bits in the hwcontext value. | ||
15 | */ | ||
16 | #define CREATE_VPTE_OFFSET1(r1, r2) \ | ||
17 | srax r1, 10, r2 | ||
18 | #define CREATE_VPTE_OFFSET2(r1, r2) nop | ||
19 | #else /* PAGE_SHIFT */ | ||
20 | #define CREATE_VPTE_OFFSET1(r1, r2) \ | ||
21 | srax r1, PAGE_SHIFT, r2 | ||
22 | #define CREATE_VPTE_OFFSET2(r1, r2) \ | ||
23 | sllx r2, 3, r2 | ||
24 | #endif /* PAGE_SHIFT */ | ||
25 | |||
26 | |||
27 | /* Ways we can get here: | ||
28 | * | ||
29 | * 1) Nucleus instruction misses from module code. | ||
30 | * 2) All user instruction misses. | ||
31 | * | ||
32 | * All real page faults merge their code paths to the | ||
33 | * sparc64_realfault_common label below. | ||
34 | */ | ||
35 | |||
36 | /* ITLB ** ICACHE line 1: Quick user TLB misses */ | ||
37 | mov TLB_SFSR, %g1 | ||
38 | ldxa [%g1 + %g1] ASI_IMMU, %g4 ! Get TAG_ACCESS | ||
39 | CREATE_VPTE_OFFSET1(%g4, %g6) ! Create VPTE offset | ||
40 | CREATE_VPTE_OFFSET2(%g4, %g6) ! Create VPTE offset | ||
41 | ldxa [%g3 + %g6] ASI_P, %g5 ! Load VPTE | ||
42 | 1: brgez,pn %g5, 3f ! Not valid, branch out | ||
43 | sethi %hi(_PAGE_EXEC), %g4 ! Delay-slot | ||
44 | andcc %g5, %g4, %g0 ! Executable? | ||
45 | |||
46 | /* ITLB ** ICACHE line 2: Real faults */ | ||
47 | be,pn %xcc, 3f ! Nope, branch. | ||
48 | nop ! Delay-slot | ||
49 | 2: stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Load PTE into TLB | ||
50 | retry ! Trap return | ||
51 | 3: rdpr %pstate, %g4 ! Move into alt-globals | ||
52 | wrpr %g4, PSTATE_AG|PSTATE_MG, %pstate | ||
53 | rdpr %tpc, %g5 ! And load faulting VA | ||
54 | mov FAULT_CODE_ITLB, %g4 ! It was read from ITLB | ||
55 | |||
56 | /* ITLB ** ICACHE line 3: Finish faults */ | ||
57 | sparc64_realfault_common: ! Called by dtlb_miss | ||
58 | stb %g4, [%g6 + TI_FAULT_CODE] | ||
59 | stx %g5, [%g6 + TI_FAULT_ADDR] | ||
60 | ba,pt %xcc, etrap ! Save state | ||
61 | 1: rd %pc, %g7 ! ... | ||
62 | call do_sparc64_fault ! Call fault handler | ||
63 | add %sp, PTREGS_OFF, %o0! Compute pt_regs arg | ||
64 | ba,pt %xcc, rtrap_clr_l6 ! Restore cpu state | ||
65 | nop | ||
66 | |||
67 | /* ITLB ** ICACHE line 4: Window fixups */ | ||
68 | winfix_trampoline: | ||
69 | rdpr %tpc, %g3 ! Prepare winfixup TNPC | ||
70 | or %g3, 0x7c, %g3 ! Compute branch offset | ||
71 | wrpr %g3, %tnpc ! Write it into TNPC | ||
72 | done ! Do it to it | ||
73 | nop | ||
74 | nop | ||
75 | nop | ||
76 | nop | ||
77 | |||
78 | #undef CREATE_VPTE_OFFSET1 | ||
79 | #undef CREATE_VPTE_OFFSET2 | ||
diff --git a/arch/sparc64/kernel/itlb_miss.S b/arch/sparc64/kernel/itlb_miss.S new file mode 100644 index 000000000000..6b6c8fee04bd --- /dev/null +++ b/arch/sparc64/kernel/itlb_miss.S | |||
@@ -0,0 +1,39 @@ | |||
1 | /* ITLB ** ICACHE line 1: Context 0 check and TSB load */ | ||
2 | ldxa [%g0] ASI_IMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer | ||
3 | ldxa [%g0] ASI_IMMU, %g6 ! Get TAG TARGET | ||
4 | srlx %g6, 48, %g5 ! Get context | ||
5 | brz,pn %g5, kvmap_itlb ! Context 0 processing | ||
6 | nop ! Delay slot (fill me) | ||
7 | ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB entry | ||
8 | cmp %g4, %g6 ! Compare TAG | ||
9 | sethi %hi(_PAGE_EXEC), %g4 ! Setup exec check | ||
10 | |||
11 | /* ITLB ** ICACHE line 2: TSB compare and TLB load */ | ||
12 | bne,pn %xcc, tsb_miss_itlb ! Miss | ||
13 | mov FAULT_CODE_ITLB, %g3 | ||
14 | andcc %g5, %g4, %g0 ! Executable? | ||
15 | be,pn %xcc, tsb_do_fault | ||
16 | nop ! Delay slot, fill me | ||
17 | stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Load TLB | ||
18 | retry ! Trap done | ||
19 | nop | ||
20 | |||
21 | /* ITLB ** ICACHE line 3: */ | ||
22 | nop | ||
23 | nop | ||
24 | nop | ||
25 | nop | ||
26 | nop | ||
27 | nop | ||
28 | nop | ||
29 | nop | ||
30 | |||
31 | /* ITLB ** ICACHE line 4: */ | ||
32 | nop | ||
33 | nop | ||
34 | nop | ||
35 | nop | ||
36 | nop | ||
37 | nop | ||
38 | nop | ||
39 | nop | ||
diff --git a/arch/sparc64/kernel/ktlb.S b/arch/sparc64/kernel/ktlb.S index d9244d3c9f73..2b5e71b68882 100644 --- a/arch/sparc64/kernel/ktlb.S +++ b/arch/sparc64/kernel/ktlb.S | |||
@@ -4,191 +4,170 @@ | |||
4 | * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de) | 4 | * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de) |
5 | * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) | 5 | * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) |
6 | * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) | 6 | * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) |
7 | */ | 7 | */ |
8 | 8 | ||
9 | #include <linux/config.h> | 9 | #include <linux/config.h> |
10 | #include <asm/head.h> | 10 | #include <asm/head.h> |
11 | #include <asm/asi.h> | 11 | #include <asm/asi.h> |
12 | #include <asm/page.h> | 12 | #include <asm/page.h> |
13 | #include <asm/pgtable.h> | 13 | #include <asm/pgtable.h> |
14 | #include <asm/tsb.h> | ||
14 | 15 | ||
15 | .text | 16 | .text |
16 | .align 32 | 17 | .align 32 |
17 | 18 | ||
18 | /* | 19 | .globl kvmap_itlb |
19 | * On a second level vpte miss, check whether the original fault is to the OBP | 20 | kvmap_itlb: |
20 | * range (note that this is only possible for instruction miss, data misses to | 21 | /* g6: TAG TARGET */ |
21 | * obp range do not use vpte). If so, go back directly to the faulting address. | 22 | mov TLB_TAG_ACCESS, %g4 |
22 | * This is because we want to read the tpc, otherwise we have no way of knowing | 23 | ldxa [%g4] ASI_IMMU, %g4 |
23 | * the 8k aligned faulting address if we are using >8k kernel pagesize. This | 24 | |
24 | * also ensures no vpte range addresses are dropped into tlb while obp is | 25 | kvmap_itlb_nonlinear: |
25 | * executing (see inherit_locked_prom_mappings() rant). | 26 | /* Catch kernel NULL pointer calls. */ |
26 | */ | 27 | sethi %hi(PAGE_SIZE), %g5 |
27 | sparc64_vpte_nucleus: | 28 | cmp %g4, %g5 |
28 | /* Note that kvmap below has verified that the address is | 29 | bleu,pn %xcc, kvmap_dtlb_longpath |
29 | * in the range MODULES_VADDR --> VMALLOC_END already. So | 30 | nop |
30 | * here we need only check if it is an OBP address or not. | 31 | |
31 | */ | 32 | KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load) |
33 | |||
34 | kvmap_itlb_tsb_miss: | ||
32 | sethi %hi(LOW_OBP_ADDRESS), %g5 | 35 | sethi %hi(LOW_OBP_ADDRESS), %g5 |
33 | cmp %g4, %g5 | 36 | cmp %g4, %g5 |
34 | blu,pn %xcc, kern_vpte | 37 | blu,pn %xcc, kvmap_itlb_vmalloc_addr |
35 | mov 0x1, %g5 | 38 | mov 0x1, %g5 |
36 | sllx %g5, 32, %g5 | 39 | sllx %g5, 32, %g5 |
37 | cmp %g4, %g5 | 40 | cmp %g4, %g5 |
38 | blu,pn %xcc, vpte_insn_obp | 41 | blu,pn %xcc, kvmap_itlb_obp |
39 | nop | 42 | nop |
40 | 43 | ||
41 | /* These two instructions are patched by paginig_init(). */ | 44 | kvmap_itlb_vmalloc_addr: |
42 | kern_vpte: | 45 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) |
43 | sethi %hi(swapper_pgd_zero), %g5 | 46 | |
44 | lduw [%g5 + %lo(swapper_pgd_zero)], %g5 | 47 | TSB_LOCK_TAG(%g1, %g2, %g4) |
45 | 48 | ||
46 | /* With kernel PGD in %g5, branch back into dtlb_backend. */ | 49 | /* Load and check PTE. */ |
47 | ba,pt %xcc, sparc64_kpte_continue | 50 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 |
48 | andn %g1, 0x3, %g1 /* Finish PMD offset adjustment. */ | 51 | brgez,a,pn %g5, kvmap_itlb_longpath |
49 | 52 | stx %g0, [%g1] | |
50 | vpte_noent: | ||
51 | /* Restore previous TAG_ACCESS, %g5 is zero, and we will | ||
52 | * skip over the trap instruction so that the top level | ||
53 | * TLB miss handler will thing this %g5 value is just an | ||
54 | * invalid PTE, thus branching to full fault processing. | ||
55 | */ | ||
56 | mov TLB_SFSR, %g1 | ||
57 | stxa %g4, [%g1 + %g1] ASI_DMMU | ||
58 | done | ||
59 | |||
60 | vpte_insn_obp: | ||
61 | /* Behave as if we are at TL0. */ | ||
62 | wrpr %g0, 1, %tl | ||
63 | rdpr %tpc, %g4 /* Find original faulting iaddr */ | ||
64 | srlx %g4, 13, %g4 /* Throw out context bits */ | ||
65 | sllx %g4, 13, %g4 /* g4 has vpn + ctx0 now */ | ||
66 | |||
67 | /* Restore previous TAG_ACCESS. */ | ||
68 | mov TLB_SFSR, %g1 | ||
69 | stxa %g4, [%g1 + %g1] ASI_IMMU | ||
70 | |||
71 | sethi %hi(prom_trans), %g5 | ||
72 | or %g5, %lo(prom_trans), %g5 | ||
73 | |||
74 | 1: ldx [%g5 + 0x00], %g6 ! base | ||
75 | brz,a,pn %g6, longpath ! no more entries, fail | ||
76 | mov TLB_SFSR, %g1 ! and restore %g1 | ||
77 | ldx [%g5 + 0x08], %g1 ! len | ||
78 | add %g6, %g1, %g1 ! end | ||
79 | cmp %g6, %g4 | ||
80 | bgu,pt %xcc, 2f | ||
81 | cmp %g4, %g1 | ||
82 | bgeu,pt %xcc, 2f | ||
83 | ldx [%g5 + 0x10], %g1 ! PTE | ||
84 | |||
85 | /* TLB load, restore %g1, and return from trap. */ | ||
86 | sub %g4, %g6, %g6 | ||
87 | add %g1, %g6, %g5 | ||
88 | mov TLB_SFSR, %g1 | ||
89 | stxa %g5, [%g0] ASI_ITLB_DATA_IN | ||
90 | retry | ||
91 | 53 | ||
92 | 2: ba,pt %xcc, 1b | 54 | TSB_WRITE(%g1, %g5, %g6) |
93 | add %g5, (3 * 8), %g5 ! next entry | 55 | |
94 | 56 | /* fallthrough to TLB load */ | |
95 | kvmap_do_obp: | 57 | |
96 | sethi %hi(prom_trans), %g5 | 58 | kvmap_itlb_load: |
97 | or %g5, %lo(prom_trans), %g5 | 59 | stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Reload TLB |
98 | srlx %g4, 13, %g4 | ||
99 | sllx %g4, 13, %g4 | ||
100 | |||
101 | 1: ldx [%g5 + 0x00], %g6 ! base | ||
102 | brz,a,pn %g6, longpath ! no more entries, fail | ||
103 | mov TLB_SFSR, %g1 ! and restore %g1 | ||
104 | ldx [%g5 + 0x08], %g1 ! len | ||
105 | add %g6, %g1, %g1 ! end | ||
106 | cmp %g6, %g4 | ||
107 | bgu,pt %xcc, 2f | ||
108 | cmp %g4, %g1 | ||
109 | bgeu,pt %xcc, 2f | ||
110 | ldx [%g5 + 0x10], %g1 ! PTE | ||
111 | |||
112 | /* TLB load, restore %g1, and return from trap. */ | ||
113 | sub %g4, %g6, %g6 | ||
114 | add %g1, %g6, %g5 | ||
115 | mov TLB_SFSR, %g1 | ||
116 | stxa %g5, [%g0] ASI_DTLB_DATA_IN | ||
117 | retry | 60 | retry |
118 | 61 | ||
119 | 2: ba,pt %xcc, 1b | 62 | kvmap_itlb_longpath: |
120 | add %g5, (3 * 8), %g5 ! next entry | 63 | rdpr %pstate, %g5 |
64 | wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate | ||
65 | rdpr %tpc, %g5 | ||
66 | ba,pt %xcc, sparc64_realfault_common | ||
67 | mov FAULT_CODE_ITLB, %g4 | ||
68 | |||
69 | kvmap_itlb_obp: | ||
70 | OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath) | ||
71 | |||
72 | TSB_LOCK_TAG(%g1, %g2, %g4) | ||
73 | |||
74 | TSB_WRITE(%g1, %g5, %g6) | ||
75 | |||
76 | ba,pt %xcc, kvmap_itlb_load | ||
77 | nop | ||
78 | |||
79 | kvmap_dtlb_obp: | ||
80 | OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath) | ||
81 | |||
82 | TSB_LOCK_TAG(%g1, %g2, %g4) | ||
83 | |||
84 | TSB_WRITE(%g1, %g5, %g6) | ||
85 | |||
86 | ba,pt %xcc, kvmap_dtlb_load | ||
87 | nop | ||
121 | 88 | ||
122 | /* | ||
123 | * On a first level data miss, check whether this is to the OBP range (note | ||
124 | * that such accesses can be made by prom, as well as by kernel using | ||
125 | * prom_getproperty on "address"), and if so, do not use vpte access ... | ||
126 | * rather, use information saved during inherit_prom_mappings() using 8k | ||
127 | * pagesize. | ||
128 | */ | ||
129 | .align 32 | 89 | .align 32 |
130 | kvmap: | 90 | .globl kvmap_dtlb |
131 | brgez,pn %g4, kvmap_nonlinear | 91 | kvmap_dtlb: |
92 | /* %g6: TAG TARGET */ | ||
93 | mov TLB_TAG_ACCESS, %g4 | ||
94 | ldxa [%g4] ASI_DMMU, %g4 | ||
95 | brgez,pn %g4, kvmap_dtlb_nonlinear | ||
132 | nop | 96 | nop |
133 | 97 | ||
134 | #ifdef CONFIG_DEBUG_PAGEALLOC | 98 | #define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000) |
99 | #define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W) | ||
100 | |||
101 | sethi %uhi(KERN_HIGHBITS), %g2 | ||
102 | or %g2, %ulo(KERN_HIGHBITS), %g2 | ||
103 | sllx %g2, 32, %g2 | ||
104 | or %g2, KERN_LOWBITS, %g2 | ||
105 | |||
106 | #undef KERN_HIGHBITS | ||
107 | #undef KERN_LOWBITS | ||
108 | |||
135 | .globl kvmap_linear_patch | 109 | .globl kvmap_linear_patch |
136 | kvmap_linear_patch: | 110 | kvmap_linear_patch: |
137 | #endif | 111 | ba,pt %xcc, kvmap_dtlb_load |
138 | ba,pt %xcc, kvmap_load | ||
139 | xor %g2, %g4, %g5 | 112 | xor %g2, %g4, %g5 |
140 | 113 | ||
141 | #ifdef CONFIG_DEBUG_PAGEALLOC | 114 | kvmap_dtlb_vmalloc_addr: |
142 | sethi %hi(swapper_pg_dir), %g5 | 115 | KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) |
143 | or %g5, %lo(swapper_pg_dir), %g5 | 116 | |
144 | sllx %g4, 64 - (PGDIR_SHIFT + PGDIR_BITS), %g6 | 117 | TSB_LOCK_TAG(%g1, %g2, %g4) |
145 | srlx %g6, 64 - PAGE_SHIFT, %g6 | 118 | |
146 | andn %g6, 0x3, %g6 | 119 | /* Load and check PTE. */ |
147 | lduw [%g5 + %g6], %g5 | 120 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 |
148 | brz,pn %g5, longpath | 121 | brgez,a,pn %g5, kvmap_dtlb_longpath |
149 | sllx %g4, 64 - (PMD_SHIFT + PMD_BITS), %g6 | 122 | stx %g0, [%g1] |
150 | srlx %g6, 64 - PAGE_SHIFT, %g6 | 123 | |
151 | sllx %g5, 11, %g5 | 124 | TSB_WRITE(%g1, %g5, %g6) |
152 | andn %g6, 0x3, %g6 | 125 | |
153 | lduwa [%g5 + %g6] ASI_PHYS_USE_EC, %g5 | 126 | /* fallthrough to TLB load */ |
154 | brz,pn %g5, longpath | 127 | |
155 | sllx %g4, 64 - PMD_SHIFT, %g6 | 128 | kvmap_dtlb_load: |
156 | srlx %g6, 64 - PAGE_SHIFT, %g6 | 129 | stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB |
157 | sllx %g5, 11, %g5 | 130 | retry |
158 | andn %g6, 0x7, %g6 | 131 | |
159 | ldxa [%g5 + %g6] ASI_PHYS_USE_EC, %g5 | 132 | kvmap_dtlb_nonlinear: |
160 | brz,pn %g5, longpath | 133 | /* Catch kernel NULL pointer derefs. */ |
134 | sethi %hi(PAGE_SIZE), %g5 | ||
135 | cmp %g4, %g5 | ||
136 | bleu,pn %xcc, kvmap_dtlb_longpath | ||
161 | nop | 137 | nop |
162 | ba,a,pt %xcc, kvmap_load | ||
163 | #endif | ||
164 | 138 | ||
165 | kvmap_nonlinear: | 139 | KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) |
140 | |||
141 | kvmap_dtlb_tsbmiss: | ||
166 | sethi %hi(MODULES_VADDR), %g5 | 142 | sethi %hi(MODULES_VADDR), %g5 |
167 | cmp %g4, %g5 | 143 | cmp %g4, %g5 |
168 | blu,pn %xcc, longpath | 144 | blu,pn %xcc, kvmap_dtlb_longpath |
169 | mov (VMALLOC_END >> 24), %g5 | 145 | mov (VMALLOC_END >> 24), %g5 |
170 | sllx %g5, 24, %g5 | 146 | sllx %g5, 24, %g5 |
171 | cmp %g4, %g5 | 147 | cmp %g4, %g5 |
172 | bgeu,pn %xcc, longpath | 148 | bgeu,pn %xcc, kvmap_dtlb_longpath |
173 | nop | 149 | nop |
174 | 150 | ||
175 | kvmap_check_obp: | 151 | kvmap_check_obp: |
176 | sethi %hi(LOW_OBP_ADDRESS), %g5 | 152 | sethi %hi(LOW_OBP_ADDRESS), %g5 |
177 | cmp %g4, %g5 | 153 | cmp %g4, %g5 |
178 | blu,pn %xcc, kvmap_vmalloc_addr | 154 | blu,pn %xcc, kvmap_dtlb_vmalloc_addr |
179 | mov 0x1, %g5 | 155 | mov 0x1, %g5 |
180 | sllx %g5, 32, %g5 | 156 | sllx %g5, 32, %g5 |
181 | cmp %g4, %g5 | 157 | cmp %g4, %g5 |
182 | blu,pn %xcc, kvmap_do_obp | 158 | blu,pn %xcc, kvmap_dtlb_obp |
183 | nop | 159 | nop |
184 | 160 | ba,pt %xcc, kvmap_dtlb_vmalloc_addr | |
185 | kvmap_vmalloc_addr: | ||
186 | /* If we get here, a vmalloc addr was accessed, load kernel VPTE. */ | ||
187 | ldxa [%g3 + %g6] ASI_N, %g5 | ||
188 | brgez,pn %g5, longpath | ||
189 | nop | 161 | nop |
190 | 162 | ||
191 | kvmap_load: | 163 | kvmap_dtlb_longpath: |
192 | /* PTE is valid, load into TLB and return from trap. */ | 164 | rdpr %pstate, %g5 |
193 | stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB | 165 | wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate |
194 | retry | 166 | rdpr %tl, %g4 |
167 | cmp %g4, 1 | ||
168 | mov TLB_TAG_ACCESS, %g4 | ||
169 | ldxa [%g4] ASI_DMMU, %g5 | ||
170 | be,pt %xcc, sparc64_realfault_common | ||
171 | mov FAULT_CODE_DTLB, %g4 | ||
172 | ba,pt %xcc, winfix_trampoline | ||
173 | nop | ||
diff --git a/arch/sparc64/kernel/process.c b/arch/sparc64/kernel/process.c index 059b0d025224..2784aab0d3e5 100644 --- a/arch/sparc64/kernel/process.c +++ b/arch/sparc64/kernel/process.c | |||
@@ -44,6 +44,7 @@ | |||
44 | #include <asm/fpumacro.h> | 44 | #include <asm/fpumacro.h> |
45 | #include <asm/head.h> | 45 | #include <asm/head.h> |
46 | #include <asm/cpudata.h> | 46 | #include <asm/cpudata.h> |
47 | #include <asm/mmu_context.h> | ||
47 | #include <asm/unistd.h> | 48 | #include <asm/unistd.h> |
48 | 49 | ||
49 | /* #define VERBOSE_SHOWREGS */ | 50 | /* #define VERBOSE_SHOWREGS */ |
@@ -433,30 +434,16 @@ void exit_thread(void) | |||
433 | void flush_thread(void) | 434 | void flush_thread(void) |
434 | { | 435 | { |
435 | struct thread_info *t = current_thread_info(); | 436 | struct thread_info *t = current_thread_info(); |
437 | struct mm_struct *mm; | ||
436 | 438 | ||
437 | if (t->flags & _TIF_ABI_PENDING) | 439 | if (t->flags & _TIF_ABI_PENDING) |
438 | t->flags ^= (_TIF_ABI_PENDING | _TIF_32BIT); | 440 | t->flags ^= (_TIF_ABI_PENDING | _TIF_32BIT); |
439 | 441 | ||
440 | if (t->task->mm) { | 442 | mm = t->task->mm; |
441 | unsigned long pgd_cache = 0UL; | 443 | if (mm) |
442 | if (test_thread_flag(TIF_32BIT)) { | 444 | tsb_context_switch(__pa(mm->pgd), |
443 | struct mm_struct *mm = t->task->mm; | 445 | mm->context.sparc64_tsb); |
444 | pgd_t *pgd0 = &mm->pgd[0]; | ||
445 | pud_t *pud0 = pud_offset(pgd0, 0); | ||
446 | 446 | ||
447 | if (pud_none(*pud0)) { | ||
448 | pmd_t *page = pmd_alloc_one(mm, 0); | ||
449 | pud_set(pud0, page); | ||
450 | } | ||
451 | pgd_cache = get_pgd_cache(pgd0); | ||
452 | } | ||
453 | __asm__ __volatile__("stxa %0, [%1] %2\n\t" | ||
454 | "membar #Sync" | ||
455 | : /* no outputs */ | ||
456 | : "r" (pgd_cache), | ||
457 | "r" (TSB_REG), | ||
458 | "i" (ASI_DMMU)); | ||
459 | } | ||
460 | set_thread_wsaved(0); | 447 | set_thread_wsaved(0); |
461 | 448 | ||
462 | /* Turn off performance counters if on. */ | 449 | /* Turn off performance counters if on. */ |
diff --git a/arch/sparc64/kernel/rtrap.S b/arch/sparc64/kernel/rtrap.S index b80eba0081ca..213eb4a9d8a4 100644 --- a/arch/sparc64/kernel/rtrap.S +++ b/arch/sparc64/kernel/rtrap.S | |||
@@ -223,10 +223,14 @@ rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 | |||
223 | ldx [%sp + PTREGS_OFF + PT_V9_G3], %g3 | 223 | ldx [%sp + PTREGS_OFF + PT_V9_G3], %g3 |
224 | ldx [%sp + PTREGS_OFF + PT_V9_G4], %g4 | 224 | ldx [%sp + PTREGS_OFF + PT_V9_G4], %g4 |
225 | ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5 | 225 | ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5 |
226 | #ifdef CONFIG_SMP | ||
227 | #error IMMU TSB usage must be fixed | ||
226 | mov TSB_REG, %g6 | 228 | mov TSB_REG, %g6 |
227 | brnz,a,pn %l3, 1f | 229 | brnz,a,pn %l3, 1f |
228 | ldxa [%g6] ASI_IMMU, %g5 | 230 | ldxa [%g6] ASI_IMMU, %g5 |
229 | 1: ldx [%sp + PTREGS_OFF + PT_V9_G6], %g6 | 231 | #endif |
232 | 1: | ||
233 | ldx [%sp + PTREGS_OFF + PT_V9_G6], %g6 | ||
230 | ldx [%sp + PTREGS_OFF + PT_V9_G7], %g7 | 234 | ldx [%sp + PTREGS_OFF + PT_V9_G7], %g7 |
231 | wrpr %g0, RTRAP_PSTATE_AG_IRQOFF, %pstate | 235 | wrpr %g0, RTRAP_PSTATE_AG_IRQOFF, %pstate |
232 | ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0 | 236 | ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0 |
diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c index 1f7ad8a69052..d2d3369e7b5d 100644 --- a/arch/sparc64/kernel/smp.c +++ b/arch/sparc64/kernel/smp.c | |||
@@ -123,6 +123,7 @@ extern void inherit_locked_prom_mappings(int save_p); | |||
123 | 123 | ||
124 | static inline void cpu_setup_percpu_base(unsigned long cpu_id) | 124 | static inline void cpu_setup_percpu_base(unsigned long cpu_id) |
125 | { | 125 | { |
126 | #error IMMU TSB usage must be fixed | ||
126 | __asm__ __volatile__("mov %0, %%g5\n\t" | 127 | __asm__ __volatile__("mov %0, %%g5\n\t" |
127 | "stxa %0, [%1] %2\n\t" | 128 | "stxa %0, [%1] %2\n\t" |
128 | "membar #Sync" | 129 | "membar #Sync" |
@@ -662,8 +663,6 @@ void smp_call_function_client(int irq, struct pt_regs *regs) | |||
662 | extern unsigned long xcall_flush_tlb_mm; | 663 | extern unsigned long xcall_flush_tlb_mm; |
663 | extern unsigned long xcall_flush_tlb_pending; | 664 | extern unsigned long xcall_flush_tlb_pending; |
664 | extern unsigned long xcall_flush_tlb_kernel_range; | 665 | extern unsigned long xcall_flush_tlb_kernel_range; |
665 | extern unsigned long xcall_flush_tlb_all_spitfire; | ||
666 | extern unsigned long xcall_flush_tlb_all_cheetah; | ||
667 | extern unsigned long xcall_report_regs; | 666 | extern unsigned long xcall_report_regs; |
668 | extern unsigned long xcall_receive_signal; | 667 | extern unsigned long xcall_receive_signal; |
669 | 668 | ||
@@ -794,15 +793,6 @@ void smp_report_regs(void) | |||
794 | smp_cross_call(&xcall_report_regs, 0, 0, 0); | 793 | smp_cross_call(&xcall_report_regs, 0, 0, 0); |
795 | } | 794 | } |
796 | 795 | ||
797 | void smp_flush_tlb_all(void) | ||
798 | { | ||
799 | if (tlb_type == spitfire) | ||
800 | smp_cross_call(&xcall_flush_tlb_all_spitfire, 0, 0, 0); | ||
801 | else | ||
802 | smp_cross_call(&xcall_flush_tlb_all_cheetah, 0, 0, 0); | ||
803 | __flush_tlb_all(); | ||
804 | } | ||
805 | |||
806 | /* We know that the window frames of the user have been flushed | 796 | /* We know that the window frames of the user have been flushed |
807 | * to the stack before we get here because all callers of us | 797 | * to the stack before we get here because all callers of us |
808 | * are flush_tlb_*() routines, and these run after flush_cache_*() | 798 | * are flush_tlb_*() routines, and these run after flush_cache_*() |
diff --git a/arch/sparc64/kernel/trampoline.S b/arch/sparc64/kernel/trampoline.S index 9478551cb020..782d8c4973e4 100644 --- a/arch/sparc64/kernel/trampoline.S +++ b/arch/sparc64/kernel/trampoline.S | |||
@@ -295,39 +295,6 @@ do_unlock: | |||
295 | wrpr %g5, %tba | 295 | wrpr %g5, %tba |
296 | mov %o2, %g6 | 296 | mov %o2, %g6 |
297 | 297 | ||
298 | wrpr %o1, PSTATE_MG, %pstate | ||
299 | #define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000) | ||
300 | #define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W) | ||
301 | |||
302 | mov TSB_REG, %g1 | ||
303 | stxa %g0, [%g1] ASI_DMMU | ||
304 | membar #Sync | ||
305 | mov TLB_SFSR, %g1 | ||
306 | sethi %uhi(KERN_HIGHBITS), %g2 | ||
307 | or %g2, %ulo(KERN_HIGHBITS), %g2 | ||
308 | sllx %g2, 32, %g2 | ||
309 | or %g2, KERN_LOWBITS, %g2 | ||
310 | |||
311 | BRANCH_IF_ANY_CHEETAH(g3,g7,9f) | ||
312 | |||
313 | ba,pt %xcc, 1f | ||
314 | nop | ||
315 | |||
316 | 9: | ||
317 | sethi %uhi(VPTE_BASE_CHEETAH), %g3 | ||
318 | or %g3, %ulo(VPTE_BASE_CHEETAH), %g3 | ||
319 | ba,pt %xcc, 2f | ||
320 | sllx %g3, 32, %g3 | ||
321 | 1: | ||
322 | sethi %uhi(VPTE_BASE_SPITFIRE), %g3 | ||
323 | or %g3, %ulo(VPTE_BASE_SPITFIRE), %g3 | ||
324 | sllx %g3, 32, %g3 | ||
325 | |||
326 | 2: | ||
327 | clr %g7 | ||
328 | #undef KERN_HIGHBITS | ||
329 | #undef KERN_LOWBITS | ||
330 | |||
331 | wrpr %o1, 0x0, %pstate | 298 | wrpr %o1, 0x0, %pstate |
332 | ldx [%g6 + TI_TASK], %g4 | 299 | ldx [%g6 + TI_TASK], %g4 |
333 | 300 | ||
diff --git a/arch/sparc64/kernel/tsb.S b/arch/sparc64/kernel/tsb.S new file mode 100644 index 000000000000..44b9e6fed09f --- /dev/null +++ b/arch/sparc64/kernel/tsb.S | |||
@@ -0,0 +1,169 @@ | |||
1 | /* tsb.S: Sparc64 TSB table handling. | ||
2 | * | ||
3 | * Copyright (C) 2006 David S. Miller <davem@davemloft.net> | ||
4 | */ | ||
5 | |||
6 | #include <asm/tsb.h> | ||
7 | |||
8 | .text | ||
9 | .align 32 | ||
10 | |||
11 | /* Invoked from TLB miss handler, we are in the | ||
12 | * MMU global registers and they are setup like | ||
13 | * this: | ||
14 | * | ||
15 | * %g1: TSB entry pointer | ||
16 | * %g2: available temporary | ||
17 | * %g3: FAULT_CODE_{D,I}TLB | ||
18 | * %g4: available temporary | ||
19 | * %g5: available temporary | ||
20 | * %g6: TAG TARGET | ||
21 | * %g7: physical address base of the linux page | ||
22 | * tables for the current address space | ||
23 | */ | ||
24 | .globl tsb_miss_dtlb | ||
25 | tsb_miss_dtlb: | ||
26 | mov TLB_TAG_ACCESS, %g4 | ||
27 | ldxa [%g4] ASI_DMMU, %g4 | ||
28 | ba,pt %xcc, tsb_miss_page_table_walk | ||
29 | nop | ||
30 | |||
31 | .globl tsb_miss_itlb | ||
32 | tsb_miss_itlb: | ||
33 | mov TLB_TAG_ACCESS, %g4 | ||
34 | ldxa [%g4] ASI_IMMU, %g4 | ||
35 | ba,pt %xcc, tsb_miss_page_table_walk | ||
36 | nop | ||
37 | |||
38 | tsb_miss_page_table_walk: | ||
39 | USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault) | ||
40 | |||
41 | tsb_reload: | ||
42 | TSB_LOCK_TAG(%g1, %g2, %g4) | ||
43 | |||
44 | /* Load and check PTE. */ | ||
45 | ldxa [%g5] ASI_PHYS_USE_EC, %g5 | ||
46 | brgez,a,pn %g5, tsb_do_fault | ||
47 | stx %g0, [%g1] | ||
48 | |||
49 | TSB_WRITE(%g1, %g5, %g6) | ||
50 | |||
51 | /* Finally, load TLB and return from trap. */ | ||
52 | tsb_tlb_reload: | ||
53 | cmp %g3, FAULT_CODE_DTLB | ||
54 | bne,pn %xcc, tsb_itlb_load | ||
55 | nop | ||
56 | |||
57 | tsb_dtlb_load: | ||
58 | stxa %g5, [%g0] ASI_DTLB_DATA_IN | ||
59 | retry | ||
60 | |||
61 | tsb_itlb_load: | ||
62 | stxa %g5, [%g0] ASI_ITLB_DATA_IN | ||
63 | retry | ||
64 | |||
65 | /* No valid entry in the page tables, do full fault | ||
66 | * processing. | ||
67 | */ | ||
68 | |||
69 | .globl tsb_do_fault | ||
70 | tsb_do_fault: | ||
71 | cmp %g3, FAULT_CODE_DTLB | ||
72 | rdpr %pstate, %g5 | ||
73 | bne,pn %xcc, tsb_do_itlb_fault | ||
74 | wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate | ||
75 | |||
76 | tsb_do_dtlb_fault: | ||
77 | rdpr %tl, %g4 | ||
78 | cmp %g4, 1 | ||
79 | mov TLB_TAG_ACCESS, %g4 | ||
80 | ldxa [%g4] ASI_DMMU, %g5 | ||
81 | be,pt %xcc, sparc64_realfault_common | ||
82 | mov FAULT_CODE_DTLB, %g4 | ||
83 | ba,pt %xcc, winfix_trampoline | ||
84 | nop | ||
85 | |||
86 | tsb_do_itlb_fault: | ||
87 | rdpr %tpc, %g5 | ||
88 | ba,pt %xcc, sparc64_realfault_common | ||
89 | mov FAULT_CODE_ITLB, %g4 | ||
90 | |||
91 | .globl sparc64_realfault_common | ||
92 | sparc64_realfault_common: | ||
93 | stb %g4, [%g6 + TI_FAULT_CODE] ! Save fault code | ||
94 | stx %g5, [%g6 + TI_FAULT_ADDR] ! Save fault address | ||
95 | ba,pt %xcc, etrap ! Save trap state | ||
96 | 1: rd %pc, %g7 ! ... | ||
97 | call do_sparc64_fault ! Call fault handler | ||
98 | add %sp, PTREGS_OFF, %o0 ! Compute pt_regs arg | ||
99 | ba,pt %xcc, rtrap_clr_l6 ! Restore cpu state | ||
100 | nop ! Delay slot (fill me) | ||
101 | |||
102 | .globl winfix_trampoline | ||
103 | winfix_trampoline: | ||
104 | rdpr %tpc, %g3 ! Prepare winfixup TNPC | ||
105 | or %g3, 0x7c, %g3 ! Compute branch offset | ||
106 | wrpr %g3, %tnpc ! Write it into TNPC | ||
107 | done ! Trap return | ||
108 | |||
109 | /* Reload MMU related context switch state at | ||
110 | * schedule() time. | ||
111 | * | ||
112 | * %o0: page table physical address | ||
113 | * %o1: TSB address | ||
114 | */ | ||
115 | .globl tsb_context_switch | ||
116 | tsb_context_switch: | ||
117 | wrpr %g0, PSTATE_MG | PSTATE_RMO | PSTATE_PEF | PSTATE_PRIV, %pstate | ||
118 | |||
119 | /* Set page table base alternate global. */ | ||
120 | mov %o0, %g7 | ||
121 | |||
122 | /* XXX can this happen? */ | ||
123 | brz,pn %o1, 9f | ||
124 | nop | ||
125 | |||
126 | /* Lock TSB into D-TLB. */ | ||
127 | sethi %hi(PAGE_SIZE), %o3 | ||
128 | and %o3, %o1, %o3 | ||
129 | sethi %hi(TSBMAP_BASE), %o2 | ||
130 | add %o2, %o3, %o2 | ||
131 | |||
132 | /* XXX handle PAGE_SIZE != 8K correctly... */ | ||
133 | mov TSB_REG, %g1 | ||
134 | stxa %o2, [%g1] ASI_DMMU | ||
135 | membar #Sync | ||
136 | |||
137 | stxa %o2, [%g1] ASI_IMMU | ||
138 | membar #Sync | ||
139 | |||
140 | #define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZBITS)^0xfffff80000000000) | ||
141 | #define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W | _PAGE_L) | ||
142 | sethi %uhi(KERN_HIGHBITS), %g2 | ||
143 | or %g2, %ulo(KERN_HIGHBITS), %g2 | ||
144 | sllx %g2, 32, %g2 | ||
145 | or %g2, KERN_LOWBITS, %g2 | ||
146 | #undef KERN_HIGHBITS | ||
147 | #undef KERN_LOWBITS | ||
148 | |||
149 | xor %o1, %g2, %o1 | ||
150 | |||
151 | /* We use entry 61 for this locked entry. This is the spitfire | ||
152 | * TLB entry number, and luckily cheetah masks the value with | ||
153 | * 15 ending us up with entry 13 which is what we want in that | ||
154 | * case too. | ||
155 | * | ||
156 | * XXX Interactions with prom_world()... | ||
157 | */ | ||
158 | mov TLB_TAG_ACCESS, %g1 | ||
159 | stxa %o2, [%g1] ASI_DMMU | ||
160 | membar #Sync | ||
161 | mov (61 << 3), %g1 | ||
162 | stxa %o1, [%g1] ASI_DTLB_DATA_ACCESS | ||
163 | membar #Sync | ||
164 | |||
165 | 9: | ||
166 | wrpr %g0, PSTATE_RMO | PSTATE_PEF | PSTATE_PRIV | PSTATE_IE, %pstate | ||
167 | |||
168 | retl | ||
169 | mov %o2, %o0 | ||
diff --git a/arch/sparc64/kernel/ttable.S b/arch/sparc64/kernel/ttable.S index 8365bc1f81f3..56f060c8fbf0 100644 --- a/arch/sparc64/kernel/ttable.S +++ b/arch/sparc64/kernel/ttable.S | |||
@@ -78,9 +78,9 @@ tl0_vaw: TRAP(do_vaw) | |||
78 | tl0_cee: membar #Sync | 78 | tl0_cee: membar #Sync |
79 | TRAP_NOSAVE_7INSNS(__spitfire_cee_trap) | 79 | TRAP_NOSAVE_7INSNS(__spitfire_cee_trap) |
80 | tl0_iamiss: | 80 | tl0_iamiss: |
81 | #include "itlb_base.S" | 81 | #include "itlb_miss.S" |
82 | tl0_damiss: | 82 | tl0_damiss: |
83 | #include "dtlb_base.S" | 83 | #include "dtlb_miss.S" |
84 | tl0_daprot: | 84 | tl0_daprot: |
85 | #include "dtlb_prot.S" | 85 | #include "dtlb_prot.S" |
86 | tl0_fecc: BTRAP(0x70) /* Fast-ECC on Cheetah */ | 86 | tl0_fecc: BTRAP(0x70) /* Fast-ECC on Cheetah */ |
@@ -241,7 +241,7 @@ tl1_cee: membar #Sync | |||
241 | 241 | ||
242 | tl1_iamiss: BTRAPTL1(0x64) BTRAPTL1(0x65) BTRAPTL1(0x66) BTRAPTL1(0x67) | 242 | tl1_iamiss: BTRAPTL1(0x64) BTRAPTL1(0x65) BTRAPTL1(0x66) BTRAPTL1(0x67) |
243 | tl1_damiss: | 243 | tl1_damiss: |
244 | #include "dtlb_backend.S" | 244 | #include "dtlb_miss.S" |
245 | tl1_daprot: | 245 | tl1_daprot: |
246 | #include "dtlb_prot.S" | 246 | #include "dtlb_prot.S" |
247 | tl1_fecc: BTRAPTL1(0x70) /* Fast-ECC on Cheetah */ | 247 | tl1_fecc: BTRAPTL1(0x70) /* Fast-ECC on Cheetah */ |
diff --git a/arch/sparc64/kernel/vmlinux.lds.S b/arch/sparc64/kernel/vmlinux.lds.S index 467d13a0d5c1..f018aaf45486 100644 --- a/arch/sparc64/kernel/vmlinux.lds.S +++ b/arch/sparc64/kernel/vmlinux.lds.S | |||
@@ -44,6 +44,9 @@ SECTIONS | |||
44 | __stop___ex_table = .; | 44 | __stop___ex_table = .; |
45 | 45 | ||
46 | . = ALIGN(8192); | 46 | . = ALIGN(8192); |
47 | swapper_tsb = .; | ||
48 | . += 8192; | ||
49 | . = ALIGN(8192); | ||
47 | __init_begin = .; | 50 | __init_begin = .; |
48 | .init.text : { | 51 | .init.text : { |
49 | _sinittext = .; | 52 | _sinittext = .; |
diff --git a/arch/sparc64/kernel/winfixup.S b/arch/sparc64/kernel/winfixup.S index 39160926267b..f5d93aa99cbb 100644 --- a/arch/sparc64/kernel/winfixup.S +++ b/arch/sparc64/kernel/winfixup.S | |||
@@ -85,6 +85,7 @@ fill_fixup: | |||
85 | mov %o7, %g6 | 85 | mov %o7, %g6 |
86 | ldx [%g6 + TI_TASK], %g4 | 86 | ldx [%g6 + TI_TASK], %g4 |
87 | #ifdef CONFIG_SMP | 87 | #ifdef CONFIG_SMP |
88 | #error IMMU TSB usage must be fixed | ||
88 | mov TSB_REG, %g1 | 89 | mov TSB_REG, %g1 |
89 | ldxa [%g1] ASI_IMMU, %g5 | 90 | ldxa [%g1] ASI_IMMU, %g5 |
90 | #endif | 91 | #endif |
@@ -209,6 +210,7 @@ fill_fixup_mna: | |||
209 | mov %o7, %g6 ! Get current back. | 210 | mov %o7, %g6 ! Get current back. |
210 | ldx [%g6 + TI_TASK], %g4 ! Finish it. | 211 | ldx [%g6 + TI_TASK], %g4 ! Finish it. |
211 | #ifdef CONFIG_SMP | 212 | #ifdef CONFIG_SMP |
213 | #error IMMU TSB usage must be fixed | ||
212 | mov TSB_REG, %g1 | 214 | mov TSB_REG, %g1 |
213 | ldxa [%g1] ASI_IMMU, %g5 | 215 | ldxa [%g1] ASI_IMMU, %g5 |
214 | #endif | 216 | #endif |
@@ -278,11 +280,6 @@ window_mna_from_user_common: | |||
278 | ba,pt %xcc, rtrap | 280 | ba,pt %xcc, rtrap |
279 | clr %l6 | 281 | clr %l6 |
280 | 282 | ||
281 | /* These are only needed for 64-bit mode processes which | ||
282 | * put their stack pointer into the VPTE area and there | ||
283 | * happens to be a VPTE tlb entry mapped there during | ||
284 | * a spill/fill trap to that stack frame. | ||
285 | */ | ||
286 | .globl winfix_dax, fill_fixup_dax, spill_fixup_dax | 283 | .globl winfix_dax, fill_fixup_dax, spill_fixup_dax |
287 | winfix_dax: | 284 | winfix_dax: |
288 | andn %g3, 0x7f, %g3 | 285 | andn %g3, 0x7f, %g3 |
@@ -318,6 +315,7 @@ fill_fixup_dax: | |||
318 | mov %o7, %g6 ! Get current back. | 315 | mov %o7, %g6 ! Get current back. |
319 | ldx [%g6 + TI_TASK], %g4 ! Finish it. | 316 | ldx [%g6 + TI_TASK], %g4 ! Finish it. |
320 | #ifdef CONFIG_SMP | 317 | #ifdef CONFIG_SMP |
318 | #error IMMU TSB usage must be fixed | ||
321 | mov TSB_REG, %g1 | 319 | mov TSB_REG, %g1 |
322 | ldxa [%g1] ASI_IMMU, %g5 | 320 | ldxa [%g1] ASI_IMMU, %g5 |
323 | #endif | 321 | #endif |
diff --git a/arch/sparc64/mm/Makefile b/arch/sparc64/mm/Makefile index 9d0960e69f48..e415bf942bcd 100644 --- a/arch/sparc64/mm/Makefile +++ b/arch/sparc64/mm/Makefile | |||
@@ -5,6 +5,6 @@ | |||
5 | EXTRA_AFLAGS := -ansi | 5 | EXTRA_AFLAGS := -ansi |
6 | EXTRA_CFLAGS := -Werror | 6 | EXTRA_CFLAGS := -Werror |
7 | 7 | ||
8 | obj-y := ultra.o tlb.o fault.o init.o generic.o | 8 | obj-y := ultra.o tlb.o tsb.o fault.o init.o generic.o |
9 | 9 | ||
10 | obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o | 10 | obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o |
diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c index 1e44ee26cee8..da068f6b2595 100644 --- a/arch/sparc64/mm/init.c +++ b/arch/sparc64/mm/init.c | |||
@@ -408,8 +408,7 @@ unsigned long prom_virt_to_phys(unsigned long promva, int *error) | |||
408 | 408 | ||
409 | /* The obp translations are saved based on 8k pagesize, since obp can | 409 | /* The obp translations are saved based on 8k pagesize, since obp can |
410 | * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS -> | 410 | * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS -> |
411 | * HI_OBP_ADDRESS range are handled in ktlb.S and do not use the vpte | 411 | * HI_OBP_ADDRESS range are handled in ktlb.S. |
412 | * scheme (also, see rant in inherit_locked_prom_mappings()). | ||
413 | */ | 412 | */ |
414 | static inline int in_obp_range(unsigned long vaddr) | 413 | static inline int in_obp_range(unsigned long vaddr) |
415 | { | 414 | { |
@@ -539,75 +538,6 @@ static void __init inherit_prom_mappings(void) | |||
539 | prom_printf("done.\n"); | 538 | prom_printf("done.\n"); |
540 | } | 539 | } |
541 | 540 | ||
542 | /* The OBP specifications for sun4u mark 0xfffffffc00000000 and | ||
543 | * upwards as reserved for use by the firmware (I wonder if this | ||
544 | * will be the same on Cheetah...). We use this virtual address | ||
545 | * range for the VPTE table mappings of the nucleus so we need | ||
546 | * to zap them when we enter the PROM. -DaveM | ||
547 | */ | ||
548 | static void __flush_nucleus_vptes(void) | ||
549 | { | ||
550 | unsigned long prom_reserved_base = 0xfffffffc00000000UL; | ||
551 | int i; | ||
552 | |||
553 | /* Only DTLB must be checked for VPTE entries. */ | ||
554 | if (tlb_type == spitfire) { | ||
555 | for (i = 0; i < 63; i++) { | ||
556 | unsigned long tag; | ||
557 | |||
558 | /* Spitfire Errata #32 workaround */ | ||
559 | /* NOTE: Always runs on spitfire, so no cheetah+ | ||
560 | * page size encodings. | ||
561 | */ | ||
562 | __asm__ __volatile__("stxa %0, [%1] %2\n\t" | ||
563 | "flush %%g6" | ||
564 | : /* No outputs */ | ||
565 | : "r" (0), | ||
566 | "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); | ||
567 | |||
568 | tag = spitfire_get_dtlb_tag(i); | ||
569 | if (((tag & ~(PAGE_MASK)) == 0) && | ||
570 | ((tag & (PAGE_MASK)) >= prom_reserved_base)) { | ||
571 | __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" | ||
572 | "membar #Sync" | ||
573 | : /* no outputs */ | ||
574 | : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU)); | ||
575 | spitfire_put_dtlb_data(i, 0x0UL); | ||
576 | } | ||
577 | } | ||
578 | } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { | ||
579 | for (i = 0; i < 512; i++) { | ||
580 | unsigned long tag = cheetah_get_dtlb_tag(i, 2); | ||
581 | |||
582 | if ((tag & ~PAGE_MASK) == 0 && | ||
583 | (tag & PAGE_MASK) >= prom_reserved_base) { | ||
584 | __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" | ||
585 | "membar #Sync" | ||
586 | : /* no outputs */ | ||
587 | : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU)); | ||
588 | cheetah_put_dtlb_data(i, 0x0UL, 2); | ||
589 | } | ||
590 | |||
591 | if (tlb_type != cheetah_plus) | ||
592 | continue; | ||
593 | |||
594 | tag = cheetah_get_dtlb_tag(i, 3); | ||
595 | |||
596 | if ((tag & ~PAGE_MASK) == 0 && | ||
597 | (tag & PAGE_MASK) >= prom_reserved_base) { | ||
598 | __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" | ||
599 | "membar #Sync" | ||
600 | : /* no outputs */ | ||
601 | : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU)); | ||
602 | cheetah_put_dtlb_data(i, 0x0UL, 3); | ||
603 | } | ||
604 | } | ||
605 | } else { | ||
606 | /* Implement me :-) */ | ||
607 | BUG(); | ||
608 | } | ||
609 | } | ||
610 | |||
611 | static int prom_ditlb_set; | 541 | static int prom_ditlb_set; |
612 | struct prom_tlb_entry { | 542 | struct prom_tlb_entry { |
613 | int tlb_ent; | 543 | int tlb_ent; |
@@ -635,9 +565,6 @@ void prom_world(int enter) | |||
635 | : "i" (PSTATE_IE)); | 565 | : "i" (PSTATE_IE)); |
636 | 566 | ||
637 | if (enter) { | 567 | if (enter) { |
638 | /* Kick out nucleus VPTEs. */ | ||
639 | __flush_nucleus_vptes(); | ||
640 | |||
641 | /* Install PROM world. */ | 568 | /* Install PROM world. */ |
642 | for (i = 0; i < 16; i++) { | 569 | for (i = 0; i < 16; i++) { |
643 | if (prom_dtlb[i].tlb_ent != -1) { | 570 | if (prom_dtlb[i].tlb_ent != -1) { |
@@ -1039,18 +966,7 @@ out: | |||
1039 | struct pgtable_cache_struct pgt_quicklists; | 966 | struct pgtable_cache_struct pgt_quicklists; |
1040 | #endif | 967 | #endif |
1041 | 968 | ||
1042 | /* OK, we have to color these pages. The page tables are accessed | 969 | /* XXX We don't need to color these things in the D-cache any longer. */ |
1043 | * by non-Dcache enabled mapping in the VPTE area by the dtlb_backend.S | ||
1044 | * code, as well as by PAGE_OFFSET range direct-mapped addresses by | ||
1045 | * other parts of the kernel. By coloring, we make sure that the tlbmiss | ||
1046 | * fast handlers do not get data from old/garbage dcache lines that | ||
1047 | * correspond to an old/stale virtual address (user/kernel) that | ||
1048 | * previously mapped the pagetable page while accessing vpte range | ||
1049 | * addresses. The idea is that if the vpte color and PAGE_OFFSET range | ||
1050 | * color is the same, then when the kernel initializes the pagetable | ||
1051 | * using the later address range, accesses with the first address | ||
1052 | * range will see the newly initialized data rather than the garbage. | ||
1053 | */ | ||
1054 | #ifdef DCACHE_ALIASING_POSSIBLE | 970 | #ifdef DCACHE_ALIASING_POSSIBLE |
1055 | #define DC_ALIAS_SHIFT 1 | 971 | #define DC_ALIAS_SHIFT 1 |
1056 | #else | 972 | #else |
@@ -1419,6 +1335,9 @@ void kernel_map_pages(struct page *page, int numpages, int enable) | |||
1419 | kernel_map_range(phys_start, phys_end, | 1335 | kernel_map_range(phys_start, phys_end, |
1420 | (enable ? PAGE_KERNEL : __pgprot(0))); | 1336 | (enable ? PAGE_KERNEL : __pgprot(0))); |
1421 | 1337 | ||
1338 | flush_tsb_kernel_range(PAGE_OFFSET + phys_start, | ||
1339 | PAGE_OFFSET + phys_end); | ||
1340 | |||
1422 | /* we should perform an IPI and flush all tlbs, | 1341 | /* we should perform an IPI and flush all tlbs, |
1423 | * but that can deadlock->flush only current cpu. | 1342 | * but that can deadlock->flush only current cpu. |
1424 | */ | 1343 | */ |
diff --git a/arch/sparc64/mm/tlb.c b/arch/sparc64/mm/tlb.c index 8b104be4662b..78357cc2a0b7 100644 --- a/arch/sparc64/mm/tlb.c +++ b/arch/sparc64/mm/tlb.c | |||
@@ -25,6 +25,8 @@ void flush_tlb_pending(void) | |||
25 | struct mmu_gather *mp = &__get_cpu_var(mmu_gathers); | 25 | struct mmu_gather *mp = &__get_cpu_var(mmu_gathers); |
26 | 26 | ||
27 | if (mp->tlb_nr) { | 27 | if (mp->tlb_nr) { |
28 | flush_tsb_user(mp); | ||
29 | |||
28 | if (CTX_VALID(mp->mm->context)) { | 30 | if (CTX_VALID(mp->mm->context)) { |
29 | #ifdef CONFIG_SMP | 31 | #ifdef CONFIG_SMP |
30 | smp_flush_tlb_pending(mp->mm, mp->tlb_nr, | 32 | smp_flush_tlb_pending(mp->mm, mp->tlb_nr, |
@@ -89,62 +91,3 @@ no_cache_flush: | |||
89 | if (nr >= TLB_BATCH_NR) | 91 | if (nr >= TLB_BATCH_NR) |
90 | flush_tlb_pending(); | 92 | flush_tlb_pending(); |
91 | } | 93 | } |
92 | |||
93 | void flush_tlb_pgtables(struct mm_struct *mm, unsigned long start, unsigned long end) | ||
94 | { | ||
95 | struct mmu_gather *mp = &__get_cpu_var(mmu_gathers); | ||
96 | unsigned long nr = mp->tlb_nr; | ||
97 | long s = start, e = end, vpte_base; | ||
98 | |||
99 | if (mp->fullmm) | ||
100 | return; | ||
101 | |||
102 | /* If start is greater than end, that is a real problem. */ | ||
103 | BUG_ON(start > end); | ||
104 | |||
105 | /* However, straddling the VA space hole is quite normal. */ | ||
106 | s &= PMD_MASK; | ||
107 | e = (e + PMD_SIZE - 1) & PMD_MASK; | ||
108 | |||
109 | vpte_base = (tlb_type == spitfire ? | ||
110 | VPTE_BASE_SPITFIRE : | ||
111 | VPTE_BASE_CHEETAH); | ||
112 | |||
113 | if (unlikely(nr != 0 && mm != mp->mm)) { | ||
114 | flush_tlb_pending(); | ||
115 | nr = 0; | ||
116 | } | ||
117 | |||
118 | if (nr == 0) | ||
119 | mp->mm = mm; | ||
120 | |||
121 | start = vpte_base + (s >> (PAGE_SHIFT - 3)); | ||
122 | end = vpte_base + (e >> (PAGE_SHIFT - 3)); | ||
123 | |||
124 | /* If the request straddles the VA space hole, we | ||
125 | * need to swap start and end. The reason this | ||
126 | * occurs is that "vpte_base" is the center of | ||
127 | * the linear page table mapping area. Thus, | ||
128 | * high addresses with the sign bit set map to | ||
129 | * addresses below vpte_base and non-sign bit | ||
130 | * addresses map to addresses above vpte_base. | ||
131 | */ | ||
132 | if (end < start) { | ||
133 | unsigned long tmp = start; | ||
134 | |||
135 | start = end; | ||
136 | end = tmp; | ||
137 | } | ||
138 | |||
139 | while (start < end) { | ||
140 | mp->vaddrs[nr] = start; | ||
141 | mp->tlb_nr = ++nr; | ||
142 | if (nr >= TLB_BATCH_NR) { | ||
143 | flush_tlb_pending(); | ||
144 | nr = 0; | ||
145 | } | ||
146 | start += PAGE_SIZE; | ||
147 | } | ||
148 | if (nr) | ||
149 | flush_tlb_pending(); | ||
150 | } | ||
diff --git a/arch/sparc64/mm/tsb.c b/arch/sparc64/mm/tsb.c new file mode 100644 index 000000000000..15e8af58b1d2 --- /dev/null +++ b/arch/sparc64/mm/tsb.c | |||
@@ -0,0 +1,84 @@ | |||
1 | /* arch/sparc64/mm/tsb.c | ||
2 | * | ||
3 | * Copyright (C) 2006 David S. Miller <davem@davemloft.net> | ||
4 | */ | ||
5 | |||
6 | #include <linux/kernel.h> | ||
7 | #include <asm/system.h> | ||
8 | #include <asm/page.h> | ||
9 | #include <asm/tlbflush.h> | ||
10 | #include <asm/tlb.h> | ||
11 | |||
12 | #define TSB_ENTRY_ALIGNMENT 16 | ||
13 | |||
14 | struct tsb { | ||
15 | unsigned long tag; | ||
16 | unsigned long pte; | ||
17 | } __attribute__((aligned(TSB_ENTRY_ALIGNMENT))); | ||
18 | |||
19 | /* We use an 8K TSB for the whole kernel, this allows to | ||
20 | * handle about 4MB of modules and vmalloc mappings without | ||
21 | * incurring many hash conflicts. | ||
22 | */ | ||
23 | #define KERNEL_TSB_SIZE_BYTES 8192 | ||
24 | #define KERNEL_TSB_NENTRIES \ | ||
25 | (KERNEL_TSB_SIZE_BYTES / sizeof(struct tsb)) | ||
26 | |||
27 | extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; | ||
28 | |||
29 | static inline unsigned long tsb_hash(unsigned long vaddr) | ||
30 | { | ||
31 | vaddr >>= PAGE_SHIFT; | ||
32 | return vaddr & (KERNEL_TSB_NENTRIES - 1); | ||
33 | } | ||
34 | |||
35 | static inline int tag_compare(struct tsb *entry, unsigned long vaddr, unsigned long context) | ||
36 | { | ||
37 | if (context == ~0UL) | ||
38 | return 1; | ||
39 | |||
40 | return (entry->tag == ((vaddr >> 22) | (context << 48))); | ||
41 | } | ||
42 | |||
43 | /* TSB flushes need only occur on the processor initiating the address | ||
44 | * space modification, not on each cpu the address space has run on. | ||
45 | * Only the TLB flush needs that treatment. | ||
46 | */ | ||
47 | |||
48 | void flush_tsb_kernel_range(unsigned long start, unsigned long end) | ||
49 | { | ||
50 | unsigned long v; | ||
51 | |||
52 | for (v = start; v < end; v += PAGE_SIZE) { | ||
53 | struct tsb *ent = &swapper_tsb[tsb_hash(v)]; | ||
54 | |||
55 | if (tag_compare(ent, v, 0)) { | ||
56 | ent->tag = 0UL; | ||
57 | membar_storeload_storestore(); | ||
58 | } | ||
59 | } | ||
60 | } | ||
61 | |||
62 | void flush_tsb_user(struct mmu_gather *mp) | ||
63 | { | ||
64 | struct mm_struct *mm = mp->mm; | ||
65 | struct tsb *tsb = (struct tsb *) mm->context.sparc64_tsb; | ||
66 | unsigned long ctx = ~0UL; | ||
67 | int i; | ||
68 | |||
69 | if (CTX_VALID(mm->context)) | ||
70 | ctx = CTX_HWBITS(mm->context); | ||
71 | |||
72 | for (i = 0; i < mp->tlb_nr; i++) { | ||
73 | unsigned long v = mp->vaddrs[i]; | ||
74 | struct tsb *ent; | ||
75 | |||
76 | v &= ~0x1UL; | ||
77 | |||
78 | ent = &tsb[tsb_hash(v)]; | ||
79 | if (tag_compare(ent, v, ctx)) { | ||
80 | ent->tag = 0UL; | ||
81 | membar_storeload_storestore(); | ||
82 | } | ||
83 | } | ||
84 | } | ||
diff --git a/arch/sparc64/mm/ultra.S b/arch/sparc64/mm/ultra.S index e4c9151fa116..22791f29552e 100644 --- a/arch/sparc64/mm/ultra.S +++ b/arch/sparc64/mm/ultra.S | |||
@@ -453,64 +453,6 @@ xcall_flush_dcache_page_spitfire: /* %g1 == physical page address | |||
453 | nop | 453 | nop |
454 | nop | 454 | nop |
455 | 455 | ||
456 | .data | ||
457 | |||
458 | errata32_hwbug: | ||
459 | .xword 0 | ||
460 | |||
461 | .text | ||
462 | |||
463 | /* These two are not performance critical... */ | ||
464 | .globl xcall_flush_tlb_all_spitfire | ||
465 | xcall_flush_tlb_all_spitfire: | ||
466 | /* Spitfire Errata #32 workaround. */ | ||
467 | sethi %hi(errata32_hwbug), %g4 | ||
468 | stx %g0, [%g4 + %lo(errata32_hwbug)] | ||
469 | |||
470 | clr %g2 | ||
471 | clr %g3 | ||
472 | 1: ldxa [%g3] ASI_DTLB_DATA_ACCESS, %g4 | ||
473 | and %g4, _PAGE_L, %g5 | ||
474 | brnz,pn %g5, 2f | ||
475 | mov TLB_TAG_ACCESS, %g7 | ||
476 | |||
477 | stxa %g0, [%g7] ASI_DMMU | ||
478 | membar #Sync | ||
479 | stxa %g0, [%g3] ASI_DTLB_DATA_ACCESS | ||
480 | membar #Sync | ||
481 | |||
482 | /* Spitfire Errata #32 workaround. */ | ||
483 | sethi %hi(errata32_hwbug), %g4 | ||
484 | stx %g0, [%g4 + %lo(errata32_hwbug)] | ||
485 | |||
486 | 2: ldxa [%g3] ASI_ITLB_DATA_ACCESS, %g4 | ||
487 | and %g4, _PAGE_L, %g5 | ||
488 | brnz,pn %g5, 2f | ||
489 | mov TLB_TAG_ACCESS, %g7 | ||
490 | |||
491 | stxa %g0, [%g7] ASI_IMMU | ||
492 | membar #Sync | ||
493 | stxa %g0, [%g3] ASI_ITLB_DATA_ACCESS | ||
494 | membar #Sync | ||
495 | |||
496 | /* Spitfire Errata #32 workaround. */ | ||
497 | sethi %hi(errata32_hwbug), %g4 | ||
498 | stx %g0, [%g4 + %lo(errata32_hwbug)] | ||
499 | |||
500 | 2: add %g2, 1, %g2 | ||
501 | cmp %g2, SPITFIRE_HIGHEST_LOCKED_TLBENT | ||
502 | ble,pt %icc, 1b | ||
503 | sll %g2, 3, %g3 | ||
504 | flush %g6 | ||
505 | retry | ||
506 | |||
507 | .globl xcall_flush_tlb_all_cheetah | ||
508 | xcall_flush_tlb_all_cheetah: | ||
509 | mov 0x80, %g2 | ||
510 | stxa %g0, [%g2] ASI_DMMU_DEMAP | ||
511 | stxa %g0, [%g2] ASI_IMMU_DEMAP | ||
512 | retry | ||
513 | |||
514 | /* These just get rescheduled to PIL vectors. */ | 456 | /* These just get rescheduled to PIL vectors. */ |
515 | .globl xcall_call_function | 457 | .globl xcall_call_function |
516 | xcall_call_function: | 458 | xcall_call_function: |