aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2006-01-31 21:29:18 -0500
committerDavid S. Miller <davem@sunset.davemloft.net>2006-03-20 04:11:13 -0500
commit74bf4312fff083ab25c3f357cc653ada7995e5f6 (patch)
treec23dea461e32485f4cd7ca4b8c33c632655eb906
parent30d4d1ffed7098afe2641536d67eef150499da02 (diff)
[SPARC64]: Move away from virtual page tables, part 1.
We now use the TSB hardware assist features of the UltraSPARC MMUs. SMP is currently knowingly broken, we need to find another place to store the per-cpu base pointers. We hid them away in the TSB base register, and that obviously will not work any more :-) Another known broken case is non-8KB base page size. Also noticed that flush_tlb_all() is not referenced anywhere, only the internal __flush_tlb_all() (local cpu only) is used by the sparc64 port, so we can get rid of flush_tlb_all(). The kernel gets it's own 8KB TSB (swapper_tsb) and each address space gets it's own private 8K TSB. Later we can add code to dynamically increase the size of per-process TSB as the RSS grows. An 8KB TSB is good enough for up to about a 4MB RSS, after which the TSB starts to incur many capacity and conflict misses. We even accumulate OBP translations into the kernel TSB. Another area for refinement is large page size support. We could use a secondary address space TSB to handle those. Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat
-rw-r--r--arch/sparc64/kernel/Makefile2
-rw-r--r--arch/sparc64/kernel/binfmt_aout32.c13
-rw-r--r--arch/sparc64/kernel/dtlb_backend.S170
-rw-r--r--arch/sparc64/kernel/dtlb_base.S109
-rw-r--r--arch/sparc64/kernel/dtlb_miss.S39
-rw-r--r--arch/sparc64/kernel/etrap.S2
-rw-r--r--arch/sparc64/kernel/head.S46
-rw-r--r--arch/sparc64/kernel/itlb_base.S79
-rw-r--r--arch/sparc64/kernel/itlb_miss.S39
-rw-r--r--arch/sparc64/kernel/ktlb.S263
-rw-r--r--arch/sparc64/kernel/process.c25
-rw-r--r--arch/sparc64/kernel/rtrap.S6
-rw-r--r--arch/sparc64/kernel/smp.c12
-rw-r--r--arch/sparc64/kernel/trampoline.S33
-rw-r--r--arch/sparc64/kernel/tsb.S169
-rw-r--r--arch/sparc64/kernel/ttable.S6
-rw-r--r--arch/sparc64/kernel/vmlinux.lds.S3
-rw-r--r--arch/sparc64/kernel/winfixup.S8
-rw-r--r--arch/sparc64/mm/Makefile2
-rw-r--r--arch/sparc64/mm/init.c91
-rw-r--r--arch/sparc64/mm/tlb.c61
-rw-r--r--arch/sparc64/mm/tsb.c84
-rw-r--r--arch/sparc64/mm/ultra.S58
-rw-r--r--include/asm-sparc64/mmu.h1
-rw-r--r--include/asm-sparc64/mmu_context.h46
-rw-r--r--include/asm-sparc64/pgalloc.h1
-rw-r--r--include/asm-sparc64/pgtable.h9
-rw-r--r--include/asm-sparc64/processor.h14
-rw-r--r--include/asm-sparc64/tlbflush.h25
-rw-r--r--include/asm-sparc64/tsb.h165
30 files changed, 693 insertions, 888 deletions
diff --git a/arch/sparc64/kernel/Makefile b/arch/sparc64/kernel/Makefile
index 83d67eb18895..a482a9ffe5bc 100644
--- a/arch/sparc64/kernel/Makefile
+++ b/arch/sparc64/kernel/Makefile
@@ -38,5 +38,5 @@ else
38 CMODEL_CFLAG := -m64 -mcmodel=medlow 38 CMODEL_CFLAG := -m64 -mcmodel=medlow
39endif 39endif
40 40
41head.o: head.S ttable.S itlb_base.S dtlb_base.S dtlb_backend.S dtlb_prot.S \ 41head.o: head.S ttable.S itlb_miss.S dtlb_miss.S ktlb.S tsb.S \
42 etrap.S rtrap.S winfixup.S entry.S 42 etrap.S rtrap.S winfixup.S entry.S
diff --git a/arch/sparc64/kernel/binfmt_aout32.c b/arch/sparc64/kernel/binfmt_aout32.c
index 202a80c24b6f..a57d7f2b6f13 100644
--- a/arch/sparc64/kernel/binfmt_aout32.c
+++ b/arch/sparc64/kernel/binfmt_aout32.c
@@ -31,6 +31,7 @@
31#include <asm/system.h> 31#include <asm/system.h>
32#include <asm/uaccess.h> 32#include <asm/uaccess.h>
33#include <asm/pgalloc.h> 33#include <asm/pgalloc.h>
34#include <asm/mmu_context.h>
34 35
35static int load_aout32_binary(struct linux_binprm *, struct pt_regs * regs); 36static int load_aout32_binary(struct linux_binprm *, struct pt_regs * regs);
36static int load_aout32_library(struct file*); 37static int load_aout32_library(struct file*);
@@ -329,15 +330,9 @@ beyond_if:
329 330
330 current->mm->start_stack = 331 current->mm->start_stack =
331 (unsigned long) create_aout32_tables((char __user *)bprm->p, bprm); 332 (unsigned long) create_aout32_tables((char __user *)bprm->p, bprm);
332 if (!(orig_thr_flags & _TIF_32BIT)) { 333 tsb_context_switch(__pa(current->mm->pgd),
333 unsigned long pgd_cache = get_pgd_cache(current->mm->pgd); 334 current->mm->context.sparc64_tsb);
334 335
335 __asm__ __volatile__("stxa\t%0, [%1] %2\n\t"
336 "membar #Sync"
337 : /* no outputs */
338 : "r" (pgd_cache),
339 "r" (TSB_REG), "i" (ASI_DMMU));
340 }
341 start_thread32(regs, ex.a_entry, current->mm->start_stack); 336 start_thread32(regs, ex.a_entry, current->mm->start_stack);
342 if (current->ptrace & PT_PTRACED) 337 if (current->ptrace & PT_PTRACED)
343 send_sig(SIGTRAP, current, 0); 338 send_sig(SIGTRAP, current, 0);
diff --git a/arch/sparc64/kernel/dtlb_backend.S b/arch/sparc64/kernel/dtlb_backend.S
deleted file mode 100644
index acc889a7f9c1..000000000000
--- a/arch/sparc64/kernel/dtlb_backend.S
+++ /dev/null
@@ -1,170 +0,0 @@
1/* $Id: dtlb_backend.S,v 1.16 2001/10/09 04:02:11 davem Exp $
2 * dtlb_backend.S: Back end to DTLB miss replacement strategy.
3 * This is included directly into the trap table.
4 *
5 * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com)
6 * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz)
7 */
8
9#include <asm/pgtable.h>
10#include <asm/mmu.h>
11
12#define VALID_SZ_BITS (_PAGE_VALID | _PAGE_SZBITS)
13
14#define VPTE_BITS (_PAGE_CP | _PAGE_CV | _PAGE_P )
15#define VPTE_SHIFT (PAGE_SHIFT - 3)
16
17/* Ways we can get here:
18 *
19 * 1) Nucleus loads and stores to/from PA-->VA direct mappings at tl>1.
20 * 2) Nucleus loads and stores to/from user/kernel window save areas.
21 * 3) VPTE misses from dtlb_base and itlb_base.
22 *
23 * We need to extract out the PMD and PGDIR indexes from the
24 * linear virtual page table access address. The PTE index
25 * is at the bottom, but we are not concerned with it. Bits
26 * 0 to 2 are clear since each PTE is 8 bytes in size. Each
27 * PMD and PGDIR entry are 4 bytes in size. Thus, this
28 * address looks something like:
29 *
30 * |---------------------------------------------------------------|
31 * | ... | PGDIR index | PMD index | PTE index | |
32 * |---------------------------------------------------------------|
33 * 63 F E D C B A 3 2 0 <- bit nr
34 *
35 * The variable bits above are defined as:
36 * A --> 3 + (PAGE_SHIFT - log2(8))
37 * --> 3 + (PAGE_SHIFT - 3) - 1
38 * (ie. this is "bit 3" + PAGE_SIZE - size of PTE entry in bits - 1)
39 * B --> A + 1
40 * C --> B + (PAGE_SHIFT - log2(4))
41 * --> B + (PAGE_SHIFT - 2) - 1
42 * (ie. this is "bit B" + PAGE_SIZE - size of PMD entry in bits - 1)
43 * D --> C + 1
44 * E --> D + (PAGE_SHIFT - log2(4))
45 * --> D + (PAGE_SHIFT - 2) - 1
46 * (ie. this is "bit D" + PAGE_SIZE - size of PGDIR entry in bits - 1)
47 * F --> E + 1
48 *
49 * (Note how "B" always evalutes to PAGE_SHIFT, all the other constants
50 * cancel out.)
51 *
52 * For 8K PAGE_SIZE (thus, PAGE_SHIFT of 13) the bit numbers are:
53 * A --> 12
54 * B --> 13
55 * C --> 23
56 * D --> 24
57 * E --> 34
58 * F --> 35
59 *
60 * For 64K PAGE_SIZE (thus, PAGE_SHIFT of 16) the bit numbers are:
61 * A --> 15
62 * B --> 16
63 * C --> 29
64 * D --> 30
65 * E --> 43
66 * F --> 44
67 *
68 * Because bits both above and below each PGDIR and PMD index need to
69 * be masked out, and the index can be as long as 14 bits (when using a
70 * 64K PAGE_SIZE, and thus a PAGE_SHIFT of 16), we need 3 instructions
71 * to extract each index out.
72 *
73 * Shifts do not pair very well on UltraSPARC-I, II, IIi, and IIe, so
74 * we try to avoid using them for the entire operation. We could setup
75 * a mask anywhere from bit 31 down to bit 10 using the sethi instruction.
76 *
77 * We need a mask covering bits B --> C and one covering D --> E.
78 * For 8K PAGE_SIZE these masks are 0x00ffe000 and 0x7ff000000.
79 * For 64K PAGE_SIZE these masks are 0x3fff0000 and 0xfffc0000000.
80 * The second in each set cannot be loaded with a single sethi
81 * instruction, because the upper bits are past bit 32. We would
82 * need to use a sethi + a shift.
83 *
84 * For the time being, we use 2 shifts and a simple "and" mask.
85 * We shift left to clear the bits above the index, we shift down
86 * to clear the bits below the index (sans the log2(4 or 8) bits)
87 * and a mask to clear the log2(4 or 8) bits. We need therefore
88 * define 4 shift counts, all of which are relative to PAGE_SHIFT.
89 *
90 * Although unsupportable for other reasons, this does mean that
91 * 512K and 4MB page sizes would be generaally supported by the
92 * kernel. (ELF binaries would break with > 64K PAGE_SIZE since
93 * the sections are only aligned that strongly).
94 *
95 * The operations performed for extraction are thus:
96 *
97 * ((X << FOO_SHIFT_LEFT) >> FOO_SHIFT_RIGHT) & ~0x3
98 *
99 */
100
101#define A (3 + (PAGE_SHIFT - 3) - 1)
102#define B (A + 1)
103#define C (B + (PAGE_SHIFT - 2) - 1)
104#define D (C + 1)
105#define E (D + (PAGE_SHIFT - 2) - 1)
106#define F (E + 1)
107
108#define PMD_SHIFT_LEFT (64 - D)
109#define PMD_SHIFT_RIGHT (64 - (D - B) - 2)
110#define PGDIR_SHIFT_LEFT (64 - F)
111#define PGDIR_SHIFT_RIGHT (64 - (F - D) - 2)
112#define LOW_MASK_BITS 0x3
113
114/* TLB1 ** ICACHE line 1: tl1 DTLB and quick VPTE miss */
115 ldxa [%g1 + %g1] ASI_DMMU, %g4 ! Get TAG_ACCESS
116 add %g3, %g3, %g5 ! Compute VPTE base
117 cmp %g4, %g5 ! VPTE miss?
118 bgeu,pt %xcc, 1f ! Continue here
119 andcc %g4, TAG_CONTEXT_BITS, %g5 ! tl0 miss Nucleus test
120 ba,a,pt %xcc, from_tl1_trap ! Fall to tl0 miss
1211: sllx %g6, VPTE_SHIFT, %g4 ! Position TAG_ACCESS
122 or %g4, %g5, %g4 ! Prepare TAG_ACCESS
123
124/* TLB1 ** ICACHE line 2: Quick VPTE miss */
125 mov TSB_REG, %g1 ! Grab TSB reg
126 ldxa [%g1] ASI_DMMU, %g5 ! Doing PGD caching?
127 sllx %g6, PMD_SHIFT_LEFT, %g1 ! Position PMD offset
128 be,pn %xcc, sparc64_vpte_nucleus ! Is it from Nucleus?
129 srlx %g1, PMD_SHIFT_RIGHT, %g1 ! Mask PMD offset bits
130 brnz,pt %g5, sparc64_vpte_continue ! Yep, go like smoke
131 andn %g1, LOW_MASK_BITS, %g1 ! Final PMD mask
132 sllx %g6, PGDIR_SHIFT_LEFT, %g5 ! Position PGD offset
133
134/* TLB1 ** ICACHE line 3: Quick VPTE miss */
135 srlx %g5, PGDIR_SHIFT_RIGHT, %g5 ! Mask PGD offset bits
136 andn %g5, LOW_MASK_BITS, %g5 ! Final PGD mask
137 lduwa [%g7 + %g5] ASI_PHYS_USE_EC, %g5! Load PGD
138 brz,pn %g5, vpte_noent ! Valid?
139sparc64_kpte_continue:
140 sllx %g5, 11, %g5 ! Shift into place
141sparc64_vpte_continue:
142 lduwa [%g5 + %g1] ASI_PHYS_USE_EC, %g5! Load PMD
143 sllx %g5, 11, %g5 ! Shift into place
144 brz,pn %g5, vpte_noent ! Valid?
145
146/* TLB1 ** ICACHE line 4: Quick VPTE miss */
147 mov (VALID_SZ_BITS >> 61), %g1 ! upper vpte into %g1
148 sllx %g1, 61, %g1 ! finish calc
149 or %g5, VPTE_BITS, %g5 ! Prepare VPTE data
150 or %g5, %g1, %g5 ! ...
151 mov TLB_SFSR, %g1 ! Restore %g1 value
152 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Load VPTE into TLB
153 stxa %g4, [%g1 + %g1] ASI_DMMU ! Restore previous TAG_ACCESS
154 retry ! Load PTE once again
155
156#undef VALID_SZ_BITS
157#undef VPTE_SHIFT
158#undef VPTE_BITS
159#undef A
160#undef B
161#undef C
162#undef D
163#undef E
164#undef F
165#undef PMD_SHIFT_LEFT
166#undef PMD_SHIFT_RIGHT
167#undef PGDIR_SHIFT_LEFT
168#undef PGDIR_SHIFT_RIGHT
169#undef LOW_MASK_BITS
170
diff --git a/arch/sparc64/kernel/dtlb_base.S b/arch/sparc64/kernel/dtlb_base.S
deleted file mode 100644
index 6528786840c0..000000000000
--- a/arch/sparc64/kernel/dtlb_base.S
+++ /dev/null
@@ -1,109 +0,0 @@
1/* $Id: dtlb_base.S,v 1.17 2001/10/11 22:33:52 davem Exp $
2 * dtlb_base.S: Front end to DTLB miss replacement strategy.
3 * This is included directly into the trap table.
4 *
5 * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com)
6 * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz)
7 */
8
9#include <asm/pgtable.h>
10#include <asm/mmu.h>
11
12/* %g1 TLB_SFSR (%g1 + %g1 == TLB_TAG_ACCESS)
13 * %g2 (KERN_HIGHBITS | KERN_LOWBITS)
14 * %g3 VPTE base (0xfffffffe00000000) Spitfire/Blackbird (44-bit VA space)
15 * (0xffe0000000000000) Cheetah (64-bit VA space)
16 * %g7 __pa(current->mm->pgd)
17 *
18 * The VPTE base value is completely magic, but note that
19 * few places in the kernel other than these TLB miss
20 * handlers know anything about the VPTE mechanism or
21 * how it works (see VPTE_SIZE, TASK_SIZE and PTRS_PER_PGD).
22 * Consider the 44-bit VADDR Ultra-I/II case as an example:
23 *
24 * VA[0 : (1<<43)] produce VPTE index [%g3 : 0]
25 * VA[0 : -(1<<43)] produce VPTE index [%g3-(1<<(43-PAGE_SHIFT+3)) : %g3]
26 *
27 * For Cheetah's 64-bit VADDR space this is:
28 *
29 * VA[0 : (1<<63)] produce VPTE index [%g3 : 0]
30 * VA[0 : -(1<<63)] produce VPTE index [%g3-(1<<(63-PAGE_SHIFT+3)) : %g3]
31 *
32 * If you're paying attention you'll notice that this means half of
33 * the VPTE table is above %g3 and half is below, low VA addresses
34 * map progressively upwards from %g3, and high VA addresses map
35 * progressively upwards towards %g3. This trick was needed to make
36 * the same 8 instruction handler work both for Spitfire/Blackbird's
37 * peculiar VA space hole configuration and the full 64-bit VA space
38 * one of Cheetah at the same time.
39 */
40
41/* Ways we can get here:
42 *
43 * 1) Nucleus loads and stores to/from PA-->VA direct mappings.
44 * 2) Nucleus loads and stores to/from vmalloc() areas.
45 * 3) User loads and stores.
46 * 4) User space accesses by nucleus at tl0
47 */
48
49#if PAGE_SHIFT == 13
50/*
51 * To compute vpte offset, we need to do ((addr >> 13) << 3),
52 * which can be optimized to (addr >> 10) if bits 10/11/12 can
53 * be guaranteed to be 0 ... mmu_context.h does guarantee this
54 * by only using 10 bits in the hwcontext value.
55 */
56#define CREATE_VPTE_OFFSET1(r1, r2) nop
57#define CREATE_VPTE_OFFSET2(r1, r2) \
58 srax r1, 10, r2
59#else
60#define CREATE_VPTE_OFFSET1(r1, r2) \
61 srax r1, PAGE_SHIFT, r2
62#define CREATE_VPTE_OFFSET2(r1, r2) \
63 sllx r2, 3, r2
64#endif
65
66/* DTLB ** ICACHE line 1: Quick user TLB misses */
67 mov TLB_SFSR, %g1
68 ldxa [%g1 + %g1] ASI_DMMU, %g4 ! Get TAG_ACCESS
69 andcc %g4, TAG_CONTEXT_BITS, %g0 ! From Nucleus?
70from_tl1_trap:
71 rdpr %tl, %g5 ! For TL==3 test
72 CREATE_VPTE_OFFSET1(%g4, %g6) ! Create VPTE offset
73 be,pn %xcc, kvmap ! Yep, special processing
74 CREATE_VPTE_OFFSET2(%g4, %g6) ! Create VPTE offset
75 cmp %g5, 4 ! Last trap level?
76
77/* DTLB ** ICACHE line 2: User finish + quick kernel TLB misses */
78 be,pn %xcc, longpath ! Yep, cannot risk VPTE miss
79 nop ! delay slot
80 ldxa [%g3 + %g6] ASI_S, %g5 ! Load VPTE
811: brgez,pn %g5, longpath ! Invalid, branch out
82 nop ! Delay-slot
839: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
84 retry ! Trap return
85 nop
86
87/* DTLB ** ICACHE line 3: winfixups+real_faults */
88longpath:
89 rdpr %pstate, %g5 ! Move into alternate globals
90 wrpr %g5, PSTATE_AG|PSTATE_MG, %pstate
91 rdpr %tl, %g4 ! See where we came from.
92 cmp %g4, 1 ! Is etrap/rtrap window fault?
93 mov TLB_TAG_ACCESS, %g4 ! Prepare for fault processing
94 ldxa [%g4] ASI_DMMU, %g5 ! Load faulting VA page
95 be,pt %xcc, sparc64_realfault_common ! Jump to normal fault handling
96 mov FAULT_CODE_DTLB, %g4 ! It was read from DTLB
97
98/* DTLB ** ICACHE line 4: Unused... */
99 ba,a,pt %xcc, winfix_trampoline ! Call window fixup code
100 nop
101 nop
102 nop
103 nop
104 nop
105 nop
106 nop
107
108#undef CREATE_VPTE_OFFSET1
109#undef CREATE_VPTE_OFFSET2
diff --git a/arch/sparc64/kernel/dtlb_miss.S b/arch/sparc64/kernel/dtlb_miss.S
new file mode 100644
index 000000000000..d0f1565cb564
--- /dev/null
+++ b/arch/sparc64/kernel/dtlb_miss.S
@@ -0,0 +1,39 @@
1/* DTLB ** ICACHE line 1: Context 0 check and TSB load */
2 ldxa [%g0] ASI_DMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer
3 ldxa [%g0] ASI_DMMU, %g6 ! Get TAG TARGET
4 srlx %g6, 48, %g5 ! Get context
5 brz,pn %g5, kvmap_dtlb ! Context 0 processing
6 nop ! Delay slot (fill me)
7 ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB entry
8 nop ! Push branch to next I$ line
9 cmp %g4, %g6 ! Compare TAG
10
11/* DTLB ** ICACHE line 2: TSB compare and TLB load */
12 bne,pn %xcc, tsb_miss_dtlb ! Miss
13 mov FAULT_CODE_DTLB, %g3
14 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Load TLB
15 retry ! Trap done
16 nop
17 nop
18 nop
19 nop
20
21/* DTLB ** ICACHE line 3: */
22 nop
23 nop
24 nop
25 nop
26 nop
27 nop
28 nop
29 nop
30
31/* DTLB ** ICACHE line 4: */
32 nop
33 nop
34 nop
35 nop
36 nop
37 nop
38 nop
39 nop
diff --git a/arch/sparc64/kernel/etrap.S b/arch/sparc64/kernel/etrap.S
index 0d8eba21111b..567dbb765c34 100644
--- a/arch/sparc64/kernel/etrap.S
+++ b/arch/sparc64/kernel/etrap.S
@@ -99,6 +99,7 @@ etrap_irq:
99 wrpr %g0, ETRAP_PSTATE2, %pstate 99 wrpr %g0, ETRAP_PSTATE2, %pstate
100 mov %l6, %g6 100 mov %l6, %g6
101#ifdef CONFIG_SMP 101#ifdef CONFIG_SMP
102#error IMMU TSB usage must be fixed
102 mov TSB_REG, %g3 103 mov TSB_REG, %g3
103 ldxa [%g3] ASI_IMMU, %g5 104 ldxa [%g3] ASI_IMMU, %g5
104#endif 105#endif
@@ -248,6 +249,7 @@ scetrap: rdpr %pil, %g2
248 mov %l6, %g6 249 mov %l6, %g6
249 stx %i7, [%sp + PTREGS_OFF + PT_V9_I7] 250 stx %i7, [%sp + PTREGS_OFF + PT_V9_I7]
250#ifdef CONFIG_SMP 251#ifdef CONFIG_SMP
252#error IMMU TSB usage must be fixed
251 mov TSB_REG, %g3 253 mov TSB_REG, %g3
252 ldxa [%g3] ASI_IMMU, %g5 254 ldxa [%g3] ASI_IMMU, %g5
253#endif 255#endif
diff --git a/arch/sparc64/kernel/head.S b/arch/sparc64/kernel/head.S
index b49dcd4504b0..d00e20693be1 100644
--- a/arch/sparc64/kernel/head.S
+++ b/arch/sparc64/kernel/head.S
@@ -429,17 +429,6 @@ setup_trap_table:
429 * 429 *
430 * %g6 --> current_thread_info() 430 * %g6 --> current_thread_info()
431 * 431 *
432 * MMU Globals (PSTATE_MG):
433 *
434 * %g1 --> TLB_SFSR
435 * %g2 --> ((_PAGE_VALID | _PAGE_SZ4MB |
436 * _PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
437 * ^ 0xfffff80000000000)
438 * (this %g2 value is used for computing the PAGE_OFFSET kernel
439 * TLB entries quickly, the virtual address of the fault XOR'd
440 * with this %g2 value is the PTE to load into the TLB)
441 * %g3 --> VPTE_BASE_CHEETAH or VPTE_BASE_SPITFIRE
442 *
443 * Interrupt Globals (PSTATE_IG, setup by init_irqwork_curcpu()): 432 * Interrupt Globals (PSTATE_IG, setup by init_irqwork_curcpu()):
444 * 433 *
445 * %g6 --> __irq_work[smp_processor_id()] 434 * %g6 --> __irq_work[smp_processor_id()]
@@ -450,40 +439,6 @@ setup_trap_table:
450 wrpr %o1, PSTATE_AG, %pstate 439 wrpr %o1, PSTATE_AG, %pstate
451 mov %o2, %g6 440 mov %o2, %g6
452 441
453#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
454#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
455 wrpr %o1, PSTATE_MG, %pstate
456 mov TSB_REG, %g1
457 stxa %g0, [%g1] ASI_DMMU
458 membar #Sync
459 stxa %g0, [%g1] ASI_IMMU
460 membar #Sync
461 mov TLB_SFSR, %g1
462 sethi %uhi(KERN_HIGHBITS), %g2
463 or %g2, %ulo(KERN_HIGHBITS), %g2
464 sllx %g2, 32, %g2
465 or %g2, KERN_LOWBITS, %g2
466
467 BRANCH_IF_ANY_CHEETAH(g3,g7,8f)
468 ba,pt %xcc, 9f
469 nop
470
4718:
472 sethi %uhi(VPTE_BASE_CHEETAH), %g3
473 or %g3, %ulo(VPTE_BASE_CHEETAH), %g3
474 ba,pt %xcc, 2f
475 sllx %g3, 32, %g3
476
4779:
478 sethi %uhi(VPTE_BASE_SPITFIRE), %g3
479 or %g3, %ulo(VPTE_BASE_SPITFIRE), %g3
480 sllx %g3, 32, %g3
481
4822:
483 clr %g7
484#undef KERN_HIGHBITS
485#undef KERN_LOWBITS
486
487 /* Kill PROM timer */ 442 /* Kill PROM timer */
488 sethi %hi(0x80000000), %o2 443 sethi %hi(0x80000000), %o2
489 sllx %o2, 32, %o2 444 sllx %o2, 32, %o2
@@ -538,6 +493,7 @@ sparc64_boot_end:
538 493
539#include "systbls.S" 494#include "systbls.S"
540#include "ktlb.S" 495#include "ktlb.S"
496#include "tsb.S"
541#include "etrap.S" 497#include "etrap.S"
542#include "rtrap.S" 498#include "rtrap.S"
543#include "winfixup.S" 499#include "winfixup.S"
diff --git a/arch/sparc64/kernel/itlb_base.S b/arch/sparc64/kernel/itlb_base.S
deleted file mode 100644
index 4951ff8f6877..000000000000
--- a/arch/sparc64/kernel/itlb_base.S
+++ /dev/null
@@ -1,79 +0,0 @@
1/* $Id: itlb_base.S,v 1.12 2002/02/09 19:49:30 davem Exp $
2 * itlb_base.S: Front end to ITLB miss replacement strategy.
3 * This is included directly into the trap table.
4 *
5 * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com)
6 * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz)
7 */
8
9#if PAGE_SHIFT == 13
10/*
11 * To compute vpte offset, we need to do ((addr >> 13) << 3),
12 * which can be optimized to (addr >> 10) if bits 10/11/12 can
13 * be guaranteed to be 0 ... mmu_context.h does guarantee this
14 * by only using 10 bits in the hwcontext value.
15 */
16#define CREATE_VPTE_OFFSET1(r1, r2) \
17 srax r1, 10, r2
18#define CREATE_VPTE_OFFSET2(r1, r2) nop
19#else /* PAGE_SHIFT */
20#define CREATE_VPTE_OFFSET1(r1, r2) \
21 srax r1, PAGE_SHIFT, r2
22#define CREATE_VPTE_OFFSET2(r1, r2) \
23 sllx r2, 3, r2
24#endif /* PAGE_SHIFT */
25
26
27/* Ways we can get here:
28 *
29 * 1) Nucleus instruction misses from module code.
30 * 2) All user instruction misses.
31 *
32 * All real page faults merge their code paths to the
33 * sparc64_realfault_common label below.
34 */
35
36/* ITLB ** ICACHE line 1: Quick user TLB misses */
37 mov TLB_SFSR, %g1
38 ldxa [%g1 + %g1] ASI_IMMU, %g4 ! Get TAG_ACCESS
39 CREATE_VPTE_OFFSET1(%g4, %g6) ! Create VPTE offset
40 CREATE_VPTE_OFFSET2(%g4, %g6) ! Create VPTE offset
41 ldxa [%g3 + %g6] ASI_P, %g5 ! Load VPTE
421: brgez,pn %g5, 3f ! Not valid, branch out
43 sethi %hi(_PAGE_EXEC), %g4 ! Delay-slot
44 andcc %g5, %g4, %g0 ! Executable?
45
46/* ITLB ** ICACHE line 2: Real faults */
47 be,pn %xcc, 3f ! Nope, branch.
48 nop ! Delay-slot
492: stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Load PTE into TLB
50 retry ! Trap return
513: rdpr %pstate, %g4 ! Move into alt-globals
52 wrpr %g4, PSTATE_AG|PSTATE_MG, %pstate
53 rdpr %tpc, %g5 ! And load faulting VA
54 mov FAULT_CODE_ITLB, %g4 ! It was read from ITLB
55
56/* ITLB ** ICACHE line 3: Finish faults */
57sparc64_realfault_common: ! Called by dtlb_miss
58 stb %g4, [%g6 + TI_FAULT_CODE]
59 stx %g5, [%g6 + TI_FAULT_ADDR]
60 ba,pt %xcc, etrap ! Save state
611: rd %pc, %g7 ! ...
62 call do_sparc64_fault ! Call fault handler
63 add %sp, PTREGS_OFF, %o0! Compute pt_regs arg
64 ba,pt %xcc, rtrap_clr_l6 ! Restore cpu state
65 nop
66
67/* ITLB ** ICACHE line 4: Window fixups */
68winfix_trampoline:
69 rdpr %tpc, %g3 ! Prepare winfixup TNPC
70 or %g3, 0x7c, %g3 ! Compute branch offset
71 wrpr %g3, %tnpc ! Write it into TNPC
72 done ! Do it to it
73 nop
74 nop
75 nop
76 nop
77
78#undef CREATE_VPTE_OFFSET1
79#undef CREATE_VPTE_OFFSET2
diff --git a/arch/sparc64/kernel/itlb_miss.S b/arch/sparc64/kernel/itlb_miss.S
new file mode 100644
index 000000000000..6b6c8fee04bd
--- /dev/null
+++ b/arch/sparc64/kernel/itlb_miss.S
@@ -0,0 +1,39 @@
1/* ITLB ** ICACHE line 1: Context 0 check and TSB load */
2 ldxa [%g0] ASI_IMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer
3 ldxa [%g0] ASI_IMMU, %g6 ! Get TAG TARGET
4 srlx %g6, 48, %g5 ! Get context
5 brz,pn %g5, kvmap_itlb ! Context 0 processing
6 nop ! Delay slot (fill me)
7 ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB entry
8 cmp %g4, %g6 ! Compare TAG
9 sethi %hi(_PAGE_EXEC), %g4 ! Setup exec check
10
11/* ITLB ** ICACHE line 2: TSB compare and TLB load */
12 bne,pn %xcc, tsb_miss_itlb ! Miss
13 mov FAULT_CODE_ITLB, %g3
14 andcc %g5, %g4, %g0 ! Executable?
15 be,pn %xcc, tsb_do_fault
16 nop ! Delay slot, fill me
17 stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Load TLB
18 retry ! Trap done
19 nop
20
21/* ITLB ** ICACHE line 3: */
22 nop
23 nop
24 nop
25 nop
26 nop
27 nop
28 nop
29 nop
30
31/* ITLB ** ICACHE line 4: */
32 nop
33 nop
34 nop
35 nop
36 nop
37 nop
38 nop
39 nop
diff --git a/arch/sparc64/kernel/ktlb.S b/arch/sparc64/kernel/ktlb.S
index d9244d3c9f73..2b5e71b68882 100644
--- a/arch/sparc64/kernel/ktlb.S
+++ b/arch/sparc64/kernel/ktlb.S
@@ -4,191 +4,170 @@
4 * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de) 4 * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de)
5 * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) 5 * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx)
6 * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 6 * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
7*/ 7 */
8 8
9#include <linux/config.h> 9#include <linux/config.h>
10#include <asm/head.h> 10#include <asm/head.h>
11#include <asm/asi.h> 11#include <asm/asi.h>
12#include <asm/page.h> 12#include <asm/page.h>
13#include <asm/pgtable.h> 13#include <asm/pgtable.h>
14#include <asm/tsb.h>
14 15
15 .text 16 .text
16 .align 32 17 .align 32
17 18
18/* 19 .globl kvmap_itlb
19 * On a second level vpte miss, check whether the original fault is to the OBP 20kvmap_itlb:
20 * range (note that this is only possible for instruction miss, data misses to 21 /* g6: TAG TARGET */
21 * obp range do not use vpte). If so, go back directly to the faulting address. 22 mov TLB_TAG_ACCESS, %g4
22 * This is because we want to read the tpc, otherwise we have no way of knowing 23 ldxa [%g4] ASI_IMMU, %g4
23 * the 8k aligned faulting address if we are using >8k kernel pagesize. This 24
24 * also ensures no vpte range addresses are dropped into tlb while obp is 25kvmap_itlb_nonlinear:
25 * executing (see inherit_locked_prom_mappings() rant). 26 /* Catch kernel NULL pointer calls. */
26 */ 27 sethi %hi(PAGE_SIZE), %g5
27sparc64_vpte_nucleus: 28 cmp %g4, %g5
28 /* Note that kvmap below has verified that the address is 29 bleu,pn %xcc, kvmap_dtlb_longpath
29 * in the range MODULES_VADDR --> VMALLOC_END already. So 30 nop
30 * here we need only check if it is an OBP address or not. 31
31 */ 32 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
33
34kvmap_itlb_tsb_miss:
32 sethi %hi(LOW_OBP_ADDRESS), %g5 35 sethi %hi(LOW_OBP_ADDRESS), %g5
33 cmp %g4, %g5 36 cmp %g4, %g5
34 blu,pn %xcc, kern_vpte 37 blu,pn %xcc, kvmap_itlb_vmalloc_addr
35 mov 0x1, %g5 38 mov 0x1, %g5
36 sllx %g5, 32, %g5 39 sllx %g5, 32, %g5
37 cmp %g4, %g5 40 cmp %g4, %g5
38 blu,pn %xcc, vpte_insn_obp 41 blu,pn %xcc, kvmap_itlb_obp
39 nop 42 nop
40 43
41 /* These two instructions are patched by paginig_init(). */ 44kvmap_itlb_vmalloc_addr:
42kern_vpte: 45 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
43 sethi %hi(swapper_pgd_zero), %g5 46
44 lduw [%g5 + %lo(swapper_pgd_zero)], %g5 47 TSB_LOCK_TAG(%g1, %g2, %g4)
45 48
46 /* With kernel PGD in %g5, branch back into dtlb_backend. */ 49 /* Load and check PTE. */
47 ba,pt %xcc, sparc64_kpte_continue 50 ldxa [%g5] ASI_PHYS_USE_EC, %g5
48 andn %g1, 0x3, %g1 /* Finish PMD offset adjustment. */ 51 brgez,a,pn %g5, kvmap_itlb_longpath
49 52 stx %g0, [%g1]
50vpte_noent:
51 /* Restore previous TAG_ACCESS, %g5 is zero, and we will
52 * skip over the trap instruction so that the top level
53 * TLB miss handler will thing this %g5 value is just an
54 * invalid PTE, thus branching to full fault processing.
55 */
56 mov TLB_SFSR, %g1
57 stxa %g4, [%g1 + %g1] ASI_DMMU
58 done
59
60vpte_insn_obp:
61 /* Behave as if we are at TL0. */
62 wrpr %g0, 1, %tl
63 rdpr %tpc, %g4 /* Find original faulting iaddr */
64 srlx %g4, 13, %g4 /* Throw out context bits */
65 sllx %g4, 13, %g4 /* g4 has vpn + ctx0 now */
66
67 /* Restore previous TAG_ACCESS. */
68 mov TLB_SFSR, %g1
69 stxa %g4, [%g1 + %g1] ASI_IMMU
70
71 sethi %hi(prom_trans), %g5
72 or %g5, %lo(prom_trans), %g5
73
741: ldx [%g5 + 0x00], %g6 ! base
75 brz,a,pn %g6, longpath ! no more entries, fail
76 mov TLB_SFSR, %g1 ! and restore %g1
77 ldx [%g5 + 0x08], %g1 ! len
78 add %g6, %g1, %g1 ! end
79 cmp %g6, %g4
80 bgu,pt %xcc, 2f
81 cmp %g4, %g1
82 bgeu,pt %xcc, 2f
83 ldx [%g5 + 0x10], %g1 ! PTE
84
85 /* TLB load, restore %g1, and return from trap. */
86 sub %g4, %g6, %g6
87 add %g1, %g6, %g5
88 mov TLB_SFSR, %g1
89 stxa %g5, [%g0] ASI_ITLB_DATA_IN
90 retry
91 53
922: ba,pt %xcc, 1b 54 TSB_WRITE(%g1, %g5, %g6)
93 add %g5, (3 * 8), %g5 ! next entry 55
94 56 /* fallthrough to TLB load */
95kvmap_do_obp: 57
96 sethi %hi(prom_trans), %g5 58kvmap_itlb_load:
97 or %g5, %lo(prom_trans), %g5 59 stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Reload TLB
98 srlx %g4, 13, %g4
99 sllx %g4, 13, %g4
100
1011: ldx [%g5 + 0x00], %g6 ! base
102 brz,a,pn %g6, longpath ! no more entries, fail
103 mov TLB_SFSR, %g1 ! and restore %g1
104 ldx [%g5 + 0x08], %g1 ! len
105 add %g6, %g1, %g1 ! end
106 cmp %g6, %g4
107 bgu,pt %xcc, 2f
108 cmp %g4, %g1
109 bgeu,pt %xcc, 2f
110 ldx [%g5 + 0x10], %g1 ! PTE
111
112 /* TLB load, restore %g1, and return from trap. */
113 sub %g4, %g6, %g6
114 add %g1, %g6, %g5
115 mov TLB_SFSR, %g1
116 stxa %g5, [%g0] ASI_DTLB_DATA_IN
117 retry 60 retry
118 61
1192: ba,pt %xcc, 1b 62kvmap_itlb_longpath:
120 add %g5, (3 * 8), %g5 ! next entry 63 rdpr %pstate, %g5
64 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
65 rdpr %tpc, %g5
66 ba,pt %xcc, sparc64_realfault_common
67 mov FAULT_CODE_ITLB, %g4
68
69kvmap_itlb_obp:
70 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
71
72 TSB_LOCK_TAG(%g1, %g2, %g4)
73
74 TSB_WRITE(%g1, %g5, %g6)
75
76 ba,pt %xcc, kvmap_itlb_load
77 nop
78
79kvmap_dtlb_obp:
80 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
81
82 TSB_LOCK_TAG(%g1, %g2, %g4)
83
84 TSB_WRITE(%g1, %g5, %g6)
85
86 ba,pt %xcc, kvmap_dtlb_load
87 nop
121 88
122/*
123 * On a first level data miss, check whether this is to the OBP range (note
124 * that such accesses can be made by prom, as well as by kernel using
125 * prom_getproperty on "address"), and if so, do not use vpte access ...
126 * rather, use information saved during inherit_prom_mappings() using 8k
127 * pagesize.
128 */
129 .align 32 89 .align 32
130kvmap: 90 .globl kvmap_dtlb
131 brgez,pn %g4, kvmap_nonlinear 91kvmap_dtlb:
92 /* %g6: TAG TARGET */
93 mov TLB_TAG_ACCESS, %g4
94 ldxa [%g4] ASI_DMMU, %g4
95 brgez,pn %g4, kvmap_dtlb_nonlinear
132 nop 96 nop
133 97
134#ifdef CONFIG_DEBUG_PAGEALLOC 98#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
99#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
100
101 sethi %uhi(KERN_HIGHBITS), %g2
102 or %g2, %ulo(KERN_HIGHBITS), %g2
103 sllx %g2, 32, %g2
104 or %g2, KERN_LOWBITS, %g2
105
106#undef KERN_HIGHBITS
107#undef KERN_LOWBITS
108
135 .globl kvmap_linear_patch 109 .globl kvmap_linear_patch
136kvmap_linear_patch: 110kvmap_linear_patch:
137#endif 111 ba,pt %xcc, kvmap_dtlb_load
138 ba,pt %xcc, kvmap_load
139 xor %g2, %g4, %g5 112 xor %g2, %g4, %g5
140 113
141#ifdef CONFIG_DEBUG_PAGEALLOC 114kvmap_dtlb_vmalloc_addr:
142 sethi %hi(swapper_pg_dir), %g5 115 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
143 or %g5, %lo(swapper_pg_dir), %g5 116
144 sllx %g4, 64 - (PGDIR_SHIFT + PGDIR_BITS), %g6 117 TSB_LOCK_TAG(%g1, %g2, %g4)
145 srlx %g6, 64 - PAGE_SHIFT, %g6 118
146 andn %g6, 0x3, %g6 119 /* Load and check PTE. */
147 lduw [%g5 + %g6], %g5 120 ldxa [%g5] ASI_PHYS_USE_EC, %g5
148 brz,pn %g5, longpath 121 brgez,a,pn %g5, kvmap_dtlb_longpath
149 sllx %g4, 64 - (PMD_SHIFT + PMD_BITS), %g6 122 stx %g0, [%g1]
150 srlx %g6, 64 - PAGE_SHIFT, %g6 123
151 sllx %g5, 11, %g5 124 TSB_WRITE(%g1, %g5, %g6)
152 andn %g6, 0x3, %g6 125
153 lduwa [%g5 + %g6] ASI_PHYS_USE_EC, %g5 126 /* fallthrough to TLB load */
154 brz,pn %g5, longpath 127
155 sllx %g4, 64 - PMD_SHIFT, %g6 128kvmap_dtlb_load:
156 srlx %g6, 64 - PAGE_SHIFT, %g6 129 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
157 sllx %g5, 11, %g5 130 retry
158 andn %g6, 0x7, %g6 131
159 ldxa [%g5 + %g6] ASI_PHYS_USE_EC, %g5 132kvmap_dtlb_nonlinear:
160 brz,pn %g5, longpath 133 /* Catch kernel NULL pointer derefs. */
134 sethi %hi(PAGE_SIZE), %g5
135 cmp %g4, %g5
136 bleu,pn %xcc, kvmap_dtlb_longpath
161 nop 137 nop
162 ba,a,pt %xcc, kvmap_load
163#endif
164 138
165kvmap_nonlinear: 139 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
140
141kvmap_dtlb_tsbmiss:
166 sethi %hi(MODULES_VADDR), %g5 142 sethi %hi(MODULES_VADDR), %g5
167 cmp %g4, %g5 143 cmp %g4, %g5
168 blu,pn %xcc, longpath 144 blu,pn %xcc, kvmap_dtlb_longpath
169 mov (VMALLOC_END >> 24), %g5 145 mov (VMALLOC_END >> 24), %g5
170 sllx %g5, 24, %g5 146 sllx %g5, 24, %g5
171 cmp %g4, %g5 147 cmp %g4, %g5
172 bgeu,pn %xcc, longpath 148 bgeu,pn %xcc, kvmap_dtlb_longpath
173 nop 149 nop
174 150
175kvmap_check_obp: 151kvmap_check_obp:
176 sethi %hi(LOW_OBP_ADDRESS), %g5 152 sethi %hi(LOW_OBP_ADDRESS), %g5
177 cmp %g4, %g5 153 cmp %g4, %g5
178 blu,pn %xcc, kvmap_vmalloc_addr 154 blu,pn %xcc, kvmap_dtlb_vmalloc_addr
179 mov 0x1, %g5 155 mov 0x1, %g5
180 sllx %g5, 32, %g5 156 sllx %g5, 32, %g5
181 cmp %g4, %g5 157 cmp %g4, %g5
182 blu,pn %xcc, kvmap_do_obp 158 blu,pn %xcc, kvmap_dtlb_obp
183 nop 159 nop
184 160 ba,pt %xcc, kvmap_dtlb_vmalloc_addr
185kvmap_vmalloc_addr:
186 /* If we get here, a vmalloc addr was accessed, load kernel VPTE. */
187 ldxa [%g3 + %g6] ASI_N, %g5
188 brgez,pn %g5, longpath
189 nop 161 nop
190 162
191kvmap_load: 163kvmap_dtlb_longpath:
192 /* PTE is valid, load into TLB and return from trap. */ 164 rdpr %pstate, %g5
193 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB 165 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
194 retry 166 rdpr %tl, %g4
167 cmp %g4, 1
168 mov TLB_TAG_ACCESS, %g4
169 ldxa [%g4] ASI_DMMU, %g5
170 be,pt %xcc, sparc64_realfault_common
171 mov FAULT_CODE_DTLB, %g4
172 ba,pt %xcc, winfix_trampoline
173 nop
diff --git a/arch/sparc64/kernel/process.c b/arch/sparc64/kernel/process.c
index 059b0d025224..2784aab0d3e5 100644
--- a/arch/sparc64/kernel/process.c
+++ b/arch/sparc64/kernel/process.c
@@ -44,6 +44,7 @@
44#include <asm/fpumacro.h> 44#include <asm/fpumacro.h>
45#include <asm/head.h> 45#include <asm/head.h>
46#include <asm/cpudata.h> 46#include <asm/cpudata.h>
47#include <asm/mmu_context.h>
47#include <asm/unistd.h> 48#include <asm/unistd.h>
48 49
49/* #define VERBOSE_SHOWREGS */ 50/* #define VERBOSE_SHOWREGS */
@@ -433,30 +434,16 @@ void exit_thread(void)
433void flush_thread(void) 434void flush_thread(void)
434{ 435{
435 struct thread_info *t = current_thread_info(); 436 struct thread_info *t = current_thread_info();
437 struct mm_struct *mm;
436 438
437 if (t->flags & _TIF_ABI_PENDING) 439 if (t->flags & _TIF_ABI_PENDING)
438 t->flags ^= (_TIF_ABI_PENDING | _TIF_32BIT); 440 t->flags ^= (_TIF_ABI_PENDING | _TIF_32BIT);
439 441
440 if (t->task->mm) { 442 mm = t->task->mm;
441 unsigned long pgd_cache = 0UL; 443 if (mm)
442 if (test_thread_flag(TIF_32BIT)) { 444 tsb_context_switch(__pa(mm->pgd),
443 struct mm_struct *mm = t->task->mm; 445 mm->context.sparc64_tsb);
444 pgd_t *pgd0 = &mm->pgd[0];
445 pud_t *pud0 = pud_offset(pgd0, 0);
446 446
447 if (pud_none(*pud0)) {
448 pmd_t *page = pmd_alloc_one(mm, 0);
449 pud_set(pud0, page);
450 }
451 pgd_cache = get_pgd_cache(pgd0);
452 }
453 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
454 "membar #Sync"
455 : /* no outputs */
456 : "r" (pgd_cache),
457 "r" (TSB_REG),
458 "i" (ASI_DMMU));
459 }
460 set_thread_wsaved(0); 447 set_thread_wsaved(0);
461 448
462 /* Turn off performance counters if on. */ 449 /* Turn off performance counters if on. */
diff --git a/arch/sparc64/kernel/rtrap.S b/arch/sparc64/kernel/rtrap.S
index b80eba0081ca..213eb4a9d8a4 100644
--- a/arch/sparc64/kernel/rtrap.S
+++ b/arch/sparc64/kernel/rtrap.S
@@ -223,10 +223,14 @@ rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1
223 ldx [%sp + PTREGS_OFF + PT_V9_G3], %g3 223 ldx [%sp + PTREGS_OFF + PT_V9_G3], %g3
224 ldx [%sp + PTREGS_OFF + PT_V9_G4], %g4 224 ldx [%sp + PTREGS_OFF + PT_V9_G4], %g4
225 ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5 225 ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5
226#ifdef CONFIG_SMP
227#error IMMU TSB usage must be fixed
226 mov TSB_REG, %g6 228 mov TSB_REG, %g6
227 brnz,a,pn %l3, 1f 229 brnz,a,pn %l3, 1f
228 ldxa [%g6] ASI_IMMU, %g5 230 ldxa [%g6] ASI_IMMU, %g5
2291: ldx [%sp + PTREGS_OFF + PT_V9_G6], %g6 231#endif
2321:
233 ldx [%sp + PTREGS_OFF + PT_V9_G6], %g6
230 ldx [%sp + PTREGS_OFF + PT_V9_G7], %g7 234 ldx [%sp + PTREGS_OFF + PT_V9_G7], %g7
231 wrpr %g0, RTRAP_PSTATE_AG_IRQOFF, %pstate 235 wrpr %g0, RTRAP_PSTATE_AG_IRQOFF, %pstate
232 ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0 236 ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0
diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c
index 1f7ad8a69052..d2d3369e7b5d 100644
--- a/arch/sparc64/kernel/smp.c
+++ b/arch/sparc64/kernel/smp.c
@@ -123,6 +123,7 @@ extern void inherit_locked_prom_mappings(int save_p);
123 123
124static inline void cpu_setup_percpu_base(unsigned long cpu_id) 124static inline void cpu_setup_percpu_base(unsigned long cpu_id)
125{ 125{
126#error IMMU TSB usage must be fixed
126 __asm__ __volatile__("mov %0, %%g5\n\t" 127 __asm__ __volatile__("mov %0, %%g5\n\t"
127 "stxa %0, [%1] %2\n\t" 128 "stxa %0, [%1] %2\n\t"
128 "membar #Sync" 129 "membar #Sync"
@@ -662,8 +663,6 @@ void smp_call_function_client(int irq, struct pt_regs *regs)
662extern unsigned long xcall_flush_tlb_mm; 663extern unsigned long xcall_flush_tlb_mm;
663extern unsigned long xcall_flush_tlb_pending; 664extern unsigned long xcall_flush_tlb_pending;
664extern unsigned long xcall_flush_tlb_kernel_range; 665extern unsigned long xcall_flush_tlb_kernel_range;
665extern unsigned long xcall_flush_tlb_all_spitfire;
666extern unsigned long xcall_flush_tlb_all_cheetah;
667extern unsigned long xcall_report_regs; 666extern unsigned long xcall_report_regs;
668extern unsigned long xcall_receive_signal; 667extern unsigned long xcall_receive_signal;
669 668
@@ -794,15 +793,6 @@ void smp_report_regs(void)
794 smp_cross_call(&xcall_report_regs, 0, 0, 0); 793 smp_cross_call(&xcall_report_regs, 0, 0, 0);
795} 794}
796 795
797void smp_flush_tlb_all(void)
798{
799 if (tlb_type == spitfire)
800 smp_cross_call(&xcall_flush_tlb_all_spitfire, 0, 0, 0);
801 else
802 smp_cross_call(&xcall_flush_tlb_all_cheetah, 0, 0, 0);
803 __flush_tlb_all();
804}
805
806/* We know that the window frames of the user have been flushed 796/* We know that the window frames of the user have been flushed
807 * to the stack before we get here because all callers of us 797 * to the stack before we get here because all callers of us
808 * are flush_tlb_*() routines, and these run after flush_cache_*() 798 * are flush_tlb_*() routines, and these run after flush_cache_*()
diff --git a/arch/sparc64/kernel/trampoline.S b/arch/sparc64/kernel/trampoline.S
index 9478551cb020..782d8c4973e4 100644
--- a/arch/sparc64/kernel/trampoline.S
+++ b/arch/sparc64/kernel/trampoline.S
@@ -295,39 +295,6 @@ do_unlock:
295 wrpr %g5, %tba 295 wrpr %g5, %tba
296 mov %o2, %g6 296 mov %o2, %g6
297 297
298 wrpr %o1, PSTATE_MG, %pstate
299#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
300#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
301
302 mov TSB_REG, %g1
303 stxa %g0, [%g1] ASI_DMMU
304 membar #Sync
305 mov TLB_SFSR, %g1
306 sethi %uhi(KERN_HIGHBITS), %g2
307 or %g2, %ulo(KERN_HIGHBITS), %g2
308 sllx %g2, 32, %g2
309 or %g2, KERN_LOWBITS, %g2
310
311 BRANCH_IF_ANY_CHEETAH(g3,g7,9f)
312
313 ba,pt %xcc, 1f
314 nop
315
3169:
317 sethi %uhi(VPTE_BASE_CHEETAH), %g3
318 or %g3, %ulo(VPTE_BASE_CHEETAH), %g3
319 ba,pt %xcc, 2f
320 sllx %g3, 32, %g3
3211:
322 sethi %uhi(VPTE_BASE_SPITFIRE), %g3
323 or %g3, %ulo(VPTE_BASE_SPITFIRE), %g3
324 sllx %g3, 32, %g3
325
3262:
327 clr %g7
328#undef KERN_HIGHBITS
329#undef KERN_LOWBITS
330
331 wrpr %o1, 0x0, %pstate 298 wrpr %o1, 0x0, %pstate
332 ldx [%g6 + TI_TASK], %g4 299 ldx [%g6 + TI_TASK], %g4
333 300
diff --git a/arch/sparc64/kernel/tsb.S b/arch/sparc64/kernel/tsb.S
new file mode 100644
index 000000000000..44b9e6fed09f
--- /dev/null
+++ b/arch/sparc64/kernel/tsb.S
@@ -0,0 +1,169 @@
1/* tsb.S: Sparc64 TSB table handling.
2 *
3 * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
4 */
5
6#include <asm/tsb.h>
7
8 .text
9 .align 32
10
11 /* Invoked from TLB miss handler, we are in the
12 * MMU global registers and they are setup like
13 * this:
14 *
15 * %g1: TSB entry pointer
16 * %g2: available temporary
17 * %g3: FAULT_CODE_{D,I}TLB
18 * %g4: available temporary
19 * %g5: available temporary
20 * %g6: TAG TARGET
21 * %g7: physical address base of the linux page
22 * tables for the current address space
23 */
24 .globl tsb_miss_dtlb
25tsb_miss_dtlb:
26 mov TLB_TAG_ACCESS, %g4
27 ldxa [%g4] ASI_DMMU, %g4
28 ba,pt %xcc, tsb_miss_page_table_walk
29 nop
30
31 .globl tsb_miss_itlb
32tsb_miss_itlb:
33 mov TLB_TAG_ACCESS, %g4
34 ldxa [%g4] ASI_IMMU, %g4
35 ba,pt %xcc, tsb_miss_page_table_walk
36 nop
37
38tsb_miss_page_table_walk:
39 USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault)
40
41tsb_reload:
42 TSB_LOCK_TAG(%g1, %g2, %g4)
43
44 /* Load and check PTE. */
45 ldxa [%g5] ASI_PHYS_USE_EC, %g5
46 brgez,a,pn %g5, tsb_do_fault
47 stx %g0, [%g1]
48
49 TSB_WRITE(%g1, %g5, %g6)
50
51 /* Finally, load TLB and return from trap. */
52tsb_tlb_reload:
53 cmp %g3, FAULT_CODE_DTLB
54 bne,pn %xcc, tsb_itlb_load
55 nop
56
57tsb_dtlb_load:
58 stxa %g5, [%g0] ASI_DTLB_DATA_IN
59 retry
60
61tsb_itlb_load:
62 stxa %g5, [%g0] ASI_ITLB_DATA_IN
63 retry
64
65 /* No valid entry in the page tables, do full fault
66 * processing.
67 */
68
69 .globl tsb_do_fault
70tsb_do_fault:
71 cmp %g3, FAULT_CODE_DTLB
72 rdpr %pstate, %g5
73 bne,pn %xcc, tsb_do_itlb_fault
74 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
75
76tsb_do_dtlb_fault:
77 rdpr %tl, %g4
78 cmp %g4, 1
79 mov TLB_TAG_ACCESS, %g4
80 ldxa [%g4] ASI_DMMU, %g5
81 be,pt %xcc, sparc64_realfault_common
82 mov FAULT_CODE_DTLB, %g4
83 ba,pt %xcc, winfix_trampoline
84 nop
85
86tsb_do_itlb_fault:
87 rdpr %tpc, %g5
88 ba,pt %xcc, sparc64_realfault_common
89 mov FAULT_CODE_ITLB, %g4
90
91 .globl sparc64_realfault_common
92sparc64_realfault_common:
93 stb %g4, [%g6 + TI_FAULT_CODE] ! Save fault code
94 stx %g5, [%g6 + TI_FAULT_ADDR] ! Save fault address
95 ba,pt %xcc, etrap ! Save trap state
961: rd %pc, %g7 ! ...
97 call do_sparc64_fault ! Call fault handler
98 add %sp, PTREGS_OFF, %o0 ! Compute pt_regs arg
99 ba,pt %xcc, rtrap_clr_l6 ! Restore cpu state
100 nop ! Delay slot (fill me)
101
102 .globl winfix_trampoline
103winfix_trampoline:
104 rdpr %tpc, %g3 ! Prepare winfixup TNPC
105 or %g3, 0x7c, %g3 ! Compute branch offset
106 wrpr %g3, %tnpc ! Write it into TNPC
107 done ! Trap return
108
109 /* Reload MMU related context switch state at
110 * schedule() time.
111 *
112 * %o0: page table physical address
113 * %o1: TSB address
114 */
115 .globl tsb_context_switch
116tsb_context_switch:
117 wrpr %g0, PSTATE_MG | PSTATE_RMO | PSTATE_PEF | PSTATE_PRIV, %pstate
118
119 /* Set page table base alternate global. */
120 mov %o0, %g7
121
122 /* XXX can this happen? */
123 brz,pn %o1, 9f
124 nop
125
126 /* Lock TSB into D-TLB. */
127 sethi %hi(PAGE_SIZE), %o3
128 and %o3, %o1, %o3
129 sethi %hi(TSBMAP_BASE), %o2
130 add %o2, %o3, %o2
131
132 /* XXX handle PAGE_SIZE != 8K correctly... */
133 mov TSB_REG, %g1
134 stxa %o2, [%g1] ASI_DMMU
135 membar #Sync
136
137 stxa %o2, [%g1] ASI_IMMU
138 membar #Sync
139
140#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZBITS)^0xfffff80000000000)
141#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W | _PAGE_L)
142 sethi %uhi(KERN_HIGHBITS), %g2
143 or %g2, %ulo(KERN_HIGHBITS), %g2
144 sllx %g2, 32, %g2
145 or %g2, KERN_LOWBITS, %g2
146#undef KERN_HIGHBITS
147#undef KERN_LOWBITS
148
149 xor %o1, %g2, %o1
150
151 /* We use entry 61 for this locked entry. This is the spitfire
152 * TLB entry number, and luckily cheetah masks the value with
153 * 15 ending us up with entry 13 which is what we want in that
154 * case too.
155 *
156 * XXX Interactions with prom_world()...
157 */
158 mov TLB_TAG_ACCESS, %g1
159 stxa %o2, [%g1] ASI_DMMU
160 membar #Sync
161 mov (61 << 3), %g1
162 stxa %o1, [%g1] ASI_DTLB_DATA_ACCESS
163 membar #Sync
164
1659:
166 wrpr %g0, PSTATE_RMO | PSTATE_PEF | PSTATE_PRIV | PSTATE_IE, %pstate
167
168 retl
169 mov %o2, %o0
diff --git a/arch/sparc64/kernel/ttable.S b/arch/sparc64/kernel/ttable.S
index 8365bc1f81f3..56f060c8fbf0 100644
--- a/arch/sparc64/kernel/ttable.S
+++ b/arch/sparc64/kernel/ttable.S
@@ -78,9 +78,9 @@ tl0_vaw: TRAP(do_vaw)
78tl0_cee: membar #Sync 78tl0_cee: membar #Sync
79 TRAP_NOSAVE_7INSNS(__spitfire_cee_trap) 79 TRAP_NOSAVE_7INSNS(__spitfire_cee_trap)
80tl0_iamiss: 80tl0_iamiss:
81#include "itlb_base.S" 81#include "itlb_miss.S"
82tl0_damiss: 82tl0_damiss:
83#include "dtlb_base.S" 83#include "dtlb_miss.S"
84tl0_daprot: 84tl0_daprot:
85#include "dtlb_prot.S" 85#include "dtlb_prot.S"
86tl0_fecc: BTRAP(0x70) /* Fast-ECC on Cheetah */ 86tl0_fecc: BTRAP(0x70) /* Fast-ECC on Cheetah */
@@ -241,7 +241,7 @@ tl1_cee: membar #Sync
241 241
242tl1_iamiss: BTRAPTL1(0x64) BTRAPTL1(0x65) BTRAPTL1(0x66) BTRAPTL1(0x67) 242tl1_iamiss: BTRAPTL1(0x64) BTRAPTL1(0x65) BTRAPTL1(0x66) BTRAPTL1(0x67)
243tl1_damiss: 243tl1_damiss:
244#include "dtlb_backend.S" 244#include "dtlb_miss.S"
245tl1_daprot: 245tl1_daprot:
246#include "dtlb_prot.S" 246#include "dtlb_prot.S"
247tl1_fecc: BTRAPTL1(0x70) /* Fast-ECC on Cheetah */ 247tl1_fecc: BTRAPTL1(0x70) /* Fast-ECC on Cheetah */
diff --git a/arch/sparc64/kernel/vmlinux.lds.S b/arch/sparc64/kernel/vmlinux.lds.S
index 467d13a0d5c1..f018aaf45486 100644
--- a/arch/sparc64/kernel/vmlinux.lds.S
+++ b/arch/sparc64/kernel/vmlinux.lds.S
@@ -44,6 +44,9 @@ SECTIONS
44 __stop___ex_table = .; 44 __stop___ex_table = .;
45 45
46 . = ALIGN(8192); 46 . = ALIGN(8192);
47 swapper_tsb = .;
48 . += 8192;
49 . = ALIGN(8192);
47 __init_begin = .; 50 __init_begin = .;
48 .init.text : { 51 .init.text : {
49 _sinittext = .; 52 _sinittext = .;
diff --git a/arch/sparc64/kernel/winfixup.S b/arch/sparc64/kernel/winfixup.S
index 39160926267b..f5d93aa99cbb 100644
--- a/arch/sparc64/kernel/winfixup.S
+++ b/arch/sparc64/kernel/winfixup.S
@@ -85,6 +85,7 @@ fill_fixup:
85 mov %o7, %g6 85 mov %o7, %g6
86 ldx [%g6 + TI_TASK], %g4 86 ldx [%g6 + TI_TASK], %g4
87#ifdef CONFIG_SMP 87#ifdef CONFIG_SMP
88#error IMMU TSB usage must be fixed
88 mov TSB_REG, %g1 89 mov TSB_REG, %g1
89 ldxa [%g1] ASI_IMMU, %g5 90 ldxa [%g1] ASI_IMMU, %g5
90#endif 91#endif
@@ -209,6 +210,7 @@ fill_fixup_mna:
209 mov %o7, %g6 ! Get current back. 210 mov %o7, %g6 ! Get current back.
210 ldx [%g6 + TI_TASK], %g4 ! Finish it. 211 ldx [%g6 + TI_TASK], %g4 ! Finish it.
211#ifdef CONFIG_SMP 212#ifdef CONFIG_SMP
213#error IMMU TSB usage must be fixed
212 mov TSB_REG, %g1 214 mov TSB_REG, %g1
213 ldxa [%g1] ASI_IMMU, %g5 215 ldxa [%g1] ASI_IMMU, %g5
214#endif 216#endif
@@ -278,11 +280,6 @@ window_mna_from_user_common:
278 ba,pt %xcc, rtrap 280 ba,pt %xcc, rtrap
279 clr %l6 281 clr %l6
280 282
281 /* These are only needed for 64-bit mode processes which
282 * put their stack pointer into the VPTE area and there
283 * happens to be a VPTE tlb entry mapped there during
284 * a spill/fill trap to that stack frame.
285 */
286 .globl winfix_dax, fill_fixup_dax, spill_fixup_dax 283 .globl winfix_dax, fill_fixup_dax, spill_fixup_dax
287winfix_dax: 284winfix_dax:
288 andn %g3, 0x7f, %g3 285 andn %g3, 0x7f, %g3
@@ -318,6 +315,7 @@ fill_fixup_dax:
318 mov %o7, %g6 ! Get current back. 315 mov %o7, %g6 ! Get current back.
319 ldx [%g6 + TI_TASK], %g4 ! Finish it. 316 ldx [%g6 + TI_TASK], %g4 ! Finish it.
320#ifdef CONFIG_SMP 317#ifdef CONFIG_SMP
318#error IMMU TSB usage must be fixed
321 mov TSB_REG, %g1 319 mov TSB_REG, %g1
322 ldxa [%g1] ASI_IMMU, %g5 320 ldxa [%g1] ASI_IMMU, %g5
323#endif 321#endif
diff --git a/arch/sparc64/mm/Makefile b/arch/sparc64/mm/Makefile
index 9d0960e69f48..e415bf942bcd 100644
--- a/arch/sparc64/mm/Makefile
+++ b/arch/sparc64/mm/Makefile
@@ -5,6 +5,6 @@
5EXTRA_AFLAGS := -ansi 5EXTRA_AFLAGS := -ansi
6EXTRA_CFLAGS := -Werror 6EXTRA_CFLAGS := -Werror
7 7
8obj-y := ultra.o tlb.o fault.o init.o generic.o 8obj-y := ultra.o tlb.o tsb.o fault.o init.o generic.o
9 9
10obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o 10obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
index 1e44ee26cee8..da068f6b2595 100644
--- a/arch/sparc64/mm/init.c
+++ b/arch/sparc64/mm/init.c
@@ -408,8 +408,7 @@ unsigned long prom_virt_to_phys(unsigned long promva, int *error)
408 408
409/* The obp translations are saved based on 8k pagesize, since obp can 409/* The obp translations are saved based on 8k pagesize, since obp can
410 * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS -> 410 * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
411 * HI_OBP_ADDRESS range are handled in ktlb.S and do not use the vpte 411 * HI_OBP_ADDRESS range are handled in ktlb.S.
412 * scheme (also, see rant in inherit_locked_prom_mappings()).
413 */ 412 */
414static inline int in_obp_range(unsigned long vaddr) 413static inline int in_obp_range(unsigned long vaddr)
415{ 414{
@@ -539,75 +538,6 @@ static void __init inherit_prom_mappings(void)
539 prom_printf("done.\n"); 538 prom_printf("done.\n");
540} 539}
541 540
542/* The OBP specifications for sun4u mark 0xfffffffc00000000 and
543 * upwards as reserved for use by the firmware (I wonder if this
544 * will be the same on Cheetah...). We use this virtual address
545 * range for the VPTE table mappings of the nucleus so we need
546 * to zap them when we enter the PROM. -DaveM
547 */
548static void __flush_nucleus_vptes(void)
549{
550 unsigned long prom_reserved_base = 0xfffffffc00000000UL;
551 int i;
552
553 /* Only DTLB must be checked for VPTE entries. */
554 if (tlb_type == spitfire) {
555 for (i = 0; i < 63; i++) {
556 unsigned long tag;
557
558 /* Spitfire Errata #32 workaround */
559 /* NOTE: Always runs on spitfire, so no cheetah+
560 * page size encodings.
561 */
562 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
563 "flush %%g6"
564 : /* No outputs */
565 : "r" (0),
566 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
567
568 tag = spitfire_get_dtlb_tag(i);
569 if (((tag & ~(PAGE_MASK)) == 0) &&
570 ((tag & (PAGE_MASK)) >= prom_reserved_base)) {
571 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
572 "membar #Sync"
573 : /* no outputs */
574 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
575 spitfire_put_dtlb_data(i, 0x0UL);
576 }
577 }
578 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
579 for (i = 0; i < 512; i++) {
580 unsigned long tag = cheetah_get_dtlb_tag(i, 2);
581
582 if ((tag & ~PAGE_MASK) == 0 &&
583 (tag & PAGE_MASK) >= prom_reserved_base) {
584 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
585 "membar #Sync"
586 : /* no outputs */
587 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
588 cheetah_put_dtlb_data(i, 0x0UL, 2);
589 }
590
591 if (tlb_type != cheetah_plus)
592 continue;
593
594 tag = cheetah_get_dtlb_tag(i, 3);
595
596 if ((tag & ~PAGE_MASK) == 0 &&
597 (tag & PAGE_MASK) >= prom_reserved_base) {
598 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
599 "membar #Sync"
600 : /* no outputs */
601 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
602 cheetah_put_dtlb_data(i, 0x0UL, 3);
603 }
604 }
605 } else {
606 /* Implement me :-) */
607 BUG();
608 }
609}
610
611static int prom_ditlb_set; 541static int prom_ditlb_set;
612struct prom_tlb_entry { 542struct prom_tlb_entry {
613 int tlb_ent; 543 int tlb_ent;
@@ -635,9 +565,6 @@ void prom_world(int enter)
635 : "i" (PSTATE_IE)); 565 : "i" (PSTATE_IE));
636 566
637 if (enter) { 567 if (enter) {
638 /* Kick out nucleus VPTEs. */
639 __flush_nucleus_vptes();
640
641 /* Install PROM world. */ 568 /* Install PROM world. */
642 for (i = 0; i < 16; i++) { 569 for (i = 0; i < 16; i++) {
643 if (prom_dtlb[i].tlb_ent != -1) { 570 if (prom_dtlb[i].tlb_ent != -1) {
@@ -1039,18 +966,7 @@ out:
1039struct pgtable_cache_struct pgt_quicklists; 966struct pgtable_cache_struct pgt_quicklists;
1040#endif 967#endif
1041 968
1042/* OK, we have to color these pages. The page tables are accessed 969/* XXX We don't need to color these things in the D-cache any longer. */
1043 * by non-Dcache enabled mapping in the VPTE area by the dtlb_backend.S
1044 * code, as well as by PAGE_OFFSET range direct-mapped addresses by
1045 * other parts of the kernel. By coloring, we make sure that the tlbmiss
1046 * fast handlers do not get data from old/garbage dcache lines that
1047 * correspond to an old/stale virtual address (user/kernel) that
1048 * previously mapped the pagetable page while accessing vpte range
1049 * addresses. The idea is that if the vpte color and PAGE_OFFSET range
1050 * color is the same, then when the kernel initializes the pagetable
1051 * using the later address range, accesses with the first address
1052 * range will see the newly initialized data rather than the garbage.
1053 */
1054#ifdef DCACHE_ALIASING_POSSIBLE 970#ifdef DCACHE_ALIASING_POSSIBLE
1055#define DC_ALIAS_SHIFT 1 971#define DC_ALIAS_SHIFT 1
1056#else 972#else
@@ -1419,6 +1335,9 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
1419 kernel_map_range(phys_start, phys_end, 1335 kernel_map_range(phys_start, phys_end,
1420 (enable ? PAGE_KERNEL : __pgprot(0))); 1336 (enable ? PAGE_KERNEL : __pgprot(0)));
1421 1337
1338 flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
1339 PAGE_OFFSET + phys_end);
1340
1422 /* we should perform an IPI and flush all tlbs, 1341 /* we should perform an IPI and flush all tlbs,
1423 * but that can deadlock->flush only current cpu. 1342 * but that can deadlock->flush only current cpu.
1424 */ 1343 */
diff --git a/arch/sparc64/mm/tlb.c b/arch/sparc64/mm/tlb.c
index 8b104be4662b..78357cc2a0b7 100644
--- a/arch/sparc64/mm/tlb.c
+++ b/arch/sparc64/mm/tlb.c
@@ -25,6 +25,8 @@ void flush_tlb_pending(void)
25 struct mmu_gather *mp = &__get_cpu_var(mmu_gathers); 25 struct mmu_gather *mp = &__get_cpu_var(mmu_gathers);
26 26
27 if (mp->tlb_nr) { 27 if (mp->tlb_nr) {
28 flush_tsb_user(mp);
29
28 if (CTX_VALID(mp->mm->context)) { 30 if (CTX_VALID(mp->mm->context)) {
29#ifdef CONFIG_SMP 31#ifdef CONFIG_SMP
30 smp_flush_tlb_pending(mp->mm, mp->tlb_nr, 32 smp_flush_tlb_pending(mp->mm, mp->tlb_nr,
@@ -89,62 +91,3 @@ no_cache_flush:
89 if (nr >= TLB_BATCH_NR) 91 if (nr >= TLB_BATCH_NR)
90 flush_tlb_pending(); 92 flush_tlb_pending();
91} 93}
92
93void flush_tlb_pgtables(struct mm_struct *mm, unsigned long start, unsigned long end)
94{
95 struct mmu_gather *mp = &__get_cpu_var(mmu_gathers);
96 unsigned long nr = mp->tlb_nr;
97 long s = start, e = end, vpte_base;
98
99 if (mp->fullmm)
100 return;
101
102 /* If start is greater than end, that is a real problem. */
103 BUG_ON(start > end);
104
105 /* However, straddling the VA space hole is quite normal. */
106 s &= PMD_MASK;
107 e = (e + PMD_SIZE - 1) & PMD_MASK;
108
109 vpte_base = (tlb_type == spitfire ?
110 VPTE_BASE_SPITFIRE :
111 VPTE_BASE_CHEETAH);
112
113 if (unlikely(nr != 0 && mm != mp->mm)) {
114 flush_tlb_pending();
115 nr = 0;
116 }
117
118 if (nr == 0)
119 mp->mm = mm;
120
121 start = vpte_base + (s >> (PAGE_SHIFT - 3));
122 end = vpte_base + (e >> (PAGE_SHIFT - 3));
123
124 /* If the request straddles the VA space hole, we
125 * need to swap start and end. The reason this
126 * occurs is that "vpte_base" is the center of
127 * the linear page table mapping area. Thus,
128 * high addresses with the sign bit set map to
129 * addresses below vpte_base and non-sign bit
130 * addresses map to addresses above vpte_base.
131 */
132 if (end < start) {
133 unsigned long tmp = start;
134
135 start = end;
136 end = tmp;
137 }
138
139 while (start < end) {
140 mp->vaddrs[nr] = start;
141 mp->tlb_nr = ++nr;
142 if (nr >= TLB_BATCH_NR) {
143 flush_tlb_pending();
144 nr = 0;
145 }
146 start += PAGE_SIZE;
147 }
148 if (nr)
149 flush_tlb_pending();
150}
diff --git a/arch/sparc64/mm/tsb.c b/arch/sparc64/mm/tsb.c
new file mode 100644
index 000000000000..15e8af58b1d2
--- /dev/null
+++ b/arch/sparc64/mm/tsb.c
@@ -0,0 +1,84 @@
1/* arch/sparc64/mm/tsb.c
2 *
3 * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
4 */
5
6#include <linux/kernel.h>
7#include <asm/system.h>
8#include <asm/page.h>
9#include <asm/tlbflush.h>
10#include <asm/tlb.h>
11
12#define TSB_ENTRY_ALIGNMENT 16
13
14struct tsb {
15 unsigned long tag;
16 unsigned long pte;
17} __attribute__((aligned(TSB_ENTRY_ALIGNMENT)));
18
19/* We use an 8K TSB for the whole kernel, this allows to
20 * handle about 4MB of modules and vmalloc mappings without
21 * incurring many hash conflicts.
22 */
23#define KERNEL_TSB_SIZE_BYTES 8192
24#define KERNEL_TSB_NENTRIES \
25 (KERNEL_TSB_SIZE_BYTES / sizeof(struct tsb))
26
27extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
28
29static inline unsigned long tsb_hash(unsigned long vaddr)
30{
31 vaddr >>= PAGE_SHIFT;
32 return vaddr & (KERNEL_TSB_NENTRIES - 1);
33}
34
35static inline int tag_compare(struct tsb *entry, unsigned long vaddr, unsigned long context)
36{
37 if (context == ~0UL)
38 return 1;
39
40 return (entry->tag == ((vaddr >> 22) | (context << 48)));
41}
42
43/* TSB flushes need only occur on the processor initiating the address
44 * space modification, not on each cpu the address space has run on.
45 * Only the TLB flush needs that treatment.
46 */
47
48void flush_tsb_kernel_range(unsigned long start, unsigned long end)
49{
50 unsigned long v;
51
52 for (v = start; v < end; v += PAGE_SIZE) {
53 struct tsb *ent = &swapper_tsb[tsb_hash(v)];
54
55 if (tag_compare(ent, v, 0)) {
56 ent->tag = 0UL;
57 membar_storeload_storestore();
58 }
59 }
60}
61
62void flush_tsb_user(struct mmu_gather *mp)
63{
64 struct mm_struct *mm = mp->mm;
65 struct tsb *tsb = (struct tsb *) mm->context.sparc64_tsb;
66 unsigned long ctx = ~0UL;
67 int i;
68
69 if (CTX_VALID(mm->context))
70 ctx = CTX_HWBITS(mm->context);
71
72 for (i = 0; i < mp->tlb_nr; i++) {
73 unsigned long v = mp->vaddrs[i];
74 struct tsb *ent;
75
76 v &= ~0x1UL;
77
78 ent = &tsb[tsb_hash(v)];
79 if (tag_compare(ent, v, ctx)) {
80 ent->tag = 0UL;
81 membar_storeload_storestore();
82 }
83 }
84}
diff --git a/arch/sparc64/mm/ultra.S b/arch/sparc64/mm/ultra.S
index e4c9151fa116..22791f29552e 100644
--- a/arch/sparc64/mm/ultra.S
+++ b/arch/sparc64/mm/ultra.S
@@ -453,64 +453,6 @@ xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
453 nop 453 nop
454 nop 454 nop
455 455
456 .data
457
458errata32_hwbug:
459 .xword 0
460
461 .text
462
463 /* These two are not performance critical... */
464 .globl xcall_flush_tlb_all_spitfire
465xcall_flush_tlb_all_spitfire:
466 /* Spitfire Errata #32 workaround. */
467 sethi %hi(errata32_hwbug), %g4
468 stx %g0, [%g4 + %lo(errata32_hwbug)]
469
470 clr %g2
471 clr %g3
4721: ldxa [%g3] ASI_DTLB_DATA_ACCESS, %g4
473 and %g4, _PAGE_L, %g5
474 brnz,pn %g5, 2f
475 mov TLB_TAG_ACCESS, %g7
476
477 stxa %g0, [%g7] ASI_DMMU
478 membar #Sync
479 stxa %g0, [%g3] ASI_DTLB_DATA_ACCESS
480 membar #Sync
481
482 /* Spitfire Errata #32 workaround. */
483 sethi %hi(errata32_hwbug), %g4
484 stx %g0, [%g4 + %lo(errata32_hwbug)]
485
4862: ldxa [%g3] ASI_ITLB_DATA_ACCESS, %g4
487 and %g4, _PAGE_L, %g5
488 brnz,pn %g5, 2f
489 mov TLB_TAG_ACCESS, %g7
490
491 stxa %g0, [%g7] ASI_IMMU
492 membar #Sync
493 stxa %g0, [%g3] ASI_ITLB_DATA_ACCESS
494 membar #Sync
495
496 /* Spitfire Errata #32 workaround. */
497 sethi %hi(errata32_hwbug), %g4
498 stx %g0, [%g4 + %lo(errata32_hwbug)]
499
5002: add %g2, 1, %g2
501 cmp %g2, SPITFIRE_HIGHEST_LOCKED_TLBENT
502 ble,pt %icc, 1b
503 sll %g2, 3, %g3
504 flush %g6
505 retry
506
507 .globl xcall_flush_tlb_all_cheetah
508xcall_flush_tlb_all_cheetah:
509 mov 0x80, %g2
510 stxa %g0, [%g2] ASI_DMMU_DEMAP
511 stxa %g0, [%g2] ASI_IMMU_DEMAP
512 retry
513
514 /* These just get rescheduled to PIL vectors. */ 456 /* These just get rescheduled to PIL vectors. */
515 .globl xcall_call_function 457 .globl xcall_call_function
516xcall_call_function: 458xcall_call_function:
diff --git a/include/asm-sparc64/mmu.h b/include/asm-sparc64/mmu.h
index 8627eed6e83d..36384cf7faa6 100644
--- a/include/asm-sparc64/mmu.h
+++ b/include/asm-sparc64/mmu.h
@@ -92,6 +92,7 @@
92 92
93typedef struct { 93typedef struct {
94 unsigned long sparc64_ctx_val; 94 unsigned long sparc64_ctx_val;
95 unsigned long *sparc64_tsb;
95} mm_context_t; 96} mm_context_t;
96 97
97#endif /* !__ASSEMBLY__ */ 98#endif /* !__ASSEMBLY__ */
diff --git a/include/asm-sparc64/mmu_context.h b/include/asm-sparc64/mmu_context.h
index 57ee7b306189..34640a370ab4 100644
--- a/include/asm-sparc64/mmu_context.h
+++ b/include/asm-sparc64/mmu_context.h
@@ -25,7 +25,13 @@ extern void get_new_mmu_context(struct mm_struct *mm);
25 * This just needs to set mm->context to an invalid context. 25 * This just needs to set mm->context to an invalid context.
26 */ 26 */
27#define init_new_context(__tsk, __mm) \ 27#define init_new_context(__tsk, __mm) \
28 (((__mm)->context.sparc64_ctx_val = 0UL), 0) 28({ unsigned long __pg = get_zeroed_page(GFP_KERNEL); \
29 (__mm)->context.sparc64_ctx_val = 0UL; \
30 (__mm)->context.sparc64_tsb = \
31 (unsigned long *) __pg; \
32 (__pg ? 0 : -ENOMEM); \
33})
34
29 35
30/* Destroy a dead context. This occurs when mmput drops the 36/* Destroy a dead context. This occurs when mmput drops the
31 * mm_users count to zero, the mmaps have been released, and 37 * mm_users count to zero, the mmaps have been released, and
@@ -35,7 +41,8 @@ extern void get_new_mmu_context(struct mm_struct *mm);
35 * this task if valid. 41 * this task if valid.
36 */ 42 */
37#define destroy_context(__mm) \ 43#define destroy_context(__mm) \
38do { spin_lock(&ctx_alloc_lock); \ 44do { free_page((unsigned long)(__mm)->context.sparc64_tsb); \
45 spin_lock(&ctx_alloc_lock); \
39 if (CTX_VALID((__mm)->context)) { \ 46 if (CTX_VALID((__mm)->context)) { \
40 unsigned long nr = CTX_NRBITS((__mm)->context); \ 47 unsigned long nr = CTX_NRBITS((__mm)->context); \
41 mmu_context_bmap[nr>>6] &= ~(1UL << (nr & 63)); \ 48 mmu_context_bmap[nr>>6] &= ~(1UL << (nr & 63)); \
@@ -43,35 +50,7 @@ do { spin_lock(&ctx_alloc_lock); \
43 spin_unlock(&ctx_alloc_lock); \ 50 spin_unlock(&ctx_alloc_lock); \
44} while(0) 51} while(0)
45 52
46/* Reload the two core values used by TLB miss handler 53extern unsigned long tsb_context_switch(unsigned long pgd_pa, unsigned long *tsb);
47 * processing on sparc64. They are:
48 * 1) The physical address of mm->pgd, when full page
49 * table walks are necessary, this is where the
50 * search begins.
51 * 2) A "PGD cache". For 32-bit tasks only pgd[0] is
52 * ever used since that maps the entire low 4GB
53 * completely. To speed up TLB miss processing we
54 * make this value available to the handlers. This
55 * decreases the amount of memory traffic incurred.
56 */
57#define reload_tlbmiss_state(__tsk, __mm) \
58do { \
59 register unsigned long paddr asm("o5"); \
60 register unsigned long pgd_cache asm("o4"); \
61 paddr = __pa((__mm)->pgd); \
62 pgd_cache = 0UL; \
63 if (task_thread_info(__tsk)->flags & _TIF_32BIT) \
64 pgd_cache = get_pgd_cache((__mm)->pgd); \
65 __asm__ __volatile__("wrpr %%g0, 0x494, %%pstate\n\t" \
66 "mov %3, %%g4\n\t" \
67 "mov %0, %%g7\n\t" \
68 "stxa %1, [%%g4] %2\n\t" \
69 "membar #Sync\n\t" \
70 "wrpr %%g0, 0x096, %%pstate" \
71 : /* no outputs */ \
72 : "r" (paddr), "r" (pgd_cache),\
73 "i" (ASI_DMMU), "i" (TSB_REG)); \
74} while(0)
75 54
76/* Set MMU context in the actual hardware. */ 55/* Set MMU context in the actual hardware. */
77#define load_secondary_context(__mm) \ 56#define load_secondary_context(__mm) \
@@ -101,7 +80,8 @@ static inline void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, str
101 80
102 if (!ctx_valid || (old_mm != mm)) { 81 if (!ctx_valid || (old_mm != mm)) {
103 load_secondary_context(mm); 82 load_secondary_context(mm);
104 reload_tlbmiss_state(tsk, mm); 83 tsb_context_switch(__pa(mm->pgd),
84 mm->context.sparc64_tsb);
105 } 85 }
106 86
107 /* Even if (mm == old_mm) we _must_ check 87 /* Even if (mm == old_mm) we _must_ check
@@ -139,7 +119,7 @@ static inline void activate_mm(struct mm_struct *active_mm, struct mm_struct *mm
139 119
140 load_secondary_context(mm); 120 load_secondary_context(mm);
141 __flush_tlb_mm(CTX_HWBITS(mm->context), SECONDARY_CONTEXT); 121 __flush_tlb_mm(CTX_HWBITS(mm->context), SECONDARY_CONTEXT);
142 reload_tlbmiss_state(current, mm); 122 tsb_context_switch(__pa(mm->pgd), mm->context.sparc64_tsb);
143} 123}
144 124
145#endif /* !(__ASSEMBLY__) */ 125#endif /* !(__ASSEMBLY__) */
diff --git a/include/asm-sparc64/pgalloc.h b/include/asm-sparc64/pgalloc.h
index a96067cca963..baf59c00ea47 100644
--- a/include/asm-sparc64/pgalloc.h
+++ b/include/asm-sparc64/pgalloc.h
@@ -61,6 +61,7 @@ static __inline__ void free_pgd_slow(pgd_t *pgd)
61 free_page((unsigned long)pgd); 61 free_page((unsigned long)pgd);
62} 62}
63 63
64/* XXX This crap can die, no longer using virtual page tables... */
64#ifdef DCACHE_ALIASING_POSSIBLE 65#ifdef DCACHE_ALIASING_POSSIBLE
65#define VPTE_COLOR(address) (((address) >> (PAGE_SHIFT + 10)) & 1UL) 66#define VPTE_COLOR(address) (((address) >> (PAGE_SHIFT + 10)) & 1UL)
66#define DCACHE_COLOR(address) (((address) >> PAGE_SHIFT) & 1UL) 67#define DCACHE_COLOR(address) (((address) >> PAGE_SHIFT) & 1UL)
diff --git a/include/asm-sparc64/pgtable.h b/include/asm-sparc64/pgtable.h
index f0a9b44d3eb5..f3ba1e058195 100644
--- a/include/asm-sparc64/pgtable.h
+++ b/include/asm-sparc64/pgtable.h
@@ -25,7 +25,8 @@
25#include <asm/const.h> 25#include <asm/const.h>
26 26
27/* The kernel image occupies 0x4000000 to 0x1000000 (4MB --> 32MB). 27/* The kernel image occupies 0x4000000 to 0x1000000 (4MB --> 32MB).
28 * The page copy blockops can use 0x2000000 to 0x10000000. 28 * The page copy blockops can use 0x2000000 to 0x4000000.
29 * The TSB is mapped in the 0x4000000 to 0x6000000 range.
29 * The PROM resides in an area spanning 0xf0000000 to 0x100000000. 30 * The PROM resides in an area spanning 0xf0000000 to 0x100000000.
30 * The vmalloc area spans 0x100000000 to 0x200000000. 31 * The vmalloc area spans 0x100000000 to 0x200000000.
31 * Since modules need to be in the lowest 32-bits of the address space, 32 * Since modules need to be in the lowest 32-bits of the address space,
@@ -34,6 +35,7 @@
34 * 0x400000000. 35 * 0x400000000.
35 */ 36 */
36#define TLBTEMP_BASE _AC(0x0000000002000000,UL) 37#define TLBTEMP_BASE _AC(0x0000000002000000,UL)
38#define TSBMAP_BASE _AC(0x0000000004000000,UL)
37#define MODULES_VADDR _AC(0x0000000010000000,UL) 39#define MODULES_VADDR _AC(0x0000000010000000,UL)
38#define MODULES_LEN _AC(0x00000000e0000000,UL) 40#define MODULES_LEN _AC(0x00000000e0000000,UL)
39#define MODULES_END _AC(0x00000000f0000000,UL) 41#define MODULES_END _AC(0x00000000f0000000,UL)
@@ -296,11 +298,6 @@ static inline pte_t pte_modify(pte_t orig_pte, pgprot_t new_prot)
296/* to find an entry in a kernel page-table-directory */ 298/* to find an entry in a kernel page-table-directory */
297#define pgd_offset_k(address) pgd_offset(&init_mm, address) 299#define pgd_offset_k(address) pgd_offset(&init_mm, address)
298 300
299/* extract the pgd cache used for optimizing the tlb miss
300 * slow path when executing 32-bit compat processes
301 */
302#define get_pgd_cache(pgd) ((unsigned long) pgd_val(*pgd) << 11)
303
304/* Find an entry in the second-level page table.. */ 301/* Find an entry in the second-level page table.. */
305#define pmd_offset(pudp, address) \ 302#define pmd_offset(pudp, address) \
306 ((pmd_t *) pud_page(*(pudp)) + \ 303 ((pmd_t *) pud_page(*(pudp)) + \
diff --git a/include/asm-sparc64/processor.h b/include/asm-sparc64/processor.h
index cd8d9b4c8658..b3889f3f943a 100644
--- a/include/asm-sparc64/processor.h
+++ b/include/asm-sparc64/processor.h
@@ -28,6 +28,8 @@
28 * User lives in his very own context, and cannot reference us. Note 28 * User lives in his very own context, and cannot reference us. Note
29 * that TASK_SIZE is a misnomer, it really gives maximum user virtual 29 * that TASK_SIZE is a misnomer, it really gives maximum user virtual
30 * address that the kernel will allocate out. 30 * address that the kernel will allocate out.
31 *
32 * XXX No longer using virtual page tables, kill this upper limit...
31 */ 33 */
32#define VA_BITS 44 34#define VA_BITS 44
33#ifndef __ASSEMBLY__ 35#ifndef __ASSEMBLY__
@@ -37,18 +39,6 @@
37#endif 39#endif
38#define TASK_SIZE ((unsigned long)-VPTE_SIZE) 40#define TASK_SIZE ((unsigned long)-VPTE_SIZE)
39 41
40/*
41 * The vpte base must be able to hold the entire vpte, half
42 * of which lives above, and half below, the base. And it
43 * is placed as close to the highest address range as possible.
44 */
45#define VPTE_BASE_SPITFIRE (-(VPTE_SIZE/2))
46#if 1
47#define VPTE_BASE_CHEETAH VPTE_BASE_SPITFIRE
48#else
49#define VPTE_BASE_CHEETAH 0xffe0000000000000
50#endif
51
52#ifndef __ASSEMBLY__ 42#ifndef __ASSEMBLY__
53 43
54typedef struct { 44typedef struct {
diff --git a/include/asm-sparc64/tlbflush.h b/include/asm-sparc64/tlbflush.h
index 3ef9909ac3ac..9ad5d9c51d42 100644
--- a/include/asm-sparc64/tlbflush.h
+++ b/include/asm-sparc64/tlbflush.h
@@ -5,6 +5,11 @@
5#include <linux/mm.h> 5#include <linux/mm.h>
6#include <asm/mmu_context.h> 6#include <asm/mmu_context.h>
7 7
8/* TSB flush operations. */
9struct mmu_gather;
10extern void flush_tsb_kernel_range(unsigned long start, unsigned long end);
11extern void flush_tsb_user(struct mmu_gather *mp);
12
8/* TLB flush operations. */ 13/* TLB flush operations. */
9 14
10extern void flush_tlb_pending(void); 15extern void flush_tlb_pending(void);
@@ -14,28 +19,36 @@ extern void flush_tlb_pending(void);
14#define flush_tlb_page(vma,addr) flush_tlb_pending() 19#define flush_tlb_page(vma,addr) flush_tlb_pending()
15#define flush_tlb_mm(mm) flush_tlb_pending() 20#define flush_tlb_mm(mm) flush_tlb_pending()
16 21
22/* Local cpu only. */
17extern void __flush_tlb_all(void); 23extern void __flush_tlb_all(void);
24
18extern void __flush_tlb_page(unsigned long context, unsigned long page, unsigned long r); 25extern void __flush_tlb_page(unsigned long context, unsigned long page, unsigned long r);
19 26
20extern void __flush_tlb_kernel_range(unsigned long start, unsigned long end); 27extern void __flush_tlb_kernel_range(unsigned long start, unsigned long end);
21 28
22#ifndef CONFIG_SMP 29#ifndef CONFIG_SMP
23 30
24#define flush_tlb_all() __flush_tlb_all()
25#define flush_tlb_kernel_range(start,end) \ 31#define flush_tlb_kernel_range(start,end) \
26 __flush_tlb_kernel_range(start,end) 32do { flush_tsb_kernel_range(start,end); \
33 __flush_tlb_kernel_range(start,end); \
34} while (0)
27 35
28#else /* CONFIG_SMP */ 36#else /* CONFIG_SMP */
29 37
30extern void smp_flush_tlb_all(void);
31extern void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end); 38extern void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end);
32 39
33#define flush_tlb_all() smp_flush_tlb_all()
34#define flush_tlb_kernel_range(start, end) \ 40#define flush_tlb_kernel_range(start, end) \
35 smp_flush_tlb_kernel_range(start, end) 41do { flush_tsb_kernel_range(start,end); \
42 smp_flush_tlb_kernel_range(start, end); \
43} while (0)
36 44
37#endif /* ! CONFIG_SMP */ 45#endif /* ! CONFIG_SMP */
38 46
39extern void flush_tlb_pgtables(struct mm_struct *, unsigned long, unsigned long); 47static inline void flush_tlb_pgtables(struct mm_struct *mm, unsigned long start, unsigned long end)
48{
49 /* We don't use virtual page tables for TLB miss processing
50 * any more. Nowadays we use the TSB.
51 */
52}
40 53
41#endif /* _SPARC64_TLBFLUSH_H */ 54#endif /* _SPARC64_TLBFLUSH_H */
diff --git a/include/asm-sparc64/tsb.h b/include/asm-sparc64/tsb.h
new file mode 100644
index 000000000000..03d272e0e477
--- /dev/null
+++ b/include/asm-sparc64/tsb.h
@@ -0,0 +1,165 @@
1#ifndef _SPARC64_TSB_H
2#define _SPARC64_TSB_H
3
4/* The sparc64 TSB is similar to the powerpc hashtables. It's a
5 * power-of-2 sized table of TAG/PTE pairs. The cpu precomputes
6 * pointers into this table for 8K and 64K page sizes, and also a
7 * comparison TAG based upon the virtual address and context which
8 * faults.
9 *
10 * TLB miss trap handler software does the actual lookup via something
11 * of the form:
12 *
13 * ldxa [%g0] ASI_{D,I}MMU_TSB_8KB_PTR, %g1
14 * ldxa [%g0] ASI_{D,I}MMU, %g6
15 * ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4
16 * cmp %g4, %g6
17 * bne,pn %xcc, tsb_miss_{d,i}tlb
18 * mov FAULT_CODE_{D,I}TLB, %g3
19 * stxa %g5, [%g0] ASI_{D,I}TLB_DATA_IN
20 * retry
21 *
22
23 * Each 16-byte slot of the TSB is the 8-byte tag and then the 8-byte
24 * PTE. The TAG is of the same layout as the TLB TAG TARGET mmu
25 * register which is:
26 *
27 * -------------------------------------------------
28 * | - | CONTEXT | - | VADDR bits 63:22 |
29 * -------------------------------------------------
30 * 63 61 60 48 47 42 41 0
31 *
32 * Like the powerpc hashtables we need to use locking in order to
33 * synchronize while we update the entries. PTE updates need locking
34 * as well.
35 *
36 * We need to carefully choose a lock bits for the TSB entry. We
37 * choose to use bit 47 in the tag. Also, since we never map anything
38 * at page zero in context zero, we use zero as an invalid tag entry.
39 * When the lock bit is set, this forces a tag comparison failure.
40 *
41 * Currently, we allocate an 8K TSB per-process and we use it for both
42 * I-TLB and D-TLB misses. Perhaps at some point we'll add code that
43 * monitors the number of active pages in the process as we get
44 * major/minor faults, and grow the TSB in response. The only trick
45 * in implementing that is synchronizing the freeing of the old TSB
46 * wrt. parallel TSB updates occuring on other processors. On
47 * possible solution is to use RCU for the freeing of the TSB.
48 */
49
50#define TSB_TAG_LOCK (1 << (47 - 32))
51
52#define TSB_MEMBAR membar #StoreStore
53
54#define TSB_LOCK_TAG(TSB, REG1, REG2) \
5599: lduwa [TSB] ASI_N, REG1; \
56 sethi %hi(TSB_TAG_LOCK), REG2;\
57 andcc REG1, REG2, %g0; \
58 bne,pn %icc, 99b; \
59 nop; \
60 casa [TSB] ASI_N, REG1, REG2;\
61 cmp REG1, REG2; \
62 bne,pn %icc, 99b; \
63 nop; \
64 TSB_MEMBAR
65
66#define TSB_WRITE(TSB, TTE, TAG) \
67 stx TTE, [TSB + 0x08]; \
68 TSB_MEMBAR; \
69 stx TAG, [TSB + 0x00];
70
71 /* Do a kernel page table walk. Leaves physical PTE pointer in
72 * REG1. Jumps to FAIL_LABEL on early page table walk termination.
73 * VADDR will not be clobbered, but REG2 will.
74 */
75#define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL) \
76 sethi %hi(swapper_pg_dir), REG1; \
77 or REG1, %lo(swapper_pg_dir), REG1; \
78 sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
79 srlx REG2, 64 - PAGE_SHIFT, REG2; \
80 andn REG2, 0x3, REG2; \
81 lduw [REG1 + REG2], REG1; \
82 brz,pn REG1, FAIL_LABEL; \
83 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
84 srlx REG2, 64 - PAGE_SHIFT, REG2; \
85 sllx REG1, 11, REG1; \
86 andn REG2, 0x3, REG2; \
87 lduwa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
88 brz,pn REG1, FAIL_LABEL; \
89 sllx VADDR, 64 - PMD_SHIFT, REG2; \
90 srlx REG2, 64 - PAGE_SHIFT, REG2; \
91 sllx REG1, 11, REG1; \
92 andn REG2, 0x7, REG2; \
93 add REG1, REG2, REG1;
94
95 /* Do a user page table walk in MMU globals. Leaves physical PTE
96 * pointer in REG1. Jumps to FAIL_LABEL on early page table walk
97 * termination. Physical base of page tables is in PHYS_PGD which
98 * will not be modified.
99 *
100 * VADDR will not be clobbered, but REG1 and REG2 will.
101 */
102#define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL) \
103 sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
104 srlx REG2, 64 - PAGE_SHIFT, REG2; \
105 andn REG2, 0x3, REG2; \
106 lduwa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \
107 brz,pn REG1, FAIL_LABEL; \
108 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
109 srlx REG2, 64 - PAGE_SHIFT, REG2; \
110 sllx REG1, 11, REG1; \
111 andn REG2, 0x3, REG2; \
112 lduwa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
113 brz,pn REG1, FAIL_LABEL; \
114 sllx VADDR, 64 - PMD_SHIFT, REG2; \
115 srlx REG2, 64 - PAGE_SHIFT, REG2; \
116 sllx REG1, 11, REG1; \
117 andn REG2, 0x7, REG2; \
118 add REG1, REG2, REG1;
119
120/* Lookup a OBP mapping on VADDR in the prom_trans[] table at TL>0.
121 * If no entry is found, FAIL_LABEL will be branched to. On success
122 * the resulting PTE value will be left in REG1. VADDR is preserved
123 * by this routine.
124 */
125#define OBP_TRANS_LOOKUP(VADDR, REG1, REG2, REG3, FAIL_LABEL) \
126 sethi %hi(prom_trans), REG1; \
127 or REG1, %lo(prom_trans), REG1; \
12897: ldx [REG1 + 0x00], REG2; \
129 brz,pn REG2, FAIL_LABEL; \
130 nop; \
131 ldx [REG1 + 0x08], REG3; \
132 add REG2, REG3, REG3; \
133 cmp REG2, VADDR; \
134 bgu,pt %xcc, 98f; \
135 cmp VADDR, REG3; \
136 bgeu,pt %xcc, 98f; \
137 ldx [REG1 + 0x10], REG3; \
138 sub VADDR, REG2, REG2; \
139 ba,pt %xcc, 99f; \
140 add REG3, REG2, REG1; \
14198: ba,pt %xcc, 97b; \
142 add REG1, (3 * 8), REG1; \
14399:
144
145 /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL
146 * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries
147 * and the found TTE will be left in REG1. REG3 and REG4 must
148 * be an even/odd pair of registers.
149 *
150 * VADDR and TAG will be preserved and not clobbered by this macro.
151 */
152 /* XXX non-8K base page size support... */
153#define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
154 sethi %hi(swapper_tsb), REG1; \
155 or REG1, %lo(swapper_tsb), REG1; \
156 srlx VADDR, 13, REG2; \
157 and REG2, (512 - 1), REG2; \
158 sllx REG2, 4, REG2; \
159 add REG1, REG2, REG2; \
160 ldda [REG2] ASI_NUCLEUS_QUAD_LDD, REG3; \
161 cmp REG3, TAG; \
162 be,a,pt %xcc, OK_LABEL; \
163 mov REG4, REG1;
164
165#endif /* !(_SPARC64_TSB_H) */
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-11 14:10:08 -0500