aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/sparc64/kernel/Makefile2
-rw-r--r--arch/sparc64/kernel/binfmt_aout32.c13
-rw-r--r--arch/sparc64/kernel/dtlb_backend.S170
-rw-r--r--arch/sparc64/kernel/dtlb_base.S109
-rw-r--r--arch/sparc64/kernel/dtlb_miss.S39
-rw-r--r--arch/sparc64/kernel/etrap.S2
-rw-r--r--arch/sparc64/kernel/head.S46
-rw-r--r--arch/sparc64/kernel/itlb_base.S79
-rw-r--r--arch/sparc64/kernel/itlb_miss.S39
-rw-r--r--arch/sparc64/kernel/ktlb.S263
-rw-r--r--arch/sparc64/kernel/process.c25
-rw-r--r--arch/sparc64/kernel/rtrap.S6
-rw-r--r--arch/sparc64/kernel/smp.c12
-rw-r--r--arch/sparc64/kernel/trampoline.S33
-rw-r--r--arch/sparc64/kernel/tsb.S169
-rw-r--r--arch/sparc64/kernel/ttable.S6
-rw-r--r--arch/sparc64/kernel/vmlinux.lds.S3
-rw-r--r--arch/sparc64/kernel/winfixup.S8
-rw-r--r--arch/sparc64/mm/Makefile2
-rw-r--r--arch/sparc64/mm/init.c91
-rw-r--r--arch/sparc64/mm/tlb.c61
-rw-r--r--arch/sparc64/mm/tsb.c84
-rw-r--r--arch/sparc64/mm/ultra.S58
-rw-r--r--include/asm-sparc64/mmu.h1
-rw-r--r--include/asm-sparc64/mmu_context.h46
-rw-r--r--include/asm-sparc64/pgalloc.h1
-rw-r--r--include/asm-sparc64/pgtable.h9
-rw-r--r--include/asm-sparc64/processor.h14
-rw-r--r--include/asm-sparc64/tlbflush.h25
-rw-r--r--include/asm-sparc64/tsb.h165
30 files changed, 693 insertions, 888 deletions
diff --git a/arch/sparc64/kernel/Makefile b/arch/sparc64/kernel/Makefile
index 83d67eb18895..a482a9ffe5bc 100644
--- a/arch/sparc64/kernel/Makefile
+++ b/arch/sparc64/kernel/Makefile
@@ -38,5 +38,5 @@ else
38 CMODEL_CFLAG := -m64 -mcmodel=medlow 38 CMODEL_CFLAG := -m64 -mcmodel=medlow
39endif 39endif
40 40
41head.o: head.S ttable.S itlb_base.S dtlb_base.S dtlb_backend.S dtlb_prot.S \ 41head.o: head.S ttable.S itlb_miss.S dtlb_miss.S ktlb.S tsb.S \
42 etrap.S rtrap.S winfixup.S entry.S 42 etrap.S rtrap.S winfixup.S entry.S
diff --git a/arch/sparc64/kernel/binfmt_aout32.c b/arch/sparc64/kernel/binfmt_aout32.c
index 202a80c24b6f..a57d7f2b6f13 100644
--- a/arch/sparc64/kernel/binfmt_aout32.c
+++ b/arch/sparc64/kernel/binfmt_aout32.c
@@ -31,6 +31,7 @@
31#include <asm/system.h> 31#include <asm/system.h>
32#include <asm/uaccess.h> 32#include <asm/uaccess.h>
33#include <asm/pgalloc.h> 33#include <asm/pgalloc.h>
34#include <asm/mmu_context.h>
34 35
35static int load_aout32_binary(struct linux_binprm *, struct pt_regs * regs); 36static int load_aout32_binary(struct linux_binprm *, struct pt_regs * regs);
36static int load_aout32_library(struct file*); 37static int load_aout32_library(struct file*);
@@ -329,15 +330,9 @@ beyond_if:
329 330
330 current->mm->start_stack = 331 current->mm->start_stack =
331 (unsigned long) create_aout32_tables((char __user *)bprm->p, bprm); 332 (unsigned long) create_aout32_tables((char __user *)bprm->p, bprm);
332 if (!(orig_thr_flags & _TIF_32BIT)) { 333 tsb_context_switch(__pa(current->mm->pgd),
333 unsigned long pgd_cache = get_pgd_cache(current->mm->pgd); 334 current->mm->context.sparc64_tsb);
334 335
335 __asm__ __volatile__("stxa\t%0, [%1] %2\n\t"
336 "membar #Sync"
337 : /* no outputs */
338 : "r" (pgd_cache),
339 "r" (TSB_REG), "i" (ASI_DMMU));
340 }
341 start_thread32(regs, ex.a_entry, current->mm->start_stack); 336 start_thread32(regs, ex.a_entry, current->mm->start_stack);
342 if (current->ptrace & PT_PTRACED) 337 if (current->ptrace & PT_PTRACED)
343 send_sig(SIGTRAP, current, 0); 338 send_sig(SIGTRAP, current, 0);
diff --git a/arch/sparc64/kernel/dtlb_backend.S b/arch/sparc64/kernel/dtlb_backend.S
deleted file mode 100644
index acc889a7f9c1..000000000000
--- a/arch/sparc64/kernel/dtlb_backend.S
+++ /dev/null
@@ -1,170 +0,0 @@
1/* $Id: dtlb_backend.S,v 1.16 2001/10/09 04:02:11 davem Exp $
2 * dtlb_backend.S: Back end to DTLB miss replacement strategy.
3 * This is included directly into the trap table.
4 *
5 * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com)
6 * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz)
7 */
8
9#include <asm/pgtable.h>
10#include <asm/mmu.h>
11
12#define VALID_SZ_BITS (_PAGE_VALID | _PAGE_SZBITS)
13
14#define VPTE_BITS (_PAGE_CP | _PAGE_CV | _PAGE_P )
15#define VPTE_SHIFT (PAGE_SHIFT - 3)
16
17/* Ways we can get here:
18 *
19 * 1) Nucleus loads and stores to/from PA-->VA direct mappings at tl>1.
20 * 2) Nucleus loads and stores to/from user/kernel window save areas.
21 * 3) VPTE misses from dtlb_base and itlb_base.
22 *
23 * We need to extract out the PMD and PGDIR indexes from the
24 * linear virtual page table access address. The PTE index
25 * is at the bottom, but we are not concerned with it. Bits
26 * 0 to 2 are clear since each PTE is 8 bytes in size. Each
27 * PMD and PGDIR entry are 4 bytes in size. Thus, this
28 * address looks something like:
29 *
30 * |---------------------------------------------------------------|
31 * | ... | PGDIR index | PMD index | PTE index | |
32 * |---------------------------------------------------------------|
33 * 63 F E D C B A 3 2 0 <- bit nr
34 *
35 * The variable bits above are defined as:
36 * A --> 3 + (PAGE_SHIFT - log2(8))
37 * --> 3 + (PAGE_SHIFT - 3) - 1
38 * (ie. this is "bit 3" + PAGE_SIZE - size of PTE entry in bits - 1)
39 * B --> A + 1
40 * C --> B + (PAGE_SHIFT - log2(4))
41 * --> B + (PAGE_SHIFT - 2) - 1
42 * (ie. this is "bit B" + PAGE_SIZE - size of PMD entry in bits - 1)
43 * D --> C + 1
44 * E --> D + (PAGE_SHIFT - log2(4))
45 * --> D + (PAGE_SHIFT - 2) - 1
46 * (ie. this is "bit D" + PAGE_SIZE - size of PGDIR entry in bits - 1)
47 * F --> E + 1
48 *
49 * (Note how "B" always evalutes to PAGE_SHIFT, all the other constants
50 * cancel out.)
51 *
52 * For 8K PAGE_SIZE (thus, PAGE_SHIFT of 13) the bit numbers are:
53 * A --> 12
54 * B --> 13
55 * C --> 23
56 * D --> 24
57 * E --> 34
58 * F --> 35
59 *
60 * For 64K PAGE_SIZE (thus, PAGE_SHIFT of 16) the bit numbers are:
61 * A --> 15
62 * B --> 16
63 * C --> 29
64 * D --> 30
65 * E --> 43
66 * F --> 44
67 *
68 * Because bits both above and below each PGDIR and PMD index need to
69 * be masked out, and the index can be as long as 14 bits (when using a
70 * 64K PAGE_SIZE, and thus a PAGE_SHIFT of 16), we need 3 instructions
71 * to extract each index out.
72 *
73 * Shifts do not pair very well on UltraSPARC-I, II, IIi, and IIe, so
74 * we try to avoid using them for the entire operation. We could setup
75 * a mask anywhere from bit 31 down to bit 10 using the sethi instruction.
76 *
77 * We need a mask covering bits B --> C and one covering D --> E.
78 * For 8K PAGE_SIZE these masks are 0x00ffe000 and 0x7ff000000.
79 * For 64K PAGE_SIZE these masks are 0x3fff0000 and 0xfffc0000000.
80 * The second in each set cannot be loaded with a single sethi
81 * instruction, because the upper bits are past bit 32. We would
82 * need to use a sethi + a shift.
83 *
84 * For the time being, we use 2 shifts and a simple "and" mask.
85 * We shift left to clear the bits above the index, we shift down
86 * to clear the bits below the index (sans the log2(4 or 8) bits)
87 * and a mask to clear the log2(4 or 8) bits. We need therefore
88 * define 4 shift counts, all of which are relative to PAGE_SHIFT.
89 *
90 * Although unsupportable for other reasons, this does mean that
91 * 512K and 4MB page sizes would be generaally supported by the
92 * kernel. (ELF binaries would break with > 64K PAGE_SIZE since
93 * the sections are only aligned that strongly).
94 *
95 * The operations performed for extraction are thus:
96 *
97 * ((X << FOO_SHIFT_LEFT) >> FOO_SHIFT_RIGHT) & ~0x3
98 *
99 */
100
101#define A (3 + (PAGE_SHIFT - 3) - 1)
102#define B (A + 1)
103#define C (B + (PAGE_SHIFT - 2) - 1)
104#define D (C + 1)
105#define E (D + (PAGE_SHIFT - 2) - 1)
106#define F (E + 1)
107
108#define PMD_SHIFT_LEFT (64 - D)
109#define PMD_SHIFT_RIGHT (64 - (D - B) - 2)
110#define PGDIR_SHIFT_LEFT (64 - F)
111#define PGDIR_SHIFT_RIGHT (64 - (F - D) - 2)
112#define LOW_MASK_BITS 0x3
113
114/* TLB1 ** ICACHE line 1: tl1 DTLB and quick VPTE miss */
115 ldxa [%g1 + %g1] ASI_DMMU, %g4 ! Get TAG_ACCESS
116 add %g3, %g3, %g5 ! Compute VPTE base
117 cmp %g4, %g5 ! VPTE miss?
118 bgeu,pt %xcc, 1f ! Continue here
119 andcc %g4, TAG_CONTEXT_BITS, %g5 ! tl0 miss Nucleus test
120 ba,a,pt %xcc, from_tl1_trap ! Fall to tl0 miss
1211: sllx %g6, VPTE_SHIFT, %g4 ! Position TAG_ACCESS
122 or %g4, %g5, %g4 ! Prepare TAG_ACCESS
123
124/* TLB1 ** ICACHE line 2: Quick VPTE miss */
125 mov TSB_REG, %g1 ! Grab TSB reg
126 ldxa [%g1] ASI_DMMU, %g5 ! Doing PGD caching?
127 sllx %g6, PMD_SHIFT_LEFT, %g1 ! Position PMD offset
128 be,pn %xcc, sparc64_vpte_nucleus ! Is it from Nucleus?
129 srlx %g1, PMD_SHIFT_RIGHT, %g1 ! Mask PMD offset bits
130 brnz,pt %g5, sparc64_vpte_continue ! Yep, go like smoke
131 andn %g1, LOW_MASK_BITS, %g1 ! Final PMD mask
132 sllx %g6, PGDIR_SHIFT_LEFT, %g5 ! Position PGD offset
133
134/* TLB1 ** ICACHE line 3: Quick VPTE miss */
135 srlx %g5, PGDIR_SHIFT_RIGHT, %g5 ! Mask PGD offset bits
136 andn %g5, LOW_MASK_BITS, %g5 ! Final PGD mask
137 lduwa [%g7 + %g5] ASI_PHYS_USE_EC, %g5! Load PGD
138 brz,pn %g5, vpte_noent ! Valid?
139sparc64_kpte_continue:
140 sllx %g5, 11, %g5 ! Shift into place
141sparc64_vpte_continue:
142 lduwa [%g5 + %g1] ASI_PHYS_USE_EC, %g5! Load PMD
143 sllx %g5, 11, %g5 ! Shift into place
144 brz,pn %g5, vpte_noent ! Valid?
145
146/* TLB1 ** ICACHE line 4: Quick VPTE miss */
147 mov (VALID_SZ_BITS >> 61), %g1 ! upper vpte into %g1
148 sllx %g1, 61, %g1 ! finish calc
149 or %g5, VPTE_BITS, %g5 ! Prepare VPTE data
150 or %g5, %g1, %g5 ! ...
151 mov TLB_SFSR, %g1 ! Restore %g1 value
152 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Load VPTE into TLB
153 stxa %g4, [%g1 + %g1] ASI_DMMU ! Restore previous TAG_ACCESS
154 retry ! Load PTE once again
155
156#undef VALID_SZ_BITS
157#undef VPTE_SHIFT
158#undef VPTE_BITS
159#undef A
160#undef B
161#undef C
162#undef D
163#undef E
164#undef F
165#undef PMD_SHIFT_LEFT
166#undef PMD_SHIFT_RIGHT
167#undef PGDIR_SHIFT_LEFT
168#undef PGDIR_SHIFT_RIGHT
169#undef LOW_MASK_BITS
170
diff --git a/arch/sparc64/kernel/dtlb_base.S b/arch/sparc64/kernel/dtlb_base.S
deleted file mode 100644
index 6528786840c0..000000000000
--- a/arch/sparc64/kernel/dtlb_base.S
+++ /dev/null
@@ -1,109 +0,0 @@
1/* $Id: dtlb_base.S,v 1.17 2001/10/11 22:33:52 davem Exp $
2 * dtlb_base.S: Front end to DTLB miss replacement strategy.
3 * This is included directly into the trap table.
4 *
5 * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com)
6 * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz)
7 */
8
9#include <asm/pgtable.h>
10#include <asm/mmu.h>
11
12/* %g1 TLB_SFSR (%g1 + %g1 == TLB_TAG_ACCESS)
13 * %g2 (KERN_HIGHBITS | KERN_LOWBITS)
14 * %g3 VPTE base (0xfffffffe00000000) Spitfire/Blackbird (44-bit VA space)
15 * (0xffe0000000000000) Cheetah (64-bit VA space)
16 * %g7 __pa(current->mm->pgd)
17 *
18 * The VPTE base value is completely magic, but note that
19 * few places in the kernel other than these TLB miss
20 * handlers know anything about the VPTE mechanism or
21 * how it works (see VPTE_SIZE, TASK_SIZE and PTRS_PER_PGD).
22 * Consider the 44-bit VADDR Ultra-I/II case as an example:
23 *
24 * VA[0 : (1<<43)] produce VPTE index [%g3 : 0]
25 * VA[0 : -(1<<43)] produce VPTE index [%g3-(1<<(43-PAGE_SHIFT+3)) : %g3]
26 *
27 * For Cheetah's 64-bit VADDR space this is:
28 *
29 * VA[0 : (1<<63)] produce VPTE index [%g3 : 0]
30 * VA[0 : -(1<<63)] produce VPTE index [%g3-(1<<(63-PAGE_SHIFT+3)) : %g3]
31 *
32 * If you're paying attention you'll notice that this means half of
33 * the VPTE table is above %g3 and half is below, low VA addresses
34 * map progressively upwards from %g3, and high VA addresses map
35 * progressively upwards towards %g3. This trick was needed to make
36 * the same 8 instruction handler work both for Spitfire/Blackbird's
37 * peculiar VA space hole configuration and the full 64-bit VA space
38 * one of Cheetah at the same time.
39 */
40
41/* Ways we can get here:
42 *
43 * 1) Nucleus loads and stores to/from PA-->VA direct mappings.
44 * 2) Nucleus loads and stores to/from vmalloc() areas.
45 * 3) User loads and stores.
46 * 4) User space accesses by nucleus at tl0
47 */
48
49#if PAGE_SHIFT == 13
50/*
51 * To compute vpte offset, we need to do ((addr >> 13) << 3),
52 * which can be optimized to (addr >> 10) if bits 10/11/12 can
53 * be guaranteed to be 0 ... mmu_context.h does guarantee this
54 * by only using 10 bits in the hwcontext value.
55 */
56#define CREATE_VPTE_OFFSET1(r1, r2) nop
57#define CREATE_VPTE_OFFSET2(r1, r2) \
58 srax r1, 10, r2
59#else
60#define CREATE_VPTE_OFFSET1(r1, r2) \
61 srax r1, PAGE_SHIFT, r2
62#define CREATE_VPTE_OFFSET2(r1, r2) \
63 sllx r2, 3, r2
64#endif
65
66/* DTLB ** ICACHE line 1: Quick user TLB misses */
67 mov TLB_SFSR, %g1
68 ldxa [%g1 + %g1] ASI_DMMU, %g4 ! Get TAG_ACCESS
69 andcc %g4, TAG_CONTEXT_BITS, %g0 ! From Nucleus?
70from_tl1_trap:
71 rdpr %tl, %g5 ! For TL==3 test
72 CREATE_VPTE_OFFSET1(%g4, %g6) ! Create VPTE offset
73 be,pn %xcc, kvmap ! Yep, special processing
74 CREATE_VPTE_OFFSET2(%g4, %g6) ! Create VPTE offset
75 cmp %g5, 4 ! Last trap level?
76
77/* DTLB ** ICACHE line 2: User finish + quick kernel TLB misses */
78 be,pn %xcc, longpath ! Yep, cannot risk VPTE miss
79 nop ! delay slot
80 ldxa [%g3 + %g6] ASI_S, %g5 ! Load VPTE
811: brgez,pn %g5, longpath ! Invalid, branch out
82 nop ! Delay-slot
839: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
84 retry ! Trap return
85 nop
86
87/* DTLB ** ICACHE line 3: winfixups+real_faults */
88longpath:
89 rdpr %pstate, %g5 ! Move into alternate globals
90 wrpr %g5, PSTATE_AG|PSTATE_MG, %pstate
91 rdpr %tl, %g4 ! See where we came from.
92 cmp %g4, 1 ! Is etrap/rtrap window fault?
93 mov TLB_TAG_ACCESS, %g4 ! Prepare for fault processing
94 ldxa [%g4] ASI_DMMU, %g5 ! Load faulting VA page
95 be,pt %xcc, sparc64_realfault_common ! Jump to normal fault handling
96 mov FAULT_CODE_DTLB, %g4 ! It was read from DTLB
97
98/* DTLB ** ICACHE line 4: Unused... */
99 ba,a,pt %xcc, winfix_trampoline ! Call window fixup code
100 nop
101 nop
102 nop
103 nop
104 nop
105 nop
106 nop
107
108#undef CREATE_VPTE_OFFSET1
109#undef CREATE_VPTE_OFFSET2
diff --git a/arch/sparc64/kernel/dtlb_miss.S b/arch/sparc64/kernel/dtlb_miss.S
new file mode 100644
index 000000000000..d0f1565cb564
--- /dev/null
+++ b/arch/sparc64/kernel/dtlb_miss.S
@@ -0,0 +1,39 @@
1/* DTLB ** ICACHE line 1: Context 0 check and TSB load */
2 ldxa [%g0] ASI_DMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer
3 ldxa [%g0] ASI_DMMU, %g6 ! Get TAG TARGET
4 srlx %g6, 48, %g5 ! Get context
5 brz,pn %g5, kvmap_dtlb ! Context 0 processing
6 nop ! Delay slot (fill me)
7 ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB entry
8 nop ! Push branch to next I$ line
9 cmp %g4, %g6 ! Compare TAG
10
11/* DTLB ** ICACHE line 2: TSB compare and TLB load */
12 bne,pn %xcc, tsb_miss_dtlb ! Miss
13 mov FAULT_CODE_DTLB, %g3
14 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Load TLB
15 retry ! Trap done
16 nop
17 nop
18 nop
19 nop
20
21/* DTLB ** ICACHE line 3: */
22 nop
23 nop
24 nop
25 nop
26 nop
27 nop
28 nop
29 nop
30
31/* DTLB ** ICACHE line 4: */
32 nop
33 nop
34 nop
35 nop
36 nop
37 nop
38 nop
39 nop
diff --git a/arch/sparc64/kernel/etrap.S b/arch/sparc64/kernel/etrap.S
index 0d8eba21111b..567dbb765c34 100644
--- a/arch/sparc64/kernel/etrap.S
+++ b/arch/sparc64/kernel/etrap.S
@@ -99,6 +99,7 @@ etrap_irq:
99 wrpr %g0, ETRAP_PSTATE2, %pstate 99 wrpr %g0, ETRAP_PSTATE2, %pstate
100 mov %l6, %g6 100 mov %l6, %g6
101#ifdef CONFIG_SMP 101#ifdef CONFIG_SMP
102#error IMMU TSB usage must be fixed
102 mov TSB_REG, %g3 103 mov TSB_REG, %g3
103 ldxa [%g3] ASI_IMMU, %g5 104 ldxa [%g3] ASI_IMMU, %g5
104#endif 105#endif
@@ -248,6 +249,7 @@ scetrap: rdpr %pil, %g2
248 mov %l6, %g6 249 mov %l6, %g6
249 stx %i7, [%sp + PTREGS_OFF + PT_V9_I7] 250 stx %i7, [%sp + PTREGS_OFF + PT_V9_I7]
250#ifdef CONFIG_SMP 251#ifdef CONFIG_SMP
252#error IMMU TSB usage must be fixed
251 mov TSB_REG, %g3 253 mov TSB_REG, %g3
252 ldxa [%g3] ASI_IMMU, %g5 254 ldxa [%g3] ASI_IMMU, %g5
253#endif 255#endif
diff --git a/arch/sparc64/kernel/head.S b/arch/sparc64/kernel/head.S
index b49dcd4504b0..d00e20693be1 100644
--- a/arch/sparc64/kernel/head.S
+++ b/arch/sparc64/kernel/head.S
@@ -429,17 +429,6 @@ setup_trap_table:
429 * 429 *
430 * %g6 --> current_thread_info() 430 * %g6 --> current_thread_info()
431 * 431 *
432 * MMU Globals (PSTATE_MG):
433 *
434 * %g1 --> TLB_SFSR
435 * %g2 --> ((_PAGE_VALID | _PAGE_SZ4MB |
436 * _PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
437 * ^ 0xfffff80000000000)
438 * (this %g2 value is used for computing the PAGE_OFFSET kernel
439 * TLB entries quickly, the virtual address of the fault XOR'd
440 * with this %g2 value is the PTE to load into the TLB)
441 * %g3 --> VPTE_BASE_CHEETAH or VPTE_BASE_SPITFIRE
442 *
443 * Interrupt Globals (PSTATE_IG, setup by init_irqwork_curcpu()): 432 * Interrupt Globals (PSTATE_IG, setup by init_irqwork_curcpu()):
444 * 433 *
445 * %g6 --> __irq_work[smp_processor_id()] 434 * %g6 --> __irq_work[smp_processor_id()]
@@ -450,40 +439,6 @@ setup_trap_table:
450 wrpr %o1, PSTATE_AG, %pstate 439 wrpr %o1, PSTATE_AG, %pstate
451 mov %o2, %g6 440 mov %o2, %g6
452 441
453#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
454#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
455 wrpr %o1, PSTATE_MG, %pstate
456 mov TSB_REG, %g1
457 stxa %g0, [%g1] ASI_DMMU
458 membar #Sync
459 stxa %g0, [%g1] ASI_IMMU
460 membar #Sync
461 mov TLB_SFSR, %g1
462 sethi %uhi(KERN_HIGHBITS), %g2
463 or %g2, %ulo(KERN_HIGHBITS), %g2
464 sllx %g2, 32, %g2
465 or %g2, KERN_LOWBITS, %g2
466
467 BRANCH_IF_ANY_CHEETAH(g3,g7,8f)
468 ba,pt %xcc, 9f
469 nop
470
4718:
472 sethi %uhi(VPTE_BASE_CHEETAH), %g3
473 or %g3, %ulo(VPTE_BASE_CHEETAH), %g3
474 ba,pt %xcc, 2f
475 sllx %g3, 32, %g3
476
4779:
478 sethi %uhi(VPTE_BASE_SPITFIRE), %g3
479 or %g3, %ulo(VPTE_BASE_SPITFIRE), %g3
480 sllx %g3, 32, %g3
481
4822:
483 clr %g7
484#undef KERN_HIGHBITS
485#undef KERN_LOWBITS
486
487 /* Kill PROM timer */ 442 /* Kill PROM timer */
488 sethi %hi(0x80000000), %o2 443 sethi %hi(0x80000000), %o2
489 sllx %o2, 32, %o2 444 sllx %o2, 32, %o2
@@ -538,6 +493,7 @@ sparc64_boot_end:
538 493
539#include "systbls.S" 494#include "systbls.S"
540#include "ktlb.S" 495#include "ktlb.S"
496#include "tsb.S"
541#include "etrap.S" 497#include "etrap.S"
542#include "rtrap.S" 498#include "rtrap.S"
543#include "winfixup.S" 499#include "winfixup.S"
diff --git a/arch/sparc64/kernel/itlb_base.S b/arch/sparc64/kernel/itlb_base.S
deleted file mode 100644
index 4951ff8f6877..000000000000
--- a/arch/sparc64/kernel/itlb_base.S
+++ /dev/null
@@ -1,79 +0,0 @@
1/* $Id: itlb_base.S,v 1.12 2002/02/09 19:49:30 davem Exp $
2 * itlb_base.S: Front end to ITLB miss replacement strategy.
3 * This is included directly into the trap table.
4 *
5 * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com)
6 * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz)
7 */
8
9#if PAGE_SHIFT == 13
10/*
11 * To compute vpte offset, we need to do ((addr >> 13) << 3),
12 * which can be optimized to (addr >> 10) if bits 10/11/12 can
13 * be guaranteed to be 0 ... mmu_context.h does guarantee this
14 * by only using 10 bits in the hwcontext value.
15 */
16#define CREATE_VPTE_OFFSET1(r1, r2) \
17 srax r1, 10, r2
18#define CREATE_VPTE_OFFSET2(r1, r2) nop
19#else /* PAGE_SHIFT */
20#define CREATE_VPTE_OFFSET1(r1, r2) \
21 srax r1, PAGE_SHIFT, r2
22#define CREATE_VPTE_OFFSET2(r1, r2) \
23 sllx r2, 3, r2
24#endif /* PAGE_SHIFT */
25
26
27/* Ways we can get here:
28 *
29 * 1) Nucleus instruction misses from module code.
30 * 2) All user instruction misses.
31 *
32 * All real page faults merge their code paths to the
33 * sparc64_realfault_common label below.
34 */
35
36/* ITLB ** ICACHE line 1: Quick user TLB misses */
37 mov TLB_SFSR, %g1
38 ldxa [%g1 + %g1] ASI_IMMU, %g4 ! Get TAG_ACCESS
39 CREATE_VPTE_OFFSET1(%g4, %g6) ! Create VPTE offset
40 CREATE_VPTE_OFFSET2(%g4, %g6) ! Create VPTE offset
41 ldxa [%g3 + %g6] ASI_P, %g5 ! Load VPTE
421: brgez,pn %g5, 3f ! Not valid, branch out
43 sethi %hi(_PAGE_EXEC), %g4 ! Delay-slot
44 andcc %g5, %g4, %g0 ! Executable?
45
46/* ITLB ** ICACHE line 2: Real faults */
47 be,pn %xcc, 3f ! Nope, branch.
48 nop ! Delay-slot
492: stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Load PTE into TLB
50 retry ! Trap return
513: rdpr %pstate, %g4 ! Move into alt-globals
52 wrpr %g4, PSTATE_AG|PSTATE_MG, %pstate
53 rdpr %tpc, %g5 ! And load faulting VA
54 mov FAULT_CODE_ITLB, %g4 ! It was read from ITLB
55
56/* ITLB ** ICACHE line 3: Finish faults */
57sparc64_realfault_common: ! Called by dtlb_miss
58 stb %g4, [%g6 + TI_FAULT_CODE]
59 stx %g5, [%g6 + TI_FAULT_ADDR]
60 ba,pt %xcc, etrap ! Save state
611: rd %pc, %g7 ! ...
62 call do_sparc64_fault ! Call fault handler
63 add %sp, PTREGS_OFF, %o0! Compute pt_regs arg
64 ba,pt %xcc, rtrap_clr_l6 ! Restore cpu state
65 nop
66
67/* ITLB ** ICACHE line 4: Window fixups */
68winfix_trampoline:
69 rdpr %tpc, %g3 ! Prepare winfixup TNPC
70 or %g3, 0x7c, %g3 ! Compute branch offset
71 wrpr %g3, %tnpc ! Write it into TNPC
72 done ! Do it to it
73 nop
74 nop
75 nop
76 nop
77
78#undef CREATE_VPTE_OFFSET1
79#undef CREATE_VPTE_OFFSET2
diff --git a/arch/sparc64/kernel/itlb_miss.S b/arch/sparc64/kernel/itlb_miss.S
new file mode 100644
index 000000000000..6b6c8fee04bd
--- /dev/null
+++ b/arch/sparc64/kernel/itlb_miss.S
@@ -0,0 +1,39 @@
1/* ITLB ** ICACHE line 1: Context 0 check and TSB load */
2 ldxa [%g0] ASI_IMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer
3 ldxa [%g0] ASI_IMMU, %g6 ! Get TAG TARGET
4 srlx %g6, 48, %g5 ! Get context
5 brz,pn %g5, kvmap_itlb ! Context 0 processing
6 nop ! Delay slot (fill me)
7 ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB entry
8 cmp %g4, %g6 ! Compare TAG
9 sethi %hi(_PAGE_EXEC), %g4 ! Setup exec check
10
11/* ITLB ** ICACHE line 2: TSB compare and TLB load */
12 bne,pn %xcc, tsb_miss_itlb ! Miss
13 mov FAULT_CODE_ITLB, %g3
14 andcc %g5, %g4, %g0 ! Executable?
15 be,pn %xcc, tsb_do_fault
16 nop ! Delay slot, fill me
17 stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Load TLB
18 retry ! Trap done
19 nop
20
21/* ITLB ** ICACHE line 3: */
22 nop
23 nop
24 nop
25 nop
26 nop
27 nop
28 nop
29 nop
30
31/* ITLB ** ICACHE line 4: */
32 nop
33 nop
34 nop
35 nop
36 nop
37 nop
38 nop
39 nop
diff --git a/arch/sparc64/kernel/ktlb.S b/arch/sparc64/kernel/ktlb.S
index d9244d3c9f73..2b5e71b68882 100644
--- a/arch/sparc64/kernel/ktlb.S
+++ b/arch/sparc64/kernel/ktlb.S
@@ -4,191 +4,170 @@
4 * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de) 4 * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de)
5 * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) 5 * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx)
6 * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 6 * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
7*/ 7 */
8 8
9#include <linux/config.h> 9#include <linux/config.h>
10#include <asm/head.h> 10#include <asm/head.h>
11#include <asm/asi.h> 11#include <asm/asi.h>
12#include <asm/page.h> 12#include <asm/page.h>
13#include <asm/pgtable.h> 13#include <asm/pgtable.h>
14#include <asm/tsb.h>
14 15
15 .text 16 .text
16 .align 32 17 .align 32
17 18
18/* 19 .globl kvmap_itlb
19 * On a second level vpte miss, check whether the original fault is to the OBP 20kvmap_itlb:
20 * range (note that this is only possible for instruction miss, data misses to 21 /* g6: TAG TARGET */
21 * obp range do not use vpte). If so, go back directly to the faulting address. 22 mov TLB_TAG_ACCESS, %g4
22 * This is because we want to read the tpc, otherwise we have no way of knowing 23 ldxa [%g4] ASI_IMMU, %g4
23 * the 8k aligned faulting address if we are using >8k kernel pagesize. This 24
24 * also ensures no vpte range addresses are dropped into tlb while obp is 25kvmap_itlb_nonlinear:
25 * executing (see inherit_locked_prom_mappings() rant). 26 /* Catch kernel NULL pointer calls. */
26 */ 27 sethi %hi(PAGE_SIZE), %g5
27sparc64_vpte_nucleus: 28 cmp %g4, %g5
28 /* Note that kvmap below has verified that the address is 29 bleu,pn %xcc, kvmap_dtlb_longpath
29 * in the range MODULES_VADDR --> VMALLOC_END already. So 30 nop
30 * here we need only check if it is an OBP address or not. 31
31 */ 32 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
33
34kvmap_itlb_tsb_miss:
32 sethi %hi(LOW_OBP_ADDRESS), %g5 35 sethi %hi(LOW_OBP_ADDRESS), %g5
33 cmp %g4, %g5 36 cmp %g4, %g5
34 blu,pn %xcc, kern_vpte 37 blu,pn %xcc, kvmap_itlb_vmalloc_addr
35 mov 0x1, %g5 38 mov 0x1, %g5
36 sllx %g5, 32, %g5 39 sllx %g5, 32, %g5
37 cmp %g4, %g5 40 cmp %g4, %g5
38 blu,pn %xcc, vpte_insn_obp 41 blu,pn %xcc, kvmap_itlb_obp
39 nop 42 nop
40 43
41 /* These two instructions are patched by paginig_init(). */ 44kvmap_itlb_vmalloc_addr:
42kern_vpte: 45 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
43 sethi %hi(swapper_pgd_zero), %g5 46
44 lduw [%g5 + %lo(swapper_pgd_zero)], %g5 47 TSB_LOCK_TAG(%g1, %g2, %g4)
45 48
46 /* With kernel PGD in %g5, branch back into dtlb_backend. */ 49 /* Load and check PTE. */
47 ba,pt %xcc, sparc64_kpte_continue 50 ldxa [%g5] ASI_PHYS_USE_EC, %g5
48 andn %g1, 0x3, %g1 /* Finish PMD offset adjustment. */ 51 brgez,a,pn %g5, kvmap_itlb_longpath
49 52 stx %g0, [%g1]
50vpte_noent:
51 /* Restore previous TAG_ACCESS, %g5 is zero, and we will
52 * skip over the trap instruction so that the top level
53 * TLB miss handler will thing this %g5 value is just an
54 * invalid PTE, thus branching to full fault processing.
55 */
56 mov TLB_SFSR, %g1
57 stxa %g4, [%g1 + %g1] ASI_DMMU
58 done
59
60vpte_insn_obp:
61 /* Behave as if we are at TL0. */
62 wrpr %g0, 1, %tl
63 rdpr %tpc, %g4 /* Find original faulting iaddr */
64 srlx %g4, 13, %g4 /* Throw out context bits */
65 sllx %g4, 13, %g4 /* g4 has vpn + ctx0 now */
66
67 /* Restore previous TAG_ACCESS. */
68 mov TLB_SFSR, %g1
69 stxa %g4, [%g1 + %g1] ASI_IMMU
70
71 sethi %hi(prom_trans), %g5
72 or %g5, %lo(prom_trans), %g5
73
741: ldx [%g5 + 0x00], %g6 ! base
75 brz,a,pn %g6, longpath ! no more entries, fail
76 mov TLB_SFSR, %g1 ! and restore %g1
77 ldx [%g5 + 0x08], %g1 ! len
78 add %g6, %g1, %g1 ! end
79 cmp %g6, %g4
80 bgu,pt %xcc, 2f
81 cmp %g4, %g1
82 bgeu,pt %xcc, 2f
83 ldx [%g5 + 0x10], %g1 ! PTE
84
85 /* TLB load, restore %g1, and return from trap. */
86 sub %g4, %g6, %g6
87 add %g1, %g6, %g5
88 mov TLB_SFSR, %g1
89 stxa %g5, [%g0] ASI_ITLB_DATA_IN
90 retry
91 53
922: ba,pt %xcc, 1b 54 TSB_WRITE(%g1, %g5, %g6)
93 add %g5, (3 * 8), %g5 ! next entry 55
94 56 /* fallthrough to TLB load */
95kvmap_do_obp: 57
96 sethi %hi(prom_trans), %g5 58kvmap_itlb_load:
97 or %g5, %lo(prom_trans), %g5 59 stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Reload TLB
98 srlx %g4, 13, %g4
99 sllx %g4, 13, %g4
100
1011: ldx [%g5 + 0x00], %g6 ! base
102 brz,a,pn %g6, longpath ! no more entries, fail
103 mov TLB_SFSR, %g1 ! and restore %g1
104 ldx [%g5 + 0x08], %g1 ! len
105 add %g6, %g1, %g1 ! end
106 cmp %g6, %g4
107 bgu,pt %xcc, 2f
108 cmp %g4, %g1
109 bgeu,pt %xcc, 2f
110 ldx [%g5 + 0x10], %g1 ! PTE
111
112 /* TLB load, restore %g1, and return from trap. */
113 sub %g4, %g6, %g6
114 add %g1, %g6, %g5
115 mov TLB_SFSR, %g1
116 stxa %g5, [%g0] ASI_DTLB_DATA_IN
117 retry 60 retry
118 61
1192: ba,pt %xcc, 1b 62kvmap_itlb_longpath:
120 add %g5, (3 * 8), %g5 ! next entry 63 rdpr %pstate, %g5
64 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
65 rdpr %tpc, %g5
66 ba,pt %xcc, sparc64_realfault_common
67 mov FAULT_CODE_ITLB, %g4
68
69kvmap_itlb_obp:
70 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
71
72 TSB_LOCK_TAG(%g1, %g2, %g4)
73
74 TSB_WRITE(%g1, %g5, %g6)
75
76 ba,pt %xcc, kvmap_itlb_load
77 nop
78
79kvmap_dtlb_obp:
80 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
81
82 TSB_LOCK_TAG(%g1, %g2, %g4)
83
84 TSB_WRITE(%g1, %g5, %g6)
85
86 ba,pt %xcc, kvmap_dtlb_load
87 nop
121 88
122/*
123 * On a first level data miss, check whether this is to the OBP range (note
124 * that such accesses can be made by prom, as well as by kernel using
125 * prom_getproperty on "address"), and if so, do not use vpte access ...
126 * rather, use information saved during inherit_prom_mappings() using 8k
127 * pagesize.
128 */
129 .align 32 89 .align 32
130kvmap: 90 .globl kvmap_dtlb
131 brgez,pn %g4, kvmap_nonlinear 91kvmap_dtlb:
92 /* %g6: TAG TARGET */
93 mov TLB_TAG_ACCESS, %g4
94 ldxa [%g4] ASI_DMMU, %g4
95 brgez,pn %g4, kvmap_dtlb_nonlinear
132 nop 96 nop
133 97
134#ifdef CONFIG_DEBUG_PAGEALLOC 98#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
99#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
100
101 sethi %uhi(KERN_HIGHBITS), %g2
102 or %g2, %ulo(KERN_HIGHBITS), %g2
103 sllx %g2, 32, %g2
104 or %g2, KERN_LOWBITS, %g2
105
106#undef KERN_HIGHBITS
107#undef KERN_LOWBITS
108
135 .globl kvmap_linear_patch 109 .globl kvmap_linear_patch
136kvmap_linear_patch: 110kvmap_linear_patch:
137#endif 111 ba,pt %xcc, kvmap_dtlb_load
138 ba,pt %xcc, kvmap_load
139 xor %g2, %g4, %g5 112 xor %g2, %g4, %g5
140 113
141#ifdef CONFIG_DEBUG_PAGEALLOC 114kvmap_dtlb_vmalloc_addr:
142 sethi %hi(swapper_pg_dir), %g5 115 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
143 or %g5, %lo(swapper_pg_dir), %g5 116
144 sllx %g4, 64 - (PGDIR_SHIFT + PGDIR_BITS), %g6 117 TSB_LOCK_TAG(%g1, %g2, %g4)
145 srlx %g6, 64 - PAGE_SHIFT, %g6 118
146 andn %g6, 0x3, %g6 119 /* Load and check PTE. */
147 lduw [%g5 + %g6], %g5 120 ldxa [%g5] ASI_PHYS_USE_EC, %g5
148 brz,pn %g5, longpath 121 brgez,a,pn %g5, kvmap_dtlb_longpath
149 sllx %g4, 64 - (PMD_SHIFT + PMD_BITS), %g6 122 stx %g0, [%g1]
150 srlx %g6, 64 - PAGE_SHIFT, %g6 123
151 sllx %g5, 11, %g5 124 TSB_WRITE(%g1, %g5, %g6)
152 andn %g6, 0x3, %g6 125
153 lduwa [%g5 + %g6] ASI_PHYS_USE_EC, %g5 126 /* fallthrough to TLB load */
154 brz,pn %g5, longpath 127
155 sllx %g4, 64 - PMD_SHIFT, %g6 128kvmap_dtlb_load:
156 srlx %g6, 64 - PAGE_SHIFT, %g6 129 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
157 sllx %g5, 11, %g5 130 retry
158 andn %g6, 0x7, %g6 131
159 ldxa [%g5 + %g6] ASI_PHYS_USE_EC, %g5 132kvmap_dtlb_nonlinear:
160 brz,pn %g5, longpath 133 /* Catch kernel NULL pointer derefs. */
134 sethi %hi(PAGE_SIZE), %g5
135 cmp %g4, %g5
136 bleu,pn %xcc, kvmap_dtlb_longpath
161 nop 137 nop
162 ba,a,pt %xcc, kvmap_load
163#endif
164 138
165kvmap_nonlinear: 139 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
140
141kvmap_dtlb_tsbmiss:
166 sethi %hi(MODULES_VADDR), %g5 142 sethi %hi(MODULES_VADDR), %g5
167 cmp %g4, %g5 143 cmp %g4, %g5
168 blu,pn %xcc, longpath 144 blu,pn %xcc, kvmap_dtlb_longpath
169 mov (VMALLOC_END >> 24), %g5 145 mov (VMALLOC_END >> 24), %g5
170 sllx %g5, 24, %g5 146 sllx %g5, 24, %g5
171 cmp %g4, %g5 147 cmp %g4, %g5
172 bgeu,pn %xcc, longpath 148 bgeu,pn %xcc, kvmap_dtlb_longpath
173 nop 149 nop
174 150
175kvmap_check_obp: 151kvmap_check_obp:
176 sethi %hi(LOW_OBP_ADDRESS), %g5 152 sethi %hi(LOW_OBP_ADDRESS), %g5
177 cmp %g4, %g5 153 cmp %g4, %g5
178 blu,pn %xcc, kvmap_vmalloc_addr 154 blu,pn %xcc, kvmap_dtlb_vmalloc_addr
179 mov 0x1, %g5 155 mov 0x1, %g5
180 sllx %g5, 32, %g5 156 sllx %g5, 32, %g5
181 cmp %g4, %g5 157 cmp %g4, %g5
182 blu,pn %xcc, kvmap_do_obp 158 blu,pn %xcc, kvmap_dtlb_obp
183 nop 159 nop
184 160 ba,pt %xcc, kvmap_dtlb_vmalloc_addr
185kvmap_vmalloc_addr:
186 /* If we get here, a vmalloc addr was accessed, load kernel VPTE. */
187 ldxa [%g3 + %g6] ASI_N, %g5
188 brgez,pn %g5, longpath
189 nop 161 nop
190 162
191kvmap_load: 163kvmap_dtlb_longpath:
192 /* PTE is valid, load into TLB and return from trap. */ 164 rdpr %pstate, %g5
193 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB 165 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
194 retry 166 rdpr %tl, %g4
167 cmp %g4, 1
168 mov TLB_TAG_ACCESS, %g4
169 ldxa [%g4] ASI_DMMU, %g5
170 be,pt %xcc, sparc64_realfault_common
171 mov FAULT_CODE_DTLB, %g4
172 ba,pt %xcc, winfix_trampoline
173 nop
diff --git a/arch/sparc64/kernel/process.c b/arch/sparc64/kernel/process.c
index 059b0d025224..2784aab0d3e5 100644
--- a/arch/sparc64/kernel/process.c
+++ b/arch/sparc64/kernel/process.c
@@ -44,6 +44,7 @@
44#include <asm/fpumacro.h> 44#include <asm/fpumacro.h>
45#include <asm/head.h> 45#include <asm/head.h>
46#include <asm/cpudata.h> 46#include <asm/cpudata.h>
47#include <asm/mmu_context.h>
47#include <asm/unistd.h> 48#include <asm/unistd.h>
48 49
49/* #define VERBOSE_SHOWREGS */ 50/* #define VERBOSE_SHOWREGS */
@@ -433,30 +434,16 @@ void exit_thread(void)
433void flush_thread(void) 434void flush_thread(void)
434{ 435{
435 struct thread_info *t = current_thread_info(); 436 struct thread_info *t = current_thread_info();
437 struct mm_struct *mm;
436 438
437 if (t->flags & _TIF_ABI_PENDING) 439 if (t->flags & _TIF_ABI_PENDING)
438 t->flags ^= (_TIF_ABI_PENDING | _TIF_32BIT); 440 t->flags ^= (_TIF_ABI_PENDING | _TIF_32BIT);
439 441
440 if (t->task->mm) { 442 mm = t->task->mm;
441 unsigned long pgd_cache = 0UL; 443 if (mm)
442 if (test_thread_flag(TIF_32BIT)) { 444 tsb_context_switch(__pa(mm->pgd),
443 struct mm_struct *mm = t->task->mm; 445 mm->context.sparc64_tsb);
444 pgd_t *pgd0 = &mm->pgd[0];
445 pud_t *pud0 = pud_offset(pgd0, 0);
446 446
447 if (pud_none(*pud0)) {
448 pmd_t *page = pmd_alloc_one(mm, 0);
449 pud_set(pud0, page);
450 }
451 pgd_cache = get_pgd_cache(pgd0);
452 }
453 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
454 "membar #Sync"
455 : /* no outputs */
456 : "r" (pgd_cache),
457 "r" (TSB_REG),
458 "i" (ASI_DMMU));
459 }
460 set_thread_wsaved(0); 447 set_thread_wsaved(0);
461 448
462 /* Turn off performance counters if on. */ 449 /* Turn off performance counters if on. */
diff --git a/arch/sparc64/kernel/rtrap.S b/arch/sparc64/kernel/rtrap.S
index b80eba0081ca..213eb4a9d8a4 100644
--- a/arch/sparc64/kernel/rtrap.S
+++ b/arch/sparc64/kernel/rtrap.S
@@ -223,10 +223,14 @@ rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1
223 ldx [%sp + PTREGS_OFF + PT_V9_G3], %g3 223 ldx [%sp + PTREGS_OFF + PT_V9_G3], %g3
224 ldx [%sp + PTREGS_OFF + PT_V9_G4], %g4 224 ldx [%sp + PTREGS_OFF + PT_V9_G4], %g4
225 ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5 225 ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5
226#ifdef CONFIG_SMP
227#error IMMU TSB usage must be fixed
226 mov TSB_REG, %g6 228 mov TSB_REG, %g6
227 brnz,a,pn %l3, 1f 229 brnz,a,pn %l3, 1f
228 ldxa [%g6] ASI_IMMU, %g5 230 ldxa [%g6] ASI_IMMU, %g5
2291: ldx [%sp + PTREGS_OFF + PT_V9_G6], %g6 231#endif
2321:
233 ldx [%sp + PTREGS_OFF + PT_V9_G6], %g6
230 ldx [%sp + PTREGS_OFF + PT_V9_G7], %g7 234 ldx [%sp + PTREGS_OFF + PT_V9_G7], %g7
231 wrpr %g0, RTRAP_PSTATE_AG_IRQOFF, %pstate 235 wrpr %g0, RTRAP_PSTATE_AG_IRQOFF, %pstate
232 ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0 236 ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0
diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c
index 1f7ad8a69052..d2d3369e7b5d 100644
--- a/arch/sparc64/kernel/smp.c
+++ b/arch/sparc64/kernel/smp.c
@@ -123,6 +123,7 @@ extern void inherit_locked_prom_mappings(int save_p);
123 123
124static inline void cpu_setup_percpu_base(unsigned long cpu_id) 124static inline void cpu_setup_percpu_base(unsigned long cpu_id)
125{ 125{
126#error IMMU TSB usage must be fixed
126 __asm__ __volatile__("mov %0, %%g5\n\t" 127 __asm__ __volatile__("mov %0, %%g5\n\t"
127 "stxa %0, [%1] %2\n\t" 128 "stxa %0, [%1] %2\n\t"
128 "membar #Sync" 129 "membar #Sync"
@@ -662,8 +663,6 @@ void smp_call_function_client(int irq, struct pt_regs *regs)
662extern unsigned long xcall_flush_tlb_mm; 663extern unsigned long xcall_flush_tlb_mm;
663extern unsigned long xcall_flush_tlb_pending; 664extern unsigned long xcall_flush_tlb_pending;
664extern unsigned long xcall_flush_tlb_kernel_range; 665extern unsigned long xcall_flush_tlb_kernel_range;
665extern unsigned long xcall_flush_tlb_all_spitfire;
666extern unsigned long xcall_flush_tlb_all_cheetah;
667extern unsigned long xcall_report_regs; 666extern unsigned long xcall_report_regs;
668extern unsigned long xcall_receive_signal; 667extern unsigned long xcall_receive_signal;
669 668
@@ -794,15 +793,6 @@ void smp_report_regs(void)
794 smp_cross_call(&xcall_report_regs, 0, 0, 0); 793 smp_cross_call(&xcall_report_regs, 0, 0, 0);
795} 794}
796 795
797void smp_flush_tlb_all(void)
798{
799 if (tlb_type == spitfire)
800 smp_cross_call(&xcall_flush_tlb_all_spitfire, 0, 0, 0);
801 else
802 smp_cross_call(&xcall_flush_tlb_all_cheetah, 0, 0, 0);
803 __flush_tlb_all();
804}
805
806/* We know that the window frames of the user have been flushed 796/* We know that the window frames of the user have been flushed
807 * to the stack before we get here because all callers of us 797 * to the stack before we get here because all callers of us
808 * are flush_tlb_*() routines, and these run after flush_cache_*() 798 * are flush_tlb_*() routines, and these run after flush_cache_*()
diff --git a/arch/sparc64/kernel/trampoline.S b/arch/sparc64/kernel/trampoline.S
index 9478551cb020..782d8c4973e4 100644
--- a/arch/sparc64/kernel/trampoline.S
+++ b/arch/sparc64/kernel/trampoline.S
@@ -295,39 +295,6 @@ do_unlock:
295 wrpr %g5, %tba 295 wrpr %g5, %tba
296 mov %o2, %g6 296 mov %o2, %g6
297 297
298 wrpr %o1, PSTATE_MG, %pstate
299#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
300#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
301
302 mov TSB_REG, %g1
303 stxa %g0, [%g1] ASI_DMMU
304 membar #Sync
305 mov TLB_SFSR, %g1
306 sethi %uhi(KERN_HIGHBITS), %g2
307 or %g2, %ulo(KERN_HIGHBITS), %g2
308 sllx %g2, 32, %g2
309 or %g2, KERN_LOWBITS, %g2
310
311 BRANCH_IF_ANY_CHEETAH(g3,g7,9f)
312
313 ba,pt %xcc, 1f
314 nop
315
3169:
317 sethi %uhi(VPTE_BASE_CHEETAH), %g3
318 or %g3, %ulo(VPTE_BASE_CHEETAH), %g3
319 ba,pt %xcc, 2f
320 sllx %g3, 32, %g3
3211:
322 sethi %uhi(VPTE_BASE_SPITFIRE), %g3
323 or %g3, %ulo(VPTE_BASE_SPITFIRE), %g3
324 sllx %g3, 32, %g3
325
3262:
327 clr %g7
328#undef KERN_HIGHBITS
329#undef KERN_LOWBITS
330
331 wrpr %o1, 0x0, %pstate 298 wrpr %o1, 0x0, %pstate
332 ldx [%g6 + TI_TASK], %g4 299 ldx [%g6 + TI_TASK], %g4
333 300
diff --git a/arch/sparc64/kernel/tsb.S b/arch/sparc64/kernel/tsb.S
new file mode 100644
index 000000000000..44b9e6fed09f
--- /dev/null
+++ b/arch/sparc64/kernel/tsb.S
@@ -0,0 +1,169 @@
1/* tsb.S: Sparc64 TSB table handling.
2 *
3 * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
4 */
5
6#include <asm/tsb.h>
7
8 .text
9 .align 32
10
11 /* Invoked from TLB miss handler, we are in the
12 * MMU global registers and they are setup like
13 * this:
14 *
15 * %g1: TSB entry pointer
16 * %g2: available temporary
17 * %g3: FAULT_CODE_{D,I}TLB
18 * %g4: available temporary
19 * %g5: available temporary
20 * %g6: TAG TARGET
21 * %g7: physical address base of the linux page
22 * tables for the current address space
23 */
24 .globl tsb_miss_dtlb
25tsb_miss_dtlb:
26 mov TLB_TAG_ACCESS, %g4
27 ldxa [%g4] ASI_DMMU, %g4
28 ba,pt %xcc, tsb_miss_page_table_walk
29 nop
30
31 .globl tsb_miss_itlb
32tsb_miss_itlb:
33 mov TLB_TAG_ACCESS, %g4
34 ldxa [%g4] ASI_IMMU, %g4
35 ba,pt %xcc, tsb_miss_page_table_walk
36 nop
37
38tsb_miss_page_table_walk:
39 USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault)
40
41tsb_reload:
42 TSB_LOCK_TAG(%g1, %g2, %g4)
43
44 /* Load and check PTE. */
45 ldxa [%g5] ASI_PHYS_USE_EC, %g5
46 brgez,a,pn %g5, tsb_do_fault
47 stx %g0, [%g1]
48
49 TSB_WRITE(%g1, %g5, %g6)
50
51 /* Finally, load TLB and return from trap. */
52tsb_tlb_reload:
53 cmp %g3, FAULT_CODE_DTLB
54 bne,pn %xcc, tsb_itlb_load
55 nop
56
57tsb_dtlb_load:
58 stxa %g5, [%g0] ASI_DTLB_DATA_IN
59 retry
60
61tsb_itlb_load:
62 stxa %g5, [%g0] ASI_ITLB_DATA_IN
63 retry
64
65 /* No valid entry in the page tables, do full fault
66 * processing.
67 */
68
69 .globl tsb_do_fault
70tsb_do_fault:
71 cmp %g3, FAULT_CODE_DTLB
72 rdpr %pstate, %g5
73 bne,pn %xcc, tsb_do_itlb_fault
74 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
75
76tsb_do_dtlb_fault:
77 rdpr %tl, %g4
78 cmp %g4, 1
79 mov TLB_TAG_ACCESS, %g4
80 ldxa [%g4] ASI_DMMU, %g5
81 be,pt %xcc, sparc64_realfault_common
82 mov FAULT_CODE_DTLB, %g4
83 ba,pt %xcc, winfix_trampoline
84 nop
85
86tsb_do_itlb_fault:
87 rdpr %tpc, %g5
88 ba,pt %xcc, sparc64_realfault_common
89 mov FAULT_CODE_ITLB, %g4
90
91 .globl sparc64_realfault_common
92sparc64_realfault_common:
93 stb %g4, [%g6 + TI_FAULT_CODE] ! Save fault code
94 stx %g5, [%g6 + TI_FAULT_ADDR] ! Save fault address
95 ba,pt %xcc, etrap ! Save trap state
961: rd %pc, %g7 ! ...
97 call do_sparc64_fault ! Call fault handler
98 add %sp, PTREGS_OFF, %o0 ! Compute pt_regs arg
99 ba,pt %xcc, rtrap_clr_l6 ! Restore cpu state
100 nop ! Delay slot (fill me)
101
102 .globl winfix_trampoline
103winfix_trampoline:
104 rdpr %tpc, %g3 ! Prepare winfixup TNPC
105 or %g3, 0x7c, %g3 ! Compute branch offset
106 wrpr %g3, %tnpc ! Write it into TNPC
107 done ! Trap return
108
109 /* Reload MMU related context switch state at
110 * schedule() time.
111 *
112 * %o0: page table physical address
113 * %o1: TSB address
114 */
115 .globl tsb_context_switch
116tsb_context_switch:
117 wrpr %g0, PSTATE_MG | PSTATE_RMO | PSTATE_PEF | PSTATE_PRIV, %pstate
118
119 /* Set page table base alternate global. */
120 mov %o0, %g7
121
122 /* XXX can this happen? */
123 brz,pn %o1, 9f
124 nop
125
126 /* Lock TSB into D-TLB. */
127 sethi %hi(PAGE_SIZE), %o3
128 and %o3, %o1, %o3
129 sethi %hi(TSBMAP_BASE), %o2
130 add %o2, %o3, %o2
131
132 /* XXX handle PAGE_SIZE != 8K correctly... */
133 mov TSB_REG, %g1
134 stxa %o2, [%g1] ASI_DMMU
135 membar #Sync
136
137 stxa %o2, [%g1] ASI_IMMU
138 membar #Sync
139
140#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZBITS)^0xfffff80000000000)
141#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W | _PAGE_L)
142 sethi %uhi(KERN_HIGHBITS), %g2
143 or %g2, %ulo(KERN_HIGHBITS), %g2
144 sllx %g2, 32, %g2
145 or %g2, KERN_LOWBITS, %g2
146#undef KERN_HIGHBITS
147#undef KERN_LOWBITS
148
149 xor %o1, %g2, %o1
150
151 /* We use entry 61 for this locked entry. This is the spitfire
152 * TLB entry number, and luckily cheetah masks the value with
153 * 15 ending us up with entry 13 which is what we want in that
154 * case too.
155 *
156 * XXX Interactions with prom_world()...
157 */
158 mov TLB_TAG_ACCESS, %g1
159 stxa %o2, [%g1] ASI_DMMU
160 membar #Sync
161 mov (61 << 3), %g1
162 stxa %o1, [%g1] ASI_DTLB_DATA_ACCESS
163 membar #Sync
164
1659:
166 wrpr %g0, PSTATE_RMO | PSTATE_PEF | PSTATE_PRIV | PSTATE_IE, %pstate
167
168 retl
169 mov %o2, %o0
diff --git a/arch/sparc64/kernel/ttable.S b/arch/sparc64/kernel/ttable.S
index 8365bc1f81f3..56f060c8fbf0 100644
--- a/arch/sparc64/kernel/ttable.S
+++ b/arch/sparc64/kernel/ttable.S
@@ -78,9 +78,9 @@ tl0_vaw: TRAP(do_vaw)
78tl0_cee: membar #Sync 78tl0_cee: membar #Sync
79 TRAP_NOSAVE_7INSNS(__spitfire_cee_trap) 79 TRAP_NOSAVE_7INSNS(__spitfire_cee_trap)
80tl0_iamiss: 80tl0_iamiss:
81#include "itlb_base.S" 81#include "itlb_miss.S"
82tl0_damiss: 82tl0_damiss:
83#include "dtlb_base.S" 83#include "dtlb_miss.S"
84tl0_daprot: 84tl0_daprot:
85#include "dtlb_prot.S" 85#include "dtlb_prot.S"
86tl0_fecc: BTRAP(0x70) /* Fast-ECC on Cheetah */ 86tl0_fecc: BTRAP(0x70) /* Fast-ECC on Cheetah */
@@ -241,7 +241,7 @@ tl1_cee: membar #Sync
241 241
242tl1_iamiss: BTRAPTL1(0x64) BTRAPTL1(0x65) BTRAPTL1(0x66) BTRAPTL1(0x67) 242tl1_iamiss: BTRAPTL1(0x64) BTRAPTL1(0x65) BTRAPTL1(0x66) BTRAPTL1(0x67)
243tl1_damiss: 243tl1_damiss:
244#include "dtlb_backend.S" 244#include "dtlb_miss.S"
245tl1_daprot: 245tl1_daprot:
246#include "dtlb_prot.S" 246#include "dtlb_prot.S"
247tl1_fecc: BTRAPTL1(0x70) /* Fast-ECC on Cheetah */ 247tl1_fecc: BTRAPTL1(0x70) /* Fast-ECC on Cheetah */
diff --git a/arch/sparc64/kernel/vmlinux.lds.S b/arch/sparc64/kernel/vmlinux.lds.S
index 467d13a0d5c1..f018aaf45486 100644
--- a/arch/sparc64/kernel/vmlinux.lds.S
+++ b/arch/sparc64/kernel/vmlinux.lds.S
@@ -44,6 +44,9 @@ SECTIONS
44 __stop___ex_table = .; 44 __stop___ex_table = .;
45 45
46 . = ALIGN(8192); 46 . = ALIGN(8192);
47 swapper_tsb = .;
48 . += 8192;
49 . = ALIGN(8192);
47 __init_begin = .; 50 __init_begin = .;
48 .init.text : { 51 .init.text : {
49 _sinittext = .; 52 _sinittext = .;
diff --git a/arch/sparc64/kernel/winfixup.S b/arch/sparc64/kernel/winfixup.S
index 39160926267b..f5d93aa99cbb 100644
--- a/arch/sparc64/kernel/winfixup.S
+++ b/arch/sparc64/kernel/winfixup.S
@@ -85,6 +85,7 @@ fill_fixup:
85 mov %o7, %g6 85 mov %o7, %g6
86 ldx [%g6 + TI_TASK], %g4 86 ldx [%g6 + TI_TASK], %g4
87#ifdef CONFIG_SMP 87#ifdef CONFIG_SMP
88#error IMMU TSB usage must be fixed
88 mov TSB_REG, %g1 89 mov TSB_REG, %g1
89 ldxa [%g1] ASI_IMMU, %g5 90 ldxa [%g1] ASI_IMMU, %g5
90#endif 91#endif
@@ -209,6 +210,7 @@ fill_fixup_mna:
209 mov %o7, %g6 ! Get current back. 210 mov %o7, %g6 ! Get current back.
210 ldx [%g6 + TI_TASK], %g4 ! Finish it. 211 ldx [%g6 + TI_TASK], %g4 ! Finish it.
211#ifdef CONFIG_SMP 212#ifdef CONFIG_SMP
213#error IMMU TSB usage must be fixed
212 mov TSB_REG, %g1 214 mov TSB_REG, %g1
213 ldxa [%g1] ASI_IMMU, %g5 215 ldxa [%g1] ASI_IMMU, %g5
214#endif 216#endif
@@ -278,11 +280,6 @@ window_mna_from_user_common:
278 ba,pt %xcc, rtrap 280 ba,pt %xcc, rtrap
279 clr %l6 281 clr %l6
280 282
281 /* These are only needed for 64-bit mode processes which
282 * put their stack pointer into the VPTE area and there
283 * happens to be a VPTE tlb entry mapped there during
284 * a spill/fill trap to that stack frame.
285 */
286 .globl winfix_dax, fill_fixup_dax, spill_fixup_dax 283 .globl winfix_dax, fill_fixup_dax, spill_fixup_dax
287winfix_dax: 284winfix_dax:
288 andn %g3, 0x7f, %g3 285 andn %g3, 0x7f, %g3
@@ -318,6 +315,7 @@ fill_fixup_dax:
318 mov %o7, %g6 ! Get current back. 315 mov %o7, %g6 ! Get current back.
319 ldx [%g6 + TI_TASK], %g4 ! Finish it. 316 ldx [%g6 + TI_TASK], %g4 ! Finish it.
320#ifdef CONFIG_SMP 317#ifdef CONFIG_SMP
318#error IMMU TSB usage must be fixed
321 mov TSB_REG, %g1 319 mov TSB_REG, %g1
322 ldxa [%g1] ASI_IMMU, %g5 320 ldxa [%g1] ASI_IMMU, %g5
323#endif 321#endif
diff --git a/arch/sparc64/mm/Makefile b/arch/sparc64/mm/Makefile
index 9d0960e69f48..e415bf942bcd 100644
--- a/arch/sparc64/mm/Makefile
+++ b/arch/sparc64/mm/Makefile
@@ -5,6 +5,6 @@
5EXTRA_AFLAGS := -ansi 5EXTRA_AFLAGS := -ansi
6EXTRA_CFLAGS := -Werror 6EXTRA_CFLAGS := -Werror
7 7
8obj-y := ultra.o tlb.o fault.o init.o generic.o 8obj-y := ultra.o tlb.o tsb.o fault.o init.o generic.o
9 9
10obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o 10obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
index 1e44ee26cee8..da068f6b2595 100644
--- a/arch/sparc64/mm/init.c
+++ b/arch/sparc64/mm/init.c
@@ -408,8 +408,7 @@ unsigned long prom_virt_to_phys(unsigned long promva, int *error)
408 408
409/* The obp translations are saved based on 8k pagesize, since obp can 409/* The obp translations are saved based on 8k pagesize, since obp can
410 * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS -> 410 * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
411 * HI_OBP_ADDRESS range are handled in ktlb.S and do not use the vpte 411 * HI_OBP_ADDRESS range are handled in ktlb.S.
412 * scheme (also, see rant in inherit_locked_prom_mappings()).
413 */ 412 */
414static inline int in_obp_range(unsigned long vaddr) 413static inline int in_obp_range(unsigned long vaddr)
415{ 414{
@@ -539,75 +538,6 @@ static void __init inherit_prom_mappings(void)
539 prom_printf("done.\n"); 538 prom_printf("done.\n");
540} 539}
541 540
542/* The OBP specifications for sun4u mark 0xfffffffc00000000 and
543 * upwards as reserved for use by the firmware (I wonder if this
544 * will be the same on Cheetah...). We use this virtual address
545 * range for the VPTE table mappings of the nucleus so we need
546 * to zap them when we enter the PROM. -DaveM
547 */
548static void __flush_nucleus_vptes(void)
549{
550 unsigned long prom_reserved_base = 0xfffffffc00000000UL;
551 int i;
552
553 /* Only DTLB must be checked for VPTE entries. */
554 if (tlb_type == spitfire) {
555 for (i = 0; i < 63; i++) {
556 unsigned long tag;
557
558 /* Spitfire Errata #32 workaround */
559 /* NOTE: Always runs on spitfire, so no cheetah+
560 * page size encodings.
561 */
562 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
563 "flush %%g6"
564 : /* No outputs */
565 : "r" (0),
566 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
567
568 tag = spitfire_get_dtlb_tag(i);
569 if (((tag & ~(PAGE_MASK)) == 0) &&
570 ((tag & (PAGE_MASK)) >= prom_reserved_base)) {
571 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
572 "membar #Sync"
573 : /* no outputs */
574 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
575 spitfire_put_dtlb_data(i, 0x0UL);
576 }
577 }
578 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
579 for (i = 0; i < 512; i++) {
580 unsigned long tag = cheetah_get_dtlb_tag(i, 2);
581
582 if ((tag & ~PAGE_MASK) == 0 &&
583 (tag & PAGE_MASK) >= prom_reserved_base) {
584 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
585 "membar #Sync"
586 : /* no outputs */
587 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
588 cheetah_put_dtlb_data(i, 0x0UL, 2);
589 }
590
591 if (tlb_type != cheetah_plus)
592 continue;
593
594 tag = cheetah_get_dtlb_tag(i, 3);
595
596 if ((tag & ~PAGE_MASK) == 0 &&
597 (tag & PAGE_MASK) >= prom_reserved_base) {
598 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
599 "membar #Sync"
600 : /* no outputs */
601 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
602 cheetah_put_dtlb_data(i, 0x0UL, 3);
603 }
604 }
605 } else {
606 /* Implement me :-) */
607 BUG();
608 }
609}
610
611static int prom_ditlb_set; 541static int prom_ditlb_set;
612struct prom_tlb_entry { 542struct prom_tlb_entry {
613 int tlb_ent; 543 int tlb_ent;
@@ -635,9 +565,6 @@ void prom_world(int enter)
635 : "i" (PSTATE_IE)); 565 : "i" (PSTATE_IE));
636 566
637 if (enter) { 567 if (enter) {
638 /* Kick out nucleus VPTEs. */
639 __flush_nucleus_vptes();
640
641 /* Install PROM world. */ 568 /* Install PROM world. */
642 for (i = 0; i < 16; i++) { 569 for (i = 0; i < 16; i++) {
643 if (prom_dtlb[i].tlb_ent != -1) { 570 if (prom_dtlb[i].tlb_ent != -1) {
@@ -1039,18 +966,7 @@ out:
1039struct pgtable_cache_struct pgt_quicklists; 966struct pgtable_cache_struct pgt_quicklists;
1040#endif 967#endif
1041 968
1042/* OK, we have to color these pages. The page tables are accessed 969/* XXX We don't need to color these things in the D-cache any longer. */
1043 * by non-Dcache enabled mapping in the VPTE area by the dtlb_backend.S
1044 * code, as well as by PAGE_OFFSET range direct-mapped addresses by
1045 * other parts of the kernel. By coloring, we make sure that the tlbmiss
1046 * fast handlers do not get data from old/garbage dcache lines that
1047 * correspond to an old/stale virtual address (user/kernel) that
1048 * previously mapped the pagetable page while accessing vpte range
1049 * addresses. The idea is that if the vpte color and PAGE_OFFSET range
1050 * color is the same, then when the kernel initializes the pagetable
1051 * using the later address range, accesses with the first address
1052 * range will see the newly initialized data rather than the garbage.
1053 */
1054#ifdef DCACHE_ALIASING_POSSIBLE 970#ifdef DCACHE_ALIASING_POSSIBLE
1055#define DC_ALIAS_SHIFT 1 971#define DC_ALIAS_SHIFT 1
1056#else 972#else
@@ -1419,6 +1335,9 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
1419 kernel_map_range(phys_start, phys_end, 1335 kernel_map_range(phys_start, phys_end,
1420 (enable ? PAGE_KERNEL : __pgprot(0))); 1336 (enable ? PAGE_KERNEL : __pgprot(0)));
1421 1337
1338 flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
1339 PAGE_OFFSET + phys_end);
1340
1422 /* we should perform an IPI and flush all tlbs, 1341 /* we should perform an IPI and flush all tlbs,
1423 * but that can deadlock->flush only current cpu. 1342 * but that can deadlock->flush only current cpu.
1424 */ 1343 */
diff --git a/arch/sparc64/mm/tlb.c b/arch/sparc64/mm/tlb.c
index 8b104be4662b..78357cc2a0b7 100644
--- a/arch/sparc64/mm/tlb.c
+++ b/arch/sparc64/mm/tlb.c
@@ -25,6 +25,8 @@ void flush_tlb_pending(void)
25 struct mmu_gather *mp = &__get_cpu_var(mmu_gathers); 25 struct mmu_gather *mp = &__get_cpu_var(mmu_gathers);
26 26
27 if (mp->tlb_nr) { 27 if (mp->tlb_nr) {
28 flush_tsb_user(mp);
29
28 if (CTX_VALID(mp->mm->context)) { 30 if (CTX_VALID(mp->mm->context)) {
29#ifdef CONFIG_SMP 31#ifdef CONFIG_SMP
30 smp_flush_tlb_pending(mp->mm, mp->tlb_nr, 32 smp_flush_tlb_pending(mp->mm, mp->tlb_nr,
@@ -89,62 +91,3 @@ no_cache_flush:
89 if (nr >= TLB_BATCH_NR) 91 if (nr >= TLB_BATCH_NR)
90 flush_tlb_pending(); 92 flush_tlb_pending();
91} 93}
92
93void flush_tlb_pgtables(struct mm_struct *mm, unsigned long start, unsigned long end)
94{
95 struct mmu_gather *mp = &__get_cpu_var(mmu_gathers);
96 unsigned long nr = mp->tlb_nr;
97 long s = start, e = end, vpte_base;
98
99 if (mp->fullmm)
100 return;
101
102 /* If start is greater than end, that is a real problem. */
103 BUG_ON(start > end);
104
105 /* However, straddling the VA space hole is quite normal. */
106 s &= PMD_MASK;
107 e = (e + PMD_SIZE - 1) & PMD_MASK;
108
109 vpte_base = (tlb_type == spitfire ?
110 VPTE_BASE_SPITFIRE :
111 VPTE_BASE_CHEETAH);
112
113 if (unlikely(nr != 0 && mm != mp->mm)) {
114 flush_tlb_pending();
115 nr = 0;
116 }
117
118 if (nr == 0)
119 mp->mm = mm;
120
121 start = vpte_base + (s >> (PAGE_SHIFT - 3));
122 end = vpte_base + (e >> (PAGE_SHIFT - 3));
123
124 /* If the request straddles the VA space hole, we
125 * need to swap start and end. The reason this
126 * occurs is that "vpte_base" is the center of
127 * the linear page table mapping area. Thus,
128 * high addresses with the sign bit set map to
129 * addresses below vpte_base and non-sign bit
130 * addresses map to addresses above vpte_base.
131 */
132 if (end < start) {
133 unsigned long tmp = start;
134
135 start = end;
136 end = tmp;
137 }
138
139 while (start < end) {
140 mp->vaddrs[nr] = start;
141 mp->tlb_nr = ++nr;
142 if (nr >= TLB_BATCH_NR) {
143 flush_tlb_pending();
144 nr = 0;
145 }
146 start += PAGE_SIZE;
147 }
148 if (nr)
149 flush_tlb_pending();
150}
diff --git a/arch/sparc64/mm/tsb.c b/arch/sparc64/mm/tsb.c
new file mode 100644
index 000000000000..15e8af58b1d2
--- /dev/null
+++ b/arch/sparc64/mm/tsb.c
@@ -0,0 +1,84 @@
1/* arch/sparc64/mm/tsb.c
2 *
3 * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
4 */
5
6#include <linux/kernel.h>
7#include <asm/system.h>
8#include <asm/page.h>
9#include <asm/tlbflush.h>
10#include <asm/tlb.h>
11
12#define TSB_ENTRY_ALIGNMENT 16
13
14struct tsb {
15 unsigned long tag;
16 unsigned long pte;
17} __attribute__((aligned(TSB_ENTRY_ALIGNMENT)));
18
19/* We use an 8K TSB for the whole kernel, this allows to
20 * handle about 4MB of modules and vmalloc mappings without
21 * incurring many hash conflicts.
22 */
23#define KERNEL_TSB_SIZE_BYTES 8192
24#define KERNEL_TSB_NENTRIES \
25 (KERNEL_TSB_SIZE_BYTES / sizeof(struct tsb))
26
27extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
28
29static inline unsigned long tsb_hash(unsigned long vaddr)
30{
31 vaddr >>= PAGE_SHIFT;
32 return vaddr & (KERNEL_TSB_NENTRIES - 1);
33}
34
35static inline int tag_compare(struct tsb *entry, unsigned long vaddr, unsigned long context)
36{
37 if (context == ~0UL)
38 return 1;
39
40 return (entry->tag == ((vaddr >> 22) | (context << 48)));
41}
42
43/* TSB flushes need only occur on the processor initiating the address
44 * space modification, not on each cpu the address space has run on.
45 * Only the TLB flush needs that treatment.
46 */
47
48void flush_tsb_kernel_range(unsigned long start, unsigned long end)
49{
50 unsigned long v;
51
52 for (v = start; v < end; v += PAGE_SIZE) {
53 struct tsb *ent = &swapper_tsb[tsb_hash(v)];
54
55 if (tag_compare(ent, v, 0)) {
56 ent->tag = 0UL;
57 membar_storeload_storestore();
58 }
59 }
60}
61
62void flush_tsb_user(struct mmu_gather *mp)
63{
64 struct mm_struct *mm = mp->mm;
65 struct tsb *tsb = (struct tsb *) mm->context.sparc64_tsb;
66 unsigned long ctx = ~0UL;
67 int i;
68
69 if (CTX_VALID(mm->context))
70 ctx = CTX_HWBITS(mm->context);
71
72 for (i = 0; i < mp->tlb_nr; i++) {
73 unsigned long v = mp->vaddrs[i];
74 struct tsb *ent;
75
76 v &= ~0x1UL;
77
78 ent = &tsb[tsb_hash(v)];
79 if (tag_compare(ent, v, ctx)) {
80 ent->tag = 0UL;
81 membar_storeload_storestore();
82 }
83 }
84}
diff --git a/arch/sparc64/mm/ultra.S b/arch/sparc64/mm/ultra.S
index e4c9151fa116..22791f29552e 100644
--- a/arch/sparc64/mm/ultra.S
+++ b/arch/sparc64/mm/ultra.S
@@ -453,64 +453,6 @@ xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
453 nop 453 nop
454 nop 454 nop
455 455
456 .data
457
458errata32_hwbug:
459 .xword 0
460
461 .text
462
463 /* These two are not performance critical... */
464 .globl xcall_flush_tlb_all_spitfire
465xcall_flush_tlb_all_spitfire:
466 /* Spitfire Errata #32 workaround. */
467 sethi %hi(errata32_hwbug), %g4
468 stx %g0, [%g4 + %lo(errata32_hwbug)]
469
470 clr %g2
471 clr %g3
4721: ldxa [%g3] ASI_DTLB_DATA_ACCESS, %g4
473 and %g4, _PAGE_L, %g5
474 brnz,pn %g5, 2f
475 mov TLB_TAG_ACCESS, %g7
476
477 stxa %g0, [%g7] ASI_DMMU
478 membar #Sync
479 stxa %g0, [%g3] ASI_DTLB_DATA_ACCESS
480 membar #Sync
481
482 /* Spitfire Errata #32 workaround. */
483 sethi %hi(errata32_hwbug), %g4
484 stx %g0, [%g4 + %lo(errata32_hwbug)]
485
4862: ldxa [%g3] ASI_ITLB_DATA_ACCESS, %g4
487 and %g4, _PAGE_L, %g5
488 brnz,pn %g5, 2f
489 mov TLB_TAG_ACCESS, %g7
490
491 stxa %g0, [%g7] ASI_IMMU
492 membar #Sync
493 stxa %g0, [%g3] ASI_ITLB_DATA_ACCESS
494 membar #Sync
495
496 /* Spitfire Errata #32 workaround. */
497 sethi %hi(errata32_hwbug), %g4
498 stx %g0, [%g4 + %lo(errata32_hwbug)]
499
5002: add %g2, 1, %g2
501 cmp %g2, SPITFIRE_HIGHEST_LOCKED_TLBENT
502 ble,pt %icc, 1b
503 sll %g2, 3, %g3
504 flush %g6
505 retry
506
507 .globl xcall_flush_tlb_all_cheetah
508xcall_flush_tlb_all_cheetah:
509 mov 0x80, %g2
510 stxa %g0, [%g2] ASI_DMMU_DEMAP
511 stxa %g0, [%g2] ASI_IMMU_DEMAP
512 retry
513
514 /* These just get rescheduled to PIL vectors. */ 456 /* These just get rescheduled to PIL vectors. */
515 .globl xcall_call_function 457 .globl xcall_call_function
516xcall_call_function: 458xcall_call_function:
diff --git a/include/asm-sparc64/mmu.h b/include/asm-sparc64/mmu.h
index 8627eed6e83d..36384cf7faa6 100644
--- a/include/asm-sparc64/mmu.h
+++ b/include/asm-sparc64/mmu.h
@@ -92,6 +92,7 @@
92 92
93typedef struct { 93typedef struct {
94 unsigned long sparc64_ctx_val; 94 unsigned long sparc64_ctx_val;
95 unsigned long *sparc64_tsb;
95} mm_context_t; 96} mm_context_t;
96 97
97#endif /* !__ASSEMBLY__ */ 98#endif /* !__ASSEMBLY__ */
diff --git a/include/asm-sparc64/mmu_context.h b/include/asm-sparc64/mmu_context.h
index 57ee7b306189..34640a370ab4 100644
--- a/include/asm-sparc64/mmu_context.h
+++ b/include/asm-sparc64/mmu_context.h
@@ -25,7 +25,13 @@ extern void get_new_mmu_context(struct mm_struct *mm);
25 * This just needs to set mm->context to an invalid context. 25 * This just needs to set mm->context to an invalid context.
26 */ 26 */
27#define init_new_context(__tsk, __mm) \ 27#define init_new_context(__tsk, __mm) \
28 (((__mm)->context.sparc64_ctx_val = 0UL), 0) 28({ unsigned long __pg = get_zeroed_page(GFP_KERNEL); \
29 (__mm)->context.sparc64_ctx_val = 0UL; \
30 (__mm)->context.sparc64_tsb = \
31 (unsigned long *) __pg; \
32 (__pg ? 0 : -ENOMEM); \
33})
34
29 35
30/* Destroy a dead context. This occurs when mmput drops the 36/* Destroy a dead context. This occurs when mmput drops the
31 * mm_users count to zero, the mmaps have been released, and 37 * mm_users count to zero, the mmaps have been released, and
@@ -35,7 +41,8 @@ extern void get_new_mmu_context(struct mm_struct *mm);
35 * this task if valid. 41 * this task if valid.
36 */ 42 */
37#define destroy_context(__mm) \ 43#define destroy_context(__mm) \
38do { spin_lock(&ctx_alloc_lock); \ 44do { free_page((unsigned long)(__mm)->context.sparc64_tsb); \
45 spin_lock(&ctx_alloc_lock); \
39 if (CTX_VALID((__mm)->context)) { \ 46 if (CTX_VALID((__mm)->context)) { \
40 unsigned long nr = CTX_NRBITS((__mm)->context); \ 47 unsigned long nr = CTX_NRBITS((__mm)->context); \
41 mmu_context_bmap[nr>>6] &= ~(1UL << (nr & 63)); \ 48 mmu_context_bmap[nr>>6] &= ~(1UL << (nr & 63)); \
@@ -43,35 +50,7 @@ do { spin_lock(&ctx_alloc_lock); \
43 spin_unlock(&ctx_alloc_lock); \ 50 spin_unlock(&ctx_alloc_lock); \
44} while(0) 51} while(0)
45 52
46/* Reload the two core values used by TLB miss handler 53extern unsigned long tsb_context_switch(unsigned long pgd_pa, unsigned long *tsb);
47 * processing on sparc64. They are:
48 * 1) The physical address of mm->pgd, when full page
49 * table walks are necessary, this is where the
50 * search begins.
51 * 2) A "PGD cache". For 32-bit tasks only pgd[0] is
52 * ever used since that maps the entire low 4GB
53 * completely. To speed up TLB miss processing we
54 * make this value available to the handlers. This
55 * decreases the amount of memory traffic incurred.
56 */
57#define reload_tlbmiss_state(__tsk, __mm) \
58do { \
59 register unsigned long paddr asm("o5"); \
60 register unsigned long pgd_cache asm("o4"); \
61 paddr = __pa((__mm)->pgd); \
62 pgd_cache = 0UL; \
63 if (task_thread_info(__tsk)->flags & _TIF_32BIT) \
64 pgd_cache = get_pgd_cache((__mm)->pgd); \
65 __asm__ __volatile__("wrpr %%g0, 0x494, %%pstate\n\t" \
66 "mov %3, %%g4\n\t" \
67 "mov %0, %%g7\n\t" \
68 "stxa %1, [%%g4] %2\n\t" \
69 "membar #Sync\n\t" \
70 "wrpr %%g0, 0x096, %%pstate" \
71 : /* no outputs */ \
72 : "r" (paddr), "r" (pgd_cache),\
73 "i" (ASI_DMMU), "i" (TSB_REG)); \
74} while(0)
75 54
76/* Set MMU context in the actual hardware. */ 55/* Set MMU context in the actual hardware. */
77#define load_secondary_context(__mm) \ 56#define load_secondary_context(__mm) \
@@ -101,7 +80,8 @@ static inline void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, str
101 80
102 if (!ctx_valid || (old_mm != mm)) { 81 if (!ctx_valid || (old_mm != mm)) {
103 load_secondary_context(mm); 82 load_secondary_context(mm);
104 reload_tlbmiss_state(tsk, mm); 83 tsb_context_switch(__pa(mm->pgd),
84 mm->context.sparc64_tsb);
105 } 85 }
106 86
107 /* Even if (mm == old_mm) we _must_ check 87 /* Even if (mm == old_mm) we _must_ check
@@ -139,7 +119,7 @@ static inline void activate_mm(struct mm_struct *active_mm, struct mm_struct *mm
139 119
140 load_secondary_context(mm); 120 load_secondary_context(mm);
141 __flush_tlb_mm(CTX_HWBITS(mm->context), SECONDARY_CONTEXT); 121 __flush_tlb_mm(CTX_HWBITS(mm->context), SECONDARY_CONTEXT);
142 reload_tlbmiss_state(current, mm); 122 tsb_context_switch(__pa(mm->pgd), mm->context.sparc64_tsb);
143} 123}
144 124
145#endif /* !(__ASSEMBLY__) */ 125#endif /* !(__ASSEMBLY__) */
diff --git a/include/asm-sparc64/pgalloc.h b/include/asm-sparc64/pgalloc.h
index a96067cca963..baf59c00ea47 100644
--- a/include/asm-sparc64/pgalloc.h
+++ b/include/asm-sparc64/pgalloc.h
@@ -61,6 +61,7 @@ static __inline__ void free_pgd_slow(pgd_t *pgd)
61 free_page((unsigned long)pgd); 61 free_page((unsigned long)pgd);
62} 62}
63 63
64/* XXX This crap can die, no longer using virtual page tables... */
64#ifdef DCACHE_ALIASING_POSSIBLE 65#ifdef DCACHE_ALIASING_POSSIBLE
65#define VPTE_COLOR(address) (((address) >> (PAGE_SHIFT + 10)) & 1UL) 66#define VPTE_COLOR(address) (((address) >> (PAGE_SHIFT + 10)) & 1UL)
66#define DCACHE_COLOR(address) (((address) >> PAGE_SHIFT) & 1UL) 67#define DCACHE_COLOR(address) (((address) >> PAGE_SHIFT) & 1UL)
diff --git a/include/asm-sparc64/pgtable.h b/include/asm-sparc64/pgtable.h
index f0a9b44d3eb5..f3ba1e058195 100644
--- a/include/asm-sparc64/pgtable.h
+++ b/include/asm-sparc64/pgtable.h
@@ -25,7 +25,8 @@
25#include <asm/const.h> 25#include <asm/const.h>
26 26
27/* The kernel image occupies 0x4000000 to 0x1000000 (4MB --> 32MB). 27/* The kernel image occupies 0x4000000 to 0x1000000 (4MB --> 32MB).
28 * The page copy blockops can use 0x2000000 to 0x10000000. 28 * The page copy blockops can use 0x2000000 to 0x4000000.
29 * The TSB is mapped in the 0x4000000 to 0x6000000 range.
29 * The PROM resides in an area spanning 0xf0000000 to 0x100000000. 30 * The PROM resides in an area spanning 0xf0000000 to 0x100000000.
30 * The vmalloc area spans 0x100000000 to 0x200000000. 31 * The vmalloc area spans 0x100000000 to 0x200000000.
31 * Since modules need to be in the lowest 32-bits of the address space, 32 * Since modules need to be in the lowest 32-bits of the address space,
@@ -34,6 +35,7 @@
34 * 0x400000000. 35 * 0x400000000.
35 */ 36 */
36#define TLBTEMP_BASE _AC(0x0000000002000000,UL) 37#define TLBTEMP_BASE _AC(0x0000000002000000,UL)
38#define TSBMAP_BASE _AC(0x0000000004000000,UL)
37#define MODULES_VADDR _AC(0x0000000010000000,UL) 39#define MODULES_VADDR _AC(0x0000000010000000,UL)
38#define MODULES_LEN _AC(0x00000000e0000000,UL) 40#define MODULES_LEN _AC(0x00000000e0000000,UL)
39#define MODULES_END _AC(0x00000000f0000000,UL) 41#define MODULES_END _AC(0x00000000f0000000,UL)
@@ -296,11 +298,6 @@ static inline pte_t pte_modify(pte_t orig_pte, pgprot_t new_prot)
296/* to find an entry in a kernel page-table-directory */ 298/* to find an entry in a kernel page-table-directory */
297#define pgd_offset_k(address) pgd_offset(&init_mm, address) 299#define pgd_offset_k(address) pgd_offset(&init_mm, address)
298 300
299/* extract the pgd cache used for optimizing the tlb miss
300 * slow path when executing 32-bit compat processes
301 */
302#define get_pgd_cache(pgd) ((unsigned long) pgd_val(*pgd) << 11)
303
304/* Find an entry in the second-level page table.. */ 301/* Find an entry in the second-level page table.. */
305#define pmd_offset(pudp, address) \ 302#define pmd_offset(pudp, address) \
306 ((pmd_t *) pud_page(*(pudp)) + \ 303 ((pmd_t *) pud_page(*(pudp)) + \
diff --git a/include/asm-sparc64/processor.h b/include/asm-sparc64/processor.h
index cd8d9b4c8658..b3889f3f943a 100644
--- a/include/asm-sparc64/processor.h
+++ b/include/asm-sparc64/processor.h
@@ -28,6 +28,8 @@
28 * User lives in his very own context, and cannot reference us. Note 28 * User lives in his very own context, and cannot reference us. Note
29 * that TASK_SIZE is a misnomer, it really gives maximum user virtual 29 * that TASK_SIZE is a misnomer, it really gives maximum user virtual
30 * address that the kernel will allocate out. 30 * address that the kernel will allocate out.
31 *
32 * XXX No longer using virtual page tables, kill this upper limit...
31 */ 33 */
32#define VA_BITS 44 34#define VA_BITS 44
33#ifndef __ASSEMBLY__ 35#ifndef __ASSEMBLY__
@@ -37,18 +39,6 @@
37#endif 39#endif
38#define TASK_SIZE ((unsigned long)-VPTE_SIZE) 40#define TASK_SIZE ((unsigned long)-VPTE_SIZE)
39 41
40/*
41 * The vpte base must be able to hold the entire vpte, half
42 * of which lives above, and half below, the base. And it
43 * is placed as close to the highest address range as possible.
44 */
45#define VPTE_BASE_SPITFIRE (-(VPTE_SIZE/2))
46#if 1
47#define VPTE_BASE_CHEETAH VPTE_BASE_SPITFIRE
48#else
49#define VPTE_BASE_CHEETAH 0xffe0000000000000
50#endif
51
52#ifndef __ASSEMBLY__ 42#ifndef __ASSEMBLY__
53 43
54typedef struct { 44typedef struct {
diff --git a/include/asm-sparc64/tlbflush.h b/include/asm-sparc64/tlbflush.h
index 3ef9909ac3ac..9ad5d9c51d42 100644
--- a/include/asm-sparc64/tlbflush.h
+++ b/include/asm-sparc64/tlbflush.h
@@ -5,6 +5,11 @@
5#include <linux/mm.h> 5#include <linux/mm.h>
6#include <asm/mmu_context.h> 6#include <asm/mmu_context.h>
7 7
8/* TSB flush operations. */
9struct mmu_gather;
10extern void flush_tsb_kernel_range(unsigned long start, unsigned long end);
11extern void flush_tsb_user(struct mmu_gather *mp);
12
8/* TLB flush operations. */ 13/* TLB flush operations. */
9 14
10extern void flush_tlb_pending(void); 15extern void flush_tlb_pending(void);
@@ -14,28 +19,36 @@ extern void flush_tlb_pending(void);
14#define flush_tlb_page(vma,addr) flush_tlb_pending() 19#define flush_tlb_page(vma,addr) flush_tlb_pending()
15#define flush_tlb_mm(mm) flush_tlb_pending() 20#define flush_tlb_mm(mm) flush_tlb_pending()
16 21
22/* Local cpu only. */
17extern void __flush_tlb_all(void); 23extern void __flush_tlb_all(void);
24
18extern void __flush_tlb_page(unsigned long context, unsigned long page, unsigned long r); 25extern void __flush_tlb_page(unsigned long context, unsigned long page, unsigned long r);
19 26
20extern void __flush_tlb_kernel_range(unsigned long start, unsigned long end); 27extern void __flush_tlb_kernel_range(unsigned long start, unsigned long end);
21 28
22#ifndef CONFIG_SMP 29#ifndef CONFIG_SMP
23 30
24#define flush_tlb_all() __flush_tlb_all()
25#define flush_tlb_kernel_range(start,end) \ 31#define flush_tlb_kernel_range(start,end) \
26 __flush_tlb_kernel_range(start,end) 32do { flush_tsb_kernel_range(start,end); \
33 __flush_tlb_kernel_range(start,end); \
34} while (0)
27 35
28#else /* CONFIG_SMP */ 36#else /* CONFIG_SMP */
29 37
30extern void smp_flush_tlb_all(void);
31extern void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end); 38extern void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end);
32 39
33#define flush_tlb_all() smp_flush_tlb_all()
34#define flush_tlb_kernel_range(start, end) \ 40#define flush_tlb_kernel_range(start, end) \
35 smp_flush_tlb_kernel_range(start, end) 41do { flush_tsb_kernel_range(start,end); \
42 smp_flush_tlb_kernel_range(start, end); \
43} while (0)
36 44
37#endif /* ! CONFIG_SMP */ 45#endif /* ! CONFIG_SMP */
38 46
39extern void flush_tlb_pgtables(struct mm_struct *, unsigned long, unsigned long); 47static inline void flush_tlb_pgtables(struct mm_struct *mm, unsigned long start, unsigned long end)
48{
49 /* We don't use virtual page tables for TLB miss processing
50 * any more. Nowadays we use the TSB.
51 */
52}
40 53
41#endif /* _SPARC64_TLBFLUSH_H */ 54#endif /* _SPARC64_TLBFLUSH_H */
diff --git a/include/asm-sparc64/tsb.h b/include/asm-sparc64/tsb.h
new file mode 100644
index 000000000000..03d272e0e477
--- /dev/null
+++ b/include/asm-sparc64/tsb.h
@@ -0,0 +1,165 @@
1#ifndef _SPARC64_TSB_H
2#define _SPARC64_TSB_H
3
4/* The sparc64 TSB is similar to the powerpc hashtables. It's a
5 * power-of-2 sized table of TAG/PTE pairs. The cpu precomputes
6 * pointers into this table for 8K and 64K page sizes, and also a
7 * comparison TAG based upon the virtual address and context which
8 * faults.
9 *
10 * TLB miss trap handler software does the actual lookup via something
11 * of the form:
12 *
13 * ldxa [%g0] ASI_{D,I}MMU_TSB_8KB_PTR, %g1
14 * ldxa [%g0] ASI_{D,I}MMU, %g6
15 * ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4
16 * cmp %g4, %g6
17 * bne,pn %xcc, tsb_miss_{d,i}tlb
18 * mov FAULT_CODE_{D,I}TLB, %g3
19 * stxa %g5, [%g0] ASI_{D,I}TLB_DATA_IN
20 * retry
21 *
22
23 * Each 16-byte slot of the TSB is the 8-byte tag and then the 8-byte
24 * PTE. The TAG is of the same layout as the TLB TAG TARGET mmu
25 * register which is:
26 *
27 * -------------------------------------------------
28 * | - | CONTEXT | - | VADDR bits 63:22 |
29 * -------------------------------------------------
30 * 63 61 60 48 47 42 41 0
31 *
32 * Like the powerpc hashtables we need to use locking in order to
33 * synchronize while we update the entries. PTE updates need locking
34 * as well.
35 *
36 * We need to carefully choose a lock bits for the TSB entry. We
37 * choose to use bit 47 in the tag. Also, since we never map anything
38 * at page zero in context zero, we use zero as an invalid tag entry.
39 * When the lock bit is set, this forces a tag comparison failure.
40 *
41 * Currently, we allocate an 8K TSB per-process and we use it for both
42 * I-TLB and D-TLB misses. Perhaps at some point we'll add code that
43 * monitors the number of active pages in the process as we get
44 * major/minor faults, and grow the TSB in response. The only trick
45 * in implementing that is synchronizing the freeing of the old TSB
46 * wrt. parallel TSB updates occuring on other processors. On
47 * possible solution is to use RCU for the freeing of the TSB.
48 */
49
50#define TSB_TAG_LOCK (1 << (47 - 32))
51
52#define TSB_MEMBAR membar #StoreStore
53
54#define TSB_LOCK_TAG(TSB, REG1, REG2) \
5599: lduwa [TSB] ASI_N, REG1; \
56 sethi %hi(TSB_TAG_LOCK), REG2;\
57 andcc REG1, REG2, %g0; \
58 bne,pn %icc, 99b; \
59 nop; \
60 casa [TSB] ASI_N, REG1, REG2;\
61 cmp REG1, REG2; \
62 bne,pn %icc, 99b; \
63 nop; \
64 TSB_MEMBAR
65
66#define TSB_WRITE(TSB, TTE, TAG) \
67 stx TTE, [TSB + 0x08]; \
68 TSB_MEMBAR; \
69 stx TAG, [TSB + 0x00];
70
71 /* Do a kernel page table walk. Leaves physical PTE pointer in
72 * REG1. Jumps to FAIL_LABEL on early page table walk termination.
73 * VADDR will not be clobbered, but REG2 will.
74 */
75#define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL) \
76 sethi %hi(swapper_pg_dir), REG1; \
77 or REG1, %lo(swapper_pg_dir), REG1; \
78 sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
79 srlx REG2, 64 - PAGE_SHIFT, REG2; \
80 andn REG2, 0x3, REG2; \
81 lduw [REG1 + REG2], REG1; \
82 brz,pn REG1, FAIL_LABEL; \
83 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
84 srlx REG2, 64 - PAGE_SHIFT, REG2; \
85 sllx REG1, 11, REG1; \
86 andn REG2, 0x3, REG2; \
87 lduwa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
88 brz,pn REG1, FAIL_LABEL; \
89 sllx VADDR, 64 - PMD_SHIFT, REG2; \
90 srlx REG2, 64 - PAGE_SHIFT, REG2; \
91 sllx REG1, 11, REG1; \
92 andn REG2, 0x7, REG2; \
93 add REG1, REG2, REG1;
94
95 /* Do a user page table walk in MMU globals. Leaves physical PTE
96 * pointer in REG1. Jumps to FAIL_LABEL on early page table walk
97 * termination. Physical base of page tables is in PHYS_PGD which
98 * will not be modified.
99 *
100 * VADDR will not be clobbered, but REG1 and REG2 will.
101 */
102#define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL) \
103 sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
104 srlx REG2, 64 - PAGE_SHIFT, REG2; \
105 andn REG2, 0x3, REG2; \
106 lduwa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \
107 brz,pn REG1, FAIL_LABEL; \
108 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
109 srlx REG2, 64 - PAGE_SHIFT, REG2; \
110 sllx REG1, 11, REG1; \
111 andn REG2, 0x3, REG2; \
112 lduwa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
113 brz,pn REG1, FAIL_LABEL; \
114 sllx VADDR, 64 - PMD_SHIFT, REG2; \
115 srlx REG2, 64 - PAGE_SHIFT, REG2; \
116 sllx REG1, 11, REG1; \
117 andn REG2, 0x7, REG2; \
118 add REG1, REG2, REG1;
119
120/* Lookup a OBP mapping on VADDR in the prom_trans[] table at TL>0.
121 * If no entry is found, FAIL_LABEL will be branched to. On success
122 * the resulting PTE value will be left in REG1. VADDR is preserved
123 * by this routine.
124 */
125#define OBP_TRANS_LOOKUP(VADDR, REG1, REG2, REG3, FAIL_LABEL) \
126 sethi %hi(prom_trans), REG1; \
127 or REG1, %lo(prom_trans), REG1; \
12897: ldx [REG1 + 0x00], REG2; \
129 brz,pn REG2, FAIL_LABEL; \
130 nop; \
131 ldx [REG1 + 0x08], REG3; \
132 add REG2, REG3, REG3; \
133 cmp REG2, VADDR; \
134 bgu,pt %xcc, 98f; \
135 cmp VADDR, REG3; \
136 bgeu,pt %xcc, 98f; \
137 ldx [REG1 + 0x10], REG3; \
138 sub VADDR, REG2, REG2; \
139 ba,pt %xcc, 99f; \
140 add REG3, REG2, REG1; \
14198: ba,pt %xcc, 97b; \
142 add REG1, (3 * 8), REG1; \
14399:
144
145 /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL
146 * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries
147 * and the found TTE will be left in REG1. REG3 and REG4 must
148 * be an even/odd pair of registers.
149 *
150 * VADDR and TAG will be preserved and not clobbered by this macro.
151 */
152 /* XXX non-8K base page size support... */
153#define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
154 sethi %hi(swapper_tsb), REG1; \
155 or REG1, %lo(swapper_tsb), REG1; \
156 srlx VADDR, 13, REG2; \
157 and REG2, (512 - 1), REG2; \
158 sllx REG2, 4, REG2; \
159 add REG1, REG2, REG2; \
160 ldda [REG2] ASI_NUCLEUS_QUAD_LDD, REG3; \
161 cmp REG3, TAG; \
162 be,a,pt %xcc, OK_LABEL; \
163 mov REG4, REG1;
164
165#endif /* !(_SPARC64_TSB_H) */
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-10 07:29:54 -0500