aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc64
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc64')
-rw-r--r--arch/sparc64/kernel/Makefile2
-rw-r--r--arch/sparc64/kernel/binfmt_aout32.c13
-rw-r--r--arch/sparc64/kernel/dtlb_backend.S170
-rw-r--r--arch/sparc64/kernel/dtlb_base.S109
-rw-r--r--arch/sparc64/kernel/dtlb_miss.S39
-rw-r--r--arch/sparc64/kernel/etrap.S2
-rw-r--r--arch/sparc64/kernel/head.S46
-rw-r--r--arch/sparc64/kernel/itlb_base.S79
-rw-r--r--arch/sparc64/kernel/itlb_miss.S39
-rw-r--r--arch/sparc64/kernel/ktlb.S263
-rw-r--r--arch/sparc64/kernel/process.c25
-rw-r--r--arch/sparc64/kernel/rtrap.S6
-rw-r--r--arch/sparc64/kernel/smp.c12
-rw-r--r--arch/sparc64/kernel/trampoline.S33
-rw-r--r--arch/sparc64/kernel/tsb.S169
-rw-r--r--arch/sparc64/kernel/ttable.S6
-rw-r--r--arch/sparc64/kernel/vmlinux.lds.S3
-rw-r--r--arch/sparc64/kernel/winfixup.S8
-rw-r--r--arch/sparc64/mm/Makefile2
-rw-r--r--arch/sparc64/mm/init.c91
-rw-r--r--arch/sparc64/mm/tlb.c61
-rw-r--r--arch/sparc64/mm/tsb.c84
-rw-r--r--arch/sparc64/mm/ultra.S58
23 files changed, 489 insertions, 831 deletions
diff --git a/arch/sparc64/kernel/Makefile b/arch/sparc64/kernel/Makefile
index 83d67eb18895..a482a9ffe5bc 100644
--- a/arch/sparc64/kernel/Makefile
+++ b/arch/sparc64/kernel/Makefile
@@ -38,5 +38,5 @@ else
38 CMODEL_CFLAG := -m64 -mcmodel=medlow 38 CMODEL_CFLAG := -m64 -mcmodel=medlow
39endif 39endif
40 40
41head.o: head.S ttable.S itlb_base.S dtlb_base.S dtlb_backend.S dtlb_prot.S \ 41head.o: head.S ttable.S itlb_miss.S dtlb_miss.S ktlb.S tsb.S \
42 etrap.S rtrap.S winfixup.S entry.S 42 etrap.S rtrap.S winfixup.S entry.S
diff --git a/arch/sparc64/kernel/binfmt_aout32.c b/arch/sparc64/kernel/binfmt_aout32.c
index 202a80c24b6f..a57d7f2b6f13 100644
--- a/arch/sparc64/kernel/binfmt_aout32.c
+++ b/arch/sparc64/kernel/binfmt_aout32.c
@@ -31,6 +31,7 @@
31#include <asm/system.h> 31#include <asm/system.h>
32#include <asm/uaccess.h> 32#include <asm/uaccess.h>
33#include <asm/pgalloc.h> 33#include <asm/pgalloc.h>
34#include <asm/mmu_context.h>
34 35
35static int load_aout32_binary(struct linux_binprm *, struct pt_regs * regs); 36static int load_aout32_binary(struct linux_binprm *, struct pt_regs * regs);
36static int load_aout32_library(struct file*); 37static int load_aout32_library(struct file*);
@@ -329,15 +330,9 @@ beyond_if:
329 330
330 current->mm->start_stack = 331 current->mm->start_stack =
331 (unsigned long) create_aout32_tables((char __user *)bprm->p, bprm); 332 (unsigned long) create_aout32_tables((char __user *)bprm->p, bprm);
332 if (!(orig_thr_flags & _TIF_32BIT)) { 333 tsb_context_switch(__pa(current->mm->pgd),
333 unsigned long pgd_cache = get_pgd_cache(current->mm->pgd); 334 current->mm->context.sparc64_tsb);
334 335
335 __asm__ __volatile__("stxa\t%0, [%1] %2\n\t"
336 "membar #Sync"
337 : /* no outputs */
338 : "r" (pgd_cache),
339 "r" (TSB_REG), "i" (ASI_DMMU));
340 }
341 start_thread32(regs, ex.a_entry, current->mm->start_stack); 336 start_thread32(regs, ex.a_entry, current->mm->start_stack);
342 if (current->ptrace & PT_PTRACED) 337 if (current->ptrace & PT_PTRACED)
343 send_sig(SIGTRAP, current, 0); 338 send_sig(SIGTRAP, current, 0);
diff --git a/arch/sparc64/kernel/dtlb_backend.S b/arch/sparc64/kernel/dtlb_backend.S
deleted file mode 100644
index acc889a7f9c1..000000000000
--- a/arch/sparc64/kernel/dtlb_backend.S
+++ /dev/null
@@ -1,170 +0,0 @@
1/* $Id: dtlb_backend.S,v 1.16 2001/10/09 04:02:11 davem Exp $
2 * dtlb_backend.S: Back end to DTLB miss replacement strategy.
3 * This is included directly into the trap table.
4 *
5 * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com)
6 * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz)
7 */
8
9#include <asm/pgtable.h>
10#include <asm/mmu.h>
11
12#define VALID_SZ_BITS (_PAGE_VALID | _PAGE_SZBITS)
13
14#define VPTE_BITS (_PAGE_CP | _PAGE_CV | _PAGE_P )
15#define VPTE_SHIFT (PAGE_SHIFT - 3)
16
17/* Ways we can get here:
18 *
19 * 1) Nucleus loads and stores to/from PA-->VA direct mappings at tl>1.
20 * 2) Nucleus loads and stores to/from user/kernel window save areas.
21 * 3) VPTE misses from dtlb_base and itlb_base.
22 *
23 * We need to extract out the PMD and PGDIR indexes from the
24 * linear virtual page table access address. The PTE index
25 * is at the bottom, but we are not concerned with it. Bits
26 * 0 to 2 are clear since each PTE is 8 bytes in size. Each
27 * PMD and PGDIR entry are 4 bytes in size. Thus, this
28 * address looks something like:
29 *
30 * |---------------------------------------------------------------|
31 * | ... | PGDIR index | PMD index | PTE index | |
32 * |---------------------------------------------------------------|
33 * 63 F E D C B A 3 2 0 <- bit nr
34 *
35 * The variable bits above are defined as:
36 * A --> 3 + (PAGE_SHIFT - log2(8))
37 * --> 3 + (PAGE_SHIFT - 3) - 1
38 * (ie. this is "bit 3" + PAGE_SIZE - size of PTE entry in bits - 1)
39 * B --> A + 1
40 * C --> B + (PAGE_SHIFT - log2(4))
41 * --> B + (PAGE_SHIFT - 2) - 1
42 * (ie. this is "bit B" + PAGE_SIZE - size of PMD entry in bits - 1)
43 * D --> C + 1
44 * E --> D + (PAGE_SHIFT - log2(4))
45 * --> D + (PAGE_SHIFT - 2) - 1
46 * (ie. this is "bit D" + PAGE_SIZE - size of PGDIR entry in bits - 1)
47 * F --> E + 1
48 *
49 * (Note how "B" always evalutes to PAGE_SHIFT, all the other constants
50 * cancel out.)
51 *
52 * For 8K PAGE_SIZE (thus, PAGE_SHIFT of 13) the bit numbers are:
53 * A --> 12
54 * B --> 13
55 * C --> 23
56 * D --> 24
57 * E --> 34
58 * F --> 35
59 *
60 * For 64K PAGE_SIZE (thus, PAGE_SHIFT of 16) the bit numbers are:
61 * A --> 15
62 * B --> 16
63 * C --> 29
64 * D --> 30
65 * E --> 43
66 * F --> 44
67 *
68 * Because bits both above and below each PGDIR and PMD index need to
69 * be masked out, and the index can be as long as 14 bits (when using a
70 * 64K PAGE_SIZE, and thus a PAGE_SHIFT of 16), we need 3 instructions
71 * to extract each index out.
72 *
73 * Shifts do not pair very well on UltraSPARC-I, II, IIi, and IIe, so
74 * we try to avoid using them for the entire operation. We could setup
75 * a mask anywhere from bit 31 down to bit 10 using the sethi instruction.
76 *
77 * We need a mask covering bits B --> C and one covering D --> E.
78 * For 8K PAGE_SIZE these masks are 0x00ffe000 and 0x7ff000000.
79 * For 64K PAGE_SIZE these masks are 0x3fff0000 and 0xfffc0000000.
80 * The second in each set cannot be loaded with a single sethi
81 * instruction, because the upper bits are past bit 32. We would
82 * need to use a sethi + a shift.
83 *
84 * For the time being, we use 2 shifts and a simple "and" mask.
85 * We shift left to clear the bits above the index, we shift down
86 * to clear the bits below the index (sans the log2(4 or 8) bits)
87 * and a mask to clear the log2(4 or 8) bits. We need therefore
88 * define 4 shift counts, all of which are relative to PAGE_SHIFT.
89 *
90 * Although unsupportable for other reasons, this does mean that
91 * 512K and 4MB page sizes would be generaally supported by the
92 * kernel. (ELF binaries would break with > 64K PAGE_SIZE since
93 * the sections are only aligned that strongly).
94 *
95 * The operations performed for extraction are thus:
96 *
97 * ((X << FOO_SHIFT_LEFT) >> FOO_SHIFT_RIGHT) & ~0x3
98 *
99 */
100
101#define A (3 + (PAGE_SHIFT - 3) - 1)
102#define B (A + 1)
103#define C (B + (PAGE_SHIFT - 2) - 1)
104#define D (C + 1)
105#define E (D + (PAGE_SHIFT - 2) - 1)
106#define F (E + 1)
107
108#define PMD_SHIFT_LEFT (64 - D)
109#define PMD_SHIFT_RIGHT (64 - (D - B) - 2)
110#define PGDIR_SHIFT_LEFT (64 - F)
111#define PGDIR_SHIFT_RIGHT (64 - (F - D) - 2)
112#define LOW_MASK_BITS 0x3
113
114/* TLB1 ** ICACHE line 1: tl1 DTLB and quick VPTE miss */
115 ldxa [%g1 + %g1] ASI_DMMU, %g4 ! Get TAG_ACCESS
116 add %g3, %g3, %g5 ! Compute VPTE base
117 cmp %g4, %g5 ! VPTE miss?
118 bgeu,pt %xcc, 1f ! Continue here
119 andcc %g4, TAG_CONTEXT_BITS, %g5 ! tl0 miss Nucleus test
120 ba,a,pt %xcc, from_tl1_trap ! Fall to tl0 miss
1211: sllx %g6, VPTE_SHIFT, %g4 ! Position TAG_ACCESS
122 or %g4, %g5, %g4 ! Prepare TAG_ACCESS
123
124/* TLB1 ** ICACHE line 2: Quick VPTE miss */
125 mov TSB_REG, %g1 ! Grab TSB reg
126 ldxa [%g1] ASI_DMMU, %g5 ! Doing PGD caching?
127 sllx %g6, PMD_SHIFT_LEFT, %g1 ! Position PMD offset
128 be,pn %xcc, sparc64_vpte_nucleus ! Is it from Nucleus?
129 srlx %g1, PMD_SHIFT_RIGHT, %g1 ! Mask PMD offset bits
130 brnz,pt %g5, sparc64_vpte_continue ! Yep, go like smoke
131 andn %g1, LOW_MASK_BITS, %g1 ! Final PMD mask
132 sllx %g6, PGDIR_SHIFT_LEFT, %g5 ! Position PGD offset
133
134/* TLB1 ** ICACHE line 3: Quick VPTE miss */
135 srlx %g5, PGDIR_SHIFT_RIGHT, %g5 ! Mask PGD offset bits
136 andn %g5, LOW_MASK_BITS, %g5 ! Final PGD mask
137 lduwa [%g7 + %g5] ASI_PHYS_USE_EC, %g5! Load PGD
138 brz,pn %g5, vpte_noent ! Valid?
139sparc64_kpte_continue:
140 sllx %g5, 11, %g5 ! Shift into place
141sparc64_vpte_continue:
142 lduwa [%g5 + %g1] ASI_PHYS_USE_EC, %g5! Load PMD
143 sllx %g5, 11, %g5 ! Shift into place
144 brz,pn %g5, vpte_noent ! Valid?
145
146/* TLB1 ** ICACHE line 4: Quick VPTE miss */
147 mov (VALID_SZ_BITS >> 61), %g1 ! upper vpte into %g1
148 sllx %g1, 61, %g1 ! finish calc
149 or %g5, VPTE_BITS, %g5 ! Prepare VPTE data
150 or %g5, %g1, %g5 ! ...
151 mov TLB_SFSR, %g1 ! Restore %g1 value
152 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Load VPTE into TLB
153 stxa %g4, [%g1 + %g1] ASI_DMMU ! Restore previous TAG_ACCESS
154 retry ! Load PTE once again
155
156#undef VALID_SZ_BITS
157#undef VPTE_SHIFT
158#undef VPTE_BITS
159#undef A
160#undef B
161#undef C
162#undef D
163#undef E
164#undef F
165#undef PMD_SHIFT_LEFT
166#undef PMD_SHIFT_RIGHT
167#undef PGDIR_SHIFT_LEFT
168#undef PGDIR_SHIFT_RIGHT
169#undef LOW_MASK_BITS
170
diff --git a/arch/sparc64/kernel/dtlb_base.S b/arch/sparc64/kernel/dtlb_base.S
deleted file mode 100644
index 6528786840c0..000000000000
--- a/arch/sparc64/kernel/dtlb_base.S
+++ /dev/null
@@ -1,109 +0,0 @@
1/* $Id: dtlb_base.S,v 1.17 2001/10/11 22:33:52 davem Exp $
2 * dtlb_base.S: Front end to DTLB miss replacement strategy.
3 * This is included directly into the trap table.
4 *
5 * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com)
6 * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz)
7 */
8
9#include <asm/pgtable.h>
10#include <asm/mmu.h>
11
12/* %g1 TLB_SFSR (%g1 + %g1 == TLB_TAG_ACCESS)
13 * %g2 (KERN_HIGHBITS | KERN_LOWBITS)
14 * %g3 VPTE base (0xfffffffe00000000) Spitfire/Blackbird (44-bit VA space)
15 * (0xffe0000000000000) Cheetah (64-bit VA space)
16 * %g7 __pa(current->mm->pgd)
17 *
18 * The VPTE base value is completely magic, but note that
19 * few places in the kernel other than these TLB miss
20 * handlers know anything about the VPTE mechanism or
21 * how it works (see VPTE_SIZE, TASK_SIZE and PTRS_PER_PGD).
22 * Consider the 44-bit VADDR Ultra-I/II case as an example:
23 *
24 * VA[0 : (1<<43)] produce VPTE index [%g3 : 0]
25 * VA[0 : -(1<<43)] produce VPTE index [%g3-(1<<(43-PAGE_SHIFT+3)) : %g3]
26 *
27 * For Cheetah's 64-bit VADDR space this is:
28 *
29 * VA[0 : (1<<63)] produce VPTE index [%g3 : 0]
30 * VA[0 : -(1<<63)] produce VPTE index [%g3-(1<<(63-PAGE_SHIFT+3)) : %g3]
31 *
32 * If you're paying attention you'll notice that this means half of
33 * the VPTE table is above %g3 and half is below, low VA addresses
34 * map progressively upwards from %g3, and high VA addresses map
35 * progressively upwards towards %g3. This trick was needed to make
36 * the same 8 instruction handler work both for Spitfire/Blackbird's
37 * peculiar VA space hole configuration and the full 64-bit VA space
38 * one of Cheetah at the same time.
39 */
40
41/* Ways we can get here:
42 *
43 * 1) Nucleus loads and stores to/from PA-->VA direct mappings.
44 * 2) Nucleus loads and stores to/from vmalloc() areas.
45 * 3) User loads and stores.
46 * 4) User space accesses by nucleus at tl0
47 */
48
49#if PAGE_SHIFT == 13
50/*
51 * To compute vpte offset, we need to do ((addr >> 13) << 3),
52 * which can be optimized to (addr >> 10) if bits 10/11/12 can
53 * be guaranteed to be 0 ... mmu_context.h does guarantee this
54 * by only using 10 bits in the hwcontext value.
55 */
56#define CREATE_VPTE_OFFSET1(r1, r2) nop
57#define CREATE_VPTE_OFFSET2(r1, r2) \
58 srax r1, 10, r2
59#else
60#define CREATE_VPTE_OFFSET1(r1, r2) \
61 srax r1, PAGE_SHIFT, r2
62#define CREATE_VPTE_OFFSET2(r1, r2) \
63 sllx r2, 3, r2
64#endif
65
66/* DTLB ** ICACHE line 1: Quick user TLB misses */
67 mov TLB_SFSR, %g1
68 ldxa [%g1 + %g1] ASI_DMMU, %g4 ! Get TAG_ACCESS
69 andcc %g4, TAG_CONTEXT_BITS, %g0 ! From Nucleus?
70from_tl1_trap:
71 rdpr %tl, %g5 ! For TL==3 test
72 CREATE_VPTE_OFFSET1(%g4, %g6) ! Create VPTE offset
73 be,pn %xcc, kvmap ! Yep, special processing
74 CREATE_VPTE_OFFSET2(%g4, %g6) ! Create VPTE offset
75 cmp %g5, 4 ! Last trap level?
76
77/* DTLB ** ICACHE line 2: User finish + quick kernel TLB misses */
78 be,pn %xcc, longpath ! Yep, cannot risk VPTE miss
79 nop ! delay slot
80 ldxa [%g3 + %g6] ASI_S, %g5 ! Load VPTE
811: brgez,pn %g5, longpath ! Invalid, branch out
82 nop ! Delay-slot
839: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
84 retry ! Trap return
85 nop
86
87/* DTLB ** ICACHE line 3: winfixups+real_faults */
88longpath:
89 rdpr %pstate, %g5 ! Move into alternate globals
90 wrpr %g5, PSTATE_AG|PSTATE_MG, %pstate
91 rdpr %tl, %g4 ! See where we came from.
92 cmp %g4, 1 ! Is etrap/rtrap window fault?
93 mov TLB_TAG_ACCESS, %g4 ! Prepare for fault processing
94 ldxa [%g4] ASI_DMMU, %g5 ! Load faulting VA page
95 be,pt %xcc, sparc64_realfault_common ! Jump to normal fault handling
96 mov FAULT_CODE_DTLB, %g4 ! It was read from DTLB
97
98/* DTLB ** ICACHE line 4: Unused... */
99 ba,a,pt %xcc, winfix_trampoline ! Call window fixup code
100 nop
101 nop
102 nop
103 nop
104 nop
105 nop
106 nop
107
108#undef CREATE_VPTE_OFFSET1
109#undef CREATE_VPTE_OFFSET2
diff --git a/arch/sparc64/kernel/dtlb_miss.S b/arch/sparc64/kernel/dtlb_miss.S
new file mode 100644
index 000000000000..d0f1565cb564
--- /dev/null
+++ b/arch/sparc64/kernel/dtlb_miss.S
@@ -0,0 +1,39 @@
1/* DTLB ** ICACHE line 1: Context 0 check and TSB load */
2 ldxa [%g0] ASI_DMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer
3 ldxa [%g0] ASI_DMMU, %g6 ! Get TAG TARGET
4 srlx %g6, 48, %g5 ! Get context
5 brz,pn %g5, kvmap_dtlb ! Context 0 processing
6 nop ! Delay slot (fill me)
7 ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB entry
8 nop ! Push branch to next I$ line
9 cmp %g4, %g6 ! Compare TAG
10
11/* DTLB ** ICACHE line 2: TSB compare and TLB load */
12 bne,pn %xcc, tsb_miss_dtlb ! Miss
13 mov FAULT_CODE_DTLB, %g3
14 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Load TLB
15 retry ! Trap done
16 nop
17 nop
18 nop
19 nop
20
21/* DTLB ** ICACHE line 3: */
22 nop
23 nop
24 nop
25 nop
26 nop
27 nop
28 nop
29 nop
30
31/* DTLB ** ICACHE line 4: */
32 nop
33 nop
34 nop
35 nop
36 nop
37 nop
38 nop
39 nop
diff --git a/arch/sparc64/kernel/etrap.S b/arch/sparc64/kernel/etrap.S
index 0d8eba21111b..567dbb765c34 100644
--- a/arch/sparc64/kernel/etrap.S
+++ b/arch/sparc64/kernel/etrap.S
@@ -99,6 +99,7 @@ etrap_irq:
99 wrpr %g0, ETRAP_PSTATE2, %pstate 99 wrpr %g0, ETRAP_PSTATE2, %pstate
100 mov %l6, %g6 100 mov %l6, %g6
101#ifdef CONFIG_SMP 101#ifdef CONFIG_SMP
102#error IMMU TSB usage must be fixed
102 mov TSB_REG, %g3 103 mov TSB_REG, %g3
103 ldxa [%g3] ASI_IMMU, %g5 104 ldxa [%g3] ASI_IMMU, %g5
104#endif 105#endif
@@ -248,6 +249,7 @@ scetrap: rdpr %pil, %g2
248 mov %l6, %g6 249 mov %l6, %g6
249 stx %i7, [%sp + PTREGS_OFF + PT_V9_I7] 250 stx %i7, [%sp + PTREGS_OFF + PT_V9_I7]
250#ifdef CONFIG_SMP 251#ifdef CONFIG_SMP
252#error IMMU TSB usage must be fixed
251 mov TSB_REG, %g3 253 mov TSB_REG, %g3
252 ldxa [%g3] ASI_IMMU, %g5 254 ldxa [%g3] ASI_IMMU, %g5
253#endif 255#endif
diff --git a/arch/sparc64/kernel/head.S b/arch/sparc64/kernel/head.S
index b49dcd4504b0..d00e20693be1 100644
--- a/arch/sparc64/kernel/head.S
+++ b/arch/sparc64/kernel/head.S
@@ -429,17 +429,6 @@ setup_trap_table:
429 * 429 *
430 * %g6 --> current_thread_info() 430 * %g6 --> current_thread_info()
431 * 431 *
432 * MMU Globals (PSTATE_MG):
433 *
434 * %g1 --> TLB_SFSR
435 * %g2 --> ((_PAGE_VALID | _PAGE_SZ4MB |
436 * _PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
437 * ^ 0xfffff80000000000)
438 * (this %g2 value is used for computing the PAGE_OFFSET kernel
439 * TLB entries quickly, the virtual address of the fault XOR'd
440 * with this %g2 value is the PTE to load into the TLB)
441 * %g3 --> VPTE_BASE_CHEETAH or VPTE_BASE_SPITFIRE
442 *
443 * Interrupt Globals (PSTATE_IG, setup by init_irqwork_curcpu()): 432 * Interrupt Globals (PSTATE_IG, setup by init_irqwork_curcpu()):
444 * 433 *
445 * %g6 --> __irq_work[smp_processor_id()] 434 * %g6 --> __irq_work[smp_processor_id()]
@@ -450,40 +439,6 @@ setup_trap_table:
450 wrpr %o1, PSTATE_AG, %pstate 439 wrpr %o1, PSTATE_AG, %pstate
451 mov %o2, %g6 440 mov %o2, %g6
452 441
453#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
454#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
455 wrpr %o1, PSTATE_MG, %pstate
456 mov TSB_REG, %g1
457 stxa %g0, [%g1] ASI_DMMU
458 membar #Sync
459 stxa %g0, [%g1] ASI_IMMU
460 membar #Sync
461 mov TLB_SFSR, %g1
462 sethi %uhi(KERN_HIGHBITS), %g2
463 or %g2, %ulo(KERN_HIGHBITS), %g2
464 sllx %g2, 32, %g2
465 or %g2, KERN_LOWBITS, %g2
466
467 BRANCH_IF_ANY_CHEETAH(g3,g7,8f)
468 ba,pt %xcc, 9f
469 nop
470
4718:
472 sethi %uhi(VPTE_BASE_CHEETAH), %g3
473 or %g3, %ulo(VPTE_BASE_CHEETAH), %g3
474 ba,pt %xcc, 2f
475 sllx %g3, 32, %g3
476
4779:
478 sethi %uhi(VPTE_BASE_SPITFIRE), %g3
479 or %g3, %ulo(VPTE_BASE_SPITFIRE), %g3
480 sllx %g3, 32, %g3
481
4822:
483 clr %g7
484#undef KERN_HIGHBITS
485#undef KERN_LOWBITS
486
487 /* Kill PROM timer */ 442 /* Kill PROM timer */
488 sethi %hi(0x80000000), %o2 443 sethi %hi(0x80000000), %o2
489 sllx %o2, 32, %o2 444 sllx %o2, 32, %o2
@@ -538,6 +493,7 @@ sparc64_boot_end:
538 493
539#include "systbls.S" 494#include "systbls.S"
540#include "ktlb.S" 495#include "ktlb.S"
496#include "tsb.S"
541#include "etrap.S" 497#include "etrap.S"
542#include "rtrap.S" 498#include "rtrap.S"
543#include "winfixup.S" 499#include "winfixup.S"
diff --git a/arch/sparc64/kernel/itlb_base.S b/arch/sparc64/kernel/itlb_base.S
deleted file mode 100644
index 4951ff8f6877..000000000000
--- a/arch/sparc64/kernel/itlb_base.S
+++ /dev/null
@@ -1,79 +0,0 @@
1/* $Id: itlb_base.S,v 1.12 2002/02/09 19:49:30 davem Exp $
2 * itlb_base.S: Front end to ITLB miss replacement strategy.
3 * This is included directly into the trap table.
4 *
5 * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com)
6 * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz)
7 */
8
9#if PAGE_SHIFT == 13
10/*
11 * To compute vpte offset, we need to do ((addr >> 13) << 3),
12 * which can be optimized to (addr >> 10) if bits 10/11/12 can
13 * be guaranteed to be 0 ... mmu_context.h does guarantee this
14 * by only using 10 bits in the hwcontext value.
15 */
16#define CREATE_VPTE_OFFSET1(r1, r2) \
17 srax r1, 10, r2
18#define CREATE_VPTE_OFFSET2(r1, r2) nop
19#else /* PAGE_SHIFT */
20#define CREATE_VPTE_OFFSET1(r1, r2) \
21 srax r1, PAGE_SHIFT, r2
22#define CREATE_VPTE_OFFSET2(r1, r2) \
23 sllx r2, 3, r2
24#endif /* PAGE_SHIFT */
25
26
27/* Ways we can get here:
28 *
29 * 1) Nucleus instruction misses from module code.
30 * 2) All user instruction misses.
31 *
32 * All real page faults merge their code paths to the
33 * sparc64_realfault_common label below.
34 */
35
36/* ITLB ** ICACHE line 1: Quick user TLB misses */
37 mov TLB_SFSR, %g1
38 ldxa [%g1 + %g1] ASI_IMMU, %g4 ! Get TAG_ACCESS
39 CREATE_VPTE_OFFSET1(%g4, %g6) ! Create VPTE offset
40 CREATE_VPTE_OFFSET2(%g4, %g6) ! Create VPTE offset
41 ldxa [%g3 + %g6] ASI_P, %g5 ! Load VPTE
421: brgez,pn %g5, 3f ! Not valid, branch out
43 sethi %hi(_PAGE_EXEC), %g4 ! Delay-slot
44 andcc %g5, %g4, %g0 ! Executable?
45
46/* ITLB ** ICACHE line 2: Real faults */
47 be,pn %xcc, 3f ! Nope, branch.
48 nop ! Delay-slot
492: stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Load PTE into TLB
50 retry ! Trap return
513: rdpr %pstate, %g4 ! Move into alt-globals
52 wrpr %g4, PSTATE_AG|PSTATE_MG, %pstate
53 rdpr %tpc, %g5 ! And load faulting VA
54 mov FAULT_CODE_ITLB, %g4 ! It was read from ITLB
55
56/* ITLB ** ICACHE line 3: Finish faults */
57sparc64_realfault_common: ! Called by dtlb_miss
58 stb %g4, [%g6 + TI_FAULT_CODE]
59 stx %g5, [%g6 + TI_FAULT_ADDR]
60 ba,pt %xcc, etrap ! Save state
611: rd %pc, %g7 ! ...
62 call do_sparc64_fault ! Call fault handler
63 add %sp, PTREGS_OFF, %o0! Compute pt_regs arg
64 ba,pt %xcc, rtrap_clr_l6 ! Restore cpu state
65 nop
66
67/* ITLB ** ICACHE line 4: Window fixups */
68winfix_trampoline:
69 rdpr %tpc, %g3 ! Prepare winfixup TNPC
70 or %g3, 0x7c, %g3 ! Compute branch offset
71 wrpr %g3, %tnpc ! Write it into TNPC
72 done ! Do it to it
73 nop
74 nop
75 nop
76 nop
77
78#undef CREATE_VPTE_OFFSET1
79#undef CREATE_VPTE_OFFSET2
diff --git a/arch/sparc64/kernel/itlb_miss.S b/arch/sparc64/kernel/itlb_miss.S
new file mode 100644
index 000000000000..6b6c8fee04bd
--- /dev/null
+++ b/arch/sparc64/kernel/itlb_miss.S
@@ -0,0 +1,39 @@
1/* ITLB ** ICACHE line 1: Context 0 check and TSB load */
2 ldxa [%g0] ASI_IMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer
3 ldxa [%g0] ASI_IMMU, %g6 ! Get TAG TARGET
4 srlx %g6, 48, %g5 ! Get context
5 brz,pn %g5, kvmap_itlb ! Context 0 processing
6 nop ! Delay slot (fill me)
7 ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB entry
8 cmp %g4, %g6 ! Compare TAG
9 sethi %hi(_PAGE_EXEC), %g4 ! Setup exec check
10
11/* ITLB ** ICACHE line 2: TSB compare and TLB load */
12 bne,pn %xcc, tsb_miss_itlb ! Miss
13 mov FAULT_CODE_ITLB, %g3
14 andcc %g5, %g4, %g0 ! Executable?
15 be,pn %xcc, tsb_do_fault
16 nop ! Delay slot, fill me
17 stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Load TLB
18 retry ! Trap done
19 nop
20
21/* ITLB ** ICACHE line 3: */
22 nop
23 nop
24 nop
25 nop
26 nop
27 nop
28 nop
29 nop
30
31/* ITLB ** ICACHE line 4: */
32 nop
33 nop
34 nop
35 nop
36 nop
37 nop
38 nop
39 nop
diff --git a/arch/sparc64/kernel/ktlb.S b/arch/sparc64/kernel/ktlb.S
index d9244d3c9f73..2b5e71b68882 100644
--- a/arch/sparc64/kernel/ktlb.S
+++ b/arch/sparc64/kernel/ktlb.S
@@ -4,191 +4,170 @@
4 * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de) 4 * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de)
5 * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) 5 * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx)
6 * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 6 * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
7*/ 7 */
8 8
9#include <linux/config.h> 9#include <linux/config.h>
10#include <asm/head.h> 10#include <asm/head.h>
11#include <asm/asi.h> 11#include <asm/asi.h>
12#include <asm/page.h> 12#include <asm/page.h>
13#include <asm/pgtable.h> 13#include <asm/pgtable.h>
14#include <asm/tsb.h>
14 15
15 .text 16 .text
16 .align 32 17 .align 32
17 18
18/* 19 .globl kvmap_itlb
19 * On a second level vpte miss, check whether the original fault is to the OBP 20kvmap_itlb:
20 * range (note that this is only possible for instruction miss, data misses to 21 /* g6: TAG TARGET */
21 * obp range do not use vpte). If so, go back directly to the faulting address. 22 mov TLB_TAG_ACCESS, %g4
22 * This is because we want to read the tpc, otherwise we have no way of knowing 23 ldxa [%g4] ASI_IMMU, %g4
23 * the 8k aligned faulting address if we are using >8k kernel pagesize. This 24
24 * also ensures no vpte range addresses are dropped into tlb while obp is 25kvmap_itlb_nonlinear:
25 * executing (see inherit_locked_prom_mappings() rant). 26 /* Catch kernel NULL pointer calls. */
26 */ 27 sethi %hi(PAGE_SIZE), %g5
27sparc64_vpte_nucleus: 28 cmp %g4, %g5
28 /* Note that kvmap below has verified that the address is 29 bleu,pn %xcc, kvmap_dtlb_longpath
29 * in the range MODULES_VADDR --> VMALLOC_END already. So 30 nop
30 * here we need only check if it is an OBP address or not. 31
31 */ 32 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
33
34kvmap_itlb_tsb_miss:
32 sethi %hi(LOW_OBP_ADDRESS), %g5 35 sethi %hi(LOW_OBP_ADDRESS), %g5
33 cmp %g4, %g5 36 cmp %g4, %g5
34 blu,pn %xcc, kern_vpte 37 blu,pn %xcc, kvmap_itlb_vmalloc_addr
35 mov 0x1, %g5 38 mov 0x1, %g5
36 sllx %g5, 32, %g5 39 sllx %g5, 32, %g5
37 cmp %g4, %g5 40 cmp %g4, %g5
38 blu,pn %xcc, vpte_insn_obp 41 blu,pn %xcc, kvmap_itlb_obp
39 nop 42 nop
40 43
41 /* These two instructions are patched by paginig_init(). */ 44kvmap_itlb_vmalloc_addr:
42kern_vpte: 45 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
43 sethi %hi(swapper_pgd_zero), %g5 46
44 lduw [%g5 + %lo(swapper_pgd_zero)], %g5 47 TSB_LOCK_TAG(%g1, %g2, %g4)
45 48
46 /* With kernel PGD in %g5, branch back into dtlb_backend. */ 49 /* Load and check PTE. */
47 ba,pt %xcc, sparc64_kpte_continue 50 ldxa [%g5] ASI_PHYS_USE_EC, %g5
48 andn %g1, 0x3, %g1 /* Finish PMD offset adjustment. */ 51 brgez,a,pn %g5, kvmap_itlb_longpath
49 52 stx %g0, [%g1]
50vpte_noent:
51 /* Restore previous TAG_ACCESS, %g5 is zero, and we will
52 * skip over the trap instruction so that the top level
53 * TLB miss handler will thing this %g5 value is just an
54 * invalid PTE, thus branching to full fault processing.
55 */
56 mov TLB_SFSR, %g1
57 stxa %g4, [%g1 + %g1] ASI_DMMU
58 done
59
60vpte_insn_obp:
61 /* Behave as if we are at TL0. */
62 wrpr %g0, 1, %tl
63 rdpr %tpc, %g4 /* Find original faulting iaddr */
64 srlx %g4, 13, %g4 /* Throw out context bits */
65 sllx %g4, 13, %g4 /* g4 has vpn + ctx0 now */
66
67 /* Restore previous TAG_ACCESS. */
68 mov TLB_SFSR, %g1
69 stxa %g4, [%g1 + %g1] ASI_IMMU
70
71 sethi %hi(prom_trans), %g5
72 or %g5, %lo(prom_trans), %g5
73
741: ldx [%g5 + 0x00], %g6 ! base
75 brz,a,pn %g6, longpath ! no more entries, fail
76 mov TLB_SFSR, %g1 ! and restore %g1
77 ldx [%g5 + 0x08], %g1 ! len
78 add %g6, %g1, %g1 ! end
79 cmp %g6, %g4
80 bgu,pt %xcc, 2f
81 cmp %g4, %g1
82 bgeu,pt %xcc, 2f
83 ldx [%g5 + 0x10], %g1 ! PTE
84
85 /* TLB load, restore %g1, and return from trap. */
86 sub %g4, %g6, %g6
87 add %g1, %g6, %g5
88 mov TLB_SFSR, %g1
89 stxa %g5, [%g0] ASI_ITLB_DATA_IN
90 retry
91 53
922: ba,pt %xcc, 1b 54 TSB_WRITE(%g1, %g5, %g6)
93 add %g5, (3 * 8), %g5 ! next entry 55
94 56 /* fallthrough to TLB load */
95kvmap_do_obp: 57
96 sethi %hi(prom_trans), %g5 58kvmap_itlb_load:
97 or %g5, %lo(prom_trans), %g5 59 stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Reload TLB
98 srlx %g4, 13, %g4
99 sllx %g4, 13, %g4
100
1011: ldx [%g5 + 0x00], %g6 ! base
102 brz,a,pn %g6, longpath ! no more entries, fail
103 mov TLB_SFSR, %g1 ! and restore %g1
104 ldx [%g5 + 0x08], %g1 ! len
105 add %g6, %g1, %g1 ! end
106 cmp %g6, %g4
107 bgu,pt %xcc, 2f
108 cmp %g4, %g1
109 bgeu,pt %xcc, 2f
110 ldx [%g5 + 0x10], %g1 ! PTE
111
112 /* TLB load, restore %g1, and return from trap. */
113 sub %g4, %g6, %g6
114 add %g1, %g6, %g5
115 mov TLB_SFSR, %g1
116 stxa %g5, [%g0] ASI_DTLB_DATA_IN
117 retry 60 retry
118 61
1192: ba,pt %xcc, 1b 62kvmap_itlb_longpath:
120 add %g5, (3 * 8), %g5 ! next entry 63 rdpr %pstate, %g5
64 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
65 rdpr %tpc, %g5
66 ba,pt %xcc, sparc64_realfault_common
67 mov FAULT_CODE_ITLB, %g4
68
69kvmap_itlb_obp:
70 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
71
72 TSB_LOCK_TAG(%g1, %g2, %g4)
73
74 TSB_WRITE(%g1, %g5, %g6)
75
76 ba,pt %xcc, kvmap_itlb_load
77 nop
78
79kvmap_dtlb_obp:
80 OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
81
82 TSB_LOCK_TAG(%g1, %g2, %g4)
83
84 TSB_WRITE(%g1, %g5, %g6)
85
86 ba,pt %xcc, kvmap_dtlb_load
87 nop
121 88
122/*
123 * On a first level data miss, check whether this is to the OBP range (note
124 * that such accesses can be made by prom, as well as by kernel using
125 * prom_getproperty on "address"), and if so, do not use vpte access ...
126 * rather, use information saved during inherit_prom_mappings() using 8k
127 * pagesize.
128 */
129 .align 32 89 .align 32
130kvmap: 90 .globl kvmap_dtlb
131 brgez,pn %g4, kvmap_nonlinear 91kvmap_dtlb:
92 /* %g6: TAG TARGET */
93 mov TLB_TAG_ACCESS, %g4
94 ldxa [%g4] ASI_DMMU, %g4
95 brgez,pn %g4, kvmap_dtlb_nonlinear
132 nop 96 nop
133 97
134#ifdef CONFIG_DEBUG_PAGEALLOC 98#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
99#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
100
101 sethi %uhi(KERN_HIGHBITS), %g2
102 or %g2, %ulo(KERN_HIGHBITS), %g2
103 sllx %g2, 32, %g2
104 or %g2, KERN_LOWBITS, %g2
105
106#undef KERN_HIGHBITS
107#undef KERN_LOWBITS
108
135 .globl kvmap_linear_patch 109 .globl kvmap_linear_patch
136kvmap_linear_patch: 110kvmap_linear_patch:
137#endif 111 ba,pt %xcc, kvmap_dtlb_load
138 ba,pt %xcc, kvmap_load
139 xor %g2, %g4, %g5 112 xor %g2, %g4, %g5
140 113
141#ifdef CONFIG_DEBUG_PAGEALLOC 114kvmap_dtlb_vmalloc_addr:
142 sethi %hi(swapper_pg_dir), %g5 115 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
143 or %g5, %lo(swapper_pg_dir), %g5 116
144 sllx %g4, 64 - (PGDIR_SHIFT + PGDIR_BITS), %g6 117 TSB_LOCK_TAG(%g1, %g2, %g4)
145 srlx %g6, 64 - PAGE_SHIFT, %g6 118
146 andn %g6, 0x3, %g6 119 /* Load and check PTE. */
147 lduw [%g5 + %g6], %g5 120 ldxa [%g5] ASI_PHYS_USE_EC, %g5
148 brz,pn %g5, longpath 121 brgez,a,pn %g5, kvmap_dtlb_longpath
149 sllx %g4, 64 - (PMD_SHIFT + PMD_BITS), %g6 122 stx %g0, [%g1]
150 srlx %g6, 64 - PAGE_SHIFT, %g6 123
151 sllx %g5, 11, %g5 124 TSB_WRITE(%g1, %g5, %g6)
152 andn %g6, 0x3, %g6 125
153 lduwa [%g5 + %g6] ASI_PHYS_USE_EC, %g5 126 /* fallthrough to TLB load */
154 brz,pn %g5, longpath 127
155 sllx %g4, 64 - PMD_SHIFT, %g6 128kvmap_dtlb_load:
156 srlx %g6, 64 - PAGE_SHIFT, %g6 129 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
157 sllx %g5, 11, %g5 130 retry
158 andn %g6, 0x7, %g6 131
159 ldxa [%g5 + %g6] ASI_PHYS_USE_EC, %g5 132kvmap_dtlb_nonlinear:
160 brz,pn %g5, longpath 133 /* Catch kernel NULL pointer derefs. */
134 sethi %hi(PAGE_SIZE), %g5
135 cmp %g4, %g5
136 bleu,pn %xcc, kvmap_dtlb_longpath
161 nop 137 nop
162 ba,a,pt %xcc, kvmap_load
163#endif
164 138
165kvmap_nonlinear: 139 KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
140
141kvmap_dtlb_tsbmiss:
166 sethi %hi(MODULES_VADDR), %g5 142 sethi %hi(MODULES_VADDR), %g5
167 cmp %g4, %g5 143 cmp %g4, %g5
168 blu,pn %xcc, longpath 144 blu,pn %xcc, kvmap_dtlb_longpath
169 mov (VMALLOC_END >> 24), %g5 145 mov (VMALLOC_END >> 24), %g5
170 sllx %g5, 24, %g5 146 sllx %g5, 24, %g5
171 cmp %g4, %g5 147 cmp %g4, %g5
172 bgeu,pn %xcc, longpath 148 bgeu,pn %xcc, kvmap_dtlb_longpath
173 nop 149 nop
174 150
175kvmap_check_obp: 151kvmap_check_obp:
176 sethi %hi(LOW_OBP_ADDRESS), %g5 152 sethi %hi(LOW_OBP_ADDRESS), %g5
177 cmp %g4, %g5 153 cmp %g4, %g5
178 blu,pn %xcc, kvmap_vmalloc_addr 154 blu,pn %xcc, kvmap_dtlb_vmalloc_addr
179 mov 0x1, %g5 155 mov 0x1, %g5
180 sllx %g5, 32, %g5 156 sllx %g5, 32, %g5
181 cmp %g4, %g5 157 cmp %g4, %g5
182 blu,pn %xcc, kvmap_do_obp 158 blu,pn %xcc, kvmap_dtlb_obp
183 nop 159 nop
184 160 ba,pt %xcc, kvmap_dtlb_vmalloc_addr
185kvmap_vmalloc_addr:
186 /* If we get here, a vmalloc addr was accessed, load kernel VPTE. */
187 ldxa [%g3 + %g6] ASI_N, %g5
188 brgez,pn %g5, longpath
189 nop 161 nop
190 162
191kvmap_load: 163kvmap_dtlb_longpath:
192 /* PTE is valid, load into TLB and return from trap. */ 164 rdpr %pstate, %g5
193 stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB 165 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
194 retry 166 rdpr %tl, %g4
167 cmp %g4, 1
168 mov TLB_TAG_ACCESS, %g4
169 ldxa [%g4] ASI_DMMU, %g5
170 be,pt %xcc, sparc64_realfault_common
171 mov FAULT_CODE_DTLB, %g4
172 ba,pt %xcc, winfix_trampoline
173 nop
diff --git a/arch/sparc64/kernel/process.c b/arch/sparc64/kernel/process.c
index 059b0d025224..2784aab0d3e5 100644
--- a/arch/sparc64/kernel/process.c
+++ b/arch/sparc64/kernel/process.c
@@ -44,6 +44,7 @@
44#include <asm/fpumacro.h> 44#include <asm/fpumacro.h>
45#include <asm/head.h> 45#include <asm/head.h>
46#include <asm/cpudata.h> 46#include <asm/cpudata.h>
47#include <asm/mmu_context.h>
47#include <asm/unistd.h> 48#include <asm/unistd.h>
48 49
49/* #define VERBOSE_SHOWREGS */ 50/* #define VERBOSE_SHOWREGS */
@@ -433,30 +434,16 @@ void exit_thread(void)
433void flush_thread(void) 434void flush_thread(void)
434{ 435{
435 struct thread_info *t = current_thread_info(); 436 struct thread_info *t = current_thread_info();
437 struct mm_struct *mm;
436 438
437 if (t->flags & _TIF_ABI_PENDING) 439 if (t->flags & _TIF_ABI_PENDING)
438 t->flags ^= (_TIF_ABI_PENDING | _TIF_32BIT); 440 t->flags ^= (_TIF_ABI_PENDING | _TIF_32BIT);
439 441
440 if (t->task->mm) { 442 mm = t->task->mm;
441 unsigned long pgd_cache = 0UL; 443 if (mm)
442 if (test_thread_flag(TIF_32BIT)) { 444 tsb_context_switch(__pa(mm->pgd),
443 struct mm_struct *mm = t->task->mm; 445 mm->context.sparc64_tsb);
444 pgd_t *pgd0 = &mm->pgd[0];
445 pud_t *pud0 = pud_offset(pgd0, 0);
446 446
447 if (pud_none(*pud0)) {
448 pmd_t *page = pmd_alloc_one(mm, 0);
449 pud_set(pud0, page);
450 }
451 pgd_cache = get_pgd_cache(pgd0);
452 }
453 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
454 "membar #Sync"
455 : /* no outputs */
456 : "r" (pgd_cache),
457 "r" (TSB_REG),
458 "i" (ASI_DMMU));
459 }
460 set_thread_wsaved(0); 447 set_thread_wsaved(0);
461 448
462 /* Turn off performance counters if on. */ 449 /* Turn off performance counters if on. */
diff --git a/arch/sparc64/kernel/rtrap.S b/arch/sparc64/kernel/rtrap.S
index b80eba0081ca..213eb4a9d8a4 100644
--- a/arch/sparc64/kernel/rtrap.S
+++ b/arch/sparc64/kernel/rtrap.S
@@ -223,10 +223,14 @@ rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1
223 ldx [%sp + PTREGS_OFF + PT_V9_G3], %g3 223 ldx [%sp + PTREGS_OFF + PT_V9_G3], %g3
224 ldx [%sp + PTREGS_OFF + PT_V9_G4], %g4 224 ldx [%sp + PTREGS_OFF + PT_V9_G4], %g4
225 ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5 225 ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5
226#ifdef CONFIG_SMP
227#error IMMU TSB usage must be fixed
226 mov TSB_REG, %g6 228 mov TSB_REG, %g6
227 brnz,a,pn %l3, 1f 229 brnz,a,pn %l3, 1f
228 ldxa [%g6] ASI_IMMU, %g5 230 ldxa [%g6] ASI_IMMU, %g5
2291: ldx [%sp + PTREGS_OFF + PT_V9_G6], %g6 231#endif
2321:
233 ldx [%sp + PTREGS_OFF + PT_V9_G6], %g6
230 ldx [%sp + PTREGS_OFF + PT_V9_G7], %g7 234 ldx [%sp + PTREGS_OFF + PT_V9_G7], %g7
231 wrpr %g0, RTRAP_PSTATE_AG_IRQOFF, %pstate 235 wrpr %g0, RTRAP_PSTATE_AG_IRQOFF, %pstate
232 ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0 236 ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0
diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c
index 1f7ad8a69052..d2d3369e7b5d 100644
--- a/arch/sparc64/kernel/smp.c
+++ b/arch/sparc64/kernel/smp.c
@@ -123,6 +123,7 @@ extern void inherit_locked_prom_mappings(int save_p);
123 123
124static inline void cpu_setup_percpu_base(unsigned long cpu_id) 124static inline void cpu_setup_percpu_base(unsigned long cpu_id)
125{ 125{
126#error IMMU TSB usage must be fixed
126 __asm__ __volatile__("mov %0, %%g5\n\t" 127 __asm__ __volatile__("mov %0, %%g5\n\t"
127 "stxa %0, [%1] %2\n\t" 128 "stxa %0, [%1] %2\n\t"
128 "membar #Sync" 129 "membar #Sync"
@@ -662,8 +663,6 @@ void smp_call_function_client(int irq, struct pt_regs *regs)
662extern unsigned long xcall_flush_tlb_mm; 663extern unsigned long xcall_flush_tlb_mm;
663extern unsigned long xcall_flush_tlb_pending; 664extern unsigned long xcall_flush_tlb_pending;
664extern unsigned long xcall_flush_tlb_kernel_range; 665extern unsigned long xcall_flush_tlb_kernel_range;
665extern unsigned long xcall_flush_tlb_all_spitfire;
666extern unsigned long xcall_flush_tlb_all_cheetah;
667extern unsigned long xcall_report_regs; 666extern unsigned long xcall_report_regs;
668extern unsigned long xcall_receive_signal; 667extern unsigned long xcall_receive_signal;
669 668
@@ -794,15 +793,6 @@ void smp_report_regs(void)
794 smp_cross_call(&xcall_report_regs, 0, 0, 0); 793 smp_cross_call(&xcall_report_regs, 0, 0, 0);
795} 794}
796 795
797void smp_flush_tlb_all(void)
798{
799 if (tlb_type == spitfire)
800 smp_cross_call(&xcall_flush_tlb_all_spitfire, 0, 0, 0);
801 else
802 smp_cross_call(&xcall_flush_tlb_all_cheetah, 0, 0, 0);
803 __flush_tlb_all();
804}
805
806/* We know that the window frames of the user have been flushed 796/* We know that the window frames of the user have been flushed
807 * to the stack before we get here because all callers of us 797 * to the stack before we get here because all callers of us
808 * are flush_tlb_*() routines, and these run after flush_cache_*() 798 * are flush_tlb_*() routines, and these run after flush_cache_*()
diff --git a/arch/sparc64/kernel/trampoline.S b/arch/sparc64/kernel/trampoline.S
index 9478551cb020..782d8c4973e4 100644
--- a/arch/sparc64/kernel/trampoline.S
+++ b/arch/sparc64/kernel/trampoline.S
@@ -295,39 +295,6 @@ do_unlock:
295 wrpr %g5, %tba 295 wrpr %g5, %tba
296 mov %o2, %g6 296 mov %o2, %g6
297 297
298 wrpr %o1, PSTATE_MG, %pstate
299#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
300#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)
301
302 mov TSB_REG, %g1
303 stxa %g0, [%g1] ASI_DMMU
304 membar #Sync
305 mov TLB_SFSR, %g1
306 sethi %uhi(KERN_HIGHBITS), %g2
307 or %g2, %ulo(KERN_HIGHBITS), %g2
308 sllx %g2, 32, %g2
309 or %g2, KERN_LOWBITS, %g2
310
311 BRANCH_IF_ANY_CHEETAH(g3,g7,9f)
312
313 ba,pt %xcc, 1f
314 nop
315
3169:
317 sethi %uhi(VPTE_BASE_CHEETAH), %g3
318 or %g3, %ulo(VPTE_BASE_CHEETAH), %g3
319 ba,pt %xcc, 2f
320 sllx %g3, 32, %g3
3211:
322 sethi %uhi(VPTE_BASE_SPITFIRE), %g3
323 or %g3, %ulo(VPTE_BASE_SPITFIRE), %g3
324 sllx %g3, 32, %g3
325
3262:
327 clr %g7
328#undef KERN_HIGHBITS
329#undef KERN_LOWBITS
330
331 wrpr %o1, 0x0, %pstate 298 wrpr %o1, 0x0, %pstate
332 ldx [%g6 + TI_TASK], %g4 299 ldx [%g6 + TI_TASK], %g4
333 300
diff --git a/arch/sparc64/kernel/tsb.S b/arch/sparc64/kernel/tsb.S
new file mode 100644
index 000000000000..44b9e6fed09f
--- /dev/null
+++ b/arch/sparc64/kernel/tsb.S
@@ -0,0 +1,169 @@
1/* tsb.S: Sparc64 TSB table handling.
2 *
3 * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
4 */
5
6#include <asm/tsb.h>
7
8 .text
9 .align 32
10
11 /* Invoked from TLB miss handler, we are in the
12 * MMU global registers and they are setup like
13 * this:
14 *
15 * %g1: TSB entry pointer
16 * %g2: available temporary
17 * %g3: FAULT_CODE_{D,I}TLB
18 * %g4: available temporary
19 * %g5: available temporary
20 * %g6: TAG TARGET
21 * %g7: physical address base of the linux page
22 * tables for the current address space
23 */
24 .globl tsb_miss_dtlb
25tsb_miss_dtlb:
26 mov TLB_TAG_ACCESS, %g4
27 ldxa [%g4] ASI_DMMU, %g4
28 ba,pt %xcc, tsb_miss_page_table_walk
29 nop
30
31 .globl tsb_miss_itlb
32tsb_miss_itlb:
33 mov TLB_TAG_ACCESS, %g4
34 ldxa [%g4] ASI_IMMU, %g4
35 ba,pt %xcc, tsb_miss_page_table_walk
36 nop
37
38tsb_miss_page_table_walk:
39 USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault)
40
41tsb_reload:
42 TSB_LOCK_TAG(%g1, %g2, %g4)
43
44 /* Load and check PTE. */
45 ldxa [%g5] ASI_PHYS_USE_EC, %g5
46 brgez,a,pn %g5, tsb_do_fault
47 stx %g0, [%g1]
48
49 TSB_WRITE(%g1, %g5, %g6)
50
51 /* Finally, load TLB and return from trap. */
52tsb_tlb_reload:
53 cmp %g3, FAULT_CODE_DTLB
54 bne,pn %xcc, tsb_itlb_load
55 nop
56
57tsb_dtlb_load:
58 stxa %g5, [%g0] ASI_DTLB_DATA_IN
59 retry
60
61tsb_itlb_load:
62 stxa %g5, [%g0] ASI_ITLB_DATA_IN
63 retry
64
65 /* No valid entry in the page tables, do full fault
66 * processing.
67 */
68
69 .globl tsb_do_fault
70tsb_do_fault:
71 cmp %g3, FAULT_CODE_DTLB
72 rdpr %pstate, %g5
73 bne,pn %xcc, tsb_do_itlb_fault
74 wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate
75
76tsb_do_dtlb_fault:
77 rdpr %tl, %g4
78 cmp %g4, 1
79 mov TLB_TAG_ACCESS, %g4
80 ldxa [%g4] ASI_DMMU, %g5
81 be,pt %xcc, sparc64_realfault_common
82 mov FAULT_CODE_DTLB, %g4
83 ba,pt %xcc, winfix_trampoline
84 nop
85
86tsb_do_itlb_fault:
87 rdpr %tpc, %g5
88 ba,pt %xcc, sparc64_realfault_common
89 mov FAULT_CODE_ITLB, %g4
90
91 .globl sparc64_realfault_common
92sparc64_realfault_common:
93 stb %g4, [%g6 + TI_FAULT_CODE] ! Save fault code
94 stx %g5, [%g6 + TI_FAULT_ADDR] ! Save fault address
95 ba,pt %xcc, etrap ! Save trap state
961: rd %pc, %g7 ! ...
97 call do_sparc64_fault ! Call fault handler
98 add %sp, PTREGS_OFF, %o0 ! Compute pt_regs arg
99 ba,pt %xcc, rtrap_clr_l6 ! Restore cpu state
100 nop ! Delay slot (fill me)
101
102 .globl winfix_trampoline
103winfix_trampoline:
104 rdpr %tpc, %g3 ! Prepare winfixup TNPC
105 or %g3, 0x7c, %g3 ! Compute branch offset
106 wrpr %g3, %tnpc ! Write it into TNPC
107 done ! Trap return
108
109 /* Reload MMU related context switch state at
110 * schedule() time.
111 *
112 * %o0: page table physical address
113 * %o1: TSB address
114 */
115 .globl tsb_context_switch
116tsb_context_switch:
117 wrpr %g0, PSTATE_MG | PSTATE_RMO | PSTATE_PEF | PSTATE_PRIV, %pstate
118
119 /* Set page table base alternate global. */
120 mov %o0, %g7
121
122 /* XXX can this happen? */
123 brz,pn %o1, 9f
124 nop
125
126 /* Lock TSB into D-TLB. */
127 sethi %hi(PAGE_SIZE), %o3
128 and %o3, %o1, %o3
129 sethi %hi(TSBMAP_BASE), %o2
130 add %o2, %o3, %o2
131
132 /* XXX handle PAGE_SIZE != 8K correctly... */
133 mov TSB_REG, %g1
134 stxa %o2, [%g1] ASI_DMMU
135 membar #Sync
136
137 stxa %o2, [%g1] ASI_IMMU
138 membar #Sync
139
140#define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZBITS)^0xfffff80000000000)
141#define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W | _PAGE_L)
142 sethi %uhi(KERN_HIGHBITS), %g2
143 or %g2, %ulo(KERN_HIGHBITS), %g2
144 sllx %g2, 32, %g2
145 or %g2, KERN_LOWBITS, %g2
146#undef KERN_HIGHBITS
147#undef KERN_LOWBITS
148
149 xor %o1, %g2, %o1
150
151 /* We use entry 61 for this locked entry. This is the spitfire
152 * TLB entry number, and luckily cheetah masks the value with
153 * 15 ending us up with entry 13 which is what we want in that
154 * case too.
155 *
156 * XXX Interactions with prom_world()...
157 */
158 mov TLB_TAG_ACCESS, %g1
159 stxa %o2, [%g1] ASI_DMMU
160 membar #Sync
161 mov (61 << 3), %g1
162 stxa %o1, [%g1] ASI_DTLB_DATA_ACCESS
163 membar #Sync
164
1659:
166 wrpr %g0, PSTATE_RMO | PSTATE_PEF | PSTATE_PRIV | PSTATE_IE, %pstate
167
168 retl
169 mov %o2, %o0
diff --git a/arch/sparc64/kernel/ttable.S b/arch/sparc64/kernel/ttable.S
index 8365bc1f81f3..56f060c8fbf0 100644
--- a/arch/sparc64/kernel/ttable.S
+++ b/arch/sparc64/kernel/ttable.S
@@ -78,9 +78,9 @@ tl0_vaw: TRAP(do_vaw)
78tl0_cee: membar #Sync 78tl0_cee: membar #Sync
79 TRAP_NOSAVE_7INSNS(__spitfire_cee_trap) 79 TRAP_NOSAVE_7INSNS(__spitfire_cee_trap)
80tl0_iamiss: 80tl0_iamiss:
81#include "itlb_base.S" 81#include "itlb_miss.S"
82tl0_damiss: 82tl0_damiss:
83#include "dtlb_base.S" 83#include "dtlb_miss.S"
84tl0_daprot: 84tl0_daprot:
85#include "dtlb_prot.S" 85#include "dtlb_prot.S"
86tl0_fecc: BTRAP(0x70) /* Fast-ECC on Cheetah */ 86tl0_fecc: BTRAP(0x70) /* Fast-ECC on Cheetah */
@@ -241,7 +241,7 @@ tl1_cee: membar #Sync
241 241
242tl1_iamiss: BTRAPTL1(0x64) BTRAPTL1(0x65) BTRAPTL1(0x66) BTRAPTL1(0x67) 242tl1_iamiss: BTRAPTL1(0x64) BTRAPTL1(0x65) BTRAPTL1(0x66) BTRAPTL1(0x67)
243tl1_damiss: 243tl1_damiss:
244#include "dtlb_backend.S" 244#include "dtlb_miss.S"
245tl1_daprot: 245tl1_daprot:
246#include "dtlb_prot.S" 246#include "dtlb_prot.S"
247tl1_fecc: BTRAPTL1(0x70) /* Fast-ECC on Cheetah */ 247tl1_fecc: BTRAPTL1(0x70) /* Fast-ECC on Cheetah */
diff --git a/arch/sparc64/kernel/vmlinux.lds.S b/arch/sparc64/kernel/vmlinux.lds.S
index 467d13a0d5c1..f018aaf45486 100644
--- a/arch/sparc64/kernel/vmlinux.lds.S
+++ b/arch/sparc64/kernel/vmlinux.lds.S
@@ -44,6 +44,9 @@ SECTIONS
44 __stop___ex_table = .; 44 __stop___ex_table = .;
45 45
46 . = ALIGN(8192); 46 . = ALIGN(8192);
47 swapper_tsb = .;
48 . += 8192;
49 . = ALIGN(8192);
47 __init_begin = .; 50 __init_begin = .;
48 .init.text : { 51 .init.text : {
49 _sinittext = .; 52 _sinittext = .;
diff --git a/arch/sparc64/kernel/winfixup.S b/arch/sparc64/kernel/winfixup.S
index 39160926267b..f5d93aa99cbb 100644
--- a/arch/sparc64/kernel/winfixup.S
+++ b/arch/sparc64/kernel/winfixup.S
@@ -85,6 +85,7 @@ fill_fixup:
85 mov %o7, %g6 85 mov %o7, %g6
86 ldx [%g6 + TI_TASK], %g4 86 ldx [%g6 + TI_TASK], %g4
87#ifdef CONFIG_SMP 87#ifdef CONFIG_SMP
88#error IMMU TSB usage must be fixed
88 mov TSB_REG, %g1 89 mov TSB_REG, %g1
89 ldxa [%g1] ASI_IMMU, %g5 90 ldxa [%g1] ASI_IMMU, %g5
90#endif 91#endif
@@ -209,6 +210,7 @@ fill_fixup_mna:
209 mov %o7, %g6 ! Get current back. 210 mov %o7, %g6 ! Get current back.
210 ldx [%g6 + TI_TASK], %g4 ! Finish it. 211 ldx [%g6 + TI_TASK], %g4 ! Finish it.
211#ifdef CONFIG_SMP 212#ifdef CONFIG_SMP
213#error IMMU TSB usage must be fixed
212 mov TSB_REG, %g1 214 mov TSB_REG, %g1
213 ldxa [%g1] ASI_IMMU, %g5 215 ldxa [%g1] ASI_IMMU, %g5
214#endif 216#endif
@@ -278,11 +280,6 @@ window_mna_from_user_common:
278 ba,pt %xcc, rtrap 280 ba,pt %xcc, rtrap
279 clr %l6 281 clr %l6
280 282
281 /* These are only needed for 64-bit mode processes which
282 * put their stack pointer into the VPTE area and there
283 * happens to be a VPTE tlb entry mapped there during
284 * a spill/fill trap to that stack frame.
285 */
286 .globl winfix_dax, fill_fixup_dax, spill_fixup_dax 283 .globl winfix_dax, fill_fixup_dax, spill_fixup_dax
287winfix_dax: 284winfix_dax:
288 andn %g3, 0x7f, %g3 285 andn %g3, 0x7f, %g3
@@ -318,6 +315,7 @@ fill_fixup_dax:
318 mov %o7, %g6 ! Get current back. 315 mov %o7, %g6 ! Get current back.
319 ldx [%g6 + TI_TASK], %g4 ! Finish it. 316 ldx [%g6 + TI_TASK], %g4 ! Finish it.
320#ifdef CONFIG_SMP 317#ifdef CONFIG_SMP
318#error IMMU TSB usage must be fixed
321 mov TSB_REG, %g1 319 mov TSB_REG, %g1
322 ldxa [%g1] ASI_IMMU, %g5 320 ldxa [%g1] ASI_IMMU, %g5
323#endif 321#endif
diff --git a/arch/sparc64/mm/Makefile b/arch/sparc64/mm/Makefile
index 9d0960e69f48..e415bf942bcd 100644
--- a/arch/sparc64/mm/Makefile
+++ b/arch/sparc64/mm/Makefile
@@ -5,6 +5,6 @@
5EXTRA_AFLAGS := -ansi 5EXTRA_AFLAGS := -ansi
6EXTRA_CFLAGS := -Werror 6EXTRA_CFLAGS := -Werror
7 7
8obj-y := ultra.o tlb.o fault.o init.o generic.o 8obj-y := ultra.o tlb.o tsb.o fault.o init.o generic.o
9 9
10obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o 10obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
index 1e44ee26cee8..da068f6b2595 100644
--- a/arch/sparc64/mm/init.c
+++ b/arch/sparc64/mm/init.c
@@ -408,8 +408,7 @@ unsigned long prom_virt_to_phys(unsigned long promva, int *error)
408 408
409/* The obp translations are saved based on 8k pagesize, since obp can 409/* The obp translations are saved based on 8k pagesize, since obp can
410 * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS -> 410 * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
411 * HI_OBP_ADDRESS range are handled in ktlb.S and do not use the vpte 411 * HI_OBP_ADDRESS range are handled in ktlb.S.
412 * scheme (also, see rant in inherit_locked_prom_mappings()).
413 */ 412 */
414static inline int in_obp_range(unsigned long vaddr) 413static inline int in_obp_range(unsigned long vaddr)
415{ 414{
@@ -539,75 +538,6 @@ static void __init inherit_prom_mappings(void)
539 prom_printf("done.\n"); 538 prom_printf("done.\n");
540} 539}
541 540
542/* The OBP specifications for sun4u mark 0xfffffffc00000000 and
543 * upwards as reserved for use by the firmware (I wonder if this
544 * will be the same on Cheetah...). We use this virtual address
545 * range for the VPTE table mappings of the nucleus so we need
546 * to zap them when we enter the PROM. -DaveM
547 */
548static void __flush_nucleus_vptes(void)
549{
550 unsigned long prom_reserved_base = 0xfffffffc00000000UL;
551 int i;
552
553 /* Only DTLB must be checked for VPTE entries. */
554 if (tlb_type == spitfire) {
555 for (i = 0; i < 63; i++) {
556 unsigned long tag;
557
558 /* Spitfire Errata #32 workaround */
559 /* NOTE: Always runs on spitfire, so no cheetah+
560 * page size encodings.
561 */
562 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
563 "flush %%g6"
564 : /* No outputs */
565 : "r" (0),
566 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
567
568 tag = spitfire_get_dtlb_tag(i);
569 if (((tag & ~(PAGE_MASK)) == 0) &&
570 ((tag & (PAGE_MASK)) >= prom_reserved_base)) {
571 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
572 "membar #Sync"
573 : /* no outputs */
574 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
575 spitfire_put_dtlb_data(i, 0x0UL);
576 }
577 }
578 } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
579 for (i = 0; i < 512; i++) {
580 unsigned long tag = cheetah_get_dtlb_tag(i, 2);
581
582 if ((tag & ~PAGE_MASK) == 0 &&
583 (tag & PAGE_MASK) >= prom_reserved_base) {
584 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
585 "membar #Sync"
586 : /* no outputs */
587 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
588 cheetah_put_dtlb_data(i, 0x0UL, 2);
589 }
590
591 if (tlb_type != cheetah_plus)
592 continue;
593
594 tag = cheetah_get_dtlb_tag(i, 3);
595
596 if ((tag & ~PAGE_MASK) == 0 &&
597 (tag & PAGE_MASK) >= prom_reserved_base) {
598 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
599 "membar #Sync"
600 : /* no outputs */
601 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
602 cheetah_put_dtlb_data(i, 0x0UL, 3);
603 }
604 }
605 } else {
606 /* Implement me :-) */
607 BUG();
608 }
609}
610
611static int prom_ditlb_set; 541static int prom_ditlb_set;
612struct prom_tlb_entry { 542struct prom_tlb_entry {
613 int tlb_ent; 543 int tlb_ent;
@@ -635,9 +565,6 @@ void prom_world(int enter)
635 : "i" (PSTATE_IE)); 565 : "i" (PSTATE_IE));
636 566
637 if (enter) { 567 if (enter) {
638 /* Kick out nucleus VPTEs. */
639 __flush_nucleus_vptes();
640
641 /* Install PROM world. */ 568 /* Install PROM world. */
642 for (i = 0; i < 16; i++) { 569 for (i = 0; i < 16; i++) {
643 if (prom_dtlb[i].tlb_ent != -1) { 570 if (prom_dtlb[i].tlb_ent != -1) {
@@ -1039,18 +966,7 @@ out:
1039struct pgtable_cache_struct pgt_quicklists; 966struct pgtable_cache_struct pgt_quicklists;
1040#endif 967#endif
1041 968
1042/* OK, we have to color these pages. The page tables are accessed 969/* XXX We don't need to color these things in the D-cache any longer. */
1043 * by non-Dcache enabled mapping in the VPTE area by the dtlb_backend.S
1044 * code, as well as by PAGE_OFFSET range direct-mapped addresses by
1045 * other parts of the kernel. By coloring, we make sure that the tlbmiss
1046 * fast handlers do not get data from old/garbage dcache lines that
1047 * correspond to an old/stale virtual address (user/kernel) that
1048 * previously mapped the pagetable page while accessing vpte range
1049 * addresses. The idea is that if the vpte color and PAGE_OFFSET range
1050 * color is the same, then when the kernel initializes the pagetable
1051 * using the later address range, accesses with the first address
1052 * range will see the newly initialized data rather than the garbage.
1053 */
1054#ifdef DCACHE_ALIASING_POSSIBLE 970#ifdef DCACHE_ALIASING_POSSIBLE
1055#define DC_ALIAS_SHIFT 1 971#define DC_ALIAS_SHIFT 1
1056#else 972#else
@@ -1419,6 +1335,9 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
1419 kernel_map_range(phys_start, phys_end, 1335 kernel_map_range(phys_start, phys_end,
1420 (enable ? PAGE_KERNEL : __pgprot(0))); 1336 (enable ? PAGE_KERNEL : __pgprot(0)));
1421 1337
1338 flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
1339 PAGE_OFFSET + phys_end);
1340
1422 /* we should perform an IPI and flush all tlbs, 1341 /* we should perform an IPI and flush all tlbs,
1423 * but that can deadlock->flush only current cpu. 1342 * but that can deadlock->flush only current cpu.
1424 */ 1343 */
diff --git a/arch/sparc64/mm/tlb.c b/arch/sparc64/mm/tlb.c
index 8b104be4662b..78357cc2a0b7 100644
--- a/arch/sparc64/mm/tlb.c
+++ b/arch/sparc64/mm/tlb.c
@@ -25,6 +25,8 @@ void flush_tlb_pending(void)
25 struct mmu_gather *mp = &__get_cpu_var(mmu_gathers); 25 struct mmu_gather *mp = &__get_cpu_var(mmu_gathers);
26 26
27 if (mp->tlb_nr) { 27 if (mp->tlb_nr) {
28 flush_tsb_user(mp);
29
28 if (CTX_VALID(mp->mm->context)) { 30 if (CTX_VALID(mp->mm->context)) {
29#ifdef CONFIG_SMP 31#ifdef CONFIG_SMP
30 smp_flush_tlb_pending(mp->mm, mp->tlb_nr, 32 smp_flush_tlb_pending(mp->mm, mp->tlb_nr,
@@ -89,62 +91,3 @@ no_cache_flush:
89 if (nr >= TLB_BATCH_NR) 91 if (nr >= TLB_BATCH_NR)
90 flush_tlb_pending(); 92 flush_tlb_pending();
91} 93}
92
93void flush_tlb_pgtables(struct mm_struct *mm, unsigned long start, unsigned long end)
94{
95 struct mmu_gather *mp = &__get_cpu_var(mmu_gathers);
96 unsigned long nr = mp->tlb_nr;
97 long s = start, e = end, vpte_base;
98
99 if (mp->fullmm)
100 return;
101
102 /* If start is greater than end, that is a real problem. */
103 BUG_ON(start > end);
104
105 /* However, straddling the VA space hole is quite normal. */
106 s &= PMD_MASK;
107 e = (e + PMD_SIZE - 1) & PMD_MASK;
108
109 vpte_base = (tlb_type == spitfire ?
110 VPTE_BASE_SPITFIRE :
111 VPTE_BASE_CHEETAH);
112
113 if (unlikely(nr != 0 && mm != mp->mm)) {
114 flush_tlb_pending();
115 nr = 0;
116 }
117
118 if (nr == 0)
119 mp->mm = mm;
120
121 start = vpte_base + (s >> (PAGE_SHIFT - 3));
122 end = vpte_base + (e >> (PAGE_SHIFT - 3));
123
124 /* If the request straddles the VA space hole, we
125 * need to swap start and end. The reason this
126 * occurs is that "vpte_base" is the center of
127 * the linear page table mapping area. Thus,
128 * high addresses with the sign bit set map to
129 * addresses below vpte_base and non-sign bit
130 * addresses map to addresses above vpte_base.
131 */
132 if (end < start) {
133 unsigned long tmp = start;
134
135 start = end;
136 end = tmp;
137 }
138
139 while (start < end) {
140 mp->vaddrs[nr] = start;
141 mp->tlb_nr = ++nr;
142 if (nr >= TLB_BATCH_NR) {
143 flush_tlb_pending();
144 nr = 0;
145 }
146 start += PAGE_SIZE;
147 }
148 if (nr)
149 flush_tlb_pending();
150}
diff --git a/arch/sparc64/mm/tsb.c b/arch/sparc64/mm/tsb.c
new file mode 100644
index 000000000000..15e8af58b1d2
--- /dev/null
+++ b/arch/sparc64/mm/tsb.c
@@ -0,0 +1,84 @@
1/* arch/sparc64/mm/tsb.c
2 *
3 * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
4 */
5
6#include <linux/kernel.h>
7#include <asm/system.h>
8#include <asm/page.h>
9#include <asm/tlbflush.h>
10#include <asm/tlb.h>
11
12#define TSB_ENTRY_ALIGNMENT 16
13
14struct tsb {
15 unsigned long tag;
16 unsigned long pte;
17} __attribute__((aligned(TSB_ENTRY_ALIGNMENT)));
18
19/* We use an 8K TSB for the whole kernel, this allows to
20 * handle about 4MB of modules and vmalloc mappings without
21 * incurring many hash conflicts.
22 */
23#define KERNEL_TSB_SIZE_BYTES 8192
24#define KERNEL_TSB_NENTRIES \
25 (KERNEL_TSB_SIZE_BYTES / sizeof(struct tsb))
26
27extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
28
29static inline unsigned long tsb_hash(unsigned long vaddr)
30{
31 vaddr >>= PAGE_SHIFT;
32 return vaddr & (KERNEL_TSB_NENTRIES - 1);
33}
34
35static inline int tag_compare(struct tsb *entry, unsigned long vaddr, unsigned long context)
36{
37 if (context == ~0UL)
38 return 1;
39
40 return (entry->tag == ((vaddr >> 22) | (context << 48)));
41}
42
43/* TSB flushes need only occur on the processor initiating the address
44 * space modification, not on each cpu the address space has run on.
45 * Only the TLB flush needs that treatment.
46 */
47
48void flush_tsb_kernel_range(unsigned long start, unsigned long end)
49{
50 unsigned long v;
51
52 for (v = start; v < end; v += PAGE_SIZE) {
53 struct tsb *ent = &swapper_tsb[tsb_hash(v)];
54
55 if (tag_compare(ent, v, 0)) {
56 ent->tag = 0UL;
57 membar_storeload_storestore();
58 }
59 }
60}
61
62void flush_tsb_user(struct mmu_gather *mp)
63{
64 struct mm_struct *mm = mp->mm;
65 struct tsb *tsb = (struct tsb *) mm->context.sparc64_tsb;
66 unsigned long ctx = ~0UL;
67 int i;
68
69 if (CTX_VALID(mm->context))
70 ctx = CTX_HWBITS(mm->context);
71
72 for (i = 0; i < mp->tlb_nr; i++) {
73 unsigned long v = mp->vaddrs[i];
74 struct tsb *ent;
75
76 v &= ~0x1UL;
77
78 ent = &tsb[tsb_hash(v)];
79 if (tag_compare(ent, v, ctx)) {
80 ent->tag = 0UL;
81 membar_storeload_storestore();
82 }
83 }
84}
diff --git a/arch/sparc64/mm/ultra.S b/arch/sparc64/mm/ultra.S
index e4c9151fa116..22791f29552e 100644
--- a/arch/sparc64/mm/ultra.S
+++ b/arch/sparc64/mm/ultra.S
@@ -453,64 +453,6 @@ xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
453 nop 453 nop
454 nop 454 nop
455 455
456 .data
457
458errata32_hwbug:
459 .xword 0
460
461 .text
462
463 /* These two are not performance critical... */
464 .globl xcall_flush_tlb_all_spitfire
465xcall_flush_tlb_all_spitfire:
466 /* Spitfire Errata #32 workaround. */
467 sethi %hi(errata32_hwbug), %g4
468 stx %g0, [%g4 + %lo(errata32_hwbug)]
469
470 clr %g2
471 clr %g3
4721: ldxa [%g3] ASI_DTLB_DATA_ACCESS, %g4
473 and %g4, _PAGE_L, %g5
474 brnz,pn %g5, 2f
475 mov TLB_TAG_ACCESS, %g7
476
477 stxa %g0, [%g7] ASI_DMMU
478 membar #Sync
479 stxa %g0, [%g3] ASI_DTLB_DATA_ACCESS
480 membar #Sync
481
482 /* Spitfire Errata #32 workaround. */
483 sethi %hi(errata32_hwbug), %g4
484 stx %g0, [%g4 + %lo(errata32_hwbug)]
485
4862: ldxa [%g3] ASI_ITLB_DATA_ACCESS, %g4
487 and %g4, _PAGE_L, %g5
488 brnz,pn %g5, 2f
489 mov TLB_TAG_ACCESS, %g7
490
491 stxa %g0, [%g7] ASI_IMMU
492 membar #Sync
493 stxa %g0, [%g3] ASI_ITLB_DATA_ACCESS
494 membar #Sync
495
496 /* Spitfire Errata #32 workaround. */
497 sethi %hi(errata32_hwbug), %g4
498 stx %g0, [%g4 + %lo(errata32_hwbug)]
499
5002: add %g2, 1, %g2
501 cmp %g2, SPITFIRE_HIGHEST_LOCKED_TLBENT
502 ble,pt %icc, 1b
503 sll %g2, 3, %g3
504 flush %g6
505 retry
506
507 .globl xcall_flush_tlb_all_cheetah
508xcall_flush_tlb_all_cheetah:
509 mov 0x80, %g2
510 stxa %g0, [%g2] ASI_DMMU_DEMAP
511 stxa %g0, [%g2] ASI_IMMU_DEMAP
512 retry
513
514 /* These just get rescheduled to PIL vectors. */ 456 /* These just get rescheduled to PIL vectors. */
515 .globl xcall_call_function 457 .globl xcall_call_function
516xcall_call_function: 458xcall_call_function:
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-26 22:22:07 -0400