diff options
author | David S. Miller <davem@davemloft.net> | 2006-01-31 21:31:20 -0500 |
---|---|---|
committer | David S. Miller <davem@sunset.davemloft.net> | 2006-03-20 04:11:17 -0500 |
commit | 98c5584cfc47932c4f3ccf5eee2e0bae1447b85e (patch) | |
tree | c067ac8bfc081bbe0b3073374cb15708458e04ab /arch/sparc64 | |
parent | 09f94287f7260e03bbeab497e743691fafcc22c3 (diff) |
[SPARC64]: Add infrastructure for dynamic TSB sizing.
This also cleans up tsb_context_switch(). The assembler
routine is now __tsb_context_switch() and the former is
an inline function that picks out the bits from the mm_struct
and passes it into the assembler code as arguments.
setup_tsb_parms() computes the locked TLB entry to map the
TSB. Later when we support using the physical address quad
load instructions of Cheetah+ and later, we'll simply use
the physical address for the TSB register value and set
the map virtual and PTE both to zero.
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'arch/sparc64')
-rw-r--r-- | arch/sparc64/kernel/binfmt_aout32.c | 3 | ||||
-rw-r--r-- | arch/sparc64/kernel/process.c | 3 | ||||
-rw-r--r-- | arch/sparc64/kernel/tsb.S | 55 | ||||
-rw-r--r-- | arch/sparc64/mm/tsb.c | 109 |
4 files changed, 118 insertions, 52 deletions
diff --git a/arch/sparc64/kernel/binfmt_aout32.c b/arch/sparc64/kernel/binfmt_aout32.c index a57d7f2b6f13..181c8cdf9541 100644 --- a/arch/sparc64/kernel/binfmt_aout32.c +++ b/arch/sparc64/kernel/binfmt_aout32.c | |||
@@ -330,8 +330,7 @@ beyond_if: | |||
330 | 330 | ||
331 | current->mm->start_stack = | 331 | current->mm->start_stack = |
332 | (unsigned long) create_aout32_tables((char __user *)bprm->p, bprm); | 332 | (unsigned long) create_aout32_tables((char __user *)bprm->p, bprm); |
333 | tsb_context_switch(__pa(current->mm->pgd), | 333 | tsb_context_switch(mm); |
334 | current->mm->context.sparc64_tsb); | ||
335 | 334 | ||
336 | start_thread32(regs, ex.a_entry, current->mm->start_stack); | 335 | start_thread32(regs, ex.a_entry, current->mm->start_stack); |
337 | if (current->ptrace & PT_PTRACED) | 336 | if (current->ptrace & PT_PTRACED) |
diff --git a/arch/sparc64/kernel/process.c b/arch/sparc64/kernel/process.c index 2784aab0d3e5..26548fc604b6 100644 --- a/arch/sparc64/kernel/process.c +++ b/arch/sparc64/kernel/process.c | |||
@@ -441,8 +441,7 @@ void flush_thread(void) | |||
441 | 441 | ||
442 | mm = t->task->mm; | 442 | mm = t->task->mm; |
443 | if (mm) | 443 | if (mm) |
444 | tsb_context_switch(__pa(mm->pgd), | 444 | tsb_context_switch(mm); |
445 | mm->context.sparc64_tsb); | ||
446 | 445 | ||
447 | set_thread_wsaved(0); | 446 | set_thread_wsaved(0); |
448 | 447 | ||
diff --git a/arch/sparc64/kernel/tsb.S b/arch/sparc64/kernel/tsb.S index 76f2c0b01f36..fe266bad0a28 100644 --- a/arch/sparc64/kernel/tsb.S +++ b/arch/sparc64/kernel/tsb.S | |||
@@ -130,48 +130,36 @@ winfix_trampoline: | |||
130 | * schedule() time. | 130 | * schedule() time. |
131 | * | 131 | * |
132 | * %o0: page table physical address | 132 | * %o0: page table physical address |
133 | * %o1: TSB address | 133 | * %o1: TSB register value |
134 | * %o2: TSB virtual address | ||
135 | * %o3: TSB mapping locked PTE | ||
136 | * | ||
137 | * We have to run this whole thing with interrupts | ||
138 | * disabled so that the current cpu doesn't change | ||
139 | * due to preemption. | ||
134 | */ | 140 | */ |
135 | .align 32 | 141 | .align 32 |
136 | .globl tsb_context_switch | 142 | .globl __tsb_context_switch |
137 | tsb_context_switch: | 143 | __tsb_context_switch: |
138 | rdpr %pstate, %o5 | 144 | rdpr %pstate, %o5 |
139 | wrpr %o5, PSTATE_IE, %pstate | 145 | wrpr %o5, PSTATE_IE, %pstate |
140 | 146 | ||
141 | ldub [%g6 + TI_CPU], %o3 | 147 | ldub [%g6 + TI_CPU], %g1 |
142 | sethi %hi(trap_block), %o4 | 148 | sethi %hi(trap_block), %g2 |
143 | sllx %o3, TRAP_BLOCK_SZ_SHIFT, %o3 | 149 | sllx %g1, TRAP_BLOCK_SZ_SHIFT, %g1 |
144 | or %o4, %lo(trap_block), %o4 | 150 | or %g2, %lo(trap_block), %g2 |
145 | add %o4, %o3, %o4 | 151 | add %g2, %g1, %g2 |
146 | stx %o0, [%o4 + TRAP_PER_CPU_PGD_PADDR] | 152 | stx %o0, [%g2 + TRAP_PER_CPU_PGD_PADDR] |
147 | |||
148 | brgez %o1, 9f | ||
149 | nop | ||
150 | |||
151 | /* Lock TSB into D-TLB. */ | ||
152 | sethi %hi(PAGE_SIZE), %o3 | ||
153 | and %o3, %o1, %o3 | ||
154 | sethi %hi(TSBMAP_BASE), %o2 | ||
155 | add %o2, %o3, %o2 | ||
156 | 153 | ||
157 | /* XXX handle PAGE_SIZE != 8K correctly... */ | ||
158 | mov TSB_REG, %g1 | 154 | mov TSB_REG, %g1 |
159 | stxa %o2, [%g1] ASI_DMMU | 155 | stxa %o1, [%g1] ASI_DMMU |
160 | membar #Sync | 156 | membar #Sync |
161 | 157 | ||
162 | stxa %o2, [%g1] ASI_IMMU | 158 | stxa %o1, [%g1] ASI_IMMU |
163 | membar #Sync | 159 | membar #Sync |
164 | 160 | ||
165 | #define KERN_HIGHBITS ((_PAGE_VALID|_PAGE_SZBITS)^0xfffff80000000000) | 161 | brz %o2, 9f |
166 | #define KERN_LOWBITS (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W | _PAGE_L) | 162 | nop |
167 | sethi %uhi(KERN_HIGHBITS), %g2 | ||
168 | or %g2, %ulo(KERN_HIGHBITS), %g2 | ||
169 | sllx %g2, 32, %g2 | ||
170 | or %g2, KERN_LOWBITS, %g2 | ||
171 | #undef KERN_HIGHBITS | ||
172 | #undef KERN_LOWBITS | ||
173 | |||
174 | xor %o1, %g2, %o1 | ||
175 | 163 | ||
176 | /* We use entry 61 for this locked entry. This is the spitfire | 164 | /* We use entry 61 for this locked entry. This is the spitfire |
177 | * TLB entry number, and luckily cheetah masks the value with | 165 | * TLB entry number, and luckily cheetah masks the value with |
@@ -184,11 +172,10 @@ tsb_context_switch: | |||
184 | stxa %o2, [%g1] ASI_DMMU | 172 | stxa %o2, [%g1] ASI_DMMU |
185 | membar #Sync | 173 | membar #Sync |
186 | mov (61 << 3), %g1 | 174 | mov (61 << 3), %g1 |
187 | stxa %o1, [%g1] ASI_DTLB_DATA_ACCESS | 175 | stxa %o3, [%g1] ASI_DTLB_DATA_ACCESS |
188 | membar #Sync | 176 | membar #Sync |
189 | |||
190 | 9: | 177 | 9: |
191 | wrpr %o5, %pstate | 178 | wrpr %o5, %pstate |
192 | 179 | ||
193 | retl | 180 | retl |
194 | mov %o2, %o0 | 181 | nop |
diff --git a/arch/sparc64/mm/tsb.c b/arch/sparc64/mm/tsb.c index 2f84cef6c1b5..dfe7144fcdf6 100644 --- a/arch/sparc64/mm/tsb.c +++ b/arch/sparc64/mm/tsb.c | |||
@@ -9,13 +9,7 @@ | |||
9 | #include <asm/tlbflush.h> | 9 | #include <asm/tlbflush.h> |
10 | #include <asm/tlb.h> | 10 | #include <asm/tlb.h> |
11 | #include <asm/mmu_context.h> | 11 | #include <asm/mmu_context.h> |
12 | 12 | #include <asm/pgtable.h> | |
13 | #define TSB_ENTRY_ALIGNMENT 16 | ||
14 | |||
15 | struct tsb { | ||
16 | unsigned long tag; | ||
17 | unsigned long pte; | ||
18 | } __attribute__((aligned(TSB_ENTRY_ALIGNMENT))); | ||
19 | 13 | ||
20 | /* We use an 8K TSB for the whole kernel, this allows to | 14 | /* We use an 8K TSB for the whole kernel, this allows to |
21 | * handle about 4MB of modules and vmalloc mappings without | 15 | * handle about 4MB of modules and vmalloc mappings without |
@@ -27,10 +21,10 @@ struct tsb { | |||
27 | 21 | ||
28 | extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; | 22 | extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; |
29 | 23 | ||
30 | static inline unsigned long tsb_hash(unsigned long vaddr) | 24 | static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long nentries) |
31 | { | 25 | { |
32 | vaddr >>= PAGE_SHIFT; | 26 | vaddr >>= PAGE_SHIFT; |
33 | return vaddr & (KERNEL_TSB_NENTRIES - 1); | 27 | return vaddr & (nentries - 1); |
34 | } | 28 | } |
35 | 29 | ||
36 | static inline int tag_compare(struct tsb *entry, unsigned long vaddr, unsigned long context) | 30 | static inline int tag_compare(struct tsb *entry, unsigned long vaddr, unsigned long context) |
@@ -51,7 +45,8 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) | |||
51 | unsigned long v; | 45 | unsigned long v; |
52 | 46 | ||
53 | for (v = start; v < end; v += PAGE_SIZE) { | 47 | for (v = start; v < end; v += PAGE_SIZE) { |
54 | struct tsb *ent = &swapper_tsb[tsb_hash(v)]; | 48 | unsigned long hash = tsb_hash(v, KERNEL_TSB_NENTRIES); |
49 | struct tsb *ent = &swapper_tsb[hash]; | ||
55 | 50 | ||
56 | if (tag_compare(ent, v, 0)) { | 51 | if (tag_compare(ent, v, 0)) { |
57 | ent->tag = 0UL; | 52 | ent->tag = 0UL; |
@@ -63,8 +58,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) | |||
63 | void flush_tsb_user(struct mmu_gather *mp) | 58 | void flush_tsb_user(struct mmu_gather *mp) |
64 | { | 59 | { |
65 | struct mm_struct *mm = mp->mm; | 60 | struct mm_struct *mm = mp->mm; |
66 | struct tsb *tsb = (struct tsb *) mm->context.sparc64_tsb; | 61 | struct tsb *tsb = mm->context.tsb; |
67 | unsigned long ctx = ~0UL; | 62 | unsigned long ctx = ~0UL; |
63 | unsigned long nentries = mm->context.tsb_nentries; | ||
68 | int i; | 64 | int i; |
69 | 65 | ||
70 | if (CTX_VALID(mm->context)) | 66 | if (CTX_VALID(mm->context)) |
@@ -76,7 +72,7 @@ void flush_tsb_user(struct mmu_gather *mp) | |||
76 | 72 | ||
77 | v &= ~0x1UL; | 73 | v &= ~0x1UL; |
78 | 74 | ||
79 | ent = &tsb[tsb_hash(v)]; | 75 | ent = &tsb[tsb_hash(v, nentries)]; |
80 | if (tag_compare(ent, v, ctx)) { | 76 | if (tag_compare(ent, v, ctx)) { |
81 | ent->tag = 0UL; | 77 | ent->tag = 0UL; |
82 | membar_storeload_storestore(); | 78 | membar_storeload_storestore(); |
@@ -84,6 +80,83 @@ void flush_tsb_user(struct mmu_gather *mp) | |||
84 | } | 80 | } |
85 | } | 81 | } |
86 | 82 | ||
83 | static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_bytes) | ||
84 | { | ||
85 | unsigned long tsb_reg, base, tsb_paddr; | ||
86 | unsigned long page_sz, tte; | ||
87 | |||
88 | mm->context.tsb_nentries = tsb_bytes / sizeof(struct tsb); | ||
89 | |||
90 | base = TSBMAP_BASE; | ||
91 | tte = (_PAGE_VALID | _PAGE_L | _PAGE_CP | | ||
92 | _PAGE_CV | _PAGE_P | _PAGE_W); | ||
93 | tsb_paddr = __pa(mm->context.tsb); | ||
94 | |||
95 | /* Use the smallest page size that can map the whole TSB | ||
96 | * in one TLB entry. | ||
97 | */ | ||
98 | switch (tsb_bytes) { | ||
99 | case 8192 << 0: | ||
100 | tsb_reg = 0x0UL; | ||
101 | #ifdef DCACHE_ALIASING_POSSIBLE | ||
102 | base += (tsb_paddr & 8192); | ||
103 | #endif | ||
104 | tte |= _PAGE_SZ8K; | ||
105 | page_sz = 8192; | ||
106 | break; | ||
107 | |||
108 | case 8192 << 1: | ||
109 | tsb_reg = 0x1UL; | ||
110 | tte |= _PAGE_SZ64K; | ||
111 | page_sz = 64 * 1024; | ||
112 | break; | ||
113 | |||
114 | case 8192 << 2: | ||
115 | tsb_reg = 0x2UL; | ||
116 | tte |= _PAGE_SZ64K; | ||
117 | page_sz = 64 * 1024; | ||
118 | break; | ||
119 | |||
120 | case 8192 << 3: | ||
121 | tsb_reg = 0x3UL; | ||
122 | tte |= _PAGE_SZ64K; | ||
123 | page_sz = 64 * 1024; | ||
124 | break; | ||
125 | |||
126 | case 8192 << 4: | ||
127 | tsb_reg = 0x4UL; | ||
128 | tte |= _PAGE_SZ512K; | ||
129 | page_sz = 512 * 1024; | ||
130 | break; | ||
131 | |||
132 | case 8192 << 5: | ||
133 | tsb_reg = 0x5UL; | ||
134 | tte |= _PAGE_SZ512K; | ||
135 | page_sz = 512 * 1024; | ||
136 | break; | ||
137 | |||
138 | case 8192 << 6: | ||
139 | tsb_reg = 0x6UL; | ||
140 | tte |= _PAGE_SZ512K; | ||
141 | page_sz = 512 * 1024; | ||
142 | break; | ||
143 | |||
144 | case 8192 << 7: | ||
145 | tsb_reg = 0x7UL; | ||
146 | tte |= _PAGE_SZ4MB; | ||
147 | page_sz = 4 * 1024 * 1024; | ||
148 | break; | ||
149 | }; | ||
150 | |||
151 | tsb_reg |= base; | ||
152 | tsb_reg |= (tsb_paddr & (page_sz - 1UL)); | ||
153 | tte |= (tsb_paddr & ~(page_sz - 1UL)); | ||
154 | |||
155 | mm->context.tsb_reg_val = tsb_reg; | ||
156 | mm->context.tsb_map_vaddr = base; | ||
157 | mm->context.tsb_map_pte = tte; | ||
158 | } | ||
159 | |||
87 | int init_new_context(struct task_struct *tsk, struct mm_struct *mm) | 160 | int init_new_context(struct task_struct *tsk, struct mm_struct *mm) |
88 | { | 161 | { |
89 | unsigned long page = get_zeroed_page(GFP_KERNEL); | 162 | unsigned long page = get_zeroed_page(GFP_KERNEL); |
@@ -92,14 +165,22 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) | |||
92 | if (unlikely(!page)) | 165 | if (unlikely(!page)) |
93 | return -ENOMEM; | 166 | return -ENOMEM; |
94 | 167 | ||
95 | mm->context.sparc64_tsb = (unsigned long *) page; | 168 | mm->context.tsb = (struct tsb *) page; |
169 | setup_tsb_params(mm, PAGE_SIZE); | ||
96 | 170 | ||
97 | return 0; | 171 | return 0; |
98 | } | 172 | } |
99 | 173 | ||
100 | void destroy_context(struct mm_struct *mm) | 174 | void destroy_context(struct mm_struct *mm) |
101 | { | 175 | { |
102 | free_page((unsigned long) mm->context.sparc64_tsb); | 176 | free_page((unsigned long) mm->context.tsb); |
177 | |||
178 | /* We can remove these later, but for now it's useful | ||
179 | * to catch any bogus post-destroy_context() references | ||
180 | * to the TSB. | ||
181 | */ | ||
182 | mm->context.tsb = NULL; | ||
183 | mm->context.tsb_reg_val = 0UL; | ||
103 | 184 | ||
104 | spin_lock(&ctx_alloc_lock); | 185 | spin_lock(&ctx_alloc_lock); |
105 | 186 | ||