aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel/smp_64.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/kernel/smp_64.c')
-rw-r--r--arch/x86/kernel/smp_64.c275
1 files changed, 0 insertions, 275 deletions
diff --git a/arch/x86/kernel/smp_64.c b/arch/x86/kernel/smp_64.c
index d28e8685709d..26448fff0abd 100644
--- a/arch/x86/kernel/smp_64.c
+++ b/arch/x86/kernel/smp_64.c
@@ -8,278 +8,3 @@
8 * This code is released under the GNU General Public License version 2 or 8 * This code is released under the GNU General Public License version 2 or
9 * later. 9 * later.
10 */ 10 */
11
12#include <linux/init.h>
13
14#include <linux/mm.h>
15#include <linux/delay.h>
16#include <linux/spinlock.h>
17#include <linux/smp.h>
18#include <linux/kernel_stat.h>
19#include <linux/mc146818rtc.h>
20#include <linux/interrupt.h>
21
22#include <asm/mtrr.h>
23#include <asm/pgalloc.h>
24#include <asm/tlbflush.h>
25#include <asm/mach_apic.h>
26#include <asm/mmu_context.h>
27#include <asm/proto.h>
28#include <asm/apicdef.h>
29#include <asm/idle.h>
30
31/*
32 * Smarter SMP flushing macros.
33 * c/o Linus Torvalds.
34 *
35 * These mean you can really definitely utterly forget about
36 * writing to user space from interrupts. (Its not allowed anyway).
37 *
38 * Optimizations Manfred Spraul <manfred@colorfullife.com>
39 *
40 * More scalable flush, from Andi Kleen
41 *
42 * To avoid global state use 8 different call vectors.
43 * Each CPU uses a specific vector to trigger flushes on other
44 * CPUs. Depending on the received vector the target CPUs look into
45 * the right per cpu variable for the flush data.
46 *
47 * With more than 8 CPUs they are hashed to the 8 available
48 * vectors. The limited global vector space forces us to this right now.
49 * In future when interrupts are split into per CPU domains this could be
50 * fixed, at the cost of triggering multiple IPIs in some cases.
51 */
52
53union smp_flush_state {
54 struct {
55 cpumask_t flush_cpumask;
56 struct mm_struct *flush_mm;
57 unsigned long flush_va;
58 spinlock_t tlbstate_lock;
59 };
60 char pad[SMP_CACHE_BYTES];
61} ____cacheline_aligned;
62
63/* State is put into the per CPU data section, but padded
64 to a full cache line because other CPUs can access it and we don't
65 want false sharing in the per cpu data segment. */
66static DEFINE_PER_CPU(union smp_flush_state, flush_state);
67
68/*
69 * We cannot call mmdrop() because we are in interrupt context,
70 * instead update mm->cpu_vm_mask.
71 */
72void leave_mm(int cpu)
73{
74 if (read_pda(mmu_state) == TLBSTATE_OK)
75 BUG();
76 cpu_clear(cpu, read_pda(active_mm)->cpu_vm_mask);
77 load_cr3(swapper_pg_dir);
78}
79EXPORT_SYMBOL_GPL(leave_mm);
80
81/*
82 *
83 * The flush IPI assumes that a thread switch happens in this order:
84 * [cpu0: the cpu that switches]
85 * 1) switch_mm() either 1a) or 1b)
86 * 1a) thread switch to a different mm
87 * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask);
88 * Stop ipi delivery for the old mm. This is not synchronized with
89 * the other cpus, but smp_invalidate_interrupt ignore flush ipis
90 * for the wrong mm, and in the worst case we perform a superfluous
91 * tlb flush.
92 * 1a2) set cpu mmu_state to TLBSTATE_OK
93 * Now the smp_invalidate_interrupt won't call leave_mm if cpu0
94 * was in lazy tlb mode.
95 * 1a3) update cpu active_mm
96 * Now cpu0 accepts tlb flushes for the new mm.
97 * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask);
98 * Now the other cpus will send tlb flush ipis.
99 * 1a4) change cr3.
100 * 1b) thread switch without mm change
101 * cpu active_mm is correct, cpu0 already handles
102 * flush ipis.
103 * 1b1) set cpu mmu_state to TLBSTATE_OK
104 * 1b2) test_and_set the cpu bit in cpu_vm_mask.
105 * Atomically set the bit [other cpus will start sending flush ipis],
106 * and test the bit.
107 * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
108 * 2) switch %%esp, ie current
109 *
110 * The interrupt must handle 2 special cases:
111 * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
112 * - the cpu performs speculative tlb reads, i.e. even if the cpu only
113 * runs in kernel space, the cpu could load tlb entries for user space
114 * pages.
115 *
116 * The good news is that cpu mmu_state is local to each cpu, no
117 * write/read ordering problems.
118 */
119
120/*
121 * TLB flush IPI:
122 *
123 * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
124 * 2) Leave the mm if we are in the lazy tlb mode.
125 *
126 * Interrupts are disabled.
127 */
128
129asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
130{
131 int cpu;
132 int sender;
133 union smp_flush_state *f;
134
135 cpu = smp_processor_id();
136 /*
137 * orig_rax contains the negated interrupt vector.
138 * Use that to determine where the sender put the data.
139 */
140 sender = ~regs->orig_ax - INVALIDATE_TLB_VECTOR_START;
141 f = &per_cpu(flush_state, sender);
142
143 if (!cpu_isset(cpu, f->flush_cpumask))
144 goto out;
145 /*
146 * This was a BUG() but until someone can quote me the
147 * line from the intel manual that guarantees an IPI to
148 * multiple CPUs is retried _only_ on the erroring CPUs
149 * its staying as a return
150 *
151 * BUG();
152 */
153
154 if (f->flush_mm == read_pda(active_mm)) {
155 if (read_pda(mmu_state) == TLBSTATE_OK) {
156 if (f->flush_va == TLB_FLUSH_ALL)
157 local_flush_tlb();
158 else
159 __flush_tlb_one(f->flush_va);
160 } else
161 leave_mm(cpu);
162 }
163out:
164 ack_APIC_irq();
165 cpu_clear(cpu, f->flush_cpumask);
166 add_pda(irq_tlb_count, 1);
167}
168
169void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
170 unsigned long va)
171{
172 int sender;
173 union smp_flush_state *f;
174 cpumask_t cpumask = *cpumaskp;
175
176 /* Caller has disabled preemption */
177 sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
178 f = &per_cpu(flush_state, sender);
179
180 /*
181 * Could avoid this lock when
182 * num_online_cpus() <= NUM_INVALIDATE_TLB_VECTORS, but it is
183 * probably not worth checking this for a cache-hot lock.
184 */
185 spin_lock(&f->tlbstate_lock);
186
187 f->flush_mm = mm;
188 f->flush_va = va;
189 cpus_or(f->flush_cpumask, cpumask, f->flush_cpumask);
190
191 /*
192 * We have to send the IPI only to
193 * CPUs affected.
194 */
195 send_IPI_mask(cpumask, INVALIDATE_TLB_VECTOR_START + sender);
196
197 while (!cpus_empty(f->flush_cpumask))
198 cpu_relax();
199
200 f->flush_mm = NULL;
201 f->flush_va = 0;
202 spin_unlock(&f->tlbstate_lock);
203}
204
205int __cpuinit init_smp_flush(void)
206{
207 int i;
208
209 for_each_cpu_mask(i, cpu_possible_map) {
210 spin_lock_init(&per_cpu(flush_state, i).tlbstate_lock);
211 }
212 return 0;
213}
214core_initcall(init_smp_flush);
215
216void flush_tlb_current_task(void)
217{
218 struct mm_struct *mm = current->mm;
219 cpumask_t cpu_mask;
220
221 preempt_disable();
222 cpu_mask = mm->cpu_vm_mask;
223 cpu_clear(smp_processor_id(), cpu_mask);
224
225 local_flush_tlb();
226 if (!cpus_empty(cpu_mask))
227 flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
228 preempt_enable();
229}
230
231void flush_tlb_mm (struct mm_struct * mm)
232{
233 cpumask_t cpu_mask;
234
235 preempt_disable();
236 cpu_mask = mm->cpu_vm_mask;
237 cpu_clear(smp_processor_id(), cpu_mask);
238
239 if (current->active_mm == mm) {
240 if (current->mm)
241 local_flush_tlb();
242 else
243 leave_mm(smp_processor_id());
244 }
245 if (!cpus_empty(cpu_mask))
246 flush_tlb_others(cpu_mask, mm, TLB_FLUSH_ALL);
247
248 preempt_enable();
249}
250
251void flush_tlb_page(struct vm_area_struct * vma, unsigned long va)
252{
253 struct mm_struct *mm = vma->vm_mm;
254 cpumask_t cpu_mask;
255
256 preempt_disable();
257 cpu_mask = mm->cpu_vm_mask;
258 cpu_clear(smp_processor_id(), cpu_mask);
259
260 if (current->active_mm == mm) {
261 if(current->mm)
262 __flush_tlb_one(va);
263 else
264 leave_mm(smp_processor_id());
265 }
266
267 if (!cpus_empty(cpu_mask))
268 flush_tlb_others(cpu_mask, mm, va);
269
270 preempt_enable();
271}
272
273static void do_flush_tlb_all(void* info)
274{
275 unsigned long cpu = smp_processor_id();
276
277 __flush_tlb_all();
278 if (read_pda(mmu_state) == TLBSTATE_LAZY)
279 leave_mm(cpu);
280}
281
282void flush_tlb_all(void)
283{
284 on_each_cpu(do_flush_tlb_all, NULL, 1, 1);
285}
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-20 06:08:18 -0500