diff options
Diffstat (limited to 'arch/x86/kernel/tlb_uv.c')
-rw-r--r-- | arch/x86/kernel/tlb_uv.c | 77 |
1 files changed, 45 insertions, 32 deletions
diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c index 6812b829ed83..8afb69180c9b 100644 --- a/arch/x86/kernel/tlb_uv.c +++ b/arch/x86/kernel/tlb_uv.c | |||
@@ -11,16 +11,15 @@ | |||
11 | #include <linux/kernel.h> | 11 | #include <linux/kernel.h> |
12 | 12 | ||
13 | #include <asm/mmu_context.h> | 13 | #include <asm/mmu_context.h> |
14 | #include <asm/uv/uv.h> | ||
14 | #include <asm/uv/uv_mmrs.h> | 15 | #include <asm/uv/uv_mmrs.h> |
15 | #include <asm/uv/uv_hub.h> | 16 | #include <asm/uv/uv_hub.h> |
16 | #include <asm/uv/uv_bau.h> | 17 | #include <asm/uv/uv_bau.h> |
17 | #include <asm/genapic.h> | 18 | #include <asm/apic.h> |
18 | #include <asm/idle.h> | 19 | #include <asm/idle.h> |
19 | #include <asm/tsc.h> | 20 | #include <asm/tsc.h> |
20 | #include <asm/irq_vectors.h> | 21 | #include <asm/irq_vectors.h> |
21 | 22 | ||
22 | #include <mach_apic.h> | ||
23 | |||
24 | static struct bau_control **uv_bau_table_bases __read_mostly; | 23 | static struct bau_control **uv_bau_table_bases __read_mostly; |
25 | static int uv_bau_retry_limit __read_mostly; | 24 | static int uv_bau_retry_limit __read_mostly; |
26 | 25 | ||
@@ -210,14 +209,15 @@ static int uv_wait_completion(struct bau_desc *bau_desc, | |||
210 | * | 209 | * |
211 | * Send a broadcast and wait for a broadcast message to complete. | 210 | * Send a broadcast and wait for a broadcast message to complete. |
212 | * | 211 | * |
213 | * The cpumaskp mask contains the cpus the broadcast was sent to. | 212 | * The flush_mask contains the cpus the broadcast was sent to. |
214 | * | 213 | * |
215 | * Returns 1 if all remote flushing was done. The mask is zeroed. | 214 | * Returns NULL if all remote flushing was done. The mask is zeroed. |
216 | * Returns 0 if some remote flushing remains to be done. The mask is left | 215 | * Returns @flush_mask if some remote flushing remains to be done. The |
217 | * unchanged. | 216 | * mask will have some bits still set. |
218 | */ | 217 | */ |
219 | int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc, | 218 | const struct cpumask *uv_flush_send_and_wait(int cpu, int this_blade, |
220 | cpumask_t *cpumaskp) | 219 | struct bau_desc *bau_desc, |
220 | struct cpumask *flush_mask) | ||
221 | { | 221 | { |
222 | int completion_status = 0; | 222 | int completion_status = 0; |
223 | int right_shift; | 223 | int right_shift; |
@@ -257,66 +257,75 @@ int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc, | |||
257 | * the cpu's, all of which are still in the mask. | 257 | * the cpu's, all of which are still in the mask. |
258 | */ | 258 | */ |
259 | __get_cpu_var(ptcstats).ptc_i++; | 259 | __get_cpu_var(ptcstats).ptc_i++; |
260 | return 0; | 260 | return flush_mask; |
261 | } | 261 | } |
262 | 262 | ||
263 | /* | 263 | /* |
264 | * Success, so clear the remote cpu's from the mask so we don't | 264 | * Success, so clear the remote cpu's from the mask so we don't |
265 | * use the IPI method of shootdown on them. | 265 | * use the IPI method of shootdown on them. |
266 | */ | 266 | */ |
267 | for_each_cpu_mask(bit, *cpumaskp) { | 267 | for_each_cpu(bit, flush_mask) { |
268 | blade = uv_cpu_to_blade_id(bit); | 268 | blade = uv_cpu_to_blade_id(bit); |
269 | if (blade == this_blade) | 269 | if (blade == this_blade) |
270 | continue; | 270 | continue; |
271 | cpu_clear(bit, *cpumaskp); | 271 | cpumask_clear_cpu(bit, flush_mask); |
272 | } | 272 | } |
273 | if (!cpus_empty(*cpumaskp)) | 273 | if (!cpumask_empty(flush_mask)) |
274 | return 0; | 274 | return flush_mask; |
275 | return 1; | 275 | return NULL; |
276 | } | 276 | } |
277 | 277 | ||
278 | static DEFINE_PER_CPU(cpumask_var_t, uv_flush_tlb_mask); | ||
279 | |||
278 | /** | 280 | /** |
279 | * uv_flush_tlb_others - globally purge translation cache of a virtual | 281 | * uv_flush_tlb_others - globally purge translation cache of a virtual |
280 | * address or all TLB's | 282 | * address or all TLB's |
281 | * @cpumaskp: mask of all cpu's in which the address is to be removed | 283 | * @cpumask: mask of all cpu's in which the address is to be removed |
282 | * @mm: mm_struct containing virtual address range | 284 | * @mm: mm_struct containing virtual address range |
283 | * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu) | 285 | * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu) |
286 | * @cpu: the current cpu | ||
284 | * | 287 | * |
285 | * This is the entry point for initiating any UV global TLB shootdown. | 288 | * This is the entry point for initiating any UV global TLB shootdown. |
286 | * | 289 | * |
287 | * Purges the translation caches of all specified processors of the given | 290 | * Purges the translation caches of all specified processors of the given |
288 | * virtual address, or purges all TLB's on specified processors. | 291 | * virtual address, or purges all TLB's on specified processors. |
289 | * | 292 | * |
290 | * The caller has derived the cpumaskp from the mm_struct and has subtracted | 293 | * The caller has derived the cpumask from the mm_struct. This function |
291 | * the local cpu from the mask. This function is called only if there | 294 | * is called only if there are bits set in the mask. (e.g. flush_tlb_page()) |
292 | * are bits set in the mask. (e.g. flush_tlb_page()) | ||
293 | * | 295 | * |
294 | * The cpumaskp is converted into a nodemask of the nodes containing | 296 | * The cpumask is converted into a nodemask of the nodes containing |
295 | * the cpus. | 297 | * the cpus. |
296 | * | 298 | * |
297 | * Returns 1 if all remote flushing was done. | 299 | * Note that this function should be called with preemption disabled. |
298 | * Returns 0 if some remote flushing remains to be done. | 300 | * |
301 | * Returns NULL if all remote flushing was done. | ||
302 | * Returns pointer to cpumask if some remote flushing remains to be | ||
303 | * done. The returned pointer is valid till preemption is re-enabled. | ||
299 | */ | 304 | */ |
300 | int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, | 305 | const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask, |
301 | unsigned long va) | 306 | struct mm_struct *mm, |
307 | unsigned long va, unsigned int cpu) | ||
302 | { | 308 | { |
309 | struct cpumask *flush_mask = __get_cpu_var(uv_flush_tlb_mask); | ||
303 | int i; | 310 | int i; |
304 | int bit; | 311 | int bit; |
305 | int blade; | 312 | int blade; |
306 | int cpu; | 313 | int uv_cpu; |
307 | int this_blade; | 314 | int this_blade; |
308 | int locals = 0; | 315 | int locals = 0; |
309 | struct bau_desc *bau_desc; | 316 | struct bau_desc *bau_desc; |
310 | 317 | ||
311 | cpu = uv_blade_processor_id(); | 318 | cpumask_andnot(flush_mask, cpumask, cpumask_of(cpu)); |
319 | |||
320 | uv_cpu = uv_blade_processor_id(); | ||
312 | this_blade = uv_numa_blade_id(); | 321 | this_blade = uv_numa_blade_id(); |
313 | bau_desc = __get_cpu_var(bau_control).descriptor_base; | 322 | bau_desc = __get_cpu_var(bau_control).descriptor_base; |
314 | bau_desc += UV_ITEMS_PER_DESCRIPTOR * cpu; | 323 | bau_desc += UV_ITEMS_PER_DESCRIPTOR * uv_cpu; |
315 | 324 | ||
316 | bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE); | 325 | bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE); |
317 | 326 | ||
318 | i = 0; | 327 | i = 0; |
319 | for_each_cpu_mask(bit, *cpumaskp) { | 328 | for_each_cpu(bit, flush_mask) { |
320 | blade = uv_cpu_to_blade_id(bit); | 329 | blade = uv_cpu_to_blade_id(bit); |
321 | BUG_ON(blade > (UV_DISTRIBUTION_SIZE - 1)); | 330 | BUG_ON(blade > (UV_DISTRIBUTION_SIZE - 1)); |
322 | if (blade == this_blade) { | 331 | if (blade == this_blade) { |
@@ -331,17 +340,17 @@ int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, | |||
331 | * no off_node flushing; return status for local node | 340 | * no off_node flushing; return status for local node |
332 | */ | 341 | */ |
333 | if (locals) | 342 | if (locals) |
334 | return 0; | 343 | return flush_mask; |
335 | else | 344 | else |
336 | return 1; | 345 | return NULL; |
337 | } | 346 | } |
338 | __get_cpu_var(ptcstats).requestor++; | 347 | __get_cpu_var(ptcstats).requestor++; |
339 | __get_cpu_var(ptcstats).ntargeted += i; | 348 | __get_cpu_var(ptcstats).ntargeted += i; |
340 | 349 | ||
341 | bau_desc->payload.address = va; | 350 | bau_desc->payload.address = va; |
342 | bau_desc->payload.sending_cpu = smp_processor_id(); | 351 | bau_desc->payload.sending_cpu = cpu; |
343 | 352 | ||
344 | return uv_flush_send_and_wait(cpu, this_blade, bau_desc, cpumaskp); | 353 | return uv_flush_send_and_wait(uv_cpu, this_blade, bau_desc, flush_mask); |
345 | } | 354 | } |
346 | 355 | ||
347 | /* | 356 | /* |
@@ -747,6 +756,10 @@ static int __init uv_bau_init(void) | |||
747 | if (!is_uv_system()) | 756 | if (!is_uv_system()) |
748 | return 0; | 757 | return 0; |
749 | 758 | ||
759 | for_each_possible_cpu(cur_cpu) | ||
760 | alloc_cpumask_var_node(&per_cpu(uv_flush_tlb_mask, cur_cpu), | ||
761 | GFP_KERNEL, cpu_to_node(cur_cpu)); | ||
762 | |||
750 | uv_bau_retry_limit = 1; | 763 | uv_bau_retry_limit = 1; |
751 | uv_nshift = uv_hub_info->n_val; | 764 | uv_nshift = uv_hub_info->n_val; |
752 | uv_mmask = (1UL << uv_hub_info->n_val) - 1; | 765 | uv_mmask = (1UL << uv_hub_info->n_val) - 1; |