diff options
Diffstat (limited to 'arch/arm/include/asm/tlbflush.h')
-rw-r--r-- | arch/arm/include/asm/tlbflush.h | 181 |
1 files changed, 149 insertions, 32 deletions
diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h index f467e9b3f8d5..38960264040c 100644 --- a/arch/arm/include/asm/tlbflush.h +++ b/arch/arm/include/asm/tlbflush.h | |||
@@ -319,67 +319,110 @@ extern struct cpu_tlb_fns cpu_tlb; | |||
319 | #define tlb_op(f, regs, arg) __tlb_op(f, "p15, 0, %0, " regs, arg) | 319 | #define tlb_op(f, regs, arg) __tlb_op(f, "p15, 0, %0, " regs, arg) |
320 | #define tlb_l2_op(f, regs, arg) __tlb_op(f, "p15, 1, %0, " regs, arg) | 320 | #define tlb_l2_op(f, regs, arg) __tlb_op(f, "p15, 1, %0, " regs, arg) |
321 | 321 | ||
322 | static inline void local_flush_tlb_all(void) | 322 | static inline void __local_flush_tlb_all(void) |
323 | { | 323 | { |
324 | const int zero = 0; | 324 | const int zero = 0; |
325 | const unsigned int __tlb_flag = __cpu_tlb_flags; | 325 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
326 | 326 | ||
327 | if (tlb_flag(TLB_WB)) | ||
328 | dsb(); | ||
329 | |||
330 | tlb_op(TLB_V4_U_FULL | TLB_V6_U_FULL, "c8, c7, 0", zero); | 327 | tlb_op(TLB_V4_U_FULL | TLB_V6_U_FULL, "c8, c7, 0", zero); |
331 | tlb_op(TLB_V4_D_FULL | TLB_V6_D_FULL, "c8, c6, 0", zero); | 328 | tlb_op(TLB_V4_D_FULL | TLB_V6_D_FULL, "c8, c6, 0", zero); |
332 | tlb_op(TLB_V4_I_FULL | TLB_V6_I_FULL, "c8, c5, 0", zero); | 329 | tlb_op(TLB_V4_I_FULL | TLB_V6_I_FULL, "c8, c5, 0", zero); |
333 | tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero); | 330 | } |
331 | |||
332 | static inline void local_flush_tlb_all(void) | ||
333 | { | ||
334 | const int zero = 0; | ||
335 | const unsigned int __tlb_flag = __cpu_tlb_flags; | ||
336 | |||
337 | if (tlb_flag(TLB_WB)) | ||
338 | dsb(nshst); | ||
339 | |||
340 | __local_flush_tlb_all(); | ||
341 | tlb_op(TLB_V7_UIS_FULL, "c8, c7, 0", zero); | ||
334 | 342 | ||
335 | if (tlb_flag(TLB_BARRIER)) { | 343 | if (tlb_flag(TLB_BARRIER)) { |
336 | dsb(); | 344 | dsb(nsh); |
337 | isb(); | 345 | isb(); |
338 | } | 346 | } |
339 | } | 347 | } |
340 | 348 | ||
341 | static inline void local_flush_tlb_mm(struct mm_struct *mm) | 349 | static inline void __flush_tlb_all(void) |
342 | { | 350 | { |
343 | const int zero = 0; | 351 | const int zero = 0; |
344 | const int asid = ASID(mm); | ||
345 | const unsigned int __tlb_flag = __cpu_tlb_flags; | 352 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
346 | 353 | ||
347 | if (tlb_flag(TLB_WB)) | 354 | if (tlb_flag(TLB_WB)) |
348 | dsb(); | 355 | dsb(ishst); |
356 | |||
357 | __local_flush_tlb_all(); | ||
358 | tlb_op(TLB_V7_UIS_FULL, "c8, c3, 0", zero); | ||
359 | |||
360 | if (tlb_flag(TLB_BARRIER)) { | ||
361 | dsb(ish); | ||
362 | isb(); | ||
363 | } | ||
364 | } | ||
365 | |||
366 | static inline void __local_flush_tlb_mm(struct mm_struct *mm) | ||
367 | { | ||
368 | const int zero = 0; | ||
369 | const int asid = ASID(mm); | ||
370 | const unsigned int __tlb_flag = __cpu_tlb_flags; | ||
349 | 371 | ||
350 | if (possible_tlb_flags & (TLB_V4_U_FULL|TLB_V4_D_FULL|TLB_V4_I_FULL)) { | 372 | if (possible_tlb_flags & (TLB_V4_U_FULL|TLB_V4_D_FULL|TLB_V4_I_FULL)) { |
351 | if (cpumask_test_cpu(get_cpu(), mm_cpumask(mm))) { | 373 | if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm))) { |
352 | tlb_op(TLB_V4_U_FULL, "c8, c7, 0", zero); | 374 | tlb_op(TLB_V4_U_FULL, "c8, c7, 0", zero); |
353 | tlb_op(TLB_V4_D_FULL, "c8, c6, 0", zero); | 375 | tlb_op(TLB_V4_D_FULL, "c8, c6, 0", zero); |
354 | tlb_op(TLB_V4_I_FULL, "c8, c5, 0", zero); | 376 | tlb_op(TLB_V4_I_FULL, "c8, c5, 0", zero); |
355 | } | 377 | } |
356 | put_cpu(); | ||
357 | } | 378 | } |
358 | 379 | ||
359 | tlb_op(TLB_V6_U_ASID, "c8, c7, 2", asid); | 380 | tlb_op(TLB_V6_U_ASID, "c8, c7, 2", asid); |
360 | tlb_op(TLB_V6_D_ASID, "c8, c6, 2", asid); | 381 | tlb_op(TLB_V6_D_ASID, "c8, c6, 2", asid); |
361 | tlb_op(TLB_V6_I_ASID, "c8, c5, 2", asid); | 382 | tlb_op(TLB_V6_I_ASID, "c8, c5, 2", asid); |
383 | } | ||
384 | |||
385 | static inline void local_flush_tlb_mm(struct mm_struct *mm) | ||
386 | { | ||
387 | const int asid = ASID(mm); | ||
388 | const unsigned int __tlb_flag = __cpu_tlb_flags; | ||
389 | |||
390 | if (tlb_flag(TLB_WB)) | ||
391 | dsb(nshst); | ||
392 | |||
393 | __local_flush_tlb_mm(mm); | ||
394 | tlb_op(TLB_V7_UIS_ASID, "c8, c7, 2", asid); | ||
395 | |||
396 | if (tlb_flag(TLB_BARRIER)) | ||
397 | dsb(nsh); | ||
398 | } | ||
399 | |||
400 | static inline void __flush_tlb_mm(struct mm_struct *mm) | ||
401 | { | ||
402 | const unsigned int __tlb_flag = __cpu_tlb_flags; | ||
403 | |||
404 | if (tlb_flag(TLB_WB)) | ||
405 | dsb(ishst); | ||
406 | |||
407 | __local_flush_tlb_mm(mm); | ||
362 | #ifdef CONFIG_ARM_ERRATA_720789 | 408 | #ifdef CONFIG_ARM_ERRATA_720789 |
363 | tlb_op(TLB_V7_UIS_ASID, "c8, c3, 0", zero); | 409 | tlb_op(TLB_V7_UIS_ASID, "c8, c3, 0", 0); |
364 | #else | 410 | #else |
365 | tlb_op(TLB_V7_UIS_ASID, "c8, c3, 2", asid); | 411 | tlb_op(TLB_V7_UIS_ASID, "c8, c3, 2", ASID(mm)); |
366 | #endif | 412 | #endif |
367 | 413 | ||
368 | if (tlb_flag(TLB_BARRIER)) | 414 | if (tlb_flag(TLB_BARRIER)) |
369 | dsb(); | 415 | dsb(ish); |
370 | } | 416 | } |
371 | 417 | ||
372 | static inline void | 418 | static inline void |
373 | local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) | 419 | __local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) |
374 | { | 420 | { |
375 | const int zero = 0; | 421 | const int zero = 0; |
376 | const unsigned int __tlb_flag = __cpu_tlb_flags; | 422 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
377 | 423 | ||
378 | uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); | 424 | uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); |
379 | 425 | ||
380 | if (tlb_flag(TLB_WB)) | ||
381 | dsb(); | ||
382 | |||
383 | if (possible_tlb_flags & (TLB_V4_U_PAGE|TLB_V4_D_PAGE|TLB_V4_I_PAGE|TLB_V4_I_FULL) && | 426 | if (possible_tlb_flags & (TLB_V4_U_PAGE|TLB_V4_D_PAGE|TLB_V4_I_PAGE|TLB_V4_I_FULL) && |
384 | cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) { | 427 | cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) { |
385 | tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", uaddr); | 428 | tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", uaddr); |
@@ -392,6 +435,36 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) | |||
392 | tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", uaddr); | 435 | tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", uaddr); |
393 | tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", uaddr); | 436 | tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", uaddr); |
394 | tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", uaddr); | 437 | tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", uaddr); |
438 | } | ||
439 | |||
440 | static inline void | ||
441 | local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) | ||
442 | { | ||
443 | const unsigned int __tlb_flag = __cpu_tlb_flags; | ||
444 | |||
445 | uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); | ||
446 | |||
447 | if (tlb_flag(TLB_WB)) | ||
448 | dsb(nshst); | ||
449 | |||
450 | __local_flush_tlb_page(vma, uaddr); | ||
451 | tlb_op(TLB_V7_UIS_PAGE, "c8, c7, 1", uaddr); | ||
452 | |||
453 | if (tlb_flag(TLB_BARRIER)) | ||
454 | dsb(nsh); | ||
455 | } | ||
456 | |||
457 | static inline void | ||
458 | __flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) | ||
459 | { | ||
460 | const unsigned int __tlb_flag = __cpu_tlb_flags; | ||
461 | |||
462 | uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); | ||
463 | |||
464 | if (tlb_flag(TLB_WB)) | ||
465 | dsb(ishst); | ||
466 | |||
467 | __local_flush_tlb_page(vma, uaddr); | ||
395 | #ifdef CONFIG_ARM_ERRATA_720789 | 468 | #ifdef CONFIG_ARM_ERRATA_720789 |
396 | tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 3", uaddr & PAGE_MASK); | 469 | tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 3", uaddr & PAGE_MASK); |
397 | #else | 470 | #else |
@@ -399,19 +472,14 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) | |||
399 | #endif | 472 | #endif |
400 | 473 | ||
401 | if (tlb_flag(TLB_BARRIER)) | 474 | if (tlb_flag(TLB_BARRIER)) |
402 | dsb(); | 475 | dsb(ish); |
403 | } | 476 | } |
404 | 477 | ||
405 | static inline void local_flush_tlb_kernel_page(unsigned long kaddr) | 478 | static inline void __local_flush_tlb_kernel_page(unsigned long kaddr) |
406 | { | 479 | { |
407 | const int zero = 0; | 480 | const int zero = 0; |
408 | const unsigned int __tlb_flag = __cpu_tlb_flags; | 481 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
409 | 482 | ||
410 | kaddr &= PAGE_MASK; | ||
411 | |||
412 | if (tlb_flag(TLB_WB)) | ||
413 | dsb(); | ||
414 | |||
415 | tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", kaddr); | 483 | tlb_op(TLB_V4_U_PAGE, "c8, c7, 1", kaddr); |
416 | tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", kaddr); | 484 | tlb_op(TLB_V4_D_PAGE, "c8, c6, 1", kaddr); |
417 | tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", kaddr); | 485 | tlb_op(TLB_V4_I_PAGE, "c8, c5, 1", kaddr); |
@@ -421,26 +489,75 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr) | |||
421 | tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", kaddr); | 489 | tlb_op(TLB_V6_U_PAGE, "c8, c7, 1", kaddr); |
422 | tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", kaddr); | 490 | tlb_op(TLB_V6_D_PAGE, "c8, c6, 1", kaddr); |
423 | tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", kaddr); | 491 | tlb_op(TLB_V6_I_PAGE, "c8, c5, 1", kaddr); |
492 | } | ||
493 | |||
494 | static inline void local_flush_tlb_kernel_page(unsigned long kaddr) | ||
495 | { | ||
496 | const unsigned int __tlb_flag = __cpu_tlb_flags; | ||
497 | |||
498 | kaddr &= PAGE_MASK; | ||
499 | |||
500 | if (tlb_flag(TLB_WB)) | ||
501 | dsb(nshst); | ||
502 | |||
503 | __local_flush_tlb_kernel_page(kaddr); | ||
504 | tlb_op(TLB_V7_UIS_PAGE, "c8, c7, 1", kaddr); | ||
505 | |||
506 | if (tlb_flag(TLB_BARRIER)) { | ||
507 | dsb(nsh); | ||
508 | isb(); | ||
509 | } | ||
510 | } | ||
511 | |||
512 | static inline void __flush_tlb_kernel_page(unsigned long kaddr) | ||
513 | { | ||
514 | const unsigned int __tlb_flag = __cpu_tlb_flags; | ||
515 | |||
516 | kaddr &= PAGE_MASK; | ||
517 | |||
518 | if (tlb_flag(TLB_WB)) | ||
519 | dsb(ishst); | ||
520 | |||
521 | __local_flush_tlb_kernel_page(kaddr); | ||
424 | tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", kaddr); | 522 | tlb_op(TLB_V7_UIS_PAGE, "c8, c3, 1", kaddr); |
425 | 523 | ||
426 | if (tlb_flag(TLB_BARRIER)) { | 524 | if (tlb_flag(TLB_BARRIER)) { |
427 | dsb(); | 525 | dsb(ish); |
428 | isb(); | 526 | isb(); |
429 | } | 527 | } |
430 | } | 528 | } |
431 | 529 | ||
530 | /* | ||
531 | * Branch predictor maintenance is paired with full TLB invalidation, so | ||
532 | * there is no need for any barriers here. | ||
533 | */ | ||
534 | static inline void __local_flush_bp_all(void) | ||
535 | { | ||
536 | const int zero = 0; | ||
537 | const unsigned int __tlb_flag = __cpu_tlb_flags; | ||
538 | |||
539 | if (tlb_flag(TLB_V6_BP)) | ||
540 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero)); | ||
541 | } | ||
542 | |||
432 | static inline void local_flush_bp_all(void) | 543 | static inline void local_flush_bp_all(void) |
433 | { | 544 | { |
434 | const int zero = 0; | 545 | const int zero = 0; |
435 | const unsigned int __tlb_flag = __cpu_tlb_flags; | 546 | const unsigned int __tlb_flag = __cpu_tlb_flags; |
436 | 547 | ||
548 | __local_flush_bp_all(); | ||
437 | if (tlb_flag(TLB_V7_UIS_BP)) | 549 | if (tlb_flag(TLB_V7_UIS_BP)) |
438 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero)); | ||
439 | else if (tlb_flag(TLB_V6_BP)) | ||
440 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero)); | 550 | asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero)); |
551 | } | ||
441 | 552 | ||
442 | if (tlb_flag(TLB_BARRIER)) | 553 | static inline void __flush_bp_all(void) |
443 | isb(); | 554 | { |
555 | const int zero = 0; | ||
556 | const unsigned int __tlb_flag = __cpu_tlb_flags; | ||
557 | |||
558 | __local_flush_bp_all(); | ||
559 | if (tlb_flag(TLB_V7_UIS_BP)) | ||
560 | asm("mcr p15, 0, %0, c7, c1, 6" : : "r" (zero)); | ||
444 | } | 561 | } |
445 | 562 | ||
446 | #include <asm/cputype.h> | 563 | #include <asm/cputype.h> |
@@ -461,7 +578,7 @@ static inline void dummy_flush_tlb_a15_erratum(void) | |||
461 | * Dummy TLBIMVAIS. Using the unmapped address 0 and ASID 0. | 578 | * Dummy TLBIMVAIS. Using the unmapped address 0 and ASID 0. |
462 | */ | 579 | */ |
463 | asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0)); | 580 | asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0)); |
464 | dsb(); | 581 | dsb(ish); |
465 | } | 582 | } |
466 | #else | 583 | #else |
467 | static inline int erratum_a15_798181(void) | 584 | static inline int erratum_a15_798181(void) |
@@ -495,7 +612,7 @@ static inline void flush_pmd_entry(void *pmd) | |||
495 | tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1 @ L2 flush_pmd", pmd); | 612 | tlb_l2_op(TLB_L2CLEAN_FR, "c15, c9, 1 @ L2 flush_pmd", pmd); |
496 | 613 | ||
497 | if (tlb_flag(TLB_WB)) | 614 | if (tlb_flag(TLB_WB)) |
498 | dsb(); | 615 | dsb(ishst); |
499 | } | 616 | } |
500 | 617 | ||
501 | static inline void clean_pmd_entry(void *pmd) | 618 | static inline void clean_pmd_entry(void *pmd) |