aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2016-10-27 12:04:54 -0400
committerDavid S. Miller <davem@davemloft.net>2016-10-27 12:11:05 -0400
commita74ad5e660a9ee1d071665e7e8ad822784a2dc7f (patch)
treec6b23f8e7461c6f1ad9b7f739b4e54346da642e5
parenta236441bb69723032db94128761a469030c3fe6d (diff)
sparc64: Handle extremely large kernel TLB range flushes more gracefully.
When the vmalloc area gets fragmented, and because the firmware mapping area sits between where modules live and the vmalloc area, we can sometimes receive requests for enormous kernel TLB range flushes. When this happens the cpu just spins flushing billions of pages and this triggers the NMI watchdog and other problems. We took care of this on the TSB side by doing a linear scan of the table once we pass a certain threshold. Do something similar for the TLB flush, however we are limited by the TLB flush facilities provided by the different chip variants. First of all we use an (mostly arbitrary) cut-off of 256K which is about 32 pages. This can be tuned in the future. The huge range code path for each chip works as follows: 1) On spitfire we flush all non-locked TLB entries using diagnostic acceses. 2) On cheetah we use the "flush all" TLB flush. 3) On sun4v/hypervisor we do a TLB context flush on context 0, which unlike previous chips does not remove "permanent" or locked entries. We could probably do something better on spitfire, such as limiting the flush to kernel TLB entries or even doing range comparisons. However that probably isn't worth it since those chips are old and the TLB only had 64 entries. Reported-by: James Clarke <jrtc27@jrtc27.com> Tested-by: James Clarke <jrtc27@jrtc27.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat
-rw-r--r--arch/sparc/mm/ultra.S283
1 files changed, 228 insertions, 55 deletions
diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
index 0fa2e6202c1f..5d2fd6cd3189 100644
--- a/arch/sparc/mm/ultra.S
+++ b/arch/sparc/mm/ultra.S
@@ -113,12 +113,14 @@ __flush_tlb_pending: /* 27 insns */
113 113
114 .align 32 114 .align 32
115 .globl __flush_tlb_kernel_range 115 .globl __flush_tlb_kernel_range
116__flush_tlb_kernel_range: /* 19 insns */ 116__flush_tlb_kernel_range: /* 31 insns */
117 /* %o0=start, %o1=end */ 117 /* %o0=start, %o1=end */
118 cmp %o0, %o1 118 cmp %o0, %o1
119 be,pn %xcc, 2f 119 be,pn %xcc, 2f
120 sub %o1, %o0, %o3
121 srlx %o3, 18, %o4
122 brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow
120 sethi %hi(PAGE_SIZE), %o4 123 sethi %hi(PAGE_SIZE), %o4
121 sub %o1, %o0, %o3
122 sub %o3, %o4, %o3 124 sub %o3, %o4, %o3
123 or %o0, 0x20, %o0 ! Nucleus 125 or %o0, 0x20, %o0 ! Nucleus
1241: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP 1261: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
@@ -134,6 +136,38 @@ __flush_tlb_kernel_range: /* 19 insns */
134 nop 136 nop
135 nop 137 nop
136 nop 138 nop
139 nop
140 nop
141 nop
142 nop
143 nop
144 nop
145 nop
146 nop
147 nop
148 nop
149
150__spitfire_flush_tlb_kernel_range_slow:
151 mov 63 * 8, %o4
1521: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3
153 andcc %o3, 0x40, %g0 /* _PAGE_L_4U */
154 bne,pn %xcc, 2f
155 mov TLB_TAG_ACCESS, %o3
156 stxa %g0, [%o3] ASI_IMMU
157 stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS
158 membar #Sync
1592: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3
160 andcc %o3, 0x40, %g0
161 bne,pn %xcc, 2f
162 mov TLB_TAG_ACCESS, %o3
163 stxa %g0, [%o3] ASI_DMMU
164 stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS
165 membar #Sync
1662: sub %o4, 8, %o4
167 brgez,pt %o4, 1b
168 nop
169 retl
170 nop
137 171
138__spitfire_flush_tlb_mm_slow: 172__spitfire_flush_tlb_mm_slow:
139 rdpr %pstate, %g1 173 rdpr %pstate, %g1
@@ -288,6 +322,40 @@ __cheetah_flush_tlb_pending: /* 27 insns */
288 retl 322 retl
289 wrpr %g7, 0x0, %pstate 323 wrpr %g7, 0x0, %pstate
290 324
325__cheetah_flush_tlb_kernel_range: /* 31 insns */
326 /* %o0=start, %o1=end */
327 cmp %o0, %o1
328 be,pn %xcc, 2f
329 sub %o1, %o0, %o3
330 srlx %o3, 18, %o4
331 brnz,pn %o4, 3f
332 sethi %hi(PAGE_SIZE), %o4
333 sub %o3, %o4, %o3
334 or %o0, 0x20, %o0 ! Nucleus
3351: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
336 stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP
337 membar #Sync
338 brnz,pt %o3, 1b
339 sub %o3, %o4, %o3
3402: sethi %hi(KERNBASE), %o3
341 flush %o3
342 retl
343 nop
3443: mov 0x80, %o4
345 stxa %g0, [%o4] ASI_DMMU_DEMAP
346 membar #Sync
347 stxa %g0, [%o4] ASI_IMMU_DEMAP
348 membar #Sync
349 retl
350 nop
351 nop
352 nop
353 nop
354 nop
355 nop
356 nop
357 nop
358
291#ifdef DCACHE_ALIASING_POSSIBLE 359#ifdef DCACHE_ALIASING_POSSIBLE
292__cheetah_flush_dcache_page: /* 11 insns */ 360__cheetah_flush_dcache_page: /* 11 insns */
293 sethi %hi(PAGE_OFFSET), %g1 361 sethi %hi(PAGE_OFFSET), %g1
@@ -388,13 +456,15 @@ __hypervisor_flush_tlb_pending: /* 27 insns */
388 nop 456 nop
389 nop 457 nop
390 458
391__hypervisor_flush_tlb_kernel_range: /* 19 insns */ 459__hypervisor_flush_tlb_kernel_range: /* 31 insns */
392 /* %o0=start, %o1=end */ 460 /* %o0=start, %o1=end */
393 cmp %o0, %o1 461 cmp %o0, %o1
394 be,pn %xcc, 2f 462 be,pn %xcc, 2f
395 sethi %hi(PAGE_SIZE), %g3 463 sub %o1, %o0, %g2
396 mov %o0, %g1 464 srlx %g2, 18, %g3
397 sub %o1, %g1, %g2 465 brnz,pn %g3, 4f
466 mov %o0, %g1
467 sethi %hi(PAGE_SIZE), %g3
398 sub %g2, %g3, %g2 468 sub %g2, %g3, %g2
3991: add %g1, %g2, %o0 /* ARG0: virtual address */ 4691: add %g1, %g2, %o0 /* ARG0: virtual address */
400 mov 0, %o1 /* ARG1: mmu context */ 470 mov 0, %o1 /* ARG1: mmu context */
@@ -409,6 +479,16 @@ __hypervisor_flush_tlb_kernel_range: /* 19 insns */
4093: sethi %hi(__hypervisor_tlb_tl0_error), %o2 4793: sethi %hi(__hypervisor_tlb_tl0_error), %o2
410 jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 480 jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0
411 nop 481 nop
4824: mov 0, %o0 /* ARG0: CPU lists unimplemented */
483 mov 0, %o1 /* ARG1: CPU lists unimplemented */
484 mov 0, %o2 /* ARG2: mmu context == nucleus */
485 mov HV_MMU_ALL, %o3 /* ARG3: flags */
486 mov HV_FAST_MMU_DEMAP_CTX, %o5
487 ta HV_FAST_TRAP
488 brnz,pn %o0, 3b
489 mov HV_FAST_MMU_DEMAP_CTX, %o1
490 retl
491 nop
412 492
413#ifdef DCACHE_ALIASING_POSSIBLE 493#ifdef DCACHE_ALIASING_POSSIBLE
414 /* XXX Niagara and friends have an 8K cache, so no aliasing is 494 /* XXX Niagara and friends have an 8K cache, so no aliasing is
@@ -431,43 +511,6 @@ tlb_patch_one:
431 retl 511 retl
432 nop 512 nop
433 513
434 .globl cheetah_patch_cachetlbops
435cheetah_patch_cachetlbops:
436 save %sp, -128, %sp
437
438 sethi %hi(__flush_tlb_mm), %o0
439 or %o0, %lo(__flush_tlb_mm), %o0
440 sethi %hi(__cheetah_flush_tlb_mm), %o1
441 or %o1, %lo(__cheetah_flush_tlb_mm), %o1
442 call tlb_patch_one
443 mov 19, %o2
444
445 sethi %hi(__flush_tlb_page), %o0
446 or %o0, %lo(__flush_tlb_page), %o0
447 sethi %hi(__cheetah_flush_tlb_page), %o1
448 or %o1, %lo(__cheetah_flush_tlb_page), %o1
449 call tlb_patch_one
450 mov 22, %o2
451
452 sethi %hi(__flush_tlb_pending), %o0
453 or %o0, %lo(__flush_tlb_pending), %o0
454 sethi %hi(__cheetah_flush_tlb_pending), %o1
455 or %o1, %lo(__cheetah_flush_tlb_pending), %o1
456 call tlb_patch_one
457 mov 27, %o2
458
459#ifdef DCACHE_ALIASING_POSSIBLE
460 sethi %hi(__flush_dcache_page), %o0
461 or %o0, %lo(__flush_dcache_page), %o0
462 sethi %hi(__cheetah_flush_dcache_page), %o1
463 or %o1, %lo(__cheetah_flush_dcache_page), %o1
464 call tlb_patch_one
465 mov 11, %o2
466#endif /* DCACHE_ALIASING_POSSIBLE */
467
468 ret
469 restore
470
471#ifdef CONFIG_SMP 514#ifdef CONFIG_SMP
472 /* These are all called by the slaves of a cross call, at 515 /* These are all called by the slaves of a cross call, at
473 * trap level 1, with interrupts fully disabled. 516 * trap level 1, with interrupts fully disabled.
@@ -535,13 +578,15 @@ xcall_flush_tlb_page: /* 20 insns */
535 nop 578 nop
536 579
537 .globl xcall_flush_tlb_kernel_range 580 .globl xcall_flush_tlb_kernel_range
538xcall_flush_tlb_kernel_range: /* 28 insns */ 581xcall_flush_tlb_kernel_range: /* 44 insns */
539 sethi %hi(PAGE_SIZE - 1), %g2 582 sethi %hi(PAGE_SIZE - 1), %g2
540 or %g2, %lo(PAGE_SIZE - 1), %g2 583 or %g2, %lo(PAGE_SIZE - 1), %g2
541 andn %g1, %g2, %g1 584 andn %g1, %g2, %g1
542 andn %g7, %g2, %g7 585 andn %g7, %g2, %g7
543 sub %g7, %g1, %g3 586 sub %g7, %g1, %g3
544 add %g2, 1, %g2 587 srlx %g3, 18, %g2
588 brnz,pn %g2, 2f
589 add %g2, 1, %g2
545 sub %g3, %g2, %g3 590 sub %g3, %g2, %g3
546 or %g1, 0x20, %g1 ! Nucleus 591 or %g1, 0x20, %g1 ! Nucleus
5471: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP 5921: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
@@ -550,11 +595,25 @@ xcall_flush_tlb_kernel_range: /* 28 insns */
550 brnz,pt %g3, 1b 595 brnz,pt %g3, 1b
551 sub %g3, %g2, %g3 596 sub %g3, %g2, %g3
552 retry 597 retry
553 nop 5982: mov 63 * 8, %g1
554 nop 5991: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2
555 nop 600 andcc %g2, 0x40, %g0 /* _PAGE_L_4U */
556 nop 601 bne,pn %xcc, 2f
557 nop 602 mov TLB_TAG_ACCESS, %g2
603 stxa %g0, [%g2] ASI_IMMU
604 stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS
605 membar #Sync
6062: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2
607 andcc %g2, 0x40, %g0
608 bne,pn %xcc, 2f
609 mov TLB_TAG_ACCESS, %g2
610 stxa %g0, [%g2] ASI_DMMU
611 stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS
612 membar #Sync
6132: sub %g1, 8, %g1
614 brgez,pt %g1, 1b
615 nop
616 retry
558 nop 617 nop
559 nop 618 nop
560 nop 619 nop
@@ -683,6 +742,52 @@ xcall_fetch_glob_pmu_n4:
683 742
684 retry 743 retry
685 744
745__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */
746 sethi %hi(PAGE_SIZE - 1), %g2
747 or %g2, %lo(PAGE_SIZE - 1), %g2
748 andn %g1, %g2, %g1
749 andn %g7, %g2, %g7
750 sub %g7, %g1, %g3
751 srlx %g3, 18, %g2
752 brnz,pn %g2, 2f
753 add %g2, 1, %g2
754 sub %g3, %g2, %g3
755 or %g1, 0x20, %g1 ! Nucleus
7561: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
757 stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP
758 membar #Sync
759 brnz,pt %g3, 1b
760 sub %g3, %g2, %g3
761 retry
7622: mov 0x80, %g2
763 stxa %g0, [%g2] ASI_DMMU_DEMAP
764 membar #Sync
765 stxa %g0, [%g2] ASI_IMMU_DEMAP
766 membar #Sync
767 retry
768 nop
769 nop
770 nop
771 nop
772 nop
773 nop
774 nop
775 nop
776 nop
777 nop
778 nop
779 nop
780 nop
781 nop
782 nop
783 nop
784 nop
785 nop
786 nop
787 nop
788 nop
789 nop
790
686#ifdef DCACHE_ALIASING_POSSIBLE 791#ifdef DCACHE_ALIASING_POSSIBLE
687 .align 32 792 .align 32
688 .globl xcall_flush_dcache_page_cheetah 793 .globl xcall_flush_dcache_page_cheetah
@@ -798,18 +903,20 @@ __hypervisor_xcall_flush_tlb_page: /* 20 insns */
798 nop 903 nop
799 904
800 .globl __hypervisor_xcall_flush_tlb_kernel_range 905 .globl __hypervisor_xcall_flush_tlb_kernel_range
801__hypervisor_xcall_flush_tlb_kernel_range: /* 28 insns */ 906__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
802 /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ 907 /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
803 sethi %hi(PAGE_SIZE - 1), %g2 908 sethi %hi(PAGE_SIZE - 1), %g2
804 or %g2, %lo(PAGE_SIZE - 1), %g2 909 or %g2, %lo(PAGE_SIZE - 1), %g2
805 andn %g1, %g2, %g1 910 andn %g1, %g2, %g1
806 andn %g7, %g2, %g7 911 andn %g7, %g2, %g7
807 sub %g7, %g1, %g3 912 sub %g7, %g1, %g3
913 srlx %g3, 18, %g7
808 add %g2, 1, %g2 914 add %g2, 1, %g2
809 sub %g3, %g2, %g3 915 sub %g3, %g2, %g3
810 mov %o0, %g2 916 mov %o0, %g2
811 mov %o1, %g4 917 mov %o1, %g4
812 mov %o2, %g7 918 brnz,pn %g7, 2f
919 mov %o2, %g7
8131: add %g1, %g3, %o0 /* ARG0: virtual address */ 9201: add %g1, %g3, %o0 /* ARG0: virtual address */
814 mov 0, %o1 /* ARG1: mmu context */ 921 mov 0, %o1 /* ARG1: mmu context */
815 mov HV_MMU_ALL, %o2 /* ARG2: flags */ 922 mov HV_MMU_ALL, %o2 /* ARG2: flags */
@@ -820,7 +927,7 @@ __hypervisor_xcall_flush_tlb_kernel_range: /* 28 insns */
820 sethi %hi(PAGE_SIZE), %o2 927 sethi %hi(PAGE_SIZE), %o2
821 brnz,pt %g3, 1b 928 brnz,pt %g3, 1b
822 sub %g3, %o2, %g3 929 sub %g3, %o2, %g3
823 mov %g2, %o0 9305: mov %g2, %o0
824 mov %g4, %o1 931 mov %g4, %o1
825 mov %g7, %o2 932 mov %g7, %o2
826 membar #Sync 933 membar #Sync
@@ -828,6 +935,20 @@ __hypervisor_xcall_flush_tlb_kernel_range: /* 28 insns */
8281: sethi %hi(__hypervisor_tlb_xcall_error), %g4 9351: sethi %hi(__hypervisor_tlb_xcall_error), %g4
829 jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 936 jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0
830 nop 937 nop
9382: mov %o3, %g1
939 mov %o5, %g3
940 mov 0, %o0 /* ARG0: CPU lists unimplemented */
941 mov 0, %o1 /* ARG1: CPU lists unimplemented */
942 mov 0, %o2 /* ARG2: mmu context == nucleus */
943 mov HV_MMU_ALL, %o3 /* ARG3: flags */
944 mov HV_FAST_MMU_DEMAP_CTX, %o5
945 ta HV_FAST_TRAP
946 mov %g1, %o3
947 brz,pt %o0, 5b
948 mov %g3, %o5
949 mov HV_FAST_MMU_DEMAP_CTX, %g6
950 ba,pt %xcc, 1b
951 clr %g5
831 952
832 /* These just get rescheduled to PIL vectors. */ 953 /* These just get rescheduled to PIL vectors. */
833 .globl xcall_call_function 954 .globl xcall_call_function
@@ -864,6 +985,58 @@ xcall_kgdb_capture:
864 985
865#endif /* CONFIG_SMP */ 986#endif /* CONFIG_SMP */
866 987
988 .globl cheetah_patch_cachetlbops
989cheetah_patch_cachetlbops:
990 save %sp, -128, %sp
991
992 sethi %hi(__flush_tlb_mm), %o0
993 or %o0, %lo(__flush_tlb_mm), %o0
994 sethi %hi(__cheetah_flush_tlb_mm), %o1
995 or %o1, %lo(__cheetah_flush_tlb_mm), %o1
996 call tlb_patch_one
997 mov 19, %o2
998
999 sethi %hi(__flush_tlb_page), %o0
1000 or %o0, %lo(__flush_tlb_page), %o0
1001 sethi %hi(__cheetah_flush_tlb_page), %o1
1002 or %o1, %lo(__cheetah_flush_tlb_page), %o1
1003 call tlb_patch_one
1004 mov 22, %o2
1005
1006 sethi %hi(__flush_tlb_pending), %o0
1007 or %o0, %lo(__flush_tlb_pending), %o0
1008 sethi %hi(__cheetah_flush_tlb_pending), %o1
1009 or %o1, %lo(__cheetah_flush_tlb_pending), %o1
1010 call tlb_patch_one
1011 mov 27, %o2
1012
1013 sethi %hi(__flush_tlb_kernel_range), %o0
1014 or %o0, %lo(__flush_tlb_kernel_range), %o0
1015 sethi %hi(__cheetah_flush_tlb_kernel_range), %o1
1016 or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
1017 call tlb_patch_one
1018 mov 31, %o2
1019
1020#ifdef DCACHE_ALIASING_POSSIBLE
1021 sethi %hi(__flush_dcache_page), %o0
1022 or %o0, %lo(__flush_dcache_page), %o0
1023 sethi %hi(__cheetah_flush_dcache_page), %o1
1024 or %o1, %lo(__cheetah_flush_dcache_page), %o1
1025 call tlb_patch_one
1026 mov 11, %o2
1027#endif /* DCACHE_ALIASING_POSSIBLE */
1028
1029#ifdef CONFIG_SMP
1030 sethi %hi(xcall_flush_tlb_kernel_range), %o0
1031 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
1032 sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
1033 or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
1034 call tlb_patch_one
1035 mov 44, %o2
1036#endif /* CONFIG_SMP */
1037
1038 ret
1039 restore
867 1040
868 .globl hypervisor_patch_cachetlbops 1041 .globl hypervisor_patch_cachetlbops
869hypervisor_patch_cachetlbops: 1042hypervisor_patch_cachetlbops:
@@ -895,7 +1068,7 @@ hypervisor_patch_cachetlbops:
895 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 1068 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1
896 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 1069 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
897 call tlb_patch_one 1070 call tlb_patch_one
898 mov 19, %o2 1071 mov 31, %o2
899 1072
900#ifdef DCACHE_ALIASING_POSSIBLE 1073#ifdef DCACHE_ALIASING_POSSIBLE
901 sethi %hi(__flush_dcache_page), %o0 1074 sethi %hi(__flush_dcache_page), %o0
@@ -926,7 +1099,7 @@ hypervisor_patch_cachetlbops:
926 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 1099 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
927 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 1100 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
928 call tlb_patch_one 1101 call tlb_patch_one
929 mov 28, %o2 1102 mov 44, %o2
930#endif /* CONFIG_SMP */ 1103#endif /* CONFIG_SMP */
931 1104
932 ret 1105 ret
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-24 18:03:41 -0400