sparc: Access kernel TSB using physical addressing when possible.

On sun4v this is basically required since we point the hypervisor and
the TSB walking hardware at these tables using physical addressing
too.

Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 39 insertions, 1 deletions

diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 3fd8e18bed80..adfac23d976a 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -1597,6 +1597,42 @@ static void __init tsb_phys_patch(void)
 static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR];
 extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
 
+static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa)
+{
+        pa >>= KTSB_PHYS_SHIFT;
+
+        while (start < end) {
+                unsigned int *ia = (unsigned int *)(unsigned long)*start;
+
+                ia[0] = (ia[0] & ~0x3fffff) | (pa >> 10);
+                __asm__ __volatile__("flush     %0" : : "r" (ia));
+
+                ia[1] = (ia[1] & ~0x3ff) | (pa & 0x3ff);
+                __asm__ __volatile__("flush     %0" : : "r" (ia + 1));
+
+                start++;
+        }
+}
+
+static void ktsb_phys_patch(void)
+{
+        extern unsigned int __swapper_tsb_phys_patch;
+        extern unsigned int __swapper_tsb_phys_patch_end;
+        extern unsigned int __swapper_4m_tsb_phys_patch;
+        extern unsigned int __swapper_4m_tsb_phys_patch_end;
+        unsigned long ktsb_pa;
+
+        ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
+        patch_one_ktsb_phys(&__swapper_tsb_phys_patch,
+                            &__swapper_tsb_phys_patch_end, ktsb_pa);
+#ifndef CONFIG_DEBUG_PAGEALLOC
+        ktsb_pa = (kern_base +
+                   ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
+        patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch,
+                            &__swapper_4m_tsb_phys_patch_end, ktsb_pa);
+#endif
+}
+
 static void __init sun4v_ktsb_init(void)
 {
         unsigned long ktsb_pa;
@@ -1716,8 +1752,10 @@ void __init paging_init(void)
                 sun4u_pgprot_init();
 
         if (tlb_type == cheetah_plus ||
-            tlb_type == hypervisor)
+            tlb_type == hypervisor) {
                 tsb_phys_patch();
+                ktsb_phys_patch();
+        }
 
         if (tlb_type == hypervisor) {
                 sun4v_patch_tlb_handlers();