MIPS: Use C0_KScratch (if present) to hold PGD pointer.

Decide at runtime to use either Context or KScratch to hold the PGD
pointer.

Signed-off-by: David Daney <ddaney@caviumnetworks.com>
To: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/1876/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>

3 files changed, 108 insertions, 18 deletions

diff --git a/arch/mips/include/asm/mmu_context.h b/arch/mips/include/asm/mmu_context.h
index d9592733a7ba..73c0d45798de 100644
--- a/arch/mips/include/asm/mmu_context.h
+++ b/arch/mips/include/asm/mmu_context.h
@@ -29,13 +29,7 @@
 #define TLBMISS_HANDLER_SETUP_PGD(pgd)                          \
         tlbmiss_handler_setup_pgd((unsigned long)(pgd))
 
-static inline void tlbmiss_handler_setup_pgd(unsigned long pgd)
-{
-        /* Check for swapper_pg_dir and convert to physical address. */
-        if ((pgd & CKSEG3) == CKSEG0)
-                pgd = CPHYSADDR(pgd);
-        write_c0_context(pgd << 11);
-}
+extern void tlbmiss_handler_setup_pgd(unsigned long pgd);
 
 #define TLBMISS_HANDLER_SETUP()                                         \
         do {                                                            \
diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index e97104302541..71350f7f2d88 100644
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -1592,7 +1592,6 @@ void __cpuinit per_cpu_trap_init(void)
 #endif /* CONFIG_MIPS_MT_SMTC */
 
         cpu_data[cpu].asid_cache = ASID_FIRST_VERSION;
-        TLBMISS_HANDLER_SETUP();
 
         atomic_inc(&init_mm.mm_count);
         current->active_mm = &init_mm;
@@ -1614,6 +1613,7 @@ void __cpuinit per_cpu_trap_init(void)
                 write_c0_wired(0);
         }
 #endif /* CONFIG_MIPS_MT_SMTC */
+        TLBMISS_HANDLER_SETUP();
 }
 
 /* Install CPU exception handler */
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 93816f3bca67..0bb4c3b89c78 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -26,8 +26,10 @@
 #include <linux/smp.h>
 #include <linux/string.h>
 #include <linux/init.h>
+#include <linux/cache.h>
 
-#include <asm/mmu_context.h>
+#include <asm/cacheflush.h>
+#include <asm/pgtable.h>
 #include <asm/war.h>
 #include <asm/uasm.h>
 
@@ -173,11 +175,38 @@ static struct uasm_reloc relocs[128] __cpuinitdata;
 static int check_for_high_segbits __cpuinitdata;
 #endif
 
-#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+
+static unsigned int kscratch_used_mask __cpuinitdata;
+
+static int __cpuinit allocate_kscratch(void)
+{
+        int r;
+        unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
+
+        r = ffs(a);
+
+        if (r == 0)
+                return -1;
+
+        r--; /* make it zero based */
+
+        kscratch_used_mask |= (1 << r);
+
+        return r;
+}
+
+static int pgd_reg __cpuinitdata;
+
+#else /* !CONFIG_MIPS_PGD_C0_CONTEXT*/
 /*
  * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
  * we cannot do r3000 under these circumstances.
+ *
+ * Declare pgd_current here instead of including mmu_context.h to avoid type
+ * conflicts for tlbmiss_handler_setup_pgd
  */
+extern unsigned long pgd_current[];
 
 /*
  * The R3000 TLB handler is simple.
@@ -573,13 +602,22 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
         /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
 
 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT
-        /*
-         * &pgd << 11 stored in CONTEXT [23..63].
-         */
-        UASM_i_MFC0(p, ptr, C0_CONTEXT);
-        uasm_i_dins(p, ptr, 0, 0, 23); /* Clear lower 23 bits of context. */
-        uasm_i_ori(p, ptr, ptr, 0x540); /* 1 0  1 0 1  << 6  xkphys cached */
-        uasm_i_drotr(p, ptr, ptr, 11);
+        if (pgd_reg != -1) {
+                /* pgd is in pgd_reg */
+                UASM_i_MFC0(p, ptr, 31, pgd_reg);
+        } else {
+                /*
+                 * &pgd << 11 stored in CONTEXT [23..63].
+                 */
+                UASM_i_MFC0(p, ptr, C0_CONTEXT);
+
+                /* Clear lower 23 bits of context. */
+                uasm_i_dins(p, ptr, 0, 0, 23);
+
+                /* 1 0  1 0 1  << 6  xkphys cached */
+                uasm_i_ori(p, ptr, ptr, 0x540);
+                uasm_i_drotr(p, ptr, ptr, 11);
+        }
 #elif defined(CONFIG_SMP)
 # ifdef  CONFIG_MIPS_MT_SMTC
         /*
@@ -1014,6 +1052,55 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
 u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
 u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
 u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned;
+
+static void __cpuinit build_r4000_setup_pgd(void)
+{
+        const int a0 = 4;
+        const int a1 = 5;
+        u32 *p = tlbmiss_handler_setup_pgd;
+        struct uasm_label *l = labels;
+        struct uasm_reloc *r = relocs;
+
+        memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd));
+        memset(labels, 0, sizeof(labels));
+        memset(relocs, 0, sizeof(relocs));
+
+        pgd_reg = allocate_kscratch();
+
+        if (pgd_reg == -1) {
+                /* PGD << 11 in c0_Context */
+                /*
+                 * If it is a ckseg0 address, convert to a physical
+                 * address.  Shifting right by 29 and adding 4 will
+                 * result in zero for these addresses.
+                 *
+                 */
+                UASM_i_SRA(&p, a1, a0, 29);
+                UASM_i_ADDIU(&p, a1, a1, 4);
+                uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1);
+                uasm_i_nop(&p);
+                uasm_i_dinsm(&p, a0, 0, 29, 64 - 29);
+                uasm_l_tlbl_goaround1(&l, p);
+                UASM_i_SLL(&p, a0, a0, 11);
+                uasm_i_jr(&p, 31);
+                UASM_i_MTC0(&p, a0, C0_CONTEXT);
+        } else {
+                /* PGD in c0_KScratch */
+                uasm_i_jr(&p, 31);
+                UASM_i_MTC0(&p, a0, 31, pgd_reg);
+        }
+        if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd))
+                panic("tlbmiss_handler_setup_pgd space exceeded");
+        uasm_resolve_relocs(relocs, labels);
+        pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
+                 (unsigned int)(p - tlbmiss_handler_setup_pgd));
+
+        dump_handler(tlbmiss_handler_setup_pgd,
+                     ARRAY_SIZE(tlbmiss_handler_setup_pgd));
+}
+#endif
 
 static void __cpuinit
 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
@@ -1161,6 +1248,8 @@ build_pte_modifiable(u32 **p, struct uasm_reloc **r,
 }
 
 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
+
+
 /*
  * R3000 style TLB load/store/modify handlers.
  */
@@ -1623,13 +1712,16 @@ void __cpuinit build_tlb_refill_handler(void)
                 break;
 
         default:
-                build_r4000_tlb_refill_handler();
                 if (!run_once) {
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+                        build_r4000_setup_pgd();
+#endif
                         build_r4000_tlb_load_handler();
                         build_r4000_tlb_store_handler();
                         build_r4000_tlb_modify_handler();
                         run_once++;
                 }
+                build_r4000_tlb_refill_handler();
         }
 }
 
@@ -1641,4 +1733,8 @@ void __cpuinit flush_tlb_handlers(void)
                            (unsigned long)handle_tlbs + sizeof(handle_tlbs));
         local_flush_icache_range((unsigned long)handle_tlbm,
                            (unsigned long)handle_tlbm + sizeof(handle_tlbm));
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+        local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
+                           (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm));
+#endif
 }