MIPS: Put PGD in C0_CONTEXT for 64-bit R2 processors.

Processors that support the mips64r2 ISA can in four instructions
convert a shifted PGD pointer stored in the upper bits of c0_context
into a usable pointer.  By doing this we save a memory load and
associated potential cache miss in the TLB exception handlers.

Since the upper bits of c0_context were holding the CPU number, we
move this to the upper bits of c0_xcontext which doesn't have enough
bits to hold the PGD pointer, but has plenty for the CPU number.

Signed-off-by: David Daney <ddaney@caviumnetworks.com>
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>

2 files changed, 38 insertions, 11 deletions

diff --git a/arch/mips/include/asm/mmu_context.h b/arch/mips/include/asm/mmu_context.h
index 6083db586500..145bb81ccaa5 100644
--- a/arch/mips/include/asm/mmu_context.h
+++ b/arch/mips/include/asm/mmu_context.h
@@ -24,6 +24,33 @@
 #endif /* SMTC */
 #include <asm-generic/mm_hooks.h>
 
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+
+#define TLBMISS_HANDLER_SETUP_PGD(pgd)                          \
+        tlbmiss_handler_setup_pgd((unsigned long)(pgd))
+
+static inline void tlbmiss_handler_setup_pgd(unsigned long pgd)
+{
+        /* Check for swapper_pg_dir and convert to physical address. */
+        if ((pgd & CKSEG3) == CKSEG0)
+                pgd = CPHYSADDR(pgd);
+        write_c0_context(pgd << 11);
+}
+
+#define TLBMISS_HANDLER_SETUP()                                         \
+        do {                                                            \
+                TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir);              \
+                write_c0_xcontext((unsigned long) smp_processor_id() << 51); \
+        } while (0)
+
+
+static inline unsigned long get_current_pgd(void)
+{
+        return PHYS_TO_XKSEG_CACHED((read_c0_context() >> 11) & ~0xfffUL);
+}
+
+#else /* CONFIG_MIPS_PGD_C0_CONTEXT: using  pgd_current*/
+
 /*
  * For the fast tlb miss handlers, we keep a per cpu array of pointers
  * to the current pgd for each processor. Also, the proc. id is stuffed
@@ -46,7 +73,7 @@ extern unsigned long pgd_current[];
         back_to_back_c0_hazard();                                       \
         TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir)
 #endif
-
+#endif /* CONFIG_MIPS_PGD_C0_CONTEXT*/
 #if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)
 
 #define ASID_INC        0x40
diff --git a/arch/mips/include/asm/stackframe.h b/arch/mips/include/asm/stackframe.h
index dd7e220e087b..3b6da3330e32 100644
--- a/arch/mips/include/asm/stackframe.h
+++ b/arch/mips/include/asm/stackframe.h
@@ -87,15 +87,19 @@
 #ifdef CONFIG_SMP
 #ifdef CONFIG_MIPS_MT_SMTC
 #define PTEBASE_SHIFT   19      /* TCBIND */
+#define CPU_ID_REG CP0_TCBIND
+#define CPU_ID_MFC0 mfc0
+#elif defined(CONFIG_MIPS_PGD_C0_CONTEXT)
+#define PTEBASE_SHIFT   48      /* XCONTEXT */
+#define CPU_ID_REG CP0_XCONTEXT
+#define CPU_ID_MFC0 MFC0
 #else
 #define PTEBASE_SHIFT   23      /* CONTEXT */
+#define CPU_ID_REG CP0_CONTEXT
+#define CPU_ID_MFC0 MFC0
 #endif
                 .macro  get_saved_sp    /* SMP variation */
-#ifdef CONFIG_MIPS_MT_SMTC
-                mfc0    k0, CP0_TCBIND
-#else
-                MFC0    k0, CP0_CONTEXT
-#endif
+                CPU_ID_MFC0     k0, CPU_ID_REG
 #if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
                 lui     k1, %hi(kernelsp)
 #else
@@ -111,11 +115,7 @@
                 .endm
 
                 .macro  set_saved_sp stackp temp temp2
-#ifdef CONFIG_MIPS_MT_SMTC
-                mfc0    \temp, CP0_TCBIND
-#else
-                MFC0    \temp, CP0_CONTEXT
-#endif
+                CPU_ID_MFC0     \temp, CPU_ID_REG
                 LONG_SRL        \temp, PTEBASE_SHIFT
                 LONG_S  \stackp, kernelsp(\temp)
                 .endm