1 files changed, 283 insertions, 83 deletions

diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index bb1719a55d22..86f004dc8355 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -29,8 +29,17 @@
 
 #include <asm/mmu_context.h>
 #include <asm/war.h>
+#include <asm/uasm.h>
+
+/*
+ * TLB load/store/modify handlers.
+ *
+ * Only the fastpath gets synthesized at runtime, the slowpath for
+ * do_page_fault remains normal asm.
+ */
+extern void tlb_do_page_fault_0(void);
+extern void tlb_do_page_fault_1(void);
 
-#include "uasm.h"
 
 static inline int r45k_bvahwbug(void)
 {
@@ -73,18 +82,18 @@ static int __cpuinit m4kc_tlbp_war(void)
 enum label_id {
         label_second_part = 1,
         label_leave,
-#ifdef MODULE_START
-        label_module_alloc,
-#endif
         label_vmalloc,
         label_vmalloc_done,
         label_tlbw_hazard,
         label_split,
+        label_tlbl_goaround1,
+        label_tlbl_goaround2,
         label_nopage_tlbl,
         label_nopage_tlbs,
         label_nopage_tlbm,
         label_smp_pgtable_change,
         label_r3000_write_probe_fail,
+        label_large_segbits_fault,
 #ifdef CONFIG_HUGETLB_PAGE
         label_tlb_huge_update,
 #endif
@@ -92,18 +101,18 @@ enum label_id {
 
 UASM_L_LA(_second_part)
 UASM_L_LA(_leave)
-#ifdef MODULE_START
-UASM_L_LA(_module_alloc)
-#endif
 UASM_L_LA(_vmalloc)
 UASM_L_LA(_vmalloc_done)
 UASM_L_LA(_tlbw_hazard)
 UASM_L_LA(_split)
+UASM_L_LA(_tlbl_goaround1)
+UASM_L_LA(_tlbl_goaround2)
 UASM_L_LA(_nopage_tlbl)
 UASM_L_LA(_nopage_tlbs)
 UASM_L_LA(_nopage_tlbm)
 UASM_L_LA(_smp_pgtable_change)
 UASM_L_LA(_r3000_write_probe_fail)
+UASM_L_LA(_large_segbits_fault)
 #ifdef CONFIG_HUGETLB_PAGE
 UASM_L_LA(_tlb_huge_update)
 #endif
@@ -160,6 +169,16 @@ static u32 tlb_handler[128] __cpuinitdata;
 static struct uasm_label labels[128] __cpuinitdata;
 static struct uasm_reloc relocs[128] __cpuinitdata;
 
+#ifdef CONFIG_64BIT
+static int check_for_high_segbits __cpuinitdata;
+#endif
+
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
+/*
+ * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
+ * we cannot do r3000 under these circumstances.
+ */
+
 /*
  * The R3000 TLB handler is simple.
  */
@@ -199,6 +218,7 @@ static void __cpuinit build_r3000_tlb_refill_handler(void)
 
         dump_handler((u32 *)ebase, 32);
 }
+#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
 
 /*
  * The R4000 TLB handler is much more complicated. We have two
@@ -396,36 +416,60 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
         }
 }
 
-#ifdef CONFIG_HUGETLB_PAGE
-static __cpuinit void build_huge_tlb_write_entry(u32 **p,
-                                                 struct uasm_label **l,
-                                                 struct uasm_reloc **r,
-                                                 unsigned int tmp,
-                                                 enum tlb_write_entry wmode)
+static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
+                                                                  unsigned int reg)
 {
-        /* Set huge page tlb entry size */
-        uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
-        uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
-        uasm_i_mtc0(p, tmp, C0_PAGEMASK);
+        if (kernel_uses_smartmips_rixi) {
+                UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC));
+                UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+        } else {
+#ifdef CONFIG_64BIT_PHYS_ADDR
+                uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL));
+#else
+                UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL));
+#endif
+        }
+}
 
-        build_tlb_write_entry(p, l, r, wmode);
+#ifdef CONFIG_HUGETLB_PAGE
 
+static __cpuinit void build_restore_pagemask(u32 **p,
+                                             struct uasm_reloc **r,
+                                             unsigned int tmp,
+                                             enum label_id lid)
+{
         /* Reset default page size */
         if (PM_DEFAULT_MASK >> 16) {
                 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
                 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
-                uasm_il_b(p, r, label_leave);
+                uasm_il_b(p, r, lid);
                 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
         } else if (PM_DEFAULT_MASK) {
                 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
-                uasm_il_b(p, r, label_leave);
+                uasm_il_b(p, r, lid);
                 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
         } else {
-                uasm_il_b(p, r, label_leave);
+                uasm_il_b(p, r, lid);
                 uasm_i_mtc0(p, 0, C0_PAGEMASK);
         }
 }
 
+static __cpuinit void build_huge_tlb_write_entry(u32 **p,
+                                                 struct uasm_label **l,
+                                                 struct uasm_reloc **r,
+                                                 unsigned int tmp,
+                                                 enum tlb_write_entry wmode)
+{
+        /* Set huge page tlb entry size */
+        uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
+        uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
+        uasm_i_mtc0(p, tmp, C0_PAGEMASK);
+
+        build_tlb_write_entry(p, l, r, wmode);
+
+        build_restore_pagemask(p, r, tmp, label_leave);
+}
+
 /*
  * Check if Huge PTE is present, if so then jump to LABEL.
  */
@@ -459,15 +503,15 @@ static __cpuinit void build_huge_update_entries(u32 **p,
         if (!small_sequence)
                 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16));
 
-        UASM_i_SRL(p, pte, pte, 6); /* convert to entrylo */
-        uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* load it */
+        build_convert_pte_to_entrylo(p, pte);
+        UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */
         /* convert to entrylo1 */
         if (small_sequence)
                 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7);
         else
                 UASM_i_ADDU(p, pte, pte, tmp);
 
-        uasm_i_mtc0(p, pte, C0_ENTRYLO1); /* load it */
+        UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */
 }
 
 static __cpuinit void build_huge_handler_tail(u32 **p,
@@ -497,30 +541,56 @@ static void __cpuinit
 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
                  unsigned int tmp, unsigned int ptr)
 {
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
         long pgdc = (long)pgd_current;
-
+#endif
         /*
          * The vmalloc handling is not in the hotpath.
          */
         uasm_i_dmfc0(p, tmp, C0_BADVADDR);
-        uasm_il_bltz(p, r, tmp, label_vmalloc);
+
+        if (check_for_high_segbits) {
+                /*
+                 * The kernel currently implicitely assumes that the
+                 * MIPS SEGBITS parameter for the processor is
+                 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never
+                 * allocate virtual addresses outside the maximum
+                 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But
+                 * that doesn't prevent user code from accessing the
+                 * higher xuseg addresses.  Here, we make sure that
+                 * everything but the lower xuseg addresses goes down
+                 * the module_alloc/vmalloc path.
+                 */
+                uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
+                uasm_il_bnez(p, r, ptr, label_vmalloc);
+        } else {
+                uasm_il_bltz(p, r, tmp, label_vmalloc);
+        }
         /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
 
-#ifdef CONFIG_SMP
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
+        /*
+         * &pgd << 11 stored in CONTEXT [23..63].
+         */
+        UASM_i_MFC0(p, ptr, C0_CONTEXT);
+        uasm_i_dins(p, ptr, 0, 0, 23); /* Clear lower 23 bits of context. */
+        uasm_i_ori(p, ptr, ptr, 0x540); /* 1 0  1 0 1  << 6  xkphys cached */
+        uasm_i_drotr(p, ptr, ptr, 11);
+#elif defined(CONFIG_SMP)
 # ifdef  CONFIG_MIPS_MT_SMTC
         /*
          * SMTC uses TCBind value as "CPU" index
          */
         uasm_i_mfc0(p, ptr, C0_TCBIND);
-        uasm_i_dsrl(p, ptr, ptr, 19);
+        uasm_i_dsrl_safe(p, ptr, ptr, 19);
 # else
         /*
          * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
          * stored in CONTEXT.
          */
         uasm_i_dmfc0(p, ptr, C0_CONTEXT);
-        uasm_i_dsrl(p, ptr, ptr, 23);
-#endif
+        uasm_i_dsrl_safe(p, ptr, ptr, 23);
+# endif
         UASM_i_LA_mostly(p, tmp, pgdc);
         uasm_i_daddu(p, ptr, ptr, tmp);
         uasm_i_dmfc0(p, tmp, C0_BADVADDR);
@@ -532,42 +602,78 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
 
         uasm_l_vmalloc_done(l, *p);
 
-        if (PGDIR_SHIFT - 3 < 32)               /* get pgd offset in bytes */
-                uasm_i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3);
-        else
-                uasm_i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32);
+        /* get pgd offset in bytes */
+        uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3);
 
         uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
         uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
+#ifndef __PAGETABLE_PMD_FOLDED
         uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
         uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */
-        uasm_i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
+        uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
         uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
         uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
+#endif
 }
 
+enum vmalloc64_mode {not_refill, refill};
 /*
  * BVADDR is the faulting address, PTR is scratch.
  * PTR will hold the pgd for vmalloc.
  */
 static void __cpuinit
 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
-                        unsigned int bvaddr, unsigned int ptr)
+                        unsigned int bvaddr, unsigned int ptr,
+                        enum vmalloc64_mode mode)
 {
         long swpd = (long)swapper_pg_dir;
+        int single_insn_swpd;
+        int did_vmalloc_branch = 0;
+
+        single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd);
 
         uasm_l_vmalloc(l, *p);
 
-        if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
-                uasm_il_b(p, r, label_vmalloc_done);
-                uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
-        } else {
-                UASM_i_LA_mostly(p, ptr, swpd);
-                uasm_il_b(p, r, label_vmalloc_done);
-                if (uasm_in_compat_space_p(swpd))
-                        uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
-                else
-                        uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
+        if (mode == refill && check_for_high_segbits) {
+                if (single_insn_swpd) {
+                        uasm_il_bltz(p, r, bvaddr, label_vmalloc_done);
+                        uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
+                        did_vmalloc_branch = 1;
+                        /* fall through */
+                } else {
+                        uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault);
+                }
+        }
+        if (!did_vmalloc_branch) {
+                if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
+                        uasm_il_b(p, r, label_vmalloc_done);
+                        uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
+                } else {
+                        UASM_i_LA_mostly(p, ptr, swpd);
+                        uasm_il_b(p, r, label_vmalloc_done);
+                        if (uasm_in_compat_space_p(swpd))
+                                uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
+                        else
+                                uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
+                }
+        }
+        if (mode == refill && check_for_high_segbits) {
+                uasm_l_large_segbits_fault(l, *p);
+                /*
+                 * We get here if we are an xsseg address, or if we are
+                 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary.
+                 *
+                 * Ignoring xsseg (assume disabled so would generate
+                 * (address errors?), the only remaining possibility
+                 * is the upper xuseg addresses.  On processors with
+                 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these
+                 * addresses would have taken an address error. We try
+                 * to mimic that here by taking a load/istream page
+                 * fault.
+                 */
+                UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0);
+                uasm_i_jr(p, ptr);
+                uasm_i_nop(p);
         }
 }
 
@@ -674,35 +780,53 @@ static void __cpuinit build_update_entries(u32 **p, unsigned int tmp,
         if (cpu_has_64bits) {
                 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */
                 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
-                uasm_i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
-                uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
-                uasm_i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
-                uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
+                if (kernel_uses_smartmips_rixi) {
+                        UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC));
+                        UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC));
+                        UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+                        UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+                        UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+                } else {
+                        uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */
+                        UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+                        uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */
+                }
+                UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
         } else {
                 int pte_off_even = sizeof(pte_t) / 2;
                 int pte_off_odd = pte_off_even + sizeof(pte_t);
 
                 /* The pte entries are pre-shifted */
                 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
-                uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
+                UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
                 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
-                uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
+                UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
         }
 #else
         UASM_i_LW(p, tmp, 0, ptep); /* get even pte */
         UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
         if (r45k_bvahwbug())
                 build_tlb_probe_entry(p);
-        UASM_i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
-        if (r4k_250MHZhwbug())
-                uasm_i_mtc0(p, 0, C0_ENTRYLO0);
-        uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
-        UASM_i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
-        if (r45k_bvahwbug())
-                uasm_i_mfc0(p, tmp, C0_INDEX);
+        if (kernel_uses_smartmips_rixi) {
+                UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC));
+                UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC));
+                UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+                if (r4k_250MHZhwbug())
+                        UASM_i_MTC0(p, 0, C0_ENTRYLO0);
+                UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+                UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+        } else {
+                UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */
+                if (r4k_250MHZhwbug())
+                        UASM_i_MTC0(p, 0, C0_ENTRYLO0);
+                UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+                UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */
+                if (r45k_bvahwbug())
+                        uasm_i_mfc0(p, tmp, C0_INDEX);
+        }
         if (r4k_250MHZhwbug())
-                uasm_i_mtc0(p, 0, C0_ENTRYLO1);
-        uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
+                UASM_i_MTC0(p, 0, C0_ENTRYLO1);
+        UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
 #endif
 }
 
@@ -731,10 +855,15 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
          * create the plain linear handler
          */
         if (bcm1250_m3_war()) {
-                UASM_i_MFC0(&p, K0, C0_BADVADDR);
-                UASM_i_MFC0(&p, K1, C0_ENTRYHI);
+                unsigned int segbits = 44;
+
+                uasm_i_dmfc0(&p, K0, C0_BADVADDR);
+                uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
                 uasm_i_xor(&p, K0, K0, K1);
-                UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
+                uasm_i_dsrl_safe(&p, K1, K0, 62);
+                uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
+                uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
+                uasm_i_or(&p, K0, K0, K1);
                 uasm_il_bnez(&p, &r, K0, label_leave);
                 /* No need for uasm_i_nop */
         }
@@ -763,7 +892,7 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
 #endif
 
 #ifdef CONFIG_64BIT
-        build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
+        build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, refill);
 #endif
 
         /*
@@ -802,8 +931,6 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
         } else {
 #if defined(CONFIG_HUGETLB_PAGE)
                 const enum label_id ls = label_tlb_huge_update;
-#elif defined(MODULE_START)
-                const enum label_id ls = label_module_alloc;
 #else
                 const enum label_id ls = label_vmalloc;
 #endif
@@ -875,15 +1002,6 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
 }
 
 /*
- * TLB load/store/modify handlers.
- *
- * Only the fastpath gets synthesized at runtime, the slowpath for
- * do_page_fault remains normal asm.
- */
-extern void tlb_do_page_fault_0(void);
-extern void tlb_do_page_fault_1(void);
-
-/*
  * 128 instructions for the fastpath handler is generous and should
  * never be exceeded.
  */
@@ -977,9 +1095,14 @@ static void __cpuinit
 build_pte_present(u32 **p, struct uasm_reloc **r,
                   unsigned int pte, unsigned int ptr, enum label_id lid)
 {
-        uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
-        uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
-        uasm_il_bnez(p, r, pte, lid);
+        if (kernel_uses_smartmips_rixi) {
+                uasm_i_andi(p, pte, pte, _PAGE_PRESENT);
+                uasm_il_beqz(p, r, pte, lid);
+        } else {
+                uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
+                uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
+                uasm_il_bnez(p, r, pte, lid);
+        }
         iPTE_LW(p, pte, ptr);
 }
 
@@ -1033,6 +1156,7 @@ build_pte_modifiable(u32 **p, struct uasm_reloc **r,
         iPTE_LW(p, pte, ptr);
 }
 
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 /*
  * R3000 style TLB load/store/modify handlers.
  */
@@ -1184,6 +1308,7 @@ static void __cpuinit build_r3000_tlb_modify_handler(void)
 
         dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
 }
+#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
 
 /*
  * R4000 style TLB load/store/modify handlers.
@@ -1235,7 +1360,7 @@ build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
         uasm_i_eret(p); /* return from trap */
 
 #ifdef CONFIG_64BIT
-        build_get_pgd_vmalloc64(p, l, r, tmp, ptr);
+        build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill);
 #endif
 }
 
@@ -1250,10 +1375,15 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
         memset(relocs, 0, sizeof(relocs));
 
         if (bcm1250_m3_war()) {
-                UASM_i_MFC0(&p, K0, C0_BADVADDR);
-                UASM_i_MFC0(&p, K1, C0_ENTRYHI);
+                unsigned int segbits = 44;
+
+                uasm_i_dmfc0(&p, K0, C0_BADVADDR);
+                uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
                 uasm_i_xor(&p, K0, K0, K1);
-                UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
+                uasm_i_dsrl_safe(&p, K1, K0, 62);
+                uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
+                uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
+                uasm_i_or(&p, K0, K0, K1);
                 uasm_il_bnez(&p, &r, K0, label_leave);
                 /* No need for uasm_i_nop */
         }
@@ -1262,6 +1392,34 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
         build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
         if (m4kc_tlbp_war())
                 build_tlb_probe_entry(&p);
+
+        if (kernel_uses_smartmips_rixi) {
+                /*
+                 * If the page is not _PAGE_VALID, RI or XI could not
+                 * have triggered it.  Skip the expensive test..
+                 */
+                uasm_i_andi(&p, K0, K0, _PAGE_VALID);
+                uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1);
+                uasm_i_nop(&p);
+
+                uasm_i_tlbr(&p);
+                /* Examine  entrylo 0 or 1 based on ptr. */
+                uasm_i_andi(&p, K0, K1, sizeof(pte_t));
+                uasm_i_beqz(&p, K0, 8);
+
+                UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
+                UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
+                /*
+                 * If the entryLo (now in K0) is valid (bit 1), RI or
+                 * XI must have triggered it.
+                 */
+                uasm_i_andi(&p, K0, K0, 2);
+                uasm_il_bnez(&p, &r, K0, label_nopage_tlbl);
+
+                uasm_l_tlbl_goaround1(&l, p);
+                /* Reload the PTE value */
+                iPTE_LW(&p, K0, K1);
+        }
         build_make_valid(&p, &r, K0, K1);
         build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
 
@@ -1274,6 +1432,40 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
         iPTE_LW(&p, K0, K1);
         build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
         build_tlb_probe_entry(&p);
+
+        if (kernel_uses_smartmips_rixi) {
+                /*
+                 * If the page is not _PAGE_VALID, RI or XI could not
+                 * have triggered it.  Skip the expensive test..
+                 */
+                uasm_i_andi(&p, K0, K0, _PAGE_VALID);
+                uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
+                uasm_i_nop(&p);
+
+                uasm_i_tlbr(&p);
+                /* Examine  entrylo 0 or 1 based on ptr. */
+                uasm_i_andi(&p, K0, K1, sizeof(pte_t));
+                uasm_i_beqz(&p, K0, 8);
+
+                UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
+                UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
+                /*
+                 * If the entryLo (now in K0) is valid (bit 1), RI or
+                 * XI must have triggered it.
+                 */
+                uasm_i_andi(&p, K0, K0, 2);
+                uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
+                /* Reload the PTE value */
+                iPTE_LW(&p, K0, K1);
+
+                /*
+                 * We clobbered C0_PAGEMASK, restore it.  On the other branch
+                 * it is restored in build_huge_tlb_write_entry.
+                 */
+                build_restore_pagemask(&p, &r, K0, label_nopage_tlbl);
+
+                uasm_l_tlbl_goaround2(&l, p);
+        }
         uasm_i_ori(&p, K0, K0, (_PAGE_ACCESSED | _PAGE_VALID));
         build_huge_handler_tail(&p, &r, &l, K0, K1);
 #endif
@@ -1392,6 +1584,10 @@ void __cpuinit build_tlb_refill_handler(void)
          */
         static int run_once = 0;
 
+#ifdef CONFIG_64BIT
+        check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
+#endif
+
         switch (current_cpu_type()) {
         case CPU_R2000:
         case CPU_R3000:
@@ -1400,6 +1596,7 @@ void __cpuinit build_tlb_refill_handler(void)
         case CPU_TX3912:
         case CPU_TX3922:
         case CPU_TX3927:
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
                 build_r3000_tlb_refill_handler();
                 if (!run_once) {
                         build_r3000_tlb_load_handler();
@@ -1407,6 +1604,9 @@ void __cpuinit build_tlb_refill_handler(void)
                         build_r3000_tlb_modify_handler();
                         run_once++;
                 }
+#else
+                panic("No R3000 TLB refill handler");
+#endif
                 break;
 
         case CPU_R6000: