MIPS: Loongson: Invalidate special TLBs when needed

Loongson-2 has a 4 entry itlb which is a subset of jtlb, Loongson-3 has
a 4 entry itlb and a 4 entry dtlb which are subsets of jtlb. We should
write diag register to invalidate itlb/dtlb when flushing jtlb because
itlb/dtlb are not totally transparent to software.

For Loongson-3A R2 (and newer), we should invalidate ITLB, DTLB, VTLB
and FTLB before we enable/disable FTLB.

Signed-off-by: Huacai Chen <chenhc@lemote.com>
Cc: Aurelien Jarno <aurelien@aurel32.net>
Cc: Steven J . Hill <sjhill@realitydiluted.com>
Cc: Fuxin Zhang <zhangfx@lemote.com>
Cc: Zhangjin Wu <wuzhangjin@gmail.com>
Cc: linux-mips@linux-mips.org
Patchwork: https://patchwork.linux-mips.org/patch/12753/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>

3 files changed, 27 insertions, 12 deletions

diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h
index 28ded49d25c6..0d0bd161fc10 100644
--- a/arch/mips/include/asm/mipsregs.h
+++ b/arch/mips/include/asm/mipsregs.h
@@ -780,6 +780,15 @@
 /* Disable Branch Return Cache */
 #define R10K_DIAG_D_BRC         (_ULCAST_(1) << 22)
 
+/* Flush ITLB */
+#define LOONGSON_DIAG_ITLB      (_ULCAST_(1) << 2)
+/* Flush DTLB */
+#define LOONGSON_DIAG_DTLB      (_ULCAST_(1) << 3)
+/* Flush VTLB */
+#define LOONGSON_DIAG_VTLB      (_ULCAST_(1) << 12)
+/* Flush FTLB */
+#define LOONGSON_DIAG_FTLB      (_ULCAST_(1) << 13)
+
 /*
  * Coprocessor 1 (FPU) register names
  */
diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c
index 2bfd48375b46..515dd1f8901e 100644
--- a/arch/mips/kernel/cpu-probe.c
+++ b/arch/mips/kernel/cpu-probe.c
@@ -563,6 +563,9 @@ static int set_ftlb_enable(struct cpuinfo_mips *c, int enable)
                                            << MIPS_CONF7_FTLBP_SHIFT));
                 break;
         case CPU_LOONGSON3:
+                /* Flush ITLB, DTLB, VTLB and FTLB */
+                write_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB |
+                              LOONGSON_DIAG_VTLB | LOONGSON_DIAG_FTLB);
                 /* Loongson-3 cores use Config6 to enable the FTLB */
                 config = read_c0_config6();
                 if (enable)
diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c
index c17d7627f872..4c3362b46066 100644
--- a/arch/mips/mm/tlb-r4k.c
+++ b/arch/mips/mm/tlb-r4k.c
@@ -28,25 +28,28 @@
 extern void build_tlb_refill_handler(void);
 
 /*
- * LOONGSON2/3 has a 4 entry itlb which is a subset of dtlb,
- * unfortunately, itlb is not totally transparent to software.
+ * LOONGSON-2 has a 4 entry itlb which is a subset of jtlb, LOONGSON-3 has
+ * a 4 entry itlb and a 4 entry dtlb which are subsets of jtlb. Unfortunately,
+ * itlb/dtlb are not totally transparent to software.
  */
-static inline void flush_itlb(void)
+static inline void flush_micro_tlb(void)
 {
         switch (current_cpu_type()) {
         case CPU_LOONGSON2:
+                write_c0_diag(LOONGSON_DIAG_ITLB);
+                break;
         case CPU_LOONGSON3:
-                write_c0_diag(4);
+                write_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB);
                 break;
         default:
                 break;
         }
 }
 
-static inline void flush_itlb_vm(struct vm_area_struct *vma)
+static inline void flush_micro_tlb_vm(struct vm_area_struct *vma)
 {
         if (vma->vm_flags & VM_EXEC)
-                flush_itlb();
+                flush_micro_tlb();
 }
 
 void local_flush_tlb_all(void)
@@ -93,7 +96,7 @@ void local_flush_tlb_all(void)
         tlbw_use_hazard();
         write_c0_entryhi(old_ctx);
         htw_start();
-        flush_itlb();
+        flush_micro_tlb();
         local_irq_restore(flags);
 }
 EXPORT_SYMBOL(local_flush_tlb_all);
@@ -159,7 +162,7 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
                 } else {
                         drop_mmu_context(mm, cpu);
                 }
-                flush_itlb();
+                flush_micro_tlb();
                 local_irq_restore(flags);
         }
 }
@@ -205,7 +208,7 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
         } else {
                 local_flush_tlb_all();
         }
-        flush_itlb();
+        flush_micro_tlb();
         local_irq_restore(flags);
 }
 
@@ -240,7 +243,7 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
         finish:
                 write_c0_entryhi(oldpid);
                 htw_start();
-                flush_itlb_vm(vma);
+                flush_micro_tlb_vm(vma);
                 local_irq_restore(flags);
         }
 }
@@ -274,7 +277,7 @@ void local_flush_tlb_one(unsigned long page)
         }
         write_c0_entryhi(oldpid);
         htw_start();
-        flush_itlb();
+        flush_micro_tlb();
         local_irq_restore(flags);
 }
 
@@ -357,7 +360,7 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
         }
         tlbw_use_hazard();
         htw_start();
-        flush_itlb_vm(vma);
+        flush_micro_tlb_vm(vma);
         local_irq_restore(flags);
 }