[MIPS] Fix potential latency problem due to non-atomic cpu_wait.

If an interrupt happened between checking of NEED_RESCHED and WAIT
instruction, adjust EPC to restart from checking of NEED_RESCHED.

Signed-off-by: Atsushi Nemoto <anemo@mba.ocn.ne.jp>
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>

1 files changed, 37 insertions, 0 deletions

diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S
index c6ada98ee042..f886dd7f708e 100644
--- a/arch/mips/kernel/genex.S
+++ b/arch/mips/kernel/genex.S
@@ -20,6 +20,7 @@
 #include <asm/stackframe.h>
 #include <asm/war.h>
 #include <asm/page.h>
+#include <asm/thread_info.h>
 
 #define PANIC_PIC(msg)                                  \
                 .set push;                              \
@@ -126,7 +127,42 @@ handle_vcei:
 
         __FINIT
 
+        .align  5       /* 32 byte rollback region */
+LEAF(r4k_wait)
+        .set    push
+        .set    noreorder
+        /* start of rollback region */
+        LONG_L  t0, TI_FLAGS($28)
+        nop
+        andi    t0, _TIF_NEED_RESCHED
+        bnez    t0, 1f
+         nop
+        nop
+        nop
+        .set    mips3
+        wait
+        /* end of rollback region (the region size must be power of two) */
+        .set    pop
+1:
+        jr      ra
+        END(r4k_wait)
+
+        .macro  BUILD_ROLLBACK_PROLOGUE handler
+        FEXPORT(rollback_\handler)
+        .set    push
+        .set    noat
+        MFC0    k0, CP0_EPC
+        PTR_LA  k1, r4k_wait
+        ori     k0, 0x1f        /* 32 byte rollback region */
+        xori    k0, 0x1f
+        bne     k0, k1, 9f
+        MTC0    k0, CP0_EPC
+9:
+        .set pop
+        .endm
+
         .align  5
+BUILD_ROLLBACK_PROLOGUE handle_int
 NESTED(handle_int, PT_SIZE, sp)
 #ifdef CONFIG_TRACE_IRQFLAGS
         /*
@@ -201,6 +237,7 @@ NESTED(except_vec_ejtag_debug, 0, sp)
  * This prototype is copied to ebase + n*IntCtl.VS and patched
  * to invoke the handler
  */
+BUILD_ROLLBACK_PROLOGUE except_vec_vi
 NESTED(except_vec_vi, 0, sp)
         SAVE_SOME
         SAVE_AT