aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorAtsushi Nemoto <anemo@mba.ocn.ne.jp>2007-11-11 12:05:18 -0500
committerRalf Baechle <ralf@linux-mips.org>2008-09-21 08:52:57 -0400
commitc65a5480ff29198c241f11d39d9e34ab22c72c69 (patch)
treebb6fadfbb27c607b4c99745844339f9f1e740940 /arch
parent6657fe0a0281e06ff46eb42bc18ecb68d8b14b27 (diff)
[MIPS] Fix potential latency problem due to non-atomic cpu_wait.
If an interrupt happened between checking of NEED_RESCHED and WAIT instruction, adjust EPC to restart from checking of NEED_RESCHED. Signed-off-by: Atsushi Nemoto <anemo@mba.ocn.ne.jp> Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
Diffstat (limited to 'arch')
-rw-r--r--arch/mips/kernel/cpu-probe.c16
-rw-r--r--arch/mips/kernel/genex.S37
-rw-r--r--arch/mips/kernel/traps.c22
3 files changed, 55 insertions, 20 deletions
diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c
index 335a6ae3d594..11c92dc53791 100644
--- a/arch/mips/kernel/cpu-probe.c
+++ b/arch/mips/kernel/cpu-probe.c
@@ -45,18 +45,7 @@ static void r39xx_wait(void)
45 local_irq_enable(); 45 local_irq_enable();
46} 46}
47 47
48/* 48extern void r4k_wait(void);
49 * There is a race when WAIT instruction executed with interrupt
50 * enabled.
51 * But it is implementation-dependent wheter the pipelie restarts when
52 * a non-enabled interrupt is requested.
53 */
54static void r4k_wait(void)
55{
56 __asm__(" .set mips3 \n"
57 " wait \n"
58 " .set mips0 \n");
59}
60 49
61/* 50/*
62 * This variant is preferable as it allows testing need_resched and going to 51 * This variant is preferable as it allows testing need_resched and going to
@@ -128,7 +117,7 @@ static int __init wait_disable(char *s)
128 117
129__setup("nowait", wait_disable); 118__setup("nowait", wait_disable);
130 119
131static inline void check_wait(void) 120void __init check_wait(void)
132{ 121{
133 struct cpuinfo_mips *c = &current_cpu_data; 122 struct cpuinfo_mips *c = &current_cpu_data;
134 123
@@ -242,7 +231,6 @@ static inline void check_errata(void)
242 231
243void __init check_bugs32(void) 232void __init check_bugs32(void)
244{ 233{
245 check_wait();
246 check_errata(); 234 check_errata();
247} 235}
248 236
diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S
index c6ada98ee042..f886dd7f708e 100644
--- a/arch/mips/kernel/genex.S
+++ b/arch/mips/kernel/genex.S
@@ -20,6 +20,7 @@
20#include <asm/stackframe.h> 20#include <asm/stackframe.h>
21#include <asm/war.h> 21#include <asm/war.h>
22#include <asm/page.h> 22#include <asm/page.h>
23#include <asm/thread_info.h>
23 24
24#define PANIC_PIC(msg) \ 25#define PANIC_PIC(msg) \
25 .set push; \ 26 .set push; \
@@ -126,7 +127,42 @@ handle_vcei:
126 127
127 __FINIT 128 __FINIT
128 129
130 .align 5 /* 32 byte rollback region */
131LEAF(r4k_wait)
132 .set push
133 .set noreorder
134 /* start of rollback region */
135 LONG_L t0, TI_FLAGS($28)
136 nop
137 andi t0, _TIF_NEED_RESCHED
138 bnez t0, 1f
139 nop
140 nop
141 nop
142 .set mips3
143 wait
144 /* end of rollback region (the region size must be power of two) */
145 .set pop
1461:
147 jr ra
148 END(r4k_wait)
149
150 .macro BUILD_ROLLBACK_PROLOGUE handler
151 FEXPORT(rollback_\handler)
152 .set push
153 .set noat
154 MFC0 k0, CP0_EPC
155 PTR_LA k1, r4k_wait
156 ori k0, 0x1f /* 32 byte rollback region */
157 xori k0, 0x1f
158 bne k0, k1, 9f
159 MTC0 k0, CP0_EPC
1609:
161 .set pop
162 .endm
163
129 .align 5 164 .align 5
165BUILD_ROLLBACK_PROLOGUE handle_int
130NESTED(handle_int, PT_SIZE, sp) 166NESTED(handle_int, PT_SIZE, sp)
131#ifdef CONFIG_TRACE_IRQFLAGS 167#ifdef CONFIG_TRACE_IRQFLAGS
132 /* 168 /*
@@ -201,6 +237,7 @@ NESTED(except_vec_ejtag_debug, 0, sp)
201 * This prototype is copied to ebase + n*IntCtl.VS and patched 237 * This prototype is copied to ebase + n*IntCtl.VS and patched
202 * to invoke the handler 238 * to invoke the handler
203 */ 239 */
240BUILD_ROLLBACK_PROLOGUE except_vec_vi
204NESTED(except_vec_vi, 0, sp) 241NESTED(except_vec_vi, 0, sp)
205 SAVE_SOME 242 SAVE_SOME
206 SAVE_AT 243 SAVE_AT
diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index 6bee29097a56..5fd0cd020af5 100644
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -46,6 +46,9 @@
46#include <asm/types.h> 46#include <asm/types.h>
47#include <asm/stacktrace.h> 47#include <asm/stacktrace.h>
48 48
49extern void check_wait(void);
50extern asmlinkage void r4k_wait(void);
51extern asmlinkage void rollback_handle_int(void);
49extern asmlinkage void handle_int(void); 52extern asmlinkage void handle_int(void);
50extern asmlinkage void handle_tlbm(void); 53extern asmlinkage void handle_tlbm(void);
51extern asmlinkage void handle_tlbl(void); 54extern asmlinkage void handle_tlbl(void);
@@ -1251,6 +1254,9 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs)
1251 1254
1252 extern char except_vec_vi, except_vec_vi_lui; 1255 extern char except_vec_vi, except_vec_vi_lui;
1253 extern char except_vec_vi_ori, except_vec_vi_end; 1256 extern char except_vec_vi_ori, except_vec_vi_end;
1257 extern char rollback_except_vec_vi;
1258 char *vec_start = (cpu_wait == r4k_wait) ?
1259 &rollback_except_vec_vi : &except_vec_vi;
1254#ifdef CONFIG_MIPS_MT_SMTC 1260#ifdef CONFIG_MIPS_MT_SMTC
1255 /* 1261 /*
1256 * We need to provide the SMTC vectored interrupt handler 1262 * We need to provide the SMTC vectored interrupt handler
@@ -1258,11 +1264,11 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs)
1258 * Status.IM bit to be masked before going there. 1264 * Status.IM bit to be masked before going there.
1259 */ 1265 */
1260 extern char except_vec_vi_mori; 1266 extern char except_vec_vi_mori;
1261 const int mori_offset = &except_vec_vi_mori - &except_vec_vi; 1267 const int mori_offset = &except_vec_vi_mori - vec_start;
1262#endif /* CONFIG_MIPS_MT_SMTC */ 1268#endif /* CONFIG_MIPS_MT_SMTC */
1263 const int handler_len = &except_vec_vi_end - &except_vec_vi; 1269 const int handler_len = &except_vec_vi_end - vec_start;
1264 const int lui_offset = &except_vec_vi_lui - &except_vec_vi; 1270 const int lui_offset = &except_vec_vi_lui - vec_start;
1265 const int ori_offset = &except_vec_vi_ori - &except_vec_vi; 1271 const int ori_offset = &except_vec_vi_ori - vec_start;
1266 1272
1267 if (handler_len > VECTORSPACING) { 1273 if (handler_len > VECTORSPACING) {
1268 /* 1274 /*
@@ -1272,7 +1278,7 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs)
1272 panic("VECTORSPACING too small"); 1278 panic("VECTORSPACING too small");
1273 } 1279 }
1274 1280
1275 memcpy(b, &except_vec_vi, handler_len); 1281 memcpy(b, vec_start, handler_len);
1276#ifdef CONFIG_MIPS_MT_SMTC 1282#ifdef CONFIG_MIPS_MT_SMTC
1277 BUG_ON(n > 7); /* Vector index %d exceeds SMTC maximum. */ 1283 BUG_ON(n > 7); /* Vector index %d exceeds SMTC maximum. */
1278 1284
@@ -1554,6 +1560,10 @@ void __init trap_init(void)
1554 extern char except_vec3_generic, except_vec3_r4000; 1560 extern char except_vec3_generic, except_vec3_r4000;
1555 extern char except_vec4; 1561 extern char except_vec4;
1556 unsigned long i; 1562 unsigned long i;
1563 int rollback;
1564
1565 check_wait();
1566 rollback = (cpu_wait == r4k_wait);
1557 1567
1558#if defined(CONFIG_KGDB) 1568#if defined(CONFIG_KGDB)
1559 if (kgdb_early_setup) 1569 if (kgdb_early_setup)
@@ -1618,7 +1628,7 @@ void __init trap_init(void)
1618 if (board_be_init) 1628 if (board_be_init)
1619 board_be_init(); 1629 board_be_init();
1620 1630
1621 set_except_vector(0, handle_int); 1631 set_except_vector(0, rollback ? rollback_handle_int : handle_int);
1622 set_except_vector(1, handle_tlbm); 1632 set_except_vector(1, handle_tlbm);
1623 set_except_vector(2, handle_tlbl); 1633 set_except_vector(2, handle_tlbl);
1624 set_except_vector(3, handle_tlbs); 1634 set_except_vector(3, handle_tlbs);