aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2005-08-29 15:46:22 -0400
committerDavid S. Miller <davem@davemloft.net>2005-08-29 15:46:22 -0400
commit4f07118f656c179740cad35b827032e2e29b1210 (patch)
tree7ddeb17346fe25ae75aa5373659c053afb9ef5f5
parent442464a50077ff00454ff8d7628cbe1b8eacc034 (diff)
[SPARC64]: More fully work around Spitfire Errata 51.
It appears that a memory barrier soon after a mispredicted branch, not just in the delay slot, can cause the hang condition of this cpu errata. So move them out-of-line, and explicitly put them into a "branch always, predict taken" delay slot which should fully kill this problem. Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat
-rw-r--r--arch/sparc64/kernel/pci_iommu.c2
-rw-r--r--arch/sparc64/kernel/process.c2
-rw-r--r--arch/sparc64/kernel/sbus.c2
-rw-r--r--arch/sparc64/kernel/signal32.c22
-rw-r--r--arch/sparc64/kernel/smp.c30
-rw-r--r--arch/sparc64/kernel/sparc64_ksyms.c9
-rw-r--r--arch/sparc64/lib/Makefile2
-rw-r--r--arch/sparc64/lib/debuglocks.c16
-rw-r--r--arch/sparc64/lib/mb.S73
-rw-r--r--arch/sparc64/solaris/misc.c6
-rw-r--r--include/asm-sparc64/atomic.h8
-rw-r--r--include/asm-sparc64/bitops.h4
-rw-r--r--include/asm-sparc64/spinlock.h4
-rw-r--r--include/asm-sparc64/system.h17
14 files changed, 143 insertions, 54 deletions
diff --git a/arch/sparc64/kernel/pci_iommu.c b/arch/sparc64/kernel/pci_iommu.c
index 2803bc7c2c79..425c60cfea19 100644
--- a/arch/sparc64/kernel/pci_iommu.c
+++ b/arch/sparc64/kernel/pci_iommu.c
@@ -466,7 +466,7 @@ do_flush_sync:
466 if (!limit) 466 if (!limit)
467 break; 467 break;
468 udelay(1); 468 udelay(1);
469 membar("#LoadLoad"); 469 rmb();
470 } 470 }
471 if (!limit) 471 if (!limit)
472 printk(KERN_WARNING "pci_strbuf_flush: flushflag timeout " 472 printk(KERN_WARNING "pci_strbuf_flush: flushflag timeout "
diff --git a/arch/sparc64/kernel/process.c b/arch/sparc64/kernel/process.c
index 07424b075938..66255434128a 100644
--- a/arch/sparc64/kernel/process.c
+++ b/arch/sparc64/kernel/process.c
@@ -103,7 +103,7 @@ void cpu_idle(void)
103 * other cpus see our increasing idleness for the buddy 103 * other cpus see our increasing idleness for the buddy
104 * redistribution algorithm. -DaveM 104 * redistribution algorithm. -DaveM
105 */ 105 */
106 membar("#StoreStore | #StoreLoad"); 106 membar_storeload_storestore();
107 } 107 }
108} 108}
109 109
diff --git a/arch/sparc64/kernel/sbus.c b/arch/sparc64/kernel/sbus.c
index 89f5e019f24c..e09ddf927655 100644
--- a/arch/sparc64/kernel/sbus.c
+++ b/arch/sparc64/kernel/sbus.c
@@ -147,7 +147,7 @@ static void sbus_strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long
147 if (!limit) 147 if (!limit)
148 break; 148 break;
149 udelay(1); 149 udelay(1);
150 membar("#LoadLoad"); 150 rmb();
151 } 151 }
152 if (!limit) 152 if (!limit)
153 printk(KERN_WARNING "sbus_strbuf_flush: flushflag timeout " 153 printk(KERN_WARNING "sbus_strbuf_flush: flushflag timeout "
diff --git a/arch/sparc64/kernel/signal32.c b/arch/sparc64/kernel/signal32.c
index b1ed23091fbb..aecccd0df1d1 100644
--- a/arch/sparc64/kernel/signal32.c
+++ b/arch/sparc64/kernel/signal32.c
@@ -877,11 +877,12 @@ static void new_setup_frame32(struct k_sigaction *ka, struct pt_regs *regs,
877 unsigned long page = (unsigned long) 877 unsigned long page = (unsigned long)
878 page_address(pte_page(*ptep)); 878 page_address(pte_page(*ptep));
879 879
880 __asm__ __volatile__( 880 wmb();
881 " membar #StoreStore\n" 881 __asm__ __volatile__("flush %0 + %1"
882 " flush %0 + %1" 882 : /* no outputs */
883 : : "r" (page), "r" (address & (PAGE_SIZE - 1)) 883 : "r" (page),
884 : "memory"); 884 "r" (address & (PAGE_SIZE - 1))
885 : "memory");
885 } 886 }
886 pte_unmap(ptep); 887 pte_unmap(ptep);
887 preempt_enable(); 888 preempt_enable();
@@ -1292,11 +1293,12 @@ static void setup_rt_frame32(struct k_sigaction *ka, struct pt_regs *regs,
1292 unsigned long page = (unsigned long) 1293 unsigned long page = (unsigned long)
1293 page_address(pte_page(*ptep)); 1294 page_address(pte_page(*ptep));
1294 1295
1295 __asm__ __volatile__( 1296 wmb();
1296 " membar #StoreStore\n" 1297 __asm__ __volatile__("flush %0 + %1"
1297 " flush %0 + %1" 1298 : /* no outputs */
1298 : : "r" (page), "r" (address & (PAGE_SIZE - 1)) 1299 : "r" (page),
1299 : "memory"); 1300 "r" (address & (PAGE_SIZE - 1))
1301 : "memory");
1300 } 1302 }
1301 pte_unmap(ptep); 1303 pte_unmap(ptep);
1302 preempt_enable(); 1304 preempt_enable();
diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c
index b9b42491e118..b4fc6a5462b2 100644
--- a/arch/sparc64/kernel/smp.c
+++ b/arch/sparc64/kernel/smp.c
@@ -144,7 +144,7 @@ void __init smp_callin(void)
144 current->active_mm = &init_mm; 144 current->active_mm = &init_mm;
145 145
146 while (!cpu_isset(cpuid, smp_commenced_mask)) 146 while (!cpu_isset(cpuid, smp_commenced_mask))
147 membar("#LoadLoad"); 147 rmb();
148 148
149 cpu_set(cpuid, cpu_online_map); 149 cpu_set(cpuid, cpu_online_map);
150} 150}
@@ -184,11 +184,11 @@ static inline long get_delta (long *rt, long *master)
184 for (i = 0; i < NUM_ITERS; i++) { 184 for (i = 0; i < NUM_ITERS; i++) {
185 t0 = tick_ops->get_tick(); 185 t0 = tick_ops->get_tick();
186 go[MASTER] = 1; 186 go[MASTER] = 1;
187 membar("#StoreLoad"); 187 membar_storeload();
188 while (!(tm = go[SLAVE])) 188 while (!(tm = go[SLAVE]))
189 membar("#LoadLoad"); 189 rmb();
190 go[SLAVE] = 0; 190 go[SLAVE] = 0;
191 membar("#StoreStore"); 191 wmb();
192 t1 = tick_ops->get_tick(); 192 t1 = tick_ops->get_tick();
193 193
194 if (t1 - t0 < best_t1 - best_t0) 194 if (t1 - t0 < best_t1 - best_t0)
@@ -221,7 +221,7 @@ void smp_synchronize_tick_client(void)
221 go[MASTER] = 1; 221 go[MASTER] = 1;
222 222
223 while (go[MASTER]) 223 while (go[MASTER])
224 membar("#LoadLoad"); 224 rmb();
225 225
226 local_irq_save(flags); 226 local_irq_save(flags);
227 { 227 {
@@ -273,21 +273,21 @@ static void smp_synchronize_one_tick(int cpu)
273 273
274 /* wait for client to be ready */ 274 /* wait for client to be ready */
275 while (!go[MASTER]) 275 while (!go[MASTER])
276 membar("#LoadLoad"); 276 rmb();
277 277
278 /* now let the client proceed into his loop */ 278 /* now let the client proceed into his loop */
279 go[MASTER] = 0; 279 go[MASTER] = 0;
280 membar("#StoreLoad"); 280 membar_storeload();
281 281
282 spin_lock_irqsave(&itc_sync_lock, flags); 282 spin_lock_irqsave(&itc_sync_lock, flags);
283 { 283 {
284 for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) { 284 for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) {
285 while (!go[MASTER]) 285 while (!go[MASTER])
286 membar("#LoadLoad"); 286 rmb();
287 go[MASTER] = 0; 287 go[MASTER] = 0;
288 membar("#StoreStore"); 288 wmb();
289 go[SLAVE] = tick_ops->get_tick(); 289 go[SLAVE] = tick_ops->get_tick();
290 membar("#StoreLoad"); 290 membar_storeload();
291 } 291 }
292 } 292 }
293 spin_unlock_irqrestore(&itc_sync_lock, flags); 293 spin_unlock_irqrestore(&itc_sync_lock, flags);
@@ -927,11 +927,11 @@ void smp_capture(void)
927 smp_processor_id()); 927 smp_processor_id());
928#endif 928#endif
929 penguins_are_doing_time = 1; 929 penguins_are_doing_time = 1;
930 membar("#StoreStore | #LoadStore"); 930 membar_storestore_loadstore();
931 atomic_inc(&smp_capture_registry); 931 atomic_inc(&smp_capture_registry);
932 smp_cross_call(&xcall_capture, 0, 0, 0); 932 smp_cross_call(&xcall_capture, 0, 0, 0);
933 while (atomic_read(&smp_capture_registry) != ncpus) 933 while (atomic_read(&smp_capture_registry) != ncpus)
934 membar("#LoadLoad"); 934 rmb();
935#ifdef CAPTURE_DEBUG 935#ifdef CAPTURE_DEBUG
936 printk("done\n"); 936 printk("done\n");
937#endif 937#endif
@@ -947,7 +947,7 @@ void smp_release(void)
947 smp_processor_id()); 947 smp_processor_id());
948#endif 948#endif
949 penguins_are_doing_time = 0; 949 penguins_are_doing_time = 0;
950 membar("#StoreStore | #StoreLoad"); 950 membar_storeload_storestore();
951 atomic_dec(&smp_capture_registry); 951 atomic_dec(&smp_capture_registry);
952 } 952 }
953} 953}
@@ -970,9 +970,9 @@ void smp_penguin_jailcell(int irq, struct pt_regs *regs)
970 save_alternate_globals(global_save); 970 save_alternate_globals(global_save);
971 prom_world(1); 971 prom_world(1);
972 atomic_inc(&smp_capture_registry); 972 atomic_inc(&smp_capture_registry);
973 membar("#StoreLoad | #StoreStore"); 973 membar_storeload_storestore();
974 while (penguins_are_doing_time) 974 while (penguins_are_doing_time)
975 membar("#LoadLoad"); 975 rmb();
976 restore_alternate_globals(global_save); 976 restore_alternate_globals(global_save);
977 atomic_dec(&smp_capture_registry); 977 atomic_dec(&smp_capture_registry);
978 prom_world(0); 978 prom_world(0);
diff --git a/arch/sparc64/kernel/sparc64_ksyms.c b/arch/sparc64/kernel/sparc64_ksyms.c
index 0764b9330784..a3ea697f1adb 100644
--- a/arch/sparc64/kernel/sparc64_ksyms.c
+++ b/arch/sparc64/kernel/sparc64_ksyms.c
@@ -406,3 +406,12 @@ EXPORT_SYMBOL(xor_vis_4);
406EXPORT_SYMBOL(xor_vis_5); 406EXPORT_SYMBOL(xor_vis_5);
407 407
408EXPORT_SYMBOL(prom_palette); 408EXPORT_SYMBOL(prom_palette);
409
410/* memory barriers */
411EXPORT_SYMBOL(mb);
412EXPORT_SYMBOL(rmb);
413EXPORT_SYMBOL(wmb);
414EXPORT_SYMBOL(membar_storeload);
415EXPORT_SYMBOL(membar_storeload_storestore);
416EXPORT_SYMBOL(membar_storeload_loadload);
417EXPORT_SYMBOL(membar_storestore_loadstore);
diff --git a/arch/sparc64/lib/Makefile b/arch/sparc64/lib/Makefile
index 40dbeec7e5d6..6201f1040982 100644
--- a/arch/sparc64/lib/Makefile
+++ b/arch/sparc64/lib/Makefile
@@ -12,7 +12,7 @@ lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \
12 U1memcpy.o U1copy_from_user.o U1copy_to_user.o \ 12 U1memcpy.o U1copy_from_user.o U1copy_to_user.o \
13 U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \ 13 U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \
14 copy_in_user.o user_fixup.o memmove.o \ 14 copy_in_user.o user_fixup.o memmove.o \
15 mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o 15 mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o mb.o
16 16
17lib-$(CONFIG_DEBUG_SPINLOCK) += debuglocks.o 17lib-$(CONFIG_DEBUG_SPINLOCK) += debuglocks.o
18lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o 18lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
diff --git a/arch/sparc64/lib/debuglocks.c b/arch/sparc64/lib/debuglocks.c
index 7f6ccc4114c5..f5f0b5586f01 100644
--- a/arch/sparc64/lib/debuglocks.c
+++ b/arch/sparc64/lib/debuglocks.c
@@ -61,7 +61,7 @@ again:
61 : "=r" (val) 61 : "=r" (val)
62 : "r" (&(lock->lock)) 62 : "r" (&(lock->lock))
63 : "memory"); 63 : "memory");
64 membar("#StoreLoad | #StoreStore"); 64 membar_storeload_storestore();
65 if (val) { 65 if (val) {
66 while (lock->lock) { 66 while (lock->lock) {
67 if (!--stuck) { 67 if (!--stuck) {
@@ -69,7 +69,7 @@ again:
69 show(str, lock, caller); 69 show(str, lock, caller);
70 stuck = INIT_STUCK; 70 stuck = INIT_STUCK;
71 } 71 }
72 membar("#LoadLoad"); 72 rmb();
73 } 73 }
74 goto again; 74 goto again;
75 } 75 }
@@ -90,7 +90,7 @@ int _do_spin_trylock(spinlock_t *lock, unsigned long caller)
90 : "=r" (val) 90 : "=r" (val)
91 : "r" (&(lock->lock)) 91 : "r" (&(lock->lock))
92 : "memory"); 92 : "memory");
93 membar("#StoreLoad | #StoreStore"); 93 membar_storeload_storestore();
94 if (!val) { 94 if (!val) {
95 lock->owner_pc = ((unsigned int)caller); 95 lock->owner_pc = ((unsigned int)caller);
96 lock->owner_cpu = cpu; 96 lock->owner_cpu = cpu;
@@ -107,7 +107,7 @@ void _do_spin_unlock(spinlock_t *lock)
107{ 107{
108 lock->owner_pc = 0; 108 lock->owner_pc = 0;
109 lock->owner_cpu = NO_PROC_ID; 109 lock->owner_cpu = NO_PROC_ID;
110 membar("#StoreStore | #LoadStore"); 110 membar_storestore_loadstore();
111 lock->lock = 0; 111 lock->lock = 0;
112 current->thread.smp_lock_count--; 112 current->thread.smp_lock_count--;
113} 113}
@@ -129,7 +129,7 @@ wlock_again:
129 show_read(str, rw, caller); 129 show_read(str, rw, caller);
130 stuck = INIT_STUCK; 130 stuck = INIT_STUCK;
131 } 131 }
132 membar("#LoadLoad"); 132 rmb();
133 } 133 }
134 /* Try once to increment the counter. */ 134 /* Try once to increment the counter. */
135 __asm__ __volatile__( 135 __asm__ __volatile__(
@@ -142,7 +142,7 @@ wlock_again:
142"2:" : "=r" (val) 142"2:" : "=r" (val)
143 : "0" (&(rw->lock)) 143 : "0" (&(rw->lock))
144 : "g1", "g7", "memory"); 144 : "g1", "g7", "memory");
145 membar("#StoreLoad | #StoreStore"); 145 membar_storeload_storestore();
146 if (val) 146 if (val)
147 goto wlock_again; 147 goto wlock_again;
148 rw->reader_pc[cpu] = ((unsigned int)caller); 148 rw->reader_pc[cpu] = ((unsigned int)caller);
@@ -201,7 +201,7 @@ wlock_again:
201 show_write(str, rw, caller); 201 show_write(str, rw, caller);
202 stuck = INIT_STUCK; 202 stuck = INIT_STUCK;
203 } 203 }
204 membar("#LoadLoad"); 204 rmb();
205 } 205 }
206 206
207 /* Try to acuire the write bit. */ 207 /* Try to acuire the write bit. */
@@ -256,7 +256,7 @@ wlock_again:
256 show_write(str, rw, caller); 256 show_write(str, rw, caller);
257 stuck = INIT_STUCK; 257 stuck = INIT_STUCK;
258 } 258 }
259 membar("#LoadLoad"); 259 rmb();
260 } 260 }
261 goto wlock_again; 261 goto wlock_again;
262 } 262 }
diff --git a/arch/sparc64/lib/mb.S b/arch/sparc64/lib/mb.S
new file mode 100644
index 000000000000..4004f748619f
--- /dev/null
+++ b/arch/sparc64/lib/mb.S
@@ -0,0 +1,73 @@
1/* mb.S: Out of line memory barriers.
2 *
3 * Copyright (C) 2005 David S. Miller (davem@davemloft.net)
4 */
5
6 /* These are here in an effort to more fully work around
7 * Spitfire Errata #51. Essentially, if a memory barrier
8 * occurs soon after a mispredicted branch, the chip can stop
9 * executing instructions until a trap occurs. Therefore, if
10 * interrupts are disabled, the chip can hang forever.
11 *
12 * It used to be believed that the memory barrier had to be
13 * right in the delay slot, but a case has been traced
14 * recently wherein the memory barrier was one instruction
15 * after the branch delay slot and the chip still hung. The
16 * offending sequence was the following in sym_wakeup_done()
17 * of the sym53c8xx_2 driver:
18 *
19 * call sym_ccb_from_dsa, 0
20 * movge %icc, 0, %l0
21 * brz,pn %o0, .LL1303
22 * mov %o0, %l2
23 * membar #LoadLoad
24 *
25 * The branch has to be mispredicted for the bug to occur.
26 * Therefore, we put the memory barrier explicitly into a
27 * "branch always, predicted taken" delay slot to avoid the
28 * problem case.
29 */
30
31 .text
32
3399: retl
34 nop
35
36 .globl mb
37mb: ba,pt %xcc, 99b
38 membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad
39 .size mb, .-mb
40
41 .globl rmb
42rmb: ba,pt %xcc, 99b
43 membar #LoadLoad
44 .size rmb, .-rmb
45
46 .globl wmb
47wmb: ba,pt %xcc, 99b
48 membar #StoreStore
49 .size wmb, .-wmb
50
51 .globl membar_storeload
52membar_storeload:
53 ba,pt %xcc, 99b
54 membar #StoreLoad
55 .size membar_storeload, .-membar_storeload
56
57 .globl membar_storeload_storestore
58membar_storeload_storestore:
59 ba,pt %xcc, 99b
60 membar #StoreLoad | #StoreStore
61 .size membar_storeload_storestore, .-membar_storeload_storestore
62
63 .globl membar_storeload_loadload
64membar_storeload_loadload:
65 ba,pt %xcc, 99b
66 membar #StoreLoad | #LoadLoad
67 .size membar_storeload_loadload, .-membar_storeload_loadload
68
69 .globl membar_storestore_loadstore
70membar_storestore_loadstore:
71 ba,pt %xcc, 99b
72 membar #StoreStore | #LoadStore
73 .size membar_storestore_loadstore, .-membar_storestore_loadstore
diff --git a/arch/sparc64/solaris/misc.c b/arch/sparc64/solaris/misc.c
index 15b4cfe07557..302efbcba70e 100644
--- a/arch/sparc64/solaris/misc.c
+++ b/arch/sparc64/solaris/misc.c
@@ -737,7 +737,8 @@ MODULE_LICENSE("GPL");
737extern u32 tl0_solaris[8]; 737extern u32 tl0_solaris[8];
738#define update_ttable(x) \ 738#define update_ttable(x) \
739 tl0_solaris[3] = (((long)(x) - (long)tl0_solaris - 3) >> 2) | 0x40000000; \ 739 tl0_solaris[3] = (((long)(x) - (long)tl0_solaris - 3) >> 2) | 0x40000000; \
740 __asm__ __volatile__ ("membar #StoreStore; flush %0" : : "r" (&tl0_solaris[3])) 740 wmb(); \
741 __asm__ __volatile__ ("flush %0" : : "r" (&tl0_solaris[3]))
741#else 742#else
742#endif 743#endif
743 744
@@ -761,7 +762,8 @@ int init_module(void)
761 entry64_personality_patch |= 762 entry64_personality_patch |=
762 (offsetof(struct task_struct, personality) + 763 (offsetof(struct task_struct, personality) +
763 (sizeof(unsigned long) - 1)); 764 (sizeof(unsigned long) - 1));
764 __asm__ __volatile__("membar #StoreStore; flush %0" 765 wmb();
766 __asm__ __volatile__("flush %0"
765 : : "r" (&entry64_personality_patch)); 767 : : "r" (&entry64_personality_patch));
766 return 0; 768 return 0;
767} 769}
diff --git a/include/asm-sparc64/atomic.h b/include/asm-sparc64/atomic.h
index d80f3379669b..e175afcf2cde 100644
--- a/include/asm-sparc64/atomic.h
+++ b/include/asm-sparc64/atomic.h
@@ -72,10 +72,10 @@ extern int atomic64_sub_ret(int, atomic64_t *);
72 72
73/* Atomic operations are already serializing */ 73/* Atomic operations are already serializing */
74#ifdef CONFIG_SMP 74#ifdef CONFIG_SMP
75#define smp_mb__before_atomic_dec() membar("#StoreLoad | #LoadLoad") 75#define smp_mb__before_atomic_dec() membar_storeload_loadload();
76#define smp_mb__after_atomic_dec() membar("#StoreLoad | #StoreStore") 76#define smp_mb__after_atomic_dec() membar_storeload_storestore();
77#define smp_mb__before_atomic_inc() membar("#StoreLoad | #LoadLoad") 77#define smp_mb__before_atomic_inc() membar_storeload_loadload();
78#define smp_mb__after_atomic_inc() membar("#StoreLoad | #StoreStore") 78#define smp_mb__after_atomic_inc() membar_storeload_storestore();
79#else 79#else
80#define smp_mb__before_atomic_dec() barrier() 80#define smp_mb__before_atomic_dec() barrier()
81#define smp_mb__after_atomic_dec() barrier() 81#define smp_mb__after_atomic_dec() barrier()
diff --git a/include/asm-sparc64/bitops.h b/include/asm-sparc64/bitops.h
index 9c5e71970287..6388b8376c50 100644
--- a/include/asm-sparc64/bitops.h
+++ b/include/asm-sparc64/bitops.h
@@ -72,8 +72,8 @@ static inline int __test_and_change_bit(int nr, volatile unsigned long *addr)
72} 72}
73 73
74#ifdef CONFIG_SMP 74#ifdef CONFIG_SMP
75#define smp_mb__before_clear_bit() membar("#StoreLoad | #LoadLoad") 75#define smp_mb__before_clear_bit() membar_storeload_loadload()
76#define smp_mb__after_clear_bit() membar("#StoreLoad | #StoreStore") 76#define smp_mb__after_clear_bit() membar_storeload_storestore()
77#else 77#else
78#define smp_mb__before_clear_bit() barrier() 78#define smp_mb__before_clear_bit() barrier()
79#define smp_mb__after_clear_bit() barrier() 79#define smp_mb__after_clear_bit() barrier()
diff --git a/include/asm-sparc64/spinlock.h b/include/asm-sparc64/spinlock.h
index d265bf6570fe..a02c4370eb42 100644
--- a/include/asm-sparc64/spinlock.h
+++ b/include/asm-sparc64/spinlock.h
@@ -43,7 +43,7 @@ typedef struct {
43#define spin_is_locked(lp) ((lp)->lock != 0) 43#define spin_is_locked(lp) ((lp)->lock != 0)
44 44
45#define spin_unlock_wait(lp) \ 45#define spin_unlock_wait(lp) \
46do { membar("#LoadLoad"); \ 46do { rmb(); \
47} while((lp)->lock) 47} while((lp)->lock)
48 48
49static inline void _raw_spin_lock(spinlock_t *lock) 49static inline void _raw_spin_lock(spinlock_t *lock)
@@ -129,7 +129,7 @@ typedef struct {
129#define spin_is_locked(__lock) ((__lock)->lock != 0) 129#define spin_is_locked(__lock) ((__lock)->lock != 0)
130#define spin_unlock_wait(__lock) \ 130#define spin_unlock_wait(__lock) \
131do { \ 131do { \
132 membar("#LoadLoad"); \ 132 rmb(); \
133} while((__lock)->lock) 133} while((__lock)->lock)
134 134
135extern void _do_spin_lock(spinlock_t *lock, char *str, unsigned long caller); 135extern void _do_spin_lock(spinlock_t *lock, char *str, unsigned long caller);
diff --git a/include/asm-sparc64/system.h b/include/asm-sparc64/system.h
index ee4bdfc6b88f..5e94c05dc2fc 100644
--- a/include/asm-sparc64/system.h
+++ b/include/asm-sparc64/system.h
@@ -28,6 +28,14 @@ enum sparc_cpu {
28#define ARCH_SUN4C_SUN4 0 28#define ARCH_SUN4C_SUN4 0
29#define ARCH_SUN4 0 29#define ARCH_SUN4 0
30 30
31extern void mb(void);
32extern void rmb(void);
33extern void wmb(void);
34extern void membar_storeload(void);
35extern void membar_storeload_storestore(void);
36extern void membar_storeload_loadload(void);
37extern void membar_storestore_loadstore(void);
38
31#endif 39#endif
32 40
33#define setipl(__new_ipl) \ 41#define setipl(__new_ipl) \
@@ -78,16 +86,11 @@ enum sparc_cpu {
78 86
79#define nop() __asm__ __volatile__ ("nop") 87#define nop() __asm__ __volatile__ ("nop")
80 88
81#define membar(type) __asm__ __volatile__ ("membar " type : : : "memory")
82#define mb() \
83 membar("#LoadLoad | #LoadStore | #StoreStore | #StoreLoad")
84#define rmb() membar("#LoadLoad")
85#define wmb() membar("#StoreStore")
86#define read_barrier_depends() do { } while(0) 89#define read_barrier_depends() do { } while(0)
87#define set_mb(__var, __value) \ 90#define set_mb(__var, __value) \
88 do { __var = __value; membar("#StoreLoad | #StoreStore"); } while(0) 91 do { __var = __value; membar_storeload_storestore(); } while(0)
89#define set_wmb(__var, __value) \ 92#define set_wmb(__var, __value) \
90 do { __var = __value; membar("#StoreStore"); } while(0) 93 do { __var = __value; wmb(); } while(0)
91 94
92#ifdef CONFIG_SMP 95#ifdef CONFIG_SMP
93#define smp_mb() mb() 96#define smp_mb() mb()
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-21 04:38:28 -0400