[SPARC64]: Avoid membar instructions in delay slots.

In particular, avoid membar instructions in the delay
slot of a jmpl instruction.

UltraSPARC-I, II, IIi, and IIe have a bug, documented in
the UltraSPARC-IIi User's Manual, Appendix K, Erratum 51

The long and short of it is that if the IMU unit misses
on a branch or jmpl, and there is a store buffer synchronizing
membar in the delay slot, the chip can stop fetching instructions.

If interrupts are enabled or some other trap is enabled, the
chip will unwedge itself, but performance will suffer.

We already had a workaround for this bug in a few spots, but
it's better to have the entire tree sanitized for this rule.

Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 20 insertions, 11 deletions

diff --git a/arch/sparc64/lib/bitops.S b/arch/sparc64/lib/bitops.S
index 886dcd2b376a..31afbfe6c1e8 100644
--- a/arch/sparc64/lib/bitops.S
+++ b/arch/sparc64/lib/bitops.S
@@ -7,20 +7,26 @@
 #include <linux/config.h>
 #include <asm/asi.h>
 
+        .text
+
         /* On SMP we need to use memory barriers to ensure
          * correct memory operation ordering, nop these out
          * for uniprocessor.
          */
+
 #ifdef CONFIG_SMP
 #define BITOP_PRE_BARRIER       membar #StoreLoad | #LoadLoad
-#define BITOP_POST_BARRIER      membar #StoreLoad | #StoreStore
+#define BITOP_POST_BARRIER      \
+        ba,pt   %xcc, 80b;      \
+        membar #StoreLoad | #StoreStore
+
+80:     retl
+         nop
 #else
-#define BITOP_PRE_BARRIER       nop
-#define BITOP_POST_BARRIER      nop
+#define BITOP_PRE_BARRIER
+#define BITOP_POST_BARRIER
 #endif
 
-        .text
-
         .globl  test_and_set_bit
         .type   test_and_set_bit,#function
 test_and_set_bit:       /* %o0=nr, %o1=addr */
@@ -37,10 +43,11 @@ test_and_set_bit:	/* %o0=nr, %o1=addr */
         cmp     %g7, %g1
         bne,pn  %xcc, 1b
          and    %g7, %o2, %g2
-        BITOP_POST_BARRIER
         clr     %o0
+        movrne  %g2, 1, %o0
+        BITOP_POST_BARRIER
         retl
-         movrne %g2, 1, %o0
+         nop
         .size   test_and_set_bit, .-test_and_set_bit
 
         .globl  test_and_clear_bit
@@ -59,10 +66,11 @@ test_and_clear_bit:	/* %o0=nr, %o1=addr */
         cmp     %g7, %g1
         bne,pn  %xcc, 1b
          and    %g7, %o2, %g2
-        BITOP_POST_BARRIER
         clr     %o0
+        movrne  %g2, 1, %o0
+        BITOP_POST_BARRIER
         retl
-         movrne %g2, 1, %o0
+         nop
         .size   test_and_clear_bit, .-test_and_clear_bit
 
         .globl  test_and_change_bit
@@ -81,10 +89,11 @@ test_and_change_bit:	/* %o0=nr, %o1=addr */
         cmp     %g7, %g1
         bne,pn  %xcc, 1b
          and    %g7, %o2, %g2
-        BITOP_POST_BARRIER
         clr     %o0
+        movrne  %g2, 1, %o0
+        BITOP_POST_BARRIER
         retl
-         movrne %g2, 1, %o0
+         nop
         .size   test_and_change_bit, .-test_and_change_bit
 
         .globl  set_bit