sparc64: Make lock backoff really a NOP on UP builds.

As noticed by Mikulas Patocka, the backoff macros don't
completely nop out for UP builds, we still get a
branch always and a delay slot nop.

Fix this by making the branch to the backoff spin loop
selective, then we can nop out the spin loop completely.

Signed-off-by: David S. Miller <davem@davemloft.net>

3 files changed, 22 insertions, 17 deletions

diff --git a/arch/sparc/include/asm/backoff.h b/arch/sparc/include/asm/backoff.h
index fa1fdf67e350..db3af0d30fb1 100644
--- a/arch/sparc/include/asm/backoff.h
+++ b/arch/sparc/include/asm/backoff.h
@@ -8,6 +8,9 @@
 #define BACKOFF_SETUP(reg)      \
         mov     1, reg
 
+#define BACKOFF_LABEL(spin_label, continue_label) \
+        spin_label
+
 #define BACKOFF_SPIN(reg, tmp, label)   \
         mov     reg, tmp; \
 88:     brnz,pt tmp, 88b; \
@@ -22,9 +25,11 @@
 #else
 
 #define BACKOFF_SETUP(reg)
-#define BACKOFF_SPIN(reg, tmp, label) \
-        ba,pt   %xcc, label; \
-         nop;
+
+#define BACKOFF_LABEL(spin_label, continue_label) \
+        continue_label
+
+#define BACKOFF_SPIN(reg, tmp, label)
 
 #endif
 
diff --git a/arch/sparc/lib/atomic_64.S b/arch/sparc/lib/atomic_64.S
index 703c9c3935b7..59186e0fcf39 100644
--- a/arch/sparc/lib/atomic_64.S
+++ b/arch/sparc/lib/atomic_64.S
@@ -21,7 +21,7 @@ atomic_add: /* %o0 = increment, %o1 = atomic_ptr */
         add     %g1, %o0, %g7
         cas     [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %icc, 2f
+        bne,pn  %icc, BACKOFF_LABEL(2f, 1b)
          nop
         retl
          nop
@@ -36,7 +36,7 @@ atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
         sub     %g1, %o0, %g7
         cas     [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %icc, 2f
+        bne,pn  %icc, BACKOFF_LABEL(2f, 1b)
          nop
         retl
          nop
@@ -51,7 +51,7 @@ atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
         add     %g1, %o0, %g7
         cas     [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %icc, 2f
+        bne,pn  %icc, BACKOFF_LABEL(2f, 1b)
          add    %g1, %o0, %g1
         retl
          sra    %g1, 0, %o0
@@ -66,7 +66,7 @@ atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
         sub     %g1, %o0, %g7
         cas     [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %icc, 2f
+        bne,pn  %icc, BACKOFF_LABEL(2f, 1b)
          sub    %g1, %o0, %g1
         retl
          sra    %g1, 0, %o0
@@ -81,7 +81,7 @@ atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */
         add     %g1, %o0, %g7
         casx    [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %xcc, 2f
+        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b)
          nop
         retl
          nop
@@ -96,7 +96,7 @@ atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */
         sub     %g1, %o0, %g7
         casx    [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %xcc, 2f
+        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b)
          nop
         retl
          nop
@@ -111,7 +111,7 @@ atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
         add     %g1, %o0, %g7
         casx    [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %xcc, 2f
+        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b)
          nop
         retl
          add    %g1, %o0, %o0
@@ -126,7 +126,7 @@ atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
         sub     %g1, %o0, %g7
         casx    [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %xcc, 2f
+        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b)
          nop
         retl
          sub    %g1, %o0, %o0
diff --git a/arch/sparc/lib/bitops.S b/arch/sparc/lib/bitops.S
index 2b7228cb8c22..3dc61d5537c0 100644
--- a/arch/sparc/lib/bitops.S
+++ b/arch/sparc/lib/bitops.S
@@ -22,7 +22,7 @@ test_and_set_bit:	/* %o0=nr, %o1=addr */
         or      %g7, %o2, %g1
         casx    [%o1], %g7, %g1
         cmp     %g7, %g1
-        bne,pn  %xcc, 2f
+        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b)
          and    %g7, %o2, %g2
         clr     %o0
         movrne  %g2, 1, %o0
@@ -45,7 +45,7 @@ test_and_clear_bit:	/* %o0=nr, %o1=addr */
         andn    %g7, %o2, %g1
         casx    [%o1], %g7, %g1
         cmp     %g7, %g1
-        bne,pn  %xcc, 2f
+        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b)
          and    %g7, %o2, %g2
         clr     %o0
         movrne  %g2, 1, %o0
@@ -68,7 +68,7 @@ test_and_change_bit:	/* %o0=nr, %o1=addr */
         xor     %g7, %o2, %g1
         casx    [%o1], %g7, %g1
         cmp     %g7, %g1
-        bne,pn  %xcc, 2f
+        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b)
          and    %g7, %o2, %g2
         clr     %o0
         movrne  %g2, 1, %o0
@@ -91,7 +91,7 @@ set_bit:		/* %o0=nr, %o1=addr */
         or      %g7, %o2, %g1
         casx    [%o1], %g7, %g1
         cmp     %g7, %g1
-        bne,pn  %xcc, 2f
+        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b)
          nop
         retl
          nop
@@ -112,7 +112,7 @@ clear_bit:		/* %o0=nr, %o1=addr */
         andn    %g7, %o2, %g1
         casx    [%o1], %g7, %g1
         cmp     %g7, %g1
-        bne,pn  %xcc, 2f
+        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b)
          nop
         retl
          nop
@@ -133,7 +133,7 @@ change_bit:		/* %o0=nr, %o1=addr */
         xor     %g7, %o2, %g1
         casx    [%o1], %g7, %g1
         cmp     %g7, %g1
-        bne,pn  %xcc, 2f
+        bne,pn  %xcc, BACKOFF_LABEL(2f, 1b)
          nop
         retl
          nop