[SPARC64]: Implement atomic backoff.

When the cpu count is high and contention hits an atomic object, the
processors can synchronize such that some cpus continually get knocked
out and cannot complete the atomic update.

So implement an exponential backoff when SMP.

Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 27 insertions, 11 deletions

diff --git a/arch/sparc64/lib/atomic.S b/arch/sparc64/lib/atomic.S
index 9633750167d..70ac4186f62 100644
--- a/arch/sparc64/lib/atomic.S
+++ b/arch/sparc64/lib/atomic.S
@@ -1,10 +1,10 @@
-/* $Id: atomic.S,v 1.4 2001/11/18 00:12:56 davem Exp $
- * atomic.S: These things are too big to do inline.
+/* atomic.S: These things are too big to do inline.
  *
- * Copyright (C) 1999 David S. Miller (davem@redhat.com)
+ * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net)
  */
 
 #include <asm/asi.h>
+#include <asm/backoff.h>
 
         .text
 
@@ -16,27 +16,31 @@
         .globl  atomic_add
         .type   atomic_add,#function
 atomic_add: /* %o0 = increment, %o1 = atomic_ptr */
+        BACKOFF_SETUP(%o2)
 1:      lduw    [%o1], %g1
         add     %g1, %o0, %g7
         cas     [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %icc, 1b
+        bne,pn  %icc, 2f
          nop
         retl
          nop
+2:      BACKOFF_SPIN(%o2, %o3, 1b)
         .size   atomic_add, .-atomic_add
 
         .globl  atomic_sub
         .type   atomic_sub,#function
 atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
+        BACKOFF_SETUP(%o2)
 1:      lduw    [%o1], %g1
         sub     %g1, %o0, %g7
         cas     [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %icc, 1b
+        bne,pn  %icc, 2f
          nop
         retl
          nop
+2:      BACKOFF_SPIN(%o2, %o3, 1b)
         .size   atomic_sub, .-atomic_sub
 
         /* On SMP we need to use memory barriers to ensure
@@ -60,89 +64,101 @@ atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
         .globl  atomic_add_ret
         .type   atomic_add_ret,#function
 atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
+        BACKOFF_SETUP(%o2)
         ATOMIC_PRE_BARRIER
 1:      lduw    [%o1], %g1
         add     %g1, %o0, %g7
         cas     [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %icc, 1b
+        bne,pn  %icc, 2f
          add    %g7, %o0, %g7
         sra     %g7, 0, %o0
         ATOMIC_POST_BARRIER
         retl
          nop
+2:      BACKOFF_SPIN(%o2, %o3, 1b)
         .size   atomic_add_ret, .-atomic_add_ret
 
         .globl  atomic_sub_ret
         .type   atomic_sub_ret,#function
 atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
+        BACKOFF_SETUP(%o2)
         ATOMIC_PRE_BARRIER
 1:      lduw    [%o1], %g1
         sub     %g1, %o0, %g7
         cas     [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %icc, 1b
+        bne,pn  %icc, 2f
          sub    %g7, %o0, %g7
         sra     %g7, 0, %o0
         ATOMIC_POST_BARRIER
         retl
          nop
+2:      BACKOFF_SPIN(%o2, %o3, 1b)
         .size   atomic_sub_ret, .-atomic_sub_ret
 
         .globl  atomic64_add
         .type   atomic64_add,#function
 atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */
+        BACKOFF_SETUP(%o2)
 1:      ldx     [%o1], %g1
         add     %g1, %o0, %g7
         casx    [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %xcc, 1b
+        bne,pn  %xcc, 2f
          nop
         retl
          nop
+2:      BACKOFF_SPIN(%o2, %o3, 1b)
         .size   atomic64_add, .-atomic64_add
 
         .globl  atomic64_sub
         .type   atomic64_sub,#function
 atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */
+        BACKOFF_SETUP(%o2)
 1:      ldx     [%o1], %g1
         sub     %g1, %o0, %g7
         casx    [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %xcc, 1b
+        bne,pn  %xcc, 2f
          nop
         retl
          nop
+2:      BACKOFF_SPIN(%o2, %o3, 1b)
         .size   atomic64_sub, .-atomic64_sub
 
         .globl  atomic64_add_ret
         .type   atomic64_add_ret,#function
 atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
+        BACKOFF_SETUP(%o2)
         ATOMIC_PRE_BARRIER
 1:      ldx     [%o1], %g1
         add     %g1, %o0, %g7
         casx    [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %xcc, 1b
+        bne,pn  %xcc, 2f
          add    %g7, %o0, %g7
         mov     %g7, %o0
         ATOMIC_POST_BARRIER
         retl
          nop
+2:      BACKOFF_SPIN(%o2, %o3, 1b)
         .size   atomic64_add_ret, .-atomic64_add_ret
 
         .globl  atomic64_sub_ret
         .type   atomic64_sub_ret,#function
 atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
+        BACKOFF_SETUP(%o2)
         ATOMIC_PRE_BARRIER
 1:      ldx     [%o1], %g1
         sub     %g1, %o0, %g7
         casx    [%o1], %g1, %g7
         cmp     %g1, %g7
-        bne,pn  %xcc, 1b
+        bne,pn  %xcc, 2f
          sub    %g7, %o0, %g7
         mov     %g7, %o0
         ATOMIC_POST_BARRIER
         retl
          nop
+2:      BACKOFF_SPIN(%o2, %o3, 1b)
         .size   atomic64_sub_ret, .-atomic64_sub_ret