[SPARC64]: Implement atomic backoff.

When the cpu count is high and contention hits an atomic object, the
processors can synchronize such that some cpus continually get knocked
out and cannot complete the atomic update.

So implement an exponential backoff when SMP.

Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 21 insertions, 9 deletions

diff --git a/arch/sparc64/lib/bitops.S b/arch/sparc64/lib/bitops.S
index 892431a82131..6b015a6eefb5 100644
--- a/arch/sparc64/lib/bitops.S
+++ b/arch/sparc64/lib/bitops.S
@@ -1,10 +1,10 @@
-/* $Id: bitops.S,v 1.3 2001/11/18 00:12:56 davem Exp $
- * bitops.S: Sparc64 atomic bit operations.
+/* bitops.S: Sparc64 atomic bit operations.
  *
- * Copyright (C) 2000 David S. Miller (davem@redhat.com)
+ * Copyright (C) 2000, 2007 David S. Miller (davem@davemloft.net)
  */
 
 #include <asm/asi.h>
+#include <asm/backoff.h>
 
         .text
 
@@ -29,6 +29,7 @@
         .globl  test_and_set_bit
         .type   test_and_set_bit,#function
 test_and_set_bit:       /* %o0=nr, %o1=addr */
+        BACKOFF_SETUP(%o3)
         BITOP_PRE_BARRIER
         srlx    %o0, 6, %g1
         mov     1, %o2
@@ -40,18 +41,20 @@ test_and_set_bit:	/* %o0=nr, %o1=addr */
         or      %g7, %o2, %g1
         casx    [%o1], %g7, %g1
         cmp     %g7, %g1
-        bne,pn  %xcc, 1b
+        bne,pn  %xcc, 2f
          and    %g7, %o2, %g2
         clr     %o0
         movrne  %g2, 1, %o0
         BITOP_POST_BARRIER
         retl
          nop
+2:      BACKOFF_SPIN(%o3, %o4, 1b)
         .size   test_and_set_bit, .-test_and_set_bit
 
         .globl  test_and_clear_bit
         .type   test_and_clear_bit,#function
 test_and_clear_bit:     /* %o0=nr, %o1=addr */
+        BACKOFF_SETUP(%o3)
         BITOP_PRE_BARRIER
         srlx    %o0, 6, %g1
         mov     1, %o2
@@ -63,18 +66,20 @@ test_and_clear_bit:	/* %o0=nr, %o1=addr */
         andn    %g7, %o2, %g1
         casx    [%o1], %g7, %g1
         cmp     %g7, %g1
-        bne,pn  %xcc, 1b
+        bne,pn  %xcc, 2f
          and    %g7, %o2, %g2
         clr     %o0
         movrne  %g2, 1, %o0
         BITOP_POST_BARRIER
         retl
          nop
+2:      BACKOFF_SPIN(%o3, %o4, 1b)
         .size   test_and_clear_bit, .-test_and_clear_bit
 
         .globl  test_and_change_bit
         .type   test_and_change_bit,#function
 test_and_change_bit:    /* %o0=nr, %o1=addr */
+        BACKOFF_SETUP(%o3)
         BITOP_PRE_BARRIER
         srlx    %o0, 6, %g1
         mov     1, %o2
@@ -86,18 +91,20 @@ test_and_change_bit:	/* %o0=nr, %o1=addr */
         xor     %g7, %o2, %g1
         casx    [%o1], %g7, %g1
         cmp     %g7, %g1
-        bne,pn  %xcc, 1b
+        bne,pn  %xcc, 2f
          and    %g7, %o2, %g2
         clr     %o0
         movrne  %g2, 1, %o0
         BITOP_POST_BARRIER
         retl
          nop
+2:      BACKOFF_SPIN(%o3, %o4, 1b)
         .size   test_and_change_bit, .-test_and_change_bit
 
         .globl  set_bit
         .type   set_bit,#function
 set_bit:                /* %o0=nr, %o1=addr */
+        BACKOFF_SETUP(%o3)
         srlx    %o0, 6, %g1
         mov     1, %o2
         sllx    %g1, 3, %g3
@@ -108,15 +115,17 @@ set_bit:		/* %o0=nr, %o1=addr */
         or      %g7, %o2, %g1
         casx    [%o1], %g7, %g1
         cmp     %g7, %g1
-        bne,pn  %xcc, 1b
+        bne,pn  %xcc, 2f
          nop
         retl
          nop
+2:      BACKOFF_SPIN(%o3, %o4, 1b)
         .size   set_bit, .-set_bit
 
         .globl  clear_bit
         .type   clear_bit,#function
 clear_bit:              /* %o0=nr, %o1=addr */
+        BACKOFF_SETUP(%o3)
         srlx    %o0, 6, %g1
         mov     1, %o2
         sllx    %g1, 3, %g3
@@ -127,15 +136,17 @@ clear_bit:		/* %o0=nr, %o1=addr */
         andn    %g7, %o2, %g1
         casx    [%o1], %g7, %g1
         cmp     %g7, %g1
-        bne,pn  %xcc, 1b
+        bne,pn  %xcc, 2f
          nop
         retl
          nop
+2:      BACKOFF_SPIN(%o3, %o4, 1b)
         .size   clear_bit, .-clear_bit
 
         .globl  change_bit
         .type   change_bit,#function
 change_bit:             /* %o0=nr, %o1=addr */
+        BACKOFF_SETUP(%o3)
         srlx    %o0, 6, %g1
         mov     1, %o2
         sllx    %g1, 3, %g3
@@ -146,8 +157,9 @@ change_bit:		/* %o0=nr, %o1=addr */
         xor     %g7, %o2, %g1
         casx    [%o1], %g7, %g1
         cmp     %g7, %g1
-        bne,pn  %xcc, 1b
+        bne,pn  %xcc, 2f
          nop
         retl
          nop
+2:      BACKOFF_SPIN(%o3, %o4, 1b)
         .size   change_bit, .-change_bit