[SPARC64]: More fully work around Spitfire Errata 51.

It appears that a memory barrier soon after a mispredicted
branch, not just in the delay slot, can cause the hang
condition of this cpu errata.

So move them out-of-line, and explicitly put them into
a "branch always, predict taken" delay slot which should
fully kill this problem.

Signed-off-by: David S. Miller <davem@davemloft.net>

3 files changed, 82 insertions, 9 deletions

diff --git a/arch/sparc64/lib/Makefile b/arch/sparc64/lib/Makefile
index 40dbeec7e5d6..6201f1040982 100644
--- a/arch/sparc64/lib/Makefile
+++ b/arch/sparc64/lib/Makefile
@@ -12,7 +12,7 @@ lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \
          U1memcpy.o U1copy_from_user.o U1copy_to_user.o \
          U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \
          copy_in_user.o user_fixup.o memmove.o \
-         mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o
+         mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o mb.o
 
 lib-$(CONFIG_DEBUG_SPINLOCK) += debuglocks.o
 lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
diff --git a/arch/sparc64/lib/debuglocks.c b/arch/sparc64/lib/debuglocks.c
index 7f6ccc4114c5..f5f0b5586f01 100644
--- a/arch/sparc64/lib/debuglocks.c
+++ b/arch/sparc64/lib/debuglocks.c
@@ -61,7 +61,7 @@ again:
                              : "=r" (val)
                              : "r" (&(lock->lock))
                              : "memory");
-        membar("#StoreLoad | #StoreStore");
+        membar_storeload_storestore();
         if (val) {
                 while (lock->lock) {
                         if (!--stuck) {
@@ -69,7 +69,7 @@ again:
                                         show(str, lock, caller);
                                 stuck = INIT_STUCK;
                         }
-                        membar("#LoadLoad");
+                        rmb();
                 }
                 goto again;
         }
@@ -90,7 +90,7 @@ int _do_spin_trylock(spinlock_t *lock, unsigned long caller)
                              : "=r" (val)
                              : "r" (&(lock->lock))
                              : "memory");
-        membar("#StoreLoad | #StoreStore");
+        membar_storeload_storestore();
         if (!val) {
                 lock->owner_pc = ((unsigned int)caller);
                 lock->owner_cpu = cpu;
@@ -107,7 +107,7 @@ void _do_spin_unlock(spinlock_t *lock)
 {
         lock->owner_pc = 0;
         lock->owner_cpu = NO_PROC_ID;
-        membar("#StoreStore | #LoadStore");
+        membar_storestore_loadstore();
         lock->lock = 0;
         current->thread.smp_lock_count--;
 }
@@ -129,7 +129,7 @@ wlock_again:
                                 show_read(str, rw, caller);
                         stuck = INIT_STUCK;
                 }
-                membar("#LoadLoad");
+                rmb();
         }
         /* Try once to increment the counter.  */
         __asm__ __volatile__(
@@ -142,7 +142,7 @@ wlock_again:
 "2:"    : "=r" (val)
         : "0" (&(rw->lock))
         : "g1", "g7", "memory");
-        membar("#StoreLoad | #StoreStore");
+        membar_storeload_storestore();
         if (val)
                 goto wlock_again;
         rw->reader_pc[cpu] = ((unsigned int)caller);
@@ -201,7 +201,7 @@ wlock_again:
                                 show_write(str, rw, caller);
                         stuck = INIT_STUCK;
                 }
-                membar("#LoadLoad");
+                rmb();
         }
 
         /* Try to acuire the write bit.  */
@@ -256,7 +256,7 @@ wlock_again:
                                         show_write(str, rw, caller);
                                 stuck = INIT_STUCK;
                         }
-                        membar("#LoadLoad");
+                        rmb();
                 }
                 goto wlock_again;
         }
diff --git a/arch/sparc64/lib/mb.S b/arch/sparc64/lib/mb.S
new file mode 100644
index 000000000000..4004f748619f
--- /dev/null
+++ b/arch/sparc64/lib/mb.S
@@ -0,0 +1,73 @@
+/* mb.S: Out of line memory barriers.
+ *
+ * Copyright (C) 2005 David S. Miller (davem@davemloft.net)
+ */
+
+        /* These are here in an effort to more fully work around
+         * Spitfire Errata #51.  Essentially, if a memory barrier
+         * occurs soon after a mispredicted branch, the chip can stop
+         * executing instructions until a trap occurs.  Therefore, if
+         * interrupts are disabled, the chip can hang forever.
+         *
+         * It used to be believed that the memory barrier had to be
+         * right in the delay slot, but a case has been traced
+         * recently wherein the memory barrier was one instruction
+         * after the branch delay slot and the chip still hung.  The
+         * offending sequence was the following in sym_wakeup_done()
+         * of the sym53c8xx_2 driver:
+         *
+         *      call    sym_ccb_from_dsa, 0
+         *       movge  %icc, 0, %l0
+         *      brz,pn  %o0, .LL1303
+         *       mov    %o0, %l2
+         *      membar  #LoadLoad
+         *
+         * The branch has to be mispredicted for the bug to occur.
+         * Therefore, we put the memory barrier explicitly into a
+         * "branch always, predicted taken" delay slot to avoid the
+         * problem case.
+         */
+
+        .text
+
+99:     retl
+         nop
+
+        .globl  mb
+mb:     ba,pt   %xcc, 99b
+         membar #LoadLoad | #LoadStore | #StoreStore | #StoreLoad
+        .size   mb, .-mb
+
+        .globl  rmb
+rmb:    ba,pt   %xcc, 99b
+         membar #LoadLoad
+        .size   rmb, .-rmb
+
+        .globl  wmb
+wmb:    ba,pt   %xcc, 99b
+         membar #StoreStore
+        .size   wmb, .-wmb
+
+        .globl  membar_storeload
+membar_storeload:
+        ba,pt   %xcc, 99b
+         membar #StoreLoad
+        .size   membar_storeload, .-membar_storeload
+
+        .globl  membar_storeload_storestore
+membar_storeload_storestore:
+        ba,pt   %xcc, 99b
+         membar #StoreLoad | #StoreStore
+        .size   membar_storeload_storestore, .-membar_storeload_storestore
+
+        .globl  membar_storeload_loadload
+membar_storeload_loadload:
+        ba,pt   %xcc, 99b
+         membar #StoreLoad | #LoadLoad
+        .size   membar_storeload_loadload, .-membar_storeload_loadload
+
+        .globl  membar_storestore_loadstore
+membar_storestore_loadstore:
+        ba,pt   %xcc, 99b
+         membar #StoreStore | #LoadStore
+        .size   membar_storestore_loadstore, .-membar_storestore_loadstore