12 files changed, 99 insertions, 26 deletions

diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt
index 7ee2ae6d5451..70a09f8a0383 100644
--- a/Documentation/memory-barriers.txt
+++ b/Documentation/memory-barriers.txt
@@ -1633,6 +1633,48 @@ There are some more advanced barrier functions:
      operations" subsection for information on where to use these.
 
 
+ (*) dma_wmb();
+ (*) dma_rmb();
+
+     These are for use with consistent memory to guarantee the ordering
+     of writes or reads of shared memory accessible to both the CPU and a
+     DMA capable device.
+
+     For example, consider a device driver that shares memory with a device
+     and uses a descriptor status value to indicate if the descriptor belongs
+     to the device or the CPU, and a doorbell to notify it when new
+     descriptors are available:
+
+        if (desc->status != DEVICE_OWN) {
+                /* do not read data until we own descriptor */
+                dma_rmb();
+
+                /* read/modify data */
+                read_data = desc->data;
+                desc->data = write_data;
+
+                /* flush modifications before status update */
+                dma_wmb();
+
+                /* assign ownership */
+                desc->status = DEVICE_OWN;
+
+                /* force memory to sync before notifying device via MMIO */
+                wmb();
+
+                /* notify device of new descriptors */
+                writel(DESC_NOTIFY, doorbell);
+        }
+
+     The dma_rmb() allows us guarantee the device has released ownership
+     before we read the data from the descriptor, and he dma_wmb() allows
+     us to guarantee the data is written to the descriptor before the device
+     can see it now has ownership.  The wmb() is needed to guarantee that the
+     cache coherent memory writes have completed before attempting a write to
+     the cache incoherent MMIO region.
+
+     See Documentation/DMA-API.txt for more information on consistent memory.
+
 MMIO WRITE BARRIER
 ------------------
 
diff --git a/arch/arm/include/asm/barrier.h b/arch/arm/include/asm/barrier.h
index c6a3e73a6e24..d2f81e6b8c1c 100644
--- a/arch/arm/include/asm/barrier.h
+++ b/arch/arm/include/asm/barrier.h
@@ -43,10 +43,14 @@
 #define mb()            do { dsb(); outer_sync(); } while (0)
 #define rmb()           dsb()
 #define wmb()           do { dsb(st); outer_sync(); } while (0)
+#define dma_rmb()       dmb(osh)
+#define dma_wmb()       dmb(oshst)
 #else
 #define mb()            barrier()
 #define rmb()           barrier()
 #define wmb()           barrier()
+#define dma_rmb()       barrier()
+#define dma_wmb()       barrier()
 #endif
 
 #ifndef CONFIG_SMP
diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h
index 6389d60574d9..a5abb0062d6e 100644
--- a/arch/arm64/include/asm/barrier.h
+++ b/arch/arm64/include/asm/barrier.h
@@ -32,6 +32,9 @@
 #define rmb()           dsb(ld)
 #define wmb()           dsb(st)
 
+#define dma_rmb()       dmb(oshld)
+#define dma_wmb()       dmb(oshst)
+
 #ifndef CONFIG_SMP
 #define smp_mb()        barrier()
 #define smp_rmb()       barrier()
diff --git a/arch/ia64/include/asm/barrier.h b/arch/ia64/include/asm/barrier.h
index e8fffb03963c..f6769eb2bbf9 100644
--- a/arch/ia64/include/asm/barrier.h
+++ b/arch/ia64/include/asm/barrier.h
@@ -39,6 +39,9 @@
 #define rmb()           mb()
 #define wmb()           mb()
 
+#define dma_rmb()       mb()
+#define dma_wmb()       mb()
+
 #ifdef CONFIG_SMP
 # define smp_mb()       mb()
 #else
diff --git a/arch/metag/include/asm/barrier.h b/arch/metag/include/asm/barrier.h
index 6d8b8c9b7c25..d703d8e26a65 100644
--- a/arch/metag/include/asm/barrier.h
+++ b/arch/metag/include/asm/barrier.h
@@ -4,8 +4,6 @@
 #include <asm/metag_mem.h>
 
 #define nop()           asm volatile ("NOP")
-#define mb()            wmb()
-#define rmb()           barrier()
 
 #ifdef CONFIG_METAG_META21
 
@@ -41,11 +39,13 @@ static inline void wr_fence(void)
 
 #endif /* !CONFIG_METAG_META21 */
 
-static inline void wmb(void)
-{
-        /* flush writes through the write combiner */
-        wr_fence();
-}
+/* flush writes through the write combiner */
+#define mb()            wr_fence()
+#define rmb()           barrier()
+#define wmb()           mb()
+
+#define dma_rmb()       rmb()
+#define dma_wmb()       wmb()
 
 #ifndef CONFIG_SMP
 #define fence()         do { } while (0)
diff --git a/arch/mips/include/asm/barrier.h b/arch/mips/include/asm/barrier.h
index 3d69aa829a76..2b8bbbcb9be0 100644
--- a/arch/mips/include/asm/barrier.h
+++ b/arch/mips/include/asm/barrier.h
@@ -75,20 +75,21 @@
 
 #include <asm/wbflush.h>
 
-#define wmb()           fast_wmb()
-#define rmb()           fast_rmb()
 #define mb()            wbflush()
 #define iob()           wbflush()
 
 #else /* !CONFIG_CPU_HAS_WB */
 
-#define wmb()           fast_wmb()
-#define rmb()           fast_rmb()
 #define mb()            fast_mb()
 #define iob()           fast_iob()
 
 #endif /* !CONFIG_CPU_HAS_WB */
 
+#define wmb()           fast_wmb()
+#define rmb()           fast_rmb()
+#define dma_wmb()       fast_wmb()
+#define dma_rmb()       fast_rmb()
+
 #if defined(CONFIG_WEAK_ORDERING) && defined(CONFIG_SMP)
 # ifdef CONFIG_CPU_CAVIUM_OCTEON
 #  define smp_mb()      __sync()
diff --git a/arch/powerpc/include/asm/barrier.h b/arch/powerpc/include/asm/barrier.h
index cb6d66c6e3e1..a3bf5be111ff 100644
--- a/arch/powerpc/include/asm/barrier.h
+++ b/arch/powerpc/include/asm/barrier.h
@@ -36,8 +36,6 @@
 
 #define set_mb(var, value)      do { var = value; mb(); } while (0)
 
-#ifdef CONFIG_SMP
-
 #ifdef __SUBARCH_HAS_LWSYNC
 #    define SMPWMB      LWSYNC
 #else
@@ -45,12 +43,17 @@
 #endif
 
 #define __lwsync()      __asm__ __volatile__ (stringify_in_c(LWSYNC) : : :"memory")
+#define dma_rmb()       __lwsync()
+#define dma_wmb()       __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory")
+
+#ifdef CONFIG_SMP
+#define smp_lwsync()    __lwsync()
 
 #define smp_mb()        mb()
 #define smp_rmb()       __lwsync()
 #define smp_wmb()       __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory")
 #else
-#define __lwsync()      barrier()
+#define smp_lwsync()    barrier()
 
 #define smp_mb()        barrier()
 #define smp_rmb()       barrier()
@@ -72,7 +75,7 @@
 #define smp_store_release(p, v)                                         \
 do {                                                                    \
         compiletime_assert_atomic_type(*p);                             \
-        __lwsync();                                                     \
+        smp_lwsync();                                                   \
         ACCESS_ONCE(*p) = (v);                                          \
 } while (0)
 
@@ -80,7 +83,7 @@ do {									\
 ({                                                                      \
         typeof(*p) ___p1 = ACCESS_ONCE(*p);                             \
         compiletime_assert_atomic_type(*p);                             \
-        __lwsync();                                                     \
+        smp_lwsync();                                                   \
         ___p1;                                                          \
 })
 
diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h
index 33d191d295e4..8d724718ec21 100644
--- a/arch/s390/include/asm/barrier.h
+++ b/arch/s390/include/asm/barrier.h
@@ -24,6 +24,8 @@
 
 #define rmb()                           mb()
 #define wmb()                           mb()
+#define dma_rmb()                       rmb()
+#define dma_wmb()                       wmb()
 #define smp_mb()                        mb()
 #define smp_rmb()                       rmb()
 #define smp_wmb()                       wmb()
diff --git a/arch/sparc/include/asm/barrier_64.h b/arch/sparc/include/asm/barrier_64.h
index 6c974c0977ad..76648941fea7 100644
--- a/arch/sparc/include/asm/barrier_64.h
+++ b/arch/sparc/include/asm/barrier_64.h
@@ -37,6 +37,9 @@ do {	__asm__ __volatile__("ba,pt	%%xcc, 1f\n\t" \
 #define rmb()   __asm__ __volatile__("":::"memory")
 #define wmb()   __asm__ __volatile__("":::"memory")
 
+#define dma_rmb()       rmb()
+#define dma_wmb()       wmb()
+
 #define set_mb(__var, __value) \
         do { __var = __value; membar_safe("#StoreLoad"); } while(0)
 
diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h
index 5238000285c1..2ab1eb33106e 100644
--- a/arch/x86/include/asm/barrier.h
+++ b/arch/x86/include/asm/barrier.h
@@ -24,13 +24,16 @@
 #define wmb()   asm volatile("sfence" ::: "memory")
 #endif
 
-#ifdef CONFIG_SMP
-#define smp_mb()        mb()
 #ifdef CONFIG_X86_PPRO_FENCE
-# define smp_rmb()      rmb()
+#define dma_rmb()       rmb()
 #else
-# define smp_rmb()      barrier()
+#define dma_rmb()       barrier()
 #endif
+#define dma_wmb()       barrier()
+
+#ifdef CONFIG_SMP
+#define smp_mb()        mb()
+#define smp_rmb()       dma_rmb()
 #define smp_wmb()       barrier()
 #define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
 #else /* !SMP */
diff --git a/arch/x86/um/asm/barrier.h b/arch/x86/um/asm/barrier.h
index d6511d954e2b..2d7d9a1f5b53 100644
--- a/arch/x86/um/asm/barrier.h
+++ b/arch/x86/um/asm/barrier.h
@@ -29,17 +29,18 @@
 
 #endif /* CONFIG_X86_32 */
 
-#ifdef CONFIG_SMP
-
-#define smp_mb()        mb()
 #ifdef CONFIG_X86_PPRO_FENCE
-#define smp_rmb()       rmb()
+#define dma_rmb()       rmb()
 #else /* CONFIG_X86_PPRO_FENCE */
-#define smp_rmb()       barrier()
+#define dma_rmb()       barrier()
 #endif /* CONFIG_X86_PPRO_FENCE */
+#define dma_wmb()       barrier()
 
-#define smp_wmb()       barrier()
+#ifdef CONFIG_SMP
 
+#define smp_mb()        mb()
+#define smp_rmb()       dma_rmb()
+#define smp_wmb()       barrier()
 #define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
 
 #else /* CONFIG_SMP */
diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h
index 1402fa855388..f5c40b0fadc2 100644
--- a/include/asm-generic/barrier.h
+++ b/include/asm-generic/barrier.h
@@ -42,6 +42,14 @@
 #define wmb()   mb()
 #endif
 
+#ifndef dma_rmb
+#define dma_rmb()       rmb()
+#endif
+
+#ifndef dma_wmb
+#define dma_wmb()       wmb()
+#endif
+
 #ifndef read_barrier_depends
 #define read_barrier_depends()          do { } while (0)
 #endif