4 files changed, 80 insertions, 67 deletions

diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h
index 730aefcfbee3..be8b4d79cf41 100644
--- a/arch/arm/include/asm/cacheflush.h
+++ b/arch/arm/include/asm/cacheflush.h
@@ -182,21 +182,6 @@
  *      DMA Cache Coherency
  *      ===================
  *
- *      dma_inv_range(start, end)
- *
- *              Invalidate (discard) the specified virtual address range.
- *              May not write back any entries.  If 'start' or 'end'
- *              are not cache line aligned, those lines must be written
- *              back.
- *              - start  - virtual start address
- *              - end    - virtual end address
- *
- *      dma_clean_range(start, end)
- *
- *              Clean (write back) the specified virtual address range.
- *              - start  - virtual start address
- *              - end    - virtual end address
- *
  *      dma_flush_range(start, end)
  *
  *              Clean and invalidate the specified virtual address range.
@@ -213,8 +198,9 @@ struct cpu_cache_fns {
         void (*coherent_user_range)(unsigned long, unsigned long);
         void (*flush_kern_dcache_area)(void *, size_t);
 
-        void (*dma_inv_range)(const void *, const void *);
-        void (*dma_clean_range)(const void *, const void *);
+        void (*dma_map_area)(const void *, size_t, int);
+        void (*dma_unmap_area)(const void *, size_t, int);
+
         void (*dma_flush_range)(const void *, const void *);
 };
 
@@ -244,8 +230,8 @@ extern struct cpu_cache_fns cpu_cache;
  * is visible to DMA, or data written by DMA to system memory is
  * visible to the CPU.
  */
-#define dmac_inv_range                  cpu_cache.dma_inv_range
-#define dmac_clean_range                cpu_cache.dma_clean_range
+#define dmac_map_area                   cpu_cache.dma_map_area
+#define dmac_unmap_area         cpu_cache.dma_unmap_area
 #define dmac_flush_range                cpu_cache.dma_flush_range
 
 #else
@@ -270,12 +256,12 @@ extern void __cpuc_flush_dcache_area(void *, size_t);
  * is visible to DMA, or data written by DMA to system memory is
  * visible to the CPU.
  */
-#define dmac_inv_range                  __glue(_CACHE,_dma_inv_range)
-#define dmac_clean_range                __glue(_CACHE,_dma_clean_range)
+#define dmac_map_area                   __glue(_CACHE,_dma_map_area)
+#define dmac_unmap_area         __glue(_CACHE,_dma_unmap_area)
 #define dmac_flush_range                __glue(_CACHE,_dma_flush_range)
 
-extern void dmac_inv_range(const void *, const void *);
-extern void dmac_clean_range(const void *, const void *);
+extern void dmac_map_area(const void *, size_t, int);
+extern void dmac_unmap_area(const void *, size_t, int);
 extern void dmac_flush_range(const void *, const void *);
 
 #endif
@@ -316,12 +302,8 @@ static inline void outer_flush_range(unsigned long start, unsigned long end)
  * processes address space.  Really, we want to allow our "user
  * space" model to handle this.
  */
-#define copy_to_user_page(vma, page, vaddr, dst, src, len) \
-        do {                                                    \
-                memcpy(dst, src, len);                          \
-                flush_ptrace_access(vma, page, vaddr, dst, len, 1);\
-        } while (0)
-
+extern void copy_to_user_page(struct vm_area_struct *, struct page *,
+        unsigned long, void *, const void *, unsigned long);
 #define copy_from_user_page(vma, page, vaddr, dst, src, len) \
         do {                                                    \
                 memcpy(dst, src, len);                          \
@@ -355,17 +337,6 @@ vivt_flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig
         }
 }
 
-static inline void
-vivt_flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
-                         unsigned long uaddr, void *kaddr,
-                         unsigned long len, int write)
-{
-        if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) {
-                unsigned long addr = (unsigned long)kaddr;
-                __cpuc_coherent_kern_range(addr, addr + len);
-        }
-}
-
 #ifndef CONFIG_CPU_CACHE_VIPT
 #define flush_cache_mm(mm) \
                 vivt_flush_cache_mm(mm)
@@ -373,15 +344,10 @@ vivt_flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
                 vivt_flush_cache_range(vma,start,end)
 #define flush_cache_page(vma,addr,pfn) \
                 vivt_flush_cache_page(vma,addr,pfn)
-#define flush_ptrace_access(vma,page,ua,ka,len,write) \
-                vivt_flush_ptrace_access(vma,page,ua,ka,len,write)
 #else
 extern void flush_cache_mm(struct mm_struct *mm);
 extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 extern void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn);
-extern void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
-                                unsigned long uaddr, void *kaddr,
-                                unsigned long len, int write);
 #endif
 
 #define flush_cache_dup_mm(mm) flush_cache_mm(mm)
diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index a96300bf83fd..256ee1c9f51a 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -57,18 +57,58 @@ static inline dma_addr_t virt_to_dma(struct device *dev, void *addr)
 #endif
 
 /*
- * DMA-consistent mapping functions.  These allocate/free a region of
- * uncached, unwrite-buffered mapped memory space for use with DMA
- * devices.  This is the "generic" version.  The PCI specific version
- * is in pci.h
+ * The DMA API is built upon the notion of "buffer ownership".  A buffer
+ * is either exclusively owned by the CPU (and therefore may be accessed
+ * by it) or exclusively owned by the DMA device.  These helper functions
+ * represent the transitions between these two ownership states.
  *
- * Note: Drivers should NOT use this function directly, as it will break
- * platforms with CONFIG_DMABOUNCE.
- * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
+ * Note, however, that on later ARMs, this notion does not work due to
+ * speculative prefetches.  We model our approach on the assumption that
+ * the CPU does do speculative prefetches, which means we clean caches
+ * before transfers and delay cache invalidation until transfer completion.
+ *
+ * Private support functions: these are not part of the API and are
+ * liable to change.  Drivers must not use these.
  */
-extern void dma_cache_maint(const void *kaddr, size_t size, int rw);
-extern void dma_cache_maint_page(struct page *page, unsigned long offset,
-                                 size_t size, int rw);
+static inline void __dma_single_cpu_to_dev(const void *kaddr, size_t size,
+        enum dma_data_direction dir)
+{
+        extern void ___dma_single_cpu_to_dev(const void *, size_t,
+                enum dma_data_direction);
+
+        if (!arch_is_coherent())
+                ___dma_single_cpu_to_dev(kaddr, size, dir);
+}
+
+static inline void __dma_single_dev_to_cpu(const void *kaddr, size_t size,
+        enum dma_data_direction dir)
+{
+        extern void ___dma_single_dev_to_cpu(const void *, size_t,
+                enum dma_data_direction);
+
+        if (!arch_is_coherent())
+                ___dma_single_dev_to_cpu(kaddr, size, dir);
+}
+
+static inline void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
+        size_t size, enum dma_data_direction dir)
+{
+        extern void ___dma_page_cpu_to_dev(struct page *, unsigned long,
+                size_t, enum dma_data_direction);
+
+        if (!arch_is_coherent())
+                ___dma_page_cpu_to_dev(page, off, size, dir);
+}
+
+static inline void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
+        size_t size, enum dma_data_direction dir)
+{
+        extern void ___dma_page_dev_to_cpu(struct page *, unsigned long,
+                size_t, enum dma_data_direction);
+
+        if (!arch_is_coherent())
+                ___dma_page_dev_to_cpu(page, off, size, dir);
+}
 
 /*
  * Return whether the given device DMA address mask can be supported
@@ -304,8 +344,7 @@ static inline dma_addr_t dma_map_single(struct device *dev, void *cpu_addr,
 {
         BUG_ON(!valid_dma_direction(dir));
 
-        if (!arch_is_coherent())
-                dma_cache_maint(cpu_addr, size, dir);
+        __dma_single_cpu_to_dev(cpu_addr, size, dir);
 
         return virt_to_dma(dev, cpu_addr);
 }
@@ -329,8 +368,7 @@ static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
 {
         BUG_ON(!valid_dma_direction(dir));
 
-        if (!arch_is_coherent())
-                dma_cache_maint_page(page, offset, size, dir);
+        __dma_page_cpu_to_dev(page, offset, size, dir);
 
         return page_to_dma(dev, page) + offset;
 }
@@ -352,7 +390,7 @@ static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
 static inline void dma_unmap_single(struct device *dev, dma_addr_t handle,
                 size_t size, enum dma_data_direction dir)
 {
-        /* nothing to do */
+        __dma_single_dev_to_cpu(dma_to_virt(dev, handle), size, dir);
 }
 
 /**
@@ -372,7 +410,8 @@ static inline void dma_unmap_single(struct device *dev, dma_addr_t handle,
 static inline void dma_unmap_page(struct device *dev, dma_addr_t handle,
                 size_t size, enum dma_data_direction dir)
 {
-        /* nothing to do */
+        __dma_page_dev_to_cpu(dma_to_page(dev, handle), handle & ~PAGE_MASK,
+                size, dir);
 }
 #endif /* CONFIG_DMABOUNCE */
 
@@ -400,7 +439,10 @@ static inline void dma_sync_single_range_for_cpu(struct device *dev,
 {
         BUG_ON(!valid_dma_direction(dir));
 
-        dmabounce_sync_for_cpu(dev, handle, offset, size, dir);
+        if (!dmabounce_sync_for_cpu(dev, handle, offset, size, dir))
+                return;
+
+        __dma_single_dev_to_cpu(dma_to_virt(dev, handle) + offset, size, dir);
 }
 
 static inline void dma_sync_single_range_for_device(struct device *dev,
@@ -412,8 +454,7 @@ static inline void dma_sync_single_range_for_device(struct device *dev,
         if (!dmabounce_sync_for_device(dev, handle, offset, size, dir))
                 return;
 
-        if (!arch_is_coherent())
-                dma_cache_maint(dma_to_virt(dev, handle) + offset, size, dir);
+        __dma_single_cpu_to_dev(dma_to_virt(dev, handle) + offset, size, dir);
 }
 
 static inline void dma_sync_single_for_cpu(struct device *dev,
diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h
index 3a32af4cce30..a485ac3c8696 100644
--- a/arch/arm/include/asm/page.h
+++ b/arch/arm/include/asm/page.h
@@ -117,11 +117,12 @@
 #endif
 
 struct page;
+struct vm_area_struct;
 
 struct cpu_user_fns {
         void (*cpu_clear_user_highpage)(struct page *page, unsigned long vaddr);
         void (*cpu_copy_user_highpage)(struct page *to, struct page *from,
-                        unsigned long vaddr);
+                        unsigned long vaddr, struct vm_area_struct *vma);
 };
 
 #ifdef MULTI_USER
@@ -137,7 +138,7 @@ extern struct cpu_user_fns cpu_user;
 
 extern void __cpu_clear_user_highpage(struct page *page, unsigned long vaddr);
 extern void __cpu_copy_user_highpage(struct page *to, struct page *from,
-                        unsigned long vaddr);
+                        unsigned long vaddr, struct vm_area_struct *vma);
 #endif
 
 #define clear_user_highpage(page,vaddr)         \
@@ -145,7 +146,7 @@ extern void __cpu_copy_user_highpage(struct page *to, struct page *from,
 
 #define __HAVE_ARCH_COPY_USER_HIGHPAGE
 #define copy_user_highpage(to,from,vaddr,vma)   \
-        __cpu_copy_user_highpage(to, from, vaddr)
+        __cpu_copy_user_highpage(to, from, vaddr, vma)
 
 #define clear_page(page)        memset((void *)(page), 0, PAGE_SIZE)
 extern void copy_page(void *to, const void *from);
diff --git a/arch/arm/include/asm/smp_plat.h b/arch/arm/include/asm/smp_plat.h
index 59303e200845..e6215305544a 100644
--- a/arch/arm/include/asm/smp_plat.h
+++ b/arch/arm/include/asm/smp_plat.h
@@ -13,4 +13,9 @@ static inline int tlb_ops_need_broadcast(void)
         return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2;
 }
 
+static inline int cache_ops_need_broadcast(void)
+{
+        return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 1;
+}
+
 #endif