aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMarek Szyprowski <m.szyprowski@samsung.com>2012-05-22 02:55:43 -0400
committerMarek Szyprowski <m.szyprowski@samsung.com>2012-05-22 02:55:43 -0400
commit0f51596bd39a5c928307ffcffc9ba07f90f42a8b (patch)
treeb636403815316ecad2170092b70f1079df260a95
parent61f6c7a47a2f84b7ba4b65240ffe9247df772b06 (diff)
parent4ce63fcd919c32d22528e54dcd89506962933719 (diff)
Merge branch 'for-next-arm-dma' into for-linus
Conflicts: arch/arm/Kconfig arch/arm/mm/dma-mapping.c Signed-off-by: Marek Szyprowski <m.szyprowski@samsung.com>
Diffstat
-rw-r--r--arch/arm/Kconfig9
-rw-r--r--arch/arm/common/dmabounce.c84
-rw-r--r--arch/arm/include/asm/device.h4
-rw-r--r--arch/arm/include/asm/dma-iommu.h34
-rw-r--r--arch/arm/include/asm/dma-mapping.h407
-rw-r--r--arch/arm/mm/dma-mapping.c998
-rw-r--r--arch/arm/mm/vmregion.h2
-rw-r--r--drivers/base/dma-coherent.c42
-rw-r--r--include/asm-generic/dma-coherent.h4
9 files changed, 1123 insertions, 461 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index cbbbc45f6b67..24d3302a1b8e 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -4,6 +4,7 @@ config ARM
4 select HAVE_AOUT 4 select HAVE_AOUT
5 select HAVE_DMA_API_DEBUG 5 select HAVE_DMA_API_DEBUG
6 select HAVE_IDE if PCI || ISA || PCMCIA 6 select HAVE_IDE if PCI || ISA || PCMCIA
7 select HAVE_DMA_ATTRS
7 select HAVE_DMA_CONTIGUOUS if (CPU_V6 || CPU_V6K || CPU_V7) 8 select HAVE_DMA_CONTIGUOUS if (CPU_V6 || CPU_V6K || CPU_V7)
8 select CMA if (CPU_V6 || CPU_V6K || CPU_V7) 9 select CMA if (CPU_V6 || CPU_V6K || CPU_V7)
9 select HAVE_MEMBLOCK 10 select HAVE_MEMBLOCK
@@ -47,6 +48,14 @@ config ARM
47config ARM_HAS_SG_CHAIN 48config ARM_HAS_SG_CHAIN
48 bool 49 bool
49 50
51config NEED_SG_DMA_LENGTH
52 bool
53
54config ARM_DMA_USE_IOMMU
55 select NEED_SG_DMA_LENGTH
56 select ARM_HAS_SG_CHAIN
57 bool
58
50config HAVE_PWM 59config HAVE_PWM
51 bool 60 bool
52 61
diff --git a/arch/arm/common/dmabounce.c b/arch/arm/common/dmabounce.c
index 595ecd290ebf..9d7eb530f95f 100644
--- a/arch/arm/common/dmabounce.c
+++ b/arch/arm/common/dmabounce.c
@@ -173,7 +173,8 @@ find_safe_buffer(struct dmabounce_device_info *device_info, dma_addr_t safe_dma_
173 read_lock_irqsave(&device_info->lock, flags); 173 read_lock_irqsave(&device_info->lock, flags);
174 174
175 list_for_each_entry(b, &device_info->safe_buffers, node) 175 list_for_each_entry(b, &device_info->safe_buffers, node)
176 if (b->safe_dma_addr == safe_dma_addr) { 176 if (b->safe_dma_addr <= safe_dma_addr &&
177 b->safe_dma_addr + b->size > safe_dma_addr) {
177 rb = b; 178 rb = b;
178 break; 179 break;
179 } 180 }
@@ -254,7 +255,7 @@ static inline dma_addr_t map_single(struct device *dev, void *ptr, size_t size,
254 if (buf == NULL) { 255 if (buf == NULL) {
255 dev_err(dev, "%s: unable to map unsafe buffer %p!\n", 256 dev_err(dev, "%s: unable to map unsafe buffer %p!\n",
256 __func__, ptr); 257 __func__, ptr);
257 return ~0; 258 return DMA_ERROR_CODE;
258 } 259 }
259 260
260 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n", 261 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n",
@@ -307,8 +308,9 @@ static inline void unmap_single(struct device *dev, struct safe_buffer *buf,
307 * substitute the safe buffer for the unsafe one. 308 * substitute the safe buffer for the unsafe one.
308 * (basically move the buffer from an unsafe area to a safe one) 309 * (basically move the buffer from an unsafe area to a safe one)
309 */ 310 */
310dma_addr_t __dma_map_page(struct device *dev, struct page *page, 311static dma_addr_t dmabounce_map_page(struct device *dev, struct page *page,
311 unsigned long offset, size_t size, enum dma_data_direction dir) 312 unsigned long offset, size_t size, enum dma_data_direction dir,
313 struct dma_attrs *attrs)
312{ 314{
313 dma_addr_t dma_addr; 315 dma_addr_t dma_addr;
314 int ret; 316 int ret;
@@ -320,21 +322,20 @@ dma_addr_t __dma_map_page(struct device *dev, struct page *page,
320 322
321 ret = needs_bounce(dev, dma_addr, size); 323 ret = needs_bounce(dev, dma_addr, size);
322 if (ret < 0) 324 if (ret < 0)
323 return ~0; 325 return DMA_ERROR_CODE;
324 326
325 if (ret == 0) { 327 if (ret == 0) {
326 __dma_page_cpu_to_dev(page, offset, size, dir); 328 arm_dma_ops.sync_single_for_device(dev, dma_addr, size, dir);
327 return dma_addr; 329 return dma_addr;
328 } 330 }
329 331
330 if (PageHighMem(page)) { 332 if (PageHighMem(page)) {
331 dev_err(dev, "DMA buffer bouncing of HIGHMEM pages is not supported\n"); 333 dev_err(dev, "DMA buffer bouncing of HIGHMEM pages is not supported\n");
332 return ~0; 334 return DMA_ERROR_CODE;
333 } 335 }
334 336
335 return map_single(dev, page_address(page) + offset, size, dir); 337 return map_single(dev, page_address(page) + offset, size, dir);
336} 338}
337EXPORT_SYMBOL(__dma_map_page);
338 339
339/* 340/*
340 * see if a mapped address was really a "safe" buffer and if so, copy 341 * see if a mapped address was really a "safe" buffer and if so, copy
@@ -342,8 +343,8 @@ EXPORT_SYMBOL(__dma_map_page);
342 * the safe buffer. (basically return things back to the way they 343 * the safe buffer. (basically return things back to the way they
343 * should be) 344 * should be)
344 */ 345 */
345void __dma_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, 346static void dmabounce_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
346 enum dma_data_direction dir) 347 enum dma_data_direction dir, struct dma_attrs *attrs)
347{ 348{
348 struct safe_buffer *buf; 349 struct safe_buffer *buf;
349 350
@@ -352,19 +353,18 @@ void __dma_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
352 353
353 buf = find_safe_buffer_dev(dev, dma_addr, __func__); 354 buf = find_safe_buffer_dev(dev, dma_addr, __func__);
354 if (!buf) { 355 if (!buf) {
355 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, dma_addr)), 356 arm_dma_ops.sync_single_for_cpu(dev, dma_addr, size, dir);
356 dma_addr & ~PAGE_MASK, size, dir);
357 return; 357 return;
358 } 358 }
359 359
360 unmap_single(dev, buf, size, dir); 360 unmap_single(dev, buf, size, dir);
361} 361}
362EXPORT_SYMBOL(__dma_unmap_page);
363 362
364int dmabounce_sync_for_cpu(struct device *dev, dma_addr_t addr, 363static int __dmabounce_sync_for_cpu(struct device *dev, dma_addr_t addr,
365 unsigned long off, size_t sz, enum dma_data_direction dir) 364 size_t sz, enum dma_data_direction dir)
366{ 365{
367 struct safe_buffer *buf; 366 struct safe_buffer *buf;
367 unsigned long off;
368 368
369 dev_dbg(dev, "%s(dma=%#x,off=%#lx,sz=%zx,dir=%x)\n", 369 dev_dbg(dev, "%s(dma=%#x,off=%#lx,sz=%zx,dir=%x)\n",
370 __func__, addr, off, sz, dir); 370 __func__, addr, off, sz, dir);
@@ -373,6 +373,8 @@ int dmabounce_sync_for_cpu(struct device *dev, dma_addr_t addr,
373 if (!buf) 373 if (!buf)
374 return 1; 374 return 1;
375 375
376 off = addr - buf->safe_dma_addr;
377
376 BUG_ON(buf->direction != dir); 378 BUG_ON(buf->direction != dir);
377 379
378 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n", 380 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n",
@@ -388,12 +390,21 @@ int dmabounce_sync_for_cpu(struct device *dev, dma_addr_t addr,
388 } 390 }
389 return 0; 391 return 0;
390} 392}
391EXPORT_SYMBOL(dmabounce_sync_for_cpu);
392 393
393int dmabounce_sync_for_device(struct device *dev, dma_addr_t addr, 394static void dmabounce_sync_for_cpu(struct device *dev,
394 unsigned long off, size_t sz, enum dma_data_direction dir) 395 dma_addr_t handle, size_t size, enum dma_data_direction dir)
396{
397 if (!__dmabounce_sync_for_cpu(dev, handle, size, dir))
398 return;
399
400 arm_dma_ops.sync_single_for_cpu(dev, handle, size, dir);
401}
402
403static int __dmabounce_sync_for_device(struct device *dev, dma_addr_t addr,
404 size_t sz, enum dma_data_direction dir)
395{ 405{
396 struct safe_buffer *buf; 406 struct safe_buffer *buf;
407 unsigned long off;
397 408
398 dev_dbg(dev, "%s(dma=%#x,off=%#lx,sz=%zx,dir=%x)\n", 409 dev_dbg(dev, "%s(dma=%#x,off=%#lx,sz=%zx,dir=%x)\n",
399 __func__, addr, off, sz, dir); 410 __func__, addr, off, sz, dir);
@@ -402,6 +413,8 @@ int dmabounce_sync_for_device(struct device *dev, dma_addr_t addr,
402 if (!buf) 413 if (!buf)
403 return 1; 414 return 1;
404 415
416 off = addr - buf->safe_dma_addr;
417
405 BUG_ON(buf->direction != dir); 418 BUG_ON(buf->direction != dir);
406 419
407 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n", 420 dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n",
@@ -417,7 +430,38 @@ int dmabounce_sync_for_device(struct device *dev, dma_addr_t addr,
417 } 430 }
418 return 0; 431 return 0;
419} 432}
420EXPORT_SYMBOL(dmabounce_sync_for_device); 433
434static void dmabounce_sync_for_device(struct device *dev,
435 dma_addr_t handle, size_t size, enum dma_data_direction dir)
436{
437 if (!__dmabounce_sync_for_device(dev, handle, size, dir))
438 return;
439
440 arm_dma_ops.sync_single_for_device(dev, handle, size, dir);
441}
442
443static int dmabounce_set_mask(struct device *dev, u64 dma_mask)
444{
445 if (dev->archdata.dmabounce)
446 return 0;
447
448 return arm_dma_ops.set_dma_mask(dev, dma_mask);
449}
450
451static struct dma_map_ops dmabounce_ops = {
452 .alloc = arm_dma_alloc,
453 .free = arm_dma_free,
454 .mmap = arm_dma_mmap,
455 .map_page = dmabounce_map_page,
456 .unmap_page = dmabounce_unmap_page,
457 .sync_single_for_cpu = dmabounce_sync_for_cpu,
458 .sync_single_for_device = dmabounce_sync_for_device,
459 .map_sg = arm_dma_map_sg,
460 .unmap_sg = arm_dma_unmap_sg,
461 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,
462 .sync_sg_for_device = arm_dma_sync_sg_for_device,
463 .set_dma_mask = dmabounce_set_mask,
464};
421 465
422static int dmabounce_init_pool(struct dmabounce_pool *pool, struct device *dev, 466static int dmabounce_init_pool(struct dmabounce_pool *pool, struct device *dev,
423 const char *name, unsigned long size) 467 const char *name, unsigned long size)
@@ -479,6 +523,7 @@ int dmabounce_register_dev(struct device *dev, unsigned long small_buffer_size,
479#endif 523#endif
480 524
481 dev->archdata.dmabounce = device_info; 525 dev->archdata.dmabounce = device_info;
526 set_dma_ops(dev, &dmabounce_ops);
482 527
483 dev_info(dev, "dmabounce: registered device\n"); 528 dev_info(dev, "dmabounce: registered device\n");
484 529
@@ -497,6 +542,7 @@ void dmabounce_unregister_dev(struct device *dev)
497 struct dmabounce_device_info *device_info = dev->archdata.dmabounce; 542 struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
498 543
499 dev->archdata.dmabounce = NULL; 544 dev->archdata.dmabounce = NULL;
545 set_dma_ops(dev, NULL);
500 546
501 if (!device_info) { 547 if (!device_info) {
502 dev_warn(dev, 548 dev_warn(dev,
diff --git a/arch/arm/include/asm/device.h b/arch/arm/include/asm/device.h
index 7aa368003b05..b69c0d3285f8 100644
--- a/arch/arm/include/asm/device.h
+++ b/arch/arm/include/asm/device.h
@@ -7,12 +7,16 @@
7#define ASMARM_DEVICE_H 7#define ASMARM_DEVICE_H
8 8
9struct dev_archdata { 9struct dev_archdata {
10 struct dma_map_ops *dma_ops;
10#ifdef CONFIG_DMABOUNCE 11#ifdef CONFIG_DMABOUNCE
11 struct dmabounce_device_info *dmabounce; 12 struct dmabounce_device_info *dmabounce;
12#endif 13#endif
13#ifdef CONFIG_IOMMU_API 14#ifdef CONFIG_IOMMU_API
14 void *iommu; /* private IOMMU data */ 15 void *iommu; /* private IOMMU data */
15#endif 16#endif
17#ifdef CONFIG_ARM_DMA_USE_IOMMU
18 struct dma_iommu_mapping *mapping;
19#endif
16}; 20};
17 21
18struct omap_device; 22struct omap_device;
diff --git a/arch/arm/include/asm/dma-iommu.h b/arch/arm/include/asm/dma-iommu.h
new file mode 100644
index 000000000000..799b09409fad
--- /dev/null
+++ b/arch/arm/include/asm/dma-iommu.h
@@ -0,0 +1,34 @@
1#ifndef ASMARM_DMA_IOMMU_H
2#define ASMARM_DMA_IOMMU_H
3
4#ifdef __KERNEL__
5
6#include <linux/mm_types.h>
7#include <linux/scatterlist.h>
8#include <linux/dma-debug.h>
9#include <linux/kmemcheck.h>
10
11struct dma_iommu_mapping {
12 /* iommu specific data */
13 struct iommu_domain *domain;
14
15 void *bitmap;
16 size_t bits;
17 unsigned int order;
18 dma_addr_t base;
19
20 spinlock_t lock;
21 struct kref kref;
22};
23
24struct dma_iommu_mapping *
25arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size,
26 int order);
27
28void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping);
29
30int arm_iommu_attach_device(struct device *dev,
31 struct dma_iommu_mapping *mapping);
32
33#endif /* __KERNEL__ */
34#endif
diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index cb3b7c981c4b..bbef15d04890 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -5,11 +5,35 @@
5 5
6#include <linux/mm_types.h> 6#include <linux/mm_types.h>
7#include <linux/scatterlist.h> 7#include <linux/scatterlist.h>
8#include <linux/dma-attrs.h>
8#include <linux/dma-debug.h> 9#include <linux/dma-debug.h>
9 10
10#include <asm-generic/dma-coherent.h> 11#include <asm-generic/dma-coherent.h>
11#include <asm/memory.h> 12#include <asm/memory.h>
12 13
14#define DMA_ERROR_CODE (~0)
15extern struct dma_map_ops arm_dma_ops;
16
17static inline struct dma_map_ops *get_dma_ops(struct device *dev)
18{
19 if (dev && dev->archdata.dma_ops)
20 return dev->archdata.dma_ops;
21 return &arm_dma_ops;
22}
23
24static inline void set_dma_ops(struct device *dev, struct dma_map_ops *ops)
25{
26 BUG_ON(!dev);
27 dev->archdata.dma_ops = ops;
28}
29
30#include <asm-generic/dma-mapping-common.h>
31
32static inline int dma_set_mask(struct device *dev, u64 mask)
33{
34 return get_dma_ops(dev)->set_dma_mask(dev, mask);
35}
36
13#ifdef __arch_page_to_dma 37#ifdef __arch_page_to_dma
14#error Please update to __arch_pfn_to_dma 38#error Please update to __arch_pfn_to_dma
15#endif 39#endif
@@ -62,68 +86,11 @@ static inline dma_addr_t virt_to_dma(struct device *dev, void *addr)
62#endif 86#endif
63 87
64/* 88/*
65 * The DMA API is built upon the notion of "buffer ownership". A buffer
66 * is either exclusively owned by the CPU (and therefore may be accessed
67 * by it) or exclusively owned by the DMA device. These helper functions
68 * represent the transitions between these two ownership states.
69 *
70 * Note, however, that on later ARMs, this notion does not work due to
71 * speculative prefetches. We model our approach on the assumption that
72 * the CPU does do speculative prefetches, which means we clean caches
73 * before transfers and delay cache invalidation until transfer completion.
74 *
75 * Private support functions: these are not part of the API and are
76 * liable to change. Drivers must not use these.
77 */
78static inline void __dma_single_cpu_to_dev(const void *kaddr, size_t size,
79 enum dma_data_direction dir)
80{
81 extern void ___dma_single_cpu_to_dev(const void *, size_t,
82 enum dma_data_direction);
83
84 if (!arch_is_coherent())
85 ___dma_single_cpu_to_dev(kaddr, size, dir);
86}
87
88static inline void __dma_single_dev_to_cpu(const void *kaddr, size_t size,
89 enum dma_data_direction dir)
90{
91 extern void ___dma_single_dev_to_cpu(const void *, size_t,
92 enum dma_data_direction);
93
94 if (!arch_is_coherent())
95 ___dma_single_dev_to_cpu(kaddr, size, dir);
96}
97
98static inline void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
99 size_t size, enum dma_data_direction dir)
100{
101 extern void ___dma_page_cpu_to_dev(struct page *, unsigned long,
102 size_t, enum dma_data_direction);
103
104 if (!arch_is_coherent())
105 ___dma_page_cpu_to_dev(page, off, size, dir);
106}
107
108static inline void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
109 size_t size, enum dma_data_direction dir)
110{
111 extern void ___dma_page_dev_to_cpu(struct page *, unsigned long,
112 size_t, enum dma_data_direction);
113
114 if (!arch_is_coherent())
115 ___dma_page_dev_to_cpu(page, off, size, dir);
116}
117
118extern int dma_supported(struct device *, u64);
119extern int dma_set_mask(struct device *, u64);
120
121/*
122 * DMA errors are defined by all-bits-set in the DMA address. 89 * DMA errors are defined by all-bits-set in the DMA address.
123 */ 90 */
124static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 91static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
125{ 92{
126 return dma_addr == ~0; 93 return dma_addr == DMA_ERROR_CODE;
127} 94}
128 95
129/* 96/*
@@ -141,69 +108,118 @@ static inline void dma_free_noncoherent(struct device *dev, size_t size,
141{ 108{
142} 109}
143 110
111extern int dma_supported(struct device *dev, u64 mask);
112
144/** 113/**
145 * dma_alloc_coherent - allocate consistent memory for DMA 114 * arm_dma_alloc - allocate consistent memory for DMA
146 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 115 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
147 * @size: required memory size 116 * @size: required memory size
148 * @handle: bus-specific DMA address 117 * @handle: bus-specific DMA address
118 * @attrs: optinal attributes that specific mapping properties
149 * 119 *
150 * Allocate some uncached, unbuffered memory for a device for 120 * Allocate some memory for a device for performing DMA. This function
151 * performing DMA. This function allocates pages, and will 121 * allocates pages, and will return the CPU-viewed address, and sets @handle
152 * return the CPU-viewed address, and sets @handle to be the 122 * to be the device-viewed address.
153 * device-viewed address.
154 */ 123 */
155extern void *dma_alloc_coherent(struct device *, size_t, dma_addr_t *, gfp_t); 124extern void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
125 gfp_t gfp, struct dma_attrs *attrs);
126
127#define dma_alloc_coherent(d, s, h, f) dma_alloc_attrs(d, s, h, f, NULL)
128
129static inline void *dma_alloc_attrs(struct device *dev, size_t size,
130 dma_addr_t *dma_handle, gfp_t flag,
131 struct dma_attrs *attrs)
132{
133 struct dma_map_ops *ops = get_dma_ops(dev);
134 void *cpu_addr;
135 BUG_ON(!ops);
136
137 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
138 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr);
139 return cpu_addr;
140}
156 141
157/** 142/**
158 * dma_free_coherent - free memory allocated by dma_alloc_coherent 143 * arm_dma_free - free memory allocated by arm_dma_alloc
159 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 144 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
160 * @size: size of memory originally requested in dma_alloc_coherent 145 * @size: size of memory originally requested in dma_alloc_coherent
161 * @cpu_addr: CPU-view address returned from dma_alloc_coherent 146 * @cpu_addr: CPU-view address returned from dma_alloc_coherent
162 * @handle: device-view address returned from dma_alloc_coherent 147 * @handle: device-view address returned from dma_alloc_coherent
148 * @attrs: optinal attributes that specific mapping properties
163 * 149 *
164 * Free (and unmap) a DMA buffer previously allocated by 150 * Free (and unmap) a DMA buffer previously allocated by
165 * dma_alloc_coherent(). 151 * arm_dma_alloc().
166 * 152 *
167 * References to memory and mappings associated with cpu_addr/handle 153 * References to memory and mappings associated with cpu_addr/handle
168 * during and after this call executing are illegal. 154 * during and after this call executing are illegal.
169 */ 155 */
170extern void dma_free_coherent(struct device *, size_t, void *, dma_addr_t); 156extern void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
157 dma_addr_t handle, struct dma_attrs *attrs);
158
159#define dma_free_coherent(d, s, c, h) dma_free_attrs(d, s, c, h, NULL)
160
161static inline void dma_free_attrs(struct device *dev, size_t size,
162 void *cpu_addr, dma_addr_t dma_handle,
163 struct dma_attrs *attrs)
164{
165 struct dma_map_ops *ops = get_dma_ops(dev);
166 BUG_ON(!ops);
167
168 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
169 ops->free(dev, size, cpu_addr, dma_handle, attrs);
170}
171 171
172/** 172/**
173 * dma_mmap_coherent - map a coherent DMA allocation into user space 173 * arm_dma_mmap - map a coherent DMA allocation into user space
174 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 174 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
175 * @vma: vm_area_struct describing requested user mapping 175 * @vma: vm_area_struct describing requested user mapping
176 * @cpu_addr: kernel CPU-view address returned from dma_alloc_coherent 176 * @cpu_addr: kernel CPU-view address returned from dma_alloc_coherent
177 * @handle: device-view address returned from dma_alloc_coherent 177 * @handle: device-view address returned from dma_alloc_coherent
178 * @size: size of memory originally requested in dma_alloc_coherent 178 * @size: size of memory originally requested in dma_alloc_coherent
179 * @attrs: optinal attributes that specific mapping properties
179 * 180 *
180 * Map a coherent DMA buffer previously allocated by dma_alloc_coherent 181 * Map a coherent DMA buffer previously allocated by dma_alloc_coherent
181 * into user space. The coherent DMA buffer must not be freed by the 182 * into user space. The coherent DMA buffer must not be freed by the
182 * driver until the user space mapping has been released. 183 * driver until the user space mapping has been released.
183 */ 184 */
184int dma_mmap_coherent(struct device *, struct vm_area_struct *, 185extern int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
185 void *, dma_addr_t, size_t); 186 void *cpu_addr, dma_addr_t dma_addr, size_t size,
187 struct dma_attrs *attrs);
186 188
189#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, NULL)
187 190
188/** 191static inline int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
189 * dma_alloc_writecombine - allocate writecombining memory for DMA 192 void *cpu_addr, dma_addr_t dma_addr,
190 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 193 size_t size, struct dma_attrs *attrs)
191 * @size: required memory size 194{
192 * @handle: bus-specific DMA address 195 struct dma_map_ops *ops = get_dma_ops(dev);
193 * 196 BUG_ON(!ops);
194 * Allocate some uncached, buffered memory for a device for 197 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
195 * performing DMA. This function allocates pages, and will 198}
196 * return the CPU-viewed address, and sets @handle to be the 199
197 * device-viewed address. 200static inline void *dma_alloc_writecombine(struct device *dev, size_t size,
198 */ 201 dma_addr_t *dma_handle, gfp_t flag)
199extern void *dma_alloc_writecombine(struct device *, size_t, dma_addr_t *, 202{
200 gfp_t); 203 DEFINE_DMA_ATTRS(attrs);
204 dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
205 return dma_alloc_attrs(dev, size, dma_handle, flag, &attrs);
206}
201 207
202#define dma_free_writecombine(dev,size,cpu_addr,handle) \ 208static inline void dma_free_writecombine(struct device *dev, size_t size,
203 dma_free_coherent(dev,size,cpu_addr,handle) 209 void *cpu_addr, dma_addr_t dma_handle)
210{
211 DEFINE_DMA_ATTRS(attrs);
212 dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
213 return dma_free_attrs(dev, size, cpu_addr, dma_handle, &attrs);
214}
204 215
205int dma_mmap_writecombine(struct device *, struct vm_area_struct *, 216static inline int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
206 void *, dma_addr_t, size_t); 217 void *cpu_addr, dma_addr_t dma_addr, size_t size)
218{
219 DEFINE_DMA_ATTRS(attrs);
220 dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
221 return dma_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, &attrs);
222}
207 223
208/* 224/*
209 * This can be called during boot to increase the size of the consistent 225 * This can be called during boot to increase the size of the consistent
@@ -212,8 +228,6 @@ int dma_mmap_writecombine(struct device *, struct vm_area_struct *,
212 */ 228 */
213extern void __init init_consistent_dma_size(unsigned long size); 229extern void __init init_consistent_dma_size(unsigned long size);
214 230
215
216#ifdef CONFIG_DMABOUNCE
217/* 231/*
218 * For SA-1111, IXP425, and ADI systems the dma-mapping functions are "magic" 232 * For SA-1111, IXP425, and ADI systems the dma-mapping functions are "magic"
219 * and utilize bounce buffers as needed to work around limited DMA windows. 233 * and utilize bounce buffers as needed to work around limited DMA windows.
@@ -253,222 +267,19 @@ extern int dmabounce_register_dev(struct device *, unsigned long,
253 */ 267 */
254extern void dmabounce_unregister_dev(struct device *); 268extern void dmabounce_unregister_dev(struct device *);
255 269
256/*
257 * The DMA API, implemented by dmabounce.c. See below for descriptions.
258 */
259extern dma_addr_t __dma_map_page(struct device *, struct page *,
260 unsigned long, size_t, enum dma_data_direction);
261extern void __dma_unmap_page(struct device *, dma_addr_t, size_t,
262 enum dma_data_direction);
263
264/*
265 * Private functions
266 */
267int dmabounce_sync_for_cpu(struct device *, dma_addr_t, unsigned long,
268 size_t, enum dma_data_direction);
269int dmabounce_sync_for_device(struct device *, dma_addr_t, unsigned long,
270 size_t, enum dma_data_direction);
271#else
272static inline int dmabounce_sync_for_cpu(struct device *d, dma_addr_t addr,
273 unsigned long offset, size_t size, enum dma_data_direction dir)
274{
275 return 1;
276}
277 270
278static inline int dmabounce_sync_for_device(struct device *d, dma_addr_t addr,
279 unsigned long offset, size_t size, enum dma_data_direction dir)
280{
281 return 1;
282}
283
284
285static inline dma_addr_t __dma_map_page(struct device *dev, struct page *page,
286 unsigned long offset, size_t size, enum dma_data_direction dir)
287{
288 __dma_page_cpu_to_dev(page, offset, size, dir);
289 return pfn_to_dma(dev, page_to_pfn(page)) + offset;
290}
291
292static inline void __dma_unmap_page(struct device *dev, dma_addr_t handle,
293 size_t size, enum dma_data_direction dir)
294{
295 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
296 handle & ~PAGE_MASK, size, dir);
297}
298#endif /* CONFIG_DMABOUNCE */
299
300/**
301 * dma_map_single - map a single buffer for streaming DMA
302 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
303 * @cpu_addr: CPU direct mapped address of buffer
304 * @size: size of buffer to map
305 * @dir: DMA transfer direction
306 *
307 * Ensure that any data held in the cache is appropriately discarded
308 * or written back.
309 *
310 * The device owns this memory once this call has completed. The CPU
311 * can regain ownership by calling dma_unmap_single() or
312 * dma_sync_single_for_cpu().
313 */
314static inline dma_addr_t dma_map_single(struct device *dev, void *cpu_addr,
315 size_t size, enum dma_data_direction dir)
316{
317 unsigned long offset;
318 struct page *page;
319 dma_addr_t addr;
320
321 BUG_ON(!virt_addr_valid(cpu_addr));
322 BUG_ON(!virt_addr_valid(cpu_addr + size - 1));
323 BUG_ON(!valid_dma_direction(dir));
324
325 page = virt_to_page(cpu_addr);
326 offset = (unsigned long)cpu_addr & ~PAGE_MASK;
327 addr = __dma_map_page(dev, page, offset, size, dir);
328 debug_dma_map_page(dev, page, offset, size, dir, addr, true);
329
330 return addr;
331}
332
333/**
334 * dma_map_page - map a portion of a page for streaming DMA
335 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
336 * @page: page that buffer resides in
337 * @offset: offset into page for start of buffer
338 * @size: size of buffer to map
339 * @dir: DMA transfer direction
340 *
341 * Ensure that any data held in the cache is appropriately discarded
342 * or written back.
343 *
344 * The device owns this memory once this call has completed. The CPU
345 * can regain ownership by calling dma_unmap_page().
346 */
347static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
348 unsigned long offset, size_t size, enum dma_data_direction dir)
349{
350 dma_addr_t addr;
351
352 BUG_ON(!valid_dma_direction(dir));
353
354 addr = __dma_map_page(dev, page, offset, size, dir);
355 debug_dma_map_page(dev, page, offset, size, dir, addr, false);
356
357 return addr;
358}
359
360/**
361 * dma_unmap_single - unmap a single buffer previously mapped
362 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
363 * @handle: DMA address of buffer
364 * @size: size of buffer (same as passed to dma_map_single)
365 * @dir: DMA transfer direction (same as passed to dma_map_single)
366 *
367 * Unmap a single streaming mode DMA translation. The handle and size
368 * must match what was provided in the previous dma_map_single() call.
369 * All other usages are undefined.
370 *
371 * After this call, reads by the CPU to the buffer are guaranteed to see
372 * whatever the device wrote there.
373 */
374static inline void dma_unmap_single(struct device *dev, dma_addr_t handle,
375 size_t size, enum dma_data_direction dir)
376{
377 debug_dma_unmap_page(dev, handle, size, dir, true);
378 __dma_unmap_page(dev, handle, size, dir);
379}
380
381/**
382 * dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
383 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
384 * @handle: DMA address of buffer
385 * @size: size of buffer (same as passed to dma_map_page)
386 * @dir: DMA transfer direction (same as passed to dma_map_page)
387 *
388 * Unmap a page streaming mode DMA translation. The handle and size
389 * must match what was provided in the previous dma_map_page() call.
390 * All other usages are undefined.
391 *
392 * After this call, reads by the CPU to the buffer are guaranteed to see
393 * whatever the device wrote there.
394 */
395static inline void dma_unmap_page(struct device *dev, dma_addr_t handle,
396 size_t size, enum dma_data_direction dir)
397{
398 debug_dma_unmap_page(dev, handle, size, dir, false);
399 __dma_unmap_page(dev, handle, size, dir);
400}
401
402/**
403 * dma_sync_single_range_for_cpu
404 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
405 * @handle: DMA address of buffer
406 * @offset: offset of region to start sync
407 * @size: size of region to sync
408 * @dir: DMA transfer direction (same as passed to dma_map_single)
409 *
410 * Make physical memory consistent for a single streaming mode DMA
411 * translation after a transfer.
412 *
413 * If you perform a dma_map_single() but wish to interrogate the
414 * buffer using the cpu, yet do not wish to teardown the PCI dma
415 * mapping, you must call this function before doing so. At the
416 * next point you give the PCI dma address back to the card, you
417 * must first the perform a dma_sync_for_device, and then the
418 * device again owns the buffer.
419 */
420static inline void dma_sync_single_range_for_cpu(struct device *dev,
421 dma_addr_t handle, unsigned long offset, size_t size,
422 enum dma_data_direction dir)
423{
424 BUG_ON(!valid_dma_direction(dir));
425
426 debug_dma_sync_single_for_cpu(dev, handle + offset, size, dir);
427
428 if (!dmabounce_sync_for_cpu(dev, handle, offset, size, dir))
429 return;
430
431 __dma_single_dev_to_cpu(dma_to_virt(dev, handle) + offset, size, dir);
432}
433
434static inline void dma_sync_single_range_for_device(struct device *dev,
435 dma_addr_t handle, unsigned long offset, size_t size,
436 enum dma_data_direction dir)
437{
438 BUG_ON(!valid_dma_direction(dir));
439
440 debug_dma_sync_single_for_device(dev, handle + offset, size, dir);
441
442 if (!dmabounce_sync_for_device(dev, handle, offset, size, dir))
443 return;
444
445 __dma_single_cpu_to_dev(dma_to_virt(dev, handle) + offset, size, dir);
446}
447
448static inline void dma_sync_single_for_cpu(struct device *dev,
449 dma_addr_t handle, size_t size, enum dma_data_direction dir)
450{
451 dma_sync_single_range_for_cpu(dev, handle, 0, size, dir);
452}
453
454static inline void dma_sync_single_for_device(struct device *dev,
455 dma_addr_t handle, size_t size, enum dma_data_direction dir)
456{
457 dma_sync_single_range_for_device(dev, handle, 0, size, dir);
458}
459 271
460/* 272/*
461 * The scatter list versions of the above methods. 273 * The scatter list versions of the above methods.
462 */ 274 */
463extern int dma_map_sg(struct device *, struct scatterlist *, int, 275extern int arm_dma_map_sg(struct device *, struct scatterlist *, int,
464 enum dma_data_direction); 276 enum dma_data_direction, struct dma_attrs *attrs);
465extern void dma_unmap_sg(struct device *, struct scatterlist *, int, 277extern void arm_dma_unmap_sg(struct device *, struct scatterlist *, int,
278 enum dma_data_direction, struct dma_attrs *attrs);
279extern void arm_dma_sync_sg_for_cpu(struct device *, struct scatterlist *, int,
466 enum dma_data_direction); 280 enum dma_data_direction);
467extern void dma_sync_sg_for_cpu(struct device *, struct scatterlist *, int, 281extern void arm_dma_sync_sg_for_device(struct device *, struct scatterlist *, int,
468 enum dma_data_direction); 282 enum dma_data_direction);
469extern void dma_sync_sg_for_device(struct device *, struct scatterlist *, int,
470 enum dma_data_direction);
471
472 283
473#endif /* __KERNEL__ */ 284#endif /* __KERNEL__ */
474#endif 285#endif
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 153f5559406a..ea6b43154090 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -21,6 +21,8 @@
21#include <linux/highmem.h> 21#include <linux/highmem.h>
22#include <linux/memblock.h> 22#include <linux/memblock.h>
23#include <linux/slab.h> 23#include <linux/slab.h>
24#include <linux/iommu.h>
25#include <linux/vmalloc.h>
24 26
25#include <asm/memory.h> 27#include <asm/memory.h>
26#include <asm/highmem.h> 28#include <asm/highmem.h>
@@ -28,12 +30,112 @@
28#include <asm/tlbflush.h> 30#include <asm/tlbflush.h>
29#include <asm/sizes.h> 31#include <asm/sizes.h>
30#include <asm/mach/arch.h> 32#include <asm/mach/arch.h>
33#include <asm/dma-iommu.h>
31#include <asm/mach/map.h> 34#include <asm/mach/map.h>
32#include <asm/system_info.h> 35#include <asm/system_info.h>
33#include <asm/dma-contiguous.h> 36#include <asm/dma-contiguous.h>
34 37
35#include "mm.h" 38#include "mm.h"
36 39
40/*
41 * The DMA API is built upon the notion of "buffer ownership". A buffer
42 * is either exclusively owned by the CPU (and therefore may be accessed
43 * by it) or exclusively owned by the DMA device. These helper functions
44 * represent the transitions between these two ownership states.
45 *
46 * Note, however, that on later ARMs, this notion does not work due to
47 * speculative prefetches. We model our approach on the assumption that
48 * the CPU does do speculative prefetches, which means we clean caches
49 * before transfers and delay cache invalidation until transfer completion.
50 *
51 */
52static void __dma_page_cpu_to_dev(struct page *, unsigned long,
53 size_t, enum dma_data_direction);
54static void __dma_page_dev_to_cpu(struct page *, unsigned long,
55 size_t, enum dma_data_direction);
56
57/**
58 * arm_dma_map_page - map a portion of a page for streaming DMA
59 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
60 * @page: page that buffer resides in
61 * @offset: offset into page for start of buffer
62 * @size: size of buffer to map
63 * @dir: DMA transfer direction
64 *
65 * Ensure that any data held in the cache is appropriately discarded
66 * or written back.
67 *
68 * The device owns this memory once this call has completed. The CPU
69 * can regain ownership by calling dma_unmap_page().
70 */
71static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
72 unsigned long offset, size_t size, enum dma_data_direction dir,
73 struct dma_attrs *attrs)
74{
75 if (!arch_is_coherent())
76 __dma_page_cpu_to_dev(page, offset, size, dir);
77 return pfn_to_dma(dev, page_to_pfn(page)) + offset;
78}
79
80/**
81 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
82 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
83 * @handle: DMA address of buffer
84 * @size: size of buffer (same as passed to dma_map_page)
85 * @dir: DMA transfer direction (same as passed to dma_map_page)
86 *
87 * Unmap a page streaming mode DMA translation. The handle and size
88 * must match what was provided in the previous dma_map_page() call.
89 * All other usages are undefined.
90 *
91 * After this call, reads by the CPU to the buffer are guaranteed to see
92 * whatever the device wrote there.
93 */
94static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
95 size_t size, enum dma_data_direction dir,
96 struct dma_attrs *attrs)
97{
98 if (!arch_is_coherent())
99 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
100 handle & ~PAGE_MASK, size, dir);
101}
102
103static void arm_dma_sync_single_for_cpu(struct device *dev,
104 dma_addr_t handle, size_t size, enum dma_data_direction dir)
105{
106 unsigned int offset = handle & (PAGE_SIZE - 1);
107 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
108 if (!arch_is_coherent())
109 __dma_page_dev_to_cpu(page, offset, size, dir);
110}
111
112static void arm_dma_sync_single_for_device(struct device *dev,
113 dma_addr_t handle, size_t size, enum dma_data_direction dir)
114{
115 unsigned int offset = handle & (PAGE_SIZE - 1);
116 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
117 if (!arch_is_coherent())
118 __dma_page_cpu_to_dev(page, offset, size, dir);
119}
120
121static int arm_dma_set_mask(struct device *dev, u64 dma_mask);
122
123struct dma_map_ops arm_dma_ops = {
124 .alloc = arm_dma_alloc,
125 .free = arm_dma_free,
126 .mmap = arm_dma_mmap,
127 .map_page = arm_dma_map_page,
128 .unmap_page = arm_dma_unmap_page,
129 .map_sg = arm_dma_map_sg,
130 .unmap_sg = arm_dma_unmap_sg,
131 .sync_single_for_cpu = arm_dma_sync_single_for_cpu,
132 .sync_single_for_device = arm_dma_sync_single_for_device,
133 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,
134 .sync_sg_for_device = arm_dma_sync_sg_for_device,
135 .set_dma_mask = arm_dma_set_mask,
136};
137EXPORT_SYMBOL(arm_dma_ops);
138
37static u64 get_coherent_dma_mask(struct device *dev) 139static u64 get_coherent_dma_mask(struct device *dev)
38{ 140{
39 u64 mask = (u64)arm_dma_limit; 141 u64 mask = (u64)arm_dma_limit;
@@ -69,9 +171,11 @@ static void __dma_clear_buffer(struct page *page, size_t size)
69 * lurking in the kernel direct-mapped region is invalidated. 171 * lurking in the kernel direct-mapped region is invalidated.
70 */ 172 */
71 ptr = page_address(page); 173 ptr = page_address(page);
72 memset(ptr, 0, size); 174 if (ptr) {
73 dmac_flush_range(ptr, ptr + size); 175 memset(ptr, 0, size);
74 outer_flush_range(__pa(ptr), __pa(ptr) + size); 176 dmac_flush_range(ptr, ptr + size);
177 outer_flush_range(__pa(ptr), __pa(ptr) + size);
178 }
75} 179}
76 180
77/* 181/*
@@ -164,8 +268,10 @@ static int __init consistent_init(void)
164 unsigned long base = consistent_base; 268 unsigned long base = consistent_base;
165 unsigned long num_ptes = (CONSISTENT_END - base) >> PMD_SHIFT; 269 unsigned long num_ptes = (CONSISTENT_END - base) >> PMD_SHIFT;
166 270
271#ifndef CONFIG_ARM_DMA_USE_IOMMU
167 if (cpu_architecture() >= CPU_ARCH_ARMv6) 272 if (cpu_architecture() >= CPU_ARCH_ARMv6)
168 return 0; 273 return 0;
274#endif
169 275
170 consistent_pte = kmalloc(num_ptes * sizeof(pte_t), GFP_KERNEL); 276 consistent_pte = kmalloc(num_ptes * sizeof(pte_t), GFP_KERNEL);
171 if (!consistent_pte) { 277 if (!consistent_pte) {
@@ -181,14 +287,14 @@ static int __init consistent_init(void)
181 287
182 pud = pud_alloc(&init_mm, pgd, base); 288 pud = pud_alloc(&init_mm, pgd, base);
183 if (!pud) { 289 if (!pud) {
184 printk(KERN_ERR "%s: no pud tables\n", __func__); 290 pr_err("%s: no pud tables\n", __func__);
185 ret = -ENOMEM; 291 ret = -ENOMEM;
186 break; 292 break;
187 } 293 }
188 294
189 pmd = pmd_alloc(&init_mm, pud, base); 295 pmd = pmd_alloc(&init_mm, pud, base);
190 if (!pmd) { 296 if (!pmd) {
191 printk(KERN_ERR "%s: no pmd tables\n", __func__); 297 pr_err("%s: no pmd tables\n", __func__);
192 ret = -ENOMEM; 298 ret = -ENOMEM;
193 break; 299 break;
194 } 300 }
@@ -196,7 +302,7 @@ static int __init consistent_init(void)
196 302
197 pte = pte_alloc_kernel(pmd, base); 303 pte = pte_alloc_kernel(pmd, base);
198 if (!pte) { 304 if (!pte) {
199 printk(KERN_ERR "%s: no pte tables\n", __func__); 305 pr_err("%s: no pte tables\n", __func__);
200 ret = -ENOMEM; 306 ret = -ENOMEM;
201 break; 307 break;
202 } 308 }
@@ -311,7 +417,7 @@ __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
311 int bit; 417 int bit;
312 418
313 if (!consistent_pte) { 419 if (!consistent_pte) {
314 printk(KERN_ERR "%s: not initialised\n", __func__); 420 pr_err("%s: not initialised\n", __func__);
315 dump_stack(); 421 dump_stack();
316 return NULL; 422 return NULL;
317 } 423 }
@@ -338,7 +444,7 @@ __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
338 u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1); 444 u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
339 445
340 pte = consistent_pte[idx] + off; 446 pte = consistent_pte[idx] + off;
341 c->vm_pages = page; 447 c->priv = page;
342 448
343 do { 449 do {
344 BUG_ON(!pte_none(*pte)); 450 BUG_ON(!pte_none(*pte));
@@ -370,14 +476,14 @@ static void __dma_free_remap(void *cpu_addr, size_t size)
370 476
371 c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr); 477 c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr);
372 if (!c) { 478 if (!c) {
373 printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n", 479 pr_err("%s: trying to free invalid coherent area: %p\n",
374 __func__, cpu_addr); 480 __func__, cpu_addr);
375 dump_stack(); 481 dump_stack();
376 return; 482 return;
377 } 483 }
378 484
379 if ((c->vm_end - c->vm_start) != size) { 485 if ((c->vm_end - c->vm_start) != size) {
380 printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n", 486 pr_err("%s: freeing wrong coherent size (%ld != %d)\n",
381 __func__, c->vm_end - c->vm_start, size); 487 __func__, c->vm_end - c->vm_start, size);
382 dump_stack(); 488 dump_stack();
383 size = c->vm_end - c->vm_start; 489 size = c->vm_end - c->vm_start;
@@ -399,8 +505,8 @@ static void __dma_free_remap(void *cpu_addr, size_t size)
399 } 505 }
400 506
401 if (pte_none(pte) || !pte_present(pte)) 507 if (pte_none(pte) || !pte_present(pte))
402 printk(KERN_CRIT "%s: bad page in kernel page table\n", 508 pr_crit("%s: bad page in kernel page table\n",
403 __func__); 509 __func__);
404 } while (size -= PAGE_SIZE); 510 } while (size -= PAGE_SIZE);
405 511
406 flush_tlb_kernel_range(c->vm_start, c->vm_end); 512 flush_tlb_kernel_range(c->vm_start, c->vm_end);
@@ -524,12 +630,21 @@ static void __free_from_contiguous(struct device *dev, struct page *page,
524 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); 630 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
525} 631}
526 632
633static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
634{
635 prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?
636 pgprot_writecombine(prot) :
637 pgprot_dmacoherent(prot);
638 return prot;
639}
640
527#define nommu() 0 641#define nommu() 0
528 642
529#else /* !CONFIG_MMU */ 643#else /* !CONFIG_MMU */
530 644
531#define nommu() 1 645#define nommu() 1
532 646
647#define __get_dma_pgprot(attrs, prot) __pgprot(0)
533#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL 648#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL
534#define __alloc_from_pool(dev, size, ret_page, c) NULL 649#define __alloc_from_pool(dev, size, ret_page, c) NULL
535#define __alloc_from_contiguous(dev, size, prot, ret) NULL 650#define __alloc_from_contiguous(dev, size, prot, ret) NULL
@@ -584,7 +699,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
584 */ 699 */
585 gfp &= ~(__GFP_COMP); 700 gfp &= ~(__GFP_COMP);
586 701
587 *handle = ~0; 702 *handle = DMA_ERROR_CODE;
588 size = PAGE_ALIGN(size); 703 size = PAGE_ALIGN(size);
589 704
590 if (arch_is_coherent() || nommu()) 705 if (arch_is_coherent() || nommu())
@@ -606,39 +721,34 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
606 * Allocate DMA-coherent memory space and return both the kernel remapped 721 * Allocate DMA-coherent memory space and return both the kernel remapped
607 * virtual and bus address for that space. 722 * virtual and bus address for that space.
608 */ 723 */
609void *dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, 724void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
610 gfp_t gfp) 725 gfp_t gfp, struct dma_attrs *attrs)
611{ 726{
727 pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
612 void *memory; 728 void *memory;
613 729
614 if (dma_alloc_from_coherent(dev, size, handle, &memory)) 730 if (dma_alloc_from_coherent(dev, size, handle, &memory))
615 return memory; 731 return memory;
616 732
617 return __dma_alloc(dev, size, handle, gfp, 733 return __dma_alloc(dev, size, handle, gfp, prot,
618 pgprot_dmacoherent(pgprot_kernel),
619 __builtin_return_address(0)); 734 __builtin_return_address(0));
620} 735}
621EXPORT_SYMBOL(dma_alloc_coherent);
622 736
623/* 737/*
624 * Allocate a writecombining region, in much the same way as 738 * Create userspace mapping for the DMA-coherent memory.
625 * dma_alloc_coherent above.
626 */ 739 */
627void * 740int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
628dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp) 741 void *cpu_addr, dma_addr_t dma_addr, size_t size,
629{ 742 struct dma_attrs *attrs)
630 return __dma_alloc(dev, size, handle, gfp,
631 pgprot_writecombine(pgprot_kernel),
632 __builtin_return_address(0));
633}
634EXPORT_SYMBOL(dma_alloc_writecombine);
635
636static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
637 void *cpu_addr, dma_addr_t dma_addr, size_t size)
638{ 743{
639 int ret = -ENXIO; 744 int ret = -ENXIO;
640#ifdef CONFIG_MMU 745#ifdef CONFIG_MMU
641 unsigned long pfn = dma_to_pfn(dev, dma_addr); 746 unsigned long pfn = dma_to_pfn(dev, dma_addr);
747 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
748
749 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
750 return ret;
751
642 ret = remap_pfn_range(vma, vma->vm_start, 752 ret = remap_pfn_range(vma, vma->vm_start,
643 pfn + vma->vm_pgoff, 753 pfn + vma->vm_pgoff,
644 vma->vm_end - vma->vm_start, 754 vma->vm_end - vma->vm_start,
@@ -648,27 +758,11 @@ static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
648 return ret; 758 return ret;
649} 759}
650 760
651int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
652 void *cpu_addr, dma_addr_t dma_addr, size_t size)
653{
654 vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
655 return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
656}
657EXPORT_SYMBOL(dma_mmap_coherent);
658
659int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
660 void *cpu_addr, dma_addr_t dma_addr, size_t size)
661{
662 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
663 return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
664}
665EXPORT_SYMBOL(dma_mmap_writecombine);
666
667
668/* 761/*
669 * Free a buffer as defined by the above mapping. 762 * Free a buffer as defined by the above mapping.
670 */ 763 */
671void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle) 764void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
765 dma_addr_t handle, struct dma_attrs *attrs)
672{ 766{
673 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 767 struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
674 768
@@ -692,48 +786,6 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr
692 __free_from_contiguous(dev, page, size); 786 __free_from_contiguous(dev, page, size);
693 } 787 }
694} 788}
695EXPORT_SYMBOL(dma_free_coherent);
696
697/*
698 * Make an area consistent for devices.
699 * Note: Drivers should NOT use this function directly, as it will break
700 * platforms with CONFIG_DMABOUNCE.
701 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
702 */
703void ___dma_single_cpu_to_dev(const void *kaddr, size_t size,
704 enum dma_data_direction dir)
705{
706 unsigned long paddr;
707
708 BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));
709
710 dmac_map_area(kaddr, size, dir);
711
712 paddr = __pa(kaddr);
713 if (dir == DMA_FROM_DEVICE) {
714 outer_inv_range(paddr, paddr + size);
715 } else {
716 outer_clean_range(paddr, paddr + size);
717 }
718 /* FIXME: non-speculating: flush on bidirectional mappings? */
719}
720EXPORT_SYMBOL(___dma_single_cpu_to_dev);
721
722void ___dma_single_dev_to_cpu(const void *kaddr, size_t size,
723 enum dma_data_direction dir)
724{
725 BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));
726
727 /* FIXME: non-speculating: not required */
728 /* don't bother invalidating if DMA to device */
729 if (dir != DMA_TO_DEVICE) {
730 unsigned long paddr = __pa(kaddr);
731 outer_inv_range(paddr, paddr + size);
732 }
733
734 dmac_unmap_area(kaddr, size, dir);
735}
736EXPORT_SYMBOL(___dma_single_dev_to_cpu);
737 789
738static void dma_cache_maint_page(struct page *page, unsigned long offset, 790static void dma_cache_maint_page(struct page *page, unsigned long offset,
739 size_t size, enum dma_data_direction dir, 791 size_t size, enum dma_data_direction dir,
@@ -779,7 +831,13 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset,
779 } while (left); 831 } while (left);
780} 832}
781 833
782void ___dma_page_cpu_to_dev(struct page *page, unsigned long off, 834/*
835 * Make an area consistent for devices.
836 * Note: Drivers should NOT use this function directly, as it will break
837 * platforms with CONFIG_DMABOUNCE.
838 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
839 */
840static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
783 size_t size, enum dma_data_direction dir) 841 size_t size, enum dma_data_direction dir)
784{ 842{
785 unsigned long paddr; 843 unsigned long paddr;
@@ -794,9 +852,8 @@ void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
794 } 852 }
795 /* FIXME: non-speculating: flush on bidirectional mappings? */ 853 /* FIXME: non-speculating: flush on bidirectional mappings? */
796} 854}
797EXPORT_SYMBOL(___dma_page_cpu_to_dev);
798 855
799void ___dma_page_dev_to_cpu(struct page *page, unsigned long off, 856static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
800 size_t size, enum dma_data_direction dir) 857 size_t size, enum dma_data_direction dir)
801{ 858{
802 unsigned long paddr = page_to_phys(page) + off; 859 unsigned long paddr = page_to_phys(page) + off;
@@ -814,10 +871,9 @@ void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
814 if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE) 871 if (dir != DMA_TO_DEVICE && off == 0 && size >= PAGE_SIZE)
815 set_bit(PG_dcache_clean, &page->flags); 872 set_bit(PG_dcache_clean, &page->flags);
816} 873}
817EXPORT_SYMBOL(___dma_page_dev_to_cpu);
818 874
819/** 875/**
820 * dma_map_sg - map a set of SG buffers for streaming mode DMA 876 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
821 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 877 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
822 * @sg: list of buffers 878 * @sg: list of buffers
823 * @nents: number of buffers to map 879 * @nents: number of buffers to map
@@ -832,32 +888,32 @@ EXPORT_SYMBOL(___dma_page_dev_to_cpu);
832 * Device ownership issues as mentioned for dma_map_single are the same 888 * Device ownership issues as mentioned for dma_map_single are the same
833 * here. 889 * here.
834 */ 890 */
835int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 891int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
836 enum dma_data_direction dir) 892 enum dma_data_direction dir, struct dma_attrs *attrs)
837{ 893{
894 struct dma_map_ops *ops = get_dma_ops(dev);
838 struct scatterlist *s; 895 struct scatterlist *s;
839 int i, j; 896 int i, j;
840 897
841 BUG_ON(!valid_dma_direction(dir));
842
843 for_each_sg(sg, s, nents, i) { 898 for_each_sg(sg, s, nents, i) {
844 s->dma_address = __dma_map_page(dev, sg_page(s), s->offset, 899#ifdef CONFIG_NEED_SG_DMA_LENGTH
845 s->length, dir); 900 s->dma_length = s->length;
901#endif
902 s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
903 s->length, dir, attrs);
846 if (dma_mapping_error(dev, s->dma_address)) 904 if (dma_mapping_error(dev, s->dma_address))
847 goto bad_mapping; 905 goto bad_mapping;
848 } 906 }
849 debug_dma_map_sg(dev, sg, nents, nents, dir);
850 return nents; 907 return nents;
851 908
852 bad_mapping: 909 bad_mapping:
853 for_each_sg(sg, s, i, j) 910 for_each_sg(sg, s, i, j)
854 __dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir); 911 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
855 return 0; 912 return 0;
856} 913}
857EXPORT_SYMBOL(dma_map_sg);
858 914
859/** 915/**
860 * dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 916 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
861 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 917 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
862 * @sg: list of buffers 918 * @sg: list of buffers
863 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 919 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
@@ -866,70 +922,55 @@ EXPORT_SYMBOL(dma_map_sg);
866 * Unmap a set of streaming mode DMA translations. Again, CPU access 922 * Unmap a set of streaming mode DMA translations. Again, CPU access
867 * rules concerning calls here are the same as for dma_unmap_single(). 923 * rules concerning calls here are the same as for dma_unmap_single().
868 */ 924 */
869void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 925void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
870 enum dma_data_direction dir) 926 enum dma_data_direction dir, struct dma_attrs *attrs)
871{ 927{
928 struct dma_map_ops *ops = get_dma_ops(dev);
872 struct scatterlist *s; 929 struct scatterlist *s;
873 int i;
874 930
875 debug_dma_unmap_sg(dev, sg, nents, dir); 931 int i;
876 932
877 for_each_sg(sg, s, nents, i) 933 for_each_sg(sg, s, nents, i)
878 __dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir); 934 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
879} 935}
880EXPORT_SYMBOL(dma_unmap_sg);
881 936
882/** 937/**
883 * dma_sync_sg_for_cpu 938 * arm_dma_sync_sg_for_cpu
884 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 939 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
885 * @sg: list of buffers 940 * @sg: list of buffers
886 * @nents: number of buffers to map (returned from dma_map_sg) 941 * @nents: number of buffers to map (returned from dma_map_sg)
887 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 942 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
888 */ 943 */
889void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 944void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
890 int nents, enum dma_data_direction dir) 945 int nents, enum dma_data_direction dir)
891{ 946{
947 struct dma_map_ops *ops = get_dma_ops(dev);
892 struct scatterlist *s; 948 struct scatterlist *s;
893 int i; 949 int i;
894 950
895 for_each_sg(sg, s, nents, i) { 951 for_each_sg(sg, s, nents, i)
896 if (!dmabounce_sync_for_cpu(dev, sg_dma_address(s), 0, 952 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
897 sg_dma_len(s), dir)) 953 dir);
898 continue;
899
900 __dma_page_dev_to_cpu(sg_page(s), s->offset,
901 s->length, dir);
902 }
903
904 debug_dma_sync_sg_for_cpu(dev, sg, nents, dir);
905} 954}
906EXPORT_SYMBOL(dma_sync_sg_for_cpu);
907 955
908/** 956/**
909 * dma_sync_sg_for_device 957 * arm_dma_sync_sg_for_device
910 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 958 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
911 * @sg: list of buffers 959 * @sg: list of buffers
912 * @nents: number of buffers to map (returned from dma_map_sg) 960 * @nents: number of buffers to map (returned from dma_map_sg)
913 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 961 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
914 */ 962 */
915void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 963void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
916 int nents, enum dma_data_direction dir) 964 int nents, enum dma_data_direction dir)
917{ 965{
966 struct dma_map_ops *ops = get_dma_ops(dev);
918 struct scatterlist *s; 967 struct scatterlist *s;
919 int i; 968 int i;
920 969
921 for_each_sg(sg, s, nents, i) { 970 for_each_sg(sg, s, nents, i)
922 if (!dmabounce_sync_for_device(dev, sg_dma_address(s), 0, 971 ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
923 sg_dma_len(s), dir)) 972 dir);
924 continue;
925
926 __dma_page_cpu_to_dev(sg_page(s), s->offset,
927 s->length, dir);
928 }
929
930 debug_dma_sync_sg_for_device(dev, sg, nents, dir);
931} 973}
932EXPORT_SYMBOL(dma_sync_sg_for_device);
933 974
934/* 975/*
935 * Return whether the given device DMA address mask can be supported 976 * Return whether the given device DMA address mask can be supported
@@ -945,18 +986,15 @@ int dma_supported(struct device *dev, u64 mask)
945} 986}
946EXPORT_SYMBOL(dma_supported); 987EXPORT_SYMBOL(dma_supported);
947 988
948int dma_set_mask(struct device *dev, u64 dma_mask) 989static int arm_dma_set_mask(struct device *dev, u64 dma_mask)
949{ 990{
950 if (!dev->dma_mask || !dma_supported(dev, dma_mask)) 991 if (!dev->dma_mask || !dma_supported(dev, dma_mask))
951 return -EIO; 992 return -EIO;
952 993
953#ifndef CONFIG_DMABOUNCE
954 *dev->dma_mask = dma_mask; 994 *dev->dma_mask = dma_mask;
955#endif
956 995
957 return 0; 996 return 0;
958} 997}
959EXPORT_SYMBOL(dma_set_mask);
960 998
961#define PREALLOC_DMA_DEBUG_ENTRIES 4096 999#define PREALLOC_DMA_DEBUG_ENTRIES 4096
962 1000
@@ -969,3 +1007,679 @@ static int __init dma_debug_do_init(void)
969 return 0; 1007 return 0;
970} 1008}
971fs_initcall(dma_debug_do_init); 1009fs_initcall(dma_debug_do_init);
1010
1011#ifdef CONFIG_ARM_DMA_USE_IOMMU
1012
1013/* IOMMU */
1014
1015static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
1016 size_t size)
1017{
1018 unsigned int order = get_order(size);
1019 unsigned int align = 0;
1020 unsigned int count, start;
1021 unsigned long flags;
1022
1023 count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) +
1024 (1 << mapping->order) - 1) >> mapping->order;
1025
1026 if (order > mapping->order)
1027 align = (1 << (order - mapping->order)) - 1;
1028
1029 spin_lock_irqsave(&mapping->lock, flags);
1030 start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0,
1031 count, align);
1032 if (start > mapping->bits) {
1033 spin_unlock_irqrestore(&mapping->lock, flags);
1034 return DMA_ERROR_CODE;
1035 }
1036
1037 bitmap_set(mapping->bitmap, start, count);
1038 spin_unlock_irqrestore(&mapping->lock, flags);
1039
1040 return mapping->base + (start << (mapping->order + PAGE_SHIFT));
1041}
1042
1043static inline void __free_iova(struct dma_iommu_mapping *mapping,
1044 dma_addr_t addr, size_t size)
1045{
1046 unsigned int start = (addr - mapping->base) >>
1047 (mapping->order + PAGE_SHIFT);
1048 unsigned int count = ((size >> PAGE_SHIFT) +
1049 (1 << mapping->order) - 1) >> mapping->order;
1050 unsigned long flags;
1051
1052 spin_lock_irqsave(&mapping->lock, flags);
1053 bitmap_clear(mapping->bitmap, start, count);
1054 spin_unlock_irqrestore(&mapping->lock, flags);
1055}
1056
1057static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
1058{
1059 struct page **pages;
1060 int count = size >> PAGE_SHIFT;
1061 int array_size = count * sizeof(struct page *);
1062 int i = 0;
1063
1064 if (array_size <= PAGE_SIZE)
1065 pages = kzalloc(array_size, gfp);
1066 else
1067 pages = vzalloc(array_size);
1068 if (!pages)
1069 return NULL;
1070
1071 while (count) {
1072 int j, order = __ffs(count);
1073
1074 pages[i] = alloc_pages(gfp | __GFP_NOWARN, order);
1075 while (!pages[i] && order)
1076 pages[i] = alloc_pages(gfp | __GFP_NOWARN, --order);
1077 if (!pages[i])
1078 goto error;
1079
1080 if (order)
1081 split_page(pages[i], order);
1082 j = 1 << order;
1083 while (--j)
1084 pages[i + j] = pages[i] + j;
1085
1086 __dma_clear_buffer(pages[i], PAGE_SIZE << order);
1087 i += 1 << order;
1088 count -= 1 << order;
1089 }
1090
1091 return pages;
1092error:
1093 while (--i)
1094 if (pages[i])
1095 __free_pages(pages[i], 0);
1096 if (array_size < PAGE_SIZE)
1097 kfree(pages);
1098 else
1099 vfree(pages);
1100 return NULL;
1101}
1102
1103static int __iommu_free_buffer(struct device *dev, struct page **pages, size_t size)
1104{
1105 int count = size >> PAGE_SHIFT;
1106 int array_size = count * sizeof(struct page *);
1107 int i;
1108 for (i = 0; i < count; i++)
1109 if (pages[i])
1110 __free_pages(pages[i], 0);
1111 if (array_size < PAGE_SIZE)
1112 kfree(pages);
1113 else
1114 vfree(pages);
1115 return 0;
1116}
1117
1118/*
1119 * Create a CPU mapping for a specified pages
1120 */
1121static void *
1122__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot)
1123{
1124 struct arm_vmregion *c;
1125 size_t align;
1126 size_t count = size >> PAGE_SHIFT;
1127 int bit;
1128
1129 if (!consistent_pte[0]) {
1130 pr_err("%s: not initialised\n", __func__);
1131 dump_stack();
1132 return NULL;
1133 }
1134
1135 /*
1136 * Align the virtual region allocation - maximum alignment is
1137 * a section size, minimum is a page size. This helps reduce
1138 * fragmentation of the DMA space, and also prevents allocations
1139 * smaller than a section from crossing a section boundary.
1140 */
1141 bit = fls(size - 1);
1142 if (bit > SECTION_SHIFT)
1143 bit = SECTION_SHIFT;
1144 align = 1 << bit;
1145
1146 /*
1147 * Allocate a virtual address in the consistent mapping region.
1148 */
1149 c = arm_vmregion_alloc(&consistent_head, align, size,
1150 gfp & ~(__GFP_DMA | __GFP_HIGHMEM), NULL);
1151 if (c) {
1152 pte_t *pte;
1153 int idx = CONSISTENT_PTE_INDEX(c->vm_start);
1154 int i = 0;
1155 u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1);
1156
1157 pte = consistent_pte[idx] + off;
1158 c->priv = pages;
1159
1160 do {
1161 BUG_ON(!pte_none(*pte));
1162
1163 set_pte_ext(pte, mk_pte(pages[i], prot), 0);
1164 pte++;
1165 off++;
1166 i++;
1167 if (off >= PTRS_PER_PTE) {
1168 off = 0;
1169 pte = consistent_pte[++idx];
1170 }
1171 } while (i < count);
1172
1173 dsb();
1174
1175 return (void *)c->vm_start;
1176 }
1177 return NULL;
1178}
1179
1180/*
1181 * Create a mapping in device IO address space for specified pages
1182 */
1183static dma_addr_t
1184__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
1185{
1186 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1187 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1188 dma_addr_t dma_addr, iova;
1189 int i, ret = DMA_ERROR_CODE;
1190
1191 dma_addr = __alloc_iova(mapping, size);
1192 if (dma_addr == DMA_ERROR_CODE)
1193 return dma_addr;
1194
1195 iova = dma_addr;
1196 for (i = 0; i < count; ) {
1197 unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
1198 phys_addr_t phys = page_to_phys(pages[i]);
1199 unsigned int len, j;
1200
1201 for (j = i + 1; j < count; j++, next_pfn++)
1202 if (page_to_pfn(pages[j]) != next_pfn)
1203 break;
1204
1205 len = (j - i) << PAGE_SHIFT;
1206 ret = iommu_map(mapping->domain, iova, phys, len, 0);
1207 if (ret < 0)
1208 goto fail;
1209 iova += len;
1210 i = j;
1211 }
1212 return dma_addr;
1213fail:
1214 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
1215 __free_iova(mapping, dma_addr, size);
1216 return DMA_ERROR_CODE;
1217}
1218
1219static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
1220{
1221 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1222
1223 /*
1224 * add optional in-page offset from iova to size and align
1225 * result to page size
1226 */
1227 size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
1228 iova &= PAGE_MASK;
1229
1230 iommu_unmap(mapping->domain, iova, size);
1231 __free_iova(mapping, iova, size);
1232 return 0;
1233}
1234
1235static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
1236 dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
1237{
1238 pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
1239 struct page **pages;
1240 void *addr = NULL;
1241
1242 *handle = DMA_ERROR_CODE;
1243 size = PAGE_ALIGN(size);
1244
1245 pages = __iommu_alloc_buffer(dev, size, gfp);
1246 if (!pages)
1247 return NULL;
1248
1249 *handle = __iommu_create_mapping(dev, pages, size);
1250 if (*handle == DMA_ERROR_CODE)
1251 goto err_buffer;
1252
1253 addr = __iommu_alloc_remap(pages, size, gfp, prot);
1254 if (!addr)
1255 goto err_mapping;
1256
1257 return addr;
1258
1259err_mapping:
1260 __iommu_remove_mapping(dev, *handle, size);
1261err_buffer:
1262 __iommu_free_buffer(dev, pages, size);
1263 return NULL;
1264}
1265
1266static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
1267 void *cpu_addr, dma_addr_t dma_addr, size_t size,
1268 struct dma_attrs *attrs)
1269{
1270 struct arm_vmregion *c;
1271
1272 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
1273 c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
1274
1275 if (c) {
1276 struct page **pages = c->priv;
1277
1278 unsigned long uaddr = vma->vm_start;
1279 unsigned long usize = vma->vm_end - vma->vm_start;
1280 int i = 0;
1281
1282 do {
1283 int ret;
1284
1285 ret = vm_insert_page(vma, uaddr, pages[i++]);
1286 if (ret) {
1287 pr_err("Remapping memory, error: %d\n", ret);
1288 return ret;
1289 }
1290
1291 uaddr += PAGE_SIZE;
1292 usize -= PAGE_SIZE;
1293 } while (usize > 0);
1294 }
1295 return 0;
1296}
1297
1298/*
1299 * free a page as defined by the above mapping.
1300 * Must not be called with IRQs disabled.
1301 */
1302void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
1303 dma_addr_t handle, struct dma_attrs *attrs)
1304{
1305 struct arm_vmregion *c;
1306 size = PAGE_ALIGN(size);
1307
1308 c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr);
1309 if (c) {
1310 struct page **pages = c->priv;
1311 __dma_free_remap(cpu_addr, size);
1312 __iommu_remove_mapping(dev, handle, size);
1313 __iommu_free_buffer(dev, pages, size);
1314 }
1315}
1316
1317/*
1318 * Map a part of the scatter-gather list into contiguous io address space
1319 */
1320static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
1321 size_t size, dma_addr_t *handle,
1322 enum dma_data_direction dir)
1323{
1324 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1325 dma_addr_t iova, iova_base;
1326 int ret = 0;
1327 unsigned int count;
1328 struct scatterlist *s;
1329
1330 size = PAGE_ALIGN(size);
1331 *handle = DMA_ERROR_CODE;
1332
1333 iova_base = iova = __alloc_iova(mapping, size);
1334 if (iova == DMA_ERROR_CODE)
1335 return -ENOMEM;
1336
1337 for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
1338 phys_addr_t phys = page_to_phys(sg_page(s));
1339 unsigned int len = PAGE_ALIGN(s->offset + s->length);
1340
1341 if (!arch_is_coherent())
1342 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1343
1344 ret = iommu_map(mapping->domain, iova, phys, len, 0);
1345 if (ret < 0)
1346 goto fail;
1347 count += len >> PAGE_SHIFT;
1348 iova += len;
1349 }
1350 *handle = iova_base;
1351
1352 return 0;
1353fail:
1354 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
1355 __free_iova(mapping, iova_base, size);
1356 return ret;
1357}
1358
1359/**
1360 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1361 * @dev: valid struct device pointer
1362 * @sg: list of buffers
1363 * @nents: number of buffers to map
1364 * @dir: DMA transfer direction
1365 *
1366 * Map a set of buffers described by scatterlist in streaming mode for DMA.
1367 * The scatter gather list elements are merged together (if possible) and
1368 * tagged with the appropriate dma address and length. They are obtained via
1369 * sg_dma_{address,length}.
1370 */
1371int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1372 enum dma_data_direction dir, struct dma_attrs *attrs)
1373{
1374 struct scatterlist *s = sg, *dma = sg, *start = sg;
1375 int i, count = 0;
1376 unsigned int offset = s->offset;
1377 unsigned int size = s->offset + s->length;
1378 unsigned int max = dma_get_max_seg_size(dev);
1379
1380 for (i = 1; i < nents; i++) {
1381 s = sg_next(s);
1382
1383 s->dma_address = DMA_ERROR_CODE;
1384 s->dma_length = 0;
1385
1386 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
1387 if (__map_sg_chunk(dev, start, size, &dma->dma_address,
1388 dir) < 0)
1389 goto bad_mapping;
1390
1391 dma->dma_address += offset;
1392 dma->dma_length = size - offset;
1393
1394 size = offset = s->offset;
1395 start = s;
1396 dma = sg_next(dma);
1397 count += 1;
1398 }
1399 size += s->length;
1400 }
1401 if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir) < 0)
1402 goto bad_mapping;
1403
1404 dma->dma_address += offset;
1405 dma->dma_length = size - offset;
1406
1407 return count+1;
1408
1409bad_mapping:
1410 for_each_sg(sg, s, count, i)
1411 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
1412 return 0;
1413}
1414
1415/**
1416 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1417 * @dev: valid struct device pointer
1418 * @sg: list of buffers
1419 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1420 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1421 *
1422 * Unmap a set of streaming mode DMA translations. Again, CPU access
1423 * rules concerning calls here are the same as for dma_unmap_single().
1424 */
1425void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1426 enum dma_data_direction dir, struct dma_attrs *attrs)
1427{
1428 struct scatterlist *s;
1429 int i;
1430
1431 for_each_sg(sg, s, nents, i) {
1432 if (sg_dma_len(s))
1433 __iommu_remove_mapping(dev, sg_dma_address(s),
1434 sg_dma_len(s));
1435 if (!arch_is_coherent())
1436 __dma_page_dev_to_cpu(sg_page(s), s->offset,
1437 s->length, dir);
1438 }
1439}
1440
1441/**
1442 * arm_iommu_sync_sg_for_cpu
1443 * @dev: valid struct device pointer
1444 * @sg: list of buffers
1445 * @nents: number of buffers to map (returned from dma_map_sg)
1446 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1447 */
1448void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1449 int nents, enum dma_data_direction dir)
1450{
1451 struct scatterlist *s;
1452 int i;
1453
1454 for_each_sg(sg, s, nents, i)
1455 if (!arch_is_coherent())
1456 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
1457
1458}
1459
1460/**
1461 * arm_iommu_sync_sg_for_device
1462 * @dev: valid struct device pointer
1463 * @sg: list of buffers
1464 * @nents: number of buffers to map (returned from dma_map_sg)
1465 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1466 */
1467void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1468 int nents, enum dma_data_direction dir)
1469{
1470 struct scatterlist *s;
1471 int i;
1472
1473 for_each_sg(sg, s, nents, i)
1474 if (!arch_is_coherent())
1475 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1476}
1477
1478
1479/**
1480 * arm_iommu_map_page
1481 * @dev: valid struct device pointer
1482 * @page: page that buffer resides in
1483 * @offset: offset into page for start of buffer
1484 * @size: size of buffer to map
1485 * @dir: DMA transfer direction
1486 *
1487 * IOMMU aware version of arm_dma_map_page()
1488 */
1489static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
1490 unsigned long offset, size_t size, enum dma_data_direction dir,
1491 struct dma_attrs *attrs)
1492{
1493 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1494 dma_addr_t dma_addr;
1495 int ret, len = PAGE_ALIGN(size + offset);
1496
1497 if (!arch_is_coherent())
1498 __dma_page_cpu_to_dev(page, offset, size, dir);
1499
1500 dma_addr = __alloc_iova(mapping, len);
1501 if (dma_addr == DMA_ERROR_CODE)
1502 return dma_addr;
1503
1504 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 0);
1505 if (ret < 0)
1506 goto fail;
1507
1508 return dma_addr + offset;
1509fail:
1510 __free_iova(mapping, dma_addr, len);
1511 return DMA_ERROR_CODE;
1512}
1513
1514/**
1515 * arm_iommu_unmap_page
1516 * @dev: valid struct device pointer
1517 * @handle: DMA address of buffer
1518 * @size: size of buffer (same as passed to dma_map_page)
1519 * @dir: DMA transfer direction (same as passed to dma_map_page)
1520 *
1521 * IOMMU aware version of arm_dma_unmap_page()
1522 */
1523static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1524 size_t size, enum dma_data_direction dir,
1525 struct dma_attrs *attrs)
1526{
1527 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1528 dma_addr_t iova = handle & PAGE_MASK;
1529 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1530 int offset = handle & ~PAGE_MASK;
1531 int len = PAGE_ALIGN(size + offset);
1532
1533 if (!iova)
1534 return;
1535
1536 if (!arch_is_coherent())
1537 __dma_page_dev_to_cpu(page, offset, size, dir);
1538
1539 iommu_unmap(mapping->domain, iova, len);
1540 __free_iova(mapping, iova, len);
1541}
1542
1543static void arm_iommu_sync_single_for_cpu(struct device *dev,
1544 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1545{
1546 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1547 dma_addr_t iova = handle & PAGE_MASK;
1548 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1549 unsigned int offset = handle & ~PAGE_MASK;
1550
1551 if (!iova)
1552 return;
1553
1554 if (!arch_is_coherent())
1555 __dma_page_dev_to_cpu(page, offset, size, dir);
1556}
1557
1558static void arm_iommu_sync_single_for_device(struct device *dev,
1559 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1560{
1561 struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1562 dma_addr_t iova = handle & PAGE_MASK;
1563 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1564 unsigned int offset = handle & ~PAGE_MASK;
1565
1566 if (!iova)
1567 return;
1568
1569 __dma_page_cpu_to_dev(page, offset, size, dir);
1570}
1571
1572struct dma_map_ops iommu_ops = {
1573 .alloc = arm_iommu_alloc_attrs,
1574 .free = arm_iommu_free_attrs,
1575 .mmap = arm_iommu_mmap_attrs,
1576
1577 .map_page = arm_iommu_map_page,
1578 .unmap_page = arm_iommu_unmap_page,
1579 .sync_single_for_cpu = arm_iommu_sync_single_for_cpu,
1580 .sync_single_for_device = arm_iommu_sync_single_for_device,
1581
1582 .map_sg = arm_iommu_map_sg,
1583 .unmap_sg = arm_iommu_unmap_sg,
1584 .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu,
1585 .sync_sg_for_device = arm_iommu_sync_sg_for_device,
1586};
1587
1588/**
1589 * arm_iommu_create_mapping
1590 * @bus: pointer to the bus holding the client device (for IOMMU calls)
1591 * @base: start address of the valid IO address space
1592 * @size: size of the valid IO address space
1593 * @order: accuracy of the IO addresses allocations
1594 *
1595 * Creates a mapping structure which holds information about used/unused
1596 * IO address ranges, which is required to perform memory allocation and
1597 * mapping with IOMMU aware functions.
1598 *
1599 * The client device need to be attached to the mapping with
1600 * arm_iommu_attach_device function.
1601 */
1602struct dma_iommu_mapping *
1603arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size,
1604 int order)
1605{
1606 unsigned int count = size >> (PAGE_SHIFT + order);
1607 unsigned int bitmap_size = BITS_TO_LONGS(count) * sizeof(long);
1608 struct dma_iommu_mapping *mapping;
1609 int err = -ENOMEM;
1610
1611 if (!count)
1612 return ERR_PTR(-EINVAL);
1613
1614 mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
1615 if (!mapping)
1616 goto err;
1617
1618 mapping->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
1619 if (!mapping->bitmap)
1620 goto err2;
1621
1622 mapping->base = base;
1623 mapping->bits = BITS_PER_BYTE * bitmap_size;
1624 mapping->order = order;
1625 spin_lock_init(&mapping->lock);
1626
1627 mapping->domain = iommu_domain_alloc(bus);
1628 if (!mapping->domain)
1629 goto err3;
1630
1631 kref_init(&mapping->kref);
1632 return mapping;
1633err3:
1634 kfree(mapping->bitmap);
1635err2:
1636 kfree(mapping);
1637err:
1638 return ERR_PTR(err);
1639}
1640
1641static void release_iommu_mapping(struct kref *kref)
1642{
1643 struct dma_iommu_mapping *mapping =
1644 container_of(kref, struct dma_iommu_mapping, kref);
1645
1646 iommu_domain_free(mapping->domain);
1647 kfree(mapping->bitmap);
1648 kfree(mapping);
1649}
1650
1651void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
1652{
1653 if (mapping)
1654 kref_put(&mapping->kref, release_iommu_mapping);
1655}
1656
1657/**
1658 * arm_iommu_attach_device
1659 * @dev: valid struct device pointer
1660 * @mapping: io address space mapping structure (returned from
1661 * arm_iommu_create_mapping)
1662 *
1663 * Attaches specified io address space mapping to the provided device,
1664 * this replaces the dma operations (dma_map_ops pointer) with the
1665 * IOMMU aware version. More than one client might be attached to
1666 * the same io address space mapping.
1667 */
1668int arm_iommu_attach_device(struct device *dev,
1669 struct dma_iommu_mapping *mapping)
1670{
1671 int err;
1672
1673 err = iommu_attach_device(mapping->domain, dev);
1674 if (err)
1675 return err;
1676
1677 kref_get(&mapping->kref);
1678 dev->archdata.mapping = mapping;
1679 set_dma_ops(dev, &iommu_ops);
1680
1681 pr_info("Attached IOMMU controller to %s device.\n", dev_name(dev));
1682 return 0;
1683}
1684
1685#endif
diff --git a/arch/arm/mm/vmregion.h b/arch/arm/mm/vmregion.h
index 162be662c088..bf312c354a21 100644
--- a/arch/arm/mm/vmregion.h
+++ b/arch/arm/mm/vmregion.h
@@ -17,7 +17,7 @@ struct arm_vmregion {
17 struct list_head vm_list; 17 struct list_head vm_list;
18 unsigned long vm_start; 18 unsigned long vm_start;
19 unsigned long vm_end; 19 unsigned long vm_end;
20 struct page *vm_pages; 20 void *priv;
21 int vm_active; 21 int vm_active;
22 const void *caller; 22 const void *caller;
23}; 23};
diff --git a/drivers/base/dma-coherent.c b/drivers/base/dma-coherent.c
index bb0025c510b3..1b85949e3d2f 100644
--- a/drivers/base/dma-coherent.c
+++ b/drivers/base/dma-coherent.c
@@ -10,6 +10,7 @@
10struct dma_coherent_mem { 10struct dma_coherent_mem {
11 void *virt_base; 11 void *virt_base;
12 dma_addr_t device_base; 12 dma_addr_t device_base;
13 phys_addr_t pfn_base;
13 int size; 14 int size;
14 int flags; 15 int flags;
15 unsigned long *bitmap; 16 unsigned long *bitmap;
@@ -44,6 +45,7 @@ int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
44 45
45 dev->dma_mem->virt_base = mem_base; 46 dev->dma_mem->virt_base = mem_base;
46 dev->dma_mem->device_base = device_addr; 47 dev->dma_mem->device_base = device_addr;
48 dev->dma_mem->pfn_base = PFN_DOWN(bus_addr);
47 dev->dma_mem->size = pages; 49 dev->dma_mem->size = pages;
48 dev->dma_mem->flags = flags; 50 dev->dma_mem->flags = flags;
49 51
@@ -176,3 +178,43 @@ int dma_release_from_coherent(struct device *dev, int order, void *vaddr)
176 return 0; 178 return 0;
177} 179}
178EXPORT_SYMBOL(dma_release_from_coherent); 180EXPORT_SYMBOL(dma_release_from_coherent);
181
182/**
183 * dma_mmap_from_coherent() - try to mmap the memory allocated from
184 * per-device coherent memory pool to userspace
185 * @dev: device from which the memory was allocated
186 * @vma: vm_area for the userspace memory
187 * @vaddr: cpu address returned by dma_alloc_from_coherent
188 * @size: size of the memory buffer allocated by dma_alloc_from_coherent
189 *
190 * This checks whether the memory was allocated from the per-device
191 * coherent memory pool and if so, maps that memory to the provided vma.
192 *
193 * Returns 1 if we correctly mapped the memory, or 0 if
194 * dma_release_coherent() should proceed with mapping memory from
195 * generic pools.
196 */
197int dma_mmap_from_coherent(struct device *dev, struct vm_area_struct *vma,
198 void *vaddr, size_t size, int *ret)
199{
200 struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
201
202 if (mem && vaddr >= mem->virt_base && vaddr + size <=
203 (mem->virt_base + (mem->size << PAGE_SHIFT))) {
204 unsigned long off = vma->vm_pgoff;
205 int start = (vaddr - mem->virt_base) >> PAGE_SHIFT;
206 int user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
207 int count = size >> PAGE_SHIFT;
208
209 *ret = -ENXIO;
210 if (off < count && user_count <= count - off) {
211 unsigned pfn = mem->pfn_base + start + off;
212 *ret = remap_pfn_range(vma, vma->vm_start, pfn,
213 user_count << PAGE_SHIFT,
214 vma->vm_page_prot);
215 }
216 return 1;
217 }
218 return 0;
219}
220EXPORT_SYMBOL(dma_mmap_from_coherent);
diff --git a/include/asm-generic/dma-coherent.h b/include/asm-generic/dma-coherent.h
index 85a3ffaa0242..abfb2682de7f 100644
--- a/include/asm-generic/dma-coherent.h
+++ b/include/asm-generic/dma-coherent.h
@@ -3,13 +3,15 @@
3 3
4#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT 4#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
5/* 5/*
6 * These two functions are only for dma allocator. 6 * These three functions are only for dma allocator.
7 * Don't use them in device drivers. 7 * Don't use them in device drivers.
8 */ 8 */
9int dma_alloc_from_coherent(struct device *dev, ssize_t size, 9int dma_alloc_from_coherent(struct device *dev, ssize_t size,
10 dma_addr_t *dma_handle, void **ret); 10 dma_addr_t *dma_handle, void **ret);
11int dma_release_from_coherent(struct device *dev, int order, void *vaddr); 11int dma_release_from_coherent(struct device *dev, int order, void *vaddr);
12 12
13int dma_mmap_from_coherent(struct device *dev, struct vm_area_struct *vma,
14 void *cpu_addr, size_t size, int *ret);
13/* 15/*
14 * Standard interface 16 * Standard interface
15 */ 17 */
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-01 23:32:36 -0500