aboutsummaryrefslogtreecommitdiffstats
path: root/lib/swiotlb.c
diff options
context:
space:
mode:
authorIngo Molnar <mingo@elte.hu>2009-01-12 05:32:03 -0500
committerIngo Molnar <mingo@elte.hu>2009-01-12 05:32:03 -0500
commite3ee1e123183ca9847e74b7b8e2694c9e3b817a6 (patch)
tree652a84674ed05eaa46a813de2223af0bd0168a5a /lib/swiotlb.c
parent5762ba1873b0bb9faa631aaa02f533c2b9837f82 (diff)
parentc59765042f53a79a7a65585042ff463b69cb248c (diff)
Merge commit 'v2.6.29-rc1' into timers/hrtimers
Conflicts: kernel/time/tick-common.c
Diffstat (limited to 'lib/swiotlb.c')
-rw-r--r--lib/swiotlb.c259
1 files changed, 171 insertions, 88 deletions
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 5f6c629a924d..1f991acc2a05 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -14,6 +14,7 @@
14 * 04/07/.. ak Better overflow handling. Assorted fixes. 14 * 04/07/.. ak Better overflow handling. Assorted fixes.
15 * 05/09/10 linville Add support for syncing ranges, support syncing for 15 * 05/09/10 linville Add support for syncing ranges, support syncing for
16 * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup. 16 * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
17 * 08/12/11 beckyb Add highmem support
17 */ 18 */
18 19
19#include <linux/cache.h> 20#include <linux/cache.h>
@@ -22,8 +23,11 @@
22#include <linux/module.h> 23#include <linux/module.h>
23#include <linux/spinlock.h> 24#include <linux/spinlock.h>
24#include <linux/string.h> 25#include <linux/string.h>
26#include <linux/swiotlb.h>
27#include <linux/pfn.h>
25#include <linux/types.h> 28#include <linux/types.h>
26#include <linux/ctype.h> 29#include <linux/ctype.h>
30#include <linux/highmem.h>
27 31
28#include <asm/io.h> 32#include <asm/io.h>
29#include <asm/dma.h> 33#include <asm/dma.h>
@@ -36,22 +40,6 @@
36#define OFFSET(val,align) ((unsigned long) \ 40#define OFFSET(val,align) ((unsigned long) \
37 ( (val) & ( (align) - 1))) 41 ( (val) & ( (align) - 1)))
38 42
39#define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
40#define SG_ENT_PHYS_ADDRESS(sg) virt_to_bus(SG_ENT_VIRT_ADDRESS(sg))
41
42/*
43 * Maximum allowable number of contiguous slabs to map,
44 * must be a power of 2. What is the appropriate value ?
45 * The complexity of {map,unmap}_single is linearly dependent on this value.
46 */
47#define IO_TLB_SEGSIZE 128
48
49/*
50 * log of the size of each IO TLB slab. The number of slabs is command line
51 * controllable.
52 */
53#define IO_TLB_SHIFT 11
54
55#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT)) 43#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
56 44
57/* 45/*
@@ -102,7 +90,7 @@ static unsigned int io_tlb_index;
102 * We need to save away the original address corresponding to a mapped entry 90 * We need to save away the original address corresponding to a mapped entry
103 * for the sync operations. 91 * for the sync operations.
104 */ 92 */
105static unsigned char **io_tlb_orig_addr; 93static phys_addr_t *io_tlb_orig_addr;
106 94
107/* 95/*
108 * Protect the above data structures in the map and unmap calls 96 * Protect the above data structures in the map and unmap calls
@@ -126,6 +114,56 @@ setup_io_tlb_npages(char *str)
126__setup("swiotlb=", setup_io_tlb_npages); 114__setup("swiotlb=", setup_io_tlb_npages);
127/* make io_tlb_overflow tunable too? */ 115/* make io_tlb_overflow tunable too? */
128 116
117void * __weak __init swiotlb_alloc_boot(size_t size, unsigned long nslabs)
118{
119 return alloc_bootmem_low_pages(size);
120}
121
122void * __weak swiotlb_alloc(unsigned order, unsigned long nslabs)
123{
124 return (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN, order);
125}
126
127dma_addr_t __weak swiotlb_phys_to_bus(struct device *hwdev, phys_addr_t paddr)
128{
129 return paddr;
130}
131
132phys_addr_t __weak swiotlb_bus_to_phys(dma_addr_t baddr)
133{
134 return baddr;
135}
136
137static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev,
138 volatile void *address)
139{
140 return swiotlb_phys_to_bus(hwdev, virt_to_phys(address));
141}
142
143static void *swiotlb_bus_to_virt(dma_addr_t address)
144{
145 return phys_to_virt(swiotlb_bus_to_phys(address));
146}
147
148int __weak swiotlb_arch_range_needs_mapping(void *ptr, size_t size)
149{
150 return 0;
151}
152
153static void swiotlb_print_info(unsigned long bytes)
154{
155 phys_addr_t pstart, pend;
156
157 pstart = virt_to_phys(io_tlb_start);
158 pend = virt_to_phys(io_tlb_end);
159
160 printk(KERN_INFO "Placing %luMB software IO TLB between %p - %p\n",
161 bytes >> 20, io_tlb_start, io_tlb_end);
162 printk(KERN_INFO "software IO TLB at phys %#llx - %#llx\n",
163 (unsigned long long)pstart,
164 (unsigned long long)pend);
165}
166
129/* 167/*
130 * Statically reserve bounce buffer space and initialize bounce buffer data 168 * Statically reserve bounce buffer space and initialize bounce buffer data
131 * structures for the software IO TLB used to implement the DMA API. 169 * structures for the software IO TLB used to implement the DMA API.
@@ -145,7 +183,7 @@ swiotlb_init_with_default_size(size_t default_size)
145 /* 183 /*
146 * Get IO TLB memory from the low pages 184 * Get IO TLB memory from the low pages
147 */ 185 */
148 io_tlb_start = alloc_bootmem_low_pages(bytes); 186 io_tlb_start = swiotlb_alloc_boot(bytes, io_tlb_nslabs);
149 if (!io_tlb_start) 187 if (!io_tlb_start)
150 panic("Cannot allocate SWIOTLB buffer"); 188 panic("Cannot allocate SWIOTLB buffer");
151 io_tlb_end = io_tlb_start + bytes; 189 io_tlb_end = io_tlb_start + bytes;
@@ -159,7 +197,7 @@ swiotlb_init_with_default_size(size_t default_size)
159 for (i = 0; i < io_tlb_nslabs; i++) 197 for (i = 0; i < io_tlb_nslabs; i++)
160 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); 198 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
161 io_tlb_index = 0; 199 io_tlb_index = 0;
162 io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(char *)); 200 io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(phys_addr_t));
163 201
164 /* 202 /*
165 * Get the overflow emergency buffer 203 * Get the overflow emergency buffer
@@ -168,8 +206,7 @@ swiotlb_init_with_default_size(size_t default_size)
168 if (!io_tlb_overflow_buffer) 206 if (!io_tlb_overflow_buffer)
169 panic("Cannot allocate SWIOTLB overflow buffer!\n"); 207 panic("Cannot allocate SWIOTLB overflow buffer!\n");
170 208
171 printk(KERN_INFO "Placing software IO TLB between 0x%lx - 0x%lx\n", 209 swiotlb_print_info(bytes);
172 virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end));
173} 210}
174 211
175void __init 212void __init
@@ -202,8 +239,7 @@ swiotlb_late_init_with_default_size(size_t default_size)
202 bytes = io_tlb_nslabs << IO_TLB_SHIFT; 239 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
203 240
204 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) { 241 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
205 io_tlb_start = (char *)__get_free_pages(GFP_DMA | __GFP_NOWARN, 242 io_tlb_start = swiotlb_alloc(order, io_tlb_nslabs);
206 order);
207 if (io_tlb_start) 243 if (io_tlb_start)
208 break; 244 break;
209 order--; 245 order--;
@@ -235,12 +271,14 @@ swiotlb_late_init_with_default_size(size_t default_size)
235 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); 271 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
236 io_tlb_index = 0; 272 io_tlb_index = 0;
237 273
238 io_tlb_orig_addr = (unsigned char **)__get_free_pages(GFP_KERNEL, 274 io_tlb_orig_addr = (phys_addr_t *)
239 get_order(io_tlb_nslabs * sizeof(char *))); 275 __get_free_pages(GFP_KERNEL,
276 get_order(io_tlb_nslabs *
277 sizeof(phys_addr_t)));
240 if (!io_tlb_orig_addr) 278 if (!io_tlb_orig_addr)
241 goto cleanup3; 279 goto cleanup3;
242 280
243 memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(char *)); 281 memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(phys_addr_t));
244 282
245 /* 283 /*
246 * Get the overflow emergency buffer 284 * Get the overflow emergency buffer
@@ -250,15 +288,13 @@ swiotlb_late_init_with_default_size(size_t default_size)
250 if (!io_tlb_overflow_buffer) 288 if (!io_tlb_overflow_buffer)
251 goto cleanup4; 289 goto cleanup4;
252 290
253 printk(KERN_INFO "Placing %luMB software IO TLB between 0x%lx - " 291 swiotlb_print_info(bytes);
254 "0x%lx\n", bytes >> 20,
255 virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end));
256 292
257 return 0; 293 return 0;
258 294
259cleanup4: 295cleanup4:
260 free_pages((unsigned long)io_tlb_orig_addr, get_order(io_tlb_nslabs * 296 free_pages((unsigned long)io_tlb_orig_addr,
261 sizeof(char *))); 297 get_order(io_tlb_nslabs * sizeof(phys_addr_t)));
262 io_tlb_orig_addr = NULL; 298 io_tlb_orig_addr = NULL;
263cleanup3: 299cleanup3:
264 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs * 300 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
@@ -279,16 +315,62 @@ address_needs_mapping(struct device *hwdev, dma_addr_t addr, size_t size)
279 return !is_buffer_dma_capable(dma_get_mask(hwdev), addr, size); 315 return !is_buffer_dma_capable(dma_get_mask(hwdev), addr, size);
280} 316}
281 317
318static inline int range_needs_mapping(void *ptr, size_t size)
319{
320 return swiotlb_force || swiotlb_arch_range_needs_mapping(ptr, size);
321}
322
282static int is_swiotlb_buffer(char *addr) 323static int is_swiotlb_buffer(char *addr)
283{ 324{
284 return addr >= io_tlb_start && addr < io_tlb_end; 325 return addr >= io_tlb_start && addr < io_tlb_end;
285} 326}
286 327
287/* 328/*
329 * Bounce: copy the swiotlb buffer back to the original dma location
330 */
331static void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size,
332 enum dma_data_direction dir)
333{
334 unsigned long pfn = PFN_DOWN(phys);
335
336 if (PageHighMem(pfn_to_page(pfn))) {
337 /* The buffer does not have a mapping. Map it in and copy */
338 unsigned int offset = phys & ~PAGE_MASK;
339 char *buffer;
340 unsigned int sz = 0;
341 unsigned long flags;
342
343 while (size) {
344 sz = min(PAGE_SIZE - offset, size);
345
346 local_irq_save(flags);
347 buffer = kmap_atomic(pfn_to_page(pfn),
348 KM_BOUNCE_READ);
349 if (dir == DMA_TO_DEVICE)
350 memcpy(dma_addr, buffer + offset, sz);
351 else
352 memcpy(buffer + offset, dma_addr, sz);
353 kunmap_atomic(buffer, KM_BOUNCE_READ);
354 local_irq_restore(flags);
355
356 size -= sz;
357 pfn++;
358 dma_addr += sz;
359 offset = 0;
360 }
361 } else {
362 if (dir == DMA_TO_DEVICE)
363 memcpy(dma_addr, phys_to_virt(phys), size);
364 else
365 memcpy(phys_to_virt(phys), dma_addr, size);
366 }
367}
368
369/*
288 * Allocates bounce buffer and returns its kernel virtual address. 370 * Allocates bounce buffer and returns its kernel virtual address.
289 */ 371 */
290static void * 372static void *
291map_single(struct device *hwdev, char *buffer, size_t size, int dir) 373map_single(struct device *hwdev, phys_addr_t phys, size_t size, int dir)
292{ 374{
293 unsigned long flags; 375 unsigned long flags;
294 char *dma_addr; 376 char *dma_addr;
@@ -300,9 +382,13 @@ map_single(struct device *hwdev, char *buffer, size_t size, int dir)
300 unsigned long max_slots; 382 unsigned long max_slots;
301 383
302 mask = dma_get_seg_boundary(hwdev); 384 mask = dma_get_seg_boundary(hwdev);
303 start_dma_addr = virt_to_bus(io_tlb_start) & mask; 385 start_dma_addr = swiotlb_virt_to_bus(hwdev, io_tlb_start) & mask;
304 386
305 offset_slots = ALIGN(start_dma_addr, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; 387 offset_slots = ALIGN(start_dma_addr, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
388
389 /*
390 * Carefully handle integer overflow which can occur when mask == ~0UL.
391 */
306 max_slots = mask + 1 392 max_slots = mask + 1
307 ? ALIGN(mask + 1, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT 393 ? ALIGN(mask + 1, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT
308 : 1UL << (BITS_PER_LONG - IO_TLB_SHIFT); 394 : 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
@@ -379,9 +465,9 @@ found:
379 * needed. 465 * needed.
380 */ 466 */
381 for (i = 0; i < nslots; i++) 467 for (i = 0; i < nslots; i++)
382 io_tlb_orig_addr[index+i] = buffer + (i << IO_TLB_SHIFT); 468 io_tlb_orig_addr[index+i] = phys + (i << IO_TLB_SHIFT);
383 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) 469 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
384 memcpy(dma_addr, buffer, size); 470 swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);
385 471
386 return dma_addr; 472 return dma_addr;
387} 473}
@@ -395,17 +481,13 @@ unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
395 unsigned long flags; 481 unsigned long flags;
396 int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; 482 int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
397 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; 483 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
398 char *buffer = io_tlb_orig_addr[index]; 484 phys_addr_t phys = io_tlb_orig_addr[index];
399 485
400 /* 486 /*
401 * First, sync the memory before unmapping the entry 487 * First, sync the memory before unmapping the entry
402 */ 488 */
403 if (buffer && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))) 489 if (phys && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
404 /* 490 swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);
405 * bounce... copy the data back into the original buffer * and
406 * delete the bounce buffer.
407 */
408 memcpy(buffer, dma_addr, size);
409 491
410 /* 492 /*
411 * Return the buffer to the free list by setting the corresponding 493 * Return the buffer to the free list by setting the corresponding
@@ -438,20 +520,20 @@ sync_single(struct device *hwdev, char *dma_addr, size_t size,
438 int dir, int target) 520 int dir, int target)
439{ 521{
440 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; 522 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
441 char *buffer = io_tlb_orig_addr[index]; 523 phys_addr_t phys = io_tlb_orig_addr[index];
442 524
443 buffer += ((unsigned long)dma_addr & ((1 << IO_TLB_SHIFT) - 1)); 525 phys += ((unsigned long)dma_addr & ((1 << IO_TLB_SHIFT) - 1));
444 526
445 switch (target) { 527 switch (target) {
446 case SYNC_FOR_CPU: 528 case SYNC_FOR_CPU:
447 if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)) 529 if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
448 memcpy(buffer, dma_addr, size); 530 swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);
449 else 531 else
450 BUG_ON(dir != DMA_TO_DEVICE); 532 BUG_ON(dir != DMA_TO_DEVICE);
451 break; 533 break;
452 case SYNC_FOR_DEVICE: 534 case SYNC_FOR_DEVICE:
453 if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) 535 if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
454 memcpy(dma_addr, buffer, size); 536 swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);
455 else 537 else
456 BUG_ON(dir != DMA_FROM_DEVICE); 538 BUG_ON(dir != DMA_FROM_DEVICE);
457 break; 539 break;
@@ -473,7 +555,9 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
473 dma_mask = hwdev->coherent_dma_mask; 555 dma_mask = hwdev->coherent_dma_mask;
474 556
475 ret = (void *)__get_free_pages(flags, order); 557 ret = (void *)__get_free_pages(flags, order);
476 if (ret && !is_buffer_dma_capable(dma_mask, virt_to_bus(ret), size)) { 558 if (ret &&
559 !is_buffer_dma_capable(dma_mask, swiotlb_virt_to_bus(hwdev, ret),
560 size)) {
477 /* 561 /*
478 * The allocated memory isn't reachable by the device. 562 * The allocated memory isn't reachable by the device.
479 * Fall back on swiotlb_map_single(). 563 * Fall back on swiotlb_map_single().
@@ -488,13 +572,13 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
488 * swiotlb_map_single(), which will grab memory from 572 * swiotlb_map_single(), which will grab memory from
489 * the lowest available address range. 573 * the lowest available address range.
490 */ 574 */
491 ret = map_single(hwdev, NULL, size, DMA_FROM_DEVICE); 575 ret = map_single(hwdev, 0, size, DMA_FROM_DEVICE);
492 if (!ret) 576 if (!ret)
493 return NULL; 577 return NULL;
494 } 578 }
495 579
496 memset(ret, 0, size); 580 memset(ret, 0, size);
497 dev_addr = virt_to_bus(ret); 581 dev_addr = swiotlb_virt_to_bus(hwdev, ret);
498 582
499 /* Confirm address can be DMA'd by device */ 583 /* Confirm address can be DMA'd by device */
500 if (!is_buffer_dma_capable(dma_mask, dev_addr, size)) { 584 if (!is_buffer_dma_capable(dma_mask, dev_addr, size)) {
@@ -509,6 +593,7 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
509 *dma_handle = dev_addr; 593 *dma_handle = dev_addr;
510 return ret; 594 return ret;
511} 595}
596EXPORT_SYMBOL(swiotlb_alloc_coherent);
512 597
513void 598void
514swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, 599swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
@@ -521,6 +606,7 @@ swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
521 /* DMA_TO_DEVICE to avoid memcpy in unmap_single */ 606 /* DMA_TO_DEVICE to avoid memcpy in unmap_single */
522 unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE); 607 unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE);
523} 608}
609EXPORT_SYMBOL(swiotlb_free_coherent);
524 610
525static void 611static void
526swiotlb_full(struct device *dev, size_t size, int dir, int do_panic) 612swiotlb_full(struct device *dev, size_t size, int dir, int do_panic)
@@ -533,7 +619,7 @@ swiotlb_full(struct device *dev, size_t size, int dir, int do_panic)
533 * the damage, or panic when the transfer is too big. 619 * the damage, or panic when the transfer is too big.
534 */ 620 */
535 printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at " 621 printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at "
536 "device %s\n", size, dev ? dev->bus_id : "?"); 622 "device %s\n", size, dev ? dev_name(dev) : "?");
537 623
538 if (size > io_tlb_overflow && do_panic) { 624 if (size > io_tlb_overflow && do_panic) {
539 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) 625 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
@@ -554,7 +640,7 @@ dma_addr_t
554swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size, 640swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size,
555 int dir, struct dma_attrs *attrs) 641 int dir, struct dma_attrs *attrs)
556{ 642{
557 dma_addr_t dev_addr = virt_to_bus(ptr); 643 dma_addr_t dev_addr = swiotlb_virt_to_bus(hwdev, ptr);
558 void *map; 644 void *map;
559 645
560 BUG_ON(dir == DMA_NONE); 646 BUG_ON(dir == DMA_NONE);
@@ -563,19 +649,20 @@ swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size,
563 * we can safely return the device addr and not worry about bounce 649 * we can safely return the device addr and not worry about bounce
564 * buffering it. 650 * buffering it.
565 */ 651 */
566 if (!address_needs_mapping(hwdev, dev_addr, size) && !swiotlb_force) 652 if (!address_needs_mapping(hwdev, dev_addr, size) &&
653 !range_needs_mapping(ptr, size))
567 return dev_addr; 654 return dev_addr;
568 655
569 /* 656 /*
570 * Oh well, have to allocate and map a bounce buffer. 657 * Oh well, have to allocate and map a bounce buffer.
571 */ 658 */
572 map = map_single(hwdev, ptr, size, dir); 659 map = map_single(hwdev, virt_to_phys(ptr), size, dir);
573 if (!map) { 660 if (!map) {
574 swiotlb_full(hwdev, size, dir, 1); 661 swiotlb_full(hwdev, size, dir, 1);
575 map = io_tlb_overflow_buffer; 662 map = io_tlb_overflow_buffer;
576 } 663 }
577 664
578 dev_addr = virt_to_bus(map); 665 dev_addr = swiotlb_virt_to_bus(hwdev, map);
579 666
580 /* 667 /*
581 * Ensure that the address returned is DMA'ble 668 * Ensure that the address returned is DMA'ble
@@ -592,6 +679,7 @@ swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
592{ 679{
593 return swiotlb_map_single_attrs(hwdev, ptr, size, dir, NULL); 680 return swiotlb_map_single_attrs(hwdev, ptr, size, dir, NULL);
594} 681}
682EXPORT_SYMBOL(swiotlb_map_single);
595 683
596/* 684/*
597 * Unmap a single streaming mode DMA translation. The dma_addr and size must 685 * Unmap a single streaming mode DMA translation. The dma_addr and size must
@@ -605,7 +693,7 @@ void
605swiotlb_unmap_single_attrs(struct device *hwdev, dma_addr_t dev_addr, 693swiotlb_unmap_single_attrs(struct device *hwdev, dma_addr_t dev_addr,
606 size_t size, int dir, struct dma_attrs *attrs) 694 size_t size, int dir, struct dma_attrs *attrs)
607{ 695{
608 char *dma_addr = bus_to_virt(dev_addr); 696 char *dma_addr = swiotlb_bus_to_virt(dev_addr);
609 697
610 BUG_ON(dir == DMA_NONE); 698 BUG_ON(dir == DMA_NONE);
611 if (is_swiotlb_buffer(dma_addr)) 699 if (is_swiotlb_buffer(dma_addr))
@@ -621,6 +709,8 @@ swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
621{ 709{
622 return swiotlb_unmap_single_attrs(hwdev, dev_addr, size, dir, NULL); 710 return swiotlb_unmap_single_attrs(hwdev, dev_addr, size, dir, NULL);
623} 711}
712EXPORT_SYMBOL(swiotlb_unmap_single);
713
624/* 714/*
625 * Make physical memory consistent for a single streaming mode DMA translation 715 * Make physical memory consistent for a single streaming mode DMA translation
626 * after a transfer. 716 * after a transfer.
@@ -635,7 +725,7 @@ static void
635swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr, 725swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
636 size_t size, int dir, int target) 726 size_t size, int dir, int target)
637{ 727{
638 char *dma_addr = bus_to_virt(dev_addr); 728 char *dma_addr = swiotlb_bus_to_virt(dev_addr);
639 729
640 BUG_ON(dir == DMA_NONE); 730 BUG_ON(dir == DMA_NONE);
641 if (is_swiotlb_buffer(dma_addr)) 731 if (is_swiotlb_buffer(dma_addr))
@@ -650,6 +740,7 @@ swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
650{ 740{
651 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU); 741 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
652} 742}
743EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
653 744
654void 745void
655swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, 746swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
@@ -657,6 +748,7 @@ swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
657{ 748{
658 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE); 749 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
659} 750}
751EXPORT_SYMBOL(swiotlb_sync_single_for_device);
660 752
661/* 753/*
662 * Same as above, but for a sub-range of the mapping. 754 * Same as above, but for a sub-range of the mapping.
@@ -666,7 +758,7 @@ swiotlb_sync_single_range(struct device *hwdev, dma_addr_t dev_addr,
666 unsigned long offset, size_t size, 758 unsigned long offset, size_t size,
667 int dir, int target) 759 int dir, int target)
668{ 760{
669 char *dma_addr = bus_to_virt(dev_addr) + offset; 761 char *dma_addr = swiotlb_bus_to_virt(dev_addr) + offset;
670 762
671 BUG_ON(dir == DMA_NONE); 763 BUG_ON(dir == DMA_NONE);
672 if (is_swiotlb_buffer(dma_addr)) 764 if (is_swiotlb_buffer(dma_addr))
@@ -682,6 +774,7 @@ swiotlb_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
682 swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir, 774 swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir,
683 SYNC_FOR_CPU); 775 SYNC_FOR_CPU);
684} 776}
777EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_cpu);
685 778
686void 779void
687swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr, 780swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr,
@@ -690,9 +783,8 @@ swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr,
690 swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir, 783 swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir,
691 SYNC_FOR_DEVICE); 784 SYNC_FOR_DEVICE);
692} 785}
786EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device);
693 787
694void swiotlb_unmap_sg_attrs(struct device *, struct scatterlist *, int, int,
695 struct dma_attrs *);
696/* 788/*
697 * Map a set of buffers described by scatterlist in streaming mode for DMA. 789 * Map a set of buffers described by scatterlist in streaming mode for DMA.
698 * This is the scatter-gather version of the above swiotlb_map_single 790 * This is the scatter-gather version of the above swiotlb_map_single
@@ -714,18 +806,18 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
714 int dir, struct dma_attrs *attrs) 806 int dir, struct dma_attrs *attrs)
715{ 807{
716 struct scatterlist *sg; 808 struct scatterlist *sg;
717 void *addr;
718 dma_addr_t dev_addr;
719 int i; 809 int i;
720 810
721 BUG_ON(dir == DMA_NONE); 811 BUG_ON(dir == DMA_NONE);
722 812
723 for_each_sg(sgl, sg, nelems, i) { 813 for_each_sg(sgl, sg, nelems, i) {
724 addr = SG_ENT_VIRT_ADDRESS(sg); 814 void *addr = sg_virt(sg);
725 dev_addr = virt_to_bus(addr); 815 dma_addr_t dev_addr = swiotlb_virt_to_bus(hwdev, addr);
726 if (swiotlb_force || 816
817 if (range_needs_mapping(addr, sg->length) ||
727 address_needs_mapping(hwdev, dev_addr, sg->length)) { 818 address_needs_mapping(hwdev, dev_addr, sg->length)) {
728 void *map = map_single(hwdev, addr, sg->length, dir); 819 void *map = map_single(hwdev, sg_phys(sg),
820 sg->length, dir);
729 if (!map) { 821 if (!map) {
730 /* Don't panic here, we expect map_sg users 822 /* Don't panic here, we expect map_sg users
731 to do proper error handling. */ 823 to do proper error handling. */
@@ -735,7 +827,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
735 sgl[0].dma_length = 0; 827 sgl[0].dma_length = 0;
736 return 0; 828 return 0;
737 } 829 }
738 sg->dma_address = virt_to_bus(map); 830 sg->dma_address = swiotlb_virt_to_bus(hwdev, map);
739 } else 831 } else
740 sg->dma_address = dev_addr; 832 sg->dma_address = dev_addr;
741 sg->dma_length = sg->length; 833 sg->dma_length = sg->length;
@@ -750,6 +842,7 @@ swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
750{ 842{
751 return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL); 843 return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL);
752} 844}
845EXPORT_SYMBOL(swiotlb_map_sg);
753 846
754/* 847/*
755 * Unmap a set of streaming mode DMA translations. Again, cpu read rules 848 * Unmap a set of streaming mode DMA translations. Again, cpu read rules
@@ -765,11 +858,11 @@ swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
765 BUG_ON(dir == DMA_NONE); 858 BUG_ON(dir == DMA_NONE);
766 859
767 for_each_sg(sgl, sg, nelems, i) { 860 for_each_sg(sgl, sg, nelems, i) {
768 if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg)) 861 if (sg->dma_address != swiotlb_virt_to_bus(hwdev, sg_virt(sg)))
769 unmap_single(hwdev, bus_to_virt(sg->dma_address), 862 unmap_single(hwdev, swiotlb_bus_to_virt(sg->dma_address),
770 sg->dma_length, dir); 863 sg->dma_length, dir);
771 else if (dir == DMA_FROM_DEVICE) 864 else if (dir == DMA_FROM_DEVICE)
772 dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length); 865 dma_mark_clean(sg_virt(sg), sg->dma_length);
773 } 866 }
774} 867}
775EXPORT_SYMBOL(swiotlb_unmap_sg_attrs); 868EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);
@@ -780,6 +873,7 @@ swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
780{ 873{
781 return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL); 874 return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL);
782} 875}
876EXPORT_SYMBOL(swiotlb_unmap_sg);
783 877
784/* 878/*
785 * Make physical memory consistent for a set of streaming mode DMA translations 879 * Make physical memory consistent for a set of streaming mode DMA translations
@@ -798,11 +892,11 @@ swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl,
798 BUG_ON(dir == DMA_NONE); 892 BUG_ON(dir == DMA_NONE);
799 893
800 for_each_sg(sgl, sg, nelems, i) { 894 for_each_sg(sgl, sg, nelems, i) {
801 if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg)) 895 if (sg->dma_address != swiotlb_virt_to_bus(hwdev, sg_virt(sg)))
802 sync_single(hwdev, bus_to_virt(sg->dma_address), 896 sync_single(hwdev, swiotlb_bus_to_virt(sg->dma_address),
803 sg->dma_length, dir, target); 897 sg->dma_length, dir, target);
804 else if (dir == DMA_FROM_DEVICE) 898 else if (dir == DMA_FROM_DEVICE)
805 dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length); 899 dma_mark_clean(sg_virt(sg), sg->dma_length);
806 } 900 }
807} 901}
808 902
@@ -812,6 +906,7 @@ swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
812{ 906{
813 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU); 907 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
814} 908}
909EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
815 910
816void 911void
817swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, 912swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
@@ -819,12 +914,14 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
819{ 914{
820 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE); 915 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
821} 916}
917EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
822 918
823int 919int
824swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) 920swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
825{ 921{
826 return (dma_addr == virt_to_bus(io_tlb_overflow_buffer)); 922 return (dma_addr == swiotlb_virt_to_bus(hwdev, io_tlb_overflow_buffer));
827} 923}
924EXPORT_SYMBOL(swiotlb_dma_mapping_error);
828 925
829/* 926/*
830 * Return whether the given device DMA address mask can be supported 927 * Return whether the given device DMA address mask can be supported
@@ -835,20 +932,6 @@ swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
835int 932int
836swiotlb_dma_supported(struct device *hwdev, u64 mask) 933swiotlb_dma_supported(struct device *hwdev, u64 mask)
837{ 934{
838 return virt_to_bus(io_tlb_end - 1) <= mask; 935 return swiotlb_virt_to_bus(hwdev, io_tlb_end - 1) <= mask;
839} 936}
840
841EXPORT_SYMBOL(swiotlb_map_single);
842EXPORT_SYMBOL(swiotlb_unmap_single);
843EXPORT_SYMBOL(swiotlb_map_sg);
844EXPORT_SYMBOL(swiotlb_unmap_sg);
845EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
846EXPORT_SYMBOL(swiotlb_sync_single_for_device);
847EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_cpu);
848EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device);
849EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
850EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
851EXPORT_SYMBOL(swiotlb_dma_mapping_error);
852EXPORT_SYMBOL(swiotlb_alloc_coherent);
853EXPORT_SYMBOL(swiotlb_free_coherent);
854EXPORT_SYMBOL(swiotlb_dma_supported); 937EXPORT_SYMBOL(swiotlb_dma_supported);