aboutsummaryrefslogtreecommitdiffstats
path: root/lib/swiotlb.c
diff options
context:
space:
mode:
authorJeremy Fitzhardinge <jeremy@goop.org>2008-12-16 15:17:33 -0500
committerIngo Molnar <mingo@elte.hu>2008-12-17 12:58:15 -0500
commitef9b189352f2eb78f14e52996f4780a523b04a49 (patch)
tree801c230f291d36db2f86404a91c1443bf6636f56 /lib/swiotlb.c
parent1b548f667c1487d92e794a9f7a67788f49b952d8 (diff)
swiotlb: support bouncing of HighMem pages
Impact: prepare the swiotlb code for HighMem struct pages This requires us to treat DMA regions in terms of page+offset rather than virtual addressing since a HighMem page may not have a mapping. Signed-off-by: Ian Campbell <ian.campbell@citrix.com> Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'lib/swiotlb.c')
-rw-r--r--lib/swiotlb.c122
1 files changed, 89 insertions, 33 deletions
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index a0b4039e2880..1661af593914 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -26,6 +26,7 @@
26#include <linux/swiotlb.h> 26#include <linux/swiotlb.h>
27#include <linux/types.h> 27#include <linux/types.h>
28#include <linux/ctype.h> 28#include <linux/ctype.h>
29#include <linux/highmem.h>
29 30
30#include <asm/io.h> 31#include <asm/io.h>
31#include <asm/dma.h> 32#include <asm/dma.h>
@@ -38,9 +39,6 @@
38#define OFFSET(val,align) ((unsigned long) \ 39#define OFFSET(val,align) ((unsigned long) \
39 ( (val) & ( (align) - 1))) 40 ( (val) & ( (align) - 1)))
40 41
41#define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
42#define SG_ENT_PHYS_ADDRESS(sg) virt_to_bus(SG_ENT_VIRT_ADDRESS(sg))
43
44#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT)) 42#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
45 43
46/* 44/*
@@ -91,7 +89,10 @@ static unsigned int io_tlb_index;
91 * We need to save away the original address corresponding to a mapped entry 89 * We need to save away the original address corresponding to a mapped entry
92 * for the sync operations. 90 * for the sync operations.
93 */ 91 */
94static unsigned char **io_tlb_orig_addr; 92static struct swiotlb_phys_addr {
93 struct page *page;
94 unsigned int offset;
95} *io_tlb_orig_addr;
95 96
96/* 97/*
97 * Protect the above data structures in the map and unmap calls 98 * Protect the above data structures in the map and unmap calls
@@ -150,6 +151,11 @@ int __weak swiotlb_arch_range_needs_mapping(void *ptr, size_t size)
150 return 0; 151 return 0;
151} 152}
152 153
154static dma_addr_t swiotlb_sg_to_bus(struct scatterlist *sg)
155{
156 return swiotlb_phys_to_bus(page_to_phys(sg_page(sg)) + sg->offset);
157}
158
153/* 159/*
154 * Statically reserve bounce buffer space and initialize bounce buffer data 160 * Statically reserve bounce buffer space and initialize bounce buffer data
155 * structures for the software IO TLB used to implement the DMA API. 161 * structures for the software IO TLB used to implement the DMA API.
@@ -183,7 +189,7 @@ swiotlb_init_with_default_size(size_t default_size)
183 for (i = 0; i < io_tlb_nslabs; i++) 189 for (i = 0; i < io_tlb_nslabs; i++)
184 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); 190 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
185 io_tlb_index = 0; 191 io_tlb_index = 0;
186 io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(char *)); 192 io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(struct swiotlb_phys_addr));
187 193
188 /* 194 /*
189 * Get the overflow emergency buffer 195 * Get the overflow emergency buffer
@@ -258,12 +264,12 @@ swiotlb_late_init_with_default_size(size_t default_size)
258 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); 264 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
259 io_tlb_index = 0; 265 io_tlb_index = 0;
260 266
261 io_tlb_orig_addr = (unsigned char **)__get_free_pages(GFP_KERNEL, 267 io_tlb_orig_addr = (struct swiotlb_phys_addr *)__get_free_pages(GFP_KERNEL,
262 get_order(io_tlb_nslabs * sizeof(char *))); 268 get_order(io_tlb_nslabs * sizeof(struct swiotlb_phys_addr)));
263 if (!io_tlb_orig_addr) 269 if (!io_tlb_orig_addr)
264 goto cleanup3; 270 goto cleanup3;
265 271
266 memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(char *)); 272 memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(struct swiotlb_phys_addr));
267 273
268 /* 274 /*
269 * Get the overflow emergency buffer 275 * Get the overflow emergency buffer
@@ -312,20 +318,59 @@ static int is_swiotlb_buffer(char *addr)
312 return addr >= io_tlb_start && addr < io_tlb_end; 318 return addr >= io_tlb_start && addr < io_tlb_end;
313} 319}
314 320
315static void 321static struct swiotlb_phys_addr swiotlb_bus_to_phys_addr(char *dma_addr)
316__sync_single(char *buffer, char *dma_addr, size_t size, int dir)
317{ 322{
318 if (dir == DMA_TO_DEVICE) 323 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
319 memcpy(dma_addr, buffer, size); 324 struct swiotlb_phys_addr buffer = io_tlb_orig_addr[index];
320 else 325 buffer.offset += (long)dma_addr & ((1 << IO_TLB_SHIFT) - 1);
321 memcpy(buffer, dma_addr, size); 326 buffer.page += buffer.offset >> PAGE_SHIFT;
327 buffer.offset &= PAGE_SIZE - 1;
328 return buffer;
329}
330
331static void
332__sync_single(struct swiotlb_phys_addr buffer, char *dma_addr, size_t size, int dir)
333{
334 if (PageHighMem(buffer.page)) {
335 size_t len, bytes;
336 char *dev, *host, *kmp;
337
338 len = size;
339 while (len != 0) {
340 unsigned long flags;
341
342 bytes = len;
343 if ((bytes + buffer.offset) > PAGE_SIZE)
344 bytes = PAGE_SIZE - buffer.offset;
345 local_irq_save(flags); /* protects KM_BOUNCE_READ */
346 kmp = kmap_atomic(buffer.page, KM_BOUNCE_READ);
347 dev = dma_addr + size - len;
348 host = kmp + buffer.offset;
349 if (dir == DMA_FROM_DEVICE)
350 memcpy(host, dev, bytes);
351 else
352 memcpy(dev, host, bytes);
353 kunmap_atomic(kmp, KM_BOUNCE_READ);
354 local_irq_restore(flags);
355 len -= bytes;
356 buffer.page++;
357 buffer.offset = 0;
358 }
359 } else {
360 void *v = page_address(buffer.page) + buffer.offset;
361
362 if (dir == DMA_TO_DEVICE)
363 memcpy(dma_addr, v, size);
364 else
365 memcpy(v, dma_addr, size);
366 }
322} 367}
323 368
324/* 369/*
325 * Allocates bounce buffer and returns its kernel virtual address. 370 * Allocates bounce buffer and returns its kernel virtual address.
326 */ 371 */
327static void * 372static void *
328map_single(struct device *hwdev, char *buffer, size_t size, int dir) 373map_single(struct device *hwdev, struct swiotlb_phys_addr buffer, size_t size, int dir)
329{ 374{
330 unsigned long flags; 375 unsigned long flags;
331 char *dma_addr; 376 char *dma_addr;
@@ -335,6 +380,7 @@ map_single(struct device *hwdev, char *buffer, size_t size, int dir)
335 unsigned long mask; 380 unsigned long mask;
336 unsigned long offset_slots; 381 unsigned long offset_slots;
337 unsigned long max_slots; 382 unsigned long max_slots;
383 struct swiotlb_phys_addr slot_buf;
338 384
339 mask = dma_get_seg_boundary(hwdev); 385 mask = dma_get_seg_boundary(hwdev);
340 start_dma_addr = swiotlb_virt_to_bus(io_tlb_start) & mask; 386 start_dma_addr = swiotlb_virt_to_bus(io_tlb_start) & mask;
@@ -419,8 +465,13 @@ found:
419 * This is needed when we sync the memory. Then we sync the buffer if 465 * This is needed when we sync the memory. Then we sync the buffer if
420 * needed. 466 * needed.
421 */ 467 */
422 for (i = 0; i < nslots; i++) 468 slot_buf = buffer;
423 io_tlb_orig_addr[index+i] = buffer + (i << IO_TLB_SHIFT); 469 for (i = 0; i < nslots; i++) {
470 slot_buf.page += slot_buf.offset >> PAGE_SHIFT;
471 slot_buf.offset &= PAGE_SIZE - 1;
472 io_tlb_orig_addr[index+i] = slot_buf;
473 slot_buf.offset += 1 << IO_TLB_SHIFT;
474 }
424 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) 475 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
425 __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE); 476 __sync_single(buffer, dma_addr, size, DMA_TO_DEVICE);
426 477
@@ -436,12 +487,12 @@ unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
436 unsigned long flags; 487 unsigned long flags;
437 int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; 488 int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
438 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; 489 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
439 char *buffer = io_tlb_orig_addr[index]; 490 struct swiotlb_phys_addr buffer = swiotlb_bus_to_phys_addr(dma_addr);
440 491
441 /* 492 /*
442 * First, sync the memory before unmapping the entry 493 * First, sync the memory before unmapping the entry
443 */ 494 */
444 if (buffer && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))) 495 if ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))
445 /* 496 /*
446 * bounce... copy the data back into the original buffer * and 497 * bounce... copy the data back into the original buffer * and
447 * delete the bounce buffer. 498 * delete the bounce buffer.
@@ -478,10 +529,7 @@ static void
478sync_single(struct device *hwdev, char *dma_addr, size_t size, 529sync_single(struct device *hwdev, char *dma_addr, size_t size,
479 int dir, int target) 530 int dir, int target)
480{ 531{
481 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; 532 struct swiotlb_phys_addr buffer = swiotlb_bus_to_phys_addr(dma_addr);
482 char *buffer = io_tlb_orig_addr[index];
483
484 buffer += ((unsigned long)dma_addr & ((1 << IO_TLB_SHIFT) - 1));
485 533
486 switch (target) { 534 switch (target) {
487 case SYNC_FOR_CPU: 535 case SYNC_FOR_CPU:
@@ -529,7 +577,10 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
529 * swiotlb_map_single(), which will grab memory from 577 * swiotlb_map_single(), which will grab memory from
530 * the lowest available address range. 578 * the lowest available address range.
531 */ 579 */
532 ret = map_single(hwdev, NULL, size, DMA_FROM_DEVICE); 580 struct swiotlb_phys_addr buffer;
581 buffer.page = virt_to_page(NULL);
582 buffer.offset = 0;
583 ret = map_single(hwdev, buffer, size, DMA_FROM_DEVICE);
533 if (!ret) 584 if (!ret)
534 return NULL; 585 return NULL;
535 } 586 }
@@ -597,6 +648,7 @@ swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size,
597{ 648{
598 dma_addr_t dev_addr = swiotlb_virt_to_bus(ptr); 649 dma_addr_t dev_addr = swiotlb_virt_to_bus(ptr);
599 void *map; 650 void *map;
651 struct swiotlb_phys_addr buffer;
600 652
601 BUG_ON(dir == DMA_NONE); 653 BUG_ON(dir == DMA_NONE);
602 /* 654 /*
@@ -611,7 +663,9 @@ swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size,
611 /* 663 /*
612 * Oh well, have to allocate and map a bounce buffer. 664 * Oh well, have to allocate and map a bounce buffer.
613 */ 665 */
614 map = map_single(hwdev, ptr, size, dir); 666 buffer.page = virt_to_page(ptr);
667 buffer.offset = (unsigned long)ptr & ~PAGE_MASK;
668 map = map_single(hwdev, buffer, size, dir);
615 if (!map) { 669 if (!map) {
616 swiotlb_full(hwdev, size, dir, 1); 670 swiotlb_full(hwdev, size, dir, 1);
617 map = io_tlb_overflow_buffer; 671 map = io_tlb_overflow_buffer;
@@ -756,18 +810,20 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
756 int dir, struct dma_attrs *attrs) 810 int dir, struct dma_attrs *attrs)
757{ 811{
758 struct scatterlist *sg; 812 struct scatterlist *sg;
759 void *addr; 813 struct swiotlb_phys_addr buffer;
760 dma_addr_t dev_addr; 814 dma_addr_t dev_addr;
761 int i; 815 int i;
762 816
763 BUG_ON(dir == DMA_NONE); 817 BUG_ON(dir == DMA_NONE);
764 818
765 for_each_sg(sgl, sg, nelems, i) { 819 for_each_sg(sgl, sg, nelems, i) {
766 addr = SG_ENT_VIRT_ADDRESS(sg); 820 dev_addr = swiotlb_sg_to_bus(sg);
767 dev_addr = swiotlb_virt_to_bus(addr);
768 if (range_needs_mapping(sg_virt(sg), sg->length) || 821 if (range_needs_mapping(sg_virt(sg), sg->length) ||
769 address_needs_mapping(hwdev, dev_addr, sg->length)) { 822 address_needs_mapping(hwdev, dev_addr, sg->length)) {
770 void *map = map_single(hwdev, addr, sg->length, dir); 823 void *map;
824 buffer.page = sg_page(sg);
825 buffer.offset = sg->offset;
826 map = map_single(hwdev, buffer, sg->length, dir);
771 if (!map) { 827 if (!map) {
772 /* Don't panic here, we expect map_sg users 828 /* Don't panic here, we expect map_sg users
773 to do proper error handling. */ 829 to do proper error handling. */
@@ -807,11 +863,11 @@ swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
807 BUG_ON(dir == DMA_NONE); 863 BUG_ON(dir == DMA_NONE);
808 864
809 for_each_sg(sgl, sg, nelems, i) { 865 for_each_sg(sgl, sg, nelems, i) {
810 if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg)) 866 if (sg->dma_address != swiotlb_sg_to_bus(sg))
811 unmap_single(hwdev, swiotlb_bus_to_virt(sg->dma_address), 867 unmap_single(hwdev, swiotlb_bus_to_virt(sg->dma_address),
812 sg->dma_length, dir); 868 sg->dma_length, dir);
813 else if (dir == DMA_FROM_DEVICE) 869 else if (dir == DMA_FROM_DEVICE)
814 dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length); 870 dma_mark_clean(swiotlb_bus_to_virt(sg->dma_address), sg->dma_length);
815 } 871 }
816} 872}
817EXPORT_SYMBOL(swiotlb_unmap_sg_attrs); 873EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);
@@ -840,11 +896,11 @@ swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl,
840 BUG_ON(dir == DMA_NONE); 896 BUG_ON(dir == DMA_NONE);
841 897
842 for_each_sg(sgl, sg, nelems, i) { 898 for_each_sg(sgl, sg, nelems, i) {
843 if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg)) 899 if (sg->dma_address != swiotlb_sg_to_bus(sg))
844 sync_single(hwdev, swiotlb_bus_to_virt(sg->dma_address), 900 sync_single(hwdev, swiotlb_bus_to_virt(sg->dma_address),
845 sg->dma_length, dir, target); 901 sg->dma_length, dir, target);
846 else if (dir == DMA_FROM_DEVICE) 902 else if (dir == DMA_FROM_DEVICE)
847 dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length); 903 dma_mark_clean(swiotlb_bus_to_virt(sg->dma_address), sg->dma_length);
848 } 904 }
849} 905}
850 906