diff options
author | Ingo Molnar <mingo@elte.hu> | 2009-01-12 05:32:03 -0500 |
---|---|---|
committer | Ingo Molnar <mingo@elte.hu> | 2009-01-12 05:32:03 -0500 |
commit | e3ee1e123183ca9847e74b7b8e2694c9e3b817a6 (patch) | |
tree | 652a84674ed05eaa46a813de2223af0bd0168a5a /lib/swiotlb.c | |
parent | 5762ba1873b0bb9faa631aaa02f533c2b9837f82 (diff) | |
parent | c59765042f53a79a7a65585042ff463b69cb248c (diff) |
Merge commit 'v2.6.29-rc1' into timers/hrtimers
Conflicts:
kernel/time/tick-common.c
Diffstat (limited to 'lib/swiotlb.c')
-rw-r--r-- | lib/swiotlb.c | 259 |
1 files changed, 171 insertions, 88 deletions
diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 5f6c629a924d..1f991acc2a05 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c | |||
@@ -14,6 +14,7 @@ | |||
14 | * 04/07/.. ak Better overflow handling. Assorted fixes. | 14 | * 04/07/.. ak Better overflow handling. Assorted fixes. |
15 | * 05/09/10 linville Add support for syncing ranges, support syncing for | 15 | * 05/09/10 linville Add support for syncing ranges, support syncing for |
16 | * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup. | 16 | * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup. |
17 | * 08/12/11 beckyb Add highmem support | ||
17 | */ | 18 | */ |
18 | 19 | ||
19 | #include <linux/cache.h> | 20 | #include <linux/cache.h> |
@@ -22,8 +23,11 @@ | |||
22 | #include <linux/module.h> | 23 | #include <linux/module.h> |
23 | #include <linux/spinlock.h> | 24 | #include <linux/spinlock.h> |
24 | #include <linux/string.h> | 25 | #include <linux/string.h> |
26 | #include <linux/swiotlb.h> | ||
27 | #include <linux/pfn.h> | ||
25 | #include <linux/types.h> | 28 | #include <linux/types.h> |
26 | #include <linux/ctype.h> | 29 | #include <linux/ctype.h> |
30 | #include <linux/highmem.h> | ||
27 | 31 | ||
28 | #include <asm/io.h> | 32 | #include <asm/io.h> |
29 | #include <asm/dma.h> | 33 | #include <asm/dma.h> |
@@ -36,22 +40,6 @@ | |||
36 | #define OFFSET(val,align) ((unsigned long) \ | 40 | #define OFFSET(val,align) ((unsigned long) \ |
37 | ( (val) & ( (align) - 1))) | 41 | ( (val) & ( (align) - 1))) |
38 | 42 | ||
39 | #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg))) | ||
40 | #define SG_ENT_PHYS_ADDRESS(sg) virt_to_bus(SG_ENT_VIRT_ADDRESS(sg)) | ||
41 | |||
42 | /* | ||
43 | * Maximum allowable number of contiguous slabs to map, | ||
44 | * must be a power of 2. What is the appropriate value ? | ||
45 | * The complexity of {map,unmap}_single is linearly dependent on this value. | ||
46 | */ | ||
47 | #define IO_TLB_SEGSIZE 128 | ||
48 | |||
49 | /* | ||
50 | * log of the size of each IO TLB slab. The number of slabs is command line | ||
51 | * controllable. | ||
52 | */ | ||
53 | #define IO_TLB_SHIFT 11 | ||
54 | |||
55 | #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT)) | 43 | #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT)) |
56 | 44 | ||
57 | /* | 45 | /* |
@@ -102,7 +90,7 @@ static unsigned int io_tlb_index; | |||
102 | * We need to save away the original address corresponding to a mapped entry | 90 | * We need to save away the original address corresponding to a mapped entry |
103 | * for the sync operations. | 91 | * for the sync operations. |
104 | */ | 92 | */ |
105 | static unsigned char **io_tlb_orig_addr; | 93 | static phys_addr_t *io_tlb_orig_addr; |
106 | 94 | ||
107 | /* | 95 | /* |
108 | * Protect the above data structures in the map and unmap calls | 96 | * Protect the above data structures in the map and unmap calls |
@@ -126,6 +114,56 @@ setup_io_tlb_npages(char *str) | |||
126 | __setup("swiotlb=", setup_io_tlb_npages); | 114 | __setup("swiotlb=", setup_io_tlb_npages); |
127 | /* make io_tlb_overflow tunable too? */ | 115 | /* make io_tlb_overflow tunable too? */ |
128 | 116 | ||
117 | void * __weak __init swiotlb_alloc_boot(size_t size, unsigned long nslabs) | ||
118 | { | ||
119 | return alloc_bootmem_low_pages(size); | ||
120 | } | ||
121 | |||
122 | void * __weak swiotlb_alloc(unsigned order, unsigned long nslabs) | ||
123 | { | ||
124 | return (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN, order); | ||
125 | } | ||
126 | |||
127 | dma_addr_t __weak swiotlb_phys_to_bus(struct device *hwdev, phys_addr_t paddr) | ||
128 | { | ||
129 | return paddr; | ||
130 | } | ||
131 | |||
132 | phys_addr_t __weak swiotlb_bus_to_phys(dma_addr_t baddr) | ||
133 | { | ||
134 | return baddr; | ||
135 | } | ||
136 | |||
137 | static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev, | ||
138 | volatile void *address) | ||
139 | { | ||
140 | return swiotlb_phys_to_bus(hwdev, virt_to_phys(address)); | ||
141 | } | ||
142 | |||
143 | static void *swiotlb_bus_to_virt(dma_addr_t address) | ||
144 | { | ||
145 | return phys_to_virt(swiotlb_bus_to_phys(address)); | ||
146 | } | ||
147 | |||
148 | int __weak swiotlb_arch_range_needs_mapping(void *ptr, size_t size) | ||
149 | { | ||
150 | return 0; | ||
151 | } | ||
152 | |||
153 | static void swiotlb_print_info(unsigned long bytes) | ||
154 | { | ||
155 | phys_addr_t pstart, pend; | ||
156 | |||
157 | pstart = virt_to_phys(io_tlb_start); | ||
158 | pend = virt_to_phys(io_tlb_end); | ||
159 | |||
160 | printk(KERN_INFO "Placing %luMB software IO TLB between %p - %p\n", | ||
161 | bytes >> 20, io_tlb_start, io_tlb_end); | ||
162 | printk(KERN_INFO "software IO TLB at phys %#llx - %#llx\n", | ||
163 | (unsigned long long)pstart, | ||
164 | (unsigned long long)pend); | ||
165 | } | ||
166 | |||
129 | /* | 167 | /* |
130 | * Statically reserve bounce buffer space and initialize bounce buffer data | 168 | * Statically reserve bounce buffer space and initialize bounce buffer data |
131 | * structures for the software IO TLB used to implement the DMA API. | 169 | * structures for the software IO TLB used to implement the DMA API. |
@@ -145,7 +183,7 @@ swiotlb_init_with_default_size(size_t default_size) | |||
145 | /* | 183 | /* |
146 | * Get IO TLB memory from the low pages | 184 | * Get IO TLB memory from the low pages |
147 | */ | 185 | */ |
148 | io_tlb_start = alloc_bootmem_low_pages(bytes); | 186 | io_tlb_start = swiotlb_alloc_boot(bytes, io_tlb_nslabs); |
149 | if (!io_tlb_start) | 187 | if (!io_tlb_start) |
150 | panic("Cannot allocate SWIOTLB buffer"); | 188 | panic("Cannot allocate SWIOTLB buffer"); |
151 | io_tlb_end = io_tlb_start + bytes; | 189 | io_tlb_end = io_tlb_start + bytes; |
@@ -159,7 +197,7 @@ swiotlb_init_with_default_size(size_t default_size) | |||
159 | for (i = 0; i < io_tlb_nslabs; i++) | 197 | for (i = 0; i < io_tlb_nslabs; i++) |
160 | io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); | 198 | io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); |
161 | io_tlb_index = 0; | 199 | io_tlb_index = 0; |
162 | io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(char *)); | 200 | io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(phys_addr_t)); |
163 | 201 | ||
164 | /* | 202 | /* |
165 | * Get the overflow emergency buffer | 203 | * Get the overflow emergency buffer |
@@ -168,8 +206,7 @@ swiotlb_init_with_default_size(size_t default_size) | |||
168 | if (!io_tlb_overflow_buffer) | 206 | if (!io_tlb_overflow_buffer) |
169 | panic("Cannot allocate SWIOTLB overflow buffer!\n"); | 207 | panic("Cannot allocate SWIOTLB overflow buffer!\n"); |
170 | 208 | ||
171 | printk(KERN_INFO "Placing software IO TLB between 0x%lx - 0x%lx\n", | 209 | swiotlb_print_info(bytes); |
172 | virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end)); | ||
173 | } | 210 | } |
174 | 211 | ||
175 | void __init | 212 | void __init |
@@ -202,8 +239,7 @@ swiotlb_late_init_with_default_size(size_t default_size) | |||
202 | bytes = io_tlb_nslabs << IO_TLB_SHIFT; | 239 | bytes = io_tlb_nslabs << IO_TLB_SHIFT; |
203 | 240 | ||
204 | while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) { | 241 | while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) { |
205 | io_tlb_start = (char *)__get_free_pages(GFP_DMA | __GFP_NOWARN, | 242 | io_tlb_start = swiotlb_alloc(order, io_tlb_nslabs); |
206 | order); | ||
207 | if (io_tlb_start) | 243 | if (io_tlb_start) |
208 | break; | 244 | break; |
209 | order--; | 245 | order--; |
@@ -235,12 +271,14 @@ swiotlb_late_init_with_default_size(size_t default_size) | |||
235 | io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); | 271 | io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); |
236 | io_tlb_index = 0; | 272 | io_tlb_index = 0; |
237 | 273 | ||
238 | io_tlb_orig_addr = (unsigned char **)__get_free_pages(GFP_KERNEL, | 274 | io_tlb_orig_addr = (phys_addr_t *) |
239 | get_order(io_tlb_nslabs * sizeof(char *))); | 275 | __get_free_pages(GFP_KERNEL, |
276 | get_order(io_tlb_nslabs * | ||
277 | sizeof(phys_addr_t))); | ||
240 | if (!io_tlb_orig_addr) | 278 | if (!io_tlb_orig_addr) |
241 | goto cleanup3; | 279 | goto cleanup3; |
242 | 280 | ||
243 | memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(char *)); | 281 | memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(phys_addr_t)); |
244 | 282 | ||
245 | /* | 283 | /* |
246 | * Get the overflow emergency buffer | 284 | * Get the overflow emergency buffer |
@@ -250,15 +288,13 @@ swiotlb_late_init_with_default_size(size_t default_size) | |||
250 | if (!io_tlb_overflow_buffer) | 288 | if (!io_tlb_overflow_buffer) |
251 | goto cleanup4; | 289 | goto cleanup4; |
252 | 290 | ||
253 | printk(KERN_INFO "Placing %luMB software IO TLB between 0x%lx - " | 291 | swiotlb_print_info(bytes); |
254 | "0x%lx\n", bytes >> 20, | ||
255 | virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end)); | ||
256 | 292 | ||
257 | return 0; | 293 | return 0; |
258 | 294 | ||
259 | cleanup4: | 295 | cleanup4: |
260 | free_pages((unsigned long)io_tlb_orig_addr, get_order(io_tlb_nslabs * | 296 | free_pages((unsigned long)io_tlb_orig_addr, |
261 | sizeof(char *))); | 297 | get_order(io_tlb_nslabs * sizeof(phys_addr_t))); |
262 | io_tlb_orig_addr = NULL; | 298 | io_tlb_orig_addr = NULL; |
263 | cleanup3: | 299 | cleanup3: |
264 | free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs * | 300 | free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs * |
@@ -279,16 +315,62 @@ address_needs_mapping(struct device *hwdev, dma_addr_t addr, size_t size) | |||
279 | return !is_buffer_dma_capable(dma_get_mask(hwdev), addr, size); | 315 | return !is_buffer_dma_capable(dma_get_mask(hwdev), addr, size); |
280 | } | 316 | } |
281 | 317 | ||
318 | static inline int range_needs_mapping(void *ptr, size_t size) | ||
319 | { | ||
320 | return swiotlb_force || swiotlb_arch_range_needs_mapping(ptr, size); | ||
321 | } | ||
322 | |||
282 | static int is_swiotlb_buffer(char *addr) | 323 | static int is_swiotlb_buffer(char *addr) |
283 | { | 324 | { |
284 | return addr >= io_tlb_start && addr < io_tlb_end; | 325 | return addr >= io_tlb_start && addr < io_tlb_end; |
285 | } | 326 | } |
286 | 327 | ||
287 | /* | 328 | /* |
329 | * Bounce: copy the swiotlb buffer back to the original dma location | ||
330 | */ | ||
331 | static void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size, | ||
332 | enum dma_data_direction dir) | ||
333 | { | ||
334 | unsigned long pfn = PFN_DOWN(phys); | ||
335 | |||
336 | if (PageHighMem(pfn_to_page(pfn))) { | ||
337 | /* The buffer does not have a mapping. Map it in and copy */ | ||
338 | unsigned int offset = phys & ~PAGE_MASK; | ||
339 | char *buffer; | ||
340 | unsigned int sz = 0; | ||
341 | unsigned long flags; | ||
342 | |||
343 | while (size) { | ||
344 | sz = min(PAGE_SIZE - offset, size); | ||
345 | |||
346 | local_irq_save(flags); | ||
347 | buffer = kmap_atomic(pfn_to_page(pfn), | ||
348 | KM_BOUNCE_READ); | ||
349 | if (dir == DMA_TO_DEVICE) | ||
350 | memcpy(dma_addr, buffer + offset, sz); | ||
351 | else | ||
352 | memcpy(buffer + offset, dma_addr, sz); | ||
353 | kunmap_atomic(buffer, KM_BOUNCE_READ); | ||
354 | local_irq_restore(flags); | ||
355 | |||
356 | size -= sz; | ||
357 | pfn++; | ||
358 | dma_addr += sz; | ||
359 | offset = 0; | ||
360 | } | ||
361 | } else { | ||
362 | if (dir == DMA_TO_DEVICE) | ||
363 | memcpy(dma_addr, phys_to_virt(phys), size); | ||
364 | else | ||
365 | memcpy(phys_to_virt(phys), dma_addr, size); | ||
366 | } | ||
367 | } | ||
368 | |||
369 | /* | ||
288 | * Allocates bounce buffer and returns its kernel virtual address. | 370 | * Allocates bounce buffer and returns its kernel virtual address. |
289 | */ | 371 | */ |
290 | static void * | 372 | static void * |
291 | map_single(struct device *hwdev, char *buffer, size_t size, int dir) | 373 | map_single(struct device *hwdev, phys_addr_t phys, size_t size, int dir) |
292 | { | 374 | { |
293 | unsigned long flags; | 375 | unsigned long flags; |
294 | char *dma_addr; | 376 | char *dma_addr; |
@@ -300,9 +382,13 @@ map_single(struct device *hwdev, char *buffer, size_t size, int dir) | |||
300 | unsigned long max_slots; | 382 | unsigned long max_slots; |
301 | 383 | ||
302 | mask = dma_get_seg_boundary(hwdev); | 384 | mask = dma_get_seg_boundary(hwdev); |
303 | start_dma_addr = virt_to_bus(io_tlb_start) & mask; | 385 | start_dma_addr = swiotlb_virt_to_bus(hwdev, io_tlb_start) & mask; |
304 | 386 | ||
305 | offset_slots = ALIGN(start_dma_addr, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; | 387 | offset_slots = ALIGN(start_dma_addr, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; |
388 | |||
389 | /* | ||
390 | * Carefully handle integer overflow which can occur when mask == ~0UL. | ||
391 | */ | ||
306 | max_slots = mask + 1 | 392 | max_slots = mask + 1 |
307 | ? ALIGN(mask + 1, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT | 393 | ? ALIGN(mask + 1, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT |
308 | : 1UL << (BITS_PER_LONG - IO_TLB_SHIFT); | 394 | : 1UL << (BITS_PER_LONG - IO_TLB_SHIFT); |
@@ -379,9 +465,9 @@ found: | |||
379 | * needed. | 465 | * needed. |
380 | */ | 466 | */ |
381 | for (i = 0; i < nslots; i++) | 467 | for (i = 0; i < nslots; i++) |
382 | io_tlb_orig_addr[index+i] = buffer + (i << IO_TLB_SHIFT); | 468 | io_tlb_orig_addr[index+i] = phys + (i << IO_TLB_SHIFT); |
383 | if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) | 469 | if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) |
384 | memcpy(dma_addr, buffer, size); | 470 | swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE); |
385 | 471 | ||
386 | return dma_addr; | 472 | return dma_addr; |
387 | } | 473 | } |
@@ -395,17 +481,13 @@ unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir) | |||
395 | unsigned long flags; | 481 | unsigned long flags; |
396 | int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; | 482 | int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; |
397 | int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; | 483 | int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; |
398 | char *buffer = io_tlb_orig_addr[index]; | 484 | phys_addr_t phys = io_tlb_orig_addr[index]; |
399 | 485 | ||
400 | /* | 486 | /* |
401 | * First, sync the memory before unmapping the entry | 487 | * First, sync the memory before unmapping the entry |
402 | */ | 488 | */ |
403 | if (buffer && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))) | 489 | if (phys && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))) |
404 | /* | 490 | swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE); |
405 | * bounce... copy the data back into the original buffer * and | ||
406 | * delete the bounce buffer. | ||
407 | */ | ||
408 | memcpy(buffer, dma_addr, size); | ||
409 | 491 | ||
410 | /* | 492 | /* |
411 | * Return the buffer to the free list by setting the corresponding | 493 | * Return the buffer to the free list by setting the corresponding |
@@ -438,20 +520,20 @@ sync_single(struct device *hwdev, char *dma_addr, size_t size, | |||
438 | int dir, int target) | 520 | int dir, int target) |
439 | { | 521 | { |
440 | int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; | 522 | int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; |
441 | char *buffer = io_tlb_orig_addr[index]; | 523 | phys_addr_t phys = io_tlb_orig_addr[index]; |
442 | 524 | ||
443 | buffer += ((unsigned long)dma_addr & ((1 << IO_TLB_SHIFT) - 1)); | 525 | phys += ((unsigned long)dma_addr & ((1 << IO_TLB_SHIFT) - 1)); |
444 | 526 | ||
445 | switch (target) { | 527 | switch (target) { |
446 | case SYNC_FOR_CPU: | 528 | case SYNC_FOR_CPU: |
447 | if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)) | 529 | if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)) |
448 | memcpy(buffer, dma_addr, size); | 530 | swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE); |
449 | else | 531 | else |
450 | BUG_ON(dir != DMA_TO_DEVICE); | 532 | BUG_ON(dir != DMA_TO_DEVICE); |
451 | break; | 533 | break; |
452 | case SYNC_FOR_DEVICE: | 534 | case SYNC_FOR_DEVICE: |
453 | if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) | 535 | if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) |
454 | memcpy(dma_addr, buffer, size); | 536 | swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE); |
455 | else | 537 | else |
456 | BUG_ON(dir != DMA_FROM_DEVICE); | 538 | BUG_ON(dir != DMA_FROM_DEVICE); |
457 | break; | 539 | break; |
@@ -473,7 +555,9 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, | |||
473 | dma_mask = hwdev->coherent_dma_mask; | 555 | dma_mask = hwdev->coherent_dma_mask; |
474 | 556 | ||
475 | ret = (void *)__get_free_pages(flags, order); | 557 | ret = (void *)__get_free_pages(flags, order); |
476 | if (ret && !is_buffer_dma_capable(dma_mask, virt_to_bus(ret), size)) { | 558 | if (ret && |
559 | !is_buffer_dma_capable(dma_mask, swiotlb_virt_to_bus(hwdev, ret), | ||
560 | size)) { | ||
477 | /* | 561 | /* |
478 | * The allocated memory isn't reachable by the device. | 562 | * The allocated memory isn't reachable by the device. |
479 | * Fall back on swiotlb_map_single(). | 563 | * Fall back on swiotlb_map_single(). |
@@ -488,13 +572,13 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, | |||
488 | * swiotlb_map_single(), which will grab memory from | 572 | * swiotlb_map_single(), which will grab memory from |
489 | * the lowest available address range. | 573 | * the lowest available address range. |
490 | */ | 574 | */ |
491 | ret = map_single(hwdev, NULL, size, DMA_FROM_DEVICE); | 575 | ret = map_single(hwdev, 0, size, DMA_FROM_DEVICE); |
492 | if (!ret) | 576 | if (!ret) |
493 | return NULL; | 577 | return NULL; |
494 | } | 578 | } |
495 | 579 | ||
496 | memset(ret, 0, size); | 580 | memset(ret, 0, size); |
497 | dev_addr = virt_to_bus(ret); | 581 | dev_addr = swiotlb_virt_to_bus(hwdev, ret); |
498 | 582 | ||
499 | /* Confirm address can be DMA'd by device */ | 583 | /* Confirm address can be DMA'd by device */ |
500 | if (!is_buffer_dma_capable(dma_mask, dev_addr, size)) { | 584 | if (!is_buffer_dma_capable(dma_mask, dev_addr, size)) { |
@@ -509,6 +593,7 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, | |||
509 | *dma_handle = dev_addr; | 593 | *dma_handle = dev_addr; |
510 | return ret; | 594 | return ret; |
511 | } | 595 | } |
596 | EXPORT_SYMBOL(swiotlb_alloc_coherent); | ||
512 | 597 | ||
513 | void | 598 | void |
514 | swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, | 599 | swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, |
@@ -521,6 +606,7 @@ swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, | |||
521 | /* DMA_TO_DEVICE to avoid memcpy in unmap_single */ | 606 | /* DMA_TO_DEVICE to avoid memcpy in unmap_single */ |
522 | unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE); | 607 | unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE); |
523 | } | 608 | } |
609 | EXPORT_SYMBOL(swiotlb_free_coherent); | ||
524 | 610 | ||
525 | static void | 611 | static void |
526 | swiotlb_full(struct device *dev, size_t size, int dir, int do_panic) | 612 | swiotlb_full(struct device *dev, size_t size, int dir, int do_panic) |
@@ -533,7 +619,7 @@ swiotlb_full(struct device *dev, size_t size, int dir, int do_panic) | |||
533 | * the damage, or panic when the transfer is too big. | 619 | * the damage, or panic when the transfer is too big. |
534 | */ | 620 | */ |
535 | printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at " | 621 | printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at " |
536 | "device %s\n", size, dev ? dev->bus_id : "?"); | 622 | "device %s\n", size, dev ? dev_name(dev) : "?"); |
537 | 623 | ||
538 | if (size > io_tlb_overflow && do_panic) { | 624 | if (size > io_tlb_overflow && do_panic) { |
539 | if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) | 625 | if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) |
@@ -554,7 +640,7 @@ dma_addr_t | |||
554 | swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size, | 640 | swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size, |
555 | int dir, struct dma_attrs *attrs) | 641 | int dir, struct dma_attrs *attrs) |
556 | { | 642 | { |
557 | dma_addr_t dev_addr = virt_to_bus(ptr); | 643 | dma_addr_t dev_addr = swiotlb_virt_to_bus(hwdev, ptr); |
558 | void *map; | 644 | void *map; |
559 | 645 | ||
560 | BUG_ON(dir == DMA_NONE); | 646 | BUG_ON(dir == DMA_NONE); |
@@ -563,19 +649,20 @@ swiotlb_map_single_attrs(struct device *hwdev, void *ptr, size_t size, | |||
563 | * we can safely return the device addr and not worry about bounce | 649 | * we can safely return the device addr and not worry about bounce |
564 | * buffering it. | 650 | * buffering it. |
565 | */ | 651 | */ |
566 | if (!address_needs_mapping(hwdev, dev_addr, size) && !swiotlb_force) | 652 | if (!address_needs_mapping(hwdev, dev_addr, size) && |
653 | !range_needs_mapping(ptr, size)) | ||
567 | return dev_addr; | 654 | return dev_addr; |
568 | 655 | ||
569 | /* | 656 | /* |
570 | * Oh well, have to allocate and map a bounce buffer. | 657 | * Oh well, have to allocate and map a bounce buffer. |
571 | */ | 658 | */ |
572 | map = map_single(hwdev, ptr, size, dir); | 659 | map = map_single(hwdev, virt_to_phys(ptr), size, dir); |
573 | if (!map) { | 660 | if (!map) { |
574 | swiotlb_full(hwdev, size, dir, 1); | 661 | swiotlb_full(hwdev, size, dir, 1); |
575 | map = io_tlb_overflow_buffer; | 662 | map = io_tlb_overflow_buffer; |
576 | } | 663 | } |
577 | 664 | ||
578 | dev_addr = virt_to_bus(map); | 665 | dev_addr = swiotlb_virt_to_bus(hwdev, map); |
579 | 666 | ||
580 | /* | 667 | /* |
581 | * Ensure that the address returned is DMA'ble | 668 | * Ensure that the address returned is DMA'ble |
@@ -592,6 +679,7 @@ swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir) | |||
592 | { | 679 | { |
593 | return swiotlb_map_single_attrs(hwdev, ptr, size, dir, NULL); | 680 | return swiotlb_map_single_attrs(hwdev, ptr, size, dir, NULL); |
594 | } | 681 | } |
682 | EXPORT_SYMBOL(swiotlb_map_single); | ||
595 | 683 | ||
596 | /* | 684 | /* |
597 | * Unmap a single streaming mode DMA translation. The dma_addr and size must | 685 | * Unmap a single streaming mode DMA translation. The dma_addr and size must |
@@ -605,7 +693,7 @@ void | |||
605 | swiotlb_unmap_single_attrs(struct device *hwdev, dma_addr_t dev_addr, | 693 | swiotlb_unmap_single_attrs(struct device *hwdev, dma_addr_t dev_addr, |
606 | size_t size, int dir, struct dma_attrs *attrs) | 694 | size_t size, int dir, struct dma_attrs *attrs) |
607 | { | 695 | { |
608 | char *dma_addr = bus_to_virt(dev_addr); | 696 | char *dma_addr = swiotlb_bus_to_virt(dev_addr); |
609 | 697 | ||
610 | BUG_ON(dir == DMA_NONE); | 698 | BUG_ON(dir == DMA_NONE); |
611 | if (is_swiotlb_buffer(dma_addr)) | 699 | if (is_swiotlb_buffer(dma_addr)) |
@@ -621,6 +709,8 @@ swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size, | |||
621 | { | 709 | { |
622 | return swiotlb_unmap_single_attrs(hwdev, dev_addr, size, dir, NULL); | 710 | return swiotlb_unmap_single_attrs(hwdev, dev_addr, size, dir, NULL); |
623 | } | 711 | } |
712 | EXPORT_SYMBOL(swiotlb_unmap_single); | ||
713 | |||
624 | /* | 714 | /* |
625 | * Make physical memory consistent for a single streaming mode DMA translation | 715 | * Make physical memory consistent for a single streaming mode DMA translation |
626 | * after a transfer. | 716 | * after a transfer. |
@@ -635,7 +725,7 @@ static void | |||
635 | swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr, | 725 | swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr, |
636 | size_t size, int dir, int target) | 726 | size_t size, int dir, int target) |
637 | { | 727 | { |
638 | char *dma_addr = bus_to_virt(dev_addr); | 728 | char *dma_addr = swiotlb_bus_to_virt(dev_addr); |
639 | 729 | ||
640 | BUG_ON(dir == DMA_NONE); | 730 | BUG_ON(dir == DMA_NONE); |
641 | if (is_swiotlb_buffer(dma_addr)) | 731 | if (is_swiotlb_buffer(dma_addr)) |
@@ -650,6 +740,7 @@ swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr, | |||
650 | { | 740 | { |
651 | swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU); | 741 | swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU); |
652 | } | 742 | } |
743 | EXPORT_SYMBOL(swiotlb_sync_single_for_cpu); | ||
653 | 744 | ||
654 | void | 745 | void |
655 | swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, | 746 | swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, |
@@ -657,6 +748,7 @@ swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, | |||
657 | { | 748 | { |
658 | swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE); | 749 | swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE); |
659 | } | 750 | } |
751 | EXPORT_SYMBOL(swiotlb_sync_single_for_device); | ||
660 | 752 | ||
661 | /* | 753 | /* |
662 | * Same as above, but for a sub-range of the mapping. | 754 | * Same as above, but for a sub-range of the mapping. |
@@ -666,7 +758,7 @@ swiotlb_sync_single_range(struct device *hwdev, dma_addr_t dev_addr, | |||
666 | unsigned long offset, size_t size, | 758 | unsigned long offset, size_t size, |
667 | int dir, int target) | 759 | int dir, int target) |
668 | { | 760 | { |
669 | char *dma_addr = bus_to_virt(dev_addr) + offset; | 761 | char *dma_addr = swiotlb_bus_to_virt(dev_addr) + offset; |
670 | 762 | ||
671 | BUG_ON(dir == DMA_NONE); | 763 | BUG_ON(dir == DMA_NONE); |
672 | if (is_swiotlb_buffer(dma_addr)) | 764 | if (is_swiotlb_buffer(dma_addr)) |
@@ -682,6 +774,7 @@ swiotlb_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dev_addr, | |||
682 | swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir, | 774 | swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir, |
683 | SYNC_FOR_CPU); | 775 | SYNC_FOR_CPU); |
684 | } | 776 | } |
777 | EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_cpu); | ||
685 | 778 | ||
686 | void | 779 | void |
687 | swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr, | 780 | swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr, |
@@ -690,9 +783,8 @@ swiotlb_sync_single_range_for_device(struct device *hwdev, dma_addr_t dev_addr, | |||
690 | swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir, | 783 | swiotlb_sync_single_range(hwdev, dev_addr, offset, size, dir, |
691 | SYNC_FOR_DEVICE); | 784 | SYNC_FOR_DEVICE); |
692 | } | 785 | } |
786 | EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device); | ||
693 | 787 | ||
694 | void swiotlb_unmap_sg_attrs(struct device *, struct scatterlist *, int, int, | ||
695 | struct dma_attrs *); | ||
696 | /* | 788 | /* |
697 | * Map a set of buffers described by scatterlist in streaming mode for DMA. | 789 | * Map a set of buffers described by scatterlist in streaming mode for DMA. |
698 | * This is the scatter-gather version of the above swiotlb_map_single | 790 | * This is the scatter-gather version of the above swiotlb_map_single |
@@ -714,18 +806,18 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, | |||
714 | int dir, struct dma_attrs *attrs) | 806 | int dir, struct dma_attrs *attrs) |
715 | { | 807 | { |
716 | struct scatterlist *sg; | 808 | struct scatterlist *sg; |
717 | void *addr; | ||
718 | dma_addr_t dev_addr; | ||
719 | int i; | 809 | int i; |
720 | 810 | ||
721 | BUG_ON(dir == DMA_NONE); | 811 | BUG_ON(dir == DMA_NONE); |
722 | 812 | ||
723 | for_each_sg(sgl, sg, nelems, i) { | 813 | for_each_sg(sgl, sg, nelems, i) { |
724 | addr = SG_ENT_VIRT_ADDRESS(sg); | 814 | void *addr = sg_virt(sg); |
725 | dev_addr = virt_to_bus(addr); | 815 | dma_addr_t dev_addr = swiotlb_virt_to_bus(hwdev, addr); |
726 | if (swiotlb_force || | 816 | |
817 | if (range_needs_mapping(addr, sg->length) || | ||
727 | address_needs_mapping(hwdev, dev_addr, sg->length)) { | 818 | address_needs_mapping(hwdev, dev_addr, sg->length)) { |
728 | void *map = map_single(hwdev, addr, sg->length, dir); | 819 | void *map = map_single(hwdev, sg_phys(sg), |
820 | sg->length, dir); | ||
729 | if (!map) { | 821 | if (!map) { |
730 | /* Don't panic here, we expect map_sg users | 822 | /* Don't panic here, we expect map_sg users |
731 | to do proper error handling. */ | 823 | to do proper error handling. */ |
@@ -735,7 +827,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, | |||
735 | sgl[0].dma_length = 0; | 827 | sgl[0].dma_length = 0; |
736 | return 0; | 828 | return 0; |
737 | } | 829 | } |
738 | sg->dma_address = virt_to_bus(map); | 830 | sg->dma_address = swiotlb_virt_to_bus(hwdev, map); |
739 | } else | 831 | } else |
740 | sg->dma_address = dev_addr; | 832 | sg->dma_address = dev_addr; |
741 | sg->dma_length = sg->length; | 833 | sg->dma_length = sg->length; |
@@ -750,6 +842,7 @@ swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems, | |||
750 | { | 842 | { |
751 | return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL); | 843 | return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL); |
752 | } | 844 | } |
845 | EXPORT_SYMBOL(swiotlb_map_sg); | ||
753 | 846 | ||
754 | /* | 847 | /* |
755 | * Unmap a set of streaming mode DMA translations. Again, cpu read rules | 848 | * Unmap a set of streaming mode DMA translations. Again, cpu read rules |
@@ -765,11 +858,11 @@ swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, | |||
765 | BUG_ON(dir == DMA_NONE); | 858 | BUG_ON(dir == DMA_NONE); |
766 | 859 | ||
767 | for_each_sg(sgl, sg, nelems, i) { | 860 | for_each_sg(sgl, sg, nelems, i) { |
768 | if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg)) | 861 | if (sg->dma_address != swiotlb_virt_to_bus(hwdev, sg_virt(sg))) |
769 | unmap_single(hwdev, bus_to_virt(sg->dma_address), | 862 | unmap_single(hwdev, swiotlb_bus_to_virt(sg->dma_address), |
770 | sg->dma_length, dir); | 863 | sg->dma_length, dir); |
771 | else if (dir == DMA_FROM_DEVICE) | 864 | else if (dir == DMA_FROM_DEVICE) |
772 | dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length); | 865 | dma_mark_clean(sg_virt(sg), sg->dma_length); |
773 | } | 866 | } |
774 | } | 867 | } |
775 | EXPORT_SYMBOL(swiotlb_unmap_sg_attrs); | 868 | EXPORT_SYMBOL(swiotlb_unmap_sg_attrs); |
@@ -780,6 +873,7 @@ swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems, | |||
780 | { | 873 | { |
781 | return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL); | 874 | return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL); |
782 | } | 875 | } |
876 | EXPORT_SYMBOL(swiotlb_unmap_sg); | ||
783 | 877 | ||
784 | /* | 878 | /* |
785 | * Make physical memory consistent for a set of streaming mode DMA translations | 879 | * Make physical memory consistent for a set of streaming mode DMA translations |
@@ -798,11 +892,11 @@ swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl, | |||
798 | BUG_ON(dir == DMA_NONE); | 892 | BUG_ON(dir == DMA_NONE); |
799 | 893 | ||
800 | for_each_sg(sgl, sg, nelems, i) { | 894 | for_each_sg(sgl, sg, nelems, i) { |
801 | if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg)) | 895 | if (sg->dma_address != swiotlb_virt_to_bus(hwdev, sg_virt(sg))) |
802 | sync_single(hwdev, bus_to_virt(sg->dma_address), | 896 | sync_single(hwdev, swiotlb_bus_to_virt(sg->dma_address), |
803 | sg->dma_length, dir, target); | 897 | sg->dma_length, dir, target); |
804 | else if (dir == DMA_FROM_DEVICE) | 898 | else if (dir == DMA_FROM_DEVICE) |
805 | dma_mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length); | 899 | dma_mark_clean(sg_virt(sg), sg->dma_length); |
806 | } | 900 | } |
807 | } | 901 | } |
808 | 902 | ||
@@ -812,6 +906,7 @@ swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, | |||
812 | { | 906 | { |
813 | swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU); | 907 | swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU); |
814 | } | 908 | } |
909 | EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu); | ||
815 | 910 | ||
816 | void | 911 | void |
817 | swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, | 912 | swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, |
@@ -819,12 +914,14 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, | |||
819 | { | 914 | { |
820 | swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE); | 915 | swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE); |
821 | } | 916 | } |
917 | EXPORT_SYMBOL(swiotlb_sync_sg_for_device); | ||
822 | 918 | ||
823 | int | 919 | int |
824 | swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) | 920 | swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) |
825 | { | 921 | { |
826 | return (dma_addr == virt_to_bus(io_tlb_overflow_buffer)); | 922 | return (dma_addr == swiotlb_virt_to_bus(hwdev, io_tlb_overflow_buffer)); |
827 | } | 923 | } |
924 | EXPORT_SYMBOL(swiotlb_dma_mapping_error); | ||
828 | 925 | ||
829 | /* | 926 | /* |
830 | * Return whether the given device DMA address mask can be supported | 927 | * Return whether the given device DMA address mask can be supported |
@@ -835,20 +932,6 @@ swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) | |||
835 | int | 932 | int |
836 | swiotlb_dma_supported(struct device *hwdev, u64 mask) | 933 | swiotlb_dma_supported(struct device *hwdev, u64 mask) |
837 | { | 934 | { |
838 | return virt_to_bus(io_tlb_end - 1) <= mask; | 935 | return swiotlb_virt_to_bus(hwdev, io_tlb_end - 1) <= mask; |
839 | } | 936 | } |
840 | |||
841 | EXPORT_SYMBOL(swiotlb_map_single); | ||
842 | EXPORT_SYMBOL(swiotlb_unmap_single); | ||
843 | EXPORT_SYMBOL(swiotlb_map_sg); | ||
844 | EXPORT_SYMBOL(swiotlb_unmap_sg); | ||
845 | EXPORT_SYMBOL(swiotlb_sync_single_for_cpu); | ||
846 | EXPORT_SYMBOL(swiotlb_sync_single_for_device); | ||
847 | EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_cpu); | ||
848 | EXPORT_SYMBOL_GPL(swiotlb_sync_single_range_for_device); | ||
849 | EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu); | ||
850 | EXPORT_SYMBOL(swiotlb_sync_sg_for_device); | ||
851 | EXPORT_SYMBOL(swiotlb_dma_mapping_error); | ||
852 | EXPORT_SYMBOL(swiotlb_alloc_coherent); | ||
853 | EXPORT_SYMBOL(swiotlb_free_coherent); | ||
854 | EXPORT_SYMBOL(swiotlb_dma_supported); | 937 | EXPORT_SYMBOL(swiotlb_dma_supported); |