diff options
Diffstat (limited to 'drivers/xen')
-rw-r--r-- | drivers/xen/Kconfig | 1 | ||||
-rw-r--r-- | drivers/xen/grant-table.c | 19 | ||||
-rw-r--r-- | drivers/xen/swiotlb-xen.c | 115 |
3 files changed, 106 insertions, 29 deletions
diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig index 23eae5cb69c2..c794ea182140 100644 --- a/drivers/xen/Kconfig +++ b/drivers/xen/Kconfig | |||
@@ -140,7 +140,6 @@ config XEN_GRANT_DEV_ALLOC | |||
140 | 140 | ||
141 | config SWIOTLB_XEN | 141 | config SWIOTLB_XEN |
142 | def_bool y | 142 | def_bool y |
143 | depends on PCI && X86 | ||
144 | select SWIOTLB | 143 | select SWIOTLB |
145 | 144 | ||
146 | config XEN_TMEM | 145 | config XEN_TMEM |
diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c index c4d2298893b1..62ccf5424ba8 100644 --- a/drivers/xen/grant-table.c +++ b/drivers/xen/grant-table.c | |||
@@ -49,6 +49,7 @@ | |||
49 | #include <xen/grant_table.h> | 49 | #include <xen/grant_table.h> |
50 | #include <xen/interface/memory.h> | 50 | #include <xen/interface/memory.h> |
51 | #include <xen/hvc-console.h> | 51 | #include <xen/hvc-console.h> |
52 | #include <xen/swiotlb-xen.h> | ||
52 | #include <asm/xen/hypercall.h> | 53 | #include <asm/xen/hypercall.h> |
53 | #include <asm/xen/interface.h> | 54 | #include <asm/xen/interface.h> |
54 | 55 | ||
@@ -898,8 +899,16 @@ int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops, | |||
898 | gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i, | 899 | gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i, |
899 | &map_ops[i].status, __func__); | 900 | &map_ops[i].status, __func__); |
900 | 901 | ||
901 | if (xen_feature(XENFEAT_auto_translated_physmap)) | 902 | /* this is basically a nop on x86 */ |
903 | if (xen_feature(XENFEAT_auto_translated_physmap)) { | ||
904 | for (i = 0; i < count; i++) { | ||
905 | if (map_ops[i].status) | ||
906 | continue; | ||
907 | set_phys_to_machine(map_ops[i].host_addr >> PAGE_SHIFT, | ||
908 | map_ops[i].dev_bus_addr >> PAGE_SHIFT); | ||
909 | } | ||
902 | return ret; | 910 | return ret; |
911 | } | ||
903 | 912 | ||
904 | if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { | 913 | if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { |
905 | arch_enter_lazy_mmu_mode(); | 914 | arch_enter_lazy_mmu_mode(); |
@@ -942,8 +951,14 @@ int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, | |||
942 | if (ret) | 951 | if (ret) |
943 | return ret; | 952 | return ret; |
944 | 953 | ||
945 | if (xen_feature(XENFEAT_auto_translated_physmap)) | 954 | /* this is basically a nop on x86 */ |
955 | if (xen_feature(XENFEAT_auto_translated_physmap)) { | ||
956 | for (i = 0; i < count; i++) { | ||
957 | set_phys_to_machine(unmap_ops[i].host_addr >> PAGE_SHIFT, | ||
958 | INVALID_P2M_ENTRY); | ||
959 | } | ||
946 | return ret; | 960 | return ret; |
961 | } | ||
947 | 962 | ||
948 | if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { | 963 | if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { |
949 | arch_enter_lazy_mmu_mode(); | 964 | arch_enter_lazy_mmu_mode(); |
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 44af9d8577de..a224bc74b6b9 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c | |||
@@ -43,6 +43,9 @@ | |||
43 | #include <xen/xen-ops.h> | 43 | #include <xen/xen-ops.h> |
44 | #include <xen/hvc-console.h> | 44 | #include <xen/hvc-console.h> |
45 | 45 | ||
46 | #include <asm/dma-mapping.h> | ||
47 | #include <asm/xen/page-coherent.h> | ||
48 | |||
46 | #include <trace/events/swiotlb.h> | 49 | #include <trace/events/swiotlb.h> |
47 | /* | 50 | /* |
48 | * Used to do a quick range check in swiotlb_tbl_unmap_single and | 51 | * Used to do a quick range check in swiotlb_tbl_unmap_single and |
@@ -50,6 +53,20 @@ | |||
50 | * API. | 53 | * API. |
51 | */ | 54 | */ |
52 | 55 | ||
56 | #ifndef CONFIG_X86 | ||
57 | static unsigned long dma_alloc_coherent_mask(struct device *dev, | ||
58 | gfp_t gfp) | ||
59 | { | ||
60 | unsigned long dma_mask = 0; | ||
61 | |||
62 | dma_mask = dev->coherent_dma_mask; | ||
63 | if (!dma_mask) | ||
64 | dma_mask = (gfp & GFP_DMA) ? DMA_BIT_MASK(24) : DMA_BIT_MASK(32); | ||
65 | |||
66 | return dma_mask; | ||
67 | } | ||
68 | #endif | ||
69 | |||
53 | static char *xen_io_tlb_start, *xen_io_tlb_end; | 70 | static char *xen_io_tlb_start, *xen_io_tlb_end; |
54 | static unsigned long xen_io_tlb_nslabs; | 71 | static unsigned long xen_io_tlb_nslabs; |
55 | /* | 72 | /* |
@@ -58,17 +75,17 @@ static unsigned long xen_io_tlb_nslabs; | |||
58 | 75 | ||
59 | static u64 start_dma_addr; | 76 | static u64 start_dma_addr; |
60 | 77 | ||
61 | static dma_addr_t xen_phys_to_bus(phys_addr_t paddr) | 78 | static inline dma_addr_t xen_phys_to_bus(phys_addr_t paddr) |
62 | { | 79 | { |
63 | return phys_to_machine(XPADDR(paddr)).maddr; | 80 | return phys_to_machine(XPADDR(paddr)).maddr; |
64 | } | 81 | } |
65 | 82 | ||
66 | static phys_addr_t xen_bus_to_phys(dma_addr_t baddr) | 83 | static inline phys_addr_t xen_bus_to_phys(dma_addr_t baddr) |
67 | { | 84 | { |
68 | return machine_to_phys(XMADDR(baddr)).paddr; | 85 | return machine_to_phys(XMADDR(baddr)).paddr; |
69 | } | 86 | } |
70 | 87 | ||
71 | static dma_addr_t xen_virt_to_bus(void *address) | 88 | static inline dma_addr_t xen_virt_to_bus(void *address) |
72 | { | 89 | { |
73 | return xen_phys_to_bus(virt_to_phys(address)); | 90 | return xen_phys_to_bus(virt_to_phys(address)); |
74 | } | 91 | } |
@@ -91,7 +108,7 @@ static int check_pages_physically_contiguous(unsigned long pfn, | |||
91 | return 1; | 108 | return 1; |
92 | } | 109 | } |
93 | 110 | ||
94 | static int range_straddles_page_boundary(phys_addr_t p, size_t size) | 111 | static inline int range_straddles_page_boundary(phys_addr_t p, size_t size) |
95 | { | 112 | { |
96 | unsigned long pfn = PFN_DOWN(p); | 113 | unsigned long pfn = PFN_DOWN(p); |
97 | unsigned int offset = p & ~PAGE_MASK; | 114 | unsigned int offset = p & ~PAGE_MASK; |
@@ -128,6 +145,8 @@ xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs) | |||
128 | { | 145 | { |
129 | int i, rc; | 146 | int i, rc; |
130 | int dma_bits; | 147 | int dma_bits; |
148 | dma_addr_t dma_handle; | ||
149 | phys_addr_t p = virt_to_phys(buf); | ||
131 | 150 | ||
132 | dma_bits = get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT) + PAGE_SHIFT; | 151 | dma_bits = get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT) + PAGE_SHIFT; |
133 | 152 | ||
@@ -137,9 +156,9 @@ xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs) | |||
137 | 156 | ||
138 | do { | 157 | do { |
139 | rc = xen_create_contiguous_region( | 158 | rc = xen_create_contiguous_region( |
140 | (unsigned long)buf + (i << IO_TLB_SHIFT), | 159 | p + (i << IO_TLB_SHIFT), |
141 | get_order(slabs << IO_TLB_SHIFT), | 160 | get_order(slabs << IO_TLB_SHIFT), |
142 | dma_bits); | 161 | dma_bits, &dma_handle); |
143 | } while (rc && dma_bits++ < max_dma_bits); | 162 | } while (rc && dma_bits++ < max_dma_bits); |
144 | if (rc) | 163 | if (rc) |
145 | return rc; | 164 | return rc; |
@@ -265,7 +284,6 @@ xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size, | |||
265 | void *ret; | 284 | void *ret; |
266 | int order = get_order(size); | 285 | int order = get_order(size); |
267 | u64 dma_mask = DMA_BIT_MASK(32); | 286 | u64 dma_mask = DMA_BIT_MASK(32); |
268 | unsigned long vstart; | ||
269 | phys_addr_t phys; | 287 | phys_addr_t phys; |
270 | dma_addr_t dev_addr; | 288 | dma_addr_t dev_addr; |
271 | 289 | ||
@@ -280,8 +298,12 @@ xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size, | |||
280 | if (dma_alloc_from_coherent(hwdev, size, dma_handle, &ret)) | 298 | if (dma_alloc_from_coherent(hwdev, size, dma_handle, &ret)) |
281 | return ret; | 299 | return ret; |
282 | 300 | ||
283 | vstart = __get_free_pages(flags, order); | 301 | /* On ARM this function returns an ioremap'ped virtual address for |
284 | ret = (void *)vstart; | 302 | * which virt_to_phys doesn't return the corresponding physical |
303 | * address. In fact on ARM virt_to_phys only works for kernel direct | ||
304 | * mapped RAM memory. Also see comment below. | ||
305 | */ | ||
306 | ret = xen_alloc_coherent_pages(hwdev, size, dma_handle, flags, attrs); | ||
285 | 307 | ||
286 | if (!ret) | 308 | if (!ret) |
287 | return ret; | 309 | return ret; |
@@ -289,18 +311,21 @@ xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size, | |||
289 | if (hwdev && hwdev->coherent_dma_mask) | 311 | if (hwdev && hwdev->coherent_dma_mask) |
290 | dma_mask = dma_alloc_coherent_mask(hwdev, flags); | 312 | dma_mask = dma_alloc_coherent_mask(hwdev, flags); |
291 | 313 | ||
292 | phys = virt_to_phys(ret); | 314 | /* At this point dma_handle is the physical address, next we are |
315 | * going to set it to the machine address. | ||
316 | * Do not use virt_to_phys(ret) because on ARM it doesn't correspond | ||
317 | * to *dma_handle. */ | ||
318 | phys = *dma_handle; | ||
293 | dev_addr = xen_phys_to_bus(phys); | 319 | dev_addr = xen_phys_to_bus(phys); |
294 | if (((dev_addr + size - 1 <= dma_mask)) && | 320 | if (((dev_addr + size - 1 <= dma_mask)) && |
295 | !range_straddles_page_boundary(phys, size)) | 321 | !range_straddles_page_boundary(phys, size)) |
296 | *dma_handle = dev_addr; | 322 | *dma_handle = dev_addr; |
297 | else { | 323 | else { |
298 | if (xen_create_contiguous_region(vstart, order, | 324 | if (xen_create_contiguous_region(phys, order, |
299 | fls64(dma_mask)) != 0) { | 325 | fls64(dma_mask), dma_handle) != 0) { |
300 | free_pages(vstart, order); | 326 | xen_free_coherent_pages(hwdev, size, ret, (dma_addr_t)phys, attrs); |
301 | return NULL; | 327 | return NULL; |
302 | } | 328 | } |
303 | *dma_handle = virt_to_machine(ret).maddr; | ||
304 | } | 329 | } |
305 | memset(ret, 0, size); | 330 | memset(ret, 0, size); |
306 | return ret; | 331 | return ret; |
@@ -321,13 +346,15 @@ xen_swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, | |||
321 | if (hwdev && hwdev->coherent_dma_mask) | 346 | if (hwdev && hwdev->coherent_dma_mask) |
322 | dma_mask = hwdev->coherent_dma_mask; | 347 | dma_mask = hwdev->coherent_dma_mask; |
323 | 348 | ||
324 | phys = virt_to_phys(vaddr); | 349 | /* do not use virt_to_phys because on ARM it doesn't return you the |
350 | * physical address */ | ||
351 | phys = xen_bus_to_phys(dev_addr); | ||
325 | 352 | ||
326 | if (((dev_addr + size - 1 > dma_mask)) || | 353 | if (((dev_addr + size - 1 > dma_mask)) || |
327 | range_straddles_page_boundary(phys, size)) | 354 | range_straddles_page_boundary(phys, size)) |
328 | xen_destroy_contiguous_region((unsigned long)vaddr, order); | 355 | xen_destroy_contiguous_region(phys, order); |
329 | 356 | ||
330 | free_pages((unsigned long)vaddr, order); | 357 | xen_free_coherent_pages(hwdev, size, vaddr, (dma_addr_t)phys, attrs); |
331 | } | 358 | } |
332 | EXPORT_SYMBOL_GPL(xen_swiotlb_free_coherent); | 359 | EXPORT_SYMBOL_GPL(xen_swiotlb_free_coherent); |
333 | 360 | ||
@@ -354,8 +381,13 @@ dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, | |||
354 | * buffering it. | 381 | * buffering it. |
355 | */ | 382 | */ |
356 | if (dma_capable(dev, dev_addr, size) && | 383 | if (dma_capable(dev, dev_addr, size) && |
357 | !range_straddles_page_boundary(phys, size) && !swiotlb_force) | 384 | !range_straddles_page_boundary(phys, size) && !swiotlb_force) { |
385 | /* we are not interested in the dma_addr returned by | ||
386 | * xen_dma_map_page, only in the potential cache flushes executed | ||
387 | * by the function. */ | ||
388 | xen_dma_map_page(dev, page, offset, size, dir, attrs); | ||
358 | return dev_addr; | 389 | return dev_addr; |
390 | } | ||
359 | 391 | ||
360 | /* | 392 | /* |
361 | * Oh well, have to allocate and map a bounce buffer. | 393 | * Oh well, have to allocate and map a bounce buffer. |
@@ -366,6 +398,8 @@ dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, | |||
366 | if (map == SWIOTLB_MAP_ERROR) | 398 | if (map == SWIOTLB_MAP_ERROR) |
367 | return DMA_ERROR_CODE; | 399 | return DMA_ERROR_CODE; |
368 | 400 | ||
401 | xen_dma_map_page(dev, pfn_to_page(map >> PAGE_SHIFT), | ||
402 | map & ~PAGE_MASK, size, dir, attrs); | ||
369 | dev_addr = xen_phys_to_bus(map); | 403 | dev_addr = xen_phys_to_bus(map); |
370 | 404 | ||
371 | /* | 405 | /* |
@@ -388,12 +422,15 @@ EXPORT_SYMBOL_GPL(xen_swiotlb_map_page); | |||
388 | * whatever the device wrote there. | 422 | * whatever the device wrote there. |
389 | */ | 423 | */ |
390 | static void xen_unmap_single(struct device *hwdev, dma_addr_t dev_addr, | 424 | static void xen_unmap_single(struct device *hwdev, dma_addr_t dev_addr, |
391 | size_t size, enum dma_data_direction dir) | 425 | size_t size, enum dma_data_direction dir, |
426 | struct dma_attrs *attrs) | ||
392 | { | 427 | { |
393 | phys_addr_t paddr = xen_bus_to_phys(dev_addr); | 428 | phys_addr_t paddr = xen_bus_to_phys(dev_addr); |
394 | 429 | ||
395 | BUG_ON(dir == DMA_NONE); | 430 | BUG_ON(dir == DMA_NONE); |
396 | 431 | ||
432 | xen_dma_unmap_page(hwdev, paddr, size, dir, attrs); | ||
433 | |||
397 | /* NOTE: We use dev_addr here, not paddr! */ | 434 | /* NOTE: We use dev_addr here, not paddr! */ |
398 | if (is_xen_swiotlb_buffer(dev_addr)) { | 435 | if (is_xen_swiotlb_buffer(dev_addr)) { |
399 | swiotlb_tbl_unmap_single(hwdev, paddr, size, dir); | 436 | swiotlb_tbl_unmap_single(hwdev, paddr, size, dir); |
@@ -416,7 +453,7 @@ void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, | |||
416 | size_t size, enum dma_data_direction dir, | 453 | size_t size, enum dma_data_direction dir, |
417 | struct dma_attrs *attrs) | 454 | struct dma_attrs *attrs) |
418 | { | 455 | { |
419 | xen_unmap_single(hwdev, dev_addr, size, dir); | 456 | xen_unmap_single(hwdev, dev_addr, size, dir, attrs); |
420 | } | 457 | } |
421 | EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_page); | 458 | EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_page); |
422 | 459 | ||
@@ -439,11 +476,15 @@ xen_swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr, | |||
439 | 476 | ||
440 | BUG_ON(dir == DMA_NONE); | 477 | BUG_ON(dir == DMA_NONE); |
441 | 478 | ||
479 | if (target == SYNC_FOR_CPU) | ||
480 | xen_dma_sync_single_for_cpu(hwdev, paddr, size, dir); | ||
481 | |||
442 | /* NOTE: We use dev_addr here, not paddr! */ | 482 | /* NOTE: We use dev_addr here, not paddr! */ |
443 | if (is_xen_swiotlb_buffer(dev_addr)) { | 483 | if (is_xen_swiotlb_buffer(dev_addr)) |
444 | swiotlb_tbl_sync_single(hwdev, paddr, size, dir, target); | 484 | swiotlb_tbl_sync_single(hwdev, paddr, size, dir, target); |
445 | return; | 485 | |
446 | } | 486 | if (target == SYNC_FOR_DEVICE) |
487 | xen_dma_sync_single_for_cpu(hwdev, paddr, size, dir); | ||
447 | 488 | ||
448 | if (dir != DMA_FROM_DEVICE) | 489 | if (dir != DMA_FROM_DEVICE) |
449 | return; | 490 | return; |
@@ -506,16 +547,26 @@ xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, | |||
506 | sg->length, | 547 | sg->length, |
507 | dir); | 548 | dir); |
508 | if (map == SWIOTLB_MAP_ERROR) { | 549 | if (map == SWIOTLB_MAP_ERROR) { |
550 | dev_warn(hwdev, "swiotlb buffer is full\n"); | ||
509 | /* Don't panic here, we expect map_sg users | 551 | /* Don't panic here, we expect map_sg users |
510 | to do proper error handling. */ | 552 | to do proper error handling. */ |
511 | xen_swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir, | 553 | xen_swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir, |
512 | attrs); | 554 | attrs); |
513 | sg_dma_len(sgl) = 0; | 555 | sg_dma_len(sgl) = 0; |
514 | return DMA_ERROR_CODE; | 556 | return 0; |
515 | } | 557 | } |
516 | sg->dma_address = xen_phys_to_bus(map); | 558 | sg->dma_address = xen_phys_to_bus(map); |
517 | } else | 559 | } else { |
560 | /* we are not interested in the dma_addr returned by | ||
561 | * xen_dma_map_page, only in the potential cache flushes executed | ||
562 | * by the function. */ | ||
563 | xen_dma_map_page(hwdev, pfn_to_page(paddr >> PAGE_SHIFT), | ||
564 | paddr & ~PAGE_MASK, | ||
565 | sg->length, | ||
566 | dir, | ||
567 | attrs); | ||
518 | sg->dma_address = dev_addr; | 568 | sg->dma_address = dev_addr; |
569 | } | ||
519 | sg_dma_len(sg) = sg->length; | 570 | sg_dma_len(sg) = sg->length; |
520 | } | 571 | } |
521 | return nelems; | 572 | return nelems; |
@@ -537,7 +588,7 @@ xen_swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, | |||
537 | BUG_ON(dir == DMA_NONE); | 588 | BUG_ON(dir == DMA_NONE); |
538 | 589 | ||
539 | for_each_sg(sgl, sg, nelems, i) | 590 | for_each_sg(sgl, sg, nelems, i) |
540 | xen_unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir); | 591 | xen_unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir, attrs); |
541 | 592 | ||
542 | } | 593 | } |
543 | EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_sg_attrs); | 594 | EXPORT_SYMBOL_GPL(xen_swiotlb_unmap_sg_attrs); |
@@ -597,3 +648,15 @@ xen_swiotlb_dma_supported(struct device *hwdev, u64 mask) | |||
597 | return xen_virt_to_bus(xen_io_tlb_end - 1) <= mask; | 648 | return xen_virt_to_bus(xen_io_tlb_end - 1) <= mask; |
598 | } | 649 | } |
599 | EXPORT_SYMBOL_GPL(xen_swiotlb_dma_supported); | 650 | EXPORT_SYMBOL_GPL(xen_swiotlb_dma_supported); |
651 | |||
652 | int | ||
653 | xen_swiotlb_set_dma_mask(struct device *dev, u64 dma_mask) | ||
654 | { | ||
655 | if (!dev->dma_mask || !xen_swiotlb_dma_supported(dev, dma_mask)) | ||
656 | return -EIO; | ||
657 | |||
658 | *dev->dma_mask = dma_mask; | ||
659 | |||
660 | return 0; | ||
661 | } | ||
662 | EXPORT_SYMBOL_GPL(xen_swiotlb_set_dma_mask); | ||