diff options
-rw-r--r-- | drivers/dma/dmaengine.c | 182 | ||||
-rw-r--r-- | drivers/dma/ioatdma.c | 295 | ||||
-rw-r--r-- | drivers/dma/ioatdma.h | 13 | ||||
-rw-r--r-- | include/linux/dmaengine.h | 237 |
4 files changed, 474 insertions, 253 deletions
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 828310d8be80..404cc7b6e705 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c | |||
@@ -59,6 +59,7 @@ | |||
59 | 59 | ||
60 | #include <linux/init.h> | 60 | #include <linux/init.h> |
61 | #include <linux/module.h> | 61 | #include <linux/module.h> |
62 | #include <linux/mm.h> | ||
62 | #include <linux/device.h> | 63 | #include <linux/device.h> |
63 | #include <linux/dmaengine.h> | 64 | #include <linux/dmaengine.h> |
64 | #include <linux/hardirq.h> | 65 | #include <linux/hardirq.h> |
@@ -66,6 +67,7 @@ | |||
66 | #include <linux/percpu.h> | 67 | #include <linux/percpu.h> |
67 | #include <linux/rcupdate.h> | 68 | #include <linux/rcupdate.h> |
68 | #include <linux/mutex.h> | 69 | #include <linux/mutex.h> |
70 | #include <linux/jiffies.h> | ||
69 | 71 | ||
70 | static DEFINE_MUTEX(dma_list_mutex); | 72 | static DEFINE_MUTEX(dma_list_mutex); |
71 | static LIST_HEAD(dma_device_list); | 73 | static LIST_HEAD(dma_device_list); |
@@ -165,6 +167,24 @@ static struct dma_chan *dma_client_chan_alloc(struct dma_client *client) | |||
165 | return NULL; | 167 | return NULL; |
166 | } | 168 | } |
167 | 169 | ||
170 | enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie) | ||
171 | { | ||
172 | enum dma_status status; | ||
173 | unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000); | ||
174 | |||
175 | dma_async_issue_pending(chan); | ||
176 | do { | ||
177 | status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); | ||
178 | if (time_after_eq(jiffies, dma_sync_wait_timeout)) { | ||
179 | printk(KERN_ERR "dma_sync_wait_timeout!\n"); | ||
180 | return DMA_ERROR; | ||
181 | } | ||
182 | } while (status == DMA_IN_PROGRESS); | ||
183 | |||
184 | return status; | ||
185 | } | ||
186 | EXPORT_SYMBOL(dma_sync_wait); | ||
187 | |||
168 | /** | 188 | /** |
169 | * dma_chan_cleanup - release a DMA channel's resources | 189 | * dma_chan_cleanup - release a DMA channel's resources |
170 | * @kref: kernel reference structure that contains the DMA channel device | 190 | * @kref: kernel reference structure that contains the DMA channel device |
@@ -322,6 +342,25 @@ int dma_async_device_register(struct dma_device *device) | |||
322 | if (!device) | 342 | if (!device) |
323 | return -ENODEV; | 343 | return -ENODEV; |
324 | 344 | ||
345 | /* validate device routines */ | ||
346 | BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) && | ||
347 | !device->device_prep_dma_memcpy); | ||
348 | BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) && | ||
349 | !device->device_prep_dma_xor); | ||
350 | BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) && | ||
351 | !device->device_prep_dma_zero_sum); | ||
352 | BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) && | ||
353 | !device->device_prep_dma_memset); | ||
354 | BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) && | ||
355 | !device->device_prep_dma_interrupt); | ||
356 | |||
357 | BUG_ON(!device->device_alloc_chan_resources); | ||
358 | BUG_ON(!device->device_free_chan_resources); | ||
359 | BUG_ON(!device->device_dependency_added); | ||
360 | BUG_ON(!device->device_is_tx_complete); | ||
361 | BUG_ON(!device->device_issue_pending); | ||
362 | BUG_ON(!device->dev); | ||
363 | |||
325 | init_completion(&device->done); | 364 | init_completion(&device->done); |
326 | kref_init(&device->refcount); | 365 | kref_init(&device->refcount); |
327 | device->dev_id = id++; | 366 | device->dev_id = id++; |
@@ -415,6 +454,149 @@ void dma_async_device_unregister(struct dma_device *device) | |||
415 | } | 454 | } |
416 | EXPORT_SYMBOL(dma_async_device_unregister); | 455 | EXPORT_SYMBOL(dma_async_device_unregister); |
417 | 456 | ||
457 | /** | ||
458 | * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses | ||
459 | * @chan: DMA channel to offload copy to | ||
460 | * @dest: destination address (virtual) | ||
461 | * @src: source address (virtual) | ||
462 | * @len: length | ||
463 | * | ||
464 | * Both @dest and @src must be mappable to a bus address according to the | ||
465 | * DMA mapping API rules for streaming mappings. | ||
466 | * Both @dest and @src must stay memory resident (kernel memory or locked | ||
467 | * user space pages). | ||
468 | */ | ||
469 | dma_cookie_t | ||
470 | dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest, | ||
471 | void *src, size_t len) | ||
472 | { | ||
473 | struct dma_device *dev = chan->device; | ||
474 | struct dma_async_tx_descriptor *tx; | ||
475 | dma_addr_t addr; | ||
476 | dma_cookie_t cookie; | ||
477 | int cpu; | ||
478 | |||
479 | tx = dev->device_prep_dma_memcpy(chan, len, 0); | ||
480 | if (!tx) | ||
481 | return -ENOMEM; | ||
482 | |||
483 | tx->ack = 1; | ||
484 | tx->callback = NULL; | ||
485 | addr = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE); | ||
486 | tx->tx_set_src(addr, tx, 0); | ||
487 | addr = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE); | ||
488 | tx->tx_set_dest(addr, tx, 0); | ||
489 | cookie = tx->tx_submit(tx); | ||
490 | |||
491 | cpu = get_cpu(); | ||
492 | per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; | ||
493 | per_cpu_ptr(chan->local, cpu)->memcpy_count++; | ||
494 | put_cpu(); | ||
495 | |||
496 | return cookie; | ||
497 | } | ||
498 | EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf); | ||
499 | |||
500 | /** | ||
501 | * dma_async_memcpy_buf_to_pg - offloaded copy from address to page | ||
502 | * @chan: DMA channel to offload copy to | ||
503 | * @page: destination page | ||
504 | * @offset: offset in page to copy to | ||
505 | * @kdata: source address (virtual) | ||
506 | * @len: length | ||
507 | * | ||
508 | * Both @page/@offset and @kdata must be mappable to a bus address according | ||
509 | * to the DMA mapping API rules for streaming mappings. | ||
510 | * Both @page/@offset and @kdata must stay memory resident (kernel memory or | ||
511 | * locked user space pages) | ||
512 | */ | ||
513 | dma_cookie_t | ||
514 | dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page, | ||
515 | unsigned int offset, void *kdata, size_t len) | ||
516 | { | ||
517 | struct dma_device *dev = chan->device; | ||
518 | struct dma_async_tx_descriptor *tx; | ||
519 | dma_addr_t addr; | ||
520 | dma_cookie_t cookie; | ||
521 | int cpu; | ||
522 | |||
523 | tx = dev->device_prep_dma_memcpy(chan, len, 0); | ||
524 | if (!tx) | ||
525 | return -ENOMEM; | ||
526 | |||
527 | tx->ack = 1; | ||
528 | tx->callback = NULL; | ||
529 | addr = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE); | ||
530 | tx->tx_set_src(addr, tx, 0); | ||
531 | addr = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE); | ||
532 | tx->tx_set_dest(addr, tx, 0); | ||
533 | cookie = tx->tx_submit(tx); | ||
534 | |||
535 | cpu = get_cpu(); | ||
536 | per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; | ||
537 | per_cpu_ptr(chan->local, cpu)->memcpy_count++; | ||
538 | put_cpu(); | ||
539 | |||
540 | return cookie; | ||
541 | } | ||
542 | EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg); | ||
543 | |||
544 | /** | ||
545 | * dma_async_memcpy_pg_to_pg - offloaded copy from page to page | ||
546 | * @chan: DMA channel to offload copy to | ||
547 | * @dest_pg: destination page | ||
548 | * @dest_off: offset in page to copy to | ||
549 | * @src_pg: source page | ||
550 | * @src_off: offset in page to copy from | ||
551 | * @len: length | ||
552 | * | ||
553 | * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus | ||
554 | * address according to the DMA mapping API rules for streaming mappings. | ||
555 | * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident | ||
556 | * (kernel memory or locked user space pages). | ||
557 | */ | ||
558 | dma_cookie_t | ||
559 | dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg, | ||
560 | unsigned int dest_off, struct page *src_pg, unsigned int src_off, | ||
561 | size_t len) | ||
562 | { | ||
563 | struct dma_device *dev = chan->device; | ||
564 | struct dma_async_tx_descriptor *tx; | ||
565 | dma_addr_t addr; | ||
566 | dma_cookie_t cookie; | ||
567 | int cpu; | ||
568 | |||
569 | tx = dev->device_prep_dma_memcpy(chan, len, 0); | ||
570 | if (!tx) | ||
571 | return -ENOMEM; | ||
572 | |||
573 | tx->ack = 1; | ||
574 | tx->callback = NULL; | ||
575 | addr = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE); | ||
576 | tx->tx_set_src(addr, tx, 0); | ||
577 | addr = dma_map_page(dev->dev, dest_pg, dest_off, len, DMA_FROM_DEVICE); | ||
578 | tx->tx_set_dest(addr, tx, 0); | ||
579 | cookie = tx->tx_submit(tx); | ||
580 | |||
581 | cpu = get_cpu(); | ||
582 | per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; | ||
583 | per_cpu_ptr(chan->local, cpu)->memcpy_count++; | ||
584 | put_cpu(); | ||
585 | |||
586 | return cookie; | ||
587 | } | ||
588 | EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg); | ||
589 | |||
590 | void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, | ||
591 | struct dma_chan *chan) | ||
592 | { | ||
593 | tx->chan = chan; | ||
594 | spin_lock_init(&tx->lock); | ||
595 | INIT_LIST_HEAD(&tx->depend_node); | ||
596 | INIT_LIST_HEAD(&tx->depend_list); | ||
597 | } | ||
598 | EXPORT_SYMBOL(dma_async_tx_descriptor_init); | ||
599 | |||
418 | static int __init dma_bus_init(void) | 600 | static int __init dma_bus_init(void) |
419 | { | 601 | { |
420 | mutex_init(&dma_list_mutex); | 602 | mutex_init(&dma_list_mutex); |
diff --git a/drivers/dma/ioatdma.c b/drivers/dma/ioatdma.c index c4209af4fde4..171044930282 100644 --- a/drivers/dma/ioatdma.c +++ b/drivers/dma/ioatdma.c | |||
@@ -38,6 +38,7 @@ | |||
38 | #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common) | 38 | #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common) |
39 | #define to_ioat_device(dev) container_of(dev, struct ioat_device, common) | 39 | #define to_ioat_device(dev) container_of(dev, struct ioat_device, common) |
40 | #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) | 40 | #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) |
41 | #define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx) | ||
41 | 42 | ||
42 | /* internal functions */ | 43 | /* internal functions */ |
43 | static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent); | 44 | static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent); |
@@ -78,6 +79,73 @@ static int enumerate_dma_channels(struct ioat_device *device) | |||
78 | return device->common.chancnt; | 79 | return device->common.chancnt; |
79 | } | 80 | } |
80 | 81 | ||
82 | static void | ||
83 | ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index) | ||
84 | { | ||
85 | struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx); | ||
86 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); | ||
87 | |||
88 | pci_unmap_addr_set(desc, src, addr); | ||
89 | |||
90 | list_for_each_entry(iter, &desc->async_tx.tx_list, node) { | ||
91 | iter->hw->src_addr = addr; | ||
92 | addr += ioat_chan->xfercap; | ||
93 | } | ||
94 | |||
95 | } | ||
96 | |||
97 | static void | ||
98 | ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index) | ||
99 | { | ||
100 | struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx); | ||
101 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); | ||
102 | |||
103 | pci_unmap_addr_set(desc, dst, addr); | ||
104 | |||
105 | list_for_each_entry(iter, &desc->async_tx.tx_list, node) { | ||
106 | iter->hw->dst_addr = addr; | ||
107 | addr += ioat_chan->xfercap; | ||
108 | } | ||
109 | } | ||
110 | |||
111 | static dma_cookie_t | ||
112 | ioat_tx_submit(struct dma_async_tx_descriptor *tx) | ||
113 | { | ||
114 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan); | ||
115 | struct ioat_desc_sw *desc = tx_to_ioat_desc(tx); | ||
116 | int append = 0; | ||
117 | dma_cookie_t cookie; | ||
118 | struct ioat_desc_sw *group_start; | ||
119 | |||
120 | group_start = list_entry(desc->async_tx.tx_list.next, | ||
121 | struct ioat_desc_sw, node); | ||
122 | spin_lock_bh(&ioat_chan->desc_lock); | ||
123 | /* cookie incr and addition to used_list must be atomic */ | ||
124 | cookie = ioat_chan->common.cookie; | ||
125 | cookie++; | ||
126 | if (cookie < 0) | ||
127 | cookie = 1; | ||
128 | ioat_chan->common.cookie = desc->async_tx.cookie = cookie; | ||
129 | |||
130 | /* write address into NextDescriptor field of last desc in chain */ | ||
131 | to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = | ||
132 | group_start->async_tx.phys; | ||
133 | list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev); | ||
134 | |||
135 | ioat_chan->pending += desc->tx_cnt; | ||
136 | if (ioat_chan->pending >= 4) { | ||
137 | append = 1; | ||
138 | ioat_chan->pending = 0; | ||
139 | } | ||
140 | spin_unlock_bh(&ioat_chan->desc_lock); | ||
141 | |||
142 | if (append) | ||
143 | writeb(IOAT_CHANCMD_APPEND, | ||
144 | ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); | ||
145 | |||
146 | return cookie; | ||
147 | } | ||
148 | |||
81 | static struct ioat_desc_sw *ioat_dma_alloc_descriptor( | 149 | static struct ioat_desc_sw *ioat_dma_alloc_descriptor( |
82 | struct ioat_dma_chan *ioat_chan, | 150 | struct ioat_dma_chan *ioat_chan, |
83 | gfp_t flags) | 151 | gfp_t flags) |
@@ -99,8 +167,13 @@ static struct ioat_desc_sw *ioat_dma_alloc_descriptor( | |||
99 | } | 167 | } |
100 | 168 | ||
101 | memset(desc, 0, sizeof(*desc)); | 169 | memset(desc, 0, sizeof(*desc)); |
170 | dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common); | ||
171 | desc_sw->async_tx.tx_set_src = ioat_set_src; | ||
172 | desc_sw->async_tx.tx_set_dest = ioat_set_dest; | ||
173 | desc_sw->async_tx.tx_submit = ioat_tx_submit; | ||
174 | INIT_LIST_HEAD(&desc_sw->async_tx.tx_list); | ||
102 | desc_sw->hw = desc; | 175 | desc_sw->hw = desc; |
103 | desc_sw->phys = phys; | 176 | desc_sw->async_tx.phys = phys; |
104 | 177 | ||
105 | return desc_sw; | 178 | return desc_sw; |
106 | } | 179 | } |
@@ -188,12 +261,14 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan) | |||
188 | list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { | 261 | list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { |
189 | in_use_descs++; | 262 | in_use_descs++; |
190 | list_del(&desc->node); | 263 | list_del(&desc->node); |
191 | pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); | 264 | pci_pool_free(ioat_device->dma_pool, desc->hw, |
265 | desc->async_tx.phys); | ||
192 | kfree(desc); | 266 | kfree(desc); |
193 | } | 267 | } |
194 | list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) { | 268 | list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) { |
195 | list_del(&desc->node); | 269 | list_del(&desc->node); |
196 | pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); | 270 | pci_pool_free(ioat_device->dma_pool, desc->hw, |
271 | desc->async_tx.phys); | ||
197 | kfree(desc); | 272 | kfree(desc); |
198 | } | 273 | } |
199 | spin_unlock_bh(&ioat_chan->desc_lock); | 274 | spin_unlock_bh(&ioat_chan->desc_lock); |
@@ -215,45 +290,25 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan) | |||
215 | writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); | 290 | writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); |
216 | } | 291 | } |
217 | 292 | ||
218 | /** | 293 | static struct dma_async_tx_descriptor * |
219 | * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction | 294 | ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en) |
220 | * @ioat_chan: IOAT DMA channel handle | ||
221 | * @dest: DMA destination address | ||
222 | * @src: DMA source address | ||
223 | * @len: transaction length in bytes | ||
224 | */ | ||
225 | |||
226 | static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan, | ||
227 | dma_addr_t dest, | ||
228 | dma_addr_t src, | ||
229 | size_t len) | ||
230 | { | 295 | { |
231 | struct ioat_desc_sw *first; | 296 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); |
232 | struct ioat_desc_sw *prev; | 297 | struct ioat_desc_sw *first, *prev, *new; |
233 | struct ioat_desc_sw *new; | ||
234 | dma_cookie_t cookie; | ||
235 | LIST_HEAD(new_chain); | 298 | LIST_HEAD(new_chain); |
236 | u32 copy; | 299 | u32 copy; |
237 | size_t orig_len; | 300 | size_t orig_len; |
238 | dma_addr_t orig_src, orig_dst; | 301 | int desc_count = 0; |
239 | unsigned int desc_count = 0; | ||
240 | unsigned int append = 0; | ||
241 | |||
242 | if (!ioat_chan || !dest || !src) | ||
243 | return -EFAULT; | ||
244 | 302 | ||
245 | if (!len) | 303 | if (!len) |
246 | return ioat_chan->common.cookie; | 304 | return NULL; |
247 | 305 | ||
248 | orig_len = len; | 306 | orig_len = len; |
249 | orig_src = src; | ||
250 | orig_dst = dest; | ||
251 | 307 | ||
252 | first = NULL; | 308 | first = NULL; |
253 | prev = NULL; | 309 | prev = NULL; |
254 | 310 | ||
255 | spin_lock_bh(&ioat_chan->desc_lock); | 311 | spin_lock_bh(&ioat_chan->desc_lock); |
256 | |||
257 | while (len) { | 312 | while (len) { |
258 | if (!list_empty(&ioat_chan->free_desc)) { | 313 | if (!list_empty(&ioat_chan->free_desc)) { |
259 | new = to_ioat_desc(ioat_chan->free_desc.next); | 314 | new = to_ioat_desc(ioat_chan->free_desc.next); |
@@ -270,140 +325,36 @@ static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan, | |||
270 | 325 | ||
271 | new->hw->size = copy; | 326 | new->hw->size = copy; |
272 | new->hw->ctl = 0; | 327 | new->hw->ctl = 0; |
273 | new->hw->src_addr = src; | 328 | new->async_tx.cookie = 0; |
274 | new->hw->dst_addr = dest; | 329 | new->async_tx.ack = 1; |
275 | new->cookie = 0; | ||
276 | 330 | ||
277 | /* chain together the physical address list for the HW */ | 331 | /* chain together the physical address list for the HW */ |
278 | if (!first) | 332 | if (!first) |
279 | first = new; | 333 | first = new; |
280 | else | 334 | else |
281 | prev->hw->next = (u64) new->phys; | 335 | prev->hw->next = (u64) new->async_tx.phys; |
282 | 336 | ||
283 | prev = new; | 337 | prev = new; |
284 | |||
285 | len -= copy; | 338 | len -= copy; |
286 | dest += copy; | ||
287 | src += copy; | ||
288 | |||
289 | list_add_tail(&new->node, &new_chain); | 339 | list_add_tail(&new->node, &new_chain); |
290 | desc_count++; | 340 | desc_count++; |
291 | } | 341 | } |
292 | new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS; | ||
293 | new->hw->next = 0; | ||
294 | 342 | ||
295 | /* cookie incr and addition to used_list must be atomic */ | 343 | list_splice(&new_chain, &new->async_tx.tx_list); |
296 | 344 | ||
297 | cookie = ioat_chan->common.cookie; | 345 | new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS; |
298 | cookie++; | 346 | new->hw->next = 0; |
299 | if (cookie < 0) | 347 | new->tx_cnt = desc_count; |
300 | cookie = 1; | 348 | new->async_tx.ack = 0; /* client is in control of this ack */ |
301 | ioat_chan->common.cookie = new->cookie = cookie; | 349 | new->async_tx.cookie = -EBUSY; |
302 | 350 | ||
303 | pci_unmap_addr_set(new, src, orig_src); | ||
304 | pci_unmap_addr_set(new, dst, orig_dst); | ||
305 | pci_unmap_len_set(new, src_len, orig_len); | 351 | pci_unmap_len_set(new, src_len, orig_len); |
306 | pci_unmap_len_set(new, dst_len, orig_len); | 352 | pci_unmap_len_set(new, dst_len, orig_len); |
307 | |||
308 | /* write address into NextDescriptor field of last desc in chain */ | ||
309 | to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys; | ||
310 | list_splice_init(&new_chain, ioat_chan->used_desc.prev); | ||
311 | |||
312 | ioat_chan->pending += desc_count; | ||
313 | if (ioat_chan->pending >= 4) { | ||
314 | append = 1; | ||
315 | ioat_chan->pending = 0; | ||
316 | } | ||
317 | |||
318 | spin_unlock_bh(&ioat_chan->desc_lock); | 353 | spin_unlock_bh(&ioat_chan->desc_lock); |
319 | 354 | ||
320 | if (append) | 355 | return new ? &new->async_tx : NULL; |
321 | writeb(IOAT_CHANCMD_APPEND, | ||
322 | ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); | ||
323 | return cookie; | ||
324 | } | 356 | } |
325 | 357 | ||
326 | /** | ||
327 | * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs | ||
328 | * @chan: IOAT DMA channel handle | ||
329 | * @dest: DMA destination address | ||
330 | * @src: DMA source address | ||
331 | * @len: transaction length in bytes | ||
332 | */ | ||
333 | |||
334 | static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan, | ||
335 | void *dest, | ||
336 | void *src, | ||
337 | size_t len) | ||
338 | { | ||
339 | dma_addr_t dest_addr; | ||
340 | dma_addr_t src_addr; | ||
341 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | ||
342 | |||
343 | dest_addr = pci_map_single(ioat_chan->device->pdev, | ||
344 | dest, len, PCI_DMA_FROMDEVICE); | ||
345 | src_addr = pci_map_single(ioat_chan->device->pdev, | ||
346 | src, len, PCI_DMA_TODEVICE); | ||
347 | |||
348 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | ||
349 | } | ||
350 | |||
351 | /** | ||
352 | * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page | ||
353 | * @chan: IOAT DMA channel handle | ||
354 | * @page: pointer to the page to copy to | ||
355 | * @offset: offset into that page | ||
356 | * @src: DMA source address | ||
357 | * @len: transaction length in bytes | ||
358 | */ | ||
359 | |||
360 | static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan, | ||
361 | struct page *page, | ||
362 | unsigned int offset, | ||
363 | void *src, | ||
364 | size_t len) | ||
365 | { | ||
366 | dma_addr_t dest_addr; | ||
367 | dma_addr_t src_addr; | ||
368 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | ||
369 | |||
370 | dest_addr = pci_map_page(ioat_chan->device->pdev, | ||
371 | page, offset, len, PCI_DMA_FROMDEVICE); | ||
372 | src_addr = pci_map_single(ioat_chan->device->pdev, | ||
373 | src, len, PCI_DMA_TODEVICE); | ||
374 | |||
375 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | ||
376 | } | ||
377 | |||
378 | /** | ||
379 | * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages | ||
380 | * @chan: IOAT DMA channel handle | ||
381 | * @dest_pg: pointer to the page to copy to | ||
382 | * @dest_off: offset into that page | ||
383 | * @src_pg: pointer to the page to copy from | ||
384 | * @src_off: offset into that page | ||
385 | * @len: transaction length in bytes. This is guaranteed not to make a copy | ||
386 | * across a page boundary. | ||
387 | */ | ||
388 | |||
389 | static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan, | ||
390 | struct page *dest_pg, | ||
391 | unsigned int dest_off, | ||
392 | struct page *src_pg, | ||
393 | unsigned int src_off, | ||
394 | size_t len) | ||
395 | { | ||
396 | dma_addr_t dest_addr; | ||
397 | dma_addr_t src_addr; | ||
398 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | ||
399 | |||
400 | dest_addr = pci_map_page(ioat_chan->device->pdev, | ||
401 | dest_pg, dest_off, len, PCI_DMA_FROMDEVICE); | ||
402 | src_addr = pci_map_page(ioat_chan->device->pdev, | ||
403 | src_pg, src_off, len, PCI_DMA_TODEVICE); | ||
404 | |||
405 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | ||
406 | } | ||
407 | 358 | ||
408 | /** | 359 | /** |
409 | * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw | 360 | * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw |
@@ -465,8 +416,8 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) | |||
465 | * exceeding xfercap, perhaps. If so, only the last one will | 416 | * exceeding xfercap, perhaps. If so, only the last one will |
466 | * have a cookie, and require unmapping. | 417 | * have a cookie, and require unmapping. |
467 | */ | 418 | */ |
468 | if (desc->cookie) { | 419 | if (desc->async_tx.cookie) { |
469 | cookie = desc->cookie; | 420 | cookie = desc->async_tx.cookie; |
470 | 421 | ||
471 | /* yes we are unmapping both _page and _single alloc'd | 422 | /* yes we are unmapping both _page and _single alloc'd |
472 | regions with unmap_page. Is this *really* that bad? | 423 | regions with unmap_page. Is this *really* that bad? |
@@ -481,14 +432,19 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) | |||
481 | PCI_DMA_TODEVICE); | 432 | PCI_DMA_TODEVICE); |
482 | } | 433 | } |
483 | 434 | ||
484 | if (desc->phys != phys_complete) { | 435 | if (desc->async_tx.phys != phys_complete) { |
485 | /* a completed entry, but not the last, so cleanup */ | 436 | /* a completed entry, but not the last, so cleanup |
486 | list_del(&desc->node); | 437 | * if the client is done with the descriptor |
487 | list_add_tail(&desc->node, &chan->free_desc); | 438 | */ |
439 | if (desc->async_tx.ack) { | ||
440 | list_del(&desc->node); | ||
441 | list_add_tail(&desc->node, &chan->free_desc); | ||
442 | } else | ||
443 | desc->async_tx.cookie = 0; | ||
488 | } else { | 444 | } else { |
489 | /* last used desc. Do not remove, so we can append from | 445 | /* last used desc. Do not remove, so we can append from |
490 | it, but don't look at it next time, either */ | 446 | it, but don't look at it next time, either */ |
491 | desc->cookie = 0; | 447 | desc->async_tx.cookie = 0; |
492 | 448 | ||
493 | /* TODO check status bits? */ | 449 | /* TODO check status bits? */ |
494 | break; | 450 | break; |
@@ -504,6 +460,17 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) | |||
504 | spin_unlock(&chan->cleanup_lock); | 460 | spin_unlock(&chan->cleanup_lock); |
505 | } | 461 | } |
506 | 462 | ||
463 | static void ioat_dma_dependency_added(struct dma_chan *chan) | ||
464 | { | ||
465 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | ||
466 | spin_lock_bh(&ioat_chan->desc_lock); | ||
467 | if (ioat_chan->pending == 0) { | ||
468 | spin_unlock_bh(&ioat_chan->desc_lock); | ||
469 | ioat_dma_memcpy_cleanup(ioat_chan); | ||
470 | } else | ||
471 | spin_unlock_bh(&ioat_chan->desc_lock); | ||
472 | } | ||
473 | |||
507 | /** | 474 | /** |
508 | * ioat_dma_is_complete - poll the status of a IOAT DMA transaction | 475 | * ioat_dma_is_complete - poll the status of a IOAT DMA transaction |
509 | * @chan: IOAT DMA channel handle | 476 | * @chan: IOAT DMA channel handle |
@@ -606,13 +573,14 @@ static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan) | |||
606 | 573 | ||
607 | desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; | 574 | desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; |
608 | desc->hw->next = 0; | 575 | desc->hw->next = 0; |
576 | desc->async_tx.ack = 1; | ||
609 | 577 | ||
610 | list_add_tail(&desc->node, &ioat_chan->used_desc); | 578 | list_add_tail(&desc->node, &ioat_chan->used_desc); |
611 | spin_unlock_bh(&ioat_chan->desc_lock); | 579 | spin_unlock_bh(&ioat_chan->desc_lock); |
612 | 580 | ||
613 | writel(((u64) desc->phys) & 0x00000000FFFFFFFF, | 581 | writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF, |
614 | ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW); | 582 | ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW); |
615 | writel(((u64) desc->phys) >> 32, | 583 | writel(((u64) desc->async_tx.phys) >> 32, |
616 | ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH); | 584 | ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH); |
617 | 585 | ||
618 | writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); | 586 | writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); |
@@ -629,6 +597,8 @@ static int ioat_self_test(struct ioat_device *device) | |||
629 | u8 *src; | 597 | u8 *src; |
630 | u8 *dest; | 598 | u8 *dest; |
631 | struct dma_chan *dma_chan; | 599 | struct dma_chan *dma_chan; |
600 | struct dma_async_tx_descriptor *tx; | ||
601 | dma_addr_t addr; | ||
632 | dma_cookie_t cookie; | 602 | dma_cookie_t cookie; |
633 | int err = 0; | 603 | int err = 0; |
634 | 604 | ||
@@ -654,7 +624,15 @@ static int ioat_self_test(struct ioat_device *device) | |||
654 | goto out; | 624 | goto out; |
655 | } | 625 | } |
656 | 626 | ||
657 | cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE); | 627 | tx = ioat_dma_prep_memcpy(dma_chan, IOAT_TEST_SIZE, 0); |
628 | async_tx_ack(tx); | ||
629 | addr = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE, | ||
630 | DMA_TO_DEVICE); | ||
631 | ioat_set_src(addr, tx, 0); | ||
632 | addr = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE, | ||
633 | DMA_FROM_DEVICE); | ||
634 | ioat_set_dest(addr, tx, 0); | ||
635 | cookie = ioat_tx_submit(tx); | ||
658 | ioat_dma_memcpy_issue_pending(dma_chan); | 636 | ioat_dma_memcpy_issue_pending(dma_chan); |
659 | msleep(1); | 637 | msleep(1); |
660 | 638 | ||
@@ -750,13 +728,14 @@ static int __devinit ioat_probe(struct pci_dev *pdev, | |||
750 | INIT_LIST_HEAD(&device->common.channels); | 728 | INIT_LIST_HEAD(&device->common.channels); |
751 | enumerate_dma_channels(device); | 729 | enumerate_dma_channels(device); |
752 | 730 | ||
731 | dma_cap_set(DMA_MEMCPY, device->common.cap_mask); | ||
753 | device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; | 732 | device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; |
754 | device->common.device_free_chan_resources = ioat_dma_free_chan_resources; | 733 | device->common.device_free_chan_resources = ioat_dma_free_chan_resources; |
755 | device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf; | 734 | device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy; |
756 | device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg; | 735 | device->common.device_is_tx_complete = ioat_dma_is_complete; |
757 | device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg; | 736 | device->common.device_issue_pending = ioat_dma_memcpy_issue_pending; |
758 | device->common.device_memcpy_complete = ioat_dma_is_complete; | 737 | device->common.device_dependency_added = ioat_dma_dependency_added; |
759 | device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending; | 738 | device->common.dev = &pdev->dev; |
760 | printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n", | 739 | printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n", |
761 | device->common.chancnt); | 740 | device->common.chancnt); |
762 | 741 | ||
diff --git a/drivers/dma/ioatdma.h b/drivers/dma/ioatdma.h index 62b26a9be4c9..d3f69bb15fa0 100644 --- a/drivers/dma/ioatdma.h +++ b/drivers/dma/ioatdma.h | |||
@@ -105,21 +105,20 @@ struct ioat_dma_chan { | |||
105 | /** | 105 | /** |
106 | * struct ioat_desc_sw - wrapper around hardware descriptor | 106 | * struct ioat_desc_sw - wrapper around hardware descriptor |
107 | * @hw: hardware DMA descriptor | 107 | * @hw: hardware DMA descriptor |
108 | * @node: | 108 | * @node: this descriptor will either be on the free list, |
109 | * @cookie: | 109 | * or attached to a transaction list (async_tx.tx_list) |
110 | * @phys: | 110 | * @tx_cnt: number of descriptors required to complete the transaction |
111 | * @async_tx: the generic software descriptor for all engines | ||
111 | */ | 112 | */ |
112 | |||
113 | struct ioat_desc_sw { | 113 | struct ioat_desc_sw { |
114 | struct ioat_dma_descriptor *hw; | 114 | struct ioat_dma_descriptor *hw; |
115 | struct list_head node; | 115 | struct list_head node; |
116 | dma_cookie_t cookie; | 116 | int tx_cnt; |
117 | dma_addr_t phys; | ||
118 | DECLARE_PCI_UNMAP_ADDR(src) | 117 | DECLARE_PCI_UNMAP_ADDR(src) |
119 | DECLARE_PCI_UNMAP_LEN(src_len) | 118 | DECLARE_PCI_UNMAP_LEN(src_len) |
120 | DECLARE_PCI_UNMAP_ADDR(dst) | 119 | DECLARE_PCI_UNMAP_ADDR(dst) |
121 | DECLARE_PCI_UNMAP_LEN(dst_len) | 120 | DECLARE_PCI_UNMAP_LEN(dst_len) |
121 | struct dma_async_tx_descriptor async_tx; | ||
122 | }; | 122 | }; |
123 | 123 | ||
124 | #endif /* IOATDMA_H */ | 124 | #endif /* IOATDMA_H */ |
125 | |||
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index c94d8f1d62e5..3de1cf71031a 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h | |||
@@ -21,13 +21,12 @@ | |||
21 | #ifndef DMAENGINE_H | 21 | #ifndef DMAENGINE_H |
22 | #define DMAENGINE_H | 22 | #define DMAENGINE_H |
23 | 23 | ||
24 | #ifdef CONFIG_DMA_ENGINE | ||
25 | |||
26 | #include <linux/device.h> | 24 | #include <linux/device.h> |
27 | #include <linux/uio.h> | 25 | #include <linux/uio.h> |
28 | #include <linux/kref.h> | 26 | #include <linux/kref.h> |
29 | #include <linux/completion.h> | 27 | #include <linux/completion.h> |
30 | #include <linux/rcupdate.h> | 28 | #include <linux/rcupdate.h> |
29 | #include <linux/dma-mapping.h> | ||
31 | 30 | ||
32 | /** | 31 | /** |
33 | * enum dma_event - resource PNP/power managment events | 32 | * enum dma_event - resource PNP/power managment events |
@@ -65,6 +64,31 @@ enum dma_status { | |||
65 | }; | 64 | }; |
66 | 65 | ||
67 | /** | 66 | /** |
67 | * enum dma_transaction_type - DMA transaction types/indexes | ||
68 | */ | ||
69 | enum dma_transaction_type { | ||
70 | DMA_MEMCPY, | ||
71 | DMA_XOR, | ||
72 | DMA_PQ_XOR, | ||
73 | DMA_DUAL_XOR, | ||
74 | DMA_PQ_UPDATE, | ||
75 | DMA_ZERO_SUM, | ||
76 | DMA_PQ_ZERO_SUM, | ||
77 | DMA_MEMSET, | ||
78 | DMA_MEMCPY_CRC32C, | ||
79 | DMA_INTERRUPT, | ||
80 | }; | ||
81 | |||
82 | /* last transaction type for creation of the capabilities mask */ | ||
83 | #define DMA_TX_TYPE_END (DMA_INTERRUPT + 1) | ||
84 | |||
85 | /** | ||
86 | * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t. | ||
87 | * See linux/cpumask.h | ||
88 | */ | ||
89 | typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t; | ||
90 | |||
91 | /** | ||
68 | * struct dma_chan_percpu - the per-CPU part of struct dma_chan | 92 | * struct dma_chan_percpu - the per-CPU part of struct dma_chan |
69 | * @refcount: local_t used for open-coded "bigref" counting | 93 | * @refcount: local_t used for open-coded "bigref" counting |
70 | * @memcpy_count: transaction counter | 94 | * @memcpy_count: transaction counter |
@@ -157,48 +181,106 @@ struct dma_client { | |||
157 | struct list_head global_node; | 181 | struct list_head global_node; |
158 | }; | 182 | }; |
159 | 183 | ||
184 | typedef void (*dma_async_tx_callback)(void *dma_async_param); | ||
185 | /** | ||
186 | * struct dma_async_tx_descriptor - async transaction descriptor | ||
187 | * ---dma generic offload fields--- | ||
188 | * @cookie: tracking cookie for this transaction, set to -EBUSY if | ||
189 | * this tx is sitting on a dependency list | ||
190 | * @ack: the descriptor can not be reused until the client acknowledges | ||
191 | * receipt, i.e. has has a chance to establish any dependency chains | ||
192 | * @phys: physical address of the descriptor | ||
193 | * @tx_list: driver common field for operations that require multiple | ||
194 | * descriptors | ||
195 | * @chan: target channel for this operation | ||
196 | * @tx_submit: set the prepared descriptor(s) to be executed by the engine | ||
197 | * @tx_set_dest: set a destination address in a hardware descriptor | ||
198 | * @tx_set_src: set a source address in a hardware descriptor | ||
199 | * @callback: routine to call after this operation is complete | ||
200 | * @callback_param: general parameter to pass to the callback routine | ||
201 | * ---async_tx api specific fields--- | ||
202 | * @depend_list: at completion this list of transactions are submitted | ||
203 | * @depend_node: allow this transaction to be executed after another | ||
204 | * transaction has completed, possibly on another channel | ||
205 | * @parent: pointer to the next level up in the dependency chain | ||
206 | * @lock: protect the dependency list | ||
207 | */ | ||
208 | struct dma_async_tx_descriptor { | ||
209 | dma_cookie_t cookie; | ||
210 | int ack; | ||
211 | dma_addr_t phys; | ||
212 | struct list_head tx_list; | ||
213 | struct dma_chan *chan; | ||
214 | dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx); | ||
215 | void (*tx_set_dest)(dma_addr_t addr, | ||
216 | struct dma_async_tx_descriptor *tx, int index); | ||
217 | void (*tx_set_src)(dma_addr_t addr, | ||
218 | struct dma_async_tx_descriptor *tx, int index); | ||
219 | dma_async_tx_callback callback; | ||
220 | void *callback_param; | ||
221 | struct list_head depend_list; | ||
222 | struct list_head depend_node; | ||
223 | struct dma_async_tx_descriptor *parent; | ||
224 | spinlock_t lock; | ||
225 | }; | ||
226 | |||
160 | /** | 227 | /** |
161 | * struct dma_device - info on the entity supplying DMA services | 228 | * struct dma_device - info on the entity supplying DMA services |
162 | * @chancnt: how many DMA channels are supported | 229 | * @chancnt: how many DMA channels are supported |
163 | * @channels: the list of struct dma_chan | 230 | * @channels: the list of struct dma_chan |
164 | * @global_node: list_head for global dma_device_list | 231 | * @global_node: list_head for global dma_device_list |
232 | * @cap_mask: one or more dma_capability flags | ||
233 | * @max_xor: maximum number of xor sources, 0 if no capability | ||
165 | * @refcount: reference count | 234 | * @refcount: reference count |
166 | * @done: IO completion struct | 235 | * @done: IO completion struct |
167 | * @dev_id: unique device ID | 236 | * @dev_id: unique device ID |
237 | * @dev: struct device reference for dma mapping api | ||
168 | * @device_alloc_chan_resources: allocate resources and return the | 238 | * @device_alloc_chan_resources: allocate resources and return the |
169 | * number of allocated descriptors | 239 | * number of allocated descriptors |
170 | * @device_free_chan_resources: release DMA channel's resources | 240 | * @device_free_chan_resources: release DMA channel's resources |
171 | * @device_memcpy_buf_to_buf: memcpy buf pointer to buf pointer | 241 | * @device_prep_dma_memcpy: prepares a memcpy operation |
172 | * @device_memcpy_buf_to_pg: memcpy buf pointer to struct page | 242 | * @device_prep_dma_xor: prepares a xor operation |
173 | * @device_memcpy_pg_to_pg: memcpy struct page/offset to struct page/offset | 243 | * @device_prep_dma_zero_sum: prepares a zero_sum operation |
174 | * @device_memcpy_complete: poll the status of an IOAT DMA transaction | 244 | * @device_prep_dma_memset: prepares a memset operation |
175 | * @device_memcpy_issue_pending: push appended descriptors to hardware | 245 | * @device_prep_dma_interrupt: prepares an end of chain interrupt operation |
246 | * @device_dependency_added: async_tx notifies the channel about new deps | ||
247 | * @device_issue_pending: push pending transactions to hardware | ||
176 | */ | 248 | */ |
177 | struct dma_device { | 249 | struct dma_device { |
178 | 250 | ||
179 | unsigned int chancnt; | 251 | unsigned int chancnt; |
180 | struct list_head channels; | 252 | struct list_head channels; |
181 | struct list_head global_node; | 253 | struct list_head global_node; |
254 | dma_cap_mask_t cap_mask; | ||
255 | int max_xor; | ||
182 | 256 | ||
183 | struct kref refcount; | 257 | struct kref refcount; |
184 | struct completion done; | 258 | struct completion done; |
185 | 259 | ||
186 | int dev_id; | 260 | int dev_id; |
261 | struct device *dev; | ||
187 | 262 | ||
188 | int (*device_alloc_chan_resources)(struct dma_chan *chan); | 263 | int (*device_alloc_chan_resources)(struct dma_chan *chan); |
189 | void (*device_free_chan_resources)(struct dma_chan *chan); | 264 | void (*device_free_chan_resources)(struct dma_chan *chan); |
190 | dma_cookie_t (*device_memcpy_buf_to_buf)(struct dma_chan *chan, | 265 | |
191 | void *dest, void *src, size_t len); | 266 | struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)( |
192 | dma_cookie_t (*device_memcpy_buf_to_pg)(struct dma_chan *chan, | 267 | struct dma_chan *chan, size_t len, int int_en); |
193 | struct page *page, unsigned int offset, void *kdata, | 268 | struct dma_async_tx_descriptor *(*device_prep_dma_xor)( |
194 | size_t len); | 269 | struct dma_chan *chan, unsigned int src_cnt, size_t len, |
195 | dma_cookie_t (*device_memcpy_pg_to_pg)(struct dma_chan *chan, | 270 | int int_en); |
196 | struct page *dest_pg, unsigned int dest_off, | 271 | struct dma_async_tx_descriptor *(*device_prep_dma_zero_sum)( |
197 | struct page *src_pg, unsigned int src_off, size_t len); | 272 | struct dma_chan *chan, unsigned int src_cnt, size_t len, |
198 | enum dma_status (*device_memcpy_complete)(struct dma_chan *chan, | 273 | u32 *result, int int_en); |
274 | struct dma_async_tx_descriptor *(*device_prep_dma_memset)( | ||
275 | struct dma_chan *chan, int value, size_t len, int int_en); | ||
276 | struct dma_async_tx_descriptor *(*device_prep_dma_interrupt)( | ||
277 | struct dma_chan *chan); | ||
278 | |||
279 | void (*device_dependency_added)(struct dma_chan *chan); | ||
280 | enum dma_status (*device_is_tx_complete)(struct dma_chan *chan, | ||
199 | dma_cookie_t cookie, dma_cookie_t *last, | 281 | dma_cookie_t cookie, dma_cookie_t *last, |
200 | dma_cookie_t *used); | 282 | dma_cookie_t *used); |
201 | void (*device_memcpy_issue_pending)(struct dma_chan *chan); | 283 | void (*device_issue_pending)(struct dma_chan *chan); |
202 | }; | 284 | }; |
203 | 285 | ||
204 | /* --- public DMA engine API --- */ | 286 | /* --- public DMA engine API --- */ |
@@ -207,96 +289,72 @@ struct dma_client *dma_async_client_register(dma_event_callback event_callback); | |||
207 | void dma_async_client_unregister(struct dma_client *client); | 289 | void dma_async_client_unregister(struct dma_client *client); |
208 | void dma_async_client_chan_request(struct dma_client *client, | 290 | void dma_async_client_chan_request(struct dma_client *client, |
209 | unsigned int number); | 291 | unsigned int number); |
292 | dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan, | ||
293 | void *dest, void *src, size_t len); | ||
294 | dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan, | ||
295 | struct page *page, unsigned int offset, void *kdata, size_t len); | ||
296 | dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan, | ||
297 | struct page *dest_pg, unsigned int dest_off, struct page *src_pg, | ||
298 | unsigned int src_off, size_t len); | ||
299 | void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, | ||
300 | struct dma_chan *chan); | ||
210 | 301 | ||
211 | /** | ||
212 | * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses | ||
213 | * @chan: DMA channel to offload copy to | ||
214 | * @dest: destination address (virtual) | ||
215 | * @src: source address (virtual) | ||
216 | * @len: length | ||
217 | * | ||
218 | * Both @dest and @src must be mappable to a bus address according to the | ||
219 | * DMA mapping API rules for streaming mappings. | ||
220 | * Both @dest and @src must stay memory resident (kernel memory or locked | ||
221 | * user space pages). | ||
222 | */ | ||
223 | static inline dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan, | ||
224 | void *dest, void *src, size_t len) | ||
225 | { | ||
226 | int cpu = get_cpu(); | ||
227 | per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; | ||
228 | per_cpu_ptr(chan->local, cpu)->memcpy_count++; | ||
229 | put_cpu(); | ||
230 | 302 | ||
231 | return chan->device->device_memcpy_buf_to_buf(chan, dest, src, len); | 303 | static inline void |
304 | async_tx_ack(struct dma_async_tx_descriptor *tx) | ||
305 | { | ||
306 | tx->ack = 1; | ||
232 | } | 307 | } |
233 | 308 | ||
234 | /** | 309 | #define first_dma_cap(mask) __first_dma_cap(&(mask)) |
235 | * dma_async_memcpy_buf_to_pg - offloaded copy from address to page | 310 | static inline int __first_dma_cap(const dma_cap_mask_t *srcp) |
236 | * @chan: DMA channel to offload copy to | ||
237 | * @page: destination page | ||
238 | * @offset: offset in page to copy to | ||
239 | * @kdata: source address (virtual) | ||
240 | * @len: length | ||
241 | * | ||
242 | * Both @page/@offset and @kdata must be mappable to a bus address according | ||
243 | * to the DMA mapping API rules for streaming mappings. | ||
244 | * Both @page/@offset and @kdata must stay memory resident (kernel memory or | ||
245 | * locked user space pages) | ||
246 | */ | ||
247 | static inline dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan, | ||
248 | struct page *page, unsigned int offset, void *kdata, size_t len) | ||
249 | { | 311 | { |
250 | int cpu = get_cpu(); | 312 | return min_t(int, DMA_TX_TYPE_END, |
251 | per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; | 313 | find_first_bit(srcp->bits, DMA_TX_TYPE_END)); |
252 | per_cpu_ptr(chan->local, cpu)->memcpy_count++; | 314 | } |
253 | put_cpu(); | ||
254 | 315 | ||
255 | return chan->device->device_memcpy_buf_to_pg(chan, page, offset, | 316 | #define next_dma_cap(n, mask) __next_dma_cap((n), &(mask)) |
256 | kdata, len); | 317 | static inline int __next_dma_cap(int n, const dma_cap_mask_t *srcp) |
318 | { | ||
319 | return min_t(int, DMA_TX_TYPE_END, | ||
320 | find_next_bit(srcp->bits, DMA_TX_TYPE_END, n+1)); | ||
257 | } | 321 | } |
258 | 322 | ||
259 | /** | 323 | #define dma_cap_set(tx, mask) __dma_cap_set((tx), &(mask)) |
260 | * dma_async_memcpy_pg_to_pg - offloaded copy from page to page | 324 | static inline void |
261 | * @chan: DMA channel to offload copy to | 325 | __dma_cap_set(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp) |
262 | * @dest_pg: destination page | ||
263 | * @dest_off: offset in page to copy to | ||
264 | * @src_pg: source page | ||
265 | * @src_off: offset in page to copy from | ||
266 | * @len: length | ||
267 | * | ||
268 | * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus | ||
269 | * address according to the DMA mapping API rules for streaming mappings. | ||
270 | * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident | ||
271 | * (kernel memory or locked user space pages). | ||
272 | */ | ||
273 | static inline dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan, | ||
274 | struct page *dest_pg, unsigned int dest_off, struct page *src_pg, | ||
275 | unsigned int src_off, size_t len) | ||
276 | { | 326 | { |
277 | int cpu = get_cpu(); | 327 | set_bit(tx_type, dstp->bits); |
278 | per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; | 328 | } |
279 | per_cpu_ptr(chan->local, cpu)->memcpy_count++; | ||
280 | put_cpu(); | ||
281 | 329 | ||
282 | return chan->device->device_memcpy_pg_to_pg(chan, dest_pg, dest_off, | 330 | #define dma_has_cap(tx, mask) __dma_has_cap((tx), &(mask)) |
283 | src_pg, src_off, len); | 331 | static inline int |
332 | __dma_has_cap(enum dma_transaction_type tx_type, dma_cap_mask_t *srcp) | ||
333 | { | ||
334 | return test_bit(tx_type, srcp->bits); | ||
284 | } | 335 | } |
285 | 336 | ||
337 | #define for_each_dma_cap_mask(cap, mask) \ | ||
338 | for ((cap) = first_dma_cap(mask); \ | ||
339 | (cap) < DMA_TX_TYPE_END; \ | ||
340 | (cap) = next_dma_cap((cap), (mask))) | ||
341 | |||
286 | /** | 342 | /** |
287 | * dma_async_memcpy_issue_pending - flush pending copies to HW | 343 | * dma_async_issue_pending - flush pending transactions to HW |
288 | * @chan: target DMA channel | 344 | * @chan: target DMA channel |
289 | * | 345 | * |
290 | * This allows drivers to push copies to HW in batches, | 346 | * This allows drivers to push copies to HW in batches, |
291 | * reducing MMIO writes where possible. | 347 | * reducing MMIO writes where possible. |
292 | */ | 348 | */ |
293 | static inline void dma_async_memcpy_issue_pending(struct dma_chan *chan) | 349 | static inline void dma_async_issue_pending(struct dma_chan *chan) |
294 | { | 350 | { |
295 | return chan->device->device_memcpy_issue_pending(chan); | 351 | return chan->device->device_issue_pending(chan); |
296 | } | 352 | } |
297 | 353 | ||
354 | #define dma_async_memcpy_issue_pending(chan) dma_async_issue_pending(chan) | ||
355 | |||
298 | /** | 356 | /** |
299 | * dma_async_memcpy_complete - poll for transaction completion | 357 | * dma_async_is_tx_complete - poll for transaction completion |
300 | * @chan: DMA channel | 358 | * @chan: DMA channel |
301 | * @cookie: transaction identifier to check status of | 359 | * @cookie: transaction identifier to check status of |
302 | * @last: returns last completed cookie, can be NULL | 360 | * @last: returns last completed cookie, can be NULL |
@@ -306,12 +364,15 @@ static inline void dma_async_memcpy_issue_pending(struct dma_chan *chan) | |||
306 | * internal state and can be used with dma_async_is_complete() to check | 364 | * internal state and can be used with dma_async_is_complete() to check |
307 | * the status of multiple cookies without re-checking hardware state. | 365 | * the status of multiple cookies without re-checking hardware state. |
308 | */ | 366 | */ |
309 | static inline enum dma_status dma_async_memcpy_complete(struct dma_chan *chan, | 367 | static inline enum dma_status dma_async_is_tx_complete(struct dma_chan *chan, |
310 | dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used) | 368 | dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used) |
311 | { | 369 | { |
312 | return chan->device->device_memcpy_complete(chan, cookie, last, used); | 370 | return chan->device->device_is_tx_complete(chan, cookie, last, used); |
313 | } | 371 | } |
314 | 372 | ||
373 | #define dma_async_memcpy_complete(chan, cookie, last, used)\ | ||
374 | dma_async_is_tx_complete(chan, cookie, last, used) | ||
375 | |||
315 | /** | 376 | /** |
316 | * dma_async_is_complete - test a cookie against chan state | 377 | * dma_async_is_complete - test a cookie against chan state |
317 | * @cookie: transaction identifier to test status of | 378 | * @cookie: transaction identifier to test status of |
@@ -334,6 +395,7 @@ static inline enum dma_status dma_async_is_complete(dma_cookie_t cookie, | |||
334 | return DMA_IN_PROGRESS; | 395 | return DMA_IN_PROGRESS; |
335 | } | 396 | } |
336 | 397 | ||
398 | enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie); | ||
337 | 399 | ||
338 | /* --- DMA device --- */ | 400 | /* --- DMA device --- */ |
339 | 401 | ||
@@ -362,5 +424,4 @@ dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov, | |||
362 | struct dma_pinned_list *pinned_list, struct page *page, | 424 | struct dma_pinned_list *pinned_list, struct page *page, |
363 | unsigned int offset, size_t len); | 425 | unsigned int offset, size_t len); |
364 | 426 | ||
365 | #endif /* CONFIG_DMA_ENGINE */ | ||
366 | #endif /* DMAENGINE_H */ | 427 | #endif /* DMAENGINE_H */ |