aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--drivers/dma/dmaengine.c182
-rw-r--r--drivers/dma/ioatdma.c295
-rw-r--r--drivers/dma/ioatdma.h13
-rw-r--r--include/linux/dmaengine.h237
4 files changed, 474 insertions, 253 deletions
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 828310d8be80..404cc7b6e705 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -59,6 +59,7 @@
59 59
60#include <linux/init.h> 60#include <linux/init.h>
61#include <linux/module.h> 61#include <linux/module.h>
62#include <linux/mm.h>
62#include <linux/device.h> 63#include <linux/device.h>
63#include <linux/dmaengine.h> 64#include <linux/dmaengine.h>
64#include <linux/hardirq.h> 65#include <linux/hardirq.h>
@@ -66,6 +67,7 @@
66#include <linux/percpu.h> 67#include <linux/percpu.h>
67#include <linux/rcupdate.h> 68#include <linux/rcupdate.h>
68#include <linux/mutex.h> 69#include <linux/mutex.h>
70#include <linux/jiffies.h>
69 71
70static DEFINE_MUTEX(dma_list_mutex); 72static DEFINE_MUTEX(dma_list_mutex);
71static LIST_HEAD(dma_device_list); 73static LIST_HEAD(dma_device_list);
@@ -165,6 +167,24 @@ static struct dma_chan *dma_client_chan_alloc(struct dma_client *client)
165 return NULL; 167 return NULL;
166} 168}
167 169
170enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
171{
172 enum dma_status status;
173 unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
174
175 dma_async_issue_pending(chan);
176 do {
177 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
178 if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
179 printk(KERN_ERR "dma_sync_wait_timeout!\n");
180 return DMA_ERROR;
181 }
182 } while (status == DMA_IN_PROGRESS);
183
184 return status;
185}
186EXPORT_SYMBOL(dma_sync_wait);
187
168/** 188/**
169 * dma_chan_cleanup - release a DMA channel's resources 189 * dma_chan_cleanup - release a DMA channel's resources
170 * @kref: kernel reference structure that contains the DMA channel device 190 * @kref: kernel reference structure that contains the DMA channel device
@@ -322,6 +342,25 @@ int dma_async_device_register(struct dma_device *device)
322 if (!device) 342 if (!device)
323 return -ENODEV; 343 return -ENODEV;
324 344
345 /* validate device routines */
346 BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) &&
347 !device->device_prep_dma_memcpy);
348 BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) &&
349 !device->device_prep_dma_xor);
350 BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
351 !device->device_prep_dma_zero_sum);
352 BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) &&
353 !device->device_prep_dma_memset);
354 BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
355 !device->device_prep_dma_interrupt);
356
357 BUG_ON(!device->device_alloc_chan_resources);
358 BUG_ON(!device->device_free_chan_resources);
359 BUG_ON(!device->device_dependency_added);
360 BUG_ON(!device->device_is_tx_complete);
361 BUG_ON(!device->device_issue_pending);
362 BUG_ON(!device->dev);
363
325 init_completion(&device->done); 364 init_completion(&device->done);
326 kref_init(&device->refcount); 365 kref_init(&device->refcount);
327 device->dev_id = id++; 366 device->dev_id = id++;
@@ -415,6 +454,149 @@ void dma_async_device_unregister(struct dma_device *device)
415} 454}
416EXPORT_SYMBOL(dma_async_device_unregister); 455EXPORT_SYMBOL(dma_async_device_unregister);
417 456
457/**
458 * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
459 * @chan: DMA channel to offload copy to
460 * @dest: destination address (virtual)
461 * @src: source address (virtual)
462 * @len: length
463 *
464 * Both @dest and @src must be mappable to a bus address according to the
465 * DMA mapping API rules for streaming mappings.
466 * Both @dest and @src must stay memory resident (kernel memory or locked
467 * user space pages).
468 */
469dma_cookie_t
470dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest,
471 void *src, size_t len)
472{
473 struct dma_device *dev = chan->device;
474 struct dma_async_tx_descriptor *tx;
475 dma_addr_t addr;
476 dma_cookie_t cookie;
477 int cpu;
478
479 tx = dev->device_prep_dma_memcpy(chan, len, 0);
480 if (!tx)
481 return -ENOMEM;
482
483 tx->ack = 1;
484 tx->callback = NULL;
485 addr = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE);
486 tx->tx_set_src(addr, tx, 0);
487 addr = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE);
488 tx->tx_set_dest(addr, tx, 0);
489 cookie = tx->tx_submit(tx);
490
491 cpu = get_cpu();
492 per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
493 per_cpu_ptr(chan->local, cpu)->memcpy_count++;
494 put_cpu();
495
496 return cookie;
497}
498EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf);
499
500/**
501 * dma_async_memcpy_buf_to_pg - offloaded copy from address to page
502 * @chan: DMA channel to offload copy to
503 * @page: destination page
504 * @offset: offset in page to copy to
505 * @kdata: source address (virtual)
506 * @len: length
507 *
508 * Both @page/@offset and @kdata must be mappable to a bus address according
509 * to the DMA mapping API rules for streaming mappings.
510 * Both @page/@offset and @kdata must stay memory resident (kernel memory or
511 * locked user space pages)
512 */
513dma_cookie_t
514dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page,
515 unsigned int offset, void *kdata, size_t len)
516{
517 struct dma_device *dev = chan->device;
518 struct dma_async_tx_descriptor *tx;
519 dma_addr_t addr;
520 dma_cookie_t cookie;
521 int cpu;
522
523 tx = dev->device_prep_dma_memcpy(chan, len, 0);
524 if (!tx)
525 return -ENOMEM;
526
527 tx->ack = 1;
528 tx->callback = NULL;
529 addr = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE);
530 tx->tx_set_src(addr, tx, 0);
531 addr = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE);
532 tx->tx_set_dest(addr, tx, 0);
533 cookie = tx->tx_submit(tx);
534
535 cpu = get_cpu();
536 per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
537 per_cpu_ptr(chan->local, cpu)->memcpy_count++;
538 put_cpu();
539
540 return cookie;
541}
542EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg);
543
544/**
545 * dma_async_memcpy_pg_to_pg - offloaded copy from page to page
546 * @chan: DMA channel to offload copy to
547 * @dest_pg: destination page
548 * @dest_off: offset in page to copy to
549 * @src_pg: source page
550 * @src_off: offset in page to copy from
551 * @len: length
552 *
553 * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus
554 * address according to the DMA mapping API rules for streaming mappings.
555 * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident
556 * (kernel memory or locked user space pages).
557 */
558dma_cookie_t
559dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg,
560 unsigned int dest_off, struct page *src_pg, unsigned int src_off,
561 size_t len)
562{
563 struct dma_device *dev = chan->device;
564 struct dma_async_tx_descriptor *tx;
565 dma_addr_t addr;
566 dma_cookie_t cookie;
567 int cpu;
568
569 tx = dev->device_prep_dma_memcpy(chan, len, 0);
570 if (!tx)
571 return -ENOMEM;
572
573 tx->ack = 1;
574 tx->callback = NULL;
575 addr = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE);
576 tx->tx_set_src(addr, tx, 0);
577 addr = dma_map_page(dev->dev, dest_pg, dest_off, len, DMA_FROM_DEVICE);
578 tx->tx_set_dest(addr, tx, 0);
579 cookie = tx->tx_submit(tx);
580
581 cpu = get_cpu();
582 per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
583 per_cpu_ptr(chan->local, cpu)->memcpy_count++;
584 put_cpu();
585
586 return cookie;
587}
588EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg);
589
590void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
591 struct dma_chan *chan)
592{
593 tx->chan = chan;
594 spin_lock_init(&tx->lock);
595 INIT_LIST_HEAD(&tx->depend_node);
596 INIT_LIST_HEAD(&tx->depend_list);
597}
598EXPORT_SYMBOL(dma_async_tx_descriptor_init);
599
418static int __init dma_bus_init(void) 600static int __init dma_bus_init(void)
419{ 601{
420 mutex_init(&dma_list_mutex); 602 mutex_init(&dma_list_mutex);
diff --git a/drivers/dma/ioatdma.c b/drivers/dma/ioatdma.c
index c4209af4fde4..171044930282 100644
--- a/drivers/dma/ioatdma.c
+++ b/drivers/dma/ioatdma.c
@@ -38,6 +38,7 @@
38#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common) 38#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
39#define to_ioat_device(dev) container_of(dev, struct ioat_device, common) 39#define to_ioat_device(dev) container_of(dev, struct ioat_device, common)
40#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) 40#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
41#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)
41 42
42/* internal functions */ 43/* internal functions */
43static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent); 44static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
@@ -78,6 +79,73 @@ static int enumerate_dma_channels(struct ioat_device *device)
78 return device->common.chancnt; 79 return device->common.chancnt;
79} 80}
80 81
82static void
83ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
84{
85 struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
86 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
87
88 pci_unmap_addr_set(desc, src, addr);
89
90 list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
91 iter->hw->src_addr = addr;
92 addr += ioat_chan->xfercap;
93 }
94
95}
96
97static void
98ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
99{
100 struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
101 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
102
103 pci_unmap_addr_set(desc, dst, addr);
104
105 list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
106 iter->hw->dst_addr = addr;
107 addr += ioat_chan->xfercap;
108 }
109}
110
111static dma_cookie_t
112ioat_tx_submit(struct dma_async_tx_descriptor *tx)
113{
114 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
115 struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
116 int append = 0;
117 dma_cookie_t cookie;
118 struct ioat_desc_sw *group_start;
119
120 group_start = list_entry(desc->async_tx.tx_list.next,
121 struct ioat_desc_sw, node);
122 spin_lock_bh(&ioat_chan->desc_lock);
123 /* cookie incr and addition to used_list must be atomic */
124 cookie = ioat_chan->common.cookie;
125 cookie++;
126 if (cookie < 0)
127 cookie = 1;
128 ioat_chan->common.cookie = desc->async_tx.cookie = cookie;
129
130 /* write address into NextDescriptor field of last desc in chain */
131 to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
132 group_start->async_tx.phys;
133 list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev);
134
135 ioat_chan->pending += desc->tx_cnt;
136 if (ioat_chan->pending >= 4) {
137 append = 1;
138 ioat_chan->pending = 0;
139 }
140 spin_unlock_bh(&ioat_chan->desc_lock);
141
142 if (append)
143 writeb(IOAT_CHANCMD_APPEND,
144 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
145
146 return cookie;
147}
148
81static struct ioat_desc_sw *ioat_dma_alloc_descriptor( 149static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
82 struct ioat_dma_chan *ioat_chan, 150 struct ioat_dma_chan *ioat_chan,
83 gfp_t flags) 151 gfp_t flags)
@@ -99,8 +167,13 @@ static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
99 } 167 }
100 168
101 memset(desc, 0, sizeof(*desc)); 169 memset(desc, 0, sizeof(*desc));
170 dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);
171 desc_sw->async_tx.tx_set_src = ioat_set_src;
172 desc_sw->async_tx.tx_set_dest = ioat_set_dest;
173 desc_sw->async_tx.tx_submit = ioat_tx_submit;
174 INIT_LIST_HEAD(&desc_sw->async_tx.tx_list);
102 desc_sw->hw = desc; 175 desc_sw->hw = desc;
103 desc_sw->phys = phys; 176 desc_sw->async_tx.phys = phys;
104 177
105 return desc_sw; 178 return desc_sw;
106} 179}
@@ -188,12 +261,14 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan)
188 list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { 261 list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
189 in_use_descs++; 262 in_use_descs++;
190 list_del(&desc->node); 263 list_del(&desc->node);
191 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); 264 pci_pool_free(ioat_device->dma_pool, desc->hw,
265 desc->async_tx.phys);
192 kfree(desc); 266 kfree(desc);
193 } 267 }
194 list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) { 268 list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
195 list_del(&desc->node); 269 list_del(&desc->node);
196 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); 270 pci_pool_free(ioat_device->dma_pool, desc->hw,
271 desc->async_tx.phys);
197 kfree(desc); 272 kfree(desc);
198 } 273 }
199 spin_unlock_bh(&ioat_chan->desc_lock); 274 spin_unlock_bh(&ioat_chan->desc_lock);
@@ -215,45 +290,25 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan)
215 writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); 290 writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
216} 291}
217 292
218/** 293static struct dma_async_tx_descriptor *
219 * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction 294ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en)
220 * @ioat_chan: IOAT DMA channel handle
221 * @dest: DMA destination address
222 * @src: DMA source address
223 * @len: transaction length in bytes
224 */
225
226static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
227 dma_addr_t dest,
228 dma_addr_t src,
229 size_t len)
230{ 295{
231 struct ioat_desc_sw *first; 296 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
232 struct ioat_desc_sw *prev; 297 struct ioat_desc_sw *first, *prev, *new;
233 struct ioat_desc_sw *new;
234 dma_cookie_t cookie;
235 LIST_HEAD(new_chain); 298 LIST_HEAD(new_chain);
236 u32 copy; 299 u32 copy;
237 size_t orig_len; 300 size_t orig_len;
238 dma_addr_t orig_src, orig_dst; 301 int desc_count = 0;
239 unsigned int desc_count = 0;
240 unsigned int append = 0;
241
242 if (!ioat_chan || !dest || !src)
243 return -EFAULT;
244 302
245 if (!len) 303 if (!len)
246 return ioat_chan->common.cookie; 304 return NULL;
247 305
248 orig_len = len; 306 orig_len = len;
249 orig_src = src;
250 orig_dst = dest;
251 307
252 first = NULL; 308 first = NULL;
253 prev = NULL; 309 prev = NULL;
254 310
255 spin_lock_bh(&ioat_chan->desc_lock); 311 spin_lock_bh(&ioat_chan->desc_lock);
256
257 while (len) { 312 while (len) {
258 if (!list_empty(&ioat_chan->free_desc)) { 313 if (!list_empty(&ioat_chan->free_desc)) {
259 new = to_ioat_desc(ioat_chan->free_desc.next); 314 new = to_ioat_desc(ioat_chan->free_desc.next);
@@ -270,140 +325,36 @@ static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
270 325
271 new->hw->size = copy; 326 new->hw->size = copy;
272 new->hw->ctl = 0; 327 new->hw->ctl = 0;
273 new->hw->src_addr = src; 328 new->async_tx.cookie = 0;
274 new->hw->dst_addr = dest; 329 new->async_tx.ack = 1;
275 new->cookie = 0;
276 330
277 /* chain together the physical address list for the HW */ 331 /* chain together the physical address list for the HW */
278 if (!first) 332 if (!first)
279 first = new; 333 first = new;
280 else 334 else
281 prev->hw->next = (u64) new->phys; 335 prev->hw->next = (u64) new->async_tx.phys;
282 336
283 prev = new; 337 prev = new;
284
285 len -= copy; 338 len -= copy;
286 dest += copy;
287 src += copy;
288
289 list_add_tail(&new->node, &new_chain); 339 list_add_tail(&new->node, &new_chain);
290 desc_count++; 340 desc_count++;
291 } 341 }
292 new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
293 new->hw->next = 0;
294 342
295 /* cookie incr and addition to used_list must be atomic */ 343 list_splice(&new_chain, &new->async_tx.tx_list);
296 344
297 cookie = ioat_chan->common.cookie; 345 new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
298 cookie++; 346 new->hw->next = 0;
299 if (cookie < 0) 347 new->tx_cnt = desc_count;
300 cookie = 1; 348 new->async_tx.ack = 0; /* client is in control of this ack */
301 ioat_chan->common.cookie = new->cookie = cookie; 349 new->async_tx.cookie = -EBUSY;
302 350
303 pci_unmap_addr_set(new, src, orig_src);
304 pci_unmap_addr_set(new, dst, orig_dst);
305 pci_unmap_len_set(new, src_len, orig_len); 351 pci_unmap_len_set(new, src_len, orig_len);
306 pci_unmap_len_set(new, dst_len, orig_len); 352 pci_unmap_len_set(new, dst_len, orig_len);
307
308 /* write address into NextDescriptor field of last desc in chain */
309 to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys;
310 list_splice_init(&new_chain, ioat_chan->used_desc.prev);
311
312 ioat_chan->pending += desc_count;
313 if (ioat_chan->pending >= 4) {
314 append = 1;
315 ioat_chan->pending = 0;
316 }
317
318 spin_unlock_bh(&ioat_chan->desc_lock); 353 spin_unlock_bh(&ioat_chan->desc_lock);
319 354
320 if (append) 355 return new ? &new->async_tx : NULL;
321 writeb(IOAT_CHANCMD_APPEND,
322 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
323 return cookie;
324} 356}
325 357
326/**
327 * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs
328 * @chan: IOAT DMA channel handle
329 * @dest: DMA destination address
330 * @src: DMA source address
331 * @len: transaction length in bytes
332 */
333
334static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan,
335 void *dest,
336 void *src,
337 size_t len)
338{
339 dma_addr_t dest_addr;
340 dma_addr_t src_addr;
341 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
342
343 dest_addr = pci_map_single(ioat_chan->device->pdev,
344 dest, len, PCI_DMA_FROMDEVICE);
345 src_addr = pci_map_single(ioat_chan->device->pdev,
346 src, len, PCI_DMA_TODEVICE);
347
348 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
349}
350
351/**
352 * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page
353 * @chan: IOAT DMA channel handle
354 * @page: pointer to the page to copy to
355 * @offset: offset into that page
356 * @src: DMA source address
357 * @len: transaction length in bytes
358 */
359
360static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan,
361 struct page *page,
362 unsigned int offset,
363 void *src,
364 size_t len)
365{
366 dma_addr_t dest_addr;
367 dma_addr_t src_addr;
368 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
369
370 dest_addr = pci_map_page(ioat_chan->device->pdev,
371 page, offset, len, PCI_DMA_FROMDEVICE);
372 src_addr = pci_map_single(ioat_chan->device->pdev,
373 src, len, PCI_DMA_TODEVICE);
374
375 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
376}
377
378/**
379 * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages
380 * @chan: IOAT DMA channel handle
381 * @dest_pg: pointer to the page to copy to
382 * @dest_off: offset into that page
383 * @src_pg: pointer to the page to copy from
384 * @src_off: offset into that page
385 * @len: transaction length in bytes. This is guaranteed not to make a copy
386 * across a page boundary.
387 */
388
389static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan,
390 struct page *dest_pg,
391 unsigned int dest_off,
392 struct page *src_pg,
393 unsigned int src_off,
394 size_t len)
395{
396 dma_addr_t dest_addr;
397 dma_addr_t src_addr;
398 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
399
400 dest_addr = pci_map_page(ioat_chan->device->pdev,
401 dest_pg, dest_off, len, PCI_DMA_FROMDEVICE);
402 src_addr = pci_map_page(ioat_chan->device->pdev,
403 src_pg, src_off, len, PCI_DMA_TODEVICE);
404
405 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
406}
407 358
408/** 359/**
409 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw 360 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw
@@ -465,8 +416,8 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
465 * exceeding xfercap, perhaps. If so, only the last one will 416 * exceeding xfercap, perhaps. If so, only the last one will
466 * have a cookie, and require unmapping. 417 * have a cookie, and require unmapping.
467 */ 418 */
468 if (desc->cookie) { 419 if (desc->async_tx.cookie) {
469 cookie = desc->cookie; 420 cookie = desc->async_tx.cookie;
470 421
471 /* yes we are unmapping both _page and _single alloc'd 422 /* yes we are unmapping both _page and _single alloc'd
472 regions with unmap_page. Is this *really* that bad? 423 regions with unmap_page. Is this *really* that bad?
@@ -481,14 +432,19 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
481 PCI_DMA_TODEVICE); 432 PCI_DMA_TODEVICE);
482 } 433 }
483 434
484 if (desc->phys != phys_complete) { 435 if (desc->async_tx.phys != phys_complete) {
485 /* a completed entry, but not the last, so cleanup */ 436 /* a completed entry, but not the last, so cleanup
486 list_del(&desc->node); 437 * if the client is done with the descriptor
487 list_add_tail(&desc->node, &chan->free_desc); 438 */
439 if (desc->async_tx.ack) {
440 list_del(&desc->node);
441 list_add_tail(&desc->node, &chan->free_desc);
442 } else
443 desc->async_tx.cookie = 0;
488 } else { 444 } else {
489 /* last used desc. Do not remove, so we can append from 445 /* last used desc. Do not remove, so we can append from
490 it, but don't look at it next time, either */ 446 it, but don't look at it next time, either */
491 desc->cookie = 0; 447 desc->async_tx.cookie = 0;
492 448
493 /* TODO check status bits? */ 449 /* TODO check status bits? */
494 break; 450 break;
@@ -504,6 +460,17 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
504 spin_unlock(&chan->cleanup_lock); 460 spin_unlock(&chan->cleanup_lock);
505} 461}
506 462
463static void ioat_dma_dependency_added(struct dma_chan *chan)
464{
465 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
466 spin_lock_bh(&ioat_chan->desc_lock);
467 if (ioat_chan->pending == 0) {
468 spin_unlock_bh(&ioat_chan->desc_lock);
469 ioat_dma_memcpy_cleanup(ioat_chan);
470 } else
471 spin_unlock_bh(&ioat_chan->desc_lock);
472}
473
507/** 474/**
508 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction 475 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
509 * @chan: IOAT DMA channel handle 476 * @chan: IOAT DMA channel handle
@@ -606,13 +573,14 @@ static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan)
606 573
607 desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; 574 desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
608 desc->hw->next = 0; 575 desc->hw->next = 0;
576 desc->async_tx.ack = 1;
609 577
610 list_add_tail(&desc->node, &ioat_chan->used_desc); 578 list_add_tail(&desc->node, &ioat_chan->used_desc);
611 spin_unlock_bh(&ioat_chan->desc_lock); 579 spin_unlock_bh(&ioat_chan->desc_lock);
612 580
613 writel(((u64) desc->phys) & 0x00000000FFFFFFFF, 581 writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
614 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW); 582 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW);
615 writel(((u64) desc->phys) >> 32, 583 writel(((u64) desc->async_tx.phys) >> 32,
616 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH); 584 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH);
617 585
618 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET); 586 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
@@ -629,6 +597,8 @@ static int ioat_self_test(struct ioat_device *device)
629 u8 *src; 597 u8 *src;
630 u8 *dest; 598 u8 *dest;
631 struct dma_chan *dma_chan; 599 struct dma_chan *dma_chan;
600 struct dma_async_tx_descriptor *tx;
601 dma_addr_t addr;
632 dma_cookie_t cookie; 602 dma_cookie_t cookie;
633 int err = 0; 603 int err = 0;
634 604
@@ -654,7 +624,15 @@ static int ioat_self_test(struct ioat_device *device)
654 goto out; 624 goto out;
655 } 625 }
656 626
657 cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE); 627 tx = ioat_dma_prep_memcpy(dma_chan, IOAT_TEST_SIZE, 0);
628 async_tx_ack(tx);
629 addr = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE,
630 DMA_TO_DEVICE);
631 ioat_set_src(addr, tx, 0);
632 addr = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE,
633 DMA_FROM_DEVICE);
634 ioat_set_dest(addr, tx, 0);
635 cookie = ioat_tx_submit(tx);
658 ioat_dma_memcpy_issue_pending(dma_chan); 636 ioat_dma_memcpy_issue_pending(dma_chan);
659 msleep(1); 637 msleep(1);
660 638
@@ -750,13 +728,14 @@ static int __devinit ioat_probe(struct pci_dev *pdev,
750 INIT_LIST_HEAD(&device->common.channels); 728 INIT_LIST_HEAD(&device->common.channels);
751 enumerate_dma_channels(device); 729 enumerate_dma_channels(device);
752 730
731 dma_cap_set(DMA_MEMCPY, device->common.cap_mask);
753 device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; 732 device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
754 device->common.device_free_chan_resources = ioat_dma_free_chan_resources; 733 device->common.device_free_chan_resources = ioat_dma_free_chan_resources;
755 device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf; 734 device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy;
756 device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg; 735 device->common.device_is_tx_complete = ioat_dma_is_complete;
757 device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg; 736 device->common.device_issue_pending = ioat_dma_memcpy_issue_pending;
758 device->common.device_memcpy_complete = ioat_dma_is_complete; 737 device->common.device_dependency_added = ioat_dma_dependency_added;
759 device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending; 738 device->common.dev = &pdev->dev;
760 printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n", 739 printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n",
761 device->common.chancnt); 740 device->common.chancnt);
762 741
diff --git a/drivers/dma/ioatdma.h b/drivers/dma/ioatdma.h
index 62b26a9be4c9..d3f69bb15fa0 100644
--- a/drivers/dma/ioatdma.h
+++ b/drivers/dma/ioatdma.h
@@ -105,21 +105,20 @@ struct ioat_dma_chan {
105/** 105/**
106 * struct ioat_desc_sw - wrapper around hardware descriptor 106 * struct ioat_desc_sw - wrapper around hardware descriptor
107 * @hw: hardware DMA descriptor 107 * @hw: hardware DMA descriptor
108 * @node: 108 * @node: this descriptor will either be on the free list,
109 * @cookie: 109 * or attached to a transaction list (async_tx.tx_list)
110 * @phys: 110 * @tx_cnt: number of descriptors required to complete the transaction
111 * @async_tx: the generic software descriptor for all engines
111 */ 112 */
112
113struct ioat_desc_sw { 113struct ioat_desc_sw {
114 struct ioat_dma_descriptor *hw; 114 struct ioat_dma_descriptor *hw;
115 struct list_head node; 115 struct list_head node;
116 dma_cookie_t cookie; 116 int tx_cnt;
117 dma_addr_t phys;
118 DECLARE_PCI_UNMAP_ADDR(src) 117 DECLARE_PCI_UNMAP_ADDR(src)
119 DECLARE_PCI_UNMAP_LEN(src_len) 118 DECLARE_PCI_UNMAP_LEN(src_len)
120 DECLARE_PCI_UNMAP_ADDR(dst) 119 DECLARE_PCI_UNMAP_ADDR(dst)
121 DECLARE_PCI_UNMAP_LEN(dst_len) 120 DECLARE_PCI_UNMAP_LEN(dst_len)
121 struct dma_async_tx_descriptor async_tx;
122}; 122};
123 123
124#endif /* IOATDMA_H */ 124#endif /* IOATDMA_H */
125
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index c94d8f1d62e5..3de1cf71031a 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -21,13 +21,12 @@
21#ifndef DMAENGINE_H 21#ifndef DMAENGINE_H
22#define DMAENGINE_H 22#define DMAENGINE_H
23 23
24#ifdef CONFIG_DMA_ENGINE
25
26#include <linux/device.h> 24#include <linux/device.h>
27#include <linux/uio.h> 25#include <linux/uio.h>
28#include <linux/kref.h> 26#include <linux/kref.h>
29#include <linux/completion.h> 27#include <linux/completion.h>
30#include <linux/rcupdate.h> 28#include <linux/rcupdate.h>
29#include <linux/dma-mapping.h>
31 30
32/** 31/**
33 * enum dma_event - resource PNP/power managment events 32 * enum dma_event - resource PNP/power managment events
@@ -65,6 +64,31 @@ enum dma_status {
65}; 64};
66 65
67/** 66/**
67 * enum dma_transaction_type - DMA transaction types/indexes
68 */
69enum dma_transaction_type {
70 DMA_MEMCPY,
71 DMA_XOR,
72 DMA_PQ_XOR,
73 DMA_DUAL_XOR,
74 DMA_PQ_UPDATE,
75 DMA_ZERO_SUM,
76 DMA_PQ_ZERO_SUM,
77 DMA_MEMSET,
78 DMA_MEMCPY_CRC32C,
79 DMA_INTERRUPT,
80};
81
82/* last transaction type for creation of the capabilities mask */
83#define DMA_TX_TYPE_END (DMA_INTERRUPT + 1)
84
85/**
86 * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t.
87 * See linux/cpumask.h
88 */
89typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t;
90
91/**
68 * struct dma_chan_percpu - the per-CPU part of struct dma_chan 92 * struct dma_chan_percpu - the per-CPU part of struct dma_chan
69 * @refcount: local_t used for open-coded "bigref" counting 93 * @refcount: local_t used for open-coded "bigref" counting
70 * @memcpy_count: transaction counter 94 * @memcpy_count: transaction counter
@@ -157,48 +181,106 @@ struct dma_client {
157 struct list_head global_node; 181 struct list_head global_node;
158}; 182};
159 183
184typedef void (*dma_async_tx_callback)(void *dma_async_param);
185/**
186 * struct dma_async_tx_descriptor - async transaction descriptor
187 * ---dma generic offload fields---
188 * @cookie: tracking cookie for this transaction, set to -EBUSY if
189 * this tx is sitting on a dependency list
190 * @ack: the descriptor can not be reused until the client acknowledges
191 * receipt, i.e. has has a chance to establish any dependency chains
192 * @phys: physical address of the descriptor
193 * @tx_list: driver common field for operations that require multiple
194 * descriptors
195 * @chan: target channel for this operation
196 * @tx_submit: set the prepared descriptor(s) to be executed by the engine
197 * @tx_set_dest: set a destination address in a hardware descriptor
198 * @tx_set_src: set a source address in a hardware descriptor
199 * @callback: routine to call after this operation is complete
200 * @callback_param: general parameter to pass to the callback routine
201 * ---async_tx api specific fields---
202 * @depend_list: at completion this list of transactions are submitted
203 * @depend_node: allow this transaction to be executed after another
204 * transaction has completed, possibly on another channel
205 * @parent: pointer to the next level up in the dependency chain
206 * @lock: protect the dependency list
207 */
208struct dma_async_tx_descriptor {
209 dma_cookie_t cookie;
210 int ack;
211 dma_addr_t phys;
212 struct list_head tx_list;
213 struct dma_chan *chan;
214 dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx);
215 void (*tx_set_dest)(dma_addr_t addr,
216 struct dma_async_tx_descriptor *tx, int index);
217 void (*tx_set_src)(dma_addr_t addr,
218 struct dma_async_tx_descriptor *tx, int index);
219 dma_async_tx_callback callback;
220 void *callback_param;
221 struct list_head depend_list;
222 struct list_head depend_node;
223 struct dma_async_tx_descriptor *parent;
224 spinlock_t lock;
225};
226
160/** 227/**
161 * struct dma_device - info on the entity supplying DMA services 228 * struct dma_device - info on the entity supplying DMA services
162 * @chancnt: how many DMA channels are supported 229 * @chancnt: how many DMA channels are supported
163 * @channels: the list of struct dma_chan 230 * @channels: the list of struct dma_chan
164 * @global_node: list_head for global dma_device_list 231 * @global_node: list_head for global dma_device_list
232 * @cap_mask: one or more dma_capability flags
233 * @max_xor: maximum number of xor sources, 0 if no capability
165 * @refcount: reference count 234 * @refcount: reference count
166 * @done: IO completion struct 235 * @done: IO completion struct
167 * @dev_id: unique device ID 236 * @dev_id: unique device ID
237 * @dev: struct device reference for dma mapping api
168 * @device_alloc_chan_resources: allocate resources and return the 238 * @device_alloc_chan_resources: allocate resources and return the
169 * number of allocated descriptors 239 * number of allocated descriptors
170 * @device_free_chan_resources: release DMA channel's resources 240 * @device_free_chan_resources: release DMA channel's resources
171 * @device_memcpy_buf_to_buf: memcpy buf pointer to buf pointer 241 * @device_prep_dma_memcpy: prepares a memcpy operation
172 * @device_memcpy_buf_to_pg: memcpy buf pointer to struct page 242 * @device_prep_dma_xor: prepares a xor operation
173 * @device_memcpy_pg_to_pg: memcpy struct page/offset to struct page/offset 243 * @device_prep_dma_zero_sum: prepares a zero_sum operation
174 * @device_memcpy_complete: poll the status of an IOAT DMA transaction 244 * @device_prep_dma_memset: prepares a memset operation
175 * @device_memcpy_issue_pending: push appended descriptors to hardware 245 * @device_prep_dma_interrupt: prepares an end of chain interrupt operation
246 * @device_dependency_added: async_tx notifies the channel about new deps
247 * @device_issue_pending: push pending transactions to hardware
176 */ 248 */
177struct dma_device { 249struct dma_device {
178 250
179 unsigned int chancnt; 251 unsigned int chancnt;
180 struct list_head channels; 252 struct list_head channels;
181 struct list_head global_node; 253 struct list_head global_node;
254 dma_cap_mask_t cap_mask;
255 int max_xor;
182 256
183 struct kref refcount; 257 struct kref refcount;
184 struct completion done; 258 struct completion done;
185 259
186 int dev_id; 260 int dev_id;
261 struct device *dev;
187 262
188 int (*device_alloc_chan_resources)(struct dma_chan *chan); 263 int (*device_alloc_chan_resources)(struct dma_chan *chan);
189 void (*device_free_chan_resources)(struct dma_chan *chan); 264 void (*device_free_chan_resources)(struct dma_chan *chan);
190 dma_cookie_t (*device_memcpy_buf_to_buf)(struct dma_chan *chan, 265
191 void *dest, void *src, size_t len); 266 struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)(
192 dma_cookie_t (*device_memcpy_buf_to_pg)(struct dma_chan *chan, 267 struct dma_chan *chan, size_t len, int int_en);
193 struct page *page, unsigned int offset, void *kdata, 268 struct dma_async_tx_descriptor *(*device_prep_dma_xor)(
194 size_t len); 269 struct dma_chan *chan, unsigned int src_cnt, size_t len,
195 dma_cookie_t (*device_memcpy_pg_to_pg)(struct dma_chan *chan, 270 int int_en);
196 struct page *dest_pg, unsigned int dest_off, 271 struct dma_async_tx_descriptor *(*device_prep_dma_zero_sum)(
197 struct page *src_pg, unsigned int src_off, size_t len); 272 struct dma_chan *chan, unsigned int src_cnt, size_t len,
198 enum dma_status (*device_memcpy_complete)(struct dma_chan *chan, 273 u32 *result, int int_en);
274 struct dma_async_tx_descriptor *(*device_prep_dma_memset)(
275 struct dma_chan *chan, int value, size_t len, int int_en);
276 struct dma_async_tx_descriptor *(*device_prep_dma_interrupt)(
277 struct dma_chan *chan);
278
279 void (*device_dependency_added)(struct dma_chan *chan);
280 enum dma_status (*device_is_tx_complete)(struct dma_chan *chan,
199 dma_cookie_t cookie, dma_cookie_t *last, 281 dma_cookie_t cookie, dma_cookie_t *last,
200 dma_cookie_t *used); 282 dma_cookie_t *used);
201 void (*device_memcpy_issue_pending)(struct dma_chan *chan); 283 void (*device_issue_pending)(struct dma_chan *chan);
202}; 284};
203 285
204/* --- public DMA engine API --- */ 286/* --- public DMA engine API --- */
@@ -207,96 +289,72 @@ struct dma_client *dma_async_client_register(dma_event_callback event_callback);
207void dma_async_client_unregister(struct dma_client *client); 289void dma_async_client_unregister(struct dma_client *client);
208void dma_async_client_chan_request(struct dma_client *client, 290void dma_async_client_chan_request(struct dma_client *client,
209 unsigned int number); 291 unsigned int number);
292dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan,
293 void *dest, void *src, size_t len);
294dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan,
295 struct page *page, unsigned int offset, void *kdata, size_t len);
296dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan,
297 struct page *dest_pg, unsigned int dest_off, struct page *src_pg,
298 unsigned int src_off, size_t len);
299void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
300 struct dma_chan *chan);
210 301
211/**
212 * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
213 * @chan: DMA channel to offload copy to
214 * @dest: destination address (virtual)
215 * @src: source address (virtual)
216 * @len: length
217 *
218 * Both @dest and @src must be mappable to a bus address according to the
219 * DMA mapping API rules for streaming mappings.
220 * Both @dest and @src must stay memory resident (kernel memory or locked
221 * user space pages).
222 */
223static inline dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan,
224 void *dest, void *src, size_t len)
225{
226 int cpu = get_cpu();
227 per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
228 per_cpu_ptr(chan->local, cpu)->memcpy_count++;
229 put_cpu();
230 302
231 return chan->device->device_memcpy_buf_to_buf(chan, dest, src, len); 303static inline void
304async_tx_ack(struct dma_async_tx_descriptor *tx)
305{
306 tx->ack = 1;
232} 307}
233 308
234/** 309#define first_dma_cap(mask) __first_dma_cap(&(mask))
235 * dma_async_memcpy_buf_to_pg - offloaded copy from address to page 310static inline int __first_dma_cap(const dma_cap_mask_t *srcp)
236 * @chan: DMA channel to offload copy to
237 * @page: destination page
238 * @offset: offset in page to copy to
239 * @kdata: source address (virtual)
240 * @len: length
241 *
242 * Both @page/@offset and @kdata must be mappable to a bus address according
243 * to the DMA mapping API rules for streaming mappings.
244 * Both @page/@offset and @kdata must stay memory resident (kernel memory or
245 * locked user space pages)
246 */
247static inline dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan,
248 struct page *page, unsigned int offset, void *kdata, size_t len)
249{ 311{
250 int cpu = get_cpu(); 312 return min_t(int, DMA_TX_TYPE_END,
251 per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; 313 find_first_bit(srcp->bits, DMA_TX_TYPE_END));
252 per_cpu_ptr(chan->local, cpu)->memcpy_count++; 314}
253 put_cpu();
254 315
255 return chan->device->device_memcpy_buf_to_pg(chan, page, offset, 316#define next_dma_cap(n, mask) __next_dma_cap((n), &(mask))
256 kdata, len); 317static inline int __next_dma_cap(int n, const dma_cap_mask_t *srcp)
318{
319 return min_t(int, DMA_TX_TYPE_END,
320 find_next_bit(srcp->bits, DMA_TX_TYPE_END, n+1));
257} 321}
258 322
259/** 323#define dma_cap_set(tx, mask) __dma_cap_set((tx), &(mask))
260 * dma_async_memcpy_pg_to_pg - offloaded copy from page to page 324static inline void
261 * @chan: DMA channel to offload copy to 325__dma_cap_set(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp)
262 * @dest_pg: destination page
263 * @dest_off: offset in page to copy to
264 * @src_pg: source page
265 * @src_off: offset in page to copy from
266 * @len: length
267 *
268 * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus
269 * address according to the DMA mapping API rules for streaming mappings.
270 * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident
271 * (kernel memory or locked user space pages).
272 */
273static inline dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan,
274 struct page *dest_pg, unsigned int dest_off, struct page *src_pg,
275 unsigned int src_off, size_t len)
276{ 326{
277 int cpu = get_cpu(); 327 set_bit(tx_type, dstp->bits);
278 per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; 328}
279 per_cpu_ptr(chan->local, cpu)->memcpy_count++;
280 put_cpu();
281 329
282 return chan->device->device_memcpy_pg_to_pg(chan, dest_pg, dest_off, 330#define dma_has_cap(tx, mask) __dma_has_cap((tx), &(mask))
283 src_pg, src_off, len); 331static inline int
332__dma_has_cap(enum dma_transaction_type tx_type, dma_cap_mask_t *srcp)
333{
334 return test_bit(tx_type, srcp->bits);
284} 335}
285 336
337#define for_each_dma_cap_mask(cap, mask) \
338 for ((cap) = first_dma_cap(mask); \
339 (cap) < DMA_TX_TYPE_END; \
340 (cap) = next_dma_cap((cap), (mask)))
341
286/** 342/**
287 * dma_async_memcpy_issue_pending - flush pending copies to HW 343 * dma_async_issue_pending - flush pending transactions to HW
288 * @chan: target DMA channel 344 * @chan: target DMA channel
289 * 345 *
290 * This allows drivers to push copies to HW in batches, 346 * This allows drivers to push copies to HW in batches,
291 * reducing MMIO writes where possible. 347 * reducing MMIO writes where possible.
292 */ 348 */
293static inline void dma_async_memcpy_issue_pending(struct dma_chan *chan) 349static inline void dma_async_issue_pending(struct dma_chan *chan)
294{ 350{
295 return chan->device->device_memcpy_issue_pending(chan); 351 return chan->device->device_issue_pending(chan);
296} 352}
297 353
354#define dma_async_memcpy_issue_pending(chan) dma_async_issue_pending(chan)
355
298/** 356/**
299 * dma_async_memcpy_complete - poll for transaction completion 357 * dma_async_is_tx_complete - poll for transaction completion
300 * @chan: DMA channel 358 * @chan: DMA channel
301 * @cookie: transaction identifier to check status of 359 * @cookie: transaction identifier to check status of
302 * @last: returns last completed cookie, can be NULL 360 * @last: returns last completed cookie, can be NULL
@@ -306,12 +364,15 @@ static inline void dma_async_memcpy_issue_pending(struct dma_chan *chan)
306 * internal state and can be used with dma_async_is_complete() to check 364 * internal state and can be used with dma_async_is_complete() to check
307 * the status of multiple cookies without re-checking hardware state. 365 * the status of multiple cookies without re-checking hardware state.
308 */ 366 */
309static inline enum dma_status dma_async_memcpy_complete(struct dma_chan *chan, 367static inline enum dma_status dma_async_is_tx_complete(struct dma_chan *chan,
310 dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used) 368 dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used)
311{ 369{
312 return chan->device->device_memcpy_complete(chan, cookie, last, used); 370 return chan->device->device_is_tx_complete(chan, cookie, last, used);
313} 371}
314 372
373#define dma_async_memcpy_complete(chan, cookie, last, used)\
374 dma_async_is_tx_complete(chan, cookie, last, used)
375
315/** 376/**
316 * dma_async_is_complete - test a cookie against chan state 377 * dma_async_is_complete - test a cookie against chan state
317 * @cookie: transaction identifier to test status of 378 * @cookie: transaction identifier to test status of
@@ -334,6 +395,7 @@ static inline enum dma_status dma_async_is_complete(dma_cookie_t cookie,
334 return DMA_IN_PROGRESS; 395 return DMA_IN_PROGRESS;
335} 396}
336 397
398enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie);
337 399
338/* --- DMA device --- */ 400/* --- DMA device --- */
339 401
@@ -362,5 +424,4 @@ dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
362 struct dma_pinned_list *pinned_list, struct page *page, 424 struct dma_pinned_list *pinned_list, struct page *page,
363 unsigned int offset, size_t len); 425 unsigned int offset, size_t len);
364 426
365#endif /* CONFIG_DMA_ENGINE */
366#endif /* DMAENGINE_H */ 427#endif /* DMAENGINE_H */