aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/dma/ioat/Makefile2
-rw-r--r--drivers/dma/ioat/dma.c874
-rw-r--r--drivers/dma/ioat/dma.h50
-rw-r--r--drivers/dma/ioat/dma_v2.c750
-rw-r--r--drivers/dma/ioat/dma_v2.h131
-rw-r--r--drivers/dma/ioat/pci.c1
6 files changed, 1122 insertions, 686 deletions
diff --git a/drivers/dma/ioat/Makefile b/drivers/dma/ioat/Makefile
index 2ce3d3a4270b..205a639e84df 100644
--- a/drivers/dma/ioat/Makefile
+++ b/drivers/dma/ioat/Makefile
@@ -1,2 +1,2 @@
1obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o 1obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
2ioatdma-objs := pci.o dma.o dca.o 2ioatdma-objs := pci.o dma.o dma_v2.o dca.o
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 2e81e0c76e61..64b4d75059aa 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -38,28 +38,14 @@
38#include "registers.h" 38#include "registers.h"
39#include "hw.h" 39#include "hw.h"
40 40
41static int ioat_pending_level = 4; 41int ioat_pending_level = 4;
42module_param(ioat_pending_level, int, 0644); 42module_param(ioat_pending_level, int, 0644);
43MODULE_PARM_DESC(ioat_pending_level, 43MODULE_PARM_DESC(ioat_pending_level,
44 "high-water mark for pushing ioat descriptors (default: 4)"); 44 "high-water mark for pushing ioat descriptors (default: 4)");
45 45
46static void ioat_dma_chan_reset_part2(struct work_struct *work);
47static void ioat_dma_chan_watchdog(struct work_struct *work);
48
49/* internal functions */ 46/* internal functions */
50static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat); 47static void ioat1_cleanup(struct ioat_dma_chan *ioat);
51static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat); 48static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat);
52
53static struct ioat_desc_sw *
54ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat);
55static struct ioat_desc_sw *
56ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat);
57
58static inline struct ioat_chan_common *
59ioat_chan_by_index(struct ioatdma_device *device, int index)
60{
61 return device->idx[index];
62}
63 49
64/** 50/**
65 * ioat_dma_do_interrupt - handler used for single vector interrupt mode 51 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
@@ -108,18 +94,38 @@ static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
108 return IRQ_HANDLED; 94 return IRQ_HANDLED;
109} 95}
110 96
111static void ioat_dma_cleanup_tasklet(unsigned long data); 97static void ioat1_cleanup_tasklet(unsigned long data);
98
99/* common channel initialization */
100void ioat_init_channel(struct ioatdma_device *device,
101 struct ioat_chan_common *chan, int idx,
102 work_func_t work_fn, void (*tasklet)(unsigned long),
103 unsigned long tasklet_data)
104{
105 struct dma_device *dma = &device->common;
106
107 chan->device = device;
108 chan->reg_base = device->reg_base + (0x80 * (idx + 1));
109 INIT_DELAYED_WORK(&chan->work, work_fn);
110 spin_lock_init(&chan->cleanup_lock);
111 chan->common.device = dma;
112 list_add_tail(&chan->common.device_node, &dma->channels);
113 device->idx[idx] = chan;
114 tasklet_init(&chan->cleanup_task, tasklet, tasklet_data);
115 tasklet_disable(&chan->cleanup_task);
116}
117
118static void ioat1_reset_part2(struct work_struct *work);
112 119
113/** 120/**
114 * ioat_dma_enumerate_channels - find and initialize the device's channels 121 * ioat1_dma_enumerate_channels - find and initialize the device's channels
115 * @device: the device to be enumerated 122 * @device: the device to be enumerated
116 */ 123 */
117static int ioat_dma_enumerate_channels(struct ioatdma_device *device) 124static int ioat1_enumerate_channels(struct ioatdma_device *device)
118{ 125{
119 u8 xfercap_scale; 126 u8 xfercap_scale;
120 u32 xfercap; 127 u32 xfercap;
121 int i; 128 int i;
122 struct ioat_chan_common *chan;
123 struct ioat_dma_chan *ioat; 129 struct ioat_dma_chan *ioat;
124 struct device *dev = &device->pdev->dev; 130 struct device *dev = &device->pdev->dev;
125 struct dma_device *dma = &device->common; 131 struct dma_device *dma = &device->common;
@@ -135,31 +141,20 @@ static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
135#endif 141#endif
136 for (i = 0; i < dma->chancnt; i++) { 142 for (i = 0; i < dma->chancnt; i++) {
137 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL); 143 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
138 if (!ioat) { 144 if (!ioat)
139 dma->chancnt = i;
140 break; 145 break;
141 }
142 146
143 chan = &ioat->base; 147 ioat_init_channel(device, &ioat->base, i,
144 chan->device = device; 148 ioat1_reset_part2,
145 chan->reg_base = device->reg_base + (0x80 * (i + 1)); 149 ioat1_cleanup_tasklet,
150 (unsigned long) ioat);
146 ioat->xfercap = xfercap; 151 ioat->xfercap = xfercap;
147 ioat->desccount = 0;
148 INIT_DELAYED_WORK(&chan->work, ioat_dma_chan_reset_part2);
149 spin_lock_init(&chan->cleanup_lock);
150 spin_lock_init(&ioat->desc_lock); 152 spin_lock_init(&ioat->desc_lock);
151 INIT_LIST_HEAD(&ioat->free_desc); 153 INIT_LIST_HEAD(&ioat->free_desc);
152 INIT_LIST_HEAD(&ioat->used_desc); 154 INIT_LIST_HEAD(&ioat->used_desc);
153 /* This should be made common somewhere in dmaengine.c */
154 chan->common.device = &device->common;
155 list_add_tail(&chan->common.device_node, &dma->channels);
156 device->idx[i] = chan;
157 tasklet_init(&chan->cleanup_task,
158 ioat_dma_cleanup_tasklet,
159 (unsigned long) ioat);
160 tasklet_disable(&chan->cleanup_task);
161 } 155 }
162 return dma->chancnt; 156 dma->chancnt = i;
157 return i;
163} 158}
164 159
165/** 160/**
@@ -187,35 +182,16 @@ static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
187 } 182 }
188} 183}
189 184
190static inline void
191__ioat2_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
192{
193 void __iomem *reg_base = ioat->base.reg_base;
194
195 ioat->pending = 0;
196 writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
197}
198
199static void ioat2_dma_memcpy_issue_pending(struct dma_chan *chan)
200{
201 struct ioat_dma_chan *ioat = to_ioat_chan(chan);
202
203 if (ioat->pending > 0) {
204 spin_lock_bh(&ioat->desc_lock);
205 __ioat2_dma_memcpy_issue_pending(ioat);
206 spin_unlock_bh(&ioat->desc_lock);
207 }
208}
209
210
211/** 185/**
212 * ioat_dma_chan_reset_part2 - reinit the channel after a reset 186 * ioat1_reset_part2 - reinit the channel after a reset
213 */ 187 */
214static void ioat_dma_chan_reset_part2(struct work_struct *work) 188static void ioat1_reset_part2(struct work_struct *work)
215{ 189{
216 struct ioat_chan_common *chan; 190 struct ioat_chan_common *chan;
217 struct ioat_dma_chan *ioat; 191 struct ioat_dma_chan *ioat;
218 struct ioat_desc_sw *desc; 192 struct ioat_desc_sw *desc;
193 int dmacount;
194 bool start_null = false;
219 195
220 chan = container_of(work, struct ioat_chan_common, work.work); 196 chan = container_of(work, struct ioat_chan_common, work.work);
221 ioat = container_of(chan, struct ioat_dma_chan, base); 197 ioat = container_of(chan, struct ioat_dma_chan, base);
@@ -226,26 +202,22 @@ static void ioat_dma_chan_reset_part2(struct work_struct *work)
226 chan->completion_virt->high = 0; 202 chan->completion_virt->high = 0;
227 ioat->pending = 0; 203 ioat->pending = 0;
228 204
229 /* 205 /* count the descriptors waiting */
230 * count the descriptors waiting, and be sure to do it 206 dmacount = 0;
231 * right for both the CB1 line and the CB2 ring
232 */
233 ioat->dmacount = 0;
234 if (ioat->used_desc.prev) { 207 if (ioat->used_desc.prev) {
235 desc = to_ioat_desc(ioat->used_desc.prev); 208 desc = to_ioat_desc(ioat->used_desc.prev);
236 do { 209 do {
237 ioat->dmacount++; 210 dmacount++;
238 desc = to_ioat_desc(desc->node.next); 211 desc = to_ioat_desc(desc->node.next);
239 } while (&desc->node != ioat->used_desc.next); 212 } while (&desc->node != ioat->used_desc.next);
240 } 213 }
241 214
242 /* 215 if (dmacount) {
243 * write the new starting descriptor address 216 /*
244 * this puts channel engine into ARMED state 217 * write the new starting descriptor address
245 */ 218 * this puts channel engine into ARMED state
246 desc = to_ioat_desc(ioat->used_desc.prev); 219 */
247 switch (chan->device->version) { 220 desc = to_ioat_desc(ioat->used_desc.prev);
248 case IOAT_VER_1_2:
249 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF, 221 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
250 chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW); 222 chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
251 writel(((u64) desc->txd.phys) >> 32, 223 writel(((u64) desc->txd.phys) >> 32,
@@ -253,32 +225,24 @@ static void ioat_dma_chan_reset_part2(struct work_struct *work)
253 225
254 writeb(IOAT_CHANCMD_START, chan->reg_base 226 writeb(IOAT_CHANCMD_START, chan->reg_base
255 + IOAT_CHANCMD_OFFSET(chan->device->version)); 227 + IOAT_CHANCMD_OFFSET(chan->device->version));
256 break; 228 } else
257 case IOAT_VER_2_0: 229 start_null = true;
258 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF, 230 spin_unlock_bh(&ioat->desc_lock);
259 chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW); 231 spin_unlock_bh(&chan->cleanup_lock);
260 writel(((u64) desc->txd.phys) >> 32,
261 chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
262
263 /* tell the engine to go with what's left to be done */
264 writew(ioat->dmacount,
265 chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
266 232
267 break;
268 }
269 dev_err(to_dev(chan), 233 dev_err(to_dev(chan),
270 "chan%d reset - %d descs waiting, %d total desc\n", 234 "chan%d reset - %d descs waiting, %d total desc\n",
271 chan_num(chan), ioat->dmacount, ioat->desccount); 235 chan_num(chan), dmacount, ioat->desccount);
272 236
273 spin_unlock_bh(&ioat->desc_lock); 237 if (start_null)
274 spin_unlock_bh(&chan->cleanup_lock); 238 ioat1_dma_start_null_desc(ioat);
275} 239}
276 240
277/** 241/**
278 * ioat_dma_reset_channel - restart a channel 242 * ioat1_reset_channel - restart a channel
279 * @ioat: IOAT DMA channel handle 243 * @ioat: IOAT DMA channel handle
280 */ 244 */
281static void ioat_dma_reset_channel(struct ioat_dma_chan *ioat) 245static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
282{ 246{
283 struct ioat_chan_common *chan = &ioat->base; 247 struct ioat_chan_common *chan = &ioat->base;
284 void __iomem *reg_base = chan->reg_base; 248 void __iomem *reg_base = chan->reg_base;
@@ -316,9 +280,9 @@ static void ioat_dma_reset_channel(struct ioat_dma_chan *ioat)
316} 280}
317 281
318/** 282/**
319 * ioat_dma_chan_watchdog - watch for stuck channels 283 * ioat1_chan_watchdog - watch for stuck channels
320 */ 284 */
321static void ioat_dma_chan_watchdog(struct work_struct *work) 285static void ioat1_chan_watchdog(struct work_struct *work)
322{ 286{
323 struct ioatdma_device *device = 287 struct ioatdma_device *device =
324 container_of(work, struct ioatdma_device, work.work); 288 container_of(work, struct ioatdma_device, work.work);
@@ -339,16 +303,15 @@ static void ioat_dma_chan_watchdog(struct work_struct *work)
339 chan = ioat_chan_by_index(device, i); 303 chan = ioat_chan_by_index(device, i);
340 ioat = container_of(chan, struct ioat_dma_chan, base); 304 ioat = container_of(chan, struct ioat_dma_chan, base);
341 305
342 if (chan->device->version == IOAT_VER_1_2 306 if (/* have we started processing anything yet */
343 /* have we started processing anything yet */ 307 chan->last_completion
344 && chan->last_completion 308 /* have we completed any since last watchdog cycle? */
345 /* have we completed any since last watchdog cycle? */
346 && (chan->last_completion == chan->watchdog_completion) 309 && (chan->last_completion == chan->watchdog_completion)
347 /* has TCP stuck on one cookie since last watchdog? */ 310 /* has TCP stuck on one cookie since last watchdog? */
348 && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie) 311 && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie)
349 && (chan->watchdog_tcp_cookie != chan->completed_cookie) 312 && (chan->watchdog_tcp_cookie != chan->completed_cookie)
350 /* is there something in the chain to be processed? */ 313 /* is there something in the chain to be processed? */
351 /* CB1 chain always has at least the last one processed */ 314 /* CB1 chain always has at least the last one processed */
352 && (ioat->used_desc.prev != ioat->used_desc.next) 315 && (ioat->used_desc.prev != ioat->used_desc.next)
353 && ioat->pending == 0) { 316 && ioat->pending == 0) {
354 317
@@ -387,34 +350,15 @@ static void ioat_dma_chan_watchdog(struct work_struct *work)
387 chan->completion_virt->low = completion_hw.low; 350 chan->completion_virt->low = completion_hw.low;
388 chan->completion_virt->high = completion_hw.high; 351 chan->completion_virt->high = completion_hw.high;
389 } else { 352 } else {
390 ioat_dma_reset_channel(ioat); 353 ioat1_reset_channel(ioat);
391 chan->watchdog_completion = 0; 354 chan->watchdog_completion = 0;
392 chan->last_compl_desc_addr_hw = 0; 355 chan->last_compl_desc_addr_hw = 0;
393 } 356 }
394
395 /*
396 * for version 2.0 if there are descriptors yet to be processed
397 * and the last completed hasn't changed since the last watchdog
398 * if they haven't hit the pending level
399 * issue the pending to push them through
400 * else
401 * try resetting the channel
402 */
403 } else if (chan->device->version == IOAT_VER_2_0
404 && ioat->used_desc.prev
405 && chan->last_completion
406 && chan->last_completion == chan->watchdog_completion) {
407
408 if (ioat->pending < ioat_pending_level)
409 ioat2_dma_memcpy_issue_pending(&chan->common);
410 else {
411 ioat_dma_reset_channel(ioat);
412 chan->watchdog_completion = 0;
413 }
414 } else { 357 } else {
415 chan->last_compl_desc_addr_hw = 0; 358 chan->last_compl_desc_addr_hw = 0;
416 chan->watchdog_completion = chan->last_completion; 359 chan->watchdog_completion = chan->last_completion;
417 } 360 }
361
418 chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie; 362 chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
419 } 363 }
420 364
@@ -447,7 +391,6 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
447 chain_tail->hw->next = first->txd.phys; 391 chain_tail->hw->next = first->txd.phys;
448 list_splice_tail_init(&tx->tx_list, &ioat->used_desc); 392 list_splice_tail_init(&tx->tx_list, &ioat->used_desc);
449 393
450 ioat->dmacount += desc->tx_cnt;
451 ioat->pending += desc->tx_cnt; 394 ioat->pending += desc->tx_cnt;
452 if (ioat->pending >= ioat_pending_level) 395 if (ioat->pending >= ioat_pending_level)
453 __ioat1_dma_memcpy_issue_pending(ioat); 396 __ioat1_dma_memcpy_issue_pending(ioat);
@@ -456,92 +399,6 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
456 return cookie; 399 return cookie;
457} 400}
458 401
459static dma_cookie_t ioat2_tx_submit(struct dma_async_tx_descriptor *tx)
460{
461 struct ioat_dma_chan *ioat = to_ioat_chan(tx->chan);
462 struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
463 struct ioat_desc_sw *new;
464 struct ioat_dma_descriptor *hw;
465 dma_cookie_t cookie;
466 u32 copy;
467 size_t len;
468 dma_addr_t src, dst;
469 unsigned long orig_flags;
470 unsigned int desc_count = 0;
471
472 /* src and dest and len are stored in the initial descriptor */
473 len = first->len;
474 src = first->src;
475 dst = first->dst;
476 orig_flags = first->txd.flags;
477 new = first;
478
479 /*
480 * ioat->desc_lock is still in force in version 2 path
481 * it gets unlocked at end of this function
482 */
483 do {
484 copy = min_t(size_t, len, ioat->xfercap);
485
486 async_tx_ack(&new->txd);
487
488 hw = new->hw;
489 hw->size = copy;
490 hw->ctl = 0;
491 hw->src_addr = src;
492 hw->dst_addr = dst;
493
494 len -= copy;
495 dst += copy;
496 src += copy;
497 desc_count++;
498 } while (len && (new = ioat2_dma_get_next_descriptor(ioat)));
499
500 if (!new) {
501 dev_err(to_dev(&ioat->base), "tx submit failed\n");
502 spin_unlock_bh(&ioat->desc_lock);
503 return -ENOMEM;
504 }
505
506 hw->ctl_f.compl_write = 1;
507 if (first->txd.callback) {
508 hw->ctl_f.int_en = 1;
509 if (first != new) {
510 /* move callback into to last desc */
511 new->txd.callback = first->txd.callback;
512 new->txd.callback_param
513 = first->txd.callback_param;
514 first->txd.callback = NULL;
515 first->txd.callback_param = NULL;
516 }
517 }
518
519 new->tx_cnt = desc_count;
520 new->txd.flags = orig_flags; /* client is in control of this ack */
521
522 /* store the original values for use in later cleanup */
523 if (new != first) {
524 new->src = first->src;
525 new->dst = first->dst;
526 new->len = first->len;
527 }
528
529 /* cookie incr and addition to used_list must be atomic */
530 cookie = ioat->base.common.cookie;
531 cookie++;
532 if (cookie < 0)
533 cookie = 1;
534 ioat->base.common.cookie = new->txd.cookie = cookie;
535
536 ioat->dmacount += desc_count;
537 ioat->pending += desc_count;
538 if (ioat->pending >= ioat_pending_level)
539 __ioat2_dma_memcpy_issue_pending(ioat);
540 spin_unlock_bh(&ioat->desc_lock);
541
542 return cookie;
543}
544
545/** 402/**
546 * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair 403 * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
547 * @ioat: the channel supplying the memory pool for the descriptors 404 * @ioat: the channel supplying the memory pool for the descriptors
@@ -567,17 +424,9 @@ ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
567 } 424 }
568 425
569 memset(desc, 0, sizeof(*desc)); 426 memset(desc, 0, sizeof(*desc));
570 dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
571 switch (ioatdma_device->version) {
572 case IOAT_VER_1_2:
573 desc_sw->txd.tx_submit = ioat1_tx_submit;
574 break;
575 case IOAT_VER_2_0:
576 case IOAT_VER_3_0:
577 desc_sw->txd.tx_submit = ioat2_tx_submit;
578 break;
579 }
580 427
428 dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
429 desc_sw->txd.tx_submit = ioat1_tx_submit;
581 desc_sw->hw = desc; 430 desc_sw->hw = desc;
582 desc_sw->txd.phys = phys; 431 desc_sw->txd.phys = phys;
583 432
@@ -587,39 +436,12 @@ ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
587static int ioat_initial_desc_count = 256; 436static int ioat_initial_desc_count = 256;
588module_param(ioat_initial_desc_count, int, 0644); 437module_param(ioat_initial_desc_count, int, 0644);
589MODULE_PARM_DESC(ioat_initial_desc_count, 438MODULE_PARM_DESC(ioat_initial_desc_count,
590 "initial descriptors per channel (default: 256)"); 439 "ioat1: initial descriptors per channel (default: 256)");
591
592/**
593 * ioat2_dma_massage_chan_desc - link the descriptors into a circle
594 * @ioat: the channel to be massaged
595 */
596static void ioat2_dma_massage_chan_desc(struct ioat_dma_chan *ioat)
597{
598 struct ioat_desc_sw *desc, *_desc;
599
600 /* setup used_desc */
601 ioat->used_desc.next = ioat->free_desc.next;
602 ioat->used_desc.prev = NULL;
603
604 /* pull free_desc out of the circle so that every node is a hw
605 * descriptor, but leave it pointing to the list
606 */
607 ioat->free_desc.prev->next = ioat->free_desc.next;
608 ioat->free_desc.next->prev = ioat->free_desc.prev;
609
610 /* circle link the hw descriptors */
611 desc = to_ioat_desc(ioat->free_desc.next);
612 desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
613 list_for_each_entry_safe(desc, _desc, ioat->free_desc.next, node) {
614 desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
615 }
616}
617
618/** 440/**
619 * ioat_dma_alloc_chan_resources - returns the number of allocated descriptors 441 * ioat1_dma_alloc_chan_resources - returns the number of allocated descriptors
620 * @chan: the channel to be filled out 442 * @chan: the channel to be filled out
621 */ 443 */
622static int ioat_dma_alloc_chan_resources(struct dma_chan *c) 444static int ioat1_dma_alloc_chan_resources(struct dma_chan *c)
623{ 445{
624 struct ioat_dma_chan *ioat = to_ioat_chan(c); 446 struct ioat_dma_chan *ioat = to_ioat_chan(c);
625 struct ioat_chan_common *chan = &ioat->base; 447 struct ioat_chan_common *chan = &ioat->base;
@@ -657,8 +479,6 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *c)
657 spin_lock_bh(&ioat->desc_lock); 479 spin_lock_bh(&ioat->desc_lock);
658 ioat->desccount = i; 480 ioat->desccount = i;
659 list_splice(&tmp_list, &ioat->free_desc); 481 list_splice(&tmp_list, &ioat->free_desc);
660 if (chan->device->version != IOAT_VER_1_2)
661 ioat2_dma_massage_chan_desc(ioat);
662 spin_unlock_bh(&ioat->desc_lock); 482 spin_unlock_bh(&ioat->desc_lock);
663 483
664 /* allocate a completion writeback area */ 484 /* allocate a completion writeback area */
@@ -674,15 +494,15 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *c)
674 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); 494 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
675 495
676 tasklet_enable(&chan->cleanup_task); 496 tasklet_enable(&chan->cleanup_task);
677 ioat_dma_start_null_desc(ioat); /* give chain to dma device */ 497 ioat1_dma_start_null_desc(ioat); /* give chain to dma device */
678 return ioat->desccount; 498 return ioat->desccount;
679} 499}
680 500
681/** 501/**
682 * ioat_dma_free_chan_resources - release all the descriptors 502 * ioat1_dma_free_chan_resources - release all the descriptors
683 * @chan: the channel to be cleaned 503 * @chan: the channel to be cleaned
684 */ 504 */
685static void ioat_dma_free_chan_resources(struct dma_chan *c) 505static void ioat1_dma_free_chan_resources(struct dma_chan *c)
686{ 506{
687 struct ioat_dma_chan *ioat = to_ioat_chan(c); 507 struct ioat_dma_chan *ioat = to_ioat_chan(c);
688 struct ioat_chan_common *chan = &ioat->base; 508 struct ioat_chan_common *chan = &ioat->base;
@@ -697,7 +517,7 @@ static void ioat_dma_free_chan_resources(struct dma_chan *c)
697 return; 517 return;
698 518
699 tasklet_disable(&chan->cleanup_task); 519 tasklet_disable(&chan->cleanup_task);
700 ioat_dma_memcpy_cleanup(ioat); 520 ioat1_cleanup(ioat);
701 521
702 /* Delay 100ms after reset to allow internal DMA logic to quiesce 522 /* Delay 100ms after reset to allow internal DMA logic to quiesce
703 * before removing DMA descriptor resources. 523 * before removing DMA descriptor resources.
@@ -707,40 +527,20 @@ static void ioat_dma_free_chan_resources(struct dma_chan *c)
707 mdelay(100); 527 mdelay(100);
708 528
709 spin_lock_bh(&ioat->desc_lock); 529 spin_lock_bh(&ioat->desc_lock);
710 switch (chan->device->version) { 530 list_for_each_entry_safe(desc, _desc,
711 case IOAT_VER_1_2: 531 &ioat->used_desc, node) {
712 list_for_each_entry_safe(desc, _desc, 532 in_use_descs++;
713 &ioat->used_desc, node) { 533 list_del(&desc->node);
714 in_use_descs++; 534 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
715 list_del(&desc->node); 535 desc->txd.phys);
716 pci_pool_free(ioatdma_device->dma_pool, desc->hw, 536 kfree(desc);
717 desc->txd.phys); 537 }
718 kfree(desc); 538 list_for_each_entry_safe(desc, _desc,
719 } 539 &ioat->free_desc, node) {
720 list_for_each_entry_safe(desc, _desc, 540 list_del(&desc->node);
721 &ioat->free_desc, node) {
722 list_del(&desc->node);
723 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
724 desc->txd.phys);
725 kfree(desc);
726 }
727 break;
728 case IOAT_VER_2_0:
729 case IOAT_VER_3_0:
730 list_for_each_entry_safe(desc, _desc,
731 ioat->free_desc.next, node) {
732 list_del(&desc->node);
733 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
734 desc->txd.phys);
735 kfree(desc);
736 }
737 desc = to_ioat_desc(ioat->free_desc.next);
738 pci_pool_free(ioatdma_device->dma_pool, desc->hw, 541 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
739 desc->txd.phys); 542 desc->txd.phys);
740 kfree(desc); 543 kfree(desc);
741 INIT_LIST_HEAD(&ioat->free_desc);
742 INIT_LIST_HEAD(&ioat->used_desc);
743 break;
744 } 544 }
745 spin_unlock_bh(&ioat->desc_lock); 545 spin_unlock_bh(&ioat->desc_lock);
746 546
@@ -758,7 +558,6 @@ static void ioat_dma_free_chan_resources(struct dma_chan *c)
758 chan->last_compl_desc_addr_hw = 0; 558 chan->last_compl_desc_addr_hw = 0;
759 chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0; 559 chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
760 ioat->pending = 0; 560 ioat->pending = 0;
761 ioat->dmacount = 0;
762 ioat->desccount = 0; 561 ioat->desccount = 0;
763} 562}
764 563
@@ -791,86 +590,6 @@ ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
791 return new; 590 return new;
792} 591}
793 592
794static struct ioat_desc_sw *
795ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
796{
797 struct ioat_desc_sw *new;
798
799 /*
800 * used.prev points to where to start processing
801 * used.next points to next free descriptor
802 * if used.prev == NULL, there are none waiting to be processed
803 * if used.next == used.prev.prev, there is only one free descriptor,
804 * and we need to use it to as a noop descriptor before
805 * linking in a new set of descriptors, since the device
806 * has probably already read the pointer to it
807 */
808 if (ioat->used_desc.prev &&
809 ioat->used_desc.next == ioat->used_desc.prev->prev) {
810
811 struct ioat_desc_sw *desc;
812 struct ioat_desc_sw *noop_desc;
813 int i;
814
815 /* set up the noop descriptor */
816 noop_desc = to_ioat_desc(ioat->used_desc.next);
817 /* set size to non-zero value (channel returns error when size is 0) */
818 noop_desc->hw->size = NULL_DESC_BUFFER_SIZE;
819 noop_desc->hw->ctl = 0;
820 noop_desc->hw->ctl_f.null = 1;
821 noop_desc->hw->src_addr = 0;
822 noop_desc->hw->dst_addr = 0;
823
824 ioat->used_desc.next = ioat->used_desc.next->next;
825 ioat->pending++;
826 ioat->dmacount++;
827
828 /* try to get a few more descriptors */
829 for (i = 16; i; i--) {
830 desc = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
831 if (!desc) {
832 dev_err(to_dev(&ioat->base),
833 "alloc failed\n");
834 break;
835 }
836 list_add_tail(&desc->node, ioat->used_desc.next);
837
838 desc->hw->next
839 = to_ioat_desc(desc->node.next)->txd.phys;
840 to_ioat_desc(desc->node.prev)->hw->next
841 = desc->txd.phys;
842 ioat->desccount++;
843 }
844
845 ioat->used_desc.next = noop_desc->node.next;
846 }
847 new = to_ioat_desc(ioat->used_desc.next);
848 prefetch(new);
849 ioat->used_desc.next = new->node.next;
850
851 if (ioat->used_desc.prev == NULL)
852 ioat->used_desc.prev = &new->node;
853
854 prefetch(new->hw);
855 return new;
856}
857
858static struct ioat_desc_sw *
859ioat_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
860{
861 if (!ioat)
862 return NULL;
863
864 switch (ioat->base.device->version) {
865 case IOAT_VER_1_2:
866 return ioat1_dma_get_next_descriptor(ioat);
867 case IOAT_VER_2_0:
868 case IOAT_VER_3_0:
869 return ioat2_dma_get_next_descriptor(ioat);
870 }
871 return NULL;
872}
873
874static struct dma_async_tx_descriptor * 593static struct dma_async_tx_descriptor *
875ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest, 594ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
876 dma_addr_t dma_src, size_t len, unsigned long flags) 595 dma_addr_t dma_src, size_t len, unsigned long flags)
@@ -886,7 +605,7 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
886 int tx_cnt = 0; 605 int tx_cnt = 0;
887 606
888 spin_lock_bh(&ioat->desc_lock); 607 spin_lock_bh(&ioat->desc_lock);
889 desc = ioat_dma_get_next_descriptor(ioat); 608 desc = ioat1_dma_get_next_descriptor(ioat);
890 do { 609 do {
891 if (!desc) 610 if (!desc)
892 break; 611 break;
@@ -909,7 +628,7 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
909 struct ioat_desc_sw *next; 628 struct ioat_desc_sw *next;
910 629
911 async_tx_ack(&desc->txd); 630 async_tx_ack(&desc->txd);
912 next = ioat_dma_get_next_descriptor(ioat); 631 next = ioat1_dma_get_next_descriptor(ioat);
913 hw->next = next ? next->txd.phys : 0; 632 hw->next = next ? next->txd.phys : 0;
914 desc = next; 633 desc = next;
915 } else 634 } else
@@ -920,8 +639,7 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
920 struct ioat_chan_common *chan = &ioat->base; 639 struct ioat_chan_common *chan = &ioat->base;
921 640
922 dev_err(to_dev(chan), 641 dev_err(to_dev(chan),
923 "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n", 642 "chan%d - get_next_desc failed\n", chan_num(chan));
924 chan_num(chan), ioat->dmacount, ioat->desccount);
925 list_splice(&chain, &ioat->free_desc); 643 list_splice(&chain, &ioat->free_desc);
926 spin_unlock_bh(&ioat->desc_lock); 644 spin_unlock_bh(&ioat->desc_lock);
927 return NULL; 645 return NULL;
@@ -940,94 +658,43 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
940 return &desc->txd; 658 return &desc->txd;
941} 659}
942 660
943static struct dma_async_tx_descriptor * 661static void ioat1_cleanup_tasklet(unsigned long data)
944ioat2_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
945 dma_addr_t dma_src, size_t len, unsigned long flags)
946{
947 struct ioat_dma_chan *ioat = to_ioat_chan(c);
948 struct ioat_desc_sw *new;
949
950 spin_lock_bh(&ioat->desc_lock);
951 new = ioat2_dma_get_next_descriptor(ioat);
952
953 /*
954 * leave ioat->desc_lock set in ioat 2 path
955 * it will get unlocked at end of tx_submit
956 */
957
958 if (new) {
959 new->len = len;
960 new->dst = dma_dest;
961 new->src = dma_src;
962 new->txd.flags = flags;
963 return &new->txd;
964 } else {
965 struct ioat_chan_common *chan = &ioat->base;
966
967 spin_unlock_bh(&ioat->desc_lock);
968 dev_err(to_dev(chan),
969 "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
970 chan_num(chan), ioat->dmacount, ioat->desccount);
971 return NULL;
972 }
973}
974
975static void ioat_dma_cleanup_tasklet(unsigned long data)
976{ 662{
977 struct ioat_dma_chan *chan = (void *)data; 663 struct ioat_dma_chan *chan = (void *)data;
978 ioat_dma_memcpy_cleanup(chan); 664 ioat1_cleanup(chan);
979 writew(IOAT_CHANCTRL_INT_DISABLE, 665 writew(IOAT_CHANCTRL_INT_DISABLE,
980 chan->base.reg_base + IOAT_CHANCTRL_OFFSET); 666 chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
981} 667}
982 668
983static void 669static void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
984ioat_dma_unmap(struct ioat_chan_common *chan, struct ioat_desc_sw *desc) 670 int direction, enum dma_ctrl_flags flags, bool dst)
985{ 671{
986 if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { 672 if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) ||
987 if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE) 673 (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE)))
988 pci_unmap_single(chan->device->pdev, 674 pci_unmap_single(pdev, addr, len, direction);
989 pci_unmap_addr(desc, dst), 675 else
990 pci_unmap_len(desc, len), 676 pci_unmap_page(pdev, addr, len, direction);
991 PCI_DMA_FROMDEVICE);
992 else
993 pci_unmap_page(chan->device->pdev,
994 pci_unmap_addr(desc, dst),
995 pci_unmap_len(desc, len),
996 PCI_DMA_FROMDEVICE);
997 }
998
999 if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1000 if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
1001 pci_unmap_single(chan->device->pdev,
1002 pci_unmap_addr(desc, src),
1003 pci_unmap_len(desc, len),
1004 PCI_DMA_TODEVICE);
1005 else
1006 pci_unmap_page(chan->device->pdev,
1007 pci_unmap_addr(desc, src),
1008 pci_unmap_len(desc, len),
1009 PCI_DMA_TODEVICE);
1010 }
1011} 677}
1012 678
1013/** 679
1014 * ioat_dma_memcpy_cleanup - cleanup up finished descriptors 680void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
1015 * @chan: ioat channel to be cleaned up 681 size_t len, struct ioat_dma_descriptor *hw)
1016 */
1017static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat)
1018{ 682{
1019 struct ioat_chan_common *chan = &ioat->base; 683 struct pci_dev *pdev = chan->device->pdev;
1020 unsigned long phys_complete; 684 size_t offset = len - hw->size;
1021 struct ioat_desc_sw *desc, *_desc;
1022 dma_cookie_t cookie = 0;
1023 unsigned long desc_phys;
1024 struct ioat_desc_sw *latest_desc;
1025 struct dma_async_tx_descriptor *tx;
1026 685
1027 prefetch(chan->completion_virt); 686 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
687 ioat_unmap(pdev, hw->dst_addr - offset, len,
688 PCI_DMA_FROMDEVICE, flags, 1);
1028 689
1029 if (!spin_trylock_bh(&chan->cleanup_lock)) 690 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP))
1030 return; 691 ioat_unmap(pdev, hw->src_addr - offset, len,
692 PCI_DMA_TODEVICE, flags, 0);
693}
694
695unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
696{
697 unsigned long phys_complete;
1031 698
1032 /* The completion writeback can happen at any time, 699 /* The completion writeback can happen at any time,
1033 so reads by the driver need to be atomic operations 700 so reads by the driver need to be atomic operations
@@ -1051,18 +718,37 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat)
1051 /* TODO do something to salvage the situation */ 718 /* TODO do something to salvage the situation */
1052 } 719 }
1053 720
721 return phys_complete;
722}
723
724/**
725 * ioat1_cleanup - cleanup up finished descriptors
726 * @chan: ioat channel to be cleaned up
727 */
728static void ioat1_cleanup(struct ioat_dma_chan *ioat)
729{
730 struct ioat_chan_common *chan = &ioat->base;
731 unsigned long phys_complete;
732 struct ioat_desc_sw *desc, *_desc;
733 dma_cookie_t cookie = 0;
734 struct dma_async_tx_descriptor *tx;
735
736 prefetch(chan->completion_virt);
737
738 if (!spin_trylock_bh(&chan->cleanup_lock))
739 return;
740
741 phys_complete = ioat_get_current_completion(chan);
1054 if (phys_complete == chan->last_completion) { 742 if (phys_complete == chan->last_completion) {
1055 spin_unlock_bh(&chan->cleanup_lock); 743 spin_unlock_bh(&chan->cleanup_lock);
1056 /* 744 /*
1057 * perhaps we're stuck so hard that the watchdog can't go off? 745 * perhaps we're stuck so hard that the watchdog can't go off?
1058 * try to catch it after 2 seconds 746 * try to catch it after 2 seconds
1059 */ 747 */
1060 if (chan->device->version != IOAT_VER_3_0) { 748 if (time_after(jiffies,
1061 if (time_after(jiffies, 749 chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
1062 chan->last_completion_time + HZ*WATCHDOG_DELAY)) { 750 ioat1_chan_watchdog(&(chan->device->work.work));
1063 ioat_dma_chan_watchdog(&(chan->device->work.work)); 751 chan->last_completion_time = jiffies;
1064 chan->last_completion_time = jiffies;
1065 }
1066 } 752 }
1067 return; 753 return;
1068 } 754 }
@@ -1074,91 +760,42 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat)
1074 return; 760 return;
1075 } 761 }
1076 762
1077 switch (chan->device->version) { 763 list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
1078 case IOAT_VER_1_2: 764 tx = &desc->txd;
1079 list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) { 765 /*
1080 tx = &desc->txd; 766 * Incoming DMA requests may use multiple descriptors,
1081 /* 767 * due to exceeding xfercap, perhaps. If so, only the
1082 * Incoming DMA requests may use multiple descriptors, 768 * last one will have a cookie, and require unmapping.
1083 * due to exceeding xfercap, perhaps. If so, only the 769 */
1084 * last one will have a cookie, and require unmapping. 770 if (tx->cookie) {
1085 */ 771 cookie = tx->cookie;
1086 if (tx->cookie) { 772 ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
1087 cookie = tx->cookie; 773 if (tx->callback) {
1088 ioat_dma_unmap(chan, desc); 774 tx->callback(tx->callback_param);
1089 if (tx->callback) { 775 tx->callback = NULL;
1090 tx->callback(tx->callback_param);
1091 tx->callback = NULL;
1092 }
1093 } 776 }
777 }
1094 778
1095 if (tx->phys != phys_complete) { 779 if (tx->phys != phys_complete) {
1096 /* 780 /*
1097 * a completed entry, but not the last, so clean 781 * a completed entry, but not the last, so clean
1098 * up if the client is done with the descriptor 782 * up if the client is done with the descriptor
1099 */ 783 */
1100 if (async_tx_test_ack(tx)) { 784 if (async_tx_test_ack(tx))
1101 list_move_tail(&desc->node, 785 list_move_tail(&desc->node, &ioat->free_desc);
1102 &ioat->free_desc); 786 else
1103 } else
1104 tx->cookie = 0;
1105 } else {
1106 /*
1107 * last used desc. Do not remove, so we can
1108 * append from it, but don't look at it next
1109 * time, either
1110 */
1111 tx->cookie = 0; 787 tx->cookie = 0;
788 } else {
789 /*
790 * last used desc. Do not remove, so we can
791 * append from it, but don't look at it next
792 * time, either
793 */
794 tx->cookie = 0;
1112 795
1113 /* TODO check status bits? */ 796 /* TODO check status bits? */
1114 break;
1115 }
1116 }
1117 break;
1118 case IOAT_VER_2_0:
1119 case IOAT_VER_3_0:
1120 /* has some other thread has already cleaned up? */
1121 if (ioat->used_desc.prev == NULL)
1122 break; 797 break;
1123
1124 /* work backwards to find latest finished desc */
1125 desc = to_ioat_desc(ioat->used_desc.next);
1126 tx = &desc->txd;
1127 latest_desc = NULL;
1128 do {
1129 desc = to_ioat_desc(desc->node.prev);
1130 desc_phys = (unsigned long)tx->phys
1131 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
1132 if (desc_phys == phys_complete) {
1133 latest_desc = desc;
1134 break;
1135 }
1136 } while (&desc->node != ioat->used_desc.prev);
1137
1138 if (latest_desc != NULL) {
1139 /* work forwards to clear finished descriptors */
1140 for (desc = to_ioat_desc(ioat->used_desc.prev);
1141 &desc->node != latest_desc->node.next &&
1142 &desc->node != ioat->used_desc.next;
1143 desc = to_ioat_desc(desc->node.next)) {
1144 if (tx->cookie) {
1145 cookie = tx->cookie;
1146 tx->cookie = 0;
1147 ioat_dma_unmap(chan, desc);
1148 if (tx->callback) {
1149 tx->callback(tx->callback_param);
1150 tx->callback = NULL;
1151 }
1152 }
1153 }
1154
1155 /* move used.prev up beyond those that are finished */
1156 if (&desc->node == ioat->used_desc.next)
1157 ioat->used_desc.prev = NULL;
1158 else
1159 ioat->used_desc.prev = &desc->node;
1160 } 798 }
1161 break;
1162 } 799 }
1163 800
1164 spin_unlock_bh(&ioat->desc_lock); 801 spin_unlock_bh(&ioat->desc_lock);
@@ -1170,50 +807,21 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat)
1170 spin_unlock_bh(&chan->cleanup_lock); 807 spin_unlock_bh(&chan->cleanup_lock);
1171} 808}
1172 809
1173/**
1174 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
1175 * @chan: IOAT DMA channel handle
1176 * @cookie: DMA transaction identifier
1177 * @done: if not %NULL, updated with last completed transaction
1178 * @used: if not %NULL, updated with last used transaction
1179 */
1180static enum dma_status 810static enum dma_status
1181ioat_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie, 811ioat1_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie,
1182 dma_cookie_t *done, dma_cookie_t *used) 812 dma_cookie_t *done, dma_cookie_t *used)
1183{ 813{
1184 struct ioat_dma_chan *ioat = to_ioat_chan(c); 814 struct ioat_dma_chan *ioat = to_ioat_chan(c);
1185 struct ioat_chan_common *chan = &ioat->base;
1186 dma_cookie_t last_used;
1187 dma_cookie_t last_complete;
1188 enum dma_status ret;
1189
1190 last_used = c->cookie;
1191 last_complete = chan->completed_cookie;
1192 chan->watchdog_tcp_cookie = cookie;
1193
1194 if (done)
1195 *done = last_complete;
1196 if (used)
1197 *used = last_used;
1198
1199 ret = dma_async_is_complete(cookie, last_complete, last_used);
1200 if (ret == DMA_SUCCESS)
1201 return ret;
1202 815
1203 ioat_dma_memcpy_cleanup(ioat); 816 if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
817 return DMA_SUCCESS;
1204 818
1205 last_used = c->cookie; 819 ioat1_cleanup(ioat);
1206 last_complete = chan->completed_cookie;
1207 820
1208 if (done) 821 return ioat_is_complete(c, cookie, done, used);
1209 *done = last_complete;
1210 if (used)
1211 *used = last_used;
1212
1213 return dma_async_is_complete(cookie, last_complete, last_used);
1214} 822}
1215 823
1216static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat) 824static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat)
1217{ 825{
1218 struct ioat_chan_common *chan = &ioat->base; 826 struct ioat_chan_common *chan = &ioat->base;
1219 struct ioat_desc_sw *desc; 827 struct ioat_desc_sw *desc;
@@ -1221,7 +829,7 @@ static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat)
1221 829
1222 spin_lock_bh(&ioat->desc_lock); 830 spin_lock_bh(&ioat->desc_lock);
1223 831
1224 desc = ioat_dma_get_next_descriptor(ioat); 832 desc = ioat1_dma_get_next_descriptor(ioat);
1225 833
1226 if (!desc) { 834 if (!desc) {
1227 dev_err(to_dev(chan), 835 dev_err(to_dev(chan),
@@ -1240,30 +848,16 @@ static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat)
1240 hw->src_addr = 0; 848 hw->src_addr = 0;
1241 hw->dst_addr = 0; 849 hw->dst_addr = 0;
1242 async_tx_ack(&desc->txd); 850 async_tx_ack(&desc->txd);
1243 switch (chan->device->version) { 851 hw->next = 0;
1244 case IOAT_VER_1_2: 852 list_add_tail(&desc->node, &ioat->used_desc);
1245 hw->next = 0;
1246 list_add_tail(&desc->node, &ioat->used_desc);
1247 853
1248 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF, 854 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1249 chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW); 855 chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
1250 writel(((u64) desc->txd.phys) >> 32, 856 writel(((u64) desc->txd.phys) >> 32,
1251 chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH); 857 chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
1252
1253 writeb(IOAT_CHANCMD_START, chan->reg_base
1254 + IOAT_CHANCMD_OFFSET(chan->device->version));
1255 break;
1256 case IOAT_VER_2_0:
1257 case IOAT_VER_3_0:
1258 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1259 chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
1260 writel(((u64) desc->txd.phys) >> 32,
1261 chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
1262 858
1263 ioat->dmacount++; 859 writeb(IOAT_CHANCMD_START, chan->reg_base
1264 __ioat2_dma_memcpy_issue_pending(ioat); 860 + IOAT_CHANCMD_OFFSET(chan->device->version));
1265 break;
1266 }
1267 spin_unlock_bh(&ioat->desc_lock); 861 spin_unlock_bh(&ioat->desc_lock);
1268} 862}
1269 863
@@ -1484,7 +1078,7 @@ static void ioat_disable_interrupts(struct ioatdma_device *device)
1484 writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET); 1078 writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1485} 1079}
1486 1080
1487static int ioat_probe(struct ioatdma_device *device) 1081int ioat_probe(struct ioatdma_device *device)
1488{ 1082{
1489 int err = -ENODEV; 1083 int err = -ENODEV;
1490 struct dma_device *dma = &device->common; 1084 struct dma_device *dma = &device->common;
@@ -1503,17 +1097,15 @@ static int ioat_probe(struct ioatdma_device *device)
1503 device->completion_pool = pci_pool_create("completion_pool", pdev, 1097 device->completion_pool = pci_pool_create("completion_pool", pdev,
1504 sizeof(u64), SMP_CACHE_BYTES, 1098 sizeof(u64), SMP_CACHE_BYTES,
1505 SMP_CACHE_BYTES); 1099 SMP_CACHE_BYTES);
1100
1506 if (!device->completion_pool) { 1101 if (!device->completion_pool) {
1507 err = -ENOMEM; 1102 err = -ENOMEM;
1508 goto err_completion_pool; 1103 goto err_completion_pool;
1509 } 1104 }
1510 1105
1511 ioat_dma_enumerate_channels(device); 1106 device->enumerate_channels(device);
1512 1107
1513 dma_cap_set(DMA_MEMCPY, dma->cap_mask); 1108 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
1514 dma->device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
1515 dma->device_free_chan_resources = ioat_dma_free_chan_resources;
1516 dma->device_is_tx_complete = ioat_dma_is_complete;
1517 dma->dev = &pdev->dev; 1109 dma->dev = &pdev->dev;
1518 1110
1519 dev_err(dev, "Intel(R) I/OAT DMA Engine found," 1111 dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
@@ -1546,7 +1138,7 @@ err_dma_pool:
1546 return err; 1138 return err;
1547} 1139}
1548 1140
1549static int ioat_register(struct ioatdma_device *device) 1141int ioat_register(struct ioatdma_device *device)
1550{ 1142{
1551 int err = dma_async_device_register(&device->common); 1143 int err = dma_async_device_register(&device->common);
1552 1144
@@ -1580,9 +1172,13 @@ int ioat1_dma_probe(struct ioatdma_device *device, int dca)
1580 int err; 1172 int err;
1581 1173
1582 device->intr_quirk = ioat1_intr_quirk; 1174 device->intr_quirk = ioat1_intr_quirk;
1175 device->enumerate_channels = ioat1_enumerate_channels;
1583 dma = &device->common; 1176 dma = &device->common;
1584 dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy; 1177 dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1585 dma->device_issue_pending = ioat1_dma_memcpy_issue_pending; 1178 dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
1179 dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources;
1180 dma->device_free_chan_resources = ioat1_dma_free_chan_resources;
1181 dma->device_is_tx_complete = ioat1_dma_is_complete;
1586 1182
1587 err = ioat_probe(device); 1183 err = ioat_probe(device);
1588 if (err) 1184 if (err)
@@ -1594,93 +1190,12 @@ int ioat1_dma_probe(struct ioatdma_device *device, int dca)
1594 if (dca) 1190 if (dca)
1595 device->dca = ioat_dca_init(pdev, device->reg_base); 1191 device->dca = ioat_dca_init(pdev, device->reg_base);
1596 1192
1597 INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog); 1193 INIT_DELAYED_WORK(&device->work, ioat1_chan_watchdog);
1598 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1599
1600 return err;
1601}
1602
1603int ioat2_dma_probe(struct ioatdma_device *device, int dca)
1604{
1605 struct pci_dev *pdev = device->pdev;
1606 struct dma_device *dma;
1607 struct dma_chan *c;
1608 struct ioat_chan_common *chan;
1609 int err;
1610
1611 dma = &device->common;
1612 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1613 dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1614
1615 err = ioat_probe(device);
1616 if (err)
1617 return err;
1618 ioat_set_tcp_copy_break(2048);
1619
1620 list_for_each_entry(c, &dma->channels, device_node) {
1621 chan = to_chan_common(c);
1622 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
1623 chan->reg_base + IOAT_DCACTRL_OFFSET);
1624 }
1625
1626 err = ioat_register(device);
1627 if (err)
1628 return err;
1629 if (dca)
1630 device->dca = ioat2_dca_init(pdev, device->reg_base);
1631
1632 INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
1633 schedule_delayed_work(&device->work, WATCHDOG_DELAY); 1194 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1634 1195
1635 return err; 1196 return err;
1636} 1197}
1637 1198
1638int ioat3_dma_probe(struct ioatdma_device *device, int dca)
1639{
1640 struct pci_dev *pdev = device->pdev;
1641 struct dma_device *dma;
1642 struct dma_chan *c;
1643 struct ioat_chan_common *chan;
1644 int err;
1645 u16 dev_id;
1646
1647 dma = &device->common;
1648 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1649 dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1650
1651 /* -= IOAT ver.3 workarounds =- */
1652 /* Write CHANERRMSK_INT with 3E07h to mask out the errors
1653 * that can cause stability issues for IOAT ver.3
1654 */
1655 pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
1656
1657 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1658 * (workaround for spurious config parity error after restart)
1659 */
1660 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1661 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
1662 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
1663
1664 err = ioat_probe(device);
1665 if (err)
1666 return err;
1667 ioat_set_tcp_copy_break(262144);
1668
1669 list_for_each_entry(c, &dma->channels, device_node) {
1670 chan = to_chan_common(c);
1671 writel(IOAT_DMA_DCA_ANY_CPU,
1672 chan->reg_base + IOAT_DCACTRL_OFFSET);
1673 }
1674
1675 err = ioat_register(device);
1676 if (err)
1677 return err;
1678 if (dca)
1679 device->dca = ioat3_dca_init(pdev, device->reg_base);
1680
1681 return err;
1682}
1683
1684void ioat_dma_remove(struct ioatdma_device *device) 1199void ioat_dma_remove(struct ioatdma_device *device)
1685{ 1200{
1686 struct dma_device *dma = &device->common; 1201 struct dma_device *dma = &device->common;
@@ -1697,4 +1212,3 @@ void ioat_dma_remove(struct ioatdma_device *device)
1697 1212
1698 INIT_LIST_HEAD(&dma->channels); 1213 INIT_LIST_HEAD(&dma->channels);
1699} 1214}
1700
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index 5b31db73ad8e..84065dfa4d40 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -62,6 +62,7 @@
62 * @idx: per channel data 62 * @idx: per channel data
63 * @dca: direct cache access context 63 * @dca: direct cache access context
64 * @intr_quirk: interrupt setup quirk (for ioat_v1 devices) 64 * @intr_quirk: interrupt setup quirk (for ioat_v1 devices)
65 * @enumerate_channels: hw version specific channel enumeration
65 */ 66 */
66 67
67struct ioatdma_device { 68struct ioatdma_device {
@@ -76,6 +77,7 @@ struct ioatdma_device {
76 struct ioat_chan_common *idx[4]; 77 struct ioat_chan_common *idx[4];
77 struct dca_provider *dca; 78 struct dca_provider *dca;
78 void (*intr_quirk)(struct ioatdma_device *device); 79 void (*intr_quirk)(struct ioatdma_device *device);
80 int (*enumerate_channels)(struct ioatdma_device *device);
79}; 81};
80 82
81struct ioat_chan_common { 83struct ioat_chan_common {
@@ -106,6 +108,7 @@ struct ioat_chan_common {
106 struct tasklet_struct cleanup_task; 108 struct tasklet_struct cleanup_task;
107}; 109};
108 110
111
109/** 112/**
110 * struct ioat_dma_chan - internal representation of a DMA channel 113 * struct ioat_dma_chan - internal representation of a DMA channel
111 */ 114 */
@@ -119,7 +122,6 @@ struct ioat_dma_chan {
119 struct list_head used_desc; 122 struct list_head used_desc;
120 123
121 int pending; 124 int pending;
122 u16 dmacount;
123 u16 desccount; 125 u16 desccount;
124}; 126};
125 127
@@ -135,6 +137,33 @@ static inline struct ioat_dma_chan *to_ioat_chan(struct dma_chan *c)
135 return container_of(chan, struct ioat_dma_chan, base); 137 return container_of(chan, struct ioat_dma_chan, base);
136} 138}
137 139
140/**
141 * ioat_is_complete - poll the status of an ioat transaction
142 * @c: channel handle
143 * @cookie: transaction identifier
144 * @done: if set, updated with last completed transaction
145 * @used: if set, updated with last used transaction
146 */
147static inline enum dma_status
148ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie,
149 dma_cookie_t *done, dma_cookie_t *used)
150{
151 struct ioat_chan_common *chan = to_chan_common(c);
152 dma_cookie_t last_used;
153 dma_cookie_t last_complete;
154
155 last_used = c->cookie;
156 last_complete = chan->completed_cookie;
157 chan->watchdog_tcp_cookie = cookie;
158
159 if (done)
160 *done = last_complete;
161 if (used)
162 *used = last_used;
163
164 return dma_async_is_complete(cookie, last_complete, last_used);
165}
166
138/* wrapper around hardware descriptor format + additional software fields */ 167/* wrapper around hardware descriptor format + additional software fields */
139 168
140/** 169/**
@@ -162,11 +191,22 @@ static inline void ioat_set_tcp_copy_break(unsigned long copybreak)
162 #endif 191 #endif
163} 192}
164 193
194static inline struct ioat_chan_common *
195ioat_chan_by_index(struct ioatdma_device *device, int index)
196{
197 return device->idx[index];
198}
199
200int ioat_probe(struct ioatdma_device *device);
201int ioat_register(struct ioatdma_device *device);
165int ioat1_dma_probe(struct ioatdma_device *dev, int dca); 202int ioat1_dma_probe(struct ioatdma_device *dev, int dca);
166int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
167int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
168void ioat_dma_remove(struct ioatdma_device *device); 203void ioat_dma_remove(struct ioatdma_device *device);
169struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); 204struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
170struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase); 205unsigned long ioat_get_current_completion(struct ioat_chan_common *chan);
171struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); 206void ioat_init_channel(struct ioatdma_device *device,
207 struct ioat_chan_common *chan, int idx,
208 work_func_t work_fn, void (*tasklet)(unsigned long),
209 unsigned long tasklet_data);
210void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
211 size_t len, struct ioat_dma_descriptor *hw);
172#endif /* IOATDMA_H */ 212#endif /* IOATDMA_H */
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
new file mode 100644
index 000000000000..49ba1c73d95e
--- /dev/null
+++ b/drivers/dma/ioat/dma_v2.c
@@ -0,0 +1,750 @@
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2009 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 */
22
23/*
24 * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
25 * does asynchronous data movement and checksumming operations.
26 */
27
28#include <linux/init.h>
29#include <linux/module.h>
30#include <linux/pci.h>
31#include <linux/interrupt.h>
32#include <linux/dmaengine.h>
33#include <linux/delay.h>
34#include <linux/dma-mapping.h>
35#include <linux/workqueue.h>
36#include <linux/i7300_idle.h>
37#include "dma.h"
38#include "dma_v2.h"
39#include "registers.h"
40#include "hw.h"
41
42static int ioat_ring_alloc_order = 8;
43module_param(ioat_ring_alloc_order, int, 0644);
44MODULE_PARM_DESC(ioat_ring_alloc_order,
45 "ioat2+: allocate 2^n descriptors per channel (default: n=8)");
46
47static void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
48{
49 void * __iomem reg_base = ioat->base.reg_base;
50
51 ioat->pending = 0;
52 ioat->dmacount += ioat2_ring_pending(ioat);
53 ioat->issued = ioat->head;
54 /* make descriptor updates globally visible before notifying channel */
55 wmb();
56 writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
57
58}
59
60static void ioat2_issue_pending(struct dma_chan *chan)
61{
62 struct ioat2_dma_chan *ioat = to_ioat2_chan(chan);
63
64 spin_lock_bh(&ioat->ring_lock);
65 if (ioat->pending == 1)
66 __ioat2_issue_pending(ioat);
67 spin_unlock_bh(&ioat->ring_lock);
68}
69
70/**
71 * ioat2_update_pending - log pending descriptors
72 * @ioat: ioat2+ channel
73 *
74 * set pending to '1' unless pending is already set to '2', pending == 2
75 * indicates that submission is temporarily blocked due to an in-flight
76 * reset. If we are already above the ioat_pending_level threshold then
77 * just issue pending.
78 *
79 * called with ring_lock held
80 */
81static void ioat2_update_pending(struct ioat2_dma_chan *ioat)
82{
83 if (unlikely(ioat->pending == 2))
84 return;
85 else if (ioat2_ring_pending(ioat) > ioat_pending_level)
86 __ioat2_issue_pending(ioat);
87 else
88 ioat->pending = 1;
89}
90
91static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
92{
93 void __iomem *reg_base = ioat->base.reg_base;
94 struct ioat_ring_ent *desc;
95 struct ioat_dma_descriptor *hw;
96 int idx;
97
98 if (ioat2_ring_space(ioat) < 1) {
99 dev_err(to_dev(&ioat->base),
100 "Unable to start null desc - ring full\n");
101 return;
102 }
103
104 idx = ioat2_desc_alloc(ioat, 1);
105 desc = ioat2_get_ring_ent(ioat, idx);
106
107 hw = desc->hw;
108 hw->ctl = 0;
109 hw->ctl_f.null = 1;
110 hw->ctl_f.int_en = 1;
111 hw->ctl_f.compl_write = 1;
112 /* set size to non-zero value (channel returns error when size is 0) */
113 hw->size = NULL_DESC_BUFFER_SIZE;
114 hw->src_addr = 0;
115 hw->dst_addr = 0;
116 async_tx_ack(&desc->txd);
117 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
118 reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
119 writel(((u64) desc->txd.phys) >> 32,
120 reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
121 __ioat2_issue_pending(ioat);
122}
123
124static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
125{
126 spin_lock_bh(&ioat->ring_lock);
127 __ioat2_start_null_desc(ioat);
128 spin_unlock_bh(&ioat->ring_lock);
129}
130
131static void ioat2_cleanup(struct ioat2_dma_chan *ioat);
132
133/**
134 * ioat2_reset_part2 - reinit the channel after a reset
135 */
136static void ioat2_reset_part2(struct work_struct *work)
137{
138 struct ioat_chan_common *chan;
139 struct ioat2_dma_chan *ioat;
140
141 chan = container_of(work, struct ioat_chan_common, work.work);
142 ioat = container_of(chan, struct ioat2_dma_chan, base);
143
144 /* ensure that ->tail points to the stalled descriptor
145 * (ioat->pending is set to 2 at this point so no new
146 * descriptors will be issued while we perform this cleanup)
147 */
148 ioat2_cleanup(ioat);
149
150 spin_lock_bh(&chan->cleanup_lock);
151 spin_lock_bh(&ioat->ring_lock);
152
153 /* set the tail to be re-issued */
154 ioat->issued = ioat->tail;
155 ioat->dmacount = 0;
156
157 if (ioat2_ring_pending(ioat)) {
158 struct ioat_ring_ent *desc;
159
160 desc = ioat2_get_ring_ent(ioat, ioat->tail);
161 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
162 chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
163 writel(((u64) desc->txd.phys) >> 32,
164 chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
165 __ioat2_issue_pending(ioat);
166 } else
167 __ioat2_start_null_desc(ioat);
168
169 spin_unlock_bh(&ioat->ring_lock);
170 spin_unlock_bh(&chan->cleanup_lock);
171
172 dev_info(to_dev(chan),
173 "chan%d reset - %d descs waiting, %d total desc\n",
174 chan_num(chan), ioat->dmacount, 1 << ioat->alloc_order);
175}
176
177/**
178 * ioat2_reset_channel - restart a channel
179 * @ioat: IOAT DMA channel handle
180 */
181static void ioat2_reset_channel(struct ioat2_dma_chan *ioat)
182{
183 u32 chansts, chanerr;
184 struct ioat_chan_common *chan = &ioat->base;
185 u16 active;
186
187 spin_lock_bh(&ioat->ring_lock);
188 active = ioat2_ring_active(ioat);
189 spin_unlock_bh(&ioat->ring_lock);
190 if (!active)
191 return;
192
193 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
194 chansts = (chan->completion_virt->low
195 & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
196 if (chanerr) {
197 dev_err(to_dev(chan),
198 "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
199 chan_num(chan), chansts, chanerr);
200 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
201 }
202
203 spin_lock_bh(&ioat->ring_lock);
204 ioat->pending = 2;
205 writeb(IOAT_CHANCMD_RESET,
206 chan->reg_base
207 + IOAT_CHANCMD_OFFSET(chan->device->version));
208 spin_unlock_bh(&ioat->ring_lock);
209 schedule_delayed_work(&chan->work, RESET_DELAY);
210}
211
212/**
213 * ioat2_chan_watchdog - watch for stuck channels
214 */
215static void ioat2_chan_watchdog(struct work_struct *work)
216{
217 struct ioatdma_device *device =
218 container_of(work, struct ioatdma_device, work.work);
219 struct ioat2_dma_chan *ioat;
220 struct ioat_chan_common *chan;
221 u16 active;
222 int i;
223
224 for (i = 0; i < device->common.chancnt; i++) {
225 chan = ioat_chan_by_index(device, i);
226 ioat = container_of(chan, struct ioat2_dma_chan, base);
227
228 /*
229 * for version 2.0 if there are descriptors yet to be processed
230 * and the last completed hasn't changed since the last watchdog
231 * if they haven't hit the pending level
232 * issue the pending to push them through
233 * else
234 * try resetting the channel
235 */
236 spin_lock_bh(&ioat->ring_lock);
237 active = ioat2_ring_active(ioat);
238 spin_unlock_bh(&ioat->ring_lock);
239
240 if (active &&
241 chan->last_completion &&
242 chan->last_completion == chan->watchdog_completion) {
243
244 if (ioat->pending == 1)
245 ioat2_issue_pending(&chan->common);
246 else {
247 ioat2_reset_channel(ioat);
248 chan->watchdog_completion = 0;
249 }
250 } else {
251 chan->last_compl_desc_addr_hw = 0;
252 chan->watchdog_completion = chan->last_completion;
253 }
254 chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
255 }
256 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
257}
258
259/**
260 * ioat2_cleanup - clean finished descriptors (advance tail pointer)
261 * @chan: ioat channel to be cleaned up
262 */
263static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
264{
265 struct ioat_chan_common *chan = &ioat->base;
266 unsigned long phys_complete;
267 struct ioat_ring_ent *desc;
268 bool seen_current = false;
269 u16 active;
270 int i;
271 struct dma_async_tx_descriptor *tx;
272
273 prefetch(chan->completion_virt);
274
275 spin_lock_bh(&chan->cleanup_lock);
276 phys_complete = ioat_get_current_completion(chan);
277 if (phys_complete == chan->last_completion) {
278 spin_unlock_bh(&chan->cleanup_lock);
279 /*
280 * perhaps we're stuck so hard that the watchdog can't go off?
281 * try to catch it after WATCHDOG_DELAY seconds
282 */
283 if (chan->device->version < IOAT_VER_3_0) {
284 unsigned long tmo;
285
286 tmo = chan->last_completion_time + HZ*WATCHDOG_DELAY;
287 if (time_after(jiffies, tmo)) {
288 ioat2_chan_watchdog(&(chan->device->work.work));
289 chan->last_completion_time = jiffies;
290 }
291 }
292 return;
293 }
294 chan->last_completion_time = jiffies;
295
296 spin_lock_bh(&ioat->ring_lock);
297
298 active = ioat2_ring_active(ioat);
299 for (i = 0; i < active && !seen_current; i++) {
300 prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
301 desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
302 tx = &desc->txd;
303 if (tx->cookie) {
304 ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
305 chan->completed_cookie = tx->cookie;
306 tx->cookie = 0;
307 if (tx->callback) {
308 tx->callback(tx->callback_param);
309 tx->callback = NULL;
310 }
311 }
312
313 if (tx->phys == phys_complete)
314 seen_current = true;
315 }
316 ioat->tail += i;
317 BUG_ON(!seen_current); /* no active descs have written a completion? */
318 spin_unlock_bh(&ioat->ring_lock);
319
320 chan->last_completion = phys_complete;
321
322 spin_unlock_bh(&chan->cleanup_lock);
323}
324
325static void ioat2_cleanup_tasklet(unsigned long data)
326{
327 struct ioat2_dma_chan *ioat = (void *) data;
328
329 ioat2_cleanup(ioat);
330 writew(IOAT_CHANCTRL_INT_DISABLE,
331 ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
332}
333
334/**
335 * ioat2_enumerate_channels - find and initialize the device's channels
336 * @device: the device to be enumerated
337 */
338static int ioat2_enumerate_channels(struct ioatdma_device *device)
339{
340 struct ioat2_dma_chan *ioat;
341 struct device *dev = &device->pdev->dev;
342 struct dma_device *dma = &device->common;
343 u8 xfercap_log;
344 int i;
345
346 INIT_LIST_HEAD(&dma->channels);
347 dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
348 xfercap_log = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
349 if (xfercap_log == 0)
350 return 0;
351
352 /* FIXME which i/oat version is i7300? */
353#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
354 if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
355 dma->chancnt--;
356#endif
357 for (i = 0; i < dma->chancnt; i++) {
358 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
359 if (!ioat)
360 break;
361
362 ioat_init_channel(device, &ioat->base, i,
363 ioat2_reset_part2,
364 ioat2_cleanup_tasklet,
365 (unsigned long) ioat);
366 ioat->xfercap_log = xfercap_log;
367 spin_lock_init(&ioat->ring_lock);
368 }
369 dma->chancnt = i;
370 return i;
371}
372
373static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
374{
375 struct dma_chan *c = tx->chan;
376 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
377 dma_cookie_t cookie = c->cookie;
378
379 cookie++;
380 if (cookie < 0)
381 cookie = 1;
382 tx->cookie = cookie;
383 c->cookie = cookie;
384 ioat2_update_pending(ioat);
385 spin_unlock_bh(&ioat->ring_lock);
386
387 return cookie;
388}
389
390static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan)
391{
392 struct ioat_dma_descriptor *hw;
393 struct ioat_ring_ent *desc;
394 struct ioatdma_device *dma;
395 dma_addr_t phys;
396
397 dma = to_ioatdma_device(chan->device);
398 hw = pci_pool_alloc(dma->dma_pool, GFP_KERNEL, &phys);
399 if (!hw)
400 return NULL;
401 memset(hw, 0, sizeof(*hw));
402
403 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
404 if (!desc) {
405 pci_pool_free(dma->dma_pool, hw, phys);
406 return NULL;
407 }
408
409 dma_async_tx_descriptor_init(&desc->txd, chan);
410 desc->txd.tx_submit = ioat2_tx_submit_unlock;
411 desc->hw = hw;
412 desc->txd.phys = phys;
413 return desc;
414}
415
416static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
417{
418 struct ioatdma_device *dma;
419
420 dma = to_ioatdma_device(chan->device);
421 pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys);
422 kfree(desc);
423}
424
425/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
426 * @chan: channel to be initialized
427 */
428static int ioat2_alloc_chan_resources(struct dma_chan *c)
429{
430 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
431 struct ioat_chan_common *chan = &ioat->base;
432 struct ioat_ring_ent **ring;
433 u16 chanctrl;
434 u32 chanerr;
435 int descs;
436 int i;
437
438 /* have we already been set up? */
439 if (ioat->ring)
440 return 1 << ioat->alloc_order;
441
442 /* Setup register to interrupt and write completion status on error */
443 chanctrl = IOAT_CHANCTRL_ERR_INT_EN | IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
444 IOAT_CHANCTRL_ERR_COMPLETION_EN;
445 writew(chanctrl, chan->reg_base + IOAT_CHANCTRL_OFFSET);
446
447 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
448 if (chanerr) {
449 dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
450 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
451 }
452
453 /* allocate a completion writeback area */
454 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
455 chan->completion_virt = pci_pool_alloc(chan->device->completion_pool,
456 GFP_KERNEL,
457 &chan->completion_addr);
458 if (!chan->completion_virt)
459 return -ENOMEM;
460
461 memset(chan->completion_virt, 0,
462 sizeof(*chan->completion_virt));
463 writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF,
464 chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
465 writel(((u64) chan->completion_addr) >> 32,
466 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
467
468 ioat->alloc_order = ioat_get_alloc_order();
469 descs = 1 << ioat->alloc_order;
470
471 /* allocate the array to hold the software ring */
472 ring = kcalloc(descs, sizeof(*ring), GFP_KERNEL);
473 if (!ring)
474 return -ENOMEM;
475 for (i = 0; i < descs; i++) {
476 ring[i] = ioat2_alloc_ring_ent(c);
477 if (!ring[i]) {
478 while (i--)
479 ioat2_free_ring_ent(ring[i], c);
480 kfree(ring);
481 return -ENOMEM;
482 }
483 }
484
485 /* link descs */
486 for (i = 0; i < descs-1; i++) {
487 struct ioat_ring_ent *next = ring[i+1];
488 struct ioat_dma_descriptor *hw = ring[i]->hw;
489
490 hw->next = next->txd.phys;
491 }
492 ring[i]->hw->next = ring[0]->txd.phys;
493
494 spin_lock_bh(&ioat->ring_lock);
495 ioat->ring = ring;
496 ioat->head = 0;
497 ioat->issued = 0;
498 ioat->tail = 0;
499 ioat->pending = 0;
500 spin_unlock_bh(&ioat->ring_lock);
501
502 tasklet_enable(&chan->cleanup_task);
503 ioat2_start_null_desc(ioat);
504
505 return descs;
506}
507
508/**
509 * ioat2_alloc_and_lock - common descriptor alloc boilerplate for ioat2,3 ops
510 * @idx: gets starting descriptor index on successful allocation
511 * @ioat: ioat2,3 channel (ring) to operate on
512 * @num_descs: allocation length
513 */
514static int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs)
515{
516 struct ioat_chan_common *chan = &ioat->base;
517
518 spin_lock_bh(&ioat->ring_lock);
519 if (unlikely(ioat2_ring_space(ioat) < num_descs)) {
520 if (printk_ratelimit())
521 dev_dbg(to_dev(chan),
522 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
523 __func__, num_descs, ioat->head, ioat->tail,
524 ioat->issued);
525 spin_unlock_bh(&ioat->ring_lock);
526
527 /* do direct reclaim in the allocation failure case */
528 ioat2_cleanup(ioat);
529
530 return -ENOMEM;
531 }
532
533 dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n",
534 __func__, num_descs, ioat->head, ioat->tail, ioat->issued);
535
536 *idx = ioat2_desc_alloc(ioat, num_descs);
537 return 0; /* with ioat->ring_lock held */
538}
539
540static struct dma_async_tx_descriptor *
541ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
542 dma_addr_t dma_src, size_t len, unsigned long flags)
543{
544 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
545 struct ioat_dma_descriptor *hw;
546 struct ioat_ring_ent *desc;
547 dma_addr_t dst = dma_dest;
548 dma_addr_t src = dma_src;
549 size_t total_len = len;
550 int num_descs;
551 u16 idx;
552 int i;
553
554 num_descs = ioat2_xferlen_to_descs(ioat, len);
555 if (likely(num_descs) &&
556 ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0)
557 /* pass */;
558 else
559 return NULL;
560 for (i = 0; i < num_descs; i++) {
561 size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log);
562
563 desc = ioat2_get_ring_ent(ioat, idx + i);
564 hw = desc->hw;
565
566 hw->size = copy;
567 hw->ctl = 0;
568 hw->src_addr = src;
569 hw->dst_addr = dst;
570
571 len -= copy;
572 dst += copy;
573 src += copy;
574 }
575
576 desc->txd.flags = flags;
577 desc->len = total_len;
578 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
579 hw->ctl_f.compl_write = 1;
580 /* we leave the channel locked to ensure in order submission */
581
582 return &desc->txd;
583}
584
585/**
586 * ioat2_free_chan_resources - release all the descriptors
587 * @chan: the channel to be cleaned
588 */
589static void ioat2_free_chan_resources(struct dma_chan *c)
590{
591 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
592 struct ioat_chan_common *chan = &ioat->base;
593 struct ioatdma_device *ioatdma_device = chan->device;
594 struct ioat_ring_ent *desc;
595 const u16 total_descs = 1 << ioat->alloc_order;
596 int descs;
597 int i;
598
599 /* Before freeing channel resources first check
600 * if they have been previously allocated for this channel.
601 */
602 if (!ioat->ring)
603 return;
604
605 tasklet_disable(&chan->cleanup_task);
606 ioat2_cleanup(ioat);
607
608 /* Delay 100ms after reset to allow internal DMA logic to quiesce
609 * before removing DMA descriptor resources.
610 */
611 writeb(IOAT_CHANCMD_RESET,
612 chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
613 mdelay(100);
614
615 spin_lock_bh(&ioat->ring_lock);
616 descs = ioat2_ring_space(ioat);
617 for (i = 0; i < descs; i++) {
618 desc = ioat2_get_ring_ent(ioat, ioat->head + i);
619 ioat2_free_ring_ent(desc, c);
620 }
621
622 if (descs < total_descs)
623 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
624 total_descs - descs);
625
626 for (i = 0; i < total_descs - descs; i++) {
627 desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
628 ioat2_free_ring_ent(desc, c);
629 }
630
631 kfree(ioat->ring);
632 ioat->ring = NULL;
633 ioat->alloc_order = 0;
634 pci_pool_free(ioatdma_device->completion_pool,
635 chan->completion_virt,
636 chan->completion_addr);
637 spin_unlock_bh(&ioat->ring_lock);
638
639 chan->last_completion = 0;
640 chan->completion_addr = 0;
641 ioat->pending = 0;
642 ioat->dmacount = 0;
643 chan->watchdog_completion = 0;
644 chan->last_compl_desc_addr_hw = 0;
645 chan->watchdog_tcp_cookie = 0;
646 chan->watchdog_last_tcp_cookie = 0;
647}
648
649static enum dma_status
650ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
651 dma_cookie_t *done, dma_cookie_t *used)
652{
653 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
654
655 if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
656 return DMA_SUCCESS;
657
658 ioat2_cleanup(ioat);
659
660 return ioat_is_complete(c, cookie, done, used);
661}
662
663int ioat2_dma_probe(struct ioatdma_device *device, int dca)
664{
665 struct pci_dev *pdev = device->pdev;
666 struct dma_device *dma;
667 struct dma_chan *c;
668 struct ioat_chan_common *chan;
669 int err;
670
671 device->enumerate_channels = ioat2_enumerate_channels;
672 dma = &device->common;
673 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
674 dma->device_issue_pending = ioat2_issue_pending;
675 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
676 dma->device_free_chan_resources = ioat2_free_chan_resources;
677 dma->device_is_tx_complete = ioat2_is_complete;
678
679 err = ioat_probe(device);
680 if (err)
681 return err;
682 ioat_set_tcp_copy_break(2048);
683
684 list_for_each_entry(c, &dma->channels, device_node) {
685 chan = to_chan_common(c);
686 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
687 chan->reg_base + IOAT_DCACTRL_OFFSET);
688 }
689
690 err = ioat_register(device);
691 if (err)
692 return err;
693 if (dca)
694 device->dca = ioat2_dca_init(pdev, device->reg_base);
695
696 INIT_DELAYED_WORK(&device->work, ioat2_chan_watchdog);
697 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
698
699 return err;
700}
701
702int ioat3_dma_probe(struct ioatdma_device *device, int dca)
703{
704 struct pci_dev *pdev = device->pdev;
705 struct dma_device *dma;
706 struct dma_chan *c;
707 struct ioat_chan_common *chan;
708 int err;
709 u16 dev_id;
710
711 device->enumerate_channels = ioat2_enumerate_channels;
712 dma = &device->common;
713 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
714 dma->device_issue_pending = ioat2_issue_pending;
715 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
716 dma->device_free_chan_resources = ioat2_free_chan_resources;
717 dma->device_is_tx_complete = ioat2_is_complete;
718
719 /* -= IOAT ver.3 workarounds =- */
720 /* Write CHANERRMSK_INT with 3E07h to mask out the errors
721 * that can cause stability issues for IOAT ver.3
722 */
723 pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
724
725 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
726 * (workaround for spurious config parity error after restart)
727 */
728 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
729 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
730 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
731
732 err = ioat_probe(device);
733 if (err)
734 return err;
735 ioat_set_tcp_copy_break(262144);
736
737 list_for_each_entry(c, &dma->channels, device_node) {
738 chan = to_chan_common(c);
739 writel(IOAT_DMA_DCA_ANY_CPU,
740 chan->reg_base + IOAT_DCACTRL_OFFSET);
741 }
742
743 err = ioat_register(device);
744 if (err)
745 return err;
746 if (dca)
747 device->dca = ioat3_dca_init(pdev, device->reg_base);
748
749 return err;
750}
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
new file mode 100644
index 000000000000..94a553eacdbd
--- /dev/null
+++ b/drivers/dma/ioat/dma_v2.h
@@ -0,0 +1,131 @@
1/*
2 * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21#ifndef IOATDMA_V2_H
22#define IOATDMA_V2_H
23
24#include <linux/dmaengine.h>
25#include "dma.h"
26#include "hw.h"
27
28
29extern int ioat_pending_level;
30
31/*
32 * workaround for IOAT ver.3.0 null descriptor issue
33 * (channel returns error when size is 0)
34 */
35#define NULL_DESC_BUFFER_SIZE 1
36
37#define IOAT_MAX_ORDER 16
38#define ioat_get_alloc_order() \
39 (min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
40
41/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes
42 * @base: common ioat channel parameters
43 * @xfercap_log; log2 of channel max transfer length (for fast division)
44 * @head: allocated index
45 * @issued: hardware notification point
46 * @tail: cleanup index
47 * @pending: lock free indicator for issued != head
48 * @dmacount: identical to 'head' except for occasionally resetting to zero
49 * @alloc_order: log2 of the number of allocated descriptors
50 * @ring: software ring buffer implementation of hardware ring
51 * @ring_lock: protects ring attributes
52 */
53struct ioat2_dma_chan {
54 struct ioat_chan_common base;
55 size_t xfercap_log;
56 u16 head;
57 u16 issued;
58 u16 tail;
59 u16 dmacount;
60 u16 alloc_order;
61 int pending;
62 struct ioat_ring_ent **ring;
63 spinlock_t ring_lock;
64};
65
66static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c)
67{
68 struct ioat_chan_common *chan = to_chan_common(c);
69
70 return container_of(chan, struct ioat2_dma_chan, base);
71}
72
73static inline u16 ioat2_ring_mask(struct ioat2_dma_chan *ioat)
74{
75 return (1 << ioat->alloc_order) - 1;
76}
77
78/* count of descriptors in flight with the engine */
79static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat)
80{
81 return (ioat->head - ioat->tail) & ioat2_ring_mask(ioat);
82}
83
84/* count of descriptors pending submission to hardware */
85static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat)
86{
87 return (ioat->head - ioat->issued) & ioat2_ring_mask(ioat);
88}
89
90static inline u16 ioat2_ring_space(struct ioat2_dma_chan *ioat)
91{
92 u16 num_descs = ioat2_ring_mask(ioat) + 1;
93 u16 active = ioat2_ring_active(ioat);
94
95 BUG_ON(active > num_descs);
96
97 return num_descs - active;
98}
99
100/* assumes caller already checked space */
101static inline u16 ioat2_desc_alloc(struct ioat2_dma_chan *ioat, u16 len)
102{
103 ioat->head += len;
104 return ioat->head - len;
105}
106
107static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len)
108{
109 u16 num_descs = len >> ioat->xfercap_log;
110
111 num_descs += !!(len & ((1 << ioat->xfercap_log) - 1));
112 return num_descs;
113}
114
115struct ioat_ring_ent {
116 struct ioat_dma_descriptor *hw;
117 struct dma_async_tx_descriptor txd;
118 size_t len;
119};
120
121static inline struct ioat_ring_ent *
122ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx)
123{
124 return ioat->ring[idx & ioat2_ring_mask(ioat)];
125}
126
127int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
128int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
129struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
130struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
131#endif /* IOATDMA_V2_H */
diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c
index 55414d88ac1b..c4e432269252 100644
--- a/drivers/dma/ioat/pci.c
+++ b/drivers/dma/ioat/pci.c
@@ -31,6 +31,7 @@
31#include <linux/interrupt.h> 31#include <linux/interrupt.h>
32#include <linux/dca.h> 32#include <linux/dca.h>
33#include "dma.h" 33#include "dma.h"
34#include "dma_v2.h"
34#include "registers.h" 35#include "registers.h"
35#include "hw.h" 36#include "hw.h"
36 37
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-05 20:45:09 -0500