aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma
diff options
context:
space:
mode:
authorDan Williams <dan.j.williams@intel.com>2009-08-26 16:01:44 -0400
committerDan Williams <dan.j.williams@intel.com>2009-09-08 20:29:55 -0400
commit5cbafa65b92ee4f5b8ba915cddf94b91f186b989 (patch)
treef074c9dbcdedf05c5567a4e456a15120895363a6 /drivers/dma
parentdcbc853af6f0c056088e4df0794d9bf36184809e (diff)
ioat2,3: convert to a true ring buffer
Replace the current linked list munged into a ring with a native ring buffer implementation. The benefit of this approach is reduced overhead as many parameters can be derived from ring position with simple pointer comparisons and descriptor allocation/freeing becomes just a manipulation of head/tail pointers. It requires a contiguous allocation for the software descriptor information. Since this arrangement is significantly different from the ioat1 chain, move ioat2,3 support into its own file and header. Common routines are exported from driver/dma/ioat/dma.[ch]. Signed-off-by: Maciej Sosnowski <maciej.sosnowski@intel.com> Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Diffstat (limited to 'drivers/dma')
-rw-r--r--drivers/dma/ioat/Makefile2
-rw-r--r--drivers/dma/ioat/dma.c874
-rw-r--r--drivers/dma/ioat/dma.h50
-rw-r--r--drivers/dma/ioat/dma_v2.c750
-rw-r--r--drivers/dma/ioat/dma_v2.h131
-rw-r--r--drivers/dma/ioat/pci.c1
6 files changed, 1122 insertions, 686 deletions
diff --git a/drivers/dma/ioat/Makefile b/drivers/dma/ioat/Makefile
index 2ce3d3a4270b..205a639e84df 100644
--- a/drivers/dma/ioat/Makefile
+++ b/drivers/dma/ioat/Makefile
@@ -1,2 +1,2 @@
1obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o 1obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
2ioatdma-objs := pci.o dma.o dca.o 2ioatdma-objs := pci.o dma.o dma_v2.o dca.o
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 2e81e0c76e61..64b4d75059aa 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -38,28 +38,14 @@
38#include "registers.h" 38#include "registers.h"
39#include "hw.h" 39#include "hw.h"
40 40
41static int ioat_pending_level = 4; 41int ioat_pending_level = 4;
42module_param(ioat_pending_level, int, 0644); 42module_param(ioat_pending_level, int, 0644);
43MODULE_PARM_DESC(ioat_pending_level, 43MODULE_PARM_DESC(ioat_pending_level,
44 "high-water mark for pushing ioat descriptors (default: 4)"); 44 "high-water mark for pushing ioat descriptors (default: 4)");
45 45
46static void ioat_dma_chan_reset_part2(struct work_struct *work);
47static void ioat_dma_chan_watchdog(struct work_struct *work);
48
49/* internal functions */ 46/* internal functions */
50static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat); 47static void ioat1_cleanup(struct ioat_dma_chan *ioat);
51static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat); 48static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat);
52
53static struct ioat_desc_sw *
54ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat);
55static struct ioat_desc_sw *
56ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat);
57
58static inline struct ioat_chan_common *
59ioat_chan_by_index(struct ioatdma_device *device, int index)
60{
61 return device->idx[index];
62}
63 49
64/** 50/**
65 * ioat_dma_do_interrupt - handler used for single vector interrupt mode 51 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
@@ -108,18 +94,38 @@ static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
108 return IRQ_HANDLED; 94 return IRQ_HANDLED;
109} 95}
110 96
111static void ioat_dma_cleanup_tasklet(unsigned long data); 97static void ioat1_cleanup_tasklet(unsigned long data);
98
99/* common channel initialization */
100void ioat_init_channel(struct ioatdma_device *device,
101 struct ioat_chan_common *chan, int idx,
102 work_func_t work_fn, void (*tasklet)(unsigned long),
103 unsigned long tasklet_data)
104{
105 struct dma_device *dma = &device->common;
106
107 chan->device = device;
108 chan->reg_base = device->reg_base + (0x80 * (idx + 1));
109 INIT_DELAYED_WORK(&chan->work, work_fn);
110 spin_lock_init(&chan->cleanup_lock);
111 chan->common.device = dma;
112 list_add_tail(&chan->common.device_node, &dma->channels);
113 device->idx[idx] = chan;
114 tasklet_init(&chan->cleanup_task, tasklet, tasklet_data);
115 tasklet_disable(&chan->cleanup_task);
116}
117
118static void ioat1_reset_part2(struct work_struct *work);
112 119
113/** 120/**
114 * ioat_dma_enumerate_channels - find and initialize the device's channels 121 * ioat1_dma_enumerate_channels - find and initialize the device's channels
115 * @device: the device to be enumerated 122 * @device: the device to be enumerated
116 */ 123 */
117static int ioat_dma_enumerate_channels(struct ioatdma_device *device) 124static int ioat1_enumerate_channels(struct ioatdma_device *device)
118{ 125{
119 u8 xfercap_scale; 126 u8 xfercap_scale;
120 u32 xfercap; 127 u32 xfercap;
121 int i; 128 int i;
122 struct ioat_chan_common *chan;
123 struct ioat_dma_chan *ioat; 129 struct ioat_dma_chan *ioat;
124 struct device *dev = &device->pdev->dev; 130 struct device *dev = &device->pdev->dev;
125 struct dma_device *dma = &device->common; 131 struct dma_device *dma = &device->common;
@@ -135,31 +141,20 @@ static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
135#endif 141#endif
136 for (i = 0; i < dma->chancnt; i++) { 142 for (i = 0; i < dma->chancnt; i++) {
137 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL); 143 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
138 if (!ioat) { 144 if (!ioat)
139 dma->chancnt = i;
140 break; 145 break;
141 }
142 146
143 chan = &ioat->base; 147 ioat_init_channel(device, &ioat->base, i,
144 chan->device = device; 148 ioat1_reset_part2,
145 chan->reg_base = device->reg_base + (0x80 * (i + 1)); 149 ioat1_cleanup_tasklet,
150 (unsigned long) ioat);
146 ioat->xfercap = xfercap; 151 ioat->xfercap = xfercap;
147 ioat->desccount = 0;
148 INIT_DELAYED_WORK(&chan->work, ioat_dma_chan_reset_part2);
149 spin_lock_init(&chan->cleanup_lock);
150 spin_lock_init(&ioat->desc_lock); 152 spin_lock_init(&ioat->desc_lock);
151 INIT_LIST_HEAD(&ioat->free_desc); 153 INIT_LIST_HEAD(&ioat->free_desc);
152 INIT_LIST_HEAD(&ioat->used_desc); 154 INIT_LIST_HEAD(&ioat->used_desc);
153 /* This should be made common somewhere in dmaengine.c */
154 chan->common.device = &device->common;
155 list_add_tail(&chan->common.device_node, &dma->channels);
156 device->idx[i] = chan;
157 tasklet_init(&chan->cleanup_task,
158 ioat_dma_cleanup_tasklet,
159 (unsigned long) ioat);
160 tasklet_disable(&chan->cleanup_task);
161 } 155 }
162 return dma->chancnt; 156 dma->chancnt = i;
157 return i;
163} 158}
164 159
165/** 160/**
@@ -187,35 +182,16 @@ static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
187 } 182 }
188} 183}
189 184
190static inline void
191__ioat2_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
192{
193 void __iomem *reg_base = ioat->base.reg_base;
194
195 ioat->pending = 0;
196 writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
197}
198
199static void ioat2_dma_memcpy_issue_pending(struct dma_chan *chan)
200{
201 struct ioat_dma_chan *ioat = to_ioat_chan(chan);
202
203 if (ioat->pending > 0) {
204 spin_lock_bh(&ioat->desc_lock);
205 __ioat2_dma_memcpy_issue_pending(ioat);
206 spin_unlock_bh(&ioat->desc_lock);
207 }
208}
209
210
211/** 185/**
212 * ioat_dma_chan_reset_part2 - reinit the channel after a reset 186 * ioat1_reset_part2 - reinit the channel after a reset
213 */ 187 */
214static void ioat_dma_chan_reset_part2(struct work_struct *work) 188static void ioat1_reset_part2(struct work_struct *work)
215{ 189{
216 struct ioat_chan_common *chan; 190 struct ioat_chan_common *chan;
217 struct ioat_dma_chan *ioat; 191 struct ioat_dma_chan *ioat;
218 struct ioat_desc_sw *desc; 192 struct ioat_desc_sw *desc;
193 int dmacount;
194 bool start_null = false;
219 195
220 chan = container_of(work, struct ioat_chan_common, work.work); 196 chan = container_of(work, struct ioat_chan_common, work.work);
221 ioat = container_of(chan, struct ioat_dma_chan, base); 197 ioat = container_of(chan, struct ioat_dma_chan, base);
@@ -226,26 +202,22 @@ static void ioat_dma_chan_reset_part2(struct work_struct *work)
226 chan->completion_virt->high = 0; 202 chan->completion_virt->high = 0;
227 ioat->pending = 0; 203 ioat->pending = 0;
228 204
229 /* 205 /* count the descriptors waiting */
230 * count the descriptors waiting, and be sure to do it 206 dmacount = 0;
231 * right for both the CB1 line and the CB2 ring
232 */
233 ioat->dmacount = 0;
234 if (ioat->used_desc.prev) { 207 if (ioat->used_desc.prev) {
235 desc = to_ioat_desc(ioat->used_desc.prev); 208 desc = to_ioat_desc(ioat->used_desc.prev);
236 do { 209 do {
237 ioat->dmacount++; 210 dmacount++;
238 desc = to_ioat_desc(desc->node.next); 211 desc = to_ioat_desc(desc->node.next);
239 } while (&desc->node != ioat->used_desc.next); 212 } while (&desc->node != ioat->used_desc.next);
240 } 213 }
241 214
242 /* 215 if (dmacount) {
243 * write the new starting descriptor address 216 /*
244 * this puts channel engine into ARMED state 217 * write the new starting descriptor address
245 */ 218 * this puts channel engine into ARMED state
246 desc = to_ioat_desc(ioat->used_desc.prev); 219 */
247 switch (chan->device->version) { 220 desc = to_ioat_desc(ioat->used_desc.prev);
248 case IOAT_VER_1_2:
249 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF, 221 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
250 chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW); 222 chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
251 writel(((u64) desc->txd.phys) >> 32, 223 writel(((u64) desc->txd.phys) >> 32,
@@ -253,32 +225,24 @@ static void ioat_dma_chan_reset_part2(struct work_struct *work)
253 225
254 writeb(IOAT_CHANCMD_START, chan->reg_base 226 writeb(IOAT_CHANCMD_START, chan->reg_base
255 + IOAT_CHANCMD_OFFSET(chan->device->version)); 227 + IOAT_CHANCMD_OFFSET(chan->device->version));
256 break; 228 } else
257 case IOAT_VER_2_0: 229 start_null = true;
258 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF, 230 spin_unlock_bh(&ioat->desc_lock);
259 chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW); 231 spin_unlock_bh(&chan->cleanup_lock);
260 writel(((u64) desc->txd.phys) >> 32,
261 chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
262
263 /* tell the engine to go with what's left to be done */
264 writew(ioat->dmacount,
265 chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
266 232
267 break;
268 }
269 dev_err(to_dev(chan), 233 dev_err(to_dev(chan),
270 "chan%d reset - %d descs waiting, %d total desc\n", 234 "chan%d reset - %d descs waiting, %d total desc\n",
271 chan_num(chan), ioat->dmacount, ioat->desccount); 235 chan_num(chan), dmacount, ioat->desccount);
272 236
273 spin_unlock_bh(&ioat->desc_lock); 237 if (start_null)
274 spin_unlock_bh(&chan->cleanup_lock); 238 ioat1_dma_start_null_desc(ioat);
275} 239}
276 240
277/** 241/**
278 * ioat_dma_reset_channel - restart a channel 242 * ioat1_reset_channel - restart a channel
279 * @ioat: IOAT DMA channel handle 243 * @ioat: IOAT DMA channel handle
280 */ 244 */
281static void ioat_dma_reset_channel(struct ioat_dma_chan *ioat) 245static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
282{ 246{
283 struct ioat_chan_common *chan = &ioat->base; 247 struct ioat_chan_common *chan = &ioat->base;
284 void __iomem *reg_base = chan->reg_base; 248 void __iomem *reg_base = chan->reg_base;
@@ -316,9 +280,9 @@ static void ioat_dma_reset_channel(struct ioat_dma_chan *ioat)
316} 280}
317 281
318/** 282/**
319 * ioat_dma_chan_watchdog - watch for stuck channels 283 * ioat1_chan_watchdog - watch for stuck channels
320 */ 284 */
321static void ioat_dma_chan_watchdog(struct work_struct *work) 285static void ioat1_chan_watchdog(struct work_struct *work)
322{ 286{
323 struct ioatdma_device *device = 287 struct ioatdma_device *device =
324 container_of(work, struct ioatdma_device, work.work); 288 container_of(work, struct ioatdma_device, work.work);
@@ -339,16 +303,15 @@ static void ioat_dma_chan_watchdog(struct work_struct *work)
339 chan = ioat_chan_by_index(device, i); 303 chan = ioat_chan_by_index(device, i);
340 ioat = container_of(chan, struct ioat_dma_chan, base); 304 ioat = container_of(chan, struct ioat_dma_chan, base);
341 305
342 if (chan->device->version == IOAT_VER_1_2 306 if (/* have we started processing anything yet */
343 /* have we started processing anything yet */ 307 chan->last_completion
344 && chan->last_completion 308 /* have we completed any since last watchdog cycle? */
345 /* have we completed any since last watchdog cycle? */
346 && (chan->last_completion == chan->watchdog_completion) 309 && (chan->last_completion == chan->watchdog_completion)
347 /* has TCP stuck on one cookie since last watchdog? */ 310 /* has TCP stuck on one cookie since last watchdog? */
348 && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie) 311 && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie)
349 && (chan->watchdog_tcp_cookie != chan->completed_cookie) 312 && (chan->watchdog_tcp_cookie != chan->completed_cookie)
350 /* is there something in the chain to be processed? */ 313 /* is there something in the chain to be processed? */
351 /* CB1 chain always has at least the last one processed */ 314 /* CB1 chain always has at least the last one processed */
352 && (ioat->used_desc.prev != ioat->used_desc.next) 315 && (ioat->used_desc.prev != ioat->used_desc.next)
353 && ioat->pending == 0) { 316 && ioat->pending == 0) {
354 317
@@ -387,34 +350,15 @@ static void ioat_dma_chan_watchdog(struct work_struct *work)
387 chan->completion_virt->low = completion_hw.low; 350 chan->completion_virt->low = completion_hw.low;
388 chan->completion_virt->high = completion_hw.high; 351 chan->completion_virt->high = completion_hw.high;
389 } else { 352 } else {
390 ioat_dma_reset_channel(ioat); 353 ioat1_reset_channel(ioat);
391 chan->watchdog_completion = 0; 354 chan->watchdog_completion = 0;
392 chan->last_compl_desc_addr_hw = 0; 355 chan->last_compl_desc_addr_hw = 0;
393 } 356 }
394
395 /*
396 * for version 2.0 if there are descriptors yet to be processed
397 * and the last completed hasn't changed since the last watchdog
398 * if they haven't hit the pending level
399 * issue the pending to push them through
400 * else
401 * try resetting the channel
402 */
403 } else if (chan->device->version == IOAT_VER_2_0
404 && ioat->used_desc.prev
405 && chan->last_completion
406 && chan->last_completion == chan->watchdog_completion) {
407
408 if (ioat->pending < ioat_pending_level)
409 ioat2_dma_memcpy_issue_pending(&chan->common);
410 else {
411 ioat_dma_reset_channel(ioat);
412 chan->watchdog_completion = 0;
413 }
414 } else { 357 } else {
415 chan->last_compl_desc_addr_hw = 0; 358 chan->last_compl_desc_addr_hw = 0;
416 chan->watchdog_completion = chan->last_completion; 359 chan->watchdog_completion = chan->last_completion;
417 } 360 }
361
418 chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie; 362 chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
419 } 363 }
420 364
@@ -447,7 +391,6 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
447 chain_tail->hw->next = first->txd.phys; 391 chain_tail->hw->next = first->txd.phys;
448 list_splice_tail_init(&tx->tx_list, &ioat->used_desc); 392 list_splice_tail_init(&tx->tx_list, &ioat->used_desc);
449 393
450 ioat->dmacount += desc->tx_cnt;
451 ioat->pending += desc->tx_cnt; 394 ioat->pending += desc->tx_cnt;
452 if (ioat->pending >= ioat_pending_level) 395 if (ioat->pending >= ioat_pending_level)
453 __ioat1_dma_memcpy_issue_pending(ioat); 396 __ioat1_dma_memcpy_issue_pending(ioat);
@@ -456,92 +399,6 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
456 return cookie; 399 return cookie;
457} 400}
458 401
459static dma_cookie_t ioat2_tx_submit(struct dma_async_tx_descriptor *tx)
460{
461 struct ioat_dma_chan *ioat = to_ioat_chan(tx->chan);
462 struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
463 struct ioat_desc_sw *new;
464 struct ioat_dma_descriptor *hw;
465 dma_cookie_t cookie;
466 u32 copy;
467 size_t len;
468 dma_addr_t src, dst;
469 unsigned long orig_flags;
470 unsigned int desc_count = 0;
471
472 /* src and dest and len are stored in the initial descriptor */
473 len = first->len;
474 src = first->src;
475 dst = first->dst;
476 orig_flags = first->txd.flags;
477 new = first;
478
479 /*
480 * ioat->desc_lock is still in force in version 2 path
481 * it gets unlocked at end of this function
482 */
483 do {
484 copy = min_t(size_t, len, ioat->xfercap);
485
486 async_tx_ack(&new->txd);
487
488 hw = new->hw;
489 hw->size = copy;
490 hw->ctl = 0;
491 hw->src_addr = src;
492 hw->dst_addr = dst;
493
494 len -= copy;
495 dst += copy;
496 src += copy;
497 desc_count++;
498 } while (len && (new = ioat2_dma_get_next_descriptor(ioat)));
499
500 if (!new) {
501 dev_err(to_dev(&ioat->base), "tx submit failed\n");
502 spin_unlock_bh(&ioat->desc_lock);
503 return -ENOMEM;
504 }
505
506 hw->ctl_f.compl_write = 1;
507 if (first->txd.callback) {
508 hw->ctl_f.int_en = 1;
509 if (first != new) {
510 /* move callback into to last desc */
511 new->txd.callback = first->txd.callback;
512 new->txd.callback_param
513 = first->txd.callback_param;
514 first->txd.callback = NULL;
515 first->txd.callback_param = NULL;
516 }
517 }
518
519 new->tx_cnt = desc_count;
520 new->txd.flags = orig_flags; /* client is in control of this ack */
521
522 /* store the original values for use in later cleanup */
523 if (new != first) {
524 new->src = first->src;
525 new->dst = first->dst;
526 new->len = first->len;
527 }
528
529 /* cookie incr and addition to used_list must be atomic */
530 cookie = ioat->base.common.cookie;
531 cookie++;
532 if (cookie < 0)
533 cookie = 1;
534 ioat->base.common.cookie = new->txd.cookie = cookie;
535
536 ioat->dmacount += desc_count;
537 ioat->pending += desc_count;
538 if (ioat->pending >= ioat_pending_level)
539 __ioat2_dma_memcpy_issue_pending(ioat);
540 spin_unlock_bh(&ioat->desc_lock);
541
542 return cookie;
543}
544
545/** 402/**
546 * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair 403 * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
547 * @ioat: the channel supplying the memory pool for the descriptors 404 * @ioat: the channel supplying the memory pool for the descriptors
@@ -567,17 +424,9 @@ ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
567 } 424 }
568 425
569 memset(desc, 0, sizeof(*desc)); 426 memset(desc, 0, sizeof(*desc));
570 dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
571 switch (ioatdma_device->version) {
572 case IOAT_VER_1_2:
573 desc_sw->txd.tx_submit = ioat1_tx_submit;
574 break;
575 case IOAT_VER_2_0:
576 case IOAT_VER_3_0:
577 desc_sw->txd.tx_submit = ioat2_tx_submit;
578 break;
579 }
580 427
428 dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
429 desc_sw->txd.tx_submit = ioat1_tx_submit;
581 desc_sw->hw = desc; 430 desc_sw->hw = desc;
582 desc_sw->txd.phys = phys; 431 desc_sw->txd.phys = phys;
583 432
@@ -587,39 +436,12 @@ ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
587static int ioat_initial_desc_count = 256; 436static int ioat_initial_desc_count = 256;
588module_param(ioat_initial_desc_count, int, 0644); 437module_param(ioat_initial_desc_count, int, 0644);
589MODULE_PARM_DESC(ioat_initial_desc_count, 438MODULE_PARM_DESC(ioat_initial_desc_count,
590 "initial descriptors per channel (default: 256)"); 439 "ioat1: initial descriptors per channel (default: 256)");
591
592/**
593 * ioat2_dma_massage_chan_desc - link the descriptors into a circle
594 * @ioat: the channel to be massaged
595 */
596static void ioat2_dma_massage_chan_desc(struct ioat_dma_chan *ioat)
597{
598 struct ioat_desc_sw *desc, *_desc;
599
600 /* setup used_desc */
601 ioat->used_desc.next = ioat->free_desc.next;
602 ioat->used_desc.prev = NULL;
603
604 /* pull free_desc out of the circle so that every node is a hw
605 * descriptor, but leave it pointing to the list
606 */
607 ioat->free_desc.prev->next = ioat->free_desc.next;
608 ioat->free_desc.next->prev = ioat->free_desc.prev;
609
610 /* circle link the hw descriptors */
611 desc = to_ioat_desc(ioat->free_desc.next);
612 desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
613 list_for_each_entry_safe(desc, _desc, ioat->free_desc.next, node) {
614 desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
615 }
616}
617
618/** 440/**
619 * ioat_dma_alloc_chan_resources - returns the number of allocated descriptors 441 * ioat1_dma_alloc_chan_resources - returns the number of allocated descriptors
620 * @chan: the channel to be filled out 442 * @chan: the channel to be filled out
621 */ 443 */
622static int ioat_dma_alloc_chan_resources(struct dma_chan *c) 444static int ioat1_dma_alloc_chan_resources(struct dma_chan *c)
623{ 445{
624 struct ioat_dma_chan *ioat = to_ioat_chan(c); 446 struct ioat_dma_chan *ioat = to_ioat_chan(c);
625 struct ioat_chan_common *chan = &ioat->base; 447 struct ioat_chan_common *chan = &ioat->base;
@@ -657,8 +479,6 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *c)
657 spin_lock_bh(&ioat->desc_lock); 479 spin_lock_bh(&ioat->desc_lock);
658 ioat->desccount = i; 480 ioat->desccount = i;
659 list_splice(&tmp_list, &ioat->free_desc); 481 list_splice(&tmp_list, &ioat->free_desc);
660 if (chan->device->version != IOAT_VER_1_2)
661 ioat2_dma_massage_chan_desc(ioat);
662 spin_unlock_bh(&ioat->desc_lock); 482 spin_unlock_bh(&ioat->desc_lock);
663 483
664 /* allocate a completion writeback area */ 484 /* allocate a completion writeback area */
@@ -674,15 +494,15 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *c)
674 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); 494 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
675 495
676 tasklet_enable(&chan->cleanup_task); 496 tasklet_enable(&chan->cleanup_task);
677 ioat_dma_start_null_desc(ioat); /* give chain to dma device */ 497 ioat1_dma_start_null_desc(ioat); /* give chain to dma device */
678 return ioat->desccount; 498 return ioat->desccount;
679} 499}
680 500
681/** 501/**
682 * ioat_dma_free_chan_resources - release all the descriptors 502 * ioat1_dma_free_chan_resources - release all the descriptors
683 * @chan: the channel to be cleaned 503 * @chan: the channel to be cleaned
684 */ 504 */
685static void ioat_dma_free_chan_resources(struct dma_chan *c) 505static void ioat1_dma_free_chan_resources(struct dma_chan *c)
686{ 506{
687 struct ioat_dma_chan *ioat = to_ioat_chan(c); 507 struct ioat_dma_chan *ioat = to_ioat_chan(c);
688 struct ioat_chan_common *chan = &ioat->base; 508 struct ioat_chan_common *chan = &ioat->base;
@@ -697,7 +517,7 @@ static void ioat_dma_free_chan_resources(struct dma_chan *c)
697 return; 517 return;
698 518
699 tasklet_disable(&chan->cleanup_task); 519 tasklet_disable(&chan->cleanup_task);
700 ioat_dma_memcpy_cleanup(ioat); 520 ioat1_cleanup(ioat);
701 521
702 /* Delay 100ms after reset to allow internal DMA logic to quiesce 522 /* Delay 100ms after reset to allow internal DMA logic to quiesce
703 * before removing DMA descriptor resources. 523 * before removing DMA descriptor resources.
@@ -707,40 +527,20 @@ static void ioat_dma_free_chan_resources(struct dma_chan *c)
707 mdelay(100); 527 mdelay(100);
708 528
709 spin_lock_bh(&ioat->desc_lock); 529 spin_lock_bh(&ioat->desc_lock);
710 switch (chan->device->version) { 530 list_for_each_entry_safe(desc, _desc,
711 case IOAT_VER_1_2: 531 &ioat->used_desc, node) {
712 list_for_each_entry_safe(desc, _desc, 532 in_use_descs++;
713 &ioat->used_desc, node) { 533 list_del(&desc->node);
714 in_use_descs++; 534 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
715 list_del(&desc->node); 535 desc->txd.phys);
716 pci_pool_free(ioatdma_device->dma_pool, desc->hw, 536 kfree(desc);
717 desc->txd.phys); 537 }
718 kfree(desc); 538 list_for_each_entry_safe(desc, _desc,
719 } 539 &ioat->free_desc, node) {
720 list_for_each_entry_safe(desc, _desc, 540 list_del(&desc->node);
721 &ioat->free_desc, node) {
722 list_del(&desc->node);
723 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
724 desc->txd.phys);
725 kfree(desc);
726 }
727 break;
728 case IOAT_VER_2_0:
729 case IOAT_VER_3_0:
730 list_for_each_entry_safe(desc, _desc,
731 ioat->free_desc.next, node) {
732 list_del(&desc->node);
733 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
734 desc->txd.phys);
735 kfree(desc);
736 }
737 desc = to_ioat_desc(ioat->free_desc.next);
738 pci_pool_free(ioatdma_device->dma_pool, desc->hw, 541 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
739 desc->txd.phys); 542 desc->txd.phys);
740 kfree(desc); 543 kfree(desc);
741 INIT_LIST_HEAD(&ioat->free_desc);
742 INIT_LIST_HEAD(&ioat->used_desc);
743 break;
744 } 544 }
745 spin_unlock_bh(&ioat->desc_lock); 545 spin_unlock_bh(&ioat->desc_lock);
746 546
@@ -758,7 +558,6 @@ static void ioat_dma_free_chan_resources(struct dma_chan *c)
758 chan->last_compl_desc_addr_hw = 0; 558 chan->last_compl_desc_addr_hw = 0;
759 chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0; 559 chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
760 ioat->pending = 0; 560 ioat->pending = 0;
761 ioat->dmacount = 0;
762 ioat->desccount = 0; 561 ioat->desccount = 0;
763} 562}
764 563
@@ -791,86 +590,6 @@ ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
791 return new; 590 return new;
792} 591}
793 592
794static struct ioat_desc_sw *
795ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
796{
797 struct ioat_desc_sw *new;
798
799 /*
800 * used.prev points to where to start processing
801 * used.next points to next free descriptor
802 * if used.prev == NULL, there are none waiting to be processed
803 * if used.next == used.prev.prev, there is only one free descriptor,
804 * and we need to use it to as a noop descriptor before
805 * linking in a new set of descriptors, since the device
806 * has probably already read the pointer to it
807 */
808 if (ioat->used_desc.prev &&
809 ioat->used_desc.next == ioat->used_desc.prev->prev) {
810
811 struct ioat_desc_sw *desc;
812 struct ioat_desc_sw *noop_desc;
813 int i;
814
815 /* set up the noop descriptor */
816 noop_desc = to_ioat_desc(ioat->used_desc.next);
817 /* set size to non-zero value (channel returns error when size is 0) */
818 noop_desc->hw->size = NULL_DESC_BUFFER_SIZE;
819 noop_desc->hw->ctl = 0;
820 noop_desc->hw->ctl_f.null = 1;
821 noop_desc->hw->src_addr = 0;
822 noop_desc->hw->dst_addr = 0;
823
824 ioat->used_desc.next = ioat->used_desc.next->next;
825 ioat->pending++;
826 ioat->dmacount++;
827
828 /* try to get a few more descriptors */
829 for (i = 16; i; i--) {
830 desc = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
831 if (!desc) {
832 dev_err(to_dev(&ioat->base),
833 "alloc failed\n");
834 break;
835 }
836 list_add_tail(&desc->node, ioat->used_desc.next);
837
838 desc->hw->next
839 = to_ioat_desc(desc->node.next)->txd.phys;
840 to_ioat_desc(desc->node.prev)->hw->next
841 = desc->txd.phys;
842 ioat->desccount++;
843 }
844
845 ioat->used_desc.next = noop_desc->node.next;
846 }
847 new = to_ioat_desc(ioat->used_desc.next);
848 prefetch(new);
849 ioat->used_desc.next = new->node.next;
850
851 if (ioat->used_desc.prev == NULL)
852 ioat->used_desc.prev = &new->node;
853
854 prefetch(new->hw);
855 return new;
856}
857
858static struct ioat_desc_sw *
859ioat_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
860{
861 if (!ioat)
862 return NULL;
863
864 switch (ioat->base.device->version) {
865 case IOAT_VER_1_2:
866 return ioat1_dma_get_next_descriptor(ioat);
867 case IOAT_VER_2_0:
868 case IOAT_VER_3_0:
869 return ioat2_dma_get_next_descriptor(ioat);
870 }
871 return NULL;
872}
873
874static struct dma_async_tx_descriptor * 593static struct dma_async_tx_descriptor *
875ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest, 594ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
876 dma_addr_t dma_src, size_t len, unsigned long flags) 595 dma_addr_t dma_src, size_t len, unsigned long flags)
@@ -886,7 +605,7 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
886 int tx_cnt = 0; 605 int tx_cnt = 0;
887 606
888 spin_lock_bh(&ioat->desc_lock); 607 spin_lock_bh(&ioat->desc_lock);
889 desc = ioat_dma_get_next_descriptor(ioat); 608 desc = ioat1_dma_get_next_descriptor(ioat);
890 do { 609 do {
891 if (!desc) 610 if (!desc)
892 break; 611 break;
@@ -909,7 +628,7 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
909 struct ioat_desc_sw *next; 628 struct ioat_desc_sw *next;
910 629
911 async_tx_ack(&desc->txd); 630 async_tx_ack(&desc->txd);
912 next = ioat_dma_get_next_descriptor(ioat); 631 next = ioat1_dma_get_next_descriptor(ioat);
913 hw->next = next ? next->txd.phys : 0; 632 hw->next = next ? next->txd.phys : 0;
914 desc = next; 633 desc = next;
915 } else 634 } else
@@ -920,8 +639,7 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
920 struct ioat_chan_common *chan = &ioat->base; 639 struct ioat_chan_common *chan = &ioat->base;
921 640
922 dev_err(to_dev(chan), 641 dev_err(to_dev(chan),
923 "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n", 642 "chan%d - get_next_desc failed\n", chan_num(chan));
924 chan_num(chan), ioat->dmacount, ioat->desccount);
925 list_splice(&chain, &ioat->free_desc); 643 list_splice(&chain, &ioat->free_desc);
926 spin_unlock_bh(&ioat->desc_lock); 644 spin_unlock_bh(&ioat->desc_lock);
927 return NULL; 645 return NULL;
@@ -940,94 +658,43 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
940 return &desc->txd; 658 return &desc->txd;
941} 659}
942 660
943static struct dma_async_tx_descriptor * 661static void ioat1_cleanup_tasklet(unsigned long data)
944ioat2_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
945 dma_addr_t dma_src, size_t len, unsigned long flags)
946{
947 struct ioat_dma_chan *ioat = to_ioat_chan(c);
948 struct ioat_desc_sw *new;
949
950 spin_lock_bh(&ioat->desc_lock);
951 new = ioat2_dma_get_next_descriptor(ioat);
952
953 /*
954 * leave ioat->desc_lock set in ioat 2 path
955 * it will get unlocked at end of tx_submit
956 */
957
958 if (new) {
959 new->len = len;
960 new->dst = dma_dest;
961 new->src = dma_src;
962 new->txd.flags = flags;
963 return &new->txd;
964 } else {
965 struct ioat_chan_common *chan = &ioat->base;
966
967 spin_unlock_bh(&ioat->desc_lock);
968 dev_err(to_dev(chan),
969 "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
970 chan_num(chan), ioat->dmacount, ioat->desccount);
971 return NULL;
972 }
973}
974
975static void ioat_dma_cleanup_tasklet(unsigned long data)
976{ 662{
977 struct ioat_dma_chan *chan = (void *)data; 663 struct ioat_dma_chan *chan = (void *)data;
978 ioat_dma_memcpy_cleanup(chan); 664 ioat1_cleanup(chan);
979 writew(IOAT_CHANCTRL_INT_DISABLE, 665 writew(IOAT_CHANCTRL_INT_DISABLE,
980 chan->base.reg_base + IOAT_CHANCTRL_OFFSET); 666 chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
981} 667}
982 668
983static void 669static void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
984ioat_dma_unmap(struct ioat_chan_common *chan, struct ioat_desc_sw *desc) 670 int direction, enum dma_ctrl_flags flags, bool dst)
985{ 671{
986 if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { 672 if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) ||
987 if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE) 673 (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE)))
988 pci_unmap_single(chan->device->pdev, 674 pci_unmap_single(pdev, addr, len, direction);
989 pci_unmap_addr(desc, dst), 675 else
990 pci_unmap_len(desc, len), 676 pci_unmap_page(pdev, addr, len, direction);
991 PCI_DMA_FROMDEVICE);
992 else
993 pci_unmap_page(chan->device->pdev,
994 pci_unmap_addr(desc, dst),
995 pci_unmap_len(desc, len),
996 PCI_DMA_FROMDEVICE);
997 }
998
999 if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1000 if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
1001 pci_unmap_single(chan->device->pdev,
1002 pci_unmap_addr(desc, src),
1003 pci_unmap_len(desc, len),
1004 PCI_DMA_TODEVICE);
1005 else
1006 pci_unmap_page(chan->device->pdev,
1007 pci_unmap_addr(desc, src),
1008 pci_unmap_len(desc, len),
1009 PCI_DMA_TODEVICE);
1010 }
1011} 677}
1012 678
1013/** 679
1014 * ioat_dma_memcpy_cleanup - cleanup up finished descriptors 680void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
1015 * @chan: ioat channel to be cleaned up 681 size_t len, struct ioat_dma_descriptor *hw)
1016 */
1017static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat)
1018{ 682{
1019 struct ioat_chan_common *chan = &ioat->base; 683 struct pci_dev *pdev = chan->device->pdev;
1020 unsigned long phys_complete; 684 size_t offset = len - hw->size;
1021 struct ioat_desc_sw *desc, *_desc;
1022 dma_cookie_t cookie = 0;
1023 unsigned long desc_phys;
1024 struct ioat_desc_sw *latest_desc;
1025 struct dma_async_tx_descriptor *tx;
1026 685
1027 prefetch(chan->completion_virt); 686 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
687 ioat_unmap(pdev, hw->dst_addr - offset, len,
688 PCI_DMA_FROMDEVICE, flags, 1);
1028 689
1029 if (!spin_trylock_bh(&chan->cleanup_lock)) 690 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP))
1030 return; 691 ioat_unmap(pdev, hw->src_addr - offset, len,
692 PCI_DMA_TODEVICE, flags, 0);
693}
694
695unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
696{
697 unsigned long phys_complete;
1031 698
1032 /* The completion writeback can happen at any time, 699 /* The completion writeback can happen at any time,
1033 so reads by the driver need to be atomic operations 700 so reads by the driver need to be atomic operations
@@ -1051,18 +718,37 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat)
1051 /* TODO do something to salvage the situation */ 718 /* TODO do something to salvage the situation */
1052 } 719 }
1053 720
721 return phys_complete;
722}
723
724/**
725 * ioat1_cleanup - cleanup up finished descriptors
726 * @chan: ioat channel to be cleaned up
727 */
728static void ioat1_cleanup(struct ioat_dma_chan *ioat)
729{
730 struct ioat_chan_common *chan = &ioat->base;
731 unsigned long phys_complete;
732 struct ioat_desc_sw *desc, *_desc;
733 dma_cookie_t cookie = 0;
734 struct dma_async_tx_descriptor *tx;
735
736 prefetch(chan->completion_virt);
737
738 if (!spin_trylock_bh(&chan->cleanup_lock))
739 return;
740
741 phys_complete = ioat_get_current_completion(chan);
1054 if (phys_complete == chan->last_completion) { 742 if (phys_complete == chan->last_completion) {
1055 spin_unlock_bh(&chan->cleanup_lock); 743 spin_unlock_bh(&chan->cleanup_lock);
1056 /* 744 /*
1057 * perhaps we're stuck so hard that the watchdog can't go off? 745 * perhaps we're stuck so hard that the watchdog can't go off?
1058 * try to catch it after 2 seconds 746 * try to catch it after 2 seconds
1059 */ 747 */
1060 if (chan->device->version != IOAT_VER_3_0) { 748 if (time_after(jiffies,
1061 if (time_after(jiffies, 749 chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
1062 chan->last_completion_time + HZ*WATCHDOG_DELAY)) { 750 ioat1_chan_watchdog(&(chan->device->work.work));
1063 ioat_dma_chan_watchdog(&(chan->device->work.work)); 751 chan->last_completion_time = jiffies;
1064 chan->last_completion_time = jiffies;
1065 }
1066 } 752 }
1067 return; 753 return;
1068 } 754 }
@@ -1074,91 +760,42 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat)
1074 return; 760 return;
1075 } 761 }
1076 762
1077 switch (chan->device->version) { 763 list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
1078 case IOAT_VER_1_2: 764 tx = &desc->txd;
1079 list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) { 765 /*
1080 tx = &desc->txd; 766 * Incoming DMA requests may use multiple descriptors,
1081 /* 767 * due to exceeding xfercap, perhaps. If so, only the
1082 * Incoming DMA requests may use multiple descriptors, 768 * last one will have a cookie, and require unmapping.
1083 * due to exceeding xfercap, perhaps. If so, only the 769 */
1084 * last one will have a cookie, and require unmapping. 770 if (tx->cookie) {
1085 */ 771 cookie = tx->cookie;
1086 if (tx->cookie) { 772 ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
1087 cookie = tx->cookie; 773 if (tx->callback) {
1088 ioat_dma_unmap(chan, desc); 774 tx->callback(tx->callback_param);
1089 if (tx->callback) { 775 tx->callback = NULL;
1090 tx->callback(tx->callback_param);
1091 tx->callback = NULL;
1092 }
1093 } 776 }
777 }
1094 778
1095 if (tx->phys != phys_complete) { 779 if (tx->phys != phys_complete) {
1096 /* 780 /*
1097 * a completed entry, but not the last, so clean 781 * a completed entry, but not the last, so clean
1098 * up if the client is done with the descriptor 782 * up if the client is done with the descriptor
1099 */ 783 */
1100 if (async_tx_test_ack(tx)) { 784 if (async_tx_test_ack(tx))
1101 list_move_tail(&desc->node, 785 list_move_tail(&desc->node, &ioat->free_desc);
1102 &ioat->free_desc); 786 else
1103 } else
1104 tx->cookie = 0;
1105 } else {
1106 /*
1107 * last used desc. Do not remove, so we can
1108 * append from it, but don't look at it next
1109 * time, either
1110 */
1111 tx->cookie = 0; 787 tx->cookie = 0;
788 } else {
789 /*
790 * last used desc. Do not remove, so we can
791 * append from it, but don't look at it next
792 * time, either
793 */
794 tx->cookie = 0;
1112 795
1113 /* TODO check status bits? */ 796 /* TODO check status bits? */
1114 break;
1115 }
1116 }
1117 break;
1118 case IOAT_VER_2_0:
1119 case IOAT_VER_3_0:
1120 /* has some other thread has already cleaned up? */
1121 if (ioat->used_desc.prev == NULL)
1122 break; 797 break;
1123
1124 /* work backwards to find latest finished desc */
1125 desc = to_ioat_desc(ioat->used_desc.next);
1126 tx = &desc->txd;
1127 latest_desc = NULL;
1128 do {
1129 desc = to_ioat_desc(desc->node.prev);
1130 desc_phys = (unsigned long)tx->phys
1131 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
1132 if (desc_phys == phys_complete) {
1133 latest_desc = desc;
1134 break;
1135 }
1136 } while (&desc->node != ioat->used_desc.prev);
1137
1138 if (latest_desc != NULL) {
1139 /* work forwards to clear finished descriptors */
1140 for (desc = to_ioat_desc(ioat->used_desc.prev);
1141 &desc->node != latest_desc->node.next &&
1142 &desc->node != ioat->used_desc.next;
1143 desc = to_ioat_desc(desc->node.next)) {
1144 if (tx->cookie) {
1145 cookie = tx->cookie;
1146 tx->cookie = 0;
1147 ioat_dma_unmap(chan, desc);
1148 if (tx->callback) {
1149 tx->callback(tx->callback_param);
1150 tx->callback = NULL;
1151 }
1152 }
1153 }
1154
1155 /* move used.prev up beyond those that are finished */
1156 if (&desc->node == ioat->used_desc.next)
1157 ioat->used_desc.prev = NULL;
1158 else
1159 ioat->used_desc.prev = &desc->node;
1160 } 798 }
1161 break;
1162 } 799 }
1163 800
1164 spin_unlock_bh(&ioat->desc_lock); 801 spin_unlock_bh(&ioat->desc_lock);
@@ -1170,50 +807,21 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat)
1170 spin_unlock_bh(&chan->cleanup_lock); 807 spin_unlock_bh(&chan->cleanup_lock);
1171} 808}
1172 809
1173/**
1174 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
1175 * @chan: IOAT DMA channel handle
1176 * @cookie: DMA transaction identifier
1177 * @done: if not %NULL, updated with last completed transaction
1178 * @used: if not %NULL, updated with last used transaction
1179 */
1180static enum dma_status 810static enum dma_status
1181ioat_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie, 811ioat1_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie,
1182 dma_cookie_t *done, dma_cookie_t *used) 812 dma_cookie_t *done, dma_cookie_t *used)
1183{ 813{
1184 struct ioat_dma_chan *ioat = to_ioat_chan(c); 814 struct ioat_dma_chan *ioat = to_ioat_chan(c);
1185 struct ioat_chan_common *chan = &ioat->base;
1186 dma_cookie_t last_used;
1187 dma_cookie_t last_complete;
1188 enum dma_status ret;
1189
1190 last_used = c->cookie;
1191 last_complete = chan->completed_cookie;
1192 chan->watchdog_tcp_cookie = cookie;
1193
1194 if (done)
1195 *done = last_complete;
1196 if (used)
1197 *used = last_used;
1198
1199 ret = dma_async_is_complete(cookie, last_complete, last_used);
1200 if (ret == DMA_SUCCESS)
1201 return ret;
1202 815
1203 ioat_dma_memcpy_cleanup(ioat); 816 if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
817 return DMA_SUCCESS;
1204 818
1205 last_used = c->cookie; 819 ioat1_cleanup(ioat);
1206 last_complete = chan->completed_cookie;
1207 820
1208 if (done) 821 return ioat_is_complete(c, cookie, done, used);
1209 *done = last_complete;
1210 if (used)
1211 *used = last_used;
1212
1213 return dma_async_is_complete(cookie, last_complete, last_used);
1214} 822}
1215 823
1216static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat) 824static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat)
1217{ 825{
1218 struct ioat_chan_common *chan = &ioat->base; 826 struct ioat_chan_common *chan = &ioat->base;
1219 struct ioat_desc_sw *desc; 827 struct ioat_desc_sw *desc;
@@ -1221,7 +829,7 @@ static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat)
1221 829
1222 spin_lock_bh(&ioat->desc_lock); 830 spin_lock_bh(&ioat->desc_lock);
1223 831
1224 desc = ioat_dma_get_next_descriptor(ioat); 832 desc = ioat1_dma_get_next_descriptor(ioat);
1225 833
1226 if (!desc) { 834 if (!desc) {
1227 dev_err(to_dev(chan), 835 dev_err(to_dev(chan),
@@ -1240,30 +848,16 @@ static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat)
1240 hw->src_addr = 0; 848 hw->src_addr = 0;
1241 hw->dst_addr = 0; 849 hw->dst_addr = 0;
1242 async_tx_ack(&desc->txd); 850 async_tx_ack(&desc->txd);
1243 switch (chan->device->version) { 851 hw->next = 0;
1244 case IOAT_VER_1_2: 852 list_add_tail(&desc->node, &ioat->used_desc);
1245 hw->next = 0;
1246 list_add_tail(&desc->node, &ioat->used_desc);
1247 853
1248 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF, 854 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1249 chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW); 855 chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
1250 writel(((u64) desc->txd.phys) >> 32, 856 writel(((u64) desc->txd.phys) >> 32,
1251 chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH); 857 chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
1252
1253 writeb(IOAT_CHANCMD_START, chan->reg_base
1254 + IOAT_CHANCMD_OFFSET(chan->device->version));
1255 break;
1256 case IOAT_VER_2_0:
1257 case IOAT_VER_3_0:
1258 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1259 chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
1260 writel(((u64) desc->txd.phys) >> 32,
1261 chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
1262 858
1263 ioat->dmacount++; 859 writeb(IOAT_CHANCMD_START, chan->reg_base
1264 __ioat2_dma_memcpy_issue_pending(ioat); 860 + IOAT_CHANCMD_OFFSET(chan->device->version));
1265 break;
1266 }
1267 spin_unlock_bh(&ioat->desc_lock); 861 spin_unlock_bh(&ioat->desc_lock);
1268} 862}
1269 863
@@ -1484,7 +1078,7 @@ static void ioat_disable_interrupts(struct ioatdma_device *device)
1484 writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET); 1078 writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1485} 1079}
1486 1080
1487static int ioat_probe(struct ioatdma_device *device) 1081int ioat_probe(struct ioatdma_device *device)
1488{ 1082{
1489 int err = -ENODEV; 1083 int err = -ENODEV;
1490 struct dma_device *dma = &device->common; 1084 struct dma_device *dma = &device->common;
@@ -1503,17 +1097,15 @@ static int ioat_probe(struct ioatdma_device *device)
1503 device->completion_pool = pci_pool_create("completion_pool", pdev, 1097 device->completion_pool = pci_pool_create("completion_pool", pdev,
1504 sizeof(u64), SMP_CACHE_BYTES, 1098 sizeof(u64), SMP_CACHE_BYTES,
1505 SMP_CACHE_BYTES); 1099 SMP_CACHE_BYTES);
1100
1506 if (!device->completion_pool) { 1101 if (!device->completion_pool) {
1507 err = -ENOMEM; 1102 err = -ENOMEM;
1508 goto err_completion_pool; 1103 goto err_completion_pool;
1509 } 1104 }
1510 1105
1511 ioat_dma_enumerate_channels(device); 1106 device->enumerate_channels(device);
1512 1107
1513 dma_cap_set(DMA_MEMCPY, dma->cap_mask); 1108 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
1514 dma->device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
1515 dma->device_free_chan_resources = ioat_dma_free_chan_resources;
1516 dma->device_is_tx_complete = ioat_dma_is_complete;
1517 dma->dev = &pdev->dev; 1109 dma->dev = &pdev->dev;
1518 1110
1519 dev_err(dev, "Intel(R) I/OAT DMA Engine found," 1111 dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
@@ -1546,7 +1138,7 @@ err_dma_pool:
1546 return err; 1138 return err;
1547} 1139}
1548 1140
1549static int ioat_register(struct ioatdma_device *device) 1141int ioat_register(struct ioatdma_device *device)
1550{ 1142{
1551 int err = dma_async_device_register(&device->common); 1143 int err = dma_async_device_register(&device->common);
1552 1144
@@ -1580,9 +1172,13 @@ int ioat1_dma_probe(struct ioatdma_device *device, int dca)
1580 int err; 1172 int err;
1581 1173
1582 device->intr_quirk = ioat1_intr_quirk; 1174 device->intr_quirk = ioat1_intr_quirk;
1175 device->enumerate_channels = ioat1_enumerate_channels;
1583 dma = &device->common; 1176 dma = &device->common;
1584 dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy; 1177 dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1585 dma->device_issue_pending = ioat1_dma_memcpy_issue_pending; 1178 dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
1179 dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources;
1180 dma->device_free_chan_resources = ioat1_dma_free_chan_resources;
1181 dma->device_is_tx_complete = ioat1_dma_is_complete;
1586 1182
1587 err = ioat_probe(device); 1183 err = ioat_probe(device);
1588 if (err) 1184 if (err)
@@ -1594,93 +1190,12 @@ int ioat1_dma_probe(struct ioatdma_device *device, int dca)
1594 if (dca) 1190 if (dca)
1595 device->dca = ioat_dca_init(pdev, device->reg_base); 1191 device->dca = ioat_dca_init(pdev, device->reg_base);
1596 1192
1597 INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog); 1193 INIT_DELAYED_WORK(&device->work, ioat1_chan_watchdog);
1598 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1599
1600 return err;
1601}
1602
1603int ioat2_dma_probe(struct ioatdma_device *device, int dca)
1604{
1605 struct pci_dev *pdev = device->pdev;
1606 struct dma_device *dma;
1607 struct dma_chan *c;
1608 struct ioat_chan_common *chan;
1609 int err;
1610
1611 dma = &device->common;
1612 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1613 dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1614
1615 err = ioat_probe(device);
1616 if (err)
1617 return err;
1618 ioat_set_tcp_copy_break(2048);
1619
1620 list_for_each_entry(c, &dma->channels, device_node) {
1621 chan = to_chan_common(c);
1622 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
1623 chan->reg_base + IOAT_DCACTRL_OFFSET);
1624 }
1625
1626 err = ioat_register(device);
1627 if (err)
1628 return err;
1629 if (dca)
1630 device->dca = ioat2_dca_init(pdev, device->reg_base);
1631
1632 INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
1633 schedule_delayed_work(&device->work, WATCHDOG_DELAY); 1194 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1634 1195
1635 return err; 1196 return err;
1636} 1197}
1637 1198
1638int ioat3_dma_probe(struct ioatdma_device *device, int dca)
1639{
1640 struct pci_dev *pdev = device->pdev;
1641 struct dma_device *dma;
1642 struct dma_chan *c;
1643 struct ioat_chan_common *chan;
1644 int err;
1645 u16 dev_id;
1646
1647 dma = &device->common;
1648 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1649 dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1650
1651 /* -= IOAT ver.3 workarounds =- */
1652 /* Write CHANERRMSK_INT with 3E07h to mask out the errors
1653 * that can cause stability issues for IOAT ver.3
1654 */
1655 pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
1656
1657 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1658 * (workaround for spurious config parity error after restart)
1659 */
1660 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1661 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
1662 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
1663
1664 err = ioat_probe(device);
1665 if (err)
1666 return err;
1667 ioat_set_tcp_copy_break(262144);
1668
1669 list_for_each_entry(c, &dma->channels, device_node) {
1670 chan = to_chan_common(c);
1671 writel(IOAT_DMA_DCA_ANY_CPU,
1672 chan->reg_base + IOAT_DCACTRL_OFFSET);
1673 }
1674
1675 err = ioat_register(device);
1676 if (err)
1677 return err;
1678 if (dca)
1679 device->dca = ioat3_dca_init(pdev, device->reg_base);
1680
1681 return err;
1682}
1683
1684void ioat_dma_remove(struct ioatdma_device *device) 1199void ioat_dma_remove(struct ioatdma_device *device)
1685{ 1200{
1686 struct dma_device *dma = &device->common; 1201 struct dma_device *dma = &device->common;
@@ -1697,4 +1212,3 @@ void ioat_dma_remove(struct ioatdma_device *device)
1697 1212
1698 INIT_LIST_HEAD(&dma->channels); 1213 INIT_LIST_HEAD(&dma->channels);
1699} 1214}
1700
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index 5b31db73ad8e..84065dfa4d40 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -62,6 +62,7 @@
62 * @idx: per channel data 62 * @idx: per channel data
63 * @dca: direct cache access context 63 * @dca: direct cache access context
64 * @intr_quirk: interrupt setup quirk (for ioat_v1 devices) 64 * @intr_quirk: interrupt setup quirk (for ioat_v1 devices)
65 * @enumerate_channels: hw version specific channel enumeration
65 */ 66 */
66 67
67struct ioatdma_device { 68struct ioatdma_device {
@@ -76,6 +77,7 @@ struct ioatdma_device {
76 struct ioat_chan_common *idx[4]; 77 struct ioat_chan_common *idx[4];
77 struct dca_provider *dca; 78 struct dca_provider *dca;
78 void (*intr_quirk)(struct ioatdma_device *device); 79 void (*intr_quirk)(struct ioatdma_device *device);
80 int (*enumerate_channels)(struct ioatdma_device *device);
79}; 81};
80 82
81struct ioat_chan_common { 83struct ioat_chan_common {
@@ -106,6 +108,7 @@ struct ioat_chan_common {
106 struct tasklet_struct cleanup_task; 108 struct tasklet_struct cleanup_task;
107}; 109};
108 110
111
109/** 112/**
110 * struct ioat_dma_chan - internal representation of a DMA channel 113 * struct ioat_dma_chan - internal representation of a DMA channel
111 */ 114 */
@@ -119,7 +122,6 @@ struct ioat_dma_chan {
119 struct list_head used_desc; 122 struct list_head used_desc;
120 123
121 int pending; 124 int pending;
122 u16 dmacount;
123 u16 desccount; 125 u16 desccount;
124}; 126};
125 127
@@ -135,6 +137,33 @@ static inline struct ioat_dma_chan *to_ioat_chan(struct dma_chan *c)
135 return container_of(chan, struct ioat_dma_chan, base); 137 return container_of(chan, struct ioat_dma_chan, base);
136} 138}
137 139
140/**
141 * ioat_is_complete - poll the status of an ioat transaction
142 * @c: channel handle
143 * @cookie: transaction identifier
144 * @done: if set, updated with last completed transaction
145 * @used: if set, updated with last used transaction
146 */
147static inline enum dma_status
148ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie,
149 dma_cookie_t *done, dma_cookie_t *used)
150{
151 struct ioat_chan_common *chan = to_chan_common(c);
152 dma_cookie_t last_used;
153 dma_cookie_t last_complete;
154
155 last_used = c->cookie;
156 last_complete = chan->completed_cookie;
157 chan->watchdog_tcp_cookie = cookie;
158
159 if (done)
160 *done = last_complete;
161 if (used)
162 *used = last_used;
163
164 return dma_async_is_complete(cookie, last_complete, last_used);
165}
166
138/* wrapper around hardware descriptor format + additional software fields */ 167/* wrapper around hardware descriptor format + additional software fields */
139 168
140/** 169/**
@@ -162,11 +191,22 @@ static inline void ioat_set_tcp_copy_break(unsigned long copybreak)
162 #endif 191 #endif
163} 192}
164 193
194static inline struct ioat_chan_common *
195ioat_chan_by_index(struct ioatdma_device *device, int index)
196{
197 return device->idx[index];
198}
199
200int ioat_probe(struct ioatdma_device *device);
201int ioat_register(struct ioatdma_device *device);
165int ioat1_dma_probe(struct ioatdma_device *dev, int dca); 202int ioat1_dma_probe(struct ioatdma_device *dev, int dca);
166int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
167int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
168void ioat_dma_remove(struct ioatdma_device *device); 203void ioat_dma_remove(struct ioatdma_device *device);
169struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); 204struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
170struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase); 205unsigned long ioat_get_current_completion(struct ioat_chan_common *chan);
171struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); 206void ioat_init_channel(struct ioatdma_device *device,
207 struct ioat_chan_common *chan, int idx,
208 work_func_t work_fn, void (*tasklet)(unsigned long),
209 unsigned long tasklet_data);
210void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
211 size_t len, struct ioat_dma_descriptor *hw);
172#endif /* IOATDMA_H */ 212#endif /* IOATDMA_H */
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
new file mode 100644
index 000000000000..49ba1c73d95e
--- /dev/null
+++ b/drivers/dma/ioat/dma_v2.c
@@ -0,0 +1,750 @@
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2009 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 */
22
23/*
24 * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
25 * does asynchronous data movement and checksumming operations.
26 */
27
28#include <linux/init.h>
29#include <linux/module.h>
30#include <linux/pci.h>
31#include <linux/interrupt.h>
32#include <linux/dmaengine.h>
33#include <linux/delay.h>
34#include <linux/dma-mapping.h>
35#include <linux/workqueue.h>
36#include <linux/i7300_idle.h>
37#include "dma.h"
38#include "dma_v2.h"
39#include "registers.h"
40#include "hw.h"
41
42static int ioat_ring_alloc_order = 8;
43module_param(ioat_ring_alloc_order, int, 0644);
44MODULE_PARM_DESC(ioat_ring_alloc_order,
45 "ioat2+: allocate 2^n descriptors per channel (default: n=8)");
46
47static void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
48{
49 void * __iomem reg_base = ioat->base.reg_base;
50
51 ioat->pending = 0;
52 ioat->dmacount += ioat2_ring_pending(ioat);
53 ioat->issued = ioat->head;
54 /* make descriptor updates globally visible before notifying channel */
55 wmb();
56 writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
57
58}
59
60static void ioat2_issue_pending(struct dma_chan *chan)
61{
62 struct ioat2_dma_chan *ioat = to_ioat2_chan(chan);
63
64 spin_lock_bh(&ioat->ring_lock);
65 if (ioat->pending == 1)
66 __ioat2_issue_pending(ioat);
67 spin_unlock_bh(&ioat->ring_lock);
68}
69
70/**
71 * ioat2_update_pending - log pending descriptors
72 * @ioat: ioat2+ channel
73 *
74 * set pending to '1' unless pending is already set to '2', pending == 2
75 * indicates that submission is temporarily blocked due to an in-flight
76 * reset. If we are already above the ioat_pending_level threshold then
77 * just issue pending.
78 *
79 * called with ring_lock held
80 */
81static void ioat2_update_pending(struct ioat2_dma_chan *ioat)
82{
83 if (unlikely(ioat->pending == 2))
84 return;
85 else if (ioat2_ring_pending(ioat) > ioat_pending_level)
86 __ioat2_issue_pending(ioat);
87 else
88 ioat->pending = 1;
89}
90
91static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
92{
93 void __iomem *reg_base = ioat->base.reg_base;
94 struct ioat_ring_ent *desc;
95 struct ioat_dma_descriptor *hw;
96 int idx;
97
98 if (ioat2_ring_space(ioat) < 1) {
99 dev_err(to_dev(&ioat->base),
100 "Unable to start null desc - ring full\n");
101 return;
102 }
103
104 idx = ioat2_desc_alloc(ioat, 1);
105 desc = ioat2_get_ring_ent(ioat, idx);
106
107 hw = desc->hw;
108 hw->ctl = 0;
109 hw->ctl_f.null = 1;
110 hw->ctl_f.int_en = 1;
111 hw->ctl_f.compl_write = 1;
112 /* set size to non-zero value (channel returns error when size is 0) */
113 hw->size = NULL_DESC_BUFFER_SIZE;
114 hw->src_addr = 0;
115 hw->dst_addr = 0;
116 async_tx_ack(&desc->txd);
117 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
118 reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
119 writel(((u64) desc->txd.phys) >> 32,
120 reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
121 __ioat2_issue_pending(ioat);
122}
123
124static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
125{
126 spin_lock_bh(&ioat->ring_lock);
127 __ioat2_start_null_desc(ioat);
128 spin_unlock_bh(&ioat->ring_lock);
129}
130
131static void ioat2_cleanup(struct ioat2_dma_chan *ioat);
132
133/**
134 * ioat2_reset_part2 - reinit the channel after a reset
135 */
136static void ioat2_reset_part2(struct work_struct *work)
137{
138 struct ioat_chan_common *chan;
139 struct ioat2_dma_chan *ioat;
140
141 chan = container_of(work, struct ioat_chan_common, work.work);
142 ioat = container_of(chan, struct ioat2_dma_chan, base);
143
144 /* ensure that ->tail points to the stalled descriptor
145 * (ioat->pending is set to 2 at this point so no new
146 * descriptors will be issued while we perform this cleanup)
147 */
148 ioat2_cleanup(ioat);
149
150 spin_lock_bh(&chan->cleanup_lock);
151 spin_lock_bh(&ioat->ring_lock);
152
153 /* set the tail to be re-issued */
154 ioat->issued = ioat->tail;
155 ioat->dmacount = 0;
156
157 if (ioat2_ring_pending(ioat)) {
158 struct ioat_ring_ent *desc;
159
160 desc = ioat2_get_ring_ent(ioat, ioat->tail);
161 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
162 chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
163 writel(((u64) desc->txd.phys) >> 32,
164 chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
165 __ioat2_issue_pending(ioat);
166 } else
167 __ioat2_start_null_desc(ioat);
168
169 spin_unlock_bh(&ioat->ring_lock);
170 spin_unlock_bh(&chan->cleanup_lock);
171
172 dev_info(to_dev(chan),
173 "chan%d reset - %d descs waiting, %d total desc\n",
174 chan_num(chan), ioat->dmacount, 1 << ioat->alloc_order);
175}
176
177/**
178 * ioat2_reset_channel - restart a channel
179 * @ioat: IOAT DMA channel handle
180 */
181static void ioat2_reset_channel(struct ioat2_dma_chan *ioat)
182{
183 u32 chansts, chanerr;
184 struct ioat_chan_common *chan = &ioat->base;
185 u16 active;
186
187 spin_lock_bh(&ioat->ring_lock);
188 active = ioat2_ring_active(ioat);
189 spin_unlock_bh(&ioat->ring_lock);
190 if (!active)
191 return;
192
193 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
194 chansts = (chan->completion_virt->low
195 & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
196 if (chanerr) {
197 dev_err(to_dev(chan),
198 "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
199 chan_num(chan), chansts, chanerr);
200 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
201 }
202
203 spin_lock_bh(&ioat->ring_lock);
204 ioat->pending = 2;
205 writeb(IOAT_CHANCMD_RESET,
206 chan->reg_base
207 + IOAT_CHANCMD_OFFSET(chan->device->version));
208 spin_unlock_bh(&ioat->ring_lock);
209 schedule_delayed_work(&chan->work, RESET_DELAY);
210}
211
212/**
213 * ioat2_chan_watchdog - watch for stuck channels
214 */
215static void ioat2_chan_watchdog(struct work_struct *work)
216{
217 struct ioatdma_device *device =
218 container_of(work, struct ioatdma_device, work.work);
219 struct ioat2_dma_chan *ioat;
220 struct ioat_chan_common *chan;
221 u16 active;
222 int i;
223
224 for (i = 0; i < device->common.chancnt; i++) {
225 chan = ioat_chan_by_index(device, i);
226 ioat = container_of(chan, struct ioat2_dma_chan, base);
227
228 /*
229 * for version 2.0 if there are descriptors yet to be processed
230 * and the last completed hasn't changed since the last watchdog
231 * if they haven't hit the pending level
232 * issue the pending to push them through
233 * else
234 * try resetting the channel
235 */
236 spin_lock_bh(&ioat->ring_lock);
237 active = ioat2_ring_active(ioat);
238 spin_unlock_bh(&ioat->ring_lock);
239
240 if (active &&
241 chan->last_completion &&
242 chan->last_completion == chan->watchdog_completion) {
243
244 if (ioat->pending == 1)
245 ioat2_issue_pending(&chan->common);
246 else {
247 ioat2_reset_channel(ioat);
248 chan->watchdog_completion = 0;
249 }
250 } else {
251 chan->last_compl_desc_addr_hw = 0;
252 chan->watchdog_completion = chan->last_completion;
253 }
254 chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
255 }
256 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
257}
258
259/**
260 * ioat2_cleanup - clean finished descriptors (advance tail pointer)
261 * @chan: ioat channel to be cleaned up
262 */
263static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
264{
265 struct ioat_chan_common *chan = &ioat->base;
266 unsigned long phys_complete;
267 struct ioat_ring_ent *desc;
268 bool seen_current = false;
269 u16 active;
270 int i;
271 struct dma_async_tx_descriptor *tx;
272
273 prefetch(chan->completion_virt);
274
275 spin_lock_bh(&chan->cleanup_lock);
276 phys_complete = ioat_get_current_completion(chan);
277 if (phys_complete == chan->last_completion) {
278 spin_unlock_bh(&chan->cleanup_lock);
279 /*
280 * perhaps we're stuck so hard that the watchdog can't go off?
281 * try to catch it after WATCHDOG_DELAY seconds
282 */
283 if (chan->device->version < IOAT_VER_3_0) {
284 unsigned long tmo;
285
286 tmo = chan->last_completion_time + HZ*WATCHDOG_DELAY;
287 if (time_after(jiffies, tmo)) {
288 ioat2_chan_watchdog(&(chan->device->work.work));
289 chan->last_completion_time = jiffies;
290 }
291 }
292 return;
293 }
294 chan->last_completion_time = jiffies;
295
296 spin_lock_bh(&ioat->ring_lock);
297
298 active = ioat2_ring_active(ioat);
299 for (i = 0; i < active && !seen_current; i++) {
300 prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
301 desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
302 tx = &desc->txd;
303 if (tx->cookie) {
304 ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
305 chan->completed_cookie = tx->cookie;
306 tx->cookie = 0;
307 if (tx->callback) {
308 tx->callback(tx->callback_param);
309 tx->callback = NULL;
310 }
311 }
312
313 if (tx->phys == phys_complete)
314 seen_current = true;
315 }
316 ioat->tail += i;
317 BUG_ON(!seen_current); /* no active descs have written a completion? */
318 spin_unlock_bh(&ioat->ring_lock);
319
320 chan->last_completion = phys_complete;
321
322 spin_unlock_bh(&chan->cleanup_lock);
323}
324
325static void ioat2_cleanup_tasklet(unsigned long data)
326{
327 struct ioat2_dma_chan *ioat = (void *) data;
328
329 ioat2_cleanup(ioat);
330 writew(IOAT_CHANCTRL_INT_DISABLE,
331 ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
332}
333
334/**
335 * ioat2_enumerate_channels - find and initialize the device's channels
336 * @device: the device to be enumerated
337 */
338static int ioat2_enumerate_channels(struct ioatdma_device *device)
339{
340 struct ioat2_dma_chan *ioat;
341 struct device *dev = &device->pdev->dev;
342 struct dma_device *dma = &device->common;
343 u8 xfercap_log;
344 int i;
345
346 INIT_LIST_HEAD(&dma->channels);
347 dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
348 xfercap_log = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
349 if (xfercap_log == 0)
350 return 0;
351
352 /* FIXME which i/oat version is i7300? */
353#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
354 if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
355 dma->chancnt--;
356#endif
357 for (i = 0; i < dma->chancnt; i++) {
358 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
359 if (!ioat)
360 break;
361
362 ioat_init_channel(device, &ioat->base, i,
363 ioat2_reset_part2,
364 ioat2_cleanup_tasklet,
365 (unsigned long) ioat);
366 ioat->xfercap_log = xfercap_log;
367 spin_lock_init(&ioat->ring_lock);
368 }
369 dma->chancnt = i;
370 return i;
371}
372
373static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
374{
375 struct dma_chan *c = tx->chan;
376 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
377 dma_cookie_t cookie = c->cookie;
378
379 cookie++;
380 if (cookie < 0)
381 cookie = 1;
382 tx->cookie = cookie;
383 c->cookie = cookie;
384 ioat2_update_pending(ioat);
385 spin_unlock_bh(&ioat->ring_lock);
386
387 return cookie;
388}
389
390static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan)
391{
392 struct ioat_dma_descriptor *hw;
393 struct ioat_ring_ent *desc;
394 struct ioatdma_device *dma;
395 dma_addr_t phys;
396
397 dma = to_ioatdma_device(chan->device);
398 hw = pci_pool_alloc(dma->dma_pool, GFP_KERNEL, &phys);
399 if (!hw)
400 return NULL;
401 memset(hw, 0, sizeof(*hw));
402
403 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
404 if (!desc) {
405 pci_pool_free(dma->dma_pool, hw, phys);
406 return NULL;
407 }
408
409 dma_async_tx_descriptor_init(&desc->txd, chan);
410 desc->txd.tx_submit = ioat2_tx_submit_unlock;
411 desc->hw = hw;
412 desc->txd.phys = phys;
413 return desc;
414}
415
416static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
417{
418 struct ioatdma_device *dma;
419
420 dma = to_ioatdma_device(chan->device);
421 pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys);
422 kfree(desc);
423}
424
425/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
426 * @chan: channel to be initialized
427 */
428static int ioat2_alloc_chan_resources(struct dma_chan *c)
429{
430 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
431 struct ioat_chan_common *chan = &ioat->base;
432 struct ioat_ring_ent **ring;
433 u16 chanctrl;
434 u32 chanerr;
435 int descs;
436 int i;
437
438 /* have we already been set up? */
439 if (ioat->ring)
440 return 1 << ioat->alloc_order;
441
442 /* Setup register to interrupt and write completion status on error */
443 chanctrl = IOAT_CHANCTRL_ERR_INT_EN | IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
444 IOAT_CHANCTRL_ERR_COMPLETION_EN;
445 writew(chanctrl, chan->reg_base + IOAT_CHANCTRL_OFFSET);
446
447 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
448 if (chanerr) {
449 dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
450 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
451 }
452
453 /* allocate a completion writeback area */
454 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
455 chan->completion_virt = pci_pool_alloc(chan->device->completion_pool,
456 GFP_KERNEL,
457 &chan->completion_addr);
458 if (!chan->completion_virt)
459 return -ENOMEM;
460
461 memset(chan->completion_virt, 0,
462 sizeof(*chan->completion_virt));
463 writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF,
464 chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
465 writel(((u64) chan->completion_addr) >> 32,
466 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
467
468 ioat->alloc_order = ioat_get_alloc_order();
469 descs = 1 << ioat->alloc_order;
470
471 /* allocate the array to hold the software ring */
472 ring = kcalloc(descs, sizeof(*ring), GFP_KERNEL);
473 if (!ring)
474 return -ENOMEM;
475 for (i = 0; i < descs; i++) {
476 ring[i] = ioat2_alloc_ring_ent(c);
477 if (!ring[i]) {
478 while (i--)
479 ioat2_free_ring_ent(ring[i], c);
480 kfree(ring);
481 return -ENOMEM;
482 }
483 }
484
485 /* link descs */
486 for (i = 0; i < descs-1; i++) {
487 struct ioat_ring_ent *next = ring[i+1];
488 struct ioat_dma_descriptor *hw = ring[i]->hw;
489
490 hw->next = next->txd.phys;
491 }
492 ring[i]->hw->next = ring[0]->txd.phys;
493
494 spin_lock_bh(&ioat->ring_lock);
495 ioat->ring = ring;
496 ioat->head = 0;
497 ioat->issued = 0;
498 ioat->tail = 0;
499 ioat->pending = 0;
500 spin_unlock_bh(&ioat->ring_lock);
501
502 tasklet_enable(&chan->cleanup_task);
503 ioat2_start_null_desc(ioat);
504
505 return descs;
506}
507
508/**
509 * ioat2_alloc_and_lock - common descriptor alloc boilerplate for ioat2,3 ops
510 * @idx: gets starting descriptor index on successful allocation
511 * @ioat: ioat2,3 channel (ring) to operate on
512 * @num_descs: allocation length
513 */
514static int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs)
515{
516 struct ioat_chan_common *chan = &ioat->base;
517
518 spin_lock_bh(&ioat->ring_lock);
519 if (unlikely(ioat2_ring_space(ioat) < num_descs)) {
520 if (printk_ratelimit())
521 dev_dbg(to_dev(chan),
522 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
523 __func__, num_descs, ioat->head, ioat->tail,
524 ioat->issued);
525 spin_unlock_bh(&ioat->ring_lock);
526
527 /* do direct reclaim in the allocation failure case */
528 ioat2_cleanup(ioat);
529
530 return -ENOMEM;
531 }
532
533 dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n",
534 __func__, num_descs, ioat->head, ioat->tail, ioat->issued);
535
536 *idx = ioat2_desc_alloc(ioat, num_descs);
537 return 0; /* with ioat->ring_lock held */
538}
539
540static struct dma_async_tx_descriptor *
541ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
542 dma_addr_t dma_src, size_t len, unsigned long flags)
543{
544 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
545 struct ioat_dma_descriptor *hw;
546 struct ioat_ring_ent *desc;
547 dma_addr_t dst = dma_dest;
548 dma_addr_t src = dma_src;
549 size_t total_len = len;
550 int num_descs;
551 u16 idx;
552 int i;
553
554 num_descs = ioat2_xferlen_to_descs(ioat, len);
555 if (likely(num_descs) &&
556 ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0)
557 /* pass */;
558 else
559 return NULL;
560 for (i = 0; i < num_descs; i++) {
561 size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log);
562
563 desc = ioat2_get_ring_ent(ioat, idx + i);
564 hw = desc->hw;
565
566 hw->size = copy;
567 hw->ctl = 0;
568 hw->src_addr = src;
569 hw->dst_addr = dst;
570
571 len -= copy;
572 dst += copy;
573 src += copy;
574 }
575
576 desc->txd.flags = flags;
577 desc->len = total_len;
578 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
579 hw->ctl_f.compl_write = 1;
580 /* we leave the channel locked to ensure in order submission */
581
582 return &desc->txd;
583}
584
585/**
586 * ioat2_free_chan_resources - release all the descriptors
587 * @chan: the channel to be cleaned
588 */
589static void ioat2_free_chan_resources(struct dma_chan *c)
590{
591 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
592 struct ioat_chan_common *chan = &ioat->base;
593 struct ioatdma_device *ioatdma_device = chan->device;
594 struct ioat_ring_ent *desc;
595 const u16 total_descs = 1 << ioat->alloc_order;
596 int descs;
597 int i;
598
599 /* Before freeing channel resources first check
600 * if they have been previously allocated for this channel.
601 */
602 if (!ioat->ring)
603 return;
604
605 tasklet_disable(&chan->cleanup_task);
606 ioat2_cleanup(ioat);
607
608 /* Delay 100ms after reset to allow internal DMA logic to quiesce
609 * before removing DMA descriptor resources.
610 */
611 writeb(IOAT_CHANCMD_RESET,
612 chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
613 mdelay(100);
614
615 spin_lock_bh(&ioat->ring_lock);
616 descs = ioat2_ring_space(ioat);
617 for (i = 0; i < descs; i++) {
618 desc = ioat2_get_ring_ent(ioat, ioat->head + i);
619 ioat2_free_ring_ent(desc, c);
620 }
621
622 if (descs < total_descs)
623 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
624 total_descs - descs);
625
626 for (i = 0; i < total_descs - descs; i++) {
627 desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
628 ioat2_free_ring_ent(desc, c);
629 }
630
631 kfree(ioat->ring);
632 ioat->ring = NULL;
633 ioat->alloc_order = 0;
634 pci_pool_free(ioatdma_device->completion_pool,
635 chan->completion_virt,
636 chan->completion_addr);
637 spin_unlock_bh(&ioat->ring_lock);
638
639 chan->last_completion = 0;
640 chan->completion_addr = 0;
641 ioat->pending = 0;
642 ioat->dmacount = 0;
643 chan->watchdog_completion = 0;
644 chan->last_compl_desc_addr_hw = 0;
645 chan->watchdog_tcp_cookie = 0;
646 chan->watchdog_last_tcp_cookie = 0;
647}
648
649static enum dma_status
650ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
651 dma_cookie_t *done, dma_cookie_t *used)
652{
653 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
654
655 if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
656 return DMA_SUCCESS;
657
658 ioat2_cleanup(ioat);
659
660 return ioat_is_complete(c, cookie, done, used);
661}
662
663int ioat2_dma_probe(struct ioatdma_device *device, int dca)
664{
665 struct pci_dev *pdev = device->pdev;
666 struct dma_device *dma;
667 struct dma_chan *c;
668 struct ioat_chan_common *chan;
669 int err;
670
671 device->enumerate_channels = ioat2_enumerate_channels;
672 dma = &device->common;
673 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
674 dma->device_issue_pending = ioat2_issue_pending;
675 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
676 dma->device_free_chan_resources = ioat2_free_chan_resources;
677 dma->device_is_tx_complete = ioat2_is_complete;
678
679 err = ioat_probe(device);
680 if (err)
681 return err;
682 ioat_set_tcp_copy_break(2048);
683
684 list_for_each_entry(c, &dma->channels, device_node) {
685 chan = to_chan_common(c);
686 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
687 chan->reg_base + IOAT_DCACTRL_OFFSET);
688 }
689
690 err = ioat_register(device);
691 if (err)
692 return err;
693 if (dca)
694 device->dca = ioat2_dca_init(pdev, device->reg_base);
695
696 INIT_DELAYED_WORK(&device->work, ioat2_chan_watchdog);
697 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
698
699 return err;
700}
701
702int ioat3_dma_probe(struct ioatdma_device *device, int dca)
703{
704 struct pci_dev *pdev = device->pdev;
705 struct dma_device *dma;
706 struct dma_chan *c;
707 struct ioat_chan_common *chan;
708 int err;
709 u16 dev_id;
710
711 device->enumerate_channels = ioat2_enumerate_channels;
712 dma = &device->common;
713 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
714 dma->device_issue_pending = ioat2_issue_pending;
715 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
716 dma->device_free_chan_resources = ioat2_free_chan_resources;
717 dma->device_is_tx_complete = ioat2_is_complete;
718
719 /* -= IOAT ver.3 workarounds =- */
720 /* Write CHANERRMSK_INT with 3E07h to mask out the errors
721 * that can cause stability issues for IOAT ver.3
722 */
723 pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
724
725 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
726 * (workaround for spurious config parity error after restart)
727 */
728 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
729 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
730 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
731
732 err = ioat_probe(device);
733 if (err)
734 return err;
735 ioat_set_tcp_copy_break(262144);
736
737 list_for_each_entry(c, &dma->channels, device_node) {
738 chan = to_chan_common(c);
739 writel(IOAT_DMA_DCA_ANY_CPU,
740 chan->reg_base + IOAT_DCACTRL_OFFSET);
741 }
742
743 err = ioat_register(device);
744 if (err)
745 return err;
746 if (dca)
747 device->dca = ioat3_dca_init(pdev, device->reg_base);
748
749 return err;
750}
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
new file mode 100644
index 000000000000..94a553eacdbd
--- /dev/null
+++ b/drivers/dma/ioat/dma_v2.h
@@ -0,0 +1,131 @@
1/*
2 * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21#ifndef IOATDMA_V2_H
22#define IOATDMA_V2_H
23
24#include <linux/dmaengine.h>
25#include "dma.h"
26#include "hw.h"
27
28
29extern int ioat_pending_level;
30
31/*
32 * workaround for IOAT ver.3.0 null descriptor issue
33 * (channel returns error when size is 0)
34 */
35#define NULL_DESC_BUFFER_SIZE 1
36
37#define IOAT_MAX_ORDER 16
38#define ioat_get_alloc_order() \
39 (min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
40
41/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes
42 * @base: common ioat channel parameters
43 * @xfercap_log; log2 of channel max transfer length (for fast division)
44 * @head: allocated index
45 * @issued: hardware notification point
46 * @tail: cleanup index
47 * @pending: lock free indicator for issued != head
48 * @dmacount: identical to 'head' except for occasionally resetting to zero
49 * @alloc_order: log2 of the number of allocated descriptors
50 * @ring: software ring buffer implementation of hardware ring
51 * @ring_lock: protects ring attributes
52 */
53struct ioat2_dma_chan {
54 struct ioat_chan_common base;
55 size_t xfercap_log;
56 u16 head;
57 u16 issued;
58 u16 tail;
59 u16 dmacount;
60 u16 alloc_order;
61 int pending;
62 struct ioat_ring_ent **ring;
63 spinlock_t ring_lock;
64};
65
66static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c)
67{
68 struct ioat_chan_common *chan = to_chan_common(c);
69
70 return container_of(chan, struct ioat2_dma_chan, base);
71}
72
73static inline u16 ioat2_ring_mask(struct ioat2_dma_chan *ioat)
74{
75 return (1 << ioat->alloc_order) - 1;
76}
77
78/* count of descriptors in flight with the engine */
79static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat)
80{
81 return (ioat->head - ioat->tail) & ioat2_ring_mask(ioat);
82}
83
84/* count of descriptors pending submission to hardware */
85static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat)
86{
87 return (ioat->head - ioat->issued) & ioat2_ring_mask(ioat);
88}
89
90static inline u16 ioat2_ring_space(struct ioat2_dma_chan *ioat)
91{
92 u16 num_descs = ioat2_ring_mask(ioat) + 1;
93 u16 active = ioat2_ring_active(ioat);
94
95 BUG_ON(active > num_descs);
96
97 return num_descs - active;
98}
99
100/* assumes caller already checked space */
101static inline u16 ioat2_desc_alloc(struct ioat2_dma_chan *ioat, u16 len)
102{
103 ioat->head += len;
104 return ioat->head - len;
105}
106
107static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len)
108{
109 u16 num_descs = len >> ioat->xfercap_log;
110
111 num_descs += !!(len & ((1 << ioat->xfercap_log) - 1));
112 return num_descs;
113}
114
115struct ioat_ring_ent {
116 struct ioat_dma_descriptor *hw;
117 struct dma_async_tx_descriptor txd;
118 size_t len;
119};
120
121static inline struct ioat_ring_ent *
122ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx)
123{
124 return ioat->ring[idx & ioat2_ring_mask(ioat)];
125}
126
127int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
128int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
129struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
130struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
131#endif /* IOATDMA_V2_H */
diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c
index 55414d88ac1b..c4e432269252 100644
--- a/drivers/dma/ioat/pci.c
+++ b/drivers/dma/ioat/pci.c
@@ -31,6 +31,7 @@
31#include <linux/interrupt.h> 31#include <linux/interrupt.h>
32#include <linux/dca.h> 32#include <linux/dca.h>
33#include "dma.h" 33#include "dma.h"
34#include "dma_v2.h"
34#include "registers.h" 35#include "registers.h"
35#include "hw.h" 36#include "hw.h"
36 37
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-23 00:06:47 -0500