aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma/ioat/dma_v2.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma/ioat/dma_v2.c')
-rw-r--r--drivers/dma/ioat/dma_v2.c750
1 files changed, 750 insertions, 0 deletions
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
new file mode 100644
index 000000000000..49ba1c73d95e
--- /dev/null
+++ b/drivers/dma/ioat/dma_v2.c
@@ -0,0 +1,750 @@
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2009 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 */
22
23/*
24 * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
25 * does asynchronous data movement and checksumming operations.
26 */
27
28#include <linux/init.h>
29#include <linux/module.h>
30#include <linux/pci.h>
31#include <linux/interrupt.h>
32#include <linux/dmaengine.h>
33#include <linux/delay.h>
34#include <linux/dma-mapping.h>
35#include <linux/workqueue.h>
36#include <linux/i7300_idle.h>
37#include "dma.h"
38#include "dma_v2.h"
39#include "registers.h"
40#include "hw.h"
41
42static int ioat_ring_alloc_order = 8;
43module_param(ioat_ring_alloc_order, int, 0644);
44MODULE_PARM_DESC(ioat_ring_alloc_order,
45 "ioat2+: allocate 2^n descriptors per channel (default: n=8)");
46
47static void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
48{
49 void * __iomem reg_base = ioat->base.reg_base;
50
51 ioat->pending = 0;
52 ioat->dmacount += ioat2_ring_pending(ioat);
53 ioat->issued = ioat->head;
54 /* make descriptor updates globally visible before notifying channel */
55 wmb();
56 writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
57
58}
59
60static void ioat2_issue_pending(struct dma_chan *chan)
61{
62 struct ioat2_dma_chan *ioat = to_ioat2_chan(chan);
63
64 spin_lock_bh(&ioat->ring_lock);
65 if (ioat->pending == 1)
66 __ioat2_issue_pending(ioat);
67 spin_unlock_bh(&ioat->ring_lock);
68}
69
70/**
71 * ioat2_update_pending - log pending descriptors
72 * @ioat: ioat2+ channel
73 *
74 * set pending to '1' unless pending is already set to '2', pending == 2
75 * indicates that submission is temporarily blocked due to an in-flight
76 * reset. If we are already above the ioat_pending_level threshold then
77 * just issue pending.
78 *
79 * called with ring_lock held
80 */
81static void ioat2_update_pending(struct ioat2_dma_chan *ioat)
82{
83 if (unlikely(ioat->pending == 2))
84 return;
85 else if (ioat2_ring_pending(ioat) > ioat_pending_level)
86 __ioat2_issue_pending(ioat);
87 else
88 ioat->pending = 1;
89}
90
91static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
92{
93 void __iomem *reg_base = ioat->base.reg_base;
94 struct ioat_ring_ent *desc;
95 struct ioat_dma_descriptor *hw;
96 int idx;
97
98 if (ioat2_ring_space(ioat) < 1) {
99 dev_err(to_dev(&ioat->base),
100 "Unable to start null desc - ring full\n");
101 return;
102 }
103
104 idx = ioat2_desc_alloc(ioat, 1);
105 desc = ioat2_get_ring_ent(ioat, idx);
106
107 hw = desc->hw;
108 hw->ctl = 0;
109 hw->ctl_f.null = 1;
110 hw->ctl_f.int_en = 1;
111 hw->ctl_f.compl_write = 1;
112 /* set size to non-zero value (channel returns error when size is 0) */
113 hw->size = NULL_DESC_BUFFER_SIZE;
114 hw->src_addr = 0;
115 hw->dst_addr = 0;
116 async_tx_ack(&desc->txd);
117 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
118 reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
119 writel(((u64) desc->txd.phys) >> 32,
120 reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
121 __ioat2_issue_pending(ioat);
122}
123
124static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
125{
126 spin_lock_bh(&ioat->ring_lock);
127 __ioat2_start_null_desc(ioat);
128 spin_unlock_bh(&ioat->ring_lock);
129}
130
131static void ioat2_cleanup(struct ioat2_dma_chan *ioat);
132
133/**
134 * ioat2_reset_part2 - reinit the channel after a reset
135 */
136static void ioat2_reset_part2(struct work_struct *work)
137{
138 struct ioat_chan_common *chan;
139 struct ioat2_dma_chan *ioat;
140
141 chan = container_of(work, struct ioat_chan_common, work.work);
142 ioat = container_of(chan, struct ioat2_dma_chan, base);
143
144 /* ensure that ->tail points to the stalled descriptor
145 * (ioat->pending is set to 2 at this point so no new
146 * descriptors will be issued while we perform this cleanup)
147 */
148 ioat2_cleanup(ioat);
149
150 spin_lock_bh(&chan->cleanup_lock);
151 spin_lock_bh(&ioat->ring_lock);
152
153 /* set the tail to be re-issued */
154 ioat->issued = ioat->tail;
155 ioat->dmacount = 0;
156
157 if (ioat2_ring_pending(ioat)) {
158 struct ioat_ring_ent *desc;
159
160 desc = ioat2_get_ring_ent(ioat, ioat->tail);
161 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
162 chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
163 writel(((u64) desc->txd.phys) >> 32,
164 chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
165 __ioat2_issue_pending(ioat);
166 } else
167 __ioat2_start_null_desc(ioat);
168
169 spin_unlock_bh(&ioat->ring_lock);
170 spin_unlock_bh(&chan->cleanup_lock);
171
172 dev_info(to_dev(chan),
173 "chan%d reset - %d descs waiting, %d total desc\n",
174 chan_num(chan), ioat->dmacount, 1 << ioat->alloc_order);
175}
176
177/**
178 * ioat2_reset_channel - restart a channel
179 * @ioat: IOAT DMA channel handle
180 */
181static void ioat2_reset_channel(struct ioat2_dma_chan *ioat)
182{
183 u32 chansts, chanerr;
184 struct ioat_chan_common *chan = &ioat->base;
185 u16 active;
186
187 spin_lock_bh(&ioat->ring_lock);
188 active = ioat2_ring_active(ioat);
189 spin_unlock_bh(&ioat->ring_lock);
190 if (!active)
191 return;
192
193 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
194 chansts = (chan->completion_virt->low
195 & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
196 if (chanerr) {
197 dev_err(to_dev(chan),
198 "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
199 chan_num(chan), chansts, chanerr);
200 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
201 }
202
203 spin_lock_bh(&ioat->ring_lock);
204 ioat->pending = 2;
205 writeb(IOAT_CHANCMD_RESET,
206 chan->reg_base
207 + IOAT_CHANCMD_OFFSET(chan->device->version));
208 spin_unlock_bh(&ioat->ring_lock);
209 schedule_delayed_work(&chan->work, RESET_DELAY);
210}
211
212/**
213 * ioat2_chan_watchdog - watch for stuck channels
214 */
215static void ioat2_chan_watchdog(struct work_struct *work)
216{
217 struct ioatdma_device *device =
218 container_of(work, struct ioatdma_device, work.work);
219 struct ioat2_dma_chan *ioat;
220 struct ioat_chan_common *chan;
221 u16 active;
222 int i;
223
224 for (i = 0; i < device->common.chancnt; i++) {
225 chan = ioat_chan_by_index(device, i);
226 ioat = container_of(chan, struct ioat2_dma_chan, base);
227
228 /*
229 * for version 2.0 if there are descriptors yet to be processed
230 * and the last completed hasn't changed since the last watchdog
231 * if they haven't hit the pending level
232 * issue the pending to push them through
233 * else
234 * try resetting the channel
235 */
236 spin_lock_bh(&ioat->ring_lock);
237 active = ioat2_ring_active(ioat);
238 spin_unlock_bh(&ioat->ring_lock);
239
240 if (active &&
241 chan->last_completion &&
242 chan->last_completion == chan->watchdog_completion) {
243
244 if (ioat->pending == 1)
245 ioat2_issue_pending(&chan->common);
246 else {
247 ioat2_reset_channel(ioat);
248 chan->watchdog_completion = 0;
249 }
250 } else {
251 chan->last_compl_desc_addr_hw = 0;
252 chan->watchdog_completion = chan->last_completion;
253 }
254 chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
255 }
256 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
257}
258
259/**
260 * ioat2_cleanup - clean finished descriptors (advance tail pointer)
261 * @chan: ioat channel to be cleaned up
262 */
263static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
264{
265 struct ioat_chan_common *chan = &ioat->base;
266 unsigned long phys_complete;
267 struct ioat_ring_ent *desc;
268 bool seen_current = false;
269 u16 active;
270 int i;
271 struct dma_async_tx_descriptor *tx;
272
273 prefetch(chan->completion_virt);
274
275 spin_lock_bh(&chan->cleanup_lock);
276 phys_complete = ioat_get_current_completion(chan);
277 if (phys_complete == chan->last_completion) {
278 spin_unlock_bh(&chan->cleanup_lock);
279 /*
280 * perhaps we're stuck so hard that the watchdog can't go off?
281 * try to catch it after WATCHDOG_DELAY seconds
282 */
283 if (chan->device->version < IOAT_VER_3_0) {
284 unsigned long tmo;
285
286 tmo = chan->last_completion_time + HZ*WATCHDOG_DELAY;
287 if (time_after(jiffies, tmo)) {
288 ioat2_chan_watchdog(&(chan->device->work.work));
289 chan->last_completion_time = jiffies;
290 }
291 }
292 return;
293 }
294 chan->last_completion_time = jiffies;
295
296 spin_lock_bh(&ioat->ring_lock);
297
298 active = ioat2_ring_active(ioat);
299 for (i = 0; i < active && !seen_current; i++) {
300 prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
301 desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
302 tx = &desc->txd;
303 if (tx->cookie) {
304 ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
305 chan->completed_cookie = tx->cookie;
306 tx->cookie = 0;
307 if (tx->callback) {
308 tx->callback(tx->callback_param);
309 tx->callback = NULL;
310 }
311 }
312
313 if (tx->phys == phys_complete)
314 seen_current = true;
315 }
316 ioat->tail += i;
317 BUG_ON(!seen_current); /* no active descs have written a completion? */
318 spin_unlock_bh(&ioat->ring_lock);
319
320 chan->last_completion = phys_complete;
321
322 spin_unlock_bh(&chan->cleanup_lock);
323}
324
325static void ioat2_cleanup_tasklet(unsigned long data)
326{
327 struct ioat2_dma_chan *ioat = (void *) data;
328
329 ioat2_cleanup(ioat);
330 writew(IOAT_CHANCTRL_INT_DISABLE,
331 ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
332}
333
334/**
335 * ioat2_enumerate_channels - find and initialize the device's channels
336 * @device: the device to be enumerated
337 */
338static int ioat2_enumerate_channels(struct ioatdma_device *device)
339{
340 struct ioat2_dma_chan *ioat;
341 struct device *dev = &device->pdev->dev;
342 struct dma_device *dma = &device->common;
343 u8 xfercap_log;
344 int i;
345
346 INIT_LIST_HEAD(&dma->channels);
347 dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
348 xfercap_log = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
349 if (xfercap_log == 0)
350 return 0;
351
352 /* FIXME which i/oat version is i7300? */
353#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
354 if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
355 dma->chancnt--;
356#endif
357 for (i = 0; i < dma->chancnt; i++) {
358 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
359 if (!ioat)
360 break;
361
362 ioat_init_channel(device, &ioat->base, i,
363 ioat2_reset_part2,
364 ioat2_cleanup_tasklet,
365 (unsigned long) ioat);
366 ioat->xfercap_log = xfercap_log;
367 spin_lock_init(&ioat->ring_lock);
368 }
369 dma->chancnt = i;
370 return i;
371}
372
373static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
374{
375 struct dma_chan *c = tx->chan;
376 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
377 dma_cookie_t cookie = c->cookie;
378
379 cookie++;
380 if (cookie < 0)
381 cookie = 1;
382 tx->cookie = cookie;
383 c->cookie = cookie;
384 ioat2_update_pending(ioat);
385 spin_unlock_bh(&ioat->ring_lock);
386
387 return cookie;
388}
389
390static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan)
391{
392 struct ioat_dma_descriptor *hw;
393 struct ioat_ring_ent *desc;
394 struct ioatdma_device *dma;
395 dma_addr_t phys;
396
397 dma = to_ioatdma_device(chan->device);
398 hw = pci_pool_alloc(dma->dma_pool, GFP_KERNEL, &phys);
399 if (!hw)
400 return NULL;
401 memset(hw, 0, sizeof(*hw));
402
403 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
404 if (!desc) {
405 pci_pool_free(dma->dma_pool, hw, phys);
406 return NULL;
407 }
408
409 dma_async_tx_descriptor_init(&desc->txd, chan);
410 desc->txd.tx_submit = ioat2_tx_submit_unlock;
411 desc->hw = hw;
412 desc->txd.phys = phys;
413 return desc;
414}
415
416static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
417{
418 struct ioatdma_device *dma;
419
420 dma = to_ioatdma_device(chan->device);
421 pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys);
422 kfree(desc);
423}
424
425/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
426 * @chan: channel to be initialized
427 */
428static int ioat2_alloc_chan_resources(struct dma_chan *c)
429{
430 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
431 struct ioat_chan_common *chan = &ioat->base;
432 struct ioat_ring_ent **ring;
433 u16 chanctrl;
434 u32 chanerr;
435 int descs;
436 int i;
437
438 /* have we already been set up? */
439 if (ioat->ring)
440 return 1 << ioat->alloc_order;
441
442 /* Setup register to interrupt and write completion status on error */
443 chanctrl = IOAT_CHANCTRL_ERR_INT_EN | IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
444 IOAT_CHANCTRL_ERR_COMPLETION_EN;
445 writew(chanctrl, chan->reg_base + IOAT_CHANCTRL_OFFSET);
446
447 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
448 if (chanerr) {
449 dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
450 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
451 }
452
453 /* allocate a completion writeback area */
454 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
455 chan->completion_virt = pci_pool_alloc(chan->device->completion_pool,
456 GFP_KERNEL,
457 &chan->completion_addr);
458 if (!chan->completion_virt)
459 return -ENOMEM;
460
461 memset(chan->completion_virt, 0,
462 sizeof(*chan->completion_virt));
463 writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF,
464 chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
465 writel(((u64) chan->completion_addr) >> 32,
466 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
467
468 ioat->alloc_order = ioat_get_alloc_order();
469 descs = 1 << ioat->alloc_order;
470
471 /* allocate the array to hold the software ring */
472 ring = kcalloc(descs, sizeof(*ring), GFP_KERNEL);
473 if (!ring)
474 return -ENOMEM;
475 for (i = 0; i < descs; i++) {
476 ring[i] = ioat2_alloc_ring_ent(c);
477 if (!ring[i]) {
478 while (i--)
479 ioat2_free_ring_ent(ring[i], c);
480 kfree(ring);
481 return -ENOMEM;
482 }
483 }
484
485 /* link descs */
486 for (i = 0; i < descs-1; i++) {
487 struct ioat_ring_ent *next = ring[i+1];
488 struct ioat_dma_descriptor *hw = ring[i]->hw;
489
490 hw->next = next->txd.phys;
491 }
492 ring[i]->hw->next = ring[0]->txd.phys;
493
494 spin_lock_bh(&ioat->ring_lock);
495 ioat->ring = ring;
496 ioat->head = 0;
497 ioat->issued = 0;
498 ioat->tail = 0;
499 ioat->pending = 0;
500 spin_unlock_bh(&ioat->ring_lock);
501
502 tasklet_enable(&chan->cleanup_task);
503 ioat2_start_null_desc(ioat);
504
505 return descs;
506}
507
508/**
509 * ioat2_alloc_and_lock - common descriptor alloc boilerplate for ioat2,3 ops
510 * @idx: gets starting descriptor index on successful allocation
511 * @ioat: ioat2,3 channel (ring) to operate on
512 * @num_descs: allocation length
513 */
514static int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs)
515{
516 struct ioat_chan_common *chan = &ioat->base;
517
518 spin_lock_bh(&ioat->ring_lock);
519 if (unlikely(ioat2_ring_space(ioat) < num_descs)) {
520 if (printk_ratelimit())
521 dev_dbg(to_dev(chan),
522 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
523 __func__, num_descs, ioat->head, ioat->tail,
524 ioat->issued);
525 spin_unlock_bh(&ioat->ring_lock);
526
527 /* do direct reclaim in the allocation failure case */
528 ioat2_cleanup(ioat);
529
530 return -ENOMEM;
531 }
532
533 dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n",
534 __func__, num_descs, ioat->head, ioat->tail, ioat->issued);
535
536 *idx = ioat2_desc_alloc(ioat, num_descs);
537 return 0; /* with ioat->ring_lock held */
538}
539
540static struct dma_async_tx_descriptor *
541ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
542 dma_addr_t dma_src, size_t len, unsigned long flags)
543{
544 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
545 struct ioat_dma_descriptor *hw;
546 struct ioat_ring_ent *desc;
547 dma_addr_t dst = dma_dest;
548 dma_addr_t src = dma_src;
549 size_t total_len = len;
550 int num_descs;
551 u16 idx;
552 int i;
553
554 num_descs = ioat2_xferlen_to_descs(ioat, len);
555 if (likely(num_descs) &&
556 ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0)
557 /* pass */;
558 else
559 return NULL;
560 for (i = 0; i < num_descs; i++) {
561 size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log);
562
563 desc = ioat2_get_ring_ent(ioat, idx + i);
564 hw = desc->hw;
565
566 hw->size = copy;
567 hw->ctl = 0;
568 hw->src_addr = src;
569 hw->dst_addr = dst;
570
571 len -= copy;
572 dst += copy;
573 src += copy;
574 }
575
576 desc->txd.flags = flags;
577 desc->len = total_len;
578 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
579 hw->ctl_f.compl_write = 1;
580 /* we leave the channel locked to ensure in order submission */
581
582 return &desc->txd;
583}
584
585/**
586 * ioat2_free_chan_resources - release all the descriptors
587 * @chan: the channel to be cleaned
588 */
589static void ioat2_free_chan_resources(struct dma_chan *c)
590{
591 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
592 struct ioat_chan_common *chan = &ioat->base;
593 struct ioatdma_device *ioatdma_device = chan->device;
594 struct ioat_ring_ent *desc;
595 const u16 total_descs = 1 << ioat->alloc_order;
596 int descs;
597 int i;
598
599 /* Before freeing channel resources first check
600 * if they have been previously allocated for this channel.
601 */
602 if (!ioat->ring)
603 return;
604
605 tasklet_disable(&chan->cleanup_task);
606 ioat2_cleanup(ioat);
607
608 /* Delay 100ms after reset to allow internal DMA logic to quiesce
609 * before removing DMA descriptor resources.
610 */
611 writeb(IOAT_CHANCMD_RESET,
612 chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
613 mdelay(100);
614
615 spin_lock_bh(&ioat->ring_lock);
616 descs = ioat2_ring_space(ioat);
617 for (i = 0; i < descs; i++) {
618 desc = ioat2_get_ring_ent(ioat, ioat->head + i);
619 ioat2_free_ring_ent(desc, c);
620 }
621
622 if (descs < total_descs)
623 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
624 total_descs - descs);
625
626 for (i = 0; i < total_descs - descs; i++) {
627 desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
628 ioat2_free_ring_ent(desc, c);
629 }
630
631 kfree(ioat->ring);
632 ioat->ring = NULL;
633 ioat->alloc_order = 0;
634 pci_pool_free(ioatdma_device->completion_pool,
635 chan->completion_virt,
636 chan->completion_addr);
637 spin_unlock_bh(&ioat->ring_lock);
638
639 chan->last_completion = 0;
640 chan->completion_addr = 0;
641 ioat->pending = 0;
642 ioat->dmacount = 0;
643 chan->watchdog_completion = 0;
644 chan->last_compl_desc_addr_hw = 0;
645 chan->watchdog_tcp_cookie = 0;
646 chan->watchdog_last_tcp_cookie = 0;
647}
648
649static enum dma_status
650ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
651 dma_cookie_t *done, dma_cookie_t *used)
652{
653 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
654
655 if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
656 return DMA_SUCCESS;
657
658 ioat2_cleanup(ioat);
659
660 return ioat_is_complete(c, cookie, done, used);
661}
662
663int ioat2_dma_probe(struct ioatdma_device *device, int dca)
664{
665 struct pci_dev *pdev = device->pdev;
666 struct dma_device *dma;
667 struct dma_chan *c;
668 struct ioat_chan_common *chan;
669 int err;
670
671 device->enumerate_channels = ioat2_enumerate_channels;
672 dma = &device->common;
673 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
674 dma->device_issue_pending = ioat2_issue_pending;
675 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
676 dma->device_free_chan_resources = ioat2_free_chan_resources;
677 dma->device_is_tx_complete = ioat2_is_complete;
678
679 err = ioat_probe(device);
680 if (err)
681 return err;
682 ioat_set_tcp_copy_break(2048);
683
684 list_for_each_entry(c, &dma->channels, device_node) {
685 chan = to_chan_common(c);
686 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
687 chan->reg_base + IOAT_DCACTRL_OFFSET);
688 }
689
690 err = ioat_register(device);
691 if (err)
692 return err;
693 if (dca)
694 device->dca = ioat2_dca_init(pdev, device->reg_base);
695
696 INIT_DELAYED_WORK(&device->work, ioat2_chan_watchdog);
697 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
698
699 return err;
700}
701
702int ioat3_dma_probe(struct ioatdma_device *device, int dca)
703{
704 struct pci_dev *pdev = device->pdev;
705 struct dma_device *dma;
706 struct dma_chan *c;
707 struct ioat_chan_common *chan;
708 int err;
709 u16 dev_id;
710
711 device->enumerate_channels = ioat2_enumerate_channels;
712 dma = &device->common;
713 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
714 dma->device_issue_pending = ioat2_issue_pending;
715 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
716 dma->device_free_chan_resources = ioat2_free_chan_resources;
717 dma->device_is_tx_complete = ioat2_is_complete;
718
719 /* -= IOAT ver.3 workarounds =- */
720 /* Write CHANERRMSK_INT with 3E07h to mask out the errors
721 * that can cause stability issues for IOAT ver.3
722 */
723 pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
724
725 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
726 * (workaround for spurious config parity error after restart)
727 */
728 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
729 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
730 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
731
732 err = ioat_probe(device);
733 if (err)
734 return err;
735 ioat_set_tcp_copy_break(262144);
736
737 list_for_each_entry(c, &dma->channels, device_node) {
738 chan = to_chan_common(c);
739 writel(IOAT_DMA_DCA_ANY_CPU,
740 chan->reg_base + IOAT_DCACTRL_OFFSET);
741 }
742
743 err = ioat_register(device);
744 if (err)
745 return err;
746 if (dca)
747 device->dca = ioat3_dca_init(pdev, device->reg_base);
748
749 return err;
750}
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-22 15:55:14 -0500