aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma/ioat_dma.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma/ioat_dma.c')
-rw-r--r--drivers/dma/ioat_dma.c818
1 files changed, 818 insertions, 0 deletions
diff --git a/drivers/dma/ioat_dma.c b/drivers/dma/ioat_dma.c
new file mode 100644
index 000000000000..f0be71cf1eee
--- /dev/null
+++ b/drivers/dma/ioat_dma.c
@@ -0,0 +1,818 @@
1/*
2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21
22/*
23 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
24 * copy operations.
25 */
26
27#include <linux/init.h>
28#include <linux/module.h>
29#include <linux/pci.h>
30#include <linux/interrupt.h>
31#include <linux/dmaengine.h>
32#include <linux/delay.h>
33#include <linux/dma-mapping.h>
34#include "ioatdma.h"
35#include "ioatdma_registers.h"
36#include "ioatdma_hw.h"
37
38#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
39#define to_ioat_device(dev) container_of(dev, struct ioat_device, common)
40#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
41#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)
42
43/* internal functions */
44static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
45static void ioat_shutdown(struct pci_dev *pdev);
46static void __devexit ioat_remove(struct pci_dev *pdev);
47
48static int enumerate_dma_channels(struct ioat_device *device)
49{
50 u8 xfercap_scale;
51 u32 xfercap;
52 int i;
53 struct ioat_dma_chan *ioat_chan;
54
55 device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
56 xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
57 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
58
59 for (i = 0; i < device->common.chancnt; i++) {
60 ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
61 if (!ioat_chan) {
62 device->common.chancnt = i;
63 break;
64 }
65
66 ioat_chan->device = device;
67 ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
68 ioat_chan->xfercap = xfercap;
69 spin_lock_init(&ioat_chan->cleanup_lock);
70 spin_lock_init(&ioat_chan->desc_lock);
71 INIT_LIST_HEAD(&ioat_chan->free_desc);
72 INIT_LIST_HEAD(&ioat_chan->used_desc);
73 /* This should be made common somewhere in dmaengine.c */
74 ioat_chan->common.device = &device->common;
75 list_add_tail(&ioat_chan->common.device_node,
76 &device->common.channels);
77 }
78 return device->common.chancnt;
79}
80
81static void
82ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
83{
84 struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
85 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
86
87 pci_unmap_addr_set(desc, src, addr);
88
89 list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
90 iter->hw->src_addr = addr;
91 addr += ioat_chan->xfercap;
92 }
93
94}
95
96static void
97ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
98{
99 struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
100 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
101
102 pci_unmap_addr_set(desc, dst, addr);
103
104 list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
105 iter->hw->dst_addr = addr;
106 addr += ioat_chan->xfercap;
107 }
108}
109
110static dma_cookie_t
111ioat_tx_submit(struct dma_async_tx_descriptor *tx)
112{
113 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
114 struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
115 int append = 0;
116 dma_cookie_t cookie;
117 struct ioat_desc_sw *group_start;
118
119 group_start = list_entry(desc->async_tx.tx_list.next,
120 struct ioat_desc_sw, node);
121 spin_lock_bh(&ioat_chan->desc_lock);
122 /* cookie incr and addition to used_list must be atomic */
123 cookie = ioat_chan->common.cookie;
124 cookie++;
125 if (cookie < 0)
126 cookie = 1;
127 ioat_chan->common.cookie = desc->async_tx.cookie = cookie;
128
129 /* write address into NextDescriptor field of last desc in chain */
130 to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
131 group_start->async_tx.phys;
132 list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev);
133
134 ioat_chan->pending += desc->tx_cnt;
135 if (ioat_chan->pending >= 4) {
136 append = 1;
137 ioat_chan->pending = 0;
138 }
139 spin_unlock_bh(&ioat_chan->desc_lock);
140
141 if (append)
142 writeb(IOAT_CHANCMD_APPEND,
143 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
144
145 return cookie;
146}
147
148static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
149 struct ioat_dma_chan *ioat_chan,
150 gfp_t flags)
151{
152 struct ioat_dma_descriptor *desc;
153 struct ioat_desc_sw *desc_sw;
154 struct ioat_device *ioat_device;
155 dma_addr_t phys;
156
157 ioat_device = to_ioat_device(ioat_chan->common.device);
158 desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys);
159 if (unlikely(!desc))
160 return NULL;
161
162 desc_sw = kzalloc(sizeof(*desc_sw), flags);
163 if (unlikely(!desc_sw)) {
164 pci_pool_free(ioat_device->dma_pool, desc, phys);
165 return NULL;
166 }
167
168 memset(desc, 0, sizeof(*desc));
169 dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);
170 desc_sw->async_tx.tx_set_src = ioat_set_src;
171 desc_sw->async_tx.tx_set_dest = ioat_set_dest;
172 desc_sw->async_tx.tx_submit = ioat_tx_submit;
173 INIT_LIST_HEAD(&desc_sw->async_tx.tx_list);
174 desc_sw->hw = desc;
175 desc_sw->async_tx.phys = phys;
176
177 return desc_sw;
178}
179
180#define INITIAL_IOAT_DESC_COUNT 128
181
182static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan);
183
184/* returns the actual number of allocated descriptors */
185static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
186{
187 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
188 struct ioat_desc_sw *desc = NULL;
189 u16 chanctrl;
190 u32 chanerr;
191 int i;
192 LIST_HEAD(tmp_list);
193
194 /* have we already been set up? */
195 if (!list_empty(&ioat_chan->free_desc))
196 return INITIAL_IOAT_DESC_COUNT;
197
198 /* Setup register to interrupt and write completion status on error */
199 chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
200 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
201 IOAT_CHANCTRL_ERR_COMPLETION_EN;
202 writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
203
204 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
205 if (chanerr) {
206 printk("IOAT: CHANERR = %x, clearing\n", chanerr);
207 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
208 }
209
210 /* Allocate descriptors */
211 for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) {
212 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
213 if (!desc) {
214 printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i);
215 break;
216 }
217 list_add_tail(&desc->node, &tmp_list);
218 }
219 spin_lock_bh(&ioat_chan->desc_lock);
220 list_splice(&tmp_list, &ioat_chan->free_desc);
221 spin_unlock_bh(&ioat_chan->desc_lock);
222
223 /* allocate a completion writeback area */
224 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
225 ioat_chan->completion_virt =
226 pci_pool_alloc(ioat_chan->device->completion_pool,
227 GFP_KERNEL,
228 &ioat_chan->completion_addr);
229 memset(ioat_chan->completion_virt, 0,
230 sizeof(*ioat_chan->completion_virt));
231 writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
232 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
233 writel(((u64) ioat_chan->completion_addr) >> 32,
234 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
235
236 ioat_start_null_desc(ioat_chan);
237 return i;
238}
239
240static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
241
242static void ioat_dma_free_chan_resources(struct dma_chan *chan)
243{
244 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
245 struct ioat_device *ioat_device = to_ioat_device(chan->device);
246 struct ioat_desc_sw *desc, *_desc;
247 u16 chanctrl;
248 int in_use_descs = 0;
249
250 ioat_dma_memcpy_cleanup(ioat_chan);
251
252 writeb(IOAT_CHANCMD_RESET, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
253
254 spin_lock_bh(&ioat_chan->desc_lock);
255 list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
256 in_use_descs++;
257 list_del(&desc->node);
258 pci_pool_free(ioat_device->dma_pool, desc->hw,
259 desc->async_tx.phys);
260 kfree(desc);
261 }
262 list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
263 list_del(&desc->node);
264 pci_pool_free(ioat_device->dma_pool, desc->hw,
265 desc->async_tx.phys);
266 kfree(desc);
267 }
268 spin_unlock_bh(&ioat_chan->desc_lock);
269
270 pci_pool_free(ioat_device->completion_pool,
271 ioat_chan->completion_virt,
272 ioat_chan->completion_addr);
273
274 /* one is ok since we left it on there on purpose */
275 if (in_use_descs > 1)
276 printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n",
277 in_use_descs - 1);
278
279 ioat_chan->last_completion = ioat_chan->completion_addr = 0;
280}
281
282static struct dma_async_tx_descriptor *
283ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en)
284{
285 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
286 struct ioat_desc_sw *first, *prev, *new;
287 LIST_HEAD(new_chain);
288 u32 copy;
289 size_t orig_len;
290 int desc_count = 0;
291
292 if (!len)
293 return NULL;
294
295 orig_len = len;
296
297 first = NULL;
298 prev = NULL;
299
300 spin_lock_bh(&ioat_chan->desc_lock);
301 while (len) {
302 if (!list_empty(&ioat_chan->free_desc)) {
303 new = to_ioat_desc(ioat_chan->free_desc.next);
304 list_del(&new->node);
305 } else {
306 /* try to get another desc */
307 new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
308 /* will this ever happen? */
309 /* TODO add upper limit on these */
310 BUG_ON(!new);
311 }
312
313 copy = min((u32) len, ioat_chan->xfercap);
314
315 new->hw->size = copy;
316 new->hw->ctl = 0;
317 new->async_tx.cookie = 0;
318 new->async_tx.ack = 1;
319
320 /* chain together the physical address list for the HW */
321 if (!first)
322 first = new;
323 else
324 prev->hw->next = (u64) new->async_tx.phys;
325
326 prev = new;
327 len -= copy;
328 list_add_tail(&new->node, &new_chain);
329 desc_count++;
330 }
331
332 list_splice(&new_chain, &new->async_tx.tx_list);
333
334 new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
335 new->hw->next = 0;
336 new->tx_cnt = desc_count;
337 new->async_tx.ack = 0; /* client is in control of this ack */
338 new->async_tx.cookie = -EBUSY;
339
340 pci_unmap_len_set(new, len, orig_len);
341 spin_unlock_bh(&ioat_chan->desc_lock);
342
343 return new ? &new->async_tx : NULL;
344}
345
346
347/**
348 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw
349 * @chan: DMA channel handle
350 */
351
352static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan)
353{
354 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
355
356 if (ioat_chan->pending != 0) {
357 ioat_chan->pending = 0;
358 writeb(IOAT_CHANCMD_APPEND,
359 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
360 }
361}
362
363static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
364{
365 unsigned long phys_complete;
366 struct ioat_desc_sw *desc, *_desc;
367 dma_cookie_t cookie = 0;
368
369 prefetch(chan->completion_virt);
370
371 if (!spin_trylock(&chan->cleanup_lock))
372 return;
373
374 /* The completion writeback can happen at any time,
375 so reads by the driver need to be atomic operations
376 The descriptor physical addresses are limited to 32-bits
377 when the CPU can only do a 32-bit mov */
378
379#if (BITS_PER_LONG == 64)
380 phys_complete =
381 chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
382#else
383 phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
384#endif
385
386 if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
387 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
388 printk("IOAT: Channel halted, chanerr = %x\n",
389 readl(chan->reg_base + IOAT_CHANERR_OFFSET));
390
391 /* TODO do something to salvage the situation */
392 }
393
394 if (phys_complete == chan->last_completion) {
395 spin_unlock(&chan->cleanup_lock);
396 return;
397 }
398
399 spin_lock_bh(&chan->desc_lock);
400 list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) {
401
402 /*
403 * Incoming DMA requests may use multiple descriptors, due to
404 * exceeding xfercap, perhaps. If so, only the last one will
405 * have a cookie, and require unmapping.
406 */
407 if (desc->async_tx.cookie) {
408 cookie = desc->async_tx.cookie;
409
410 /* yes we are unmapping both _page and _single alloc'd
411 regions with unmap_page. Is this *really* that bad?
412 */
413 pci_unmap_page(chan->device->pdev,
414 pci_unmap_addr(desc, dst),
415 pci_unmap_len(desc, len),
416 PCI_DMA_FROMDEVICE);
417 pci_unmap_page(chan->device->pdev,
418 pci_unmap_addr(desc, src),
419 pci_unmap_len(desc, len),
420 PCI_DMA_TODEVICE);
421 }
422
423 if (desc->async_tx.phys != phys_complete) {
424 /* a completed entry, but not the last, so cleanup
425 * if the client is done with the descriptor
426 */
427 if (desc->async_tx.ack) {
428 list_del(&desc->node);
429 list_add_tail(&desc->node, &chan->free_desc);
430 } else
431 desc->async_tx.cookie = 0;
432 } else {
433 /* last used desc. Do not remove, so we can append from
434 it, but don't look at it next time, either */
435 desc->async_tx.cookie = 0;
436
437 /* TODO check status bits? */
438 break;
439 }
440 }
441
442 spin_unlock_bh(&chan->desc_lock);
443
444 chan->last_completion = phys_complete;
445 if (cookie != 0)
446 chan->completed_cookie = cookie;
447
448 spin_unlock(&chan->cleanup_lock);
449}
450
451static void ioat_dma_dependency_added(struct dma_chan *chan)
452{
453 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
454 spin_lock_bh(&ioat_chan->desc_lock);
455 if (ioat_chan->pending == 0) {
456 spin_unlock_bh(&ioat_chan->desc_lock);
457 ioat_dma_memcpy_cleanup(ioat_chan);
458 } else
459 spin_unlock_bh(&ioat_chan->desc_lock);
460}
461
462/**
463 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
464 * @chan: IOAT DMA channel handle
465 * @cookie: DMA transaction identifier
466 * @done: if not %NULL, updated with last completed transaction
467 * @used: if not %NULL, updated with last used transaction
468 */
469
470static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
471 dma_cookie_t cookie,
472 dma_cookie_t *done,
473 dma_cookie_t *used)
474{
475 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
476 dma_cookie_t last_used;
477 dma_cookie_t last_complete;
478 enum dma_status ret;
479
480 last_used = chan->cookie;
481 last_complete = ioat_chan->completed_cookie;
482
483 if (done)
484 *done= last_complete;
485 if (used)
486 *used = last_used;
487
488 ret = dma_async_is_complete(cookie, last_complete, last_used);
489 if (ret == DMA_SUCCESS)
490 return ret;
491
492 ioat_dma_memcpy_cleanup(ioat_chan);
493
494 last_used = chan->cookie;
495 last_complete = ioat_chan->completed_cookie;
496
497 if (done)
498 *done= last_complete;
499 if (used)
500 *used = last_used;
501
502 return dma_async_is_complete(cookie, last_complete, last_used);
503}
504
505/* PCI API */
506
507static struct pci_device_id ioat_pci_tbl[] = {
508 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
509 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB) },
510 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) },
511 { PCI_DEVICE(PCI_VENDOR_ID_UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) },
512 { 0, }
513};
514
515static struct pci_driver ioat_pci_driver = {
516 .name = "ioatdma",
517 .id_table = ioat_pci_tbl,
518 .probe = ioat_probe,
519 .shutdown = ioat_shutdown,
520 .remove = __devexit_p(ioat_remove),
521};
522
523static irqreturn_t ioat_do_interrupt(int irq, void *data)
524{
525 struct ioat_device *instance = data;
526 unsigned long attnstatus;
527 u8 intrctrl;
528
529 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
530
531 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
532 return IRQ_NONE;
533
534 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
535 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
536 return IRQ_NONE;
537 }
538
539 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
540
541 printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus);
542
543 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
544 return IRQ_HANDLED;
545}
546
547static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan)
548{
549 struct ioat_desc_sw *desc;
550
551 spin_lock_bh(&ioat_chan->desc_lock);
552
553 if (!list_empty(&ioat_chan->free_desc)) {
554 desc = to_ioat_desc(ioat_chan->free_desc.next);
555 list_del(&desc->node);
556 } else {
557 /* try to get another desc */
558 spin_unlock_bh(&ioat_chan->desc_lock);
559 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
560 spin_lock_bh(&ioat_chan->desc_lock);
561 /* will this ever happen? */
562 BUG_ON(!desc);
563 }
564
565 desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
566 desc->hw->next = 0;
567 desc->async_tx.ack = 1;
568
569 list_add_tail(&desc->node, &ioat_chan->used_desc);
570 spin_unlock_bh(&ioat_chan->desc_lock);
571
572 writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
573 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW);
574 writel(((u64) desc->async_tx.phys) >> 32,
575 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH);
576
577 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
578}
579
580/*
581 * Perform a IOAT transaction to verify the HW works.
582 */
583#define IOAT_TEST_SIZE 2000
584
585static int ioat_self_test(struct ioat_device *device)
586{
587 int i;
588 u8 *src;
589 u8 *dest;
590 struct dma_chan *dma_chan;
591 struct dma_async_tx_descriptor *tx;
592 dma_addr_t addr;
593 dma_cookie_t cookie;
594 int err = 0;
595
596 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
597 if (!src)
598 return -ENOMEM;
599 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
600 if (!dest) {
601 kfree(src);
602 return -ENOMEM;
603 }
604
605 /* Fill in src buffer */
606 for (i = 0; i < IOAT_TEST_SIZE; i++)
607 src[i] = (u8)i;
608
609 /* Start copy, using first DMA channel */
610 dma_chan = container_of(device->common.channels.next,
611 struct dma_chan,
612 device_node);
613 if (ioat_dma_alloc_chan_resources(dma_chan) < 1) {
614 err = -ENODEV;
615 goto out;
616 }
617
618 tx = ioat_dma_prep_memcpy(dma_chan, IOAT_TEST_SIZE, 0);
619 async_tx_ack(tx);
620 addr = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE,
621 DMA_TO_DEVICE);
622 ioat_set_src(addr, tx, 0);
623 addr = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE,
624 DMA_FROM_DEVICE);
625 ioat_set_dest(addr, tx, 0);
626 cookie = ioat_tx_submit(tx);
627 ioat_dma_memcpy_issue_pending(dma_chan);
628 msleep(1);
629
630 if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
631 printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n");
632 err = -ENODEV;
633 goto free_resources;
634 }
635 if (memcmp(src, dest, IOAT_TEST_SIZE)) {
636 printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n");
637 err = -ENODEV;
638 goto free_resources;
639 }
640
641free_resources:
642 ioat_dma_free_chan_resources(dma_chan);
643out:
644 kfree(src);
645 kfree(dest);
646 return err;
647}
648
649static int __devinit ioat_probe(struct pci_dev *pdev,
650 const struct pci_device_id *ent)
651{
652 int err;
653 unsigned long mmio_start, mmio_len;
654 void __iomem *reg_base;
655 struct ioat_device *device;
656
657 err = pci_enable_device(pdev);
658 if (err)
659 goto err_enable_device;
660
661 err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
662 if (err)
663 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
664 if (err)
665 goto err_set_dma_mask;
666
667 err = pci_request_regions(pdev, ioat_pci_driver.name);
668 if (err)
669 goto err_request_regions;
670
671 mmio_start = pci_resource_start(pdev, 0);
672 mmio_len = pci_resource_len(pdev, 0);
673
674 reg_base = ioremap(mmio_start, mmio_len);
675 if (!reg_base) {
676 err = -ENOMEM;
677 goto err_ioremap;
678 }
679
680 device = kzalloc(sizeof(*device), GFP_KERNEL);
681 if (!device) {
682 err = -ENOMEM;
683 goto err_kzalloc;
684 }
685
686 /* DMA coherent memory pool for DMA descriptor allocations */
687 device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
688 sizeof(struct ioat_dma_descriptor), 64, 0);
689 if (!device->dma_pool) {
690 err = -ENOMEM;
691 goto err_dma_pool;
692 }
693
694 device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES);
695 if (!device->completion_pool) {
696 err = -ENOMEM;
697 goto err_completion_pool;
698 }
699
700 device->pdev = pdev;
701 pci_set_drvdata(pdev, device);
702#ifdef CONFIG_PCI_MSI
703 if (pci_enable_msi(pdev) == 0) {
704 device->msi = 1;
705 } else {
706 device->msi = 0;
707 }
708#endif
709 err = request_irq(pdev->irq, &ioat_do_interrupt, IRQF_SHARED, "ioat",
710 device);
711 if (err)
712 goto err_irq;
713
714 device->reg_base = reg_base;
715
716 writeb(IOAT_INTRCTRL_MASTER_INT_EN, device->reg_base + IOAT_INTRCTRL_OFFSET);
717 pci_set_master(pdev);
718
719 INIT_LIST_HEAD(&device->common.channels);
720 enumerate_dma_channels(device);
721
722 dma_cap_set(DMA_MEMCPY, device->common.cap_mask);
723 device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
724 device->common.device_free_chan_resources = ioat_dma_free_chan_resources;
725 device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy;
726 device->common.device_is_tx_complete = ioat_dma_is_complete;
727 device->common.device_issue_pending = ioat_dma_memcpy_issue_pending;
728 device->common.device_dependency_added = ioat_dma_dependency_added;
729 device->common.dev = &pdev->dev;
730 printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n",
731 device->common.chancnt);
732
733 err = ioat_self_test(device);
734 if (err)
735 goto err_self_test;
736
737 dma_async_device_register(&device->common);
738
739 return 0;
740
741err_self_test:
742err_irq:
743 pci_pool_destroy(device->completion_pool);
744err_completion_pool:
745 pci_pool_destroy(device->dma_pool);
746err_dma_pool:
747 kfree(device);
748err_kzalloc:
749 iounmap(reg_base);
750err_ioremap:
751 pci_release_regions(pdev);
752err_request_regions:
753err_set_dma_mask:
754 pci_disable_device(pdev);
755err_enable_device:
756
757 printk(KERN_ERR "Intel(R) I/OAT DMA Engine initialization failed\n");
758
759 return err;
760}
761
762static void ioat_shutdown(struct pci_dev *pdev)
763{
764 struct ioat_device *device;
765 device = pci_get_drvdata(pdev);
766
767 dma_async_device_unregister(&device->common);
768}
769
770static void __devexit ioat_remove(struct pci_dev *pdev)
771{
772 struct ioat_device *device;
773 struct dma_chan *chan, *_chan;
774 struct ioat_dma_chan *ioat_chan;
775
776 device = pci_get_drvdata(pdev);
777 dma_async_device_unregister(&device->common);
778
779 free_irq(device->pdev->irq, device);
780#ifdef CONFIG_PCI_MSI
781 if (device->msi)
782 pci_disable_msi(device->pdev);
783#endif
784 pci_pool_destroy(device->dma_pool);
785 pci_pool_destroy(device->completion_pool);
786 iounmap(device->reg_base);
787 pci_release_regions(pdev);
788 pci_disable_device(pdev);
789 list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) {
790 ioat_chan = to_ioat_chan(chan);
791 list_del(&chan->device_node);
792 kfree(ioat_chan);
793 }
794 kfree(device);
795}
796
797/* MODULE API */
798MODULE_VERSION("1.9");
799MODULE_LICENSE("GPL");
800MODULE_AUTHOR("Intel Corporation");
801
802static int __init ioat_init_module(void)
803{
804 /* it's currently unsafe to unload this module */
805 /* if forced, worst case is that rmmod hangs */
806 __unsafe(THIS_MODULE);
807
808 return pci_register_driver(&ioat_pci_driver);
809}
810
811module_init(ioat_init_module);
812
813static void __exit ioat_exit_module(void)
814{
815 pci_unregister_driver(&ioat_pci_driver);
816}
817
818module_exit(ioat_exit_module);
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-18 04:55:59 -0500