aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma/ioatdma.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma/ioatdma.c')
-rw-r--r--drivers/dma/ioatdma.c839
1 files changed, 839 insertions, 0 deletions
diff --git a/drivers/dma/ioatdma.c b/drivers/dma/ioatdma.c
new file mode 100644
index 000000000000..11d48b97b36e
--- /dev/null
+++ b/drivers/dma/ioatdma.c
@@ -0,0 +1,839 @@
1/*
2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21
22/*
23 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
24 * copy operations.
25 */
26
27#include <linux/init.h>
28#include <linux/module.h>
29#include <linux/pci.h>
30#include <linux/interrupt.h>
31#include <linux/dmaengine.h>
32#include <linux/delay.h>
33#include "ioatdma.h"
34#include "ioatdma_io.h"
35#include "ioatdma_registers.h"
36#include "ioatdma_hw.h"
37
38#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
39#define to_ioat_device(dev) container_of(dev, struct ioat_device, common)
40#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
41
42/* internal functions */
43static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
44static void __devexit ioat_remove(struct pci_dev *pdev);
45
46static int enumerate_dma_channels(struct ioat_device *device)
47{
48 u8 xfercap_scale;
49 u32 xfercap;
50 int i;
51 struct ioat_dma_chan *ioat_chan;
52
53 device->common.chancnt = ioatdma_read8(device, IOAT_CHANCNT_OFFSET);
54 xfercap_scale = ioatdma_read8(device, IOAT_XFERCAP_OFFSET);
55 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
56
57 for (i = 0; i < device->common.chancnt; i++) {
58 ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
59 if (!ioat_chan) {
60 device->common.chancnt = i;
61 break;
62 }
63
64 ioat_chan->device = device;
65 ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
66 ioat_chan->xfercap = xfercap;
67 spin_lock_init(&ioat_chan->cleanup_lock);
68 spin_lock_init(&ioat_chan->desc_lock);
69 INIT_LIST_HEAD(&ioat_chan->free_desc);
70 INIT_LIST_HEAD(&ioat_chan->used_desc);
71 /* This should be made common somewhere in dmaengine.c */
72 ioat_chan->common.device = &device->common;
73 ioat_chan->common.client = NULL;
74 list_add_tail(&ioat_chan->common.device_node,
75 &device->common.channels);
76 }
77 return device->common.chancnt;
78}
79
80static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
81 struct ioat_dma_chan *ioat_chan,
82 int flags)
83{
84 struct ioat_dma_descriptor *desc;
85 struct ioat_desc_sw *desc_sw;
86 struct ioat_device *ioat_device;
87 dma_addr_t phys;
88
89 ioat_device = to_ioat_device(ioat_chan->common.device);
90 desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys);
91 if (unlikely(!desc))
92 return NULL;
93
94 desc_sw = kzalloc(sizeof(*desc_sw), flags);
95 if (unlikely(!desc_sw)) {
96 pci_pool_free(ioat_device->dma_pool, desc, phys);
97 return NULL;
98 }
99
100 memset(desc, 0, sizeof(*desc));
101 desc_sw->hw = desc;
102 desc_sw->phys = phys;
103
104 return desc_sw;
105}
106
107#define INITIAL_IOAT_DESC_COUNT 128
108
109static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan);
110
111/* returns the actual number of allocated descriptors */
112static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
113{
114 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
115 struct ioat_desc_sw *desc = NULL;
116 u16 chanctrl;
117 u32 chanerr;
118 int i;
119 LIST_HEAD(tmp_list);
120
121 /*
122 * In-use bit automatically set by reading chanctrl
123 * If 0, we got it, if 1, someone else did
124 */
125 chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET);
126 if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE)
127 return -EBUSY;
128
129 /* Setup register to interrupt and write completion status on error */
130 chanctrl = IOAT_CHANCTRL_CHANNEL_IN_USE |
131 IOAT_CHANCTRL_ERR_INT_EN |
132 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
133 IOAT_CHANCTRL_ERR_COMPLETION_EN;
134 ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl);
135
136 chanerr = ioatdma_chan_read32(ioat_chan, IOAT_CHANERR_OFFSET);
137 if (chanerr) {
138 printk("IOAT: CHANERR = %x, clearing\n", chanerr);
139 ioatdma_chan_write32(ioat_chan, IOAT_CHANERR_OFFSET, chanerr);
140 }
141
142 /* Allocate descriptors */
143 for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) {
144 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
145 if (!desc) {
146 printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i);
147 break;
148 }
149 list_add_tail(&desc->node, &tmp_list);
150 }
151 spin_lock_bh(&ioat_chan->desc_lock);
152 list_splice(&tmp_list, &ioat_chan->free_desc);
153 spin_unlock_bh(&ioat_chan->desc_lock);
154
155 /* allocate a completion writeback area */
156 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
157 ioat_chan->completion_virt =
158 pci_pool_alloc(ioat_chan->device->completion_pool,
159 GFP_KERNEL,
160 &ioat_chan->completion_addr);
161 memset(ioat_chan->completion_virt, 0,
162 sizeof(*ioat_chan->completion_virt));
163 ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_LOW,
164 ((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF);
165 ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_HIGH,
166 ((u64) ioat_chan->completion_addr) >> 32);
167
168 ioat_start_null_desc(ioat_chan);
169 return i;
170}
171
172static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
173
174static void ioat_dma_free_chan_resources(struct dma_chan *chan)
175{
176 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
177 struct ioat_device *ioat_device = to_ioat_device(chan->device);
178 struct ioat_desc_sw *desc, *_desc;
179 u16 chanctrl;
180 int in_use_descs = 0;
181
182 ioat_dma_memcpy_cleanup(ioat_chan);
183
184 ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET);
185
186 spin_lock_bh(&ioat_chan->desc_lock);
187 list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
188 in_use_descs++;
189 list_del(&desc->node);
190 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
191 kfree(desc);
192 }
193 list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
194 list_del(&desc->node);
195 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
196 kfree(desc);
197 }
198 spin_unlock_bh(&ioat_chan->desc_lock);
199
200 pci_pool_free(ioat_device->completion_pool,
201 ioat_chan->completion_virt,
202 ioat_chan->completion_addr);
203
204 /* one is ok since we left it on there on purpose */
205 if (in_use_descs > 1)
206 printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n",
207 in_use_descs - 1);
208
209 ioat_chan->last_completion = ioat_chan->completion_addr = 0;
210
211 /* Tell hw the chan is free */
212 chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET);
213 chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE;
214 ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl);
215}
216
217/**
218 * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction
219 * @chan: IOAT DMA channel handle
220 * @dest: DMA destination address
221 * @src: DMA source address
222 * @len: transaction length in bytes
223 */
224
225static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
226 dma_addr_t dest,
227 dma_addr_t src,
228 size_t len)
229{
230 struct ioat_desc_sw *first;
231 struct ioat_desc_sw *prev;
232 struct ioat_desc_sw *new;
233 dma_cookie_t cookie;
234 LIST_HEAD(new_chain);
235 u32 copy;
236 size_t orig_len;
237 dma_addr_t orig_src, orig_dst;
238 unsigned int desc_count = 0;
239 unsigned int append = 0;
240
241 if (!ioat_chan || !dest || !src)
242 return -EFAULT;
243
244 if (!len)
245 return ioat_chan->common.cookie;
246
247 orig_len = len;
248 orig_src = src;
249 orig_dst = dest;
250
251 first = NULL;
252 prev = NULL;
253
254 spin_lock_bh(&ioat_chan->desc_lock);
255
256 while (len) {
257 if (!list_empty(&ioat_chan->free_desc)) {
258 new = to_ioat_desc(ioat_chan->free_desc.next);
259 list_del(&new->node);
260 } else {
261 /* try to get another desc */
262 new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
263 /* will this ever happen? */
264 /* TODO add upper limit on these */
265 BUG_ON(!new);
266 }
267
268 copy = min((u32) len, ioat_chan->xfercap);
269
270 new->hw->size = copy;
271 new->hw->ctl = 0;
272 new->hw->src_addr = src;
273 new->hw->dst_addr = dest;
274 new->cookie = 0;
275
276 /* chain together the physical address list for the HW */
277 if (!first)
278 first = new;
279 else
280 prev->hw->next = (u64) new->phys;
281
282 prev = new;
283
284 len -= copy;
285 dest += copy;
286 src += copy;
287
288 list_add_tail(&new->node, &new_chain);
289 desc_count++;
290 }
291 new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
292 new->hw->next = 0;
293
294 /* cookie incr and addition to used_list must be atomic */
295
296 cookie = ioat_chan->common.cookie;
297 cookie++;
298 if (cookie < 0)
299 cookie = 1;
300 ioat_chan->common.cookie = new->cookie = cookie;
301
302 pci_unmap_addr_set(new, src, orig_src);
303 pci_unmap_addr_set(new, dst, orig_dst);
304 pci_unmap_len_set(new, src_len, orig_len);
305 pci_unmap_len_set(new, dst_len, orig_len);
306
307 /* write address into NextDescriptor field of last desc in chain */
308 to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys;
309 list_splice_init(&new_chain, ioat_chan->used_desc.prev);
310
311 ioat_chan->pending += desc_count;
312 if (ioat_chan->pending >= 20) {
313 append = 1;
314 ioat_chan->pending = 0;
315 }
316
317 spin_unlock_bh(&ioat_chan->desc_lock);
318
319 if (append)
320 ioatdma_chan_write8(ioat_chan,
321 IOAT_CHANCMD_OFFSET,
322 IOAT_CHANCMD_APPEND);
323 return cookie;
324}
325
326/**
327 * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs
328 * @chan: IOAT DMA channel handle
329 * @dest: DMA destination address
330 * @src: DMA source address
331 * @len: transaction length in bytes
332 */
333
334static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan,
335 void *dest,
336 void *src,
337 size_t len)
338{
339 dma_addr_t dest_addr;
340 dma_addr_t src_addr;
341 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
342
343 dest_addr = pci_map_single(ioat_chan->device->pdev,
344 dest, len, PCI_DMA_FROMDEVICE);
345 src_addr = pci_map_single(ioat_chan->device->pdev,
346 src, len, PCI_DMA_TODEVICE);
347
348 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
349}
350
351/**
352 * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page
353 * @chan: IOAT DMA channel handle
354 * @page: pointer to the page to copy to
355 * @offset: offset into that page
356 * @src: DMA source address
357 * @len: transaction length in bytes
358 */
359
360static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan,
361 struct page *page,
362 unsigned int offset,
363 void *src,
364 size_t len)
365{
366 dma_addr_t dest_addr;
367 dma_addr_t src_addr;
368 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
369
370 dest_addr = pci_map_page(ioat_chan->device->pdev,
371 page, offset, len, PCI_DMA_FROMDEVICE);
372 src_addr = pci_map_single(ioat_chan->device->pdev,
373 src, len, PCI_DMA_TODEVICE);
374
375 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
376}
377
378/**
379 * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages
380 * @chan: IOAT DMA channel handle
381 * @dest_pg: pointer to the page to copy to
382 * @dest_off: offset into that page
383 * @src_pg: pointer to the page to copy from
384 * @src_off: offset into that page
385 * @len: transaction length in bytes. This is guaranteed to not make a copy
386 * across a page boundary.
387 */
388
389static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan,
390 struct page *dest_pg,
391 unsigned int dest_off,
392 struct page *src_pg,
393 unsigned int src_off,
394 size_t len)
395{
396 dma_addr_t dest_addr;
397 dma_addr_t src_addr;
398 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
399
400 dest_addr = pci_map_page(ioat_chan->device->pdev,
401 dest_pg, dest_off, len, PCI_DMA_FROMDEVICE);
402 src_addr = pci_map_page(ioat_chan->device->pdev,
403 src_pg, src_off, len, PCI_DMA_TODEVICE);
404
405 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
406}
407
408/**
409 * ioat_dma_memcpy_issue_pending - push potentially unrecognoized appended descriptors to hw
410 * @chan: DMA channel handle
411 */
412
413static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan)
414{
415 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
416
417 if (ioat_chan->pending != 0) {
418 ioat_chan->pending = 0;
419 ioatdma_chan_write8(ioat_chan,
420 IOAT_CHANCMD_OFFSET,
421 IOAT_CHANCMD_APPEND);
422 }
423}
424
425static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
426{
427 unsigned long phys_complete;
428 struct ioat_desc_sw *desc, *_desc;
429 dma_cookie_t cookie = 0;
430
431 prefetch(chan->completion_virt);
432
433 if (!spin_trylock(&chan->cleanup_lock))
434 return;
435
436 /* The completion writeback can happen at any time,
437 so reads by the driver need to be atomic operations
438 The descriptor physical addresses are limited to 32-bits
439 when the CPU can only do a 32-bit mov */
440
441#if (BITS_PER_LONG == 64)
442 phys_complete =
443 chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
444#else
445 phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
446#endif
447
448 if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
449 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
450 printk("IOAT: Channel halted, chanerr = %x\n",
451 ioatdma_chan_read32(chan, IOAT_CHANERR_OFFSET));
452
453 /* TODO do something to salvage the situation */
454 }
455
456 if (phys_complete == chan->last_completion) {
457 spin_unlock(&chan->cleanup_lock);
458 return;
459 }
460
461 spin_lock_bh(&chan->desc_lock);
462 list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) {
463
464 /*
465 * Incoming DMA requests may use multiple descriptors, due to
466 * exceeding xfercap, perhaps. If so, only the last one will
467 * have a cookie, and require unmapping.
468 */
469 if (desc->cookie) {
470 cookie = desc->cookie;
471
472 /* yes we are unmapping both _page and _single alloc'd
473 regions with unmap_page. Is this *really* that bad?
474 */
475 pci_unmap_page(chan->device->pdev,
476 pci_unmap_addr(desc, dst),
477 pci_unmap_len(desc, dst_len),
478 PCI_DMA_FROMDEVICE);
479 pci_unmap_page(chan->device->pdev,
480 pci_unmap_addr(desc, src),
481 pci_unmap_len(desc, src_len),
482 PCI_DMA_TODEVICE);
483 }
484
485 if (desc->phys != phys_complete) {
486 /* a completed entry, but not the last, so cleanup */
487 list_del(&desc->node);
488 list_add_tail(&desc->node, &chan->free_desc);
489 } else {
490 /* last used desc. Do not remove, so we can append from
491 it, but don't look at it next time, either */
492 desc->cookie = 0;
493
494 /* TODO check status bits? */
495 break;
496 }
497 }
498
499 spin_unlock_bh(&chan->desc_lock);
500
501 chan->last_completion = phys_complete;
502 if (cookie != 0)
503 chan->completed_cookie = cookie;
504
505 spin_unlock(&chan->cleanup_lock);
506}
507
508/**
509 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
510 * @chan: IOAT DMA channel handle
511 * @cookie: DMA transaction identifier
512 */
513
514static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
515 dma_cookie_t cookie,
516 dma_cookie_t *done,
517 dma_cookie_t *used)
518{
519 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
520 dma_cookie_t last_used;
521 dma_cookie_t last_complete;
522 enum dma_status ret;
523
524 last_used = chan->cookie;
525 last_complete = ioat_chan->completed_cookie;
526
527 if (done)
528 *done= last_complete;
529 if (used)
530 *used = last_used;
531
532 ret = dma_async_is_complete(cookie, last_complete, last_used);
533 if (ret == DMA_SUCCESS)
534 return ret;
535
536 ioat_dma_memcpy_cleanup(ioat_chan);
537
538 last_used = chan->cookie;
539 last_complete = ioat_chan->completed_cookie;
540
541 if (done)
542 *done= last_complete;
543 if (used)
544 *used = last_used;
545
546 return dma_async_is_complete(cookie, last_complete, last_used);
547}
548
549/* PCI API */
550
551static struct pci_device_id ioat_pci_tbl[] = {
552 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
553 { 0, }
554};
555
556static struct pci_driver ioat_pci_drv = {
557 .name = "ioatdma",
558 .id_table = ioat_pci_tbl,
559 .probe = ioat_probe,
560 .remove = __devexit_p(ioat_remove),
561};
562
563static irqreturn_t ioat_do_interrupt(int irq, void *data, struct pt_regs *regs)
564{
565 struct ioat_device *instance = data;
566 unsigned long attnstatus;
567 u8 intrctrl;
568
569 intrctrl = ioatdma_read8(instance, IOAT_INTRCTRL_OFFSET);
570
571 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
572 return IRQ_NONE;
573
574 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
575 ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl);
576 return IRQ_NONE;
577 }
578
579 attnstatus = ioatdma_read32(instance, IOAT_ATTNSTATUS_OFFSET);
580
581 printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus);
582
583 ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl);
584 return IRQ_HANDLED;
585}
586
587static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan)
588{
589 struct ioat_desc_sw *desc;
590
591 spin_lock_bh(&ioat_chan->desc_lock);
592
593 if (!list_empty(&ioat_chan->free_desc)) {
594 desc = to_ioat_desc(ioat_chan->free_desc.next);
595 list_del(&desc->node);
596 } else {
597 /* try to get another desc */
598 spin_unlock_bh(&ioat_chan->desc_lock);
599 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
600 spin_lock_bh(&ioat_chan->desc_lock);
601 /* will this ever happen? */
602 BUG_ON(!desc);
603 }
604
605 desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
606 desc->hw->next = 0;
607
608 list_add_tail(&desc->node, &ioat_chan->used_desc);
609 spin_unlock_bh(&ioat_chan->desc_lock);
610
611#if (BITS_PER_LONG == 64)
612 ioatdma_chan_write64(ioat_chan, IOAT_CHAINADDR_OFFSET, desc->phys);
613#else
614 ioatdma_chan_write32(ioat_chan,
615 IOAT_CHAINADDR_OFFSET_LOW,
616 (u32) desc->phys);
617 ioatdma_chan_write32(ioat_chan, IOAT_CHAINADDR_OFFSET_HIGH, 0);
618#endif
619 ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_START);
620}
621
622/*
623 * Perform a IOAT transaction to verify the HW works.
624 */
625#define IOAT_TEST_SIZE 2000
626
627static int ioat_self_test(struct ioat_device *device)
628{
629 int i;
630 u8 *src;
631 u8 *dest;
632 struct dma_chan *dma_chan;
633 dma_cookie_t cookie;
634 int err = 0;
635
636 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, SLAB_KERNEL);
637 if (!src)
638 return -ENOMEM;
639 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, SLAB_KERNEL);
640 if (!dest) {
641 kfree(src);
642 return -ENOMEM;
643 }
644
645 /* Fill in src buffer */
646 for (i = 0; i < IOAT_TEST_SIZE; i++)
647 src[i] = (u8)i;
648
649 /* Start copy, using first DMA channel */
650 dma_chan = container_of(device->common.channels.next,
651 struct dma_chan,
652 device_node);
653 if (ioat_dma_alloc_chan_resources(dma_chan) < 1) {
654 err = -ENODEV;
655 goto out;
656 }
657
658 cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE);
659 ioat_dma_memcpy_issue_pending(dma_chan);
660 msleep(1);
661
662 if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
663 printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n");
664 err = -ENODEV;
665 goto free_resources;
666 }
667 if (memcmp(src, dest, IOAT_TEST_SIZE)) {
668 printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n");
669 err = -ENODEV;
670 goto free_resources;
671 }
672
673free_resources:
674 ioat_dma_free_chan_resources(dma_chan);
675out:
676 kfree(src);
677 kfree(dest);
678 return err;
679}
680
681static int __devinit ioat_probe(struct pci_dev *pdev,
682 const struct pci_device_id *ent)
683{
684 int err;
685 unsigned long mmio_start, mmio_len;
686 void *reg_base;
687 struct ioat_device *device;
688
689 err = pci_enable_device(pdev);
690 if (err)
691 goto err_enable_device;
692
693 err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
694 if (err)
695 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
696 if (err)
697 goto err_set_dma_mask;
698
699 err = pci_request_regions(pdev, ioat_pci_drv.name);
700 if (err)
701 goto err_request_regions;
702
703 mmio_start = pci_resource_start(pdev, 0);
704 mmio_len = pci_resource_len(pdev, 0);
705
706 reg_base = ioremap(mmio_start, mmio_len);
707 if (!reg_base) {
708 err = -ENOMEM;
709 goto err_ioremap;
710 }
711
712 device = kzalloc(sizeof(*device), GFP_KERNEL);
713 if (!device) {
714 err = -ENOMEM;
715 goto err_kzalloc;
716 }
717
718 /* DMA coherent memory pool for DMA descriptor allocations */
719 device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
720 sizeof(struct ioat_dma_descriptor), 64, 0);
721 if (!device->dma_pool) {
722 err = -ENOMEM;
723 goto err_dma_pool;
724 }
725
726 device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES);
727 if (!device->completion_pool) {
728 err = -ENOMEM;
729 goto err_completion_pool;
730 }
731
732 device->pdev = pdev;
733 pci_set_drvdata(pdev, device);
734#ifdef CONFIG_PCI_MSI
735 if (pci_enable_msi(pdev) == 0) {
736 device->msi = 1;
737 } else {
738 device->msi = 0;
739 }
740#endif
741 err = request_irq(pdev->irq, &ioat_do_interrupt, SA_SHIRQ, "ioat",
742 device);
743 if (err)
744 goto err_irq;
745
746 device->reg_base = reg_base;
747
748 ioatdma_write8(device, IOAT_INTRCTRL_OFFSET, IOAT_INTRCTRL_MASTER_INT_EN);
749 pci_set_master(pdev);
750
751 INIT_LIST_HEAD(&device->common.channels);
752 enumerate_dma_channels(device);
753
754 device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
755 device->common.device_free_chan_resources = ioat_dma_free_chan_resources;
756 device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf;
757 device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg;
758 device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg;
759 device->common.device_memcpy_complete = ioat_dma_is_complete;
760 device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending;
761 printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n",
762 device->common.chancnt);
763
764 err = ioat_self_test(device);
765 if (err)
766 goto err_self_test;
767
768 dma_async_device_register(&device->common);
769
770 return 0;
771
772err_self_test:
773err_irq:
774 pci_pool_destroy(device->completion_pool);
775err_completion_pool:
776 pci_pool_destroy(device->dma_pool);
777err_dma_pool:
778 kfree(device);
779err_kzalloc:
780 iounmap(reg_base);
781err_ioremap:
782 pci_release_regions(pdev);
783err_request_regions:
784err_set_dma_mask:
785 pci_disable_device(pdev);
786err_enable_device:
787 return err;
788}
789
790static void __devexit ioat_remove(struct pci_dev *pdev)
791{
792 struct ioat_device *device;
793 struct dma_chan *chan, *_chan;
794 struct ioat_dma_chan *ioat_chan;
795
796 device = pci_get_drvdata(pdev);
797 dma_async_device_unregister(&device->common);
798
799 free_irq(device->pdev->irq, device);
800#ifdef CONFIG_PCI_MSI
801 if (device->msi)
802 pci_disable_msi(device->pdev);
803#endif
804 pci_pool_destroy(device->dma_pool);
805 pci_pool_destroy(device->completion_pool);
806 iounmap(device->reg_base);
807 pci_release_regions(pdev);
808 pci_disable_device(pdev);
809 list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) {
810 ioat_chan = to_ioat_chan(chan);
811 list_del(&chan->device_node);
812 kfree(ioat_chan);
813 }
814 kfree(device);
815}
816
817/* MODULE API */
818MODULE_VERSION("1.7");
819MODULE_LICENSE("GPL");
820MODULE_AUTHOR("Intel Corporation");
821
822static int __init ioat_init_module(void)
823{
824 /* it's currently unsafe to unload this module */
825 /* if forced, worst case is that rmmod hangs */
826 if (THIS_MODULE != NULL)
827 THIS_MODULE->unsafe = 1;
828
829 return pci_module_init(&ioat_pci_drv);
830}
831
832module_init(ioat_init_module);
833
834static void __exit ioat_exit_module(void)
835{
836 pci_unregister_driver(&ioat_pci_drv);
837}
838
839module_exit(ioat_exit_module);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-27 02:49:34 -0500