aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma')
-rw-r--r--drivers/dma/Kconfig4
-rw-r--r--drivers/dma/dmaengine.c93
-rw-r--r--drivers/dma/dmatest.c40
-rw-r--r--drivers/dma/ioat/Makefile2
-rw-r--r--drivers/dma/ioat/dma.c136
-rw-r--r--drivers/dma/ioat/dma.h34
-rw-r--r--drivers/dma/ioat/dma_v2.c129
-rw-r--r--drivers/dma/ioat/dma_v2.h45
-rw-r--r--drivers/dma/ioat/dma_v3.c1220
-rw-r--r--drivers/dma/ioat/hw.h142
-rw-r--r--drivers/dma/ioat/pci.c45
-rw-r--r--drivers/dma/ioat/registers.h20
-rw-r--r--drivers/dma/iop-adma.c482
13 files changed, 2179 insertions, 213 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 81e1020fb514..fe1f3717b1ff 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -17,11 +17,15 @@ if DMADEVICES
17 17
18comment "DMA Devices" 18comment "DMA Devices"
19 19
20config ASYNC_TX_DISABLE_CHANNEL_SWITCH
21 bool
22
20config INTEL_IOATDMA 23config INTEL_IOATDMA
21 tristate "Intel I/OAT DMA support" 24 tristate "Intel I/OAT DMA support"
22 depends on PCI && X86 25 depends on PCI && X86
23 select DMA_ENGINE 26 select DMA_ENGINE
24 select DCA 27 select DCA
28 select ASYNC_TX_DISABLE_CHANNEL_SWITCH
25 help 29 help
26 Enable support for the Intel(R) I/OAT DMA engine present 30 Enable support for the Intel(R) I/OAT DMA engine present
27 in recent Intel Xeon chipsets. 31 in recent Intel Xeon chipsets.
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 562d182eae66..bd0b248de2cf 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -608,6 +608,40 @@ void dmaengine_put(void)
608} 608}
609EXPORT_SYMBOL(dmaengine_put); 609EXPORT_SYMBOL(dmaengine_put);
610 610
611static bool device_has_all_tx_types(struct dma_device *device)
612{
613 /* A device that satisfies this test has channels that will never cause
614 * an async_tx channel switch event as all possible operation types can
615 * be handled.
616 */
617 #ifdef CONFIG_ASYNC_TX_DMA
618 if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))
619 return false;
620 #endif
621
622 #if defined(CONFIG_ASYNC_MEMCPY) || defined(CONFIG_ASYNC_MEMCPY_MODULE)
623 if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))
624 return false;
625 #endif
626
627 #if defined(CONFIG_ASYNC_MEMSET) || defined(CONFIG_ASYNC_MEMSET_MODULE)
628 if (!dma_has_cap(DMA_MEMSET, device->cap_mask))
629 return false;
630 #endif
631
632 #if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE)
633 if (!dma_has_cap(DMA_XOR, device->cap_mask))
634 return false;
635 #endif
636
637 #if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE)
638 if (!dma_has_cap(DMA_PQ, device->cap_mask))
639 return false;
640 #endif
641
642 return true;
643}
644
611static int get_dma_id(struct dma_device *device) 645static int get_dma_id(struct dma_device *device)
612{ 646{
613 int rc; 647 int rc;
@@ -644,8 +678,12 @@ int dma_async_device_register(struct dma_device *device)
644 !device->device_prep_dma_memcpy); 678 !device->device_prep_dma_memcpy);
645 BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) && 679 BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) &&
646 !device->device_prep_dma_xor); 680 !device->device_prep_dma_xor);
647 BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) && 681 BUG_ON(dma_has_cap(DMA_XOR_VAL, device->cap_mask) &&
648 !device->device_prep_dma_zero_sum); 682 !device->device_prep_dma_xor_val);
683 BUG_ON(dma_has_cap(DMA_PQ, device->cap_mask) &&
684 !device->device_prep_dma_pq);
685 BUG_ON(dma_has_cap(DMA_PQ_VAL, device->cap_mask) &&
686 !device->device_prep_dma_pq_val);
649 BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) && 687 BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) &&
650 !device->device_prep_dma_memset); 688 !device->device_prep_dma_memset);
651 BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) && 689 BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) &&
@@ -661,6 +699,12 @@ int dma_async_device_register(struct dma_device *device)
661 BUG_ON(!device->device_issue_pending); 699 BUG_ON(!device->device_issue_pending);
662 BUG_ON(!device->dev); 700 BUG_ON(!device->dev);
663 701
702 /* note: this only matters in the
703 * CONFIG_ASYNC_TX_DISABLE_CHANNEL_SWITCH=y case
704 */
705 if (device_has_all_tx_types(device))
706 dma_cap_set(DMA_ASYNC_TX, device->cap_mask);
707
664 idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL); 708 idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL);
665 if (!idr_ref) 709 if (!idr_ref)
666 return -ENOMEM; 710 return -ENOMEM;
@@ -938,49 +982,24 @@ EXPORT_SYMBOL(dma_async_tx_descriptor_init);
938 982
939/* dma_wait_for_async_tx - spin wait for a transaction to complete 983/* dma_wait_for_async_tx - spin wait for a transaction to complete
940 * @tx: in-flight transaction to wait on 984 * @tx: in-flight transaction to wait on
941 *
942 * This routine assumes that tx was obtained from a call to async_memcpy,
943 * async_xor, async_memset, etc which ensures that tx is "in-flight" (prepped
944 * and submitted). Walking the parent chain is only meant to cover for DMA
945 * drivers that do not implement the DMA_INTERRUPT capability and may race with
946 * the driver's descriptor cleanup routine.
947 */ 985 */
948enum dma_status 986enum dma_status
949dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx) 987dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
950{ 988{
951 enum dma_status status; 989 unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
952 struct dma_async_tx_descriptor *iter;
953 struct dma_async_tx_descriptor *parent;
954 990
955 if (!tx) 991 if (!tx)
956 return DMA_SUCCESS; 992 return DMA_SUCCESS;
957 993
958 WARN_ONCE(tx->parent, "%s: speculatively walking dependency chain for" 994 while (tx->cookie == -EBUSY) {
959 " %s\n", __func__, dma_chan_name(tx->chan)); 995 if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
960 996 pr_err("%s timeout waiting for descriptor submission\n",
961 /* poll through the dependency chain, return when tx is complete */ 997 __func__);
962 do { 998 return DMA_ERROR;
963 iter = tx; 999 }
964 1000 cpu_relax();
965 /* find the root of the unsubmitted dependency chain */ 1001 }
966 do { 1002 return dma_sync_wait(tx->chan, tx->cookie);
967 parent = iter->parent;
968 if (!parent)
969 break;
970 else
971 iter = parent;
972 } while (parent);
973
974 /* there is a small window for ->parent == NULL and
975 * ->cookie == -EBUSY
976 */
977 while (iter->cookie == -EBUSY)
978 cpu_relax();
979
980 status = dma_sync_wait(iter->chan, iter->cookie);
981 } while (status == DMA_IN_PROGRESS || (iter != tx));
982
983 return status;
984} 1003}
985EXPORT_SYMBOL_GPL(dma_wait_for_async_tx); 1004EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
986 1005
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index d93017fc7872..a32a4cf7b1e0 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -48,6 +48,11 @@ module_param(xor_sources, uint, S_IRUGO);
48MODULE_PARM_DESC(xor_sources, 48MODULE_PARM_DESC(xor_sources,
49 "Number of xor source buffers (default: 3)"); 49 "Number of xor source buffers (default: 3)");
50 50
51static unsigned int pq_sources = 3;
52module_param(pq_sources, uint, S_IRUGO);
53MODULE_PARM_DESC(pq_sources,
54 "Number of p+q source buffers (default: 3)");
55
51/* 56/*
52 * Initialization patterns. All bytes in the source buffer has bit 7 57 * Initialization patterns. All bytes in the source buffer has bit 7
53 * set, all bytes in the destination buffer has bit 7 cleared. 58 * set, all bytes in the destination buffer has bit 7 cleared.
@@ -232,6 +237,7 @@ static int dmatest_func(void *data)
232 dma_cookie_t cookie; 237 dma_cookie_t cookie;
233 enum dma_status status; 238 enum dma_status status;
234 enum dma_ctrl_flags flags; 239 enum dma_ctrl_flags flags;
240 u8 pq_coefs[pq_sources];
235 int ret; 241 int ret;
236 int src_cnt; 242 int src_cnt;
237 int dst_cnt; 243 int dst_cnt;
@@ -248,6 +254,11 @@ static int dmatest_func(void *data)
248 else if (thread->type == DMA_XOR) { 254 else if (thread->type == DMA_XOR) {
249 src_cnt = xor_sources | 1; /* force odd to ensure dst = src */ 255 src_cnt = xor_sources | 1; /* force odd to ensure dst = src */
250 dst_cnt = 1; 256 dst_cnt = 1;
257 } else if (thread->type == DMA_PQ) {
258 src_cnt = pq_sources | 1; /* force odd to ensure dst = src */
259 dst_cnt = 2;
260 for (i = 0; i < pq_sources; i++)
261 pq_coefs[i] = 1;
251 } else 262 } else
252 goto err_srcs; 263 goto err_srcs;
253 264
@@ -283,6 +294,7 @@ static int dmatest_func(void *data)
283 dma_addr_t dma_dsts[dst_cnt]; 294 dma_addr_t dma_dsts[dst_cnt];
284 struct completion cmp; 295 struct completion cmp;
285 unsigned long tmo = msecs_to_jiffies(3000); 296 unsigned long tmo = msecs_to_jiffies(3000);
297 u8 align = 0;
286 298
287 total_tests++; 299 total_tests++;
288 300
@@ -290,6 +302,18 @@ static int dmatest_func(void *data)
290 src_off = dmatest_random() % (test_buf_size - len + 1); 302 src_off = dmatest_random() % (test_buf_size - len + 1);
291 dst_off = dmatest_random() % (test_buf_size - len + 1); 303 dst_off = dmatest_random() % (test_buf_size - len + 1);
292 304
305 /* honor alignment restrictions */
306 if (thread->type == DMA_MEMCPY)
307 align = dev->copy_align;
308 else if (thread->type == DMA_XOR)
309 align = dev->xor_align;
310 else if (thread->type == DMA_PQ)
311 align = dev->pq_align;
312
313 len = (len >> align) << align;
314 src_off = (src_off >> align) << align;
315 dst_off = (dst_off >> align) << align;
316
293 dmatest_init_srcs(thread->srcs, src_off, len); 317 dmatest_init_srcs(thread->srcs, src_off, len);
294 dmatest_init_dsts(thread->dsts, dst_off, len); 318 dmatest_init_dsts(thread->dsts, dst_off, len);
295 319
@@ -306,6 +330,7 @@ static int dmatest_func(void *data)
306 DMA_BIDIRECTIONAL); 330 DMA_BIDIRECTIONAL);
307 } 331 }
308 332
333
309 if (thread->type == DMA_MEMCPY) 334 if (thread->type == DMA_MEMCPY)
310 tx = dev->device_prep_dma_memcpy(chan, 335 tx = dev->device_prep_dma_memcpy(chan,
311 dma_dsts[0] + dst_off, 336 dma_dsts[0] + dst_off,
@@ -316,6 +341,15 @@ static int dmatest_func(void *data)
316 dma_dsts[0] + dst_off, 341 dma_dsts[0] + dst_off,
317 dma_srcs, xor_sources, 342 dma_srcs, xor_sources,
318 len, flags); 343 len, flags);
344 else if (thread->type == DMA_PQ) {
345 dma_addr_t dma_pq[dst_cnt];
346
347 for (i = 0; i < dst_cnt; i++)
348 dma_pq[i] = dma_dsts[i] + dst_off;
349 tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
350 pq_sources, pq_coefs,
351 len, flags);
352 }
319 353
320 if (!tx) { 354 if (!tx) {
321 for (i = 0; i < src_cnt; i++) 355 for (i = 0; i < src_cnt; i++)
@@ -459,6 +493,8 @@ static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_ty
459 op = "copy"; 493 op = "copy";
460 else if (type == DMA_XOR) 494 else if (type == DMA_XOR)
461 op = "xor"; 495 op = "xor";
496 else if (type == DMA_PQ)
497 op = "pq";
462 else 498 else
463 return -EINVAL; 499 return -EINVAL;
464 500
@@ -514,6 +550,10 @@ static int dmatest_add_channel(struct dma_chan *chan)
514 cnt = dmatest_add_threads(dtc, DMA_XOR); 550 cnt = dmatest_add_threads(dtc, DMA_XOR);
515 thread_count += cnt > 0 ? cnt : 0; 551 thread_count += cnt > 0 ? cnt : 0;
516 } 552 }
553 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
554 cnt = dmatest_add_threads(dtc, DMA_PQ);
555 thread_count += cnt > 0 ?: 0;
556 }
517 557
518 pr_info("dmatest: Started %u threads using %s\n", 558 pr_info("dmatest: Started %u threads using %s\n",
519 thread_count, dma_chan_name(chan)); 559 thread_count, dma_chan_name(chan));
diff --git a/drivers/dma/ioat/Makefile b/drivers/dma/ioat/Makefile
index 205a639e84df..8997d3fb9051 100644
--- a/drivers/dma/ioat/Makefile
+++ b/drivers/dma/ioat/Makefile
@@ -1,2 +1,2 @@
1obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o 1obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
2ioatdma-objs := pci.o dma.o dma_v2.o dca.o 2ioatdma-objs := pci.o dma.o dma_v2.o dma_v3.o dca.o
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 21527b89590c..c524d36d3c2e 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -263,6 +263,7 @@ static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
263 if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state)) 263 if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
264 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); 264 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
265 265
266 ioat->active += desc->hw->tx_cnt;
266 ioat->pending += desc->hw->tx_cnt; 267 ioat->pending += desc->hw->tx_cnt;
267 if (ioat->pending >= ioat_pending_level) 268 if (ioat->pending >= ioat_pending_level)
268 __ioat1_dma_memcpy_issue_pending(ioat); 269 __ioat1_dma_memcpy_issue_pending(ioat);
@@ -539,17 +540,6 @@ static void ioat1_cleanup_tasklet(unsigned long data)
539 writew(IOAT_CHANCTRL_RUN, chan->base.reg_base + IOAT_CHANCTRL_OFFSET); 540 writew(IOAT_CHANCTRL_RUN, chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
540} 541}
541 542
542static void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
543 int direction, enum dma_ctrl_flags flags, bool dst)
544{
545 if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) ||
546 (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE)))
547 pci_unmap_single(pdev, addr, len, direction);
548 else
549 pci_unmap_page(pdev, addr, len, direction);
550}
551
552
553void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags, 543void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
554 size_t len, struct ioat_dma_descriptor *hw) 544 size_t len, struct ioat_dma_descriptor *hw)
555{ 545{
@@ -623,6 +613,7 @@ static void __cleanup(struct ioat_dma_chan *ioat, unsigned long phys_complete)
623 chan->completed_cookie = tx->cookie; 613 chan->completed_cookie = tx->cookie;
624 tx->cookie = 0; 614 tx->cookie = 0;
625 ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw); 615 ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
616 ioat->active -= desc->hw->tx_cnt;
626 if (tx->callback) { 617 if (tx->callback) {
627 tx->callback(tx->callback_param); 618 tx->callback(tx->callback_param);
628 tx->callback = NULL; 619 tx->callback = NULL;
@@ -809,7 +800,7 @@ static void __devinit ioat_dma_test_callback(void *dma_async_param)
809 * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works. 800 * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
810 * @device: device to be tested 801 * @device: device to be tested
811 */ 802 */
812static int __devinit ioat_dma_self_test(struct ioatdma_device *device) 803int __devinit ioat_dma_self_test(struct ioatdma_device *device)
813{ 804{
814 int i; 805 int i;
815 u8 *src; 806 u8 *src;
@@ -1040,13 +1031,8 @@ int __devinit ioat_probe(struct ioatdma_device *device)
1040 dma_cap_set(DMA_MEMCPY, dma->cap_mask); 1031 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
1041 dma->dev = &pdev->dev; 1032 dma->dev = &pdev->dev;
1042 1033
1043 dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
1044 " %d channels, device version 0x%02x, driver version %s\n",
1045 dma->chancnt, device->version, IOAT_DMA_VERSION);
1046
1047 if (!dma->chancnt) { 1034 if (!dma->chancnt) {
1048 dev_err(dev, "Intel(R) I/OAT DMA Engine problem found: " 1035 dev_err(dev, "zero channels detected\n");
1049 "zero channels detected\n");
1050 goto err_setup_interrupts; 1036 goto err_setup_interrupts;
1051 } 1037 }
1052 1038
@@ -1054,7 +1040,7 @@ int __devinit ioat_probe(struct ioatdma_device *device)
1054 if (err) 1040 if (err)
1055 goto err_setup_interrupts; 1041 goto err_setup_interrupts;
1056 1042
1057 err = ioat_dma_self_test(device); 1043 err = device->self_test(device);
1058 if (err) 1044 if (err)
1059 goto err_self_test; 1045 goto err_self_test;
1060 1046
@@ -1097,6 +1083,113 @@ static void ioat1_intr_quirk(struct ioatdma_device *device)
1097 pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl); 1083 pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
1098} 1084}
1099 1085
1086static ssize_t ring_size_show(struct dma_chan *c, char *page)
1087{
1088 struct ioat_dma_chan *ioat = to_ioat_chan(c);
1089
1090 return sprintf(page, "%d\n", ioat->desccount);
1091}
1092static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
1093
1094static ssize_t ring_active_show(struct dma_chan *c, char *page)
1095{
1096 struct ioat_dma_chan *ioat = to_ioat_chan(c);
1097
1098 return sprintf(page, "%d\n", ioat->active);
1099}
1100static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
1101
1102static ssize_t cap_show(struct dma_chan *c, char *page)
1103{
1104 struct dma_device *dma = c->device;
1105
1106 return sprintf(page, "copy%s%s%s%s%s%s\n",
1107 dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "",
1108 dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "",
1109 dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "",
1110 dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "",
1111 dma_has_cap(DMA_MEMSET, dma->cap_mask) ? " fill" : "",
1112 dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : "");
1113
1114}
1115struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap);
1116
1117static ssize_t version_show(struct dma_chan *c, char *page)
1118{
1119 struct dma_device *dma = c->device;
1120 struct ioatdma_device *device = to_ioatdma_device(dma);
1121
1122 return sprintf(page, "%d.%d\n",
1123 device->version >> 4, device->version & 0xf);
1124}
1125struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version);
1126
1127static struct attribute *ioat1_attrs[] = {
1128 &ring_size_attr.attr,
1129 &ring_active_attr.attr,
1130 &ioat_cap_attr.attr,
1131 &ioat_version_attr.attr,
1132 NULL,
1133};
1134
1135static ssize_t
1136ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
1137{
1138 struct ioat_sysfs_entry *entry;
1139 struct ioat_chan_common *chan;
1140
1141 entry = container_of(attr, struct ioat_sysfs_entry, attr);
1142 chan = container_of(kobj, struct ioat_chan_common, kobj);
1143
1144 if (!entry->show)
1145 return -EIO;
1146 return entry->show(&chan->common, page);
1147}
1148
1149struct sysfs_ops ioat_sysfs_ops = {
1150 .show = ioat_attr_show,
1151};
1152
1153static struct kobj_type ioat1_ktype = {
1154 .sysfs_ops = &ioat_sysfs_ops,
1155 .default_attrs = ioat1_attrs,
1156};
1157
1158void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type)
1159{
1160 struct dma_device *dma = &device->common;
1161 struct dma_chan *c;
1162
1163 list_for_each_entry(c, &dma->channels, device_node) {
1164 struct ioat_chan_common *chan = to_chan_common(c);
1165 struct kobject *parent = &c->dev->device.kobj;
1166 int err;
1167
1168 err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata");
1169 if (err) {
1170 dev_warn(to_dev(chan),
1171 "sysfs init error (%d), continuing...\n", err);
1172 kobject_put(&chan->kobj);
1173 set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state);
1174 }
1175 }
1176}
1177
1178void ioat_kobject_del(struct ioatdma_device *device)
1179{
1180 struct dma_device *dma = &device->common;
1181 struct dma_chan *c;
1182
1183 list_for_each_entry(c, &dma->channels, device_node) {
1184 struct ioat_chan_common *chan = to_chan_common(c);
1185
1186 if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) {
1187 kobject_del(&chan->kobj);
1188 kobject_put(&chan->kobj);
1189 }
1190 }
1191}
1192
1100int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca) 1193int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca)
1101{ 1194{
1102 struct pci_dev *pdev = device->pdev; 1195 struct pci_dev *pdev = device->pdev;
@@ -1105,6 +1198,7 @@ int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca)
1105 1198
1106 device->intr_quirk = ioat1_intr_quirk; 1199 device->intr_quirk = ioat1_intr_quirk;
1107 device->enumerate_channels = ioat1_enumerate_channels; 1200 device->enumerate_channels = ioat1_enumerate_channels;
1201 device->self_test = ioat_dma_self_test;
1108 dma = &device->common; 1202 dma = &device->common;
1109 dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy; 1203 dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1110 dma->device_issue_pending = ioat1_dma_memcpy_issue_pending; 1204 dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
@@ -1119,6 +1213,8 @@ int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca)
1119 err = ioat_register(device); 1213 err = ioat_register(device);
1120 if (err) 1214 if (err)
1121 return err; 1215 return err;
1216 ioat_kobject_add(device, &ioat1_ktype);
1217
1122 if (dca) 1218 if (dca)
1123 device->dca = ioat_dca_init(pdev, device->reg_base); 1219 device->dca = ioat_dca_init(pdev, device->reg_base);
1124 1220
@@ -1131,6 +1227,8 @@ void __devexit ioat_dma_remove(struct ioatdma_device *device)
1131 1227
1132 ioat_disable_interrupts(device); 1228 ioat_disable_interrupts(device);
1133 1229
1230 ioat_kobject_del(device);
1231
1134 dma_async_device_unregister(dma); 1232 dma_async_device_unregister(dma);
1135 1233
1136 pci_pool_destroy(device->dma_pool); 1234 pci_pool_destroy(device->dma_pool);
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index 8966fa5453a7..6a675a2a2d1c 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -60,8 +60,12 @@
60 * @dca: direct cache access context 60 * @dca: direct cache access context
61 * @intr_quirk: interrupt setup quirk (for ioat_v1 devices) 61 * @intr_quirk: interrupt setup quirk (for ioat_v1 devices)
62 * @enumerate_channels: hw version specific channel enumeration 62 * @enumerate_channels: hw version specific channel enumeration
63 * @cleanup_tasklet: select between the v2 and v3 cleanup routines
64 * @timer_fn: select between the v2 and v3 timer watchdog routines
65 * @self_test: hardware version specific self test for each supported op type
66 *
67 * Note: the v3 cleanup routine supports raid operations
63 */ 68 */
64
65struct ioatdma_device { 69struct ioatdma_device {
66 struct pci_dev *pdev; 70 struct pci_dev *pdev;
67 void __iomem *reg_base; 71 void __iomem *reg_base;
@@ -74,6 +78,9 @@ struct ioatdma_device {
74 struct dca_provider *dca; 78 struct dca_provider *dca;
75 void (*intr_quirk)(struct ioatdma_device *device); 79 void (*intr_quirk)(struct ioatdma_device *device);
76 int (*enumerate_channels)(struct ioatdma_device *device); 80 int (*enumerate_channels)(struct ioatdma_device *device);
81 void (*cleanup_tasklet)(unsigned long data);
82 void (*timer_fn)(unsigned long data);
83 int (*self_test)(struct ioatdma_device *device);
77}; 84};
78 85
79struct ioat_chan_common { 86struct ioat_chan_common {
@@ -86,6 +93,7 @@ struct ioat_chan_common {
86 #define IOAT_COMPLETION_PENDING 0 93 #define IOAT_COMPLETION_PENDING 0
87 #define IOAT_COMPLETION_ACK 1 94 #define IOAT_COMPLETION_ACK 1
88 #define IOAT_RESET_PENDING 2 95 #define IOAT_RESET_PENDING 2
96 #define IOAT_KOBJ_INIT_FAIL 3
89 struct timer_list timer; 97 struct timer_list timer;
90 #define COMPLETION_TIMEOUT msecs_to_jiffies(100) 98 #define COMPLETION_TIMEOUT msecs_to_jiffies(100)
91 #define IDLE_TIMEOUT msecs_to_jiffies(2000) 99 #define IDLE_TIMEOUT msecs_to_jiffies(2000)
@@ -94,8 +102,13 @@ struct ioat_chan_common {
94 dma_addr_t completion_dma; 102 dma_addr_t completion_dma;
95 u64 *completion; 103 u64 *completion;
96 struct tasklet_struct cleanup_task; 104 struct tasklet_struct cleanup_task;
105 struct kobject kobj;
97}; 106};
98 107
108struct ioat_sysfs_entry {
109 struct attribute attr;
110 ssize_t (*show)(struct dma_chan *, char *);
111};
99 112
100/** 113/**
101 * struct ioat_dma_chan - internal representation of a DMA channel 114 * struct ioat_dma_chan - internal representation of a DMA channel
@@ -111,6 +124,7 @@ struct ioat_dma_chan {
111 124
112 int pending; 125 int pending;
113 u16 desccount; 126 u16 desccount;
127 u16 active;
114}; 128};
115 129
116static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c) 130static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c)
@@ -155,7 +169,7 @@ ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie,
155 169
156/** 170/**
157 * struct ioat_desc_sw - wrapper around hardware descriptor 171 * struct ioat_desc_sw - wrapper around hardware descriptor
158 * @hw: hardware DMA descriptor 172 * @hw: hardware DMA descriptor (for memcpy)
159 * @node: this descriptor will either be on the free list, 173 * @node: this descriptor will either be on the free list,
160 * or attached to a transaction list (tx_list) 174 * or attached to a transaction list (tx_list)
161 * @txd: the generic software descriptor for all engines 175 * @txd: the generic software descriptor for all engines
@@ -288,9 +302,20 @@ static inline bool is_ioat_bug(unsigned long err)
288 IOAT_CHANERR_LENGTH_ERR)); 302 IOAT_CHANERR_LENGTH_ERR));
289} 303}
290 304
305static inline void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
306 int direction, enum dma_ctrl_flags flags, bool dst)
307{
308 if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) ||
309 (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE)))
310 pci_unmap_single(pdev, addr, len, direction);
311 else
312 pci_unmap_page(pdev, addr, len, direction);
313}
314
291int __devinit ioat_probe(struct ioatdma_device *device); 315int __devinit ioat_probe(struct ioatdma_device *device);
292int __devinit ioat_register(struct ioatdma_device *device); 316int __devinit ioat_register(struct ioatdma_device *device);
293int __devinit ioat1_dma_probe(struct ioatdma_device *dev, int dca); 317int __devinit ioat1_dma_probe(struct ioatdma_device *dev, int dca);
318int __devinit ioat_dma_self_test(struct ioatdma_device *device);
294void __devexit ioat_dma_remove(struct ioatdma_device *device); 319void __devexit ioat_dma_remove(struct ioatdma_device *device);
295struct dca_provider * __devinit ioat_dca_init(struct pci_dev *pdev, 320struct dca_provider * __devinit ioat_dca_init(struct pci_dev *pdev,
296 void __iomem *iobase); 321 void __iomem *iobase);
@@ -304,4 +329,9 @@ void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
304 size_t len, struct ioat_dma_descriptor *hw); 329 size_t len, struct ioat_dma_descriptor *hw);
305bool ioat_cleanup_preamble(struct ioat_chan_common *chan, 330bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
306 unsigned long *phys_complete); 331 unsigned long *phys_complete);
332void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type);
333void ioat_kobject_del(struct ioatdma_device *device);
334extern struct sysfs_ops ioat_sysfs_ops;
335extern struct ioat_sysfs_entry ioat_version_attr;
336extern struct ioat_sysfs_entry ioat_cap_attr;
307#endif /* IOATDMA_H */ 337#endif /* IOATDMA_H */
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
index fa3d6db6624c..5d6ac49e0d32 100644
--- a/drivers/dma/ioat/dma_v2.c
+++ b/drivers/dma/ioat/dma_v2.c
@@ -39,7 +39,7 @@
39#include "registers.h" 39#include "registers.h"
40#include "hw.h" 40#include "hw.h"
41 41
42static int ioat_ring_alloc_order = 8; 42int ioat_ring_alloc_order = 8;
43module_param(ioat_ring_alloc_order, int, 0644); 43module_param(ioat_ring_alloc_order, int, 0644);
44MODULE_PARM_DESC(ioat_ring_alloc_order, 44MODULE_PARM_DESC(ioat_ring_alloc_order,
45 "ioat2+: allocate 2^n descriptors per channel (default: n=8)"); 45 "ioat2+: allocate 2^n descriptors per channel (default: n=8)");
@@ -48,7 +48,7 @@ module_param(ioat_ring_max_alloc_order, int, 0644);
48MODULE_PARM_DESC(ioat_ring_max_alloc_order, 48MODULE_PARM_DESC(ioat_ring_max_alloc_order,
49 "ioat2+: upper limit for dynamic ring resizing (default: n=16)"); 49 "ioat2+: upper limit for dynamic ring resizing (default: n=16)");
50 50
51static void __ioat2_issue_pending(struct ioat2_dma_chan *ioat) 51void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
52{ 52{
53 void * __iomem reg_base = ioat->base.reg_base; 53 void * __iomem reg_base = ioat->base.reg_base;
54 54
@@ -63,7 +63,7 @@ static void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
63 __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount); 63 __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
64} 64}
65 65
66static void ioat2_issue_pending(struct dma_chan *chan) 66void ioat2_issue_pending(struct dma_chan *chan)
67{ 67{
68 struct ioat2_dma_chan *ioat = to_ioat2_chan(chan); 68 struct ioat2_dma_chan *ioat = to_ioat2_chan(chan);
69 69
@@ -206,7 +206,7 @@ static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
206 spin_unlock_bh(&chan->cleanup_lock); 206 spin_unlock_bh(&chan->cleanup_lock);
207} 207}
208 208
209static void ioat2_cleanup_tasklet(unsigned long data) 209void ioat2_cleanup_tasklet(unsigned long data)
210{ 210{
211 struct ioat2_dma_chan *ioat = (void *) data; 211 struct ioat2_dma_chan *ioat = (void *) data;
212 212
@@ -214,7 +214,7 @@ static void ioat2_cleanup_tasklet(unsigned long data)
214 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); 214 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
215} 215}
216 216
217static void __restart_chan(struct ioat2_dma_chan *ioat) 217void __ioat2_restart_chan(struct ioat2_dma_chan *ioat)
218{ 218{
219 struct ioat_chan_common *chan = &ioat->base; 219 struct ioat_chan_common *chan = &ioat->base;
220 220
@@ -255,12 +255,10 @@ static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
255 if (ioat_cleanup_preamble(chan, &phys_complete)) 255 if (ioat_cleanup_preamble(chan, &phys_complete))
256 __cleanup(ioat, phys_complete); 256 __cleanup(ioat, phys_complete);
257 257
258 __restart_chan(ioat); 258 __ioat2_restart_chan(ioat);
259} 259}
260 260
261static bool reshape_ring(struct ioat2_dma_chan *ioat, int order); 261void ioat2_timer_event(unsigned long data)
262
263static void ioat2_timer_event(unsigned long data)
264{ 262{
265 struct ioat2_dma_chan *ioat = (void *) data; 263 struct ioat2_dma_chan *ioat = (void *) data;
266 struct ioat_chan_common *chan = &ioat->base; 264 struct ioat_chan_common *chan = &ioat->base;
@@ -321,7 +319,7 @@ static void ioat2_timer_event(unsigned long data)
321 * ioat2_enumerate_channels - find and initialize the device's channels 319 * ioat2_enumerate_channels - find and initialize the device's channels
322 * @device: the device to be enumerated 320 * @device: the device to be enumerated
323 */ 321 */
324static int ioat2_enumerate_channels(struct ioatdma_device *device) 322int ioat2_enumerate_channels(struct ioatdma_device *device)
325{ 323{
326 struct ioat2_dma_chan *ioat; 324 struct ioat2_dma_chan *ioat;
327 struct device *dev = &device->pdev->dev; 325 struct device *dev = &device->pdev->dev;
@@ -354,8 +352,8 @@ static int ioat2_enumerate_channels(struct ioatdma_device *device)
354 break; 352 break;
355 353
356 ioat_init_channel(device, &ioat->base, i, 354 ioat_init_channel(device, &ioat->base, i,
357 ioat2_timer_event, 355 device->timer_fn,
358 ioat2_cleanup_tasklet, 356 device->cleanup_tasklet,
359 (unsigned long) ioat); 357 (unsigned long) ioat);
360 ioat->xfercap_log = xfercap_log; 358 ioat->xfercap_log = xfercap_log;
361 spin_lock_init(&ioat->ring_lock); 359 spin_lock_init(&ioat->ring_lock);
@@ -461,7 +459,7 @@ static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gf
461/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring 459/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
462 * @chan: channel to be initialized 460 * @chan: channel to be initialized
463 */ 461 */
464static int ioat2_alloc_chan_resources(struct dma_chan *c) 462int ioat2_alloc_chan_resources(struct dma_chan *c)
465{ 463{
466 struct ioat2_dma_chan *ioat = to_ioat2_chan(c); 464 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
467 struct ioat_chan_common *chan = &ioat->base; 465 struct ioat_chan_common *chan = &ioat->base;
@@ -515,7 +513,7 @@ static int ioat2_alloc_chan_resources(struct dma_chan *c)
515 return 1 << ioat->alloc_order; 513 return 1 << ioat->alloc_order;
516} 514}
517 515
518static bool reshape_ring(struct ioat2_dma_chan *ioat, int order) 516bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
519{ 517{
520 /* reshape differs from normal ring allocation in that we want 518 /* reshape differs from normal ring allocation in that we want
521 * to allocate a new software ring while only 519 * to allocate a new software ring while only
@@ -628,7 +626,7 @@ static bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
628 * @ioat: ioat2,3 channel (ring) to operate on 626 * @ioat: ioat2,3 channel (ring) to operate on
629 * @num_descs: allocation length 627 * @num_descs: allocation length
630 */ 628 */
631static int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs) 629int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs)
632{ 630{
633 struct ioat_chan_common *chan = &ioat->base; 631 struct ioat_chan_common *chan = &ioat->base;
634 632
@@ -656,9 +654,11 @@ static int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_d
656 spin_lock_bh(&chan->cleanup_lock); 654 spin_lock_bh(&chan->cleanup_lock);
657 if (jiffies > chan->timer.expires && 655 if (jiffies > chan->timer.expires &&
658 timer_pending(&chan->timer)) { 656 timer_pending(&chan->timer)) {
657 struct ioatdma_device *device = chan->device;
658
659 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); 659 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
660 spin_unlock_bh(&chan->cleanup_lock); 660 spin_unlock_bh(&chan->cleanup_lock);
661 ioat2_timer_event((unsigned long) ioat); 661 device->timer_fn((unsigned long) ioat);
662 } else 662 } else
663 spin_unlock_bh(&chan->cleanup_lock); 663 spin_unlock_bh(&chan->cleanup_lock);
664 return -ENOMEM; 664 return -ENOMEM;
@@ -671,7 +671,7 @@ static int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_d
671 return 0; /* with ioat->ring_lock held */ 671 return 0; /* with ioat->ring_lock held */
672} 672}
673 673
674static struct dma_async_tx_descriptor * 674struct dma_async_tx_descriptor *
675ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, 675ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
676 dma_addr_t dma_src, size_t len, unsigned long flags) 676 dma_addr_t dma_src, size_t len, unsigned long flags)
677{ 677{
@@ -711,6 +711,7 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
711 desc->txd.flags = flags; 711 desc->txd.flags = flags;
712 desc->len = total_len; 712 desc->len = total_len;
713 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 713 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
714 hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
714 hw->ctl_f.compl_write = 1; 715 hw->ctl_f.compl_write = 1;
715 dump_desc_dbg(ioat, desc); 716 dump_desc_dbg(ioat, desc);
716 /* we leave the channel locked to ensure in order submission */ 717 /* we leave the channel locked to ensure in order submission */
@@ -722,11 +723,11 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
722 * ioat2_free_chan_resources - release all the descriptors 723 * ioat2_free_chan_resources - release all the descriptors
723 * @chan: the channel to be cleaned 724 * @chan: the channel to be cleaned
724 */ 725 */
725static void ioat2_free_chan_resources(struct dma_chan *c) 726void ioat2_free_chan_resources(struct dma_chan *c)
726{ 727{
727 struct ioat2_dma_chan *ioat = to_ioat2_chan(c); 728 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
728 struct ioat_chan_common *chan = &ioat->base; 729 struct ioat_chan_common *chan = &ioat->base;
729 struct ioatdma_device *ioatdma_device = chan->device; 730 struct ioatdma_device *device = chan->device;
730 struct ioat_ring_ent *desc; 731 struct ioat_ring_ent *desc;
731 const u16 total_descs = 1 << ioat->alloc_order; 732 const u16 total_descs = 1 << ioat->alloc_order;
732 int descs; 733 int descs;
@@ -740,7 +741,7 @@ static void ioat2_free_chan_resources(struct dma_chan *c)
740 741
741 tasklet_disable(&chan->cleanup_task); 742 tasklet_disable(&chan->cleanup_task);
742 del_timer_sync(&chan->timer); 743 del_timer_sync(&chan->timer);
743 ioat2_cleanup(ioat); 744 device->cleanup_tasklet((unsigned long) ioat);
744 745
745 /* Delay 100ms after reset to allow internal DMA logic to quiesce 746 /* Delay 100ms after reset to allow internal DMA logic to quiesce
746 * before removing DMA descriptor resources. 747 * before removing DMA descriptor resources.
@@ -770,8 +771,7 @@ static void ioat2_free_chan_resources(struct dma_chan *c)
770 kfree(ioat->ring); 771 kfree(ioat->ring);
771 ioat->ring = NULL; 772 ioat->ring = NULL;
772 ioat->alloc_order = 0; 773 ioat->alloc_order = 0;
773 pci_pool_free(ioatdma_device->completion_pool, 774 pci_pool_free(device->completion_pool, chan->completion,
774 chan->completion,
775 chan->completion_dma); 775 chan->completion_dma);
776 spin_unlock_bh(&ioat->ring_lock); 776 spin_unlock_bh(&ioat->ring_lock);
777 777
@@ -781,66 +781,63 @@ static void ioat2_free_chan_resources(struct dma_chan *c)
781 ioat->dmacount = 0; 781 ioat->dmacount = 0;
782} 782}
783 783
784static enum dma_status 784enum dma_status
785ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie, 785ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
786 dma_cookie_t *done, dma_cookie_t *used) 786 dma_cookie_t *done, dma_cookie_t *used)
787{ 787{
788 struct ioat2_dma_chan *ioat = to_ioat2_chan(c); 788 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
789 struct ioatdma_device *device = ioat->base.device;
789 790
790 if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS) 791 if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
791 return DMA_SUCCESS; 792 return DMA_SUCCESS;
792 793
793 ioat2_cleanup(ioat); 794 device->cleanup_tasklet((unsigned long) ioat);
794 795
795 return ioat_is_complete(c, cookie, done, used); 796 return ioat_is_complete(c, cookie, done, used);
796} 797}
797 798
798int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca) 799static ssize_t ring_size_show(struct dma_chan *c, char *page)
799{ 800{
800 struct pci_dev *pdev = device->pdev; 801 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
801 struct dma_device *dma;
802 struct dma_chan *c;
803 struct ioat_chan_common *chan;
804 int err;
805 802
806 device->enumerate_channels = ioat2_enumerate_channels; 803 return sprintf(page, "%d\n", (1 << ioat->alloc_order) & ~1);
807 dma = &device->common; 804}
808 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock; 805static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
809 dma->device_issue_pending = ioat2_issue_pending;
810 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
811 dma->device_free_chan_resources = ioat2_free_chan_resources;
812 dma->device_is_tx_complete = ioat2_is_complete;
813 806
814 err = ioat_probe(device); 807static ssize_t ring_active_show(struct dma_chan *c, char *page)
815 if (err) 808{
816 return err; 809 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
817 ioat_set_tcp_copy_break(2048);
818 810
819 list_for_each_entry(c, &dma->channels, device_node) { 811 /* ...taken outside the lock, no need to be precise */
820 chan = to_chan_common(c); 812 return sprintf(page, "%d\n", ioat2_ring_active(ioat));
821 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU, 813}
822 chan->reg_base + IOAT_DCACTRL_OFFSET); 814static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
823 }
824 815
825 err = ioat_register(device); 816static struct attribute *ioat2_attrs[] = {
826 if (err) 817 &ring_size_attr.attr,
827 return err; 818 &ring_active_attr.attr,
828 if (dca) 819 &ioat_cap_attr.attr,
829 device->dca = ioat2_dca_init(pdev, device->reg_base); 820 &ioat_version_attr.attr,
821 NULL,
822};
830 823
831 return err; 824struct kobj_type ioat2_ktype = {
832} 825 .sysfs_ops = &ioat_sysfs_ops,
826 .default_attrs = ioat2_attrs,
827};
833 828
834int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) 829int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca)
835{ 830{
836 struct pci_dev *pdev = device->pdev; 831 struct pci_dev *pdev = device->pdev;
837 struct dma_device *dma; 832 struct dma_device *dma;
838 struct dma_chan *c; 833 struct dma_chan *c;
839 struct ioat_chan_common *chan; 834 struct ioat_chan_common *chan;
840 int err; 835 int err;
841 u16 dev_id;
842 836
843 device->enumerate_channels = ioat2_enumerate_channels; 837 device->enumerate_channels = ioat2_enumerate_channels;
838 device->cleanup_tasklet = ioat2_cleanup_tasklet;
839 device->timer_fn = ioat2_timer_event;
840 device->self_test = ioat_dma_self_test;
844 dma = &device->common; 841 dma = &device->common;
845 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock; 842 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
846 dma->device_issue_pending = ioat2_issue_pending; 843 dma->device_issue_pending = ioat2_issue_pending;
@@ -848,35 +845,25 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca)
848 dma->device_free_chan_resources = ioat2_free_chan_resources; 845 dma->device_free_chan_resources = ioat2_free_chan_resources;
849 dma->device_is_tx_complete = ioat2_is_complete; 846 dma->device_is_tx_complete = ioat2_is_complete;
850 847
851 /* -= IOAT ver.3 workarounds =- */
852 /* Write CHANERRMSK_INT with 3E07h to mask out the errors
853 * that can cause stability issues for IOAT ver.3
854 */
855 pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
856
857 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
858 * (workaround for spurious config parity error after restart)
859 */
860 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
861 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
862 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
863
864 err = ioat_probe(device); 848 err = ioat_probe(device);
865 if (err) 849 if (err)
866 return err; 850 return err;
867 ioat_set_tcp_copy_break(262144); 851 ioat_set_tcp_copy_break(2048);
868 852
869 list_for_each_entry(c, &dma->channels, device_node) { 853 list_for_each_entry(c, &dma->channels, device_node) {
870 chan = to_chan_common(c); 854 chan = to_chan_common(c);
871 writel(IOAT_DMA_DCA_ANY_CPU, 855 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
872 chan->reg_base + IOAT_DCACTRL_OFFSET); 856 chan->reg_base + IOAT_DCACTRL_OFFSET);
873 } 857 }
874 858
875 err = ioat_register(device); 859 err = ioat_register(device);
876 if (err) 860 if (err)
877 return err; 861 return err;
862
863 ioat_kobject_add(device, &ioat2_ktype);
864
878 if (dca) 865 if (dca)
879 device->dca = ioat3_dca_init(pdev, device->reg_base); 866 device->dca = ioat2_dca_init(pdev, device->reg_base);
880 867
881 return err; 868 return err;
882} 869}
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
index ac00adc81974..1d849ef74d5f 100644
--- a/drivers/dma/ioat/dma_v2.h
+++ b/drivers/dma/ioat/dma_v2.h
@@ -27,6 +27,7 @@
27 27
28 28
29extern int ioat_pending_level; 29extern int ioat_pending_level;
30extern int ioat_ring_alloc_order;
30 31
31/* 32/*
32 * workaround for IOAT ver.3.0 null descriptor issue 33 * workaround for IOAT ver.3.0 null descriptor issue
@@ -114,10 +115,36 @@ static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len
114 return num_descs; 115 return num_descs;
115} 116}
116 117
118/**
119 * struct ioat_ring_ent - wrapper around hardware descriptor
120 * @hw: hardware DMA descriptor (for memcpy)
121 * @fill: hardware fill descriptor
122 * @xor: hardware xor descriptor
123 * @xor_ex: hardware xor extension descriptor
124 * @pq: hardware pq descriptor
125 * @pq_ex: hardware pq extension descriptor
126 * @pqu: hardware pq update descriptor
127 * @raw: hardware raw (un-typed) descriptor
128 * @txd: the generic software descriptor for all engines
129 * @len: total transaction length for unmap
130 * @result: asynchronous result of validate operations
131 * @id: identifier for debug
132 */
133
117struct ioat_ring_ent { 134struct ioat_ring_ent {
118 struct ioat_dma_descriptor *hw; 135 union {
136 struct ioat_dma_descriptor *hw;
137 struct ioat_fill_descriptor *fill;
138 struct ioat_xor_descriptor *xor;
139 struct ioat_xor_ext_descriptor *xor_ex;
140 struct ioat_pq_descriptor *pq;
141 struct ioat_pq_ext_descriptor *pq_ex;
142 struct ioat_pq_update_descriptor *pqu;
143 struct ioat_raw_descriptor *raw;
144 };
119 size_t len; 145 size_t len;
120 struct dma_async_tx_descriptor txd; 146 struct dma_async_tx_descriptor txd;
147 enum sum_check_flags *result;
121 #ifdef DEBUG 148 #ifdef DEBUG
122 int id; 149 int id;
123 #endif 150 #endif
@@ -143,5 +170,21 @@ int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca);
143int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca); 170int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca);
144struct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase); 171struct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
145struct dca_provider * __devinit ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); 172struct dca_provider * __devinit ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
173int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs);
174int ioat2_enumerate_channels(struct ioatdma_device *device);
175struct dma_async_tx_descriptor *
176ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
177 dma_addr_t dma_src, size_t len, unsigned long flags);
178void ioat2_issue_pending(struct dma_chan *chan);
179int ioat2_alloc_chan_resources(struct dma_chan *c);
180void ioat2_free_chan_resources(struct dma_chan *c);
181enum dma_status ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
182 dma_cookie_t *done, dma_cookie_t *used);
183void __ioat2_restart_chan(struct ioat2_dma_chan *ioat);
184bool reshape_ring(struct ioat2_dma_chan *ioat, int order);
185void __ioat2_issue_pending(struct ioat2_dma_chan *ioat);
186void ioat2_cleanup_tasklet(unsigned long data);
187void ioat2_timer_event(unsigned long data);
188extern struct kobj_type ioat2_ktype;
146extern struct kmem_cache *ioat2_cache; 189extern struct kmem_cache *ioat2_cache;
147#endif /* IOATDMA_V2_H */ 190#endif /* IOATDMA_V2_H */
diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c
new file mode 100644
index 000000000000..3686dddf6bff
--- /dev/null
+++ b/drivers/dma/ioat/dma_v3.c
@@ -0,0 +1,1220 @@
1/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
8 *
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms and conditions of the GNU General Public License,
11 * version 2, as published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
21 *
22 * The full GNU General Public License is included in this distribution in
23 * the file called "COPYING".
24 *
25 * BSD LICENSE
26 *
27 * Copyright(c) 2004-2009 Intel Corporation. All rights reserved.
28 *
29 * Redistribution and use in source and binary forms, with or without
30 * modification, are permitted provided that the following conditions are met:
31 *
32 * * Redistributions of source code must retain the above copyright
33 * notice, this list of conditions and the following disclaimer.
34 * * Redistributions in binary form must reproduce the above copyright
35 * notice, this list of conditions and the following disclaimer in
36 * the documentation and/or other materials provided with the
37 * distribution.
38 * * Neither the name of Intel Corporation nor the names of its
39 * contributors may be used to endorse or promote products derived
40 * from this software without specific prior written permission.
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
43 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
44 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
45 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
46 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
47 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
48 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
49 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
50 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
51 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
52 * POSSIBILITY OF SUCH DAMAGE.
53 */
54
55/*
56 * Support routines for v3+ hardware
57 */
58
59#include <linux/pci.h>
60#include <linux/dmaengine.h>
61#include <linux/dma-mapping.h>
62#include "registers.h"
63#include "hw.h"
64#include "dma.h"
65#include "dma_v2.h"
66
67/* ioat hardware assumes at least two sources for raid operations */
68#define src_cnt_to_sw(x) ((x) + 2)
69#define src_cnt_to_hw(x) ((x) - 2)
70
71/* provide a lookup table for setting the source address in the base or
72 * extended descriptor of an xor or pq descriptor
73 */
74static const u8 xor_idx_to_desc __read_mostly = 0xd0;
75static const u8 xor_idx_to_field[] __read_mostly = { 1, 4, 5, 6, 7, 0, 1, 2 };
76static const u8 pq_idx_to_desc __read_mostly = 0xf8;
77static const u8 pq_idx_to_field[] __read_mostly = { 1, 4, 5, 0, 1, 2, 4, 5 };
78
79static dma_addr_t xor_get_src(struct ioat_raw_descriptor *descs[2], int idx)
80{
81 struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
82
83 return raw->field[xor_idx_to_field[idx]];
84}
85
86static void xor_set_src(struct ioat_raw_descriptor *descs[2],
87 dma_addr_t addr, u32 offset, int idx)
88{
89 struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
90
91 raw->field[xor_idx_to_field[idx]] = addr + offset;
92}
93
94static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx)
95{
96 struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
97
98 return raw->field[pq_idx_to_field[idx]];
99}
100
101static void pq_set_src(struct ioat_raw_descriptor *descs[2],
102 dma_addr_t addr, u32 offset, u8 coef, int idx)
103{
104 struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0];
105 struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
106
107 raw->field[pq_idx_to_field[idx]] = addr + offset;
108 pq->coef[idx] = coef;
109}
110
111static void ioat3_dma_unmap(struct ioat2_dma_chan *ioat,
112 struct ioat_ring_ent *desc, int idx)
113{
114 struct ioat_chan_common *chan = &ioat->base;
115 struct pci_dev *pdev = chan->device->pdev;
116 size_t len = desc->len;
117 size_t offset = len - desc->hw->size;
118 struct dma_async_tx_descriptor *tx = &desc->txd;
119 enum dma_ctrl_flags flags = tx->flags;
120
121 switch (desc->hw->ctl_f.op) {
122 case IOAT_OP_COPY:
123 if (!desc->hw->ctl_f.null) /* skip 'interrupt' ops */
124 ioat_dma_unmap(chan, flags, len, desc->hw);
125 break;
126 case IOAT_OP_FILL: {
127 struct ioat_fill_descriptor *hw = desc->fill;
128
129 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
130 ioat_unmap(pdev, hw->dst_addr - offset, len,
131 PCI_DMA_FROMDEVICE, flags, 1);
132 break;
133 }
134 case IOAT_OP_XOR_VAL:
135 case IOAT_OP_XOR: {
136 struct ioat_xor_descriptor *xor = desc->xor;
137 struct ioat_ring_ent *ext;
138 struct ioat_xor_ext_descriptor *xor_ex = NULL;
139 int src_cnt = src_cnt_to_sw(xor->ctl_f.src_cnt);
140 struct ioat_raw_descriptor *descs[2];
141 int i;
142
143 if (src_cnt > 5) {
144 ext = ioat2_get_ring_ent(ioat, idx + 1);
145 xor_ex = ext->xor_ex;
146 }
147
148 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
149 descs[0] = (struct ioat_raw_descriptor *) xor;
150 descs[1] = (struct ioat_raw_descriptor *) xor_ex;
151 for (i = 0; i < src_cnt; i++) {
152 dma_addr_t src = xor_get_src(descs, i);
153
154 ioat_unmap(pdev, src - offset, len,
155 PCI_DMA_TODEVICE, flags, 0);
156 }
157
158 /* dest is a source in xor validate operations */
159 if (xor->ctl_f.op == IOAT_OP_XOR_VAL) {
160 ioat_unmap(pdev, xor->dst_addr - offset, len,
161 PCI_DMA_TODEVICE, flags, 1);
162 break;
163 }
164 }
165
166 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
167 ioat_unmap(pdev, xor->dst_addr - offset, len,
168 PCI_DMA_FROMDEVICE, flags, 1);
169 break;
170 }
171 case IOAT_OP_PQ_VAL:
172 case IOAT_OP_PQ: {
173 struct ioat_pq_descriptor *pq = desc->pq;
174 struct ioat_ring_ent *ext;
175 struct ioat_pq_ext_descriptor *pq_ex = NULL;
176 int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
177 struct ioat_raw_descriptor *descs[2];
178 int i;
179
180 if (src_cnt > 3) {
181 ext = ioat2_get_ring_ent(ioat, idx + 1);
182 pq_ex = ext->pq_ex;
183 }
184
185 /* in the 'continue' case don't unmap the dests as sources */
186 if (dmaf_p_disabled_continue(flags))
187 src_cnt--;
188 else if (dmaf_continue(flags))
189 src_cnt -= 3;
190
191 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
192 descs[0] = (struct ioat_raw_descriptor *) pq;
193 descs[1] = (struct ioat_raw_descriptor *) pq_ex;
194 for (i = 0; i < src_cnt; i++) {
195 dma_addr_t src = pq_get_src(descs, i);
196
197 ioat_unmap(pdev, src - offset, len,
198 PCI_DMA_TODEVICE, flags, 0);
199 }
200
201 /* the dests are sources in pq validate operations */
202 if (pq->ctl_f.op == IOAT_OP_XOR_VAL) {
203 if (!(flags & DMA_PREP_PQ_DISABLE_P))
204 ioat_unmap(pdev, pq->p_addr - offset,
205 len, PCI_DMA_TODEVICE, flags, 0);
206 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
207 ioat_unmap(pdev, pq->q_addr - offset,
208 len, PCI_DMA_TODEVICE, flags, 0);
209 break;
210 }
211 }
212
213 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
214 if (!(flags & DMA_PREP_PQ_DISABLE_P))
215 ioat_unmap(pdev, pq->p_addr - offset, len,
216 PCI_DMA_BIDIRECTIONAL, flags, 1);
217 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
218 ioat_unmap(pdev, pq->q_addr - offset, len,
219 PCI_DMA_BIDIRECTIONAL, flags, 1);
220 }
221 break;
222 }
223 default:
224 dev_err(&pdev->dev, "%s: unknown op type: %#x\n",
225 __func__, desc->hw->ctl_f.op);
226 }
227}
228
229static bool desc_has_ext(struct ioat_ring_ent *desc)
230{
231 struct ioat_dma_descriptor *hw = desc->hw;
232
233 if (hw->ctl_f.op == IOAT_OP_XOR ||
234 hw->ctl_f.op == IOAT_OP_XOR_VAL) {
235 struct ioat_xor_descriptor *xor = desc->xor;
236
237 if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
238 return true;
239 } else if (hw->ctl_f.op == IOAT_OP_PQ ||
240 hw->ctl_f.op == IOAT_OP_PQ_VAL) {
241 struct ioat_pq_descriptor *pq = desc->pq;
242
243 if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3)
244 return true;
245 }
246
247 return false;
248}
249
250/**
251 * __cleanup - reclaim used descriptors
252 * @ioat: channel (ring) to clean
253 *
254 * The difference from the dma_v2.c __cleanup() is that this routine
255 * handles extended descriptors and dma-unmapping raid operations.
256 */
257static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
258{
259 struct ioat_chan_common *chan = &ioat->base;
260 struct ioat_ring_ent *desc;
261 bool seen_current = false;
262 u16 active;
263 int i;
264
265 dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
266 __func__, ioat->head, ioat->tail, ioat->issued);
267
268 active = ioat2_ring_active(ioat);
269 for (i = 0; i < active && !seen_current; i++) {
270 struct dma_async_tx_descriptor *tx;
271
272 prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
273 desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
274 dump_desc_dbg(ioat, desc);
275 tx = &desc->txd;
276 if (tx->cookie) {
277 chan->completed_cookie = tx->cookie;
278 ioat3_dma_unmap(ioat, desc, ioat->tail + i);
279 tx->cookie = 0;
280 if (tx->callback) {
281 tx->callback(tx->callback_param);
282 tx->callback = NULL;
283 }
284 }
285
286 if (tx->phys == phys_complete)
287 seen_current = true;
288
289 /* skip extended descriptors */
290 if (desc_has_ext(desc)) {
291 BUG_ON(i + 1 >= active);
292 i++;
293 }
294 }
295 ioat->tail += i;
296 BUG_ON(!seen_current); /* no active descs have written a completion? */
297 chan->last_completion = phys_complete;
298 if (ioat->head == ioat->tail) {
299 dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
300 __func__);
301 clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
302 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
303 }
304}
305
306static void ioat3_cleanup(struct ioat2_dma_chan *ioat)
307{
308 struct ioat_chan_common *chan = &ioat->base;
309 unsigned long phys_complete;
310
311 prefetch(chan->completion);
312
313 if (!spin_trylock_bh(&chan->cleanup_lock))
314 return;
315
316 if (!ioat_cleanup_preamble(chan, &phys_complete)) {
317 spin_unlock_bh(&chan->cleanup_lock);
318 return;
319 }
320
321 if (!spin_trylock_bh(&ioat->ring_lock)) {
322 spin_unlock_bh(&chan->cleanup_lock);
323 return;
324 }
325
326 __cleanup(ioat, phys_complete);
327
328 spin_unlock_bh(&ioat->ring_lock);
329 spin_unlock_bh(&chan->cleanup_lock);
330}
331
332static void ioat3_cleanup_tasklet(unsigned long data)
333{
334 struct ioat2_dma_chan *ioat = (void *) data;
335
336 ioat3_cleanup(ioat);
337 writew(IOAT_CHANCTRL_RUN | IOAT3_CHANCTRL_COMPL_DCA_EN,
338 ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
339}
340
341static void ioat3_restart_channel(struct ioat2_dma_chan *ioat)
342{
343 struct ioat_chan_common *chan = &ioat->base;
344 unsigned long phys_complete;
345 u32 status;
346
347 status = ioat_chansts(chan);
348 if (is_ioat_active(status) || is_ioat_idle(status))
349 ioat_suspend(chan);
350 while (is_ioat_active(status) || is_ioat_idle(status)) {
351 status = ioat_chansts(chan);
352 cpu_relax();
353 }
354
355 if (ioat_cleanup_preamble(chan, &phys_complete))
356 __cleanup(ioat, phys_complete);
357
358 __ioat2_restart_chan(ioat);
359}
360
361static void ioat3_timer_event(unsigned long data)
362{
363 struct ioat2_dma_chan *ioat = (void *) data;
364 struct ioat_chan_common *chan = &ioat->base;
365
366 spin_lock_bh(&chan->cleanup_lock);
367 if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
368 unsigned long phys_complete;
369 u64 status;
370
371 spin_lock_bh(&ioat->ring_lock);
372 status = ioat_chansts(chan);
373
374 /* when halted due to errors check for channel
375 * programming errors before advancing the completion state
376 */
377 if (is_ioat_halted(status)) {
378 u32 chanerr;
379
380 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
381 BUG_ON(is_ioat_bug(chanerr));
382 }
383
384 /* if we haven't made progress and we have already
385 * acknowledged a pending completion once, then be more
386 * forceful with a restart
387 */
388 if (ioat_cleanup_preamble(chan, &phys_complete))
389 __cleanup(ioat, phys_complete);
390 else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
391 ioat3_restart_channel(ioat);
392 else {
393 set_bit(IOAT_COMPLETION_ACK, &chan->state);
394 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
395 }
396 spin_unlock_bh(&ioat->ring_lock);
397 } else {
398 u16 active;
399
400 /* if the ring is idle, empty, and oversized try to step
401 * down the size
402 */
403 spin_lock_bh(&ioat->ring_lock);
404 active = ioat2_ring_active(ioat);
405 if (active == 0 && ioat->alloc_order > ioat_get_alloc_order())
406 reshape_ring(ioat, ioat->alloc_order-1);
407 spin_unlock_bh(&ioat->ring_lock);
408
409 /* keep shrinking until we get back to our minimum
410 * default size
411 */
412 if (ioat->alloc_order > ioat_get_alloc_order())
413 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
414 }
415 spin_unlock_bh(&chan->cleanup_lock);
416}
417
418static enum dma_status
419ioat3_is_complete(struct dma_chan *c, dma_cookie_t cookie,
420 dma_cookie_t *done, dma_cookie_t *used)
421{
422 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
423
424 if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
425 return DMA_SUCCESS;
426
427 ioat3_cleanup(ioat);
428
429 return ioat_is_complete(c, cookie, done, used);
430}
431
432static struct dma_async_tx_descriptor *
433ioat3_prep_memset_lock(struct dma_chan *c, dma_addr_t dest, int value,
434 size_t len, unsigned long flags)
435{
436 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
437 struct ioat_ring_ent *desc;
438 size_t total_len = len;
439 struct ioat_fill_descriptor *fill;
440 int num_descs;
441 u64 src_data = (0x0101010101010101ULL) * (value & 0xff);
442 u16 idx;
443 int i;
444
445 num_descs = ioat2_xferlen_to_descs(ioat, len);
446 if (likely(num_descs) &&
447 ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0)
448 /* pass */;
449 else
450 return NULL;
451 for (i = 0; i < num_descs; i++) {
452 size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
453
454 desc = ioat2_get_ring_ent(ioat, idx + i);
455 fill = desc->fill;
456
457 fill->size = xfer_size;
458 fill->src_data = src_data;
459 fill->dst_addr = dest;
460 fill->ctl = 0;
461 fill->ctl_f.op = IOAT_OP_FILL;
462
463 len -= xfer_size;
464 dest += xfer_size;
465 dump_desc_dbg(ioat, desc);
466 }
467
468 desc->txd.flags = flags;
469 desc->len = total_len;
470 fill->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
471 fill->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
472 fill->ctl_f.compl_write = 1;
473 dump_desc_dbg(ioat, desc);
474
475 /* we leave the channel locked to ensure in order submission */
476 return &desc->txd;
477}
478
479static struct dma_async_tx_descriptor *
480__ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
481 dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt,
482 size_t len, unsigned long flags)
483{
484 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
485 struct ioat_ring_ent *compl_desc;
486 struct ioat_ring_ent *desc;
487 struct ioat_ring_ent *ext;
488 size_t total_len = len;
489 struct ioat_xor_descriptor *xor;
490 struct ioat_xor_ext_descriptor *xor_ex = NULL;
491 struct ioat_dma_descriptor *hw;
492 u32 offset = 0;
493 int num_descs;
494 int with_ext;
495 int i;
496 u16 idx;
497 u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR;
498
499 BUG_ON(src_cnt < 2);
500
501 num_descs = ioat2_xferlen_to_descs(ioat, len);
502 /* we need 2x the number of descriptors to cover greater than 5
503 * sources
504 */
505 if (src_cnt > 5) {
506 with_ext = 1;
507 num_descs *= 2;
508 } else
509 with_ext = 0;
510
511 /* completion writes from the raid engine may pass completion
512 * writes from the legacy engine, so we need one extra null
513 * (legacy) descriptor to ensure all completion writes arrive in
514 * order.
515 */
516 if (likely(num_descs) &&
517 ioat2_alloc_and_lock(&idx, ioat, num_descs+1) == 0)
518 /* pass */;
519 else
520 return NULL;
521 for (i = 0; i < num_descs; i += 1 + with_ext) {
522 struct ioat_raw_descriptor *descs[2];
523 size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
524 int s;
525
526 desc = ioat2_get_ring_ent(ioat, idx + i);
527 xor = desc->xor;
528
529 /* save a branch by unconditionally retrieving the
530 * extended descriptor xor_set_src() knows to not write
531 * to it in the single descriptor case
532 */
533 ext = ioat2_get_ring_ent(ioat, idx + i + 1);
534 xor_ex = ext->xor_ex;
535
536 descs[0] = (struct ioat_raw_descriptor *) xor;
537 descs[1] = (struct ioat_raw_descriptor *) xor_ex;
538 for (s = 0; s < src_cnt; s++)
539 xor_set_src(descs, src[s], offset, s);
540 xor->size = xfer_size;
541 xor->dst_addr = dest + offset;
542 xor->ctl = 0;
543 xor->ctl_f.op = op;
544 xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt);
545
546 len -= xfer_size;
547 offset += xfer_size;
548 dump_desc_dbg(ioat, desc);
549 }
550
551 /* last xor descriptor carries the unmap parameters and fence bit */
552 desc->txd.flags = flags;
553 desc->len = total_len;
554 if (result)
555 desc->result = result;
556 xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
557
558 /* completion descriptor carries interrupt bit */
559 compl_desc = ioat2_get_ring_ent(ioat, idx + i);
560 compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
561 hw = compl_desc->hw;
562 hw->ctl = 0;
563 hw->ctl_f.null = 1;
564 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
565 hw->ctl_f.compl_write = 1;
566 hw->size = NULL_DESC_BUFFER_SIZE;
567 dump_desc_dbg(ioat, compl_desc);
568
569 /* we leave the channel locked to ensure in order submission */
570 return &desc->txd;
571}
572
573static struct dma_async_tx_descriptor *
574ioat3_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
575 unsigned int src_cnt, size_t len, unsigned long flags)
576{
577 return __ioat3_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags);
578}
579
580struct dma_async_tx_descriptor *
581ioat3_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
582 unsigned int src_cnt, size_t len,
583 enum sum_check_flags *result, unsigned long flags)
584{
585 /* the cleanup routine only sets bits on validate failure, it
586 * does not clear bits on validate success... so clear it here
587 */
588 *result = 0;
589
590 return __ioat3_prep_xor_lock(chan, result, src[0], &src[1],
591 src_cnt - 1, len, flags);
592}
593
594static void
595dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext)
596{
597 struct device *dev = to_dev(&ioat->base);
598 struct ioat_pq_descriptor *pq = desc->pq;
599 struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL;
600 struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex };
601 int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
602 int i;
603
604 dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
605 " sz: %#x ctl: %#x (op: %d int: %d compl: %d pq: '%s%s' src_cnt: %d)\n",
606 desc_id(desc), (unsigned long long) desc->txd.phys,
607 (unsigned long long) (pq_ex ? pq_ex->next : pq->next),
608 desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op, pq->ctl_f.int_en,
609 pq->ctl_f.compl_write,
610 pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
611 pq->ctl_f.src_cnt);
612 for (i = 0; i < src_cnt; i++)
613 dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
614 (unsigned long long) pq_get_src(descs, i), pq->coef[i]);
615 dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
616 dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
617}
618
619static struct dma_async_tx_descriptor *
620__ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
621 const dma_addr_t *dst, const dma_addr_t *src,
622 unsigned int src_cnt, const unsigned char *scf,
623 size_t len, unsigned long flags)
624{
625 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
626 struct ioat_chan_common *chan = &ioat->base;
627 struct ioat_ring_ent *compl_desc;
628 struct ioat_ring_ent *desc;
629 struct ioat_ring_ent *ext;
630 size_t total_len = len;
631 struct ioat_pq_descriptor *pq;
632 struct ioat_pq_ext_descriptor *pq_ex = NULL;
633 struct ioat_dma_descriptor *hw;
634 u32 offset = 0;
635 int num_descs;
636 int with_ext;
637 int i, s;
638 u16 idx;
639 u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ;
640
641 dev_dbg(to_dev(chan), "%s\n", __func__);
642 /* the engine requires at least two sources (we provide
643 * at least 1 implied source in the DMA_PREP_CONTINUE case)
644 */
645 BUG_ON(src_cnt + dmaf_continue(flags) < 2);
646
647 num_descs = ioat2_xferlen_to_descs(ioat, len);
648 /* we need 2x the number of descriptors to cover greater than 3
649 * sources
650 */
651 if (src_cnt > 3 || flags & DMA_PREP_CONTINUE) {
652 with_ext = 1;
653 num_descs *= 2;
654 } else
655 with_ext = 0;
656
657 /* completion writes from the raid engine may pass completion
658 * writes from the legacy engine, so we need one extra null
659 * (legacy) descriptor to ensure all completion writes arrive in
660 * order.
661 */
662 if (likely(num_descs) &&
663 ioat2_alloc_and_lock(&idx, ioat, num_descs+1) == 0)
664 /* pass */;
665 else
666 return NULL;
667 for (i = 0; i < num_descs; i += 1 + with_ext) {
668 struct ioat_raw_descriptor *descs[2];
669 size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
670
671 desc = ioat2_get_ring_ent(ioat, idx + i);
672 pq = desc->pq;
673
674 /* save a branch by unconditionally retrieving the
675 * extended descriptor pq_set_src() knows to not write
676 * to it in the single descriptor case
677 */
678 ext = ioat2_get_ring_ent(ioat, idx + i + with_ext);
679 pq_ex = ext->pq_ex;
680
681 descs[0] = (struct ioat_raw_descriptor *) pq;
682 descs[1] = (struct ioat_raw_descriptor *) pq_ex;
683
684 for (s = 0; s < src_cnt; s++)
685 pq_set_src(descs, src[s], offset, scf[s], s);
686
687 /* see the comment for dma_maxpq in include/linux/dmaengine.h */
688 if (dmaf_p_disabled_continue(flags))
689 pq_set_src(descs, dst[1], offset, 1, s++);
690 else if (dmaf_continue(flags)) {
691 pq_set_src(descs, dst[0], offset, 0, s++);
692 pq_set_src(descs, dst[1], offset, 1, s++);
693 pq_set_src(descs, dst[1], offset, 0, s++);
694 }
695 pq->size = xfer_size;
696 pq->p_addr = dst[0] + offset;
697 pq->q_addr = dst[1] + offset;
698 pq->ctl = 0;
699 pq->ctl_f.op = op;
700 pq->ctl_f.src_cnt = src_cnt_to_hw(s);
701 pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
702 pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
703
704 len -= xfer_size;
705 offset += xfer_size;
706 }
707
708 /* last pq descriptor carries the unmap parameters and fence bit */
709 desc->txd.flags = flags;
710 desc->len = total_len;
711 if (result)
712 desc->result = result;
713 pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
714 dump_pq_desc_dbg(ioat, desc, ext);
715
716 /* completion descriptor carries interrupt bit */
717 compl_desc = ioat2_get_ring_ent(ioat, idx + i);
718 compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
719 hw = compl_desc->hw;
720 hw->ctl = 0;
721 hw->ctl_f.null = 1;
722 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
723 hw->ctl_f.compl_write = 1;
724 hw->size = NULL_DESC_BUFFER_SIZE;
725 dump_desc_dbg(ioat, compl_desc);
726
727 /* we leave the channel locked to ensure in order submission */
728 return &desc->txd;
729}
730
731static struct dma_async_tx_descriptor *
732ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
733 unsigned int src_cnt, const unsigned char *scf, size_t len,
734 unsigned long flags)
735{
736 /* handle the single source multiply case from the raid6
737 * recovery path
738 */
739 if (unlikely((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1)) {
740 dma_addr_t single_source[2];
741 unsigned char single_source_coef[2];
742
743 BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q);
744 single_source[0] = src[0];
745 single_source[1] = src[0];
746 single_source_coef[0] = scf[0];
747 single_source_coef[1] = 0;
748
749 return __ioat3_prep_pq_lock(chan, NULL, dst, single_source, 2,
750 single_source_coef, len, flags);
751 } else
752 return __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt, scf,
753 len, flags);
754}
755
756struct dma_async_tx_descriptor *
757ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
758 unsigned int src_cnt, const unsigned char *scf, size_t len,
759 enum sum_check_flags *pqres, unsigned long flags)
760{
761 /* the cleanup routine only sets bits on validate failure, it
762 * does not clear bits on validate success... so clear it here
763 */
764 *pqres = 0;
765
766 return __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,
767 flags);
768}
769
770static struct dma_async_tx_descriptor *
771ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
772 unsigned int src_cnt, size_t len, unsigned long flags)
773{
774 unsigned char scf[src_cnt];
775 dma_addr_t pq[2];
776
777 memset(scf, 0, src_cnt);
778 flags |= DMA_PREP_PQ_DISABLE_Q;
779 pq[0] = dst;
780 pq[1] = ~0;
781
782 return __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,
783 flags);
784}
785
786struct dma_async_tx_descriptor *
787ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
788 unsigned int src_cnt, size_t len,
789 enum sum_check_flags *result, unsigned long flags)
790{
791 unsigned char scf[src_cnt];
792 dma_addr_t pq[2];
793
794 /* the cleanup routine only sets bits on validate failure, it
795 * does not clear bits on validate success... so clear it here
796 */
797 *result = 0;
798
799 memset(scf, 0, src_cnt);
800 flags |= DMA_PREP_PQ_DISABLE_Q;
801 pq[0] = src[0];
802 pq[1] = ~0;
803
804 return __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, scf,
805 len, flags);
806}
807
808static struct dma_async_tx_descriptor *
809ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags)
810{
811 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
812 struct ioat_ring_ent *desc;
813 struct ioat_dma_descriptor *hw;
814 u16 idx;
815
816 if (ioat2_alloc_and_lock(&idx, ioat, 1) == 0)
817 desc = ioat2_get_ring_ent(ioat, idx);
818 else
819 return NULL;
820
821 hw = desc->hw;
822 hw->ctl = 0;
823 hw->ctl_f.null = 1;
824 hw->ctl_f.int_en = 1;
825 hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
826 hw->ctl_f.compl_write = 1;
827 hw->size = NULL_DESC_BUFFER_SIZE;
828 hw->src_addr = 0;
829 hw->dst_addr = 0;
830
831 desc->txd.flags = flags;
832 desc->len = 1;
833
834 dump_desc_dbg(ioat, desc);
835
836 /* we leave the channel locked to ensure in order submission */
837 return &desc->txd;
838}
839
840static void __devinit ioat3_dma_test_callback(void *dma_async_param)
841{
842 struct completion *cmp = dma_async_param;
843
844 complete(cmp);
845}
846
847#define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */
848static int __devinit ioat_xor_val_self_test(struct ioatdma_device *device)
849{
850 int i, src_idx;
851 struct page *dest;
852 struct page *xor_srcs[IOAT_NUM_SRC_TEST];
853 struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1];
854 dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1];
855 dma_addr_t dma_addr, dest_dma;
856 struct dma_async_tx_descriptor *tx;
857 struct dma_chan *dma_chan;
858 dma_cookie_t cookie;
859 u8 cmp_byte = 0;
860 u32 cmp_word;
861 u32 xor_val_result;
862 int err = 0;
863 struct completion cmp;
864 unsigned long tmo;
865 struct device *dev = &device->pdev->dev;
866 struct dma_device *dma = &device->common;
867
868 dev_dbg(dev, "%s\n", __func__);
869
870 if (!dma_has_cap(DMA_XOR, dma->cap_mask))
871 return 0;
872
873 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
874 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
875 if (!xor_srcs[src_idx]) {
876 while (src_idx--)
877 __free_page(xor_srcs[src_idx]);
878 return -ENOMEM;
879 }
880 }
881
882 dest = alloc_page(GFP_KERNEL);
883 if (!dest) {
884 while (src_idx--)
885 __free_page(xor_srcs[src_idx]);
886 return -ENOMEM;
887 }
888
889 /* Fill in src buffers */
890 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
891 u8 *ptr = page_address(xor_srcs[src_idx]);
892 for (i = 0; i < PAGE_SIZE; i++)
893 ptr[i] = (1 << src_idx);
894 }
895
896 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++)
897 cmp_byte ^= (u8) (1 << src_idx);
898
899 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
900 (cmp_byte << 8) | cmp_byte;
901
902 memset(page_address(dest), 0, PAGE_SIZE);
903
904 dma_chan = container_of(dma->channels.next, struct dma_chan,
905 device_node);
906 if (dma->device_alloc_chan_resources(dma_chan) < 1) {
907 err = -ENODEV;
908 goto out;
909 }
910
911 /* test xor */
912 dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE);
913 for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
914 dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE,
915 DMA_TO_DEVICE);
916 tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
917 IOAT_NUM_SRC_TEST, PAGE_SIZE,
918 DMA_PREP_INTERRUPT);
919
920 if (!tx) {
921 dev_err(dev, "Self-test xor prep failed\n");
922 err = -ENODEV;
923 goto free_resources;
924 }
925
926 async_tx_ack(tx);
927 init_completion(&cmp);
928 tx->callback = ioat3_dma_test_callback;
929 tx->callback_param = &cmp;
930 cookie = tx->tx_submit(tx);
931 if (cookie < 0) {
932 dev_err(dev, "Self-test xor setup failed\n");
933 err = -ENODEV;
934 goto free_resources;
935 }
936 dma->device_issue_pending(dma_chan);
937
938 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
939
940 if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
941 dev_err(dev, "Self-test xor timed out\n");
942 err = -ENODEV;
943 goto free_resources;
944 }
945
946 dma_sync_single_for_cpu(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
947 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
948 u32 *ptr = page_address(dest);
949 if (ptr[i] != cmp_word) {
950 dev_err(dev, "Self-test xor failed compare\n");
951 err = -ENODEV;
952 goto free_resources;
953 }
954 }
955 dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_TO_DEVICE);
956
957 /* skip validate if the capability is not present */
958 if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
959 goto free_resources;
960
961 /* validate the sources with the destintation page */
962 for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
963 xor_val_srcs[i] = xor_srcs[i];
964 xor_val_srcs[i] = dest;
965
966 xor_val_result = 1;
967
968 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
969 dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
970 DMA_TO_DEVICE);
971 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
972 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
973 &xor_val_result, DMA_PREP_INTERRUPT);
974 if (!tx) {
975 dev_err(dev, "Self-test zero prep failed\n");
976 err = -ENODEV;
977 goto free_resources;
978 }
979
980 async_tx_ack(tx);
981 init_completion(&cmp);
982 tx->callback = ioat3_dma_test_callback;
983 tx->callback_param = &cmp;
984 cookie = tx->tx_submit(tx);
985 if (cookie < 0) {
986 dev_err(dev, "Self-test zero setup failed\n");
987 err = -ENODEV;
988 goto free_resources;
989 }
990 dma->device_issue_pending(dma_chan);
991
992 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
993
994 if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
995 dev_err(dev, "Self-test validate timed out\n");
996 err = -ENODEV;
997 goto free_resources;
998 }
999
1000 if (xor_val_result != 0) {
1001 dev_err(dev, "Self-test validate failed compare\n");
1002 err = -ENODEV;
1003 goto free_resources;
1004 }
1005
1006 /* skip memset if the capability is not present */
1007 if (!dma_has_cap(DMA_MEMSET, dma_chan->device->cap_mask))
1008 goto free_resources;
1009
1010 /* test memset */
1011 dma_addr = dma_map_page(dev, dest, 0,
1012 PAGE_SIZE, DMA_FROM_DEVICE);
1013 tx = dma->device_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE,
1014 DMA_PREP_INTERRUPT);
1015 if (!tx) {
1016 dev_err(dev, "Self-test memset prep failed\n");
1017 err = -ENODEV;
1018 goto free_resources;
1019 }
1020
1021 async_tx_ack(tx);
1022 init_completion(&cmp);
1023 tx->callback = ioat3_dma_test_callback;
1024 tx->callback_param = &cmp;
1025 cookie = tx->tx_submit(tx);
1026 if (cookie < 0) {
1027 dev_err(dev, "Self-test memset setup failed\n");
1028 err = -ENODEV;
1029 goto free_resources;
1030 }
1031 dma->device_issue_pending(dma_chan);
1032
1033 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1034
1035 if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1036 dev_err(dev, "Self-test memset timed out\n");
1037 err = -ENODEV;
1038 goto free_resources;
1039 }
1040
1041 for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) {
1042 u32 *ptr = page_address(dest);
1043 if (ptr[i]) {
1044 dev_err(dev, "Self-test memset failed compare\n");
1045 err = -ENODEV;
1046 goto free_resources;
1047 }
1048 }
1049
1050 /* test for non-zero parity sum */
1051 xor_val_result = 0;
1052 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
1053 dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
1054 DMA_TO_DEVICE);
1055 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
1056 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
1057 &xor_val_result, DMA_PREP_INTERRUPT);
1058 if (!tx) {
1059 dev_err(dev, "Self-test 2nd zero prep failed\n");
1060 err = -ENODEV;
1061 goto free_resources;
1062 }
1063
1064 async_tx_ack(tx);
1065 init_completion(&cmp);
1066 tx->callback = ioat3_dma_test_callback;
1067 tx->callback_param = &cmp;
1068 cookie = tx->tx_submit(tx);
1069 if (cookie < 0) {
1070 dev_err(dev, "Self-test 2nd zero setup failed\n");
1071 err = -ENODEV;
1072 goto free_resources;
1073 }
1074 dma->device_issue_pending(dma_chan);
1075
1076 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1077
1078 if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1079 dev_err(dev, "Self-test 2nd validate timed out\n");
1080 err = -ENODEV;
1081 goto free_resources;
1082 }
1083
1084 if (xor_val_result != SUM_CHECK_P_RESULT) {
1085 dev_err(dev, "Self-test validate failed compare\n");
1086 err = -ENODEV;
1087 goto free_resources;
1088 }
1089
1090free_resources:
1091 dma->device_free_chan_resources(dma_chan);
1092out:
1093 src_idx = IOAT_NUM_SRC_TEST;
1094 while (src_idx--)
1095 __free_page(xor_srcs[src_idx]);
1096 __free_page(dest);
1097 return err;
1098}
1099
1100static int __devinit ioat3_dma_self_test(struct ioatdma_device *device)
1101{
1102 int rc = ioat_dma_self_test(device);
1103
1104 if (rc)
1105 return rc;
1106
1107 rc = ioat_xor_val_self_test(device);
1108 if (rc)
1109 return rc;
1110
1111 return 0;
1112}
1113
1114int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca)
1115{
1116 struct pci_dev *pdev = device->pdev;
1117 struct dma_device *dma;
1118 struct dma_chan *c;
1119 struct ioat_chan_common *chan;
1120 bool is_raid_device = false;
1121 int err;
1122 u16 dev_id;
1123 u32 cap;
1124
1125 device->enumerate_channels = ioat2_enumerate_channels;
1126 device->self_test = ioat3_dma_self_test;
1127 dma = &device->common;
1128 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
1129 dma->device_issue_pending = ioat2_issue_pending;
1130 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
1131 dma->device_free_chan_resources = ioat2_free_chan_resources;
1132
1133 dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
1134 dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock;
1135
1136 cap = readl(device->reg_base + IOAT_DMA_CAP_OFFSET);
1137 if (cap & IOAT_CAP_XOR) {
1138 is_raid_device = true;
1139 dma->max_xor = 8;
1140 dma->xor_align = 2;
1141
1142 dma_cap_set(DMA_XOR, dma->cap_mask);
1143 dma->device_prep_dma_xor = ioat3_prep_xor;
1144
1145 dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
1146 dma->device_prep_dma_xor_val = ioat3_prep_xor_val;
1147 }
1148 if (cap & IOAT_CAP_PQ) {
1149 is_raid_device = true;
1150 dma_set_maxpq(dma, 8, 0);
1151 dma->pq_align = 2;
1152
1153 dma_cap_set(DMA_PQ, dma->cap_mask);
1154 dma->device_prep_dma_pq = ioat3_prep_pq;
1155
1156 dma_cap_set(DMA_PQ_VAL, dma->cap_mask);
1157 dma->device_prep_dma_pq_val = ioat3_prep_pq_val;
1158
1159 if (!(cap & IOAT_CAP_XOR)) {
1160 dma->max_xor = 8;
1161 dma->xor_align = 2;
1162
1163 dma_cap_set(DMA_XOR, dma->cap_mask);
1164 dma->device_prep_dma_xor = ioat3_prep_pqxor;
1165
1166 dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
1167 dma->device_prep_dma_xor_val = ioat3_prep_pqxor_val;
1168 }
1169 }
1170 if (is_raid_device && (cap & IOAT_CAP_FILL_BLOCK)) {
1171 dma_cap_set(DMA_MEMSET, dma->cap_mask);
1172 dma->device_prep_dma_memset = ioat3_prep_memset_lock;
1173 }
1174
1175
1176 if (is_raid_device) {
1177 dma->device_is_tx_complete = ioat3_is_complete;
1178 device->cleanup_tasklet = ioat3_cleanup_tasklet;
1179 device->timer_fn = ioat3_timer_event;
1180 } else {
1181 dma->device_is_tx_complete = ioat2_is_complete;
1182 device->cleanup_tasklet = ioat2_cleanup_tasklet;
1183 device->timer_fn = ioat2_timer_event;
1184 }
1185
1186 /* -= IOAT ver.3 workarounds =- */
1187 /* Write CHANERRMSK_INT with 3E07h to mask out the errors
1188 * that can cause stability issues for IOAT ver.3
1189 */
1190 pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
1191
1192 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1193 * (workaround for spurious config parity error after restart)
1194 */
1195 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1196 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
1197 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
1198
1199 err = ioat_probe(device);
1200 if (err)
1201 return err;
1202 ioat_set_tcp_copy_break(262144);
1203
1204 list_for_each_entry(c, &dma->channels, device_node) {
1205 chan = to_chan_common(c);
1206 writel(IOAT_DMA_DCA_ANY_CPU,
1207 chan->reg_base + IOAT_DCACTRL_OFFSET);
1208 }
1209
1210 err = ioat_register(device);
1211 if (err)
1212 return err;
1213
1214 ioat_kobject_add(device, &ioat2_ktype);
1215
1216 if (dca)
1217 device->dca = ioat3_dca_init(pdev, device->reg_base);
1218
1219 return 0;
1220}
diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h
index 7481fb13ce00..99afb12bd409 100644
--- a/drivers/dma/ioat/hw.h
+++ b/drivers/dma/ioat/hw.h
@@ -37,6 +37,7 @@
37#define IOAT_VER_1_2 0x12 /* Version 1.2 */ 37#define IOAT_VER_1_2 0x12 /* Version 1.2 */
38#define IOAT_VER_2_0 0x20 /* Version 2.0 */ 38#define IOAT_VER_2_0 0x20 /* Version 2.0 */
39#define IOAT_VER_3_0 0x30 /* Version 3.0 */ 39#define IOAT_VER_3_0 0x30 /* Version 3.0 */
40#define IOAT_VER_3_2 0x32 /* Version 3.2 */
40 41
41struct ioat_dma_descriptor { 42struct ioat_dma_descriptor {
42 uint32_t size; 43 uint32_t size;
@@ -55,6 +56,7 @@ struct ioat_dma_descriptor {
55 unsigned int dest_dca:1; 56 unsigned int dest_dca:1;
56 unsigned int hint:1; 57 unsigned int hint:1;
57 unsigned int rsvd2:13; 58 unsigned int rsvd2:13;
59 #define IOAT_OP_COPY 0x00
58 unsigned int op:8; 60 unsigned int op:8;
59 } ctl_f; 61 } ctl_f;
60 }; 62 };
@@ -70,4 +72,144 @@ struct ioat_dma_descriptor {
70 }; 72 };
71 uint64_t user2; 73 uint64_t user2;
72}; 74};
75
76struct ioat_fill_descriptor {
77 uint32_t size;
78 union {
79 uint32_t ctl;
80 struct {
81 unsigned int int_en:1;
82 unsigned int rsvd:1;
83 unsigned int dest_snoop_dis:1;
84 unsigned int compl_write:1;
85 unsigned int fence:1;
86 unsigned int rsvd2:2;
87 unsigned int dest_brk:1;
88 unsigned int bundle:1;
89 unsigned int rsvd4:15;
90 #define IOAT_OP_FILL 0x01
91 unsigned int op:8;
92 } ctl_f;
93 };
94 uint64_t src_data;
95 uint64_t dst_addr;
96 uint64_t next;
97 uint64_t rsv1;
98 uint64_t next_dst_addr;
99 uint64_t user1;
100 uint64_t user2;
101};
102
103struct ioat_xor_descriptor {
104 uint32_t size;
105 union {
106 uint32_t ctl;
107 struct {
108 unsigned int int_en:1;
109 unsigned int src_snoop_dis:1;
110 unsigned int dest_snoop_dis:1;
111 unsigned int compl_write:1;
112 unsigned int fence:1;
113 unsigned int src_cnt:3;
114 unsigned int bundle:1;
115 unsigned int dest_dca:1;
116 unsigned int hint:1;
117 unsigned int rsvd:13;
118 #define IOAT_OP_XOR 0x87
119 #define IOAT_OP_XOR_VAL 0x88
120 unsigned int op:8;
121 } ctl_f;
122 };
123 uint64_t src_addr;
124 uint64_t dst_addr;
125 uint64_t next;
126 uint64_t src_addr2;
127 uint64_t src_addr3;
128 uint64_t src_addr4;
129 uint64_t src_addr5;
130};
131
132struct ioat_xor_ext_descriptor {
133 uint64_t src_addr6;
134 uint64_t src_addr7;
135 uint64_t src_addr8;
136 uint64_t next;
137 uint64_t rsvd[4];
138};
139
140struct ioat_pq_descriptor {
141 uint32_t size;
142 union {
143 uint32_t ctl;
144 struct {
145 unsigned int int_en:1;
146 unsigned int src_snoop_dis:1;
147 unsigned int dest_snoop_dis:1;
148 unsigned int compl_write:1;
149 unsigned int fence:1;
150 unsigned int src_cnt:3;
151 unsigned int bundle:1;
152 unsigned int dest_dca:1;
153 unsigned int hint:1;
154 unsigned int p_disable:1;
155 unsigned int q_disable:1;
156 unsigned int rsvd:11;
157 #define IOAT_OP_PQ 0x89
158 #define IOAT_OP_PQ_VAL 0x8a
159 unsigned int op:8;
160 } ctl_f;
161 };
162 uint64_t src_addr;
163 uint64_t p_addr;
164 uint64_t next;
165 uint64_t src_addr2;
166 uint64_t src_addr3;
167 uint8_t coef[8];
168 uint64_t q_addr;
169};
170
171struct ioat_pq_ext_descriptor {
172 uint64_t src_addr4;
173 uint64_t src_addr5;
174 uint64_t src_addr6;
175 uint64_t next;
176 uint64_t src_addr7;
177 uint64_t src_addr8;
178 uint64_t rsvd[2];
179};
180
181struct ioat_pq_update_descriptor {
182 uint32_t size;
183 union {
184 uint32_t ctl;
185 struct {
186 unsigned int int_en:1;
187 unsigned int src_snoop_dis:1;
188 unsigned int dest_snoop_dis:1;
189 unsigned int compl_write:1;
190 unsigned int fence:1;
191 unsigned int src_cnt:3;
192 unsigned int bundle:1;
193 unsigned int dest_dca:1;
194 unsigned int hint:1;
195 unsigned int p_disable:1;
196 unsigned int q_disable:1;
197 unsigned int rsvd:3;
198 unsigned int coef:8;
199 #define IOAT_OP_PQ_UP 0x8b
200 unsigned int op:8;
201 } ctl_f;
202 };
203 uint64_t src_addr;
204 uint64_t p_addr;
205 uint64_t next;
206 uint64_t src_addr2;
207 uint64_t p_src;
208 uint64_t q_src;
209 uint64_t q_addr;
210};
211
212struct ioat_raw_descriptor {
213 uint64_t field[8];
214};
73#endif 215#endif
diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c
index 61086c6bbf42..c788fa266470 100644
--- a/drivers/dma/ioat/pci.c
+++ b/drivers/dma/ioat/pci.c
@@ -36,30 +36,44 @@
36#include "hw.h" 36#include "hw.h"
37 37
38MODULE_VERSION(IOAT_DMA_VERSION); 38MODULE_VERSION(IOAT_DMA_VERSION);
39MODULE_LICENSE("GPL"); 39MODULE_LICENSE("Dual BSD/GPL");
40MODULE_AUTHOR("Intel Corporation"); 40MODULE_AUTHOR("Intel Corporation");
41 41
42static struct pci_device_id ioat_pci_tbl[] = { 42static struct pci_device_id ioat_pci_tbl[] = {
43 /* I/OAT v1 platforms */ 43 /* I/OAT v1 platforms */
44 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, 44 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
45 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB) }, 45 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB) },
46 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) }, 46 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) },
47 { PCI_DEVICE(PCI_VENDOR_ID_UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) }, 47 { PCI_VDEVICE(UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) },
48 48
49 /* I/OAT v2 platforms */ 49 /* I/OAT v2 platforms */
50 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) }, 50 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) },
51 51
52 /* I/OAT v3 platforms */ 52 /* I/OAT v3 platforms */
53 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) }, 53 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
54 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) }, 54 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
55 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) }, 55 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },
56 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) }, 56 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) },
57 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) }, 57 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) },
58 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) }, 58 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) },
59 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) }, 59 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) },
60 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) }, 60 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) },
61
62 /* I/OAT v3.2 platforms */
63 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) },
64 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) },
65 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) },
66 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) },
67 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) },
68 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) },
69 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) },
70 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) },
71 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) },
72 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) },
73
61 { 0, } 74 { 0, }
62}; 75};
76MODULE_DEVICE_TABLE(pci, ioat_pci_tbl);
63 77
64static int __devinit ioat_pci_probe(struct pci_dev *pdev, 78static int __devinit ioat_pci_probe(struct pci_dev *pdev,
65 const struct pci_device_id *id); 79 const struct pci_device_id *id);
@@ -172,6 +186,9 @@ static int __init ioat_init_module(void)
172{ 186{
173 int err; 187 int err;
174 188
189 pr_info("%s: Intel(R) QuickData Technology Driver %s\n",
190 DRV_NAME, IOAT_DMA_VERSION);
191
175 ioat2_cache = kmem_cache_create("ioat2", sizeof(struct ioat_ring_ent), 192 ioat2_cache = kmem_cache_create("ioat2", sizeof(struct ioat_ring_ent),
176 0, SLAB_HWCACHE_ALIGN, NULL); 193 0, SLAB_HWCACHE_ALIGN, NULL);
177 if (!ioat2_cache) 194 if (!ioat2_cache)
diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h
index e4334a195380..63038e18ab03 100644
--- a/drivers/dma/ioat/registers.h
+++ b/drivers/dma/ioat/registers.h
@@ -64,12 +64,27 @@
64 64
65#define IOAT_DEVICE_STATUS_OFFSET 0x0E /* 16-bit */ 65#define IOAT_DEVICE_STATUS_OFFSET 0x0E /* 16-bit */
66#define IOAT_DEVICE_STATUS_DEGRADED_MODE 0x0001 66#define IOAT_DEVICE_STATUS_DEGRADED_MODE 0x0001
67#define IOAT_DEVICE_MMIO_RESTRICTED 0x0002
68#define IOAT_DEVICE_MEMORY_BYPASS 0x0004
69#define IOAT_DEVICE_ADDRESS_REMAPPING 0x0008
70
71#define IOAT_DMA_CAP_OFFSET 0x10 /* 32-bit */
72#define IOAT_CAP_PAGE_BREAK 0x00000001
73#define IOAT_CAP_CRC 0x00000002
74#define IOAT_CAP_SKIP_MARKER 0x00000004
75#define IOAT_CAP_DCA 0x00000010
76#define IOAT_CAP_CRC_MOVE 0x00000020
77#define IOAT_CAP_FILL_BLOCK 0x00000040
78#define IOAT_CAP_APIC 0x00000080
79#define IOAT_CAP_XOR 0x00000100
80#define IOAT_CAP_PQ 0x00000200
67 81
68#define IOAT_CHANNEL_MMIO_SIZE 0x80 /* Each Channel MMIO space is this size */ 82#define IOAT_CHANNEL_MMIO_SIZE 0x80 /* Each Channel MMIO space is this size */
69 83
70/* DMA Channel Registers */ 84/* DMA Channel Registers */
71#define IOAT_CHANCTRL_OFFSET 0x00 /* 16-bit Channel Control Register */ 85#define IOAT_CHANCTRL_OFFSET 0x00 /* 16-bit Channel Control Register */
72#define IOAT_CHANCTRL_CHANNEL_PRIORITY_MASK 0xF000 86#define IOAT_CHANCTRL_CHANNEL_PRIORITY_MASK 0xF000
87#define IOAT3_CHANCTRL_COMPL_DCA_EN 0x0200
73#define IOAT_CHANCTRL_CHANNEL_IN_USE 0x0100 88#define IOAT_CHANCTRL_CHANNEL_IN_USE 0x0100
74#define IOAT_CHANCTRL_DESCRIPTOR_ADDR_SNOOP_CONTROL 0x0020 89#define IOAT_CHANCTRL_DESCRIPTOR_ADDR_SNOOP_CONTROL 0x0020
75#define IOAT_CHANCTRL_ERR_INT_EN 0x0010 90#define IOAT_CHANCTRL_ERR_INT_EN 0x0010
@@ -224,6 +239,11 @@
224#define IOAT_CHANERR_INT_CONFIGURATION_ERR 0x2000 239#define IOAT_CHANERR_INT_CONFIGURATION_ERR 0x2000
225#define IOAT_CHANERR_SOFT_ERR 0x4000 240#define IOAT_CHANERR_SOFT_ERR 0x4000
226#define IOAT_CHANERR_UNAFFILIATED_ERR 0x8000 241#define IOAT_CHANERR_UNAFFILIATED_ERR 0x8000
242#define IOAT_CHANERR_XOR_P_OR_CRC_ERR 0x10000
243#define IOAT_CHANERR_XOR_Q_ERR 0x20000
244#define IOAT_CHANERR_DESCRIPTOR_COUNT_ERR 0x40000
245
246#define IOAT_CHANERR_HANDLE_MASK (IOAT_CHANERR_XOR_P_OR_CRC_ERR | IOAT_CHANERR_XOR_Q_ERR)
227 247
228#define IOAT_CHANERR_MASK_OFFSET 0x2C /* 32-bit Channel Error Register */ 248#define IOAT_CHANERR_MASK_OFFSET 0x2C /* 32-bit Channel Error Register */
229 249
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
index 9f6c16f8e2be..645ca8d54ec4 100644
--- a/drivers/dma/iop-adma.c
+++ b/drivers/dma/iop-adma.c
@@ -31,6 +31,7 @@
31#include <linux/platform_device.h> 31#include <linux/platform_device.h>
32#include <linux/memory.h> 32#include <linux/memory.h>
33#include <linux/ioport.h> 33#include <linux/ioport.h>
34#include <linux/raid/pq.h>
34 35
35#include <mach/adma.h> 36#include <mach/adma.h>
36 37
@@ -57,65 +58,110 @@ static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
57 } 58 }
58} 59}
59 60
61static void
62iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
63{
64 struct dma_async_tx_descriptor *tx = &desc->async_tx;
65 struct iop_adma_desc_slot *unmap = desc->group_head;
66 struct device *dev = &iop_chan->device->pdev->dev;
67 u32 len = unmap->unmap_len;
68 enum dma_ctrl_flags flags = tx->flags;
69 u32 src_cnt;
70 dma_addr_t addr;
71 dma_addr_t dest;
72
73 src_cnt = unmap->unmap_src_cnt;
74 dest = iop_desc_get_dest_addr(unmap, iop_chan);
75 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
76 enum dma_data_direction dir;
77
78 if (src_cnt > 1) /* is xor? */
79 dir = DMA_BIDIRECTIONAL;
80 else
81 dir = DMA_FROM_DEVICE;
82
83 dma_unmap_page(dev, dest, len, dir);
84 }
85
86 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
87 while (src_cnt--) {
88 addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt);
89 if (addr == dest)
90 continue;
91 dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
92 }
93 }
94 desc->group_head = NULL;
95}
96
97static void
98iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
99{
100 struct dma_async_tx_descriptor *tx = &desc->async_tx;
101 struct iop_adma_desc_slot *unmap = desc->group_head;
102 struct device *dev = &iop_chan->device->pdev->dev;
103 u32 len = unmap->unmap_len;
104 enum dma_ctrl_flags flags = tx->flags;
105 u32 src_cnt = unmap->unmap_src_cnt;
106 dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan);
107 dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan);
108 int i;
109
110 if (tx->flags & DMA_PREP_CONTINUE)
111 src_cnt -= 3;
112
113 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) {
114 dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL);
115 dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL);
116 }
117
118 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
119 dma_addr_t addr;
120
121 for (i = 0; i < src_cnt; i++) {
122 addr = iop_desc_get_src_addr(unmap, iop_chan, i);
123 dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
124 }
125 if (desc->pq_check_result) {
126 dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE);
127 dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE);
128 }
129 }
130
131 desc->group_head = NULL;
132}
133
134
60static dma_cookie_t 135static dma_cookie_t
61iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, 136iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
62 struct iop_adma_chan *iop_chan, dma_cookie_t cookie) 137 struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
63{ 138{
64 BUG_ON(desc->async_tx.cookie < 0); 139 struct dma_async_tx_descriptor *tx = &desc->async_tx;
65 if (desc->async_tx.cookie > 0) { 140
66 cookie = desc->async_tx.cookie; 141 BUG_ON(tx->cookie < 0);
67 desc->async_tx.cookie = 0; 142 if (tx->cookie > 0) {
143 cookie = tx->cookie;
144 tx->cookie = 0;
68 145
69 /* call the callback (must not sleep or submit new 146 /* call the callback (must not sleep or submit new
70 * operations to this channel) 147 * operations to this channel)
71 */ 148 */
72 if (desc->async_tx.callback) 149 if (tx->callback)
73 desc->async_tx.callback( 150 tx->callback(tx->callback_param);
74 desc->async_tx.callback_param);
75 151
76 /* unmap dma addresses 152 /* unmap dma addresses
77 * (unmap_single vs unmap_page?) 153 * (unmap_single vs unmap_page?)
78 */ 154 */
79 if (desc->group_head && desc->unmap_len) { 155 if (desc->group_head && desc->unmap_len) {
80 struct iop_adma_desc_slot *unmap = desc->group_head; 156 if (iop_desc_is_pq(desc))
81 struct device *dev = 157 iop_desc_unmap_pq(iop_chan, desc);
82 &iop_chan->device->pdev->dev; 158 else
83 u32 len = unmap->unmap_len; 159 iop_desc_unmap(iop_chan, desc);
84 enum dma_ctrl_flags flags = desc->async_tx.flags;
85 u32 src_cnt;
86 dma_addr_t addr;
87 dma_addr_t dest;
88
89 src_cnt = unmap->unmap_src_cnt;
90 dest = iop_desc_get_dest_addr(unmap, iop_chan);
91 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
92 enum dma_data_direction dir;
93
94 if (src_cnt > 1) /* is xor? */
95 dir = DMA_BIDIRECTIONAL;
96 else
97 dir = DMA_FROM_DEVICE;
98
99 dma_unmap_page(dev, dest, len, dir);
100 }
101
102 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
103 while (src_cnt--) {
104 addr = iop_desc_get_src_addr(unmap,
105 iop_chan,
106 src_cnt);
107 if (addr == dest)
108 continue;
109 dma_unmap_page(dev, addr, len,
110 DMA_TO_DEVICE);
111 }
112 }
113 desc->group_head = NULL;
114 } 160 }
115 } 161 }
116 162
117 /* run dependent operations */ 163 /* run dependent operations */
118 dma_run_dependencies(&desc->async_tx); 164 dma_run_dependencies(tx);
119 165
120 return cookie; 166 return cookie;
121} 167}
@@ -287,7 +333,12 @@ static void iop_adma_tasklet(unsigned long data)
287{ 333{
288 struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data; 334 struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
289 335
290 spin_lock(&iop_chan->lock); 336 /* lockdep will flag depedency submissions as potentially
337 * recursive locking, this is not the case as a dependency
338 * submission will never recurse a channels submit routine.
339 * There are checks in async_tx.c to prevent this.
340 */
341 spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
291 __iop_adma_slot_cleanup(iop_chan); 342 __iop_adma_slot_cleanup(iop_chan);
292 spin_unlock(&iop_chan->lock); 343 spin_unlock(&iop_chan->lock);
293} 344}
@@ -661,9 +712,9 @@ iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
661} 712}
662 713
663static struct dma_async_tx_descriptor * 714static struct dma_async_tx_descriptor *
664iop_adma_prep_dma_zero_sum(struct dma_chan *chan, dma_addr_t *dma_src, 715iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
665 unsigned int src_cnt, size_t len, u32 *result, 716 unsigned int src_cnt, size_t len, u32 *result,
666 unsigned long flags) 717 unsigned long flags)
667{ 718{
668 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 719 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
669 struct iop_adma_desc_slot *sw_desc, *grp_start; 720 struct iop_adma_desc_slot *sw_desc, *grp_start;
@@ -697,6 +748,118 @@ iop_adma_prep_dma_zero_sum(struct dma_chan *chan, dma_addr_t *dma_src,
697 return sw_desc ? &sw_desc->async_tx : NULL; 748 return sw_desc ? &sw_desc->async_tx : NULL;
698} 749}
699 750
751static struct dma_async_tx_descriptor *
752iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
753 unsigned int src_cnt, const unsigned char *scf, size_t len,
754 unsigned long flags)
755{
756 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
757 struct iop_adma_desc_slot *sw_desc, *g;
758 int slot_cnt, slots_per_op;
759 int continue_srcs;
760
761 if (unlikely(!len))
762 return NULL;
763 BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
764
765 dev_dbg(iop_chan->device->common.dev,
766 "%s src_cnt: %d len: %u flags: %lx\n",
767 __func__, src_cnt, len, flags);
768
769 if (dmaf_p_disabled_continue(flags))
770 continue_srcs = 1+src_cnt;
771 else if (dmaf_continue(flags))
772 continue_srcs = 3+src_cnt;
773 else
774 continue_srcs = 0+src_cnt;
775
776 spin_lock_bh(&iop_chan->lock);
777 slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
778 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
779 if (sw_desc) {
780 int i;
781
782 g = sw_desc->group_head;
783 iop_desc_set_byte_count(g, iop_chan, len);
784
785 /* even if P is disabled its destination address (bits
786 * [3:0]) must match Q. It is ok if P points to an
787 * invalid address, it won't be written.
788 */
789 if (flags & DMA_PREP_PQ_DISABLE_P)
790 dst[0] = dst[1] & 0x7;
791
792 iop_desc_set_pq_addr(g, dst);
793 sw_desc->unmap_src_cnt = src_cnt;
794 sw_desc->unmap_len = len;
795 sw_desc->async_tx.flags = flags;
796 for (i = 0; i < src_cnt; i++)
797 iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
798
799 /* if we are continuing a previous operation factor in
800 * the old p and q values, see the comment for dma_maxpq
801 * in include/linux/dmaengine.h
802 */
803 if (dmaf_p_disabled_continue(flags))
804 iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
805 else if (dmaf_continue(flags)) {
806 iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
807 iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
808 iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
809 }
810 iop_desc_init_pq(g, i, flags);
811 }
812 spin_unlock_bh(&iop_chan->lock);
813
814 return sw_desc ? &sw_desc->async_tx : NULL;
815}
816
817static struct dma_async_tx_descriptor *
818iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
819 unsigned int src_cnt, const unsigned char *scf,
820 size_t len, enum sum_check_flags *pqres,
821 unsigned long flags)
822{
823 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
824 struct iop_adma_desc_slot *sw_desc, *g;
825 int slot_cnt, slots_per_op;
826
827 if (unlikely(!len))
828 return NULL;
829 BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
830
831 dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
832 __func__, src_cnt, len);
833
834 spin_lock_bh(&iop_chan->lock);
835 slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
836 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
837 if (sw_desc) {
838 /* for validate operations p and q are tagged onto the
839 * end of the source list
840 */
841 int pq_idx = src_cnt;
842
843 g = sw_desc->group_head;
844 iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
845 iop_desc_set_pq_zero_sum_byte_count(g, len);
846 g->pq_check_result = pqres;
847 pr_debug("\t%s: g->pq_check_result: %p\n",
848 __func__, g->pq_check_result);
849 sw_desc->unmap_src_cnt = src_cnt+2;
850 sw_desc->unmap_len = len;
851 sw_desc->async_tx.flags = flags;
852 while (src_cnt--)
853 iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
854 src[src_cnt],
855 scf[src_cnt]);
856 iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
857 }
858 spin_unlock_bh(&iop_chan->lock);
859
860 return sw_desc ? &sw_desc->async_tx : NULL;
861}
862
700static void iop_adma_free_chan_resources(struct dma_chan *chan) 863static void iop_adma_free_chan_resources(struct dma_chan *chan)
701{ 864{
702 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); 865 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
@@ -907,7 +1070,7 @@ out:
907 1070
908#define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */ 1071#define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
909static int __devinit 1072static int __devinit
910iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device) 1073iop_adma_xor_val_self_test(struct iop_adma_device *device)
911{ 1074{
912 int i, src_idx; 1075 int i, src_idx;
913 struct page *dest; 1076 struct page *dest;
@@ -1003,7 +1166,7 @@ iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
1003 PAGE_SIZE, DMA_TO_DEVICE); 1166 PAGE_SIZE, DMA_TO_DEVICE);
1004 1167
1005 /* skip zero sum if the capability is not present */ 1168 /* skip zero sum if the capability is not present */
1006 if (!dma_has_cap(DMA_ZERO_SUM, dma_chan->device->cap_mask)) 1169 if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
1007 goto free_resources; 1170 goto free_resources;
1008 1171
1009 /* zero sum the sources with the destintation page */ 1172 /* zero sum the sources with the destintation page */
@@ -1017,10 +1180,10 @@ iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
1017 dma_srcs[i] = dma_map_page(dma_chan->device->dev, 1180 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1018 zero_sum_srcs[i], 0, PAGE_SIZE, 1181 zero_sum_srcs[i], 0, PAGE_SIZE,
1019 DMA_TO_DEVICE); 1182 DMA_TO_DEVICE);
1020 tx = iop_adma_prep_dma_zero_sum(dma_chan, dma_srcs, 1183 tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1021 IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, 1184 IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1022 &zero_sum_result, 1185 &zero_sum_result,
1023 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1186 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1024 1187
1025 cookie = iop_adma_tx_submit(tx); 1188 cookie = iop_adma_tx_submit(tx);
1026 iop_adma_issue_pending(dma_chan); 1189 iop_adma_issue_pending(dma_chan);
@@ -1073,10 +1236,10 @@ iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
1073 dma_srcs[i] = dma_map_page(dma_chan->device->dev, 1236 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1074 zero_sum_srcs[i], 0, PAGE_SIZE, 1237 zero_sum_srcs[i], 0, PAGE_SIZE,
1075 DMA_TO_DEVICE); 1238 DMA_TO_DEVICE);
1076 tx = iop_adma_prep_dma_zero_sum(dma_chan, dma_srcs, 1239 tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1077 IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, 1240 IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1078 &zero_sum_result, 1241 &zero_sum_result,
1079 DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 1242 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1080 1243
1081 cookie = iop_adma_tx_submit(tx); 1244 cookie = iop_adma_tx_submit(tx);
1082 iop_adma_issue_pending(dma_chan); 1245 iop_adma_issue_pending(dma_chan);
@@ -1106,6 +1269,170 @@ out:
1106 return err; 1269 return err;
1107} 1270}
1108 1271
1272#ifdef CONFIG_MD_RAID6_PQ
1273static int __devinit
1274iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
1275{
1276 /* combined sources, software pq results, and extra hw pq results */
1277 struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
1278 /* ptr to the extra hw pq buffers defined above */
1279 struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
1280 /* address conversion buffers (dma_map / page_address) */
1281 void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
1282 dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST];
1283 dma_addr_t pq_dest[2];
1284
1285 int i;
1286 struct dma_async_tx_descriptor *tx;
1287 struct dma_chan *dma_chan;
1288 dma_cookie_t cookie;
1289 u32 zero_sum_result;
1290 int err = 0;
1291 struct device *dev;
1292
1293 dev_dbg(device->common.dev, "%s\n", __func__);
1294
1295 for (i = 0; i < ARRAY_SIZE(pq); i++) {
1296 pq[i] = alloc_page(GFP_KERNEL);
1297 if (!pq[i]) {
1298 while (i--)
1299 __free_page(pq[i]);
1300 return -ENOMEM;
1301 }
1302 }
1303
1304 /* Fill in src buffers */
1305 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
1306 pq_sw[i] = page_address(pq[i]);
1307 memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
1308 }
1309 pq_sw[i] = page_address(pq[i]);
1310 pq_sw[i+1] = page_address(pq[i+1]);
1311
1312 dma_chan = container_of(device->common.channels.next,
1313 struct dma_chan,
1314 device_node);
1315 if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1316 err = -ENODEV;
1317 goto out;
1318 }
1319
1320 dev = dma_chan->device->dev;
1321
1322 /* initialize the dests */
1323 memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
1324 memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);
1325
1326 /* test pq */
1327 pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1328 pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1329 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1330 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1331 DMA_TO_DEVICE);
1332
1333 tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
1334 IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
1335 PAGE_SIZE,
1336 DMA_PREP_INTERRUPT |
1337 DMA_CTRL_ACK);
1338
1339 cookie = iop_adma_tx_submit(tx);
1340 iop_adma_issue_pending(dma_chan);
1341 msleep(8);
1342
1343 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
1344 DMA_SUCCESS) {
1345 dev_err(dev, "Self-test pq timed out, disabling\n");
1346 err = -ENODEV;
1347 goto free_resources;
1348 }
1349
1350 raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);
1351
1352 if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
1353 page_address(pq_hw[0]), PAGE_SIZE) != 0) {
1354 dev_err(dev, "Self-test p failed compare, disabling\n");
1355 err = -ENODEV;
1356 goto free_resources;
1357 }
1358 if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
1359 page_address(pq_hw[1]), PAGE_SIZE) != 0) {
1360 dev_err(dev, "Self-test q failed compare, disabling\n");
1361 err = -ENODEV;
1362 goto free_resources;
1363 }
1364
1365 /* test correct zero sum using the software generated pq values */
1366 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1367 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1368 DMA_TO_DEVICE);
1369
1370 zero_sum_result = ~0;
1371 tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1372 pq_src, IOP_ADMA_NUM_SRC_TEST,
1373 raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1374 DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1375
1376 cookie = iop_adma_tx_submit(tx);
1377 iop_adma_issue_pending(dma_chan);
1378 msleep(8);
1379
1380 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
1381 DMA_SUCCESS) {
1382 dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
1383 err = -ENODEV;
1384 goto free_resources;
1385 }
1386
1387 if (zero_sum_result != 0) {
1388 dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
1389 zero_sum_result);
1390 err = -ENODEV;
1391 goto free_resources;
1392 }
1393
1394 /* test incorrect zero sum */
1395 i = IOP_ADMA_NUM_SRC_TEST;
1396 memset(pq_sw[i] + 100, 0, 100);
1397 memset(pq_sw[i+1] + 200, 0, 200);
1398 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1399 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1400 DMA_TO_DEVICE);
1401
1402 zero_sum_result = 0;
1403 tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1404 pq_src, IOP_ADMA_NUM_SRC_TEST,
1405 raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1406 DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1407
1408 cookie = iop_adma_tx_submit(tx);
1409 iop_adma_issue_pending(dma_chan);
1410 msleep(8);
1411
1412 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
1413 DMA_SUCCESS) {
1414 dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
1415 err = -ENODEV;
1416 goto free_resources;
1417 }
1418
1419 if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) {
1420 dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
1421 zero_sum_result);
1422 err = -ENODEV;
1423 goto free_resources;
1424 }
1425
1426free_resources:
1427 iop_adma_free_chan_resources(dma_chan);
1428out:
1429 i = ARRAY_SIZE(pq);
1430 while (i--)
1431 __free_page(pq[i]);
1432 return err;
1433}
1434#endif
1435
1109static int __devexit iop_adma_remove(struct platform_device *dev) 1436static int __devexit iop_adma_remove(struct platform_device *dev)
1110{ 1437{
1111 struct iop_adma_device *device = platform_get_drvdata(dev); 1438 struct iop_adma_device *device = platform_get_drvdata(dev);
@@ -1193,9 +1520,16 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
1193 dma_dev->max_xor = iop_adma_get_max_xor(); 1520 dma_dev->max_xor = iop_adma_get_max_xor();
1194 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor; 1521 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
1195 } 1522 }
1196 if (dma_has_cap(DMA_ZERO_SUM, dma_dev->cap_mask)) 1523 if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
1197 dma_dev->device_prep_dma_zero_sum = 1524 dma_dev->device_prep_dma_xor_val =
1198 iop_adma_prep_dma_zero_sum; 1525 iop_adma_prep_dma_xor_val;
1526 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1527 dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
1528 dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
1529 }
1530 if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
1531 dma_dev->device_prep_dma_pq_val =
1532 iop_adma_prep_dma_pq_val;
1199 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) 1533 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1200 dma_dev->device_prep_dma_interrupt = 1534 dma_dev->device_prep_dma_interrupt =
1201 iop_adma_prep_dma_interrupt; 1535 iop_adma_prep_dma_interrupt;
@@ -1249,23 +1583,35 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
1249 } 1583 }
1250 1584
1251 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask) || 1585 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask) ||
1252 dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) { 1586 dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
1253 ret = iop_adma_xor_zero_sum_self_test(adev); 1587 ret = iop_adma_xor_val_self_test(adev);
1254 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); 1588 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1255 if (ret) 1589 if (ret)
1256 goto err_free_iop_chan; 1590 goto err_free_iop_chan;
1257 } 1591 }
1258 1592
1593 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
1594 dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
1595 #ifdef CONFIG_MD_RAID6_PQ
1596 ret = iop_adma_pq_zero_sum_self_test(adev);
1597 dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
1598 #else
1599 /* can not test raid6, so do not publish capability */
1600 dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
1601 dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
1602 ret = 0;
1603 #endif
1604 if (ret)
1605 goto err_free_iop_chan;
1606 }
1607
1259 dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: " 1608 dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: "
1260 "( %s%s%s%s%s%s%s%s%s%s)\n", 1609 "( %s%s%s%s%s%s%s)\n",
1261 dma_has_cap(DMA_PQ_XOR, dma_dev->cap_mask) ? "pq_xor " : "", 1610 dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",
1262 dma_has_cap(DMA_PQ_UPDATE, dma_dev->cap_mask) ? "pq_update " : "", 1611 dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "",
1263 dma_has_cap(DMA_PQ_ZERO_SUM, dma_dev->cap_mask) ? "pq_zero_sum " : "",
1264 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1612 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1265 dma_has_cap(DMA_DUAL_XOR, dma_dev->cap_mask) ? "dual_xor " : "", 1613 dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "",
1266 dma_has_cap(DMA_ZERO_SUM, dma_dev->cap_mask) ? "xor_zero_sum " : "",
1267 dma_has_cap(DMA_MEMSET, dma_dev->cap_mask) ? "fill " : "", 1614 dma_has_cap(DMA_MEMSET, dma_dev->cap_mask) ? "fill " : "",
1268 dma_has_cap(DMA_MEMCPY_CRC32C, dma_dev->cap_mask) ? "cpy+crc " : "",
1269 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", 1615 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1270 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); 1616 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1271 1617
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-28 03:05:25 -0500