Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx into for-linus

32 files changed, 6071 insertions, 2403 deletions

diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 81e1020fb514..5903a88351bf 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -17,11 +17,15 @@ if DMADEVICES
 
 comment "DMA Devices"
 
+config ASYNC_TX_DISABLE_CHANNEL_SWITCH
+        bool
+
 config INTEL_IOATDMA
         tristate "Intel I/OAT DMA support"
         depends on PCI && X86
         select DMA_ENGINE
         select DCA
+        select ASYNC_TX_DISABLE_CHANNEL_SWITCH
         help
           Enable support for the Intel(R) I/OAT DMA engine present
           in recent Intel Xeon chipsets.
@@ -97,6 +101,14 @@ config TXX9_DMAC
           Support the TXx9 SoC internal DMA controller.  This can be
           integrated in chips such as the Toshiba TX4927/38/39.
 
+config SH_DMAE
+        tristate "Renesas SuperH DMAC support"
+        depends on SUPERH && SH_DMA
+        depends on !SH_DMA_API
+        select DMA_ENGINE
+        help
+          Enable support for the Renesas SuperH DMA controllers.
+
 config DMA_ENGINE
         bool
 
@@ -116,7 +128,7 @@ config NET_DMA
 
 config ASYNC_TX_DMA
         bool "Async_tx: Offload support for the async_tx api"
-        depends on DMA_ENGINE && !HIGHMEM64G
+        depends on DMA_ENGINE
         help
           This allows the async_tx api to take advantage of offload engines for
           memcpy, memset, xor, and raid6 p+q operations.  If your platform has
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index 40e1e0083571..eca71ba78ae9 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -1,8 +1,7 @@
 obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
 obj-$(CONFIG_NET_DMA) += iovlock.o
 obj-$(CONFIG_DMATEST) += dmatest.o
-obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
-ioatdma-objs := ioat.o ioat_dma.o ioat_dca.o
+obj-$(CONFIG_INTEL_IOATDMA) += ioat/
 obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
 obj-$(CONFIG_FSL_DMA) += fsldma.o
 obj-$(CONFIG_MV_XOR) += mv_xor.o
@@ -10,3 +9,4 @@ obj-$(CONFIG_DW_DMAC) += dw_dmac.o
 obj-$(CONFIG_AT_HDMAC) += at_hdmac.o
 obj-$(CONFIG_MX3_IPU) += ipu/
 obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
+obj-$(CONFIG_SH_DMAE) += shdma.o
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index c8522e6f1ad2..7585c4164bd5 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -87,6 +87,7 @@ static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan,
         desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);
         if (desc) {
                 memset(desc, 0, sizeof(struct at_desc));
+                INIT_LIST_HEAD(&desc->tx_list);
                 dma_async_tx_descriptor_init(&desc->txd, chan);
                 /* txd.flags will be overwritten in prep functions */
                 desc->txd.flags = DMA_CTRL_ACK;
@@ -150,11 +151,11 @@ static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc)
                 struct at_desc *child;
 
                 spin_lock_bh(&atchan->lock);
-                list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+                list_for_each_entry(child, &desc->tx_list, desc_node)
                         dev_vdbg(chan2dev(&atchan->chan_common),
                                         "moving child desc %p to freelist\n",
                                         child);
-                list_splice_init(&desc->txd.tx_list, &atchan->free_list);
+                list_splice_init(&desc->tx_list, &atchan->free_list);
                 dev_vdbg(chan2dev(&atchan->chan_common),
                          "moving desc %p to freelist\n", desc);
                 list_add(&desc->desc_node, &atchan->free_list);
@@ -247,30 +248,33 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
         param = txd->callback_param;
 
         /* move children to free_list */
-        list_splice_init(&txd->tx_list, &atchan->free_list);
+        list_splice_init(&desc->tx_list, &atchan->free_list);
         /* move myself to free_list */
         list_move(&desc->desc_node, &atchan->free_list);
 
         /* unmap dma addresses */
-        if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
-                if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
-                        dma_unmap_single(chan2parent(&atchan->chan_common),
-                                        desc->lli.daddr,
-                                        desc->len, DMA_FROM_DEVICE);
-                else
-                        dma_unmap_page(chan2parent(&atchan->chan_common),
-                                        desc->lli.daddr,
-                                        desc->len, DMA_FROM_DEVICE);
-        }
-        if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
-                if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
-                        dma_unmap_single(chan2parent(&atchan->chan_common),
-                                        desc->lli.saddr,
-                                        desc->len, DMA_TO_DEVICE);
-                else
-                        dma_unmap_page(chan2parent(&atchan->chan_common),
-                                        desc->lli.saddr,
-                                        desc->len, DMA_TO_DEVICE);
+        if (!atchan->chan_common.private) {
+                struct device *parent = chan2parent(&atchan->chan_common);
+                if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+                        if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
+                                dma_unmap_single(parent,
+                                                desc->lli.daddr,
+                                                desc->len, DMA_FROM_DEVICE);
+                        else
+                                dma_unmap_page(parent,
+                                                desc->lli.daddr,
+                                                desc->len, DMA_FROM_DEVICE);
+                }
+                if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                        if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
+                                dma_unmap_single(parent,
+                                                desc->lli.saddr,
+                                                desc->len, DMA_TO_DEVICE);
+                        else
+                                dma_unmap_page(parent,
+                                                desc->lli.saddr,
+                                                desc->len, DMA_TO_DEVICE);
+                }
         }
 
         /*
@@ -334,7 +338,7 @@ static void atc_cleanup_descriptors(struct at_dma_chan *atchan)
                         /* This one is currently in progress */
                         return;
 
-                list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+                list_for_each_entry(child, &desc->tx_list, desc_node)
                         if (!(child->lli.ctrla & ATC_DONE))
                                 /* Currently in progress */
                                 return;
@@ -407,7 +411,7 @@ static void atc_handle_error(struct at_dma_chan *atchan)
         dev_crit(chan2dev(&atchan->chan_common),
                         "  cookie: %d\n", bad_desc->txd.cookie);
         atc_dump_lli(atchan, &bad_desc->lli);
-        list_for_each_entry(child, &bad_desc->txd.tx_list, desc_node)
+        list_for_each_entry(child, &bad_desc->tx_list, desc_node)
                 atc_dump_lli(atchan, &child->lli);
 
         /* Pretend the descriptor completed successfully */
@@ -587,7 +591,7 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
                         prev->lli.dscr = desc->txd.phys;
                         /* insert the link descriptor to the LD ring */
                         list_add_tail(&desc->desc_node,
-                                        &first->txd.tx_list);
+                                        &first->tx_list);
                 }
                 prev = desc;
         }
@@ -646,8 +650,6 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 
         reg_width = atslave->reg_width;
 
-        sg_len = dma_map_sg(chan2parent(chan), sgl, sg_len, direction);
-
         ctrla = ATC_DEFAULT_CTRLA | atslave->ctrla;
         ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN;
 
@@ -687,7 +689,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                 prev->lli.dscr = desc->txd.phys;
                                 /* insert the link descriptor to the LD ring */
                                 list_add_tail(&desc->desc_node,
-                                                &first->txd.tx_list);
+                                                &first->tx_list);
                         }
                         prev = desc;
                         total_len += len;
@@ -729,7 +731,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                 prev->lli.dscr = desc->txd.phys;
                                 /* insert the link descriptor to the LD ring */
                                 list_add_tail(&desc->desc_node,
-                                                &first->txd.tx_list);
+                                                &first->tx_list);
                         }
                         prev = desc;
                         total_len += len;
diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h
index 4c972afc49ec..495457e3dc4b 100644
--- a/drivers/dma/at_hdmac_regs.h
+++ b/drivers/dma/at_hdmac_regs.h
@@ -165,6 +165,7 @@ struct at_desc {
         struct at_lli                   lli;
 
         /* THEN values for driver housekeeping */
+        struct list_head                tx_list;
         struct dma_async_tx_descriptor  txd;
         struct list_head                desc_node;
         size_t                          len;
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 5a87384ea4ff..bd0b248de2cf 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -608,6 +608,40 @@ void dmaengine_put(void)
 }
 EXPORT_SYMBOL(dmaengine_put);
 
+static bool device_has_all_tx_types(struct dma_device *device)
+{
+        /* A device that satisfies this test has channels that will never cause
+         * an async_tx channel switch event as all possible operation types can
+         * be handled.
+         */
+        #ifdef CONFIG_ASYNC_TX_DMA
+        if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))
+                return false;
+        #endif
+
+        #if defined(CONFIG_ASYNC_MEMCPY) || defined(CONFIG_ASYNC_MEMCPY_MODULE)
+        if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))
+                return false;
+        #endif
+
+        #if defined(CONFIG_ASYNC_MEMSET) || defined(CONFIG_ASYNC_MEMSET_MODULE)
+        if (!dma_has_cap(DMA_MEMSET, device->cap_mask))
+                return false;
+        #endif
+
+        #if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE)
+        if (!dma_has_cap(DMA_XOR, device->cap_mask))
+                return false;
+        #endif
+
+        #if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE)
+        if (!dma_has_cap(DMA_PQ, device->cap_mask))
+                return false;
+        #endif
+
+        return true;
+}
+
 static int get_dma_id(struct dma_device *device)
 {
         int rc;
@@ -644,8 +678,12 @@ int dma_async_device_register(struct dma_device *device)
                 !device->device_prep_dma_memcpy);
         BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) &&
                 !device->device_prep_dma_xor);
-        BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
-                !device->device_prep_dma_zero_sum);
+        BUG_ON(dma_has_cap(DMA_XOR_VAL, device->cap_mask) &&
+                !device->device_prep_dma_xor_val);
+        BUG_ON(dma_has_cap(DMA_PQ, device->cap_mask) &&
+                !device->device_prep_dma_pq);
+        BUG_ON(dma_has_cap(DMA_PQ_VAL, device->cap_mask) &&
+                !device->device_prep_dma_pq_val);
         BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) &&
                 !device->device_prep_dma_memset);
         BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) &&
@@ -661,6 +699,12 @@ int dma_async_device_register(struct dma_device *device)
         BUG_ON(!device->device_issue_pending);
         BUG_ON(!device->dev);
 
+        /* note: this only matters in the
+         * CONFIG_ASYNC_TX_DISABLE_CHANNEL_SWITCH=y case
+         */
+        if (device_has_all_tx_types(device))
+                dma_cap_set(DMA_ASYNC_TX, device->cap_mask);
+
         idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL);
         if (!idr_ref)
                 return -ENOMEM;
@@ -933,55 +977,29 @@ void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
 {
         tx->chan = chan;
         spin_lock_init(&tx->lock);
-        INIT_LIST_HEAD(&tx->tx_list);
 }
 EXPORT_SYMBOL(dma_async_tx_descriptor_init);
 
 /* dma_wait_for_async_tx - spin wait for a transaction to complete
  * @tx: in-flight transaction to wait on
- *
- * This routine assumes that tx was obtained from a call to async_memcpy,
- * async_xor, async_memset, etc which ensures that tx is "in-flight" (prepped
- * and submitted).  Walking the parent chain is only meant to cover for DMA
- * drivers that do not implement the DMA_INTERRUPT capability and may race with
- * the driver's descriptor cleanup routine.
  */
 enum dma_status
 dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
 {
-        enum dma_status status;
-        struct dma_async_tx_descriptor *iter;
-        struct dma_async_tx_descriptor *parent;
+        unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
 
         if (!tx)
                 return DMA_SUCCESS;
 
-        WARN_ONCE(tx->parent, "%s: speculatively walking dependency chain for"
-                  " %s\n", __func__, dma_chan_name(tx->chan));
-
-        /* poll through the dependency chain, return when tx is complete */
-        do {
-                iter = tx;
-
-                /* find the root of the unsubmitted dependency chain */
-                do {
-                        parent = iter->parent;
-                        if (!parent)
-                                break;
-                        else
-                                iter = parent;
-                } while (parent);
-
-                /* there is a small window for ->parent == NULL and
-                 * ->cookie == -EBUSY
-                 */
-                while (iter->cookie == -EBUSY)
-                        cpu_relax();
-
-                status = dma_sync_wait(iter->chan, iter->cookie);
-        } while (status == DMA_IN_PROGRESS || (iter != tx));
-
-        return status;
+        while (tx->cookie == -EBUSY) {
+                if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
+                        pr_err("%s timeout waiting for descriptor submission\n",
+                                __func__);
+                        return DMA_ERROR;
+                }
+                cpu_relax();
+        }
+        return dma_sync_wait(tx->chan, tx->cookie);
 }
 EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
 
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index d93017fc7872..a32a4cf7b1e0 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -48,6 +48,11 @@ module_param(xor_sources, uint, S_IRUGO);
 MODULE_PARM_DESC(xor_sources,
                 "Number of xor source buffers (default: 3)");
 
+static unsigned int pq_sources = 3;
+module_param(pq_sources, uint, S_IRUGO);
+MODULE_PARM_DESC(pq_sources,
+                "Number of p+q source buffers (default: 3)");
+
 /*
  * Initialization patterns. All bytes in the source buffer has bit 7
  * set, all bytes in the destination buffer has bit 7 cleared.
@@ -232,6 +237,7 @@ static int dmatest_func(void *data)
         dma_cookie_t            cookie;
         enum dma_status         status;
         enum dma_ctrl_flags     flags;
+        u8                      pq_coefs[pq_sources];
         int                     ret;
         int                     src_cnt;
         int                     dst_cnt;
@@ -248,6 +254,11 @@ static int dmatest_func(void *data)
         else if (thread->type == DMA_XOR) {
                 src_cnt = xor_sources | 1; /* force odd to ensure dst = src */
                 dst_cnt = 1;
+        } else if (thread->type == DMA_PQ) {
+                src_cnt = pq_sources | 1; /* force odd to ensure dst = src */
+                dst_cnt = 2;
+                for (i = 0; i < pq_sources; i++)
+                        pq_coefs[i] = 1;
         } else
                 goto err_srcs;
 
@@ -283,6 +294,7 @@ static int dmatest_func(void *data)
                 dma_addr_t dma_dsts[dst_cnt];
                 struct completion cmp;
                 unsigned long tmo = msecs_to_jiffies(3000);
+                u8 align = 0;
 
                 total_tests++;
 
@@ -290,6 +302,18 @@ static int dmatest_func(void *data)
                 src_off = dmatest_random() % (test_buf_size - len + 1);
                 dst_off = dmatest_random() % (test_buf_size - len + 1);
 
+                /* honor alignment restrictions */
+                if (thread->type == DMA_MEMCPY)
+                        align = dev->copy_align;
+                else if (thread->type == DMA_XOR)
+                        align = dev->xor_align;
+                else if (thread->type == DMA_PQ)
+                        align = dev->pq_align;
+
+                len = (len >> align) << align;
+                src_off = (src_off >> align) << align;
+                dst_off = (dst_off >> align) << align;
+
                 dmatest_init_srcs(thread->srcs, src_off, len);
                 dmatest_init_dsts(thread->dsts, dst_off, len);
 
@@ -306,6 +330,7 @@ static int dmatest_func(void *data)
                                                      DMA_BIDIRECTIONAL);
                 }
 
+
                 if (thread->type == DMA_MEMCPY)
                         tx = dev->device_prep_dma_memcpy(chan,
                                                          dma_dsts[0] + dst_off,
@@ -316,6 +341,15 @@ static int dmatest_func(void *data)
                                                       dma_dsts[0] + dst_off,
                                                       dma_srcs, xor_sources,
                                                       len, flags);
+                else if (thread->type == DMA_PQ) {
+                        dma_addr_t dma_pq[dst_cnt];
+
+                        for (i = 0; i < dst_cnt; i++)
+                                dma_pq[i] = dma_dsts[i] + dst_off;
+                        tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
+                                                     pq_sources, pq_coefs,
+                                                     len, flags);
+                }
 
                 if (!tx) {
                         for (i = 0; i < src_cnt; i++)
@@ -459,6 +493,8 @@ static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_ty
                 op = "copy";
         else if (type == DMA_XOR)
                 op = "xor";
+        else if (type == DMA_PQ)
+                op = "pq";
         else
                 return -EINVAL;
 
@@ -514,6 +550,10 @@ static int dmatest_add_channel(struct dma_chan *chan)
                 cnt = dmatest_add_threads(dtc, DMA_XOR);
                 thread_count += cnt > 0 ? cnt : 0;
         }
+        if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
+                cnt = dmatest_add_threads(dtc, DMA_PQ);
+                thread_count += cnt > 0 ?: 0;
+        }
 
         pr_info("dmatest: Started %u threads using %s\n",
                 thread_count, dma_chan_name(chan));
diff --git a/drivers/dma/dw_dmac.c b/drivers/dma/dw_dmac.c
index 933c143b6a74..2eea823516a7 100644
--- a/drivers/dma/dw_dmac.c
+++ b/drivers/dma/dw_dmac.c
@@ -116,7 +116,7 @@ static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
 {
         struct dw_desc  *child;
 
-        list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+        list_for_each_entry(child, &desc->tx_list, desc_node)
                 dma_sync_single_for_cpu(chan2parent(&dwc->chan),
                                 child->txd.phys, sizeof(child->lli),
                                 DMA_TO_DEVICE);
@@ -137,11 +137,11 @@ static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
                 dwc_sync_desc_for_cpu(dwc, desc);
 
                 spin_lock_bh(&dwc->lock);
-                list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+                list_for_each_entry(child, &desc->tx_list, desc_node)
                         dev_vdbg(chan2dev(&dwc->chan),
                                         "moving child desc %p to freelist\n",
                                         child);
-                list_splice_init(&desc->txd.tx_list, &dwc->free_list);
+                list_splice_init(&desc->tx_list, &dwc->free_list);
                 dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
                 list_add(&desc->desc_node, &dwc->free_list);
                 spin_unlock_bh(&dwc->lock);
@@ -209,19 +209,28 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc)
         param = txd->callback_param;
 
         dwc_sync_desc_for_cpu(dwc, desc);
-        list_splice_init(&txd->tx_list, &dwc->free_list);
+        list_splice_init(&desc->tx_list, &dwc->free_list);
         list_move(&desc->desc_node, &dwc->free_list);
 
-        /*
-         * We use dma_unmap_page() regardless of how the buffers were
-         * mapped before they were submitted...
-         */
-        if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP))
-                dma_unmap_page(chan2parent(&dwc->chan), desc->lli.dar,
-                               desc->len, DMA_FROM_DEVICE);
-        if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP))
-                dma_unmap_page(chan2parent(&dwc->chan), desc->lli.sar,
-                               desc->len, DMA_TO_DEVICE);
+        if (!dwc->chan.private) {
+                struct device *parent = chan2parent(&dwc->chan);
+                if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+                        if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
+                                dma_unmap_single(parent, desc->lli.dar,
+                                                desc->len, DMA_FROM_DEVICE);
+                        else
+                                dma_unmap_page(parent, desc->lli.dar,
+                                                desc->len, DMA_FROM_DEVICE);
+                }
+                if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                        if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
+                                dma_unmap_single(parent, desc->lli.sar,
+                                                desc->len, DMA_TO_DEVICE);
+                        else
+                                dma_unmap_page(parent, desc->lli.sar,
+                                                desc->len, DMA_TO_DEVICE);
+                }
+        }
 
         /*
          * The API requires that no submissions are done from a
@@ -289,7 +298,7 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
                         /* This one is currently in progress */
                         return;
 
-                list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+                list_for_each_entry(child, &desc->tx_list, desc_node)
                         if (child->lli.llp == llp)
                                 /* Currently in progress */
                                 return;
@@ -356,7 +365,7 @@ static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
         dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
                         "  cookie: %d\n", bad_desc->txd.cookie);
         dwc_dump_lli(dwc, &bad_desc->lli);
-        list_for_each_entry(child, &bad_desc->txd.tx_list, desc_node)
+        list_for_each_entry(child, &bad_desc->tx_list, desc_node)
                 dwc_dump_lli(dwc, &child->lli);
 
         /* Pretend the descriptor completed successfully */
@@ -608,7 +617,7 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
                                         prev->txd.phys, sizeof(prev->lli),
                                         DMA_TO_DEVICE);
                         list_add_tail(&desc->desc_node,
-                                        &first->txd.tx_list);
+                                        &first->tx_list);
                 }
                 prev = desc;
         }
@@ -658,8 +667,6 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
         reg_width = dws->reg_width;
         prev = first = NULL;
 
-        sg_len = dma_map_sg(chan2parent(chan), sgl, sg_len, direction);
-
         switch (direction) {
         case DMA_TO_DEVICE:
                 ctllo = (DWC_DEFAULT_CTLLO
@@ -700,7 +707,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                                 sizeof(prev->lli),
                                                 DMA_TO_DEVICE);
                                 list_add_tail(&desc->desc_node,
-                                                &first->txd.tx_list);
+                                                &first->tx_list);
                         }
                         prev = desc;
                         total_len += len;
@@ -746,7 +753,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                                 sizeof(prev->lli),
                                                 DMA_TO_DEVICE);
                                 list_add_tail(&desc->desc_node,
-                                                &first->txd.tx_list);
+                                                &first->tx_list);
                         }
                         prev = desc;
                         total_len += len;
@@ -902,6 +909,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
                         break;
                 }
 
+                INIT_LIST_HEAD(&desc->tx_list);
                 dma_async_tx_descriptor_init(&desc->txd, chan);
                 desc->txd.tx_submit = dwc_tx_submit;
                 desc->txd.flags = DMA_CTRL_ACK;
diff --git a/drivers/dma/dw_dmac_regs.h b/drivers/dma/dw_dmac_regs.h
index 13a580767031..d9a939f67f46 100644
--- a/drivers/dma/dw_dmac_regs.h
+++ b/drivers/dma/dw_dmac_regs.h
@@ -217,6 +217,7 @@ struct dw_desc {
 
         /* THEN values for driver housekeeping */
         struct list_head                desc_node;
+        struct list_head                tx_list;
         struct dma_async_tx_descriptor  txd;
         size_t                          len;
 };
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index ef87a8984145..296f9e747fac 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -34,6 +34,7 @@
 #include <linux/dmapool.h>
 #include <linux/of_platform.h>
 
+#include <asm/fsldma.h>
 #include "fsldma.h"
 
 static void dma_init(struct fsl_dma_chan *fsl_chan)
@@ -280,28 +281,40 @@ static void fsl_chan_set_dest_loop_size(struct fsl_dma_chan *fsl_chan, int size)
 }
 
 /**
- * fsl_chan_toggle_ext_pause - Toggle channel external pause status
+ * fsl_chan_set_request_count - Set DMA Request Count for external control
  * @fsl_chan : Freescale DMA channel
- * @size     : Pause control size, 0 for disable external pause control.
- *             The maximum is 1024.
+ * @size     : Number of bytes to transfer in a single request
+ *
+ * The Freescale DMA channel can be controlled by the external signal DREQ#.
+ * The DMA request count is how many bytes are allowed to transfer before
+ * pausing the channel, after which a new assertion of DREQ# resumes channel
+ * operation.
  *
- * The Freescale DMA channel can be controlled by the external
- * signal DREQ#. The pause control size is how many bytes are allowed
- * to transfer before pausing the channel, after which a new assertion
- * of DREQ# resumes channel operation.
+ * A size of 0 disables external pause control. The maximum size is 1024.
  */
-static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int size)
+static void fsl_chan_set_request_count(struct fsl_dma_chan *fsl_chan, int size)
 {
-        if (size > 1024)
-                return;
+        BUG_ON(size > 1024);
+        DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
+                DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
+                        | ((__ilog2(size) << 24) & 0x0f000000),
+                32);
+}
 
-        if (size) {
-                DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
-                        DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
-                                | ((__ilog2(size) << 24) & 0x0f000000),
-                        32);
+/**
+ * fsl_chan_toggle_ext_pause - Toggle channel external pause status
+ * @fsl_chan : Freescale DMA channel
+ * @enable   : 0 is disabled, 1 is enabled.
+ *
+ * The Freescale DMA channel can be controlled by the external signal DREQ#.
+ * The DMA Request Count feature should be used in addition to this feature
+ * to set the number of bytes to transfer before pausing the channel.
+ */
+static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int enable)
+{
+        if (enable)
                 fsl_chan->feature |= FSL_DMA_CHAN_PAUSE_EXT;
-        } else
+        else
                 fsl_chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT;
 }
 
@@ -326,7 +339,8 @@ static void fsl_chan_toggle_ext_start(struct fsl_dma_chan *fsl_chan, int enable)
 static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
 {
         struct fsl_dma_chan *fsl_chan = to_fsl_chan(tx->chan);
-        struct fsl_desc_sw *desc;
+        struct fsl_desc_sw *desc = tx_to_fsl_desc(tx);
+        struct fsl_desc_sw *child;
         unsigned long flags;
         dma_cookie_t cookie;
 
@@ -334,7 +348,7 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
         spin_lock_irqsave(&fsl_chan->desc_lock, flags);
 
         cookie = fsl_chan->common.cookie;
-        list_for_each_entry(desc, &tx->tx_list, node) {
+        list_for_each_entry(child, &desc->tx_list, node) {
                 cookie++;
                 if (cookie < 0)
                         cookie = 1;
@@ -343,8 +357,8 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
         }
 
         fsl_chan->common.cookie = cookie;
-        append_ld_queue(fsl_chan, tx_to_fsl_desc(tx));
-        list_splice_init(&tx->tx_list, fsl_chan->ld_queue.prev);
+        append_ld_queue(fsl_chan, desc);
+        list_splice_init(&desc->tx_list, fsl_chan->ld_queue.prev);
 
         spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
 
@@ -366,6 +380,7 @@ static struct fsl_desc_sw *fsl_dma_alloc_descriptor(
         desc_sw = dma_pool_alloc(fsl_chan->desc_pool, GFP_ATOMIC, &pdesc);
         if (desc_sw) {
                 memset(desc_sw, 0, sizeof(struct fsl_desc_sw));
+                INIT_LIST_HEAD(&desc_sw->tx_list);
                 dma_async_tx_descriptor_init(&desc_sw->async_tx,
                                                 &fsl_chan->common);
                 desc_sw->async_tx.tx_submit = fsl_dma_tx_submit;
@@ -455,7 +470,7 @@ fsl_dma_prep_interrupt(struct dma_chan *chan, unsigned long flags)
         new->async_tx.flags = flags;
 
         /* Insert the link descriptor to the LD ring */
-        list_add_tail(&new->node, &new->async_tx.tx_list);
+        list_add_tail(&new->node, &new->tx_list);
 
         /* Set End-of-link to the last link descriptor of new list*/
         set_ld_eol(fsl_chan, new);
@@ -513,7 +528,7 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy(
                 dma_dest += copy;
 
                 /* Insert the link descriptor to the LD ring */
-                list_add_tail(&new->node, &first->async_tx.tx_list);
+                list_add_tail(&new->node, &first->tx_list);
         } while (len);
 
         new->async_tx.flags = flags; /* client is in control of this ack */
@@ -528,7 +543,7 @@ fail:
         if (!first)
                 return NULL;
 
-        list = &first->async_tx.tx_list;
+        list = &first->tx_list;
         list_for_each_entry_safe_reverse(new, prev, list, node) {
                 list_del(&new->node);
                 dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys);
@@ -538,6 +553,229 @@ fail:
 }
 
 /**
+ * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
+ * @chan: DMA channel
+ * @sgl: scatterlist to transfer to/from
+ * @sg_len: number of entries in @scatterlist
+ * @direction: DMA direction
+ * @flags: DMAEngine flags
+ *
+ * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
+ * DMA_SLAVE API, this gets the device-specific information from the
+ * chan->private variable.
+ */
+static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg(
+        struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
+        enum dma_data_direction direction, unsigned long flags)
+{
+        struct fsl_dma_chan *fsl_chan;
+        struct fsl_desc_sw *first = NULL, *prev = NULL, *new = NULL;
+        struct fsl_dma_slave *slave;
+        struct list_head *tx_list;
+        size_t copy;
+
+        int i;
+        struct scatterlist *sg;
+        size_t sg_used;
+        size_t hw_used;
+        struct fsl_dma_hw_addr *hw;
+        dma_addr_t dma_dst, dma_src;
+
+        if (!chan)
+                return NULL;
+
+        if (!chan->private)
+                return NULL;
+
+        fsl_chan = to_fsl_chan(chan);
+        slave = chan->private;
+
+        if (list_empty(&slave->addresses))
+                return NULL;
+
+        hw = list_first_entry(&slave->addresses, struct fsl_dma_hw_addr, entry);
+        hw_used = 0;
+
+        /*
+         * Build the hardware transaction to copy from the scatterlist to
+         * the hardware, or from the hardware to the scatterlist
+         *
+         * If you are copying from the hardware to the scatterlist and it
+         * takes two hardware entries to fill an entire page, then both
+         * hardware entries will be coalesced into the same page
+         *
+         * If you are copying from the scatterlist to the hardware and a
+         * single page can fill two hardware entries, then the data will
+         * be read out of the page into the first hardware entry, and so on
+         */
+        for_each_sg(sgl, sg, sg_len, i) {
+                sg_used = 0;
+
+                /* Loop until the entire scatterlist entry is used */
+                while (sg_used < sg_dma_len(sg)) {
+
+                        /*
+                         * If we've used up the current hardware address/length
+                         * pair, we need to load a new one
+                         *
+                         * This is done in a while loop so that descriptors with
+                         * length == 0 will be skipped
+                         */
+                        while (hw_used >= hw->length) {
+
+                                /*
+                                 * If the current hardware entry is the last
+                                 * entry in the list, we're finished
+                                 */
+                                if (list_is_last(&hw->entry, &slave->addresses))
+                                        goto finished;
+
+                                /* Get the next hardware address/length pair */
+                                hw = list_entry(hw->entry.next,
+                                                struct fsl_dma_hw_addr, entry);
+                                hw_used = 0;
+                        }
+
+                        /* Allocate the link descriptor from DMA pool */
+                        new = fsl_dma_alloc_descriptor(fsl_chan);
+                        if (!new) {
+                                dev_err(fsl_chan->dev, "No free memory for "
+                                                       "link descriptor\n");
+                                goto fail;
+                        }
+#ifdef FSL_DMA_LD_DEBUG
+                        dev_dbg(fsl_chan->dev, "new link desc alloc %p\n", new);
+#endif
+
+                        /*
+                         * Calculate the maximum number of bytes to transfer,
+                         * making sure it is less than the DMA controller limit
+                         */
+                        copy = min_t(size_t, sg_dma_len(sg) - sg_used,
+                                             hw->length - hw_used);
+                        copy = min_t(size_t, copy, FSL_DMA_BCR_MAX_CNT);
+
+                        /*
+                         * DMA_FROM_DEVICE
+                         * from the hardware to the scatterlist
+                         *
+                         * DMA_TO_DEVICE
+                         * from the scatterlist to the hardware
+                         */
+                        if (direction == DMA_FROM_DEVICE) {
+                                dma_src = hw->address + hw_used;
+                                dma_dst = sg_dma_address(sg) + sg_used;
+                        } else {
+                                dma_src = sg_dma_address(sg) + sg_used;
+                                dma_dst = hw->address + hw_used;
+                        }
+
+                        /* Fill in the descriptor */
+                        set_desc_cnt(fsl_chan, &new->hw, copy);
+                        set_desc_src(fsl_chan, &new->hw, dma_src);
+                        set_desc_dest(fsl_chan, &new->hw, dma_dst);
+
+                        /*
+                         * If this is not the first descriptor, chain the
+                         * current descriptor after the previous descriptor
+                         */
+                        if (!first) {
+                                first = new;
+                        } else {
+                                set_desc_next(fsl_chan, &prev->hw,
+                                              new->async_tx.phys);
+                        }
+
+                        new->async_tx.cookie = 0;
+                        async_tx_ack(&new->async_tx);
+
+                        prev = new;
+                        sg_used += copy;
+                        hw_used += copy;
+
+                        /* Insert the link descriptor into the LD ring */
+                        list_add_tail(&new->node, &first->tx_list);
+                }
+        }
+
+finished:
+
+        /* All of the hardware address/length pairs had length == 0 */
+        if (!first || !new)
+                return NULL;
+
+        new->async_tx.flags = flags;
+        new->async_tx.cookie = -EBUSY;
+
+        /* Set End-of-link to the last link descriptor of new list */
+        set_ld_eol(fsl_chan, new);
+
+        /* Enable extra controller features */
+        if (fsl_chan->set_src_loop_size)
+                fsl_chan->set_src_loop_size(fsl_chan, slave->src_loop_size);
+
+        if (fsl_chan->set_dest_loop_size)
+                fsl_chan->set_dest_loop_size(fsl_chan, slave->dst_loop_size);
+
+        if (fsl_chan->toggle_ext_start)
+                fsl_chan->toggle_ext_start(fsl_chan, slave->external_start);
+
+        if (fsl_chan->toggle_ext_pause)
+                fsl_chan->toggle_ext_pause(fsl_chan, slave->external_pause);
+
+        if (fsl_chan->set_request_count)
+                fsl_chan->set_request_count(fsl_chan, slave->request_count);
+
+        return &first->async_tx;
+
+fail:
+        /* If first was not set, then we failed to allocate the very first
+         * descriptor, and we're done */
+        if (!first)
+                return NULL;
+
+        /*
+         * First is set, so all of the descriptors we allocated have been added
+         * to first->tx_list, INCLUDING "first" itself. Therefore we
+         * must traverse the list backwards freeing each descriptor in turn
+         *
+         * We're re-using variables for the loop, oh well
+         */
+        tx_list = &first->tx_list;
+        list_for_each_entry_safe_reverse(new, prev, tx_list, node) {
+                list_del_init(&new->node);
+                dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys);
+        }
+
+        return NULL;
+}
+
+static void fsl_dma_device_terminate_all(struct dma_chan *chan)
+{
+        struct fsl_dma_chan *fsl_chan;
+        struct fsl_desc_sw *desc, *tmp;
+        unsigned long flags;
+
+        if (!chan)
+                return;
+
+        fsl_chan = to_fsl_chan(chan);
+
+        /* Halt the DMA engine */
+        dma_halt(fsl_chan);
+
+        spin_lock_irqsave(&fsl_chan->desc_lock, flags);
+
+        /* Remove and free all of the descriptors in the LD queue */
+        list_for_each_entry_safe(desc, tmp, &fsl_chan->ld_queue, node) {
+                list_del(&desc->node);
+                dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys);
+        }
+
+        spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
+}
+
+/**
  * fsl_dma_update_completed_cookie - Update the completed cookie.
  * @fsl_chan : Freescale DMA channel
  */
@@ -883,6 +1121,7 @@ static int __devinit fsl_dma_chan_probe(struct fsl_dma_device *fdev,
                 new_fsl_chan->toggle_ext_start = fsl_chan_toggle_ext_start;
                 new_fsl_chan->set_src_loop_size = fsl_chan_set_src_loop_size;
                 new_fsl_chan->set_dest_loop_size = fsl_chan_set_dest_loop_size;
+                new_fsl_chan->set_request_count = fsl_chan_set_request_count;
         }
 
         spin_lock_init(&new_fsl_chan->desc_lock);
@@ -962,12 +1201,15 @@ static int __devinit of_fsl_dma_probe(struct of_device *dev,
 
         dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
         dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask);
+        dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
         fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
         fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
         fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt;
         fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
         fdev->common.device_is_tx_complete = fsl_dma_is_complete;
         fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
+        fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg;
+        fdev->common.device_terminate_all = fsl_dma_device_terminate_all;
         fdev->common.dev = &dev->dev;
 
         fdev->irq = irq_of_parse_and_map(dev->node, 0);
diff --git a/drivers/dma/fsldma.h b/drivers/dma/fsldma.h
index dc7f26865797..0df14cbb8ca3 100644
--- a/drivers/dma/fsldma.h
+++ b/drivers/dma/fsldma.h
@@ -90,6 +90,7 @@ struct fsl_dma_ld_hw {
 struct fsl_desc_sw {
         struct fsl_dma_ld_hw hw;
         struct list_head node;
+        struct list_head tx_list;
         struct dma_async_tx_descriptor async_tx;
         struct list_head *ld;
         void *priv;
@@ -143,10 +144,11 @@ struct fsl_dma_chan {
         struct tasklet_struct tasklet;
         u32 feature;
 
-        void (*toggle_ext_pause)(struct fsl_dma_chan *fsl_chan, int size);
+        void (*toggle_ext_pause)(struct fsl_dma_chan *fsl_chan, int enable);
         void (*toggle_ext_start)(struct fsl_dma_chan *fsl_chan, int enable);
         void (*set_src_loop_size)(struct fsl_dma_chan *fsl_chan, int size);
         void (*set_dest_loop_size)(struct fsl_dma_chan *fsl_chan, int size);
+        void (*set_request_count)(struct fsl_dma_chan *fsl_chan, int size);
 };
 
 #define to_fsl_chan(chan) container_of(chan, struct fsl_dma_chan, common)
diff --git a/drivers/dma/ioat.c b/drivers/dma/ioat.c
deleted file mode 100644
index 2225bb6ba3d1..000000000000
--- a/drivers/dma/ioat.c
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Intel I/OAT DMA Linux driver
- * Copyright(c) 2007 - 2009 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- */
-
-/*
- * This driver supports an Intel I/OAT DMA engine, which does asynchronous
- * copy operations.
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/pci.h>
-#include <linux/interrupt.h>
-#include <linux/dca.h>
-#include "ioatdma.h"
-#include "ioatdma_registers.h"
-#include "ioatdma_hw.h"
-
-MODULE_VERSION(IOAT_DMA_VERSION);
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Intel Corporation");
-
-static struct pci_device_id ioat_pci_tbl[] = {
-        /* I/OAT v1 platforms */
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB)  },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) },
-        { PCI_DEVICE(PCI_VENDOR_ID_UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) },
-
-        /* I/OAT v2 platforms */
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) },
-
-        /* I/OAT v3 platforms */
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) },
-        { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) },
-        { 0, }
-};
-
-struct ioat_device {
-        struct pci_dev          *pdev;
-        void __iomem            *iobase;
-        struct ioatdma_device   *dma;
-        struct dca_provider     *dca;
-};
-
-static int __devinit ioat_probe(struct pci_dev *pdev,
-                                const struct pci_device_id *id);
-static void __devexit ioat_remove(struct pci_dev *pdev);
-
-static int ioat_dca_enabled = 1;
-module_param(ioat_dca_enabled, int, 0644);
-MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
-
-static struct pci_driver ioat_pci_driver = {
-        .name           = "ioatdma",
-        .id_table       = ioat_pci_tbl,
-        .probe          = ioat_probe,
-        .remove         = __devexit_p(ioat_remove),
-};
-
-static int __devinit ioat_probe(struct pci_dev *pdev,
-                                const struct pci_device_id *id)
-{
-        void __iomem *iobase;
-        struct ioat_device *device;
-        unsigned long mmio_start, mmio_len;
-        int err;
-
-        err = pci_enable_device(pdev);
-        if (err)
-                goto err_enable_device;
-
-        err = pci_request_regions(pdev, ioat_pci_driver.name);
-        if (err)
-                goto err_request_regions;
-
-        err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
-        if (err)
-                err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
-        if (err)
-                goto err_set_dma_mask;
-
-        err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
-        if (err)
-                err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
-        if (err)
-                goto err_set_dma_mask;
-
-        mmio_start = pci_resource_start(pdev, 0);
-        mmio_len = pci_resource_len(pdev, 0);
-        iobase = ioremap(mmio_start, mmio_len);
-        if (!iobase) {
-                err = -ENOMEM;
-                goto err_ioremap;
-        }
-
-        device = kzalloc(sizeof(*device), GFP_KERNEL);
-        if (!device) {
-                err = -ENOMEM;
-                goto err_kzalloc;
-        }
-        device->pdev = pdev;
-        pci_set_drvdata(pdev, device);
-        device->iobase = iobase;
-
-        pci_set_master(pdev);
-
-        switch (readb(iobase + IOAT_VER_OFFSET)) {
-        case IOAT_VER_1_2:
-                device->dma = ioat_dma_probe(pdev, iobase);
-                if (device->dma && ioat_dca_enabled)
-                        device->dca = ioat_dca_init(pdev, iobase);
-                break;
-        case IOAT_VER_2_0:
-                device->dma = ioat_dma_probe(pdev, iobase);
-                if (device->dma && ioat_dca_enabled)
-                        device->dca = ioat2_dca_init(pdev, iobase);
-                break;
-        case IOAT_VER_3_0:
-                device->dma = ioat_dma_probe(pdev, iobase);
-                if (device->dma && ioat_dca_enabled)
-                        device->dca = ioat3_dca_init(pdev, iobase);
-                break;
-        default:
-                err = -ENODEV;
-                break;
-        }
-        if (!device->dma)
-                err = -ENODEV;
-
-        if (err)
-                goto err_version;
-
-        return 0;
-
-err_version:
-        kfree(device);
-err_kzalloc:
-        iounmap(iobase);
-err_ioremap:
-err_set_dma_mask:
-        pci_release_regions(pdev);
-        pci_disable_device(pdev);
-err_request_regions:
-err_enable_device:
-        return err;
-}
-
-static void __devexit ioat_remove(struct pci_dev *pdev)
-{
-        struct ioat_device *device = pci_get_drvdata(pdev);
-
-        dev_err(&pdev->dev, "Removing dma and dca services\n");
-        if (device->dca) {
-                unregister_dca_provider(device->dca);
-                free_dca_provider(device->dca);
-                device->dca = NULL;
-        }
-
-        if (device->dma) {
-                ioat_dma_remove(device->dma);
-                device->dma = NULL;
-        }
-
-        kfree(device);
-}
-
-static int __init ioat_init_module(void)
-{
-        return pci_register_driver(&ioat_pci_driver);
-}
-module_init(ioat_init_module);
-
-static void __exit ioat_exit_module(void)
-{
-        pci_unregister_driver(&ioat_pci_driver);
-}
-module_exit(ioat_exit_module);
diff --git a/drivers/dma/ioat/Makefile b/drivers/dma/ioat/Makefile
new file mode 100644
index 000000000000..8997d3fb9051
--- /dev/null
+++ b/drivers/dma/ioat/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
+ioatdma-objs := pci.o dma.o dma_v2.o dma_v3.o dca.o
diff --git a/drivers/dma/ioat_dca.c b/drivers/dma/ioat/dca.c
index c012a1e15043..69d02615c4d6 100644
--- a/drivers/dma/ioat_dca.c
+++ b/drivers/dma/ioat/dca.c
@@ -33,8 +33,8 @@
 #define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
 #endif
 
-#include "ioatdma.h"
-#include "ioatdma_registers.h"
+#include "dma.h"
+#include "registers.h"
 
 /*
  * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
@@ -242,7 +242,8 @@ static struct dca_ops ioat_dca_ops = {
 };
 
 
-struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
+struct dca_provider * __devinit
+ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 {
         struct dca_provider *dca;
         struct ioat_dca_priv *ioatdca;
@@ -407,7 +408,8 @@ static int ioat2_dca_count_dca_slots(void __iomem *iobase, u16 dca_offset)
         return slots;
 }
 
-struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
+struct dca_provider * __devinit
+ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 {
         struct dca_provider *dca;
         struct ioat_dca_priv *ioatdca;
@@ -602,7 +604,8 @@ static int ioat3_dca_count_dca_slots(void *iobase, u16 dca_offset)
         return slots;
 }
 
-struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
+struct dca_provider * __devinit
+ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 {
         struct dca_provider *dca;
         struct ioat_dca_priv *ioatdca;
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
new file mode 100644
index 000000000000..c524d36d3c2e
--- /dev/null
+++ b/drivers/dma/ioat/dma.c
@@ -0,0 +1,1238 @@
+/*
+ * Intel I/OAT DMA Linux driver
+ * Copyright(c) 2004 - 2009 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ */
+
+/*
+ * This driver supports an Intel I/OAT DMA engine, which does asynchronous
+ * copy operations.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/dmaengine.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/workqueue.h>
+#include <linux/i7300_idle.h>
+#include "dma.h"
+#include "registers.h"
+#include "hw.h"
+
+int ioat_pending_level = 4;
+module_param(ioat_pending_level, int, 0644);
+MODULE_PARM_DESC(ioat_pending_level,
+                 "high-water mark for pushing ioat descriptors (default: 4)");
+
+/* internal functions */
+static void ioat1_cleanup(struct ioat_dma_chan *ioat);
+static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat);
+
+/**
+ * ioat_dma_do_interrupt - handler used for single vector interrupt mode
+ * @irq: interrupt id
+ * @data: interrupt data
+ */
+static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
+{
+        struct ioatdma_device *instance = data;
+        struct ioat_chan_common *chan;
+        unsigned long attnstatus;
+        int bit;
+        u8 intrctrl;
+
+        intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
+
+        if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
+                return IRQ_NONE;
+
+        if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
+                writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
+                return IRQ_NONE;
+        }
+
+        attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
+        for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
+                chan = ioat_chan_by_index(instance, bit);
+                tasklet_schedule(&chan->cleanup_task);
+        }
+
+        writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
+        return IRQ_HANDLED;
+}
+
+/**
+ * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
+ * @irq: interrupt id
+ * @data: interrupt data
+ */
+static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
+{
+        struct ioat_chan_common *chan = data;
+
+        tasklet_schedule(&chan->cleanup_task);
+
+        return IRQ_HANDLED;
+}
+
+static void ioat1_cleanup_tasklet(unsigned long data);
+
+/* common channel initialization */
+void ioat_init_channel(struct ioatdma_device *device,
+                       struct ioat_chan_common *chan, int idx,
+                       void (*timer_fn)(unsigned long),
+                       void (*tasklet)(unsigned long),
+                       unsigned long ioat)
+{
+        struct dma_device *dma = &device->common;
+
+        chan->device = device;
+        chan->reg_base = device->reg_base + (0x80 * (idx + 1));
+        spin_lock_init(&chan->cleanup_lock);
+        chan->common.device = dma;
+        list_add_tail(&chan->common.device_node, &dma->channels);
+        device->idx[idx] = chan;
+        init_timer(&chan->timer);
+        chan->timer.function = timer_fn;
+        chan->timer.data = ioat;
+        tasklet_init(&chan->cleanup_task, tasklet, ioat);
+        tasklet_disable(&chan->cleanup_task);
+}
+
+static void ioat1_timer_event(unsigned long data);
+
+/**
+ * ioat1_dma_enumerate_channels - find and initialize the device's channels
+ * @device: the device to be enumerated
+ */
+static int ioat1_enumerate_channels(struct ioatdma_device *device)
+{
+        u8 xfercap_scale;
+        u32 xfercap;
+        int i;
+        struct ioat_dma_chan *ioat;
+        struct device *dev = &device->pdev->dev;
+        struct dma_device *dma = &device->common;
+
+        INIT_LIST_HEAD(&dma->channels);
+        dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
+        dma->chancnt &= 0x1f; /* bits [4:0] valid */
+        if (dma->chancnt > ARRAY_SIZE(device->idx)) {
+                dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
+                         dma->chancnt, ARRAY_SIZE(device->idx));
+                dma->chancnt = ARRAY_SIZE(device->idx);
+        }
+        xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
+        xfercap_scale &= 0x1f; /* bits [4:0] valid */
+        xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
+        dev_dbg(dev, "%s: xfercap = %d\n", __func__, xfercap);
+
+#ifdef  CONFIG_I7300_IDLE_IOAT_CHANNEL
+        if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
+                dma->chancnt--;
+#endif
+        for (i = 0; i < dma->chancnt; i++) {
+                ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
+                if (!ioat)
+                        break;
+
+                ioat_init_channel(device, &ioat->base, i,
+                                  ioat1_timer_event,
+                                  ioat1_cleanup_tasklet,
+                                  (unsigned long) ioat);
+                ioat->xfercap = xfercap;
+                spin_lock_init(&ioat->desc_lock);
+                INIT_LIST_HEAD(&ioat->free_desc);
+                INIT_LIST_HEAD(&ioat->used_desc);
+        }
+        dma->chancnt = i;
+        return i;
+}
+
+/**
+ * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
+ *                                 descriptors to hw
+ * @chan: DMA channel handle
+ */
+static inline void
+__ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
+{
+        void __iomem *reg_base = ioat->base.reg_base;
+
+        dev_dbg(to_dev(&ioat->base), "%s: pending: %d\n",
+                __func__, ioat->pending);
+        ioat->pending = 0;
+        writeb(IOAT_CHANCMD_APPEND, reg_base + IOAT1_CHANCMD_OFFSET);
+}
+
+static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(chan);
+
+        if (ioat->pending > 0) {
+                spin_lock_bh(&ioat->desc_lock);
+                __ioat1_dma_memcpy_issue_pending(ioat);
+                spin_unlock_bh(&ioat->desc_lock);
+        }
+}
+
+/**
+ * ioat1_reset_channel - restart a channel
+ * @ioat: IOAT DMA channel handle
+ */
+static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        void __iomem *reg_base = chan->reg_base;
+        u32 chansts, chanerr;
+
+        dev_warn(to_dev(chan), "reset\n");
+        chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
+        chansts = *chan->completion & IOAT_CHANSTS_STATUS;
+        if (chanerr) {
+                dev_err(to_dev(chan),
+                        "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
+                        chan_num(chan), chansts, chanerr);
+                writel(chanerr, reg_base + IOAT_CHANERR_OFFSET);
+        }
+
+        /*
+         * whack it upside the head with a reset
+         * and wait for things to settle out.
+         * force the pending count to a really big negative
+         * to make sure no one forces an issue_pending
+         * while we're waiting.
+         */
+
+        ioat->pending = INT_MIN;
+        writeb(IOAT_CHANCMD_RESET,
+               reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
+        set_bit(IOAT_RESET_PENDING, &chan->state);
+        mod_timer(&chan->timer, jiffies + RESET_DELAY);
+}
+
+static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+        struct dma_chan *c = tx->chan;
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+        struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioat_desc_sw *first;
+        struct ioat_desc_sw *chain_tail;
+        dma_cookie_t cookie;
+
+        spin_lock_bh(&ioat->desc_lock);
+        /* cookie incr and addition to used_list must be atomic */
+        cookie = c->cookie;
+        cookie++;
+        if (cookie < 0)
+                cookie = 1;
+        c->cookie = cookie;
+        tx->cookie = cookie;
+        dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
+
+        /* write address into NextDescriptor field of last desc in chain */
+        first = to_ioat_desc(desc->tx_list.next);
+        chain_tail = to_ioat_desc(ioat->used_desc.prev);
+        /* make descriptor updates globally visible before chaining */
+        wmb();
+        chain_tail->hw->next = first->txd.phys;
+        list_splice_tail_init(&desc->tx_list, &ioat->used_desc);
+        dump_desc_dbg(ioat, chain_tail);
+        dump_desc_dbg(ioat, first);
+
+        if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
+                mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
+        ioat->active += desc->hw->tx_cnt;
+        ioat->pending += desc->hw->tx_cnt;
+        if (ioat->pending >= ioat_pending_level)
+                __ioat1_dma_memcpy_issue_pending(ioat);
+        spin_unlock_bh(&ioat->desc_lock);
+
+        return cookie;
+}
+
+/**
+ * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
+ * @ioat: the channel supplying the memory pool for the descriptors
+ * @flags: allocation flags
+ */
+static struct ioat_desc_sw *
+ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
+{
+        struct ioat_dma_descriptor *desc;
+        struct ioat_desc_sw *desc_sw;
+        struct ioatdma_device *ioatdma_device;
+        dma_addr_t phys;
+
+        ioatdma_device = ioat->base.device;
+        desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
+        if (unlikely(!desc))
+                return NULL;
+
+        desc_sw = kzalloc(sizeof(*desc_sw), flags);
+        if (unlikely(!desc_sw)) {
+                pci_pool_free(ioatdma_device->dma_pool, desc, phys);
+                return NULL;
+        }
+
+        memset(desc, 0, sizeof(*desc));
+
+        INIT_LIST_HEAD(&desc_sw->tx_list);
+        dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
+        desc_sw->txd.tx_submit = ioat1_tx_submit;
+        desc_sw->hw = desc;
+        desc_sw->txd.phys = phys;
+        set_desc_id(desc_sw, -1);
+
+        return desc_sw;
+}
+
+static int ioat_initial_desc_count = 256;
+module_param(ioat_initial_desc_count, int, 0644);
+MODULE_PARM_DESC(ioat_initial_desc_count,
+                 "ioat1: initial descriptors per channel (default: 256)");
+/**
+ * ioat1_dma_alloc_chan_resources - returns the number of allocated descriptors
+ * @chan: the channel to be filled out
+ */
+static int ioat1_dma_alloc_chan_resources(struct dma_chan *c)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioat_desc_sw *desc;
+        u32 chanerr;
+        int i;
+        LIST_HEAD(tmp_list);
+
+        /* have we already been set up? */
+        if (!list_empty(&ioat->free_desc))
+                return ioat->desccount;
+
+        /* Setup register to interrupt and write completion status on error */
+        writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET);
+
+        chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+        if (chanerr) {
+                dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
+                writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
+        }
+
+        /* Allocate descriptors */
+        for (i = 0; i < ioat_initial_desc_count; i++) {
+                desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL);
+                if (!desc) {
+                        dev_err(to_dev(chan), "Only %d initial descriptors\n", i);
+                        break;
+                }
+                set_desc_id(desc, i);
+                list_add_tail(&desc->node, &tmp_list);
+        }
+        spin_lock_bh(&ioat->desc_lock);
+        ioat->desccount = i;
+        list_splice(&tmp_list, &ioat->free_desc);
+        spin_unlock_bh(&ioat->desc_lock);
+
+        /* allocate a completion writeback area */
+        /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
+        chan->completion = pci_pool_alloc(chan->device->completion_pool,
+                                          GFP_KERNEL, &chan->completion_dma);
+        memset(chan->completion, 0, sizeof(*chan->completion));
+        writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
+               chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
+        writel(((u64) chan->completion_dma) >> 32,
+               chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
+
+        tasklet_enable(&chan->cleanup_task);
+        ioat1_dma_start_null_desc(ioat);  /* give chain to dma device */
+        dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
+                __func__, ioat->desccount);
+        return ioat->desccount;
+}
+
+/**
+ * ioat1_dma_free_chan_resources - release all the descriptors
+ * @chan: the channel to be cleaned
+ */
+static void ioat1_dma_free_chan_resources(struct dma_chan *c)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioatdma_device *ioatdma_device = chan->device;
+        struct ioat_desc_sw *desc, *_desc;
+        int in_use_descs = 0;
+
+        /* Before freeing channel resources first check
+         * if they have been previously allocated for this channel.
+         */
+        if (ioat->desccount == 0)
+                return;
+
+        tasklet_disable(&chan->cleanup_task);
+        del_timer_sync(&chan->timer);
+        ioat1_cleanup(ioat);
+
+        /* Delay 100ms after reset to allow internal DMA logic to quiesce
+         * before removing DMA descriptor resources.
+         */
+        writeb(IOAT_CHANCMD_RESET,
+               chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
+        mdelay(100);
+
+        spin_lock_bh(&ioat->desc_lock);
+        list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
+                dev_dbg(to_dev(chan), "%s: freeing %d from used list\n",
+                        __func__, desc_id(desc));
+                dump_desc_dbg(ioat, desc);
+                in_use_descs++;
+                list_del(&desc->node);
+                pci_pool_free(ioatdma_device->dma_pool, desc->hw,
+                              desc->txd.phys);
+                kfree(desc);
+        }
+        list_for_each_entry_safe(desc, _desc,
+                                 &ioat->free_desc, node) {
+                list_del(&desc->node);
+                pci_pool_free(ioatdma_device->dma_pool, desc->hw,
+                              desc->txd.phys);
+                kfree(desc);
+        }
+        spin_unlock_bh(&ioat->desc_lock);
+
+        pci_pool_free(ioatdma_device->completion_pool,
+                      chan->completion,
+                      chan->completion_dma);
+
+        /* one is ok since we left it on there on purpose */
+        if (in_use_descs > 1)
+                dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
+                        in_use_descs - 1);
+
+        chan->last_completion = 0;
+        chan->completion_dma = 0;
+        ioat->pending = 0;
+        ioat->desccount = 0;
+}
+
+/**
+ * ioat1_dma_get_next_descriptor - return the next available descriptor
+ * @ioat: IOAT DMA channel handle
+ *
+ * Gets the next descriptor from the chain, and must be called with the
+ * channel's desc_lock held.  Allocates more descriptors if the channel
+ * has run out.
+ */
+static struct ioat_desc_sw *
+ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
+{
+        struct ioat_desc_sw *new;
+
+        if (!list_empty(&ioat->free_desc)) {
+                new = to_ioat_desc(ioat->free_desc.next);
+                list_del(&new->node);
+        } else {
+                /* try to get another desc */
+                new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
+                if (!new) {
+                        dev_err(to_dev(&ioat->base), "alloc failed\n");
+                        return NULL;
+                }
+        }
+        dev_dbg(to_dev(&ioat->base), "%s: allocated: %d\n",
+                __func__, desc_id(new));
+        prefetch(new->hw);
+        return new;
+}
+
+static struct dma_async_tx_descriptor *
+ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
+                      dma_addr_t dma_src, size_t len, unsigned long flags)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+        struct ioat_desc_sw *desc;
+        size_t copy;
+        LIST_HEAD(chain);
+        dma_addr_t src = dma_src;
+        dma_addr_t dest = dma_dest;
+        size_t total_len = len;
+        struct ioat_dma_descriptor *hw = NULL;
+        int tx_cnt = 0;
+
+        spin_lock_bh(&ioat->desc_lock);
+        desc = ioat1_dma_get_next_descriptor(ioat);
+        do {
+                if (!desc)
+                        break;
+
+                tx_cnt++;
+                copy = min_t(size_t, len, ioat->xfercap);
+
+                hw = desc->hw;
+                hw->size = copy;
+                hw->ctl = 0;
+                hw->src_addr = src;
+                hw->dst_addr = dest;
+
+                list_add_tail(&desc->node, &chain);
+
+                len -= copy;
+                dest += copy;
+                src += copy;
+                if (len) {
+                        struct ioat_desc_sw *next;
+
+                        async_tx_ack(&desc->txd);
+                        next = ioat1_dma_get_next_descriptor(ioat);
+                        hw->next = next ? next->txd.phys : 0;
+                        dump_desc_dbg(ioat, desc);
+                        desc = next;
+                } else
+                        hw->next = 0;
+        } while (len);
+
+        if (!desc) {
+                struct ioat_chan_common *chan = &ioat->base;
+
+                dev_err(to_dev(chan),
+                        "chan%d - get_next_desc failed\n", chan_num(chan));
+                list_splice(&chain, &ioat->free_desc);
+                spin_unlock_bh(&ioat->desc_lock);
+                return NULL;
+        }
+        spin_unlock_bh(&ioat->desc_lock);
+
+        desc->txd.flags = flags;
+        desc->len = total_len;
+        list_splice(&chain, &desc->tx_list);
+        hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+        hw->ctl_f.compl_write = 1;
+        hw->tx_cnt = tx_cnt;
+        dump_desc_dbg(ioat, desc);
+
+        return &desc->txd;
+}
+
+static void ioat1_cleanup_tasklet(unsigned long data)
+{
+        struct ioat_dma_chan *chan = (void *)data;
+
+        ioat1_cleanup(chan);
+        writew(IOAT_CHANCTRL_RUN, chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
+}
+
+void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
+                    size_t len, struct ioat_dma_descriptor *hw)
+{
+        struct pci_dev *pdev = chan->device->pdev;
+        size_t offset = len - hw->size;
+
+        if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
+                ioat_unmap(pdev, hw->dst_addr - offset, len,
+                           PCI_DMA_FROMDEVICE, flags, 1);
+
+        if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP))
+                ioat_unmap(pdev, hw->src_addr - offset, len,
+                           PCI_DMA_TODEVICE, flags, 0);
+}
+
+unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
+{
+        unsigned long phys_complete;
+        u64 completion;
+
+        completion = *chan->completion;
+        phys_complete = ioat_chansts_to_addr(completion);
+
+        dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
+                (unsigned long long) phys_complete);
+
+        if (is_ioat_halted(completion)) {
+                u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+                dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
+                        chanerr);
+
+                /* TODO do something to salvage the situation */
+        }
+
+        return phys_complete;
+}
+
+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
+                           unsigned long *phys_complete)
+{
+        *phys_complete = ioat_get_current_completion(chan);
+        if (*phys_complete == chan->last_completion)
+                return false;
+        clear_bit(IOAT_COMPLETION_ACK, &chan->state);
+        mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
+        return true;
+}
+
+static void __cleanup(struct ioat_dma_chan *ioat, unsigned long phys_complete)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        struct list_head *_desc, *n;
+        struct dma_async_tx_descriptor *tx;
+
+        dev_dbg(to_dev(chan), "%s: phys_complete: %lx\n",
+                 __func__, phys_complete);
+        list_for_each_safe(_desc, n, &ioat->used_desc) {
+                struct ioat_desc_sw *desc;
+
+                prefetch(n);
+                desc = list_entry(_desc, typeof(*desc), node);
+                tx = &desc->txd;
+                /*
+                 * Incoming DMA requests may use multiple descriptors,
+                 * due to exceeding xfercap, perhaps. If so, only the
+                 * last one will have a cookie, and require unmapping.
+                 */
+                dump_desc_dbg(ioat, desc);
+                if (tx->cookie) {
+                        chan->completed_cookie = tx->cookie;
+                        tx->cookie = 0;
+                        ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
+                        ioat->active -= desc->hw->tx_cnt;
+                        if (tx->callback) {
+                                tx->callback(tx->callback_param);
+                                tx->callback = NULL;
+                        }
+                }
+
+                if (tx->phys != phys_complete) {
+                        /*
+                         * a completed entry, but not the last, so clean
+                         * up if the client is done with the descriptor
+                         */
+                        if (async_tx_test_ack(tx))
+                                list_move_tail(&desc->node, &ioat->free_desc);
+                } else {
+                        /*
+                         * last used desc. Do not remove, so we can
+                         * append from it.
+                         */
+
+                        /* if nothing else is pending, cancel the
+                         * completion timeout
+                         */
+                        if (n == &ioat->used_desc) {
+                                dev_dbg(to_dev(chan),
+                                        "%s cancel completion timeout\n",
+                                        __func__);
+                                clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+                        }
+
+                        /* TODO check status bits? */
+                        break;
+                }
+        }
+
+        chan->last_completion = phys_complete;
+}
+
+/**
+ * ioat1_cleanup - cleanup up finished descriptors
+ * @chan: ioat channel to be cleaned up
+ *
+ * To prevent lock contention we defer cleanup when the locks are
+ * contended with a terminal timeout that forces cleanup and catches
+ * completion notification errors.
+ */
+static void ioat1_cleanup(struct ioat_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        unsigned long phys_complete;
+
+        prefetch(chan->completion);
+
+        if (!spin_trylock_bh(&chan->cleanup_lock))
+                return;
+
+        if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+
+        if (!spin_trylock_bh(&ioat->desc_lock)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+
+        __cleanup(ioat, phys_complete);
+
+        spin_unlock_bh(&ioat->desc_lock);
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static void ioat1_timer_event(unsigned long data)
+{
+        struct ioat_dma_chan *ioat = (void *) data;
+        struct ioat_chan_common *chan = &ioat->base;
+
+        dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state);
+
+        spin_lock_bh(&chan->cleanup_lock);
+        if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) {
+                struct ioat_desc_sw *desc;
+
+                spin_lock_bh(&ioat->desc_lock);
+
+                /* restart active descriptors */
+                desc = to_ioat_desc(ioat->used_desc.prev);
+                ioat_set_chainaddr(ioat, desc->txd.phys);
+                ioat_start(chan);
+
+                ioat->pending = 0;
+                set_bit(IOAT_COMPLETION_PENDING, &chan->state);
+                mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+                spin_unlock_bh(&ioat->desc_lock);
+        } else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+                unsigned long phys_complete;
+
+                spin_lock_bh(&ioat->desc_lock);
+                /* if we haven't made progress and we have already
+                 * acknowledged a pending completion once, then be more
+                 * forceful with a restart
+                 */
+                if (ioat_cleanup_preamble(chan, &phys_complete))
+                        __cleanup(ioat, phys_complete);
+                else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+                        ioat1_reset_channel(ioat);
+                else {
+                        u64 status = ioat_chansts(chan);
+
+                        /* manually update the last completion address */
+                        if (ioat_chansts_to_addr(status) != 0)
+                                *chan->completion = status;
+
+                        set_bit(IOAT_COMPLETION_ACK, &chan->state);
+                        mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+                }
+                spin_unlock_bh(&ioat->desc_lock);
+        }
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static enum dma_status
+ioat1_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                      dma_cookie_t *done, dma_cookie_t *used)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+
+        if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
+                return DMA_SUCCESS;
+
+        ioat1_cleanup(ioat);
+
+        return ioat_is_complete(c, cookie, done, used);
+}
+
+static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioat_desc_sw *desc;
+        struct ioat_dma_descriptor *hw;
+
+        spin_lock_bh(&ioat->desc_lock);
+
+        desc = ioat1_dma_get_next_descriptor(ioat);
+
+        if (!desc) {
+                dev_err(to_dev(chan),
+                        "Unable to start null desc - get next desc failed\n");
+                spin_unlock_bh(&ioat->desc_lock);
+                return;
+        }
+
+        hw = desc->hw;
+        hw->ctl = 0;
+        hw->ctl_f.null = 1;
+        hw->ctl_f.int_en = 1;
+        hw->ctl_f.compl_write = 1;
+        /* set size to non-zero value (channel returns error when size is 0) */
+        hw->size = NULL_DESC_BUFFER_SIZE;
+        hw->src_addr = 0;
+        hw->dst_addr = 0;
+        async_tx_ack(&desc->txd);
+        hw->next = 0;
+        list_add_tail(&desc->node, &ioat->used_desc);
+        dump_desc_dbg(ioat, desc);
+
+        ioat_set_chainaddr(ioat, desc->txd.phys);
+        ioat_start(chan);
+        spin_unlock_bh(&ioat->desc_lock);
+}
+
+/*
+ * Perform a IOAT transaction to verify the HW works.
+ */
+#define IOAT_TEST_SIZE 2000
+
+static void __devinit ioat_dma_test_callback(void *dma_async_param)
+{
+        struct completion *cmp = dma_async_param;
+
+        complete(cmp);
+}
+
+/**
+ * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
+ * @device: device to be tested
+ */
+int __devinit ioat_dma_self_test(struct ioatdma_device *device)
+{
+        int i;
+        u8 *src;
+        u8 *dest;
+        struct dma_device *dma = &device->common;
+        struct device *dev = &device->pdev->dev;
+        struct dma_chan *dma_chan;
+        struct dma_async_tx_descriptor *tx;
+        dma_addr_t dma_dest, dma_src;
+        dma_cookie_t cookie;
+        int err = 0;
+        struct completion cmp;
+        unsigned long tmo;
+        unsigned long flags;
+
+        src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
+        if (!src)
+                return -ENOMEM;
+        dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
+        if (!dest) {
+                kfree(src);
+                return -ENOMEM;
+        }
+
+        /* Fill in src buffer */
+        for (i = 0; i < IOAT_TEST_SIZE; i++)
+                src[i] = (u8)i;
+
+        /* Start copy, using first DMA channel */
+        dma_chan = container_of(dma->channels.next, struct dma_chan,
+                                device_node);
+        if (dma->device_alloc_chan_resources(dma_chan) < 1) {
+                dev_err(dev, "selftest cannot allocate chan resource\n");
+                err = -ENODEV;
+                goto out;
+        }
+
+        dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
+        dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
+        flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE |
+                DMA_PREP_INTERRUPT;
+        tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
+                                                   IOAT_TEST_SIZE, flags);
+        if (!tx) {
+                dev_err(dev, "Self-test prep failed, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        async_tx_ack(tx);
+        init_completion(&cmp);
+        tx->callback = ioat_dma_test_callback;
+        tx->callback_param = &cmp;
+        cookie = tx->tx_submit(tx);
+        if (cookie < 0) {
+                dev_err(dev, "Self-test setup failed, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        dma->device_issue_pending(dma_chan);
+
+        tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+        if (tmo == 0 ||
+            dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL)
+                                        != DMA_SUCCESS) {
+                dev_err(dev, "Self-test copy timed out, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        if (memcmp(src, dest, IOAT_TEST_SIZE)) {
+                dev_err(dev, "Self-test copy failed compare, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+free_resources:
+        dma->device_free_chan_resources(dma_chan);
+out:
+        kfree(src);
+        kfree(dest);
+        return err;
+}
+
+static char ioat_interrupt_style[32] = "msix";
+module_param_string(ioat_interrupt_style, ioat_interrupt_style,
+                    sizeof(ioat_interrupt_style), 0644);
+MODULE_PARM_DESC(ioat_interrupt_style,
+                 "set ioat interrupt style: msix (default), "
+                 "msix-single-vector, msi, intx)");
+
+/**
+ * ioat_dma_setup_interrupts - setup interrupt handler
+ * @device: ioat device
+ */
+static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
+{
+        struct ioat_chan_common *chan;
+        struct pci_dev *pdev = device->pdev;
+        struct device *dev = &pdev->dev;
+        struct msix_entry *msix;
+        int i, j, msixcnt;
+        int err = -EINVAL;
+        u8 intrctrl = 0;
+
+        if (!strcmp(ioat_interrupt_style, "msix"))
+                goto msix;
+        if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
+                goto msix_single_vector;
+        if (!strcmp(ioat_interrupt_style, "msi"))
+                goto msi;
+        if (!strcmp(ioat_interrupt_style, "intx"))
+                goto intx;
+        dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
+        goto err_no_irq;
+
+msix:
+        /* The number of MSI-X vectors should equal the number of channels */
+        msixcnt = device->common.chancnt;
+        for (i = 0; i < msixcnt; i++)
+                device->msix_entries[i].entry = i;
+
+        err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
+        if (err < 0)
+                goto msi;
+        if (err > 0)
+                goto msix_single_vector;
+
+        for (i = 0; i < msixcnt; i++) {
+                msix = &device->msix_entries[i];
+                chan = ioat_chan_by_index(device, i);
+                err = devm_request_irq(dev, msix->vector,
+                                       ioat_dma_do_interrupt_msix, 0,
+                                       "ioat-msix", chan);
+                if (err) {
+                        for (j = 0; j < i; j++) {
+                                msix = &device->msix_entries[j];
+                                chan = ioat_chan_by_index(device, j);
+                                devm_free_irq(dev, msix->vector, chan);
+                        }
+                        goto msix_single_vector;
+                }
+        }
+        intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
+        goto done;
+
+msix_single_vector:
+        msix = &device->msix_entries[0];
+        msix->entry = 0;
+        err = pci_enable_msix(pdev, device->msix_entries, 1);
+        if (err)
+                goto msi;
+
+        err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
+                               "ioat-msix", device);
+        if (err) {
+                pci_disable_msix(pdev);
+                goto msi;
+        }
+        goto done;
+
+msi:
+        err = pci_enable_msi(pdev);
+        if (err)
+                goto intx;
+
+        err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
+                               "ioat-msi", device);
+        if (err) {
+                pci_disable_msi(pdev);
+                goto intx;
+        }
+        goto done;
+
+intx:
+        err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
+                               IRQF_SHARED, "ioat-intx", device);
+        if (err)
+                goto err_no_irq;
+
+done:
+        if (device->intr_quirk)
+                device->intr_quirk(device);
+        intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
+        writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
+        return 0;
+
+err_no_irq:
+        /* Disable all interrupt generation */
+        writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
+        dev_err(dev, "no usable interrupts\n");
+        return err;
+}
+
+static void ioat_disable_interrupts(struct ioatdma_device *device)
+{
+        /* Disable all interrupt generation */
+        writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
+}
+
+int __devinit ioat_probe(struct ioatdma_device *device)
+{
+        int err = -ENODEV;
+        struct dma_device *dma = &device->common;
+        struct pci_dev *pdev = device->pdev;
+        struct device *dev = &pdev->dev;
+
+        /* DMA coherent memory pool for DMA descriptor allocations */
+        device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
+                                           sizeof(struct ioat_dma_descriptor),
+                                           64, 0);
+        if (!device->dma_pool) {
+                err = -ENOMEM;
+                goto err_dma_pool;
+        }
+
+        device->completion_pool = pci_pool_create("completion_pool", pdev,
+                                                  sizeof(u64), SMP_CACHE_BYTES,
+                                                  SMP_CACHE_BYTES);
+
+        if (!device->completion_pool) {
+                err = -ENOMEM;
+                goto err_completion_pool;
+        }
+
+        device->enumerate_channels(device);
+
+        dma_cap_set(DMA_MEMCPY, dma->cap_mask);
+        dma->dev = &pdev->dev;
+
+        if (!dma->chancnt) {
+                dev_err(dev, "zero channels detected\n");
+                goto err_setup_interrupts;
+        }
+
+        err = ioat_dma_setup_interrupts(device);
+        if (err)
+                goto err_setup_interrupts;
+
+        err = device->self_test(device);
+        if (err)
+                goto err_self_test;
+
+        return 0;
+
+err_self_test:
+        ioat_disable_interrupts(device);
+err_setup_interrupts:
+        pci_pool_destroy(device->completion_pool);
+err_completion_pool:
+        pci_pool_destroy(device->dma_pool);
+err_dma_pool:
+        return err;
+}
+
+int __devinit ioat_register(struct ioatdma_device *device)
+{
+        int err = dma_async_device_register(&device->common);
+
+        if (err) {
+                ioat_disable_interrupts(device);
+                pci_pool_destroy(device->completion_pool);
+                pci_pool_destroy(device->dma_pool);
+        }
+
+        return err;
+}
+
+/* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
+static void ioat1_intr_quirk(struct ioatdma_device *device)
+{
+        struct pci_dev *pdev = device->pdev;
+        u32 dmactrl;
+
+        pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
+        if (pdev->msi_enabled)
+                dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
+        else
+                dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
+        pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
+}
+
+static ssize_t ring_size_show(struct dma_chan *c, char *page)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+
+        return sprintf(page, "%d\n", ioat->desccount);
+}
+static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
+
+static ssize_t ring_active_show(struct dma_chan *c, char *page)
+{
+        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+
+        return sprintf(page, "%d\n", ioat->active);
+}
+static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
+
+static ssize_t cap_show(struct dma_chan *c, char *page)
+{
+        struct dma_device *dma = c->device;
+
+        return sprintf(page, "copy%s%s%s%s%s%s\n",
+                       dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "",
+                       dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "",
+                       dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "",
+                       dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "",
+                       dma_has_cap(DMA_MEMSET, dma->cap_mask)  ? " fill" : "",
+                       dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : "");
+
+}
+struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap);
+
+static ssize_t version_show(struct dma_chan *c, char *page)
+{
+        struct dma_device *dma = c->device;
+        struct ioatdma_device *device = to_ioatdma_device(dma);
+
+        return sprintf(page, "%d.%d\n",
+                       device->version >> 4, device->version & 0xf);
+}
+struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version);
+
+static struct attribute *ioat1_attrs[] = {
+        &ring_size_attr.attr,
+        &ring_active_attr.attr,
+        &ioat_cap_attr.attr,
+        &ioat_version_attr.attr,
+        NULL,
+};
+
+static ssize_t
+ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
+{
+        struct ioat_sysfs_entry *entry;
+        struct ioat_chan_common *chan;
+
+        entry = container_of(attr, struct ioat_sysfs_entry, attr);
+        chan = container_of(kobj, struct ioat_chan_common, kobj);
+
+        if (!entry->show)
+                return -EIO;
+        return entry->show(&chan->common, page);
+}
+
+struct sysfs_ops ioat_sysfs_ops = {
+        .show   = ioat_attr_show,
+};
+
+static struct kobj_type ioat1_ktype = {
+        .sysfs_ops = &ioat_sysfs_ops,
+        .default_attrs = ioat1_attrs,
+};
+
+void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type)
+{
+        struct dma_device *dma = &device->common;
+        struct dma_chan *c;
+
+        list_for_each_entry(c, &dma->channels, device_node) {
+                struct ioat_chan_common *chan = to_chan_common(c);
+                struct kobject *parent = &c->dev->device.kobj;
+                int err;
+
+                err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata");
+                if (err) {
+                        dev_warn(to_dev(chan),
+                                 "sysfs init error (%d), continuing...\n", err);
+                        kobject_put(&chan->kobj);
+                        set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state);
+                }
+        }
+}
+
+void ioat_kobject_del(struct ioatdma_device *device)
+{
+        struct dma_device *dma = &device->common;
+        struct dma_chan *c;
+
+        list_for_each_entry(c, &dma->channels, device_node) {
+                struct ioat_chan_common *chan = to_chan_common(c);
+
+                if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) {
+                        kobject_del(&chan->kobj);
+                        kobject_put(&chan->kobj);
+                }
+        }
+}
+
+int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca)
+{
+        struct pci_dev *pdev = device->pdev;
+        struct dma_device *dma;
+        int err;
+
+        device->intr_quirk = ioat1_intr_quirk;
+        device->enumerate_channels = ioat1_enumerate_channels;
+        device->self_test = ioat_dma_self_test;
+        dma = &device->common;
+        dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
+        dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
+        dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources;
+        dma->device_free_chan_resources = ioat1_dma_free_chan_resources;
+        dma->device_is_tx_complete = ioat1_dma_is_complete;
+
+        err = ioat_probe(device);
+        if (err)
+                return err;
+        ioat_set_tcp_copy_break(4096);
+        err = ioat_register(device);
+        if (err)
+                return err;
+        ioat_kobject_add(device, &ioat1_ktype);
+
+        if (dca)
+                device->dca = ioat_dca_init(pdev, device->reg_base);
+
+        return err;
+}
+
+void __devexit ioat_dma_remove(struct ioatdma_device *device)
+{
+        struct dma_device *dma = &device->common;
+
+        ioat_disable_interrupts(device);
+
+        ioat_kobject_del(device);
+
+        dma_async_device_unregister(dma);
+
+        pci_pool_destroy(device->dma_pool);
+        pci_pool_destroy(device->completion_pool);
+
+        INIT_LIST_HEAD(&dma->channels);
+}
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
new file mode 100644
index 000000000000..c14fdfeb7f33
--- /dev/null
+++ b/drivers/dma/ioat/dma.h
@@ -0,0 +1,337 @@
+/*
+ * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called COPYING.
+ */
+#ifndef IOATDMA_H
+#define IOATDMA_H
+
+#include <linux/dmaengine.h>
+#include "hw.h"
+#include "registers.h"
+#include <linux/init.h>
+#include <linux/dmapool.h>
+#include <linux/cache.h>
+#include <linux/pci_ids.h>
+#include <net/tcp.h>
+
+#define IOAT_DMA_VERSION  "4.00"
+
+#define IOAT_LOW_COMPLETION_MASK        0xffffffc0
+#define IOAT_DMA_DCA_ANY_CPU            ~0
+
+#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
+#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
+#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, txd)
+#define to_dev(ioat_chan) (&(ioat_chan)->device->pdev->dev)
+
+#define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80)
+
+/*
+ * workaround for IOAT ver.3.0 null descriptor issue
+ * (channel returns error when size is 0)
+ */
+#define NULL_DESC_BUFFER_SIZE 1
+
+/**
+ * struct ioatdma_device - internal representation of a IOAT device
+ * @pdev: PCI-Express device
+ * @reg_base: MMIO register space base address
+ * @dma_pool: for allocating DMA descriptors
+ * @common: embedded struct dma_device
+ * @version: version of ioatdma device
+ * @msix_entries: irq handlers
+ * @idx: per channel data
+ * @dca: direct cache access context
+ * @intr_quirk: interrupt setup quirk (for ioat_v1 devices)
+ * @enumerate_channels: hw version specific channel enumeration
+ * @cleanup_tasklet: select between the v2 and v3 cleanup routines
+ * @timer_fn: select between the v2 and v3 timer watchdog routines
+ * @self_test: hardware version specific self test for each supported op type
+ *
+ * Note: the v3 cleanup routine supports raid operations
+ */
+struct ioatdma_device {
+        struct pci_dev *pdev;
+        void __iomem *reg_base;
+        struct pci_pool *dma_pool;
+        struct pci_pool *completion_pool;
+        struct dma_device common;
+        u8 version;
+        struct msix_entry msix_entries[4];
+        struct ioat_chan_common *idx[4];
+        struct dca_provider *dca;
+        void (*intr_quirk)(struct ioatdma_device *device);
+        int (*enumerate_channels)(struct ioatdma_device *device);
+        void (*cleanup_tasklet)(unsigned long data);
+        void (*timer_fn)(unsigned long data);
+        int (*self_test)(struct ioatdma_device *device);
+};
+
+struct ioat_chan_common {
+        struct dma_chan common;
+        void __iomem *reg_base;
+        unsigned long last_completion;
+        spinlock_t cleanup_lock;
+        dma_cookie_t completed_cookie;
+        unsigned long state;
+        #define IOAT_COMPLETION_PENDING 0
+        #define IOAT_COMPLETION_ACK 1
+        #define IOAT_RESET_PENDING 2
+        #define IOAT_KOBJ_INIT_FAIL 3
+        struct timer_list timer;
+        #define COMPLETION_TIMEOUT msecs_to_jiffies(100)
+        #define IDLE_TIMEOUT msecs_to_jiffies(2000)
+        #define RESET_DELAY msecs_to_jiffies(100)
+        struct ioatdma_device *device;
+        dma_addr_t completion_dma;
+        u64 *completion;
+        struct tasklet_struct cleanup_task;
+        struct kobject kobj;
+};
+
+struct ioat_sysfs_entry {
+        struct attribute attr;
+        ssize_t (*show)(struct dma_chan *, char *);
+};
+
+/**
+ * struct ioat_dma_chan - internal representation of a DMA channel
+ */
+struct ioat_dma_chan {
+        struct ioat_chan_common base;
+
+        size_t xfercap; /* XFERCAP register value expanded out */
+
+        spinlock_t desc_lock;
+        struct list_head free_desc;
+        struct list_head used_desc;
+
+        int pending;
+        u16 desccount;
+        u16 active;
+};
+
+static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c)
+{
+        return container_of(c, struct ioat_chan_common, common);
+}
+
+static inline struct ioat_dma_chan *to_ioat_chan(struct dma_chan *c)
+{
+        struct ioat_chan_common *chan = to_chan_common(c);
+
+        return container_of(chan, struct ioat_dma_chan, base);
+}
+
+/**
+ * ioat_is_complete - poll the status of an ioat transaction
+ * @c: channel handle
+ * @cookie: transaction identifier
+ * @done: if set, updated with last completed transaction
+ * @used: if set, updated with last used transaction
+ */
+static inline enum dma_status
+ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                 dma_cookie_t *done, dma_cookie_t *used)
+{
+        struct ioat_chan_common *chan = to_chan_common(c);
+        dma_cookie_t last_used;
+        dma_cookie_t last_complete;
+
+        last_used = c->cookie;
+        last_complete = chan->completed_cookie;
+
+        if (done)
+                *done = last_complete;
+        if (used)
+                *used = last_used;
+
+        return dma_async_is_complete(cookie, last_complete, last_used);
+}
+
+/* wrapper around hardware descriptor format + additional software fields */
+
+/**
+ * struct ioat_desc_sw - wrapper around hardware descriptor
+ * @hw: hardware DMA descriptor (for memcpy)
+ * @node: this descriptor will either be on the free list,
+ *     or attached to a transaction list (tx_list)
+ * @txd: the generic software descriptor for all engines
+ * @id: identifier for debug
+ */
+struct ioat_desc_sw {
+        struct ioat_dma_descriptor *hw;
+        struct list_head node;
+        size_t len;
+        struct list_head tx_list;
+        struct dma_async_tx_descriptor txd;
+        #ifdef DEBUG
+        int id;
+        #endif
+};
+
+#ifdef DEBUG
+#define set_desc_id(desc, i) ((desc)->id = (i))
+#define desc_id(desc) ((desc)->id)
+#else
+#define set_desc_id(desc, i)
+#define desc_id(desc) (0)
+#endif
+
+static inline void
+__dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw,
+                struct dma_async_tx_descriptor *tx, int id)
+{
+        struct device *dev = to_dev(chan);
+
+        dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x"
+                " ctl: %#x (op: %d int_en: %d compl: %d)\n", id,
+                (unsigned long long) tx->phys,
+                (unsigned long long) hw->next, tx->cookie, tx->flags,
+                hw->ctl, hw->ctl_f.op, hw->ctl_f.int_en, hw->ctl_f.compl_write);
+}
+
+#define dump_desc_dbg(c, d) \
+        ({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; })
+
+static inline void ioat_set_tcp_copy_break(unsigned long copybreak)
+{
+        #ifdef CONFIG_NET_DMA
+        sysctl_tcp_dma_copybreak = copybreak;
+        #endif
+}
+
+static inline struct ioat_chan_common *
+ioat_chan_by_index(struct ioatdma_device *device, int index)
+{
+        return device->idx[index];
+}
+
+static inline u64 ioat_chansts(struct ioat_chan_common *chan)
+{
+        u8 ver = chan->device->version;
+        u64 status;
+        u32 status_lo;
+
+        /* We need to read the low address first as this causes the
+         * chipset to latch the upper bits for the subsequent read
+         */
+        status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver));
+        status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver));
+        status <<= 32;
+        status |= status_lo;
+
+        return status;
+}
+
+static inline void ioat_start(struct ioat_chan_common *chan)
+{
+        u8 ver = chan->device->version;
+
+        writeb(IOAT_CHANCMD_START, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
+}
+
+static inline u64 ioat_chansts_to_addr(u64 status)
+{
+        return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
+}
+
+static inline u32 ioat_chanerr(struct ioat_chan_common *chan)
+{
+        return readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+}
+
+static inline void ioat_suspend(struct ioat_chan_common *chan)
+{
+        u8 ver = chan->device->version;
+
+        writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
+}
+
+static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+
+        writel(addr & 0x00000000FFFFFFFF,
+               chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
+        writel(addr >> 32,
+               chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
+}
+
+static inline bool is_ioat_active(unsigned long status)
+{
+        return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
+}
+
+static inline bool is_ioat_idle(unsigned long status)
+{
+        return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_DONE);
+}
+
+static inline bool is_ioat_halted(unsigned long status)
+{
+        return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED);
+}
+
+static inline bool is_ioat_suspended(unsigned long status)
+{
+        return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED);
+}
+
+/* channel was fatally programmed */
+static inline bool is_ioat_bug(unsigned long err)
+{
+        return !!(err & (IOAT_CHANERR_SRC_ADDR_ERR|IOAT_CHANERR_DEST_ADDR_ERR|
+                         IOAT_CHANERR_NEXT_ADDR_ERR|IOAT_CHANERR_CONTROL_ERR|
+                         IOAT_CHANERR_LENGTH_ERR));
+}
+
+static inline void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
+                              int direction, enum dma_ctrl_flags flags, bool dst)
+{
+        if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) ||
+            (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE)))
+                pci_unmap_single(pdev, addr, len, direction);
+        else
+                pci_unmap_page(pdev, addr, len, direction);
+}
+
+int __devinit ioat_probe(struct ioatdma_device *device);
+int __devinit ioat_register(struct ioatdma_device *device);
+int __devinit ioat1_dma_probe(struct ioatdma_device *dev, int dca);
+int __devinit ioat_dma_self_test(struct ioatdma_device *device);
+void __devexit ioat_dma_remove(struct ioatdma_device *device);
+struct dca_provider * __devinit ioat_dca_init(struct pci_dev *pdev,
+                                              void __iomem *iobase);
+unsigned long ioat_get_current_completion(struct ioat_chan_common *chan);
+void ioat_init_channel(struct ioatdma_device *device,
+                       struct ioat_chan_common *chan, int idx,
+                       void (*timer_fn)(unsigned long),
+                       void (*tasklet)(unsigned long),
+                       unsigned long ioat);
+void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
+                    size_t len, struct ioat_dma_descriptor *hw);
+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
+                           unsigned long *phys_complete);
+void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type);
+void ioat_kobject_del(struct ioatdma_device *device);
+extern struct sysfs_ops ioat_sysfs_ops;
+extern struct ioat_sysfs_entry ioat_version_attr;
+extern struct ioat_sysfs_entry ioat_cap_attr;
+#endif /* IOATDMA_H */
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
new file mode 100644
index 000000000000..96ffab7d37a7
--- /dev/null
+++ b/drivers/dma/ioat/dma_v2.c
@@ -0,0 +1,871 @@
+/*
+ * Intel I/OAT DMA Linux driver
+ * Copyright(c) 2004 - 2009 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ */
+
+/*
+ * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
+ * does asynchronous data movement and checksumming operations.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/dmaengine.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/workqueue.h>
+#include <linux/i7300_idle.h>
+#include "dma.h"
+#include "dma_v2.h"
+#include "registers.h"
+#include "hw.h"
+
+int ioat_ring_alloc_order = 8;
+module_param(ioat_ring_alloc_order, int, 0644);
+MODULE_PARM_DESC(ioat_ring_alloc_order,
+                 "ioat2+: allocate 2^n descriptors per channel"
+                 " (default: 8 max: 16)");
+static int ioat_ring_max_alloc_order = IOAT_MAX_ORDER;
+module_param(ioat_ring_max_alloc_order, int, 0644);
+MODULE_PARM_DESC(ioat_ring_max_alloc_order,
+                 "ioat2+: upper limit for ring size (default: 16)");
+
+void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
+{
+        void * __iomem reg_base = ioat->base.reg_base;
+
+        ioat->pending = 0;
+        ioat->dmacount += ioat2_ring_pending(ioat);
+        ioat->issued = ioat->head;
+        /* make descriptor updates globally visible before notifying channel */
+        wmb();
+        writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
+        dev_dbg(to_dev(&ioat->base),
+                "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
+                __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
+}
+
+void ioat2_issue_pending(struct dma_chan *chan)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(chan);
+
+        spin_lock_bh(&ioat->ring_lock);
+        if (ioat->pending == 1)
+                __ioat2_issue_pending(ioat);
+        spin_unlock_bh(&ioat->ring_lock);
+}
+
+/**
+ * ioat2_update_pending - log pending descriptors
+ * @ioat: ioat2+ channel
+ *
+ * set pending to '1' unless pending is already set to '2', pending == 2
+ * indicates that submission is temporarily blocked due to an in-flight
+ * reset.  If we are already above the ioat_pending_level threshold then
+ * just issue pending.
+ *
+ * called with ring_lock held
+ */
+static void ioat2_update_pending(struct ioat2_dma_chan *ioat)
+{
+        if (unlikely(ioat->pending == 2))
+                return;
+        else if (ioat2_ring_pending(ioat) > ioat_pending_level)
+                __ioat2_issue_pending(ioat);
+        else
+                ioat->pending = 1;
+}
+
+static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_ring_ent *desc;
+        struct ioat_dma_descriptor *hw;
+        int idx;
+
+        if (ioat2_ring_space(ioat) < 1) {
+                dev_err(to_dev(&ioat->base),
+                        "Unable to start null desc - ring full\n");
+                return;
+        }
+
+        dev_dbg(to_dev(&ioat->base), "%s: head: %#x tail: %#x issued: %#x\n",
+                __func__, ioat->head, ioat->tail, ioat->issued);
+        idx = ioat2_desc_alloc(ioat, 1);
+        desc = ioat2_get_ring_ent(ioat, idx);
+
+        hw = desc->hw;
+        hw->ctl = 0;
+        hw->ctl_f.null = 1;
+        hw->ctl_f.int_en = 1;
+        hw->ctl_f.compl_write = 1;
+        /* set size to non-zero value (channel returns error when size is 0) */
+        hw->size = NULL_DESC_BUFFER_SIZE;
+        hw->src_addr = 0;
+        hw->dst_addr = 0;
+        async_tx_ack(&desc->txd);
+        ioat2_set_chainaddr(ioat, desc->txd.phys);
+        dump_desc_dbg(ioat, desc);
+        __ioat2_issue_pending(ioat);
+}
+
+static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
+{
+        spin_lock_bh(&ioat->ring_lock);
+        __ioat2_start_null_desc(ioat);
+        spin_unlock_bh(&ioat->ring_lock);
+}
+
+static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        struct dma_async_tx_descriptor *tx;
+        struct ioat_ring_ent *desc;
+        bool seen_current = false;
+        u16 active;
+        int i;
+
+        dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
+                __func__, ioat->head, ioat->tail, ioat->issued);
+
+        active = ioat2_ring_active(ioat);
+        for (i = 0; i < active && !seen_current; i++) {
+                prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
+                desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
+                tx = &desc->txd;
+                dump_desc_dbg(ioat, desc);
+                if (tx->cookie) {
+                        ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
+                        chan->completed_cookie = tx->cookie;
+                        tx->cookie = 0;
+                        if (tx->callback) {
+                                tx->callback(tx->callback_param);
+                                tx->callback = NULL;
+                        }
+                }
+
+                if (tx->phys == phys_complete)
+                        seen_current = true;
+        }
+        ioat->tail += i;
+        BUG_ON(!seen_current); /* no active descs have written a completion? */
+
+        chan->last_completion = phys_complete;
+        if (ioat->head == ioat->tail) {
+                dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
+                        __func__);
+                clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+                mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
+        }
+}
+
+/**
+ * ioat2_cleanup - clean finished descriptors (advance tail pointer)
+ * @chan: ioat channel to be cleaned up
+ */
+static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        unsigned long phys_complete;
+
+        prefetch(chan->completion);
+
+        if (!spin_trylock_bh(&chan->cleanup_lock))
+                return;
+
+        if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+
+        if (!spin_trylock_bh(&ioat->ring_lock)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+
+        __cleanup(ioat, phys_complete);
+
+        spin_unlock_bh(&ioat->ring_lock);
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+void ioat2_cleanup_tasklet(unsigned long data)
+{
+        struct ioat2_dma_chan *ioat = (void *) data;
+
+        ioat2_cleanup(ioat);
+        writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
+}
+
+void __ioat2_restart_chan(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+
+        /* set the tail to be re-issued */
+        ioat->issued = ioat->tail;
+        ioat->dmacount = 0;
+        set_bit(IOAT_COMPLETION_PENDING, &chan->state);
+        mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
+        dev_dbg(to_dev(chan),
+                "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
+                __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
+
+        if (ioat2_ring_pending(ioat)) {
+                struct ioat_ring_ent *desc;
+
+                desc = ioat2_get_ring_ent(ioat, ioat->tail);
+                ioat2_set_chainaddr(ioat, desc->txd.phys);
+                __ioat2_issue_pending(ioat);
+        } else
+                __ioat2_start_null_desc(ioat);
+}
+
+static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        unsigned long phys_complete;
+        u32 status;
+
+        status = ioat_chansts(chan);
+        if (is_ioat_active(status) || is_ioat_idle(status))
+                ioat_suspend(chan);
+        while (is_ioat_active(status) || is_ioat_idle(status)) {
+                status = ioat_chansts(chan);
+                cpu_relax();
+        }
+
+        if (ioat_cleanup_preamble(chan, &phys_complete))
+                __cleanup(ioat, phys_complete);
+
+        __ioat2_restart_chan(ioat);
+}
+
+void ioat2_timer_event(unsigned long data)
+{
+        struct ioat2_dma_chan *ioat = (void *) data;
+        struct ioat_chan_common *chan = &ioat->base;
+
+        spin_lock_bh(&chan->cleanup_lock);
+        if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+                unsigned long phys_complete;
+                u64 status;
+
+                spin_lock_bh(&ioat->ring_lock);
+                status = ioat_chansts(chan);
+
+                /* when halted due to errors check for channel
+                 * programming errors before advancing the completion state
+                 */
+                if (is_ioat_halted(status)) {
+                        u32 chanerr;
+
+                        chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+                        BUG_ON(is_ioat_bug(chanerr));
+                }
+
+                /* if we haven't made progress and we have already
+                 * acknowledged a pending completion once, then be more
+                 * forceful with a restart
+                 */
+                if (ioat_cleanup_preamble(chan, &phys_complete))
+                        __cleanup(ioat, phys_complete);
+                else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+                        ioat2_restart_channel(ioat);
+                else {
+                        set_bit(IOAT_COMPLETION_ACK, &chan->state);
+                        mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+                }
+                spin_unlock_bh(&ioat->ring_lock);
+        } else {
+                u16 active;
+
+                /* if the ring is idle, empty, and oversized try to step
+                 * down the size
+                 */
+                spin_lock_bh(&ioat->ring_lock);
+                active = ioat2_ring_active(ioat);
+                if (active == 0 && ioat->alloc_order > ioat_get_alloc_order())
+                        reshape_ring(ioat, ioat->alloc_order-1);
+                spin_unlock_bh(&ioat->ring_lock);
+
+                /* keep shrinking until we get back to our minimum
+                 * default size
+                 */
+                if (ioat->alloc_order > ioat_get_alloc_order())
+                        mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
+        }
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+/**
+ * ioat2_enumerate_channels - find and initialize the device's channels
+ * @device: the device to be enumerated
+ */
+int ioat2_enumerate_channels(struct ioatdma_device *device)
+{
+        struct ioat2_dma_chan *ioat;
+        struct device *dev = &device->pdev->dev;
+        struct dma_device *dma = &device->common;
+        u8 xfercap_log;
+        int i;
+
+        INIT_LIST_HEAD(&dma->channels);
+        dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
+        dma->chancnt &= 0x1f; /* bits [4:0] valid */
+        if (dma->chancnt > ARRAY_SIZE(device->idx)) {
+                dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
+                         dma->chancnt, ARRAY_SIZE(device->idx));
+                dma->chancnt = ARRAY_SIZE(device->idx);
+        }
+        xfercap_log = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
+        xfercap_log &= 0x1f; /* bits [4:0] valid */
+        if (xfercap_log == 0)
+                return 0;
+        dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
+
+        /* FIXME which i/oat version is i7300? */
+#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
+        if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
+                dma->chancnt--;
+#endif
+        for (i = 0; i < dma->chancnt; i++) {
+                ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
+                if (!ioat)
+                        break;
+
+                ioat_init_channel(device, &ioat->base, i,
+                                  device->timer_fn,
+                                  device->cleanup_tasklet,
+                                  (unsigned long) ioat);
+                ioat->xfercap_log = xfercap_log;
+                spin_lock_init(&ioat->ring_lock);
+        }
+        dma->chancnt = i;
+        return i;
+}
+
+static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
+{
+        struct dma_chan *c = tx->chan;
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_chan_common *chan = &ioat->base;
+        dma_cookie_t cookie = c->cookie;
+
+        cookie++;
+        if (cookie < 0)
+                cookie = 1;
+        tx->cookie = cookie;
+        c->cookie = cookie;
+        dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
+
+        if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
+                mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+        ioat2_update_pending(ioat);
+        spin_unlock_bh(&ioat->ring_lock);
+
+        return cookie;
+}
+
+static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
+{
+        struct ioat_dma_descriptor *hw;
+        struct ioat_ring_ent *desc;
+        struct ioatdma_device *dma;
+        dma_addr_t phys;
+
+        dma = to_ioatdma_device(chan->device);
+        hw = pci_pool_alloc(dma->dma_pool, flags, &phys);
+        if (!hw)
+                return NULL;
+        memset(hw, 0, sizeof(*hw));
+
+        desc = kmem_cache_alloc(ioat2_cache, flags);
+        if (!desc) {
+                pci_pool_free(dma->dma_pool, hw, phys);
+                return NULL;
+        }
+        memset(desc, 0, sizeof(*desc));
+
+        dma_async_tx_descriptor_init(&desc->txd, chan);
+        desc->txd.tx_submit = ioat2_tx_submit_unlock;
+        desc->hw = hw;
+        desc->txd.phys = phys;
+        return desc;
+}
+
+static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
+{
+        struct ioatdma_device *dma;
+
+        dma = to_ioatdma_device(chan->device);
+        pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys);
+        kmem_cache_free(ioat2_cache, desc);
+}
+
+static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
+{
+        struct ioat_ring_ent **ring;
+        int descs = 1 << order;
+        int i;
+
+        if (order > ioat_get_max_alloc_order())
+                return NULL;
+
+        /* allocate the array to hold the software ring */
+        ring = kcalloc(descs, sizeof(*ring), flags);
+        if (!ring)
+                return NULL;
+        for (i = 0; i < descs; i++) {
+                ring[i] = ioat2_alloc_ring_ent(c, flags);
+                if (!ring[i]) {
+                        while (i--)
+                                ioat2_free_ring_ent(ring[i], c);
+                        kfree(ring);
+                        return NULL;
+                }
+                set_desc_id(ring[i], i);
+        }
+
+        /* link descs */
+        for (i = 0; i < descs-1; i++) {
+                struct ioat_ring_ent *next = ring[i+1];
+                struct ioat_dma_descriptor *hw = ring[i]->hw;
+
+                hw->next = next->txd.phys;
+        }
+        ring[i]->hw->next = ring[0]->txd.phys;
+
+        return ring;
+}
+
+/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
+ * @chan: channel to be initialized
+ */
+int ioat2_alloc_chan_resources(struct dma_chan *c)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioat_ring_ent **ring;
+        u32 chanerr;
+        int order;
+
+        /* have we already been set up? */
+        if (ioat->ring)
+                return 1 << ioat->alloc_order;
+
+        /* Setup register to interrupt and write completion status on error */
+        writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET);
+
+        chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+        if (chanerr) {
+                dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
+                writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
+        }
+
+        /* allocate a completion writeback area */
+        /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
+        chan->completion = pci_pool_alloc(chan->device->completion_pool,
+                                          GFP_KERNEL, &chan->completion_dma);
+        if (!chan->completion)
+                return -ENOMEM;
+
+        memset(chan->completion, 0, sizeof(*chan->completion));
+        writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
+               chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
+        writel(((u64) chan->completion_dma) >> 32,
+               chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
+
+        order = ioat_get_alloc_order();
+        ring = ioat2_alloc_ring(c, order, GFP_KERNEL);
+        if (!ring)
+                return -ENOMEM;
+
+        spin_lock_bh(&ioat->ring_lock);
+        ioat->ring = ring;
+        ioat->head = 0;
+        ioat->issued = 0;
+        ioat->tail = 0;
+        ioat->pending = 0;
+        ioat->alloc_order = order;
+        spin_unlock_bh(&ioat->ring_lock);
+
+        tasklet_enable(&chan->cleanup_task);
+        ioat2_start_null_desc(ioat);
+
+        return 1 << ioat->alloc_order;
+}
+
+bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
+{
+        /* reshape differs from normal ring allocation in that we want
+         * to allocate a new software ring while only
+         * extending/truncating the hardware ring
+         */
+        struct ioat_chan_common *chan = &ioat->base;
+        struct dma_chan *c = &chan->common;
+        const u16 curr_size = ioat2_ring_mask(ioat) + 1;
+        const u16 active = ioat2_ring_active(ioat);
+        const u16 new_size = 1 << order;
+        struct ioat_ring_ent **ring;
+        u16 i;
+
+        if (order > ioat_get_max_alloc_order())
+                return false;
+
+        /* double check that we have at least 1 free descriptor */
+        if (active == curr_size)
+                return false;
+
+        /* when shrinking, verify that we can hold the current active
+         * set in the new ring
+         */
+        if (active >= new_size)
+                return false;
+
+        /* allocate the array to hold the software ring */
+        ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT);
+        if (!ring)
+                return false;
+
+        /* allocate/trim descriptors as needed */
+        if (new_size > curr_size) {
+                /* copy current descriptors to the new ring */
+                for (i = 0; i < curr_size; i++) {
+                        u16 curr_idx = (ioat->tail+i) & (curr_size-1);
+                        u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+                        ring[new_idx] = ioat->ring[curr_idx];
+                        set_desc_id(ring[new_idx], new_idx);
+                }
+
+                /* add new descriptors to the ring */
+                for (i = curr_size; i < new_size; i++) {
+                        u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+                        ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT);
+                        if (!ring[new_idx]) {
+                                while (i--) {
+                                        u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+                                        ioat2_free_ring_ent(ring[new_idx], c);
+                                }
+                                kfree(ring);
+                                return false;
+                        }
+                        set_desc_id(ring[new_idx], new_idx);
+                }
+
+                /* hw link new descriptors */
+                for (i = curr_size-1; i < new_size; i++) {
+                        u16 new_idx = (ioat->tail+i) & (new_size-1);
+                        struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)];
+                        struct ioat_dma_descriptor *hw = ring[new_idx]->hw;
+
+                        hw->next = next->txd.phys;
+                }
+        } else {
+                struct ioat_dma_descriptor *hw;
+                struct ioat_ring_ent *next;
+
+                /* copy current descriptors to the new ring, dropping the
+                 * removed descriptors
+                 */
+                for (i = 0; i < new_size; i++) {
+                        u16 curr_idx = (ioat->tail+i) & (curr_size-1);
+                        u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+                        ring[new_idx] = ioat->ring[curr_idx];
+                        set_desc_id(ring[new_idx], new_idx);
+                }
+
+                /* free deleted descriptors */
+                for (i = new_size; i < curr_size; i++) {
+                        struct ioat_ring_ent *ent;
+
+                        ent = ioat2_get_ring_ent(ioat, ioat->tail+i);
+                        ioat2_free_ring_ent(ent, c);
+                }
+
+                /* fix up hardware ring */
+                hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->hw;
+                next = ring[(ioat->tail+new_size) & (new_size-1)];
+                hw->next = next->txd.phys;
+        }
+
+        dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
+                __func__, new_size);
+
+        kfree(ioat->ring);
+        ioat->ring = ring;
+        ioat->alloc_order = order;
+
+        return true;
+}
+
+/**
+ * ioat2_alloc_and_lock - common descriptor alloc boilerplate for ioat2,3 ops
+ * @idx: gets starting descriptor index on successful allocation
+ * @ioat: ioat2,3 channel (ring) to operate on
+ * @num_descs: allocation length
+ */
+int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+
+        spin_lock_bh(&ioat->ring_lock);
+        /* never allow the last descriptor to be consumed, we need at
+         * least one free at all times to allow for on-the-fly ring
+         * resizing.
+         */
+        while (unlikely(ioat2_ring_space(ioat) <= num_descs)) {
+                if (reshape_ring(ioat, ioat->alloc_order + 1) &&
+                    ioat2_ring_space(ioat) > num_descs)
+                                break;
+
+                if (printk_ratelimit())
+                        dev_dbg(to_dev(chan),
+                                "%s: ring full! num_descs: %d (%x:%x:%x)\n",
+                                __func__, num_descs, ioat->head, ioat->tail,
+                                ioat->issued);
+                spin_unlock_bh(&ioat->ring_lock);
+
+                /* progress reclaim in the allocation failure case we
+                 * may be called under bh_disabled so we need to trigger
+                 * the timer event directly
+                 */
+                spin_lock_bh(&chan->cleanup_lock);
+                if (jiffies > chan->timer.expires &&
+                    timer_pending(&chan->timer)) {
+                        struct ioatdma_device *device = chan->device;
+
+                        mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+                        spin_unlock_bh(&chan->cleanup_lock);
+                        device->timer_fn((unsigned long) ioat);
+                } else
+                        spin_unlock_bh(&chan->cleanup_lock);
+                return -ENOMEM;
+        }
+
+        dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n",
+                __func__, num_descs, ioat->head, ioat->tail, ioat->issued);
+
+        *idx = ioat2_desc_alloc(ioat, num_descs);
+        return 0;  /* with ioat->ring_lock held */
+}
+
+struct dma_async_tx_descriptor *
+ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
+                           dma_addr_t dma_src, size_t len, unsigned long flags)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_dma_descriptor *hw;
+        struct ioat_ring_ent *desc;
+        dma_addr_t dst = dma_dest;
+        dma_addr_t src = dma_src;
+        size_t total_len = len;
+        int num_descs;
+        u16 idx;
+        int i;
+
+        num_descs = ioat2_xferlen_to_descs(ioat, len);
+        if (likely(num_descs) &&
+            ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0)
+                /* pass */;
+        else
+                return NULL;
+        i = 0;
+        do {
+                size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log);
+
+                desc = ioat2_get_ring_ent(ioat, idx + i);
+                hw = desc->hw;
+
+                hw->size = copy;
+                hw->ctl = 0;
+                hw->src_addr = src;
+                hw->dst_addr = dst;
+
+                len -= copy;
+                dst += copy;
+                src += copy;
+                dump_desc_dbg(ioat, desc);
+        } while (++i < num_descs);
+
+        desc->txd.flags = flags;
+        desc->len = total_len;
+        hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+        hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+        hw->ctl_f.compl_write = 1;
+        dump_desc_dbg(ioat, desc);
+        /* we leave the channel locked to ensure in order submission */
+
+        return &desc->txd;
+}
+
+/**
+ * ioat2_free_chan_resources - release all the descriptors
+ * @chan: the channel to be cleaned
+ */
+void ioat2_free_chan_resources(struct dma_chan *c)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioatdma_device *device = chan->device;
+        struct ioat_ring_ent *desc;
+        const u16 total_descs = 1 << ioat->alloc_order;
+        int descs;
+        int i;
+
+        /* Before freeing channel resources first check
+         * if they have been previously allocated for this channel.
+         */
+        if (!ioat->ring)
+                return;
+
+        tasklet_disable(&chan->cleanup_task);
+        del_timer_sync(&chan->timer);
+        device->cleanup_tasklet((unsigned long) ioat);
+
+        /* Delay 100ms after reset to allow internal DMA logic to quiesce
+         * before removing DMA descriptor resources.
+         */
+        writeb(IOAT_CHANCMD_RESET,
+               chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
+        mdelay(100);
+
+        spin_lock_bh(&ioat->ring_lock);
+        descs = ioat2_ring_space(ioat);
+        dev_dbg(to_dev(chan), "freeing %d idle descriptors\n", descs);
+        for (i = 0; i < descs; i++) {
+                desc = ioat2_get_ring_ent(ioat, ioat->head + i);
+                ioat2_free_ring_ent(desc, c);
+        }
+
+        if (descs < total_descs)
+                dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
+                        total_descs - descs);
+
+        for (i = 0; i < total_descs - descs; i++) {
+                desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
+                dump_desc_dbg(ioat, desc);
+                ioat2_free_ring_ent(desc, c);
+        }
+
+        kfree(ioat->ring);
+        ioat->ring = NULL;
+        ioat->alloc_order = 0;
+        pci_pool_free(device->completion_pool, chan->completion,
+                      chan->completion_dma);
+        spin_unlock_bh(&ioat->ring_lock);
+
+        chan->last_completion = 0;
+        chan->completion_dma = 0;
+        ioat->pending = 0;
+        ioat->dmacount = 0;
+}
+
+enum dma_status
+ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                     dma_cookie_t *done, dma_cookie_t *used)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioatdma_device *device = ioat->base.device;
+
+        if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
+                return DMA_SUCCESS;
+
+        device->cleanup_tasklet((unsigned long) ioat);
+
+        return ioat_is_complete(c, cookie, done, used);
+}
+
+static ssize_t ring_size_show(struct dma_chan *c, char *page)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+
+        return sprintf(page, "%d\n", (1 << ioat->alloc_order) & ~1);
+}
+static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
+
+static ssize_t ring_active_show(struct dma_chan *c, char *page)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+
+        /* ...taken outside the lock, no need to be precise */
+        return sprintf(page, "%d\n", ioat2_ring_active(ioat));
+}
+static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
+
+static struct attribute *ioat2_attrs[] = {
+        &ring_size_attr.attr,
+        &ring_active_attr.attr,
+        &ioat_cap_attr.attr,
+        &ioat_version_attr.attr,
+        NULL,
+};
+
+struct kobj_type ioat2_ktype = {
+        .sysfs_ops = &ioat_sysfs_ops,
+        .default_attrs = ioat2_attrs,
+};
+
+int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca)
+{
+        struct pci_dev *pdev = device->pdev;
+        struct dma_device *dma;
+        struct dma_chan *c;
+        struct ioat_chan_common *chan;
+        int err;
+
+        device->enumerate_channels = ioat2_enumerate_channels;
+        device->cleanup_tasklet = ioat2_cleanup_tasklet;
+        device->timer_fn = ioat2_timer_event;
+        device->self_test = ioat_dma_self_test;
+        dma = &device->common;
+        dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
+        dma->device_issue_pending = ioat2_issue_pending;
+        dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
+        dma->device_free_chan_resources = ioat2_free_chan_resources;
+        dma->device_is_tx_complete = ioat2_is_complete;
+
+        err = ioat_probe(device);
+        if (err)
+                return err;
+        ioat_set_tcp_copy_break(2048);
+
+        list_for_each_entry(c, &dma->channels, device_node) {
+                chan = to_chan_common(c);
+                writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
+                       chan->reg_base + IOAT_DCACTRL_OFFSET);
+        }
+
+        err = ioat_register(device);
+        if (err)
+                return err;
+
+        ioat_kobject_add(device, &ioat2_ktype);
+
+        if (dca)
+                device->dca = ioat2_dca_init(pdev, device->reg_base);
+
+        return err;
+}
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
new file mode 100644
index 000000000000..1d849ef74d5f
--- /dev/null
+++ b/drivers/dma/ioat/dma_v2.h
@@ -0,0 +1,190 @@
+/*
+ * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called COPYING.
+ */
+#ifndef IOATDMA_V2_H
+#define IOATDMA_V2_H
+
+#include <linux/dmaengine.h>
+#include "dma.h"
+#include "hw.h"
+
+
+extern int ioat_pending_level;
+extern int ioat_ring_alloc_order;
+
+/*
+ * workaround for IOAT ver.3.0 null descriptor issue
+ * (channel returns error when size is 0)
+ */
+#define NULL_DESC_BUFFER_SIZE 1
+
+#define IOAT_MAX_ORDER 16
+#define ioat_get_alloc_order() \
+        (min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
+#define ioat_get_max_alloc_order() \
+        (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
+
+/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes
+ * @base: common ioat channel parameters
+ * @xfercap_log; log2 of channel max transfer length (for fast division)
+ * @head: allocated index
+ * @issued: hardware notification point
+ * @tail: cleanup index
+ * @pending: lock free indicator for issued != head
+ * @dmacount: identical to 'head' except for occasionally resetting to zero
+ * @alloc_order: log2 of the number of allocated descriptors
+ * @ring: software ring buffer implementation of hardware ring
+ * @ring_lock: protects ring attributes
+ */
+struct ioat2_dma_chan {
+        struct ioat_chan_common base;
+        size_t xfercap_log;
+        u16 head;
+        u16 issued;
+        u16 tail;
+        u16 dmacount;
+        u16 alloc_order;
+        int pending;
+        struct ioat_ring_ent **ring;
+        spinlock_t ring_lock;
+};
+
+static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c)
+{
+        struct ioat_chan_common *chan = to_chan_common(c);
+
+        return container_of(chan, struct ioat2_dma_chan, base);
+}
+
+static inline u16 ioat2_ring_mask(struct ioat2_dma_chan *ioat)
+{
+        return (1 << ioat->alloc_order) - 1;
+}
+
+/* count of descriptors in flight with the engine */
+static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat)
+{
+        return (ioat->head - ioat->tail) & ioat2_ring_mask(ioat);
+}
+
+/* count of descriptors pending submission to hardware */
+static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat)
+{
+        return (ioat->head - ioat->issued) & ioat2_ring_mask(ioat);
+}
+
+static inline u16 ioat2_ring_space(struct ioat2_dma_chan *ioat)
+{
+        u16 num_descs = ioat2_ring_mask(ioat) + 1;
+        u16 active = ioat2_ring_active(ioat);
+
+        BUG_ON(active > num_descs);
+
+        return num_descs - active;
+}
+
+/* assumes caller already checked space */
+static inline u16 ioat2_desc_alloc(struct ioat2_dma_chan *ioat, u16 len)
+{
+        ioat->head += len;
+        return ioat->head - len;
+}
+
+static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len)
+{
+        u16 num_descs = len >> ioat->xfercap_log;
+
+        num_descs += !!(len & ((1 << ioat->xfercap_log) - 1));
+        return num_descs;
+}
+
+/**
+ * struct ioat_ring_ent - wrapper around hardware descriptor
+ * @hw: hardware DMA descriptor (for memcpy)
+ * @fill: hardware fill descriptor
+ * @xor: hardware xor descriptor
+ * @xor_ex: hardware xor extension descriptor
+ * @pq: hardware pq descriptor
+ * @pq_ex: hardware pq extension descriptor
+ * @pqu: hardware pq update descriptor
+ * @raw: hardware raw (un-typed) descriptor
+ * @txd: the generic software descriptor for all engines
+ * @len: total transaction length for unmap
+ * @result: asynchronous result of validate operations
+ * @id: identifier for debug
+ */
+
+struct ioat_ring_ent {
+        union {
+                struct ioat_dma_descriptor *hw;
+                struct ioat_fill_descriptor *fill;
+                struct ioat_xor_descriptor *xor;
+                struct ioat_xor_ext_descriptor *xor_ex;
+                struct ioat_pq_descriptor *pq;
+                struct ioat_pq_ext_descriptor *pq_ex;
+                struct ioat_pq_update_descriptor *pqu;
+                struct ioat_raw_descriptor *raw;
+        };
+        size_t len;
+        struct dma_async_tx_descriptor txd;
+        enum sum_check_flags *result;
+        #ifdef DEBUG
+        int id;
+        #endif
+};
+
+static inline struct ioat_ring_ent *
+ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx)
+{
+        return ioat->ring[idx & ioat2_ring_mask(ioat)];
+}
+
+static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+
+        writel(addr & 0x00000000FFFFFFFF,
+               chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
+        writel(addr >> 32,
+               chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
+}
+
+int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca);
+int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca);
+struct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
+struct dca_provider * __devinit ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
+int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs);
+int ioat2_enumerate_channels(struct ioatdma_device *device);
+struct dma_async_tx_descriptor *
+ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
+                           dma_addr_t dma_src, size_t len, unsigned long flags);
+void ioat2_issue_pending(struct dma_chan *chan);
+int ioat2_alloc_chan_resources(struct dma_chan *c);
+void ioat2_free_chan_resources(struct dma_chan *c);
+enum dma_status ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                                  dma_cookie_t *done, dma_cookie_t *used);
+void __ioat2_restart_chan(struct ioat2_dma_chan *ioat);
+bool reshape_ring(struct ioat2_dma_chan *ioat, int order);
+void __ioat2_issue_pending(struct ioat2_dma_chan *ioat);
+void ioat2_cleanup_tasklet(unsigned long data);
+void ioat2_timer_event(unsigned long data);
+extern struct kobj_type ioat2_ktype;
+extern struct kmem_cache *ioat2_cache;
+#endif /* IOATDMA_V2_H */
diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c
new file mode 100644
index 000000000000..35d1e33afd5b
--- /dev/null
+++ b/drivers/dma/ioat/dma_v3.c
@@ -0,0 +1,1223 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2004-2009 Intel Corporation. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in
+ *     the documentation and/or other materials provided with the
+ *     distribution.
+ *   * Neither the name of Intel Corporation nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Support routines for v3+ hardware
+ */
+
+#include <linux/pci.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include "registers.h"
+#include "hw.h"
+#include "dma.h"
+#include "dma_v2.h"
+
+/* ioat hardware assumes at least two sources for raid operations */
+#define src_cnt_to_sw(x) ((x) + 2)
+#define src_cnt_to_hw(x) ((x) - 2)
+
+/* provide a lookup table for setting the source address in the base or
+ * extended descriptor of an xor or pq descriptor
+ */
+static const u8 xor_idx_to_desc __read_mostly = 0xd0;
+static const u8 xor_idx_to_field[] __read_mostly = { 1, 4, 5, 6, 7, 0, 1, 2 };
+static const u8 pq_idx_to_desc __read_mostly = 0xf8;
+static const u8 pq_idx_to_field[] __read_mostly = { 1, 4, 5, 0, 1, 2, 4, 5 };
+
+static dma_addr_t xor_get_src(struct ioat_raw_descriptor *descs[2], int idx)
+{
+        struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
+
+        return raw->field[xor_idx_to_field[idx]];
+}
+
+static void xor_set_src(struct ioat_raw_descriptor *descs[2],
+                        dma_addr_t addr, u32 offset, int idx)
+{
+        struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
+
+        raw->field[xor_idx_to_field[idx]] = addr + offset;
+}
+
+static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx)
+{
+        struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
+
+        return raw->field[pq_idx_to_field[idx]];
+}
+
+static void pq_set_src(struct ioat_raw_descriptor *descs[2],
+                       dma_addr_t addr, u32 offset, u8 coef, int idx)
+{
+        struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0];
+        struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
+
+        raw->field[pq_idx_to_field[idx]] = addr + offset;
+        pq->coef[idx] = coef;
+}
+
+static void ioat3_dma_unmap(struct ioat2_dma_chan *ioat,
+                            struct ioat_ring_ent *desc, int idx)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        struct pci_dev *pdev = chan->device->pdev;
+        size_t len = desc->len;
+        size_t offset = len - desc->hw->size;
+        struct dma_async_tx_descriptor *tx = &desc->txd;
+        enum dma_ctrl_flags flags = tx->flags;
+
+        switch (desc->hw->ctl_f.op) {
+        case IOAT_OP_COPY:
+                if (!desc->hw->ctl_f.null) /* skip 'interrupt' ops */
+                        ioat_dma_unmap(chan, flags, len, desc->hw);
+                break;
+        case IOAT_OP_FILL: {
+                struct ioat_fill_descriptor *hw = desc->fill;
+
+                if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
+                        ioat_unmap(pdev, hw->dst_addr - offset, len,
+                                   PCI_DMA_FROMDEVICE, flags, 1);
+                break;
+        }
+        case IOAT_OP_XOR_VAL:
+        case IOAT_OP_XOR: {
+                struct ioat_xor_descriptor *xor = desc->xor;
+                struct ioat_ring_ent *ext;
+                struct ioat_xor_ext_descriptor *xor_ex = NULL;
+                int src_cnt = src_cnt_to_sw(xor->ctl_f.src_cnt);
+                struct ioat_raw_descriptor *descs[2];
+                int i;
+
+                if (src_cnt > 5) {
+                        ext = ioat2_get_ring_ent(ioat, idx + 1);
+                        xor_ex = ext->xor_ex;
+                }
+
+                if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                        descs[0] = (struct ioat_raw_descriptor *) xor;
+                        descs[1] = (struct ioat_raw_descriptor *) xor_ex;
+                        for (i = 0; i < src_cnt; i++) {
+                                dma_addr_t src = xor_get_src(descs, i);
+
+                                ioat_unmap(pdev, src - offset, len,
+                                           PCI_DMA_TODEVICE, flags, 0);
+                        }
+
+                        /* dest is a source in xor validate operations */
+                        if (xor->ctl_f.op == IOAT_OP_XOR_VAL) {
+                                ioat_unmap(pdev, xor->dst_addr - offset, len,
+                                           PCI_DMA_TODEVICE, flags, 1);
+                                break;
+                        }
+                }
+
+                if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
+                        ioat_unmap(pdev, xor->dst_addr - offset, len,
+                                   PCI_DMA_FROMDEVICE, flags, 1);
+                break;
+        }
+        case IOAT_OP_PQ_VAL:
+        case IOAT_OP_PQ: {
+                struct ioat_pq_descriptor *pq = desc->pq;
+                struct ioat_ring_ent *ext;
+                struct ioat_pq_ext_descriptor *pq_ex = NULL;
+                int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
+                struct ioat_raw_descriptor *descs[2];
+                int i;
+
+                if (src_cnt > 3) {
+                        ext = ioat2_get_ring_ent(ioat, idx + 1);
+                        pq_ex = ext->pq_ex;
+                }
+
+                /* in the 'continue' case don't unmap the dests as sources */
+                if (dmaf_p_disabled_continue(flags))
+                        src_cnt--;
+                else if (dmaf_continue(flags))
+                        src_cnt -= 3;
+
+                if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                        descs[0] = (struct ioat_raw_descriptor *) pq;
+                        descs[1] = (struct ioat_raw_descriptor *) pq_ex;
+                        for (i = 0; i < src_cnt; i++) {
+                                dma_addr_t src = pq_get_src(descs, i);
+
+                                ioat_unmap(pdev, src - offset, len,
+                                           PCI_DMA_TODEVICE, flags, 0);
+                        }
+
+                        /* the dests are sources in pq validate operations */
+                        if (pq->ctl_f.op == IOAT_OP_XOR_VAL) {
+                                if (!(flags & DMA_PREP_PQ_DISABLE_P))
+                                        ioat_unmap(pdev, pq->p_addr - offset,
+                                                   len, PCI_DMA_TODEVICE, flags, 0);
+                                if (!(flags & DMA_PREP_PQ_DISABLE_Q))
+                                        ioat_unmap(pdev, pq->q_addr - offset,
+                                                   len, PCI_DMA_TODEVICE, flags, 0);
+                                break;
+                        }
+                }
+
+                if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+                        if (!(flags & DMA_PREP_PQ_DISABLE_P))
+                                ioat_unmap(pdev, pq->p_addr - offset, len,
+                                           PCI_DMA_BIDIRECTIONAL, flags, 1);
+                        if (!(flags & DMA_PREP_PQ_DISABLE_Q))
+                                ioat_unmap(pdev, pq->q_addr - offset, len,
+                                           PCI_DMA_BIDIRECTIONAL, flags, 1);
+                }
+                break;
+        }
+        default:
+                dev_err(&pdev->dev, "%s: unknown op type: %#x\n",
+                        __func__, desc->hw->ctl_f.op);
+        }
+}
+
+static bool desc_has_ext(struct ioat_ring_ent *desc)
+{
+        struct ioat_dma_descriptor *hw = desc->hw;
+
+        if (hw->ctl_f.op == IOAT_OP_XOR ||
+            hw->ctl_f.op == IOAT_OP_XOR_VAL) {
+                struct ioat_xor_descriptor *xor = desc->xor;
+
+                if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
+                        return true;
+        } else if (hw->ctl_f.op == IOAT_OP_PQ ||
+                   hw->ctl_f.op == IOAT_OP_PQ_VAL) {
+                struct ioat_pq_descriptor *pq = desc->pq;
+
+                if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3)
+                        return true;
+        }
+
+        return false;
+}
+
+/**
+ * __cleanup - reclaim used descriptors
+ * @ioat: channel (ring) to clean
+ *
+ * The difference from the dma_v2.c __cleanup() is that this routine
+ * handles extended descriptors and dma-unmapping raid operations.
+ */
+static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioat_ring_ent *desc;
+        bool seen_current = false;
+        u16 active;
+        int i;
+
+        dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
+                __func__, ioat->head, ioat->tail, ioat->issued);
+
+        active = ioat2_ring_active(ioat);
+        for (i = 0; i < active && !seen_current; i++) {
+                struct dma_async_tx_descriptor *tx;
+
+                prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
+                desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
+                dump_desc_dbg(ioat, desc);
+                tx = &desc->txd;
+                if (tx->cookie) {
+                        chan->completed_cookie = tx->cookie;
+                        ioat3_dma_unmap(ioat, desc, ioat->tail + i);
+                        tx->cookie = 0;
+                        if (tx->callback) {
+                                tx->callback(tx->callback_param);
+                                tx->callback = NULL;
+                        }
+                }
+
+                if (tx->phys == phys_complete)
+                        seen_current = true;
+
+                /* skip extended descriptors */
+                if (desc_has_ext(desc)) {
+                        BUG_ON(i + 1 >= active);
+                        i++;
+                }
+        }
+        ioat->tail += i;
+        BUG_ON(!seen_current); /* no active descs have written a completion? */
+        chan->last_completion = phys_complete;
+        if (ioat->head == ioat->tail) {
+                dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
+                        __func__);
+                clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+                mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
+        }
+}
+
+static void ioat3_cleanup(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        unsigned long phys_complete;
+
+        prefetch(chan->completion);
+
+        if (!spin_trylock_bh(&chan->cleanup_lock))
+                return;
+
+        if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+
+        if (!spin_trylock_bh(&ioat->ring_lock)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+
+        __cleanup(ioat, phys_complete);
+
+        spin_unlock_bh(&ioat->ring_lock);
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static void ioat3_cleanup_tasklet(unsigned long data)
+{
+        struct ioat2_dma_chan *ioat = (void *) data;
+
+        ioat3_cleanup(ioat);
+        writew(IOAT_CHANCTRL_RUN | IOAT3_CHANCTRL_COMPL_DCA_EN,
+               ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
+}
+
+static void ioat3_restart_channel(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        unsigned long phys_complete;
+        u32 status;
+
+        status = ioat_chansts(chan);
+        if (is_ioat_active(status) || is_ioat_idle(status))
+                ioat_suspend(chan);
+        while (is_ioat_active(status) || is_ioat_idle(status)) {
+                status = ioat_chansts(chan);
+                cpu_relax();
+        }
+
+        if (ioat_cleanup_preamble(chan, &phys_complete))
+                __cleanup(ioat, phys_complete);
+
+        __ioat2_restart_chan(ioat);
+}
+
+static void ioat3_timer_event(unsigned long data)
+{
+        struct ioat2_dma_chan *ioat = (void *) data;
+        struct ioat_chan_common *chan = &ioat->base;
+
+        spin_lock_bh(&chan->cleanup_lock);
+        if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+                unsigned long phys_complete;
+                u64 status;
+
+                spin_lock_bh(&ioat->ring_lock);
+                status = ioat_chansts(chan);
+
+                /* when halted due to errors check for channel
+                 * programming errors before advancing the completion state
+                 */
+                if (is_ioat_halted(status)) {
+                        u32 chanerr;
+
+                        chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+                        BUG_ON(is_ioat_bug(chanerr));
+                }
+
+                /* if we haven't made progress and we have already
+                 * acknowledged a pending completion once, then be more
+                 * forceful with a restart
+                 */
+                if (ioat_cleanup_preamble(chan, &phys_complete))
+                        __cleanup(ioat, phys_complete);
+                else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+                        ioat3_restart_channel(ioat);
+                else {
+                        set_bit(IOAT_COMPLETION_ACK, &chan->state);
+                        mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+                }
+                spin_unlock_bh(&ioat->ring_lock);
+        } else {
+                u16 active;
+
+                /* if the ring is idle, empty, and oversized try to step
+                 * down the size
+                 */
+                spin_lock_bh(&ioat->ring_lock);
+                active = ioat2_ring_active(ioat);
+                if (active == 0 && ioat->alloc_order > ioat_get_alloc_order())
+                        reshape_ring(ioat, ioat->alloc_order-1);
+                spin_unlock_bh(&ioat->ring_lock);
+
+                /* keep shrinking until we get back to our minimum
+                 * default size
+                 */
+                if (ioat->alloc_order > ioat_get_alloc_order())
+                        mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
+        }
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static enum dma_status
+ioat3_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                  dma_cookie_t *done, dma_cookie_t *used)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+
+        if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
+                return DMA_SUCCESS;
+
+        ioat3_cleanup(ioat);
+
+        return ioat_is_complete(c, cookie, done, used);
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_memset_lock(struct dma_chan *c, dma_addr_t dest, int value,
+                       size_t len, unsigned long flags)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_ring_ent *desc;
+        size_t total_len = len;
+        struct ioat_fill_descriptor *fill;
+        int num_descs;
+        u64 src_data = (0x0101010101010101ULL) * (value & 0xff);
+        u16 idx;
+        int i;
+
+        num_descs = ioat2_xferlen_to_descs(ioat, len);
+        if (likely(num_descs) &&
+            ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0)
+                /* pass */;
+        else
+                return NULL;
+        i = 0;
+        do {
+                size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
+
+                desc = ioat2_get_ring_ent(ioat, idx + i);
+                fill = desc->fill;
+
+                fill->size = xfer_size;
+                fill->src_data = src_data;
+                fill->dst_addr = dest;
+                fill->ctl = 0;
+                fill->ctl_f.op = IOAT_OP_FILL;
+
+                len -= xfer_size;
+                dest += xfer_size;
+                dump_desc_dbg(ioat, desc);
+        } while (++i < num_descs);
+
+        desc->txd.flags = flags;
+        desc->len = total_len;
+        fill->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+        fill->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+        fill->ctl_f.compl_write = 1;
+        dump_desc_dbg(ioat, desc);
+
+        /* we leave the channel locked to ensure in order submission */
+        return &desc->txd;
+}
+
+static struct dma_async_tx_descriptor *
+__ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
+                      dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt,
+                      size_t len, unsigned long flags)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_ring_ent *compl_desc;
+        struct ioat_ring_ent *desc;
+        struct ioat_ring_ent *ext;
+        size_t total_len = len;
+        struct ioat_xor_descriptor *xor;
+        struct ioat_xor_ext_descriptor *xor_ex = NULL;
+        struct ioat_dma_descriptor *hw;
+        u32 offset = 0;
+        int num_descs;
+        int with_ext;
+        int i;
+        u16 idx;
+        u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR;
+
+        BUG_ON(src_cnt < 2);
+
+        num_descs = ioat2_xferlen_to_descs(ioat, len);
+        /* we need 2x the number of descriptors to cover greater than 5
+         * sources
+         */
+        if (src_cnt > 5) {
+                with_ext = 1;
+                num_descs *= 2;
+        } else
+                with_ext = 0;
+
+        /* completion writes from the raid engine may pass completion
+         * writes from the legacy engine, so we need one extra null
+         * (legacy) descriptor to ensure all completion writes arrive in
+         * order.
+         */
+        if (likely(num_descs) &&
+            ioat2_alloc_and_lock(&idx, ioat, num_descs+1) == 0)
+                /* pass */;
+        else
+                return NULL;
+        i = 0;
+        do {
+                struct ioat_raw_descriptor *descs[2];
+                size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
+                int s;
+
+                desc = ioat2_get_ring_ent(ioat, idx + i);
+                xor = desc->xor;
+
+                /* save a branch by unconditionally retrieving the
+                 * extended descriptor xor_set_src() knows to not write
+                 * to it in the single descriptor case
+                 */
+                ext = ioat2_get_ring_ent(ioat, idx + i + 1);
+                xor_ex = ext->xor_ex;
+
+                descs[0] = (struct ioat_raw_descriptor *) xor;
+                descs[1] = (struct ioat_raw_descriptor *) xor_ex;
+                for (s = 0; s < src_cnt; s++)
+                        xor_set_src(descs, src[s], offset, s);
+                xor->size = xfer_size;
+                xor->dst_addr = dest + offset;
+                xor->ctl = 0;
+                xor->ctl_f.op = op;
+                xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt);
+
+                len -= xfer_size;
+                offset += xfer_size;
+                dump_desc_dbg(ioat, desc);
+        } while ((i += 1 + with_ext) < num_descs);
+
+        /* last xor descriptor carries the unmap parameters and fence bit */
+        desc->txd.flags = flags;
+        desc->len = total_len;
+        if (result)
+                desc->result = result;
+        xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+
+        /* completion descriptor carries interrupt bit */
+        compl_desc = ioat2_get_ring_ent(ioat, idx + i);
+        compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
+        hw = compl_desc->hw;
+        hw->ctl = 0;
+        hw->ctl_f.null = 1;
+        hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+        hw->ctl_f.compl_write = 1;
+        hw->size = NULL_DESC_BUFFER_SIZE;
+        dump_desc_dbg(ioat, compl_desc);
+
+        /* we leave the channel locked to ensure in order submission */
+        return &desc->txd;
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
+               unsigned int src_cnt, size_t len, unsigned long flags)
+{
+        return __ioat3_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags);
+}
+
+struct dma_async_tx_descriptor *
+ioat3_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
+                    unsigned int src_cnt, size_t len,
+                    enum sum_check_flags *result, unsigned long flags)
+{
+        /* the cleanup routine only sets bits on validate failure, it
+         * does not clear bits on validate success... so clear it here
+         */
+        *result = 0;
+
+        return __ioat3_prep_xor_lock(chan, result, src[0], &src[1],
+                                     src_cnt - 1, len, flags);
+}
+
+static void
+dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext)
+{
+        struct device *dev = to_dev(&ioat->base);
+        struct ioat_pq_descriptor *pq = desc->pq;
+        struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL;
+        struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex };
+        int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
+        int i;
+
+        dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
+                " sz: %#x ctl: %#x (op: %d int: %d compl: %d pq: '%s%s' src_cnt: %d)\n",
+                desc_id(desc), (unsigned long long) desc->txd.phys,
+                (unsigned long long) (pq_ex ? pq_ex->next : pq->next),
+                desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op, pq->ctl_f.int_en,
+                pq->ctl_f.compl_write,
+                pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
+                pq->ctl_f.src_cnt);
+        for (i = 0; i < src_cnt; i++)
+                dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
+                        (unsigned long long) pq_get_src(descs, i), pq->coef[i]);
+        dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
+        dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
+}
+
+static struct dma_async_tx_descriptor *
+__ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
+                     const dma_addr_t *dst, const dma_addr_t *src,
+                     unsigned int src_cnt, const unsigned char *scf,
+                     size_t len, unsigned long flags)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_chan_common *chan = &ioat->base;
+        struct ioat_ring_ent *compl_desc;
+        struct ioat_ring_ent *desc;
+        struct ioat_ring_ent *ext;
+        size_t total_len = len;
+        struct ioat_pq_descriptor *pq;
+        struct ioat_pq_ext_descriptor *pq_ex = NULL;
+        struct ioat_dma_descriptor *hw;
+        u32 offset = 0;
+        int num_descs;
+        int with_ext;
+        int i, s;
+        u16 idx;
+        u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ;
+
+        dev_dbg(to_dev(chan), "%s\n", __func__);
+        /* the engine requires at least two sources (we provide
+         * at least 1 implied source in the DMA_PREP_CONTINUE case)
+         */
+        BUG_ON(src_cnt + dmaf_continue(flags) < 2);
+
+        num_descs = ioat2_xferlen_to_descs(ioat, len);
+        /* we need 2x the number of descriptors to cover greater than 3
+         * sources
+         */
+        if (src_cnt > 3 || flags & DMA_PREP_CONTINUE) {
+                with_ext = 1;
+                num_descs *= 2;
+        } else
+                with_ext = 0;
+
+        /* completion writes from the raid engine may pass completion
+         * writes from the legacy engine, so we need one extra null
+         * (legacy) descriptor to ensure all completion writes arrive in
+         * order.
+         */
+        if (likely(num_descs) &&
+            ioat2_alloc_and_lock(&idx, ioat, num_descs+1) == 0)
+                /* pass */;
+        else
+                return NULL;
+        i = 0;
+        do {
+                struct ioat_raw_descriptor *descs[2];
+                size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
+
+                desc = ioat2_get_ring_ent(ioat, idx + i);
+                pq = desc->pq;
+
+                /* save a branch by unconditionally retrieving the
+                 * extended descriptor pq_set_src() knows to not write
+                 * to it in the single descriptor case
+                 */
+                ext = ioat2_get_ring_ent(ioat, idx + i + with_ext);
+                pq_ex = ext->pq_ex;
+
+                descs[0] = (struct ioat_raw_descriptor *) pq;
+                descs[1] = (struct ioat_raw_descriptor *) pq_ex;
+
+                for (s = 0; s < src_cnt; s++)
+                        pq_set_src(descs, src[s], offset, scf[s], s);
+
+                /* see the comment for dma_maxpq in include/linux/dmaengine.h */
+                if (dmaf_p_disabled_continue(flags))
+                        pq_set_src(descs, dst[1], offset, 1, s++);
+                else if (dmaf_continue(flags)) {
+                        pq_set_src(descs, dst[0], offset, 0, s++);
+                        pq_set_src(descs, dst[1], offset, 1, s++);
+                        pq_set_src(descs, dst[1], offset, 0, s++);
+                }
+                pq->size = xfer_size;
+                pq->p_addr = dst[0] + offset;
+                pq->q_addr = dst[1] + offset;
+                pq->ctl = 0;
+                pq->ctl_f.op = op;
+                pq->ctl_f.src_cnt = src_cnt_to_hw(s);
+                pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
+                pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
+
+                len -= xfer_size;
+                offset += xfer_size;
+        } while ((i += 1 + with_ext) < num_descs);
+
+        /* last pq descriptor carries the unmap parameters and fence bit */
+        desc->txd.flags = flags;
+        desc->len = total_len;
+        if (result)
+                desc->result = result;
+        pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+        dump_pq_desc_dbg(ioat, desc, ext);
+
+        /* completion descriptor carries interrupt bit */
+        compl_desc = ioat2_get_ring_ent(ioat, idx + i);
+        compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
+        hw = compl_desc->hw;
+        hw->ctl = 0;
+        hw->ctl_f.null = 1;
+        hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+        hw->ctl_f.compl_write = 1;
+        hw->size = NULL_DESC_BUFFER_SIZE;
+        dump_desc_dbg(ioat, compl_desc);
+
+        /* we leave the channel locked to ensure in order submission */
+        return &desc->txd;
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
+              unsigned int src_cnt, const unsigned char *scf, size_t len,
+              unsigned long flags)
+{
+        /* handle the single source multiply case from the raid6
+         * recovery path
+         */
+        if (unlikely((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1)) {
+                dma_addr_t single_source[2];
+                unsigned char single_source_coef[2];
+
+                BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q);
+                single_source[0] = src[0];
+                single_source[1] = src[0];
+                single_source_coef[0] = scf[0];
+                single_source_coef[1] = 0;
+
+                return __ioat3_prep_pq_lock(chan, NULL, dst, single_source, 2,
+                                            single_source_coef, len, flags);
+        } else
+                return __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt, scf,
+                                            len, flags);
+}
+
+struct dma_async_tx_descriptor *
+ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
+                  unsigned int src_cnt, const unsigned char *scf, size_t len,
+                  enum sum_check_flags *pqres, unsigned long flags)
+{
+        /* the cleanup routine only sets bits on validate failure, it
+         * does not clear bits on validate success... so clear it here
+         */
+        *pqres = 0;
+
+        return __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,
+                                    flags);
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
+                 unsigned int src_cnt, size_t len, unsigned long flags)
+{
+        unsigned char scf[src_cnt];
+        dma_addr_t pq[2];
+
+        memset(scf, 0, src_cnt);
+        flags |= DMA_PREP_PQ_DISABLE_Q;
+        pq[0] = dst;
+        pq[1] = ~0;
+
+        return __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,
+                                    flags);
+}
+
+struct dma_async_tx_descriptor *
+ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
+                     unsigned int src_cnt, size_t len,
+                     enum sum_check_flags *result, unsigned long flags)
+{
+        unsigned char scf[src_cnt];
+        dma_addr_t pq[2];
+
+        /* the cleanup routine only sets bits on validate failure, it
+         * does not clear bits on validate success... so clear it here
+         */
+        *result = 0;
+
+        memset(scf, 0, src_cnt);
+        flags |= DMA_PREP_PQ_DISABLE_Q;
+        pq[0] = src[0];
+        pq[1] = ~0;
+
+        return __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, scf,
+                                    len, flags);
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags)
+{
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+        struct ioat_ring_ent *desc;
+        struct ioat_dma_descriptor *hw;
+        u16 idx;
+
+        if (ioat2_alloc_and_lock(&idx, ioat, 1) == 0)
+                desc = ioat2_get_ring_ent(ioat, idx);
+        else
+                return NULL;
+
+        hw = desc->hw;
+        hw->ctl = 0;
+        hw->ctl_f.null = 1;
+        hw->ctl_f.int_en = 1;
+        hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+        hw->ctl_f.compl_write = 1;
+        hw->size = NULL_DESC_BUFFER_SIZE;
+        hw->src_addr = 0;
+        hw->dst_addr = 0;
+
+        desc->txd.flags = flags;
+        desc->len = 1;
+
+        dump_desc_dbg(ioat, desc);
+
+        /* we leave the channel locked to ensure in order submission */
+        return &desc->txd;
+}
+
+static void __devinit ioat3_dma_test_callback(void *dma_async_param)
+{
+        struct completion *cmp = dma_async_param;
+
+        complete(cmp);
+}
+
+#define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */
+static int __devinit ioat_xor_val_self_test(struct ioatdma_device *device)
+{
+        int i, src_idx;
+        struct page *dest;
+        struct page *xor_srcs[IOAT_NUM_SRC_TEST];
+        struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1];
+        dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1];
+        dma_addr_t dma_addr, dest_dma;
+        struct dma_async_tx_descriptor *tx;
+        struct dma_chan *dma_chan;
+        dma_cookie_t cookie;
+        u8 cmp_byte = 0;
+        u32 cmp_word;
+        u32 xor_val_result;
+        int err = 0;
+        struct completion cmp;
+        unsigned long tmo;
+        struct device *dev = &device->pdev->dev;
+        struct dma_device *dma = &device->common;
+
+        dev_dbg(dev, "%s\n", __func__);
+
+        if (!dma_has_cap(DMA_XOR, dma->cap_mask))
+                return 0;
+
+        for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
+                xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
+                if (!xor_srcs[src_idx]) {
+                        while (src_idx--)
+                                __free_page(xor_srcs[src_idx]);
+                        return -ENOMEM;
+                }
+        }
+
+        dest = alloc_page(GFP_KERNEL);
+        if (!dest) {
+                while (src_idx--)
+                        __free_page(xor_srcs[src_idx]);
+                return -ENOMEM;
+        }
+
+        /* Fill in src buffers */
+        for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
+                u8 *ptr = page_address(xor_srcs[src_idx]);
+                for (i = 0; i < PAGE_SIZE; i++)
+                        ptr[i] = (1 << src_idx);
+        }
+
+        for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++)
+                cmp_byte ^= (u8) (1 << src_idx);
+
+        cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
+                        (cmp_byte << 8) | cmp_byte;
+
+        memset(page_address(dest), 0, PAGE_SIZE);
+
+        dma_chan = container_of(dma->channels.next, struct dma_chan,
+                                device_node);
+        if (dma->device_alloc_chan_resources(dma_chan) < 1) {
+                err = -ENODEV;
+                goto out;
+        }
+
+        /* test xor */
+        dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE);
+        for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
+                dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE,
+                                           DMA_TO_DEVICE);
+        tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
+                                      IOAT_NUM_SRC_TEST, PAGE_SIZE,
+                                      DMA_PREP_INTERRUPT);
+
+        if (!tx) {
+                dev_err(dev, "Self-test xor prep failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        async_tx_ack(tx);
+        init_completion(&cmp);
+        tx->callback = ioat3_dma_test_callback;
+        tx->callback_param = &cmp;
+        cookie = tx->tx_submit(tx);
+        if (cookie < 0) {
+                dev_err(dev, "Self-test xor setup failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        dma->device_issue_pending(dma_chan);
+
+        tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+        if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
+                dev_err(dev, "Self-test xor timed out\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        dma_sync_single_for_cpu(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
+        for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
+                u32 *ptr = page_address(dest);
+                if (ptr[i] != cmp_word) {
+                        dev_err(dev, "Self-test xor failed compare\n");
+                        err = -ENODEV;
+                        goto free_resources;
+                }
+        }
+        dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_TO_DEVICE);
+
+        /* skip validate if the capability is not present */
+        if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
+                goto free_resources;
+
+        /* validate the sources with the destintation page */
+        for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
+                xor_val_srcs[i] = xor_srcs[i];
+        xor_val_srcs[i] = dest;
+
+        xor_val_result = 1;
+
+        for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
+                dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
+                                           DMA_TO_DEVICE);
+        tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
+                                          IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
+                                          &xor_val_result, DMA_PREP_INTERRUPT);
+        if (!tx) {
+                dev_err(dev, "Self-test zero prep failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        async_tx_ack(tx);
+        init_completion(&cmp);
+        tx->callback = ioat3_dma_test_callback;
+        tx->callback_param = &cmp;
+        cookie = tx->tx_submit(tx);
+        if (cookie < 0) {
+                dev_err(dev, "Self-test zero setup failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        dma->device_issue_pending(dma_chan);
+
+        tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+        if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
+                dev_err(dev, "Self-test validate timed out\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        if (xor_val_result != 0) {
+                dev_err(dev, "Self-test validate failed compare\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        /* skip memset if the capability is not present */
+        if (!dma_has_cap(DMA_MEMSET, dma_chan->device->cap_mask))
+                goto free_resources;
+
+        /* test memset */
+        dma_addr = dma_map_page(dev, dest, 0,
+                        PAGE_SIZE, DMA_FROM_DEVICE);
+        tx = dma->device_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE,
+                                         DMA_PREP_INTERRUPT);
+        if (!tx) {
+                dev_err(dev, "Self-test memset prep failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        async_tx_ack(tx);
+        init_completion(&cmp);
+        tx->callback = ioat3_dma_test_callback;
+        tx->callback_param = &cmp;
+        cookie = tx->tx_submit(tx);
+        if (cookie < 0) {
+                dev_err(dev, "Self-test memset setup failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        dma->device_issue_pending(dma_chan);
+
+        tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+        if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
+                dev_err(dev, "Self-test memset timed out\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) {
+                u32 *ptr = page_address(dest);
+                if (ptr[i]) {
+                        dev_err(dev, "Self-test memset failed compare\n");
+                        err = -ENODEV;
+                        goto free_resources;
+                }
+        }
+
+        /* test for non-zero parity sum */
+        xor_val_result = 0;
+        for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
+                dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
+                                           DMA_TO_DEVICE);
+        tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
+                                          IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
+                                          &xor_val_result, DMA_PREP_INTERRUPT);
+        if (!tx) {
+                dev_err(dev, "Self-test 2nd zero prep failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        async_tx_ack(tx);
+        init_completion(&cmp);
+        tx->callback = ioat3_dma_test_callback;
+        tx->callback_param = &cmp;
+        cookie = tx->tx_submit(tx);
+        if (cookie < 0) {
+                dev_err(dev, "Self-test  2nd zero setup failed\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        dma->device_issue_pending(dma_chan);
+
+        tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+        if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
+                dev_err(dev, "Self-test 2nd validate timed out\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        if (xor_val_result != SUM_CHECK_P_RESULT) {
+                dev_err(dev, "Self-test validate failed compare\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+free_resources:
+        dma->device_free_chan_resources(dma_chan);
+out:
+        src_idx = IOAT_NUM_SRC_TEST;
+        while (src_idx--)
+                __free_page(xor_srcs[src_idx]);
+        __free_page(dest);
+        return err;
+}
+
+static int __devinit ioat3_dma_self_test(struct ioatdma_device *device)
+{
+        int rc = ioat_dma_self_test(device);
+
+        if (rc)
+                return rc;
+
+        rc = ioat_xor_val_self_test(device);
+        if (rc)
+                return rc;
+
+        return 0;
+}
+
+int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca)
+{
+        struct pci_dev *pdev = device->pdev;
+        struct dma_device *dma;
+        struct dma_chan *c;
+        struct ioat_chan_common *chan;
+        bool is_raid_device = false;
+        int err;
+        u16 dev_id;
+        u32 cap;
+
+        device->enumerate_channels = ioat2_enumerate_channels;
+        device->self_test = ioat3_dma_self_test;
+        dma = &device->common;
+        dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
+        dma->device_issue_pending = ioat2_issue_pending;
+        dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
+        dma->device_free_chan_resources = ioat2_free_chan_resources;
+
+        dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
+        dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock;
+
+        cap = readl(device->reg_base + IOAT_DMA_CAP_OFFSET);
+        if (cap & IOAT_CAP_XOR) {
+                is_raid_device = true;
+                dma->max_xor = 8;
+                dma->xor_align = 2;
+
+                dma_cap_set(DMA_XOR, dma->cap_mask);
+                dma->device_prep_dma_xor = ioat3_prep_xor;
+
+                dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
+                dma->device_prep_dma_xor_val = ioat3_prep_xor_val;
+        }
+        if (cap & IOAT_CAP_PQ) {
+                is_raid_device = true;
+                dma_set_maxpq(dma, 8, 0);
+                dma->pq_align = 2;
+
+                dma_cap_set(DMA_PQ, dma->cap_mask);
+                dma->device_prep_dma_pq = ioat3_prep_pq;
+
+                dma_cap_set(DMA_PQ_VAL, dma->cap_mask);
+                dma->device_prep_dma_pq_val = ioat3_prep_pq_val;
+
+                if (!(cap & IOAT_CAP_XOR)) {
+                        dma->max_xor = 8;
+                        dma->xor_align = 2;
+
+                        dma_cap_set(DMA_XOR, dma->cap_mask);
+                        dma->device_prep_dma_xor = ioat3_prep_pqxor;
+
+                        dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
+                        dma->device_prep_dma_xor_val = ioat3_prep_pqxor_val;
+                }
+        }
+        if (is_raid_device && (cap & IOAT_CAP_FILL_BLOCK)) {
+                dma_cap_set(DMA_MEMSET, dma->cap_mask);
+                dma->device_prep_dma_memset = ioat3_prep_memset_lock;
+        }
+
+
+        if (is_raid_device) {
+                dma->device_is_tx_complete = ioat3_is_complete;
+                device->cleanup_tasklet = ioat3_cleanup_tasklet;
+                device->timer_fn = ioat3_timer_event;
+        } else {
+                dma->device_is_tx_complete = ioat2_is_complete;
+                device->cleanup_tasklet = ioat2_cleanup_tasklet;
+                device->timer_fn = ioat2_timer_event;
+        }
+
+        /* -= IOAT ver.3 workarounds =- */
+        /* Write CHANERRMSK_INT with 3E07h to mask out the errors
+         * that can cause stability issues for IOAT ver.3
+         */
+        pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
+
+        /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
+         * (workaround for spurious config parity error after restart)
+         */
+        pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
+        if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
+                pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
+
+        err = ioat_probe(device);
+        if (err)
+                return err;
+        ioat_set_tcp_copy_break(262144);
+
+        list_for_each_entry(c, &dma->channels, device_node) {
+                chan = to_chan_common(c);
+                writel(IOAT_DMA_DCA_ANY_CPU,
+                       chan->reg_base + IOAT_DCACTRL_OFFSET);
+        }
+
+        err = ioat_register(device);
+        if (err)
+                return err;
+
+        ioat_kobject_add(device, &ioat2_ktype);
+
+        if (dca)
+                device->dca = ioat3_dca_init(pdev, device->reg_base);
+
+        return 0;
+}
diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h
new file mode 100644
index 000000000000..99afb12bd409
--- /dev/null
+++ b/drivers/dma/ioat/hw.h
@@ -0,0 +1,215 @@
+/*
+ * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called COPYING.
+ */
+#ifndef _IOAT_HW_H_
+#define _IOAT_HW_H_
+
+/* PCI Configuration Space Values */
+#define IOAT_PCI_VID            0x8086
+#define IOAT_MMIO_BAR           0
+
+/* CB device ID's */
+#define IOAT_PCI_DID_5000       0x1A38
+#define IOAT_PCI_DID_CNB        0x360B
+#define IOAT_PCI_DID_SCNB       0x65FF
+#define IOAT_PCI_DID_SNB        0x402F
+
+#define IOAT_PCI_RID            0x00
+#define IOAT_PCI_SVID           0x8086
+#define IOAT_PCI_SID            0x8086
+#define IOAT_VER_1_2            0x12    /* Version 1.2 */
+#define IOAT_VER_2_0            0x20    /* Version 2.0 */
+#define IOAT_VER_3_0            0x30    /* Version 3.0 */
+#define IOAT_VER_3_2            0x32    /* Version 3.2 */
+
+struct ioat_dma_descriptor {
+        uint32_t        size;
+        union {
+                uint32_t ctl;
+                struct {
+                        unsigned int int_en:1;
+                        unsigned int src_snoop_dis:1;
+                        unsigned int dest_snoop_dis:1;
+                        unsigned int compl_write:1;
+                        unsigned int fence:1;
+                        unsigned int null:1;
+                        unsigned int src_brk:1;
+                        unsigned int dest_brk:1;
+                        unsigned int bundle:1;
+                        unsigned int dest_dca:1;
+                        unsigned int hint:1;
+                        unsigned int rsvd2:13;
+                        #define IOAT_OP_COPY 0x00
+                        unsigned int op:8;
+                } ctl_f;
+        };
+        uint64_t        src_addr;
+        uint64_t        dst_addr;
+        uint64_t        next;
+        uint64_t        rsv1;
+        uint64_t        rsv2;
+        /* store some driver data in an unused portion of the descriptor */
+        union {
+                uint64_t        user1;
+                uint64_t        tx_cnt;
+        };
+        uint64_t        user2;
+};
+
+struct ioat_fill_descriptor {
+        uint32_t        size;
+        union {
+                uint32_t ctl;
+                struct {
+                        unsigned int int_en:1;
+                        unsigned int rsvd:1;
+                        unsigned int dest_snoop_dis:1;
+                        unsigned int compl_write:1;
+                        unsigned int fence:1;
+                        unsigned int rsvd2:2;
+                        unsigned int dest_brk:1;
+                        unsigned int bundle:1;
+                        unsigned int rsvd4:15;
+                        #define IOAT_OP_FILL 0x01
+                        unsigned int op:8;
+                } ctl_f;
+        };
+        uint64_t        src_data;
+        uint64_t        dst_addr;
+        uint64_t        next;
+        uint64_t        rsv1;
+        uint64_t        next_dst_addr;
+        uint64_t        user1;
+        uint64_t        user2;
+};
+
+struct ioat_xor_descriptor {
+        uint32_t        size;
+        union {
+                uint32_t ctl;
+                struct {
+                        unsigned int int_en:1;
+                        unsigned int src_snoop_dis:1;
+                        unsigned int dest_snoop_dis:1;
+                        unsigned int compl_write:1;
+                        unsigned int fence:1;
+                        unsigned int src_cnt:3;
+                        unsigned int bundle:1;
+                        unsigned int dest_dca:1;
+                        unsigned int hint:1;
+                        unsigned int rsvd:13;
+                        #define IOAT_OP_XOR 0x87
+                        #define IOAT_OP_XOR_VAL 0x88
+                        unsigned int op:8;
+                } ctl_f;
+        };
+        uint64_t        src_addr;
+        uint64_t        dst_addr;
+        uint64_t        next;
+        uint64_t        src_addr2;
+        uint64_t        src_addr3;
+        uint64_t        src_addr4;
+        uint64_t        src_addr5;
+};
+
+struct ioat_xor_ext_descriptor {
+        uint64_t        src_addr6;
+        uint64_t        src_addr7;
+        uint64_t        src_addr8;
+        uint64_t        next;
+        uint64_t        rsvd[4];
+};
+
+struct ioat_pq_descriptor {
+        uint32_t        size;
+        union {
+                uint32_t ctl;
+                struct {
+                        unsigned int int_en:1;
+                        unsigned int src_snoop_dis:1;
+                        unsigned int dest_snoop_dis:1;
+                        unsigned int compl_write:1;
+                        unsigned int fence:1;
+                        unsigned int src_cnt:3;
+                        unsigned int bundle:1;
+                        unsigned int dest_dca:1;
+                        unsigned int hint:1;
+                        unsigned int p_disable:1;
+                        unsigned int q_disable:1;
+                        unsigned int rsvd:11;
+                        #define IOAT_OP_PQ 0x89
+                        #define IOAT_OP_PQ_VAL 0x8a
+                        unsigned int op:8;
+                } ctl_f;
+        };
+        uint64_t        src_addr;
+        uint64_t        p_addr;
+        uint64_t        next;
+        uint64_t        src_addr2;
+        uint64_t        src_addr3;
+        uint8_t         coef[8];
+        uint64_t        q_addr;
+};
+
+struct ioat_pq_ext_descriptor {
+        uint64_t        src_addr4;
+        uint64_t        src_addr5;
+        uint64_t        src_addr6;
+        uint64_t        next;
+        uint64_t        src_addr7;
+        uint64_t        src_addr8;
+        uint64_t        rsvd[2];
+};
+
+struct ioat_pq_update_descriptor {
+        uint32_t        size;
+        union {
+                uint32_t ctl;
+                struct {
+                        unsigned int int_en:1;
+                        unsigned int src_snoop_dis:1;
+                        unsigned int dest_snoop_dis:1;
+                        unsigned int compl_write:1;
+                        unsigned int fence:1;
+                        unsigned int src_cnt:3;
+                        unsigned int bundle:1;
+                        unsigned int dest_dca:1;
+                        unsigned int hint:1;
+                        unsigned int p_disable:1;
+                        unsigned int q_disable:1;
+                        unsigned int rsvd:3;
+                        unsigned int coef:8;
+                        #define IOAT_OP_PQ_UP 0x8b
+                        unsigned int op:8;
+                } ctl_f;
+        };
+        uint64_t        src_addr;
+        uint64_t        p_addr;
+        uint64_t        next;
+        uint64_t        src_addr2;
+        uint64_t        p_src;
+        uint64_t        q_src;
+        uint64_t        q_addr;
+};
+
+struct ioat_raw_descriptor {
+        uint64_t        field[8];
+};
+#endif
diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c
new file mode 100644
index 000000000000..d545fae30f37
--- /dev/null
+++ b/drivers/dma/ioat/pci.c
@@ -0,0 +1,210 @@
+/*
+ * Intel I/OAT DMA Linux driver
+ * Copyright(c) 2007 - 2009 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ */
+
+/*
+ * This driver supports an Intel I/OAT DMA engine, which does asynchronous
+ * copy operations.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/dca.h>
+#include "dma.h"
+#include "dma_v2.h"
+#include "registers.h"
+#include "hw.h"
+
+MODULE_VERSION(IOAT_DMA_VERSION);
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_AUTHOR("Intel Corporation");
+
+static struct pci_device_id ioat_pci_tbl[] = {
+        /* I/OAT v1 platforms */
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB)  },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) },
+        { PCI_VDEVICE(UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) },
+
+        /* I/OAT v2 platforms */
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) },
+
+        /* I/OAT v3 platforms */
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) },
+
+        /* I/OAT v3.2 platforms */
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) },
+        { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) },
+
+        { 0, }
+};
+MODULE_DEVICE_TABLE(pci, ioat_pci_tbl);
+
+static int __devinit ioat_pci_probe(struct pci_dev *pdev,
+                                    const struct pci_device_id *id);
+static void __devexit ioat_remove(struct pci_dev *pdev);
+
+static int ioat_dca_enabled = 1;
+module_param(ioat_dca_enabled, int, 0644);
+MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
+
+struct kmem_cache *ioat2_cache;
+
+#define DRV_NAME "ioatdma"
+
+static struct pci_driver ioat_pci_driver = {
+        .name           = DRV_NAME,
+        .id_table       = ioat_pci_tbl,
+        .probe          = ioat_pci_probe,
+        .remove         = __devexit_p(ioat_remove),
+};
+
+static struct ioatdma_device *
+alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase)
+{
+        struct device *dev = &pdev->dev;
+        struct ioatdma_device *d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL);
+
+        if (!d)
+                return NULL;
+        d->pdev = pdev;
+        d->reg_base = iobase;
+        return d;
+}
+
+static int __devinit ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+        void __iomem * const *iomap;
+        struct device *dev = &pdev->dev;
+        struct ioatdma_device *device;
+        int err;
+
+        err = pcim_enable_device(pdev);
+        if (err)
+                return err;
+
+        err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME);
+        if (err)
+                return err;
+        iomap = pcim_iomap_table(pdev);
+        if (!iomap)
+                return -ENOMEM;
+
+        err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
+        if (err)
+                err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+        if (err)
+                return err;
+
+        err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
+        if (err)
+                err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
+        if (err)
+                return err;
+
+        device = devm_kzalloc(dev, sizeof(*device), GFP_KERNEL);
+        if (!device)
+                return -ENOMEM;
+
+        pci_set_master(pdev);
+
+        device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]);
+        if (!device)
+                return -ENOMEM;
+        pci_set_drvdata(pdev, device);
+
+        device->version = readb(device->reg_base + IOAT_VER_OFFSET);
+        if (device->version == IOAT_VER_1_2)
+                err = ioat1_dma_probe(device, ioat_dca_enabled);
+        else if (device->version == IOAT_VER_2_0)
+                err = ioat2_dma_probe(device, ioat_dca_enabled);
+        else if (device->version >= IOAT_VER_3_0)
+                err = ioat3_dma_probe(device, ioat_dca_enabled);
+        else
+                return -ENODEV;
+
+        if (err) {
+                dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n");
+                return -ENODEV;
+        }
+
+        return 0;
+}
+
+static void __devexit ioat_remove(struct pci_dev *pdev)
+{
+        struct ioatdma_device *device = pci_get_drvdata(pdev);
+
+        if (!device)
+                return;
+
+        dev_err(&pdev->dev, "Removing dma and dca services\n");
+        if (device->dca) {
+                unregister_dca_provider(device->dca, &pdev->dev);
+                free_dca_provider(device->dca);
+                device->dca = NULL;
+        }
+        ioat_dma_remove(device);
+}
+
+static int __init ioat_init_module(void)
+{
+        int err;
+
+        pr_info("%s: Intel(R) QuickData Technology Driver %s\n",
+                DRV_NAME, IOAT_DMA_VERSION);
+
+        ioat2_cache = kmem_cache_create("ioat2", sizeof(struct ioat_ring_ent),
+                                        0, SLAB_HWCACHE_ALIGN, NULL);
+        if (!ioat2_cache)
+                return -ENOMEM;
+
+        err = pci_register_driver(&ioat_pci_driver);
+        if (err)
+                kmem_cache_destroy(ioat2_cache);
+
+        return err;
+}
+module_init(ioat_init_module);
+
+static void __exit ioat_exit_module(void)
+{
+        pci_unregister_driver(&ioat_pci_driver);
+        kmem_cache_destroy(ioat2_cache);
+}
+module_exit(ioat_exit_module);
diff --git a/drivers/dma/ioatdma_registers.h b/drivers/dma/ioat/registers.h
index 49bc277424f8..63038e18ab03 100644
--- a/drivers/dma/ioatdma_registers.h
+++ b/drivers/dma/ioat/registers.h
@@ -64,18 +64,37 @@
 
 #define IOAT_DEVICE_STATUS_OFFSET               0x0E    /* 16-bit */
 #define IOAT_DEVICE_STATUS_DEGRADED_MODE        0x0001
+#define IOAT_DEVICE_MMIO_RESTRICTED             0x0002
+#define IOAT_DEVICE_MEMORY_BYPASS               0x0004
+#define IOAT_DEVICE_ADDRESS_REMAPPING           0x0008
+
+#define IOAT_DMA_CAP_OFFSET                     0x10    /* 32-bit */
+#define IOAT_CAP_PAGE_BREAK                     0x00000001
+#define IOAT_CAP_CRC                            0x00000002
+#define IOAT_CAP_SKIP_MARKER                    0x00000004
+#define IOAT_CAP_DCA                            0x00000010
+#define IOAT_CAP_CRC_MOVE                       0x00000020
+#define IOAT_CAP_FILL_BLOCK                     0x00000040
+#define IOAT_CAP_APIC                           0x00000080
+#define IOAT_CAP_XOR                            0x00000100
+#define IOAT_CAP_PQ                             0x00000200
 
 #define IOAT_CHANNEL_MMIO_SIZE                  0x80    /* Each Channel MMIO space is this size */
 
 /* DMA Channel Registers */
 #define IOAT_CHANCTRL_OFFSET                    0x00    /* 16-bit Channel Control Register */
 #define IOAT_CHANCTRL_CHANNEL_PRIORITY_MASK     0xF000
+#define IOAT3_CHANCTRL_COMPL_DCA_EN             0x0200
 #define IOAT_CHANCTRL_CHANNEL_IN_USE            0x0100
 #define IOAT_CHANCTRL_DESCRIPTOR_ADDR_SNOOP_CONTROL     0x0020
 #define IOAT_CHANCTRL_ERR_INT_EN                0x0010
 #define IOAT_CHANCTRL_ANY_ERR_ABORT_EN          0x0008
 #define IOAT_CHANCTRL_ERR_COMPLETION_EN         0x0004
-#define IOAT_CHANCTRL_INT_DISABLE               0x0001
+#define IOAT_CHANCTRL_INT_REARM                 0x0001
+#define IOAT_CHANCTRL_RUN                       (IOAT_CHANCTRL_INT_REARM |\
+                                                 IOAT_CHANCTRL_ERR_COMPLETION_EN |\
+                                                 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |\
+                                                 IOAT_CHANCTRL_ERR_INT_EN)
 
 #define IOAT_DMA_COMP_OFFSET                    0x02    /* 16-bit DMA channel compatibility */
 #define IOAT_DMA_COMP_V1                        0x0001  /* Compatibility with DMA version 1 */
@@ -94,14 +113,14 @@
 #define IOAT2_CHANSTS_OFFSET_HIGH       0x0C
 #define IOAT_CHANSTS_OFFSET_HIGH(ver)           ((ver) < IOAT_VER_2_0 \
                                                 ? IOAT1_CHANSTS_OFFSET_HIGH : IOAT2_CHANSTS_OFFSET_HIGH)
-#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR  ~0x3F
-#define IOAT_CHANSTS_SOFT_ERR                   0x0000000000000010
-#define IOAT_CHANSTS_UNAFFILIATED_ERR           0x0000000000000008
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS        0x0000000000000007
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE 0x0
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE   0x1
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_SUSPENDED      0x2
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED 0x3
+#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR  (~0x3fULL)
+#define IOAT_CHANSTS_SOFT_ERR                   0x10ULL
+#define IOAT_CHANSTS_UNAFFILIATED_ERR           0x8ULL
+#define IOAT_CHANSTS_STATUS     0x7ULL
+#define IOAT_CHANSTS_ACTIVE     0x0
+#define IOAT_CHANSTS_DONE       0x1
+#define IOAT_CHANSTS_SUSPENDED  0x2
+#define IOAT_CHANSTS_HALTED     0x3
 
 
 
@@ -204,22 +223,27 @@
 #define IOAT_CDAR_OFFSET_HIGH                   0x24
 
 #define IOAT_CHANERR_OFFSET                     0x28    /* 32-bit Channel Error Register */
-#define IOAT_CHANERR_DMA_TRANSFER_SRC_ADDR_ERR  0x0001
-#define IOAT_CHANERR_DMA_TRANSFER_DEST_ADDR_ERR 0x0002
-#define IOAT_CHANERR_NEXT_DESCRIPTOR_ADDR_ERR   0x0004
-#define IOAT_CHANERR_NEXT_DESCRIPTOR_ALIGNMENT_ERR      0x0008
+#define IOAT_CHANERR_SRC_ADDR_ERR       0x0001
+#define IOAT_CHANERR_DEST_ADDR_ERR      0x0002
+#define IOAT_CHANERR_NEXT_ADDR_ERR      0x0004
+#define IOAT_CHANERR_NEXT_DESC_ALIGN_ERR        0x0008
 #define IOAT_CHANERR_CHAIN_ADDR_VALUE_ERR       0x0010
 #define IOAT_CHANERR_CHANCMD_ERR                0x0020
 #define IOAT_CHANERR_CHIPSET_UNCORRECTABLE_DATA_INTEGRITY_ERR   0x0040
 #define IOAT_CHANERR_DMA_UNCORRECTABLE_DATA_INTEGRITY_ERR       0x0080
 #define IOAT_CHANERR_READ_DATA_ERR              0x0100
 #define IOAT_CHANERR_WRITE_DATA_ERR             0x0200
-#define IOAT_CHANERR_DESCRIPTOR_CONTROL_ERR     0x0400
-#define IOAT_CHANERR_DESCRIPTOR_LENGTH_ERR      0x0800
+#define IOAT_CHANERR_CONTROL_ERR        0x0400
+#define IOAT_CHANERR_LENGTH_ERR 0x0800
 #define IOAT_CHANERR_COMPLETION_ADDR_ERR        0x1000
 #define IOAT_CHANERR_INT_CONFIGURATION_ERR      0x2000
 #define IOAT_CHANERR_SOFT_ERR                   0x4000
 #define IOAT_CHANERR_UNAFFILIATED_ERR           0x8000
+#define IOAT_CHANERR_XOR_P_OR_CRC_ERR           0x10000
+#define IOAT_CHANERR_XOR_Q_ERR                  0x20000
+#define IOAT_CHANERR_DESCRIPTOR_COUNT_ERR       0x40000
+
+#define IOAT_CHANERR_HANDLE_MASK (IOAT_CHANERR_XOR_P_OR_CRC_ERR | IOAT_CHANERR_XOR_Q_ERR)
 
 #define IOAT_CHANERR_MASK_OFFSET                0x2C    /* 32-bit Channel Error Register */
 
diff --git a/drivers/dma/ioat_dma.c b/drivers/dma/ioat_dma.c
deleted file mode 100644
index a600fc0f7962..000000000000
--- a/drivers/dma/ioat_dma.c
+++ /dev/null
@@ -1,1741 +0,0 @@
-/*
- * Intel I/OAT DMA Linux driver
- * Copyright(c) 2004 - 2009 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- */
-
-/*
- * This driver supports an Intel I/OAT DMA engine, which does asynchronous
- * copy operations.
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/pci.h>
-#include <linux/interrupt.h>
-#include <linux/dmaengine.h>
-#include <linux/delay.h>
-#include <linux/dma-mapping.h>
-#include <linux/workqueue.h>
-#include <linux/i7300_idle.h>
-#include "ioatdma.h"
-#include "ioatdma_registers.h"
-#include "ioatdma_hw.h"
-
-#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
-#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
-#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
-#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)
-
-#define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80)
-static int ioat_pending_level = 4;
-module_param(ioat_pending_level, int, 0644);
-MODULE_PARM_DESC(ioat_pending_level,
-                 "high-water mark for pushing ioat descriptors (default: 4)");
-
-#define RESET_DELAY  msecs_to_jiffies(100)
-#define WATCHDOG_DELAY  round_jiffies(msecs_to_jiffies(2000))
-static void ioat_dma_chan_reset_part2(struct work_struct *work);
-static void ioat_dma_chan_watchdog(struct work_struct *work);
-
-/*
- * workaround for IOAT ver.3.0 null descriptor issue
- * (channel returns error when size is 0)
- */
-#define NULL_DESC_BUFFER_SIZE 1
-
-/* internal functions */
-static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan);
-static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
-
-static struct ioat_desc_sw *
-ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);
-static struct ioat_desc_sw *
-ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);
-
-static inline struct ioat_dma_chan *ioat_lookup_chan_by_index(
-                                                struct ioatdma_device *device,
-                                                int index)
-{
-        return device->idx[index];
-}
-
-/**
- * ioat_dma_do_interrupt - handler used for single vector interrupt mode
- * @irq: interrupt id
- * @data: interrupt data
- */
-static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
-{
-        struct ioatdma_device *instance = data;
-        struct ioat_dma_chan *ioat_chan;
-        unsigned long attnstatus;
-        int bit;
-        u8 intrctrl;
-
-        intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
-
-        if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
-                return IRQ_NONE;
-
-        if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
-                writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
-                return IRQ_NONE;
-        }
-
-        attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
-        for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
-                ioat_chan = ioat_lookup_chan_by_index(instance, bit);
-                tasklet_schedule(&ioat_chan->cleanup_task);
-        }
-
-        writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
-        return IRQ_HANDLED;
-}
-
-/**
- * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
- * @irq: interrupt id
- * @data: interrupt data
- */
-static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
-{
-        struct ioat_dma_chan *ioat_chan = data;
-
-        tasklet_schedule(&ioat_chan->cleanup_task);
-
-        return IRQ_HANDLED;
-}
-
-static void ioat_dma_cleanup_tasklet(unsigned long data);
-
-/**
- * ioat_dma_enumerate_channels - find and initialize the device's channels
- * @device: the device to be enumerated
- */
-static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
-{
-        u8 xfercap_scale;
-        u32 xfercap;
-        int i;
-        struct ioat_dma_chan *ioat_chan;
-
-        /*
-         * IOAT ver.3 workarounds
-         */
-        if (device->version == IOAT_VER_3_0) {
-                u32 chan_err_mask;
-                u16 dev_id;
-                u32 dmauncerrsts;
-
-                /*
-                 * Write CHANERRMSK_INT with 3E07h to mask out the errors
-                 * that can cause stability issues for IOAT ver.3
-                 */
-                chan_err_mask = 0x3E07;
-                pci_write_config_dword(device->pdev,
-                        IOAT_PCI_CHANERRMASK_INT_OFFSET,
-                        chan_err_mask);
-
-                /*
-                 * Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
-                 * (workaround for spurious config parity error after restart)
-                 */
-                pci_read_config_word(device->pdev,
-                        IOAT_PCI_DEVICE_ID_OFFSET,
-                        &dev_id);
-                if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) {
-                        dmauncerrsts = 0x10;
-                        pci_write_config_dword(device->pdev,
-                                IOAT_PCI_DMAUNCERRSTS_OFFSET,
-                                dmauncerrsts);
-                }
-        }
-
-        device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
-        xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
-        xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
-
-#ifdef  CONFIG_I7300_IDLE_IOAT_CHANNEL
-        if (i7300_idle_platform_probe(NULL, NULL, 1) == 0) {
-                device->common.chancnt--;
-        }
-#endif
-        for (i = 0; i < device->common.chancnt; i++) {
-                ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
-                if (!ioat_chan) {
-                        device->common.chancnt = i;
-                        break;
-                }
-
-                ioat_chan->device = device;
-                ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
-                ioat_chan->xfercap = xfercap;
-                ioat_chan->desccount = 0;
-                INIT_DELAYED_WORK(&ioat_chan->work, ioat_dma_chan_reset_part2);
-                if (ioat_chan->device->version == IOAT_VER_2_0)
-                        writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE |
-                               IOAT_DMA_DCA_ANY_CPU,
-                               ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
-                else if (ioat_chan->device->version == IOAT_VER_3_0)
-                        writel(IOAT_DMA_DCA_ANY_CPU,
-                               ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
-                spin_lock_init(&ioat_chan->cleanup_lock);
-                spin_lock_init(&ioat_chan->desc_lock);
-                INIT_LIST_HEAD(&ioat_chan->free_desc);
-                INIT_LIST_HEAD(&ioat_chan->used_desc);
-                /* This should be made common somewhere in dmaengine.c */
-                ioat_chan->common.device = &device->common;
-                list_add_tail(&ioat_chan->common.device_node,
-                              &device->common.channels);
-                device->idx[i] = ioat_chan;
-                tasklet_init(&ioat_chan->cleanup_task,
-                             ioat_dma_cleanup_tasklet,
-                             (unsigned long) ioat_chan);
-                tasklet_disable(&ioat_chan->cleanup_task);
-        }
-        return device->common.chancnt;
-}
-
-/**
- * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
- *                                 descriptors to hw
- * @chan: DMA channel handle
- */
-static inline void __ioat1_dma_memcpy_issue_pending(
-                                                struct ioat_dma_chan *ioat_chan)
-{
-        ioat_chan->pending = 0;
-        writeb(IOAT_CHANCMD_APPEND, ioat_chan->reg_base + IOAT1_CHANCMD_OFFSET);
-}
-
-static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-
-        if (ioat_chan->pending > 0) {
-                spin_lock_bh(&ioat_chan->desc_lock);
-                __ioat1_dma_memcpy_issue_pending(ioat_chan);
-                spin_unlock_bh(&ioat_chan->desc_lock);
-        }
-}
-
-static inline void __ioat2_dma_memcpy_issue_pending(
-                                                struct ioat_dma_chan *ioat_chan)
-{
-        ioat_chan->pending = 0;
-        writew(ioat_chan->dmacount,
-               ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
-}
-
-static void ioat2_dma_memcpy_issue_pending(struct dma_chan *chan)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-
-        if (ioat_chan->pending > 0) {
-                spin_lock_bh(&ioat_chan->desc_lock);
-                __ioat2_dma_memcpy_issue_pending(ioat_chan);
-                spin_unlock_bh(&ioat_chan->desc_lock);
-        }
-}
-
-
-/**
- * ioat_dma_chan_reset_part2 - reinit the channel after a reset
- */
-static void ioat_dma_chan_reset_part2(struct work_struct *work)
-{
-        struct ioat_dma_chan *ioat_chan =
-                container_of(work, struct ioat_dma_chan, work.work);
-        struct ioat_desc_sw *desc;
-
-        spin_lock_bh(&ioat_chan->cleanup_lock);
-        spin_lock_bh(&ioat_chan->desc_lock);
-
-        ioat_chan->completion_virt->low = 0;
-        ioat_chan->completion_virt->high = 0;
-        ioat_chan->pending = 0;
-
-        /*
-         * count the descriptors waiting, and be sure to do it
-         * right for both the CB1 line and the CB2 ring
-         */
-        ioat_chan->dmacount = 0;
-        if (ioat_chan->used_desc.prev) {
-                desc = to_ioat_desc(ioat_chan->used_desc.prev);
-                do {
-                        ioat_chan->dmacount++;
-                        desc = to_ioat_desc(desc->node.next);
-                } while (&desc->node != ioat_chan->used_desc.next);
-        }
-
-        /*
-         * write the new starting descriptor address
-         * this puts channel engine into ARMED state
-         */
-        desc = to_ioat_desc(ioat_chan->used_desc.prev);
-        switch (ioat_chan->device->version) {
-        case IOAT_VER_1_2:
-                writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
-                       ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
-                writel(((u64) desc->async_tx.phys) >> 32,
-                       ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
-
-                writeb(IOAT_CHANCMD_START, ioat_chan->reg_base
-                        + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
-                break;
-        case IOAT_VER_2_0:
-                writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
-                       ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
-                writel(((u64) desc->async_tx.phys) >> 32,
-                       ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
-
-                /* tell the engine to go with what's left to be done */
-                writew(ioat_chan->dmacount,
-                       ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
-
-                break;
-        }
-        dev_err(&ioat_chan->device->pdev->dev,
-                "chan%d reset - %d descs waiting, %d total desc\n",
-                chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
-
-        spin_unlock_bh(&ioat_chan->desc_lock);
-        spin_unlock_bh(&ioat_chan->cleanup_lock);
-}
-
-/**
- * ioat_dma_reset_channel - restart a channel
- * @ioat_chan: IOAT DMA channel handle
- */
-static void ioat_dma_reset_channel(struct ioat_dma_chan *ioat_chan)
-{
-        u32 chansts, chanerr;
-
-        if (!ioat_chan->used_desc.prev)
-                return;
-
-        chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
-        chansts = (ioat_chan->completion_virt->low
-                                        & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
-        if (chanerr) {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
-                        chan_num(ioat_chan), chansts, chanerr);
-                writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
-        }
-
-        /*
-         * whack it upside the head with a reset
-         * and wait for things to settle out.
-         * force the pending count to a really big negative
-         * to make sure no one forces an issue_pending
-         * while we're waiting.
-         */
-
-        spin_lock_bh(&ioat_chan->desc_lock);
-        ioat_chan->pending = INT_MIN;
-        writeb(IOAT_CHANCMD_RESET,
-               ioat_chan->reg_base
-               + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        /* schedule the 2nd half instead of sleeping a long time */
-        schedule_delayed_work(&ioat_chan->work, RESET_DELAY);
-}
-
-/**
- * ioat_dma_chan_watchdog - watch for stuck channels
- */
-static void ioat_dma_chan_watchdog(struct work_struct *work)
-{
-        struct ioatdma_device *device =
-                container_of(work, struct ioatdma_device, work.work);
-        struct ioat_dma_chan *ioat_chan;
-        int i;
-
-        union {
-                u64 full;
-                struct {
-                        u32 low;
-                        u32 high;
-                };
-        } completion_hw;
-        unsigned long compl_desc_addr_hw;
-
-        for (i = 0; i < device->common.chancnt; i++) {
-                ioat_chan = ioat_lookup_chan_by_index(device, i);
-
-                if (ioat_chan->device->version == IOAT_VER_1_2
-                        /* have we started processing anything yet */
-                    && ioat_chan->last_completion
-                        /* have we completed any since last watchdog cycle? */
-                    && (ioat_chan->last_completion ==
-                                ioat_chan->watchdog_completion)
-                        /* has TCP stuck on one cookie since last watchdog? */
-                    && (ioat_chan->watchdog_tcp_cookie ==
-                                ioat_chan->watchdog_last_tcp_cookie)
-                    && (ioat_chan->watchdog_tcp_cookie !=
-                                ioat_chan->completed_cookie)
-                        /* is there something in the chain to be processed? */
-                        /* CB1 chain always has at least the last one processed */
-                    && (ioat_chan->used_desc.prev != ioat_chan->used_desc.next)
-                    && ioat_chan->pending == 0) {
-
-                        /*
-                         * check CHANSTS register for completed
-                         * descriptor address.
-                         * if it is different than completion writeback,
-                         * it is not zero
-                         * and it has changed since the last watchdog
-                         *     we can assume that channel
-                         *     is still working correctly
-                         *     and the problem is in completion writeback.
-                         *     update completion writeback
-                         *     with actual CHANSTS value
-                         * else
-                         *     try resetting the channel
-                         */
-
-                        completion_hw.low = readl(ioat_chan->reg_base +
-                                IOAT_CHANSTS_OFFSET_LOW(ioat_chan->device->version));
-                        completion_hw.high = readl(ioat_chan->reg_base +
-                                IOAT_CHANSTS_OFFSET_HIGH(ioat_chan->device->version));
-#if (BITS_PER_LONG == 64)
-                        compl_desc_addr_hw =
-                                completion_hw.full
-                                & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
-#else
-                        compl_desc_addr_hw =
-                                completion_hw.low & IOAT_LOW_COMPLETION_MASK;
-#endif
-
-                        if ((compl_desc_addr_hw != 0)
-                           && (compl_desc_addr_hw != ioat_chan->watchdog_completion)
-                           && (compl_desc_addr_hw != ioat_chan->last_compl_desc_addr_hw)) {
-                                ioat_chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
-                                ioat_chan->completion_virt->low = completion_hw.low;
-                                ioat_chan->completion_virt->high = completion_hw.high;
-                        } else {
-                                ioat_dma_reset_channel(ioat_chan);
-                                ioat_chan->watchdog_completion = 0;
-                                ioat_chan->last_compl_desc_addr_hw = 0;
-                        }
-
-                /*
-                 * for version 2.0 if there are descriptors yet to be processed
-                 * and the last completed hasn't changed since the last watchdog
-                 *      if they haven't hit the pending level
-                 *          issue the pending to push them through
-                 *      else
-                 *          try resetting the channel
-                 */
-                } else if (ioat_chan->device->version == IOAT_VER_2_0
-                    && ioat_chan->used_desc.prev
-                    && ioat_chan->last_completion
-                    && ioat_chan->last_completion == ioat_chan->watchdog_completion) {
-
-                        if (ioat_chan->pending < ioat_pending_level)
-                                ioat2_dma_memcpy_issue_pending(&ioat_chan->common);
-                        else {
-                                ioat_dma_reset_channel(ioat_chan);
-                                ioat_chan->watchdog_completion = 0;
-                        }
-                } else {
-                        ioat_chan->last_compl_desc_addr_hw = 0;
-                        ioat_chan->watchdog_completion
-                                        = ioat_chan->last_completion;
-                }
-
-                ioat_chan->watchdog_last_tcp_cookie =
-                        ioat_chan->watchdog_tcp_cookie;
-        }
-
-        schedule_delayed_work(&device->work, WATCHDOG_DELAY);
-}
-
-static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
-        struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
-        struct ioat_desc_sw *prev, *new;
-        struct ioat_dma_descriptor *hw;
-        dma_cookie_t cookie;
-        LIST_HEAD(new_chain);
-        u32 copy;
-        size_t len;
-        dma_addr_t src, dst;
-        unsigned long orig_flags;
-        unsigned int desc_count = 0;
-
-        /* src and dest and len are stored in the initial descriptor */
-        len = first->len;
-        src = first->src;
-        dst = first->dst;
-        orig_flags = first->async_tx.flags;
-        new = first;
-
-        spin_lock_bh(&ioat_chan->desc_lock);
-        prev = to_ioat_desc(ioat_chan->used_desc.prev);
-        prefetch(prev->hw);
-        do {
-                copy = min_t(size_t, len, ioat_chan->xfercap);
-
-                async_tx_ack(&new->async_tx);
-
-                hw = new->hw;
-                hw->size = copy;
-                hw->ctl = 0;
-                hw->src_addr = src;
-                hw->dst_addr = dst;
-                hw->next = 0;
-
-                /* chain together the physical address list for the HW */
-                wmb();
-                prev->hw->next = (u64) new->async_tx.phys;
-
-                len -= copy;
-                dst += copy;
-                src += copy;
-
-                list_add_tail(&new->node, &new_chain);
-                desc_count++;
-                prev = new;
-        } while (len && (new = ioat1_dma_get_next_descriptor(ioat_chan)));
-
-        if (!new) {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "tx submit failed\n");
-                spin_unlock_bh(&ioat_chan->desc_lock);
-                return -ENOMEM;
-        }
-
-        hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
-        if (first->async_tx.callback) {
-                hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;
-                if (first != new) {
-                        /* move callback into to last desc */
-                        new->async_tx.callback = first->async_tx.callback;
-                        new->async_tx.callback_param
-                                        = first->async_tx.callback_param;
-                        first->async_tx.callback = NULL;
-                        first->async_tx.callback_param = NULL;
-                }
-        }
-
-        new->tx_cnt = desc_count;
-        new->async_tx.flags = orig_flags; /* client is in control of this ack */
-
-        /* store the original values for use in later cleanup */
-        if (new != first) {
-                new->src = first->src;
-                new->dst = first->dst;
-                new->len = first->len;
-        }
-
-        /* cookie incr and addition to used_list must be atomic */
-        cookie = ioat_chan->common.cookie;
-        cookie++;
-        if (cookie < 0)
-                cookie = 1;
-        ioat_chan->common.cookie = new->async_tx.cookie = cookie;
-
-        /* write address into NextDescriptor field of last desc in chain */
-        to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
-                                                        first->async_tx.phys;
-        list_splice_tail(&new_chain, &ioat_chan->used_desc);
-
-        ioat_chan->dmacount += desc_count;
-        ioat_chan->pending += desc_count;
-        if (ioat_chan->pending >= ioat_pending_level)
-                __ioat1_dma_memcpy_issue_pending(ioat_chan);
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        return cookie;
-}
-
-static dma_cookie_t ioat2_tx_submit(struct dma_async_tx_descriptor *tx)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
-        struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
-        struct ioat_desc_sw *new;
-        struct ioat_dma_descriptor *hw;
-        dma_cookie_t cookie;
-        u32 copy;
-        size_t len;
-        dma_addr_t src, dst;
-        unsigned long orig_flags;
-        unsigned int desc_count = 0;
-
-        /* src and dest and len are stored in the initial descriptor */
-        len = first->len;
-        src = first->src;
-        dst = first->dst;
-        orig_flags = first->async_tx.flags;
-        new = first;
-
-        /*
-         * ioat_chan->desc_lock is still in force in version 2 path
-         * it gets unlocked at end of this function
-         */
-        do {
-                copy = min_t(size_t, len, ioat_chan->xfercap);
-
-                async_tx_ack(&new->async_tx);
-
-                hw = new->hw;
-                hw->size = copy;
-                hw->ctl = 0;
-                hw->src_addr = src;
-                hw->dst_addr = dst;
-
-                len -= copy;
-                dst += copy;
-                src += copy;
-                desc_count++;
-        } while (len && (new = ioat2_dma_get_next_descriptor(ioat_chan)));
-
-        if (!new) {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "tx submit failed\n");
-                spin_unlock_bh(&ioat_chan->desc_lock);
-                return -ENOMEM;
-        }
-
-        hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
-        if (first->async_tx.callback) {
-                hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;
-                if (first != new) {
-                        /* move callback into to last desc */
-                        new->async_tx.callback = first->async_tx.callback;
-                        new->async_tx.callback_param
-                                        = first->async_tx.callback_param;
-                        first->async_tx.callback = NULL;
-                        first->async_tx.callback_param = NULL;
-                }
-        }
-
-        new->tx_cnt = desc_count;
-        new->async_tx.flags = orig_flags; /* client is in control of this ack */
-
-        /* store the original values for use in later cleanup */
-        if (new != first) {
-                new->src = first->src;
-                new->dst = first->dst;
-                new->len = first->len;
-        }
-
-        /* cookie incr and addition to used_list must be atomic */
-        cookie = ioat_chan->common.cookie;
-        cookie++;
-        if (cookie < 0)
-                cookie = 1;
-        ioat_chan->common.cookie = new->async_tx.cookie = cookie;
-
-        ioat_chan->dmacount += desc_count;
-        ioat_chan->pending += desc_count;
-        if (ioat_chan->pending >= ioat_pending_level)
-                __ioat2_dma_memcpy_issue_pending(ioat_chan);
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        return cookie;
-}
-
-/**
- * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
- * @ioat_chan: the channel supplying the memory pool for the descriptors
- * @flags: allocation flags
- */
-static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
-                                        struct ioat_dma_chan *ioat_chan,
-                                        gfp_t flags)
-{
-        struct ioat_dma_descriptor *desc;
-        struct ioat_desc_sw *desc_sw;
-        struct ioatdma_device *ioatdma_device;
-        dma_addr_t phys;
-
-        ioatdma_device = to_ioatdma_device(ioat_chan->common.device);
-        desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
-        if (unlikely(!desc))
-                return NULL;
-
-        desc_sw = kzalloc(sizeof(*desc_sw), flags);
-        if (unlikely(!desc_sw)) {
-                pci_pool_free(ioatdma_device->dma_pool, desc, phys);
-                return NULL;
-        }
-
-        memset(desc, 0, sizeof(*desc));
-        dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);
-        switch (ioat_chan->device->version) {
-        case IOAT_VER_1_2:
-                desc_sw->async_tx.tx_submit = ioat1_tx_submit;
-                break;
-        case IOAT_VER_2_0:
-        case IOAT_VER_3_0:
-                desc_sw->async_tx.tx_submit = ioat2_tx_submit;
-                break;
-        }
-
-        desc_sw->hw = desc;
-        desc_sw->async_tx.phys = phys;
-
-        return desc_sw;
-}
-
-static int ioat_initial_desc_count = 256;
-module_param(ioat_initial_desc_count, int, 0644);
-MODULE_PARM_DESC(ioat_initial_desc_count,
-                 "initial descriptors per channel (default: 256)");
-
-/**
- * ioat2_dma_massage_chan_desc - link the descriptors into a circle
- * @ioat_chan: the channel to be massaged
- */
-static void ioat2_dma_massage_chan_desc(struct ioat_dma_chan *ioat_chan)
-{
-        struct ioat_desc_sw *desc, *_desc;
-
-        /* setup used_desc */
-        ioat_chan->used_desc.next = ioat_chan->free_desc.next;
-        ioat_chan->used_desc.prev = NULL;
-
-        /* pull free_desc out of the circle so that every node is a hw
-         * descriptor, but leave it pointing to the list
-         */
-        ioat_chan->free_desc.prev->next = ioat_chan->free_desc.next;
-        ioat_chan->free_desc.next->prev = ioat_chan->free_desc.prev;
-
-        /* circle link the hw descriptors */
-        desc = to_ioat_desc(ioat_chan->free_desc.next);
-        desc->hw->next = to_ioat_desc(desc->node.next)->async_tx.phys;
-        list_for_each_entry_safe(desc, _desc, ioat_chan->free_desc.next, node) {
-                desc->hw->next = to_ioat_desc(desc->node.next)->async_tx.phys;
-        }
-}
-
-/**
- * ioat_dma_alloc_chan_resources - returns the number of allocated descriptors
- * @chan: the channel to be filled out
- */
-static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-        struct ioat_desc_sw *desc;
-        u16 chanctrl;
-        u32 chanerr;
-        int i;
-        LIST_HEAD(tmp_list);
-
-        /* have we already been set up? */
-        if (!list_empty(&ioat_chan->free_desc))
-                return ioat_chan->desccount;
-
-        /* Setup register to interrupt and write completion status on error */
-        chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
-                IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
-                IOAT_CHANCTRL_ERR_COMPLETION_EN;
-        writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
-
-        chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
-        if (chanerr) {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "CHANERR = %x, clearing\n", chanerr);
-                writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
-        }
-
-        /* Allocate descriptors */
-        for (i = 0; i < ioat_initial_desc_count; i++) {
-                desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
-                if (!desc) {
-                        dev_err(&ioat_chan->device->pdev->dev,
-                                "Only %d initial descriptors\n", i);
-                        break;
-                }
-                list_add_tail(&desc->node, &tmp_list);
-        }
-        spin_lock_bh(&ioat_chan->desc_lock);
-        ioat_chan->desccount = i;
-        list_splice(&tmp_list, &ioat_chan->free_desc);
-        if (ioat_chan->device->version != IOAT_VER_1_2)
-                ioat2_dma_massage_chan_desc(ioat_chan);
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        /* allocate a completion writeback area */
-        /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
-        ioat_chan->completion_virt =
-                pci_pool_alloc(ioat_chan->device->completion_pool,
-                               GFP_KERNEL,
-                               &ioat_chan->completion_addr);
-        memset(ioat_chan->completion_virt, 0,
-               sizeof(*ioat_chan->completion_virt));
-        writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
-               ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
-        writel(((u64) ioat_chan->completion_addr) >> 32,
-               ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
-
-        tasklet_enable(&ioat_chan->cleanup_task);
-        ioat_dma_start_null_desc(ioat_chan);  /* give chain to dma device */
-        return ioat_chan->desccount;
-}
-
-/**
- * ioat_dma_free_chan_resources - release all the descriptors
- * @chan: the channel to be cleaned
- */
-static void ioat_dma_free_chan_resources(struct dma_chan *chan)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-        struct ioatdma_device *ioatdma_device = to_ioatdma_device(chan->device);
-        struct ioat_desc_sw *desc, *_desc;
-        int in_use_descs = 0;
-
-        /* Before freeing channel resources first check
-         * if they have been previously allocated for this channel.
-         */
-        if (ioat_chan->desccount == 0)
-                return;
-
-        tasklet_disable(&ioat_chan->cleanup_task);
-        ioat_dma_memcpy_cleanup(ioat_chan);
-
-        /* Delay 100ms after reset to allow internal DMA logic to quiesce
-         * before removing DMA descriptor resources.
-         */
-        writeb(IOAT_CHANCMD_RESET,
-               ioat_chan->reg_base
-                        + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
-        mdelay(100);
-
-        spin_lock_bh(&ioat_chan->desc_lock);
-        switch (ioat_chan->device->version) {
-        case IOAT_VER_1_2:
-                list_for_each_entry_safe(desc, _desc,
-                                         &ioat_chan->used_desc, node) {
-                        in_use_descs++;
-                        list_del(&desc->node);
-                        pci_pool_free(ioatdma_device->dma_pool, desc->hw,
-                                      desc->async_tx.phys);
-                        kfree(desc);
-                }
-                list_for_each_entry_safe(desc, _desc,
-                                         &ioat_chan->free_desc, node) {
-                        list_del(&desc->node);
-                        pci_pool_free(ioatdma_device->dma_pool, desc->hw,
-                                      desc->async_tx.phys);
-                        kfree(desc);
-                }
-                break;
-        case IOAT_VER_2_0:
-        case IOAT_VER_3_0:
-                list_for_each_entry_safe(desc, _desc,
-                                         ioat_chan->free_desc.next, node) {
-                        list_del(&desc->node);
-                        pci_pool_free(ioatdma_device->dma_pool, desc->hw,
-                                      desc->async_tx.phys);
-                        kfree(desc);
-                }
-                desc = to_ioat_desc(ioat_chan->free_desc.next);
-                pci_pool_free(ioatdma_device->dma_pool, desc->hw,
-                              desc->async_tx.phys);
-                kfree(desc);
-                INIT_LIST_HEAD(&ioat_chan->free_desc);
-                INIT_LIST_HEAD(&ioat_chan->used_desc);
-                break;
-        }
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        pci_pool_free(ioatdma_device->completion_pool,
-                      ioat_chan->completion_virt,
-                      ioat_chan->completion_addr);
-
-        /* one is ok since we left it on there on purpose */
-        if (in_use_descs > 1)
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "Freeing %d in use descriptors!\n",
-                        in_use_descs - 1);
-
-        ioat_chan->last_completion = ioat_chan->completion_addr = 0;
-        ioat_chan->pending = 0;
-        ioat_chan->dmacount = 0;
-        ioat_chan->desccount = 0;
-        ioat_chan->watchdog_completion = 0;
-        ioat_chan->last_compl_desc_addr_hw = 0;
-        ioat_chan->watchdog_tcp_cookie =
-                ioat_chan->watchdog_last_tcp_cookie = 0;
-}
-
-/**
- * ioat_dma_get_next_descriptor - return the next available descriptor
- * @ioat_chan: IOAT DMA channel handle
- *
- * Gets the next descriptor from the chain, and must be called with the
- * channel's desc_lock held.  Allocates more descriptors if the channel
- * has run out.
- */
-static struct ioat_desc_sw *
-ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
-{
-        struct ioat_desc_sw *new;
-
-        if (!list_empty(&ioat_chan->free_desc)) {
-                new = to_ioat_desc(ioat_chan->free_desc.next);
-                list_del(&new->node);
-        } else {
-                /* try to get another desc */
-                new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
-                if (!new) {
-                        dev_err(&ioat_chan->device->pdev->dev,
-                                "alloc failed\n");
-                        return NULL;
-                }
-        }
-
-        prefetch(new->hw);
-        return new;
-}
-
-static struct ioat_desc_sw *
-ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
-{
-        struct ioat_desc_sw *new;
-
-        /*
-         * used.prev points to where to start processing
-         * used.next points to next free descriptor
-         * if used.prev == NULL, there are none waiting to be processed
-         * if used.next == used.prev.prev, there is only one free descriptor,
-         *      and we need to use it to as a noop descriptor before
-         *      linking in a new set of descriptors, since the device
-         *      has probably already read the pointer to it
-         */
-        if (ioat_chan->used_desc.prev &&
-            ioat_chan->used_desc.next == ioat_chan->used_desc.prev->prev) {
-
-                struct ioat_desc_sw *desc;
-                struct ioat_desc_sw *noop_desc;
-                int i;
-
-                /* set up the noop descriptor */
-                noop_desc = to_ioat_desc(ioat_chan->used_desc.next);
-                /* set size to non-zero value (channel returns error when size is 0) */
-                noop_desc->hw->size = NULL_DESC_BUFFER_SIZE;
-                noop_desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
-                noop_desc->hw->src_addr = 0;
-                noop_desc->hw->dst_addr = 0;
-
-                ioat_chan->used_desc.next = ioat_chan->used_desc.next->next;
-                ioat_chan->pending++;
-                ioat_chan->dmacount++;
-
-                /* try to get a few more descriptors */
-                for (i = 16; i; i--) {
-                        desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
-                        if (!desc) {
-                                dev_err(&ioat_chan->device->pdev->dev,
-                                        "alloc failed\n");
-                                break;
-                        }
-                        list_add_tail(&desc->node, ioat_chan->used_desc.next);
-
-                        desc->hw->next
-                                = to_ioat_desc(desc->node.next)->async_tx.phys;
-                        to_ioat_desc(desc->node.prev)->hw->next
-                                = desc->async_tx.phys;
-                        ioat_chan->desccount++;
-                }
-
-                ioat_chan->used_desc.next = noop_desc->node.next;
-        }
-        new = to_ioat_desc(ioat_chan->used_desc.next);
-        prefetch(new);
-        ioat_chan->used_desc.next = new->node.next;
-
-        if (ioat_chan->used_desc.prev == NULL)
-                ioat_chan->used_desc.prev = &new->node;
-
-        prefetch(new->hw);
-        return new;
-}
-
-static struct ioat_desc_sw *ioat_dma_get_next_descriptor(
-                                                struct ioat_dma_chan *ioat_chan)
-{
-        if (!ioat_chan)
-                return NULL;
-
-        switch (ioat_chan->device->version) {
-        case IOAT_VER_1_2:
-                return ioat1_dma_get_next_descriptor(ioat_chan);
-        case IOAT_VER_2_0:
-        case IOAT_VER_3_0:
-                return ioat2_dma_get_next_descriptor(ioat_chan);
-        }
-        return NULL;
-}
-
-static struct dma_async_tx_descriptor *ioat1_dma_prep_memcpy(
-                                                struct dma_chan *chan,
-                                                dma_addr_t dma_dest,
-                                                dma_addr_t dma_src,
-                                                size_t len,
-                                                unsigned long flags)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-        struct ioat_desc_sw *new;
-
-        spin_lock_bh(&ioat_chan->desc_lock);
-        new = ioat_dma_get_next_descriptor(ioat_chan);
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        if (new) {
-                new->len = len;
-                new->dst = dma_dest;
-                new->src = dma_src;
-                new->async_tx.flags = flags;
-                return &new->async_tx;
-        } else {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
-                        chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
-                return NULL;
-        }
-}
-
-static struct dma_async_tx_descriptor *ioat2_dma_prep_memcpy(
-                                                struct dma_chan *chan,
-                                                dma_addr_t dma_dest,
-                                                dma_addr_t dma_src,
-                                                size_t len,
-                                                unsigned long flags)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-        struct ioat_desc_sw *new;
-
-        spin_lock_bh(&ioat_chan->desc_lock);
-        new = ioat2_dma_get_next_descriptor(ioat_chan);
-
-        /*
-         * leave ioat_chan->desc_lock set in ioat 2 path
-         * it will get unlocked at end of tx_submit
-         */
-
-        if (new) {
-                new->len = len;
-                new->dst = dma_dest;
-                new->src = dma_src;
-                new->async_tx.flags = flags;
-                return &new->async_tx;
-        } else {
-                spin_unlock_bh(&ioat_chan->desc_lock);
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
-                        chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
-                return NULL;
-        }
-}
-
-static void ioat_dma_cleanup_tasklet(unsigned long data)
-{
-        struct ioat_dma_chan *chan = (void *)data;
-        ioat_dma_memcpy_cleanup(chan);
-        writew(IOAT_CHANCTRL_INT_DISABLE,
-               chan->reg_base + IOAT_CHANCTRL_OFFSET);
-}
-
-static void
-ioat_dma_unmap(struct ioat_dma_chan *ioat_chan, struct ioat_desc_sw *desc)
-{
-        if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
-                if (desc->async_tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
-                        pci_unmap_single(ioat_chan->device->pdev,
-                                         pci_unmap_addr(desc, dst),
-                                         pci_unmap_len(desc, len),
-                                         PCI_DMA_FROMDEVICE);
-                else
-                        pci_unmap_page(ioat_chan->device->pdev,
-                                       pci_unmap_addr(desc, dst),
-                                       pci_unmap_len(desc, len),
-                                       PCI_DMA_FROMDEVICE);
-        }
-
-        if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
-                if (desc->async_tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
-                        pci_unmap_single(ioat_chan->device->pdev,
-                                         pci_unmap_addr(desc, src),
-                                         pci_unmap_len(desc, len),
-                                         PCI_DMA_TODEVICE);
-                else
-                        pci_unmap_page(ioat_chan->device->pdev,
-                                       pci_unmap_addr(desc, src),
-                                       pci_unmap_len(desc, len),
-                                       PCI_DMA_TODEVICE);
-        }
-}
-
-/**
- * ioat_dma_memcpy_cleanup - cleanup up finished descriptors
- * @chan: ioat channel to be cleaned up
- */
-static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan)
-{
-        unsigned long phys_complete;
-        struct ioat_desc_sw *desc, *_desc;
-        dma_cookie_t cookie = 0;
-        unsigned long desc_phys;
-        struct ioat_desc_sw *latest_desc;
-
-        prefetch(ioat_chan->completion_virt);
-
-        if (!spin_trylock_bh(&ioat_chan->cleanup_lock))
-                return;
-
-        /* The completion writeback can happen at any time,
-           so reads by the driver need to be atomic operations
-           The descriptor physical addresses are limited to 32-bits
-           when the CPU can only do a 32-bit mov */
-
-#if (BITS_PER_LONG == 64)
-        phys_complete =
-                ioat_chan->completion_virt->full
-                & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
-#else
-        phys_complete =
-                ioat_chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
-#endif
-
-        if ((ioat_chan->completion_virt->full
-                & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
-                                IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "Channel halted, chanerr = %x\n",
-                        readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET));
-
-                /* TODO do something to salvage the situation */
-        }
-
-        if (phys_complete == ioat_chan->last_completion) {
-                spin_unlock_bh(&ioat_chan->cleanup_lock);
-                /*
-                 * perhaps we're stuck so hard that the watchdog can't go off?
-                 * try to catch it after 2 seconds
-                 */
-                if (ioat_chan->device->version != IOAT_VER_3_0) {
-                        if (time_after(jiffies,
-                                       ioat_chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
-                                ioat_dma_chan_watchdog(&(ioat_chan->device->work.work));
-                                ioat_chan->last_completion_time = jiffies;
-                        }
-                }
-                return;
-        }
-        ioat_chan->last_completion_time = jiffies;
-
-        cookie = 0;
-        if (!spin_trylock_bh(&ioat_chan->desc_lock)) {
-                spin_unlock_bh(&ioat_chan->cleanup_lock);
-                return;
-        }
-
-        switch (ioat_chan->device->version) {
-        case IOAT_VER_1_2:
-                list_for_each_entry_safe(desc, _desc,
-                                         &ioat_chan->used_desc, node) {
-
-                        /*
-                         * Incoming DMA requests may use multiple descriptors,
-                         * due to exceeding xfercap, perhaps. If so, only the
-                         * last one will have a cookie, and require unmapping.
-                         */
-                        if (desc->async_tx.cookie) {
-                                cookie = desc->async_tx.cookie;
-                                ioat_dma_unmap(ioat_chan, desc);
-                                if (desc->async_tx.callback) {
-                                        desc->async_tx.callback(desc->async_tx.callback_param);
-                                        desc->async_tx.callback = NULL;
-                                }
-                        }
-
-                        if (desc->async_tx.phys != phys_complete) {
-                                /*
-                                 * a completed entry, but not the last, so clean
-                                 * up if the client is done with the descriptor
-                                 */
-                                if (async_tx_test_ack(&desc->async_tx)) {
-                                        list_move_tail(&desc->node,
-                                                       &ioat_chan->free_desc);
-                                } else
-                                        desc->async_tx.cookie = 0;
-                        } else {
-                                /*
-                                 * last used desc. Do not remove, so we can
-                                 * append from it, but don't look at it next
-                                 * time, either
-                                 */
-                                desc->async_tx.cookie = 0;
-
-                                /* TODO check status bits? */
-                                break;
-                        }
-                }
-                break;
-        case IOAT_VER_2_0:
-        case IOAT_VER_3_0:
-                /* has some other thread has already cleaned up? */
-                if (ioat_chan->used_desc.prev == NULL)
-                        break;
-
-                /* work backwards to find latest finished desc */
-                desc = to_ioat_desc(ioat_chan->used_desc.next);
-                latest_desc = NULL;
-                do {
-                        desc = to_ioat_desc(desc->node.prev);
-                        desc_phys = (unsigned long)desc->async_tx.phys
-                                       & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
-                        if (desc_phys == phys_complete) {
-                                latest_desc = desc;
-                                break;
-                        }
-                } while (&desc->node != ioat_chan->used_desc.prev);
-
-                if (latest_desc != NULL) {
-
-                        /* work forwards to clear finished descriptors */
-                        for (desc = to_ioat_desc(ioat_chan->used_desc.prev);
-                             &desc->node != latest_desc->node.next &&
-                             &desc->node != ioat_chan->used_desc.next;
-                             desc = to_ioat_desc(desc->node.next)) {
-                                if (desc->async_tx.cookie) {
-                                        cookie = desc->async_tx.cookie;
-                                        desc->async_tx.cookie = 0;
-                                        ioat_dma_unmap(ioat_chan, desc);
-                                        if (desc->async_tx.callback) {
-                                                desc->async_tx.callback(desc->async_tx.callback_param);
-                                                desc->async_tx.callback = NULL;
-                                        }
-                                }
-                        }
-
-                        /* move used.prev up beyond those that are finished */
-                        if (&desc->node == ioat_chan->used_desc.next)
-                                ioat_chan->used_desc.prev = NULL;
-                        else
-                                ioat_chan->used_desc.prev = &desc->node;
-                }
-                break;
-        }
-
-        spin_unlock_bh(&ioat_chan->desc_lock);
-
-        ioat_chan->last_completion = phys_complete;
-        if (cookie != 0)
-                ioat_chan->completed_cookie = cookie;
-
-        spin_unlock_bh(&ioat_chan->cleanup_lock);
-}
-
-/**
- * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
- * @chan: IOAT DMA channel handle
- * @cookie: DMA transaction identifier
- * @done: if not %NULL, updated with last completed transaction
- * @used: if not %NULL, updated with last used transaction
- */
-static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
-                                            dma_cookie_t cookie,
-                                            dma_cookie_t *done,
-                                            dma_cookie_t *used)
-{
-        struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
-        dma_cookie_t last_used;
-        dma_cookie_t last_complete;
-        enum dma_status ret;
-
-        last_used = chan->cookie;
-        last_complete = ioat_chan->completed_cookie;
-        ioat_chan->watchdog_tcp_cookie = cookie;
-
-        if (done)
-                *done = last_complete;
-        if (used)
-                *used = last_used;
-
-        ret = dma_async_is_complete(cookie, last_complete, last_used);
-        if (ret == DMA_SUCCESS)
-                return ret;
-
-        ioat_dma_memcpy_cleanup(ioat_chan);
-
-        last_used = chan->cookie;
-        last_complete = ioat_chan->completed_cookie;
-
-        if (done)
-                *done = last_complete;
-        if (used)
-                *used = last_used;
-
-        return dma_async_is_complete(cookie, last_complete, last_used);
-}
-
-static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan)
-{
-        struct ioat_desc_sw *desc;
-
-        spin_lock_bh(&ioat_chan->desc_lock);
-
-        desc = ioat_dma_get_next_descriptor(ioat_chan);
-
-        if (!desc) {
-                dev_err(&ioat_chan->device->pdev->dev,
-                        "Unable to start null desc - get next desc failed\n");
-                spin_unlock_bh(&ioat_chan->desc_lock);
-                return;
-        }
-
-        desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL
-                                | IOAT_DMA_DESCRIPTOR_CTL_INT_GN
-                                | IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
-        /* set size to non-zero value (channel returns error when size is 0) */
-        desc->hw->size = NULL_DESC_BUFFER_SIZE;
-        desc->hw->src_addr = 0;
-        desc->hw->dst_addr = 0;
-        async_tx_ack(&desc->async_tx);
-        switch (ioat_chan->device->version) {
-        case IOAT_VER_1_2:
-                desc->hw->next = 0;
-                list_add_tail(&desc->node, &ioat_chan->used_desc);
-
-                writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
-                       ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
-                writel(((u64) desc->async_tx.phys) >> 32,
-                       ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
-
-                writeb(IOAT_CHANCMD_START, ioat_chan->reg_base
-                        + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
-                break;
-        case IOAT_VER_2_0:
-        case IOAT_VER_3_0:
-                writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
-                       ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
-                writel(((u64) desc->async_tx.phys) >> 32,
-                       ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
-
-                ioat_chan->dmacount++;
-                __ioat2_dma_memcpy_issue_pending(ioat_chan);
-                break;
-        }
-        spin_unlock_bh(&ioat_chan->desc_lock);
-}
-
-/*
- * Perform a IOAT transaction to verify the HW works.
- */
-#define IOAT_TEST_SIZE 2000
-
-static void ioat_dma_test_callback(void *dma_async_param)
-{
-        struct completion *cmp = dma_async_param;
-
-        complete(cmp);
-}
-
-/**
- * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
- * @device: device to be tested
- */
-static int ioat_dma_self_test(struct ioatdma_device *device)
-{
-        int i;
-        u8 *src;
-        u8 *dest;
-        struct dma_chan *dma_chan;
-        struct dma_async_tx_descriptor *tx;
-        dma_addr_t dma_dest, dma_src;
-        dma_cookie_t cookie;
-        int err = 0;
-        struct completion cmp;
-        unsigned long tmo;
-        unsigned long flags;
-
-        src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
-        if (!src)
-                return -ENOMEM;
-        dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
-        if (!dest) {
-                kfree(src);
-                return -ENOMEM;
-        }
-
-        /* Fill in src buffer */
-        for (i = 0; i < IOAT_TEST_SIZE; i++)
-                src[i] = (u8)i;
-
-        /* Start copy, using first DMA channel */
-        dma_chan = container_of(device->common.channels.next,
-                                struct dma_chan,
-                                device_node);
-        if (device->common.device_alloc_chan_resources(dma_chan) < 1) {
-                dev_err(&device->pdev->dev,
-                        "selftest cannot allocate chan resource\n");
-                err = -ENODEV;
-                goto out;
-        }
-
-        dma_src = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE,
-                                 DMA_TO_DEVICE);
-        dma_dest = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE,
-                                  DMA_FROM_DEVICE);
-        flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE;
-        tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
-                                                   IOAT_TEST_SIZE, flags);
-        if (!tx) {
-                dev_err(&device->pdev->dev,
-                        "Self-test prep failed, disabling\n");
-                err = -ENODEV;
-                goto free_resources;
-        }
-
-        async_tx_ack(tx);
-        init_completion(&cmp);
-        tx->callback = ioat_dma_test_callback;
-        tx->callback_param = &cmp;
-        cookie = tx->tx_submit(tx);
-        if (cookie < 0) {
-                dev_err(&device->pdev->dev,
-                        "Self-test setup failed, disabling\n");
-                err = -ENODEV;
-                goto free_resources;
-        }
-        device->common.device_issue_pending(dma_chan);
-
-        tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
-
-        if (tmo == 0 ||
-            device->common.device_is_tx_complete(dma_chan, cookie, NULL, NULL)
-                                        != DMA_SUCCESS) {
-                dev_err(&device->pdev->dev,
-                        "Self-test copy timed out, disabling\n");
-                err = -ENODEV;
-                goto free_resources;
-        }
-        if (memcmp(src, dest, IOAT_TEST_SIZE)) {
-                dev_err(&device->pdev->dev,
-                        "Self-test copy failed compare, disabling\n");
-                err = -ENODEV;
-                goto free_resources;
-        }
-
-free_resources:
-        device->common.device_free_chan_resources(dma_chan);
-out:
-        kfree(src);
-        kfree(dest);
-        return err;
-}
-
-static char ioat_interrupt_style[32] = "msix";
-module_param_string(ioat_interrupt_style, ioat_interrupt_style,
-                    sizeof(ioat_interrupt_style), 0644);
-MODULE_PARM_DESC(ioat_interrupt_style,
-                 "set ioat interrupt style: msix (default), "
-                 "msix-single-vector, msi, intx)");
-
-/**
- * ioat_dma_setup_interrupts - setup interrupt handler
- * @device: ioat device
- */
-static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
-{
-        struct ioat_dma_chan *ioat_chan;
-        int err, i, j, msixcnt;
-        u8 intrctrl = 0;
-
-        if (!strcmp(ioat_interrupt_style, "msix"))
-                goto msix;
-        if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
-                goto msix_single_vector;
-        if (!strcmp(ioat_interrupt_style, "msi"))
-                goto msi;
-        if (!strcmp(ioat_interrupt_style, "intx"))
-                goto intx;
-        dev_err(&device->pdev->dev, "invalid ioat_interrupt_style %s\n",
-                ioat_interrupt_style);
-        goto err_no_irq;
-
-msix:
-        /* The number of MSI-X vectors should equal the number of channels */
-        msixcnt = device->common.chancnt;
-        for (i = 0; i < msixcnt; i++)
-                device->msix_entries[i].entry = i;
-
-        err = pci_enable_msix(device->pdev, device->msix_entries, msixcnt);
-        if (err < 0)
-                goto msi;
-        if (err > 0)
-                goto msix_single_vector;
-
-        for (i = 0; i < msixcnt; i++) {
-                ioat_chan = ioat_lookup_chan_by_index(device, i);
-                err = request_irq(device->msix_entries[i].vector,
-                                  ioat_dma_do_interrupt_msix,
-                                  0, "ioat-msix", ioat_chan);
-                if (err) {
-                        for (j = 0; j < i; j++) {
-                                ioat_chan =
-                                        ioat_lookup_chan_by_index(device, j);
-                                free_irq(device->msix_entries[j].vector,
-                                         ioat_chan);
-                        }
-                        goto msix_single_vector;
-                }
-        }
-        intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
-        device->irq_mode = msix_multi_vector;
-        goto done;
-
-msix_single_vector:
-        device->msix_entries[0].entry = 0;
-        err = pci_enable_msix(device->pdev, device->msix_entries, 1);
-        if (err)
-                goto msi;
-
-        err = request_irq(device->msix_entries[0].vector, ioat_dma_do_interrupt,
-                          0, "ioat-msix", device);
-        if (err) {
-                pci_disable_msix(device->pdev);
-                goto msi;
-        }
-        device->irq_mode = msix_single_vector;
-        goto done;
-
-msi:
-        err = pci_enable_msi(device->pdev);
-        if (err)
-                goto intx;
-
-        err = request_irq(device->pdev->irq, ioat_dma_do_interrupt,
-                          0, "ioat-msi", device);
-        if (err) {
-                pci_disable_msi(device->pdev);
-                goto intx;
-        }
-        /*
-         * CB 1.2 devices need a bit set in configuration space to enable MSI
-         */
-        if (device->version == IOAT_VER_1_2) {
-                u32 dmactrl;
-                pci_read_config_dword(device->pdev,
-                                      IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
-                dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
-                pci_write_config_dword(device->pdev,
-                                       IOAT_PCI_DMACTRL_OFFSET, dmactrl);
-        }
-        device->irq_mode = msi;
-        goto done;
-
-intx:
-        err = request_irq(device->pdev->irq, ioat_dma_do_interrupt,
-                          IRQF_SHARED, "ioat-intx", device);
-        if (err)
-                goto err_no_irq;
-        device->irq_mode = intx;
-
-done:
-        intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
-        writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
-        return 0;
-
-err_no_irq:
-        /* Disable all interrupt generation */
-        writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
-        dev_err(&device->pdev->dev, "no usable interrupts\n");
-        device->irq_mode = none;
-        return -1;
-}
-
-/**
- * ioat_dma_remove_interrupts - remove whatever interrupts were set
- * @device: ioat device
- */
-static void ioat_dma_remove_interrupts(struct ioatdma_device *device)
-{
-        struct ioat_dma_chan *ioat_chan;
-        int i;
-
-        /* Disable all interrupt generation */
-        writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
-
-        switch (device->irq_mode) {
-        case msix_multi_vector:
-                for (i = 0; i < device->common.chancnt; i++) {
-                        ioat_chan = ioat_lookup_chan_by_index(device, i);
-                        free_irq(device->msix_entries[i].vector, ioat_chan);
-                }
-                pci_disable_msix(device->pdev);
-                break;
-        case msix_single_vector:
-                free_irq(device->msix_entries[0].vector, device);
-                pci_disable_msix(device->pdev);
-                break;
-        case msi:
-                free_irq(device->pdev->irq, device);
-                pci_disable_msi(device->pdev);
-                break;
-        case intx:
-                free_irq(device->pdev->irq, device);
-                break;
-        case none:
-                dev_warn(&device->pdev->dev,
-                         "call to %s without interrupts setup\n", __func__);
-        }
-        device->irq_mode = none;
-}
-
-struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev,
-                                      void __iomem *iobase)
-{
-        int err;
-        struct ioatdma_device *device;
-
-        device = kzalloc(sizeof(*device), GFP_KERNEL);
-        if (!device) {
-                err = -ENOMEM;
-                goto err_kzalloc;
-        }
-        device->pdev = pdev;
-        device->reg_base = iobase;
-        device->version = readb(device->reg_base + IOAT_VER_OFFSET);
-
-        /* DMA coherent memory pool for DMA descriptor allocations */
-        device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
-                                           sizeof(struct ioat_dma_descriptor),
-                                           64, 0);
-        if (!device->dma_pool) {
-                err = -ENOMEM;
-                goto err_dma_pool;
-        }
-
-        device->completion_pool = pci_pool_create("completion_pool", pdev,
-                                                  sizeof(u64), SMP_CACHE_BYTES,
-                                                  SMP_CACHE_BYTES);
-        if (!device->completion_pool) {
-                err = -ENOMEM;
-                goto err_completion_pool;
-        }
-
-        INIT_LIST_HEAD(&device->common.channels);
-        ioat_dma_enumerate_channels(device);
-
-        device->common.device_alloc_chan_resources =
-                                                ioat_dma_alloc_chan_resources;
-        device->common.device_free_chan_resources =
-                                                ioat_dma_free_chan_resources;
-        device->common.dev = &pdev->dev;
-
-        dma_cap_set(DMA_MEMCPY, device->common.cap_mask);
-        device->common.device_is_tx_complete = ioat_dma_is_complete;
-        switch (device->version) {
-        case IOAT_VER_1_2:
-                device->common.device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
-                device->common.device_issue_pending =
-                                                ioat1_dma_memcpy_issue_pending;
-                break;
-        case IOAT_VER_2_0:
-        case IOAT_VER_3_0:
-                device->common.device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
-                device->common.device_issue_pending =
-                                                ioat2_dma_memcpy_issue_pending;
-                break;
-        }
-
-        dev_err(&device->pdev->dev,
-                "Intel(R) I/OAT DMA Engine found,"
-                " %d channels, device version 0x%02x, driver version %s\n",
-                device->common.chancnt, device->version, IOAT_DMA_VERSION);
-
-        if (!device->common.chancnt) {
-                dev_err(&device->pdev->dev,
-                        "Intel(R) I/OAT DMA Engine problem found: "
-                        "zero channels detected\n");
-                goto err_setup_interrupts;
-        }
-
-        err = ioat_dma_setup_interrupts(device);
-        if (err)
-                goto err_setup_interrupts;
-
-        err = ioat_dma_self_test(device);
-        if (err)
-                goto err_self_test;
-
-        ioat_set_tcp_copy_break(device);
-
-        dma_async_device_register(&device->common);
-
-        if (device->version != IOAT_VER_3_0) {
-                INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
-                schedule_delayed_work(&device->work,
-                                      WATCHDOG_DELAY);
-        }
-
-        return device;
-
-err_self_test:
-        ioat_dma_remove_interrupts(device);
-err_setup_interrupts:
-        pci_pool_destroy(device->completion_pool);
-err_completion_pool:
-        pci_pool_destroy(device->dma_pool);
-err_dma_pool:
-        kfree(device);
-err_kzalloc:
-        dev_err(&pdev->dev,
-                "Intel(R) I/OAT DMA Engine initialization failed\n");
-        return NULL;
-}
-
-void ioat_dma_remove(struct ioatdma_device *device)
-{
-        struct dma_chan *chan, *_chan;
-        struct ioat_dma_chan *ioat_chan;
-
-        if (device->version != IOAT_VER_3_0)
-                cancel_delayed_work(&device->work);
-
-        ioat_dma_remove_interrupts(device);
-
-        dma_async_device_unregister(&device->common);
-
-        pci_pool_destroy(device->dma_pool);
-        pci_pool_destroy(device->completion_pool);
-
-        iounmap(device->reg_base);
-        pci_release_regions(device->pdev);
-        pci_disable_device(device->pdev);
-
-        list_for_each_entry_safe(chan, _chan,
-                                 &device->common.channels, device_node) {
-                ioat_chan = to_ioat_chan(chan);
-                list_del(&chan->device_node);
-                kfree(ioat_chan);
-        }
-        kfree(device);
-}
-
diff --git a/drivers/dma/ioatdma.h b/drivers/dma/ioatdma.h
deleted file mode 100644
index a52ff4bd4601..000000000000
--- a/drivers/dma/ioatdma.h
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc., 59
- * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
- *
- * The full GNU General Public License is included in this distribution in the
- * file called COPYING.
- */
-#ifndef IOATDMA_H
-#define IOATDMA_H
-
-#include <linux/dmaengine.h>
-#include "ioatdma_hw.h"
-#include <linux/init.h>
-#include <linux/dmapool.h>
-#include <linux/cache.h>
-#include <linux/pci_ids.h>
-#include <net/tcp.h>
-
-#define IOAT_DMA_VERSION  "3.64"
-
-enum ioat_interrupt {
-        none = 0,
-        msix_multi_vector = 1,
-        msix_single_vector = 2,
-        msi = 3,
-        intx = 4,
-};
-
-#define IOAT_LOW_COMPLETION_MASK        0xffffffc0
-#define IOAT_DMA_DCA_ANY_CPU            ~0
-#define IOAT_WATCHDOG_PERIOD            (2 * HZ)
-
-
-/**
- * struct ioatdma_device - internal representation of a IOAT device
- * @pdev: PCI-Express device
- * @reg_base: MMIO register space base address
- * @dma_pool: for allocating DMA descriptors
- * @common: embedded struct dma_device
- * @version: version of ioatdma device
- * @irq_mode: which style irq to use
- * @msix_entries: irq handlers
- * @idx: per channel data
- */
-
-struct ioatdma_device {
-        struct pci_dev *pdev;
-        void __iomem *reg_base;
-        struct pci_pool *dma_pool;
-        struct pci_pool *completion_pool;
-        struct dma_device common;
-        u8 version;
-        enum ioat_interrupt irq_mode;
-        struct delayed_work work;
-        struct msix_entry msix_entries[4];
-        struct ioat_dma_chan *idx[4];
-};
-
-/**
- * struct ioat_dma_chan - internal representation of a DMA channel
- */
-struct ioat_dma_chan {
-
-        void __iomem *reg_base;
-
-        dma_cookie_t completed_cookie;
-        unsigned long last_completion;
-        unsigned long last_completion_time;
-
-        size_t xfercap; /* XFERCAP register value expanded out */
-
-        spinlock_t cleanup_lock;
-        spinlock_t desc_lock;
-        struct list_head free_desc;
-        struct list_head used_desc;
-        unsigned long watchdog_completion;
-        int watchdog_tcp_cookie;
-        u32 watchdog_last_tcp_cookie;
-        struct delayed_work work;
-
-        int pending;
-        int dmacount;
-        int desccount;
-
-        struct ioatdma_device *device;
-        struct dma_chan common;
-
-        dma_addr_t completion_addr;
-        union {
-                u64 full; /* HW completion writeback */
-                struct {
-                        u32 low;
-                        u32 high;
-                };
-        } *completion_virt;
-        unsigned long last_compl_desc_addr_hw;
-        struct tasklet_struct cleanup_task;
-};
-
-/* wrapper around hardware descriptor format + additional software fields */
-
-/**
- * struct ioat_desc_sw - wrapper around hardware descriptor
- * @hw: hardware DMA descriptor
- * @node: this descriptor will either be on the free list,
- *     or attached to a transaction list (async_tx.tx_list)
- * @tx_cnt: number of descriptors required to complete the transaction
- * @async_tx: the generic software descriptor for all engines
- */
-struct ioat_desc_sw {
-        struct ioat_dma_descriptor *hw;
-        struct list_head node;
-        int tx_cnt;
-        size_t len;
-        dma_addr_t src;
-        dma_addr_t dst;
-        struct dma_async_tx_descriptor async_tx;
-};
-
-static inline void ioat_set_tcp_copy_break(struct ioatdma_device *dev)
-{
-        #ifdef CONFIG_NET_DMA
-        switch (dev->version) {
-        case IOAT_VER_1_2:
-                sysctl_tcp_dma_copybreak = 4096;
-                break;
-        case IOAT_VER_2_0:
-                sysctl_tcp_dma_copybreak = 2048;
-                break;
-        case IOAT_VER_3_0:
-                sysctl_tcp_dma_copybreak = 262144;
-                break;
-        }
-        #endif
-}
-
-#if defined(CONFIG_INTEL_IOATDMA) || defined(CONFIG_INTEL_IOATDMA_MODULE)
-struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev,
-                                      void __iomem *iobase);
-void ioat_dma_remove(struct ioatdma_device *device);
-struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
-struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
-struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
-#else
-#define ioat_dma_probe(pdev, iobase)    NULL
-#define ioat_dma_remove(device)         do { } while (0)
-#define ioat_dca_init(pdev, iobase)     NULL
-#define ioat2_dca_init(pdev, iobase)    NULL
-#define ioat3_dca_init(pdev, iobase)    NULL
-#endif
-
-#endif /* IOATDMA_H */
diff --git a/drivers/dma/ioatdma_hw.h b/drivers/dma/ioatdma_hw.h
deleted file mode 100644
index afa57eef86c9..000000000000
--- a/drivers/dma/ioatdma_hw.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc., 59
- * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
- *
- * The full GNU General Public License is included in this distribution in the
- * file called COPYING.
- */
-#ifndef _IOAT_HW_H_
-#define _IOAT_HW_H_
-
-/* PCI Configuration Space Values */
-#define IOAT_PCI_VID            0x8086
-
-/* CB device ID's */
-#define IOAT_PCI_DID_5000       0x1A38
-#define IOAT_PCI_DID_CNB        0x360B
-#define IOAT_PCI_DID_SCNB       0x65FF
-#define IOAT_PCI_DID_SNB        0x402F
-
-#define IOAT_PCI_RID            0x00
-#define IOAT_PCI_SVID           0x8086
-#define IOAT_PCI_SID            0x8086
-#define IOAT_VER_1_2            0x12    /* Version 1.2 */
-#define IOAT_VER_2_0            0x20    /* Version 2.0 */
-#define IOAT_VER_3_0            0x30    /* Version 3.0 */
-
-struct ioat_dma_descriptor {
-        uint32_t        size;
-        uint32_t        ctl;
-        uint64_t        src_addr;
-        uint64_t        dst_addr;
-        uint64_t        next;
-        uint64_t        rsv1;
-        uint64_t        rsv2;
-        uint64_t        user1;
-        uint64_t        user2;
-};
-
-#define IOAT_DMA_DESCRIPTOR_CTL_INT_GN  0x00000001
-#define IOAT_DMA_DESCRIPTOR_CTL_SRC_SN  0x00000002
-#define IOAT_DMA_DESCRIPTOR_CTL_DST_SN  0x00000004
-#define IOAT_DMA_DESCRIPTOR_CTL_CP_STS  0x00000008
-#define IOAT_DMA_DESCRIPTOR_CTL_FRAME   0x00000010
-#define IOAT_DMA_DESCRIPTOR_NUL         0x00000020
-#define IOAT_DMA_DESCRIPTOR_CTL_SP_BRK  0x00000040
-#define IOAT_DMA_DESCRIPTOR_CTL_DP_BRK  0x00000080
-#define IOAT_DMA_DESCRIPTOR_CTL_BNDL    0x00000100
-#define IOAT_DMA_DESCRIPTOR_CTL_DCA     0x00000200
-#define IOAT_DMA_DESCRIPTOR_CTL_BUFHINT 0x00000400
-
-#define IOAT_DMA_DESCRIPTOR_CTL_OPCODE_CONTEXT  0xFF000000
-#define IOAT_DMA_DESCRIPTOR_CTL_OPCODE_DMA      0x00000000
-
-#define IOAT_DMA_DESCRIPTOR_CTL_CONTEXT_DCA     0x00000001
-#define IOAT_DMA_DESCRIPTOR_CTL_OPCODE_MASK     0xFF000000
-
-#endif
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
index 2f052265122f..645ca8d54ec4 100644
--- a/drivers/dma/iop-adma.c
+++ b/drivers/dma/iop-adma.c
@@ -31,6 +31,7 @@
 #include <linux/platform_device.h>
 #include <linux/memory.h>
 #include <linux/ioport.h>
+#include <linux/raid/pq.h>
 
 #include <mach/adma.h>
 
@@ -57,65 +58,110 @@ static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
         }
 }
 
+static void
+iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
+{
+        struct dma_async_tx_descriptor *tx = &desc->async_tx;
+        struct iop_adma_desc_slot *unmap = desc->group_head;
+        struct device *dev = &iop_chan->device->pdev->dev;
+        u32 len = unmap->unmap_len;
+        enum dma_ctrl_flags flags = tx->flags;
+        u32 src_cnt;
+        dma_addr_t addr;
+        dma_addr_t dest;
+
+        src_cnt = unmap->unmap_src_cnt;
+        dest = iop_desc_get_dest_addr(unmap, iop_chan);
+        if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+                enum dma_data_direction dir;
+
+                if (src_cnt > 1) /* is xor? */
+                        dir = DMA_BIDIRECTIONAL;
+                else
+                        dir = DMA_FROM_DEVICE;
+
+                dma_unmap_page(dev, dest, len, dir);
+        }
+
+        if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                while (src_cnt--) {
+                        addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt);
+                        if (addr == dest)
+                                continue;
+                        dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
+                }
+        }
+        desc->group_head = NULL;
+}
+
+static void
+iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
+{
+        struct dma_async_tx_descriptor *tx = &desc->async_tx;
+        struct iop_adma_desc_slot *unmap = desc->group_head;
+        struct device *dev = &iop_chan->device->pdev->dev;
+        u32 len = unmap->unmap_len;
+        enum dma_ctrl_flags flags = tx->flags;
+        u32 src_cnt = unmap->unmap_src_cnt;
+        dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan);
+        dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan);
+        int i;
+
+        if (tx->flags & DMA_PREP_CONTINUE)
+                src_cnt -= 3;
+
+        if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) {
+                dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL);
+                dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL);
+        }
+
+        if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                dma_addr_t addr;
+
+                for (i = 0; i < src_cnt; i++) {
+                        addr = iop_desc_get_src_addr(unmap, iop_chan, i);
+                        dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
+                }
+                if (desc->pq_check_result) {
+                        dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE);
+                        dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE);
+                }
+        }
+
+        desc->group_head = NULL;
+}
+
+
 static dma_cookie_t
 iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
         struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
 {
-        BUG_ON(desc->async_tx.cookie < 0);
-        if (desc->async_tx.cookie > 0) {
-                cookie = desc->async_tx.cookie;
-                desc->async_tx.cookie = 0;
+        struct dma_async_tx_descriptor *tx = &desc->async_tx;
+
+        BUG_ON(tx->cookie < 0);
+        if (tx->cookie > 0) {
+                cookie = tx->cookie;
+                tx->cookie = 0;
 
                 /* call the callback (must not sleep or submit new
                  * operations to this channel)
                  */
-                if (desc->async_tx.callback)
-                        desc->async_tx.callback(
-                                desc->async_tx.callback_param);
+                if (tx->callback)
+                        tx->callback(tx->callback_param);
 
                 /* unmap dma addresses
                  * (unmap_single vs unmap_page?)
                  */
                 if (desc->group_head && desc->unmap_len) {
-                        struct iop_adma_desc_slot *unmap = desc->group_head;
-                        struct device *dev =
-                                &iop_chan->device->pdev->dev;
-                        u32 len = unmap->unmap_len;
-                        enum dma_ctrl_flags flags = desc->async_tx.flags;
-                        u32 src_cnt;
-                        dma_addr_t addr;
-                        dma_addr_t dest;
-
-                        src_cnt = unmap->unmap_src_cnt;
-                        dest = iop_desc_get_dest_addr(unmap, iop_chan);
-                        if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
-                                enum dma_data_direction dir;
-
-                                if (src_cnt > 1) /* is xor? */
-                                        dir = DMA_BIDIRECTIONAL;
-                                else
-                                        dir = DMA_FROM_DEVICE;
-
-                                dma_unmap_page(dev, dest, len, dir);
-                        }
-
-                        if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
-                                while (src_cnt--) {
-                                        addr = iop_desc_get_src_addr(unmap,
-                                                                     iop_chan,
-                                                                     src_cnt);
-                                        if (addr == dest)
-                                                continue;
-                                        dma_unmap_page(dev, addr, len,
-                                                       DMA_TO_DEVICE);
-                                }
-                        }
-                        desc->group_head = NULL;
+                        if (iop_desc_is_pq(desc))
+                                iop_desc_unmap_pq(iop_chan, desc);
+                        else
+                                iop_desc_unmap(iop_chan, desc);
                 }
         }
 
         /* run dependent operations */
-        dma_run_dependencies(&desc->async_tx);
+        dma_run_dependencies(tx);
 
         return cookie;
 }
@@ -287,7 +333,12 @@ static void iop_adma_tasklet(unsigned long data)
 {
         struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
 
-        spin_lock(&iop_chan->lock);
+        /* lockdep will flag depedency submissions as potentially
+         * recursive locking, this is not the case as a dependency
+         * submission will never recurse a channels submit routine.
+         * There are checks in async_tx.c to prevent this.
+         */
+        spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
         __iop_adma_slot_cleanup(iop_chan);
         spin_unlock(&iop_chan->lock);
 }
@@ -370,7 +421,7 @@ retry:
                         }
                         alloc_tail->group_head = alloc_start;
                         alloc_tail->async_tx.cookie = -EBUSY;
-                        list_splice(&chain, &alloc_tail->async_tx.tx_list);
+                        list_splice(&chain, &alloc_tail->tx_list);
                         iop_chan->last_used = last_used;
                         iop_desc_clear_next_desc(alloc_start);
                         iop_desc_clear_next_desc(alloc_tail);
@@ -429,7 +480,7 @@ iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
 
         old_chain_tail = list_entry(iop_chan->chain.prev,
                 struct iop_adma_desc_slot, chain_node);
-        list_splice_init(&sw_desc->async_tx.tx_list,
+        list_splice_init(&sw_desc->tx_list,
                          &old_chain_tail->chain_node);
 
         /* fix up the hardware chain */
@@ -496,6 +547,7 @@ static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
 
                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
                 slot->async_tx.tx_submit = iop_adma_tx_submit;
+                INIT_LIST_HEAD(&slot->tx_list);
                 INIT_LIST_HEAD(&slot->chain_node);
                 INIT_LIST_HEAD(&slot->slot_node);
                 hw_desc = (char *) iop_chan->device->dma_desc_pool;
@@ -660,9 +712,9 @@ iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
 }
 
 static struct dma_async_tx_descriptor *
-iop_adma_prep_dma_zero_sum(struct dma_chan *chan, dma_addr_t *dma_src,
-                           unsigned int src_cnt, size_t len, u32 *result,
-                           unsigned long flags)
+iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
+                          unsigned int src_cnt, size_t len, u32 *result,
+                          unsigned long flags)
 {
         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
         struct iop_adma_desc_slot *sw_desc, *grp_start;
@@ -696,6 +748,118 @@ iop_adma_prep_dma_zero_sum(struct dma_chan *chan, dma_addr_t *dma_src,
         return sw_desc ? &sw_desc->async_tx : NULL;
 }
 
+static struct dma_async_tx_descriptor *
+iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
+                     unsigned int src_cnt, const unsigned char *scf, size_t len,
+                     unsigned long flags)
+{
+        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
+        struct iop_adma_desc_slot *sw_desc, *g;
+        int slot_cnt, slots_per_op;
+        int continue_srcs;
+
+        if (unlikely(!len))
+                return NULL;
+        BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
+
+        dev_dbg(iop_chan->device->common.dev,
+                "%s src_cnt: %d len: %u flags: %lx\n",
+                __func__, src_cnt, len, flags);
+
+        if (dmaf_p_disabled_continue(flags))
+                continue_srcs = 1+src_cnt;
+        else if (dmaf_continue(flags))
+                continue_srcs = 3+src_cnt;
+        else
+                continue_srcs = 0+src_cnt;
+
+        spin_lock_bh(&iop_chan->lock);
+        slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
+        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
+        if (sw_desc) {
+                int i;
+
+                g = sw_desc->group_head;
+                iop_desc_set_byte_count(g, iop_chan, len);
+
+                /* even if P is disabled its destination address (bits
+                 * [3:0]) must match Q.  It is ok if P points to an
+                 * invalid address, it won't be written.
+                 */
+                if (flags & DMA_PREP_PQ_DISABLE_P)
+                        dst[0] = dst[1] & 0x7;
+
+                iop_desc_set_pq_addr(g, dst);
+                sw_desc->unmap_src_cnt = src_cnt;
+                sw_desc->unmap_len = len;
+                sw_desc->async_tx.flags = flags;
+                for (i = 0; i < src_cnt; i++)
+                        iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
+
+                /* if we are continuing a previous operation factor in
+                 * the old p and q values, see the comment for dma_maxpq
+                 * in include/linux/dmaengine.h
+                 */
+                if (dmaf_p_disabled_continue(flags))
+                        iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
+                else if (dmaf_continue(flags)) {
+                        iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
+                        iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
+                        iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
+                }
+                iop_desc_init_pq(g, i, flags);
+        }
+        spin_unlock_bh(&iop_chan->lock);
+
+        return sw_desc ? &sw_desc->async_tx : NULL;
+}
+
+static struct dma_async_tx_descriptor *
+iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
+                         unsigned int src_cnt, const unsigned char *scf,
+                         size_t len, enum sum_check_flags *pqres,
+                         unsigned long flags)
+{
+        struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
+        struct iop_adma_desc_slot *sw_desc, *g;
+        int slot_cnt, slots_per_op;
+
+        if (unlikely(!len))
+                return NULL;
+        BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
+
+        dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
+                __func__, src_cnt, len);
+
+        spin_lock_bh(&iop_chan->lock);
+        slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
+        sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
+        if (sw_desc) {
+                /* for validate operations p and q are tagged onto the
+                 * end of the source list
+                 */
+                int pq_idx = src_cnt;
+
+                g = sw_desc->group_head;
+                iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
+                iop_desc_set_pq_zero_sum_byte_count(g, len);
+                g->pq_check_result = pqres;
+                pr_debug("\t%s: g->pq_check_result: %p\n",
+                        __func__, g->pq_check_result);
+                sw_desc->unmap_src_cnt = src_cnt+2;
+                sw_desc->unmap_len = len;
+                sw_desc->async_tx.flags = flags;
+                while (src_cnt--)
+                        iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
+                                                          src[src_cnt],
+                                                          scf[src_cnt]);
+                iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
+        }
+        spin_unlock_bh(&iop_chan->lock);
+
+        return sw_desc ? &sw_desc->async_tx : NULL;
+}
+
 static void iop_adma_free_chan_resources(struct dma_chan *chan)
 {
         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
@@ -906,7 +1070,7 @@ out:
 
 #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
 static int __devinit
-iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
+iop_adma_xor_val_self_test(struct iop_adma_device *device)
 {
         int i, src_idx;
         struct page *dest;
@@ -1002,7 +1166,7 @@ iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
                 PAGE_SIZE, DMA_TO_DEVICE);
 
         /* skip zero sum if the capability is not present */
-        if (!dma_has_cap(DMA_ZERO_SUM, dma_chan->device->cap_mask))
+        if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
                 goto free_resources;
 
         /* zero sum the sources with the destintation page */
@@ -1016,10 +1180,10 @@ iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
                                            zero_sum_srcs[i], 0, PAGE_SIZE,
                                            DMA_TO_DEVICE);
-        tx = iop_adma_prep_dma_zero_sum(dma_chan, dma_srcs,
-                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
-                                        &zero_sum_result,
-                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+        tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
+                                       IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
+                                       &zero_sum_result,
+                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 
         cookie = iop_adma_tx_submit(tx);
         iop_adma_issue_pending(dma_chan);
@@ -1072,10 +1236,10 @@ iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
                                            zero_sum_srcs[i], 0, PAGE_SIZE,
                                            DMA_TO_DEVICE);
-        tx = iop_adma_prep_dma_zero_sum(dma_chan, dma_srcs,
-                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
-                                        &zero_sum_result,
-                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+        tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
+                                       IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
+                                       &zero_sum_result,
+                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 
         cookie = iop_adma_tx_submit(tx);
         iop_adma_issue_pending(dma_chan);
@@ -1105,6 +1269,170 @@ out:
         return err;
 }
 
+#ifdef CONFIG_MD_RAID6_PQ
+static int __devinit
+iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
+{
+        /* combined sources, software pq results, and extra hw pq results */
+        struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
+        /* ptr to the extra hw pq buffers defined above */
+        struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
+        /* address conversion buffers (dma_map / page_address) */
+        void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
+        dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST];
+        dma_addr_t pq_dest[2];
+
+        int i;
+        struct dma_async_tx_descriptor *tx;
+        struct dma_chan *dma_chan;
+        dma_cookie_t cookie;
+        u32 zero_sum_result;
+        int err = 0;
+        struct device *dev;
+
+        dev_dbg(device->common.dev, "%s\n", __func__);
+
+        for (i = 0; i < ARRAY_SIZE(pq); i++) {
+                pq[i] = alloc_page(GFP_KERNEL);
+                if (!pq[i]) {
+                        while (i--)
+                                __free_page(pq[i]);
+                        return -ENOMEM;
+                }
+        }
+
+        /* Fill in src buffers */
+        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
+                pq_sw[i] = page_address(pq[i]);
+                memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
+        }
+        pq_sw[i] = page_address(pq[i]);
+        pq_sw[i+1] = page_address(pq[i+1]);
+
+        dma_chan = container_of(device->common.channels.next,
+                                struct dma_chan,
+                                device_node);
+        if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
+                err = -ENODEV;
+                goto out;
+        }
+
+        dev = dma_chan->device->dev;
+
+        /* initialize the dests */
+        memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
+        memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);
+
+        /* test pq */
+        pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
+        pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
+        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
+                pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
+                                         DMA_TO_DEVICE);
+
+        tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
+                                  IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
+                                  PAGE_SIZE,
+                                  DMA_PREP_INTERRUPT |
+                                  DMA_CTRL_ACK);
+
+        cookie = iop_adma_tx_submit(tx);
+        iop_adma_issue_pending(dma_chan);
+        msleep(8);
+
+        if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
+                DMA_SUCCESS) {
+                dev_err(dev, "Self-test pq timed out, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);
+
+        if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
+                   page_address(pq_hw[0]), PAGE_SIZE) != 0) {
+                dev_err(dev, "Self-test p failed compare, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+        if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
+                   page_address(pq_hw[1]), PAGE_SIZE) != 0) {
+                dev_err(dev, "Self-test q failed compare, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        /* test correct zero sum using the software generated pq values */
+        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
+                pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
+                                         DMA_TO_DEVICE);
+
+        zero_sum_result = ~0;
+        tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
+                                      pq_src, IOP_ADMA_NUM_SRC_TEST,
+                                      raid6_gfexp, PAGE_SIZE, &zero_sum_result,
+                                      DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
+
+        cookie = iop_adma_tx_submit(tx);
+        iop_adma_issue_pending(dma_chan);
+        msleep(8);
+
+        if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
+                DMA_SUCCESS) {
+                dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        if (zero_sum_result != 0) {
+                dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
+                        zero_sum_result);
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        /* test incorrect zero sum */
+        i = IOP_ADMA_NUM_SRC_TEST;
+        memset(pq_sw[i] + 100, 0, 100);
+        memset(pq_sw[i+1] + 200, 0, 200);
+        for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
+                pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
+                                         DMA_TO_DEVICE);
+
+        zero_sum_result = 0;
+        tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
+                                      pq_src, IOP_ADMA_NUM_SRC_TEST,
+                                      raid6_gfexp, PAGE_SIZE, &zero_sum_result,
+                                      DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
+
+        cookie = iop_adma_tx_submit(tx);
+        iop_adma_issue_pending(dma_chan);
+        msleep(8);
+
+        if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
+                DMA_SUCCESS) {
+                dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+        if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) {
+                dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
+                        zero_sum_result);
+                err = -ENODEV;
+                goto free_resources;
+        }
+
+free_resources:
+        iop_adma_free_chan_resources(dma_chan);
+out:
+        i = ARRAY_SIZE(pq);
+        while (i--)
+                __free_page(pq[i]);
+        return err;
+}
+#endif
+
 static int __devexit iop_adma_remove(struct platform_device *dev)
 {
         struct iop_adma_device *device = platform_get_drvdata(dev);
@@ -1192,9 +1520,16 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
                 dma_dev->max_xor = iop_adma_get_max_xor();
                 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
         }
-        if (dma_has_cap(DMA_ZERO_SUM, dma_dev->cap_mask))
-                dma_dev->device_prep_dma_zero_sum =
-                        iop_adma_prep_dma_zero_sum;
+        if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
+                dma_dev->device_prep_dma_xor_val =
+                        iop_adma_prep_dma_xor_val;
+        if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
+                dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
+                dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
+        }
+        if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
+                dma_dev->device_prep_dma_pq_val =
+                        iop_adma_prep_dma_pq_val;
         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
                 dma_dev->device_prep_dma_interrupt =
                         iop_adma_prep_dma_interrupt;
@@ -1248,23 +1583,35 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
         }
 
         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask) ||
-                dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
-                ret = iop_adma_xor_zero_sum_self_test(adev);
+            dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
+                ret = iop_adma_xor_val_self_test(adev);
                 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
                 if (ret)
                         goto err_free_iop_chan;
         }
 
+        if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
+            dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
+                #ifdef CONFIG_MD_RAID6_PQ
+                ret = iop_adma_pq_zero_sum_self_test(adev);
+                dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
+                #else
+                /* can not test raid6, so do not publish capability */
+                dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
+                dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
+                ret = 0;
+                #endif
+                if (ret)
+                        goto err_free_iop_chan;
+        }
+
         dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: "
-          "( %s%s%s%s%s%s%s%s%s%s)\n",
-          dma_has_cap(DMA_PQ_XOR, dma_dev->cap_mask) ? "pq_xor " : "",
-          dma_has_cap(DMA_PQ_UPDATE, dma_dev->cap_mask) ? "pq_update " : "",
-          dma_has_cap(DMA_PQ_ZERO_SUM, dma_dev->cap_mask) ? "pq_zero_sum " : "",
+          "( %s%s%s%s%s%s%s)\n",
+          dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",
+          dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "",
           dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
-          dma_has_cap(DMA_DUAL_XOR, dma_dev->cap_mask) ? "dual_xor " : "",
-          dma_has_cap(DMA_ZERO_SUM, dma_dev->cap_mask) ? "xor_zero_sum " : "",
+          dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "",
           dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)  ? "fill " : "",
-          dma_has_cap(DMA_MEMCPY_CRC32C, dma_dev->cap_mask) ? "cpy+crc " : "",
           dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
           dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
 
@@ -1296,7 +1643,7 @@ static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
         if (sw_desc) {
                 grp_start = sw_desc->group_head;
 
-                list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
+                list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
                 async_tx_ack(&sw_desc->async_tx);
                 iop_desc_init_memcpy(grp_start, 0);
                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
@@ -1352,7 +1699,7 @@ static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
         if (sw_desc) {
                 grp_start = sw_desc->group_head;
-                list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
+                list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
                 async_tx_ack(&sw_desc->async_tx);
                 iop_desc_init_null_xor(grp_start, 2, 0);
                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
diff --git a/drivers/dma/iovlock.c b/drivers/dma/iovlock.c
index 9f6fe46a9b87..c0a272c73682 100644
--- a/drivers/dma/iovlock.c
+++ b/drivers/dma/iovlock.c
@@ -183,6 +183,11 @@ dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
                                         iov_byte_offset,
                                         kdata,
                                         copy);
+                        /* poll for a descriptor slot */
+                        if (unlikely(dma_cookie < 0)) {
+                                dma_async_issue_pending(chan);
+                                continue;
+                        }
 
                         len -= copy;
                         iov[iovec_idx].iov_len -= copy;
@@ -248,6 +253,11 @@ dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
                                         page,
                                         offset,
                                         copy);
+                        /* poll for a descriptor slot */
+                        if (unlikely(dma_cookie < 0)) {
+                                dma_async_issue_pending(chan);
+                                continue;
+                        }
 
                         len -= copy;
                         iov[iovec_idx].iov_len -= copy;
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index 3f23eabe09f2..466ab10c1ff1 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -517,7 +517,7 @@ retry:
                         }
                         alloc_tail->group_head = alloc_start;
                         alloc_tail->async_tx.cookie = -EBUSY;
-                        list_splice(&chain, &alloc_tail->async_tx.tx_list);
+                        list_splice(&chain, &alloc_tail->tx_list);
                         mv_chan->last_used = last_used;
                         mv_desc_clear_next_desc(alloc_start);
                         mv_desc_clear_next_desc(alloc_tail);
@@ -565,14 +565,14 @@ mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
         cookie = mv_desc_assign_cookie(mv_chan, sw_desc);
 
         if (list_empty(&mv_chan->chain))
-                list_splice_init(&sw_desc->async_tx.tx_list, &mv_chan->chain);
+                list_splice_init(&sw_desc->tx_list, &mv_chan->chain);
         else {
                 new_hw_chain = 0;
 
                 old_chain_tail = list_entry(mv_chan->chain.prev,
                                             struct mv_xor_desc_slot,
                                             chain_node);
-                list_splice_init(&grp_start->async_tx.tx_list,
+                list_splice_init(&grp_start->tx_list,
                                  &old_chain_tail->chain_node);
 
                 if (!mv_can_chain(grp_start))
@@ -632,6 +632,7 @@ static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
                 slot->async_tx.tx_submit = mv_xor_tx_submit;
                 INIT_LIST_HEAD(&slot->chain_node);
                 INIT_LIST_HEAD(&slot->slot_node);
+                INIT_LIST_HEAD(&slot->tx_list);
                 hw_desc = (char *) mv_chan->device->dma_desc_pool;
                 slot->async_tx.phys =
                         (dma_addr_t) &hw_desc[idx * MV_XOR_SLOT_SIZE];
diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h
index 06cafe1ef521..977b592e976b 100644
--- a/drivers/dma/mv_xor.h
+++ b/drivers/dma/mv_xor.h
@@ -126,9 +126,8 @@ struct mv_xor_chan {
  * @idx: pool index
  * @unmap_src_cnt: number of xor sources
  * @unmap_len: transaction bytecount
+ * @tx_list: list of slots that make up a multi-descriptor transaction
  * @async_tx: support for the async_tx api
- * @group_list: list of slots that make up a multi-descriptor transaction
- *      for example transfer lengths larger than the supported hw max
  * @xor_check_result: result of zero sum
  * @crc32_result: result crc calculation
  */
@@ -145,6 +144,7 @@ struct mv_xor_desc_slot {
         u16                     unmap_src_cnt;
         u32                     value;
         size_t                  unmap_len;
+        struct list_head        tx_list;
         struct dma_async_tx_descriptor  async_tx;
         union {
                 u32             *xor_check_result;
diff --git a/drivers/dma/shdma.c b/drivers/dma/shdma.c
new file mode 100644
index 000000000000..b3b065c4e5c1
--- /dev/null
+++ b/drivers/dma/shdma.c
@@ -0,0 +1,786 @@
+/*
+ * Renesas SuperH DMA Engine support
+ *
+ * base is drivers/dma/flsdma.c
+ *
+ * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>
+ * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved.
+ * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * - DMA of SuperH does not have Hardware DMA chain mode.
+ * - MAX DMA size is 16MB.
+ *
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/dmaengine.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/platform_device.h>
+#include <cpu/dma.h>
+#include <asm/dma-sh.h>
+#include "shdma.h"
+
+/* DMA descriptor control */
+#define DESC_LAST       (-1)
+#define DESC_COMP       (1)
+#define DESC_NCOMP      (0)
+
+#define NR_DESCS_PER_CHANNEL 32
+/*
+ * Define the default configuration for dual address memory-memory transfer.
+ * The 0x400 value represents auto-request, external->external.
+ *
+ * And this driver set 4byte burst mode.
+ * If you want to change mode, you need to change RS_DEFAULT of value.
+ * (ex 1byte burst mode -> (RS_DUAL & ~TS_32)
+ */
+#define RS_DEFAULT  (RS_DUAL)
+
+#define SH_DMAC_CHAN_BASE(id) (dma_base_addr[id])
+static void sh_dmae_writel(struct sh_dmae_chan *sh_dc, u32 data, u32 reg)
+{
+        ctrl_outl(data, (SH_DMAC_CHAN_BASE(sh_dc->id) + reg));
+}
+
+static u32 sh_dmae_readl(struct sh_dmae_chan *sh_dc, u32 reg)
+{
+        return ctrl_inl((SH_DMAC_CHAN_BASE(sh_dc->id) + reg));
+}
+
+static void dmae_init(struct sh_dmae_chan *sh_chan)
+{
+        u32 chcr = RS_DEFAULT; /* default is DUAL mode */
+        sh_dmae_writel(sh_chan, chcr, CHCR);
+}
+
+/*
+ * Reset DMA controller
+ *
+ * SH7780 has two DMAOR register
+ */
+static void sh_dmae_ctl_stop(int id)
+{
+        unsigned short dmaor = dmaor_read_reg(id);
+
+        dmaor &= ~(DMAOR_NMIF | DMAOR_AE);
+        dmaor_write_reg(id, dmaor);
+}
+
+static int sh_dmae_rst(int id)
+{
+        unsigned short dmaor;
+
+        sh_dmae_ctl_stop(id);
+        dmaor = (dmaor_read_reg(id)|DMAOR_INIT);
+
+        dmaor_write_reg(id, dmaor);
+        if ((dmaor_read_reg(id) & (DMAOR_AE | DMAOR_NMIF))) {
+                pr_warning(KERN_ERR "dma-sh: Can't initialize DMAOR.\n");
+                return -EINVAL;
+        }
+        return 0;
+}
+
+static int dmae_is_idle(struct sh_dmae_chan *sh_chan)
+{
+        u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+        if (chcr & CHCR_DE) {
+                if (!(chcr & CHCR_TE))
+                        return -EBUSY; /* working */
+        }
+        return 0; /* waiting */
+}
+
+static inline unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan)
+{
+        u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+        return ts_shift[(chcr & CHCR_TS_MASK) >> CHCR_TS_SHIFT];
+}
+
+static void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs hw)
+{
+        sh_dmae_writel(sh_chan, hw.sar, SAR);
+        sh_dmae_writel(sh_chan, hw.dar, DAR);
+        sh_dmae_writel(sh_chan,
+                (hw.tcr >> calc_xmit_shift(sh_chan)), TCR);
+}
+
+static void dmae_start(struct sh_dmae_chan *sh_chan)
+{
+        u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+
+        chcr |= (CHCR_DE|CHCR_IE);
+        sh_dmae_writel(sh_chan, chcr, CHCR);
+}
+
+static void dmae_halt(struct sh_dmae_chan *sh_chan)
+{
+        u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+
+        chcr &= ~(CHCR_DE | CHCR_TE | CHCR_IE);
+        sh_dmae_writel(sh_chan, chcr, CHCR);
+}
+
+static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val)
+{
+        int ret = dmae_is_idle(sh_chan);
+        /* When DMA was working, can not set data to CHCR */
+        if (ret)
+                return ret;
+
+        sh_dmae_writel(sh_chan, val, CHCR);
+        return 0;
+}
+
+#define DMARS1_ADDR     0x04
+#define DMARS2_ADDR     0x08
+#define DMARS_SHIFT 8
+#define DMARS_CHAN_MSK 0x01
+static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val)
+{
+        u32 addr;
+        int shift = 0;
+        int ret = dmae_is_idle(sh_chan);
+        if (ret)
+                return ret;
+
+        if (sh_chan->id & DMARS_CHAN_MSK)
+                shift = DMARS_SHIFT;
+
+        switch (sh_chan->id) {
+        /* DMARS0 */
+        case 0:
+        case 1:
+                addr = SH_DMARS_BASE;
+                break;
+        /* DMARS1 */
+        case 2:
+        case 3:
+                addr = (SH_DMARS_BASE + DMARS1_ADDR);
+                break;
+        /* DMARS2 */
+        case 4:
+        case 5:
+                addr = (SH_DMARS_BASE + DMARS2_ADDR);
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        ctrl_outw((val << shift) |
+                (ctrl_inw(addr) & (shift ? 0xFF00 : 0x00FF)),
+                addr);
+
+        return 0;
+}
+
+static dma_cookie_t sh_dmae_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+        struct sh_desc *desc = tx_to_sh_desc(tx);
+        struct sh_dmae_chan *sh_chan = to_sh_chan(tx->chan);
+        dma_cookie_t cookie;
+
+        spin_lock_bh(&sh_chan->desc_lock);
+
+        cookie = sh_chan->common.cookie;
+        cookie++;
+        if (cookie < 0)
+                cookie = 1;
+
+        /* If desc only in the case of 1 */
+        if (desc->async_tx.cookie != -EBUSY)
+                desc->async_tx.cookie = cookie;
+        sh_chan->common.cookie = desc->async_tx.cookie;
+
+        list_splice_init(&desc->tx_list, sh_chan->ld_queue.prev);
+
+        spin_unlock_bh(&sh_chan->desc_lock);
+
+        return cookie;
+}
+
+static struct sh_desc *sh_dmae_get_desc(struct sh_dmae_chan *sh_chan)
+{
+        struct sh_desc *desc, *_desc, *ret = NULL;
+
+        spin_lock_bh(&sh_chan->desc_lock);
+        list_for_each_entry_safe(desc, _desc, &sh_chan->ld_free, node) {
+                if (async_tx_test_ack(&desc->async_tx)) {
+                        list_del(&desc->node);
+                        ret = desc;
+                        break;
+                }
+        }
+        spin_unlock_bh(&sh_chan->desc_lock);
+
+        return ret;
+}
+
+static void sh_dmae_put_desc(struct sh_dmae_chan *sh_chan, struct sh_desc *desc)
+{
+        if (desc) {
+                spin_lock_bh(&sh_chan->desc_lock);
+
+                list_splice_init(&desc->tx_list, &sh_chan->ld_free);
+                list_add(&desc->node, &sh_chan->ld_free);
+
+                spin_unlock_bh(&sh_chan->desc_lock);
+        }
+}
+
+static int sh_dmae_alloc_chan_resources(struct dma_chan *chan)
+{
+        struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
+        struct sh_desc *desc;
+
+        spin_lock_bh(&sh_chan->desc_lock);
+        while (sh_chan->descs_allocated < NR_DESCS_PER_CHANNEL) {
+                spin_unlock_bh(&sh_chan->desc_lock);
+                desc = kzalloc(sizeof(struct sh_desc), GFP_KERNEL);
+                if (!desc) {
+                        spin_lock_bh(&sh_chan->desc_lock);
+                        break;
+                }
+                dma_async_tx_descriptor_init(&desc->async_tx,
+                                        &sh_chan->common);
+                desc->async_tx.tx_submit = sh_dmae_tx_submit;
+                desc->async_tx.flags = DMA_CTRL_ACK;
+                INIT_LIST_HEAD(&desc->tx_list);
+                sh_dmae_put_desc(sh_chan, desc);
+
+                spin_lock_bh(&sh_chan->desc_lock);
+                sh_chan->descs_allocated++;
+        }
+        spin_unlock_bh(&sh_chan->desc_lock);
+
+        return sh_chan->descs_allocated;
+}
+
+/*
+ * sh_dma_free_chan_resources - Free all resources of the channel.
+ */
+static void sh_dmae_free_chan_resources(struct dma_chan *chan)
+{
+        struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
+        struct sh_desc *desc, *_desc;
+        LIST_HEAD(list);
+
+        BUG_ON(!list_empty(&sh_chan->ld_queue));
+        spin_lock_bh(&sh_chan->desc_lock);
+
+        list_splice_init(&sh_chan->ld_free, &list);
+        sh_chan->descs_allocated = 0;
+
+        spin_unlock_bh(&sh_chan->desc_lock);
+
+        list_for_each_entry_safe(desc, _desc, &list, node)
+                kfree(desc);
+}
+
+static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy(
+        struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src,
+        size_t len, unsigned long flags)
+{
+        struct sh_dmae_chan *sh_chan;
+        struct sh_desc *first = NULL, *prev = NULL, *new;
+        size_t copy_size;
+
+        if (!chan)
+                return NULL;
+
+        if (!len)
+                return NULL;
+
+        sh_chan = to_sh_chan(chan);
+
+        do {
+                /* Allocate the link descriptor from DMA pool */
+                new = sh_dmae_get_desc(sh_chan);
+                if (!new) {
+                        dev_err(sh_chan->dev,
+                                        "No free memory for link descriptor\n");
+                        goto err_get_desc;
+                }
+
+                copy_size = min(len, (size_t)SH_DMA_TCR_MAX);
+
+                new->hw.sar = dma_src;
+                new->hw.dar = dma_dest;
+                new->hw.tcr = copy_size;
+                if (!first)
+                        first = new;
+
+                new->mark = DESC_NCOMP;
+                async_tx_ack(&new->async_tx);
+
+                prev = new;
+                len -= copy_size;
+                dma_src += copy_size;
+                dma_dest += copy_size;
+                /* Insert the link descriptor to the LD ring */
+                list_add_tail(&new->node, &first->tx_list);
+        } while (len);
+
+        new->async_tx.flags = flags; /* client is in control of this ack */
+        new->async_tx.cookie = -EBUSY; /* Last desc */
+
+        return &first->async_tx;
+
+err_get_desc:
+        sh_dmae_put_desc(sh_chan, first);
+        return NULL;
+
+}
+
+/*
+ * sh_chan_ld_cleanup - Clean up link descriptors
+ *
+ * This function clean up the ld_queue of DMA channel.
+ */
+static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan)
+{
+        struct sh_desc *desc, *_desc;
+
+        spin_lock_bh(&sh_chan->desc_lock);
+        list_for_each_entry_safe(desc, _desc, &sh_chan->ld_queue, node) {
+                dma_async_tx_callback callback;
+                void *callback_param;
+
+                /* non send data */
+                if (desc->mark == DESC_NCOMP)
+                        break;
+
+                /* send data sesc */
+                callback = desc->async_tx.callback;
+                callback_param = desc->async_tx.callback_param;
+
+                /* Remove from ld_queue list */
+                list_splice_init(&desc->tx_list, &sh_chan->ld_free);
+
+                dev_dbg(sh_chan->dev, "link descriptor %p will be recycle.\n",
+                                desc);
+
+                list_move(&desc->node, &sh_chan->ld_free);
+                /* Run the link descriptor callback function */
+                if (callback) {
+                        spin_unlock_bh(&sh_chan->desc_lock);
+                        dev_dbg(sh_chan->dev, "link descriptor %p callback\n",
+                                        desc);
+                        callback(callback_param);
+                        spin_lock_bh(&sh_chan->desc_lock);
+                }
+        }
+        spin_unlock_bh(&sh_chan->desc_lock);
+}
+
+static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan)
+{
+        struct list_head *ld_node;
+        struct sh_dmae_regs hw;
+
+        /* DMA work check */
+        if (dmae_is_idle(sh_chan))
+                return;
+
+        /* Find the first un-transfer desciptor */
+        for (ld_node = sh_chan->ld_queue.next;
+                (ld_node != &sh_chan->ld_queue)
+                        && (to_sh_desc(ld_node)->mark == DESC_COMP);
+                ld_node = ld_node->next)
+                cpu_relax();
+
+        if (ld_node != &sh_chan->ld_queue) {
+                /* Get the ld start address from ld_queue */
+                hw = to_sh_desc(ld_node)->hw;
+                dmae_set_reg(sh_chan, hw);
+                dmae_start(sh_chan);
+        }
+}
+
+static void sh_dmae_memcpy_issue_pending(struct dma_chan *chan)
+{
+        struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
+        sh_chan_xfer_ld_queue(sh_chan);
+}
+
+static enum dma_status sh_dmae_is_complete(struct dma_chan *chan,
+                                        dma_cookie_t cookie,
+                                        dma_cookie_t *done,
+                                        dma_cookie_t *used)
+{
+        struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
+        dma_cookie_t last_used;
+        dma_cookie_t last_complete;
+
+        sh_dmae_chan_ld_cleanup(sh_chan);
+
+        last_used = chan->cookie;
+        last_complete = sh_chan->completed_cookie;
+        if (last_complete == -EBUSY)
+                last_complete = last_used;
+
+        if (done)
+                *done = last_complete;
+
+        if (used)
+                *used = last_used;
+
+        return dma_async_is_complete(cookie, last_complete, last_used);
+}
+
+static irqreturn_t sh_dmae_interrupt(int irq, void *data)
+{
+        irqreturn_t ret = IRQ_NONE;
+        struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data;
+        u32 chcr = sh_dmae_readl(sh_chan, CHCR);
+
+        if (chcr & CHCR_TE) {
+                /* DMA stop */
+                dmae_halt(sh_chan);
+
+                ret = IRQ_HANDLED;
+                tasklet_schedule(&sh_chan->tasklet);
+        }
+
+        return ret;
+}
+
+#if defined(CONFIG_CPU_SH4)
+static irqreturn_t sh_dmae_err(int irq, void *data)
+{
+        int err = 0;
+        struct sh_dmae_device *shdev = (struct sh_dmae_device *)data;
+
+        /* IRQ Multi */
+        if (shdev->pdata.mode & SHDMA_MIX_IRQ) {
+                int cnt = 0;
+                switch (irq) {
+#if defined(DMTE6_IRQ) && defined(DMAE1_IRQ)
+                case DMTE6_IRQ:
+                        cnt++;
+#endif
+                case DMTE0_IRQ:
+                        if (dmaor_read_reg(cnt) & (DMAOR_NMIF | DMAOR_AE)) {
+                                disable_irq(irq);
+                                return IRQ_HANDLED;
+                        }
+                default:
+                        return IRQ_NONE;
+                }
+        } else {
+                /* reset dma controller */
+                err = sh_dmae_rst(0);
+                if (err)
+                        return err;
+                if (shdev->pdata.mode & SHDMA_DMAOR1) {
+                        err = sh_dmae_rst(1);
+                        if (err)
+                                return err;
+                }
+                disable_irq(irq);
+                return IRQ_HANDLED;
+        }
+}
+#endif
+
+static void dmae_do_tasklet(unsigned long data)
+{
+        struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data;
+        struct sh_desc *desc, *_desc, *cur_desc = NULL;
+        u32 sar_buf = sh_dmae_readl(sh_chan, SAR);
+        list_for_each_entry_safe(desc, _desc,
+                                        &sh_chan->ld_queue, node) {
+                if ((desc->hw.sar + desc->hw.tcr) == sar_buf) {
+                        cur_desc = desc;
+                        break;
+                }
+        }
+
+        if (cur_desc) {
+                switch (cur_desc->async_tx.cookie) {
+                case 0: /* other desc data */
+                        break;
+                case -EBUSY: /* last desc */
+                sh_chan->completed_cookie =
+                                cur_desc->async_tx.cookie;
+                        break;
+                default: /* first desc ( 0 < )*/
+                        sh_chan->completed_cookie =
+                                cur_desc->async_tx.cookie - 1;
+                        break;
+                }
+                cur_desc->mark = DESC_COMP;
+        }
+        /* Next desc */
+        sh_chan_xfer_ld_queue(sh_chan);
+        sh_dmae_chan_ld_cleanup(sh_chan);
+}
+
+static unsigned int get_dmae_irq(unsigned int id)
+{
+        unsigned int irq = 0;
+        if (id < ARRAY_SIZE(dmte_irq_map))
+                irq = dmte_irq_map[id];
+        return irq;
+}
+
+static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id)
+{
+        int err;
+        unsigned int irq = get_dmae_irq(id);
+        unsigned long irqflags = IRQF_DISABLED;
+        struct sh_dmae_chan *new_sh_chan;
+
+        /* alloc channel */
+        new_sh_chan = kzalloc(sizeof(struct sh_dmae_chan), GFP_KERNEL);
+        if (!new_sh_chan) {
+                dev_err(shdev->common.dev, "No free memory for allocating "
+                                "dma channels!\n");
+                return -ENOMEM;
+        }
+
+        new_sh_chan->dev = shdev->common.dev;
+        new_sh_chan->id = id;
+
+        /* Init DMA tasklet */
+        tasklet_init(&new_sh_chan->tasklet, dmae_do_tasklet,
+                        (unsigned long)new_sh_chan);
+
+        /* Init the channel */
+        dmae_init(new_sh_chan);
+
+        spin_lock_init(&new_sh_chan->desc_lock);
+
+        /* Init descripter manage list */
+        INIT_LIST_HEAD(&new_sh_chan->ld_queue);
+        INIT_LIST_HEAD(&new_sh_chan->ld_free);
+
+        /* copy struct dma_device */
+        new_sh_chan->common.device = &shdev->common;
+
+        /* Add the channel to DMA device channel list */
+        list_add_tail(&new_sh_chan->common.device_node,
+                        &shdev->common.channels);
+        shdev->common.chancnt++;
+
+        if (shdev->pdata.mode & SHDMA_MIX_IRQ) {
+                irqflags = IRQF_SHARED;
+#if defined(DMTE6_IRQ)
+                if (irq >= DMTE6_IRQ)
+                        irq = DMTE6_IRQ;
+                else
+#endif
+                        irq = DMTE0_IRQ;
+        }
+
+        snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id),
+                        "sh-dmae%d", new_sh_chan->id);
+
+        /* set up channel irq */
+        err = request_irq(irq, &sh_dmae_interrupt,
+                irqflags, new_sh_chan->dev_id, new_sh_chan);
+        if (err) {
+                dev_err(shdev->common.dev, "DMA channel %d request_irq error "
+                        "with return %d\n", id, err);
+                goto err_no_irq;
+        }
+
+        /* CHCR register control function */
+        new_sh_chan->set_chcr = dmae_set_chcr;
+        /* DMARS register control function */
+        new_sh_chan->set_dmars = dmae_set_dmars;
+
+        shdev->chan[id] = new_sh_chan;
+        return 0;
+
+err_no_irq:
+        /* remove from dmaengine device node */
+        list_del(&new_sh_chan->common.device_node);
+        kfree(new_sh_chan);
+        return err;
+}
+
+static void sh_dmae_chan_remove(struct sh_dmae_device *shdev)
+{
+        int i;
+
+        for (i = shdev->common.chancnt - 1 ; i >= 0 ; i--) {
+                if (shdev->chan[i]) {
+                        struct sh_dmae_chan *shchan = shdev->chan[i];
+                        if (!(shdev->pdata.mode & SHDMA_MIX_IRQ))
+                                free_irq(dmte_irq_map[i], shchan);
+
+                        list_del(&shchan->common.device_node);
+                        kfree(shchan);
+                        shdev->chan[i] = NULL;
+                }
+        }
+        shdev->common.chancnt = 0;
+}
+
+static int __init sh_dmae_probe(struct platform_device *pdev)
+{
+        int err = 0, cnt, ecnt;
+        unsigned long irqflags = IRQF_DISABLED;
+#if defined(CONFIG_CPU_SH4)
+        int eirq[] = { DMAE0_IRQ,
+#if defined(DMAE1_IRQ)
+                        DMAE1_IRQ
+#endif
+                };
+#endif
+        struct sh_dmae_device *shdev;
+
+        shdev = kzalloc(sizeof(struct sh_dmae_device), GFP_KERNEL);
+        if (!shdev) {
+                dev_err(&pdev->dev, "No enough memory\n");
+                err = -ENOMEM;
+                goto shdev_err;
+        }
+
+        /* get platform data */
+        if (!pdev->dev.platform_data)
+                goto shdev_err;
+
+        /* platform data */
+        memcpy(&shdev->pdata, pdev->dev.platform_data,
+                        sizeof(struct sh_dmae_pdata));
+
+        /* reset dma controller */
+        err = sh_dmae_rst(0);
+        if (err)
+                goto rst_err;
+
+        /* SH7780/85/23 has DMAOR1 */
+        if (shdev->pdata.mode & SHDMA_DMAOR1) {
+                err = sh_dmae_rst(1);
+                if (err)
+                        goto rst_err;
+        }
+
+        INIT_LIST_HEAD(&shdev->common.channels);
+
+        dma_cap_set(DMA_MEMCPY, shdev->common.cap_mask);
+        shdev->common.device_alloc_chan_resources
+                = sh_dmae_alloc_chan_resources;
+        shdev->common.device_free_chan_resources = sh_dmae_free_chan_resources;
+        shdev->common.device_prep_dma_memcpy = sh_dmae_prep_memcpy;
+        shdev->common.device_is_tx_complete = sh_dmae_is_complete;
+        shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending;
+        shdev->common.dev = &pdev->dev;
+
+#if defined(CONFIG_CPU_SH4)
+        /* Non Mix IRQ mode SH7722/SH7730 etc... */
+        if (shdev->pdata.mode & SHDMA_MIX_IRQ) {
+                irqflags = IRQF_SHARED;
+                eirq[0] = DMTE0_IRQ;
+#if defined(DMTE6_IRQ) && defined(DMAE1_IRQ)
+                eirq[1] = DMTE6_IRQ;
+#endif
+        }
+
+        for (ecnt = 0 ; ecnt < ARRAY_SIZE(eirq); ecnt++) {
+                err = request_irq(eirq[ecnt], sh_dmae_err,
+                        irqflags, "DMAC Address Error", shdev);
+                if (err) {
+                        dev_err(&pdev->dev, "DMA device request_irq"
+                                "error (irq %d) with return %d\n",
+                                eirq[ecnt], err);
+                        goto eirq_err;
+                }
+        }
+#endif /* CONFIG_CPU_SH4 */
+
+        /* Create DMA Channel */
+        for (cnt = 0 ; cnt < MAX_DMA_CHANNELS ; cnt++) {
+                err = sh_dmae_chan_probe(shdev, cnt);
+                if (err)
+                        goto chan_probe_err;
+        }
+
+        platform_set_drvdata(pdev, shdev);
+        dma_async_device_register(&shdev->common);
+
+        return err;
+
+chan_probe_err:
+        sh_dmae_chan_remove(shdev);
+
+eirq_err:
+        for (ecnt-- ; ecnt >= 0; ecnt--)
+                free_irq(eirq[ecnt], shdev);
+
+rst_err:
+        kfree(shdev);
+
+shdev_err:
+        return err;
+}
+
+static int __exit sh_dmae_remove(struct platform_device *pdev)
+{
+        struct sh_dmae_device *shdev = platform_get_drvdata(pdev);
+
+        dma_async_device_unregister(&shdev->common);
+
+        if (shdev->pdata.mode & SHDMA_MIX_IRQ) {
+                free_irq(DMTE0_IRQ, shdev);
+#if defined(DMTE6_IRQ)
+                free_irq(DMTE6_IRQ, shdev);
+#endif
+        }
+
+        /* channel data remove */
+        sh_dmae_chan_remove(shdev);
+
+        if (!(shdev->pdata.mode & SHDMA_MIX_IRQ)) {
+                free_irq(DMAE0_IRQ, shdev);
+#if defined(DMAE1_IRQ)
+                free_irq(DMAE1_IRQ, shdev);
+#endif
+        }
+        kfree(shdev);
+
+        return 0;
+}
+
+static void sh_dmae_shutdown(struct platform_device *pdev)
+{
+        struct sh_dmae_device *shdev = platform_get_drvdata(pdev);
+        sh_dmae_ctl_stop(0);
+        if (shdev->pdata.mode & SHDMA_DMAOR1)
+                sh_dmae_ctl_stop(1);
+}
+
+static struct platform_driver sh_dmae_driver = {
+        .remove         = __exit_p(sh_dmae_remove),
+        .shutdown       = sh_dmae_shutdown,
+        .driver = {
+                .name   = "sh-dma-engine",
+        },
+};
+
+static int __init sh_dmae_init(void)
+{
+        return platform_driver_probe(&sh_dmae_driver, sh_dmae_probe);
+}
+module_init(sh_dmae_init);
+
+static void __exit sh_dmae_exit(void)
+{
+        platform_driver_unregister(&sh_dmae_driver);
+}
+module_exit(sh_dmae_exit);
+
+MODULE_AUTHOR("Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>");
+MODULE_DESCRIPTION("Renesas SH DMA Engine driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/dma/shdma.h b/drivers/dma/shdma.h
new file mode 100644
index 000000000000..2b4bc15a2c0a
--- /dev/null
+++ b/drivers/dma/shdma.h
@@ -0,0 +1,64 @@
+/*
+ * Renesas SuperH DMA Engine support
+ *
+ * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>
+ * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved.
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+#ifndef __DMA_SHDMA_H
+#define __DMA_SHDMA_H
+
+#include <linux/device.h>
+#include <linux/dmapool.h>
+#include <linux/dmaengine.h>
+
+#define SH_DMA_TCR_MAX 0x00FFFFFF       /* 16MB */
+
+struct sh_dmae_regs {
+        u32 sar; /* SAR / source address */
+        u32 dar; /* DAR / destination address */
+        u32 tcr; /* TCR / transfer count */
+};
+
+struct sh_desc {
+        struct list_head tx_list;
+        struct sh_dmae_regs hw;
+        struct list_head node;
+        struct dma_async_tx_descriptor async_tx;
+        int mark;
+};
+
+struct sh_dmae_chan {
+        dma_cookie_t completed_cookie;  /* The maximum cookie completed */
+        spinlock_t desc_lock;                   /* Descriptor operation lock */
+        struct list_head ld_queue;              /* Link descriptors queue */
+        struct list_head ld_free;               /* Link descriptors free */
+        struct dma_chan common;                 /* DMA common channel */
+        struct device *dev;                             /* Channel device */
+        struct tasklet_struct tasklet;  /* Tasklet */
+        int descs_allocated;                    /* desc count */
+        int id;                         /* Raw id of this channel */
+        char dev_id[16];        /* unique name per DMAC of channel */
+
+        /* Set chcr */
+        int (*set_chcr)(struct sh_dmae_chan *sh_chan, u32 regs);
+        /* Set DMA resource */
+        int (*set_dmars)(struct sh_dmae_chan *sh_chan, u16 res);
+};
+
+struct sh_dmae_device {
+        struct dma_device common;
+        struct sh_dmae_chan *chan[MAX_DMA_CHANNELS];
+        struct sh_dmae_pdata pdata;
+};
+
+#define to_sh_chan(chan) container_of(chan, struct sh_dmae_chan, common)
+#define to_sh_desc(lh) container_of(lh, struct sh_desc, node)
+#define tx_to_sh_desc(tx) container_of(tx, struct sh_desc, async_tx)
+
+#endif  /* __DMA_SHDMA_H */
diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c
index 7837930146a4..fb6bb64e8861 100644
--- a/drivers/dma/txx9dmac.c
+++ b/drivers/dma/txx9dmac.c
@@ -180,9 +180,8 @@ static struct txx9dmac_desc *txx9dmac_first_queued(struct txx9dmac_chan *dc)
 
 static struct txx9dmac_desc *txx9dmac_last_child(struct txx9dmac_desc *desc)
 {
-        if (!list_empty(&desc->txd.tx_list))
-                desc = list_entry(desc->txd.tx_list.prev,
-                                  struct txx9dmac_desc, desc_node);
+        if (!list_empty(&desc->tx_list))
+                desc = list_entry(desc->tx_list.prev, typeof(*desc), desc_node);
         return desc;
 }
 
@@ -197,6 +196,7 @@ static struct txx9dmac_desc *txx9dmac_desc_alloc(struct txx9dmac_chan *dc,
         desc = kzalloc(sizeof(*desc), flags);
         if (!desc)
                 return NULL;
+        INIT_LIST_HEAD(&desc->tx_list);
         dma_async_tx_descriptor_init(&desc->txd, &dc->chan);
         desc->txd.tx_submit = txx9dmac_tx_submit;
         /* txd.flags will be overwritten in prep funcs */
@@ -245,7 +245,7 @@ static void txx9dmac_sync_desc_for_cpu(struct txx9dmac_chan *dc,
         struct txx9dmac_dev *ddev = dc->ddev;
         struct txx9dmac_desc *child;
 
-        list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+        list_for_each_entry(child, &desc->tx_list, desc_node)
                 dma_sync_single_for_cpu(chan2parent(&dc->chan),
                                 child->txd.phys, ddev->descsize,
                                 DMA_TO_DEVICE);
@@ -267,11 +267,11 @@ static void txx9dmac_desc_put(struct txx9dmac_chan *dc,
                 txx9dmac_sync_desc_for_cpu(dc, desc);
 
                 spin_lock_bh(&dc->lock);
-                list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+                list_for_each_entry(child, &desc->tx_list, desc_node)
                         dev_vdbg(chan2dev(&dc->chan),
                                  "moving child desc %p to freelist\n",
                                  child);
-                list_splice_init(&desc->txd.tx_list, &dc->free_list);
+                list_splice_init(&desc->tx_list, &dc->free_list);
                 dev_vdbg(chan2dev(&dc->chan), "moving desc %p to freelist\n",
                          desc);
                 list_add(&desc->desc_node, &dc->free_list);
@@ -429,7 +429,7 @@ txx9dmac_descriptor_complete(struct txx9dmac_chan *dc,
         param = txd->callback_param;
 
         txx9dmac_sync_desc_for_cpu(dc, desc);
-        list_splice_init(&txd->tx_list, &dc->free_list);
+        list_splice_init(&desc->tx_list, &dc->free_list);
         list_move(&desc->desc_node, &dc->free_list);
 
         if (!ds) {
@@ -571,7 +571,7 @@ static void txx9dmac_handle_error(struct txx9dmac_chan *dc, u32 csr)
                  "Bad descriptor submitted for DMA! (cookie: %d)\n",
                  bad_desc->txd.cookie);
         txx9dmac_dump_desc(dc, &bad_desc->hwdesc);
-        list_for_each_entry(child, &bad_desc->txd.tx_list, desc_node)
+        list_for_each_entry(child, &bad_desc->tx_list, desc_node)
                 txx9dmac_dump_desc(dc, &child->hwdesc);
         /* Pretend the descriptor completed successfully */
         txx9dmac_descriptor_complete(dc, bad_desc);
@@ -613,7 +613,7 @@ static void txx9dmac_scan_descriptors(struct txx9dmac_chan *dc)
                         return;
                 }
 
-                list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+                list_for_each_entry(child, &desc->tx_list, desc_node)
                         if (desc_read_CHAR(dc, child) == chain) {
                                 /* Currently in progress */
                                 if (csr & TXX9_DMA_CSR_ABCHC)
@@ -823,8 +823,7 @@ txx9dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
                         dma_sync_single_for_device(chan2parent(&dc->chan),
                                         prev->txd.phys, ddev->descsize,
                                         DMA_TO_DEVICE);
-                        list_add_tail(&desc->desc_node,
-                                        &first->txd.tx_list);
+                        list_add_tail(&desc->desc_node, &first->tx_list);
                 }
                 prev = desc;
         }
@@ -919,8 +918,7 @@ txx9dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                         prev->txd.phys,
                                         ddev->descsize,
                                         DMA_TO_DEVICE);
-                        list_add_tail(&desc->desc_node,
-                                        &first->txd.tx_list);
+                        list_add_tail(&desc->desc_node, &first->tx_list);
                 }
                 prev = desc;
         }
diff --git a/drivers/dma/txx9dmac.h b/drivers/dma/txx9dmac.h
index c907ff01d276..365d42366b9f 100644
--- a/drivers/dma/txx9dmac.h
+++ b/drivers/dma/txx9dmac.h
@@ -231,6 +231,7 @@ struct txx9dmac_desc {
 
         /* THEN values for driver housekeeping */
         struct list_head                desc_node ____cacheline_aligned;
+        struct list_head                tx_list;
         struct dma_async_tx_descriptor  txd;
         size_t                          len;
 };