6 files changed, 90 insertions, 109 deletions

diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index dcc4ab78b32b..5d0e42b263df 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -94,16 +94,12 @@ static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
         return IRQ_HANDLED;
 }
 
-static void ioat1_cleanup_tasklet(unsigned long data);
-
 /* common channel initialization */
-void ioat_init_channel(struct ioatdma_device *device,
-                       struct ioat_chan_common *chan, int idx,
-                       void (*timer_fn)(unsigned long),
-                       void (*tasklet)(unsigned long),
-                       unsigned long ioat)
+void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx)
 {
         struct dma_device *dma = &device->common;
+        struct dma_chan *c = &chan->common;
+        unsigned long data = (unsigned long) c;
 
         chan->device = device;
         chan->reg_base = device->reg_base + (0x80 * (idx + 1));
@@ -112,14 +108,12 @@ void ioat_init_channel(struct ioatdma_device *device,
         list_add_tail(&chan->common.device_node, &dma->channels);
         device->idx[idx] = chan;
         init_timer(&chan->timer);
-        chan->timer.function = timer_fn;
-        chan->timer.data = ioat;
-        tasklet_init(&chan->cleanup_task, tasklet, ioat);
+        chan->timer.function = device->timer_fn;
+        chan->timer.data = data;
+        tasklet_init(&chan->cleanup_task, device->cleanup_fn, data);
         tasklet_disable(&chan->cleanup_task);
 }
 
-static void ioat1_timer_event(unsigned long data);
-
 /**
  * ioat1_dma_enumerate_channels - find and initialize the device's channels
  * @device: the device to be enumerated
@@ -155,10 +149,7 @@ static int ioat1_enumerate_channels(struct ioatdma_device *device)
                 if (!ioat)
                         break;
 
-                ioat_init_channel(device, &ioat->base, i,
-                                  ioat1_timer_event,
-                                  ioat1_cleanup_tasklet,
-                                  (unsigned long) ioat);
+                ioat_init_channel(device, &ioat->base, i);
                 ioat->xfercap = xfercap;
                 spin_lock_init(&ioat->desc_lock);
                 INIT_LIST_HEAD(&ioat->free_desc);
@@ -532,12 +523,12 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
         return &desc->txd;
 }
 
-static void ioat1_cleanup_tasklet(unsigned long data)
+static void ioat1_cleanup_event(unsigned long data)
 {
-        struct ioat_dma_chan *chan = (void *)data;
+        struct ioat_dma_chan *ioat = to_ioat_chan((void *) data);
 
-        ioat1_cleanup(chan);
-        writew(IOAT_CHANCTRL_RUN, chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
+        ioat1_cleanup(ioat);
+        writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
 }
 
 void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
@@ -687,7 +678,7 @@ static void ioat1_cleanup(struct ioat_dma_chan *ioat)
 
 static void ioat1_timer_event(unsigned long data)
 {
-        struct ioat_dma_chan *ioat = (void *) data;
+        struct ioat_dma_chan *ioat = to_ioat_chan((void *) data);
         struct ioat_chan_common *chan = &ioat->base;
 
         dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state);
@@ -734,16 +725,17 @@ static void ioat1_timer_event(unsigned long data)
         spin_unlock_bh(&chan->cleanup_lock);
 }
 
-static enum dma_status
-ioat1_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+enum dma_status
+ioat_is_dma_complete(struct dma_chan *c, dma_cookie_t cookie,
                       dma_cookie_t *done, dma_cookie_t *used)
 {
-        struct ioat_dma_chan *ioat = to_ioat_chan(c);
+        struct ioat_chan_common *chan = to_chan_common(c);
+        struct ioatdma_device *device = chan->device;
 
         if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
                 return DMA_SUCCESS;
 
-        ioat1_cleanup(ioat);
+        device->cleanup_fn((unsigned long) c);
 
         return ioat_is_complete(c, cookie, done, used);
 }
@@ -1199,12 +1191,14 @@ int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca)
         device->intr_quirk = ioat1_intr_quirk;
         device->enumerate_channels = ioat1_enumerate_channels;
         device->self_test = ioat_dma_self_test;
+        device->timer_fn = ioat1_timer_event;
+        device->cleanup_fn = ioat1_cleanup_event;
         dma = &device->common;
         dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
         dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
         dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources;
         dma->device_free_chan_resources = ioat1_dma_free_chan_resources;
-        dma->device_is_tx_complete = ioat1_dma_is_complete;
+        dma->device_is_tx_complete = ioat_is_dma_complete;
 
         err = ioat_probe(device);
         if (err)
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index bbc3e78ef333..4f747a254074 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -61,7 +61,7 @@
  * @intr_quirk: interrupt setup quirk (for ioat_v1 devices)
  * @enumerate_channels: hw version specific channel enumeration
  * @reset_hw: hw version specific channel (re)initialization
- * @cleanup_tasklet: select between the v2 and v3 cleanup routines
+ * @cleanup_fn: select between the v2 and v3 cleanup routines
  * @timer_fn: select between the v2 and v3 timer watchdog routines
  * @self_test: hardware version specific self test for each supported op type
  *
@@ -80,7 +80,7 @@ struct ioatdma_device {
         void (*intr_quirk)(struct ioatdma_device *device);
         int (*enumerate_channels)(struct ioatdma_device *device);
         int (*reset_hw)(struct ioat_chan_common *chan);
-        void (*cleanup_tasklet)(unsigned long data);
+        void (*cleanup_fn)(unsigned long data);
         void (*timer_fn)(unsigned long data);
         int (*self_test)(struct ioatdma_device *device);
 };
@@ -337,10 +337,9 @@ struct dca_provider * __devinit ioat_dca_init(struct pci_dev *pdev,
                                               void __iomem *iobase);
 unsigned long ioat_get_current_completion(struct ioat_chan_common *chan);
 void ioat_init_channel(struct ioatdma_device *device,
-                       struct ioat_chan_common *chan, int idx,
-                       void (*timer_fn)(unsigned long),
-                       void (*tasklet)(unsigned long),
-                       unsigned long ioat);
+                       struct ioat_chan_common *chan, int idx);
+enum dma_status ioat_is_dma_complete(struct dma_chan *c, dma_cookie_t cookie,
+                                     dma_cookie_t *done, dma_cookie_t *used);
 void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
                     size_t len, struct ioat_dma_descriptor *hw);
 bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
index 5cc37afe2bc1..1ed5d66d7dca 100644
--- a/drivers/dma/ioat/dma_v2.c
+++ b/drivers/dma/ioat/dma_v2.c
@@ -51,48 +51,40 @@ MODULE_PARM_DESC(ioat_ring_max_alloc_order,
 
 void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
 {
-        void * __iomem reg_base = ioat->base.reg_base;
+        struct ioat_chan_common *chan = &ioat->base;
 
-        ioat->pending = 0;
         ioat->dmacount += ioat2_ring_pending(ioat);
         ioat->issued = ioat->head;
         /* make descriptor updates globally visible before notifying channel */
         wmb();
-        writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
-        dev_dbg(to_dev(&ioat->base),
+        writew(ioat->dmacount, chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
+        dev_dbg(to_dev(chan),
                 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
                 __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
 }
 
-void ioat2_issue_pending(struct dma_chan *chan)
+void ioat2_issue_pending(struct dma_chan *c)
 {
-        struct ioat2_dma_chan *ioat = to_ioat2_chan(chan);
+        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
 
-        spin_lock_bh(&ioat->ring_lock);
-        if (ioat->pending == 1)
+        if (ioat2_ring_pending(ioat)) {
+                spin_lock_bh(&ioat->ring_lock);
                 __ioat2_issue_pending(ioat);
-        spin_unlock_bh(&ioat->ring_lock);
+                spin_unlock_bh(&ioat->ring_lock);
+        }
 }
 
 /**
  * ioat2_update_pending - log pending descriptors
  * @ioat: ioat2+ channel
  *
- * set pending to '1' unless pending is already set to '2', pending == 2
- * indicates that submission is temporarily blocked due to an in-flight
- * reset.  If we are already above the ioat_pending_level threshold then
- * just issue pending.
- *
- * called with ring_lock held
+ * Check if the number of unsubmitted descriptors has exceeded the
+ * watermark.  Called with ring_lock held
  */
 static void ioat2_update_pending(struct ioat2_dma_chan *ioat)
 {
-        if (unlikely(ioat->pending == 2))
-                return;
-        else if (ioat2_ring_pending(ioat) > ioat_pending_level)
+        if (ioat2_ring_pending(ioat) > ioat_pending_level)
                 __ioat2_issue_pending(ioat);
-        else
-                ioat->pending = 1;
 }
 
 static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
@@ -166,7 +158,7 @@ static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
                         seen_current = true;
         }
         ioat->tail += i;
-        BUG_ON(!seen_current); /* no active descs have written a completion? */
+        BUG_ON(active && !seen_current); /* no active descs have written a completion? */
 
         chan->last_completion = phys_complete;
         if (ioat->head == ioat->tail) {
@@ -207,9 +199,9 @@ static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
         spin_unlock_bh(&chan->cleanup_lock);
 }
 
-void ioat2_cleanup_tasklet(unsigned long data)
+void ioat2_cleanup_event(unsigned long data)
 {
-        struct ioat2_dma_chan *ioat = (void *) data;
+        struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);
 
         ioat2_cleanup(ioat);
         writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
@@ -291,7 +283,7 @@ static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
 
 void ioat2_timer_event(unsigned long data)
 {
-        struct ioat2_dma_chan *ioat = (void *) data;
+        struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);
         struct ioat_chan_common *chan = &ioat->base;
 
         spin_lock_bh(&chan->cleanup_lock);
@@ -397,10 +389,7 @@ int ioat2_enumerate_channels(struct ioatdma_device *device)
                 if (!ioat)
                         break;
 
-                ioat_init_channel(device, &ioat->base, i,
-                                  device->timer_fn,
-                                  device->cleanup_tasklet,
-                                  (unsigned long) ioat);
+                ioat_init_channel(device, &ioat->base, i);
                 ioat->xfercap_log = xfercap_log;
                 spin_lock_init(&ioat->ring_lock);
                 if (device->reset_hw(&ioat->base)) {
@@ -546,7 +535,6 @@ int ioat2_alloc_chan_resources(struct dma_chan *c)
         ioat->head = 0;
         ioat->issued = 0;
         ioat->tail = 0;
-        ioat->pending = 0;
         ioat->alloc_order = order;
         spin_unlock_bh(&ioat->ring_lock);
 
@@ -701,7 +689,7 @@ int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs)
 
                         mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
                         spin_unlock_bh(&chan->cleanup_lock);
-                        device->timer_fn((unsigned long) ioat);
+                        device->timer_fn((unsigned long) &chan->common);
                 } else
                         spin_unlock_bh(&chan->cleanup_lock);
                 return -ENOMEM;
@@ -785,7 +773,7 @@ void ioat2_free_chan_resources(struct dma_chan *c)
 
         tasklet_disable(&chan->cleanup_task);
         del_timer_sync(&chan->timer);
-        device->cleanup_tasklet((unsigned long) ioat);
+        device->cleanup_fn((unsigned long) c);
         device->reset_hw(chan);
 
         spin_lock_bh(&ioat->ring_lock);
@@ -815,25 +803,9 @@ void ioat2_free_chan_resources(struct dma_chan *c)
 
         chan->last_completion = 0;
         chan->completion_dma = 0;
-        ioat->pending = 0;
         ioat->dmacount = 0;
 }
 
-enum dma_status
-ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
-                     dma_cookie_t *done, dma_cookie_t *used)
-{
-        struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
-        struct ioatdma_device *device = ioat->base.device;
-
-        if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
-                return DMA_SUCCESS;
-
-        device->cleanup_tasklet((unsigned long) ioat);
-
-        return ioat_is_complete(c, cookie, done, used);
-}
-
 static ssize_t ring_size_show(struct dma_chan *c, char *page)
 {
         struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
@@ -874,7 +846,7 @@ int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca)
 
         device->enumerate_channels = ioat2_enumerate_channels;
         device->reset_hw = ioat2_reset_hw;
-        device->cleanup_tasklet = ioat2_cleanup_tasklet;
+        device->cleanup_fn = ioat2_cleanup_event;
         device->timer_fn = ioat2_timer_event;
         device->self_test = ioat_dma_self_test;
         dma = &device->common;
@@ -882,7 +854,7 @@ int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca)
         dma->device_issue_pending = ioat2_issue_pending;
         dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
         dma->device_free_chan_resources = ioat2_free_chan_resources;
-        dma->device_is_tx_complete = ioat2_is_complete;
+        dma->device_is_tx_complete = ioat_is_dma_complete;
 
         err = ioat_probe(device);
         if (err)
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
index 3afad8da43cc..ef2871fd7868 100644
--- a/drivers/dma/ioat/dma_v2.h
+++ b/drivers/dma/ioat/dma_v2.h
@@ -47,7 +47,6 @@ extern int ioat_ring_alloc_order;
  * @head: allocated index
  * @issued: hardware notification point
  * @tail: cleanup index
- * @pending: lock free indicator for issued != head
  * @dmacount: identical to 'head' except for occasionally resetting to zero
  * @alloc_order: log2 of the number of allocated descriptors
  * @ring: software ring buffer implementation of hardware ring
@@ -61,7 +60,6 @@ struct ioat2_dma_chan {
         u16 tail;
         u16 dmacount;
         u16 alloc_order;
-        int pending;
         struct ioat_ring_ent **ring;
         spinlock_t ring_lock;
 };
@@ -178,12 +176,10 @@ ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
 void ioat2_issue_pending(struct dma_chan *chan);
 int ioat2_alloc_chan_resources(struct dma_chan *c);
 void ioat2_free_chan_resources(struct dma_chan *c);
-enum dma_status ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
-                                  dma_cookie_t *done, dma_cookie_t *used);
 void __ioat2_restart_chan(struct ioat2_dma_chan *ioat);
 bool reshape_ring(struct ioat2_dma_chan *ioat, int order);
 void __ioat2_issue_pending(struct ioat2_dma_chan *ioat);
-void ioat2_cleanup_tasklet(unsigned long data);
+void ioat2_cleanup_event(unsigned long data);
 void ioat2_timer_event(unsigned long data);
 int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo);
 int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo);
diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c
index 9908c9e94b2d..26febc56dab1 100644
--- a/drivers/dma/ioat/dma_v3.c
+++ b/drivers/dma/ioat/dma_v3.c
@@ -293,17 +293,25 @@ static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
                 }
         }
         ioat->tail += i;
-        BUG_ON(!seen_current); /* no active descs have written a completion? */
+        BUG_ON(active && !seen_current); /* no active descs have written a completion? */
         chan->last_completion = phys_complete;
-        if (ioat->head == ioat->tail) {
+
+        active = ioat2_ring_active(ioat);
+        if (active == 0) {
                 dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
                         __func__);
                 clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
                 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
         }
+        /* 5 microsecond delay per pending descriptor */
+        writew(min((5 * active), IOAT_INTRDELAY_MASK),
+               chan->device->reg_base + IOAT_INTRDELAY_OFFSET);
 }
 
-static void ioat3_cleanup(struct ioat2_dma_chan *ioat)
+/* try to cleanup, but yield (via spin_trylock) to incoming submissions
+ * with the expectation that we will immediately poll again shortly
+ */
+static void ioat3_cleanup_poll(struct ioat2_dma_chan *ioat)
 {
         struct ioat_chan_common *chan = &ioat->base;
         unsigned long phys_complete;
@@ -329,29 +337,41 @@ static void ioat3_cleanup(struct ioat2_dma_chan *ioat)
         spin_unlock_bh(&chan->cleanup_lock);
 }
 
-static void ioat3_cleanup_tasklet(unsigned long data)
+/* run cleanup now because we already delayed the interrupt via INTRDELAY */
+static void ioat3_cleanup_sync(struct ioat2_dma_chan *ioat)
+{
+        struct ioat_chan_common *chan = &ioat->base;
+        unsigned long phys_complete;
+
+        prefetch(chan->completion);
+
+        spin_lock_bh(&chan->cleanup_lock);
+        if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+                spin_unlock_bh(&chan->cleanup_lock);
+                return;
+        }
+        spin_lock_bh(&ioat->ring_lock);
+
+        __cleanup(ioat, phys_complete);
+
+        spin_unlock_bh(&ioat->ring_lock);
+        spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static void ioat3_cleanup_event(unsigned long data)
 {
-        struct ioat2_dma_chan *ioat = (void *) data;
+        struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);
 
-        ioat3_cleanup(ioat);
-        writew(IOAT_CHANCTRL_RUN | IOAT3_CHANCTRL_COMPL_DCA_EN,
-               ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
+        ioat3_cleanup_sync(ioat);
+        writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
 }
 
 static void ioat3_restart_channel(struct ioat2_dma_chan *ioat)
 {
         struct ioat_chan_common *chan = &ioat->base;
         unsigned long phys_complete;
-        u32 status;
-
-        status = ioat_chansts(chan);
-        if (is_ioat_active(status) || is_ioat_idle(status))
-                ioat_suspend(chan);
-        while (is_ioat_active(status) || is_ioat_idle(status)) {
-                status = ioat_chansts(chan);
-                cpu_relax();
-        }
 
+        ioat2_quiesce(chan, 0);
         if (ioat_cleanup_preamble(chan, &phys_complete))
                 __cleanup(ioat, phys_complete);
 
@@ -360,7 +380,7 @@ static void ioat3_restart_channel(struct ioat2_dma_chan *ioat)
 
 static void ioat3_timer_event(unsigned long data)
 {
-        struct ioat2_dma_chan *ioat = (void *) data;
+        struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);
         struct ioat_chan_common *chan = &ioat->base;
 
         spin_lock_bh(&chan->cleanup_lock);
@@ -426,7 +446,7 @@ ioat3_is_complete(struct dma_chan *c, dma_cookie_t cookie,
         if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
                 return DMA_SUCCESS;
 
-        ioat3_cleanup(ioat);
+        ioat3_cleanup_poll(ioat);
 
         return ioat_is_complete(c, cookie, done, used);
 }
@@ -1239,11 +1259,11 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca)
 
         if (is_raid_device) {
                 dma->device_is_tx_complete = ioat3_is_complete;
-                device->cleanup_tasklet = ioat3_cleanup_tasklet;
+                device->cleanup_fn = ioat3_cleanup_event;
                 device->timer_fn = ioat3_timer_event;
         } else {
-                dma->device_is_tx_complete = ioat2_is_complete;
-                device->cleanup_tasklet = ioat2_cleanup_tasklet;
+                dma->device_is_tx_complete = ioat_is_dma_complete;
+                device->cleanup_fn = ioat2_cleanup_event;
                 device->timer_fn = ioat2_timer_event;
         }
 
diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h
index e8ae63baf588..1391798542b6 100644
--- a/drivers/dma/ioat/registers.h
+++ b/drivers/dma/ioat/registers.h
@@ -60,7 +60,7 @@
 #define IOAT_PERPORTOFFSET_OFFSET               0x0A    /* 16-bit */
 
 #define IOAT_INTRDELAY_OFFSET                   0x0C    /* 16-bit */
-#define IOAT_INTRDELAY_INT_DELAY_MASK           0x3FFF  /* Interrupt Delay Time */
+#define IOAT_INTRDELAY_MASK                     0x3FFF  /* Interrupt Delay Time */
 #define IOAT_INTRDELAY_COALESE_SUPPORT          0x8000  /* Interrupt Coalescing Supported */
 
 #define IOAT_DEVICE_STATUS_OFFSET               0x0E    /* 16-bit */