rsxx: Restructured DMA cancel scheme.

Before, DMAs would never be cancelled if there was a data stall
or an EEH Permenant failure which would cause an unrecoverable
I/O hang.

The DMA cancellation mechanism has been modified to fix
these issues and allows DMAs to be cancelled during the
above mentioned events.

Signed-off-by: Philip J Kelleher <pjk1939@linux.vnet.ibm.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

4 files changed, 95 insertions, 93 deletions

diff --git a/drivers/block/rsxx/core.c b/drivers/block/rsxx/core.c
index 774f810c6a9c..aca3f198e5cd 100644
--- a/drivers/block/rsxx/core.c
+++ b/drivers/block/rsxx/core.c
@@ -368,15 +368,26 @@ static void rsxx_eeh_failure(struct pci_dev *dev)
 {
         struct rsxx_cardinfo *card = pci_get_drvdata(dev);
         int i;
+        int cnt = 0;
 
         dev_err(&dev->dev, "IBM FlashSystem PCI: disabling failed card.\n");
 
         card->eeh_state = 1;
+        card->halt = 1;
+
+        for (i = 0; i < card->n_targets; i++) {
+                spin_lock_bh(&card->ctrl[i].queue_lock);
+                cnt = rsxx_cleanup_dma_queue(&card->ctrl[i],
+                                             &card->ctrl[i].queue);
+                spin_unlock_bh(&card->ctrl[i].queue_lock);
 
-        for (i = 0; i < card->n_targets; i++)
-                del_timer_sync(&card->ctrl[i].activity_timer);
+                cnt += rsxx_dma_cancel(&card->ctrl[i]);
 
-        rsxx_eeh_cancel_dmas(card);
+                if (cnt)
+                        dev_info(CARD_TO_DEV(card),
+                                "Freed %d queued DMAs on channel %d\n",
+                                cnt, card->ctrl[i].id);
+        }
 }
 
 static int rsxx_eeh_fifo_flush_poll(struct rsxx_cardinfo *card)
diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c
index 4346d17d2949..604ad2dafa43 100644
--- a/drivers/block/rsxx/dev.c
+++ b/drivers/block/rsxx/dev.c
@@ -155,7 +155,8 @@ static void bio_dma_done_cb(struct rsxx_cardinfo *card,
                 atomic_set(&meta->error, 1);
 
         if (atomic_dec_and_test(&meta->pending_dmas)) {
-                disk_stats_complete(card, meta->bio, meta->start_time);
+                if (!card->eeh_state && card->gendisk)
+                        disk_stats_complete(card, meta->bio, meta->start_time);
 
                 bio_endio(meta->bio, atomic_read(&meta->error) ? -EIO : 0);
                 kmem_cache_free(bio_meta_pool, meta);
@@ -196,7 +197,8 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio)
         atomic_set(&bio_meta->pending_dmas, 0);
         bio_meta->start_time = jiffies;
 
-        disk_stats_start(card, bio);
+        if (!unlikely(card->halt))
+                disk_stats_start(card, bio);
 
         dev_dbg(CARD_TO_DEV(card), "BIO[%c]: meta: %p addr8: x%llx size: %d\n",
                  bio_data_dir(bio) ? 'W' : 'R', bio_meta,
diff --git a/drivers/block/rsxx/dma.c b/drivers/block/rsxx/dma.c
index 0607513cfb41..213e40e4bd92 100644
--- a/drivers/block/rsxx/dma.c
+++ b/drivers/block/rsxx/dma.c
@@ -245,6 +245,22 @@ static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
         kmem_cache_free(rsxx_dma_pool, dma);
 }
 
+int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
+                           struct list_head *q)
+{
+        struct rsxx_dma *dma;
+        struct rsxx_dma *tmp;
+        int cnt = 0;
+
+        list_for_each_entry_safe(dma, tmp, q, list) {
+                list_del(&dma->list);
+                rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
+                cnt++;
+        }
+
+        return cnt;
+}
+
 static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
                                  struct rsxx_dma *dma)
 {
@@ -252,9 +268,9 @@ static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
          * Requeued DMAs go to the front of the queue so they are issued
          * first.
          */
-        spin_lock(&ctrl->queue_lock);
+        spin_lock_bh(&ctrl->queue_lock);
         list_add(&dma->list, &ctrl->queue);
-        spin_unlock(&ctrl->queue_lock);
+        spin_unlock_bh(&ctrl->queue_lock);
 }
 
 static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
@@ -329,6 +345,7 @@ static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
 static void dma_engine_stalled(unsigned long data)
 {
         struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data;
+        int cnt;
 
         if (atomic_read(&ctrl->stats.hw_q_depth) == 0 ||
             unlikely(ctrl->card->eeh_state))
@@ -349,6 +366,18 @@ static void dma_engine_stalled(unsigned long data)
                         "DMA channel %d has stalled, faulting interface.\n",
                         ctrl->id);
                 ctrl->card->dma_fault = 1;
+
+                /* Clean up the DMA queue */
+                spin_lock(&ctrl->queue_lock);
+                cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue);
+                spin_unlock(&ctrl->queue_lock);
+
+                cnt += rsxx_dma_cancel(ctrl);
+
+                if (cnt)
+                        dev_info(CARD_TO_DEV(ctrl->card),
+                                "Freed %d queued DMAs on channel %d\n",
+                                cnt, ctrl->id);
         }
 }
 
@@ -368,22 +397,22 @@ static void rsxx_issue_dmas(struct work_struct *work)
                 return;
 
         while (1) {
-                spin_lock(&ctrl->queue_lock);
+                spin_lock_bh(&ctrl->queue_lock);
                 if (list_empty(&ctrl->queue)) {
-                        spin_unlock(&ctrl->queue_lock);
+                        spin_unlock_bh(&ctrl->queue_lock);
                         break;
                 }
-                spin_unlock(&ctrl->queue_lock);
+                spin_unlock_bh(&ctrl->queue_lock);
 
                 tag = pop_tracker(ctrl->trackers);
                 if (tag == -1)
                         break;
 
-                spin_lock(&ctrl->queue_lock);
+                spin_lock_bh(&ctrl->queue_lock);
                 dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
                 list_del(&dma->list);
                 ctrl->stats.sw_q_depth--;
-                spin_unlock(&ctrl->queue_lock);
+                spin_unlock_bh(&ctrl->queue_lock);
 
                 /*
                  * This will catch any DMAs that slipped in right before the
@@ -520,33 +549,10 @@ static void rsxx_dma_done(struct work_struct *work)
         rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
         spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
 
-        spin_lock(&ctrl->queue_lock);
+        spin_lock_bh(&ctrl->queue_lock);
         if (ctrl->stats.sw_q_depth)
                 queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
-        spin_unlock(&ctrl->queue_lock);
-}
-
-static int rsxx_cleanup_dma_queue(struct rsxx_cardinfo *card,
-                                      struct list_head *q)
-{
-        struct rsxx_dma *dma;
-        struct rsxx_dma *tmp;
-        int cnt = 0;
-
-        list_for_each_entry_safe(dma, tmp, q, list) {
-                list_del(&dma->list);
-
-                if (dma->dma_addr)
-                        pci_unmap_page(card->dev, dma->dma_addr,
-                                       get_dma_size(dma),
-                                       (dma->cmd == HW_CMD_BLK_WRITE) ?
-                                       PCI_DMA_TODEVICE :
-                                       PCI_DMA_FROMDEVICE);
-                kmem_cache_free(rsxx_dma_pool, dma);
-                cnt++;
-        }
-
-        return cnt;
+        spin_unlock_bh(&ctrl->queue_lock);
 }
 
 static int rsxx_queue_discard(struct rsxx_cardinfo *card,
@@ -698,10 +704,10 @@ int rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
 
         for (i = 0; i < card->n_targets; i++) {
                 if (!list_empty(&dma_list[i])) {
-                        spin_lock(&card->ctrl[i].queue_lock);
+                        spin_lock_bh(&card->ctrl[i].queue_lock);
                         card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
                         list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
-                        spin_unlock(&card->ctrl[i].queue_lock);
+                        spin_unlock_bh(&card->ctrl[i].queue_lock);
 
                         queue_work(card->ctrl[i].issue_wq,
                                    &card->ctrl[i].issue_dma_work);
@@ -711,8 +717,11 @@ int rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
         return 0;
 
 bvec_err:
-        for (i = 0; i < card->n_targets; i++)
-                rsxx_cleanup_dma_queue(card, &dma_list[i]);
+        for (i = 0; i < card->n_targets; i++) {
+                spin_lock_bh(&card->ctrl[i].queue_lock);
+                rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i]);
+                spin_unlock_bh(&card->ctrl[i].queue_lock);
+        }
 
         return st;
 }
@@ -918,13 +927,30 @@ failed_dma_setup:
         return st;
 }
 
+int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl)
+{
+        struct rsxx_dma *dma;
+        int i;
+        int cnt = 0;
+
+        /* Clean up issued DMAs */
+        for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
+                dma = get_tracker_dma(ctrl->trackers, i);
+                if (dma) {
+                        atomic_dec(&ctrl->stats.hw_q_depth);
+                        rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
+                        push_tracker(ctrl->trackers, i);
+                        cnt++;
+                }
+        }
+
+        return cnt;
+}
 
 void rsxx_dma_destroy(struct rsxx_cardinfo *card)
 {
         struct rsxx_dma_ctrl *ctrl;
-        struct rsxx_dma *dma;
-        int i, j;
-        int cnt = 0;
+        int i;
 
         for (i = 0; i < card->n_targets; i++) {
                 ctrl = &card->ctrl[i];
@@ -943,33 +969,11 @@ void rsxx_dma_destroy(struct rsxx_cardinfo *card)
                         del_timer_sync(&ctrl->activity_timer);
 
                 /* Clean up the DMA queue */
-                spin_lock(&ctrl->queue_lock);
-                cnt = rsxx_cleanup_dma_queue(card, &ctrl->queue);
-                spin_unlock(&ctrl->queue_lock);
+                spin_lock_bh(&ctrl->queue_lock);
+                rsxx_cleanup_dma_queue(ctrl, &ctrl->queue);
+                spin_unlock_bh(&ctrl->queue_lock);
 
-                if (cnt)
-                        dev_info(CARD_TO_DEV(card),
-                                "Freed %d queued DMAs on channel %d\n",
-                                cnt, i);
-
-                /* Clean up issued DMAs */
-                for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
-                        dma = get_tracker_dma(ctrl->trackers, j);
-                        if (dma) {
-                                pci_unmap_page(card->dev, dma->dma_addr,
-                                               get_dma_size(dma),
-                                               (dma->cmd == HW_CMD_BLK_WRITE) ?
-                                               PCI_DMA_TODEVICE :
-                                               PCI_DMA_FROMDEVICE);
-                                kmem_cache_free(rsxx_dma_pool, dma);
-                                cnt++;
-                        }
-                }
-
-                if (cnt)
-                        dev_info(CARD_TO_DEV(card),
-                                "Freed %d pending DMAs on channel %d\n",
-                                cnt, i);
+                rsxx_dma_cancel(ctrl);
 
                 vfree(ctrl->trackers);
 
@@ -1013,7 +1017,7 @@ int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
                         cnt++;
                 }
 
-                spin_lock(&card->ctrl[i].queue_lock);
+                spin_lock_bh(&card->ctrl[i].queue_lock);
                 list_splice(&issued_dmas[i], &card->ctrl[i].queue);
 
                 atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
@@ -1028,7 +1032,7 @@ int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
                                                PCI_DMA_TODEVICE :
                                                PCI_DMA_FROMDEVICE);
                 }
-                spin_unlock(&card->ctrl[i].queue_lock);
+                spin_unlock_bh(&card->ctrl[i].queue_lock);
         }
 
         kfree(issued_dmas);
@@ -1036,30 +1040,13 @@ int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
         return 0;
 }
 
-void rsxx_eeh_cancel_dmas(struct rsxx_cardinfo *card)
-{
-        struct rsxx_dma *dma;
-        struct rsxx_dma *tmp;
-        int i;
-
-        for (i = 0; i < card->n_targets; i++) {
-                spin_lock(&card->ctrl[i].queue_lock);
-                list_for_each_entry_safe(dma, tmp, &card->ctrl[i].queue, list) {
-                        list_del(&dma->list);
-
-                        rsxx_complete_dma(&card->ctrl[i], dma, DMA_CANCELLED);
-                }
-                spin_unlock(&card->ctrl[i].queue_lock);
-        }
-}
-
 int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card)
 {
         struct rsxx_dma *dma;
         int i;
 
         for (i = 0; i < card->n_targets; i++) {
-                spin_lock(&card->ctrl[i].queue_lock);
+                spin_lock_bh(&card->ctrl[i].queue_lock);
                 list_for_each_entry(dma, &card->ctrl[i].queue, list) {
                         dma->dma_addr = pci_map_page(card->dev, dma->page,
                                         dma->pg_off, get_dma_size(dma),
@@ -1067,12 +1054,12 @@ int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card)
                                         PCI_DMA_TODEVICE :
                                         PCI_DMA_FROMDEVICE);
                         if (!dma->dma_addr) {
-                                spin_unlock(&card->ctrl[i].queue_lock);
+                                spin_unlock_bh(&card->ctrl[i].queue_lock);
                                 kmem_cache_free(rsxx_dma_pool, dma);
                                 return -ENOMEM;
                         }
                 }
-                spin_unlock(&card->ctrl[i].queue_lock);
+                spin_unlock_bh(&card->ctrl[i].queue_lock);
         }
 
         return 0;
diff --git a/drivers/block/rsxx/rsxx_priv.h b/drivers/block/rsxx/rsxx_priv.h
index 0dd62d966772..60b6ed6779ac 100644
--- a/drivers/block/rsxx/rsxx_priv.h
+++ b/drivers/block/rsxx/rsxx_priv.h
@@ -39,6 +39,7 @@
 #include <linux/vmalloc.h>
 #include <linux/timer.h>
 #include <linux/ioctl.h>
+#include <linux/delay.h>
 
 #include "rsxx.h"
 #include "rsxx_cfg.h"
@@ -374,6 +375,8 @@ typedef void (*rsxx_dma_cb)(struct rsxx_cardinfo *card,
 int rsxx_dma_setup(struct rsxx_cardinfo *card);
 void rsxx_dma_destroy(struct rsxx_cardinfo *card);
 int rsxx_dma_init(void);
+int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl, struct list_head *q);
+int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl);
 void rsxx_dma_cleanup(void);
 void rsxx_dma_queue_reset(struct rsxx_cardinfo *card);
 int rsxx_dma_configure(struct rsxx_cardinfo *card);
@@ -384,7 +387,6 @@ int rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
                            void *cb_data);
 int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl);
 int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card);
-void rsxx_eeh_cancel_dmas(struct rsxx_cardinfo *card);
 int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card);
 
 /***** cregs.c *****/