async_tx: remove walk of tx->parent chain in dma_wait_for_async_tx

We currently walk the parent chain when waiting for a given tx to
complete however this walk may race with the driver cleanup routine.
The routines in async_raid6_recov.c may fall back to the synchronous
path at any point so we need to be prepared to call async_tx_quiesce()
(which calls  dma_wait_for_async_tx).  To remove the ->parent walk we
guarantee that every time a dependency is attached ->issue_pending() is
invoked, then we can simply poll the initial descriptor until
completion.

This also allows for a lighter weight 'issue pending' implementation as
there is no longer a requirement to iterate through all the channels'
->issue_pending() routines as long as operations have been submitted in
an ordered chain.  async_tx_issue_pending() is added for this case.

Signed-off-by: Dan Williams <dan.j.williams@intel.com>

1 files changed, 7 insertions, 6 deletions

diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c
index 6e37ad3f4417..60615fedcf5e 100644
--- a/crypto/async_tx/async_tx.c
+++ b/crypto/async_tx/async_tx.c
@@ -77,8 +77,8 @@ static void
 async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
                         struct dma_async_tx_descriptor *tx)
 {
-        struct dma_chan *chan;
-        struct dma_device *device;
+        struct dma_chan *chan = depend_tx->chan;
+        struct dma_device *device = chan->device;
         struct dma_async_tx_descriptor *intr_tx = (void *) ~0;
 
         /* first check to see if we can still append to depend_tx */
@@ -90,11 +90,11 @@ async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
         }
         spin_unlock_bh(&depend_tx->lock);
 
-        if (!intr_tx)
+        /* attached dependency, flush the parent channel */
+        if (!intr_tx) {
+                device->device_issue_pending(chan);
                 return;
-
-        chan = depend_tx->chan;
-        device = chan->device;
+        }
 
         /* see if we can schedule an interrupt
          * otherwise poll for completion
@@ -128,6 +128,7 @@ async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
                         intr_tx->tx_submit(intr_tx);
                         async_tx_ack(intr_tx);
                 }
+                device->device_issue_pending(chan);
         } else {
                 if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
                         panic("%s: DMA_ERROR waiting for depend_tx\n",