2 files changed, 48 insertions, 8 deletions

diff --git a/block/blk-ioc.c b/block/blk-ioc.c
index f53c80ecaf07..92bf55540d87 100644
--- a/block/blk-ioc.c
+++ b/block/blk-ioc.c
@@ -36,11 +36,23 @@ static void icq_free_icq_rcu(struct rcu_head *head)
         kmem_cache_free(icq->__rcu_icq_cache, icq);
 }
 
-/*
- * Exit and free an icq.  Called with both ioc and q locked.
- */
+/* Exit an icq. Called with both ioc and q locked. */
 static void ioc_exit_icq(struct io_cq *icq)
 {
+        struct elevator_type *et = icq->q->elevator->type;
+
+        if (icq->flags & ICQ_EXITED)
+                return;
+
+        if (et->ops.elevator_exit_icq_fn)
+                et->ops.elevator_exit_icq_fn(icq);
+
+        icq->flags |= ICQ_EXITED;
+}
+
+/* Release an icq.  Called with both ioc and q locked. */
+static void ioc_destroy_icq(struct io_cq *icq)
+{
         struct io_context *ioc = icq->ioc;
         struct request_queue *q = icq->q;
         struct elevator_type *et = q->elevator->type;
@@ -60,8 +72,7 @@ static void ioc_exit_icq(struct io_cq *icq)
         if (rcu_dereference_raw(ioc->icq_hint) == icq)
                 rcu_assign_pointer(ioc->icq_hint, NULL);
 
-        if (et->ops.elevator_exit_icq_fn)
-                et->ops.elevator_exit_icq_fn(icq);
+        ioc_exit_icq(icq);
 
         /*
          * @icq->q might have gone away by the time RCU callback runs
@@ -95,7 +106,7 @@ static void ioc_release_fn(struct work_struct *work)
                 struct request_queue *q = icq->q;
 
                 if (spin_trylock(q->queue_lock)) {
-                        ioc_exit_icq(icq);
+                        ioc_destroy_icq(icq);
                         spin_unlock(q->queue_lock);
                 } else {
                         spin_unlock_irqrestore(&ioc->lock, flags);
@@ -142,13 +153,41 @@ EXPORT_SYMBOL(put_io_context);
 void exit_io_context(struct task_struct *task)
 {
         struct io_context *ioc;
+        struct io_cq *icq;
+        struct hlist_node *n;
+        unsigned long flags;
 
         task_lock(task);
         ioc = task->io_context;
         task->io_context = NULL;
         task_unlock(task);
 
-        atomic_dec(&ioc->nr_tasks);
+        if (!atomic_dec_and_test(&ioc->nr_tasks)) {
+                put_io_context(ioc);
+                return;
+        }
+
+        /*
+         * Need ioc lock to walk icq_list and q lock to exit icq.  Perform
+         * reverse double locking.  Read comment in ioc_release_fn() for
+         * explanation on the nested locking annotation.
+         */
+retry:
+        spin_lock_irqsave_nested(&ioc->lock, flags, 1);
+        hlist_for_each_entry(icq, n, &ioc->icq_list, ioc_node) {
+                if (icq->flags & ICQ_EXITED)
+                        continue;
+                if (spin_trylock(icq->q->queue_lock)) {
+                        ioc_exit_icq(icq);
+                        spin_unlock(icq->q->queue_lock);
+                } else {
+                        spin_unlock_irqrestore(&ioc->lock, flags);
+                        cpu_relax();
+                        goto retry;
+                }
+        }
+        spin_unlock_irqrestore(&ioc->lock, flags);
+
         put_io_context(ioc);
 }
 
@@ -168,7 +207,7 @@ void ioc_clear_queue(struct request_queue *q)
                 struct io_context *ioc = icq->ioc;
 
                 spin_lock(&ioc->lock);
-                ioc_exit_icq(icq);
+                ioc_destroy_icq(icq);
                 spin_unlock(&ioc->lock);
         }
 }
diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h
index 17839c7b9614..1a3018063034 100644
--- a/include/linux/iocontext.h
+++ b/include/linux/iocontext.h
@@ -8,6 +8,7 @@
 enum {
         ICQ_IOPRIO_CHANGED      = 1 << 0,
         ICQ_CGROUP_CHANGED      = 1 << 1,
+        ICQ_EXITED              = 1 << 2,
 
         ICQ_CHANGED_MASK        = ICQ_IOPRIO_CHANGED | ICQ_CGROUP_CHANGED,
 };