7 files changed, 124 insertions, 65 deletions

diff --git a/block/blk-core.c b/block/blk-core.c
index 90f22cc30799..5fa3dd2705c6 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -198,26 +198,13 @@ void blk_dump_rq_flags(struct request *rq, char *msg)
 }
 EXPORT_SYMBOL(blk_dump_rq_flags);
 
-/*
- * Make sure that plugs that were pending when this function was entered,
- * are now complete and requests pushed to the queue.
-*/
-static inline void queue_sync_plugs(struct request_queue *q)
-{
-        /*
-         * If the current process is plugged and has barriers submitted,
-         * we will livelock if we don't unplug first.
-         */
-        blk_flush_plug(current);
-}
-
 static void blk_delay_work(struct work_struct *work)
 {
         struct request_queue *q;
 
         q = container_of(work, struct request_queue, delay_work.work);
         spin_lock_irq(q->queue_lock);
-        __blk_run_queue(q, false);
+        __blk_run_queue(q);
         spin_unlock_irq(q->queue_lock);
 }
 
@@ -233,7 +220,8 @@ static void blk_delay_work(struct work_struct *work)
  */
 void blk_delay_queue(struct request_queue *q, unsigned long msecs)
 {
-        schedule_delayed_work(&q->delay_work, msecs_to_jiffies(msecs));
+        queue_delayed_work(kblockd_workqueue, &q->delay_work,
+                                msecs_to_jiffies(msecs));
 }
 EXPORT_SYMBOL(blk_delay_queue);
 
@@ -251,7 +239,7 @@ void blk_start_queue(struct request_queue *q)
         WARN_ON(!irqs_disabled());
 
         queue_flag_clear(QUEUE_FLAG_STOPPED, q);
-        __blk_run_queue(q, false);
+        __blk_run_queue(q);
 }
 EXPORT_SYMBOL(blk_start_queue);
 
@@ -298,7 +286,6 @@ void blk_sync_queue(struct request_queue *q)
 {
         del_timer_sync(&q->timeout);
         cancel_delayed_work_sync(&q->delay_work);
-        queue_sync_plugs(q);
 }
 EXPORT_SYMBOL(blk_sync_queue);
 
@@ -310,9 +297,8 @@ EXPORT_SYMBOL(blk_sync_queue);
  * Description:
  *    See @blk_run_queue. This variant must be called with the queue lock
  *    held and interrupts disabled.
- *
  */
-void __blk_run_queue(struct request_queue *q, bool force_kblockd)
+void __blk_run_queue(struct request_queue *q)
 {
         if (unlikely(blk_queue_stopped(q)))
                 return;
@@ -321,7 +307,7 @@ void __blk_run_queue(struct request_queue *q, bool force_kblockd)
          * Only recurse once to avoid overrunning the stack, let the unplug
          * handling reinvoke the handler shortly if we already got there.
          */
-        if (!force_kblockd && !queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
+        if (!queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
                 q->request_fn(q);
                 queue_flag_clear(QUEUE_FLAG_REENTER, q);
         } else
@@ -330,6 +316,20 @@ void __blk_run_queue(struct request_queue *q, bool force_kblockd)
 EXPORT_SYMBOL(__blk_run_queue);
 
 /**
+ * blk_run_queue_async - run a single device queue in workqueue context
+ * @q:  The queue to run
+ *
+ * Description:
+ *    Tells kblockd to perform the equivalent of @blk_run_queue on behalf
+ *    of us.
+ */
+void blk_run_queue_async(struct request_queue *q)
+{
+        if (likely(!blk_queue_stopped(q)))
+                queue_delayed_work(kblockd_workqueue, &q->delay_work, 0);
+}
+
+/**
  * blk_run_queue - run a single device queue
  * @q: The queue to run
  *
@@ -342,7 +342,7 @@ void blk_run_queue(struct request_queue *q)
         unsigned long flags;
 
         spin_lock_irqsave(q->queue_lock, flags);
-        __blk_run_queue(q, false);
+        __blk_run_queue(q);
         spin_unlock_irqrestore(q->queue_lock, flags);
 }
 EXPORT_SYMBOL(blk_run_queue);
@@ -991,7 +991,7 @@ void blk_insert_request(struct request_queue *q, struct request *rq,
                 blk_queue_end_tag(q, rq);
 
         add_acct_request(q, rq, where);
-        __blk_run_queue(q, false);
+        __blk_run_queue(q);
         spin_unlock_irqrestore(q->queue_lock, flags);
 }
 EXPORT_SYMBOL(blk_insert_request);
@@ -1311,7 +1311,15 @@ get_rq:
 
         plug = current->plug;
         if (plug) {
-                if (!plug->should_sort && !list_empty(&plug->list)) {
+                /*
+                 * If this is the first request added after a plug, fire
+                 * of a plug trace. If others have been added before, check
+                 * if we have multiple devices in this plug. If so, make a
+                 * note to sort the list before dispatch.
+                 */
+                if (list_empty(&plug->list))
+                        trace_block_plug(q);
+                else if (!plug->should_sort) {
                         struct request *__rq;
 
                         __rq = list_entry_rq(plug->list.prev);
@@ -1327,7 +1335,7 @@ get_rq:
         } else {
                 spin_lock_irq(q->queue_lock);
                 add_acct_request(q, req, where);
-                __blk_run_queue(q, false);
+                __blk_run_queue(q);
 out_unlock:
                 spin_unlock_irq(q->queue_lock);
         }
@@ -2644,6 +2652,7 @@ void blk_start_plug(struct blk_plug *plug)
 
         plug->magic = PLUG_MAGIC;
         INIT_LIST_HEAD(&plug->list);
+        INIT_LIST_HEAD(&plug->cb_list);
         plug->should_sort = 0;
 
         /*
@@ -2668,33 +2677,93 @@ static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
         return !(rqa->q <= rqb->q);
 }
 
-static void flush_plug_list(struct blk_plug *plug)
+/*
+ * If 'from_schedule' is true, then postpone the dispatch of requests
+ * until a safe kblockd context. We due this to avoid accidental big
+ * additional stack usage in driver dispatch, in places where the originally
+ * plugger did not intend it.
+ */
+static void queue_unplugged(struct request_queue *q, unsigned int depth,
+                            bool from_schedule)
+        __releases(q->queue_lock)
+{
+        trace_block_unplug(q, depth, !from_schedule);
+
+        /*
+         * If we are punting this to kblockd, then we can safely drop
+         * the queue_lock before waking kblockd (which needs to take
+         * this lock).
+         */
+        if (from_schedule) {
+                spin_unlock(q->queue_lock);
+                blk_run_queue_async(q);
+        } else {
+                __blk_run_queue(q);
+                spin_unlock(q->queue_lock);
+        }
+
+}
+
+static void flush_plug_callbacks(struct blk_plug *plug)
+{
+        LIST_HEAD(callbacks);
+
+        if (list_empty(&plug->cb_list))
+                return;
+
+        list_splice_init(&plug->cb_list, &callbacks);
+
+        while (!list_empty(&callbacks)) {
+                struct blk_plug_cb *cb = list_first_entry(&callbacks,
+                                                          struct blk_plug_cb,
+                                                          list);
+                list_del(&cb->list);
+                cb->callback(cb);
+        }
+}
+
+void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
 {
         struct request_queue *q;
         unsigned long flags;
         struct request *rq;
+        LIST_HEAD(list);
+        unsigned int depth;
 
         BUG_ON(plug->magic != PLUG_MAGIC);
 
+        flush_plug_callbacks(plug);
         if (list_empty(&plug->list))
                 return;
 
-        if (plug->should_sort)
-                list_sort(NULL, &plug->list, plug_rq_cmp);
+        list_splice_init(&plug->list, &list);
+
+        if (plug->should_sort) {
+                list_sort(NULL, &list, plug_rq_cmp);
+                plug->should_sort = 0;
+        }
 
         q = NULL;
+        depth = 0;
+
+        /*
+         * Save and disable interrupts here, to avoid doing it for every
+         * queue lock we have to take.
+         */
         local_irq_save(flags);
-        while (!list_empty(&plug->list)) {
-                rq = list_entry_rq(plug->list.next);
+        while (!list_empty(&list)) {
+                rq = list_entry_rq(list.next);
                 list_del_init(&rq->queuelist);
                 BUG_ON(!(rq->cmd_flags & REQ_ON_PLUG));
                 BUG_ON(!rq->q);
                 if (rq->q != q) {
-                        if (q) {
-                                __blk_run_queue(q, false);
-                                spin_unlock(q->queue_lock);
-                        }
+                        /*
+                         * This drops the queue lock
+                         */
+                        if (q)
+                                queue_unplugged(q, depth, from_schedule);
                         q = rq->q;
+                        depth = 0;
                         spin_lock(q->queue_lock);
                 }
                 rq->cmd_flags &= ~REQ_ON_PLUG;
@@ -2706,38 +2775,28 @@ static void flush_plug_list(struct blk_plug *plug)
                         __elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH);
                 else
                         __elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE);
-        }
 
-        if (q) {
-                __blk_run_queue(q, false);
-                spin_unlock(q->queue_lock);
+                depth++;
         }
 
-        BUG_ON(!list_empty(&plug->list));
-        local_irq_restore(flags);
-}
-
-static void __blk_finish_plug(struct task_struct *tsk, struct blk_plug *plug)
-{
-        flush_plug_list(plug);
+        /*
+         * This drops the queue lock
+         */
+        if (q)
+                queue_unplugged(q, depth, from_schedule);
 
-        if (plug == tsk->plug)
-                tsk->plug = NULL;
+        local_irq_restore(flags);
 }
+EXPORT_SYMBOL(blk_flush_plug_list);
 
 void blk_finish_plug(struct blk_plug *plug)
 {
-        if (plug)
-                __blk_finish_plug(current, plug);
-}
-EXPORT_SYMBOL(blk_finish_plug);
+        blk_flush_plug_list(plug, false);
 
-void __blk_flush_plug(struct task_struct *tsk, struct blk_plug *plug)
-{
-        __blk_finish_plug(tsk, plug);
-        tsk->plug = plug;
+        if (plug == current->plug)
+                current->plug = NULL;
 }
-EXPORT_SYMBOL(__blk_flush_plug);
+EXPORT_SYMBOL(blk_finish_plug);
 
 int __init blk_dev_init(void)
 {
diff --git a/block/blk-exec.c b/block/blk-exec.c
index 7482b7fa863b..81e31819a597 100644
--- a/block/blk-exec.c
+++ b/block/blk-exec.c
@@ -55,7 +55,7 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
         WARN_ON(irqs_disabled());
         spin_lock_irq(q->queue_lock);
         __elv_add_request(q, rq, where);
-        __blk_run_queue(q, false);
+        __blk_run_queue(q);
         /* the queue is stopped so it won't be plugged+unplugged */
         if (rq->cmd_type == REQ_TYPE_PM_RESUME)
                 q->request_fn(q);
diff --git a/block/blk-flush.c b/block/blk-flush.c
index eba4a2790c6c..6c9b5e189e62 100644
--- a/block/blk-flush.c
+++ b/block/blk-flush.c
@@ -218,7 +218,7 @@ static void flush_end_io(struct request *flush_rq, int error)
          * request_fn may confuse the driver.  Always use kblockd.
          */
         if (queued)
-                __blk_run_queue(q, true);
+                blk_run_queue_async(q);
 }
 
 /**
@@ -274,7 +274,7 @@ static void flush_data_end_io(struct request *rq, int error)
          * the comment in flush_end_io().
          */
         if (blk_flush_complete_seq(rq, REQ_FSEQ_DATA, error))
-                __blk_run_queue(q, true);
+                blk_run_queue_async(q);
 }
 
 /**
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 261c75c665ae..6d735122bc59 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -498,7 +498,6 @@ int blk_register_queue(struct gendisk *disk)
 {
         int ret;
         struct device *dev = disk_to_dev(disk);
-
         struct request_queue *q = disk->queue;
 
         if (WARN_ON(!q))
@@ -521,7 +520,7 @@ int blk_register_queue(struct gendisk *disk)
         if (ret) {
                 kobject_uevent(&q->kobj, KOBJ_REMOVE);
                 kobject_del(&q->kobj);
-                blk_trace_remove_sysfs(disk_to_dev(disk));
+                blk_trace_remove_sysfs(dev);
                 kobject_put(&dev->kobj);
                 return ret;
         }
diff --git a/block/blk.h b/block/blk.h
index 61263463e38e..c9df8fc3c999 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -22,6 +22,7 @@ void blk_rq_timed_out_timer(unsigned long data);
 void blk_delete_timer(struct request *);
 void blk_add_timer(struct request *);
 void __generic_unplug_device(struct request_queue *);
+void blk_run_queue_async(struct request_queue *q);
 
 /*
  * Internal atomic flags for request handling
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 3be881ec95ad..46b0a1d1d925 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -3368,7 +3368,7 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                             cfqd->busy_queues > 1) {
                                 cfq_del_timer(cfqd, cfqq);
                                 cfq_clear_cfqq_wait_request(cfqq);
-                                __blk_run_queue(cfqd->queue, false);
+                                __blk_run_queue(cfqd->queue);
                         } else {
                                 cfq_blkiocg_update_idle_time_stats(
                                                 &cfqq->cfqg->blkg);
@@ -3383,7 +3383,7 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                  * this new queue is RT and the current one is BE
                  */
                 cfq_preempt_queue(cfqd, cfqq);
-                __blk_run_queue(cfqd->queue, false);
+                __blk_run_queue(cfqd->queue);
         }
 }
 
@@ -3743,7 +3743,7 @@ static void cfq_kick_queue(struct work_struct *work)
         struct request_queue *q = cfqd->queue;
 
         spin_lock_irq(q->queue_lock);
-        __blk_run_queue(cfqd->queue, false);
+        __blk_run_queue(cfqd->queue);
         spin_unlock_irq(q->queue_lock);
 }
 
diff --git a/block/elevator.c b/block/elevator.c
index 0cdb4e7ebab4..6f6abc08bb56 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -642,7 +642,7 @@ void elv_quiesce_start(struct request_queue *q)
          */
         elv_drain_elevator(q);
         while (q->rq.elvpriv) {
-                __blk_run_queue(q, false);
+                __blk_run_queue(q);
                 spin_unlock_irq(q->queue_lock);
                 msleep(10);
                 spin_lock_irq(q->queue_lock);
@@ -695,7 +695,7 @@ void __elv_add_request(struct request_queue *q, struct request *rq, int where)
                  *   with anything.  There's no point in delaying queue
                  *   processing.
                  */
-                __blk_run_queue(q, false);
+                __blk_run_queue(q);
                 break;
 
         case ELEVATOR_INSERT_SORT_MERGE: