block: add @force_kblockd to __blk_run_queue()

__blk_run_queue() automatically either calls q->request_fn() directly
or schedules kblockd depending on whether the function is recursed.
blk-flush implementation needs to be able to explicitly choose
kblockd.  Add @force_kblockd.

All the current users are converted to specify %false for the
parameter and this patch doesn't introduce any behavior change.

stable: This is prerequisite for fixing ide oops caused by the new
        blk-flush implementation.

Signed-off-by: Tejun Heo <tj@kernel.org>
Cc: Jan Beulich <JBeulich@novell.com>
Cc: James Bottomley <James.Bottomley@HansenPartnership.com>
Cc: stable@kernel.org
Signed-off-by: Jens Axboe <jaxboe@fusionio.com>

7 files changed, 15 insertions, 14 deletions

diff --git a/block/blk-core.c b/block/blk-core.c
index 792ece276160..518dd423a5fe 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -352,7 +352,7 @@ void blk_start_queue(struct request_queue *q)
         WARN_ON(!irqs_disabled());
 
         queue_flag_clear(QUEUE_FLAG_STOPPED, q);
-        __blk_run_queue(q);
+        __blk_run_queue(q, false);
 }
 EXPORT_SYMBOL(blk_start_queue);
 
@@ -403,13 +403,14 @@ EXPORT_SYMBOL(blk_sync_queue);
 /**
  * __blk_run_queue - run a single device queue
  * @q:  The queue to run
+ * @force_kblockd: Don't run @q->request_fn directly.  Use kblockd.
  *
  * Description:
  *    See @blk_run_queue. This variant must be called with the queue lock
  *    held and interrupts disabled.
  *
  */
-void __blk_run_queue(struct request_queue *q)
+void __blk_run_queue(struct request_queue *q, bool force_kblockd)
 {
         blk_remove_plug(q);
 
@@ -423,7 +424,7 @@ void __blk_run_queue(struct request_queue *q)
          * Only recurse once to avoid overrunning the stack, let the unplug
          * handling reinvoke the handler shortly if we already got there.
          */
-        if (!queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
+        if (!force_kblockd && !queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
                 q->request_fn(q);
                 queue_flag_clear(QUEUE_FLAG_REENTER, q);
         } else {
@@ -446,7 +447,7 @@ void blk_run_queue(struct request_queue *q)
         unsigned long flags;
 
         spin_lock_irqsave(q->queue_lock, flags);
-        __blk_run_queue(q);
+        __blk_run_queue(q, false);
         spin_unlock_irqrestore(q->queue_lock, flags);
 }
 EXPORT_SYMBOL(blk_run_queue);
@@ -1053,7 +1054,7 @@ void blk_insert_request(struct request_queue *q, struct request *rq,
 
         drive_stat_acct(rq, 1);
         __elv_add_request(q, rq, where, 0);
-        __blk_run_queue(q);
+        __blk_run_queue(q, false);
         spin_unlock_irqrestore(q->queue_lock, flags);
 }
 EXPORT_SYMBOL(blk_insert_request);
diff --git a/block/blk-flush.c b/block/blk-flush.c
index 54b123d6563e..56adaa8d55cd 100644
--- a/block/blk-flush.c
+++ b/block/blk-flush.c
@@ -69,7 +69,7 @@ static void blk_flush_complete_seq_end_io(struct request_queue *q,
          * queue.  Kick the queue in those cases.
          */
         if (was_empty && next_rq)
-                __blk_run_queue(q);
+                __blk_run_queue(q, false);
 }
 
 static void pre_flush_end_io(struct request *rq, int error)
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 7be4c7959625..ea83a4f0c27d 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -3355,7 +3355,7 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                             cfqd->busy_queues > 1) {
                                 cfq_del_timer(cfqd, cfqq);
                                 cfq_clear_cfqq_wait_request(cfqq);
-                                __blk_run_queue(cfqd->queue);
+                                __blk_run_queue(cfqd->queue, false);
                         } else {
                                 cfq_blkiocg_update_idle_time_stats(
                                                 &cfqq->cfqg->blkg);
@@ -3370,7 +3370,7 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                  * this new queue is RT and the current one is BE
                  */
                 cfq_preempt_queue(cfqd, cfqq);
-                __blk_run_queue(cfqd->queue);
+                __blk_run_queue(cfqd->queue, false);
         }
 }
 
@@ -3731,7 +3731,7 @@ static void cfq_kick_queue(struct work_struct *work)
         struct request_queue *q = cfqd->queue;
 
         spin_lock_irq(q->queue_lock);
-        __blk_run_queue(cfqd->queue);
+        __blk_run_queue(cfqd->queue, false);
         spin_unlock_irq(q->queue_lock);
 }
 
diff --git a/block/elevator.c b/block/elevator.c
index 2569512830d3..236e93c1f46c 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -602,7 +602,7 @@ void elv_quiesce_start(struct request_queue *q)
          */
         elv_drain_elevator(q);
         while (q->rq.elvpriv) {
-                __blk_run_queue(q);
+                __blk_run_queue(q, false);
                 spin_unlock_irq(q->queue_lock);
                 msleep(10);
                 spin_lock_irq(q->queue_lock);
@@ -651,7 +651,7 @@ void elv_insert(struct request_queue *q, struct request *rq, int where)
                  *   with anything.  There's no point in delaying queue
                  *   processing.
                  */
-                __blk_run_queue(q);
+                __blk_run_queue(q, false);
                 break;
 
         case ELEVATOR_INSERT_SORT:
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 9045c52abd25..fb2bb35c62cb 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -443,7 +443,7 @@ static void scsi_run_queue(struct request_queue *q)
                                         &sdev->request_queue->queue_flags);
                 if (flagset)
                         queue_flag_set(QUEUE_FLAG_REENTER, sdev->request_queue);
-                __blk_run_queue(sdev->request_queue);
+                __blk_run_queue(sdev->request_queue, false);
                 if (flagset)
                         queue_flag_clear(QUEUE_FLAG_REENTER, sdev->request_queue);
                 spin_unlock(sdev->request_queue->queue_lock);
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index 998c01be3234..5c3ccfc6b622 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -3829,7 +3829,7 @@ fc_bsg_goose_queue(struct fc_rport *rport)
                   !test_bit(QUEUE_FLAG_REENTER, &rport->rqst_q->queue_flags);
         if (flagset)
                 queue_flag_set(QUEUE_FLAG_REENTER, rport->rqst_q);
-        __blk_run_queue(rport->rqst_q);
+        __blk_run_queue(rport->rqst_q, false);
         if (flagset)
                 queue_flag_clear(QUEUE_FLAG_REENTER, rport->rqst_q);
         spin_unlock_irqrestore(rport->rqst_q->queue_lock, flags);
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index dd8cd0f47e3a..d5063e1b5555 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -699,7 +699,7 @@ extern void blk_start_queue(struct request_queue *q);
 extern void blk_stop_queue(struct request_queue *q);
 extern void blk_sync_queue(struct request_queue *q);
 extern void __blk_stop_queue(struct request_queue *q);
-extern void __blk_run_queue(struct request_queue *);
+extern void __blk_run_queue(struct request_queue *q, bool force_kblockd);
 extern void blk_run_queue(struct request_queue *);
 extern int blk_rq_map_user(struct request_queue *, struct request *,
                            struct rq_map_data *, void __user *, unsigned long,