4 files changed, 37 insertions, 24 deletions

diff --git a/block/blk-mq.c b/block/blk-mq.c
index cee96234bf58..14c8f35946e1 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -326,7 +326,7 @@ static void blk_mq_bio_endio(struct request *rq, struct bio *bio, int error)
                 bio_endio(bio, error);
 }
 
-void blk_mq_complete_request(struct request *rq, int error)
+void blk_mq_end_io(struct request *rq, int error)
 {
         struct bio *bio = rq->bio;
         unsigned int bytes = 0;
@@ -351,46 +351,53 @@ void blk_mq_complete_request(struct request *rq, int error)
         else
                 blk_mq_free_request(rq);
 }
+EXPORT_SYMBOL(blk_mq_end_io);
 
-void __blk_mq_end_io(struct request *rq, int error)
-{
-        if (!blk_mark_rq_complete(rq))
-                blk_mq_complete_request(rq, error);
-}
-
-static void blk_mq_end_io_remote(void *data)
+static void __blk_mq_complete_request_remote(void *data)
 {
         struct request *rq = data;
 
-        __blk_mq_end_io(rq, rq->errors);
+        rq->q->softirq_done_fn(rq);
 }
 
-/*
- * End IO on this request on a multiqueue enabled driver. We'll either do
- * it directly inline, or punt to a local IPI handler on the matching
- * remote CPU.
- */
-void blk_mq_end_io(struct request *rq, int error)
+void __blk_mq_complete_request(struct request *rq)
 {
         struct blk_mq_ctx *ctx = rq->mq_ctx;
         int cpu;
 
-        if (!ctx->ipi_redirect)
-                return __blk_mq_end_io(rq, error);
+        if (!ctx->ipi_redirect) {
+                rq->q->softirq_done_fn(rq);
+                return;
+        }
 
         cpu = get_cpu();
         if (cpu != ctx->cpu && cpu_online(ctx->cpu)) {
-                rq->errors = error;
-                rq->csd.func = blk_mq_end_io_remote;
+                rq->csd.func = __blk_mq_complete_request_remote;
                 rq->csd.info = rq;
                 rq->csd.flags = 0;
                 __smp_call_function_single(ctx->cpu, &rq->csd, 0);
         } else {
-                __blk_mq_end_io(rq, error);
+                rq->q->softirq_done_fn(rq);
         }
         put_cpu();
 }
-EXPORT_SYMBOL(blk_mq_end_io);
+
+/**
+ * blk_mq_complete_request - end I/O on a request
+ * @rq:         the request being processed
+ *
+ * Description:
+ *      Ends all I/O on a request. It does not handle partial completions.
+ *      The actual completion happens out-of-order, through a IPI handler.
+ **/
+void blk_mq_complete_request(struct request *rq)
+{
+        if (unlikely(blk_should_fake_timeout(rq->q)))
+                return;
+        if (!blk_mark_rq_complete(rq))
+                __blk_mq_complete_request(rq);
+}
+EXPORT_SYMBOL(blk_mq_complete_request);
 
 static void blk_mq_start_request(struct request *rq)
 {
@@ -1399,6 +1406,9 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_reg *reg,
         if (reg->timeout)
                 blk_queue_rq_timeout(q, reg->timeout);
 
+        if (reg->ops->complete)
+                blk_queue_softirq_done(q, reg->ops->complete);
+
         blk_mq_init_flush(q);
         blk_mq_init_cpu_queues(q, reg->nr_hw_queues);
 
diff --git a/block/blk-mq.h b/block/blk-mq.h
index 5c3917984b00..f29b645f0e1c 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -22,8 +22,7 @@ struct blk_mq_ctx {
         struct kobject          kobj;
 };
 
-void __blk_mq_end_io(struct request *rq, int error);
-void blk_mq_complete_request(struct request *rq, int error);
+void __blk_mq_complete_request(struct request *rq);
 void blk_mq_run_request(struct request *rq, bool run_queue, bool async);
 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
 void blk_mq_init_flush(struct request_queue *q);
diff --git a/block/blk-timeout.c b/block/blk-timeout.c
index bba81c9348e1..d96f7061c6fd 100644
--- a/block/blk-timeout.c
+++ b/block/blk-timeout.c
@@ -91,7 +91,7 @@ static void blk_rq_timed_out(struct request *req)
         case BLK_EH_HANDLED:
                 /* Can we use req->errors here? */
                 if (q->mq_ops)
-                        blk_mq_complete_request(req, req->errors);
+                        __blk_mq_complete_request(req);
                 else
                         __blk_complete_request(req);
                 break;
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index b7638be58599..468be242db90 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -86,6 +86,8 @@ struct blk_mq_ops {
          */
         rq_timed_out_fn         *timeout;
 
+        softirq_done_fn         *complete;
+
         /*
          * Override for hctx allocations (should probably go)
          */
@@ -137,6 +139,8 @@ void blk_mq_free_single_hw_queue(struct blk_mq_hw_ctx *, unsigned int);
 
 void blk_mq_end_io(struct request *rq, int error);
 
+void blk_mq_complete_request(struct request *rq);
+
 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
 void blk_mq_stop_hw_queues(struct request_queue *q);