aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJens Axboe <axboe@kernel.dk>2017-11-10 11:13:21 -0500
committerJens Axboe <axboe@kernel.dk>2017-11-10 21:55:57 -0500
commit79f720a751cad613620d0237e3b44f89f4a69181 (patch)
tree4f530507baac038f38fa6974734f3a0595b2abb1
parentf0fba398fec65ff5a2e1bf8ae62718ec0450abaf (diff)
blk-mq: only run the hardware queue if IO is pending
Currently we are inconsistent in when we decide to run the queue. Using blk_mq_run_hw_queues() we check if the hctx has pending IO before running it, but we don't do that from the individual queue run function, blk_mq_run_hw_queue(). This results in a lot of extra and pointless queue runs, potentially, on flush requests and (much worse) on tag starvation situations. This is observable just looking at top output, with lots of kworkers active. For the !async runs, it just adds to the CPU overhead of blk-mq. Move the has-pending check into the run function instead of having callers do it. Reviewed-by: Christoph Hellwig <hch@lst.de> Signed-off-by: Jens Axboe <axboe@kernel.dk>
-rw-r--r--block/blk-mq-sched.c7
-rw-r--r--block/blk-mq.c18
-rw-r--r--block/blk-mq.h2
-rw-r--r--include/linux/blk-mq.h2
4 files changed, 13 insertions, 16 deletions
diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index 6f4bdb8209f7..c117bd8fd1f6 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -81,12 +81,7 @@ static bool blk_mq_sched_restart_hctx(struct blk_mq_hw_ctx *hctx)
81 } else 81 } else
82 clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state); 82 clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
83 83
84 if (blk_mq_hctx_has_pending(hctx)) { 84 return blk_mq_run_hw_queue(hctx, true);
85 blk_mq_run_hw_queue(hctx, true);
86 return true;
87 }
88
89 return false;
90} 85}
91 86
92/* 87/*
diff --git a/block/blk-mq.c b/block/blk-mq.c
index bfe24a5b62a3..a2a4271f5ab8 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -61,10 +61,10 @@ static int blk_mq_poll_stats_bkt(const struct request *rq)
61/* 61/*
62 * Check if any of the ctx's have pending work in this hardware queue 62 * Check if any of the ctx's have pending work in this hardware queue
63 */ 63 */
64bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) 64static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
65{ 65{
66 return sbitmap_any_bit_set(&hctx->ctx_map) || 66 return !list_empty_careful(&hctx->dispatch) ||
67 !list_empty_careful(&hctx->dispatch) || 67 sbitmap_any_bit_set(&hctx->ctx_map) ||
68 blk_mq_sched_has_work(hctx); 68 blk_mq_sched_has_work(hctx);
69} 69}
70 70
@@ -1253,9 +1253,14 @@ void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
1253} 1253}
1254EXPORT_SYMBOL(blk_mq_delay_run_hw_queue); 1254EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
1255 1255
1256void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 1256bool blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1257{ 1257{
1258 __blk_mq_delay_run_hw_queue(hctx, async, 0); 1258 if (blk_mq_hctx_has_pending(hctx)) {
1259 __blk_mq_delay_run_hw_queue(hctx, async, 0);
1260 return true;
1261 }
1262
1263 return false;
1259} 1264}
1260EXPORT_SYMBOL(blk_mq_run_hw_queue); 1265EXPORT_SYMBOL(blk_mq_run_hw_queue);
1261 1266
@@ -1265,8 +1270,7 @@ void blk_mq_run_hw_queues(struct request_queue *q, bool async)
1265 int i; 1270 int i;
1266 1271
1267 queue_for_each_hw_ctx(q, hctx, i) { 1272 queue_for_each_hw_ctx(q, hctx, i) {
1268 if (!blk_mq_hctx_has_pending(hctx) || 1273 if (blk_mq_hctx_stopped(hctx))
1269 blk_mq_hctx_stopped(hctx))
1270 continue; 1274 continue;
1271 1275
1272 blk_mq_run_hw_queue(hctx, async); 1276 blk_mq_run_hw_queue(hctx, async);
diff --git a/block/blk-mq.h b/block/blk-mq.h
index 99a19c5523e2..dcf379a892dd 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -26,14 +26,12 @@ struct blk_mq_ctx {
26 struct kobject kobj; 26 struct kobject kobj;
27} ____cacheline_aligned_in_smp; 27} ____cacheline_aligned_in_smp;
28 28
29void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
30void blk_mq_freeze_queue(struct request_queue *q); 29void blk_mq_freeze_queue(struct request_queue *q);
31void blk_mq_free_queue(struct request_queue *q); 30void blk_mq_free_queue(struct request_queue *q);
32int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr); 31int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
33void blk_mq_wake_waiters(struct request_queue *q); 32void blk_mq_wake_waiters(struct request_queue *q);
34bool blk_mq_dispatch_rq_list(struct request_queue *, struct list_head *, bool); 33bool blk_mq_dispatch_rq_list(struct request_queue *, struct list_head *, bool);
35void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list); 34void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list);
36bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx);
37bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx, 35bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx,
38 bool wait); 36 bool wait);
39struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, 37struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index b326208277ee..eb1e2cdffb31 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -266,7 +266,7 @@ void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async);
266void blk_mq_quiesce_queue(struct request_queue *q); 266void blk_mq_quiesce_queue(struct request_queue *q);
267void blk_mq_unquiesce_queue(struct request_queue *q); 267void blk_mq_unquiesce_queue(struct request_queue *q);
268void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); 268void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
269void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async); 269bool blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
270void blk_mq_run_hw_queues(struct request_queue *q, bool async); 270void blk_mq_run_hw_queues(struct request_queue *q, bool async);
271void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); 271void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
272void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, 272void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,