diff options
Diffstat (limited to 'block/blk-mq.c')
-rw-r--r-- | block/blk-mq.c | 50 |
1 files changed, 24 insertions, 26 deletions
diff --git a/block/blk-mq.c b/block/blk-mq.c index 53b6def12fc4..4e7a31466139 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c | |||
@@ -281,26 +281,6 @@ void blk_mq_free_request(struct request *rq) | |||
281 | __blk_mq_free_request(hctx, ctx, rq); | 281 | __blk_mq_free_request(hctx, ctx, rq); |
282 | } | 282 | } |
283 | 283 | ||
284 | /* | ||
285 | * Clone all relevant state from a request that has been put on hold in | ||
286 | * the flush state machine into the preallocated flush request that hangs | ||
287 | * off the request queue. | ||
288 | * | ||
289 | * For a driver the flush request should be invisible, that's why we are | ||
290 | * impersonating the original request here. | ||
291 | */ | ||
292 | void blk_mq_clone_flush_request(struct request *flush_rq, | ||
293 | struct request *orig_rq) | ||
294 | { | ||
295 | struct blk_mq_hw_ctx *hctx = | ||
296 | orig_rq->q->mq_ops->map_queue(orig_rq->q, orig_rq->mq_ctx->cpu); | ||
297 | |||
298 | flush_rq->mq_ctx = orig_rq->mq_ctx; | ||
299 | flush_rq->tag = orig_rq->tag; | ||
300 | memcpy(blk_mq_rq_to_pdu(flush_rq), blk_mq_rq_to_pdu(orig_rq), | ||
301 | hctx->cmd_size); | ||
302 | } | ||
303 | |||
304 | inline void __blk_mq_end_request(struct request *rq, int error) | 284 | inline void __blk_mq_end_request(struct request *rq, int error) |
305 | { | 285 | { |
306 | blk_account_io_done(rq); | 286 | blk_account_io_done(rq); |
@@ -1516,12 +1496,20 @@ static void blk_mq_exit_hctx(struct request_queue *q, | |||
1516 | struct blk_mq_tag_set *set, | 1496 | struct blk_mq_tag_set *set, |
1517 | struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) | 1497 | struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) |
1518 | { | 1498 | { |
1499 | unsigned flush_start_tag = set->queue_depth; | ||
1500 | |||
1519 | blk_mq_tag_idle(hctx); | 1501 | blk_mq_tag_idle(hctx); |
1520 | 1502 | ||
1503 | if (set->ops->exit_request) | ||
1504 | set->ops->exit_request(set->driver_data, | ||
1505 | hctx->fq->flush_rq, hctx_idx, | ||
1506 | flush_start_tag + hctx_idx); | ||
1507 | |||
1521 | if (set->ops->exit_hctx) | 1508 | if (set->ops->exit_hctx) |
1522 | set->ops->exit_hctx(hctx, hctx_idx); | 1509 | set->ops->exit_hctx(hctx, hctx_idx); |
1523 | 1510 | ||
1524 | blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier); | 1511 | blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier); |
1512 | blk_free_flush_queue(hctx->fq); | ||
1525 | kfree(hctx->ctxs); | 1513 | kfree(hctx->ctxs); |
1526 | blk_mq_free_bitmap(&hctx->ctx_map); | 1514 | blk_mq_free_bitmap(&hctx->ctx_map); |
1527 | } | 1515 | } |
@@ -1556,6 +1544,7 @@ static int blk_mq_init_hctx(struct request_queue *q, | |||
1556 | struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) | 1544 | struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) |
1557 | { | 1545 | { |
1558 | int node; | 1546 | int node; |
1547 | unsigned flush_start_tag = set->queue_depth; | ||
1559 | 1548 | ||
1560 | node = hctx->numa_node; | 1549 | node = hctx->numa_node; |
1561 | if (node == NUMA_NO_NODE) | 1550 | if (node == NUMA_NO_NODE) |
@@ -1594,8 +1583,23 @@ static int blk_mq_init_hctx(struct request_queue *q, | |||
1594 | set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) | 1583 | set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) |
1595 | goto free_bitmap; | 1584 | goto free_bitmap; |
1596 | 1585 | ||
1586 | hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size); | ||
1587 | if (!hctx->fq) | ||
1588 | goto exit_hctx; | ||
1589 | |||
1590 | if (set->ops->init_request && | ||
1591 | set->ops->init_request(set->driver_data, | ||
1592 | hctx->fq->flush_rq, hctx_idx, | ||
1593 | flush_start_tag + hctx_idx, node)) | ||
1594 | goto free_fq; | ||
1595 | |||
1597 | return 0; | 1596 | return 0; |
1598 | 1597 | ||
1598 | free_fq: | ||
1599 | kfree(hctx->fq); | ||
1600 | exit_hctx: | ||
1601 | if (set->ops->exit_hctx) | ||
1602 | set->ops->exit_hctx(hctx, hctx_idx); | ||
1599 | free_bitmap: | 1603 | free_bitmap: |
1600 | blk_mq_free_bitmap(&hctx->ctx_map); | 1604 | blk_mq_free_bitmap(&hctx->ctx_map); |
1601 | free_ctxs: | 1605 | free_ctxs: |
@@ -1862,16 +1866,10 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) | |||
1862 | 1866 | ||
1863 | blk_mq_add_queue_tag_set(set, q); | 1867 | blk_mq_add_queue_tag_set(set, q); |
1864 | 1868 | ||
1865 | q->fq = blk_alloc_flush_queue(q); | ||
1866 | if (!q->fq) | ||
1867 | goto err_hw_queues; | ||
1868 | |||
1869 | blk_mq_map_swqueue(q); | 1869 | blk_mq_map_swqueue(q); |
1870 | 1870 | ||
1871 | return q; | 1871 | return q; |
1872 | 1872 | ||
1873 | err_hw_queues: | ||
1874 | blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); | ||
1875 | err_hw: | 1873 | err_hw: |
1876 | blk_cleanup_queue(q); | 1874 | blk_cleanup_queue(q); |
1877 | err_hctxs: | 1875 | err_hctxs: |