summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--block/blk-core.c76
-rw-r--r--block/blk-merge.c2
-rw-r--r--block/blk-mq-sched.c35
-rw-r--r--block/blk-mq.c32
-rw-r--r--block/blk.h2
-rw-r--r--block/cfq-iosched.c4
-rw-r--r--block/deadline-iosched.c12
-rw-r--r--block/elevator.c10
-rw-r--r--block/mq-deadline.c2
-rw-r--r--include/linux/elevator.h28
10 files changed, 102 insertions, 101 deletions
diff --git a/block/blk-core.c b/block/blk-core.c
index 1d263311353a..00e053c704a1 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -1511,12 +1511,11 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1511{ 1511{
1512 struct blk_plug *plug; 1512 struct blk_plug *plug;
1513 struct request *rq; 1513 struct request *rq;
1514 bool ret = false;
1515 struct list_head *plug_list; 1514 struct list_head *plug_list;
1516 1515
1517 plug = current->plug; 1516 plug = current->plug;
1518 if (!plug) 1517 if (!plug)
1519 goto out; 1518 return false;
1520 *request_count = 0; 1519 *request_count = 0;
1521 1520
1522 if (q->mq_ops) 1521 if (q->mq_ops)
@@ -1525,7 +1524,7 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1525 plug_list = &plug->list; 1524 plug_list = &plug->list;
1526 1525
1527 list_for_each_entry_reverse(rq, plug_list, queuelist) { 1526 list_for_each_entry_reverse(rq, plug_list, queuelist) {
1528 int el_ret; 1527 bool merged = false;
1529 1528
1530 if (rq->q == q) { 1529 if (rq->q == q) {
1531 (*request_count)++; 1530 (*request_count)++;
@@ -1541,19 +1540,22 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1541 if (rq->q != q || !blk_rq_merge_ok(rq, bio)) 1540 if (rq->q != q || !blk_rq_merge_ok(rq, bio))
1542 continue; 1541 continue;
1543 1542
1544 el_ret = blk_try_merge(rq, bio); 1543 switch (blk_try_merge(rq, bio)) {
1545 if (el_ret == ELEVATOR_BACK_MERGE) { 1544 case ELEVATOR_BACK_MERGE:
1546 ret = bio_attempt_back_merge(q, rq, bio); 1545 merged = bio_attempt_back_merge(q, rq, bio);
1547 if (ret) 1546 break;
1548 break; 1547 case ELEVATOR_FRONT_MERGE:
1549 } else if (el_ret == ELEVATOR_FRONT_MERGE) { 1548 merged = bio_attempt_front_merge(q, rq, bio);
1550 ret = bio_attempt_front_merge(q, rq, bio); 1549 break;
1551 if (ret) 1550 default:
1552 break; 1551 break;
1553 } 1552 }
1553
1554 if (merged)
1555 return true;
1554 } 1556 }
1555out: 1557
1556 return ret; 1558 return false;
1557} 1559}
1558 1560
1559unsigned int blk_plug_queued_count(struct request_queue *q) 1561unsigned int blk_plug_queued_count(struct request_queue *q)
@@ -1595,7 +1597,7 @@ void init_request_from_bio(struct request *req, struct bio *bio)
1595static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio) 1597static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
1596{ 1598{
1597 struct blk_plug *plug; 1599 struct blk_plug *plug;
1598 int el_ret, where = ELEVATOR_INSERT_SORT; 1600 int where = ELEVATOR_INSERT_SORT;
1599 struct request *req, *free; 1601 struct request *req, *free;
1600 unsigned int request_count = 0; 1602 unsigned int request_count = 0;
1601 unsigned int wb_acct; 1603 unsigned int wb_acct;
@@ -1633,27 +1635,29 @@ static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
1633 1635
1634 spin_lock_irq(q->queue_lock); 1636 spin_lock_irq(q->queue_lock);
1635 1637
1636 el_ret = elv_merge(q, &req, bio); 1638 switch (elv_merge(q, &req, bio)) {
1637 if (el_ret == ELEVATOR_BACK_MERGE) { 1639 case ELEVATOR_BACK_MERGE:
1638 if (bio_attempt_back_merge(q, req, bio)) { 1640 if (!bio_attempt_back_merge(q, req, bio))
1639 elv_bio_merged(q, req, bio); 1641 break;
1640 free = attempt_back_merge(q, req); 1642 elv_bio_merged(q, req, bio);
1641 if (!free) 1643 free = attempt_back_merge(q, req);
1642 elv_merged_request(q, req, el_ret); 1644 if (free)
1643 else 1645 __blk_put_request(q, free);
1644 __blk_put_request(q, free); 1646 else
1645 goto out_unlock; 1647 elv_merged_request(q, req, ELEVATOR_BACK_MERGE);
1646 } 1648 goto out_unlock;
1647 } else if (el_ret == ELEVATOR_FRONT_MERGE) { 1649 case ELEVATOR_FRONT_MERGE:
1648 if (bio_attempt_front_merge(q, req, bio)) { 1650 if (!bio_attempt_front_merge(q, req, bio))
1649 elv_bio_merged(q, req, bio); 1651 break;
1650 free = attempt_front_merge(q, req); 1652 elv_bio_merged(q, req, bio);
1651 if (!free) 1653 free = attempt_front_merge(q, req);
1652 elv_merged_request(q, req, el_ret); 1654 if (free)
1653 else 1655 __blk_put_request(q, free);
1654 __blk_put_request(q, free); 1656 else
1655 goto out_unlock; 1657 elv_merged_request(q, req, ELEVATOR_FRONT_MERGE);
1656 } 1658 goto out_unlock;
1659 default:
1660 break;
1657 } 1661 }
1658 1662
1659get_rq: 1663get_rq:
diff --git a/block/blk-merge.c b/block/blk-merge.c
index c956d9e7aafd..6cbd90ad5f90 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -801,7 +801,7 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
801 return true; 801 return true;
802} 802}
803 803
804int blk_try_merge(struct request *rq, struct bio *bio) 804enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)
805{ 805{
806 if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector) 806 if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)
807 return ELEVATOR_BACK_MERGE; 807 return ELEVATOR_BACK_MERGE;
diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index ee455e7cf9d8..72d0d8361175 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -238,30 +238,29 @@ bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
238 struct request **merged_request) 238 struct request **merged_request)
239{ 239{
240 struct request *rq; 240 struct request *rq;
241 int ret;
242 241
243 ret = elv_merge(q, &rq, bio); 242 switch (elv_merge(q, &rq, bio)) {
244 if (ret == ELEVATOR_BACK_MERGE) { 243 case ELEVATOR_BACK_MERGE:
245 if (!blk_mq_sched_allow_merge(q, rq, bio)) 244 if (!blk_mq_sched_allow_merge(q, rq, bio))
246 return false; 245 return false;
247 if (bio_attempt_back_merge(q, rq, bio)) { 246 if (!bio_attempt_back_merge(q, rq, bio))
248 *merged_request = attempt_back_merge(q, rq); 247 return false;
249 if (!*merged_request) 248 *merged_request = attempt_back_merge(q, rq);
250 elv_merged_request(q, rq, ret); 249 if (!*merged_request)
251 return true; 250 elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
252 } 251 return true;
253 } else if (ret == ELEVATOR_FRONT_MERGE) { 252 case ELEVATOR_FRONT_MERGE:
254 if (!blk_mq_sched_allow_merge(q, rq, bio)) 253 if (!blk_mq_sched_allow_merge(q, rq, bio))
255 return false; 254 return false;
256 if (bio_attempt_front_merge(q, rq, bio)) { 255 if (!bio_attempt_front_merge(q, rq, bio))
257 *merged_request = attempt_front_merge(q, rq); 256 return false;
258 if (!*merged_request) 257 *merged_request = attempt_front_merge(q, rq);
259 elv_merged_request(q, rq, ret); 258 if (!*merged_request)
260 return true; 259 elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
261 } 260 return true;
261 default:
262 return false;
262 } 263 }
263
264 return false;
265} 264}
266EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge); 265EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
267 266
diff --git a/block/blk-mq.c b/block/blk-mq.c
index e9957df05700..dd9722df4afe 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -763,7 +763,7 @@ static bool blk_mq_attempt_merge(struct request_queue *q,
763 int checked = 8; 763 int checked = 8;
764 764
765 list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) { 765 list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
766 int el_ret; 766 bool merged = false;
767 767
768 if (!checked--) 768 if (!checked--)
769 break; 769 break;
@@ -771,26 +771,22 @@ static bool blk_mq_attempt_merge(struct request_queue *q,
771 if (!blk_rq_merge_ok(rq, bio)) 771 if (!blk_rq_merge_ok(rq, bio))
772 continue; 772 continue;
773 773
774 el_ret = blk_try_merge(rq, bio); 774 switch (blk_try_merge(rq, bio)) {
775 if (el_ret == ELEVATOR_NO_MERGE) 775 case ELEVATOR_BACK_MERGE:
776 continue; 776 if (blk_mq_sched_allow_merge(q, rq, bio))
777 777 merged = bio_attempt_back_merge(q, rq, bio);
778 if (!blk_mq_sched_allow_merge(q, rq, bio))
779 break; 778 break;
780 779 case ELEVATOR_FRONT_MERGE:
781 if (el_ret == ELEVATOR_BACK_MERGE) { 780 if (blk_mq_sched_allow_merge(q, rq, bio))
782 if (bio_attempt_back_merge(q, rq, bio)) { 781 merged = bio_attempt_front_merge(q, rq, bio);
783 ctx->rq_merged++;
784 return true;
785 }
786 break;
787 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
788 if (bio_attempt_front_merge(q, rq, bio)) {
789 ctx->rq_merged++;
790 return true;
791 }
792 break; 782 break;
783 default:
784 continue;
793 } 785 }
786
787 if (merged)
788 ctx->rq_merged++;
789 return merged;
794 } 790 }
795 791
796 return false; 792 return false;
diff --git a/block/blk.h b/block/blk.h
index 3e08703902a9..ae82f2ac4019 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -215,7 +215,7 @@ int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
215void blk_recalc_rq_segments(struct request *rq); 215void blk_recalc_rq_segments(struct request *rq);
216void blk_rq_set_mixed_merge(struct request *rq); 216void blk_rq_set_mixed_merge(struct request *rq);
217bool blk_rq_merge_ok(struct request *rq, struct bio *bio); 217bool blk_rq_merge_ok(struct request *rq, struct bio *bio);
218int blk_try_merge(struct request *rq, struct bio *bio); 218enum elv_merge blk_try_merge(struct request *rq, struct bio *bio);
219 219
220void blk_queue_congestion_threshold(struct request_queue *q); 220void blk_queue_congestion_threshold(struct request_queue *q);
221 221
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index f0f29ee731e1..921262770636 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -2528,7 +2528,7 @@ static void cfq_remove_request(struct request *rq)
2528 } 2528 }
2529} 2529}
2530 2530
2531static int cfq_merge(struct request_queue *q, struct request **req, 2531static enum elv_merge cfq_merge(struct request_queue *q, struct request **req,
2532 struct bio *bio) 2532 struct bio *bio)
2533{ 2533{
2534 struct cfq_data *cfqd = q->elevator->elevator_data; 2534 struct cfq_data *cfqd = q->elevator->elevator_data;
@@ -2544,7 +2544,7 @@ static int cfq_merge(struct request_queue *q, struct request **req,
2544} 2544}
2545 2545
2546static void cfq_merged_request(struct request_queue *q, struct request *req, 2546static void cfq_merged_request(struct request_queue *q, struct request *req,
2547 int type) 2547 enum elv_merge type)
2548{ 2548{
2549 if (type == ELEVATOR_FRONT_MERGE) { 2549 if (type == ELEVATOR_FRONT_MERGE) {
2550 struct cfq_queue *cfqq = RQ_CFQQ(req); 2550 struct cfq_queue *cfqq = RQ_CFQQ(req);
diff --git a/block/deadline-iosched.c b/block/deadline-iosched.c
index 05fc0ea25a98..c68f6bbc0dcd 100644
--- a/block/deadline-iosched.c
+++ b/block/deadline-iosched.c
@@ -120,12 +120,11 @@ static void deadline_remove_request(struct request_queue *q, struct request *rq)
120 deadline_del_rq_rb(dd, rq); 120 deadline_del_rq_rb(dd, rq);
121} 121}
122 122
123static int 123static enum elv_merge
124deadline_merge(struct request_queue *q, struct request **req, struct bio *bio) 124deadline_merge(struct request_queue *q, struct request **req, struct bio *bio)
125{ 125{
126 struct deadline_data *dd = q->elevator->elevator_data; 126 struct deadline_data *dd = q->elevator->elevator_data;
127 struct request *__rq; 127 struct request *__rq;
128 int ret;
129 128
130 /* 129 /*
131 * check for front merge 130 * check for front merge
@@ -138,20 +137,17 @@ deadline_merge(struct request_queue *q, struct request **req, struct bio *bio)
138 BUG_ON(sector != blk_rq_pos(__rq)); 137 BUG_ON(sector != blk_rq_pos(__rq));
139 138
140 if (elv_bio_merge_ok(__rq, bio)) { 139 if (elv_bio_merge_ok(__rq, bio)) {
141 ret = ELEVATOR_FRONT_MERGE; 140 *req = __rq;
142 goto out; 141 return ELEVATOR_FRONT_MERGE;
143 } 142 }
144 } 143 }
145 } 144 }
146 145
147 return ELEVATOR_NO_MERGE; 146 return ELEVATOR_NO_MERGE;
148out:
149 *req = __rq;
150 return ret;
151} 147}
152 148
153static void deadline_merged_request(struct request_queue *q, 149static void deadline_merged_request(struct request_queue *q,
154 struct request *req, int type) 150 struct request *req, enum elv_merge type)
155{ 151{
156 struct deadline_data *dd = q->elevator->elevator_data; 152 struct deadline_data *dd = q->elevator->elevator_data;
157 153
diff --git a/block/elevator.c b/block/elevator.c
index 7e4f5880dd64..27ff1ed5a6fa 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -427,11 +427,11 @@ void elv_dispatch_add_tail(struct request_queue *q, struct request *rq)
427} 427}
428EXPORT_SYMBOL(elv_dispatch_add_tail); 428EXPORT_SYMBOL(elv_dispatch_add_tail);
429 429
430int elv_merge(struct request_queue *q, struct request **req, struct bio *bio) 430enum elv_merge elv_merge(struct request_queue *q, struct request **req,
431 struct bio *bio)
431{ 432{
432 struct elevator_queue *e = q->elevator; 433 struct elevator_queue *e = q->elevator;
433 struct request *__rq; 434 struct request *__rq;
434 int ret;
435 435
436 /* 436 /*
437 * Levels of merges: 437 * Levels of merges:
@@ -446,7 +446,8 @@ int elv_merge(struct request_queue *q, struct request **req, struct bio *bio)
446 * First try one-hit cache. 446 * First try one-hit cache.
447 */ 447 */
448 if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) { 448 if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) {
449 ret = blk_try_merge(q->last_merge, bio); 449 enum elv_merge ret = blk_try_merge(q->last_merge, bio);
450
450 if (ret != ELEVATOR_NO_MERGE) { 451 if (ret != ELEVATOR_NO_MERGE) {
451 *req = q->last_merge; 452 *req = q->last_merge;
452 return ret; 453 return ret;
@@ -514,7 +515,8 @@ bool elv_attempt_insert_merge(struct request_queue *q, struct request *rq)
514 return ret; 515 return ret;
515} 516}
516 517
517void elv_merged_request(struct request_queue *q, struct request *rq, int type) 518void elv_merged_request(struct request_queue *q, struct request *rq,
519 enum elv_merge type)
518{ 520{
519 struct elevator_queue *e = q->elevator; 521 struct elevator_queue *e = q->elevator;
520 522
diff --git a/block/mq-deadline.c b/block/mq-deadline.c
index d68d9c273a66..236121633ca0 100644
--- a/block/mq-deadline.c
+++ b/block/mq-deadline.c
@@ -121,7 +121,7 @@ static void deadline_remove_request(struct request_queue *q, struct request *rq)
121} 121}
122 122
123static void dd_request_merged(struct request_queue *q, struct request *req, 123static void dd_request_merged(struct request_queue *q, struct request *req,
124 int type) 124 enum elv_merge type)
125{ 125{
126 struct deadline_data *dd = q->elevator->elevator_data; 126 struct deadline_data *dd = q->elevator->elevator_data;
127 127
diff --git a/include/linux/elevator.h b/include/linux/elevator.h
index b5825c4f06f7..b38b4e651ea6 100644
--- a/include/linux/elevator.h
+++ b/include/linux/elevator.h
@@ -9,12 +9,21 @@
9struct io_cq; 9struct io_cq;
10struct elevator_type; 10struct elevator_type;
11 11
12typedef int (elevator_merge_fn) (struct request_queue *, struct request **, 12/*
13 * Return values from elevator merger
14 */
15enum elv_merge {
16 ELEVATOR_NO_MERGE = 0,
17 ELEVATOR_FRONT_MERGE = 1,
18 ELEVATOR_BACK_MERGE = 2,
19};
20
21typedef enum elv_merge (elevator_merge_fn) (struct request_queue *, struct request **,
13 struct bio *); 22 struct bio *);
14 23
15typedef void (elevator_merge_req_fn) (struct request_queue *, struct request *, struct request *); 24typedef void (elevator_merge_req_fn) (struct request_queue *, struct request *, struct request *);
16 25
17typedef void (elevator_merged_fn) (struct request_queue *, struct request *, int); 26typedef void (elevator_merged_fn) (struct request_queue *, struct request *, enum elv_merge);
18 27
19typedef int (elevator_allow_bio_merge_fn) (struct request_queue *, 28typedef int (elevator_allow_bio_merge_fn) (struct request_queue *,
20 struct request *, struct bio *); 29 struct request *, struct bio *);
@@ -87,7 +96,7 @@ struct elevator_mq_ops {
87 bool (*allow_merge)(struct request_queue *, struct request *, struct bio *); 96 bool (*allow_merge)(struct request_queue *, struct request *, struct bio *);
88 bool (*bio_merge)(struct blk_mq_hw_ctx *, struct bio *); 97 bool (*bio_merge)(struct blk_mq_hw_ctx *, struct bio *);
89 int (*request_merge)(struct request_queue *q, struct request **, struct bio *); 98 int (*request_merge)(struct request_queue *q, struct request **, struct bio *);
90 void (*request_merged)(struct request_queue *, struct request *, int); 99 void (*request_merged)(struct request_queue *, struct request *, enum elv_merge);
91 void (*requests_merged)(struct request_queue *, struct request *, struct request *); 100 void (*requests_merged)(struct request_queue *, struct request *, struct request *);
92 struct request *(*get_request)(struct request_queue *, unsigned int, struct blk_mq_alloc_data *); 101 struct request *(*get_request)(struct request_queue *, unsigned int, struct blk_mq_alloc_data *);
93 void (*put_request)(struct request *); 102 void (*put_request)(struct request *);
@@ -166,10 +175,12 @@ extern void elv_dispatch_sort(struct request_queue *, struct request *);
166extern void elv_dispatch_add_tail(struct request_queue *, struct request *); 175extern void elv_dispatch_add_tail(struct request_queue *, struct request *);
167extern void elv_add_request(struct request_queue *, struct request *, int); 176extern void elv_add_request(struct request_queue *, struct request *, int);
168extern void __elv_add_request(struct request_queue *, struct request *, int); 177extern void __elv_add_request(struct request_queue *, struct request *, int);
169extern int elv_merge(struct request_queue *, struct request **, struct bio *); 178extern enum elv_merge elv_merge(struct request_queue *, struct request **,
179 struct bio *);
170extern void elv_merge_requests(struct request_queue *, struct request *, 180extern void elv_merge_requests(struct request_queue *, struct request *,
171 struct request *); 181 struct request *);
172extern void elv_merged_request(struct request_queue *, struct request *, int); 182extern void elv_merged_request(struct request_queue *, struct request *,
183 enum elv_merge);
173extern void elv_bio_merged(struct request_queue *q, struct request *, 184extern void elv_bio_merged(struct request_queue *q, struct request *,
174 struct bio *); 185 struct bio *);
175extern bool elv_attempt_insert_merge(struct request_queue *, struct request *); 186extern bool elv_attempt_insert_merge(struct request_queue *, struct request *);
@@ -219,13 +230,6 @@ extern void elv_rb_del(struct rb_root *, struct request *);
219extern struct request *elv_rb_find(struct rb_root *, sector_t); 230extern struct request *elv_rb_find(struct rb_root *, sector_t);
220 231
221/* 232/*
222 * Return values from elevator merger
223 */
224#define ELEVATOR_NO_MERGE 0
225#define ELEVATOR_FRONT_MERGE 1
226#define ELEVATOR_BACK_MERGE 2
227
228/*
229 * Insertion selection 233 * Insertion selection
230 */ 234 */
231#define ELEVATOR_INSERT_FRONT 1 235#define ELEVATOR_INSERT_FRONT 1