block: add queue flag for disabling SG merging

If devices are not SG starved, we waste a lot of time potentially
collapsing SG segments. Enough that 1.5% of the CPU time goes
to this, at only 400K IOPS. Add a queue flag, QUEUE_FLAG_NO_SG_MERGE,
which just returns the number of vectors in a bio instead of looping
over all segments and checking for collapsible ones.

Add a BLK_MQ_F_SG_MERGE flag so that drivers can opt-in on the sg
merging, if they so desire.

Signed-off-by: Jens Axboe <axboe@fb.com>

2 files changed, 24 insertions, 7 deletions

diff --git a/block/blk-merge.c b/block/blk-merge.c
index 6c583f9c5b65..b3bf0df0f4c2 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -13,7 +13,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
                                              struct bio *bio)
 {
         struct bio_vec bv, bvprv = { NULL };
-        int cluster, high, highprv = 1;
+        int cluster, high, highprv = 1, no_sg_merge;
         unsigned int seg_size, nr_phys_segs;
         struct bio *fbio, *bbio;
         struct bvec_iter iter;
@@ -35,12 +35,21 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
         cluster = blk_queue_cluster(q);
         seg_size = 0;
         nr_phys_segs = 0;
+        no_sg_merge = test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags);
+        high = 0;
         for_each_bio(bio) {
                 bio_for_each_segment(bv, bio, iter) {
                         /*
+                         * If SG merging is disabled, each bio vector is
+                         * a segment
+                         */
+                        if (no_sg_merge)
+                                goto new_segment;
+
+                        /*
                          * the trick here is making sure that a high page is
-                         * never considered part of another segment, since that
-                         * might change with the bounce page.
+                         * never considered part of another segment, since
+                         * that might change with the bounce page.
                          */
                         high = page_to_pfn(bv.bv_page) > queue_bounce_pfn(q);
                         if (!high && !highprv && cluster) {
@@ -84,11 +93,16 @@ void blk_recalc_rq_segments(struct request *rq)
 
 void blk_recount_segments(struct request_queue *q, struct bio *bio)
 {
-        struct bio *nxt = bio->bi_next;
+        if (test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags))
+                bio->bi_phys_segments = bio->bi_vcnt;
+        else {
+                struct bio *nxt = bio->bi_next;
+
+                bio->bi_next = NULL;
+                bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio);
+                bio->bi_next = nxt;
+        }
 
-        bio->bi_next = NULL;
-        bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio);
-        bio->bi_next = nxt;
         bio->bi_flags |= (1 << BIO_SEG_VALID);
 }
 EXPORT_SYMBOL(blk_recount_segments);
diff --git a/block/blk-mq.c b/block/blk-mq.c
index f27fe44230c2..f98d977fd150 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -1829,6 +1829,9 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
         q->mq_ops = set->ops;
         q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
 
+        if (!(set->flags & BLK_MQ_F_SG_MERGE))
+                q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;
+
         q->sg_reserved_size = INT_MAX;
 
         INIT_WORK(&q->requeue_work, blk_mq_requeue_work);