1 files changed, 149 insertions, 10 deletions

diff --git a/block/blk-merge.c b/block/blk-merge.c
index a455b9860143..d9c3a75e4a60 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -9,12 +9,158 @@
 
 #include "blk.h"
 
+static struct bio *blk_bio_discard_split(struct request_queue *q,
+                                         struct bio *bio,
+                                         struct bio_set *bs)
+{
+        unsigned int max_discard_sectors, granularity;
+        int alignment;
+        sector_t tmp;
+        unsigned split_sectors;
+
+        /* Zero-sector (unknown) and one-sector granularities are the same.  */
+        granularity = max(q->limits.discard_granularity >> 9, 1U);
+
+        max_discard_sectors = min(q->limits.max_discard_sectors, UINT_MAX >> 9);
+        max_discard_sectors -= max_discard_sectors % granularity;
+
+        if (unlikely(!max_discard_sectors)) {
+                /* XXX: warn */
+                return NULL;
+        }
+
+        if (bio_sectors(bio) <= max_discard_sectors)
+                return NULL;
+
+        split_sectors = max_discard_sectors;
+
+        /*
+         * If the next starting sector would be misaligned, stop the discard at
+         * the previous aligned sector.
+         */
+        alignment = (q->limits.discard_alignment >> 9) % granularity;
+
+        tmp = bio->bi_iter.bi_sector + split_sectors - alignment;
+        tmp = sector_div(tmp, granularity);
+
+        if (split_sectors > tmp)
+                split_sectors -= tmp;
+
+        return bio_split(bio, split_sectors, GFP_NOIO, bs);
+}
+
+static struct bio *blk_bio_write_same_split(struct request_queue *q,
+                                            struct bio *bio,
+                                            struct bio_set *bs)
+{
+        if (!q->limits.max_write_same_sectors)
+                return NULL;
+
+        if (bio_sectors(bio) <= q->limits.max_write_same_sectors)
+                return NULL;
+
+        return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs);
+}
+
+static struct bio *blk_bio_segment_split(struct request_queue *q,
+                                         struct bio *bio,
+                                         struct bio_set *bs)
+{
+        struct bio *split;
+        struct bio_vec bv, bvprv;
+        struct bvec_iter iter;
+        unsigned seg_size = 0, nsegs = 0;
+        int prev = 0;
+
+        struct bvec_merge_data bvm = {
+                .bi_bdev        = bio->bi_bdev,
+                .bi_sector      = bio->bi_iter.bi_sector,
+                .bi_size        = 0,
+                .bi_rw          = bio->bi_rw,
+        };
+
+        bio_for_each_segment(bv, bio, iter) {
+                if (q->merge_bvec_fn &&
+                    q->merge_bvec_fn(q, &bvm, &bv) < (int) bv.bv_len)
+                        goto split;
+
+                bvm.bi_size += bv.bv_len;
+
+                if (bvm.bi_size >> 9 > queue_max_sectors(q))
+                        goto split;
+
+                /*
+                 * If the queue doesn't support SG gaps and adding this
+                 * offset would create a gap, disallow it.
+                 */
+                if (q->queue_flags & (1 << QUEUE_FLAG_SG_GAPS) &&
+                    prev && bvec_gap_to_prev(&bvprv, bv.bv_offset))
+                        goto split;
+
+                if (prev && blk_queue_cluster(q)) {
+                        if (seg_size + bv.bv_len > queue_max_segment_size(q))
+                                goto new_segment;
+                        if (!BIOVEC_PHYS_MERGEABLE(&bvprv, &bv))
+                                goto new_segment;
+                        if (!BIOVEC_SEG_BOUNDARY(q, &bvprv, &bv))
+                                goto new_segment;
+
+                        seg_size += bv.bv_len;
+                        bvprv = bv;
+                        prev = 1;
+                        continue;
+                }
+new_segment:
+                if (nsegs == queue_max_segments(q))
+                        goto split;
+
+                nsegs++;
+                bvprv = bv;
+                prev = 1;
+                seg_size = bv.bv_len;
+        }
+
+        return NULL;
+split:
+        split = bio_clone_bioset(bio, GFP_NOIO, bs);
+
+        split->bi_iter.bi_size -= iter.bi_size;
+        bio->bi_iter = iter;
+
+        if (bio_integrity(bio)) {
+                bio_integrity_advance(bio, split->bi_iter.bi_size);
+                bio_integrity_trim(split, 0, bio_sectors(split));
+        }
+
+        return split;
+}
+
+void blk_queue_split(struct request_queue *q, struct bio **bio,
+                     struct bio_set *bs)
+{
+        struct bio *split;
+
+        if ((*bio)->bi_rw & REQ_DISCARD)
+                split = blk_bio_discard_split(q, *bio, bs);
+        else if ((*bio)->bi_rw & REQ_WRITE_SAME)
+                split = blk_bio_write_same_split(q, *bio, bs);
+        else
+                split = blk_bio_segment_split(q, *bio, q->bio_split);
+
+        if (split) {
+                bio_chain(split, *bio);
+                generic_make_request(*bio);
+                *bio = split;
+        }
+}
+EXPORT_SYMBOL(blk_queue_split);
+
 static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
                                              struct bio *bio,
                                              bool no_sg_merge)
 {
         struct bio_vec bv, bvprv = { NULL };
-        int cluster, high, highprv = 1;
+        int cluster, prev = 0;
         unsigned int seg_size, nr_phys_segs;
         struct bio *fbio, *bbio;
         struct bvec_iter iter;
@@ -36,7 +182,6 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
         cluster = blk_queue_cluster(q);
         seg_size = 0;
         nr_phys_segs = 0;
-        high = 0;
         for_each_bio(bio) {
                 bio_for_each_segment(bv, bio, iter) {
                         /*
@@ -46,13 +191,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
                         if (no_sg_merge)
                                 goto new_segment;
 
-                        /*
-                         * the trick here is making sure that a high page is
-                         * never considered part of another segment, since
-                         * that might change with the bounce page.
-                         */
-                        high = page_to_pfn(bv.bv_page) > queue_bounce_pfn(q);
-                        if (!high && !highprv && cluster) {
+                        if (prev && cluster) {
                                 if (seg_size + bv.bv_len
                                     > queue_max_segment_size(q))
                                         goto new_segment;
@@ -72,8 +211,8 @@ new_segment:
 
                         nr_phys_segs++;
                         bvprv = bv;
+                        prev = 1;
                         seg_size = bv.bv_len;
-                        highprv = high;
                 }
                 bbio = bio;
         }