block: Introduce new bio_split()

The new bio_split() can split arbitrary bios - it's not restricted to
single page bios, like the old bio_split() (previously renamed to
bio_pair_split()). It also has different semantics - it doesn't allocate
a struct bio_pair, leaving it up to the caller to handle completions.

Then convert the existing bio_pair_split() users to the new bio_split()
- and also nvme, which was open coding bio splitting.

(We have to take that BUG_ON() out of bio_integrity_trim() because this
bio_split() needs to use it, and there's no reason it has to be used on
bios marked as cloned; BIO_CLONED doesn't seem to have clearly
documented semantics anyways.)

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: Matthew Wilcox <matthew.r.wilcox@intel.com>
Cc: Keith Busch <keith.busch@intel.com>
Cc: Vishal Verma <vishal.l.verma@intel.com>
Cc: Jiri Kosina <jkosina@suse.cz>
Cc: Neil Brown <neilb@suse.de>

6 files changed, 131 insertions, 250 deletions

diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index 6b6fe935be73..964353c5329d 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -901,7 +901,6 @@ void bch_bbio_endio(struct cache_set *, struct bio *, int, const char *);
 void bch_bbio_free(struct bio *, struct cache_set *);
 struct bio *bch_bbio_alloc(struct cache_set *);
 
-struct bio *bch_bio_split(struct bio *, int, gfp_t, struct bio_set *);
 void bch_generic_make_request(struct bio *, struct bio_split_pool *);
 void __bch_submit_bbio(struct bio *, struct cache_set *);
 void bch_submit_bbio(struct bio *, struct cache_set *, struct bkey *, unsigned);
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index 522f95778443..fa028fa82df4 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -11,84 +11,6 @@
 
 #include <linux/blkdev.h>
 
-/**
- * bch_bio_split - split a bio
- * @bio:        bio to split
- * @sectors:    number of sectors to split from the front of @bio
- * @gfp:        gfp mask
- * @bs:         bio set to allocate from
- *
- * Allocates and returns a new bio which represents @sectors from the start of
- * @bio, and updates @bio to represent the remaining sectors.
- *
- * If bio_sectors(@bio) was less than or equal to @sectors, returns @bio
- * unchanged.
- *
- * The newly allocated bio will point to @bio's bi_io_vec, if the split was on a
- * bvec boundry; it is the caller's responsibility to ensure that @bio is not
- * freed before the split.
- */
-struct bio *bch_bio_split(struct bio *bio, int sectors,
-                          gfp_t gfp, struct bio_set *bs)
-{
-        unsigned vcnt = 0, nbytes = sectors << 9;
-        struct bio_vec bv;
-        struct bvec_iter iter;
-        struct bio *ret = NULL;
-
-        BUG_ON(sectors <= 0);
-
-        if (sectors >= bio_sectors(bio))
-                return bio;
-
-        if (bio->bi_rw & REQ_DISCARD) {
-                ret = bio_alloc_bioset(gfp, 1, bs);
-                if (!ret)
-                        return NULL;
-                goto out;
-        }
-
-        bio_for_each_segment(bv, bio, iter) {
-                vcnt++;
-
-                if (nbytes <= bv.bv_len)
-                        break;
-
-                nbytes -= bv.bv_len;
-        }
-
-        ret = bio_alloc_bioset(gfp, vcnt, bs);
-        if (!ret)
-                return NULL;
-
-        bio_for_each_segment(bv, bio, iter) {
-                ret->bi_io_vec[ret->bi_vcnt++] = bv;
-
-                if (ret->bi_vcnt == vcnt)
-                        break;
-        }
-
-        ret->bi_io_vec[ret->bi_vcnt - 1].bv_len = nbytes;
-out:
-        ret->bi_bdev    = bio->bi_bdev;
-        ret->bi_iter.bi_sector  = bio->bi_iter.bi_sector;
-        ret->bi_iter.bi_size    = sectors << 9;
-        ret->bi_rw      = bio->bi_rw;
-
-        if (bio_integrity(bio)) {
-                if (bio_integrity_clone(ret, bio, gfp)) {
-                        bio_put(ret);
-                        return NULL;
-                }
-
-                bio_integrity_trim(ret, 0, bio_sectors(ret));
-        }
-
-        bio_advance(bio, ret->bi_iter.bi_size);
-
-        return ret;
-}
-
 static unsigned bch_bio_max_sectors(struct bio *bio)
 {
         struct request_queue *q = bdev_get_queue(bio->bi_bdev);
@@ -172,8 +94,8 @@ void bch_generic_make_request(struct bio *bio, struct bio_split_pool *p)
         bio_get(bio);
 
         do {
-                n = bch_bio_split(bio, bch_bio_max_sectors(bio),
-                                  GFP_NOIO, s->p->bio_split);
+                n = bio_next_split(bio, bch_bio_max_sectors(bio),
+                                   GFP_NOIO, s->p->bio_split);
 
                 n->bi_end_io    = bch_bio_submit_split_endio;
                 n->bi_private   = &s->cl;
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index 63451c724781..5878cdb39529 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -371,7 +371,7 @@ static void bch_data_insert_start(struct closure *cl)
                                        op->writeback))
                         goto err;
 
-                n = bch_bio_split(bio, KEY_SIZE(k), GFP_NOIO, split);
+                n = bio_next_split(bio, KEY_SIZE(k), GFP_NOIO, split);
 
                 n->bi_end_io    = bch_data_insert_endio;
                 n->bi_private   = cl;
@@ -679,9 +679,9 @@ static int cache_lookup_fn(struct btree_op *op, struct btree *b, struct bkey *k)
         if (KEY_DIRTY(k))
                 s->read_dirty_data = true;
 
-        n = bch_bio_split(bio, min_t(uint64_t, INT_MAX,
-                                     KEY_OFFSET(k) - bio->bi_iter.bi_sector),
-                          GFP_NOIO, s->d->bio_split);
+        n = bio_next_split(bio, min_t(uint64_t, INT_MAX,
+                                      KEY_OFFSET(k) - bio->bi_iter.bi_sector),
+                           GFP_NOIO, s->d->bio_split);
 
         bio_key = &container_of(n, struct bbio, bio)->key;
         bch_bkey_copy_single_ptr(bio_key, k, ptr);
@@ -920,7 +920,7 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s,
         struct bio *miss, *cache_bio;
 
         if (s->cache_miss || s->iop.bypass) {
-                miss = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split);
+                miss = bio_next_split(bio, sectors, GFP_NOIO, s->d->bio_split);
                 ret = miss == bio ? MAP_DONE : MAP_CONTINUE;
                 goto out_submit;
         }
@@ -943,7 +943,7 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s,
 
         s->iop.replace = true;
 
-        miss = bch_bio_split(bio, sectors, GFP_NOIO, s->d->bio_split);
+        miss = bio_next_split(bio, sectors, GFP_NOIO, s->d->bio_split);
 
         /* btree_search_recurse()'s btree iterator is no good anymore */
         ret = miss == bio ? MAP_DONE : -EINTR;
diff --git a/drivers/md/linear.c b/drivers/md/linear.c
index e9b53e9793bf..56f534b4a2d2 100644
--- a/drivers/md/linear.c
+++ b/drivers/md/linear.c
@@ -288,65 +288,65 @@ static int linear_stop (struct mddev *mddev)
 
 static void linear_make_request(struct mddev *mddev, struct bio *bio)
 {
+        char b[BDEVNAME_SIZE];
         struct dev_info *tmp_dev;
-        sector_t start_sector;
+        struct bio *split;
+        sector_t start_sector, end_sector, data_offset;
 
         if (unlikely(bio->bi_rw & REQ_FLUSH)) {
                 md_flush_request(mddev, bio);
                 return;
         }
 
-        rcu_read_lock();
-        tmp_dev = which_dev(mddev, bio->bi_iter.bi_sector);
-        start_sector = tmp_dev->end_sector - tmp_dev->rdev->sectors;
-
-
-        if (unlikely(bio->bi_iter.bi_sector >= (tmp_dev->end_sector)
-                     || (bio->bi_iter.bi_sector < start_sector))) {
-                char b[BDEVNAME_SIZE];
-
-                printk(KERN_ERR
-                       "md/linear:%s: make_request: Sector %llu out of bounds on "
-                       "dev %s: %llu sectors, offset %llu\n",
-                       mdname(mddev),
-                       (unsigned long long)bio->bi_iter.bi_sector,
-                       bdevname(tmp_dev->rdev->bdev, b),
-                       (unsigned long long)tmp_dev->rdev->sectors,
-                       (unsigned long long)start_sector);
-                rcu_read_unlock();
-                bio_io_error(bio);
-                return;
-        }
-        if (unlikely(bio_end_sector(bio) > tmp_dev->end_sector)) {
-                /* This bio crosses a device boundary, so we have to
-                 * split it.
-                 */
-                struct bio_pair *bp;
-                sector_t end_sector = tmp_dev->end_sector;
+        do {
+                rcu_read_lock();
 
-                rcu_read_unlock();
-
-                bp = bio_pair_split(bio, end_sector - bio->bi_iter.bi_sector);
+                tmp_dev = which_dev(mddev, bio->bi_iter.bi_sector);
+                start_sector = tmp_dev->end_sector - tmp_dev->rdev->sectors;
+                end_sector = tmp_dev->end_sector;
+                data_offset = tmp_dev->rdev->data_offset;
+                bio->bi_bdev = tmp_dev->rdev->bdev;
 
-                linear_make_request(mddev, &bp->bio1);
-                linear_make_request(mddev, &bp->bio2);
-                bio_pair_release(bp);
-                return;
-        }
-                    
-        bio->bi_bdev = tmp_dev->rdev->bdev;
-        bio->bi_iter.bi_sector = bio->bi_iter.bi_sector - start_sector
-                + tmp_dev->rdev->data_offset;
-        rcu_read_unlock();
+                rcu_read_unlock();
 
-        if (unlikely((bio->bi_rw & REQ_DISCARD) &&
-                     !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) {
-                /* Just ignore it */
-                bio_endio(bio, 0);
-                return;
-        }
+                if (unlikely(bio->bi_iter.bi_sector >= end_sector ||
+                             bio->bi_iter.bi_sector < start_sector))
+                        goto out_of_bounds;
+
+                if (unlikely(bio_end_sector(bio) > end_sector)) {
+                        /* This bio crosses a device boundary, so we have to
+                         * split it.
+                         */
+                        split = bio_split(bio, end_sector -
+                                          bio->bi_iter.bi_sector,
+                                          GFP_NOIO, fs_bio_set);
+                        bio_chain(split, bio);
+                } else {
+                        split = bio;
+                }
 
-        generic_make_request(bio);
+                split->bi_iter.bi_sector = split->bi_iter.bi_sector -
+                        start_sector + data_offset;
+
+                if (unlikely((split->bi_rw & REQ_DISCARD) &&
+                         !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) {
+                        /* Just ignore it */
+                        bio_endio(split, 0);
+                } else
+                        generic_make_request(split);
+        } while (split != bio);
+        return;
+
+out_of_bounds:
+        printk(KERN_ERR
+               "md/linear:%s: make_request: Sector %llu out of bounds on "
+               "dev %s: %llu sectors, offset %llu\n",
+               mdname(mddev),
+               (unsigned long long)bio->bi_iter.bi_sector,
+               bdevname(tmp_dev->rdev->bdev, b),
+               (unsigned long long)tmp_dev->rdev->sectors,
+               (unsigned long long)start_sector);
+        bio_io_error(bio);
 }
 
 static void linear_status (struct seq_file *seq, struct mddev *mddev)
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
index ea754dd1a5f5..407a99e46f69 100644
--- a/drivers/md/raid0.c
+++ b/drivers/md/raid0.c
@@ -513,65 +513,44 @@ static inline int is_io_in_chunk_boundary(struct mddev *mddev,
 
 static void raid0_make_request(struct mddev *mddev, struct bio *bio)
 {
-        unsigned int chunk_sects;
-        sector_t sector_offset;
         struct strip_zone *zone;
         struct md_rdev *tmp_dev;
+        struct bio *split;
 
         if (unlikely(bio->bi_rw & REQ_FLUSH)) {
                 md_flush_request(mddev, bio);
                 return;
         }
 
-        chunk_sects = mddev->chunk_sectors;
-        if (unlikely(!is_io_in_chunk_boundary(mddev, chunk_sects, bio))) {
+        do {
                 sector_t sector = bio->bi_iter.bi_sector;
-                struct bio_pair *bp;
-                /* Sanity check -- queue functions should prevent this happening */
-                if (bio_multiple_segments(bio))
-                        goto bad_map;
-                /* This is a one page bio that upper layers
-                 * refuse to split for us, so we need to split it.
-                 */
-                if (likely(is_power_of_2(chunk_sects)))
-                        bp = bio_pair_split(bio, chunk_sects - (sector &
-                                                           (chunk_sects-1)));
-                else
-                        bp = bio_pair_split(bio, chunk_sects -
-                                            sector_div(sector, chunk_sects));
-                raid0_make_request(mddev, &bp->bio1);
-                raid0_make_request(mddev, &bp->bio2);
-                bio_pair_release(bp);
-                return;
-        }
-
-        sector_offset = bio->bi_iter.bi_sector;
-        zone = find_zone(mddev->private, &sector_offset);
-        tmp_dev = map_sector(mddev, zone, bio->bi_iter.bi_sector,
-                             &sector_offset);
-        bio->bi_bdev = tmp_dev->bdev;
-        bio->bi_iter.bi_sector = sector_offset + zone->dev_start +
-                tmp_dev->data_offset;
-
-        if (unlikely((bio->bi_rw & REQ_DISCARD) &&
-                     !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) {
-                /* Just ignore it */
-                bio_endio(bio, 0);
-                return;
-        }
-
-        generic_make_request(bio);
-        return;
-
-bad_map:
-        printk("md/raid0:%s: make_request bug: can't convert block across chunks"
-               " or bigger than %dk %llu %d\n",
-               mdname(mddev), chunk_sects / 2,
-               (unsigned long long)bio->bi_iter.bi_sector,
-               bio_sectors(bio) / 2);
+                unsigned chunk_sects = mddev->chunk_sectors;
+
+                unsigned sectors = chunk_sects -
+                        (likely(is_power_of_2(chunk_sects))
+                         ? (sector & (chunk_sects-1))
+                         : sector_div(sector, chunk_sects));
+
+                if (sectors < bio_sectors(bio)) {
+                        split = bio_split(bio, sectors, GFP_NOIO, fs_bio_set);
+                        bio_chain(split, bio);
+                } else {
+                        split = bio;
+                }
 
-        bio_io_error(bio);
-        return;
+                zone = find_zone(mddev->private, &sector);
+                tmp_dev = map_sector(mddev, zone, sector, &sector);
+                split->bi_bdev = tmp_dev->bdev;
+                split->bi_iter.bi_sector = sector + zone->dev_start +
+                        tmp_dev->data_offset;
+
+                if (unlikely((split->bi_rw & REQ_DISCARD) &&
+                         !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) {
+                        /* Just ignore it */
+                        bio_endio(split, 0);
+                } else
+                        generic_make_request(split);
+        } while (split != bio);
 }
 
 static void raid0_status(struct seq_file *seq, struct mddev *mddev)
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 69c1bc8da88f..6d43d88657aa 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -1152,14 +1152,12 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
         kfree(plug);
 }
 
-static void make_request(struct mddev *mddev, struct bio * bio)
+static void __make_request(struct mddev *mddev, struct bio *bio)
 {
         struct r10conf *conf = mddev->private;
         struct r10bio *r10_bio;
         struct bio *read_bio;
         int i;
-        sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
-        int chunk_sects = chunk_mask + 1;
         const int rw = bio_data_dir(bio);
         const unsigned long do_sync = (bio->bi_rw & REQ_SYNC);
         const unsigned long do_fua = (bio->bi_rw & REQ_FUA);
@@ -1174,69 +1172,6 @@ static void make_request(struct mddev *mddev, struct bio * bio)
         int max_sectors;
         int sectors;
 
-        if (unlikely(bio->bi_rw & REQ_FLUSH)) {
-                md_flush_request(mddev, bio);
-                return;
-        }
-
-        /* If this request crosses a chunk boundary, we need to
-         * split it.  This will only happen for 1 PAGE (or less) requests.
-         */
-        if (unlikely((bio->bi_iter.bi_sector & chunk_mask) + bio_sectors(bio)
-                     > chunk_sects
-                     && (conf->geo.near_copies < conf->geo.raid_disks
-                         || conf->prev.near_copies < conf->prev.raid_disks))) {
-                struct bio_pair *bp;
-                /* Sanity check -- queue functions should prevent this happening */
-                if (bio_multiple_segments(bio))
-                        goto bad_map;
-                /* This is a one page bio that upper layers
-                 * refuse to split for us, so we need to split it.
-                 */
-                bp = bio_pair_split(bio, chunk_sects -
-                               (bio->bi_iter.bi_sector & (chunk_sects - 1)));
-
-                /* Each of these 'make_request' calls will call 'wait_barrier'.
-                 * If the first succeeds but the second blocks due to the resync
-                 * thread raising the barrier, we will deadlock because the
-                 * IO to the underlying device will be queued in generic_make_request
-                 * and will never complete, so will never reduce nr_pending.
-                 * So increment nr_waiting here so no new raise_barriers will
-                 * succeed, and so the second wait_barrier cannot block.
-                 */
-                spin_lock_irq(&conf->resync_lock);
-                conf->nr_waiting++;
-                spin_unlock_irq(&conf->resync_lock);
-
-                make_request(mddev, &bp->bio1);
-                make_request(mddev, &bp->bio2);
-
-                spin_lock_irq(&conf->resync_lock);
-                conf->nr_waiting--;
-                wake_up(&conf->wait_barrier);
-                spin_unlock_irq(&conf->resync_lock);
-
-                bio_pair_release(bp);
-                return;
-        bad_map:
-                printk("md/raid10:%s: make_request bug: can't convert block across chunks"
-                       " or bigger than %dk %llu %d\n", mdname(mddev), chunk_sects/2,
-                       (unsigned long long)bio->bi_iter.bi_sector,
-                       bio_sectors(bio) / 2);
-
-                bio_io_error(bio);
-                return;
-        }
-
-        md_write_start(mddev, bio);
-
-        /*
-         * Register the new request and wait if the reconstruction
-         * thread has put up a bar for new requests.
-         * Continue immediately if no resync is active currently.
-         */
-        wait_barrier(conf);
-
         sectors = bio_sectors(bio);
         while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
             bio->bi_iter.bi_sector < conf->reshape_progress &&
@@ -1600,6 +1535,52 @@ retry_write:
                 goto retry_write;
         }
         one_write_done(r10_bio);
+}
+
+static void make_request(struct mddev *mddev, struct bio *bio)
+{
+        struct r10conf *conf = mddev->private;
+        sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
+        int chunk_sects = chunk_mask + 1;
+
+        struct bio *split;
+
+        if (unlikely(bio->bi_rw & REQ_FLUSH)) {
+                md_flush_request(mddev, bio);
+                return;
+        }
+
+        md_write_start(mddev, bio);
+
+        /*
+         * Register the new request and wait if the reconstruction
+         * thread has put up a bar for new requests.
+         * Continue immediately if no resync is active currently.
+         */
+        wait_barrier(conf);
+
+        do {
+
+                /*
+                 * If this request crosses a chunk boundary, we need to split
+                 * it.
+                 */
+                if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +
+                             bio_sectors(bio) > chunk_sects
+                             && (conf->geo.near_copies < conf->geo.raid_disks
+                                 || conf->prev.near_copies <
+                                 conf->prev.raid_disks))) {
+                        split = bio_split(bio, chunk_sects -
+                                          (bio->bi_iter.bi_sector &
+                                           (chunk_sects - 1)),
+                                          GFP_NOIO, fs_bio_set);
+                        bio_chain(split, bio);
+                } else {
+                        split = bio;
+                }
+
+                __make_request(mddev, split);
+        } while (split != bio);
 
         /* In case raid10d snuck in to freeze_array */
         wake_up(&conf->wait_barrier);