block: Introduce new bio_split()

The new bio_split() can split arbitrary bios - it's not restricted to
single page bios, like the old bio_split() (previously renamed to
bio_pair_split()). It also has different semantics - it doesn't allocate
a struct bio_pair, leaving it up to the caller to handle completions.

Then convert the existing bio_pair_split() users to the new bio_split()
- and also nvme, which was open coding bio splitting.

(We have to take that BUG_ON() out of bio_integrity_trim() because this
bio_split() needs to use it, and there's no reason it has to be used on
bios marked as cloned; BIO_CLONED doesn't seem to have clearly
documented semantics anyways.)

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: Matthew Wilcox <matthew.r.wilcox@intel.com>
Cc: Keith Busch <keith.busch@intel.com>
Cc: Vishal Verma <vishal.l.verma@intel.com>
Cc: Jiri Kosina <jkosina@suse.cz>
Cc: Neil Brown <neilb@suse.de>

2 files changed, 83 insertions, 159 deletions

diff --git a/drivers/block/nvme-core.c b/drivers/block/nvme-core.c
index 5539d2920872..1f14ac403945 100644
--- a/drivers/block/nvme-core.c
+++ b/drivers/block/nvme-core.c
@@ -441,104 +441,19 @@ int nvme_setup_prps(struct nvme_dev *dev, struct nvme_common_command *cmd,
         return total_len;
 }
 
-struct nvme_bio_pair {
-        struct bio b1, b2, *parent;
-        struct bio_vec *bv1, *bv2;
-        int err;
-        atomic_t cnt;
-};
-
-static void nvme_bio_pair_endio(struct bio *bio, int err)
-{
-        struct nvme_bio_pair *bp = bio->bi_private;
-
-        if (err)
-                bp->err = err;
-
-        if (atomic_dec_and_test(&bp->cnt)) {
-                bio_endio(bp->parent, bp->err);
-                kfree(bp->bv1);
-                kfree(bp->bv2);
-                kfree(bp);
-        }
-}
-
-static struct nvme_bio_pair *nvme_bio_split(struct bio *bio, int idx,
-                                                        int len, int offset)
-{
-        struct nvme_bio_pair *bp;
-
-        BUG_ON(len > bio->bi_iter.bi_size);
-        BUG_ON(idx > bio->bi_vcnt);
-
-        bp = kmalloc(sizeof(*bp), GFP_ATOMIC);
-        if (!bp)
-                return NULL;
-        bp->err = 0;
-
-        bp->b1 = *bio;
-        bp->b2 = *bio;
-
-        bp->b1.bi_iter.bi_size = len;
-        bp->b2.bi_iter.bi_size -= len;
-        bp->b1.bi_vcnt = idx;
-        bp->b2.bi_iter.bi_idx = idx;
-        bp->b2.bi_iter.bi_sector += len >> 9;
-
-        if (offset) {
-                bp->bv1 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec),
-                                                                GFP_ATOMIC);
-                if (!bp->bv1)
-                        goto split_fail_1;
-
-                bp->bv2 = kmalloc(bio->bi_max_vecs * sizeof(struct bio_vec),
-                                                                GFP_ATOMIC);
-                if (!bp->bv2)
-                        goto split_fail_2;
-
-                memcpy(bp->bv1, bio->bi_io_vec,
-                        bio->bi_max_vecs * sizeof(struct bio_vec));
-                memcpy(bp->bv2, bio->bi_io_vec,
-                        bio->bi_max_vecs * sizeof(struct bio_vec));
-
-                bp->b1.bi_io_vec = bp->bv1;
-                bp->b2.bi_io_vec = bp->bv2;
-                bp->b2.bi_io_vec[idx].bv_offset += offset;
-                bp->b2.bi_io_vec[idx].bv_len -= offset;
-                bp->b1.bi_io_vec[idx].bv_len = offset;
-                bp->b1.bi_vcnt++;
-        } else
-                bp->bv1 = bp->bv2 = NULL;
-
-        bp->b1.bi_private = bp;
-        bp->b2.bi_private = bp;
-
-        bp->b1.bi_end_io = nvme_bio_pair_endio;
-        bp->b2.bi_end_io = nvme_bio_pair_endio;
-
-        bp->parent = bio;
-        atomic_set(&bp->cnt, 2);
-
-        return bp;
-
- split_fail_2:
-        kfree(bp->bv1);
- split_fail_1:
-        kfree(bp);
-        return NULL;
-}
-
 static int nvme_split_and_submit(struct bio *bio, struct nvme_queue *nvmeq,
-                                                int idx, int len, int offset)
+                                 int len)
 {
-        struct nvme_bio_pair *bp = nvme_bio_split(bio, idx, len, offset);
-        if (!bp)
+        struct bio *split = bio_split(bio, len >> 9, GFP_ATOMIC, NULL);
+        if (!split)
                 return -ENOMEM;
 
+        bio_chain(split, bio);
+
         if (bio_list_empty(&nvmeq->sq_cong))
                 add_wait_queue(&nvmeq->sq_full, &nvmeq->sq_cong_wait);
-        bio_list_add(&nvmeq->sq_cong, &bp->b1);
-        bio_list_add(&nvmeq->sq_cong, &bp->b2);
+        bio_list_add(&nvmeq->sq_cong, split);
+        bio_list_add(&nvmeq->sq_cong, bio);
 
         return 0;
 }
@@ -568,8 +483,7 @@ static int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod,
                 } else {
                         if (!first && BIOVEC_NOT_VIRT_MERGEABLE(&bvprv, &bvec))
                                 return nvme_split_and_submit(bio, nvmeq,
-                                                             iter.bi_idx,
-                                                             length, 0);
+                                                             length);
 
                         sg = sg ? sg + 1 : iod->sg;
                         sg_set_page(sg, bvec.bv_page,
@@ -578,9 +492,7 @@ static int nvme_map_bio(struct nvme_queue *nvmeq, struct nvme_iod *iod,
                 }
 
                 if (split_len - length < bvec.bv_len)
-                        return nvme_split_and_submit(bio, nvmeq, iter.bi_idx,
-                                                     split_len,
-                                                     split_len - length);
+                        return nvme_split_and_submit(bio, nvmeq, split_len);
                 length += bvec.bv_len;
                 bvprv = bvec;
                 first = 0;
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index 28789b82ae7d..3dda09a5ec41 100644
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -2338,75 +2338,29 @@ static void pkt_end_io_read_cloned(struct bio *bio, int err)
         pkt_bio_finished(pd);
 }
 
-static void pkt_make_request(struct request_queue *q, struct bio *bio)
+static void pkt_make_request_read(struct pktcdvd_device *pd, struct bio *bio)
 {
-        struct pktcdvd_device *pd;
-        char b[BDEVNAME_SIZE];
+        struct bio *cloned_bio = bio_clone(bio, GFP_NOIO);
+        struct packet_stacked_data *psd = mempool_alloc(psd_pool, GFP_NOIO);
+
+        psd->pd = pd;
+        psd->bio = bio;
+        cloned_bio->bi_bdev = pd->bdev;
+        cloned_bio->bi_private = psd;
+        cloned_bio->bi_end_io = pkt_end_io_read_cloned;
+        pd->stats.secs_r += bio_sectors(bio);
+        pkt_queue_bio(pd, cloned_bio);
+}
+
+static void pkt_make_request_write(struct request_queue *q, struct bio *bio)
+{
+        struct pktcdvd_device *pd = q->queuedata;
         sector_t zone;
         struct packet_data *pkt;
         int was_empty, blocked_bio;
         struct pkt_rb_node *node;
 
-        pd = q->queuedata;
-        if (!pd) {
-                pr_err("%s incorrect request queue\n",
-                       bdevname(bio->bi_bdev, b));
-                goto end_io;
-        }
-
-        /*
-         * Clone READ bios so we can have our own bi_end_io callback.
-         */
-        if (bio_data_dir(bio) == READ) {
-                struct bio *cloned_bio = bio_clone(bio, GFP_NOIO);
-                struct packet_stacked_data *psd = mempool_alloc(psd_pool, GFP_NOIO);
-
-                psd->pd = pd;
-                psd->bio = bio;
-                cloned_bio->bi_bdev = pd->bdev;
-                cloned_bio->bi_private = psd;
-                cloned_bio->bi_end_io = pkt_end_io_read_cloned;
-                pd->stats.secs_r += bio_sectors(bio);
-                pkt_queue_bio(pd, cloned_bio);
-                return;
-        }
-
-        if (!test_bit(PACKET_WRITABLE, &pd->flags)) {
-                pkt_notice(pd, "WRITE for ro device (%llu)\n",
-                           (unsigned long long)bio->bi_iter.bi_sector);
-                goto end_io;
-        }
-
-        if (!bio->bi_iter.bi_size || (bio->bi_iter.bi_size % CD_FRAMESIZE)) {
-                pkt_err(pd, "wrong bio size\n");
-                goto end_io;
-        }
-
-        blk_queue_bounce(q, &bio);
-
         zone = get_zone(bio->bi_iter.bi_sector, pd);
-        pkt_dbg(2, pd, "start = %6llx stop = %6llx\n",
-                (unsigned long long)bio->bi_iter.bi_sector,
-                (unsigned long long)bio_end_sector(bio));
-
-        /* Check if we have to split the bio */
-        {
-                struct bio_pair *bp;
-                sector_t last_zone;
-                int first_sectors;
-
-                last_zone = get_zone(bio_end_sector(bio) - 1, pd);
-                if (last_zone != zone) {
-                        BUG_ON(last_zone != zone + pd->settings.size);
-                        first_sectors = last_zone - bio->bi_iter.bi_sector;
-                        bp = bio_pair_split(bio, first_sectors);
-                        BUG_ON(!bp);
-                        pkt_make_request(q, &bp->bio1);
-                        pkt_make_request(q, &bp->bio2);
-                        bio_pair_release(bp);
-                        return;
-                }
-        }
 
         /*
          * If we find a matching packet in state WAITING or READ_WAIT, we can
@@ -2480,6 +2434,64 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio)
                  */
                 wake_up(&pd->wqueue);
         }
+}
+
+static void pkt_make_request(struct request_queue *q, struct bio *bio)
+{
+        struct pktcdvd_device *pd;
+        char b[BDEVNAME_SIZE];
+        struct bio *split;
+
+        pd = q->queuedata;
+        if (!pd) {
+                pr_err("%s incorrect request queue\n",
+                       bdevname(bio->bi_bdev, b));
+                goto end_io;
+        }
+
+        pkt_dbg(2, pd, "start = %6llx stop = %6llx\n",
+                (unsigned long long)bio->bi_iter.bi_sector,
+                (unsigned long long)bio_end_sector(bio));
+
+        /*
+         * Clone READ bios so we can have our own bi_end_io callback.
+         */
+        if (bio_data_dir(bio) == READ) {
+                pkt_make_request_read(pd, bio);
+                return;
+        }
+
+        if (!test_bit(PACKET_WRITABLE, &pd->flags)) {
+                pkt_notice(pd, "WRITE for ro device (%llu)\n",
+                           (unsigned long long)bio->bi_iter.bi_sector);
+                goto end_io;
+        }
+
+        if (!bio->bi_iter.bi_size || (bio->bi_iter.bi_size % CD_FRAMESIZE)) {
+                pkt_err(pd, "wrong bio size\n");
+                goto end_io;
+        }
+
+        blk_queue_bounce(q, &bio);
+
+        do {
+                sector_t zone = get_zone(bio->bi_iter.bi_sector, pd);
+                sector_t last_zone = get_zone(bio_end_sector(bio) - 1, pd);
+
+                if (last_zone != zone) {
+                        BUG_ON(last_zone != zone + pd->settings.size);
+
+                        split = bio_split(bio, last_zone -
+                                          bio->bi_iter.bi_sector,
+                                          GFP_NOIO, fs_bio_set);
+                        bio_chain(split, bio);
+                } else {
+                        split = bio;
+                }
+
+                pkt_make_request_write(q, split);
+        } while (split != bio);
+
         return;
 end_io:
         bio_io_error(bio);