block: Generic bio chaining

This adds a generic mechanism for chaining bio completions. This is
going to be used for a bio_split() replacement, and it turns out to be
very useful in a fair amount of driver code - a fair number of drivers
were implementing this in their own roundabout ways, often painfully.

Note that this means it's no longer to call bio_endio() more than once
on the same bio! This can cause problems for drivers that save/restore
bi_end_io. Arguably they shouldn't be saving/restoring bi_end_io at all
- in all but the simplest cases they'd be better off just cloning the
bio, and immutable biovecs is making bio cloning cheaper. But for now,
we add a bio_endio_nodec() for these cases.

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
Cc: Jens Axboe <axboe@kernel.dk>

5 files changed, 15 insertions, 4 deletions

diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index 0f0ab659914d..522f95778443 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -133,7 +133,7 @@ static void bch_bio_submit_split_done(struct closure *cl)
 
         s->bio->bi_end_io = s->bi_end_io;
         s->bio->bi_private = s->bi_private;
-        bio_endio(s->bio, 0);
+        bio_endio_nodec(s->bio, 0);
 
         closure_debug_destroy(&s->cl);
         mempool_free(s, s->p->bio_split_hook);
diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index 86f9c83eb30c..bf3a206abd78 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -765,6 +765,12 @@ static void writethrough_endio(struct bio *bio, int err)
 
         dm_unhook_bio(&pb->hook_info, bio);
 
+        /*
+         * Must bump bi_remaining to allow bio to complete with
+         * restored bi_end_io.
+         */
+        atomic_inc(&bio->bi_remaining);
+
         if (err) {
                 bio_endio(bio, err);
                 return;
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index 3ded8c729dfb..80b5cabbea29 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -1415,6 +1415,7 @@ out:
         if (full_bio) {
                 full_bio->bi_end_io = pe->full_bio_end_io;
                 full_bio->bi_private = pe->full_bio_private;
+                atomic_inc(&full_bio->bi_remaining);
         }
         free_pending_exception(pe);
 
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index a65402480c8c..1abb4a24c338 100644
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -611,8 +611,10 @@ static void cell_defer_no_holder(struct thin_c *tc, struct dm_bio_prison_cell *c
 
 static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
 {
-        if (m->bio)
+        if (m->bio) {
                 m->bio->bi_end_io = m->saved_bi_end_io;
+                atomic_inc(&m->bio->bi_remaining);
+        }
         cell_error(m->tc->pool, m->cell);
         list_del(&m->list);
         mempool_free(m, m->tc->pool->mapping_pool);
@@ -626,8 +628,10 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m)
         int r;
 
         bio = m->bio;
-        if (bio)
+        if (bio) {
                 bio->bi_end_io = m->saved_bi_end_io;
+                atomic_inc(&bio->bi_remaining);
+        }
 
         if (m->err) {
                 cell_error(pool, m->cell);
diff --git a/drivers/md/dm-verity.c b/drivers/md/dm-verity.c
index ac35e959d49b..796007a5e0e1 100644
--- a/drivers/md/dm-verity.c
+++ b/drivers/md/dm-verity.c
@@ -385,7 +385,7 @@ static void verity_finish_io(struct dm_verity_io *io, int error)
         bio->bi_end_io = io->orig_bi_end_io;
         bio->bi_private = io->orig_bi_private;
 
-        bio_endio(bio, error);
+        bio_endio_nodec(bio, error);
 }
 
 static void verity_work(struct work_struct *w)