10 files changed, 65 insertions, 160 deletions

diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index fa028fa82df4..cb64e64a4789 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -55,7 +55,7 @@ static void bch_bio_submit_split_done(struct closure *cl)
 
         s->bio->bi_end_io = s->bi_end_io;
         s->bio->bi_private = s->bi_private;
-        bio_endio_nodec(s->bio, 0);
+        bio_endio(s->bio, 0);
 
         closure_debug_destroy(&s->cl);
         mempool_free(s, s->p->bio_split_hook);
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index ab43faddb447..1616f668a4cb 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -619,7 +619,7 @@ static void do_bio_hook(struct search *s, struct bio *orig_bio)
         bio->bi_end_io          = request_endio;
         bio->bi_private         = &s->cl;
 
-        atomic_set(&bio->bi_cnt, 3);
+        bio_cnt_set(bio, 3);
 }
 
 static void search_free(struct closure *cl)
diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index 7755af351867..41b2594a80c6 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -86,12 +86,6 @@ static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio)
 {
         bio->bi_end_io = h->bi_end_io;
         bio->bi_private = h->bi_private;
-
-        /*
-         * Must bump bi_remaining to allow bio to complete with
-         * restored bi_end_io.
-         */
-        atomic_inc(&bio->bi_remaining);
 }
 
 /*----------------------------------------------------------------*/
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index 089d62751f7f..743fa9bbae9e 100644
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -1254,8 +1254,6 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, int error)
                         dm_bio_restore(bd, bio);
                         bio_record->details.bi_bdev = NULL;
 
-                        atomic_inc(&bio->bi_remaining);
-
                         queue_bio(ms, bio, rw);
                         return DM_ENDIO_INCOMPLETE;
                 }
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index f83a0f3fc365..7c82d3ccce87 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -1478,7 +1478,6 @@ out:
         if (full_bio) {
                 full_bio->bi_end_io = pe->full_bio_end_io;
                 full_bio->bi_private = pe->full_bio_private;
-                atomic_inc(&full_bio->bi_remaining);
         }
         increment_pending_exceptions_done_count();
 
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index 16ba55ad7089..a5f94125ad01 100644
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -942,21 +942,28 @@ static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *
 {
         unsigned type = dm_table_get_type(t);
         unsigned per_bio_data_size = 0;
-        struct dm_target *tgt;
         unsigned i;
 
-        if (unlikely(type == DM_TYPE_NONE)) {
+        switch (type) {
+        case DM_TYPE_BIO_BASED:
+                for (i = 0; i < t->num_targets; i++) {
+                        struct dm_target *tgt = t->targets + i;
+
+                        per_bio_data_size = max(per_bio_data_size,
+                                                tgt->per_bio_data_size);
+                }
+                t->mempools = dm_alloc_bio_mempools(t->integrity_supported,
+                                                    per_bio_data_size);
+                break;
+        case DM_TYPE_REQUEST_BASED:
+        case DM_TYPE_MQ_REQUEST_BASED:
+                t->mempools = dm_alloc_rq_mempools(md, type);
+                break;
+        default:
                 DMWARN("no table type is set, can't allocate mempools");
                 return -EINVAL;
         }
 
-        if (type == DM_TYPE_BIO_BASED)
-                for (i = 0; i < t->num_targets; i++) {
-                        tgt = t->targets + i;
-                        per_bio_data_size = max(per_bio_data_size, tgt->per_bio_data_size);
-                }
-
-        t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported, per_bio_data_size);
         if (!t->mempools)
                 return -ENOMEM;
 
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index 921aafd12aee..e852602c0091 100644
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -793,10 +793,9 @@ static void inc_remap_and_issue_cell(struct thin_c *tc,
 
 static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m)
 {
-        if (m->bio) {
+        if (m->bio)
                 m->bio->bi_end_io = m->saved_bi_end_io;
-                atomic_inc(&m->bio->bi_remaining);
-        }
+
         cell_error(m->tc->pool, m->cell);
         list_del(&m->list);
         mempool_free(m, m->tc->pool->mapping_pool);
@@ -810,10 +809,8 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m)
         int r;
 
         bio = m->bio;
-        if (bio) {
+        if (bio)
                 bio->bi_end_io = m->saved_bi_end_io;
-                atomic_inc(&bio->bi_remaining);
-        }
 
         if (m->err) {
                 cell_error(pool, m->cell);
diff --git a/drivers/md/dm-verity.c b/drivers/md/dm-verity.c
index 66616db33e6f..bb9c6a00e4b0 100644
--- a/drivers/md/dm-verity.c
+++ b/drivers/md/dm-verity.c
@@ -459,7 +459,7 @@ static void verity_finish_io(struct dm_verity_io *io, int error)
         bio->bi_end_io = io->orig_bi_end_io;
         bio->bi_private = io->orig_bi_private;
 
-        bio_endio_nodec(bio, error);
+        bio_endio(bio, error);
 }
 
 static void verity_work(struct work_struct *w)
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 2caf492890d6..4d6f089a0e9e 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -990,57 +990,6 @@ static void clone_endio(struct bio *bio, int error)
         dec_pending(io, error);
 }
 
-/*
- * Partial completion handling for request-based dm
- */
-static void end_clone_bio(struct bio *clone, int error)
-{
-        struct dm_rq_clone_bio_info *info =
-                container_of(clone, struct dm_rq_clone_bio_info, clone);
-        struct dm_rq_target_io *tio = info->tio;
-        struct bio *bio = info->orig;
-        unsigned int nr_bytes = info->orig->bi_iter.bi_size;
-
-        bio_put(clone);
-
-        if (tio->error)
-                /*
-                 * An error has already been detected on the request.
-                 * Once error occurred, just let clone->end_io() handle
-                 * the remainder.
-                 */
-                return;
-        else if (error) {
-                /*
-                 * Don't notice the error to the upper layer yet.
-                 * The error handling decision is made by the target driver,
-                 * when the request is completed.
-                 */
-                tio->error = error;
-                return;
-        }
-
-        /*
-         * I/O for the bio successfully completed.
-         * Notice the data completion to the upper layer.
-         */
-
-        /*
-         * bios are processed from the head of the list.
-         * So the completing bio should always be rq->bio.
-         * If it's not, something wrong is happening.
-         */
-        if (tio->orig->bio != bio)
-                DMERR("bio completion is going in the middle of the request");
-
-        /*
-         * Update the original request.
-         * Do not use blk_end_request() here, because it may complete
-         * the original request before the clone, and break the ordering.
-         */
-        blk_update_request(tio->orig, 0, nr_bytes);
-}
-
 static struct dm_rq_target_io *tio_from_request(struct request *rq)
 {
         return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special);
@@ -1087,8 +1036,6 @@ static void free_rq_clone(struct request *clone)
         struct dm_rq_target_io *tio = clone->end_io_data;
         struct mapped_device *md = tio->md;
 
-        blk_rq_unprep_clone(clone);
-
         if (md->type == DM_TYPE_MQ_REQUEST_BASED)
                 /* stacked on blk-mq queue(s) */
                 tio->ti->type->release_clone_rq(clone);
@@ -1827,39 +1774,13 @@ static void dm_dispatch_clone_request(struct request *clone, struct request *rq)
                 dm_complete_request(rq, r);
 }
 
-static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
-                                 void *data)
+static void setup_clone(struct request *clone, struct request *rq,
+                        struct dm_rq_target_io *tio)
 {
-        struct dm_rq_target_io *tio = data;
-        struct dm_rq_clone_bio_info *info =
-                container_of(bio, struct dm_rq_clone_bio_info, clone);
-
-        info->orig = bio_orig;
-        info->tio = tio;
-        bio->bi_end_io = end_clone_bio;
-
-        return 0;
-}
-
-static int setup_clone(struct request *clone, struct request *rq,
-                       struct dm_rq_target_io *tio, gfp_t gfp_mask)
-{
-        int r;
-
-        r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask,
-                              dm_rq_bio_constructor, tio);
-        if (r)
-                return r;
-
-        clone->cmd = rq->cmd;
-        clone->cmd_len = rq->cmd_len;
-        clone->sense = rq->sense;
+        blk_rq_prep_clone(clone, rq);
         clone->end_io = end_clone_request;
         clone->end_io_data = tio;
-
         tio->clone = clone;
-
-        return 0;
 }
 
 static struct request *clone_rq(struct request *rq, struct mapped_device *md,
@@ -1880,12 +1801,7 @@ static struct request *clone_rq(struct request *rq, struct mapped_device *md,
                 clone = tio->clone;
 
         blk_rq_init(NULL, clone);
-        if (setup_clone(clone, rq, tio, gfp_mask)) {
-                /* -ENOMEM */
-                if (alloc_clone)
-                        free_clone_request(md, clone);
-                return NULL;
-        }
+        setup_clone(clone, rq, tio);
 
         return clone;
 }
@@ -1979,11 +1895,7 @@ static int map_request(struct dm_rq_target_io *tio, struct request *rq,
                 }
                 if (r != DM_MAPIO_REMAPPED)
                         return r;
-                if (setup_clone(clone, rq, tio, GFP_ATOMIC)) {
-                        /* -ENOMEM */
-                        ti->type->release_clone_rq(clone);
-                        return DM_MAPIO_REQUEUE;
-                }
+                setup_clone(clone, rq, tio);
         }
 
         switch (r) {
@@ -2437,8 +2349,6 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
                 goto out;
         }
 
-        BUG_ON(!p || md->io_pool || md->rq_pool || md->bs);
-
         md->io_pool = p->io_pool;
         p->io_pool = NULL;
         md->rq_pool = p->rq_pool;
@@ -3544,48 +3454,23 @@ int dm_noflush_suspending(struct dm_target *ti)
 }
 EXPORT_SYMBOL_GPL(dm_noflush_suspending);
 
-struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned type,
-                                            unsigned integrity, unsigned per_bio_data_size)
+struct dm_md_mempools *dm_alloc_bio_mempools(unsigned integrity,
+                                             unsigned per_bio_data_size)
 {
-        struct dm_md_mempools *pools = kzalloc(sizeof(*pools), GFP_KERNEL);
-        struct kmem_cache *cachep = NULL;
-        unsigned int pool_size = 0;
+        struct dm_md_mempools *pools;
+        unsigned int pool_size = dm_get_reserved_bio_based_ios();
         unsigned int front_pad;
 
+        pools = kzalloc(sizeof(*pools), GFP_KERNEL);
         if (!pools)
                 return NULL;
 
-        type = filter_md_type(type, md);
+        front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) +
+                offsetof(struct dm_target_io, clone);
 
-        switch (type) {
-        case DM_TYPE_BIO_BASED:
-                cachep = _io_cache;
-                pool_size = dm_get_reserved_bio_based_ios();
-                front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
-                break;
-        case DM_TYPE_REQUEST_BASED:
-                cachep = _rq_tio_cache;
-                pool_size = dm_get_reserved_rq_based_ios();
-                pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache);
-                if (!pools->rq_pool)
-                        goto out;
-                /* fall through to setup remaining rq-based pools */
-        case DM_TYPE_MQ_REQUEST_BASED:
-                if (!pool_size)
-                        pool_size = dm_get_reserved_rq_based_ios();
-                front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
-                /* per_bio_data_size is not used. See __bind_mempools(). */
-                WARN_ON(per_bio_data_size != 0);
-                break;
-        default:
-                BUG();
-        }
-
-        if (cachep) {
-                pools->io_pool = mempool_create_slab_pool(pool_size, cachep);
-                if (!pools->io_pool)
-                        goto out;
-        }
+        pools->io_pool = mempool_create_slab_pool(pool_size, _io_cache);
+        if (!pools->io_pool)
+                goto out;
 
         pools->bs = bioset_create_nobvec(pool_size, front_pad);
         if (!pools->bs)
@@ -3595,10 +3480,34 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned t
                 goto out;
 
         return pools;
-
 out:
         dm_free_md_mempools(pools);
+        return NULL;
+}
+
+struct dm_md_mempools *dm_alloc_rq_mempools(struct mapped_device *md,
+                                            unsigned type)
+{
+        unsigned int pool_size = dm_get_reserved_rq_based_ios();
+        struct dm_md_mempools *pools;
+
+        pools = kzalloc(sizeof(*pools), GFP_KERNEL);
+        if (!pools)
+                return NULL;
+
+        if (filter_md_type(type, md) == DM_TYPE_REQUEST_BASED) {
+                pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache);
+                if (!pools->rq_pool)
+                        goto out;
+        }
 
+        pools->io_pool = mempool_create_slab_pool(pool_size, _rq_tio_cache);
+        if (!pools->io_pool)
+                goto out;
+
+        return pools;
+out:
+        dm_free_md_mempools(pools);
         return NULL;
 }
 
diff --git a/drivers/md/dm.h b/drivers/md/dm.h
index 6123c2bf9150..e6e66d087b26 100644
--- a/drivers/md/dm.h
+++ b/drivers/md/dm.h
@@ -222,8 +222,9 @@ void dm_kcopyd_exit(void);
 /*
  * Mempool operations
  */
-struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned type,
-                                            unsigned integrity, unsigned per_bio_data_size);
+struct dm_md_mempools *dm_alloc_bio_mempools(unsigned integrity,
+                                             unsigned per_bio_data_size);
+struct dm_md_mempools *dm_alloc_rq_mempools(struct mapped_device *md, unsigned type);
 void dm_free_md_mempools(struct dm_md_mempools *pools);
 
 /*