dm: prepare for allocating blk-mq clone requests in target

For blk-mq request-based DM the responsibility of allocating a cloned
request will be transfered from DM core to the target type.

To prepare for conditionally using this new model the original
request's 'special' now points to the dm_rq_target_io because the
clone is allocated later in the block layer rather than in DM core.

Signed-off-by: Keith Busch <keith.busch@intel.com>
Signed-off-by: Mike Snitzer <snitzer@redhat.com>

1 files changed, 66 insertions, 68 deletions

diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index f0e34070c11d..ae1219893948 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -1016,7 +1016,7 @@ static void end_clone_bio(struct bio *clone, int error)
  * the md may be freed in dm_put() at the end of this function.
  * Or do dm_get() before calling this function and dm_put() later.
  */
-static void rq_completed(struct mapped_device *md, int rw, int run_queue)
+static void rq_completed(struct mapped_device *md, int rw, bool run_queue)
 {
         atomic_dec(&md->pending[rw]);
 
@@ -1050,7 +1050,8 @@ static void free_rq_clone(struct request *clone)
 
 /*
  * Complete the clone and the original request.
- * Must be called without queue lock.
+ * Must be called without clone's queue lock held,
+ * see end_clone_request() for more details.
  */
 static void dm_end_request(struct request *clone, int error)
 {
@@ -1079,7 +1080,8 @@ static void dm_end_request(struct request *clone, int error)
 
 static void dm_unprep_request(struct request *rq)
 {
-        struct request *clone = rq->special;
+        struct dm_rq_target_io *tio = rq->special;
+        struct request *clone = tio->clone;
 
         rq->special = NULL;
         rq->cmd_flags &= ~REQ_DONTPREP;
@@ -1090,12 +1092,10 @@ static void dm_unprep_request(struct request *rq)
 /*
  * Requeue the original request of a clone.
  */
-static void dm_requeue_unmapped_request(struct request *clone)
+static void dm_requeue_unmapped_original_request(struct mapped_device *md,
+                                                 struct request *rq)
 {
-        int rw = rq_data_dir(clone);
-        struct dm_rq_target_io *tio = clone->end_io_data;
-        struct mapped_device *md = tio->md;
-        struct request *rq = tio->orig;
+        int rw = rq_data_dir(rq);
         struct request_queue *q = rq->q;
         unsigned long flags;
 
@@ -1105,7 +1105,14 @@ static void dm_requeue_unmapped_request(struct request *clone)
         blk_requeue_request(q, rq);
         spin_unlock_irqrestore(q->queue_lock, flags);
 
-        rq_completed(md, rw, 0);
+        rq_completed(md, rw, false);
+}
+
+static void dm_requeue_unmapped_request(struct request *clone)
+{
+        struct dm_rq_target_io *tio = clone->end_io_data;
+
+        dm_requeue_unmapped_original_request(tio->md, tio->orig);
 }
 
 static void __stop_queue(struct request_queue *q)
@@ -1175,8 +1182,8 @@ static void dm_done(struct request *clone, int error, bool mapped)
 static void dm_softirq_done(struct request *rq)
 {
         bool mapped = true;
-        struct request *clone = rq->completion_data;
-        struct dm_rq_target_io *tio = clone->end_io_data;
+        struct dm_rq_target_io *tio = rq->special;
+        struct request *clone = tio->clone;
 
         if (rq->cmd_flags & REQ_FAILED)
                 mapped = false;
@@ -1188,13 +1195,11 @@ static void dm_softirq_done(struct request *rq)
  * Complete the clone and the original request with the error status
  * through softirq context.
  */
-static void dm_complete_request(struct request *clone, int error)
+static void dm_complete_request(struct request *rq, int error)
 {
-        struct dm_rq_target_io *tio = clone->end_io_data;
-        struct request *rq = tio->orig;
+        struct dm_rq_target_io *tio = rq->special;
 
         tio->error = error;
-        rq->completion_data = clone;
         blk_complete_request(rq);
 }
 
@@ -1204,20 +1209,19 @@ static void dm_complete_request(struct request *clone, int error)
  * Target's rq_end_io() function isn't called.
  * This may be used when the target's map_rq() function fails.
  */
-static void dm_kill_unmapped_request(struct request *clone, int error)
+static void dm_kill_unmapped_request(struct request *rq, int error)
 {
-        struct dm_rq_target_io *tio = clone->end_io_data;
-        struct request *rq = tio->orig;
-
         rq->cmd_flags |= REQ_FAILED;
-        dm_complete_request(clone, error);
+        dm_complete_request(rq, error);
 }
 
 /*
- * Called with the queue lock held
+ * Called with the clone's queue lock held
  */
 static void end_clone_request(struct request *clone, int error)
 {
+        struct dm_rq_target_io *tio = clone->end_io_data;
+
         /*
          * For just cleaning up the information of the queue in which
          * the clone was dispatched.
@@ -1228,13 +1232,13 @@ static void end_clone_request(struct request *clone, int error)
 
         /*
          * Actual request completion is done in a softirq context which doesn't
-         * hold the queue lock.  Otherwise, deadlock could occur because:
+         * hold the clone's queue lock.  Otherwise, deadlock could occur because:
          *     - another request may be submitted by the upper level driver
          *       of the stacking during the completion
          *     - the submission which requires queue lock may be done
-         *       against this queue
+         *       against this clone's queue
          */
-        dm_complete_request(clone, error);
+        dm_complete_request(tio->orig, error);
 }
 
 /*
@@ -1712,16 +1716,17 @@ static void dm_request(struct request_queue *q, struct bio *bio)
                 _dm_request(q, bio);
 }
 
-static void dm_dispatch_request(struct request *rq)
+static void dm_dispatch_clone_request(struct request *clone, struct request *rq)
 {
         int r;
 
-        if (blk_queue_io_stat(rq->q))
-                rq->cmd_flags |= REQ_IO_STAT;
+        if (blk_queue_io_stat(clone->q))
+                clone->cmd_flags |= REQ_IO_STAT;
 
-        rq->start_time = jiffies;
-        r = blk_insert_cloned_request(rq->q, rq);
+        clone->start_time = jiffies;
+        r = blk_insert_cloned_request(clone->q, clone);
         if (r)
+                /* must complete clone in terms of original request */
                 dm_complete_request(rq, r);
 }
 
@@ -1760,8 +1765,8 @@ static int setup_clone(struct request *clone, struct request *rq,
         return 0;
 }
 
-static struct request *__clone_rq(struct request *rq, struct mapped_device *md,
-                                  struct dm_rq_target_io *tio, gfp_t gfp_mask)
+static struct request *clone_rq(struct request *rq, struct mapped_device *md,
+                                struct dm_rq_target_io *tio, gfp_t gfp_mask)
 {
         struct request *clone = alloc_clone_request(md, gfp_mask);
 
@@ -1780,10 +1785,9 @@ static struct request *__clone_rq(struct request *rq, struct mapped_device *md,
 
 static void map_tio_request(struct kthread_work *work);
 
-static struct request *clone_rq(struct request *rq, struct mapped_device *md,
-                                gfp_t gfp_mask)
+static struct dm_rq_target_io *prep_tio(struct request *rq,
+                                        struct mapped_device *md, gfp_t gfp_mask)
 {
-        struct request *clone;
         struct dm_rq_target_io *tio;
 
         tio = alloc_rq_tio(md, gfp_mask);
@@ -1798,13 +1802,12 @@ static struct request *clone_rq(struct request *rq, struct mapped_device *md,
         memset(&tio->info, 0, sizeof(tio->info));
         init_kthread_work(&tio->work, map_tio_request);
 
-        clone = __clone_rq(rq, md, tio, GFP_ATOMIC);
-        if (!clone) {
+        if (!clone_rq(rq, md, tio, gfp_mask)) {
                 free_rq_tio(tio);
                 return NULL;
         }
 
-        return clone;
+        return tio;
 }
 
 /*
@@ -1813,18 +1816,18 @@ static struct request *clone_rq(struct request *rq, struct mapped_device *md,
 static int dm_prep_fn(struct request_queue *q, struct request *rq)
 {
         struct mapped_device *md = q->queuedata;
-        struct request *clone;
+        struct dm_rq_target_io *tio;
 
         if (unlikely(rq->special)) {
                 DMWARN("Already has something in rq->special.");
                 return BLKPREP_KILL;
         }
 
-        clone = clone_rq(rq, md, GFP_ATOMIC);
-        if (!clone)
+        tio = prep_tio(rq, md, GFP_ATOMIC);
+        if (!tio)
                 return BLKPREP_DEFER;
 
-        rq->special = clone;
+        rq->special = tio;
         rq->cmd_flags |= REQ_DONTPREP;
 
         return BLKPREP_OK;
@@ -1835,11 +1838,12 @@ static int dm_prep_fn(struct request_queue *q, struct request *rq)
  * 0  : the request has been processed (not requeued)
  * !0 : the request has been requeued
  */
-static int map_request(struct dm_target *ti, struct request *clone,
+static int map_request(struct dm_target *ti, struct request *rq,
                        struct mapped_device *md)
 {
         int r, requeued = 0;
-        struct dm_rq_target_io *tio = clone->end_io_data;
+        struct dm_rq_target_io *tio = rq->special;
+        struct request *clone = tio->clone;
 
         r = ti->type->map_rq(ti, clone, &tio->info);
         switch (r) {
@@ -1849,8 +1853,8 @@ static int map_request(struct dm_target *ti, struct request *clone,
         case DM_MAPIO_REMAPPED:
                 /* The target has remapped the I/O so dispatch it */
                 trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),
-                                     blk_rq_pos(tio->orig));
-                dm_dispatch_request(clone);
+                                     blk_rq_pos(rq));
+                dm_dispatch_clone_request(clone, rq);
                 break;
         case DM_MAPIO_REQUEUE:
                 /* The target wants to requeue the I/O */
@@ -1864,7 +1868,7 @@ static int map_request(struct dm_target *ti, struct request *clone,
                 }
 
                 /* The target wants to complete the I/O */
-                dm_kill_unmapped_request(clone, r);
+                dm_kill_unmapped_request(rq, r);
                 break;
         }
 
@@ -1875,16 +1879,13 @@ static void map_tio_request(struct kthread_work *work)
 {
         struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work);
 
-        map_request(tio->ti, tio->clone, tio->md);
+        map_request(tio->ti, tio->orig, tio->md);
 }
 
-static struct request *dm_start_request(struct mapped_device *md, struct request *orig)
+static void dm_start_request(struct mapped_device *md, struct request *orig)
 {
-        struct request *clone;
-
         blk_start_request(orig);
-        clone = orig->special;
-        atomic_inc(&md->pending[rq_data_dir(clone)]);
+        atomic_inc(&md->pending[rq_data_dir(orig)]);
 
         /*
          * Hold the md reference here for the in-flight I/O.
@@ -1894,8 +1895,6 @@ static struct request *dm_start_request(struct mapped_device *md, struct request
          * See the comment in rq_completed() too.
          */
         dm_get(md);
-
-        return clone;
 }
 
 /*
@@ -1908,7 +1907,7 @@ static void dm_request_fn(struct request_queue *q)
         int srcu_idx;
         struct dm_table *map = dm_get_live_table(md, &srcu_idx);
         struct dm_target *ti;
-        struct request *rq, *clone;
+        struct request *rq;
         struct dm_rq_target_io *tio;
         sector_t pos;
 
@@ -1931,19 +1930,19 @@ static void dm_request_fn(struct request_queue *q)
                 ti = dm_table_find_target(map, pos);
                 if (!dm_target_is_valid(ti)) {
                         /*
-                         * Must perform setup, that dm_done() requires,
+                         * Must perform setup, that rq_completed() requires,
                          * before calling dm_kill_unmapped_request
                          */
                         DMERR_LIMIT("request attempted access beyond the end of device");
-                        clone = dm_start_request(md, rq);
-                        dm_kill_unmapped_request(clone, -EIO);
+                        dm_start_request(md, rq);
+                        dm_kill_unmapped_request(rq, -EIO);
                         continue;
                 }
 
                 if (ti->type->busy && ti->type->busy(ti))
                         goto delay_and_out;
 
-                clone = dm_start_request(md, rq);
+                dm_start_request(md, rq);
 
                 tio = rq->special;
                 /* Establish tio->ti before queuing work (map_tio_request) */
@@ -2240,16 +2239,15 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
                         bioset_free(md->bs);
                         md->bs = p->bs;
                         p->bs = NULL;
-                } else if (dm_table_get_type(t) == DM_TYPE_REQUEST_BASED) {
-                        /*
-                         * There's no need to reload with request-based dm
-                         * because the size of front_pad doesn't change.
-                         * Note for future: If you are to reload bioset,
-                         * prep-ed requests in the queue may refer
-                         * to bio from the old bioset, so you must walk
-                         * through the queue to unprep.
-                         */
                 }
+                /*
+                 * There's no need to reload with request-based dm
+                 * because the size of front_pad doesn't change.
+                 * Note for future: If you are to reload bioset,
+                 * prep-ed requests in the queue may refer
+                 * to bio from the old bioset, so you must walk
+                 * through the queue to unprep.
+                 */
                 goto out;
         }