dm mpath: fix missing call of path selector type->end_io

After commit 396eaf21ee17 ("blk-mq: improve DM's blk-mq IO merging via
blk_insert_cloned_request feedback"), map_request() will requeue the tio
when issued clone request return BLK_STS_RESOURCE or BLK_STS_DEV_RESOURCE.

Thus, if device driver status is error, a tio may be requeued multiple
times until the return value is not DM_MAPIO_REQUEUE.  That means
type->start_io may be called multiple times, while type->end_io is only
called when IO complete.

In fact, even without commit 396eaf21ee17, setup_clone() failure can
also cause tio requeue and associated missed call to type->end_io.

The service-time path selector selects path based on in_flight_size,
which is increased by st_start_io() and decreased by st_end_io().
Missed calls to st_end_io() can lead to in_flight_size count error and
will cause the selector to make the wrong choice.  In addition,
queue-length path selector will also be affected.

To fix the problem, call type->end_io in ->release_clone_rq before tio
requeue.  map_info is passed to ->release_clone_rq() for map_request()
error path that result in requeue.

Fixes: 396eaf21ee17 ("blk-mq: improve DM's blk-mq IO merging via blk_insert_cloned_request feedback")
Cc: stable@vger.kernl.org
Signed-off-by: Yufen Yu <yuyufen@huawei.com>
Signed-off-by: Mike Snitzer <snitzer@redhat.com>

4 files changed, 24 insertions, 7 deletions

diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index 2ee5e357a0a7..07cfef67c0ba 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -544,8 +544,23 @@ static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
         return DM_MAPIO_REMAPPED;
 }
 
-static void multipath_release_clone(struct request *clone)
+static void multipath_release_clone(struct request *clone,
+                                    union map_info *map_context)
 {
+        if (unlikely(map_context)) {
+                /*
+                 * non-NULL map_context means caller is still map
+                 * method; must undo multipath_clone_and_map()
+                 */
+                struct dm_mpath_io *mpio = get_mpio(map_context);
+                struct pgpath *pgpath = mpio->pgpath;
+
+                if (pgpath && pgpath->pg->ps.type->end_io)
+                        pgpath->pg->ps.type->end_io(&pgpath->pg->ps,
+                                                    &pgpath->path,
+                                                    mpio->nr_bytes);
+        }
+
         blk_put_request(clone);
 }
 
diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c
index b66745bd08bb..5f7063f05ae0 100644
--- a/drivers/md/dm-rq.c
+++ b/drivers/md/dm-rq.c
@@ -168,7 +168,7 @@ static void dm_end_request(struct request *clone, blk_status_t error)
         struct request *rq = tio->orig;
 
         blk_rq_unprep_clone(clone);
-        tio->ti->type->release_clone_rq(clone);
+        tio->ti->type->release_clone_rq(clone, NULL);
 
         rq_end_stats(md, rq);
         blk_mq_end_request(rq, error);
@@ -201,7 +201,7 @@ static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_
         rq_end_stats(md, rq);
         if (tio->clone) {
                 blk_rq_unprep_clone(tio->clone);
-                tio->ti->type->release_clone_rq(tio->clone);
+                tio->ti->type->release_clone_rq(tio->clone, NULL);
         }
 
         dm_mq_delay_requeue_request(rq, delay_ms);
@@ -398,7 +398,7 @@ static int map_request(struct dm_rq_target_io *tio)
         case DM_MAPIO_REMAPPED:
                 if (setup_clone(clone, rq, tio, GFP_ATOMIC)) {
                         /* -ENOMEM */
-                        ti->type->release_clone_rq(clone);
+                        ti->type->release_clone_rq(clone, &tio->info);
                         return DM_MAPIO_REQUEUE;
                 }
 
@@ -408,7 +408,7 @@ static int map_request(struct dm_rq_target_io *tio)
                 ret = dm_dispatch_clone_request(clone, rq);
                 if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) {
                         blk_rq_unprep_clone(clone);
-                        tio->ti->type->release_clone_rq(clone);
+                        tio->ti->type->release_clone_rq(clone, &tio->info);
                         tio->clone = NULL;
                         return DM_MAPIO_REQUEUE;
                 }
diff --git a/drivers/md/dm-target.c b/drivers/md/dm-target.c
index 314d17ca6466..64dd0b34fcf4 100644
--- a/drivers/md/dm-target.c
+++ b/drivers/md/dm-target.c
@@ -136,7 +136,8 @@ static int io_err_clone_and_map_rq(struct dm_target *ti, struct request *rq,
         return DM_MAPIO_KILL;
 }
 
-static void io_err_release_clone_rq(struct request *clone)
+static void io_err_release_clone_rq(struct request *clone,
+                                    union map_info *map_context)
 {
 }
 
diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h
index b0672756d056..e1f51d607cc5 100644
--- a/include/linux/device-mapper.h
+++ b/include/linux/device-mapper.h
@@ -62,7 +62,8 @@ typedef int (*dm_clone_and_map_request_fn) (struct dm_target *ti,
                                             struct request *rq,
                                             union map_info *map_context,
                                             struct request **clone);
-typedef void (*dm_release_clone_request_fn) (struct request *clone);
+typedef void (*dm_release_clone_request_fn) (struct request *clone,
+                                             union map_info *map_context);
 
 /*
  * Returns: