Merge tag 'dm-3.20-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm

Pull device mapper changes from Mike Snitzer:

 - The most significant change this cycle is request-based DM now
   supports stacking ontop of blk-mq devices.  This blk-mq support
   changes the model request-based DM uses for cloning a request to
   relying on calling blk_get_request() directly from the underlying
   blk-mq device.

   An early consumer of this code is Intel's emerging NVMe hardware;
   thanks to Keith Busch for working on, and pushing for, these changes.

 - A few other small fixes and cleanups across other DM targets.

* tag 'dm-3.20-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm:
  dm: inherit QUEUE_FLAG_SG_GAPS flags from underlying queues
  dm snapshot: remove unnecessary NULL checks before vfree() calls
  dm mpath: simplify failure path of dm_multipath_init()
  dm thin metadata: remove unused dm_pool_get_data_block_size()
  dm ioctl: fix stale comment above dm_get_inactive_table()
  dm crypt: update url in CONFIG_DM_CRYPT help text
  dm bufio: fix time comparison to use time_after_eq()
  dm: use time_in_range() and time_after()
  dm raid: fix a couple integer overflows
  dm table: train hybrid target type detection to select blk-mq if appropriate
  dm: allocate requests in target when stacking on blk-mq devices
  dm: prepare for allocating blk-mq clone requests in target
  dm: submit stacked requests in irq enabled context
  dm: split request structure out from dm_rq_target_io structure
  dm: remove exports for request-based interfaces without external callers

15 files changed, 407 insertions, 193 deletions

diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index c355a226a024..c39644478aa4 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -231,9 +231,8 @@ config DM_CRYPT
           transparently encrypts the data on it. You'll need to activate
           the ciphers you're going to use in the cryptoapi configuration.
 
-          Information on how to use dm-crypt can be found on
-
-          <http://www.saout.de/misc/dm-crypt/>
+          For further information on dm-crypt and userspace tools see:
+          <http://code.google.com/p/cryptsetup/wiki/DMCrypt>
 
           To compile this code as a module, choose M here: the module will
           be called dm-crypt.
diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index c33b49792b87..86dbbc737402 100644
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -11,6 +11,7 @@
 #include <linux/device-mapper.h>
 #include <linux/dm-io.h>
 #include <linux/slab.h>
+#include <linux/jiffies.h>
 #include <linux/vmalloc.h>
 #include <linux/shrinker.h>
 #include <linux/module.h>
@@ -1739,7 +1740,7 @@ static unsigned get_max_age_hz(void)
 
 static bool older_than(struct dm_buffer *b, unsigned long age_hz)
 {
-        return (jiffies - b->last_accessed) >= age_hz;
+        return time_after_eq(jiffies, b->last_accessed + age_hz);
 }
 
 static void __evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz)
diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index e1650539cc2f..7755af351867 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -11,6 +11,7 @@
 
 #include <linux/dm-io.h>
 #include <linux/dm-kcopyd.h>
+#include <linux/jiffies.h>
 #include <linux/init.h>
 #include <linux/mempool.h>
 #include <linux/module.h>
@@ -1562,8 +1563,8 @@ static void process_bio(struct cache *cache, struct prealloc *structs,
 
 static int need_commit_due_to_time(struct cache *cache)
 {
-        return jiffies < cache->last_commit_jiffies ||
-               jiffies > cache->last_commit_jiffies + COMMIT_PERIOD;
+        return !time_in_range(jiffies, cache->last_commit_jiffies,
+                              cache->last_commit_jiffies + COMMIT_PERIOD);
 }
 
 static int commit_if_needed(struct cache *cache)
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index 73f791bb9ea4..c8a18e4ee9dc 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -639,8 +639,8 @@ static int check_name(const char *name)
 
 /*
  * On successful return, the caller must not attempt to acquire
- * _hash_lock without first calling dm_table_put, because dm_table_destroy
- * waits for this dm_table_put and could be called under this lock.
+ * _hash_lock without first calling dm_put_live_table, because dm_table_destroy
+ * waits for this dm_put_live_table and could be called under this lock.
  */
 static struct dm_table *dm_get_inactive_table(struct mapped_device *md, int *srcu_idx)
 {
diff --git a/drivers/md/dm-log-userspace-base.c b/drivers/md/dm-log-userspace-base.c
index b953db6cc229..03177ca0b009 100644
--- a/drivers/md/dm-log-userspace-base.c
+++ b/drivers/md/dm-log-userspace-base.c
@@ -6,6 +6,7 @@
 
 #include <linux/bio.h>
 #include <linux/slab.h>
+#include <linux/jiffies.h>
 #include <linux/dm-dirty-log.h>
 #include <linux/device-mapper.h>
 #include <linux/dm-log-userspace.h>
@@ -829,7 +830,7 @@ static int userspace_is_remote_recovering(struct dm_dirty_log *log,
         int r;
         uint64_t region64 = region;
         struct log_c *lc = log->context;
-        static unsigned long long limit;
+        static unsigned long limit;
         struct {
                 int64_t is_recovering;
                 uint64_t in_sync_hint;
@@ -845,7 +846,7 @@ static int userspace_is_remote_recovering(struct dm_dirty_log *log,
          */
         if (region < lc->in_sync_hint)
                 return 0;
-        else if (jiffies < limit)
+        else if (time_after(limit, jiffies))
                 return 1;
 
         limit = jiffies + (HZ / 4);
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index 7b6b0f0f831a..d376dc87716e 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -11,6 +11,7 @@
 #include "dm-path-selector.h"
 #include "dm-uevent.h"
 
+#include <linux/blkdev.h>
 #include <linux/ctype.h>
 #include <linux/init.h>
 #include <linux/mempool.h>
@@ -378,18 +379,18 @@ static int __must_push_back(struct multipath *m)
 /*
  * Map cloned requests
  */
-static int multipath_map(struct dm_target *ti, struct request *clone,
-                         union map_info *map_context)
+static int __multipath_map(struct dm_target *ti, struct request *clone,
+                           union map_info *map_context,
+                           struct request *rq, struct request **__clone)
 {
         struct multipath *m = (struct multipath *) ti->private;
         int r = DM_MAPIO_REQUEUE;
-        size_t nr_bytes = blk_rq_bytes(clone);
-        unsigned long flags;
+        size_t nr_bytes = clone ? blk_rq_bytes(clone) : blk_rq_bytes(rq);
         struct pgpath *pgpath;
         struct block_device *bdev;
         struct dm_mpath_io *mpio;
 
-        spin_lock_irqsave(&m->lock, flags);
+        spin_lock_irq(&m->lock);
 
         /* Do we need to select a new pgpath? */
         if (!m->current_pgpath ||
@@ -411,25 +412,61 @@ static int multipath_map(struct dm_target *ti, struct request *clone,
                 /* ENOMEM, requeue */
                 goto out_unlock;
 
-        bdev = pgpath->path.dev->bdev;
-        clone->q = bdev_get_queue(bdev);
-        clone->rq_disk = bdev->bd_disk;
-        clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
         mpio = map_context->ptr;
         mpio->pgpath = pgpath;
         mpio->nr_bytes = nr_bytes;
+
+        bdev = pgpath->path.dev->bdev;
+
+        spin_unlock_irq(&m->lock);
+
+        if (clone) {
+                /* Old request-based interface: allocated clone is passed in */
+                clone->q = bdev_get_queue(bdev);
+                clone->rq_disk = bdev->bd_disk;
+                clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
+        } else {
+                /* blk-mq request-based interface */
+                *__clone = blk_get_request(bdev_get_queue(bdev),
+                                           rq_data_dir(rq), GFP_KERNEL);
+                if (IS_ERR(*__clone))
+                        /* ENOMEM, requeue */
+                        return r;
+                (*__clone)->bio = (*__clone)->biotail = NULL;
+                (*__clone)->rq_disk = bdev->bd_disk;
+                (*__clone)->cmd_flags |= REQ_FAILFAST_TRANSPORT;
+        }
+
         if (pgpath->pg->ps.type->start_io)
                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
                                               &pgpath->path,
                                               nr_bytes);
-        r = DM_MAPIO_REMAPPED;
+        return DM_MAPIO_REMAPPED;
 
 out_unlock:
-        spin_unlock_irqrestore(&m->lock, flags);
+        spin_unlock_irq(&m->lock);
 
         return r;
 }
 
+static int multipath_map(struct dm_target *ti, struct request *clone,
+                         union map_info *map_context)
+{
+        return __multipath_map(ti, clone, map_context, NULL, NULL);
+}
+
+static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
+                                   union map_info *map_context,
+                                   struct request **clone)
+{
+        return __multipath_map(ti, NULL, map_context, rq, clone);
+}
+
+static void multipath_release_clone(struct request *clone)
+{
+        blk_put_request(clone);
+}
+
 /*
  * If we run out of usable paths, should we queue I/O or error it?
  */
@@ -1666,11 +1703,13 @@ out:
  *---------------------------------------------------------------*/
 static struct target_type multipath_target = {
         .name = "multipath",
-        .version = {1, 7, 0},
+        .version = {1, 8, 0},
         .module = THIS_MODULE,
         .ctr = multipath_ctr,
         .dtr = multipath_dtr,
         .map_rq = multipath_map,
+        .clone_and_map_rq = multipath_clone_and_map,
+        .release_clone_rq = multipath_release_clone,
         .rq_end_io = multipath_end_io,
         .presuspend = multipath_presuspend,
         .postsuspend = multipath_postsuspend,
@@ -1694,16 +1733,15 @@ static int __init dm_multipath_init(void)
         r = dm_register_target(&multipath_target);
         if (r < 0) {
                 DMERR("register failed %d", r);
-                kmem_cache_destroy(_mpio_cache);
-                return -EINVAL;
+                r = -EINVAL;
+                goto bad_register_target;
         }
 
         kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
         if (!kmultipathd) {
                 DMERR("failed to create workqueue kmpathd");
-                dm_unregister_target(&multipath_target);
-                kmem_cache_destroy(_mpio_cache);
-                return -ENOMEM;
+                r = -ENOMEM;
+                goto bad_alloc_kmultipathd;
         }
 
         /*
@@ -1716,16 +1754,23 @@ static int __init dm_multipath_init(void)
                                                   WQ_MEM_RECLAIM);
         if (!kmpath_handlerd) {
                 DMERR("failed to create workqueue kmpath_handlerd");
-                destroy_workqueue(kmultipathd);
-                dm_unregister_target(&multipath_target);
-                kmem_cache_destroy(_mpio_cache);
-                return -ENOMEM;
+                r = -ENOMEM;
+                goto bad_alloc_kmpath_handlerd;
         }
 
         DMINFO("version %u.%u.%u loaded",
                multipath_target.version[0], multipath_target.version[1],
                multipath_target.version[2]);
 
+        return 0;
+
+bad_alloc_kmpath_handlerd:
+        destroy_workqueue(kmultipathd);
+bad_alloc_kmultipathd:
+        dm_unregister_target(&multipath_target);
+bad_register_target:
+        kmem_cache_destroy(_mpio_cache);
+
         return r;
 }
 
diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c
index 777d9ba2acad..88e4c7f24986 100644
--- a/drivers/md/dm-raid.c
+++ b/drivers/md/dm-raid.c
@@ -1237,7 +1237,7 @@ static int raid_ctr(struct dm_target *ti, unsigned argc, char **argv)
         argv++;
 
         /* Skip over RAID params for now and find out # of devices */
-        if (num_raid_params + 1 > argc) {
+        if (num_raid_params >= argc) {
                 ti->error = "Arguments do not agree with counts given";
                 return -EINVAL;
         }
@@ -1248,6 +1248,12 @@ static int raid_ctr(struct dm_target *ti, unsigned argc, char **argv)
                 return -EINVAL;
         }
 
+        argc -= num_raid_params + 1; /* +1: we already have num_raid_devs */
+        if (argc != (num_raid_devs * 2)) {
+                ti->error = "Supplied RAID devices does not match the count given";
+                return -EINVAL;
+        }
+
         rs = context_alloc(ti, rt, (unsigned)num_raid_devs);
         if (IS_ERR(rs))
                 return PTR_ERR(rs);
@@ -1256,16 +1262,8 @@ static int raid_ctr(struct dm_target *ti, unsigned argc, char **argv)
         if (ret)
                 goto bad;
 
-        ret = -EINVAL;
-
-        argc -= num_raid_params + 1; /* +1: we already have num_raid_devs */
         argv += num_raid_params + 1;
 
-        if (argc != (num_raid_devs * 2)) {
-                ti->error = "Supplied RAID devices does not match the count given";
-                goto bad;
-        }
-
         ret = dev_parms(rs, argv);
         if (ret)
                 goto bad;
diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c
index d6e88178d22c..808b8419bc48 100644
--- a/drivers/md/dm-snap-persistent.c
+++ b/drivers/md/dm-snap-persistent.c
@@ -200,16 +200,11 @@ err_area:
 
 static void free_area(struct pstore *ps)
 {
-        if (ps->area)
-                vfree(ps->area);
+        vfree(ps->area);
         ps->area = NULL;
-
-        if (ps->zero_area)
-                vfree(ps->zero_area);
+        vfree(ps->zero_area);
         ps->zero_area = NULL;
-
-        if (ps->header_area)
-                vfree(ps->header_area);
+        vfree(ps->header_area);
         ps->header_area = NULL;
 }
 
@@ -605,8 +600,7 @@ static void persistent_dtr(struct dm_exception_store *store)
         free_area(ps);
 
         /* Allocated in persistent_read_metadata */
-        if (ps->callbacks)
-                vfree(ps->callbacks);
+        vfree(ps->callbacks);
 
         kfree(ps);
 }
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index 3afae9e062f8..6554d9148927 100644
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -827,10 +827,11 @@ static int dm_table_set_type(struct dm_table *t)
 {
         unsigned i;
         unsigned bio_based = 0, request_based = 0, hybrid = 0;
+        bool use_blk_mq = false;
         struct dm_target *tgt;
         struct dm_dev_internal *dd;
         struct list_head *devices;
-        unsigned live_md_type;
+        unsigned live_md_type = dm_get_md_type(t->md);
 
         for (i = 0; i < t->num_targets; i++) {
                 tgt = t->targets + i;
@@ -854,8 +855,8 @@ static int dm_table_set_type(struct dm_table *t)
                  * Determine the type from the live device.
                  * Default to bio-based if device is new.
                  */
-                live_md_type = dm_get_md_type(t->md);
-                if (live_md_type == DM_TYPE_REQUEST_BASED)
+                if (live_md_type == DM_TYPE_REQUEST_BASED ||
+                    live_md_type == DM_TYPE_MQ_REQUEST_BASED)
                         request_based = 1;
                 else
                         bio_based = 1;
@@ -869,16 +870,6 @@ static int dm_table_set_type(struct dm_table *t)
 
         BUG_ON(!request_based); /* No targets in this table */
 
-        /* Non-request-stackable devices can't be used for request-based dm */
-        devices = dm_table_get_devices(t);
-        list_for_each_entry(dd, devices, list) {
-                if (!blk_queue_stackable(bdev_get_queue(dd->dm_dev->bdev))) {
-                        DMWARN("table load rejected: including"
-                               " non-request-stackable devices");
-                        return -EINVAL;
-                }
-        }
-
         /*
          * Request-based dm supports only tables that have a single target now.
          * To support multiple targets, request splitting support is needed,
@@ -890,7 +881,37 @@ static int dm_table_set_type(struct dm_table *t)
                 return -EINVAL;
         }
 
-        t->type = DM_TYPE_REQUEST_BASED;
+        /* Non-request-stackable devices can't be used for request-based dm */
+        devices = dm_table_get_devices(t);
+        list_for_each_entry(dd, devices, list) {
+                struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev);
+
+                if (!blk_queue_stackable(q)) {
+                        DMERR("table load rejected: including"
+                              " non-request-stackable devices");
+                        return -EINVAL;
+                }
+
+                if (q->mq_ops)
+                        use_blk_mq = true;
+        }
+
+        if (use_blk_mq) {
+                /* verify _all_ devices in the table are blk-mq devices */
+                list_for_each_entry(dd, devices, list)
+                        if (!bdev_get_queue(dd->dm_dev->bdev)->mq_ops) {
+                                DMERR("table load rejected: not all devices"
+                                      " are blk-mq request-stackable");
+                                return -EINVAL;
+                        }
+                t->type = DM_TYPE_MQ_REQUEST_BASED;
+
+        } else if (hybrid && list_empty(devices) && live_md_type != DM_TYPE_NONE) {
+                /* inherit live MD type */
+                t->type = live_md_type;
+
+        } else
+                t->type = DM_TYPE_REQUEST_BASED;
 
         return 0;
 }
@@ -907,7 +928,15 @@ struct target_type *dm_table_get_immutable_target_type(struct dm_table *t)
 
 bool dm_table_request_based(struct dm_table *t)
 {
-        return dm_table_get_type(t) == DM_TYPE_REQUEST_BASED;
+        unsigned table_type = dm_table_get_type(t);
+
+        return (table_type == DM_TYPE_REQUEST_BASED ||
+                table_type == DM_TYPE_MQ_REQUEST_BASED);
+}
+
+bool dm_table_mq_request_based(struct dm_table *t)
+{
+        return dm_table_get_type(t) == DM_TYPE_MQ_REQUEST_BASED;
 }
 
 static int dm_table_alloc_md_mempools(struct dm_table *t)
@@ -1360,6 +1389,14 @@ static int queue_supports_sg_merge(struct dm_target *ti, struct dm_dev *dev,
         return q && !test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags);
 }
 
+static int queue_supports_sg_gaps(struct dm_target *ti, struct dm_dev *dev,
+                                  sector_t start, sector_t len, void *data)
+{
+        struct request_queue *q = bdev_get_queue(dev->bdev);
+
+        return q && !test_bit(QUEUE_FLAG_SG_GAPS, &q->queue_flags);
+}
+
 static bool dm_table_all_devices_attribute(struct dm_table *t,
                                            iterate_devices_callout_fn func)
 {
@@ -1480,6 +1517,11 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
         else
                 queue_flag_set_unlocked(QUEUE_FLAG_NO_SG_MERGE, q);
 
+        if (dm_table_all_devices_attribute(t, queue_supports_sg_gaps))
+                queue_flag_clear_unlocked(QUEUE_FLAG_SG_GAPS, q);
+        else
+                queue_flag_set_unlocked(QUEUE_FLAG_SG_GAPS, q);
+
         dm_table_set_integrity(t);
 
         /*
diff --git a/drivers/md/dm-target.c b/drivers/md/dm-target.c
index 242e3cec397a..925ec1b15e75 100644
--- a/drivers/md/dm-target.c
+++ b/drivers/md/dm-target.c
@@ -137,13 +137,26 @@ static int io_err_map_rq(struct dm_target *ti, struct request *clone,
         return -EIO;
 }
 
+static int io_err_clone_and_map_rq(struct dm_target *ti, struct request *rq,
+                                   union map_info *map_context,
+                                   struct request **clone)
+{
+        return -EIO;
+}
+
+static void io_err_release_clone_rq(struct request *clone)
+{
+}
+
 static struct target_type error_target = {
         .name = "error",
-        .version = {1, 2, 0},
+        .version = {1, 3, 0},
         .ctr  = io_err_ctr,
         .dtr  = io_err_dtr,
         .map  = io_err_map,
         .map_rq = io_err_map_rq,
+        .clone_and_map_rq = io_err_clone_and_map_rq,
+        .release_clone_rq = io_err_release_clone_rq,
 };
 
 int __init dm_target_init(void)
diff --git a/drivers/md/dm-thin-metadata.c b/drivers/md/dm-thin-metadata.c
index 43adbb863f5a..79f694120ddf 100644
--- a/drivers/md/dm-thin-metadata.c
+++ b/drivers/md/dm-thin-metadata.c
@@ -1635,15 +1635,6 @@ int dm_pool_get_metadata_dev_size(struct dm_pool_metadata *pmd,
         return r;
 }
 
-int dm_pool_get_data_block_size(struct dm_pool_metadata *pmd, sector_t *result)
-{
-        down_read(&pmd->root_lock);
-        *result = pmd->data_block_size;
-        up_read(&pmd->root_lock);
-
-        return 0;
-}
-
 int dm_pool_get_data_dev_size(struct dm_pool_metadata *pmd, dm_block_t *result)
 {
         int r = -EINVAL;
diff --git a/drivers/md/dm-thin-metadata.h b/drivers/md/dm-thin-metadata.h
index 921d15ee56a0..fac01a96d303 100644
--- a/drivers/md/dm-thin-metadata.h
+++ b/drivers/md/dm-thin-metadata.h
@@ -182,8 +182,6 @@ int dm_pool_get_free_metadata_block_count(struct dm_pool_metadata *pmd,
 int dm_pool_get_metadata_dev_size(struct dm_pool_metadata *pmd,
                                   dm_block_t *result);
 
-int dm_pool_get_data_block_size(struct dm_pool_metadata *pmd, sector_t *result);
-
 int dm_pool_get_data_dev_size(struct dm_pool_metadata *pmd, dm_block_t *result);
 
 int dm_pool_block_is_used(struct dm_pool_metadata *pmd, dm_block_t b, bool *result);
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index 07705ee181e3..654773cb1eee 100644
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -11,6 +11,7 @@
 #include <linux/device-mapper.h>
 #include <linux/dm-io.h>
 #include <linux/dm-kcopyd.h>
+#include <linux/jiffies.h>
 #include <linux/log2.h>
 #include <linux/list.h>
 #include <linux/rculist.h>
@@ -1700,8 +1701,8 @@ static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell
  */
 static int need_commit_due_to_time(struct pool *pool)
 {
-        return jiffies < pool->last_commit_jiffies ||
-               jiffies > pool->last_commit_jiffies + COMMIT_PERIOD;
+        return !time_in_range(jiffies, pool->last_commit_jiffies,
+                              pool->last_commit_jiffies + COMMIT_PERIOD);
 }
 
 #define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node)
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 68c1b535c52e..ec1444f49de1 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -20,6 +20,7 @@
 #include <linux/hdreg.h>
 #include <linux/delay.h>
 #include <linux/wait.h>
+#include <linux/kthread.h>
 
 #include <trace/events/block.h>
 
@@ -78,7 +79,8 @@ struct dm_io {
 struct dm_rq_target_io {
         struct mapped_device *md;
         struct dm_target *ti;
-        struct request *orig, clone;
+        struct request *orig, *clone;
+        struct kthread_work work;
         int error;
         union map_info info;
 };
@@ -179,6 +181,7 @@ struct mapped_device {
          * io objects are allocated from here.
          */
         mempool_t *io_pool;
+        mempool_t *rq_pool;
 
         struct bio_set *bs;
 
@@ -210,6 +213,9 @@ struct mapped_device {
         unsigned internal_suspend_count;
 
         struct dm_stats stats;
+
+        struct kthread_worker kworker;
+        struct task_struct *kworker_task;
 };
 
 /*
@@ -217,6 +223,7 @@ struct mapped_device {
  */
 struct dm_md_mempools {
         mempool_t *io_pool;
+        mempool_t *rq_pool;
         struct bio_set *bs;
 };
 
@@ -231,6 +238,7 @@ struct table_device {
 #define RESERVED_MAX_IOS                1024
 static struct kmem_cache *_io_cache;
 static struct kmem_cache *_rq_tio_cache;
+static struct kmem_cache *_rq_cache;
 
 /*
  * Bio-based DM's mempools' reserved IOs set by the user.
@@ -288,9 +296,14 @@ static int __init local_init(void)
         if (!_rq_tio_cache)
                 goto out_free_io_cache;
 
+        _rq_cache = kmem_cache_create("dm_clone_request", sizeof(struct request),
+                                      __alignof__(struct request), 0, NULL);
+        if (!_rq_cache)
+                goto out_free_rq_tio_cache;
+
         r = dm_uevent_init();
         if (r)
-                goto out_free_rq_tio_cache;
+                goto out_free_rq_cache;
 
         deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
         if (!deferred_remove_workqueue) {
@@ -312,6 +325,8 @@ out_free_workqueue:
         destroy_workqueue(deferred_remove_workqueue);
 out_uevent_exit:
         dm_uevent_exit();
+out_free_rq_cache:
+        kmem_cache_destroy(_rq_cache);
 out_free_rq_tio_cache:
         kmem_cache_destroy(_rq_tio_cache);
 out_free_io_cache:
@@ -325,6 +340,7 @@ static void local_exit(void)
         flush_scheduled_work();
         destroy_workqueue(deferred_remove_workqueue);
 
+        kmem_cache_destroy(_rq_cache);
         kmem_cache_destroy(_rq_tio_cache);
         kmem_cache_destroy(_io_cache);
         unregister_blkdev(_major, _name);
@@ -577,6 +593,17 @@ static void free_rq_tio(struct dm_rq_target_io *tio)
         mempool_free(tio, tio->md->io_pool);
 }
 
+static struct request *alloc_clone_request(struct mapped_device *md,
+                                           gfp_t gfp_mask)
+{
+        return mempool_alloc(md->rq_pool, gfp_mask);
+}
+
+static void free_clone_request(struct mapped_device *md, struct request *rq)
+{
+        mempool_free(rq, md->rq_pool);
+}
+
 static int md_in_flight(struct mapped_device *md)
 {
         return atomic_read(&md->pending[READ]) +
@@ -992,7 +1019,7 @@ static void end_clone_bio(struct bio *clone, int error)
  * the md may be freed in dm_put() at the end of this function.
  * Or do dm_get() before calling this function and dm_put() later.
  */
-static void rq_completed(struct mapped_device *md, int rw, int run_queue)
+static void rq_completed(struct mapped_device *md, int rw, bool run_queue)
 {
         atomic_dec(&md->pending[rw]);
 
@@ -1020,12 +1047,17 @@ static void free_rq_clone(struct request *clone)
         struct dm_rq_target_io *tio = clone->end_io_data;
 
         blk_rq_unprep_clone(clone);
+        if (clone->q && clone->q->mq_ops)
+                tio->ti->type->release_clone_rq(clone);
+        else
+                free_clone_request(tio->md, clone);
         free_rq_tio(tio);
 }
 
 /*
  * Complete the clone and the original request.
- * Must be called without queue lock.
+ * Must be called without clone's queue lock held,
+ * see end_clone_request() for more details.
  */
 static void dm_end_request(struct request *clone, int error)
 {
@@ -1054,23 +1086,23 @@ static void dm_end_request(struct request *clone, int error)
 
 static void dm_unprep_request(struct request *rq)
 {
-        struct request *clone = rq->special;
+        struct dm_rq_target_io *tio = rq->special;
+        struct request *clone = tio->clone;
 
         rq->special = NULL;
         rq->cmd_flags &= ~REQ_DONTPREP;
 
-        free_rq_clone(clone);
+        if (clone)
+                free_rq_clone(clone);
 }
 
 /*
  * Requeue the original request of a clone.
  */
-void dm_requeue_unmapped_request(struct request *clone)
+static void dm_requeue_unmapped_original_request(struct mapped_device *md,
+                                                 struct request *rq)
 {
-        int rw = rq_data_dir(clone);
-        struct dm_rq_target_io *tio = clone->end_io_data;
-        struct mapped_device *md = tio->md;
-        struct request *rq = tio->orig;
+        int rw = rq_data_dir(rq);
         struct request_queue *q = rq->q;
         unsigned long flags;
 
@@ -1080,9 +1112,15 @@ void dm_requeue_unmapped_request(struct request *clone)
         blk_requeue_request(q, rq);
         spin_unlock_irqrestore(q->queue_lock, flags);
 
-        rq_completed(md, rw, 0);
+        rq_completed(md, rw, false);
+}
+
+static void dm_requeue_unmapped_request(struct request *clone)
+{
+        struct dm_rq_target_io *tio = clone->end_io_data;
+
+        dm_requeue_unmapped_original_request(tio->md, tio->orig);
 }
-EXPORT_SYMBOL_GPL(dm_requeue_unmapped_request);
 
 static void __stop_queue(struct request_queue *q)
 {
@@ -1151,8 +1189,15 @@ static void dm_done(struct request *clone, int error, bool mapped)
 static void dm_softirq_done(struct request *rq)
 {
         bool mapped = true;
-        struct request *clone = rq->completion_data;
-        struct dm_rq_target_io *tio = clone->end_io_data;
+        struct dm_rq_target_io *tio = rq->special;
+        struct request *clone = tio->clone;
+
+        if (!clone) {
+                blk_end_request_all(rq, tio->error);
+                rq_completed(tio->md, rq_data_dir(rq), false);
+                free_rq_tio(tio);
+                return;
+        }
 
         if (rq->cmd_flags & REQ_FAILED)
                 mapped = false;
@@ -1164,13 +1209,11 @@ static void dm_softirq_done(struct request *rq)
  * Complete the clone and the original request with the error status
  * through softirq context.
  */
-static void dm_complete_request(struct request *clone, int error)
+static void dm_complete_request(struct request *rq, int error)
 {
-        struct dm_rq_target_io *tio = clone->end_io_data;
-        struct request *rq = tio->orig;
+        struct dm_rq_target_io *tio = rq->special;
 
         tio->error = error;
-        rq->completion_data = clone;
         blk_complete_request(rq);
 }
 
@@ -1178,40 +1221,40 @@ static void dm_complete_request(struct request *clone, int error)
  * Complete the not-mapped clone and the original request with the error status
  * through softirq context.
  * Target's rq_end_io() function isn't called.
- * This may be used when the target's map_rq() function fails.
+ * This may be used when the target's map_rq() or clone_and_map_rq() functions fail.
  */
-void dm_kill_unmapped_request(struct request *clone, int error)
+static void dm_kill_unmapped_request(struct request *rq, int error)
 {
-        struct dm_rq_target_io *tio = clone->end_io_data;
-        struct request *rq = tio->orig;
-
         rq->cmd_flags |= REQ_FAILED;
-        dm_complete_request(clone, error);
+        dm_complete_request(rq, error);
 }
-EXPORT_SYMBOL_GPL(dm_kill_unmapped_request);
 
 /*
- * Called with the queue lock held
+ * Called with the clone's queue lock held
  */
 static void end_clone_request(struct request *clone, int error)
 {
-        /*
-         * For just cleaning up the information of the queue in which
-         * the clone was dispatched.
-         * The clone is *NOT* freed actually here because it is alloced from
-         * dm own mempool and REQ_ALLOCED isn't set in clone->cmd_flags.
-         */
-        __blk_put_request(clone->q, clone);
+        struct dm_rq_target_io *tio = clone->end_io_data;
+
+        if (!clone->q->mq_ops) {
+                /*
+                 * For just cleaning up the information of the queue in which
+                 * the clone was dispatched.
+                 * The clone is *NOT* freed actually here because it is alloced
+                 * from dm own mempool (REQ_ALLOCED isn't set).
+                 */
+                __blk_put_request(clone->q, clone);
+        }
 
         /*
          * Actual request completion is done in a softirq context which doesn't
-         * hold the queue lock.  Otherwise, deadlock could occur because:
+         * hold the clone's queue lock.  Otherwise, deadlock could occur because:
          *     - another request may be submitted by the upper level driver
          *       of the stacking during the completion
          *     - the submission which requires queue lock may be done
-         *       against this queue
+         *       against this clone's queue
          */
-        dm_complete_request(clone, error);
+        dm_complete_request(tio->orig, error);
 }
 
 /*
@@ -1689,19 +1732,19 @@ static void dm_request(struct request_queue *q, struct bio *bio)
                 _dm_request(q, bio);
 }
 
-void dm_dispatch_request(struct request *rq)
+static void dm_dispatch_clone_request(struct request *clone, struct request *rq)
 {
         int r;
 
-        if (blk_queue_io_stat(rq->q))
-                rq->cmd_flags |= REQ_IO_STAT;
+        if (blk_queue_io_stat(clone->q))
+                clone->cmd_flags |= REQ_IO_STAT;
 
-        rq->start_time = jiffies;
-        r = blk_insert_cloned_request(rq->q, rq);
+        clone->start_time = jiffies;
+        r = blk_insert_cloned_request(clone->q, clone);
         if (r)
+                /* must complete clone in terms of original request */
                 dm_complete_request(rq, r);
 }
-EXPORT_SYMBOL_GPL(dm_dispatch_request);
 
 static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
                                  void *data)
@@ -1718,12 +1761,11 @@ static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
 }
 
 static int setup_clone(struct request *clone, struct request *rq,
-                       struct dm_rq_target_io *tio)
+                       struct dm_rq_target_io *tio, gfp_t gfp_mask)
 {
         int r;
 
-        blk_rq_init(NULL, clone);
-        r = blk_rq_prep_clone(clone, rq, tio->md->bs, GFP_ATOMIC,
+        r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask,
                               dm_rq_bio_constructor, tio);
         if (r)
                 return r;
@@ -1734,14 +1776,37 @@ static int setup_clone(struct request *clone, struct request *rq,
         clone->end_io = end_clone_request;
         clone->end_io_data = tio;
 
+        tio->clone = clone;
+
         return 0;
 }
 
 static struct request *clone_rq(struct request *rq, struct mapped_device *md,
-                                gfp_t gfp_mask)
+                                struct dm_rq_target_io *tio, gfp_t gfp_mask)
+{
+        struct request *clone = alloc_clone_request(md, gfp_mask);
+
+        if (!clone)
+                return NULL;
+
+        blk_rq_init(NULL, clone);
+        if (setup_clone(clone, rq, tio, gfp_mask)) {
+                /* -ENOMEM */
+                free_clone_request(md, clone);
+                return NULL;
+        }
+
+        return clone;
+}
+
+static void map_tio_request(struct kthread_work *work);
+
+static struct dm_rq_target_io *prep_tio(struct request *rq,
+                                        struct mapped_device *md, gfp_t gfp_mask)
 {
-        struct request *clone;
         struct dm_rq_target_io *tio;
+        int srcu_idx;
+        struct dm_table *table;
 
         tio = alloc_rq_tio(md, gfp_mask);
         if (!tio)
@@ -1749,18 +1814,23 @@ static struct request *clone_rq(struct request *rq, struct mapped_device *md,
 
         tio->md = md;
         tio->ti = NULL;
+        tio->clone = NULL;
         tio->orig = rq;
         tio->error = 0;
         memset(&tio->info, 0, sizeof(tio->info));
-
-        clone = &tio->clone;
-        if (setup_clone(clone, rq, tio)) {
-                /* -ENOMEM */
-                free_rq_tio(tio);
-                return NULL;
+        init_kthread_work(&tio->work, map_tio_request);
+
+        table = dm_get_live_table(md, &srcu_idx);
+        if (!dm_table_mq_request_based(table)) {
+                if (!clone_rq(rq, md, tio, gfp_mask)) {
+                        dm_put_live_table(md, srcu_idx);
+                        free_rq_tio(tio);
+                        return NULL;
+                }
         }
+        dm_put_live_table(md, srcu_idx);
 
-        return clone;
+        return tio;
 }
 
 /*
@@ -1769,18 +1839,18 @@ static struct request *clone_rq(struct request *rq, struct mapped_device *md,
 static int dm_prep_fn(struct request_queue *q, struct request *rq)
 {
         struct mapped_device *md = q->queuedata;
-        struct request *clone;
+        struct dm_rq_target_io *tio;
 
         if (unlikely(rq->special)) {
                 DMWARN("Already has something in rq->special.");
                 return BLKPREP_KILL;
         }
 
-        clone = clone_rq(rq, md, GFP_ATOMIC);
-        if (!clone)
+        tio = prep_tio(rq, md, GFP_ATOMIC);
+        if (!tio)
                 return BLKPREP_DEFER;
 
-        rq->special = clone;
+        rq->special = tio;
         rq->cmd_flags |= REQ_DONTPREP;
 
         return BLKPREP_OK;
@@ -1788,17 +1858,36 @@ static int dm_prep_fn(struct request_queue *q, struct request *rq)
 
 /*
  * Returns:
- * 0  : the request has been processed (not requeued)
- * !0 : the request has been requeued
+ * 0                : the request has been processed
+ * DM_MAPIO_REQUEUE : the original request needs to be requeued
+ * < 0              : the request was completed due to failure
  */
-static int map_request(struct dm_target *ti, struct request *clone,
+static int map_request(struct dm_target *ti, struct request *rq,
                        struct mapped_device *md)
 {
-        int r, requeued = 0;
-        struct dm_rq_target_io *tio = clone->end_io_data;
+        int r;
+        struct dm_rq_target_io *tio = rq->special;
+        struct request *clone = NULL;
+
+        if (tio->clone) {
+                clone = tio->clone;
+                r = ti->type->map_rq(ti, clone, &tio->info);
+        } else {
+                r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);
+                if (r < 0) {
+                        /* The target wants to complete the I/O */
+                        dm_kill_unmapped_request(rq, r);
+                        return r;
+                }
+                if (IS_ERR(clone))
+                        return DM_MAPIO_REQUEUE;
+                if (setup_clone(clone, rq, tio, GFP_KERNEL)) {
+                        /* -ENOMEM */
+                        ti->type->release_clone_rq(clone);
+                        return DM_MAPIO_REQUEUE;
+                }
+        }
 
-        tio->ti = ti;
-        r = ti->type->map_rq(ti, clone, &tio->info);
         switch (r) {
         case DM_MAPIO_SUBMITTED:
                 /* The target has taken the I/O to submit by itself later */
@@ -1806,13 +1895,12 @@ static int map_request(struct dm_target *ti, struct request *clone,
         case DM_MAPIO_REMAPPED:
                 /* The target has remapped the I/O so dispatch it */
                 trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),
-                                     blk_rq_pos(tio->orig));
-                dm_dispatch_request(clone);
+                                     blk_rq_pos(rq));
+                dm_dispatch_clone_request(clone, rq);
                 break;
         case DM_MAPIO_REQUEUE:
                 /* The target wants to requeue the I/O */
                 dm_requeue_unmapped_request(clone);
-                requeued = 1;
                 break;
         default:
                 if (r > 0) {
@@ -1821,20 +1909,27 @@ static int map_request(struct dm_target *ti, struct request *clone,
                 }
 
                 /* The target wants to complete the I/O */
-                dm_kill_unmapped_request(clone, r);
-                break;
+                dm_kill_unmapped_request(rq, r);
+                return r;
         }
 
-        return requeued;
+        return 0;
 }
 
-static struct request *dm_start_request(struct mapped_device *md, struct request *orig)
+static void map_tio_request(struct kthread_work *work)
 {
-        struct request *clone;
+        struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work);
+        struct request *rq = tio->orig;
+        struct mapped_device *md = tio->md;
 
+        if (map_request(tio->ti, rq, md) == DM_MAPIO_REQUEUE)
+                dm_requeue_unmapped_original_request(md, rq);
+}
+
+static void dm_start_request(struct mapped_device *md, struct request *orig)
+{
         blk_start_request(orig);
-        clone = orig->special;
-        atomic_inc(&md->pending[rq_data_dir(clone)]);
+        atomic_inc(&md->pending[rq_data_dir(orig)]);
 
         /*
          * Hold the md reference here for the in-flight I/O.
@@ -1844,8 +1939,6 @@ static struct request *dm_start_request(struct mapped_device *md, struct request
          * See the comment in rq_completed() too.
          */
         dm_get(md);
-
-        return clone;
 }
 
 /*
@@ -1858,7 +1951,8 @@ static void dm_request_fn(struct request_queue *q)
         int srcu_idx;
         struct dm_table *map = dm_get_live_table(md, &srcu_idx);
         struct dm_target *ti;
-        struct request *rq, *clone;
+        struct request *rq;
+        struct dm_rq_target_io *tio;
         sector_t pos;
 
         /*
@@ -1880,34 +1974,29 @@ static void dm_request_fn(struct request_queue *q)
                 ti = dm_table_find_target(map, pos);
                 if (!dm_target_is_valid(ti)) {
                         /*
-                         * Must perform setup, that dm_done() requires,
+                         * Must perform setup, that rq_completed() requires,
                          * before calling dm_kill_unmapped_request
                          */
                         DMERR_LIMIT("request attempted access beyond the end of device");
-                        clone = dm_start_request(md, rq);
-                        dm_kill_unmapped_request(clone, -EIO);
+                        dm_start_request(md, rq);
+                        dm_kill_unmapped_request(rq, -EIO);
                         continue;
                 }
 
                 if (ti->type->busy && ti->type->busy(ti))
                         goto delay_and_out;
 
-                clone = dm_start_request(md, rq);
-
-                spin_unlock(q->queue_lock);
-                if (map_request(ti, clone, md))
-                        goto requeued;
+                dm_start_request(md, rq);
 
+                tio = rq->special;
+                /* Establish tio->ti before queuing work (map_tio_request) */
+                tio->ti = ti;
+                queue_kthread_work(&md->kworker, &tio->work);
                 BUG_ON(!irqs_disabled());
-                spin_lock(q->queue_lock);
         }
 
         goto out;
 
-requeued:
-        BUG_ON(!irqs_disabled());
-        spin_lock(q->queue_lock);
-
 delay_and_out:
         blk_delay_queue(q, HZ / 10);
 out:
@@ -2093,6 +2182,7 @@ static struct mapped_device *alloc_dev(int minor)
         INIT_WORK(&md->work, dm_wq_work);
         init_waitqueue_head(&md->eventq);
         init_completion(&md->kobj_holder.completion);
+        md->kworker_task = NULL;
 
         md->disk->major = _major;
         md->disk->first_minor = minor;
@@ -2153,8 +2243,13 @@ static void free_dev(struct mapped_device *md)
         unlock_fs(md);
         bdput(md->bdev);
         destroy_workqueue(md->wq);
+
+        if (md->kworker_task)
+                kthread_stop(md->kworker_task);
         if (md->io_pool)
                 mempool_destroy(md->io_pool);
+        if (md->rq_pool)
+                mempool_destroy(md->rq_pool);
         if (md->bs)
                 bioset_free(md->bs);
         blk_integrity_unregister(md->disk);
@@ -2188,23 +2283,24 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
                         bioset_free(md->bs);
                         md->bs = p->bs;
                         p->bs = NULL;
-                } else if (dm_table_get_type(t) == DM_TYPE_REQUEST_BASED) {
-                        /*
-                         * There's no need to reload with request-based dm
-                         * because the size of front_pad doesn't change.
-                         * Note for future: If you are to reload bioset,
-                         * prep-ed requests in the queue may refer
-                         * to bio from the old bioset, so you must walk
-                         * through the queue to unprep.
-                         */
                 }
+                /*
+                 * There's no need to reload with request-based dm
+                 * because the size of front_pad doesn't change.
+                 * Note for future: If you are to reload bioset,
+                 * prep-ed requests in the queue may refer
+                 * to bio from the old bioset, so you must walk
+                 * through the queue to unprep.
+                 */
                 goto out;
         }
 
-        BUG_ON(!p || md->io_pool || md->bs);
+        BUG_ON(!p || md->io_pool || md->rq_pool || md->bs);
 
         md->io_pool = p->io_pool;
         p->io_pool = NULL;
+        md->rq_pool = p->rq_pool;
+        p->rq_pool = NULL;
         md->bs = p->bs;
         p->bs = NULL;
 
@@ -2407,6 +2503,14 @@ unsigned dm_get_md_type(struct mapped_device *md)
         return md->type;
 }
 
+static bool dm_md_type_request_based(struct mapped_device *md)
+{
+        unsigned table_type = dm_get_md_type(md);
+
+        return (table_type == DM_TYPE_REQUEST_BASED ||
+                table_type == DM_TYPE_MQ_REQUEST_BASED);
+}
+
 struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
 {
         return md->immutable_target_type;
@@ -2444,6 +2548,11 @@ static int dm_init_request_based_queue(struct mapped_device *md)
         blk_queue_prep_rq(md->queue, dm_prep_fn);
         blk_queue_lld_busy(md->queue, dm_lld_busy);
 
+        /* Also initialize the request-based DM worker thread */
+        init_kthread_worker(&md->kworker);
+        md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker,
+                                       "kdmwork-%s", dm_device_name(md));
+
         elv_register_queue(md->queue);
 
         return 1;
@@ -2454,8 +2563,7 @@ static int dm_init_request_based_queue(struct mapped_device *md)
  */
 int dm_setup_md_queue(struct mapped_device *md)
 {
-        if ((dm_get_md_type(md) == DM_TYPE_REQUEST_BASED) &&
-            !dm_init_request_based_queue(md)) {
+        if (dm_md_type_request_based(md) && !dm_init_request_based_queue(md)) {
                 DMWARN("Cannot initialize queue for request-based mapped device");
                 return -EINVAL;
         }
@@ -2534,6 +2642,9 @@ static void __dm_destroy(struct mapped_device *md, bool wait)
         set_bit(DMF_FREEING, &md->flags);
         spin_unlock(&_minor_lock);
 
+        if (dm_request_based(md))
+                flush_kthread_worker(&md->kworker);
+
         if (!dm_suspended_md(md)) {
                 dm_table_presuspend_targets(map);
                 dm_table_postsuspend_targets(map);
@@ -2777,8 +2888,10 @@ static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
          * Stop md->queue before flushing md->wq in case request-based
          * dm defers requests to md->wq from md->queue.
          */
-        if (dm_request_based(md))
+        if (dm_request_based(md)) {
                 stop_queue(md->queue);
+                flush_kthread_worker(&md->kworker);
+        }
 
         flush_workqueue(md->wq);
 
@@ -3124,24 +3237,35 @@ struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, u
 {
         struct dm_md_mempools *pools = kzalloc(sizeof(*pools), GFP_KERNEL);
         struct kmem_cache *cachep;
-        unsigned int pool_size;
+        unsigned int pool_size = 0;
         unsigned int front_pad;
 
         if (!pools)
                 return NULL;
 
-        if (type == DM_TYPE_BIO_BASED) {
+        switch (type) {
+        case DM_TYPE_BIO_BASED:
                 cachep = _io_cache;
                 pool_size = dm_get_reserved_bio_based_ios();
                 front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
-        } else if (type == DM_TYPE_REQUEST_BASED) {
-                cachep = _rq_tio_cache;
+                break;
+        case DM_TYPE_REQUEST_BASED:
                 pool_size = dm_get_reserved_rq_based_ios();
+                pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache);
+                if (!pools->rq_pool)
+                        goto out;
+                /* fall through to setup remaining rq-based pools */
+        case DM_TYPE_MQ_REQUEST_BASED:
+                cachep = _rq_tio_cache;
+                if (!pool_size)
+                        pool_size = dm_get_reserved_rq_based_ios();
                 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
                 /* per_bio_data_size is not used. See __bind_mempools(). */
                 WARN_ON(per_bio_data_size != 0);
-        } else
+                break;
+        default:
                 goto out;
+        }
 
         pools->io_pool = mempool_create_slab_pool(pool_size, cachep);
         if (!pools->io_pool)
@@ -3170,6 +3294,9 @@ void dm_free_md_mempools(struct dm_md_mempools *pools)
         if (pools->io_pool)
                 mempool_destroy(pools->io_pool);
 
+        if (pools->rq_pool)
+                mempool_destroy(pools->rq_pool);
+
         if (pools->bs)
                 bioset_free(pools->bs);
 
diff --git a/drivers/md/dm.h b/drivers/md/dm.h
index 84b0f9e4ba6c..59f53e79db82 100644
--- a/drivers/md/dm.h
+++ b/drivers/md/dm.h
@@ -34,9 +34,10 @@
 /*
  * Type of table and mapped_device's mempool
  */
-#define DM_TYPE_NONE            0
-#define DM_TYPE_BIO_BASED       1
-#define DM_TYPE_REQUEST_BASED   2
+#define DM_TYPE_NONE                    0
+#define DM_TYPE_BIO_BASED               1
+#define DM_TYPE_REQUEST_BASED           2
+#define DM_TYPE_MQ_REQUEST_BASED        3
 
 /*
  * List of devices that a metadevice uses and should open/close.
@@ -73,6 +74,7 @@ int dm_table_any_busy_target(struct dm_table *t);
 unsigned dm_table_get_type(struct dm_table *t);
 struct target_type *dm_table_get_immutable_target_type(struct dm_table *t);
 bool dm_table_request_based(struct dm_table *t);
+bool dm_table_mq_request_based(struct dm_table *t);
 void dm_table_free_md_mempools(struct dm_table *t);
 struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t);
 
@@ -99,7 +101,8 @@ int dm_setup_md_queue(struct mapped_device *md);
 /*
  * To check whether the target type is request-based or not (bio-based).
  */
-#define dm_target_request_based(t) ((t)->type->map_rq != NULL)
+#define dm_target_request_based(t) (((t)->type->map_rq != NULL) || \
+                                    ((t)->type->clone_and_map_rq != NULL))
 
 /*
  * To check whether the target type is a hybrid (capable of being