17 files changed, 226 insertions, 89 deletions

diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index b39f6f0b45f2..0f12382aa35d 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -498,7 +498,7 @@ struct cached_dev {
          */
         atomic_t                has_dirty;
 
-        struct ratelimit        writeback_rate;
+        struct bch_ratelimit    writeback_rate;
         struct delayed_work     writeback_rate_update;
 
         /*
@@ -507,10 +507,9 @@ struct cached_dev {
          */
         sector_t                last_read;
 
-        /* Number of writeback bios in flight */
-        atomic_t                in_flight;
+        /* Limit number of writeback bios in flight */
+        struct semaphore        in_flight;
         struct closure_with_timer writeback;
-        struct closure_waitlist writeback_wait;
 
         struct keybuf           writeback_keys;
 
diff --git a/drivers/md/bcache/bset.c b/drivers/md/bcache/bset.c
index 8010eed06a51..22d1ae72c282 100644
--- a/drivers/md/bcache/bset.c
+++ b/drivers/md/bcache/bset.c
@@ -926,28 +926,45 @@ struct bkey *bch_next_recurse_key(struct btree *b, struct bkey *search)
 
 /* Mergesort */
 
+static void sort_key_next(struct btree_iter *iter,
+                          struct btree_iter_set *i)
+{
+        i->k = bkey_next(i->k);
+
+        if (i->k == i->end)
+                *i = iter->data[--iter->used];
+}
+
 static void btree_sort_fixup(struct btree_iter *iter)
 {
         while (iter->used > 1) {
                 struct btree_iter_set *top = iter->data, *i = top + 1;
-                struct bkey *k;
 
                 if (iter->used > 2 &&
                     btree_iter_cmp(i[0], i[1]))
                         i++;
 
-                for (k = i->k;
-                     k != i->end && bkey_cmp(top->k, &START_KEY(k)) > 0;
-                     k = bkey_next(k))
-                        if (top->k > i->k)
-                                __bch_cut_front(top->k, k);
-                        else if (KEY_SIZE(k))
-                                bch_cut_back(&START_KEY(k), top->k);
-
-                if (top->k < i->k || k == i->k)
+                if (bkey_cmp(top->k, &START_KEY(i->k)) <= 0)
                         break;
 
-                heap_sift(iter, i - top, btree_iter_cmp);
+                if (!KEY_SIZE(i->k)) {
+                        sort_key_next(iter, i);
+                        heap_sift(iter, i - top, btree_iter_cmp);
+                        continue;
+                }
+
+                if (top->k > i->k) {
+                        if (bkey_cmp(top->k, i->k) >= 0)
+                                sort_key_next(iter, i);
+                        else
+                                bch_cut_front(top->k, i->k);
+
+                        heap_sift(iter, i - top, btree_iter_cmp);
+                } else {
+                        /* can't happen because of comparison func */
+                        BUG_ON(!bkey_cmp(&START_KEY(top->k), &START_KEY(i->k)));
+                        bch_cut_back(&START_KEY(i->k), top->k);
+                }
         }
 }
 
diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
index f9764e61978b..f42fc7ed9cd6 100644
--- a/drivers/md/bcache/btree.c
+++ b/drivers/md/bcache/btree.c
@@ -255,7 +255,7 @@ void bch_btree_node_read(struct btree *b)
 
         return;
 err:
-        bch_cache_set_error(b->c, "io error reading bucket %lu",
+        bch_cache_set_error(b->c, "io error reading bucket %zu",
                             PTR_BUCKET_NR(b->c, &b->key, 0));
 }
 
@@ -612,7 +612,7 @@ static unsigned long bch_mca_scan(struct shrinker *shrink,
                 return SHRINK_STOP;
 
         /* Return -1 if we can't do anything right now */
-        if (sc->gfp_mask & __GFP_WAIT)
+        if (sc->gfp_mask & __GFP_IO)
                 mutex_lock(&c->bucket_lock);
         else if (!mutex_trylock(&c->bucket_lock))
                 return -1;
diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c
index ba95ab84b2be..8435f81e5d85 100644
--- a/drivers/md/bcache/journal.c
+++ b/drivers/md/bcache/journal.c
@@ -153,7 +153,8 @@ int bch_journal_read(struct cache_set *c, struct list_head *list,
                 bitmap_zero(bitmap, SB_JOURNAL_BUCKETS);
                 pr_debug("%u journal buckets", ca->sb.njournal_buckets);
 
-                /* Read journal buckets ordered by golden ratio hash to quickly
+                /*
+                 * Read journal buckets ordered by golden ratio hash to quickly
                  * find a sequence of buckets with valid journal entries
                  */
                 for (i = 0; i < ca->sb.njournal_buckets; i++) {
@@ -166,18 +167,20 @@ int bch_journal_read(struct cache_set *c, struct list_head *list,
                                 goto bsearch;
                 }
 
-                /* If that fails, check all the buckets we haven't checked
+                /*
+                 * If that fails, check all the buckets we haven't checked
                  * already
                  */
                 pr_debug("falling back to linear search");
 
-                for (l = 0; l < ca->sb.njournal_buckets; l++) {
-                        if (test_bit(l, bitmap))
-                                continue;
-
+                for (l = find_first_zero_bit(bitmap, ca->sb.njournal_buckets);
+                     l < ca->sb.njournal_buckets;
+                     l = find_next_zero_bit(bitmap, ca->sb.njournal_buckets, l + 1))
                         if (read_bucket(l))
                                 goto bsearch;
-                }
+
+                if (list_empty(list))
+                        continue;
 bsearch:
                 /* Binary search */
                 m = r = find_next_bit(bitmap, ca->sb.njournal_buckets, l + 1);
@@ -197,10 +200,12 @@ bsearch:
                                 r = m;
                 }
 
-                /* Read buckets in reverse order until we stop finding more
+                /*
+                 * Read buckets in reverse order until we stop finding more
                  * journal entries
                  */
-                pr_debug("finishing up");
+                pr_debug("finishing up: m %u njournal_buckets %u",
+                         m, ca->sb.njournal_buckets);
                 l = m;
 
                 while (1) {
@@ -228,9 +233,10 @@ bsearch:
                         }
         }
 
-        c->journal.seq = list_entry(list->prev,
-                                    struct journal_replay,
-                                    list)->j.seq;
+        if (!list_empty(list))
+                c->journal.seq = list_entry(list->prev,
+                                            struct journal_replay,
+                                            list)->j.seq;
 
         return 0;
 #undef read_bucket
@@ -428,7 +434,7 @@ static void do_journal_discard(struct cache *ca)
                 return;
         }
 
-        switch (atomic_read(&ja->discard_in_flight) == DISCARD_IN_FLIGHT) {
+        switch (atomic_read(&ja->discard_in_flight)) {
         case DISCARD_IN_FLIGHT:
                 return;
 
@@ -689,6 +695,7 @@ void bch_journal_meta(struct cache_set *c, struct closure *cl)
                 if (cl)
                         BUG_ON(!closure_wait(&w->wait, cl));
 
+                closure_flush(&c->journal.io);
                 __journal_try_write(c, true);
         }
 }
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index 786a1a4f74d8..71eb233b9ace 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -997,14 +997,17 @@ static void request_write(struct cached_dev *dc, struct search *s)
         } else {
                 bch_writeback_add(dc);
 
-                if (s->op.flush_journal) {
+                if (bio->bi_rw & REQ_FLUSH) {
                         /* Also need to send a flush to the backing device */
-                        s->op.cache_bio = bio_clone_bioset(bio, GFP_NOIO,
-                                                           dc->disk.bio_split);
+                        struct bio *flush = bio_alloc_bioset(0, GFP_NOIO,
+                                                             dc->disk.bio_split);
 
-                        bio->bi_size = 0;
-                        bio->bi_vcnt = 0;
-                        closure_bio_submit(bio, cl, s->d);
+                        flush->bi_rw    = WRITE_FLUSH;
+                        flush->bi_bdev  = bio->bi_bdev;
+                        flush->bi_end_io = request_endio;
+                        flush->bi_private = cl;
+
+                        closure_bio_submit(flush, cl, s->d);
                 } else {
                         s->op.cache_bio = bio;
                 }
diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c
index 4fe6ab2fbe2e..924dcfdae111 100644
--- a/drivers/md/bcache/sysfs.c
+++ b/drivers/md/bcache/sysfs.c
@@ -223,8 +223,13 @@ STORE(__cached_dev)
         }
 
         if (attr == &sysfs_label) {
-                /* note: endlines are preserved */
-                memcpy(dc->sb.label, buf, SB_LABEL_SIZE);
+                if (size > SB_LABEL_SIZE)
+                        return -EINVAL;
+                memcpy(dc->sb.label, buf, size);
+                if (size < SB_LABEL_SIZE)
+                        dc->sb.label[size] = '\0';
+                if (size && dc->sb.label[size - 1] == '\n')
+                        dc->sb.label[size - 1] = '\0';
                 bch_write_bdev_super(dc, NULL);
                 if (dc->disk.c) {
                         memcpy(dc->disk.c->uuids[dc->disk.id].label,
diff --git a/drivers/md/bcache/util.c b/drivers/md/bcache/util.c
index 98eb81159a22..420dad545c7d 100644
--- a/drivers/md/bcache/util.c
+++ b/drivers/md/bcache/util.c
@@ -190,7 +190,16 @@ void bch_time_stats_update(struct time_stats *stats, uint64_t start_time)
         stats->last = now ?: 1;
 }
 
-unsigned bch_next_delay(struct ratelimit *d, uint64_t done)
+/**
+ * bch_next_delay() - increment @d by the amount of work done, and return how
+ * long to delay until the next time to do some work.
+ *
+ * @d - the struct bch_ratelimit to update
+ * @done - the amount of work done, in arbitrary units
+ *
+ * Returns the amount of time to delay by, in jiffies
+ */
+uint64_t bch_next_delay(struct bch_ratelimit *d, uint64_t done)
 {
         uint64_t now = local_clock();
 
diff --git a/drivers/md/bcache/util.h b/drivers/md/bcache/util.h
index 1ae2a73ad85f..ea345c6896f4 100644
--- a/drivers/md/bcache/util.h
+++ b/drivers/md/bcache/util.h
@@ -450,17 +450,23 @@ read_attribute(name ## _last_ ## frequency_units)
         (ewma) >> factor;                                               \
 })
 
-struct ratelimit {
+struct bch_ratelimit {
+        /* Next time we want to do some work, in nanoseconds */
         uint64_t                next;
+
+        /*
+         * Rate at which we want to do work, in units per nanosecond
+         * The units here correspond to the units passed to bch_next_delay()
+         */
         unsigned                rate;
 };
 
-static inline void ratelimit_reset(struct ratelimit *d)
+static inline void bch_ratelimit_reset(struct bch_ratelimit *d)
 {
         d->next = local_clock();
 }
 
-unsigned bch_next_delay(struct ratelimit *d, uint64_t done);
+uint64_t bch_next_delay(struct bch_ratelimit *d, uint64_t done);
 
 #define __DIV_SAFE(n, d, zero)                                          \
 ({                                                                      \
diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c
index 22cbff551628..ba3ee48320f2 100644
--- a/drivers/md/bcache/writeback.c
+++ b/drivers/md/bcache/writeback.c
@@ -94,11 +94,15 @@ static void update_writeback_rate(struct work_struct *work)
 
 static unsigned writeback_delay(struct cached_dev *dc, unsigned sectors)
 {
+        uint64_t ret;
+
         if (atomic_read(&dc->disk.detaching) ||
             !dc->writeback_percent)
                 return 0;
 
-        return bch_next_delay(&dc->writeback_rate, sectors * 10000000ULL);
+        ret = bch_next_delay(&dc->writeback_rate, sectors * 10000000ULL);
+
+        return min_t(uint64_t, ret, HZ);
 }
 
 /* Background writeback */
@@ -208,7 +212,7 @@ normal_refill:
 
         up_write(&dc->writeback_lock);
 
-        ratelimit_reset(&dc->writeback_rate);
+        bch_ratelimit_reset(&dc->writeback_rate);
 
         /* Punt to workqueue only so we don't recurse and blow the stack */
         continue_at(cl, read_dirty, dirty_wq);
@@ -318,9 +322,7 @@ static void write_dirty_finish(struct closure *cl)
         }
 
         bch_keybuf_del(&dc->writeback_keys, w);
-        atomic_dec_bug(&dc->in_flight);
-
-        closure_wake_up(&dc->writeback_wait);
+        up(&dc->in_flight);
 
         closure_return_with_destructor(cl, dirty_io_destructor);
 }
@@ -349,7 +351,7 @@ static void write_dirty(struct closure *cl)
 
         closure_bio_submit(&io->bio, cl, &io->dc->disk);
 
-        continue_at(cl, write_dirty_finish, dirty_wq);
+        continue_at(cl, write_dirty_finish, system_wq);
 }
 
 static void read_dirty_endio(struct bio *bio, int error)
@@ -369,7 +371,7 @@ static void read_dirty_submit(struct closure *cl)
 
         closure_bio_submit(&io->bio, cl, &io->dc->disk);
 
-        continue_at(cl, write_dirty, dirty_wq);
+        continue_at(cl, write_dirty, system_wq);
 }
 
 static void read_dirty(struct closure *cl)
@@ -394,12 +396,8 @@ static void read_dirty(struct closure *cl)
 
                 if (delay > 0 &&
                     (KEY_START(&w->key) != dc->last_read ||
-                     jiffies_to_msecs(delay) > 50)) {
-                        w->private = NULL;
-
-                        closure_delay(&dc->writeback, delay);
-                        continue_at(cl, read_dirty, dirty_wq);
-                }
+                     jiffies_to_msecs(delay) > 50))
+                        delay = schedule_timeout_uninterruptible(delay);
 
                 dc->last_read   = KEY_OFFSET(&w->key);
 
@@ -424,15 +422,10 @@ static void read_dirty(struct closure *cl)
 
                 trace_bcache_writeback(&w->key);
 
-                closure_call(&io->cl, read_dirty_submit, NULL, &dc->disk.cl);
+                down(&dc->in_flight);
+                closure_call(&io->cl, read_dirty_submit, NULL, cl);
 
                 delay = writeback_delay(dc, KEY_SIZE(&w->key));
-
-                atomic_inc(&dc->in_flight);
-
-                if (!closure_wait_event(&dc->writeback_wait, cl,
-                                        atomic_read(&dc->in_flight) < 64))
-                        continue_at(cl, read_dirty, dirty_wq);
         }
 
         if (0) {
@@ -442,7 +435,11 @@ err:
                 bch_keybuf_del(&dc->writeback_keys, w);
         }
 
-        refill_dirty(cl);
+        /*
+         * Wait for outstanding writeback IOs to finish (and keybuf slots to be
+         * freed) before refilling again
+         */
+        continue_at(cl, refill_dirty, dirty_wq);
 }
 
 /* Init */
@@ -484,6 +481,7 @@ void bch_sectors_dirty_init(struct cached_dev *dc)
 
 void bch_cached_dev_writeback_init(struct cached_dev *dc)
 {
+        sema_init(&dc->in_flight, 64);
         closure_init_unlocked(&dc->writeback);
         init_rwsem(&dc->writeback_lock);
 
@@ -513,7 +511,7 @@ void bch_writeback_exit(void)
 
 int __init bch_writeback_init(void)
 {
-        dirty_wq = create_singlethread_workqueue("bcache_writeback");
+        dirty_wq = create_workqueue("bcache_writeback");
         if (!dirty_wq)
                 return -ENOMEM;
 
diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
index ea49834377c8..2a20986a2fec 100644
--- a/drivers/md/dm-io.c
+++ b/drivers/md/dm-io.c
@@ -19,8 +19,6 @@
 #define DM_MSG_PREFIX "io"
 
 #define DM_IO_MAX_REGIONS       BITS_PER_LONG
-#define MIN_IOS         16
-#define MIN_BIOS        16
 
 struct dm_io_client {
         mempool_t *pool;
@@ -50,16 +48,17 @@ static struct kmem_cache *_dm_io_cache;
 struct dm_io_client *dm_io_client_create(void)
 {
         struct dm_io_client *client;
+        unsigned min_ios = dm_get_reserved_bio_based_ios();
 
         client = kmalloc(sizeof(*client), GFP_KERNEL);
         if (!client)
                 return ERR_PTR(-ENOMEM);
 
-        client->pool = mempool_create_slab_pool(MIN_IOS, _dm_io_cache);
+        client->pool = mempool_create_slab_pool(min_ios, _dm_io_cache);
         if (!client->pool)
                 goto bad;
 
-        client->bios = bioset_create(MIN_BIOS, 0);
+        client->bios = bioset_create(min_ios, 0);
         if (!client->bios)
                 goto bad;
 
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index b759a127f9c3..de570a558764 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -7,6 +7,7 @@
 
 #include <linux/device-mapper.h>
 
+#include "dm.h"
 #include "dm-path-selector.h"
 #include "dm-uevent.h"
 
@@ -116,8 +117,6 @@ struct dm_mpath_io {
 
 typedef int (*action_fn) (struct pgpath *pgpath);
 
-#define MIN_IOS 256     /* Mempool size */
-
 static struct kmem_cache *_mpio_cache;
 
 static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
@@ -190,6 +189,7 @@ static void free_priority_group(struct priority_group *pg,
 static struct multipath *alloc_multipath(struct dm_target *ti)
 {
         struct multipath *m;
+        unsigned min_ios = dm_get_reserved_rq_based_ios();
 
         m = kzalloc(sizeof(*m), GFP_KERNEL);
         if (m) {
@@ -202,7 +202,7 @@ static struct multipath *alloc_multipath(struct dm_target *ti)
                 INIT_WORK(&m->trigger_event, trigger_event);
                 init_waitqueue_head(&m->pg_init_wait);
                 mutex_init(&m->work_mutex);
-                m->mpio_pool = mempool_create_slab_pool(MIN_IOS, _mpio_cache);
+                m->mpio_pool = mempool_create_slab_pool(min_ios, _mpio_cache);
                 if (!m->mpio_pool) {
                         kfree(m);
                         return NULL;
@@ -1268,6 +1268,7 @@ static int noretry_error(int error)
         case -EREMOTEIO:
         case -EILSEQ:
         case -ENODATA:
+        case -ENOSPC:
                 return 1;
         }
 
@@ -1298,8 +1299,17 @@ static int do_end_io(struct multipath *m, struct request *clone,
         if (!error && !clone->errors)
                 return 0;       /* I/O complete */
 
-        if (noretry_error(error))
+        if (noretry_error(error)) {
+                if ((clone->cmd_flags & REQ_WRITE_SAME) &&
+                    !clone->q->limits.max_write_same_sectors) {
+                        struct queue_limits *limits;
+
+                        /* device doesn't really support WRITE SAME, disable it */
+                        limits = dm_get_queue_limits(dm_table_get_md(m->ti->table));
+                        limits->max_write_same_sectors = 0;
+                }
                 return error;
+        }
 
         if (mpio->pgpath)
                 fail_path(mpio->pgpath);
diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c
index 3ac415675b6c..4caa8e6d59d7 100644
--- a/drivers/md/dm-snap-persistent.c
+++ b/drivers/md/dm-snap-persistent.c
@@ -256,7 +256,7 @@ static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, int rw,
          */
         INIT_WORK_ONSTACK(&req.work, do_metadata);
         queue_work(ps->metadata_wq, &req.work);
-        flush_work(&req.work);
+        flush_workqueue(ps->metadata_wq);
 
         return req.result;
 }
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index c434e5aab2df..aec57d76db5d 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -725,17 +725,16 @@ static int calc_max_buckets(void)
  */
 static int init_hash_tables(struct dm_snapshot *s)
 {
-        sector_t hash_size, cow_dev_size, origin_dev_size, max_buckets;
+        sector_t hash_size, cow_dev_size, max_buckets;
 
         /*
          * Calculate based on the size of the original volume or
          * the COW volume...
          */
         cow_dev_size = get_dev_size(s->cow->bdev);
-        origin_dev_size = get_dev_size(s->origin->bdev);
         max_buckets = calc_max_buckets();
 
-        hash_size = min(origin_dev_size, cow_dev_size) >> s->store->chunk_shift;
+        hash_size = cow_dev_size >> s->store->chunk_shift;
         hash_size = min(hash_size, max_buckets);
 
         if (hash_size < 64)
diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c
index 8ae31e8d3d64..3d404c1371ed 100644
--- a/drivers/md/dm-stats.c
+++ b/drivers/md/dm-stats.c
@@ -451,19 +451,26 @@ static void dm_stat_for_entry(struct dm_stat *s, size_t entry,
         struct dm_stat_percpu *p;
 
         /*
-         * For strict correctness we should use local_irq_disable/enable
+         * For strict correctness we should use local_irq_save/restore
          * instead of preempt_disable/enable.
          *
-         * This is racy if the driver finishes bios from non-interrupt
-         * context as well as from interrupt context or from more different
-         * interrupts.
+         * preempt_disable/enable is racy if the driver finishes bios
+         * from non-interrupt context as well as from interrupt context
+         * or from more different interrupts.
          *
-         * However, the race only results in not counting some events,
-         * so it is acceptable.
+         * On 64-bit architectures the race only results in not counting some
+         * events, so it is acceptable.  On 32-bit architectures the race could
+         * cause the counter going off by 2^32, so we need to do proper locking
+         * there.
          *
          * part_stat_lock()/part_stat_unlock() have this race too.
          */
+#if BITS_PER_LONG == 32
+        unsigned long flags;
+        local_irq_save(flags);
+#else
         preempt_disable();
+#endif
         p = &s->stat_percpu[smp_processor_id()][entry];
 
         if (!end) {
@@ -478,7 +485,11 @@ static void dm_stat_for_entry(struct dm_stat *s, size_t entry,
                 p->ticks[idx] += duration;
         }
 
+#if BITS_PER_LONG == 32
+        local_irq_restore(flags);
+#else
         preempt_enable();
+#endif
 }
 
 static void __dm_stat_bio(struct dm_stat *s, unsigned long bi_rw,
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index ed063427d676..2c0cf511ec23 100644
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -2095,6 +2095,7 @@ static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv)
          * them down to the data device.  The thin device's discard
          * processing will cause mappings to be removed from the btree.
          */
+        ti->discard_zeroes_data_unsupported = true;
         if (pf.discard_enabled && pf.discard_passdown) {
                 ti->num_discard_bios = 1;
 
@@ -2104,7 +2105,6 @@ static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv)
                  * thin devices' discard limits consistent).
                  */
                 ti->discards_supported = true;
-                ti->discard_zeroes_data_unsupported = true;
         }
         ti->private = pt;
 
@@ -2689,8 +2689,16 @@ static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
          * They get transferred to the live pool in bind_control_target()
          * called from pool_preresume().
          */
-        if (!pt->adjusted_pf.discard_enabled)
+        if (!pt->adjusted_pf.discard_enabled) {
+                /*
+                 * Must explicitly disallow stacking discard limits otherwise the
+                 * block layer will stack them if pool's data device has support.
+                 * QUEUE_FLAG_DISCARD wouldn't be set but there is no way for the
+                 * user to see that, so make sure to set all discard limits to 0.
+                 */
+                limits->discard_granularity = 0;
                 return;
+        }
 
         disable_passdown_if_not_supported(pt);
 
@@ -2826,10 +2834,10 @@ static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
         ti->per_bio_data_size = sizeof(struct dm_thin_endio_hook);
 
         /* In case the pool supports discards, pass them on. */
+        ti->discard_zeroes_data_unsupported = true;
         if (tc->pool->pf.discard_enabled) {
                 ti->discards_supported = true;
                 ti->num_discard_bios = 1;
-                ti->discard_zeroes_data_unsupported = true;
                 /* Discard bios must be split on a block boundary */
                 ti->split_discard_bios = true;
         }
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 6a5e9ed2fcc3..b3e26c7d1417 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -211,10 +211,55 @@ struct dm_md_mempools {
         struct bio_set *bs;
 };
 
-#define MIN_IOS 256
+#define RESERVED_BIO_BASED_IOS          16
+#define RESERVED_REQUEST_BASED_IOS      256
+#define RESERVED_MAX_IOS                1024
 static struct kmem_cache *_io_cache;
 static struct kmem_cache *_rq_tio_cache;
 
+/*
+ * Bio-based DM's mempools' reserved IOs set by the user.
+ */
+static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
+
+/*
+ * Request-based DM's mempools' reserved IOs set by the user.
+ */
+static unsigned reserved_rq_based_ios = RESERVED_REQUEST_BASED_IOS;
+
+static unsigned __dm_get_reserved_ios(unsigned *reserved_ios,
+                                      unsigned def, unsigned max)
+{
+        unsigned ios = ACCESS_ONCE(*reserved_ios);
+        unsigned modified_ios = 0;
+
+        if (!ios)
+                modified_ios = def;
+        else if (ios > max)
+                modified_ios = max;
+
+        if (modified_ios) {
+                (void)cmpxchg(reserved_ios, ios, modified_ios);
+                ios = modified_ios;
+        }
+
+        return ios;
+}
+
+unsigned dm_get_reserved_bio_based_ios(void)
+{
+        return __dm_get_reserved_ios(&reserved_bio_based_ios,
+                                     RESERVED_BIO_BASED_IOS, RESERVED_MAX_IOS);
+}
+EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
+
+unsigned dm_get_reserved_rq_based_ios(void)
+{
+        return __dm_get_reserved_ios(&reserved_rq_based_ios,
+                                     RESERVED_REQUEST_BASED_IOS, RESERVED_MAX_IOS);
+}
+EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios);
+
 static int __init local_init(void)
 {
         int r = -ENOMEM;
@@ -2278,6 +2323,17 @@ struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
 }
 
 /*
+ * The queue_limits are only valid as long as you have a reference
+ * count on 'md'.
+ */
+struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
+{
+        BUG_ON(!atomic_read(&md->holders));
+        return &md->queue->limits;
+}
+EXPORT_SYMBOL_GPL(dm_get_queue_limits);
+
+/*
  * Fully initialize a request-based queue (->elevator, ->request_fn, etc).
  */
 static int dm_init_request_based_queue(struct mapped_device *md)
@@ -2862,18 +2918,18 @@ struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, u
 
         if (type == DM_TYPE_BIO_BASED) {
                 cachep = _io_cache;
-                pool_size = 16;
+                pool_size = dm_get_reserved_bio_based_ios();
                 front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
         } else if (type == DM_TYPE_REQUEST_BASED) {
                 cachep = _rq_tio_cache;
-                pool_size = MIN_IOS;
+                pool_size = dm_get_reserved_rq_based_ios();
                 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
                 /* per_bio_data_size is not used. See __bind_mempools(). */
                 WARN_ON(per_bio_data_size != 0);
         } else
                 goto out;
 
-        pools->io_pool = mempool_create_slab_pool(MIN_IOS, cachep);
+        pools->io_pool = mempool_create_slab_pool(pool_size, cachep);
         if (!pools->io_pool)
                 goto out;
 
@@ -2924,6 +2980,13 @@ module_exit(dm_exit);
 
 module_param(major, uint, 0);
 MODULE_PARM_DESC(major, "The major number of the device mapper");
+
+module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
+
+module_param(reserved_rq_based_ios, uint, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(reserved_rq_based_ios, "Reserved IOs in request-based mempools");
+
 MODULE_DESCRIPTION(DM_NAME " driver");
 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm.h b/drivers/md/dm.h
index 5e604cc7b4aa..1d1ad7b7e527 100644
--- a/drivers/md/dm.h
+++ b/drivers/md/dm.h
@@ -184,6 +184,9 @@ void dm_free_md_mempools(struct dm_md_mempools *pools);
 /*
  * Helpers that are used by DM core
  */
+unsigned dm_get_reserved_bio_based_ios(void);
+unsigned dm_get_reserved_rq_based_ios(void);
+
 static inline bool dm_message_test_buffer_overflow(char *result, unsigned maxlen)
 {
         return !maxlen || strlen(result) + 1 >= maxlen;