7 files changed, 143 insertions, 75 deletions

diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index 1bf4a71919ec..5f054c44b485 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -2178,6 +2178,8 @@ static int cache_create(struct cache_args *ca, struct cache **result)
         ti->num_discard_bios = 1;
         ti->discards_supported = true;
         ti->discard_zeroes_data_unsupported = true;
+        /* Discard bios must be split on a block boundary */
+        ti->split_discard_bios = true;
 
         cache->features = ca->features;
         ti->per_bio_data_size = get_per_bio_data_size(cache);
@@ -2488,6 +2490,7 @@ static int cache_map(struct dm_target *ti, struct bio *bio)
 
                 } else {
                         inc_hit_counter(cache, bio);
+                        pb->all_io_entry = dm_deferred_entry_inc(cache->all_io_ds);
 
                         if (bio_data_dir(bio) == WRITE && writethrough_mode(&cache->features) &&
                             !is_dirty(cache, lookup_result.cblock))
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 784695d22fde..53b213226c01 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -19,7 +19,6 @@
 #include <linux/crypto.h>
 #include <linux/workqueue.h>
 #include <linux/backing-dev.h>
-#include <linux/percpu.h>
 #include <linux/atomic.h>
 #include <linux/scatterlist.h>
 #include <asm/page.h>
@@ -43,6 +42,7 @@ struct convert_context {
         struct bvec_iter iter_out;
         sector_t cc_sector;
         atomic_t cc_pending;
+        struct ablkcipher_request *req;
 };
 
 /*
@@ -111,15 +111,7 @@ struct iv_tcw_private {
 enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID };
 
 /*
- * Duplicated per-CPU state for cipher.
- */
-struct crypt_cpu {
-        struct ablkcipher_request *req;
-};
-
-/*
- * The fields in here must be read only after initialization,
- * changing state should be in crypt_cpu.
+ * The fields in here must be read only after initialization.
  */
 struct crypt_config {
         struct dm_dev *dev;
@@ -150,12 +142,6 @@ struct crypt_config {
         sector_t iv_offset;
         unsigned int iv_size;
 
-        /*
-         * Duplicated per cpu state. Access through
-         * per_cpu_ptr() only.
-         */
-        struct crypt_cpu __percpu *cpu;
-
         /* ESSIV: struct crypto_cipher *essiv_tfm */
         void *iv_private;
         struct crypto_ablkcipher **tfms;
@@ -192,11 +178,6 @@ static void clone_init(struct dm_crypt_io *, struct bio *);
 static void kcryptd_queue_crypt(struct dm_crypt_io *io);
 static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq);
 
-static struct crypt_cpu *this_crypt_config(struct crypt_config *cc)
-{
-        return this_cpu_ptr(cc->cpu);
-}
-
 /*
  * Use this to access cipher attributes that are the same for each CPU.
  */
@@ -903,16 +884,15 @@ static void kcryptd_async_done(struct crypto_async_request *async_req,
 static void crypt_alloc_req(struct crypt_config *cc,
                             struct convert_context *ctx)
 {
-        struct crypt_cpu *this_cc = this_crypt_config(cc);
         unsigned key_index = ctx->cc_sector & (cc->tfms_count - 1);
 
-        if (!this_cc->req)
-                this_cc->req = mempool_alloc(cc->req_pool, GFP_NOIO);
+        if (!ctx->req)
+                ctx->req = mempool_alloc(cc->req_pool, GFP_NOIO);
 
-        ablkcipher_request_set_tfm(this_cc->req, cc->tfms[key_index]);
-        ablkcipher_request_set_callback(this_cc->req,
+        ablkcipher_request_set_tfm(ctx->req, cc->tfms[key_index]);
+        ablkcipher_request_set_callback(ctx->req,
             CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
-            kcryptd_async_done, dmreq_of_req(cc, this_cc->req));
+            kcryptd_async_done, dmreq_of_req(cc, ctx->req));
 }
 
 /*
@@ -921,7 +901,6 @@ static void crypt_alloc_req(struct crypt_config *cc,
 static int crypt_convert(struct crypt_config *cc,
                          struct convert_context *ctx)
 {
-        struct crypt_cpu *this_cc = this_crypt_config(cc);
         int r;
 
         atomic_set(&ctx->cc_pending, 1);
@@ -932,7 +911,7 @@ static int crypt_convert(struct crypt_config *cc,
 
                 atomic_inc(&ctx->cc_pending);
 
-                r = crypt_convert_block(cc, ctx, this_cc->req);
+                r = crypt_convert_block(cc, ctx, ctx->req);
 
                 switch (r) {
                 /* async */
@@ -941,7 +920,7 @@ static int crypt_convert(struct crypt_config *cc,
                         reinit_completion(&ctx->restart);
                         /* fall through*/
                 case -EINPROGRESS:
-                        this_cc->req = NULL;
+                        ctx->req = NULL;
                         ctx->cc_sector++;
                         continue;
 
@@ -1040,6 +1019,7 @@ static struct dm_crypt_io *crypt_io_alloc(struct crypt_config *cc,
         io->sector = sector;
         io->error = 0;
         io->base_io = NULL;
+        io->ctx.req = NULL;
         atomic_set(&io->io_pending, 0);
 
         return io;
@@ -1065,6 +1045,8 @@ static void crypt_dec_pending(struct dm_crypt_io *io)
         if (!atomic_dec_and_test(&io->io_pending))
                 return;
 
+        if (io->ctx.req)
+                mempool_free(io->ctx.req, cc->req_pool);
         mempool_free(io, cc->io_pool);
 
         if (likely(!base_io))
@@ -1492,8 +1474,6 @@ static int crypt_wipe_key(struct crypt_config *cc)
 static void crypt_dtr(struct dm_target *ti)
 {
         struct crypt_config *cc = ti->private;
-        struct crypt_cpu *cpu_cc;
-        int cpu;
 
         ti->private = NULL;
 
@@ -1505,13 +1485,6 @@ static void crypt_dtr(struct dm_target *ti)
         if (cc->crypt_queue)
                 destroy_workqueue(cc->crypt_queue);
 
-        if (cc->cpu)
-                for_each_possible_cpu(cpu) {
-                        cpu_cc = per_cpu_ptr(cc->cpu, cpu);
-                        if (cpu_cc->req)
-                                mempool_free(cpu_cc->req, cc->req_pool);
-                }
-
         crypt_free_tfms(cc);
 
         if (cc->bs)
@@ -1530,9 +1503,6 @@ static void crypt_dtr(struct dm_target *ti)
         if (cc->dev)
                 dm_put_device(ti, cc->dev);
 
-        if (cc->cpu)
-                free_percpu(cc->cpu);
-
         kzfree(cc->cipher);
         kzfree(cc->cipher_string);
 
@@ -1588,13 +1558,6 @@ static int crypt_ctr_cipher(struct dm_target *ti,
         if (tmp)
                 DMWARN("Ignoring unexpected additional cipher options");
 
-        cc->cpu = __alloc_percpu(sizeof(*(cc->cpu)),
-                                 __alignof__(struct crypt_cpu));
-        if (!cc->cpu) {
-                ti->error = "Cannot allocate per cpu state";
-                goto bad_mem;
-        }
-
         /*
          * For compatibility with the original dm-crypt mapping format, if
          * only the cipher name is supplied, use cbc-plain.
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index aa009e865871..ebfa411d1a7d 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -445,11 +445,11 @@ static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path,
         else
                 m->saved_queue_if_no_path = queue_if_no_path;
         m->queue_if_no_path = queue_if_no_path;
-        if (!m->queue_if_no_path)
-                dm_table_run_md_queue_async(m->ti->table);
-
         spin_unlock_irqrestore(&m->lock, flags);
 
+        if (!queue_if_no_path)
+                dm_table_run_md_queue_async(m->ti->table);
+
         return 0;
 }
 
@@ -954,7 +954,7 @@ out:
  */
 static int reinstate_path(struct pgpath *pgpath)
 {
-        int r = 0;
+        int r = 0, run_queue = 0;
         unsigned long flags;
         struct multipath *m = pgpath->pg->m;
 
@@ -978,7 +978,7 @@ static int reinstate_path(struct pgpath *pgpath)
 
         if (!m->nr_valid_paths++) {
                 m->current_pgpath = NULL;
-                dm_table_run_md_queue_async(m->ti->table);
+                run_queue = 1;
         } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
                 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
                         m->pg_init_in_progress++;
@@ -991,6 +991,8 @@ static int reinstate_path(struct pgpath *pgpath)
 
 out:
         spin_unlock_irqrestore(&m->lock, flags);
+        if (run_queue)
+                dm_table_run_md_queue_async(m->ti->table);
 
         return r;
 }
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index 53728be84dee..242ac2ea5f29 100644
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -27,6 +27,9 @@
 #define MAPPING_POOL_SIZE 1024
 #define PRISON_CELLS 1024
 #define COMMIT_PERIOD HZ
+#define NO_SPACE_TIMEOUT_SECS 60
+
+static unsigned no_space_timeout_secs = NO_SPACE_TIMEOUT_SECS;
 
 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle,
                 "A percentage of time allocated for copy on write");
@@ -175,6 +178,7 @@ struct pool {
         struct workqueue_struct *wq;
         struct work_struct worker;
         struct delayed_work waker;
+        struct delayed_work no_space_timeout;
 
         unsigned long last_commit_jiffies;
         unsigned ref_count;
@@ -232,6 +236,13 @@ struct thin_c {
         struct bio_list deferred_bio_list;
         struct bio_list retry_on_resume_list;
         struct rb_root sort_bio_list; /* sorted list of deferred bios */
+
+        /*
+         * Ensures the thin is not destroyed until the worker has finished
+         * iterating the active_thins list.
+         */
+        atomic_t refcount;
+        struct completion can_destroy;
 };
 
 /*----------------------------------------------------------------*/
@@ -928,7 +939,7 @@ static int commit(struct pool *pool)
 {
         int r;
 
-        if (get_pool_mode(pool) != PM_WRITE)
+        if (get_pool_mode(pool) >= PM_READ_ONLY)
                 return -EINVAL;
 
         r = dm_pool_commit_metadata(pool->pmd);
@@ -1486,6 +1497,45 @@ static void process_thin_deferred_bios(struct thin_c *tc)
         blk_finish_plug(&plug);
 }
 
+static void thin_get(struct thin_c *tc);
+static void thin_put(struct thin_c *tc);
+
+/*
+ * We can't hold rcu_read_lock() around code that can block.  So we
+ * find a thin with the rcu lock held; bump a refcount; then drop
+ * the lock.
+ */
+static struct thin_c *get_first_thin(struct pool *pool)
+{
+        struct thin_c *tc = NULL;
+
+        rcu_read_lock();
+        if (!list_empty(&pool->active_thins)) {
+                tc = list_entry_rcu(pool->active_thins.next, struct thin_c, list);
+                thin_get(tc);
+        }
+        rcu_read_unlock();
+
+        return tc;
+}
+
+static struct thin_c *get_next_thin(struct pool *pool, struct thin_c *tc)
+{
+        struct thin_c *old_tc = tc;
+
+        rcu_read_lock();
+        list_for_each_entry_continue_rcu(tc, &pool->active_thins, list) {
+                thin_get(tc);
+                thin_put(old_tc);
+                rcu_read_unlock();
+                return tc;
+        }
+        thin_put(old_tc);
+        rcu_read_unlock();
+
+        return NULL;
+}
+
 static void process_deferred_bios(struct pool *pool)
 {
         unsigned long flags;
@@ -1493,10 +1543,11 @@ static void process_deferred_bios(struct pool *pool)
         struct bio_list bios;
         struct thin_c *tc;
 
-        rcu_read_lock();
-        list_for_each_entry_rcu(tc, &pool->active_thins, list)
+        tc = get_first_thin(pool);
+        while (tc) {
                 process_thin_deferred_bios(tc);
-        rcu_read_unlock();
+                tc = get_next_thin(pool, tc);
+        }
 
         /*
          * If there are any deferred flush bios, we must commit
@@ -1543,6 +1594,20 @@ static void do_waker(struct work_struct *ws)
         queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD);
 }
 
+/*
+ * We're holding onto IO to allow userland time to react.  After the
+ * timeout either the pool will have been resized (and thus back in
+ * PM_WRITE mode), or we degrade to PM_READ_ONLY and start erroring IO.
+ */
+static void do_no_space_timeout(struct work_struct *ws)
+{
+        struct pool *pool = container_of(to_delayed_work(ws), struct pool,
+                                         no_space_timeout);
+
+        if (get_pool_mode(pool) == PM_OUT_OF_DATA_SPACE && !pool->pf.error_if_no_space)
+                set_pool_mode(pool, PM_READ_ONLY);
+}
+
 /*----------------------------------------------------------------*/
 
 struct noflush_work {
@@ -1578,7 +1643,7 @@ static void noflush_work(struct thin_c *tc, void (*fn)(struct work_struct *))
 {
         struct noflush_work w;
 
-        INIT_WORK(&w.worker, fn);
+        INIT_WORK_ONSTACK(&w.worker, fn);
         w.tc = tc;
         atomic_set(&w.complete, 0);
         init_waitqueue_head(&w.wait);
@@ -1607,6 +1672,7 @@ static void set_pool_mode(struct pool *pool, enum pool_mode new_mode)
         struct pool_c *pt = pool->ti->private;
         bool needs_check = dm_pool_metadata_needs_check(pool->pmd);
         enum pool_mode old_mode = get_pool_mode(pool);
+        unsigned long no_space_timeout = ACCESS_ONCE(no_space_timeout_secs) * HZ;
 
         /*
          * Never allow the pool to transition to PM_WRITE mode if user
@@ -1668,6 +1734,9 @@ static void set_pool_mode(struct pool *pool, enum pool_mode new_mode)
                 pool->process_discard = process_discard;
                 pool->process_prepared_mapping = process_prepared_mapping;
                 pool->process_prepared_discard = process_prepared_discard_passdown;
+
+                if (!pool->pf.error_if_no_space && no_space_timeout)
+                        queue_delayed_work(pool->wq, &pool->no_space_timeout, no_space_timeout);
                 break;
 
         case PM_WRITE:
@@ -2053,6 +2122,7 @@ static struct pool *pool_create(struct mapped_device *pool_md,
 
         INIT_WORK(&pool->worker, do_worker);
         INIT_DELAYED_WORK(&pool->waker, do_waker);
+        INIT_DELAYED_WORK(&pool->no_space_timeout, do_no_space_timeout);
         spin_lock_init(&pool->lock);
         bio_list_init(&pool->deferred_flush_bios);
         INIT_LIST_HEAD(&pool->prepared_mappings);
@@ -2615,6 +2685,7 @@ static void pool_postsuspend(struct dm_target *ti)
         struct pool *pool = pt->pool;
 
         cancel_delayed_work(&pool->waker);
+        cancel_delayed_work(&pool->no_space_timeout);
         flush_workqueue(pool->wq);
         (void) commit(pool);
 }
@@ -3061,11 +3132,25 @@ static struct target_type pool_target = {
 /*----------------------------------------------------------------
  * Thin target methods
  *--------------------------------------------------------------*/
+static void thin_get(struct thin_c *tc)
+{
+        atomic_inc(&tc->refcount);
+}
+
+static void thin_put(struct thin_c *tc)
+{
+        if (atomic_dec_and_test(&tc->refcount))
+                complete(&tc->can_destroy);
+}
+
 static void thin_dtr(struct dm_target *ti)
 {
         struct thin_c *tc = ti->private;
         unsigned long flags;
 
+        thin_put(tc);
+        wait_for_completion(&tc->can_destroy);
+
         spin_lock_irqsave(&tc->pool->lock, flags);
         list_del_rcu(&tc->list);
         spin_unlock_irqrestore(&tc->pool->lock, flags);
@@ -3101,6 +3186,7 @@ static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
         struct thin_c *tc;
         struct dm_dev *pool_dev, *origin_dev;
         struct mapped_device *pool_md;
+        unsigned long flags;
 
         mutex_lock(&dm_thin_pool_table.mutex);
 
@@ -3191,9 +3277,12 @@ static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
 
         mutex_unlock(&dm_thin_pool_table.mutex);
 
-        spin_lock(&tc->pool->lock);
+        atomic_set(&tc->refcount, 1);
+        init_completion(&tc->can_destroy);
+
+        spin_lock_irqsave(&tc->pool->lock, flags);
         list_add_tail_rcu(&tc->list, &tc->pool->active_thins);
-        spin_unlock(&tc->pool->lock);
+        spin_unlock_irqrestore(&tc->pool->lock, flags);
         /*
          * This synchronize_rcu() call is needed here otherwise we risk a
          * wake_worker() call finding no bios to process (because the newly
@@ -3422,6 +3511,9 @@ static void dm_thin_exit(void)
 module_init(dm_thin_init);
 module_exit(dm_thin_exit);
 
+module_param_named(no_space_timeout, no_space_timeout_secs, uint, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(no_space_timeout, "Out of data space queue IO timeout in seconds");
+
 MODULE_DESCRIPTION(DM_NAME " thin provisioning target");
 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-verity.c b/drivers/md/dm-verity.c
index 796007a5e0e1..7a7bab8947ae 100644
--- a/drivers/md/dm-verity.c
+++ b/drivers/md/dm-verity.c
@@ -330,15 +330,17 @@ test_block_hash:
                                 return r;
                         }
                 }
-
                 todo = 1 << v->data_dev_block_bits;
-                while (io->iter.bi_size) {
+                do {
                         u8 *page;
+                        unsigned len;
                         struct bio_vec bv = bio_iter_iovec(bio, io->iter);
 
                         page = kmap_atomic(bv.bv_page);
-                        r = crypto_shash_update(desc, page + bv.bv_offset,
-                                                bv.bv_len);
+                        len = bv.bv_len;
+                        if (likely(len >= todo))
+                                len = todo;
+                        r = crypto_shash_update(desc, page + bv.bv_offset, len);
                         kunmap_atomic(page);
 
                         if (r < 0) {
@@ -346,8 +348,9 @@ test_block_hash:
                                 return r;
                         }
 
-                        bio_advance_iter(bio, &io->iter, bv.bv_len);
-                }
+                        bio_advance_iter(bio, &io->iter, len);
+                        todo -= len;
+                } while (todo);
 
                 if (!v->version) {
                         r = crypto_shash_update(desc, v->salt, v->salt_size);
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 8fda38d23e38..2382cfc9bb3f 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -7381,8 +7381,10 @@ void md_do_sync(struct md_thread *thread)
         /* just incase thread restarts... */
         if (test_bit(MD_RECOVERY_DONE, &mddev->recovery))
                 return;
-        if (mddev->ro) /* never try to sync a read-only array */
+        if (mddev->ro) {/* never try to sync a read-only array */
+                set_bit(MD_RECOVERY_INTR, &mddev->recovery);
                 return;
+        }
 
         if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
                 if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) {
@@ -7824,6 +7826,7 @@ void md_check_recovery(struct mddev *mddev)
                         /* There is no thread, but we need to call
                          * ->spare_active and clear saved_raid_disk
                          */
+                        set_bit(MD_RECOVERY_INTR, &mddev->recovery);
                         md_reap_sync_thread(mddev);
                         clear_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
                         goto unlock;
@@ -8516,7 +8519,8 @@ static int md_notify_reboot(struct notifier_block *this,
                 if (mddev_trylock(mddev)) {
                         if (mddev->pers)
                                 __md_stop_writes(mddev);
-                        mddev->safemode = 2;
+                        if (mddev->persistent)
+                                mddev->safemode = 2;
                         mddev_unlock(mddev);
                 }
                 need_delay = 1;
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 33fc408e5eac..cb882aae9e20 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -1172,6 +1172,13 @@ static void __make_request(struct mddev *mddev, struct bio *bio)
         int max_sectors;
         int sectors;
 
+        /*
+         * Register the new request and wait if the reconstruction
+         * thread has put up a bar for new requests.
+         * Continue immediately if no resync is active currently.
+         */
+        wait_barrier(conf);
+
         sectors = bio_sectors(bio);
         while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
             bio->bi_iter.bi_sector < conf->reshape_progress &&
@@ -1552,12 +1559,6 @@ static void make_request(struct mddev *mddev, struct bio *bio)
 
         md_write_start(mddev, bio);
 
-        /*
-         * Register the new request and wait if the reconstruction
-         * thread has put up a bar for new requests.
-         * Continue immediately if no resync is active currently.
-         */
-        wait_barrier(conf);
 
         do {