1 files changed, 102 insertions, 68 deletions

diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 4a5849b55197..0eb5416798bd 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -36,7 +36,6 @@ struct dm_crypt_io {
         struct work_struct work;
         atomic_t pending;
         int error;
-        int post_process;
 };
 
 /*
@@ -57,7 +56,7 @@ struct crypt_config;
 
 struct crypt_iv_operations {
         int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
-                   const char *opts);
+                   const char *opts);
         void (*dtr)(struct crypt_config *cc);
         const char *(*status)(struct crypt_config *cc);
         int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector);
@@ -80,6 +79,8 @@ struct crypt_config {
         mempool_t *page_pool;
         struct bio_set *bs;
 
+        struct workqueue_struct *io_queue;
+        struct workqueue_struct *crypt_queue;
         /*
          * crypto related data
          */
@@ -137,7 +138,7 @@ static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
 }
 
 static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
-                              const char *opts)
+                              const char *opts)
 {
         struct crypto_cipher *essiv_tfm;
         struct crypto_hash *hash_tfm;
@@ -175,6 +176,7 @@ static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
 
         if (err) {
                 ti->error = "Error calculating hash in ESSIV";
+                kfree(salt);
                 return err;
         }
 
@@ -188,7 +190,7 @@ static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
         if (crypto_cipher_blocksize(essiv_tfm) !=
             crypto_blkcipher_ivsize(cc->tfm)) {
                 ti->error = "Block size of ESSIV cipher does "
-                                "not match IV size of block cipher";
+                            "not match IV size of block cipher";
                 crypto_free_cipher(essiv_tfm);
                 kfree(salt);
                 return -EINVAL;
@@ -319,10 +321,10 @@ crypt_convert_scatterlist(struct crypt_config *cc, struct scatterlist *out,
         return r;
 }
 
-static void
-crypt_convert_init(struct crypt_config *cc, struct convert_context *ctx,
-                   struct bio *bio_out, struct bio *bio_in,
-                   sector_t sector, int write)
+static void crypt_convert_init(struct crypt_config *cc,
+                               struct convert_context *ctx,
+                               struct bio *bio_out, struct bio *bio_in,
+                               sector_t sector, int write)
 {
         ctx->bio_in = bio_in;
         ctx->bio_out = bio_out;
@@ -338,7 +340,7 @@ crypt_convert_init(struct crypt_config *cc, struct convert_context *ctx,
  * Encrypt / decrypt data from one bio to another one (can be the same one)
  */
 static int crypt_convert(struct crypt_config *cc,
-                         struct convert_context *ctx)
+                         struct convert_context *ctx)
 {
         int r = 0;
 
@@ -370,7 +372,7 @@ static int crypt_convert(struct crypt_config *cc,
                 }
 
                 r = crypt_convert_scatterlist(cc, &sg_out, &sg_in, sg_in.length,
-                                              ctx->write, ctx->sector);
+                                              ctx->write, ctx->sector);
                 if (r < 0)
                         break;
 
@@ -380,13 +382,13 @@ static int crypt_convert(struct crypt_config *cc,
         return r;
 }
 
- static void dm_crypt_bio_destructor(struct bio *bio)
- {
+static void dm_crypt_bio_destructor(struct bio *bio)
+{
         struct dm_crypt_io *io = bio->bi_private;
         struct crypt_config *cc = io->target->private;
 
         bio_free(bio, cc->bs);
- }
+}
 
 /*
  * Generate a new unfragmented bio with the given size
@@ -458,7 +460,7 @@ static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
  * One of the bios was finished. Check for completion of
  * the whole request and correctly clean up the buffer.
  */
-static void dec_pending(struct dm_crypt_io *io, int error)
+static void crypt_dec_pending(struct dm_crypt_io *io, int error)
 {
         struct crypt_config *cc = (struct crypt_config *) io->target->private;
 
@@ -474,18 +476,36 @@ static void dec_pending(struct dm_crypt_io *io, int error)
 }
 
 /*
- * kcryptd:
+ * kcryptd/kcryptd_io:
  *
  * Needed because it would be very unwise to do decryption in an
  * interrupt context.
+ *
+ * kcryptd performs the actual encryption or decryption.
+ *
+ * kcryptd_io performs the IO submission.
+ *
+ * They must be separated as otherwise the final stages could be
+ * starved by new requests which can block in the first stages due
+ * to memory allocation.
  */
-static struct workqueue_struct *_kcryptd_workqueue;
 static void kcryptd_do_work(struct work_struct *work);
+static void kcryptd_do_crypt(struct work_struct *work);
 
 static void kcryptd_queue_io(struct dm_crypt_io *io)
 {
+        struct crypt_config *cc = io->target->private;
+
         INIT_WORK(&io->work, kcryptd_do_work);
-        queue_work(_kcryptd_workqueue, &io->work);
+        queue_work(cc->io_queue, &io->work);
+}
+
+static void kcryptd_queue_crypt(struct dm_crypt_io *io)
+{
+        struct crypt_config *cc = io->target->private;
+
+        INIT_WORK(&io->work, kcryptd_do_crypt);
+        queue_work(cc->crypt_queue, &io->work);
 }
 
 static void crypt_endio(struct bio *clone, int error)
@@ -508,13 +528,12 @@ static void crypt_endio(struct bio *clone, int error)
         }
 
         bio_put(clone);
-        io->post_process = 1;
-        kcryptd_queue_io(io);
+        kcryptd_queue_crypt(io);
         return;
 
 out:
         bio_put(clone);
-        dec_pending(io, error);
+        crypt_dec_pending(io, error);
 }
 
 static void clone_init(struct dm_crypt_io *io, struct bio *clone)
@@ -544,7 +563,7 @@ static void process_read(struct dm_crypt_io *io)
          */
         clone = bio_alloc_bioset(GFP_NOIO, bio_segments(base_bio), cc->bs);
         if (unlikely(!clone)) {
-                dec_pending(io, -ENOMEM);
+                crypt_dec_pending(io, -ENOMEM);
                 return;
         }
 
@@ -579,7 +598,7 @@ static void process_write(struct dm_crypt_io *io)
         while (remaining) {
                 clone = crypt_alloc_buffer(io, remaining);
                 if (unlikely(!clone)) {
-                        dec_pending(io, -ENOMEM);
+                        crypt_dec_pending(io, -ENOMEM);
                         return;
                 }
 
@@ -589,7 +608,7 @@ static void process_write(struct dm_crypt_io *io)
                 if (unlikely(crypt_convert(cc, &ctx) < 0)) {
                         crypt_free_buffer_pages(cc, clone);
                         bio_put(clone);
-                        dec_pending(io, -EIO);
+                        crypt_dec_pending(io, -EIO);
                         return;
                 }
 
@@ -624,17 +643,23 @@ static void process_read_endio(struct dm_crypt_io *io)
         crypt_convert_init(cc, &ctx, io->base_bio, io->base_bio,
                            io->base_bio->bi_sector - io->target->begin, 0);
 
-        dec_pending(io, crypt_convert(cc, &ctx));
+        crypt_dec_pending(io, crypt_convert(cc, &ctx));
 }
 
 static void kcryptd_do_work(struct work_struct *work)
 {
         struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
 
-        if (io->post_process)
-                process_read_endio(io);
-        else if (bio_data_dir(io->base_bio) == READ)
+        if (bio_data_dir(io->base_bio) == READ)
                 process_read(io);
+}
+
+static void kcryptd_do_crypt(struct work_struct *work)
+{
+        struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
+
+        if (bio_data_dir(io->base_bio) == READ)
+                process_read_endio(io);
         else
                 process_write(io);
 }
@@ -690,7 +715,7 @@ static int crypt_set_key(struct crypt_config *cc, char *key)
         cc->key_size = key_size; /* initial settings */
 
         if ((!key_size && strcmp(key, "-")) ||
-            (key_size && crypt_decode_key(cc->key, key, key_size) < 0))
+           (key_size && crypt_decode_key(cc->key, key, key_size) < 0))
                 return -EINVAL;
 
         set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
@@ -746,7 +771,7 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 
         if (crypt_set_key(cc, argv[1])) {
                 ti->error = "Error decoding key";
-                goto bad1;
+                goto bad_cipher;
         }
 
         /* Compatiblity mode for old dm-crypt cipher strings */
@@ -757,19 +782,19 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 
         if (strcmp(chainmode, "ecb") && !ivmode) {
                 ti->error = "This chaining mode requires an IV mechanism";
-                goto bad1;
+                goto bad_cipher;
         }
 
-        if (snprintf(cc->cipher, CRYPTO_MAX_ALG_NAME, "%s(%s)", chainmode, 
-                     cipher) >= CRYPTO_MAX_ALG_NAME) {
+        if (snprintf(cc->cipher, CRYPTO_MAX_ALG_NAME, "%s(%s)",
+                     chainmode, cipher) >= CRYPTO_MAX_ALG_NAME) {
                 ti->error = "Chain mode + cipher name is too long";
-                goto bad1;
+                goto bad_cipher;
         }
 
         tfm = crypto_alloc_blkcipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
         if (IS_ERR(tfm)) {
                 ti->error = "Error allocating crypto tfm";
-                goto bad1;
+                goto bad_cipher;
         }
 
         strcpy(cc->cipher, cipher);
@@ -793,18 +818,18 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
                 cc->iv_gen_ops = &crypt_iv_null_ops;
         else {
                 ti->error = "Invalid IV mode";
-                goto bad2;
+                goto bad_ivmode;
         }
 
         if (cc->iv_gen_ops && cc->iv_gen_ops->ctr &&
             cc->iv_gen_ops->ctr(cc, ti, ivopts) < 0)
-                goto bad2;
+                goto bad_ivmode;
 
         cc->iv_size = crypto_blkcipher_ivsize(tfm);
         if (cc->iv_size)
                 /* at least a 64 bit sector number should fit in our buffer */
                 cc->iv_size = max(cc->iv_size,
-                                  (unsigned int)(sizeof(u64) / sizeof(u8)));
+                                  (unsigned int)(sizeof(u64) / sizeof(u8)));
         else {
                 if (cc->iv_gen_ops) {
                         DMWARN("Selected cipher does not support IVs");
@@ -817,13 +842,13 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
         cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool);
         if (!cc->io_pool) {
                 ti->error = "Cannot allocate crypt io mempool";
-                goto bad3;
+                goto bad_slab_pool;
         }
 
         cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0);
         if (!cc->page_pool) {
                 ti->error = "Cannot allocate page mempool";
-                goto bad4;
+                goto bad_page_pool;
         }
 
         cc->bs = bioset_create(MIN_IOS, MIN_IOS);
@@ -834,25 +859,25 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 
         if (crypto_blkcipher_setkey(tfm, cc->key, key_size) < 0) {
                 ti->error = "Error setting key";
-                goto bad5;
+                goto bad_device;
         }
 
         if (sscanf(argv[2], "%llu", &tmpll) != 1) {
                 ti->error = "Invalid iv_offset sector";
-                goto bad5;
+                goto bad_device;
         }
         cc->iv_offset = tmpll;
 
         if (sscanf(argv[4], "%llu", &tmpll) != 1) {
                 ti->error = "Invalid device sector";
-                goto bad5;
+                goto bad_device;
         }
         cc->start = tmpll;
 
         if (dm_get_device(ti, argv[3], cc->start, ti->len,
-                          dm_table_get_mode(ti->table), &cc->dev)) {
+                          dm_table_get_mode(ti->table), &cc->dev)) {
                 ti->error = "Device lookup failed";
-                goto bad5;
+                goto bad_device;
         }
 
         if (ivmode && cc->iv_gen_ops) {
@@ -861,27 +886,45 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
                 cc->iv_mode = kmalloc(strlen(ivmode) + 1, GFP_KERNEL);
                 if (!cc->iv_mode) {
                         ti->error = "Error kmallocing iv_mode string";
-                        goto bad5;
+                        goto bad_ivmode_string;
                 }
                 strcpy(cc->iv_mode, ivmode);
         } else
                 cc->iv_mode = NULL;
 
+        cc->io_queue = create_singlethread_workqueue("kcryptd_io");
+        if (!cc->io_queue) {
+                ti->error = "Couldn't create kcryptd io queue";
+                goto bad_io_queue;
+        }
+
+        cc->crypt_queue = create_singlethread_workqueue("kcryptd");
+        if (!cc->crypt_queue) {
+                ti->error = "Couldn't create kcryptd queue";
+                goto bad_crypt_queue;
+        }
+
         ti->private = cc;
         return 0;
 
-bad5:
+bad_crypt_queue:
+        destroy_workqueue(cc->io_queue);
+bad_io_queue:
+        kfree(cc->iv_mode);
+bad_ivmode_string:
+        dm_put_device(ti, cc->dev);
+bad_device:
         bioset_free(cc->bs);
 bad_bs:
         mempool_destroy(cc->page_pool);
-bad4:
+bad_page_pool:
         mempool_destroy(cc->io_pool);
-bad3:
+bad_slab_pool:
         if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
                 cc->iv_gen_ops->dtr(cc);
-bad2:
+bad_ivmode:
         crypto_free_blkcipher(tfm);
-bad1:
+bad_cipher:
         /* Must zero key material before freeing */
         memset(cc, 0, sizeof(*cc) + cc->key_size * sizeof(u8));
         kfree(cc);
@@ -892,7 +935,8 @@ static void crypt_dtr(struct dm_target *ti)
 {
         struct crypt_config *cc = (struct crypt_config *) ti->private;
 
-        flush_workqueue(_kcryptd_workqueue);
+        destroy_workqueue(cc->io_queue);
+        destroy_workqueue(cc->crypt_queue);
 
         bioset_free(cc->bs);
         mempool_destroy(cc->page_pool);
@@ -918,9 +962,13 @@ static int crypt_map(struct dm_target *ti, struct bio *bio,
         io = mempool_alloc(cc->io_pool, GFP_NOIO);
         io->target = ti;
         io->base_bio = bio;
-        io->error = io->post_process = 0;
+        io->error = 0;
         atomic_set(&io->pending, 0);
-        kcryptd_queue_io(io);
+
+        if (bio_data_dir(io->base_bio) == READ)
+                kcryptd_queue_io(io);
+        else
+                kcryptd_queue_crypt(io);
 
         return DM_MAPIO_SUBMITTED;
 }
@@ -1037,25 +1085,12 @@ static int __init dm_crypt_init(void)
         if (!_crypt_io_pool)
                 return -ENOMEM;
 
-        _kcryptd_workqueue = create_workqueue("kcryptd");
-        if (!_kcryptd_workqueue) {
-                r = -ENOMEM;
-                DMERR("couldn't create kcryptd");
-                goto bad1;
-        }
-
         r = dm_register_target(&crypt_target);
         if (r < 0) {
                 DMERR("register failed %d", r);
-                goto bad2;
+                kmem_cache_destroy(_crypt_io_pool);
         }
 
-        return 0;
-
-bad2:
-        destroy_workqueue(_kcryptd_workqueue);
-bad1:
-        kmem_cache_destroy(_crypt_io_pool);
         return r;
 }
 
@@ -1066,7 +1101,6 @@ static void __exit dm_crypt_exit(void)
         if (r < 0)
                 DMERR("unregister failed %d", r);
 
-        destroy_workqueue(_kcryptd_workqueue);
         kmem_cache_destroy(_crypt_io_pool);
 }