block: Convert integrity to bvec_alloc_bs()

This adds a pointer to the bvec array to struct bio_integrity_payload,
instead of the bvecs always being inline; then the bvecs are allocated
with bvec_alloc_bs().

Changed bvec_alloc_bs() and bvec_free_bs() to take a pointer to a
mempool instead of the bioset, so that bio integrity can use a different
mempool for its bvecs, and thus avoid a potential deadlock.

This is eventually for immutable bio vecs - immutable bvecs aren't
useful if we still have to copy them, hence the need for the pointer.
Less code is always nice too, though.

Also, bio_integrity_alloc() was using fs_bio_set if no bio_set was
specified. This was wrong - using the bio_set doesn't protect us from
memory allocation failures, because we just used kmalloc for the
bio_integrity_payload. But it does introduce the possibility of
deadlock, if for some reason we weren't supposed to be using fs_bio_set.

Signed-off-by: Kent Overstreet <koverstreet@google.com>
CC: Jens Axboe <axboe@kernel.dk>
CC: Martin K. Petersen <martin.petersen@oracle.com>

1 files changed, 48 insertions, 84 deletions

diff --git a/fs/bio-integrity.c b/fs/bio-integrity.c
index 94fa1c562c0e..8c4c604c840d 100644
--- a/fs/bio-integrity.c
+++ b/fs/bio-integrity.c
@@ -27,48 +27,11 @@
 #include <linux/workqueue.h>
 #include <linux/slab.h>
 
-struct integrity_slab {
-        struct kmem_cache *slab;
-        unsigned short nr_vecs;
-        char name[8];
-};
-
-#define IS(x) { .nr_vecs = x, .name = "bip-"__stringify(x) }
-struct integrity_slab bip_slab[BIOVEC_NR_POOLS] __read_mostly = {
-        IS(1), IS(4), IS(16), IS(64), IS(128), IS(BIO_MAX_PAGES),
-};
-#undef IS
+#define BIP_INLINE_VECS 4
 
+static struct kmem_cache *bip_slab;
 static struct workqueue_struct *kintegrityd_wq;
 
-static inline unsigned int vecs_to_idx(unsigned int nr)
-{
-        switch (nr) {
-        case 1:
-                return 0;
-        case 2 ... 4:
-                return 1;
-        case 5 ... 16:
-                return 2;
-        case 17 ... 64:
-                return 3;
-        case 65 ... 128:
-                return 4;
-        case 129 ... BIO_MAX_PAGES:
-                return 5;
-        default:
-                BUG();
-        }
-}
-
-static inline int use_bip_pool(unsigned int idx)
-{
-        if (idx == BIOVEC_MAX_IDX)
-                return 1;
-
-        return 0;
-}
-
 /**
  * bio_integrity_alloc - Allocate integrity payload and attach it to bio
  * @bio:        bio to attach integrity metadata to
@@ -84,38 +47,41 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
                                                   unsigned int nr_vecs)
 {
         struct bio_integrity_payload *bip;
-        unsigned int idx = vecs_to_idx(nr_vecs);
         struct bio_set *bs = bio->bi_pool;
-
-        if (!bs)
-                bs = fs_bio_set;
-
-        BUG_ON(bio == NULL);
-        bip = NULL;
-
-        /* Lower order allocations come straight from slab */
-        if (!use_bip_pool(idx))
-                bip = kmem_cache_alloc(bip_slab[idx].slab, gfp_mask);
-
-        /* Use mempool if lower order alloc failed or max vecs were requested */
-        if (bip == NULL) {
-                idx = BIOVEC_MAX_IDX;  /* so we free the payload properly later */
+        unsigned long idx = BIO_POOL_NONE;
+        unsigned inline_vecs;
+
+        if (!bs) {
+                bip = kmalloc(sizeof(struct bio_integrity_payload) +
+                              sizeof(struct bio_vec) * nr_vecs, gfp_mask);
+                inline_vecs = nr_vecs;
+        } else {
                 bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask);
-
-                if (unlikely(bip == NULL)) {
-                        printk(KERN_ERR "%s: could not alloc bip\n", __func__);
-                        return NULL;
-                }
+                inline_vecs = BIP_INLINE_VECS;
         }
 
+        if (unlikely(!bip))
+                return NULL;
+
         memset(bip, 0, sizeof(*bip));
 
+        if (nr_vecs > inline_vecs) {
+                bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,
+                                          bs->bvec_integrity_pool);
+                if (!bip->bip_vec)
+                        goto err;
+        } else {
+                bip->bip_vec = bip->bip_inline_vecs;
+        }
+
         bip->bip_slab = idx;
         bip->bip_bio = bio;
-        bip->bip_vec = bip->bip_inline_vecs;
         bio->bi_integrity = bip;
 
         return bip;
+err:
+        mempool_free(bip, bs->bio_integrity_pool);
+        return NULL;
 }
 EXPORT_SYMBOL(bio_integrity_alloc);
 
@@ -131,20 +97,20 @@ void bio_integrity_free(struct bio *bio)
         struct bio_integrity_payload *bip = bio->bi_integrity;
         struct bio_set *bs = bio->bi_pool;
 
-        if (!bs)
-                bs = fs_bio_set;
-
-        BUG_ON(bip == NULL);
-
         /* A cloned bio doesn't own the integrity metadata */
         if (!bio_flagged(bio, BIO_CLONED) && !bio_flagged(bio, BIO_FS_INTEGRITY)
             && bip->bip_buf != NULL)
                 kfree(bip->bip_buf);
 
-        if (use_bip_pool(bip->bip_slab))
+        if (bs) {
+                if (bip->bip_slab != BIO_POOL_NONE)
+                        bvec_free(bs->bvec_integrity_pool, bip->bip_vec,
+                                  bip->bip_slab);
+
                 mempool_free(bip, bs->bio_integrity_pool);
-        else
-                kmem_cache_free(bip_slab[bip->bip_slab].slab, bip);
+        } else {
+                kfree(bip);
+        }
 
         bio->bi_integrity = NULL;
 }
@@ -747,13 +713,14 @@ EXPORT_SYMBOL(bio_integrity_clone);
 
 int bioset_integrity_create(struct bio_set *bs, int pool_size)
 {
-        unsigned int max_slab = vecs_to_idx(BIO_MAX_PAGES);
-
         if (bs->bio_integrity_pool)
                 return 0;
 
-        bs->bio_integrity_pool =
-                mempool_create_slab_pool(pool_size, bip_slab[max_slab].slab);
+        bs->bio_integrity_pool = mempool_create_slab_pool(pool_size, bip_slab);
+
+        bs->bvec_integrity_pool = biovec_create_pool(bs, pool_size);
+        if (!bs->bvec_integrity_pool)
+                return -1;
 
         if (!bs->bio_integrity_pool)
                 return -1;
@@ -766,13 +733,14 @@ void bioset_integrity_free(struct bio_set *bs)
 {
         if (bs->bio_integrity_pool)
                 mempool_destroy(bs->bio_integrity_pool);
+
+        if (bs->bvec_integrity_pool)
+                mempool_destroy(bs->bio_integrity_pool);
 }
 EXPORT_SYMBOL(bioset_integrity_free);
 
 void __init bio_integrity_init(void)
 {
-        unsigned int i;
-
         /*
          * kintegrityd won't block much but may burn a lot of CPU cycles.
          * Make it highpri CPU intensive wq with max concurrency of 1.
@@ -782,14 +750,10 @@ void __init bio_integrity_init(void)
         if (!kintegrityd_wq)
                 panic("Failed to create kintegrityd\n");
 
-        for (i = 0 ; i < BIOVEC_NR_POOLS ; i++) {
-                unsigned int size;
-
-                size = sizeof(struct bio_integrity_payload)
-                        + bip_slab[i].nr_vecs * sizeof(struct bio_vec);
-
-                bip_slab[i].slab =
-                        kmem_cache_create(bip_slab[i].name, size, 0,
-                                          SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
-        }
+        bip_slab = kmem_cache_create("bio_integrity_payload",
+                                     sizeof(struct bio_integrity_payload) +
+                                     sizeof(struct bio_vec) * BIP_INLINE_VECS,
+                                     0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
+        if (!bip_slab)
+                panic("Failed to create slab\n");
 }