block: Create bip slabs with embedded integrity vectors

This patch restores stacking ability to the block layer integrity
infrastructure by creating a set of dedicated bip slabs.  Each bip slab
has an embedded bio_vec array at the end.  This cuts down on memory
allocations and also simplifies the code compared to the original bvec
version.  Only the largest bip slab is backed by a mempool.  The pool is
contained in the bio_set so stacking drivers can ensure forward
progress.

Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
Signed-off-by: Jens Axboe <axboe@carl.(none)>

1 files changed, 16 insertions, 6 deletions

diff --git a/include/linux/bio.h b/include/linux/bio.h
index 2a04eb54c0dd..2892b710771c 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -319,7 +319,6 @@ static inline int bio_has_allocated_vec(struct bio *bio)
  */
 struct bio_integrity_payload {
         struct bio              *bip_bio;       /* parent bio */
-        struct bio_vec          *bip_vec;       /* integrity data vector */
 
         sector_t                bip_sector;     /* virtual start sector */
 
@@ -328,11 +327,12 @@ struct bio_integrity_payload {
 
         unsigned int            bip_size;
 
-        unsigned short          bip_pool;       /* pool the ivec came from */
+        unsigned short          bip_slab;       /* slab the bip came from */
         unsigned short          bip_vcnt;       /* # of integrity bio_vecs */
         unsigned short          bip_idx;        /* current bip_vec index */
 
         struct work_struct      bip_work;       /* I/O completion */
+        struct bio_vec          bip_vec[0];     /* embedded bvec array */
 };
 #endif /* CONFIG_BLK_DEV_INTEGRITY */
 
@@ -430,6 +430,9 @@ struct bio_set {
         unsigned int front_pad;
 
         mempool_t *bio_pool;
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+        mempool_t *bio_integrity_pool;
+#endif
         mempool_t *bvec_pool;
 };
 
@@ -634,8 +637,9 @@ static inline struct bio *bio_list_get(struct bio_list *bl)
 
 #define bio_integrity(bio) (bio->bi_integrity != NULL)
 
+extern struct bio_integrity_payload *bio_integrity_alloc_bioset(struct bio *, gfp_t, unsigned int, struct bio_set *);
 extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
-extern void bio_integrity_free(struct bio *);
+extern void bio_integrity_free(struct bio *, struct bio_set *);
 extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
 extern int bio_integrity_enabled(struct bio *bio);
 extern int bio_integrity_set_tag(struct bio *, void *, unsigned int);
@@ -645,21 +649,27 @@ extern void bio_integrity_endio(struct bio *, int);
 extern void bio_integrity_advance(struct bio *, unsigned int);
 extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int);
 extern void bio_integrity_split(struct bio *, struct bio_pair *, int);
-extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
+extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t, struct bio_set *);
+extern int bioset_integrity_create(struct bio_set *, int);
+extern void bioset_integrity_free(struct bio_set *);
+extern void bio_integrity_init(void);
 
 #else /* CONFIG_BLK_DEV_INTEGRITY */
 
 #define bio_integrity(a)                (0)
+#define bioset_integrity_create(a, b)   (0)
 #define bio_integrity_prep(a)           (0)
 #define bio_integrity_enabled(a)        (0)
-#define bio_integrity_clone(a, b, c)    (0)
-#define bio_integrity_free(a)           do { } while (0)
+#define bio_integrity_clone(a, b, c, d) (0)
+#define bioset_integrity_free(a)        do { } while (0)
+#define bio_integrity_free(a, b)        do { } while (0)
 #define bio_integrity_endio(a, b)       do { } while (0)
 #define bio_integrity_advance(a, b)     do { } while (0)
 #define bio_integrity_trim(a, b, c)     do { } while (0)
 #define bio_integrity_split(a, b, c)    do { } while (0)
 #define bio_integrity_set_tag(a, b, c)  do { } while (0)
 #define bio_integrity_get_tag(a, b, c)  do { } while (0)
+#define bio_integrity_init(a)           do { } while (0)
 
 #endif /* CONFIG_BLK_DEV_INTEGRITY */