block: Block layer data integrity support

Some block devices support verifying the integrity of requests by way
of checksums or other protection information that is submitted along
with the I/O.

This patch implements support for generating and verifying integrity
metadata, as well as correctly merging, splitting and cloning bios and
requests that have this extra information attached.

See Documentation/block/data-integrity.txt for more information.

Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

3 files changed, 198 insertions, 4 deletions

diff --git a/include/linux/bio.h b/include/linux/bio.h
index 49dfb3cb7460..6bfc3e8d9d89 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -64,6 +64,7 @@ struct bio_vec {
 
 struct bio_set;
 struct bio;
+struct bio_integrity_payload;
 typedef void (bio_end_io_t) (struct bio *, int);
 typedef void (bio_destructor_t) (struct bio *);
 
@@ -112,6 +113,9 @@ struct bio {
         atomic_t                bi_cnt;         /* pin count */
 
         void                    *bi_private;
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+        struct bio_integrity_payload *bi_integrity;  /* data integrity */
+#endif
 
         bio_destructor_t        *bi_destructor; /* destructor */
 };
@@ -271,6 +275,29 @@ static inline void *bio_data(struct bio *bio)
  */
 #define bio_get(bio)    atomic_inc(&(bio)->bi_cnt)
 
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+/*
+ * bio integrity payload
+ */
+struct bio_integrity_payload {
+        struct bio              *bip_bio;       /* parent bio */
+        struct bio_vec          *bip_vec;       /* integrity data vector */
+
+        sector_t                bip_sector;     /* virtual start sector */
+
+        void                    *bip_buf;       /* generated integrity data */
+        bio_end_io_t            *bip_end_io;    /* saved I/O completion fn */
+
+        int                     bip_error;      /* saved I/O error */
+        unsigned int            bip_size;
+
+        unsigned short          bip_pool;       /* pool the ivec came from */
+        unsigned short          bip_vcnt;       /* # of integrity bio_vecs */
+        unsigned short          bip_idx;        /* current bip_vec index */
+
+        struct work_struct      bip_work;       /* I/O completion */
+};
+#endif /* CONFIG_BLK_DEV_INTEGRITY */
 
 /*
  * A bio_pair is used when we need to split a bio.
@@ -283,10 +310,14 @@ static inline void *bio_data(struct bio *bio)
  *   in bio2.bi_private
  */
 struct bio_pair {
-        struct bio      bio1, bio2;
-        struct bio_vec  bv1, bv2;
-        atomic_t        cnt;
-        int             error;
+        struct bio                      bio1, bio2;
+        struct bio_vec                  bv1, bv2;
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+        struct bio_integrity_payload    bip1, bip2;
+        struct bio_vec                  iv1, iv2;
+#endif
+        atomic_t                        cnt;
+        int                             error;
 };
 extern struct bio_pair *bio_split(struct bio *bi, mempool_t *pool,
                                   int first_sectors);
@@ -334,6 +365,7 @@ extern struct bio *bio_copy_user_iov(struct request_queue *, struct sg_iovec *,
 extern int bio_uncopy_user(struct bio *);
 void zero_fill_bio(struct bio *bio);
 extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set *);
+extern unsigned int bvec_nr_vecs(unsigned short idx);
 
 /*
  * bio_set is used to allow other portions of the IO system to
@@ -346,6 +378,9 @@ extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set
 
 struct bio_set {
         mempool_t *bio_pool;
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+        mempool_t *bio_integrity_pool;
+#endif
         mempool_t *bvec_pools[BIOVEC_NR_POOLS];
 };
 
@@ -410,5 +445,56 @@ static inline char *__bio_kmap_irq(struct bio *bio, unsigned short idx,
         __bio_kmap_irq((bio), (bio)->bi_idx, (flags))
 #define bio_kunmap_irq(buf,flags)       __bio_kunmap_irq(buf, flags)
 
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+
+#define bip_vec_idx(bip, idx)   (&(bip->bip_vec[(idx)]))
+#define bip_vec(bip)            bip_vec_idx(bip, 0)
+
+#define __bip_for_each_vec(bvl, bip, i, start_idx)                      \
+        for (bvl = bip_vec_idx((bip), (start_idx)), i = (start_idx);    \
+             i < (bip)->bip_vcnt;                                       \
+             bvl++, i++)
+
+#define bip_for_each_vec(bvl, bip, i)                                   \
+        __bip_for_each_vec(bvl, bip, i, (bip)->bip_idx)
+
+#define bio_integrity(bio)      ((bio)->bi_integrity ? 1 : 0)
+
+extern struct bio_integrity_payload *bio_integrity_alloc_bioset(struct bio *, gfp_t, unsigned int, struct bio_set *);
+extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
+extern void bio_integrity_free(struct bio *, struct bio_set *);
+extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
+extern int bio_integrity_enabled(struct bio *bio);
+extern int bio_integrity_set_tag(struct bio *, void *, unsigned int);
+extern int bio_integrity_get_tag(struct bio *, void *, unsigned int);
+extern int bio_integrity_prep(struct bio *);
+extern void bio_integrity_endio(struct bio *, int);
+extern void bio_integrity_advance(struct bio *, unsigned int);
+extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int);
+extern void bio_integrity_split(struct bio *, struct bio_pair *, int);
+extern int bio_integrity_clone(struct bio *, struct bio *, struct bio_set *);
+extern int bioset_integrity_create(struct bio_set *, int);
+extern void bioset_integrity_free(struct bio_set *);
+extern void bio_integrity_init_slab(void);
+
+#else /* CONFIG_BLK_DEV_INTEGRITY */
+
+#define bio_integrity(a)                (0)
+#define bioset_integrity_create(a, b)   (0)
+#define bio_integrity_prep(a)           (0)
+#define bio_integrity_enabled(a)        (0)
+#define bio_integrity_clone(a, b, c)    (0)
+#define bioset_integrity_free(a)        do { } while (0)
+#define bio_integrity_free(a, b)        do { } while (0)
+#define bio_integrity_endio(a, b)       do { } while (0)
+#define bio_integrity_advance(a, b)     do { } while (0)
+#define bio_integrity_trim(a, b, c)     do { } while (0)
+#define bio_integrity_split(a, b, c)    do { } while (0)
+#define bio_integrity_set_tag(a, b, c)  do { } while (0)
+#define bio_integrity_get_tag(a, b, c)  do { } while (0)
+#define bio_integrity_init_slab(a)      do { } while (0)
+
+#endif /* CONFIG_BLK_DEV_INTEGRITY */
+
 #endif /* CONFIG_BLOCK */
 #endif /* __LINUX_BIO_H */
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 6a3da6717135..4a9ed45270ff 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -112,6 +112,7 @@ enum rq_flag_bits {
         __REQ_ALLOCED,          /* request came from our alloc pool */
         __REQ_RW_META,          /* metadata io request */
         __REQ_COPY_USER,        /* contains copies of user pages */
+        __REQ_INTEGRITY,        /* integrity metadata has been remapped */
         __REQ_NR_BITS,          /* stops here */
 };
 
@@ -134,6 +135,7 @@ enum rq_flag_bits {
 #define REQ_ALLOCED     (1 << __REQ_ALLOCED)
 #define REQ_RW_META     (1 << __REQ_RW_META)
 #define REQ_COPY_USER   (1 << __REQ_COPY_USER)
+#define REQ_INTEGRITY   (1 << __REQ_INTEGRITY)
 
 #define BLK_MAX_CDB     16
 
@@ -865,6 +867,109 @@ void kblockd_flush_work(struct work_struct *work);
         MODULE_ALIAS("block-major-" __stringify(major) "-*")
 
 
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+
+#define INTEGRITY_FLAG_READ     1       /* verify data integrity on read */
+#define INTEGRITY_FLAG_WRITE    2       /* generate data integrity on write */
+
+struct blk_integrity_exchg {
+        void                    *prot_buf;
+        void                    *data_buf;
+        sector_t                sector;
+        unsigned int            data_size;
+        unsigned short          sector_size;
+        const char              *disk_name;
+};
+
+typedef void (integrity_gen_fn) (struct blk_integrity_exchg *);
+typedef int (integrity_vrfy_fn) (struct blk_integrity_exchg *);
+typedef void (integrity_set_tag_fn) (void *, void *, unsigned int);
+typedef void (integrity_get_tag_fn) (void *, void *, unsigned int);
+
+struct blk_integrity {
+        integrity_gen_fn        *generate_fn;
+        integrity_vrfy_fn       *verify_fn;
+        integrity_set_tag_fn    *set_tag_fn;
+        integrity_get_tag_fn    *get_tag_fn;
+
+        unsigned short          flags;
+        unsigned short          tuple_size;
+        unsigned short          sector_size;
+        unsigned short          tag_size;
+
+        const char              *name;
+
+        struct kobject          kobj;
+};
+
+extern int blk_integrity_register(struct gendisk *, struct blk_integrity *);
+extern void blk_integrity_unregister(struct gendisk *);
+extern int blk_integrity_compare(struct block_device *, struct block_device *);
+extern int blk_rq_map_integrity_sg(struct request *, struct scatterlist *);
+extern int blk_rq_count_integrity_sg(struct request *);
+
+static inline unsigned short blk_integrity_tuple_size(struct blk_integrity *bi)
+{
+        if (bi)
+                return bi->tuple_size;
+
+        return 0;
+}
+
+static inline struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
+{
+        return bdev->bd_disk->integrity;
+}
+
+static inline unsigned int bdev_get_tag_size(struct block_device *bdev)
+{
+        struct blk_integrity *bi = bdev_get_integrity(bdev);
+
+        if (bi)
+                return bi->tag_size;
+
+        return 0;
+}
+
+static inline int bdev_integrity_enabled(struct block_device *bdev, int rw)
+{
+        struct blk_integrity *bi = bdev_get_integrity(bdev);
+
+        if (bi == NULL)
+                return 0;
+
+        if (rw == READ && bi->verify_fn != NULL &&
+            test_bit(INTEGRITY_FLAG_READ, &bi->flags))
+                return 1;
+
+        if (rw == WRITE && bi->generate_fn != NULL &&
+            test_bit(INTEGRITY_FLAG_WRITE, &bi->flags))
+                return 1;
+
+        return 0;
+}
+
+static inline int blk_integrity_rq(struct request *rq)
+{
+        BUG_ON(rq->bio == NULL);
+
+        return bio_integrity(rq->bio);
+}
+
+#else /* CONFIG_BLK_DEV_INTEGRITY */
+
+#define blk_integrity_rq(rq)                    (0)
+#define blk_rq_count_integrity_sg(a)            (0)
+#define blk_rq_map_integrity_sg(a, b)           (0)
+#define bdev_get_integrity(a)                   (0)
+#define bdev_get_tag_size(a)                    (0)
+#define blk_integrity_compare(a, b)             (0)
+#define blk_integrity_register(a, b)            (0)
+#define blk_integrity_unregister(a)             do { } while (0);
+
+#endif /* CONFIG_BLK_DEV_INTEGRITY */
+
+
 #else /* CONFIG_BLOCK */
 /*
  * stubs for when the block layer is configured out
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index ae7aec3cabee..524ec96f5a23 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -141,6 +141,9 @@ struct gendisk {
         struct disk_stats dkstats;
 #endif
         struct work_struct async_notify;
+#ifdef  CONFIG_BLK_DEV_INTEGRITY
+        struct blk_integrity *integrity;
+#endif
 };
 
 /*