aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin K. Petersen <martin.petersen@oracle.com>2008-06-30 14:04:41 -0400
committerJens Axboe <jens.axboe@oracle.com>2008-07-03 07:21:13 -0400
commit7ba1ba12eeef0aa7113beb16410ef8b7c748e18b (patch)
tree4629aabe88bf095d58eabd2f451207695bb35b08
parent51d654e1d885607a6edd02b337105fa5c28b6d33 (diff)
block: Block layer data integrity support
Some block devices support verifying the integrity of requests by way of checksums or other protection information that is submitted along with the I/O. This patch implements support for generating and verifying integrity metadata, as well as correctly merging, splitting and cloning bios and requests that have this extra information attached. See Documentation/block/data-integrity.txt for more information. Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com> Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
-rw-r--r--block/Kconfig12
-rw-r--r--block/Makefile1
-rw-r--r--block/blk-core.c7
-rw-r--r--block/blk-integrity.c382
-rw-r--r--block/blk-merge.c3
-rw-r--r--block/blk.h8
-rw-r--r--block/elevator.c6
-rw-r--r--fs/Makefile1
-rw-r--r--fs/bio-integrity.c708
-rw-r--r--fs/bio.c32
-rw-r--r--include/linux/bio.h94
-rw-r--r--include/linux/blkdev.h105
-rw-r--r--include/linux/genhd.h3
13 files changed, 1355 insertions, 7 deletions
diff --git a/block/Kconfig b/block/Kconfig
index 3e97f2bc446f..1ab7c15c8d7a 100644
--- a/block/Kconfig
+++ b/block/Kconfig
@@ -81,6 +81,18 @@ config BLK_DEV_BSG
81 81
82 If unsure, say N. 82 If unsure, say N.
83 83
84config BLK_DEV_INTEGRITY
85 bool "Block layer data integrity support"
86 ---help---
87 Some storage devices allow extra information to be
88 stored/retrieved to help protect the data. The block layer
89 data integrity option provides hooks which can be used by
90 filesystems to ensure better data integrity.
91
92 Say yes here if you have a storage device that provides the
93 T10/SCSI Data Integrity Field or the T13/ATA External Path
94 Protection. If in doubt, say N.
95
84endif # BLOCK 96endif # BLOCK
85 97
86config BLOCK_COMPAT 98config BLOCK_COMPAT
diff --git a/block/Makefile b/block/Makefile
index 5a43c7d79594..045f7b62e4bb 100644
--- a/block/Makefile
+++ b/block/Makefile
@@ -14,3 +14,4 @@ obj-$(CONFIG_IOSCHED_CFQ) += cfq-iosched.o
14 14
15obj-$(CONFIG_BLK_DEV_IO_TRACE) += blktrace.o 15obj-$(CONFIG_BLK_DEV_IO_TRACE) += blktrace.o
16obj-$(CONFIG_BLOCK_COMPAT) += compat_ioctl.o 16obj-$(CONFIG_BLOCK_COMPAT) += compat_ioctl.o
17obj-$(CONFIG_BLK_DEV_INTEGRITY) += blk-integrity.o
diff --git a/block/blk-core.c b/block/blk-core.c
index 1905aaba49fb..e0fb0bcc0c17 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -143,6 +143,10 @@ static void req_bio_endio(struct request *rq, struct bio *bio,
143 143
144 bio->bi_size -= nbytes; 144 bio->bi_size -= nbytes;
145 bio->bi_sector += (nbytes >> 9); 145 bio->bi_sector += (nbytes >> 9);
146
147 if (bio_integrity(bio))
148 bio_integrity_advance(bio, nbytes);
149
146 if (bio->bi_size == 0) 150 if (bio->bi_size == 0)
147 bio_endio(bio, error); 151 bio_endio(bio, error);
148 } else { 152 } else {
@@ -1381,6 +1385,9 @@ end_io:
1381 */ 1385 */
1382 blk_partition_remap(bio); 1386 blk_partition_remap(bio);
1383 1387
1388 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio))
1389 goto end_io;
1390
1384 if (old_sector != -1) 1391 if (old_sector != -1)
1385 blk_add_trace_remap(q, bio, old_dev, bio->bi_sector, 1392 blk_add_trace_remap(q, bio, old_dev, bio->bi_sector,
1386 old_sector); 1393 old_sector);
diff --git a/block/blk-integrity.c b/block/blk-integrity.c
new file mode 100644
index 000000000000..65f23ef38bbe
--- /dev/null
+++ b/block/blk-integrity.c
@@ -0,0 +1,382 @@
1/*
2 * blk-integrity.c - Block layer data integrity extensions
3 *
4 * Copyright (C) 2007, 2008 Oracle Corporation
5 * Written by: Martin K. Petersen <martin.petersen@oracle.com>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License version
9 * 2 as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; see the file COPYING. If not, write to
18 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
19 * USA.
20 *
21 */
22
23#include <linux/blkdev.h>
24#include <linux/mempool.h>
25#include <linux/bio.h>
26#include <linux/scatterlist.h>
27
28#include "blk.h"
29
30static struct kmem_cache *integrity_cachep;
31
32/**
33 * blk_rq_count_integrity_sg - Count number of integrity scatterlist elements
34 * @rq: request with integrity metadata attached
35 *
36 * Description: Returns the number of elements required in a
37 * scatterlist corresponding to the integrity metadata in a request.
38 */
39int blk_rq_count_integrity_sg(struct request *rq)
40{
41 struct bio_vec *iv, *ivprv;
42 struct req_iterator iter;
43 unsigned int segments;
44
45 ivprv = NULL;
46 segments = 0;
47
48 rq_for_each_integrity_segment(iv, rq, iter) {
49
50 if (!ivprv || !BIOVEC_PHYS_MERGEABLE(ivprv, iv))
51 segments++;
52
53 ivprv = iv;
54 }
55
56 return segments;
57}
58EXPORT_SYMBOL(blk_rq_count_integrity_sg);
59
60/**
61 * blk_rq_map_integrity_sg - Map integrity metadata into a scatterlist
62 * @rq: request with integrity metadata attached
63 * @sglist: target scatterlist
64 *
65 * Description: Map the integrity vectors in request into a
66 * scatterlist. The scatterlist must be big enough to hold all
67 * elements. I.e. sized using blk_rq_count_integrity_sg().
68 */
69int blk_rq_map_integrity_sg(struct request *rq, struct scatterlist *sglist)
70{
71 struct bio_vec *iv, *ivprv;
72 struct req_iterator iter;
73 struct scatterlist *sg;
74 unsigned int segments;
75
76 ivprv = NULL;
77 sg = NULL;
78 segments = 0;
79
80 rq_for_each_integrity_segment(iv, rq, iter) {
81
82 if (ivprv) {
83 if (!BIOVEC_PHYS_MERGEABLE(ivprv, iv))
84 goto new_segment;
85
86 sg->length += iv->bv_len;
87 } else {
88new_segment:
89 if (!sg)
90 sg = sglist;
91 else {
92 sg->page_link &= ~0x02;
93 sg = sg_next(sg);
94 }
95
96 sg_set_page(sg, iv->bv_page, iv->bv_len, iv->bv_offset);
97 segments++;
98 }
99
100 ivprv = iv;
101 }
102
103 if (sg)
104 sg_mark_end(sg);
105
106 return segments;
107}
108EXPORT_SYMBOL(blk_rq_map_integrity_sg);
109
110/**
111 * blk_integrity_compare - Compare integrity profile of two block devices
112 * @b1: Device to compare
113 * @b2: Device to compare
114 *
115 * Description: Meta-devices like DM and MD need to verify that all
116 * sub-devices use the same integrity format before advertising to
117 * upper layers that they can send/receive integrity metadata. This
118 * function can be used to check whether two block devices have
119 * compatible integrity formats.
120 */
121int blk_integrity_compare(struct block_device *bd1, struct block_device *bd2)
122{
123 struct blk_integrity *b1 = bd1->bd_disk->integrity;
124 struct blk_integrity *b2 = bd2->bd_disk->integrity;
125
126 BUG_ON(bd1->bd_disk == NULL);
127 BUG_ON(bd2->bd_disk == NULL);
128
129 if (!b1 || !b2)
130 return 0;
131
132 if (b1->sector_size != b2->sector_size) {
133 printk(KERN_ERR "%s: %s/%s sector sz %u != %u\n", __func__,
134 bd1->bd_disk->disk_name, bd2->bd_disk->disk_name,
135 b1->sector_size, b2->sector_size);
136 return -1;
137 }
138
139 if (b1->tuple_size != b2->tuple_size) {
140 printk(KERN_ERR "%s: %s/%s tuple sz %u != %u\n", __func__,
141 bd1->bd_disk->disk_name, bd2->bd_disk->disk_name,
142 b1->tuple_size, b2->tuple_size);
143 return -1;
144 }
145
146 if (b1->tag_size && b2->tag_size && (b1->tag_size != b2->tag_size)) {
147 printk(KERN_ERR "%s: %s/%s tag sz %u != %u\n", __func__,
148 bd1->bd_disk->disk_name, bd2->bd_disk->disk_name,
149 b1->tag_size, b2->tag_size);
150 return -1;
151 }
152
153 if (strcmp(b1->name, b2->name)) {
154 printk(KERN_ERR "%s: %s/%s type %s != %s\n", __func__,
155 bd1->bd_disk->disk_name, bd2->bd_disk->disk_name,
156 b1->name, b2->name);
157 return -1;
158 }
159
160 return 0;
161}
162EXPORT_SYMBOL(blk_integrity_compare);
163
164struct integrity_sysfs_entry {
165 struct attribute attr;
166 ssize_t (*show)(struct blk_integrity *, char *);
167 ssize_t (*store)(struct blk_integrity *, const char *, size_t);
168};
169
170static ssize_t integrity_attr_show(struct kobject *kobj, struct attribute *attr,
171 char *page)
172{
173 struct blk_integrity *bi =
174 container_of(kobj, struct blk_integrity, kobj);
175 struct integrity_sysfs_entry *entry =
176 container_of(attr, struct integrity_sysfs_entry, attr);
177
178 return entry->show(bi, page);
179}
180
181static ssize_t integrity_attr_store(struct kobject *kobj, struct attribute *attr,
182 const char *page, size_t count)
183{
184 struct blk_integrity *bi =
185 container_of(kobj, struct blk_integrity, kobj);
186 struct integrity_sysfs_entry *entry =
187 container_of(attr, struct integrity_sysfs_entry, attr);
188 ssize_t ret = 0;
189
190 if (entry->store)
191 ret = entry->store(bi, page, count);
192
193 return ret;
194}
195
196static ssize_t integrity_format_show(struct blk_integrity *bi, char *page)
197{
198 if (bi != NULL && bi->name != NULL)
199 return sprintf(page, "%s\n", bi->name);
200 else
201 return sprintf(page, "none\n");
202}
203
204static ssize_t integrity_tag_size_show(struct blk_integrity *bi, char *page)
205{
206 if (bi != NULL)
207 return sprintf(page, "%u\n", bi->tag_size);
208 else
209 return sprintf(page, "0\n");
210}
211
212static ssize_t integrity_read_store(struct blk_integrity *bi,
213 const char *page, size_t count)
214{
215 char *p = (char *) page;
216 unsigned long val = simple_strtoul(p, &p, 10);
217
218 if (val)
219 set_bit(INTEGRITY_FLAG_READ, &bi->flags);
220 else
221 clear_bit(INTEGRITY_FLAG_READ, &bi->flags);
222
223 return count;
224}
225
226static ssize_t integrity_read_show(struct blk_integrity *bi, char *page)
227{
228 return sprintf(page, "%d\n",
229 test_bit(INTEGRITY_FLAG_READ, &bi->flags) ? 1 : 0);
230}
231
232static ssize_t integrity_write_store(struct blk_integrity *bi,
233 const char *page, size_t count)
234{
235 char *p = (char *) page;
236 unsigned long val = simple_strtoul(p, &p, 10);
237
238 if (val)
239 set_bit(INTEGRITY_FLAG_WRITE, &bi->flags);
240 else
241 clear_bit(INTEGRITY_FLAG_WRITE, &bi->flags);
242
243 return count;
244}
245
246static ssize_t integrity_write_show(struct blk_integrity *bi, char *page)
247{
248 return sprintf(page, "%d\n",
249 test_bit(INTEGRITY_FLAG_WRITE, &bi->flags) ? 1 : 0);
250}
251
252static struct integrity_sysfs_entry integrity_format_entry = {
253 .attr = { .name = "format", .mode = S_IRUGO },
254 .show = integrity_format_show,
255};
256
257static struct integrity_sysfs_entry integrity_tag_size_entry = {
258 .attr = { .name = "tag_size", .mode = S_IRUGO },
259 .show = integrity_tag_size_show,
260};
261
262static struct integrity_sysfs_entry integrity_read_entry = {
263 .attr = { .name = "read_verify", .mode = S_IRUGO | S_IWUSR },
264 .show = integrity_read_show,
265 .store = integrity_read_store,
266};
267
268static struct integrity_sysfs_entry integrity_write_entry = {
269 .attr = { .name = "write_generate", .mode = S_IRUGO | S_IWUSR },
270 .show = integrity_write_show,
271 .store = integrity_write_store,
272};
273
274static struct attribute *integrity_attrs[] = {
275 &integrity_format_entry.attr,
276 &integrity_tag_size_entry.attr,
277 &integrity_read_entry.attr,
278 &integrity_write_entry.attr,
279 NULL,
280};
281
282static struct sysfs_ops integrity_ops = {
283 .show = &integrity_attr_show,
284 .store = &integrity_attr_store,
285};
286
287static int __init blk_dev_integrity_init(void)
288{
289 integrity_cachep = kmem_cache_create("blkdev_integrity",
290 sizeof(struct blk_integrity),
291 0, SLAB_PANIC, NULL);
292 return 0;
293}
294subsys_initcall(blk_dev_integrity_init);
295
296static void blk_integrity_release(struct kobject *kobj)
297{
298 struct blk_integrity *bi =
299 container_of(kobj, struct blk_integrity, kobj);
300
301 kmem_cache_free(integrity_cachep, bi);
302}
303
304static struct kobj_type integrity_ktype = {
305 .default_attrs = integrity_attrs,
306 .sysfs_ops = &integrity_ops,
307 .release = blk_integrity_release,
308};
309
310/**
311 * blk_integrity_register - Register a gendisk as being integrity-capable
312 * @disk: struct gendisk pointer to make integrity-aware
313 * @template: integrity profile
314 *
315 * Description: When a device needs to advertise itself as being able
316 * to send/receive integrity metadata it must use this function to
317 * register the capability with the block layer. The template is a
318 * blk_integrity struct with values appropriate for the underlying
319 * hardware. See Documentation/block/data-integrity.txt.
320 */
321int blk_integrity_register(struct gendisk *disk, struct blk_integrity *template)
322{
323 struct blk_integrity *bi;
324
325 BUG_ON(disk == NULL);
326 BUG_ON(template == NULL);
327
328 if (disk->integrity == NULL) {
329 bi = kmem_cache_alloc(integrity_cachep, GFP_KERNEL | __GFP_ZERO);
330 if (!bi)
331 return -1;
332
333 if (kobject_init_and_add(&bi->kobj, &integrity_ktype,
334 &disk->dev.kobj, "%s", "integrity")) {
335 kmem_cache_free(integrity_cachep, bi);
336 return -1;
337 }
338
339 kobject_uevent(&bi->kobj, KOBJ_ADD);
340
341 set_bit(INTEGRITY_FLAG_READ, &bi->flags);
342 set_bit(INTEGRITY_FLAG_WRITE, &bi->flags);
343 bi->sector_size = disk->queue->hardsect_size;
344 disk->integrity = bi;
345 } else
346 bi = disk->integrity;
347
348 /* Use the provided profile as template */
349 bi->name = template->name;
350 bi->generate_fn = template->generate_fn;
351 bi->verify_fn = template->verify_fn;
352 bi->tuple_size = template->tuple_size;
353 bi->set_tag_fn = template->set_tag_fn;
354 bi->get_tag_fn = template->get_tag_fn;
355 bi->tag_size = template->tag_size;
356
357 return 0;
358}
359EXPORT_SYMBOL(blk_integrity_register);
360
361/**
362 * blk_integrity_unregister - Remove block integrity profile
363 * @disk: disk whose integrity profile to deallocate
364 *
365 * Description: This function frees all memory used by the block
366 * integrity profile. To be called at device teardown.
367 */
368void blk_integrity_unregister(struct gendisk *disk)
369{
370 struct blk_integrity *bi;
371
372 if (!disk || !disk->integrity)
373 return;
374
375 bi = disk->integrity;
376
377 kobject_uevent(&bi->kobj, KOBJ_REMOVE);
378 kobject_del(&bi->kobj);
379 kobject_put(&disk->dev.kobj);
380 kmem_cache_free(integrity_cachep, bi);
381}
382EXPORT_SYMBOL(blk_integrity_unregister);
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 651136aae76e..5efc9e7a68b7 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -441,6 +441,9 @@ static int attempt_merge(struct request_queue *q, struct request *req,
441 || next->special) 441 || next->special)
442 return 0; 442 return 0;
443 443
444 if (blk_integrity_rq(req) != blk_integrity_rq(next))
445 return 0;
446
444 /* 447 /*
445 * If we are allowed to merge, then append bio list 448 * If we are allowed to merge, then append bio list
446 * from next to rq and release next. merge_requests_fn 449 * from next to rq and release next. merge_requests_fn
diff --git a/block/blk.h b/block/blk.h
index 59776ab4742a..c79f30e1df52 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -51,4 +51,12 @@ static inline int queue_congestion_off_threshold(struct request_queue *q)
51 return q->nr_congestion_off; 51 return q->nr_congestion_off;
52} 52}
53 53
54#if defined(CONFIG_BLK_DEV_INTEGRITY)
55
56#define rq_for_each_integrity_segment(bvl, _rq, _iter) \
57 __rq_for_each_bio(_iter.bio, _rq) \
58 bip_for_each_vec(bvl, _iter.bio->bi_integrity, _iter.i)
59
60#endif /* BLK_DEV_INTEGRITY */
61
54#endif 62#endif
diff --git a/block/elevator.c b/block/elevator.c
index 902dd1344d56..1f5bfe696026 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -86,6 +86,12 @@ int elv_rq_merge_ok(struct request *rq, struct bio *bio)
86 if (rq->rq_disk != bio->bi_bdev->bd_disk || rq->special) 86 if (rq->rq_disk != bio->bi_bdev->bd_disk || rq->special)
87 return 0; 87 return 0;
88 88
89 /*
90 * only merge integrity protected bio into ditto rq
91 */
92 if (bio_integrity(bio) != blk_integrity_rq(rq))
93 return 0;
94
89 if (!elv_iosched_allow_merge(rq, bio)) 95 if (!elv_iosched_allow_merge(rq, bio))
90 return 0; 96 return 0;
91 97
diff --git a/fs/Makefile b/fs/Makefile
index 1e7a11bd4da1..277b079dec9e 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -19,6 +19,7 @@ else
19obj-y += no-block.o 19obj-y += no-block.o
20endif 20endif
21 21
22obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o
22obj-$(CONFIG_INOTIFY) += inotify.o 23obj-$(CONFIG_INOTIFY) += inotify.o
23obj-$(CONFIG_INOTIFY_USER) += inotify_user.o 24obj-$(CONFIG_INOTIFY_USER) += inotify_user.o
24obj-$(CONFIG_EPOLL) += eventpoll.o 25obj-$(CONFIG_EPOLL) += eventpoll.o
diff --git a/fs/bio-integrity.c b/fs/bio-integrity.c
new file mode 100644
index 000000000000..31b08878913d
--- /dev/null
+++ b/fs/bio-integrity.c
@@ -0,0 +1,708 @@
1/*
2 * bio-integrity.c - bio data integrity extensions
3 *
4 * Copyright (C) 2007, 2008 Oracle Corporation
5 * Written by: Martin K. Petersen <martin.petersen@oracle.com>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License version
9 * 2 as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; see the file COPYING. If not, write to
18 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
19 * USA.
20 *
21 */
22
23#include <linux/blkdev.h>
24#include <linux/mempool.h>
25#include <linux/bio.h>
26#include <linux/workqueue.h>
27
28static struct kmem_cache *bio_integrity_slab __read_mostly;
29static struct workqueue_struct *kintegrityd_wq;
30
31/**
32 * bio_integrity_alloc_bioset - Allocate integrity payload and attach it to bio
33 * @bio: bio to attach integrity metadata to
34 * @gfp_mask: Memory allocation mask
35 * @nr_vecs: Number of integrity metadata scatter-gather elements
36 * @bs: bio_set to allocate from
37 *
38 * Description: This function prepares a bio for attaching integrity
39 * metadata. nr_vecs specifies the maximum number of pages containing
40 * integrity metadata that can be attached.
41 */
42struct bio_integrity_payload *bio_integrity_alloc_bioset(struct bio *bio, gfp_t gfp_mask, unsigned int nr_vecs, struct bio_set *bs)
43{
44 struct bio_integrity_payload *bip;
45 struct bio_vec *iv;
46 unsigned long idx;
47
48 BUG_ON(bio == NULL);
49
50 bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask);
51 if (unlikely(bip == NULL)) {
52 printk(KERN_ERR "%s: could not alloc bip\n", __func__);
53 return NULL;
54 }
55
56 memset(bip, 0, sizeof(*bip));
57
58 iv = bvec_alloc_bs(gfp_mask, nr_vecs, &idx, bs);
59 if (unlikely(iv == NULL)) {
60 printk(KERN_ERR "%s: could not alloc bip_vec\n", __func__);
61 mempool_free(bip, bs->bio_integrity_pool);
62 return NULL;
63 }
64
65 bip->bip_pool = idx;
66 bip->bip_vec = iv;
67 bip->bip_bio = bio;
68 bio->bi_integrity = bip;
69
70 return bip;
71}
72EXPORT_SYMBOL(bio_integrity_alloc_bioset);
73
74/**
75 * bio_integrity_alloc - Allocate integrity payload and attach it to bio
76 * @bio: bio to attach integrity metadata to
77 * @gfp_mask: Memory allocation mask
78 * @nr_vecs: Number of integrity metadata scatter-gather elements
79 *
80 * Description: This function prepares a bio for attaching integrity
81 * metadata. nr_vecs specifies the maximum number of pages containing
82 * integrity metadata that can be attached.
83 */
84struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, gfp_t gfp_mask, unsigned int nr_vecs)
85{
86 return bio_integrity_alloc_bioset(bio, gfp_mask, nr_vecs, fs_bio_set);
87}
88EXPORT_SYMBOL(bio_integrity_alloc);
89
90/**
91 * bio_integrity_free - Free bio integrity payload
92 * @bio: bio containing bip to be freed
93 * @bs: bio_set this bio was allocated from
94 *
95 * Description: Used to free the integrity portion of a bio. Usually
96 * called from bio_free().
97 */
98void bio_integrity_free(struct bio *bio, struct bio_set *bs)
99{
100 struct bio_integrity_payload *bip = bio->bi_integrity;
101
102 BUG_ON(bip == NULL);
103
104 /* A cloned bio doesn't own the integrity metadata */
105 if (!bio_flagged(bio, BIO_CLONED) && bip->bip_buf != NULL)
106 kfree(bip->bip_buf);
107
108 mempool_free(bip->bip_vec, bs->bvec_pools[bip->bip_pool]);
109 mempool_free(bip, bs->bio_integrity_pool);
110
111 bio->bi_integrity = NULL;
112}
113EXPORT_SYMBOL(bio_integrity_free);
114
115/**
116 * bio_integrity_add_page - Attach integrity metadata
117 * @bio: bio to update
118 * @page: page containing integrity metadata
119 * @len: number of bytes of integrity metadata in page
120 * @offset: start offset within page
121 *
122 * Description: Attach a page containing integrity metadata to bio.
123 */
124int bio_integrity_add_page(struct bio *bio, struct page *page,
125 unsigned int len, unsigned int offset)
126{
127 struct bio_integrity_payload *bip = bio->bi_integrity;
128 struct bio_vec *iv;
129
130 if (bip->bip_vcnt >= bvec_nr_vecs(bip->bip_pool)) {
131 printk(KERN_ERR "%s: bip_vec full\n", __func__);
132 return 0;
133 }
134
135 iv = bip_vec_idx(bip, bip->bip_vcnt);
136 BUG_ON(iv == NULL);
137 BUG_ON(iv->bv_page != NULL);
138
139 iv->bv_page = page;
140 iv->bv_len = len;
141 iv->bv_offset = offset;
142 bip->bip_vcnt++;
143
144 return len;
145}
146EXPORT_SYMBOL(bio_integrity_add_page);
147
148/**
149 * bio_integrity_enabled - Check whether integrity can be passed
150 * @bio: bio to check
151 *
152 * Description: Determines whether bio_integrity_prep() can be called
153 * on this bio or not. bio data direction and target device must be
154 * set prior to calling. The functions honors the write_generate and
155 * read_verify flags in sysfs.
156 */
157int bio_integrity_enabled(struct bio *bio)
158{
159 /* Already protected? */
160 if (bio_integrity(bio))
161 return 0;
162
163 return bdev_integrity_enabled(bio->bi_bdev, bio_data_dir(bio));
164}
165EXPORT_SYMBOL(bio_integrity_enabled);
166
167/**
168 * bio_integrity_hw_sectors - Convert 512b sectors to hardware ditto
169 * @bi: blk_integrity profile for device
170 * @sectors: Number of 512 sectors to convert
171 *
172 * Description: The block layer calculates everything in 512 byte
173 * sectors but integrity metadata is done in terms of the hardware
174 * sector size of the storage device. Convert the block layer sectors
175 * to physical sectors.
176 */
177static inline unsigned int bio_integrity_hw_sectors(struct blk_integrity *bi, unsigned int sectors)
178{
179 /* At this point there are only 512b or 4096b DIF/EPP devices */
180 if (bi->sector_size == 4096)
181 return sectors >>= 3;
182
183 return sectors;
184}
185
186/**
187 * bio_integrity_tag_size - Retrieve integrity tag space
188 * @bio: bio to inspect
189 *
190 * Description: Returns the maximum number of tag bytes that can be
191 * attached to this bio. Filesystems can use this to determine how
192 * much metadata to attach to an I/O.
193 */
194unsigned int bio_integrity_tag_size(struct bio *bio)
195{
196 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
197
198 BUG_ON(bio->bi_size == 0);
199
200 return bi->tag_size * (bio->bi_size / bi->sector_size);
201}
202EXPORT_SYMBOL(bio_integrity_tag_size);
203
204int bio_integrity_tag(struct bio *bio, void *tag_buf, unsigned int len, int set)
205{
206 struct bio_integrity_payload *bip = bio->bi_integrity;
207 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
208 unsigned int nr_sectors;
209
210 BUG_ON(bip->bip_buf == NULL);
211
212 if (bi->tag_size == 0)
213 return -1;
214
215 nr_sectors = bio_integrity_hw_sectors(bi, DIV_ROUND_UP(len, bi->tag_size));
216
217 if (nr_sectors * bi->tuple_size > bip->bip_size) {
218 printk(KERN_ERR "%s: tag too big for bio: %u > %u\n",
219 __func__, nr_sectors * bi->tuple_size, bip->bip_size);
220 return -1;
221 }
222
223 if (set)
224 bi->set_tag_fn(bip->bip_buf, tag_buf, nr_sectors);
225 else
226 bi->get_tag_fn(bip->bip_buf, tag_buf, nr_sectors);
227
228 return 0;
229}
230
231/**
232 * bio_integrity_set_tag - Attach a tag buffer to a bio
233 * @bio: bio to attach buffer to
234 * @tag_buf: Pointer to a buffer containing tag data
235 * @len: Length of the included buffer
236 *
237 * Description: Use this function to tag a bio by leveraging the extra
238 * space provided by devices formatted with integrity protection. The
239 * size of the integrity buffer must be <= to the size reported by
240 * bio_integrity_tag_size().
241 */
242int bio_integrity_set_tag(struct bio *bio, void *tag_buf, unsigned int len)
243{
244 BUG_ON(bio_data_dir(bio) != WRITE);
245
246 return bio_integrity_tag(bio, tag_buf, len, 1);
247}
248EXPORT_SYMBOL(bio_integrity_set_tag);
249
250/**
251 * bio_integrity_get_tag - Retrieve a tag buffer from a bio
252 * @bio: bio to retrieve buffer from
253 * @tag_buf: Pointer to a buffer for the tag data
254 * @len: Length of the target buffer
255 *
256 * Description: Use this function to retrieve the tag buffer from a
257 * completed I/O. The size of the integrity buffer must be <= to the
258 * size reported by bio_integrity_tag_size().
259 */
260int bio_integrity_get_tag(struct bio *bio, void *tag_buf, unsigned int len)
261{
262 BUG_ON(bio_data_dir(bio) != READ);
263
264 return bio_integrity_tag(bio, tag_buf, len, 0);
265}
266EXPORT_SYMBOL(bio_integrity_get_tag);
267
268/**
269 * bio_integrity_generate - Generate integrity metadata for a bio
270 * @bio: bio to generate integrity metadata for
271 *
272 * Description: Generates integrity metadata for a bio by calling the
273 * block device's generation callback function. The bio must have a
274 * bip attached with enough room to accommodate the generated
275 * integrity metadata.
276 */
277static void bio_integrity_generate(struct bio *bio)
278{
279 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
280 struct blk_integrity_exchg bix;
281 struct bio_vec *bv;
282 sector_t sector = bio->bi_sector;
283 unsigned int i, sectors, total;
284 void *prot_buf = bio->bi_integrity->bip_buf;
285
286 total = 0;
287 bix.disk_name = bio->bi_bdev->bd_disk->disk_name;
288 bix.sector_size = bi->sector_size;
289
290 bio_for_each_segment(bv, bio, i) {
291 void *kaddr = kmap_atomic(bv->bv_page, KM_USER0);
292 bix.data_buf = kaddr + bv->bv_offset;
293 bix.data_size = bv->bv_len;
294 bix.prot_buf = prot_buf;
295 bix.sector = sector;
296
297 bi->generate_fn(&bix);
298
299 sectors = bv->bv_len / bi->sector_size;
300 sector += sectors;
301 prot_buf += sectors * bi->tuple_size;
302 total += sectors * bi->tuple_size;
303 BUG_ON(total > bio->bi_integrity->bip_size);
304
305 kunmap_atomic(kaddr, KM_USER0);
306 }
307}
308
309/**
310 * bio_integrity_prep - Prepare bio for integrity I/O
311 * @bio: bio to prepare
312 *
313 * Description: Allocates a buffer for integrity metadata, maps the
314 * pages and attaches them to a bio. The bio must have data
315 * direction, target device and start sector set priot to calling. In
316 * the WRITE case, integrity metadata will be generated using the
317 * block device's integrity function. In the READ case, the buffer
318 * will be prepared for DMA and a suitable end_io handler set up.
319 */
320int bio_integrity_prep(struct bio *bio)
321{
322 struct bio_integrity_payload *bip;
323 struct blk_integrity *bi;
324 struct request_queue *q;
325 void *buf;
326 unsigned long start, end;
327 unsigned int len, nr_pages;
328 unsigned int bytes, offset, i;
329 unsigned int sectors;
330
331 bi = bdev_get_integrity(bio->bi_bdev);
332 q = bdev_get_queue(bio->bi_bdev);
333 BUG_ON(bi == NULL);
334 BUG_ON(bio_integrity(bio));
335
336 sectors = bio_integrity_hw_sectors(bi, bio_sectors(bio));
337
338 /* Allocate kernel buffer for protection data */
339 len = sectors * blk_integrity_tuple_size(bi);
340 buf = kmalloc(len, GFP_NOIO | __GFP_NOFAIL | q->bounce_gfp);
341 if (unlikely(buf == NULL)) {
342 printk(KERN_ERR "could not allocate integrity buffer\n");
343 return -EIO;
344 }
345
346 end = (((unsigned long) buf) + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
347 start = ((unsigned long) buf) >> PAGE_SHIFT;
348 nr_pages = end - start;
349
350 /* Allocate bio integrity payload and integrity vectors */
351 bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages);
352 if (unlikely(bip == NULL)) {
353 printk(KERN_ERR "could not allocate data integrity bioset\n");
354 kfree(buf);
355 return -EIO;
356 }
357
358 bip->bip_buf = buf;
359 bip->bip_size = len;
360 bip->bip_sector = bio->bi_sector;
361
362 /* Map it */
363 offset = offset_in_page(buf);
364 for (i = 0 ; i < nr_pages ; i++) {
365 int ret;
366 bytes = PAGE_SIZE - offset;
367
368 if (len <= 0)
369 break;
370
371 if (bytes > len)
372 bytes = len;
373
374 ret = bio_integrity_add_page(bio, virt_to_page(buf),
375 bytes, offset);
376
377 if (ret == 0)
378 return 0;
379
380 if (ret < bytes)
381 break;
382
383 buf += bytes;
384 len -= bytes;
385 offset = 0;
386 }
387
388 /* Install custom I/O completion handler if read verify is enabled */
389 if (bio_data_dir(bio) == READ) {
390 bip->bip_end_io = bio->bi_end_io;
391 bio->bi_end_io = bio_integrity_endio;
392 }
393
394 /* Auto-generate integrity metadata if this is a write */
395 if (bio_data_dir(bio) == WRITE)
396 bio_integrity_generate(bio);
397
398 return 0;
399}
400EXPORT_SYMBOL(bio_integrity_prep);
401
402/**
403 * bio_integrity_verify - Verify integrity metadata for a bio
404 * @bio: bio to verify
405 *
406 * Description: This function is called to verify the integrity of a
407 * bio. The data in the bio io_vec is compared to the integrity
408 * metadata returned by the HBA.
409 */
410static int bio_integrity_verify(struct bio *bio)
411{
412 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
413 struct blk_integrity_exchg bix;
414 struct bio_vec *bv;
415 sector_t sector = bio->bi_integrity->bip_sector;
416 unsigned int i, sectors, total, ret;
417 void *prot_buf = bio->bi_integrity->bip_buf;
418
419 ret = total = 0;
420 bix.disk_name = bio->bi_bdev->bd_disk->disk_name;
421 bix.sector_size = bi->sector_size;
422
423 bio_for_each_segment(bv, bio, i) {
424 void *kaddr = kmap_atomic(bv->bv_page, KM_USER0);
425 bix.data_buf = kaddr + bv->bv_offset;
426 bix.data_size = bv->bv_len;
427 bix.prot_buf = prot_buf;
428 bix.sector = sector;
429
430 ret = bi->verify_fn(&bix);
431
432 if (ret) {
433 kunmap_atomic(kaddr, KM_USER0);
434 break;
435 }
436
437 sectors = bv->bv_len / bi->sector_size;
438 sector += sectors;
439 prot_buf += sectors * bi->tuple_size;
440 total += sectors * bi->tuple_size;
441 BUG_ON(total > bio->bi_integrity->bip_size);
442
443 kunmap_atomic(kaddr, KM_USER0);
444 }
445
446 return ret;
447}
448
449/**
450 * bio_integrity_verify_fn - Integrity I/O completion worker
451 * @work: Work struct stored in bio to be verified
452 *
453 * Description: This workqueue function is called to complete a READ
454 * request. The function verifies the transferred integrity metadata
455 * and then calls the original bio end_io function.
456 */
457static void bio_integrity_verify_fn(struct work_struct *work)
458{
459 struct bio_integrity_payload *bip =
460 container_of(work, struct bio_integrity_payload, bip_work);
461 struct bio *bio = bip->bip_bio;
462 int error = bip->bip_error;
463
464 if (bio_integrity_verify(bio)) {
465 clear_bit(BIO_UPTODATE, &bio->bi_flags);
466 error = -EIO;
467 }
468
469 /* Restore original bio completion handler */
470 bio->bi_end_io = bip->bip_end_io;
471
472 if (bio->bi_end_io)
473 bio->bi_end_io(bio, error);
474}
475
476/**
477 * bio_integrity_endio - Integrity I/O completion function
478 * @bio: Protected bio
479 * @error: Pointer to errno
480 *
481 * Description: Completion for integrity I/O
482 *
483 * Normally I/O completion is done in interrupt context. However,
484 * verifying I/O integrity is a time-consuming task which must be run
485 * in process context. This function postpones completion
486 * accordingly.
487 */
488void bio_integrity_endio(struct bio *bio, int error)
489{
490 struct bio_integrity_payload *bip = bio->bi_integrity;
491
492 BUG_ON(bip->bip_bio != bio);
493
494 bip->bip_error = error;
495 INIT_WORK(&bip->bip_work, bio_integrity_verify_fn);
496 queue_work(kintegrityd_wq, &bip->bip_work);
497}
498EXPORT_SYMBOL(bio_integrity_endio);
499
500/**
501 * bio_integrity_mark_head - Advance bip_vec skip bytes
502 * @bip: Integrity vector to advance
503 * @skip: Number of bytes to advance it
504 */
505void bio_integrity_mark_head(struct bio_integrity_payload *bip, unsigned int skip)
506{
507 struct bio_vec *iv;
508 unsigned int i;
509
510 bip_for_each_vec(iv, bip, i) {
511 if (skip == 0) {
512 bip->bip_idx = i;
513 return;
514 } else if (skip >= iv->bv_len) {
515 skip -= iv->bv_len;
516 } else { /* skip < iv->bv_len) */
517 iv->bv_offset += skip;
518 iv->bv_len -= skip;
519 bip->bip_idx = i;
520 return;
521 }
522 }
523}
524
525/**
526 * bio_integrity_mark_tail - Truncate bip_vec to be len bytes long
527 * @bip: Integrity vector to truncate
528 * @len: New length of integrity vector
529 */
530void bio_integrity_mark_tail(struct bio_integrity_payload *bip, unsigned int len)
531{
532 struct bio_vec *iv;
533 unsigned int i;
534
535 bip_for_each_vec(iv, bip, i) {
536 if (len == 0) {
537 bip->bip_vcnt = i;
538 return;
539 } else if (len >= iv->bv_len) {
540 len -= iv->bv_len;
541 } else { /* len < iv->bv_len) */
542 iv->bv_len = len;
543 len = 0;
544 }
545 }
546}
547
548/**
549 * bio_integrity_advance - Advance integrity vector
550 * @bio: bio whose integrity vector to update
551 * @bytes_done: number of data bytes that have been completed
552 *
553 * Description: This function calculates how many integrity bytes the
554 * number of completed data bytes correspond to and advances the
555 * integrity vector accordingly.
556 */
557void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
558{
559 struct bio_integrity_payload *bip = bio->bi_integrity;
560 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
561 unsigned int nr_sectors;
562
563 BUG_ON(bip == NULL);
564 BUG_ON(bi == NULL);
565
566 nr_sectors = bio_integrity_hw_sectors(bi, bytes_done >> 9);
567 bio_integrity_mark_head(bip, nr_sectors * bi->tuple_size);
568}
569EXPORT_SYMBOL(bio_integrity_advance);
570
571/**
572 * bio_integrity_trim - Trim integrity vector
573 * @bio: bio whose integrity vector to update
574 * @offset: offset to first data sector
575 * @sectors: number of data sectors
576 *
577 * Description: Used to trim the integrity vector in a cloned bio.
578 * The ivec will be advanced corresponding to 'offset' data sectors
579 * and the length will be truncated corresponding to 'len' data
580 * sectors.
581 */
582void bio_integrity_trim(struct bio *bio, unsigned int offset, unsigned int sectors)
583{
584 struct bio_integrity_payload *bip = bio->bi_integrity;
585 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
586 unsigned int nr_sectors;
587
588 BUG_ON(bip == NULL);
589 BUG_ON(bi == NULL);
590 BUG_ON(!bio_flagged(bio, BIO_CLONED));
591
592 nr_sectors = bio_integrity_hw_sectors(bi, sectors);
593 bip->bip_sector = bip->bip_sector + offset;
594 bio_integrity_mark_head(bip, offset * bi->tuple_size);
595 bio_integrity_mark_tail(bip, sectors * bi->tuple_size);
596}
597EXPORT_SYMBOL(bio_integrity_trim);
598
599/**
600 * bio_integrity_split - Split integrity metadata
601 * @bio: Protected bio
602 * @bp: Resulting bio_pair
603 * @sectors: Offset
604 *
605 * Description: Splits an integrity page into a bio_pair.
606 */
607void bio_integrity_split(struct bio *bio, struct bio_pair *bp, int sectors)
608{
609 struct blk_integrity *bi;
610 struct bio_integrity_payload *bip = bio->bi_integrity;
611 unsigned int nr_sectors;
612
613 if (bio_integrity(bio) == 0)
614 return;
615
616 bi = bdev_get_integrity(bio->bi_bdev);
617 BUG_ON(bi == NULL);
618 BUG_ON(bip->bip_vcnt != 1);
619
620 nr_sectors = bio_integrity_hw_sectors(bi, sectors);
621
622 bp->bio1.bi_integrity = &bp->bip1;
623 bp->bio2.bi_integrity = &bp->bip2;
624
625 bp->iv1 = bip->bip_vec[0];
626 bp->iv2 = bip->bip_vec[0];
627
628 bp->bip1.bip_vec = &bp->iv1;
629 bp->bip2.bip_vec = &bp->iv2;
630
631 bp->iv1.bv_len = sectors * bi->tuple_size;
632 bp->iv2.bv_offset += sectors * bi->tuple_size;
633 bp->iv2.bv_len -= sectors * bi->tuple_size;
634
635 bp->bip1.bip_sector = bio->bi_integrity->bip_sector;
636 bp->bip2.bip_sector = bio->bi_integrity->bip_sector + nr_sectors;
637
638 bp->bip1.bip_vcnt = bp->bip2.bip_vcnt = 1;
639 bp->bip1.bip_idx = bp->bip2.bip_idx = 0;
640}
641EXPORT_SYMBOL(bio_integrity_split);
642
643/**
644 * bio_integrity_clone - Callback for cloning bios with integrity metadata
645 * @bio: New bio
646 * @bio_src: Original bio
647 * @bs: bio_set to allocate bip from
648 *
649 * Description: Called to allocate a bip when cloning a bio
650 */
651int bio_integrity_clone(struct bio *bio, struct bio *bio_src, struct bio_set *bs)
652{
653 struct bio_integrity_payload *bip_src = bio_src->bi_integrity;
654 struct bio_integrity_payload *bip;
655
656 BUG_ON(bip_src == NULL);
657
658 bip = bio_integrity_alloc_bioset(bio, GFP_NOIO, bip_src->bip_vcnt, bs);
659
660 if (bip == NULL)
661 return -EIO;
662
663 memcpy(bip->bip_vec, bip_src->bip_vec,
664 bip_src->bip_vcnt * sizeof(struct bio_vec));
665
666 bip->bip_sector = bip_src->bip_sector;
667 bip->bip_vcnt = bip_src->bip_vcnt;
668 bip->bip_idx = bip_src->bip_idx;
669
670 return 0;
671}
672EXPORT_SYMBOL(bio_integrity_clone);
673
674int bioset_integrity_create(struct bio_set *bs, int pool_size)
675{
676 bs->bio_integrity_pool = mempool_create_slab_pool(pool_size,
677 bio_integrity_slab);
678 if (!bs->bio_integrity_pool)
679 return -1;
680
681 return 0;
682}
683EXPORT_SYMBOL(bioset_integrity_create);
684
685void bioset_integrity_free(struct bio_set *bs)
686{
687 if (bs->bio_integrity_pool)
688 mempool_destroy(bs->bio_integrity_pool);
689}
690EXPORT_SYMBOL(bioset_integrity_free);
691
692void __init bio_integrity_init_slab(void)
693{
694 bio_integrity_slab = KMEM_CACHE(bio_integrity_payload,
695 SLAB_HWCACHE_ALIGN|SLAB_PANIC);
696}
697EXPORT_SYMBOL(bio_integrity_init_slab);
698
699static int __init integrity_init(void)
700{
701 kintegrityd_wq = create_workqueue("kintegrityd");
702
703 if (!kintegrityd_wq)
704 panic("Failed to create kintegrityd\n");
705
706 return 0;
707}
708subsys_initcall(integrity_init);
diff --git a/fs/bio.c b/fs/bio.c
index 7a6598abc967..7761c84c7032 100644
--- a/fs/bio.c
+++ b/fs/bio.c
@@ -50,6 +50,11 @@ static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = {
50 */ 50 */
51struct bio_set *fs_bio_set; 51struct bio_set *fs_bio_set;
52 52
53unsigned int bvec_nr_vecs(unsigned short idx)
54{
55 return bvec_slabs[idx].nr_vecs;
56}
57
53struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx, struct bio_set *bs) 58struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx, struct bio_set *bs)
54{ 59{
55 struct bio_vec *bvl; 60 struct bio_vec *bvl;
@@ -91,6 +96,9 @@ void bio_free(struct bio *bio, struct bio_set *bio_set)
91 mempool_free(bio->bi_io_vec, bio_set->bvec_pools[pool_idx]); 96 mempool_free(bio->bi_io_vec, bio_set->bvec_pools[pool_idx]);
92 } 97 }
93 98
99 if (bio_integrity(bio))
100 bio_integrity_free(bio, bio_set);
101
94 mempool_free(bio, bio_set->bio_pool); 102 mempool_free(bio, bio_set->bio_pool);
95} 103}
96 104
@@ -249,9 +257,19 @@ struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask)
249{ 257{
250 struct bio *b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, fs_bio_set); 258 struct bio *b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, fs_bio_set);
251 259
252 if (b) { 260 if (!b)
253 b->bi_destructor = bio_fs_destructor; 261 return NULL;
254 __bio_clone(b, bio); 262
263 b->bi_destructor = bio_fs_destructor;
264 __bio_clone(b, bio);
265
266 if (bio_integrity(bio)) {
267 int ret;
268
269 ret = bio_integrity_clone(b, bio, fs_bio_set);
270
271 if (ret < 0)
272 return NULL;
255 } 273 }
256 274
257 return b; 275 return b;
@@ -1223,6 +1241,9 @@ struct bio_pair *bio_split(struct bio *bi, mempool_t *pool, int first_sectors)
1223 bp->bio1.bi_private = bi; 1241 bp->bio1.bi_private = bi;
1224 bp->bio2.bi_private = pool; 1242 bp->bio2.bi_private = pool;
1225 1243
1244 if (bio_integrity(bi))
1245 bio_integrity_split(bi, bp, first_sectors);
1246
1226 return bp; 1247 return bp;
1227} 1248}
1228 1249
@@ -1264,6 +1285,7 @@ void bioset_free(struct bio_set *bs)
1264 if (bs->bio_pool) 1285 if (bs->bio_pool)
1265 mempool_destroy(bs->bio_pool); 1286 mempool_destroy(bs->bio_pool);
1266 1287
1288 bioset_integrity_free(bs);
1267 biovec_free_pools(bs); 1289 biovec_free_pools(bs);
1268 1290
1269 kfree(bs); 1291 kfree(bs);
@@ -1280,6 +1302,9 @@ struct bio_set *bioset_create(int bio_pool_size, int bvec_pool_size)
1280 if (!bs->bio_pool) 1302 if (!bs->bio_pool)
1281 goto bad; 1303 goto bad;
1282 1304
1305 if (bioset_integrity_create(bs, bio_pool_size))
1306 goto bad;
1307
1283 if (!biovec_create_pools(bs, bvec_pool_size)) 1308 if (!biovec_create_pools(bs, bvec_pool_size))
1284 return bs; 1309 return bs;
1285 1310
@@ -1306,6 +1331,7 @@ static int __init init_bio(void)
1306{ 1331{
1307 bio_slab = KMEM_CACHE(bio, SLAB_HWCACHE_ALIGN|SLAB_PANIC); 1332 bio_slab = KMEM_CACHE(bio, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
1308 1333
1334 bio_integrity_init_slab();
1309 biovec_init_slabs(); 1335 biovec_init_slabs();
1310 1336
1311 fs_bio_set = bioset_create(BIO_POOL_SIZE, 2); 1337 fs_bio_set = bioset_create(BIO_POOL_SIZE, 2);
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 49dfb3cb7460..6bfc3e8d9d89 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -64,6 +64,7 @@ struct bio_vec {
64 64
65struct bio_set; 65struct bio_set;
66struct bio; 66struct bio;
67struct bio_integrity_payload;
67typedef void (bio_end_io_t) (struct bio *, int); 68typedef void (bio_end_io_t) (struct bio *, int);
68typedef void (bio_destructor_t) (struct bio *); 69typedef void (bio_destructor_t) (struct bio *);
69 70
@@ -112,6 +113,9 @@ struct bio {
112 atomic_t bi_cnt; /* pin count */ 113 atomic_t bi_cnt; /* pin count */
113 114
114 void *bi_private; 115 void *bi_private;
116#if defined(CONFIG_BLK_DEV_INTEGRITY)
117 struct bio_integrity_payload *bi_integrity; /* data integrity */
118#endif
115 119
116 bio_destructor_t *bi_destructor; /* destructor */ 120 bio_destructor_t *bi_destructor; /* destructor */
117}; 121};
@@ -271,6 +275,29 @@ static inline void *bio_data(struct bio *bio)
271 */ 275 */
272#define bio_get(bio) atomic_inc(&(bio)->bi_cnt) 276#define bio_get(bio) atomic_inc(&(bio)->bi_cnt)
273 277
278#if defined(CONFIG_BLK_DEV_INTEGRITY)
279/*
280 * bio integrity payload
281 */
282struct bio_integrity_payload {
283 struct bio *bip_bio; /* parent bio */
284 struct bio_vec *bip_vec; /* integrity data vector */
285
286 sector_t bip_sector; /* virtual start sector */
287
288 void *bip_buf; /* generated integrity data */
289 bio_end_io_t *bip_end_io; /* saved I/O completion fn */
290
291 int bip_error; /* saved I/O error */
292 unsigned int bip_size;
293
294 unsigned short bip_pool; /* pool the ivec came from */
295 unsigned short bip_vcnt; /* # of integrity bio_vecs */
296 unsigned short bip_idx; /* current bip_vec index */
297
298 struct work_struct bip_work; /* I/O completion */
299};
300#endif /* CONFIG_BLK_DEV_INTEGRITY */
274 301
275/* 302/*
276 * A bio_pair is used when we need to split a bio. 303 * A bio_pair is used when we need to split a bio.
@@ -283,10 +310,14 @@ static inline void *bio_data(struct bio *bio)
283 * in bio2.bi_private 310 * in bio2.bi_private
284 */ 311 */
285struct bio_pair { 312struct bio_pair {
286 struct bio bio1, bio2; 313 struct bio bio1, bio2;
287 struct bio_vec bv1, bv2; 314 struct bio_vec bv1, bv2;
288 atomic_t cnt; 315#if defined(CONFIG_BLK_DEV_INTEGRITY)
289 int error; 316 struct bio_integrity_payload bip1, bip2;
317 struct bio_vec iv1, iv2;
318#endif
319 atomic_t cnt;
320 int error;
290}; 321};
291extern struct bio_pair *bio_split(struct bio *bi, mempool_t *pool, 322extern struct bio_pair *bio_split(struct bio *bi, mempool_t *pool,
292 int first_sectors); 323 int first_sectors);
@@ -334,6 +365,7 @@ extern struct bio *bio_copy_user_iov(struct request_queue *, struct sg_iovec *,
334extern int bio_uncopy_user(struct bio *); 365extern int bio_uncopy_user(struct bio *);
335void zero_fill_bio(struct bio *bio); 366void zero_fill_bio(struct bio *bio);
336extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set *); 367extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set *);
368extern unsigned int bvec_nr_vecs(unsigned short idx);
337 369
338/* 370/*
339 * bio_set is used to allow other portions of the IO system to 371 * bio_set is used to allow other portions of the IO system to
@@ -346,6 +378,9 @@ extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set
346 378
347struct bio_set { 379struct bio_set {
348 mempool_t *bio_pool; 380 mempool_t *bio_pool;
381#if defined(CONFIG_BLK_DEV_INTEGRITY)
382 mempool_t *bio_integrity_pool;
383#endif
349 mempool_t *bvec_pools[BIOVEC_NR_POOLS]; 384 mempool_t *bvec_pools[BIOVEC_NR_POOLS];
350}; 385};
351 386
@@ -410,5 +445,56 @@ static inline char *__bio_kmap_irq(struct bio *bio, unsigned short idx,
410 __bio_kmap_irq((bio), (bio)->bi_idx, (flags)) 445 __bio_kmap_irq((bio), (bio)->bi_idx, (flags))
411#define bio_kunmap_irq(buf,flags) __bio_kunmap_irq(buf, flags) 446#define bio_kunmap_irq(buf,flags) __bio_kunmap_irq(buf, flags)
412 447
448#if defined(CONFIG_BLK_DEV_INTEGRITY)
449
450#define bip_vec_idx(bip, idx) (&(bip->bip_vec[(idx)]))
451#define bip_vec(bip) bip_vec_idx(bip, 0)
452
453#define __bip_for_each_vec(bvl, bip, i, start_idx) \
454 for (bvl = bip_vec_idx((bip), (start_idx)), i = (start_idx); \
455 i < (bip)->bip_vcnt; \
456 bvl++, i++)
457
458#define bip_for_each_vec(bvl, bip, i) \
459 __bip_for_each_vec(bvl, bip, i, (bip)->bip_idx)
460
461#define bio_integrity(bio) ((bio)->bi_integrity ? 1 : 0)
462
463extern struct bio_integrity_payload *bio_integrity_alloc_bioset(struct bio *, gfp_t, unsigned int, struct bio_set *);
464extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
465extern void bio_integrity_free(struct bio *, struct bio_set *);
466extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
467extern int bio_integrity_enabled(struct bio *bio);
468extern int bio_integrity_set_tag(struct bio *, void *, unsigned int);
469extern int bio_integrity_get_tag(struct bio *, void *, unsigned int);
470extern int bio_integrity_prep(struct bio *);
471extern void bio_integrity_endio(struct bio *, int);
472extern void bio_integrity_advance(struct bio *, unsigned int);
473extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int);
474extern void bio_integrity_split(struct bio *, struct bio_pair *, int);
475extern int bio_integrity_clone(struct bio *, struct bio *, struct bio_set *);
476extern int bioset_integrity_create(struct bio_set *, int);
477extern void bioset_integrity_free(struct bio_set *);
478extern void bio_integrity_init_slab(void);
479
480#else /* CONFIG_BLK_DEV_INTEGRITY */
481
482#define bio_integrity(a) (0)
483#define bioset_integrity_create(a, b) (0)
484#define bio_integrity_prep(a) (0)
485#define bio_integrity_enabled(a) (0)
486#define bio_integrity_clone(a, b, c) (0)
487#define bioset_integrity_free(a) do { } while (0)
488#define bio_integrity_free(a, b) do { } while (0)
489#define bio_integrity_endio(a, b) do { } while (0)
490#define bio_integrity_advance(a, b) do { } while (0)
491#define bio_integrity_trim(a, b, c) do { } while (0)
492#define bio_integrity_split(a, b, c) do { } while (0)
493#define bio_integrity_set_tag(a, b, c) do { } while (0)
494#define bio_integrity_get_tag(a, b, c) do { } while (0)
495#define bio_integrity_init_slab(a) do { } while (0)
496
497#endif /* CONFIG_BLK_DEV_INTEGRITY */
498
413#endif /* CONFIG_BLOCK */ 499#endif /* CONFIG_BLOCK */
414#endif /* __LINUX_BIO_H */ 500#endif /* __LINUX_BIO_H */
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 6a3da6717135..4a9ed45270ff 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -112,6 +112,7 @@ enum rq_flag_bits {
112 __REQ_ALLOCED, /* request came from our alloc pool */ 112 __REQ_ALLOCED, /* request came from our alloc pool */
113 __REQ_RW_META, /* metadata io request */ 113 __REQ_RW_META, /* metadata io request */
114 __REQ_COPY_USER, /* contains copies of user pages */ 114 __REQ_COPY_USER, /* contains copies of user pages */
115 __REQ_INTEGRITY, /* integrity metadata has been remapped */
115 __REQ_NR_BITS, /* stops here */ 116 __REQ_NR_BITS, /* stops here */
116}; 117};
117 118
@@ -134,6 +135,7 @@ enum rq_flag_bits {
134#define REQ_ALLOCED (1 << __REQ_ALLOCED) 135#define REQ_ALLOCED (1 << __REQ_ALLOCED)
135#define REQ_RW_META (1 << __REQ_RW_META) 136#define REQ_RW_META (1 << __REQ_RW_META)
136#define REQ_COPY_USER (1 << __REQ_COPY_USER) 137#define REQ_COPY_USER (1 << __REQ_COPY_USER)
138#define REQ_INTEGRITY (1 << __REQ_INTEGRITY)
137 139
138#define BLK_MAX_CDB 16 140#define BLK_MAX_CDB 16
139 141
@@ -865,6 +867,109 @@ void kblockd_flush_work(struct work_struct *work);
865 MODULE_ALIAS("block-major-" __stringify(major) "-*") 867 MODULE_ALIAS("block-major-" __stringify(major) "-*")
866 868
867 869
870#if defined(CONFIG_BLK_DEV_INTEGRITY)
871
872#define INTEGRITY_FLAG_READ 1 /* verify data integrity on read */
873#define INTEGRITY_FLAG_WRITE 2 /* generate data integrity on write */
874
875struct blk_integrity_exchg {
876 void *prot_buf;
877 void *data_buf;
878 sector_t sector;
879 unsigned int data_size;
880 unsigned short sector_size;
881 const char *disk_name;
882};
883
884typedef void (integrity_gen_fn) (struct blk_integrity_exchg *);
885typedef int (integrity_vrfy_fn) (struct blk_integrity_exchg *);
886typedef void (integrity_set_tag_fn) (void *, void *, unsigned int);
887typedef void (integrity_get_tag_fn) (void *, void *, unsigned int);
888
889struct blk_integrity {
890 integrity_gen_fn *generate_fn;
891 integrity_vrfy_fn *verify_fn;
892 integrity_set_tag_fn *set_tag_fn;
893 integrity_get_tag_fn *get_tag_fn;
894
895 unsigned short flags;
896 unsigned short tuple_size;
897 unsigned short sector_size;
898 unsigned short tag_size;
899
900 const char *name;
901
902 struct kobject kobj;
903};
904
905extern int blk_integrity_register(struct gendisk *, struct blk_integrity *);
906extern void blk_integrity_unregister(struct gendisk *);
907extern int blk_integrity_compare(struct block_device *, struct block_device *);
908extern int blk_rq_map_integrity_sg(struct request *, struct scatterlist *);
909extern int blk_rq_count_integrity_sg(struct request *);
910
911static inline unsigned short blk_integrity_tuple_size(struct blk_integrity *bi)
912{
913 if (bi)
914 return bi->tuple_size;
915
916 return 0;
917}
918
919static inline struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
920{
921 return bdev->bd_disk->integrity;
922}
923
924static inline unsigned int bdev_get_tag_size(struct block_device *bdev)
925{
926 struct blk_integrity *bi = bdev_get_integrity(bdev);
927
928 if (bi)
929 return bi->tag_size;
930
931 return 0;
932}
933
934static inline int bdev_integrity_enabled(struct block_device *bdev, int rw)
935{
936 struct blk_integrity *bi = bdev_get_integrity(bdev);
937
938 if (bi == NULL)
939 return 0;
940
941 if (rw == READ && bi->verify_fn != NULL &&
942 test_bit(INTEGRITY_FLAG_READ, &bi->flags))
943 return 1;
944
945 if (rw == WRITE && bi->generate_fn != NULL &&
946 test_bit(INTEGRITY_FLAG_WRITE, &bi->flags))
947 return 1;
948
949 return 0;
950}
951
952static inline int blk_integrity_rq(struct request *rq)
953{
954 BUG_ON(rq->bio == NULL);
955
956 return bio_integrity(rq->bio);
957}
958
959#else /* CONFIG_BLK_DEV_INTEGRITY */
960
961#define blk_integrity_rq(rq) (0)
962#define blk_rq_count_integrity_sg(a) (0)
963#define blk_rq_map_integrity_sg(a, b) (0)
964#define bdev_get_integrity(a) (0)
965#define bdev_get_tag_size(a) (0)
966#define blk_integrity_compare(a, b) (0)
967#define blk_integrity_register(a, b) (0)
968#define blk_integrity_unregister(a) do { } while (0);
969
970#endif /* CONFIG_BLK_DEV_INTEGRITY */
971
972
868#else /* CONFIG_BLOCK */ 973#else /* CONFIG_BLOCK */
869/* 974/*
870 * stubs for when the block layer is configured out 975 * stubs for when the block layer is configured out
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index ae7aec3cabee..524ec96f5a23 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -141,6 +141,9 @@ struct gendisk {
141 struct disk_stats dkstats; 141 struct disk_stats dkstats;
142#endif 142#endif
143 struct work_struct async_notify; 143 struct work_struct async_notify;
144#ifdef CONFIG_BLK_DEV_INTEGRITY
145 struct blk_integrity *integrity;
146#endif
144}; 147};
145 148
146/* 149/*