diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2009-07-01 13:41:09 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2009-07-01 13:41:09 -0400 |
commit | 2027bd9f9290ede1664d5fa18b3fbd6c5f7401d3 (patch) | |
tree | ba5a8fe8e2373744edc22974571a953840d6c9e8 /fs | |
parent | 544ae5f96e14998cabc637fa20cf409eb92a0dd0 (diff) | |
parent | b706f64281b24d8b1fdc8ae883700131d365c412 (diff) |
Merge branch 'for-linus' of git://git.kernel.dk/linux-2.6-block
* 'for-linus' of git://git.kernel.dk/linux-2.6-block:
cfq-iosched: remove redundant check for NULL cfqq in cfq_set_request()
blocK: Restore barrier support for md and probably other virtual devices.
block: get rid of queue-private command filter
block: Create bip slabs with embedded integrity vectors
cfq-iosched: get rid of the need for __GFP_NOFAIL in cfq_find_alloc_queue()
cfq-iosched: move cfqq initialization out of cfq_find_alloc_queue()
Trivial typo fixes in Documentation/block/data-integrity.txt.
Diffstat (limited to 'fs')
-rw-r--r-- | fs/bio-integrity.c | 170 | ||||
-rw-r--r-- | fs/bio.c | 11 |
2 files changed, 133 insertions, 48 deletions
diff --git a/fs/bio-integrity.c b/fs/bio-integrity.c index 31c46a241bac..49a34e7f7306 100644 --- a/fs/bio-integrity.c +++ b/fs/bio-integrity.c | |||
@@ -1,7 +1,7 @@ | |||
1 | /* | 1 | /* |
2 | * bio-integrity.c - bio data integrity extensions | 2 | * bio-integrity.c - bio data integrity extensions |
3 | * | 3 | * |
4 | * Copyright (C) 2007, 2008 Oracle Corporation | 4 | * Copyright (C) 2007, 2008, 2009 Oracle Corporation |
5 | * Written by: Martin K. Petersen <martin.petersen@oracle.com> | 5 | * Written by: Martin K. Petersen <martin.petersen@oracle.com> |
6 | * | 6 | * |
7 | * This program is free software; you can redistribute it and/or | 7 | * This program is free software; you can redistribute it and/or |
@@ -25,63 +25,121 @@ | |||
25 | #include <linux/bio.h> | 25 | #include <linux/bio.h> |
26 | #include <linux/workqueue.h> | 26 | #include <linux/workqueue.h> |
27 | 27 | ||
28 | static struct kmem_cache *bio_integrity_slab __read_mostly; | 28 | struct integrity_slab { |
29 | static mempool_t *bio_integrity_pool; | 29 | struct kmem_cache *slab; |
30 | static struct bio_set *integrity_bio_set; | 30 | unsigned short nr_vecs; |
31 | char name[8]; | ||
32 | }; | ||
33 | |||
34 | #define IS(x) { .nr_vecs = x, .name = "bip-"__stringify(x) } | ||
35 | struct integrity_slab bip_slab[BIOVEC_NR_POOLS] __read_mostly = { | ||
36 | IS(1), IS(4), IS(16), IS(64), IS(128), IS(BIO_MAX_PAGES), | ||
37 | }; | ||
38 | #undef IS | ||
39 | |||
31 | static struct workqueue_struct *kintegrityd_wq; | 40 | static struct workqueue_struct *kintegrityd_wq; |
32 | 41 | ||
42 | static inline unsigned int vecs_to_idx(unsigned int nr) | ||
43 | { | ||
44 | switch (nr) { | ||
45 | case 1: | ||
46 | return 0; | ||
47 | case 2 ... 4: | ||
48 | return 1; | ||
49 | case 5 ... 16: | ||
50 | return 2; | ||
51 | case 17 ... 64: | ||
52 | return 3; | ||
53 | case 65 ... 128: | ||
54 | return 4; | ||
55 | case 129 ... BIO_MAX_PAGES: | ||
56 | return 5; | ||
57 | default: | ||
58 | BUG(); | ||
59 | } | ||
60 | } | ||
61 | |||
62 | static inline int use_bip_pool(unsigned int idx) | ||
63 | { | ||
64 | if (idx == BIOVEC_NR_POOLS) | ||
65 | return 1; | ||
66 | |||
67 | return 0; | ||
68 | } | ||
69 | |||
33 | /** | 70 | /** |
34 | * bio_integrity_alloc - Allocate integrity payload and attach it to bio | 71 | * bio_integrity_alloc_bioset - Allocate integrity payload and attach it to bio |
35 | * @bio: bio to attach integrity metadata to | 72 | * @bio: bio to attach integrity metadata to |
36 | * @gfp_mask: Memory allocation mask | 73 | * @gfp_mask: Memory allocation mask |
37 | * @nr_vecs: Number of integrity metadata scatter-gather elements | 74 | * @nr_vecs: Number of integrity metadata scatter-gather elements |
75 | * @bs: bio_set to allocate from | ||
38 | * | 76 | * |
39 | * Description: This function prepares a bio for attaching integrity | 77 | * Description: This function prepares a bio for attaching integrity |
40 | * metadata. nr_vecs specifies the maximum number of pages containing | 78 | * metadata. nr_vecs specifies the maximum number of pages containing |
41 | * integrity metadata that can be attached. | 79 | * integrity metadata that can be attached. |
42 | */ | 80 | */ |
43 | struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, | 81 | struct bio_integrity_payload *bio_integrity_alloc_bioset(struct bio *bio, |
44 | gfp_t gfp_mask, | 82 | gfp_t gfp_mask, |
45 | unsigned int nr_vecs) | 83 | unsigned int nr_vecs, |
84 | struct bio_set *bs) | ||
46 | { | 85 | { |
47 | struct bio_integrity_payload *bip; | 86 | struct bio_integrity_payload *bip; |
48 | struct bio_vec *iv; | 87 | unsigned int idx = vecs_to_idx(nr_vecs); |
49 | unsigned long idx; | ||
50 | 88 | ||
51 | BUG_ON(bio == NULL); | 89 | BUG_ON(bio == NULL); |
90 | bip = NULL; | ||
52 | 91 | ||
53 | bip = mempool_alloc(bio_integrity_pool, gfp_mask); | 92 | /* Lower order allocations come straight from slab */ |
54 | if (unlikely(bip == NULL)) { | 93 | if (!use_bip_pool(idx)) |
55 | printk(KERN_ERR "%s: could not alloc bip\n", __func__); | 94 | bip = kmem_cache_alloc(bip_slab[idx].slab, gfp_mask); |
56 | return NULL; | ||
57 | } | ||
58 | 95 | ||
59 | memset(bip, 0, sizeof(*bip)); | 96 | /* Use mempool if lower order alloc failed or max vecs were requested */ |
97 | if (bip == NULL) { | ||
98 | bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask); | ||
60 | 99 | ||
61 | iv = bvec_alloc_bs(gfp_mask, nr_vecs, &idx, integrity_bio_set); | 100 | if (unlikely(bip == NULL)) { |
62 | if (unlikely(iv == NULL)) { | 101 | printk(KERN_ERR "%s: could not alloc bip\n", __func__); |
63 | printk(KERN_ERR "%s: could not alloc bip_vec\n", __func__); | 102 | return NULL; |
64 | mempool_free(bip, bio_integrity_pool); | 103 | } |
65 | return NULL; | ||
66 | } | 104 | } |
67 | 105 | ||
68 | bip->bip_pool = idx; | 106 | memset(bip, 0, sizeof(*bip)); |
69 | bip->bip_vec = iv; | 107 | |
108 | bip->bip_slab = idx; | ||
70 | bip->bip_bio = bio; | 109 | bip->bip_bio = bio; |
71 | bio->bi_integrity = bip; | 110 | bio->bi_integrity = bip; |
72 | 111 | ||
73 | return bip; | 112 | return bip; |
74 | } | 113 | } |
114 | EXPORT_SYMBOL(bio_integrity_alloc_bioset); | ||
115 | |||
116 | /** | ||
117 | * bio_integrity_alloc - Allocate integrity payload and attach it to bio | ||
118 | * @bio: bio to attach integrity metadata to | ||
119 | * @gfp_mask: Memory allocation mask | ||
120 | * @nr_vecs: Number of integrity metadata scatter-gather elements | ||
121 | * | ||
122 | * Description: This function prepares a bio for attaching integrity | ||
123 | * metadata. nr_vecs specifies the maximum number of pages containing | ||
124 | * integrity metadata that can be attached. | ||
125 | */ | ||
126 | struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, | ||
127 | gfp_t gfp_mask, | ||
128 | unsigned int nr_vecs) | ||
129 | { | ||
130 | return bio_integrity_alloc_bioset(bio, gfp_mask, nr_vecs, fs_bio_set); | ||
131 | } | ||
75 | EXPORT_SYMBOL(bio_integrity_alloc); | 132 | EXPORT_SYMBOL(bio_integrity_alloc); |
76 | 133 | ||
77 | /** | 134 | /** |
78 | * bio_integrity_free - Free bio integrity payload | 135 | * bio_integrity_free - Free bio integrity payload |
79 | * @bio: bio containing bip to be freed | 136 | * @bio: bio containing bip to be freed |
137 | * @bs: bio_set this bio was allocated from | ||
80 | * | 138 | * |
81 | * Description: Used to free the integrity portion of a bio. Usually | 139 | * Description: Used to free the integrity portion of a bio. Usually |
82 | * called from bio_free(). | 140 | * called from bio_free(). |
83 | */ | 141 | */ |
84 | void bio_integrity_free(struct bio *bio) | 142 | void bio_integrity_free(struct bio *bio, struct bio_set *bs) |
85 | { | 143 | { |
86 | struct bio_integrity_payload *bip = bio->bi_integrity; | 144 | struct bio_integrity_payload *bip = bio->bi_integrity; |
87 | 145 | ||
@@ -92,8 +150,10 @@ void bio_integrity_free(struct bio *bio) | |||
92 | && bip->bip_buf != NULL) | 150 | && bip->bip_buf != NULL) |
93 | kfree(bip->bip_buf); | 151 | kfree(bip->bip_buf); |
94 | 152 | ||
95 | bvec_free_bs(integrity_bio_set, bip->bip_vec, bip->bip_pool); | 153 | if (use_bip_pool(bip->bip_slab)) |
96 | mempool_free(bip, bio_integrity_pool); | 154 | mempool_free(bip, bs->bio_integrity_pool); |
155 | else | ||
156 | kmem_cache_free(bip_slab[bip->bip_slab].slab, bip); | ||
97 | 157 | ||
98 | bio->bi_integrity = NULL; | 158 | bio->bi_integrity = NULL; |
99 | } | 159 | } |
@@ -114,7 +174,7 @@ int bio_integrity_add_page(struct bio *bio, struct page *page, | |||
114 | struct bio_integrity_payload *bip = bio->bi_integrity; | 174 | struct bio_integrity_payload *bip = bio->bi_integrity; |
115 | struct bio_vec *iv; | 175 | struct bio_vec *iv; |
116 | 176 | ||
117 | if (bip->bip_vcnt >= bvec_nr_vecs(bip->bip_pool)) { | 177 | if (bip->bip_vcnt >= bvec_nr_vecs(bip->bip_slab)) { |
118 | printk(KERN_ERR "%s: bip_vec full\n", __func__); | 178 | printk(KERN_ERR "%s: bip_vec full\n", __func__); |
119 | return 0; | 179 | return 0; |
120 | } | 180 | } |
@@ -647,8 +707,8 @@ void bio_integrity_split(struct bio *bio, struct bio_pair *bp, int sectors) | |||
647 | bp->iv1 = bip->bip_vec[0]; | 707 | bp->iv1 = bip->bip_vec[0]; |
648 | bp->iv2 = bip->bip_vec[0]; | 708 | bp->iv2 = bip->bip_vec[0]; |
649 | 709 | ||
650 | bp->bip1.bip_vec = &bp->iv1; | 710 | bp->bip1.bip_vec[0] = bp->iv1; |
651 | bp->bip2.bip_vec = &bp->iv2; | 711 | bp->bip2.bip_vec[0] = bp->iv2; |
652 | 712 | ||
653 | bp->iv1.bv_len = sectors * bi->tuple_size; | 713 | bp->iv1.bv_len = sectors * bi->tuple_size; |
654 | bp->iv2.bv_offset += sectors * bi->tuple_size; | 714 | bp->iv2.bv_offset += sectors * bi->tuple_size; |
@@ -667,17 +727,19 @@ EXPORT_SYMBOL(bio_integrity_split); | |||
667 | * @bio: New bio | 727 | * @bio: New bio |
668 | * @bio_src: Original bio | 728 | * @bio_src: Original bio |
669 | * @gfp_mask: Memory allocation mask | 729 | * @gfp_mask: Memory allocation mask |
730 | * @bs: bio_set to allocate bip from | ||
670 | * | 731 | * |
671 | * Description: Called to allocate a bip when cloning a bio | 732 | * Description: Called to allocate a bip when cloning a bio |
672 | */ | 733 | */ |
673 | int bio_integrity_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp_mask) | 734 | int bio_integrity_clone(struct bio *bio, struct bio *bio_src, |
735 | gfp_t gfp_mask, struct bio_set *bs) | ||
674 | { | 736 | { |
675 | struct bio_integrity_payload *bip_src = bio_src->bi_integrity; | 737 | struct bio_integrity_payload *bip_src = bio_src->bi_integrity; |
676 | struct bio_integrity_payload *bip; | 738 | struct bio_integrity_payload *bip; |
677 | 739 | ||
678 | BUG_ON(bip_src == NULL); | 740 | BUG_ON(bip_src == NULL); |
679 | 741 | ||
680 | bip = bio_integrity_alloc(bio, gfp_mask, bip_src->bip_vcnt); | 742 | bip = bio_integrity_alloc_bioset(bio, gfp_mask, bip_src->bip_vcnt, bs); |
681 | 743 | ||
682 | if (bip == NULL) | 744 | if (bip == NULL) |
683 | return -EIO; | 745 | return -EIO; |
@@ -693,25 +755,43 @@ int bio_integrity_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp_mask) | |||
693 | } | 755 | } |
694 | EXPORT_SYMBOL(bio_integrity_clone); | 756 | EXPORT_SYMBOL(bio_integrity_clone); |
695 | 757 | ||
696 | static int __init bio_integrity_init(void) | 758 | int bioset_integrity_create(struct bio_set *bs, int pool_size) |
697 | { | 759 | { |
698 | kintegrityd_wq = create_workqueue("kintegrityd"); | 760 | unsigned int max_slab = vecs_to_idx(BIO_MAX_PAGES); |
761 | |||
762 | bs->bio_integrity_pool = | ||
763 | mempool_create_slab_pool(pool_size, bip_slab[max_slab].slab); | ||
699 | 764 | ||
765 | if (!bs->bio_integrity_pool) | ||
766 | return -1; | ||
767 | |||
768 | return 0; | ||
769 | } | ||
770 | EXPORT_SYMBOL(bioset_integrity_create); | ||
771 | |||
772 | void bioset_integrity_free(struct bio_set *bs) | ||
773 | { | ||
774 | if (bs->bio_integrity_pool) | ||
775 | mempool_destroy(bs->bio_integrity_pool); | ||
776 | } | ||
777 | EXPORT_SYMBOL(bioset_integrity_free); | ||
778 | |||
779 | void __init bio_integrity_init(void) | ||
780 | { | ||
781 | unsigned int i; | ||
782 | |||
783 | kintegrityd_wq = create_workqueue("kintegrityd"); | ||
700 | if (!kintegrityd_wq) | 784 | if (!kintegrityd_wq) |
701 | panic("Failed to create kintegrityd\n"); | 785 | panic("Failed to create kintegrityd\n"); |
702 | 786 | ||
703 | bio_integrity_slab = KMEM_CACHE(bio_integrity_payload, | 787 | for (i = 0 ; i < BIOVEC_NR_POOLS ; i++) { |
704 | SLAB_HWCACHE_ALIGN|SLAB_PANIC); | 788 | unsigned int size; |
705 | 789 | ||
706 | bio_integrity_pool = mempool_create_slab_pool(BIO_POOL_SIZE, | 790 | size = sizeof(struct bio_integrity_payload) |
707 | bio_integrity_slab); | 791 | + bip_slab[i].nr_vecs * sizeof(struct bio_vec); |
708 | if (!bio_integrity_pool) | ||
709 | panic("bio_integrity: can't allocate bip pool\n"); | ||
710 | 792 | ||
711 | integrity_bio_set = bioset_create(BIO_POOL_SIZE, 0); | 793 | bip_slab[i].slab = |
712 | if (!integrity_bio_set) | 794 | kmem_cache_create(bip_slab[i].name, size, 0, |
713 | panic("bio_integrity: can't allocate bio_set\n"); | 795 | SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); |
714 | 796 | } | |
715 | return 0; | ||
716 | } | 797 | } |
717 | subsys_initcall(bio_integrity_init); | ||
@@ -238,7 +238,7 @@ void bio_free(struct bio *bio, struct bio_set *bs) | |||
238 | bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio)); | 238 | bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio)); |
239 | 239 | ||
240 | if (bio_integrity(bio)) | 240 | if (bio_integrity(bio)) |
241 | bio_integrity_free(bio); | 241 | bio_integrity_free(bio, bs); |
242 | 242 | ||
243 | /* | 243 | /* |
244 | * If we have front padding, adjust the bio pointer before freeing | 244 | * If we have front padding, adjust the bio pointer before freeing |
@@ -341,7 +341,7 @@ struct bio *bio_alloc(gfp_t gfp_mask, int nr_iovecs) | |||
341 | static void bio_kmalloc_destructor(struct bio *bio) | 341 | static void bio_kmalloc_destructor(struct bio *bio) |
342 | { | 342 | { |
343 | if (bio_integrity(bio)) | 343 | if (bio_integrity(bio)) |
344 | bio_integrity_free(bio); | 344 | bio_integrity_free(bio, fs_bio_set); |
345 | kfree(bio); | 345 | kfree(bio); |
346 | } | 346 | } |
347 | 347 | ||
@@ -472,7 +472,7 @@ struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask) | |||
472 | if (bio_integrity(bio)) { | 472 | if (bio_integrity(bio)) { |
473 | int ret; | 473 | int ret; |
474 | 474 | ||
475 | ret = bio_integrity_clone(b, bio, gfp_mask); | 475 | ret = bio_integrity_clone(b, bio, gfp_mask, fs_bio_set); |
476 | 476 | ||
477 | if (ret < 0) { | 477 | if (ret < 0) { |
478 | bio_put(b); | 478 | bio_put(b); |
@@ -1539,6 +1539,7 @@ void bioset_free(struct bio_set *bs) | |||
1539 | if (bs->bio_pool) | 1539 | if (bs->bio_pool) |
1540 | mempool_destroy(bs->bio_pool); | 1540 | mempool_destroy(bs->bio_pool); |
1541 | 1541 | ||
1542 | bioset_integrity_free(bs); | ||
1542 | biovec_free_pools(bs); | 1543 | biovec_free_pools(bs); |
1543 | bio_put_slab(bs); | 1544 | bio_put_slab(bs); |
1544 | 1545 | ||
@@ -1579,6 +1580,9 @@ struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad) | |||
1579 | if (!bs->bio_pool) | 1580 | if (!bs->bio_pool) |
1580 | goto bad; | 1581 | goto bad; |
1581 | 1582 | ||
1583 | if (bioset_integrity_create(bs, pool_size)) | ||
1584 | goto bad; | ||
1585 | |||
1582 | if (!biovec_create_pools(bs, pool_size)) | 1586 | if (!biovec_create_pools(bs, pool_size)) |
1583 | return bs; | 1587 | return bs; |
1584 | 1588 | ||
@@ -1616,6 +1620,7 @@ static int __init init_bio(void) | |||
1616 | if (!bio_slabs) | 1620 | if (!bio_slabs) |
1617 | panic("bio: can't allocate bios\n"); | 1621 | panic("bio: can't allocate bios\n"); |
1618 | 1622 | ||
1623 | bio_integrity_init(); | ||
1619 | biovec_init_slabs(); | 1624 | biovec_init_slabs(); |
1620 | 1625 | ||
1621 | fs_bio_set = bioset_create(BIO_POOL_SIZE, 0); | 1626 | fs_bio_set = bioset_create(BIO_POOL_SIZE, 0); |