aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--fs/bio.c27
-rw-r--r--include/linux/bio.h12
2 files changed, 34 insertions, 5 deletions
diff --git a/fs/bio.c b/fs/bio.c
index 0146f80789e9..75e6be18ecd3 100644
--- a/fs/bio.c
+++ b/fs/bio.c
@@ -31,6 +31,12 @@
31 31
32DEFINE_TRACE(block_split); 32DEFINE_TRACE(block_split);
33 33
34/*
35 * Test patch to inline a certain number of bi_io_vec's inside the bio
36 * itself, to shrink a bio data allocation from two mempool calls to one
37 */
38#define BIO_INLINE_VECS 4
39
34static mempool_t *bio_split_pool __read_mostly; 40static mempool_t *bio_split_pool __read_mostly;
35 41
36/* 42/*
@@ -241,7 +247,7 @@ void bio_free(struct bio *bio, struct bio_set *bs)
241{ 247{
242 void *p; 248 void *p;
243 249
244 if (bio->bi_io_vec) 250 if (bio_has_allocated_vec(bio))
245 bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio)); 251 bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio));
246 252
247 if (bio_integrity(bio)) 253 if (bio_integrity(bio))
@@ -267,7 +273,8 @@ static void bio_fs_destructor(struct bio *bio)
267 273
268static void bio_kmalloc_destructor(struct bio *bio) 274static void bio_kmalloc_destructor(struct bio *bio)
269{ 275{
270 kfree(bio->bi_io_vec); 276 if (bio_has_allocated_vec(bio))
277 kfree(bio->bi_io_vec);
271 kfree(bio); 278 kfree(bio);
272} 279}
273 280
@@ -314,7 +321,16 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
314 if (likely(nr_iovecs)) { 321 if (likely(nr_iovecs)) {
315 unsigned long uninitialized_var(idx); 322 unsigned long uninitialized_var(idx);
316 323
317 bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs); 324 if (nr_iovecs <= BIO_INLINE_VECS) {
325 idx = 0;
326 bvl = bio->bi_inline_vecs;
327 nr_iovecs = BIO_INLINE_VECS;
328 memset(bvl, 0, BIO_INLINE_VECS * sizeof(*bvl));
329 } else {
330 bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx,
331 bs);
332 nr_iovecs = bvec_nr_vecs(idx);
333 }
318 if (unlikely(!bvl)) { 334 if (unlikely(!bvl)) {
319 if (bs) 335 if (bs)
320 mempool_free(bio, bs->bio_pool); 336 mempool_free(bio, bs->bio_pool);
@@ -324,7 +340,7 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
324 goto out; 340 goto out;
325 } 341 }
326 bio->bi_flags |= idx << BIO_POOL_OFFSET; 342 bio->bi_flags |= idx << BIO_POOL_OFFSET;
327 bio->bi_max_vecs = bvec_nr_vecs(idx); 343 bio->bi_max_vecs = nr_iovecs;
328 } 344 }
329 bio->bi_io_vec = bvl; 345 bio->bi_io_vec = bvl;
330 } 346 }
@@ -1525,6 +1541,7 @@ void bioset_free(struct bio_set *bs)
1525 */ 1541 */
1526struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad) 1542struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
1527{ 1543{
1544 unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1528 struct bio_set *bs; 1545 struct bio_set *bs;
1529 1546
1530 bs = kzalloc(sizeof(*bs), GFP_KERNEL); 1547 bs = kzalloc(sizeof(*bs), GFP_KERNEL);
@@ -1533,7 +1550,7 @@ struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
1533 1550
1534 bs->front_pad = front_pad; 1551 bs->front_pad = front_pad;
1535 1552
1536 bs->bio_slab = bio_find_or_create_slab(front_pad); 1553 bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
1537 if (!bs->bio_slab) { 1554 if (!bs->bio_slab) {
1538 kfree(bs); 1555 kfree(bs);
1539 return NULL; 1556 return NULL;
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 4b80d3537f97..18462c5b8fff 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -102,6 +102,13 @@ struct bio {
102#endif 102#endif
103 103
104 bio_destructor_t *bi_destructor; /* destructor */ 104 bio_destructor_t *bi_destructor; /* destructor */
105
106 /*
107 * We can inline a number of vecs at the end of the bio, to avoid
108 * double allocations for a small number of bio_vecs. This member
109 * MUST obviously be kept at the very end of the bio.
110 */
111 struct bio_vec bi_inline_vecs[0];
105}; 112};
106 113
107/* 114/*
@@ -213,6 +220,11 @@ static inline void *bio_data(struct bio *bio)
213 return NULL; 220 return NULL;
214} 221}
215 222
223static inline int bio_has_allocated_vec(struct bio *bio)
224{
225 return bio->bi_io_vec && bio->bi_io_vec != bio->bi_inline_vecs;
226}
227
216/* 228/*
217 * will die 229 * will die
218 */ 230 */