aboutsummaryrefslogtreecommitdiffstats
path: root/block/bio.c
diff options
context:
space:
mode:
Diffstat (limited to 'block/bio.c')
-rw-r--r--block/bio.c2038
1 files changed, 2038 insertions, 0 deletions
diff --git a/block/bio.c b/block/bio.c
new file mode 100644
index 000000000000..96d28eee8a1e
--- /dev/null
+++ b/block/bio.c
@@ -0,0 +1,2038 @@
1/*
2 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public Licens
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
16 *
17 */
18#include <linux/mm.h>
19#include <linux/swap.h>
20#include <linux/bio.h>
21#include <linux/blkdev.h>
22#include <linux/uio.h>
23#include <linux/iocontext.h>
24#include <linux/slab.h>
25#include <linux/init.h>
26#include <linux/kernel.h>
27#include <linux/export.h>
28#include <linux/mempool.h>
29#include <linux/workqueue.h>
30#include <linux/cgroup.h>
31#include <scsi/sg.h> /* for struct sg_iovec */
32
33#include <trace/events/block.h>
34
35/*
36 * Test patch to inline a certain number of bi_io_vec's inside the bio
37 * itself, to shrink a bio data allocation from two mempool calls to one
38 */
39#define BIO_INLINE_VECS 4
40
41/*
42 * if you change this list, also change bvec_alloc or things will
43 * break badly! cannot be bigger than what you can fit into an
44 * unsigned short
45 */
46#define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }
47static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = {
48 BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),
49};
50#undef BV
51
52/*
53 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
54 * IO code that does not need private memory pools.
55 */
56struct bio_set *fs_bio_set;
57EXPORT_SYMBOL(fs_bio_set);
58
59/*
60 * Our slab pool management
61 */
62struct bio_slab {
63 struct kmem_cache *slab;
64 unsigned int slab_ref;
65 unsigned int slab_size;
66 char name[8];
67};
68static DEFINE_MUTEX(bio_slab_lock);
69static struct bio_slab *bio_slabs;
70static unsigned int bio_slab_nr, bio_slab_max;
71
72static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)
73{
74 unsigned int sz = sizeof(struct bio) + extra_size;
75 struct kmem_cache *slab = NULL;
76 struct bio_slab *bslab, *new_bio_slabs;
77 unsigned int new_bio_slab_max;
78 unsigned int i, entry = -1;
79
80 mutex_lock(&bio_slab_lock);
81
82 i = 0;
83 while (i < bio_slab_nr) {
84 bslab = &bio_slabs[i];
85
86 if (!bslab->slab && entry == -1)
87 entry = i;
88 else if (bslab->slab_size == sz) {
89 slab = bslab->slab;
90 bslab->slab_ref++;
91 break;
92 }
93 i++;
94 }
95
96 if (slab)
97 goto out_unlock;
98
99 if (bio_slab_nr == bio_slab_max && entry == -1) {
100 new_bio_slab_max = bio_slab_max << 1;
101 new_bio_slabs = krealloc(bio_slabs,
102 new_bio_slab_max * sizeof(struct bio_slab),
103 GFP_KERNEL);
104 if (!new_bio_slabs)
105 goto out_unlock;
106 bio_slab_max = new_bio_slab_max;
107 bio_slabs = new_bio_slabs;
108 }
109 if (entry == -1)
110 entry = bio_slab_nr++;
111
112 bslab = &bio_slabs[entry];
113
114 snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);
115 slab = kmem_cache_create(bslab->name, sz, 0, SLAB_HWCACHE_ALIGN, NULL);
116 if (!slab)
117 goto out_unlock;
118
119 bslab->slab = slab;
120 bslab->slab_ref = 1;
121 bslab->slab_size = sz;
122out_unlock:
123 mutex_unlock(&bio_slab_lock);
124 return slab;
125}
126
127static void bio_put_slab(struct bio_set *bs)
128{
129 struct bio_slab *bslab = NULL;
130 unsigned int i;
131
132 mutex_lock(&bio_slab_lock);
133
134 for (i = 0; i < bio_slab_nr; i++) {
135 if (bs->bio_slab == bio_slabs[i].slab) {
136 bslab = &bio_slabs[i];
137 break;
138 }
139 }
140
141 if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
142 goto out;
143
144 WARN_ON(!bslab->slab_ref);
145
146 if (--bslab->slab_ref)
147 goto out;
148
149 kmem_cache_destroy(bslab->slab);
150 bslab->slab = NULL;
151
152out:
153 mutex_unlock(&bio_slab_lock);
154}
155
156unsigned int bvec_nr_vecs(unsigned short idx)
157{
158 return bvec_slabs[idx].nr_vecs;
159}
160
161void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
162{
163 BIO_BUG_ON(idx >= BIOVEC_NR_POOLS);
164
165 if (idx == BIOVEC_MAX_IDX)
166 mempool_free(bv, pool);
167 else {
168 struct biovec_slab *bvs = bvec_slabs + idx;
169
170 kmem_cache_free(bvs->slab, bv);
171 }
172}
173
174struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
175 mempool_t *pool)
176{
177 struct bio_vec *bvl;
178
179 /*
180 * see comment near bvec_array define!
181 */
182 switch (nr) {
183 case 1:
184 *idx = 0;
185 break;
186 case 2 ... 4:
187 *idx = 1;
188 break;
189 case 5 ... 16:
190 *idx = 2;
191 break;
192 case 17 ... 64:
193 *idx = 3;
194 break;
195 case 65 ... 128:
196 *idx = 4;
197 break;
198 case 129 ... BIO_MAX_PAGES:
199 *idx = 5;
200 break;
201 default:
202 return NULL;
203 }
204
205 /*
206 * idx now points to the pool we want to allocate from. only the
207 * 1-vec entry pool is mempool backed.
208 */
209 if (*idx == BIOVEC_MAX_IDX) {
210fallback:
211 bvl = mempool_alloc(pool, gfp_mask);
212 } else {
213 struct biovec_slab *bvs = bvec_slabs + *idx;
214 gfp_t __gfp_mask = gfp_mask & ~(__GFP_WAIT | __GFP_IO);
215
216 /*
217 * Make this allocation restricted and don't dump info on
218 * allocation failures, since we'll fallback to the mempool
219 * in case of failure.
220 */
221 __gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
222
223 /*
224 * Try a slab allocation. If this fails and __GFP_WAIT
225 * is set, retry with the 1-entry mempool
226 */
227 bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
228 if (unlikely(!bvl && (gfp_mask & __GFP_WAIT))) {
229 *idx = BIOVEC_MAX_IDX;
230 goto fallback;
231 }
232 }
233
234 return bvl;
235}
236
237static void __bio_free(struct bio *bio)
238{
239 bio_disassociate_task(bio);
240
241 if (bio_integrity(bio))
242 bio_integrity_free(bio);
243}
244
245static void bio_free(struct bio *bio)
246{
247 struct bio_set *bs = bio->bi_pool;
248 void *p;
249
250 __bio_free(bio);
251
252 if (bs) {
253 if (bio_flagged(bio, BIO_OWNS_VEC))
254 bvec_free(bs->bvec_pool, bio->bi_io_vec, BIO_POOL_IDX(bio));
255
256 /*
257 * If we have front padding, adjust the bio pointer before freeing
258 */
259 p = bio;
260 p -= bs->front_pad;
261
262 mempool_free(p, bs->bio_pool);
263 } else {
264 /* Bio was allocated by bio_kmalloc() */
265 kfree(bio);
266 }
267}
268
269void bio_init(struct bio *bio)
270{
271 memset(bio, 0, sizeof(*bio));
272 bio->bi_flags = 1 << BIO_UPTODATE;
273 atomic_set(&bio->bi_remaining, 1);
274 atomic_set(&bio->bi_cnt, 1);
275}
276EXPORT_SYMBOL(bio_init);
277
278/**
279 * bio_reset - reinitialize a bio
280 * @bio: bio to reset
281 *
282 * Description:
283 * After calling bio_reset(), @bio will be in the same state as a freshly
284 * allocated bio returned bio bio_alloc_bioset() - the only fields that are
285 * preserved are the ones that are initialized by bio_alloc_bioset(). See
286 * comment in struct bio.
287 */
288void bio_reset(struct bio *bio)
289{
290 unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);
291
292 __bio_free(bio);
293
294 memset(bio, 0, BIO_RESET_BYTES);
295 bio->bi_flags = flags|(1 << BIO_UPTODATE);
296 atomic_set(&bio->bi_remaining, 1);
297}
298EXPORT_SYMBOL(bio_reset);
299
300static void bio_chain_endio(struct bio *bio, int error)
301{
302 bio_endio(bio->bi_private, error);
303 bio_put(bio);
304}
305
306/**
307 * bio_chain - chain bio completions
308 * @bio: the target bio
309 * @parent: the @bio's parent bio
310 *
311 * The caller won't have a bi_end_io called when @bio completes - instead,
312 * @parent's bi_end_io won't be called until both @parent and @bio have
313 * completed; the chained bio will also be freed when it completes.
314 *
315 * The caller must not set bi_private or bi_end_io in @bio.
316 */
317void bio_chain(struct bio *bio, struct bio *parent)
318{
319 BUG_ON(bio->bi_private || bio->bi_end_io);
320
321 bio->bi_private = parent;
322 bio->bi_end_io = bio_chain_endio;
323 atomic_inc(&parent->bi_remaining);
324}
325EXPORT_SYMBOL(bio_chain);
326
327static void bio_alloc_rescue(struct work_struct *work)
328{
329 struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
330 struct bio *bio;
331
332 while (1) {
333 spin_lock(&bs->rescue_lock);
334 bio = bio_list_pop(&bs->rescue_list);
335 spin_unlock(&bs->rescue_lock);
336
337 if (!bio)
338 break;
339
340 generic_make_request(bio);
341 }
342}
343
344static void punt_bios_to_rescuer(struct bio_set *bs)
345{
346 struct bio_list punt, nopunt;
347 struct bio *bio;
348
349 /*
350 * In order to guarantee forward progress we must punt only bios that
351 * were allocated from this bio_set; otherwise, if there was a bio on
352 * there for a stacking driver higher up in the stack, processing it
353 * could require allocating bios from this bio_set, and doing that from
354 * our own rescuer would be bad.
355 *
356 * Since bio lists are singly linked, pop them all instead of trying to
357 * remove from the middle of the list:
358 */
359
360 bio_list_init(&punt);
361 bio_list_init(&nopunt);
362
363 while ((bio = bio_list_pop(current->bio_list)))
364 bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
365
366 *current->bio_list = nopunt;
367
368 spin_lock(&bs->rescue_lock);
369 bio_list_merge(&bs->rescue_list, &punt);
370 spin_unlock(&bs->rescue_lock);
371
372 queue_work(bs->rescue_workqueue, &bs->rescue_work);
373}
374
375/**
376 * bio_alloc_bioset - allocate a bio for I/O
377 * @gfp_mask: the GFP_ mask given to the slab allocator
378 * @nr_iovecs: number of iovecs to pre-allocate
379 * @bs: the bio_set to allocate from.
380 *
381 * Description:
382 * If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
383 * backed by the @bs's mempool.
384 *
385 * When @bs is not NULL, if %__GFP_WAIT is set then bio_alloc will always be
386 * able to allocate a bio. This is due to the mempool guarantees. To make this
387 * work, callers must never allocate more than 1 bio at a time from this pool.
388 * Callers that need to allocate more than 1 bio must always submit the
389 * previously allocated bio for IO before attempting to allocate a new one.
390 * Failure to do so can cause deadlocks under memory pressure.
391 *
392 * Note that when running under generic_make_request() (i.e. any block
393 * driver), bios are not submitted until after you return - see the code in
394 * generic_make_request() that converts recursion into iteration, to prevent
395 * stack overflows.
396 *
397 * This would normally mean allocating multiple bios under
398 * generic_make_request() would be susceptible to deadlocks, but we have
399 * deadlock avoidance code that resubmits any blocked bios from a rescuer
400 * thread.
401 *
402 * However, we do not guarantee forward progress for allocations from other
403 * mempools. Doing multiple allocations from the same mempool under
404 * generic_make_request() should be avoided - instead, use bio_set's front_pad
405 * for per bio allocations.
406 *
407 * RETURNS:
408 * Pointer to new bio on success, NULL on failure.
409 */
410struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
411{
412 gfp_t saved_gfp = gfp_mask;
413 unsigned front_pad;
414 unsigned inline_vecs;
415 unsigned long idx = BIO_POOL_NONE;
416 struct bio_vec *bvl = NULL;
417 struct bio *bio;
418 void *p;
419
420 if (!bs) {
421 if (nr_iovecs > UIO_MAXIOV)
422 return NULL;
423
424 p = kmalloc(sizeof(struct bio) +
425 nr_iovecs * sizeof(struct bio_vec),
426 gfp_mask);
427 front_pad = 0;
428 inline_vecs = nr_iovecs;
429 } else {
430 /*
431 * generic_make_request() converts recursion to iteration; this
432 * means if we're running beneath it, any bios we allocate and
433 * submit will not be submitted (and thus freed) until after we
434 * return.
435 *
436 * This exposes us to a potential deadlock if we allocate
437 * multiple bios from the same bio_set() while running
438 * underneath generic_make_request(). If we were to allocate
439 * multiple bios (say a stacking block driver that was splitting
440 * bios), we would deadlock if we exhausted the mempool's
441 * reserve.
442 *
443 * We solve this, and guarantee forward progress, with a rescuer
444 * workqueue per bio_set. If we go to allocate and there are
445 * bios on current->bio_list, we first try the allocation
446 * without __GFP_WAIT; if that fails, we punt those bios we
447 * would be blocking to the rescuer workqueue before we retry
448 * with the original gfp_flags.
449 */
450
451 if (current->bio_list && !bio_list_empty(current->bio_list))
452 gfp_mask &= ~__GFP_WAIT;
453
454 p = mempool_alloc(bs->bio_pool, gfp_mask);
455 if (!p && gfp_mask != saved_gfp) {
456 punt_bios_to_rescuer(bs);
457 gfp_mask = saved_gfp;
458 p = mempool_alloc(bs->bio_pool, gfp_mask);
459 }
460
461 front_pad = bs->front_pad;
462 inline_vecs = BIO_INLINE_VECS;
463 }
464
465 if (unlikely(!p))
466 return NULL;
467
468 bio = p + front_pad;
469 bio_init(bio);
470
471 if (nr_iovecs > inline_vecs) {
472 bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
473 if (!bvl && gfp_mask != saved_gfp) {
474 punt_bios_to_rescuer(bs);
475 gfp_mask = saved_gfp;
476 bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
477 }
478
479 if (unlikely(!bvl))
480 goto err_free;
481
482 bio->bi_flags |= 1 << BIO_OWNS_VEC;
483 } else if (nr_iovecs) {
484 bvl = bio->bi_inline_vecs;
485 }
486
487 bio->bi_pool = bs;
488 bio->bi_flags |= idx << BIO_POOL_OFFSET;
489 bio->bi_max_vecs = nr_iovecs;
490 bio->bi_io_vec = bvl;
491 return bio;
492
493err_free:
494 mempool_free(p, bs->bio_pool);
495 return NULL;
496}
497EXPORT_SYMBOL(bio_alloc_bioset);
498
499void zero_fill_bio(struct bio *bio)
500{
501 unsigned long flags;
502 struct bio_vec bv;
503 struct bvec_iter iter;
504
505 bio_for_each_segment(bv, bio, iter) {
506 char *data = bvec_kmap_irq(&bv, &flags);
507 memset(data, 0, bv.bv_len);
508 flush_dcache_page(bv.bv_page);
509 bvec_kunmap_irq(data, &flags);
510 }
511}
512EXPORT_SYMBOL(zero_fill_bio);
513
514/**
515 * bio_put - release a reference to a bio
516 * @bio: bio to release reference to
517 *
518 * Description:
519 * Put a reference to a &struct bio, either one you have gotten with
520 * bio_alloc, bio_get or bio_clone. The last put of a bio will free it.
521 **/
522void bio_put(struct bio *bio)
523{
524 BIO_BUG_ON(!atomic_read(&bio->bi_cnt));
525
526 /*
527 * last put frees it
528 */
529 if (atomic_dec_and_test(&bio->bi_cnt))
530 bio_free(bio);
531}
532EXPORT_SYMBOL(bio_put);
533
534inline int bio_phys_segments(struct request_queue *q, struct bio *bio)
535{
536 if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
537 blk_recount_segments(q, bio);
538
539 return bio->bi_phys_segments;
540}
541EXPORT_SYMBOL(bio_phys_segments);
542
543/**
544 * __bio_clone_fast - clone a bio that shares the original bio's biovec
545 * @bio: destination bio
546 * @bio_src: bio to clone
547 *
548 * Clone a &bio. Caller will own the returned bio, but not
549 * the actual data it points to. Reference count of returned
550 * bio will be one.
551 *
552 * Caller must ensure that @bio_src is not freed before @bio.
553 */
554void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
555{
556 BUG_ON(bio->bi_pool && BIO_POOL_IDX(bio) != BIO_POOL_NONE);
557
558 /*
559 * most users will be overriding ->bi_bdev with a new target,
560 * so we don't set nor calculate new physical/hw segment counts here
561 */
562 bio->bi_bdev = bio_src->bi_bdev;
563 bio->bi_flags |= 1 << BIO_CLONED;
564 bio->bi_rw = bio_src->bi_rw;
565 bio->bi_iter = bio_src->bi_iter;
566 bio->bi_io_vec = bio_src->bi_io_vec;
567}
568EXPORT_SYMBOL(__bio_clone_fast);
569
570/**
571 * bio_clone_fast - clone a bio that shares the original bio's biovec
572 * @bio: bio to clone
573 * @gfp_mask: allocation priority
574 * @bs: bio_set to allocate from
575 *
576 * Like __bio_clone_fast, only also allocates the returned bio
577 */
578struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
579{
580 struct bio *b;
581
582 b = bio_alloc_bioset(gfp_mask, 0, bs);
583 if (!b)
584 return NULL;
585
586 __bio_clone_fast(b, bio);
587
588 if (bio_integrity(bio)) {
589 int ret;
590
591 ret = bio_integrity_clone(b, bio, gfp_mask);
592
593 if (ret < 0) {
594 bio_put(b);
595 return NULL;
596 }
597 }
598
599 return b;
600}
601EXPORT_SYMBOL(bio_clone_fast);
602
603/**
604 * bio_clone_bioset - clone a bio
605 * @bio_src: bio to clone
606 * @gfp_mask: allocation priority
607 * @bs: bio_set to allocate from
608 *
609 * Clone bio. Caller will own the returned bio, but not the actual data it
610 * points to. Reference count of returned bio will be one.
611 */
612struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask,
613 struct bio_set *bs)
614{
615 struct bvec_iter iter;
616 struct bio_vec bv;
617 struct bio *bio;
618
619 /*
620 * Pre immutable biovecs, __bio_clone() used to just do a memcpy from
621 * bio_src->bi_io_vec to bio->bi_io_vec.
622 *
623 * We can't do that anymore, because:
624 *
625 * - The point of cloning the biovec is to produce a bio with a biovec
626 * the caller can modify: bi_idx and bi_bvec_done should be 0.
627 *
628 * - The original bio could've had more than BIO_MAX_PAGES biovecs; if
629 * we tried to clone the whole thing bio_alloc_bioset() would fail.
630 * But the clone should succeed as long as the number of biovecs we
631 * actually need to allocate is fewer than BIO_MAX_PAGES.
632 *
633 * - Lastly, bi_vcnt should not be looked at or relied upon by code
634 * that does not own the bio - reason being drivers don't use it for
635 * iterating over the biovec anymore, so expecting it to be kept up
636 * to date (i.e. for clones that share the parent biovec) is just
637 * asking for trouble and would force extra work on
638 * __bio_clone_fast() anyways.
639 */
640
641 bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
642 if (!bio)
643 return NULL;
644
645 bio->bi_bdev = bio_src->bi_bdev;
646 bio->bi_rw = bio_src->bi_rw;
647 bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
648 bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
649
650 if (bio->bi_rw & REQ_DISCARD)
651 goto integrity_clone;
652
653 if (bio->bi_rw & REQ_WRITE_SAME) {
654 bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0];
655 goto integrity_clone;
656 }
657
658 bio_for_each_segment(bv, bio_src, iter)
659 bio->bi_io_vec[bio->bi_vcnt++] = bv;
660
661integrity_clone:
662 if (bio_integrity(bio_src)) {
663 int ret;
664
665 ret = bio_integrity_clone(bio, bio_src, gfp_mask);
666 if (ret < 0) {
667 bio_put(bio);
668 return NULL;
669 }
670 }
671
672 return bio;
673}
674EXPORT_SYMBOL(bio_clone_bioset);
675
676/**
677 * bio_get_nr_vecs - return approx number of vecs
678 * @bdev: I/O target
679 *
680 * Return the approximate number of pages we can send to this target.
681 * There's no guarantee that you will be able to fit this number of pages
682 * into a bio, it does not account for dynamic restrictions that vary
683 * on offset.
684 */
685int bio_get_nr_vecs(struct block_device *bdev)
686{
687 struct request_queue *q = bdev_get_queue(bdev);
688 int nr_pages;
689
690 nr_pages = min_t(unsigned,
691 queue_max_segments(q),
692 queue_max_sectors(q) / (PAGE_SIZE >> 9) + 1);
693
694 return min_t(unsigned, nr_pages, BIO_MAX_PAGES);
695
696}
697EXPORT_SYMBOL(bio_get_nr_vecs);
698
699static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
700 *page, unsigned int len, unsigned int offset,
701 unsigned int max_sectors)
702{
703 int retried_segments = 0;
704 struct bio_vec *bvec;
705
706 /*
707 * cloned bio must not modify vec list
708 */
709 if (unlikely(bio_flagged(bio, BIO_CLONED)))
710 return 0;
711
712 if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors)
713 return 0;
714
715 /*
716 * For filesystems with a blocksize smaller than the pagesize
717 * we will often be called with the same page as last time and
718 * a consecutive offset. Optimize this special case.
719 */
720 if (bio->bi_vcnt > 0) {
721 struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
722
723 if (page == prev->bv_page &&
724 offset == prev->bv_offset + prev->bv_len) {
725 unsigned int prev_bv_len = prev->bv_len;
726 prev->bv_len += len;
727
728 if (q->merge_bvec_fn) {
729 struct bvec_merge_data bvm = {
730 /* prev_bvec is already charged in
731 bi_size, discharge it in order to
732 simulate merging updated prev_bvec
733 as new bvec. */
734 .bi_bdev = bio->bi_bdev,
735 .bi_sector = bio->bi_iter.bi_sector,
736 .bi_size = bio->bi_iter.bi_size -
737 prev_bv_len,
738 .bi_rw = bio->bi_rw,
739 };
740
741 if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len) {
742 prev->bv_len -= len;
743 return 0;
744 }
745 }
746
747 goto done;
748 }
749 }
750
751 if (bio->bi_vcnt >= bio->bi_max_vecs)
752 return 0;
753
754 /*
755 * we might lose a segment or two here, but rather that than
756 * make this too complex.
757 */
758
759 while (bio->bi_phys_segments >= queue_max_segments(q)) {
760
761 if (retried_segments)
762 return 0;
763
764 retried_segments = 1;
765 blk_recount_segments(q, bio);
766 }
767
768 /*
769 * setup the new entry, we might clear it again later if we
770 * cannot add the page
771 */
772 bvec = &bio->bi_io_vec[bio->bi_vcnt];
773 bvec->bv_page = page;
774 bvec->bv_len = len;
775 bvec->bv_offset = offset;
776
777 /*
778 * if queue has other restrictions (eg varying max sector size
779 * depending on offset), it can specify a merge_bvec_fn in the
780 * queue to get further control
781 */
782 if (q->merge_bvec_fn) {
783 struct bvec_merge_data bvm = {
784 .bi_bdev = bio->bi_bdev,
785 .bi_sector = bio->bi_iter.bi_sector,
786 .bi_size = bio->bi_iter.bi_size,
787 .bi_rw = bio->bi_rw,
788 };
789
790 /*
791 * merge_bvec_fn() returns number of bytes it can accept
792 * at this offset
793 */
794 if (q->merge_bvec_fn(q, &bvm, bvec) < bvec->bv_len) {
795 bvec->bv_page = NULL;
796 bvec->bv_len = 0;
797 bvec->bv_offset = 0;
798 return 0;
799 }
800 }
801
802 /* If we may be able to merge these biovecs, force a recount */
803 if (bio->bi_vcnt && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
804 bio->bi_flags &= ~(1 << BIO_SEG_VALID);
805
806 bio->bi_vcnt++;
807 bio->bi_phys_segments++;
808 done:
809 bio->bi_iter.bi_size += len;
810 return len;
811}
812
813/**
814 * bio_add_pc_page - attempt to add page to bio
815 * @q: the target queue
816 * @bio: destination bio
817 * @page: page to add
818 * @len: vec entry length
819 * @offset: vec entry offset
820 *
821 * Attempt to add a page to the bio_vec maplist. This can fail for a
822 * number of reasons, such as the bio being full or target block device
823 * limitations. The target block device must allow bio's up to PAGE_SIZE,
824 * so it is always possible to add a single page to an empty bio.
825 *
826 * This should only be used by REQ_PC bios.
827 */
828int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page,
829 unsigned int len, unsigned int offset)
830{
831 return __bio_add_page(q, bio, page, len, offset,
832 queue_max_hw_sectors(q));
833}
834EXPORT_SYMBOL(bio_add_pc_page);
835
836/**
837 * bio_add_page - attempt to add page to bio
838 * @bio: destination bio
839 * @page: page to add
840 * @len: vec entry length
841 * @offset: vec entry offset
842 *
843 * Attempt to add a page to the bio_vec maplist. This can fail for a
844 * number of reasons, such as the bio being full or target block device
845 * limitations. The target block device must allow bio's up to PAGE_SIZE,
846 * so it is always possible to add a single page to an empty bio.
847 */
848int bio_add_page(struct bio *bio, struct page *page, unsigned int len,
849 unsigned int offset)
850{
851 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
852 return __bio_add_page(q, bio, page, len, offset, queue_max_sectors(q));
853}
854EXPORT_SYMBOL(bio_add_page);
855
856struct submit_bio_ret {
857 struct completion event;
858 int error;
859};
860
861static void submit_bio_wait_endio(struct bio *bio, int error)
862{
863 struct submit_bio_ret *ret = bio->bi_private;
864
865 ret->error = error;
866 complete(&ret->event);
867}
868
869/**
870 * submit_bio_wait - submit a bio, and wait until it completes
871 * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
872 * @bio: The &struct bio which describes the I/O
873 *
874 * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
875 * bio_endio() on failure.
876 */
877int submit_bio_wait(int rw, struct bio *bio)
878{
879 struct submit_bio_ret ret;
880
881 rw |= REQ_SYNC;
882 init_completion(&ret.event);
883 bio->bi_private = &ret;
884 bio->bi_end_io = submit_bio_wait_endio;
885 submit_bio(rw, bio);
886 wait_for_completion(&ret.event);
887
888 return ret.error;
889}
890EXPORT_SYMBOL(submit_bio_wait);
891
892/**
893 * bio_advance - increment/complete a bio by some number of bytes
894 * @bio: bio to advance
895 * @bytes: number of bytes to complete
896 *
897 * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
898 * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
899 * be updated on the last bvec as well.
900 *
901 * @bio will then represent the remaining, uncompleted portion of the io.
902 */
903void bio_advance(struct bio *bio, unsigned bytes)
904{
905 if (bio_integrity(bio))
906 bio_integrity_advance(bio, bytes);
907
908 bio_advance_iter(bio, &bio->bi_iter, bytes);
909}
910EXPORT_SYMBOL(bio_advance);
911
912/**
913 * bio_alloc_pages - allocates a single page for each bvec in a bio
914 * @bio: bio to allocate pages for
915 * @gfp_mask: flags for allocation
916 *
917 * Allocates pages up to @bio->bi_vcnt.
918 *
919 * Returns 0 on success, -ENOMEM on failure. On failure, any allocated pages are
920 * freed.
921 */
922int bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
923{
924 int i;
925 struct bio_vec *bv;
926
927 bio_for_each_segment_all(bv, bio, i) {
928 bv->bv_page = alloc_page(gfp_mask);
929 if (!bv->bv_page) {
930 while (--bv >= bio->bi_io_vec)
931 __free_page(bv->bv_page);
932 return -ENOMEM;
933 }
934 }
935
936 return 0;
937}
938EXPORT_SYMBOL(bio_alloc_pages);
939
940/**
941 * bio_copy_data - copy contents of data buffers from one chain of bios to
942 * another
943 * @src: source bio list
944 * @dst: destination bio list
945 *
946 * If @src and @dst are single bios, bi_next must be NULL - otherwise, treats
947 * @src and @dst as linked lists of bios.
948 *
949 * Stops when it reaches the end of either @src or @dst - that is, copies
950 * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
951 */
952void bio_copy_data(struct bio *dst, struct bio *src)
953{
954 struct bvec_iter src_iter, dst_iter;
955 struct bio_vec src_bv, dst_bv;
956 void *src_p, *dst_p;
957 unsigned bytes;
958
959 src_iter = src->bi_iter;
960 dst_iter = dst->bi_iter;
961
962 while (1) {
963 if (!src_iter.bi_size) {
964 src = src->bi_next;
965 if (!src)
966 break;
967
968 src_iter = src->bi_iter;
969 }
970
971 if (!dst_iter.bi_size) {
972 dst = dst->bi_next;
973 if (!dst)
974 break;
975
976 dst_iter = dst->bi_iter;
977 }
978
979 src_bv = bio_iter_iovec(src, src_iter);
980 dst_bv = bio_iter_iovec(dst, dst_iter);
981
982 bytes = min(src_bv.bv_len, dst_bv.bv_len);
983
984 src_p = kmap_atomic(src_bv.bv_page);
985 dst_p = kmap_atomic(dst_bv.bv_page);
986
987 memcpy(dst_p + dst_bv.bv_offset,
988 src_p + src_bv.bv_offset,
989 bytes);
990
991 kunmap_atomic(dst_p);
992 kunmap_atomic(src_p);
993
994 bio_advance_iter(src, &src_iter, bytes);
995 bio_advance_iter(dst, &dst_iter, bytes);
996 }
997}
998EXPORT_SYMBOL(bio_copy_data);
999
1000struct bio_map_data {
1001 int nr_sgvecs;
1002 int is_our_pages;
1003 struct sg_iovec sgvecs[];
1004};
1005
1006static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio,
1007 const struct sg_iovec *iov, int iov_count,
1008 int is_our_pages)
1009{
1010 memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count);
1011 bmd->nr_sgvecs = iov_count;
1012 bmd->is_our_pages = is_our_pages;
1013 bio->bi_private = bmd;
1014}
1015
1016static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count,
1017 gfp_t gfp_mask)
1018{
1019 if (iov_count > UIO_MAXIOV)
1020 return NULL;
1021
1022 return kmalloc(sizeof(struct bio_map_data) +
1023 sizeof(struct sg_iovec) * iov_count, gfp_mask);
1024}
1025
1026static int __bio_copy_iov(struct bio *bio, const struct sg_iovec *iov, int iov_count,
1027 int to_user, int from_user, int do_free_page)
1028{
1029 int ret = 0, i;
1030 struct bio_vec *bvec;
1031 int iov_idx = 0;
1032 unsigned int iov_off = 0;
1033
1034 bio_for_each_segment_all(bvec, bio, i) {
1035 char *bv_addr = page_address(bvec->bv_page);
1036 unsigned int bv_len = bvec->bv_len;
1037
1038 while (bv_len && iov_idx < iov_count) {
1039 unsigned int bytes;
1040 char __user *iov_addr;
1041
1042 bytes = min_t(unsigned int,
1043 iov[iov_idx].iov_len - iov_off, bv_len);
1044 iov_addr = iov[iov_idx].iov_base + iov_off;
1045
1046 if (!ret) {
1047 if (to_user)
1048 ret = copy_to_user(iov_addr, bv_addr,
1049 bytes);
1050
1051 if (from_user)
1052 ret = copy_from_user(bv_addr, iov_addr,
1053 bytes);
1054
1055 if (ret)
1056 ret = -EFAULT;
1057 }
1058
1059 bv_len -= bytes;
1060 bv_addr += bytes;
1061 iov_addr += bytes;
1062 iov_off += bytes;
1063
1064 if (iov[iov_idx].iov_len == iov_off) {
1065 iov_idx++;
1066 iov_off = 0;
1067 }
1068 }
1069
1070 if (do_free_page)
1071 __free_page(bvec->bv_page);
1072 }
1073
1074 return ret;
1075}
1076
1077/**
1078 * bio_uncopy_user - finish previously mapped bio
1079 * @bio: bio being terminated
1080 *
1081 * Free pages allocated from bio_copy_user() and write back data
1082 * to user space in case of a read.
1083 */
1084int bio_uncopy_user(struct bio *bio)
1085{
1086 struct bio_map_data *bmd = bio->bi_private;
1087 struct bio_vec *bvec;
1088 int ret = 0, i;
1089
1090 if (!bio_flagged(bio, BIO_NULL_MAPPED)) {
1091 /*
1092 * if we're in a workqueue, the request is orphaned, so
1093 * don't copy into a random user address space, just free.
1094 */
1095 if (current->mm)
1096 ret = __bio_copy_iov(bio, bmd->sgvecs, bmd->nr_sgvecs,
1097 bio_data_dir(bio) == READ,
1098 0, bmd->is_our_pages);
1099 else if (bmd->is_our_pages)
1100 bio_for_each_segment_all(bvec, bio, i)
1101 __free_page(bvec->bv_page);
1102 }
1103 kfree(bmd);
1104 bio_put(bio);
1105 return ret;
1106}
1107EXPORT_SYMBOL(bio_uncopy_user);
1108
1109/**
1110 * bio_copy_user_iov - copy user data to bio
1111 * @q: destination block queue
1112 * @map_data: pointer to the rq_map_data holding pages (if necessary)
1113 * @iov: the iovec.
1114 * @iov_count: number of elements in the iovec
1115 * @write_to_vm: bool indicating writing to pages or not
1116 * @gfp_mask: memory allocation flags
1117 *
1118 * Prepares and returns a bio for indirect user io, bouncing data
1119 * to/from kernel pages as necessary. Must be paired with
1120 * call bio_uncopy_user() on io completion.
1121 */
1122struct bio *bio_copy_user_iov(struct request_queue *q,
1123 struct rq_map_data *map_data,
1124 const struct sg_iovec *iov, int iov_count,
1125 int write_to_vm, gfp_t gfp_mask)
1126{
1127 struct bio_map_data *bmd;
1128 struct bio_vec *bvec;
1129 struct page *page;
1130 struct bio *bio;
1131 int i, ret;
1132 int nr_pages = 0;
1133 unsigned int len = 0;
1134 unsigned int offset = map_data ? map_data->offset & ~PAGE_MASK : 0;
1135
1136 for (i = 0; i < iov_count; i++) {
1137 unsigned long uaddr;
1138 unsigned long end;
1139 unsigned long start;
1140
1141 uaddr = (unsigned long)iov[i].iov_base;
1142 end = (uaddr + iov[i].iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1143 start = uaddr >> PAGE_SHIFT;
1144
1145 /*
1146 * Overflow, abort
1147 */
1148 if (end < start)
1149 return ERR_PTR(-EINVAL);
1150
1151 nr_pages += end - start;
1152 len += iov[i].iov_len;
1153 }
1154
1155 if (offset)
1156 nr_pages++;
1157
1158 bmd = bio_alloc_map_data(iov_count, gfp_mask);
1159 if (!bmd)
1160 return ERR_PTR(-ENOMEM);
1161
1162 ret = -ENOMEM;
1163 bio = bio_kmalloc(gfp_mask, nr_pages);
1164 if (!bio)
1165 goto out_bmd;
1166
1167 if (!write_to_vm)
1168 bio->bi_rw |= REQ_WRITE;
1169
1170 ret = 0;
1171
1172 if (map_data) {
1173 nr_pages = 1 << map_data->page_order;
1174 i = map_data->offset / PAGE_SIZE;
1175 }
1176 while (len) {
1177 unsigned int bytes = PAGE_SIZE;
1178
1179 bytes -= offset;
1180
1181 if (bytes > len)
1182 bytes = len;
1183
1184 if (map_data) {
1185 if (i == map_data->nr_entries * nr_pages) {
1186 ret = -ENOMEM;
1187 break;
1188 }
1189
1190 page = map_data->pages[i / nr_pages];
1191 page += (i % nr_pages);
1192
1193 i++;
1194 } else {
1195 page = alloc_page(q->bounce_gfp | gfp_mask);
1196 if (!page) {
1197 ret = -ENOMEM;
1198 break;
1199 }
1200 }
1201
1202 if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes)
1203 break;
1204
1205 len -= bytes;
1206 offset = 0;
1207 }
1208
1209 if (ret)
1210 goto cleanup;
1211
1212 /*
1213 * success
1214 */
1215 if ((!write_to_vm && (!map_data || !map_data->null_mapped)) ||
1216 (map_data && map_data->from_user)) {
1217 ret = __bio_copy_iov(bio, iov, iov_count, 0, 1, 0);
1218 if (ret)
1219 goto cleanup;
1220 }
1221
1222 bio_set_map_data(bmd, bio, iov, iov_count, map_data ? 0 : 1);
1223 return bio;
1224cleanup:
1225 if (!map_data)
1226 bio_for_each_segment_all(bvec, bio, i)
1227 __free_page(bvec->bv_page);
1228
1229 bio_put(bio);
1230out_bmd:
1231 kfree(bmd);
1232 return ERR_PTR(ret);
1233}
1234
1235/**
1236 * bio_copy_user - copy user data to bio
1237 * @q: destination block queue
1238 * @map_data: pointer to the rq_map_data holding pages (if necessary)
1239 * @uaddr: start of user address
1240 * @len: length in bytes
1241 * @write_to_vm: bool indicating writing to pages or not
1242 * @gfp_mask: memory allocation flags
1243 *
1244 * Prepares and returns a bio for indirect user io, bouncing data
1245 * to/from kernel pages as necessary. Must be paired with
1246 * call bio_uncopy_user() on io completion.
1247 */
1248struct bio *bio_copy_user(struct request_queue *q, struct rq_map_data *map_data,
1249 unsigned long uaddr, unsigned int len,
1250 int write_to_vm, gfp_t gfp_mask)
1251{
1252 struct sg_iovec iov;
1253
1254 iov.iov_base = (void __user *)uaddr;
1255 iov.iov_len = len;
1256
1257 return bio_copy_user_iov(q, map_data, &iov, 1, write_to_vm, gfp_mask);
1258}
1259EXPORT_SYMBOL(bio_copy_user);
1260
1261static struct bio *__bio_map_user_iov(struct request_queue *q,
1262 struct block_device *bdev,
1263 const struct sg_iovec *iov, int iov_count,
1264 int write_to_vm, gfp_t gfp_mask)
1265{
1266 int i, j;
1267 int nr_pages = 0;
1268 struct page **pages;
1269 struct bio *bio;
1270 int cur_page = 0;
1271 int ret, offset;
1272
1273 for (i = 0; i < iov_count; i++) {
1274 unsigned long uaddr = (unsigned long)iov[i].iov_base;
1275 unsigned long len = iov[i].iov_len;
1276 unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1277 unsigned long start = uaddr >> PAGE_SHIFT;
1278
1279 /*
1280 * Overflow, abort
1281 */
1282 if (end < start)
1283 return ERR_PTR(-EINVAL);
1284
1285 nr_pages += end - start;
1286 /*
1287 * buffer must be aligned to at least hardsector size for now
1288 */
1289 if (uaddr & queue_dma_alignment(q))
1290 return ERR_PTR(-EINVAL);
1291 }
1292
1293 if (!nr_pages)
1294 return ERR_PTR(-EINVAL);
1295
1296 bio = bio_kmalloc(gfp_mask, nr_pages);
1297 if (!bio)
1298 return ERR_PTR(-ENOMEM);
1299
1300 ret = -ENOMEM;
1301 pages = kcalloc(nr_pages, sizeof(struct page *), gfp_mask);
1302 if (!pages)
1303 goto out;
1304
1305 for (i = 0; i < iov_count; i++) {
1306 unsigned long uaddr = (unsigned long)iov[i].iov_base;
1307 unsigned long len = iov[i].iov_len;
1308 unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1309 unsigned long start = uaddr >> PAGE_SHIFT;
1310 const int local_nr_pages = end - start;
1311 const int page_limit = cur_page + local_nr_pages;
1312
1313 ret = get_user_pages_fast(uaddr, local_nr_pages,
1314 write_to_vm, &pages[cur_page]);
1315 if (ret < local_nr_pages) {
1316 ret = -EFAULT;
1317 goto out_unmap;
1318 }
1319
1320 offset = uaddr & ~PAGE_MASK;
1321 for (j = cur_page; j < page_limit; j++) {
1322 unsigned int bytes = PAGE_SIZE - offset;
1323
1324 if (len <= 0)
1325 break;
1326
1327 if (bytes > len)
1328 bytes = len;
1329
1330 /*
1331 * sorry...
1332 */
1333 if (bio_add_pc_page(q, bio, pages[j], bytes, offset) <
1334 bytes)
1335 break;
1336
1337 len -= bytes;
1338 offset = 0;
1339 }
1340
1341 cur_page = j;
1342 /*
1343 * release the pages we didn't map into the bio, if any
1344 */
1345 while (j < page_limit)
1346 page_cache_release(pages[j++]);
1347 }
1348
1349 kfree(pages);
1350
1351 /*
1352 * set data direction, and check if mapped pages need bouncing
1353 */
1354 if (!write_to_vm)
1355 bio->bi_rw |= REQ_WRITE;
1356
1357 bio->bi_bdev = bdev;
1358 bio->bi_flags |= (1 << BIO_USER_MAPPED);
1359 return bio;
1360
1361 out_unmap:
1362 for (i = 0; i < nr_pages; i++) {
1363 if(!pages[i])
1364 break;
1365 page_cache_release(pages[i]);
1366 }
1367 out:
1368 kfree(pages);
1369 bio_put(bio);
1370 return ERR_PTR(ret);
1371}
1372
1373/**
1374 * bio_map_user - map user address into bio
1375 * @q: the struct request_queue for the bio
1376 * @bdev: destination block device
1377 * @uaddr: start of user address
1378 * @len: length in bytes
1379 * @write_to_vm: bool indicating writing to pages or not
1380 * @gfp_mask: memory allocation flags
1381 *
1382 * Map the user space address into a bio suitable for io to a block
1383 * device. Returns an error pointer in case of error.
1384 */
1385struct bio *bio_map_user(struct request_queue *q, struct block_device *bdev,
1386 unsigned long uaddr, unsigned int len, int write_to_vm,
1387 gfp_t gfp_mask)
1388{
1389 struct sg_iovec iov;
1390
1391 iov.iov_base = (void __user *)uaddr;
1392 iov.iov_len = len;
1393
1394 return bio_map_user_iov(q, bdev, &iov, 1, write_to_vm, gfp_mask);
1395}
1396EXPORT_SYMBOL(bio_map_user);
1397
1398/**
1399 * bio_map_user_iov - map user sg_iovec table into bio
1400 * @q: the struct request_queue for the bio
1401 * @bdev: destination block device
1402 * @iov: the iovec.
1403 * @iov_count: number of elements in the iovec
1404 * @write_to_vm: bool indicating writing to pages or not
1405 * @gfp_mask: memory allocation flags
1406 *
1407 * Map the user space address into a bio suitable for io to a block
1408 * device. Returns an error pointer in case of error.
1409 */
1410struct bio *bio_map_user_iov(struct request_queue *q, struct block_device *bdev,
1411 const struct sg_iovec *iov, int iov_count,
1412 int write_to_vm, gfp_t gfp_mask)
1413{
1414 struct bio *bio;
1415
1416 bio = __bio_map_user_iov(q, bdev, iov, iov_count, write_to_vm,
1417 gfp_mask);
1418 if (IS_ERR(bio))
1419 return bio;
1420
1421 /*
1422 * subtle -- if __bio_map_user() ended up bouncing a bio,
1423 * it would normally disappear when its bi_end_io is run.
1424 * however, we need it for the unmap, so grab an extra
1425 * reference to it
1426 */
1427 bio_get(bio);
1428
1429 return bio;
1430}
1431
1432static void __bio_unmap_user(struct bio *bio)
1433{
1434 struct bio_vec *bvec;
1435 int i;
1436
1437 /*
1438 * make sure we dirty pages we wrote to
1439 */
1440 bio_for_each_segment_all(bvec, bio, i) {
1441 if (bio_data_dir(bio) == READ)
1442 set_page_dirty_lock(bvec->bv_page);
1443
1444 page_cache_release(bvec->bv_page);
1445 }
1446
1447 bio_put(bio);
1448}
1449
1450/**
1451 * bio_unmap_user - unmap a bio
1452 * @bio: the bio being unmapped
1453 *
1454 * Unmap a bio previously mapped by bio_map_user(). Must be called with
1455 * a process context.
1456 *
1457 * bio_unmap_user() may sleep.
1458 */
1459void bio_unmap_user(struct bio *bio)
1460{
1461 __bio_unmap_user(bio);
1462 bio_put(bio);
1463}
1464EXPORT_SYMBOL(bio_unmap_user);
1465
1466static void bio_map_kern_endio(struct bio *bio, int err)
1467{
1468 bio_put(bio);
1469}
1470
1471static struct bio *__bio_map_kern(struct request_queue *q, void *data,
1472 unsigned int len, gfp_t gfp_mask)
1473{
1474 unsigned long kaddr = (unsigned long)data;
1475 unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1476 unsigned long start = kaddr >> PAGE_SHIFT;
1477 const int nr_pages = end - start;
1478 int offset, i;
1479 struct bio *bio;
1480
1481 bio = bio_kmalloc(gfp_mask, nr_pages);
1482 if (!bio)
1483 return ERR_PTR(-ENOMEM);
1484
1485 offset = offset_in_page(kaddr);
1486 for (i = 0; i < nr_pages; i++) {
1487 unsigned int bytes = PAGE_SIZE - offset;
1488
1489 if (len <= 0)
1490 break;
1491
1492 if (bytes > len)
1493 bytes = len;
1494
1495 if (bio_add_pc_page(q, bio, virt_to_page(data), bytes,
1496 offset) < bytes)
1497 break;
1498
1499 data += bytes;
1500 len -= bytes;
1501 offset = 0;
1502 }
1503
1504 bio->bi_end_io = bio_map_kern_endio;
1505 return bio;
1506}
1507
1508/**
1509 * bio_map_kern - map kernel address into bio
1510 * @q: the struct request_queue for the bio
1511 * @data: pointer to buffer to map
1512 * @len: length in bytes
1513 * @gfp_mask: allocation flags for bio allocation
1514 *
1515 * Map the kernel address into a bio suitable for io to a block
1516 * device. Returns an error pointer in case of error.
1517 */
1518struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
1519 gfp_t gfp_mask)
1520{
1521 struct bio *bio;
1522
1523 bio = __bio_map_kern(q, data, len, gfp_mask);
1524 if (IS_ERR(bio))
1525 return bio;
1526
1527 if (bio->bi_iter.bi_size == len)
1528 return bio;
1529
1530 /*
1531 * Don't support partial mappings.
1532 */
1533 bio_put(bio);
1534 return ERR_PTR(-EINVAL);
1535}
1536EXPORT_SYMBOL(bio_map_kern);
1537
1538static void bio_copy_kern_endio(struct bio *bio, int err)
1539{
1540 struct bio_vec *bvec;
1541 const int read = bio_data_dir(bio) == READ;
1542 struct bio_map_data *bmd = bio->bi_private;
1543 int i;
1544 char *p = bmd->sgvecs[0].iov_base;
1545
1546 bio_for_each_segment_all(bvec, bio, i) {
1547 char *addr = page_address(bvec->bv_page);
1548
1549 if (read)
1550 memcpy(p, addr, bvec->bv_len);
1551
1552 __free_page(bvec->bv_page);
1553 p += bvec->bv_len;
1554 }
1555
1556 kfree(bmd);
1557 bio_put(bio);
1558}
1559
1560/**
1561 * bio_copy_kern - copy kernel address into bio
1562 * @q: the struct request_queue for the bio
1563 * @data: pointer to buffer to copy
1564 * @len: length in bytes
1565 * @gfp_mask: allocation flags for bio and page allocation
1566 * @reading: data direction is READ
1567 *
1568 * copy the kernel address into a bio suitable for io to a block
1569 * device. Returns an error pointer in case of error.
1570 */
1571struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,
1572 gfp_t gfp_mask, int reading)
1573{
1574 struct bio *bio;
1575 struct bio_vec *bvec;
1576 int i;
1577
1578 bio = bio_copy_user(q, NULL, (unsigned long)data, len, 1, gfp_mask);
1579 if (IS_ERR(bio))
1580 return bio;
1581
1582 if (!reading) {
1583 void *p = data;
1584
1585 bio_for_each_segment_all(bvec, bio, i) {
1586 char *addr = page_address(bvec->bv_page);
1587
1588 memcpy(addr, p, bvec->bv_len);
1589 p += bvec->bv_len;
1590 }
1591 }
1592
1593 bio->bi_end_io = bio_copy_kern_endio;
1594
1595 return bio;
1596}
1597EXPORT_SYMBOL(bio_copy_kern);
1598
1599/*
1600 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
1601 * for performing direct-IO in BIOs.
1602 *
1603 * The problem is that we cannot run set_page_dirty() from interrupt context
1604 * because the required locks are not interrupt-safe. So what we can do is to
1605 * mark the pages dirty _before_ performing IO. And in interrupt context,
1606 * check that the pages are still dirty. If so, fine. If not, redirty them
1607 * in process context.
1608 *
1609 * We special-case compound pages here: normally this means reads into hugetlb
1610 * pages. The logic in here doesn't really work right for compound pages
1611 * because the VM does not uniformly chase down the head page in all cases.
1612 * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't
1613 * handle them at all. So we skip compound pages here at an early stage.
1614 *
1615 * Note that this code is very hard to test under normal circumstances because
1616 * direct-io pins the pages with get_user_pages(). This makes
1617 * is_page_cache_freeable return false, and the VM will not clean the pages.
1618 * But other code (eg, flusher threads) could clean the pages if they are mapped
1619 * pagecache.
1620 *
1621 * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
1622 * deferred bio dirtying paths.
1623 */
1624
1625/*
1626 * bio_set_pages_dirty() will mark all the bio's pages as dirty.
1627 */
1628void bio_set_pages_dirty(struct bio *bio)
1629{
1630 struct bio_vec *bvec;
1631 int i;
1632
1633 bio_for_each_segment_all(bvec, bio, i) {
1634 struct page *page = bvec->bv_page;
1635
1636 if (page && !PageCompound(page))
1637 set_page_dirty_lock(page);
1638 }
1639}
1640
1641static void bio_release_pages(struct bio *bio)
1642{
1643 struct bio_vec *bvec;
1644 int i;
1645
1646 bio_for_each_segment_all(bvec, bio, i) {
1647 struct page *page = bvec->bv_page;
1648
1649 if (page)
1650 put_page(page);
1651 }
1652}
1653
1654/*
1655 * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
1656 * If they are, then fine. If, however, some pages are clean then they must
1657 * have been written out during the direct-IO read. So we take another ref on
1658 * the BIO and the offending pages and re-dirty the pages in process context.
1659 *
1660 * It is expected that bio_check_pages_dirty() will wholly own the BIO from
1661 * here on. It will run one page_cache_release() against each page and will
1662 * run one bio_put() against the BIO.
1663 */
1664
1665static void bio_dirty_fn(struct work_struct *work);
1666
1667static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
1668static DEFINE_SPINLOCK(bio_dirty_lock);
1669static struct bio *bio_dirty_list;
1670
1671/*
1672 * This runs in process context
1673 */
1674static void bio_dirty_fn(struct work_struct *work)
1675{
1676 unsigned long flags;
1677 struct bio *bio;
1678
1679 spin_lock_irqsave(&bio_dirty_lock, flags);
1680 bio = bio_dirty_list;
1681 bio_dirty_list = NULL;
1682 spin_unlock_irqrestore(&bio_dirty_lock, flags);
1683
1684 while (bio) {
1685 struct bio *next = bio->bi_private;
1686
1687 bio_set_pages_dirty(bio);
1688 bio_release_pages(bio);
1689 bio_put(bio);
1690 bio = next;
1691 }
1692}
1693
1694void bio_check_pages_dirty(struct bio *bio)
1695{
1696 struct bio_vec *bvec;
1697 int nr_clean_pages = 0;
1698 int i;
1699
1700 bio_for_each_segment_all(bvec, bio, i) {
1701 struct page *page = bvec->bv_page;
1702
1703 if (PageDirty(page) || PageCompound(page)) {
1704 page_cache_release(page);
1705 bvec->bv_page = NULL;
1706 } else {
1707 nr_clean_pages++;
1708 }
1709 }
1710
1711 if (nr_clean_pages) {
1712 unsigned long flags;
1713
1714 spin_lock_irqsave(&bio_dirty_lock, flags);
1715 bio->bi_private = bio_dirty_list;
1716 bio_dirty_list = bio;
1717 spin_unlock_irqrestore(&bio_dirty_lock, flags);
1718 schedule_work(&bio_dirty_work);
1719 } else {
1720 bio_put(bio);
1721 }
1722}
1723
1724#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
1725void bio_flush_dcache_pages(struct bio *bi)
1726{
1727 struct bio_vec bvec;
1728 struct bvec_iter iter;
1729
1730 bio_for_each_segment(bvec, bi, iter)
1731 flush_dcache_page(bvec.bv_page);
1732}
1733EXPORT_SYMBOL(bio_flush_dcache_pages);
1734#endif
1735
1736/**
1737 * bio_endio - end I/O on a bio
1738 * @bio: bio
1739 * @error: error, if any
1740 *
1741 * Description:
1742 * bio_endio() will end I/O on the whole bio. bio_endio() is the
1743 * preferred way to end I/O on a bio, it takes care of clearing
1744 * BIO_UPTODATE on error. @error is 0 on success, and and one of the
1745 * established -Exxxx (-EIO, for instance) error values in case
1746 * something went wrong. No one should call bi_end_io() directly on a
1747 * bio unless they own it and thus know that it has an end_io
1748 * function.
1749 **/
1750void bio_endio(struct bio *bio, int error)
1751{
1752 while (bio) {
1753 BUG_ON(atomic_read(&bio->bi_remaining) <= 0);
1754
1755 if (error)
1756 clear_bit(BIO_UPTODATE, &bio->bi_flags);
1757 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1758 error = -EIO;
1759
1760 if (!atomic_dec_and_test(&bio->bi_remaining))
1761 return;
1762
1763 /*
1764 * Need to have a real endio function for chained bios,
1765 * otherwise various corner cases will break (like stacking
1766 * block devices that save/restore bi_end_io) - however, we want
1767 * to avoid unbounded recursion and blowing the stack. Tail call
1768 * optimization would handle this, but compiling with frame
1769 * pointers also disables gcc's sibling call optimization.
1770 */
1771 if (bio->bi_end_io == bio_chain_endio) {
1772 struct bio *parent = bio->bi_private;
1773 bio_put(bio);
1774 bio = parent;
1775 } else {
1776 if (bio->bi_end_io)
1777 bio->bi_end_io(bio, error);
1778 bio = NULL;
1779 }
1780 }
1781}
1782EXPORT_SYMBOL(bio_endio);
1783
1784/**
1785 * bio_endio_nodec - end I/O on a bio, without decrementing bi_remaining
1786 * @bio: bio
1787 * @error: error, if any
1788 *
1789 * For code that has saved and restored bi_end_io; thing hard before using this
1790 * function, probably you should've cloned the entire bio.
1791 **/
1792void bio_endio_nodec(struct bio *bio, int error)
1793{
1794 atomic_inc(&bio->bi_remaining);
1795 bio_endio(bio, error);
1796}
1797EXPORT_SYMBOL(bio_endio_nodec);
1798
1799/**
1800 * bio_split - split a bio
1801 * @bio: bio to split
1802 * @sectors: number of sectors to split from the front of @bio
1803 * @gfp: gfp mask
1804 * @bs: bio set to allocate from
1805 *
1806 * Allocates and returns a new bio which represents @sectors from the start of
1807 * @bio, and updates @bio to represent the remaining sectors.
1808 *
1809 * The newly allocated bio will point to @bio's bi_io_vec; it is the caller's
1810 * responsibility to ensure that @bio is not freed before the split.
1811 */
1812struct bio *bio_split(struct bio *bio, int sectors,
1813 gfp_t gfp, struct bio_set *bs)
1814{
1815 struct bio *split = NULL;
1816
1817 BUG_ON(sectors <= 0);
1818 BUG_ON(sectors >= bio_sectors(bio));
1819
1820 split = bio_clone_fast(bio, gfp, bs);
1821 if (!split)
1822 return NULL;
1823
1824 split->bi_iter.bi_size = sectors << 9;
1825
1826 if (bio_integrity(split))
1827 bio_integrity_trim(split, 0, sectors);
1828
1829 bio_advance(bio, split->bi_iter.bi_size);
1830
1831 return split;
1832}
1833EXPORT_SYMBOL(bio_split);
1834
1835/**
1836 * bio_trim - trim a bio
1837 * @bio: bio to trim
1838 * @offset: number of sectors to trim from the front of @bio
1839 * @size: size we want to trim @bio to, in sectors
1840 */
1841void bio_trim(struct bio *bio, int offset, int size)
1842{
1843 /* 'bio' is a cloned bio which we need to trim to match
1844 * the given offset and size.
1845 */
1846
1847 size <<= 9;
1848 if (offset == 0 && size == bio->bi_iter.bi_size)
1849 return;
1850
1851 clear_bit(BIO_SEG_VALID, &bio->bi_flags);
1852
1853 bio_advance(bio, offset << 9);
1854
1855 bio->bi_iter.bi_size = size;
1856}
1857EXPORT_SYMBOL_GPL(bio_trim);
1858
1859/*
1860 * create memory pools for biovec's in a bio_set.
1861 * use the global biovec slabs created for general use.
1862 */
1863mempool_t *biovec_create_pool(int pool_entries)
1864{
1865 struct biovec_slab *bp = bvec_slabs + BIOVEC_MAX_IDX;
1866
1867 return mempool_create_slab_pool(pool_entries, bp->slab);
1868}
1869
1870void bioset_free(struct bio_set *bs)
1871{
1872 if (bs->rescue_workqueue)
1873 destroy_workqueue(bs->rescue_workqueue);
1874
1875 if (bs->bio_pool)
1876 mempool_destroy(bs->bio_pool);
1877
1878 if (bs->bvec_pool)
1879 mempool_destroy(bs->bvec_pool);
1880
1881 bioset_integrity_free(bs);
1882 bio_put_slab(bs);
1883
1884 kfree(bs);
1885}
1886EXPORT_SYMBOL(bioset_free);
1887
1888/**
1889 * bioset_create - Create a bio_set
1890 * @pool_size: Number of bio and bio_vecs to cache in the mempool
1891 * @front_pad: Number of bytes to allocate in front of the returned bio
1892 *
1893 * Description:
1894 * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
1895 * to ask for a number of bytes to be allocated in front of the bio.
1896 * Front pad allocation is useful for embedding the bio inside
1897 * another structure, to avoid allocating extra data to go with the bio.
1898 * Note that the bio must be embedded at the END of that structure always,
1899 * or things will break badly.
1900 */
1901struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
1902{
1903 unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1904 struct bio_set *bs;
1905
1906 bs = kzalloc(sizeof(*bs), GFP_KERNEL);
1907 if (!bs)
1908 return NULL;
1909
1910 bs->front_pad = front_pad;
1911
1912 spin_lock_init(&bs->rescue_lock);
1913 bio_list_init(&bs->rescue_list);
1914 INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
1915
1916 bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
1917 if (!bs->bio_slab) {
1918 kfree(bs);
1919 return NULL;
1920 }
1921
1922 bs->bio_pool = mempool_create_slab_pool(pool_size, bs->bio_slab);
1923 if (!bs->bio_pool)
1924 goto bad;
1925
1926 bs->bvec_pool = biovec_create_pool(pool_size);
1927 if (!bs->bvec_pool)
1928 goto bad;
1929
1930 bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
1931 if (!bs->rescue_workqueue)
1932 goto bad;
1933
1934 return bs;
1935bad:
1936 bioset_free(bs);
1937 return NULL;
1938}
1939EXPORT_SYMBOL(bioset_create);
1940
1941#ifdef CONFIG_BLK_CGROUP
1942/**
1943 * bio_associate_current - associate a bio with %current
1944 * @bio: target bio
1945 *
1946 * Associate @bio with %current if it hasn't been associated yet. Block
1947 * layer will treat @bio as if it were issued by %current no matter which
1948 * task actually issues it.
1949 *
1950 * This function takes an extra reference of @task's io_context and blkcg
1951 * which will be put when @bio is released. The caller must own @bio,
1952 * ensure %current->io_context exists, and is responsible for synchronizing
1953 * calls to this function.
1954 */
1955int bio_associate_current(struct bio *bio)
1956{
1957 struct io_context *ioc;
1958 struct cgroup_subsys_state *css;
1959
1960 if (bio->bi_ioc)
1961 return -EBUSY;
1962
1963 ioc = current->io_context;
1964 if (!ioc)
1965 return -ENOENT;
1966
1967 /* acquire active ref on @ioc and associate */
1968 get_io_context_active(ioc);
1969 bio->bi_ioc = ioc;
1970
1971 /* associate blkcg if exists */
1972 rcu_read_lock();
1973 css = task_css(current, blkio_cgrp_id);
1974 if (css && css_tryget(css))
1975 bio->bi_css = css;
1976 rcu_read_unlock();
1977
1978 return 0;
1979}
1980
1981/**
1982 * bio_disassociate_task - undo bio_associate_current()
1983 * @bio: target bio
1984 */
1985void bio_disassociate_task(struct bio *bio)
1986{
1987 if (bio->bi_ioc) {
1988 put_io_context(bio->bi_ioc);
1989 bio->bi_ioc = NULL;
1990 }
1991 if (bio->bi_css) {
1992 css_put(bio->bi_css);
1993 bio->bi_css = NULL;
1994 }
1995}
1996
1997#endif /* CONFIG_BLK_CGROUP */
1998
1999static void __init biovec_init_slabs(void)
2000{
2001 int i;
2002
2003 for (i = 0; i < BIOVEC_NR_POOLS; i++) {
2004 int size;
2005 struct biovec_slab *bvs = bvec_slabs + i;
2006
2007 if (bvs->nr_vecs <= BIO_INLINE_VECS) {
2008 bvs->slab = NULL;
2009 continue;
2010 }
2011
2012 size = bvs->nr_vecs * sizeof(struct bio_vec);
2013 bvs->slab = kmem_cache_create(bvs->name, size, 0,
2014 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2015 }
2016}
2017
2018static int __init init_bio(void)
2019{
2020 bio_slab_max = 2;
2021 bio_slab_nr = 0;
2022 bio_slabs = kzalloc(bio_slab_max * sizeof(struct bio_slab), GFP_KERNEL);
2023 if (!bio_slabs)
2024 panic("bio: can't allocate bios\n");
2025
2026 bio_integrity_init();
2027 biovec_init_slabs();
2028
2029 fs_bio_set = bioset_create(BIO_POOL_SIZE, 0);
2030 if (!fs_bio_set)
2031 panic("bio: can't allocate bios\n");
2032
2033 if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE))
2034 panic("bio: can't create integrity pool\n");
2035
2036 return 0;
2037}
2038subsys_initcall(init_bio);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-23 15:08:40 -0500