aboutsummaryrefslogtreecommitdiffstats
path: root/fs/bio.c
diff options
context:
space:
mode:
Diffstat (limited to 'fs/bio.c')
-rw-r--r--fs/bio.c502
1 files changed, 230 insertions, 272 deletions
diff --git a/fs/bio.c b/fs/bio.c
index 33d79a4eb92d..75c49a382239 100644
--- a/fs/bio.c
+++ b/fs/bio.c
@@ -38,8 +38,6 @@
38 */ 38 */
39#define BIO_INLINE_VECS 4 39#define BIO_INLINE_VECS 4
40 40
41static mempool_t *bio_split_pool __read_mostly;
42
43/* 41/*
44 * if you change this list, also change bvec_alloc or things will 42 * if you change this list, also change bvec_alloc or things will
45 * break badly! cannot be bigger than what you can fit into an 43 * break badly! cannot be bigger than what you can fit into an
@@ -273,6 +271,7 @@ void bio_init(struct bio *bio)
273{ 271{
274 memset(bio, 0, sizeof(*bio)); 272 memset(bio, 0, sizeof(*bio));
275 bio->bi_flags = 1 << BIO_UPTODATE; 273 bio->bi_flags = 1 << BIO_UPTODATE;
274 atomic_set(&bio->bi_remaining, 1);
276 atomic_set(&bio->bi_cnt, 1); 275 atomic_set(&bio->bi_cnt, 1);
277} 276}
278EXPORT_SYMBOL(bio_init); 277EXPORT_SYMBOL(bio_init);
@@ -295,9 +294,35 @@ void bio_reset(struct bio *bio)
295 294
296 memset(bio, 0, BIO_RESET_BYTES); 295 memset(bio, 0, BIO_RESET_BYTES);
297 bio->bi_flags = flags|(1 << BIO_UPTODATE); 296 bio->bi_flags = flags|(1 << BIO_UPTODATE);
297 atomic_set(&bio->bi_remaining, 1);
298} 298}
299EXPORT_SYMBOL(bio_reset); 299EXPORT_SYMBOL(bio_reset);
300 300
301static void bio_chain_endio(struct bio *bio, int error)
302{
303 bio_endio(bio->bi_private, error);
304 bio_put(bio);
305}
306
307/**
308 * bio_chain - chain bio completions
309 *
310 * The caller won't have a bi_end_io called when @bio completes - instead,
311 * @parent's bi_end_io won't be called until both @parent and @bio have
312 * completed; the chained bio will also be freed when it completes.
313 *
314 * The caller must not set bi_private or bi_end_io in @bio.
315 */
316void bio_chain(struct bio *bio, struct bio *parent)
317{
318 BUG_ON(bio->bi_private || bio->bi_end_io);
319
320 bio->bi_private = parent;
321 bio->bi_end_io = bio_chain_endio;
322 atomic_inc(&parent->bi_remaining);
323}
324EXPORT_SYMBOL(bio_chain);
325
301static void bio_alloc_rescue(struct work_struct *work) 326static void bio_alloc_rescue(struct work_struct *work)
302{ 327{
303 struct bio_set *bs = container_of(work, struct bio_set, rescue_work); 328 struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
@@ -473,13 +498,13 @@ EXPORT_SYMBOL(bio_alloc_bioset);
473void zero_fill_bio(struct bio *bio) 498void zero_fill_bio(struct bio *bio)
474{ 499{
475 unsigned long flags; 500 unsigned long flags;
476 struct bio_vec *bv; 501 struct bio_vec bv;
477 int i; 502 struct bvec_iter iter;
478 503
479 bio_for_each_segment(bv, bio, i) { 504 bio_for_each_segment(bv, bio, iter) {
480 char *data = bvec_kmap_irq(bv, &flags); 505 char *data = bvec_kmap_irq(&bv, &flags);
481 memset(data, 0, bv->bv_len); 506 memset(data, 0, bv.bv_len);
482 flush_dcache_page(bv->bv_page); 507 flush_dcache_page(bv.bv_page);
483 bvec_kunmap_irq(data, &flags); 508 bvec_kunmap_irq(data, &flags);
484 } 509 }
485} 510}
@@ -515,51 +540,49 @@ inline int bio_phys_segments(struct request_queue *q, struct bio *bio)
515EXPORT_SYMBOL(bio_phys_segments); 540EXPORT_SYMBOL(bio_phys_segments);
516 541
517/** 542/**
518 * __bio_clone - clone a bio 543 * __bio_clone_fast - clone a bio that shares the original bio's biovec
519 * @bio: destination bio 544 * @bio: destination bio
520 * @bio_src: bio to clone 545 * @bio_src: bio to clone
521 * 546 *
522 * Clone a &bio. Caller will own the returned bio, but not 547 * Clone a &bio. Caller will own the returned bio, but not
523 * the actual data it points to. Reference count of returned 548 * the actual data it points to. Reference count of returned
524 * bio will be one. 549 * bio will be one.
550 *
551 * Caller must ensure that @bio_src is not freed before @bio.
525 */ 552 */
526void __bio_clone(struct bio *bio, struct bio *bio_src) 553void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
527{ 554{
528 memcpy(bio->bi_io_vec, bio_src->bi_io_vec, 555 BUG_ON(bio->bi_pool && BIO_POOL_IDX(bio) != BIO_POOL_NONE);
529 bio_src->bi_max_vecs * sizeof(struct bio_vec));
530 556
531 /* 557 /*
532 * most users will be overriding ->bi_bdev with a new target, 558 * most users will be overriding ->bi_bdev with a new target,
533 * so we don't set nor calculate new physical/hw segment counts here 559 * so we don't set nor calculate new physical/hw segment counts here
534 */ 560 */
535 bio->bi_sector = bio_src->bi_sector;
536 bio->bi_bdev = bio_src->bi_bdev; 561 bio->bi_bdev = bio_src->bi_bdev;
537 bio->bi_flags |= 1 << BIO_CLONED; 562 bio->bi_flags |= 1 << BIO_CLONED;
538 bio->bi_rw = bio_src->bi_rw; 563 bio->bi_rw = bio_src->bi_rw;
539 bio->bi_vcnt = bio_src->bi_vcnt; 564 bio->bi_iter = bio_src->bi_iter;
540 bio->bi_size = bio_src->bi_size; 565 bio->bi_io_vec = bio_src->bi_io_vec;
541 bio->bi_idx = bio_src->bi_idx;
542} 566}
543EXPORT_SYMBOL(__bio_clone); 567EXPORT_SYMBOL(__bio_clone_fast);
544 568
545/** 569/**
546 * bio_clone_bioset - clone a bio 570 * bio_clone_fast - clone a bio that shares the original bio's biovec
547 * @bio: bio to clone 571 * @bio: bio to clone
548 * @gfp_mask: allocation priority 572 * @gfp_mask: allocation priority
549 * @bs: bio_set to allocate from 573 * @bs: bio_set to allocate from
550 * 574 *
551 * Like __bio_clone, only also allocates the returned bio 575 * Like __bio_clone_fast, only also allocates the returned bio
552 */ 576 */
553struct bio *bio_clone_bioset(struct bio *bio, gfp_t gfp_mask, 577struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
554 struct bio_set *bs)
555{ 578{
556 struct bio *b; 579 struct bio *b;
557 580
558 b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, bs); 581 b = bio_alloc_bioset(gfp_mask, 0, bs);
559 if (!b) 582 if (!b)
560 return NULL; 583 return NULL;
561 584
562 __bio_clone(b, bio); 585 __bio_clone_fast(b, bio);
563 586
564 if (bio_integrity(bio)) { 587 if (bio_integrity(bio)) {
565 int ret; 588 int ret;
@@ -574,6 +597,74 @@ struct bio *bio_clone_bioset(struct bio *bio, gfp_t gfp_mask,
574 597
575 return b; 598 return b;
576} 599}
600EXPORT_SYMBOL(bio_clone_fast);
601
602/**
603 * bio_clone_bioset - clone a bio
604 * @bio_src: bio to clone
605 * @gfp_mask: allocation priority
606 * @bs: bio_set to allocate from
607 *
608 * Clone bio. Caller will own the returned bio, but not the actual data it
609 * points to. Reference count of returned bio will be one.
610 */
611struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask,
612 struct bio_set *bs)
613{
614 unsigned nr_iovecs = 0;
615 struct bvec_iter iter;
616 struct bio_vec bv;
617 struct bio *bio;
618
619 /*
620 * Pre immutable biovecs, __bio_clone() used to just do a memcpy from
621 * bio_src->bi_io_vec to bio->bi_io_vec.
622 *
623 * We can't do that anymore, because:
624 *
625 * - The point of cloning the biovec is to produce a bio with a biovec
626 * the caller can modify: bi_idx and bi_bvec_done should be 0.
627 *
628 * - The original bio could've had more than BIO_MAX_PAGES biovecs; if
629 * we tried to clone the whole thing bio_alloc_bioset() would fail.
630 * But the clone should succeed as long as the number of biovecs we
631 * actually need to allocate is fewer than BIO_MAX_PAGES.
632 *
633 * - Lastly, bi_vcnt should not be looked at or relied upon by code
634 * that does not own the bio - reason being drivers don't use it for
635 * iterating over the biovec anymore, so expecting it to be kept up
636 * to date (i.e. for clones that share the parent biovec) is just
637 * asking for trouble and would force extra work on
638 * __bio_clone_fast() anyways.
639 */
640
641 bio_for_each_segment(bv, bio_src, iter)
642 nr_iovecs++;
643
644 bio = bio_alloc_bioset(gfp_mask, nr_iovecs, bs);
645 if (!bio)
646 return NULL;
647
648 bio->bi_bdev = bio_src->bi_bdev;
649 bio->bi_rw = bio_src->bi_rw;
650 bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
651 bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
652
653 bio_for_each_segment(bv, bio_src, iter)
654 bio->bi_io_vec[bio->bi_vcnt++] = bv;
655
656 if (bio_integrity(bio_src)) {
657 int ret;
658
659 ret = bio_integrity_clone(bio, bio_src, gfp_mask);
660 if (ret < 0) {
661 bio_put(bio);
662 return NULL;
663 }
664 }
665
666 return bio;
667}
577EXPORT_SYMBOL(bio_clone_bioset); 668EXPORT_SYMBOL(bio_clone_bioset);
578 669
579/** 670/**
@@ -612,7 +703,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
612 if (unlikely(bio_flagged(bio, BIO_CLONED))) 703 if (unlikely(bio_flagged(bio, BIO_CLONED)))
613 return 0; 704 return 0;
614 705
615 if (((bio->bi_size + len) >> 9) > max_sectors) 706 if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors)
616 return 0; 707 return 0;
617 708
618 /* 709 /*
@@ -635,8 +726,9 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
635 simulate merging updated prev_bvec 726 simulate merging updated prev_bvec
636 as new bvec. */ 727 as new bvec. */
637 .bi_bdev = bio->bi_bdev, 728 .bi_bdev = bio->bi_bdev,
638 .bi_sector = bio->bi_sector, 729 .bi_sector = bio->bi_iter.bi_sector,
639 .bi_size = bio->bi_size - prev_bv_len, 730 .bi_size = bio->bi_iter.bi_size -
731 prev_bv_len,
640 .bi_rw = bio->bi_rw, 732 .bi_rw = bio->bi_rw,
641 }; 733 };
642 734
@@ -684,8 +776,8 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
684 if (q->merge_bvec_fn) { 776 if (q->merge_bvec_fn) {
685 struct bvec_merge_data bvm = { 777 struct bvec_merge_data bvm = {
686 .bi_bdev = bio->bi_bdev, 778 .bi_bdev = bio->bi_bdev,
687 .bi_sector = bio->bi_sector, 779 .bi_sector = bio->bi_iter.bi_sector,
688 .bi_size = bio->bi_size, 780 .bi_size = bio->bi_iter.bi_size,
689 .bi_rw = bio->bi_rw, 781 .bi_rw = bio->bi_rw,
690 }; 782 };
691 783
@@ -708,7 +800,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
708 bio->bi_vcnt++; 800 bio->bi_vcnt++;
709 bio->bi_phys_segments++; 801 bio->bi_phys_segments++;
710 done: 802 done:
711 bio->bi_size += len; 803 bio->bi_iter.bi_size += len;
712 return len; 804 return len;
713} 805}
714 806
@@ -807,28 +899,7 @@ void bio_advance(struct bio *bio, unsigned bytes)
807 if (bio_integrity(bio)) 899 if (bio_integrity(bio))
808 bio_integrity_advance(bio, bytes); 900 bio_integrity_advance(bio, bytes);
809 901
810 bio->bi_sector += bytes >> 9; 902 bio_advance_iter(bio, &bio->bi_iter, bytes);
811 bio->bi_size -= bytes;
812
813 if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK)
814 return;
815
816 while (bytes) {
817 if (unlikely(bio->bi_idx >= bio->bi_vcnt)) {
818 WARN_ONCE(1, "bio idx %d >= vcnt %d\n",
819 bio->bi_idx, bio->bi_vcnt);
820 break;
821 }
822
823 if (bytes >= bio_iovec(bio)->bv_len) {
824 bytes -= bio_iovec(bio)->bv_len;
825 bio->bi_idx++;
826 } else {
827 bio_iovec(bio)->bv_len -= bytes;
828 bio_iovec(bio)->bv_offset += bytes;
829 bytes = 0;
830 }
831 }
832} 903}
833EXPORT_SYMBOL(bio_advance); 904EXPORT_SYMBOL(bio_advance);
834 905
@@ -874,117 +945,80 @@ EXPORT_SYMBOL(bio_alloc_pages);
874 */ 945 */
875void bio_copy_data(struct bio *dst, struct bio *src) 946void bio_copy_data(struct bio *dst, struct bio *src)
876{ 947{
877 struct bio_vec *src_bv, *dst_bv; 948 struct bvec_iter src_iter, dst_iter;
878 unsigned src_offset, dst_offset, bytes; 949 struct bio_vec src_bv, dst_bv;
879 void *src_p, *dst_p; 950 void *src_p, *dst_p;
951 unsigned bytes;
880 952
881 src_bv = bio_iovec(src); 953 src_iter = src->bi_iter;
882 dst_bv = bio_iovec(dst); 954 dst_iter = dst->bi_iter;
883
884 src_offset = src_bv->bv_offset;
885 dst_offset = dst_bv->bv_offset;
886 955
887 while (1) { 956 while (1) {
888 if (src_offset == src_bv->bv_offset + src_bv->bv_len) { 957 if (!src_iter.bi_size) {
889 src_bv++; 958 src = src->bi_next;
890 if (src_bv == bio_iovec_idx(src, src->bi_vcnt)) { 959 if (!src)
891 src = src->bi_next; 960 break;
892 if (!src)
893 break;
894
895 src_bv = bio_iovec(src);
896 }
897 961
898 src_offset = src_bv->bv_offset; 962 src_iter = src->bi_iter;
899 } 963 }
900 964
901 if (dst_offset == dst_bv->bv_offset + dst_bv->bv_len) { 965 if (!dst_iter.bi_size) {
902 dst_bv++; 966 dst = dst->bi_next;
903 if (dst_bv == bio_iovec_idx(dst, dst->bi_vcnt)) { 967 if (!dst)
904 dst = dst->bi_next; 968 break;
905 if (!dst)
906 break;
907
908 dst_bv = bio_iovec(dst);
909 }
910 969
911 dst_offset = dst_bv->bv_offset; 970 dst_iter = dst->bi_iter;
912 } 971 }
913 972
914 bytes = min(dst_bv->bv_offset + dst_bv->bv_len - dst_offset, 973 src_bv = bio_iter_iovec(src, src_iter);
915 src_bv->bv_offset + src_bv->bv_len - src_offset); 974 dst_bv = bio_iter_iovec(dst, dst_iter);
975
976 bytes = min(src_bv.bv_len, dst_bv.bv_len);
916 977
917 src_p = kmap_atomic(src_bv->bv_page); 978 src_p = kmap_atomic(src_bv.bv_page);
918 dst_p = kmap_atomic(dst_bv->bv_page); 979 dst_p = kmap_atomic(dst_bv.bv_page);
919 980
920 memcpy(dst_p + dst_offset, 981 memcpy(dst_p + dst_bv.bv_offset,
921 src_p + src_offset, 982 src_p + src_bv.bv_offset,
922 bytes); 983 bytes);
923 984
924 kunmap_atomic(dst_p); 985 kunmap_atomic(dst_p);
925 kunmap_atomic(src_p); 986 kunmap_atomic(src_p);
926 987
927 src_offset += bytes; 988 bio_advance_iter(src, &src_iter, bytes);
928 dst_offset += bytes; 989 bio_advance_iter(dst, &dst_iter, bytes);
929 } 990 }
930} 991}
931EXPORT_SYMBOL(bio_copy_data); 992EXPORT_SYMBOL(bio_copy_data);
932 993
933struct bio_map_data { 994struct bio_map_data {
934 struct bio_vec *iovecs;
935 struct sg_iovec *sgvecs;
936 int nr_sgvecs; 995 int nr_sgvecs;
937 int is_our_pages; 996 int is_our_pages;
997 struct sg_iovec sgvecs[];
938}; 998};
939 999
940static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio, 1000static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio,
941 struct sg_iovec *iov, int iov_count, 1001 struct sg_iovec *iov, int iov_count,
942 int is_our_pages) 1002 int is_our_pages)
943{ 1003{
944 memcpy(bmd->iovecs, bio->bi_io_vec, sizeof(struct bio_vec) * bio->bi_vcnt);
945 memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count); 1004 memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count);
946 bmd->nr_sgvecs = iov_count; 1005 bmd->nr_sgvecs = iov_count;
947 bmd->is_our_pages = is_our_pages; 1006 bmd->is_our_pages = is_our_pages;
948 bio->bi_private = bmd; 1007 bio->bi_private = bmd;
949} 1008}
950 1009
951static void bio_free_map_data(struct bio_map_data *bmd)
952{
953 kfree(bmd->iovecs);
954 kfree(bmd->sgvecs);
955 kfree(bmd);
956}
957
958static struct bio_map_data *bio_alloc_map_data(int nr_segs, 1010static struct bio_map_data *bio_alloc_map_data(int nr_segs,
959 unsigned int iov_count, 1011 unsigned int iov_count,
960 gfp_t gfp_mask) 1012 gfp_t gfp_mask)
961{ 1013{
962 struct bio_map_data *bmd;
963
964 if (iov_count > UIO_MAXIOV) 1014 if (iov_count > UIO_MAXIOV)
965 return NULL; 1015 return NULL;
966 1016
967 bmd = kmalloc(sizeof(*bmd), gfp_mask); 1017 return kmalloc(sizeof(struct bio_map_data) +
968 if (!bmd) 1018 sizeof(struct sg_iovec) * iov_count, gfp_mask);
969 return NULL;
970
971 bmd->iovecs = kmalloc(sizeof(struct bio_vec) * nr_segs, gfp_mask);
972 if (!bmd->iovecs) {
973 kfree(bmd);
974 return NULL;
975 }
976
977 bmd->sgvecs = kmalloc(sizeof(struct sg_iovec) * iov_count, gfp_mask);
978 if (bmd->sgvecs)
979 return bmd;
980
981 kfree(bmd->iovecs);
982 kfree(bmd);
983 return NULL;
984} 1019}
985 1020
986static int __bio_copy_iov(struct bio *bio, struct bio_vec *iovecs, 1021static int __bio_copy_iov(struct bio *bio, struct sg_iovec *iov, int iov_count,
987 struct sg_iovec *iov, int iov_count,
988 int to_user, int from_user, int do_free_page) 1022 int to_user, int from_user, int do_free_page)
989{ 1023{
990 int ret = 0, i; 1024 int ret = 0, i;
@@ -994,7 +1028,7 @@ static int __bio_copy_iov(struct bio *bio, struct bio_vec *iovecs,
994 1028
995 bio_for_each_segment_all(bvec, bio, i) { 1029 bio_for_each_segment_all(bvec, bio, i) {
996 char *bv_addr = page_address(bvec->bv_page); 1030 char *bv_addr = page_address(bvec->bv_page);
997 unsigned int bv_len = iovecs[i].bv_len; 1031 unsigned int bv_len = bvec->bv_len;
998 1032
999 while (bv_len && iov_idx < iov_count) { 1033 while (bv_len && iov_idx < iov_count) {
1000 unsigned int bytes; 1034 unsigned int bytes;
@@ -1054,14 +1088,14 @@ int bio_uncopy_user(struct bio *bio)
1054 * don't copy into a random user address space, just free. 1088 * don't copy into a random user address space, just free.
1055 */ 1089 */
1056 if (current->mm) 1090 if (current->mm)
1057 ret = __bio_copy_iov(bio, bmd->iovecs, bmd->sgvecs, 1091 ret = __bio_copy_iov(bio, bmd->sgvecs, bmd->nr_sgvecs,
1058 bmd->nr_sgvecs, bio_data_dir(bio) == READ, 1092 bio_data_dir(bio) == READ,
1059 0, bmd->is_our_pages); 1093 0, bmd->is_our_pages);
1060 else if (bmd->is_our_pages) 1094 else if (bmd->is_our_pages)
1061 bio_for_each_segment_all(bvec, bio, i) 1095 bio_for_each_segment_all(bvec, bio, i)
1062 __free_page(bvec->bv_page); 1096 __free_page(bvec->bv_page);
1063 } 1097 }
1064 bio_free_map_data(bmd); 1098 kfree(bmd);
1065 bio_put(bio); 1099 bio_put(bio);
1066 return ret; 1100 return ret;
1067} 1101}
@@ -1175,7 +1209,7 @@ struct bio *bio_copy_user_iov(struct request_queue *q,
1175 */ 1209 */
1176 if ((!write_to_vm && (!map_data || !map_data->null_mapped)) || 1210 if ((!write_to_vm && (!map_data || !map_data->null_mapped)) ||
1177 (map_data && map_data->from_user)) { 1211 (map_data && map_data->from_user)) {
1178 ret = __bio_copy_iov(bio, bio->bi_io_vec, iov, iov_count, 0, 1, 0); 1212 ret = __bio_copy_iov(bio, iov, iov_count, 0, 1, 0);
1179 if (ret) 1213 if (ret)
1180 goto cleanup; 1214 goto cleanup;
1181 } 1215 }
@@ -1189,7 +1223,7 @@ cleanup:
1189 1223
1190 bio_put(bio); 1224 bio_put(bio);
1191out_bmd: 1225out_bmd:
1192 bio_free_map_data(bmd); 1226 kfree(bmd);
1193 return ERR_PTR(ret); 1227 return ERR_PTR(ret);
1194} 1228}
1195 1229
@@ -1485,7 +1519,7 @@ struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
1485 if (IS_ERR(bio)) 1519 if (IS_ERR(bio))
1486 return bio; 1520 return bio;
1487 1521
1488 if (bio->bi_size == len) 1522 if (bio->bi_iter.bi_size == len)
1489 return bio; 1523 return bio;
1490 1524
1491 /* 1525 /*
@@ -1506,16 +1540,15 @@ static void bio_copy_kern_endio(struct bio *bio, int err)
1506 1540
1507 bio_for_each_segment_all(bvec, bio, i) { 1541 bio_for_each_segment_all(bvec, bio, i) {
1508 char *addr = page_address(bvec->bv_page); 1542 char *addr = page_address(bvec->bv_page);
1509 int len = bmd->iovecs[i].bv_len;
1510 1543
1511 if (read) 1544 if (read)
1512 memcpy(p, addr, len); 1545 memcpy(p, addr, bvec->bv_len);
1513 1546
1514 __free_page(bvec->bv_page); 1547 __free_page(bvec->bv_page);
1515 p += len; 1548 p += bvec->bv_len;
1516 } 1549 }
1517 1550
1518 bio_free_map_data(bmd); 1551 kfree(bmd);
1519 bio_put(bio); 1552 bio_put(bio);
1520} 1553}
1521 1554
@@ -1686,11 +1719,11 @@ void bio_check_pages_dirty(struct bio *bio)
1686#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1719#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
1687void bio_flush_dcache_pages(struct bio *bi) 1720void bio_flush_dcache_pages(struct bio *bi)
1688{ 1721{
1689 int i; 1722 struct bio_vec bvec;
1690 struct bio_vec *bvec; 1723 struct bvec_iter iter;
1691 1724
1692 bio_for_each_segment(bvec, bi, i) 1725 bio_for_each_segment(bvec, bi, iter)
1693 flush_dcache_page(bvec->bv_page); 1726 flush_dcache_page(bvec.bv_page);
1694} 1727}
1695EXPORT_SYMBOL(bio_flush_dcache_pages); 1728EXPORT_SYMBOL(bio_flush_dcache_pages);
1696#endif 1729#endif
@@ -1711,96 +1744,86 @@ EXPORT_SYMBOL(bio_flush_dcache_pages);
1711 **/ 1744 **/
1712void bio_endio(struct bio *bio, int error) 1745void bio_endio(struct bio *bio, int error)
1713{ 1746{
1714 if (error) 1747 while (bio) {
1715 clear_bit(BIO_UPTODATE, &bio->bi_flags); 1748 BUG_ON(atomic_read(&bio->bi_remaining) <= 0);
1716 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1717 error = -EIO;
1718 1749
1719 if (bio->bi_end_io) 1750 if (error)
1720 bio->bi_end_io(bio, error); 1751 clear_bit(BIO_UPTODATE, &bio->bi_flags);
1721} 1752 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1722EXPORT_SYMBOL(bio_endio); 1753 error = -EIO;
1723 1754
1724void bio_pair_release(struct bio_pair *bp) 1755 if (!atomic_dec_and_test(&bio->bi_remaining))
1725{ 1756 return;
1726 if (atomic_dec_and_test(&bp->cnt)) {
1727 struct bio *master = bp->bio1.bi_private;
1728 1757
1729 bio_endio(master, bp->error); 1758 /*
1730 mempool_free(bp, bp->bio2.bi_private); 1759 * Need to have a real endio function for chained bios,
1760 * otherwise various corner cases will break (like stacking
1761 * block devices that save/restore bi_end_io) - however, we want
1762 * to avoid unbounded recursion and blowing the stack. Tail call
1763 * optimization would handle this, but compiling with frame
1764 * pointers also disables gcc's sibling call optimization.
1765 */
1766 if (bio->bi_end_io == bio_chain_endio) {
1767 struct bio *parent = bio->bi_private;
1768 bio_put(bio);
1769 bio = parent;
1770 } else {
1771 if (bio->bi_end_io)
1772 bio->bi_end_io(bio, error);
1773 bio = NULL;
1774 }
1731 } 1775 }
1732} 1776}
1733EXPORT_SYMBOL(bio_pair_release); 1777EXPORT_SYMBOL(bio_endio);
1734 1778
1735static void bio_pair_end_1(struct bio *bi, int err) 1779/**
1780 * bio_endio_nodec - end I/O on a bio, without decrementing bi_remaining
1781 * @bio: bio
1782 * @error: error, if any
1783 *
1784 * For code that has saved and restored bi_end_io; thing hard before using this
1785 * function, probably you should've cloned the entire bio.
1786 **/
1787void bio_endio_nodec(struct bio *bio, int error)
1736{ 1788{
1737 struct bio_pair *bp = container_of(bi, struct bio_pair, bio1); 1789 atomic_inc(&bio->bi_remaining);
1738 1790 bio_endio(bio, error);
1739 if (err)
1740 bp->error = err;
1741
1742 bio_pair_release(bp);
1743} 1791}
1792EXPORT_SYMBOL(bio_endio_nodec);
1744 1793
1745static void bio_pair_end_2(struct bio *bi, int err) 1794/**
1746{ 1795 * bio_split - split a bio
1747 struct bio_pair *bp = container_of(bi, struct bio_pair, bio2); 1796 * @bio: bio to split
1748 1797 * @sectors: number of sectors to split from the front of @bio
1749 if (err) 1798 * @gfp: gfp mask
1750 bp->error = err; 1799 * @bs: bio set to allocate from
1751 1800 *
1752 bio_pair_release(bp); 1801 * Allocates and returns a new bio which represents @sectors from the start of
1753} 1802 * @bio, and updates @bio to represent the remaining sectors.
1754 1803 *
1755/* 1804 * The newly allocated bio will point to @bio's bi_io_vec; it is the caller's
1756 * split a bio - only worry about a bio with a single page in its iovec 1805 * responsibility to ensure that @bio is not freed before the split.
1757 */ 1806 */
1758struct bio_pair *bio_split(struct bio *bi, int first_sectors) 1807struct bio *bio_split(struct bio *bio, int sectors,
1808 gfp_t gfp, struct bio_set *bs)
1759{ 1809{
1760 struct bio_pair *bp = mempool_alloc(bio_split_pool, GFP_NOIO); 1810 struct bio *split = NULL;
1761
1762 if (!bp)
1763 return bp;
1764
1765 trace_block_split(bdev_get_queue(bi->bi_bdev), bi,
1766 bi->bi_sector + first_sectors);
1767
1768 BUG_ON(bio_segments(bi) > 1);
1769 atomic_set(&bp->cnt, 3);
1770 bp->error = 0;
1771 bp->bio1 = *bi;
1772 bp->bio2 = *bi;
1773 bp->bio2.bi_sector += first_sectors;
1774 bp->bio2.bi_size -= first_sectors << 9;
1775 bp->bio1.bi_size = first_sectors << 9;
1776
1777 if (bi->bi_vcnt != 0) {
1778 bp->bv1 = *bio_iovec(bi);
1779 bp->bv2 = *bio_iovec(bi);
1780
1781 if (bio_is_rw(bi)) {
1782 bp->bv2.bv_offset += first_sectors << 9;
1783 bp->bv2.bv_len -= first_sectors << 9;
1784 bp->bv1.bv_len = first_sectors << 9;
1785 }
1786 1811
1787 bp->bio1.bi_io_vec = &bp->bv1; 1812 BUG_ON(sectors <= 0);
1788 bp->bio2.bi_io_vec = &bp->bv2; 1813 BUG_ON(sectors >= bio_sectors(bio));
1789 1814
1790 bp->bio1.bi_max_vecs = 1; 1815 split = bio_clone_fast(bio, gfp, bs);
1791 bp->bio2.bi_max_vecs = 1; 1816 if (!split)
1792 } 1817 return NULL;
1793 1818
1794 bp->bio1.bi_end_io = bio_pair_end_1; 1819 split->bi_iter.bi_size = sectors << 9;
1795 bp->bio2.bi_end_io = bio_pair_end_2;
1796 1820
1797 bp->bio1.bi_private = bi; 1821 if (bio_integrity(split))
1798 bp->bio2.bi_private = bio_split_pool; 1822 bio_integrity_trim(split, 0, sectors);
1799 1823
1800 if (bio_integrity(bi)) 1824 bio_advance(bio, split->bi_iter.bi_size);
1801 bio_integrity_split(bi, bp, first_sectors);
1802 1825
1803 return bp; 1826 return split;
1804} 1827}
1805EXPORT_SYMBOL(bio_split); 1828EXPORT_SYMBOL(bio_split);
1806 1829
@@ -1814,80 +1837,20 @@ void bio_trim(struct bio *bio, int offset, int size)
1814{ 1837{
1815 /* 'bio' is a cloned bio which we need to trim to match 1838 /* 'bio' is a cloned bio which we need to trim to match
1816 * the given offset and size. 1839 * the given offset and size.
1817 * This requires adjusting bi_sector, bi_size, and bi_io_vec
1818 */ 1840 */
1819 int i;
1820 struct bio_vec *bvec;
1821 int sofar = 0;
1822 1841
1823 size <<= 9; 1842 size <<= 9;
1824 if (offset == 0 && size == bio->bi_size) 1843 if (offset == 0 && size == bio->bi_iter.bi_size)
1825 return; 1844 return;
1826 1845
1827 clear_bit(BIO_SEG_VALID, &bio->bi_flags); 1846 clear_bit(BIO_SEG_VALID, &bio->bi_flags);
1828 1847
1829 bio_advance(bio, offset << 9); 1848 bio_advance(bio, offset << 9);
1830 1849
1831 bio->bi_size = size; 1850 bio->bi_iter.bi_size = size;
1832
1833 /* avoid any complications with bi_idx being non-zero*/
1834 if (bio->bi_idx) {
1835 memmove(bio->bi_io_vec, bio->bi_io_vec+bio->bi_idx,
1836 (bio->bi_vcnt - bio->bi_idx) * sizeof(struct bio_vec));
1837 bio->bi_vcnt -= bio->bi_idx;
1838 bio->bi_idx = 0;
1839 }
1840 /* Make sure vcnt and last bv are not too big */
1841 bio_for_each_segment(bvec, bio, i) {
1842 if (sofar + bvec->bv_len > size)
1843 bvec->bv_len = size - sofar;
1844 if (bvec->bv_len == 0) {
1845 bio->bi_vcnt = i;
1846 break;
1847 }
1848 sofar += bvec->bv_len;
1849 }
1850} 1851}
1851EXPORT_SYMBOL_GPL(bio_trim); 1852EXPORT_SYMBOL_GPL(bio_trim);
1852 1853
1853/**
1854 * bio_sector_offset - Find hardware sector offset in bio
1855 * @bio: bio to inspect
1856 * @index: bio_vec index
1857 * @offset: offset in bv_page
1858 *
1859 * Return the number of hardware sectors between beginning of bio
1860 * and an end point indicated by a bio_vec index and an offset
1861 * within that vector's page.
1862 */
1863sector_t bio_sector_offset(struct bio *bio, unsigned short index,
1864 unsigned int offset)
1865{
1866 unsigned int sector_sz;
1867 struct bio_vec *bv;
1868 sector_t sectors;
1869 int i;
1870
1871 sector_sz = queue_logical_block_size(bio->bi_bdev->bd_disk->queue);
1872 sectors = 0;
1873
1874 if (index >= bio->bi_idx)
1875 index = bio->bi_vcnt - 1;
1876
1877 bio_for_each_segment_all(bv, bio, i) {
1878 if (i == index) {
1879 if (offset > bv->bv_offset)
1880 sectors += (offset - bv->bv_offset) / sector_sz;
1881 break;
1882 }
1883
1884 sectors += bv->bv_len / sector_sz;
1885 }
1886
1887 return sectors;
1888}
1889EXPORT_SYMBOL(bio_sector_offset);
1890
1891/* 1854/*
1892 * create memory pools for biovec's in a bio_set. 1855 * create memory pools for biovec's in a bio_set.
1893 * use the global biovec slabs created for general use. 1856 * use the global biovec slabs created for general use.
@@ -2065,11 +2028,6 @@ static int __init init_bio(void)
2065 if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE)) 2028 if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE))
2066 panic("bio: can't create integrity pool\n"); 2029 panic("bio: can't create integrity pool\n");
2067 2030
2068 bio_split_pool = mempool_create_kmalloc_pool(BIO_SPLIT_ENTRIES,
2069 sizeof(struct bio_pair));
2070 if (!bio_split_pool)
2071 panic("bio: can't create split pool\n");
2072
2073 return 0; 2031 return 0;
2074} 2032}
2075subsys_initcall(init_bio); 2033subsys_initcall(init_bio);
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-23 05:38:31 -0500