aboutsummaryrefslogtreecommitdiffstats
path: root/fs
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2014-01-30 14:19:05 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2014-01-30 14:19:05 -0500
commitf568849edac8611d603e00bd6cbbcfea09395ae6 (patch)
treeb9472d640fe5d87426d38c9d81d946cf197ad3fb /fs
parentd9894c228b11273e720bb63ba120d1d326fe9d94 (diff)
parent675675ada486dde5bf9aa51665e90706bff11a35 (diff)
Merge branch 'for-3.14/core' of git://git.kernel.dk/linux-block
Pull core block IO changes from Jens Axboe: "The major piece in here is the immutable bio_ve series from Kent, the rest is fairly minor. It was supposed to go in last round, but various issues pushed it to this release instead. The pull request contains: - Various smaller blk-mq fixes from different folks. Nothing major here, just minor fixes and cleanups. - Fix for a memory leak in the error path in the block ioctl code from Christian Engelmayer. - Header export fix from CaiZhiyong. - Finally the immutable biovec changes from Kent Overstreet. This enables some nice future work on making arbitrarily sized bios possible, and splitting more efficient. Related fixes to immutable bio_vecs: - dm-cache immutable fixup from Mike Snitzer. - btrfs immutable fixup from Muthu Kumar. - bio-integrity fix from Nic Bellinger, which is also going to stable" * 'for-3.14/core' of git://git.kernel.dk/linux-block: (44 commits) xtensa: fixup simdisk driver to work with immutable bio_vecs block/blk-mq-cpu.c: use hotcpu_notifier() blk-mq: for_each_* macro correctness block: Fix memory leak in rw_copy_check_uvector() handling bio-integrity: Fix bio_integrity_verify segment start bug block: remove unrelated header files and export symbol blk-mq: uses page->list incorrectly blk-mq: use __smp_call_function_single directly btrfs: fix missing increment of bi_remaining Revert "block: Warn and free bio if bi_end_io is not set" block: Warn and free bio if bi_end_io is not set blk-mq: fix initializing request's start time block: blk-mq: don't export blk_mq_free_queue() block: blk-mq: make blk_sync_queue support mq block: blk-mq: support draining mq queue dm cache: increment bi_remaining when bi_end_io is restored block: fixup for generic bio chaining block: Really silence spurious compiler warnings block: Silence spurious compiler warnings block: Kill bio_pair_split() ...
Diffstat (limited to 'fs')
-rw-r--r--fs/bio-integrity.c170
-rw-r--r--fs/bio.c502
-rw-r--r--fs/btrfs/check-integrity.c8
-rw-r--r--fs/btrfs/compression.c27
-rw-r--r--fs/btrfs/disk-io.c13
-rw-r--r--fs/btrfs/extent_io.c49
-rw-r--r--fs/btrfs/file-item.c19
-rw-r--r--fs/btrfs/inode.c37
-rw-r--r--fs/btrfs/raid56.c22
-rw-r--r--fs/btrfs/scrub.c12
-rw-r--r--fs/btrfs/volumes.c19
-rw-r--r--fs/buffer.c12
-rw-r--r--fs/direct-io.c4
-rw-r--r--fs/ext4/page-io.c8
-rw-r--r--fs/f2fs/data.c35
-rw-r--r--fs/gfs2/lops.c2
-rw-r--r--fs/gfs2/ops_fstype.c2
-rw-r--r--fs/hfsplus/wrapper.c2
-rw-r--r--fs/jfs/jfs_logmgr.c12
-rw-r--r--fs/jfs/jfs_metapage.c9
-rw-r--r--fs/logfs/dev_bdev.c38
-rw-r--r--fs/mpage.c19
-rw-r--r--fs/nfs/blocklayout/blocklayout.c43
-rw-r--r--fs/nilfs2/segbuf.c3
-rw-r--r--fs/ocfs2/cluster/heartbeat.c2
-rw-r--r--fs/xfs/xfs_aops.c2
-rw-r--r--fs/xfs/xfs_buf.c4
27 files changed, 456 insertions, 619 deletions
diff --git a/fs/bio-integrity.c b/fs/bio-integrity.c
index fc60b31453ee..0bad24ddc2e7 100644
--- a/fs/bio-integrity.c
+++ b/fs/bio-integrity.c
@@ -134,8 +134,7 @@ int bio_integrity_add_page(struct bio *bio, struct page *page,
134 return 0; 134 return 0;
135 } 135 }
136 136
137 iv = bip_vec_idx(bip, bip->bip_vcnt); 137 iv = bip->bip_vec + bip->bip_vcnt;
138 BUG_ON(iv == NULL);
139 138
140 iv->bv_page = page; 139 iv->bv_page = page;
141 iv->bv_len = len; 140 iv->bv_len = len;
@@ -203,6 +202,12 @@ static inline unsigned int bio_integrity_hw_sectors(struct blk_integrity *bi,
203 return sectors; 202 return sectors;
204} 203}
205 204
205static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi,
206 unsigned int sectors)
207{
208 return bio_integrity_hw_sectors(bi, sectors) * bi->tuple_size;
209}
210
206/** 211/**
207 * bio_integrity_tag_size - Retrieve integrity tag space 212 * bio_integrity_tag_size - Retrieve integrity tag space
208 * @bio: bio to inspect 213 * @bio: bio to inspect
@@ -215,9 +220,9 @@ unsigned int bio_integrity_tag_size(struct bio *bio)
215{ 220{
216 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 221 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
217 222
218 BUG_ON(bio->bi_size == 0); 223 BUG_ON(bio->bi_iter.bi_size == 0);
219 224
220 return bi->tag_size * (bio->bi_size / bi->sector_size); 225 return bi->tag_size * (bio->bi_iter.bi_size / bi->sector_size);
221} 226}
222EXPORT_SYMBOL(bio_integrity_tag_size); 227EXPORT_SYMBOL(bio_integrity_tag_size);
223 228
@@ -235,9 +240,9 @@ int bio_integrity_tag(struct bio *bio, void *tag_buf, unsigned int len, int set)
235 nr_sectors = bio_integrity_hw_sectors(bi, 240 nr_sectors = bio_integrity_hw_sectors(bi,
236 DIV_ROUND_UP(len, bi->tag_size)); 241 DIV_ROUND_UP(len, bi->tag_size));
237 242
238 if (nr_sectors * bi->tuple_size > bip->bip_size) { 243 if (nr_sectors * bi->tuple_size > bip->bip_iter.bi_size) {
239 printk(KERN_ERR "%s: tag too big for bio: %u > %u\n", 244 printk(KERN_ERR "%s: tag too big for bio: %u > %u\n", __func__,
240 __func__, nr_sectors * bi->tuple_size, bip->bip_size); 245 nr_sectors * bi->tuple_size, bip->bip_iter.bi_size);
241 return -1; 246 return -1;
242 } 247 }
243 248
@@ -299,29 +304,30 @@ static void bio_integrity_generate(struct bio *bio)
299{ 304{
300 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 305 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
301 struct blk_integrity_exchg bix; 306 struct blk_integrity_exchg bix;
302 struct bio_vec *bv; 307 struct bio_vec bv;
303 sector_t sector = bio->bi_sector; 308 struct bvec_iter iter;
304 unsigned int i, sectors, total; 309 sector_t sector = bio->bi_iter.bi_sector;
310 unsigned int sectors, total;
305 void *prot_buf = bio->bi_integrity->bip_buf; 311 void *prot_buf = bio->bi_integrity->bip_buf;
306 312
307 total = 0; 313 total = 0;
308 bix.disk_name = bio->bi_bdev->bd_disk->disk_name; 314 bix.disk_name = bio->bi_bdev->bd_disk->disk_name;
309 bix.sector_size = bi->sector_size; 315 bix.sector_size = bi->sector_size;
310 316
311 bio_for_each_segment(bv, bio, i) { 317 bio_for_each_segment(bv, bio, iter) {
312 void *kaddr = kmap_atomic(bv->bv_page); 318 void *kaddr = kmap_atomic(bv.bv_page);
313 bix.data_buf = kaddr + bv->bv_offset; 319 bix.data_buf = kaddr + bv.bv_offset;
314 bix.data_size = bv->bv_len; 320 bix.data_size = bv.bv_len;
315 bix.prot_buf = prot_buf; 321 bix.prot_buf = prot_buf;
316 bix.sector = sector; 322 bix.sector = sector;
317 323
318 bi->generate_fn(&bix); 324 bi->generate_fn(&bix);
319 325
320 sectors = bv->bv_len / bi->sector_size; 326 sectors = bv.bv_len / bi->sector_size;
321 sector += sectors; 327 sector += sectors;
322 prot_buf += sectors * bi->tuple_size; 328 prot_buf += sectors * bi->tuple_size;
323 total += sectors * bi->tuple_size; 329 total += sectors * bi->tuple_size;
324 BUG_ON(total > bio->bi_integrity->bip_size); 330 BUG_ON(total > bio->bi_integrity->bip_iter.bi_size);
325 331
326 kunmap_atomic(kaddr); 332 kunmap_atomic(kaddr);
327 } 333 }
@@ -386,8 +392,8 @@ int bio_integrity_prep(struct bio *bio)
386 392
387 bip->bip_owns_buf = 1; 393 bip->bip_owns_buf = 1;
388 bip->bip_buf = buf; 394 bip->bip_buf = buf;
389 bip->bip_size = len; 395 bip->bip_iter.bi_size = len;
390 bip->bip_sector = bio->bi_sector; 396 bip->bip_iter.bi_sector = bio->bi_iter.bi_sector;
391 397
392 /* Map it */ 398 /* Map it */
393 offset = offset_in_page(buf); 399 offset = offset_in_page(buf);
@@ -442,16 +448,18 @@ static int bio_integrity_verify(struct bio *bio)
442 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 448 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
443 struct blk_integrity_exchg bix; 449 struct blk_integrity_exchg bix;
444 struct bio_vec *bv; 450 struct bio_vec *bv;
445 sector_t sector = bio->bi_integrity->bip_sector; 451 sector_t sector = bio->bi_integrity->bip_iter.bi_sector;
446 unsigned int i, sectors, total, ret; 452 unsigned int sectors, total, ret;
447 void *prot_buf = bio->bi_integrity->bip_buf; 453 void *prot_buf = bio->bi_integrity->bip_buf;
454 int i;
448 455
449 ret = total = 0; 456 ret = total = 0;
450 bix.disk_name = bio->bi_bdev->bd_disk->disk_name; 457 bix.disk_name = bio->bi_bdev->bd_disk->disk_name;
451 bix.sector_size = bi->sector_size; 458 bix.sector_size = bi->sector_size;
452 459
453 bio_for_each_segment(bv, bio, i) { 460 bio_for_each_segment_all(bv, bio, i) {
454 void *kaddr = kmap_atomic(bv->bv_page); 461 void *kaddr = kmap_atomic(bv->bv_page);
462
455 bix.data_buf = kaddr + bv->bv_offset; 463 bix.data_buf = kaddr + bv->bv_offset;
456 bix.data_size = bv->bv_len; 464 bix.data_size = bv->bv_len;
457 bix.prot_buf = prot_buf; 465 bix.prot_buf = prot_buf;
@@ -468,7 +476,7 @@ static int bio_integrity_verify(struct bio *bio)
468 sector += sectors; 476 sector += sectors;
469 prot_buf += sectors * bi->tuple_size; 477 prot_buf += sectors * bi->tuple_size;
470 total += sectors * bi->tuple_size; 478 total += sectors * bi->tuple_size;
471 BUG_ON(total > bio->bi_integrity->bip_size); 479 BUG_ON(total > bio->bi_integrity->bip_iter.bi_size);
472 480
473 kunmap_atomic(kaddr); 481 kunmap_atomic(kaddr);
474 } 482 }
@@ -495,7 +503,7 @@ static void bio_integrity_verify_fn(struct work_struct *work)
495 503
496 /* Restore original bio completion handler */ 504 /* Restore original bio completion handler */
497 bio->bi_end_io = bip->bip_end_io; 505 bio->bi_end_io = bip->bip_end_io;
498 bio_endio(bio, error); 506 bio_endio_nodec(bio, error);
499} 507}
500 508
501/** 509/**
@@ -533,56 +541,6 @@ void bio_integrity_endio(struct bio *bio, int error)
533EXPORT_SYMBOL(bio_integrity_endio); 541EXPORT_SYMBOL(bio_integrity_endio);
534 542
535/** 543/**
536 * bio_integrity_mark_head - Advance bip_vec skip bytes
537 * @bip: Integrity vector to advance
538 * @skip: Number of bytes to advance it
539 */
540void bio_integrity_mark_head(struct bio_integrity_payload *bip,
541 unsigned int skip)
542{
543 struct bio_vec *iv;
544 unsigned int i;
545
546 bip_for_each_vec(iv, bip, i) {
547 if (skip == 0) {
548 bip->bip_idx = i;
549 return;
550 } else if (skip >= iv->bv_len) {
551 skip -= iv->bv_len;
552 } else { /* skip < iv->bv_len) */
553 iv->bv_offset += skip;
554 iv->bv_len -= skip;
555 bip->bip_idx = i;
556 return;
557 }
558 }
559}
560
561/**
562 * bio_integrity_mark_tail - Truncate bip_vec to be len bytes long
563 * @bip: Integrity vector to truncate
564 * @len: New length of integrity vector
565 */
566void bio_integrity_mark_tail(struct bio_integrity_payload *bip,
567 unsigned int len)
568{
569 struct bio_vec *iv;
570 unsigned int i;
571
572 bip_for_each_vec(iv, bip, i) {
573 if (len == 0) {
574 bip->bip_vcnt = i;
575 return;
576 } else if (len >= iv->bv_len) {
577 len -= iv->bv_len;
578 } else { /* len < iv->bv_len) */
579 iv->bv_len = len;
580 len = 0;
581 }
582 }
583}
584
585/**
586 * bio_integrity_advance - Advance integrity vector 544 * bio_integrity_advance - Advance integrity vector
587 * @bio: bio whose integrity vector to update 545 * @bio: bio whose integrity vector to update
588 * @bytes_done: number of data bytes that have been completed 546 * @bytes_done: number of data bytes that have been completed
@@ -595,13 +553,9 @@ void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
595{ 553{
596 struct bio_integrity_payload *bip = bio->bi_integrity; 554 struct bio_integrity_payload *bip = bio->bi_integrity;
597 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 555 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
598 unsigned int nr_sectors; 556 unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
599 557
600 BUG_ON(bip == NULL); 558 bvec_iter_advance(bip->bip_vec, &bip->bip_iter, bytes);
601 BUG_ON(bi == NULL);
602
603 nr_sectors = bio_integrity_hw_sectors(bi, bytes_done >> 9);
604 bio_integrity_mark_head(bip, nr_sectors * bi->tuple_size);
605} 559}
606EXPORT_SYMBOL(bio_integrity_advance); 560EXPORT_SYMBOL(bio_integrity_advance);
607 561
@@ -621,64 +575,13 @@ void bio_integrity_trim(struct bio *bio, unsigned int offset,
621{ 575{
622 struct bio_integrity_payload *bip = bio->bi_integrity; 576 struct bio_integrity_payload *bip = bio->bi_integrity;
623 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 577 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
624 unsigned int nr_sectors;
625
626 BUG_ON(bip == NULL);
627 BUG_ON(bi == NULL);
628 BUG_ON(!bio_flagged(bio, BIO_CLONED));
629 578
630 nr_sectors = bio_integrity_hw_sectors(bi, sectors); 579 bio_integrity_advance(bio, offset << 9);
631 bip->bip_sector = bip->bip_sector + offset; 580 bip->bip_iter.bi_size = bio_integrity_bytes(bi, sectors);
632 bio_integrity_mark_head(bip, offset * bi->tuple_size);
633 bio_integrity_mark_tail(bip, sectors * bi->tuple_size);
634} 581}
635EXPORT_SYMBOL(bio_integrity_trim); 582EXPORT_SYMBOL(bio_integrity_trim);
636 583
637/** 584/**
638 * bio_integrity_split - Split integrity metadata
639 * @bio: Protected bio
640 * @bp: Resulting bio_pair
641 * @sectors: Offset
642 *
643 * Description: Splits an integrity page into a bio_pair.
644 */
645void bio_integrity_split(struct bio *bio, struct bio_pair *bp, int sectors)
646{
647 struct blk_integrity *bi;
648 struct bio_integrity_payload *bip = bio->bi_integrity;
649 unsigned int nr_sectors;
650
651 if (bio_integrity(bio) == 0)
652 return;
653
654 bi = bdev_get_integrity(bio->bi_bdev);
655 BUG_ON(bi == NULL);
656 BUG_ON(bip->bip_vcnt != 1);
657
658 nr_sectors = bio_integrity_hw_sectors(bi, sectors);
659
660 bp->bio1.bi_integrity = &bp->bip1;
661 bp->bio2.bi_integrity = &bp->bip2;
662
663 bp->iv1 = bip->bip_vec[bip->bip_idx];
664 bp->iv2 = bip->bip_vec[bip->bip_idx];
665
666 bp->bip1.bip_vec = &bp->iv1;
667 bp->bip2.bip_vec = &bp->iv2;
668
669 bp->iv1.bv_len = sectors * bi->tuple_size;
670 bp->iv2.bv_offset += sectors * bi->tuple_size;
671 bp->iv2.bv_len -= sectors * bi->tuple_size;
672
673 bp->bip1.bip_sector = bio->bi_integrity->bip_sector;
674 bp->bip2.bip_sector = bio->bi_integrity->bip_sector + nr_sectors;
675
676 bp->bip1.bip_vcnt = bp->bip2.bip_vcnt = 1;
677 bp->bip1.bip_idx = bp->bip2.bip_idx = 0;
678}
679EXPORT_SYMBOL(bio_integrity_split);
680
681/**
682 * bio_integrity_clone - Callback for cloning bios with integrity metadata 585 * bio_integrity_clone - Callback for cloning bios with integrity metadata
683 * @bio: New bio 586 * @bio: New bio
684 * @bio_src: Original bio 587 * @bio_src: Original bio
@@ -702,9 +605,8 @@ int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
702 memcpy(bip->bip_vec, bip_src->bip_vec, 605 memcpy(bip->bip_vec, bip_src->bip_vec,
703 bip_src->bip_vcnt * sizeof(struct bio_vec)); 606 bip_src->bip_vcnt * sizeof(struct bio_vec));
704 607
705 bip->bip_sector = bip_src->bip_sector;
706 bip->bip_vcnt = bip_src->bip_vcnt; 608 bip->bip_vcnt = bip_src->bip_vcnt;
707 bip->bip_idx = bip_src->bip_idx; 609 bip->bip_iter = bip_src->bip_iter;
708 610
709 return 0; 611 return 0;
710} 612}
diff --git a/fs/bio.c b/fs/bio.c
index 33d79a4eb92d..75c49a382239 100644
--- a/fs/bio.c
+++ b/fs/bio.c
@@ -38,8 +38,6 @@
38 */ 38 */
39#define BIO_INLINE_VECS 4 39#define BIO_INLINE_VECS 4
40 40
41static mempool_t *bio_split_pool __read_mostly;
42
43/* 41/*
44 * if you change this list, also change bvec_alloc or things will 42 * if you change this list, also change bvec_alloc or things will
45 * break badly! cannot be bigger than what you can fit into an 43 * break badly! cannot be bigger than what you can fit into an
@@ -273,6 +271,7 @@ void bio_init(struct bio *bio)
273{ 271{
274 memset(bio, 0, sizeof(*bio)); 272 memset(bio, 0, sizeof(*bio));
275 bio->bi_flags = 1 << BIO_UPTODATE; 273 bio->bi_flags = 1 << BIO_UPTODATE;
274 atomic_set(&bio->bi_remaining, 1);
276 atomic_set(&bio->bi_cnt, 1); 275 atomic_set(&bio->bi_cnt, 1);
277} 276}
278EXPORT_SYMBOL(bio_init); 277EXPORT_SYMBOL(bio_init);
@@ -295,9 +294,35 @@ void bio_reset(struct bio *bio)
295 294
296 memset(bio, 0, BIO_RESET_BYTES); 295 memset(bio, 0, BIO_RESET_BYTES);
297 bio->bi_flags = flags|(1 << BIO_UPTODATE); 296 bio->bi_flags = flags|(1 << BIO_UPTODATE);
297 atomic_set(&bio->bi_remaining, 1);
298} 298}
299EXPORT_SYMBOL(bio_reset); 299EXPORT_SYMBOL(bio_reset);
300 300
301static void bio_chain_endio(struct bio *bio, int error)
302{
303 bio_endio(bio->bi_private, error);
304 bio_put(bio);
305}
306
307/**
308 * bio_chain - chain bio completions
309 *
310 * The caller won't have a bi_end_io called when @bio completes - instead,
311 * @parent's bi_end_io won't be called until both @parent and @bio have
312 * completed; the chained bio will also be freed when it completes.
313 *
314 * The caller must not set bi_private or bi_end_io in @bio.
315 */
316void bio_chain(struct bio *bio, struct bio *parent)
317{
318 BUG_ON(bio->bi_private || bio->bi_end_io);
319
320 bio->bi_private = parent;
321 bio->bi_end_io = bio_chain_endio;
322 atomic_inc(&parent->bi_remaining);
323}
324EXPORT_SYMBOL(bio_chain);
325
301static void bio_alloc_rescue(struct work_struct *work) 326static void bio_alloc_rescue(struct work_struct *work)
302{ 327{
303 struct bio_set *bs = container_of(work, struct bio_set, rescue_work); 328 struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
@@ -473,13 +498,13 @@ EXPORT_SYMBOL(bio_alloc_bioset);
473void zero_fill_bio(struct bio *bio) 498void zero_fill_bio(struct bio *bio)
474{ 499{
475 unsigned long flags; 500 unsigned long flags;
476 struct bio_vec *bv; 501 struct bio_vec bv;
477 int i; 502 struct bvec_iter iter;
478 503
479 bio_for_each_segment(bv, bio, i) { 504 bio_for_each_segment(bv, bio, iter) {
480 char *data = bvec_kmap_irq(bv, &flags); 505 char *data = bvec_kmap_irq(&bv, &flags);
481 memset(data, 0, bv->bv_len); 506 memset(data, 0, bv.bv_len);
482 flush_dcache_page(bv->bv_page); 507 flush_dcache_page(bv.bv_page);
483 bvec_kunmap_irq(data, &flags); 508 bvec_kunmap_irq(data, &flags);
484 } 509 }
485} 510}
@@ -515,51 +540,49 @@ inline int bio_phys_segments(struct request_queue *q, struct bio *bio)
515EXPORT_SYMBOL(bio_phys_segments); 540EXPORT_SYMBOL(bio_phys_segments);
516 541
517/** 542/**
518 * __bio_clone - clone a bio 543 * __bio_clone_fast - clone a bio that shares the original bio's biovec
519 * @bio: destination bio 544 * @bio: destination bio
520 * @bio_src: bio to clone 545 * @bio_src: bio to clone
521 * 546 *
522 * Clone a &bio. Caller will own the returned bio, but not 547 * Clone a &bio. Caller will own the returned bio, but not
523 * the actual data it points to. Reference count of returned 548 * the actual data it points to. Reference count of returned
524 * bio will be one. 549 * bio will be one.
550 *
551 * Caller must ensure that @bio_src is not freed before @bio.
525 */ 552 */
526void __bio_clone(struct bio *bio, struct bio *bio_src) 553void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
527{ 554{
528 memcpy(bio->bi_io_vec, bio_src->bi_io_vec, 555 BUG_ON(bio->bi_pool && BIO_POOL_IDX(bio) != BIO_POOL_NONE);
529 bio_src->bi_max_vecs * sizeof(struct bio_vec));
530 556
531 /* 557 /*
532 * most users will be overriding ->bi_bdev with a new target, 558 * most users will be overriding ->bi_bdev with a new target,
533 * so we don't set nor calculate new physical/hw segment counts here 559 * so we don't set nor calculate new physical/hw segment counts here
534 */ 560 */
535 bio->bi_sector = bio_src->bi_sector;
536 bio->bi_bdev = bio_src->bi_bdev; 561 bio->bi_bdev = bio_src->bi_bdev;
537 bio->bi_flags |= 1 << BIO_CLONED; 562 bio->bi_flags |= 1 << BIO_CLONED;
538 bio->bi_rw = bio_src->bi_rw; 563 bio->bi_rw = bio_src->bi_rw;
539 bio->bi_vcnt = bio_src->bi_vcnt; 564 bio->bi_iter = bio_src->bi_iter;
540 bio->bi_size = bio_src->bi_size; 565 bio->bi_io_vec = bio_src->bi_io_vec;
541 bio->bi_idx = bio_src->bi_idx;
542} 566}
543EXPORT_SYMBOL(__bio_clone); 567EXPORT_SYMBOL(__bio_clone_fast);
544 568
545/** 569/**
546 * bio_clone_bioset - clone a bio 570 * bio_clone_fast - clone a bio that shares the original bio's biovec
547 * @bio: bio to clone 571 * @bio: bio to clone
548 * @gfp_mask: allocation priority 572 * @gfp_mask: allocation priority
549 * @bs: bio_set to allocate from 573 * @bs: bio_set to allocate from
550 * 574 *
551 * Like __bio_clone, only also allocates the returned bio 575 * Like __bio_clone_fast, only also allocates the returned bio
552 */ 576 */
553struct bio *bio_clone_bioset(struct bio *bio, gfp_t gfp_mask, 577struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
554 struct bio_set *bs)
555{ 578{
556 struct bio *b; 579 struct bio *b;
557 580
558 b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, bs); 581 b = bio_alloc_bioset(gfp_mask, 0, bs);
559 if (!b) 582 if (!b)
560 return NULL; 583 return NULL;
561 584
562 __bio_clone(b, bio); 585 __bio_clone_fast(b, bio);
563 586
564 if (bio_integrity(bio)) { 587 if (bio_integrity(bio)) {
565 int ret; 588 int ret;
@@ -574,6 +597,74 @@ struct bio *bio_clone_bioset(struct bio *bio, gfp_t gfp_mask,
574 597
575 return b; 598 return b;
576} 599}
600EXPORT_SYMBOL(bio_clone_fast);
601
602/**
603 * bio_clone_bioset - clone a bio
604 * @bio_src: bio to clone
605 * @gfp_mask: allocation priority
606 * @bs: bio_set to allocate from
607 *
608 * Clone bio. Caller will own the returned bio, but not the actual data it
609 * points to. Reference count of returned bio will be one.
610 */
611struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask,
612 struct bio_set *bs)
613{
614 unsigned nr_iovecs = 0;
615 struct bvec_iter iter;
616 struct bio_vec bv;
617 struct bio *bio;
618
619 /*
620 * Pre immutable biovecs, __bio_clone() used to just do a memcpy from
621 * bio_src->bi_io_vec to bio->bi_io_vec.
622 *
623 * We can't do that anymore, because:
624 *
625 * - The point of cloning the biovec is to produce a bio with a biovec
626 * the caller can modify: bi_idx and bi_bvec_done should be 0.
627 *
628 * - The original bio could've had more than BIO_MAX_PAGES biovecs; if
629 * we tried to clone the whole thing bio_alloc_bioset() would fail.
630 * But the clone should succeed as long as the number of biovecs we
631 * actually need to allocate is fewer than BIO_MAX_PAGES.
632 *
633 * - Lastly, bi_vcnt should not be looked at or relied upon by code
634 * that does not own the bio - reason being drivers don't use it for
635 * iterating over the biovec anymore, so expecting it to be kept up
636 * to date (i.e. for clones that share the parent biovec) is just
637 * asking for trouble and would force extra work on
638 * __bio_clone_fast() anyways.
639 */
640
641 bio_for_each_segment(bv, bio_src, iter)
642 nr_iovecs++;
643
644 bio = bio_alloc_bioset(gfp_mask, nr_iovecs, bs);
645 if (!bio)
646 return NULL;
647
648 bio->bi_bdev = bio_src->bi_bdev;
649 bio->bi_rw = bio_src->bi_rw;
650 bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
651 bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
652
653 bio_for_each_segment(bv, bio_src, iter)
654 bio->bi_io_vec[bio->bi_vcnt++] = bv;
655
656 if (bio_integrity(bio_src)) {
657 int ret;
658
659 ret = bio_integrity_clone(bio, bio_src, gfp_mask);
660 if (ret < 0) {
661 bio_put(bio);
662 return NULL;
663 }
664 }
665
666 return bio;
667}
577EXPORT_SYMBOL(bio_clone_bioset); 668EXPORT_SYMBOL(bio_clone_bioset);
578 669
579/** 670/**
@@ -612,7 +703,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
612 if (unlikely(bio_flagged(bio, BIO_CLONED))) 703 if (unlikely(bio_flagged(bio, BIO_CLONED)))
613 return 0; 704 return 0;
614 705
615 if (((bio->bi_size + len) >> 9) > max_sectors) 706 if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors)
616 return 0; 707 return 0;
617 708
618 /* 709 /*
@@ -635,8 +726,9 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
635 simulate merging updated prev_bvec 726 simulate merging updated prev_bvec
636 as new bvec. */ 727 as new bvec. */
637 .bi_bdev = bio->bi_bdev, 728 .bi_bdev = bio->bi_bdev,
638 .bi_sector = bio->bi_sector, 729 .bi_sector = bio->bi_iter.bi_sector,
639 .bi_size = bio->bi_size - prev_bv_len, 730 .bi_size = bio->bi_iter.bi_size -
731 prev_bv_len,
640 .bi_rw = bio->bi_rw, 732 .bi_rw = bio->bi_rw,
641 }; 733 };
642 734
@@ -684,8 +776,8 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
684 if (q->merge_bvec_fn) { 776 if (q->merge_bvec_fn) {
685 struct bvec_merge_data bvm = { 777 struct bvec_merge_data bvm = {
686 .bi_bdev = bio->bi_bdev, 778 .bi_bdev = bio->bi_bdev,
687 .bi_sector = bio->bi_sector, 779 .bi_sector = bio->bi_iter.bi_sector,
688 .bi_size = bio->bi_size, 780 .bi_size = bio->bi_iter.bi_size,
689 .bi_rw = bio->bi_rw, 781 .bi_rw = bio->bi_rw,
690 }; 782 };
691 783
@@ -708,7 +800,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
708 bio->bi_vcnt++; 800 bio->bi_vcnt++;
709 bio->bi_phys_segments++; 801 bio->bi_phys_segments++;
710 done: 802 done:
711 bio->bi_size += len; 803 bio->bi_iter.bi_size += len;
712 return len; 804 return len;
713} 805}
714 806
@@ -807,28 +899,7 @@ void bio_advance(struct bio *bio, unsigned bytes)
807 if (bio_integrity(bio)) 899 if (bio_integrity(bio))
808 bio_integrity_advance(bio, bytes); 900 bio_integrity_advance(bio, bytes);
809 901
810 bio->bi_sector += bytes >> 9; 902 bio_advance_iter(bio, &bio->bi_iter, bytes);
811 bio->bi_size -= bytes;
812
813 if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK)
814 return;
815
816 while (bytes) {
817 if (unlikely(bio->bi_idx >= bio->bi_vcnt)) {
818 WARN_ONCE(1, "bio idx %d >= vcnt %d\n",
819 bio->bi_idx, bio->bi_vcnt);
820 break;
821 }
822
823 if (bytes >= bio_iovec(bio)->bv_len) {
824 bytes -= bio_iovec(bio)->bv_len;
825 bio->bi_idx++;
826 } else {
827 bio_iovec(bio)->bv_len -= bytes;
828 bio_iovec(bio)->bv_offset += bytes;
829 bytes = 0;
830 }
831 }
832} 903}
833EXPORT_SYMBOL(bio_advance); 904EXPORT_SYMBOL(bio_advance);
834 905
@@ -874,117 +945,80 @@ EXPORT_SYMBOL(bio_alloc_pages);
874 */ 945 */
875void bio_copy_data(struct bio *dst, struct bio *src) 946void bio_copy_data(struct bio *dst, struct bio *src)
876{ 947{
877 struct bio_vec *src_bv, *dst_bv; 948 struct bvec_iter src_iter, dst_iter;
878 unsigned src_offset, dst_offset, bytes; 949 struct bio_vec src_bv, dst_bv;
879 void *src_p, *dst_p; 950 void *src_p, *dst_p;
951 unsigned bytes;
880 952
881 src_bv = bio_iovec(src); 953 src_iter = src->bi_iter;
882 dst_bv = bio_iovec(dst); 954 dst_iter = dst->bi_iter;
883
884 src_offset = src_bv->bv_offset;
885 dst_offset = dst_bv->bv_offset;
886 955
887 while (1) { 956 while (1) {
888 if (src_offset == src_bv->bv_offset + src_bv->bv_len) { 957 if (!src_iter.bi_size) {
889 src_bv++; 958 src = src->bi_next;
890 if (src_bv == bio_iovec_idx(src, src->bi_vcnt)) { 959 if (!src)
891 src = src->bi_next; 960 break;
892 if (!src)
893 break;
894
895 src_bv = bio_iovec(src);
896 }
897 961
898 src_offset = src_bv->bv_offset; 962 src_iter = src->bi_iter;
899 } 963 }
900 964
901 if (dst_offset == dst_bv->bv_offset + dst_bv->bv_len) { 965 if (!dst_iter.bi_size) {
902 dst_bv++; 966 dst = dst->bi_next;
903 if (dst_bv == bio_iovec_idx(dst, dst->bi_vcnt)) { 967 if (!dst)
904 dst = dst->bi_next; 968 break;
905 if (!dst)
906 break;
907
908 dst_bv = bio_iovec(dst);
909 }
910 969
911 dst_offset = dst_bv->bv_offset; 970 dst_iter = dst->bi_iter;
912 } 971 }
913 972
914 bytes = min(dst_bv->bv_offset + dst_bv->bv_len - dst_offset, 973 src_bv = bio_iter_iovec(src, src_iter);
915 src_bv->bv_offset + src_bv->bv_len - src_offset); 974 dst_bv = bio_iter_iovec(dst, dst_iter);
975
976 bytes = min(src_bv.bv_len, dst_bv.bv_len);
916 977
917 src_p = kmap_atomic(src_bv->bv_page); 978 src_p = kmap_atomic(src_bv.bv_page);
918 dst_p = kmap_atomic(dst_bv->bv_page); 979 dst_p = kmap_atomic(dst_bv.bv_page);
919 980
920 memcpy(dst_p + dst_offset, 981 memcpy(dst_p + dst_bv.bv_offset,
921 src_p + src_offset, 982 src_p + src_bv.bv_offset,
922 bytes); 983 bytes);
923 984
924 kunmap_atomic(dst_p); 985 kunmap_atomic(dst_p);
925 kunmap_atomic(src_p); 986 kunmap_atomic(src_p);
926 987
927 src_offset += bytes; 988 bio_advance_iter(src, &src_iter, bytes);
928 dst_offset += bytes; 989 bio_advance_iter(dst, &dst_iter, bytes);
929 } 990 }
930} 991}
931EXPORT_SYMBOL(bio_copy_data); 992EXPORT_SYMBOL(bio_copy_data);
932 993
933struct bio_map_data { 994struct bio_map_data {
934 struct bio_vec *iovecs;
935 struct sg_iovec *sgvecs;
936 int nr_sgvecs; 995 int nr_sgvecs;
937 int is_our_pages; 996 int is_our_pages;
997 struct sg_iovec sgvecs[];
938}; 998};
939 999
940static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio, 1000static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio,
941 struct sg_iovec *iov, int iov_count, 1001 struct sg_iovec *iov, int iov_count,
942 int is_our_pages) 1002 int is_our_pages)
943{ 1003{
944 memcpy(bmd->iovecs, bio->bi_io_vec, sizeof(struct bio_vec) * bio->bi_vcnt);
945 memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count); 1004 memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count);
946 bmd->nr_sgvecs = iov_count; 1005 bmd->nr_sgvecs = iov_count;
947 bmd->is_our_pages = is_our_pages; 1006 bmd->is_our_pages = is_our_pages;
948 bio->bi_private = bmd; 1007 bio->bi_private = bmd;
949} 1008}
950 1009
951static void bio_free_map_data(struct bio_map_data *bmd)
952{
953 kfree(bmd->iovecs);
954 kfree(bmd->sgvecs);
955 kfree(bmd);
956}
957
958static struct bio_map_data *bio_alloc_map_data(int nr_segs, 1010static struct bio_map_data *bio_alloc_map_data(int nr_segs,
959 unsigned int iov_count, 1011 unsigned int iov_count,
960 gfp_t gfp_mask) 1012 gfp_t gfp_mask)
961{ 1013{
962 struct bio_map_data *bmd;
963
964 if (iov_count > UIO_MAXIOV) 1014 if (iov_count > UIO_MAXIOV)
965 return NULL; 1015 return NULL;
966 1016
967 bmd = kmalloc(sizeof(*bmd), gfp_mask); 1017 return kmalloc(sizeof(struct bio_map_data) +
968 if (!bmd) 1018 sizeof(struct sg_iovec) * iov_count, gfp_mask);
969 return NULL;
970
971 bmd->iovecs = kmalloc(sizeof(struct bio_vec) * nr_segs, gfp_mask);
972 if (!bmd->iovecs) {
973 kfree(bmd);
974 return NULL;
975 }
976
977 bmd->sgvecs = kmalloc(sizeof(struct sg_iovec) * iov_count, gfp_mask);
978 if (bmd->sgvecs)
979 return bmd;
980
981 kfree(bmd->iovecs);
982 kfree(bmd);
983 return NULL;
984} 1019}
985 1020
986static int __bio_copy_iov(struct bio *bio, struct bio_vec *iovecs, 1021static int __bio_copy_iov(struct bio *bio, struct sg_iovec *iov, int iov_count,
987 struct sg_iovec *iov, int iov_count,
988 int to_user, int from_user, int do_free_page) 1022 int to_user, int from_user, int do_free_page)
989{ 1023{
990 int ret = 0, i; 1024 int ret = 0, i;
@@ -994,7 +1028,7 @@ static int __bio_copy_iov(struct bio *bio, struct bio_vec *iovecs,
994 1028
995 bio_for_each_segment_all(bvec, bio, i) { 1029 bio_for_each_segment_all(bvec, bio, i) {
996 char *bv_addr = page_address(bvec->bv_page); 1030 char *bv_addr = page_address(bvec->bv_page);
997 unsigned int bv_len = iovecs[i].bv_len; 1031 unsigned int bv_len = bvec->bv_len;
998 1032
999 while (bv_len && iov_idx < iov_count) { 1033 while (bv_len && iov_idx < iov_count) {
1000 unsigned int bytes; 1034 unsigned int bytes;
@@ -1054,14 +1088,14 @@ int bio_uncopy_user(struct bio *bio)
1054 * don't copy into a random user address space, just free. 1088 * don't copy into a random user address space, just free.
1055 */ 1089 */
1056 if (current->mm) 1090 if (current->mm)
1057 ret = __bio_copy_iov(bio, bmd->iovecs, bmd->sgvecs, 1091 ret = __bio_copy_iov(bio, bmd->sgvecs, bmd->nr_sgvecs,
1058 bmd->nr_sgvecs, bio_data_dir(bio) == READ, 1092 bio_data_dir(bio) == READ,
1059 0, bmd->is_our_pages); 1093 0, bmd->is_our_pages);
1060 else if (bmd->is_our_pages) 1094 else if (bmd->is_our_pages)
1061 bio_for_each_segment_all(bvec, bio, i) 1095 bio_for_each_segment_all(bvec, bio, i)
1062 __free_page(bvec->bv_page); 1096 __free_page(bvec->bv_page);
1063 } 1097 }
1064 bio_free_map_data(bmd); 1098 kfree(bmd);
1065 bio_put(bio); 1099 bio_put(bio);
1066 return ret; 1100 return ret;
1067} 1101}
@@ -1175,7 +1209,7 @@ struct bio *bio_copy_user_iov(struct request_queue *q,
1175 */ 1209 */
1176 if ((!write_to_vm && (!map_data || !map_data->null_mapped)) || 1210 if ((!write_to_vm && (!map_data || !map_data->null_mapped)) ||
1177 (map_data && map_data->from_user)) { 1211 (map_data && map_data->from_user)) {
1178 ret = __bio_copy_iov(bio, bio->bi_io_vec, iov, iov_count, 0, 1, 0); 1212 ret = __bio_copy_iov(bio, iov, iov_count, 0, 1, 0);
1179 if (ret) 1213 if (ret)
1180 goto cleanup; 1214 goto cleanup;
1181 } 1215 }
@@ -1189,7 +1223,7 @@ cleanup:
1189 1223
1190 bio_put(bio); 1224 bio_put(bio);
1191out_bmd: 1225out_bmd:
1192 bio_free_map_data(bmd); 1226 kfree(bmd);
1193 return ERR_PTR(ret); 1227 return ERR_PTR(ret);
1194} 1228}
1195 1229
@@ -1485,7 +1519,7 @@ struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
1485 if (IS_ERR(bio)) 1519 if (IS_ERR(bio))
1486 return bio; 1520 return bio;
1487 1521
1488 if (bio->bi_size == len) 1522 if (bio->bi_iter.bi_size == len)
1489 return bio; 1523 return bio;
1490 1524
1491 /* 1525 /*
@@ -1506,16 +1540,15 @@ static void bio_copy_kern_endio(struct bio *bio, int err)
1506 1540
1507 bio_for_each_segment_all(bvec, bio, i) { 1541 bio_for_each_segment_all(bvec, bio, i) {
1508 char *addr = page_address(bvec->bv_page); 1542 char *addr = page_address(bvec->bv_page);
1509 int len = bmd->iovecs[i].bv_len;
1510 1543
1511 if (read) 1544 if (read)
1512 memcpy(p, addr, len); 1545 memcpy(p, addr, bvec->bv_len);
1513 1546
1514 __free_page(bvec->bv_page); 1547 __free_page(bvec->bv_page);
1515 p += len; 1548 p += bvec->bv_len;
1516 } 1549 }
1517 1550
1518 bio_free_map_data(bmd); 1551 kfree(bmd);
1519 bio_put(bio); 1552 bio_put(bio);
1520} 1553}
1521 1554
@@ -1686,11 +1719,11 @@ void bio_check_pages_dirty(struct bio *bio)
1686#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1719#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
1687void bio_flush_dcache_pages(struct bio *bi) 1720void bio_flush_dcache_pages(struct bio *bi)
1688{ 1721{
1689 int i; 1722 struct bio_vec bvec;
1690 struct bio_vec *bvec; 1723 struct bvec_iter iter;
1691 1724
1692 bio_for_each_segment(bvec, bi, i) 1725 bio_for_each_segment(bvec, bi, iter)
1693 flush_dcache_page(bvec->bv_page); 1726 flush_dcache_page(bvec.bv_page);
1694} 1727}
1695EXPORT_SYMBOL(bio_flush_dcache_pages); 1728EXPORT_SYMBOL(bio_flush_dcache_pages);
1696#endif 1729#endif
@@ -1711,96 +1744,86 @@ EXPORT_SYMBOL(bio_flush_dcache_pages);
1711 **/ 1744 **/
1712void bio_endio(struct bio *bio, int error) 1745void bio_endio(struct bio *bio, int error)
1713{ 1746{
1714 if (error) 1747 while (bio) {
1715 clear_bit(BIO_UPTODATE, &bio->bi_flags); 1748 BUG_ON(atomic_read(&bio->bi_remaining) <= 0);
1716 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1717 error = -EIO;
1718 1749
1719 if (bio->bi_end_io) 1750 if (error)
1720 bio->bi_end_io(bio, error); 1751 clear_bit(BIO_UPTODATE, &bio->bi_flags);
1721} 1752 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1722EXPORT_SYMBOL(bio_endio); 1753 error = -EIO;
1723 1754
1724void bio_pair_release(struct bio_pair *bp) 1755 if (!atomic_dec_and_test(&bio->bi_remaining))
1725{ 1756 return;
1726 if (atomic_dec_and_test(&bp->cnt)) {
1727 struct bio *master = bp->bio1.bi_private;
1728 1757
1729 bio_endio(master, bp->error); 1758 /*
1730 mempool_free(bp, bp->bio2.bi_private); 1759 * Need to have a real endio function for chained bios,
1760 * otherwise various corner cases will break (like stacking
1761 * block devices that save/restore bi_end_io) - however, we want
1762 * to avoid unbounded recursion and blowing the stack. Tail call
1763 * optimization would handle this, but compiling with frame
1764 * pointers also disables gcc's sibling call optimization.
1765 */
1766 if (bio->bi_end_io == bio_chain_endio) {
1767 struct bio *parent = bio->bi_private;
1768 bio_put(bio);
1769 bio = parent;
1770 } else {
1771 if (bio->bi_end_io)
1772 bio->bi_end_io(bio, error);
1773 bio = NULL;
1774 }
1731 } 1775 }
1732} 1776}
1733EXPORT_SYMBOL(bio_pair_release); 1777EXPORT_SYMBOL(bio_endio);
1734 1778
1735static void bio_pair_end_1(struct bio *bi, int err) 1779/**
1780 * bio_endio_nodec - end I/O on a bio, without decrementing bi_remaining
1781 * @bio: bio
1782 * @error: error, if any
1783 *
1784 * For code that has saved and restored bi_end_io; thing hard before using this
1785 * function, probably you should've cloned the entire bio.
1786 **/
1787void bio_endio_nodec(struct bio *bio, int error)
1736{ 1788{
1737 struct bio_pair *bp = container_of(bi, struct bio_pair, bio1); 1789 atomic_inc(&bio->bi_remaining);
1738 1790 bio_endio(bio, error);
1739 if (err)
1740 bp->error = err;
1741
1742 bio_pair_release(bp);
1743} 1791}
1792EXPORT_SYMBOL(bio_endio_nodec);
1744 1793
1745static void bio_pair_end_2(struct bio *bi, int err) 1794/**
1746{ 1795 * bio_split - split a bio
1747 struct bio_pair *bp = container_of(bi, struct bio_pair, bio2); 1796 * @bio: bio to split
1748 1797 * @sectors: number of sectors to split from the front of @bio
1749 if (err) 1798 * @gfp: gfp mask
1750 bp->error = err; 1799 * @bs: bio set to allocate from
1751 1800 *
1752 bio_pair_release(bp); 1801 * Allocates and returns a new bio which represents @sectors from the start of
1753} 1802 * @bio, and updates @bio to represent the remaining sectors.
1754 1803 *
1755/* 1804 * The newly allocated bio will point to @bio's bi_io_vec; it is the caller's
1756 * split a bio - only worry about a bio with a single page in its iovec 1805 * responsibility to ensure that @bio is not freed before the split.
1757 */ 1806 */
1758struct bio_pair *bio_split(struct bio *bi, int first_sectors) 1807struct bio *bio_split(struct bio *bio, int sectors,
1808 gfp_t gfp, struct bio_set *bs)
1759{ 1809{
1760 struct bio_pair *bp = mempool_alloc(bio_split_pool, GFP_NOIO); 1810 struct bio *split = NULL;
1761
1762 if (!bp)
1763 return bp;
1764
1765 trace_block_split(bdev_get_queue(bi->bi_bdev), bi,
1766 bi->bi_sector + first_sectors);
1767
1768 BUG_ON(bio_segments(bi) > 1);
1769 atomic_set(&bp->cnt, 3);
1770 bp->error = 0;
1771 bp->bio1 = *bi;
1772 bp->bio2 = *bi;
1773 bp->bio2.bi_sector += first_sectors;
1774 bp->bio2.bi_size -= first_sectors << 9;
1775 bp->bio1.bi_size = first_sectors << 9;
1776
1777 if (bi->bi_vcnt != 0) {
1778 bp->bv1 = *bio_iovec(bi);
1779 bp->bv2 = *bio_iovec(bi);
1780
1781 if (bio_is_rw(bi)) {
1782 bp->bv2.bv_offset += first_sectors << 9;
1783 bp->bv2.bv_len -= first_sectors << 9;
1784 bp->bv1.bv_len = first_sectors << 9;
1785 }
1786 1811
1787 bp->bio1.bi_io_vec = &bp->bv1; 1812 BUG_ON(sectors <= 0);
1788 bp->bio2.bi_io_vec = &bp->bv2; 1813 BUG_ON(sectors >= bio_sectors(bio));
1789 1814
1790 bp->bio1.bi_max_vecs = 1; 1815 split = bio_clone_fast(bio, gfp, bs);
1791 bp->bio2.bi_max_vecs = 1; 1816 if (!split)
1792 } 1817 return NULL;
1793 1818
1794 bp->bio1.bi_end_io = bio_pair_end_1; 1819 split->bi_iter.bi_size = sectors << 9;
1795 bp->bio2.bi_end_io = bio_pair_end_2;
1796 1820
1797 bp->bio1.bi_private = bi; 1821 if (bio_integrity(split))
1798 bp->bio2.bi_private = bio_split_pool; 1822 bio_integrity_trim(split, 0, sectors);
1799 1823
1800 if (bio_integrity(bi)) 1824 bio_advance(bio, split->bi_iter.bi_size);
1801 bio_integrity_split(bi, bp, first_sectors);
1802 1825
1803 return bp; 1826 return split;
1804} 1827}
1805EXPORT_SYMBOL(bio_split); 1828EXPORT_SYMBOL(bio_split);
1806 1829
@@ -1814,80 +1837,20 @@ void bio_trim(struct bio *bio, int offset, int size)
1814{ 1837{
1815 /* 'bio' is a cloned bio which we need to trim to match 1838 /* 'bio' is a cloned bio which we need to trim to match
1816 * the given offset and size. 1839 * the given offset and size.
1817 * This requires adjusting bi_sector, bi_size, and bi_io_vec
1818 */ 1840 */
1819 int i;
1820 struct bio_vec *bvec;
1821 int sofar = 0;
1822 1841
1823 size <<= 9; 1842 size <<= 9;
1824 if (offset == 0 && size == bio->bi_size) 1843 if (offset == 0 && size == bio->bi_iter.bi_size)
1825 return; 1844 return;
1826 1845
1827 clear_bit(BIO_SEG_VALID, &bio->bi_flags); 1846 clear_bit(BIO_SEG_VALID, &bio->bi_flags);
1828 1847
1829 bio_advance(bio, offset << 9); 1848 bio_advance(bio, offset << 9);
1830 1849
1831 bio->bi_size = size; 1850 bio->bi_iter.bi_size = size;
1832
1833 /* avoid any complications with bi_idx being non-zero*/
1834 if (bio->bi_idx) {
1835 memmove(bio->bi_io_vec, bio->bi_io_vec+bio->bi_idx,
1836 (bio->bi_vcnt - bio->bi_idx) * sizeof(struct bio_vec));
1837 bio->bi_vcnt -= bio->bi_idx;
1838 bio->bi_idx = 0;
1839 }
1840 /* Make sure vcnt and last bv are not too big */
1841 bio_for_each_segment(bvec, bio, i) {
1842 if (sofar + bvec->bv_len > size)
1843 bvec->bv_len = size - sofar;
1844 if (bvec->bv_len == 0) {
1845 bio->bi_vcnt = i;
1846 break;
1847 }
1848 sofar += bvec->bv_len;
1849 }
1850} 1851}
1851EXPORT_SYMBOL_GPL(bio_trim); 1852EXPORT_SYMBOL_GPL(bio_trim);
1852 1853
1853/**
1854 * bio_sector_offset - Find hardware sector offset in bio
1855 * @bio: bio to inspect
1856 * @index: bio_vec index
1857 * @offset: offset in bv_page
1858 *
1859 * Return the number of hardware sectors between beginning of bio
1860 * and an end point indicated by a bio_vec index and an offset
1861 * within that vector's page.
1862 */
1863sector_t bio_sector_offset(struct bio *bio, unsigned short index,
1864 unsigned int offset)
1865{
1866 unsigned int sector_sz;
1867 struct bio_vec *bv;
1868 sector_t sectors;
1869 int i;
1870
1871 sector_sz = queue_logical_block_size(bio->bi_bdev->bd_disk->queue);
1872 sectors = 0;
1873
1874 if (index >= bio->bi_idx)
1875 index = bio->bi_vcnt - 1;
1876
1877 bio_for_each_segment_all(bv, bio, i) {
1878 if (i == index) {
1879 if (offset > bv->bv_offset)
1880 sectors += (offset - bv->bv_offset) / sector_sz;
1881 break;
1882 }
1883
1884 sectors += bv->bv_len / sector_sz;
1885 }
1886
1887 return sectors;
1888}
1889EXPORT_SYMBOL(bio_sector_offset);
1890
1891/* 1854/*
1892 * create memory pools for biovec's in a bio_set. 1855 * create memory pools for biovec's in a bio_set.
1893 * use the global biovec slabs created for general use. 1856 * use the global biovec slabs created for general use.
@@ -2065,11 +2028,6 @@ static int __init init_bio(void)
2065 if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE)) 2028 if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE))
2066 panic("bio: can't create integrity pool\n"); 2029 panic("bio: can't create integrity pool\n");
2067 2030
2068 bio_split_pool = mempool_create_kmalloc_pool(BIO_SPLIT_ENTRIES,
2069 sizeof(struct bio_pair));
2070 if (!bio_split_pool)
2071 panic("bio: can't create split pool\n");
2072
2073 return 0; 2031 return 0;
2074} 2032}
2075subsys_initcall(init_bio); 2033subsys_initcall(init_bio);
diff --git a/fs/btrfs/check-integrity.c b/fs/btrfs/check-integrity.c
index 131d82800b3a..cb05e1c842c5 100644
--- a/fs/btrfs/check-integrity.c
+++ b/fs/btrfs/check-integrity.c
@@ -1695,7 +1695,7 @@ static int btrfsic_read_block(struct btrfsic_state *state,
1695 return -1; 1695 return -1;
1696 } 1696 }
1697 bio->bi_bdev = block_ctx->dev->bdev; 1697 bio->bi_bdev = block_ctx->dev->bdev;
1698 bio->bi_sector = dev_bytenr >> 9; 1698 bio->bi_iter.bi_sector = dev_bytenr >> 9;
1699 1699
1700 for (j = i; j < num_pages; j++) { 1700 for (j = i; j < num_pages; j++) {
1701 ret = bio_add_page(bio, block_ctx->pagev[j], 1701 ret = bio_add_page(bio, block_ctx->pagev[j],
@@ -3013,7 +3013,7 @@ static void __btrfsic_submit_bio(int rw, struct bio *bio)
3013 int bio_is_patched; 3013 int bio_is_patched;
3014 char **mapped_datav; 3014 char **mapped_datav;
3015 3015
3016 dev_bytenr = 512 * bio->bi_sector; 3016 dev_bytenr = 512 * bio->bi_iter.bi_sector;
3017 bio_is_patched = 0; 3017 bio_is_patched = 0;
3018 if (dev_state->state->print_mask & 3018 if (dev_state->state->print_mask &
3019 BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH) 3019 BTRFSIC_PRINT_MASK_SUBMIT_BIO_BH)
@@ -3021,8 +3021,8 @@ static void __btrfsic_submit_bio(int rw, struct bio *bio)
3021 "submit_bio(rw=0x%x, bi_vcnt=%u," 3021 "submit_bio(rw=0x%x, bi_vcnt=%u,"
3022 " bi_sector=%llu (bytenr %llu), bi_bdev=%p)\n", 3022 " bi_sector=%llu (bytenr %llu), bi_bdev=%p)\n",
3023 rw, bio->bi_vcnt, 3023 rw, bio->bi_vcnt,
3024 (unsigned long long)bio->bi_sector, dev_bytenr, 3024 (unsigned long long)bio->bi_iter.bi_sector,
3025 bio->bi_bdev); 3025 dev_bytenr, bio->bi_bdev);
3026 3026
3027 mapped_datav = kmalloc(sizeof(*mapped_datav) * bio->bi_vcnt, 3027 mapped_datav = kmalloc(sizeof(*mapped_datav) * bio->bi_vcnt,
3028 GFP_NOFS); 3028 GFP_NOFS);
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
index 1499b27b4186..f5cdeb4b5538 100644
--- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c
@@ -172,7 +172,8 @@ static void end_compressed_bio_read(struct bio *bio, int err)
172 goto out; 172 goto out;
173 173
174 inode = cb->inode; 174 inode = cb->inode;
175 ret = check_compressed_csum(inode, cb, (u64)bio->bi_sector << 9); 175 ret = check_compressed_csum(inode, cb,
176 (u64)bio->bi_iter.bi_sector << 9);
176 if (ret) 177 if (ret)
177 goto csum_failed; 178 goto csum_failed;
178 179
@@ -201,18 +202,16 @@ csum_failed:
201 if (cb->errors) { 202 if (cb->errors) {
202 bio_io_error(cb->orig_bio); 203 bio_io_error(cb->orig_bio);
203 } else { 204 } else {
204 int bio_index = 0; 205 int i;
205 struct bio_vec *bvec = cb->orig_bio->bi_io_vec; 206 struct bio_vec *bvec;
206 207
207 /* 208 /*
208 * we have verified the checksum already, set page 209 * we have verified the checksum already, set page
209 * checked so the end_io handlers know about it 210 * checked so the end_io handlers know about it
210 */ 211 */
211 while (bio_index < cb->orig_bio->bi_vcnt) { 212 bio_for_each_segment_all(bvec, cb->orig_bio, i)
212 SetPageChecked(bvec->bv_page); 213 SetPageChecked(bvec->bv_page);
213 bvec++; 214
214 bio_index++;
215 }
216 bio_endio(cb->orig_bio, 0); 215 bio_endio(cb->orig_bio, 0);
217 } 216 }
218 217
@@ -372,7 +371,7 @@ int btrfs_submit_compressed_write(struct inode *inode, u64 start,
372 for (pg_index = 0; pg_index < cb->nr_pages; pg_index++) { 371 for (pg_index = 0; pg_index < cb->nr_pages; pg_index++) {
373 page = compressed_pages[pg_index]; 372 page = compressed_pages[pg_index];
374 page->mapping = inode->i_mapping; 373 page->mapping = inode->i_mapping;
375 if (bio->bi_size) 374 if (bio->bi_iter.bi_size)
376 ret = io_tree->ops->merge_bio_hook(WRITE, page, 0, 375 ret = io_tree->ops->merge_bio_hook(WRITE, page, 0,
377 PAGE_CACHE_SIZE, 376 PAGE_CACHE_SIZE,
378 bio, 0); 377 bio, 0);
@@ -506,7 +505,7 @@ static noinline int add_ra_bio_pages(struct inode *inode,
506 505
507 if (!em || last_offset < em->start || 506 if (!em || last_offset < em->start ||
508 (last_offset + PAGE_CACHE_SIZE > extent_map_end(em)) || 507 (last_offset + PAGE_CACHE_SIZE > extent_map_end(em)) ||
509 (em->block_start >> 9) != cb->orig_bio->bi_sector) { 508 (em->block_start >> 9) != cb->orig_bio->bi_iter.bi_sector) {
510 free_extent_map(em); 509 free_extent_map(em);
511 unlock_extent(tree, last_offset, end); 510 unlock_extent(tree, last_offset, end);
512 unlock_page(page); 511 unlock_page(page);
@@ -552,7 +551,7 @@ next:
552 * in it. We don't actually do IO on those pages but allocate new ones 551 * in it. We don't actually do IO on those pages but allocate new ones
553 * to hold the compressed pages on disk. 552 * to hold the compressed pages on disk.
554 * 553 *
555 * bio->bi_sector points to the compressed extent on disk 554 * bio->bi_iter.bi_sector points to the compressed extent on disk
556 * bio->bi_io_vec points to all of the inode pages 555 * bio->bi_io_vec points to all of the inode pages
557 * bio->bi_vcnt is a count of pages 556 * bio->bi_vcnt is a count of pages
558 * 557 *
@@ -573,7 +572,7 @@ int btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
573 struct page *page; 572 struct page *page;
574 struct block_device *bdev; 573 struct block_device *bdev;
575 struct bio *comp_bio; 574 struct bio *comp_bio;
576 u64 cur_disk_byte = (u64)bio->bi_sector << 9; 575 u64 cur_disk_byte = (u64)bio->bi_iter.bi_sector << 9;
577 u64 em_len; 576 u64 em_len;
578 u64 em_start; 577 u64 em_start;
579 struct extent_map *em; 578 struct extent_map *em;
@@ -659,7 +658,7 @@ int btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
659 page->mapping = inode->i_mapping; 658 page->mapping = inode->i_mapping;
660 page->index = em_start >> PAGE_CACHE_SHIFT; 659 page->index = em_start >> PAGE_CACHE_SHIFT;
661 660
662 if (comp_bio->bi_size) 661 if (comp_bio->bi_iter.bi_size)
663 ret = tree->ops->merge_bio_hook(READ, page, 0, 662 ret = tree->ops->merge_bio_hook(READ, page, 0,
664 PAGE_CACHE_SIZE, 663 PAGE_CACHE_SIZE,
665 comp_bio, 0); 664 comp_bio, 0);
@@ -687,8 +686,8 @@ int btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
687 comp_bio, sums); 686 comp_bio, sums);
688 BUG_ON(ret); /* -ENOMEM */ 687 BUG_ON(ret); /* -ENOMEM */
689 } 688 }
690 sums += (comp_bio->bi_size + root->sectorsize - 1) / 689 sums += (comp_bio->bi_iter.bi_size +
691 root->sectorsize; 690 root->sectorsize - 1) / root->sectorsize;
692 691
693 ret = btrfs_map_bio(root, READ, comp_bio, 692 ret = btrfs_map_bio(root, READ, comp_bio,
694 mirror_num, 0); 693 mirror_num, 0);
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 8072cfa8a3b1..e71039ea66cf 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -842,20 +842,17 @@ int btrfs_wq_submit_bio(struct btrfs_fs_info *fs_info, struct inode *inode,
842 842
843static int btree_csum_one_bio(struct bio *bio) 843static int btree_csum_one_bio(struct bio *bio)
844{ 844{
845 struct bio_vec *bvec = bio->bi_io_vec; 845 struct bio_vec *bvec;
846 int bio_index = 0;
847 struct btrfs_root *root; 846 struct btrfs_root *root;
848 int ret = 0; 847 int i, ret = 0;
849 848
850 WARN_ON(bio->bi_vcnt <= 0); 849 bio_for_each_segment_all(bvec, bio, i) {
851 while (bio_index < bio->bi_vcnt) {
852 root = BTRFS_I(bvec->bv_page->mapping->host)->root; 850 root = BTRFS_I(bvec->bv_page->mapping->host)->root;
853 ret = csum_dirty_buffer(root, bvec->bv_page); 851 ret = csum_dirty_buffer(root, bvec->bv_page);
854 if (ret) 852 if (ret)
855 break; 853 break;
856 bio_index++;
857 bvec++;
858 } 854 }
855
859 return ret; 856 return ret;
860} 857}
861 858
@@ -1695,7 +1692,7 @@ static void end_workqueue_fn(struct btrfs_work *work)
1695 bio->bi_private = end_io_wq->private; 1692 bio->bi_private = end_io_wq->private;
1696 bio->bi_end_io = end_io_wq->end_io; 1693 bio->bi_end_io = end_io_wq->end_io;
1697 kfree(end_io_wq); 1694 kfree(end_io_wq);
1698 bio_endio(bio, error); 1695 bio_endio_nodec(bio, error);
1699} 1696}
1700 1697
1701static int cleaner_kthread(void *arg) 1698static int cleaner_kthread(void *arg)
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index ff43802a7c88..bcb6f1b780d6 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -1984,7 +1984,7 @@ int repair_io_failure(struct btrfs_fs_info *fs_info, u64 start,
1984 bio = btrfs_io_bio_alloc(GFP_NOFS, 1); 1984 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1985 if (!bio) 1985 if (!bio)
1986 return -EIO; 1986 return -EIO;
1987 bio->bi_size = 0; 1987 bio->bi_iter.bi_size = 0;
1988 map_length = length; 1988 map_length = length;
1989 1989
1990 ret = btrfs_map_block(fs_info, WRITE, logical, 1990 ret = btrfs_map_block(fs_info, WRITE, logical,
@@ -1995,7 +1995,7 @@ int repair_io_failure(struct btrfs_fs_info *fs_info, u64 start,
1995 } 1995 }
1996 BUG_ON(mirror_num != bbio->mirror_num); 1996 BUG_ON(mirror_num != bbio->mirror_num);
1997 sector = bbio->stripes[mirror_num-1].physical >> 9; 1997 sector = bbio->stripes[mirror_num-1].physical >> 9;
1998 bio->bi_sector = sector; 1998 bio->bi_iter.bi_sector = sector;
1999 dev = bbio->stripes[mirror_num-1].dev; 1999 dev = bbio->stripes[mirror_num-1].dev;
2000 kfree(bbio); 2000 kfree(bbio);
2001 if (!dev || !dev->bdev || !dev->writeable) { 2001 if (!dev || !dev->bdev || !dev->writeable) {
@@ -2268,9 +2268,9 @@ static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2268 return -EIO; 2268 return -EIO;
2269 } 2269 }
2270 bio->bi_end_io = failed_bio->bi_end_io; 2270 bio->bi_end_io = failed_bio->bi_end_io;
2271 bio->bi_sector = failrec->logical >> 9; 2271 bio->bi_iter.bi_sector = failrec->logical >> 9;
2272 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev; 2272 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
2273 bio->bi_size = 0; 2273 bio->bi_iter.bi_size = 0;
2274 2274
2275 btrfs_failed_bio = btrfs_io_bio(failed_bio); 2275 btrfs_failed_bio = btrfs_io_bio(failed_bio);
2276 if (btrfs_failed_bio->csum) { 2276 if (btrfs_failed_bio->csum) {
@@ -2332,12 +2332,13 @@ int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2332 */ 2332 */
2333static void end_bio_extent_writepage(struct bio *bio, int err) 2333static void end_bio_extent_writepage(struct bio *bio, int err)
2334{ 2334{
2335 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 2335 struct bio_vec *bvec;
2336 struct extent_io_tree *tree; 2336 struct extent_io_tree *tree;
2337 u64 start; 2337 u64 start;
2338 u64 end; 2338 u64 end;
2339 int i;
2339 2340
2340 do { 2341 bio_for_each_segment_all(bvec, bio, i) {
2341 struct page *page = bvec->bv_page; 2342 struct page *page = bvec->bv_page;
2342 tree = &BTRFS_I(page->mapping->host)->io_tree; 2343 tree = &BTRFS_I(page->mapping->host)->io_tree;
2343 2344
@@ -2355,14 +2356,11 @@ static void end_bio_extent_writepage(struct bio *bio, int err)
2355 start = page_offset(page); 2356 start = page_offset(page);
2356 end = start + bvec->bv_offset + bvec->bv_len - 1; 2357 end = start + bvec->bv_offset + bvec->bv_len - 1;
2357 2358
2358 if (--bvec >= bio->bi_io_vec)
2359 prefetchw(&bvec->bv_page->flags);
2360
2361 if (end_extent_writepage(page, err, start, end)) 2359 if (end_extent_writepage(page, err, start, end))
2362 continue; 2360 continue;
2363 2361
2364 end_page_writeback(page); 2362 end_page_writeback(page);
2365 } while (bvec >= bio->bi_io_vec); 2363 }
2366 2364
2367 bio_put(bio); 2365 bio_put(bio);
2368} 2366}
@@ -2392,9 +2390,8 @@ endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2392 */ 2390 */
2393static void end_bio_extent_readpage(struct bio *bio, int err) 2391static void end_bio_extent_readpage(struct bio *bio, int err)
2394{ 2392{
2393 struct bio_vec *bvec;
2395 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 2394 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
2396 struct bio_vec *bvec_end = bio->bi_io_vec + bio->bi_vcnt - 1;
2397 struct bio_vec *bvec = bio->bi_io_vec;
2398 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 2395 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
2399 struct extent_io_tree *tree; 2396 struct extent_io_tree *tree;
2400 u64 offset = 0; 2397 u64 offset = 0;
@@ -2405,16 +2402,17 @@ static void end_bio_extent_readpage(struct bio *bio, int err)
2405 u64 extent_len = 0; 2402 u64 extent_len = 0;
2406 int mirror; 2403 int mirror;
2407 int ret; 2404 int ret;
2405 int i;
2408 2406
2409 if (err) 2407 if (err)
2410 uptodate = 0; 2408 uptodate = 0;
2411 2409
2412 do { 2410 bio_for_each_segment_all(bvec, bio, i) {
2413 struct page *page = bvec->bv_page; 2411 struct page *page = bvec->bv_page;
2414 struct inode *inode = page->mapping->host; 2412 struct inode *inode = page->mapping->host;
2415 2413
2416 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, " 2414 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
2417 "mirror=%lu\n", (u64)bio->bi_sector, err, 2415 "mirror=%lu\n", (u64)bio->bi_iter.bi_sector, err,
2418 io_bio->mirror_num); 2416 io_bio->mirror_num);
2419 tree = &BTRFS_I(inode)->io_tree; 2417 tree = &BTRFS_I(inode)->io_tree;
2420 2418
@@ -2433,9 +2431,6 @@ static void end_bio_extent_readpage(struct bio *bio, int err)
2433 end = start + bvec->bv_offset + bvec->bv_len - 1; 2431 end = start + bvec->bv_offset + bvec->bv_len - 1;
2434 len = bvec->bv_len; 2432 len = bvec->bv_len;
2435 2433
2436 if (++bvec <= bvec_end)
2437 prefetchw(&bvec->bv_page->flags);
2438
2439 mirror = io_bio->mirror_num; 2434 mirror = io_bio->mirror_num;
2440 if (likely(uptodate && tree->ops && 2435 if (likely(uptodate && tree->ops &&
2441 tree->ops->readpage_end_io_hook)) { 2436 tree->ops->readpage_end_io_hook)) {
@@ -2516,7 +2511,7 @@ readpage_ok:
2516 extent_start = start; 2511 extent_start = start;
2517 extent_len = end + 1 - start; 2512 extent_len = end + 1 - start;
2518 } 2513 }
2519 } while (bvec <= bvec_end); 2514 }
2520 2515
2521 if (extent_len) 2516 if (extent_len)
2522 endio_readpage_release_extent(tree, extent_start, extent_len, 2517 endio_readpage_release_extent(tree, extent_start, extent_len,
@@ -2547,9 +2542,8 @@ btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2547 } 2542 }
2548 2543
2549 if (bio) { 2544 if (bio) {
2550 bio->bi_size = 0;
2551 bio->bi_bdev = bdev; 2545 bio->bi_bdev = bdev;
2552 bio->bi_sector = first_sector; 2546 bio->bi_iter.bi_sector = first_sector;
2553 btrfs_bio = btrfs_io_bio(bio); 2547 btrfs_bio = btrfs_io_bio(bio);
2554 btrfs_bio->csum = NULL; 2548 btrfs_bio->csum = NULL;
2555 btrfs_bio->csum_allocated = NULL; 2549 btrfs_bio->csum_allocated = NULL;
@@ -2643,7 +2637,7 @@ static int submit_extent_page(int rw, struct extent_io_tree *tree,
2643 if (bio_ret && *bio_ret) { 2637 if (bio_ret && *bio_ret) {
2644 bio = *bio_ret; 2638 bio = *bio_ret;
2645 if (old_compressed) 2639 if (old_compressed)
2646 contig = bio->bi_sector == sector; 2640 contig = bio->bi_iter.bi_sector == sector;
2647 else 2641 else
2648 contig = bio_end_sector(bio) == sector; 2642 contig = bio_end_sector(bio) == sector;
2649 2643
@@ -3410,20 +3404,18 @@ static void end_extent_buffer_writeback(struct extent_buffer *eb)
3410 3404
3411static void end_bio_extent_buffer_writepage(struct bio *bio, int err) 3405static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3412{ 3406{
3413 int uptodate = err == 0; 3407 struct bio_vec *bvec;
3414 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
3415 struct extent_buffer *eb; 3408 struct extent_buffer *eb;
3416 int done; 3409 int i, done;
3417 3410
3418 do { 3411 bio_for_each_segment_all(bvec, bio, i) {
3419 struct page *page = bvec->bv_page; 3412 struct page *page = bvec->bv_page;
3420 3413
3421 bvec--;
3422 eb = (struct extent_buffer *)page->private; 3414 eb = (struct extent_buffer *)page->private;
3423 BUG_ON(!eb); 3415 BUG_ON(!eb);
3424 done = atomic_dec_and_test(&eb->io_pages); 3416 done = atomic_dec_and_test(&eb->io_pages);
3425 3417
3426 if (!uptodate || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) { 3418 if (err || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) {
3427 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags); 3419 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3428 ClearPageUptodate(page); 3420 ClearPageUptodate(page);
3429 SetPageError(page); 3421 SetPageError(page);
@@ -3435,10 +3427,9 @@ static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3435 continue; 3427 continue;
3436 3428
3437 end_extent_buffer_writeback(eb); 3429 end_extent_buffer_writeback(eb);
3438 } while (bvec >= bio->bi_io_vec); 3430 }
3439 3431
3440 bio_put(bio); 3432 bio_put(bio);
3441
3442} 3433}
3443 3434
3444static int write_one_eb(struct extent_buffer *eb, 3435static int write_one_eb(struct extent_buffer *eb,
diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c
index 6f3848860283..84a46a42d262 100644
--- a/fs/btrfs/file-item.c
+++ b/fs/btrfs/file-item.c
@@ -182,7 +182,7 @@ static int __btrfs_lookup_bio_sums(struct btrfs_root *root,
182 if (!path) 182 if (!path)
183 return -ENOMEM; 183 return -ENOMEM;
184 184
185 nblocks = bio->bi_size >> inode->i_sb->s_blocksize_bits; 185 nblocks = bio->bi_iter.bi_size >> inode->i_sb->s_blocksize_bits;
186 if (!dst) { 186 if (!dst) {
187 if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) { 187 if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) {
188 btrfs_bio->csum_allocated = kmalloc(nblocks * csum_size, 188 btrfs_bio->csum_allocated = kmalloc(nblocks * csum_size,
@@ -201,7 +201,7 @@ static int __btrfs_lookup_bio_sums(struct btrfs_root *root,
201 csum = (u8 *)dst; 201 csum = (u8 *)dst;
202 } 202 }
203 203
204 if (bio->bi_size > PAGE_CACHE_SIZE * 8) 204 if (bio->bi_iter.bi_size > PAGE_CACHE_SIZE * 8)
205 path->reada = 2; 205 path->reada = 2;
206 206
207 WARN_ON(bio->bi_vcnt <= 0); 207 WARN_ON(bio->bi_vcnt <= 0);
@@ -217,7 +217,7 @@ static int __btrfs_lookup_bio_sums(struct btrfs_root *root,
217 path->skip_locking = 1; 217 path->skip_locking = 1;
218 } 218 }
219 219
220 disk_bytenr = (u64)bio->bi_sector << 9; 220 disk_bytenr = (u64)bio->bi_iter.bi_sector << 9;
221 if (dio) 221 if (dio)
222 offset = logical_offset; 222 offset = logical_offset;
223 while (bio_index < bio->bi_vcnt) { 223 while (bio_index < bio->bi_vcnt) {
@@ -302,7 +302,7 @@ int btrfs_lookup_bio_sums_dio(struct btrfs_root *root, struct inode *inode,
302 struct btrfs_dio_private *dip, struct bio *bio, 302 struct btrfs_dio_private *dip, struct bio *bio,
303 u64 offset) 303 u64 offset)
304{ 304{
305 int len = (bio->bi_sector << 9) - dip->disk_bytenr; 305 int len = (bio->bi_iter.bi_sector << 9) - dip->disk_bytenr;
306 u16 csum_size = btrfs_super_csum_size(root->fs_info->super_copy); 306 u16 csum_size = btrfs_super_csum_size(root->fs_info->super_copy);
307 int ret; 307 int ret;
308 308
@@ -447,11 +447,12 @@ int btrfs_csum_one_bio(struct btrfs_root *root, struct inode *inode,
447 u64 offset; 447 u64 offset;
448 448
449 WARN_ON(bio->bi_vcnt <= 0); 449 WARN_ON(bio->bi_vcnt <= 0);
450 sums = kzalloc(btrfs_ordered_sum_size(root, bio->bi_size), GFP_NOFS); 450 sums = kzalloc(btrfs_ordered_sum_size(root, bio->bi_iter.bi_size),
451 GFP_NOFS);
451 if (!sums) 452 if (!sums)
452 return -ENOMEM; 453 return -ENOMEM;
453 454
454 sums->len = bio->bi_size; 455 sums->len = bio->bi_iter.bi_size;
455 INIT_LIST_HEAD(&sums->list); 456 INIT_LIST_HEAD(&sums->list);
456 457
457 if (contig) 458 if (contig)
@@ -461,7 +462,7 @@ int btrfs_csum_one_bio(struct btrfs_root *root, struct inode *inode,
461 462
462 ordered = btrfs_lookup_ordered_extent(inode, offset); 463 ordered = btrfs_lookup_ordered_extent(inode, offset);
463 BUG_ON(!ordered); /* Logic error */ 464 BUG_ON(!ordered); /* Logic error */
464 sums->bytenr = (u64)bio->bi_sector << 9; 465 sums->bytenr = (u64)bio->bi_iter.bi_sector << 9;
465 index = 0; 466 index = 0;
466 467
467 while (bio_index < bio->bi_vcnt) { 468 while (bio_index < bio->bi_vcnt) {
@@ -476,7 +477,7 @@ int btrfs_csum_one_bio(struct btrfs_root *root, struct inode *inode,
476 btrfs_add_ordered_sum(inode, ordered, sums); 477 btrfs_add_ordered_sum(inode, ordered, sums);
477 btrfs_put_ordered_extent(ordered); 478 btrfs_put_ordered_extent(ordered);
478 479
479 bytes_left = bio->bi_size - total_bytes; 480 bytes_left = bio->bi_iter.bi_size - total_bytes;
480 481
481 sums = kzalloc(btrfs_ordered_sum_size(root, bytes_left), 482 sums = kzalloc(btrfs_ordered_sum_size(root, bytes_left),
482 GFP_NOFS); 483 GFP_NOFS);
@@ -484,7 +485,7 @@ int btrfs_csum_one_bio(struct btrfs_root *root, struct inode *inode,
484 sums->len = bytes_left; 485 sums->len = bytes_left;
485 ordered = btrfs_lookup_ordered_extent(inode, offset); 486 ordered = btrfs_lookup_ordered_extent(inode, offset);
486 BUG_ON(!ordered); /* Logic error */ 487 BUG_ON(!ordered); /* Logic error */
487 sums->bytenr = ((u64)bio->bi_sector << 9) + 488 sums->bytenr = ((u64)bio->bi_iter.bi_sector << 9) +
488 total_bytes; 489 total_bytes;
489 index = 0; 490 index = 0;
490 } 491 }
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 514b291b1354..d546d8c3038b 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -1577,7 +1577,7 @@ int btrfs_merge_bio_hook(int rw, struct page *page, unsigned long offset,
1577 unsigned long bio_flags) 1577 unsigned long bio_flags)
1578{ 1578{
1579 struct btrfs_root *root = BTRFS_I(page->mapping->host)->root; 1579 struct btrfs_root *root = BTRFS_I(page->mapping->host)->root;
1580 u64 logical = (u64)bio->bi_sector << 9; 1580 u64 logical = (u64)bio->bi_iter.bi_sector << 9;
1581 u64 length = 0; 1581 u64 length = 0;
1582 u64 map_length; 1582 u64 map_length;
1583 int ret; 1583 int ret;
@@ -1585,7 +1585,7 @@ int btrfs_merge_bio_hook(int rw, struct page *page, unsigned long offset,
1585 if (bio_flags & EXTENT_BIO_COMPRESSED) 1585 if (bio_flags & EXTENT_BIO_COMPRESSED)
1586 return 0; 1586 return 0;
1587 1587
1588 length = bio->bi_size; 1588 length = bio->bi_iter.bi_size;
1589 map_length = length; 1589 map_length = length;
1590 ret = btrfs_map_block(root->fs_info, rw, logical, 1590 ret = btrfs_map_block(root->fs_info, rw, logical,
1591 &map_length, NULL, 0); 1591 &map_length, NULL, 0);
@@ -6783,17 +6783,16 @@ unlock_err:
6783static void btrfs_endio_direct_read(struct bio *bio, int err) 6783static void btrfs_endio_direct_read(struct bio *bio, int err)
6784{ 6784{
6785 struct btrfs_dio_private *dip = bio->bi_private; 6785 struct btrfs_dio_private *dip = bio->bi_private;
6786 struct bio_vec *bvec_end = bio->bi_io_vec + bio->bi_vcnt - 1; 6786 struct bio_vec *bvec;
6787 struct bio_vec *bvec = bio->bi_io_vec;
6788 struct inode *inode = dip->inode; 6787 struct inode *inode = dip->inode;
6789 struct btrfs_root *root = BTRFS_I(inode)->root; 6788 struct btrfs_root *root = BTRFS_I(inode)->root;
6790 struct bio *dio_bio; 6789 struct bio *dio_bio;
6791 u32 *csums = (u32 *)dip->csum; 6790 u32 *csums = (u32 *)dip->csum;
6792 int index = 0;
6793 u64 start; 6791 u64 start;
6792 int i;
6794 6793
6795 start = dip->logical_offset; 6794 start = dip->logical_offset;
6796 do { 6795 bio_for_each_segment_all(bvec, bio, i) {
6797 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) { 6796 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) {
6798 struct page *page = bvec->bv_page; 6797 struct page *page = bvec->bv_page;
6799 char *kaddr; 6798 char *kaddr;
@@ -6809,18 +6808,16 @@ static void btrfs_endio_direct_read(struct bio *bio, int err)
6809 local_irq_restore(flags); 6808 local_irq_restore(flags);
6810 6809
6811 flush_dcache_page(bvec->bv_page); 6810 flush_dcache_page(bvec->bv_page);
6812 if (csum != csums[index]) { 6811 if (csum != csums[i]) {
6813 btrfs_err(root->fs_info, "csum failed ino %llu off %llu csum %u expected csum %u", 6812 btrfs_err(root->fs_info, "csum failed ino %llu off %llu csum %u expected csum %u",
6814 btrfs_ino(inode), start, csum, 6813 btrfs_ino(inode), start, csum,
6815 csums[index]); 6814 csums[i]);
6816 err = -EIO; 6815 err = -EIO;
6817 } 6816 }
6818 } 6817 }
6819 6818
6820 start += bvec->bv_len; 6819 start += bvec->bv_len;
6821 bvec++; 6820 }
6822 index++;
6823 } while (bvec <= bvec_end);
6824 6821
6825 unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset, 6822 unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset,
6826 dip->logical_offset + dip->bytes - 1); 6823 dip->logical_offset + dip->bytes - 1);
@@ -6901,7 +6898,8 @@ static void btrfs_end_dio_bio(struct bio *bio, int err)
6901 printk(KERN_ERR "btrfs direct IO failed ino %llu rw %lu " 6898 printk(KERN_ERR "btrfs direct IO failed ino %llu rw %lu "
6902 "sector %#Lx len %u err no %d\n", 6899 "sector %#Lx len %u err no %d\n",
6903 btrfs_ino(dip->inode), bio->bi_rw, 6900 btrfs_ino(dip->inode), bio->bi_rw,
6904 (unsigned long long)bio->bi_sector, bio->bi_size, err); 6901 (unsigned long long)bio->bi_iter.bi_sector,
6902 bio->bi_iter.bi_size, err);
6905 dip->errors = 1; 6903 dip->errors = 1;
6906 6904
6907 /* 6905 /*
@@ -6992,7 +6990,7 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,
6992 struct bio *bio; 6990 struct bio *bio;
6993 struct bio *orig_bio = dip->orig_bio; 6991 struct bio *orig_bio = dip->orig_bio;
6994 struct bio_vec *bvec = orig_bio->bi_io_vec; 6992 struct bio_vec *bvec = orig_bio->bi_io_vec;
6995 u64 start_sector = orig_bio->bi_sector; 6993 u64 start_sector = orig_bio->bi_iter.bi_sector;
6996 u64 file_offset = dip->logical_offset; 6994 u64 file_offset = dip->logical_offset;
6997 u64 submit_len = 0; 6995 u64 submit_len = 0;
6998 u64 map_length; 6996 u64 map_length;
@@ -7000,7 +6998,7 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,
7000 int ret = 0; 6998 int ret = 0;
7001 int async_submit = 0; 6999 int async_submit = 0;
7002 7000
7003 map_length = orig_bio->bi_size; 7001 map_length = orig_bio->bi_iter.bi_size;
7004 ret = btrfs_map_block(root->fs_info, rw, start_sector << 9, 7002 ret = btrfs_map_block(root->fs_info, rw, start_sector << 9,
7005 &map_length, NULL, 0); 7003 &map_length, NULL, 0);
7006 if (ret) { 7004 if (ret) {
@@ -7008,7 +7006,7 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,
7008 return -EIO; 7006 return -EIO;
7009 } 7007 }
7010 7008
7011 if (map_length >= orig_bio->bi_size) { 7009 if (map_length >= orig_bio->bi_iter.bi_size) {
7012 bio = orig_bio; 7010 bio = orig_bio;
7013 goto submit; 7011 goto submit;
7014 } 7012 }
@@ -7060,7 +7058,7 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,
7060 bio->bi_private = dip; 7058 bio->bi_private = dip;
7061 bio->bi_end_io = btrfs_end_dio_bio; 7059 bio->bi_end_io = btrfs_end_dio_bio;
7062 7060
7063 map_length = orig_bio->bi_size; 7061 map_length = orig_bio->bi_iter.bi_size;
7064 ret = btrfs_map_block(root->fs_info, rw, 7062 ret = btrfs_map_block(root->fs_info, rw,
7065 start_sector << 9, 7063 start_sector << 9,
7066 &map_length, NULL, 0); 7064 &map_length, NULL, 0);
@@ -7118,7 +7116,8 @@ static void btrfs_submit_direct(int rw, struct bio *dio_bio,
7118 7116
7119 if (!skip_sum && !write) { 7117 if (!skip_sum && !write) {
7120 csum_size = btrfs_super_csum_size(root->fs_info->super_copy); 7118 csum_size = btrfs_super_csum_size(root->fs_info->super_copy);
7121 sum_len = dio_bio->bi_size >> inode->i_sb->s_blocksize_bits; 7119 sum_len = dio_bio->bi_iter.bi_size >>
7120 inode->i_sb->s_blocksize_bits;
7122 sum_len *= csum_size; 7121 sum_len *= csum_size;
7123 } else { 7122 } else {
7124 sum_len = 0; 7123 sum_len = 0;
@@ -7133,8 +7132,8 @@ static void btrfs_submit_direct(int rw, struct bio *dio_bio,
7133 dip->private = dio_bio->bi_private; 7132 dip->private = dio_bio->bi_private;
7134 dip->inode = inode; 7133 dip->inode = inode;
7135 dip->logical_offset = file_offset; 7134 dip->logical_offset = file_offset;
7136 dip->bytes = dio_bio->bi_size; 7135 dip->bytes = dio_bio->bi_iter.bi_size;
7137 dip->disk_bytenr = (u64)dio_bio->bi_sector << 9; 7136 dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
7138 io_bio->bi_private = dip; 7137 io_bio->bi_private = dip;
7139 dip->errors = 0; 7138 dip->errors = 0;
7140 dip->orig_bio = io_bio; 7139 dip->orig_bio = io_bio;
diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c
index 24ac21840a9a..9af0b25d991a 100644
--- a/fs/btrfs/raid56.c
+++ b/fs/btrfs/raid56.c
@@ -1032,8 +1032,8 @@ static int rbio_add_io_page(struct btrfs_raid_bio *rbio,
1032 1032
1033 /* see if we can add this page onto our existing bio */ 1033 /* see if we can add this page onto our existing bio */
1034 if (last) { 1034 if (last) {
1035 last_end = (u64)last->bi_sector << 9; 1035 last_end = (u64)last->bi_iter.bi_sector << 9;
1036 last_end += last->bi_size; 1036 last_end += last->bi_iter.bi_size;
1037 1037
1038 /* 1038 /*
1039 * we can't merge these if they are from different 1039 * we can't merge these if they are from different
@@ -1053,9 +1053,9 @@ static int rbio_add_io_page(struct btrfs_raid_bio *rbio,
1053 if (!bio) 1053 if (!bio)
1054 return -ENOMEM; 1054 return -ENOMEM;
1055 1055
1056 bio->bi_size = 0; 1056 bio->bi_iter.bi_size = 0;
1057 bio->bi_bdev = stripe->dev->bdev; 1057 bio->bi_bdev = stripe->dev->bdev;
1058 bio->bi_sector = disk_start >> 9; 1058 bio->bi_iter.bi_sector = disk_start >> 9;
1059 set_bit(BIO_UPTODATE, &bio->bi_flags); 1059 set_bit(BIO_UPTODATE, &bio->bi_flags);
1060 1060
1061 bio_add_page(bio, page, PAGE_CACHE_SIZE, 0); 1061 bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
@@ -1111,7 +1111,7 @@ static void index_rbio_pages(struct btrfs_raid_bio *rbio)
1111 1111
1112 spin_lock_irq(&rbio->bio_list_lock); 1112 spin_lock_irq(&rbio->bio_list_lock);
1113 bio_list_for_each(bio, &rbio->bio_list) { 1113 bio_list_for_each(bio, &rbio->bio_list) {
1114 start = (u64)bio->bi_sector << 9; 1114 start = (u64)bio->bi_iter.bi_sector << 9;
1115 stripe_offset = start - rbio->raid_map[0]; 1115 stripe_offset = start - rbio->raid_map[0];
1116 page_index = stripe_offset >> PAGE_CACHE_SHIFT; 1116 page_index = stripe_offset >> PAGE_CACHE_SHIFT;
1117 1117
@@ -1272,7 +1272,7 @@ cleanup:
1272static int find_bio_stripe(struct btrfs_raid_bio *rbio, 1272static int find_bio_stripe(struct btrfs_raid_bio *rbio,
1273 struct bio *bio) 1273 struct bio *bio)
1274{ 1274{
1275 u64 physical = bio->bi_sector; 1275 u64 physical = bio->bi_iter.bi_sector;
1276 u64 stripe_start; 1276 u64 stripe_start;
1277 int i; 1277 int i;
1278 struct btrfs_bio_stripe *stripe; 1278 struct btrfs_bio_stripe *stripe;
@@ -1298,7 +1298,7 @@ static int find_bio_stripe(struct btrfs_raid_bio *rbio,
1298static int find_logical_bio_stripe(struct btrfs_raid_bio *rbio, 1298static int find_logical_bio_stripe(struct btrfs_raid_bio *rbio,
1299 struct bio *bio) 1299 struct bio *bio)
1300{ 1300{
1301 u64 logical = bio->bi_sector; 1301 u64 logical = bio->bi_iter.bi_sector;
1302 u64 stripe_start; 1302 u64 stripe_start;
1303 int i; 1303 int i;
1304 1304
@@ -1602,8 +1602,8 @@ static int plug_cmp(void *priv, struct list_head *a, struct list_head *b)
1602 plug_list); 1602 plug_list);
1603 struct btrfs_raid_bio *rb = container_of(b, struct btrfs_raid_bio, 1603 struct btrfs_raid_bio *rb = container_of(b, struct btrfs_raid_bio,
1604 plug_list); 1604 plug_list);
1605 u64 a_sector = ra->bio_list.head->bi_sector; 1605 u64 a_sector = ra->bio_list.head->bi_iter.bi_sector;
1606 u64 b_sector = rb->bio_list.head->bi_sector; 1606 u64 b_sector = rb->bio_list.head->bi_iter.bi_sector;
1607 1607
1608 if (a_sector < b_sector) 1608 if (a_sector < b_sector)
1609 return -1; 1609 return -1;
@@ -1691,7 +1691,7 @@ int raid56_parity_write(struct btrfs_root *root, struct bio *bio,
1691 if (IS_ERR(rbio)) 1691 if (IS_ERR(rbio))
1692 return PTR_ERR(rbio); 1692 return PTR_ERR(rbio);
1693 bio_list_add(&rbio->bio_list, bio); 1693 bio_list_add(&rbio->bio_list, bio);
1694 rbio->bio_list_bytes = bio->bi_size; 1694 rbio->bio_list_bytes = bio->bi_iter.bi_size;
1695 1695
1696 /* 1696 /*
1697 * don't plug on full rbios, just get them out the door 1697 * don't plug on full rbios, just get them out the door
@@ -2044,7 +2044,7 @@ int raid56_parity_recover(struct btrfs_root *root, struct bio *bio,
2044 2044
2045 rbio->read_rebuild = 1; 2045 rbio->read_rebuild = 1;
2046 bio_list_add(&rbio->bio_list, bio); 2046 bio_list_add(&rbio->bio_list, bio);
2047 rbio->bio_list_bytes = bio->bi_size; 2047 rbio->bio_list_bytes = bio->bi_iter.bi_size;
2048 2048
2049 rbio->faila = find_logical_bio_stripe(rbio, bio); 2049 rbio->faila = find_logical_bio_stripe(rbio, bio);
2050 if (rbio->faila == -1) { 2050 if (rbio->faila == -1) {
diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c
index 1fd3f33c330a..bb9a928fa3a8 100644
--- a/fs/btrfs/scrub.c
+++ b/fs/btrfs/scrub.c
@@ -1308,7 +1308,7 @@ static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1308 continue; 1308 continue;
1309 } 1309 }
1310 bio->bi_bdev = page->dev->bdev; 1310 bio->bi_bdev = page->dev->bdev;
1311 bio->bi_sector = page->physical >> 9; 1311 bio->bi_iter.bi_sector = page->physical >> 9;
1312 1312
1313 bio_add_page(bio, page->page, PAGE_SIZE, 0); 1313 bio_add_page(bio, page->page, PAGE_SIZE, 0);
1314 if (btrfsic_submit_bio_wait(READ, bio)) 1314 if (btrfsic_submit_bio_wait(READ, bio))
@@ -1427,7 +1427,7 @@ static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1427 if (!bio) 1427 if (!bio)
1428 return -EIO; 1428 return -EIO;
1429 bio->bi_bdev = page_bad->dev->bdev; 1429 bio->bi_bdev = page_bad->dev->bdev;
1430 bio->bi_sector = page_bad->physical >> 9; 1430 bio->bi_iter.bi_sector = page_bad->physical >> 9;
1431 1431
1432 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0); 1432 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1433 if (PAGE_SIZE != ret) { 1433 if (PAGE_SIZE != ret) {
@@ -1520,7 +1520,7 @@ again:
1520 bio->bi_private = sbio; 1520 bio->bi_private = sbio;
1521 bio->bi_end_io = scrub_wr_bio_end_io; 1521 bio->bi_end_io = scrub_wr_bio_end_io;
1522 bio->bi_bdev = sbio->dev->bdev; 1522 bio->bi_bdev = sbio->dev->bdev;
1523 bio->bi_sector = sbio->physical >> 9; 1523 bio->bi_iter.bi_sector = sbio->physical >> 9;
1524 sbio->err = 0; 1524 sbio->err = 0;
1525 } else if (sbio->physical + sbio->page_count * PAGE_SIZE != 1525 } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1526 spage->physical_for_dev_replace || 1526 spage->physical_for_dev_replace ||
@@ -1926,7 +1926,7 @@ again:
1926 bio->bi_private = sbio; 1926 bio->bi_private = sbio;
1927 bio->bi_end_io = scrub_bio_end_io; 1927 bio->bi_end_io = scrub_bio_end_io;
1928 bio->bi_bdev = sbio->dev->bdev; 1928 bio->bi_bdev = sbio->dev->bdev;
1929 bio->bi_sector = sbio->physical >> 9; 1929 bio->bi_iter.bi_sector = sbio->physical >> 9;
1930 sbio->err = 0; 1930 sbio->err = 0;
1931 } else if (sbio->physical + sbio->page_count * PAGE_SIZE != 1931 } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1932 spage->physical || 1932 spage->physical ||
@@ -3371,8 +3371,8 @@ static int write_page_nocow(struct scrub_ctx *sctx,
3371 spin_unlock(&sctx->stat_lock); 3371 spin_unlock(&sctx->stat_lock);
3372 return -ENOMEM; 3372 return -ENOMEM;
3373 } 3373 }
3374 bio->bi_size = 0; 3374 bio->bi_iter.bi_size = 0;
3375 bio->bi_sector = physical_for_dev_replace >> 9; 3375 bio->bi_iter.bi_sector = physical_for_dev_replace >> 9;
3376 bio->bi_bdev = dev->bdev; 3376 bio->bi_bdev = dev->bdev;
3377 ret = bio_add_page(bio, page, PAGE_CACHE_SIZE, 0); 3377 ret = bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
3378 if (ret != PAGE_CACHE_SIZE) { 3378 if (ret != PAGE_CACHE_SIZE) {
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index 92303f42baaa..54d2685a3071 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -5298,6 +5298,13 @@ static void btrfs_end_bio(struct bio *bio, int err)
5298 bio_put(bio); 5298 bio_put(bio);
5299 bio = bbio->orig_bio; 5299 bio = bbio->orig_bio;
5300 } 5300 }
5301
5302 /*
5303 * We have original bio now. So increment bi_remaining to
5304 * account for it in endio
5305 */
5306 atomic_inc(&bio->bi_remaining);
5307
5301 bio->bi_private = bbio->private; 5308 bio->bi_private = bbio->private;
5302 bio->bi_end_io = bbio->end_io; 5309 bio->bi_end_io = bbio->end_io;
5303 btrfs_io_bio(bio)->mirror_num = bbio->mirror_num; 5310 btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
@@ -5411,7 +5418,7 @@ static int bio_size_ok(struct block_device *bdev, struct bio *bio,
5411 if (!q->merge_bvec_fn) 5418 if (!q->merge_bvec_fn)
5412 return 1; 5419 return 1;
5413 5420
5414 bvm.bi_size = bio->bi_size - prev->bv_len; 5421 bvm.bi_size = bio->bi_iter.bi_size - prev->bv_len;
5415 if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len) 5422 if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len)
5416 return 0; 5423 return 0;
5417 return 1; 5424 return 1;
@@ -5426,7 +5433,7 @@ static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,
5426 bio->bi_private = bbio; 5433 bio->bi_private = bbio;
5427 btrfs_io_bio(bio)->stripe_index = dev_nr; 5434 btrfs_io_bio(bio)->stripe_index = dev_nr;
5428 bio->bi_end_io = btrfs_end_bio; 5435 bio->bi_end_io = btrfs_end_bio;
5429 bio->bi_sector = physical >> 9; 5436 bio->bi_iter.bi_sector = physical >> 9;
5430#ifdef DEBUG 5437#ifdef DEBUG
5431 { 5438 {
5432 struct rcu_string *name; 5439 struct rcu_string *name;
@@ -5464,7 +5471,7 @@ again:
5464 while (bvec <= (first_bio->bi_io_vec + first_bio->bi_vcnt - 1)) { 5471 while (bvec <= (first_bio->bi_io_vec + first_bio->bi_vcnt - 1)) {
5465 if (bio_add_page(bio, bvec->bv_page, bvec->bv_len, 5472 if (bio_add_page(bio, bvec->bv_page, bvec->bv_len,
5466 bvec->bv_offset) < bvec->bv_len) { 5473 bvec->bv_offset) < bvec->bv_len) {
5467 u64 len = bio->bi_size; 5474 u64 len = bio->bi_iter.bi_size;
5468 5475
5469 atomic_inc(&bbio->stripes_pending); 5476 atomic_inc(&bbio->stripes_pending);
5470 submit_stripe_bio(root, bbio, bio, physical, dev_nr, 5477 submit_stripe_bio(root, bbio, bio, physical, dev_nr,
@@ -5486,7 +5493,7 @@ static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical)
5486 bio->bi_private = bbio->private; 5493 bio->bi_private = bbio->private;
5487 bio->bi_end_io = bbio->end_io; 5494 bio->bi_end_io = bbio->end_io;
5488 btrfs_io_bio(bio)->mirror_num = bbio->mirror_num; 5495 btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
5489 bio->bi_sector = logical >> 9; 5496 bio->bi_iter.bi_sector = logical >> 9;
5490 kfree(bbio); 5497 kfree(bbio);
5491 bio_endio(bio, -EIO); 5498 bio_endio(bio, -EIO);
5492 } 5499 }
@@ -5497,7 +5504,7 @@ int btrfs_map_bio(struct btrfs_root *root, int rw, struct bio *bio,
5497{ 5504{
5498 struct btrfs_device *dev; 5505 struct btrfs_device *dev;
5499 struct bio *first_bio = bio; 5506 struct bio *first_bio = bio;
5500 u64 logical = (u64)bio->bi_sector << 9; 5507 u64 logical = (u64)bio->bi_iter.bi_sector << 9;
5501 u64 length = 0; 5508 u64 length = 0;
5502 u64 map_length; 5509 u64 map_length;
5503 u64 *raid_map = NULL; 5510 u64 *raid_map = NULL;
@@ -5506,7 +5513,7 @@ int btrfs_map_bio(struct btrfs_root *root, int rw, struct bio *bio,
5506 int total_devs = 1; 5513 int total_devs = 1;
5507 struct btrfs_bio *bbio = NULL; 5514 struct btrfs_bio *bbio = NULL;
5508 5515
5509 length = bio->bi_size; 5516 length = bio->bi_iter.bi_size;
5510 map_length = length; 5517 map_length = length;
5511 5518
5512 ret = __btrfs_map_block(root->fs_info, rw, logical, &map_length, &bbio, 5519 ret = __btrfs_map_block(root->fs_info, rw, logical, &map_length, &bbio,
diff --git a/fs/buffer.c b/fs/buffer.c
index 6024877335ca..1c04ec66974e 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -2982,11 +2982,11 @@ static void guard_bh_eod(int rw, struct bio *bio, struct buffer_head *bh)
2982 * let it through, and the IO layer will turn it into 2982 * let it through, and the IO layer will turn it into
2983 * an EIO. 2983 * an EIO.
2984 */ 2984 */
2985 if (unlikely(bio->bi_sector >= maxsector)) 2985 if (unlikely(bio->bi_iter.bi_sector >= maxsector))
2986 return; 2986 return;
2987 2987
2988 maxsector -= bio->bi_sector; 2988 maxsector -= bio->bi_iter.bi_sector;
2989 bytes = bio->bi_size; 2989 bytes = bio->bi_iter.bi_size;
2990 if (likely((bytes >> 9) <= maxsector)) 2990 if (likely((bytes >> 9) <= maxsector))
2991 return; 2991 return;
2992 2992
@@ -2994,7 +2994,7 @@ static void guard_bh_eod(int rw, struct bio *bio, struct buffer_head *bh)
2994 bytes = maxsector << 9; 2994 bytes = maxsector << 9;
2995 2995
2996 /* Truncate the bio.. */ 2996 /* Truncate the bio.. */
2997 bio->bi_size = bytes; 2997 bio->bi_iter.bi_size = bytes;
2998 bio->bi_io_vec[0].bv_len = bytes; 2998 bio->bi_io_vec[0].bv_len = bytes;
2999 2999
3000 /* ..and clear the end of the buffer for reads */ 3000 /* ..and clear the end of the buffer for reads */
@@ -3029,14 +3029,14 @@ int _submit_bh(int rw, struct buffer_head *bh, unsigned long bio_flags)
3029 */ 3029 */
3030 bio = bio_alloc(GFP_NOIO, 1); 3030 bio = bio_alloc(GFP_NOIO, 1);
3031 3031
3032 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9); 3032 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
3033 bio->bi_bdev = bh->b_bdev; 3033 bio->bi_bdev = bh->b_bdev;
3034 bio->bi_io_vec[0].bv_page = bh->b_page; 3034 bio->bi_io_vec[0].bv_page = bh->b_page;
3035 bio->bi_io_vec[0].bv_len = bh->b_size; 3035 bio->bi_io_vec[0].bv_len = bh->b_size;
3036 bio->bi_io_vec[0].bv_offset = bh_offset(bh); 3036 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
3037 3037
3038 bio->bi_vcnt = 1; 3038 bio->bi_vcnt = 1;
3039 bio->bi_size = bh->b_size; 3039 bio->bi_iter.bi_size = bh->b_size;
3040 3040
3041 bio->bi_end_io = end_bio_bh_io_sync; 3041 bio->bi_end_io = end_bio_bh_io_sync;
3042 bio->bi_private = bh; 3042 bio->bi_private = bh;
diff --git a/fs/direct-io.c b/fs/direct-io.c
index 0e04142d5962..160a5489a939 100644
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -375,7 +375,7 @@ dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,
375 bio = bio_alloc(GFP_KERNEL, nr_vecs); 375 bio = bio_alloc(GFP_KERNEL, nr_vecs);
376 376
377 bio->bi_bdev = bdev; 377 bio->bi_bdev = bdev;
378 bio->bi_sector = first_sector; 378 bio->bi_iter.bi_sector = first_sector;
379 if (dio->is_async) 379 if (dio->is_async)
380 bio->bi_end_io = dio_bio_end_aio; 380 bio->bi_end_io = dio_bio_end_aio;
381 else 381 else
@@ -719,7 +719,7 @@ static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio,
719 if (sdio->bio) { 719 if (sdio->bio) {
720 loff_t cur_offset = sdio->cur_page_fs_offset; 720 loff_t cur_offset = sdio->cur_page_fs_offset;
721 loff_t bio_next_offset = sdio->logical_offset_in_bio + 721 loff_t bio_next_offset = sdio->logical_offset_in_bio +
722 sdio->bio->bi_size; 722 sdio->bio->bi_iter.bi_size;
723 723
724 /* 724 /*
725 * See whether this new request is contiguous with the old. 725 * See whether this new request is contiguous with the old.
diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c
index d488f80ee32d..ab95508e3d40 100644
--- a/fs/ext4/page-io.c
+++ b/fs/ext4/page-io.c
@@ -65,9 +65,9 @@ static void ext4_finish_bio(struct bio *bio)
65{ 65{
66 int i; 66 int i;
67 int error = !test_bit(BIO_UPTODATE, &bio->bi_flags); 67 int error = !test_bit(BIO_UPTODATE, &bio->bi_flags);
68 struct bio_vec *bvec;
68 69
69 for (i = 0; i < bio->bi_vcnt; i++) { 70 bio_for_each_segment_all(bvec, bio, i) {
70 struct bio_vec *bvec = &bio->bi_io_vec[i];
71 struct page *page = bvec->bv_page; 71 struct page *page = bvec->bv_page;
72 struct buffer_head *bh, *head; 72 struct buffer_head *bh, *head;
73 unsigned bio_start = bvec->bv_offset; 73 unsigned bio_start = bvec->bv_offset;
@@ -298,7 +298,7 @@ ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end)
298static void ext4_end_bio(struct bio *bio, int error) 298static void ext4_end_bio(struct bio *bio, int error)
299{ 299{
300 ext4_io_end_t *io_end = bio->bi_private; 300 ext4_io_end_t *io_end = bio->bi_private;
301 sector_t bi_sector = bio->bi_sector; 301 sector_t bi_sector = bio->bi_iter.bi_sector;
302 302
303 BUG_ON(!io_end); 303 BUG_ON(!io_end);
304 bio->bi_end_io = NULL; 304 bio->bi_end_io = NULL;
@@ -366,7 +366,7 @@ static int io_submit_init_bio(struct ext4_io_submit *io,
366 bio = bio_alloc(GFP_NOIO, min(nvecs, BIO_MAX_PAGES)); 366 bio = bio_alloc(GFP_NOIO, min(nvecs, BIO_MAX_PAGES));
367 if (!bio) 367 if (!bio)
368 return -ENOMEM; 368 return -ENOMEM;
369 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9); 369 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
370 bio->bi_bdev = bh->b_bdev; 370 bio->bi_bdev = bh->b_bdev;
371 bio->bi_end_io = ext4_end_bio; 371 bio->bi_end_io = ext4_end_bio;
372 bio->bi_private = ext4_get_io_end(io->io_end); 372 bio->bi_private = ext4_get_io_end(io->io_end);
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index 0ae558723506..2261ccdd0b5f 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -26,40 +26,33 @@
26 26
27static void f2fs_read_end_io(struct bio *bio, int err) 27static void f2fs_read_end_io(struct bio *bio, int err)
28{ 28{
29 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 29 struct bio_vec *bvec;
30 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 30 int i;
31 31
32 do { 32 bio_for_each_segment_all(bvec, bio, i) {
33 struct page *page = bvec->bv_page; 33 struct page *page = bvec->bv_page;
34 34
35 if (--bvec >= bio->bi_io_vec) 35 if (!err) {
36 prefetchw(&bvec->bv_page->flags); 36 SetPageUptodate(page);
37 37 } else {
38 if (unlikely(!uptodate)) {
39 ClearPageUptodate(page); 38 ClearPageUptodate(page);
40 SetPageError(page); 39 SetPageError(page);
41 } else {
42 SetPageUptodate(page);
43 } 40 }
44 unlock_page(page); 41 unlock_page(page);
45 } while (bvec >= bio->bi_io_vec); 42 }
46
47 bio_put(bio); 43 bio_put(bio);
48} 44}
49 45
50static void f2fs_write_end_io(struct bio *bio, int err) 46static void f2fs_write_end_io(struct bio *bio, int err)
51{ 47{
52 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 48 struct f2fs_sb_info *sbi = F2FS_SB(bio->bi_io_vec->bv_page->mapping->host->i_sb);
53 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 49 struct bio_vec *bvec;
54 struct f2fs_sb_info *sbi = F2FS_SB(bvec->bv_page->mapping->host->i_sb); 50 int i;
55 51
56 do { 52 bio_for_each_segment_all(bvec, bio, i) {
57 struct page *page = bvec->bv_page; 53 struct page *page = bvec->bv_page;
58 54
59 if (--bvec >= bio->bi_io_vec) 55 if (unlikely(err)) {
60 prefetchw(&bvec->bv_page->flags);
61
62 if (unlikely(!uptodate)) {
63 SetPageError(page); 56 SetPageError(page);
64 set_bit(AS_EIO, &page->mapping->flags); 57 set_bit(AS_EIO, &page->mapping->flags);
65 set_ckpt_flags(sbi->ckpt, CP_ERROR_FLAG); 58 set_ckpt_flags(sbi->ckpt, CP_ERROR_FLAG);
@@ -67,7 +60,7 @@ static void f2fs_write_end_io(struct bio *bio, int err)
67 } 60 }
68 end_page_writeback(page); 61 end_page_writeback(page);
69 dec_page_count(sbi, F2FS_WRITEBACK); 62 dec_page_count(sbi, F2FS_WRITEBACK);
70 } while (bvec >= bio->bi_io_vec); 63 }
71 64
72 if (bio->bi_private) 65 if (bio->bi_private)
73 complete(bio->bi_private); 66 complete(bio->bi_private);
@@ -91,7 +84,7 @@ static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
91 bio = bio_alloc(GFP_NOIO, npages); 84 bio = bio_alloc(GFP_NOIO, npages);
92 85
93 bio->bi_bdev = sbi->sb->s_bdev; 86 bio->bi_bdev = sbi->sb->s_bdev;
94 bio->bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr); 87 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr);
95 bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io; 88 bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io;
96 89
97 return bio; 90 return bio;
diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c
index 58f06400b7b8..76693793cedd 100644
--- a/fs/gfs2/lops.c
+++ b/fs/gfs2/lops.c
@@ -273,7 +273,7 @@ static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno)
273 nrvecs = max(nrvecs/2, 1U); 273 nrvecs = max(nrvecs/2, 1U);
274 } 274 }
275 275
276 bio->bi_sector = blkno * (sb->s_blocksize >> 9); 276 bio->bi_iter.bi_sector = blkno * (sb->s_blocksize >> 9);
277 bio->bi_bdev = sb->s_bdev; 277 bio->bi_bdev = sb->s_bdev;
278 bio->bi_end_io = gfs2_end_log_write; 278 bio->bi_end_io = gfs2_end_log_write;
279 bio->bi_private = sdp; 279 bio->bi_private = sdp;
diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c
index 1e712b566d76..c6872d09561a 100644
--- a/fs/gfs2/ops_fstype.c
+++ b/fs/gfs2/ops_fstype.c
@@ -238,7 +238,7 @@ static int gfs2_read_super(struct gfs2_sbd *sdp, sector_t sector, int silent)
238 lock_page(page); 238 lock_page(page);
239 239
240 bio = bio_alloc(GFP_NOFS, 1); 240 bio = bio_alloc(GFP_NOFS, 1);
241 bio->bi_sector = sector * (sb->s_blocksize >> 9); 241 bio->bi_iter.bi_sector = sector * (sb->s_blocksize >> 9);
242 bio->bi_bdev = sb->s_bdev; 242 bio->bi_bdev = sb->s_bdev;
243 bio_add_page(bio, page, PAGE_SIZE, 0); 243 bio_add_page(bio, page, PAGE_SIZE, 0);
244 244
diff --git a/fs/hfsplus/wrapper.c b/fs/hfsplus/wrapper.c
index e9a97a0d4314..3f999649587f 100644
--- a/fs/hfsplus/wrapper.c
+++ b/fs/hfsplus/wrapper.c
@@ -63,7 +63,7 @@ int hfsplus_submit_bio(struct super_block *sb, sector_t sector,
63 sector &= ~((io_size >> HFSPLUS_SECTOR_SHIFT) - 1); 63 sector &= ~((io_size >> HFSPLUS_SECTOR_SHIFT) - 1);
64 64
65 bio = bio_alloc(GFP_NOIO, 1); 65 bio = bio_alloc(GFP_NOIO, 1);
66 bio->bi_sector = sector; 66 bio->bi_iter.bi_sector = sector;
67 bio->bi_bdev = sb->s_bdev; 67 bio->bi_bdev = sb->s_bdev;
68 68
69 if (!(rw & WRITE) && data) 69 if (!(rw & WRITE) && data)
diff --git a/fs/jfs/jfs_logmgr.c b/fs/jfs/jfs_logmgr.c
index 360d27c48887..8d811e02b4b9 100644
--- a/fs/jfs/jfs_logmgr.c
+++ b/fs/jfs/jfs_logmgr.c
@@ -1998,20 +1998,20 @@ static int lbmRead(struct jfs_log * log, int pn, struct lbuf ** bpp)
1998 1998
1999 bio = bio_alloc(GFP_NOFS, 1); 1999 bio = bio_alloc(GFP_NOFS, 1);
2000 2000
2001 bio->bi_sector = bp->l_blkno << (log->l2bsize - 9); 2001 bio->bi_iter.bi_sector = bp->l_blkno << (log->l2bsize - 9);
2002 bio->bi_bdev = log->bdev; 2002 bio->bi_bdev = log->bdev;
2003 bio->bi_io_vec[0].bv_page = bp->l_page; 2003 bio->bi_io_vec[0].bv_page = bp->l_page;
2004 bio->bi_io_vec[0].bv_len = LOGPSIZE; 2004 bio->bi_io_vec[0].bv_len = LOGPSIZE;
2005 bio->bi_io_vec[0].bv_offset = bp->l_offset; 2005 bio->bi_io_vec[0].bv_offset = bp->l_offset;
2006 2006
2007 bio->bi_vcnt = 1; 2007 bio->bi_vcnt = 1;
2008 bio->bi_size = LOGPSIZE; 2008 bio->bi_iter.bi_size = LOGPSIZE;
2009 2009
2010 bio->bi_end_io = lbmIODone; 2010 bio->bi_end_io = lbmIODone;
2011 bio->bi_private = bp; 2011 bio->bi_private = bp;
2012 /*check if journaling to disk has been disabled*/ 2012 /*check if journaling to disk has been disabled*/
2013 if (log->no_integrity) { 2013 if (log->no_integrity) {
2014 bio->bi_size = 0; 2014 bio->bi_iter.bi_size = 0;
2015 lbmIODone(bio, 0); 2015 lbmIODone(bio, 0);
2016 } else { 2016 } else {
2017 submit_bio(READ_SYNC, bio); 2017 submit_bio(READ_SYNC, bio);
@@ -2144,21 +2144,21 @@ static void lbmStartIO(struct lbuf * bp)
2144 jfs_info("lbmStartIO\n"); 2144 jfs_info("lbmStartIO\n");
2145 2145
2146 bio = bio_alloc(GFP_NOFS, 1); 2146 bio = bio_alloc(GFP_NOFS, 1);
2147 bio->bi_sector = bp->l_blkno << (log->l2bsize - 9); 2147 bio->bi_iter.bi_sector = bp->l_blkno << (log->l2bsize - 9);
2148 bio->bi_bdev = log->bdev; 2148 bio->bi_bdev = log->bdev;
2149 bio->bi_io_vec[0].bv_page = bp->l_page; 2149 bio->bi_io_vec[0].bv_page = bp->l_page;
2150 bio->bi_io_vec[0].bv_len = LOGPSIZE; 2150 bio->bi_io_vec[0].bv_len = LOGPSIZE;
2151 bio->bi_io_vec[0].bv_offset = bp->l_offset; 2151 bio->bi_io_vec[0].bv_offset = bp->l_offset;
2152 2152
2153 bio->bi_vcnt = 1; 2153 bio->bi_vcnt = 1;
2154 bio->bi_size = LOGPSIZE; 2154 bio->bi_iter.bi_size = LOGPSIZE;
2155 2155
2156 bio->bi_end_io = lbmIODone; 2156 bio->bi_end_io = lbmIODone;
2157 bio->bi_private = bp; 2157 bio->bi_private = bp;
2158 2158
2159 /* check if journaling to disk has been disabled */ 2159 /* check if journaling to disk has been disabled */
2160 if (log->no_integrity) { 2160 if (log->no_integrity) {
2161 bio->bi_size = 0; 2161 bio->bi_iter.bi_size = 0;
2162 lbmIODone(bio, 0); 2162 lbmIODone(bio, 0);
2163 } else { 2163 } else {
2164 submit_bio(WRITE_SYNC, bio); 2164 submit_bio(WRITE_SYNC, bio);
diff --git a/fs/jfs/jfs_metapage.c b/fs/jfs/jfs_metapage.c
index d165cde0c68d..49ba7ff1bbb9 100644
--- a/fs/jfs/jfs_metapage.c
+++ b/fs/jfs/jfs_metapage.c
@@ -416,7 +416,7 @@ static int metapage_writepage(struct page *page, struct writeback_control *wbc)
416 * count from hitting zero before we're through 416 * count from hitting zero before we're through
417 */ 417 */
418 inc_io(page); 418 inc_io(page);
419 if (!bio->bi_size) 419 if (!bio->bi_iter.bi_size)
420 goto dump_bio; 420 goto dump_bio;
421 submit_bio(WRITE, bio); 421 submit_bio(WRITE, bio);
422 nr_underway++; 422 nr_underway++;
@@ -438,7 +438,7 @@ static int metapage_writepage(struct page *page, struct writeback_control *wbc)
438 438
439 bio = bio_alloc(GFP_NOFS, 1); 439 bio = bio_alloc(GFP_NOFS, 1);
440 bio->bi_bdev = inode->i_sb->s_bdev; 440 bio->bi_bdev = inode->i_sb->s_bdev;
441 bio->bi_sector = pblock << (inode->i_blkbits - 9); 441 bio->bi_iter.bi_sector = pblock << (inode->i_blkbits - 9);
442 bio->bi_end_io = metapage_write_end_io; 442 bio->bi_end_io = metapage_write_end_io;
443 bio->bi_private = page; 443 bio->bi_private = page;
444 444
@@ -452,7 +452,7 @@ static int metapage_writepage(struct page *page, struct writeback_control *wbc)
452 if (bio) { 452 if (bio) {
453 if (bio_add_page(bio, page, bio_bytes, bio_offset) < bio_bytes) 453 if (bio_add_page(bio, page, bio_bytes, bio_offset) < bio_bytes)
454 goto add_failed; 454 goto add_failed;
455 if (!bio->bi_size) 455 if (!bio->bi_iter.bi_size)
456 goto dump_bio; 456 goto dump_bio;
457 457
458 submit_bio(WRITE, bio); 458 submit_bio(WRITE, bio);
@@ -517,7 +517,8 @@ static int metapage_readpage(struct file *fp, struct page *page)
517 517
518 bio = bio_alloc(GFP_NOFS, 1); 518 bio = bio_alloc(GFP_NOFS, 1);
519 bio->bi_bdev = inode->i_sb->s_bdev; 519 bio->bi_bdev = inode->i_sb->s_bdev;
520 bio->bi_sector = pblock << (inode->i_blkbits - 9); 520 bio->bi_iter.bi_sector =
521 pblock << (inode->i_blkbits - 9);
521 bio->bi_end_io = metapage_read_end_io; 522 bio->bi_end_io = metapage_read_end_io;
522 bio->bi_private = page; 523 bio->bi_private = page;
523 len = xlen << inode->i_blkbits; 524 len = xlen << inode->i_blkbits;
diff --git a/fs/logfs/dev_bdev.c b/fs/logfs/dev_bdev.c
index 0f95f0d0b313..76279e11982d 100644
--- a/fs/logfs/dev_bdev.c
+++ b/fs/logfs/dev_bdev.c
@@ -26,9 +26,9 @@ static int sync_request(struct page *page, struct block_device *bdev, int rw)
26 bio_vec.bv_len = PAGE_SIZE; 26 bio_vec.bv_len = PAGE_SIZE;
27 bio_vec.bv_offset = 0; 27 bio_vec.bv_offset = 0;
28 bio.bi_vcnt = 1; 28 bio.bi_vcnt = 1;
29 bio.bi_size = PAGE_SIZE;
30 bio.bi_bdev = bdev; 29 bio.bi_bdev = bdev;
31 bio.bi_sector = page->index * (PAGE_SIZE >> 9); 30 bio.bi_iter.bi_sector = page->index * (PAGE_SIZE >> 9);
31 bio.bi_iter.bi_size = PAGE_SIZE;
32 32
33 return submit_bio_wait(rw, &bio); 33 return submit_bio_wait(rw, &bio);
34} 34}
@@ -56,22 +56,18 @@ static DECLARE_WAIT_QUEUE_HEAD(wq);
56static void writeseg_end_io(struct bio *bio, int err) 56static void writeseg_end_io(struct bio *bio, int err)
57{ 57{
58 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 58 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
59 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 59 struct bio_vec *bvec;
60 int i;
60 struct super_block *sb = bio->bi_private; 61 struct super_block *sb = bio->bi_private;
61 struct logfs_super *super = logfs_super(sb); 62 struct logfs_super *super = logfs_super(sb);
62 struct page *page;
63 63
64 BUG_ON(!uptodate); /* FIXME: Retry io or write elsewhere */ 64 BUG_ON(!uptodate); /* FIXME: Retry io or write elsewhere */
65 BUG_ON(err); 65 BUG_ON(err);
66 BUG_ON(bio->bi_vcnt == 0); 66
67 do { 67 bio_for_each_segment_all(bvec, bio, i) {
68 page = bvec->bv_page; 68 end_page_writeback(bvec->bv_page);
69 if (--bvec >= bio->bi_io_vec) 69 page_cache_release(bvec->bv_page);
70 prefetchw(&bvec->bv_page->flags); 70 }
71
72 end_page_writeback(page);
73 page_cache_release(page);
74 } while (bvec >= bio->bi_io_vec);
75 bio_put(bio); 71 bio_put(bio);
76 if (atomic_dec_and_test(&super->s_pending_writes)) 72 if (atomic_dec_and_test(&super->s_pending_writes))
77 wake_up(&wq); 73 wake_up(&wq);
@@ -96,9 +92,9 @@ static int __bdev_writeseg(struct super_block *sb, u64 ofs, pgoff_t index,
96 if (i >= max_pages) { 92 if (i >= max_pages) {
97 /* Block layer cannot split bios :( */ 93 /* Block layer cannot split bios :( */
98 bio->bi_vcnt = i; 94 bio->bi_vcnt = i;
99 bio->bi_size = i * PAGE_SIZE; 95 bio->bi_iter.bi_size = i * PAGE_SIZE;
100 bio->bi_bdev = super->s_bdev; 96 bio->bi_bdev = super->s_bdev;
101 bio->bi_sector = ofs >> 9; 97 bio->bi_iter.bi_sector = ofs >> 9;
102 bio->bi_private = sb; 98 bio->bi_private = sb;
103 bio->bi_end_io = writeseg_end_io; 99 bio->bi_end_io = writeseg_end_io;
104 atomic_inc(&super->s_pending_writes); 100 atomic_inc(&super->s_pending_writes);
@@ -123,9 +119,9 @@ static int __bdev_writeseg(struct super_block *sb, u64 ofs, pgoff_t index,
123 unlock_page(page); 119 unlock_page(page);
124 } 120 }
125 bio->bi_vcnt = nr_pages; 121 bio->bi_vcnt = nr_pages;
126 bio->bi_size = nr_pages * PAGE_SIZE; 122 bio->bi_iter.bi_size = nr_pages * PAGE_SIZE;
127 bio->bi_bdev = super->s_bdev; 123 bio->bi_bdev = super->s_bdev;
128 bio->bi_sector = ofs >> 9; 124 bio->bi_iter.bi_sector = ofs >> 9;
129 bio->bi_private = sb; 125 bio->bi_private = sb;
130 bio->bi_end_io = writeseg_end_io; 126 bio->bi_end_io = writeseg_end_io;
131 atomic_inc(&super->s_pending_writes); 127 atomic_inc(&super->s_pending_writes);
@@ -188,9 +184,9 @@ static int do_erase(struct super_block *sb, u64 ofs, pgoff_t index,
188 if (i >= max_pages) { 184 if (i >= max_pages) {
189 /* Block layer cannot split bios :( */ 185 /* Block layer cannot split bios :( */
190 bio->bi_vcnt = i; 186 bio->bi_vcnt = i;
191 bio->bi_size = i * PAGE_SIZE; 187 bio->bi_iter.bi_size = i * PAGE_SIZE;
192 bio->bi_bdev = super->s_bdev; 188 bio->bi_bdev = super->s_bdev;
193 bio->bi_sector = ofs >> 9; 189 bio->bi_iter.bi_sector = ofs >> 9;
194 bio->bi_private = sb; 190 bio->bi_private = sb;
195 bio->bi_end_io = erase_end_io; 191 bio->bi_end_io = erase_end_io;
196 atomic_inc(&super->s_pending_writes); 192 atomic_inc(&super->s_pending_writes);
@@ -209,9 +205,9 @@ static int do_erase(struct super_block *sb, u64 ofs, pgoff_t index,
209 bio->bi_io_vec[i].bv_offset = 0; 205 bio->bi_io_vec[i].bv_offset = 0;
210 } 206 }
211 bio->bi_vcnt = nr_pages; 207 bio->bi_vcnt = nr_pages;
212 bio->bi_size = nr_pages * PAGE_SIZE; 208 bio->bi_iter.bi_size = nr_pages * PAGE_SIZE;
213 bio->bi_bdev = super->s_bdev; 209 bio->bi_bdev = super->s_bdev;
214 bio->bi_sector = ofs >> 9; 210 bio->bi_iter.bi_sector = ofs >> 9;
215 bio->bi_private = sb; 211 bio->bi_private = sb;
216 bio->bi_end_io = erase_end_io; 212 bio->bi_end_io = erase_end_io;
217 atomic_inc(&super->s_pending_writes); 213 atomic_inc(&super->s_pending_writes);
diff --git a/fs/mpage.c b/fs/mpage.c
index 0face1c4d4c6..4979ffa60aaa 100644
--- a/fs/mpage.c
+++ b/fs/mpage.c
@@ -43,16 +43,14 @@
43 */ 43 */
44static void mpage_end_io(struct bio *bio, int err) 44static void mpage_end_io(struct bio *bio, int err)
45{ 45{
46 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 46 struct bio_vec *bv;
47 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 47 int i;
48 48
49 do { 49 bio_for_each_segment_all(bv, bio, i) {
50 struct page *page = bvec->bv_page; 50 struct page *page = bv->bv_page;
51 51
52 if (--bvec >= bio->bi_io_vec)
53 prefetchw(&bvec->bv_page->flags);
54 if (bio_data_dir(bio) == READ) { 52 if (bio_data_dir(bio) == READ) {
55 if (uptodate) { 53 if (!err) {
56 SetPageUptodate(page); 54 SetPageUptodate(page);
57 } else { 55 } else {
58 ClearPageUptodate(page); 56 ClearPageUptodate(page);
@@ -60,14 +58,15 @@ static void mpage_end_io(struct bio *bio, int err)
60 } 58 }
61 unlock_page(page); 59 unlock_page(page);
62 } else { /* bio_data_dir(bio) == WRITE */ 60 } else { /* bio_data_dir(bio) == WRITE */
63 if (!uptodate) { 61 if (err) {
64 SetPageError(page); 62 SetPageError(page);
65 if (page->mapping) 63 if (page->mapping)
66 set_bit(AS_EIO, &page->mapping->flags); 64 set_bit(AS_EIO, &page->mapping->flags);
67 } 65 }
68 end_page_writeback(page); 66 end_page_writeback(page);
69 } 67 }
70 } while (bvec >= bio->bi_io_vec); 68 }
69
71 bio_put(bio); 70 bio_put(bio);
72} 71}
73 72
@@ -94,7 +93,7 @@ mpage_alloc(struct block_device *bdev,
94 93
95 if (bio) { 94 if (bio) {
96 bio->bi_bdev = bdev; 95 bio->bi_bdev = bdev;
97 bio->bi_sector = first_sector; 96 bio->bi_iter.bi_sector = first_sector;
98 } 97 }
99 return bio; 98 return bio;
100} 99}
diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c
index e242bbf72972..56ff823ca82e 100644
--- a/fs/nfs/blocklayout/blocklayout.c
+++ b/fs/nfs/blocklayout/blocklayout.c
@@ -134,8 +134,8 @@ bl_submit_bio(int rw, struct bio *bio)
134 if (bio) { 134 if (bio) {
135 get_parallel(bio->bi_private); 135 get_parallel(bio->bi_private);
136 dprintk("%s submitting %s bio %u@%llu\n", __func__, 136 dprintk("%s submitting %s bio %u@%llu\n", __func__,
137 rw == READ ? "read" : "write", 137 rw == READ ? "read" : "write", bio->bi_iter.bi_size,
138 bio->bi_size, (unsigned long long)bio->bi_sector); 138 (unsigned long long)bio->bi_iter.bi_sector);
139 submit_bio(rw, bio); 139 submit_bio(rw, bio);
140 } 140 }
141 return NULL; 141 return NULL;
@@ -156,7 +156,8 @@ static struct bio *bl_alloc_init_bio(int npg, sector_t isect,
156 } 156 }
157 157
158 if (bio) { 158 if (bio) {
159 bio->bi_sector = isect - be->be_f_offset + be->be_v_offset; 159 bio->bi_iter.bi_sector = isect - be->be_f_offset +
160 be->be_v_offset;
160 bio->bi_bdev = be->be_mdev; 161 bio->bi_bdev = be->be_mdev;
161 bio->bi_end_io = end_io; 162 bio->bi_end_io = end_io;
162 bio->bi_private = par; 163 bio->bi_private = par;
@@ -201,18 +202,14 @@ static struct bio *bl_add_page_to_bio(struct bio *bio, int npg, int rw,
201static void bl_end_io_read(struct bio *bio, int err) 202static void bl_end_io_read(struct bio *bio, int err)
202{ 203{
203 struct parallel_io *par = bio->bi_private; 204 struct parallel_io *par = bio->bi_private;
204 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 205 struct bio_vec *bvec;
205 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 206 int i;
206 207
207 do { 208 if (!err)
208 struct page *page = bvec->bv_page; 209 bio_for_each_segment_all(bvec, bio, i)
210 SetPageUptodate(bvec->bv_page);
209 211
210 if (--bvec >= bio->bi_io_vec) 212 if (err) {
211 prefetchw(&bvec->bv_page->flags);
212 if (uptodate)
213 SetPageUptodate(page);
214 } while (bvec >= bio->bi_io_vec);
215 if (!uptodate) {
216 struct nfs_read_data *rdata = par->data; 213 struct nfs_read_data *rdata = par->data;
217 struct nfs_pgio_header *header = rdata->header; 214 struct nfs_pgio_header *header = rdata->header;
218 215
@@ -383,20 +380,16 @@ static void mark_extents_written(struct pnfs_block_layout *bl,
383static void bl_end_io_write_zero(struct bio *bio, int err) 380static void bl_end_io_write_zero(struct bio *bio, int err)
384{ 381{
385 struct parallel_io *par = bio->bi_private; 382 struct parallel_io *par = bio->bi_private;
386 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 383 struct bio_vec *bvec;
387 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 384 int i;
388
389 do {
390 struct page *page = bvec->bv_page;
391 385
392 if (--bvec >= bio->bi_io_vec) 386 bio_for_each_segment_all(bvec, bio, i) {
393 prefetchw(&bvec->bv_page->flags);
394 /* This is the zeroing page we added */ 387 /* This is the zeroing page we added */
395 end_page_writeback(page); 388 end_page_writeback(bvec->bv_page);
396 page_cache_release(page); 389 page_cache_release(bvec->bv_page);
397 } while (bvec >= bio->bi_io_vec); 390 }
398 391
399 if (unlikely(!uptodate)) { 392 if (unlikely(err)) {
400 struct nfs_write_data *data = par->data; 393 struct nfs_write_data *data = par->data;
401 struct nfs_pgio_header *header = data->header; 394 struct nfs_pgio_header *header = data->header;
402 395
@@ -519,7 +512,7 @@ bl_do_readpage_sync(struct page *page, struct pnfs_block_extent *be,
519 isect = (page->index << PAGE_CACHE_SECTOR_SHIFT) + 512 isect = (page->index << PAGE_CACHE_SECTOR_SHIFT) +
520 (offset / SECTOR_SIZE); 513 (offset / SECTOR_SIZE);
521 514
522 bio->bi_sector = isect - be->be_f_offset + be->be_v_offset; 515 bio->bi_iter.bi_sector = isect - be->be_f_offset + be->be_v_offset;
523 bio->bi_bdev = be->be_mdev; 516 bio->bi_bdev = be->be_mdev;
524 bio->bi_end_io = bl_read_single_end_io; 517 bio->bi_end_io = bl_read_single_end_io;
525 518
diff --git a/fs/nilfs2/segbuf.c b/fs/nilfs2/segbuf.c
index 2d8be51f90dc..dc3a9efdaab8 100644
--- a/fs/nilfs2/segbuf.c
+++ b/fs/nilfs2/segbuf.c
@@ -416,7 +416,8 @@ static struct bio *nilfs_alloc_seg_bio(struct the_nilfs *nilfs, sector_t start,
416 } 416 }
417 if (likely(bio)) { 417 if (likely(bio)) {
418 bio->bi_bdev = nilfs->ns_bdev; 418 bio->bi_bdev = nilfs->ns_bdev;
419 bio->bi_sector = start << (nilfs->ns_blocksize_bits - 9); 419 bio->bi_iter.bi_sector =
420 start << (nilfs->ns_blocksize_bits - 9);
420 } 421 }
421 return bio; 422 return bio;
422} 423}
diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c
index 73920ffda05b..bf482dfed14f 100644
--- a/fs/ocfs2/cluster/heartbeat.c
+++ b/fs/ocfs2/cluster/heartbeat.c
@@ -413,7 +413,7 @@ static struct bio *o2hb_setup_one_bio(struct o2hb_region *reg,
413 } 413 }
414 414
415 /* Must put everything in 512 byte sectors for the bio... */ 415 /* Must put everything in 512 byte sectors for the bio... */
416 bio->bi_sector = (reg->hr_start_block + cs) << (bits - 9); 416 bio->bi_iter.bi_sector = (reg->hr_start_block + cs) << (bits - 9);
417 bio->bi_bdev = reg->hr_bdev; 417 bio->bi_bdev = reg->hr_bdev;
418 bio->bi_private = wc; 418 bio->bi_private = wc;
419 bio->bi_end_io = o2hb_bio_end_io; 419 bio->bi_end_io = o2hb_bio_end_io;
diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c
index a26739451b53..db2cfb067d0b 100644
--- a/fs/xfs/xfs_aops.c
+++ b/fs/xfs/xfs_aops.c
@@ -407,7 +407,7 @@ xfs_alloc_ioend_bio(
407 struct bio *bio = bio_alloc(GFP_NOIO, nvecs); 407 struct bio *bio = bio_alloc(GFP_NOIO, nvecs);
408 408
409 ASSERT(bio->bi_private == NULL); 409 ASSERT(bio->bi_private == NULL);
410 bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9); 410 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
411 bio->bi_bdev = bh->b_bdev; 411 bio->bi_bdev = bh->b_bdev;
412 return bio; 412 return bio;
413} 413}
diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c
index 51757113a822..9c061ef2b0d9 100644
--- a/fs/xfs/xfs_buf.c
+++ b/fs/xfs/xfs_buf.c
@@ -1240,7 +1240,7 @@ next_chunk:
1240 1240
1241 bio = bio_alloc(GFP_NOIO, nr_pages); 1241 bio = bio_alloc(GFP_NOIO, nr_pages);
1242 bio->bi_bdev = bp->b_target->bt_bdev; 1242 bio->bi_bdev = bp->b_target->bt_bdev;
1243 bio->bi_sector = sector; 1243 bio->bi_iter.bi_sector = sector;
1244 bio->bi_end_io = xfs_buf_bio_end_io; 1244 bio->bi_end_io = xfs_buf_bio_end_io;
1245 bio->bi_private = bp; 1245 bio->bi_private = bp;
1246 1246
@@ -1262,7 +1262,7 @@ next_chunk:
1262 total_nr_pages--; 1262 total_nr_pages--;
1263 } 1263 }
1264 1264
1265 if (likely(bio->bi_size)) { 1265 if (likely(bio->bi_iter.bi_size)) {
1266 if (xfs_buf_is_vmapped(bp)) { 1266 if (xfs_buf_is_vmapped(bp)) {
1267 flush_kernel_vmap_range(bp->b_addr, 1267 flush_kernel_vmap_range(bp->b_addr,
1268 xfs_buf_vmap_len(bp)); 1268 xfs_buf_vmap_len(bp));
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-22 18:50:25 -0500