aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Documentation/block/biodoc.txt2
-rw-r--r--Documentation/block/biovecs.txt10
-rw-r--r--Documentation/block/queue-sysfs.txt10
-rw-r--r--arch/m68k/emu/nfblock.c2
-rw-r--r--arch/powerpc/sysdev/axonram.c2
-rw-r--r--arch/xtensa/platforms/iss/simdisk.c12
-rw-r--r--block/bio-integrity.c11
-rw-r--r--block/bio.c212
-rw-r--r--block/blk-core.c36
-rw-r--r--block/blk-flush.c15
-rw-r--r--block/blk-lib.c77
-rw-r--r--block/blk-map.c4
-rw-r--r--block/blk-merge.c169
-rw-r--r--block/blk-mq-sysfs.c25
-rw-r--r--block/blk-mq-tag.c4
-rw-r--r--block/blk-mq-tag.h12
-rw-r--r--block/blk-mq.c26
-rw-r--r--block/blk-settings.c44
-rw-r--r--block/blk-sysfs.c43
-rw-r--r--block/blk.h6
-rw-r--r--block/bounce.c29
-rw-r--r--block/genhd.c9
-rw-r--r--block/partition-generic.c12
-rw-r--r--drivers/block/aoe/aoeblk.c2
-rw-r--r--drivers/block/aoe/aoecmd.c10
-rw-r--r--drivers/block/aoe/aoedev.c2
-rw-r--r--drivers/block/brd.c15
-rw-r--r--drivers/block/drbd/drbd_actlog.c4
-rw-r--r--drivers/block/drbd/drbd_bitmap.c19
-rw-r--r--drivers/block/drbd/drbd_int.h12
-rw-r--r--drivers/block/drbd/drbd_main.c1
-rw-r--r--drivers/block/drbd/drbd_nl.c4
-rw-r--r--drivers/block/drbd/drbd_req.c47
-rw-r--r--drivers/block/drbd/drbd_worker.c44
-rw-r--r--drivers/block/floppy.c7
-rw-r--r--drivers/block/loop.c4
-rw-r--r--drivers/block/nbd.c2
-rw-r--r--drivers/block/null_blk.c2
-rw-r--r--drivers/block/nvme-core.c4
-rw-r--r--drivers/block/pktcdvd.c59
-rw-r--r--drivers/block/ps3vram.c5
-rw-r--r--drivers/block/rbd.c49
-rw-r--r--drivers/block/rsxx/dev.c11
-rw-r--r--drivers/block/skd_main.c2
-rw-r--r--drivers/block/umem.c6
-rw-r--r--drivers/block/xen-blkback/blkback.c4
-rw-r--r--drivers/block/xen-blkfront.c9
-rw-r--r--drivers/block/zram/zram_drv.c9
-rw-r--r--drivers/md/bcache/bcache.h18
-rw-r--r--drivers/md/bcache/btree.c10
-rw-r--r--drivers/md/bcache/closure.h2
-rw-r--r--drivers/md/bcache/io.c101
-rw-r--r--drivers/md/bcache/journal.c12
-rw-r--r--drivers/md/bcache/movinggc.c8
-rw-r--r--drivers/md/bcache/request.c43
-rw-r--r--drivers/md/bcache/super.c48
-rw-r--r--drivers/md/bcache/util.h5
-rw-r--r--drivers/md/bcache/writeback.c14
-rw-r--r--drivers/md/dm-bio-prison.c6
-rw-r--r--drivers/md/dm-bufio.c26
-rw-r--r--drivers/md/dm-cache-target.c45
-rw-r--r--drivers/md/dm-crypt.c26
-rw-r--r--drivers/md/dm-era-target.c15
-rw-r--r--drivers/md/dm-flakey.c18
-rw-r--r--drivers/md/dm-io.c8
-rw-r--r--drivers/md/dm-linear.c16
-rw-r--r--drivers/md/dm-log-writes.c27
-rw-r--r--drivers/md/dm-raid.c19
-rw-r--r--drivers/md/dm-raid1.c24
-rw-r--r--drivers/md/dm-snap.c21
-rw-r--r--drivers/md/dm-stripe.c23
-rw-r--r--drivers/md/dm-table.c21
-rw-r--r--drivers/md/dm-thin.c72
-rw-r--r--drivers/md/dm-verity.c25
-rw-r--r--drivers/md/dm-zero.c2
-rw-r--r--drivers/md/dm.c137
-rw-r--r--drivers/md/dm.h2
-rw-r--r--drivers/md/faulty.c4
-rw-r--r--drivers/md/linear.c45
-rw-r--r--drivers/md/md.c46
-rw-r--r--drivers/md/md.h12
-rw-r--r--drivers/md/multipath.c33
-rw-r--r--drivers/md/raid0.c58
-rw-r--r--drivers/md/raid0.h2
-rw-r--r--drivers/md/raid1.c115
-rw-r--r--drivers/md/raid10.c182
-rw-r--r--drivers/md/raid5.c147
-rw-r--r--drivers/mmc/card/queue.c2
-rw-r--r--drivers/mtd/mtd_blkdevs.c2
-rw-r--r--drivers/nvdimm/blk.c5
-rw-r--r--drivers/nvdimm/btt.c5
-rw-r--r--drivers/nvdimm/pmem.c2
-rw-r--r--drivers/s390/block/dcssblk.c4
-rw-r--r--drivers/s390/block/xpram.c5
-rw-r--r--drivers/scsi/sd.c4
-rw-r--r--drivers/staging/lustre/lustre/llite/lloop.c5
-rw-r--r--drivers/target/target_core_iblock.c21
-rw-r--r--drivers/target/target_core_pscsi.c6
-rw-r--r--fs/btrfs/check-integrity.c10
-rw-r--r--fs/btrfs/compression.c29
-rw-r--r--fs/btrfs/disk-io.c35
-rw-r--r--fs/btrfs/extent_io.c39
-rw-r--r--fs/btrfs/inode.c53
-rw-r--r--fs/btrfs/raid56.c62
-rw-r--r--fs/btrfs/scrub.c40
-rw-r--r--fs/btrfs/volumes.c95
-rw-r--r--fs/buffer.c13
-rw-r--r--fs/direct-io.c18
-rw-r--r--fs/ext4/page-io.c18
-rw-r--r--fs/ext4/readpage.c8
-rw-r--r--fs/f2fs/data.c12
-rw-r--r--fs/gfs2/lops.c19
-rw-r--r--fs/gfs2/ops_fstype.c6
-rw-r--r--fs/jfs/jfs_logmgr.c22
-rw-r--r--fs/jfs/jfs_metapage.c8
-rw-r--r--fs/logfs/dev_bdev.c16
-rw-r--r--fs/mpage.c8
-rw-r--r--fs/nfs/blocklayout/blocklayout.c14
-rw-r--r--fs/nilfs2/segbuf.c7
-rw-r--r--fs/ocfs2/cluster/heartbeat.c9
-rw-r--r--fs/xfs/xfs_aops.c8
-rw-r--r--fs/xfs/xfs_buf.c7
-rw-r--r--include/linux/bio.h38
-rw-r--r--include/linux/blk-cgroup.h2
-rw-r--r--include/linux/blk_types.h10
-rw-r--r--include/linux/blkdev.h36
-rw-r--r--include/linux/device-mapper.h4
-rw-r--r--include/linux/genhd.h33
-rw-r--r--include/linux/swap.h4
-rw-r--r--kernel/power/swap.c12
-rw-r--r--kernel/trace/blktrace.c10
-rw-r--r--mm/page_io.c20
132 files changed, 1199 insertions, 2130 deletions
diff --git a/Documentation/block/biodoc.txt b/Documentation/block/biodoc.txt
index fd12c0d835fd..5be8a7f4cc7f 100644
--- a/Documentation/block/biodoc.txt
+++ b/Documentation/block/biodoc.txt
@@ -1109,7 +1109,7 @@ it will loop and handle as many sectors (on a bio-segment granularity)
1109as specified. 1109as specified.
1110 1110
1111Now bh->b_end_io is replaced by bio->bi_end_io, but most of the time the 1111Now bh->b_end_io is replaced by bio->bi_end_io, but most of the time the
1112right thing to use is bio_endio(bio, uptodate) instead. 1112right thing to use is bio_endio(bio) instead.
1113 1113
1114If the driver is dropping the io_request_lock from its request_fn strategy, 1114If the driver is dropping the io_request_lock from its request_fn strategy,
1115then it just needs to replace that with q->queue_lock instead. 1115then it just needs to replace that with q->queue_lock instead.
diff --git a/Documentation/block/biovecs.txt b/Documentation/block/biovecs.txt
index 74a32ad52f53..25689584e6e0 100644
--- a/Documentation/block/biovecs.txt
+++ b/Documentation/block/biovecs.txt
@@ -24,7 +24,7 @@ particular, presenting the illusion of partially completed biovecs so that
24normal code doesn't have to deal with bi_bvec_done. 24normal code doesn't have to deal with bi_bvec_done.
25 25
26 * Driver code should no longer refer to biovecs directly; we now have 26 * Driver code should no longer refer to biovecs directly; we now have
27 bio_iovec() and bio_iovec_iter() macros that return literal struct biovecs, 27 bio_iovec() and bio_iter_iovec() macros that return literal struct biovecs,
28 constructed from the raw biovecs but taking into account bi_bvec_done and 28 constructed from the raw biovecs but taking into account bi_bvec_done and
29 bi_size. 29 bi_size.
30 30
@@ -109,3 +109,11 @@ Other implications:
109 over all the biovecs in the new bio - which is silly as it's not needed. 109 over all the biovecs in the new bio - which is silly as it's not needed.
110 110
111 So, don't use bi_vcnt anymore. 111 So, don't use bi_vcnt anymore.
112
113 * The current interface allows the block layer to split bios as needed, so we
114 could eliminate a lot of complexity particularly in stacked drivers. Code
115 that creates bios can then create whatever size bios are convenient, and
116 more importantly stacked drivers don't have to deal with both their own bio
117 size limitations and the limitations of the underlying devices. Thus
118 there's no need to define ->merge_bvec_fn() callbacks for individual block
119 drivers.
diff --git a/Documentation/block/queue-sysfs.txt b/Documentation/block/queue-sysfs.txt
index 3a29f8914df9..e5d914845be6 100644
--- a/Documentation/block/queue-sysfs.txt
+++ b/Documentation/block/queue-sysfs.txt
@@ -20,7 +20,7 @@ This shows the size of internal allocation of the device in bytes, if
20reported by the device. A value of '0' means device does not support 20reported by the device. A value of '0' means device does not support
21the discard functionality. 21the discard functionality.
22 22
23discard_max_bytes (RO) 23discard_max_hw_bytes (RO)
24---------------------- 24----------------------
25Devices that support discard functionality may have internal limits on 25Devices that support discard functionality may have internal limits on
26the number of bytes that can be trimmed or unmapped in a single operation. 26the number of bytes that can be trimmed or unmapped in a single operation.
@@ -29,6 +29,14 @@ number of bytes that can be discarded in a single operation. Discard
29requests issued to the device must not exceed this limit. A discard_max_bytes 29requests issued to the device must not exceed this limit. A discard_max_bytes
30value of 0 means that the device does not support discard functionality. 30value of 0 means that the device does not support discard functionality.
31 31
32discard_max_bytes (RW)
33----------------------
34While discard_max_hw_bytes is the hardware limit for the device, this
35setting is the software limit. Some devices exhibit large latencies when
36large discards are issued, setting this value lower will make Linux issue
37smaller discards and potentially help reduce latencies induced by large
38discard operations.
39
32discard_zeroes_data (RO) 40discard_zeroes_data (RO)
33------------------------ 41------------------------
34When read, this file will show if the discarded block are zeroed by the 42When read, this file will show if the discarded block are zeroed by the
diff --git a/arch/m68k/emu/nfblock.c b/arch/m68k/emu/nfblock.c
index 2d75ae246167..f2a00c591bf7 100644
--- a/arch/m68k/emu/nfblock.c
+++ b/arch/m68k/emu/nfblock.c
@@ -76,7 +76,7 @@ static void nfhd_make_request(struct request_queue *queue, struct bio *bio)
76 bvec_to_phys(&bvec)); 76 bvec_to_phys(&bvec));
77 sec += len; 77 sec += len;
78 } 78 }
79 bio_endio(bio, 0); 79 bio_endio(bio);
80} 80}
81 81
82static int nfhd_getgeo(struct block_device *bdev, struct hd_geometry *geo) 82static int nfhd_getgeo(struct block_device *bdev, struct hd_geometry *geo)
diff --git a/arch/powerpc/sysdev/axonram.c b/arch/powerpc/sysdev/axonram.c
index ee90db17b097..f86250c48b53 100644
--- a/arch/powerpc/sysdev/axonram.c
+++ b/arch/powerpc/sysdev/axonram.c
@@ -132,7 +132,7 @@ axon_ram_make_request(struct request_queue *queue, struct bio *bio)
132 phys_mem += vec.bv_len; 132 phys_mem += vec.bv_len;
133 transfered += vec.bv_len; 133 transfered += vec.bv_len;
134 } 134 }
135 bio_endio(bio, 0); 135 bio_endio(bio);
136} 136}
137 137
138/** 138/**
diff --git a/arch/xtensa/platforms/iss/simdisk.c b/arch/xtensa/platforms/iss/simdisk.c
index 48eebacdf5fe..fa84ca990caa 100644
--- a/arch/xtensa/platforms/iss/simdisk.c
+++ b/arch/xtensa/platforms/iss/simdisk.c
@@ -101,8 +101,9 @@ static void simdisk_transfer(struct simdisk *dev, unsigned long sector,
101 spin_unlock(&dev->lock); 101 spin_unlock(&dev->lock);
102} 102}
103 103
104static int simdisk_xfer_bio(struct simdisk *dev, struct bio *bio) 104static void simdisk_make_request(struct request_queue *q, struct bio *bio)
105{ 105{
106 struct simdisk *dev = q->queuedata;
106 struct bio_vec bvec; 107 struct bio_vec bvec;
107 struct bvec_iter iter; 108 struct bvec_iter iter;
108 sector_t sector = bio->bi_iter.bi_sector; 109 sector_t sector = bio->bi_iter.bi_sector;
@@ -116,17 +117,10 @@ static int simdisk_xfer_bio(struct simdisk *dev, struct bio *bio)
116 sector += len; 117 sector += len;
117 __bio_kunmap_atomic(buffer); 118 __bio_kunmap_atomic(buffer);
118 } 119 }
119 return 0;
120}
121 120
122static void simdisk_make_request(struct request_queue *q, struct bio *bio) 121 bio_endio(bio);
123{
124 struct simdisk *dev = q->queuedata;
125 int status = simdisk_xfer_bio(dev, bio);
126 bio_endio(bio, status);
127} 122}
128 123
129
130static int simdisk_open(struct block_device *bdev, fmode_t mode) 124static int simdisk_open(struct block_device *bdev, fmode_t mode)
131{ 125{
132 struct simdisk *dev = bdev->bd_disk->private_data; 126 struct simdisk *dev = bdev->bd_disk->private_data;
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index 719b7152aed1..4aecca79374a 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -355,13 +355,12 @@ static void bio_integrity_verify_fn(struct work_struct *work)
355 container_of(work, struct bio_integrity_payload, bip_work); 355 container_of(work, struct bio_integrity_payload, bip_work);
356 struct bio *bio = bip->bip_bio; 356 struct bio *bio = bip->bip_bio;
357 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 357 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
358 int error;
359 358
360 error = bio_integrity_process(bio, bi->verify_fn); 359 bio->bi_error = bio_integrity_process(bio, bi->verify_fn);
361 360
362 /* Restore original bio completion handler */ 361 /* Restore original bio completion handler */
363 bio->bi_end_io = bip->bip_end_io; 362 bio->bi_end_io = bip->bip_end_io;
364 bio_endio(bio, error); 363 bio_endio(bio);
365} 364}
366 365
367/** 366/**
@@ -376,7 +375,7 @@ static void bio_integrity_verify_fn(struct work_struct *work)
376 * in process context. This function postpones completion 375 * in process context. This function postpones completion
377 * accordingly. 376 * accordingly.
378 */ 377 */
379void bio_integrity_endio(struct bio *bio, int error) 378void bio_integrity_endio(struct bio *bio)
380{ 379{
381 struct bio_integrity_payload *bip = bio_integrity(bio); 380 struct bio_integrity_payload *bip = bio_integrity(bio);
382 381
@@ -386,9 +385,9 @@ void bio_integrity_endio(struct bio *bio, int error)
386 * integrity metadata. Restore original bio end_io handler 385 * integrity metadata. Restore original bio end_io handler
387 * and run it. 386 * and run it.
388 */ 387 */
389 if (error) { 388 if (bio->bi_error) {
390 bio->bi_end_io = bip->bip_end_io; 389 bio->bi_end_io = bip->bip_end_io;
391 bio_endio(bio, error); 390 bio_endio(bio);
392 391
393 return; 392 return;
394 } 393 }
diff --git a/block/bio.c b/block/bio.c
index d6e5ba3399f0..515b5434fe2d 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -269,7 +269,6 @@ static void bio_free(struct bio *bio)
269void bio_init(struct bio *bio) 269void bio_init(struct bio *bio)
270{ 270{
271 memset(bio, 0, sizeof(*bio)); 271 memset(bio, 0, sizeof(*bio));
272 bio->bi_flags = 1 << BIO_UPTODATE;
273 atomic_set(&bio->__bi_remaining, 1); 272 atomic_set(&bio->__bi_remaining, 1);
274 atomic_set(&bio->__bi_cnt, 1); 273 atomic_set(&bio->__bi_cnt, 1);
275} 274}
@@ -292,14 +291,17 @@ void bio_reset(struct bio *bio)
292 __bio_free(bio); 291 __bio_free(bio);
293 292
294 memset(bio, 0, BIO_RESET_BYTES); 293 memset(bio, 0, BIO_RESET_BYTES);
295 bio->bi_flags = flags | (1 << BIO_UPTODATE); 294 bio->bi_flags = flags;
296 atomic_set(&bio->__bi_remaining, 1); 295 atomic_set(&bio->__bi_remaining, 1);
297} 296}
298EXPORT_SYMBOL(bio_reset); 297EXPORT_SYMBOL(bio_reset);
299 298
300static void bio_chain_endio(struct bio *bio, int error) 299static void bio_chain_endio(struct bio *bio)
301{ 300{
302 bio_endio(bio->bi_private, error); 301 struct bio *parent = bio->bi_private;
302
303 parent->bi_error = bio->bi_error;
304 bio_endio(parent);
303 bio_put(bio); 305 bio_put(bio);
304} 306}
305 307
@@ -309,7 +311,7 @@ static void bio_chain_endio(struct bio *bio, int error)
309 */ 311 */
310static inline void bio_inc_remaining(struct bio *bio) 312static inline void bio_inc_remaining(struct bio *bio)
311{ 313{
312 bio->bi_flags |= (1 << BIO_CHAIN); 314 bio_set_flag(bio, BIO_CHAIN);
313 smp_mb__before_atomic(); 315 smp_mb__before_atomic();
314 atomic_inc(&bio->__bi_remaining); 316 atomic_inc(&bio->__bi_remaining);
315} 317}
@@ -493,7 +495,7 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
493 if (unlikely(!bvl)) 495 if (unlikely(!bvl))
494 goto err_free; 496 goto err_free;
495 497
496 bio->bi_flags |= 1 << BIO_OWNS_VEC; 498 bio_set_flag(bio, BIO_OWNS_VEC);
497 } else if (nr_iovecs) { 499 } else if (nr_iovecs) {
498 bvl = bio->bi_inline_vecs; 500 bvl = bio->bi_inline_vecs;
499 } 501 }
@@ -578,7 +580,7 @@ void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
578 * so we don't set nor calculate new physical/hw segment counts here 580 * so we don't set nor calculate new physical/hw segment counts here
579 */ 581 */
580 bio->bi_bdev = bio_src->bi_bdev; 582 bio->bi_bdev = bio_src->bi_bdev;
581 bio->bi_flags |= 1 << BIO_CLONED; 583 bio_set_flag(bio, BIO_CLONED);
582 bio->bi_rw = bio_src->bi_rw; 584 bio->bi_rw = bio_src->bi_rw;
583 bio->bi_iter = bio_src->bi_iter; 585 bio->bi_iter = bio_src->bi_iter;
584 bio->bi_io_vec = bio_src->bi_io_vec; 586 bio->bi_io_vec = bio_src->bi_io_vec;
@@ -692,31 +694,22 @@ integrity_clone:
692EXPORT_SYMBOL(bio_clone_bioset); 694EXPORT_SYMBOL(bio_clone_bioset);
693 695
694/** 696/**
695 * bio_get_nr_vecs - return approx number of vecs 697 * bio_add_pc_page - attempt to add page to bio
696 * @bdev: I/O target 698 * @q: the target queue
699 * @bio: destination bio
700 * @page: page to add
701 * @len: vec entry length
702 * @offset: vec entry offset
697 * 703 *
698 * Return the approximate number of pages we can send to this target. 704 * Attempt to add a page to the bio_vec maplist. This can fail for a
699 * There's no guarantee that you will be able to fit this number of pages 705 * number of reasons, such as the bio being full or target block device
700 * into a bio, it does not account for dynamic restrictions that vary 706 * limitations. The target block device must allow bio's up to PAGE_SIZE,
701 * on offset. 707 * so it is always possible to add a single page to an empty bio.
708 *
709 * This should only be used by REQ_PC bios.
702 */ 710 */
703int bio_get_nr_vecs(struct block_device *bdev) 711int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page
704{ 712 *page, unsigned int len, unsigned int offset)
705 struct request_queue *q = bdev_get_queue(bdev);
706 int nr_pages;
707
708 nr_pages = min_t(unsigned,
709 queue_max_segments(q),
710 queue_max_sectors(q) / (PAGE_SIZE >> 9) + 1);
711
712 return min_t(unsigned, nr_pages, BIO_MAX_PAGES);
713
714}
715EXPORT_SYMBOL(bio_get_nr_vecs);
716
717static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
718 *page, unsigned int len, unsigned int offset,
719 unsigned int max_sectors)
720{ 713{
721 int retried_segments = 0; 714 int retried_segments = 0;
722 struct bio_vec *bvec; 715 struct bio_vec *bvec;
@@ -727,7 +720,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
727 if (unlikely(bio_flagged(bio, BIO_CLONED))) 720 if (unlikely(bio_flagged(bio, BIO_CLONED)))
728 return 0; 721 return 0;
729 722
730 if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors) 723 if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q))
731 return 0; 724 return 0;
732 725
733 /* 726 /*
@@ -740,28 +733,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
740 733
741 if (page == prev->bv_page && 734 if (page == prev->bv_page &&
742 offset == prev->bv_offset + prev->bv_len) { 735 offset == prev->bv_offset + prev->bv_len) {
743 unsigned int prev_bv_len = prev->bv_len;
744 prev->bv_len += len; 736 prev->bv_len += len;
745
746 if (q->merge_bvec_fn) {
747 struct bvec_merge_data bvm = {
748 /* prev_bvec is already charged in
749 bi_size, discharge it in order to
750 simulate merging updated prev_bvec
751 as new bvec. */
752 .bi_bdev = bio->bi_bdev,
753 .bi_sector = bio->bi_iter.bi_sector,
754 .bi_size = bio->bi_iter.bi_size -
755 prev_bv_len,
756 .bi_rw = bio->bi_rw,
757 };
758
759 if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len) {
760 prev->bv_len -= len;
761 return 0;
762 }
763 }
764
765 bio->bi_iter.bi_size += len; 737 bio->bi_iter.bi_size += len;
766 goto done; 738 goto done;
767 } 739 }
@@ -770,8 +742,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
770 * If the queue doesn't support SG gaps and adding this 742 * If the queue doesn't support SG gaps and adding this
771 * offset would create a gap, disallow it. 743 * offset would create a gap, disallow it.
772 */ 744 */
773 if (q->queue_flags & (1 << QUEUE_FLAG_SG_GAPS) && 745 if (bvec_gap_to_prev(q, prev, offset))
774 bvec_gap_to_prev(prev, offset))
775 return 0; 746 return 0;
776 } 747 }
777 748
@@ -804,30 +775,9 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
804 blk_recount_segments(q, bio); 775 blk_recount_segments(q, bio);
805 } 776 }
806 777
807 /*
808 * if queue has other restrictions (eg varying max sector size
809 * depending on offset), it can specify a merge_bvec_fn in the
810 * queue to get further control
811 */
812 if (q->merge_bvec_fn) {
813 struct bvec_merge_data bvm = {
814 .bi_bdev = bio->bi_bdev,
815 .bi_sector = bio->bi_iter.bi_sector,
816 .bi_size = bio->bi_iter.bi_size - len,
817 .bi_rw = bio->bi_rw,
818 };
819
820 /*
821 * merge_bvec_fn() returns number of bytes it can accept
822 * at this offset
823 */
824 if (q->merge_bvec_fn(q, &bvm, bvec) < bvec->bv_len)
825 goto failed;
826 }
827
828 /* If we may be able to merge these biovecs, force a recount */ 778 /* If we may be able to merge these biovecs, force a recount */
829 if (bio->bi_vcnt > 1 && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec))) 779 if (bio->bi_vcnt > 1 && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
830 bio->bi_flags &= ~(1 << BIO_SEG_VALID); 780 bio_clear_flag(bio, BIO_SEG_VALID);
831 781
832 done: 782 done:
833 return len; 783 return len;
@@ -841,28 +791,6 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
841 blk_recount_segments(q, bio); 791 blk_recount_segments(q, bio);
842 return 0; 792 return 0;
843} 793}
844
845/**
846 * bio_add_pc_page - attempt to add page to bio
847 * @q: the target queue
848 * @bio: destination bio
849 * @page: page to add
850 * @len: vec entry length
851 * @offset: vec entry offset
852 *
853 * Attempt to add a page to the bio_vec maplist. This can fail for a
854 * number of reasons, such as the bio being full or target block device
855 * limitations. The target block device must allow bio's up to PAGE_SIZE,
856 * so it is always possible to add a single page to an empty bio.
857 *
858 * This should only be used by REQ_PC bios.
859 */
860int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page,
861 unsigned int len, unsigned int offset)
862{
863 return __bio_add_page(q, bio, page, len, offset,
864 queue_max_hw_sectors(q));
865}
866EXPORT_SYMBOL(bio_add_pc_page); 794EXPORT_SYMBOL(bio_add_pc_page);
867 795
868/** 796/**
@@ -872,22 +800,47 @@ EXPORT_SYMBOL(bio_add_pc_page);
872 * @len: vec entry length 800 * @len: vec entry length
873 * @offset: vec entry offset 801 * @offset: vec entry offset
874 * 802 *
875 * Attempt to add a page to the bio_vec maplist. This can fail for a 803 * Attempt to add a page to the bio_vec maplist. This will only fail
876 * number of reasons, such as the bio being full or target block device 804 * if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
877 * limitations. The target block device must allow bio's up to PAGE_SIZE,
878 * so it is always possible to add a single page to an empty bio.
879 */ 805 */
880int bio_add_page(struct bio *bio, struct page *page, unsigned int len, 806int bio_add_page(struct bio *bio, struct page *page,
881 unsigned int offset) 807 unsigned int len, unsigned int offset)
882{ 808{
883 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 809 struct bio_vec *bv;
884 unsigned int max_sectors; 810
811 /*
812 * cloned bio must not modify vec list
813 */
814 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
815 return 0;
885 816
886 max_sectors = blk_max_size_offset(q, bio->bi_iter.bi_sector); 817 /*
887 if ((max_sectors < (len >> 9)) && !bio->bi_iter.bi_size) 818 * For filesystems with a blocksize smaller than the pagesize
888 max_sectors = len >> 9; 819 * we will often be called with the same page as last time and
820 * a consecutive offset. Optimize this special case.
821 */
822 if (bio->bi_vcnt > 0) {
823 bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
889 824
890 return __bio_add_page(q, bio, page, len, offset, max_sectors); 825 if (page == bv->bv_page &&
826 offset == bv->bv_offset + bv->bv_len) {
827 bv->bv_len += len;
828 goto done;
829 }
830 }
831
832 if (bio->bi_vcnt >= bio->bi_max_vecs)
833 return 0;
834
835 bv = &bio->bi_io_vec[bio->bi_vcnt];
836 bv->bv_page = page;
837 bv->bv_len = len;
838 bv->bv_offset = offset;
839
840 bio->bi_vcnt++;
841done:
842 bio->bi_iter.bi_size += len;
843 return len;
891} 844}
892EXPORT_SYMBOL(bio_add_page); 845EXPORT_SYMBOL(bio_add_page);
893 846
@@ -896,11 +849,11 @@ struct submit_bio_ret {
896 int error; 849 int error;
897}; 850};
898 851
899static void submit_bio_wait_endio(struct bio *bio, int error) 852static void submit_bio_wait_endio(struct bio *bio)
900{ 853{
901 struct submit_bio_ret *ret = bio->bi_private; 854 struct submit_bio_ret *ret = bio->bi_private;
902 855
903 ret->error = error; 856 ret->error = bio->bi_error;
904 complete(&ret->event); 857 complete(&ret->event);
905} 858}
906 859
@@ -1388,7 +1341,7 @@ struct bio *bio_map_user_iov(struct request_queue *q,
1388 if (iter->type & WRITE) 1341 if (iter->type & WRITE)
1389 bio->bi_rw |= REQ_WRITE; 1342 bio->bi_rw |= REQ_WRITE;
1390 1343
1391 bio->bi_flags |= (1 << BIO_USER_MAPPED); 1344 bio_set_flag(bio, BIO_USER_MAPPED);
1392 1345
1393 /* 1346 /*
1394 * subtle -- if __bio_map_user() ended up bouncing a bio, 1347 * subtle -- if __bio_map_user() ended up bouncing a bio,
@@ -1445,7 +1398,7 @@ void bio_unmap_user(struct bio *bio)
1445} 1398}
1446EXPORT_SYMBOL(bio_unmap_user); 1399EXPORT_SYMBOL(bio_unmap_user);
1447 1400
1448static void bio_map_kern_endio(struct bio *bio, int err) 1401static void bio_map_kern_endio(struct bio *bio)
1449{ 1402{
1450 bio_put(bio); 1403 bio_put(bio);
1451} 1404}
@@ -1501,13 +1454,13 @@ struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
1501} 1454}
1502EXPORT_SYMBOL(bio_map_kern); 1455EXPORT_SYMBOL(bio_map_kern);
1503 1456
1504static void bio_copy_kern_endio(struct bio *bio, int err) 1457static void bio_copy_kern_endio(struct bio *bio)
1505{ 1458{
1506 bio_free_pages(bio); 1459 bio_free_pages(bio);
1507 bio_put(bio); 1460 bio_put(bio);
1508} 1461}
1509 1462
1510static void bio_copy_kern_endio_read(struct bio *bio, int err) 1463static void bio_copy_kern_endio_read(struct bio *bio)
1511{ 1464{
1512 char *p = bio->bi_private; 1465 char *p = bio->bi_private;
1513 struct bio_vec *bvec; 1466 struct bio_vec *bvec;
@@ -1518,7 +1471,7 @@ static void bio_copy_kern_endio_read(struct bio *bio, int err)
1518 p += bvec->bv_len; 1471 p += bvec->bv_len;
1519 } 1472 }
1520 1473
1521 bio_copy_kern_endio(bio, err); 1474 bio_copy_kern_endio(bio);
1522} 1475}
1523 1476
1524/** 1477/**
@@ -1768,7 +1721,7 @@ static inline bool bio_remaining_done(struct bio *bio)
1768 BUG_ON(atomic_read(&bio->__bi_remaining) <= 0); 1721 BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
1769 1722
1770 if (atomic_dec_and_test(&bio->__bi_remaining)) { 1723 if (atomic_dec_and_test(&bio->__bi_remaining)) {
1771 clear_bit(BIO_CHAIN, &bio->bi_flags); 1724 bio_clear_flag(bio, BIO_CHAIN);
1772 return true; 1725 return true;
1773 } 1726 }
1774 1727
@@ -1778,25 +1731,15 @@ static inline bool bio_remaining_done(struct bio *bio)
1778/** 1731/**
1779 * bio_endio - end I/O on a bio 1732 * bio_endio - end I/O on a bio
1780 * @bio: bio 1733 * @bio: bio
1781 * @error: error, if any
1782 * 1734 *
1783 * Description: 1735 * Description:
1784 * bio_endio() will end I/O on the whole bio. bio_endio() is the 1736 * bio_endio() will end I/O on the whole bio. bio_endio() is the preferred
1785 * preferred way to end I/O on a bio, it takes care of clearing 1737 * way to end I/O on a bio. No one should call bi_end_io() directly on a
1786 * BIO_UPTODATE on error. @error is 0 on success, and and one of the 1738 * bio unless they own it and thus know that it has an end_io function.
1787 * established -Exxxx (-EIO, for instance) error values in case
1788 * something went wrong. No one should call bi_end_io() directly on a
1789 * bio unless they own it and thus know that it has an end_io
1790 * function.
1791 **/ 1739 **/
1792void bio_endio(struct bio *bio, int error) 1740void bio_endio(struct bio *bio)
1793{ 1741{
1794 while (bio) { 1742 while (bio) {
1795 if (error)
1796 clear_bit(BIO_UPTODATE, &bio->bi_flags);
1797 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1798 error = -EIO;
1799
1800 if (unlikely(!bio_remaining_done(bio))) 1743 if (unlikely(!bio_remaining_done(bio)))
1801 break; 1744 break;
1802 1745
@@ -1810,11 +1753,12 @@ void bio_endio(struct bio *bio, int error)
1810 */ 1753 */
1811 if (bio->bi_end_io == bio_chain_endio) { 1754 if (bio->bi_end_io == bio_chain_endio) {
1812 struct bio *parent = bio->bi_private; 1755 struct bio *parent = bio->bi_private;
1756 parent->bi_error = bio->bi_error;
1813 bio_put(bio); 1757 bio_put(bio);
1814 bio = parent; 1758 bio = parent;
1815 } else { 1759 } else {
1816 if (bio->bi_end_io) 1760 if (bio->bi_end_io)
1817 bio->bi_end_io(bio, error); 1761 bio->bi_end_io(bio);
1818 bio = NULL; 1762 bio = NULL;
1819 } 1763 }
1820 } 1764 }
@@ -1882,7 +1826,7 @@ void bio_trim(struct bio *bio, int offset, int size)
1882 if (offset == 0 && size == bio->bi_iter.bi_size) 1826 if (offset == 0 && size == bio->bi_iter.bi_size)
1883 return; 1827 return;
1884 1828
1885 clear_bit(BIO_SEG_VALID, &bio->bi_flags); 1829 bio_clear_flag(bio, BIO_SEG_VALID);
1886 1830
1887 bio_advance(bio, offset << 9); 1831 bio_advance(bio, offset << 9);
1888 1832
diff --git a/block/blk-core.c b/block/blk-core.c
index 627ed0c593fb..60912e983f16 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -143,18 +143,16 @@ static void req_bio_endio(struct request *rq, struct bio *bio,
143 unsigned int nbytes, int error) 143 unsigned int nbytes, int error)
144{ 144{
145 if (error) 145 if (error)
146 clear_bit(BIO_UPTODATE, &bio->bi_flags); 146 bio->bi_error = error;
147 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
148 error = -EIO;
149 147
150 if (unlikely(rq->cmd_flags & REQ_QUIET)) 148 if (unlikely(rq->cmd_flags & REQ_QUIET))
151 set_bit(BIO_QUIET, &bio->bi_flags); 149 bio_set_flag(bio, BIO_QUIET);
152 150
153 bio_advance(bio, nbytes); 151 bio_advance(bio, nbytes);
154 152
155 /* don't actually finish bio if it's part of flush sequence */ 153 /* don't actually finish bio if it's part of flush sequence */
156 if (bio->bi_iter.bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ)) 154 if (bio->bi_iter.bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ))
157 bio_endio(bio, error); 155 bio_endio(bio);
158} 156}
159 157
160void blk_dump_rq_flags(struct request *rq, char *msg) 158void blk_dump_rq_flags(struct request *rq, char *msg)
@@ -645,6 +643,10 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
645 if (q->id < 0) 643 if (q->id < 0)
646 goto fail_q; 644 goto fail_q;
647 645
646 q->bio_split = bioset_create(BIO_POOL_SIZE, 0);
647 if (!q->bio_split)
648 goto fail_id;
649
648 q->backing_dev_info.ra_pages = 650 q->backing_dev_info.ra_pages =
649 (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; 651 (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
650 q->backing_dev_info.capabilities = BDI_CAP_CGROUP_WRITEBACK; 652 q->backing_dev_info.capabilities = BDI_CAP_CGROUP_WRITEBACK;
@@ -653,7 +655,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
653 655
654 err = bdi_init(&q->backing_dev_info); 656 err = bdi_init(&q->backing_dev_info);
655 if (err) 657 if (err)
656 goto fail_id; 658 goto fail_split;
657 659
658 setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, 660 setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
659 laptop_mode_timer_fn, (unsigned long) q); 661 laptop_mode_timer_fn, (unsigned long) q);
@@ -695,6 +697,8 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
695 697
696fail_bdi: 698fail_bdi:
697 bdi_destroy(&q->backing_dev_info); 699 bdi_destroy(&q->backing_dev_info);
700fail_split:
701 bioset_free(q->bio_split);
698fail_id: 702fail_id:
699 ida_simple_remove(&blk_queue_ida, q->id); 703 ida_simple_remove(&blk_queue_ida, q->id);
700fail_q: 704fail_q:
@@ -1612,6 +1616,8 @@ static void blk_queue_bio(struct request_queue *q, struct bio *bio)
1612 struct request *req; 1616 struct request *req;
1613 unsigned int request_count = 0; 1617 unsigned int request_count = 0;
1614 1618
1619 blk_queue_split(q, &bio, q->bio_split);
1620
1615 /* 1621 /*
1616 * low level driver can indicate that it wants pages above a 1622 * low level driver can indicate that it wants pages above a
1617 * certain limit bounced to low memory (ie for highmem, or even 1623 * certain limit bounced to low memory (ie for highmem, or even
@@ -1620,7 +1626,8 @@ static void blk_queue_bio(struct request_queue *q, struct bio *bio)
1620 blk_queue_bounce(q, &bio); 1626 blk_queue_bounce(q, &bio);
1621 1627
1622 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { 1628 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1623 bio_endio(bio, -EIO); 1629 bio->bi_error = -EIO;
1630 bio_endio(bio);
1624 return; 1631 return;
1625 } 1632 }
1626 1633
@@ -1673,7 +1680,8 @@ get_rq:
1673 */ 1680 */
1674 req = get_request(q, rw_flags, bio, GFP_NOIO); 1681 req = get_request(q, rw_flags, bio, GFP_NOIO);
1675 if (IS_ERR(req)) { 1682 if (IS_ERR(req)) {
1676 bio_endio(bio, PTR_ERR(req)); /* @q is dead */ 1683 bio->bi_error = PTR_ERR(req);
1684 bio_endio(bio);
1677 goto out_unlock; 1685 goto out_unlock;
1678 } 1686 }
1679 1687
@@ -1832,15 +1840,6 @@ generic_make_request_checks(struct bio *bio)
1832 goto end_io; 1840 goto end_io;
1833 } 1841 }
1834 1842
1835 if (likely(bio_is_rw(bio) &&
1836 nr_sectors > queue_max_hw_sectors(q))) {
1837 printk(KERN_ERR "bio too big device %s (%u > %u)\n",
1838 bdevname(bio->bi_bdev, b),
1839 bio_sectors(bio),
1840 queue_max_hw_sectors(q));
1841 goto end_io;
1842 }
1843
1844 part = bio->bi_bdev->bd_part; 1843 part = bio->bi_bdev->bd_part;
1845 if (should_fail_request(part, bio->bi_iter.bi_size) || 1844 if (should_fail_request(part, bio->bi_iter.bi_size) ||
1846 should_fail_request(&part_to_disk(part)->part0, 1845 should_fail_request(&part_to_disk(part)->part0,
@@ -1896,7 +1895,8 @@ generic_make_request_checks(struct bio *bio)
1896 return true; 1895 return true;
1897 1896
1898end_io: 1897end_io:
1899 bio_endio(bio, err); 1898 bio->bi_error = err;
1899 bio_endio(bio);
1900 return false; 1900 return false;
1901} 1901}
1902 1902
diff --git a/block/blk-flush.c b/block/blk-flush.c
index 20badd7b9d1b..9c423e53324a 100644
--- a/block/blk-flush.c
+++ b/block/blk-flush.c
@@ -73,6 +73,7 @@
73 73
74#include "blk.h" 74#include "blk.h"
75#include "blk-mq.h" 75#include "blk-mq.h"
76#include "blk-mq-tag.h"
76 77
77/* FLUSH/FUA sequences */ 78/* FLUSH/FUA sequences */
78enum { 79enum {
@@ -226,7 +227,12 @@ static void flush_end_io(struct request *flush_rq, int error)
226 struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx); 227 struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx);
227 228
228 if (q->mq_ops) { 229 if (q->mq_ops) {
230 struct blk_mq_hw_ctx *hctx;
231
232 /* release the tag's ownership to the req cloned from */
229 spin_lock_irqsave(&fq->mq_flush_lock, flags); 233 spin_lock_irqsave(&fq->mq_flush_lock, flags);
234 hctx = q->mq_ops->map_queue(q, flush_rq->mq_ctx->cpu);
235 blk_mq_tag_set_rq(hctx, flush_rq->tag, fq->orig_rq);
230 flush_rq->tag = -1; 236 flush_rq->tag = -1;
231 } 237 }
232 238
@@ -308,11 +314,18 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq)
308 314
309 /* 315 /*
310 * Borrow tag from the first request since they can't 316 * Borrow tag from the first request since they can't
311 * be in flight at the same time. 317 * be in flight at the same time. And acquire the tag's
318 * ownership for flush req.
312 */ 319 */
313 if (q->mq_ops) { 320 if (q->mq_ops) {
321 struct blk_mq_hw_ctx *hctx;
322
314 flush_rq->mq_ctx = first_rq->mq_ctx; 323 flush_rq->mq_ctx = first_rq->mq_ctx;
315 flush_rq->tag = first_rq->tag; 324 flush_rq->tag = first_rq->tag;
325 fq->orig_rq = first_rq;
326
327 hctx = q->mq_ops->map_queue(q, first_rq->mq_ctx->cpu);
328 blk_mq_tag_set_rq(hctx, first_rq->tag, flush_rq);
316 } 329 }
317 330
318 flush_rq->cmd_type = REQ_TYPE_FS; 331 flush_rq->cmd_type = REQ_TYPE_FS;
diff --git a/block/blk-lib.c b/block/blk-lib.c
index 7688ee3f5d72..bd40292e5009 100644
--- a/block/blk-lib.c
+++ b/block/blk-lib.c
@@ -11,21 +11,28 @@
11 11
12struct bio_batch { 12struct bio_batch {
13 atomic_t done; 13 atomic_t done;
14 unsigned long flags; 14 int error;
15 struct completion *wait; 15 struct completion *wait;
16}; 16};
17 17
18static void bio_batch_end_io(struct bio *bio, int err) 18static void bio_batch_end_io(struct bio *bio)
19{ 19{
20 struct bio_batch *bb = bio->bi_private; 20 struct bio_batch *bb = bio->bi_private;
21 21
22 if (err && (err != -EOPNOTSUPP)) 22 if (bio->bi_error && bio->bi_error != -EOPNOTSUPP)
23 clear_bit(BIO_UPTODATE, &bb->flags); 23 bb->error = bio->bi_error;
24 if (atomic_dec_and_test(&bb->done)) 24 if (atomic_dec_and_test(&bb->done))
25 complete(bb->wait); 25 complete(bb->wait);
26 bio_put(bio); 26 bio_put(bio);
27} 27}
28 28
29/*
30 * Ensure that max discard sectors doesn't overflow bi_size and hopefully
31 * it is of the proper granularity as long as the granularity is a power
32 * of two.
33 */
34#define MAX_BIO_SECTORS ((1U << 31) >> 9)
35
29/** 36/**
30 * blkdev_issue_discard - queue a discard 37 * blkdev_issue_discard - queue a discard
31 * @bdev: blockdev to issue discard for 38 * @bdev: blockdev to issue discard for
@@ -43,8 +50,6 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
43 DECLARE_COMPLETION_ONSTACK(wait); 50 DECLARE_COMPLETION_ONSTACK(wait);
44 struct request_queue *q = bdev_get_queue(bdev); 51 struct request_queue *q = bdev_get_queue(bdev);
45 int type = REQ_WRITE | REQ_DISCARD; 52 int type = REQ_WRITE | REQ_DISCARD;
46 unsigned int max_discard_sectors, granularity;
47 int alignment;
48 struct bio_batch bb; 53 struct bio_batch bb;
49 struct bio *bio; 54 struct bio *bio;
50 int ret = 0; 55 int ret = 0;
@@ -56,21 +61,6 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
56 if (!blk_queue_discard(q)) 61 if (!blk_queue_discard(q))
57 return -EOPNOTSUPP; 62 return -EOPNOTSUPP;
58 63
59 /* Zero-sector (unknown) and one-sector granularities are the same. */
60 granularity = max(q->limits.discard_granularity >> 9, 1U);
61 alignment = (bdev_discard_alignment(bdev) >> 9) % granularity;
62
63 /*
64 * Ensure that max_discard_sectors is of the proper
65 * granularity, so that requests stay aligned after a split.
66 */
67 max_discard_sectors = min(q->limits.max_discard_sectors, UINT_MAX >> 9);
68 max_discard_sectors -= max_discard_sectors % granularity;
69 if (unlikely(!max_discard_sectors)) {
70 /* Avoid infinite loop below. Being cautious never hurts. */
71 return -EOPNOTSUPP;
72 }
73
74 if (flags & BLKDEV_DISCARD_SECURE) { 64 if (flags & BLKDEV_DISCARD_SECURE) {
75 if (!blk_queue_secdiscard(q)) 65 if (!blk_queue_secdiscard(q))
76 return -EOPNOTSUPP; 66 return -EOPNOTSUPP;
@@ -78,13 +68,13 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
78 } 68 }
79 69
80 atomic_set(&bb.done, 1); 70 atomic_set(&bb.done, 1);
81 bb.flags = 1 << BIO_UPTODATE; 71 bb.error = 0;
82 bb.wait = &wait; 72 bb.wait = &wait;
83 73
84 blk_start_plug(&plug); 74 blk_start_plug(&plug);
85 while (nr_sects) { 75 while (nr_sects) {
86 unsigned int req_sects; 76 unsigned int req_sects;
87 sector_t end_sect, tmp; 77 sector_t end_sect;
88 78
89 bio = bio_alloc(gfp_mask, 1); 79 bio = bio_alloc(gfp_mask, 1);
90 if (!bio) { 80 if (!bio) {
@@ -92,21 +82,8 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
92 break; 82 break;
93 } 83 }
94 84
95 req_sects = min_t(sector_t, nr_sects, max_discard_sectors); 85 req_sects = min_t(sector_t, nr_sects, MAX_BIO_SECTORS);
96
97 /*
98 * If splitting a request, and the next starting sector would be
99 * misaligned, stop the discard at the previous aligned sector.
100 */
101 end_sect = sector + req_sects; 86 end_sect = sector + req_sects;
102 tmp = end_sect;
103 if (req_sects < nr_sects &&
104 sector_div(tmp, granularity) != alignment) {
105 end_sect = end_sect - alignment;
106 sector_div(end_sect, granularity);
107 end_sect = end_sect * granularity + alignment;
108 req_sects = end_sect - sector;
109 }
110 87
111 bio->bi_iter.bi_sector = sector; 88 bio->bi_iter.bi_sector = sector;
112 bio->bi_end_io = bio_batch_end_io; 89 bio->bi_end_io = bio_batch_end_io;
@@ -134,9 +111,8 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
134 if (!atomic_dec_and_test(&bb.done)) 111 if (!atomic_dec_and_test(&bb.done))
135 wait_for_completion_io(&wait); 112 wait_for_completion_io(&wait);
136 113
137 if (!test_bit(BIO_UPTODATE, &bb.flags)) 114 if (bb.error)
138 ret = -EIO; 115 return bb.error;
139
140 return ret; 116 return ret;
141} 117}
142EXPORT_SYMBOL(blkdev_issue_discard); 118EXPORT_SYMBOL(blkdev_issue_discard);
@@ -166,13 +142,11 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
166 if (!q) 142 if (!q)
167 return -ENXIO; 143 return -ENXIO;
168 144
169 max_write_same_sectors = q->limits.max_write_same_sectors; 145 /* Ensure that max_write_same_sectors doesn't overflow bi_size */
170 146 max_write_same_sectors = UINT_MAX >> 9;
171 if (max_write_same_sectors == 0)
172 return -EOPNOTSUPP;
173 147
174 atomic_set(&bb.done, 1); 148 atomic_set(&bb.done, 1);
175 bb.flags = 1 << BIO_UPTODATE; 149 bb.error = 0;
176 bb.wait = &wait; 150 bb.wait = &wait;
177 151
178 while (nr_sects) { 152 while (nr_sects) {
@@ -208,9 +182,8 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
208 if (!atomic_dec_and_test(&bb.done)) 182 if (!atomic_dec_and_test(&bb.done))
209 wait_for_completion_io(&wait); 183 wait_for_completion_io(&wait);
210 184
211 if (!test_bit(BIO_UPTODATE, &bb.flags)) 185 if (bb.error)
212 ret = -ENOTSUPP; 186 return bb.error;
213
214 return ret; 187 return ret;
215} 188}
216EXPORT_SYMBOL(blkdev_issue_write_same); 189EXPORT_SYMBOL(blkdev_issue_write_same);
@@ -236,7 +209,7 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
236 DECLARE_COMPLETION_ONSTACK(wait); 209 DECLARE_COMPLETION_ONSTACK(wait);
237 210
238 atomic_set(&bb.done, 1); 211 atomic_set(&bb.done, 1);
239 bb.flags = 1 << BIO_UPTODATE; 212 bb.error = 0;
240 bb.wait = &wait; 213 bb.wait = &wait;
241 214
242 ret = 0; 215 ret = 0;
@@ -270,10 +243,8 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
270 if (!atomic_dec_and_test(&bb.done)) 243 if (!atomic_dec_and_test(&bb.done))
271 wait_for_completion_io(&wait); 244 wait_for_completion_io(&wait);
272 245
273 if (!test_bit(BIO_UPTODATE, &bb.flags)) 246 if (bb.error)
274 /* One of bios in the batch was completed with error.*/ 247 return bb.error;
275 ret = -EIO;
276
277 return ret; 248 return ret;
278} 249}
279 250
diff --git a/block/blk-map.c b/block/blk-map.c
index da310a105429..233841644c9d 100644
--- a/block/blk-map.c
+++ b/block/blk-map.c
@@ -94,7 +94,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
94 return PTR_ERR(bio); 94 return PTR_ERR(bio);
95 95
96 if (map_data && map_data->null_mapped) 96 if (map_data && map_data->null_mapped)
97 bio->bi_flags |= (1 << BIO_NULL_MAPPED); 97 bio_set_flag(bio, BIO_NULL_MAPPED);
98 98
99 if (bio->bi_iter.bi_size != iter->count) { 99 if (bio->bi_iter.bi_size != iter->count) {
100 /* 100 /*
@@ -103,7 +103,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
103 * normal IO completion path 103 * normal IO completion path
104 */ 104 */
105 bio_get(bio); 105 bio_get(bio);
106 bio_endio(bio, 0); 106 bio_endio(bio);
107 __blk_rq_unmap_user(bio); 107 __blk_rq_unmap_user(bio);
108 return -EINVAL; 108 return -EINVAL;
109 } 109 }
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 30a0d9f89017..b2625271a572 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -9,12 +9,146 @@
9 9
10#include "blk.h" 10#include "blk.h"
11 11
12static struct bio *blk_bio_discard_split(struct request_queue *q,
13 struct bio *bio,
14 struct bio_set *bs)
15{
16 unsigned int max_discard_sectors, granularity;
17 int alignment;
18 sector_t tmp;
19 unsigned split_sectors;
20
21 /* Zero-sector (unknown) and one-sector granularities are the same. */
22 granularity = max(q->limits.discard_granularity >> 9, 1U);
23
24 max_discard_sectors = min(q->limits.max_discard_sectors, UINT_MAX >> 9);
25 max_discard_sectors -= max_discard_sectors % granularity;
26
27 if (unlikely(!max_discard_sectors)) {
28 /* XXX: warn */
29 return NULL;
30 }
31
32 if (bio_sectors(bio) <= max_discard_sectors)
33 return NULL;
34
35 split_sectors = max_discard_sectors;
36
37 /*
38 * If the next starting sector would be misaligned, stop the discard at
39 * the previous aligned sector.
40 */
41 alignment = (q->limits.discard_alignment >> 9) % granularity;
42
43 tmp = bio->bi_iter.bi_sector + split_sectors - alignment;
44 tmp = sector_div(tmp, granularity);
45
46 if (split_sectors > tmp)
47 split_sectors -= tmp;
48
49 return bio_split(bio, split_sectors, GFP_NOIO, bs);
50}
51
52static struct bio *blk_bio_write_same_split(struct request_queue *q,
53 struct bio *bio,
54 struct bio_set *bs)
55{
56 if (!q->limits.max_write_same_sectors)
57 return NULL;
58
59 if (bio_sectors(bio) <= q->limits.max_write_same_sectors)
60 return NULL;
61
62 return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs);
63}
64
65static struct bio *blk_bio_segment_split(struct request_queue *q,
66 struct bio *bio,
67 struct bio_set *bs)
68{
69 struct bio *split;
70 struct bio_vec bv, bvprv;
71 struct bvec_iter iter;
72 unsigned seg_size = 0, nsegs = 0, sectors = 0;
73 int prev = 0;
74
75 bio_for_each_segment(bv, bio, iter) {
76 sectors += bv.bv_len >> 9;
77
78 if (sectors > queue_max_sectors(q))
79 goto split;
80
81 /*
82 * If the queue doesn't support SG gaps and adding this
83 * offset would create a gap, disallow it.
84 */
85 if (prev && bvec_gap_to_prev(q, &bvprv, bv.bv_offset))
86 goto split;
87
88 if (prev && blk_queue_cluster(q)) {
89 if (seg_size + bv.bv_len > queue_max_segment_size(q))
90 goto new_segment;
91 if (!BIOVEC_PHYS_MERGEABLE(&bvprv, &bv))
92 goto new_segment;
93 if (!BIOVEC_SEG_BOUNDARY(q, &bvprv, &bv))
94 goto new_segment;
95
96 seg_size += bv.bv_len;
97 bvprv = bv;
98 prev = 1;
99 continue;
100 }
101new_segment:
102 if (nsegs == queue_max_segments(q))
103 goto split;
104
105 nsegs++;
106 bvprv = bv;
107 prev = 1;
108 seg_size = bv.bv_len;
109 }
110
111 return NULL;
112split:
113 split = bio_clone_bioset(bio, GFP_NOIO, bs);
114
115 split->bi_iter.bi_size -= iter.bi_size;
116 bio->bi_iter = iter;
117
118 if (bio_integrity(bio)) {
119 bio_integrity_advance(bio, split->bi_iter.bi_size);
120 bio_integrity_trim(split, 0, bio_sectors(split));
121 }
122
123 return split;
124}
125
126void blk_queue_split(struct request_queue *q, struct bio **bio,
127 struct bio_set *bs)
128{
129 struct bio *split;
130
131 if ((*bio)->bi_rw & REQ_DISCARD)
132 split = blk_bio_discard_split(q, *bio, bs);
133 else if ((*bio)->bi_rw & REQ_WRITE_SAME)
134 split = blk_bio_write_same_split(q, *bio, bs);
135 else
136 split = blk_bio_segment_split(q, *bio, q->bio_split);
137
138 if (split) {
139 bio_chain(split, *bio);
140 generic_make_request(*bio);
141 *bio = split;
142 }
143}
144EXPORT_SYMBOL(blk_queue_split);
145
12static unsigned int __blk_recalc_rq_segments(struct request_queue *q, 146static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
13 struct bio *bio, 147 struct bio *bio,
14 bool no_sg_merge) 148 bool no_sg_merge)
15{ 149{
16 struct bio_vec bv, bvprv = { NULL }; 150 struct bio_vec bv, bvprv = { NULL };
17 int cluster, high, highprv = 1; 151 int cluster, prev = 0;
18 unsigned int seg_size, nr_phys_segs; 152 unsigned int seg_size, nr_phys_segs;
19 struct bio *fbio, *bbio; 153 struct bio *fbio, *bbio;
20 struct bvec_iter iter; 154 struct bvec_iter iter;
@@ -36,7 +170,6 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
36 cluster = blk_queue_cluster(q); 170 cluster = blk_queue_cluster(q);
37 seg_size = 0; 171 seg_size = 0;
38 nr_phys_segs = 0; 172 nr_phys_segs = 0;
39 high = 0;
40 for_each_bio(bio) { 173 for_each_bio(bio) {
41 bio_for_each_segment(bv, bio, iter) { 174 bio_for_each_segment(bv, bio, iter) {
42 /* 175 /*
@@ -46,13 +179,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
46 if (no_sg_merge) 179 if (no_sg_merge)
47 goto new_segment; 180 goto new_segment;
48 181
49 /* 182 if (prev && cluster) {
50 * the trick here is making sure that a high page is
51 * never considered part of another segment, since
52 * that might change with the bounce page.
53 */
54 high = page_to_pfn(bv.bv_page) > queue_bounce_pfn(q);
55 if (!high && !highprv && cluster) {
56 if (seg_size + bv.bv_len 183 if (seg_size + bv.bv_len
57 > queue_max_segment_size(q)) 184 > queue_max_segment_size(q))
58 goto new_segment; 185 goto new_segment;
@@ -72,8 +199,8 @@ new_segment:
72 199
73 nr_phys_segs++; 200 nr_phys_segs++;
74 bvprv = bv; 201 bvprv = bv;
202 prev = 1;
75 seg_size = bv.bv_len; 203 seg_size = bv.bv_len;
76 highprv = high;
77 } 204 }
78 bbio = bio; 205 bbio = bio;
79 } 206 }
@@ -116,7 +243,7 @@ void blk_recount_segments(struct request_queue *q, struct bio *bio)
116 bio->bi_next = nxt; 243 bio->bi_next = nxt;
117 } 244 }
118 245
119 bio->bi_flags |= (1 << BIO_SEG_VALID); 246 bio_set_flag(bio, BIO_SEG_VALID);
120} 247}
121EXPORT_SYMBOL(blk_recount_segments); 248EXPORT_SYMBOL(blk_recount_segments);
122 249
@@ -356,12 +483,12 @@ static bool req_no_special_merge(struct request *req)
356 return !q->mq_ops && req->special; 483 return !q->mq_ops && req->special;
357} 484}
358 485
359static int req_gap_to_prev(struct request *req, struct request *next) 486static int req_gap_to_prev(struct request *req, struct bio *next)
360{ 487{
361 struct bio *prev = req->biotail; 488 struct bio *prev = req->biotail;
362 489
363 return bvec_gap_to_prev(&prev->bi_io_vec[prev->bi_vcnt - 1], 490 return bvec_gap_to_prev(req->q, &prev->bi_io_vec[prev->bi_vcnt - 1],
364 next->bio->bi_io_vec[0].bv_offset); 491 next->bi_io_vec[0].bv_offset);
365} 492}
366 493
367static int ll_merge_requests_fn(struct request_queue *q, struct request *req, 494static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
@@ -378,8 +505,7 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
378 if (req_no_special_merge(req) || req_no_special_merge(next)) 505 if (req_no_special_merge(req) || req_no_special_merge(next))
379 return 0; 506 return 0;
380 507
381 if (test_bit(QUEUE_FLAG_SG_GAPS, &q->queue_flags) && 508 if (req_gap_to_prev(req, next->bio))
382 req_gap_to_prev(req, next))
383 return 0; 509 return 0;
384 510
385 /* 511 /*
@@ -564,8 +690,6 @@ int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
564 690
565bool blk_rq_merge_ok(struct request *rq, struct bio *bio) 691bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
566{ 692{
567 struct request_queue *q = rq->q;
568
569 if (!rq_mergeable(rq) || !bio_mergeable(bio)) 693 if (!rq_mergeable(rq) || !bio_mergeable(bio))
570 return false; 694 return false;
571 695
@@ -590,13 +714,8 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
590 return false; 714 return false;
591 715
592 /* Only check gaps if the bio carries data */ 716 /* Only check gaps if the bio carries data */
593 if (q->queue_flags & (1 << QUEUE_FLAG_SG_GAPS) && bio_has_data(bio)) { 717 if (bio_has_data(bio) && req_gap_to_prev(rq, bio))
594 struct bio_vec *bprev; 718 return false;
595
596 bprev = &rq->biotail->bi_io_vec[rq->biotail->bi_vcnt - 1];
597 if (bvec_gap_to_prev(bprev, bio->bi_io_vec[0].bv_offset))
598 return false;
599 }
600 719
601 return true; 720 return true;
602} 721}
diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c
index b79685e06b70..279c5d674edf 100644
--- a/block/blk-mq-sysfs.c
+++ b/block/blk-mq-sysfs.c
@@ -141,15 +141,26 @@ static ssize_t blk_mq_sysfs_completed_show(struct blk_mq_ctx *ctx, char *page)
141 141
142static ssize_t sysfs_list_show(char *page, struct list_head *list, char *msg) 142static ssize_t sysfs_list_show(char *page, struct list_head *list, char *msg)
143{ 143{
144 char *start_page = page;
145 struct request *rq; 144 struct request *rq;
145 int len = snprintf(page, PAGE_SIZE - 1, "%s:\n", msg);
146
147 list_for_each_entry(rq, list, queuelist) {
148 const int rq_len = 2 * sizeof(rq) + 2;
149
150 /* if the output will be truncated */
151 if (PAGE_SIZE - 1 < len + rq_len) {
152 /* backspacing if it can't hold '\t...\n' */
153 if (PAGE_SIZE - 1 < len + 5)
154 len -= rq_len;
155 len += snprintf(page + len, PAGE_SIZE - 1 - len,
156 "\t...\n");
157 break;
158 }
159 len += snprintf(page + len, PAGE_SIZE - 1 - len,
160 "\t%p\n", rq);
161 }
146 162
147 page += sprintf(page, "%s:\n", msg); 163 return len;
148
149 list_for_each_entry(rq, list, queuelist)
150 page += sprintf(page, "\t%p\n", rq);
151
152 return page - start_page;
153} 164}
154 165
155static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx *ctx, char *page) 166static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx *ctx, char *page)
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 9b6e28830b82..9115c6d59948 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -429,7 +429,7 @@ static void bt_for_each(struct blk_mq_hw_ctx *hctx,
429 for (bit = find_first_bit(&bm->word, bm->depth); 429 for (bit = find_first_bit(&bm->word, bm->depth);
430 bit < bm->depth; 430 bit < bm->depth;
431 bit = find_next_bit(&bm->word, bm->depth, bit + 1)) { 431 bit = find_next_bit(&bm->word, bm->depth, bit + 1)) {
432 rq = blk_mq_tag_to_rq(hctx->tags, off + bit); 432 rq = hctx->tags->rqs[off + bit];
433 if (rq->q == hctx->queue) 433 if (rq->q == hctx->queue)
434 fn(hctx, rq, data, reserved); 434 fn(hctx, rq, data, reserved);
435 } 435 }
@@ -453,7 +453,7 @@ static void bt_tags_for_each(struct blk_mq_tags *tags,
453 for (bit = find_first_bit(&bm->word, bm->depth); 453 for (bit = find_first_bit(&bm->word, bm->depth);
454 bit < bm->depth; 454 bit < bm->depth;
455 bit = find_next_bit(&bm->word, bm->depth, bit + 1)) { 455 bit = find_next_bit(&bm->word, bm->depth, bit + 1)) {
456 rq = blk_mq_tag_to_rq(tags, off + bit); 456 rq = tags->rqs[off + bit];
457 fn(rq, data, reserved); 457 fn(rq, data, reserved);
458 } 458 }
459 459
diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
index 75893a34237d..9eb2cf4f01cb 100644
--- a/block/blk-mq-tag.h
+++ b/block/blk-mq-tag.h
@@ -89,4 +89,16 @@ static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
89 __blk_mq_tag_idle(hctx); 89 __blk_mq_tag_idle(hctx);
90} 90}
91 91
92/*
93 * This helper should only be used for flush request to share tag
94 * with the request cloned from, and both the two requests can't be
95 * in flight at the same time. The caller has to make sure the tag
96 * can't be freed.
97 */
98static inline void blk_mq_tag_set_rq(struct blk_mq_hw_ctx *hctx,
99 unsigned int tag, struct request *rq)
100{
101 hctx->tags->rqs[tag] = rq;
102}
103
92#endif 104#endif
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 7d842db59699..f2d67b4047a0 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -559,23 +559,9 @@ void blk_mq_abort_requeue_list(struct request_queue *q)
559} 559}
560EXPORT_SYMBOL(blk_mq_abort_requeue_list); 560EXPORT_SYMBOL(blk_mq_abort_requeue_list);
561 561
562static inline bool is_flush_request(struct request *rq,
563 struct blk_flush_queue *fq, unsigned int tag)
564{
565 return ((rq->cmd_flags & REQ_FLUSH_SEQ) &&
566 fq->flush_rq->tag == tag);
567}
568
569struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) 562struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
570{ 563{
571 struct request *rq = tags->rqs[tag]; 564 return tags->rqs[tag];
572 /* mq_ctx of flush rq is always cloned from the corresponding req */
573 struct blk_flush_queue *fq = blk_get_flush_queue(rq->q, rq->mq_ctx);
574
575 if (!is_flush_request(rq, fq, tag))
576 return rq;
577
578 return fq->flush_rq;
579} 565}
580EXPORT_SYMBOL(blk_mq_tag_to_rq); 566EXPORT_SYMBOL(blk_mq_tag_to_rq);
581 567
@@ -1199,7 +1185,7 @@ static struct request *blk_mq_map_request(struct request_queue *q,
1199 struct blk_mq_alloc_data alloc_data; 1185 struct blk_mq_alloc_data alloc_data;
1200 1186
1201 if (unlikely(blk_mq_queue_enter(q, GFP_KERNEL))) { 1187 if (unlikely(blk_mq_queue_enter(q, GFP_KERNEL))) {
1202 bio_endio(bio, -EIO); 1188 bio_io_error(bio);
1203 return NULL; 1189 return NULL;
1204 } 1190 }
1205 1191
@@ -1283,10 +1269,12 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
1283 blk_queue_bounce(q, &bio); 1269 blk_queue_bounce(q, &bio);
1284 1270
1285 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { 1271 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1286 bio_endio(bio, -EIO); 1272 bio_io_error(bio);
1287 return; 1273 return;
1288 } 1274 }
1289 1275
1276 blk_queue_split(q, &bio, q->bio_split);
1277
1290 if (!is_flush_fua && !blk_queue_nomerges(q) && 1278 if (!is_flush_fua && !blk_queue_nomerges(q) &&
1291 blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq)) 1279 blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
1292 return; 1280 return;
@@ -1368,10 +1356,12 @@ static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
1368 blk_queue_bounce(q, &bio); 1356 blk_queue_bounce(q, &bio);
1369 1357
1370 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { 1358 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1371 bio_endio(bio, -EIO); 1359 bio_io_error(bio);
1372 return; 1360 return;
1373 } 1361 }
1374 1362
1363 blk_queue_split(q, &bio, q->bio_split);
1364
1375 if (!is_flush_fua && !blk_queue_nomerges(q) && 1365 if (!is_flush_fua && !blk_queue_nomerges(q) &&
1376 blk_attempt_plug_merge(q, bio, &request_count, NULL)) 1366 blk_attempt_plug_merge(q, bio, &request_count, NULL))
1377 return; 1367 return;
diff --git a/block/blk-settings.c b/block/blk-settings.c
index e0057d035200..7d8f129a1516 100644
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -53,28 +53,6 @@ void blk_queue_unprep_rq(struct request_queue *q, unprep_rq_fn *ufn)
53} 53}
54EXPORT_SYMBOL(blk_queue_unprep_rq); 54EXPORT_SYMBOL(blk_queue_unprep_rq);
55 55
56/**
57 * blk_queue_merge_bvec - set a merge_bvec function for queue
58 * @q: queue
59 * @mbfn: merge_bvec_fn
60 *
61 * Usually queues have static limitations on the max sectors or segments that
62 * we can put in a request. Stacking drivers may have some settings that
63 * are dynamic, and thus we have to query the queue whether it is ok to
64 * add a new bio_vec to a bio at a given offset or not. If the block device
65 * has such limitations, it needs to register a merge_bvec_fn to control
66 * the size of bio's sent to it. Note that a block device *must* allow a
67 * single page to be added to an empty bio. The block device driver may want
68 * to use the bio_split() function to deal with these bio's. By default
69 * no merge_bvec_fn is defined for a queue, and only the fixed limits are
70 * honored.
71 */
72void blk_queue_merge_bvec(struct request_queue *q, merge_bvec_fn *mbfn)
73{
74 q->merge_bvec_fn = mbfn;
75}
76EXPORT_SYMBOL(blk_queue_merge_bvec);
77
78void blk_queue_softirq_done(struct request_queue *q, softirq_done_fn *fn) 56void blk_queue_softirq_done(struct request_queue *q, softirq_done_fn *fn)
79{ 57{
80 q->softirq_done_fn = fn; 58 q->softirq_done_fn = fn;
@@ -111,11 +89,13 @@ void blk_set_default_limits(struct queue_limits *lim)
111 lim->max_segments = BLK_MAX_SEGMENTS; 89 lim->max_segments = BLK_MAX_SEGMENTS;
112 lim->max_integrity_segments = 0; 90 lim->max_integrity_segments = 0;
113 lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK; 91 lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK;
92 lim->virt_boundary_mask = 0;
114 lim->max_segment_size = BLK_MAX_SEGMENT_SIZE; 93 lim->max_segment_size = BLK_MAX_SEGMENT_SIZE;
115 lim->max_sectors = lim->max_hw_sectors = BLK_SAFE_MAX_SECTORS; 94 lim->max_sectors = lim->max_hw_sectors = BLK_SAFE_MAX_SECTORS;
116 lim->chunk_sectors = 0; 95 lim->chunk_sectors = 0;
117 lim->max_write_same_sectors = 0; 96 lim->max_write_same_sectors = 0;
118 lim->max_discard_sectors = 0; 97 lim->max_discard_sectors = 0;
98 lim->max_hw_discard_sectors = 0;
119 lim->discard_granularity = 0; 99 lim->discard_granularity = 0;
120 lim->discard_alignment = 0; 100 lim->discard_alignment = 0;
121 lim->discard_misaligned = 0; 101 lim->discard_misaligned = 0;
@@ -257,7 +237,9 @@ void blk_limits_max_hw_sectors(struct queue_limits *limits, unsigned int max_hw_
257 __func__, max_hw_sectors); 237 __func__, max_hw_sectors);
258 } 238 }
259 239
260 limits->max_sectors = limits->max_hw_sectors = max_hw_sectors; 240 limits->max_hw_sectors = max_hw_sectors;
241 limits->max_sectors = min_t(unsigned int, max_hw_sectors,
242 BLK_DEF_MAX_SECTORS);
261} 243}
262EXPORT_SYMBOL(blk_limits_max_hw_sectors); 244EXPORT_SYMBOL(blk_limits_max_hw_sectors);
263 245
@@ -303,6 +285,7 @@ EXPORT_SYMBOL(blk_queue_chunk_sectors);
303void blk_queue_max_discard_sectors(struct request_queue *q, 285void blk_queue_max_discard_sectors(struct request_queue *q,
304 unsigned int max_discard_sectors) 286 unsigned int max_discard_sectors)
305{ 287{
288 q->limits.max_hw_discard_sectors = max_discard_sectors;
306 q->limits.max_discard_sectors = max_discard_sectors; 289 q->limits.max_discard_sectors = max_discard_sectors;
307} 290}
308EXPORT_SYMBOL(blk_queue_max_discard_sectors); 291EXPORT_SYMBOL(blk_queue_max_discard_sectors);
@@ -550,6 +533,8 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
550 533
551 t->seg_boundary_mask = min_not_zero(t->seg_boundary_mask, 534 t->seg_boundary_mask = min_not_zero(t->seg_boundary_mask,
552 b->seg_boundary_mask); 535 b->seg_boundary_mask);
536 t->virt_boundary_mask = min_not_zero(t->virt_boundary_mask,
537 b->virt_boundary_mask);
553 538
554 t->max_segments = min_not_zero(t->max_segments, b->max_segments); 539 t->max_segments = min_not_zero(t->max_segments, b->max_segments);
555 t->max_integrity_segments = min_not_zero(t->max_integrity_segments, 540 t->max_integrity_segments = min_not_zero(t->max_integrity_segments,
@@ -641,6 +626,8 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
641 626
642 t->max_discard_sectors = min_not_zero(t->max_discard_sectors, 627 t->max_discard_sectors = min_not_zero(t->max_discard_sectors,
643 b->max_discard_sectors); 628 b->max_discard_sectors);
629 t->max_hw_discard_sectors = min_not_zero(t->max_hw_discard_sectors,
630 b->max_hw_discard_sectors);
644 t->discard_granularity = max(t->discard_granularity, 631 t->discard_granularity = max(t->discard_granularity,
645 b->discard_granularity); 632 b->discard_granularity);
646 t->discard_alignment = lcm_not_zero(t->discard_alignment, alignment) % 633 t->discard_alignment = lcm_not_zero(t->discard_alignment, alignment) %
@@ -788,6 +775,17 @@ void blk_queue_segment_boundary(struct request_queue *q, unsigned long mask)
788EXPORT_SYMBOL(blk_queue_segment_boundary); 775EXPORT_SYMBOL(blk_queue_segment_boundary);
789 776
790/** 777/**
778 * blk_queue_virt_boundary - set boundary rules for bio merging
779 * @q: the request queue for the device
780 * @mask: the memory boundary mask
781 **/
782void blk_queue_virt_boundary(struct request_queue *q, unsigned long mask)
783{
784 q->limits.virt_boundary_mask = mask;
785}
786EXPORT_SYMBOL(blk_queue_virt_boundary);
787
788/**
791 * blk_queue_dma_alignment - set dma length and memory alignment 789 * blk_queue_dma_alignment - set dma length and memory alignment
792 * @q: the request queue for the device 790 * @q: the request queue for the device
793 * @mask: alignment mask 791 * @mask: alignment mask
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 6264b382d4d1..3e44a9da2a13 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -145,12 +145,43 @@ static ssize_t queue_discard_granularity_show(struct request_queue *q, char *pag
145 return queue_var_show(q->limits.discard_granularity, page); 145 return queue_var_show(q->limits.discard_granularity, page);
146} 146}
147 147
148static ssize_t queue_discard_max_hw_show(struct request_queue *q, char *page)
149{
150 unsigned long long val;
151
152 val = q->limits.max_hw_discard_sectors << 9;
153 return sprintf(page, "%llu\n", val);
154}
155
148static ssize_t queue_discard_max_show(struct request_queue *q, char *page) 156static ssize_t queue_discard_max_show(struct request_queue *q, char *page)
149{ 157{
150 return sprintf(page, "%llu\n", 158 return sprintf(page, "%llu\n",
151 (unsigned long long)q->limits.max_discard_sectors << 9); 159 (unsigned long long)q->limits.max_discard_sectors << 9);
152} 160}
153 161
162static ssize_t queue_discard_max_store(struct request_queue *q,
163 const char *page, size_t count)
164{
165 unsigned long max_discard;
166 ssize_t ret = queue_var_store(&max_discard, page, count);
167
168 if (ret < 0)
169 return ret;
170
171 if (max_discard & (q->limits.discard_granularity - 1))
172 return -EINVAL;
173
174 max_discard >>= 9;
175 if (max_discard > UINT_MAX)
176 return -EINVAL;
177
178 if (max_discard > q->limits.max_hw_discard_sectors)
179 max_discard = q->limits.max_hw_discard_sectors;
180
181 q->limits.max_discard_sectors = max_discard;
182 return ret;
183}
184
154static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page) 185static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page)
155{ 186{
156 return queue_var_show(queue_discard_zeroes_data(q), page); 187 return queue_var_show(queue_discard_zeroes_data(q), page);
@@ -360,9 +391,15 @@ static struct queue_sysfs_entry queue_discard_granularity_entry = {
360 .show = queue_discard_granularity_show, 391 .show = queue_discard_granularity_show,
361}; 392};
362 393
394static struct queue_sysfs_entry queue_discard_max_hw_entry = {
395 .attr = {.name = "discard_max_hw_bytes", .mode = S_IRUGO },
396 .show = queue_discard_max_hw_show,
397};
398
363static struct queue_sysfs_entry queue_discard_max_entry = { 399static struct queue_sysfs_entry queue_discard_max_entry = {
364 .attr = {.name = "discard_max_bytes", .mode = S_IRUGO }, 400 .attr = {.name = "discard_max_bytes", .mode = S_IRUGO | S_IWUSR },
365 .show = queue_discard_max_show, 401 .show = queue_discard_max_show,
402 .store = queue_discard_max_store,
366}; 403};
367 404
368static struct queue_sysfs_entry queue_discard_zeroes_data_entry = { 405static struct queue_sysfs_entry queue_discard_zeroes_data_entry = {
@@ -421,6 +458,7 @@ static struct attribute *default_attrs[] = {
421 &queue_io_opt_entry.attr, 458 &queue_io_opt_entry.attr,
422 &queue_discard_granularity_entry.attr, 459 &queue_discard_granularity_entry.attr,
423 &queue_discard_max_entry.attr, 460 &queue_discard_max_entry.attr,
461 &queue_discard_max_hw_entry.attr,
424 &queue_discard_zeroes_data_entry.attr, 462 &queue_discard_zeroes_data_entry.attr,
425 &queue_write_same_max_entry.attr, 463 &queue_write_same_max_entry.attr,
426 &queue_nonrot_entry.attr, 464 &queue_nonrot_entry.attr,
@@ -523,6 +561,9 @@ static void blk_release_queue(struct kobject *kobj)
523 561
524 blk_trace_shutdown(q); 562 blk_trace_shutdown(q);
525 563
564 if (q->bio_split)
565 bioset_free(q->bio_split);
566
526 ida_simple_remove(&blk_queue_ida, q->id); 567 ida_simple_remove(&blk_queue_ida, q->id);
527 call_rcu(&q->rcu_head, blk_free_queue_rcu); 568 call_rcu(&q->rcu_head, blk_free_queue_rcu);
528} 569}
diff --git a/block/blk.h b/block/blk.h
index 026d9594142b..838188b35a83 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -22,6 +22,12 @@ struct blk_flush_queue {
22 struct list_head flush_queue[2]; 22 struct list_head flush_queue[2];
23 struct list_head flush_data_in_flight; 23 struct list_head flush_data_in_flight;
24 struct request *flush_rq; 24 struct request *flush_rq;
25
26 /*
27 * flush_rq shares tag with this rq, both can't be active
28 * at the same time
29 */
30 struct request *orig_rq;
25 spinlock_t mq_flush_lock; 31 spinlock_t mq_flush_lock;
26}; 32};
27 33
diff --git a/block/bounce.c b/block/bounce.c
index b17311227c12..2c310ea007ee 100644
--- a/block/bounce.c
+++ b/block/bounce.c
@@ -123,7 +123,7 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio *from)
123 } 123 }
124} 124}
125 125
126static void bounce_end_io(struct bio *bio, mempool_t *pool, int err) 126static void bounce_end_io(struct bio *bio, mempool_t *pool)
127{ 127{
128 struct bio *bio_orig = bio->bi_private; 128 struct bio *bio_orig = bio->bi_private;
129 struct bio_vec *bvec, *org_vec; 129 struct bio_vec *bvec, *org_vec;
@@ -141,39 +141,40 @@ static void bounce_end_io(struct bio *bio, mempool_t *pool, int err)
141 mempool_free(bvec->bv_page, pool); 141 mempool_free(bvec->bv_page, pool);
142 } 142 }
143 143
144 bio_endio(bio_orig, err); 144 bio_orig->bi_error = bio->bi_error;
145 bio_endio(bio_orig);
145 bio_put(bio); 146 bio_put(bio);
146} 147}
147 148
148static void bounce_end_io_write(struct bio *bio, int err) 149static void bounce_end_io_write(struct bio *bio)
149{ 150{
150 bounce_end_io(bio, page_pool, err); 151 bounce_end_io(bio, page_pool);
151} 152}
152 153
153static void bounce_end_io_write_isa(struct bio *bio, int err) 154static void bounce_end_io_write_isa(struct bio *bio)
154{ 155{
155 156
156 bounce_end_io(bio, isa_page_pool, err); 157 bounce_end_io(bio, isa_page_pool);
157} 158}
158 159
159static void __bounce_end_io_read(struct bio *bio, mempool_t *pool, int err) 160static void __bounce_end_io_read(struct bio *bio, mempool_t *pool)
160{ 161{
161 struct bio *bio_orig = bio->bi_private; 162 struct bio *bio_orig = bio->bi_private;
162 163
163 if (test_bit(BIO_UPTODATE, &bio->bi_flags)) 164 if (!bio->bi_error)
164 copy_to_high_bio_irq(bio_orig, bio); 165 copy_to_high_bio_irq(bio_orig, bio);
165 166
166 bounce_end_io(bio, pool, err); 167 bounce_end_io(bio, pool);
167} 168}
168 169
169static void bounce_end_io_read(struct bio *bio, int err) 170static void bounce_end_io_read(struct bio *bio)
170{ 171{
171 __bounce_end_io_read(bio, page_pool, err); 172 __bounce_end_io_read(bio, page_pool);
172} 173}
173 174
174static void bounce_end_io_read_isa(struct bio *bio, int err) 175static void bounce_end_io_read_isa(struct bio *bio)
175{ 176{
176 __bounce_end_io_read(bio, isa_page_pool, err); 177 __bounce_end_io_read(bio, isa_page_pool);
177} 178}
178 179
179#ifdef CONFIG_NEED_BOUNCE_POOL 180#ifdef CONFIG_NEED_BOUNCE_POOL
@@ -185,7 +186,7 @@ static int must_snapshot_stable_pages(struct request_queue *q, struct bio *bio)
185 if (!bdi_cap_stable_pages_required(&q->backing_dev_info)) 186 if (!bdi_cap_stable_pages_required(&q->backing_dev_info))
186 return 0; 187 return 0;
187 188
188 return test_bit(BIO_SNAP_STABLE, &bio->bi_flags); 189 return bio_flagged(bio, BIO_SNAP_STABLE);
189} 190}
190#else 191#else
191static int must_snapshot_stable_pages(struct request_queue *q, struct bio *bio) 192static int must_snapshot_stable_pages(struct request_queue *q, struct bio *bio)
diff --git a/block/genhd.c b/block/genhd.c
index 59a1395eedac..0c706f33a599 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -1110,8 +1110,7 @@ static void disk_release(struct device *dev)
1110 disk_release_events(disk); 1110 disk_release_events(disk);
1111 kfree(disk->random); 1111 kfree(disk->random);
1112 disk_replace_part_tbl(disk, NULL); 1112 disk_replace_part_tbl(disk, NULL);
1113 free_part_stats(&disk->part0); 1113 hd_free_part(&disk->part0);
1114 free_part_info(&disk->part0);
1115 if (disk->queue) 1114 if (disk->queue)
1116 blk_put_queue(disk->queue); 1115 blk_put_queue(disk->queue);
1117 kfree(disk); 1116 kfree(disk);
@@ -1285,7 +1284,11 @@ struct gendisk *alloc_disk_node(int minors, int node_id)
1285 * converted to make use of bd_mutex and sequence counters. 1284 * converted to make use of bd_mutex and sequence counters.
1286 */ 1285 */
1287 seqcount_init(&disk->part0.nr_sects_seq); 1286 seqcount_init(&disk->part0.nr_sects_seq);
1288 hd_ref_init(&disk->part0); 1287 if (hd_ref_init(&disk->part0)) {
1288 hd_free_part(&disk->part0);
1289 kfree(disk);
1290 return NULL;
1291 }
1289 1292
1290 disk->minors = minors; 1293 disk->minors = minors;
1291 rand_initialize_disk(disk); 1294 rand_initialize_disk(disk);
diff --git a/block/partition-generic.c b/block/partition-generic.c
index 0d9e5f97f0a8..e7711133284e 100644
--- a/block/partition-generic.c
+++ b/block/partition-generic.c
@@ -212,8 +212,7 @@ static void part_release(struct device *dev)
212{ 212{
213 struct hd_struct *p = dev_to_part(dev); 213 struct hd_struct *p = dev_to_part(dev);
214 blk_free_devt(dev->devt); 214 blk_free_devt(dev->devt);
215 free_part_stats(p); 215 hd_free_part(p);
216 free_part_info(p);
217 kfree(p); 216 kfree(p);
218} 217}
219 218
@@ -233,8 +232,9 @@ static void delete_partition_rcu_cb(struct rcu_head *head)
233 put_device(part_to_dev(part)); 232 put_device(part_to_dev(part));
234} 233}
235 234
236void __delete_partition(struct hd_struct *part) 235void __delete_partition(struct percpu_ref *ref)
237{ 236{
237 struct hd_struct *part = container_of(ref, struct hd_struct, ref);
238 call_rcu(&part->rcu_head, delete_partition_rcu_cb); 238 call_rcu(&part->rcu_head, delete_partition_rcu_cb);
239} 239}
240 240
@@ -255,7 +255,7 @@ void delete_partition(struct gendisk *disk, int partno)
255 kobject_put(part->holder_dir); 255 kobject_put(part->holder_dir);
256 device_del(part_to_dev(part)); 256 device_del(part_to_dev(part));
257 257
258 hd_struct_put(part); 258 hd_struct_kill(part);
259} 259}
260 260
261static ssize_t whole_disk_show(struct device *dev, 261static ssize_t whole_disk_show(struct device *dev,
@@ -356,8 +356,8 @@ struct hd_struct *add_partition(struct gendisk *disk, int partno,
356 if (!dev_get_uevent_suppress(ddev)) 356 if (!dev_get_uevent_suppress(ddev))
357 kobject_uevent(&pdev->kobj, KOBJ_ADD); 357 kobject_uevent(&pdev->kobj, KOBJ_ADD);
358 358
359 hd_ref_init(p); 359 if (!hd_ref_init(p))
360 return p; 360 return p;
361 361
362out_free_info: 362out_free_info:
363 free_part_info(p); 363 free_part_info(p);
diff --git a/drivers/block/aoe/aoeblk.c b/drivers/block/aoe/aoeblk.c
index 46c282fff104..dd73e1ff1759 100644
--- a/drivers/block/aoe/aoeblk.c
+++ b/drivers/block/aoe/aoeblk.c
@@ -395,7 +395,7 @@ aoeblk_gdalloc(void *vp)
395 WARN_ON(d->flags & DEVFL_TKILL); 395 WARN_ON(d->flags & DEVFL_TKILL);
396 WARN_ON(d->gd); 396 WARN_ON(d->gd);
397 WARN_ON(d->flags & DEVFL_UP); 397 WARN_ON(d->flags & DEVFL_UP);
398 blk_queue_max_hw_sectors(q, 1024); 398 blk_queue_max_hw_sectors(q, BLK_DEF_MAX_SECTORS);
399 q->backing_dev_info.name = "aoe"; 399 q->backing_dev_info.name = "aoe";
400 q->backing_dev_info.ra_pages = READ_AHEAD / PAGE_CACHE_SIZE; 400 q->backing_dev_info.ra_pages = READ_AHEAD / PAGE_CACHE_SIZE;
401 d->bufpool = mp; 401 d->bufpool = mp;
diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c
index 422b7d84f686..ad80c85e0857 100644
--- a/drivers/block/aoe/aoecmd.c
+++ b/drivers/block/aoe/aoecmd.c
@@ -1110,7 +1110,7 @@ aoe_end_request(struct aoedev *d, struct request *rq, int fastfail)
1110 d->ip.rq = NULL; 1110 d->ip.rq = NULL;
1111 do { 1111 do {
1112 bio = rq->bio; 1112 bio = rq->bio;
1113 bok = !fastfail && test_bit(BIO_UPTODATE, &bio->bi_flags); 1113 bok = !fastfail && !bio->bi_error;
1114 } while (__blk_end_request(rq, bok ? 0 : -EIO, bio->bi_iter.bi_size)); 1114 } while (__blk_end_request(rq, bok ? 0 : -EIO, bio->bi_iter.bi_size));
1115 1115
1116 /* cf. http://lkml.org/lkml/2006/10/31/28 */ 1116 /* cf. http://lkml.org/lkml/2006/10/31/28 */
@@ -1172,7 +1172,7 @@ ktiocomplete(struct frame *f)
1172 ahout->cmdstat, ahin->cmdstat, 1172 ahout->cmdstat, ahin->cmdstat,
1173 d->aoemajor, d->aoeminor); 1173 d->aoemajor, d->aoeminor);
1174noskb: if (buf) 1174noskb: if (buf)
1175 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); 1175 buf->bio->bi_error = -EIO;
1176 goto out; 1176 goto out;
1177 } 1177 }
1178 1178
@@ -1185,7 +1185,7 @@ noskb: if (buf)
1185 "aoe: runt data size in read from", 1185 "aoe: runt data size in read from",
1186 (long) d->aoemajor, d->aoeminor, 1186 (long) d->aoemajor, d->aoeminor,
1187 skb->len, n); 1187 skb->len, n);
1188 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); 1188 buf->bio->bi_error = -EIO;
1189 break; 1189 break;
1190 } 1190 }
1191 if (n > f->iter.bi_size) { 1191 if (n > f->iter.bi_size) {
@@ -1193,7 +1193,7 @@ noskb: if (buf)
1193 "aoe: too-large data size in read from", 1193 "aoe: too-large data size in read from",
1194 (long) d->aoemajor, d->aoeminor, 1194 (long) d->aoemajor, d->aoeminor,
1195 n, f->iter.bi_size); 1195 n, f->iter.bi_size);
1196 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); 1196 buf->bio->bi_error = -EIO;
1197 break; 1197 break;
1198 } 1198 }
1199 bvcpy(skb, f->buf->bio, f->iter, n); 1199 bvcpy(skb, f->buf->bio, f->iter, n);
@@ -1695,7 +1695,7 @@ aoe_failbuf(struct aoedev *d, struct buf *buf)
1695 if (buf == NULL) 1695 if (buf == NULL)
1696 return; 1696 return;
1697 buf->iter.bi_size = 0; 1697 buf->iter.bi_size = 0;
1698 clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); 1698 buf->bio->bi_error = -EIO;
1699 if (buf->nframesout == 0) 1699 if (buf->nframesout == 0)
1700 aoe_end_buf(d, buf); 1700 aoe_end_buf(d, buf);
1701} 1701}
diff --git a/drivers/block/aoe/aoedev.c b/drivers/block/aoe/aoedev.c
index e774c50b6842..ffd1947500c6 100644
--- a/drivers/block/aoe/aoedev.c
+++ b/drivers/block/aoe/aoedev.c
@@ -170,7 +170,7 @@ aoe_failip(struct aoedev *d)
170 if (rq == NULL) 170 if (rq == NULL)
171 return; 171 return;
172 while ((bio = d->ip.nxbio)) { 172 while ((bio = d->ip.nxbio)) {
173 clear_bit(BIO_UPTODATE, &bio->bi_flags); 173 bio->bi_error = -EIO;
174 d->ip.nxbio = bio->bi_next; 174 d->ip.nxbio = bio->bi_next;
175 n = (unsigned long) rq->special; 175 n = (unsigned long) rq->special;
176 rq->special = (void *) --n; 176 rq->special = (void *) --n;
diff --git a/drivers/block/brd.c b/drivers/block/brd.c
index 64ab4951e9d6..f9ab74505e69 100644
--- a/drivers/block/brd.c
+++ b/drivers/block/brd.c
@@ -331,14 +331,12 @@ static void brd_make_request(struct request_queue *q, struct bio *bio)
331 struct bio_vec bvec; 331 struct bio_vec bvec;
332 sector_t sector; 332 sector_t sector;
333 struct bvec_iter iter; 333 struct bvec_iter iter;
334 int err = -EIO;
335 334
336 sector = bio->bi_iter.bi_sector; 335 sector = bio->bi_iter.bi_sector;
337 if (bio_end_sector(bio) > get_capacity(bdev->bd_disk)) 336 if (bio_end_sector(bio) > get_capacity(bdev->bd_disk))
338 goto out; 337 goto io_error;
339 338
340 if (unlikely(bio->bi_rw & REQ_DISCARD)) { 339 if (unlikely(bio->bi_rw & REQ_DISCARD)) {
341 err = 0;
342 discard_from_brd(brd, sector, bio->bi_iter.bi_size); 340 discard_from_brd(brd, sector, bio->bi_iter.bi_size);
343 goto out; 341 goto out;
344 } 342 }
@@ -349,15 +347,20 @@ static void brd_make_request(struct request_queue *q, struct bio *bio)
349 347
350 bio_for_each_segment(bvec, bio, iter) { 348 bio_for_each_segment(bvec, bio, iter) {
351 unsigned int len = bvec.bv_len; 349 unsigned int len = bvec.bv_len;
350 int err;
351
352 err = brd_do_bvec(brd, bvec.bv_page, len, 352 err = brd_do_bvec(brd, bvec.bv_page, len,
353 bvec.bv_offset, rw, sector); 353 bvec.bv_offset, rw, sector);
354 if (err) 354 if (err)
355 break; 355 goto io_error;
356 sector += len >> SECTOR_SHIFT; 356 sector += len >> SECTOR_SHIFT;
357 } 357 }
358 358
359out: 359out:
360 bio_endio(bio, err); 360 bio_endio(bio);
361 return;
362io_error:
363 bio_io_error(bio);
361} 364}
362 365
363static int brd_rw_page(struct block_device *bdev, sector_t sector, 366static int brd_rw_page(struct block_device *bdev, sector_t sector,
@@ -500,7 +503,7 @@ static struct brd_device *brd_alloc(int i)
500 blk_queue_physical_block_size(brd->brd_queue, PAGE_SIZE); 503 blk_queue_physical_block_size(brd->brd_queue, PAGE_SIZE);
501 504
502 brd->brd_queue->limits.discard_granularity = PAGE_SIZE; 505 brd->brd_queue->limits.discard_granularity = PAGE_SIZE;
503 brd->brd_queue->limits.max_discard_sectors = UINT_MAX; 506 blk_queue_max_discard_sectors(brd->brd_queue, UINT_MAX);
504 brd->brd_queue->limits.discard_zeroes_data = 1; 507 brd->brd_queue->limits.discard_zeroes_data = 1;
505 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, brd->brd_queue); 508 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, brd->brd_queue);
506 509
diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c
index 1318e3217cb0..b3868e7a1ffd 100644
--- a/drivers/block/drbd/drbd_actlog.c
+++ b/drivers/block/drbd/drbd_actlog.c
@@ -175,11 +175,11 @@ static int _drbd_md_sync_page_io(struct drbd_device *device,
175 atomic_inc(&device->md_io.in_use); /* drbd_md_put_buffer() is in the completion handler */ 175 atomic_inc(&device->md_io.in_use); /* drbd_md_put_buffer() is in the completion handler */
176 device->md_io.submit_jif = jiffies; 176 device->md_io.submit_jif = jiffies;
177 if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) 177 if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD))
178 bio_endio(bio, -EIO); 178 bio_io_error(bio);
179 else 179 else
180 submit_bio(rw, bio); 180 submit_bio(rw, bio);
181 wait_until_done_or_force_detached(device, bdev, &device->md_io.done); 181 wait_until_done_or_force_detached(device, bdev, &device->md_io.done);
182 if (bio_flagged(bio, BIO_UPTODATE)) 182 if (!bio->bi_error)
183 err = device->md_io.error; 183 err = device->md_io.error;
184 184
185 out: 185 out:
diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c
index 434c77dcc99e..e5e0f19ceda0 100644
--- a/drivers/block/drbd/drbd_bitmap.c
+++ b/drivers/block/drbd/drbd_bitmap.c
@@ -941,36 +941,27 @@ static void drbd_bm_aio_ctx_destroy(struct kref *kref)
941} 941}
942 942
943/* bv_page may be a copy, or may be the original */ 943/* bv_page may be a copy, or may be the original */
944static void drbd_bm_endio(struct bio *bio, int error) 944static void drbd_bm_endio(struct bio *bio)
945{ 945{
946 struct drbd_bm_aio_ctx *ctx = bio->bi_private; 946 struct drbd_bm_aio_ctx *ctx = bio->bi_private;
947 struct drbd_device *device = ctx->device; 947 struct drbd_device *device = ctx->device;
948 struct drbd_bitmap *b = device->bitmap; 948 struct drbd_bitmap *b = device->bitmap;
949 unsigned int idx = bm_page_to_idx(bio->bi_io_vec[0].bv_page); 949 unsigned int idx = bm_page_to_idx(bio->bi_io_vec[0].bv_page);
950 int uptodate = bio_flagged(bio, BIO_UPTODATE);
951
952
953 /* strange behavior of some lower level drivers...
954 * fail the request by clearing the uptodate flag,
955 * but do not return any error?!
956 * do we want to WARN() on this? */
957 if (!error && !uptodate)
958 error = -EIO;
959 950
960 if ((ctx->flags & BM_AIO_COPY_PAGES) == 0 && 951 if ((ctx->flags & BM_AIO_COPY_PAGES) == 0 &&
961 !bm_test_page_unchanged(b->bm_pages[idx])) 952 !bm_test_page_unchanged(b->bm_pages[idx]))
962 drbd_warn(device, "bitmap page idx %u changed during IO!\n", idx); 953 drbd_warn(device, "bitmap page idx %u changed during IO!\n", idx);
963 954
964 if (error) { 955 if (bio->bi_error) {
965 /* ctx error will hold the completed-last non-zero error code, 956 /* ctx error will hold the completed-last non-zero error code,
966 * in case error codes differ. */ 957 * in case error codes differ. */
967 ctx->error = error; 958 ctx->error = bio->bi_error;
968 bm_set_page_io_err(b->bm_pages[idx]); 959 bm_set_page_io_err(b->bm_pages[idx]);
969 /* Not identical to on disk version of it. 960 /* Not identical to on disk version of it.
970 * Is BM_PAGE_IO_ERROR enough? */ 961 * Is BM_PAGE_IO_ERROR enough? */
971 if (__ratelimit(&drbd_ratelimit_state)) 962 if (__ratelimit(&drbd_ratelimit_state))
972 drbd_err(device, "IO ERROR %d on bitmap page idx %u\n", 963 drbd_err(device, "IO ERROR %d on bitmap page idx %u\n",
973 error, idx); 964 bio->bi_error, idx);
974 } else { 965 } else {
975 bm_clear_page_io_err(b->bm_pages[idx]); 966 bm_clear_page_io_err(b->bm_pages[idx]);
976 dynamic_drbd_dbg(device, "bitmap page idx %u completed\n", idx); 967 dynamic_drbd_dbg(device, "bitmap page idx %u completed\n", idx);
@@ -1031,7 +1022,7 @@ static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_ho
1031 1022
1032 if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) { 1023 if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) {
1033 bio->bi_rw |= rw; 1024 bio->bi_rw |= rw;
1034 bio_endio(bio, -EIO); 1025 bio_io_error(bio);
1035 } else { 1026 } else {
1036 submit_bio(rw, bio); 1027 submit_bio(rw, bio);
1037 /* this should not count as user activity and cause the 1028 /* this should not count as user activity and cause the
diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index efd19c2da9c2..015c6e91b756 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -1450,7 +1450,6 @@ extern void do_submit(struct work_struct *ws);
1450extern void __drbd_make_request(struct drbd_device *, struct bio *, unsigned long); 1450extern void __drbd_make_request(struct drbd_device *, struct bio *, unsigned long);
1451extern void drbd_make_request(struct request_queue *q, struct bio *bio); 1451extern void drbd_make_request(struct request_queue *q, struct bio *bio);
1452extern int drbd_read_remote(struct drbd_device *device, struct drbd_request *req); 1452extern int drbd_read_remote(struct drbd_device *device, struct drbd_request *req);
1453extern int drbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bvm, struct bio_vec *bvec);
1454extern int is_valid_ar_handle(struct drbd_request *, sector_t); 1453extern int is_valid_ar_handle(struct drbd_request *, sector_t);
1455 1454
1456 1455
@@ -1481,9 +1480,9 @@ extern int drbd_khelper(struct drbd_device *device, char *cmd);
1481 1480
1482/* drbd_worker.c */ 1481/* drbd_worker.c */
1483/* bi_end_io handlers */ 1482/* bi_end_io handlers */
1484extern void drbd_md_endio(struct bio *bio, int error); 1483extern void drbd_md_endio(struct bio *bio);
1485extern void drbd_peer_request_endio(struct bio *bio, int error); 1484extern void drbd_peer_request_endio(struct bio *bio);
1486extern void drbd_request_endio(struct bio *bio, int error); 1485extern void drbd_request_endio(struct bio *bio);
1487extern int drbd_worker(struct drbd_thread *thi); 1486extern int drbd_worker(struct drbd_thread *thi);
1488enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor); 1487enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor);
1489void drbd_resync_after_changed(struct drbd_device *device); 1488void drbd_resync_after_changed(struct drbd_device *device);
@@ -1604,12 +1603,13 @@ static inline void drbd_generic_make_request(struct drbd_device *device,
1604 __release(local); 1603 __release(local);
1605 if (!bio->bi_bdev) { 1604 if (!bio->bi_bdev) {
1606 drbd_err(device, "drbd_generic_make_request: bio->bi_bdev == NULL\n"); 1605 drbd_err(device, "drbd_generic_make_request: bio->bi_bdev == NULL\n");
1607 bio_endio(bio, -ENODEV); 1606 bio->bi_error = -ENODEV;
1607 bio_endio(bio);
1608 return; 1608 return;
1609 } 1609 }
1610 1610
1611 if (drbd_insert_fault(device, fault_type)) 1611 if (drbd_insert_fault(device, fault_type))
1612 bio_endio(bio, -EIO); 1612 bio_io_error(bio);
1613 else 1613 else
1614 generic_make_request(bio); 1614 generic_make_request(bio);
1615} 1615}
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index a1518539b858..74d97f4bac34 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -2774,7 +2774,6 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig
2774 This triggers a max_bio_size message upon first attach or connect */ 2774 This triggers a max_bio_size message upon first attach or connect */
2775 blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8); 2775 blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8);
2776 blk_queue_bounce_limit(q, BLK_BOUNCE_ANY); 2776 blk_queue_bounce_limit(q, BLK_BOUNCE_ANY);
2777 blk_queue_merge_bvec(q, drbd_merge_bvec);
2778 q->queue_lock = &resource->req_lock; 2777 q->queue_lock = &resource->req_lock;
2779 2778
2780 device->md_io.page = alloc_page(GFP_KERNEL); 2779 device->md_io.page = alloc_page(GFP_KERNEL);
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index 74df8cfad414..e80cbefbc2b5 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -1156,14 +1156,14 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi
1156 /* For now, don't allow more than one activity log extent worth of data 1156 /* For now, don't allow more than one activity log extent worth of data
1157 * to be discarded in one go. We may need to rework drbd_al_begin_io() 1157 * to be discarded in one go. We may need to rework drbd_al_begin_io()
1158 * to allow for even larger discard ranges */ 1158 * to allow for even larger discard ranges */
1159 q->limits.max_discard_sectors = DRBD_MAX_DISCARD_SECTORS; 1159 blk_queue_max_discard_sectors(q, DRBD_MAX_DISCARD_SECTORS);
1160 1160
1161 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); 1161 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
1162 /* REALLY? Is stacking secdiscard "legal"? */ 1162 /* REALLY? Is stacking secdiscard "legal"? */
1163 if (blk_queue_secdiscard(b)) 1163 if (blk_queue_secdiscard(b))
1164 queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, q); 1164 queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, q);
1165 } else { 1165 } else {
1166 q->limits.max_discard_sectors = 0; 1166 blk_queue_max_discard_sectors(q, 0);
1167 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); 1167 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q);
1168 queue_flag_clear_unlocked(QUEUE_FLAG_SECDISCARD, q); 1168 queue_flag_clear_unlocked(QUEUE_FLAG_SECDISCARD, q);
1169 } 1169 }
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index 3907202fb9d9..211592682169 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -201,7 +201,8 @@ void start_new_tl_epoch(struct drbd_connection *connection)
201void complete_master_bio(struct drbd_device *device, 201void complete_master_bio(struct drbd_device *device,
202 struct bio_and_error *m) 202 struct bio_and_error *m)
203{ 203{
204 bio_endio(m->bio, m->error); 204 m->bio->bi_error = m->error;
205 bio_endio(m->bio);
205 dec_ap_bio(device); 206 dec_ap_bio(device);
206} 207}
207 208
@@ -1153,12 +1154,12 @@ drbd_submit_req_private_bio(struct drbd_request *req)
1153 rw == WRITE ? DRBD_FAULT_DT_WR 1154 rw == WRITE ? DRBD_FAULT_DT_WR
1154 : rw == READ ? DRBD_FAULT_DT_RD 1155 : rw == READ ? DRBD_FAULT_DT_RD
1155 : DRBD_FAULT_DT_RA)) 1156 : DRBD_FAULT_DT_RA))
1156 bio_endio(bio, -EIO); 1157 bio_io_error(bio);
1157 else 1158 else
1158 generic_make_request(bio); 1159 generic_make_request(bio);
1159 put_ldev(device); 1160 put_ldev(device);
1160 } else 1161 } else
1161 bio_endio(bio, -EIO); 1162 bio_io_error(bio);
1162} 1163}
1163 1164
1164static void drbd_queue_write(struct drbd_device *device, struct drbd_request *req) 1165static void drbd_queue_write(struct drbd_device *device, struct drbd_request *req)
@@ -1191,7 +1192,8 @@ drbd_request_prepare(struct drbd_device *device, struct bio *bio, unsigned long
1191 /* only pass the error to the upper layers. 1192 /* only pass the error to the upper layers.
1192 * if user cannot handle io errors, that's not our business. */ 1193 * if user cannot handle io errors, that's not our business. */
1193 drbd_err(device, "could not kmalloc() req\n"); 1194 drbd_err(device, "could not kmalloc() req\n");
1194 bio_endio(bio, -ENOMEM); 1195 bio->bi_error = -ENOMEM;
1196 bio_endio(bio);
1195 return ERR_PTR(-ENOMEM); 1197 return ERR_PTR(-ENOMEM);
1196 } 1198 }
1197 req->start_jif = start_jif; 1199 req->start_jif = start_jif;
@@ -1497,6 +1499,8 @@ void drbd_make_request(struct request_queue *q, struct bio *bio)
1497 struct drbd_device *device = (struct drbd_device *) q->queuedata; 1499 struct drbd_device *device = (struct drbd_device *) q->queuedata;
1498 unsigned long start_jif; 1500 unsigned long start_jif;
1499 1501
1502 blk_queue_split(q, &bio, q->bio_split);
1503
1500 start_jif = jiffies; 1504 start_jif = jiffies;
1501 1505
1502 /* 1506 /*
@@ -1508,41 +1512,6 @@ void drbd_make_request(struct request_queue *q, struct bio *bio)
1508 __drbd_make_request(device, bio, start_jif); 1512 __drbd_make_request(device, bio, start_jif);
1509} 1513}
1510 1514
1511/* This is called by bio_add_page().
1512 *
1513 * q->max_hw_sectors and other global limits are already enforced there.
1514 *
1515 * We need to call down to our lower level device,
1516 * in case it has special restrictions.
1517 *
1518 * We also may need to enforce configured max-bio-bvecs limits.
1519 *
1520 * As long as the BIO is empty we have to allow at least one bvec,
1521 * regardless of size and offset, so no need to ask lower levels.
1522 */
1523int drbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bvm, struct bio_vec *bvec)
1524{
1525 struct drbd_device *device = (struct drbd_device *) q->queuedata;
1526 unsigned int bio_size = bvm->bi_size;
1527 int limit = DRBD_MAX_BIO_SIZE;
1528 int backing_limit;
1529
1530 if (bio_size && get_ldev(device)) {
1531 unsigned int max_hw_sectors = queue_max_hw_sectors(q);
1532 struct request_queue * const b =
1533 device->ldev->backing_bdev->bd_disk->queue;
1534 if (b->merge_bvec_fn) {
1535 bvm->bi_bdev = device->ldev->backing_bdev;
1536 backing_limit = b->merge_bvec_fn(b, bvm, bvec);
1537 limit = min(limit, backing_limit);
1538 }
1539 put_ldev(device);
1540 if ((limit >> 9) > max_hw_sectors)
1541 limit = max_hw_sectors << 9;
1542 }
1543 return limit;
1544}
1545
1546void request_timer_fn(unsigned long data) 1515void request_timer_fn(unsigned long data)
1547{ 1516{
1548 struct drbd_device *device = (struct drbd_device *) data; 1517 struct drbd_device *device = (struct drbd_device *) data;
diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c
index d0fae55d871d..5578c1477ba6 100644
--- a/drivers/block/drbd/drbd_worker.c
+++ b/drivers/block/drbd/drbd_worker.c
@@ -65,12 +65,12 @@ rwlock_t global_state_lock;
65/* used for synchronous meta data and bitmap IO 65/* used for synchronous meta data and bitmap IO
66 * submitted by drbd_md_sync_page_io() 66 * submitted by drbd_md_sync_page_io()
67 */ 67 */
68void drbd_md_endio(struct bio *bio, int error) 68void drbd_md_endio(struct bio *bio)
69{ 69{
70 struct drbd_device *device; 70 struct drbd_device *device;
71 71
72 device = bio->bi_private; 72 device = bio->bi_private;
73 device->md_io.error = error; 73 device->md_io.error = bio->bi_error;
74 74
75 /* We grabbed an extra reference in _drbd_md_sync_page_io() to be able 75 /* We grabbed an extra reference in _drbd_md_sync_page_io() to be able
76 * to timeout on the lower level device, and eventually detach from it. 76 * to timeout on the lower level device, and eventually detach from it.
@@ -170,31 +170,20 @@ void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(l
170/* writes on behalf of the partner, or resync writes, 170/* writes on behalf of the partner, or resync writes,
171 * "submitted" by the receiver. 171 * "submitted" by the receiver.
172 */ 172 */
173void drbd_peer_request_endio(struct bio *bio, int error) 173void drbd_peer_request_endio(struct bio *bio)
174{ 174{
175 struct drbd_peer_request *peer_req = bio->bi_private; 175 struct drbd_peer_request *peer_req = bio->bi_private;
176 struct drbd_device *device = peer_req->peer_device->device; 176 struct drbd_device *device = peer_req->peer_device->device;
177 int uptodate = bio_flagged(bio, BIO_UPTODATE);
178 int is_write = bio_data_dir(bio) == WRITE; 177 int is_write = bio_data_dir(bio) == WRITE;
179 int is_discard = !!(bio->bi_rw & REQ_DISCARD); 178 int is_discard = !!(bio->bi_rw & REQ_DISCARD);
180 179
181 if (error && __ratelimit(&drbd_ratelimit_state)) 180 if (bio->bi_error && __ratelimit(&drbd_ratelimit_state))
182 drbd_warn(device, "%s: error=%d s=%llus\n", 181 drbd_warn(device, "%s: error=%d s=%llus\n",
183 is_write ? (is_discard ? "discard" : "write") 182 is_write ? (is_discard ? "discard" : "write")
184 : "read", error, 183 : "read", bio->bi_error,
185 (unsigned long long)peer_req->i.sector); 184 (unsigned long long)peer_req->i.sector);
186 if (!error && !uptodate) {
187 if (__ratelimit(&drbd_ratelimit_state))
188 drbd_warn(device, "%s: setting error to -EIO s=%llus\n",
189 is_write ? "write" : "read",
190 (unsigned long long)peer_req->i.sector);
191 /* strange behavior of some lower level drivers...
192 * fail the request by clearing the uptodate flag,
193 * but do not return any error?! */
194 error = -EIO;
195 }
196 185
197 if (error) 186 if (bio->bi_error)
198 set_bit(__EE_WAS_ERROR, &peer_req->flags); 187 set_bit(__EE_WAS_ERROR, &peer_req->flags);
199 188
200 bio_put(bio); /* no need for the bio anymore */ 189 bio_put(bio); /* no need for the bio anymore */
@@ -208,24 +197,13 @@ void drbd_peer_request_endio(struct bio *bio, int error)
208 197
209/* read, readA or write requests on R_PRIMARY coming from drbd_make_request 198/* read, readA or write requests on R_PRIMARY coming from drbd_make_request
210 */ 199 */
211void drbd_request_endio(struct bio *bio, int error) 200void drbd_request_endio(struct bio *bio)
212{ 201{
213 unsigned long flags; 202 unsigned long flags;
214 struct drbd_request *req = bio->bi_private; 203 struct drbd_request *req = bio->bi_private;
215 struct drbd_device *device = req->device; 204 struct drbd_device *device = req->device;
216 struct bio_and_error m; 205 struct bio_and_error m;
217 enum drbd_req_event what; 206 enum drbd_req_event what;
218 int uptodate = bio_flagged(bio, BIO_UPTODATE);
219
220 if (!error && !uptodate) {
221 drbd_warn(device, "p %s: setting error to -EIO\n",
222 bio_data_dir(bio) == WRITE ? "write" : "read");
223 /* strange behavior of some lower level drivers...
224 * fail the request by clearing the uptodate flag,
225 * but do not return any error?! */
226 error = -EIO;
227 }
228
229 207
230 /* If this request was aborted locally before, 208 /* If this request was aborted locally before,
231 * but now was completed "successfully", 209 * but now was completed "successfully",
@@ -259,14 +237,14 @@ void drbd_request_endio(struct bio *bio, int error)
259 if (__ratelimit(&drbd_ratelimit_state)) 237 if (__ratelimit(&drbd_ratelimit_state))
260 drbd_emerg(device, "delayed completion of aborted local request; disk-timeout may be too aggressive\n"); 238 drbd_emerg(device, "delayed completion of aborted local request; disk-timeout may be too aggressive\n");
261 239
262 if (!error) 240 if (!bio->bi_error)
263 panic("possible random memory corruption caused by delayed completion of aborted local request\n"); 241 panic("possible random memory corruption caused by delayed completion of aborted local request\n");
264 } 242 }
265 243
266 /* to avoid recursion in __req_mod */ 244 /* to avoid recursion in __req_mod */
267 if (unlikely(error)) { 245 if (unlikely(bio->bi_error)) {
268 if (bio->bi_rw & REQ_DISCARD) 246 if (bio->bi_rw & REQ_DISCARD)
269 what = (error == -EOPNOTSUPP) 247 what = (bio->bi_error == -EOPNOTSUPP)
270 ? DISCARD_COMPLETED_NOTSUPP 248 ? DISCARD_COMPLETED_NOTSUPP
271 : DISCARD_COMPLETED_WITH_ERROR; 249 : DISCARD_COMPLETED_WITH_ERROR;
272 else 250 else
@@ -279,7 +257,7 @@ void drbd_request_endio(struct bio *bio, int error)
279 what = COMPLETED_OK; 257 what = COMPLETED_OK;
280 258
281 bio_put(req->private_bio); 259 bio_put(req->private_bio);
282 req->private_bio = ERR_PTR(error); 260 req->private_bio = ERR_PTR(bio->bi_error);
283 261
284 /* not req_mod(), we need irqsave here! */ 262 /* not req_mod(), we need irqsave here! */
285 spin_lock_irqsave(&device->resource->req_lock, flags); 263 spin_lock_irqsave(&device->resource->req_lock, flags);
diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c
index a08cda955285..331363e7de0f 100644
--- a/drivers/block/floppy.c
+++ b/drivers/block/floppy.c
@@ -3771,13 +3771,14 @@ struct rb0_cbdata {
3771 struct completion complete; 3771 struct completion complete;
3772}; 3772};
3773 3773
3774static void floppy_rb0_cb(struct bio *bio, int err) 3774static void floppy_rb0_cb(struct bio *bio)
3775{ 3775{
3776 struct rb0_cbdata *cbdata = (struct rb0_cbdata *)bio->bi_private; 3776 struct rb0_cbdata *cbdata = (struct rb0_cbdata *)bio->bi_private;
3777 int drive = cbdata->drive; 3777 int drive = cbdata->drive;
3778 3778
3779 if (err) { 3779 if (bio->bi_error) {
3780 pr_info("floppy: error %d while reading block 0\n", err); 3780 pr_info("floppy: error %d while reading block 0\n",
3781 bio->bi_error);
3781 set_bit(FD_OPEN_SHOULD_FAIL_BIT, &UDRS->flags); 3782 set_bit(FD_OPEN_SHOULD_FAIL_BIT, &UDRS->flags);
3782 } 3783 }
3783 complete(&cbdata->complete); 3784 complete(&cbdata->complete);
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index f7a4c9d7f721..f9889b6bc02c 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -675,7 +675,7 @@ static void loop_config_discard(struct loop_device *lo)
675 lo->lo_encrypt_key_size) { 675 lo->lo_encrypt_key_size) {
676 q->limits.discard_granularity = 0; 676 q->limits.discard_granularity = 0;
677 q->limits.discard_alignment = 0; 677 q->limits.discard_alignment = 0;
678 q->limits.max_discard_sectors = 0; 678 blk_queue_max_discard_sectors(q, 0);
679 q->limits.discard_zeroes_data = 0; 679 q->limits.discard_zeroes_data = 0;
680 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); 680 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q);
681 return; 681 return;
@@ -683,7 +683,7 @@ static void loop_config_discard(struct loop_device *lo)
683 683
684 q->limits.discard_granularity = inode->i_sb->s_blocksize; 684 q->limits.discard_granularity = inode->i_sb->s_blocksize;
685 q->limits.discard_alignment = 0; 685 q->limits.discard_alignment = 0;
686 q->limits.max_discard_sectors = UINT_MAX >> 9; 686 blk_queue_max_discard_sectors(q, UINT_MAX >> 9);
687 q->limits.discard_zeroes_data = 1; 687 q->limits.discard_zeroes_data = 1;
688 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); 688 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
689} 689}
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 0e385d8e9b86..f169faf9838a 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -822,7 +822,7 @@ static int __init nbd_init(void)
822 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue); 822 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue);
823 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, disk->queue); 823 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, disk->queue);
824 disk->queue->limits.discard_granularity = 512; 824 disk->queue->limits.discard_granularity = 512;
825 disk->queue->limits.max_discard_sectors = UINT_MAX; 825 blk_queue_max_discard_sectors(disk->queue, UINT_MAX);
826 disk->queue->limits.discard_zeroes_data = 0; 826 disk->queue->limits.discard_zeroes_data = 0;
827 blk_queue_max_hw_sectors(disk->queue, 65536); 827 blk_queue_max_hw_sectors(disk->queue, 65536);
828 disk->queue->limits.max_sectors = 256; 828 disk->queue->limits.max_sectors = 256;
diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c
index 3177b245d2bd..17269a3b85f2 100644
--- a/drivers/block/null_blk.c
+++ b/drivers/block/null_blk.c
@@ -222,7 +222,7 @@ static void end_cmd(struct nullb_cmd *cmd)
222 blk_end_request_all(cmd->rq, 0); 222 blk_end_request_all(cmd->rq, 0);
223 break; 223 break;
224 case NULL_Q_BIO: 224 case NULL_Q_BIO:
225 bio_endio(cmd->bio, 0); 225 bio_endio(cmd->bio);
226 break; 226 break;
227 } 227 }
228 228
diff --git a/drivers/block/nvme-core.c b/drivers/block/nvme-core.c
index 7920c2741b47..2f694d78da55 100644
--- a/drivers/block/nvme-core.c
+++ b/drivers/block/nvme-core.c
@@ -1935,7 +1935,7 @@ static void nvme_config_discard(struct nvme_ns *ns)
1935 ns->queue->limits.discard_zeroes_data = 0; 1935 ns->queue->limits.discard_zeroes_data = 0;
1936 ns->queue->limits.discard_alignment = logical_block_size; 1936 ns->queue->limits.discard_alignment = logical_block_size;
1937 ns->queue->limits.discard_granularity = logical_block_size; 1937 ns->queue->limits.discard_granularity = logical_block_size;
1938 ns->queue->limits.max_discard_sectors = 0xffffffff; 1938 blk_queue_max_discard_sectors(ns->queue, 0xffffffff);
1939 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue); 1939 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
1940} 1940}
1941 1941
@@ -2067,7 +2067,6 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
2067 goto out_free_ns; 2067 goto out_free_ns;
2068 queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue); 2068 queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
2069 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue); 2069 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
2070 queue_flag_set_unlocked(QUEUE_FLAG_SG_GAPS, ns->queue);
2071 ns->dev = dev; 2070 ns->dev = dev;
2072 ns->queue->queuedata = ns; 2071 ns->queue->queuedata = ns;
2073 2072
@@ -2087,6 +2086,7 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
2087 blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9); 2086 blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9);
2088 if (dev->vwc & NVME_CTRL_VWC_PRESENT) 2087 if (dev->vwc & NVME_CTRL_VWC_PRESENT)
2089 blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA); 2088 blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
2089 blk_queue_virt_boundary(ns->queue, dev->page_size - 1);
2090 2090
2091 disk->major = nvme_major; 2091 disk->major = nvme_major;
2092 disk->first_minor = 0; 2092 disk->first_minor = 0;
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index 4c20c228184c..7be2375db7f2 100644
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -977,7 +977,7 @@ static void pkt_make_local_copy(struct packet_data *pkt, struct bio_vec *bvec)
977 } 977 }
978} 978}
979 979
980static void pkt_end_io_read(struct bio *bio, int err) 980static void pkt_end_io_read(struct bio *bio)
981{ 981{
982 struct packet_data *pkt = bio->bi_private; 982 struct packet_data *pkt = bio->bi_private;
983 struct pktcdvd_device *pd = pkt->pd; 983 struct pktcdvd_device *pd = pkt->pd;
@@ -985,9 +985,9 @@ static void pkt_end_io_read(struct bio *bio, int err)
985 985
986 pkt_dbg(2, pd, "bio=%p sec0=%llx sec=%llx err=%d\n", 986 pkt_dbg(2, pd, "bio=%p sec0=%llx sec=%llx err=%d\n",
987 bio, (unsigned long long)pkt->sector, 987 bio, (unsigned long long)pkt->sector,
988 (unsigned long long)bio->bi_iter.bi_sector, err); 988 (unsigned long long)bio->bi_iter.bi_sector, bio->bi_error);
989 989
990 if (err) 990 if (bio->bi_error)
991 atomic_inc(&pkt->io_errors); 991 atomic_inc(&pkt->io_errors);
992 if (atomic_dec_and_test(&pkt->io_wait)) { 992 if (atomic_dec_and_test(&pkt->io_wait)) {
993 atomic_inc(&pkt->run_sm); 993 atomic_inc(&pkt->run_sm);
@@ -996,13 +996,13 @@ static void pkt_end_io_read(struct bio *bio, int err)
996 pkt_bio_finished(pd); 996 pkt_bio_finished(pd);
997} 997}
998 998
999static void pkt_end_io_packet_write(struct bio *bio, int err) 999static void pkt_end_io_packet_write(struct bio *bio)
1000{ 1000{
1001 struct packet_data *pkt = bio->bi_private; 1001 struct packet_data *pkt = bio->bi_private;
1002 struct pktcdvd_device *pd = pkt->pd; 1002 struct pktcdvd_device *pd = pkt->pd;
1003 BUG_ON(!pd); 1003 BUG_ON(!pd);
1004 1004
1005 pkt_dbg(2, pd, "id=%d, err=%d\n", pkt->id, err); 1005 pkt_dbg(2, pd, "id=%d, err=%d\n", pkt->id, bio->bi_error);
1006 1006
1007 pd->stats.pkt_ended++; 1007 pd->stats.pkt_ended++;
1008 1008
@@ -1340,22 +1340,22 @@ static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt)
1340 pkt_queue_bio(pd, pkt->w_bio); 1340 pkt_queue_bio(pd, pkt->w_bio);
1341} 1341}
1342 1342
1343static void pkt_finish_packet(struct packet_data *pkt, int uptodate) 1343static void pkt_finish_packet(struct packet_data *pkt, int error)
1344{ 1344{
1345 struct bio *bio; 1345 struct bio *bio;
1346 1346
1347 if (!uptodate) 1347 if (error)
1348 pkt->cache_valid = 0; 1348 pkt->cache_valid = 0;
1349 1349
1350 /* Finish all bios corresponding to this packet */ 1350 /* Finish all bios corresponding to this packet */
1351 while ((bio = bio_list_pop(&pkt->orig_bios))) 1351 while ((bio = bio_list_pop(&pkt->orig_bios))) {
1352 bio_endio(bio, uptodate ? 0 : -EIO); 1352 bio->bi_error = error;
1353 bio_endio(bio);
1354 }
1353} 1355}
1354 1356
1355static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data *pkt) 1357static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data *pkt)
1356{ 1358{
1357 int uptodate;
1358
1359 pkt_dbg(2, pd, "pkt %d\n", pkt->id); 1359 pkt_dbg(2, pd, "pkt %d\n", pkt->id);
1360 1360
1361 for (;;) { 1361 for (;;) {
@@ -1384,7 +1384,7 @@ static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data
1384 if (atomic_read(&pkt->io_wait) > 0) 1384 if (atomic_read(&pkt->io_wait) > 0)
1385 return; 1385 return;
1386 1386
1387 if (test_bit(BIO_UPTODATE, &pkt->w_bio->bi_flags)) { 1387 if (!pkt->w_bio->bi_error) {
1388 pkt_set_state(pkt, PACKET_FINISHED_STATE); 1388 pkt_set_state(pkt, PACKET_FINISHED_STATE);
1389 } else { 1389 } else {
1390 pkt_set_state(pkt, PACKET_RECOVERY_STATE); 1390 pkt_set_state(pkt, PACKET_RECOVERY_STATE);
@@ -1401,8 +1401,7 @@ static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data
1401 break; 1401 break;
1402 1402
1403 case PACKET_FINISHED_STATE: 1403 case PACKET_FINISHED_STATE:
1404 uptodate = test_bit(BIO_UPTODATE, &pkt->w_bio->bi_flags); 1404 pkt_finish_packet(pkt, pkt->w_bio->bi_error);
1405 pkt_finish_packet(pkt, uptodate);
1406 return; 1405 return;
1407 1406
1408 default: 1407 default:
@@ -2332,13 +2331,14 @@ static void pkt_close(struct gendisk *disk, fmode_t mode)
2332} 2331}
2333 2332
2334 2333
2335static void pkt_end_io_read_cloned(struct bio *bio, int err) 2334static void pkt_end_io_read_cloned(struct bio *bio)
2336{ 2335{
2337 struct packet_stacked_data *psd = bio->bi_private; 2336 struct packet_stacked_data *psd = bio->bi_private;
2338 struct pktcdvd_device *pd = psd->pd; 2337 struct pktcdvd_device *pd = psd->pd;
2339 2338
2339 psd->bio->bi_error = bio->bi_error;
2340 bio_put(bio); 2340 bio_put(bio);
2341 bio_endio(psd->bio, err); 2341 bio_endio(psd->bio);
2342 mempool_free(psd, psd_pool); 2342 mempool_free(psd, psd_pool);
2343 pkt_bio_finished(pd); 2343 pkt_bio_finished(pd);
2344} 2344}
@@ -2447,6 +2447,10 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio)
2447 char b[BDEVNAME_SIZE]; 2447 char b[BDEVNAME_SIZE];
2448 struct bio *split; 2448 struct bio *split;
2449 2449
2450 blk_queue_bounce(q, &bio);
2451
2452 blk_queue_split(q, &bio, q->bio_split);
2453
2450 pd = q->queuedata; 2454 pd = q->queuedata;
2451 if (!pd) { 2455 if (!pd) {
2452 pr_err("%s incorrect request queue\n", 2456 pr_err("%s incorrect request queue\n",
@@ -2477,8 +2481,6 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio)
2477 goto end_io; 2481 goto end_io;
2478 } 2482 }
2479 2483
2480 blk_queue_bounce(q, &bio);
2481
2482 do { 2484 do {
2483 sector_t zone = get_zone(bio->bi_iter.bi_sector, pd); 2485 sector_t zone = get_zone(bio->bi_iter.bi_sector, pd);
2484 sector_t last_zone = get_zone(bio_end_sector(bio) - 1, pd); 2486 sector_t last_zone = get_zone(bio_end_sector(bio) - 1, pd);
@@ -2504,26 +2506,6 @@ end_io:
2504 2506
2505 2507
2506 2508
2507static int pkt_merge_bvec(struct request_queue *q, struct bvec_merge_data *bmd,
2508 struct bio_vec *bvec)
2509{
2510 struct pktcdvd_device *pd = q->queuedata;
2511 sector_t zone = get_zone(bmd->bi_sector, pd);
2512 int used = ((bmd->bi_sector - zone) << 9) + bmd->bi_size;
2513 int remaining = (pd->settings.size << 9) - used;
2514 int remaining2;
2515
2516 /*
2517 * A bio <= PAGE_SIZE must be allowed. If it crosses a packet
2518 * boundary, pkt_make_request() will split the bio.
2519 */
2520 remaining2 = PAGE_SIZE - bmd->bi_size;
2521 remaining = max(remaining, remaining2);
2522
2523 BUG_ON(remaining < 0);
2524 return remaining;
2525}
2526
2527static void pkt_init_queue(struct pktcdvd_device *pd) 2509static void pkt_init_queue(struct pktcdvd_device *pd)
2528{ 2510{
2529 struct request_queue *q = pd->disk->queue; 2511 struct request_queue *q = pd->disk->queue;
@@ -2531,7 +2513,6 @@ static void pkt_init_queue(struct pktcdvd_device *pd)
2531 blk_queue_make_request(q, pkt_make_request); 2513 blk_queue_make_request(q, pkt_make_request);
2532 blk_queue_logical_block_size(q, CD_FRAMESIZE); 2514 blk_queue_logical_block_size(q, CD_FRAMESIZE);
2533 blk_queue_max_hw_sectors(q, PACKET_MAX_SECTORS); 2515 blk_queue_max_hw_sectors(q, PACKET_MAX_SECTORS);
2534 blk_queue_merge_bvec(q, pkt_merge_bvec);
2535 q->queuedata = pd; 2516 q->queuedata = pd;
2536} 2517}
2537 2518
diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c
index b1612eb16172..d89fcac59515 100644
--- a/drivers/block/ps3vram.c
+++ b/drivers/block/ps3vram.c
@@ -593,7 +593,8 @@ out:
593 next = bio_list_peek(&priv->list); 593 next = bio_list_peek(&priv->list);
594 spin_unlock_irq(&priv->lock); 594 spin_unlock_irq(&priv->lock);
595 595
596 bio_endio(bio, error); 596 bio->bi_error = error;
597 bio_endio(bio);
597 return next; 598 return next;
598} 599}
599 600
@@ -605,6 +606,8 @@ static void ps3vram_make_request(struct request_queue *q, struct bio *bio)
605 606
606 dev_dbg(&dev->core, "%s\n", __func__); 607 dev_dbg(&dev->core, "%s\n", __func__);
607 608
609 blk_queue_split(q, &bio, q->bio_split);
610
608 spin_lock_irq(&priv->lock); 611 spin_lock_irq(&priv->lock);
609 busy = !bio_list_empty(&priv->list); 612 busy = !bio_list_empty(&priv->list);
610 bio_list_add(&priv->list, bio); 613 bio_list_add(&priv->list, bio);
diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index bc67a93aa4f4..698f761037ce 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -3474,52 +3474,6 @@ static int rbd_queue_rq(struct blk_mq_hw_ctx *hctx,
3474 return BLK_MQ_RQ_QUEUE_OK; 3474 return BLK_MQ_RQ_QUEUE_OK;
3475} 3475}
3476 3476
3477/*
3478 * a queue callback. Makes sure that we don't create a bio that spans across
3479 * multiple osd objects. One exception would be with a single page bios,
3480 * which we handle later at bio_chain_clone_range()
3481 */
3482static int rbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bmd,
3483 struct bio_vec *bvec)
3484{
3485 struct rbd_device *rbd_dev = q->queuedata;
3486 sector_t sector_offset;
3487 sector_t sectors_per_obj;
3488 sector_t obj_sector_offset;
3489 int ret;
3490
3491 /*
3492 * Find how far into its rbd object the partition-relative
3493 * bio start sector is to offset relative to the enclosing
3494 * device.
3495 */
3496 sector_offset = get_start_sect(bmd->bi_bdev) + bmd->bi_sector;
3497 sectors_per_obj = 1 << (rbd_dev->header.obj_order - SECTOR_SHIFT);
3498 obj_sector_offset = sector_offset & (sectors_per_obj - 1);
3499
3500 /*
3501 * Compute the number of bytes from that offset to the end
3502 * of the object. Account for what's already used by the bio.
3503 */
3504 ret = (int) (sectors_per_obj - obj_sector_offset) << SECTOR_SHIFT;
3505 if (ret > bmd->bi_size)
3506 ret -= bmd->bi_size;
3507 else
3508 ret = 0;
3509
3510 /*
3511 * Don't send back more than was asked for. And if the bio
3512 * was empty, let the whole thing through because: "Note
3513 * that a block device *must* allow a single page to be
3514 * added to an empty bio."
3515 */
3516 rbd_assert(bvec->bv_len <= PAGE_SIZE);
3517 if (ret > (int) bvec->bv_len || !bmd->bi_size)
3518 ret = (int) bvec->bv_len;
3519
3520 return ret;
3521}
3522
3523static void rbd_free_disk(struct rbd_device *rbd_dev) 3477static void rbd_free_disk(struct rbd_device *rbd_dev)
3524{ 3478{
3525 struct gendisk *disk = rbd_dev->disk; 3479 struct gendisk *disk = rbd_dev->disk;
@@ -3815,10 +3769,9 @@ static int rbd_init_disk(struct rbd_device *rbd_dev)
3815 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); 3769 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
3816 q->limits.discard_granularity = segment_size; 3770 q->limits.discard_granularity = segment_size;
3817 q->limits.discard_alignment = segment_size; 3771 q->limits.discard_alignment = segment_size;
3818 q->limits.max_discard_sectors = segment_size / SECTOR_SIZE; 3772 blk_queue_max_discard_sectors(q, segment_size / SECTOR_SIZE);
3819 q->limits.discard_zeroes_data = 1; 3773 q->limits.discard_zeroes_data = 1;
3820 3774
3821 blk_queue_merge_bvec(q, rbd_merge_bvec);
3822 disk->queue = q; 3775 disk->queue = q;
3823 3776
3824 q->queuedata = rbd_dev; 3777 q->queuedata = rbd_dev;
diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c
index ac8c62cb4875..3163e4cdc2cc 100644
--- a/drivers/block/rsxx/dev.c
+++ b/drivers/block/rsxx/dev.c
@@ -137,7 +137,10 @@ static void bio_dma_done_cb(struct rsxx_cardinfo *card,
137 if (!card->eeh_state && card->gendisk) 137 if (!card->eeh_state && card->gendisk)
138 disk_stats_complete(card, meta->bio, meta->start_time); 138 disk_stats_complete(card, meta->bio, meta->start_time);
139 139
140 bio_endio(meta->bio, atomic_read(&meta->error) ? -EIO : 0); 140 if (atomic_read(&meta->error))
141 bio_io_error(meta->bio);
142 else
143 bio_endio(meta->bio);
141 kmem_cache_free(bio_meta_pool, meta); 144 kmem_cache_free(bio_meta_pool, meta);
142 } 145 }
143} 146}
@@ -148,6 +151,8 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio)
148 struct rsxx_bio_meta *bio_meta; 151 struct rsxx_bio_meta *bio_meta;
149 int st = -EINVAL; 152 int st = -EINVAL;
150 153
154 blk_queue_split(q, &bio, q->bio_split);
155
151 might_sleep(); 156 might_sleep();
152 157
153 if (!card) 158 if (!card)
@@ -199,7 +204,9 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio)
199queue_err: 204queue_err:
200 kmem_cache_free(bio_meta_pool, bio_meta); 205 kmem_cache_free(bio_meta_pool, bio_meta);
201req_err: 206req_err:
202 bio_endio(bio, st); 207 if (st)
208 bio->bi_error = st;
209 bio_endio(bio);
203} 210}
204 211
205/*----------------- Device Setup -------------------*/ 212/*----------------- Device Setup -------------------*/
diff --git a/drivers/block/skd_main.c b/drivers/block/skd_main.c
index 1e46eb2305c0..586f9168ffa4 100644
--- a/drivers/block/skd_main.c
+++ b/drivers/block/skd_main.c
@@ -4422,7 +4422,7 @@ static int skd_cons_disk(struct skd_device *skdev)
4422 /* DISCARD Flag initialization. */ 4422 /* DISCARD Flag initialization. */
4423 q->limits.discard_granularity = 8192; 4423 q->limits.discard_granularity = 8192;
4424 q->limits.discard_alignment = 0; 4424 q->limits.discard_alignment = 0;
4425 q->limits.max_discard_sectors = UINT_MAX >> 9; 4425 blk_queue_max_discard_sectors(q, UINT_MAX >> 9);
4426 q->limits.discard_zeroes_data = 1; 4426 q->limits.discard_zeroes_data = 1;
4427 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); 4427 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
4428 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q); 4428 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
diff --git a/drivers/block/umem.c b/drivers/block/umem.c
index 4cf81b5bf0f7..04d65790a886 100644
--- a/drivers/block/umem.c
+++ b/drivers/block/umem.c
@@ -456,7 +456,7 @@ static void process_page(unsigned long data)
456 PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE); 456 PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE);
457 if (control & DMASCR_HARD_ERROR) { 457 if (control & DMASCR_HARD_ERROR) {
458 /* error */ 458 /* error */
459 clear_bit(BIO_UPTODATE, &bio->bi_flags); 459 bio->bi_error = -EIO;
460 dev_printk(KERN_WARNING, &card->dev->dev, 460 dev_printk(KERN_WARNING, &card->dev->dev,
461 "I/O error on sector %d/%d\n", 461 "I/O error on sector %d/%d\n",
462 le32_to_cpu(desc->local_addr)>>9, 462 le32_to_cpu(desc->local_addr)>>9,
@@ -505,7 +505,7 @@ static void process_page(unsigned long data)
505 505
506 return_bio = bio->bi_next; 506 return_bio = bio->bi_next;
507 bio->bi_next = NULL; 507 bio->bi_next = NULL;
508 bio_endio(bio, 0); 508 bio_endio(bio);
509 } 509 }
510} 510}
511 511
@@ -531,6 +531,8 @@ static void mm_make_request(struct request_queue *q, struct bio *bio)
531 (unsigned long long)bio->bi_iter.bi_sector, 531 (unsigned long long)bio->bi_iter.bi_sector,
532 bio->bi_iter.bi_size); 532 bio->bi_iter.bi_size);
533 533
534 blk_queue_split(q, &bio, q->bio_split);
535
534 spin_lock_irq(&card->lock); 536 spin_lock_irq(&card->lock);
535 *card->biotail = bio; 537 *card->biotail = bio;
536 bio->bi_next = NULL; 538 bio->bi_next = NULL;
diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c
index 954c0029fb3b..6a685aec6994 100644
--- a/drivers/block/xen-blkback/blkback.c
+++ b/drivers/block/xen-blkback/blkback.c
@@ -1078,9 +1078,9 @@ static void __end_block_io_op(struct pending_req *pending_req, int error)
1078/* 1078/*
1079 * bio callback. 1079 * bio callback.
1080 */ 1080 */
1081static void end_block_io_op(struct bio *bio, int error) 1081static void end_block_io_op(struct bio *bio)
1082{ 1082{
1083 __end_block_io_op(bio->bi_private, error); 1083 __end_block_io_op(bio->bi_private, bio->bi_error);
1084 bio_put(bio); 1084 bio_put(bio);
1085} 1085}
1086 1086
diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c
index 7a8a73f1fc04..5f6b3be0a93c 100644
--- a/drivers/block/xen-blkfront.c
+++ b/drivers/block/xen-blkfront.c
@@ -82,7 +82,6 @@ struct blk_shadow {
82struct split_bio { 82struct split_bio {
83 struct bio *bio; 83 struct bio *bio;
84 atomic_t pending; 84 atomic_t pending;
85 int err;
86}; 85};
87 86
88static DEFINE_MUTEX(blkfront_mutex); 87static DEFINE_MUTEX(blkfront_mutex);
@@ -1481,16 +1480,14 @@ static int blkfront_probe(struct xenbus_device *dev,
1481 return 0; 1480 return 0;
1482} 1481}
1483 1482
1484static void split_bio_end(struct bio *bio, int error) 1483static void split_bio_end(struct bio *bio)
1485{ 1484{
1486 struct split_bio *split_bio = bio->bi_private; 1485 struct split_bio *split_bio = bio->bi_private;
1487 1486
1488 if (error)
1489 split_bio->err = error;
1490
1491 if (atomic_dec_and_test(&split_bio->pending)) { 1487 if (atomic_dec_and_test(&split_bio->pending)) {
1492 split_bio->bio->bi_phys_segments = 0; 1488 split_bio->bio->bi_phys_segments = 0;
1493 bio_endio(split_bio->bio, split_bio->err); 1489 split_bio->bio->bi_error = bio->bi_error;
1490 bio_endio(split_bio->bio);
1494 kfree(split_bio); 1491 kfree(split_bio);
1495 } 1492 }
1496 bio_put(bio); 1493 bio_put(bio);
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index 763301c7828c..9c01f5bfa33f 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -848,7 +848,7 @@ static void __zram_make_request(struct zram *zram, struct bio *bio)
848 848
849 if (unlikely(bio->bi_rw & REQ_DISCARD)) { 849 if (unlikely(bio->bi_rw & REQ_DISCARD)) {
850 zram_bio_discard(zram, index, offset, bio); 850 zram_bio_discard(zram, index, offset, bio);
851 bio_endio(bio, 0); 851 bio_endio(bio);
852 return; 852 return;
853 } 853 }
854 854
@@ -881,8 +881,7 @@ static void __zram_make_request(struct zram *zram, struct bio *bio)
881 update_position(&index, &offset, &bvec); 881 update_position(&index, &offset, &bvec);
882 } 882 }
883 883
884 set_bit(BIO_UPTODATE, &bio->bi_flags); 884 bio_endio(bio);
885 bio_endio(bio, 0);
886 return; 885 return;
887 886
888out: 887out:
@@ -899,6 +898,8 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio)
899 if (unlikely(!zram_meta_get(zram))) 898 if (unlikely(!zram_meta_get(zram)))
900 goto error; 899 goto error;
901 900
901 blk_queue_split(queue, &bio, queue->bio_split);
902
902 if (!valid_io_request(zram, bio->bi_iter.bi_sector, 903 if (!valid_io_request(zram, bio->bi_iter.bi_sector,
903 bio->bi_iter.bi_size)) { 904 bio->bi_iter.bi_size)) {
904 atomic64_inc(&zram->stats.invalid_io); 905 atomic64_inc(&zram->stats.invalid_io);
@@ -1242,7 +1243,7 @@ static int zram_add(void)
1242 blk_queue_io_min(zram->disk->queue, PAGE_SIZE); 1243 blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
1243 blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); 1244 blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
1244 zram->disk->queue->limits.discard_granularity = PAGE_SIZE; 1245 zram->disk->queue->limits.discard_granularity = PAGE_SIZE;
1245 zram->disk->queue->limits.max_discard_sectors = UINT_MAX; 1246 blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX);
1246 /* 1247 /*
1247 * zram_bio_discard() will clear all logical blocks if logical block 1248 * zram_bio_discard() will clear all logical blocks if logical block
1248 * size is identical with physical block size(PAGE_SIZE). But if it is 1249 * size is identical with physical block size(PAGE_SIZE). But if it is
diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index 04f7bc28ef83..6b420a55c745 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -243,19 +243,6 @@ struct keybuf {
243 DECLARE_ARRAY_ALLOCATOR(struct keybuf_key, freelist, KEYBUF_NR); 243 DECLARE_ARRAY_ALLOCATOR(struct keybuf_key, freelist, KEYBUF_NR);
244}; 244};
245 245
246struct bio_split_pool {
247 struct bio_set *bio_split;
248 mempool_t *bio_split_hook;
249};
250
251struct bio_split_hook {
252 struct closure cl;
253 struct bio_split_pool *p;
254 struct bio *bio;
255 bio_end_io_t *bi_end_io;
256 void *bi_private;
257};
258
259struct bcache_device { 246struct bcache_device {
260 struct closure cl; 247 struct closure cl;
261 248
@@ -288,8 +275,6 @@ struct bcache_device {
288 int (*cache_miss)(struct btree *, struct search *, 275 int (*cache_miss)(struct btree *, struct search *,
289 struct bio *, unsigned); 276 struct bio *, unsigned);
290 int (*ioctl) (struct bcache_device *, fmode_t, unsigned, unsigned long); 277 int (*ioctl) (struct bcache_device *, fmode_t, unsigned, unsigned long);
291
292 struct bio_split_pool bio_split_hook;
293}; 278};
294 279
295struct io { 280struct io {
@@ -454,8 +439,6 @@ struct cache {
454 atomic_long_t meta_sectors_written; 439 atomic_long_t meta_sectors_written;
455 atomic_long_t btree_sectors_written; 440 atomic_long_t btree_sectors_written;
456 atomic_long_t sectors_written; 441 atomic_long_t sectors_written;
457
458 struct bio_split_pool bio_split_hook;
459}; 442};
460 443
461struct gc_stat { 444struct gc_stat {
@@ -873,7 +856,6 @@ void bch_bbio_endio(struct cache_set *, struct bio *, int, const char *);
873void bch_bbio_free(struct bio *, struct cache_set *); 856void bch_bbio_free(struct bio *, struct cache_set *);
874struct bio *bch_bbio_alloc(struct cache_set *); 857struct bio *bch_bbio_alloc(struct cache_set *);
875 858
876void bch_generic_make_request(struct bio *, struct bio_split_pool *);
877void __bch_submit_bbio(struct bio *, struct cache_set *); 859void __bch_submit_bbio(struct bio *, struct cache_set *);
878void bch_submit_bbio(struct bio *, struct cache_set *, struct bkey *, unsigned); 860void bch_submit_bbio(struct bio *, struct cache_set *, struct bkey *, unsigned);
879 861
diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
index 00cde40db572..83392f856dfd 100644
--- a/drivers/md/bcache/btree.c
+++ b/drivers/md/bcache/btree.c
@@ -278,7 +278,7 @@ err:
278 goto out; 278 goto out;
279} 279}
280 280
281static void btree_node_read_endio(struct bio *bio, int error) 281static void btree_node_read_endio(struct bio *bio)
282{ 282{
283 struct closure *cl = bio->bi_private; 283 struct closure *cl = bio->bi_private;
284 closure_put(cl); 284 closure_put(cl);
@@ -305,7 +305,7 @@ static void bch_btree_node_read(struct btree *b)
305 bch_submit_bbio(bio, b->c, &b->key, 0); 305 bch_submit_bbio(bio, b->c, &b->key, 0);
306 closure_sync(&cl); 306 closure_sync(&cl);
307 307
308 if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) 308 if (bio->bi_error)
309 set_btree_node_io_error(b); 309 set_btree_node_io_error(b);
310 310
311 bch_bbio_free(bio, b->c); 311 bch_bbio_free(bio, b->c);
@@ -371,15 +371,15 @@ static void btree_node_write_done(struct closure *cl)
371 __btree_node_write_done(cl); 371 __btree_node_write_done(cl);
372} 372}
373 373
374static void btree_node_write_endio(struct bio *bio, int error) 374static void btree_node_write_endio(struct bio *bio)
375{ 375{
376 struct closure *cl = bio->bi_private; 376 struct closure *cl = bio->bi_private;
377 struct btree *b = container_of(cl, struct btree, io); 377 struct btree *b = container_of(cl, struct btree, io);
378 378
379 if (error) 379 if (bio->bi_error)
380 set_btree_node_io_error(b); 380 set_btree_node_io_error(b);
381 381
382 bch_bbio_count_io_errors(b->c, bio, error, "writing btree"); 382 bch_bbio_count_io_errors(b->c, bio, bio->bi_error, "writing btree");
383 closure_put(cl); 383 closure_put(cl);
384} 384}
385 385
diff --git a/drivers/md/bcache/closure.h b/drivers/md/bcache/closure.h
index 79a6d63e8ed3..782cc2c8a185 100644
--- a/drivers/md/bcache/closure.h
+++ b/drivers/md/bcache/closure.h
@@ -38,7 +38,7 @@
38 * they are running owned by the thread that is running them. Otherwise, suppose 38 * they are running owned by the thread that is running them. Otherwise, suppose
39 * you submit some bios and wish to have a function run when they all complete: 39 * you submit some bios and wish to have a function run when they all complete:
40 * 40 *
41 * foo_endio(struct bio *bio, int error) 41 * foo_endio(struct bio *bio)
42 * { 42 * {
43 * closure_put(cl); 43 * closure_put(cl);
44 * } 44 * }
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index bf6a9ca18403..86a0bb87124e 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -11,105 +11,6 @@
11 11
12#include <linux/blkdev.h> 12#include <linux/blkdev.h>
13 13
14static unsigned bch_bio_max_sectors(struct bio *bio)
15{
16 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
17 struct bio_vec bv;
18 struct bvec_iter iter;
19 unsigned ret = 0, seg = 0;
20
21 if (bio->bi_rw & REQ_DISCARD)
22 return min(bio_sectors(bio), q->limits.max_discard_sectors);
23
24 bio_for_each_segment(bv, bio, iter) {
25 struct bvec_merge_data bvm = {
26 .bi_bdev = bio->bi_bdev,
27 .bi_sector = bio->bi_iter.bi_sector,
28 .bi_size = ret << 9,
29 .bi_rw = bio->bi_rw,
30 };
31
32 if (seg == min_t(unsigned, BIO_MAX_PAGES,
33 queue_max_segments(q)))
34 break;
35
36 if (q->merge_bvec_fn &&
37 q->merge_bvec_fn(q, &bvm, &bv) < (int) bv.bv_len)
38 break;
39
40 seg++;
41 ret += bv.bv_len >> 9;
42 }
43
44 ret = min(ret, queue_max_sectors(q));
45
46 WARN_ON(!ret);
47 ret = max_t(int, ret, bio_iovec(bio).bv_len >> 9);
48
49 return ret;
50}
51
52static void bch_bio_submit_split_done(struct closure *cl)
53{
54 struct bio_split_hook *s = container_of(cl, struct bio_split_hook, cl);
55
56 s->bio->bi_end_io = s->bi_end_io;
57 s->bio->bi_private = s->bi_private;
58 bio_endio(s->bio, 0);
59
60 closure_debug_destroy(&s->cl);
61 mempool_free(s, s->p->bio_split_hook);
62}
63
64static void bch_bio_submit_split_endio(struct bio *bio, int error)
65{
66 struct closure *cl = bio->bi_private;
67 struct bio_split_hook *s = container_of(cl, struct bio_split_hook, cl);
68
69 if (error)
70 clear_bit(BIO_UPTODATE, &s->bio->bi_flags);
71
72 bio_put(bio);
73 closure_put(cl);
74}
75
76void bch_generic_make_request(struct bio *bio, struct bio_split_pool *p)
77{
78 struct bio_split_hook *s;
79 struct bio *n;
80
81 if (!bio_has_data(bio) && !(bio->bi_rw & REQ_DISCARD))
82 goto submit;
83
84 if (bio_sectors(bio) <= bch_bio_max_sectors(bio))
85 goto submit;
86
87 s = mempool_alloc(p->bio_split_hook, GFP_NOIO);
88 closure_init(&s->cl, NULL);
89
90 s->bio = bio;
91 s->p = p;
92 s->bi_end_io = bio->bi_end_io;
93 s->bi_private = bio->bi_private;
94 bio_get(bio);
95
96 do {
97 n = bio_next_split(bio, bch_bio_max_sectors(bio),
98 GFP_NOIO, s->p->bio_split);
99
100 n->bi_end_io = bch_bio_submit_split_endio;
101 n->bi_private = &s->cl;
102
103 closure_get(&s->cl);
104 generic_make_request(n);
105 } while (n != bio);
106
107 continue_at(&s->cl, bch_bio_submit_split_done, NULL);
108 return;
109submit:
110 generic_make_request(bio);
111}
112
113/* Bios with headers */ 14/* Bios with headers */
114 15
115void bch_bbio_free(struct bio *bio, struct cache_set *c) 16void bch_bbio_free(struct bio *bio, struct cache_set *c)
@@ -139,7 +40,7 @@ void __bch_submit_bbio(struct bio *bio, struct cache_set *c)
139 bio->bi_bdev = PTR_CACHE(c, &b->key, 0)->bdev; 40 bio->bi_bdev = PTR_CACHE(c, &b->key, 0)->bdev;
140 41
141 b->submit_time_us = local_clock_us(); 42 b->submit_time_us = local_clock_us();
142 closure_bio_submit(bio, bio->bi_private, PTR_CACHE(c, &b->key, 0)); 43 closure_bio_submit(bio, bio->bi_private);
143} 44}
144 45
145void bch_submit_bbio(struct bio *bio, struct cache_set *c, 46void bch_submit_bbio(struct bio *bio, struct cache_set *c,
diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c
index 418607a6ba33..29eba7219b01 100644
--- a/drivers/md/bcache/journal.c
+++ b/drivers/md/bcache/journal.c
@@ -24,7 +24,7 @@
24 * bit. 24 * bit.
25 */ 25 */
26 26
27static void journal_read_endio(struct bio *bio, int error) 27static void journal_read_endio(struct bio *bio)
28{ 28{
29 struct closure *cl = bio->bi_private; 29 struct closure *cl = bio->bi_private;
30 closure_put(cl); 30 closure_put(cl);
@@ -61,7 +61,7 @@ reread: left = ca->sb.bucket_size - offset;
61 bio->bi_private = &cl; 61 bio->bi_private = &cl;
62 bch_bio_map(bio, data); 62 bch_bio_map(bio, data);
63 63
64 closure_bio_submit(bio, &cl, ca); 64 closure_bio_submit(bio, &cl);
65 closure_sync(&cl); 65 closure_sync(&cl);
66 66
67 /* This function could be simpler now since we no longer write 67 /* This function could be simpler now since we no longer write
@@ -401,7 +401,7 @@ retry:
401 401
402#define last_seq(j) ((j)->seq - fifo_used(&(j)->pin) + 1) 402#define last_seq(j) ((j)->seq - fifo_used(&(j)->pin) + 1)
403 403
404static void journal_discard_endio(struct bio *bio, int error) 404static void journal_discard_endio(struct bio *bio)
405{ 405{
406 struct journal_device *ja = 406 struct journal_device *ja =
407 container_of(bio, struct journal_device, discard_bio); 407 container_of(bio, struct journal_device, discard_bio);
@@ -547,11 +547,11 @@ void bch_journal_next(struct journal *j)
547 pr_debug("journal_pin full (%zu)", fifo_used(&j->pin)); 547 pr_debug("journal_pin full (%zu)", fifo_used(&j->pin));
548} 548}
549 549
550static void journal_write_endio(struct bio *bio, int error) 550static void journal_write_endio(struct bio *bio)
551{ 551{
552 struct journal_write *w = bio->bi_private; 552 struct journal_write *w = bio->bi_private;
553 553
554 cache_set_err_on(error, w->c, "journal io error"); 554 cache_set_err_on(bio->bi_error, w->c, "journal io error");
555 closure_put(&w->c->journal.io); 555 closure_put(&w->c->journal.io);
556} 556}
557 557
@@ -648,7 +648,7 @@ static void journal_write_unlocked(struct closure *cl)
648 spin_unlock(&c->journal.lock); 648 spin_unlock(&c->journal.lock);
649 649
650 while ((bio = bio_list_pop(&list))) 650 while ((bio = bio_list_pop(&list)))
651 closure_bio_submit(bio, cl, c->cache[0]); 651 closure_bio_submit(bio, cl);
652 652
653 continue_at(cl, journal_write_done, NULL); 653 continue_at(cl, journal_write_done, NULL);
654} 654}
diff --git a/drivers/md/bcache/movinggc.c b/drivers/md/bcache/movinggc.c
index cd7490311e51..b929fc944e9c 100644
--- a/drivers/md/bcache/movinggc.c
+++ b/drivers/md/bcache/movinggc.c
@@ -60,20 +60,20 @@ static void write_moving_finish(struct closure *cl)
60 closure_return_with_destructor(cl, moving_io_destructor); 60 closure_return_with_destructor(cl, moving_io_destructor);
61} 61}
62 62
63static void read_moving_endio(struct bio *bio, int error) 63static void read_moving_endio(struct bio *bio)
64{ 64{
65 struct bbio *b = container_of(bio, struct bbio, bio); 65 struct bbio *b = container_of(bio, struct bbio, bio);
66 struct moving_io *io = container_of(bio->bi_private, 66 struct moving_io *io = container_of(bio->bi_private,
67 struct moving_io, cl); 67 struct moving_io, cl);
68 68
69 if (error) 69 if (bio->bi_error)
70 io->op.error = error; 70 io->op.error = bio->bi_error;
71 else if (!KEY_DIRTY(&b->key) && 71 else if (!KEY_DIRTY(&b->key) &&
72 ptr_stale(io->op.c, &b->key, 0)) { 72 ptr_stale(io->op.c, &b->key, 0)) {
73 io->op.error = -EINTR; 73 io->op.error = -EINTR;
74 } 74 }
75 75
76 bch_bbio_endio(io->op.c, bio, error, "reading data to move"); 76 bch_bbio_endio(io->op.c, bio, bio->bi_error, "reading data to move");
77} 77}
78 78
79static void moving_init(struct moving_io *io) 79static void moving_init(struct moving_io *io)
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index f292790997d7..8e9877b04637 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -173,22 +173,22 @@ static void bch_data_insert_error(struct closure *cl)
173 bch_data_insert_keys(cl); 173 bch_data_insert_keys(cl);
174} 174}
175 175
176static void bch_data_insert_endio(struct bio *bio, int error) 176static void bch_data_insert_endio(struct bio *bio)
177{ 177{
178 struct closure *cl = bio->bi_private; 178 struct closure *cl = bio->bi_private;
179 struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); 179 struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
180 180
181 if (error) { 181 if (bio->bi_error) {
182 /* TODO: We could try to recover from this. */ 182 /* TODO: We could try to recover from this. */
183 if (op->writeback) 183 if (op->writeback)
184 op->error = error; 184 op->error = bio->bi_error;
185 else if (!op->replace) 185 else if (!op->replace)
186 set_closure_fn(cl, bch_data_insert_error, op->wq); 186 set_closure_fn(cl, bch_data_insert_error, op->wq);
187 else 187 else
188 set_closure_fn(cl, NULL, NULL); 188 set_closure_fn(cl, NULL, NULL);
189 } 189 }
190 190
191 bch_bbio_endio(op->c, bio, error, "writing data to cache"); 191 bch_bbio_endio(op->c, bio, bio->bi_error, "writing data to cache");
192} 192}
193 193
194static void bch_data_insert_start(struct closure *cl) 194static void bch_data_insert_start(struct closure *cl)
@@ -477,7 +477,7 @@ struct search {
477 struct data_insert_op iop; 477 struct data_insert_op iop;
478}; 478};
479 479
480static void bch_cache_read_endio(struct bio *bio, int error) 480static void bch_cache_read_endio(struct bio *bio)
481{ 481{
482 struct bbio *b = container_of(bio, struct bbio, bio); 482 struct bbio *b = container_of(bio, struct bbio, bio);
483 struct closure *cl = bio->bi_private; 483 struct closure *cl = bio->bi_private;
@@ -490,15 +490,15 @@ static void bch_cache_read_endio(struct bio *bio, int error)
490 * from the backing device. 490 * from the backing device.
491 */ 491 */
492 492
493 if (error) 493 if (bio->bi_error)
494 s->iop.error = error; 494 s->iop.error = bio->bi_error;
495 else if (!KEY_DIRTY(&b->key) && 495 else if (!KEY_DIRTY(&b->key) &&
496 ptr_stale(s->iop.c, &b->key, 0)) { 496 ptr_stale(s->iop.c, &b->key, 0)) {
497 atomic_long_inc(&s->iop.c->cache_read_races); 497 atomic_long_inc(&s->iop.c->cache_read_races);
498 s->iop.error = -EINTR; 498 s->iop.error = -EINTR;
499 } 499 }
500 500
501 bch_bbio_endio(s->iop.c, bio, error, "reading from cache"); 501 bch_bbio_endio(s->iop.c, bio, bio->bi_error, "reading from cache");
502} 502}
503 503
504/* 504/*
@@ -591,13 +591,13 @@ static void cache_lookup(struct closure *cl)
591 591
592/* Common code for the make_request functions */ 592/* Common code for the make_request functions */
593 593
594static void request_endio(struct bio *bio, int error) 594static void request_endio(struct bio *bio)
595{ 595{
596 struct closure *cl = bio->bi_private; 596 struct closure *cl = bio->bi_private;
597 597
598 if (error) { 598 if (bio->bi_error) {
599 struct search *s = container_of(cl, struct search, cl); 599 struct search *s = container_of(cl, struct search, cl);
600 s->iop.error = error; 600 s->iop.error = bio->bi_error;
601 /* Only cache read errors are recoverable */ 601 /* Only cache read errors are recoverable */
602 s->recoverable = false; 602 s->recoverable = false;
603 } 603 }
@@ -613,7 +613,8 @@ static void bio_complete(struct search *s)
613 &s->d->disk->part0, s->start_time); 613 &s->d->disk->part0, s->start_time);
614 614
615 trace_bcache_request_end(s->d, s->orig_bio); 615 trace_bcache_request_end(s->d, s->orig_bio);
616 bio_endio(s->orig_bio, s->iop.error); 616 s->orig_bio->bi_error = s->iop.error;
617 bio_endio(s->orig_bio);
617 s->orig_bio = NULL; 618 s->orig_bio = NULL;
618 } 619 }
619} 620}
@@ -718,7 +719,7 @@ static void cached_dev_read_error(struct closure *cl)
718 719
719 /* XXX: invalidate cache */ 720 /* XXX: invalidate cache */
720 721
721 closure_bio_submit(bio, cl, s->d); 722 closure_bio_submit(bio, cl);
722 } 723 }
723 724
724 continue_at(cl, cached_dev_cache_miss_done, NULL); 725 continue_at(cl, cached_dev_cache_miss_done, NULL);
@@ -841,7 +842,7 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s,
841 s->cache_miss = miss; 842 s->cache_miss = miss;
842 s->iop.bio = cache_bio; 843 s->iop.bio = cache_bio;
843 bio_get(cache_bio); 844 bio_get(cache_bio);
844 closure_bio_submit(cache_bio, &s->cl, s->d); 845 closure_bio_submit(cache_bio, &s->cl);
845 846
846 return ret; 847 return ret;
847out_put: 848out_put:
@@ -849,7 +850,7 @@ out_put:
849out_submit: 850out_submit:
850 miss->bi_end_io = request_endio; 851 miss->bi_end_io = request_endio;
851 miss->bi_private = &s->cl; 852 miss->bi_private = &s->cl;
852 closure_bio_submit(miss, &s->cl, s->d); 853 closure_bio_submit(miss, &s->cl);
853 return ret; 854 return ret;
854} 855}
855 856
@@ -914,7 +915,7 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)
914 915
915 if (!(bio->bi_rw & REQ_DISCARD) || 916 if (!(bio->bi_rw & REQ_DISCARD) ||
916 blk_queue_discard(bdev_get_queue(dc->bdev))) 917 blk_queue_discard(bdev_get_queue(dc->bdev)))
917 closure_bio_submit(bio, cl, s->d); 918 closure_bio_submit(bio, cl);
918 } else if (s->iop.writeback) { 919 } else if (s->iop.writeback) {
919 bch_writeback_add(dc); 920 bch_writeback_add(dc);
920 s->iop.bio = bio; 921 s->iop.bio = bio;
@@ -929,12 +930,12 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)
929 flush->bi_end_io = request_endio; 930 flush->bi_end_io = request_endio;
930 flush->bi_private = cl; 931 flush->bi_private = cl;
931 932
932 closure_bio_submit(flush, cl, s->d); 933 closure_bio_submit(flush, cl);
933 } 934 }
934 } else { 935 } else {
935 s->iop.bio = bio_clone_fast(bio, GFP_NOIO, dc->disk.bio_split); 936 s->iop.bio = bio_clone_fast(bio, GFP_NOIO, dc->disk.bio_split);
936 937
937 closure_bio_submit(bio, cl, s->d); 938 closure_bio_submit(bio, cl);
938 } 939 }
939 940
940 closure_call(&s->iop.cl, bch_data_insert, NULL, cl); 941 closure_call(&s->iop.cl, bch_data_insert, NULL, cl);
@@ -950,7 +951,7 @@ static void cached_dev_nodata(struct closure *cl)
950 bch_journal_meta(s->iop.c, cl); 951 bch_journal_meta(s->iop.c, cl);
951 952
952 /* If it's a flush, we send the flush to the backing device too */ 953 /* If it's a flush, we send the flush to the backing device too */
953 closure_bio_submit(bio, cl, s->d); 954 closure_bio_submit(bio, cl);
954 955
955 continue_at(cl, cached_dev_bio_complete, NULL); 956 continue_at(cl, cached_dev_bio_complete, NULL);
956} 957}
@@ -992,9 +993,9 @@ static void cached_dev_make_request(struct request_queue *q, struct bio *bio)
992 } else { 993 } else {
993 if ((bio->bi_rw & REQ_DISCARD) && 994 if ((bio->bi_rw & REQ_DISCARD) &&
994 !blk_queue_discard(bdev_get_queue(dc->bdev))) 995 !blk_queue_discard(bdev_get_queue(dc->bdev)))
995 bio_endio(bio, 0); 996 bio_endio(bio);
996 else 997 else
997 bch_generic_make_request(bio, &d->bio_split_hook); 998 generic_make_request(bio);
998 } 999 }
999} 1000}
1000 1001
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index 94980bfca434..679a093a3bf6 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -59,29 +59,6 @@ struct workqueue_struct *bcache_wq;
59 59
60#define BTREE_MAX_PAGES (256 * 1024 / PAGE_SIZE) 60#define BTREE_MAX_PAGES (256 * 1024 / PAGE_SIZE)
61 61
62static void bio_split_pool_free(struct bio_split_pool *p)
63{
64 if (p->bio_split_hook)
65 mempool_destroy(p->bio_split_hook);
66
67 if (p->bio_split)
68 bioset_free(p->bio_split);
69}
70
71static int bio_split_pool_init(struct bio_split_pool *p)
72{
73 p->bio_split = bioset_create(4, 0);
74 if (!p->bio_split)
75 return -ENOMEM;
76
77 p->bio_split_hook = mempool_create_kmalloc_pool(4,
78 sizeof(struct bio_split_hook));
79 if (!p->bio_split_hook)
80 return -ENOMEM;
81
82 return 0;
83}
84
85/* Superblock */ 62/* Superblock */
86 63
87static const char *read_super(struct cache_sb *sb, struct block_device *bdev, 64static const char *read_super(struct cache_sb *sb, struct block_device *bdev,
@@ -221,7 +198,7 @@ err:
221 return err; 198 return err;
222} 199}
223 200
224static void write_bdev_super_endio(struct bio *bio, int error) 201static void write_bdev_super_endio(struct bio *bio)
225{ 202{
226 struct cached_dev *dc = bio->bi_private; 203 struct cached_dev *dc = bio->bi_private;
227 /* XXX: error checking */ 204 /* XXX: error checking */
@@ -290,11 +267,11 @@ void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent)
290 closure_return_with_destructor(cl, bch_write_bdev_super_unlock); 267 closure_return_with_destructor(cl, bch_write_bdev_super_unlock);
291} 268}
292 269
293static void write_super_endio(struct bio *bio, int error) 270static void write_super_endio(struct bio *bio)
294{ 271{
295 struct cache *ca = bio->bi_private; 272 struct cache *ca = bio->bi_private;
296 273
297 bch_count_io_errors(ca, error, "writing superblock"); 274 bch_count_io_errors(ca, bio->bi_error, "writing superblock");
298 closure_put(&ca->set->sb_write); 275 closure_put(&ca->set->sb_write);
299} 276}
300 277
@@ -339,12 +316,12 @@ void bcache_write_super(struct cache_set *c)
339 316
340/* UUID io */ 317/* UUID io */
341 318
342static void uuid_endio(struct bio *bio, int error) 319static void uuid_endio(struct bio *bio)
343{ 320{
344 struct closure *cl = bio->bi_private; 321 struct closure *cl = bio->bi_private;
345 struct cache_set *c = container_of(cl, struct cache_set, uuid_write); 322 struct cache_set *c = container_of(cl, struct cache_set, uuid_write);
346 323
347 cache_set_err_on(error, c, "accessing uuids"); 324 cache_set_err_on(bio->bi_error, c, "accessing uuids");
348 bch_bbio_free(bio, c); 325 bch_bbio_free(bio, c);
349 closure_put(cl); 326 closure_put(cl);
350} 327}
@@ -512,11 +489,11 @@ static struct uuid_entry *uuid_find_empty(struct cache_set *c)
512 * disk. 489 * disk.
513 */ 490 */
514 491
515static void prio_endio(struct bio *bio, int error) 492static void prio_endio(struct bio *bio)
516{ 493{
517 struct cache *ca = bio->bi_private; 494 struct cache *ca = bio->bi_private;
518 495
519 cache_set_err_on(error, ca->set, "accessing priorities"); 496 cache_set_err_on(bio->bi_error, ca->set, "accessing priorities");
520 bch_bbio_free(bio, ca->set); 497 bch_bbio_free(bio, ca->set);
521 closure_put(&ca->prio); 498 closure_put(&ca->prio);
522} 499}
@@ -537,7 +514,7 @@ static void prio_io(struct cache *ca, uint64_t bucket, unsigned long rw)
537 bio->bi_private = ca; 514 bio->bi_private = ca;
538 bch_bio_map(bio, ca->disk_buckets); 515 bch_bio_map(bio, ca->disk_buckets);
539 516
540 closure_bio_submit(bio, &ca->prio, ca); 517 closure_bio_submit(bio, &ca->prio);
541 closure_sync(cl); 518 closure_sync(cl);
542} 519}
543 520
@@ -757,7 +734,6 @@ static void bcache_device_free(struct bcache_device *d)
757 put_disk(d->disk); 734 put_disk(d->disk);
758 } 735 }
759 736
760 bio_split_pool_free(&d->bio_split_hook);
761 if (d->bio_split) 737 if (d->bio_split)
762 bioset_free(d->bio_split); 738 bioset_free(d->bio_split);
763 kvfree(d->full_dirty_stripes); 739 kvfree(d->full_dirty_stripes);
@@ -804,7 +780,6 @@ static int bcache_device_init(struct bcache_device *d, unsigned block_size,
804 return minor; 780 return minor;
805 781
806 if (!(d->bio_split = bioset_create(4, offsetof(struct bbio, bio))) || 782 if (!(d->bio_split = bioset_create(4, offsetof(struct bbio, bio))) ||
807 bio_split_pool_init(&d->bio_split_hook) ||
808 !(d->disk = alloc_disk(1))) { 783 !(d->disk = alloc_disk(1))) {
809 ida_simple_remove(&bcache_minor, minor); 784 ida_simple_remove(&bcache_minor, minor);
810 return -ENOMEM; 785 return -ENOMEM;
@@ -830,7 +805,7 @@ static int bcache_device_init(struct bcache_device *d, unsigned block_size,
830 q->limits.max_sectors = UINT_MAX; 805 q->limits.max_sectors = UINT_MAX;
831 q->limits.max_segment_size = UINT_MAX; 806 q->limits.max_segment_size = UINT_MAX;
832 q->limits.max_segments = BIO_MAX_PAGES; 807 q->limits.max_segments = BIO_MAX_PAGES;
833 q->limits.max_discard_sectors = UINT_MAX; 808 blk_queue_max_discard_sectors(q, UINT_MAX);
834 q->limits.discard_granularity = 512; 809 q->limits.discard_granularity = 512;
835 q->limits.io_min = block_size; 810 q->limits.io_min = block_size;
836 q->limits.logical_block_size = block_size; 811 q->limits.logical_block_size = block_size;
@@ -1793,8 +1768,6 @@ void bch_cache_release(struct kobject *kobj)
1793 ca->set->cache[ca->sb.nr_this_dev] = NULL; 1768 ca->set->cache[ca->sb.nr_this_dev] = NULL;
1794 } 1769 }
1795 1770
1796 bio_split_pool_free(&ca->bio_split_hook);
1797
1798 free_pages((unsigned long) ca->disk_buckets, ilog2(bucket_pages(ca))); 1771 free_pages((unsigned long) ca->disk_buckets, ilog2(bucket_pages(ca)));
1799 kfree(ca->prio_buckets); 1772 kfree(ca->prio_buckets);
1800 vfree(ca->buckets); 1773 vfree(ca->buckets);
@@ -1839,8 +1812,7 @@ static int cache_alloc(struct cache_sb *sb, struct cache *ca)
1839 ca->sb.nbuckets)) || 1812 ca->sb.nbuckets)) ||
1840 !(ca->prio_buckets = kzalloc(sizeof(uint64_t) * prio_buckets(ca) * 1813 !(ca->prio_buckets = kzalloc(sizeof(uint64_t) * prio_buckets(ca) *
1841 2, GFP_KERNEL)) || 1814 2, GFP_KERNEL)) ||
1842 !(ca->disk_buckets = alloc_bucket_pages(GFP_KERNEL, ca)) || 1815 !(ca->disk_buckets = alloc_bucket_pages(GFP_KERNEL, ca)))
1843 bio_split_pool_init(&ca->bio_split_hook))
1844 return -ENOMEM; 1816 return -ENOMEM;
1845 1817
1846 ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca); 1818 ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);
diff --git a/drivers/md/bcache/util.h b/drivers/md/bcache/util.h
index 1d04c4859c70..cf2cbc211d83 100644
--- a/drivers/md/bcache/util.h
+++ b/drivers/md/bcache/util.h
@@ -4,6 +4,7 @@
4 4
5#include <linux/blkdev.h> 5#include <linux/blkdev.h>
6#include <linux/errno.h> 6#include <linux/errno.h>
7#include <linux/blkdev.h>
7#include <linux/kernel.h> 8#include <linux/kernel.h>
8#include <linux/llist.h> 9#include <linux/llist.h>
9#include <linux/ratelimit.h> 10#include <linux/ratelimit.h>
@@ -570,10 +571,10 @@ static inline sector_t bdev_sectors(struct block_device *bdev)
570 return bdev->bd_inode->i_size >> 9; 571 return bdev->bd_inode->i_size >> 9;
571} 572}
572 573
573#define closure_bio_submit(bio, cl, dev) \ 574#define closure_bio_submit(bio, cl) \
574do { \ 575do { \
575 closure_get(cl); \ 576 closure_get(cl); \
576 bch_generic_make_request(bio, &(dev)->bio_split_hook); \ 577 generic_make_request(bio); \
577} while (0) 578} while (0)
578 579
579uint64_t bch_crc64_update(uint64_t, const void *, size_t); 580uint64_t bch_crc64_update(uint64_t, const void *, size_t);
diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c
index f1986bcd1bf0..b23f88d9f18c 100644
--- a/drivers/md/bcache/writeback.c
+++ b/drivers/md/bcache/writeback.c
@@ -166,12 +166,12 @@ static void write_dirty_finish(struct closure *cl)
166 closure_return_with_destructor(cl, dirty_io_destructor); 166 closure_return_with_destructor(cl, dirty_io_destructor);
167} 167}
168 168
169static void dirty_endio(struct bio *bio, int error) 169static void dirty_endio(struct bio *bio)
170{ 170{
171 struct keybuf_key *w = bio->bi_private; 171 struct keybuf_key *w = bio->bi_private;
172 struct dirty_io *io = w->private; 172 struct dirty_io *io = w->private;
173 173
174 if (error) 174 if (bio->bi_error)
175 SET_KEY_DIRTY(&w->key, false); 175 SET_KEY_DIRTY(&w->key, false);
176 176
177 closure_put(&io->cl); 177 closure_put(&io->cl);
@@ -188,27 +188,27 @@ static void write_dirty(struct closure *cl)
188 io->bio.bi_bdev = io->dc->bdev; 188 io->bio.bi_bdev = io->dc->bdev;
189 io->bio.bi_end_io = dirty_endio; 189 io->bio.bi_end_io = dirty_endio;
190 190
191 closure_bio_submit(&io->bio, cl, &io->dc->disk); 191 closure_bio_submit(&io->bio, cl);
192 192
193 continue_at(cl, write_dirty_finish, system_wq); 193 continue_at(cl, write_dirty_finish, system_wq);
194} 194}
195 195
196static void read_dirty_endio(struct bio *bio, int error) 196static void read_dirty_endio(struct bio *bio)
197{ 197{
198 struct keybuf_key *w = bio->bi_private; 198 struct keybuf_key *w = bio->bi_private;
199 struct dirty_io *io = w->private; 199 struct dirty_io *io = w->private;
200 200
201 bch_count_io_errors(PTR_CACHE(io->dc->disk.c, &w->key, 0), 201 bch_count_io_errors(PTR_CACHE(io->dc->disk.c, &w->key, 0),
202 error, "reading dirty data from cache"); 202 bio->bi_error, "reading dirty data from cache");
203 203
204 dirty_endio(bio, error); 204 dirty_endio(bio);
205} 205}
206 206
207static void read_dirty_submit(struct closure *cl) 207static void read_dirty_submit(struct closure *cl)
208{ 208{
209 struct dirty_io *io = container_of(cl, struct dirty_io, cl); 209 struct dirty_io *io = container_of(cl, struct dirty_io, cl);
210 210
211 closure_bio_submit(&io->bio, cl, &io->dc->disk); 211 closure_bio_submit(&io->bio, cl);
212 212
213 continue_at(cl, write_dirty, system_wq); 213 continue_at(cl, write_dirty, system_wq);
214} 214}
diff --git a/drivers/md/dm-bio-prison.c b/drivers/md/dm-bio-prison.c
index cd6d1d21e057..03af174485d3 100644
--- a/drivers/md/dm-bio-prison.c
+++ b/drivers/md/dm-bio-prison.c
@@ -236,8 +236,10 @@ void dm_cell_error(struct dm_bio_prison *prison,
236 bio_list_init(&bios); 236 bio_list_init(&bios);
237 dm_cell_release(prison, cell, &bios); 237 dm_cell_release(prison, cell, &bios);
238 238
239 while ((bio = bio_list_pop(&bios))) 239 while ((bio = bio_list_pop(&bios))) {
240 bio_endio(bio, error); 240 bio->bi_error = error;
241 bio_endio(bio);
242 }
241} 243}
242EXPORT_SYMBOL_GPL(dm_cell_error); 244EXPORT_SYMBOL_GPL(dm_cell_error);
243 245
diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index 86dbbc737402..83cc52eaf56d 100644
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -545,7 +545,8 @@ static void dmio_complete(unsigned long error, void *context)
545{ 545{
546 struct dm_buffer *b = context; 546 struct dm_buffer *b = context;
547 547
548 b->bio.bi_end_io(&b->bio, error ? -EIO : 0); 548 b->bio.bi_error = error ? -EIO : 0;
549 b->bio.bi_end_io(&b->bio);
549} 550}
550 551
551static void use_dmio(struct dm_buffer *b, int rw, sector_t block, 552static void use_dmio(struct dm_buffer *b, int rw, sector_t block,
@@ -575,13 +576,16 @@ static void use_dmio(struct dm_buffer *b, int rw, sector_t block,
575 b->bio.bi_end_io = end_io; 576 b->bio.bi_end_io = end_io;
576 577
577 r = dm_io(&io_req, 1, &region, NULL); 578 r = dm_io(&io_req, 1, &region, NULL);
578 if (r) 579 if (r) {
579 end_io(&b->bio, r); 580 b->bio.bi_error = r;
581 end_io(&b->bio);
582 }
580} 583}
581 584
582static void inline_endio(struct bio *bio, int error) 585static void inline_endio(struct bio *bio)
583{ 586{
584 bio_end_io_t *end_fn = bio->bi_private; 587 bio_end_io_t *end_fn = bio->bi_private;
588 int error = bio->bi_error;
585 589
586 /* 590 /*
587 * Reset the bio to free any attached resources 591 * Reset the bio to free any attached resources
@@ -589,7 +593,8 @@ static void inline_endio(struct bio *bio, int error)
589 */ 593 */
590 bio_reset(bio); 594 bio_reset(bio);
591 595
592 end_fn(bio, error); 596 bio->bi_error = error;
597 end_fn(bio);
593} 598}
594 599
595static void use_inline_bio(struct dm_buffer *b, int rw, sector_t block, 600static void use_inline_bio(struct dm_buffer *b, int rw, sector_t block,
@@ -661,13 +666,14 @@ static void submit_io(struct dm_buffer *b, int rw, sector_t block,
661 * Set the error, clear B_WRITING bit and wake anyone who was waiting on 666 * Set the error, clear B_WRITING bit and wake anyone who was waiting on
662 * it. 667 * it.
663 */ 668 */
664static void write_endio(struct bio *bio, int error) 669static void write_endio(struct bio *bio)
665{ 670{
666 struct dm_buffer *b = container_of(bio, struct dm_buffer, bio); 671 struct dm_buffer *b = container_of(bio, struct dm_buffer, bio);
667 672
668 b->write_error = error; 673 b->write_error = bio->bi_error;
669 if (unlikely(error)) { 674 if (unlikely(bio->bi_error)) {
670 struct dm_bufio_client *c = b->c; 675 struct dm_bufio_client *c = b->c;
676 int error = bio->bi_error;
671 (void)cmpxchg(&c->async_write_error, 0, error); 677 (void)cmpxchg(&c->async_write_error, 0, error);
672 } 678 }
673 679
@@ -1026,11 +1032,11 @@ found_buffer:
1026 * The endio routine for reading: set the error, clear the bit and wake up 1032 * The endio routine for reading: set the error, clear the bit and wake up
1027 * anyone waiting on the buffer. 1033 * anyone waiting on the buffer.
1028 */ 1034 */
1029static void read_endio(struct bio *bio, int error) 1035static void read_endio(struct bio *bio)
1030{ 1036{
1031 struct dm_buffer *b = container_of(bio, struct dm_buffer, bio); 1037 struct dm_buffer *b = container_of(bio, struct dm_buffer, bio);
1032 1038
1033 b->read_error = error; 1039 b->read_error = bio->bi_error;
1034 1040
1035 BUG_ON(!test_bit(B_READING, &b->state)); 1041 BUG_ON(!test_bit(B_READING, &b->state));
1036 1042
diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index 1fe93cfea7d3..7245071778db 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -919,14 +919,14 @@ static void defer_writethrough_bio(struct cache *cache, struct bio *bio)
919 wake_worker(cache); 919 wake_worker(cache);
920} 920}
921 921
922static void writethrough_endio(struct bio *bio, int err) 922static void writethrough_endio(struct bio *bio)
923{ 923{
924 struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT); 924 struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT);
925 925
926 dm_unhook_bio(&pb->hook_info, bio); 926 dm_unhook_bio(&pb->hook_info, bio);
927 927
928 if (err) { 928 if (bio->bi_error) {
929 bio_endio(bio, err); 929 bio_endio(bio);
930 return; 930 return;
931 } 931 }
932 932
@@ -1231,7 +1231,7 @@ static void migration_success_post_commit(struct dm_cache_migration *mg)
1231 * The block was promoted via an overwrite, so it's dirty. 1231 * The block was promoted via an overwrite, so it's dirty.
1232 */ 1232 */
1233 set_dirty(cache, mg->new_oblock, mg->cblock); 1233 set_dirty(cache, mg->new_oblock, mg->cblock);
1234 bio_endio(mg->new_ocell->holder, 0); 1234 bio_endio(mg->new_ocell->holder);
1235 cell_defer(cache, mg->new_ocell, false); 1235 cell_defer(cache, mg->new_ocell, false);
1236 } 1236 }
1237 free_io_migration(mg); 1237 free_io_migration(mg);
@@ -1284,7 +1284,7 @@ static void issue_copy(struct dm_cache_migration *mg)
1284 } 1284 }
1285} 1285}
1286 1286
1287static void overwrite_endio(struct bio *bio, int err) 1287static void overwrite_endio(struct bio *bio)
1288{ 1288{
1289 struct dm_cache_migration *mg = bio->bi_private; 1289 struct dm_cache_migration *mg = bio->bi_private;
1290 struct cache *cache = mg->cache; 1290 struct cache *cache = mg->cache;
@@ -1294,7 +1294,7 @@ static void overwrite_endio(struct bio *bio, int err)
1294 1294
1295 dm_unhook_bio(&pb->hook_info, bio); 1295 dm_unhook_bio(&pb->hook_info, bio);
1296 1296
1297 if (err) 1297 if (bio->bi_error)
1298 mg->err = true; 1298 mg->err = true;
1299 1299
1300 mg->requeue_holder = false; 1300 mg->requeue_holder = false;
@@ -1358,7 +1358,7 @@ static void issue_discard(struct dm_cache_migration *mg)
1358 b = to_dblock(from_dblock(b) + 1); 1358 b = to_dblock(from_dblock(b) + 1);
1359 } 1359 }
1360 1360
1361 bio_endio(bio, 0); 1361 bio_endio(bio);
1362 cell_defer(mg->cache, mg->new_ocell, false); 1362 cell_defer(mg->cache, mg->new_ocell, false);
1363 free_migration(mg); 1363 free_migration(mg);
1364} 1364}
@@ -1631,7 +1631,7 @@ static void process_discard_bio(struct cache *cache, struct prealloc *structs,
1631 1631
1632 calc_discard_block_range(cache, bio, &b, &e); 1632 calc_discard_block_range(cache, bio, &b, &e);
1633 if (b == e) { 1633 if (b == e) {
1634 bio_endio(bio, 0); 1634 bio_endio(bio);
1635 return; 1635 return;
1636 } 1636 }
1637 1637
@@ -2217,8 +2217,10 @@ static void requeue_deferred_bios(struct cache *cache)
2217 bio_list_merge(&bios, &cache->deferred_bios); 2217 bio_list_merge(&bios, &cache->deferred_bios);
2218 bio_list_init(&cache->deferred_bios); 2218 bio_list_init(&cache->deferred_bios);
2219 2219
2220 while ((bio = bio_list_pop(&bios))) 2220 while ((bio = bio_list_pop(&bios))) {
2221 bio_endio(bio, DM_ENDIO_REQUEUE); 2221 bio->bi_error = DM_ENDIO_REQUEUE;
2222 bio_endio(bio);
2223 }
2222} 2224}
2223 2225
2224static int more_work(struct cache *cache) 2226static int more_work(struct cache *cache)
@@ -3123,7 +3125,7 @@ static int cache_map(struct dm_target *ti, struct bio *bio)
3123 * This is a duplicate writethrough io that is no 3125 * This is a duplicate writethrough io that is no
3124 * longer needed because the block has been demoted. 3126 * longer needed because the block has been demoted.
3125 */ 3127 */
3126 bio_endio(bio, 0); 3128 bio_endio(bio);
3127 // FIXME: remap everything as a miss 3129 // FIXME: remap everything as a miss
3128 cell_defer(cache, cell, false); 3130 cell_defer(cache, cell, false);
3129 r = DM_MAPIO_SUBMITTED; 3131 r = DM_MAPIO_SUBMITTED;
@@ -3778,26 +3780,6 @@ static int cache_iterate_devices(struct dm_target *ti,
3778 return r; 3780 return r;
3779} 3781}
3780 3782
3781/*
3782 * We assume I/O is going to the origin (which is the volume
3783 * more likely to have restrictions e.g. by being striped).
3784 * (Looking up the exact location of the data would be expensive
3785 * and could always be out of date by the time the bio is submitted.)
3786 */
3787static int cache_bvec_merge(struct dm_target *ti,
3788 struct bvec_merge_data *bvm,
3789 struct bio_vec *biovec, int max_size)
3790{
3791 struct cache *cache = ti->private;
3792 struct request_queue *q = bdev_get_queue(cache->origin_dev->bdev);
3793
3794 if (!q->merge_bvec_fn)
3795 return max_size;
3796
3797 bvm->bi_bdev = cache->origin_dev->bdev;
3798 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
3799}
3800
3801static void set_discard_limits(struct cache *cache, struct queue_limits *limits) 3783static void set_discard_limits(struct cache *cache, struct queue_limits *limits)
3802{ 3784{
3803 /* 3785 /*
@@ -3841,7 +3823,6 @@ static struct target_type cache_target = {
3841 .status = cache_status, 3823 .status = cache_status,
3842 .message = cache_message, 3824 .message = cache_message,
3843 .iterate_devices = cache_iterate_devices, 3825 .iterate_devices = cache_iterate_devices,
3844 .merge = cache_bvec_merge,
3845 .io_hints = cache_io_hints, 3826 .io_hints = cache_io_hints,
3846}; 3827};
3847 3828
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 0f48fed44a17..ba5c2105f4e6 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -1076,7 +1076,8 @@ static void crypt_dec_pending(struct dm_crypt_io *io)
1076 if (io->ctx.req) 1076 if (io->ctx.req)
1077 crypt_free_req(cc, io->ctx.req, base_bio); 1077 crypt_free_req(cc, io->ctx.req, base_bio);
1078 1078
1079 bio_endio(base_bio, error); 1079 base_bio->bi_error = error;
1080 bio_endio(base_bio);
1080} 1081}
1081 1082
1082/* 1083/*
@@ -1096,14 +1097,12 @@ static void crypt_dec_pending(struct dm_crypt_io *io)
1096 * The work is done per CPU global for all dm-crypt instances. 1097 * The work is done per CPU global for all dm-crypt instances.
1097 * They should not depend on each other and do not block. 1098 * They should not depend on each other and do not block.
1098 */ 1099 */
1099static void crypt_endio(struct bio *clone, int error) 1100static void crypt_endio(struct bio *clone)
1100{ 1101{
1101 struct dm_crypt_io *io = clone->bi_private; 1102 struct dm_crypt_io *io = clone->bi_private;
1102 struct crypt_config *cc = io->cc; 1103 struct crypt_config *cc = io->cc;
1103 unsigned rw = bio_data_dir(clone); 1104 unsigned rw = bio_data_dir(clone);
1104 1105 int error;
1105 if (unlikely(!bio_flagged(clone, BIO_UPTODATE) && !error))
1106 error = -EIO;
1107 1106
1108 /* 1107 /*
1109 * free the processed pages 1108 * free the processed pages
@@ -1111,6 +1110,7 @@ static void crypt_endio(struct bio *clone, int error)
1111 if (rw == WRITE) 1110 if (rw == WRITE)
1112 crypt_free_buffer_pages(cc, clone); 1111 crypt_free_buffer_pages(cc, clone);
1113 1112
1113 error = clone->bi_error;
1114 bio_put(clone); 1114 bio_put(clone);
1115 1115
1116 if (rw == READ && !error) { 1116 if (rw == READ && !error) {
@@ -2035,21 +2035,6 @@ error:
2035 return -EINVAL; 2035 return -EINVAL;
2036} 2036}
2037 2037
2038static int crypt_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
2039 struct bio_vec *biovec, int max_size)
2040{
2041 struct crypt_config *cc = ti->private;
2042 struct request_queue *q = bdev_get_queue(cc->dev->bdev);
2043
2044 if (!q->merge_bvec_fn)
2045 return max_size;
2046
2047 bvm->bi_bdev = cc->dev->bdev;
2048 bvm->bi_sector = cc->start + dm_target_offset(ti, bvm->bi_sector);
2049
2050 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
2051}
2052
2053static int crypt_iterate_devices(struct dm_target *ti, 2038static int crypt_iterate_devices(struct dm_target *ti,
2054 iterate_devices_callout_fn fn, void *data) 2039 iterate_devices_callout_fn fn, void *data)
2055{ 2040{
@@ -2070,7 +2055,6 @@ static struct target_type crypt_target = {
2070 .preresume = crypt_preresume, 2055 .preresume = crypt_preresume,
2071 .resume = crypt_resume, 2056 .resume = crypt_resume,
2072 .message = crypt_message, 2057 .message = crypt_message,
2073 .merge = crypt_merge,
2074 .iterate_devices = crypt_iterate_devices, 2058 .iterate_devices = crypt_iterate_devices,
2075}; 2059};
2076 2060
diff --git a/drivers/md/dm-era-target.c b/drivers/md/dm-era-target.c
index ad913cd4aded..0119ebfb3d49 100644
--- a/drivers/md/dm-era-target.c
+++ b/drivers/md/dm-era-target.c
@@ -1673,20 +1673,6 @@ static int era_iterate_devices(struct dm_target *ti,
1673 return fn(ti, era->origin_dev, 0, get_dev_size(era->origin_dev), data); 1673 return fn(ti, era->origin_dev, 0, get_dev_size(era->origin_dev), data);
1674} 1674}
1675 1675
1676static int era_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
1677 struct bio_vec *biovec, int max_size)
1678{
1679 struct era *era = ti->private;
1680 struct request_queue *q = bdev_get_queue(era->origin_dev->bdev);
1681
1682 if (!q->merge_bvec_fn)
1683 return max_size;
1684
1685 bvm->bi_bdev = era->origin_dev->bdev;
1686
1687 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
1688}
1689
1690static void era_io_hints(struct dm_target *ti, struct queue_limits *limits) 1676static void era_io_hints(struct dm_target *ti, struct queue_limits *limits)
1691{ 1677{
1692 struct era *era = ti->private; 1678 struct era *era = ti->private;
@@ -1717,7 +1703,6 @@ static struct target_type era_target = {
1717 .status = era_status, 1703 .status = era_status,
1718 .message = era_message, 1704 .message = era_message,
1719 .iterate_devices = era_iterate_devices, 1705 .iterate_devices = era_iterate_devices,
1720 .merge = era_merge,
1721 .io_hints = era_io_hints 1706 .io_hints = era_io_hints
1722}; 1707};
1723 1708
diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c
index b257e46876d3..afab13bd683e 100644
--- a/drivers/md/dm-flakey.c
+++ b/drivers/md/dm-flakey.c
@@ -296,7 +296,7 @@ static int flakey_map(struct dm_target *ti, struct bio *bio)
296 * Drop writes? 296 * Drop writes?
297 */ 297 */
298 if (test_bit(DROP_WRITES, &fc->flags)) { 298 if (test_bit(DROP_WRITES, &fc->flags)) {
299 bio_endio(bio, 0); 299 bio_endio(bio);
300 return DM_MAPIO_SUBMITTED; 300 return DM_MAPIO_SUBMITTED;
301 } 301 }
302 302
@@ -387,21 +387,6 @@ static int flakey_ioctl(struct dm_target *ti, unsigned int cmd, unsigned long ar
387 return r ? : __blkdev_driver_ioctl(dev->bdev, dev->mode, cmd, arg); 387 return r ? : __blkdev_driver_ioctl(dev->bdev, dev->mode, cmd, arg);
388} 388}
389 389
390static int flakey_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
391 struct bio_vec *biovec, int max_size)
392{
393 struct flakey_c *fc = ti->private;
394 struct request_queue *q = bdev_get_queue(fc->dev->bdev);
395
396 if (!q->merge_bvec_fn)
397 return max_size;
398
399 bvm->bi_bdev = fc->dev->bdev;
400 bvm->bi_sector = flakey_map_sector(ti, bvm->bi_sector);
401
402 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
403}
404
405static int flakey_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data) 390static int flakey_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data)
406{ 391{
407 struct flakey_c *fc = ti->private; 392 struct flakey_c *fc = ti->private;
@@ -419,7 +404,6 @@ static struct target_type flakey_target = {
419 .end_io = flakey_end_io, 404 .end_io = flakey_end_io,
420 .status = flakey_status, 405 .status = flakey_status,
421 .ioctl = flakey_ioctl, 406 .ioctl = flakey_ioctl,
422 .merge = flakey_merge,
423 .iterate_devices = flakey_iterate_devices, 407 .iterate_devices = flakey_iterate_devices,
424}; 408};
425 409
diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
index 74adcd2c967e..6f8e83b2a6f8 100644
--- a/drivers/md/dm-io.c
+++ b/drivers/md/dm-io.c
@@ -134,12 +134,13 @@ static void dec_count(struct io *io, unsigned int region, int error)
134 complete_io(io); 134 complete_io(io);
135} 135}
136 136
137static void endio(struct bio *bio, int error) 137static void endio(struct bio *bio)
138{ 138{
139 struct io *io; 139 struct io *io;
140 unsigned region; 140 unsigned region;
141 int error;
141 142
142 if (error && bio_data_dir(bio) == READ) 143 if (bio->bi_error && bio_data_dir(bio) == READ)
143 zero_fill_bio(bio); 144 zero_fill_bio(bio);
144 145
145 /* 146 /*
@@ -147,6 +148,7 @@ static void endio(struct bio *bio, int error)
147 */ 148 */
148 retrieve_io_and_region_from_bio(bio, &io, &region); 149 retrieve_io_and_region_from_bio(bio, &io, &region);
149 150
151 error = bio->bi_error;
150 bio_put(bio); 152 bio_put(bio);
151 153
152 dec_count(io, region, error); 154 dec_count(io, region, error);
@@ -314,7 +316,7 @@ static void do_region(int rw, unsigned region, struct dm_io_region *where,
314 if ((rw & REQ_DISCARD) || (rw & REQ_WRITE_SAME)) 316 if ((rw & REQ_DISCARD) || (rw & REQ_WRITE_SAME))
315 num_bvecs = 1; 317 num_bvecs = 1;
316 else 318 else
317 num_bvecs = min_t(int, bio_get_nr_vecs(where->bdev), 319 num_bvecs = min_t(int, BIO_MAX_PAGES,
318 dm_sector_div_up(remaining, (PAGE_SIZE >> SECTOR_SHIFT))); 320 dm_sector_div_up(remaining, (PAGE_SIZE >> SECTOR_SHIFT)));
319 321
320 bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, io->client->bios); 322 bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, io->client->bios);
diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c
index 53e848c10939..7dd5fc8e3eea 100644
--- a/drivers/md/dm-linear.c
+++ b/drivers/md/dm-linear.c
@@ -130,21 +130,6 @@ static int linear_ioctl(struct dm_target *ti, unsigned int cmd,
130 return r ? : __blkdev_driver_ioctl(dev->bdev, dev->mode, cmd, arg); 130 return r ? : __blkdev_driver_ioctl(dev->bdev, dev->mode, cmd, arg);
131} 131}
132 132
133static int linear_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
134 struct bio_vec *biovec, int max_size)
135{
136 struct linear_c *lc = ti->private;
137 struct request_queue *q = bdev_get_queue(lc->dev->bdev);
138
139 if (!q->merge_bvec_fn)
140 return max_size;
141
142 bvm->bi_bdev = lc->dev->bdev;
143 bvm->bi_sector = linear_map_sector(ti, bvm->bi_sector);
144
145 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
146}
147
148static int linear_iterate_devices(struct dm_target *ti, 133static int linear_iterate_devices(struct dm_target *ti,
149 iterate_devices_callout_fn fn, void *data) 134 iterate_devices_callout_fn fn, void *data)
150{ 135{
@@ -162,7 +147,6 @@ static struct target_type linear_target = {
162 .map = linear_map, 147 .map = linear_map,
163 .status = linear_status, 148 .status = linear_status,
164 .ioctl = linear_ioctl, 149 .ioctl = linear_ioctl,
165 .merge = linear_merge,
166 .iterate_devices = linear_iterate_devices, 150 .iterate_devices = linear_iterate_devices,
167}; 151};
168 152
diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c
index ad1b049ae2ab..316cc3fb741f 100644
--- a/drivers/md/dm-log-writes.c
+++ b/drivers/md/dm-log-writes.c
@@ -146,16 +146,16 @@ static void put_io_block(struct log_writes_c *lc)
146 } 146 }
147} 147}
148 148
149static void log_end_io(struct bio *bio, int err) 149static void log_end_io(struct bio *bio)
150{ 150{
151 struct log_writes_c *lc = bio->bi_private; 151 struct log_writes_c *lc = bio->bi_private;
152 struct bio_vec *bvec; 152 struct bio_vec *bvec;
153 int i; 153 int i;
154 154
155 if (err) { 155 if (bio->bi_error) {
156 unsigned long flags; 156 unsigned long flags;
157 157
158 DMERR("Error writing log block, error=%d", err); 158 DMERR("Error writing log block, error=%d", bio->bi_error);
159 spin_lock_irqsave(&lc->blocks_lock, flags); 159 spin_lock_irqsave(&lc->blocks_lock, flags);
160 lc->logging_enabled = false; 160 lc->logging_enabled = false;
161 spin_unlock_irqrestore(&lc->blocks_lock, flags); 161 spin_unlock_irqrestore(&lc->blocks_lock, flags);
@@ -205,7 +205,6 @@ static int write_metadata(struct log_writes_c *lc, void *entry,
205 bio->bi_bdev = lc->logdev->bdev; 205 bio->bi_bdev = lc->logdev->bdev;
206 bio->bi_end_io = log_end_io; 206 bio->bi_end_io = log_end_io;
207 bio->bi_private = lc; 207 bio->bi_private = lc;
208 set_bit(BIO_UPTODATE, &bio->bi_flags);
209 208
210 page = alloc_page(GFP_KERNEL); 209 page = alloc_page(GFP_KERNEL);
211 if (!page) { 210 if (!page) {
@@ -270,7 +269,6 @@ static int log_one_block(struct log_writes_c *lc,
270 bio->bi_bdev = lc->logdev->bdev; 269 bio->bi_bdev = lc->logdev->bdev;
271 bio->bi_end_io = log_end_io; 270 bio->bi_end_io = log_end_io;
272 bio->bi_private = lc; 271 bio->bi_private = lc;
273 set_bit(BIO_UPTODATE, &bio->bi_flags);
274 272
275 for (i = 0; i < block->vec_cnt; i++) { 273 for (i = 0; i < block->vec_cnt; i++) {
276 /* 274 /*
@@ -292,7 +290,6 @@ static int log_one_block(struct log_writes_c *lc,
292 bio->bi_bdev = lc->logdev->bdev; 290 bio->bi_bdev = lc->logdev->bdev;
293 bio->bi_end_io = log_end_io; 291 bio->bi_end_io = log_end_io;
294 bio->bi_private = lc; 292 bio->bi_private = lc;
295 set_bit(BIO_UPTODATE, &bio->bi_flags);
296 293
297 ret = bio_add_page(bio, block->vecs[i].bv_page, 294 ret = bio_add_page(bio, block->vecs[i].bv_page,
298 block->vecs[i].bv_len, 0); 295 block->vecs[i].bv_len, 0);
@@ -606,7 +603,7 @@ static int log_writes_map(struct dm_target *ti, struct bio *bio)
606 WARN_ON(flush_bio || fua_bio); 603 WARN_ON(flush_bio || fua_bio);
607 if (lc->device_supports_discard) 604 if (lc->device_supports_discard)
608 goto map_bio; 605 goto map_bio;
609 bio_endio(bio, 0); 606 bio_endio(bio);
610 return DM_MAPIO_SUBMITTED; 607 return DM_MAPIO_SUBMITTED;
611 } 608 }
612 609
@@ -728,21 +725,6 @@ static int log_writes_ioctl(struct dm_target *ti, unsigned int cmd,
728 return r ? : __blkdev_driver_ioctl(dev->bdev, dev->mode, cmd, arg); 725 return r ? : __blkdev_driver_ioctl(dev->bdev, dev->mode, cmd, arg);
729} 726}
730 727
731static int log_writes_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
732 struct bio_vec *biovec, int max_size)
733{
734 struct log_writes_c *lc = ti->private;
735 struct request_queue *q = bdev_get_queue(lc->dev->bdev);
736
737 if (!q->merge_bvec_fn)
738 return max_size;
739
740 bvm->bi_bdev = lc->dev->bdev;
741 bvm->bi_sector = dm_target_offset(ti, bvm->bi_sector);
742
743 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
744}
745
746static int log_writes_iterate_devices(struct dm_target *ti, 728static int log_writes_iterate_devices(struct dm_target *ti,
747 iterate_devices_callout_fn fn, 729 iterate_devices_callout_fn fn,
748 void *data) 730 void *data)
@@ -796,7 +778,6 @@ static struct target_type log_writes_target = {
796 .end_io = normal_end_io, 778 .end_io = normal_end_io,
797 .status = log_writes_status, 779 .status = log_writes_status,
798 .ioctl = log_writes_ioctl, 780 .ioctl = log_writes_ioctl,
799 .merge = log_writes_merge,
800 .message = log_writes_message, 781 .message = log_writes_message,
801 .iterate_devices = log_writes_iterate_devices, 782 .iterate_devices = log_writes_iterate_devices,
802 .io_hints = log_writes_io_hints, 783 .io_hints = log_writes_io_hints,
diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c
index 2daa67793511..97e165183e79 100644
--- a/drivers/md/dm-raid.c
+++ b/drivers/md/dm-raid.c
@@ -1717,24 +1717,6 @@ static void raid_resume(struct dm_target *ti)
1717 mddev_resume(&rs->md); 1717 mddev_resume(&rs->md);
1718} 1718}
1719 1719
1720static int raid_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
1721 struct bio_vec *biovec, int max_size)
1722{
1723 struct raid_set *rs = ti->private;
1724 struct md_personality *pers = rs->md.pers;
1725
1726 if (pers && pers->mergeable_bvec)
1727 return min(max_size, pers->mergeable_bvec(&rs->md, bvm, biovec));
1728
1729 /*
1730 * In case we can't request the personality because
1731 * the raid set is not running yet
1732 *
1733 * -> return safe minimum
1734 */
1735 return rs->md.chunk_sectors;
1736}
1737
1738static struct target_type raid_target = { 1720static struct target_type raid_target = {
1739 .name = "raid", 1721 .name = "raid",
1740 .version = {1, 7, 0}, 1722 .version = {1, 7, 0},
@@ -1749,7 +1731,6 @@ static struct target_type raid_target = {
1749 .presuspend = raid_presuspend, 1731 .presuspend = raid_presuspend,
1750 .postsuspend = raid_postsuspend, 1732 .postsuspend = raid_postsuspend,
1751 .resume = raid_resume, 1733 .resume = raid_resume,
1752 .merge = raid_merge,
1753}; 1734};
1754 1735
1755static int __init dm_raid_init(void) 1736static int __init dm_raid_init(void)
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index d83696bf403b..e1eabfb2f52d 100644
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -490,9 +490,11 @@ static void hold_bio(struct mirror_set *ms, struct bio *bio)
490 * If device is suspended, complete the bio. 490 * If device is suspended, complete the bio.
491 */ 491 */
492 if (dm_noflush_suspending(ms->ti)) 492 if (dm_noflush_suspending(ms->ti))
493 bio_endio(bio, DM_ENDIO_REQUEUE); 493 bio->bi_error = DM_ENDIO_REQUEUE;
494 else 494 else
495 bio_endio(bio, -EIO); 495 bio->bi_error = -EIO;
496
497 bio_endio(bio);
496 return; 498 return;
497 } 499 }
498 500
@@ -515,7 +517,7 @@ static void read_callback(unsigned long error, void *context)
515 bio_set_m(bio, NULL); 517 bio_set_m(bio, NULL);
516 518
517 if (likely(!error)) { 519 if (likely(!error)) {
518 bio_endio(bio, 0); 520 bio_endio(bio);
519 return; 521 return;
520 } 522 }
521 523
@@ -531,7 +533,7 @@ static void read_callback(unsigned long error, void *context)
531 533
532 DMERR_LIMIT("Read failure on mirror device %s. Failing I/O.", 534 DMERR_LIMIT("Read failure on mirror device %s. Failing I/O.",
533 m->dev->name); 535 m->dev->name);
534 bio_endio(bio, -EIO); 536 bio_io_error(bio);
535} 537}
536 538
537/* Asynchronous read. */ 539/* Asynchronous read. */
@@ -580,7 +582,7 @@ static void do_reads(struct mirror_set *ms, struct bio_list *reads)
580 if (likely(m)) 582 if (likely(m))
581 read_async_bio(m, bio); 583 read_async_bio(m, bio);
582 else 584 else
583 bio_endio(bio, -EIO); 585 bio_io_error(bio);
584 } 586 }
585} 587}
586 588
@@ -598,7 +600,7 @@ static void do_reads(struct mirror_set *ms, struct bio_list *reads)
598 600
599static void write_callback(unsigned long error, void *context) 601static void write_callback(unsigned long error, void *context)
600{ 602{
601 unsigned i, ret = 0; 603 unsigned i;
602 struct bio *bio = (struct bio *) context; 604 struct bio *bio = (struct bio *) context;
603 struct mirror_set *ms; 605 struct mirror_set *ms;
604 int should_wake = 0; 606 int should_wake = 0;
@@ -614,7 +616,7 @@ static void write_callback(unsigned long error, void *context)
614 * regions with the same code. 616 * regions with the same code.
615 */ 617 */
616 if (likely(!error)) { 618 if (likely(!error)) {
617 bio_endio(bio, ret); 619 bio_endio(bio);
618 return; 620 return;
619 } 621 }
620 622
@@ -623,7 +625,8 @@ static void write_callback(unsigned long error, void *context)
623 * degrade the array. 625 * degrade the array.
624 */ 626 */
625 if (bio->bi_rw & REQ_DISCARD) { 627 if (bio->bi_rw & REQ_DISCARD) {
626 bio_endio(bio, -EOPNOTSUPP); 628 bio->bi_error = -EOPNOTSUPP;
629 bio_endio(bio);
627 return; 630 return;
628 } 631 }
629 632
@@ -828,13 +831,12 @@ static void do_failures(struct mirror_set *ms, struct bio_list *failures)
828 * be wrong if the failed leg returned after reboot and 831 * be wrong if the failed leg returned after reboot and
829 * got replicated back to the good legs.) 832 * got replicated back to the good legs.)
830 */ 833 */
831
832 if (unlikely(!get_valid_mirror(ms) || (keep_log(ms) && ms->log_failure))) 834 if (unlikely(!get_valid_mirror(ms) || (keep_log(ms) && ms->log_failure)))
833 bio_endio(bio, -EIO); 835 bio_io_error(bio);
834 else if (errors_handled(ms) && !keep_log(ms)) 836 else if (errors_handled(ms) && !keep_log(ms))
835 hold_bio(ms, bio); 837 hold_bio(ms, bio);
836 else 838 else
837 bio_endio(bio, 0); 839 bio_endio(bio);
838 } 840 }
839} 841}
840 842
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index 7c82d3ccce87..d10b6876018e 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -1490,7 +1490,7 @@ out:
1490 error_bios(snapshot_bios); 1490 error_bios(snapshot_bios);
1491 } else { 1491 } else {
1492 if (full_bio) 1492 if (full_bio)
1493 bio_endio(full_bio, 0); 1493 bio_endio(full_bio);
1494 flush_bios(snapshot_bios); 1494 flush_bios(snapshot_bios);
1495 } 1495 }
1496 1496
@@ -1580,11 +1580,11 @@ static void start_copy(struct dm_snap_pending_exception *pe)
1580 dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe); 1580 dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe);
1581} 1581}
1582 1582
1583static void full_bio_end_io(struct bio *bio, int error) 1583static void full_bio_end_io(struct bio *bio)
1584{ 1584{
1585 void *callback_data = bio->bi_private; 1585 void *callback_data = bio->bi_private;
1586 1586
1587 dm_kcopyd_do_callback(callback_data, 0, error ? 1 : 0); 1587 dm_kcopyd_do_callback(callback_data, 0, bio->bi_error ? 1 : 0);
1588} 1588}
1589 1589
1590static void start_full_bio(struct dm_snap_pending_exception *pe, 1590static void start_full_bio(struct dm_snap_pending_exception *pe,
@@ -2330,20 +2330,6 @@ static void origin_status(struct dm_target *ti, status_type_t type,
2330 } 2330 }
2331} 2331}
2332 2332
2333static int origin_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
2334 struct bio_vec *biovec, int max_size)
2335{
2336 struct dm_origin *o = ti->private;
2337 struct request_queue *q = bdev_get_queue(o->dev->bdev);
2338
2339 if (!q->merge_bvec_fn)
2340 return max_size;
2341
2342 bvm->bi_bdev = o->dev->bdev;
2343
2344 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
2345}
2346
2347static int origin_iterate_devices(struct dm_target *ti, 2333static int origin_iterate_devices(struct dm_target *ti,
2348 iterate_devices_callout_fn fn, void *data) 2334 iterate_devices_callout_fn fn, void *data)
2349{ 2335{
@@ -2362,7 +2348,6 @@ static struct target_type origin_target = {
2362 .resume = origin_resume, 2348 .resume = origin_resume,
2363 .postsuspend = origin_postsuspend, 2349 .postsuspend = origin_postsuspend,
2364 .status = origin_status, 2350 .status = origin_status,
2365 .merge = origin_merge,
2366 .iterate_devices = origin_iterate_devices, 2351 .iterate_devices = origin_iterate_devices,
2367}; 2352};
2368 2353
diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c
index a672a1502c14..484029db8cba 100644
--- a/drivers/md/dm-stripe.c
+++ b/drivers/md/dm-stripe.c
@@ -273,7 +273,7 @@ static int stripe_map_range(struct stripe_c *sc, struct bio *bio,
273 return DM_MAPIO_REMAPPED; 273 return DM_MAPIO_REMAPPED;
274 } else { 274 } else {
275 /* The range doesn't map to the target stripe */ 275 /* The range doesn't map to the target stripe */
276 bio_endio(bio, 0); 276 bio_endio(bio);
277 return DM_MAPIO_SUBMITTED; 277 return DM_MAPIO_SUBMITTED;
278 } 278 }
279} 279}
@@ -412,26 +412,6 @@ static void stripe_io_hints(struct dm_target *ti,
412 blk_limits_io_opt(limits, chunk_size * sc->stripes); 412 blk_limits_io_opt(limits, chunk_size * sc->stripes);
413} 413}
414 414
415static int stripe_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
416 struct bio_vec *biovec, int max_size)
417{
418 struct stripe_c *sc = ti->private;
419 sector_t bvm_sector = bvm->bi_sector;
420 uint32_t stripe;
421 struct request_queue *q;
422
423 stripe_map_sector(sc, bvm_sector, &stripe, &bvm_sector);
424
425 q = bdev_get_queue(sc->stripe[stripe].dev->bdev);
426 if (!q->merge_bvec_fn)
427 return max_size;
428
429 bvm->bi_bdev = sc->stripe[stripe].dev->bdev;
430 bvm->bi_sector = sc->stripe[stripe].physical_start + bvm_sector;
431
432 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
433}
434
435static struct target_type stripe_target = { 415static struct target_type stripe_target = {
436 .name = "striped", 416 .name = "striped",
437 .version = {1, 5, 1}, 417 .version = {1, 5, 1},
@@ -443,7 +423,6 @@ static struct target_type stripe_target = {
443 .status = stripe_status, 423 .status = stripe_status,
444 .iterate_devices = stripe_iterate_devices, 424 .iterate_devices = stripe_iterate_devices,
445 .io_hints = stripe_io_hints, 425 .io_hints = stripe_io_hints,
446 .merge = stripe_merge,
447}; 426};
448 427
449int __init dm_stripe_init(void) 428int __init dm_stripe_init(void)
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index 16ba55ad7089..e76ed003769e 100644
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -440,14 +440,6 @@ static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev,
440 q->limits.alignment_offset, 440 q->limits.alignment_offset,
441 (unsigned long long) start << SECTOR_SHIFT); 441 (unsigned long long) start << SECTOR_SHIFT);
442 442
443 /*
444 * Check if merge fn is supported.
445 * If not we'll force DM to use PAGE_SIZE or
446 * smaller I/O, just to be safe.
447 */
448 if (dm_queue_merge_is_compulsory(q) && !ti->type->merge)
449 blk_limits_max_hw_sectors(limits,
450 (unsigned int) (PAGE_SIZE >> 9));
451 return 0; 443 return 0;
452} 444}
453 445
@@ -1388,14 +1380,6 @@ static int queue_supports_sg_merge(struct dm_target *ti, struct dm_dev *dev,
1388 return q && !test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags); 1380 return q && !test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags);
1389} 1381}
1390 1382
1391static int queue_supports_sg_gaps(struct dm_target *ti, struct dm_dev *dev,
1392 sector_t start, sector_t len, void *data)
1393{
1394 struct request_queue *q = bdev_get_queue(dev->bdev);
1395
1396 return q && !test_bit(QUEUE_FLAG_SG_GAPS, &q->queue_flags);
1397}
1398
1399static bool dm_table_all_devices_attribute(struct dm_table *t, 1383static bool dm_table_all_devices_attribute(struct dm_table *t,
1400 iterate_devices_callout_fn func) 1384 iterate_devices_callout_fn func)
1401{ 1385{
@@ -1516,11 +1500,6 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
1516 else 1500 else
1517 queue_flag_set_unlocked(QUEUE_FLAG_NO_SG_MERGE, q); 1501 queue_flag_set_unlocked(QUEUE_FLAG_NO_SG_MERGE, q);
1518 1502
1519 if (dm_table_all_devices_attribute(t, queue_supports_sg_gaps))
1520 queue_flag_clear_unlocked(QUEUE_FLAG_SG_GAPS, q);
1521 else
1522 queue_flag_set_unlocked(QUEUE_FLAG_SG_GAPS, q);
1523
1524 dm_table_set_integrity(t); 1503 dm_table_set_integrity(t);
1525 1504
1526 /* 1505 /*
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index d2bbe8cc1e97..271a66249363 100644
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -615,8 +615,10 @@ static void error_bio_list(struct bio_list *bios, int error)
615{ 615{
616 struct bio *bio; 616 struct bio *bio;
617 617
618 while ((bio = bio_list_pop(bios))) 618 while ((bio = bio_list_pop(bios))) {
619 bio_endio(bio, error); 619 bio->bi_error = error;
620 bio_endio(bio);
621 }
620} 622}
621 623
622static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master, int error) 624static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master, int error)
@@ -870,14 +872,14 @@ static void copy_complete(int read_err, unsigned long write_err, void *context)
870 complete_mapping_preparation(m); 872 complete_mapping_preparation(m);
871} 873}
872 874
873static void overwrite_endio(struct bio *bio, int err) 875static void overwrite_endio(struct bio *bio)
874{ 876{
875 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); 877 struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
876 struct dm_thin_new_mapping *m = h->overwrite_mapping; 878 struct dm_thin_new_mapping *m = h->overwrite_mapping;
877 879
878 bio->bi_end_io = m->saved_bi_end_io; 880 bio->bi_end_io = m->saved_bi_end_io;
879 881
880 m->err = err; 882 m->err = bio->bi_error;
881 complete_mapping_preparation(m); 883 complete_mapping_preparation(m);
882} 884}
883 885
@@ -1002,7 +1004,7 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m)
1002 */ 1004 */
1003 if (bio) { 1005 if (bio) {
1004 inc_remap_and_issue_cell(tc, m->cell, m->data_block); 1006 inc_remap_and_issue_cell(tc, m->cell, m->data_block);
1005 bio_endio(bio, 0); 1007 bio_endio(bio);
1006 } else { 1008 } else {
1007 inc_all_io_entry(tc->pool, m->cell->holder); 1009 inc_all_io_entry(tc->pool, m->cell->holder);
1008 remap_and_issue(tc, m->cell->holder, m->data_block); 1010 remap_and_issue(tc, m->cell->holder, m->data_block);
@@ -1032,7 +1034,7 @@ static void process_prepared_discard_fail(struct dm_thin_new_mapping *m)
1032 1034
1033static void process_prepared_discard_success(struct dm_thin_new_mapping *m) 1035static void process_prepared_discard_success(struct dm_thin_new_mapping *m)
1034{ 1036{
1035 bio_endio(m->bio, 0); 1037 bio_endio(m->bio);
1036 free_discard_mapping(m); 1038 free_discard_mapping(m);
1037} 1039}
1038 1040
@@ -1046,7 +1048,7 @@ static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m)
1046 metadata_operation_failed(tc->pool, "dm_thin_remove_range", r); 1048 metadata_operation_failed(tc->pool, "dm_thin_remove_range", r);
1047 bio_io_error(m->bio); 1049 bio_io_error(m->bio);
1048 } else 1050 } else
1049 bio_endio(m->bio, 0); 1051 bio_endio(m->bio);
1050 1052
1051 cell_defer_no_holder(tc, m->cell); 1053 cell_defer_no_holder(tc, m->cell);
1052 mempool_free(m, tc->pool->mapping_pool); 1054 mempool_free(m, tc->pool->mapping_pool);
@@ -1117,7 +1119,8 @@ static void process_prepared_discard_passdown(struct dm_thin_new_mapping *m)
1117 * Even if r is set, there could be sub discards in flight that we 1119 * Even if r is set, there could be sub discards in flight that we
1118 * need to wait for. 1120 * need to wait for.
1119 */ 1121 */
1120 bio_endio(m->bio, r); 1122 m->bio->bi_error = r;
1123 bio_endio(m->bio);
1121 cell_defer_no_holder(tc, m->cell); 1124 cell_defer_no_holder(tc, m->cell);
1122 mempool_free(m, pool->mapping_pool); 1125 mempool_free(m, pool->mapping_pool);
1123} 1126}
@@ -1493,9 +1496,10 @@ static void handle_unserviceable_bio(struct pool *pool, struct bio *bio)
1493{ 1496{
1494 int error = should_error_unserviceable_bio(pool); 1497 int error = should_error_unserviceable_bio(pool);
1495 1498
1496 if (error) 1499 if (error) {
1497 bio_endio(bio, error); 1500 bio->bi_error = error;
1498 else 1501 bio_endio(bio);
1502 } else
1499 retry_on_resume(bio); 1503 retry_on_resume(bio);
1500} 1504}
1501 1505
@@ -1631,7 +1635,7 @@ static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_priso
1631 * will prevent completion until the sub range discards have 1635 * will prevent completion until the sub range discards have
1632 * completed. 1636 * completed.
1633 */ 1637 */
1634 bio_endio(bio, 0); 1638 bio_endio(bio);
1635} 1639}
1636 1640
1637static void process_discard_bio(struct thin_c *tc, struct bio *bio) 1641static void process_discard_bio(struct thin_c *tc, struct bio *bio)
@@ -1645,7 +1649,7 @@ static void process_discard_bio(struct thin_c *tc, struct bio *bio)
1645 /* 1649 /*
1646 * The discard covers less than a block. 1650 * The discard covers less than a block.
1647 */ 1651 */
1648 bio_endio(bio, 0); 1652 bio_endio(bio);
1649 return; 1653 return;
1650 } 1654 }
1651 1655
@@ -1790,7 +1794,7 @@ static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block
1790 if (bio_data_dir(bio) == READ) { 1794 if (bio_data_dir(bio) == READ) {
1791 zero_fill_bio(bio); 1795 zero_fill_bio(bio);
1792 cell_defer_no_holder(tc, cell); 1796 cell_defer_no_holder(tc, cell);
1793 bio_endio(bio, 0); 1797 bio_endio(bio);
1794 return; 1798 return;
1795 } 1799 }
1796 1800
@@ -1855,7 +1859,7 @@ static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell)
1855 1859
1856 } else { 1860 } else {
1857 zero_fill_bio(bio); 1861 zero_fill_bio(bio);
1858 bio_endio(bio, 0); 1862 bio_endio(bio);
1859 } 1863 }
1860 } else 1864 } else
1861 provision_block(tc, bio, block, cell); 1865 provision_block(tc, bio, block, cell);
@@ -1926,7 +1930,7 @@ static void __process_bio_read_only(struct thin_c *tc, struct bio *bio,
1926 } 1930 }
1927 1931
1928 zero_fill_bio(bio); 1932 zero_fill_bio(bio);
1929 bio_endio(bio, 0); 1933 bio_endio(bio);
1930 break; 1934 break;
1931 1935
1932 default: 1936 default:
@@ -1951,7 +1955,7 @@ static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell
1951 1955
1952static void process_bio_success(struct thin_c *tc, struct bio *bio) 1956static void process_bio_success(struct thin_c *tc, struct bio *bio)
1953{ 1957{
1954 bio_endio(bio, 0); 1958 bio_endio(bio);
1955} 1959}
1956 1960
1957static void process_bio_fail(struct thin_c *tc, struct bio *bio) 1961static void process_bio_fail(struct thin_c *tc, struct bio *bio)
@@ -2600,7 +2604,8 @@ static int thin_bio_map(struct dm_target *ti, struct bio *bio)
2600 thin_hook_bio(tc, bio); 2604 thin_hook_bio(tc, bio);
2601 2605
2602 if (tc->requeue_mode) { 2606 if (tc->requeue_mode) {
2603 bio_endio(bio, DM_ENDIO_REQUEUE); 2607 bio->bi_error = DM_ENDIO_REQUEUE;
2608 bio_endio(bio);
2604 return DM_MAPIO_SUBMITTED; 2609 return DM_MAPIO_SUBMITTED;
2605 } 2610 }
2606 2611
@@ -3875,20 +3880,6 @@ static int pool_iterate_devices(struct dm_target *ti,
3875 return fn(ti, pt->data_dev, 0, ti->len, data); 3880 return fn(ti, pt->data_dev, 0, ti->len, data);
3876} 3881}
3877 3882
3878static int pool_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
3879 struct bio_vec *biovec, int max_size)
3880{
3881 struct pool_c *pt = ti->private;
3882 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
3883
3884 if (!q->merge_bvec_fn)
3885 return max_size;
3886
3887 bvm->bi_bdev = pt->data_dev->bdev;
3888
3889 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
3890}
3891
3892static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits) 3883static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
3893{ 3884{
3894 struct pool_c *pt = ti->private; 3885 struct pool_c *pt = ti->private;
@@ -3965,7 +3956,6 @@ static struct target_type pool_target = {
3965 .resume = pool_resume, 3956 .resume = pool_resume,
3966 .message = pool_message, 3957 .message = pool_message,
3967 .status = pool_status, 3958 .status = pool_status,
3968 .merge = pool_merge,
3969 .iterate_devices = pool_iterate_devices, 3959 .iterate_devices = pool_iterate_devices,
3970 .io_hints = pool_io_hints, 3960 .io_hints = pool_io_hints,
3971}; 3961};
@@ -4292,21 +4282,6 @@ err:
4292 DMEMIT("Error"); 4282 DMEMIT("Error");
4293} 4283}
4294 4284
4295static int thin_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
4296 struct bio_vec *biovec, int max_size)
4297{
4298 struct thin_c *tc = ti->private;
4299 struct request_queue *q = bdev_get_queue(tc->pool_dev->bdev);
4300
4301 if (!q->merge_bvec_fn)
4302 return max_size;
4303
4304 bvm->bi_bdev = tc->pool_dev->bdev;
4305 bvm->bi_sector = dm_target_offset(ti, bvm->bi_sector);
4306
4307 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
4308}
4309
4310static int thin_iterate_devices(struct dm_target *ti, 4285static int thin_iterate_devices(struct dm_target *ti,
4311 iterate_devices_callout_fn fn, void *data) 4286 iterate_devices_callout_fn fn, void *data)
4312{ 4287{
@@ -4350,7 +4325,6 @@ static struct target_type thin_target = {
4350 .presuspend = thin_presuspend, 4325 .presuspend = thin_presuspend,
4351 .postsuspend = thin_postsuspend, 4326 .postsuspend = thin_postsuspend,
4352 .status = thin_status, 4327 .status = thin_status,
4353 .merge = thin_merge,
4354 .iterate_devices = thin_iterate_devices, 4328 .iterate_devices = thin_iterate_devices,
4355 .io_hints = thin_io_hints, 4329 .io_hints = thin_io_hints,
4356}; 4330};
diff --git a/drivers/md/dm-verity.c b/drivers/md/dm-verity.c
index bb9c6a00e4b0..c137dcb147b8 100644
--- a/drivers/md/dm-verity.c
+++ b/drivers/md/dm-verity.c
@@ -458,8 +458,9 @@ static void verity_finish_io(struct dm_verity_io *io, int error)
458 458
459 bio->bi_end_io = io->orig_bi_end_io; 459 bio->bi_end_io = io->orig_bi_end_io;
460 bio->bi_private = io->orig_bi_private; 460 bio->bi_private = io->orig_bi_private;
461 bio->bi_error = error;
461 462
462 bio_endio(bio, error); 463 bio_endio(bio);
463} 464}
464 465
465static void verity_work(struct work_struct *w) 466static void verity_work(struct work_struct *w)
@@ -469,12 +470,12 @@ static void verity_work(struct work_struct *w)
469 verity_finish_io(io, verity_verify_io(io)); 470 verity_finish_io(io, verity_verify_io(io));
470} 471}
471 472
472static void verity_end_io(struct bio *bio, int error) 473static void verity_end_io(struct bio *bio)
473{ 474{
474 struct dm_verity_io *io = bio->bi_private; 475 struct dm_verity_io *io = bio->bi_private;
475 476
476 if (error) { 477 if (bio->bi_error) {
477 verity_finish_io(io, error); 478 verity_finish_io(io, bio->bi_error);
478 return; 479 return;
479 } 480 }
480 481
@@ -648,21 +649,6 @@ static int verity_ioctl(struct dm_target *ti, unsigned cmd,
648 cmd, arg); 649 cmd, arg);
649} 650}
650 651
651static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
652 struct bio_vec *biovec, int max_size)
653{
654 struct dm_verity *v = ti->private;
655 struct request_queue *q = bdev_get_queue(v->data_dev->bdev);
656
657 if (!q->merge_bvec_fn)
658 return max_size;
659
660 bvm->bi_bdev = v->data_dev->bdev;
661 bvm->bi_sector = verity_map_sector(v, bvm->bi_sector);
662
663 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
664}
665
666static int verity_iterate_devices(struct dm_target *ti, 652static int verity_iterate_devices(struct dm_target *ti,
667 iterate_devices_callout_fn fn, void *data) 653 iterate_devices_callout_fn fn, void *data)
668{ 654{
@@ -995,7 +981,6 @@ static struct target_type verity_target = {
995 .map = verity_map, 981 .map = verity_map,
996 .status = verity_status, 982 .status = verity_status,
997 .ioctl = verity_ioctl, 983 .ioctl = verity_ioctl,
998 .merge = verity_merge,
999 .iterate_devices = verity_iterate_devices, 984 .iterate_devices = verity_iterate_devices,
1000 .io_hints = verity_io_hints, 985 .io_hints = verity_io_hints,
1001}; 986};
diff --git a/drivers/md/dm-zero.c b/drivers/md/dm-zero.c
index b9a64bbce304..766bc93006e6 100644
--- a/drivers/md/dm-zero.c
+++ b/drivers/md/dm-zero.c
@@ -47,7 +47,7 @@ static int zero_map(struct dm_target *ti, struct bio *bio)
47 break; 47 break;
48 } 48 }
49 49
50 bio_endio(bio, 0); 50 bio_endio(bio);
51 51
52 /* accepted bio, don't make new request */ 52 /* accepted bio, don't make new request */
53 return DM_MAPIO_SUBMITTED; 53 return DM_MAPIO_SUBMITTED;
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 0d7ab20c58df..6ffc01bb85f2 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -124,9 +124,8 @@ EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo);
124#define DMF_FREEING 3 124#define DMF_FREEING 3
125#define DMF_DELETING 4 125#define DMF_DELETING 4
126#define DMF_NOFLUSH_SUSPENDING 5 126#define DMF_NOFLUSH_SUSPENDING 5
127#define DMF_MERGE_IS_OPTIONAL 6 127#define DMF_DEFERRED_REMOVE 6
128#define DMF_DEFERRED_REMOVE 7 128#define DMF_SUSPENDED_INTERNALLY 7
129#define DMF_SUSPENDED_INTERNALLY 8
130 129
131/* 130/*
132 * A dummy definition to make RCU happy. 131 * A dummy definition to make RCU happy.
@@ -944,7 +943,8 @@ static void dec_pending(struct dm_io *io, int error)
944 } else { 943 } else {
945 /* done with normal IO or empty flush */ 944 /* done with normal IO or empty flush */
946 trace_block_bio_complete(md->queue, bio, io_error); 945 trace_block_bio_complete(md->queue, bio, io_error);
947 bio_endio(bio, io_error); 946 bio->bi_error = io_error;
947 bio_endio(bio);
948 } 948 }
949 } 949 }
950} 950}
@@ -957,17 +957,15 @@ static void disable_write_same(struct mapped_device *md)
957 limits->max_write_same_sectors = 0; 957 limits->max_write_same_sectors = 0;
958} 958}
959 959
960static void clone_endio(struct bio *bio, int error) 960static void clone_endio(struct bio *bio)
961{ 961{
962 int error = bio->bi_error;
962 int r = error; 963 int r = error;
963 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 964 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
964 struct dm_io *io = tio->io; 965 struct dm_io *io = tio->io;
965 struct mapped_device *md = tio->io->md; 966 struct mapped_device *md = tio->io->md;
966 dm_endio_fn endio = tio->ti->type->end_io; 967 dm_endio_fn endio = tio->ti->type->end_io;
967 968
968 if (!bio_flagged(bio, BIO_UPTODATE) && !error)
969 error = -EIO;
970
971 if (endio) { 969 if (endio) {
972 r = endio(tio->ti, bio, error); 970 r = endio(tio->ti, bio, error);
973 if (r < 0 || r == DM_ENDIO_REQUEUE) 971 if (r < 0 || r == DM_ENDIO_REQUEUE)
@@ -996,7 +994,7 @@ static void clone_endio(struct bio *bio, int error)
996/* 994/*
997 * Partial completion handling for request-based dm 995 * Partial completion handling for request-based dm
998 */ 996 */
999static void end_clone_bio(struct bio *clone, int error) 997static void end_clone_bio(struct bio *clone)
1000{ 998{
1001 struct dm_rq_clone_bio_info *info = 999 struct dm_rq_clone_bio_info *info =
1002 container_of(clone, struct dm_rq_clone_bio_info, clone); 1000 container_of(clone, struct dm_rq_clone_bio_info, clone);
@@ -1013,13 +1011,13 @@ static void end_clone_bio(struct bio *clone, int error)
1013 * the remainder. 1011 * the remainder.
1014 */ 1012 */
1015 return; 1013 return;
1016 else if (error) { 1014 else if (bio->bi_error) {
1017 /* 1015 /*
1018 * Don't notice the error to the upper layer yet. 1016 * Don't notice the error to the upper layer yet.
1019 * The error handling decision is made by the target driver, 1017 * The error handling decision is made by the target driver,
1020 * when the request is completed. 1018 * when the request is completed.
1021 */ 1019 */
1022 tio->error = error; 1020 tio->error = bio->bi_error;
1023 return; 1021 return;
1024 } 1022 }
1025 1023
@@ -1722,60 +1720,6 @@ static void __split_and_process_bio(struct mapped_device *md,
1722 * CRUD END 1720 * CRUD END
1723 *---------------------------------------------------------------*/ 1721 *---------------------------------------------------------------*/
1724 1722
1725static int dm_merge_bvec(struct request_queue *q,
1726 struct bvec_merge_data *bvm,
1727 struct bio_vec *biovec)
1728{
1729 struct mapped_device *md = q->queuedata;
1730 struct dm_table *map = dm_get_live_table_fast(md);
1731 struct dm_target *ti;
1732 sector_t max_sectors;
1733 int max_size = 0;
1734
1735 if (unlikely(!map))
1736 goto out;
1737
1738 ti = dm_table_find_target(map, bvm->bi_sector);
1739 if (!dm_target_is_valid(ti))
1740 goto out;
1741
1742 /*
1743 * Find maximum amount of I/O that won't need splitting
1744 */
1745 max_sectors = min(max_io_len(bvm->bi_sector, ti),
1746 (sector_t) BIO_MAX_SECTORS);
1747 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
1748 if (max_size < 0)
1749 max_size = 0;
1750
1751 /*
1752 * merge_bvec_fn() returns number of bytes
1753 * it can accept at this offset
1754 * max is precomputed maximal io size
1755 */
1756 if (max_size && ti->type->merge)
1757 max_size = ti->type->merge(ti, bvm, biovec, max_size);
1758 /*
1759 * If the target doesn't support merge method and some of the devices
1760 * provided their merge_bvec method (we know this by looking at
1761 * queue_max_hw_sectors), then we can't allow bios with multiple vector
1762 * entries. So always set max_size to 0, and the code below allows
1763 * just one page.
1764 */
1765 else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)
1766 max_size = 0;
1767
1768out:
1769 dm_put_live_table_fast(md);
1770 /*
1771 * Always allow an entire first page
1772 */
1773 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
1774 max_size = biovec->bv_len;
1775
1776 return max_size;
1777}
1778
1779/* 1723/*
1780 * The request function that just remaps the bio built up by 1724 * The request function that just remaps the bio built up by
1781 * dm_merge_bvec. 1725 * dm_merge_bvec.
@@ -1789,6 +1733,8 @@ static void dm_make_request(struct request_queue *q, struct bio *bio)
1789 1733
1790 map = dm_get_live_table(md, &srcu_idx); 1734 map = dm_get_live_table(md, &srcu_idx);
1791 1735
1736 blk_queue_split(q, &bio, q->bio_split);
1737
1792 generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0); 1738 generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0);
1793 1739
1794 /* if we're suspended, we have to queue this io for later */ 1740 /* if we're suspended, we have to queue this io for later */
@@ -2496,59 +2442,6 @@ static void __set_size(struct mapped_device *md, sector_t size)
2496} 2442}
2497 2443
2498/* 2444/*
2499 * Return 1 if the queue has a compulsory merge_bvec_fn function.
2500 *
2501 * If this function returns 0, then the device is either a non-dm
2502 * device without a merge_bvec_fn, or it is a dm device that is
2503 * able to split any bios it receives that are too big.
2504 */
2505int dm_queue_merge_is_compulsory(struct request_queue *q)
2506{
2507 struct mapped_device *dev_md;
2508
2509 if (!q->merge_bvec_fn)
2510 return 0;
2511
2512 if (q->make_request_fn == dm_make_request) {
2513 dev_md = q->queuedata;
2514 if (test_bit(DMF_MERGE_IS_OPTIONAL, &dev_md->flags))
2515 return 0;
2516 }
2517
2518 return 1;
2519}
2520
2521static int dm_device_merge_is_compulsory(struct dm_target *ti,
2522 struct dm_dev *dev, sector_t start,
2523 sector_t len, void *data)
2524{
2525 struct block_device *bdev = dev->bdev;
2526 struct request_queue *q = bdev_get_queue(bdev);
2527
2528 return dm_queue_merge_is_compulsory(q);
2529}
2530
2531/*
2532 * Return 1 if it is acceptable to ignore merge_bvec_fn based
2533 * on the properties of the underlying devices.
2534 */
2535static int dm_table_merge_is_optional(struct dm_table *table)
2536{
2537 unsigned i = 0;
2538 struct dm_target *ti;
2539
2540 while (i < dm_table_get_num_targets(table)) {
2541 ti = dm_table_get_target(table, i++);
2542
2543 if (ti->type->iterate_devices &&
2544 ti->type->iterate_devices(ti, dm_device_merge_is_compulsory, NULL))
2545 return 0;
2546 }
2547
2548 return 1;
2549}
2550
2551/*
2552 * Returns old map, which caller must destroy. 2445 * Returns old map, which caller must destroy.
2553 */ 2446 */
2554static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, 2447static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
@@ -2557,7 +2450,6 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2557 struct dm_table *old_map; 2450 struct dm_table *old_map;
2558 struct request_queue *q = md->queue; 2451 struct request_queue *q = md->queue;
2559 sector_t size; 2452 sector_t size;
2560 int merge_is_optional;
2561 2453
2562 size = dm_table_get_size(t); 2454 size = dm_table_get_size(t);
2563 2455
@@ -2583,17 +2475,11 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2583 2475
2584 __bind_mempools(md, t); 2476 __bind_mempools(md, t);
2585 2477
2586 merge_is_optional = dm_table_merge_is_optional(t);
2587
2588 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2478 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2589 rcu_assign_pointer(md->map, t); 2479 rcu_assign_pointer(md->map, t);
2590 md->immutable_target_type = dm_table_get_immutable_target_type(t); 2480 md->immutable_target_type = dm_table_get_immutable_target_type(t);
2591 2481
2592 dm_table_set_restrictions(t, q, limits); 2482 dm_table_set_restrictions(t, q, limits);
2593 if (merge_is_optional)
2594 set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2595 else
2596 clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2597 if (old_map) 2483 if (old_map)
2598 dm_sync_table(md); 2484 dm_sync_table(md);
2599 2485
@@ -2874,7 +2760,6 @@ int dm_setup_md_queue(struct mapped_device *md)
2874 case DM_TYPE_BIO_BASED: 2760 case DM_TYPE_BIO_BASED:
2875 dm_init_old_md_queue(md); 2761 dm_init_old_md_queue(md);
2876 blk_queue_make_request(md->queue, dm_make_request); 2762 blk_queue_make_request(md->queue, dm_make_request);
2877 blk_queue_merge_bvec(md->queue, dm_merge_bvec);
2878 break; 2763 break;
2879 } 2764 }
2880 2765
diff --git a/drivers/md/dm.h b/drivers/md/dm.h
index 4e984993d40a..7edcf97dfa5a 100644
--- a/drivers/md/dm.h
+++ b/drivers/md/dm.h
@@ -78,8 +78,6 @@ bool dm_table_mq_request_based(struct dm_table *t);
78void dm_table_free_md_mempools(struct dm_table *t); 78void dm_table_free_md_mempools(struct dm_table *t);
79struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t); 79struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t);
80 80
81int dm_queue_merge_is_compulsory(struct request_queue *q);
82
83void dm_lock_md_type(struct mapped_device *md); 81void dm_lock_md_type(struct mapped_device *md);
84void dm_unlock_md_type(struct mapped_device *md); 82void dm_unlock_md_type(struct mapped_device *md);
85void dm_set_md_type(struct mapped_device *md, unsigned type); 83void dm_set_md_type(struct mapped_device *md, unsigned type);
diff --git a/drivers/md/faulty.c b/drivers/md/faulty.c
index 1277eb26b58a..4a8e15058e8b 100644
--- a/drivers/md/faulty.c
+++ b/drivers/md/faulty.c
@@ -70,7 +70,7 @@
70#include <linux/seq_file.h> 70#include <linux/seq_file.h>
71 71
72 72
73static void faulty_fail(struct bio *bio, int error) 73static void faulty_fail(struct bio *bio)
74{ 74{
75 struct bio *b = bio->bi_private; 75 struct bio *b = bio->bi_private;
76 76
@@ -181,7 +181,7 @@ static void make_request(struct mddev *mddev, struct bio *bio)
181 /* special case - don't decrement, don't generic_make_request, 181 /* special case - don't decrement, don't generic_make_request,
182 * just fail immediately 182 * just fail immediately
183 */ 183 */
184 bio_endio(bio, -EIO); 184 bio_io_error(bio);
185 return; 185 return;
186 } 186 }
187 187
diff --git a/drivers/md/linear.c b/drivers/md/linear.c
index fa7d577f3d12..b7fe7e9fc777 100644
--- a/drivers/md/linear.c
+++ b/drivers/md/linear.c
@@ -52,48 +52,6 @@ static inline struct dev_info *which_dev(struct mddev *mddev, sector_t sector)
52 return conf->disks + lo; 52 return conf->disks + lo;
53} 53}
54 54
55/**
56 * linear_mergeable_bvec -- tell bio layer if two requests can be merged
57 * @q: request queue
58 * @bvm: properties of new bio
59 * @biovec: the request that could be merged to it.
60 *
61 * Return amount of bytes we can take at this offset
62 */
63static int linear_mergeable_bvec(struct mddev *mddev,
64 struct bvec_merge_data *bvm,
65 struct bio_vec *biovec)
66{
67 struct dev_info *dev0;
68 unsigned long maxsectors, bio_sectors = bvm->bi_size >> 9;
69 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
70 int maxbytes = biovec->bv_len;
71 struct request_queue *subq;
72
73 dev0 = which_dev(mddev, sector);
74 maxsectors = dev0->end_sector - sector;
75 subq = bdev_get_queue(dev0->rdev->bdev);
76 if (subq->merge_bvec_fn) {
77 bvm->bi_bdev = dev0->rdev->bdev;
78 bvm->bi_sector -= dev0->end_sector - dev0->rdev->sectors;
79 maxbytes = min(maxbytes, subq->merge_bvec_fn(subq, bvm,
80 biovec));
81 }
82
83 if (maxsectors < bio_sectors)
84 maxsectors = 0;
85 else
86 maxsectors -= bio_sectors;
87
88 if (maxsectors <= (PAGE_SIZE >> 9 ) && bio_sectors == 0)
89 return maxbytes;
90
91 if (maxsectors > (maxbytes >> 9))
92 return maxbytes;
93 else
94 return maxsectors << 9;
95}
96
97static int linear_congested(struct mddev *mddev, int bits) 55static int linear_congested(struct mddev *mddev, int bits)
98{ 56{
99 struct linear_conf *conf; 57 struct linear_conf *conf;
@@ -297,7 +255,7 @@ static void linear_make_request(struct mddev *mddev, struct bio *bio)
297 if (unlikely((split->bi_rw & REQ_DISCARD) && 255 if (unlikely((split->bi_rw & REQ_DISCARD) &&
298 !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) { 256 !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) {
299 /* Just ignore it */ 257 /* Just ignore it */
300 bio_endio(split, 0); 258 bio_endio(split);
301 } else 259 } else
302 generic_make_request(split); 260 generic_make_request(split);
303 } while (split != bio); 261 } while (split != bio);
@@ -338,7 +296,6 @@ static struct md_personality linear_personality =
338 .size = linear_size, 296 .size = linear_size,
339 .quiesce = linear_quiesce, 297 .quiesce = linear_quiesce,
340 .congested = linear_congested, 298 .congested = linear_congested,
341 .mergeable_bvec = linear_mergeable_bvec,
342}; 299};
343 300
344static int __init linear_init (void) 301static int __init linear_init (void)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index e25f00f0138a..40332625713b 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -257,13 +257,17 @@ static void md_make_request(struct request_queue *q, struct bio *bio)
257 unsigned int sectors; 257 unsigned int sectors;
258 int cpu; 258 int cpu;
259 259
260 blk_queue_split(q, &bio, q->bio_split);
261
260 if (mddev == NULL || mddev->pers == NULL 262 if (mddev == NULL || mddev->pers == NULL
261 || !mddev->ready) { 263 || !mddev->ready) {
262 bio_io_error(bio); 264 bio_io_error(bio);
263 return; 265 return;
264 } 266 }
265 if (mddev->ro == 1 && unlikely(rw == WRITE)) { 267 if (mddev->ro == 1 && unlikely(rw == WRITE)) {
266 bio_endio(bio, bio_sectors(bio) == 0 ? 0 : -EROFS); 268 if (bio_sectors(bio) != 0)
269 bio->bi_error = -EROFS;
270 bio_endio(bio);
267 return; 271 return;
268 } 272 }
269 smp_rmb(); /* Ensure implications of 'active' are visible */ 273 smp_rmb(); /* Ensure implications of 'active' are visible */
@@ -350,34 +354,11 @@ static int md_congested(void *data, int bits)
350 return mddev_congested(mddev, bits); 354 return mddev_congested(mddev, bits);
351} 355}
352 356
353static int md_mergeable_bvec(struct request_queue *q,
354 struct bvec_merge_data *bvm,
355 struct bio_vec *biovec)
356{
357 struct mddev *mddev = q->queuedata;
358 int ret;
359 rcu_read_lock();
360 if (mddev->suspended) {
361 /* Must always allow one vec */
362 if (bvm->bi_size == 0)
363 ret = biovec->bv_len;
364 else
365 ret = 0;
366 } else {
367 struct md_personality *pers = mddev->pers;
368 if (pers && pers->mergeable_bvec)
369 ret = pers->mergeable_bvec(mddev, bvm, biovec);
370 else
371 ret = biovec->bv_len;
372 }
373 rcu_read_unlock();
374 return ret;
375}
376/* 357/*
377 * Generic flush handling for md 358 * Generic flush handling for md
378 */ 359 */
379 360
380static void md_end_flush(struct bio *bio, int err) 361static void md_end_flush(struct bio *bio)
381{ 362{
382 struct md_rdev *rdev = bio->bi_private; 363 struct md_rdev *rdev = bio->bi_private;
383 struct mddev *mddev = rdev->mddev; 364 struct mddev *mddev = rdev->mddev;
@@ -433,7 +414,7 @@ static void md_submit_flush_data(struct work_struct *ws)
433 414
434 if (bio->bi_iter.bi_size == 0) 415 if (bio->bi_iter.bi_size == 0)
435 /* an empty barrier - all done */ 416 /* an empty barrier - all done */
436 bio_endio(bio, 0); 417 bio_endio(bio);
437 else { 418 else {
438 bio->bi_rw &= ~REQ_FLUSH; 419 bio->bi_rw &= ~REQ_FLUSH;
439 mddev->pers->make_request(mddev, bio); 420 mddev->pers->make_request(mddev, bio);
@@ -728,15 +709,13 @@ void md_rdev_clear(struct md_rdev *rdev)
728} 709}
729EXPORT_SYMBOL_GPL(md_rdev_clear); 710EXPORT_SYMBOL_GPL(md_rdev_clear);
730 711
731static void super_written(struct bio *bio, int error) 712static void super_written(struct bio *bio)
732{ 713{
733 struct md_rdev *rdev = bio->bi_private; 714 struct md_rdev *rdev = bio->bi_private;
734 struct mddev *mddev = rdev->mddev; 715 struct mddev *mddev = rdev->mddev;
735 716
736 if (error || !test_bit(BIO_UPTODATE, &bio->bi_flags)) { 717 if (bio->bi_error) {
737 printk("md: super_written gets error=%d, uptodate=%d\n", 718 printk("md: super_written gets error=%d\n", bio->bi_error);
738 error, test_bit(BIO_UPTODATE, &bio->bi_flags));
739 WARN_ON(test_bit(BIO_UPTODATE, &bio->bi_flags));
740 md_error(mddev, rdev); 719 md_error(mddev, rdev);
741 } 720 }
742 721
@@ -791,7 +770,7 @@ int sync_page_io(struct md_rdev *rdev, sector_t sector, int size,
791 bio_add_page(bio, page, size, 0); 770 bio_add_page(bio, page, size, 0);
792 submit_bio_wait(rw, bio); 771 submit_bio_wait(rw, bio);
793 772
794 ret = test_bit(BIO_UPTODATE, &bio->bi_flags); 773 ret = !bio->bi_error;
795 bio_put(bio); 774 bio_put(bio);
796 return ret; 775 return ret;
797} 776}
@@ -5186,7 +5165,6 @@ int md_run(struct mddev *mddev)
5186 if (mddev->queue) { 5165 if (mddev->queue) {
5187 mddev->queue->backing_dev_info.congested_data = mddev; 5166 mddev->queue->backing_dev_info.congested_data = mddev;
5188 mddev->queue->backing_dev_info.congested_fn = md_congested; 5167 mddev->queue->backing_dev_info.congested_fn = md_congested;
5189 blk_queue_merge_bvec(mddev->queue, md_mergeable_bvec);
5190 } 5168 }
5191 if (pers->sync_request) { 5169 if (pers->sync_request) {
5192 if (mddev->kobj.sd && 5170 if (mddev->kobj.sd &&
@@ -5315,7 +5293,6 @@ static void md_clean(struct mddev *mddev)
5315 mddev->degraded = 0; 5293 mddev->degraded = 0;
5316 mddev->safemode = 0; 5294 mddev->safemode = 0;
5317 mddev->private = NULL; 5295 mddev->private = NULL;
5318 mddev->merge_check_needed = 0;
5319 mddev->bitmap_info.offset = 0; 5296 mddev->bitmap_info.offset = 0;
5320 mddev->bitmap_info.default_offset = 0; 5297 mddev->bitmap_info.default_offset = 0;
5321 mddev->bitmap_info.default_space = 0; 5298 mddev->bitmap_info.default_space = 0;
@@ -5514,7 +5491,6 @@ static int do_md_stop(struct mddev *mddev, int mode,
5514 5491
5515 __md_stop_writes(mddev); 5492 __md_stop_writes(mddev);
5516 __md_stop(mddev); 5493 __md_stop(mddev);
5517 mddev->queue->merge_bvec_fn = NULL;
5518 mddev->queue->backing_dev_info.congested_fn = NULL; 5494 mddev->queue->backing_dev_info.congested_fn = NULL;
5519 5495
5520 /* tell userspace to handle 'inactive' */ 5496 /* tell userspace to handle 'inactive' */
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 7da6e9c3cb53..ab339571e57f 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -134,10 +134,6 @@ enum flag_bits {
134 Bitmap_sync, /* ..actually, not quite In_sync. Need a 134 Bitmap_sync, /* ..actually, not quite In_sync. Need a
135 * bitmap-based recovery to get fully in sync 135 * bitmap-based recovery to get fully in sync
136 */ 136 */
137 Unmerged, /* device is being added to array and should
138 * be considerred for bvec_merge_fn but not
139 * yet for actual IO
140 */
141 WriteMostly, /* Avoid reading if at all possible */ 137 WriteMostly, /* Avoid reading if at all possible */
142 AutoDetected, /* added by auto-detect */ 138 AutoDetected, /* added by auto-detect */
143 Blocked, /* An error occurred but has not yet 139 Blocked, /* An error occurred but has not yet
@@ -374,10 +370,6 @@ struct mddev {
374 int degraded; /* whether md should consider 370 int degraded; /* whether md should consider
375 * adding a spare 371 * adding a spare
376 */ 372 */
377 int merge_check_needed; /* at least one
378 * member device
379 * has a
380 * merge_bvec_fn */
381 373
382 atomic_t recovery_active; /* blocks scheduled, but not written */ 374 atomic_t recovery_active; /* blocks scheduled, but not written */
383 wait_queue_head_t recovery_wait; 375 wait_queue_head_t recovery_wait;
@@ -532,10 +524,6 @@ struct md_personality
532 /* congested implements bdi.congested_fn(). 524 /* congested implements bdi.congested_fn().
533 * Will not be called while array is 'suspended' */ 525 * Will not be called while array is 'suspended' */
534 int (*congested)(struct mddev *mddev, int bits); 526 int (*congested)(struct mddev *mddev, int bits);
535 /* mergeable_bvec is use to implement ->merge_bvec_fn */
536 int (*mergeable_bvec)(struct mddev *mddev,
537 struct bvec_merge_data *bvm,
538 struct bio_vec *biovec);
539}; 527};
540 528
541struct md_sysfs_entry { 529struct md_sysfs_entry {
diff --git a/drivers/md/multipath.c b/drivers/md/multipath.c
index ac3ede2bd00e..d222522c52e0 100644
--- a/drivers/md/multipath.c
+++ b/drivers/md/multipath.c
@@ -77,18 +77,18 @@ static void multipath_end_bh_io (struct multipath_bh *mp_bh, int err)
77 struct bio *bio = mp_bh->master_bio; 77 struct bio *bio = mp_bh->master_bio;
78 struct mpconf *conf = mp_bh->mddev->private; 78 struct mpconf *conf = mp_bh->mddev->private;
79 79
80 bio_endio(bio, err); 80 bio->bi_error = err;
81 bio_endio(bio);
81 mempool_free(mp_bh, conf->pool); 82 mempool_free(mp_bh, conf->pool);
82} 83}
83 84
84static void multipath_end_request(struct bio *bio, int error) 85static void multipath_end_request(struct bio *bio)
85{ 86{
86 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
87 struct multipath_bh *mp_bh = bio->bi_private; 87 struct multipath_bh *mp_bh = bio->bi_private;
88 struct mpconf *conf = mp_bh->mddev->private; 88 struct mpconf *conf = mp_bh->mddev->private;
89 struct md_rdev *rdev = conf->multipaths[mp_bh->path].rdev; 89 struct md_rdev *rdev = conf->multipaths[mp_bh->path].rdev;
90 90
91 if (uptodate) 91 if (!bio->bi_error)
92 multipath_end_bh_io(mp_bh, 0); 92 multipath_end_bh_io(mp_bh, 0);
93 else if (!(bio->bi_rw & REQ_RAHEAD)) { 93 else if (!(bio->bi_rw & REQ_RAHEAD)) {
94 /* 94 /*
@@ -101,7 +101,7 @@ static void multipath_end_request(struct bio *bio, int error)
101 (unsigned long long)bio->bi_iter.bi_sector); 101 (unsigned long long)bio->bi_iter.bi_sector);
102 multipath_reschedule_retry(mp_bh); 102 multipath_reschedule_retry(mp_bh);
103 } else 103 } else
104 multipath_end_bh_io(mp_bh, error); 104 multipath_end_bh_io(mp_bh, bio->bi_error);
105 rdev_dec_pending(rdev, conf->mddev); 105 rdev_dec_pending(rdev, conf->mddev);
106} 106}
107 107
@@ -123,7 +123,7 @@ static void multipath_make_request(struct mddev *mddev, struct bio * bio)
123 123
124 mp_bh->path = multipath_map(conf); 124 mp_bh->path = multipath_map(conf);
125 if (mp_bh->path < 0) { 125 if (mp_bh->path < 0) {
126 bio_endio(bio, -EIO); 126 bio_io_error(bio);
127 mempool_free(mp_bh, conf->pool); 127 mempool_free(mp_bh, conf->pool);
128 return; 128 return;
129 } 129 }
@@ -257,18 +257,6 @@ static int multipath_add_disk(struct mddev *mddev, struct md_rdev *rdev)
257 disk_stack_limits(mddev->gendisk, rdev->bdev, 257 disk_stack_limits(mddev->gendisk, rdev->bdev,
258 rdev->data_offset << 9); 258 rdev->data_offset << 9);
259 259
260 /* as we don't honour merge_bvec_fn, we must never risk
261 * violating it, so limit ->max_segments to one, lying
262 * within a single page.
263 * (Note: it is very unlikely that a device with
264 * merge_bvec_fn will be involved in multipath.)
265 */
266 if (q->merge_bvec_fn) {
267 blk_queue_max_segments(mddev->queue, 1);
268 blk_queue_segment_boundary(mddev->queue,
269 PAGE_CACHE_SIZE - 1);
270 }
271
272 spin_lock_irq(&conf->device_lock); 260 spin_lock_irq(&conf->device_lock);
273 mddev->degraded--; 261 mddev->degraded--;
274 rdev->raid_disk = path; 262 rdev->raid_disk = path;
@@ -432,15 +420,6 @@ static int multipath_run (struct mddev *mddev)
432 disk_stack_limits(mddev->gendisk, rdev->bdev, 420 disk_stack_limits(mddev->gendisk, rdev->bdev,
433 rdev->data_offset << 9); 421 rdev->data_offset << 9);
434 422
435 /* as we don't honour merge_bvec_fn, we must never risk
436 * violating it, not that we ever expect a device with
437 * a merge_bvec_fn to be involved in multipath */
438 if (rdev->bdev->bd_disk->queue->merge_bvec_fn) {
439 blk_queue_max_segments(mddev->queue, 1);
440 blk_queue_segment_boundary(mddev->queue,
441 PAGE_CACHE_SIZE - 1);
442 }
443
444 if (!test_bit(Faulty, &rdev->flags)) 423 if (!test_bit(Faulty, &rdev->flags))
445 working_disks++; 424 working_disks++;
446 } 425 }
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
index efb654eb5399..59cda501a224 100644
--- a/drivers/md/raid0.c
+++ b/drivers/md/raid0.c
@@ -192,9 +192,6 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf)
192 disk_stack_limits(mddev->gendisk, rdev1->bdev, 192 disk_stack_limits(mddev->gendisk, rdev1->bdev,
193 rdev1->data_offset << 9); 193 rdev1->data_offset << 9);
194 194
195 if (rdev1->bdev->bd_disk->queue->merge_bvec_fn)
196 conf->has_merge_bvec = 1;
197
198 if (!smallest || (rdev1->sectors < smallest->sectors)) 195 if (!smallest || (rdev1->sectors < smallest->sectors))
199 smallest = rdev1; 196 smallest = rdev1;
200 cnt++; 197 cnt++;
@@ -351,58 +348,6 @@ static struct md_rdev *map_sector(struct mddev *mddev, struct strip_zone *zone,
351 + sector_div(sector, zone->nb_dev)]; 348 + sector_div(sector, zone->nb_dev)];
352} 349}
353 350
354/**
355 * raid0_mergeable_bvec -- tell bio layer if two requests can be merged
356 * @mddev: the md device
357 * @bvm: properties of new bio
358 * @biovec: the request that could be merged to it.
359 *
360 * Return amount of bytes we can accept at this offset
361 */
362static int raid0_mergeable_bvec(struct mddev *mddev,
363 struct bvec_merge_data *bvm,
364 struct bio_vec *biovec)
365{
366 struct r0conf *conf = mddev->private;
367 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
368 sector_t sector_offset = sector;
369 int max;
370 unsigned int chunk_sectors = mddev->chunk_sectors;
371 unsigned int bio_sectors = bvm->bi_size >> 9;
372 struct strip_zone *zone;
373 struct md_rdev *rdev;
374 struct request_queue *subq;
375
376 if (is_power_of_2(chunk_sectors))
377 max = (chunk_sectors - ((sector & (chunk_sectors-1))
378 + bio_sectors)) << 9;
379 else
380 max = (chunk_sectors - (sector_div(sector, chunk_sectors)
381 + bio_sectors)) << 9;
382 if (max < 0)
383 max = 0; /* bio_add cannot handle a negative return */
384 if (max <= biovec->bv_len && bio_sectors == 0)
385 return biovec->bv_len;
386 if (max < biovec->bv_len)
387 /* too small already, no need to check further */
388 return max;
389 if (!conf->has_merge_bvec)
390 return max;
391
392 /* May need to check subordinate device */
393 sector = sector_offset;
394 zone = find_zone(mddev->private, &sector_offset);
395 rdev = map_sector(mddev, zone, sector, &sector_offset);
396 subq = bdev_get_queue(rdev->bdev);
397 if (subq->merge_bvec_fn) {
398 bvm->bi_bdev = rdev->bdev;
399 bvm->bi_sector = sector_offset + zone->dev_start +
400 rdev->data_offset;
401 return min(max, subq->merge_bvec_fn(subq, bvm, biovec));
402 } else
403 return max;
404}
405
406static sector_t raid0_size(struct mddev *mddev, sector_t sectors, int raid_disks) 351static sector_t raid0_size(struct mddev *mddev, sector_t sectors, int raid_disks)
407{ 352{
408 sector_t array_sectors = 0; 353 sector_t array_sectors = 0;
@@ -543,7 +488,7 @@ static void raid0_make_request(struct mddev *mddev, struct bio *bio)
543 if (unlikely((split->bi_rw & REQ_DISCARD) && 488 if (unlikely((split->bi_rw & REQ_DISCARD) &&
544 !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) { 489 !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) {
545 /* Just ignore it */ 490 /* Just ignore it */
546 bio_endio(split, 0); 491 bio_endio(split);
547 } else 492 } else
548 generic_make_request(split); 493 generic_make_request(split);
549 } while (split != bio); 494 } while (split != bio);
@@ -727,7 +672,6 @@ static struct md_personality raid0_personality=
727 .takeover = raid0_takeover, 672 .takeover = raid0_takeover,
728 .quiesce = raid0_quiesce, 673 .quiesce = raid0_quiesce,
729 .congested = raid0_congested, 674 .congested = raid0_congested,
730 .mergeable_bvec = raid0_mergeable_bvec,
731}; 675};
732 676
733static int __init raid0_init (void) 677static int __init raid0_init (void)
diff --git a/drivers/md/raid0.h b/drivers/md/raid0.h
index 05539d9c97f0..7127a623f5da 100644
--- a/drivers/md/raid0.h
+++ b/drivers/md/raid0.h
@@ -12,8 +12,6 @@ struct r0conf {
12 struct md_rdev **devlist; /* lists of rdevs, pointed to 12 struct md_rdev **devlist; /* lists of rdevs, pointed to
13 * by strip_zone->dev */ 13 * by strip_zone->dev */
14 int nr_strip_zones; 14 int nr_strip_zones;
15 int has_merge_bvec; /* at least one member has
16 * a merge_bvec_fn */
17}; 15};
18 16
19#endif 17#endif
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 967a4ed73929..f39d69f884de 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -255,9 +255,10 @@ static void call_bio_endio(struct r1bio *r1_bio)
255 done = 1; 255 done = 1;
256 256
257 if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) 257 if (!test_bit(R1BIO_Uptodate, &r1_bio->state))
258 clear_bit(BIO_UPTODATE, &bio->bi_flags); 258 bio->bi_error = -EIO;
259
259 if (done) { 260 if (done) {
260 bio_endio(bio, 0); 261 bio_endio(bio);
261 /* 262 /*
262 * Wake up any possible resync thread that waits for the device 263 * Wake up any possible resync thread that waits for the device
263 * to go idle. 264 * to go idle.
@@ -312,9 +313,9 @@ static int find_bio_disk(struct r1bio *r1_bio, struct bio *bio)
312 return mirror; 313 return mirror;
313} 314}
314 315
315static void raid1_end_read_request(struct bio *bio, int error) 316static void raid1_end_read_request(struct bio *bio)
316{ 317{
317 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 318 int uptodate = !bio->bi_error;
318 struct r1bio *r1_bio = bio->bi_private; 319 struct r1bio *r1_bio = bio->bi_private;
319 int mirror; 320 int mirror;
320 struct r1conf *conf = r1_bio->mddev->private; 321 struct r1conf *conf = r1_bio->mddev->private;
@@ -397,9 +398,8 @@ static void r1_bio_write_done(struct r1bio *r1_bio)
397 } 398 }
398} 399}
399 400
400static void raid1_end_write_request(struct bio *bio, int error) 401static void raid1_end_write_request(struct bio *bio)
401{ 402{
402 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
403 struct r1bio *r1_bio = bio->bi_private; 403 struct r1bio *r1_bio = bio->bi_private;
404 int mirror, behind = test_bit(R1BIO_BehindIO, &r1_bio->state); 404 int mirror, behind = test_bit(R1BIO_BehindIO, &r1_bio->state);
405 struct r1conf *conf = r1_bio->mddev->private; 405 struct r1conf *conf = r1_bio->mddev->private;
@@ -410,7 +410,7 @@ static void raid1_end_write_request(struct bio *bio, int error)
410 /* 410 /*
411 * 'one mirror IO has finished' event handler: 411 * 'one mirror IO has finished' event handler:
412 */ 412 */
413 if (!uptodate) { 413 if (bio->bi_error) {
414 set_bit(WriteErrorSeen, 414 set_bit(WriteErrorSeen,
415 &conf->mirrors[mirror].rdev->flags); 415 &conf->mirrors[mirror].rdev->flags);
416 if (!test_and_set_bit(WantReplacement, 416 if (!test_and_set_bit(WantReplacement,
@@ -557,7 +557,6 @@ static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sect
557 rdev = rcu_dereference(conf->mirrors[disk].rdev); 557 rdev = rcu_dereference(conf->mirrors[disk].rdev);
558 if (r1_bio->bios[disk] == IO_BLOCKED 558 if (r1_bio->bios[disk] == IO_BLOCKED
559 || rdev == NULL 559 || rdev == NULL
560 || test_bit(Unmerged, &rdev->flags)
561 || test_bit(Faulty, &rdev->flags)) 560 || test_bit(Faulty, &rdev->flags))
562 continue; 561 continue;
563 if (!test_bit(In_sync, &rdev->flags) && 562 if (!test_bit(In_sync, &rdev->flags) &&
@@ -708,38 +707,6 @@ static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sect
708 return best_disk; 707 return best_disk;
709} 708}
710 709
711static int raid1_mergeable_bvec(struct mddev *mddev,
712 struct bvec_merge_data *bvm,
713 struct bio_vec *biovec)
714{
715 struct r1conf *conf = mddev->private;
716 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
717 int max = biovec->bv_len;
718
719 if (mddev->merge_check_needed) {
720 int disk;
721 rcu_read_lock();
722 for (disk = 0; disk < conf->raid_disks * 2; disk++) {
723 struct md_rdev *rdev = rcu_dereference(
724 conf->mirrors[disk].rdev);
725 if (rdev && !test_bit(Faulty, &rdev->flags)) {
726 struct request_queue *q =
727 bdev_get_queue(rdev->bdev);
728 if (q->merge_bvec_fn) {
729 bvm->bi_sector = sector +
730 rdev->data_offset;
731 bvm->bi_bdev = rdev->bdev;
732 max = min(max, q->merge_bvec_fn(
733 q, bvm, biovec));
734 }
735 }
736 }
737 rcu_read_unlock();
738 }
739 return max;
740
741}
742
743static int raid1_congested(struct mddev *mddev, int bits) 710static int raid1_congested(struct mddev *mddev, int bits)
744{ 711{
745 struct r1conf *conf = mddev->private; 712 struct r1conf *conf = mddev->private;
@@ -793,7 +760,7 @@ static void flush_pending_writes(struct r1conf *conf)
793 if (unlikely((bio->bi_rw & REQ_DISCARD) && 760 if (unlikely((bio->bi_rw & REQ_DISCARD) &&
794 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 761 !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
795 /* Just ignore it */ 762 /* Just ignore it */
796 bio_endio(bio, 0); 763 bio_endio(bio);
797 else 764 else
798 generic_make_request(bio); 765 generic_make_request(bio);
799 bio = next; 766 bio = next;
@@ -1068,7 +1035,7 @@ static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule)
1068 if (unlikely((bio->bi_rw & REQ_DISCARD) && 1035 if (unlikely((bio->bi_rw & REQ_DISCARD) &&
1069 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 1036 !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
1070 /* Just ignore it */ 1037 /* Just ignore it */
1071 bio_endio(bio, 0); 1038 bio_endio(bio);
1072 else 1039 else
1073 generic_make_request(bio); 1040 generic_make_request(bio);
1074 bio = next; 1041 bio = next;
@@ -1158,7 +1125,7 @@ static void make_request(struct mddev *mddev, struct bio * bio)
1158 * non-zero, then it is the number of not-completed requests. 1125 * non-zero, then it is the number of not-completed requests.
1159 */ 1126 */
1160 bio->bi_phys_segments = 0; 1127 bio->bi_phys_segments = 0;
1161 clear_bit(BIO_SEG_VALID, &bio->bi_flags); 1128 bio_clear_flag(bio, BIO_SEG_VALID);
1162 1129
1163 if (rw == READ) { 1130 if (rw == READ) {
1164 /* 1131 /*
@@ -1269,8 +1236,7 @@ read_again:
1269 break; 1236 break;
1270 } 1237 }
1271 r1_bio->bios[i] = NULL; 1238 r1_bio->bios[i] = NULL;
1272 if (!rdev || test_bit(Faulty, &rdev->flags) 1239 if (!rdev || test_bit(Faulty, &rdev->flags)) {
1273 || test_bit(Unmerged, &rdev->flags)) {
1274 if (i < conf->raid_disks) 1240 if (i < conf->raid_disks)
1275 set_bit(R1BIO_Degraded, &r1_bio->state); 1241 set_bit(R1BIO_Degraded, &r1_bio->state);
1276 continue; 1242 continue;
@@ -1617,7 +1583,6 @@ static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1617 struct raid1_info *p; 1583 struct raid1_info *p;
1618 int first = 0; 1584 int first = 0;
1619 int last = conf->raid_disks - 1; 1585 int last = conf->raid_disks - 1;
1620 struct request_queue *q = bdev_get_queue(rdev->bdev);
1621 1586
1622 if (mddev->recovery_disabled == conf->recovery_disabled) 1587 if (mddev->recovery_disabled == conf->recovery_disabled)
1623 return -EBUSY; 1588 return -EBUSY;
@@ -1625,11 +1590,6 @@ static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1625 if (rdev->raid_disk >= 0) 1590 if (rdev->raid_disk >= 0)
1626 first = last = rdev->raid_disk; 1591 first = last = rdev->raid_disk;
1627 1592
1628 if (q->merge_bvec_fn) {
1629 set_bit(Unmerged, &rdev->flags);
1630 mddev->merge_check_needed = 1;
1631 }
1632
1633 for (mirror = first; mirror <= last; mirror++) { 1593 for (mirror = first; mirror <= last; mirror++) {
1634 p = conf->mirrors+mirror; 1594 p = conf->mirrors+mirror;
1635 if (!p->rdev) { 1595 if (!p->rdev) {
@@ -1661,19 +1621,6 @@ static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1661 break; 1621 break;
1662 } 1622 }
1663 } 1623 }
1664 if (err == 0 && test_bit(Unmerged, &rdev->flags)) {
1665 /* Some requests might not have seen this new
1666 * merge_bvec_fn. We must wait for them to complete
1667 * before merging the device fully.
1668 * First we make sure any code which has tested
1669 * our function has submitted the request, then
1670 * we wait for all outstanding requests to complete.
1671 */
1672 synchronize_sched();
1673 freeze_array(conf, 0);
1674 unfreeze_array(conf);
1675 clear_bit(Unmerged, &rdev->flags);
1676 }
1677 md_integrity_add_rdev(rdev, mddev); 1624 md_integrity_add_rdev(rdev, mddev);
1678 if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev))) 1625 if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
1679 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue); 1626 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);
@@ -1737,7 +1684,7 @@ abort:
1737 return err; 1684 return err;
1738} 1685}
1739 1686
1740static void end_sync_read(struct bio *bio, int error) 1687static void end_sync_read(struct bio *bio)
1741{ 1688{
1742 struct r1bio *r1_bio = bio->bi_private; 1689 struct r1bio *r1_bio = bio->bi_private;
1743 1690
@@ -1748,16 +1695,16 @@ static void end_sync_read(struct bio *bio, int error)
1748 * or re-read if the read failed. 1695 * or re-read if the read failed.
1749 * We don't do much here, just schedule handling by raid1d 1696 * We don't do much here, just schedule handling by raid1d
1750 */ 1697 */
1751 if (test_bit(BIO_UPTODATE, &bio->bi_flags)) 1698 if (!bio->bi_error)
1752 set_bit(R1BIO_Uptodate, &r1_bio->state); 1699 set_bit(R1BIO_Uptodate, &r1_bio->state);
1753 1700
1754 if (atomic_dec_and_test(&r1_bio->remaining)) 1701 if (atomic_dec_and_test(&r1_bio->remaining))
1755 reschedule_retry(r1_bio); 1702 reschedule_retry(r1_bio);
1756} 1703}
1757 1704
1758static void end_sync_write(struct bio *bio, int error) 1705static void end_sync_write(struct bio *bio)
1759{ 1706{
1760 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 1707 int uptodate = !bio->bi_error;
1761 struct r1bio *r1_bio = bio->bi_private; 1708 struct r1bio *r1_bio = bio->bi_private;
1762 struct mddev *mddev = r1_bio->mddev; 1709 struct mddev *mddev = r1_bio->mddev;
1763 struct r1conf *conf = mddev->private; 1710 struct r1conf *conf = mddev->private;
@@ -1944,7 +1891,7 @@ static int fix_sync_read_error(struct r1bio *r1_bio)
1944 idx ++; 1891 idx ++;
1945 } 1892 }
1946 set_bit(R1BIO_Uptodate, &r1_bio->state); 1893 set_bit(R1BIO_Uptodate, &r1_bio->state);
1947 set_bit(BIO_UPTODATE, &bio->bi_flags); 1894 bio->bi_error = 0;
1948 return 1; 1895 return 1;
1949} 1896}
1950 1897
@@ -1968,15 +1915,14 @@ static void process_checks(struct r1bio *r1_bio)
1968 for (i = 0; i < conf->raid_disks * 2; i++) { 1915 for (i = 0; i < conf->raid_disks * 2; i++) {
1969 int j; 1916 int j;
1970 int size; 1917 int size;
1971 int uptodate; 1918 int error;
1972 struct bio *b = r1_bio->bios[i]; 1919 struct bio *b = r1_bio->bios[i];
1973 if (b->bi_end_io != end_sync_read) 1920 if (b->bi_end_io != end_sync_read)
1974 continue; 1921 continue;
1975 /* fixup the bio for reuse, but preserve BIO_UPTODATE */ 1922 /* fixup the bio for reuse, but preserve errno */
1976 uptodate = test_bit(BIO_UPTODATE, &b->bi_flags); 1923 error = b->bi_error;
1977 bio_reset(b); 1924 bio_reset(b);
1978 if (!uptodate) 1925 b->bi_error = error;
1979 clear_bit(BIO_UPTODATE, &b->bi_flags);
1980 b->bi_vcnt = vcnt; 1926 b->bi_vcnt = vcnt;
1981 b->bi_iter.bi_size = r1_bio->sectors << 9; 1927 b->bi_iter.bi_size = r1_bio->sectors << 9;
1982 b->bi_iter.bi_sector = r1_bio->sector + 1928 b->bi_iter.bi_sector = r1_bio->sector +
@@ -1999,7 +1945,7 @@ static void process_checks(struct r1bio *r1_bio)
1999 } 1945 }
2000 for (primary = 0; primary < conf->raid_disks * 2; primary++) 1946 for (primary = 0; primary < conf->raid_disks * 2; primary++)
2001 if (r1_bio->bios[primary]->bi_end_io == end_sync_read && 1947 if (r1_bio->bios[primary]->bi_end_io == end_sync_read &&
2002 test_bit(BIO_UPTODATE, &r1_bio->bios[primary]->bi_flags)) { 1948 !r1_bio->bios[primary]->bi_error) {
2003 r1_bio->bios[primary]->bi_end_io = NULL; 1949 r1_bio->bios[primary]->bi_end_io = NULL;
2004 rdev_dec_pending(conf->mirrors[primary].rdev, mddev); 1950 rdev_dec_pending(conf->mirrors[primary].rdev, mddev);
2005 break; 1951 break;
@@ -2009,14 +1955,14 @@ static void process_checks(struct r1bio *r1_bio)
2009 int j; 1955 int j;
2010 struct bio *pbio = r1_bio->bios[primary]; 1956 struct bio *pbio = r1_bio->bios[primary];
2011 struct bio *sbio = r1_bio->bios[i]; 1957 struct bio *sbio = r1_bio->bios[i];
2012 int uptodate = test_bit(BIO_UPTODATE, &sbio->bi_flags); 1958 int error = sbio->bi_error;
2013 1959
2014 if (sbio->bi_end_io != end_sync_read) 1960 if (sbio->bi_end_io != end_sync_read)
2015 continue; 1961 continue;
2016 /* Now we can 'fixup' the BIO_UPTODATE flag */ 1962 /* Now we can 'fixup' the error value */
2017 set_bit(BIO_UPTODATE, &sbio->bi_flags); 1963 sbio->bi_error = 0;
2018 1964
2019 if (uptodate) { 1965 if (!error) {
2020 for (j = vcnt; j-- ; ) { 1966 for (j = vcnt; j-- ; ) {
2021 struct page *p, *s; 1967 struct page *p, *s;
2022 p = pbio->bi_io_vec[j].bv_page; 1968 p = pbio->bi_io_vec[j].bv_page;
@@ -2031,7 +1977,7 @@ static void process_checks(struct r1bio *r1_bio)
2031 if (j >= 0) 1977 if (j >= 0)
2032 atomic64_add(r1_bio->sectors, &mddev->resync_mismatches); 1978 atomic64_add(r1_bio->sectors, &mddev->resync_mismatches);
2033 if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery) 1979 if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)
2034 && uptodate)) { 1980 && !error)) {
2035 /* No need to write to this device. */ 1981 /* No need to write to this device. */
2036 sbio->bi_end_io = NULL; 1982 sbio->bi_end_io = NULL;
2037 rdev_dec_pending(conf->mirrors[i].rdev, mddev); 1983 rdev_dec_pending(conf->mirrors[i].rdev, mddev);
@@ -2272,11 +2218,11 @@ static void handle_sync_write_finished(struct r1conf *conf, struct r1bio *r1_bio
2272 struct bio *bio = r1_bio->bios[m]; 2218 struct bio *bio = r1_bio->bios[m];
2273 if (bio->bi_end_io == NULL) 2219 if (bio->bi_end_io == NULL)
2274 continue; 2220 continue;
2275 if (test_bit(BIO_UPTODATE, &bio->bi_flags) && 2221 if (!bio->bi_error &&
2276 test_bit(R1BIO_MadeGood, &r1_bio->state)) { 2222 test_bit(R1BIO_MadeGood, &r1_bio->state)) {
2277 rdev_clear_badblocks(rdev, r1_bio->sector, s, 0); 2223 rdev_clear_badblocks(rdev, r1_bio->sector, s, 0);
2278 } 2224 }
2279 if (!test_bit(BIO_UPTODATE, &bio->bi_flags) && 2225 if (bio->bi_error &&
2280 test_bit(R1BIO_WriteError, &r1_bio->state)) { 2226 test_bit(R1BIO_WriteError, &r1_bio->state)) {
2281 if (!rdev_set_badblocks(rdev, r1_bio->sector, s, 0)) 2227 if (!rdev_set_badblocks(rdev, r1_bio->sector, s, 0))
2282 md_error(conf->mddev, rdev); 2228 md_error(conf->mddev, rdev);
@@ -2715,7 +2661,7 @@ static sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipp
2715 /* remove last page from this bio */ 2661 /* remove last page from this bio */
2716 bio->bi_vcnt--; 2662 bio->bi_vcnt--;
2717 bio->bi_iter.bi_size -= len; 2663 bio->bi_iter.bi_size -= len;
2718 __clear_bit(BIO_SEG_VALID, &bio->bi_flags); 2664 bio_clear_flag(bio, BIO_SEG_VALID);
2719 } 2665 }
2720 goto bio_full; 2666 goto bio_full;
2721 } 2667 }
@@ -2810,8 +2756,6 @@ static struct r1conf *setup_conf(struct mddev *mddev)
2810 goto abort; 2756 goto abort;
2811 disk->rdev = rdev; 2757 disk->rdev = rdev;
2812 q = bdev_get_queue(rdev->bdev); 2758 q = bdev_get_queue(rdev->bdev);
2813 if (q->merge_bvec_fn)
2814 mddev->merge_check_needed = 1;
2815 2759
2816 disk->head_position = 0; 2760 disk->head_position = 0;
2817 disk->seq_start = MaxSector; 2761 disk->seq_start = MaxSector;
@@ -3176,7 +3120,6 @@ static struct md_personality raid1_personality =
3176 .quiesce = raid1_quiesce, 3120 .quiesce = raid1_quiesce,
3177 .takeover = raid1_takeover, 3121 .takeover = raid1_takeover,
3178 .congested = raid1_congested, 3122 .congested = raid1_congested,
3179 .mergeable_bvec = raid1_mergeable_bvec,
3180}; 3123};
3181 3124
3182static int __init raid_init(void) 3125static int __init raid_init(void)
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index 38c58e19cfce..b0fce2ebf7ad 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -101,7 +101,7 @@ static int _enough(struct r10conf *conf, int previous, int ignore);
101static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 101static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
102 int *skipped); 102 int *skipped);
103static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio); 103static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
104static void end_reshape_write(struct bio *bio, int error); 104static void end_reshape_write(struct bio *bio);
105static void end_reshape(struct r10conf *conf); 105static void end_reshape(struct r10conf *conf);
106 106
107static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) 107static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
@@ -307,9 +307,9 @@ static void raid_end_bio_io(struct r10bio *r10_bio)
307 } else 307 } else
308 done = 1; 308 done = 1;
309 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 309 if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
310 clear_bit(BIO_UPTODATE, &bio->bi_flags); 310 bio->bi_error = -EIO;
311 if (done) { 311 if (done) {
312 bio_endio(bio, 0); 312 bio_endio(bio);
313 /* 313 /*
314 * Wake up any possible resync thread that waits for the device 314 * Wake up any possible resync thread that waits for the device
315 * to go idle. 315 * to go idle.
@@ -358,9 +358,9 @@ static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
358 return r10_bio->devs[slot].devnum; 358 return r10_bio->devs[slot].devnum;
359} 359}
360 360
361static void raid10_end_read_request(struct bio *bio, int error) 361static void raid10_end_read_request(struct bio *bio)
362{ 362{
363 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 363 int uptodate = !bio->bi_error;
364 struct r10bio *r10_bio = bio->bi_private; 364 struct r10bio *r10_bio = bio->bi_private;
365 int slot, dev; 365 int slot, dev;
366 struct md_rdev *rdev; 366 struct md_rdev *rdev;
@@ -438,9 +438,8 @@ static void one_write_done(struct r10bio *r10_bio)
438 } 438 }
439} 439}
440 440
441static void raid10_end_write_request(struct bio *bio, int error) 441static void raid10_end_write_request(struct bio *bio)
442{ 442{
443 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
444 struct r10bio *r10_bio = bio->bi_private; 443 struct r10bio *r10_bio = bio->bi_private;
445 int dev; 444 int dev;
446 int dec_rdev = 1; 445 int dec_rdev = 1;
@@ -460,7 +459,7 @@ static void raid10_end_write_request(struct bio *bio, int error)
460 /* 459 /*
461 * this branch is our 'one mirror IO has finished' event handler: 460 * this branch is our 'one mirror IO has finished' event handler:
462 */ 461 */
463 if (!uptodate) { 462 if (bio->bi_error) {
464 if (repl) 463 if (repl)
465 /* Never record new bad blocks to replacement, 464 /* Never record new bad blocks to replacement,
466 * just fail it. 465 * just fail it.
@@ -672,93 +671,6 @@ static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
672 return (vchunk << geo->chunk_shift) + offset; 671 return (vchunk << geo->chunk_shift) + offset;
673} 672}
674 673
675/**
676 * raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
677 * @mddev: the md device
678 * @bvm: properties of new bio
679 * @biovec: the request that could be merged to it.
680 *
681 * Return amount of bytes we can accept at this offset
682 * This requires checking for end-of-chunk if near_copies != raid_disks,
683 * and for subordinate merge_bvec_fns if merge_check_needed.
684 */
685static int raid10_mergeable_bvec(struct mddev *mddev,
686 struct bvec_merge_data *bvm,
687 struct bio_vec *biovec)
688{
689 struct r10conf *conf = mddev->private;
690 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
691 int max;
692 unsigned int chunk_sectors;
693 unsigned int bio_sectors = bvm->bi_size >> 9;
694 struct geom *geo = &conf->geo;
695
696 chunk_sectors = (conf->geo.chunk_mask & conf->prev.chunk_mask) + 1;
697 if (conf->reshape_progress != MaxSector &&
698 ((sector >= conf->reshape_progress) !=
699 conf->mddev->reshape_backwards))
700 geo = &conf->prev;
701
702 if (geo->near_copies < geo->raid_disks) {
703 max = (chunk_sectors - ((sector & (chunk_sectors - 1))
704 + bio_sectors)) << 9;
705 if (max < 0)
706 /* bio_add cannot handle a negative return */
707 max = 0;
708 if (max <= biovec->bv_len && bio_sectors == 0)
709 return biovec->bv_len;
710 } else
711 max = biovec->bv_len;
712
713 if (mddev->merge_check_needed) {
714 struct {
715 struct r10bio r10_bio;
716 struct r10dev devs[conf->copies];
717 } on_stack;
718 struct r10bio *r10_bio = &on_stack.r10_bio;
719 int s;
720 if (conf->reshape_progress != MaxSector) {
721 /* Cannot give any guidance during reshape */
722 if (max <= biovec->bv_len && bio_sectors == 0)
723 return biovec->bv_len;
724 return 0;
725 }
726 r10_bio->sector = sector;
727 raid10_find_phys(conf, r10_bio);
728 rcu_read_lock();
729 for (s = 0; s < conf->copies; s++) {
730 int disk = r10_bio->devs[s].devnum;
731 struct md_rdev *rdev = rcu_dereference(
732 conf->mirrors[disk].rdev);
733 if (rdev && !test_bit(Faulty, &rdev->flags)) {
734 struct request_queue *q =
735 bdev_get_queue(rdev->bdev);
736 if (q->merge_bvec_fn) {
737 bvm->bi_sector = r10_bio->devs[s].addr
738 + rdev->data_offset;
739 bvm->bi_bdev = rdev->bdev;
740 max = min(max, q->merge_bvec_fn(
741 q, bvm, biovec));
742 }
743 }
744 rdev = rcu_dereference(conf->mirrors[disk].replacement);
745 if (rdev && !test_bit(Faulty, &rdev->flags)) {
746 struct request_queue *q =
747 bdev_get_queue(rdev->bdev);
748 if (q->merge_bvec_fn) {
749 bvm->bi_sector = r10_bio->devs[s].addr
750 + rdev->data_offset;
751 bvm->bi_bdev = rdev->bdev;
752 max = min(max, q->merge_bvec_fn(
753 q, bvm, biovec));
754 }
755 }
756 }
757 rcu_read_unlock();
758 }
759 return max;
760}
761
762/* 674/*
763 * This routine returns the disk from which the requested read should 675 * This routine returns the disk from which the requested read should
764 * be done. There is a per-array 'next expected sequential IO' sector 676 * be done. There is a per-array 'next expected sequential IO' sector
@@ -821,12 +733,10 @@ retry:
821 disk = r10_bio->devs[slot].devnum; 733 disk = r10_bio->devs[slot].devnum;
822 rdev = rcu_dereference(conf->mirrors[disk].replacement); 734 rdev = rcu_dereference(conf->mirrors[disk].replacement);
823 if (rdev == NULL || test_bit(Faulty, &rdev->flags) || 735 if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
824 test_bit(Unmerged, &rdev->flags) ||
825 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 736 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
826 rdev = rcu_dereference(conf->mirrors[disk].rdev); 737 rdev = rcu_dereference(conf->mirrors[disk].rdev);
827 if (rdev == NULL || 738 if (rdev == NULL ||
828 test_bit(Faulty, &rdev->flags) || 739 test_bit(Faulty, &rdev->flags))
829 test_bit(Unmerged, &rdev->flags))
830 continue; 740 continue;
831 if (!test_bit(In_sync, &rdev->flags) && 741 if (!test_bit(In_sync, &rdev->flags) &&
832 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 742 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
@@ -957,7 +867,7 @@ static void flush_pending_writes(struct r10conf *conf)
957 if (unlikely((bio->bi_rw & REQ_DISCARD) && 867 if (unlikely((bio->bi_rw & REQ_DISCARD) &&
958 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 868 !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
959 /* Just ignore it */ 869 /* Just ignore it */
960 bio_endio(bio, 0); 870 bio_endio(bio);
961 else 871 else
962 generic_make_request(bio); 872 generic_make_request(bio);
963 bio = next; 873 bio = next;
@@ -1133,7 +1043,7 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
1133 if (unlikely((bio->bi_rw & REQ_DISCARD) && 1043 if (unlikely((bio->bi_rw & REQ_DISCARD) &&
1134 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 1044 !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
1135 /* Just ignore it */ 1045 /* Just ignore it */
1136 bio_endio(bio, 0); 1046 bio_endio(bio);
1137 else 1047 else
1138 generic_make_request(bio); 1048 generic_make_request(bio);
1139 bio = next; 1049 bio = next;
@@ -1217,7 +1127,7 @@ static void __make_request(struct mddev *mddev, struct bio *bio)
1217 * non-zero, then it is the number of not-completed requests. 1127 * non-zero, then it is the number of not-completed requests.
1218 */ 1128 */
1219 bio->bi_phys_segments = 0; 1129 bio->bi_phys_segments = 0;
1220 clear_bit(BIO_SEG_VALID, &bio->bi_flags); 1130 bio_clear_flag(bio, BIO_SEG_VALID);
1221 1131
1222 if (rw == READ) { 1132 if (rw == READ) {
1223 /* 1133 /*
@@ -1326,11 +1236,9 @@ retry_write:
1326 blocked_rdev = rrdev; 1236 blocked_rdev = rrdev;
1327 break; 1237 break;
1328 } 1238 }
1329 if (rdev && (test_bit(Faulty, &rdev->flags) 1239 if (rdev && (test_bit(Faulty, &rdev->flags)))
1330 || test_bit(Unmerged, &rdev->flags)))
1331 rdev = NULL; 1240 rdev = NULL;
1332 if (rrdev && (test_bit(Faulty, &rrdev->flags) 1241 if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1333 || test_bit(Unmerged, &rrdev->flags)))
1334 rrdev = NULL; 1242 rrdev = NULL;
1335 1243
1336 r10_bio->devs[i].bio = NULL; 1244 r10_bio->devs[i].bio = NULL;
@@ -1777,7 +1685,6 @@ static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1777 int mirror; 1685 int mirror;
1778 int first = 0; 1686 int first = 0;
1779 int last = conf->geo.raid_disks - 1; 1687 int last = conf->geo.raid_disks - 1;
1780 struct request_queue *q = bdev_get_queue(rdev->bdev);
1781 1688
1782 if (mddev->recovery_cp < MaxSector) 1689 if (mddev->recovery_cp < MaxSector)
1783 /* only hot-add to in-sync arrays, as recovery is 1690 /* only hot-add to in-sync arrays, as recovery is
@@ -1790,11 +1697,6 @@ static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1790 if (rdev->raid_disk >= 0) 1697 if (rdev->raid_disk >= 0)
1791 first = last = rdev->raid_disk; 1698 first = last = rdev->raid_disk;
1792 1699
1793 if (q->merge_bvec_fn) {
1794 set_bit(Unmerged, &rdev->flags);
1795 mddev->merge_check_needed = 1;
1796 }
1797
1798 if (rdev->saved_raid_disk >= first && 1700 if (rdev->saved_raid_disk >= first &&
1799 conf->mirrors[rdev->saved_raid_disk].rdev == NULL) 1701 conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
1800 mirror = rdev->saved_raid_disk; 1702 mirror = rdev->saved_raid_disk;
@@ -1833,19 +1735,6 @@ static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1833 rcu_assign_pointer(p->rdev, rdev); 1735 rcu_assign_pointer(p->rdev, rdev);
1834 break; 1736 break;
1835 } 1737 }
1836 if (err == 0 && test_bit(Unmerged, &rdev->flags)) {
1837 /* Some requests might not have seen this new
1838 * merge_bvec_fn. We must wait for them to complete
1839 * before merging the device fully.
1840 * First we make sure any code which has tested
1841 * our function has submitted the request, then
1842 * we wait for all outstanding requests to complete.
1843 */
1844 synchronize_sched();
1845 freeze_array(conf, 0);
1846 unfreeze_array(conf);
1847 clear_bit(Unmerged, &rdev->flags);
1848 }
1849 md_integrity_add_rdev(rdev, mddev); 1738 md_integrity_add_rdev(rdev, mddev);
1850 if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev))) 1739 if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
1851 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue); 1740 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);
@@ -1916,7 +1805,7 @@ abort:
1916 return err; 1805 return err;
1917} 1806}
1918 1807
1919static void end_sync_read(struct bio *bio, int error) 1808static void end_sync_read(struct bio *bio)
1920{ 1809{
1921 struct r10bio *r10_bio = bio->bi_private; 1810 struct r10bio *r10_bio = bio->bi_private;
1922 struct r10conf *conf = r10_bio->mddev->private; 1811 struct r10conf *conf = r10_bio->mddev->private;
@@ -1928,7 +1817,7 @@ static void end_sync_read(struct bio *bio, int error)
1928 } else 1817 } else
1929 d = find_bio_disk(conf, r10_bio, bio, NULL, NULL); 1818 d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
1930 1819
1931 if (test_bit(BIO_UPTODATE, &bio->bi_flags)) 1820 if (!bio->bi_error)
1932 set_bit(R10BIO_Uptodate, &r10_bio->state); 1821 set_bit(R10BIO_Uptodate, &r10_bio->state);
1933 else 1822 else
1934 /* The write handler will notice the lack of 1823 /* The write handler will notice the lack of
@@ -1977,9 +1866,8 @@ static void end_sync_request(struct r10bio *r10_bio)
1977 } 1866 }
1978} 1867}
1979 1868
1980static void end_sync_write(struct bio *bio, int error) 1869static void end_sync_write(struct bio *bio)
1981{ 1870{
1982 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1983 struct r10bio *r10_bio = bio->bi_private; 1871 struct r10bio *r10_bio = bio->bi_private;
1984 struct mddev *mddev = r10_bio->mddev; 1872 struct mddev *mddev = r10_bio->mddev;
1985 struct r10conf *conf = mddev->private; 1873 struct r10conf *conf = mddev->private;
@@ -1996,7 +1884,7 @@ static void end_sync_write(struct bio *bio, int error)
1996 else 1884 else
1997 rdev = conf->mirrors[d].rdev; 1885 rdev = conf->mirrors[d].rdev;
1998 1886
1999 if (!uptodate) { 1887 if (bio->bi_error) {
2000 if (repl) 1888 if (repl)
2001 md_error(mddev, rdev); 1889 md_error(mddev, rdev);
2002 else { 1890 else {
@@ -2044,7 +1932,7 @@ static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2044 1932
2045 /* find the first device with a block */ 1933 /* find the first device with a block */
2046 for (i=0; i<conf->copies; i++) 1934 for (i=0; i<conf->copies; i++)
2047 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) 1935 if (!r10_bio->devs[i].bio->bi_error)
2048 break; 1936 break;
2049 1937
2050 if (i == conf->copies) 1938 if (i == conf->copies)
@@ -2064,7 +1952,7 @@ static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
2064 continue; 1952 continue;
2065 if (i == first) 1953 if (i == first)
2066 continue; 1954 continue;
2067 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) { 1955 if (!r10_bio->devs[i].bio->bi_error) {
2068 /* We know that the bi_io_vec layout is the same for 1956 /* We know that the bi_io_vec layout is the same for
2069 * both 'first' and 'i', so we just compare them. 1957 * both 'first' and 'i', so we just compare them.
2070 * All vec entries are PAGE_SIZE; 1958 * All vec entries are PAGE_SIZE;
@@ -2394,7 +2282,6 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10
2394 d = r10_bio->devs[sl].devnum; 2282 d = r10_bio->devs[sl].devnum;
2395 rdev = rcu_dereference(conf->mirrors[d].rdev); 2283 rdev = rcu_dereference(conf->mirrors[d].rdev);
2396 if (rdev && 2284 if (rdev &&
2397 !test_bit(Unmerged, &rdev->flags) &&
2398 test_bit(In_sync, &rdev->flags) && 2285 test_bit(In_sync, &rdev->flags) &&
2399 is_badblock(rdev, r10_bio->devs[sl].addr + sect, s, 2286 is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
2400 &first_bad, &bad_sectors) == 0) { 2287 &first_bad, &bad_sectors) == 0) {
@@ -2448,7 +2335,6 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10
2448 d = r10_bio->devs[sl].devnum; 2335 d = r10_bio->devs[sl].devnum;
2449 rdev = rcu_dereference(conf->mirrors[d].rdev); 2336 rdev = rcu_dereference(conf->mirrors[d].rdev);
2450 if (!rdev || 2337 if (!rdev ||
2451 test_bit(Unmerged, &rdev->flags) ||
2452 !test_bit(In_sync, &rdev->flags)) 2338 !test_bit(In_sync, &rdev->flags))
2453 continue; 2339 continue;
2454 2340
@@ -2706,8 +2592,7 @@ static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2706 rdev = conf->mirrors[dev].rdev; 2592 rdev = conf->mirrors[dev].rdev;
2707 if (r10_bio->devs[m].bio == NULL) 2593 if (r10_bio->devs[m].bio == NULL)
2708 continue; 2594 continue;
2709 if (test_bit(BIO_UPTODATE, 2595 if (!r10_bio->devs[m].bio->bi_error) {
2710 &r10_bio->devs[m].bio->bi_flags)) {
2711 rdev_clear_badblocks( 2596 rdev_clear_badblocks(
2712 rdev, 2597 rdev,
2713 r10_bio->devs[m].addr, 2598 r10_bio->devs[m].addr,
@@ -2722,8 +2607,8 @@ static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2722 rdev = conf->mirrors[dev].replacement; 2607 rdev = conf->mirrors[dev].replacement;
2723 if (r10_bio->devs[m].repl_bio == NULL) 2608 if (r10_bio->devs[m].repl_bio == NULL)
2724 continue; 2609 continue;
2725 if (test_bit(BIO_UPTODATE, 2610
2726 &r10_bio->devs[m].repl_bio->bi_flags)) { 2611 if (!r10_bio->devs[m].repl_bio->bi_error) {
2727 rdev_clear_badblocks( 2612 rdev_clear_badblocks(
2728 rdev, 2613 rdev,
2729 r10_bio->devs[m].addr, 2614 r10_bio->devs[m].addr,
@@ -2748,8 +2633,7 @@ static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2748 r10_bio->devs[m].addr, 2633 r10_bio->devs[m].addr,
2749 r10_bio->sectors, 0); 2634 r10_bio->sectors, 0);
2750 rdev_dec_pending(rdev, conf->mddev); 2635 rdev_dec_pending(rdev, conf->mddev);
2751 } else if (bio != NULL && 2636 } else if (bio != NULL && bio->bi_error) {
2752 !test_bit(BIO_UPTODATE, &bio->bi_flags)) {
2753 if (!narrow_write_error(r10_bio, m)) { 2637 if (!narrow_write_error(r10_bio, m)) {
2754 md_error(conf->mddev, rdev); 2638 md_error(conf->mddev, rdev);
2755 set_bit(R10BIO_Degraded, 2639 set_bit(R10BIO_Degraded,
@@ -3263,7 +3147,7 @@ static sector_t sync_request(struct mddev *mddev, sector_t sector_nr,
3263 3147
3264 bio = r10_bio->devs[i].bio; 3148 bio = r10_bio->devs[i].bio;
3265 bio_reset(bio); 3149 bio_reset(bio);
3266 clear_bit(BIO_UPTODATE, &bio->bi_flags); 3150 bio->bi_error = -EIO;
3267 if (conf->mirrors[d].rdev == NULL || 3151 if (conf->mirrors[d].rdev == NULL ||
3268 test_bit(Faulty, &conf->mirrors[d].rdev->flags)) 3152 test_bit(Faulty, &conf->mirrors[d].rdev->flags))
3269 continue; 3153 continue;
@@ -3300,7 +3184,7 @@ static sector_t sync_request(struct mddev *mddev, sector_t sector_nr,
3300 /* Need to set up for writing to the replacement */ 3184 /* Need to set up for writing to the replacement */
3301 bio = r10_bio->devs[i].repl_bio; 3185 bio = r10_bio->devs[i].repl_bio;
3302 bio_reset(bio); 3186 bio_reset(bio);
3303 clear_bit(BIO_UPTODATE, &bio->bi_flags); 3187 bio->bi_error = -EIO;
3304 3188
3305 sector = r10_bio->devs[i].addr; 3189 sector = r10_bio->devs[i].addr;
3306 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 3190 atomic_inc(&conf->mirrors[d].rdev->nr_pending);
@@ -3357,7 +3241,7 @@ static sector_t sync_request(struct mddev *mddev, sector_t sector_nr,
3357 /* remove last page from this bio */ 3241 /* remove last page from this bio */
3358 bio2->bi_vcnt--; 3242 bio2->bi_vcnt--;
3359 bio2->bi_iter.bi_size -= len; 3243 bio2->bi_iter.bi_size -= len;
3360 __clear_bit(BIO_SEG_VALID, &bio2->bi_flags); 3244 bio_clear_flag(bio2, BIO_SEG_VALID);
3361 } 3245 }
3362 goto bio_full; 3246 goto bio_full;
3363 } 3247 }
@@ -3377,7 +3261,7 @@ static sector_t sync_request(struct mddev *mddev, sector_t sector_nr,
3377 3261
3378 if (bio->bi_end_io == end_sync_read) { 3262 if (bio->bi_end_io == end_sync_read) {
3379 md_sync_acct(bio->bi_bdev, nr_sectors); 3263 md_sync_acct(bio->bi_bdev, nr_sectors);
3380 set_bit(BIO_UPTODATE, &bio->bi_flags); 3264 bio->bi_error = 0;
3381 generic_make_request(bio); 3265 generic_make_request(bio);
3382 } 3266 }
3383 } 3267 }
@@ -3643,8 +3527,6 @@ static int run(struct mddev *mddev)
3643 disk->rdev = rdev; 3527 disk->rdev = rdev;
3644 } 3528 }
3645 q = bdev_get_queue(rdev->bdev); 3529 q = bdev_get_queue(rdev->bdev);
3646 if (q->merge_bvec_fn)
3647 mddev->merge_check_needed = 1;
3648 diff = (rdev->new_data_offset - rdev->data_offset); 3530 diff = (rdev->new_data_offset - rdev->data_offset);
3649 if (!mddev->reshape_backwards) 3531 if (!mddev->reshape_backwards)
3650 diff = -diff; 3532 diff = -diff;
@@ -4382,7 +4264,7 @@ read_more:
4382 read_bio->bi_end_io = end_sync_read; 4264 read_bio->bi_end_io = end_sync_read;
4383 read_bio->bi_rw = READ; 4265 read_bio->bi_rw = READ;
4384 read_bio->bi_flags &= (~0UL << BIO_RESET_BITS); 4266 read_bio->bi_flags &= (~0UL << BIO_RESET_BITS);
4385 __set_bit(BIO_UPTODATE, &read_bio->bi_flags); 4267 read_bio->bi_error = 0;
4386 read_bio->bi_vcnt = 0; 4268 read_bio->bi_vcnt = 0;
4387 read_bio->bi_iter.bi_size = 0; 4269 read_bio->bi_iter.bi_size = 0;
4388 r10_bio->master_bio = read_bio; 4270 r10_bio->master_bio = read_bio;
@@ -4439,7 +4321,7 @@ read_more:
4439 /* Remove last page from this bio */ 4321 /* Remove last page from this bio */
4440 bio2->bi_vcnt--; 4322 bio2->bi_vcnt--;
4441 bio2->bi_iter.bi_size -= len; 4323 bio2->bi_iter.bi_size -= len;
4442 __clear_bit(BIO_SEG_VALID, &bio2->bi_flags); 4324 bio_clear_flag(bio2, BIO_SEG_VALID);
4443 } 4325 }
4444 goto bio_full; 4326 goto bio_full;
4445 } 4327 }
@@ -4604,9 +4486,8 @@ static int handle_reshape_read_error(struct mddev *mddev,
4604 return 0; 4486 return 0;
4605} 4487}
4606 4488
4607static void end_reshape_write(struct bio *bio, int error) 4489static void end_reshape_write(struct bio *bio)
4608{ 4490{
4609 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
4610 struct r10bio *r10_bio = bio->bi_private; 4491 struct r10bio *r10_bio = bio->bi_private;
4611 struct mddev *mddev = r10_bio->mddev; 4492 struct mddev *mddev = r10_bio->mddev;
4612 struct r10conf *conf = mddev->private; 4493 struct r10conf *conf = mddev->private;
@@ -4623,7 +4504,7 @@ static void end_reshape_write(struct bio *bio, int error)
4623 rdev = conf->mirrors[d].rdev; 4504 rdev = conf->mirrors[d].rdev;
4624 } 4505 }
4625 4506
4626 if (!uptodate) { 4507 if (bio->bi_error) {
4627 /* FIXME should record badblock */ 4508 /* FIXME should record badblock */
4628 md_error(mddev, rdev); 4509 md_error(mddev, rdev);
4629 } 4510 }
@@ -4700,7 +4581,6 @@ static struct md_personality raid10_personality =
4700 .start_reshape = raid10_start_reshape, 4581 .start_reshape = raid10_start_reshape,
4701 .finish_reshape = raid10_finish_reshape, 4582 .finish_reshape = raid10_finish_reshape,
4702 .congested = raid10_congested, 4583 .congested = raid10_congested,
4703 .mergeable_bvec = raid10_mergeable_bvec,
4704}; 4584};
4705 4585
4706static int __init raid_init(void) 4586static int __init raid_init(void)
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index f757023fc458..b29e89cb815b 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -233,7 +233,7 @@ static void return_io(struct bio *return_bi)
233 bi->bi_iter.bi_size = 0; 233 bi->bi_iter.bi_size = 0;
234 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), 234 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
235 bi, 0); 235 bi, 0);
236 bio_endio(bi, 0); 236 bio_endio(bi);
237 bi = return_bi; 237 bi = return_bi;
238 } 238 }
239} 239}
@@ -887,9 +887,9 @@ static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
887} 887}
888 888
889static void 889static void
890raid5_end_read_request(struct bio *bi, int error); 890raid5_end_read_request(struct bio *bi);
891static void 891static void
892raid5_end_write_request(struct bio *bi, int error); 892raid5_end_write_request(struct bio *bi);
893 893
894static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) 894static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
895{ 895{
@@ -2282,12 +2282,11 @@ static void shrink_stripes(struct r5conf *conf)
2282 conf->slab_cache = NULL; 2282 conf->slab_cache = NULL;
2283} 2283}
2284 2284
2285static void raid5_end_read_request(struct bio * bi, int error) 2285static void raid5_end_read_request(struct bio * bi)
2286{ 2286{
2287 struct stripe_head *sh = bi->bi_private; 2287 struct stripe_head *sh = bi->bi_private;
2288 struct r5conf *conf = sh->raid_conf; 2288 struct r5conf *conf = sh->raid_conf;
2289 int disks = sh->disks, i; 2289 int disks = sh->disks, i;
2290 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
2291 char b[BDEVNAME_SIZE]; 2290 char b[BDEVNAME_SIZE];
2292 struct md_rdev *rdev = NULL; 2291 struct md_rdev *rdev = NULL;
2293 sector_t s; 2292 sector_t s;
@@ -2296,9 +2295,9 @@ static void raid5_end_read_request(struct bio * bi, int error)
2296 if (bi == &sh->dev[i].req) 2295 if (bi == &sh->dev[i].req)
2297 break; 2296 break;
2298 2297
2299 pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", 2298 pr_debug("end_read_request %llu/%d, count: %d, error %d.\n",
2300 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 2299 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
2301 uptodate); 2300 bi->bi_error);
2302 if (i == disks) { 2301 if (i == disks) {
2303 BUG(); 2302 BUG();
2304 return; 2303 return;
@@ -2317,7 +2316,7 @@ static void raid5_end_read_request(struct bio * bi, int error)
2317 s = sh->sector + rdev->new_data_offset; 2316 s = sh->sector + rdev->new_data_offset;
2318 else 2317 else
2319 s = sh->sector + rdev->data_offset; 2318 s = sh->sector + rdev->data_offset;
2320 if (uptodate) { 2319 if (!bi->bi_error) {
2321 set_bit(R5_UPTODATE, &sh->dev[i].flags); 2320 set_bit(R5_UPTODATE, &sh->dev[i].flags);
2322 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 2321 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2323 /* Note that this cannot happen on a 2322 /* Note that this cannot happen on a
@@ -2405,13 +2404,12 @@ static void raid5_end_read_request(struct bio * bi, int error)
2405 release_stripe(sh); 2404 release_stripe(sh);
2406} 2405}
2407 2406
2408static void raid5_end_write_request(struct bio *bi, int error) 2407static void raid5_end_write_request(struct bio *bi)
2409{ 2408{
2410 struct stripe_head *sh = bi->bi_private; 2409 struct stripe_head *sh = bi->bi_private;
2411 struct r5conf *conf = sh->raid_conf; 2410 struct r5conf *conf = sh->raid_conf;
2412 int disks = sh->disks, i; 2411 int disks = sh->disks, i;
2413 struct md_rdev *uninitialized_var(rdev); 2412 struct md_rdev *uninitialized_var(rdev);
2414 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
2415 sector_t first_bad; 2413 sector_t first_bad;
2416 int bad_sectors; 2414 int bad_sectors;
2417 int replacement = 0; 2415 int replacement = 0;
@@ -2434,23 +2432,23 @@ static void raid5_end_write_request(struct bio *bi, int error)
2434 break; 2432 break;
2435 } 2433 }
2436 } 2434 }
2437 pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", 2435 pr_debug("end_write_request %llu/%d, count %d, error: %d.\n",
2438 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 2436 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
2439 uptodate); 2437 bi->bi_error);
2440 if (i == disks) { 2438 if (i == disks) {
2441 BUG(); 2439 BUG();
2442 return; 2440 return;
2443 } 2441 }
2444 2442
2445 if (replacement) { 2443 if (replacement) {
2446 if (!uptodate) 2444 if (bi->bi_error)
2447 md_error(conf->mddev, rdev); 2445 md_error(conf->mddev, rdev);
2448 else if (is_badblock(rdev, sh->sector, 2446 else if (is_badblock(rdev, sh->sector,
2449 STRIPE_SECTORS, 2447 STRIPE_SECTORS,
2450 &first_bad, &bad_sectors)) 2448 &first_bad, &bad_sectors))
2451 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); 2449 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
2452 } else { 2450 } else {
2453 if (!uptodate) { 2451 if (bi->bi_error) {
2454 set_bit(STRIPE_DEGRADED, &sh->state); 2452 set_bit(STRIPE_DEGRADED, &sh->state);
2455 set_bit(WriteErrorSeen, &rdev->flags); 2453 set_bit(WriteErrorSeen, &rdev->flags);
2456 set_bit(R5_WriteError, &sh->dev[i].flags); 2454 set_bit(R5_WriteError, &sh->dev[i].flags);
@@ -2471,7 +2469,7 @@ static void raid5_end_write_request(struct bio *bi, int error)
2471 } 2469 }
2472 rdev_dec_pending(rdev, conf->mddev); 2470 rdev_dec_pending(rdev, conf->mddev);
2473 2471
2474 if (sh->batch_head && !uptodate && !replacement) 2472 if (sh->batch_head && bi->bi_error && !replacement)
2475 set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state); 2473 set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);
2476 2474
2477 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) 2475 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
@@ -3112,7 +3110,8 @@ handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
3112 while (bi && bi->bi_iter.bi_sector < 3110 while (bi && bi->bi_iter.bi_sector <
3113 sh->dev[i].sector + STRIPE_SECTORS) { 3111 sh->dev[i].sector + STRIPE_SECTORS) {
3114 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); 3112 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
3115 clear_bit(BIO_UPTODATE, &bi->bi_flags); 3113
3114 bi->bi_error = -EIO;
3116 if (!raid5_dec_bi_active_stripes(bi)) { 3115 if (!raid5_dec_bi_active_stripes(bi)) {
3117 md_write_end(conf->mddev); 3116 md_write_end(conf->mddev);
3118 bi->bi_next = *return_bi; 3117 bi->bi_next = *return_bi;
@@ -3136,7 +3135,8 @@ handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
3136 while (bi && bi->bi_iter.bi_sector < 3135 while (bi && bi->bi_iter.bi_sector <
3137 sh->dev[i].sector + STRIPE_SECTORS) { 3136 sh->dev[i].sector + STRIPE_SECTORS) {
3138 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); 3137 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
3139 clear_bit(BIO_UPTODATE, &bi->bi_flags); 3138
3139 bi->bi_error = -EIO;
3140 if (!raid5_dec_bi_active_stripes(bi)) { 3140 if (!raid5_dec_bi_active_stripes(bi)) {
3141 md_write_end(conf->mddev); 3141 md_write_end(conf->mddev);
3142 bi->bi_next = *return_bi; 3142 bi->bi_next = *return_bi;
@@ -3161,7 +3161,8 @@ handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
3161 sh->dev[i].sector + STRIPE_SECTORS) { 3161 sh->dev[i].sector + STRIPE_SECTORS) {
3162 struct bio *nextbi = 3162 struct bio *nextbi =
3163 r5_next_bio(bi, sh->dev[i].sector); 3163 r5_next_bio(bi, sh->dev[i].sector);
3164 clear_bit(BIO_UPTODATE, &bi->bi_flags); 3164
3165 bi->bi_error = -EIO;
3165 if (!raid5_dec_bi_active_stripes(bi)) { 3166 if (!raid5_dec_bi_active_stripes(bi)) {
3166 bi->bi_next = *return_bi; 3167 bi->bi_next = *return_bi;
3167 *return_bi = bi; 3168 *return_bi = bi;
@@ -4669,35 +4670,6 @@ static int raid5_congested(struct mddev *mddev, int bits)
4669 return 0; 4670 return 0;
4670} 4671}
4671 4672
4672/* We want read requests to align with chunks where possible,
4673 * but write requests don't need to.
4674 */
4675static int raid5_mergeable_bvec(struct mddev *mddev,
4676 struct bvec_merge_data *bvm,
4677 struct bio_vec *biovec)
4678{
4679 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
4680 int max;
4681 unsigned int chunk_sectors = mddev->chunk_sectors;
4682 unsigned int bio_sectors = bvm->bi_size >> 9;
4683
4684 /*
4685 * always allow writes to be mergeable, read as well if array
4686 * is degraded as we'll go through stripe cache anyway.
4687 */
4688 if ((bvm->bi_rw & 1) == WRITE || mddev->degraded)
4689 return biovec->bv_len;
4690
4691 if (mddev->new_chunk_sectors < mddev->chunk_sectors)
4692 chunk_sectors = mddev->new_chunk_sectors;
4693 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
4694 if (max < 0) max = 0;
4695 if (max <= biovec->bv_len && bio_sectors == 0)
4696 return biovec->bv_len;
4697 else
4698 return max;
4699}
4700
4701static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) 4673static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
4702{ 4674{
4703 sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev); 4675 sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev);
@@ -4756,13 +4728,13 @@ static struct bio *remove_bio_from_retry(struct r5conf *conf)
4756 * first). 4728 * first).
4757 * If the read failed.. 4729 * If the read failed..
4758 */ 4730 */
4759static void raid5_align_endio(struct bio *bi, int error) 4731static void raid5_align_endio(struct bio *bi)
4760{ 4732{
4761 struct bio* raid_bi = bi->bi_private; 4733 struct bio* raid_bi = bi->bi_private;
4762 struct mddev *mddev; 4734 struct mddev *mddev;
4763 struct r5conf *conf; 4735 struct r5conf *conf;
4764 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
4765 struct md_rdev *rdev; 4736 struct md_rdev *rdev;
4737 int error = bi->bi_error;
4766 4738
4767 bio_put(bi); 4739 bio_put(bi);
4768 4740
@@ -4773,10 +4745,10 @@ static void raid5_align_endio(struct bio *bi, int error)
4773 4745
4774 rdev_dec_pending(rdev, conf->mddev); 4746 rdev_dec_pending(rdev, conf->mddev);
4775 4747
4776 if (!error && uptodate) { 4748 if (!error) {
4777 trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev), 4749 trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev),
4778 raid_bi, 0); 4750 raid_bi, 0);
4779 bio_endio(raid_bi, 0); 4751 bio_endio(raid_bi);
4780 if (atomic_dec_and_test(&conf->active_aligned_reads)) 4752 if (atomic_dec_and_test(&conf->active_aligned_reads))
4781 wake_up(&conf->wait_for_quiescent); 4753 wake_up(&conf->wait_for_quiescent);
4782 return; 4754 return;
@@ -4787,26 +4759,7 @@ static void raid5_align_endio(struct bio *bi, int error)
4787 add_bio_to_retry(raid_bi, conf); 4759 add_bio_to_retry(raid_bi, conf);
4788} 4760}
4789 4761
4790static int bio_fits_rdev(struct bio *bi) 4762static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)
4791{
4792 struct request_queue *q = bdev_get_queue(bi->bi_bdev);
4793
4794 if (bio_sectors(bi) > queue_max_sectors(q))
4795 return 0;
4796 blk_recount_segments(q, bi);
4797 if (bi->bi_phys_segments > queue_max_segments(q))
4798 return 0;
4799
4800 if (q->merge_bvec_fn)
4801 /* it's too hard to apply the merge_bvec_fn at this stage,
4802 * just just give up
4803 */
4804 return 0;
4805
4806 return 1;
4807}
4808
4809static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
4810{ 4763{
4811 struct r5conf *conf = mddev->private; 4764 struct r5conf *conf = mddev->private;
4812 int dd_idx; 4765 int dd_idx;
@@ -4815,7 +4768,7 @@ static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
4815 sector_t end_sector; 4768 sector_t end_sector;
4816 4769
4817 if (!in_chunk_boundary(mddev, raid_bio)) { 4770 if (!in_chunk_boundary(mddev, raid_bio)) {
4818 pr_debug("chunk_aligned_read : non aligned\n"); 4771 pr_debug("%s: non aligned\n", __func__);
4819 return 0; 4772 return 0;
4820 } 4773 }
4821 /* 4774 /*
@@ -4857,13 +4810,11 @@ static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
4857 rcu_read_unlock(); 4810 rcu_read_unlock();
4858 raid_bio->bi_next = (void*)rdev; 4811 raid_bio->bi_next = (void*)rdev;
4859 align_bi->bi_bdev = rdev->bdev; 4812 align_bi->bi_bdev = rdev->bdev;
4860 __clear_bit(BIO_SEG_VALID, &align_bi->bi_flags); 4813 bio_clear_flag(align_bi, BIO_SEG_VALID);
4861 4814
4862 if (!bio_fits_rdev(align_bi) || 4815 if (is_badblock(rdev, align_bi->bi_iter.bi_sector,
4863 is_badblock(rdev, align_bi->bi_iter.bi_sector,
4864 bio_sectors(align_bi), 4816 bio_sectors(align_bi),
4865 &first_bad, &bad_sectors)) { 4817 &first_bad, &bad_sectors)) {
4866 /* too big in some way, or has a known bad block */
4867 bio_put(align_bi); 4818 bio_put(align_bi);
4868 rdev_dec_pending(rdev, mddev); 4819 rdev_dec_pending(rdev, mddev);
4869 return 0; 4820 return 0;
@@ -4892,6 +4843,31 @@ static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
4892 } 4843 }
4893} 4844}
4894 4845
4846static struct bio *chunk_aligned_read(struct mddev *mddev, struct bio *raid_bio)
4847{
4848 struct bio *split;
4849
4850 do {
4851 sector_t sector = raid_bio->bi_iter.bi_sector;
4852 unsigned chunk_sects = mddev->chunk_sectors;
4853 unsigned sectors = chunk_sects - (sector & (chunk_sects-1));
4854
4855 if (sectors < bio_sectors(raid_bio)) {
4856 split = bio_split(raid_bio, sectors, GFP_NOIO, fs_bio_set);
4857 bio_chain(split, raid_bio);
4858 } else
4859 split = raid_bio;
4860
4861 if (!raid5_read_one_chunk(mddev, split)) {
4862 if (split != raid_bio)
4863 generic_make_request(raid_bio);
4864 return split;
4865 }
4866 } while (split != raid_bio);
4867
4868 return NULL;
4869}
4870
4895/* __get_priority_stripe - get the next stripe to process 4871/* __get_priority_stripe - get the next stripe to process
4896 * 4872 *
4897 * Full stripe writes are allowed to pass preread active stripes up until 4873 * Full stripe writes are allowed to pass preread active stripes up until
@@ -5140,7 +5116,7 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi)
5140 remaining = raid5_dec_bi_active_stripes(bi); 5116 remaining = raid5_dec_bi_active_stripes(bi);
5141 if (remaining == 0) { 5117 if (remaining == 0) {
5142 md_write_end(mddev); 5118 md_write_end(mddev);
5143 bio_endio(bi, 0); 5119 bio_endio(bi);
5144 } 5120 }
5145} 5121}
5146 5122
@@ -5169,9 +5145,11 @@ static void make_request(struct mddev *mddev, struct bio * bi)
5169 * data on failed drives. 5145 * data on failed drives.
5170 */ 5146 */
5171 if (rw == READ && mddev->degraded == 0 && 5147 if (rw == READ && mddev->degraded == 0 &&
5172 mddev->reshape_position == MaxSector && 5148 mddev->reshape_position == MaxSector) {
5173 chunk_aligned_read(mddev,bi)) 5149 bi = chunk_aligned_read(mddev, bi);
5174 return; 5150 if (!bi)
5151 return;
5152 }
5175 5153
5176 if (unlikely(bi->bi_rw & REQ_DISCARD)) { 5154 if (unlikely(bi->bi_rw & REQ_DISCARD)) {
5177 make_discard_request(mddev, bi); 5155 make_discard_request(mddev, bi);
@@ -5304,7 +5282,7 @@ static void make_request(struct mddev *mddev, struct bio * bi)
5304 release_stripe_plug(mddev, sh); 5282 release_stripe_plug(mddev, sh);
5305 } else { 5283 } else {
5306 /* cannot get stripe for read-ahead, just give-up */ 5284 /* cannot get stripe for read-ahead, just give-up */
5307 clear_bit(BIO_UPTODATE, &bi->bi_flags); 5285 bi->bi_error = -EIO;
5308 break; 5286 break;
5309 } 5287 }
5310 } 5288 }
@@ -5318,7 +5296,7 @@ static void make_request(struct mddev *mddev, struct bio * bi)
5318 5296
5319 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), 5297 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
5320 bi, 0); 5298 bi, 0);
5321 bio_endio(bi, 0); 5299 bio_endio(bi);
5322 } 5300 }
5323} 5301}
5324 5302
@@ -5714,7 +5692,7 @@ static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
5714 if (remaining == 0) { 5692 if (remaining == 0) {
5715 trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev), 5693 trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev),
5716 raid_bio, 0); 5694 raid_bio, 0);
5717 bio_endio(raid_bio, 0); 5695 bio_endio(raid_bio);
5718 } 5696 }
5719 if (atomic_dec_and_test(&conf->active_aligned_reads)) 5697 if (atomic_dec_and_test(&conf->active_aligned_reads))
5720 wake_up(&conf->wait_for_quiescent); 5698 wake_up(&conf->wait_for_quiescent);
@@ -7779,7 +7757,6 @@ static struct md_personality raid6_personality =
7779 .quiesce = raid5_quiesce, 7757 .quiesce = raid5_quiesce,
7780 .takeover = raid6_takeover, 7758 .takeover = raid6_takeover,
7781 .congested = raid5_congested, 7759 .congested = raid5_congested,
7782 .mergeable_bvec = raid5_mergeable_bvec,
7783}; 7760};
7784static struct md_personality raid5_personality = 7761static struct md_personality raid5_personality =
7785{ 7762{
@@ -7803,7 +7780,6 @@ static struct md_personality raid5_personality =
7803 .quiesce = raid5_quiesce, 7780 .quiesce = raid5_quiesce,
7804 .takeover = raid5_takeover, 7781 .takeover = raid5_takeover,
7805 .congested = raid5_congested, 7782 .congested = raid5_congested,
7806 .mergeable_bvec = raid5_mergeable_bvec,
7807}; 7783};
7808 7784
7809static struct md_personality raid4_personality = 7785static struct md_personality raid4_personality =
@@ -7828,7 +7804,6 @@ static struct md_personality raid4_personality =
7828 .quiesce = raid5_quiesce, 7804 .quiesce = raid5_quiesce,
7829 .takeover = raid4_takeover, 7805 .takeover = raid4_takeover,
7830 .congested = raid5_congested, 7806 .congested = raid5_congested,
7831 .mergeable_bvec = raid5_mergeable_bvec,
7832}; 7807};
7833 7808
7834static int __init raid5_init(void) 7809static int __init raid5_init(void)
diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c
index b5a2b145d89f..5daf302835b1 100644
--- a/drivers/mmc/card/queue.c
+++ b/drivers/mmc/card/queue.c
@@ -165,7 +165,7 @@ static void mmc_queue_setup_discard(struct request_queue *q,
165 return; 165 return;
166 166
167 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); 167 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
168 q->limits.max_discard_sectors = max_discard; 168 blk_queue_max_discard_sectors(q, max_discard);
169 if (card->erased_byte == 0 && !mmc_can_discard(card)) 169 if (card->erased_byte == 0 && !mmc_can_discard(card))
170 q->limits.discard_zeroes_data = 1; 170 q->limits.discard_zeroes_data = 1;
171 q->limits.discard_granularity = card->pref_erase << 9; 171 q->limits.discard_granularity = card->pref_erase << 9;
diff --git a/drivers/mtd/mtd_blkdevs.c b/drivers/mtd/mtd_blkdevs.c
index 88304751eb8a..44dc965a2f7c 100644
--- a/drivers/mtd/mtd_blkdevs.c
+++ b/drivers/mtd/mtd_blkdevs.c
@@ -419,7 +419,7 @@ int add_mtd_blktrans_dev(struct mtd_blktrans_dev *new)
419 419
420 if (tr->discard) { 420 if (tr->discard) {
421 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, new->rq); 421 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, new->rq);
422 new->rq->limits.max_discard_sectors = UINT_MAX; 422 blk_queue_max_discard_sectors(new->rq, UINT_MAX);
423 } 423 }
424 424
425 gd->queue = new->rq; 425 gd->queue = new->rq;
diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c
index 4f97b248c236..0df77cb07df6 100644
--- a/drivers/nvdimm/blk.c
+++ b/drivers/nvdimm/blk.c
@@ -180,7 +180,7 @@ static void nd_blk_make_request(struct request_queue *q, struct bio *bio)
180 * another kernel subsystem, and we just pass it through. 180 * another kernel subsystem, and we just pass it through.
181 */ 181 */
182 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { 182 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
183 err = -EIO; 183 bio->bi_error = -EIO;
184 goto out; 184 goto out;
185 } 185 }
186 186
@@ -199,6 +199,7 @@ static void nd_blk_make_request(struct request_queue *q, struct bio *bio)
199 "io error in %s sector %lld, len %d,\n", 199 "io error in %s sector %lld, len %d,\n",
200 (rw == READ) ? "READ" : "WRITE", 200 (rw == READ) ? "READ" : "WRITE",
201 (unsigned long long) iter.bi_sector, len); 201 (unsigned long long) iter.bi_sector, len);
202 bio->bi_error = err;
202 break; 203 break;
203 } 204 }
204 } 205 }
@@ -206,7 +207,7 @@ static void nd_blk_make_request(struct request_queue *q, struct bio *bio)
206 nd_iostat_end(bio, start); 207 nd_iostat_end(bio, start);
207 208
208 out: 209 out:
209 bio_endio(bio, err); 210 bio_endio(bio);
210} 211}
211 212
212static int nd_blk_rw_bytes(struct nd_namespace_common *ndns, 213static int nd_blk_rw_bytes(struct nd_namespace_common *ndns,
diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c
index 411c7b2bb37a..341202ed32b4 100644
--- a/drivers/nvdimm/btt.c
+++ b/drivers/nvdimm/btt.c
@@ -1189,7 +1189,7 @@ static void btt_make_request(struct request_queue *q, struct bio *bio)
1189 * another kernel subsystem, and we just pass it through. 1189 * another kernel subsystem, and we just pass it through.
1190 */ 1190 */
1191 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { 1191 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1192 err = -EIO; 1192 bio->bi_error = -EIO;
1193 goto out; 1193 goto out;
1194 } 1194 }
1195 1195
@@ -1211,6 +1211,7 @@ static void btt_make_request(struct request_queue *q, struct bio *bio)
1211 "io error in %s sector %lld, len %d,\n", 1211 "io error in %s sector %lld, len %d,\n",
1212 (rw == READ) ? "READ" : "WRITE", 1212 (rw == READ) ? "READ" : "WRITE",
1213 (unsigned long long) iter.bi_sector, len); 1213 (unsigned long long) iter.bi_sector, len);
1214 bio->bi_error = err;
1214 break; 1215 break;
1215 } 1216 }
1216 } 1217 }
@@ -1218,7 +1219,7 @@ static void btt_make_request(struct request_queue *q, struct bio *bio)
1218 nd_iostat_end(bio, start); 1219 nd_iostat_end(bio, start);
1219 1220
1220out: 1221out:
1221 bio_endio(bio, err); 1222 bio_endio(bio);
1222} 1223}
1223 1224
1224static int btt_rw_page(struct block_device *bdev, sector_t sector, 1225static int btt_rw_page(struct block_device *bdev, sector_t sector,
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index ade9eb917a4d..4c079d5cb539 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -77,7 +77,7 @@ static void pmem_make_request(struct request_queue *q, struct bio *bio)
77 if (bio_data_dir(bio)) 77 if (bio_data_dir(bio))
78 wmb_pmem(); 78 wmb_pmem();
79 79
80 bio_endio(bio, 0); 80 bio_endio(bio);
81} 81}
82 82
83static int pmem_rw_page(struct block_device *bdev, sector_t sector, 83static int pmem_rw_page(struct block_device *bdev, sector_t sector,
diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c
index dff3fcb69a78..2b744fbba68e 100644
--- a/drivers/s390/block/dcssblk.c
+++ b/drivers/s390/block/dcssblk.c
@@ -826,6 +826,8 @@ dcssblk_make_request(struct request_queue *q, struct bio *bio)
826 unsigned long source_addr; 826 unsigned long source_addr;
827 unsigned long bytes_done; 827 unsigned long bytes_done;
828 828
829 blk_queue_split(q, &bio, q->bio_split);
830
829 bytes_done = 0; 831 bytes_done = 0;
830 dev_info = bio->bi_bdev->bd_disk->private_data; 832 dev_info = bio->bi_bdev->bd_disk->private_data;
831 if (dev_info == NULL) 833 if (dev_info == NULL)
@@ -871,7 +873,7 @@ dcssblk_make_request(struct request_queue *q, struct bio *bio)
871 } 873 }
872 bytes_done += bvec.bv_len; 874 bytes_done += bvec.bv_len;
873 } 875 }
874 bio_endio(bio, 0); 876 bio_endio(bio);
875 return; 877 return;
876fail: 878fail:
877 bio_io_error(bio); 879 bio_io_error(bio);
diff --git a/drivers/s390/block/xpram.c b/drivers/s390/block/xpram.c
index 7d4e9397ac31..02871f1db562 100644
--- a/drivers/s390/block/xpram.c
+++ b/drivers/s390/block/xpram.c
@@ -190,6 +190,8 @@ static void xpram_make_request(struct request_queue *q, struct bio *bio)
190 unsigned long page_addr; 190 unsigned long page_addr;
191 unsigned long bytes; 191 unsigned long bytes;
192 192
193 blk_queue_split(q, &bio, q->bio_split);
194
193 if ((bio->bi_iter.bi_sector & 7) != 0 || 195 if ((bio->bi_iter.bi_sector & 7) != 0 ||
194 (bio->bi_iter.bi_size & 4095) != 0) 196 (bio->bi_iter.bi_size & 4095) != 0)
195 /* Request is not page-aligned. */ 197 /* Request is not page-aligned. */
@@ -220,8 +222,7 @@ static void xpram_make_request(struct request_queue *q, struct bio *bio)
220 index++; 222 index++;
221 } 223 }
222 } 224 }
223 set_bit(BIO_UPTODATE, &bio->bi_flags); 225 bio_endio(bio);
224 bio_endio(bio, 0);
225 return; 226 return;
226fail: 227fail:
227 bio_io_error(bio); 228 bio_io_error(bio);
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index a20da8c25b4f..3f370228bf31 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -647,7 +647,7 @@ static void sd_config_discard(struct scsi_disk *sdkp, unsigned int mode)
647 switch (mode) { 647 switch (mode) {
648 648
649 case SD_LBP_DISABLE: 649 case SD_LBP_DISABLE:
650 q->limits.max_discard_sectors = 0; 650 blk_queue_max_discard_sectors(q, 0);
651 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); 651 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q);
652 return; 652 return;
653 653
@@ -675,7 +675,7 @@ static void sd_config_discard(struct scsi_disk *sdkp, unsigned int mode)
675 break; 675 break;
676 } 676 }
677 677
678 q->limits.max_discard_sectors = max_blocks * (logical_block_size >> 9); 678 blk_queue_max_discard_sectors(q, max_blocks * (logical_block_size >> 9));
679 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); 679 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
680} 680}
681 681
diff --git a/drivers/staging/lustre/lustre/llite/lloop.c b/drivers/staging/lustre/lustre/llite/lloop.c
index 06f5e51ecd9e..5f0d80cc9718 100644
--- a/drivers/staging/lustre/lustre/llite/lloop.c
+++ b/drivers/staging/lustre/lustre/llite/lloop.c
@@ -340,6 +340,8 @@ static void loop_make_request(struct request_queue *q, struct bio *old_bio)
340 int rw = bio_rw(old_bio); 340 int rw = bio_rw(old_bio);
341 int inactive; 341 int inactive;
342 342
343 blk_queue_split(q, &old_bio, q->bio_split);
344
343 if (!lo) 345 if (!lo)
344 goto err; 346 goto err;
345 347
@@ -376,7 +378,8 @@ static inline void loop_handle_bio(struct lloop_device *lo, struct bio *bio)
376 while (bio) { 378 while (bio) {
377 struct bio *tmp = bio->bi_next; 379 struct bio *tmp = bio->bi_next;
378 bio->bi_next = NULL; 380 bio->bi_next = NULL;
379 bio_endio(bio, ret); 381 bio->bi_error = ret;
382 bio_endio(bio);
380 bio = tmp; 383 bio = tmp;
381 } 384 }
382} 385}
diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c
index 6d88d24e6cce..5a9982f5d5d6 100644
--- a/drivers/target/target_core_iblock.c
+++ b/drivers/target/target_core_iblock.c
@@ -306,20 +306,13 @@ static void iblock_complete_cmd(struct se_cmd *cmd)
306 kfree(ibr); 306 kfree(ibr);
307} 307}
308 308
309static void iblock_bio_done(struct bio *bio, int err) 309static void iblock_bio_done(struct bio *bio)
310{ 310{
311 struct se_cmd *cmd = bio->bi_private; 311 struct se_cmd *cmd = bio->bi_private;
312 struct iblock_req *ibr = cmd->priv; 312 struct iblock_req *ibr = cmd->priv;
313 313
314 /* 314 if (bio->bi_error) {
315 * Set -EIO if !BIO_UPTODATE and the passed is still err=0 315 pr_err("bio error: %p, err: %d\n", bio, bio->bi_error);
316 */
317 if (!test_bit(BIO_UPTODATE, &bio->bi_flags) && !err)
318 err = -EIO;
319
320 if (err != 0) {
321 pr_err("test_bit(BIO_UPTODATE) failed for bio: %p,"
322 " err: %d\n", bio, err);
323 /* 316 /*
324 * Bump the ib_bio_err_cnt and release bio. 317 * Bump the ib_bio_err_cnt and release bio.
325 */ 318 */
@@ -370,15 +363,15 @@ static void iblock_submit_bios(struct bio_list *list, int rw)
370 blk_finish_plug(&plug); 363 blk_finish_plug(&plug);
371} 364}
372 365
373static void iblock_end_io_flush(struct bio *bio, int err) 366static void iblock_end_io_flush(struct bio *bio)
374{ 367{
375 struct se_cmd *cmd = bio->bi_private; 368 struct se_cmd *cmd = bio->bi_private;
376 369
377 if (err) 370 if (bio->bi_error)
378 pr_err("IBLOCK: cache flush failed: %d\n", err); 371 pr_err("IBLOCK: cache flush failed: %d\n", bio->bi_error);
379 372
380 if (cmd) { 373 if (cmd) {
381 if (err) 374 if (bio->bi_error)
382 target_complete_cmd(cmd, SAM_STAT_CHECK_CONDITION); 375 target_complete_cmd(cmd, SAM_STAT_CHECK_CONDITION);
383 else 376 else
384 target_complete_cmd(cmd, SAM_STAT_GOOD); 377 target_complete_cmd(cmd, SAM_STAT_GOOD);
diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c
index 08e9084ee615..de18790eb21c 100644
--- a/drivers/target/target_core_pscsi.c
+++ b/drivers/target/target_core_pscsi.c
@@ -852,7 +852,7 @@ static ssize_t pscsi_show_configfs_dev_params(struct se_device *dev, char *b)
852 return bl; 852 return bl;
853} 853}
854 854
855static void pscsi_bi_endio(struct bio *bio, int error) 855static void pscsi_bi_endio(struct bio *bio)
856{ 856{
857 bio_put(bio); 857 bio_put(bio);
858} 858}
@@ -973,7 +973,7 @@ fail:
973 while (*hbio) { 973 while (*hbio) {
974 bio = *hbio; 974 bio = *hbio;
975 *hbio = (*hbio)->bi_next; 975 *hbio = (*hbio)->bi_next;
976 bio_endio(bio, 0); /* XXX: should be error */ 976 bio_endio(bio);
977 } 977 }
978 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 978 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
979} 979}
@@ -1061,7 +1061,7 @@ fail_free_bio:
1061 while (hbio) { 1061 while (hbio) {
1062 struct bio *bio = hbio; 1062 struct bio *bio = hbio;
1063 hbio = hbio->bi_next; 1063 hbio = hbio->bi_next;
1064 bio_endio(bio, 0); /* XXX: should be error */ 1064 bio_endio(bio);
1065 } 1065 }
1066 ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 1066 ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
1067fail: 1067fail:
diff --git a/fs/btrfs/check-integrity.c b/fs/btrfs/check-integrity.c
index ce7dec88f4b8..541fbfaed276 100644
--- a/fs/btrfs/check-integrity.c
+++ b/fs/btrfs/check-integrity.c
@@ -343,7 +343,7 @@ static int btrfsic_process_written_superblock(
343 struct btrfsic_state *state, 343 struct btrfsic_state *state,
344 struct btrfsic_block *const block, 344 struct btrfsic_block *const block,
345 struct btrfs_super_block *const super_hdr); 345 struct btrfs_super_block *const super_hdr);
346static void btrfsic_bio_end_io(struct bio *bp, int bio_error_status); 346static void btrfsic_bio_end_io(struct bio *bp);
347static void btrfsic_bh_end_io(struct buffer_head *bh, int uptodate); 347static void btrfsic_bh_end_io(struct buffer_head *bh, int uptodate);
348static int btrfsic_is_block_ref_by_superblock(const struct btrfsic_state *state, 348static int btrfsic_is_block_ref_by_superblock(const struct btrfsic_state *state,
349 const struct btrfsic_block *block, 349 const struct btrfsic_block *block,
@@ -2207,7 +2207,7 @@ continue_loop:
2207 goto again; 2207 goto again;
2208} 2208}
2209 2209
2210static void btrfsic_bio_end_io(struct bio *bp, int bio_error_status) 2210static void btrfsic_bio_end_io(struct bio *bp)
2211{ 2211{
2212 struct btrfsic_block *block = (struct btrfsic_block *)bp->bi_private; 2212 struct btrfsic_block *block = (struct btrfsic_block *)bp->bi_private;
2213 int iodone_w_error; 2213 int iodone_w_error;
@@ -2215,7 +2215,7 @@ static void btrfsic_bio_end_io(struct bio *bp, int bio_error_status)
2215 /* mutex is not held! This is not save if IO is not yet completed 2215 /* mutex is not held! This is not save if IO is not yet completed
2216 * on umount */ 2216 * on umount */
2217 iodone_w_error = 0; 2217 iodone_w_error = 0;
2218 if (bio_error_status) 2218 if (bp->bi_error)
2219 iodone_w_error = 1; 2219 iodone_w_error = 1;
2220 2220
2221 BUG_ON(NULL == block); 2221 BUG_ON(NULL == block);
@@ -2230,7 +2230,7 @@ static void btrfsic_bio_end_io(struct bio *bp, int bio_error_status)
2230 BTRFSIC_PRINT_MASK_END_IO_BIO_BH)) 2230 BTRFSIC_PRINT_MASK_END_IO_BIO_BH))
2231 printk(KERN_INFO 2231 printk(KERN_INFO
2232 "bio_end_io(err=%d) for %c @%llu (%s/%llu/%d)\n", 2232 "bio_end_io(err=%d) for %c @%llu (%s/%llu/%d)\n",
2233 bio_error_status, 2233 bp->bi_error,
2234 btrfsic_get_block_type(dev_state->state, block), 2234 btrfsic_get_block_type(dev_state->state, block),
2235 block->logical_bytenr, dev_state->name, 2235 block->logical_bytenr, dev_state->name,
2236 block->dev_bytenr, block->mirror_num); 2236 block->dev_bytenr, block->mirror_num);
@@ -2252,7 +2252,7 @@ static void btrfsic_bio_end_io(struct bio *bp, int bio_error_status)
2252 block = next_block; 2252 block = next_block;
2253 } while (NULL != block); 2253 } while (NULL != block);
2254 2254
2255 bp->bi_end_io(bp, bio_error_status); 2255 bp->bi_end_io(bp);
2256} 2256}
2257 2257
2258static void btrfsic_bh_end_io(struct buffer_head *bh, int uptodate) 2258static void btrfsic_bh_end_io(struct buffer_head *bh, int uptodate)
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
index ce62324c78e7..57ee8ca29b06 100644
--- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c
@@ -97,10 +97,7 @@ static inline int compressed_bio_size(struct btrfs_root *root,
97static struct bio *compressed_bio_alloc(struct block_device *bdev, 97static struct bio *compressed_bio_alloc(struct block_device *bdev,
98 u64 first_byte, gfp_t gfp_flags) 98 u64 first_byte, gfp_t gfp_flags)
99{ 99{
100 int nr_vecs; 100 return btrfs_bio_alloc(bdev, first_byte >> 9, BIO_MAX_PAGES, gfp_flags);
101
102 nr_vecs = bio_get_nr_vecs(bdev);
103 return btrfs_bio_alloc(bdev, first_byte >> 9, nr_vecs, gfp_flags);
104} 101}
105 102
106static int check_compressed_csum(struct inode *inode, 103static int check_compressed_csum(struct inode *inode,
@@ -152,7 +149,7 @@ fail:
152 * The compressed pages are freed here, and it must be run 149 * The compressed pages are freed here, and it must be run
153 * in process context 150 * in process context
154 */ 151 */
155static void end_compressed_bio_read(struct bio *bio, int err) 152static void end_compressed_bio_read(struct bio *bio)
156{ 153{
157 struct compressed_bio *cb = bio->bi_private; 154 struct compressed_bio *cb = bio->bi_private;
158 struct inode *inode; 155 struct inode *inode;
@@ -160,7 +157,7 @@ static void end_compressed_bio_read(struct bio *bio, int err)
160 unsigned long index; 157 unsigned long index;
161 int ret; 158 int ret;
162 159
163 if (err) 160 if (bio->bi_error)
164 cb->errors = 1; 161 cb->errors = 1;
165 162
166 /* if there are more bios still pending for this compressed 163 /* if there are more bios still pending for this compressed
@@ -210,7 +207,7 @@ csum_failed:
210 bio_for_each_segment_all(bvec, cb->orig_bio, i) 207 bio_for_each_segment_all(bvec, cb->orig_bio, i)
211 SetPageChecked(bvec->bv_page); 208 SetPageChecked(bvec->bv_page);
212 209
213 bio_endio(cb->orig_bio, 0); 210 bio_endio(cb->orig_bio);
214 } 211 }
215 212
216 /* finally free the cb struct */ 213 /* finally free the cb struct */
@@ -266,7 +263,7 @@ static noinline void end_compressed_writeback(struct inode *inode,
266 * This also calls the writeback end hooks for the file pages so that 263 * This also calls the writeback end hooks for the file pages so that
267 * metadata and checksums can be updated in the file. 264 * metadata and checksums can be updated in the file.
268 */ 265 */
269static void end_compressed_bio_write(struct bio *bio, int err) 266static void end_compressed_bio_write(struct bio *bio)
270{ 267{
271 struct extent_io_tree *tree; 268 struct extent_io_tree *tree;
272 struct compressed_bio *cb = bio->bi_private; 269 struct compressed_bio *cb = bio->bi_private;
@@ -274,7 +271,7 @@ static void end_compressed_bio_write(struct bio *bio, int err)
274 struct page *page; 271 struct page *page;
275 unsigned long index; 272 unsigned long index;
276 273
277 if (err) 274 if (bio->bi_error)
278 cb->errors = 1; 275 cb->errors = 1;
279 276
280 /* if there are more bios still pending for this compressed 277 /* if there are more bios still pending for this compressed
@@ -293,7 +290,7 @@ static void end_compressed_bio_write(struct bio *bio, int err)
293 cb->start, 290 cb->start,
294 cb->start + cb->len - 1, 291 cb->start + cb->len - 1,
295 NULL, 292 NULL,
296 err ? 0 : 1); 293 bio->bi_error ? 0 : 1);
297 cb->compressed_pages[0]->mapping = NULL; 294 cb->compressed_pages[0]->mapping = NULL;
298 295
299 end_compressed_writeback(inode, cb); 296 end_compressed_writeback(inode, cb);
@@ -697,8 +694,10 @@ int btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
697 694
698 ret = btrfs_map_bio(root, READ, comp_bio, 695 ret = btrfs_map_bio(root, READ, comp_bio,
699 mirror_num, 0); 696 mirror_num, 0);
700 if (ret) 697 if (ret) {
701 bio_endio(comp_bio, ret); 698 bio->bi_error = ret;
699 bio_endio(comp_bio);
700 }
702 701
703 bio_put(comp_bio); 702 bio_put(comp_bio);
704 703
@@ -724,8 +723,10 @@ int btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
724 } 723 }
725 724
726 ret = btrfs_map_bio(root, READ, comp_bio, mirror_num, 0); 725 ret = btrfs_map_bio(root, READ, comp_bio, mirror_num, 0);
727 if (ret) 726 if (ret) {
728 bio_endio(comp_bio, ret); 727 bio->bi_error = ret;
728 bio_endio(comp_bio);
729 }
729 730
730 bio_put(comp_bio); 731 bio_put(comp_bio);
731 return 0; 732 return 0;
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index f556c3732c2c..5e307bd0471a 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -703,7 +703,7 @@ static int btree_io_failed_hook(struct page *page, int failed_mirror)
703 return -EIO; /* we fixed nothing */ 703 return -EIO; /* we fixed nothing */
704} 704}
705 705
706static void end_workqueue_bio(struct bio *bio, int err) 706static void end_workqueue_bio(struct bio *bio)
707{ 707{
708 struct btrfs_end_io_wq *end_io_wq = bio->bi_private; 708 struct btrfs_end_io_wq *end_io_wq = bio->bi_private;
709 struct btrfs_fs_info *fs_info; 709 struct btrfs_fs_info *fs_info;
@@ -711,7 +711,7 @@ static void end_workqueue_bio(struct bio *bio, int err)
711 btrfs_work_func_t func; 711 btrfs_work_func_t func;
712 712
713 fs_info = end_io_wq->info; 713 fs_info = end_io_wq->info;
714 end_io_wq->error = err; 714 end_io_wq->error = bio->bi_error;
715 715
716 if (bio->bi_rw & REQ_WRITE) { 716 if (bio->bi_rw & REQ_WRITE) {
717 if (end_io_wq->metadata == BTRFS_WQ_ENDIO_METADATA) { 717 if (end_io_wq->metadata == BTRFS_WQ_ENDIO_METADATA) {
@@ -808,7 +808,8 @@ static void run_one_async_done(struct btrfs_work *work)
808 808
809 /* If an error occured we just want to clean up the bio and move on */ 809 /* If an error occured we just want to clean up the bio and move on */
810 if (async->error) { 810 if (async->error) {
811 bio_endio(async->bio, async->error); 811 async->bio->bi_error = async->error;
812 bio_endio(async->bio);
812 return; 813 return;
813 } 814 }
814 815
@@ -908,8 +909,10 @@ static int __btree_submit_bio_done(struct inode *inode, int rw, struct bio *bio,
908 * submission context. Just jump into btrfs_map_bio 909 * submission context. Just jump into btrfs_map_bio
909 */ 910 */
910 ret = btrfs_map_bio(BTRFS_I(inode)->root, rw, bio, mirror_num, 1); 911 ret = btrfs_map_bio(BTRFS_I(inode)->root, rw, bio, mirror_num, 1);
911 if (ret) 912 if (ret) {
912 bio_endio(bio, ret); 913 bio->bi_error = ret;
914 bio_endio(bio);
915 }
913 return ret; 916 return ret;
914} 917}
915 918
@@ -960,10 +963,13 @@ static int btree_submit_bio_hook(struct inode *inode, int rw, struct bio *bio,
960 __btree_submit_bio_done); 963 __btree_submit_bio_done);
961 } 964 }
962 965
963 if (ret) { 966 if (ret)
967 goto out_w_error;
968 return 0;
969
964out_w_error: 970out_w_error:
965 bio_endio(bio, ret); 971 bio->bi_error = ret;
966 } 972 bio_endio(bio);
967 return ret; 973 return ret;
968} 974}
969 975
@@ -1735,16 +1741,15 @@ static void end_workqueue_fn(struct btrfs_work *work)
1735{ 1741{
1736 struct bio *bio; 1742 struct bio *bio;
1737 struct btrfs_end_io_wq *end_io_wq; 1743 struct btrfs_end_io_wq *end_io_wq;
1738 int error;
1739 1744
1740 end_io_wq = container_of(work, struct btrfs_end_io_wq, work); 1745 end_io_wq = container_of(work, struct btrfs_end_io_wq, work);
1741 bio = end_io_wq->bio; 1746 bio = end_io_wq->bio;
1742 1747
1743 error = end_io_wq->error; 1748 bio->bi_error = end_io_wq->error;
1744 bio->bi_private = end_io_wq->private; 1749 bio->bi_private = end_io_wq->private;
1745 bio->bi_end_io = end_io_wq->end_io; 1750 bio->bi_end_io = end_io_wq->end_io;
1746 kmem_cache_free(btrfs_end_io_wq_cache, end_io_wq); 1751 kmem_cache_free(btrfs_end_io_wq_cache, end_io_wq);
1747 bio_endio(bio, error); 1752 bio_endio(bio);
1748} 1753}
1749 1754
1750static int cleaner_kthread(void *arg) 1755static int cleaner_kthread(void *arg)
@@ -3324,10 +3329,8 @@ static int write_dev_supers(struct btrfs_device *device,
3324 * endio for the write_dev_flush, this will wake anyone waiting 3329 * endio for the write_dev_flush, this will wake anyone waiting
3325 * for the barrier when it is done 3330 * for the barrier when it is done
3326 */ 3331 */
3327static void btrfs_end_empty_barrier(struct bio *bio, int err) 3332static void btrfs_end_empty_barrier(struct bio *bio)
3328{ 3333{
3329 if (err)
3330 clear_bit(BIO_UPTODATE, &bio->bi_flags);
3331 if (bio->bi_private) 3334 if (bio->bi_private)
3332 complete(bio->bi_private); 3335 complete(bio->bi_private);
3333 bio_put(bio); 3336 bio_put(bio);
@@ -3355,8 +3358,8 @@ static int write_dev_flush(struct btrfs_device *device, int wait)
3355 3358
3356 wait_for_completion(&device->flush_wait); 3359 wait_for_completion(&device->flush_wait);
3357 3360
3358 if (!bio_flagged(bio, BIO_UPTODATE)) { 3361 if (bio->bi_error) {
3359 ret = -EIO; 3362 ret = bio->bi_error;
3360 btrfs_dev_stat_inc_and_print(device, 3363 btrfs_dev_stat_inc_and_print(device,
3361 BTRFS_DEV_STAT_FLUSH_ERRS); 3364 BTRFS_DEV_STAT_FLUSH_ERRS);
3362 } 3365 }
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 02d05817cbdf..68b12bbc709f 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -2486,7 +2486,7 @@ int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2486 * Scheduling is not allowed, so the extent state tree is expected 2486 * Scheduling is not allowed, so the extent state tree is expected
2487 * to have one and only one object corresponding to this IO. 2487 * to have one and only one object corresponding to this IO.
2488 */ 2488 */
2489static void end_bio_extent_writepage(struct bio *bio, int err) 2489static void end_bio_extent_writepage(struct bio *bio)
2490{ 2490{
2491 struct bio_vec *bvec; 2491 struct bio_vec *bvec;
2492 u64 start; 2492 u64 start;
@@ -2516,7 +2516,7 @@ static void end_bio_extent_writepage(struct bio *bio, int err)
2516 start = page_offset(page); 2516 start = page_offset(page);
2517 end = start + bvec->bv_offset + bvec->bv_len - 1; 2517 end = start + bvec->bv_offset + bvec->bv_len - 1;
2518 2518
2519 if (end_extent_writepage(page, err, start, end)) 2519 if (end_extent_writepage(page, bio->bi_error, start, end))
2520 continue; 2520 continue;
2521 2521
2522 end_page_writeback(page); 2522 end_page_writeback(page);
@@ -2548,10 +2548,10 @@ endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2548 * Scheduling is not allowed, so the extent state tree is expected 2548 * Scheduling is not allowed, so the extent state tree is expected
2549 * to have one and only one object corresponding to this IO. 2549 * to have one and only one object corresponding to this IO.
2550 */ 2550 */
2551static void end_bio_extent_readpage(struct bio *bio, int err) 2551static void end_bio_extent_readpage(struct bio *bio)
2552{ 2552{
2553 struct bio_vec *bvec; 2553 struct bio_vec *bvec;
2554 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 2554 int uptodate = !bio->bi_error;
2555 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 2555 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
2556 struct extent_io_tree *tree; 2556 struct extent_io_tree *tree;
2557 u64 offset = 0; 2557 u64 offset = 0;
@@ -2564,16 +2564,13 @@ static void end_bio_extent_readpage(struct bio *bio, int err)
2564 int ret; 2564 int ret;
2565 int i; 2565 int i;
2566 2566
2567 if (err)
2568 uptodate = 0;
2569
2570 bio_for_each_segment_all(bvec, bio, i) { 2567 bio_for_each_segment_all(bvec, bio, i) {
2571 struct page *page = bvec->bv_page; 2568 struct page *page = bvec->bv_page;
2572 struct inode *inode = page->mapping->host; 2569 struct inode *inode = page->mapping->host;
2573 2570
2574 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, " 2571 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
2575 "mirror=%u\n", (u64)bio->bi_iter.bi_sector, err, 2572 "mirror=%u\n", (u64)bio->bi_iter.bi_sector,
2576 io_bio->mirror_num); 2573 bio->bi_error, io_bio->mirror_num);
2577 tree = &BTRFS_I(inode)->io_tree; 2574 tree = &BTRFS_I(inode)->io_tree;
2578 2575
2579 /* We always issue full-page reads, but if some block 2576 /* We always issue full-page reads, but if some block
@@ -2614,8 +2611,7 @@ static void end_bio_extent_readpage(struct bio *bio, int err)
2614 2611
2615 if (tree->ops && tree->ops->readpage_io_failed_hook) { 2612 if (tree->ops && tree->ops->readpage_io_failed_hook) {
2616 ret = tree->ops->readpage_io_failed_hook(page, mirror); 2613 ret = tree->ops->readpage_io_failed_hook(page, mirror);
2617 if (!ret && !err && 2614 if (!ret && !bio->bi_error)
2618 test_bit(BIO_UPTODATE, &bio->bi_flags))
2619 uptodate = 1; 2615 uptodate = 1;
2620 } else { 2616 } else {
2621 /* 2617 /*
@@ -2631,10 +2627,7 @@ static void end_bio_extent_readpage(struct bio *bio, int err)
2631 ret = bio_readpage_error(bio, offset, page, start, end, 2627 ret = bio_readpage_error(bio, offset, page, start, end,
2632 mirror); 2628 mirror);
2633 if (ret == 0) { 2629 if (ret == 0) {
2634 uptodate = 2630 uptodate = !bio->bi_error;
2635 test_bit(BIO_UPTODATE, &bio->bi_flags);
2636 if (err)
2637 uptodate = 0;
2638 offset += len; 2631 offset += len;
2639 continue; 2632 continue;
2640 } 2633 }
@@ -2684,7 +2677,7 @@ readpage_ok:
2684 endio_readpage_release_extent(tree, extent_start, extent_len, 2677 endio_readpage_release_extent(tree, extent_start, extent_len,
2685 uptodate); 2678 uptodate);
2686 if (io_bio->end_io) 2679 if (io_bio->end_io)
2687 io_bio->end_io(io_bio, err); 2680 io_bio->end_io(io_bio, bio->bi_error);
2688 bio_put(bio); 2681 bio_put(bio);
2689} 2682}
2690 2683
@@ -2802,9 +2795,7 @@ static int submit_extent_page(int rw, struct extent_io_tree *tree,
2802{ 2795{
2803 int ret = 0; 2796 int ret = 0;
2804 struct bio *bio; 2797 struct bio *bio;
2805 int nr;
2806 int contig = 0; 2798 int contig = 0;
2807 int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
2808 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED; 2799 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
2809 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE); 2800 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
2810 2801
@@ -2829,12 +2820,9 @@ static int submit_extent_page(int rw, struct extent_io_tree *tree,
2829 return 0; 2820 return 0;
2830 } 2821 }
2831 } 2822 }
2832 if (this_compressed)
2833 nr = BIO_MAX_PAGES;
2834 else
2835 nr = bio_get_nr_vecs(bdev);
2836 2823
2837 bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH); 2824 bio = btrfs_bio_alloc(bdev, sector, BIO_MAX_PAGES,
2825 GFP_NOFS | __GFP_HIGH);
2838 if (!bio) 2826 if (!bio)
2839 return -ENOMEM; 2827 return -ENOMEM;
2840 2828
@@ -3696,7 +3684,7 @@ static void set_btree_ioerr(struct page *page)
3696 } 3684 }
3697} 3685}
3698 3686
3699static void end_bio_extent_buffer_writepage(struct bio *bio, int err) 3687static void end_bio_extent_buffer_writepage(struct bio *bio)
3700{ 3688{
3701 struct bio_vec *bvec; 3689 struct bio_vec *bvec;
3702 struct extent_buffer *eb; 3690 struct extent_buffer *eb;
@@ -3709,7 +3697,8 @@ static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3709 BUG_ON(!eb); 3697 BUG_ON(!eb);
3710 done = atomic_dec_and_test(&eb->io_pages); 3698 done = atomic_dec_and_test(&eb->io_pages);
3711 3699
3712 if (err || test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) { 3700 if (bio->bi_error ||
3701 test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) {
3713 ClearPageUptodate(page); 3702 ClearPageUptodate(page);
3714 set_btree_ioerr(page); 3703 set_btree_ioerr(page);
3715 } 3704 }
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index e33dff356460..f924d9a62700 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -1845,8 +1845,10 @@ static int __btrfs_submit_bio_done(struct inode *inode, int rw, struct bio *bio,
1845 int ret; 1845 int ret;
1846 1846
1847 ret = btrfs_map_bio(root, rw, bio, mirror_num, 1); 1847 ret = btrfs_map_bio(root, rw, bio, mirror_num, 1);
1848 if (ret) 1848 if (ret) {
1849 bio_endio(bio, ret); 1849 bio->bi_error = ret;
1850 bio_endio(bio);
1851 }
1850 return ret; 1852 return ret;
1851} 1853}
1852 1854
@@ -1906,8 +1908,10 @@ mapit:
1906 ret = btrfs_map_bio(root, rw, bio, mirror_num, 0); 1908 ret = btrfs_map_bio(root, rw, bio, mirror_num, 0);
1907 1909
1908out: 1910out:
1909 if (ret < 0) 1911 if (ret < 0) {
1910 bio_endio(bio, ret); 1912 bio->bi_error = ret;
1913 bio_endio(bio);
1914 }
1911 return ret; 1915 return ret;
1912} 1916}
1913 1917
@@ -7688,13 +7692,13 @@ struct btrfs_retry_complete {
7688 int uptodate; 7692 int uptodate;
7689}; 7693};
7690 7694
7691static void btrfs_retry_endio_nocsum(struct bio *bio, int err) 7695static void btrfs_retry_endio_nocsum(struct bio *bio)
7692{ 7696{
7693 struct btrfs_retry_complete *done = bio->bi_private; 7697 struct btrfs_retry_complete *done = bio->bi_private;
7694 struct bio_vec *bvec; 7698 struct bio_vec *bvec;
7695 int i; 7699 int i;
7696 7700
7697 if (err) 7701 if (bio->bi_error)
7698 goto end; 7702 goto end;
7699 7703
7700 done->uptodate = 1; 7704 done->uptodate = 1;
@@ -7743,7 +7747,7 @@ try_again:
7743 return 0; 7747 return 0;
7744} 7748}
7745 7749
7746static void btrfs_retry_endio(struct bio *bio, int err) 7750static void btrfs_retry_endio(struct bio *bio)
7747{ 7751{
7748 struct btrfs_retry_complete *done = bio->bi_private; 7752 struct btrfs_retry_complete *done = bio->bi_private;
7749 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 7753 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
@@ -7752,7 +7756,7 @@ static void btrfs_retry_endio(struct bio *bio, int err)
7752 int ret; 7756 int ret;
7753 int i; 7757 int i;
7754 7758
7755 if (err) 7759 if (bio->bi_error)
7756 goto end; 7760 goto end;
7757 7761
7758 uptodate = 1; 7762 uptodate = 1;
@@ -7835,12 +7839,13 @@ static int btrfs_subio_endio_read(struct inode *inode,
7835 } 7839 }
7836} 7840}
7837 7841
7838static void btrfs_endio_direct_read(struct bio *bio, int err) 7842static void btrfs_endio_direct_read(struct bio *bio)
7839{ 7843{
7840 struct btrfs_dio_private *dip = bio->bi_private; 7844 struct btrfs_dio_private *dip = bio->bi_private;
7841 struct inode *inode = dip->inode; 7845 struct inode *inode = dip->inode;
7842 struct bio *dio_bio; 7846 struct bio *dio_bio;
7843 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 7847 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
7848 int err = bio->bi_error;
7844 7849
7845 if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED) 7850 if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED)
7846 err = btrfs_subio_endio_read(inode, io_bio, err); 7851 err = btrfs_subio_endio_read(inode, io_bio, err);
@@ -7851,17 +7856,14 @@ static void btrfs_endio_direct_read(struct bio *bio, int err)
7851 7856
7852 kfree(dip); 7857 kfree(dip);
7853 7858
7854 /* If we had a csum failure make sure to clear the uptodate flag */ 7859 dio_end_io(dio_bio, bio->bi_error);
7855 if (err)
7856 clear_bit(BIO_UPTODATE, &dio_bio->bi_flags);
7857 dio_end_io(dio_bio, err);
7858 7860
7859 if (io_bio->end_io) 7861 if (io_bio->end_io)
7860 io_bio->end_io(io_bio, err); 7862 io_bio->end_io(io_bio, err);
7861 bio_put(bio); 7863 bio_put(bio);
7862} 7864}
7863 7865
7864static void btrfs_endio_direct_write(struct bio *bio, int err) 7866static void btrfs_endio_direct_write(struct bio *bio)
7865{ 7867{
7866 struct btrfs_dio_private *dip = bio->bi_private; 7868 struct btrfs_dio_private *dip = bio->bi_private;
7867 struct inode *inode = dip->inode; 7869 struct inode *inode = dip->inode;
@@ -7875,7 +7877,8 @@ static void btrfs_endio_direct_write(struct bio *bio, int err)
7875again: 7877again:
7876 ret = btrfs_dec_test_first_ordered_pending(inode, &ordered, 7878 ret = btrfs_dec_test_first_ordered_pending(inode, &ordered,
7877 &ordered_offset, 7879 &ordered_offset,
7878 ordered_bytes, !err); 7880 ordered_bytes,
7881 !bio->bi_error);
7879 if (!ret) 7882 if (!ret)
7880 goto out_test; 7883 goto out_test;
7881 7884
@@ -7898,10 +7901,7 @@ out_test:
7898 7901
7899 kfree(dip); 7902 kfree(dip);
7900 7903
7901 /* If we had an error make sure to clear the uptodate flag */ 7904 dio_end_io(dio_bio, bio->bi_error);
7902 if (err)
7903 clear_bit(BIO_UPTODATE, &dio_bio->bi_flags);
7904 dio_end_io(dio_bio, err);
7905 bio_put(bio); 7905 bio_put(bio);
7906} 7906}
7907 7907
@@ -7916,9 +7916,10 @@ static int __btrfs_submit_bio_start_direct_io(struct inode *inode, int rw,
7916 return 0; 7916 return 0;
7917} 7917}
7918 7918
7919static void btrfs_end_dio_bio(struct bio *bio, int err) 7919static void btrfs_end_dio_bio(struct bio *bio)
7920{ 7920{
7921 struct btrfs_dio_private *dip = bio->bi_private; 7921 struct btrfs_dio_private *dip = bio->bi_private;
7922 int err = bio->bi_error;
7922 7923
7923 if (err) 7924 if (err)
7924 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info, 7925 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info,
@@ -7947,8 +7948,8 @@ static void btrfs_end_dio_bio(struct bio *bio, int err)
7947 if (dip->errors) { 7948 if (dip->errors) {
7948 bio_io_error(dip->orig_bio); 7949 bio_io_error(dip->orig_bio);
7949 } else { 7950 } else {
7950 set_bit(BIO_UPTODATE, &dip->dio_bio->bi_flags); 7951 dip->dio_bio->bi_error = 0;
7951 bio_endio(dip->orig_bio, 0); 7952 bio_endio(dip->orig_bio);
7952 } 7953 }
7953out: 7954out:
7954 bio_put(bio); 7955 bio_put(bio);
@@ -7957,8 +7958,7 @@ out:
7957static struct bio *btrfs_dio_bio_alloc(struct block_device *bdev, 7958static struct bio *btrfs_dio_bio_alloc(struct block_device *bdev,
7958 u64 first_sector, gfp_t gfp_flags) 7959 u64 first_sector, gfp_t gfp_flags)
7959{ 7960{
7960 int nr_vecs = bio_get_nr_vecs(bdev); 7961 return btrfs_bio_alloc(bdev, first_sector, BIO_MAX_PAGES, gfp_flags);
7961 return btrfs_bio_alloc(bdev, first_sector, nr_vecs, gfp_flags);
7962} 7962}
7963 7963
7964static inline int btrfs_lookup_and_bind_dio_csum(struct btrfs_root *root, 7964static inline int btrfs_lookup_and_bind_dio_csum(struct btrfs_root *root,
@@ -8219,7 +8219,8 @@ free_ordered:
8219 * callbacks - they require an allocated dip and a clone of dio_bio. 8219 * callbacks - they require an allocated dip and a clone of dio_bio.
8220 */ 8220 */
8221 if (io_bio && dip) { 8221 if (io_bio && dip) {
8222 bio_endio(io_bio, ret); 8222 io_bio->bi_error = -EIO;
8223 bio_endio(io_bio);
8223 /* 8224 /*
8224 * The end io callbacks free our dip, do the final put on io_bio 8225 * The end io callbacks free our dip, do the final put on io_bio
8225 * and all the cleanup and final put for dio_bio (through 8226 * and all the cleanup and final put for dio_bio (through
@@ -8246,7 +8247,7 @@ free_ordered:
8246 unlock_extent(&BTRFS_I(inode)->io_tree, file_offset, 8247 unlock_extent(&BTRFS_I(inode)->io_tree, file_offset,
8247 file_offset + dio_bio->bi_iter.bi_size - 1); 8248 file_offset + dio_bio->bi_iter.bi_size - 1);
8248 } 8249 }
8249 clear_bit(BIO_UPTODATE, &dio_bio->bi_flags); 8250 dio_bio->bi_error = -EIO;
8250 /* 8251 /*
8251 * Releases and cleans up our dio_bio, no need to bio_put() 8252 * Releases and cleans up our dio_bio, no need to bio_put()
8252 * nor bio_endio()/bio_io_error() against dio_bio. 8253 * nor bio_endio()/bio_io_error() against dio_bio.
diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c
index fa72068bd256..0a02e24900aa 100644
--- a/fs/btrfs/raid56.c
+++ b/fs/btrfs/raid56.c
@@ -851,7 +851,7 @@ static void free_raid_bio(struct btrfs_raid_bio *rbio)
851 * this frees the rbio and runs through all the bios in the 851 * this frees the rbio and runs through all the bios in the
852 * bio_list and calls end_io on them 852 * bio_list and calls end_io on them
853 */ 853 */
854static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, int err, int uptodate) 854static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, int err)
855{ 855{
856 struct bio *cur = bio_list_get(&rbio->bio_list); 856 struct bio *cur = bio_list_get(&rbio->bio_list);
857 struct bio *next; 857 struct bio *next;
@@ -864,9 +864,8 @@ static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, int err, int uptodate)
864 while (cur) { 864 while (cur) {
865 next = cur->bi_next; 865 next = cur->bi_next;
866 cur->bi_next = NULL; 866 cur->bi_next = NULL;
867 if (uptodate) 867 cur->bi_error = err;
868 set_bit(BIO_UPTODATE, &cur->bi_flags); 868 bio_endio(cur);
869 bio_endio(cur, err);
870 cur = next; 869 cur = next;
871 } 870 }
872} 871}
@@ -875,9 +874,10 @@ static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, int err, int uptodate)
875 * end io function used by finish_rmw. When we finally 874 * end io function used by finish_rmw. When we finally
876 * get here, we've written a full stripe 875 * get here, we've written a full stripe
877 */ 876 */
878static void raid_write_end_io(struct bio *bio, int err) 877static void raid_write_end_io(struct bio *bio)
879{ 878{
880 struct btrfs_raid_bio *rbio = bio->bi_private; 879 struct btrfs_raid_bio *rbio = bio->bi_private;
880 int err = bio->bi_error;
881 881
882 if (err) 882 if (err)
883 fail_bio_stripe(rbio, bio); 883 fail_bio_stripe(rbio, bio);
@@ -893,7 +893,7 @@ static void raid_write_end_io(struct bio *bio, int err)
893 if (atomic_read(&rbio->error) > rbio->bbio->max_errors) 893 if (atomic_read(&rbio->error) > rbio->bbio->max_errors)
894 err = -EIO; 894 err = -EIO;
895 895
896 rbio_orig_end_io(rbio, err, 0); 896 rbio_orig_end_io(rbio, err);
897 return; 897 return;
898} 898}
899 899
@@ -1071,7 +1071,7 @@ static int rbio_add_io_page(struct btrfs_raid_bio *rbio,
1071 * devices or if they are not contiguous 1071 * devices or if they are not contiguous
1072 */ 1072 */
1073 if (last_end == disk_start && stripe->dev->bdev && 1073 if (last_end == disk_start && stripe->dev->bdev &&
1074 test_bit(BIO_UPTODATE, &last->bi_flags) && 1074 !last->bi_error &&
1075 last->bi_bdev == stripe->dev->bdev) { 1075 last->bi_bdev == stripe->dev->bdev) {
1076 ret = bio_add_page(last, page, PAGE_CACHE_SIZE, 0); 1076 ret = bio_add_page(last, page, PAGE_CACHE_SIZE, 0);
1077 if (ret == PAGE_CACHE_SIZE) 1077 if (ret == PAGE_CACHE_SIZE)
@@ -1087,7 +1087,6 @@ static int rbio_add_io_page(struct btrfs_raid_bio *rbio,
1087 bio->bi_iter.bi_size = 0; 1087 bio->bi_iter.bi_size = 0;
1088 bio->bi_bdev = stripe->dev->bdev; 1088 bio->bi_bdev = stripe->dev->bdev;
1089 bio->bi_iter.bi_sector = disk_start >> 9; 1089 bio->bi_iter.bi_sector = disk_start >> 9;
1090 set_bit(BIO_UPTODATE, &bio->bi_flags);
1091 1090
1092 bio_add_page(bio, page, PAGE_CACHE_SIZE, 0); 1091 bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
1093 bio_list_add(bio_list, bio); 1092 bio_list_add(bio_list, bio);
@@ -1312,13 +1311,12 @@ write_data:
1312 1311
1313 bio->bi_private = rbio; 1312 bio->bi_private = rbio;
1314 bio->bi_end_io = raid_write_end_io; 1313 bio->bi_end_io = raid_write_end_io;
1315 BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags));
1316 submit_bio(WRITE, bio); 1314 submit_bio(WRITE, bio);
1317 } 1315 }
1318 return; 1316 return;
1319 1317
1320cleanup: 1318cleanup:
1321 rbio_orig_end_io(rbio, -EIO, 0); 1319 rbio_orig_end_io(rbio, -EIO);
1322} 1320}
1323 1321
1324/* 1322/*
@@ -1441,11 +1439,11 @@ static void set_bio_pages_uptodate(struct bio *bio)
1441 * This will usually kick off finish_rmw once all the bios are read in, but it 1439 * This will usually kick off finish_rmw once all the bios are read in, but it
1442 * may trigger parity reconstruction if we had any errors along the way 1440 * may trigger parity reconstruction if we had any errors along the way
1443 */ 1441 */
1444static void raid_rmw_end_io(struct bio *bio, int err) 1442static void raid_rmw_end_io(struct bio *bio)
1445{ 1443{
1446 struct btrfs_raid_bio *rbio = bio->bi_private; 1444 struct btrfs_raid_bio *rbio = bio->bi_private;
1447 1445
1448 if (err) 1446 if (bio->bi_error)
1449 fail_bio_stripe(rbio, bio); 1447 fail_bio_stripe(rbio, bio);
1450 else 1448 else
1451 set_bio_pages_uptodate(bio); 1449 set_bio_pages_uptodate(bio);
@@ -1455,7 +1453,6 @@ static void raid_rmw_end_io(struct bio *bio, int err)
1455 if (!atomic_dec_and_test(&rbio->stripes_pending)) 1453 if (!atomic_dec_and_test(&rbio->stripes_pending))
1456 return; 1454 return;
1457 1455
1458 err = 0;
1459 if (atomic_read(&rbio->error) > rbio->bbio->max_errors) 1456 if (atomic_read(&rbio->error) > rbio->bbio->max_errors)
1460 goto cleanup; 1457 goto cleanup;
1461 1458
@@ -1469,7 +1466,7 @@ static void raid_rmw_end_io(struct bio *bio, int err)
1469 1466
1470cleanup: 1467cleanup:
1471 1468
1472 rbio_orig_end_io(rbio, -EIO, 0); 1469 rbio_orig_end_io(rbio, -EIO);
1473} 1470}
1474 1471
1475static void async_rmw_stripe(struct btrfs_raid_bio *rbio) 1472static void async_rmw_stripe(struct btrfs_raid_bio *rbio)
@@ -1572,14 +1569,13 @@ static int raid56_rmw_stripe(struct btrfs_raid_bio *rbio)
1572 btrfs_bio_wq_end_io(rbio->fs_info, bio, 1569 btrfs_bio_wq_end_io(rbio->fs_info, bio,
1573 BTRFS_WQ_ENDIO_RAID56); 1570 BTRFS_WQ_ENDIO_RAID56);
1574 1571
1575 BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags));
1576 submit_bio(READ, bio); 1572 submit_bio(READ, bio);
1577 } 1573 }
1578 /* the actual write will happen once the reads are done */ 1574 /* the actual write will happen once the reads are done */
1579 return 0; 1575 return 0;
1580 1576
1581cleanup: 1577cleanup:
1582 rbio_orig_end_io(rbio, -EIO, 0); 1578 rbio_orig_end_io(rbio, -EIO);
1583 return -EIO; 1579 return -EIO;
1584 1580
1585finish: 1581finish:
@@ -1964,7 +1960,7 @@ cleanup_io:
1964 else 1960 else
1965 clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags); 1961 clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags);
1966 1962
1967 rbio_orig_end_io(rbio, err, err == 0); 1963 rbio_orig_end_io(rbio, err);
1968 } else if (err == 0) { 1964 } else if (err == 0) {
1969 rbio->faila = -1; 1965 rbio->faila = -1;
1970 rbio->failb = -1; 1966 rbio->failb = -1;
@@ -1976,7 +1972,7 @@ cleanup_io:
1976 else 1972 else
1977 BUG(); 1973 BUG();
1978 } else { 1974 } else {
1979 rbio_orig_end_io(rbio, err, 0); 1975 rbio_orig_end_io(rbio, err);
1980 } 1976 }
1981} 1977}
1982 1978
@@ -1984,7 +1980,7 @@ cleanup_io:
1984 * This is called only for stripes we've read from disk to 1980 * This is called only for stripes we've read from disk to
1985 * reconstruct the parity. 1981 * reconstruct the parity.
1986 */ 1982 */
1987static void raid_recover_end_io(struct bio *bio, int err) 1983static void raid_recover_end_io(struct bio *bio)
1988{ 1984{
1989 struct btrfs_raid_bio *rbio = bio->bi_private; 1985 struct btrfs_raid_bio *rbio = bio->bi_private;
1990 1986
@@ -1992,7 +1988,7 @@ static void raid_recover_end_io(struct bio *bio, int err)
1992 * we only read stripe pages off the disk, set them 1988 * we only read stripe pages off the disk, set them
1993 * up to date if there were no errors 1989 * up to date if there were no errors
1994 */ 1990 */
1995 if (err) 1991 if (bio->bi_error)
1996 fail_bio_stripe(rbio, bio); 1992 fail_bio_stripe(rbio, bio);
1997 else 1993 else
1998 set_bio_pages_uptodate(bio); 1994 set_bio_pages_uptodate(bio);
@@ -2002,7 +1998,7 @@ static void raid_recover_end_io(struct bio *bio, int err)
2002 return; 1998 return;
2003 1999
2004 if (atomic_read(&rbio->error) > rbio->bbio->max_errors) 2000 if (atomic_read(&rbio->error) > rbio->bbio->max_errors)
2005 rbio_orig_end_io(rbio, -EIO, 0); 2001 rbio_orig_end_io(rbio, -EIO);
2006 else 2002 else
2007 __raid_recover_end_io(rbio); 2003 __raid_recover_end_io(rbio);
2008} 2004}
@@ -2094,7 +2090,6 @@ static int __raid56_parity_recover(struct btrfs_raid_bio *rbio)
2094 btrfs_bio_wq_end_io(rbio->fs_info, bio, 2090 btrfs_bio_wq_end_io(rbio->fs_info, bio,
2095 BTRFS_WQ_ENDIO_RAID56); 2091 BTRFS_WQ_ENDIO_RAID56);
2096 2092
2097 BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags));
2098 submit_bio(READ, bio); 2093 submit_bio(READ, bio);
2099 } 2094 }
2100out: 2095out:
@@ -2102,7 +2097,7 @@ out:
2102 2097
2103cleanup: 2098cleanup:
2104 if (rbio->operation == BTRFS_RBIO_READ_REBUILD) 2099 if (rbio->operation == BTRFS_RBIO_READ_REBUILD)
2105 rbio_orig_end_io(rbio, -EIO, 0); 2100 rbio_orig_end_io(rbio, -EIO);
2106 return -EIO; 2101 return -EIO;
2107} 2102}
2108 2103
@@ -2277,11 +2272,12 @@ static int alloc_rbio_essential_pages(struct btrfs_raid_bio *rbio)
2277 * end io function used by finish_rmw. When we finally 2272 * end io function used by finish_rmw. When we finally
2278 * get here, we've written a full stripe 2273 * get here, we've written a full stripe
2279 */ 2274 */
2280static void raid_write_parity_end_io(struct bio *bio, int err) 2275static void raid_write_parity_end_io(struct bio *bio)
2281{ 2276{
2282 struct btrfs_raid_bio *rbio = bio->bi_private; 2277 struct btrfs_raid_bio *rbio = bio->bi_private;
2278 int err = bio->bi_error;
2283 2279
2284 if (err) 2280 if (bio->bi_error)
2285 fail_bio_stripe(rbio, bio); 2281 fail_bio_stripe(rbio, bio);
2286 2282
2287 bio_put(bio); 2283 bio_put(bio);
@@ -2294,7 +2290,7 @@ static void raid_write_parity_end_io(struct bio *bio, int err)
2294 if (atomic_read(&rbio->error)) 2290 if (atomic_read(&rbio->error))
2295 err = -EIO; 2291 err = -EIO;
2296 2292
2297 rbio_orig_end_io(rbio, err, 0); 2293 rbio_orig_end_io(rbio, err);
2298} 2294}
2299 2295
2300static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio, 2296static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio,
@@ -2437,7 +2433,7 @@ submit_write:
2437 nr_data = bio_list_size(&bio_list); 2433 nr_data = bio_list_size(&bio_list);
2438 if (!nr_data) { 2434 if (!nr_data) {
2439 /* Every parity is right */ 2435 /* Every parity is right */
2440 rbio_orig_end_io(rbio, 0, 0); 2436 rbio_orig_end_io(rbio, 0);
2441 return; 2437 return;
2442 } 2438 }
2443 2439
@@ -2450,13 +2446,12 @@ submit_write:
2450 2446
2451 bio->bi_private = rbio; 2447 bio->bi_private = rbio;
2452 bio->bi_end_io = raid_write_parity_end_io; 2448 bio->bi_end_io = raid_write_parity_end_io;
2453 BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags));
2454 submit_bio(WRITE, bio); 2449 submit_bio(WRITE, bio);
2455 } 2450 }
2456 return; 2451 return;
2457 2452
2458cleanup: 2453cleanup:
2459 rbio_orig_end_io(rbio, -EIO, 0); 2454 rbio_orig_end_io(rbio, -EIO);
2460} 2455}
2461 2456
2462static inline int is_data_stripe(struct btrfs_raid_bio *rbio, int stripe) 2457static inline int is_data_stripe(struct btrfs_raid_bio *rbio, int stripe)
@@ -2524,7 +2519,7 @@ static void validate_rbio_for_parity_scrub(struct btrfs_raid_bio *rbio)
2524 return; 2519 return;
2525 2520
2526cleanup: 2521cleanup:
2527 rbio_orig_end_io(rbio, -EIO, 0); 2522 rbio_orig_end_io(rbio, -EIO);
2528} 2523}
2529 2524
2530/* 2525/*
@@ -2535,11 +2530,11 @@ cleanup:
2535 * This will usually kick off finish_rmw once all the bios are read in, but it 2530 * This will usually kick off finish_rmw once all the bios are read in, but it
2536 * may trigger parity reconstruction if we had any errors along the way 2531 * may trigger parity reconstruction if we had any errors along the way
2537 */ 2532 */
2538static void raid56_parity_scrub_end_io(struct bio *bio, int err) 2533static void raid56_parity_scrub_end_io(struct bio *bio)
2539{ 2534{
2540 struct btrfs_raid_bio *rbio = bio->bi_private; 2535 struct btrfs_raid_bio *rbio = bio->bi_private;
2541 2536
2542 if (err) 2537 if (bio->bi_error)
2543 fail_bio_stripe(rbio, bio); 2538 fail_bio_stripe(rbio, bio);
2544 else 2539 else
2545 set_bio_pages_uptodate(bio); 2540 set_bio_pages_uptodate(bio);
@@ -2632,14 +2627,13 @@ static void raid56_parity_scrub_stripe(struct btrfs_raid_bio *rbio)
2632 btrfs_bio_wq_end_io(rbio->fs_info, bio, 2627 btrfs_bio_wq_end_io(rbio->fs_info, bio,
2633 BTRFS_WQ_ENDIO_RAID56); 2628 BTRFS_WQ_ENDIO_RAID56);
2634 2629
2635 BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags));
2636 submit_bio(READ, bio); 2630 submit_bio(READ, bio);
2637 } 2631 }
2638 /* the actual write will happen once the reads are done */ 2632 /* the actual write will happen once the reads are done */
2639 return; 2633 return;
2640 2634
2641cleanup: 2635cleanup:
2642 rbio_orig_end_io(rbio, -EIO, 0); 2636 rbio_orig_end_io(rbio, -EIO);
2643 return; 2637 return;
2644 2638
2645finish: 2639finish:
diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c
index 94db0fa5225a..9c146d8307b5 100644
--- a/fs/btrfs/scrub.c
+++ b/fs/btrfs/scrub.c
@@ -278,7 +278,7 @@ static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
278 u64 physical, struct btrfs_device *dev, u64 flags, 278 u64 physical, struct btrfs_device *dev, u64 flags,
279 u64 gen, int mirror_num, u8 *csum, int force, 279 u64 gen, int mirror_num, u8 *csum, int force,
280 u64 physical_for_dev_replace); 280 u64 physical_for_dev_replace);
281static void scrub_bio_end_io(struct bio *bio, int err); 281static void scrub_bio_end_io(struct bio *bio);
282static void scrub_bio_end_io_worker(struct btrfs_work *work); 282static void scrub_bio_end_io_worker(struct btrfs_work *work);
283static void scrub_block_complete(struct scrub_block *sblock); 283static void scrub_block_complete(struct scrub_block *sblock);
284static void scrub_remap_extent(struct btrfs_fs_info *fs_info, 284static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
@@ -295,7 +295,7 @@ static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx);
295static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx, 295static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
296 struct scrub_page *spage); 296 struct scrub_page *spage);
297static void scrub_wr_submit(struct scrub_ctx *sctx); 297static void scrub_wr_submit(struct scrub_ctx *sctx);
298static void scrub_wr_bio_end_io(struct bio *bio, int err); 298static void scrub_wr_bio_end_io(struct bio *bio);
299static void scrub_wr_bio_end_io_worker(struct btrfs_work *work); 299static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
300static int write_page_nocow(struct scrub_ctx *sctx, 300static int write_page_nocow(struct scrub_ctx *sctx,
301 u64 physical_for_dev_replace, struct page *page); 301 u64 physical_for_dev_replace, struct page *page);
@@ -454,27 +454,14 @@ struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
454 struct scrub_ctx *sctx; 454 struct scrub_ctx *sctx;
455 int i; 455 int i;
456 struct btrfs_fs_info *fs_info = dev->dev_root->fs_info; 456 struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
457 int pages_per_rd_bio;
458 int ret; 457 int ret;
459 458
460 /*
461 * the setting of pages_per_rd_bio is correct for scrub but might
462 * be wrong for the dev_replace code where we might read from
463 * different devices in the initial huge bios. However, that
464 * code is able to correctly handle the case when adding a page
465 * to a bio fails.
466 */
467 if (dev->bdev)
468 pages_per_rd_bio = min_t(int, SCRUB_PAGES_PER_RD_BIO,
469 bio_get_nr_vecs(dev->bdev));
470 else
471 pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
472 sctx = kzalloc(sizeof(*sctx), GFP_NOFS); 459 sctx = kzalloc(sizeof(*sctx), GFP_NOFS);
473 if (!sctx) 460 if (!sctx)
474 goto nomem; 461 goto nomem;
475 atomic_set(&sctx->refs, 1); 462 atomic_set(&sctx->refs, 1);
476 sctx->is_dev_replace = is_dev_replace; 463 sctx->is_dev_replace = is_dev_replace;
477 sctx->pages_per_rd_bio = pages_per_rd_bio; 464 sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
478 sctx->curr = -1; 465 sctx->curr = -1;
479 sctx->dev_root = dev->dev_root; 466 sctx->dev_root = dev->dev_root;
480 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) { 467 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
@@ -1429,11 +1416,11 @@ struct scrub_bio_ret {
1429 int error; 1416 int error;
1430}; 1417};
1431 1418
1432static void scrub_bio_wait_endio(struct bio *bio, int error) 1419static void scrub_bio_wait_endio(struct bio *bio)
1433{ 1420{
1434 struct scrub_bio_ret *ret = bio->bi_private; 1421 struct scrub_bio_ret *ret = bio->bi_private;
1435 1422
1436 ret->error = error; 1423 ret->error = bio->bi_error;
1437 complete(&ret->event); 1424 complete(&ret->event);
1438} 1425}
1439 1426
@@ -1790,12 +1777,12 @@ static void scrub_wr_submit(struct scrub_ctx *sctx)
1790 btrfsic_submit_bio(WRITE, sbio->bio); 1777 btrfsic_submit_bio(WRITE, sbio->bio);
1791} 1778}
1792 1779
1793static void scrub_wr_bio_end_io(struct bio *bio, int err) 1780static void scrub_wr_bio_end_io(struct bio *bio)
1794{ 1781{
1795 struct scrub_bio *sbio = bio->bi_private; 1782 struct scrub_bio *sbio = bio->bi_private;
1796 struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info; 1783 struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
1797 1784
1798 sbio->err = err; 1785 sbio->err = bio->bi_error;
1799 sbio->bio = bio; 1786 sbio->bio = bio;
1800 1787
1801 btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper, 1788 btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,
@@ -2098,7 +2085,7 @@ static void scrub_submit(struct scrub_ctx *sctx)
2098 */ 2085 */
2099 printk_ratelimited(KERN_WARNING 2086 printk_ratelimited(KERN_WARNING
2100 "BTRFS: scrub_submit(bio bdev == NULL) is unexpected!\n"); 2087 "BTRFS: scrub_submit(bio bdev == NULL) is unexpected!\n");
2101 bio_endio(sbio->bio, -EIO); 2088 bio_io_error(sbio->bio);
2102 } else { 2089 } else {
2103 btrfsic_submit_bio(READ, sbio->bio); 2090 btrfsic_submit_bio(READ, sbio->bio);
2104 } 2091 }
@@ -2260,12 +2247,12 @@ leave_nomem:
2260 return 0; 2247 return 0;
2261} 2248}
2262 2249
2263static void scrub_bio_end_io(struct bio *bio, int err) 2250static void scrub_bio_end_io(struct bio *bio)
2264{ 2251{
2265 struct scrub_bio *sbio = bio->bi_private; 2252 struct scrub_bio *sbio = bio->bi_private;
2266 struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info; 2253 struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
2267 2254
2268 sbio->err = err; 2255 sbio->err = bio->bi_error;
2269 sbio->bio = bio; 2256 sbio->bio = bio;
2270 2257
2271 btrfs_queue_work(fs_info->scrub_workers, &sbio->work); 2258 btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
@@ -2672,11 +2659,11 @@ static void scrub_parity_bio_endio_worker(struct btrfs_work *work)
2672 scrub_pending_bio_dec(sctx); 2659 scrub_pending_bio_dec(sctx);
2673} 2660}
2674 2661
2675static void scrub_parity_bio_endio(struct bio *bio, int error) 2662static void scrub_parity_bio_endio(struct bio *bio)
2676{ 2663{
2677 struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private; 2664 struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
2678 2665
2679 if (error) 2666 if (bio->bi_error)
2680 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap, 2667 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2681 sparity->nsectors); 2668 sparity->nsectors);
2682 2669
@@ -3896,8 +3883,7 @@ static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
3896 return 0; 3883 return 0;
3897 3884
3898 WARN_ON(!dev->bdev); 3885 WARN_ON(!dev->bdev);
3899 wr_ctx->pages_per_wr_bio = min_t(int, SCRUB_PAGES_PER_WR_BIO, 3886 wr_ctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO;
3900 bio_get_nr_vecs(dev->bdev));
3901 wr_ctx->tgtdev = dev; 3887 wr_ctx->tgtdev = dev;
3902 atomic_set(&wr_ctx->flush_all_writes, 0); 3888 atomic_set(&wr_ctx->flush_all_writes, 0);
3903 return 0; 3889 return 0;
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index fbe7c104531c..762476f5f08d 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -5741,23 +5741,23 @@ int btrfs_rmap_block(struct btrfs_mapping_tree *map_tree,
5741 return 0; 5741 return 0;
5742} 5742}
5743 5743
5744static inline void btrfs_end_bbio(struct btrfs_bio *bbio, struct bio *bio, int err) 5744static inline void btrfs_end_bbio(struct btrfs_bio *bbio, struct bio *bio)
5745{ 5745{
5746 bio->bi_private = bbio->private; 5746 bio->bi_private = bbio->private;
5747 bio->bi_end_io = bbio->end_io; 5747 bio->bi_end_io = bbio->end_io;
5748 bio_endio(bio, err); 5748 bio_endio(bio);
5749 5749
5750 btrfs_put_bbio(bbio); 5750 btrfs_put_bbio(bbio);
5751} 5751}
5752 5752
5753static void btrfs_end_bio(struct bio *bio, int err) 5753static void btrfs_end_bio(struct bio *bio)
5754{ 5754{
5755 struct btrfs_bio *bbio = bio->bi_private; 5755 struct btrfs_bio *bbio = bio->bi_private;
5756 int is_orig_bio = 0; 5756 int is_orig_bio = 0;
5757 5757
5758 if (err) { 5758 if (bio->bi_error) {
5759 atomic_inc(&bbio->error); 5759 atomic_inc(&bbio->error);
5760 if (err == -EIO || err == -EREMOTEIO) { 5760 if (bio->bi_error == -EIO || bio->bi_error == -EREMOTEIO) {
5761 unsigned int stripe_index = 5761 unsigned int stripe_index =
5762 btrfs_io_bio(bio)->stripe_index; 5762 btrfs_io_bio(bio)->stripe_index;
5763 struct btrfs_device *dev; 5763 struct btrfs_device *dev;
@@ -5795,17 +5795,16 @@ static void btrfs_end_bio(struct bio *bio, int err)
5795 * beyond the tolerance of the btrfs bio 5795 * beyond the tolerance of the btrfs bio
5796 */ 5796 */
5797 if (atomic_read(&bbio->error) > bbio->max_errors) { 5797 if (atomic_read(&bbio->error) > bbio->max_errors) {
5798 err = -EIO; 5798 bio->bi_error = -EIO;
5799 } else { 5799 } else {
5800 /* 5800 /*
5801 * this bio is actually up to date, we didn't 5801 * this bio is actually up to date, we didn't
5802 * go over the max number of errors 5802 * go over the max number of errors
5803 */ 5803 */
5804 set_bit(BIO_UPTODATE, &bio->bi_flags); 5804 bio->bi_error = 0;
5805 err = 0;
5806 } 5805 }
5807 5806
5808 btrfs_end_bbio(bbio, bio, err); 5807 btrfs_end_bbio(bbio, bio);
5809 } else if (!is_orig_bio) { 5808 } else if (!is_orig_bio) {
5810 bio_put(bio); 5809 bio_put(bio);
5811 } 5810 }
@@ -5826,7 +5825,7 @@ static noinline void btrfs_schedule_bio(struct btrfs_root *root,
5826 struct btrfs_pending_bios *pending_bios; 5825 struct btrfs_pending_bios *pending_bios;
5827 5826
5828 if (device->missing || !device->bdev) { 5827 if (device->missing || !device->bdev) {
5829 bio_endio(bio, -EIO); 5828 bio_io_error(bio);
5830 return; 5829 return;
5831 } 5830 }
5832 5831
@@ -5871,34 +5870,6 @@ static noinline void btrfs_schedule_bio(struct btrfs_root *root,
5871 &device->work); 5870 &device->work);
5872} 5871}
5873 5872
5874static int bio_size_ok(struct block_device *bdev, struct bio *bio,
5875 sector_t sector)
5876{
5877 struct bio_vec *prev;
5878 struct request_queue *q = bdev_get_queue(bdev);
5879 unsigned int max_sectors = queue_max_sectors(q);
5880 struct bvec_merge_data bvm = {
5881 .bi_bdev = bdev,
5882 .bi_sector = sector,
5883 .bi_rw = bio->bi_rw,
5884 };
5885
5886 if (WARN_ON(bio->bi_vcnt == 0))
5887 return 1;
5888
5889 prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
5890 if (bio_sectors(bio) > max_sectors)
5891 return 0;
5892
5893 if (!q->merge_bvec_fn)
5894 return 1;
5895
5896 bvm.bi_size = bio->bi_iter.bi_size - prev->bv_len;
5897 if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len)
5898 return 0;
5899 return 1;
5900}
5901
5902static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio, 5873static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,
5903 struct bio *bio, u64 physical, int dev_nr, 5874 struct bio *bio, u64 physical, int dev_nr,
5904 int rw, int async) 5875 int rw, int async)
@@ -5932,38 +5903,6 @@ static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,
5932 btrfsic_submit_bio(rw, bio); 5903 btrfsic_submit_bio(rw, bio);
5933} 5904}
5934 5905
5935static int breakup_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,
5936 struct bio *first_bio, struct btrfs_device *dev,
5937 int dev_nr, int rw, int async)
5938{
5939 struct bio_vec *bvec = first_bio->bi_io_vec;
5940 struct bio *bio;
5941 int nr_vecs = bio_get_nr_vecs(dev->bdev);
5942 u64 physical = bbio->stripes[dev_nr].physical;
5943
5944again:
5945 bio = btrfs_bio_alloc(dev->bdev, physical >> 9, nr_vecs, GFP_NOFS);
5946 if (!bio)
5947 return -ENOMEM;
5948
5949 while (bvec <= (first_bio->bi_io_vec + first_bio->bi_vcnt - 1)) {
5950 if (bio_add_page(bio, bvec->bv_page, bvec->bv_len,
5951 bvec->bv_offset) < bvec->bv_len) {
5952 u64 len = bio->bi_iter.bi_size;
5953
5954 atomic_inc(&bbio->stripes_pending);
5955 submit_stripe_bio(root, bbio, bio, physical, dev_nr,
5956 rw, async);
5957 physical += len;
5958 goto again;
5959 }
5960 bvec++;
5961 }
5962
5963 submit_stripe_bio(root, bbio, bio, physical, dev_nr, rw, async);
5964 return 0;
5965}
5966
5967static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical) 5906static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical)
5968{ 5907{
5969 atomic_inc(&bbio->error); 5908 atomic_inc(&bbio->error);
@@ -5973,8 +5912,8 @@ static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical)
5973 5912
5974 btrfs_io_bio(bio)->mirror_num = bbio->mirror_num; 5913 btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
5975 bio->bi_iter.bi_sector = logical >> 9; 5914 bio->bi_iter.bi_sector = logical >> 9;
5976 5915 bio->bi_error = -EIO;
5977 btrfs_end_bbio(bbio, bio, -EIO); 5916 btrfs_end_bbio(bbio, bio);
5978 } 5917 }
5979} 5918}
5980 5919
@@ -6036,18 +5975,6 @@ int btrfs_map_bio(struct btrfs_root *root, int rw, struct bio *bio,
6036 continue; 5975 continue;
6037 } 5976 }
6038 5977
6039 /*
6040 * Check and see if we're ok with this bio based on it's size
6041 * and offset with the given device.
6042 */
6043 if (!bio_size_ok(dev->bdev, first_bio,
6044 bbio->stripes[dev_nr].physical >> 9)) {
6045 ret = breakup_stripe_bio(root, bbio, first_bio, dev,
6046 dev_nr, rw, async_submit);
6047 BUG_ON(ret);
6048 continue;
6049 }
6050
6051 if (dev_nr < total_devs - 1) { 5978 if (dev_nr < total_devs - 1) {
6052 bio = btrfs_bio_clone(first_bio, GFP_NOFS); 5979 bio = btrfs_bio_clone(first_bio, GFP_NOFS);
6053 BUG_ON(!bio); /* -ENOMEM */ 5980 BUG_ON(!bio); /* -ENOMEM */
diff --git a/fs/buffer.c b/fs/buffer.c
index 1cf7a53a0277..82283abb2795 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -2957,14 +2957,14 @@ sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2957} 2957}
2958EXPORT_SYMBOL(generic_block_bmap); 2958EXPORT_SYMBOL(generic_block_bmap);
2959 2959
2960static void end_bio_bh_io_sync(struct bio *bio, int err) 2960static void end_bio_bh_io_sync(struct bio *bio)
2961{ 2961{
2962 struct buffer_head *bh = bio->bi_private; 2962 struct buffer_head *bh = bio->bi_private;
2963 2963
2964 if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags))) 2964 if (unlikely(bio_flagged(bio, BIO_QUIET)))
2965 set_bit(BH_Quiet, &bh->b_state); 2965 set_bit(BH_Quiet, &bh->b_state);
2966 2966
2967 bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags)); 2967 bh->b_end_io(bh, !bio->bi_error);
2968 bio_put(bio); 2968 bio_put(bio);
2969} 2969}
2970 2970
@@ -3046,12 +3046,9 @@ static int submit_bh_wbc(int rw, struct buffer_head *bh,
3046 3046
3047 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); 3047 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
3048 bio->bi_bdev = bh->b_bdev; 3048 bio->bi_bdev = bh->b_bdev;
3049 bio->bi_io_vec[0].bv_page = bh->b_page;
3050 bio->bi_io_vec[0].bv_len = bh->b_size;
3051 bio->bi_io_vec[0].bv_offset = bh_offset(bh);
3052 3049
3053 bio->bi_vcnt = 1; 3050 bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
3054 bio->bi_iter.bi_size = bh->b_size; 3051 BUG_ON(bio->bi_iter.bi_size != bh->b_size);
3055 3052
3056 bio->bi_end_io = end_bio_bh_io_sync; 3053 bio->bi_end_io = end_bio_bh_io_sync;
3057 bio->bi_private = bh; 3054 bio->bi_private = bh;
diff --git a/fs/direct-io.c b/fs/direct-io.c
index 745d2342651a..11256291642e 100644
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -285,7 +285,7 @@ static int dio_bio_complete(struct dio *dio, struct bio *bio);
285/* 285/*
286 * Asynchronous IO callback. 286 * Asynchronous IO callback.
287 */ 287 */
288static void dio_bio_end_aio(struct bio *bio, int error) 288static void dio_bio_end_aio(struct bio *bio)
289{ 289{
290 struct dio *dio = bio->bi_private; 290 struct dio *dio = bio->bi_private;
291 unsigned long remaining; 291 unsigned long remaining;
@@ -318,7 +318,7 @@ static void dio_bio_end_aio(struct bio *bio, int error)
318 * During I/O bi_private points at the dio. After I/O, bi_private is used to 318 * During I/O bi_private points at the dio. After I/O, bi_private is used to
319 * implement a singly-linked list of completed BIOs, at dio->bio_list. 319 * implement a singly-linked list of completed BIOs, at dio->bio_list.
320 */ 320 */
321static void dio_bio_end_io(struct bio *bio, int error) 321static void dio_bio_end_io(struct bio *bio)
322{ 322{
323 struct dio *dio = bio->bi_private; 323 struct dio *dio = bio->bi_private;
324 unsigned long flags; 324 unsigned long flags;
@@ -345,9 +345,9 @@ void dio_end_io(struct bio *bio, int error)
345 struct dio *dio = bio->bi_private; 345 struct dio *dio = bio->bi_private;
346 346
347 if (dio->is_async) 347 if (dio->is_async)
348 dio_bio_end_aio(bio, error); 348 dio_bio_end_aio(bio);
349 else 349 else
350 dio_bio_end_io(bio, error); 350 dio_bio_end_io(bio);
351} 351}
352EXPORT_SYMBOL_GPL(dio_end_io); 352EXPORT_SYMBOL_GPL(dio_end_io);
353 353
@@ -457,15 +457,16 @@ static struct bio *dio_await_one(struct dio *dio)
457 */ 457 */
458static int dio_bio_complete(struct dio *dio, struct bio *bio) 458static int dio_bio_complete(struct dio *dio, struct bio *bio)
459{ 459{
460 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
461 struct bio_vec *bvec; 460 struct bio_vec *bvec;
462 unsigned i; 461 unsigned i;
462 int err;
463 463
464 if (!uptodate) 464 if (bio->bi_error)
465 dio->io_error = -EIO; 465 dio->io_error = -EIO;
466 466
467 if (dio->is_async && dio->rw == READ) { 467 if (dio->is_async && dio->rw == READ) {
468 bio_check_pages_dirty(bio); /* transfers ownership */ 468 bio_check_pages_dirty(bio); /* transfers ownership */
469 err = bio->bi_error;
469 } else { 470 } else {
470 bio_for_each_segment_all(bvec, bio, i) { 471 bio_for_each_segment_all(bvec, bio, i) {
471 struct page *page = bvec->bv_page; 472 struct page *page = bvec->bv_page;
@@ -474,9 +475,10 @@ static int dio_bio_complete(struct dio *dio, struct bio *bio)
474 set_page_dirty_lock(page); 475 set_page_dirty_lock(page);
475 page_cache_release(page); 476 page_cache_release(page);
476 } 477 }
478 err = bio->bi_error;
477 bio_put(bio); 479 bio_put(bio);
478 } 480 }
479 return uptodate ? 0 : -EIO; 481 return err;
480} 482}
481 483
482/* 484/*
@@ -653,7 +655,7 @@ static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio,
653 if (ret) 655 if (ret)
654 goto out; 656 goto out;
655 sector = start_sector << (sdio->blkbits - 9); 657 sector = start_sector << (sdio->blkbits - 9);
656 nr_pages = min(sdio->pages_in_io, bio_get_nr_vecs(map_bh->b_bdev)); 658 nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES);
657 BUG_ON(nr_pages <= 0); 659 BUG_ON(nr_pages <= 0);
658 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); 660 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages);
659 sdio->boundary = 0; 661 sdio->boundary = 0;
diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c
index 5602450f03f6..8a9d63a0c071 100644
--- a/fs/ext4/page-io.c
+++ b/fs/ext4/page-io.c
@@ -61,7 +61,6 @@ static void buffer_io_error(struct buffer_head *bh)
61static void ext4_finish_bio(struct bio *bio) 61static void ext4_finish_bio(struct bio *bio)
62{ 62{
63 int i; 63 int i;
64 int error = !test_bit(BIO_UPTODATE, &bio->bi_flags);
65 struct bio_vec *bvec; 64 struct bio_vec *bvec;
66 65
67 bio_for_each_segment_all(bvec, bio, i) { 66 bio_for_each_segment_all(bvec, bio, i) {
@@ -88,7 +87,7 @@ static void ext4_finish_bio(struct bio *bio)
88 } 87 }
89#endif 88#endif
90 89
91 if (error) { 90 if (bio->bi_error) {
92 SetPageError(page); 91 SetPageError(page);
93 set_bit(AS_EIO, &page->mapping->flags); 92 set_bit(AS_EIO, &page->mapping->flags);
94 } 93 }
@@ -107,7 +106,7 @@ static void ext4_finish_bio(struct bio *bio)
107 continue; 106 continue;
108 } 107 }
109 clear_buffer_async_write(bh); 108 clear_buffer_async_write(bh);
110 if (error) 109 if (bio->bi_error)
111 buffer_io_error(bh); 110 buffer_io_error(bh);
112 } while ((bh = bh->b_this_page) != head); 111 } while ((bh = bh->b_this_page) != head);
113 bit_spin_unlock(BH_Uptodate_Lock, &head->b_state); 112 bit_spin_unlock(BH_Uptodate_Lock, &head->b_state);
@@ -310,27 +309,25 @@ ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end)
310} 309}
311 310
312/* BIO completion function for page writeback */ 311/* BIO completion function for page writeback */
313static void ext4_end_bio(struct bio *bio, int error) 312static void ext4_end_bio(struct bio *bio)
314{ 313{
315 ext4_io_end_t *io_end = bio->bi_private; 314 ext4_io_end_t *io_end = bio->bi_private;
316 sector_t bi_sector = bio->bi_iter.bi_sector; 315 sector_t bi_sector = bio->bi_iter.bi_sector;
317 316
318 BUG_ON(!io_end); 317 BUG_ON(!io_end);
319 bio->bi_end_io = NULL; 318 bio->bi_end_io = NULL;
320 if (test_bit(BIO_UPTODATE, &bio->bi_flags))
321 error = 0;
322 319
323 if (error) { 320 if (bio->bi_error) {
324 struct inode *inode = io_end->inode; 321 struct inode *inode = io_end->inode;
325 322
326 ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu " 323 ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu "
327 "(offset %llu size %ld starting block %llu)", 324 "(offset %llu size %ld starting block %llu)",
328 error, inode->i_ino, 325 bio->bi_error, inode->i_ino,
329 (unsigned long long) io_end->offset, 326 (unsigned long long) io_end->offset,
330 (long) io_end->size, 327 (long) io_end->size,
331 (unsigned long long) 328 (unsigned long long)
332 bi_sector >> (inode->i_blkbits - 9)); 329 bi_sector >> (inode->i_blkbits - 9));
333 mapping_set_error(inode->i_mapping, error); 330 mapping_set_error(inode->i_mapping, bio->bi_error);
334 } 331 }
335 332
336 if (io_end->flag & EXT4_IO_END_UNWRITTEN) { 333 if (io_end->flag & EXT4_IO_END_UNWRITTEN) {
@@ -375,10 +372,9 @@ void ext4_io_submit_init(struct ext4_io_submit *io,
375static int io_submit_init_bio(struct ext4_io_submit *io, 372static int io_submit_init_bio(struct ext4_io_submit *io,
376 struct buffer_head *bh) 373 struct buffer_head *bh)
377{ 374{
378 int nvecs = bio_get_nr_vecs(bh->b_bdev);
379 struct bio *bio; 375 struct bio *bio;
380 376
381 bio = bio_alloc(GFP_NOIO, min(nvecs, BIO_MAX_PAGES)); 377 bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES);
382 if (!bio) 378 if (!bio)
383 return -ENOMEM; 379 return -ENOMEM;
384 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); 380 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c
index ec3ef93a52db..e26803fb210d 100644
--- a/fs/ext4/readpage.c
+++ b/fs/ext4/readpage.c
@@ -98,7 +98,7 @@ static inline bool ext4_bio_encrypted(struct bio *bio)
98 * status of that page is hard. See end_buffer_async_read() for the details. 98 * status of that page is hard. See end_buffer_async_read() for the details.
99 * There is no point in duplicating all that complexity. 99 * There is no point in duplicating all that complexity.
100 */ 100 */
101static void mpage_end_io(struct bio *bio, int err) 101static void mpage_end_io(struct bio *bio)
102{ 102{
103 struct bio_vec *bv; 103 struct bio_vec *bv;
104 int i; 104 int i;
@@ -106,7 +106,7 @@ static void mpage_end_io(struct bio *bio, int err)
106 if (ext4_bio_encrypted(bio)) { 106 if (ext4_bio_encrypted(bio)) {
107 struct ext4_crypto_ctx *ctx = bio->bi_private; 107 struct ext4_crypto_ctx *ctx = bio->bi_private;
108 108
109 if (err) { 109 if (bio->bi_error) {
110 ext4_release_crypto_ctx(ctx); 110 ext4_release_crypto_ctx(ctx);
111 } else { 111 } else {
112 INIT_WORK(&ctx->r.work, completion_pages); 112 INIT_WORK(&ctx->r.work, completion_pages);
@@ -118,7 +118,7 @@ static void mpage_end_io(struct bio *bio, int err)
118 bio_for_each_segment_all(bv, bio, i) { 118 bio_for_each_segment_all(bv, bio, i) {
119 struct page *page = bv->bv_page; 119 struct page *page = bv->bv_page;
120 120
121 if (!err) { 121 if (!bio->bi_error) {
122 SetPageUptodate(page); 122 SetPageUptodate(page);
123 } else { 123 } else {
124 ClearPageUptodate(page); 124 ClearPageUptodate(page);
@@ -284,7 +284,7 @@ int ext4_mpage_readpages(struct address_space *mapping,
284 goto set_error_page; 284 goto set_error_page;
285 } 285 }
286 bio = bio_alloc(GFP_KERNEL, 286 bio = bio_alloc(GFP_KERNEL,
287 min_t(int, nr_pages, bio_get_nr_vecs(bdev))); 287 min_t(int, nr_pages, BIO_MAX_PAGES));
288 if (!bio) { 288 if (!bio) {
289 if (ctx) 289 if (ctx)
290 ext4_release_crypto_ctx(ctx); 290 ext4_release_crypto_ctx(ctx);
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index f71e19a9dd3c..c414d49aa2de 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -29,13 +29,13 @@
29static struct kmem_cache *extent_tree_slab; 29static struct kmem_cache *extent_tree_slab;
30static struct kmem_cache *extent_node_slab; 30static struct kmem_cache *extent_node_slab;
31 31
32static void f2fs_read_end_io(struct bio *bio, int err) 32static void f2fs_read_end_io(struct bio *bio)
33{ 33{
34 struct bio_vec *bvec; 34 struct bio_vec *bvec;
35 int i; 35 int i;
36 36
37 if (f2fs_bio_encrypted(bio)) { 37 if (f2fs_bio_encrypted(bio)) {
38 if (err) { 38 if (bio->bi_error) {
39 f2fs_release_crypto_ctx(bio->bi_private); 39 f2fs_release_crypto_ctx(bio->bi_private);
40 } else { 40 } else {
41 f2fs_end_io_crypto_work(bio->bi_private, bio); 41 f2fs_end_io_crypto_work(bio->bi_private, bio);
@@ -46,7 +46,7 @@ static void f2fs_read_end_io(struct bio *bio, int err)
46 bio_for_each_segment_all(bvec, bio, i) { 46 bio_for_each_segment_all(bvec, bio, i) {
47 struct page *page = bvec->bv_page; 47 struct page *page = bvec->bv_page;
48 48
49 if (!err) { 49 if (!bio->bi_error) {
50 SetPageUptodate(page); 50 SetPageUptodate(page);
51 } else { 51 } else {
52 ClearPageUptodate(page); 52 ClearPageUptodate(page);
@@ -57,7 +57,7 @@ static void f2fs_read_end_io(struct bio *bio, int err)
57 bio_put(bio); 57 bio_put(bio);
58} 58}
59 59
60static void f2fs_write_end_io(struct bio *bio, int err) 60static void f2fs_write_end_io(struct bio *bio)
61{ 61{
62 struct f2fs_sb_info *sbi = bio->bi_private; 62 struct f2fs_sb_info *sbi = bio->bi_private;
63 struct bio_vec *bvec; 63 struct bio_vec *bvec;
@@ -68,7 +68,7 @@ static void f2fs_write_end_io(struct bio *bio, int err)
68 68
69 f2fs_restore_and_release_control_page(&page); 69 f2fs_restore_and_release_control_page(&page);
70 70
71 if (unlikely(err)) { 71 if (unlikely(bio->bi_error)) {
72 set_page_dirty(page); 72 set_page_dirty(page);
73 set_bit(AS_EIO, &page->mapping->flags); 73 set_bit(AS_EIO, &page->mapping->flags);
74 f2fs_stop_checkpoint(sbi); 74 f2fs_stop_checkpoint(sbi);
@@ -1552,7 +1552,7 @@ submit_and_realloc:
1552 } 1552 }
1553 1553
1554 bio = bio_alloc(GFP_KERNEL, 1554 bio = bio_alloc(GFP_KERNEL,
1555 min_t(int, nr_pages, bio_get_nr_vecs(bdev))); 1555 min_t(int, nr_pages, BIO_MAX_PAGES));
1556 if (!bio) { 1556 if (!bio) {
1557 if (ctx) 1557 if (ctx)
1558 f2fs_release_crypto_ctx(ctx); 1558 f2fs_release_crypto_ctx(ctx);
diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c
index 2c1ae861dc94..92324ac58290 100644
--- a/fs/gfs2/lops.c
+++ b/fs/gfs2/lops.c
@@ -202,22 +202,22 @@ static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp, struct bio_vec *bvec,
202 * 202 *
203 */ 203 */
204 204
205static void gfs2_end_log_write(struct bio *bio, int error) 205static void gfs2_end_log_write(struct bio *bio)
206{ 206{
207 struct gfs2_sbd *sdp = bio->bi_private; 207 struct gfs2_sbd *sdp = bio->bi_private;
208 struct bio_vec *bvec; 208 struct bio_vec *bvec;
209 struct page *page; 209 struct page *page;
210 int i; 210 int i;
211 211
212 if (error) { 212 if (bio->bi_error) {
213 sdp->sd_log_error = error; 213 sdp->sd_log_error = bio->bi_error;
214 fs_err(sdp, "Error %d writing to log\n", error); 214 fs_err(sdp, "Error %d writing to log\n", bio->bi_error);
215 } 215 }
216 216
217 bio_for_each_segment_all(bvec, bio, i) { 217 bio_for_each_segment_all(bvec, bio, i) {
218 page = bvec->bv_page; 218 page = bvec->bv_page;
219 if (page_has_buffers(page)) 219 if (page_has_buffers(page))
220 gfs2_end_log_write_bh(sdp, bvec, error); 220 gfs2_end_log_write_bh(sdp, bvec, bio->bi_error);
221 else 221 else
222 mempool_free(page, gfs2_page_pool); 222 mempool_free(page, gfs2_page_pool);
223 } 223 }
@@ -261,18 +261,11 @@ void gfs2_log_flush_bio(struct gfs2_sbd *sdp, int rw)
261static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno) 261static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno)
262{ 262{
263 struct super_block *sb = sdp->sd_vfs; 263 struct super_block *sb = sdp->sd_vfs;
264 unsigned nrvecs = bio_get_nr_vecs(sb->s_bdev);
265 struct bio *bio; 264 struct bio *bio;
266 265
267 BUG_ON(sdp->sd_log_bio); 266 BUG_ON(sdp->sd_log_bio);
268 267
269 while (1) { 268 bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES);
270 bio = bio_alloc(GFP_NOIO, nrvecs);
271 if (likely(bio))
272 break;
273 nrvecs = max(nrvecs/2, 1U);
274 }
275
276 bio->bi_iter.bi_sector = blkno * (sb->s_blocksize >> 9); 269 bio->bi_iter.bi_sector = blkno * (sb->s_blocksize >> 9);
277 bio->bi_bdev = sb->s_bdev; 270 bio->bi_bdev = sb->s_bdev;
278 bio->bi_end_io = gfs2_end_log_write; 271 bio->bi_end_io = gfs2_end_log_write;
diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c
index 1e3a93f2f71d..02586e7eb964 100644
--- a/fs/gfs2/ops_fstype.c
+++ b/fs/gfs2/ops_fstype.c
@@ -171,14 +171,14 @@ static int gfs2_check_sb(struct gfs2_sbd *sdp, int silent)
171 return -EINVAL; 171 return -EINVAL;
172} 172}
173 173
174static void end_bio_io_page(struct bio *bio, int error) 174static void end_bio_io_page(struct bio *bio)
175{ 175{
176 struct page *page = bio->bi_private; 176 struct page *page = bio->bi_private;
177 177
178 if (!error) 178 if (!bio->bi_error)
179 SetPageUptodate(page); 179 SetPageUptodate(page);
180 else 180 else
181 pr_warn("error %d reading superblock\n", error); 181 pr_warn("error %d reading superblock\n", bio->bi_error);
182 unlock_page(page); 182 unlock_page(page);
183} 183}
184 184
diff --git a/fs/jfs/jfs_logmgr.c b/fs/jfs/jfs_logmgr.c
index bc462dcd7a40..a69bdf2a1085 100644
--- a/fs/jfs/jfs_logmgr.c
+++ b/fs/jfs/jfs_logmgr.c
@@ -1999,19 +1999,16 @@ static int lbmRead(struct jfs_log * log, int pn, struct lbuf ** bpp)
1999 1999
2000 bio->bi_iter.bi_sector = bp->l_blkno << (log->l2bsize - 9); 2000 bio->bi_iter.bi_sector = bp->l_blkno << (log->l2bsize - 9);
2001 bio->bi_bdev = log->bdev; 2001 bio->bi_bdev = log->bdev;
2002 bio->bi_io_vec[0].bv_page = bp->l_page;
2003 bio->bi_io_vec[0].bv_len = LOGPSIZE;
2004 bio->bi_io_vec[0].bv_offset = bp->l_offset;
2005 2002
2006 bio->bi_vcnt = 1; 2003 bio_add_page(bio, bp->l_page, LOGPSIZE, bp->l_offset);
2007 bio->bi_iter.bi_size = LOGPSIZE; 2004 BUG_ON(bio->bi_iter.bi_size != LOGPSIZE);
2008 2005
2009 bio->bi_end_io = lbmIODone; 2006 bio->bi_end_io = lbmIODone;
2010 bio->bi_private = bp; 2007 bio->bi_private = bp;
2011 /*check if journaling to disk has been disabled*/ 2008 /*check if journaling to disk has been disabled*/
2012 if (log->no_integrity) { 2009 if (log->no_integrity) {
2013 bio->bi_iter.bi_size = 0; 2010 bio->bi_iter.bi_size = 0;
2014 lbmIODone(bio, 0); 2011 lbmIODone(bio);
2015 } else { 2012 } else {
2016 submit_bio(READ_SYNC, bio); 2013 submit_bio(READ_SYNC, bio);
2017 } 2014 }
@@ -2145,12 +2142,9 @@ static void lbmStartIO(struct lbuf * bp)
2145 bio = bio_alloc(GFP_NOFS, 1); 2142 bio = bio_alloc(GFP_NOFS, 1);
2146 bio->bi_iter.bi_sector = bp->l_blkno << (log->l2bsize - 9); 2143 bio->bi_iter.bi_sector = bp->l_blkno << (log->l2bsize - 9);
2147 bio->bi_bdev = log->bdev; 2144 bio->bi_bdev = log->bdev;
2148 bio->bi_io_vec[0].bv_page = bp->l_page;
2149 bio->bi_io_vec[0].bv_len = LOGPSIZE;
2150 bio->bi_io_vec[0].bv_offset = bp->l_offset;
2151 2145
2152 bio->bi_vcnt = 1; 2146 bio_add_page(bio, bp->l_page, LOGPSIZE, bp->l_offset);
2153 bio->bi_iter.bi_size = LOGPSIZE; 2147 BUG_ON(bio->bi_iter.bi_size != LOGPSIZE);
2154 2148
2155 bio->bi_end_io = lbmIODone; 2149 bio->bi_end_io = lbmIODone;
2156 bio->bi_private = bp; 2150 bio->bi_private = bp;
@@ -2158,7 +2152,7 @@ static void lbmStartIO(struct lbuf * bp)
2158 /* check if journaling to disk has been disabled */ 2152 /* check if journaling to disk has been disabled */
2159 if (log->no_integrity) { 2153 if (log->no_integrity) {
2160 bio->bi_iter.bi_size = 0; 2154 bio->bi_iter.bi_size = 0;
2161 lbmIODone(bio, 0); 2155 lbmIODone(bio);
2162 } else { 2156 } else {
2163 submit_bio(WRITE_SYNC, bio); 2157 submit_bio(WRITE_SYNC, bio);
2164 INCREMENT(lmStat.submitted); 2158 INCREMENT(lmStat.submitted);
@@ -2196,7 +2190,7 @@ static int lbmIOWait(struct lbuf * bp, int flag)
2196 * 2190 *
2197 * executed at INTIODONE level 2191 * executed at INTIODONE level
2198 */ 2192 */
2199static void lbmIODone(struct bio *bio, int error) 2193static void lbmIODone(struct bio *bio)
2200{ 2194{
2201 struct lbuf *bp = bio->bi_private; 2195 struct lbuf *bp = bio->bi_private;
2202 struct lbuf *nextbp, *tail; 2196 struct lbuf *nextbp, *tail;
@@ -2212,7 +2206,7 @@ static void lbmIODone(struct bio *bio, int error)
2212 2206
2213 bp->l_flag |= lbmDONE; 2207 bp->l_flag |= lbmDONE;
2214 2208
2215 if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) { 2209 if (bio->bi_error) {
2216 bp->l_flag |= lbmERROR; 2210 bp->l_flag |= lbmERROR;
2217 2211
2218 jfs_err("lbmIODone: I/O error in JFS log"); 2212 jfs_err("lbmIODone: I/O error in JFS log");
diff --git a/fs/jfs/jfs_metapage.c b/fs/jfs/jfs_metapage.c
index 16a0922beb59..a3eb316b1ac3 100644
--- a/fs/jfs/jfs_metapage.c
+++ b/fs/jfs/jfs_metapage.c
@@ -276,11 +276,11 @@ static void last_read_complete(struct page *page)
276 unlock_page(page); 276 unlock_page(page);
277} 277}
278 278
279static void metapage_read_end_io(struct bio *bio, int err) 279static void metapage_read_end_io(struct bio *bio)
280{ 280{
281 struct page *page = bio->bi_private; 281 struct page *page = bio->bi_private;
282 282
283 if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) { 283 if (bio->bi_error) {
284 printk(KERN_ERR "metapage_read_end_io: I/O error\n"); 284 printk(KERN_ERR "metapage_read_end_io: I/O error\n");
285 SetPageError(page); 285 SetPageError(page);
286 } 286 }
@@ -331,13 +331,13 @@ static void last_write_complete(struct page *page)
331 end_page_writeback(page); 331 end_page_writeback(page);
332} 332}
333 333
334static void metapage_write_end_io(struct bio *bio, int err) 334static void metapage_write_end_io(struct bio *bio)
335{ 335{
336 struct page *page = bio->bi_private; 336 struct page *page = bio->bi_private;
337 337
338 BUG_ON(!PagePrivate(page)); 338 BUG_ON(!PagePrivate(page));
339 339
340 if (! test_bit(BIO_UPTODATE, &bio->bi_flags)) { 340 if (bio->bi_error) {
341 printk(KERN_ERR "metapage_write_end_io: I/O error\n"); 341 printk(KERN_ERR "metapage_write_end_io: I/O error\n");
342 SetPageError(page); 342 SetPageError(page);
343 } 343 }
diff --git a/fs/logfs/dev_bdev.c b/fs/logfs/dev_bdev.c
index 76279e11982d..a7fdbd868474 100644
--- a/fs/logfs/dev_bdev.c
+++ b/fs/logfs/dev_bdev.c
@@ -53,16 +53,14 @@ static int bdev_readpage(void *_sb, struct page *page)
53 53
54static DECLARE_WAIT_QUEUE_HEAD(wq); 54static DECLARE_WAIT_QUEUE_HEAD(wq);
55 55
56static void writeseg_end_io(struct bio *bio, int err) 56static void writeseg_end_io(struct bio *bio)
57{ 57{
58 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
59 struct bio_vec *bvec; 58 struct bio_vec *bvec;
60 int i; 59 int i;
61 struct super_block *sb = bio->bi_private; 60 struct super_block *sb = bio->bi_private;
62 struct logfs_super *super = logfs_super(sb); 61 struct logfs_super *super = logfs_super(sb);
63 62
64 BUG_ON(!uptodate); /* FIXME: Retry io or write elsewhere */ 63 BUG_ON(bio->bi_error); /* FIXME: Retry io or write elsewhere */
65 BUG_ON(err);
66 64
67 bio_for_each_segment_all(bvec, bio, i) { 65 bio_for_each_segment_all(bvec, bio, i) {
68 end_page_writeback(bvec->bv_page); 66 end_page_writeback(bvec->bv_page);
@@ -83,7 +81,7 @@ static int __bdev_writeseg(struct super_block *sb, u64 ofs, pgoff_t index,
83 unsigned int max_pages; 81 unsigned int max_pages;
84 int i; 82 int i;
85 83
86 max_pages = min(nr_pages, (size_t) bio_get_nr_vecs(super->s_bdev)); 84 max_pages = min(nr_pages, BIO_MAX_PAGES);
87 85
88 bio = bio_alloc(GFP_NOFS, max_pages); 86 bio = bio_alloc(GFP_NOFS, max_pages);
89 BUG_ON(!bio); 87 BUG_ON(!bio);
@@ -153,14 +151,12 @@ static void bdev_writeseg(struct super_block *sb, u64 ofs, size_t len)
153} 151}
154 152
155 153
156static void erase_end_io(struct bio *bio, int err) 154static void erase_end_io(struct bio *bio)
157{ 155{
158 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
159 struct super_block *sb = bio->bi_private; 156 struct super_block *sb = bio->bi_private;
160 struct logfs_super *super = logfs_super(sb); 157 struct logfs_super *super = logfs_super(sb);
161 158
162 BUG_ON(!uptodate); /* FIXME: Retry io or write elsewhere */ 159 BUG_ON(bio->bi_error); /* FIXME: Retry io or write elsewhere */
163 BUG_ON(err);
164 BUG_ON(bio->bi_vcnt == 0); 160 BUG_ON(bio->bi_vcnt == 0);
165 bio_put(bio); 161 bio_put(bio);
166 if (atomic_dec_and_test(&super->s_pending_writes)) 162 if (atomic_dec_and_test(&super->s_pending_writes))
@@ -175,7 +171,7 @@ static int do_erase(struct super_block *sb, u64 ofs, pgoff_t index,
175 unsigned int max_pages; 171 unsigned int max_pages;
176 int i; 172 int i;
177 173
178 max_pages = min(nr_pages, (size_t) bio_get_nr_vecs(super->s_bdev)); 174 max_pages = min(nr_pages, BIO_MAX_PAGES);
179 175
180 bio = bio_alloc(GFP_NOFS, max_pages); 176 bio = bio_alloc(GFP_NOFS, max_pages);
181 BUG_ON(!bio); 177 BUG_ON(!bio);
diff --git a/fs/mpage.c b/fs/mpage.c
index ca0244b69de8..778a4ddef77a 100644
--- a/fs/mpage.c
+++ b/fs/mpage.c
@@ -42,14 +42,14 @@
42 * status of that page is hard. See end_buffer_async_read() for the details. 42 * status of that page is hard. See end_buffer_async_read() for the details.
43 * There is no point in duplicating all that complexity. 43 * There is no point in duplicating all that complexity.
44 */ 44 */
45static void mpage_end_io(struct bio *bio, int err) 45static void mpage_end_io(struct bio *bio)
46{ 46{
47 struct bio_vec *bv; 47 struct bio_vec *bv;
48 int i; 48 int i;
49 49
50 bio_for_each_segment_all(bv, bio, i) { 50 bio_for_each_segment_all(bv, bio, i) {
51 struct page *page = bv->bv_page; 51 struct page *page = bv->bv_page;
52 page_endio(page, bio_data_dir(bio), err); 52 page_endio(page, bio_data_dir(bio), bio->bi_error);
53 } 53 }
54 54
55 bio_put(bio); 55 bio_put(bio);
@@ -277,7 +277,7 @@ alloc_new:
277 goto out; 277 goto out;
278 } 278 }
279 bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9), 279 bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
280 min_t(int, nr_pages, bio_get_nr_vecs(bdev)), 280 min_t(int, nr_pages, BIO_MAX_PAGES),
281 GFP_KERNEL); 281 GFP_KERNEL);
282 if (bio == NULL) 282 if (bio == NULL)
283 goto confused; 283 goto confused;
@@ -602,7 +602,7 @@ alloc_new:
602 } 602 }
603 } 603 }
604 bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9), 604 bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
605 bio_get_nr_vecs(bdev), GFP_NOFS|__GFP_HIGH); 605 BIO_MAX_PAGES, GFP_NOFS|__GFP_HIGH);
606 if (bio == NULL) 606 if (bio == NULL)
607 goto confused; 607 goto confused;
608 608
diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c
index d2554fe140a3..9cd4eb3a1e22 100644
--- a/fs/nfs/blocklayout/blocklayout.c
+++ b/fs/nfs/blocklayout/blocklayout.c
@@ -116,7 +116,7 @@ bl_submit_bio(int rw, struct bio *bio)
116 116
117static struct bio * 117static struct bio *
118bl_alloc_init_bio(int npg, struct block_device *bdev, sector_t disk_sector, 118bl_alloc_init_bio(int npg, struct block_device *bdev, sector_t disk_sector,
119 void (*end_io)(struct bio *, int err), struct parallel_io *par) 119 bio_end_io_t end_io, struct parallel_io *par)
120{ 120{
121 struct bio *bio; 121 struct bio *bio;
122 122
@@ -139,8 +139,7 @@ bl_alloc_init_bio(int npg, struct block_device *bdev, sector_t disk_sector,
139static struct bio * 139static struct bio *
140do_add_page_to_bio(struct bio *bio, int npg, int rw, sector_t isect, 140do_add_page_to_bio(struct bio *bio, int npg, int rw, sector_t isect,
141 struct page *page, struct pnfs_block_dev_map *map, 141 struct page *page, struct pnfs_block_dev_map *map,
142 struct pnfs_block_extent *be, 142 struct pnfs_block_extent *be, bio_end_io_t end_io,
143 void (*end_io)(struct bio *, int err),
144 struct parallel_io *par, unsigned int offset, int *len) 143 struct parallel_io *par, unsigned int offset, int *len)
145{ 144{
146 struct pnfs_block_dev *dev = 145 struct pnfs_block_dev *dev =
@@ -183,11 +182,11 @@ retry:
183 return bio; 182 return bio;
184} 183}
185 184
186static void bl_end_io_read(struct bio *bio, int err) 185static void bl_end_io_read(struct bio *bio)
187{ 186{
188 struct parallel_io *par = bio->bi_private; 187 struct parallel_io *par = bio->bi_private;
189 188
190 if (err) { 189 if (bio->bi_error) {
191 struct nfs_pgio_header *header = par->data; 190 struct nfs_pgio_header *header = par->data;
192 191
193 if (!header->pnfs_error) 192 if (!header->pnfs_error)
@@ -316,13 +315,12 @@ out:
316 return PNFS_ATTEMPTED; 315 return PNFS_ATTEMPTED;
317} 316}
318 317
319static void bl_end_io_write(struct bio *bio, int err) 318static void bl_end_io_write(struct bio *bio)
320{ 319{
321 struct parallel_io *par = bio->bi_private; 320 struct parallel_io *par = bio->bi_private;
322 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
323 struct nfs_pgio_header *header = par->data; 321 struct nfs_pgio_header *header = par->data;
324 322
325 if (!uptodate) { 323 if (bio->bi_error) {
326 if (!header->pnfs_error) 324 if (!header->pnfs_error)
327 header->pnfs_error = -EIO; 325 header->pnfs_error = -EIO;
328 pnfs_set_lo_fail(header->lseg); 326 pnfs_set_lo_fail(header->lseg);
diff --git a/fs/nilfs2/segbuf.c b/fs/nilfs2/segbuf.c
index 42468e5ab3e7..f63620ce3892 100644
--- a/fs/nilfs2/segbuf.c
+++ b/fs/nilfs2/segbuf.c
@@ -338,12 +338,11 @@ void nilfs_add_checksums_on_logs(struct list_head *logs, u32 seed)
338/* 338/*
339 * BIO operations 339 * BIO operations
340 */ 340 */
341static void nilfs_end_bio_write(struct bio *bio, int err) 341static void nilfs_end_bio_write(struct bio *bio)
342{ 342{
343 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
344 struct nilfs_segment_buffer *segbuf = bio->bi_private; 343 struct nilfs_segment_buffer *segbuf = bio->bi_private;
345 344
346 if (!uptodate) 345 if (bio->bi_error)
347 atomic_inc(&segbuf->sb_err); 346 atomic_inc(&segbuf->sb_err);
348 347
349 bio_put(bio); 348 bio_put(bio);
@@ -415,7 +414,7 @@ static void nilfs_segbuf_prepare_write(struct nilfs_segment_buffer *segbuf,
415{ 414{
416 wi->bio = NULL; 415 wi->bio = NULL;
417 wi->rest_blocks = segbuf->sb_sum.nblocks; 416 wi->rest_blocks = segbuf->sb_sum.nblocks;
418 wi->max_pages = bio_get_nr_vecs(wi->nilfs->ns_bdev); 417 wi->max_pages = BIO_MAX_PAGES;
419 wi->nr_vecs = min(wi->max_pages, wi->rest_blocks); 418 wi->nr_vecs = min(wi->max_pages, wi->rest_blocks);
420 wi->start = wi->end = 0; 419 wi->start = wi->end = 0;
421 wi->blocknr = segbuf->sb_pseg_start; 420 wi->blocknr = segbuf->sb_pseg_start;
diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c
index 16eff45727ee..140de3c93d2e 100644
--- a/fs/ocfs2/cluster/heartbeat.c
+++ b/fs/ocfs2/cluster/heartbeat.c
@@ -372,14 +372,13 @@ static void o2hb_wait_on_io(struct o2hb_region *reg,
372 wait_for_completion(&wc->wc_io_complete); 372 wait_for_completion(&wc->wc_io_complete);
373} 373}
374 374
375static void o2hb_bio_end_io(struct bio *bio, 375static void o2hb_bio_end_io(struct bio *bio)
376 int error)
377{ 376{
378 struct o2hb_bio_wait_ctxt *wc = bio->bi_private; 377 struct o2hb_bio_wait_ctxt *wc = bio->bi_private;
379 378
380 if (error) { 379 if (bio->bi_error) {
381 mlog(ML_ERROR, "IO Error %d\n", error); 380 mlog(ML_ERROR, "IO Error %d\n", bio->bi_error);
382 wc->wc_error = error; 381 wc->wc_error = bio->bi_error;
383 } 382 }
384 383
385 o2hb_bio_wait_dec(wc, 1); 384 o2hb_bio_wait_dec(wc, 1);
diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c
index 3859f5e27a4d..c77499bcbd7a 100644
--- a/fs/xfs/xfs_aops.c
+++ b/fs/xfs/xfs_aops.c
@@ -351,12 +351,11 @@ xfs_imap_valid(
351 */ 351 */
352STATIC void 352STATIC void
353xfs_end_bio( 353xfs_end_bio(
354 struct bio *bio, 354 struct bio *bio)
355 int error)
356{ 355{
357 xfs_ioend_t *ioend = bio->bi_private; 356 xfs_ioend_t *ioend = bio->bi_private;
358 357
359 ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error; 358 ioend->io_error = bio->bi_error;
360 359
361 /* Toss bio and pass work off to an xfsdatad thread */ 360 /* Toss bio and pass work off to an xfsdatad thread */
362 bio->bi_private = NULL; 361 bio->bi_private = NULL;
@@ -382,8 +381,7 @@ STATIC struct bio *
382xfs_alloc_ioend_bio( 381xfs_alloc_ioend_bio(
383 struct buffer_head *bh) 382 struct buffer_head *bh)
384{ 383{
385 int nvecs = bio_get_nr_vecs(bh->b_bdev); 384 struct bio *bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES);
386 struct bio *bio = bio_alloc(GFP_NOIO, nvecs);
387 385
388 ASSERT(bio->bi_private == NULL); 386 ASSERT(bio->bi_private == NULL);
389 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); 387 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c
index a4b7d92e946c..01bd6781974e 100644
--- a/fs/xfs/xfs_buf.c
+++ b/fs/xfs/xfs_buf.c
@@ -1096,8 +1096,7 @@ xfs_bwrite(
1096 1096
1097STATIC void 1097STATIC void
1098xfs_buf_bio_end_io( 1098xfs_buf_bio_end_io(
1099 struct bio *bio, 1099 struct bio *bio)
1100 int error)
1101{ 1100{
1102 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private; 1101 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
1103 1102
@@ -1105,10 +1104,10 @@ xfs_buf_bio_end_io(
1105 * don't overwrite existing errors - otherwise we can lose errors on 1104 * don't overwrite existing errors - otherwise we can lose errors on
1106 * buffers that require multiple bios to complete. 1105 * buffers that require multiple bios to complete.
1107 */ 1106 */
1108 if (error) { 1107 if (bio->bi_error) {
1109 spin_lock(&bp->b_lock); 1108 spin_lock(&bp->b_lock);
1110 if (!bp->b_io_error) 1109 if (!bp->b_io_error)
1111 bp->b_io_error = error; 1110 bp->b_io_error = bio->bi_error;
1112 spin_unlock(&bp->b_lock); 1111 spin_unlock(&bp->b_lock);
1113 } 1112 }
1114 1113
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 5e963a6d7c14..b9b6e046b52e 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -187,17 +187,6 @@ static inline void *bio_data(struct bio *bio)
187 __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q))) 187 __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q)))
188 188
189/* 189/*
190 * Check if adding a bio_vec after bprv with offset would create a gap in
191 * the SG list. Most drivers don't care about this, but some do.
192 */
193static inline bool bvec_gap_to_prev(struct bio_vec *bprv, unsigned int offset)
194{
195 return offset || ((bprv->bv_offset + bprv->bv_len) & (PAGE_SIZE - 1));
196}
197
198#define bio_io_error(bio) bio_endio((bio), -EIO)
199
200/*
201 * drivers should _never_ use the all version - the bio may have been split 190 * drivers should _never_ use the all version - the bio may have been split
202 * before it got to the driver and the driver won't own all of it 191 * before it got to the driver and the driver won't own all of it
203 */ 192 */
@@ -306,6 +295,21 @@ static inline void bio_cnt_set(struct bio *bio, unsigned int count)
306 atomic_set(&bio->__bi_cnt, count); 295 atomic_set(&bio->__bi_cnt, count);
307} 296}
308 297
298static inline bool bio_flagged(struct bio *bio, unsigned int bit)
299{
300 return (bio->bi_flags & (1U << bit)) != 0;
301}
302
303static inline void bio_set_flag(struct bio *bio, unsigned int bit)
304{
305 bio->bi_flags |= (1U << bit);
306}
307
308static inline void bio_clear_flag(struct bio *bio, unsigned int bit)
309{
310 bio->bi_flags &= ~(1U << bit);
311}
312
309enum bip_flags { 313enum bip_flags {
310 BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */ 314 BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */
311 BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */ 315 BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */
@@ -426,7 +430,14 @@ static inline struct bio *bio_clone_kmalloc(struct bio *bio, gfp_t gfp_mask)
426 430
427} 431}
428 432
429extern void bio_endio(struct bio *, int); 433extern void bio_endio(struct bio *);
434
435static inline void bio_io_error(struct bio *bio)
436{
437 bio->bi_error = -EIO;
438 bio_endio(bio);
439}
440
430struct request_queue; 441struct request_queue;
431extern int bio_phys_segments(struct request_queue *, struct bio *); 442extern int bio_phys_segments(struct request_queue *, struct bio *);
432 443
@@ -440,7 +451,6 @@ void bio_chain(struct bio *, struct bio *);
440extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int); 451extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
441extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *, 452extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
442 unsigned int, unsigned int); 453 unsigned int, unsigned int);
443extern int bio_get_nr_vecs(struct block_device *);
444struct rq_map_data; 454struct rq_map_data;
445extern struct bio *bio_map_user_iov(struct request_queue *, 455extern struct bio *bio_map_user_iov(struct request_queue *,
446 const struct iov_iter *, gfp_t); 456 const struct iov_iter *, gfp_t);
@@ -717,7 +727,7 @@ extern void bio_integrity_free(struct bio *);
717extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int); 727extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
718extern bool bio_integrity_enabled(struct bio *bio); 728extern bool bio_integrity_enabled(struct bio *bio);
719extern int bio_integrity_prep(struct bio *); 729extern int bio_integrity_prep(struct bio *);
720extern void bio_integrity_endio(struct bio *, int); 730extern void bio_integrity_endio(struct bio *);
721extern void bio_integrity_advance(struct bio *, unsigned int); 731extern void bio_integrity_advance(struct bio *, unsigned int);
722extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int); 732extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int);
723extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t); 733extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h
index 1b62d768c7df..a4cd1641e9e2 100644
--- a/include/linux/blk-cgroup.h
+++ b/include/linux/blk-cgroup.h
@@ -374,7 +374,7 @@ static inline struct request_list *blk_get_rl(struct request_queue *q,
374 * root_rl in such cases. 374 * root_rl in such cases.
375 */ 375 */
376 blkg = blkg_lookup_create(blkcg, q); 376 blkg = blkg_lookup_create(blkcg, q);
377 if (unlikely(IS_ERR(blkg))) 377 if (IS_ERR(blkg))
378 goto root_rl; 378 goto root_rl;
379 379
380 blkg_get(blkg); 380 blkg_get(blkg);
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 7303b3405520..4b7b4ebaa633 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -14,7 +14,7 @@ struct page;
14struct block_device; 14struct block_device;
15struct io_context; 15struct io_context;
16struct cgroup_subsys_state; 16struct cgroup_subsys_state;
17typedef void (bio_end_io_t) (struct bio *, int); 17typedef void (bio_end_io_t) (struct bio *);
18typedef void (bio_destructor_t) (struct bio *); 18typedef void (bio_destructor_t) (struct bio *);
19 19
20/* 20/*
@@ -46,7 +46,8 @@ struct bvec_iter {
46struct bio { 46struct bio {
47 struct bio *bi_next; /* request queue link */ 47 struct bio *bi_next; /* request queue link */
48 struct block_device *bi_bdev; 48 struct block_device *bi_bdev;
49 unsigned long bi_flags; /* status, command, etc */ 49 unsigned int bi_flags; /* status, command, etc */
50 int bi_error;
50 unsigned long bi_rw; /* bottom bits READ/WRITE, 51 unsigned long bi_rw; /* bottom bits READ/WRITE,
51 * top bits priority 52 * top bits priority
52 */ 53 */
@@ -111,7 +112,6 @@ struct bio {
111/* 112/*
112 * bio flags 113 * bio flags
113 */ 114 */
114#define BIO_UPTODATE 0 /* ok after I/O completion */
115#define BIO_SEG_VALID 1 /* bi_phys_segments valid */ 115#define BIO_SEG_VALID 1 /* bi_phys_segments valid */
116#define BIO_CLONED 2 /* doesn't own data */ 116#define BIO_CLONED 2 /* doesn't own data */
117#define BIO_BOUNCED 3 /* bio is a bounce bio */ 117#define BIO_BOUNCED 3 /* bio is a bounce bio */
@@ -129,14 +129,12 @@ struct bio {
129#define BIO_RESET_BITS 13 129#define BIO_RESET_BITS 13
130#define BIO_OWNS_VEC 13 /* bio_free() should free bvec */ 130#define BIO_OWNS_VEC 13 /* bio_free() should free bvec */
131 131
132#define bio_flagged(bio, flag) ((bio)->bi_flags & (1 << (flag)))
133
134/* 132/*
135 * top 4 bits of bio flags indicate the pool this bio came from 133 * top 4 bits of bio flags indicate the pool this bio came from
136 */ 134 */
137#define BIO_POOL_BITS (4) 135#define BIO_POOL_BITS (4)
138#define BIO_POOL_NONE ((1UL << BIO_POOL_BITS) - 1) 136#define BIO_POOL_NONE ((1UL << BIO_POOL_BITS) - 1)
139#define BIO_POOL_OFFSET (BITS_PER_LONG - BIO_POOL_BITS) 137#define BIO_POOL_OFFSET (32 - BIO_POOL_BITS)
140#define BIO_POOL_MASK (1UL << BIO_POOL_OFFSET) 138#define BIO_POOL_MASK (1UL << BIO_POOL_OFFSET)
141#define BIO_POOL_IDX(bio) ((bio)->bi_flags >> BIO_POOL_OFFSET) 139#define BIO_POOL_IDX(bio) ((bio)->bi_flags >> BIO_POOL_OFFSET)
142 140
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index d4068c17d0df..a622f270f09e 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -213,14 +213,6 @@ typedef int (prep_rq_fn) (struct request_queue *, struct request *);
213typedef void (unprep_rq_fn) (struct request_queue *, struct request *); 213typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
214 214
215struct bio_vec; 215struct bio_vec;
216struct bvec_merge_data {
217 struct block_device *bi_bdev;
218 sector_t bi_sector;
219 unsigned bi_size;
220 unsigned long bi_rw;
221};
222typedef int (merge_bvec_fn) (struct request_queue *, struct bvec_merge_data *,
223 struct bio_vec *);
224typedef void (softirq_done_fn)(struct request *); 216typedef void (softirq_done_fn)(struct request *);
225typedef int (dma_drain_needed_fn)(struct request *); 217typedef int (dma_drain_needed_fn)(struct request *);
226typedef int (lld_busy_fn) (struct request_queue *q); 218typedef int (lld_busy_fn) (struct request_queue *q);
@@ -258,6 +250,7 @@ struct blk_queue_tag {
258struct queue_limits { 250struct queue_limits {
259 unsigned long bounce_pfn; 251 unsigned long bounce_pfn;
260 unsigned long seg_boundary_mask; 252 unsigned long seg_boundary_mask;
253 unsigned long virt_boundary_mask;
261 254
262 unsigned int max_hw_sectors; 255 unsigned int max_hw_sectors;
263 unsigned int chunk_sectors; 256 unsigned int chunk_sectors;
@@ -268,6 +261,7 @@ struct queue_limits {
268 unsigned int io_min; 261 unsigned int io_min;
269 unsigned int io_opt; 262 unsigned int io_opt;
270 unsigned int max_discard_sectors; 263 unsigned int max_discard_sectors;
264 unsigned int max_hw_discard_sectors;
271 unsigned int max_write_same_sectors; 265 unsigned int max_write_same_sectors;
272 unsigned int discard_granularity; 266 unsigned int discard_granularity;
273 unsigned int discard_alignment; 267 unsigned int discard_alignment;
@@ -305,7 +299,6 @@ struct request_queue {
305 make_request_fn *make_request_fn; 299 make_request_fn *make_request_fn;
306 prep_rq_fn *prep_rq_fn; 300 prep_rq_fn *prep_rq_fn;
307 unprep_rq_fn *unprep_rq_fn; 301 unprep_rq_fn *unprep_rq_fn;
308 merge_bvec_fn *merge_bvec_fn;
309 softirq_done_fn *softirq_done_fn; 302 softirq_done_fn *softirq_done_fn;
310 rq_timed_out_fn *rq_timed_out_fn; 303 rq_timed_out_fn *rq_timed_out_fn;
311 dma_drain_needed_fn *dma_drain_needed; 304 dma_drain_needed_fn *dma_drain_needed;
@@ -462,6 +455,7 @@ struct request_queue {
462 455
463 struct blk_mq_tag_set *tag_set; 456 struct blk_mq_tag_set *tag_set;
464 struct list_head tag_set_list; 457 struct list_head tag_set_list;
458 struct bio_set *bio_split;
465}; 459};
466 460
467#define QUEUE_FLAG_QUEUED 1 /* uses generic tag queueing */ 461#define QUEUE_FLAG_QUEUED 1 /* uses generic tag queueing */
@@ -486,7 +480,6 @@ struct request_queue {
486#define QUEUE_FLAG_DEAD 19 /* queue tear-down finished */ 480#define QUEUE_FLAG_DEAD 19 /* queue tear-down finished */
487#define QUEUE_FLAG_INIT_DONE 20 /* queue is initialized */ 481#define QUEUE_FLAG_INIT_DONE 20 /* queue is initialized */
488#define QUEUE_FLAG_NO_SG_MERGE 21 /* don't attempt to merge SG segments*/ 482#define QUEUE_FLAG_NO_SG_MERGE 21 /* don't attempt to merge SG segments*/
489#define QUEUE_FLAG_SG_GAPS 22 /* queue doesn't support SG gaps */
490 483
491#define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ 484#define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
492 (1 << QUEUE_FLAG_STACKABLE) | \ 485 (1 << QUEUE_FLAG_STACKABLE) | \
@@ -782,6 +775,8 @@ extern void blk_rq_unprep_clone(struct request *rq);
782extern int blk_insert_cloned_request(struct request_queue *q, 775extern int blk_insert_cloned_request(struct request_queue *q,
783 struct request *rq); 776 struct request *rq);
784extern void blk_delay_queue(struct request_queue *, unsigned long); 777extern void blk_delay_queue(struct request_queue *, unsigned long);
778extern void blk_queue_split(struct request_queue *, struct bio **,
779 struct bio_set *);
785extern void blk_recount_segments(struct request_queue *, struct bio *); 780extern void blk_recount_segments(struct request_queue *, struct bio *);
786extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int); 781extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
787extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t, 782extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
@@ -986,9 +981,9 @@ extern int blk_queue_dma_drain(struct request_queue *q,
986 void *buf, unsigned int size); 981 void *buf, unsigned int size);
987extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn); 982extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn);
988extern void blk_queue_segment_boundary(struct request_queue *, unsigned long); 983extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
984extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
989extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn); 985extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn);
990extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn); 986extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn);
991extern void blk_queue_merge_bvec(struct request_queue *, merge_bvec_fn *);
992extern void blk_queue_dma_alignment(struct request_queue *, int); 987extern void blk_queue_dma_alignment(struct request_queue *, int);
993extern void blk_queue_update_dma_alignment(struct request_queue *, int); 988extern void blk_queue_update_dma_alignment(struct request_queue *, int);
994extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *); 989extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *);
@@ -1138,6 +1133,7 @@ extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm);
1138enum blk_default_limits { 1133enum blk_default_limits {
1139 BLK_MAX_SEGMENTS = 128, 1134 BLK_MAX_SEGMENTS = 128,
1140 BLK_SAFE_MAX_SECTORS = 255, 1135 BLK_SAFE_MAX_SECTORS = 255,
1136 BLK_DEF_MAX_SECTORS = 2560,
1141 BLK_MAX_SEGMENT_SIZE = 65536, 1137 BLK_MAX_SEGMENT_SIZE = 65536,
1142 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, 1138 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL,
1143}; 1139};
@@ -1154,6 +1150,11 @@ static inline unsigned long queue_segment_boundary(struct request_queue *q)
1154 return q->limits.seg_boundary_mask; 1150 return q->limits.seg_boundary_mask;
1155} 1151}
1156 1152
1153static inline unsigned long queue_virt_boundary(struct request_queue *q)
1154{
1155 return q->limits.virt_boundary_mask;
1156}
1157
1157static inline unsigned int queue_max_sectors(struct request_queue *q) 1158static inline unsigned int queue_max_sectors(struct request_queue *q)
1158{ 1159{
1159 return q->limits.max_sectors; 1160 return q->limits.max_sectors;
@@ -1354,6 +1355,19 @@ static inline void put_dev_sector(Sector p)
1354 page_cache_release(p.v); 1355 page_cache_release(p.v);
1355} 1356}
1356 1357
1358/*
1359 * Check if adding a bio_vec after bprv with offset would create a gap in
1360 * the SG list. Most drivers don't care about this, but some do.
1361 */
1362static inline bool bvec_gap_to_prev(struct request_queue *q,
1363 struct bio_vec *bprv, unsigned int offset)
1364{
1365 if (!queue_virt_boundary(q))
1366 return false;
1367 return offset ||
1368 ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
1369}
1370
1357struct work_struct; 1371struct work_struct;
1358int kblockd_schedule_work(struct work_struct *work); 1372int kblockd_schedule_work(struct work_struct *work);
1359int kblockd_schedule_delayed_work(struct delayed_work *dwork, unsigned long delay); 1373int kblockd_schedule_delayed_work(struct delayed_work *dwork, unsigned long delay);
diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h
index 51cc1deb7af3..76d23fa8c7d3 100644
--- a/include/linux/device-mapper.h
+++ b/include/linux/device-mapper.h
@@ -82,9 +82,6 @@ typedef int (*dm_message_fn) (struct dm_target *ti, unsigned argc, char **argv);
82typedef int (*dm_ioctl_fn) (struct dm_target *ti, unsigned int cmd, 82typedef int (*dm_ioctl_fn) (struct dm_target *ti, unsigned int cmd,
83 unsigned long arg); 83 unsigned long arg);
84 84
85typedef int (*dm_merge_fn) (struct dm_target *ti, struct bvec_merge_data *bvm,
86 struct bio_vec *biovec, int max_size);
87
88/* 85/*
89 * These iteration functions are typically used to check (and combine) 86 * These iteration functions are typically used to check (and combine)
90 * properties of underlying devices. 87 * properties of underlying devices.
@@ -160,7 +157,6 @@ struct target_type {
160 dm_status_fn status; 157 dm_status_fn status;
161 dm_message_fn message; 158 dm_message_fn message;
162 dm_ioctl_fn ioctl; 159 dm_ioctl_fn ioctl;
163 dm_merge_fn merge;
164 dm_busy_fn busy; 160 dm_busy_fn busy;
165 dm_iterate_devices_fn iterate_devices; 161 dm_iterate_devices_fn iterate_devices;
166 dm_io_hints_fn io_hints; 162 dm_io_hints_fn io_hints;
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index ec274e0f4ed2..2adbfa6d02bc 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -13,6 +13,7 @@
13#include <linux/kdev_t.h> 13#include <linux/kdev_t.h>
14#include <linux/rcupdate.h> 14#include <linux/rcupdate.h>
15#include <linux/slab.h> 15#include <linux/slab.h>
16#include <linux/percpu-refcount.h>
16 17
17#ifdef CONFIG_BLOCK 18#ifdef CONFIG_BLOCK
18 19
@@ -124,7 +125,7 @@ struct hd_struct {
124#else 125#else
125 struct disk_stats dkstats; 126 struct disk_stats dkstats;
126#endif 127#endif
127 atomic_t ref; 128 struct percpu_ref ref;
128 struct rcu_head rcu_head; 129 struct rcu_head rcu_head;
129}; 130};
130 131
@@ -611,7 +612,7 @@ extern struct hd_struct * __must_check add_partition(struct gendisk *disk,
611 sector_t len, int flags, 612 sector_t len, int flags,
612 struct partition_meta_info 613 struct partition_meta_info
613 *info); 614 *info);
614extern void __delete_partition(struct hd_struct *); 615extern void __delete_partition(struct percpu_ref *);
615extern void delete_partition(struct gendisk *, int); 616extern void delete_partition(struct gendisk *, int);
616extern void printk_all_partitions(void); 617extern void printk_all_partitions(void);
617 618
@@ -640,27 +641,39 @@ extern ssize_t part_fail_store(struct device *dev,
640 const char *buf, size_t count); 641 const char *buf, size_t count);
641#endif /* CONFIG_FAIL_MAKE_REQUEST */ 642#endif /* CONFIG_FAIL_MAKE_REQUEST */
642 643
643static inline void hd_ref_init(struct hd_struct *part) 644static inline int hd_ref_init(struct hd_struct *part)
644{ 645{
645 atomic_set(&part->ref, 1); 646 if (percpu_ref_init(&part->ref, __delete_partition, 0,
646 smp_mb(); 647 GFP_KERNEL))
648 return -ENOMEM;
649 return 0;
647} 650}
648 651
649static inline void hd_struct_get(struct hd_struct *part) 652static inline void hd_struct_get(struct hd_struct *part)
650{ 653{
651 atomic_inc(&part->ref); 654 percpu_ref_get(&part->ref);
652 smp_mb__after_atomic();
653} 655}
654 656
655static inline int hd_struct_try_get(struct hd_struct *part) 657static inline int hd_struct_try_get(struct hd_struct *part)
656{ 658{
657 return atomic_inc_not_zero(&part->ref); 659 return percpu_ref_tryget_live(&part->ref);
658} 660}
659 661
660static inline void hd_struct_put(struct hd_struct *part) 662static inline void hd_struct_put(struct hd_struct *part)
661{ 663{
662 if (atomic_dec_and_test(&part->ref)) 664 percpu_ref_put(&part->ref);
663 __delete_partition(part); 665}
666
667static inline void hd_struct_kill(struct hd_struct *part)
668{
669 percpu_ref_kill(&part->ref);
670}
671
672static inline void hd_free_part(struct hd_struct *part)
673{
674 free_part_stats(part);
675 free_part_info(part);
676 percpu_ref_exit(&part->ref);
664} 677}
665 678
666/* 679/*
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 38874729dc5f..31496d201fdc 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -373,9 +373,9 @@ static inline void mem_cgroup_uncharge_swap(swp_entry_t entry)
373/* linux/mm/page_io.c */ 373/* linux/mm/page_io.c */
374extern int swap_readpage(struct page *); 374extern int swap_readpage(struct page *);
375extern int swap_writepage(struct page *page, struct writeback_control *wbc); 375extern int swap_writepage(struct page *page, struct writeback_control *wbc);
376extern void end_swap_bio_write(struct bio *bio, int err); 376extern void end_swap_bio_write(struct bio *bio);
377extern int __swap_writepage(struct page *page, struct writeback_control *wbc, 377extern int __swap_writepage(struct page *page, struct writeback_control *wbc,
378 void (*end_write_func)(struct bio *, int)); 378 bio_end_io_t end_write_func);
379extern int swap_set_page_dirty(struct page *page); 379extern int swap_set_page_dirty(struct page *page);
380 380
381int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, 381int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
diff --git a/kernel/power/swap.c b/kernel/power/swap.c
index 2f30ca91e4fa..b2066fb5b10f 100644
--- a/kernel/power/swap.c
+++ b/kernel/power/swap.c
@@ -227,27 +227,23 @@ static void hib_init_batch(struct hib_bio_batch *hb)
227 hb->error = 0; 227 hb->error = 0;
228} 228}
229 229
230static void hib_end_io(struct bio *bio, int error) 230static void hib_end_io(struct bio *bio)
231{ 231{
232 struct hib_bio_batch *hb = bio->bi_private; 232 struct hib_bio_batch *hb = bio->bi_private;
233 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
234 struct page *page = bio->bi_io_vec[0].bv_page; 233 struct page *page = bio->bi_io_vec[0].bv_page;
235 234
236 if (!uptodate || error) { 235 if (bio->bi_error) {
237 printk(KERN_ALERT "Read-error on swap-device (%u:%u:%Lu)\n", 236 printk(KERN_ALERT "Read-error on swap-device (%u:%u:%Lu)\n",
238 imajor(bio->bi_bdev->bd_inode), 237 imajor(bio->bi_bdev->bd_inode),
239 iminor(bio->bi_bdev->bd_inode), 238 iminor(bio->bi_bdev->bd_inode),
240 (unsigned long long)bio->bi_iter.bi_sector); 239 (unsigned long long)bio->bi_iter.bi_sector);
241
242 if (!error)
243 error = -EIO;
244 } 240 }
245 241
246 if (bio_data_dir(bio) == WRITE) 242 if (bio_data_dir(bio) == WRITE)
247 put_page(page); 243 put_page(page);
248 244
249 if (error && !hb->error) 245 if (bio->bi_error && !hb->error)
250 hb->error = error; 246 hb->error = bio->bi_error;
251 if (atomic_dec_and_test(&hb->count)) 247 if (atomic_dec_and_test(&hb->count))
252 wake_up(&hb->wait); 248 wake_up(&hb->wait);
253 249
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c
index b3e6b39b6cf9..90e72a0c3047 100644
--- a/kernel/trace/blktrace.c
+++ b/kernel/trace/blktrace.c
@@ -778,9 +778,6 @@ static void blk_add_trace_bio(struct request_queue *q, struct bio *bio,
778 if (likely(!bt)) 778 if (likely(!bt))
779 return; 779 return;
780 780
781 if (!error && !bio_flagged(bio, BIO_UPTODATE))
782 error = EIO;
783
784 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, 781 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,
785 bio->bi_rw, what, error, 0, NULL); 782 bio->bi_rw, what, error, 0, NULL);
786} 783}
@@ -887,8 +884,7 @@ static void blk_add_trace_split(void *ignore,
887 884
888 __blk_add_trace(bt, bio->bi_iter.bi_sector, 885 __blk_add_trace(bt, bio->bi_iter.bi_sector,
889 bio->bi_iter.bi_size, bio->bi_rw, BLK_TA_SPLIT, 886 bio->bi_iter.bi_size, bio->bi_rw, BLK_TA_SPLIT,
890 !bio_flagged(bio, BIO_UPTODATE), 887 bio->bi_error, sizeof(rpdu), &rpdu);
891 sizeof(rpdu), &rpdu);
892 } 888 }
893} 889}
894 890
@@ -920,8 +916,8 @@ static void blk_add_trace_bio_remap(void *ignore,
920 r.sector_from = cpu_to_be64(from); 916 r.sector_from = cpu_to_be64(from);
921 917
922 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, 918 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,
923 bio->bi_rw, BLK_TA_REMAP, 919 bio->bi_rw, BLK_TA_REMAP, bio->bi_error,
924 !bio_flagged(bio, BIO_UPTODATE), sizeof(r), &r); 920 sizeof(r), &r);
925} 921}
926 922
927/** 923/**
diff --git a/mm/page_io.c b/mm/page_io.c
index 520baa4b04d7..b995a5ba5e8f 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -33,22 +33,19 @@ static struct bio *get_swap_bio(gfp_t gfp_flags,
33 if (bio) { 33 if (bio) {
34 bio->bi_iter.bi_sector = map_swap_page(page, &bio->bi_bdev); 34 bio->bi_iter.bi_sector = map_swap_page(page, &bio->bi_bdev);
35 bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9; 35 bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9;
36 bio->bi_io_vec[0].bv_page = page;
37 bio->bi_io_vec[0].bv_len = PAGE_SIZE;
38 bio->bi_io_vec[0].bv_offset = 0;
39 bio->bi_vcnt = 1;
40 bio->bi_iter.bi_size = PAGE_SIZE;
41 bio->bi_end_io = end_io; 36 bio->bi_end_io = end_io;
37
38 bio_add_page(bio, page, PAGE_SIZE, 0);
39 BUG_ON(bio->bi_iter.bi_size != PAGE_SIZE);
42 } 40 }
43 return bio; 41 return bio;
44} 42}
45 43
46void end_swap_bio_write(struct bio *bio, int err) 44void end_swap_bio_write(struct bio *bio)
47{ 45{
48 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
49 struct page *page = bio->bi_io_vec[0].bv_page; 46 struct page *page = bio->bi_io_vec[0].bv_page;
50 47
51 if (!uptodate) { 48 if (bio->bi_error) {
52 SetPageError(page); 49 SetPageError(page);
53 /* 50 /*
54 * We failed to write the page out to swap-space. 51 * We failed to write the page out to swap-space.
@@ -69,12 +66,11 @@ void end_swap_bio_write(struct bio *bio, int err)
69 bio_put(bio); 66 bio_put(bio);
70} 67}
71 68
72static void end_swap_bio_read(struct bio *bio, int err) 69static void end_swap_bio_read(struct bio *bio)
73{ 70{
74 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
75 struct page *page = bio->bi_io_vec[0].bv_page; 71 struct page *page = bio->bi_io_vec[0].bv_page;
76 72
77 if (!uptodate) { 73 if (bio->bi_error) {
78 SetPageError(page); 74 SetPageError(page);
79 ClearPageUptodate(page); 75 ClearPageUptodate(page);
80 printk(KERN_ALERT "Read-error on swap-device (%u:%u:%Lu)\n", 76 printk(KERN_ALERT "Read-error on swap-device (%u:%u:%Lu)\n",
@@ -254,7 +250,7 @@ static sector_t swap_page_sector(struct page *page)
254} 250}
255 251
256int __swap_writepage(struct page *page, struct writeback_control *wbc, 252int __swap_writepage(struct page *page, struct writeback_control *wbc,
257 void (*end_write_func)(struct bio *, int)) 253 bio_end_io_t end_write_func)
258{ 254{
259 struct bio *bio; 255 struct bio *bio;
260 int ret, rw = WRITE; 256 int ret, rw = WRITE;