diff options
132 files changed, 1199 insertions, 2130 deletions
diff --git a/Documentation/block/biodoc.txt b/Documentation/block/biodoc.txt index fd12c0d835fd..5be8a7f4cc7f 100644 --- a/Documentation/block/biodoc.txt +++ b/Documentation/block/biodoc.txt | |||
@@ -1109,7 +1109,7 @@ it will loop and handle as many sectors (on a bio-segment granularity) | |||
1109 | as specified. | 1109 | as specified. |
1110 | 1110 | ||
1111 | Now bh->b_end_io is replaced by bio->bi_end_io, but most of the time the | 1111 | Now bh->b_end_io is replaced by bio->bi_end_io, but most of the time the |
1112 | right thing to use is bio_endio(bio, uptodate) instead. | 1112 | right thing to use is bio_endio(bio) instead. |
1113 | 1113 | ||
1114 | If the driver is dropping the io_request_lock from its request_fn strategy, | 1114 | If the driver is dropping the io_request_lock from its request_fn strategy, |
1115 | then it just needs to replace that with q->queue_lock instead. | 1115 | then it just needs to replace that with q->queue_lock instead. |
diff --git a/Documentation/block/biovecs.txt b/Documentation/block/biovecs.txt index 74a32ad52f53..25689584e6e0 100644 --- a/Documentation/block/biovecs.txt +++ b/Documentation/block/biovecs.txt | |||
@@ -24,7 +24,7 @@ particular, presenting the illusion of partially completed biovecs so that | |||
24 | normal code doesn't have to deal with bi_bvec_done. | 24 | normal code doesn't have to deal with bi_bvec_done. |
25 | 25 | ||
26 | * Driver code should no longer refer to biovecs directly; we now have | 26 | * Driver code should no longer refer to biovecs directly; we now have |
27 | bio_iovec() and bio_iovec_iter() macros that return literal struct biovecs, | 27 | bio_iovec() and bio_iter_iovec() macros that return literal struct biovecs, |
28 | constructed from the raw biovecs but taking into account bi_bvec_done and | 28 | constructed from the raw biovecs but taking into account bi_bvec_done and |
29 | bi_size. | 29 | bi_size. |
30 | 30 | ||
@@ -109,3 +109,11 @@ Other implications: | |||
109 | over all the biovecs in the new bio - which is silly as it's not needed. | 109 | over all the biovecs in the new bio - which is silly as it's not needed. |
110 | 110 | ||
111 | So, don't use bi_vcnt anymore. | 111 | So, don't use bi_vcnt anymore. |
112 | |||
113 | * The current interface allows the block layer to split bios as needed, so we | ||
114 | could eliminate a lot of complexity particularly in stacked drivers. Code | ||
115 | that creates bios can then create whatever size bios are convenient, and | ||
116 | more importantly stacked drivers don't have to deal with both their own bio | ||
117 | size limitations and the limitations of the underlying devices. Thus | ||
118 | there's no need to define ->merge_bvec_fn() callbacks for individual block | ||
119 | drivers. | ||
diff --git a/Documentation/block/queue-sysfs.txt b/Documentation/block/queue-sysfs.txt index 3a29f8914df9..e5d914845be6 100644 --- a/Documentation/block/queue-sysfs.txt +++ b/Documentation/block/queue-sysfs.txt | |||
@@ -20,7 +20,7 @@ This shows the size of internal allocation of the device in bytes, if | |||
20 | reported by the device. A value of '0' means device does not support | 20 | reported by the device. A value of '0' means device does not support |
21 | the discard functionality. | 21 | the discard functionality. |
22 | 22 | ||
23 | discard_max_bytes (RO) | 23 | discard_max_hw_bytes (RO) |
24 | ---------------------- | 24 | ---------------------- |
25 | Devices that support discard functionality may have internal limits on | 25 | Devices that support discard functionality may have internal limits on |
26 | the number of bytes that can be trimmed or unmapped in a single operation. | 26 | the number of bytes that can be trimmed or unmapped in a single operation. |
@@ -29,6 +29,14 @@ number of bytes that can be discarded in a single operation. Discard | |||
29 | requests issued to the device must not exceed this limit. A discard_max_bytes | 29 | requests issued to the device must not exceed this limit. A discard_max_bytes |
30 | value of 0 means that the device does not support discard functionality. | 30 | value of 0 means that the device does not support discard functionality. |
31 | 31 | ||
32 | discard_max_bytes (RW) | ||
33 | ---------------------- | ||
34 | While discard_max_hw_bytes is the hardware limit for the device, this | ||
35 | setting is the software limit. Some devices exhibit large latencies when | ||
36 | large discards are issued, setting this value lower will make Linux issue | ||
37 | smaller discards and potentially help reduce latencies induced by large | ||
38 | discard operations. | ||
39 | |||
32 | discard_zeroes_data (RO) | 40 | discard_zeroes_data (RO) |
33 | ------------------------ | 41 | ------------------------ |
34 | When read, this file will show if the discarded block are zeroed by the | 42 | When read, this file will show if the discarded block are zeroed by the |
diff --git a/arch/m68k/emu/nfblock.c b/arch/m68k/emu/nfblock.c index 2d75ae246167..f2a00c591bf7 100644 --- a/arch/m68k/emu/nfblock.c +++ b/arch/m68k/emu/nfblock.c | |||
@@ -76,7 +76,7 @@ static void nfhd_make_request(struct request_queue *queue, struct bio *bio) | |||
76 | bvec_to_phys(&bvec)); | 76 | bvec_to_phys(&bvec)); |
77 | sec += len; | 77 | sec += len; |
78 | } | 78 | } |
79 | bio_endio(bio, 0); | 79 | bio_endio(bio); |
80 | } | 80 | } |
81 | 81 | ||
82 | static int nfhd_getgeo(struct block_device *bdev, struct hd_geometry *geo) | 82 | static int nfhd_getgeo(struct block_device *bdev, struct hd_geometry *geo) |
diff --git a/arch/powerpc/sysdev/axonram.c b/arch/powerpc/sysdev/axonram.c index ee90db17b097..f86250c48b53 100644 --- a/arch/powerpc/sysdev/axonram.c +++ b/arch/powerpc/sysdev/axonram.c | |||
@@ -132,7 +132,7 @@ axon_ram_make_request(struct request_queue *queue, struct bio *bio) | |||
132 | phys_mem += vec.bv_len; | 132 | phys_mem += vec.bv_len; |
133 | transfered += vec.bv_len; | 133 | transfered += vec.bv_len; |
134 | } | 134 | } |
135 | bio_endio(bio, 0); | 135 | bio_endio(bio); |
136 | } | 136 | } |
137 | 137 | ||
138 | /** | 138 | /** |
diff --git a/arch/xtensa/platforms/iss/simdisk.c b/arch/xtensa/platforms/iss/simdisk.c index 48eebacdf5fe..fa84ca990caa 100644 --- a/arch/xtensa/platforms/iss/simdisk.c +++ b/arch/xtensa/platforms/iss/simdisk.c | |||
@@ -101,8 +101,9 @@ static void simdisk_transfer(struct simdisk *dev, unsigned long sector, | |||
101 | spin_unlock(&dev->lock); | 101 | spin_unlock(&dev->lock); |
102 | } | 102 | } |
103 | 103 | ||
104 | static int simdisk_xfer_bio(struct simdisk *dev, struct bio *bio) | 104 | static void simdisk_make_request(struct request_queue *q, struct bio *bio) |
105 | { | 105 | { |
106 | struct simdisk *dev = q->queuedata; | ||
106 | struct bio_vec bvec; | 107 | struct bio_vec bvec; |
107 | struct bvec_iter iter; | 108 | struct bvec_iter iter; |
108 | sector_t sector = bio->bi_iter.bi_sector; | 109 | sector_t sector = bio->bi_iter.bi_sector; |
@@ -116,17 +117,10 @@ static int simdisk_xfer_bio(struct simdisk *dev, struct bio *bio) | |||
116 | sector += len; | 117 | sector += len; |
117 | __bio_kunmap_atomic(buffer); | 118 | __bio_kunmap_atomic(buffer); |
118 | } | 119 | } |
119 | return 0; | ||
120 | } | ||
121 | 120 | ||
122 | static void simdisk_make_request(struct request_queue *q, struct bio *bio) | 121 | bio_endio(bio); |
123 | { | ||
124 | struct simdisk *dev = q->queuedata; | ||
125 | int status = simdisk_xfer_bio(dev, bio); | ||
126 | bio_endio(bio, status); | ||
127 | } | 122 | } |
128 | 123 | ||
129 | |||
130 | static int simdisk_open(struct block_device *bdev, fmode_t mode) | 124 | static int simdisk_open(struct block_device *bdev, fmode_t mode) |
131 | { | 125 | { |
132 | struct simdisk *dev = bdev->bd_disk->private_data; | 126 | struct simdisk *dev = bdev->bd_disk->private_data; |
diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 719b7152aed1..4aecca79374a 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c | |||
@@ -355,13 +355,12 @@ static void bio_integrity_verify_fn(struct work_struct *work) | |||
355 | container_of(work, struct bio_integrity_payload, bip_work); | 355 | container_of(work, struct bio_integrity_payload, bip_work); |
356 | struct bio *bio = bip->bip_bio; | 356 | struct bio *bio = bip->bip_bio; |
357 | struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); | 357 | struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); |
358 | int error; | ||
359 | 358 | ||
360 | error = bio_integrity_process(bio, bi->verify_fn); | 359 | bio->bi_error = bio_integrity_process(bio, bi->verify_fn); |
361 | 360 | ||
362 | /* Restore original bio completion handler */ | 361 | /* Restore original bio completion handler */ |
363 | bio->bi_end_io = bip->bip_end_io; | 362 | bio->bi_end_io = bip->bip_end_io; |
364 | bio_endio(bio, error); | 363 | bio_endio(bio); |
365 | } | 364 | } |
366 | 365 | ||
367 | /** | 366 | /** |
@@ -376,7 +375,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) | |||
376 | * in process context. This function postpones completion | 375 | * in process context. This function postpones completion |
377 | * accordingly. | 376 | * accordingly. |
378 | */ | 377 | */ |
379 | void bio_integrity_endio(struct bio *bio, int error) | 378 | void bio_integrity_endio(struct bio *bio) |
380 | { | 379 | { |
381 | struct bio_integrity_payload *bip = bio_integrity(bio); | 380 | struct bio_integrity_payload *bip = bio_integrity(bio); |
382 | 381 | ||
@@ -386,9 +385,9 @@ void bio_integrity_endio(struct bio *bio, int error) | |||
386 | * integrity metadata. Restore original bio end_io handler | 385 | * integrity metadata. Restore original bio end_io handler |
387 | * and run it. | 386 | * and run it. |
388 | */ | 387 | */ |
389 | if (error) { | 388 | if (bio->bi_error) { |
390 | bio->bi_end_io = bip->bip_end_io; | 389 | bio->bi_end_io = bip->bip_end_io; |
391 | bio_endio(bio, error); | 390 | bio_endio(bio); |
392 | 391 | ||
393 | return; | 392 | return; |
394 | } | 393 | } |
diff --git a/block/bio.c b/block/bio.c index d6e5ba3399f0..515b5434fe2d 100644 --- a/block/bio.c +++ b/block/bio.c | |||
@@ -269,7 +269,6 @@ static void bio_free(struct bio *bio) | |||
269 | void bio_init(struct bio *bio) | 269 | void bio_init(struct bio *bio) |
270 | { | 270 | { |
271 | memset(bio, 0, sizeof(*bio)); | 271 | memset(bio, 0, sizeof(*bio)); |
272 | bio->bi_flags = 1 << BIO_UPTODATE; | ||
273 | atomic_set(&bio->__bi_remaining, 1); | 272 | atomic_set(&bio->__bi_remaining, 1); |
274 | atomic_set(&bio->__bi_cnt, 1); | 273 | atomic_set(&bio->__bi_cnt, 1); |
275 | } | 274 | } |
@@ -292,14 +291,17 @@ void bio_reset(struct bio *bio) | |||
292 | __bio_free(bio); | 291 | __bio_free(bio); |
293 | 292 | ||
294 | memset(bio, 0, BIO_RESET_BYTES); | 293 | memset(bio, 0, BIO_RESET_BYTES); |
295 | bio->bi_flags = flags | (1 << BIO_UPTODATE); | 294 | bio->bi_flags = flags; |
296 | atomic_set(&bio->__bi_remaining, 1); | 295 | atomic_set(&bio->__bi_remaining, 1); |
297 | } | 296 | } |
298 | EXPORT_SYMBOL(bio_reset); | 297 | EXPORT_SYMBOL(bio_reset); |
299 | 298 | ||
300 | static void bio_chain_endio(struct bio *bio, int error) | 299 | static void bio_chain_endio(struct bio *bio) |
301 | { | 300 | { |
302 | bio_endio(bio->bi_private, error); | 301 | struct bio *parent = bio->bi_private; |
302 | |||
303 | parent->bi_error = bio->bi_error; | ||
304 | bio_endio(parent); | ||
303 | bio_put(bio); | 305 | bio_put(bio); |
304 | } | 306 | } |
305 | 307 | ||
@@ -309,7 +311,7 @@ static void bio_chain_endio(struct bio *bio, int error) | |||
309 | */ | 311 | */ |
310 | static inline void bio_inc_remaining(struct bio *bio) | 312 | static inline void bio_inc_remaining(struct bio *bio) |
311 | { | 313 | { |
312 | bio->bi_flags |= (1 << BIO_CHAIN); | 314 | bio_set_flag(bio, BIO_CHAIN); |
313 | smp_mb__before_atomic(); | 315 | smp_mb__before_atomic(); |
314 | atomic_inc(&bio->__bi_remaining); | 316 | atomic_inc(&bio->__bi_remaining); |
315 | } | 317 | } |
@@ -493,7 +495,7 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) | |||
493 | if (unlikely(!bvl)) | 495 | if (unlikely(!bvl)) |
494 | goto err_free; | 496 | goto err_free; |
495 | 497 | ||
496 | bio->bi_flags |= 1 << BIO_OWNS_VEC; | 498 | bio_set_flag(bio, BIO_OWNS_VEC); |
497 | } else if (nr_iovecs) { | 499 | } else if (nr_iovecs) { |
498 | bvl = bio->bi_inline_vecs; | 500 | bvl = bio->bi_inline_vecs; |
499 | } | 501 | } |
@@ -578,7 +580,7 @@ void __bio_clone_fast(struct bio *bio, struct bio *bio_src) | |||
578 | * so we don't set nor calculate new physical/hw segment counts here | 580 | * so we don't set nor calculate new physical/hw segment counts here |
579 | */ | 581 | */ |
580 | bio->bi_bdev = bio_src->bi_bdev; | 582 | bio->bi_bdev = bio_src->bi_bdev; |
581 | bio->bi_flags |= 1 << BIO_CLONED; | 583 | bio_set_flag(bio, BIO_CLONED); |
582 | bio->bi_rw = bio_src->bi_rw; | 584 | bio->bi_rw = bio_src->bi_rw; |
583 | bio->bi_iter = bio_src->bi_iter; | 585 | bio->bi_iter = bio_src->bi_iter; |
584 | bio->bi_io_vec = bio_src->bi_io_vec; | 586 | bio->bi_io_vec = bio_src->bi_io_vec; |
@@ -692,31 +694,22 @@ integrity_clone: | |||
692 | EXPORT_SYMBOL(bio_clone_bioset); | 694 | EXPORT_SYMBOL(bio_clone_bioset); |
693 | 695 | ||
694 | /** | 696 | /** |
695 | * bio_get_nr_vecs - return approx number of vecs | 697 | * bio_add_pc_page - attempt to add page to bio |
696 | * @bdev: I/O target | 698 | * @q: the target queue |
699 | * @bio: destination bio | ||
700 | * @page: page to add | ||
701 | * @len: vec entry length | ||
702 | * @offset: vec entry offset | ||
697 | * | 703 | * |
698 | * Return the approximate number of pages we can send to this target. | 704 | * Attempt to add a page to the bio_vec maplist. This can fail for a |
699 | * There's no guarantee that you will be able to fit this number of pages | 705 | * number of reasons, such as the bio being full or target block device |
700 | * into a bio, it does not account for dynamic restrictions that vary | 706 | * limitations. The target block device must allow bio's up to PAGE_SIZE, |
701 | * on offset. | 707 | * so it is always possible to add a single page to an empty bio. |
708 | * | ||
709 | * This should only be used by REQ_PC bios. | ||
702 | */ | 710 | */ |
703 | int bio_get_nr_vecs(struct block_device *bdev) | 711 | int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page |
704 | { | 712 | *page, unsigned int len, unsigned int offset) |
705 | struct request_queue *q = bdev_get_queue(bdev); | ||
706 | int nr_pages; | ||
707 | |||
708 | nr_pages = min_t(unsigned, | ||
709 | queue_max_segments(q), | ||
710 | queue_max_sectors(q) / (PAGE_SIZE >> 9) + 1); | ||
711 | |||
712 | return min_t(unsigned, nr_pages, BIO_MAX_PAGES); | ||
713 | |||
714 | } | ||
715 | EXPORT_SYMBOL(bio_get_nr_vecs); | ||
716 | |||
717 | static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page | ||
718 | *page, unsigned int len, unsigned int offset, | ||
719 | unsigned int max_sectors) | ||
720 | { | 713 | { |
721 | int retried_segments = 0; | 714 | int retried_segments = 0; |
722 | struct bio_vec *bvec; | 715 | struct bio_vec *bvec; |
@@ -727,7 +720,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page | |||
727 | if (unlikely(bio_flagged(bio, BIO_CLONED))) | 720 | if (unlikely(bio_flagged(bio, BIO_CLONED))) |
728 | return 0; | 721 | return 0; |
729 | 722 | ||
730 | if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors) | 723 | if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q)) |
731 | return 0; | 724 | return 0; |
732 | 725 | ||
733 | /* | 726 | /* |
@@ -740,28 +733,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page | |||
740 | 733 | ||
741 | if (page == prev->bv_page && | 734 | if (page == prev->bv_page && |
742 | offset == prev->bv_offset + prev->bv_len) { | 735 | offset == prev->bv_offset + prev->bv_len) { |
743 | unsigned int prev_bv_len = prev->bv_len; | ||
744 | prev->bv_len += len; | 736 | prev->bv_len += len; |
745 | |||
746 | if (q->merge_bvec_fn) { | ||
747 | struct bvec_merge_data bvm = { | ||
748 | /* prev_bvec is already charged in | ||
749 | bi_size, discharge it in order to | ||
750 | simulate merging updated prev_bvec | ||
751 | as new bvec. */ | ||
752 | .bi_bdev = bio->bi_bdev, | ||
753 | .bi_sector = bio->bi_iter.bi_sector, | ||
754 | .bi_size = bio->bi_iter.bi_size - | ||
755 | prev_bv_len, | ||
756 | .bi_rw = bio->bi_rw, | ||
757 | }; | ||
758 | |||
759 | if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len) { | ||
760 | prev->bv_len -= len; | ||
761 | return 0; | ||
762 | } | ||
763 | } | ||
764 | |||
765 | bio->bi_iter.bi_size += len; | 737 | bio->bi_iter.bi_size += len; |
766 | goto done; | 738 | goto done; |
767 | } | 739 | } |
@@ -770,8 +742,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page | |||
770 | * If the queue doesn't support SG gaps and adding this | 742 | * If the queue doesn't support SG gaps and adding this |
771 | * offset would create a gap, disallow it. | 743 | * offset would create a gap, disallow it. |
772 | */ | 744 | */ |
773 | if (q->queue_flags & (1 << QUEUE_FLAG_SG_GAPS) && | 745 | if (bvec_gap_to_prev(q, prev, offset)) |
774 | bvec_gap_to_prev(prev, offset)) | ||
775 | return 0; | 746 | return 0; |
776 | } | 747 | } |
777 | 748 | ||
@@ -804,30 +775,9 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page | |||
804 | blk_recount_segments(q, bio); | 775 | blk_recount_segments(q, bio); |
805 | } | 776 | } |
806 | 777 | ||
807 | /* | ||
808 | * if queue has other restrictions (eg varying max sector size | ||
809 | * depending on offset), it can specify a merge_bvec_fn in the | ||
810 | * queue to get further control | ||
811 | */ | ||
812 | if (q->merge_bvec_fn) { | ||
813 | struct bvec_merge_data bvm = { | ||
814 | .bi_bdev = bio->bi_bdev, | ||
815 | .bi_sector = bio->bi_iter.bi_sector, | ||
816 | .bi_size = bio->bi_iter.bi_size - len, | ||
817 | .bi_rw = bio->bi_rw, | ||
818 | }; | ||
819 | |||
820 | /* | ||
821 | * merge_bvec_fn() returns number of bytes it can accept | ||
822 | * at this offset | ||
823 | */ | ||
824 | if (q->merge_bvec_fn(q, &bvm, bvec) < bvec->bv_len) | ||
825 | goto failed; | ||
826 | } | ||
827 | |||
828 | /* If we may be able to merge these biovecs, force a recount */ | 778 | /* If we may be able to merge these biovecs, force a recount */ |
829 | if (bio->bi_vcnt > 1 && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec))) | 779 | if (bio->bi_vcnt > 1 && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec))) |
830 | bio->bi_flags &= ~(1 << BIO_SEG_VALID); | 780 | bio_clear_flag(bio, BIO_SEG_VALID); |
831 | 781 | ||
832 | done: | 782 | done: |
833 | return len; | 783 | return len; |
@@ -841,28 +791,6 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page | |||
841 | blk_recount_segments(q, bio); | 791 | blk_recount_segments(q, bio); |
842 | return 0; | 792 | return 0; |
843 | } | 793 | } |
844 | |||
845 | /** | ||
846 | * bio_add_pc_page - attempt to add page to bio | ||
847 | * @q: the target queue | ||
848 | * @bio: destination bio | ||
849 | * @page: page to add | ||
850 | * @len: vec entry length | ||
851 | * @offset: vec entry offset | ||
852 | * | ||
853 | * Attempt to add a page to the bio_vec maplist. This can fail for a | ||
854 | * number of reasons, such as the bio being full or target block device | ||
855 | * limitations. The target block device must allow bio's up to PAGE_SIZE, | ||
856 | * so it is always possible to add a single page to an empty bio. | ||
857 | * | ||
858 | * This should only be used by REQ_PC bios. | ||
859 | */ | ||
860 | int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page, | ||
861 | unsigned int len, unsigned int offset) | ||
862 | { | ||
863 | return __bio_add_page(q, bio, page, len, offset, | ||
864 | queue_max_hw_sectors(q)); | ||
865 | } | ||
866 | EXPORT_SYMBOL(bio_add_pc_page); | 794 | EXPORT_SYMBOL(bio_add_pc_page); |
867 | 795 | ||
868 | /** | 796 | /** |
@@ -872,22 +800,47 @@ EXPORT_SYMBOL(bio_add_pc_page); | |||
872 | * @len: vec entry length | 800 | * @len: vec entry length |
873 | * @offset: vec entry offset | 801 | * @offset: vec entry offset |
874 | * | 802 | * |
875 | * Attempt to add a page to the bio_vec maplist. This can fail for a | 803 | * Attempt to add a page to the bio_vec maplist. This will only fail |
876 | * number of reasons, such as the bio being full or target block device | 804 | * if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio. |
877 | * limitations. The target block device must allow bio's up to PAGE_SIZE, | ||
878 | * so it is always possible to add a single page to an empty bio. | ||
879 | */ | 805 | */ |
880 | int bio_add_page(struct bio *bio, struct page *page, unsigned int len, | 806 | int bio_add_page(struct bio *bio, struct page *page, |
881 | unsigned int offset) | 807 | unsigned int len, unsigned int offset) |
882 | { | 808 | { |
883 | struct request_queue *q = bdev_get_queue(bio->bi_bdev); | 809 | struct bio_vec *bv; |
884 | unsigned int max_sectors; | 810 | |
811 | /* | ||
812 | * cloned bio must not modify vec list | ||
813 | */ | ||
814 | if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) | ||
815 | return 0; | ||
885 | 816 | ||
886 | max_sectors = blk_max_size_offset(q, bio->bi_iter.bi_sector); | 817 | /* |
887 | if ((max_sectors < (len >> 9)) && !bio->bi_iter.bi_size) | 818 | * For filesystems with a blocksize smaller than the pagesize |
888 | max_sectors = len >> 9; | 819 | * we will often be called with the same page as last time and |
820 | * a consecutive offset. Optimize this special case. | ||
821 | */ | ||
822 | if (bio->bi_vcnt > 0) { | ||
823 | bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; | ||
889 | 824 | ||
890 | return __bio_add_page(q, bio, page, len, offset, max_sectors); | 825 | if (page == bv->bv_page && |
826 | offset == bv->bv_offset + bv->bv_len) { | ||
827 | bv->bv_len += len; | ||
828 | goto done; | ||
829 | } | ||
830 | } | ||
831 | |||
832 | if (bio->bi_vcnt >= bio->bi_max_vecs) | ||
833 | return 0; | ||
834 | |||
835 | bv = &bio->bi_io_vec[bio->bi_vcnt]; | ||
836 | bv->bv_page = page; | ||
837 | bv->bv_len = len; | ||
838 | bv->bv_offset = offset; | ||
839 | |||
840 | bio->bi_vcnt++; | ||
841 | done: | ||
842 | bio->bi_iter.bi_size += len; | ||
843 | return len; | ||
891 | } | 844 | } |
892 | EXPORT_SYMBOL(bio_add_page); | 845 | EXPORT_SYMBOL(bio_add_page); |
893 | 846 | ||
@@ -896,11 +849,11 @@ struct submit_bio_ret { | |||
896 | int error; | 849 | int error; |
897 | }; | 850 | }; |
898 | 851 | ||
899 | static void submit_bio_wait_endio(struct bio *bio, int error) | 852 | static void submit_bio_wait_endio(struct bio *bio) |
900 | { | 853 | { |
901 | struct submit_bio_ret *ret = bio->bi_private; | 854 | struct submit_bio_ret *ret = bio->bi_private; |
902 | 855 | ||
903 | ret->error = error; | 856 | ret->error = bio->bi_error; |
904 | complete(&ret->event); | 857 | complete(&ret->event); |
905 | } | 858 | } |
906 | 859 | ||
@@ -1388,7 +1341,7 @@ struct bio *bio_map_user_iov(struct request_queue *q, | |||
1388 | if (iter->type & WRITE) | 1341 | if (iter->type & WRITE) |
1389 | bio->bi_rw |= REQ_WRITE; | 1342 | bio->bi_rw |= REQ_WRITE; |
1390 | 1343 | ||
1391 | bio->bi_flags |= (1 << BIO_USER_MAPPED); | 1344 | bio_set_flag(bio, BIO_USER_MAPPED); |
1392 | 1345 | ||
1393 | /* | 1346 | /* |
1394 | * subtle -- if __bio_map_user() ended up bouncing a bio, | 1347 | * subtle -- if __bio_map_user() ended up bouncing a bio, |
@@ -1445,7 +1398,7 @@ void bio_unmap_user(struct bio *bio) | |||
1445 | } | 1398 | } |
1446 | EXPORT_SYMBOL(bio_unmap_user); | 1399 | EXPORT_SYMBOL(bio_unmap_user); |
1447 | 1400 | ||
1448 | static void bio_map_kern_endio(struct bio *bio, int err) | 1401 | static void bio_map_kern_endio(struct bio *bio) |
1449 | { | 1402 | { |
1450 | bio_put(bio); | 1403 | bio_put(bio); |
1451 | } | 1404 | } |
@@ -1501,13 +1454,13 @@ struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len, | |||
1501 | } | 1454 | } |
1502 | EXPORT_SYMBOL(bio_map_kern); | 1455 | EXPORT_SYMBOL(bio_map_kern); |
1503 | 1456 | ||
1504 | static void bio_copy_kern_endio(struct bio *bio, int err) | 1457 | static void bio_copy_kern_endio(struct bio *bio) |
1505 | { | 1458 | { |
1506 | bio_free_pages(bio); | 1459 | bio_free_pages(bio); |
1507 | bio_put(bio); | 1460 | bio_put(bio); |
1508 | } | 1461 | } |
1509 | 1462 | ||
1510 | static void bio_copy_kern_endio_read(struct bio *bio, int err) | 1463 | static void bio_copy_kern_endio_read(struct bio *bio) |
1511 | { | 1464 | { |
1512 | char *p = bio->bi_private; | 1465 | char *p = bio->bi_private; |
1513 | struct bio_vec *bvec; | 1466 | struct bio_vec *bvec; |
@@ -1518,7 +1471,7 @@ static void bio_copy_kern_endio_read(struct bio *bio, int err) | |||
1518 | p += bvec->bv_len; | 1471 | p += bvec->bv_len; |
1519 | } | 1472 | } |
1520 | 1473 | ||
1521 | bio_copy_kern_endio(bio, err); | 1474 | bio_copy_kern_endio(bio); |
1522 | } | 1475 | } |
1523 | 1476 | ||
1524 | /** | 1477 | /** |
@@ -1768,7 +1721,7 @@ static inline bool bio_remaining_done(struct bio *bio) | |||
1768 | BUG_ON(atomic_read(&bio->__bi_remaining) <= 0); | 1721 | BUG_ON(atomic_read(&bio->__bi_remaining) <= 0); |
1769 | 1722 | ||
1770 | if (atomic_dec_and_test(&bio->__bi_remaining)) { | 1723 | if (atomic_dec_and_test(&bio->__bi_remaining)) { |
1771 | clear_bit(BIO_CHAIN, &bio->bi_flags); | 1724 | bio_clear_flag(bio, BIO_CHAIN); |
1772 | return true; | 1725 | return true; |
1773 | } | 1726 | } |
1774 | 1727 | ||
@@ -1778,25 +1731,15 @@ static inline bool bio_remaining_done(struct bio *bio) | |||
1778 | /** | 1731 | /** |
1779 | * bio_endio - end I/O on a bio | 1732 | * bio_endio - end I/O on a bio |
1780 | * @bio: bio | 1733 | * @bio: bio |
1781 | * @error: error, if any | ||
1782 | * | 1734 | * |
1783 | * Description: | 1735 | * Description: |
1784 | * bio_endio() will end I/O on the whole bio. bio_endio() is the | 1736 | * bio_endio() will end I/O on the whole bio. bio_endio() is the preferred |
1785 | * preferred way to end I/O on a bio, it takes care of clearing | 1737 | * way to end I/O on a bio. No one should call bi_end_io() directly on a |
1786 | * BIO_UPTODATE on error. @error is 0 on success, and and one of the | 1738 | * bio unless they own it and thus know that it has an end_io function. |
1787 | * established -Exxxx (-EIO, for instance) error values in case | ||
1788 | * something went wrong. No one should call bi_end_io() directly on a | ||
1789 | * bio unless they own it and thus know that it has an end_io | ||
1790 | * function. | ||
1791 | **/ | 1739 | **/ |
1792 | void bio_endio(struct bio *bio, int error) | 1740 | void bio_endio(struct bio *bio) |
1793 | { | 1741 | { |
1794 | while (bio) { | 1742 | while (bio) { |
1795 | if (error) | ||
1796 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | ||
1797 | else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) | ||
1798 | error = -EIO; | ||
1799 | |||
1800 | if (unlikely(!bio_remaining_done(bio))) | 1743 | if (unlikely(!bio_remaining_done(bio))) |
1801 | break; | 1744 | break; |
1802 | 1745 | ||
@@ -1810,11 +1753,12 @@ void bio_endio(struct bio *bio, int error) | |||
1810 | */ | 1753 | */ |
1811 | if (bio->bi_end_io == bio_chain_endio) { | 1754 | if (bio->bi_end_io == bio_chain_endio) { |
1812 | struct bio *parent = bio->bi_private; | 1755 | struct bio *parent = bio->bi_private; |
1756 | parent->bi_error = bio->bi_error; | ||
1813 | bio_put(bio); | 1757 | bio_put(bio); |
1814 | bio = parent; | 1758 | bio = parent; |
1815 | } else { | 1759 | } else { |
1816 | if (bio->bi_end_io) | 1760 | if (bio->bi_end_io) |
1817 | bio->bi_end_io(bio, error); | 1761 | bio->bi_end_io(bio); |
1818 | bio = NULL; | 1762 | bio = NULL; |
1819 | } | 1763 | } |
1820 | } | 1764 | } |
@@ -1882,7 +1826,7 @@ void bio_trim(struct bio *bio, int offset, int size) | |||
1882 | if (offset == 0 && size == bio->bi_iter.bi_size) | 1826 | if (offset == 0 && size == bio->bi_iter.bi_size) |
1883 | return; | 1827 | return; |
1884 | 1828 | ||
1885 | clear_bit(BIO_SEG_VALID, &bio->bi_flags); | 1829 | bio_clear_flag(bio, BIO_SEG_VALID); |
1886 | 1830 | ||
1887 | bio_advance(bio, offset << 9); | 1831 | bio_advance(bio, offset << 9); |
1888 | 1832 | ||
diff --git a/block/blk-core.c b/block/blk-core.c index 627ed0c593fb..60912e983f16 100644 --- a/block/blk-core.c +++ b/block/blk-core.c | |||
@@ -143,18 +143,16 @@ static void req_bio_endio(struct request *rq, struct bio *bio, | |||
143 | unsigned int nbytes, int error) | 143 | unsigned int nbytes, int error) |
144 | { | 144 | { |
145 | if (error) | 145 | if (error) |
146 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | 146 | bio->bi_error = error; |
147 | else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) | ||
148 | error = -EIO; | ||
149 | 147 | ||
150 | if (unlikely(rq->cmd_flags & REQ_QUIET)) | 148 | if (unlikely(rq->cmd_flags & REQ_QUIET)) |
151 | set_bit(BIO_QUIET, &bio->bi_flags); | 149 | bio_set_flag(bio, BIO_QUIET); |
152 | 150 | ||
153 | bio_advance(bio, nbytes); | 151 | bio_advance(bio, nbytes); |
154 | 152 | ||
155 | /* don't actually finish bio if it's part of flush sequence */ | 153 | /* don't actually finish bio if it's part of flush sequence */ |
156 | if (bio->bi_iter.bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ)) | 154 | if (bio->bi_iter.bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ)) |
157 | bio_endio(bio, error); | 155 | bio_endio(bio); |
158 | } | 156 | } |
159 | 157 | ||
160 | void blk_dump_rq_flags(struct request *rq, char *msg) | 158 | void blk_dump_rq_flags(struct request *rq, char *msg) |
@@ -645,6 +643,10 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) | |||
645 | if (q->id < 0) | 643 | if (q->id < 0) |
646 | goto fail_q; | 644 | goto fail_q; |
647 | 645 | ||
646 | q->bio_split = bioset_create(BIO_POOL_SIZE, 0); | ||
647 | if (!q->bio_split) | ||
648 | goto fail_id; | ||
649 | |||
648 | q->backing_dev_info.ra_pages = | 650 | q->backing_dev_info.ra_pages = |
649 | (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; | 651 | (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; |
650 | q->backing_dev_info.capabilities = BDI_CAP_CGROUP_WRITEBACK; | 652 | q->backing_dev_info.capabilities = BDI_CAP_CGROUP_WRITEBACK; |
@@ -653,7 +655,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) | |||
653 | 655 | ||
654 | err = bdi_init(&q->backing_dev_info); | 656 | err = bdi_init(&q->backing_dev_info); |
655 | if (err) | 657 | if (err) |
656 | goto fail_id; | 658 | goto fail_split; |
657 | 659 | ||
658 | setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, | 660 | setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, |
659 | laptop_mode_timer_fn, (unsigned long) q); | 661 | laptop_mode_timer_fn, (unsigned long) q); |
@@ -695,6 +697,8 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) | |||
695 | 697 | ||
696 | fail_bdi: | 698 | fail_bdi: |
697 | bdi_destroy(&q->backing_dev_info); | 699 | bdi_destroy(&q->backing_dev_info); |
700 | fail_split: | ||
701 | bioset_free(q->bio_split); | ||
698 | fail_id: | 702 | fail_id: |
699 | ida_simple_remove(&blk_queue_ida, q->id); | 703 | ida_simple_remove(&blk_queue_ida, q->id); |
700 | fail_q: | 704 | fail_q: |
@@ -1612,6 +1616,8 @@ static void blk_queue_bio(struct request_queue *q, struct bio *bio) | |||
1612 | struct request *req; | 1616 | struct request *req; |
1613 | unsigned int request_count = 0; | 1617 | unsigned int request_count = 0; |
1614 | 1618 | ||
1619 | blk_queue_split(q, &bio, q->bio_split); | ||
1620 | |||
1615 | /* | 1621 | /* |
1616 | * low level driver can indicate that it wants pages above a | 1622 | * low level driver can indicate that it wants pages above a |
1617 | * certain limit bounced to low memory (ie for highmem, or even | 1623 | * certain limit bounced to low memory (ie for highmem, or even |
@@ -1620,7 +1626,8 @@ static void blk_queue_bio(struct request_queue *q, struct bio *bio) | |||
1620 | blk_queue_bounce(q, &bio); | 1626 | blk_queue_bounce(q, &bio); |
1621 | 1627 | ||
1622 | if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { | 1628 | if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { |
1623 | bio_endio(bio, -EIO); | 1629 | bio->bi_error = -EIO; |
1630 | bio_endio(bio); | ||
1624 | return; | 1631 | return; |
1625 | } | 1632 | } |
1626 | 1633 | ||
@@ -1673,7 +1680,8 @@ get_rq: | |||
1673 | */ | 1680 | */ |
1674 | req = get_request(q, rw_flags, bio, GFP_NOIO); | 1681 | req = get_request(q, rw_flags, bio, GFP_NOIO); |
1675 | if (IS_ERR(req)) { | 1682 | if (IS_ERR(req)) { |
1676 | bio_endio(bio, PTR_ERR(req)); /* @q is dead */ | 1683 | bio->bi_error = PTR_ERR(req); |
1684 | bio_endio(bio); | ||
1677 | goto out_unlock; | 1685 | goto out_unlock; |
1678 | } | 1686 | } |
1679 | 1687 | ||
@@ -1832,15 +1840,6 @@ generic_make_request_checks(struct bio *bio) | |||
1832 | goto end_io; | 1840 | goto end_io; |
1833 | } | 1841 | } |
1834 | 1842 | ||
1835 | if (likely(bio_is_rw(bio) && | ||
1836 | nr_sectors > queue_max_hw_sectors(q))) { | ||
1837 | printk(KERN_ERR "bio too big device %s (%u > %u)\n", | ||
1838 | bdevname(bio->bi_bdev, b), | ||
1839 | bio_sectors(bio), | ||
1840 | queue_max_hw_sectors(q)); | ||
1841 | goto end_io; | ||
1842 | } | ||
1843 | |||
1844 | part = bio->bi_bdev->bd_part; | 1843 | part = bio->bi_bdev->bd_part; |
1845 | if (should_fail_request(part, bio->bi_iter.bi_size) || | 1844 | if (should_fail_request(part, bio->bi_iter.bi_size) || |
1846 | should_fail_request(&part_to_disk(part)->part0, | 1845 | should_fail_request(&part_to_disk(part)->part0, |
@@ -1896,7 +1895,8 @@ generic_make_request_checks(struct bio *bio) | |||
1896 | return true; | 1895 | return true; |
1897 | 1896 | ||
1898 | end_io: | 1897 | end_io: |
1899 | bio_endio(bio, err); | 1898 | bio->bi_error = err; |
1899 | bio_endio(bio); | ||
1900 | return false; | 1900 | return false; |
1901 | } | 1901 | } |
1902 | 1902 | ||
diff --git a/block/blk-flush.c b/block/blk-flush.c index 20badd7b9d1b..9c423e53324a 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c | |||
@@ -73,6 +73,7 @@ | |||
73 | 73 | ||
74 | #include "blk.h" | 74 | #include "blk.h" |
75 | #include "blk-mq.h" | 75 | #include "blk-mq.h" |
76 | #include "blk-mq-tag.h" | ||
76 | 77 | ||
77 | /* FLUSH/FUA sequences */ | 78 | /* FLUSH/FUA sequences */ |
78 | enum { | 79 | enum { |
@@ -226,7 +227,12 @@ static void flush_end_io(struct request *flush_rq, int error) | |||
226 | struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx); | 227 | struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx); |
227 | 228 | ||
228 | if (q->mq_ops) { | 229 | if (q->mq_ops) { |
230 | struct blk_mq_hw_ctx *hctx; | ||
231 | |||
232 | /* release the tag's ownership to the req cloned from */ | ||
229 | spin_lock_irqsave(&fq->mq_flush_lock, flags); | 233 | spin_lock_irqsave(&fq->mq_flush_lock, flags); |
234 | hctx = q->mq_ops->map_queue(q, flush_rq->mq_ctx->cpu); | ||
235 | blk_mq_tag_set_rq(hctx, flush_rq->tag, fq->orig_rq); | ||
230 | flush_rq->tag = -1; | 236 | flush_rq->tag = -1; |
231 | } | 237 | } |
232 | 238 | ||
@@ -308,11 +314,18 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) | |||
308 | 314 | ||
309 | /* | 315 | /* |
310 | * Borrow tag from the first request since they can't | 316 | * Borrow tag from the first request since they can't |
311 | * be in flight at the same time. | 317 | * be in flight at the same time. And acquire the tag's |
318 | * ownership for flush req. | ||
312 | */ | 319 | */ |
313 | if (q->mq_ops) { | 320 | if (q->mq_ops) { |
321 | struct blk_mq_hw_ctx *hctx; | ||
322 | |||
314 | flush_rq->mq_ctx = first_rq->mq_ctx; | 323 | flush_rq->mq_ctx = first_rq->mq_ctx; |
315 | flush_rq->tag = first_rq->tag; | 324 | flush_rq->tag = first_rq->tag; |
325 | fq->orig_rq = first_rq; | ||
326 | |||
327 | hctx = q->mq_ops->map_queue(q, first_rq->mq_ctx->cpu); | ||
328 | blk_mq_tag_set_rq(hctx, first_rq->tag, flush_rq); | ||
316 | } | 329 | } |
317 | 330 | ||
318 | flush_rq->cmd_type = REQ_TYPE_FS; | 331 | flush_rq->cmd_type = REQ_TYPE_FS; |
diff --git a/block/blk-lib.c b/block/blk-lib.c index 7688ee3f5d72..bd40292e5009 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c | |||
@@ -11,21 +11,28 @@ | |||
11 | 11 | ||
12 | struct bio_batch { | 12 | struct bio_batch { |
13 | atomic_t done; | 13 | atomic_t done; |
14 | unsigned long flags; | 14 | int error; |
15 | struct completion *wait; | 15 | struct completion *wait; |
16 | }; | 16 | }; |
17 | 17 | ||
18 | static void bio_batch_end_io(struct bio *bio, int err) | 18 | static void bio_batch_end_io(struct bio *bio) |
19 | { | 19 | { |
20 | struct bio_batch *bb = bio->bi_private; | 20 | struct bio_batch *bb = bio->bi_private; |
21 | 21 | ||
22 | if (err && (err != -EOPNOTSUPP)) | 22 | if (bio->bi_error && bio->bi_error != -EOPNOTSUPP) |
23 | clear_bit(BIO_UPTODATE, &bb->flags); | 23 | bb->error = bio->bi_error; |
24 | if (atomic_dec_and_test(&bb->done)) | 24 | if (atomic_dec_and_test(&bb->done)) |
25 | complete(bb->wait); | 25 | complete(bb->wait); |
26 | bio_put(bio); | 26 | bio_put(bio); |
27 | } | 27 | } |
28 | 28 | ||
29 | /* | ||
30 | * Ensure that max discard sectors doesn't overflow bi_size and hopefully | ||
31 | * it is of the proper granularity as long as the granularity is a power | ||
32 | * of two. | ||
33 | */ | ||
34 | #define MAX_BIO_SECTORS ((1U << 31) >> 9) | ||
35 | |||
29 | /** | 36 | /** |
30 | * blkdev_issue_discard - queue a discard | 37 | * blkdev_issue_discard - queue a discard |
31 | * @bdev: blockdev to issue discard for | 38 | * @bdev: blockdev to issue discard for |
@@ -43,8 +50,6 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, | |||
43 | DECLARE_COMPLETION_ONSTACK(wait); | 50 | DECLARE_COMPLETION_ONSTACK(wait); |
44 | struct request_queue *q = bdev_get_queue(bdev); | 51 | struct request_queue *q = bdev_get_queue(bdev); |
45 | int type = REQ_WRITE | REQ_DISCARD; | 52 | int type = REQ_WRITE | REQ_DISCARD; |
46 | unsigned int max_discard_sectors, granularity; | ||
47 | int alignment; | ||
48 | struct bio_batch bb; | 53 | struct bio_batch bb; |
49 | struct bio *bio; | 54 | struct bio *bio; |
50 | int ret = 0; | 55 | int ret = 0; |
@@ -56,21 +61,6 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, | |||
56 | if (!blk_queue_discard(q)) | 61 | if (!blk_queue_discard(q)) |
57 | return -EOPNOTSUPP; | 62 | return -EOPNOTSUPP; |
58 | 63 | ||
59 | /* Zero-sector (unknown) and one-sector granularities are the same. */ | ||
60 | granularity = max(q->limits.discard_granularity >> 9, 1U); | ||
61 | alignment = (bdev_discard_alignment(bdev) >> 9) % granularity; | ||
62 | |||
63 | /* | ||
64 | * Ensure that max_discard_sectors is of the proper | ||
65 | * granularity, so that requests stay aligned after a split. | ||
66 | */ | ||
67 | max_discard_sectors = min(q->limits.max_discard_sectors, UINT_MAX >> 9); | ||
68 | max_discard_sectors -= max_discard_sectors % granularity; | ||
69 | if (unlikely(!max_discard_sectors)) { | ||
70 | /* Avoid infinite loop below. Being cautious never hurts. */ | ||
71 | return -EOPNOTSUPP; | ||
72 | } | ||
73 | |||
74 | if (flags & BLKDEV_DISCARD_SECURE) { | 64 | if (flags & BLKDEV_DISCARD_SECURE) { |
75 | if (!blk_queue_secdiscard(q)) | 65 | if (!blk_queue_secdiscard(q)) |
76 | return -EOPNOTSUPP; | 66 | return -EOPNOTSUPP; |
@@ -78,13 +68,13 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, | |||
78 | } | 68 | } |
79 | 69 | ||
80 | atomic_set(&bb.done, 1); | 70 | atomic_set(&bb.done, 1); |
81 | bb.flags = 1 << BIO_UPTODATE; | 71 | bb.error = 0; |
82 | bb.wait = &wait; | 72 | bb.wait = &wait; |
83 | 73 | ||
84 | blk_start_plug(&plug); | 74 | blk_start_plug(&plug); |
85 | while (nr_sects) { | 75 | while (nr_sects) { |
86 | unsigned int req_sects; | 76 | unsigned int req_sects; |
87 | sector_t end_sect, tmp; | 77 | sector_t end_sect; |
88 | 78 | ||
89 | bio = bio_alloc(gfp_mask, 1); | 79 | bio = bio_alloc(gfp_mask, 1); |
90 | if (!bio) { | 80 | if (!bio) { |
@@ -92,21 +82,8 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, | |||
92 | break; | 82 | break; |
93 | } | 83 | } |
94 | 84 | ||
95 | req_sects = min_t(sector_t, nr_sects, max_discard_sectors); | 85 | req_sects = min_t(sector_t, nr_sects, MAX_BIO_SECTORS); |
96 | |||
97 | /* | ||
98 | * If splitting a request, and the next starting sector would be | ||
99 | * misaligned, stop the discard at the previous aligned sector. | ||
100 | */ | ||
101 | end_sect = sector + req_sects; | 86 | end_sect = sector + req_sects; |
102 | tmp = end_sect; | ||
103 | if (req_sects < nr_sects && | ||
104 | sector_div(tmp, granularity) != alignment) { | ||
105 | end_sect = end_sect - alignment; | ||
106 | sector_div(end_sect, granularity); | ||
107 | end_sect = end_sect * granularity + alignment; | ||
108 | req_sects = end_sect - sector; | ||
109 | } | ||
110 | 87 | ||
111 | bio->bi_iter.bi_sector = sector; | 88 | bio->bi_iter.bi_sector = sector; |
112 | bio->bi_end_io = bio_batch_end_io; | 89 | bio->bi_end_io = bio_batch_end_io; |
@@ -134,9 +111,8 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, | |||
134 | if (!atomic_dec_and_test(&bb.done)) | 111 | if (!atomic_dec_and_test(&bb.done)) |
135 | wait_for_completion_io(&wait); | 112 | wait_for_completion_io(&wait); |
136 | 113 | ||
137 | if (!test_bit(BIO_UPTODATE, &bb.flags)) | 114 | if (bb.error) |
138 | ret = -EIO; | 115 | return bb.error; |
139 | |||
140 | return ret; | 116 | return ret; |
141 | } | 117 | } |
142 | EXPORT_SYMBOL(blkdev_issue_discard); | 118 | EXPORT_SYMBOL(blkdev_issue_discard); |
@@ -166,13 +142,11 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, | |||
166 | if (!q) | 142 | if (!q) |
167 | return -ENXIO; | 143 | return -ENXIO; |
168 | 144 | ||
169 | max_write_same_sectors = q->limits.max_write_same_sectors; | 145 | /* Ensure that max_write_same_sectors doesn't overflow bi_size */ |
170 | 146 | max_write_same_sectors = UINT_MAX >> 9; | |
171 | if (max_write_same_sectors == 0) | ||
172 | return -EOPNOTSUPP; | ||
173 | 147 | ||
174 | atomic_set(&bb.done, 1); | 148 | atomic_set(&bb.done, 1); |
175 | bb.flags = 1 << BIO_UPTODATE; | 149 | bb.error = 0; |
176 | bb.wait = &wait; | 150 | bb.wait = &wait; |
177 | 151 | ||
178 | while (nr_sects) { | 152 | while (nr_sects) { |
@@ -208,9 +182,8 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, | |||
208 | if (!atomic_dec_and_test(&bb.done)) | 182 | if (!atomic_dec_and_test(&bb.done)) |
209 | wait_for_completion_io(&wait); | 183 | wait_for_completion_io(&wait); |
210 | 184 | ||
211 | if (!test_bit(BIO_UPTODATE, &bb.flags)) | 185 | if (bb.error) |
212 | ret = -ENOTSUPP; | 186 | return bb.error; |
213 | |||
214 | return ret; | 187 | return ret; |
215 | } | 188 | } |
216 | EXPORT_SYMBOL(blkdev_issue_write_same); | 189 | EXPORT_SYMBOL(blkdev_issue_write_same); |
@@ -236,7 +209,7 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, | |||
236 | DECLARE_COMPLETION_ONSTACK(wait); | 209 | DECLARE_COMPLETION_ONSTACK(wait); |
237 | 210 | ||
238 | atomic_set(&bb.done, 1); | 211 | atomic_set(&bb.done, 1); |
239 | bb.flags = 1 << BIO_UPTODATE; | 212 | bb.error = 0; |
240 | bb.wait = &wait; | 213 | bb.wait = &wait; |
241 | 214 | ||
242 | ret = 0; | 215 | ret = 0; |
@@ -270,10 +243,8 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, | |||
270 | if (!atomic_dec_and_test(&bb.done)) | 243 | if (!atomic_dec_and_test(&bb.done)) |
271 | wait_for_completion_io(&wait); | 244 | wait_for_completion_io(&wait); |
272 | 245 | ||
273 | if (!test_bit(BIO_UPTODATE, &bb.flags)) | 246 | if (bb.error) |
274 | /* One of bios in the batch was completed with error.*/ | 247 | return bb.error; |
275 | ret = -EIO; | ||
276 | |||
277 | return ret; | 248 | return ret; |
278 | } | 249 | } |
279 | 250 | ||
diff --git a/block/blk-map.c b/block/blk-map.c index da310a105429..233841644c9d 100644 --- a/block/blk-map.c +++ b/block/blk-map.c | |||
@@ -94,7 +94,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, | |||
94 | return PTR_ERR(bio); | 94 | return PTR_ERR(bio); |
95 | 95 | ||
96 | if (map_data && map_data->null_mapped) | 96 | if (map_data && map_data->null_mapped) |
97 | bio->bi_flags |= (1 << BIO_NULL_MAPPED); | 97 | bio_set_flag(bio, BIO_NULL_MAPPED); |
98 | 98 | ||
99 | if (bio->bi_iter.bi_size != iter->count) { | 99 | if (bio->bi_iter.bi_size != iter->count) { |
100 | /* | 100 | /* |
@@ -103,7 +103,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, | |||
103 | * normal IO completion path | 103 | * normal IO completion path |
104 | */ | 104 | */ |
105 | bio_get(bio); | 105 | bio_get(bio); |
106 | bio_endio(bio, 0); | 106 | bio_endio(bio); |
107 | __blk_rq_unmap_user(bio); | 107 | __blk_rq_unmap_user(bio); |
108 | return -EINVAL; | 108 | return -EINVAL; |
109 | } | 109 | } |
diff --git a/block/blk-merge.c b/block/blk-merge.c index 30a0d9f89017..b2625271a572 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c | |||
@@ -9,12 +9,146 @@ | |||
9 | 9 | ||
10 | #include "blk.h" | 10 | #include "blk.h" |
11 | 11 | ||
12 | static struct bio *blk_bio_discard_split(struct request_queue *q, | ||
13 | struct bio *bio, | ||
14 | struct bio_set *bs) | ||
15 | { | ||
16 | unsigned int max_discard_sectors, granularity; | ||
17 | int alignment; | ||
18 | sector_t tmp; | ||
19 | unsigned split_sectors; | ||
20 | |||
21 | /* Zero-sector (unknown) and one-sector granularities are the same. */ | ||
22 | granularity = max(q->limits.discard_granularity >> 9, 1U); | ||
23 | |||
24 | max_discard_sectors = min(q->limits.max_discard_sectors, UINT_MAX >> 9); | ||
25 | max_discard_sectors -= max_discard_sectors % granularity; | ||
26 | |||
27 | if (unlikely(!max_discard_sectors)) { | ||
28 | /* XXX: warn */ | ||
29 | return NULL; | ||
30 | } | ||
31 | |||
32 | if (bio_sectors(bio) <= max_discard_sectors) | ||
33 | return NULL; | ||
34 | |||
35 | split_sectors = max_discard_sectors; | ||
36 | |||
37 | /* | ||
38 | * If the next starting sector would be misaligned, stop the discard at | ||
39 | * the previous aligned sector. | ||
40 | */ | ||
41 | alignment = (q->limits.discard_alignment >> 9) % granularity; | ||
42 | |||
43 | tmp = bio->bi_iter.bi_sector + split_sectors - alignment; | ||
44 | tmp = sector_div(tmp, granularity); | ||
45 | |||
46 | if (split_sectors > tmp) | ||
47 | split_sectors -= tmp; | ||
48 | |||
49 | return bio_split(bio, split_sectors, GFP_NOIO, bs); | ||
50 | } | ||
51 | |||
52 | static struct bio *blk_bio_write_same_split(struct request_queue *q, | ||
53 | struct bio *bio, | ||
54 | struct bio_set *bs) | ||
55 | { | ||
56 | if (!q->limits.max_write_same_sectors) | ||
57 | return NULL; | ||
58 | |||
59 | if (bio_sectors(bio) <= q->limits.max_write_same_sectors) | ||
60 | return NULL; | ||
61 | |||
62 | return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs); | ||
63 | } | ||
64 | |||
65 | static struct bio *blk_bio_segment_split(struct request_queue *q, | ||
66 | struct bio *bio, | ||
67 | struct bio_set *bs) | ||
68 | { | ||
69 | struct bio *split; | ||
70 | struct bio_vec bv, bvprv; | ||
71 | struct bvec_iter iter; | ||
72 | unsigned seg_size = 0, nsegs = 0, sectors = 0; | ||
73 | int prev = 0; | ||
74 | |||
75 | bio_for_each_segment(bv, bio, iter) { | ||
76 | sectors += bv.bv_len >> 9; | ||
77 | |||
78 | if (sectors > queue_max_sectors(q)) | ||
79 | goto split; | ||
80 | |||
81 | /* | ||
82 | * If the queue doesn't support SG gaps and adding this | ||
83 | * offset would create a gap, disallow it. | ||
84 | */ | ||
85 | if (prev && bvec_gap_to_prev(q, &bvprv, bv.bv_offset)) | ||
86 | goto split; | ||
87 | |||
88 | if (prev && blk_queue_cluster(q)) { | ||
89 | if (seg_size + bv.bv_len > queue_max_segment_size(q)) | ||
90 | goto new_segment; | ||
91 | if (!BIOVEC_PHYS_MERGEABLE(&bvprv, &bv)) | ||
92 | goto new_segment; | ||
93 | if (!BIOVEC_SEG_BOUNDARY(q, &bvprv, &bv)) | ||
94 | goto new_segment; | ||
95 | |||
96 | seg_size += bv.bv_len; | ||
97 | bvprv = bv; | ||
98 | prev = 1; | ||
99 | continue; | ||
100 | } | ||
101 | new_segment: | ||
102 | if (nsegs == queue_max_segments(q)) | ||
103 | goto split; | ||
104 | |||
105 | nsegs++; | ||
106 | bvprv = bv; | ||
107 | prev = 1; | ||
108 | seg_size = bv.bv_len; | ||
109 | } | ||
110 | |||
111 | return NULL; | ||
112 | split: | ||
113 | split = bio_clone_bioset(bio, GFP_NOIO, bs); | ||
114 | |||
115 | split->bi_iter.bi_size -= iter.bi_size; | ||
116 | bio->bi_iter = iter; | ||
117 | |||
118 | if (bio_integrity(bio)) { | ||
119 | bio_integrity_advance(bio, split->bi_iter.bi_size); | ||
120 | bio_integrity_trim(split, 0, bio_sectors(split)); | ||
121 | } | ||
122 | |||
123 | return split; | ||
124 | } | ||
125 | |||
126 | void blk_queue_split(struct request_queue *q, struct bio **bio, | ||
127 | struct bio_set *bs) | ||
128 | { | ||
129 | struct bio *split; | ||
130 | |||
131 | if ((*bio)->bi_rw & REQ_DISCARD) | ||
132 | split = blk_bio_discard_split(q, *bio, bs); | ||
133 | else if ((*bio)->bi_rw & REQ_WRITE_SAME) | ||
134 | split = blk_bio_write_same_split(q, *bio, bs); | ||
135 | else | ||
136 | split = blk_bio_segment_split(q, *bio, q->bio_split); | ||
137 | |||
138 | if (split) { | ||
139 | bio_chain(split, *bio); | ||
140 | generic_make_request(*bio); | ||
141 | *bio = split; | ||
142 | } | ||
143 | } | ||
144 | EXPORT_SYMBOL(blk_queue_split); | ||
145 | |||
12 | static unsigned int __blk_recalc_rq_segments(struct request_queue *q, | 146 | static unsigned int __blk_recalc_rq_segments(struct request_queue *q, |
13 | struct bio *bio, | 147 | struct bio *bio, |
14 | bool no_sg_merge) | 148 | bool no_sg_merge) |
15 | { | 149 | { |
16 | struct bio_vec bv, bvprv = { NULL }; | 150 | struct bio_vec bv, bvprv = { NULL }; |
17 | int cluster, high, highprv = 1; | 151 | int cluster, prev = 0; |
18 | unsigned int seg_size, nr_phys_segs; | 152 | unsigned int seg_size, nr_phys_segs; |
19 | struct bio *fbio, *bbio; | 153 | struct bio *fbio, *bbio; |
20 | struct bvec_iter iter; | 154 | struct bvec_iter iter; |
@@ -36,7 +170,6 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, | |||
36 | cluster = blk_queue_cluster(q); | 170 | cluster = blk_queue_cluster(q); |
37 | seg_size = 0; | 171 | seg_size = 0; |
38 | nr_phys_segs = 0; | 172 | nr_phys_segs = 0; |
39 | high = 0; | ||
40 | for_each_bio(bio) { | 173 | for_each_bio(bio) { |
41 | bio_for_each_segment(bv, bio, iter) { | 174 | bio_for_each_segment(bv, bio, iter) { |
42 | /* | 175 | /* |
@@ -46,13 +179,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, | |||
46 | if (no_sg_merge) | 179 | if (no_sg_merge) |
47 | goto new_segment; | 180 | goto new_segment; |
48 | 181 | ||
49 | /* | 182 | if (prev && cluster) { |
50 | * the trick here is making sure that a high page is | ||
51 | * never considered part of another segment, since | ||
52 | * that might change with the bounce page. | ||
53 | */ | ||
54 | high = page_to_pfn(bv.bv_page) > queue_bounce_pfn(q); | ||
55 | if (!high && !highprv && cluster) { | ||
56 | if (seg_size + bv.bv_len | 183 | if (seg_size + bv.bv_len |
57 | > queue_max_segment_size(q)) | 184 | > queue_max_segment_size(q)) |
58 | goto new_segment; | 185 | goto new_segment; |
@@ -72,8 +199,8 @@ new_segment: | |||
72 | 199 | ||
73 | nr_phys_segs++; | 200 | nr_phys_segs++; |
74 | bvprv = bv; | 201 | bvprv = bv; |
202 | prev = 1; | ||
75 | seg_size = bv.bv_len; | 203 | seg_size = bv.bv_len; |
76 | highprv = high; | ||
77 | } | 204 | } |
78 | bbio = bio; | 205 | bbio = bio; |
79 | } | 206 | } |
@@ -116,7 +243,7 @@ void blk_recount_segments(struct request_queue *q, struct bio *bio) | |||
116 | bio->bi_next = nxt; | 243 | bio->bi_next = nxt; |
117 | } | 244 | } |
118 | 245 | ||
119 | bio->bi_flags |= (1 << BIO_SEG_VALID); | 246 | bio_set_flag(bio, BIO_SEG_VALID); |
120 | } | 247 | } |
121 | EXPORT_SYMBOL(blk_recount_segments); | 248 | EXPORT_SYMBOL(blk_recount_segments); |
122 | 249 | ||
@@ -356,12 +483,12 @@ static bool req_no_special_merge(struct request *req) | |||
356 | return !q->mq_ops && req->special; | 483 | return !q->mq_ops && req->special; |
357 | } | 484 | } |
358 | 485 | ||
359 | static int req_gap_to_prev(struct request *req, struct request *next) | 486 | static int req_gap_to_prev(struct request *req, struct bio *next) |
360 | { | 487 | { |
361 | struct bio *prev = req->biotail; | 488 | struct bio *prev = req->biotail; |
362 | 489 | ||
363 | return bvec_gap_to_prev(&prev->bi_io_vec[prev->bi_vcnt - 1], | 490 | return bvec_gap_to_prev(req->q, &prev->bi_io_vec[prev->bi_vcnt - 1], |
364 | next->bio->bi_io_vec[0].bv_offset); | 491 | next->bi_io_vec[0].bv_offset); |
365 | } | 492 | } |
366 | 493 | ||
367 | static int ll_merge_requests_fn(struct request_queue *q, struct request *req, | 494 | static int ll_merge_requests_fn(struct request_queue *q, struct request *req, |
@@ -378,8 +505,7 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req, | |||
378 | if (req_no_special_merge(req) || req_no_special_merge(next)) | 505 | if (req_no_special_merge(req) || req_no_special_merge(next)) |
379 | return 0; | 506 | return 0; |
380 | 507 | ||
381 | if (test_bit(QUEUE_FLAG_SG_GAPS, &q->queue_flags) && | 508 | if (req_gap_to_prev(req, next->bio)) |
382 | req_gap_to_prev(req, next)) | ||
383 | return 0; | 509 | return 0; |
384 | 510 | ||
385 | /* | 511 | /* |
@@ -564,8 +690,6 @@ int blk_attempt_req_merge(struct request_queue *q, struct request *rq, | |||
564 | 690 | ||
565 | bool blk_rq_merge_ok(struct request *rq, struct bio *bio) | 691 | bool blk_rq_merge_ok(struct request *rq, struct bio *bio) |
566 | { | 692 | { |
567 | struct request_queue *q = rq->q; | ||
568 | |||
569 | if (!rq_mergeable(rq) || !bio_mergeable(bio)) | 693 | if (!rq_mergeable(rq) || !bio_mergeable(bio)) |
570 | return false; | 694 | return false; |
571 | 695 | ||
@@ -590,13 +714,8 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) | |||
590 | return false; | 714 | return false; |
591 | 715 | ||
592 | /* Only check gaps if the bio carries data */ | 716 | /* Only check gaps if the bio carries data */ |
593 | if (q->queue_flags & (1 << QUEUE_FLAG_SG_GAPS) && bio_has_data(bio)) { | 717 | if (bio_has_data(bio) && req_gap_to_prev(rq, bio)) |
594 | struct bio_vec *bprev; | 718 | return false; |
595 | |||
596 | bprev = &rq->biotail->bi_io_vec[rq->biotail->bi_vcnt - 1]; | ||
597 | if (bvec_gap_to_prev(bprev, bio->bi_io_vec[0].bv_offset)) | ||
598 | return false; | ||
599 | } | ||
600 | 719 | ||
601 | return true; | 720 | return true; |
602 | } | 721 | } |
diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c index b79685e06b70..279c5d674edf 100644 --- a/block/blk-mq-sysfs.c +++ b/block/blk-mq-sysfs.c | |||
@@ -141,15 +141,26 @@ static ssize_t blk_mq_sysfs_completed_show(struct blk_mq_ctx *ctx, char *page) | |||
141 | 141 | ||
142 | static ssize_t sysfs_list_show(char *page, struct list_head *list, char *msg) | 142 | static ssize_t sysfs_list_show(char *page, struct list_head *list, char *msg) |
143 | { | 143 | { |
144 | char *start_page = page; | ||
145 | struct request *rq; | 144 | struct request *rq; |
145 | int len = snprintf(page, PAGE_SIZE - 1, "%s:\n", msg); | ||
146 | |||
147 | list_for_each_entry(rq, list, queuelist) { | ||
148 | const int rq_len = 2 * sizeof(rq) + 2; | ||
149 | |||
150 | /* if the output will be truncated */ | ||
151 | if (PAGE_SIZE - 1 < len + rq_len) { | ||
152 | /* backspacing if it can't hold '\t...\n' */ | ||
153 | if (PAGE_SIZE - 1 < len + 5) | ||
154 | len -= rq_len; | ||
155 | len += snprintf(page + len, PAGE_SIZE - 1 - len, | ||
156 | "\t...\n"); | ||
157 | break; | ||
158 | } | ||
159 | len += snprintf(page + len, PAGE_SIZE - 1 - len, | ||
160 | "\t%p\n", rq); | ||
161 | } | ||
146 | 162 | ||
147 | page += sprintf(page, "%s:\n", msg); | 163 | return len; |
148 | |||
149 | list_for_each_entry(rq, list, queuelist) | ||
150 | page += sprintf(page, "\t%p\n", rq); | ||
151 | |||
152 | return page - start_page; | ||
153 | } | 164 | } |
154 | 165 | ||
155 | static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx *ctx, char *page) | 166 | static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx *ctx, char *page) |
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 9b6e28830b82..9115c6d59948 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c | |||
@@ -429,7 +429,7 @@ static void bt_for_each(struct blk_mq_hw_ctx *hctx, | |||
429 | for (bit = find_first_bit(&bm->word, bm->depth); | 429 | for (bit = find_first_bit(&bm->word, bm->depth); |
430 | bit < bm->depth; | 430 | bit < bm->depth; |
431 | bit = find_next_bit(&bm->word, bm->depth, bit + 1)) { | 431 | bit = find_next_bit(&bm->word, bm->depth, bit + 1)) { |
432 | rq = blk_mq_tag_to_rq(hctx->tags, off + bit); | 432 | rq = hctx->tags->rqs[off + bit]; |
433 | if (rq->q == hctx->queue) | 433 | if (rq->q == hctx->queue) |
434 | fn(hctx, rq, data, reserved); | 434 | fn(hctx, rq, data, reserved); |
435 | } | 435 | } |
@@ -453,7 +453,7 @@ static void bt_tags_for_each(struct blk_mq_tags *tags, | |||
453 | for (bit = find_first_bit(&bm->word, bm->depth); | 453 | for (bit = find_first_bit(&bm->word, bm->depth); |
454 | bit < bm->depth; | 454 | bit < bm->depth; |
455 | bit = find_next_bit(&bm->word, bm->depth, bit + 1)) { | 455 | bit = find_next_bit(&bm->word, bm->depth, bit + 1)) { |
456 | rq = blk_mq_tag_to_rq(tags, off + bit); | 456 | rq = tags->rqs[off + bit]; |
457 | fn(rq, data, reserved); | 457 | fn(rq, data, reserved); |
458 | } | 458 | } |
459 | 459 | ||
diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h index 75893a34237d..9eb2cf4f01cb 100644 --- a/block/blk-mq-tag.h +++ b/block/blk-mq-tag.h | |||
@@ -89,4 +89,16 @@ static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) | |||
89 | __blk_mq_tag_idle(hctx); | 89 | __blk_mq_tag_idle(hctx); |
90 | } | 90 | } |
91 | 91 | ||
92 | /* | ||
93 | * This helper should only be used for flush request to share tag | ||
94 | * with the request cloned from, and both the two requests can't be | ||
95 | * in flight at the same time. The caller has to make sure the tag | ||
96 | * can't be freed. | ||
97 | */ | ||
98 | static inline void blk_mq_tag_set_rq(struct blk_mq_hw_ctx *hctx, | ||
99 | unsigned int tag, struct request *rq) | ||
100 | { | ||
101 | hctx->tags->rqs[tag] = rq; | ||
102 | } | ||
103 | |||
92 | #endif | 104 | #endif |
diff --git a/block/blk-mq.c b/block/blk-mq.c index 7d842db59699..f2d67b4047a0 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c | |||
@@ -559,23 +559,9 @@ void blk_mq_abort_requeue_list(struct request_queue *q) | |||
559 | } | 559 | } |
560 | EXPORT_SYMBOL(blk_mq_abort_requeue_list); | 560 | EXPORT_SYMBOL(blk_mq_abort_requeue_list); |
561 | 561 | ||
562 | static inline bool is_flush_request(struct request *rq, | ||
563 | struct blk_flush_queue *fq, unsigned int tag) | ||
564 | { | ||
565 | return ((rq->cmd_flags & REQ_FLUSH_SEQ) && | ||
566 | fq->flush_rq->tag == tag); | ||
567 | } | ||
568 | |||
569 | struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) | 562 | struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) |
570 | { | 563 | { |
571 | struct request *rq = tags->rqs[tag]; | 564 | return tags->rqs[tag]; |
572 | /* mq_ctx of flush rq is always cloned from the corresponding req */ | ||
573 | struct blk_flush_queue *fq = blk_get_flush_queue(rq->q, rq->mq_ctx); | ||
574 | |||
575 | if (!is_flush_request(rq, fq, tag)) | ||
576 | return rq; | ||
577 | |||
578 | return fq->flush_rq; | ||
579 | } | 565 | } |
580 | EXPORT_SYMBOL(blk_mq_tag_to_rq); | 566 | EXPORT_SYMBOL(blk_mq_tag_to_rq); |
581 | 567 | ||
@@ -1199,7 +1185,7 @@ static struct request *blk_mq_map_request(struct request_queue *q, | |||
1199 | struct blk_mq_alloc_data alloc_data; | 1185 | struct blk_mq_alloc_data alloc_data; |
1200 | 1186 | ||
1201 | if (unlikely(blk_mq_queue_enter(q, GFP_KERNEL))) { | 1187 | if (unlikely(blk_mq_queue_enter(q, GFP_KERNEL))) { |
1202 | bio_endio(bio, -EIO); | 1188 | bio_io_error(bio); |
1203 | return NULL; | 1189 | return NULL; |
1204 | } | 1190 | } |
1205 | 1191 | ||
@@ -1283,10 +1269,12 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio) | |||
1283 | blk_queue_bounce(q, &bio); | 1269 | blk_queue_bounce(q, &bio); |
1284 | 1270 | ||
1285 | if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { | 1271 | if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { |
1286 | bio_endio(bio, -EIO); | 1272 | bio_io_error(bio); |
1287 | return; | 1273 | return; |
1288 | } | 1274 | } |
1289 | 1275 | ||
1276 | blk_queue_split(q, &bio, q->bio_split); | ||
1277 | |||
1290 | if (!is_flush_fua && !blk_queue_nomerges(q) && | 1278 | if (!is_flush_fua && !blk_queue_nomerges(q) && |
1291 | blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq)) | 1279 | blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq)) |
1292 | return; | 1280 | return; |
@@ -1368,10 +1356,12 @@ static void blk_sq_make_request(struct request_queue *q, struct bio *bio) | |||
1368 | blk_queue_bounce(q, &bio); | 1356 | blk_queue_bounce(q, &bio); |
1369 | 1357 | ||
1370 | if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { | 1358 | if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { |
1371 | bio_endio(bio, -EIO); | 1359 | bio_io_error(bio); |
1372 | return; | 1360 | return; |
1373 | } | 1361 | } |
1374 | 1362 | ||
1363 | blk_queue_split(q, &bio, q->bio_split); | ||
1364 | |||
1375 | if (!is_flush_fua && !blk_queue_nomerges(q) && | 1365 | if (!is_flush_fua && !blk_queue_nomerges(q) && |
1376 | blk_attempt_plug_merge(q, bio, &request_count, NULL)) | 1366 | blk_attempt_plug_merge(q, bio, &request_count, NULL)) |
1377 | return; | 1367 | return; |
diff --git a/block/blk-settings.c b/block/blk-settings.c index e0057d035200..7d8f129a1516 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c | |||
@@ -53,28 +53,6 @@ void blk_queue_unprep_rq(struct request_queue *q, unprep_rq_fn *ufn) | |||
53 | } | 53 | } |
54 | EXPORT_SYMBOL(blk_queue_unprep_rq); | 54 | EXPORT_SYMBOL(blk_queue_unprep_rq); |
55 | 55 | ||
56 | /** | ||
57 | * blk_queue_merge_bvec - set a merge_bvec function for queue | ||
58 | * @q: queue | ||
59 | * @mbfn: merge_bvec_fn | ||
60 | * | ||
61 | * Usually queues have static limitations on the max sectors or segments that | ||
62 | * we can put in a request. Stacking drivers may have some settings that | ||
63 | * are dynamic, and thus we have to query the queue whether it is ok to | ||
64 | * add a new bio_vec to a bio at a given offset or not. If the block device | ||
65 | * has such limitations, it needs to register a merge_bvec_fn to control | ||
66 | * the size of bio's sent to it. Note that a block device *must* allow a | ||
67 | * single page to be added to an empty bio. The block device driver may want | ||
68 | * to use the bio_split() function to deal with these bio's. By default | ||
69 | * no merge_bvec_fn is defined for a queue, and only the fixed limits are | ||
70 | * honored. | ||
71 | */ | ||
72 | void blk_queue_merge_bvec(struct request_queue *q, merge_bvec_fn *mbfn) | ||
73 | { | ||
74 | q->merge_bvec_fn = mbfn; | ||
75 | } | ||
76 | EXPORT_SYMBOL(blk_queue_merge_bvec); | ||
77 | |||
78 | void blk_queue_softirq_done(struct request_queue *q, softirq_done_fn *fn) | 56 | void blk_queue_softirq_done(struct request_queue *q, softirq_done_fn *fn) |
79 | { | 57 | { |
80 | q->softirq_done_fn = fn; | 58 | q->softirq_done_fn = fn; |
@@ -111,11 +89,13 @@ void blk_set_default_limits(struct queue_limits *lim) | |||
111 | lim->max_segments = BLK_MAX_SEGMENTS; | 89 | lim->max_segments = BLK_MAX_SEGMENTS; |
112 | lim->max_integrity_segments = 0; | 90 | lim->max_integrity_segments = 0; |
113 | lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK; | 91 | lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK; |
92 | lim->virt_boundary_mask = 0; | ||
114 | lim->max_segment_size = BLK_MAX_SEGMENT_SIZE; | 93 | lim->max_segment_size = BLK_MAX_SEGMENT_SIZE; |
115 | lim->max_sectors = lim->max_hw_sectors = BLK_SAFE_MAX_SECTORS; | 94 | lim->max_sectors = lim->max_hw_sectors = BLK_SAFE_MAX_SECTORS; |
116 | lim->chunk_sectors = 0; | 95 | lim->chunk_sectors = 0; |
117 | lim->max_write_same_sectors = 0; | 96 | lim->max_write_same_sectors = 0; |
118 | lim->max_discard_sectors = 0; | 97 | lim->max_discard_sectors = 0; |
98 | lim->max_hw_discard_sectors = 0; | ||
119 | lim->discard_granularity = 0; | 99 | lim->discard_granularity = 0; |
120 | lim->discard_alignment = 0; | 100 | lim->discard_alignment = 0; |
121 | lim->discard_misaligned = 0; | 101 | lim->discard_misaligned = 0; |
@@ -257,7 +237,9 @@ void blk_limits_max_hw_sectors(struct queue_limits *limits, unsigned int max_hw_ | |||
257 | __func__, max_hw_sectors); | 237 | __func__, max_hw_sectors); |
258 | } | 238 | } |
259 | 239 | ||
260 | limits->max_sectors = limits->max_hw_sectors = max_hw_sectors; | 240 | limits->max_hw_sectors = max_hw_sectors; |
241 | limits->max_sectors = min_t(unsigned int, max_hw_sectors, | ||
242 | BLK_DEF_MAX_SECTORS); | ||
261 | } | 243 | } |
262 | EXPORT_SYMBOL(blk_limits_max_hw_sectors); | 244 | EXPORT_SYMBOL(blk_limits_max_hw_sectors); |
263 | 245 | ||
@@ -303,6 +285,7 @@ EXPORT_SYMBOL(blk_queue_chunk_sectors); | |||
303 | void blk_queue_max_discard_sectors(struct request_queue *q, | 285 | void blk_queue_max_discard_sectors(struct request_queue *q, |
304 | unsigned int max_discard_sectors) | 286 | unsigned int max_discard_sectors) |
305 | { | 287 | { |
288 | q->limits.max_hw_discard_sectors = max_discard_sectors; | ||
306 | q->limits.max_discard_sectors = max_discard_sectors; | 289 | q->limits.max_discard_sectors = max_discard_sectors; |
307 | } | 290 | } |
308 | EXPORT_SYMBOL(blk_queue_max_discard_sectors); | 291 | EXPORT_SYMBOL(blk_queue_max_discard_sectors); |
@@ -550,6 +533,8 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, | |||
550 | 533 | ||
551 | t->seg_boundary_mask = min_not_zero(t->seg_boundary_mask, | 534 | t->seg_boundary_mask = min_not_zero(t->seg_boundary_mask, |
552 | b->seg_boundary_mask); | 535 | b->seg_boundary_mask); |
536 | t->virt_boundary_mask = min_not_zero(t->virt_boundary_mask, | ||
537 | b->virt_boundary_mask); | ||
553 | 538 | ||
554 | t->max_segments = min_not_zero(t->max_segments, b->max_segments); | 539 | t->max_segments = min_not_zero(t->max_segments, b->max_segments); |
555 | t->max_integrity_segments = min_not_zero(t->max_integrity_segments, | 540 | t->max_integrity_segments = min_not_zero(t->max_integrity_segments, |
@@ -641,6 +626,8 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, | |||
641 | 626 | ||
642 | t->max_discard_sectors = min_not_zero(t->max_discard_sectors, | 627 | t->max_discard_sectors = min_not_zero(t->max_discard_sectors, |
643 | b->max_discard_sectors); | 628 | b->max_discard_sectors); |
629 | t->max_hw_discard_sectors = min_not_zero(t->max_hw_discard_sectors, | ||
630 | b->max_hw_discard_sectors); | ||
644 | t->discard_granularity = max(t->discard_granularity, | 631 | t->discard_granularity = max(t->discard_granularity, |
645 | b->discard_granularity); | 632 | b->discard_granularity); |
646 | t->discard_alignment = lcm_not_zero(t->discard_alignment, alignment) % | 633 | t->discard_alignment = lcm_not_zero(t->discard_alignment, alignment) % |
@@ -788,6 +775,17 @@ void blk_queue_segment_boundary(struct request_queue *q, unsigned long mask) | |||
788 | EXPORT_SYMBOL(blk_queue_segment_boundary); | 775 | EXPORT_SYMBOL(blk_queue_segment_boundary); |
789 | 776 | ||
790 | /** | 777 | /** |
778 | * blk_queue_virt_boundary - set boundary rules for bio merging | ||
779 | * @q: the request queue for the device | ||
780 | * @mask: the memory boundary mask | ||
781 | **/ | ||
782 | void blk_queue_virt_boundary(struct request_queue *q, unsigned long mask) | ||
783 | { | ||
784 | q->limits.virt_boundary_mask = mask; | ||
785 | } | ||
786 | EXPORT_SYMBOL(blk_queue_virt_boundary); | ||
787 | |||
788 | /** | ||
791 | * blk_queue_dma_alignment - set dma length and memory alignment | 789 | * blk_queue_dma_alignment - set dma length and memory alignment |
792 | * @q: the request queue for the device | 790 | * @q: the request queue for the device |
793 | * @mask: alignment mask | 791 | * @mask: alignment mask |
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 6264b382d4d1..3e44a9da2a13 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c | |||
@@ -145,12 +145,43 @@ static ssize_t queue_discard_granularity_show(struct request_queue *q, char *pag | |||
145 | return queue_var_show(q->limits.discard_granularity, page); | 145 | return queue_var_show(q->limits.discard_granularity, page); |
146 | } | 146 | } |
147 | 147 | ||
148 | static ssize_t queue_discard_max_hw_show(struct request_queue *q, char *page) | ||
149 | { | ||
150 | unsigned long long val; | ||
151 | |||
152 | val = q->limits.max_hw_discard_sectors << 9; | ||
153 | return sprintf(page, "%llu\n", val); | ||
154 | } | ||
155 | |||
148 | static ssize_t queue_discard_max_show(struct request_queue *q, char *page) | 156 | static ssize_t queue_discard_max_show(struct request_queue *q, char *page) |
149 | { | 157 | { |
150 | return sprintf(page, "%llu\n", | 158 | return sprintf(page, "%llu\n", |
151 | (unsigned long long)q->limits.max_discard_sectors << 9); | 159 | (unsigned long long)q->limits.max_discard_sectors << 9); |
152 | } | 160 | } |
153 | 161 | ||
162 | static ssize_t queue_discard_max_store(struct request_queue *q, | ||
163 | const char *page, size_t count) | ||
164 | { | ||
165 | unsigned long max_discard; | ||
166 | ssize_t ret = queue_var_store(&max_discard, page, count); | ||
167 | |||
168 | if (ret < 0) | ||
169 | return ret; | ||
170 | |||
171 | if (max_discard & (q->limits.discard_granularity - 1)) | ||
172 | return -EINVAL; | ||
173 | |||
174 | max_discard >>= 9; | ||
175 | if (max_discard > UINT_MAX) | ||
176 | return -EINVAL; | ||
177 | |||
178 | if (max_discard > q->limits.max_hw_discard_sectors) | ||
179 | max_discard = q->limits.max_hw_discard_sectors; | ||
180 | |||
181 | q->limits.max_discard_sectors = max_discard; | ||
182 | return ret; | ||
183 | } | ||
184 | |||
154 | static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page) | 185 | static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page) |
155 | { | 186 | { |
156 | return queue_var_show(queue_discard_zeroes_data(q), page); | 187 | return queue_var_show(queue_discard_zeroes_data(q), page); |
@@ -360,9 +391,15 @@ static struct queue_sysfs_entry queue_discard_granularity_entry = { | |||
360 | .show = queue_discard_granularity_show, | 391 | .show = queue_discard_granularity_show, |
361 | }; | 392 | }; |
362 | 393 | ||
394 | static struct queue_sysfs_entry queue_discard_max_hw_entry = { | ||
395 | .attr = {.name = "discard_max_hw_bytes", .mode = S_IRUGO }, | ||
396 | .show = queue_discard_max_hw_show, | ||
397 | }; | ||
398 | |||
363 | static struct queue_sysfs_entry queue_discard_max_entry = { | 399 | static struct queue_sysfs_entry queue_discard_max_entry = { |
364 | .attr = {.name = "discard_max_bytes", .mode = S_IRUGO }, | 400 | .attr = {.name = "discard_max_bytes", .mode = S_IRUGO | S_IWUSR }, |
365 | .show = queue_discard_max_show, | 401 | .show = queue_discard_max_show, |
402 | .store = queue_discard_max_store, | ||
366 | }; | 403 | }; |
367 | 404 | ||
368 | static struct queue_sysfs_entry queue_discard_zeroes_data_entry = { | 405 | static struct queue_sysfs_entry queue_discard_zeroes_data_entry = { |
@@ -421,6 +458,7 @@ static struct attribute *default_attrs[] = { | |||
421 | &queue_io_opt_entry.attr, | 458 | &queue_io_opt_entry.attr, |
422 | &queue_discard_granularity_entry.attr, | 459 | &queue_discard_granularity_entry.attr, |
423 | &queue_discard_max_entry.attr, | 460 | &queue_discard_max_entry.attr, |
461 | &queue_discard_max_hw_entry.attr, | ||
424 | &queue_discard_zeroes_data_entry.attr, | 462 | &queue_discard_zeroes_data_entry.attr, |
425 | &queue_write_same_max_entry.attr, | 463 | &queue_write_same_max_entry.attr, |
426 | &queue_nonrot_entry.attr, | 464 | &queue_nonrot_entry.attr, |
@@ -523,6 +561,9 @@ static void blk_release_queue(struct kobject *kobj) | |||
523 | 561 | ||
524 | blk_trace_shutdown(q); | 562 | blk_trace_shutdown(q); |
525 | 563 | ||
564 | if (q->bio_split) | ||
565 | bioset_free(q->bio_split); | ||
566 | |||
526 | ida_simple_remove(&blk_queue_ida, q->id); | 567 | ida_simple_remove(&blk_queue_ida, q->id); |
527 | call_rcu(&q->rcu_head, blk_free_queue_rcu); | 568 | call_rcu(&q->rcu_head, blk_free_queue_rcu); |
528 | } | 569 | } |
diff --git a/block/blk.h b/block/blk.h index 026d9594142b..838188b35a83 100644 --- a/block/blk.h +++ b/block/blk.h | |||
@@ -22,6 +22,12 @@ struct blk_flush_queue { | |||
22 | struct list_head flush_queue[2]; | 22 | struct list_head flush_queue[2]; |
23 | struct list_head flush_data_in_flight; | 23 | struct list_head flush_data_in_flight; |
24 | struct request *flush_rq; | 24 | struct request *flush_rq; |
25 | |||
26 | /* | ||
27 | * flush_rq shares tag with this rq, both can't be active | ||
28 | * at the same time | ||
29 | */ | ||
30 | struct request *orig_rq; | ||
25 | spinlock_t mq_flush_lock; | 31 | spinlock_t mq_flush_lock; |
26 | }; | 32 | }; |
27 | 33 | ||
diff --git a/block/bounce.c b/block/bounce.c index b17311227c12..2c310ea007ee 100644 --- a/block/bounce.c +++ b/block/bounce.c | |||
@@ -123,7 +123,7 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio *from) | |||
123 | } | 123 | } |
124 | } | 124 | } |
125 | 125 | ||
126 | static void bounce_end_io(struct bio *bio, mempool_t *pool, int err) | 126 | static void bounce_end_io(struct bio *bio, mempool_t *pool) |
127 | { | 127 | { |
128 | struct bio *bio_orig = bio->bi_private; | 128 | struct bio *bio_orig = bio->bi_private; |
129 | struct bio_vec *bvec, *org_vec; | 129 | struct bio_vec *bvec, *org_vec; |
@@ -141,39 +141,40 @@ static void bounce_end_io(struct bio *bio, mempool_t *pool, int err) | |||
141 | mempool_free(bvec->bv_page, pool); | 141 | mempool_free(bvec->bv_page, pool); |
142 | } | 142 | } |
143 | 143 | ||
144 | bio_endio(bio_orig, err); | 144 | bio_orig->bi_error = bio->bi_error; |
145 | bio_endio(bio_orig); | ||
145 | bio_put(bio); | 146 | bio_put(bio); |
146 | } | 147 | } |
147 | 148 | ||
148 | static void bounce_end_io_write(struct bio *bio, int err) | 149 | static void bounce_end_io_write(struct bio *bio) |
149 | { | 150 | { |
150 | bounce_end_io(bio, page_pool, err); | 151 | bounce_end_io(bio, page_pool); |
151 | } | 152 | } |
152 | 153 | ||
153 | static void bounce_end_io_write_isa(struct bio *bio, int err) | 154 | static void bounce_end_io_write_isa(struct bio *bio) |
154 | { | 155 | { |
155 | 156 | ||
156 | bounce_end_io(bio, isa_page_pool, err); | 157 | bounce_end_io(bio, isa_page_pool); |
157 | } | 158 | } |
158 | 159 | ||
159 | static void __bounce_end_io_read(struct bio *bio, mempool_t *pool, int err) | 160 | static void __bounce_end_io_read(struct bio *bio, mempool_t *pool) |
160 | { | 161 | { |
161 | struct bio *bio_orig = bio->bi_private; | 162 | struct bio *bio_orig = bio->bi_private; |
162 | 163 | ||
163 | if (test_bit(BIO_UPTODATE, &bio->bi_flags)) | 164 | if (!bio->bi_error) |
164 | copy_to_high_bio_irq(bio_orig, bio); | 165 | copy_to_high_bio_irq(bio_orig, bio); |
165 | 166 | ||
166 | bounce_end_io(bio, pool, err); | 167 | bounce_end_io(bio, pool); |
167 | } | 168 | } |
168 | 169 | ||
169 | static void bounce_end_io_read(struct bio *bio, int err) | 170 | static void bounce_end_io_read(struct bio *bio) |
170 | { | 171 | { |
171 | __bounce_end_io_read(bio, page_pool, err); | 172 | __bounce_end_io_read(bio, page_pool); |
172 | } | 173 | } |
173 | 174 | ||
174 | static void bounce_end_io_read_isa(struct bio *bio, int err) | 175 | static void bounce_end_io_read_isa(struct bio *bio) |
175 | { | 176 | { |
176 | __bounce_end_io_read(bio, isa_page_pool, err); | 177 | __bounce_end_io_read(bio, isa_page_pool); |
177 | } | 178 | } |
178 | 179 | ||
179 | #ifdef CONFIG_NEED_BOUNCE_POOL | 180 | #ifdef CONFIG_NEED_BOUNCE_POOL |
@@ -185,7 +186,7 @@ static int must_snapshot_stable_pages(struct request_queue *q, struct bio *bio) | |||
185 | if (!bdi_cap_stable_pages_required(&q->backing_dev_info)) | 186 | if (!bdi_cap_stable_pages_required(&q->backing_dev_info)) |
186 | return 0; | 187 | return 0; |
187 | 188 | ||
188 | return test_bit(BIO_SNAP_STABLE, &bio->bi_flags); | 189 | return bio_flagged(bio, BIO_SNAP_STABLE); |
189 | } | 190 | } |
190 | #else | 191 | #else |
191 | static int must_snapshot_stable_pages(struct request_queue *q, struct bio *bio) | 192 | static int must_snapshot_stable_pages(struct request_queue *q, struct bio *bio) |
diff --git a/block/genhd.c b/block/genhd.c index 59a1395eedac..0c706f33a599 100644 --- a/block/genhd.c +++ b/block/genhd.c | |||
@@ -1110,8 +1110,7 @@ static void disk_release(struct device *dev) | |||
1110 | disk_release_events(disk); | 1110 | disk_release_events(disk); |
1111 | kfree(disk->random); | 1111 | kfree(disk->random); |
1112 | disk_replace_part_tbl(disk, NULL); | 1112 | disk_replace_part_tbl(disk, NULL); |
1113 | free_part_stats(&disk->part0); | 1113 | hd_free_part(&disk->part0); |
1114 | free_part_info(&disk->part0); | ||
1115 | if (disk->queue) | 1114 | if (disk->queue) |
1116 | blk_put_queue(disk->queue); | 1115 | blk_put_queue(disk->queue); |
1117 | kfree(disk); | 1116 | kfree(disk); |
@@ -1285,7 +1284,11 @@ struct gendisk *alloc_disk_node(int minors, int node_id) | |||
1285 | * converted to make use of bd_mutex and sequence counters. | 1284 | * converted to make use of bd_mutex and sequence counters. |
1286 | */ | 1285 | */ |
1287 | seqcount_init(&disk->part0.nr_sects_seq); | 1286 | seqcount_init(&disk->part0.nr_sects_seq); |
1288 | hd_ref_init(&disk->part0); | 1287 | if (hd_ref_init(&disk->part0)) { |
1288 | hd_free_part(&disk->part0); | ||
1289 | kfree(disk); | ||
1290 | return NULL; | ||
1291 | } | ||
1289 | 1292 | ||
1290 | disk->minors = minors; | 1293 | disk->minors = minors; |
1291 | rand_initialize_disk(disk); | 1294 | rand_initialize_disk(disk); |
diff --git a/block/partition-generic.c b/block/partition-generic.c index 0d9e5f97f0a8..e7711133284e 100644 --- a/block/partition-generic.c +++ b/block/partition-generic.c | |||
@@ -212,8 +212,7 @@ static void part_release(struct device *dev) | |||
212 | { | 212 | { |
213 | struct hd_struct *p = dev_to_part(dev); | 213 | struct hd_struct *p = dev_to_part(dev); |
214 | blk_free_devt(dev->devt); | 214 | blk_free_devt(dev->devt); |
215 | free_part_stats(p); | 215 | hd_free_part(p); |
216 | free_part_info(p); | ||
217 | kfree(p); | 216 | kfree(p); |
218 | } | 217 | } |
219 | 218 | ||
@@ -233,8 +232,9 @@ static void delete_partition_rcu_cb(struct rcu_head *head) | |||
233 | put_device(part_to_dev(part)); | 232 | put_device(part_to_dev(part)); |
234 | } | 233 | } |
235 | 234 | ||
236 | void __delete_partition(struct hd_struct *part) | 235 | void __delete_partition(struct percpu_ref *ref) |
237 | { | 236 | { |
237 | struct hd_struct *part = container_of(ref, struct hd_struct, ref); | ||
238 | call_rcu(&part->rcu_head, delete_partition_rcu_cb); | 238 | call_rcu(&part->rcu_head, delete_partition_rcu_cb); |
239 | } | 239 | } |
240 | 240 | ||
@@ -255,7 +255,7 @@ void delete_partition(struct gendisk *disk, int partno) | |||
255 | kobject_put(part->holder_dir); | 255 | kobject_put(part->holder_dir); |
256 | device_del(part_to_dev(part)); | 256 | device_del(part_to_dev(part)); |
257 | 257 | ||
258 | hd_struct_put(part); | 258 | hd_struct_kill(part); |
259 | } | 259 | } |
260 | 260 | ||
261 | static ssize_t whole_disk_show(struct device *dev, | 261 | static ssize_t whole_disk_show(struct device *dev, |
@@ -356,8 +356,8 @@ struct hd_struct *add_partition(struct gendisk *disk, int partno, | |||
356 | if (!dev_get_uevent_suppress(ddev)) | 356 | if (!dev_get_uevent_suppress(ddev)) |
357 | kobject_uevent(&pdev->kobj, KOBJ_ADD); | 357 | kobject_uevent(&pdev->kobj, KOBJ_ADD); |
358 | 358 | ||
359 | hd_ref_init(p); | 359 | if (!hd_ref_init(p)) |
360 | return p; | 360 | return p; |
361 | 361 | ||
362 | out_free_info: | 362 | out_free_info: |
363 | free_part_info(p); | 363 | free_part_info(p); |
diff --git a/drivers/block/aoe/aoeblk.c b/drivers/block/aoe/aoeblk.c index 46c282fff104..dd73e1ff1759 100644 --- a/drivers/block/aoe/aoeblk.c +++ b/drivers/block/aoe/aoeblk.c | |||
@@ -395,7 +395,7 @@ aoeblk_gdalloc(void *vp) | |||
395 | WARN_ON(d->flags & DEVFL_TKILL); | 395 | WARN_ON(d->flags & DEVFL_TKILL); |
396 | WARN_ON(d->gd); | 396 | WARN_ON(d->gd); |
397 | WARN_ON(d->flags & DEVFL_UP); | 397 | WARN_ON(d->flags & DEVFL_UP); |
398 | blk_queue_max_hw_sectors(q, 1024); | 398 | blk_queue_max_hw_sectors(q, BLK_DEF_MAX_SECTORS); |
399 | q->backing_dev_info.name = "aoe"; | 399 | q->backing_dev_info.name = "aoe"; |
400 | q->backing_dev_info.ra_pages = READ_AHEAD / PAGE_CACHE_SIZE; | 400 | q->backing_dev_info.ra_pages = READ_AHEAD / PAGE_CACHE_SIZE; |
401 | d->bufpool = mp; | 401 | d->bufpool = mp; |
diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c index 422b7d84f686..ad80c85e0857 100644 --- a/drivers/block/aoe/aoecmd.c +++ b/drivers/block/aoe/aoecmd.c | |||
@@ -1110,7 +1110,7 @@ aoe_end_request(struct aoedev *d, struct request *rq, int fastfail) | |||
1110 | d->ip.rq = NULL; | 1110 | d->ip.rq = NULL; |
1111 | do { | 1111 | do { |
1112 | bio = rq->bio; | 1112 | bio = rq->bio; |
1113 | bok = !fastfail && test_bit(BIO_UPTODATE, &bio->bi_flags); | 1113 | bok = !fastfail && !bio->bi_error; |
1114 | } while (__blk_end_request(rq, bok ? 0 : -EIO, bio->bi_iter.bi_size)); | 1114 | } while (__blk_end_request(rq, bok ? 0 : -EIO, bio->bi_iter.bi_size)); |
1115 | 1115 | ||
1116 | /* cf. http://lkml.org/lkml/2006/10/31/28 */ | 1116 | /* cf. http://lkml.org/lkml/2006/10/31/28 */ |
@@ -1172,7 +1172,7 @@ ktiocomplete(struct frame *f) | |||
1172 | ahout->cmdstat, ahin->cmdstat, | 1172 | ahout->cmdstat, ahin->cmdstat, |
1173 | d->aoemajor, d->aoeminor); | 1173 | d->aoemajor, d->aoeminor); |
1174 | noskb: if (buf) | 1174 | noskb: if (buf) |
1175 | clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); | 1175 | buf->bio->bi_error = -EIO; |
1176 | goto out; | 1176 | goto out; |
1177 | } | 1177 | } |
1178 | 1178 | ||
@@ -1185,7 +1185,7 @@ noskb: if (buf) | |||
1185 | "aoe: runt data size in read from", | 1185 | "aoe: runt data size in read from", |
1186 | (long) d->aoemajor, d->aoeminor, | 1186 | (long) d->aoemajor, d->aoeminor, |
1187 | skb->len, n); | 1187 | skb->len, n); |
1188 | clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); | 1188 | buf->bio->bi_error = -EIO; |
1189 | break; | 1189 | break; |
1190 | } | 1190 | } |
1191 | if (n > f->iter.bi_size) { | 1191 | if (n > f->iter.bi_size) { |
@@ -1193,7 +1193,7 @@ noskb: if (buf) | |||
1193 | "aoe: too-large data size in read from", | 1193 | "aoe: too-large data size in read from", |
1194 | (long) d->aoemajor, d->aoeminor, | 1194 | (long) d->aoemajor, d->aoeminor, |
1195 | n, f->iter.bi_size); | 1195 | n, f->iter.bi_size); |
1196 | clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); | 1196 | buf->bio->bi_error = -EIO; |
1197 | break; | 1197 | break; |
1198 | } | 1198 | } |
1199 | bvcpy(skb, f->buf->bio, f->iter, n); | 1199 | bvcpy(skb, f->buf->bio, f->iter, n); |
@@ -1695,7 +1695,7 @@ aoe_failbuf(struct aoedev *d, struct buf *buf) | |||
1695 | if (buf == NULL) | 1695 | if (buf == NULL) |
1696 | return; | 1696 | return; |
1697 | buf->iter.bi_size = 0; | 1697 | buf->iter.bi_size = 0; |
1698 | clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); | 1698 | buf->bio->bi_error = -EIO; |
1699 | if (buf->nframesout == 0) | 1699 | if (buf->nframesout == 0) |
1700 | aoe_end_buf(d, buf); | 1700 | aoe_end_buf(d, buf); |
1701 | } | 1701 | } |
diff --git a/drivers/block/aoe/aoedev.c b/drivers/block/aoe/aoedev.c index e774c50b6842..ffd1947500c6 100644 --- a/drivers/block/aoe/aoedev.c +++ b/drivers/block/aoe/aoedev.c | |||
@@ -170,7 +170,7 @@ aoe_failip(struct aoedev *d) | |||
170 | if (rq == NULL) | 170 | if (rq == NULL) |
171 | return; | 171 | return; |
172 | while ((bio = d->ip.nxbio)) { | 172 | while ((bio = d->ip.nxbio)) { |
173 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | 173 | bio->bi_error = -EIO; |
174 | d->ip.nxbio = bio->bi_next; | 174 | d->ip.nxbio = bio->bi_next; |
175 | n = (unsigned long) rq->special; | 175 | n = (unsigned long) rq->special; |
176 | rq->special = (void *) --n; | 176 | rq->special = (void *) --n; |
diff --git a/drivers/block/brd.c b/drivers/block/brd.c index 64ab4951e9d6..f9ab74505e69 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c | |||
@@ -331,14 +331,12 @@ static void brd_make_request(struct request_queue *q, struct bio *bio) | |||
331 | struct bio_vec bvec; | 331 | struct bio_vec bvec; |
332 | sector_t sector; | 332 | sector_t sector; |
333 | struct bvec_iter iter; | 333 | struct bvec_iter iter; |
334 | int err = -EIO; | ||
335 | 334 | ||
336 | sector = bio->bi_iter.bi_sector; | 335 | sector = bio->bi_iter.bi_sector; |
337 | if (bio_end_sector(bio) > get_capacity(bdev->bd_disk)) | 336 | if (bio_end_sector(bio) > get_capacity(bdev->bd_disk)) |
338 | goto out; | 337 | goto io_error; |
339 | 338 | ||
340 | if (unlikely(bio->bi_rw & REQ_DISCARD)) { | 339 | if (unlikely(bio->bi_rw & REQ_DISCARD)) { |
341 | err = 0; | ||
342 | discard_from_brd(brd, sector, bio->bi_iter.bi_size); | 340 | discard_from_brd(brd, sector, bio->bi_iter.bi_size); |
343 | goto out; | 341 | goto out; |
344 | } | 342 | } |
@@ -349,15 +347,20 @@ static void brd_make_request(struct request_queue *q, struct bio *bio) | |||
349 | 347 | ||
350 | bio_for_each_segment(bvec, bio, iter) { | 348 | bio_for_each_segment(bvec, bio, iter) { |
351 | unsigned int len = bvec.bv_len; | 349 | unsigned int len = bvec.bv_len; |
350 | int err; | ||
351 | |||
352 | err = brd_do_bvec(brd, bvec.bv_page, len, | 352 | err = brd_do_bvec(brd, bvec.bv_page, len, |
353 | bvec.bv_offset, rw, sector); | 353 | bvec.bv_offset, rw, sector); |
354 | if (err) | 354 | if (err) |
355 | break; | 355 | goto io_error; |
356 | sector += len >> SECTOR_SHIFT; | 356 | sector += len >> SECTOR_SHIFT; |
357 | } | 357 | } |
358 | 358 | ||
359 | out: | 359 | out: |
360 | bio_endio(bio, err); | 360 | bio_endio(bio); |
361 | return; | ||
362 | io_error: | ||
363 | bio_io_error(bio); | ||
361 | } | 364 | } |
362 | 365 | ||
363 | static int brd_rw_page(struct block_device *bdev, sector_t sector, | 366 | static int brd_rw_page(struct block_device *bdev, sector_t sector, |
@@ -500,7 +503,7 @@ static struct brd_device *brd_alloc(int i) | |||
500 | blk_queue_physical_block_size(brd->brd_queue, PAGE_SIZE); | 503 | blk_queue_physical_block_size(brd->brd_queue, PAGE_SIZE); |
501 | 504 | ||
502 | brd->brd_queue->limits.discard_granularity = PAGE_SIZE; | 505 | brd->brd_queue->limits.discard_granularity = PAGE_SIZE; |
503 | brd->brd_queue->limits.max_discard_sectors = UINT_MAX; | 506 | blk_queue_max_discard_sectors(brd->brd_queue, UINT_MAX); |
504 | brd->brd_queue->limits.discard_zeroes_data = 1; | 507 | brd->brd_queue->limits.discard_zeroes_data = 1; |
505 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, brd->brd_queue); | 508 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, brd->brd_queue); |
506 | 509 | ||
diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c index 1318e3217cb0..b3868e7a1ffd 100644 --- a/drivers/block/drbd/drbd_actlog.c +++ b/drivers/block/drbd/drbd_actlog.c | |||
@@ -175,11 +175,11 @@ static int _drbd_md_sync_page_io(struct drbd_device *device, | |||
175 | atomic_inc(&device->md_io.in_use); /* drbd_md_put_buffer() is in the completion handler */ | 175 | atomic_inc(&device->md_io.in_use); /* drbd_md_put_buffer() is in the completion handler */ |
176 | device->md_io.submit_jif = jiffies; | 176 | device->md_io.submit_jif = jiffies; |
177 | if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) | 177 | if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) |
178 | bio_endio(bio, -EIO); | 178 | bio_io_error(bio); |
179 | else | 179 | else |
180 | submit_bio(rw, bio); | 180 | submit_bio(rw, bio); |
181 | wait_until_done_or_force_detached(device, bdev, &device->md_io.done); | 181 | wait_until_done_or_force_detached(device, bdev, &device->md_io.done); |
182 | if (bio_flagged(bio, BIO_UPTODATE)) | 182 | if (!bio->bi_error) |
183 | err = device->md_io.error; | 183 | err = device->md_io.error; |
184 | 184 | ||
185 | out: | 185 | out: |
diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c index 434c77dcc99e..e5e0f19ceda0 100644 --- a/drivers/block/drbd/drbd_bitmap.c +++ b/drivers/block/drbd/drbd_bitmap.c | |||
@@ -941,36 +941,27 @@ static void drbd_bm_aio_ctx_destroy(struct kref *kref) | |||
941 | } | 941 | } |
942 | 942 | ||
943 | /* bv_page may be a copy, or may be the original */ | 943 | /* bv_page may be a copy, or may be the original */ |
944 | static void drbd_bm_endio(struct bio *bio, int error) | 944 | static void drbd_bm_endio(struct bio *bio) |
945 | { | 945 | { |
946 | struct drbd_bm_aio_ctx *ctx = bio->bi_private; | 946 | struct drbd_bm_aio_ctx *ctx = bio->bi_private; |
947 | struct drbd_device *device = ctx->device; | 947 | struct drbd_device *device = ctx->device; |
948 | struct drbd_bitmap *b = device->bitmap; | 948 | struct drbd_bitmap *b = device->bitmap; |
949 | unsigned int idx = bm_page_to_idx(bio->bi_io_vec[0].bv_page); | 949 | unsigned int idx = bm_page_to_idx(bio->bi_io_vec[0].bv_page); |
950 | int uptodate = bio_flagged(bio, BIO_UPTODATE); | ||
951 | |||
952 | |||
953 | /* strange behavior of some lower level drivers... | ||
954 | * fail the request by clearing the uptodate flag, | ||
955 | * but do not return any error?! | ||
956 | * do we want to WARN() on this? */ | ||
957 | if (!error && !uptodate) | ||
958 | error = -EIO; | ||
959 | 950 | ||
960 | if ((ctx->flags & BM_AIO_COPY_PAGES) == 0 && | 951 | if ((ctx->flags & BM_AIO_COPY_PAGES) == 0 && |
961 | !bm_test_page_unchanged(b->bm_pages[idx])) | 952 | !bm_test_page_unchanged(b->bm_pages[idx])) |
962 | drbd_warn(device, "bitmap page idx %u changed during IO!\n", idx); | 953 | drbd_warn(device, "bitmap page idx %u changed during IO!\n", idx); |
963 | 954 | ||
964 | if (error) { | 955 | if (bio->bi_error) { |
965 | /* ctx error will hold the completed-last non-zero error code, | 956 | /* ctx error will hold the completed-last non-zero error code, |
966 | * in case error codes differ. */ | 957 | * in case error codes differ. */ |
967 | ctx->error = error; | 958 | ctx->error = bio->bi_error; |
968 | bm_set_page_io_err(b->bm_pages[idx]); | 959 | bm_set_page_io_err(b->bm_pages[idx]); |
969 | /* Not identical to on disk version of it. | 960 | /* Not identical to on disk version of it. |
970 | * Is BM_PAGE_IO_ERROR enough? */ | 961 | * Is BM_PAGE_IO_ERROR enough? */ |
971 | if (__ratelimit(&drbd_ratelimit_state)) | 962 | if (__ratelimit(&drbd_ratelimit_state)) |
972 | drbd_err(device, "IO ERROR %d on bitmap page idx %u\n", | 963 | drbd_err(device, "IO ERROR %d on bitmap page idx %u\n", |
973 | error, idx); | 964 | bio->bi_error, idx); |
974 | } else { | 965 | } else { |
975 | bm_clear_page_io_err(b->bm_pages[idx]); | 966 | bm_clear_page_io_err(b->bm_pages[idx]); |
976 | dynamic_drbd_dbg(device, "bitmap page idx %u completed\n", idx); | 967 | dynamic_drbd_dbg(device, "bitmap page idx %u completed\n", idx); |
@@ -1031,7 +1022,7 @@ static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_ho | |||
1031 | 1022 | ||
1032 | if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) { | 1023 | if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) { |
1033 | bio->bi_rw |= rw; | 1024 | bio->bi_rw |= rw; |
1034 | bio_endio(bio, -EIO); | 1025 | bio_io_error(bio); |
1035 | } else { | 1026 | } else { |
1036 | submit_bio(rw, bio); | 1027 | submit_bio(rw, bio); |
1037 | /* this should not count as user activity and cause the | 1028 | /* this should not count as user activity and cause the |
diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index efd19c2da9c2..015c6e91b756 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h | |||
@@ -1450,7 +1450,6 @@ extern void do_submit(struct work_struct *ws); | |||
1450 | extern void __drbd_make_request(struct drbd_device *, struct bio *, unsigned long); | 1450 | extern void __drbd_make_request(struct drbd_device *, struct bio *, unsigned long); |
1451 | extern void drbd_make_request(struct request_queue *q, struct bio *bio); | 1451 | extern void drbd_make_request(struct request_queue *q, struct bio *bio); |
1452 | extern int drbd_read_remote(struct drbd_device *device, struct drbd_request *req); | 1452 | extern int drbd_read_remote(struct drbd_device *device, struct drbd_request *req); |
1453 | extern int drbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bvm, struct bio_vec *bvec); | ||
1454 | extern int is_valid_ar_handle(struct drbd_request *, sector_t); | 1453 | extern int is_valid_ar_handle(struct drbd_request *, sector_t); |
1455 | 1454 | ||
1456 | 1455 | ||
@@ -1481,9 +1480,9 @@ extern int drbd_khelper(struct drbd_device *device, char *cmd); | |||
1481 | 1480 | ||
1482 | /* drbd_worker.c */ | 1481 | /* drbd_worker.c */ |
1483 | /* bi_end_io handlers */ | 1482 | /* bi_end_io handlers */ |
1484 | extern void drbd_md_endio(struct bio *bio, int error); | 1483 | extern void drbd_md_endio(struct bio *bio); |
1485 | extern void drbd_peer_request_endio(struct bio *bio, int error); | 1484 | extern void drbd_peer_request_endio(struct bio *bio); |
1486 | extern void drbd_request_endio(struct bio *bio, int error); | 1485 | extern void drbd_request_endio(struct bio *bio); |
1487 | extern int drbd_worker(struct drbd_thread *thi); | 1486 | extern int drbd_worker(struct drbd_thread *thi); |
1488 | enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor); | 1487 | enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor); |
1489 | void drbd_resync_after_changed(struct drbd_device *device); | 1488 | void drbd_resync_after_changed(struct drbd_device *device); |
@@ -1604,12 +1603,13 @@ static inline void drbd_generic_make_request(struct drbd_device *device, | |||
1604 | __release(local); | 1603 | __release(local); |
1605 | if (!bio->bi_bdev) { | 1604 | if (!bio->bi_bdev) { |
1606 | drbd_err(device, "drbd_generic_make_request: bio->bi_bdev == NULL\n"); | 1605 | drbd_err(device, "drbd_generic_make_request: bio->bi_bdev == NULL\n"); |
1607 | bio_endio(bio, -ENODEV); | 1606 | bio->bi_error = -ENODEV; |
1607 | bio_endio(bio); | ||
1608 | return; | 1608 | return; |
1609 | } | 1609 | } |
1610 | 1610 | ||
1611 | if (drbd_insert_fault(device, fault_type)) | 1611 | if (drbd_insert_fault(device, fault_type)) |
1612 | bio_endio(bio, -EIO); | 1612 | bio_io_error(bio); |
1613 | else | 1613 | else |
1614 | generic_make_request(bio); | 1614 | generic_make_request(bio); |
1615 | } | 1615 | } |
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index a1518539b858..74d97f4bac34 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c | |||
@@ -2774,7 +2774,6 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig | |||
2774 | This triggers a max_bio_size message upon first attach or connect */ | 2774 | This triggers a max_bio_size message upon first attach or connect */ |
2775 | blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8); | 2775 | blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8); |
2776 | blk_queue_bounce_limit(q, BLK_BOUNCE_ANY); | 2776 | blk_queue_bounce_limit(q, BLK_BOUNCE_ANY); |
2777 | blk_queue_merge_bvec(q, drbd_merge_bvec); | ||
2778 | q->queue_lock = &resource->req_lock; | 2777 | q->queue_lock = &resource->req_lock; |
2779 | 2778 | ||
2780 | device->md_io.page = alloc_page(GFP_KERNEL); | 2779 | device->md_io.page = alloc_page(GFP_KERNEL); |
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index 74df8cfad414..e80cbefbc2b5 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c | |||
@@ -1156,14 +1156,14 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi | |||
1156 | /* For now, don't allow more than one activity log extent worth of data | 1156 | /* For now, don't allow more than one activity log extent worth of data |
1157 | * to be discarded in one go. We may need to rework drbd_al_begin_io() | 1157 | * to be discarded in one go. We may need to rework drbd_al_begin_io() |
1158 | * to allow for even larger discard ranges */ | 1158 | * to allow for even larger discard ranges */ |
1159 | q->limits.max_discard_sectors = DRBD_MAX_DISCARD_SECTORS; | 1159 | blk_queue_max_discard_sectors(q, DRBD_MAX_DISCARD_SECTORS); |
1160 | 1160 | ||
1161 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); | 1161 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); |
1162 | /* REALLY? Is stacking secdiscard "legal"? */ | 1162 | /* REALLY? Is stacking secdiscard "legal"? */ |
1163 | if (blk_queue_secdiscard(b)) | 1163 | if (blk_queue_secdiscard(b)) |
1164 | queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, q); | 1164 | queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, q); |
1165 | } else { | 1165 | } else { |
1166 | q->limits.max_discard_sectors = 0; | 1166 | blk_queue_max_discard_sectors(q, 0); |
1167 | queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); | 1167 | queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); |
1168 | queue_flag_clear_unlocked(QUEUE_FLAG_SECDISCARD, q); | 1168 | queue_flag_clear_unlocked(QUEUE_FLAG_SECDISCARD, q); |
1169 | } | 1169 | } |
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index 3907202fb9d9..211592682169 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c | |||
@@ -201,7 +201,8 @@ void start_new_tl_epoch(struct drbd_connection *connection) | |||
201 | void complete_master_bio(struct drbd_device *device, | 201 | void complete_master_bio(struct drbd_device *device, |
202 | struct bio_and_error *m) | 202 | struct bio_and_error *m) |
203 | { | 203 | { |
204 | bio_endio(m->bio, m->error); | 204 | m->bio->bi_error = m->error; |
205 | bio_endio(m->bio); | ||
205 | dec_ap_bio(device); | 206 | dec_ap_bio(device); |
206 | } | 207 | } |
207 | 208 | ||
@@ -1153,12 +1154,12 @@ drbd_submit_req_private_bio(struct drbd_request *req) | |||
1153 | rw == WRITE ? DRBD_FAULT_DT_WR | 1154 | rw == WRITE ? DRBD_FAULT_DT_WR |
1154 | : rw == READ ? DRBD_FAULT_DT_RD | 1155 | : rw == READ ? DRBD_FAULT_DT_RD |
1155 | : DRBD_FAULT_DT_RA)) | 1156 | : DRBD_FAULT_DT_RA)) |
1156 | bio_endio(bio, -EIO); | 1157 | bio_io_error(bio); |
1157 | else | 1158 | else |
1158 | generic_make_request(bio); | 1159 | generic_make_request(bio); |
1159 | put_ldev(device); | 1160 | put_ldev(device); |
1160 | } else | 1161 | } else |
1161 | bio_endio(bio, -EIO); | 1162 | bio_io_error(bio); |
1162 | } | 1163 | } |
1163 | 1164 | ||
1164 | static void drbd_queue_write(struct drbd_device *device, struct drbd_request *req) | 1165 | static void drbd_queue_write(struct drbd_device *device, struct drbd_request *req) |
@@ -1191,7 +1192,8 @@ drbd_request_prepare(struct drbd_device *device, struct bio *bio, unsigned long | |||
1191 | /* only pass the error to the upper layers. | 1192 | /* only pass the error to the upper layers. |
1192 | * if user cannot handle io errors, that's not our business. */ | 1193 | * if user cannot handle io errors, that's not our business. */ |
1193 | drbd_err(device, "could not kmalloc() req\n"); | 1194 | drbd_err(device, "could not kmalloc() req\n"); |
1194 | bio_endio(bio, -ENOMEM); | 1195 | bio->bi_error = -ENOMEM; |
1196 | bio_endio(bio); | ||
1195 | return ERR_PTR(-ENOMEM); | 1197 | return ERR_PTR(-ENOMEM); |
1196 | } | 1198 | } |
1197 | req->start_jif = start_jif; | 1199 | req->start_jif = start_jif; |
@@ -1497,6 +1499,8 @@ void drbd_make_request(struct request_queue *q, struct bio *bio) | |||
1497 | struct drbd_device *device = (struct drbd_device *) q->queuedata; | 1499 | struct drbd_device *device = (struct drbd_device *) q->queuedata; |
1498 | unsigned long start_jif; | 1500 | unsigned long start_jif; |
1499 | 1501 | ||
1502 | blk_queue_split(q, &bio, q->bio_split); | ||
1503 | |||
1500 | start_jif = jiffies; | 1504 | start_jif = jiffies; |
1501 | 1505 | ||
1502 | /* | 1506 | /* |
@@ -1508,41 +1512,6 @@ void drbd_make_request(struct request_queue *q, struct bio *bio) | |||
1508 | __drbd_make_request(device, bio, start_jif); | 1512 | __drbd_make_request(device, bio, start_jif); |
1509 | } | 1513 | } |
1510 | 1514 | ||
1511 | /* This is called by bio_add_page(). | ||
1512 | * | ||
1513 | * q->max_hw_sectors and other global limits are already enforced there. | ||
1514 | * | ||
1515 | * We need to call down to our lower level device, | ||
1516 | * in case it has special restrictions. | ||
1517 | * | ||
1518 | * We also may need to enforce configured max-bio-bvecs limits. | ||
1519 | * | ||
1520 | * As long as the BIO is empty we have to allow at least one bvec, | ||
1521 | * regardless of size and offset, so no need to ask lower levels. | ||
1522 | */ | ||
1523 | int drbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bvm, struct bio_vec *bvec) | ||
1524 | { | ||
1525 | struct drbd_device *device = (struct drbd_device *) q->queuedata; | ||
1526 | unsigned int bio_size = bvm->bi_size; | ||
1527 | int limit = DRBD_MAX_BIO_SIZE; | ||
1528 | int backing_limit; | ||
1529 | |||
1530 | if (bio_size && get_ldev(device)) { | ||
1531 | unsigned int max_hw_sectors = queue_max_hw_sectors(q); | ||
1532 | struct request_queue * const b = | ||
1533 | device->ldev->backing_bdev->bd_disk->queue; | ||
1534 | if (b->merge_bvec_fn) { | ||
1535 | bvm->bi_bdev = device->ldev->backing_bdev; | ||
1536 | backing_limit = b->merge_bvec_fn(b, bvm, bvec); | ||
1537 | limit = min(limit, backing_limit); | ||
1538 | } | ||
1539 | put_ldev(device); | ||
1540 | if ((limit >> 9) > max_hw_sectors) | ||
1541 | limit = max_hw_sectors << 9; | ||
1542 | } | ||
1543 | return limit; | ||
1544 | } | ||
1545 | |||
1546 | void request_timer_fn(unsigned long data) | 1515 | void request_timer_fn(unsigned long data) |
1547 | { | 1516 | { |
1548 | struct drbd_device *device = (struct drbd_device *) data; | 1517 | struct drbd_device *device = (struct drbd_device *) data; |
diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c index d0fae55d871d..5578c1477ba6 100644 --- a/drivers/block/drbd/drbd_worker.c +++ b/drivers/block/drbd/drbd_worker.c | |||
@@ -65,12 +65,12 @@ rwlock_t global_state_lock; | |||
65 | /* used for synchronous meta data and bitmap IO | 65 | /* used for synchronous meta data and bitmap IO |
66 | * submitted by drbd_md_sync_page_io() | 66 | * submitted by drbd_md_sync_page_io() |
67 | */ | 67 | */ |
68 | void drbd_md_endio(struct bio *bio, int error) | 68 | void drbd_md_endio(struct bio *bio) |
69 | { | 69 | { |
70 | struct drbd_device *device; | 70 | struct drbd_device *device; |
71 | 71 | ||
72 | device = bio->bi_private; | 72 | device = bio->bi_private; |
73 | device->md_io.error = error; | 73 | device->md_io.error = bio->bi_error; |
74 | 74 | ||
75 | /* We grabbed an extra reference in _drbd_md_sync_page_io() to be able | 75 | /* We grabbed an extra reference in _drbd_md_sync_page_io() to be able |
76 | * to timeout on the lower level device, and eventually detach from it. | 76 | * to timeout on the lower level device, and eventually detach from it. |
@@ -170,31 +170,20 @@ void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(l | |||
170 | /* writes on behalf of the partner, or resync writes, | 170 | /* writes on behalf of the partner, or resync writes, |
171 | * "submitted" by the receiver. | 171 | * "submitted" by the receiver. |
172 | */ | 172 | */ |
173 | void drbd_peer_request_endio(struct bio *bio, int error) | 173 | void drbd_peer_request_endio(struct bio *bio) |
174 | { | 174 | { |
175 | struct drbd_peer_request *peer_req = bio->bi_private; | 175 | struct drbd_peer_request *peer_req = bio->bi_private; |
176 | struct drbd_device *device = peer_req->peer_device->device; | 176 | struct drbd_device *device = peer_req->peer_device->device; |
177 | int uptodate = bio_flagged(bio, BIO_UPTODATE); | ||
178 | int is_write = bio_data_dir(bio) == WRITE; | 177 | int is_write = bio_data_dir(bio) == WRITE; |
179 | int is_discard = !!(bio->bi_rw & REQ_DISCARD); | 178 | int is_discard = !!(bio->bi_rw & REQ_DISCARD); |
180 | 179 | ||
181 | if (error && __ratelimit(&drbd_ratelimit_state)) | 180 | if (bio->bi_error && __ratelimit(&drbd_ratelimit_state)) |
182 | drbd_warn(device, "%s: error=%d s=%llus\n", | 181 | drbd_warn(device, "%s: error=%d s=%llus\n", |
183 | is_write ? (is_discard ? "discard" : "write") | 182 | is_write ? (is_discard ? "discard" : "write") |
184 | : "read", error, | 183 | : "read", bio->bi_error, |
185 | (unsigned long long)peer_req->i.sector); | 184 | (unsigned long long)peer_req->i.sector); |
186 | if (!error && !uptodate) { | ||
187 | if (__ratelimit(&drbd_ratelimit_state)) | ||
188 | drbd_warn(device, "%s: setting error to -EIO s=%llus\n", | ||
189 | is_write ? "write" : "read", | ||
190 | (unsigned long long)peer_req->i.sector); | ||
191 | /* strange behavior of some lower level drivers... | ||
192 | * fail the request by clearing the uptodate flag, | ||
193 | * but do not return any error?! */ | ||
194 | error = -EIO; | ||
195 | } | ||
196 | 185 | ||
197 | if (error) | 186 | if (bio->bi_error) |
198 | set_bit(__EE_WAS_ERROR, &peer_req->flags); | 187 | set_bit(__EE_WAS_ERROR, &peer_req->flags); |
199 | 188 | ||
200 | bio_put(bio); /* no need for the bio anymore */ | 189 | bio_put(bio); /* no need for the bio anymore */ |
@@ -208,24 +197,13 @@ void drbd_peer_request_endio(struct bio *bio, int error) | |||
208 | 197 | ||
209 | /* read, readA or write requests on R_PRIMARY coming from drbd_make_request | 198 | /* read, readA or write requests on R_PRIMARY coming from drbd_make_request |
210 | */ | 199 | */ |
211 | void drbd_request_endio(struct bio *bio, int error) | 200 | void drbd_request_endio(struct bio *bio) |
212 | { | 201 | { |
213 | unsigned long flags; | 202 | unsigned long flags; |
214 | struct drbd_request *req = bio->bi_private; | 203 | struct drbd_request *req = bio->bi_private; |
215 | struct drbd_device *device = req->device; | 204 | struct drbd_device *device = req->device; |
216 | struct bio_and_error m; | 205 | struct bio_and_error m; |
217 | enum drbd_req_event what; | 206 | enum drbd_req_event what; |
218 | int uptodate = bio_flagged(bio, BIO_UPTODATE); | ||
219 | |||
220 | if (!error && !uptodate) { | ||
221 | drbd_warn(device, "p %s: setting error to -EIO\n", | ||
222 | bio_data_dir(bio) == WRITE ? "write" : "read"); | ||
223 | /* strange behavior of some lower level drivers... | ||
224 | * fail the request by clearing the uptodate flag, | ||
225 | * but do not return any error?! */ | ||
226 | error = -EIO; | ||
227 | } | ||
228 | |||
229 | 207 | ||
230 | /* If this request was aborted locally before, | 208 | /* If this request was aborted locally before, |
231 | * but now was completed "successfully", | 209 | * but now was completed "successfully", |
@@ -259,14 +237,14 @@ void drbd_request_endio(struct bio *bio, int error) | |||
259 | if (__ratelimit(&drbd_ratelimit_state)) | 237 | if (__ratelimit(&drbd_ratelimit_state)) |
260 | drbd_emerg(device, "delayed completion of aborted local request; disk-timeout may be too aggressive\n"); | 238 | drbd_emerg(device, "delayed completion of aborted local request; disk-timeout may be too aggressive\n"); |
261 | 239 | ||
262 | if (!error) | 240 | if (!bio->bi_error) |
263 | panic("possible random memory corruption caused by delayed completion of aborted local request\n"); | 241 | panic("possible random memory corruption caused by delayed completion of aborted local request\n"); |
264 | } | 242 | } |
265 | 243 | ||
266 | /* to avoid recursion in __req_mod */ | 244 | /* to avoid recursion in __req_mod */ |
267 | if (unlikely(error)) { | 245 | if (unlikely(bio->bi_error)) { |
268 | if (bio->bi_rw & REQ_DISCARD) | 246 | if (bio->bi_rw & REQ_DISCARD) |
269 | what = (error == -EOPNOTSUPP) | 247 | what = (bio->bi_error == -EOPNOTSUPP) |
270 | ? DISCARD_COMPLETED_NOTSUPP | 248 | ? DISCARD_COMPLETED_NOTSUPP |
271 | : DISCARD_COMPLETED_WITH_ERROR; | 249 | : DISCARD_COMPLETED_WITH_ERROR; |
272 | else | 250 | else |
@@ -279,7 +257,7 @@ void drbd_request_endio(struct bio *bio, int error) | |||
279 | what = COMPLETED_OK; | 257 | what = COMPLETED_OK; |
280 | 258 | ||
281 | bio_put(req->private_bio); | 259 | bio_put(req->private_bio); |
282 | req->private_bio = ERR_PTR(error); | 260 | req->private_bio = ERR_PTR(bio->bi_error); |
283 | 261 | ||
284 | /* not req_mod(), we need irqsave here! */ | 262 | /* not req_mod(), we need irqsave here! */ |
285 | spin_lock_irqsave(&device->resource->req_lock, flags); | 263 | spin_lock_irqsave(&device->resource->req_lock, flags); |
diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index a08cda955285..331363e7de0f 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c | |||
@@ -3771,13 +3771,14 @@ struct rb0_cbdata { | |||
3771 | struct completion complete; | 3771 | struct completion complete; |
3772 | }; | 3772 | }; |
3773 | 3773 | ||
3774 | static void floppy_rb0_cb(struct bio *bio, int err) | 3774 | static void floppy_rb0_cb(struct bio *bio) |
3775 | { | 3775 | { |
3776 | struct rb0_cbdata *cbdata = (struct rb0_cbdata *)bio->bi_private; | 3776 | struct rb0_cbdata *cbdata = (struct rb0_cbdata *)bio->bi_private; |
3777 | int drive = cbdata->drive; | 3777 | int drive = cbdata->drive; |
3778 | 3778 | ||
3779 | if (err) { | 3779 | if (bio->bi_error) { |
3780 | pr_info("floppy: error %d while reading block 0\n", err); | 3780 | pr_info("floppy: error %d while reading block 0\n", |
3781 | bio->bi_error); | ||
3781 | set_bit(FD_OPEN_SHOULD_FAIL_BIT, &UDRS->flags); | 3782 | set_bit(FD_OPEN_SHOULD_FAIL_BIT, &UDRS->flags); |
3782 | } | 3783 | } |
3783 | complete(&cbdata->complete); | 3784 | complete(&cbdata->complete); |
diff --git a/drivers/block/loop.c b/drivers/block/loop.c index f7a4c9d7f721..f9889b6bc02c 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c | |||
@@ -675,7 +675,7 @@ static void loop_config_discard(struct loop_device *lo) | |||
675 | lo->lo_encrypt_key_size) { | 675 | lo->lo_encrypt_key_size) { |
676 | q->limits.discard_granularity = 0; | 676 | q->limits.discard_granularity = 0; |
677 | q->limits.discard_alignment = 0; | 677 | q->limits.discard_alignment = 0; |
678 | q->limits.max_discard_sectors = 0; | 678 | blk_queue_max_discard_sectors(q, 0); |
679 | q->limits.discard_zeroes_data = 0; | 679 | q->limits.discard_zeroes_data = 0; |
680 | queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); | 680 | queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); |
681 | return; | 681 | return; |
@@ -683,7 +683,7 @@ static void loop_config_discard(struct loop_device *lo) | |||
683 | 683 | ||
684 | q->limits.discard_granularity = inode->i_sb->s_blocksize; | 684 | q->limits.discard_granularity = inode->i_sb->s_blocksize; |
685 | q->limits.discard_alignment = 0; | 685 | q->limits.discard_alignment = 0; |
686 | q->limits.max_discard_sectors = UINT_MAX >> 9; | 686 | blk_queue_max_discard_sectors(q, UINT_MAX >> 9); |
687 | q->limits.discard_zeroes_data = 1; | 687 | q->limits.discard_zeroes_data = 1; |
688 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); | 688 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); |
689 | } | 689 | } |
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 0e385d8e9b86..f169faf9838a 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c | |||
@@ -822,7 +822,7 @@ static int __init nbd_init(void) | |||
822 | queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue); | 822 | queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue); |
823 | queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, disk->queue); | 823 | queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, disk->queue); |
824 | disk->queue->limits.discard_granularity = 512; | 824 | disk->queue->limits.discard_granularity = 512; |
825 | disk->queue->limits.max_discard_sectors = UINT_MAX; | 825 | blk_queue_max_discard_sectors(disk->queue, UINT_MAX); |
826 | disk->queue->limits.discard_zeroes_data = 0; | 826 | disk->queue->limits.discard_zeroes_data = 0; |
827 | blk_queue_max_hw_sectors(disk->queue, 65536); | 827 | blk_queue_max_hw_sectors(disk->queue, 65536); |
828 | disk->queue->limits.max_sectors = 256; | 828 | disk->queue->limits.max_sectors = 256; |
diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c index 3177b245d2bd..17269a3b85f2 100644 --- a/drivers/block/null_blk.c +++ b/drivers/block/null_blk.c | |||
@@ -222,7 +222,7 @@ static void end_cmd(struct nullb_cmd *cmd) | |||
222 | blk_end_request_all(cmd->rq, 0); | 222 | blk_end_request_all(cmd->rq, 0); |
223 | break; | 223 | break; |
224 | case NULL_Q_BIO: | 224 | case NULL_Q_BIO: |
225 | bio_endio(cmd->bio, 0); | 225 | bio_endio(cmd->bio); |
226 | break; | 226 | break; |
227 | } | 227 | } |
228 | 228 | ||
diff --git a/drivers/block/nvme-core.c b/drivers/block/nvme-core.c index 7920c2741b47..2f694d78da55 100644 --- a/drivers/block/nvme-core.c +++ b/drivers/block/nvme-core.c | |||
@@ -1935,7 +1935,7 @@ static void nvme_config_discard(struct nvme_ns *ns) | |||
1935 | ns->queue->limits.discard_zeroes_data = 0; | 1935 | ns->queue->limits.discard_zeroes_data = 0; |
1936 | ns->queue->limits.discard_alignment = logical_block_size; | 1936 | ns->queue->limits.discard_alignment = logical_block_size; |
1937 | ns->queue->limits.discard_granularity = logical_block_size; | 1937 | ns->queue->limits.discard_granularity = logical_block_size; |
1938 | ns->queue->limits.max_discard_sectors = 0xffffffff; | 1938 | blk_queue_max_discard_sectors(ns->queue, 0xffffffff); |
1939 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue); | 1939 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue); |
1940 | } | 1940 | } |
1941 | 1941 | ||
@@ -2067,7 +2067,6 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid) | |||
2067 | goto out_free_ns; | 2067 | goto out_free_ns; |
2068 | queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue); | 2068 | queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue); |
2069 | queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue); | 2069 | queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue); |
2070 | queue_flag_set_unlocked(QUEUE_FLAG_SG_GAPS, ns->queue); | ||
2071 | ns->dev = dev; | 2070 | ns->dev = dev; |
2072 | ns->queue->queuedata = ns; | 2071 | ns->queue->queuedata = ns; |
2073 | 2072 | ||
@@ -2087,6 +2086,7 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid) | |||
2087 | blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9); | 2086 | blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9); |
2088 | if (dev->vwc & NVME_CTRL_VWC_PRESENT) | 2087 | if (dev->vwc & NVME_CTRL_VWC_PRESENT) |
2089 | blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA); | 2088 | blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA); |
2089 | blk_queue_virt_boundary(ns->queue, dev->page_size - 1); | ||
2090 | 2090 | ||
2091 | disk->major = nvme_major; | 2091 | disk->major = nvme_major; |
2092 | disk->first_minor = 0; | 2092 | disk->first_minor = 0; |
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 4c20c228184c..7be2375db7f2 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c | |||
@@ -977,7 +977,7 @@ static void pkt_make_local_copy(struct packet_data *pkt, struct bio_vec *bvec) | |||
977 | } | 977 | } |
978 | } | 978 | } |
979 | 979 | ||
980 | static void pkt_end_io_read(struct bio *bio, int err) | 980 | static void pkt_end_io_read(struct bio *bio) |
981 | { | 981 | { |
982 | struct packet_data *pkt = bio->bi_private; | 982 | struct packet_data *pkt = bio->bi_private; |
983 | struct pktcdvd_device *pd = pkt->pd; | 983 | struct pktcdvd_device *pd = pkt->pd; |
@@ -985,9 +985,9 @@ static void pkt_end_io_read(struct bio *bio, int err) | |||
985 | 985 | ||
986 | pkt_dbg(2, pd, "bio=%p sec0=%llx sec=%llx err=%d\n", | 986 | pkt_dbg(2, pd, "bio=%p sec0=%llx sec=%llx err=%d\n", |
987 | bio, (unsigned long long)pkt->sector, | 987 | bio, (unsigned long long)pkt->sector, |
988 | (unsigned long long)bio->bi_iter.bi_sector, err); | 988 | (unsigned long long)bio->bi_iter.bi_sector, bio->bi_error); |
989 | 989 | ||
990 | if (err) | 990 | if (bio->bi_error) |
991 | atomic_inc(&pkt->io_errors); | 991 | atomic_inc(&pkt->io_errors); |
992 | if (atomic_dec_and_test(&pkt->io_wait)) { | 992 | if (atomic_dec_and_test(&pkt->io_wait)) { |
993 | atomic_inc(&pkt->run_sm); | 993 | atomic_inc(&pkt->run_sm); |
@@ -996,13 +996,13 @@ static void pkt_end_io_read(struct bio *bio, int err) | |||
996 | pkt_bio_finished(pd); | 996 | pkt_bio_finished(pd); |
997 | } | 997 | } |
998 | 998 | ||
999 | static void pkt_end_io_packet_write(struct bio *bio, int err) | 999 | static void pkt_end_io_packet_write(struct bio *bio) |
1000 | { | 1000 | { |
1001 | struct packet_data *pkt = bio->bi_private; | 1001 | struct packet_data *pkt = bio->bi_private; |
1002 | struct pktcdvd_device *pd = pkt->pd; | 1002 | struct pktcdvd_device *pd = pkt->pd; |
1003 | BUG_ON(!pd); | 1003 | BUG_ON(!pd); |
1004 | 1004 | ||
1005 | pkt_dbg(2, pd, "id=%d, err=%d\n", pkt->id, err); | 1005 | pkt_dbg(2, pd, "id=%d, err=%d\n", pkt->id, bio->bi_error); |
1006 | 1006 | ||
1007 | pd->stats.pkt_ended++; | 1007 | pd->stats.pkt_ended++; |
1008 | 1008 | ||
@@ -1340,22 +1340,22 @@ static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt) | |||
1340 | pkt_queue_bio(pd, pkt->w_bio); | 1340 | pkt_queue_bio(pd, pkt->w_bio); |
1341 | } | 1341 | } |
1342 | 1342 | ||
1343 | static void pkt_finish_packet(struct packet_data *pkt, int uptodate) | 1343 | static void pkt_finish_packet(struct packet_data *pkt, int error) |
1344 | { | 1344 | { |
1345 | struct bio *bio; | 1345 | struct bio *bio; |
1346 | 1346 | ||
1347 | if (!uptodate) | 1347 | if (error) |
1348 | pkt->cache_valid = 0; | 1348 | pkt->cache_valid = 0; |
1349 | 1349 | ||
1350 | /* Finish all bios corresponding to this packet */ | 1350 | /* Finish all bios corresponding to this packet */ |
1351 | while ((bio = bio_list_pop(&pkt->orig_bios))) | 1351 | while ((bio = bio_list_pop(&pkt->orig_bios))) { |
1352 | bio_endio(bio, uptodate ? 0 : -EIO); | 1352 | bio->bi_error = error; |
1353 | bio_endio(bio); | ||
1354 | } | ||
1353 | } | 1355 | } |
1354 | 1356 | ||
1355 | static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data *pkt) | 1357 | static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data *pkt) |
1356 | { | 1358 | { |
1357 | int uptodate; | ||
1358 | |||
1359 | pkt_dbg(2, pd, "pkt %d\n", pkt->id); | 1359 | pkt_dbg(2, pd, "pkt %d\n", pkt->id); |
1360 | 1360 | ||
1361 | for (;;) { | 1361 | for (;;) { |
@@ -1384,7 +1384,7 @@ static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data | |||
1384 | if (atomic_read(&pkt->io_wait) > 0) | 1384 | if (atomic_read(&pkt->io_wait) > 0) |
1385 | return; | 1385 | return; |
1386 | 1386 | ||
1387 | if (test_bit(BIO_UPTODATE, &pkt->w_bio->bi_flags)) { | 1387 | if (!pkt->w_bio->bi_error) { |
1388 | pkt_set_state(pkt, PACKET_FINISHED_STATE); | 1388 | pkt_set_state(pkt, PACKET_FINISHED_STATE); |
1389 | } else { | 1389 | } else { |
1390 | pkt_set_state(pkt, PACKET_RECOVERY_STATE); | 1390 | pkt_set_state(pkt, PACKET_RECOVERY_STATE); |
@@ -1401,8 +1401,7 @@ static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data | |||
1401 | break; | 1401 | break; |
1402 | 1402 | ||
1403 | case PACKET_FINISHED_STATE: | 1403 | case PACKET_FINISHED_STATE: |
1404 | uptodate = test_bit(BIO_UPTODATE, &pkt->w_bio->bi_flags); | 1404 | pkt_finish_packet(pkt, pkt->w_bio->bi_error); |
1405 | pkt_finish_packet(pkt, uptodate); | ||
1406 | return; | 1405 | return; |
1407 | 1406 | ||
1408 | default: | 1407 | default: |
@@ -2332,13 +2331,14 @@ static void pkt_close(struct gendisk *disk, fmode_t mode) | |||
2332 | } | 2331 | } |
2333 | 2332 | ||
2334 | 2333 | ||
2335 | static void pkt_end_io_read_cloned(struct bio *bio, int err) | 2334 | static void pkt_end_io_read_cloned(struct bio *bio) |
2336 | { | 2335 | { |
2337 | struct packet_stacked_data *psd = bio->bi_private; | 2336 | struct packet_stacked_data *psd = bio->bi_private; |
2338 | struct pktcdvd_device *pd = psd->pd; | 2337 | struct pktcdvd_device *pd = psd->pd; |
2339 | 2338 | ||
2339 | psd->bio->bi_error = bio->bi_error; | ||
2340 | bio_put(bio); | 2340 | bio_put(bio); |
2341 | bio_endio(psd->bio, err); | 2341 | bio_endio(psd->bio); |
2342 | mempool_free(psd, psd_pool); | 2342 | mempool_free(psd, psd_pool); |
2343 | pkt_bio_finished(pd); | 2343 | pkt_bio_finished(pd); |
2344 | } | 2344 | } |
@@ -2447,6 +2447,10 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio) | |||
2447 | char b[BDEVNAME_SIZE]; | 2447 | char b[BDEVNAME_SIZE]; |
2448 | struct bio *split; | 2448 | struct bio *split; |
2449 | 2449 | ||
2450 | blk_queue_bounce(q, &bio); | ||
2451 | |||
2452 | blk_queue_split(q, &bio, q->bio_split); | ||
2453 | |||
2450 | pd = q->queuedata; | 2454 | pd = q->queuedata; |
2451 | if (!pd) { | 2455 | if (!pd) { |
2452 | pr_err("%s incorrect request queue\n", | 2456 | pr_err("%s incorrect request queue\n", |
@@ -2477,8 +2481,6 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio) | |||
2477 | goto end_io; | 2481 | goto end_io; |
2478 | } | 2482 | } |
2479 | 2483 | ||
2480 | blk_queue_bounce(q, &bio); | ||
2481 | |||
2482 | do { | 2484 | do { |
2483 | sector_t zone = get_zone(bio->bi_iter.bi_sector, pd); | 2485 | sector_t zone = get_zone(bio->bi_iter.bi_sector, pd); |
2484 | sector_t last_zone = get_zone(bio_end_sector(bio) - 1, pd); | 2486 | sector_t last_zone = get_zone(bio_end_sector(bio) - 1, pd); |
@@ -2504,26 +2506,6 @@ end_io: | |||
2504 | 2506 | ||
2505 | 2507 | ||
2506 | 2508 | ||
2507 | static int pkt_merge_bvec(struct request_queue *q, struct bvec_merge_data *bmd, | ||
2508 | struct bio_vec *bvec) | ||
2509 | { | ||
2510 | struct pktcdvd_device *pd = q->queuedata; | ||
2511 | sector_t zone = get_zone(bmd->bi_sector, pd); | ||
2512 | int used = ((bmd->bi_sector - zone) << 9) + bmd->bi_size; | ||
2513 | int remaining = (pd->settings.size << 9) - used; | ||
2514 | int remaining2; | ||
2515 | |||
2516 | /* | ||
2517 | * A bio <= PAGE_SIZE must be allowed. If it crosses a packet | ||
2518 | * boundary, pkt_make_request() will split the bio. | ||
2519 | */ | ||
2520 | remaining2 = PAGE_SIZE - bmd->bi_size; | ||
2521 | remaining = max(remaining, remaining2); | ||
2522 | |||
2523 | BUG_ON(remaining < 0); | ||
2524 | return remaining; | ||
2525 | } | ||
2526 | |||
2527 | static void pkt_init_queue(struct pktcdvd_device *pd) | 2509 | static void pkt_init_queue(struct pktcdvd_device *pd) |
2528 | { | 2510 | { |
2529 | struct request_queue *q = pd->disk->queue; | 2511 | struct request_queue *q = pd->disk->queue; |
@@ -2531,7 +2513,6 @@ static void pkt_init_queue(struct pktcdvd_device *pd) | |||
2531 | blk_queue_make_request(q, pkt_make_request); | 2513 | blk_queue_make_request(q, pkt_make_request); |
2532 | blk_queue_logical_block_size(q, CD_FRAMESIZE); | 2514 | blk_queue_logical_block_size(q, CD_FRAMESIZE); |
2533 | blk_queue_max_hw_sectors(q, PACKET_MAX_SECTORS); | 2515 | blk_queue_max_hw_sectors(q, PACKET_MAX_SECTORS); |
2534 | blk_queue_merge_bvec(q, pkt_merge_bvec); | ||
2535 | q->queuedata = pd; | 2516 | q->queuedata = pd; |
2536 | } | 2517 | } |
2537 | 2518 | ||
diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c index b1612eb16172..d89fcac59515 100644 --- a/drivers/block/ps3vram.c +++ b/drivers/block/ps3vram.c | |||
@@ -593,7 +593,8 @@ out: | |||
593 | next = bio_list_peek(&priv->list); | 593 | next = bio_list_peek(&priv->list); |
594 | spin_unlock_irq(&priv->lock); | 594 | spin_unlock_irq(&priv->lock); |
595 | 595 | ||
596 | bio_endio(bio, error); | 596 | bio->bi_error = error; |
597 | bio_endio(bio); | ||
597 | return next; | 598 | return next; |
598 | } | 599 | } |
599 | 600 | ||
@@ -605,6 +606,8 @@ static void ps3vram_make_request(struct request_queue *q, struct bio *bio) | |||
605 | 606 | ||
606 | dev_dbg(&dev->core, "%s\n", __func__); | 607 | dev_dbg(&dev->core, "%s\n", __func__); |
607 | 608 | ||
609 | blk_queue_split(q, &bio, q->bio_split); | ||
610 | |||
608 | spin_lock_irq(&priv->lock); | 611 | spin_lock_irq(&priv->lock); |
609 | busy = !bio_list_empty(&priv->list); | 612 | busy = !bio_list_empty(&priv->list); |
610 | bio_list_add(&priv->list, bio); | 613 | bio_list_add(&priv->list, bio); |
diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index bc67a93aa4f4..698f761037ce 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c | |||
@@ -3474,52 +3474,6 @@ static int rbd_queue_rq(struct blk_mq_hw_ctx *hctx, | |||
3474 | return BLK_MQ_RQ_QUEUE_OK; | 3474 | return BLK_MQ_RQ_QUEUE_OK; |
3475 | } | 3475 | } |
3476 | 3476 | ||
3477 | /* | ||
3478 | * a queue callback. Makes sure that we don't create a bio that spans across | ||
3479 | * multiple osd objects. One exception would be with a single page bios, | ||
3480 | * which we handle later at bio_chain_clone_range() | ||
3481 | */ | ||
3482 | static int rbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bmd, | ||
3483 | struct bio_vec *bvec) | ||
3484 | { | ||
3485 | struct rbd_device *rbd_dev = q->queuedata; | ||
3486 | sector_t sector_offset; | ||
3487 | sector_t sectors_per_obj; | ||
3488 | sector_t obj_sector_offset; | ||
3489 | int ret; | ||
3490 | |||
3491 | /* | ||
3492 | * Find how far into its rbd object the partition-relative | ||
3493 | * bio start sector is to offset relative to the enclosing | ||
3494 | * device. | ||
3495 | */ | ||
3496 | sector_offset = get_start_sect(bmd->bi_bdev) + bmd->bi_sector; | ||
3497 | sectors_per_obj = 1 << (rbd_dev->header.obj_order - SECTOR_SHIFT); | ||
3498 | obj_sector_offset = sector_offset & (sectors_per_obj - 1); | ||
3499 | |||
3500 | /* | ||
3501 | * Compute the number of bytes from that offset to the end | ||
3502 | * of the object. Account for what's already used by the bio. | ||
3503 | */ | ||
3504 | ret = (int) (sectors_per_obj - obj_sector_offset) << SECTOR_SHIFT; | ||
3505 | if (ret > bmd->bi_size) | ||
3506 | ret -= bmd->bi_size; | ||
3507 | else | ||
3508 | ret = 0; | ||
3509 | |||
3510 | /* | ||
3511 | * Don't send back more than was asked for. And if the bio | ||
3512 | * was empty, let the whole thing through because: "Note | ||
3513 | * that a block device *must* allow a single page to be | ||
3514 | * added to an empty bio." | ||
3515 | */ | ||
3516 | rbd_assert(bvec->bv_len <= PAGE_SIZE); | ||
3517 | if (ret > (int) bvec->bv_len || !bmd->bi_size) | ||
3518 | ret = (int) bvec->bv_len; | ||
3519 | |||
3520 | return ret; | ||
3521 | } | ||
3522 | |||
3523 | static void rbd_free_disk(struct rbd_device *rbd_dev) | 3477 | static void rbd_free_disk(struct rbd_device *rbd_dev) |
3524 | { | 3478 | { |
3525 | struct gendisk *disk = rbd_dev->disk; | 3479 | struct gendisk *disk = rbd_dev->disk; |
@@ -3815,10 +3769,9 @@ static int rbd_init_disk(struct rbd_device *rbd_dev) | |||
3815 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); | 3769 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); |
3816 | q->limits.discard_granularity = segment_size; | 3770 | q->limits.discard_granularity = segment_size; |
3817 | q->limits.discard_alignment = segment_size; | 3771 | q->limits.discard_alignment = segment_size; |
3818 | q->limits.max_discard_sectors = segment_size / SECTOR_SIZE; | 3772 | blk_queue_max_discard_sectors(q, segment_size / SECTOR_SIZE); |
3819 | q->limits.discard_zeroes_data = 1; | 3773 | q->limits.discard_zeroes_data = 1; |
3820 | 3774 | ||
3821 | blk_queue_merge_bvec(q, rbd_merge_bvec); | ||
3822 | disk->queue = q; | 3775 | disk->queue = q; |
3823 | 3776 | ||
3824 | q->queuedata = rbd_dev; | 3777 | q->queuedata = rbd_dev; |
diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c index ac8c62cb4875..3163e4cdc2cc 100644 --- a/drivers/block/rsxx/dev.c +++ b/drivers/block/rsxx/dev.c | |||
@@ -137,7 +137,10 @@ static void bio_dma_done_cb(struct rsxx_cardinfo *card, | |||
137 | if (!card->eeh_state && card->gendisk) | 137 | if (!card->eeh_state && card->gendisk) |
138 | disk_stats_complete(card, meta->bio, meta->start_time); | 138 | disk_stats_complete(card, meta->bio, meta->start_time); |
139 | 139 | ||
140 | bio_endio(meta->bio, atomic_read(&meta->error) ? -EIO : 0); | 140 | if (atomic_read(&meta->error)) |
141 | bio_io_error(meta->bio); | ||
142 | else | ||
143 | bio_endio(meta->bio); | ||
141 | kmem_cache_free(bio_meta_pool, meta); | 144 | kmem_cache_free(bio_meta_pool, meta); |
142 | } | 145 | } |
143 | } | 146 | } |
@@ -148,6 +151,8 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio) | |||
148 | struct rsxx_bio_meta *bio_meta; | 151 | struct rsxx_bio_meta *bio_meta; |
149 | int st = -EINVAL; | 152 | int st = -EINVAL; |
150 | 153 | ||
154 | blk_queue_split(q, &bio, q->bio_split); | ||
155 | |||
151 | might_sleep(); | 156 | might_sleep(); |
152 | 157 | ||
153 | if (!card) | 158 | if (!card) |
@@ -199,7 +204,9 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio) | |||
199 | queue_err: | 204 | queue_err: |
200 | kmem_cache_free(bio_meta_pool, bio_meta); | 205 | kmem_cache_free(bio_meta_pool, bio_meta); |
201 | req_err: | 206 | req_err: |
202 | bio_endio(bio, st); | 207 | if (st) |
208 | bio->bi_error = st; | ||
209 | bio_endio(bio); | ||
203 | } | 210 | } |
204 | 211 | ||
205 | /*----------------- Device Setup -------------------*/ | 212 | /*----------------- Device Setup -------------------*/ |
diff --git a/drivers/block/skd_main.c b/drivers/block/skd_main.c index 1e46eb2305c0..586f9168ffa4 100644 --- a/drivers/block/skd_main.c +++ b/drivers/block/skd_main.c | |||
@@ -4422,7 +4422,7 @@ static int skd_cons_disk(struct skd_device *skdev) | |||
4422 | /* DISCARD Flag initialization. */ | 4422 | /* DISCARD Flag initialization. */ |
4423 | q->limits.discard_granularity = 8192; | 4423 | q->limits.discard_granularity = 8192; |
4424 | q->limits.discard_alignment = 0; | 4424 | q->limits.discard_alignment = 0; |
4425 | q->limits.max_discard_sectors = UINT_MAX >> 9; | 4425 | blk_queue_max_discard_sectors(q, UINT_MAX >> 9); |
4426 | q->limits.discard_zeroes_data = 1; | 4426 | q->limits.discard_zeroes_data = 1; |
4427 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); | 4427 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); |
4428 | queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q); | 4428 | queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q); |
diff --git a/drivers/block/umem.c b/drivers/block/umem.c index 4cf81b5bf0f7..04d65790a886 100644 --- a/drivers/block/umem.c +++ b/drivers/block/umem.c | |||
@@ -456,7 +456,7 @@ static void process_page(unsigned long data) | |||
456 | PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE); | 456 | PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE); |
457 | if (control & DMASCR_HARD_ERROR) { | 457 | if (control & DMASCR_HARD_ERROR) { |
458 | /* error */ | 458 | /* error */ |
459 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | 459 | bio->bi_error = -EIO; |
460 | dev_printk(KERN_WARNING, &card->dev->dev, | 460 | dev_printk(KERN_WARNING, &card->dev->dev, |
461 | "I/O error on sector %d/%d\n", | 461 | "I/O error on sector %d/%d\n", |
462 | le32_to_cpu(desc->local_addr)>>9, | 462 | le32_to_cpu(desc->local_addr)>>9, |
@@ -505,7 +505,7 @@ static void process_page(unsigned long data) | |||
505 | 505 | ||
506 | return_bio = bio->bi_next; | 506 | return_bio = bio->bi_next; |
507 | bio->bi_next = NULL; | 507 | bio->bi_next = NULL; |
508 | bio_endio(bio, 0); | 508 | bio_endio(bio); |
509 | } | 509 | } |
510 | } | 510 | } |
511 | 511 | ||
@@ -531,6 +531,8 @@ static void mm_make_request(struct request_queue *q, struct bio *bio) | |||
531 | (unsigned long long)bio->bi_iter.bi_sector, | 531 | (unsigned long long)bio->bi_iter.bi_sector, |
532 | bio->bi_iter.bi_size); | 532 | bio->bi_iter.bi_size); |
533 | 533 | ||
534 | blk_queue_split(q, &bio, q->bio_split); | ||
535 | |||
534 | spin_lock_irq(&card->lock); | 536 | spin_lock_irq(&card->lock); |
535 | *card->biotail = bio; | 537 | *card->biotail = bio; |
536 | bio->bi_next = NULL; | 538 | bio->bi_next = NULL; |
diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c index 954c0029fb3b..6a685aec6994 100644 --- a/drivers/block/xen-blkback/blkback.c +++ b/drivers/block/xen-blkback/blkback.c | |||
@@ -1078,9 +1078,9 @@ static void __end_block_io_op(struct pending_req *pending_req, int error) | |||
1078 | /* | 1078 | /* |
1079 | * bio callback. | 1079 | * bio callback. |
1080 | */ | 1080 | */ |
1081 | static void end_block_io_op(struct bio *bio, int error) | 1081 | static void end_block_io_op(struct bio *bio) |
1082 | { | 1082 | { |
1083 | __end_block_io_op(bio->bi_private, error); | 1083 | __end_block_io_op(bio->bi_private, bio->bi_error); |
1084 | bio_put(bio); | 1084 | bio_put(bio); |
1085 | } | 1085 | } |
1086 | 1086 | ||
diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index 7a8a73f1fc04..5f6b3be0a93c 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c | |||
@@ -82,7 +82,6 @@ struct blk_shadow { | |||
82 | struct split_bio { | 82 | struct split_bio { |
83 | struct bio *bio; | 83 | struct bio *bio; |
84 | atomic_t pending; | 84 | atomic_t pending; |
85 | int err; | ||
86 | }; | 85 | }; |
87 | 86 | ||
88 | static DEFINE_MUTEX(blkfront_mutex); | 87 | static DEFINE_MUTEX(blkfront_mutex); |
@@ -1481,16 +1480,14 @@ static int blkfront_probe(struct xenbus_device *dev, | |||
1481 | return 0; | 1480 | return 0; |
1482 | } | 1481 | } |
1483 | 1482 | ||
1484 | static void split_bio_end(struct bio *bio, int error) | 1483 | static void split_bio_end(struct bio *bio) |
1485 | { | 1484 | { |
1486 | struct split_bio *split_bio = bio->bi_private; | 1485 | struct split_bio *split_bio = bio->bi_private; |
1487 | 1486 | ||
1488 | if (error) | ||
1489 | split_bio->err = error; | ||
1490 | |||
1491 | if (atomic_dec_and_test(&split_bio->pending)) { | 1487 | if (atomic_dec_and_test(&split_bio->pending)) { |
1492 | split_bio->bio->bi_phys_segments = 0; | 1488 | split_bio->bio->bi_phys_segments = 0; |
1493 | bio_endio(split_bio->bio, split_bio->err); | 1489 | split_bio->bio->bi_error = bio->bi_error; |
1490 | bio_endio(split_bio->bio); | ||
1494 | kfree(split_bio); | 1491 | kfree(split_bio); |
1495 | } | 1492 | } |
1496 | bio_put(bio); | 1493 | bio_put(bio); |
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 763301c7828c..9c01f5bfa33f 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c | |||
@@ -848,7 +848,7 @@ static void __zram_make_request(struct zram *zram, struct bio *bio) | |||
848 | 848 | ||
849 | if (unlikely(bio->bi_rw & REQ_DISCARD)) { | 849 | if (unlikely(bio->bi_rw & REQ_DISCARD)) { |
850 | zram_bio_discard(zram, index, offset, bio); | 850 | zram_bio_discard(zram, index, offset, bio); |
851 | bio_endio(bio, 0); | 851 | bio_endio(bio); |
852 | return; | 852 | return; |
853 | } | 853 | } |
854 | 854 | ||
@@ -881,8 +881,7 @@ static void __zram_make_request(struct zram *zram, struct bio *bio) | |||
881 | update_position(&index, &offset, &bvec); | 881 | update_position(&index, &offset, &bvec); |
882 | } | 882 | } |
883 | 883 | ||
884 | set_bit(BIO_UPTODATE, &bio->bi_flags); | 884 | bio_endio(bio); |
885 | bio_endio(bio, 0); | ||
886 | return; | 885 | return; |
887 | 886 | ||
888 | out: | 887 | out: |
@@ -899,6 +898,8 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio) | |||
899 | if (unlikely(!zram_meta_get(zram))) | 898 | if (unlikely(!zram_meta_get(zram))) |
900 | goto error; | 899 | goto error; |
901 | 900 | ||
901 | blk_queue_split(queue, &bio, queue->bio_split); | ||
902 | |||
902 | if (!valid_io_request(zram, bio->bi_iter.bi_sector, | 903 | if (!valid_io_request(zram, bio->bi_iter.bi_sector, |
903 | bio->bi_iter.bi_size)) { | 904 | bio->bi_iter.bi_size)) { |
904 | atomic64_inc(&zram->stats.invalid_io); | 905 | atomic64_inc(&zram->stats.invalid_io); |
@@ -1242,7 +1243,7 @@ static int zram_add(void) | |||
1242 | blk_queue_io_min(zram->disk->queue, PAGE_SIZE); | 1243 | blk_queue_io_min(zram->disk->queue, PAGE_SIZE); |
1243 | blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); | 1244 | blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); |
1244 | zram->disk->queue->limits.discard_granularity = PAGE_SIZE; | 1245 | zram->disk->queue->limits.discard_granularity = PAGE_SIZE; |
1245 | zram->disk->queue->limits.max_discard_sectors = UINT_MAX; | 1246 | blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX); |
1246 | /* | 1247 | /* |
1247 | * zram_bio_discard() will clear all logical blocks if logical block | 1248 | * zram_bio_discard() will clear all logical blocks if logical block |
1248 | * size is identical with physical block size(PAGE_SIZE). But if it is | 1249 | * size is identical with physical block size(PAGE_SIZE). But if it is |
diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h index 04f7bc28ef83..6b420a55c745 100644 --- a/drivers/md/bcache/bcache.h +++ b/drivers/md/bcache/bcache.h | |||
@@ -243,19 +243,6 @@ struct keybuf { | |||
243 | DECLARE_ARRAY_ALLOCATOR(struct keybuf_key, freelist, KEYBUF_NR); | 243 | DECLARE_ARRAY_ALLOCATOR(struct keybuf_key, freelist, KEYBUF_NR); |
244 | }; | 244 | }; |
245 | 245 | ||
246 | struct bio_split_pool { | ||
247 | struct bio_set *bio_split; | ||
248 | mempool_t *bio_split_hook; | ||
249 | }; | ||
250 | |||
251 | struct bio_split_hook { | ||
252 | struct closure cl; | ||
253 | struct bio_split_pool *p; | ||
254 | struct bio *bio; | ||
255 | bio_end_io_t *bi_end_io; | ||
256 | void *bi_private; | ||
257 | }; | ||
258 | |||
259 | struct bcache_device { | 246 | struct bcache_device { |
260 | struct closure cl; | 247 | struct closure cl; |
261 | 248 | ||
@@ -288,8 +275,6 @@ struct bcache_device { | |||
288 | int (*cache_miss)(struct btree *, struct search *, | 275 | int (*cache_miss)(struct btree *, struct search *, |
289 | struct bio *, unsigned); | 276 | struct bio *, unsigned); |
290 | int (*ioctl) (struct bcache_device *, fmode_t, unsigned, unsigned long); | 277 | int (*ioctl) (struct bcache_device *, fmode_t, unsigned, unsigned long); |
291 | |||
292 | struct bio_split_pool bio_split_hook; | ||
293 | }; | 278 | }; |
294 | 279 | ||
295 | struct io { | 280 | struct io { |
@@ -454,8 +439,6 @@ struct cache { | |||
454 | atomic_long_t meta_sectors_written; | 439 | atomic_long_t meta_sectors_written; |
455 | atomic_long_t btree_sectors_written; | 440 | atomic_long_t btree_sectors_written; |
456 | atomic_long_t sectors_written; | 441 | atomic_long_t sectors_written; |
457 | |||
458 | struct bio_split_pool bio_split_hook; | ||
459 | }; | 442 | }; |
460 | 443 | ||
461 | struct gc_stat { | 444 | struct gc_stat { |
@@ -873,7 +856,6 @@ void bch_bbio_endio(struct cache_set *, struct bio *, int, const char *); | |||
873 | void bch_bbio_free(struct bio *, struct cache_set *); | 856 | void bch_bbio_free(struct bio *, struct cache_set *); |
874 | struct bio *bch_bbio_alloc(struct cache_set *); | 857 | struct bio *bch_bbio_alloc(struct cache_set *); |
875 | 858 | ||
876 | void bch_generic_make_request(struct bio *, struct bio_split_pool *); | ||
877 | void __bch_submit_bbio(struct bio *, struct cache_set *); | 859 | void __bch_submit_bbio(struct bio *, struct cache_set *); |
878 | void bch_submit_bbio(struct bio *, struct cache_set *, struct bkey *, unsigned); | 860 | void bch_submit_bbio(struct bio *, struct cache_set *, struct bkey *, unsigned); |
879 | 861 | ||
diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c index 00cde40db572..83392f856dfd 100644 --- a/drivers/md/bcache/btree.c +++ b/drivers/md/bcache/btree.c | |||
@@ -278,7 +278,7 @@ err: | |||
278 | goto out; | 278 | goto out; |
279 | } | 279 | } |
280 | 280 | ||
281 | static void btree_node_read_endio(struct bio *bio, int error) | 281 | static void btree_node_read_endio(struct bio *bio) |
282 | { | 282 | { |
283 | struct closure *cl = bio->bi_private; | 283 | struct closure *cl = bio->bi_private; |
284 | closure_put(cl); | 284 | closure_put(cl); |
@@ -305,7 +305,7 @@ static void bch_btree_node_read(struct btree *b) | |||
305 | bch_submit_bbio(bio, b->c, &b->key, 0); | 305 | bch_submit_bbio(bio, b->c, &b->key, 0); |
306 | closure_sync(&cl); | 306 | closure_sync(&cl); |
307 | 307 | ||
308 | if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) | 308 | if (bio->bi_error) |
309 | set_btree_node_io_error(b); | 309 | set_btree_node_io_error(b); |
310 | 310 | ||
311 | bch_bbio_free(bio, b->c); | 311 | bch_bbio_free(bio, b->c); |
@@ -371,15 +371,15 @@ static void btree_node_write_done(struct closure *cl) | |||
371 | __btree_node_write_done(cl); | 371 | __btree_node_write_done(cl); |
372 | } | 372 | } |
373 | 373 | ||
374 | static void btree_node_write_endio(struct bio *bio, int error) | 374 | static void btree_node_write_endio(struct bio *bio) |
375 | { | 375 | { |
376 | struct closure *cl = bio->bi_private; | 376 | struct closure *cl = bio->bi_private; |
377 | struct btree *b = container_of(cl, struct btree, io); | 377 | struct btree *b = container_of(cl, struct btree, io); |
378 | 378 | ||
379 | if (error) | 379 | if (bio->bi_error) |
380 | set_btree_node_io_error(b); | 380 | set_btree_node_io_error(b); |
381 | 381 | ||
382 | bch_bbio_count_io_errors(b->c, bio, error, "writing btree"); | 382 | bch_bbio_count_io_errors(b->c, bio, bio->bi_error, "writing btree"); |
383 | closure_put(cl); | 383 | closure_put(cl); |
384 | } | 384 | } |
385 | 385 | ||
diff --git a/drivers/md/bcache/closure.h b/drivers/md/bcache/closure.h index 79a6d63e8ed3..782cc2c8a185 100644 --- a/drivers/md/bcache/closure.h +++ b/drivers/md/bcache/closure.h | |||
@@ -38,7 +38,7 @@ | |||
38 | * they are running owned by the thread that is running them. Otherwise, suppose | 38 | * they are running owned by the thread that is running them. Otherwise, suppose |
39 | * you submit some bios and wish to have a function run when they all complete: | 39 | * you submit some bios and wish to have a function run when they all complete: |
40 | * | 40 | * |
41 | * foo_endio(struct bio *bio, int error) | 41 | * foo_endio(struct bio *bio) |
42 | * { | 42 | * { |
43 | * closure_put(cl); | 43 | * closure_put(cl); |
44 | * } | 44 | * } |
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c index bf6a9ca18403..86a0bb87124e 100644 --- a/drivers/md/bcache/io.c +++ b/drivers/md/bcache/io.c | |||
@@ -11,105 +11,6 @@ | |||
11 | 11 | ||
12 | #include <linux/blkdev.h> | 12 | #include <linux/blkdev.h> |
13 | 13 | ||
14 | static unsigned bch_bio_max_sectors(struct bio *bio) | ||
15 | { | ||
16 | struct request_queue *q = bdev_get_queue(bio->bi_bdev); | ||
17 | struct bio_vec bv; | ||
18 | struct bvec_iter iter; | ||
19 | unsigned ret = 0, seg = 0; | ||
20 | |||
21 | if (bio->bi_rw & REQ_DISCARD) | ||
22 | return min(bio_sectors(bio), q->limits.max_discard_sectors); | ||
23 | |||
24 | bio_for_each_segment(bv, bio, iter) { | ||
25 | struct bvec_merge_data bvm = { | ||
26 | .bi_bdev = bio->bi_bdev, | ||
27 | .bi_sector = bio->bi_iter.bi_sector, | ||
28 | .bi_size = ret << 9, | ||
29 | .bi_rw = bio->bi_rw, | ||
30 | }; | ||
31 | |||
32 | if (seg == min_t(unsigned, BIO_MAX_PAGES, | ||
33 | queue_max_segments(q))) | ||
34 | break; | ||
35 | |||
36 | if (q->merge_bvec_fn && | ||
37 | q->merge_bvec_fn(q, &bvm, &bv) < (int) bv.bv_len) | ||
38 | break; | ||
39 | |||
40 | seg++; | ||
41 | ret += bv.bv_len >> 9; | ||
42 | } | ||
43 | |||
44 | ret = min(ret, queue_max_sectors(q)); | ||
45 | |||
46 | WARN_ON(!ret); | ||
47 | ret = max_t(int, ret, bio_iovec(bio).bv_len >> 9); | ||
48 | |||
49 | return ret; | ||
50 | } | ||
51 | |||
52 | static void bch_bio_submit_split_done(struct closure *cl) | ||
53 | { | ||
54 | struct bio_split_hook *s = container_of(cl, struct bio_split_hook, cl); | ||
55 | |||
56 | s->bio->bi_end_io = s->bi_end_io; | ||
57 | s->bio->bi_private = s->bi_private; | ||
58 | bio_endio(s->bio, 0); | ||
59 | |||
60 | closure_debug_destroy(&s->cl); | ||
61 | mempool_free(s, s->p->bio_split_hook); | ||
62 | } | ||
63 | |||
64 | static void bch_bio_submit_split_endio(struct bio *bio, int error) | ||
65 | { | ||
66 | struct closure *cl = bio->bi_private; | ||
67 | struct bio_split_hook *s = container_of(cl, struct bio_split_hook, cl); | ||
68 | |||
69 | if (error) | ||
70 | clear_bit(BIO_UPTODATE, &s->bio->bi_flags); | ||
71 | |||
72 | bio_put(bio); | ||
73 | closure_put(cl); | ||
74 | } | ||
75 | |||
76 | void bch_generic_make_request(struct bio *bio, struct bio_split_pool *p) | ||
77 | { | ||
78 | struct bio_split_hook *s; | ||
79 | struct bio *n; | ||
80 | |||
81 | if (!bio_has_data(bio) && !(bio->bi_rw & REQ_DISCARD)) | ||
82 | goto submit; | ||
83 | |||
84 | if (bio_sectors(bio) <= bch_bio_max_sectors(bio)) | ||
85 | goto submit; | ||
86 | |||
87 | s = mempool_alloc(p->bio_split_hook, GFP_NOIO); | ||
88 | closure_init(&s->cl, NULL); | ||
89 | |||
90 | s->bio = bio; | ||
91 | s->p = p; | ||
92 | s->bi_end_io = bio->bi_end_io; | ||
93 | s->bi_private = bio->bi_private; | ||
94 | bio_get(bio); | ||
95 | |||
96 | do { | ||
97 | n = bio_next_split(bio, bch_bio_max_sectors(bio), | ||
98 | GFP_NOIO, s->p->bio_split); | ||
99 | |||
100 | n->bi_end_io = bch_bio_submit_split_endio; | ||
101 | n->bi_private = &s->cl; | ||
102 | |||
103 | closure_get(&s->cl); | ||
104 | generic_make_request(n); | ||
105 | } while (n != bio); | ||
106 | |||
107 | continue_at(&s->cl, bch_bio_submit_split_done, NULL); | ||
108 | return; | ||
109 | submit: | ||
110 | generic_make_request(bio); | ||
111 | } | ||
112 | |||
113 | /* Bios with headers */ | 14 | /* Bios with headers */ |
114 | 15 | ||
115 | void bch_bbio_free(struct bio *bio, struct cache_set *c) | 16 | void bch_bbio_free(struct bio *bio, struct cache_set *c) |
@@ -139,7 +40,7 @@ void __bch_submit_bbio(struct bio *bio, struct cache_set *c) | |||
139 | bio->bi_bdev = PTR_CACHE(c, &b->key, 0)->bdev; | 40 | bio->bi_bdev = PTR_CACHE(c, &b->key, 0)->bdev; |
140 | 41 | ||
141 | b->submit_time_us = local_clock_us(); | 42 | b->submit_time_us = local_clock_us(); |
142 | closure_bio_submit(bio, bio->bi_private, PTR_CACHE(c, &b->key, 0)); | 43 | closure_bio_submit(bio, bio->bi_private); |
143 | } | 44 | } |
144 | 45 | ||
145 | void bch_submit_bbio(struct bio *bio, struct cache_set *c, | 46 | void bch_submit_bbio(struct bio *bio, struct cache_set *c, |
diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c index 418607a6ba33..29eba7219b01 100644 --- a/drivers/md/bcache/journal.c +++ b/drivers/md/bcache/journal.c | |||
@@ -24,7 +24,7 @@ | |||
24 | * bit. | 24 | * bit. |
25 | */ | 25 | */ |
26 | 26 | ||
27 | static void journal_read_endio(struct bio *bio, int error) | 27 | static void journal_read_endio(struct bio *bio) |
28 | { | 28 | { |
29 | struct closure *cl = bio->bi_private; | 29 | struct closure *cl = bio->bi_private; |
30 | closure_put(cl); | 30 | closure_put(cl); |
@@ -61,7 +61,7 @@ reread: left = ca->sb.bucket_size - offset; | |||
61 | bio->bi_private = &cl; | 61 | bio->bi_private = &cl; |
62 | bch_bio_map(bio, data); | 62 | bch_bio_map(bio, data); |
63 | 63 | ||
64 | closure_bio_submit(bio, &cl, ca); | 64 | closure_bio_submit(bio, &cl); |
65 | closure_sync(&cl); | 65 | closure_sync(&cl); |
66 | 66 | ||
67 | /* This function could be simpler now since we no longer write | 67 | /* This function could be simpler now since we no longer write |
@@ -401,7 +401,7 @@ retry: | |||
401 | 401 | ||
402 | #define last_seq(j) ((j)->seq - fifo_used(&(j)->pin) + 1) | 402 | #define last_seq(j) ((j)->seq - fifo_used(&(j)->pin) + 1) |
403 | 403 | ||
404 | static void journal_discard_endio(struct bio *bio, int error) | 404 | static void journal_discard_endio(struct bio *bio) |
405 | { | 405 | { |
406 | struct journal_device *ja = | 406 | struct journal_device *ja = |
407 | container_of(bio, struct journal_device, discard_bio); | 407 | container_of(bio, struct journal_device, discard_bio); |
@@ -547,11 +547,11 @@ void bch_journal_next(struct journal *j) | |||
547 | pr_debug("journal_pin full (%zu)", fifo_used(&j->pin)); | 547 | pr_debug("journal_pin full (%zu)", fifo_used(&j->pin)); |
548 | } | 548 | } |
549 | 549 | ||
550 | static void journal_write_endio(struct bio *bio, int error) | 550 | static void journal_write_endio(struct bio *bio) |
551 | { | 551 | { |
552 | struct journal_write *w = bio->bi_private; | 552 | struct journal_write *w = bio->bi_private; |
553 | 553 | ||
554 | cache_set_err_on(error, w->c, "journal io error"); | 554 | cache_set_err_on(bio->bi_error, w->c, "journal io error"); |
555 | closure_put(&w->c->journal.io); | 555 | closure_put(&w->c->journal.io); |
556 | } | 556 | } |
557 | 557 | ||
@@ -648,7 +648,7 @@ static void journal_write_unlocked(struct closure *cl) | |||
648 | spin_unlock(&c->journal.lock); | 648 | spin_unlock(&c->journal.lock); |
649 | 649 | ||
650 | while ((bio = bio_list_pop(&list))) | 650 | while ((bio = bio_list_pop(&list))) |
651 | closure_bio_submit(bio, cl, c->cache[0]); | 651 | closure_bio_submit(bio, cl); |
652 | 652 | ||
653 | continue_at(cl, journal_write_done, NULL); | 653 | continue_at(cl, journal_write_done, NULL); |
654 | } | 654 | } |
diff --git a/drivers/md/bcache/movinggc.c b/drivers/md/bcache/movinggc.c index cd7490311e51..b929fc944e9c 100644 --- a/drivers/md/bcache/movinggc.c +++ b/drivers/md/bcache/movinggc.c | |||
@@ -60,20 +60,20 @@ static void write_moving_finish(struct closure *cl) | |||
60 | closure_return_with_destructor(cl, moving_io_destructor); | 60 | closure_return_with_destructor(cl, moving_io_destructor); |
61 | } | 61 | } |
62 | 62 | ||
63 | static void read_moving_endio(struct bio *bio, int error) | 63 | static void read_moving_endio(struct bio *bio) |
64 | { | 64 | { |
65 | struct bbio *b = container_of(bio, struct bbio, bio); | 65 | struct bbio *b = container_of(bio, struct bbio, bio); |
66 | struct moving_io *io = container_of(bio->bi_private, | 66 | struct moving_io *io = container_of(bio->bi_private, |
67 | struct moving_io, cl); | 67 | struct moving_io, cl); |
68 | 68 | ||
69 | if (error) | 69 | if (bio->bi_error) |
70 | io->op.error = error; | 70 | io->op.error = bio->bi_error; |
71 | else if (!KEY_DIRTY(&b->key) && | 71 | else if (!KEY_DIRTY(&b->key) && |
72 | ptr_stale(io->op.c, &b->key, 0)) { | 72 | ptr_stale(io->op.c, &b->key, 0)) { |
73 | io->op.error = -EINTR; | 73 | io->op.error = -EINTR; |
74 | } | 74 | } |
75 | 75 | ||
76 | bch_bbio_endio(io->op.c, bio, error, "reading data to move"); | 76 | bch_bbio_endio(io->op.c, bio, bio->bi_error, "reading data to move"); |
77 | } | 77 | } |
78 | 78 | ||
79 | static void moving_init(struct moving_io *io) | 79 | static void moving_init(struct moving_io *io) |
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index f292790997d7..8e9877b04637 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c | |||
@@ -173,22 +173,22 @@ static void bch_data_insert_error(struct closure *cl) | |||
173 | bch_data_insert_keys(cl); | 173 | bch_data_insert_keys(cl); |
174 | } | 174 | } |
175 | 175 | ||
176 | static void bch_data_insert_endio(struct bio *bio, int error) | 176 | static void bch_data_insert_endio(struct bio *bio) |
177 | { | 177 | { |
178 | struct closure *cl = bio->bi_private; | 178 | struct closure *cl = bio->bi_private; |
179 | struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); | 179 | struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); |
180 | 180 | ||
181 | if (error) { | 181 | if (bio->bi_error) { |
182 | /* TODO: We could try to recover from this. */ | 182 | /* TODO: We could try to recover from this. */ |
183 | if (op->writeback) | 183 | if (op->writeback) |
184 | op->error = error; | 184 | op->error = bio->bi_error; |
185 | else if (!op->replace) | 185 | else if (!op->replace) |
186 | set_closure_fn(cl, bch_data_insert_error, op->wq); | 186 | set_closure_fn(cl, bch_data_insert_error, op->wq); |
187 | else | 187 | else |
188 | set_closure_fn(cl, NULL, NULL); | 188 | set_closure_fn(cl, NULL, NULL); |
189 | } | 189 | } |
190 | 190 | ||
191 | bch_bbio_endio(op->c, bio, error, "writing data to cache"); | 191 | bch_bbio_endio(op->c, bio, bio->bi_error, "writing data to cache"); |
192 | } | 192 | } |
193 | 193 | ||
194 | static void bch_data_insert_start(struct closure *cl) | 194 | static void bch_data_insert_start(struct closure *cl) |
@@ -477,7 +477,7 @@ struct search { | |||
477 | struct data_insert_op iop; | 477 | struct data_insert_op iop; |
478 | }; | 478 | }; |
479 | 479 | ||
480 | static void bch_cache_read_endio(struct bio *bio, int error) | 480 | static void bch_cache_read_endio(struct bio *bio) |
481 | { | 481 | { |
482 | struct bbio *b = container_of(bio, struct bbio, bio); | 482 | struct bbio *b = container_of(bio, struct bbio, bio); |
483 | struct closure *cl = bio->bi_private; | 483 | struct closure *cl = bio->bi_private; |
@@ -490,15 +490,15 @@ static void bch_cache_read_endio(struct bio *bio, int error) | |||
490 | * from the backing device. | 490 | * from the backing device. |
491 | */ | 491 | */ |
492 | 492 | ||
493 | if (error) | 493 | if (bio->bi_error) |
494 | s->iop.error = error; | 494 | s->iop.error = bio->bi_error; |
495 | else if (!KEY_DIRTY(&b->key) && | 495 | else if (!KEY_DIRTY(&b->key) && |
496 | ptr_stale(s->iop.c, &b->key, 0)) { | 496 | ptr_stale(s->iop.c, &b->key, 0)) { |
497 | atomic_long_inc(&s->iop.c->cache_read_races); | 497 | atomic_long_inc(&s->iop.c->cache_read_races); |
498 | s->iop.error = -EINTR; | 498 | s->iop.error = -EINTR; |
499 | } | 499 | } |
500 | 500 | ||
501 | bch_bbio_endio(s->iop.c, bio, error, "reading from cache"); | 501 | bch_bbio_endio(s->iop.c, bio, bio->bi_error, "reading from cache"); |
502 | } | 502 | } |
503 | 503 | ||
504 | /* | 504 | /* |
@@ -591,13 +591,13 @@ static void cache_lookup(struct closure *cl) | |||
591 | 591 | ||
592 | /* Common code for the make_request functions */ | 592 | /* Common code for the make_request functions */ |
593 | 593 | ||
594 | static void request_endio(struct bio *bio, int error) | 594 | static void request_endio(struct bio *bio) |
595 | { | 595 | { |
596 | struct closure *cl = bio->bi_private; | 596 | struct closure *cl = bio->bi_private; |
597 | 597 | ||
598 | if (error) { | 598 | if (bio->bi_error) { |
599 | struct search *s = container_of(cl, struct search, cl); | 599 | struct search *s = container_of(cl, struct search, cl); |
600 | s->iop.error = error; | 600 | s->iop.error = bio->bi_error; |
601 | /* Only cache read errors are recoverable */ | 601 | /* Only cache read errors are recoverable */ |
602 | s->recoverable = false; | 602 | s->recoverable = false; |
603 | } | 603 | } |
@@ -613,7 +613,8 @@ static void bio_complete(struct search *s) | |||
613 | &s->d->disk->part0, s->start_time); | 613 | &s->d->disk->part0, s->start_time); |
614 | 614 | ||
615 | trace_bcache_request_end(s->d, s->orig_bio); | 615 | trace_bcache_request_end(s->d, s->orig_bio); |
616 | bio_endio(s->orig_bio, s->iop.error); | 616 | s->orig_bio->bi_error = s->iop.error; |
617 | bio_endio(s->orig_bio); | ||
617 | s->orig_bio = NULL; | 618 | s->orig_bio = NULL; |
618 | } | 619 | } |
619 | } | 620 | } |
@@ -718,7 +719,7 @@ static void cached_dev_read_error(struct closure *cl) | |||
718 | 719 | ||
719 | /* XXX: invalidate cache */ | 720 | /* XXX: invalidate cache */ |
720 | 721 | ||
721 | closure_bio_submit(bio, cl, s->d); | 722 | closure_bio_submit(bio, cl); |
722 | } | 723 | } |
723 | 724 | ||
724 | continue_at(cl, cached_dev_cache_miss_done, NULL); | 725 | continue_at(cl, cached_dev_cache_miss_done, NULL); |
@@ -841,7 +842,7 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s, | |||
841 | s->cache_miss = miss; | 842 | s->cache_miss = miss; |
842 | s->iop.bio = cache_bio; | 843 | s->iop.bio = cache_bio; |
843 | bio_get(cache_bio); | 844 | bio_get(cache_bio); |
844 | closure_bio_submit(cache_bio, &s->cl, s->d); | 845 | closure_bio_submit(cache_bio, &s->cl); |
845 | 846 | ||
846 | return ret; | 847 | return ret; |
847 | out_put: | 848 | out_put: |
@@ -849,7 +850,7 @@ out_put: | |||
849 | out_submit: | 850 | out_submit: |
850 | miss->bi_end_io = request_endio; | 851 | miss->bi_end_io = request_endio; |
851 | miss->bi_private = &s->cl; | 852 | miss->bi_private = &s->cl; |
852 | closure_bio_submit(miss, &s->cl, s->d); | 853 | closure_bio_submit(miss, &s->cl); |
853 | return ret; | 854 | return ret; |
854 | } | 855 | } |
855 | 856 | ||
@@ -914,7 +915,7 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s) | |||
914 | 915 | ||
915 | if (!(bio->bi_rw & REQ_DISCARD) || | 916 | if (!(bio->bi_rw & REQ_DISCARD) || |
916 | blk_queue_discard(bdev_get_queue(dc->bdev))) | 917 | blk_queue_discard(bdev_get_queue(dc->bdev))) |
917 | closure_bio_submit(bio, cl, s->d); | 918 | closure_bio_submit(bio, cl); |
918 | } else if (s->iop.writeback) { | 919 | } else if (s->iop.writeback) { |
919 | bch_writeback_add(dc); | 920 | bch_writeback_add(dc); |
920 | s->iop.bio = bio; | 921 | s->iop.bio = bio; |
@@ -929,12 +930,12 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s) | |||
929 | flush->bi_end_io = request_endio; | 930 | flush->bi_end_io = request_endio; |
930 | flush->bi_private = cl; | 931 | flush->bi_private = cl; |
931 | 932 | ||
932 | closure_bio_submit(flush, cl, s->d); | 933 | closure_bio_submit(flush, cl); |
933 | } | 934 | } |
934 | } else { | 935 | } else { |
935 | s->iop.bio = bio_clone_fast(bio, GFP_NOIO, dc->disk.bio_split); | 936 | s->iop.bio = bio_clone_fast(bio, GFP_NOIO, dc->disk.bio_split); |
936 | 937 | ||
937 | closure_bio_submit(bio, cl, s->d); | 938 | closure_bio_submit(bio, cl); |
938 | } | 939 | } |
939 | 940 | ||
940 | closure_call(&s->iop.cl, bch_data_insert, NULL, cl); | 941 | closure_call(&s->iop.cl, bch_data_insert, NULL, cl); |
@@ -950,7 +951,7 @@ static void cached_dev_nodata(struct closure *cl) | |||
950 | bch_journal_meta(s->iop.c, cl); | 951 | bch_journal_meta(s->iop.c, cl); |
951 | 952 | ||
952 | /* If it's a flush, we send the flush to the backing device too */ | 953 | /* If it's a flush, we send the flush to the backing device too */ |
953 | closure_bio_submit(bio, cl, s->d); | 954 | closure_bio_submit(bio, cl); |
954 | 955 | ||
955 | continue_at(cl, cached_dev_bio_complete, NULL); | 956 | continue_at(cl, cached_dev_bio_complete, NULL); |
956 | } | 957 | } |
@@ -992,9 +993,9 @@ static void cached_dev_make_request(struct request_queue *q, struct bio *bio) | |||
992 | } else { | 993 | } else { |
993 | if ((bio->bi_rw & REQ_DISCARD) && | 994 | if ((bio->bi_rw & REQ_DISCARD) && |
994 | !blk_queue_discard(bdev_get_queue(dc->bdev))) | 995 | !blk_queue_discard(bdev_get_queue(dc->bdev))) |
995 | bio_endio(bio, 0); | 996 | bio_endio(bio); |
996 | else | 997 | else |
997 | bch_generic_make_request(bio, &d->bio_split_hook); | 998 | generic_make_request(bio); |
998 | } | 999 | } |
999 | } | 1000 | } |
1000 | 1001 | ||
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 94980bfca434..679a093a3bf6 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c | |||
@@ -59,29 +59,6 @@ struct workqueue_struct *bcache_wq; | |||
59 | 59 | ||
60 | #define BTREE_MAX_PAGES (256 * 1024 / PAGE_SIZE) | 60 | #define BTREE_MAX_PAGES (256 * 1024 / PAGE_SIZE) |
61 | 61 | ||
62 | static void bio_split_pool_free(struct bio_split_pool *p) | ||
63 | { | ||
64 | if (p->bio_split_hook) | ||
65 | mempool_destroy(p->bio_split_hook); | ||
66 | |||
67 | if (p->bio_split) | ||
68 | bioset_free(p->bio_split); | ||
69 | } | ||
70 | |||
71 | static int bio_split_pool_init(struct bio_split_pool *p) | ||
72 | { | ||
73 | p->bio_split = bioset_create(4, 0); | ||
74 | if (!p->bio_split) | ||
75 | return -ENOMEM; | ||
76 | |||
77 | p->bio_split_hook = mempool_create_kmalloc_pool(4, | ||
78 | sizeof(struct bio_split_hook)); | ||
79 | if (!p->bio_split_hook) | ||
80 | return -ENOMEM; | ||
81 | |||
82 | return 0; | ||
83 | } | ||
84 | |||
85 | /* Superblock */ | 62 | /* Superblock */ |
86 | 63 | ||
87 | static const char *read_super(struct cache_sb *sb, struct block_device *bdev, | 64 | static const char *read_super(struct cache_sb *sb, struct block_device *bdev, |
@@ -221,7 +198,7 @@ err: | |||
221 | return err; | 198 | return err; |
222 | } | 199 | } |
223 | 200 | ||
224 | static void write_bdev_super_endio(struct bio *bio, int error) | 201 | static void write_bdev_super_endio(struct bio *bio) |
225 | { | 202 | { |
226 | struct cached_dev *dc = bio->bi_private; | 203 | struct cached_dev *dc = bio->bi_private; |
227 | /* XXX: error checking */ | 204 | /* XXX: error checking */ |
@@ -290,11 +267,11 @@ void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent) | |||
290 | closure_return_with_destructor(cl, bch_write_bdev_super_unlock); | 267 | closure_return_with_destructor(cl, bch_write_bdev_super_unlock); |
291 | } | 268 | } |
292 | 269 | ||
293 | static void write_super_endio(struct bio *bio, int error) | 270 | static void write_super_endio(struct bio *bio) |
294 | { | 271 | { |
295 | struct cache *ca = bio->bi_private; | 272 | struct cache *ca = bio->bi_private; |
296 | 273 | ||
297 | bch_count_io_errors(ca, error, "writing superblock"); | 274 | bch_count_io_errors(ca, bio->bi_error, "writing superblock"); |
298 | closure_put(&ca->set->sb_write); | 275 | closure_put(&ca->set->sb_write); |
299 | } | 276 | } |
300 | 277 | ||
@@ -339,12 +316,12 @@ void bcache_write_super(struct cache_set *c) | |||
339 | 316 | ||
340 | /* UUID io */ | 317 | /* UUID io */ |
341 | 318 | ||
342 | static void uuid_endio(struct bio *bio, int error) | 319 | static void uuid_endio(struct bio *bio) |
343 | { | 320 | { |
344 | struct closure *cl = bio->bi_private; | 321 | struct closure *cl = bio->bi_private; |
345 | struct cache_set *c = container_of(cl, struct cache_set, uuid_write); | 322 | struct cache_set *c = container_of(cl, struct cache_set, uuid_write); |
346 | 323 | ||
347 | cache_set_err_on(error, c, "accessing uuids"); | 324 | cache_set_err_on(bio->bi_error, c, "accessing uuids"); |
348 | bch_bbio_free(bio, c); | 325 | bch_bbio_free(bio, c); |
349 | closure_put(cl); | 326 | closure_put(cl); |
350 | } | 327 | } |
@@ -512,11 +489,11 @@ static struct uuid_entry *uuid_find_empty(struct cache_set *c) | |||
512 | * disk. | 489 | * disk. |
513 | */ | 490 | */ |
514 | 491 | ||
515 | static void prio_endio(struct bio *bio, int error) | 492 | static void prio_endio(struct bio *bio) |
516 | { | 493 | { |
517 | struct cache *ca = bio->bi_private; | 494 | struct cache *ca = bio->bi_private; |
518 | 495 | ||
519 | cache_set_err_on(error, ca->set, "accessing priorities"); | 496 | cache_set_err_on(bio->bi_error, ca->set, "accessing priorities"); |
520 | bch_bbio_free(bio, ca->set); | 497 | bch_bbio_free(bio, ca->set); |
521 | closure_put(&ca->prio); | 498 | closure_put(&ca->prio); |
522 | } | 499 | } |
@@ -537,7 +514,7 @@ static void prio_io(struct cache *ca, uint64_t bucket, unsigned long rw) | |||
537 | bio->bi_private = ca; | 514 | bio->bi_private = ca; |
538 | bch_bio_map(bio, ca->disk_buckets); | 515 | bch_bio_map(bio, ca->disk_buckets); |
539 | 516 | ||
540 | closure_bio_submit(bio, &ca->prio, ca); | 517 | closure_bio_submit(bio, &ca->prio); |
541 | closure_sync(cl); | 518 | closure_sync(cl); |
542 | } | 519 | } |
543 | 520 | ||
@@ -757,7 +734,6 @@ static void bcache_device_free(struct bcache_device *d) | |||
757 | put_disk(d->disk); | 734 | put_disk(d->disk); |
758 | } | 735 | } |
759 | 736 | ||
760 | bio_split_pool_free(&d->bio_split_hook); | ||
761 | if (d->bio_split) | 737 | if (d->bio_split) |
762 | bioset_free(d->bio_split); | 738 | bioset_free(d->bio_split); |
763 | kvfree(d->full_dirty_stripes); | 739 | kvfree(d->full_dirty_stripes); |
@@ -804,7 +780,6 @@ static int bcache_device_init(struct bcache_device *d, unsigned block_size, | |||
804 | return minor; | 780 | return minor; |
805 | 781 | ||
806 | if (!(d->bio_split = bioset_create(4, offsetof(struct bbio, bio))) || | 782 | if (!(d->bio_split = bioset_create(4, offsetof(struct bbio, bio))) || |
807 | bio_split_pool_init(&d->bio_split_hook) || | ||
808 | !(d->disk = alloc_disk(1))) { | 783 | !(d->disk = alloc_disk(1))) { |
809 | ida_simple_remove(&bcache_minor, minor); | 784 | ida_simple_remove(&bcache_minor, minor); |
810 | return -ENOMEM; | 785 | return -ENOMEM; |
@@ -830,7 +805,7 @@ static int bcache_device_init(struct bcache_device *d, unsigned block_size, | |||
830 | q->limits.max_sectors = UINT_MAX; | 805 | q->limits.max_sectors = UINT_MAX; |
831 | q->limits.max_segment_size = UINT_MAX; | 806 | q->limits.max_segment_size = UINT_MAX; |
832 | q->limits.max_segments = BIO_MAX_PAGES; | 807 | q->limits.max_segments = BIO_MAX_PAGES; |
833 | q->limits.max_discard_sectors = UINT_MAX; | 808 | blk_queue_max_discard_sectors(q, UINT_MAX); |
834 | q->limits.discard_granularity = 512; | 809 | q->limits.discard_granularity = 512; |
835 | q->limits.io_min = block_size; | 810 | q->limits.io_min = block_size; |
836 | q->limits.logical_block_size = block_size; | 811 | q->limits.logical_block_size = block_size; |
@@ -1793,8 +1768,6 @@ void bch_cache_release(struct kobject *kobj) | |||
1793 | ca->set->cache[ca->sb.nr_this_dev] = NULL; | 1768 | ca->set->cache[ca->sb.nr_this_dev] = NULL; |
1794 | } | 1769 | } |
1795 | 1770 | ||
1796 | bio_split_pool_free(&ca->bio_split_hook); | ||
1797 | |||
1798 | free_pages((unsigned long) ca->disk_buckets, ilog2(bucket_pages(ca))); | 1771 | free_pages((unsigned long) ca->disk_buckets, ilog2(bucket_pages(ca))); |
1799 | kfree(ca->prio_buckets); | 1772 | kfree(ca->prio_buckets); |
1800 | vfree(ca->buckets); | 1773 | vfree(ca->buckets); |
@@ -1839,8 +1812,7 @@ static int cache_alloc(struct cache_sb *sb, struct cache *ca) | |||
1839 | ca->sb.nbuckets)) || | 1812 | ca->sb.nbuckets)) || |
1840 | !(ca->prio_buckets = kzalloc(sizeof(uint64_t) * prio_buckets(ca) * | 1813 | !(ca->prio_buckets = kzalloc(sizeof(uint64_t) * prio_buckets(ca) * |
1841 | 2, GFP_KERNEL)) || | 1814 | 2, GFP_KERNEL)) || |
1842 | !(ca->disk_buckets = alloc_bucket_pages(GFP_KERNEL, ca)) || | 1815 | !(ca->disk_buckets = alloc_bucket_pages(GFP_KERNEL, ca))) |
1843 | bio_split_pool_init(&ca->bio_split_hook)) | ||
1844 | return -ENOMEM; | 1816 | return -ENOMEM; |
1845 | 1817 | ||
1846 | ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca); | 1818 | ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca); |
diff --git a/drivers/md/bcache/util.h b/drivers/md/bcache/util.h index 1d04c4859c70..cf2cbc211d83 100644 --- a/drivers/md/bcache/util.h +++ b/drivers/md/bcache/util.h | |||
@@ -4,6 +4,7 @@ | |||
4 | 4 | ||
5 | #include <linux/blkdev.h> | 5 | #include <linux/blkdev.h> |
6 | #include <linux/errno.h> | 6 | #include <linux/errno.h> |
7 | #include <linux/blkdev.h> | ||
7 | #include <linux/kernel.h> | 8 | #include <linux/kernel.h> |
8 | #include <linux/llist.h> | 9 | #include <linux/llist.h> |
9 | #include <linux/ratelimit.h> | 10 | #include <linux/ratelimit.h> |
@@ -570,10 +571,10 @@ static inline sector_t bdev_sectors(struct block_device *bdev) | |||
570 | return bdev->bd_inode->i_size >> 9; | 571 | return bdev->bd_inode->i_size >> 9; |
571 | } | 572 | } |
572 | 573 | ||
573 | #define closure_bio_submit(bio, cl, dev) \ | 574 | #define closure_bio_submit(bio, cl) \ |
574 | do { \ | 575 | do { \ |
575 | closure_get(cl); \ | 576 | closure_get(cl); \ |
576 | bch_generic_make_request(bio, &(dev)->bio_split_hook); \ | 577 | generic_make_request(bio); \ |
577 | } while (0) | 578 | } while (0) |
578 | 579 | ||
579 | uint64_t bch_crc64_update(uint64_t, const void *, size_t); | 580 | uint64_t bch_crc64_update(uint64_t, const void *, size_t); |
diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c index f1986bcd1bf0..b23f88d9f18c 100644 --- a/drivers/md/bcache/writeback.c +++ b/drivers/md/bcache/writeback.c | |||
@@ -166,12 +166,12 @@ static void write_dirty_finish(struct closure *cl) | |||
166 | closure_return_with_destructor(cl, dirty_io_destructor); | 166 | closure_return_with_destructor(cl, dirty_io_destructor); |
167 | } | 167 | } |
168 | 168 | ||
169 | static void dirty_endio(struct bio *bio, int error) | 169 | static void dirty_endio(struct bio *bio) |
170 | { | 170 | { |
171 | struct keybuf_key *w = bio->bi_private; | 171 | struct keybuf_key *w = bio->bi_private; |
172 | struct dirty_io *io = w->private; | 172 | struct dirty_io *io = w->private; |
173 | 173 | ||
174 | if (error) | 174 | if (bio->bi_error) |
175 | SET_KEY_DIRTY(&w->key, false); | 175 | SET_KEY_DIRTY(&w->key, false); |
176 | 176 | ||
177 | closure_put(&io->cl); | 177 | closure_put(&io->cl); |
@@ -188,27 +188,27 @@ static void write_dirty(struct closure *cl) | |||
188 | io->bio.bi_bdev = io->dc->bdev; | 188 | io->bio.bi_bdev = io->dc->bdev; |
189 | io->bio.bi_end_io = dirty_endio; | 189 | io->bio.bi_end_io = dirty_endio; |
190 | 190 | ||
191 | closure_bio_submit(&io->bio, cl, &io->dc->disk); | 191 | closure_bio_submit(&io->bio, cl); |
192 | 192 | ||
193 | continue_at(cl, write_dirty_finish, system_wq); | 193 | continue_at(cl, write_dirty_finish, system_wq); |
194 | } | 194 | } |
195 | 195 | ||
196 | static void read_dirty_endio(struct bio *bio, int error) | 196 | static void read_dirty_endio(struct bio *bio) |
197 | { | 197 | { |
198 | struct keybuf_key *w = bio->bi_private; | 198 | struct keybuf_key *w = bio->bi_private; |
199 | struct dirty_io *io = w->private; | 199 | struct dirty_io *io = w->private; |
200 | 200 | ||
201 | bch_count_io_errors(PTR_CACHE(io->dc->disk.c, &w->key, 0), | 201 | bch_count_io_errors(PTR_CACHE(io->dc->disk.c, &w->key, 0), |
202 | error, "reading dirty data from cache"); | 202 | bio->bi_error, "reading dirty data from cache"); |
203 | 203 | ||
204 | dirty_endio(bio, error); | 204 | dirty_endio(bio); |
205 | } | 205 | } |
206 | 206 | ||
207 | static void read_dirty_submit(struct closure *cl) | 207 | static void read_dirty_submit(struct closure *cl) |
208 | { | 208 | { |
209 | struct dirty_io *io = container_of(cl, struct dirty_io, cl); | 209 | struct dirty_io *io = container_of(cl, struct dirty_io, cl); |
210 | 210 | ||
211 | closure_bio_submit(&io->bio, cl, &io->dc->disk); | 211 | closure_bio_submit(&io->bio, cl); |
212 | 212 | ||
213 | continue_at(cl, write_dirty, system_wq); | 213 | continue_at(cl, write_dirty, system_wq); |
214 | } | 214 | } |
diff --git a/drivers/md/dm-bio-prison.c b/drivers/md/dm-bio-prison.c index cd6d1d21e057..03af174485d3 100644 --- a/drivers/md/dm-bio-prison.c +++ b/drivers/md/dm-bio-prison.c | |||
@@ -236,8 +236,10 @@ void dm_cell_error(struct dm_bio_prison *prison, | |||
236 | bio_list_init(&bios); | 236 | bio_list_init(&bios); |
237 | dm_cell_release(prison, cell, &bios); | 237 | dm_cell_release(prison, cell, &bios); |
238 | 238 | ||
239 | while ((bio = bio_list_pop(&bios))) | 239 | while ((bio = bio_list_pop(&bios))) { |
240 | bio_endio(bio, error); | 240 | bio->bi_error = error; |
241 | bio_endio(bio); | ||
242 | } | ||
241 | } | 243 | } |
242 | EXPORT_SYMBOL_GPL(dm_cell_error); | 244 | EXPORT_SYMBOL_GPL(dm_cell_error); |
243 | 245 | ||
diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index 86dbbc737402..83cc52eaf56d 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c | |||
@@ -545,7 +545,8 @@ static void dmio_complete(unsigned long error, void *context) | |||
545 | { | 545 | { |
546 | struct dm_buffer *b = context; | 546 | struct dm_buffer *b = context; |
547 | 547 | ||
548 | b->bio.bi_end_io(&b->bio, error ? -EIO : 0); | 548 | b->bio.bi_error = error ? -EIO : 0; |
549 | b->bio.bi_end_io(&b->bio); | ||
549 | } | 550 | } |
550 | 551 | ||
551 | static void use_dmio(struct dm_buffer *b, int rw, sector_t block, | 552 | static void use_dmio(struct dm_buffer *b, int rw, sector_t block, |
@@ -575,13 +576,16 @@ static void use_dmio(struct dm_buffer *b, int rw, sector_t block, | |||
575 | b->bio.bi_end_io = end_io; | 576 | b->bio.bi_end_io = end_io; |
576 | 577 | ||
577 | r = dm_io(&io_req, 1, ®ion, NULL); | 578 | r = dm_io(&io_req, 1, ®ion, NULL); |
578 | if (r) | 579 | if (r) { |
579 | end_io(&b->bio, r); | 580 | b->bio.bi_error = r; |
581 | end_io(&b->bio); | ||
582 | } | ||
580 | } | 583 | } |
581 | 584 | ||
582 | static void inline_endio(struct bio *bio, int error) | 585 | static void inline_endio(struct bio *bio) |
583 | { | 586 | { |
584 | bio_end_io_t *end_fn = bio->bi_private; | 587 | bio_end_io_t *end_fn = bio->bi_private; |
588 | int error = bio->bi_error; | ||
585 | 589 | ||
586 | /* | 590 | /* |
587 | * Reset the bio to free any attached resources | 591 | * Reset the bio to free any attached resources |
@@ -589,7 +593,8 @@ static void inline_endio(struct bio *bio, int error) | |||
589 | */ | 593 | */ |
590 | bio_reset(bio); | 594 | bio_reset(bio); |
591 | 595 | ||
592 | end_fn(bio, error); | 596 | bio->bi_error = error; |
597 | end_fn(bio); | ||
593 | } | 598 | } |
594 | 599 | ||
595 | static void use_inline_bio(struct dm_buffer *b, int rw, sector_t block, | 600 | static void use_inline_bio(struct dm_buffer *b, int rw, sector_t block, |
@@ -661,13 +666,14 @@ static void submit_io(struct dm_buffer *b, int rw, sector_t block, | |||
661 | * Set the error, clear B_WRITING bit and wake anyone who was waiting on | 666 | * Set the error, clear B_WRITING bit and wake anyone who was waiting on |
662 | * it. | 667 | * it. |
663 | */ | 668 | */ |
664 | static void write_endio(struct bio *bio, int error) | 669 | static void write_endio(struct bio *bio) |
665 | { | 670 | { |
666 | struct dm_buffer *b = container_of(bio, struct dm_buffer, bio); | 671 | struct dm_buffer *b = container_of(bio, struct dm_buffer, bio); |
667 | 672 | ||
668 | b->write_error = error; | 673 | b->write_error = bio->bi_error; |
669 | if (unlikely(error)) { | 674 | if (unlikely(bio->bi_error)) { |
670 | struct dm_bufio_client *c = b->c; | 675 | struct dm_bufio_client *c = b->c; |
676 | int error = bio->bi_error; | ||
671 | (void)cmpxchg(&c->async_write_error, 0, error); | 677 | (void)cmpxchg(&c->async_write_error, 0, error); |
672 | } | 678 | } |
673 | 679 | ||
@@ -1026,11 +1032,11 @@ found_buffer: | |||
1026 | * The endio routine for reading: set the error, clear the bit and wake up | 1032 | * The endio routine for reading: set the error, clear the bit and wake up |
1027 | * anyone waiting on the buffer. | 1033 | * anyone waiting on the buffer. |
1028 | */ | 1034 | */ |
1029 | static void read_endio(struct bio *bio, int error) | 1035 | static void read_endio(struct bio *bio) |
1030 | { | 1036 | { |
1031 | struct dm_buffer *b = container_of(bio, struct dm_buffer, bio); | 1037 | struct dm_buffer *b = container_of(bio, struct dm_buffer, bio); |
1032 | 1038 | ||
1033 | b->read_error = error; | 1039 | b->read_error = bio->bi_error; |
1034 | 1040 | ||
1035 | BUG_ON(!test_bit(B_READING, &b->state)); | 1041 | BUG_ON(!test_bit(B_READING, &b->state)); |
1036 | 1042 | ||
diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index 1fe93cfea7d3..7245071778db 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c | |||
@@ -919,14 +919,14 @@ static void defer_writethrough_bio(struct cache *cache, struct bio *bio) | |||
919 | wake_worker(cache); | 919 | wake_worker(cache); |
920 | } | 920 | } |
921 | 921 | ||
922 | static void writethrough_endio(struct bio *bio, int err) | 922 | static void writethrough_endio(struct bio *bio) |
923 | { | 923 | { |
924 | struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT); | 924 | struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT); |
925 | 925 | ||
926 | dm_unhook_bio(&pb->hook_info, bio); | 926 | dm_unhook_bio(&pb->hook_info, bio); |
927 | 927 | ||
928 | if (err) { | 928 | if (bio->bi_error) { |
929 | bio_endio(bio, err); | 929 | bio_endio(bio); |
930 | return; | 930 | return; |
931 | } | 931 | } |
932 | 932 | ||
@@ -1231,7 +1231,7 @@ static void migration_success_post_commit(struct dm_cache_migration *mg) | |||
1231 | * The block was promoted via an overwrite, so it's dirty. | 1231 | * The block was promoted via an overwrite, so it's dirty. |
1232 | */ | 1232 | */ |
1233 | set_dirty(cache, mg->new_oblock, mg->cblock); | 1233 | set_dirty(cache, mg->new_oblock, mg->cblock); |
1234 | bio_endio(mg->new_ocell->holder, 0); | 1234 | bio_endio(mg->new_ocell->holder); |
1235 | cell_defer(cache, mg->new_ocell, false); | 1235 | cell_defer(cache, mg->new_ocell, false); |
1236 | } | 1236 | } |
1237 | free_io_migration(mg); | 1237 | free_io_migration(mg); |
@@ -1284,7 +1284,7 @@ static void issue_copy(struct dm_cache_migration *mg) | |||
1284 | } | 1284 | } |
1285 | } | 1285 | } |
1286 | 1286 | ||
1287 | static void overwrite_endio(struct bio *bio, int err) | 1287 | static void overwrite_endio(struct bio *bio) |
1288 | { | 1288 | { |
1289 | struct dm_cache_migration *mg = bio->bi_private; | 1289 | struct dm_cache_migration *mg = bio->bi_private; |
1290 | struct cache *cache = mg->cache; | 1290 | struct cache *cache = mg->cache; |
@@ -1294,7 +1294,7 @@ static void overwrite_endio(struct bio *bio, int err) | |||
1294 | 1294 | ||
1295 | dm_unhook_bio(&pb->hook_info, bio); | 1295 | dm_unhook_bio(&pb->hook_info, bio); |
1296 | 1296 | ||
1297 | if (err) | 1297 | if (bio->bi_error) |
1298 | mg->err = true; | 1298 | mg->err = true; |
1299 | 1299 | ||
1300 | mg->requeue_holder = false; | 1300 | mg->requeue_holder = false; |
@@ -1358,7 +1358,7 @@ static void issue_discard(struct dm_cache_migration *mg) | |||
1358 | b = to_dblock(from_dblock(b) + 1); | 1358 | b = to_dblock(from_dblock(b) + 1); |
1359 | } | 1359 | } |
1360 | 1360 | ||
1361 | bio_endio(bio, 0); | 1361 | bio_endio(bio); |
1362 | cell_defer(mg->cache, mg->new_ocell, false); | 1362 | cell_defer(mg->cache, mg->new_ocell, false); |
1363 | free_migration(mg); | 1363 | free_migration(mg); |
1364 | } | 1364 | } |
@@ -1631,7 +1631,7 @@ static void process_discard_bio(struct cache *cache, struct prealloc *structs, | |||
1631 | 1631 | ||
1632 | calc_discard_block_range(cache, bio, &b, &e); | 1632 | calc_discard_block_range(cache, bio, &b, &e); |
1633 | if (b == e) { | 1633 | if (b == e) { |
1634 | bio_endio(bio, 0); | 1634 | bio_endio(bio); |
1635 | return; | 1635 | return; |
1636 | } | 1636 | } |
1637 | 1637 | ||
@@ -2217,8 +2217,10 @@ static void requeue_deferred_bios(struct cache *cache) | |||
2217 | bio_list_merge(&bios, &cache->deferred_bios); | 2217 | bio_list_merge(&bios, &cache->deferred_bios); |
2218 | bio_list_init(&cache->deferred_bios); | 2218 | bio_list_init(&cache->deferred_bios); |
2219 | 2219 | ||
2220 | while ((bio = bio_list_pop(&bios))) | 2220 | while ((bio = bio_list_pop(&bios))) { |
2221 | bio_endio(bio, DM_ENDIO_REQUEUE); | 2221 | bio->bi_error = DM_ENDIO_REQUEUE; |
2222 | bio_endio(bio); | ||
2223 | } | ||
2222 | } | 2224 | } |
2223 | 2225 | ||
2224 | static int more_work(struct cache *cache) | 2226 | static int more_work(struct cache *cache) |
@@ -3123,7 +3125,7 @@ static int cache_map(struct dm_target *ti, struct bio *bio) | |||
3123 | * This is a duplicate writethrough io that is no | 3125 | * This is a duplicate writethrough io that is no |
3124 | * longer needed because the block has been demoted. | 3126 | * longer needed because the block has been demoted. |
3125 | */ | 3127 | */ |
3126 | bio_endio(bio, 0); | 3128 | bio_endio(bio); |
3127 | // FIXME: remap everything as a miss | 3129 | // FIXME: remap everything as a miss |
3128 | cell_defer(cache, cell, false); | 3130 | cell_defer(cache, cell, false); |
3129 | r = DM_MAPIO_SUBMITTED; | 3131 | r = DM_MAPIO_SUBMITTED; |
@@ -3778,26 +3780,6 @@ static int cache_iterate_devices(struct dm_target *ti, | |||
3778 | return r; | 3780 | return r; |
3779 | } | 3781 | } |
3780 | 3782 | ||
3781 | /* | ||
3782 | * We assume I/O is going to the origin (which is the volume | ||
3783 | * more likely to have restrictions e.g. by being striped). | ||
3784 | * (Looking up the exact location of the data would be expensive | ||
3785 | * and could always be out of date by the time the bio is submitted.) | ||
3786 | */ | ||
3787 | static int cache_bvec_merge(struct dm_target *ti, | ||
3788 | struct bvec_merge_data *bvm, | ||
3789 | struct bio_vec *biovec, int max_size) | ||
3790 | { | ||
3791 | struct cache *cache = ti->private; | ||
3792 | struct request_queue *q = bdev_get_queue(cache->origin_dev->bdev); | ||
3793 | |||
3794 | if (!q->merge_bvec_fn) | ||
3795 | return max_size; | ||
3796 | |||
3797 | bvm->bi_bdev = cache->origin_dev->bdev; | ||
3798 | return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); | ||
3799 | } | ||
3800 | |||
3801 | static void set_discard_limits(struct cache *cache, struct queue_limits *limits) | 3783 | static void set_discard_limits(struct cache *cache, struct queue_limits *limits) |
3802 | { | 3784 | { |
3803 | /* | 3785 | /* |
@@ -3841,7 +3823,6 @@ static struct target_type cache_target = { | |||
3841 | .status = cache_status, | 3823 | .status = cache_status, |
3842 | .message = cache_message, | 3824 | .message = cache_message, |
3843 | .iterate_devices = cache_iterate_devices, | 3825 | .iterate_devices = cache_iterate_devices, |
3844 | .merge = cache_bvec_merge, | ||
3845 | .io_hints = cache_io_hints, | 3826 | .io_hints = cache_io_hints, |
3846 | }; | 3827 | }; |
3847 | 3828 | ||
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 0f48fed44a17..ba5c2105f4e6 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c | |||
@@ -1076,7 +1076,8 @@ static void crypt_dec_pending(struct dm_crypt_io *io) | |||
1076 | if (io->ctx.req) | 1076 | if (io->ctx.req) |
1077 | crypt_free_req(cc, io->ctx.req, base_bio); | 1077 | crypt_free_req(cc, io->ctx.req, base_bio); |
1078 | 1078 | ||
1079 | bio_endio(base_bio, error); | 1079 | base_bio->bi_error = error; |
1080 | bio_endio(base_bio); | ||
1080 | } | 1081 | } |
1081 | 1082 | ||
1082 | /* | 1083 | /* |
@@ -1096,14 +1097,12 @@ static void crypt_dec_pending(struct dm_crypt_io *io) | |||
1096 | * The work is done per CPU global for all dm-crypt instances. | 1097 | * The work is done per CPU global for all dm-crypt instances. |
1097 | * They should not depend on each other and do not block. | 1098 | * They should not depend on each other and do not block. |
1098 | */ | 1099 | */ |
1099 | static void crypt_endio(struct bio *clone, int error) | 1100 | static void crypt_endio(struct bio *clone) |
1100 | { | 1101 | { |
1101 | struct dm_crypt_io *io = clone->bi_private; | 1102 | struct dm_crypt_io *io = clone->bi_private; |
1102 | struct crypt_config *cc = io->cc; | 1103 | struct crypt_config *cc = io->cc; |
1103 | unsigned rw = bio_data_dir(clone); | 1104 | unsigned rw = bio_data_dir(clone); |
1104 | 1105 | int error; | |
1105 | if (unlikely(!bio_flagged(clone, BIO_UPTODATE) && !error)) | ||
1106 | error = -EIO; | ||
1107 | 1106 | ||
1108 | /* | 1107 | /* |
1109 | * free the processed pages | 1108 | * free the processed pages |
@@ -1111,6 +1110,7 @@ static void crypt_endio(struct bio *clone, int error) | |||
1111 | if (rw == WRITE) | 1110 | if (rw == WRITE) |
1112 | crypt_free_buffer_pages(cc, clone); | 1111 | crypt_free_buffer_pages(cc, clone); |
1113 | 1112 | ||
1113 | error = clone->bi_error; | ||
1114 | bio_put(clone); | 1114 | bio_put(clone); |
1115 | 1115 | ||
1116 | if (rw == READ && !error) { | 1116 | if (rw == READ && !error) { |
@@ -2035,21 +2035,6 @@ error: | |||
2035 | return -EINVAL; | 2035 | return -EINVAL; |
2036 | } | 2036 | } |
2037 | 2037 | ||
2038 | static int crypt_merge(struct dm_target *ti, struct bvec_merge_data *bvm, | ||
2039 | struct bio_vec *biovec, int max_size) | ||
2040 | { | ||
2041 | struct crypt_config *cc = ti->private; | ||
2042 | struct request_queue *q = bdev_get_queue(cc->dev->bdev); | ||
2043 | |||
2044 | if (!q->merge_bvec_fn) | ||
2045 | return max_size; | ||
2046 | |||
2047 | bvm->bi_bdev = cc->dev->bdev; | ||
2048 | bvm->bi_sector = cc->start + dm_target_offset(ti, bvm->bi_sector); | ||
2049 | |||
2050 | return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); | ||
2051 | } | ||
2052 | |||
2053 | static int crypt_iterate_devices(struct dm_target *ti, | 2038 | static int crypt_iterate_devices(struct dm_target *ti, |
2054 | iterate_devices_callout_fn fn, void *data) | 2039 | iterate_devices_callout_fn fn, void *data) |
2055 | { | 2040 | { |
@@ -2070,7 +2055,6 @@ static struct target_type crypt_target = { | |||
2070 | .preresume = crypt_preresume, | 2055 | .preresume = crypt_preresume, |
2071 | .resume = crypt_resume, | 2056 | .resume = crypt_resume, |
2072 | .message = crypt_message, | 2057 | .message = crypt_message, |
2073 | .merge = crypt_merge, | ||
2074 | .iterate_devices = crypt_iterate_devices, | 2058 | .iterate_devices = crypt_iterate_devices, |
2075 | }; | 2059 | }; |
2076 | 2060 | ||
diff --git a/drivers/md/dm-era-target.c b/drivers/md/dm-era-target.c index ad913cd4aded..0119ebfb3d49 100644 --- a/drivers/md/dm-era-target.c +++ b/drivers/md/dm-era-target.c | |||
@@ -1673,20 +1673,6 @@ static int era_iterate_devices(struct dm_target *ti, | |||
1673 | return fn(ti, era->origin_dev, 0, get_dev_size(era->origin_dev), data); | 1673 | return fn(ti, era->origin_dev, 0, get_dev_size(era->origin_dev), data); |
1674 | } | 1674 | } |
1675 | 1675 | ||
1676 | static int era_merge(struct dm_target *ti, struct bvec_merge_data *bvm, | ||
1677 | struct bio_vec *biovec, int max_size) | ||
1678 | { | ||
1679 | struct era *era = ti->private; | ||
1680 | struct request_queue *q = bdev_get_queue(era->origin_dev->bdev); | ||
1681 | |||
1682 | if (!q->merge_bvec_fn) | ||
1683 | return max_size; | ||
1684 | |||
1685 | bvm->bi_bdev = era->origin_dev->bdev; | ||
1686 | |||
1687 | return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); | ||
1688 | } | ||
1689 | |||
1690 | static void era_io_hints(struct dm_target *ti, struct queue_limits *limits) | 1676 | static void era_io_hints(struct dm_target *ti, struct queue_limits *limits) |
1691 | { | 1677 | { |
1692 | struct era *era = ti->private; | 1678 | struct era *era = ti->private; |
@@ -1717,7 +1703,6 @@ static struct target_type era_target = { | |||
1717 | .status = era_status, | 1703 | .status = era_status, |
1718 | .message = era_message, | 1704 | .message = era_message, |
1719 | .iterate_devices = era_iterate_devices, | 1705 | .iterate_devices = era_iterate_devices, |
1720 | .merge = era_merge, | ||
1721 | .io_hints = era_io_hints | 1706 | .io_hints = era_io_hints |
1722 | }; | 1707 | }; |
1723 | 1708 | ||
diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c index b257e46876d3..afab13bd683e 100644 --- a/drivers/md/dm-flakey.c +++ b/drivers/md/dm-flakey.c | |||
@@ -296,7 +296,7 @@ static int flakey_map(struct dm_target *ti, struct bio *bio) | |||
296 | * Drop writes? | 296 | * Drop writes? |
297 | */ | 297 | */ |
298 | if (test_bit(DROP_WRITES, &fc->flags)) { | 298 | if (test_bit(DROP_WRITES, &fc->flags)) { |
299 | bio_endio(bio, 0); | 299 | bio_endio(bio); |
300 | return DM_MAPIO_SUBMITTED; | 300 | return DM_MAPIO_SUBMITTED; |
301 | } | 301 | } |
302 | 302 | ||
@@ -387,21 +387,6 @@ static int flakey_ioctl(struct dm_target *ti, unsigned int cmd, unsigned long ar | |||
387 | return r ? : __blkdev_driver_ioctl(dev->bdev, dev->mode, cmd, arg); | 387 | return r ? : __blkdev_driver_ioctl(dev->bdev, dev->mode, cmd, arg); |
388 | } | 388 | } |
389 | 389 | ||
390 | static int flakey_merge(struct dm_target *ti, struct bvec_merge_data *bvm, | ||
391 | struct bio_vec *biovec, int max_size) | ||
392 | { | ||
393 | struct flakey_c *fc = ti->private; | ||
394 | struct request_queue *q = bdev_get_queue(fc->dev->bdev); | ||
395 | |||
396 | if (!q->merge_bvec_fn) | ||
397 | return max_size; | ||
398 | |||
399 | bvm->bi_bdev = fc->dev->bdev; | ||
400 | bvm->bi_sector = flakey_map_sector(ti, bvm->bi_sector); | ||
401 | |||
402 | return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); | ||
403 | } | ||
404 | |||
405 | static int flakey_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data) | 390 | static int flakey_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data) |
406 | { | 391 | { |
407 | struct flakey_c *fc = ti->private; | 392 | struct flakey_c *fc = ti->private; |
@@ -419,7 +404,6 @@ static struct target_type flakey_target = { | |||
419 | .end_io = flakey_end_io, | 404 | .end_io = flakey_end_io, |
420 | .status = flakey_status, | 405 | .status = flakey_status, |
421 | .ioctl = flakey_ioctl, | 406 | .ioctl = flakey_ioctl, |
422 | .merge = flakey_merge, | ||
423 | .iterate_devices = flakey_iterate_devices, | 407 | .iterate_devices = flakey_iterate_devices, |
424 | }; | 408 | }; |
425 | 409 | ||
diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c index 74adcd2c967e..6f8e83b2a6f8 100644 --- a/drivers/md/dm-io.c +++ b/drivers/md/dm-io.c | |||
@@ -134,12 +134,13 @@ static void dec_count(struct io *io, unsigned int region, int error) | |||
134 | complete_io(io); | 134 | complete_io(io); |
135 | } | 135 | } |
136 | 136 | ||
137 | static void endio(struct bio *bio, int error) | 137 | static void endio(struct bio *bio) |
138 | { | 138 | { |
139 | struct io *io; | 139 | struct io *io; |
140 | unsigned region; | 140 | unsigned region; |
141 | int error; | ||
141 | 142 | ||
142 | if (error && bio_data_dir(bio) == READ) | 143 | if (bio->bi_error && bio_data_dir(bio) == READ) |
143 | zero_fill_bio(bio); | 144 | zero_fill_bio(bio); |
144 | 145 | ||
145 | /* | 146 | /* |
@@ -147,6 +148,7 @@ static void endio(struct bio *bio, int error) | |||
147 | */ | 148 | */ |
148 | retrieve_io_and_region_from_bio(bio, &io, ®ion); | 149 | retrieve_io_and_region_from_bio(bio, &io, ®ion); |
149 | 150 | ||
151 | error = bio->bi_error; | ||
150 | bio_put(bio); | 152 | bio_put(bio); |
151 | 153 | ||
152 | dec_count(io, region, error); | 154 | dec_count(io, region, error); |
@@ -314,7 +316,7 @@ static void do_region(int rw, unsigned region, struct dm_io_region *where, | |||
314 | if ((rw & REQ_DISCARD) || (rw & REQ_WRITE_SAME)) | 316 | if ((rw & REQ_DISCARD) || (rw & REQ_WRITE_SAME)) |
315 | num_bvecs = 1; | 317 | num_bvecs = 1; |
316 | else | 318 | else |
317 | num_bvecs = min_t(int, bio_get_nr_vecs(where->bdev), | 319 | num_bvecs = min_t(int, BIO_MAX_PAGES, |
318 | dm_sector_div_up(remaining, (PAGE_SIZE >> SECTOR_SHIFT))); | 320 | dm_sector_div_up(remaining, (PAGE_SIZE >> SECTOR_SHIFT))); |
319 | 321 | ||
320 | bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, io->client->bios); | 322 | bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, io->client->bios); |
diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c index 53e848c10939..7dd5fc8e3eea 100644 --- a/drivers/md/dm-linear.c +++ b/drivers/md/dm-linear.c | |||
@@ -130,21 +130,6 @@ static int linear_ioctl(struct dm_target *ti, unsigned int cmd, | |||
130 | return r ? : __blkdev_driver_ioctl(dev->bdev, dev->mode, cmd, arg); | 130 | return r ? : __blkdev_driver_ioctl(dev->bdev, dev->mode, cmd, arg); |
131 | } | 131 | } |
132 | 132 | ||
133 | static int linear_merge(struct dm_target *ti, struct bvec_merge_data *bvm, | ||
134 | struct bio_vec *biovec, int max_size) | ||
135 | { | ||
136 | struct linear_c *lc = ti->private; | ||
137 | struct request_queue *q = bdev_get_queue(lc->dev->bdev); | ||
138 | |||
139 | if (!q->merge_bvec_fn) | ||
140 | return max_size; | ||
141 | |||
142 | bvm->bi_bdev = lc->dev->bdev; | ||
143 | bvm->bi_sector = linear_map_sector(ti, bvm->bi_sector); | ||
144 | |||
145 | return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); | ||
146 | } | ||
147 | |||
148 | static int linear_iterate_devices(struct dm_target *ti, | 133 | static int linear_iterate_devices(struct dm_target *ti, |
149 | iterate_devices_callout_fn fn, void *data) | 134 | iterate_devices_callout_fn fn, void *data) |
150 | { | 135 | { |
@@ -162,7 +147,6 @@ static struct target_type linear_target = { | |||
162 | .map = linear_map, | 147 | .map = linear_map, |
163 | .status = linear_status, | 148 | .status = linear_status, |
164 | .ioctl = linear_ioctl, | 149 | .ioctl = linear_ioctl, |
165 | .merge = linear_merge, | ||
166 | .iterate_devices = linear_iterate_devices, | 150 | .iterate_devices = linear_iterate_devices, |
167 | }; | 151 | }; |
168 | 152 | ||
diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c index ad1b049ae2ab..316cc3fb741f 100644 --- a/drivers/md/dm-log-writes.c +++ b/drivers/md/dm-log-writes.c | |||
@@ -146,16 +146,16 @@ static void put_io_block(struct log_writes_c *lc) | |||
146 | } | 146 | } |
147 | } | 147 | } |
148 | 148 | ||
149 | static void log_end_io(struct bio *bio, int err) | 149 | static void log_end_io(struct bio *bio) |
150 | { | 150 | { |
151 | struct log_writes_c *lc = bio->bi_private; | 151 | struct log_writes_c *lc = bio->bi_private; |
152 | struct bio_vec *bvec; | 152 | struct bio_vec *bvec; |
153 | int i; | 153 | int i; |
154 | 154 | ||
155 | if (err) { | 155 | if (bio->bi_error) { |
156 | unsigned long flags; | 156 | unsigned long flags; |
157 | 157 | ||
158 | DMERR("Error writing log block, error=%d", err); | 158 | DMERR("Error writing log block, error=%d", bio->bi_error); |
159 | spin_lock_irqsave(&lc->blocks_lock, flags); | 159 | spin_lock_irqsave(&lc->blocks_lock, flags); |
160 | lc->logging_enabled = false; | 160 | lc->logging_enabled = false; |
161 | spin_unlock_irqrestore(&lc->blocks_lock, flags); | 161 | spin_unlock_irqrestore(&lc->blocks_lock, flags); |
@@ -205,7 +205,6 @@ static int write_metadata(struct log_writes_c *lc, void *entry, | |||
205 | bio->bi_bdev = lc->logdev->bdev; | 205 | bio->bi_bdev = lc->logdev->bdev; |
206 | bio->bi_end_io = log_end_io; | 206 | bio->bi_end_io = log_end_io; |
207 | bio->bi_private = lc; | 207 | bio->bi_private = lc; |
208 | set_bit(BIO_UPTODATE, &bio->bi_flags); | ||
209 | 208 | ||
210 | page = alloc_page(GFP_KERNEL); | 209 | page = alloc_page(GFP_KERNEL); |
211 | if (!page) { | 210 | if (!page) { |
@@ -270,7 +269,6 @@ static int log_one_block(struct log_writes_c *lc, | |||
270 | bio->bi_bdev = lc->logdev->bdev; | 269 | bio->bi_bdev = lc->logdev->bdev; |
271 | bio->bi_end_io = log_end_io; | 270 | bio->bi_end_io = log_end_io; |
272 | bio->bi_private = lc; | 271 | bio->bi_private = lc; |
273 | set_bit(BIO_UPTODATE, &bio->bi_flags); | ||
274 | 272 | ||
275 | for (i = 0; i < block->vec_cnt; i++) { | 273 | for (i = 0; i < block->vec_cnt; i++) { |
276 | /* | 274 | /* |
@@ -292,7 +290,6 @@ static int log_one_block(struct log_writes_c *lc, | |||
292 | bio->bi_bdev = lc->logdev->bdev; | 290 | bio->bi_bdev = lc->logdev->bdev; |
293 | bio->bi_end_io = log_end_io; | 291 | bio->bi_end_io = log_end_io; |
294 | bio->bi_private = lc; | 292 | bio->bi_private = lc; |
295 | set_bit(BIO_UPTODATE, &bio->bi_flags); | ||
296 | 293 | ||
297 | ret = bio_add_page(bio, block->vecs[i].bv_page, | 294 | ret = bio_add_page(bio, block->vecs[i].bv_page, |
298 | block->vecs[i].bv_len, 0); | 295 | block->vecs[i].bv_len, 0); |
@@ -606,7 +603,7 @@ static int log_writes_map(struct dm_target *ti, struct bio *bio) | |||
606 | WARN_ON(flush_bio || fua_bio); | 603 | WARN_ON(flush_bio || fua_bio); |
607 | if (lc->device_supports_discard) | 604 | if (lc->device_supports_discard) |
608 | goto map_bio; | 605 | goto map_bio; |
609 | bio_endio(bio, 0); | 606 | bio_endio(bio); |
610 | return DM_MAPIO_SUBMITTED; | 607 | return DM_MAPIO_SUBMITTED; |
611 | } | 608 | } |
612 | 609 | ||
@@ -728,21 +725,6 @@ static int log_writes_ioctl(struct dm_target *ti, unsigned int cmd, | |||
728 | return r ? : __blkdev_driver_ioctl(dev->bdev, dev->mode, cmd, arg); | 725 | return r ? : __blkdev_driver_ioctl(dev->bdev, dev->mode, cmd, arg); |
729 | } | 726 | } |
730 | 727 | ||
731 | static int log_writes_merge(struct dm_target *ti, struct bvec_merge_data *bvm, | ||
732 | struct bio_vec *biovec, int max_size) | ||
733 | { | ||
734 | struct log_writes_c *lc = ti->private; | ||
735 | struct request_queue *q = bdev_get_queue(lc->dev->bdev); | ||
736 | |||
737 | if (!q->merge_bvec_fn) | ||
738 | return max_size; | ||
739 | |||
740 | bvm->bi_bdev = lc->dev->bdev; | ||
741 | bvm->bi_sector = dm_target_offset(ti, bvm->bi_sector); | ||
742 | |||
743 | return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); | ||
744 | } | ||
745 | |||
746 | static int log_writes_iterate_devices(struct dm_target *ti, | 728 | static int log_writes_iterate_devices(struct dm_target *ti, |
747 | iterate_devices_callout_fn fn, | 729 | iterate_devices_callout_fn fn, |
748 | void *data) | 730 | void *data) |
@@ -796,7 +778,6 @@ static struct target_type log_writes_target = { | |||
796 | .end_io = normal_end_io, | 778 | .end_io = normal_end_io, |
797 | .status = log_writes_status, | 779 | .status = log_writes_status, |
798 | .ioctl = log_writes_ioctl, | 780 | .ioctl = log_writes_ioctl, |
799 | .merge = log_writes_merge, | ||
800 | .message = log_writes_message, | 781 | .message = log_writes_message, |
801 | .iterate_devices = log_writes_iterate_devices, | 782 | .iterate_devices = log_writes_iterate_devices, |
802 | .io_hints = log_writes_io_hints, | 783 | .io_hints = log_writes_io_hints, |
diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index 2daa67793511..97e165183e79 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c | |||
@@ -1717,24 +1717,6 @@ static void raid_resume(struct dm_target *ti) | |||
1717 | mddev_resume(&rs->md); | 1717 | mddev_resume(&rs->md); |
1718 | } | 1718 | } |
1719 | 1719 | ||
1720 | static int raid_merge(struct dm_target *ti, struct bvec_merge_data *bvm, | ||
1721 | struct bio_vec *biovec, int max_size) | ||
1722 | { | ||
1723 | struct raid_set *rs = ti->private; | ||
1724 | struct md_personality *pers = rs->md.pers; | ||
1725 | |||
1726 | if (pers && pers->mergeable_bvec) | ||
1727 | return min(max_size, pers->mergeable_bvec(&rs->md, bvm, biovec)); | ||
1728 | |||
1729 | /* | ||
1730 | * In case we can't request the personality because | ||
1731 | * the raid set is not running yet | ||
1732 | * | ||
1733 | * -> return safe minimum | ||
1734 | */ | ||
1735 | return rs->md.chunk_sectors; | ||
1736 | } | ||
1737 | |||
1738 | static struct target_type raid_target = { | 1720 | static struct target_type raid_target = { |
1739 | .name = "raid", | 1721 | .name = "raid", |
1740 | .version = {1, 7, 0}, | 1722 | .version = {1, 7, 0}, |
@@ -1749,7 +1731,6 @@ static struct target_type raid_target = { | |||
1749 | .presuspend = raid_presuspend, | 1731 | .presuspend = raid_presuspend, |
1750 | .postsuspend = raid_postsuspend, | 1732 | .postsuspend = raid_postsuspend, |
1751 | .resume = raid_resume, | 1733 | .resume = raid_resume, |
1752 | .merge = raid_merge, | ||
1753 | }; | 1734 | }; |
1754 | 1735 | ||
1755 | static int __init dm_raid_init(void) | 1736 | static int __init dm_raid_init(void) |
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index d83696bf403b..e1eabfb2f52d 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c | |||
@@ -490,9 +490,11 @@ static void hold_bio(struct mirror_set *ms, struct bio *bio) | |||
490 | * If device is suspended, complete the bio. | 490 | * If device is suspended, complete the bio. |
491 | */ | 491 | */ |
492 | if (dm_noflush_suspending(ms->ti)) | 492 | if (dm_noflush_suspending(ms->ti)) |
493 | bio_endio(bio, DM_ENDIO_REQUEUE); | 493 | bio->bi_error = DM_ENDIO_REQUEUE; |
494 | else | 494 | else |
495 | bio_endio(bio, -EIO); | 495 | bio->bi_error = -EIO; |
496 | |||
497 | bio_endio(bio); | ||
496 | return; | 498 | return; |
497 | } | 499 | } |
498 | 500 | ||
@@ -515,7 +517,7 @@ static void read_callback(unsigned long error, void *context) | |||
515 | bio_set_m(bio, NULL); | 517 | bio_set_m(bio, NULL); |
516 | 518 | ||
517 | if (likely(!error)) { | 519 | if (likely(!error)) { |
518 | bio_endio(bio, 0); | 520 | bio_endio(bio); |
519 | return; | 521 | return; |
520 | } | 522 | } |
521 | 523 | ||
@@ -531,7 +533,7 @@ static void read_callback(unsigned long error, void *context) | |||
531 | 533 | ||
532 | DMERR_LIMIT("Read failure on mirror device %s. Failing I/O.", | 534 | DMERR_LIMIT("Read failure on mirror device %s. Failing I/O.", |
533 | m->dev->name); | 535 | m->dev->name); |
534 | bio_endio(bio, -EIO); | 536 | bio_io_error(bio); |
535 | } | 537 | } |
536 | 538 | ||
537 | /* Asynchronous read. */ | 539 | /* Asynchronous read. */ |
@@ -580,7 +582,7 @@ static void do_reads(struct mirror_set *ms, struct bio_list *reads) | |||
580 | if (likely(m)) | 582 | if (likely(m)) |
581 | read_async_bio(m, bio); | 583 | read_async_bio(m, bio); |
582 | else | 584 | else |
583 | bio_endio(bio, -EIO); | 585 | bio_io_error(bio); |
584 | } | 586 | } |
585 | } | 587 | } |
586 | 588 | ||
@@ -598,7 +600,7 @@ static void do_reads(struct mirror_set *ms, struct bio_list *reads) | |||
598 | 600 | ||
599 | static void write_callback(unsigned long error, void *context) | 601 | static void write_callback(unsigned long error, void *context) |
600 | { | 602 | { |
601 | unsigned i, ret = 0; | 603 | unsigned i; |
602 | struct bio *bio = (struct bio *) context; | 604 | struct bio *bio = (struct bio *) context; |
603 | struct mirror_set *ms; | 605 | struct mirror_set *ms; |
604 | int should_wake = 0; | 606 | int should_wake = 0; |
@@ -614,7 +616,7 @@ static void write_callback(unsigned long error, void *context) | |||
614 | * regions with the same code. | 616 | * regions with the same code. |
615 | */ | 617 | */ |
616 | if (likely(!error)) { | 618 | if (likely(!error)) { |
617 | bio_endio(bio, ret); | 619 | bio_endio(bio); |
618 | return; | 620 | return; |
619 | } | 621 | } |
620 | 622 | ||
@@ -623,7 +625,8 @@ static void write_callback(unsigned long error, void *context) | |||
623 | * degrade the array. | 625 | * degrade the array. |
624 | */ | 626 | */ |
625 | if (bio->bi_rw & REQ_DISCARD) { | 627 | if (bio->bi_rw & REQ_DISCARD) { |
626 | bio_endio(bio, -EOPNOTSUPP); | 628 | bio->bi_error = -EOPNOTSUPP; |
629 | bio_endio(bio); | ||
627 | return; | 630 | return; |
628 | } | 631 | } |
629 | 632 | ||
@@ -828,13 +831,12 @@ static void do_failures(struct mirror_set *ms, struct bio_list *failures) | |||
828 | * be wrong if the failed leg returned after reboot and | 831 | * be wrong if the failed leg returned after reboot and |
829 | * got replicated back to the good legs.) | 832 | * got replicated back to the good legs.) |
830 | */ | 833 | */ |
831 | |||
832 | if (unlikely(!get_valid_mirror(ms) || (keep_log(ms) && ms->log_failure))) | 834 | if (unlikely(!get_valid_mirror(ms) || (keep_log(ms) && ms->log_failure))) |
833 | bio_endio(bio, -EIO); | 835 | bio_io_error(bio); |
834 | else if (errors_handled(ms) && !keep_log(ms)) | 836 | else if (errors_handled(ms) && !keep_log(ms)) |
835 | hold_bio(ms, bio); | 837 | hold_bio(ms, bio); |
836 | else | 838 | else |
837 | bio_endio(bio, 0); | 839 | bio_endio(bio); |
838 | } | 840 | } |
839 | } | 841 | } |
840 | 842 | ||
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c index 7c82d3ccce87..d10b6876018e 100644 --- a/drivers/md/dm-snap.c +++ b/drivers/md/dm-snap.c | |||
@@ -1490,7 +1490,7 @@ out: | |||
1490 | error_bios(snapshot_bios); | 1490 | error_bios(snapshot_bios); |
1491 | } else { | 1491 | } else { |
1492 | if (full_bio) | 1492 | if (full_bio) |
1493 | bio_endio(full_bio, 0); | 1493 | bio_endio(full_bio); |
1494 | flush_bios(snapshot_bios); | 1494 | flush_bios(snapshot_bios); |
1495 | } | 1495 | } |
1496 | 1496 | ||
@@ -1580,11 +1580,11 @@ static void start_copy(struct dm_snap_pending_exception *pe) | |||
1580 | dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe); | 1580 | dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe); |
1581 | } | 1581 | } |
1582 | 1582 | ||
1583 | static void full_bio_end_io(struct bio *bio, int error) | 1583 | static void full_bio_end_io(struct bio *bio) |
1584 | { | 1584 | { |
1585 | void *callback_data = bio->bi_private; | 1585 | void *callback_data = bio->bi_private; |
1586 | 1586 | ||
1587 | dm_kcopyd_do_callback(callback_data, 0, error ? 1 : 0); | 1587 | dm_kcopyd_do_callback(callback_data, 0, bio->bi_error ? 1 : 0); |
1588 | } | 1588 | } |
1589 | 1589 | ||
1590 | static void start_full_bio(struct dm_snap_pending_exception *pe, | 1590 | static void start_full_bio(struct dm_snap_pending_exception *pe, |
@@ -2330,20 +2330,6 @@ static void origin_status(struct dm_target *ti, status_type_t type, | |||
2330 | } | 2330 | } |
2331 | } | 2331 | } |
2332 | 2332 | ||
2333 | static int origin_merge(struct dm_target *ti, struct bvec_merge_data *bvm, | ||
2334 | struct bio_vec *biovec, int max_size) | ||
2335 | { | ||
2336 | struct dm_origin *o = ti->private; | ||
2337 | struct request_queue *q = bdev_get_queue(o->dev->bdev); | ||
2338 | |||
2339 | if (!q->merge_bvec_fn) | ||
2340 | return max_size; | ||
2341 | |||
2342 | bvm->bi_bdev = o->dev->bdev; | ||
2343 | |||
2344 | return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); | ||
2345 | } | ||
2346 | |||
2347 | static int origin_iterate_devices(struct dm_target *ti, | 2333 | static int origin_iterate_devices(struct dm_target *ti, |
2348 | iterate_devices_callout_fn fn, void *data) | 2334 | iterate_devices_callout_fn fn, void *data) |
2349 | { | 2335 | { |
@@ -2362,7 +2348,6 @@ static struct target_type origin_target = { | |||
2362 | .resume = origin_resume, | 2348 | .resume = origin_resume, |
2363 | .postsuspend = origin_postsuspend, | 2349 | .postsuspend = origin_postsuspend, |
2364 | .status = origin_status, | 2350 | .status = origin_status, |
2365 | .merge = origin_merge, | ||
2366 | .iterate_devices = origin_iterate_devices, | 2351 | .iterate_devices = origin_iterate_devices, |
2367 | }; | 2352 | }; |
2368 | 2353 | ||
diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c index a672a1502c14..484029db8cba 100644 --- a/drivers/md/dm-stripe.c +++ b/drivers/md/dm-stripe.c | |||
@@ -273,7 +273,7 @@ static int stripe_map_range(struct stripe_c *sc, struct bio *bio, | |||
273 | return DM_MAPIO_REMAPPED; | 273 | return DM_MAPIO_REMAPPED; |
274 | } else { | 274 | } else { |
275 | /* The range doesn't map to the target stripe */ | 275 | /* The range doesn't map to the target stripe */ |
276 | bio_endio(bio, 0); | 276 | bio_endio(bio); |
277 | return DM_MAPIO_SUBMITTED; | 277 | return DM_MAPIO_SUBMITTED; |
278 | } | 278 | } |
279 | } | 279 | } |
@@ -412,26 +412,6 @@ static void stripe_io_hints(struct dm_target *ti, | |||
412 | blk_limits_io_opt(limits, chunk_size * sc->stripes); | 412 | blk_limits_io_opt(limits, chunk_size * sc->stripes); |
413 | } | 413 | } |
414 | 414 | ||
415 | static int stripe_merge(struct dm_target *ti, struct bvec_merge_data *bvm, | ||
416 | struct bio_vec *biovec, int max_size) | ||
417 | { | ||
418 | struct stripe_c *sc = ti->private; | ||
419 | sector_t bvm_sector = bvm->bi_sector; | ||
420 | uint32_t stripe; | ||
421 | struct request_queue *q; | ||
422 | |||
423 | stripe_map_sector(sc, bvm_sector, &stripe, &bvm_sector); | ||
424 | |||
425 | q = bdev_get_queue(sc->stripe[stripe].dev->bdev); | ||
426 | if (!q->merge_bvec_fn) | ||
427 | return max_size; | ||
428 | |||
429 | bvm->bi_bdev = sc->stripe[stripe].dev->bdev; | ||
430 | bvm->bi_sector = sc->stripe[stripe].physical_start + bvm_sector; | ||
431 | |||
432 | return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); | ||
433 | } | ||
434 | |||
435 | static struct target_type stripe_target = { | 415 | static struct target_type stripe_target = { |
436 | .name = "striped", | 416 | .name = "striped", |
437 | .version = {1, 5, 1}, | 417 | .version = {1, 5, 1}, |
@@ -443,7 +423,6 @@ static struct target_type stripe_target = { | |||
443 | .status = stripe_status, | 423 | .status = stripe_status, |
444 | .iterate_devices = stripe_iterate_devices, | 424 | .iterate_devices = stripe_iterate_devices, |
445 | .io_hints = stripe_io_hints, | 425 | .io_hints = stripe_io_hints, |
446 | .merge = stripe_merge, | ||
447 | }; | 426 | }; |
448 | 427 | ||
449 | int __init dm_stripe_init(void) | 428 | int __init dm_stripe_init(void) |
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 16ba55ad7089..e76ed003769e 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c | |||
@@ -440,14 +440,6 @@ static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev, | |||
440 | q->limits.alignment_offset, | 440 | q->limits.alignment_offset, |
441 | (unsigned long long) start << SECTOR_SHIFT); | 441 | (unsigned long long) start << SECTOR_SHIFT); |
442 | 442 | ||
443 | /* | ||
444 | * Check if merge fn is supported. | ||
445 | * If not we'll force DM to use PAGE_SIZE or | ||
446 | * smaller I/O, just to be safe. | ||
447 | */ | ||
448 | if (dm_queue_merge_is_compulsory(q) && !ti->type->merge) | ||
449 | blk_limits_max_hw_sectors(limits, | ||
450 | (unsigned int) (PAGE_SIZE >> 9)); | ||
451 | return 0; | 443 | return 0; |
452 | } | 444 | } |
453 | 445 | ||
@@ -1388,14 +1380,6 @@ static int queue_supports_sg_merge(struct dm_target *ti, struct dm_dev *dev, | |||
1388 | return q && !test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags); | 1380 | return q && !test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags); |
1389 | } | 1381 | } |
1390 | 1382 | ||
1391 | static int queue_supports_sg_gaps(struct dm_target *ti, struct dm_dev *dev, | ||
1392 | sector_t start, sector_t len, void *data) | ||
1393 | { | ||
1394 | struct request_queue *q = bdev_get_queue(dev->bdev); | ||
1395 | |||
1396 | return q && !test_bit(QUEUE_FLAG_SG_GAPS, &q->queue_flags); | ||
1397 | } | ||
1398 | |||
1399 | static bool dm_table_all_devices_attribute(struct dm_table *t, | 1383 | static bool dm_table_all_devices_attribute(struct dm_table *t, |
1400 | iterate_devices_callout_fn func) | 1384 | iterate_devices_callout_fn func) |
1401 | { | 1385 | { |
@@ -1516,11 +1500,6 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q, | |||
1516 | else | 1500 | else |
1517 | queue_flag_set_unlocked(QUEUE_FLAG_NO_SG_MERGE, q); | 1501 | queue_flag_set_unlocked(QUEUE_FLAG_NO_SG_MERGE, q); |
1518 | 1502 | ||
1519 | if (dm_table_all_devices_attribute(t, queue_supports_sg_gaps)) | ||
1520 | queue_flag_clear_unlocked(QUEUE_FLAG_SG_GAPS, q); | ||
1521 | else | ||
1522 | queue_flag_set_unlocked(QUEUE_FLAG_SG_GAPS, q); | ||
1523 | |||
1524 | dm_table_set_integrity(t); | 1503 | dm_table_set_integrity(t); |
1525 | 1504 | ||
1526 | /* | 1505 | /* |
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index d2bbe8cc1e97..271a66249363 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c | |||
@@ -615,8 +615,10 @@ static void error_bio_list(struct bio_list *bios, int error) | |||
615 | { | 615 | { |
616 | struct bio *bio; | 616 | struct bio *bio; |
617 | 617 | ||
618 | while ((bio = bio_list_pop(bios))) | 618 | while ((bio = bio_list_pop(bios))) { |
619 | bio_endio(bio, error); | 619 | bio->bi_error = error; |
620 | bio_endio(bio); | ||
621 | } | ||
620 | } | 622 | } |
621 | 623 | ||
622 | static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master, int error) | 624 | static void error_thin_bio_list(struct thin_c *tc, struct bio_list *master, int error) |
@@ -870,14 +872,14 @@ static void copy_complete(int read_err, unsigned long write_err, void *context) | |||
870 | complete_mapping_preparation(m); | 872 | complete_mapping_preparation(m); |
871 | } | 873 | } |
872 | 874 | ||
873 | static void overwrite_endio(struct bio *bio, int err) | 875 | static void overwrite_endio(struct bio *bio) |
874 | { | 876 | { |
875 | struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); | 877 | struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook)); |
876 | struct dm_thin_new_mapping *m = h->overwrite_mapping; | 878 | struct dm_thin_new_mapping *m = h->overwrite_mapping; |
877 | 879 | ||
878 | bio->bi_end_io = m->saved_bi_end_io; | 880 | bio->bi_end_io = m->saved_bi_end_io; |
879 | 881 | ||
880 | m->err = err; | 882 | m->err = bio->bi_error; |
881 | complete_mapping_preparation(m); | 883 | complete_mapping_preparation(m); |
882 | } | 884 | } |
883 | 885 | ||
@@ -1002,7 +1004,7 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m) | |||
1002 | */ | 1004 | */ |
1003 | if (bio) { | 1005 | if (bio) { |
1004 | inc_remap_and_issue_cell(tc, m->cell, m->data_block); | 1006 | inc_remap_and_issue_cell(tc, m->cell, m->data_block); |
1005 | bio_endio(bio, 0); | 1007 | bio_endio(bio); |
1006 | } else { | 1008 | } else { |
1007 | inc_all_io_entry(tc->pool, m->cell->holder); | 1009 | inc_all_io_entry(tc->pool, m->cell->holder); |
1008 | remap_and_issue(tc, m->cell->holder, m->data_block); | 1010 | remap_and_issue(tc, m->cell->holder, m->data_block); |
@@ -1032,7 +1034,7 @@ static void process_prepared_discard_fail(struct dm_thin_new_mapping *m) | |||
1032 | 1034 | ||
1033 | static void process_prepared_discard_success(struct dm_thin_new_mapping *m) | 1035 | static void process_prepared_discard_success(struct dm_thin_new_mapping *m) |
1034 | { | 1036 | { |
1035 | bio_endio(m->bio, 0); | 1037 | bio_endio(m->bio); |
1036 | free_discard_mapping(m); | 1038 | free_discard_mapping(m); |
1037 | } | 1039 | } |
1038 | 1040 | ||
@@ -1046,7 +1048,7 @@ static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping *m) | |||
1046 | metadata_operation_failed(tc->pool, "dm_thin_remove_range", r); | 1048 | metadata_operation_failed(tc->pool, "dm_thin_remove_range", r); |
1047 | bio_io_error(m->bio); | 1049 | bio_io_error(m->bio); |
1048 | } else | 1050 | } else |
1049 | bio_endio(m->bio, 0); | 1051 | bio_endio(m->bio); |
1050 | 1052 | ||
1051 | cell_defer_no_holder(tc, m->cell); | 1053 | cell_defer_no_holder(tc, m->cell); |
1052 | mempool_free(m, tc->pool->mapping_pool); | 1054 | mempool_free(m, tc->pool->mapping_pool); |
@@ -1117,7 +1119,8 @@ static void process_prepared_discard_passdown(struct dm_thin_new_mapping *m) | |||
1117 | * Even if r is set, there could be sub discards in flight that we | 1119 | * Even if r is set, there could be sub discards in flight that we |
1118 | * need to wait for. | 1120 | * need to wait for. |
1119 | */ | 1121 | */ |
1120 | bio_endio(m->bio, r); | 1122 | m->bio->bi_error = r; |
1123 | bio_endio(m->bio); | ||
1121 | cell_defer_no_holder(tc, m->cell); | 1124 | cell_defer_no_holder(tc, m->cell); |
1122 | mempool_free(m, pool->mapping_pool); | 1125 | mempool_free(m, pool->mapping_pool); |
1123 | } | 1126 | } |
@@ -1493,9 +1496,10 @@ static void handle_unserviceable_bio(struct pool *pool, struct bio *bio) | |||
1493 | { | 1496 | { |
1494 | int error = should_error_unserviceable_bio(pool); | 1497 | int error = should_error_unserviceable_bio(pool); |
1495 | 1498 | ||
1496 | if (error) | 1499 | if (error) { |
1497 | bio_endio(bio, error); | 1500 | bio->bi_error = error; |
1498 | else | 1501 | bio_endio(bio); |
1502 | } else | ||
1499 | retry_on_resume(bio); | 1503 | retry_on_resume(bio); |
1500 | } | 1504 | } |
1501 | 1505 | ||
@@ -1631,7 +1635,7 @@ static void process_discard_cell_passdown(struct thin_c *tc, struct dm_bio_priso | |||
1631 | * will prevent completion until the sub range discards have | 1635 | * will prevent completion until the sub range discards have |
1632 | * completed. | 1636 | * completed. |
1633 | */ | 1637 | */ |
1634 | bio_endio(bio, 0); | 1638 | bio_endio(bio); |
1635 | } | 1639 | } |
1636 | 1640 | ||
1637 | static void process_discard_bio(struct thin_c *tc, struct bio *bio) | 1641 | static void process_discard_bio(struct thin_c *tc, struct bio *bio) |
@@ -1645,7 +1649,7 @@ static void process_discard_bio(struct thin_c *tc, struct bio *bio) | |||
1645 | /* | 1649 | /* |
1646 | * The discard covers less than a block. | 1650 | * The discard covers less than a block. |
1647 | */ | 1651 | */ |
1648 | bio_endio(bio, 0); | 1652 | bio_endio(bio); |
1649 | return; | 1653 | return; |
1650 | } | 1654 | } |
1651 | 1655 | ||
@@ -1790,7 +1794,7 @@ static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block | |||
1790 | if (bio_data_dir(bio) == READ) { | 1794 | if (bio_data_dir(bio) == READ) { |
1791 | zero_fill_bio(bio); | 1795 | zero_fill_bio(bio); |
1792 | cell_defer_no_holder(tc, cell); | 1796 | cell_defer_no_holder(tc, cell); |
1793 | bio_endio(bio, 0); | 1797 | bio_endio(bio); |
1794 | return; | 1798 | return; |
1795 | } | 1799 | } |
1796 | 1800 | ||
@@ -1855,7 +1859,7 @@ static void process_cell(struct thin_c *tc, struct dm_bio_prison_cell *cell) | |||
1855 | 1859 | ||
1856 | } else { | 1860 | } else { |
1857 | zero_fill_bio(bio); | 1861 | zero_fill_bio(bio); |
1858 | bio_endio(bio, 0); | 1862 | bio_endio(bio); |
1859 | } | 1863 | } |
1860 | } else | 1864 | } else |
1861 | provision_block(tc, bio, block, cell); | 1865 | provision_block(tc, bio, block, cell); |
@@ -1926,7 +1930,7 @@ static void __process_bio_read_only(struct thin_c *tc, struct bio *bio, | |||
1926 | } | 1930 | } |
1927 | 1931 | ||
1928 | zero_fill_bio(bio); | 1932 | zero_fill_bio(bio); |
1929 | bio_endio(bio, 0); | 1933 | bio_endio(bio); |
1930 | break; | 1934 | break; |
1931 | 1935 | ||
1932 | default: | 1936 | default: |
@@ -1951,7 +1955,7 @@ static void process_cell_read_only(struct thin_c *tc, struct dm_bio_prison_cell | |||
1951 | 1955 | ||
1952 | static void process_bio_success(struct thin_c *tc, struct bio *bio) | 1956 | static void process_bio_success(struct thin_c *tc, struct bio *bio) |
1953 | { | 1957 | { |
1954 | bio_endio(bio, 0); | 1958 | bio_endio(bio); |
1955 | } | 1959 | } |
1956 | 1960 | ||
1957 | static void process_bio_fail(struct thin_c *tc, struct bio *bio) | 1961 | static void process_bio_fail(struct thin_c *tc, struct bio *bio) |
@@ -2600,7 +2604,8 @@ static int thin_bio_map(struct dm_target *ti, struct bio *bio) | |||
2600 | thin_hook_bio(tc, bio); | 2604 | thin_hook_bio(tc, bio); |
2601 | 2605 | ||
2602 | if (tc->requeue_mode) { | 2606 | if (tc->requeue_mode) { |
2603 | bio_endio(bio, DM_ENDIO_REQUEUE); | 2607 | bio->bi_error = DM_ENDIO_REQUEUE; |
2608 | bio_endio(bio); | ||
2604 | return DM_MAPIO_SUBMITTED; | 2609 | return DM_MAPIO_SUBMITTED; |
2605 | } | 2610 | } |
2606 | 2611 | ||
@@ -3875,20 +3880,6 @@ static int pool_iterate_devices(struct dm_target *ti, | |||
3875 | return fn(ti, pt->data_dev, 0, ti->len, data); | 3880 | return fn(ti, pt->data_dev, 0, ti->len, data); |
3876 | } | 3881 | } |
3877 | 3882 | ||
3878 | static int pool_merge(struct dm_target *ti, struct bvec_merge_data *bvm, | ||
3879 | struct bio_vec *biovec, int max_size) | ||
3880 | { | ||
3881 | struct pool_c *pt = ti->private; | ||
3882 | struct request_queue *q = bdev_get_queue(pt->data_dev->bdev); | ||
3883 | |||
3884 | if (!q->merge_bvec_fn) | ||
3885 | return max_size; | ||
3886 | |||
3887 | bvm->bi_bdev = pt->data_dev->bdev; | ||
3888 | |||
3889 | return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); | ||
3890 | } | ||
3891 | |||
3892 | static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits) | 3883 | static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits) |
3893 | { | 3884 | { |
3894 | struct pool_c *pt = ti->private; | 3885 | struct pool_c *pt = ti->private; |
@@ -3965,7 +3956,6 @@ static struct target_type pool_target = { | |||
3965 | .resume = pool_resume, | 3956 | .resume = pool_resume, |
3966 | .message = pool_message, | 3957 | .message = pool_message, |
3967 | .status = pool_status, | 3958 | .status = pool_status, |
3968 | .merge = pool_merge, | ||
3969 | .iterate_devices = pool_iterate_devices, | 3959 | .iterate_devices = pool_iterate_devices, |
3970 | .io_hints = pool_io_hints, | 3960 | .io_hints = pool_io_hints, |
3971 | }; | 3961 | }; |
@@ -4292,21 +4282,6 @@ err: | |||
4292 | DMEMIT("Error"); | 4282 | DMEMIT("Error"); |
4293 | } | 4283 | } |
4294 | 4284 | ||
4295 | static int thin_merge(struct dm_target *ti, struct bvec_merge_data *bvm, | ||
4296 | struct bio_vec *biovec, int max_size) | ||
4297 | { | ||
4298 | struct thin_c *tc = ti->private; | ||
4299 | struct request_queue *q = bdev_get_queue(tc->pool_dev->bdev); | ||
4300 | |||
4301 | if (!q->merge_bvec_fn) | ||
4302 | return max_size; | ||
4303 | |||
4304 | bvm->bi_bdev = tc->pool_dev->bdev; | ||
4305 | bvm->bi_sector = dm_target_offset(ti, bvm->bi_sector); | ||
4306 | |||
4307 | return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); | ||
4308 | } | ||
4309 | |||
4310 | static int thin_iterate_devices(struct dm_target *ti, | 4285 | static int thin_iterate_devices(struct dm_target *ti, |
4311 | iterate_devices_callout_fn fn, void *data) | 4286 | iterate_devices_callout_fn fn, void *data) |
4312 | { | 4287 | { |
@@ -4350,7 +4325,6 @@ static struct target_type thin_target = { | |||
4350 | .presuspend = thin_presuspend, | 4325 | .presuspend = thin_presuspend, |
4351 | .postsuspend = thin_postsuspend, | 4326 | .postsuspend = thin_postsuspend, |
4352 | .status = thin_status, | 4327 | .status = thin_status, |
4353 | .merge = thin_merge, | ||
4354 | .iterate_devices = thin_iterate_devices, | 4328 | .iterate_devices = thin_iterate_devices, |
4355 | .io_hints = thin_io_hints, | 4329 | .io_hints = thin_io_hints, |
4356 | }; | 4330 | }; |
diff --git a/drivers/md/dm-verity.c b/drivers/md/dm-verity.c index bb9c6a00e4b0..c137dcb147b8 100644 --- a/drivers/md/dm-verity.c +++ b/drivers/md/dm-verity.c | |||
@@ -458,8 +458,9 @@ static void verity_finish_io(struct dm_verity_io *io, int error) | |||
458 | 458 | ||
459 | bio->bi_end_io = io->orig_bi_end_io; | 459 | bio->bi_end_io = io->orig_bi_end_io; |
460 | bio->bi_private = io->orig_bi_private; | 460 | bio->bi_private = io->orig_bi_private; |
461 | bio->bi_error = error; | ||
461 | 462 | ||
462 | bio_endio(bio, error); | 463 | bio_endio(bio); |
463 | } | 464 | } |
464 | 465 | ||
465 | static void verity_work(struct work_struct *w) | 466 | static void verity_work(struct work_struct *w) |
@@ -469,12 +470,12 @@ static void verity_work(struct work_struct *w) | |||
469 | verity_finish_io(io, verity_verify_io(io)); | 470 | verity_finish_io(io, verity_verify_io(io)); |
470 | } | 471 | } |
471 | 472 | ||
472 | static void verity_end_io(struct bio *bio, int error) | 473 | static void verity_end_io(struct bio *bio) |
473 | { | 474 | { |
474 | struct dm_verity_io *io = bio->bi_private; | 475 | struct dm_verity_io *io = bio->bi_private; |
475 | 476 | ||
476 | if (error) { | 477 | if (bio->bi_error) { |
477 | verity_finish_io(io, error); | 478 | verity_finish_io(io, bio->bi_error); |
478 | return; | 479 | return; |
479 | } | 480 | } |
480 | 481 | ||
@@ -648,21 +649,6 @@ static int verity_ioctl(struct dm_target *ti, unsigned cmd, | |||
648 | cmd, arg); | 649 | cmd, arg); |
649 | } | 650 | } |
650 | 651 | ||
651 | static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm, | ||
652 | struct bio_vec *biovec, int max_size) | ||
653 | { | ||
654 | struct dm_verity *v = ti->private; | ||
655 | struct request_queue *q = bdev_get_queue(v->data_dev->bdev); | ||
656 | |||
657 | if (!q->merge_bvec_fn) | ||
658 | return max_size; | ||
659 | |||
660 | bvm->bi_bdev = v->data_dev->bdev; | ||
661 | bvm->bi_sector = verity_map_sector(v, bvm->bi_sector); | ||
662 | |||
663 | return min(max_size, q->merge_bvec_fn(q, bvm, biovec)); | ||
664 | } | ||
665 | |||
666 | static int verity_iterate_devices(struct dm_target *ti, | 652 | static int verity_iterate_devices(struct dm_target *ti, |
667 | iterate_devices_callout_fn fn, void *data) | 653 | iterate_devices_callout_fn fn, void *data) |
668 | { | 654 | { |
@@ -995,7 +981,6 @@ static struct target_type verity_target = { | |||
995 | .map = verity_map, | 981 | .map = verity_map, |
996 | .status = verity_status, | 982 | .status = verity_status, |
997 | .ioctl = verity_ioctl, | 983 | .ioctl = verity_ioctl, |
998 | .merge = verity_merge, | ||
999 | .iterate_devices = verity_iterate_devices, | 984 | .iterate_devices = verity_iterate_devices, |
1000 | .io_hints = verity_io_hints, | 985 | .io_hints = verity_io_hints, |
1001 | }; | 986 | }; |
diff --git a/drivers/md/dm-zero.c b/drivers/md/dm-zero.c index b9a64bbce304..766bc93006e6 100644 --- a/drivers/md/dm-zero.c +++ b/drivers/md/dm-zero.c | |||
@@ -47,7 +47,7 @@ static int zero_map(struct dm_target *ti, struct bio *bio) | |||
47 | break; | 47 | break; |
48 | } | 48 | } |
49 | 49 | ||
50 | bio_endio(bio, 0); | 50 | bio_endio(bio); |
51 | 51 | ||
52 | /* accepted bio, don't make new request */ | 52 | /* accepted bio, don't make new request */ |
53 | return DM_MAPIO_SUBMITTED; | 53 | return DM_MAPIO_SUBMITTED; |
diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 0d7ab20c58df..6ffc01bb85f2 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c | |||
@@ -124,9 +124,8 @@ EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo); | |||
124 | #define DMF_FREEING 3 | 124 | #define DMF_FREEING 3 |
125 | #define DMF_DELETING 4 | 125 | #define DMF_DELETING 4 |
126 | #define DMF_NOFLUSH_SUSPENDING 5 | 126 | #define DMF_NOFLUSH_SUSPENDING 5 |
127 | #define DMF_MERGE_IS_OPTIONAL 6 | 127 | #define DMF_DEFERRED_REMOVE 6 |
128 | #define DMF_DEFERRED_REMOVE 7 | 128 | #define DMF_SUSPENDED_INTERNALLY 7 |
129 | #define DMF_SUSPENDED_INTERNALLY 8 | ||
130 | 129 | ||
131 | /* | 130 | /* |
132 | * A dummy definition to make RCU happy. | 131 | * A dummy definition to make RCU happy. |
@@ -944,7 +943,8 @@ static void dec_pending(struct dm_io *io, int error) | |||
944 | } else { | 943 | } else { |
945 | /* done with normal IO or empty flush */ | 944 | /* done with normal IO or empty flush */ |
946 | trace_block_bio_complete(md->queue, bio, io_error); | 945 | trace_block_bio_complete(md->queue, bio, io_error); |
947 | bio_endio(bio, io_error); | 946 | bio->bi_error = io_error; |
947 | bio_endio(bio); | ||
948 | } | 948 | } |
949 | } | 949 | } |
950 | } | 950 | } |
@@ -957,17 +957,15 @@ static void disable_write_same(struct mapped_device *md) | |||
957 | limits->max_write_same_sectors = 0; | 957 | limits->max_write_same_sectors = 0; |
958 | } | 958 | } |
959 | 959 | ||
960 | static void clone_endio(struct bio *bio, int error) | 960 | static void clone_endio(struct bio *bio) |
961 | { | 961 | { |
962 | int error = bio->bi_error; | ||
962 | int r = error; | 963 | int r = error; |
963 | struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); | 964 | struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); |
964 | struct dm_io *io = tio->io; | 965 | struct dm_io *io = tio->io; |
965 | struct mapped_device *md = tio->io->md; | 966 | struct mapped_device *md = tio->io->md; |
966 | dm_endio_fn endio = tio->ti->type->end_io; | 967 | dm_endio_fn endio = tio->ti->type->end_io; |
967 | 968 | ||
968 | if (!bio_flagged(bio, BIO_UPTODATE) && !error) | ||
969 | error = -EIO; | ||
970 | |||
971 | if (endio) { | 969 | if (endio) { |
972 | r = endio(tio->ti, bio, error); | 970 | r = endio(tio->ti, bio, error); |
973 | if (r < 0 || r == DM_ENDIO_REQUEUE) | 971 | if (r < 0 || r == DM_ENDIO_REQUEUE) |
@@ -996,7 +994,7 @@ static void clone_endio(struct bio *bio, int error) | |||
996 | /* | 994 | /* |
997 | * Partial completion handling for request-based dm | 995 | * Partial completion handling for request-based dm |
998 | */ | 996 | */ |
999 | static void end_clone_bio(struct bio *clone, int error) | 997 | static void end_clone_bio(struct bio *clone) |
1000 | { | 998 | { |
1001 | struct dm_rq_clone_bio_info *info = | 999 | struct dm_rq_clone_bio_info *info = |
1002 | container_of(clone, struct dm_rq_clone_bio_info, clone); | 1000 | container_of(clone, struct dm_rq_clone_bio_info, clone); |
@@ -1013,13 +1011,13 @@ static void end_clone_bio(struct bio *clone, int error) | |||
1013 | * the remainder. | 1011 | * the remainder. |
1014 | */ | 1012 | */ |
1015 | return; | 1013 | return; |
1016 | else if (error) { | 1014 | else if (bio->bi_error) { |
1017 | /* | 1015 | /* |
1018 | * Don't notice the error to the upper layer yet. | 1016 | * Don't notice the error to the upper layer yet. |
1019 | * The error handling decision is made by the target driver, | 1017 | * The error handling decision is made by the target driver, |
1020 | * when the request is completed. | 1018 | * when the request is completed. |
1021 | */ | 1019 | */ |
1022 | tio->error = error; | 1020 | tio->error = bio->bi_error; |
1023 | return; | 1021 | return; |
1024 | } | 1022 | } |
1025 | 1023 | ||
@@ -1722,60 +1720,6 @@ static void __split_and_process_bio(struct mapped_device *md, | |||
1722 | * CRUD END | 1720 | * CRUD END |
1723 | *---------------------------------------------------------------*/ | 1721 | *---------------------------------------------------------------*/ |
1724 | 1722 | ||
1725 | static int dm_merge_bvec(struct request_queue *q, | ||
1726 | struct bvec_merge_data *bvm, | ||
1727 | struct bio_vec *biovec) | ||
1728 | { | ||
1729 | struct mapped_device *md = q->queuedata; | ||
1730 | struct dm_table *map = dm_get_live_table_fast(md); | ||
1731 | struct dm_target *ti; | ||
1732 | sector_t max_sectors; | ||
1733 | int max_size = 0; | ||
1734 | |||
1735 | if (unlikely(!map)) | ||
1736 | goto out; | ||
1737 | |||
1738 | ti = dm_table_find_target(map, bvm->bi_sector); | ||
1739 | if (!dm_target_is_valid(ti)) | ||
1740 | goto out; | ||
1741 | |||
1742 | /* | ||
1743 | * Find maximum amount of I/O that won't need splitting | ||
1744 | */ | ||
1745 | max_sectors = min(max_io_len(bvm->bi_sector, ti), | ||
1746 | (sector_t) BIO_MAX_SECTORS); | ||
1747 | max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size; | ||
1748 | if (max_size < 0) | ||
1749 | max_size = 0; | ||
1750 | |||
1751 | /* | ||
1752 | * merge_bvec_fn() returns number of bytes | ||
1753 | * it can accept at this offset | ||
1754 | * max is precomputed maximal io size | ||
1755 | */ | ||
1756 | if (max_size && ti->type->merge) | ||
1757 | max_size = ti->type->merge(ti, bvm, biovec, max_size); | ||
1758 | /* | ||
1759 | * If the target doesn't support merge method and some of the devices | ||
1760 | * provided their merge_bvec method (we know this by looking at | ||
1761 | * queue_max_hw_sectors), then we can't allow bios with multiple vector | ||
1762 | * entries. So always set max_size to 0, and the code below allows | ||
1763 | * just one page. | ||
1764 | */ | ||
1765 | else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9) | ||
1766 | max_size = 0; | ||
1767 | |||
1768 | out: | ||
1769 | dm_put_live_table_fast(md); | ||
1770 | /* | ||
1771 | * Always allow an entire first page | ||
1772 | */ | ||
1773 | if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT)) | ||
1774 | max_size = biovec->bv_len; | ||
1775 | |||
1776 | return max_size; | ||
1777 | } | ||
1778 | |||
1779 | /* | 1723 | /* |
1780 | * The request function that just remaps the bio built up by | 1724 | * The request function that just remaps the bio built up by |
1781 | * dm_merge_bvec. | 1725 | * dm_merge_bvec. |
@@ -1789,6 +1733,8 @@ static void dm_make_request(struct request_queue *q, struct bio *bio) | |||
1789 | 1733 | ||
1790 | map = dm_get_live_table(md, &srcu_idx); | 1734 | map = dm_get_live_table(md, &srcu_idx); |
1791 | 1735 | ||
1736 | blk_queue_split(q, &bio, q->bio_split); | ||
1737 | |||
1792 | generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0); | 1738 | generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0); |
1793 | 1739 | ||
1794 | /* if we're suspended, we have to queue this io for later */ | 1740 | /* if we're suspended, we have to queue this io for later */ |
@@ -2496,59 +2442,6 @@ static void __set_size(struct mapped_device *md, sector_t size) | |||
2496 | } | 2442 | } |
2497 | 2443 | ||
2498 | /* | 2444 | /* |
2499 | * Return 1 if the queue has a compulsory merge_bvec_fn function. | ||
2500 | * | ||
2501 | * If this function returns 0, then the device is either a non-dm | ||
2502 | * device without a merge_bvec_fn, or it is a dm device that is | ||
2503 | * able to split any bios it receives that are too big. | ||
2504 | */ | ||
2505 | int dm_queue_merge_is_compulsory(struct request_queue *q) | ||
2506 | { | ||
2507 | struct mapped_device *dev_md; | ||
2508 | |||
2509 | if (!q->merge_bvec_fn) | ||
2510 | return 0; | ||
2511 | |||
2512 | if (q->make_request_fn == dm_make_request) { | ||
2513 | dev_md = q->queuedata; | ||
2514 | if (test_bit(DMF_MERGE_IS_OPTIONAL, &dev_md->flags)) | ||
2515 | return 0; | ||
2516 | } | ||
2517 | |||
2518 | return 1; | ||
2519 | } | ||
2520 | |||
2521 | static int dm_device_merge_is_compulsory(struct dm_target *ti, | ||
2522 | struct dm_dev *dev, sector_t start, | ||
2523 | sector_t len, void *data) | ||
2524 | { | ||
2525 | struct block_device *bdev = dev->bdev; | ||
2526 | struct request_queue *q = bdev_get_queue(bdev); | ||
2527 | |||
2528 | return dm_queue_merge_is_compulsory(q); | ||
2529 | } | ||
2530 | |||
2531 | /* | ||
2532 | * Return 1 if it is acceptable to ignore merge_bvec_fn based | ||
2533 | * on the properties of the underlying devices. | ||
2534 | */ | ||
2535 | static int dm_table_merge_is_optional(struct dm_table *table) | ||
2536 | { | ||
2537 | unsigned i = 0; | ||
2538 | struct dm_target *ti; | ||
2539 | |||
2540 | while (i < dm_table_get_num_targets(table)) { | ||
2541 | ti = dm_table_get_target(table, i++); | ||
2542 | |||
2543 | if (ti->type->iterate_devices && | ||
2544 | ti->type->iterate_devices(ti, dm_device_merge_is_compulsory, NULL)) | ||
2545 | return 0; | ||
2546 | } | ||
2547 | |||
2548 | return 1; | ||
2549 | } | ||
2550 | |||
2551 | /* | ||
2552 | * Returns old map, which caller must destroy. | 2445 | * Returns old map, which caller must destroy. |
2553 | */ | 2446 | */ |
2554 | static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, | 2447 | static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, |
@@ -2557,7 +2450,6 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, | |||
2557 | struct dm_table *old_map; | 2450 | struct dm_table *old_map; |
2558 | struct request_queue *q = md->queue; | 2451 | struct request_queue *q = md->queue; |
2559 | sector_t size; | 2452 | sector_t size; |
2560 | int merge_is_optional; | ||
2561 | 2453 | ||
2562 | size = dm_table_get_size(t); | 2454 | size = dm_table_get_size(t); |
2563 | 2455 | ||
@@ -2583,17 +2475,11 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, | |||
2583 | 2475 | ||
2584 | __bind_mempools(md, t); | 2476 | __bind_mempools(md, t); |
2585 | 2477 | ||
2586 | merge_is_optional = dm_table_merge_is_optional(t); | ||
2587 | |||
2588 | old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); | 2478 | old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); |
2589 | rcu_assign_pointer(md->map, t); | 2479 | rcu_assign_pointer(md->map, t); |
2590 | md->immutable_target_type = dm_table_get_immutable_target_type(t); | 2480 | md->immutable_target_type = dm_table_get_immutable_target_type(t); |
2591 | 2481 | ||
2592 | dm_table_set_restrictions(t, q, limits); | 2482 | dm_table_set_restrictions(t, q, limits); |
2593 | if (merge_is_optional) | ||
2594 | set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags); | ||
2595 | else | ||
2596 | clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags); | ||
2597 | if (old_map) | 2483 | if (old_map) |
2598 | dm_sync_table(md); | 2484 | dm_sync_table(md); |
2599 | 2485 | ||
@@ -2874,7 +2760,6 @@ int dm_setup_md_queue(struct mapped_device *md) | |||
2874 | case DM_TYPE_BIO_BASED: | 2760 | case DM_TYPE_BIO_BASED: |
2875 | dm_init_old_md_queue(md); | 2761 | dm_init_old_md_queue(md); |
2876 | blk_queue_make_request(md->queue, dm_make_request); | 2762 | blk_queue_make_request(md->queue, dm_make_request); |
2877 | blk_queue_merge_bvec(md->queue, dm_merge_bvec); | ||
2878 | break; | 2763 | break; |
2879 | } | 2764 | } |
2880 | 2765 | ||
diff --git a/drivers/md/dm.h b/drivers/md/dm.h index 4e984993d40a..7edcf97dfa5a 100644 --- a/drivers/md/dm.h +++ b/drivers/md/dm.h | |||
@@ -78,8 +78,6 @@ bool dm_table_mq_request_based(struct dm_table *t); | |||
78 | void dm_table_free_md_mempools(struct dm_table *t); | 78 | void dm_table_free_md_mempools(struct dm_table *t); |
79 | struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t); | 79 | struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t); |
80 | 80 | ||
81 | int dm_queue_merge_is_compulsory(struct request_queue *q); | ||
82 | |||
83 | void dm_lock_md_type(struct mapped_device *md); | 81 | void dm_lock_md_type(struct mapped_device *md); |
84 | void dm_unlock_md_type(struct mapped_device *md); | 82 | void dm_unlock_md_type(struct mapped_device *md); |
85 | void dm_set_md_type(struct mapped_device *md, unsigned type); | 83 | void dm_set_md_type(struct mapped_device *md, unsigned type); |
diff --git a/drivers/md/faulty.c b/drivers/md/faulty.c index 1277eb26b58a..4a8e15058e8b 100644 --- a/drivers/md/faulty.c +++ b/drivers/md/faulty.c | |||
@@ -70,7 +70,7 @@ | |||
70 | #include <linux/seq_file.h> | 70 | #include <linux/seq_file.h> |
71 | 71 | ||
72 | 72 | ||
73 | static void faulty_fail(struct bio *bio, int error) | 73 | static void faulty_fail(struct bio *bio) |
74 | { | 74 | { |
75 | struct bio *b = bio->bi_private; | 75 | struct bio *b = bio->bi_private; |
76 | 76 | ||
@@ -181,7 +181,7 @@ static void make_request(struct mddev *mddev, struct bio *bio) | |||
181 | /* special case - don't decrement, don't generic_make_request, | 181 | /* special case - don't decrement, don't generic_make_request, |
182 | * just fail immediately | 182 | * just fail immediately |
183 | */ | 183 | */ |
184 | bio_endio(bio, -EIO); | 184 | bio_io_error(bio); |
185 | return; | 185 | return; |
186 | } | 186 | } |
187 | 187 | ||
diff --git a/drivers/md/linear.c b/drivers/md/linear.c index fa7d577f3d12..b7fe7e9fc777 100644 --- a/drivers/md/linear.c +++ b/drivers/md/linear.c | |||
@@ -52,48 +52,6 @@ static inline struct dev_info *which_dev(struct mddev *mddev, sector_t sector) | |||
52 | return conf->disks + lo; | 52 | return conf->disks + lo; |
53 | } | 53 | } |
54 | 54 | ||
55 | /** | ||
56 | * linear_mergeable_bvec -- tell bio layer if two requests can be merged | ||
57 | * @q: request queue | ||
58 | * @bvm: properties of new bio | ||
59 | * @biovec: the request that could be merged to it. | ||
60 | * | ||
61 | * Return amount of bytes we can take at this offset | ||
62 | */ | ||
63 | static int linear_mergeable_bvec(struct mddev *mddev, | ||
64 | struct bvec_merge_data *bvm, | ||
65 | struct bio_vec *biovec) | ||
66 | { | ||
67 | struct dev_info *dev0; | ||
68 | unsigned long maxsectors, bio_sectors = bvm->bi_size >> 9; | ||
69 | sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); | ||
70 | int maxbytes = biovec->bv_len; | ||
71 | struct request_queue *subq; | ||
72 | |||
73 | dev0 = which_dev(mddev, sector); | ||
74 | maxsectors = dev0->end_sector - sector; | ||
75 | subq = bdev_get_queue(dev0->rdev->bdev); | ||
76 | if (subq->merge_bvec_fn) { | ||
77 | bvm->bi_bdev = dev0->rdev->bdev; | ||
78 | bvm->bi_sector -= dev0->end_sector - dev0->rdev->sectors; | ||
79 | maxbytes = min(maxbytes, subq->merge_bvec_fn(subq, bvm, | ||
80 | biovec)); | ||
81 | } | ||
82 | |||
83 | if (maxsectors < bio_sectors) | ||
84 | maxsectors = 0; | ||
85 | else | ||
86 | maxsectors -= bio_sectors; | ||
87 | |||
88 | if (maxsectors <= (PAGE_SIZE >> 9 ) && bio_sectors == 0) | ||
89 | return maxbytes; | ||
90 | |||
91 | if (maxsectors > (maxbytes >> 9)) | ||
92 | return maxbytes; | ||
93 | else | ||
94 | return maxsectors << 9; | ||
95 | } | ||
96 | |||
97 | static int linear_congested(struct mddev *mddev, int bits) | 55 | static int linear_congested(struct mddev *mddev, int bits) |
98 | { | 56 | { |
99 | struct linear_conf *conf; | 57 | struct linear_conf *conf; |
@@ -297,7 +255,7 @@ static void linear_make_request(struct mddev *mddev, struct bio *bio) | |||
297 | if (unlikely((split->bi_rw & REQ_DISCARD) && | 255 | if (unlikely((split->bi_rw & REQ_DISCARD) && |
298 | !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) { | 256 | !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) { |
299 | /* Just ignore it */ | 257 | /* Just ignore it */ |
300 | bio_endio(split, 0); | 258 | bio_endio(split); |
301 | } else | 259 | } else |
302 | generic_make_request(split); | 260 | generic_make_request(split); |
303 | } while (split != bio); | 261 | } while (split != bio); |
@@ -338,7 +296,6 @@ static struct md_personality linear_personality = | |||
338 | .size = linear_size, | 296 | .size = linear_size, |
339 | .quiesce = linear_quiesce, | 297 | .quiesce = linear_quiesce, |
340 | .congested = linear_congested, | 298 | .congested = linear_congested, |
341 | .mergeable_bvec = linear_mergeable_bvec, | ||
342 | }; | 299 | }; |
343 | 300 | ||
344 | static int __init linear_init (void) | 301 | static int __init linear_init (void) |
diff --git a/drivers/md/md.c b/drivers/md/md.c index e25f00f0138a..40332625713b 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c | |||
@@ -257,13 +257,17 @@ static void md_make_request(struct request_queue *q, struct bio *bio) | |||
257 | unsigned int sectors; | 257 | unsigned int sectors; |
258 | int cpu; | 258 | int cpu; |
259 | 259 | ||
260 | blk_queue_split(q, &bio, q->bio_split); | ||
261 | |||
260 | if (mddev == NULL || mddev->pers == NULL | 262 | if (mddev == NULL || mddev->pers == NULL |
261 | || !mddev->ready) { | 263 | || !mddev->ready) { |
262 | bio_io_error(bio); | 264 | bio_io_error(bio); |
263 | return; | 265 | return; |
264 | } | 266 | } |
265 | if (mddev->ro == 1 && unlikely(rw == WRITE)) { | 267 | if (mddev->ro == 1 && unlikely(rw == WRITE)) { |
266 | bio_endio(bio, bio_sectors(bio) == 0 ? 0 : -EROFS); | 268 | if (bio_sectors(bio) != 0) |
269 | bio->bi_error = -EROFS; | ||
270 | bio_endio(bio); | ||
267 | return; | 271 | return; |
268 | } | 272 | } |
269 | smp_rmb(); /* Ensure implications of 'active' are visible */ | 273 | smp_rmb(); /* Ensure implications of 'active' are visible */ |
@@ -350,34 +354,11 @@ static int md_congested(void *data, int bits) | |||
350 | return mddev_congested(mddev, bits); | 354 | return mddev_congested(mddev, bits); |
351 | } | 355 | } |
352 | 356 | ||
353 | static int md_mergeable_bvec(struct request_queue *q, | ||
354 | struct bvec_merge_data *bvm, | ||
355 | struct bio_vec *biovec) | ||
356 | { | ||
357 | struct mddev *mddev = q->queuedata; | ||
358 | int ret; | ||
359 | rcu_read_lock(); | ||
360 | if (mddev->suspended) { | ||
361 | /* Must always allow one vec */ | ||
362 | if (bvm->bi_size == 0) | ||
363 | ret = biovec->bv_len; | ||
364 | else | ||
365 | ret = 0; | ||
366 | } else { | ||
367 | struct md_personality *pers = mddev->pers; | ||
368 | if (pers && pers->mergeable_bvec) | ||
369 | ret = pers->mergeable_bvec(mddev, bvm, biovec); | ||
370 | else | ||
371 | ret = biovec->bv_len; | ||
372 | } | ||
373 | rcu_read_unlock(); | ||
374 | return ret; | ||
375 | } | ||
376 | /* | 357 | /* |
377 | * Generic flush handling for md | 358 | * Generic flush handling for md |
378 | */ | 359 | */ |
379 | 360 | ||
380 | static void md_end_flush(struct bio *bio, int err) | 361 | static void md_end_flush(struct bio *bio) |
381 | { | 362 | { |
382 | struct md_rdev *rdev = bio->bi_private; | 363 | struct md_rdev *rdev = bio->bi_private; |
383 | struct mddev *mddev = rdev->mddev; | 364 | struct mddev *mddev = rdev->mddev; |
@@ -433,7 +414,7 @@ static void md_submit_flush_data(struct work_struct *ws) | |||
433 | 414 | ||
434 | if (bio->bi_iter.bi_size == 0) | 415 | if (bio->bi_iter.bi_size == 0) |
435 | /* an empty barrier - all done */ | 416 | /* an empty barrier - all done */ |
436 | bio_endio(bio, 0); | 417 | bio_endio(bio); |
437 | else { | 418 | else { |
438 | bio->bi_rw &= ~REQ_FLUSH; | 419 | bio->bi_rw &= ~REQ_FLUSH; |
439 | mddev->pers->make_request(mddev, bio); | 420 | mddev->pers->make_request(mddev, bio); |
@@ -728,15 +709,13 @@ void md_rdev_clear(struct md_rdev *rdev) | |||
728 | } | 709 | } |
729 | EXPORT_SYMBOL_GPL(md_rdev_clear); | 710 | EXPORT_SYMBOL_GPL(md_rdev_clear); |
730 | 711 | ||
731 | static void super_written(struct bio *bio, int error) | 712 | static void super_written(struct bio *bio) |
732 | { | 713 | { |
733 | struct md_rdev *rdev = bio->bi_private; | 714 | struct md_rdev *rdev = bio->bi_private; |
734 | struct mddev *mddev = rdev->mddev; | 715 | struct mddev *mddev = rdev->mddev; |
735 | 716 | ||
736 | if (error || !test_bit(BIO_UPTODATE, &bio->bi_flags)) { | 717 | if (bio->bi_error) { |
737 | printk("md: super_written gets error=%d, uptodate=%d\n", | 718 | printk("md: super_written gets error=%d\n", bio->bi_error); |
738 | error, test_bit(BIO_UPTODATE, &bio->bi_flags)); | ||
739 | WARN_ON(test_bit(BIO_UPTODATE, &bio->bi_flags)); | ||
740 | md_error(mddev, rdev); | 719 | md_error(mddev, rdev); |
741 | } | 720 | } |
742 | 721 | ||
@@ -791,7 +770,7 @@ int sync_page_io(struct md_rdev *rdev, sector_t sector, int size, | |||
791 | bio_add_page(bio, page, size, 0); | 770 | bio_add_page(bio, page, size, 0); |
792 | submit_bio_wait(rw, bio); | 771 | submit_bio_wait(rw, bio); |
793 | 772 | ||
794 | ret = test_bit(BIO_UPTODATE, &bio->bi_flags); | 773 | ret = !bio->bi_error; |
795 | bio_put(bio); | 774 | bio_put(bio); |
796 | return ret; | 775 | return ret; |
797 | } | 776 | } |
@@ -5186,7 +5165,6 @@ int md_run(struct mddev *mddev) | |||
5186 | if (mddev->queue) { | 5165 | if (mddev->queue) { |
5187 | mddev->queue->backing_dev_info.congested_data = mddev; | 5166 | mddev->queue->backing_dev_info.congested_data = mddev; |
5188 | mddev->queue->backing_dev_info.congested_fn = md_congested; | 5167 | mddev->queue->backing_dev_info.congested_fn = md_congested; |
5189 | blk_queue_merge_bvec(mddev->queue, md_mergeable_bvec); | ||
5190 | } | 5168 | } |
5191 | if (pers->sync_request) { | 5169 | if (pers->sync_request) { |
5192 | if (mddev->kobj.sd && | 5170 | if (mddev->kobj.sd && |
@@ -5315,7 +5293,6 @@ static void md_clean(struct mddev *mddev) | |||
5315 | mddev->degraded = 0; | 5293 | mddev->degraded = 0; |
5316 | mddev->safemode = 0; | 5294 | mddev->safemode = 0; |
5317 | mddev->private = NULL; | 5295 | mddev->private = NULL; |
5318 | mddev->merge_check_needed = 0; | ||
5319 | mddev->bitmap_info.offset = 0; | 5296 | mddev->bitmap_info.offset = 0; |
5320 | mddev->bitmap_info.default_offset = 0; | 5297 | mddev->bitmap_info.default_offset = 0; |
5321 | mddev->bitmap_info.default_space = 0; | 5298 | mddev->bitmap_info.default_space = 0; |
@@ -5514,7 +5491,6 @@ static int do_md_stop(struct mddev *mddev, int mode, | |||
5514 | 5491 | ||
5515 | __md_stop_writes(mddev); | 5492 | __md_stop_writes(mddev); |
5516 | __md_stop(mddev); | 5493 | __md_stop(mddev); |
5517 | mddev->queue->merge_bvec_fn = NULL; | ||
5518 | mddev->queue->backing_dev_info.congested_fn = NULL; | 5494 | mddev->queue->backing_dev_info.congested_fn = NULL; |
5519 | 5495 | ||
5520 | /* tell userspace to handle 'inactive' */ | 5496 | /* tell userspace to handle 'inactive' */ |
diff --git a/drivers/md/md.h b/drivers/md/md.h index 7da6e9c3cb53..ab339571e57f 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h | |||
@@ -134,10 +134,6 @@ enum flag_bits { | |||
134 | Bitmap_sync, /* ..actually, not quite In_sync. Need a | 134 | Bitmap_sync, /* ..actually, not quite In_sync. Need a |
135 | * bitmap-based recovery to get fully in sync | 135 | * bitmap-based recovery to get fully in sync |
136 | */ | 136 | */ |
137 | Unmerged, /* device is being added to array and should | ||
138 | * be considerred for bvec_merge_fn but not | ||
139 | * yet for actual IO | ||
140 | */ | ||
141 | WriteMostly, /* Avoid reading if at all possible */ | 137 | WriteMostly, /* Avoid reading if at all possible */ |
142 | AutoDetected, /* added by auto-detect */ | 138 | AutoDetected, /* added by auto-detect */ |
143 | Blocked, /* An error occurred but has not yet | 139 | Blocked, /* An error occurred but has not yet |
@@ -374,10 +370,6 @@ struct mddev { | |||
374 | int degraded; /* whether md should consider | 370 | int degraded; /* whether md should consider |
375 | * adding a spare | 371 | * adding a spare |
376 | */ | 372 | */ |
377 | int merge_check_needed; /* at least one | ||
378 | * member device | ||
379 | * has a | ||
380 | * merge_bvec_fn */ | ||
381 | 373 | ||
382 | atomic_t recovery_active; /* blocks scheduled, but not written */ | 374 | atomic_t recovery_active; /* blocks scheduled, but not written */ |
383 | wait_queue_head_t recovery_wait; | 375 | wait_queue_head_t recovery_wait; |
@@ -532,10 +524,6 @@ struct md_personality | |||
532 | /* congested implements bdi.congested_fn(). | 524 | /* congested implements bdi.congested_fn(). |
533 | * Will not be called while array is 'suspended' */ | 525 | * Will not be called while array is 'suspended' */ |
534 | int (*congested)(struct mddev *mddev, int bits); | 526 | int (*congested)(struct mddev *mddev, int bits); |
535 | /* mergeable_bvec is use to implement ->merge_bvec_fn */ | ||
536 | int (*mergeable_bvec)(struct mddev *mddev, | ||
537 | struct bvec_merge_data *bvm, | ||
538 | struct bio_vec *biovec); | ||
539 | }; | 527 | }; |
540 | 528 | ||
541 | struct md_sysfs_entry { | 529 | struct md_sysfs_entry { |
diff --git a/drivers/md/multipath.c b/drivers/md/multipath.c index ac3ede2bd00e..d222522c52e0 100644 --- a/drivers/md/multipath.c +++ b/drivers/md/multipath.c | |||
@@ -77,18 +77,18 @@ static void multipath_end_bh_io (struct multipath_bh *mp_bh, int err) | |||
77 | struct bio *bio = mp_bh->master_bio; | 77 | struct bio *bio = mp_bh->master_bio; |
78 | struct mpconf *conf = mp_bh->mddev->private; | 78 | struct mpconf *conf = mp_bh->mddev->private; |
79 | 79 | ||
80 | bio_endio(bio, err); | 80 | bio->bi_error = err; |
81 | bio_endio(bio); | ||
81 | mempool_free(mp_bh, conf->pool); | 82 | mempool_free(mp_bh, conf->pool); |
82 | } | 83 | } |
83 | 84 | ||
84 | static void multipath_end_request(struct bio *bio, int error) | 85 | static void multipath_end_request(struct bio *bio) |
85 | { | 86 | { |
86 | int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
87 | struct multipath_bh *mp_bh = bio->bi_private; | 87 | struct multipath_bh *mp_bh = bio->bi_private; |
88 | struct mpconf *conf = mp_bh->mddev->private; | 88 | struct mpconf *conf = mp_bh->mddev->private; |
89 | struct md_rdev *rdev = conf->multipaths[mp_bh->path].rdev; | 89 | struct md_rdev *rdev = conf->multipaths[mp_bh->path].rdev; |
90 | 90 | ||
91 | if (uptodate) | 91 | if (!bio->bi_error) |
92 | multipath_end_bh_io(mp_bh, 0); | 92 | multipath_end_bh_io(mp_bh, 0); |
93 | else if (!(bio->bi_rw & REQ_RAHEAD)) { | 93 | else if (!(bio->bi_rw & REQ_RAHEAD)) { |
94 | /* | 94 | /* |
@@ -101,7 +101,7 @@ static void multipath_end_request(struct bio *bio, int error) | |||
101 | (unsigned long long)bio->bi_iter.bi_sector); | 101 | (unsigned long long)bio->bi_iter.bi_sector); |
102 | multipath_reschedule_retry(mp_bh); | 102 | multipath_reschedule_retry(mp_bh); |
103 | } else | 103 | } else |
104 | multipath_end_bh_io(mp_bh, error); | 104 | multipath_end_bh_io(mp_bh, bio->bi_error); |
105 | rdev_dec_pending(rdev, conf->mddev); | 105 | rdev_dec_pending(rdev, conf->mddev); |
106 | } | 106 | } |
107 | 107 | ||
@@ -123,7 +123,7 @@ static void multipath_make_request(struct mddev *mddev, struct bio * bio) | |||
123 | 123 | ||
124 | mp_bh->path = multipath_map(conf); | 124 | mp_bh->path = multipath_map(conf); |
125 | if (mp_bh->path < 0) { | 125 | if (mp_bh->path < 0) { |
126 | bio_endio(bio, -EIO); | 126 | bio_io_error(bio); |
127 | mempool_free(mp_bh, conf->pool); | 127 | mempool_free(mp_bh, conf->pool); |
128 | return; | 128 | return; |
129 | } | 129 | } |
@@ -257,18 +257,6 @@ static int multipath_add_disk(struct mddev *mddev, struct md_rdev *rdev) | |||
257 | disk_stack_limits(mddev->gendisk, rdev->bdev, | 257 | disk_stack_limits(mddev->gendisk, rdev->bdev, |
258 | rdev->data_offset << 9); | 258 | rdev->data_offset << 9); |
259 | 259 | ||
260 | /* as we don't honour merge_bvec_fn, we must never risk | ||
261 | * violating it, so limit ->max_segments to one, lying | ||
262 | * within a single page. | ||
263 | * (Note: it is very unlikely that a device with | ||
264 | * merge_bvec_fn will be involved in multipath.) | ||
265 | */ | ||
266 | if (q->merge_bvec_fn) { | ||
267 | blk_queue_max_segments(mddev->queue, 1); | ||
268 | blk_queue_segment_boundary(mddev->queue, | ||
269 | PAGE_CACHE_SIZE - 1); | ||
270 | } | ||
271 | |||
272 | spin_lock_irq(&conf->device_lock); | 260 | spin_lock_irq(&conf->device_lock); |
273 | mddev->degraded--; | 261 | mddev->degraded--; |
274 | rdev->raid_disk = path; | 262 | rdev->raid_disk = path; |
@@ -432,15 +420,6 @@ static int multipath_run (struct mddev *mddev) | |||
432 | disk_stack_limits(mddev->gendisk, rdev->bdev, | 420 | disk_stack_limits(mddev->gendisk, rdev->bdev, |
433 | rdev->data_offset << 9); | 421 | rdev->data_offset << 9); |
434 | 422 | ||
435 | /* as we don't honour merge_bvec_fn, we must never risk | ||
436 | * violating it, not that we ever expect a device with | ||
437 | * a merge_bvec_fn to be involved in multipath */ | ||
438 | if (rdev->bdev->bd_disk->queue->merge_bvec_fn) { | ||
439 | blk_queue_max_segments(mddev->queue, 1); | ||
440 | blk_queue_segment_boundary(mddev->queue, | ||
441 | PAGE_CACHE_SIZE - 1); | ||
442 | } | ||
443 | |||
444 | if (!test_bit(Faulty, &rdev->flags)) | 423 | if (!test_bit(Faulty, &rdev->flags)) |
445 | working_disks++; | 424 | working_disks++; |
446 | } | 425 | } |
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index efb654eb5399..59cda501a224 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c | |||
@@ -192,9 +192,6 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf) | |||
192 | disk_stack_limits(mddev->gendisk, rdev1->bdev, | 192 | disk_stack_limits(mddev->gendisk, rdev1->bdev, |
193 | rdev1->data_offset << 9); | 193 | rdev1->data_offset << 9); |
194 | 194 | ||
195 | if (rdev1->bdev->bd_disk->queue->merge_bvec_fn) | ||
196 | conf->has_merge_bvec = 1; | ||
197 | |||
198 | if (!smallest || (rdev1->sectors < smallest->sectors)) | 195 | if (!smallest || (rdev1->sectors < smallest->sectors)) |
199 | smallest = rdev1; | 196 | smallest = rdev1; |
200 | cnt++; | 197 | cnt++; |
@@ -351,58 +348,6 @@ static struct md_rdev *map_sector(struct mddev *mddev, struct strip_zone *zone, | |||
351 | + sector_div(sector, zone->nb_dev)]; | 348 | + sector_div(sector, zone->nb_dev)]; |
352 | } | 349 | } |
353 | 350 | ||
354 | /** | ||
355 | * raid0_mergeable_bvec -- tell bio layer if two requests can be merged | ||
356 | * @mddev: the md device | ||
357 | * @bvm: properties of new bio | ||
358 | * @biovec: the request that could be merged to it. | ||
359 | * | ||
360 | * Return amount of bytes we can accept at this offset | ||
361 | */ | ||
362 | static int raid0_mergeable_bvec(struct mddev *mddev, | ||
363 | struct bvec_merge_data *bvm, | ||
364 | struct bio_vec *biovec) | ||
365 | { | ||
366 | struct r0conf *conf = mddev->private; | ||
367 | sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); | ||
368 | sector_t sector_offset = sector; | ||
369 | int max; | ||
370 | unsigned int chunk_sectors = mddev->chunk_sectors; | ||
371 | unsigned int bio_sectors = bvm->bi_size >> 9; | ||
372 | struct strip_zone *zone; | ||
373 | struct md_rdev *rdev; | ||
374 | struct request_queue *subq; | ||
375 | |||
376 | if (is_power_of_2(chunk_sectors)) | ||
377 | max = (chunk_sectors - ((sector & (chunk_sectors-1)) | ||
378 | + bio_sectors)) << 9; | ||
379 | else | ||
380 | max = (chunk_sectors - (sector_div(sector, chunk_sectors) | ||
381 | + bio_sectors)) << 9; | ||
382 | if (max < 0) | ||
383 | max = 0; /* bio_add cannot handle a negative return */ | ||
384 | if (max <= biovec->bv_len && bio_sectors == 0) | ||
385 | return biovec->bv_len; | ||
386 | if (max < biovec->bv_len) | ||
387 | /* too small already, no need to check further */ | ||
388 | return max; | ||
389 | if (!conf->has_merge_bvec) | ||
390 | return max; | ||
391 | |||
392 | /* May need to check subordinate device */ | ||
393 | sector = sector_offset; | ||
394 | zone = find_zone(mddev->private, §or_offset); | ||
395 | rdev = map_sector(mddev, zone, sector, §or_offset); | ||
396 | subq = bdev_get_queue(rdev->bdev); | ||
397 | if (subq->merge_bvec_fn) { | ||
398 | bvm->bi_bdev = rdev->bdev; | ||
399 | bvm->bi_sector = sector_offset + zone->dev_start + | ||
400 | rdev->data_offset; | ||
401 | return min(max, subq->merge_bvec_fn(subq, bvm, biovec)); | ||
402 | } else | ||
403 | return max; | ||
404 | } | ||
405 | |||
406 | static sector_t raid0_size(struct mddev *mddev, sector_t sectors, int raid_disks) | 351 | static sector_t raid0_size(struct mddev *mddev, sector_t sectors, int raid_disks) |
407 | { | 352 | { |
408 | sector_t array_sectors = 0; | 353 | sector_t array_sectors = 0; |
@@ -543,7 +488,7 @@ static void raid0_make_request(struct mddev *mddev, struct bio *bio) | |||
543 | if (unlikely((split->bi_rw & REQ_DISCARD) && | 488 | if (unlikely((split->bi_rw & REQ_DISCARD) && |
544 | !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) { | 489 | !blk_queue_discard(bdev_get_queue(split->bi_bdev)))) { |
545 | /* Just ignore it */ | 490 | /* Just ignore it */ |
546 | bio_endio(split, 0); | 491 | bio_endio(split); |
547 | } else | 492 | } else |
548 | generic_make_request(split); | 493 | generic_make_request(split); |
549 | } while (split != bio); | 494 | } while (split != bio); |
@@ -727,7 +672,6 @@ static struct md_personality raid0_personality= | |||
727 | .takeover = raid0_takeover, | 672 | .takeover = raid0_takeover, |
728 | .quiesce = raid0_quiesce, | 673 | .quiesce = raid0_quiesce, |
729 | .congested = raid0_congested, | 674 | .congested = raid0_congested, |
730 | .mergeable_bvec = raid0_mergeable_bvec, | ||
731 | }; | 675 | }; |
732 | 676 | ||
733 | static int __init raid0_init (void) | 677 | static int __init raid0_init (void) |
diff --git a/drivers/md/raid0.h b/drivers/md/raid0.h index 05539d9c97f0..7127a623f5da 100644 --- a/drivers/md/raid0.h +++ b/drivers/md/raid0.h | |||
@@ -12,8 +12,6 @@ struct r0conf { | |||
12 | struct md_rdev **devlist; /* lists of rdevs, pointed to | 12 | struct md_rdev **devlist; /* lists of rdevs, pointed to |
13 | * by strip_zone->dev */ | 13 | * by strip_zone->dev */ |
14 | int nr_strip_zones; | 14 | int nr_strip_zones; |
15 | int has_merge_bvec; /* at least one member has | ||
16 | * a merge_bvec_fn */ | ||
17 | }; | 15 | }; |
18 | 16 | ||
19 | #endif | 17 | #endif |
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 967a4ed73929..f39d69f884de 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c | |||
@@ -255,9 +255,10 @@ static void call_bio_endio(struct r1bio *r1_bio) | |||
255 | done = 1; | 255 | done = 1; |
256 | 256 | ||
257 | if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) | 257 | if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) |
258 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | 258 | bio->bi_error = -EIO; |
259 | |||
259 | if (done) { | 260 | if (done) { |
260 | bio_endio(bio, 0); | 261 | bio_endio(bio); |
261 | /* | 262 | /* |
262 | * Wake up any possible resync thread that waits for the device | 263 | * Wake up any possible resync thread that waits for the device |
263 | * to go idle. | 264 | * to go idle. |
@@ -312,9 +313,9 @@ static int find_bio_disk(struct r1bio *r1_bio, struct bio *bio) | |||
312 | return mirror; | 313 | return mirror; |
313 | } | 314 | } |
314 | 315 | ||
315 | static void raid1_end_read_request(struct bio *bio, int error) | 316 | static void raid1_end_read_request(struct bio *bio) |
316 | { | 317 | { |
317 | int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | 318 | int uptodate = !bio->bi_error; |
318 | struct r1bio *r1_bio = bio->bi_private; | 319 | struct r1bio *r1_bio = bio->bi_private; |
319 | int mirror; | 320 | int mirror; |
320 | struct r1conf *conf = r1_bio->mddev->private; | 321 | struct r1conf *conf = r1_bio->mddev->private; |
@@ -397,9 +398,8 @@ static void r1_bio_write_done(struct r1bio *r1_bio) | |||
397 | } | 398 | } |
398 | } | 399 | } |
399 | 400 | ||
400 | static void raid1_end_write_request(struct bio *bio, int error) | 401 | static void raid1_end_write_request(struct bio *bio) |
401 | { | 402 | { |
402 | int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
403 | struct r1bio *r1_bio = bio->bi_private; | 403 | struct r1bio *r1_bio = bio->bi_private; |
404 | int mirror, behind = test_bit(R1BIO_BehindIO, &r1_bio->state); | 404 | int mirror, behind = test_bit(R1BIO_BehindIO, &r1_bio->state); |
405 | struct r1conf *conf = r1_bio->mddev->private; | 405 | struct r1conf *conf = r1_bio->mddev->private; |
@@ -410,7 +410,7 @@ static void raid1_end_write_request(struct bio *bio, int error) | |||
410 | /* | 410 | /* |
411 | * 'one mirror IO has finished' event handler: | 411 | * 'one mirror IO has finished' event handler: |
412 | */ | 412 | */ |
413 | if (!uptodate) { | 413 | if (bio->bi_error) { |
414 | set_bit(WriteErrorSeen, | 414 | set_bit(WriteErrorSeen, |
415 | &conf->mirrors[mirror].rdev->flags); | 415 | &conf->mirrors[mirror].rdev->flags); |
416 | if (!test_and_set_bit(WantReplacement, | 416 | if (!test_and_set_bit(WantReplacement, |
@@ -557,7 +557,6 @@ static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sect | |||
557 | rdev = rcu_dereference(conf->mirrors[disk].rdev); | 557 | rdev = rcu_dereference(conf->mirrors[disk].rdev); |
558 | if (r1_bio->bios[disk] == IO_BLOCKED | 558 | if (r1_bio->bios[disk] == IO_BLOCKED |
559 | || rdev == NULL | 559 | || rdev == NULL |
560 | || test_bit(Unmerged, &rdev->flags) | ||
561 | || test_bit(Faulty, &rdev->flags)) | 560 | || test_bit(Faulty, &rdev->flags)) |
562 | continue; | 561 | continue; |
563 | if (!test_bit(In_sync, &rdev->flags) && | 562 | if (!test_bit(In_sync, &rdev->flags) && |
@@ -708,38 +707,6 @@ static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sect | |||
708 | return best_disk; | 707 | return best_disk; |
709 | } | 708 | } |
710 | 709 | ||
711 | static int raid1_mergeable_bvec(struct mddev *mddev, | ||
712 | struct bvec_merge_data *bvm, | ||
713 | struct bio_vec *biovec) | ||
714 | { | ||
715 | struct r1conf *conf = mddev->private; | ||
716 | sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); | ||
717 | int max = biovec->bv_len; | ||
718 | |||
719 | if (mddev->merge_check_needed) { | ||
720 | int disk; | ||
721 | rcu_read_lock(); | ||
722 | for (disk = 0; disk < conf->raid_disks * 2; disk++) { | ||
723 | struct md_rdev *rdev = rcu_dereference( | ||
724 | conf->mirrors[disk].rdev); | ||
725 | if (rdev && !test_bit(Faulty, &rdev->flags)) { | ||
726 | struct request_queue *q = | ||
727 | bdev_get_queue(rdev->bdev); | ||
728 | if (q->merge_bvec_fn) { | ||
729 | bvm->bi_sector = sector + | ||
730 | rdev->data_offset; | ||
731 | bvm->bi_bdev = rdev->bdev; | ||
732 | max = min(max, q->merge_bvec_fn( | ||
733 | q, bvm, biovec)); | ||
734 | } | ||
735 | } | ||
736 | } | ||
737 | rcu_read_unlock(); | ||
738 | } | ||
739 | return max; | ||
740 | |||
741 | } | ||
742 | |||
743 | static int raid1_congested(struct mddev *mddev, int bits) | 710 | static int raid1_congested(struct mddev *mddev, int bits) |
744 | { | 711 | { |
745 | struct r1conf *conf = mddev->private; | 712 | struct r1conf *conf = mddev->private; |
@@ -793,7 +760,7 @@ static void flush_pending_writes(struct r1conf *conf) | |||
793 | if (unlikely((bio->bi_rw & REQ_DISCARD) && | 760 | if (unlikely((bio->bi_rw & REQ_DISCARD) && |
794 | !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) | 761 | !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) |
795 | /* Just ignore it */ | 762 | /* Just ignore it */ |
796 | bio_endio(bio, 0); | 763 | bio_endio(bio); |
797 | else | 764 | else |
798 | generic_make_request(bio); | 765 | generic_make_request(bio); |
799 | bio = next; | 766 | bio = next; |
@@ -1068,7 +1035,7 @@ static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule) | |||
1068 | if (unlikely((bio->bi_rw & REQ_DISCARD) && | 1035 | if (unlikely((bio->bi_rw & REQ_DISCARD) && |
1069 | !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) | 1036 | !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) |
1070 | /* Just ignore it */ | 1037 | /* Just ignore it */ |
1071 | bio_endio(bio, 0); | 1038 | bio_endio(bio); |
1072 | else | 1039 | else |
1073 | generic_make_request(bio); | 1040 | generic_make_request(bio); |
1074 | bio = next; | 1041 | bio = next; |
@@ -1158,7 +1125,7 @@ static void make_request(struct mddev *mddev, struct bio * bio) | |||
1158 | * non-zero, then it is the number of not-completed requests. | 1125 | * non-zero, then it is the number of not-completed requests. |
1159 | */ | 1126 | */ |
1160 | bio->bi_phys_segments = 0; | 1127 | bio->bi_phys_segments = 0; |
1161 | clear_bit(BIO_SEG_VALID, &bio->bi_flags); | 1128 | bio_clear_flag(bio, BIO_SEG_VALID); |
1162 | 1129 | ||
1163 | if (rw == READ) { | 1130 | if (rw == READ) { |
1164 | /* | 1131 | /* |
@@ -1269,8 +1236,7 @@ read_again: | |||
1269 | break; | 1236 | break; |
1270 | } | 1237 | } |
1271 | r1_bio->bios[i] = NULL; | 1238 | r1_bio->bios[i] = NULL; |
1272 | if (!rdev || test_bit(Faulty, &rdev->flags) | 1239 | if (!rdev || test_bit(Faulty, &rdev->flags)) { |
1273 | || test_bit(Unmerged, &rdev->flags)) { | ||
1274 | if (i < conf->raid_disks) | 1240 | if (i < conf->raid_disks) |
1275 | set_bit(R1BIO_Degraded, &r1_bio->state); | 1241 | set_bit(R1BIO_Degraded, &r1_bio->state); |
1276 | continue; | 1242 | continue; |
@@ -1617,7 +1583,6 @@ static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev) | |||
1617 | struct raid1_info *p; | 1583 | struct raid1_info *p; |
1618 | int first = 0; | 1584 | int first = 0; |
1619 | int last = conf->raid_disks - 1; | 1585 | int last = conf->raid_disks - 1; |
1620 | struct request_queue *q = bdev_get_queue(rdev->bdev); | ||
1621 | 1586 | ||
1622 | if (mddev->recovery_disabled == conf->recovery_disabled) | 1587 | if (mddev->recovery_disabled == conf->recovery_disabled) |
1623 | return -EBUSY; | 1588 | return -EBUSY; |
@@ -1625,11 +1590,6 @@ static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev) | |||
1625 | if (rdev->raid_disk >= 0) | 1590 | if (rdev->raid_disk >= 0) |
1626 | first = last = rdev->raid_disk; | 1591 | first = last = rdev->raid_disk; |
1627 | 1592 | ||
1628 | if (q->merge_bvec_fn) { | ||
1629 | set_bit(Unmerged, &rdev->flags); | ||
1630 | mddev->merge_check_needed = 1; | ||
1631 | } | ||
1632 | |||
1633 | for (mirror = first; mirror <= last; mirror++) { | 1593 | for (mirror = first; mirror <= last; mirror++) { |
1634 | p = conf->mirrors+mirror; | 1594 | p = conf->mirrors+mirror; |
1635 | if (!p->rdev) { | 1595 | if (!p->rdev) { |
@@ -1661,19 +1621,6 @@ static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev) | |||
1661 | break; | 1621 | break; |
1662 | } | 1622 | } |
1663 | } | 1623 | } |
1664 | if (err == 0 && test_bit(Unmerged, &rdev->flags)) { | ||
1665 | /* Some requests might not have seen this new | ||
1666 | * merge_bvec_fn. We must wait for them to complete | ||
1667 | * before merging the device fully. | ||
1668 | * First we make sure any code which has tested | ||
1669 | * our function has submitted the request, then | ||
1670 | * we wait for all outstanding requests to complete. | ||
1671 | */ | ||
1672 | synchronize_sched(); | ||
1673 | freeze_array(conf, 0); | ||
1674 | unfreeze_array(conf); | ||
1675 | clear_bit(Unmerged, &rdev->flags); | ||
1676 | } | ||
1677 | md_integrity_add_rdev(rdev, mddev); | 1624 | md_integrity_add_rdev(rdev, mddev); |
1678 | if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev))) | 1625 | if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev))) |
1679 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue); | 1626 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue); |
@@ -1737,7 +1684,7 @@ abort: | |||
1737 | return err; | 1684 | return err; |
1738 | } | 1685 | } |
1739 | 1686 | ||
1740 | static void end_sync_read(struct bio *bio, int error) | 1687 | static void end_sync_read(struct bio *bio) |
1741 | { | 1688 | { |
1742 | struct r1bio *r1_bio = bio->bi_private; | 1689 | struct r1bio *r1_bio = bio->bi_private; |
1743 | 1690 | ||
@@ -1748,16 +1695,16 @@ static void end_sync_read(struct bio *bio, int error) | |||
1748 | * or re-read if the read failed. | 1695 | * or re-read if the read failed. |
1749 | * We don't do much here, just schedule handling by raid1d | 1696 | * We don't do much here, just schedule handling by raid1d |
1750 | */ | 1697 | */ |
1751 | if (test_bit(BIO_UPTODATE, &bio->bi_flags)) | 1698 | if (!bio->bi_error) |
1752 | set_bit(R1BIO_Uptodate, &r1_bio->state); | 1699 | set_bit(R1BIO_Uptodate, &r1_bio->state); |
1753 | 1700 | ||
1754 | if (atomic_dec_and_test(&r1_bio->remaining)) | 1701 | if (atomic_dec_and_test(&r1_bio->remaining)) |
1755 | reschedule_retry(r1_bio); | 1702 | reschedule_retry(r1_bio); |
1756 | } | 1703 | } |
1757 | 1704 | ||
1758 | static void end_sync_write(struct bio *bio, int error) | 1705 | static void end_sync_write(struct bio *bio) |
1759 | { | 1706 | { |
1760 | int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | 1707 | int uptodate = !bio->bi_error; |
1761 | struct r1bio *r1_bio = bio->bi_private; | 1708 | struct r1bio *r1_bio = bio->bi_private; |
1762 | struct mddev *mddev = r1_bio->mddev; | 1709 | struct mddev *mddev = r1_bio->mddev; |
1763 | struct r1conf *conf = mddev->private; | 1710 | struct r1conf *conf = mddev->private; |
@@ -1944,7 +1891,7 @@ static int fix_sync_read_error(struct r1bio *r1_bio) | |||
1944 | idx ++; | 1891 | idx ++; |
1945 | } | 1892 | } |
1946 | set_bit(R1BIO_Uptodate, &r1_bio->state); | 1893 | set_bit(R1BIO_Uptodate, &r1_bio->state); |
1947 | set_bit(BIO_UPTODATE, &bio->bi_flags); | 1894 | bio->bi_error = 0; |
1948 | return 1; | 1895 | return 1; |
1949 | } | 1896 | } |
1950 | 1897 | ||
@@ -1968,15 +1915,14 @@ static void process_checks(struct r1bio *r1_bio) | |||
1968 | for (i = 0; i < conf->raid_disks * 2; i++) { | 1915 | for (i = 0; i < conf->raid_disks * 2; i++) { |
1969 | int j; | 1916 | int j; |
1970 | int size; | 1917 | int size; |
1971 | int uptodate; | 1918 | int error; |
1972 | struct bio *b = r1_bio->bios[i]; | 1919 | struct bio *b = r1_bio->bios[i]; |
1973 | if (b->bi_end_io != end_sync_read) | 1920 | if (b->bi_end_io != end_sync_read) |
1974 | continue; | 1921 | continue; |
1975 | /* fixup the bio for reuse, but preserve BIO_UPTODATE */ | 1922 | /* fixup the bio for reuse, but preserve errno */ |
1976 | uptodate = test_bit(BIO_UPTODATE, &b->bi_flags); | 1923 | error = b->bi_error; |
1977 | bio_reset(b); | 1924 | bio_reset(b); |
1978 | if (!uptodate) | 1925 | b->bi_error = error; |
1979 | clear_bit(BIO_UPTODATE, &b->bi_flags); | ||
1980 | b->bi_vcnt = vcnt; | 1926 | b->bi_vcnt = vcnt; |
1981 | b->bi_iter.bi_size = r1_bio->sectors << 9; | 1927 | b->bi_iter.bi_size = r1_bio->sectors << 9; |
1982 | b->bi_iter.bi_sector = r1_bio->sector + | 1928 | b->bi_iter.bi_sector = r1_bio->sector + |
@@ -1999,7 +1945,7 @@ static void process_checks(struct r1bio *r1_bio) | |||
1999 | } | 1945 | } |
2000 | for (primary = 0; primary < conf->raid_disks * 2; primary++) | 1946 | for (primary = 0; primary < conf->raid_disks * 2; primary++) |
2001 | if (r1_bio->bios[primary]->bi_end_io == end_sync_read && | 1947 | if (r1_bio->bios[primary]->bi_end_io == end_sync_read && |
2002 | test_bit(BIO_UPTODATE, &r1_bio->bios[primary]->bi_flags)) { | 1948 | !r1_bio->bios[primary]->bi_error) { |
2003 | r1_bio->bios[primary]->bi_end_io = NULL; | 1949 | r1_bio->bios[primary]->bi_end_io = NULL; |
2004 | rdev_dec_pending(conf->mirrors[primary].rdev, mddev); | 1950 | rdev_dec_pending(conf->mirrors[primary].rdev, mddev); |
2005 | break; | 1951 | break; |
@@ -2009,14 +1955,14 @@ static void process_checks(struct r1bio *r1_bio) | |||
2009 | int j; | 1955 | int j; |
2010 | struct bio *pbio = r1_bio->bios[primary]; | 1956 | struct bio *pbio = r1_bio->bios[primary]; |
2011 | struct bio *sbio = r1_bio->bios[i]; | 1957 | struct bio *sbio = r1_bio->bios[i]; |
2012 | int uptodate = test_bit(BIO_UPTODATE, &sbio->bi_flags); | 1958 | int error = sbio->bi_error; |
2013 | 1959 | ||
2014 | if (sbio->bi_end_io != end_sync_read) | 1960 | if (sbio->bi_end_io != end_sync_read) |
2015 | continue; | 1961 | continue; |
2016 | /* Now we can 'fixup' the BIO_UPTODATE flag */ | 1962 | /* Now we can 'fixup' the error value */ |
2017 | set_bit(BIO_UPTODATE, &sbio->bi_flags); | 1963 | sbio->bi_error = 0; |
2018 | 1964 | ||
2019 | if (uptodate) { | 1965 | if (!error) { |
2020 | for (j = vcnt; j-- ; ) { | 1966 | for (j = vcnt; j-- ; ) { |
2021 | struct page *p, *s; | 1967 | struct page *p, *s; |
2022 | p = pbio->bi_io_vec[j].bv_page; | 1968 | p = pbio->bi_io_vec[j].bv_page; |
@@ -2031,7 +1977,7 @@ static void process_checks(struct r1bio *r1_bio) | |||
2031 | if (j >= 0) | 1977 | if (j >= 0) |
2032 | atomic64_add(r1_bio->sectors, &mddev->resync_mismatches); | 1978 | atomic64_add(r1_bio->sectors, &mddev->resync_mismatches); |
2033 | if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery) | 1979 | if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery) |
2034 | && uptodate)) { | 1980 | && !error)) { |
2035 | /* No need to write to this device. */ | 1981 | /* No need to write to this device. */ |
2036 | sbio->bi_end_io = NULL; | 1982 | sbio->bi_end_io = NULL; |
2037 | rdev_dec_pending(conf->mirrors[i].rdev, mddev); | 1983 | rdev_dec_pending(conf->mirrors[i].rdev, mddev); |
@@ -2272,11 +2218,11 @@ static void handle_sync_write_finished(struct r1conf *conf, struct r1bio *r1_bio | |||
2272 | struct bio *bio = r1_bio->bios[m]; | 2218 | struct bio *bio = r1_bio->bios[m]; |
2273 | if (bio->bi_end_io == NULL) | 2219 | if (bio->bi_end_io == NULL) |
2274 | continue; | 2220 | continue; |
2275 | if (test_bit(BIO_UPTODATE, &bio->bi_flags) && | 2221 | if (!bio->bi_error && |
2276 | test_bit(R1BIO_MadeGood, &r1_bio->state)) { | 2222 | test_bit(R1BIO_MadeGood, &r1_bio->state)) { |
2277 | rdev_clear_badblocks(rdev, r1_bio->sector, s, 0); | 2223 | rdev_clear_badblocks(rdev, r1_bio->sector, s, 0); |
2278 | } | 2224 | } |
2279 | if (!test_bit(BIO_UPTODATE, &bio->bi_flags) && | 2225 | if (bio->bi_error && |
2280 | test_bit(R1BIO_WriteError, &r1_bio->state)) { | 2226 | test_bit(R1BIO_WriteError, &r1_bio->state)) { |
2281 | if (!rdev_set_badblocks(rdev, r1_bio->sector, s, 0)) | 2227 | if (!rdev_set_badblocks(rdev, r1_bio->sector, s, 0)) |
2282 | md_error(conf->mddev, rdev); | 2228 | md_error(conf->mddev, rdev); |
@@ -2715,7 +2661,7 @@ static sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipp | |||
2715 | /* remove last page from this bio */ | 2661 | /* remove last page from this bio */ |
2716 | bio->bi_vcnt--; | 2662 | bio->bi_vcnt--; |
2717 | bio->bi_iter.bi_size -= len; | 2663 | bio->bi_iter.bi_size -= len; |
2718 | __clear_bit(BIO_SEG_VALID, &bio->bi_flags); | 2664 | bio_clear_flag(bio, BIO_SEG_VALID); |
2719 | } | 2665 | } |
2720 | goto bio_full; | 2666 | goto bio_full; |
2721 | } | 2667 | } |
@@ -2810,8 +2756,6 @@ static struct r1conf *setup_conf(struct mddev *mddev) | |||
2810 | goto abort; | 2756 | goto abort; |
2811 | disk->rdev = rdev; | 2757 | disk->rdev = rdev; |
2812 | q = bdev_get_queue(rdev->bdev); | 2758 | q = bdev_get_queue(rdev->bdev); |
2813 | if (q->merge_bvec_fn) | ||
2814 | mddev->merge_check_needed = 1; | ||
2815 | 2759 | ||
2816 | disk->head_position = 0; | 2760 | disk->head_position = 0; |
2817 | disk->seq_start = MaxSector; | 2761 | disk->seq_start = MaxSector; |
@@ -3176,7 +3120,6 @@ static struct md_personality raid1_personality = | |||
3176 | .quiesce = raid1_quiesce, | 3120 | .quiesce = raid1_quiesce, |
3177 | .takeover = raid1_takeover, | 3121 | .takeover = raid1_takeover, |
3178 | .congested = raid1_congested, | 3122 | .congested = raid1_congested, |
3179 | .mergeable_bvec = raid1_mergeable_bvec, | ||
3180 | }; | 3123 | }; |
3181 | 3124 | ||
3182 | static int __init raid_init(void) | 3125 | static int __init raid_init(void) |
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 38c58e19cfce..b0fce2ebf7ad 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c | |||
@@ -101,7 +101,7 @@ static int _enough(struct r10conf *conf, int previous, int ignore); | |||
101 | static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, | 101 | static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, |
102 | int *skipped); | 102 | int *skipped); |
103 | static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio); | 103 | static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio); |
104 | static void end_reshape_write(struct bio *bio, int error); | 104 | static void end_reshape_write(struct bio *bio); |
105 | static void end_reshape(struct r10conf *conf); | 105 | static void end_reshape(struct r10conf *conf); |
106 | 106 | ||
107 | static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) | 107 | static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) |
@@ -307,9 +307,9 @@ static void raid_end_bio_io(struct r10bio *r10_bio) | |||
307 | } else | 307 | } else |
308 | done = 1; | 308 | done = 1; |
309 | if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) | 309 | if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) |
310 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | 310 | bio->bi_error = -EIO; |
311 | if (done) { | 311 | if (done) { |
312 | bio_endio(bio, 0); | 312 | bio_endio(bio); |
313 | /* | 313 | /* |
314 | * Wake up any possible resync thread that waits for the device | 314 | * Wake up any possible resync thread that waits for the device |
315 | * to go idle. | 315 | * to go idle. |
@@ -358,9 +358,9 @@ static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio, | |||
358 | return r10_bio->devs[slot].devnum; | 358 | return r10_bio->devs[slot].devnum; |
359 | } | 359 | } |
360 | 360 | ||
361 | static void raid10_end_read_request(struct bio *bio, int error) | 361 | static void raid10_end_read_request(struct bio *bio) |
362 | { | 362 | { |
363 | int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | 363 | int uptodate = !bio->bi_error; |
364 | struct r10bio *r10_bio = bio->bi_private; | 364 | struct r10bio *r10_bio = bio->bi_private; |
365 | int slot, dev; | 365 | int slot, dev; |
366 | struct md_rdev *rdev; | 366 | struct md_rdev *rdev; |
@@ -438,9 +438,8 @@ static void one_write_done(struct r10bio *r10_bio) | |||
438 | } | 438 | } |
439 | } | 439 | } |
440 | 440 | ||
441 | static void raid10_end_write_request(struct bio *bio, int error) | 441 | static void raid10_end_write_request(struct bio *bio) |
442 | { | 442 | { |
443 | int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
444 | struct r10bio *r10_bio = bio->bi_private; | 443 | struct r10bio *r10_bio = bio->bi_private; |
445 | int dev; | 444 | int dev; |
446 | int dec_rdev = 1; | 445 | int dec_rdev = 1; |
@@ -460,7 +459,7 @@ static void raid10_end_write_request(struct bio *bio, int error) | |||
460 | /* | 459 | /* |
461 | * this branch is our 'one mirror IO has finished' event handler: | 460 | * this branch is our 'one mirror IO has finished' event handler: |
462 | */ | 461 | */ |
463 | if (!uptodate) { | 462 | if (bio->bi_error) { |
464 | if (repl) | 463 | if (repl) |
465 | /* Never record new bad blocks to replacement, | 464 | /* Never record new bad blocks to replacement, |
466 | * just fail it. | 465 | * just fail it. |
@@ -672,93 +671,6 @@ static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev) | |||
672 | return (vchunk << geo->chunk_shift) + offset; | 671 | return (vchunk << geo->chunk_shift) + offset; |
673 | } | 672 | } |
674 | 673 | ||
675 | /** | ||
676 | * raid10_mergeable_bvec -- tell bio layer if a two requests can be merged | ||
677 | * @mddev: the md device | ||
678 | * @bvm: properties of new bio | ||
679 | * @biovec: the request that could be merged to it. | ||
680 | * | ||
681 | * Return amount of bytes we can accept at this offset | ||
682 | * This requires checking for end-of-chunk if near_copies != raid_disks, | ||
683 | * and for subordinate merge_bvec_fns if merge_check_needed. | ||
684 | */ | ||
685 | static int raid10_mergeable_bvec(struct mddev *mddev, | ||
686 | struct bvec_merge_data *bvm, | ||
687 | struct bio_vec *biovec) | ||
688 | { | ||
689 | struct r10conf *conf = mddev->private; | ||
690 | sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); | ||
691 | int max; | ||
692 | unsigned int chunk_sectors; | ||
693 | unsigned int bio_sectors = bvm->bi_size >> 9; | ||
694 | struct geom *geo = &conf->geo; | ||
695 | |||
696 | chunk_sectors = (conf->geo.chunk_mask & conf->prev.chunk_mask) + 1; | ||
697 | if (conf->reshape_progress != MaxSector && | ||
698 | ((sector >= conf->reshape_progress) != | ||
699 | conf->mddev->reshape_backwards)) | ||
700 | geo = &conf->prev; | ||
701 | |||
702 | if (geo->near_copies < geo->raid_disks) { | ||
703 | max = (chunk_sectors - ((sector & (chunk_sectors - 1)) | ||
704 | + bio_sectors)) << 9; | ||
705 | if (max < 0) | ||
706 | /* bio_add cannot handle a negative return */ | ||
707 | max = 0; | ||
708 | if (max <= biovec->bv_len && bio_sectors == 0) | ||
709 | return biovec->bv_len; | ||
710 | } else | ||
711 | max = biovec->bv_len; | ||
712 | |||
713 | if (mddev->merge_check_needed) { | ||
714 | struct { | ||
715 | struct r10bio r10_bio; | ||
716 | struct r10dev devs[conf->copies]; | ||
717 | } on_stack; | ||
718 | struct r10bio *r10_bio = &on_stack.r10_bio; | ||
719 | int s; | ||
720 | if (conf->reshape_progress != MaxSector) { | ||
721 | /* Cannot give any guidance during reshape */ | ||
722 | if (max <= biovec->bv_len && bio_sectors == 0) | ||
723 | return biovec->bv_len; | ||
724 | return 0; | ||
725 | } | ||
726 | r10_bio->sector = sector; | ||
727 | raid10_find_phys(conf, r10_bio); | ||
728 | rcu_read_lock(); | ||
729 | for (s = 0; s < conf->copies; s++) { | ||
730 | int disk = r10_bio->devs[s].devnum; | ||
731 | struct md_rdev *rdev = rcu_dereference( | ||
732 | conf->mirrors[disk].rdev); | ||
733 | if (rdev && !test_bit(Faulty, &rdev->flags)) { | ||
734 | struct request_queue *q = | ||
735 | bdev_get_queue(rdev->bdev); | ||
736 | if (q->merge_bvec_fn) { | ||
737 | bvm->bi_sector = r10_bio->devs[s].addr | ||
738 | + rdev->data_offset; | ||
739 | bvm->bi_bdev = rdev->bdev; | ||
740 | max = min(max, q->merge_bvec_fn( | ||
741 | q, bvm, biovec)); | ||
742 | } | ||
743 | } | ||
744 | rdev = rcu_dereference(conf->mirrors[disk].replacement); | ||
745 | if (rdev && !test_bit(Faulty, &rdev->flags)) { | ||
746 | struct request_queue *q = | ||
747 | bdev_get_queue(rdev->bdev); | ||
748 | if (q->merge_bvec_fn) { | ||
749 | bvm->bi_sector = r10_bio->devs[s].addr | ||
750 | + rdev->data_offset; | ||
751 | bvm->bi_bdev = rdev->bdev; | ||
752 | max = min(max, q->merge_bvec_fn( | ||
753 | q, bvm, biovec)); | ||
754 | } | ||
755 | } | ||
756 | } | ||
757 | rcu_read_unlock(); | ||
758 | } | ||
759 | return max; | ||
760 | } | ||
761 | |||
762 | /* | 674 | /* |
763 | * This routine returns the disk from which the requested read should | 675 | * This routine returns the disk from which the requested read should |
764 | * be done. There is a per-array 'next expected sequential IO' sector | 676 | * be done. There is a per-array 'next expected sequential IO' sector |
@@ -821,12 +733,10 @@ retry: | |||
821 | disk = r10_bio->devs[slot].devnum; | 733 | disk = r10_bio->devs[slot].devnum; |
822 | rdev = rcu_dereference(conf->mirrors[disk].replacement); | 734 | rdev = rcu_dereference(conf->mirrors[disk].replacement); |
823 | if (rdev == NULL || test_bit(Faulty, &rdev->flags) || | 735 | if (rdev == NULL || test_bit(Faulty, &rdev->flags) || |
824 | test_bit(Unmerged, &rdev->flags) || | ||
825 | r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) | 736 | r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) |
826 | rdev = rcu_dereference(conf->mirrors[disk].rdev); | 737 | rdev = rcu_dereference(conf->mirrors[disk].rdev); |
827 | if (rdev == NULL || | 738 | if (rdev == NULL || |
828 | test_bit(Faulty, &rdev->flags) || | 739 | test_bit(Faulty, &rdev->flags)) |
829 | test_bit(Unmerged, &rdev->flags)) | ||
830 | continue; | 740 | continue; |
831 | if (!test_bit(In_sync, &rdev->flags) && | 741 | if (!test_bit(In_sync, &rdev->flags) && |
832 | r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) | 742 | r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) |
@@ -957,7 +867,7 @@ static void flush_pending_writes(struct r10conf *conf) | |||
957 | if (unlikely((bio->bi_rw & REQ_DISCARD) && | 867 | if (unlikely((bio->bi_rw & REQ_DISCARD) && |
958 | !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) | 868 | !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) |
959 | /* Just ignore it */ | 869 | /* Just ignore it */ |
960 | bio_endio(bio, 0); | 870 | bio_endio(bio); |
961 | else | 871 | else |
962 | generic_make_request(bio); | 872 | generic_make_request(bio); |
963 | bio = next; | 873 | bio = next; |
@@ -1133,7 +1043,7 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) | |||
1133 | if (unlikely((bio->bi_rw & REQ_DISCARD) && | 1043 | if (unlikely((bio->bi_rw & REQ_DISCARD) && |
1134 | !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) | 1044 | !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) |
1135 | /* Just ignore it */ | 1045 | /* Just ignore it */ |
1136 | bio_endio(bio, 0); | 1046 | bio_endio(bio); |
1137 | else | 1047 | else |
1138 | generic_make_request(bio); | 1048 | generic_make_request(bio); |
1139 | bio = next; | 1049 | bio = next; |
@@ -1217,7 +1127,7 @@ static void __make_request(struct mddev *mddev, struct bio *bio) | |||
1217 | * non-zero, then it is the number of not-completed requests. | 1127 | * non-zero, then it is the number of not-completed requests. |
1218 | */ | 1128 | */ |
1219 | bio->bi_phys_segments = 0; | 1129 | bio->bi_phys_segments = 0; |
1220 | clear_bit(BIO_SEG_VALID, &bio->bi_flags); | 1130 | bio_clear_flag(bio, BIO_SEG_VALID); |
1221 | 1131 | ||
1222 | if (rw == READ) { | 1132 | if (rw == READ) { |
1223 | /* | 1133 | /* |
@@ -1326,11 +1236,9 @@ retry_write: | |||
1326 | blocked_rdev = rrdev; | 1236 | blocked_rdev = rrdev; |
1327 | break; | 1237 | break; |
1328 | } | 1238 | } |
1329 | if (rdev && (test_bit(Faulty, &rdev->flags) | 1239 | if (rdev && (test_bit(Faulty, &rdev->flags))) |
1330 | || test_bit(Unmerged, &rdev->flags))) | ||
1331 | rdev = NULL; | 1240 | rdev = NULL; |
1332 | if (rrdev && (test_bit(Faulty, &rrdev->flags) | 1241 | if (rrdev && (test_bit(Faulty, &rrdev->flags))) |
1333 | || test_bit(Unmerged, &rrdev->flags))) | ||
1334 | rrdev = NULL; | 1242 | rrdev = NULL; |
1335 | 1243 | ||
1336 | r10_bio->devs[i].bio = NULL; | 1244 | r10_bio->devs[i].bio = NULL; |
@@ -1777,7 +1685,6 @@ static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev) | |||
1777 | int mirror; | 1685 | int mirror; |
1778 | int first = 0; | 1686 | int first = 0; |
1779 | int last = conf->geo.raid_disks - 1; | 1687 | int last = conf->geo.raid_disks - 1; |
1780 | struct request_queue *q = bdev_get_queue(rdev->bdev); | ||
1781 | 1688 | ||
1782 | if (mddev->recovery_cp < MaxSector) | 1689 | if (mddev->recovery_cp < MaxSector) |
1783 | /* only hot-add to in-sync arrays, as recovery is | 1690 | /* only hot-add to in-sync arrays, as recovery is |
@@ -1790,11 +1697,6 @@ static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev) | |||
1790 | if (rdev->raid_disk >= 0) | 1697 | if (rdev->raid_disk >= 0) |
1791 | first = last = rdev->raid_disk; | 1698 | first = last = rdev->raid_disk; |
1792 | 1699 | ||
1793 | if (q->merge_bvec_fn) { | ||
1794 | set_bit(Unmerged, &rdev->flags); | ||
1795 | mddev->merge_check_needed = 1; | ||
1796 | } | ||
1797 | |||
1798 | if (rdev->saved_raid_disk >= first && | 1700 | if (rdev->saved_raid_disk >= first && |
1799 | conf->mirrors[rdev->saved_raid_disk].rdev == NULL) | 1701 | conf->mirrors[rdev->saved_raid_disk].rdev == NULL) |
1800 | mirror = rdev->saved_raid_disk; | 1702 | mirror = rdev->saved_raid_disk; |
@@ -1833,19 +1735,6 @@ static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev) | |||
1833 | rcu_assign_pointer(p->rdev, rdev); | 1735 | rcu_assign_pointer(p->rdev, rdev); |
1834 | break; | 1736 | break; |
1835 | } | 1737 | } |
1836 | if (err == 0 && test_bit(Unmerged, &rdev->flags)) { | ||
1837 | /* Some requests might not have seen this new | ||
1838 | * merge_bvec_fn. We must wait for them to complete | ||
1839 | * before merging the device fully. | ||
1840 | * First we make sure any code which has tested | ||
1841 | * our function has submitted the request, then | ||
1842 | * we wait for all outstanding requests to complete. | ||
1843 | */ | ||
1844 | synchronize_sched(); | ||
1845 | freeze_array(conf, 0); | ||
1846 | unfreeze_array(conf); | ||
1847 | clear_bit(Unmerged, &rdev->flags); | ||
1848 | } | ||
1849 | md_integrity_add_rdev(rdev, mddev); | 1738 | md_integrity_add_rdev(rdev, mddev); |
1850 | if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev))) | 1739 | if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev))) |
1851 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue); | 1740 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue); |
@@ -1916,7 +1805,7 @@ abort: | |||
1916 | return err; | 1805 | return err; |
1917 | } | 1806 | } |
1918 | 1807 | ||
1919 | static void end_sync_read(struct bio *bio, int error) | 1808 | static void end_sync_read(struct bio *bio) |
1920 | { | 1809 | { |
1921 | struct r10bio *r10_bio = bio->bi_private; | 1810 | struct r10bio *r10_bio = bio->bi_private; |
1922 | struct r10conf *conf = r10_bio->mddev->private; | 1811 | struct r10conf *conf = r10_bio->mddev->private; |
@@ -1928,7 +1817,7 @@ static void end_sync_read(struct bio *bio, int error) | |||
1928 | } else | 1817 | } else |
1929 | d = find_bio_disk(conf, r10_bio, bio, NULL, NULL); | 1818 | d = find_bio_disk(conf, r10_bio, bio, NULL, NULL); |
1930 | 1819 | ||
1931 | if (test_bit(BIO_UPTODATE, &bio->bi_flags)) | 1820 | if (!bio->bi_error) |
1932 | set_bit(R10BIO_Uptodate, &r10_bio->state); | 1821 | set_bit(R10BIO_Uptodate, &r10_bio->state); |
1933 | else | 1822 | else |
1934 | /* The write handler will notice the lack of | 1823 | /* The write handler will notice the lack of |
@@ -1977,9 +1866,8 @@ static void end_sync_request(struct r10bio *r10_bio) | |||
1977 | } | 1866 | } |
1978 | } | 1867 | } |
1979 | 1868 | ||
1980 | static void end_sync_write(struct bio *bio, int error) | 1869 | static void end_sync_write(struct bio *bio) |
1981 | { | 1870 | { |
1982 | int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
1983 | struct r10bio *r10_bio = bio->bi_private; | 1871 | struct r10bio *r10_bio = bio->bi_private; |
1984 | struct mddev *mddev = r10_bio->mddev; | 1872 | struct mddev *mddev = r10_bio->mddev; |
1985 | struct r10conf *conf = mddev->private; | 1873 | struct r10conf *conf = mddev->private; |
@@ -1996,7 +1884,7 @@ static void end_sync_write(struct bio *bio, int error) | |||
1996 | else | 1884 | else |
1997 | rdev = conf->mirrors[d].rdev; | 1885 | rdev = conf->mirrors[d].rdev; |
1998 | 1886 | ||
1999 | if (!uptodate) { | 1887 | if (bio->bi_error) { |
2000 | if (repl) | 1888 | if (repl) |
2001 | md_error(mddev, rdev); | 1889 | md_error(mddev, rdev); |
2002 | else { | 1890 | else { |
@@ -2044,7 +1932,7 @@ static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio) | |||
2044 | 1932 | ||
2045 | /* find the first device with a block */ | 1933 | /* find the first device with a block */ |
2046 | for (i=0; i<conf->copies; i++) | 1934 | for (i=0; i<conf->copies; i++) |
2047 | if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) | 1935 | if (!r10_bio->devs[i].bio->bi_error) |
2048 | break; | 1936 | break; |
2049 | 1937 | ||
2050 | if (i == conf->copies) | 1938 | if (i == conf->copies) |
@@ -2064,7 +1952,7 @@ static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio) | |||
2064 | continue; | 1952 | continue; |
2065 | if (i == first) | 1953 | if (i == first) |
2066 | continue; | 1954 | continue; |
2067 | if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) { | 1955 | if (!r10_bio->devs[i].bio->bi_error) { |
2068 | /* We know that the bi_io_vec layout is the same for | 1956 | /* We know that the bi_io_vec layout is the same for |
2069 | * both 'first' and 'i', so we just compare them. | 1957 | * both 'first' and 'i', so we just compare them. |
2070 | * All vec entries are PAGE_SIZE; | 1958 | * All vec entries are PAGE_SIZE; |
@@ -2394,7 +2282,6 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10 | |||
2394 | d = r10_bio->devs[sl].devnum; | 2282 | d = r10_bio->devs[sl].devnum; |
2395 | rdev = rcu_dereference(conf->mirrors[d].rdev); | 2283 | rdev = rcu_dereference(conf->mirrors[d].rdev); |
2396 | if (rdev && | 2284 | if (rdev && |
2397 | !test_bit(Unmerged, &rdev->flags) && | ||
2398 | test_bit(In_sync, &rdev->flags) && | 2285 | test_bit(In_sync, &rdev->flags) && |
2399 | is_badblock(rdev, r10_bio->devs[sl].addr + sect, s, | 2286 | is_badblock(rdev, r10_bio->devs[sl].addr + sect, s, |
2400 | &first_bad, &bad_sectors) == 0) { | 2287 | &first_bad, &bad_sectors) == 0) { |
@@ -2448,7 +2335,6 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10 | |||
2448 | d = r10_bio->devs[sl].devnum; | 2335 | d = r10_bio->devs[sl].devnum; |
2449 | rdev = rcu_dereference(conf->mirrors[d].rdev); | 2336 | rdev = rcu_dereference(conf->mirrors[d].rdev); |
2450 | if (!rdev || | 2337 | if (!rdev || |
2451 | test_bit(Unmerged, &rdev->flags) || | ||
2452 | !test_bit(In_sync, &rdev->flags)) | 2338 | !test_bit(In_sync, &rdev->flags)) |
2453 | continue; | 2339 | continue; |
2454 | 2340 | ||
@@ -2706,8 +2592,7 @@ static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio) | |||
2706 | rdev = conf->mirrors[dev].rdev; | 2592 | rdev = conf->mirrors[dev].rdev; |
2707 | if (r10_bio->devs[m].bio == NULL) | 2593 | if (r10_bio->devs[m].bio == NULL) |
2708 | continue; | 2594 | continue; |
2709 | if (test_bit(BIO_UPTODATE, | 2595 | if (!r10_bio->devs[m].bio->bi_error) { |
2710 | &r10_bio->devs[m].bio->bi_flags)) { | ||
2711 | rdev_clear_badblocks( | 2596 | rdev_clear_badblocks( |
2712 | rdev, | 2597 | rdev, |
2713 | r10_bio->devs[m].addr, | 2598 | r10_bio->devs[m].addr, |
@@ -2722,8 +2607,8 @@ static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio) | |||
2722 | rdev = conf->mirrors[dev].replacement; | 2607 | rdev = conf->mirrors[dev].replacement; |
2723 | if (r10_bio->devs[m].repl_bio == NULL) | 2608 | if (r10_bio->devs[m].repl_bio == NULL) |
2724 | continue; | 2609 | continue; |
2725 | if (test_bit(BIO_UPTODATE, | 2610 | |
2726 | &r10_bio->devs[m].repl_bio->bi_flags)) { | 2611 | if (!r10_bio->devs[m].repl_bio->bi_error) { |
2727 | rdev_clear_badblocks( | 2612 | rdev_clear_badblocks( |
2728 | rdev, | 2613 | rdev, |
2729 | r10_bio->devs[m].addr, | 2614 | r10_bio->devs[m].addr, |
@@ -2748,8 +2633,7 @@ static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio) | |||
2748 | r10_bio->devs[m].addr, | 2633 | r10_bio->devs[m].addr, |
2749 | r10_bio->sectors, 0); | 2634 | r10_bio->sectors, 0); |
2750 | rdev_dec_pending(rdev, conf->mddev); | 2635 | rdev_dec_pending(rdev, conf->mddev); |
2751 | } else if (bio != NULL && | 2636 | } else if (bio != NULL && bio->bi_error) { |
2752 | !test_bit(BIO_UPTODATE, &bio->bi_flags)) { | ||
2753 | if (!narrow_write_error(r10_bio, m)) { | 2637 | if (!narrow_write_error(r10_bio, m)) { |
2754 | md_error(conf->mddev, rdev); | 2638 | md_error(conf->mddev, rdev); |
2755 | set_bit(R10BIO_Degraded, | 2639 | set_bit(R10BIO_Degraded, |
@@ -3263,7 +3147,7 @@ static sector_t sync_request(struct mddev *mddev, sector_t sector_nr, | |||
3263 | 3147 | ||
3264 | bio = r10_bio->devs[i].bio; | 3148 | bio = r10_bio->devs[i].bio; |
3265 | bio_reset(bio); | 3149 | bio_reset(bio); |
3266 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | 3150 | bio->bi_error = -EIO; |
3267 | if (conf->mirrors[d].rdev == NULL || | 3151 | if (conf->mirrors[d].rdev == NULL || |
3268 | test_bit(Faulty, &conf->mirrors[d].rdev->flags)) | 3152 | test_bit(Faulty, &conf->mirrors[d].rdev->flags)) |
3269 | continue; | 3153 | continue; |
@@ -3300,7 +3184,7 @@ static sector_t sync_request(struct mddev *mddev, sector_t sector_nr, | |||
3300 | /* Need to set up for writing to the replacement */ | 3184 | /* Need to set up for writing to the replacement */ |
3301 | bio = r10_bio->devs[i].repl_bio; | 3185 | bio = r10_bio->devs[i].repl_bio; |
3302 | bio_reset(bio); | 3186 | bio_reset(bio); |
3303 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | 3187 | bio->bi_error = -EIO; |
3304 | 3188 | ||
3305 | sector = r10_bio->devs[i].addr; | 3189 | sector = r10_bio->devs[i].addr; |
3306 | atomic_inc(&conf->mirrors[d].rdev->nr_pending); | 3190 | atomic_inc(&conf->mirrors[d].rdev->nr_pending); |
@@ -3357,7 +3241,7 @@ static sector_t sync_request(struct mddev *mddev, sector_t sector_nr, | |||
3357 | /* remove last page from this bio */ | 3241 | /* remove last page from this bio */ |
3358 | bio2->bi_vcnt--; | 3242 | bio2->bi_vcnt--; |
3359 | bio2->bi_iter.bi_size -= len; | 3243 | bio2->bi_iter.bi_size -= len; |
3360 | __clear_bit(BIO_SEG_VALID, &bio2->bi_flags); | 3244 | bio_clear_flag(bio2, BIO_SEG_VALID); |
3361 | } | 3245 | } |
3362 | goto bio_full; | 3246 | goto bio_full; |
3363 | } | 3247 | } |
@@ -3377,7 +3261,7 @@ static sector_t sync_request(struct mddev *mddev, sector_t sector_nr, | |||
3377 | 3261 | ||
3378 | if (bio->bi_end_io == end_sync_read) { | 3262 | if (bio->bi_end_io == end_sync_read) { |
3379 | md_sync_acct(bio->bi_bdev, nr_sectors); | 3263 | md_sync_acct(bio->bi_bdev, nr_sectors); |
3380 | set_bit(BIO_UPTODATE, &bio->bi_flags); | 3264 | bio->bi_error = 0; |
3381 | generic_make_request(bio); | 3265 | generic_make_request(bio); |
3382 | } | 3266 | } |
3383 | } | 3267 | } |
@@ -3643,8 +3527,6 @@ static int run(struct mddev *mddev) | |||
3643 | disk->rdev = rdev; | 3527 | disk->rdev = rdev; |
3644 | } | 3528 | } |
3645 | q = bdev_get_queue(rdev->bdev); | 3529 | q = bdev_get_queue(rdev->bdev); |
3646 | if (q->merge_bvec_fn) | ||
3647 | mddev->merge_check_needed = 1; | ||
3648 | diff = (rdev->new_data_offset - rdev->data_offset); | 3530 | diff = (rdev->new_data_offset - rdev->data_offset); |
3649 | if (!mddev->reshape_backwards) | 3531 | if (!mddev->reshape_backwards) |
3650 | diff = -diff; | 3532 | diff = -diff; |
@@ -4382,7 +4264,7 @@ read_more: | |||
4382 | read_bio->bi_end_io = end_sync_read; | 4264 | read_bio->bi_end_io = end_sync_read; |
4383 | read_bio->bi_rw = READ; | 4265 | read_bio->bi_rw = READ; |
4384 | read_bio->bi_flags &= (~0UL << BIO_RESET_BITS); | 4266 | read_bio->bi_flags &= (~0UL << BIO_RESET_BITS); |
4385 | __set_bit(BIO_UPTODATE, &read_bio->bi_flags); | 4267 | read_bio->bi_error = 0; |
4386 | read_bio->bi_vcnt = 0; | 4268 | read_bio->bi_vcnt = 0; |
4387 | read_bio->bi_iter.bi_size = 0; | 4269 | read_bio->bi_iter.bi_size = 0; |
4388 | r10_bio->master_bio = read_bio; | 4270 | r10_bio->master_bio = read_bio; |
@@ -4439,7 +4321,7 @@ read_more: | |||
4439 | /* Remove last page from this bio */ | 4321 | /* Remove last page from this bio */ |
4440 | bio2->bi_vcnt--; | 4322 | bio2->bi_vcnt--; |
4441 | bio2->bi_iter.bi_size -= len; | 4323 | bio2->bi_iter.bi_size -= len; |
4442 | __clear_bit(BIO_SEG_VALID, &bio2->bi_flags); | 4324 | bio_clear_flag(bio2, BIO_SEG_VALID); |
4443 | } | 4325 | } |
4444 | goto bio_full; | 4326 | goto bio_full; |
4445 | } | 4327 | } |
@@ -4604,9 +4486,8 @@ static int handle_reshape_read_error(struct mddev *mddev, | |||
4604 | return 0; | 4486 | return 0; |
4605 | } | 4487 | } |
4606 | 4488 | ||
4607 | static void end_reshape_write(struct bio *bio, int error) | 4489 | static void end_reshape_write(struct bio *bio) |
4608 | { | 4490 | { |
4609 | int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
4610 | struct r10bio *r10_bio = bio->bi_private; | 4491 | struct r10bio *r10_bio = bio->bi_private; |
4611 | struct mddev *mddev = r10_bio->mddev; | 4492 | struct mddev *mddev = r10_bio->mddev; |
4612 | struct r10conf *conf = mddev->private; | 4493 | struct r10conf *conf = mddev->private; |
@@ -4623,7 +4504,7 @@ static void end_reshape_write(struct bio *bio, int error) | |||
4623 | rdev = conf->mirrors[d].rdev; | 4504 | rdev = conf->mirrors[d].rdev; |
4624 | } | 4505 | } |
4625 | 4506 | ||
4626 | if (!uptodate) { | 4507 | if (bio->bi_error) { |
4627 | /* FIXME should record badblock */ | 4508 | /* FIXME should record badblock */ |
4628 | md_error(mddev, rdev); | 4509 | md_error(mddev, rdev); |
4629 | } | 4510 | } |
@@ -4700,7 +4581,6 @@ static struct md_personality raid10_personality = | |||
4700 | .start_reshape = raid10_start_reshape, | 4581 | .start_reshape = raid10_start_reshape, |
4701 | .finish_reshape = raid10_finish_reshape, | 4582 | .finish_reshape = raid10_finish_reshape, |
4702 | .congested = raid10_congested, | 4583 | .congested = raid10_congested, |
4703 | .mergeable_bvec = raid10_mergeable_bvec, | ||
4704 | }; | 4584 | }; |
4705 | 4585 | ||
4706 | static int __init raid_init(void) | 4586 | static int __init raid_init(void) |
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index f757023fc458..b29e89cb815b 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c | |||
@@ -233,7 +233,7 @@ static void return_io(struct bio *return_bi) | |||
233 | bi->bi_iter.bi_size = 0; | 233 | bi->bi_iter.bi_size = 0; |
234 | trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), | 234 | trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), |
235 | bi, 0); | 235 | bi, 0); |
236 | bio_endio(bi, 0); | 236 | bio_endio(bi); |
237 | bi = return_bi; | 237 | bi = return_bi; |
238 | } | 238 | } |
239 | } | 239 | } |
@@ -887,9 +887,9 @@ static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) | |||
887 | } | 887 | } |
888 | 888 | ||
889 | static void | 889 | static void |
890 | raid5_end_read_request(struct bio *bi, int error); | 890 | raid5_end_read_request(struct bio *bi); |
891 | static void | 891 | static void |
892 | raid5_end_write_request(struct bio *bi, int error); | 892 | raid5_end_write_request(struct bio *bi); |
893 | 893 | ||
894 | static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) | 894 | static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) |
895 | { | 895 | { |
@@ -2282,12 +2282,11 @@ static void shrink_stripes(struct r5conf *conf) | |||
2282 | conf->slab_cache = NULL; | 2282 | conf->slab_cache = NULL; |
2283 | } | 2283 | } |
2284 | 2284 | ||
2285 | static void raid5_end_read_request(struct bio * bi, int error) | 2285 | static void raid5_end_read_request(struct bio * bi) |
2286 | { | 2286 | { |
2287 | struct stripe_head *sh = bi->bi_private; | 2287 | struct stripe_head *sh = bi->bi_private; |
2288 | struct r5conf *conf = sh->raid_conf; | 2288 | struct r5conf *conf = sh->raid_conf; |
2289 | int disks = sh->disks, i; | 2289 | int disks = sh->disks, i; |
2290 | int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); | ||
2291 | char b[BDEVNAME_SIZE]; | 2290 | char b[BDEVNAME_SIZE]; |
2292 | struct md_rdev *rdev = NULL; | 2291 | struct md_rdev *rdev = NULL; |
2293 | sector_t s; | 2292 | sector_t s; |
@@ -2296,9 +2295,9 @@ static void raid5_end_read_request(struct bio * bi, int error) | |||
2296 | if (bi == &sh->dev[i].req) | 2295 | if (bi == &sh->dev[i].req) |
2297 | break; | 2296 | break; |
2298 | 2297 | ||
2299 | pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", | 2298 | pr_debug("end_read_request %llu/%d, count: %d, error %d.\n", |
2300 | (unsigned long long)sh->sector, i, atomic_read(&sh->count), | 2299 | (unsigned long long)sh->sector, i, atomic_read(&sh->count), |
2301 | uptodate); | 2300 | bi->bi_error); |
2302 | if (i == disks) { | 2301 | if (i == disks) { |
2303 | BUG(); | 2302 | BUG(); |
2304 | return; | 2303 | return; |
@@ -2317,7 +2316,7 @@ static void raid5_end_read_request(struct bio * bi, int error) | |||
2317 | s = sh->sector + rdev->new_data_offset; | 2316 | s = sh->sector + rdev->new_data_offset; |
2318 | else | 2317 | else |
2319 | s = sh->sector + rdev->data_offset; | 2318 | s = sh->sector + rdev->data_offset; |
2320 | if (uptodate) { | 2319 | if (!bi->bi_error) { |
2321 | set_bit(R5_UPTODATE, &sh->dev[i].flags); | 2320 | set_bit(R5_UPTODATE, &sh->dev[i].flags); |
2322 | if (test_bit(R5_ReadError, &sh->dev[i].flags)) { | 2321 | if (test_bit(R5_ReadError, &sh->dev[i].flags)) { |
2323 | /* Note that this cannot happen on a | 2322 | /* Note that this cannot happen on a |
@@ -2405,13 +2404,12 @@ static void raid5_end_read_request(struct bio * bi, int error) | |||
2405 | release_stripe(sh); | 2404 | release_stripe(sh); |
2406 | } | 2405 | } |
2407 | 2406 | ||
2408 | static void raid5_end_write_request(struct bio *bi, int error) | 2407 | static void raid5_end_write_request(struct bio *bi) |
2409 | { | 2408 | { |
2410 | struct stripe_head *sh = bi->bi_private; | 2409 | struct stripe_head *sh = bi->bi_private; |
2411 | struct r5conf *conf = sh->raid_conf; | 2410 | struct r5conf *conf = sh->raid_conf; |
2412 | int disks = sh->disks, i; | 2411 | int disks = sh->disks, i; |
2413 | struct md_rdev *uninitialized_var(rdev); | 2412 | struct md_rdev *uninitialized_var(rdev); |
2414 | int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); | ||
2415 | sector_t first_bad; | 2413 | sector_t first_bad; |
2416 | int bad_sectors; | 2414 | int bad_sectors; |
2417 | int replacement = 0; | 2415 | int replacement = 0; |
@@ -2434,23 +2432,23 @@ static void raid5_end_write_request(struct bio *bi, int error) | |||
2434 | break; | 2432 | break; |
2435 | } | 2433 | } |
2436 | } | 2434 | } |
2437 | pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", | 2435 | pr_debug("end_write_request %llu/%d, count %d, error: %d.\n", |
2438 | (unsigned long long)sh->sector, i, atomic_read(&sh->count), | 2436 | (unsigned long long)sh->sector, i, atomic_read(&sh->count), |
2439 | uptodate); | 2437 | bi->bi_error); |
2440 | if (i == disks) { | 2438 | if (i == disks) { |
2441 | BUG(); | 2439 | BUG(); |
2442 | return; | 2440 | return; |
2443 | } | 2441 | } |
2444 | 2442 | ||
2445 | if (replacement) { | 2443 | if (replacement) { |
2446 | if (!uptodate) | 2444 | if (bi->bi_error) |
2447 | md_error(conf->mddev, rdev); | 2445 | md_error(conf->mddev, rdev); |
2448 | else if (is_badblock(rdev, sh->sector, | 2446 | else if (is_badblock(rdev, sh->sector, |
2449 | STRIPE_SECTORS, | 2447 | STRIPE_SECTORS, |
2450 | &first_bad, &bad_sectors)) | 2448 | &first_bad, &bad_sectors)) |
2451 | set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); | 2449 | set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); |
2452 | } else { | 2450 | } else { |
2453 | if (!uptodate) { | 2451 | if (bi->bi_error) { |
2454 | set_bit(STRIPE_DEGRADED, &sh->state); | 2452 | set_bit(STRIPE_DEGRADED, &sh->state); |
2455 | set_bit(WriteErrorSeen, &rdev->flags); | 2453 | set_bit(WriteErrorSeen, &rdev->flags); |
2456 | set_bit(R5_WriteError, &sh->dev[i].flags); | 2454 | set_bit(R5_WriteError, &sh->dev[i].flags); |
@@ -2471,7 +2469,7 @@ static void raid5_end_write_request(struct bio *bi, int error) | |||
2471 | } | 2469 | } |
2472 | rdev_dec_pending(rdev, conf->mddev); | 2470 | rdev_dec_pending(rdev, conf->mddev); |
2473 | 2471 | ||
2474 | if (sh->batch_head && !uptodate && !replacement) | 2472 | if (sh->batch_head && bi->bi_error && !replacement) |
2475 | set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state); | 2473 | set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state); |
2476 | 2474 | ||
2477 | if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) | 2475 | if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) |
@@ -3112,7 +3110,8 @@ handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, | |||
3112 | while (bi && bi->bi_iter.bi_sector < | 3110 | while (bi && bi->bi_iter.bi_sector < |
3113 | sh->dev[i].sector + STRIPE_SECTORS) { | 3111 | sh->dev[i].sector + STRIPE_SECTORS) { |
3114 | struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); | 3112 | struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); |
3115 | clear_bit(BIO_UPTODATE, &bi->bi_flags); | 3113 | |
3114 | bi->bi_error = -EIO; | ||
3116 | if (!raid5_dec_bi_active_stripes(bi)) { | 3115 | if (!raid5_dec_bi_active_stripes(bi)) { |
3117 | md_write_end(conf->mddev); | 3116 | md_write_end(conf->mddev); |
3118 | bi->bi_next = *return_bi; | 3117 | bi->bi_next = *return_bi; |
@@ -3136,7 +3135,8 @@ handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, | |||
3136 | while (bi && bi->bi_iter.bi_sector < | 3135 | while (bi && bi->bi_iter.bi_sector < |
3137 | sh->dev[i].sector + STRIPE_SECTORS) { | 3136 | sh->dev[i].sector + STRIPE_SECTORS) { |
3138 | struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); | 3137 | struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); |
3139 | clear_bit(BIO_UPTODATE, &bi->bi_flags); | 3138 | |
3139 | bi->bi_error = -EIO; | ||
3140 | if (!raid5_dec_bi_active_stripes(bi)) { | 3140 | if (!raid5_dec_bi_active_stripes(bi)) { |
3141 | md_write_end(conf->mddev); | 3141 | md_write_end(conf->mddev); |
3142 | bi->bi_next = *return_bi; | 3142 | bi->bi_next = *return_bi; |
@@ -3161,7 +3161,8 @@ handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, | |||
3161 | sh->dev[i].sector + STRIPE_SECTORS) { | 3161 | sh->dev[i].sector + STRIPE_SECTORS) { |
3162 | struct bio *nextbi = | 3162 | struct bio *nextbi = |
3163 | r5_next_bio(bi, sh->dev[i].sector); | 3163 | r5_next_bio(bi, sh->dev[i].sector); |
3164 | clear_bit(BIO_UPTODATE, &bi->bi_flags); | 3164 | |
3165 | bi->bi_error = -EIO; | ||
3165 | if (!raid5_dec_bi_active_stripes(bi)) { | 3166 | if (!raid5_dec_bi_active_stripes(bi)) { |
3166 | bi->bi_next = *return_bi; | 3167 | bi->bi_next = *return_bi; |
3167 | *return_bi = bi; | 3168 | *return_bi = bi; |
@@ -4669,35 +4670,6 @@ static int raid5_congested(struct mddev *mddev, int bits) | |||
4669 | return 0; | 4670 | return 0; |
4670 | } | 4671 | } |
4671 | 4672 | ||
4672 | /* We want read requests to align with chunks where possible, | ||
4673 | * but write requests don't need to. | ||
4674 | */ | ||
4675 | static int raid5_mergeable_bvec(struct mddev *mddev, | ||
4676 | struct bvec_merge_data *bvm, | ||
4677 | struct bio_vec *biovec) | ||
4678 | { | ||
4679 | sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); | ||
4680 | int max; | ||
4681 | unsigned int chunk_sectors = mddev->chunk_sectors; | ||
4682 | unsigned int bio_sectors = bvm->bi_size >> 9; | ||
4683 | |||
4684 | /* | ||
4685 | * always allow writes to be mergeable, read as well if array | ||
4686 | * is degraded as we'll go through stripe cache anyway. | ||
4687 | */ | ||
4688 | if ((bvm->bi_rw & 1) == WRITE || mddev->degraded) | ||
4689 | return biovec->bv_len; | ||
4690 | |||
4691 | if (mddev->new_chunk_sectors < mddev->chunk_sectors) | ||
4692 | chunk_sectors = mddev->new_chunk_sectors; | ||
4693 | max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; | ||
4694 | if (max < 0) max = 0; | ||
4695 | if (max <= biovec->bv_len && bio_sectors == 0) | ||
4696 | return biovec->bv_len; | ||
4697 | else | ||
4698 | return max; | ||
4699 | } | ||
4700 | |||
4701 | static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) | 4673 | static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) |
4702 | { | 4674 | { |
4703 | sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev); | 4675 | sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev); |
@@ -4756,13 +4728,13 @@ static struct bio *remove_bio_from_retry(struct r5conf *conf) | |||
4756 | * first). | 4728 | * first). |
4757 | * If the read failed.. | 4729 | * If the read failed.. |
4758 | */ | 4730 | */ |
4759 | static void raid5_align_endio(struct bio *bi, int error) | 4731 | static void raid5_align_endio(struct bio *bi) |
4760 | { | 4732 | { |
4761 | struct bio* raid_bi = bi->bi_private; | 4733 | struct bio* raid_bi = bi->bi_private; |
4762 | struct mddev *mddev; | 4734 | struct mddev *mddev; |
4763 | struct r5conf *conf; | 4735 | struct r5conf *conf; |
4764 | int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); | ||
4765 | struct md_rdev *rdev; | 4736 | struct md_rdev *rdev; |
4737 | int error = bi->bi_error; | ||
4766 | 4738 | ||
4767 | bio_put(bi); | 4739 | bio_put(bi); |
4768 | 4740 | ||
@@ -4773,10 +4745,10 @@ static void raid5_align_endio(struct bio *bi, int error) | |||
4773 | 4745 | ||
4774 | rdev_dec_pending(rdev, conf->mddev); | 4746 | rdev_dec_pending(rdev, conf->mddev); |
4775 | 4747 | ||
4776 | if (!error && uptodate) { | 4748 | if (!error) { |
4777 | trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev), | 4749 | trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev), |
4778 | raid_bi, 0); | 4750 | raid_bi, 0); |
4779 | bio_endio(raid_bi, 0); | 4751 | bio_endio(raid_bi); |
4780 | if (atomic_dec_and_test(&conf->active_aligned_reads)) | 4752 | if (atomic_dec_and_test(&conf->active_aligned_reads)) |
4781 | wake_up(&conf->wait_for_quiescent); | 4753 | wake_up(&conf->wait_for_quiescent); |
4782 | return; | 4754 | return; |
@@ -4787,26 +4759,7 @@ static void raid5_align_endio(struct bio *bi, int error) | |||
4787 | add_bio_to_retry(raid_bi, conf); | 4759 | add_bio_to_retry(raid_bi, conf); |
4788 | } | 4760 | } |
4789 | 4761 | ||
4790 | static int bio_fits_rdev(struct bio *bi) | 4762 | static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio) |
4791 | { | ||
4792 | struct request_queue *q = bdev_get_queue(bi->bi_bdev); | ||
4793 | |||
4794 | if (bio_sectors(bi) > queue_max_sectors(q)) | ||
4795 | return 0; | ||
4796 | blk_recount_segments(q, bi); | ||
4797 | if (bi->bi_phys_segments > queue_max_segments(q)) | ||
4798 | return 0; | ||
4799 | |||
4800 | if (q->merge_bvec_fn) | ||
4801 | /* it's too hard to apply the merge_bvec_fn at this stage, | ||
4802 | * just just give up | ||
4803 | */ | ||
4804 | return 0; | ||
4805 | |||
4806 | return 1; | ||
4807 | } | ||
4808 | |||
4809 | static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio) | ||
4810 | { | 4763 | { |
4811 | struct r5conf *conf = mddev->private; | 4764 | struct r5conf *conf = mddev->private; |
4812 | int dd_idx; | 4765 | int dd_idx; |
@@ -4815,7 +4768,7 @@ static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio) | |||
4815 | sector_t end_sector; | 4768 | sector_t end_sector; |
4816 | 4769 | ||
4817 | if (!in_chunk_boundary(mddev, raid_bio)) { | 4770 | if (!in_chunk_boundary(mddev, raid_bio)) { |
4818 | pr_debug("chunk_aligned_read : non aligned\n"); | 4771 | pr_debug("%s: non aligned\n", __func__); |
4819 | return 0; | 4772 | return 0; |
4820 | } | 4773 | } |
4821 | /* | 4774 | /* |
@@ -4857,13 +4810,11 @@ static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio) | |||
4857 | rcu_read_unlock(); | 4810 | rcu_read_unlock(); |
4858 | raid_bio->bi_next = (void*)rdev; | 4811 | raid_bio->bi_next = (void*)rdev; |
4859 | align_bi->bi_bdev = rdev->bdev; | 4812 | align_bi->bi_bdev = rdev->bdev; |
4860 | __clear_bit(BIO_SEG_VALID, &align_bi->bi_flags); | 4813 | bio_clear_flag(align_bi, BIO_SEG_VALID); |
4861 | 4814 | ||
4862 | if (!bio_fits_rdev(align_bi) || | 4815 | if (is_badblock(rdev, align_bi->bi_iter.bi_sector, |
4863 | is_badblock(rdev, align_bi->bi_iter.bi_sector, | ||
4864 | bio_sectors(align_bi), | 4816 | bio_sectors(align_bi), |
4865 | &first_bad, &bad_sectors)) { | 4817 | &first_bad, &bad_sectors)) { |
4866 | /* too big in some way, or has a known bad block */ | ||
4867 | bio_put(align_bi); | 4818 | bio_put(align_bi); |
4868 | rdev_dec_pending(rdev, mddev); | 4819 | rdev_dec_pending(rdev, mddev); |
4869 | return 0; | 4820 | return 0; |
@@ -4892,6 +4843,31 @@ static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio) | |||
4892 | } | 4843 | } |
4893 | } | 4844 | } |
4894 | 4845 | ||
4846 | static struct bio *chunk_aligned_read(struct mddev *mddev, struct bio *raid_bio) | ||
4847 | { | ||
4848 | struct bio *split; | ||
4849 | |||
4850 | do { | ||
4851 | sector_t sector = raid_bio->bi_iter.bi_sector; | ||
4852 | unsigned chunk_sects = mddev->chunk_sectors; | ||
4853 | unsigned sectors = chunk_sects - (sector & (chunk_sects-1)); | ||
4854 | |||
4855 | if (sectors < bio_sectors(raid_bio)) { | ||
4856 | split = bio_split(raid_bio, sectors, GFP_NOIO, fs_bio_set); | ||
4857 | bio_chain(split, raid_bio); | ||
4858 | } else | ||
4859 | split = raid_bio; | ||
4860 | |||
4861 | if (!raid5_read_one_chunk(mddev, split)) { | ||
4862 | if (split != raid_bio) | ||
4863 | generic_make_request(raid_bio); | ||
4864 | return split; | ||
4865 | } | ||
4866 | } while (split != raid_bio); | ||
4867 | |||
4868 | return NULL; | ||
4869 | } | ||
4870 | |||
4895 | /* __get_priority_stripe - get the next stripe to process | 4871 | /* __get_priority_stripe - get the next stripe to process |
4896 | * | 4872 | * |
4897 | * Full stripe writes are allowed to pass preread active stripes up until | 4873 | * Full stripe writes are allowed to pass preread active stripes up until |
@@ -5140,7 +5116,7 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) | |||
5140 | remaining = raid5_dec_bi_active_stripes(bi); | 5116 | remaining = raid5_dec_bi_active_stripes(bi); |
5141 | if (remaining == 0) { | 5117 | if (remaining == 0) { |
5142 | md_write_end(mddev); | 5118 | md_write_end(mddev); |
5143 | bio_endio(bi, 0); | 5119 | bio_endio(bi); |
5144 | } | 5120 | } |
5145 | } | 5121 | } |
5146 | 5122 | ||
@@ -5169,9 +5145,11 @@ static void make_request(struct mddev *mddev, struct bio * bi) | |||
5169 | * data on failed drives. | 5145 | * data on failed drives. |
5170 | */ | 5146 | */ |
5171 | if (rw == READ && mddev->degraded == 0 && | 5147 | if (rw == READ && mddev->degraded == 0 && |
5172 | mddev->reshape_position == MaxSector && | 5148 | mddev->reshape_position == MaxSector) { |
5173 | chunk_aligned_read(mddev,bi)) | 5149 | bi = chunk_aligned_read(mddev, bi); |
5174 | return; | 5150 | if (!bi) |
5151 | return; | ||
5152 | } | ||
5175 | 5153 | ||
5176 | if (unlikely(bi->bi_rw & REQ_DISCARD)) { | 5154 | if (unlikely(bi->bi_rw & REQ_DISCARD)) { |
5177 | make_discard_request(mddev, bi); | 5155 | make_discard_request(mddev, bi); |
@@ -5304,7 +5282,7 @@ static void make_request(struct mddev *mddev, struct bio * bi) | |||
5304 | release_stripe_plug(mddev, sh); | 5282 | release_stripe_plug(mddev, sh); |
5305 | } else { | 5283 | } else { |
5306 | /* cannot get stripe for read-ahead, just give-up */ | 5284 | /* cannot get stripe for read-ahead, just give-up */ |
5307 | clear_bit(BIO_UPTODATE, &bi->bi_flags); | 5285 | bi->bi_error = -EIO; |
5308 | break; | 5286 | break; |
5309 | } | 5287 | } |
5310 | } | 5288 | } |
@@ -5318,7 +5296,7 @@ static void make_request(struct mddev *mddev, struct bio * bi) | |||
5318 | 5296 | ||
5319 | trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), | 5297 | trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), |
5320 | bi, 0); | 5298 | bi, 0); |
5321 | bio_endio(bi, 0); | 5299 | bio_endio(bi); |
5322 | } | 5300 | } |
5323 | } | 5301 | } |
5324 | 5302 | ||
@@ -5714,7 +5692,7 @@ static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio) | |||
5714 | if (remaining == 0) { | 5692 | if (remaining == 0) { |
5715 | trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev), | 5693 | trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev), |
5716 | raid_bio, 0); | 5694 | raid_bio, 0); |
5717 | bio_endio(raid_bio, 0); | 5695 | bio_endio(raid_bio); |
5718 | } | 5696 | } |
5719 | if (atomic_dec_and_test(&conf->active_aligned_reads)) | 5697 | if (atomic_dec_and_test(&conf->active_aligned_reads)) |
5720 | wake_up(&conf->wait_for_quiescent); | 5698 | wake_up(&conf->wait_for_quiescent); |
@@ -7779,7 +7757,6 @@ static struct md_personality raid6_personality = | |||
7779 | .quiesce = raid5_quiesce, | 7757 | .quiesce = raid5_quiesce, |
7780 | .takeover = raid6_takeover, | 7758 | .takeover = raid6_takeover, |
7781 | .congested = raid5_congested, | 7759 | .congested = raid5_congested, |
7782 | .mergeable_bvec = raid5_mergeable_bvec, | ||
7783 | }; | 7760 | }; |
7784 | static struct md_personality raid5_personality = | 7761 | static struct md_personality raid5_personality = |
7785 | { | 7762 | { |
@@ -7803,7 +7780,6 @@ static struct md_personality raid5_personality = | |||
7803 | .quiesce = raid5_quiesce, | 7780 | .quiesce = raid5_quiesce, |
7804 | .takeover = raid5_takeover, | 7781 | .takeover = raid5_takeover, |
7805 | .congested = raid5_congested, | 7782 | .congested = raid5_congested, |
7806 | .mergeable_bvec = raid5_mergeable_bvec, | ||
7807 | }; | 7783 | }; |
7808 | 7784 | ||
7809 | static struct md_personality raid4_personality = | 7785 | static struct md_personality raid4_personality = |
@@ -7828,7 +7804,6 @@ static struct md_personality raid4_personality = | |||
7828 | .quiesce = raid5_quiesce, | 7804 | .quiesce = raid5_quiesce, |
7829 | .takeover = raid4_takeover, | 7805 | .takeover = raid4_takeover, |
7830 | .congested = raid5_congested, | 7806 | .congested = raid5_congested, |
7831 | .mergeable_bvec = raid5_mergeable_bvec, | ||
7832 | }; | 7807 | }; |
7833 | 7808 | ||
7834 | static int __init raid5_init(void) | 7809 | static int __init raid5_init(void) |
diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c index b5a2b145d89f..5daf302835b1 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/card/queue.c | |||
@@ -165,7 +165,7 @@ static void mmc_queue_setup_discard(struct request_queue *q, | |||
165 | return; | 165 | return; |
166 | 166 | ||
167 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); | 167 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); |
168 | q->limits.max_discard_sectors = max_discard; | 168 | blk_queue_max_discard_sectors(q, max_discard); |
169 | if (card->erased_byte == 0 && !mmc_can_discard(card)) | 169 | if (card->erased_byte == 0 && !mmc_can_discard(card)) |
170 | q->limits.discard_zeroes_data = 1; | 170 | q->limits.discard_zeroes_data = 1; |
171 | q->limits.discard_granularity = card->pref_erase << 9; | 171 | q->limits.discard_granularity = card->pref_erase << 9; |
diff --git a/drivers/mtd/mtd_blkdevs.c b/drivers/mtd/mtd_blkdevs.c index 88304751eb8a..44dc965a2f7c 100644 --- a/drivers/mtd/mtd_blkdevs.c +++ b/drivers/mtd/mtd_blkdevs.c | |||
@@ -419,7 +419,7 @@ int add_mtd_blktrans_dev(struct mtd_blktrans_dev *new) | |||
419 | 419 | ||
420 | if (tr->discard) { | 420 | if (tr->discard) { |
421 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, new->rq); | 421 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, new->rq); |
422 | new->rq->limits.max_discard_sectors = UINT_MAX; | 422 | blk_queue_max_discard_sectors(new->rq, UINT_MAX); |
423 | } | 423 | } |
424 | 424 | ||
425 | gd->queue = new->rq; | 425 | gd->queue = new->rq; |
diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c index 4f97b248c236..0df77cb07df6 100644 --- a/drivers/nvdimm/blk.c +++ b/drivers/nvdimm/blk.c | |||
@@ -180,7 +180,7 @@ static void nd_blk_make_request(struct request_queue *q, struct bio *bio) | |||
180 | * another kernel subsystem, and we just pass it through. | 180 | * another kernel subsystem, and we just pass it through. |
181 | */ | 181 | */ |
182 | if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { | 182 | if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { |
183 | err = -EIO; | 183 | bio->bi_error = -EIO; |
184 | goto out; | 184 | goto out; |
185 | } | 185 | } |
186 | 186 | ||
@@ -199,6 +199,7 @@ static void nd_blk_make_request(struct request_queue *q, struct bio *bio) | |||
199 | "io error in %s sector %lld, len %d,\n", | 199 | "io error in %s sector %lld, len %d,\n", |
200 | (rw == READ) ? "READ" : "WRITE", | 200 | (rw == READ) ? "READ" : "WRITE", |
201 | (unsigned long long) iter.bi_sector, len); | 201 | (unsigned long long) iter.bi_sector, len); |
202 | bio->bi_error = err; | ||
202 | break; | 203 | break; |
203 | } | 204 | } |
204 | } | 205 | } |
@@ -206,7 +207,7 @@ static void nd_blk_make_request(struct request_queue *q, struct bio *bio) | |||
206 | nd_iostat_end(bio, start); | 207 | nd_iostat_end(bio, start); |
207 | 208 | ||
208 | out: | 209 | out: |
209 | bio_endio(bio, err); | 210 | bio_endio(bio); |
210 | } | 211 | } |
211 | 212 | ||
212 | static int nd_blk_rw_bytes(struct nd_namespace_common *ndns, | 213 | static int nd_blk_rw_bytes(struct nd_namespace_common *ndns, |
diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 411c7b2bb37a..341202ed32b4 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c | |||
@@ -1189,7 +1189,7 @@ static void btt_make_request(struct request_queue *q, struct bio *bio) | |||
1189 | * another kernel subsystem, and we just pass it through. | 1189 | * another kernel subsystem, and we just pass it through. |
1190 | */ | 1190 | */ |
1191 | if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { | 1191 | if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { |
1192 | err = -EIO; | 1192 | bio->bi_error = -EIO; |
1193 | goto out; | 1193 | goto out; |
1194 | } | 1194 | } |
1195 | 1195 | ||
@@ -1211,6 +1211,7 @@ static void btt_make_request(struct request_queue *q, struct bio *bio) | |||
1211 | "io error in %s sector %lld, len %d,\n", | 1211 | "io error in %s sector %lld, len %d,\n", |
1212 | (rw == READ) ? "READ" : "WRITE", | 1212 | (rw == READ) ? "READ" : "WRITE", |
1213 | (unsigned long long) iter.bi_sector, len); | 1213 | (unsigned long long) iter.bi_sector, len); |
1214 | bio->bi_error = err; | ||
1214 | break; | 1215 | break; |
1215 | } | 1216 | } |
1216 | } | 1217 | } |
@@ -1218,7 +1219,7 @@ static void btt_make_request(struct request_queue *q, struct bio *bio) | |||
1218 | nd_iostat_end(bio, start); | 1219 | nd_iostat_end(bio, start); |
1219 | 1220 | ||
1220 | out: | 1221 | out: |
1221 | bio_endio(bio, err); | 1222 | bio_endio(bio); |
1222 | } | 1223 | } |
1223 | 1224 | ||
1224 | static int btt_rw_page(struct block_device *bdev, sector_t sector, | 1225 | static int btt_rw_page(struct block_device *bdev, sector_t sector, |
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index ade9eb917a4d..4c079d5cb539 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c | |||
@@ -77,7 +77,7 @@ static void pmem_make_request(struct request_queue *q, struct bio *bio) | |||
77 | if (bio_data_dir(bio)) | 77 | if (bio_data_dir(bio)) |
78 | wmb_pmem(); | 78 | wmb_pmem(); |
79 | 79 | ||
80 | bio_endio(bio, 0); | 80 | bio_endio(bio); |
81 | } | 81 | } |
82 | 82 | ||
83 | static int pmem_rw_page(struct block_device *bdev, sector_t sector, | 83 | static int pmem_rw_page(struct block_device *bdev, sector_t sector, |
diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index dff3fcb69a78..2b744fbba68e 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c | |||
@@ -826,6 +826,8 @@ dcssblk_make_request(struct request_queue *q, struct bio *bio) | |||
826 | unsigned long source_addr; | 826 | unsigned long source_addr; |
827 | unsigned long bytes_done; | 827 | unsigned long bytes_done; |
828 | 828 | ||
829 | blk_queue_split(q, &bio, q->bio_split); | ||
830 | |||
829 | bytes_done = 0; | 831 | bytes_done = 0; |
830 | dev_info = bio->bi_bdev->bd_disk->private_data; | 832 | dev_info = bio->bi_bdev->bd_disk->private_data; |
831 | if (dev_info == NULL) | 833 | if (dev_info == NULL) |
@@ -871,7 +873,7 @@ dcssblk_make_request(struct request_queue *q, struct bio *bio) | |||
871 | } | 873 | } |
872 | bytes_done += bvec.bv_len; | 874 | bytes_done += bvec.bv_len; |
873 | } | 875 | } |
874 | bio_endio(bio, 0); | 876 | bio_endio(bio); |
875 | return; | 877 | return; |
876 | fail: | 878 | fail: |
877 | bio_io_error(bio); | 879 | bio_io_error(bio); |
diff --git a/drivers/s390/block/xpram.c b/drivers/s390/block/xpram.c index 7d4e9397ac31..02871f1db562 100644 --- a/drivers/s390/block/xpram.c +++ b/drivers/s390/block/xpram.c | |||
@@ -190,6 +190,8 @@ static void xpram_make_request(struct request_queue *q, struct bio *bio) | |||
190 | unsigned long page_addr; | 190 | unsigned long page_addr; |
191 | unsigned long bytes; | 191 | unsigned long bytes; |
192 | 192 | ||
193 | blk_queue_split(q, &bio, q->bio_split); | ||
194 | |||
193 | if ((bio->bi_iter.bi_sector & 7) != 0 || | 195 | if ((bio->bi_iter.bi_sector & 7) != 0 || |
194 | (bio->bi_iter.bi_size & 4095) != 0) | 196 | (bio->bi_iter.bi_size & 4095) != 0) |
195 | /* Request is not page-aligned. */ | 197 | /* Request is not page-aligned. */ |
@@ -220,8 +222,7 @@ static void xpram_make_request(struct request_queue *q, struct bio *bio) | |||
220 | index++; | 222 | index++; |
221 | } | 223 | } |
222 | } | 224 | } |
223 | set_bit(BIO_UPTODATE, &bio->bi_flags); | 225 | bio_endio(bio); |
224 | bio_endio(bio, 0); | ||
225 | return; | 226 | return; |
226 | fail: | 227 | fail: |
227 | bio_io_error(bio); | 228 | bio_io_error(bio); |
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index a20da8c25b4f..3f370228bf31 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c | |||
@@ -647,7 +647,7 @@ static void sd_config_discard(struct scsi_disk *sdkp, unsigned int mode) | |||
647 | switch (mode) { | 647 | switch (mode) { |
648 | 648 | ||
649 | case SD_LBP_DISABLE: | 649 | case SD_LBP_DISABLE: |
650 | q->limits.max_discard_sectors = 0; | 650 | blk_queue_max_discard_sectors(q, 0); |
651 | queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); | 651 | queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); |
652 | return; | 652 | return; |
653 | 653 | ||
@@ -675,7 +675,7 @@ static void sd_config_discard(struct scsi_disk *sdkp, unsigned int mode) | |||
675 | break; | 675 | break; |
676 | } | 676 | } |
677 | 677 | ||
678 | q->limits.max_discard_sectors = max_blocks * (logical_block_size >> 9); | 678 | blk_queue_max_discard_sectors(q, max_blocks * (logical_block_size >> 9)); |
679 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); | 679 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); |
680 | } | 680 | } |
681 | 681 | ||
diff --git a/drivers/staging/lustre/lustre/llite/lloop.c b/drivers/staging/lustre/lustre/llite/lloop.c index 06f5e51ecd9e..5f0d80cc9718 100644 --- a/drivers/staging/lustre/lustre/llite/lloop.c +++ b/drivers/staging/lustre/lustre/llite/lloop.c | |||
@@ -340,6 +340,8 @@ static void loop_make_request(struct request_queue *q, struct bio *old_bio) | |||
340 | int rw = bio_rw(old_bio); | 340 | int rw = bio_rw(old_bio); |
341 | int inactive; | 341 | int inactive; |
342 | 342 | ||
343 | blk_queue_split(q, &old_bio, q->bio_split); | ||
344 | |||
343 | if (!lo) | 345 | if (!lo) |
344 | goto err; | 346 | goto err; |
345 | 347 | ||
@@ -376,7 +378,8 @@ static inline void loop_handle_bio(struct lloop_device *lo, struct bio *bio) | |||
376 | while (bio) { | 378 | while (bio) { |
377 | struct bio *tmp = bio->bi_next; | 379 | struct bio *tmp = bio->bi_next; |
378 | bio->bi_next = NULL; | 380 | bio->bi_next = NULL; |
379 | bio_endio(bio, ret); | 381 | bio->bi_error = ret; |
382 | bio_endio(bio); | ||
380 | bio = tmp; | 383 | bio = tmp; |
381 | } | 384 | } |
382 | } | 385 | } |
diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c index 6d88d24e6cce..5a9982f5d5d6 100644 --- a/drivers/target/target_core_iblock.c +++ b/drivers/target/target_core_iblock.c | |||
@@ -306,20 +306,13 @@ static void iblock_complete_cmd(struct se_cmd *cmd) | |||
306 | kfree(ibr); | 306 | kfree(ibr); |
307 | } | 307 | } |
308 | 308 | ||
309 | static void iblock_bio_done(struct bio *bio, int err) | 309 | static void iblock_bio_done(struct bio *bio) |
310 | { | 310 | { |
311 | struct se_cmd *cmd = bio->bi_private; | 311 | struct se_cmd *cmd = bio->bi_private; |
312 | struct iblock_req *ibr = cmd->priv; | 312 | struct iblock_req *ibr = cmd->priv; |
313 | 313 | ||
314 | /* | 314 | if (bio->bi_error) { |
315 | * Set -EIO if !BIO_UPTODATE and the passed is still err=0 | 315 | pr_err("bio error: %p, err: %d\n", bio, bio->bi_error); |
316 | */ | ||
317 | if (!test_bit(BIO_UPTODATE, &bio->bi_flags) && !err) | ||
318 | err = -EIO; | ||
319 | |||
320 | if (err != 0) { | ||
321 | pr_err("test_bit(BIO_UPTODATE) failed for bio: %p," | ||
322 | " err: %d\n", bio, err); | ||
323 | /* | 316 | /* |
324 | * Bump the ib_bio_err_cnt and release bio. | 317 | * Bump the ib_bio_err_cnt and release bio. |
325 | */ | 318 | */ |
@@ -370,15 +363,15 @@ static void iblock_submit_bios(struct bio_list *list, int rw) | |||
370 | blk_finish_plug(&plug); | 363 | blk_finish_plug(&plug); |
371 | } | 364 | } |
372 | 365 | ||
373 | static void iblock_end_io_flush(struct bio *bio, int err) | 366 | static void iblock_end_io_flush(struct bio *bio) |
374 | { | 367 | { |
375 | struct se_cmd *cmd = bio->bi_private; | 368 | struct se_cmd *cmd = bio->bi_private; |
376 | 369 | ||
377 | if (err) | 370 | if (bio->bi_error) |
378 | pr_err("IBLOCK: cache flush failed: %d\n", err); | 371 | pr_err("IBLOCK: cache flush failed: %d\n", bio->bi_error); |
379 | 372 | ||
380 | if (cmd) { | 373 | if (cmd) { |
381 | if (err) | 374 | if (bio->bi_error) |
382 | target_complete_cmd(cmd, SAM_STAT_CHECK_CONDITION); | 375 | target_complete_cmd(cmd, SAM_STAT_CHECK_CONDITION); |
383 | else | 376 | else |
384 | target_complete_cmd(cmd, SAM_STAT_GOOD); | 377 | target_complete_cmd(cmd, SAM_STAT_GOOD); |
diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c index 08e9084ee615..de18790eb21c 100644 --- a/drivers/target/target_core_pscsi.c +++ b/drivers/target/target_core_pscsi.c | |||
@@ -852,7 +852,7 @@ static ssize_t pscsi_show_configfs_dev_params(struct se_device *dev, char *b) | |||
852 | return bl; | 852 | return bl; |
853 | } | 853 | } |
854 | 854 | ||
855 | static void pscsi_bi_endio(struct bio *bio, int error) | 855 | static void pscsi_bi_endio(struct bio *bio) |
856 | { | 856 | { |
857 | bio_put(bio); | 857 | bio_put(bio); |
858 | } | 858 | } |
@@ -973,7 +973,7 @@ fail: | |||
973 | while (*hbio) { | 973 | while (*hbio) { |
974 | bio = *hbio; | 974 | bio = *hbio; |
975 | *hbio = (*hbio)->bi_next; | 975 | *hbio = (*hbio)->bi_next; |
976 | bio_endio(bio, 0); /* XXX: should be error */ | 976 | bio_endio(bio); |
977 | } | 977 | } |
978 | return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; | 978 | return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; |
979 | } | 979 | } |
@@ -1061,7 +1061,7 @@ fail_free_bio: | |||
1061 | while (hbio) { | 1061 | while (hbio) { |
1062 | struct bio *bio = hbio; | 1062 | struct bio *bio = hbio; |
1063 | hbio = hbio->bi_next; | 1063 | hbio = hbio->bi_next; |
1064 | bio_endio(bio, 0); /* XXX: should be error */ | 1064 | bio_endio(bio); |
1065 | } | 1065 | } |
1066 | ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; | 1066 | ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; |
1067 | fail: | 1067 | fail: |
diff --git a/fs/btrfs/check-integrity.c b/fs/btrfs/check-integrity.c index ce7dec88f4b8..541fbfaed276 100644 --- a/fs/btrfs/check-integrity.c +++ b/fs/btrfs/check-integrity.c | |||
@@ -343,7 +343,7 @@ static int btrfsic_process_written_superblock( | |||
343 | struct btrfsic_state *state, | 343 | struct btrfsic_state *state, |
344 | struct btrfsic_block *const block, | 344 | struct btrfsic_block *const block, |
345 | struct btrfs_super_block *const super_hdr); | 345 | struct btrfs_super_block *const super_hdr); |
346 | static void btrfsic_bio_end_io(struct bio *bp, int bio_error_status); | 346 | static void btrfsic_bio_end_io(struct bio *bp); |
347 | static void btrfsic_bh_end_io(struct buffer_head *bh, int uptodate); | 347 | static void btrfsic_bh_end_io(struct buffer_head *bh, int uptodate); |
348 | static int btrfsic_is_block_ref_by_superblock(const struct btrfsic_state *state, | 348 | static int btrfsic_is_block_ref_by_superblock(const struct btrfsic_state *state, |
349 | const struct btrfsic_block *block, | 349 | const struct btrfsic_block *block, |
@@ -2207,7 +2207,7 @@ continue_loop: | |||
2207 | goto again; | 2207 | goto again; |
2208 | } | 2208 | } |
2209 | 2209 | ||
2210 | static void btrfsic_bio_end_io(struct bio *bp, int bio_error_status) | 2210 | static void btrfsic_bio_end_io(struct bio *bp) |
2211 | { | 2211 | { |
2212 | struct btrfsic_block *block = (struct btrfsic_block *)bp->bi_private; | 2212 | struct btrfsic_block *block = (struct btrfsic_block *)bp->bi_private; |
2213 | int iodone_w_error; | 2213 | int iodone_w_error; |
@@ -2215,7 +2215,7 @@ static void btrfsic_bio_end_io(struct bio *bp, int bio_error_status) | |||
2215 | /* mutex is not held! This is not save if IO is not yet completed | 2215 | /* mutex is not held! This is not save if IO is not yet completed |
2216 | * on umount */ | 2216 | * on umount */ |
2217 | iodone_w_error = 0; | 2217 | iodone_w_error = 0; |
2218 | if (bio_error_status) | 2218 | if (bp->bi_error) |
2219 | iodone_w_error = 1; | 2219 | iodone_w_error = 1; |
2220 | 2220 | ||
2221 | BUG_ON(NULL == block); | 2221 | BUG_ON(NULL == block); |
@@ -2230,7 +2230,7 @@ static void btrfsic_bio_end_io(struct bio *bp, int bio_error_status) | |||
2230 | BTRFSIC_PRINT_MASK_END_IO_BIO_BH)) | 2230 | BTRFSIC_PRINT_MASK_END_IO_BIO_BH)) |
2231 | printk(KERN_INFO | 2231 | printk(KERN_INFO |
2232 | "bio_end_io(err=%d) for %c @%llu (%s/%llu/%d)\n", | 2232 | "bio_end_io(err=%d) for %c @%llu (%s/%llu/%d)\n", |
2233 | bio_error_status, | 2233 | bp->bi_error, |
2234 | btrfsic_get_block_type(dev_state->state, block), | 2234 | btrfsic_get_block_type(dev_state->state, block), |
2235 | block->logical_bytenr, dev_state->name, | 2235 | block->logical_bytenr, dev_state->name, |
2236 | block->dev_bytenr, block->mirror_num); | 2236 | block->dev_bytenr, block->mirror_num); |
@@ -2252,7 +2252,7 @@ static void btrfsic_bio_end_io(struct bio *bp, int bio_error_status) | |||
2252 | block = next_block; | 2252 | block = next_block; |
2253 | } while (NULL != block); | 2253 | } while (NULL != block); |
2254 | 2254 | ||
2255 | bp->bi_end_io(bp, bio_error_status); | 2255 | bp->bi_end_io(bp); |
2256 | } | 2256 | } |
2257 | 2257 | ||
2258 | static void btrfsic_bh_end_io(struct buffer_head *bh, int uptodate) | 2258 | static void btrfsic_bh_end_io(struct buffer_head *bh, int uptodate) |
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index ce62324c78e7..57ee8ca29b06 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c | |||
@@ -97,10 +97,7 @@ static inline int compressed_bio_size(struct btrfs_root *root, | |||
97 | static struct bio *compressed_bio_alloc(struct block_device *bdev, | 97 | static struct bio *compressed_bio_alloc(struct block_device *bdev, |
98 | u64 first_byte, gfp_t gfp_flags) | 98 | u64 first_byte, gfp_t gfp_flags) |
99 | { | 99 | { |
100 | int nr_vecs; | 100 | return btrfs_bio_alloc(bdev, first_byte >> 9, BIO_MAX_PAGES, gfp_flags); |
101 | |||
102 | nr_vecs = bio_get_nr_vecs(bdev); | ||
103 | return btrfs_bio_alloc(bdev, first_byte >> 9, nr_vecs, gfp_flags); | ||
104 | } | 101 | } |
105 | 102 | ||
106 | static int check_compressed_csum(struct inode *inode, | 103 | static int check_compressed_csum(struct inode *inode, |
@@ -152,7 +149,7 @@ fail: | |||
152 | * The compressed pages are freed here, and it must be run | 149 | * The compressed pages are freed here, and it must be run |
153 | * in process context | 150 | * in process context |
154 | */ | 151 | */ |
155 | static void end_compressed_bio_read(struct bio *bio, int err) | 152 | static void end_compressed_bio_read(struct bio *bio) |
156 | { | 153 | { |
157 | struct compressed_bio *cb = bio->bi_private; | 154 | struct compressed_bio *cb = bio->bi_private; |
158 | struct inode *inode; | 155 | struct inode *inode; |
@@ -160,7 +157,7 @@ static void end_compressed_bio_read(struct bio *bio, int err) | |||
160 | unsigned long index; | 157 | unsigned long index; |
161 | int ret; | 158 | int ret; |
162 | 159 | ||
163 | if (err) | 160 | if (bio->bi_error) |
164 | cb->errors = 1; | 161 | cb->errors = 1; |
165 | 162 | ||
166 | /* if there are more bios still pending for this compressed | 163 | /* if there are more bios still pending for this compressed |
@@ -210,7 +207,7 @@ csum_failed: | |||
210 | bio_for_each_segment_all(bvec, cb->orig_bio, i) | 207 | bio_for_each_segment_all(bvec, cb->orig_bio, i) |
211 | SetPageChecked(bvec->bv_page); | 208 | SetPageChecked(bvec->bv_page); |
212 | 209 | ||
213 | bio_endio(cb->orig_bio, 0); | 210 | bio_endio(cb->orig_bio); |
214 | } | 211 | } |
215 | 212 | ||
216 | /* finally free the cb struct */ | 213 | /* finally free the cb struct */ |
@@ -266,7 +263,7 @@ static noinline void end_compressed_writeback(struct inode *inode, | |||
266 | * This also calls the writeback end hooks for the file pages so that | 263 | * This also calls the writeback end hooks for the file pages so that |
267 | * metadata and checksums can be updated in the file. | 264 | * metadata and checksums can be updated in the file. |
268 | */ | 265 | */ |
269 | static void end_compressed_bio_write(struct bio *bio, int err) | 266 | static void end_compressed_bio_write(struct bio *bio) |
270 | { | 267 | { |
271 | struct extent_io_tree *tree; | 268 | struct extent_io_tree *tree; |
272 | struct compressed_bio *cb = bio->bi_private; | 269 | struct compressed_bio *cb = bio->bi_private; |
@@ -274,7 +271,7 @@ static void end_compressed_bio_write(struct bio *bio, int err) | |||
274 | struct page *page; | 271 | struct page *page; |
275 | unsigned long index; | 272 | unsigned long index; |
276 | 273 | ||
277 | if (err) | 274 | if (bio->bi_error) |
278 | cb->errors = 1; | 275 | cb->errors = 1; |
279 | 276 | ||
280 | /* if there are more bios still pending for this compressed | 277 | /* if there are more bios still pending for this compressed |
@@ -293,7 +290,7 @@ static void end_compressed_bio_write(struct bio *bio, int err) | |||
293 | cb->start, | 290 | cb->start, |
294 | cb->start + cb->len - 1, | 291 | cb->start + cb->len - 1, |
295 | NULL, | 292 | NULL, |
296 | err ? 0 : 1); | 293 | bio->bi_error ? 0 : 1); |
297 | cb->compressed_pages[0]->mapping = NULL; | 294 | cb->compressed_pages[0]->mapping = NULL; |
298 | 295 | ||
299 | end_compressed_writeback(inode, cb); | 296 | end_compressed_writeback(inode, cb); |
@@ -697,8 +694,10 @@ int btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, | |||
697 | 694 | ||
698 | ret = btrfs_map_bio(root, READ, comp_bio, | 695 | ret = btrfs_map_bio(root, READ, comp_bio, |
699 | mirror_num, 0); | 696 | mirror_num, 0); |
700 | if (ret) | 697 | if (ret) { |
701 | bio_endio(comp_bio, ret); | 698 | bio->bi_error = ret; |
699 | bio_endio(comp_bio); | ||
700 | } | ||
702 | 701 | ||
703 | bio_put(comp_bio); | 702 | bio_put(comp_bio); |
704 | 703 | ||
@@ -724,8 +723,10 @@ int btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, | |||
724 | } | 723 | } |
725 | 724 | ||
726 | ret = btrfs_map_bio(root, READ, comp_bio, mirror_num, 0); | 725 | ret = btrfs_map_bio(root, READ, comp_bio, mirror_num, 0); |
727 | if (ret) | 726 | if (ret) { |
728 | bio_endio(comp_bio, ret); | 727 | bio->bi_error = ret; |
728 | bio_endio(comp_bio); | ||
729 | } | ||
729 | 730 | ||
730 | bio_put(comp_bio); | 731 | bio_put(comp_bio); |
731 | return 0; | 732 | return 0; |
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index f556c3732c2c..5e307bd0471a 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c | |||
@@ -703,7 +703,7 @@ static int btree_io_failed_hook(struct page *page, int failed_mirror) | |||
703 | return -EIO; /* we fixed nothing */ | 703 | return -EIO; /* we fixed nothing */ |
704 | } | 704 | } |
705 | 705 | ||
706 | static void end_workqueue_bio(struct bio *bio, int err) | 706 | static void end_workqueue_bio(struct bio *bio) |
707 | { | 707 | { |
708 | struct btrfs_end_io_wq *end_io_wq = bio->bi_private; | 708 | struct btrfs_end_io_wq *end_io_wq = bio->bi_private; |
709 | struct btrfs_fs_info *fs_info; | 709 | struct btrfs_fs_info *fs_info; |
@@ -711,7 +711,7 @@ static void end_workqueue_bio(struct bio *bio, int err) | |||
711 | btrfs_work_func_t func; | 711 | btrfs_work_func_t func; |
712 | 712 | ||
713 | fs_info = end_io_wq->info; | 713 | fs_info = end_io_wq->info; |
714 | end_io_wq->error = err; | 714 | end_io_wq->error = bio->bi_error; |
715 | 715 | ||
716 | if (bio->bi_rw & REQ_WRITE) { | 716 | if (bio->bi_rw & REQ_WRITE) { |
717 | if (end_io_wq->metadata == BTRFS_WQ_ENDIO_METADATA) { | 717 | if (end_io_wq->metadata == BTRFS_WQ_ENDIO_METADATA) { |
@@ -808,7 +808,8 @@ static void run_one_async_done(struct btrfs_work *work) | |||
808 | 808 | ||
809 | /* If an error occured we just want to clean up the bio and move on */ | 809 | /* If an error occured we just want to clean up the bio and move on */ |
810 | if (async->error) { | 810 | if (async->error) { |
811 | bio_endio(async->bio, async->error); | 811 | async->bio->bi_error = async->error; |
812 | bio_endio(async->bio); | ||
812 | return; | 813 | return; |
813 | } | 814 | } |
814 | 815 | ||
@@ -908,8 +909,10 @@ static int __btree_submit_bio_done(struct inode *inode, int rw, struct bio *bio, | |||
908 | * submission context. Just jump into btrfs_map_bio | 909 | * submission context. Just jump into btrfs_map_bio |
909 | */ | 910 | */ |
910 | ret = btrfs_map_bio(BTRFS_I(inode)->root, rw, bio, mirror_num, 1); | 911 | ret = btrfs_map_bio(BTRFS_I(inode)->root, rw, bio, mirror_num, 1); |
911 | if (ret) | 912 | if (ret) { |
912 | bio_endio(bio, ret); | 913 | bio->bi_error = ret; |
914 | bio_endio(bio); | ||
915 | } | ||
913 | return ret; | 916 | return ret; |
914 | } | 917 | } |
915 | 918 | ||
@@ -960,10 +963,13 @@ static int btree_submit_bio_hook(struct inode *inode, int rw, struct bio *bio, | |||
960 | __btree_submit_bio_done); | 963 | __btree_submit_bio_done); |
961 | } | 964 | } |
962 | 965 | ||
963 | if (ret) { | 966 | if (ret) |
967 | goto out_w_error; | ||
968 | return 0; | ||
969 | |||
964 | out_w_error: | 970 | out_w_error: |
965 | bio_endio(bio, ret); | 971 | bio->bi_error = ret; |
966 | } | 972 | bio_endio(bio); |
967 | return ret; | 973 | return ret; |
968 | } | 974 | } |
969 | 975 | ||
@@ -1735,16 +1741,15 @@ static void end_workqueue_fn(struct btrfs_work *work) | |||
1735 | { | 1741 | { |
1736 | struct bio *bio; | 1742 | struct bio *bio; |
1737 | struct btrfs_end_io_wq *end_io_wq; | 1743 | struct btrfs_end_io_wq *end_io_wq; |
1738 | int error; | ||
1739 | 1744 | ||
1740 | end_io_wq = container_of(work, struct btrfs_end_io_wq, work); | 1745 | end_io_wq = container_of(work, struct btrfs_end_io_wq, work); |
1741 | bio = end_io_wq->bio; | 1746 | bio = end_io_wq->bio; |
1742 | 1747 | ||
1743 | error = end_io_wq->error; | 1748 | bio->bi_error = end_io_wq->error; |
1744 | bio->bi_private = end_io_wq->private; | 1749 | bio->bi_private = end_io_wq->private; |
1745 | bio->bi_end_io = end_io_wq->end_io; | 1750 | bio->bi_end_io = end_io_wq->end_io; |
1746 | kmem_cache_free(btrfs_end_io_wq_cache, end_io_wq); | 1751 | kmem_cache_free(btrfs_end_io_wq_cache, end_io_wq); |
1747 | bio_endio(bio, error); | 1752 | bio_endio(bio); |
1748 | } | 1753 | } |
1749 | 1754 | ||
1750 | static int cleaner_kthread(void *arg) | 1755 | static int cleaner_kthread(void *arg) |
@@ -3324,10 +3329,8 @@ static int write_dev_supers(struct btrfs_device *device, | |||
3324 | * endio for the write_dev_flush, this will wake anyone waiting | 3329 | * endio for the write_dev_flush, this will wake anyone waiting |
3325 | * for the barrier when it is done | 3330 | * for the barrier when it is done |
3326 | */ | 3331 | */ |
3327 | static void btrfs_end_empty_barrier(struct bio *bio, int err) | 3332 | static void btrfs_end_empty_barrier(struct bio *bio) |
3328 | { | 3333 | { |
3329 | if (err) | ||
3330 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | ||
3331 | if (bio->bi_private) | 3334 | if (bio->bi_private) |
3332 | complete(bio->bi_private); | 3335 | complete(bio->bi_private); |
3333 | bio_put(bio); | 3336 | bio_put(bio); |
@@ -3355,8 +3358,8 @@ static int write_dev_flush(struct btrfs_device *device, int wait) | |||
3355 | 3358 | ||
3356 | wait_for_completion(&device->flush_wait); | 3359 | wait_for_completion(&device->flush_wait); |
3357 | 3360 | ||
3358 | if (!bio_flagged(bio, BIO_UPTODATE)) { | 3361 | if (bio->bi_error) { |
3359 | ret = -EIO; | 3362 | ret = bio->bi_error; |
3360 | btrfs_dev_stat_inc_and_print(device, | 3363 | btrfs_dev_stat_inc_and_print(device, |
3361 | BTRFS_DEV_STAT_FLUSH_ERRS); | 3364 | BTRFS_DEV_STAT_FLUSH_ERRS); |
3362 | } | 3365 | } |
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 02d05817cbdf..68b12bbc709f 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c | |||
@@ -2486,7 +2486,7 @@ int end_extent_writepage(struct page *page, int err, u64 start, u64 end) | |||
2486 | * Scheduling is not allowed, so the extent state tree is expected | 2486 | * Scheduling is not allowed, so the extent state tree is expected |
2487 | * to have one and only one object corresponding to this IO. | 2487 | * to have one and only one object corresponding to this IO. |
2488 | */ | 2488 | */ |
2489 | static void end_bio_extent_writepage(struct bio *bio, int err) | 2489 | static void end_bio_extent_writepage(struct bio *bio) |
2490 | { | 2490 | { |
2491 | struct bio_vec *bvec; | 2491 | struct bio_vec *bvec; |
2492 | u64 start; | 2492 | u64 start; |
@@ -2516,7 +2516,7 @@ static void end_bio_extent_writepage(struct bio *bio, int err) | |||
2516 | start = page_offset(page); | 2516 | start = page_offset(page); |
2517 | end = start + bvec->bv_offset + bvec->bv_len - 1; | 2517 | end = start + bvec->bv_offset + bvec->bv_len - 1; |
2518 | 2518 | ||
2519 | if (end_extent_writepage(page, err, start, end)) | 2519 | if (end_extent_writepage(page, bio->bi_error, start, end)) |
2520 | continue; | 2520 | continue; |
2521 | 2521 | ||
2522 | end_page_writeback(page); | 2522 | end_page_writeback(page); |
@@ -2548,10 +2548,10 @@ endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len, | |||
2548 | * Scheduling is not allowed, so the extent state tree is expected | 2548 | * Scheduling is not allowed, so the extent state tree is expected |
2549 | * to have one and only one object corresponding to this IO. | 2549 | * to have one and only one object corresponding to this IO. |
2550 | */ | 2550 | */ |
2551 | static void end_bio_extent_readpage(struct bio *bio, int err) | 2551 | static void end_bio_extent_readpage(struct bio *bio) |
2552 | { | 2552 | { |
2553 | struct bio_vec *bvec; | 2553 | struct bio_vec *bvec; |
2554 | int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | 2554 | int uptodate = !bio->bi_error; |
2555 | struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); | 2555 | struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); |
2556 | struct extent_io_tree *tree; | 2556 | struct extent_io_tree *tree; |
2557 | u64 offset = 0; | 2557 | u64 offset = 0; |
@@ -2564,16 +2564,13 @@ static void end_bio_extent_readpage(struct bio *bio, int err) | |||
2564 | int ret; | 2564 | int ret; |
2565 | int i; | 2565 | int i; |
2566 | 2566 | ||
2567 | if (err) | ||
2568 | uptodate = 0; | ||
2569 | |||
2570 | bio_for_each_segment_all(bvec, bio, i) { | 2567 | bio_for_each_segment_all(bvec, bio, i) { |
2571 | struct page *page = bvec->bv_page; | 2568 | struct page *page = bvec->bv_page; |
2572 | struct inode *inode = page->mapping->host; | 2569 | struct inode *inode = page->mapping->host; |
2573 | 2570 | ||
2574 | pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, " | 2571 | pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, " |
2575 | "mirror=%u\n", (u64)bio->bi_iter.bi_sector, err, | 2572 | "mirror=%u\n", (u64)bio->bi_iter.bi_sector, |
2576 | io_bio->mirror_num); | 2573 | bio->bi_error, io_bio->mirror_num); |
2577 | tree = &BTRFS_I(inode)->io_tree; | 2574 | tree = &BTRFS_I(inode)->io_tree; |
2578 | 2575 | ||
2579 | /* We always issue full-page reads, but if some block | 2576 | /* We always issue full-page reads, but if some block |
@@ -2614,8 +2611,7 @@ static void end_bio_extent_readpage(struct bio *bio, int err) | |||
2614 | 2611 | ||
2615 | if (tree->ops && tree->ops->readpage_io_failed_hook) { | 2612 | if (tree->ops && tree->ops->readpage_io_failed_hook) { |
2616 | ret = tree->ops->readpage_io_failed_hook(page, mirror); | 2613 | ret = tree->ops->readpage_io_failed_hook(page, mirror); |
2617 | if (!ret && !err && | 2614 | if (!ret && !bio->bi_error) |
2618 | test_bit(BIO_UPTODATE, &bio->bi_flags)) | ||
2619 | uptodate = 1; | 2615 | uptodate = 1; |
2620 | } else { | 2616 | } else { |
2621 | /* | 2617 | /* |
@@ -2631,10 +2627,7 @@ static void end_bio_extent_readpage(struct bio *bio, int err) | |||
2631 | ret = bio_readpage_error(bio, offset, page, start, end, | 2627 | ret = bio_readpage_error(bio, offset, page, start, end, |
2632 | mirror); | 2628 | mirror); |
2633 | if (ret == 0) { | 2629 | if (ret == 0) { |
2634 | uptodate = | 2630 | uptodate = !bio->bi_error; |
2635 | test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
2636 | if (err) | ||
2637 | uptodate = 0; | ||
2638 | offset += len; | 2631 | offset += len; |
2639 | continue; | 2632 | continue; |
2640 | } | 2633 | } |
@@ -2684,7 +2677,7 @@ readpage_ok: | |||
2684 | endio_readpage_release_extent(tree, extent_start, extent_len, | 2677 | endio_readpage_release_extent(tree, extent_start, extent_len, |
2685 | uptodate); | 2678 | uptodate); |
2686 | if (io_bio->end_io) | 2679 | if (io_bio->end_io) |
2687 | io_bio->end_io(io_bio, err); | 2680 | io_bio->end_io(io_bio, bio->bi_error); |
2688 | bio_put(bio); | 2681 | bio_put(bio); |
2689 | } | 2682 | } |
2690 | 2683 | ||
@@ -2802,9 +2795,7 @@ static int submit_extent_page(int rw, struct extent_io_tree *tree, | |||
2802 | { | 2795 | { |
2803 | int ret = 0; | 2796 | int ret = 0; |
2804 | struct bio *bio; | 2797 | struct bio *bio; |
2805 | int nr; | ||
2806 | int contig = 0; | 2798 | int contig = 0; |
2807 | int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED; | ||
2808 | int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED; | 2799 | int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED; |
2809 | size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE); | 2800 | size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE); |
2810 | 2801 | ||
@@ -2829,12 +2820,9 @@ static int submit_extent_page(int rw, struct extent_io_tree *tree, | |||
2829 | return 0; | 2820 | return 0; |
2830 | } | 2821 | } |
2831 | } | 2822 | } |
2832 | if (this_compressed) | ||
2833 | nr = BIO_MAX_PAGES; | ||
2834 | else | ||
2835 | nr = bio_get_nr_vecs(bdev); | ||
2836 | 2823 | ||
2837 | bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH); | 2824 | bio = btrfs_bio_alloc(bdev, sector, BIO_MAX_PAGES, |
2825 | GFP_NOFS | __GFP_HIGH); | ||
2838 | if (!bio) | 2826 | if (!bio) |
2839 | return -ENOMEM; | 2827 | return -ENOMEM; |
2840 | 2828 | ||
@@ -3696,7 +3684,7 @@ static void set_btree_ioerr(struct page *page) | |||
3696 | } | 3684 | } |
3697 | } | 3685 | } |
3698 | 3686 | ||
3699 | static void end_bio_extent_buffer_writepage(struct bio *bio, int err) | 3687 | static void end_bio_extent_buffer_writepage(struct bio *bio) |
3700 | { | 3688 | { |
3701 | struct bio_vec *bvec; | 3689 | struct bio_vec *bvec; |
3702 | struct extent_buffer *eb; | 3690 | struct extent_buffer *eb; |
@@ -3709,7 +3697,8 @@ static void end_bio_extent_buffer_writepage(struct bio *bio, int err) | |||
3709 | BUG_ON(!eb); | 3697 | BUG_ON(!eb); |
3710 | done = atomic_dec_and_test(&eb->io_pages); | 3698 | done = atomic_dec_and_test(&eb->io_pages); |
3711 | 3699 | ||
3712 | if (err || test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) { | 3700 | if (bio->bi_error || |
3701 | test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) { | ||
3713 | ClearPageUptodate(page); | 3702 | ClearPageUptodate(page); |
3714 | set_btree_ioerr(page); | 3703 | set_btree_ioerr(page); |
3715 | } | 3704 | } |
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index e33dff356460..f924d9a62700 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c | |||
@@ -1845,8 +1845,10 @@ static int __btrfs_submit_bio_done(struct inode *inode, int rw, struct bio *bio, | |||
1845 | int ret; | 1845 | int ret; |
1846 | 1846 | ||
1847 | ret = btrfs_map_bio(root, rw, bio, mirror_num, 1); | 1847 | ret = btrfs_map_bio(root, rw, bio, mirror_num, 1); |
1848 | if (ret) | 1848 | if (ret) { |
1849 | bio_endio(bio, ret); | 1849 | bio->bi_error = ret; |
1850 | bio_endio(bio); | ||
1851 | } | ||
1850 | return ret; | 1852 | return ret; |
1851 | } | 1853 | } |
1852 | 1854 | ||
@@ -1906,8 +1908,10 @@ mapit: | |||
1906 | ret = btrfs_map_bio(root, rw, bio, mirror_num, 0); | 1908 | ret = btrfs_map_bio(root, rw, bio, mirror_num, 0); |
1907 | 1909 | ||
1908 | out: | 1910 | out: |
1909 | if (ret < 0) | 1911 | if (ret < 0) { |
1910 | bio_endio(bio, ret); | 1912 | bio->bi_error = ret; |
1913 | bio_endio(bio); | ||
1914 | } | ||
1911 | return ret; | 1915 | return ret; |
1912 | } | 1916 | } |
1913 | 1917 | ||
@@ -7688,13 +7692,13 @@ struct btrfs_retry_complete { | |||
7688 | int uptodate; | 7692 | int uptodate; |
7689 | }; | 7693 | }; |
7690 | 7694 | ||
7691 | static void btrfs_retry_endio_nocsum(struct bio *bio, int err) | 7695 | static void btrfs_retry_endio_nocsum(struct bio *bio) |
7692 | { | 7696 | { |
7693 | struct btrfs_retry_complete *done = bio->bi_private; | 7697 | struct btrfs_retry_complete *done = bio->bi_private; |
7694 | struct bio_vec *bvec; | 7698 | struct bio_vec *bvec; |
7695 | int i; | 7699 | int i; |
7696 | 7700 | ||
7697 | if (err) | 7701 | if (bio->bi_error) |
7698 | goto end; | 7702 | goto end; |
7699 | 7703 | ||
7700 | done->uptodate = 1; | 7704 | done->uptodate = 1; |
@@ -7743,7 +7747,7 @@ try_again: | |||
7743 | return 0; | 7747 | return 0; |
7744 | } | 7748 | } |
7745 | 7749 | ||
7746 | static void btrfs_retry_endio(struct bio *bio, int err) | 7750 | static void btrfs_retry_endio(struct bio *bio) |
7747 | { | 7751 | { |
7748 | struct btrfs_retry_complete *done = bio->bi_private; | 7752 | struct btrfs_retry_complete *done = bio->bi_private; |
7749 | struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); | 7753 | struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); |
@@ -7752,7 +7756,7 @@ static void btrfs_retry_endio(struct bio *bio, int err) | |||
7752 | int ret; | 7756 | int ret; |
7753 | int i; | 7757 | int i; |
7754 | 7758 | ||
7755 | if (err) | 7759 | if (bio->bi_error) |
7756 | goto end; | 7760 | goto end; |
7757 | 7761 | ||
7758 | uptodate = 1; | 7762 | uptodate = 1; |
@@ -7835,12 +7839,13 @@ static int btrfs_subio_endio_read(struct inode *inode, | |||
7835 | } | 7839 | } |
7836 | } | 7840 | } |
7837 | 7841 | ||
7838 | static void btrfs_endio_direct_read(struct bio *bio, int err) | 7842 | static void btrfs_endio_direct_read(struct bio *bio) |
7839 | { | 7843 | { |
7840 | struct btrfs_dio_private *dip = bio->bi_private; | 7844 | struct btrfs_dio_private *dip = bio->bi_private; |
7841 | struct inode *inode = dip->inode; | 7845 | struct inode *inode = dip->inode; |
7842 | struct bio *dio_bio; | 7846 | struct bio *dio_bio; |
7843 | struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); | 7847 | struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); |
7848 | int err = bio->bi_error; | ||
7844 | 7849 | ||
7845 | if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED) | 7850 | if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED) |
7846 | err = btrfs_subio_endio_read(inode, io_bio, err); | 7851 | err = btrfs_subio_endio_read(inode, io_bio, err); |
@@ -7851,17 +7856,14 @@ static void btrfs_endio_direct_read(struct bio *bio, int err) | |||
7851 | 7856 | ||
7852 | kfree(dip); | 7857 | kfree(dip); |
7853 | 7858 | ||
7854 | /* If we had a csum failure make sure to clear the uptodate flag */ | 7859 | dio_end_io(dio_bio, bio->bi_error); |
7855 | if (err) | ||
7856 | clear_bit(BIO_UPTODATE, &dio_bio->bi_flags); | ||
7857 | dio_end_io(dio_bio, err); | ||
7858 | 7860 | ||
7859 | if (io_bio->end_io) | 7861 | if (io_bio->end_io) |
7860 | io_bio->end_io(io_bio, err); | 7862 | io_bio->end_io(io_bio, err); |
7861 | bio_put(bio); | 7863 | bio_put(bio); |
7862 | } | 7864 | } |
7863 | 7865 | ||
7864 | static void btrfs_endio_direct_write(struct bio *bio, int err) | 7866 | static void btrfs_endio_direct_write(struct bio *bio) |
7865 | { | 7867 | { |
7866 | struct btrfs_dio_private *dip = bio->bi_private; | 7868 | struct btrfs_dio_private *dip = bio->bi_private; |
7867 | struct inode *inode = dip->inode; | 7869 | struct inode *inode = dip->inode; |
@@ -7875,7 +7877,8 @@ static void btrfs_endio_direct_write(struct bio *bio, int err) | |||
7875 | again: | 7877 | again: |
7876 | ret = btrfs_dec_test_first_ordered_pending(inode, &ordered, | 7878 | ret = btrfs_dec_test_first_ordered_pending(inode, &ordered, |
7877 | &ordered_offset, | 7879 | &ordered_offset, |
7878 | ordered_bytes, !err); | 7880 | ordered_bytes, |
7881 | !bio->bi_error); | ||
7879 | if (!ret) | 7882 | if (!ret) |
7880 | goto out_test; | 7883 | goto out_test; |
7881 | 7884 | ||
@@ -7898,10 +7901,7 @@ out_test: | |||
7898 | 7901 | ||
7899 | kfree(dip); | 7902 | kfree(dip); |
7900 | 7903 | ||
7901 | /* If we had an error make sure to clear the uptodate flag */ | 7904 | dio_end_io(dio_bio, bio->bi_error); |
7902 | if (err) | ||
7903 | clear_bit(BIO_UPTODATE, &dio_bio->bi_flags); | ||
7904 | dio_end_io(dio_bio, err); | ||
7905 | bio_put(bio); | 7905 | bio_put(bio); |
7906 | } | 7906 | } |
7907 | 7907 | ||
@@ -7916,9 +7916,10 @@ static int __btrfs_submit_bio_start_direct_io(struct inode *inode, int rw, | |||
7916 | return 0; | 7916 | return 0; |
7917 | } | 7917 | } |
7918 | 7918 | ||
7919 | static void btrfs_end_dio_bio(struct bio *bio, int err) | 7919 | static void btrfs_end_dio_bio(struct bio *bio) |
7920 | { | 7920 | { |
7921 | struct btrfs_dio_private *dip = bio->bi_private; | 7921 | struct btrfs_dio_private *dip = bio->bi_private; |
7922 | int err = bio->bi_error; | ||
7922 | 7923 | ||
7923 | if (err) | 7924 | if (err) |
7924 | btrfs_warn(BTRFS_I(dip->inode)->root->fs_info, | 7925 | btrfs_warn(BTRFS_I(dip->inode)->root->fs_info, |
@@ -7947,8 +7948,8 @@ static void btrfs_end_dio_bio(struct bio *bio, int err) | |||
7947 | if (dip->errors) { | 7948 | if (dip->errors) { |
7948 | bio_io_error(dip->orig_bio); | 7949 | bio_io_error(dip->orig_bio); |
7949 | } else { | 7950 | } else { |
7950 | set_bit(BIO_UPTODATE, &dip->dio_bio->bi_flags); | 7951 | dip->dio_bio->bi_error = 0; |
7951 | bio_endio(dip->orig_bio, 0); | 7952 | bio_endio(dip->orig_bio); |
7952 | } | 7953 | } |
7953 | out: | 7954 | out: |
7954 | bio_put(bio); | 7955 | bio_put(bio); |
@@ -7957,8 +7958,7 @@ out: | |||
7957 | static struct bio *btrfs_dio_bio_alloc(struct block_device *bdev, | 7958 | static struct bio *btrfs_dio_bio_alloc(struct block_device *bdev, |
7958 | u64 first_sector, gfp_t gfp_flags) | 7959 | u64 first_sector, gfp_t gfp_flags) |
7959 | { | 7960 | { |
7960 | int nr_vecs = bio_get_nr_vecs(bdev); | 7961 | return btrfs_bio_alloc(bdev, first_sector, BIO_MAX_PAGES, gfp_flags); |
7961 | return btrfs_bio_alloc(bdev, first_sector, nr_vecs, gfp_flags); | ||
7962 | } | 7962 | } |
7963 | 7963 | ||
7964 | static inline int btrfs_lookup_and_bind_dio_csum(struct btrfs_root *root, | 7964 | static inline int btrfs_lookup_and_bind_dio_csum(struct btrfs_root *root, |
@@ -8219,7 +8219,8 @@ free_ordered: | |||
8219 | * callbacks - they require an allocated dip and a clone of dio_bio. | 8219 | * callbacks - they require an allocated dip and a clone of dio_bio. |
8220 | */ | 8220 | */ |
8221 | if (io_bio && dip) { | 8221 | if (io_bio && dip) { |
8222 | bio_endio(io_bio, ret); | 8222 | io_bio->bi_error = -EIO; |
8223 | bio_endio(io_bio); | ||
8223 | /* | 8224 | /* |
8224 | * The end io callbacks free our dip, do the final put on io_bio | 8225 | * The end io callbacks free our dip, do the final put on io_bio |
8225 | * and all the cleanup and final put for dio_bio (through | 8226 | * and all the cleanup and final put for dio_bio (through |
@@ -8246,7 +8247,7 @@ free_ordered: | |||
8246 | unlock_extent(&BTRFS_I(inode)->io_tree, file_offset, | 8247 | unlock_extent(&BTRFS_I(inode)->io_tree, file_offset, |
8247 | file_offset + dio_bio->bi_iter.bi_size - 1); | 8248 | file_offset + dio_bio->bi_iter.bi_size - 1); |
8248 | } | 8249 | } |
8249 | clear_bit(BIO_UPTODATE, &dio_bio->bi_flags); | 8250 | dio_bio->bi_error = -EIO; |
8250 | /* | 8251 | /* |
8251 | * Releases and cleans up our dio_bio, no need to bio_put() | 8252 | * Releases and cleans up our dio_bio, no need to bio_put() |
8252 | * nor bio_endio()/bio_io_error() against dio_bio. | 8253 | * nor bio_endio()/bio_io_error() against dio_bio. |
diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index fa72068bd256..0a02e24900aa 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c | |||
@@ -851,7 +851,7 @@ static void free_raid_bio(struct btrfs_raid_bio *rbio) | |||
851 | * this frees the rbio and runs through all the bios in the | 851 | * this frees the rbio and runs through all the bios in the |
852 | * bio_list and calls end_io on them | 852 | * bio_list and calls end_io on them |
853 | */ | 853 | */ |
854 | static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, int err, int uptodate) | 854 | static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, int err) |
855 | { | 855 | { |
856 | struct bio *cur = bio_list_get(&rbio->bio_list); | 856 | struct bio *cur = bio_list_get(&rbio->bio_list); |
857 | struct bio *next; | 857 | struct bio *next; |
@@ -864,9 +864,8 @@ static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, int err, int uptodate) | |||
864 | while (cur) { | 864 | while (cur) { |
865 | next = cur->bi_next; | 865 | next = cur->bi_next; |
866 | cur->bi_next = NULL; | 866 | cur->bi_next = NULL; |
867 | if (uptodate) | 867 | cur->bi_error = err; |
868 | set_bit(BIO_UPTODATE, &cur->bi_flags); | 868 | bio_endio(cur); |
869 | bio_endio(cur, err); | ||
870 | cur = next; | 869 | cur = next; |
871 | } | 870 | } |
872 | } | 871 | } |
@@ -875,9 +874,10 @@ static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, int err, int uptodate) | |||
875 | * end io function used by finish_rmw. When we finally | 874 | * end io function used by finish_rmw. When we finally |
876 | * get here, we've written a full stripe | 875 | * get here, we've written a full stripe |
877 | */ | 876 | */ |
878 | static void raid_write_end_io(struct bio *bio, int err) | 877 | static void raid_write_end_io(struct bio *bio) |
879 | { | 878 | { |
880 | struct btrfs_raid_bio *rbio = bio->bi_private; | 879 | struct btrfs_raid_bio *rbio = bio->bi_private; |
880 | int err = bio->bi_error; | ||
881 | 881 | ||
882 | if (err) | 882 | if (err) |
883 | fail_bio_stripe(rbio, bio); | 883 | fail_bio_stripe(rbio, bio); |
@@ -893,7 +893,7 @@ static void raid_write_end_io(struct bio *bio, int err) | |||
893 | if (atomic_read(&rbio->error) > rbio->bbio->max_errors) | 893 | if (atomic_read(&rbio->error) > rbio->bbio->max_errors) |
894 | err = -EIO; | 894 | err = -EIO; |
895 | 895 | ||
896 | rbio_orig_end_io(rbio, err, 0); | 896 | rbio_orig_end_io(rbio, err); |
897 | return; | 897 | return; |
898 | } | 898 | } |
899 | 899 | ||
@@ -1071,7 +1071,7 @@ static int rbio_add_io_page(struct btrfs_raid_bio *rbio, | |||
1071 | * devices or if they are not contiguous | 1071 | * devices or if they are not contiguous |
1072 | */ | 1072 | */ |
1073 | if (last_end == disk_start && stripe->dev->bdev && | 1073 | if (last_end == disk_start && stripe->dev->bdev && |
1074 | test_bit(BIO_UPTODATE, &last->bi_flags) && | 1074 | !last->bi_error && |
1075 | last->bi_bdev == stripe->dev->bdev) { | 1075 | last->bi_bdev == stripe->dev->bdev) { |
1076 | ret = bio_add_page(last, page, PAGE_CACHE_SIZE, 0); | 1076 | ret = bio_add_page(last, page, PAGE_CACHE_SIZE, 0); |
1077 | if (ret == PAGE_CACHE_SIZE) | 1077 | if (ret == PAGE_CACHE_SIZE) |
@@ -1087,7 +1087,6 @@ static int rbio_add_io_page(struct btrfs_raid_bio *rbio, | |||
1087 | bio->bi_iter.bi_size = 0; | 1087 | bio->bi_iter.bi_size = 0; |
1088 | bio->bi_bdev = stripe->dev->bdev; | 1088 | bio->bi_bdev = stripe->dev->bdev; |
1089 | bio->bi_iter.bi_sector = disk_start >> 9; | 1089 | bio->bi_iter.bi_sector = disk_start >> 9; |
1090 | set_bit(BIO_UPTODATE, &bio->bi_flags); | ||
1091 | 1090 | ||
1092 | bio_add_page(bio, page, PAGE_CACHE_SIZE, 0); | 1091 | bio_add_page(bio, page, PAGE_CACHE_SIZE, 0); |
1093 | bio_list_add(bio_list, bio); | 1092 | bio_list_add(bio_list, bio); |
@@ -1312,13 +1311,12 @@ write_data: | |||
1312 | 1311 | ||
1313 | bio->bi_private = rbio; | 1312 | bio->bi_private = rbio; |
1314 | bio->bi_end_io = raid_write_end_io; | 1313 | bio->bi_end_io = raid_write_end_io; |
1315 | BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags)); | ||
1316 | submit_bio(WRITE, bio); | 1314 | submit_bio(WRITE, bio); |
1317 | } | 1315 | } |
1318 | return; | 1316 | return; |
1319 | 1317 | ||
1320 | cleanup: | 1318 | cleanup: |
1321 | rbio_orig_end_io(rbio, -EIO, 0); | 1319 | rbio_orig_end_io(rbio, -EIO); |
1322 | } | 1320 | } |
1323 | 1321 | ||
1324 | /* | 1322 | /* |
@@ -1441,11 +1439,11 @@ static void set_bio_pages_uptodate(struct bio *bio) | |||
1441 | * This will usually kick off finish_rmw once all the bios are read in, but it | 1439 | * This will usually kick off finish_rmw once all the bios are read in, but it |
1442 | * may trigger parity reconstruction if we had any errors along the way | 1440 | * may trigger parity reconstruction if we had any errors along the way |
1443 | */ | 1441 | */ |
1444 | static void raid_rmw_end_io(struct bio *bio, int err) | 1442 | static void raid_rmw_end_io(struct bio *bio) |
1445 | { | 1443 | { |
1446 | struct btrfs_raid_bio *rbio = bio->bi_private; | 1444 | struct btrfs_raid_bio *rbio = bio->bi_private; |
1447 | 1445 | ||
1448 | if (err) | 1446 | if (bio->bi_error) |
1449 | fail_bio_stripe(rbio, bio); | 1447 | fail_bio_stripe(rbio, bio); |
1450 | else | 1448 | else |
1451 | set_bio_pages_uptodate(bio); | 1449 | set_bio_pages_uptodate(bio); |
@@ -1455,7 +1453,6 @@ static void raid_rmw_end_io(struct bio *bio, int err) | |||
1455 | if (!atomic_dec_and_test(&rbio->stripes_pending)) | 1453 | if (!atomic_dec_and_test(&rbio->stripes_pending)) |
1456 | return; | 1454 | return; |
1457 | 1455 | ||
1458 | err = 0; | ||
1459 | if (atomic_read(&rbio->error) > rbio->bbio->max_errors) | 1456 | if (atomic_read(&rbio->error) > rbio->bbio->max_errors) |
1460 | goto cleanup; | 1457 | goto cleanup; |
1461 | 1458 | ||
@@ -1469,7 +1466,7 @@ static void raid_rmw_end_io(struct bio *bio, int err) | |||
1469 | 1466 | ||
1470 | cleanup: | 1467 | cleanup: |
1471 | 1468 | ||
1472 | rbio_orig_end_io(rbio, -EIO, 0); | 1469 | rbio_orig_end_io(rbio, -EIO); |
1473 | } | 1470 | } |
1474 | 1471 | ||
1475 | static void async_rmw_stripe(struct btrfs_raid_bio *rbio) | 1472 | static void async_rmw_stripe(struct btrfs_raid_bio *rbio) |
@@ -1572,14 +1569,13 @@ static int raid56_rmw_stripe(struct btrfs_raid_bio *rbio) | |||
1572 | btrfs_bio_wq_end_io(rbio->fs_info, bio, | 1569 | btrfs_bio_wq_end_io(rbio->fs_info, bio, |
1573 | BTRFS_WQ_ENDIO_RAID56); | 1570 | BTRFS_WQ_ENDIO_RAID56); |
1574 | 1571 | ||
1575 | BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags)); | ||
1576 | submit_bio(READ, bio); | 1572 | submit_bio(READ, bio); |
1577 | } | 1573 | } |
1578 | /* the actual write will happen once the reads are done */ | 1574 | /* the actual write will happen once the reads are done */ |
1579 | return 0; | 1575 | return 0; |
1580 | 1576 | ||
1581 | cleanup: | 1577 | cleanup: |
1582 | rbio_orig_end_io(rbio, -EIO, 0); | 1578 | rbio_orig_end_io(rbio, -EIO); |
1583 | return -EIO; | 1579 | return -EIO; |
1584 | 1580 | ||
1585 | finish: | 1581 | finish: |
@@ -1964,7 +1960,7 @@ cleanup_io: | |||
1964 | else | 1960 | else |
1965 | clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags); | 1961 | clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags); |
1966 | 1962 | ||
1967 | rbio_orig_end_io(rbio, err, err == 0); | 1963 | rbio_orig_end_io(rbio, err); |
1968 | } else if (err == 0) { | 1964 | } else if (err == 0) { |
1969 | rbio->faila = -1; | 1965 | rbio->faila = -1; |
1970 | rbio->failb = -1; | 1966 | rbio->failb = -1; |
@@ -1976,7 +1972,7 @@ cleanup_io: | |||
1976 | else | 1972 | else |
1977 | BUG(); | 1973 | BUG(); |
1978 | } else { | 1974 | } else { |
1979 | rbio_orig_end_io(rbio, err, 0); | 1975 | rbio_orig_end_io(rbio, err); |
1980 | } | 1976 | } |
1981 | } | 1977 | } |
1982 | 1978 | ||
@@ -1984,7 +1980,7 @@ cleanup_io: | |||
1984 | * This is called only for stripes we've read from disk to | 1980 | * This is called only for stripes we've read from disk to |
1985 | * reconstruct the parity. | 1981 | * reconstruct the parity. |
1986 | */ | 1982 | */ |
1987 | static void raid_recover_end_io(struct bio *bio, int err) | 1983 | static void raid_recover_end_io(struct bio *bio) |
1988 | { | 1984 | { |
1989 | struct btrfs_raid_bio *rbio = bio->bi_private; | 1985 | struct btrfs_raid_bio *rbio = bio->bi_private; |
1990 | 1986 | ||
@@ -1992,7 +1988,7 @@ static void raid_recover_end_io(struct bio *bio, int err) | |||
1992 | * we only read stripe pages off the disk, set them | 1988 | * we only read stripe pages off the disk, set them |
1993 | * up to date if there were no errors | 1989 | * up to date if there were no errors |
1994 | */ | 1990 | */ |
1995 | if (err) | 1991 | if (bio->bi_error) |
1996 | fail_bio_stripe(rbio, bio); | 1992 | fail_bio_stripe(rbio, bio); |
1997 | else | 1993 | else |
1998 | set_bio_pages_uptodate(bio); | 1994 | set_bio_pages_uptodate(bio); |
@@ -2002,7 +1998,7 @@ static void raid_recover_end_io(struct bio *bio, int err) | |||
2002 | return; | 1998 | return; |
2003 | 1999 | ||
2004 | if (atomic_read(&rbio->error) > rbio->bbio->max_errors) | 2000 | if (atomic_read(&rbio->error) > rbio->bbio->max_errors) |
2005 | rbio_orig_end_io(rbio, -EIO, 0); | 2001 | rbio_orig_end_io(rbio, -EIO); |
2006 | else | 2002 | else |
2007 | __raid_recover_end_io(rbio); | 2003 | __raid_recover_end_io(rbio); |
2008 | } | 2004 | } |
@@ -2094,7 +2090,6 @@ static int __raid56_parity_recover(struct btrfs_raid_bio *rbio) | |||
2094 | btrfs_bio_wq_end_io(rbio->fs_info, bio, | 2090 | btrfs_bio_wq_end_io(rbio->fs_info, bio, |
2095 | BTRFS_WQ_ENDIO_RAID56); | 2091 | BTRFS_WQ_ENDIO_RAID56); |
2096 | 2092 | ||
2097 | BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags)); | ||
2098 | submit_bio(READ, bio); | 2093 | submit_bio(READ, bio); |
2099 | } | 2094 | } |
2100 | out: | 2095 | out: |
@@ -2102,7 +2097,7 @@ out: | |||
2102 | 2097 | ||
2103 | cleanup: | 2098 | cleanup: |
2104 | if (rbio->operation == BTRFS_RBIO_READ_REBUILD) | 2099 | if (rbio->operation == BTRFS_RBIO_READ_REBUILD) |
2105 | rbio_orig_end_io(rbio, -EIO, 0); | 2100 | rbio_orig_end_io(rbio, -EIO); |
2106 | return -EIO; | 2101 | return -EIO; |
2107 | } | 2102 | } |
2108 | 2103 | ||
@@ -2277,11 +2272,12 @@ static int alloc_rbio_essential_pages(struct btrfs_raid_bio *rbio) | |||
2277 | * end io function used by finish_rmw. When we finally | 2272 | * end io function used by finish_rmw. When we finally |
2278 | * get here, we've written a full stripe | 2273 | * get here, we've written a full stripe |
2279 | */ | 2274 | */ |
2280 | static void raid_write_parity_end_io(struct bio *bio, int err) | 2275 | static void raid_write_parity_end_io(struct bio *bio) |
2281 | { | 2276 | { |
2282 | struct btrfs_raid_bio *rbio = bio->bi_private; | 2277 | struct btrfs_raid_bio *rbio = bio->bi_private; |
2278 | int err = bio->bi_error; | ||
2283 | 2279 | ||
2284 | if (err) | 2280 | if (bio->bi_error) |
2285 | fail_bio_stripe(rbio, bio); | 2281 | fail_bio_stripe(rbio, bio); |
2286 | 2282 | ||
2287 | bio_put(bio); | 2283 | bio_put(bio); |
@@ -2294,7 +2290,7 @@ static void raid_write_parity_end_io(struct bio *bio, int err) | |||
2294 | if (atomic_read(&rbio->error)) | 2290 | if (atomic_read(&rbio->error)) |
2295 | err = -EIO; | 2291 | err = -EIO; |
2296 | 2292 | ||
2297 | rbio_orig_end_io(rbio, err, 0); | 2293 | rbio_orig_end_io(rbio, err); |
2298 | } | 2294 | } |
2299 | 2295 | ||
2300 | static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio, | 2296 | static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio, |
@@ -2437,7 +2433,7 @@ submit_write: | |||
2437 | nr_data = bio_list_size(&bio_list); | 2433 | nr_data = bio_list_size(&bio_list); |
2438 | if (!nr_data) { | 2434 | if (!nr_data) { |
2439 | /* Every parity is right */ | 2435 | /* Every parity is right */ |
2440 | rbio_orig_end_io(rbio, 0, 0); | 2436 | rbio_orig_end_io(rbio, 0); |
2441 | return; | 2437 | return; |
2442 | } | 2438 | } |
2443 | 2439 | ||
@@ -2450,13 +2446,12 @@ submit_write: | |||
2450 | 2446 | ||
2451 | bio->bi_private = rbio; | 2447 | bio->bi_private = rbio; |
2452 | bio->bi_end_io = raid_write_parity_end_io; | 2448 | bio->bi_end_io = raid_write_parity_end_io; |
2453 | BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags)); | ||
2454 | submit_bio(WRITE, bio); | 2449 | submit_bio(WRITE, bio); |
2455 | } | 2450 | } |
2456 | return; | 2451 | return; |
2457 | 2452 | ||
2458 | cleanup: | 2453 | cleanup: |
2459 | rbio_orig_end_io(rbio, -EIO, 0); | 2454 | rbio_orig_end_io(rbio, -EIO); |
2460 | } | 2455 | } |
2461 | 2456 | ||
2462 | static inline int is_data_stripe(struct btrfs_raid_bio *rbio, int stripe) | 2457 | static inline int is_data_stripe(struct btrfs_raid_bio *rbio, int stripe) |
@@ -2524,7 +2519,7 @@ static void validate_rbio_for_parity_scrub(struct btrfs_raid_bio *rbio) | |||
2524 | return; | 2519 | return; |
2525 | 2520 | ||
2526 | cleanup: | 2521 | cleanup: |
2527 | rbio_orig_end_io(rbio, -EIO, 0); | 2522 | rbio_orig_end_io(rbio, -EIO); |
2528 | } | 2523 | } |
2529 | 2524 | ||
2530 | /* | 2525 | /* |
@@ -2535,11 +2530,11 @@ cleanup: | |||
2535 | * This will usually kick off finish_rmw once all the bios are read in, but it | 2530 | * This will usually kick off finish_rmw once all the bios are read in, but it |
2536 | * may trigger parity reconstruction if we had any errors along the way | 2531 | * may trigger parity reconstruction if we had any errors along the way |
2537 | */ | 2532 | */ |
2538 | static void raid56_parity_scrub_end_io(struct bio *bio, int err) | 2533 | static void raid56_parity_scrub_end_io(struct bio *bio) |
2539 | { | 2534 | { |
2540 | struct btrfs_raid_bio *rbio = bio->bi_private; | 2535 | struct btrfs_raid_bio *rbio = bio->bi_private; |
2541 | 2536 | ||
2542 | if (err) | 2537 | if (bio->bi_error) |
2543 | fail_bio_stripe(rbio, bio); | 2538 | fail_bio_stripe(rbio, bio); |
2544 | else | 2539 | else |
2545 | set_bio_pages_uptodate(bio); | 2540 | set_bio_pages_uptodate(bio); |
@@ -2632,14 +2627,13 @@ static void raid56_parity_scrub_stripe(struct btrfs_raid_bio *rbio) | |||
2632 | btrfs_bio_wq_end_io(rbio->fs_info, bio, | 2627 | btrfs_bio_wq_end_io(rbio->fs_info, bio, |
2633 | BTRFS_WQ_ENDIO_RAID56); | 2628 | BTRFS_WQ_ENDIO_RAID56); |
2634 | 2629 | ||
2635 | BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags)); | ||
2636 | submit_bio(READ, bio); | 2630 | submit_bio(READ, bio); |
2637 | } | 2631 | } |
2638 | /* the actual write will happen once the reads are done */ | 2632 | /* the actual write will happen once the reads are done */ |
2639 | return; | 2633 | return; |
2640 | 2634 | ||
2641 | cleanup: | 2635 | cleanup: |
2642 | rbio_orig_end_io(rbio, -EIO, 0); | 2636 | rbio_orig_end_io(rbio, -EIO); |
2643 | return; | 2637 | return; |
2644 | 2638 | ||
2645 | finish: | 2639 | finish: |
diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 94db0fa5225a..9c146d8307b5 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c | |||
@@ -278,7 +278,7 @@ static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len, | |||
278 | u64 physical, struct btrfs_device *dev, u64 flags, | 278 | u64 physical, struct btrfs_device *dev, u64 flags, |
279 | u64 gen, int mirror_num, u8 *csum, int force, | 279 | u64 gen, int mirror_num, u8 *csum, int force, |
280 | u64 physical_for_dev_replace); | 280 | u64 physical_for_dev_replace); |
281 | static void scrub_bio_end_io(struct bio *bio, int err); | 281 | static void scrub_bio_end_io(struct bio *bio); |
282 | static void scrub_bio_end_io_worker(struct btrfs_work *work); | 282 | static void scrub_bio_end_io_worker(struct btrfs_work *work); |
283 | static void scrub_block_complete(struct scrub_block *sblock); | 283 | static void scrub_block_complete(struct scrub_block *sblock); |
284 | static void scrub_remap_extent(struct btrfs_fs_info *fs_info, | 284 | static void scrub_remap_extent(struct btrfs_fs_info *fs_info, |
@@ -295,7 +295,7 @@ static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx); | |||
295 | static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx, | 295 | static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx, |
296 | struct scrub_page *spage); | 296 | struct scrub_page *spage); |
297 | static void scrub_wr_submit(struct scrub_ctx *sctx); | 297 | static void scrub_wr_submit(struct scrub_ctx *sctx); |
298 | static void scrub_wr_bio_end_io(struct bio *bio, int err); | 298 | static void scrub_wr_bio_end_io(struct bio *bio); |
299 | static void scrub_wr_bio_end_io_worker(struct btrfs_work *work); | 299 | static void scrub_wr_bio_end_io_worker(struct btrfs_work *work); |
300 | static int write_page_nocow(struct scrub_ctx *sctx, | 300 | static int write_page_nocow(struct scrub_ctx *sctx, |
301 | u64 physical_for_dev_replace, struct page *page); | 301 | u64 physical_for_dev_replace, struct page *page); |
@@ -454,27 +454,14 @@ struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace) | |||
454 | struct scrub_ctx *sctx; | 454 | struct scrub_ctx *sctx; |
455 | int i; | 455 | int i; |
456 | struct btrfs_fs_info *fs_info = dev->dev_root->fs_info; | 456 | struct btrfs_fs_info *fs_info = dev->dev_root->fs_info; |
457 | int pages_per_rd_bio; | ||
458 | int ret; | 457 | int ret; |
459 | 458 | ||
460 | /* | ||
461 | * the setting of pages_per_rd_bio is correct for scrub but might | ||
462 | * be wrong for the dev_replace code where we might read from | ||
463 | * different devices in the initial huge bios. However, that | ||
464 | * code is able to correctly handle the case when adding a page | ||
465 | * to a bio fails. | ||
466 | */ | ||
467 | if (dev->bdev) | ||
468 | pages_per_rd_bio = min_t(int, SCRUB_PAGES_PER_RD_BIO, | ||
469 | bio_get_nr_vecs(dev->bdev)); | ||
470 | else | ||
471 | pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO; | ||
472 | sctx = kzalloc(sizeof(*sctx), GFP_NOFS); | 459 | sctx = kzalloc(sizeof(*sctx), GFP_NOFS); |
473 | if (!sctx) | 460 | if (!sctx) |
474 | goto nomem; | 461 | goto nomem; |
475 | atomic_set(&sctx->refs, 1); | 462 | atomic_set(&sctx->refs, 1); |
476 | sctx->is_dev_replace = is_dev_replace; | 463 | sctx->is_dev_replace = is_dev_replace; |
477 | sctx->pages_per_rd_bio = pages_per_rd_bio; | 464 | sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO; |
478 | sctx->curr = -1; | 465 | sctx->curr = -1; |
479 | sctx->dev_root = dev->dev_root; | 466 | sctx->dev_root = dev->dev_root; |
480 | for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) { | 467 | for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) { |
@@ -1429,11 +1416,11 @@ struct scrub_bio_ret { | |||
1429 | int error; | 1416 | int error; |
1430 | }; | 1417 | }; |
1431 | 1418 | ||
1432 | static void scrub_bio_wait_endio(struct bio *bio, int error) | 1419 | static void scrub_bio_wait_endio(struct bio *bio) |
1433 | { | 1420 | { |
1434 | struct scrub_bio_ret *ret = bio->bi_private; | 1421 | struct scrub_bio_ret *ret = bio->bi_private; |
1435 | 1422 | ||
1436 | ret->error = error; | 1423 | ret->error = bio->bi_error; |
1437 | complete(&ret->event); | 1424 | complete(&ret->event); |
1438 | } | 1425 | } |
1439 | 1426 | ||
@@ -1790,12 +1777,12 @@ static void scrub_wr_submit(struct scrub_ctx *sctx) | |||
1790 | btrfsic_submit_bio(WRITE, sbio->bio); | 1777 | btrfsic_submit_bio(WRITE, sbio->bio); |
1791 | } | 1778 | } |
1792 | 1779 | ||
1793 | static void scrub_wr_bio_end_io(struct bio *bio, int err) | 1780 | static void scrub_wr_bio_end_io(struct bio *bio) |
1794 | { | 1781 | { |
1795 | struct scrub_bio *sbio = bio->bi_private; | 1782 | struct scrub_bio *sbio = bio->bi_private; |
1796 | struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info; | 1783 | struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info; |
1797 | 1784 | ||
1798 | sbio->err = err; | 1785 | sbio->err = bio->bi_error; |
1799 | sbio->bio = bio; | 1786 | sbio->bio = bio; |
1800 | 1787 | ||
1801 | btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper, | 1788 | btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper, |
@@ -2098,7 +2085,7 @@ static void scrub_submit(struct scrub_ctx *sctx) | |||
2098 | */ | 2085 | */ |
2099 | printk_ratelimited(KERN_WARNING | 2086 | printk_ratelimited(KERN_WARNING |
2100 | "BTRFS: scrub_submit(bio bdev == NULL) is unexpected!\n"); | 2087 | "BTRFS: scrub_submit(bio bdev == NULL) is unexpected!\n"); |
2101 | bio_endio(sbio->bio, -EIO); | 2088 | bio_io_error(sbio->bio); |
2102 | } else { | 2089 | } else { |
2103 | btrfsic_submit_bio(READ, sbio->bio); | 2090 | btrfsic_submit_bio(READ, sbio->bio); |
2104 | } | 2091 | } |
@@ -2260,12 +2247,12 @@ leave_nomem: | |||
2260 | return 0; | 2247 | return 0; |
2261 | } | 2248 | } |
2262 | 2249 | ||
2263 | static void scrub_bio_end_io(struct bio *bio, int err) | 2250 | static void scrub_bio_end_io(struct bio *bio) |
2264 | { | 2251 | { |
2265 | struct scrub_bio *sbio = bio->bi_private; | 2252 | struct scrub_bio *sbio = bio->bi_private; |
2266 | struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info; | 2253 | struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info; |
2267 | 2254 | ||
2268 | sbio->err = err; | 2255 | sbio->err = bio->bi_error; |
2269 | sbio->bio = bio; | 2256 | sbio->bio = bio; |
2270 | 2257 | ||
2271 | btrfs_queue_work(fs_info->scrub_workers, &sbio->work); | 2258 | btrfs_queue_work(fs_info->scrub_workers, &sbio->work); |
@@ -2672,11 +2659,11 @@ static void scrub_parity_bio_endio_worker(struct btrfs_work *work) | |||
2672 | scrub_pending_bio_dec(sctx); | 2659 | scrub_pending_bio_dec(sctx); |
2673 | } | 2660 | } |
2674 | 2661 | ||
2675 | static void scrub_parity_bio_endio(struct bio *bio, int error) | 2662 | static void scrub_parity_bio_endio(struct bio *bio) |
2676 | { | 2663 | { |
2677 | struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private; | 2664 | struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private; |
2678 | 2665 | ||
2679 | if (error) | 2666 | if (bio->bi_error) |
2680 | bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap, | 2667 | bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap, |
2681 | sparity->nsectors); | 2668 | sparity->nsectors); |
2682 | 2669 | ||
@@ -3896,8 +3883,7 @@ static int scrub_setup_wr_ctx(struct scrub_ctx *sctx, | |||
3896 | return 0; | 3883 | return 0; |
3897 | 3884 | ||
3898 | WARN_ON(!dev->bdev); | 3885 | WARN_ON(!dev->bdev); |
3899 | wr_ctx->pages_per_wr_bio = min_t(int, SCRUB_PAGES_PER_WR_BIO, | 3886 | wr_ctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO; |
3900 | bio_get_nr_vecs(dev->bdev)); | ||
3901 | wr_ctx->tgtdev = dev; | 3887 | wr_ctx->tgtdev = dev; |
3902 | atomic_set(&wr_ctx->flush_all_writes, 0); | 3888 | atomic_set(&wr_ctx->flush_all_writes, 0); |
3903 | return 0; | 3889 | return 0; |
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index fbe7c104531c..762476f5f08d 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c | |||
@@ -5741,23 +5741,23 @@ int btrfs_rmap_block(struct btrfs_mapping_tree *map_tree, | |||
5741 | return 0; | 5741 | return 0; |
5742 | } | 5742 | } |
5743 | 5743 | ||
5744 | static inline void btrfs_end_bbio(struct btrfs_bio *bbio, struct bio *bio, int err) | 5744 | static inline void btrfs_end_bbio(struct btrfs_bio *bbio, struct bio *bio) |
5745 | { | 5745 | { |
5746 | bio->bi_private = bbio->private; | 5746 | bio->bi_private = bbio->private; |
5747 | bio->bi_end_io = bbio->end_io; | 5747 | bio->bi_end_io = bbio->end_io; |
5748 | bio_endio(bio, err); | 5748 | bio_endio(bio); |
5749 | 5749 | ||
5750 | btrfs_put_bbio(bbio); | 5750 | btrfs_put_bbio(bbio); |
5751 | } | 5751 | } |
5752 | 5752 | ||
5753 | static void btrfs_end_bio(struct bio *bio, int err) | 5753 | static void btrfs_end_bio(struct bio *bio) |
5754 | { | 5754 | { |
5755 | struct btrfs_bio *bbio = bio->bi_private; | 5755 | struct btrfs_bio *bbio = bio->bi_private; |
5756 | int is_orig_bio = 0; | 5756 | int is_orig_bio = 0; |
5757 | 5757 | ||
5758 | if (err) { | 5758 | if (bio->bi_error) { |
5759 | atomic_inc(&bbio->error); | 5759 | atomic_inc(&bbio->error); |
5760 | if (err == -EIO || err == -EREMOTEIO) { | 5760 | if (bio->bi_error == -EIO || bio->bi_error == -EREMOTEIO) { |
5761 | unsigned int stripe_index = | 5761 | unsigned int stripe_index = |
5762 | btrfs_io_bio(bio)->stripe_index; | 5762 | btrfs_io_bio(bio)->stripe_index; |
5763 | struct btrfs_device *dev; | 5763 | struct btrfs_device *dev; |
@@ -5795,17 +5795,16 @@ static void btrfs_end_bio(struct bio *bio, int err) | |||
5795 | * beyond the tolerance of the btrfs bio | 5795 | * beyond the tolerance of the btrfs bio |
5796 | */ | 5796 | */ |
5797 | if (atomic_read(&bbio->error) > bbio->max_errors) { | 5797 | if (atomic_read(&bbio->error) > bbio->max_errors) { |
5798 | err = -EIO; | 5798 | bio->bi_error = -EIO; |
5799 | } else { | 5799 | } else { |
5800 | /* | 5800 | /* |
5801 | * this bio is actually up to date, we didn't | 5801 | * this bio is actually up to date, we didn't |
5802 | * go over the max number of errors | 5802 | * go over the max number of errors |
5803 | */ | 5803 | */ |
5804 | set_bit(BIO_UPTODATE, &bio->bi_flags); | 5804 | bio->bi_error = 0; |
5805 | err = 0; | ||
5806 | } | 5805 | } |
5807 | 5806 | ||
5808 | btrfs_end_bbio(bbio, bio, err); | 5807 | btrfs_end_bbio(bbio, bio); |
5809 | } else if (!is_orig_bio) { | 5808 | } else if (!is_orig_bio) { |
5810 | bio_put(bio); | 5809 | bio_put(bio); |
5811 | } | 5810 | } |
@@ -5826,7 +5825,7 @@ static noinline void btrfs_schedule_bio(struct btrfs_root *root, | |||
5826 | struct btrfs_pending_bios *pending_bios; | 5825 | struct btrfs_pending_bios *pending_bios; |
5827 | 5826 | ||
5828 | if (device->missing || !device->bdev) { | 5827 | if (device->missing || !device->bdev) { |
5829 | bio_endio(bio, -EIO); | 5828 | bio_io_error(bio); |
5830 | return; | 5829 | return; |
5831 | } | 5830 | } |
5832 | 5831 | ||
@@ -5871,34 +5870,6 @@ static noinline void btrfs_schedule_bio(struct btrfs_root *root, | |||
5871 | &device->work); | 5870 | &device->work); |
5872 | } | 5871 | } |
5873 | 5872 | ||
5874 | static int bio_size_ok(struct block_device *bdev, struct bio *bio, | ||
5875 | sector_t sector) | ||
5876 | { | ||
5877 | struct bio_vec *prev; | ||
5878 | struct request_queue *q = bdev_get_queue(bdev); | ||
5879 | unsigned int max_sectors = queue_max_sectors(q); | ||
5880 | struct bvec_merge_data bvm = { | ||
5881 | .bi_bdev = bdev, | ||
5882 | .bi_sector = sector, | ||
5883 | .bi_rw = bio->bi_rw, | ||
5884 | }; | ||
5885 | |||
5886 | if (WARN_ON(bio->bi_vcnt == 0)) | ||
5887 | return 1; | ||
5888 | |||
5889 | prev = &bio->bi_io_vec[bio->bi_vcnt - 1]; | ||
5890 | if (bio_sectors(bio) > max_sectors) | ||
5891 | return 0; | ||
5892 | |||
5893 | if (!q->merge_bvec_fn) | ||
5894 | return 1; | ||
5895 | |||
5896 | bvm.bi_size = bio->bi_iter.bi_size - prev->bv_len; | ||
5897 | if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len) | ||
5898 | return 0; | ||
5899 | return 1; | ||
5900 | } | ||
5901 | |||
5902 | static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio, | 5873 | static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio, |
5903 | struct bio *bio, u64 physical, int dev_nr, | 5874 | struct bio *bio, u64 physical, int dev_nr, |
5904 | int rw, int async) | 5875 | int rw, int async) |
@@ -5932,38 +5903,6 @@ static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio, | |||
5932 | btrfsic_submit_bio(rw, bio); | 5903 | btrfsic_submit_bio(rw, bio); |
5933 | } | 5904 | } |
5934 | 5905 | ||
5935 | static int breakup_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio, | ||
5936 | struct bio *first_bio, struct btrfs_device *dev, | ||
5937 | int dev_nr, int rw, int async) | ||
5938 | { | ||
5939 | struct bio_vec *bvec = first_bio->bi_io_vec; | ||
5940 | struct bio *bio; | ||
5941 | int nr_vecs = bio_get_nr_vecs(dev->bdev); | ||
5942 | u64 physical = bbio->stripes[dev_nr].physical; | ||
5943 | |||
5944 | again: | ||
5945 | bio = btrfs_bio_alloc(dev->bdev, physical >> 9, nr_vecs, GFP_NOFS); | ||
5946 | if (!bio) | ||
5947 | return -ENOMEM; | ||
5948 | |||
5949 | while (bvec <= (first_bio->bi_io_vec + first_bio->bi_vcnt - 1)) { | ||
5950 | if (bio_add_page(bio, bvec->bv_page, bvec->bv_len, | ||
5951 | bvec->bv_offset) < bvec->bv_len) { | ||
5952 | u64 len = bio->bi_iter.bi_size; | ||
5953 | |||
5954 | atomic_inc(&bbio->stripes_pending); | ||
5955 | submit_stripe_bio(root, bbio, bio, physical, dev_nr, | ||
5956 | rw, async); | ||
5957 | physical += len; | ||
5958 | goto again; | ||
5959 | } | ||
5960 | bvec++; | ||
5961 | } | ||
5962 | |||
5963 | submit_stripe_bio(root, bbio, bio, physical, dev_nr, rw, async); | ||
5964 | return 0; | ||
5965 | } | ||
5966 | |||
5967 | static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical) | 5906 | static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical) |
5968 | { | 5907 | { |
5969 | atomic_inc(&bbio->error); | 5908 | atomic_inc(&bbio->error); |
@@ -5973,8 +5912,8 @@ static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical) | |||
5973 | 5912 | ||
5974 | btrfs_io_bio(bio)->mirror_num = bbio->mirror_num; | 5913 | btrfs_io_bio(bio)->mirror_num = bbio->mirror_num; |
5975 | bio->bi_iter.bi_sector = logical >> 9; | 5914 | bio->bi_iter.bi_sector = logical >> 9; |
5976 | 5915 | bio->bi_error = -EIO; | |
5977 | btrfs_end_bbio(bbio, bio, -EIO); | 5916 | btrfs_end_bbio(bbio, bio); |
5978 | } | 5917 | } |
5979 | } | 5918 | } |
5980 | 5919 | ||
@@ -6036,18 +5975,6 @@ int btrfs_map_bio(struct btrfs_root *root, int rw, struct bio *bio, | |||
6036 | continue; | 5975 | continue; |
6037 | } | 5976 | } |
6038 | 5977 | ||
6039 | /* | ||
6040 | * Check and see if we're ok with this bio based on it's size | ||
6041 | * and offset with the given device. | ||
6042 | */ | ||
6043 | if (!bio_size_ok(dev->bdev, first_bio, | ||
6044 | bbio->stripes[dev_nr].physical >> 9)) { | ||
6045 | ret = breakup_stripe_bio(root, bbio, first_bio, dev, | ||
6046 | dev_nr, rw, async_submit); | ||
6047 | BUG_ON(ret); | ||
6048 | continue; | ||
6049 | } | ||
6050 | |||
6051 | if (dev_nr < total_devs - 1) { | 5978 | if (dev_nr < total_devs - 1) { |
6052 | bio = btrfs_bio_clone(first_bio, GFP_NOFS); | 5979 | bio = btrfs_bio_clone(first_bio, GFP_NOFS); |
6053 | BUG_ON(!bio); /* -ENOMEM */ | 5980 | BUG_ON(!bio); /* -ENOMEM */ |
diff --git a/fs/buffer.c b/fs/buffer.c index 1cf7a53a0277..82283abb2795 100644 --- a/fs/buffer.c +++ b/fs/buffer.c | |||
@@ -2957,14 +2957,14 @@ sector_t generic_block_bmap(struct address_space *mapping, sector_t block, | |||
2957 | } | 2957 | } |
2958 | EXPORT_SYMBOL(generic_block_bmap); | 2958 | EXPORT_SYMBOL(generic_block_bmap); |
2959 | 2959 | ||
2960 | static void end_bio_bh_io_sync(struct bio *bio, int err) | 2960 | static void end_bio_bh_io_sync(struct bio *bio) |
2961 | { | 2961 | { |
2962 | struct buffer_head *bh = bio->bi_private; | 2962 | struct buffer_head *bh = bio->bi_private; |
2963 | 2963 | ||
2964 | if (unlikely (test_bit(BIO_QUIET,&bio->bi_flags))) | 2964 | if (unlikely(bio_flagged(bio, BIO_QUIET))) |
2965 | set_bit(BH_Quiet, &bh->b_state); | 2965 | set_bit(BH_Quiet, &bh->b_state); |
2966 | 2966 | ||
2967 | bh->b_end_io(bh, test_bit(BIO_UPTODATE, &bio->bi_flags)); | 2967 | bh->b_end_io(bh, !bio->bi_error); |
2968 | bio_put(bio); | 2968 | bio_put(bio); |
2969 | } | 2969 | } |
2970 | 2970 | ||
@@ -3046,12 +3046,9 @@ static int submit_bh_wbc(int rw, struct buffer_head *bh, | |||
3046 | 3046 | ||
3047 | bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); | 3047 | bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); |
3048 | bio->bi_bdev = bh->b_bdev; | 3048 | bio->bi_bdev = bh->b_bdev; |
3049 | bio->bi_io_vec[0].bv_page = bh->b_page; | ||
3050 | bio->bi_io_vec[0].bv_len = bh->b_size; | ||
3051 | bio->bi_io_vec[0].bv_offset = bh_offset(bh); | ||
3052 | 3049 | ||
3053 | bio->bi_vcnt = 1; | 3050 | bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh)); |
3054 | bio->bi_iter.bi_size = bh->b_size; | 3051 | BUG_ON(bio->bi_iter.bi_size != bh->b_size); |
3055 | 3052 | ||
3056 | bio->bi_end_io = end_bio_bh_io_sync; | 3053 | bio->bi_end_io = end_bio_bh_io_sync; |
3057 | bio->bi_private = bh; | 3054 | bio->bi_private = bh; |
diff --git a/fs/direct-io.c b/fs/direct-io.c index 745d2342651a..11256291642e 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c | |||
@@ -285,7 +285,7 @@ static int dio_bio_complete(struct dio *dio, struct bio *bio); | |||
285 | /* | 285 | /* |
286 | * Asynchronous IO callback. | 286 | * Asynchronous IO callback. |
287 | */ | 287 | */ |
288 | static void dio_bio_end_aio(struct bio *bio, int error) | 288 | static void dio_bio_end_aio(struct bio *bio) |
289 | { | 289 | { |
290 | struct dio *dio = bio->bi_private; | 290 | struct dio *dio = bio->bi_private; |
291 | unsigned long remaining; | 291 | unsigned long remaining; |
@@ -318,7 +318,7 @@ static void dio_bio_end_aio(struct bio *bio, int error) | |||
318 | * During I/O bi_private points at the dio. After I/O, bi_private is used to | 318 | * During I/O bi_private points at the dio. After I/O, bi_private is used to |
319 | * implement a singly-linked list of completed BIOs, at dio->bio_list. | 319 | * implement a singly-linked list of completed BIOs, at dio->bio_list. |
320 | */ | 320 | */ |
321 | static void dio_bio_end_io(struct bio *bio, int error) | 321 | static void dio_bio_end_io(struct bio *bio) |
322 | { | 322 | { |
323 | struct dio *dio = bio->bi_private; | 323 | struct dio *dio = bio->bi_private; |
324 | unsigned long flags; | 324 | unsigned long flags; |
@@ -345,9 +345,9 @@ void dio_end_io(struct bio *bio, int error) | |||
345 | struct dio *dio = bio->bi_private; | 345 | struct dio *dio = bio->bi_private; |
346 | 346 | ||
347 | if (dio->is_async) | 347 | if (dio->is_async) |
348 | dio_bio_end_aio(bio, error); | 348 | dio_bio_end_aio(bio); |
349 | else | 349 | else |
350 | dio_bio_end_io(bio, error); | 350 | dio_bio_end_io(bio); |
351 | } | 351 | } |
352 | EXPORT_SYMBOL_GPL(dio_end_io); | 352 | EXPORT_SYMBOL_GPL(dio_end_io); |
353 | 353 | ||
@@ -457,15 +457,16 @@ static struct bio *dio_await_one(struct dio *dio) | |||
457 | */ | 457 | */ |
458 | static int dio_bio_complete(struct dio *dio, struct bio *bio) | 458 | static int dio_bio_complete(struct dio *dio, struct bio *bio) |
459 | { | 459 | { |
460 | const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
461 | struct bio_vec *bvec; | 460 | struct bio_vec *bvec; |
462 | unsigned i; | 461 | unsigned i; |
462 | int err; | ||
463 | 463 | ||
464 | if (!uptodate) | 464 | if (bio->bi_error) |
465 | dio->io_error = -EIO; | 465 | dio->io_error = -EIO; |
466 | 466 | ||
467 | if (dio->is_async && dio->rw == READ) { | 467 | if (dio->is_async && dio->rw == READ) { |
468 | bio_check_pages_dirty(bio); /* transfers ownership */ | 468 | bio_check_pages_dirty(bio); /* transfers ownership */ |
469 | err = bio->bi_error; | ||
469 | } else { | 470 | } else { |
470 | bio_for_each_segment_all(bvec, bio, i) { | 471 | bio_for_each_segment_all(bvec, bio, i) { |
471 | struct page *page = bvec->bv_page; | 472 | struct page *page = bvec->bv_page; |
@@ -474,9 +475,10 @@ static int dio_bio_complete(struct dio *dio, struct bio *bio) | |||
474 | set_page_dirty_lock(page); | 475 | set_page_dirty_lock(page); |
475 | page_cache_release(page); | 476 | page_cache_release(page); |
476 | } | 477 | } |
478 | err = bio->bi_error; | ||
477 | bio_put(bio); | 479 | bio_put(bio); |
478 | } | 480 | } |
479 | return uptodate ? 0 : -EIO; | 481 | return err; |
480 | } | 482 | } |
481 | 483 | ||
482 | /* | 484 | /* |
@@ -653,7 +655,7 @@ static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, | |||
653 | if (ret) | 655 | if (ret) |
654 | goto out; | 656 | goto out; |
655 | sector = start_sector << (sdio->blkbits - 9); | 657 | sector = start_sector << (sdio->blkbits - 9); |
656 | nr_pages = min(sdio->pages_in_io, bio_get_nr_vecs(map_bh->b_bdev)); | 658 | nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES); |
657 | BUG_ON(nr_pages <= 0); | 659 | BUG_ON(nr_pages <= 0); |
658 | dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); | 660 | dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); |
659 | sdio->boundary = 0; | 661 | sdio->boundary = 0; |
diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index 5602450f03f6..8a9d63a0c071 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c | |||
@@ -61,7 +61,6 @@ static void buffer_io_error(struct buffer_head *bh) | |||
61 | static void ext4_finish_bio(struct bio *bio) | 61 | static void ext4_finish_bio(struct bio *bio) |
62 | { | 62 | { |
63 | int i; | 63 | int i; |
64 | int error = !test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
65 | struct bio_vec *bvec; | 64 | struct bio_vec *bvec; |
66 | 65 | ||
67 | bio_for_each_segment_all(bvec, bio, i) { | 66 | bio_for_each_segment_all(bvec, bio, i) { |
@@ -88,7 +87,7 @@ static void ext4_finish_bio(struct bio *bio) | |||
88 | } | 87 | } |
89 | #endif | 88 | #endif |
90 | 89 | ||
91 | if (error) { | 90 | if (bio->bi_error) { |
92 | SetPageError(page); | 91 | SetPageError(page); |
93 | set_bit(AS_EIO, &page->mapping->flags); | 92 | set_bit(AS_EIO, &page->mapping->flags); |
94 | } | 93 | } |
@@ -107,7 +106,7 @@ static void ext4_finish_bio(struct bio *bio) | |||
107 | continue; | 106 | continue; |
108 | } | 107 | } |
109 | clear_buffer_async_write(bh); | 108 | clear_buffer_async_write(bh); |
110 | if (error) | 109 | if (bio->bi_error) |
111 | buffer_io_error(bh); | 110 | buffer_io_error(bh); |
112 | } while ((bh = bh->b_this_page) != head); | 111 | } while ((bh = bh->b_this_page) != head); |
113 | bit_spin_unlock(BH_Uptodate_Lock, &head->b_state); | 112 | bit_spin_unlock(BH_Uptodate_Lock, &head->b_state); |
@@ -310,27 +309,25 @@ ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end) | |||
310 | } | 309 | } |
311 | 310 | ||
312 | /* BIO completion function for page writeback */ | 311 | /* BIO completion function for page writeback */ |
313 | static void ext4_end_bio(struct bio *bio, int error) | 312 | static void ext4_end_bio(struct bio *bio) |
314 | { | 313 | { |
315 | ext4_io_end_t *io_end = bio->bi_private; | 314 | ext4_io_end_t *io_end = bio->bi_private; |
316 | sector_t bi_sector = bio->bi_iter.bi_sector; | 315 | sector_t bi_sector = bio->bi_iter.bi_sector; |
317 | 316 | ||
318 | BUG_ON(!io_end); | 317 | BUG_ON(!io_end); |
319 | bio->bi_end_io = NULL; | 318 | bio->bi_end_io = NULL; |
320 | if (test_bit(BIO_UPTODATE, &bio->bi_flags)) | ||
321 | error = 0; | ||
322 | 319 | ||
323 | if (error) { | 320 | if (bio->bi_error) { |
324 | struct inode *inode = io_end->inode; | 321 | struct inode *inode = io_end->inode; |
325 | 322 | ||
326 | ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu " | 323 | ext4_warning(inode->i_sb, "I/O error %d writing to inode %lu " |
327 | "(offset %llu size %ld starting block %llu)", | 324 | "(offset %llu size %ld starting block %llu)", |
328 | error, inode->i_ino, | 325 | bio->bi_error, inode->i_ino, |
329 | (unsigned long long) io_end->offset, | 326 | (unsigned long long) io_end->offset, |
330 | (long) io_end->size, | 327 | (long) io_end->size, |
331 | (unsigned long long) | 328 | (unsigned long long) |
332 | bi_sector >> (inode->i_blkbits - 9)); | 329 | bi_sector >> (inode->i_blkbits - 9)); |
333 | mapping_set_error(inode->i_mapping, error); | 330 | mapping_set_error(inode->i_mapping, bio->bi_error); |
334 | } | 331 | } |
335 | 332 | ||
336 | if (io_end->flag & EXT4_IO_END_UNWRITTEN) { | 333 | if (io_end->flag & EXT4_IO_END_UNWRITTEN) { |
@@ -375,10 +372,9 @@ void ext4_io_submit_init(struct ext4_io_submit *io, | |||
375 | static int io_submit_init_bio(struct ext4_io_submit *io, | 372 | static int io_submit_init_bio(struct ext4_io_submit *io, |
376 | struct buffer_head *bh) | 373 | struct buffer_head *bh) |
377 | { | 374 | { |
378 | int nvecs = bio_get_nr_vecs(bh->b_bdev); | ||
379 | struct bio *bio; | 375 | struct bio *bio; |
380 | 376 | ||
381 | bio = bio_alloc(GFP_NOIO, min(nvecs, BIO_MAX_PAGES)); | 377 | bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); |
382 | if (!bio) | 378 | if (!bio) |
383 | return -ENOMEM; | 379 | return -ENOMEM; |
384 | bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); | 380 | bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); |
diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index ec3ef93a52db..e26803fb210d 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c | |||
@@ -98,7 +98,7 @@ static inline bool ext4_bio_encrypted(struct bio *bio) | |||
98 | * status of that page is hard. See end_buffer_async_read() for the details. | 98 | * status of that page is hard. See end_buffer_async_read() for the details. |
99 | * There is no point in duplicating all that complexity. | 99 | * There is no point in duplicating all that complexity. |
100 | */ | 100 | */ |
101 | static void mpage_end_io(struct bio *bio, int err) | 101 | static void mpage_end_io(struct bio *bio) |
102 | { | 102 | { |
103 | struct bio_vec *bv; | 103 | struct bio_vec *bv; |
104 | int i; | 104 | int i; |
@@ -106,7 +106,7 @@ static void mpage_end_io(struct bio *bio, int err) | |||
106 | if (ext4_bio_encrypted(bio)) { | 106 | if (ext4_bio_encrypted(bio)) { |
107 | struct ext4_crypto_ctx *ctx = bio->bi_private; | 107 | struct ext4_crypto_ctx *ctx = bio->bi_private; |
108 | 108 | ||
109 | if (err) { | 109 | if (bio->bi_error) { |
110 | ext4_release_crypto_ctx(ctx); | 110 | ext4_release_crypto_ctx(ctx); |
111 | } else { | 111 | } else { |
112 | INIT_WORK(&ctx->r.work, completion_pages); | 112 | INIT_WORK(&ctx->r.work, completion_pages); |
@@ -118,7 +118,7 @@ static void mpage_end_io(struct bio *bio, int err) | |||
118 | bio_for_each_segment_all(bv, bio, i) { | 118 | bio_for_each_segment_all(bv, bio, i) { |
119 | struct page *page = bv->bv_page; | 119 | struct page *page = bv->bv_page; |
120 | 120 | ||
121 | if (!err) { | 121 | if (!bio->bi_error) { |
122 | SetPageUptodate(page); | 122 | SetPageUptodate(page); |
123 | } else { | 123 | } else { |
124 | ClearPageUptodate(page); | 124 | ClearPageUptodate(page); |
@@ -284,7 +284,7 @@ int ext4_mpage_readpages(struct address_space *mapping, | |||
284 | goto set_error_page; | 284 | goto set_error_page; |
285 | } | 285 | } |
286 | bio = bio_alloc(GFP_KERNEL, | 286 | bio = bio_alloc(GFP_KERNEL, |
287 | min_t(int, nr_pages, bio_get_nr_vecs(bdev))); | 287 | min_t(int, nr_pages, BIO_MAX_PAGES)); |
288 | if (!bio) { | 288 | if (!bio) { |
289 | if (ctx) | 289 | if (ctx) |
290 | ext4_release_crypto_ctx(ctx); | 290 | ext4_release_crypto_ctx(ctx); |
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index f71e19a9dd3c..c414d49aa2de 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c | |||
@@ -29,13 +29,13 @@ | |||
29 | static struct kmem_cache *extent_tree_slab; | 29 | static struct kmem_cache *extent_tree_slab; |
30 | static struct kmem_cache *extent_node_slab; | 30 | static struct kmem_cache *extent_node_slab; |
31 | 31 | ||
32 | static void f2fs_read_end_io(struct bio *bio, int err) | 32 | static void f2fs_read_end_io(struct bio *bio) |
33 | { | 33 | { |
34 | struct bio_vec *bvec; | 34 | struct bio_vec *bvec; |
35 | int i; | 35 | int i; |
36 | 36 | ||
37 | if (f2fs_bio_encrypted(bio)) { | 37 | if (f2fs_bio_encrypted(bio)) { |
38 | if (err) { | 38 | if (bio->bi_error) { |
39 | f2fs_release_crypto_ctx(bio->bi_private); | 39 | f2fs_release_crypto_ctx(bio->bi_private); |
40 | } else { | 40 | } else { |
41 | f2fs_end_io_crypto_work(bio->bi_private, bio); | 41 | f2fs_end_io_crypto_work(bio->bi_private, bio); |
@@ -46,7 +46,7 @@ static void f2fs_read_end_io(struct bio *bio, int err) | |||
46 | bio_for_each_segment_all(bvec, bio, i) { | 46 | bio_for_each_segment_all(bvec, bio, i) { |
47 | struct page *page = bvec->bv_page; | 47 | struct page *page = bvec->bv_page; |
48 | 48 | ||
49 | if (!err) { | 49 | if (!bio->bi_error) { |
50 | SetPageUptodate(page); | 50 | SetPageUptodate(page); |
51 | } else { | 51 | } else { |
52 | ClearPageUptodate(page); | 52 | ClearPageUptodate(page); |
@@ -57,7 +57,7 @@ static void f2fs_read_end_io(struct bio *bio, int err) | |||
57 | bio_put(bio); | 57 | bio_put(bio); |
58 | } | 58 | } |
59 | 59 | ||
60 | static void f2fs_write_end_io(struct bio *bio, int err) | 60 | static void f2fs_write_end_io(struct bio *bio) |
61 | { | 61 | { |
62 | struct f2fs_sb_info *sbi = bio->bi_private; | 62 | struct f2fs_sb_info *sbi = bio->bi_private; |
63 | struct bio_vec *bvec; | 63 | struct bio_vec *bvec; |
@@ -68,7 +68,7 @@ static void f2fs_write_end_io(struct bio *bio, int err) | |||
68 | 68 | ||
69 | f2fs_restore_and_release_control_page(&page); | 69 | f2fs_restore_and_release_control_page(&page); |
70 | 70 | ||
71 | if (unlikely(err)) { | 71 | if (unlikely(bio->bi_error)) { |
72 | set_page_dirty(page); | 72 | set_page_dirty(page); |
73 | set_bit(AS_EIO, &page->mapping->flags); | 73 | set_bit(AS_EIO, &page->mapping->flags); |
74 | f2fs_stop_checkpoint(sbi); | 74 | f2fs_stop_checkpoint(sbi); |
@@ -1552,7 +1552,7 @@ submit_and_realloc: | |||
1552 | } | 1552 | } |
1553 | 1553 | ||
1554 | bio = bio_alloc(GFP_KERNEL, | 1554 | bio = bio_alloc(GFP_KERNEL, |
1555 | min_t(int, nr_pages, bio_get_nr_vecs(bdev))); | 1555 | min_t(int, nr_pages, BIO_MAX_PAGES)); |
1556 | if (!bio) { | 1556 | if (!bio) { |
1557 | if (ctx) | 1557 | if (ctx) |
1558 | f2fs_release_crypto_ctx(ctx); | 1558 | f2fs_release_crypto_ctx(ctx); |
diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c index 2c1ae861dc94..92324ac58290 100644 --- a/fs/gfs2/lops.c +++ b/fs/gfs2/lops.c | |||
@@ -202,22 +202,22 @@ static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp, struct bio_vec *bvec, | |||
202 | * | 202 | * |
203 | */ | 203 | */ |
204 | 204 | ||
205 | static void gfs2_end_log_write(struct bio *bio, int error) | 205 | static void gfs2_end_log_write(struct bio *bio) |
206 | { | 206 | { |
207 | struct gfs2_sbd *sdp = bio->bi_private; | 207 | struct gfs2_sbd *sdp = bio->bi_private; |
208 | struct bio_vec *bvec; | 208 | struct bio_vec *bvec; |
209 | struct page *page; | 209 | struct page *page; |
210 | int i; | 210 | int i; |
211 | 211 | ||
212 | if (error) { | 212 | if (bio->bi_error) { |
213 | sdp->sd_log_error = error; | 213 | sdp->sd_log_error = bio->bi_error; |
214 | fs_err(sdp, "Error %d writing to log\n", error); | 214 | fs_err(sdp, "Error %d writing to log\n", bio->bi_error); |
215 | } | 215 | } |
216 | 216 | ||
217 | bio_for_each_segment_all(bvec, bio, i) { | 217 | bio_for_each_segment_all(bvec, bio, i) { |
218 | page = bvec->bv_page; | 218 | page = bvec->bv_page; |
219 | if (page_has_buffers(page)) | 219 | if (page_has_buffers(page)) |
220 | gfs2_end_log_write_bh(sdp, bvec, error); | 220 | gfs2_end_log_write_bh(sdp, bvec, bio->bi_error); |
221 | else | 221 | else |
222 | mempool_free(page, gfs2_page_pool); | 222 | mempool_free(page, gfs2_page_pool); |
223 | } | 223 | } |
@@ -261,18 +261,11 @@ void gfs2_log_flush_bio(struct gfs2_sbd *sdp, int rw) | |||
261 | static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno) | 261 | static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno) |
262 | { | 262 | { |
263 | struct super_block *sb = sdp->sd_vfs; | 263 | struct super_block *sb = sdp->sd_vfs; |
264 | unsigned nrvecs = bio_get_nr_vecs(sb->s_bdev); | ||
265 | struct bio *bio; | 264 | struct bio *bio; |
266 | 265 | ||
267 | BUG_ON(sdp->sd_log_bio); | 266 | BUG_ON(sdp->sd_log_bio); |
268 | 267 | ||
269 | while (1) { | 268 | bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); |
270 | bio = bio_alloc(GFP_NOIO, nrvecs); | ||
271 | if (likely(bio)) | ||
272 | break; | ||
273 | nrvecs = max(nrvecs/2, 1U); | ||
274 | } | ||
275 | |||
276 | bio->bi_iter.bi_sector = blkno * (sb->s_blocksize >> 9); | 269 | bio->bi_iter.bi_sector = blkno * (sb->s_blocksize >> 9); |
277 | bio->bi_bdev = sb->s_bdev; | 270 | bio->bi_bdev = sb->s_bdev; |
278 | bio->bi_end_io = gfs2_end_log_write; | 271 | bio->bi_end_io = gfs2_end_log_write; |
diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index 1e3a93f2f71d..02586e7eb964 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c | |||
@@ -171,14 +171,14 @@ static int gfs2_check_sb(struct gfs2_sbd *sdp, int silent) | |||
171 | return -EINVAL; | 171 | return -EINVAL; |
172 | } | 172 | } |
173 | 173 | ||
174 | static void end_bio_io_page(struct bio *bio, int error) | 174 | static void end_bio_io_page(struct bio *bio) |
175 | { | 175 | { |
176 | struct page *page = bio->bi_private; | 176 | struct page *page = bio->bi_private; |
177 | 177 | ||
178 | if (!error) | 178 | if (!bio->bi_error) |
179 | SetPageUptodate(page); | 179 | SetPageUptodate(page); |
180 | else | 180 | else |
181 | pr_warn("error %d reading superblock\n", error); | 181 | pr_warn("error %d reading superblock\n", bio->bi_error); |
182 | unlock_page(page); | 182 | unlock_page(page); |
183 | } | 183 | } |
184 | 184 | ||
diff --git a/fs/jfs/jfs_logmgr.c b/fs/jfs/jfs_logmgr.c index bc462dcd7a40..a69bdf2a1085 100644 --- a/fs/jfs/jfs_logmgr.c +++ b/fs/jfs/jfs_logmgr.c | |||
@@ -1999,19 +1999,16 @@ static int lbmRead(struct jfs_log * log, int pn, struct lbuf ** bpp) | |||
1999 | 1999 | ||
2000 | bio->bi_iter.bi_sector = bp->l_blkno << (log->l2bsize - 9); | 2000 | bio->bi_iter.bi_sector = bp->l_blkno << (log->l2bsize - 9); |
2001 | bio->bi_bdev = log->bdev; | 2001 | bio->bi_bdev = log->bdev; |
2002 | bio->bi_io_vec[0].bv_page = bp->l_page; | ||
2003 | bio->bi_io_vec[0].bv_len = LOGPSIZE; | ||
2004 | bio->bi_io_vec[0].bv_offset = bp->l_offset; | ||
2005 | 2002 | ||
2006 | bio->bi_vcnt = 1; | 2003 | bio_add_page(bio, bp->l_page, LOGPSIZE, bp->l_offset); |
2007 | bio->bi_iter.bi_size = LOGPSIZE; | 2004 | BUG_ON(bio->bi_iter.bi_size != LOGPSIZE); |
2008 | 2005 | ||
2009 | bio->bi_end_io = lbmIODone; | 2006 | bio->bi_end_io = lbmIODone; |
2010 | bio->bi_private = bp; | 2007 | bio->bi_private = bp; |
2011 | /*check if journaling to disk has been disabled*/ | 2008 | /*check if journaling to disk has been disabled*/ |
2012 | if (log->no_integrity) { | 2009 | if (log->no_integrity) { |
2013 | bio->bi_iter.bi_size = 0; | 2010 | bio->bi_iter.bi_size = 0; |
2014 | lbmIODone(bio, 0); | 2011 | lbmIODone(bio); |
2015 | } else { | 2012 | } else { |
2016 | submit_bio(READ_SYNC, bio); | 2013 | submit_bio(READ_SYNC, bio); |
2017 | } | 2014 | } |
@@ -2145,12 +2142,9 @@ static void lbmStartIO(struct lbuf * bp) | |||
2145 | bio = bio_alloc(GFP_NOFS, 1); | 2142 | bio = bio_alloc(GFP_NOFS, 1); |
2146 | bio->bi_iter.bi_sector = bp->l_blkno << (log->l2bsize - 9); | 2143 | bio->bi_iter.bi_sector = bp->l_blkno << (log->l2bsize - 9); |
2147 | bio->bi_bdev = log->bdev; | 2144 | bio->bi_bdev = log->bdev; |
2148 | bio->bi_io_vec[0].bv_page = bp->l_page; | ||
2149 | bio->bi_io_vec[0].bv_len = LOGPSIZE; | ||
2150 | bio->bi_io_vec[0].bv_offset = bp->l_offset; | ||
2151 | 2145 | ||
2152 | bio->bi_vcnt = 1; | 2146 | bio_add_page(bio, bp->l_page, LOGPSIZE, bp->l_offset); |
2153 | bio->bi_iter.bi_size = LOGPSIZE; | 2147 | BUG_ON(bio->bi_iter.bi_size != LOGPSIZE); |
2154 | 2148 | ||
2155 | bio->bi_end_io = lbmIODone; | 2149 | bio->bi_end_io = lbmIODone; |
2156 | bio->bi_private = bp; | 2150 | bio->bi_private = bp; |
@@ -2158,7 +2152,7 @@ static void lbmStartIO(struct lbuf * bp) | |||
2158 | /* check if journaling to disk has been disabled */ | 2152 | /* check if journaling to disk has been disabled */ |
2159 | if (log->no_integrity) { | 2153 | if (log->no_integrity) { |
2160 | bio->bi_iter.bi_size = 0; | 2154 | bio->bi_iter.bi_size = 0; |
2161 | lbmIODone(bio, 0); | 2155 | lbmIODone(bio); |
2162 | } else { | 2156 | } else { |
2163 | submit_bio(WRITE_SYNC, bio); | 2157 | submit_bio(WRITE_SYNC, bio); |
2164 | INCREMENT(lmStat.submitted); | 2158 | INCREMENT(lmStat.submitted); |
@@ -2196,7 +2190,7 @@ static int lbmIOWait(struct lbuf * bp, int flag) | |||
2196 | * | 2190 | * |
2197 | * executed at INTIODONE level | 2191 | * executed at INTIODONE level |
2198 | */ | 2192 | */ |
2199 | static void lbmIODone(struct bio *bio, int error) | 2193 | static void lbmIODone(struct bio *bio) |
2200 | { | 2194 | { |
2201 | struct lbuf *bp = bio->bi_private; | 2195 | struct lbuf *bp = bio->bi_private; |
2202 | struct lbuf *nextbp, *tail; | 2196 | struct lbuf *nextbp, *tail; |
@@ -2212,7 +2206,7 @@ static void lbmIODone(struct bio *bio, int error) | |||
2212 | 2206 | ||
2213 | bp->l_flag |= lbmDONE; | 2207 | bp->l_flag |= lbmDONE; |
2214 | 2208 | ||
2215 | if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) { | 2209 | if (bio->bi_error) { |
2216 | bp->l_flag |= lbmERROR; | 2210 | bp->l_flag |= lbmERROR; |
2217 | 2211 | ||
2218 | jfs_err("lbmIODone: I/O error in JFS log"); | 2212 | jfs_err("lbmIODone: I/O error in JFS log"); |
diff --git a/fs/jfs/jfs_metapage.c b/fs/jfs/jfs_metapage.c index 16a0922beb59..a3eb316b1ac3 100644 --- a/fs/jfs/jfs_metapage.c +++ b/fs/jfs/jfs_metapage.c | |||
@@ -276,11 +276,11 @@ static void last_read_complete(struct page *page) | |||
276 | unlock_page(page); | 276 | unlock_page(page); |
277 | } | 277 | } |
278 | 278 | ||
279 | static void metapage_read_end_io(struct bio *bio, int err) | 279 | static void metapage_read_end_io(struct bio *bio) |
280 | { | 280 | { |
281 | struct page *page = bio->bi_private; | 281 | struct page *page = bio->bi_private; |
282 | 282 | ||
283 | if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) { | 283 | if (bio->bi_error) { |
284 | printk(KERN_ERR "metapage_read_end_io: I/O error\n"); | 284 | printk(KERN_ERR "metapage_read_end_io: I/O error\n"); |
285 | SetPageError(page); | 285 | SetPageError(page); |
286 | } | 286 | } |
@@ -331,13 +331,13 @@ static void last_write_complete(struct page *page) | |||
331 | end_page_writeback(page); | 331 | end_page_writeback(page); |
332 | } | 332 | } |
333 | 333 | ||
334 | static void metapage_write_end_io(struct bio *bio, int err) | 334 | static void metapage_write_end_io(struct bio *bio) |
335 | { | 335 | { |
336 | struct page *page = bio->bi_private; | 336 | struct page *page = bio->bi_private; |
337 | 337 | ||
338 | BUG_ON(!PagePrivate(page)); | 338 | BUG_ON(!PagePrivate(page)); |
339 | 339 | ||
340 | if (! test_bit(BIO_UPTODATE, &bio->bi_flags)) { | 340 | if (bio->bi_error) { |
341 | printk(KERN_ERR "metapage_write_end_io: I/O error\n"); | 341 | printk(KERN_ERR "metapage_write_end_io: I/O error\n"); |
342 | SetPageError(page); | 342 | SetPageError(page); |
343 | } | 343 | } |
diff --git a/fs/logfs/dev_bdev.c b/fs/logfs/dev_bdev.c index 76279e11982d..a7fdbd868474 100644 --- a/fs/logfs/dev_bdev.c +++ b/fs/logfs/dev_bdev.c | |||
@@ -53,16 +53,14 @@ static int bdev_readpage(void *_sb, struct page *page) | |||
53 | 53 | ||
54 | static DECLARE_WAIT_QUEUE_HEAD(wq); | 54 | static DECLARE_WAIT_QUEUE_HEAD(wq); |
55 | 55 | ||
56 | static void writeseg_end_io(struct bio *bio, int err) | 56 | static void writeseg_end_io(struct bio *bio) |
57 | { | 57 | { |
58 | const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
59 | struct bio_vec *bvec; | 58 | struct bio_vec *bvec; |
60 | int i; | 59 | int i; |
61 | struct super_block *sb = bio->bi_private; | 60 | struct super_block *sb = bio->bi_private; |
62 | struct logfs_super *super = logfs_super(sb); | 61 | struct logfs_super *super = logfs_super(sb); |
63 | 62 | ||
64 | BUG_ON(!uptodate); /* FIXME: Retry io or write elsewhere */ | 63 | BUG_ON(bio->bi_error); /* FIXME: Retry io or write elsewhere */ |
65 | BUG_ON(err); | ||
66 | 64 | ||
67 | bio_for_each_segment_all(bvec, bio, i) { | 65 | bio_for_each_segment_all(bvec, bio, i) { |
68 | end_page_writeback(bvec->bv_page); | 66 | end_page_writeback(bvec->bv_page); |
@@ -83,7 +81,7 @@ static int __bdev_writeseg(struct super_block *sb, u64 ofs, pgoff_t index, | |||
83 | unsigned int max_pages; | 81 | unsigned int max_pages; |
84 | int i; | 82 | int i; |
85 | 83 | ||
86 | max_pages = min(nr_pages, (size_t) bio_get_nr_vecs(super->s_bdev)); | 84 | max_pages = min(nr_pages, BIO_MAX_PAGES); |
87 | 85 | ||
88 | bio = bio_alloc(GFP_NOFS, max_pages); | 86 | bio = bio_alloc(GFP_NOFS, max_pages); |
89 | BUG_ON(!bio); | 87 | BUG_ON(!bio); |
@@ -153,14 +151,12 @@ static void bdev_writeseg(struct super_block *sb, u64 ofs, size_t len) | |||
153 | } | 151 | } |
154 | 152 | ||
155 | 153 | ||
156 | static void erase_end_io(struct bio *bio, int err) | 154 | static void erase_end_io(struct bio *bio) |
157 | { | 155 | { |
158 | const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
159 | struct super_block *sb = bio->bi_private; | 156 | struct super_block *sb = bio->bi_private; |
160 | struct logfs_super *super = logfs_super(sb); | 157 | struct logfs_super *super = logfs_super(sb); |
161 | 158 | ||
162 | BUG_ON(!uptodate); /* FIXME: Retry io or write elsewhere */ | 159 | BUG_ON(bio->bi_error); /* FIXME: Retry io or write elsewhere */ |
163 | BUG_ON(err); | ||
164 | BUG_ON(bio->bi_vcnt == 0); | 160 | BUG_ON(bio->bi_vcnt == 0); |
165 | bio_put(bio); | 161 | bio_put(bio); |
166 | if (atomic_dec_and_test(&super->s_pending_writes)) | 162 | if (atomic_dec_and_test(&super->s_pending_writes)) |
@@ -175,7 +171,7 @@ static int do_erase(struct super_block *sb, u64 ofs, pgoff_t index, | |||
175 | unsigned int max_pages; | 171 | unsigned int max_pages; |
176 | int i; | 172 | int i; |
177 | 173 | ||
178 | max_pages = min(nr_pages, (size_t) bio_get_nr_vecs(super->s_bdev)); | 174 | max_pages = min(nr_pages, BIO_MAX_PAGES); |
179 | 175 | ||
180 | bio = bio_alloc(GFP_NOFS, max_pages); | 176 | bio = bio_alloc(GFP_NOFS, max_pages); |
181 | BUG_ON(!bio); | 177 | BUG_ON(!bio); |
diff --git a/fs/mpage.c b/fs/mpage.c index ca0244b69de8..778a4ddef77a 100644 --- a/fs/mpage.c +++ b/fs/mpage.c | |||
@@ -42,14 +42,14 @@ | |||
42 | * status of that page is hard. See end_buffer_async_read() for the details. | 42 | * status of that page is hard. See end_buffer_async_read() for the details. |
43 | * There is no point in duplicating all that complexity. | 43 | * There is no point in duplicating all that complexity. |
44 | */ | 44 | */ |
45 | static void mpage_end_io(struct bio *bio, int err) | 45 | static void mpage_end_io(struct bio *bio) |
46 | { | 46 | { |
47 | struct bio_vec *bv; | 47 | struct bio_vec *bv; |
48 | int i; | 48 | int i; |
49 | 49 | ||
50 | bio_for_each_segment_all(bv, bio, i) { | 50 | bio_for_each_segment_all(bv, bio, i) { |
51 | struct page *page = bv->bv_page; | 51 | struct page *page = bv->bv_page; |
52 | page_endio(page, bio_data_dir(bio), err); | 52 | page_endio(page, bio_data_dir(bio), bio->bi_error); |
53 | } | 53 | } |
54 | 54 | ||
55 | bio_put(bio); | 55 | bio_put(bio); |
@@ -277,7 +277,7 @@ alloc_new: | |||
277 | goto out; | 277 | goto out; |
278 | } | 278 | } |
279 | bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9), | 279 | bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9), |
280 | min_t(int, nr_pages, bio_get_nr_vecs(bdev)), | 280 | min_t(int, nr_pages, BIO_MAX_PAGES), |
281 | GFP_KERNEL); | 281 | GFP_KERNEL); |
282 | if (bio == NULL) | 282 | if (bio == NULL) |
283 | goto confused; | 283 | goto confused; |
@@ -602,7 +602,7 @@ alloc_new: | |||
602 | } | 602 | } |
603 | } | 603 | } |
604 | bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9), | 604 | bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9), |
605 | bio_get_nr_vecs(bdev), GFP_NOFS|__GFP_HIGH); | 605 | BIO_MAX_PAGES, GFP_NOFS|__GFP_HIGH); |
606 | if (bio == NULL) | 606 | if (bio == NULL) |
607 | goto confused; | 607 | goto confused; |
608 | 608 | ||
diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c index d2554fe140a3..9cd4eb3a1e22 100644 --- a/fs/nfs/blocklayout/blocklayout.c +++ b/fs/nfs/blocklayout/blocklayout.c | |||
@@ -116,7 +116,7 @@ bl_submit_bio(int rw, struct bio *bio) | |||
116 | 116 | ||
117 | static struct bio * | 117 | static struct bio * |
118 | bl_alloc_init_bio(int npg, struct block_device *bdev, sector_t disk_sector, | 118 | bl_alloc_init_bio(int npg, struct block_device *bdev, sector_t disk_sector, |
119 | void (*end_io)(struct bio *, int err), struct parallel_io *par) | 119 | bio_end_io_t end_io, struct parallel_io *par) |
120 | { | 120 | { |
121 | struct bio *bio; | 121 | struct bio *bio; |
122 | 122 | ||
@@ -139,8 +139,7 @@ bl_alloc_init_bio(int npg, struct block_device *bdev, sector_t disk_sector, | |||
139 | static struct bio * | 139 | static struct bio * |
140 | do_add_page_to_bio(struct bio *bio, int npg, int rw, sector_t isect, | 140 | do_add_page_to_bio(struct bio *bio, int npg, int rw, sector_t isect, |
141 | struct page *page, struct pnfs_block_dev_map *map, | 141 | struct page *page, struct pnfs_block_dev_map *map, |
142 | struct pnfs_block_extent *be, | 142 | struct pnfs_block_extent *be, bio_end_io_t end_io, |
143 | void (*end_io)(struct bio *, int err), | ||
144 | struct parallel_io *par, unsigned int offset, int *len) | 143 | struct parallel_io *par, unsigned int offset, int *len) |
145 | { | 144 | { |
146 | struct pnfs_block_dev *dev = | 145 | struct pnfs_block_dev *dev = |
@@ -183,11 +182,11 @@ retry: | |||
183 | return bio; | 182 | return bio; |
184 | } | 183 | } |
185 | 184 | ||
186 | static void bl_end_io_read(struct bio *bio, int err) | 185 | static void bl_end_io_read(struct bio *bio) |
187 | { | 186 | { |
188 | struct parallel_io *par = bio->bi_private; | 187 | struct parallel_io *par = bio->bi_private; |
189 | 188 | ||
190 | if (err) { | 189 | if (bio->bi_error) { |
191 | struct nfs_pgio_header *header = par->data; | 190 | struct nfs_pgio_header *header = par->data; |
192 | 191 | ||
193 | if (!header->pnfs_error) | 192 | if (!header->pnfs_error) |
@@ -316,13 +315,12 @@ out: | |||
316 | return PNFS_ATTEMPTED; | 315 | return PNFS_ATTEMPTED; |
317 | } | 316 | } |
318 | 317 | ||
319 | static void bl_end_io_write(struct bio *bio, int err) | 318 | static void bl_end_io_write(struct bio *bio) |
320 | { | 319 | { |
321 | struct parallel_io *par = bio->bi_private; | 320 | struct parallel_io *par = bio->bi_private; |
322 | const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
323 | struct nfs_pgio_header *header = par->data; | 321 | struct nfs_pgio_header *header = par->data; |
324 | 322 | ||
325 | if (!uptodate) { | 323 | if (bio->bi_error) { |
326 | if (!header->pnfs_error) | 324 | if (!header->pnfs_error) |
327 | header->pnfs_error = -EIO; | 325 | header->pnfs_error = -EIO; |
328 | pnfs_set_lo_fail(header->lseg); | 326 | pnfs_set_lo_fail(header->lseg); |
diff --git a/fs/nilfs2/segbuf.c b/fs/nilfs2/segbuf.c index 42468e5ab3e7..f63620ce3892 100644 --- a/fs/nilfs2/segbuf.c +++ b/fs/nilfs2/segbuf.c | |||
@@ -338,12 +338,11 @@ void nilfs_add_checksums_on_logs(struct list_head *logs, u32 seed) | |||
338 | /* | 338 | /* |
339 | * BIO operations | 339 | * BIO operations |
340 | */ | 340 | */ |
341 | static void nilfs_end_bio_write(struct bio *bio, int err) | 341 | static void nilfs_end_bio_write(struct bio *bio) |
342 | { | 342 | { |
343 | const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
344 | struct nilfs_segment_buffer *segbuf = bio->bi_private; | 343 | struct nilfs_segment_buffer *segbuf = bio->bi_private; |
345 | 344 | ||
346 | if (!uptodate) | 345 | if (bio->bi_error) |
347 | atomic_inc(&segbuf->sb_err); | 346 | atomic_inc(&segbuf->sb_err); |
348 | 347 | ||
349 | bio_put(bio); | 348 | bio_put(bio); |
@@ -415,7 +414,7 @@ static void nilfs_segbuf_prepare_write(struct nilfs_segment_buffer *segbuf, | |||
415 | { | 414 | { |
416 | wi->bio = NULL; | 415 | wi->bio = NULL; |
417 | wi->rest_blocks = segbuf->sb_sum.nblocks; | 416 | wi->rest_blocks = segbuf->sb_sum.nblocks; |
418 | wi->max_pages = bio_get_nr_vecs(wi->nilfs->ns_bdev); | 417 | wi->max_pages = BIO_MAX_PAGES; |
419 | wi->nr_vecs = min(wi->max_pages, wi->rest_blocks); | 418 | wi->nr_vecs = min(wi->max_pages, wi->rest_blocks); |
420 | wi->start = wi->end = 0; | 419 | wi->start = wi->end = 0; |
421 | wi->blocknr = segbuf->sb_pseg_start; | 420 | wi->blocknr = segbuf->sb_pseg_start; |
diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c index 16eff45727ee..140de3c93d2e 100644 --- a/fs/ocfs2/cluster/heartbeat.c +++ b/fs/ocfs2/cluster/heartbeat.c | |||
@@ -372,14 +372,13 @@ static void o2hb_wait_on_io(struct o2hb_region *reg, | |||
372 | wait_for_completion(&wc->wc_io_complete); | 372 | wait_for_completion(&wc->wc_io_complete); |
373 | } | 373 | } |
374 | 374 | ||
375 | static void o2hb_bio_end_io(struct bio *bio, | 375 | static void o2hb_bio_end_io(struct bio *bio) |
376 | int error) | ||
377 | { | 376 | { |
378 | struct o2hb_bio_wait_ctxt *wc = bio->bi_private; | 377 | struct o2hb_bio_wait_ctxt *wc = bio->bi_private; |
379 | 378 | ||
380 | if (error) { | 379 | if (bio->bi_error) { |
381 | mlog(ML_ERROR, "IO Error %d\n", error); | 380 | mlog(ML_ERROR, "IO Error %d\n", bio->bi_error); |
382 | wc->wc_error = error; | 381 | wc->wc_error = bio->bi_error; |
383 | } | 382 | } |
384 | 383 | ||
385 | o2hb_bio_wait_dec(wc, 1); | 384 | o2hb_bio_wait_dec(wc, 1); |
diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index 3859f5e27a4d..c77499bcbd7a 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c | |||
@@ -351,12 +351,11 @@ xfs_imap_valid( | |||
351 | */ | 351 | */ |
352 | STATIC void | 352 | STATIC void |
353 | xfs_end_bio( | 353 | xfs_end_bio( |
354 | struct bio *bio, | 354 | struct bio *bio) |
355 | int error) | ||
356 | { | 355 | { |
357 | xfs_ioend_t *ioend = bio->bi_private; | 356 | xfs_ioend_t *ioend = bio->bi_private; |
358 | 357 | ||
359 | ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error; | 358 | ioend->io_error = bio->bi_error; |
360 | 359 | ||
361 | /* Toss bio and pass work off to an xfsdatad thread */ | 360 | /* Toss bio and pass work off to an xfsdatad thread */ |
362 | bio->bi_private = NULL; | 361 | bio->bi_private = NULL; |
@@ -382,8 +381,7 @@ STATIC struct bio * | |||
382 | xfs_alloc_ioend_bio( | 381 | xfs_alloc_ioend_bio( |
383 | struct buffer_head *bh) | 382 | struct buffer_head *bh) |
384 | { | 383 | { |
385 | int nvecs = bio_get_nr_vecs(bh->b_bdev); | 384 | struct bio *bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES); |
386 | struct bio *bio = bio_alloc(GFP_NOIO, nvecs); | ||
387 | 385 | ||
388 | ASSERT(bio->bi_private == NULL); | 386 | ASSERT(bio->bi_private == NULL); |
389 | bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); | 387 | bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); |
diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index a4b7d92e946c..01bd6781974e 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c | |||
@@ -1096,8 +1096,7 @@ xfs_bwrite( | |||
1096 | 1096 | ||
1097 | STATIC void | 1097 | STATIC void |
1098 | xfs_buf_bio_end_io( | 1098 | xfs_buf_bio_end_io( |
1099 | struct bio *bio, | 1099 | struct bio *bio) |
1100 | int error) | ||
1101 | { | 1100 | { |
1102 | xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private; | 1101 | xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private; |
1103 | 1102 | ||
@@ -1105,10 +1104,10 @@ xfs_buf_bio_end_io( | |||
1105 | * don't overwrite existing errors - otherwise we can lose errors on | 1104 | * don't overwrite existing errors - otherwise we can lose errors on |
1106 | * buffers that require multiple bios to complete. | 1105 | * buffers that require multiple bios to complete. |
1107 | */ | 1106 | */ |
1108 | if (error) { | 1107 | if (bio->bi_error) { |
1109 | spin_lock(&bp->b_lock); | 1108 | spin_lock(&bp->b_lock); |
1110 | if (!bp->b_io_error) | 1109 | if (!bp->b_io_error) |
1111 | bp->b_io_error = error; | 1110 | bp->b_io_error = bio->bi_error; |
1112 | spin_unlock(&bp->b_lock); | 1111 | spin_unlock(&bp->b_lock); |
1113 | } | 1112 | } |
1114 | 1113 | ||
diff --git a/include/linux/bio.h b/include/linux/bio.h index 5e963a6d7c14..b9b6e046b52e 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h | |||
@@ -187,17 +187,6 @@ static inline void *bio_data(struct bio *bio) | |||
187 | __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q))) | 187 | __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q))) |
188 | 188 | ||
189 | /* | 189 | /* |
190 | * Check if adding a bio_vec after bprv with offset would create a gap in | ||
191 | * the SG list. Most drivers don't care about this, but some do. | ||
192 | */ | ||
193 | static inline bool bvec_gap_to_prev(struct bio_vec *bprv, unsigned int offset) | ||
194 | { | ||
195 | return offset || ((bprv->bv_offset + bprv->bv_len) & (PAGE_SIZE - 1)); | ||
196 | } | ||
197 | |||
198 | #define bio_io_error(bio) bio_endio((bio), -EIO) | ||
199 | |||
200 | /* | ||
201 | * drivers should _never_ use the all version - the bio may have been split | 190 | * drivers should _never_ use the all version - the bio may have been split |
202 | * before it got to the driver and the driver won't own all of it | 191 | * before it got to the driver and the driver won't own all of it |
203 | */ | 192 | */ |
@@ -306,6 +295,21 @@ static inline void bio_cnt_set(struct bio *bio, unsigned int count) | |||
306 | atomic_set(&bio->__bi_cnt, count); | 295 | atomic_set(&bio->__bi_cnt, count); |
307 | } | 296 | } |
308 | 297 | ||
298 | static inline bool bio_flagged(struct bio *bio, unsigned int bit) | ||
299 | { | ||
300 | return (bio->bi_flags & (1U << bit)) != 0; | ||
301 | } | ||
302 | |||
303 | static inline void bio_set_flag(struct bio *bio, unsigned int bit) | ||
304 | { | ||
305 | bio->bi_flags |= (1U << bit); | ||
306 | } | ||
307 | |||
308 | static inline void bio_clear_flag(struct bio *bio, unsigned int bit) | ||
309 | { | ||
310 | bio->bi_flags &= ~(1U << bit); | ||
311 | } | ||
312 | |||
309 | enum bip_flags { | 313 | enum bip_flags { |
310 | BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */ | 314 | BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */ |
311 | BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */ | 315 | BIP_MAPPED_INTEGRITY = 1 << 1, /* ref tag has been remapped */ |
@@ -426,7 +430,14 @@ static inline struct bio *bio_clone_kmalloc(struct bio *bio, gfp_t gfp_mask) | |||
426 | 430 | ||
427 | } | 431 | } |
428 | 432 | ||
429 | extern void bio_endio(struct bio *, int); | 433 | extern void bio_endio(struct bio *); |
434 | |||
435 | static inline void bio_io_error(struct bio *bio) | ||
436 | { | ||
437 | bio->bi_error = -EIO; | ||
438 | bio_endio(bio); | ||
439 | } | ||
440 | |||
430 | struct request_queue; | 441 | struct request_queue; |
431 | extern int bio_phys_segments(struct request_queue *, struct bio *); | 442 | extern int bio_phys_segments(struct request_queue *, struct bio *); |
432 | 443 | ||
@@ -440,7 +451,6 @@ void bio_chain(struct bio *, struct bio *); | |||
440 | extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int); | 451 | extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int); |
441 | extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *, | 452 | extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *, |
442 | unsigned int, unsigned int); | 453 | unsigned int, unsigned int); |
443 | extern int bio_get_nr_vecs(struct block_device *); | ||
444 | struct rq_map_data; | 454 | struct rq_map_data; |
445 | extern struct bio *bio_map_user_iov(struct request_queue *, | 455 | extern struct bio *bio_map_user_iov(struct request_queue *, |
446 | const struct iov_iter *, gfp_t); | 456 | const struct iov_iter *, gfp_t); |
@@ -717,7 +727,7 @@ extern void bio_integrity_free(struct bio *); | |||
717 | extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int); | 727 | extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int); |
718 | extern bool bio_integrity_enabled(struct bio *bio); | 728 | extern bool bio_integrity_enabled(struct bio *bio); |
719 | extern int bio_integrity_prep(struct bio *); | 729 | extern int bio_integrity_prep(struct bio *); |
720 | extern void bio_integrity_endio(struct bio *, int); | 730 | extern void bio_integrity_endio(struct bio *); |
721 | extern void bio_integrity_advance(struct bio *, unsigned int); | 731 | extern void bio_integrity_advance(struct bio *, unsigned int); |
722 | extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int); | 732 | extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int); |
723 | extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t); | 733 | extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t); |
diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h index 1b62d768c7df..a4cd1641e9e2 100644 --- a/include/linux/blk-cgroup.h +++ b/include/linux/blk-cgroup.h | |||
@@ -374,7 +374,7 @@ static inline struct request_list *blk_get_rl(struct request_queue *q, | |||
374 | * root_rl in such cases. | 374 | * root_rl in such cases. |
375 | */ | 375 | */ |
376 | blkg = blkg_lookup_create(blkcg, q); | 376 | blkg = blkg_lookup_create(blkcg, q); |
377 | if (unlikely(IS_ERR(blkg))) | 377 | if (IS_ERR(blkg)) |
378 | goto root_rl; | 378 | goto root_rl; |
379 | 379 | ||
380 | blkg_get(blkg); | 380 | blkg_get(blkg); |
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 7303b3405520..4b7b4ebaa633 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h | |||
@@ -14,7 +14,7 @@ struct page; | |||
14 | struct block_device; | 14 | struct block_device; |
15 | struct io_context; | 15 | struct io_context; |
16 | struct cgroup_subsys_state; | 16 | struct cgroup_subsys_state; |
17 | typedef void (bio_end_io_t) (struct bio *, int); | 17 | typedef void (bio_end_io_t) (struct bio *); |
18 | typedef void (bio_destructor_t) (struct bio *); | 18 | typedef void (bio_destructor_t) (struct bio *); |
19 | 19 | ||
20 | /* | 20 | /* |
@@ -46,7 +46,8 @@ struct bvec_iter { | |||
46 | struct bio { | 46 | struct bio { |
47 | struct bio *bi_next; /* request queue link */ | 47 | struct bio *bi_next; /* request queue link */ |
48 | struct block_device *bi_bdev; | 48 | struct block_device *bi_bdev; |
49 | unsigned long bi_flags; /* status, command, etc */ | 49 | unsigned int bi_flags; /* status, command, etc */ |
50 | int bi_error; | ||
50 | unsigned long bi_rw; /* bottom bits READ/WRITE, | 51 | unsigned long bi_rw; /* bottom bits READ/WRITE, |
51 | * top bits priority | 52 | * top bits priority |
52 | */ | 53 | */ |
@@ -111,7 +112,6 @@ struct bio { | |||
111 | /* | 112 | /* |
112 | * bio flags | 113 | * bio flags |
113 | */ | 114 | */ |
114 | #define BIO_UPTODATE 0 /* ok after I/O completion */ | ||
115 | #define BIO_SEG_VALID 1 /* bi_phys_segments valid */ | 115 | #define BIO_SEG_VALID 1 /* bi_phys_segments valid */ |
116 | #define BIO_CLONED 2 /* doesn't own data */ | 116 | #define BIO_CLONED 2 /* doesn't own data */ |
117 | #define BIO_BOUNCED 3 /* bio is a bounce bio */ | 117 | #define BIO_BOUNCED 3 /* bio is a bounce bio */ |
@@ -129,14 +129,12 @@ struct bio { | |||
129 | #define BIO_RESET_BITS 13 | 129 | #define BIO_RESET_BITS 13 |
130 | #define BIO_OWNS_VEC 13 /* bio_free() should free bvec */ | 130 | #define BIO_OWNS_VEC 13 /* bio_free() should free bvec */ |
131 | 131 | ||
132 | #define bio_flagged(bio, flag) ((bio)->bi_flags & (1 << (flag))) | ||
133 | |||
134 | /* | 132 | /* |
135 | * top 4 bits of bio flags indicate the pool this bio came from | 133 | * top 4 bits of bio flags indicate the pool this bio came from |
136 | */ | 134 | */ |
137 | #define BIO_POOL_BITS (4) | 135 | #define BIO_POOL_BITS (4) |
138 | #define BIO_POOL_NONE ((1UL << BIO_POOL_BITS) - 1) | 136 | #define BIO_POOL_NONE ((1UL << BIO_POOL_BITS) - 1) |
139 | #define BIO_POOL_OFFSET (BITS_PER_LONG - BIO_POOL_BITS) | 137 | #define BIO_POOL_OFFSET (32 - BIO_POOL_BITS) |
140 | #define BIO_POOL_MASK (1UL << BIO_POOL_OFFSET) | 138 | #define BIO_POOL_MASK (1UL << BIO_POOL_OFFSET) |
141 | #define BIO_POOL_IDX(bio) ((bio)->bi_flags >> BIO_POOL_OFFSET) | 139 | #define BIO_POOL_IDX(bio) ((bio)->bi_flags >> BIO_POOL_OFFSET) |
142 | 140 | ||
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d4068c17d0df..a622f270f09e 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h | |||
@@ -213,14 +213,6 @@ typedef int (prep_rq_fn) (struct request_queue *, struct request *); | |||
213 | typedef void (unprep_rq_fn) (struct request_queue *, struct request *); | 213 | typedef void (unprep_rq_fn) (struct request_queue *, struct request *); |
214 | 214 | ||
215 | struct bio_vec; | 215 | struct bio_vec; |
216 | struct bvec_merge_data { | ||
217 | struct block_device *bi_bdev; | ||
218 | sector_t bi_sector; | ||
219 | unsigned bi_size; | ||
220 | unsigned long bi_rw; | ||
221 | }; | ||
222 | typedef int (merge_bvec_fn) (struct request_queue *, struct bvec_merge_data *, | ||
223 | struct bio_vec *); | ||
224 | typedef void (softirq_done_fn)(struct request *); | 216 | typedef void (softirq_done_fn)(struct request *); |
225 | typedef int (dma_drain_needed_fn)(struct request *); | 217 | typedef int (dma_drain_needed_fn)(struct request *); |
226 | typedef int (lld_busy_fn) (struct request_queue *q); | 218 | typedef int (lld_busy_fn) (struct request_queue *q); |
@@ -258,6 +250,7 @@ struct blk_queue_tag { | |||
258 | struct queue_limits { | 250 | struct queue_limits { |
259 | unsigned long bounce_pfn; | 251 | unsigned long bounce_pfn; |
260 | unsigned long seg_boundary_mask; | 252 | unsigned long seg_boundary_mask; |
253 | unsigned long virt_boundary_mask; | ||
261 | 254 | ||
262 | unsigned int max_hw_sectors; | 255 | unsigned int max_hw_sectors; |
263 | unsigned int chunk_sectors; | 256 | unsigned int chunk_sectors; |
@@ -268,6 +261,7 @@ struct queue_limits { | |||
268 | unsigned int io_min; | 261 | unsigned int io_min; |
269 | unsigned int io_opt; | 262 | unsigned int io_opt; |
270 | unsigned int max_discard_sectors; | 263 | unsigned int max_discard_sectors; |
264 | unsigned int max_hw_discard_sectors; | ||
271 | unsigned int max_write_same_sectors; | 265 | unsigned int max_write_same_sectors; |
272 | unsigned int discard_granularity; | 266 | unsigned int discard_granularity; |
273 | unsigned int discard_alignment; | 267 | unsigned int discard_alignment; |
@@ -305,7 +299,6 @@ struct request_queue { | |||
305 | make_request_fn *make_request_fn; | 299 | make_request_fn *make_request_fn; |
306 | prep_rq_fn *prep_rq_fn; | 300 | prep_rq_fn *prep_rq_fn; |
307 | unprep_rq_fn *unprep_rq_fn; | 301 | unprep_rq_fn *unprep_rq_fn; |
308 | merge_bvec_fn *merge_bvec_fn; | ||
309 | softirq_done_fn *softirq_done_fn; | 302 | softirq_done_fn *softirq_done_fn; |
310 | rq_timed_out_fn *rq_timed_out_fn; | 303 | rq_timed_out_fn *rq_timed_out_fn; |
311 | dma_drain_needed_fn *dma_drain_needed; | 304 | dma_drain_needed_fn *dma_drain_needed; |
@@ -462,6 +455,7 @@ struct request_queue { | |||
462 | 455 | ||
463 | struct blk_mq_tag_set *tag_set; | 456 | struct blk_mq_tag_set *tag_set; |
464 | struct list_head tag_set_list; | 457 | struct list_head tag_set_list; |
458 | struct bio_set *bio_split; | ||
465 | }; | 459 | }; |
466 | 460 | ||
467 | #define QUEUE_FLAG_QUEUED 1 /* uses generic tag queueing */ | 461 | #define QUEUE_FLAG_QUEUED 1 /* uses generic tag queueing */ |
@@ -486,7 +480,6 @@ struct request_queue { | |||
486 | #define QUEUE_FLAG_DEAD 19 /* queue tear-down finished */ | 480 | #define QUEUE_FLAG_DEAD 19 /* queue tear-down finished */ |
487 | #define QUEUE_FLAG_INIT_DONE 20 /* queue is initialized */ | 481 | #define QUEUE_FLAG_INIT_DONE 20 /* queue is initialized */ |
488 | #define QUEUE_FLAG_NO_SG_MERGE 21 /* don't attempt to merge SG segments*/ | 482 | #define QUEUE_FLAG_NO_SG_MERGE 21 /* don't attempt to merge SG segments*/ |
489 | #define QUEUE_FLAG_SG_GAPS 22 /* queue doesn't support SG gaps */ | ||
490 | 483 | ||
491 | #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ | 484 | #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ |
492 | (1 << QUEUE_FLAG_STACKABLE) | \ | 485 | (1 << QUEUE_FLAG_STACKABLE) | \ |
@@ -782,6 +775,8 @@ extern void blk_rq_unprep_clone(struct request *rq); | |||
782 | extern int blk_insert_cloned_request(struct request_queue *q, | 775 | extern int blk_insert_cloned_request(struct request_queue *q, |
783 | struct request *rq); | 776 | struct request *rq); |
784 | extern void blk_delay_queue(struct request_queue *, unsigned long); | 777 | extern void blk_delay_queue(struct request_queue *, unsigned long); |
778 | extern void blk_queue_split(struct request_queue *, struct bio **, | ||
779 | struct bio_set *); | ||
785 | extern void blk_recount_segments(struct request_queue *, struct bio *); | 780 | extern void blk_recount_segments(struct request_queue *, struct bio *); |
786 | extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int); | 781 | extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int); |
787 | extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t, | 782 | extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t, |
@@ -986,9 +981,9 @@ extern int blk_queue_dma_drain(struct request_queue *q, | |||
986 | void *buf, unsigned int size); | 981 | void *buf, unsigned int size); |
987 | extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn); | 982 | extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn); |
988 | extern void blk_queue_segment_boundary(struct request_queue *, unsigned long); | 983 | extern void blk_queue_segment_boundary(struct request_queue *, unsigned long); |
984 | extern void blk_queue_virt_boundary(struct request_queue *, unsigned long); | ||
989 | extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn); | 985 | extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn); |
990 | extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn); | 986 | extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn); |
991 | extern void blk_queue_merge_bvec(struct request_queue *, merge_bvec_fn *); | ||
992 | extern void blk_queue_dma_alignment(struct request_queue *, int); | 987 | extern void blk_queue_dma_alignment(struct request_queue *, int); |
993 | extern void blk_queue_update_dma_alignment(struct request_queue *, int); | 988 | extern void blk_queue_update_dma_alignment(struct request_queue *, int); |
994 | extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *); | 989 | extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *); |
@@ -1138,6 +1133,7 @@ extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm); | |||
1138 | enum blk_default_limits { | 1133 | enum blk_default_limits { |
1139 | BLK_MAX_SEGMENTS = 128, | 1134 | BLK_MAX_SEGMENTS = 128, |
1140 | BLK_SAFE_MAX_SECTORS = 255, | 1135 | BLK_SAFE_MAX_SECTORS = 255, |
1136 | BLK_DEF_MAX_SECTORS = 2560, | ||
1141 | BLK_MAX_SEGMENT_SIZE = 65536, | 1137 | BLK_MAX_SEGMENT_SIZE = 65536, |
1142 | BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, | 1138 | BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, |
1143 | }; | 1139 | }; |
@@ -1154,6 +1150,11 @@ static inline unsigned long queue_segment_boundary(struct request_queue *q) | |||
1154 | return q->limits.seg_boundary_mask; | 1150 | return q->limits.seg_boundary_mask; |
1155 | } | 1151 | } |
1156 | 1152 | ||
1153 | static inline unsigned long queue_virt_boundary(struct request_queue *q) | ||
1154 | { | ||
1155 | return q->limits.virt_boundary_mask; | ||
1156 | } | ||
1157 | |||
1157 | static inline unsigned int queue_max_sectors(struct request_queue *q) | 1158 | static inline unsigned int queue_max_sectors(struct request_queue *q) |
1158 | { | 1159 | { |
1159 | return q->limits.max_sectors; | 1160 | return q->limits.max_sectors; |
@@ -1354,6 +1355,19 @@ static inline void put_dev_sector(Sector p) | |||
1354 | page_cache_release(p.v); | 1355 | page_cache_release(p.v); |
1355 | } | 1356 | } |
1356 | 1357 | ||
1358 | /* | ||
1359 | * Check if adding a bio_vec after bprv with offset would create a gap in | ||
1360 | * the SG list. Most drivers don't care about this, but some do. | ||
1361 | */ | ||
1362 | static inline bool bvec_gap_to_prev(struct request_queue *q, | ||
1363 | struct bio_vec *bprv, unsigned int offset) | ||
1364 | { | ||
1365 | if (!queue_virt_boundary(q)) | ||
1366 | return false; | ||
1367 | return offset || | ||
1368 | ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q)); | ||
1369 | } | ||
1370 | |||
1357 | struct work_struct; | 1371 | struct work_struct; |
1358 | int kblockd_schedule_work(struct work_struct *work); | 1372 | int kblockd_schedule_work(struct work_struct *work); |
1359 | int kblockd_schedule_delayed_work(struct delayed_work *dwork, unsigned long delay); | 1373 | int kblockd_schedule_delayed_work(struct delayed_work *dwork, unsigned long delay); |
diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 51cc1deb7af3..76d23fa8c7d3 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h | |||
@@ -82,9 +82,6 @@ typedef int (*dm_message_fn) (struct dm_target *ti, unsigned argc, char **argv); | |||
82 | typedef int (*dm_ioctl_fn) (struct dm_target *ti, unsigned int cmd, | 82 | typedef int (*dm_ioctl_fn) (struct dm_target *ti, unsigned int cmd, |
83 | unsigned long arg); | 83 | unsigned long arg); |
84 | 84 | ||
85 | typedef int (*dm_merge_fn) (struct dm_target *ti, struct bvec_merge_data *bvm, | ||
86 | struct bio_vec *biovec, int max_size); | ||
87 | |||
88 | /* | 85 | /* |
89 | * These iteration functions are typically used to check (and combine) | 86 | * These iteration functions are typically used to check (and combine) |
90 | * properties of underlying devices. | 87 | * properties of underlying devices. |
@@ -160,7 +157,6 @@ struct target_type { | |||
160 | dm_status_fn status; | 157 | dm_status_fn status; |
161 | dm_message_fn message; | 158 | dm_message_fn message; |
162 | dm_ioctl_fn ioctl; | 159 | dm_ioctl_fn ioctl; |
163 | dm_merge_fn merge; | ||
164 | dm_busy_fn busy; | 160 | dm_busy_fn busy; |
165 | dm_iterate_devices_fn iterate_devices; | 161 | dm_iterate_devices_fn iterate_devices; |
166 | dm_io_hints_fn io_hints; | 162 | dm_io_hints_fn io_hints; |
diff --git a/include/linux/genhd.h b/include/linux/genhd.h index ec274e0f4ed2..2adbfa6d02bc 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h | |||
@@ -13,6 +13,7 @@ | |||
13 | #include <linux/kdev_t.h> | 13 | #include <linux/kdev_t.h> |
14 | #include <linux/rcupdate.h> | 14 | #include <linux/rcupdate.h> |
15 | #include <linux/slab.h> | 15 | #include <linux/slab.h> |
16 | #include <linux/percpu-refcount.h> | ||
16 | 17 | ||
17 | #ifdef CONFIG_BLOCK | 18 | #ifdef CONFIG_BLOCK |
18 | 19 | ||
@@ -124,7 +125,7 @@ struct hd_struct { | |||
124 | #else | 125 | #else |
125 | struct disk_stats dkstats; | 126 | struct disk_stats dkstats; |
126 | #endif | 127 | #endif |
127 | atomic_t ref; | 128 | struct percpu_ref ref; |
128 | struct rcu_head rcu_head; | 129 | struct rcu_head rcu_head; |
129 | }; | 130 | }; |
130 | 131 | ||
@@ -611,7 +612,7 @@ extern struct hd_struct * __must_check add_partition(struct gendisk *disk, | |||
611 | sector_t len, int flags, | 612 | sector_t len, int flags, |
612 | struct partition_meta_info | 613 | struct partition_meta_info |
613 | *info); | 614 | *info); |
614 | extern void __delete_partition(struct hd_struct *); | 615 | extern void __delete_partition(struct percpu_ref *); |
615 | extern void delete_partition(struct gendisk *, int); | 616 | extern void delete_partition(struct gendisk *, int); |
616 | extern void printk_all_partitions(void); | 617 | extern void printk_all_partitions(void); |
617 | 618 | ||
@@ -640,27 +641,39 @@ extern ssize_t part_fail_store(struct device *dev, | |||
640 | const char *buf, size_t count); | 641 | const char *buf, size_t count); |
641 | #endif /* CONFIG_FAIL_MAKE_REQUEST */ | 642 | #endif /* CONFIG_FAIL_MAKE_REQUEST */ |
642 | 643 | ||
643 | static inline void hd_ref_init(struct hd_struct *part) | 644 | static inline int hd_ref_init(struct hd_struct *part) |
644 | { | 645 | { |
645 | atomic_set(&part->ref, 1); | 646 | if (percpu_ref_init(&part->ref, __delete_partition, 0, |
646 | smp_mb(); | 647 | GFP_KERNEL)) |
648 | return -ENOMEM; | ||
649 | return 0; | ||
647 | } | 650 | } |
648 | 651 | ||
649 | static inline void hd_struct_get(struct hd_struct *part) | 652 | static inline void hd_struct_get(struct hd_struct *part) |
650 | { | 653 | { |
651 | atomic_inc(&part->ref); | 654 | percpu_ref_get(&part->ref); |
652 | smp_mb__after_atomic(); | ||
653 | } | 655 | } |
654 | 656 | ||
655 | static inline int hd_struct_try_get(struct hd_struct *part) | 657 | static inline int hd_struct_try_get(struct hd_struct *part) |
656 | { | 658 | { |
657 | return atomic_inc_not_zero(&part->ref); | 659 | return percpu_ref_tryget_live(&part->ref); |
658 | } | 660 | } |
659 | 661 | ||
660 | static inline void hd_struct_put(struct hd_struct *part) | 662 | static inline void hd_struct_put(struct hd_struct *part) |
661 | { | 663 | { |
662 | if (atomic_dec_and_test(&part->ref)) | 664 | percpu_ref_put(&part->ref); |
663 | __delete_partition(part); | 665 | } |
666 | |||
667 | static inline void hd_struct_kill(struct hd_struct *part) | ||
668 | { | ||
669 | percpu_ref_kill(&part->ref); | ||
670 | } | ||
671 | |||
672 | static inline void hd_free_part(struct hd_struct *part) | ||
673 | { | ||
674 | free_part_stats(part); | ||
675 | free_part_info(part); | ||
676 | percpu_ref_exit(&part->ref); | ||
664 | } | 677 | } |
665 | 678 | ||
666 | /* | 679 | /* |
diff --git a/include/linux/swap.h b/include/linux/swap.h index 38874729dc5f..31496d201fdc 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h | |||
@@ -373,9 +373,9 @@ static inline void mem_cgroup_uncharge_swap(swp_entry_t entry) | |||
373 | /* linux/mm/page_io.c */ | 373 | /* linux/mm/page_io.c */ |
374 | extern int swap_readpage(struct page *); | 374 | extern int swap_readpage(struct page *); |
375 | extern int swap_writepage(struct page *page, struct writeback_control *wbc); | 375 | extern int swap_writepage(struct page *page, struct writeback_control *wbc); |
376 | extern void end_swap_bio_write(struct bio *bio, int err); | 376 | extern void end_swap_bio_write(struct bio *bio); |
377 | extern int __swap_writepage(struct page *page, struct writeback_control *wbc, | 377 | extern int __swap_writepage(struct page *page, struct writeback_control *wbc, |
378 | void (*end_write_func)(struct bio *, int)); | 378 | bio_end_io_t end_write_func); |
379 | extern int swap_set_page_dirty(struct page *page); | 379 | extern int swap_set_page_dirty(struct page *page); |
380 | 380 | ||
381 | int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, | 381 | int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, |
diff --git a/kernel/power/swap.c b/kernel/power/swap.c index 2f30ca91e4fa..b2066fb5b10f 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c | |||
@@ -227,27 +227,23 @@ static void hib_init_batch(struct hib_bio_batch *hb) | |||
227 | hb->error = 0; | 227 | hb->error = 0; |
228 | } | 228 | } |
229 | 229 | ||
230 | static void hib_end_io(struct bio *bio, int error) | 230 | static void hib_end_io(struct bio *bio) |
231 | { | 231 | { |
232 | struct hib_bio_batch *hb = bio->bi_private; | 232 | struct hib_bio_batch *hb = bio->bi_private; |
233 | const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
234 | struct page *page = bio->bi_io_vec[0].bv_page; | 233 | struct page *page = bio->bi_io_vec[0].bv_page; |
235 | 234 | ||
236 | if (!uptodate || error) { | 235 | if (bio->bi_error) { |
237 | printk(KERN_ALERT "Read-error on swap-device (%u:%u:%Lu)\n", | 236 | printk(KERN_ALERT "Read-error on swap-device (%u:%u:%Lu)\n", |
238 | imajor(bio->bi_bdev->bd_inode), | 237 | imajor(bio->bi_bdev->bd_inode), |
239 | iminor(bio->bi_bdev->bd_inode), | 238 | iminor(bio->bi_bdev->bd_inode), |
240 | (unsigned long long)bio->bi_iter.bi_sector); | 239 | (unsigned long long)bio->bi_iter.bi_sector); |
241 | |||
242 | if (!error) | ||
243 | error = -EIO; | ||
244 | } | 240 | } |
245 | 241 | ||
246 | if (bio_data_dir(bio) == WRITE) | 242 | if (bio_data_dir(bio) == WRITE) |
247 | put_page(page); | 243 | put_page(page); |
248 | 244 | ||
249 | if (error && !hb->error) | 245 | if (bio->bi_error && !hb->error) |
250 | hb->error = error; | 246 | hb->error = bio->bi_error; |
251 | if (atomic_dec_and_test(&hb->count)) | 247 | if (atomic_dec_and_test(&hb->count)) |
252 | wake_up(&hb->wait); | 248 | wake_up(&hb->wait); |
253 | 249 | ||
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index b3e6b39b6cf9..90e72a0c3047 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c | |||
@@ -778,9 +778,6 @@ static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, | |||
778 | if (likely(!bt)) | 778 | if (likely(!bt)) |
779 | return; | 779 | return; |
780 | 780 | ||
781 | if (!error && !bio_flagged(bio, BIO_UPTODATE)) | ||
782 | error = EIO; | ||
783 | |||
784 | __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, | 781 | __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, |
785 | bio->bi_rw, what, error, 0, NULL); | 782 | bio->bi_rw, what, error, 0, NULL); |
786 | } | 783 | } |
@@ -887,8 +884,7 @@ static void blk_add_trace_split(void *ignore, | |||
887 | 884 | ||
888 | __blk_add_trace(bt, bio->bi_iter.bi_sector, | 885 | __blk_add_trace(bt, bio->bi_iter.bi_sector, |
889 | bio->bi_iter.bi_size, bio->bi_rw, BLK_TA_SPLIT, | 886 | bio->bi_iter.bi_size, bio->bi_rw, BLK_TA_SPLIT, |
890 | !bio_flagged(bio, BIO_UPTODATE), | 887 | bio->bi_error, sizeof(rpdu), &rpdu); |
891 | sizeof(rpdu), &rpdu); | ||
892 | } | 888 | } |
893 | } | 889 | } |
894 | 890 | ||
@@ -920,8 +916,8 @@ static void blk_add_trace_bio_remap(void *ignore, | |||
920 | r.sector_from = cpu_to_be64(from); | 916 | r.sector_from = cpu_to_be64(from); |
921 | 917 | ||
922 | __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, | 918 | __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, |
923 | bio->bi_rw, BLK_TA_REMAP, | 919 | bio->bi_rw, BLK_TA_REMAP, bio->bi_error, |
924 | !bio_flagged(bio, BIO_UPTODATE), sizeof(r), &r); | 920 | sizeof(r), &r); |
925 | } | 921 | } |
926 | 922 | ||
927 | /** | 923 | /** |
diff --git a/mm/page_io.c b/mm/page_io.c index 520baa4b04d7..b995a5ba5e8f 100644 --- a/mm/page_io.c +++ b/mm/page_io.c | |||
@@ -33,22 +33,19 @@ static struct bio *get_swap_bio(gfp_t gfp_flags, | |||
33 | if (bio) { | 33 | if (bio) { |
34 | bio->bi_iter.bi_sector = map_swap_page(page, &bio->bi_bdev); | 34 | bio->bi_iter.bi_sector = map_swap_page(page, &bio->bi_bdev); |
35 | bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9; | 35 | bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9; |
36 | bio->bi_io_vec[0].bv_page = page; | ||
37 | bio->bi_io_vec[0].bv_len = PAGE_SIZE; | ||
38 | bio->bi_io_vec[0].bv_offset = 0; | ||
39 | bio->bi_vcnt = 1; | ||
40 | bio->bi_iter.bi_size = PAGE_SIZE; | ||
41 | bio->bi_end_io = end_io; | 36 | bio->bi_end_io = end_io; |
37 | |||
38 | bio_add_page(bio, page, PAGE_SIZE, 0); | ||
39 | BUG_ON(bio->bi_iter.bi_size != PAGE_SIZE); | ||
42 | } | 40 | } |
43 | return bio; | 41 | return bio; |
44 | } | 42 | } |
45 | 43 | ||
46 | void end_swap_bio_write(struct bio *bio, int err) | 44 | void end_swap_bio_write(struct bio *bio) |
47 | { | 45 | { |
48 | const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
49 | struct page *page = bio->bi_io_vec[0].bv_page; | 46 | struct page *page = bio->bi_io_vec[0].bv_page; |
50 | 47 | ||
51 | if (!uptodate) { | 48 | if (bio->bi_error) { |
52 | SetPageError(page); | 49 | SetPageError(page); |
53 | /* | 50 | /* |
54 | * We failed to write the page out to swap-space. | 51 | * We failed to write the page out to swap-space. |
@@ -69,12 +66,11 @@ void end_swap_bio_write(struct bio *bio, int err) | |||
69 | bio_put(bio); | 66 | bio_put(bio); |
70 | } | 67 | } |
71 | 68 | ||
72 | static void end_swap_bio_read(struct bio *bio, int err) | 69 | static void end_swap_bio_read(struct bio *bio) |
73 | { | 70 | { |
74 | const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); | ||
75 | struct page *page = bio->bi_io_vec[0].bv_page; | 71 | struct page *page = bio->bi_io_vec[0].bv_page; |
76 | 72 | ||
77 | if (!uptodate) { | 73 | if (bio->bi_error) { |
78 | SetPageError(page); | 74 | SetPageError(page); |
79 | ClearPageUptodate(page); | 75 | ClearPageUptodate(page); |
80 | printk(KERN_ALERT "Read-error on swap-device (%u:%u:%Lu)\n", | 76 | printk(KERN_ALERT "Read-error on swap-device (%u:%u:%Lu)\n", |
@@ -254,7 +250,7 @@ static sector_t swap_page_sector(struct page *page) | |||
254 | } | 250 | } |
255 | 251 | ||
256 | int __swap_writepage(struct page *page, struct writeback_control *wbc, | 252 | int __swap_writepage(struct page *page, struct writeback_control *wbc, |
257 | void (*end_write_func)(struct bio *, int)) | 253 | bio_end_io_t end_write_func) |
258 | { | 254 | { |
259 | struct bio *bio; | 255 | struct bio *bio; |
260 | int ret, rw = WRITE; | 256 | int ret, rw = WRITE; |