11 files changed, 387 insertions, 150 deletions

diff --git a/fs/bio-integrity.c b/fs/bio-integrity.c
index a3f28f331b2b..8fb42916d8a2 100644
--- a/fs/bio-integrity.c
+++ b/fs/bio-integrity.c
@@ -27,48 +27,11 @@
 #include <linux/workqueue.h>
 #include <linux/slab.h>
 
-struct integrity_slab {
-        struct kmem_cache *slab;
-        unsigned short nr_vecs;
-        char name[8];
-};
-
-#define IS(x) { .nr_vecs = x, .name = "bip-"__stringify(x) }
-struct integrity_slab bip_slab[BIOVEC_NR_POOLS] __read_mostly = {
-        IS(1), IS(4), IS(16), IS(64), IS(128), IS(BIO_MAX_PAGES),
-};
-#undef IS
+#define BIP_INLINE_VECS 4
 
+static struct kmem_cache *bip_slab;
 static struct workqueue_struct *kintegrityd_wq;
 
-static inline unsigned int vecs_to_idx(unsigned int nr)
-{
-        switch (nr) {
-        case 1:
-                return 0;
-        case 2 ... 4:
-                return 1;
-        case 5 ... 16:
-                return 2;
-        case 17 ... 64:
-                return 3;
-        case 65 ... 128:
-                return 4;
-        case 129 ... BIO_MAX_PAGES:
-                return 5;
-        default:
-                BUG();
-        }
-}
-
-static inline int use_bip_pool(unsigned int idx)
-{
-        if (idx == BIOVEC_MAX_IDX)
-                return 1;
-
-        return 0;
-}
-
 /**
  * bio_integrity_alloc - Allocate integrity payload and attach it to bio
  * @bio:        bio to attach integrity metadata to
@@ -84,37 +47,41 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
                                                   unsigned int nr_vecs)
 {
         struct bio_integrity_payload *bip;
-        unsigned int idx = vecs_to_idx(nr_vecs);
         struct bio_set *bs = bio->bi_pool;
-
-        if (!bs)
-                bs = fs_bio_set;
-
-        BUG_ON(bio == NULL);
-        bip = NULL;
-
-        /* Lower order allocations come straight from slab */
-        if (!use_bip_pool(idx))
-                bip = kmem_cache_alloc(bip_slab[idx].slab, gfp_mask);
-
-        /* Use mempool if lower order alloc failed or max vecs were requested */
-        if (bip == NULL) {
-                idx = BIOVEC_MAX_IDX;  /* so we free the payload properly later */
+        unsigned long idx = BIO_POOL_NONE;
+        unsigned inline_vecs;
+
+        if (!bs) {
+                bip = kmalloc(sizeof(struct bio_integrity_payload) +
+                              sizeof(struct bio_vec) * nr_vecs, gfp_mask);
+                inline_vecs = nr_vecs;
+        } else {
                 bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask);
-
-                if (unlikely(bip == NULL)) {
-                        printk(KERN_ERR "%s: could not alloc bip\n", __func__);
-                        return NULL;
-                }
+                inline_vecs = BIP_INLINE_VECS;
         }
 
+        if (unlikely(!bip))
+                return NULL;
+
         memset(bip, 0, sizeof(*bip));
 
+        if (nr_vecs > inline_vecs) {
+                bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,
+                                          bs->bvec_integrity_pool);
+                if (!bip->bip_vec)
+                        goto err;
+        } else {
+                bip->bip_vec = bip->bip_inline_vecs;
+        }
+
         bip->bip_slab = idx;
         bip->bip_bio = bio;
         bio->bi_integrity = bip;
 
         return bip;
+err:
+        mempool_free(bip, bs->bio_integrity_pool);
+        return NULL;
 }
 EXPORT_SYMBOL(bio_integrity_alloc);
 
@@ -130,20 +97,18 @@ void bio_integrity_free(struct bio *bio)
         struct bio_integrity_payload *bip = bio->bi_integrity;
         struct bio_set *bs = bio->bi_pool;
 
-        if (!bs)
-                bs = fs_bio_set;
-
-        BUG_ON(bip == NULL);
-
-        /* A cloned bio doesn't own the integrity metadata */
-        if (!bio_flagged(bio, BIO_CLONED) && !bio_flagged(bio, BIO_FS_INTEGRITY)
-            && bip->bip_buf != NULL)
+        if (bip->bip_owns_buf)
                 kfree(bip->bip_buf);
 
-        if (use_bip_pool(bip->bip_slab))
+        if (bs) {
+                if (bip->bip_slab != BIO_POOL_NONE)
+                        bvec_free(bs->bvec_integrity_pool, bip->bip_vec,
+                                  bip->bip_slab);
+
                 mempool_free(bip, bs->bio_integrity_pool);
-        else
-                kmem_cache_free(bip_slab[bip->bip_slab].slab, bip);
+        } else {
+                kfree(bip);
+        }
 
         bio->bi_integrity = NULL;
 }
@@ -419,6 +384,7 @@ int bio_integrity_prep(struct bio *bio)
                 return -EIO;
         }
 
+        bip->bip_owns_buf = 1;
         bip->bip_buf = buf;
         bip->bip_size = len;
         bip->bip_sector = bio->bi_sector;
@@ -694,11 +660,11 @@ void bio_integrity_split(struct bio *bio, struct bio_pair *bp, int sectors)
         bp->bio1.bi_integrity = &bp->bip1;
         bp->bio2.bi_integrity = &bp->bip2;
 
-        bp->iv1 = bip->bip_vec[0];
-        bp->iv2 = bip->bip_vec[0];
+        bp->iv1 = bip->bip_vec[bip->bip_idx];
+        bp->iv2 = bip->bip_vec[bip->bip_idx];
 
-        bp->bip1.bip_vec[0] = bp->iv1;
-        bp->bip2.bip_vec[0] = bp->iv2;
+        bp->bip1.bip_vec = &bp->iv1;
+        bp->bip2.bip_vec = &bp->iv2;
 
         bp->iv1.bv_len = sectors * bi->tuple_size;
         bp->iv2.bv_offset += sectors * bi->tuple_size;
@@ -746,13 +712,14 @@ EXPORT_SYMBOL(bio_integrity_clone);
 
 int bioset_integrity_create(struct bio_set *bs, int pool_size)
 {
-        unsigned int max_slab = vecs_to_idx(BIO_MAX_PAGES);
-
         if (bs->bio_integrity_pool)
                 return 0;
 
-        bs->bio_integrity_pool =
-                mempool_create_slab_pool(pool_size, bip_slab[max_slab].slab);
+        bs->bio_integrity_pool = mempool_create_slab_pool(pool_size, bip_slab);
+
+        bs->bvec_integrity_pool = biovec_create_pool(bs, pool_size);
+        if (!bs->bvec_integrity_pool)
+                return -1;
 
         if (!bs->bio_integrity_pool)
                 return -1;
@@ -765,13 +732,14 @@ void bioset_integrity_free(struct bio_set *bs)
 {
         if (bs->bio_integrity_pool)
                 mempool_destroy(bs->bio_integrity_pool);
+
+        if (bs->bvec_integrity_pool)
+                mempool_destroy(bs->bio_integrity_pool);
 }
 EXPORT_SYMBOL(bioset_integrity_free);
 
 void __init bio_integrity_init(void)
 {
-        unsigned int i;
-
         /*
          * kintegrityd won't block much but may burn a lot of CPU cycles.
          * Make it highpri CPU intensive wq with max concurrency of 1.
@@ -781,14 +749,10 @@ void __init bio_integrity_init(void)
         if (!kintegrityd_wq)
                 panic("Failed to create kintegrityd\n");
 
-        for (i = 0 ; i < BIOVEC_NR_POOLS ; i++) {
-                unsigned int size;
-
-                size = sizeof(struct bio_integrity_payload)
-                        + bip_slab[i].nr_vecs * sizeof(struct bio_vec);
-
-                bip_slab[i].slab =
-                        kmem_cache_create(bip_slab[i].name, size, 0,
-                                          SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
-        }
+        bip_slab = kmem_cache_create("bio_integrity_payload",
+                                     sizeof(struct bio_integrity_payload) +
+                                     sizeof(struct bio_vec) * BIP_INLINE_VECS,
+                                     0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
+        if (!bip_slab)
+                panic("Failed to create slab\n");
 }
diff --git a/fs/bio.c b/fs/bio.c
index bb5768f59b32..9238a54b562c 100644
--- a/fs/bio.c
+++ b/fs/bio.c
@@ -160,12 +160,12 @@ unsigned int bvec_nr_vecs(unsigned short idx)
         return bvec_slabs[idx].nr_vecs;
 }
 
-void bvec_free_bs(struct bio_set *bs, struct bio_vec *bv, unsigned int idx)
+void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
 {
         BIO_BUG_ON(idx >= BIOVEC_NR_POOLS);
 
         if (idx == BIOVEC_MAX_IDX)
-                mempool_free(bv, bs->bvec_pool);
+                mempool_free(bv, pool);
         else {
                 struct biovec_slab *bvs = bvec_slabs + idx;
 
@@ -173,8 +173,8 @@ void bvec_free_bs(struct bio_set *bs, struct bio_vec *bv, unsigned int idx)
         }
 }
 
-struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx,
-                              struct bio_set *bs)
+struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
+                           mempool_t *pool)
 {
         struct bio_vec *bvl;
 
@@ -210,7 +210,7 @@ struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx,
          */
         if (*idx == BIOVEC_MAX_IDX) {
 fallback:
-                bvl = mempool_alloc(bs->bvec_pool, gfp_mask);
+                bvl = mempool_alloc(pool, gfp_mask);
         } else {
                 struct biovec_slab *bvs = bvec_slabs + *idx;
                 gfp_t __gfp_mask = gfp_mask & ~(__GFP_WAIT | __GFP_IO);
@@ -252,8 +252,8 @@ static void bio_free(struct bio *bio)
         __bio_free(bio);
 
         if (bs) {
-                if (bio_has_allocated_vec(bio))
-                        bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio));
+                if (bio_flagged(bio, BIO_OWNS_VEC))
+                        bvec_free(bs->bvec_pool, bio->bi_io_vec, BIO_POOL_IDX(bio));
 
                 /*
                  * If we have front padding, adjust the bio pointer before freeing
@@ -297,6 +297,54 @@ void bio_reset(struct bio *bio)
 }
 EXPORT_SYMBOL(bio_reset);
 
+static void bio_alloc_rescue(struct work_struct *work)
+{
+        struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
+        struct bio *bio;
+
+        while (1) {
+                spin_lock(&bs->rescue_lock);
+                bio = bio_list_pop(&bs->rescue_list);
+                spin_unlock(&bs->rescue_lock);
+
+                if (!bio)
+                        break;
+
+                generic_make_request(bio);
+        }
+}
+
+static void punt_bios_to_rescuer(struct bio_set *bs)
+{
+        struct bio_list punt, nopunt;
+        struct bio *bio;
+
+        /*
+         * In order to guarantee forward progress we must punt only bios that
+         * were allocated from this bio_set; otherwise, if there was a bio on
+         * there for a stacking driver higher up in the stack, processing it
+         * could require allocating bios from this bio_set, and doing that from
+         * our own rescuer would be bad.
+         *
+         * Since bio lists are singly linked, pop them all instead of trying to
+         * remove from the middle of the list:
+         */
+
+        bio_list_init(&punt);
+        bio_list_init(&nopunt);
+
+        while ((bio = bio_list_pop(current->bio_list)))
+                bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
+
+        *current->bio_list = nopunt;
+
+        spin_lock(&bs->rescue_lock);
+        bio_list_merge(&bs->rescue_list, &punt);
+        spin_unlock(&bs->rescue_lock);
+
+        queue_work(bs->rescue_workqueue, &bs->rescue_work);
+}
+
 /**
  * bio_alloc_bioset - allocate a bio for I/O
  * @gfp_mask:   the GFP_ mask given to the slab allocator
@@ -314,11 +362,27 @@ EXPORT_SYMBOL(bio_reset);
  *   previously allocated bio for IO before attempting to allocate a new one.
  *   Failure to do so can cause deadlocks under memory pressure.
  *
+ *   Note that when running under generic_make_request() (i.e. any block
+ *   driver), bios are not submitted until after you return - see the code in
+ *   generic_make_request() that converts recursion into iteration, to prevent
+ *   stack overflows.
+ *
+ *   This would normally mean allocating multiple bios under
+ *   generic_make_request() would be susceptible to deadlocks, but we have
+ *   deadlock avoidance code that resubmits any blocked bios from a rescuer
+ *   thread.
+ *
+ *   However, we do not guarantee forward progress for allocations from other
+ *   mempools. Doing multiple allocations from the same mempool under
+ *   generic_make_request() should be avoided - instead, use bio_set's front_pad
+ *   for per bio allocations.
+ *
  *   RETURNS:
  *   Pointer to new bio on success, NULL on failure.
  */
 struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
 {
+        gfp_t saved_gfp = gfp_mask;
         unsigned front_pad;
         unsigned inline_vecs;
         unsigned long idx = BIO_POOL_NONE;
@@ -336,7 +400,37 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
                 front_pad = 0;
                 inline_vecs = nr_iovecs;
         } else {
+                /*
+                 * generic_make_request() converts recursion to iteration; this
+                 * means if we're running beneath it, any bios we allocate and
+                 * submit will not be submitted (and thus freed) until after we
+                 * return.
+                 *
+                 * This exposes us to a potential deadlock if we allocate
+                 * multiple bios from the same bio_set() while running
+                 * underneath generic_make_request(). If we were to allocate
+                 * multiple bios (say a stacking block driver that was splitting
+                 * bios), we would deadlock if we exhausted the mempool's
+                 * reserve.
+                 *
+                 * We solve this, and guarantee forward progress, with a rescuer
+                 * workqueue per bio_set. If we go to allocate and there are
+                 * bios on current->bio_list, we first try the allocation
+                 * without __GFP_WAIT; if that fails, we punt those bios we
+                 * would be blocking to the rescuer workqueue before we retry
+                 * with the original gfp_flags.
+                 */
+
+                if (current->bio_list && !bio_list_empty(current->bio_list))
+                        gfp_mask &= ~__GFP_WAIT;
+
                 p = mempool_alloc(bs->bio_pool, gfp_mask);
+                if (!p && gfp_mask != saved_gfp) {
+                        punt_bios_to_rescuer(bs);
+                        gfp_mask = saved_gfp;
+                        p = mempool_alloc(bs->bio_pool, gfp_mask);
+                }
+
                 front_pad = bs->front_pad;
                 inline_vecs = BIO_INLINE_VECS;
         }
@@ -348,9 +442,17 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
         bio_init(bio);
 
         if (nr_iovecs > inline_vecs) {
-                bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs);
+                bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
+                if (!bvl && gfp_mask != saved_gfp) {
+                        punt_bios_to_rescuer(bs);
+                        gfp_mask = saved_gfp;
+                        bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
+                }
+
                 if (unlikely(!bvl))
                         goto err_free;
+
+                bio->bi_flags |= 1 << BIO_OWNS_VEC;
         } else if (nr_iovecs) {
                 bvl = bio->bi_inline_vecs;
         }
@@ -652,6 +754,181 @@ int bio_add_page(struct bio *bio, struct page *page, unsigned int len,
 }
 EXPORT_SYMBOL(bio_add_page);
 
+struct submit_bio_ret {
+        struct completion event;
+        int error;
+};
+
+static void submit_bio_wait_endio(struct bio *bio, int error)
+{
+        struct submit_bio_ret *ret = bio->bi_private;
+
+        ret->error = error;
+        complete(&ret->event);
+}
+
+/**
+ * submit_bio_wait - submit a bio, and wait until it completes
+ * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
+ * @bio: The &struct bio which describes the I/O
+ *
+ * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
+ * bio_endio() on failure.
+ */
+int submit_bio_wait(int rw, struct bio *bio)
+{
+        struct submit_bio_ret ret;
+
+        rw |= REQ_SYNC;
+        init_completion(&ret.event);
+        bio->bi_private = &ret;
+        bio->bi_end_io = submit_bio_wait_endio;
+        submit_bio(rw, bio);
+        wait_for_completion(&ret.event);
+
+        return ret.error;
+}
+EXPORT_SYMBOL(submit_bio_wait);
+
+/**
+ * bio_advance - increment/complete a bio by some number of bytes
+ * @bio:        bio to advance
+ * @bytes:      number of bytes to complete
+ *
+ * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
+ * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
+ * be updated on the last bvec as well.
+ *
+ * @bio will then represent the remaining, uncompleted portion of the io.
+ */
+void bio_advance(struct bio *bio, unsigned bytes)
+{
+        if (bio_integrity(bio))
+                bio_integrity_advance(bio, bytes);
+
+        bio->bi_sector += bytes >> 9;
+        bio->bi_size -= bytes;
+
+        if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK)
+                return;
+
+        while (bytes) {
+                if (unlikely(bio->bi_idx >= bio->bi_vcnt)) {
+                        WARN_ONCE(1, "bio idx %d >= vcnt %d\n",
+                                  bio->bi_idx, bio->bi_vcnt);
+                        break;
+                }
+
+                if (bytes >= bio_iovec(bio)->bv_len) {
+                        bytes -= bio_iovec(bio)->bv_len;
+                        bio->bi_idx++;
+                } else {
+                        bio_iovec(bio)->bv_len -= bytes;
+                        bio_iovec(bio)->bv_offset += bytes;
+                        bytes = 0;
+                }
+        }
+}
+EXPORT_SYMBOL(bio_advance);
+
+/**
+ * bio_alloc_pages - allocates a single page for each bvec in a bio
+ * @bio: bio to allocate pages for
+ * @gfp_mask: flags for allocation
+ *
+ * Allocates pages up to @bio->bi_vcnt.
+ *
+ * Returns 0 on success, -ENOMEM on failure. On failure, any allocated pages are
+ * freed.
+ */
+int bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
+{
+        int i;
+        struct bio_vec *bv;
+
+        bio_for_each_segment_all(bv, bio, i) {
+                bv->bv_page = alloc_page(gfp_mask);
+                if (!bv->bv_page) {
+                        while (--bv >= bio->bi_io_vec)
+                                __free_page(bv->bv_page);
+                        return -ENOMEM;
+                }
+        }
+
+        return 0;
+}
+EXPORT_SYMBOL(bio_alloc_pages);
+
+/**
+ * bio_copy_data - copy contents of data buffers from one chain of bios to
+ * another
+ * @src: source bio list
+ * @dst: destination bio list
+ *
+ * If @src and @dst are single bios, bi_next must be NULL - otherwise, treats
+ * @src and @dst as linked lists of bios.
+ *
+ * Stops when it reaches the end of either @src or @dst - that is, copies
+ * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
+ */
+void bio_copy_data(struct bio *dst, struct bio *src)
+{
+        struct bio_vec *src_bv, *dst_bv;
+        unsigned src_offset, dst_offset, bytes;
+        void *src_p, *dst_p;
+
+        src_bv = bio_iovec(src);
+        dst_bv = bio_iovec(dst);
+
+        src_offset = src_bv->bv_offset;
+        dst_offset = dst_bv->bv_offset;
+
+        while (1) {
+                if (src_offset == src_bv->bv_offset + src_bv->bv_len) {
+                        src_bv++;
+                        if (src_bv == bio_iovec_idx(src, src->bi_vcnt)) {
+                                src = src->bi_next;
+                                if (!src)
+                                        break;
+
+                                src_bv = bio_iovec(src);
+                        }
+
+                        src_offset = src_bv->bv_offset;
+                }
+
+                if (dst_offset == dst_bv->bv_offset + dst_bv->bv_len) {
+                        dst_bv++;
+                        if (dst_bv == bio_iovec_idx(dst, dst->bi_vcnt)) {
+                                dst = dst->bi_next;
+                                if (!dst)
+                                        break;
+
+                                dst_bv = bio_iovec(dst);
+                        }
+
+                        dst_offset = dst_bv->bv_offset;
+                }
+
+                bytes = min(dst_bv->bv_offset + dst_bv->bv_len - dst_offset,
+                            src_bv->bv_offset + src_bv->bv_len - src_offset);
+
+                src_p = kmap_atomic(src_bv->bv_page);
+                dst_p = kmap_atomic(dst_bv->bv_page);
+
+                memcpy(dst_p + dst_bv->bv_offset,
+                       src_p + src_bv->bv_offset,
+                       bytes);
+
+                kunmap_atomic(dst_p);
+                kunmap_atomic(src_p);
+
+                src_offset += bytes;
+                dst_offset += bytes;
+        }
+}
+EXPORT_SYMBOL(bio_copy_data);
+
 struct bio_map_data {
         struct bio_vec *iovecs;
         struct sg_iovec *sgvecs;
@@ -714,7 +991,7 @@ static int __bio_copy_iov(struct bio *bio, struct bio_vec *iovecs,
         int iov_idx = 0;
         unsigned int iov_off = 0;
 
-        __bio_for_each_segment(bvec, bio, i, 0) {
+        bio_for_each_segment_all(bvec, bio, i) {
                 char *bv_addr = page_address(bvec->bv_page);
                 unsigned int bv_len = iovecs[i].bv_len;
 
@@ -896,7 +1173,7 @@ struct bio *bio_copy_user_iov(struct request_queue *q,
         return bio;
 cleanup:
         if (!map_data)
-                bio_for_each_segment(bvec, bio, i)
+                bio_for_each_segment_all(bvec, bio, i)
                         __free_page(bvec->bv_page);
 
         bio_put(bio);
@@ -1110,7 +1387,7 @@ static void __bio_unmap_user(struct bio *bio)
         /*
          * make sure we dirty pages we wrote to
          */
-        __bio_for_each_segment(bvec, bio, i, 0) {
+        bio_for_each_segment_all(bvec, bio, i) {
                 if (bio_data_dir(bio) == READ)
                         set_page_dirty_lock(bvec->bv_page);
 
@@ -1216,7 +1493,7 @@ static void bio_copy_kern_endio(struct bio *bio, int err)
         int i;
         char *p = bmd->sgvecs[0].iov_base;
 
-        __bio_for_each_segment(bvec, bio, i, 0) {
+        bio_for_each_segment_all(bvec, bio, i) {
                 char *addr = page_address(bvec->bv_page);
                 int len = bmd->iovecs[i].bv_len;
 
@@ -1256,7 +1533,7 @@ struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,
         if (!reading) {
                 void *p = data;
 
-                bio_for_each_segment(bvec, bio, i) {
+                bio_for_each_segment_all(bvec, bio, i) {
                         char *addr = page_address(bvec->bv_page);
 
                         memcpy(addr, p, bvec->bv_len);
@@ -1301,11 +1578,11 @@ EXPORT_SYMBOL(bio_copy_kern);
  */
 void bio_set_pages_dirty(struct bio *bio)
 {
-        struct bio_vec *bvec = bio->bi_io_vec;
+        struct bio_vec *bvec;
         int i;
 
-        for (i = 0; i < bio->bi_vcnt; i++) {
-                struct page *page = bvec[i].bv_page;
+        bio_for_each_segment_all(bvec, bio, i) {
+                struct page *page = bvec->bv_page;
 
                 if (page && !PageCompound(page))
                         set_page_dirty_lock(page);
@@ -1314,11 +1591,11 @@ void bio_set_pages_dirty(struct bio *bio)
 
 static void bio_release_pages(struct bio *bio)
 {
-        struct bio_vec *bvec = bio->bi_io_vec;
+        struct bio_vec *bvec;
         int i;
 
-        for (i = 0; i < bio->bi_vcnt; i++) {
-                struct page *page = bvec[i].bv_page;
+        bio_for_each_segment_all(bvec, bio, i) {
+                struct page *page = bvec->bv_page;
 
                 if (page)
                         put_page(page);
@@ -1367,16 +1644,16 @@ static void bio_dirty_fn(struct work_struct *work)
 
 void bio_check_pages_dirty(struct bio *bio)
 {
-        struct bio_vec *bvec = bio->bi_io_vec;
+        struct bio_vec *bvec;
         int nr_clean_pages = 0;
         int i;
 
-        for (i = 0; i < bio->bi_vcnt; i++) {
-                struct page *page = bvec[i].bv_page;
+        bio_for_each_segment_all(bvec, bio, i) {
+                struct page *page = bvec->bv_page;
 
                 if (PageDirty(page) || PageCompound(page)) {
                         page_cache_release(page);
-                        bvec[i].bv_page = NULL;
+                        bvec->bv_page = NULL;
                 } else {
                         nr_clean_pages++;
                 }
@@ -1479,8 +1756,7 @@ struct bio_pair *bio_split(struct bio *bi, int first_sectors)
         trace_block_split(bdev_get_queue(bi->bi_bdev), bi,
                                 bi->bi_sector + first_sectors);
 
-        BUG_ON(bi->bi_vcnt != 1 && bi->bi_vcnt != 0);
-        BUG_ON(bi->bi_idx != 0);
+        BUG_ON(bio_segments(bi) > 1);
         atomic_set(&bp->cnt, 3);
         bp->error = 0;
         bp->bio1 = *bi;
@@ -1490,8 +1766,8 @@ struct bio_pair *bio_split(struct bio *bi, int first_sectors)
         bp->bio1.bi_size = first_sectors << 9;
 
         if (bi->bi_vcnt != 0) {
-                bp->bv1 = bi->bi_io_vec[0];
-                bp->bv2 = bi->bi_io_vec[0];
+                bp->bv1 = *bio_iovec(bi);
+                bp->bv2 = *bio_iovec(bi);
 
                 if (bio_is_rw(bi)) {
                         bp->bv2.bv_offset += first_sectors << 9;
@@ -1543,7 +1819,7 @@ sector_t bio_sector_offset(struct bio *bio, unsigned short index,
         if (index >= bio->bi_idx)
                 index = bio->bi_vcnt - 1;
 
-        __bio_for_each_segment(bv, bio, i, 0) {
+        bio_for_each_segment_all(bv, bio, i) {
                 if (i == index) {
                         if (offset > bv->bv_offset)
                                 sectors += (offset - bv->bv_offset) / sector_sz;
@@ -1561,29 +1837,25 @@ EXPORT_SYMBOL(bio_sector_offset);
  * create memory pools for biovec's in a bio_set.
  * use the global biovec slabs created for general use.
  */
-static int biovec_create_pools(struct bio_set *bs, int pool_entries)
+mempool_t *biovec_create_pool(struct bio_set *bs, int pool_entries)
 {
         struct biovec_slab *bp = bvec_slabs + BIOVEC_MAX_IDX;
 
-        bs->bvec_pool = mempool_create_slab_pool(pool_entries, bp->slab);
-        if (!bs->bvec_pool)
-                return -ENOMEM;
-
-        return 0;
-}
-
-static void biovec_free_pools(struct bio_set *bs)
-{
-        mempool_destroy(bs->bvec_pool);
+        return mempool_create_slab_pool(pool_entries, bp->slab);
 }
 
 void bioset_free(struct bio_set *bs)
 {
+        if (bs->rescue_workqueue)
+                destroy_workqueue(bs->rescue_workqueue);
+
         if (bs->bio_pool)
                 mempool_destroy(bs->bio_pool);
 
+        if (bs->bvec_pool)
+                mempool_destroy(bs->bvec_pool);
+
         bioset_integrity_free(bs);
-        biovec_free_pools(bs);
         bio_put_slab(bs);
 
         kfree(bs);
@@ -1614,6 +1886,10 @@ struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
 
         bs->front_pad = front_pad;
 
+        spin_lock_init(&bs->rescue_lock);
+        bio_list_init(&bs->rescue_list);
+        INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
+
         bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
         if (!bs->bio_slab) {
                 kfree(bs);
@@ -1624,9 +1900,15 @@ struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
         if (!bs->bio_pool)
                 goto bad;
 
-        if (!biovec_create_pools(bs, pool_size))
-                return bs;
+        bs->bvec_pool = biovec_create_pool(bs, pool_size);
+        if (!bs->bvec_pool)
+                goto bad;
+
+        bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
+        if (!bs->rescue_workqueue)
+                goto bad;
 
+        return bs;
 bad:
         bioset_free(bs);
         return NULL;
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index f173c5af6461..bed072aa461f 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -2527,8 +2527,7 @@ static int submit_extent_page(int rw, struct extent_io_tree *tree,
                 if (old_compressed)
                         contig = bio->bi_sector == sector;
                 else
-                        contig = bio->bi_sector + (bio->bi_size >> 9) ==
-                                sector;
+                        contig = bio_end_sector(bio) == sector;
 
                 if (prev_bio_flags != bio_flags || !contig ||
                     merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) ||
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index 5989a92236f7..d90e0485e01b 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -5166,7 +5166,7 @@ static int bio_size_ok(struct block_device *bdev, struct bio *bio,
         }
 
         prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
-        if ((bio->bi_size >> 9) > max_sectors)
+        if (bio_sectors(bio) > max_sectors)
                 return 0;
 
         if (!q->merge_bvec_fn)
diff --git a/fs/buffer.c b/fs/buffer.c
index b4dcb34c9635..ecd3792ae0e9 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -2979,7 +2979,6 @@ int submit_bh(int rw, struct buffer_head * bh)
         bio->bi_io_vec[0].bv_offset = bh_offset(bh);
 
         bio->bi_vcnt = 1;
-        bio->bi_idx = 0;
         bio->bi_size = bh->b_size;
 
         bio->bi_end_io = end_bio_bh_io_sync;
diff --git a/fs/direct-io.c b/fs/direct-io.c
index f853263cf74f..38484b08a39a 100644
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -441,8 +441,8 @@ static struct bio *dio_await_one(struct dio *dio)
 static int dio_bio_complete(struct dio *dio, struct bio *bio)
 {
         const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
-        struct bio_vec *bvec = bio->bi_io_vec;
-        int page_no;
+        struct bio_vec *bvec;
+        unsigned i;
 
         if (!uptodate)
                 dio->io_error = -EIO;
@@ -450,8 +450,8 @@ static int dio_bio_complete(struct dio *dio, struct bio *bio)
         if (dio->is_async && dio->rw == READ) {
                 bio_check_pages_dirty(bio);     /* transfers ownership */
         } else {
-                for (page_no = 0; page_no < bio->bi_vcnt; page_no++) {
-                        struct page *page = bvec[page_no].bv_page;
+                bio_for_each_segment_all(bvec, bio, i) {
+                        struct page *page = bvec->bv_page;
 
                         if (dio->rw == READ && !PageCompound(page))
                                 set_page_dirty_lock(page);
diff --git a/fs/exofs/ore.c b/fs/exofs/ore.c
index f936cb50dc0d..b74422888604 100644
--- a/fs/exofs/ore.c
+++ b/fs/exofs/ore.c
@@ -401,7 +401,7 @@ static void _clear_bio(struct bio *bio)
         struct bio_vec *bv;
         unsigned i;
 
-        __bio_for_each_segment(bv, bio, i, 0) {
+        bio_for_each_segment_all(bv, bio, i) {
                 unsigned this_count = bv->bv_len;
 
                 if (likely(PAGE_SIZE == this_count))
diff --git a/fs/exofs/ore_raid.c b/fs/exofs/ore_raid.c
index b963f38ac298..7682b970d0f1 100644
--- a/fs/exofs/ore_raid.c
+++ b/fs/exofs/ore_raid.c
@@ -432,7 +432,7 @@ static void _mark_read4write_pages_uptodate(struct ore_io_state *ios, int ret)
                 if (!bio)
                         continue;
 
-                __bio_for_each_segment(bv, bio, i, 0) {
+                bio_for_each_segment_all(bv, bio, i) {
                         struct page *page = bv->bv_page;
 
                         SetPageUptodate(page);
diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c
index a5055977a214..5c37ef982390 100644
--- a/fs/gfs2/lops.c
+++ b/fs/gfs2/lops.c
@@ -300,7 +300,7 @@ static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno)
         u64 nblk;
 
         if (bio) {
-                nblk = bio->bi_sector + bio_sectors(bio);
+                nblk = bio_end_sector(bio);
                 nblk >>= sdp->sd_fsb2bb_shift;
                 if (blkno == nblk)
                         return bio;
diff --git a/fs/jfs/jfs_logmgr.c b/fs/jfs/jfs_logmgr.c
index 2eb952c41a69..8ae5e350da43 100644
--- a/fs/jfs/jfs_logmgr.c
+++ b/fs/jfs/jfs_logmgr.c
@@ -2004,7 +2004,6 @@ static int lbmRead(struct jfs_log * log, int pn, struct lbuf ** bpp)
         bio->bi_io_vec[0].bv_offset = bp->l_offset;
 
         bio->bi_vcnt = 1;
-        bio->bi_idx = 0;
         bio->bi_size = LOGPSIZE;
 
         bio->bi_end_io = lbmIODone;
@@ -2145,7 +2144,6 @@ static void lbmStartIO(struct lbuf * bp)
         bio->bi_io_vec[0].bv_offset = bp->l_offset;
 
         bio->bi_vcnt = 1;
-        bio->bi_idx = 0;
         bio->bi_size = LOGPSIZE;
 
         bio->bi_end_io = lbmIODone;
diff --git a/fs/logfs/dev_bdev.c b/fs/logfs/dev_bdev.c
index e784a217b500..550475ca6a0e 100644
--- a/fs/logfs/dev_bdev.c
+++ b/fs/logfs/dev_bdev.c
@@ -32,7 +32,6 @@ static int sync_request(struct page *page, struct block_device *bdev, int rw)
         bio_vec.bv_len = PAGE_SIZE;
         bio_vec.bv_offset = 0;
         bio.bi_vcnt = 1;
-        bio.bi_idx = 0;
         bio.bi_size = PAGE_SIZE;
         bio.bi_bdev = bdev;
         bio.bi_sector = page->index * (PAGE_SIZE >> 9);
@@ -108,7 +107,6 @@ static int __bdev_writeseg(struct super_block *sb, u64 ofs, pgoff_t index,
                 if (i >= max_pages) {
                         /* Block layer cannot split bios :( */
                         bio->bi_vcnt = i;
-                        bio->bi_idx = 0;
                         bio->bi_size = i * PAGE_SIZE;
                         bio->bi_bdev = super->s_bdev;
                         bio->bi_sector = ofs >> 9;
@@ -136,7 +134,6 @@ static int __bdev_writeseg(struct super_block *sb, u64 ofs, pgoff_t index,
                 unlock_page(page);
         }
         bio->bi_vcnt = nr_pages;
-        bio->bi_idx = 0;
         bio->bi_size = nr_pages * PAGE_SIZE;
         bio->bi_bdev = super->s_bdev;
         bio->bi_sector = ofs >> 9;
@@ -202,7 +199,6 @@ static int do_erase(struct super_block *sb, u64 ofs, pgoff_t index,
                 if (i >= max_pages) {
                         /* Block layer cannot split bios :( */
                         bio->bi_vcnt = i;
-                        bio->bi_idx = 0;
                         bio->bi_size = i * PAGE_SIZE;
                         bio->bi_bdev = super->s_bdev;
                         bio->bi_sector = ofs >> 9;
@@ -224,7 +220,6 @@ static int do_erase(struct super_block *sb, u64 ofs, pgoff_t index,
                 bio->bi_io_vec[i].bv_offset = 0;
         }
         bio->bi_vcnt = nr_pages;
-        bio->bi_idx = 0;
         bio->bi_size = nr_pages * PAGE_SIZE;
         bio->bi_bdev = super->s_bdev;
         bio->bi_sector = ofs >> 9;