1 files changed, 105 insertions, 126 deletions

diff --git a/fs/bio.c b/fs/bio.c
index 71072ab99128..9298c65ad9c7 100644
--- a/fs/bio.c
+++ b/fs/bio.c
@@ -55,6 +55,7 @@ static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = {
  * IO code that does not need private memory pools.
  */
 struct bio_set *fs_bio_set;
+EXPORT_SYMBOL(fs_bio_set);
 
 /*
  * Our slab pool management
@@ -233,26 +234,37 @@ fallback:
         return bvl;
 }
 
-void bio_free(struct bio *bio, struct bio_set *bs)
+static void __bio_free(struct bio *bio)
 {
+        bio_disassociate_task(bio);
+
+        if (bio_integrity(bio))
+                bio_integrity_free(bio);
+}
+
+static void bio_free(struct bio *bio)
+{
+        struct bio_set *bs = bio->bi_pool;
         void *p;
 
-        if (bio_has_allocated_vec(bio))
-                bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio));
+        __bio_free(bio);
 
-        if (bio_integrity(bio))
-                bio_integrity_free(bio, bs);
+        if (bs) {
+                if (bio_has_allocated_vec(bio))
+                        bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio));
 
-        /*
-         * If we have front padding, adjust the bio pointer before freeing
-         */
-        p = bio;
-        if (bs->front_pad)
+                /*
+                 * If we have front padding, adjust the bio pointer before freeing
+                 */
+                p = bio;
                 p -= bs->front_pad;
 
-        mempool_free(p, bs->bio_pool);
+                mempool_free(p, bs->bio_pool);
+        } else {
+                /* Bio was allocated by bio_kmalloc() */
+                kfree(bio);
+        }
 }
-EXPORT_SYMBOL(bio_free);
 
 void bio_init(struct bio *bio)
 {
@@ -263,48 +275,85 @@ void bio_init(struct bio *bio)
 EXPORT_SYMBOL(bio_init);
 
 /**
+ * bio_reset - reinitialize a bio
+ * @bio:        bio to reset
+ *
+ * Description:
+ *   After calling bio_reset(), @bio will be in the same state as a freshly
+ *   allocated bio returned bio bio_alloc_bioset() - the only fields that are
+ *   preserved are the ones that are initialized by bio_alloc_bioset(). See
+ *   comment in struct bio.
+ */
+void bio_reset(struct bio *bio)
+{
+        unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);
+
+        __bio_free(bio);
+
+        memset(bio, 0, BIO_RESET_BYTES);
+        bio->bi_flags = flags|(1 << BIO_UPTODATE);
+}
+EXPORT_SYMBOL(bio_reset);
+
+/**
  * bio_alloc_bioset - allocate a bio for I/O
  * @gfp_mask:   the GFP_ mask given to the slab allocator
  * @nr_iovecs:  number of iovecs to pre-allocate
  * @bs:         the bio_set to allocate from.
  *
  * Description:
- *   bio_alloc_bioset will try its own mempool to satisfy the allocation.
- *   If %__GFP_WAIT is set then we will block on the internal pool waiting
- *   for a &struct bio to become free.
+ *   If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
+ *   backed by the @bs's mempool.
  *
- *   Note that the caller must set ->bi_destructor on successful return
- *   of a bio, to do the appropriate freeing of the bio once the reference
- *   count drops to zero.
- **/
+ *   When @bs is not NULL, if %__GFP_WAIT is set then bio_alloc will always be
+ *   able to allocate a bio. This is due to the mempool guarantees. To make this
+ *   work, callers must never allocate more than 1 bio at a time from this pool.
+ *   Callers that need to allocate more than 1 bio must always submit the
+ *   previously allocated bio for IO before attempting to allocate a new one.
+ *   Failure to do so can cause deadlocks under memory pressure.
+ *
+ *   RETURNS:
+ *   Pointer to new bio on success, NULL on failure.
+ */
 struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
 {
+        unsigned front_pad;
+        unsigned inline_vecs;
         unsigned long idx = BIO_POOL_NONE;
         struct bio_vec *bvl = NULL;
         struct bio *bio;
         void *p;
 
-        p = mempool_alloc(bs->bio_pool, gfp_mask);
+        if (!bs) {
+                if (nr_iovecs > UIO_MAXIOV)
+                        return NULL;
+
+                p = kmalloc(sizeof(struct bio) +
+                            nr_iovecs * sizeof(struct bio_vec),
+                            gfp_mask);
+                front_pad = 0;
+                inline_vecs = nr_iovecs;
+        } else {
+                p = mempool_alloc(bs->bio_pool, gfp_mask);
+                front_pad = bs->front_pad;
+                inline_vecs = BIO_INLINE_VECS;
+        }
+
         if (unlikely(!p))
                 return NULL;
-        bio = p + bs->front_pad;
 
+        bio = p + front_pad;
         bio_init(bio);
 
-        if (unlikely(!nr_iovecs))
-                goto out_set;
-
-        if (nr_iovecs <= BIO_INLINE_VECS) {
-                bvl = bio->bi_inline_vecs;
-                nr_iovecs = BIO_INLINE_VECS;
-        } else {
+        if (nr_iovecs > inline_vecs) {
                 bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs);
                 if (unlikely(!bvl))
                         goto err_free;
-
-                nr_iovecs = bvec_nr_vecs(idx);
+        } else if (nr_iovecs) {
+                bvl = bio->bi_inline_vecs;
         }
-out_set:
+
+        bio->bi_pool = bs;
         bio->bi_flags |= idx << BIO_POOL_OFFSET;
         bio->bi_max_vecs = nr_iovecs;
         bio->bi_io_vec = bvl;
@@ -316,80 +365,6 @@ err_free:
 }
 EXPORT_SYMBOL(bio_alloc_bioset);
 
-static void bio_fs_destructor(struct bio *bio)
-{
-        bio_free(bio, fs_bio_set);
-}
-
-/**
- *      bio_alloc - allocate a new bio, memory pool backed
- *      @gfp_mask: allocation mask to use
- *      @nr_iovecs: number of iovecs
- *
- *      bio_alloc will allocate a bio and associated bio_vec array that can hold
- *      at least @nr_iovecs entries. Allocations will be done from the
- *      fs_bio_set. Also see @bio_alloc_bioset and @bio_kmalloc.
- *
- *      If %__GFP_WAIT is set, then bio_alloc will always be able to allocate
- *      a bio. This is due to the mempool guarantees. To make this work, callers
- *      must never allocate more than 1 bio at a time from this pool. Callers
- *      that need to allocate more than 1 bio must always submit the previously
- *      allocated bio for IO before attempting to allocate a new one. Failure to
- *      do so can cause livelocks under memory pressure.
- *
- *      RETURNS:
- *      Pointer to new bio on success, NULL on failure.
- */
-struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
-{
-        struct bio *bio = bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set);
-
-        if (bio)
-                bio->bi_destructor = bio_fs_destructor;
-
-        return bio;
-}
-EXPORT_SYMBOL(bio_alloc);
-
-static void bio_kmalloc_destructor(struct bio *bio)
-{
-        if (bio_integrity(bio))
-                bio_integrity_free(bio, fs_bio_set);
-        kfree(bio);
-}
-
-/**
- * bio_kmalloc - allocate a bio for I/O using kmalloc()
- * @gfp_mask:   the GFP_ mask given to the slab allocator
- * @nr_iovecs:  number of iovecs to pre-allocate
- *
- * Description:
- *   Allocate a new bio with @nr_iovecs bvecs.  If @gfp_mask contains
- *   %__GFP_WAIT, the allocation is guaranteed to succeed.
- *
- **/
-struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)
-{
-        struct bio *bio;
-
-        if (nr_iovecs > UIO_MAXIOV)
-                return NULL;
-
-        bio = kmalloc(sizeof(struct bio) + nr_iovecs * sizeof(struct bio_vec),
-                      gfp_mask);
-        if (unlikely(!bio))
-                return NULL;
-
-        bio_init(bio);
-        bio->bi_flags |= BIO_POOL_NONE << BIO_POOL_OFFSET;
-        bio->bi_max_vecs = nr_iovecs;
-        bio->bi_io_vec = bio->bi_inline_vecs;
-        bio->bi_destructor = bio_kmalloc_destructor;
-
-        return bio;
-}
-EXPORT_SYMBOL(bio_kmalloc);
-
 void zero_fill_bio(struct bio *bio)
 {
         unsigned long flags;
@@ -420,11 +395,8 @@ void bio_put(struct bio *bio)
         /*
          * last put frees it
          */
-        if (atomic_dec_and_test(&bio->bi_cnt)) {
-                bio_disassociate_task(bio);
-                bio->bi_next = NULL;
-                bio->bi_destructor(bio);
-        }
+        if (atomic_dec_and_test(&bio->bi_cnt))
+                bio_free(bio);
 }
 EXPORT_SYMBOL(bio_put);
 
@@ -466,26 +438,28 @@ void __bio_clone(struct bio *bio, struct bio *bio_src)
 EXPORT_SYMBOL(__bio_clone);
 
 /**
- *      bio_clone       -       clone a bio
+ *      bio_clone_bioset -      clone a bio
  *      @bio: bio to clone
  *      @gfp_mask: allocation priority
+ *      @bs: bio_set to allocate from
  *
  *      Like __bio_clone, only also allocates the returned bio
  */
-struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask)
+struct bio *bio_clone_bioset(struct bio *bio, gfp_t gfp_mask,
+                             struct bio_set *bs)
 {
-        struct bio *b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, fs_bio_set);
+        struct bio *b;
 
+        b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, bs);
         if (!b)
                 return NULL;
 
-        b->bi_destructor = bio_fs_destructor;
         __bio_clone(b, bio);
 
         if (bio_integrity(bio)) {
                 int ret;
 
-                ret = bio_integrity_clone(b, bio, gfp_mask, fs_bio_set);
+                ret = bio_integrity_clone(b, bio, gfp_mask);
 
                 if (ret < 0) {
                         bio_put(b);
@@ -495,7 +469,7 @@ struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask)
 
         return b;
 }
-EXPORT_SYMBOL(bio_clone);
+EXPORT_SYMBOL(bio_clone_bioset);
 
 /**
  *      bio_get_nr_vecs         - return approx number of vecs
@@ -1501,7 +1475,7 @@ struct bio_pair *bio_split(struct bio *bi, int first_sectors)
         trace_block_split(bdev_get_queue(bi->bi_bdev), bi,
                                 bi->bi_sector + first_sectors);
 
-        BUG_ON(bi->bi_vcnt != 1);
+        BUG_ON(bi->bi_vcnt != 1 && bi->bi_vcnt != 0);
         BUG_ON(bi->bi_idx != 0);
         atomic_set(&bp->cnt, 3);
         bp->error = 0;
@@ -1511,17 +1485,22 @@ struct bio_pair *bio_split(struct bio *bi, int first_sectors)
         bp->bio2.bi_size -= first_sectors << 9;
         bp->bio1.bi_size = first_sectors << 9;
 
-        bp->bv1 = bi->bi_io_vec[0];
-        bp->bv2 = bi->bi_io_vec[0];
-        bp->bv2.bv_offset += first_sectors << 9;
-        bp->bv2.bv_len -= first_sectors << 9;
-        bp->bv1.bv_len = first_sectors << 9;
+        if (bi->bi_vcnt != 0) {
+                bp->bv1 = bi->bi_io_vec[0];
+                bp->bv2 = bi->bi_io_vec[0];
+
+                if (bio_is_rw(bi)) {
+                        bp->bv2.bv_offset += first_sectors << 9;
+                        bp->bv2.bv_len -= first_sectors << 9;
+                        bp->bv1.bv_len = first_sectors << 9;
+                }
 
-        bp->bio1.bi_io_vec = &bp->bv1;
-        bp->bio2.bi_io_vec = &bp->bv2;
+                bp->bio1.bi_io_vec = &bp->bv1;
+                bp->bio2.bi_io_vec = &bp->bv2;
 
-        bp->bio1.bi_max_vecs = 1;
-        bp->bio2.bi_max_vecs = 1;
+                bp->bio1.bi_max_vecs = 1;
+                bp->bio2.bi_max_vecs = 1;
+        }
 
         bp->bio1.bi_end_io = bio_pair_end_1;
         bp->bio2.bi_end_io = bio_pair_end_2;