1 files changed, 71 insertions, 44 deletions

diff --git a/include/linux/bio.h b/include/linux/bio.h
index 820e7aaad4fd..ef24466d8f82 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -67,6 +67,7 @@
 #define bio_offset(bio)         bio_iovec((bio))->bv_offset
 #define bio_segments(bio)       ((bio)->bi_vcnt - (bio)->bi_idx)
 #define bio_sectors(bio)        ((bio)->bi_size >> 9)
+#define bio_end_sector(bio)     ((bio)->bi_sector + bio_sectors((bio)))
 
 static inline unsigned int bio_cur_bytes(struct bio *bio)
 {
@@ -84,11 +85,6 @@ static inline void *bio_data(struct bio *bio)
         return NULL;
 }
 
-static inline int bio_has_allocated_vec(struct bio *bio)
-{
-        return bio->bi_io_vec && bio->bi_io_vec != bio->bi_inline_vecs;
-}
-
 /*
  * will die
  */
@@ -136,16 +132,27 @@ static inline int bio_has_allocated_vec(struct bio *bio)
 #define bio_io_error(bio) bio_endio((bio), -EIO)
 
 /*
- * drivers should not use the __ version unless they _really_ want to
- * run through the entire bio and not just pending pieces
+ * drivers should not use the __ version unless they _really_ know what
+ * they're doing
  */
 #define __bio_for_each_segment(bvl, bio, i, start_idx)                  \
         for (bvl = bio_iovec_idx((bio), (start_idx)), i = (start_idx);  \
              i < (bio)->bi_vcnt;                                        \
              bvl++, i++)
 
+/*
+ * drivers should _never_ use the all version - the bio may have been split
+ * before it got to the driver and the driver won't own all of it
+ */
+#define bio_for_each_segment_all(bvl, bio, i)                           \
+        for (i = 0;                                                     \
+             bvl = bio_iovec_idx((bio), (i)), i < (bio)->bi_vcnt;       \
+             i++)
+
 #define bio_for_each_segment(bvl, bio, i)                               \
-        __bio_for_each_segment(bvl, bio, i, (bio)->bi_idx)
+        for (i = (bio)->bi_idx;                                         \
+             bvl = bio_iovec_idx((bio), (i)), i < (bio)->bi_vcnt;       \
+             i++)
 
 /*
  * get a reference to a bio, so it won't disappear. the intended use is
@@ -180,9 +187,12 @@ struct bio_integrity_payload {
         unsigned short          bip_slab;       /* slab the bip came from */
         unsigned short          bip_vcnt;       /* # of integrity bio_vecs */
         unsigned short          bip_idx;        /* current bip_vec index */
+        unsigned                bip_owns_buf:1; /* should free bip_buf */
 
         struct work_struct      bip_work;       /* I/O completion */
-        struct bio_vec          bip_vec[0];     /* embedded bvec array */
+
+        struct bio_vec          *bip_vec;
+        struct bio_vec          bip_inline_vecs[0];/* embedded bvec array */
 };
 #endif /* CONFIG_BLK_DEV_INTEGRITY */
 
@@ -211,6 +221,7 @@ extern void bio_pair_release(struct bio_pair *dbio);
 
 extern struct bio_set *bioset_create(unsigned int, unsigned int);
 extern void bioset_free(struct bio_set *);
+extern mempool_t *biovec_create_pool(struct bio_set *bs, int pool_entries);
 
 extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);
 extern void bio_put(struct bio *);
@@ -245,6 +256,9 @@ extern void bio_endio(struct bio *, int);
 struct request_queue;
 extern int bio_phys_segments(struct request_queue *, struct bio *);
 
+extern int submit_bio_wait(int rw, struct bio *bio);
+extern void bio_advance(struct bio *, unsigned);
+
 extern void bio_init(struct bio *);
 extern void bio_reset(struct bio *);
 
@@ -279,6 +293,9 @@ static inline void bio_flush_dcache_pages(struct bio *bi)
 }
 #endif
 
+extern void bio_copy_data(struct bio *dst, struct bio *src);
+extern int bio_alloc_pages(struct bio *bio, gfp_t gfp);
+
 extern struct bio *bio_copy_user(struct request_queue *, struct rq_map_data *,
                                  unsigned long, unsigned int, int, gfp_t);
 extern struct bio *bio_copy_user_iov(struct request_queue *,
@@ -286,8 +303,8 @@ extern struct bio *bio_copy_user_iov(struct request_queue *,
                                      int, int, gfp_t);
 extern int bio_uncopy_user(struct bio *);
 void zero_fill_bio(struct bio *bio);
-extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set *);
-extern void bvec_free_bs(struct bio_set *, struct bio_vec *, unsigned int);
+extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
+extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int);
 extern unsigned int bvec_nr_vecs(unsigned short idx);
 
 #ifdef CONFIG_BLK_CGROUP
@@ -298,39 +315,6 @@ static inline int bio_associate_current(struct bio *bio) { return -ENOENT; }
 static inline void bio_disassociate_task(struct bio *bio) { }
 #endif  /* CONFIG_BLK_CGROUP */
 
-/*
- * bio_set is used to allow other portions of the IO system to
- * allocate their own private memory pools for bio and iovec structures.
- * These memory pools in turn all allocate from the bio_slab
- * and the bvec_slabs[].
- */
-#define BIO_POOL_SIZE 2
-#define BIOVEC_NR_POOLS 6
-#define BIOVEC_MAX_IDX  (BIOVEC_NR_POOLS - 1)
-
-struct bio_set {
-        struct kmem_cache *bio_slab;
-        unsigned int front_pad;
-
-        mempool_t *bio_pool;
-#if defined(CONFIG_BLK_DEV_INTEGRITY)
-        mempool_t *bio_integrity_pool;
-#endif
-        mempool_t *bvec_pool;
-};
-
-struct biovec_slab {
-        int nr_vecs;
-        char *name;
-        struct kmem_cache *slab;
-};
-
-/*
- * a small number of entries is fine, not going to be performance critical.
- * basically we just need to survive
- */
-#define BIO_SPLIT_ENTRIES 2
-
 #ifdef CONFIG_HIGHMEM
 /*
  * remember never ever reenable interrupts between a bvec_kmap_irq and
@@ -527,6 +511,49 @@ static inline struct bio *bio_list_get(struct bio_list *bl)
         return bio;
 }
 
+/*
+ * bio_set is used to allow other portions of the IO system to
+ * allocate their own private memory pools for bio and iovec structures.
+ * These memory pools in turn all allocate from the bio_slab
+ * and the bvec_slabs[].
+ */
+#define BIO_POOL_SIZE 2
+#define BIOVEC_NR_POOLS 6
+#define BIOVEC_MAX_IDX  (BIOVEC_NR_POOLS - 1)
+
+struct bio_set {
+        struct kmem_cache *bio_slab;
+        unsigned int front_pad;
+
+        mempool_t *bio_pool;
+        mempool_t *bvec_pool;
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+        mempool_t *bio_integrity_pool;
+        mempool_t *bvec_integrity_pool;
+#endif
+
+        /*
+         * Deadlock avoidance for stacking block drivers: see comments in
+         * bio_alloc_bioset() for details
+         */
+        spinlock_t              rescue_lock;
+        struct bio_list         rescue_list;
+        struct work_struct      rescue_work;
+        struct workqueue_struct *rescue_workqueue;
+};
+
+struct biovec_slab {
+        int nr_vecs;
+        char *name;
+        struct kmem_cache *slab;
+};
+
+/*
+ * a small number of entries is fine, not going to be performance critical.
+ * basically we just need to survive
+ */
+#define BIO_SPLIT_ENTRIES 2
+
 #if defined(CONFIG_BLK_DEV_INTEGRITY)
 
 #define bip_vec_idx(bip, idx)   (&(bip->bip_vec[(idx)]))