block: Immutable bio vecs

This adds a mechanism by which we can advance a bio by an arbitrary
number of bytes without modifying the biovec: bio->bi_iter.bi_bvec_done
indicates the number of bytes completed in the current bvec.

Various driver code still needs to be updated to not refer to the bvec
directly before we can use this for interesting things, like efficient
bio splitting.

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Lars Ellenberg <drbd-dev@lists.linbit.com>
Cc: Paul Clements <Paul.Clements@steeleye.com>
Cc: drbd-user@lists.linbit.com
Cc: nbd-general@lists.sourceforge.net

3 files changed, 78 insertions, 10 deletions

diff --git a/include/linux/bio.h b/include/linux/bio.h
index c16adb5f69f8..04e592e74c92 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -64,11 +64,38 @@
 #define bio_iovec_idx(bio, idx) (&((bio)->bi_io_vec[(idx)]))
 #define __bio_iovec(bio)        bio_iovec_idx((bio), (bio)->bi_iter.bi_idx)
 
-#define bio_iter_iovec(bio, iter) ((bio)->bi_io_vec[(iter).bi_idx])
+#define __bvec_iter_bvec(bvec, iter)    (&(bvec)[(iter).bi_idx])
 
-#define bio_page(bio)           (bio_iovec((bio)).bv_page)
-#define bio_offset(bio)         (bio_iovec((bio)).bv_offset)
-#define bio_iovec(bio)          (*__bio_iovec(bio))
+#define bvec_iter_page(bvec, iter)                              \
+        (__bvec_iter_bvec((bvec), (iter))->bv_page)
+
+#define bvec_iter_len(bvec, iter)                               \
+        min((iter).bi_size,                                     \
+            __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done)
+
+#define bvec_iter_offset(bvec, iter)                            \
+        (__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done)
+
+#define bvec_iter_bvec(bvec, iter)                              \
+((struct bio_vec) {                                             \
+        .bv_page        = bvec_iter_page((bvec), (iter)),       \
+        .bv_len         = bvec_iter_len((bvec), (iter)),        \
+        .bv_offset      = bvec_iter_offset((bvec), (iter)),     \
+})
+
+#define bio_iter_iovec(bio, iter)                               \
+        bvec_iter_bvec((bio)->bi_io_vec, (iter))
+
+#define bio_iter_page(bio, iter)                                \
+        bvec_iter_page((bio)->bi_io_vec, (iter))
+#define bio_iter_len(bio, iter)                                 \
+        bvec_iter_len((bio)->bi_io_vec, (iter))
+#define bio_iter_offset(bio, iter)                              \
+        bvec_iter_offset((bio)->bi_io_vec, (iter))
+
+#define bio_page(bio)           bio_iter_page((bio), (bio)->bi_iter)
+#define bio_offset(bio)         bio_iter_offset((bio), (bio)->bi_iter)
+#define bio_iovec(bio)          bio_iter_iovec((bio), (bio)->bi_iter)
 
 #define bio_segments(bio)       ((bio)->bi_vcnt - (bio)->bi_iter.bi_idx)
 #define bio_sectors(bio)        ((bio)->bi_iter.bi_size >> 9)
@@ -145,16 +172,54 @@ static inline void *bio_data(struct bio *bio)
              bvl = bio_iovec_idx((bio), (i)), i < (bio)->bi_vcnt;       \
              i++)
 
+static inline void bvec_iter_advance(struct bio_vec *bv, struct bvec_iter *iter,
+                                     unsigned bytes)
+{
+        WARN_ONCE(bytes > iter->bi_size,
+                  "Attempted to advance past end of bvec iter\n");
+
+        while (bytes) {
+                unsigned len = min(bytes, bvec_iter_len(bv, *iter));
+
+                bytes -= len;
+                iter->bi_size -= len;
+                iter->bi_bvec_done += len;
+
+                if (iter->bi_bvec_done == __bvec_iter_bvec(bv, *iter)->bv_len) {
+                        iter->bi_bvec_done = 0;
+                        iter->bi_idx++;
+                }
+        }
+}
+
+#define for_each_bvec(bvl, bio_vec, iter, start)                        \
+        for ((iter) = start;                                            \
+             (bvl) = bvec_iter_bvec((bio_vec), (iter)),                 \
+                (iter).bi_size;                                         \
+             bvec_iter_advance((bio_vec), &(iter), (bvl).bv_len))
+
+
+static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,
+                                    unsigned bytes)
+{
+        iter->bi_sector += bytes >> 9;
+
+        if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK)
+                iter->bi_size -= bytes;
+        else
+                bvec_iter_advance(bio->bi_io_vec, iter, bytes);
+}
+
 #define __bio_for_each_segment(bvl, bio, iter, start)                   \
         for (iter = (start);                                            \
-             bvl = bio_iter_iovec((bio), (iter)),                       \
-             (iter).bi_idx < (bio)->bi_vcnt;                            \
-             (iter).bi_idx++)
+             (iter).bi_size &&                                          \
+                ((bvl = bio_iter_iovec((bio), (iter))), 1);             \
+             bio_advance_iter((bio), &(iter), (bvl).bv_len))
 
 #define bio_for_each_segment(bvl, bio, iter)                            \
         __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
 
-#define bio_iter_last(bio, iter) ((iter).bi_idx == (bio)->bi_vcnt - 1)
+#define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
 
 /*
  * get a reference to a bio, so it won't disappear. the intended use is
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 29b5b84d8a29..d369f8f6af79 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -34,6 +34,9 @@ struct bvec_iter {
         unsigned int            bi_size;        /* residual I/O count */
 
         unsigned int            bi_idx;         /* current index into bvl_vec */
+
+        unsigned int            bi_bvec_done;   /* number of bytes completed in
+                                                   current bvec */
 };
 
 /*
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 337b92a54658..02cb6f0ea71d 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -750,9 +750,9 @@ struct req_iterator {
         __rq_for_each_bio(_iter.bio, _rq)                       \
                 bio_for_each_segment(bvl, _iter.bio, _iter.iter)
 
-#define rq_iter_last(rq, _iter)                                 \
+#define rq_iter_last(bvec, _iter)                               \
                 (_iter.bio->bi_next == NULL &&                  \
-                 bio_iter_last(_iter.bio, _iter.iter))
+                 bio_iter_last(bvec, _iter.iter))
 
 #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
 # error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"