dm cache: promotion optimisation for writes

If a write block triggers promotion and covers a whole block we can
avoid a copy.

Introduce dm_{hook,unhook}_bio to simplify saving and restoring bio
fields (bi_private is now used by overwrite).  Switch writethrough
support over to using these helpers too.

Signed-off-by: Joe Thornber <ejt@redhat.com>
Signed-off-by: Mike Snitzer <snitzer@redhat.com>

1 files changed, 87 insertions, 6 deletions

diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c
index 4b564069e08f..655994fdf308 100644
--- a/drivers/md/dm-cache-target.c
+++ b/drivers/md/dm-cache-target.c
@@ -61,6 +61,34 @@ static void free_bitset(unsigned long *bits)
 
 /*----------------------------------------------------------------*/
 
+/*
+ * There are a couple of places where we let a bio run, but want to do some
+ * work before calling its endio function.  We do this by temporarily
+ * changing the endio fn.
+ */
+struct dm_hook_info {
+        bio_end_io_t *bi_end_io;
+        void *bi_private;
+};
+
+static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio,
+                        bio_end_io_t *bi_end_io, void *bi_private)
+{
+        h->bi_end_io = bio->bi_end_io;
+        h->bi_private = bio->bi_private;
+
+        bio->bi_end_io = bi_end_io;
+        bio->bi_private = bi_private;
+}
+
+static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio)
+{
+        bio->bi_end_io = h->bi_end_io;
+        bio->bi_private = h->bi_private;
+}
+
+/*----------------------------------------------------------------*/
+
 #define PRISON_CELLS 1024
 #define MIGRATION_POOL_SIZE 128
 #define COMMIT_PERIOD HZ
@@ -214,7 +242,7 @@ struct per_bio_data {
          */
         struct cache *cache;
         dm_cblock_t cblock;
-        bio_end_io_t *saved_bi_end_io;
+        struct dm_hook_info hook_info;
         struct dm_bio_details bio_details;
 };
 
@@ -231,6 +259,7 @@ struct dm_cache_migration {
         bool writeback:1;
         bool demote:1;
         bool promote:1;
+        bool requeue_holder:1;
 
         struct dm_bio_prison_cell *old_ocell;
         struct dm_bio_prison_cell *new_ocell;
@@ -666,7 +695,8 @@ static void defer_writethrough_bio(struct cache *cache, struct bio *bio)
 static void writethrough_endio(struct bio *bio, int err)
 {
         struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT);
-        bio->bi_end_io = pb->saved_bi_end_io;
+
+        dm_unhook_bio(&pb->hook_info, bio);
 
         if (err) {
                 bio_endio(bio, err);
@@ -697,9 +727,8 @@ static void remap_to_origin_then_cache(struct cache *cache, struct bio *bio,
 
         pb->cache = cache;
         pb->cblock = cblock;
-        pb->saved_bi_end_io = bio->bi_end_io;
+        dm_hook_bio(&pb->hook_info, bio, writethrough_endio, NULL);
         dm_bio_record(&pb->bio_details, bio);
-        bio->bi_end_io = writethrough_endio;
 
         remap_to_origin_clear_discard(pb->cache, bio, oblock);
 }
@@ -841,7 +870,12 @@ static void migration_success_post_commit(struct dm_cache_migration *mg)
                         cleanup_migration(mg);
 
         } else {
-                cell_defer(cache, mg->new_ocell, true);
+                if (mg->requeue_holder)
+                        cell_defer(cache, mg->new_ocell, true);
+                else {
+                        bio_endio(mg->new_ocell->holder, 0);
+                        cell_defer(cache, mg->new_ocell, false);
+                }
                 clear_dirty(cache, mg->new_oblock, mg->cblock);
                 cleanup_migration(mg);
         }
@@ -892,6 +926,42 @@ static void issue_copy_real(struct dm_cache_migration *mg)
         }
 }
 
+static void overwrite_endio(struct bio *bio, int err)
+{
+        struct dm_cache_migration *mg = bio->bi_private;
+        struct cache *cache = mg->cache;
+        size_t pb_data_size = get_per_bio_data_size(cache);
+        struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
+        unsigned long flags;
+
+        if (err)
+                mg->err = true;
+
+        spin_lock_irqsave(&cache->lock, flags);
+        list_add_tail(&mg->list, &cache->completed_migrations);
+        dm_unhook_bio(&pb->hook_info, bio);
+        mg->requeue_holder = false;
+        spin_unlock_irqrestore(&cache->lock, flags);
+
+        wake_worker(cache);
+}
+
+static void issue_overwrite(struct dm_cache_migration *mg, struct bio *bio)
+{
+        size_t pb_data_size = get_per_bio_data_size(mg->cache);
+        struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
+
+        dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);
+        remap_to_cache_dirty(mg->cache, bio, mg->new_oblock, mg->cblock);
+        generic_make_request(bio);
+}
+
+static bool bio_writes_complete_block(struct cache *cache, struct bio *bio)
+{
+        return (bio_data_dir(bio) == WRITE) &&
+                (bio->bi_size == (cache->sectors_per_block << SECTOR_SHIFT));
+}
+
 static void avoid_copy(struct dm_cache_migration *mg)
 {
         atomic_inc(&mg->cache->stats.copies_avoided);
@@ -906,9 +976,17 @@ static void issue_copy(struct dm_cache_migration *mg)
         if (mg->writeback || mg->demote)
                 avoid = !is_dirty(cache, mg->cblock) ||
                         is_discarded_oblock(cache, mg->old_oblock);
-        else
+        else {
+                struct bio *bio = mg->new_ocell->holder;
+
                 avoid = is_discarded_oblock(cache, mg->new_oblock);
 
+                if (!avoid && bio_writes_complete_block(cache, bio)) {
+                        issue_overwrite(mg, bio);
+                        return;
+                }
+        }
+
         avoid ? avoid_copy(mg) : issue_copy_real(mg);
 }
 
@@ -998,6 +1076,7 @@ static void promote(struct cache *cache, struct prealloc *structs,
         mg->writeback = false;
         mg->demote = false;
         mg->promote = true;
+        mg->requeue_holder = true;
         mg->cache = cache;
         mg->new_oblock = oblock;
         mg->cblock = cblock;
@@ -1019,6 +1098,7 @@ static void writeback(struct cache *cache, struct prealloc *structs,
         mg->writeback = true;
         mg->demote = false;
         mg->promote = false;
+        mg->requeue_holder = true;
         mg->cache = cache;
         mg->old_oblock = oblock;
         mg->cblock = cblock;
@@ -1042,6 +1122,7 @@ static void demote_then_promote(struct cache *cache, struct prealloc *structs,
         mg->writeback = false;
         mg->demote = true;
         mg->promote = true;
+        mg->requeue_holder = true;
         mg->cache = cache;
         mg->old_oblock = old_oblock;
         mg->new_oblock = new_oblock;