1 files changed, 72 insertions, 53 deletions

diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index 7ca2bf2aafaa..188121ca00aa 100644
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -523,7 +523,7 @@ struct pool {
 
         struct bio_list retry_on_resume_list;
 
-        struct deferred_set ds; /* FIXME: move to thin_c */
+        struct deferred_set shared_read_ds;
 
         struct new_mapping *next_mapping;
         mempool_t *mapping_pool;
@@ -618,6 +618,12 @@ static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev
 
 /*----------------------------------------------------------------*/
 
+struct endio_hook {
+        struct thin_c *tc;
+        struct deferred_entry *shared_read_entry;
+        struct new_mapping *overwrite_mapping;
+};
+
 static void __requeue_bio_list(struct thin_c *tc, struct bio_list *master)
 {
         struct bio *bio;
@@ -628,7 +634,8 @@ static void __requeue_bio_list(struct thin_c *tc, struct bio_list *master)
         bio_list_init(master);
 
         while ((bio = bio_list_pop(&bios))) {
-                if (dm_get_mapinfo(bio)->ptr == tc)
+                struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
+                if (h->tc == tc)
                         bio_endio(bio, DM_ENDIO_REQUEUE);
                 else
                         bio_list_add(master, bio);
@@ -716,16 +723,11 @@ static void wake_worker(struct pool *pool)
 /*
  * Bio endio functions.
  */
-struct endio_hook {
-        struct thin_c *tc;
-        bio_end_io_t *saved_bi_end_io;
-        struct deferred_entry *entry;
-};
-
 struct new_mapping {
         struct list_head list;
 
-        int prepared;
+        unsigned quiesced:1;
+        unsigned prepared:1;
 
         struct thin_c *tc;
         dm_block_t virt_block;
@@ -747,7 +749,7 @@ static void __maybe_add_mapping(struct new_mapping *m)
 {
         struct pool *pool = m->tc->pool;
 
-        if (list_empty(&m->list) && m->prepared) {
+        if (m->quiesced && m->prepared) {
                 list_add(&m->list, &pool->prepared_mappings);
                 wake_worker(pool);
         }
@@ -770,7 +772,8 @@ static void copy_complete(int read_err, unsigned long write_err, void *context)
 static void overwrite_endio(struct bio *bio, int err)
 {
         unsigned long flags;
-        struct new_mapping *m = dm_get_mapinfo(bio)->ptr;
+        struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
+        struct new_mapping *m = h->overwrite_mapping;
         struct pool *pool = m->tc->pool;
 
         m->err = err;
@@ -781,31 +784,6 @@ static void overwrite_endio(struct bio *bio, int err)
         spin_unlock_irqrestore(&pool->lock, flags);
 }
 
-static void shared_read_endio(struct bio *bio, int err)
-{
-        struct list_head mappings;
-        struct new_mapping *m, *tmp;
-        struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
-        unsigned long flags;
-        struct pool *pool = h->tc->pool;
-
-        bio->bi_end_io = h->saved_bi_end_io;
-        bio_endio(bio, err);
-
-        INIT_LIST_HEAD(&mappings);
-        ds_dec(h->entry, &mappings);
-
-        spin_lock_irqsave(&pool->lock, flags);
-        list_for_each_entry_safe(m, tmp, &mappings, list) {
-                list_del(&m->list);
-                INIT_LIST_HEAD(&m->list);
-                __maybe_add_mapping(m);
-        }
-        spin_unlock_irqrestore(&pool->lock, flags);
-
-        mempool_free(h, pool->endio_hook_pool);
-}
-
 /*----------------------------------------------------------------*/
 
 /*
@@ -957,6 +935,7 @@ static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
         struct new_mapping *m = get_next_mapping(pool);
 
         INIT_LIST_HEAD(&m->list);
+        m->quiesced = 0;
         m->prepared = 0;
         m->tc = tc;
         m->virt_block = virt_block;
@@ -965,7 +944,8 @@ static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
         m->err = 0;
         m->bio = NULL;
 
-        ds_add_work(&pool->ds, &m->list);
+        if (!ds_add_work(&pool->shared_read_ds, &m->list))
+                m->quiesced = 1;
 
         /*
          * IO to pool_dev remaps to the pool target's data_dev.
@@ -974,9 +954,10 @@ static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
          * bio immediately. Otherwise we use kcopyd to clone the data first.
          */
         if (io_overwrites_block(pool, bio)) {
+                struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
+                h->overwrite_mapping = m;
                 m->bio = bio;
                 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
-                dm_get_mapinfo(bio)->ptr = m;
                 remap_and_issue(tc, bio, data_dest);
         } else {
                 struct dm_io_region from, to;
@@ -1023,6 +1004,7 @@ static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
         struct new_mapping *m = get_next_mapping(pool);
 
         INIT_LIST_HEAD(&m->list);
+        m->quiesced = 1;
         m->prepared = 0;
         m->tc = tc;
         m->virt_block = virt_block;
@@ -1040,9 +1022,10 @@ static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
                 process_prepared_mapping(m);
 
         else if (io_overwrites_block(pool, bio)) {
+                struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
+                h->overwrite_mapping = m;
                 m->bio = bio;
                 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
-                dm_get_mapinfo(bio)->ptr = m;
                 remap_and_issue(tc, bio, data_block);
 
         } else {
@@ -1129,7 +1112,8 @@ static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
  */
 static void retry_on_resume(struct bio *bio)
 {
-        struct thin_c *tc = dm_get_mapinfo(bio)->ptr;
+        struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
+        struct thin_c *tc = h->tc;
         struct pool *pool = tc->pool;
         unsigned long flags;
 
@@ -1195,13 +1179,9 @@ static void process_shared_bio(struct thin_c *tc, struct bio *bio,
         if (bio_data_dir(bio) == WRITE)
                 break_sharing(tc, bio, block, &key, lookup_result, cell);
         else {
-                struct endio_hook *h;
+                struct endio_hook *h = dm_get_mapinfo(bio)->ptr;