3 files changed, 103 insertions, 3 deletions

diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c
index 98725e119324..f82147029636 100644
--- a/drivers/md/dm-kcopyd.c
+++ b/drivers/md/dm-kcopyd.c
@@ -617,6 +617,37 @@ int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,
 }
 EXPORT_SYMBOL(dm_kcopyd_copy);
 
+void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc,
+                                 dm_kcopyd_notify_fn fn, void *context)
+{
+        struct kcopyd_job *job;
+
+        job = mempool_alloc(kc->job_pool, GFP_NOIO);
+
+        memset(job, 0, sizeof(struct kcopyd_job));
+        job->kc = kc;
+        job->fn = fn;
+        job->context = context;
+
+        atomic_inc(&kc->nr_jobs);
+
+        return job;
+}
+EXPORT_SYMBOL(dm_kcopyd_prepare_callback);
+
+void dm_kcopyd_do_callback(void *j, int read_err, unsigned long write_err)
+{
+        struct kcopyd_job *job = j;
+        struct dm_kcopyd_client *kc = job->kc;
+
+        job->read_err = read_err;
+        job->write_err = write_err;
+
+        push(&kc->complete_jobs, job);
+        wake(kc);
+}
+EXPORT_SYMBOL(dm_kcopyd_do_callback);
+
 /*
  * Cancels a kcopyd job, eg. someone might be deactivating a
  * mirror.
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index 94dee05dd28e..6f758870fc19 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -170,6 +170,13 @@ struct dm_snap_pending_exception {
          * kcopyd.
          */
         int started;
+
+        /*
+         * For writing a complete chunk, bypassing the copy.
+         */
+        struct bio *full_bio;
+        bio_end_io_t *full_bio_end_io;
+        void *full_bio_private;
 };
 
 /*
@@ -1369,6 +1376,7 @@ static void pending_complete(struct dm_snap_pending_exception *pe, int success)
         struct dm_snapshot *s = pe->snap;
         struct bio *origin_bios = NULL;
         struct bio *snapshot_bios = NULL;
+        struct bio *full_bio = NULL;
         int error = 0;
 
         if (!success) {
@@ -1408,6 +1416,11 @@ out:
         dm_remove_exception(&pe->e);
         snapshot_bios = bio_list_get(&pe->snapshot_bios);
         origin_bios = bio_list_get(&pe->origin_bios);
+        full_bio = pe->full_bio;
+        if (full_bio) {
+                full_bio->bi_end_io = pe->full_bio_end_io;
+                full_bio->bi_private = pe->full_bio_private;
+        }
         free_pending_exception(pe);
 
         increment_pending_exceptions_done_count();
@@ -1415,10 +1428,15 @@ out:
         up_write(&s->lock);
 
         /* Submit any pending write bios */
-        if (error)
+        if (error) {
+                if (full_bio)
+                        bio_io_error(full_bio);
                 error_bios(snapshot_bios);
-        else
+        } else {
+                if (full_bio)
+                        bio_endio(full_bio, 0);
                 flush_bios(snapshot_bios);
+        }
 
         retry_origin_bios(s, origin_bios);
 }
@@ -1472,6 +1490,32 @@ static void start_copy(struct dm_snap_pending_exception *pe)
         dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe);
 }
 
+static void full_bio_end_io(struct bio *bio, int error)
+{
+        void *callback_data = bio->bi_private;
+
+        dm_kcopyd_do_callback(callback_data, 0, error ? 1 : 0);
+}
+
+static void start_full_bio(struct dm_snap_pending_exception *pe,
+                           struct bio *bio)
+{
+        struct dm_snapshot *s = pe->snap;
+        void *callback_data;
+
+        pe->full_bio = bio;
+        pe->full_bio_end_io = bio->bi_end_io;
+        pe->full_bio_private = bio->bi_private;
+
+        callback_data = dm_kcopyd_prepare_callback(s->kcopyd_client,
+                                                   copy_callback, pe);
+
+        bio->bi_end_io = full_bio_end_io;
+        bio->bi_private = callback_data;
+
+        generic_make_request(bio);
+}
+
 static struct dm_snap_pending_exception *
 __lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk)
 {
@@ -1507,6 +1551,7 @@ __find_pending_exception(struct dm_snapshot *s,
         bio_list_init(&pe->origin_bios);
         bio_list_init(&pe->snapshot_bios);
         pe->started = 0;
+        pe->full_bio = NULL;
 
         if (s->store->type->prepare_exception(s->store, &pe->e)) {
                 free_pending_exception(pe);
@@ -1600,10 +1645,19 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio,
                 }
 
                 remap_exception(s, &pe->e, bio, chunk);
-                bio_list_add(&pe->snapshot_bios, bio);
 
                 r = DM_MAPIO_SUBMITTED;
 
+                if (!pe->started &&
+                    bio->bi_size == (s->store->chunk_size << SECTOR_SHIFT)) {
+                        pe->started = 1;
+                        up_write(&s->lock);
+                        start_full_bio(pe, bio);
+                        goto out;
+                }
+
+                bio_list_add(&pe->snapshot_bios, bio);
+
                 if (!pe->started) {
                         /* this is protected by snap->lock */
                         pe->started = 1;
diff --git a/include/linux/dm-kcopyd.h b/include/linux/dm-kcopyd.h
index 298d587e349b..5e54458e920f 100644
--- a/include/linux/dm-kcopyd.h
+++ b/include/linux/dm-kcopyd.h
@@ -42,5 +42,20 @@ int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,
                    unsigned num_dests, struct dm_io_region *dests,
                    unsigned flags, dm_kcopyd_notify_fn fn, void *context);
 
+/*
+ * Prepare a callback and submit it via the kcopyd thread.
+ *
+ * dm_kcopyd_prepare_callback allocates a callback structure and returns it.
+ * It must not be called from interrupt context.
+ * The returned value should be passed into dm_kcopyd_do_callback.
+ *
+ * dm_kcopyd_do_callback submits the callback.
+ * It may be called from interrupt context.
+ * The callback is issued from the kcopyd thread.
+ */
+void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc,
+                                 dm_kcopyd_notify_fn fn, void *context);
+void dm_kcopyd_do_callback(void *job, int read_err, unsigned long write_err);
+
 #endif  /* __KERNEL__ */
 #endif  /* _LINUX_DM_KCOPYD_H */