aboutsummaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2009-04-09 11:01:21 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2009-04-09 11:01:21 -0400
commitdf5529297ec4c32d57dd149398e3add6c37bc054 (patch)
tree24e40e2286f2d6396994ed92906420890b4cf9c3 /drivers
parent97c18e2c7a8e36d2d83d50ee070314aadac73a11 (diff)
parent340cd44451fb0bfa542365e6b4b565bbd44836e2 (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/agk/linux-2.6-dm
* git://git.kernel.org/pub/scm/linux/kernel/git/agk/linux-2.6-dm: dm kcopyd: fix callback race dm kcopyd: prepare for callback race fix dm: implement basic barrier support dm: remove dm_request loop dm: rework queueing and suspension dm: simplify dm_request loop dm: split DMF_BLOCK_IO flag into two dm: rearrange dm_wq_work dm: remove limited barrier support dm: add integrity support
Diffstat (limited to 'drivers')
-rw-r--r--drivers/md/dm-ioctl.c21
-rw-r--r--drivers/md/dm-kcopyd.c23
-rw-r--r--drivers/md/dm-linear.c1
-rw-r--r--drivers/md/dm-table.c59
-rw-r--r--drivers/md/dm.c199
-rw-r--r--drivers/md/dm.h1
6 files changed, 218 insertions, 86 deletions
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index f01096549a93..823ceba6efa8 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -1047,6 +1047,19 @@ static int populate_table(struct dm_table *table,
1047 return dm_table_complete(table); 1047 return dm_table_complete(table);
1048} 1048}
1049 1049
1050static int table_prealloc_integrity(struct dm_table *t,
1051 struct mapped_device *md)
1052{
1053 struct list_head *devices = dm_table_get_devices(t);
1054 struct dm_dev_internal *dd;
1055
1056 list_for_each_entry(dd, devices, list)
1057 if (bdev_get_integrity(dd->dm_dev.bdev))
1058 return blk_integrity_register(dm_disk(md), NULL);
1059
1060 return 0;
1061}
1062
1050static int table_load(struct dm_ioctl *param, size_t param_size) 1063static int table_load(struct dm_ioctl *param, size_t param_size)
1051{ 1064{
1052 int r; 1065 int r;
@@ -1068,6 +1081,14 @@ static int table_load(struct dm_ioctl *param, size_t param_size)
1068 goto out; 1081 goto out;
1069 } 1082 }
1070 1083
1084 r = table_prealloc_integrity(t, md);
1085 if (r) {
1086 DMERR("%s: could not register integrity profile.",
1087 dm_device_name(md));
1088 dm_table_destroy(t);
1089 goto out;
1090 }
1091
1071 down_write(&_hash_lock); 1092 down_write(&_hash_lock);
1072 hc = dm_get_mdptr(md); 1093 hc = dm_get_mdptr(md);
1073 if (!hc || hc->md != md) { 1094 if (!hc || hc->md != md) {
diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c
index 0a225da21272..3e3fc06cb861 100644
--- a/drivers/md/dm-kcopyd.c
+++ b/drivers/md/dm-kcopyd.c
@@ -297,7 +297,8 @@ static int run_complete_job(struct kcopyd_job *job)
297 dm_kcopyd_notify_fn fn = job->fn; 297 dm_kcopyd_notify_fn fn = job->fn;
298 struct dm_kcopyd_client *kc = job->kc; 298 struct dm_kcopyd_client *kc = job->kc;
299 299
300 kcopyd_put_pages(kc, job->pages); 300 if (job->pages)
301 kcopyd_put_pages(kc, job->pages);
301 mempool_free(job, kc->job_pool); 302 mempool_free(job, kc->job_pool);
302 fn(read_err, write_err, context); 303 fn(read_err, write_err, context);
303 304
@@ -461,6 +462,7 @@ static void segment_complete(int read_err, unsigned long write_err,
461 sector_t progress = 0; 462 sector_t progress = 0;
462 sector_t count = 0; 463 sector_t count = 0;
463 struct kcopyd_job *job = (struct kcopyd_job *) context; 464 struct kcopyd_job *job = (struct kcopyd_job *) context;
465 struct dm_kcopyd_client *kc = job->kc;
464 466
465 mutex_lock(&job->lock); 467 mutex_lock(&job->lock);
466 468
@@ -490,7 +492,7 @@ static void segment_complete(int read_err, unsigned long write_err,
490 492
491 if (count) { 493 if (count) {
492 int i; 494 int i;
493 struct kcopyd_job *sub_job = mempool_alloc(job->kc->job_pool, 495 struct kcopyd_job *sub_job = mempool_alloc(kc->job_pool,
494 GFP_NOIO); 496 GFP_NOIO);
495 497
496 *sub_job = *job; 498 *sub_job = *job;
@@ -509,13 +511,16 @@ static void segment_complete(int read_err, unsigned long write_err,
509 } else if (atomic_dec_and_test(&job->sub_jobs)) { 511 } else if (atomic_dec_and_test(&job->sub_jobs)) {
510 512
511 /* 513 /*
512 * To avoid a race we must keep the job around 514 * Queue the completion callback to the kcopyd thread.
513 * until after the notify function has completed. 515 *
514 * Otherwise the client may try and stop the job 516 * Some callers assume that all the completions are called
515 * after we've completed. 517 * from a single thread and don't race with each other.
518 *
519 * We must not call the callback directly here because this
520 * code may not be executing in the thread.
516 */ 521 */
517 job->fn(read_err, write_err, job->context); 522 push(&kc->complete_jobs, job);
518 mempool_free(job, job->kc->job_pool); 523 wake(kc);
519 } 524 }
520} 525}
521 526
@@ -528,6 +533,8 @@ static void split_job(struct kcopyd_job *job)
528{ 533{
529 int i; 534 int i;
530 535
536 atomic_inc(&job->kc->nr_jobs);
537
531 atomic_set(&job->sub_jobs, SPLIT_COUNT); 538 atomic_set(&job->sub_jobs, SPLIT_COUNT);
532 for (i = 0; i < SPLIT_COUNT; i++) 539 for (i = 0; i < SPLIT_COUNT; i++)
533 segment_complete(0, 0u, job); 540 segment_complete(0, 0u, job);
diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c
index bfa107f59d96..79fb53e51c70 100644
--- a/drivers/md/dm-linear.c
+++ b/drivers/md/dm-linear.c
@@ -142,7 +142,6 @@ static struct target_type linear_target = {
142 .status = linear_status, 142 .status = linear_status,
143 .ioctl = linear_ioctl, 143 .ioctl = linear_ioctl,
144 .merge = linear_merge, 144 .merge = linear_merge,
145 .features = DM_TARGET_SUPPORTS_BARRIERS,
146}; 145};
147 146
148int __init dm_linear_init(void) 147int __init dm_linear_init(void)
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index e8361b191b9b..429b50b975d5 100644
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -52,8 +52,6 @@ struct dm_table {
52 sector_t *highs; 52 sector_t *highs;
53 struct dm_target *targets; 53 struct dm_target *targets;
54 54
55 unsigned barriers_supported:1;
56
57 /* 55 /*
58 * Indicates the rw permissions for the new logical 56 * Indicates the rw permissions for the new logical
59 * device. This should be a combination of FMODE_READ 57 * device. This should be a combination of FMODE_READ
@@ -243,7 +241,6 @@ int dm_table_create(struct dm_table **result, fmode_t mode,
243 241
244 INIT_LIST_HEAD(&t->devices); 242 INIT_LIST_HEAD(&t->devices);
245 atomic_set(&t->holders, 0); 243 atomic_set(&t->holders, 0);
246 t->barriers_supported = 1;
247 244
248 if (!num_targets) 245 if (!num_targets)
249 num_targets = KEYS_PER_NODE; 246 num_targets = KEYS_PER_NODE;
@@ -751,10 +748,6 @@ int dm_table_add_target(struct dm_table *t, const char *type,
751 /* FIXME: the plan is to combine high here and then have 748 /* FIXME: the plan is to combine high here and then have
752 * the merge fn apply the target level restrictions. */ 749 * the merge fn apply the target level restrictions. */
753 combine_restrictions_low(&t->limits, &tgt->limits); 750 combine_restrictions_low(&t->limits, &tgt->limits);
754
755 if (!(tgt->type->features & DM_TARGET_SUPPORTS_BARRIERS))
756 t->barriers_supported = 0;
757
758 return 0; 751 return 0;
759 752
760 bad: 753 bad:
@@ -799,12 +792,6 @@ int dm_table_complete(struct dm_table *t)
799 792
800 check_for_valid_limits(&t->limits); 793 check_for_valid_limits(&t->limits);
801 794
802 /*
803 * We only support barriers if there is exactly one underlying device.
804 */
805 if (!list_is_singular(&t->devices))
806 t->barriers_supported = 0;
807
808 /* how many indexes will the btree have ? */ 795 /* how many indexes will the btree have ? */
809 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE); 796 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);
810 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE); 797 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);
@@ -879,6 +866,45 @@ struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)
879 return &t->targets[(KEYS_PER_NODE * n) + k]; 866 return &t->targets[(KEYS_PER_NODE * n) + k];
880} 867}
881 868
869/*
870 * Set the integrity profile for this device if all devices used have
871 * matching profiles.
872 */
873static void dm_table_set_integrity(struct dm_table *t)
874{
875 struct list_head *devices = dm_table_get_devices(t);
876 struct dm_dev_internal *prev = NULL, *dd = NULL;
877
878 if (!blk_get_integrity(dm_disk(t->md)))
879 return;
880
881 list_for_each_entry(dd, devices, list) {
882 if (prev &&
883 blk_integrity_compare(prev->dm_dev.bdev->bd_disk,
884 dd->dm_dev.bdev->bd_disk) < 0) {
885 DMWARN("%s: integrity not set: %s and %s mismatch",
886 dm_device_name(t->md),
887 prev->dm_dev.bdev->bd_disk->disk_name,
888 dd->dm_dev.bdev->bd_disk->disk_name);
889 goto no_integrity;
890 }
891 prev = dd;
892 }
893
894 if (!prev || !bdev_get_integrity(prev->dm_dev.bdev))
895 goto no_integrity;
896
897 blk_integrity_register(dm_disk(t->md),
898 bdev_get_integrity(prev->dm_dev.bdev));
899
900 return;
901
902no_integrity:
903 blk_integrity_register(dm_disk(t->md), NULL);
904
905 return;
906}
907
882void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q) 908void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q)
883{ 909{
884 /* 910 /*
@@ -899,6 +925,7 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q)
899 else 925 else
900 queue_flag_set_unlocked(QUEUE_FLAG_CLUSTER, q); 926 queue_flag_set_unlocked(QUEUE_FLAG_CLUSTER, q);
901 927
928 dm_table_set_integrity(t);
902} 929}
903 930
904unsigned int dm_table_get_num_targets(struct dm_table *t) 931unsigned int dm_table_get_num_targets(struct dm_table *t)
@@ -1019,12 +1046,6 @@ struct mapped_device *dm_table_get_md(struct dm_table *t)
1019 return t->md; 1046 return t->md;
1020} 1047}
1021 1048
1022int dm_table_barrier_ok(struct dm_table *t)
1023{
1024 return t->barriers_supported;
1025}
1026EXPORT_SYMBOL(dm_table_barrier_ok);
1027
1028EXPORT_SYMBOL(dm_vcalloc); 1049EXPORT_SYMBOL(dm_vcalloc);
1029EXPORT_SYMBOL(dm_get_device); 1050EXPORT_SYMBOL(dm_get_device);
1030EXPORT_SYMBOL(dm_put_device); 1051EXPORT_SYMBOL(dm_put_device);
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 788ba96a6256..8a994be035ba 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -89,12 +89,13 @@ union map_info *dm_get_mapinfo(struct bio *bio)
89/* 89/*
90 * Bits for the md->flags field. 90 * Bits for the md->flags field.
91 */ 91 */
92#define DMF_BLOCK_IO 0 92#define DMF_BLOCK_IO_FOR_SUSPEND 0
93#define DMF_SUSPENDED 1 93#define DMF_SUSPENDED 1
94#define DMF_FROZEN 2 94#define DMF_FROZEN 2
95#define DMF_FREEING 3 95#define DMF_FREEING 3
96#define DMF_DELETING 4 96#define DMF_DELETING 4
97#define DMF_NOFLUSH_SUSPENDING 5 97#define DMF_NOFLUSH_SUSPENDING 5
98#define DMF_QUEUE_IO_TO_THREAD 6
98 99
99/* 100/*
100 * Work processed by per-device workqueue. 101 * Work processed by per-device workqueue.
@@ -124,6 +125,11 @@ struct mapped_device {
124 spinlock_t deferred_lock; 125 spinlock_t deferred_lock;
125 126
126 /* 127 /*
128 * An error from the barrier request currently being processed.
129 */
130 int barrier_error;
131
132 /*
127 * Processing queue (flush/barriers) 133 * Processing queue (flush/barriers)
128 */ 134 */
129 struct workqueue_struct *wq; 135 struct workqueue_struct *wq;
@@ -424,6 +430,10 @@ static void end_io_acct(struct dm_io *io)
424 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration); 430 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
425 part_stat_unlock(); 431 part_stat_unlock();
426 432
433 /*
434 * After this is decremented the bio must not be touched if it is
435 * a barrier.
436 */
427 dm_disk(md)->part0.in_flight = pending = 437 dm_disk(md)->part0.in_flight = pending =
428 atomic_dec_return(&md->pending); 438 atomic_dec_return(&md->pending);
429 439
@@ -435,21 +445,18 @@ static void end_io_acct(struct dm_io *io)
435/* 445/*
436 * Add the bio to the list of deferred io. 446 * Add the bio to the list of deferred io.
437 */ 447 */
438static int queue_io(struct mapped_device *md, struct bio *bio) 448static void queue_io(struct mapped_device *md, struct bio *bio)
439{ 449{
440 down_write(&md->io_lock); 450 down_write(&md->io_lock);
441 451
442 if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
443 up_write(&md->io_lock);
444 return 1;
445 }
446
447 spin_lock_irq(&md->deferred_lock); 452 spin_lock_irq(&md->deferred_lock);
448 bio_list_add(&md->deferred, bio); 453 bio_list_add(&md->deferred, bio);
449 spin_unlock_irq(&md->deferred_lock); 454 spin_unlock_irq(&md->deferred_lock);
450 455
456 if (!test_and_set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags))
457 queue_work(md->wq, &md->work);
458
451 up_write(&md->io_lock); 459 up_write(&md->io_lock);
452 return 0; /* deferred successfully */
453} 460}
454 461
455/* 462/*
@@ -533,25 +540,35 @@ static void dec_pending(struct dm_io *io, int error)
533 */ 540 */
534 spin_lock_irqsave(&md->deferred_lock, flags); 541 spin_lock_irqsave(&md->deferred_lock, flags);
535 if (__noflush_suspending(md)) 542 if (__noflush_suspending(md))
536 bio_list_add(&md->deferred, io->bio); 543 bio_list_add_head(&md->deferred, io->bio);
537 else 544 else
538 /* noflush suspend was interrupted. */ 545 /* noflush suspend was interrupted. */
539 io->error = -EIO; 546 io->error = -EIO;
540 spin_unlock_irqrestore(&md->deferred_lock, flags); 547 spin_unlock_irqrestore(&md->deferred_lock, flags);
541 } 548 }
542 549
543 end_io_acct(io);
544
545 io_error = io->error; 550 io_error = io->error;
546 bio = io->bio; 551 bio = io->bio;
547 552
548 free_io(md, io); 553 if (bio_barrier(bio)) {
554 /*
555 * There can be just one barrier request so we use
556 * a per-device variable for error reporting.
557 * Note that you can't touch the bio after end_io_acct
558 */
559 md->barrier_error = io_error;
560 end_io_acct(io);
561 } else {
562 end_io_acct(io);
549 563
550 if (io_error != DM_ENDIO_REQUEUE) { 564 if (io_error != DM_ENDIO_REQUEUE) {
551 trace_block_bio_complete(md->queue, bio); 565 trace_block_bio_complete(md->queue, bio);
552 566
553 bio_endio(bio, io_error); 567 bio_endio(bio, io_error);
568 }
554 } 569 }
570
571 free_io(md, io);
555 } 572 }
556} 573}
557 574
@@ -693,13 +710,19 @@ static struct bio *split_bvec(struct bio *bio, sector_t sector,
693 710
694 clone->bi_sector = sector; 711 clone->bi_sector = sector;
695 clone->bi_bdev = bio->bi_bdev; 712 clone->bi_bdev = bio->bi_bdev;
696 clone->bi_rw = bio->bi_rw; 713 clone->bi_rw = bio->bi_rw & ~(1 << BIO_RW_BARRIER);
697 clone->bi_vcnt = 1; 714 clone->bi_vcnt = 1;
698 clone->bi_size = to_bytes(len); 715 clone->bi_size = to_bytes(len);
699 clone->bi_io_vec->bv_offset = offset; 716 clone->bi_io_vec->bv_offset = offset;
700 clone->bi_io_vec->bv_len = clone->bi_size; 717 clone->bi_io_vec->bv_len = clone->bi_size;
701 clone->bi_flags |= 1 << BIO_CLONED; 718 clone->bi_flags |= 1 << BIO_CLONED;
702 719
720 if (bio_integrity(bio)) {
721 bio_integrity_clone(clone, bio, GFP_NOIO);
722 bio_integrity_trim(clone,
723 bio_sector_offset(bio, idx, offset), len);
724 }
725
703 return clone; 726 return clone;
704} 727}
705 728
@@ -714,6 +737,7 @@ static struct bio *clone_bio(struct bio *bio, sector_t sector,
714 737
715 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs); 738 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
716 __bio_clone(clone, bio); 739 __bio_clone(clone, bio);
740 clone->bi_rw &= ~(1 << BIO_RW_BARRIER);
717 clone->bi_destructor = dm_bio_destructor; 741 clone->bi_destructor = dm_bio_destructor;
718 clone->bi_sector = sector; 742 clone->bi_sector = sector;
719 clone->bi_idx = idx; 743 clone->bi_idx = idx;
@@ -721,6 +745,14 @@ static struct bio *clone_bio(struct bio *bio, sector_t sector,
721 clone->bi_size = to_bytes(len); 745 clone->bi_size = to_bytes(len);
722 clone->bi_flags &= ~(1 << BIO_SEG_VALID); 746 clone->bi_flags &= ~(1 << BIO_SEG_VALID);
723 747
748 if (bio_integrity(bio)) {
749 bio_integrity_clone(clone, bio, GFP_NOIO);
750
751 if (idx != bio->bi_idx || clone->bi_size < bio->bi_size)
752 bio_integrity_trim(clone,
753 bio_sector_offset(bio, idx, 0), len);
754 }
755
724 return clone; 756 return clone;
725} 757}
726 758
@@ -834,14 +866,13 @@ static void __split_and_process_bio(struct mapped_device *md, struct bio *bio)
834 866
835 ci.map = dm_get_table(md); 867 ci.map = dm_get_table(md);
836 if (unlikely(!ci.map)) { 868 if (unlikely(!ci.map)) {
837 bio_io_error(bio); 869 if (!bio_barrier(bio))
838 return; 870 bio_io_error(bio);
839 } 871 else
840 if (unlikely(bio_barrier(bio) && !dm_table_barrier_ok(ci.map))) { 872 md->barrier_error = -EIO;
841 dm_table_put(ci.map);
842 bio_endio(bio, -EOPNOTSUPP);
843 return; 873 return;
844 } 874 }
875
845 ci.md = md; 876 ci.md = md;
846 ci.bio = bio; 877 ci.bio = bio;
847 ci.io = alloc_io(md); 878 ci.io = alloc_io(md);
@@ -918,7 +949,6 @@ out:
918 */ 949 */
919static int dm_request(struct request_queue *q, struct bio *bio) 950static int dm_request(struct request_queue *q, struct bio *bio)
920{ 951{
921 int r = -EIO;
922 int rw = bio_data_dir(bio); 952 int rw = bio_data_dir(bio);
923 struct mapped_device *md = q->queuedata; 953 struct mapped_device *md = q->queuedata;
924 int cpu; 954 int cpu;
@@ -931,34 +961,27 @@ static int dm_request(struct request_queue *q, struct bio *bio)
931 part_stat_unlock(); 961 part_stat_unlock();
932 962
933 /* 963 /*
934 * If we're suspended we have to queue 964 * If we're suspended or the thread is processing barriers
935 * this io for later. 965 * we have to queue this io for later.
936 */ 966 */
937 while (test_bit(DMF_BLOCK_IO, &md->flags)) { 967 if (unlikely(test_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags)) ||
968 unlikely(bio_barrier(bio))) {
938 up_read(&md->io_lock); 969 up_read(&md->io_lock);
939 970
940 if (bio_rw(bio) != READA) 971 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) &&
941 r = queue_io(md, bio); 972 bio_rw(bio) == READA) {
973 bio_io_error(bio);
974 return 0;
975 }
942 976
943 if (r <= 0) 977 queue_io(md, bio);
944 goto out_req;
945 978
946 /* 979 return 0;
947 * We're in a while loop, because someone could suspend
948 * before we get to the following read lock.
949 */
950 down_read(&md->io_lock);
951 } 980 }
952 981
953 __split_and_process_bio(md, bio); 982 __split_and_process_bio(md, bio);
954 up_read(&md->io_lock); 983 up_read(&md->io_lock);
955 return 0; 984 return 0;
956
957out_req:
958 if (r < 0)
959 bio_io_error(bio);
960
961 return 0;
962} 985}
963 986
964static void dm_unplug_all(struct request_queue *q) 987static void dm_unplug_all(struct request_queue *q)
@@ -978,7 +1001,7 @@ static int dm_any_congested(void *congested_data, int bdi_bits)
978 struct mapped_device *md = congested_data; 1001 struct mapped_device *md = congested_data;
979 struct dm_table *map; 1002 struct dm_table *map;
980 1003
981 if (!test_bit(DMF_BLOCK_IO, &md->flags)) { 1004 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
982 map = dm_get_table(md); 1005 map = dm_get_table(md);
983 if (map) { 1006 if (map) {
984 r = dm_table_any_congested(map, bdi_bits); 1007 r = dm_table_any_congested(map, bdi_bits);
@@ -1193,6 +1216,7 @@ static void free_dev(struct mapped_device *md)
1193 mempool_destroy(md->tio_pool); 1216 mempool_destroy(md->tio_pool);
1194 mempool_destroy(md->io_pool); 1217 mempool_destroy(md->io_pool);
1195 bioset_free(md->bs); 1218 bioset_free(md->bs);
1219 blk_integrity_unregister(md->disk);
1196 del_gendisk(md->disk); 1220 del_gendisk(md->disk);
1197 free_minor(minor); 1221 free_minor(minor);
1198 1222
@@ -1406,6 +1430,36 @@ static int dm_wait_for_completion(struct mapped_device *md, int interruptible)
1406 return r; 1430 return r;
1407} 1431}
1408 1432
1433static int dm_flush(struct mapped_device *md)
1434{
1435 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
1436 return 0;
1437}
1438
1439static void process_barrier(struct mapped_device *md, struct bio *bio)
1440{
1441 int error = dm_flush(md);
1442
1443 if (unlikely(error)) {
1444 bio_endio(bio, error);
1445 return;
1446 }
1447 if (bio_empty_barrier(bio)) {
1448 bio_endio(bio, 0);
1449 return;
1450 }
1451
1452 __split_and_process_bio(md, bio);
1453
1454 error = dm_flush(md);
1455
1456 if (!error && md->barrier_error)
1457 error = md->barrier_error;
1458
1459 if (md->barrier_error != DM_ENDIO_REQUEUE)
1460 bio_endio(bio, error);
1461}
1462
1409/* 1463/*
1410 * Process the deferred bios 1464 * Process the deferred bios
1411 */ 1465 */
@@ -1417,25 +1471,34 @@ static void dm_wq_work(struct work_struct *work)
1417 1471
1418 down_write(&md->io_lock); 1472 down_write(&md->io_lock);
1419 1473
1420next_bio: 1474 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1421 spin_lock_irq(&md->deferred_lock); 1475 spin_lock_irq(&md->deferred_lock);
1422 c = bio_list_pop(&md->deferred); 1476 c = bio_list_pop(&md->deferred);
1423 spin_unlock_irq(&md->deferred_lock); 1477 spin_unlock_irq(&md->deferred_lock);
1424 1478
1425 if (c) { 1479 if (!c) {
1426 __split_and_process_bio(md, c); 1480 clear_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags);
1427 goto next_bio; 1481 break;
1428 } 1482 }
1429 1483
1430 clear_bit(DMF_BLOCK_IO, &md->flags); 1484 up_write(&md->io_lock);
1485
1486 if (bio_barrier(c))
1487 process_barrier(md, c);
1488 else
1489 __split_and_process_bio(md, c);
1490
1491 down_write(&md->io_lock);
1492 }
1431 1493
1432 up_write(&md->io_lock); 1494 up_write(&md->io_lock);
1433} 1495}
1434 1496
1435static void dm_queue_flush(struct mapped_device *md) 1497static void dm_queue_flush(struct mapped_device *md)
1436{ 1498{
1499 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
1500 smp_mb__after_clear_bit();
1437 queue_work(md->wq, &md->work); 1501 queue_work(md->wq, &md->work);
1438 flush_workqueue(md->wq);
1439} 1502}
1440 1503
1441/* 1504/*
@@ -1553,20 +1616,36 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
1553 } 1616 }
1554 1617
1555 /* 1618 /*
1556 * First we set the BLOCK_IO flag so no more ios will be mapped. 1619 * Here we must make sure that no processes are submitting requests
1620 * to target drivers i.e. no one may be executing
1621 * __split_and_process_bio. This is called from dm_request and
1622 * dm_wq_work.
1623 *
1624 * To get all processes out of __split_and_process_bio in dm_request,
1625 * we take the write lock. To prevent any process from reentering
1626 * __split_and_process_bio from dm_request, we set
1627 * DMF_QUEUE_IO_TO_THREAD.
1628 *
1629 * To quiesce the thread (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND
1630 * and call flush_workqueue(md->wq). flush_workqueue will wait until
1631 * dm_wq_work exits and DMF_BLOCK_IO_FOR_SUSPEND will prevent any
1632 * further calls to __split_and_process_bio from dm_wq_work.
1557 */ 1633 */
1558 down_write(&md->io_lock); 1634 down_write(&md->io_lock);
1559 set_bit(DMF_BLOCK_IO, &md->flags); 1635 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
1560 1636 set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags);
1561 up_write(&md->io_lock); 1637 up_write(&md->io_lock);
1562 1638
1639 flush_workqueue(md->wq);
1640
1563 /* 1641 /*
1564 * Wait for the already-mapped ios to complete. 1642 * At this point no more requests are entering target request routines.
1643 * We call dm_wait_for_completion to wait for all existing requests
1644 * to finish.
1565 */ 1645 */
1566 r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE); 1646 r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE);
1567 1647
1568 down_write(&md->io_lock); 1648 down_write(&md->io_lock);
1569
1570 if (noflush) 1649 if (noflush)
1571 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 1650 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1572 up_write(&md->io_lock); 1651 up_write(&md->io_lock);
@@ -1579,6 +1658,12 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
1579 goto out; /* pushback list is already flushed, so skip flush */ 1658 goto out; /* pushback list is already flushed, so skip flush */
1580 } 1659 }
1581 1660
1661 /*
1662 * If dm_wait_for_completion returned 0, the device is completely
1663 * quiescent now. There is no request-processing activity. All new
1664 * requests are being added to md->deferred list.
1665 */
1666
1582 dm_table_postsuspend_targets(map); 1667 dm_table_postsuspend_targets(map);
1583 1668
1584 set_bit(DMF_SUSPENDED, &md->flags); 1669 set_bit(DMF_SUSPENDED, &md->flags);
diff --git a/drivers/md/dm.h b/drivers/md/dm.h
index b48397c0abbd..a31506d93e91 100644
--- a/drivers/md/dm.h
+++ b/drivers/md/dm.h
@@ -52,7 +52,6 @@ int dm_table_any_congested(struct dm_table *t, int bdi_bits);
52 * To check the return value from dm_table_find_target(). 52 * To check the return value from dm_table_find_target().
53 */ 53 */
54#define dm_target_is_valid(t) ((t)->table) 54#define dm_target_is_valid(t) ((t)->table)
55int dm_table_barrier_ok(struct dm_table *t);
56 55
57/*----------------------------------------------------------------- 56/*-----------------------------------------------------------------
58 * A registry of target types. 57 * A registry of target types.