diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2009-04-09 11:01:21 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2009-04-09 11:01:21 -0400 |
commit | df5529297ec4c32d57dd149398e3add6c37bc054 (patch) | |
tree | 24e40e2286f2d6396994ed92906420890b4cf9c3 /drivers | |
parent | 97c18e2c7a8e36d2d83d50ee070314aadac73a11 (diff) | |
parent | 340cd44451fb0bfa542365e6b4b565bbd44836e2 (diff) |
Merge git://git.kernel.org/pub/scm/linux/kernel/git/agk/linux-2.6-dm
* git://git.kernel.org/pub/scm/linux/kernel/git/agk/linux-2.6-dm:
dm kcopyd: fix callback race
dm kcopyd: prepare for callback race fix
dm: implement basic barrier support
dm: remove dm_request loop
dm: rework queueing and suspension
dm: simplify dm_request loop
dm: split DMF_BLOCK_IO flag into two
dm: rearrange dm_wq_work
dm: remove limited barrier support
dm: add integrity support
Diffstat (limited to 'drivers')
-rw-r--r-- | drivers/md/dm-ioctl.c | 21 | ||||
-rw-r--r-- | drivers/md/dm-kcopyd.c | 23 | ||||
-rw-r--r-- | drivers/md/dm-linear.c | 1 | ||||
-rw-r--r-- | drivers/md/dm-table.c | 59 | ||||
-rw-r--r-- | drivers/md/dm.c | 199 | ||||
-rw-r--r-- | drivers/md/dm.h | 1 |
6 files changed, 218 insertions, 86 deletions
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index f01096549a93..823ceba6efa8 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c | |||
@@ -1047,6 +1047,19 @@ static int populate_table(struct dm_table *table, | |||
1047 | return dm_table_complete(table); | 1047 | return dm_table_complete(table); |
1048 | } | 1048 | } |
1049 | 1049 | ||
1050 | static int table_prealloc_integrity(struct dm_table *t, | ||
1051 | struct mapped_device *md) | ||
1052 | { | ||
1053 | struct list_head *devices = dm_table_get_devices(t); | ||
1054 | struct dm_dev_internal *dd; | ||
1055 | |||
1056 | list_for_each_entry(dd, devices, list) | ||
1057 | if (bdev_get_integrity(dd->dm_dev.bdev)) | ||
1058 | return blk_integrity_register(dm_disk(md), NULL); | ||
1059 | |||
1060 | return 0; | ||
1061 | } | ||
1062 | |||
1050 | static int table_load(struct dm_ioctl *param, size_t param_size) | 1063 | static int table_load(struct dm_ioctl *param, size_t param_size) |
1051 | { | 1064 | { |
1052 | int r; | 1065 | int r; |
@@ -1068,6 +1081,14 @@ static int table_load(struct dm_ioctl *param, size_t param_size) | |||
1068 | goto out; | 1081 | goto out; |
1069 | } | 1082 | } |
1070 | 1083 | ||
1084 | r = table_prealloc_integrity(t, md); | ||
1085 | if (r) { | ||
1086 | DMERR("%s: could not register integrity profile.", | ||
1087 | dm_device_name(md)); | ||
1088 | dm_table_destroy(t); | ||
1089 | goto out; | ||
1090 | } | ||
1091 | |||
1071 | down_write(&_hash_lock); | 1092 | down_write(&_hash_lock); |
1072 | hc = dm_get_mdptr(md); | 1093 | hc = dm_get_mdptr(md); |
1073 | if (!hc || hc->md != md) { | 1094 | if (!hc || hc->md != md) { |
diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index 0a225da21272..3e3fc06cb861 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c | |||
@@ -297,7 +297,8 @@ static int run_complete_job(struct kcopyd_job *job) | |||
297 | dm_kcopyd_notify_fn fn = job->fn; | 297 | dm_kcopyd_notify_fn fn = job->fn; |
298 | struct dm_kcopyd_client *kc = job->kc; | 298 | struct dm_kcopyd_client *kc = job->kc; |
299 | 299 | ||
300 | kcopyd_put_pages(kc, job->pages); | 300 | if (job->pages) |
301 | kcopyd_put_pages(kc, job->pages); | ||
301 | mempool_free(job, kc->job_pool); | 302 | mempool_free(job, kc->job_pool); |
302 | fn(read_err, write_err, context); | 303 | fn(read_err, write_err, context); |
303 | 304 | ||
@@ -461,6 +462,7 @@ static void segment_complete(int read_err, unsigned long write_err, | |||
461 | sector_t progress = 0; | 462 | sector_t progress = 0; |
462 | sector_t count = 0; | 463 | sector_t count = 0; |
463 | struct kcopyd_job *job = (struct kcopyd_job *) context; | 464 | struct kcopyd_job *job = (struct kcopyd_job *) context; |
465 | struct dm_kcopyd_client *kc = job->kc; | ||
464 | 466 | ||
465 | mutex_lock(&job->lock); | 467 | mutex_lock(&job->lock); |
466 | 468 | ||
@@ -490,7 +492,7 @@ static void segment_complete(int read_err, unsigned long write_err, | |||
490 | 492 | ||
491 | if (count) { | 493 | if (count) { |
492 | int i; | 494 | int i; |
493 | struct kcopyd_job *sub_job = mempool_alloc(job->kc->job_pool, | 495 | struct kcopyd_job *sub_job = mempool_alloc(kc->job_pool, |
494 | GFP_NOIO); | 496 | GFP_NOIO); |
495 | 497 | ||
496 | *sub_job = *job; | 498 | *sub_job = *job; |
@@ -509,13 +511,16 @@ static void segment_complete(int read_err, unsigned long write_err, | |||
509 | } else if (atomic_dec_and_test(&job->sub_jobs)) { | 511 | } else if (atomic_dec_and_test(&job->sub_jobs)) { |
510 | 512 | ||
511 | /* | 513 | /* |
512 | * To avoid a race we must keep the job around | 514 | * Queue the completion callback to the kcopyd thread. |
513 | * until after the notify function has completed. | 515 | * |
514 | * Otherwise the client may try and stop the job | 516 | * Some callers assume that all the completions are called |
515 | * after we've completed. | 517 | * from a single thread and don't race with each other. |
518 | * | ||
519 | * We must not call the callback directly here because this | ||
520 | * code may not be executing in the thread. | ||
516 | */ | 521 | */ |
517 | job->fn(read_err, write_err, job->context); | 522 | push(&kc->complete_jobs, job); |
518 | mempool_free(job, job->kc->job_pool); | 523 | wake(kc); |
519 | } | 524 | } |
520 | } | 525 | } |
521 | 526 | ||
@@ -528,6 +533,8 @@ static void split_job(struct kcopyd_job *job) | |||
528 | { | 533 | { |
529 | int i; | 534 | int i; |
530 | 535 | ||
536 | atomic_inc(&job->kc->nr_jobs); | ||
537 | |||
531 | atomic_set(&job->sub_jobs, SPLIT_COUNT); | 538 | atomic_set(&job->sub_jobs, SPLIT_COUNT); |
532 | for (i = 0; i < SPLIT_COUNT; i++) | 539 | for (i = 0; i < SPLIT_COUNT; i++) |
533 | segment_complete(0, 0u, job); | 540 | segment_complete(0, 0u, job); |
diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c index bfa107f59d96..79fb53e51c70 100644 --- a/drivers/md/dm-linear.c +++ b/drivers/md/dm-linear.c | |||
@@ -142,7 +142,6 @@ static struct target_type linear_target = { | |||
142 | .status = linear_status, | 142 | .status = linear_status, |
143 | .ioctl = linear_ioctl, | 143 | .ioctl = linear_ioctl, |
144 | .merge = linear_merge, | 144 | .merge = linear_merge, |
145 | .features = DM_TARGET_SUPPORTS_BARRIERS, | ||
146 | }; | 145 | }; |
147 | 146 | ||
148 | int __init dm_linear_init(void) | 147 | int __init dm_linear_init(void) |
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index e8361b191b9b..429b50b975d5 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c | |||
@@ -52,8 +52,6 @@ struct dm_table { | |||
52 | sector_t *highs; | 52 | sector_t *highs; |
53 | struct dm_target *targets; | 53 | struct dm_target *targets; |
54 | 54 | ||
55 | unsigned barriers_supported:1; | ||
56 | |||
57 | /* | 55 | /* |
58 | * Indicates the rw permissions for the new logical | 56 | * Indicates the rw permissions for the new logical |
59 | * device. This should be a combination of FMODE_READ | 57 | * device. This should be a combination of FMODE_READ |
@@ -243,7 +241,6 @@ int dm_table_create(struct dm_table **result, fmode_t mode, | |||
243 | 241 | ||
244 | INIT_LIST_HEAD(&t->devices); | 242 | INIT_LIST_HEAD(&t->devices); |
245 | atomic_set(&t->holders, 0); | 243 | atomic_set(&t->holders, 0); |
246 | t->barriers_supported = 1; | ||
247 | 244 | ||
248 | if (!num_targets) | 245 | if (!num_targets) |
249 | num_targets = KEYS_PER_NODE; | 246 | num_targets = KEYS_PER_NODE; |
@@ -751,10 +748,6 @@ int dm_table_add_target(struct dm_table *t, const char *type, | |||
751 | /* FIXME: the plan is to combine high here and then have | 748 | /* FIXME: the plan is to combine high here and then have |
752 | * the merge fn apply the target level restrictions. */ | 749 | * the merge fn apply the target level restrictions. */ |
753 | combine_restrictions_low(&t->limits, &tgt->limits); | 750 | combine_restrictions_low(&t->limits, &tgt->limits); |
754 | |||
755 | if (!(tgt->type->features & DM_TARGET_SUPPORTS_BARRIERS)) | ||
756 | t->barriers_supported = 0; | ||
757 | |||
758 | return 0; | 751 | return 0; |
759 | 752 | ||
760 | bad: | 753 | bad: |
@@ -799,12 +792,6 @@ int dm_table_complete(struct dm_table *t) | |||
799 | 792 | ||
800 | check_for_valid_limits(&t->limits); | 793 | check_for_valid_limits(&t->limits); |
801 | 794 | ||
802 | /* | ||
803 | * We only support barriers if there is exactly one underlying device. | ||
804 | */ | ||
805 | if (!list_is_singular(&t->devices)) | ||
806 | t->barriers_supported = 0; | ||
807 | |||
808 | /* how many indexes will the btree have ? */ | 795 | /* how many indexes will the btree have ? */ |
809 | leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE); | 796 | leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE); |
810 | t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE); | 797 | t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE); |
@@ -879,6 +866,45 @@ struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector) | |||
879 | return &t->targets[(KEYS_PER_NODE * n) + k]; | 866 | return &t->targets[(KEYS_PER_NODE * n) + k]; |
880 | } | 867 | } |
881 | 868 | ||
869 | /* | ||
870 | * Set the integrity profile for this device if all devices used have | ||
871 | * matching profiles. | ||
872 | */ | ||
873 | static void dm_table_set_integrity(struct dm_table *t) | ||
874 | { | ||
875 | struct list_head *devices = dm_table_get_devices(t); | ||
876 | struct dm_dev_internal *prev = NULL, *dd = NULL; | ||
877 | |||
878 | if (!blk_get_integrity(dm_disk(t->md))) | ||
879 | return; | ||
880 | |||
881 | list_for_each_entry(dd, devices, list) { | ||
882 | if (prev && | ||
883 | blk_integrity_compare(prev->dm_dev.bdev->bd_disk, | ||
884 | dd->dm_dev.bdev->bd_disk) < 0) { | ||
885 | DMWARN("%s: integrity not set: %s and %s mismatch", | ||
886 | dm_device_name(t->md), | ||
887 | prev->dm_dev.bdev->bd_disk->disk_name, | ||
888 | dd->dm_dev.bdev->bd_disk->disk_name); | ||
889 | goto no_integrity; | ||
890 | } | ||
891 | prev = dd; | ||
892 | } | ||
893 | |||
894 | if (!prev || !bdev_get_integrity(prev->dm_dev.bdev)) | ||
895 | goto no_integrity; | ||
896 | |||
897 | blk_integrity_register(dm_disk(t->md), | ||
898 | bdev_get_integrity(prev->dm_dev.bdev)); | ||
899 | |||
900 | return; | ||
901 | |||
902 | no_integrity: | ||
903 | blk_integrity_register(dm_disk(t->md), NULL); | ||
904 | |||
905 | return; | ||
906 | } | ||
907 | |||
882 | void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q) | 908 | void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q) |
883 | { | 909 | { |
884 | /* | 910 | /* |
@@ -899,6 +925,7 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q) | |||
899 | else | 925 | else |
900 | queue_flag_set_unlocked(QUEUE_FLAG_CLUSTER, q); | 926 | queue_flag_set_unlocked(QUEUE_FLAG_CLUSTER, q); |
901 | 927 | ||
928 | dm_table_set_integrity(t); | ||
902 | } | 929 | } |
903 | 930 | ||
904 | unsigned int dm_table_get_num_targets(struct dm_table *t) | 931 | unsigned int dm_table_get_num_targets(struct dm_table *t) |
@@ -1019,12 +1046,6 @@ struct mapped_device *dm_table_get_md(struct dm_table *t) | |||
1019 | return t->md; | 1046 | return t->md; |
1020 | } | 1047 | } |
1021 | 1048 | ||
1022 | int dm_table_barrier_ok(struct dm_table *t) | ||
1023 | { | ||
1024 | return t->barriers_supported; | ||
1025 | } | ||
1026 | EXPORT_SYMBOL(dm_table_barrier_ok); | ||
1027 | |||
1028 | EXPORT_SYMBOL(dm_vcalloc); | 1049 | EXPORT_SYMBOL(dm_vcalloc); |
1029 | EXPORT_SYMBOL(dm_get_device); | 1050 | EXPORT_SYMBOL(dm_get_device); |
1030 | EXPORT_SYMBOL(dm_put_device); | 1051 | EXPORT_SYMBOL(dm_put_device); |
diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 788ba96a6256..8a994be035ba 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c | |||
@@ -89,12 +89,13 @@ union map_info *dm_get_mapinfo(struct bio *bio) | |||
89 | /* | 89 | /* |
90 | * Bits for the md->flags field. | 90 | * Bits for the md->flags field. |
91 | */ | 91 | */ |
92 | #define DMF_BLOCK_IO 0 | 92 | #define DMF_BLOCK_IO_FOR_SUSPEND 0 |
93 | #define DMF_SUSPENDED 1 | 93 | #define DMF_SUSPENDED 1 |
94 | #define DMF_FROZEN 2 | 94 | #define DMF_FROZEN 2 |
95 | #define DMF_FREEING 3 | 95 | #define DMF_FREEING 3 |
96 | #define DMF_DELETING 4 | 96 | #define DMF_DELETING 4 |
97 | #define DMF_NOFLUSH_SUSPENDING 5 | 97 | #define DMF_NOFLUSH_SUSPENDING 5 |
98 | #define DMF_QUEUE_IO_TO_THREAD 6 | ||
98 | 99 | ||
99 | /* | 100 | /* |
100 | * Work processed by per-device workqueue. | 101 | * Work processed by per-device workqueue. |
@@ -124,6 +125,11 @@ struct mapped_device { | |||
124 | spinlock_t deferred_lock; | 125 | spinlock_t deferred_lock; |
125 | 126 | ||
126 | /* | 127 | /* |
128 | * An error from the barrier request currently being processed. | ||
129 | */ | ||
130 | int barrier_error; | ||
131 | |||
132 | /* | ||
127 | * Processing queue (flush/barriers) | 133 | * Processing queue (flush/barriers) |
128 | */ | 134 | */ |
129 | struct workqueue_struct *wq; | 135 | struct workqueue_struct *wq; |
@@ -424,6 +430,10 @@ static void end_io_acct(struct dm_io *io) | |||
424 | part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration); | 430 | part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration); |
425 | part_stat_unlock(); | 431 | part_stat_unlock(); |
426 | 432 | ||
433 | /* | ||
434 | * After this is decremented the bio must not be touched if it is | ||
435 | * a barrier. | ||
436 | */ | ||
427 | dm_disk(md)->part0.in_flight = pending = | 437 | dm_disk(md)->part0.in_flight = pending = |
428 | atomic_dec_return(&md->pending); | 438 | atomic_dec_return(&md->pending); |
429 | 439 | ||
@@ -435,21 +445,18 @@ static void end_io_acct(struct dm_io *io) | |||
435 | /* | 445 | /* |
436 | * Add the bio to the list of deferred io. | 446 | * Add the bio to the list of deferred io. |
437 | */ | 447 | */ |
438 | static int queue_io(struct mapped_device *md, struct bio *bio) | 448 | static void queue_io(struct mapped_device *md, struct bio *bio) |
439 | { | 449 | { |
440 | down_write(&md->io_lock); | 450 | down_write(&md->io_lock); |
441 | 451 | ||
442 | if (!test_bit(DMF_BLOCK_IO, &md->flags)) { | ||
443 | up_write(&md->io_lock); | ||
444 | return 1; | ||
445 | } | ||
446 | |||
447 | spin_lock_irq(&md->deferred_lock); | 452 | spin_lock_irq(&md->deferred_lock); |
448 | bio_list_add(&md->deferred, bio); | 453 | bio_list_add(&md->deferred, bio); |
449 | spin_unlock_irq(&md->deferred_lock); | 454 | spin_unlock_irq(&md->deferred_lock); |
450 | 455 | ||
456 | if (!test_and_set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags)) | ||
457 | queue_work(md->wq, &md->work); | ||
458 | |||
451 | up_write(&md->io_lock); | 459 | up_write(&md->io_lock); |
452 | return 0; /* deferred successfully */ | ||
453 | } | 460 | } |
454 | 461 | ||
455 | /* | 462 | /* |
@@ -533,25 +540,35 @@ static void dec_pending(struct dm_io *io, int error) | |||
533 | */ | 540 | */ |
534 | spin_lock_irqsave(&md->deferred_lock, flags); | 541 | spin_lock_irqsave(&md->deferred_lock, flags); |
535 | if (__noflush_suspending(md)) | 542 | if (__noflush_suspending(md)) |
536 | bio_list_add(&md->deferred, io->bio); | 543 | bio_list_add_head(&md->deferred, io->bio); |
537 | else | 544 | else |
538 | /* noflush suspend was interrupted. */ | 545 | /* noflush suspend was interrupted. */ |
539 | io->error = -EIO; | 546 | io->error = -EIO; |
540 | spin_unlock_irqrestore(&md->deferred_lock, flags); | 547 | spin_unlock_irqrestore(&md->deferred_lock, flags); |
541 | } | 548 | } |
542 | 549 | ||
543 | end_io_acct(io); | ||
544 | |||
545 | io_error = io->error; | 550 | io_error = io->error; |
546 | bio = io->bio; | 551 | bio = io->bio; |
547 | 552 | ||
548 | free_io(md, io); | 553 | if (bio_barrier(bio)) { |
554 | /* | ||
555 | * There can be just one barrier request so we use | ||
556 | * a per-device variable for error reporting. | ||
557 | * Note that you can't touch the bio after end_io_acct | ||
558 | */ | ||
559 | md->barrier_error = io_error; | ||
560 | end_io_acct(io); | ||
561 | } else { | ||
562 | end_io_acct(io); | ||
549 | 563 | ||
550 | if (io_error != DM_ENDIO_REQUEUE) { | 564 | if (io_error != DM_ENDIO_REQUEUE) { |
551 | trace_block_bio_complete(md->queue, bio); | 565 | trace_block_bio_complete(md->queue, bio); |
552 | 566 | ||
553 | bio_endio(bio, io_error); | 567 | bio_endio(bio, io_error); |
568 | } | ||
554 | } | 569 | } |
570 | |||
571 | free_io(md, io); | ||
555 | } | 572 | } |
556 | } | 573 | } |
557 | 574 | ||
@@ -693,13 +710,19 @@ static struct bio *split_bvec(struct bio *bio, sector_t sector, | |||
693 | 710 | ||
694 | clone->bi_sector = sector; | 711 | clone->bi_sector = sector; |
695 | clone->bi_bdev = bio->bi_bdev; | 712 | clone->bi_bdev = bio->bi_bdev; |
696 | clone->bi_rw = bio->bi_rw; | 713 | clone->bi_rw = bio->bi_rw & ~(1 << BIO_RW_BARRIER); |
697 | clone->bi_vcnt = 1; | 714 | clone->bi_vcnt = 1; |
698 | clone->bi_size = to_bytes(len); | 715 | clone->bi_size = to_bytes(len); |
699 | clone->bi_io_vec->bv_offset = offset; | 716 | clone->bi_io_vec->bv_offset = offset; |
700 | clone->bi_io_vec->bv_len = clone->bi_size; | 717 | clone->bi_io_vec->bv_len = clone->bi_size; |
701 | clone->bi_flags |= 1 << BIO_CLONED; | 718 | clone->bi_flags |= 1 << BIO_CLONED; |
702 | 719 | ||
720 | if (bio_integrity(bio)) { | ||
721 | bio_integrity_clone(clone, bio, GFP_NOIO); | ||
722 | bio_integrity_trim(clone, | ||
723 | bio_sector_offset(bio, idx, offset), len); | ||
724 | } | ||
725 | |||
703 | return clone; | 726 | return clone; |
704 | } | 727 | } |
705 | 728 | ||
@@ -714,6 +737,7 @@ static struct bio *clone_bio(struct bio *bio, sector_t sector, | |||
714 | 737 | ||
715 | clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs); | 738 | clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs); |
716 | __bio_clone(clone, bio); | 739 | __bio_clone(clone, bio); |
740 | clone->bi_rw &= ~(1 << BIO_RW_BARRIER); | ||
717 | clone->bi_destructor = dm_bio_destructor; | 741 | clone->bi_destructor = dm_bio_destructor; |
718 | clone->bi_sector = sector; | 742 | clone->bi_sector = sector; |
719 | clone->bi_idx = idx; | 743 | clone->bi_idx = idx; |
@@ -721,6 +745,14 @@ static struct bio *clone_bio(struct bio *bio, sector_t sector, | |||
721 | clone->bi_size = to_bytes(len); | 745 | clone->bi_size = to_bytes(len); |
722 | clone->bi_flags &= ~(1 << BIO_SEG_VALID); | 746 | clone->bi_flags &= ~(1 << BIO_SEG_VALID); |
723 | 747 | ||
748 | if (bio_integrity(bio)) { | ||
749 | bio_integrity_clone(clone, bio, GFP_NOIO); | ||
750 | |||
751 | if (idx != bio->bi_idx || clone->bi_size < bio->bi_size) | ||
752 | bio_integrity_trim(clone, | ||
753 | bio_sector_offset(bio, idx, 0), len); | ||
754 | } | ||
755 | |||
724 | return clone; | 756 | return clone; |
725 | } | 757 | } |
726 | 758 | ||
@@ -834,14 +866,13 @@ static void __split_and_process_bio(struct mapped_device *md, struct bio *bio) | |||
834 | 866 | ||
835 | ci.map = dm_get_table(md); | 867 | ci.map = dm_get_table(md); |
836 | if (unlikely(!ci.map)) { | 868 | if (unlikely(!ci.map)) { |
837 | bio_io_error(bio); | 869 | if (!bio_barrier(bio)) |
838 | return; | 870 | bio_io_error(bio); |
839 | } | 871 | else |
840 | if (unlikely(bio_barrier(bio) && !dm_table_barrier_ok(ci.map))) { | 872 | md->barrier_error = -EIO; |
841 | dm_table_put(ci.map); | ||
842 | bio_endio(bio, -EOPNOTSUPP); | ||
843 | return; | 873 | return; |
844 | } | 874 | } |
875 | |||
845 | ci.md = md; | 876 | ci.md = md; |
846 | ci.bio = bio; | 877 | ci.bio = bio; |
847 | ci.io = alloc_io(md); | 878 | ci.io = alloc_io(md); |
@@ -918,7 +949,6 @@ out: | |||
918 | */ | 949 | */ |
919 | static int dm_request(struct request_queue *q, struct bio *bio) | 950 | static int dm_request(struct request_queue *q, struct bio *bio) |
920 | { | 951 | { |
921 | int r = -EIO; | ||
922 | int rw = bio_data_dir(bio); | 952 | int rw = bio_data_dir(bio); |
923 | struct mapped_device *md = q->queuedata; | 953 | struct mapped_device *md = q->queuedata; |
924 | int cpu; | 954 | int cpu; |
@@ -931,34 +961,27 @@ static int dm_request(struct request_queue *q, struct bio *bio) | |||
931 | part_stat_unlock(); | 961 | part_stat_unlock(); |
932 | 962 | ||
933 | /* | 963 | /* |
934 | * If we're suspended we have to queue | 964 | * If we're suspended or the thread is processing barriers |
935 | * this io for later. | 965 | * we have to queue this io for later. |
936 | */ | 966 | */ |
937 | while (test_bit(DMF_BLOCK_IO, &md->flags)) { | 967 | if (unlikely(test_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags)) || |
968 | unlikely(bio_barrier(bio))) { | ||
938 | up_read(&md->io_lock); | 969 | up_read(&md->io_lock); |
939 | 970 | ||
940 | if (bio_rw(bio) != READA) | 971 | if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) && |
941 | r = queue_io(md, bio); | 972 | bio_rw(bio) == READA) { |
973 | bio_io_error(bio); | ||
974 | return 0; | ||
975 | } | ||
942 | 976 | ||
943 | if (r <= 0) | 977 | queue_io(md, bio); |
944 | goto out_req; | ||
945 | 978 | ||
946 | /* | 979 | return 0; |
947 | * We're in a while loop, because someone could suspend | ||
948 | * before we get to the following read lock. | ||
949 | */ | ||
950 | down_read(&md->io_lock); | ||
951 | } | 980 | } |
952 | 981 | ||
953 | __split_and_process_bio(md, bio); | 982 | __split_and_process_bio(md, bio); |
954 | up_read(&md->io_lock); | 983 | up_read(&md->io_lock); |
955 | return 0; | 984 | return 0; |
956 | |||
957 | out_req: | ||
958 | if (r < 0) | ||
959 | bio_io_error(bio); | ||
960 | |||
961 | return 0; | ||
962 | } | 985 | } |
963 | 986 | ||
964 | static void dm_unplug_all(struct request_queue *q) | 987 | static void dm_unplug_all(struct request_queue *q) |
@@ -978,7 +1001,7 @@ static int dm_any_congested(void *congested_data, int bdi_bits) | |||
978 | struct mapped_device *md = congested_data; | 1001 | struct mapped_device *md = congested_data; |
979 | struct dm_table *map; | 1002 | struct dm_table *map; |
980 | 1003 | ||
981 | if (!test_bit(DMF_BLOCK_IO, &md->flags)) { | 1004 | if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { |
982 | map = dm_get_table(md); | 1005 | map = dm_get_table(md); |
983 | if (map) { | 1006 | if (map) { |
984 | r = dm_table_any_congested(map, bdi_bits); | 1007 | r = dm_table_any_congested(map, bdi_bits); |
@@ -1193,6 +1216,7 @@ static void free_dev(struct mapped_device *md) | |||
1193 | mempool_destroy(md->tio_pool); | 1216 | mempool_destroy(md->tio_pool); |
1194 | mempool_destroy(md->io_pool); | 1217 | mempool_destroy(md->io_pool); |
1195 | bioset_free(md->bs); | 1218 | bioset_free(md->bs); |
1219 | blk_integrity_unregister(md->disk); | ||
1196 | del_gendisk(md->disk); | 1220 | del_gendisk(md->disk); |
1197 | free_minor(minor); | 1221 | free_minor(minor); |
1198 | 1222 | ||
@@ -1406,6 +1430,36 @@ static int dm_wait_for_completion(struct mapped_device *md, int interruptible) | |||
1406 | return r; | 1430 | return r; |
1407 | } | 1431 | } |
1408 | 1432 | ||
1433 | static int dm_flush(struct mapped_device *md) | ||
1434 | { | ||
1435 | dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); | ||
1436 | return 0; | ||
1437 | } | ||
1438 | |||
1439 | static void process_barrier(struct mapped_device *md, struct bio *bio) | ||
1440 | { | ||
1441 | int error = dm_flush(md); | ||
1442 | |||
1443 | if (unlikely(error)) { | ||
1444 | bio_endio(bio, error); | ||
1445 | return; | ||
1446 | } | ||
1447 | if (bio_empty_barrier(bio)) { | ||
1448 | bio_endio(bio, 0); | ||
1449 | return; | ||
1450 | } | ||
1451 | |||
1452 | __split_and_process_bio(md, bio); | ||
1453 | |||
1454 | error = dm_flush(md); | ||
1455 | |||
1456 | if (!error && md->barrier_error) | ||
1457 | error = md->barrier_error; | ||
1458 | |||
1459 | if (md->barrier_error != DM_ENDIO_REQUEUE) | ||
1460 | bio_endio(bio, error); | ||
1461 | } | ||
1462 | |||
1409 | /* | 1463 | /* |
1410 | * Process the deferred bios | 1464 | * Process the deferred bios |
1411 | */ | 1465 | */ |
@@ -1417,25 +1471,34 @@ static void dm_wq_work(struct work_struct *work) | |||
1417 | 1471 | ||
1418 | down_write(&md->io_lock); | 1472 | down_write(&md->io_lock); |
1419 | 1473 | ||
1420 | next_bio: | 1474 | while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { |
1421 | spin_lock_irq(&md->deferred_lock); | 1475 | spin_lock_irq(&md->deferred_lock); |
1422 | c = bio_list_pop(&md->deferred); | 1476 | c = bio_list_pop(&md->deferred); |
1423 | spin_unlock_irq(&md->deferred_lock); | 1477 | spin_unlock_irq(&md->deferred_lock); |
1424 | 1478 | ||
1425 | if (c) { | 1479 | if (!c) { |
1426 | __split_and_process_bio(md, c); | 1480 | clear_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags); |
1427 | goto next_bio; | 1481 | break; |
1428 | } | 1482 | } |
1429 | 1483 | ||
1430 | clear_bit(DMF_BLOCK_IO, &md->flags); | 1484 | up_write(&md->io_lock); |
1485 | |||
1486 | if (bio_barrier(c)) | ||
1487 | process_barrier(md, c); | ||
1488 | else | ||
1489 | __split_and_process_bio(md, c); | ||
1490 | |||
1491 | down_write(&md->io_lock); | ||
1492 | } | ||
1431 | 1493 | ||
1432 | up_write(&md->io_lock); | 1494 | up_write(&md->io_lock); |
1433 | } | 1495 | } |
1434 | 1496 | ||
1435 | static void dm_queue_flush(struct mapped_device *md) | 1497 | static void dm_queue_flush(struct mapped_device *md) |
1436 | { | 1498 | { |
1499 | clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); | ||
1500 | smp_mb__after_clear_bit(); | ||
1437 | queue_work(md->wq, &md->work); | 1501 | queue_work(md->wq, &md->work); |
1438 | flush_workqueue(md->wq); | ||
1439 | } | 1502 | } |
1440 | 1503 | ||
1441 | /* | 1504 | /* |
@@ -1553,20 +1616,36 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags) | |||
1553 | } | 1616 | } |
1554 | 1617 | ||
1555 | /* | 1618 | /* |
1556 | * First we set the BLOCK_IO flag so no more ios will be mapped. | 1619 | * Here we must make sure that no processes are submitting requests |
1620 | * to target drivers i.e. no one may be executing | ||
1621 | * __split_and_process_bio. This is called from dm_request and | ||
1622 | * dm_wq_work. | ||
1623 | * | ||
1624 | * To get all processes out of __split_and_process_bio in dm_request, | ||
1625 | * we take the write lock. To prevent any process from reentering | ||
1626 | * __split_and_process_bio from dm_request, we set | ||
1627 | * DMF_QUEUE_IO_TO_THREAD. | ||
1628 | * | ||
1629 | * To quiesce the thread (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND | ||
1630 | * and call flush_workqueue(md->wq). flush_workqueue will wait until | ||
1631 | * dm_wq_work exits and DMF_BLOCK_IO_FOR_SUSPEND will prevent any | ||
1632 | * further calls to __split_and_process_bio from dm_wq_work. | ||
1557 | */ | 1633 | */ |
1558 | down_write(&md->io_lock); | 1634 | down_write(&md->io_lock); |
1559 | set_bit(DMF_BLOCK_IO, &md->flags); | 1635 | set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); |
1560 | 1636 | set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags); | |
1561 | up_write(&md->io_lock); | 1637 | up_write(&md->io_lock); |
1562 | 1638 | ||
1639 | flush_workqueue(md->wq); | ||
1640 | |||
1563 | /* | 1641 | /* |
1564 | * Wait for the already-mapped ios to complete. | 1642 | * At this point no more requests are entering target request routines. |
1643 | * We call dm_wait_for_completion to wait for all existing requests | ||
1644 | * to finish. | ||
1565 | */ | 1645 | */ |
1566 | r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE); | 1646 | r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE); |
1567 | 1647 | ||
1568 | down_write(&md->io_lock); | 1648 | down_write(&md->io_lock); |
1569 | |||
1570 | if (noflush) | 1649 | if (noflush) |
1571 | clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); | 1650 | clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); |
1572 | up_write(&md->io_lock); | 1651 | up_write(&md->io_lock); |
@@ -1579,6 +1658,12 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags) | |||
1579 | goto out; /* pushback list is already flushed, so skip flush */ | 1658 | goto out; /* pushback list is already flushed, so skip flush */ |
1580 | } | 1659 | } |
1581 | 1660 | ||
1661 | /* | ||
1662 | * If dm_wait_for_completion returned 0, the device is completely | ||
1663 | * quiescent now. There is no request-processing activity. All new | ||
1664 | * requests are being added to md->deferred list. | ||
1665 | */ | ||
1666 | |||
1582 | dm_table_postsuspend_targets(map); | 1667 | dm_table_postsuspend_targets(map); |
1583 | 1668 | ||
1584 | set_bit(DMF_SUSPENDED, &md->flags); | 1669 | set_bit(DMF_SUSPENDED, &md->flags); |
diff --git a/drivers/md/dm.h b/drivers/md/dm.h index b48397c0abbd..a31506d93e91 100644 --- a/drivers/md/dm.h +++ b/drivers/md/dm.h | |||
@@ -52,7 +52,6 @@ int dm_table_any_congested(struct dm_table *t, int bdi_bits); | |||
52 | * To check the return value from dm_table_find_target(). | 52 | * To check the return value from dm_table_find_target(). |
53 | */ | 53 | */ |
54 | #define dm_target_is_valid(t) ((t)->table) | 54 | #define dm_target_is_valid(t) ((t)->table) |
55 | int dm_table_barrier_ok(struct dm_table *t); | ||
56 | 55 | ||
57 | /*----------------------------------------------------------------- | 56 | /*----------------------------------------------------------------- |
58 | * A registry of target types. | 57 | * A registry of target types. |