aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/md
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/md')
-rw-r--r--drivers/md/dm-bio-list.h117
-rw-r--r--drivers/md/dm-delay.c2
-rw-r--r--drivers/md/dm-ioctl.c21
-rw-r--r--drivers/md/dm-kcopyd.c23
-rw-r--r--drivers/md/dm-linear.c1
-rw-r--r--drivers/md/dm-mpath.c1
-rw-r--r--drivers/md/dm-raid1.c1
-rw-r--r--drivers/md/dm-region-hash.c1
-rw-r--r--drivers/md/dm-snap.c1
-rw-r--r--drivers/md/dm-table.c59
-rw-r--r--drivers/md/dm.c200
-rw-r--r--drivers/md/dm.h1
-rw-r--r--drivers/md/raid1.c1
-rw-r--r--drivers/md/raid10.c1
14 files changed, 218 insertions, 212 deletions
diff --git a/drivers/md/dm-bio-list.h b/drivers/md/dm-bio-list.h
deleted file mode 100644
index 345098b4ca77..000000000000
--- a/drivers/md/dm-bio-list.h
+++ /dev/null
@@ -1,117 +0,0 @@
1/*
2 * Copyright (C) 2004 Red Hat UK Ltd.
3 *
4 * This file is released under the GPL.
5 */
6
7#ifndef DM_BIO_LIST_H
8#define DM_BIO_LIST_H
9
10#include <linux/bio.h>
11
12#ifdef CONFIG_BLOCK
13
14struct bio_list {
15 struct bio *head;
16 struct bio *tail;
17};
18
19static inline int bio_list_empty(const struct bio_list *bl)
20{
21 return bl->head == NULL;
22}
23
24static inline void bio_list_init(struct bio_list *bl)
25{
26 bl->head = bl->tail = NULL;
27}
28
29#define bio_list_for_each(bio, bl) \
30 for (bio = (bl)->head; bio; bio = bio->bi_next)
31
32static inline unsigned bio_list_size(const struct bio_list *bl)
33{
34 unsigned sz = 0;
35 struct bio *bio;
36
37 bio_list_for_each(bio, bl)
38 sz++;
39
40 return sz;
41}
42
43static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
44{
45 bio->bi_next = NULL;
46
47 if (bl->tail)
48 bl->tail->bi_next = bio;
49 else
50 bl->head = bio;
51
52 bl->tail = bio;
53}
54
55static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)
56{
57 bio->bi_next = bl->head;
58
59 bl->head = bio;
60
61 if (!bl->tail)
62 bl->tail = bio;
63}
64
65static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
66{
67 if (!bl2->head)
68 return;
69
70 if (bl->tail)
71 bl->tail->bi_next = bl2->head;
72 else
73 bl->head = bl2->head;
74
75 bl->tail = bl2->tail;
76}
77
78static inline void bio_list_merge_head(struct bio_list *bl,
79 struct bio_list *bl2)
80{
81 if (!bl2->head)
82 return;
83
84 if (bl->head)
85 bl2->tail->bi_next = bl->head;
86 else
87 bl->tail = bl2->tail;
88
89 bl->head = bl2->head;
90}
91
92static inline struct bio *bio_list_pop(struct bio_list *bl)
93{
94 struct bio *bio = bl->head;
95
96 if (bio) {
97 bl->head = bl->head->bi_next;
98 if (!bl->head)
99 bl->tail = NULL;
100
101 bio->bi_next = NULL;
102 }
103
104 return bio;
105}
106
107static inline struct bio *bio_list_get(struct bio_list *bl)
108{
109 struct bio *bio = bl->head;
110
111 bl->head = bl->tail = NULL;
112
113 return bio;
114}
115
116#endif /* CONFIG_BLOCK */
117#endif
diff --git a/drivers/md/dm-delay.c b/drivers/md/dm-delay.c
index 59ee1b015d2d..559dbb52bc85 100644
--- a/drivers/md/dm-delay.c
+++ b/drivers/md/dm-delay.c
@@ -15,8 +15,6 @@
15 15
16#include <linux/device-mapper.h> 16#include <linux/device-mapper.h>
17 17
18#include "dm-bio-list.h"
19
20#define DM_MSG_PREFIX "delay" 18#define DM_MSG_PREFIX "delay"
21 19
22struct delay_c { 20struct delay_c {
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index f01096549a93..823ceba6efa8 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -1047,6 +1047,19 @@ static int populate_table(struct dm_table *table,
1047 return dm_table_complete(table); 1047 return dm_table_complete(table);
1048} 1048}
1049 1049
1050static int table_prealloc_integrity(struct dm_table *t,
1051 struct mapped_device *md)
1052{
1053 struct list_head *devices = dm_table_get_devices(t);
1054 struct dm_dev_internal *dd;
1055
1056 list_for_each_entry(dd, devices, list)
1057 if (bdev_get_integrity(dd->dm_dev.bdev))
1058 return blk_integrity_register(dm_disk(md), NULL);
1059
1060 return 0;
1061}
1062
1050static int table_load(struct dm_ioctl *param, size_t param_size) 1063static int table_load(struct dm_ioctl *param, size_t param_size)
1051{ 1064{
1052 int r; 1065 int r;
@@ -1068,6 +1081,14 @@ static int table_load(struct dm_ioctl *param, size_t param_size)
1068 goto out; 1081 goto out;
1069 } 1082 }
1070 1083
1084 r = table_prealloc_integrity(t, md);
1085 if (r) {
1086 DMERR("%s: could not register integrity profile.",
1087 dm_device_name(md));
1088 dm_table_destroy(t);
1089 goto out;
1090 }
1091
1071 down_write(&_hash_lock); 1092 down_write(&_hash_lock);
1072 hc = dm_get_mdptr(md); 1093 hc = dm_get_mdptr(md);
1073 if (!hc || hc->md != md) { 1094 if (!hc || hc->md != md) {
diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c
index 0a225da21272..3e3fc06cb861 100644
--- a/drivers/md/dm-kcopyd.c
+++ b/drivers/md/dm-kcopyd.c
@@ -297,7 +297,8 @@ static int run_complete_job(struct kcopyd_job *job)
297 dm_kcopyd_notify_fn fn = job->fn; 297 dm_kcopyd_notify_fn fn = job->fn;
298 struct dm_kcopyd_client *kc = job->kc; 298 struct dm_kcopyd_client *kc = job->kc;
299 299
300 kcopyd_put_pages(kc, job->pages); 300 if (job->pages)
301 kcopyd_put_pages(kc, job->pages);
301 mempool_free(job, kc->job_pool); 302 mempool_free(job, kc->job_pool);
302 fn(read_err, write_err, context); 303 fn(read_err, write_err, context);
303 304
@@ -461,6 +462,7 @@ static void segment_complete(int read_err, unsigned long write_err,
461 sector_t progress = 0; 462 sector_t progress = 0;
462 sector_t count = 0; 463 sector_t count = 0;
463 struct kcopyd_job *job = (struct kcopyd_job *) context; 464 struct kcopyd_job *job = (struct kcopyd_job *) context;
465 struct dm_kcopyd_client *kc = job->kc;
464 466
465 mutex_lock(&job->lock); 467 mutex_lock(&job->lock);
466 468
@@ -490,7 +492,7 @@ static void segment_complete(int read_err, unsigned long write_err,
490 492
491 if (count) { 493 if (count) {
492 int i; 494 int i;
493 struct kcopyd_job *sub_job = mempool_alloc(job->kc->job_pool, 495 struct kcopyd_job *sub_job = mempool_alloc(kc->job_pool,
494 GFP_NOIO); 496 GFP_NOIO);
495 497
496 *sub_job = *job; 498 *sub_job = *job;
@@ -509,13 +511,16 @@ static void segment_complete(int read_err, unsigned long write_err,
509 } else if (atomic_dec_and_test(&job->sub_jobs)) { 511 } else if (atomic_dec_and_test(&job->sub_jobs)) {
510 512
511 /* 513 /*
512 * To avoid a race we must keep the job around 514 * Queue the completion callback to the kcopyd thread.
513 * until after the notify function has completed. 515 *
514 * Otherwise the client may try and stop the job 516 * Some callers assume that all the completions are called
515 * after we've completed. 517 * from a single thread and don't race with each other.
518 *
519 * We must not call the callback directly here because this
520 * code may not be executing in the thread.
516 */ 521 */
517 job->fn(read_err, write_err, job->context); 522 push(&kc->complete_jobs, job);
518 mempool_free(job, job->kc->job_pool); 523 wake(kc);
519 } 524 }
520} 525}
521 526
@@ -528,6 +533,8 @@ static void split_job(struct kcopyd_job *job)
528{ 533{
529 int i; 534 int i;
530 535
536 atomic_inc(&job->kc->nr_jobs);
537
531 atomic_set(&job->sub_jobs, SPLIT_COUNT); 538 atomic_set(&job->sub_jobs, SPLIT_COUNT);
532 for (i = 0; i < SPLIT_COUNT; i++) 539 for (i = 0; i < SPLIT_COUNT; i++)
533 segment_complete(0, 0u, job); 540 segment_complete(0, 0u, job);
diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c
index bfa107f59d96..79fb53e51c70 100644
--- a/drivers/md/dm-linear.c
+++ b/drivers/md/dm-linear.c
@@ -142,7 +142,6 @@ static struct target_type linear_target = {
142 .status = linear_status, 142 .status = linear_status,
143 .ioctl = linear_ioctl, 143 .ioctl = linear_ioctl,
144 .merge = linear_merge, 144 .merge = linear_merge,
145 .features = DM_TARGET_SUPPORTS_BARRIERS,
146}; 145};
147 146
148int __init dm_linear_init(void) 147int __init dm_linear_init(void)
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index 095f77bf9681..6a386ab4f7eb 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -8,7 +8,6 @@
8#include <linux/device-mapper.h> 8#include <linux/device-mapper.h>
9 9
10#include "dm-path-selector.h" 10#include "dm-path-selector.h"
11#include "dm-bio-list.h"
12#include "dm-bio-record.h" 11#include "dm-bio-record.h"
13#include "dm-uevent.h" 12#include "dm-uevent.h"
14 13
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
index 536ef0bef154..076fbb4e967a 100644
--- a/drivers/md/dm-raid1.c
+++ b/drivers/md/dm-raid1.c
@@ -5,7 +5,6 @@
5 * This file is released under the GPL. 5 * This file is released under the GPL.
6 */ 6 */
7 7
8#include "dm-bio-list.h"
9#include "dm-bio-record.h" 8#include "dm-bio-record.h"
10 9
11#include <linux/init.h> 10#include <linux/init.h>
diff --git a/drivers/md/dm-region-hash.c b/drivers/md/dm-region-hash.c
index 59f8d9df9e1a..7b899be0b087 100644
--- a/drivers/md/dm-region-hash.c
+++ b/drivers/md/dm-region-hash.c
@@ -14,7 +14,6 @@
14#include <linux/vmalloc.h> 14#include <linux/vmalloc.h>
15 15
16#include "dm.h" 16#include "dm.h"
17#include "dm-bio-list.h"
18 17
19#define DM_MSG_PREFIX "region hash" 18#define DM_MSG_PREFIX "region hash"
20 19
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index 981a0413068f..d73f17fc7778 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -22,7 +22,6 @@
22#include <linux/workqueue.h> 22#include <linux/workqueue.h>
23 23
24#include "dm-exception-store.h" 24#include "dm-exception-store.h"
25#include "dm-bio-list.h"
26 25
27#define DM_MSG_PREFIX "snapshots" 26#define DM_MSG_PREFIX "snapshots"
28 27
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index e8361b191b9b..429b50b975d5 100644
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -52,8 +52,6 @@ struct dm_table {
52 sector_t *highs; 52 sector_t *highs;
53 struct dm_target *targets; 53 struct dm_target *targets;
54 54
55 unsigned barriers_supported:1;
56
57 /* 55 /*
58 * Indicates the rw permissions for the new logical 56 * Indicates the rw permissions for the new logical
59 * device. This should be a combination of FMODE_READ 57 * device. This should be a combination of FMODE_READ
@@ -243,7 +241,6 @@ int dm_table_create(struct dm_table **result, fmode_t mode,
243 241
244 INIT_LIST_HEAD(&t->devices); 242 INIT_LIST_HEAD(&t->devices);
245 atomic_set(&t->holders, 0); 243 atomic_set(&t->holders, 0);
246 t->barriers_supported = 1;
247 244
248 if (!num_targets) 245 if (!num_targets)
249 num_targets = KEYS_PER_NODE; 246 num_targets = KEYS_PER_NODE;
@@ -751,10 +748,6 @@ int dm_table_add_target(struct dm_table *t, const char *type,
751 /* FIXME: the plan is to combine high here and then have 748 /* FIXME: the plan is to combine high here and then have
752 * the merge fn apply the target level restrictions. */ 749 * the merge fn apply the target level restrictions. */
753 combine_restrictions_low(&t->limits, &tgt->limits); 750 combine_restrictions_low(&t->limits, &tgt->limits);
754
755 if (!(tgt->type->features & DM_TARGET_SUPPORTS_BARRIERS))
756 t->barriers_supported = 0;
757
758 return 0; 751 return 0;
759 752
760 bad: 753 bad:
@@ -799,12 +792,6 @@ int dm_table_complete(struct dm_table *t)
799 792
800 check_for_valid_limits(&t->limits); 793 check_for_valid_limits(&t->limits);
801 794
802 /*
803 * We only support barriers if there is exactly one underlying device.
804 */
805 if (!list_is_singular(&t->devices))
806 t->barriers_supported = 0;
807
808 /* how many indexes will the btree have ? */ 795 /* how many indexes will the btree have ? */
809 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE); 796 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);
810 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE); 797 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);
@@ -879,6 +866,45 @@ struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)
879 return &t->targets[(KEYS_PER_NODE * n) + k]; 866 return &t->targets[(KEYS_PER_NODE * n) + k];
880} 867}
881 868
869/*
870 * Set the integrity profile for this device if all devices used have
871 * matching profiles.
872 */
873static void dm_table_set_integrity(struct dm_table *t)
874{
875 struct list_head *devices = dm_table_get_devices(t);
876 struct dm_dev_internal *prev = NULL, *dd = NULL;
877
878 if (!blk_get_integrity(dm_disk(t->md)))
879 return;
880
881 list_for_each_entry(dd, devices, list) {
882 if (prev &&
883 blk_integrity_compare(prev->dm_dev.bdev->bd_disk,
884 dd->dm_dev.bdev->bd_disk) < 0) {
885 DMWARN("%s: integrity not set: %s and %s mismatch",
886 dm_device_name(t->md),
887 prev->dm_dev.bdev->bd_disk->disk_name,
888 dd->dm_dev.bdev->bd_disk->disk_name);
889 goto no_integrity;
890 }
891 prev = dd;
892 }
893
894 if (!prev || !bdev_get_integrity(prev->dm_dev.bdev))
895 goto no_integrity;
896
897 blk_integrity_register(dm_disk(t->md),
898 bdev_get_integrity(prev->dm_dev.bdev));
899
900 return;
901
902no_integrity:
903 blk_integrity_register(dm_disk(t->md), NULL);
904
905 return;
906}
907
882void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q) 908void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q)
883{ 909{
884 /* 910 /*
@@ -899,6 +925,7 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q)
899 else 925 else
900 queue_flag_set_unlocked(QUEUE_FLAG_CLUSTER, q); 926 queue_flag_set_unlocked(QUEUE_FLAG_CLUSTER, q);
901 927
928 dm_table_set_integrity(t);
902} 929}
903 930
904unsigned int dm_table_get_num_targets(struct dm_table *t) 931unsigned int dm_table_get_num_targets(struct dm_table *t)
@@ -1019,12 +1046,6 @@ struct mapped_device *dm_table_get_md(struct dm_table *t)
1019 return t->md; 1046 return t->md;
1020} 1047}
1021 1048
1022int dm_table_barrier_ok(struct dm_table *t)
1023{
1024 return t->barriers_supported;
1025}
1026EXPORT_SYMBOL(dm_table_barrier_ok);
1027
1028EXPORT_SYMBOL(dm_vcalloc); 1049EXPORT_SYMBOL(dm_vcalloc);
1029EXPORT_SYMBOL(dm_get_device); 1050EXPORT_SYMBOL(dm_get_device);
1030EXPORT_SYMBOL(dm_put_device); 1051EXPORT_SYMBOL(dm_put_device);
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 788ba96a6256..424f7b048c30 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -6,7 +6,6 @@
6 */ 6 */
7 7
8#include "dm.h" 8#include "dm.h"
9#include "dm-bio-list.h"
10#include "dm-uevent.h" 9#include "dm-uevent.h"
11 10
12#include <linux/init.h> 11#include <linux/init.h>
@@ -89,12 +88,13 @@ union map_info *dm_get_mapinfo(struct bio *bio)
89/* 88/*
90 * Bits for the md->flags field. 89 * Bits for the md->flags field.
91 */ 90 */
92#define DMF_BLOCK_IO 0 91#define DMF_BLOCK_IO_FOR_SUSPEND 0
93#define DMF_SUSPENDED 1 92#define DMF_SUSPENDED 1
94#define DMF_FROZEN 2 93#define DMF_FROZEN 2
95#define DMF_FREEING 3 94#define DMF_FREEING 3
96#define DMF_DELETING 4 95#define DMF_DELETING 4
97#define DMF_NOFLUSH_SUSPENDING 5 96#define DMF_NOFLUSH_SUSPENDING 5
97#define DMF_QUEUE_IO_TO_THREAD 6
98 98
99/* 99/*
100 * Work processed by per-device workqueue. 100 * Work processed by per-device workqueue.
@@ -124,6 +124,11 @@ struct mapped_device {
124 spinlock_t deferred_lock; 124 spinlock_t deferred_lock;
125 125
126 /* 126 /*
127 * An error from the barrier request currently being processed.
128 */
129 int barrier_error;
130
131 /*
127 * Processing queue (flush/barriers) 132 * Processing queue (flush/barriers)
128 */ 133 */
129 struct workqueue_struct *wq; 134 struct workqueue_struct *wq;
@@ -424,6 +429,10 @@ static void end_io_acct(struct dm_io *io)
424 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration); 429 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
425 part_stat_unlock(); 430 part_stat_unlock();
426 431
432 /*
433 * After this is decremented the bio must not be touched if it is
434 * a barrier.
435 */
427 dm_disk(md)->part0.in_flight = pending = 436 dm_disk(md)->part0.in_flight = pending =
428 atomic_dec_return(&md->pending); 437 atomic_dec_return(&md->pending);
429 438
@@ -435,21 +444,18 @@ static void end_io_acct(struct dm_io *io)
435/* 444/*
436 * Add the bio to the list of deferred io. 445 * Add the bio to the list of deferred io.
437 */ 446 */
438static int queue_io(struct mapped_device *md, struct bio *bio) 447static void queue_io(struct mapped_device *md, struct bio *bio)
439{ 448{
440 down_write(&md->io_lock); 449 down_write(&md->io_lock);
441 450
442 if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
443 up_write(&md->io_lock);
444 return 1;
445 }
446
447 spin_lock_irq(&md->deferred_lock); 451 spin_lock_irq(&md->deferred_lock);
448 bio_list_add(&md->deferred, bio); 452 bio_list_add(&md->deferred, bio);
449 spin_unlock_irq(&md->deferred_lock); 453 spin_unlock_irq(&md->deferred_lock);
450 454
455 if (!test_and_set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags))
456 queue_work(md->wq, &md->work);
457
451 up_write(&md->io_lock); 458 up_write(&md->io_lock);
452 return 0; /* deferred successfully */
453} 459}
454 460
455/* 461/*
@@ -533,25 +539,35 @@ static void dec_pending(struct dm_io *io, int error)
533 */ 539 */
534 spin_lock_irqsave(&md->deferred_lock, flags); 540 spin_lock_irqsave(&md->deferred_lock, flags);
535 if (__noflush_suspending(md)) 541 if (__noflush_suspending(md))
536 bio_list_add(&md->deferred, io->bio); 542 bio_list_add_head(&md->deferred, io->bio);
537 else 543 else
538 /* noflush suspend was interrupted. */ 544 /* noflush suspend was interrupted. */
539 io->error = -EIO; 545 io->error = -EIO;
540 spin_unlock_irqrestore(&md->deferred_lock, flags); 546 spin_unlock_irqrestore(&md->deferred_lock, flags);
541 } 547 }
542 548
543 end_io_acct(io);
544
545 io_error = io->error; 549 io_error = io->error;
546 bio = io->bio; 550 bio = io->bio;
547 551
548 free_io(md, io); 552 if (bio_barrier(bio)) {
553 /*
554 * There can be just one barrier request so we use
555 * a per-device variable for error reporting.
556 * Note that you can't touch the bio after end_io_acct
557 */
558 md->barrier_error = io_error;
559 end_io_acct(io);
560 } else {
561 end_io_acct(io);
549 562
550 if (io_error != DM_ENDIO_REQUEUE) { 563 if (io_error != DM_ENDIO_REQUEUE) {
551 trace_block_bio_complete(md->queue, bio); 564 trace_block_bio_complete(md->queue, bio);
552 565
553 bio_endio(bio, io_error); 566 bio_endio(bio, io_error);
567 }
554 } 568 }
569
570 free_io(md, io);
555 } 571 }
556} 572}
557 573
@@ -693,13 +709,19 @@ static struct bio *split_bvec(struct bio *bio, sector_t sector,
693 709
694 clone->bi_sector = sector; 710 clone->bi_sector = sector;
695 clone->bi_bdev = bio->bi_bdev; 711 clone->bi_bdev = bio->bi_bdev;
696 clone->bi_rw = bio->bi_rw; 712 clone->bi_rw = bio->bi_rw & ~(1 << BIO_RW_BARRIER);
697 clone->bi_vcnt = 1; 713 clone->bi_vcnt = 1;
698 clone->bi_size = to_bytes(len); 714 clone->bi_size = to_bytes(len);
699 clone->bi_io_vec->bv_offset = offset; 715 clone->bi_io_vec->bv_offset = offset;
700 clone->bi_io_vec->bv_len = clone->bi_size; 716 clone->bi_io_vec->bv_len = clone->bi_size;
701 clone->bi_flags |= 1 << BIO_CLONED; 717 clone->bi_flags |= 1 << BIO_CLONED;
702 718
719 if (bio_integrity(bio)) {
720 bio_integrity_clone(clone, bio, GFP_NOIO);
721 bio_integrity_trim(clone,
722 bio_sector_offset(bio, idx, offset), len);
723 }
724
703 return clone; 725 return clone;
704} 726}
705 727
@@ -714,6 +736,7 @@ static struct bio *clone_bio(struct bio *bio, sector_t sector,
714 736
715 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs); 737 clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
716 __bio_clone(clone, bio); 738 __bio_clone(clone, bio);
739 clone->bi_rw &= ~(1 << BIO_RW_BARRIER);
717 clone->bi_destructor = dm_bio_destructor; 740 clone->bi_destructor = dm_bio_destructor;
718 clone->bi_sector = sector; 741 clone->bi_sector = sector;
719 clone->bi_idx = idx; 742 clone->bi_idx = idx;
@@ -721,6 +744,14 @@ static struct bio *clone_bio(struct bio *bio, sector_t sector,
721 clone->bi_size = to_bytes(len); 744 clone->bi_size = to_bytes(len);
722 clone->bi_flags &= ~(1 << BIO_SEG_VALID); 745 clone->bi_flags &= ~(1 << BIO_SEG_VALID);
723 746
747 if (bio_integrity(bio)) {
748 bio_integrity_clone(clone, bio, GFP_NOIO);
749
750 if (idx != bio->bi_idx || clone->bi_size < bio->bi_size)
751 bio_integrity_trim(clone,
752 bio_sector_offset(bio, idx, 0), len);
753 }
754
724 return clone; 755 return clone;
725} 756}
726 757
@@ -834,14 +865,13 @@ static void __split_and_process_bio(struct mapped_device *md, struct bio *bio)
834 865
835 ci.map = dm_get_table(md); 866 ci.map = dm_get_table(md);
836 if (unlikely(!ci.map)) { 867 if (unlikely(!ci.map)) {
837 bio_io_error(bio); 868 if (!bio_barrier(bio))
838 return; 869 bio_io_error(bio);
839 } 870 else
840 if (unlikely(bio_barrier(bio) && !dm_table_barrier_ok(ci.map))) { 871 md->barrier_error = -EIO;
841 dm_table_put(ci.map);
842 bio_endio(bio, -EOPNOTSUPP);
843 return; 872 return;
844 } 873 }
874
845 ci.md = md; 875 ci.md = md;
846 ci.bio = bio; 876 ci.bio = bio;
847 ci.io = alloc_io(md); 877 ci.io = alloc_io(md);
@@ -918,7 +948,6 @@ out:
918 */ 948 */
919static int dm_request(struct request_queue *q, struct bio *bio) 949static int dm_request(struct request_queue *q, struct bio *bio)
920{ 950{
921 int r = -EIO;
922 int rw = bio_data_dir(bio); 951 int rw = bio_data_dir(bio);
923 struct mapped_device *md = q->queuedata; 952 struct mapped_device *md = q->queuedata;
924 int cpu; 953 int cpu;
@@ -931,34 +960,27 @@ static int dm_request(struct request_queue *q, struct bio *bio)
931 part_stat_unlock(); 960 part_stat_unlock();
932 961
933 /* 962 /*
934 * If we're suspended we have to queue 963 * If we're suspended or the thread is processing barriers
935 * this io for later. 964 * we have to queue this io for later.
936 */ 965 */
937 while (test_bit(DMF_BLOCK_IO, &md->flags)) { 966 if (unlikely(test_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags)) ||
967 unlikely(bio_barrier(bio))) {
938 up_read(&md->io_lock); 968 up_read(&md->io_lock);
939 969
940 if (bio_rw(bio) != READA) 970 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) &&
941 r = queue_io(md, bio); 971 bio_rw(bio) == READA) {
972 bio_io_error(bio);
973 return 0;
974 }
942 975
943 if (r <= 0) 976 queue_io(md, bio);
944 goto out_req;
945 977
946 /* 978 return 0;
947 * We're in a while loop, because someone could suspend
948 * before we get to the following read lock.
949 */
950 down_read(&md->io_lock);
951 } 979 }
952 980
953 __split_and_process_bio(md, bio); 981 __split_and_process_bio(md, bio);
954 up_read(&md->io_lock); 982 up_read(&md->io_lock);
955 return 0; 983 return 0;
956
957out_req:
958 if (r < 0)
959 bio_io_error(bio);
960
961 return 0;
962} 984}
963 985
964static void dm_unplug_all(struct request_queue *q) 986static void dm_unplug_all(struct request_queue *q)
@@ -978,7 +1000,7 @@ static int dm_any_congested(void *congested_data, int bdi_bits)
978 struct mapped_device *md = congested_data; 1000 struct mapped_device *md = congested_data;
979 struct dm_table *map; 1001 struct dm_table *map;
980 1002
981 if (!test_bit(DMF_BLOCK_IO, &md->flags)) { 1003 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
982 map = dm_get_table(md); 1004 map = dm_get_table(md);
983 if (map) { 1005 if (map) {
984 r = dm_table_any_congested(map, bdi_bits); 1006 r = dm_table_any_congested(map, bdi_bits);
@@ -1193,6 +1215,7 @@ static void free_dev(struct mapped_device *md)
1193 mempool_destroy(md->tio_pool); 1215 mempool_destroy(md->tio_pool);
1194 mempool_destroy(md->io_pool); 1216 mempool_destroy(md->io_pool);
1195 bioset_free(md->bs); 1217 bioset_free(md->bs);
1218 blk_integrity_unregister(md->disk);
1196 del_gendisk(md->disk); 1219 del_gendisk(md->disk);
1197 free_minor(minor); 1220 free_minor(minor);
1198 1221
@@ -1406,6 +1429,36 @@ static int dm_wait_for_completion(struct mapped_device *md, int interruptible)
1406 return r; 1429 return r;
1407} 1430}
1408 1431
1432static int dm_flush(struct mapped_device *md)
1433{
1434 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
1435 return 0;
1436}
1437
1438static void process_barrier(struct mapped_device *md, struct bio *bio)
1439{
1440 int error = dm_flush(md);
1441
1442 if (unlikely(error)) {
1443 bio_endio(bio, error);
1444 return;
1445 }
1446 if (bio_empty_barrier(bio)) {
1447 bio_endio(bio, 0);
1448 return;
1449 }
1450
1451 __split_and_process_bio(md, bio);
1452
1453 error = dm_flush(md);
1454
1455 if (!error && md->barrier_error)
1456 error = md->barrier_error;
1457
1458 if (md->barrier_error != DM_ENDIO_REQUEUE)
1459 bio_endio(bio, error);
1460}
1461
1409/* 1462/*
1410 * Process the deferred bios 1463 * Process the deferred bios
1411 */ 1464 */
@@ -1417,25 +1470,34 @@ static void dm_wq_work(struct work_struct *work)
1417 1470
1418 down_write(&md->io_lock); 1471 down_write(&md->io_lock);
1419 1472
1420next_bio: 1473 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1421 spin_lock_irq(&md->deferred_lock); 1474 spin_lock_irq(&md->deferred_lock);
1422 c = bio_list_pop(&md->deferred); 1475 c = bio_list_pop(&md->deferred);
1423 spin_unlock_irq(&md->deferred_lock); 1476 spin_unlock_irq(&md->deferred_lock);
1424 1477
1425 if (c) { 1478 if (!c) {
1426 __split_and_process_bio(md, c); 1479 clear_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags);
1427 goto next_bio; 1480 break;
1428 } 1481 }
1429 1482
1430 clear_bit(DMF_BLOCK_IO, &md->flags); 1483 up_write(&md->io_lock);
1484
1485 if (bio_barrier(c))
1486 process_barrier(md, c);
1487 else
1488 __split_and_process_bio(md, c);
1489
1490 down_write(&md->io_lock);
1491 }
1431 1492
1432 up_write(&md->io_lock); 1493 up_write(&md->io_lock);
1433} 1494}
1434 1495
1435static void dm_queue_flush(struct mapped_device *md) 1496static void dm_queue_flush(struct mapped_device *md)
1436{ 1497{
1498 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
1499 smp_mb__after_clear_bit();
1437 queue_work(md->wq, &md->work); 1500 queue_work(md->wq, &md->work);
1438 flush_workqueue(md->wq);
1439} 1501}
1440 1502
1441/* 1503/*
@@ -1553,20 +1615,36 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
1553 } 1615 }
1554 1616
1555 /* 1617 /*
1556 * First we set the BLOCK_IO flag so no more ios will be mapped. 1618 * Here we must make sure that no processes are submitting requests
1619 * to target drivers i.e. no one may be executing
1620 * __split_and_process_bio. This is called from dm_request and
1621 * dm_wq_work.
1622 *
1623 * To get all processes out of __split_and_process_bio in dm_request,
1624 * we take the write lock. To prevent any process from reentering
1625 * __split_and_process_bio from dm_request, we set
1626 * DMF_QUEUE_IO_TO_THREAD.
1627 *
1628 * To quiesce the thread (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND
1629 * and call flush_workqueue(md->wq). flush_workqueue will wait until
1630 * dm_wq_work exits and DMF_BLOCK_IO_FOR_SUSPEND will prevent any
1631 * further calls to __split_and_process_bio from dm_wq_work.
1557 */ 1632 */
1558 down_write(&md->io_lock); 1633 down_write(&md->io_lock);
1559 set_bit(DMF_BLOCK_IO, &md->flags); 1634 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
1560 1635 set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags);
1561 up_write(&md->io_lock); 1636 up_write(&md->io_lock);
1562 1637
1638 flush_workqueue(md->wq);
1639
1563 /* 1640 /*
1564 * Wait for the already-mapped ios to complete. 1641 * At this point no more requests are entering target request routines.
1642 * We call dm_wait_for_completion to wait for all existing requests
1643 * to finish.
1565 */ 1644 */
1566 r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE); 1645 r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE);
1567 1646
1568 down_write(&md->io_lock); 1647 down_write(&md->io_lock);
1569
1570 if (noflush) 1648 if (noflush)
1571 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 1649 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1572 up_write(&md->io_lock); 1650 up_write(&md->io_lock);
@@ -1579,6 +1657,12 @@ int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
1579 goto out; /* pushback list is already flushed, so skip flush */ 1657 goto out; /* pushback list is already flushed, so skip flush */
1580 } 1658 }
1581 1659
1660 /*
1661 * If dm_wait_for_completion returned 0, the device is completely
1662 * quiescent now. There is no request-processing activity. All new
1663 * requests are being added to md->deferred list.
1664 */
1665
1582 dm_table_postsuspend_targets(map); 1666 dm_table_postsuspend_targets(map);
1583 1667
1584 set_bit(DMF_SUSPENDED, &md->flags); 1668 set_bit(DMF_SUSPENDED, &md->flags);
diff --git a/drivers/md/dm.h b/drivers/md/dm.h
index b48397c0abbd..a31506d93e91 100644
--- a/drivers/md/dm.h
+++ b/drivers/md/dm.h
@@ -52,7 +52,6 @@ int dm_table_any_congested(struct dm_table *t, int bdi_bits);
52 * To check the return value from dm_table_find_target(). 52 * To check the return value from dm_table_find_target().
53 */ 53 */
54#define dm_target_is_valid(t) ((t)->table) 54#define dm_target_is_valid(t) ((t)->table)
55int dm_table_barrier_ok(struct dm_table *t);
56 55
57/*----------------------------------------------------------------- 56/*-----------------------------------------------------------------
58 * A registry of target types. 57 * A registry of target types.
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 274b491a11c1..36df9109cde1 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -35,7 +35,6 @@
35#include <linux/blkdev.h> 35#include <linux/blkdev.h>
36#include <linux/seq_file.h> 36#include <linux/seq_file.h>
37#include "md.h" 37#include "md.h"
38#include "dm-bio-list.h"
39#include "raid1.h" 38#include "raid1.h"
40#include "bitmap.h" 39#include "bitmap.h"
41 40
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index e293d92641ac..81a54f17417e 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -22,7 +22,6 @@
22#include <linux/blkdev.h> 22#include <linux/blkdev.h>
23#include <linux/seq_file.h> 23#include <linux/seq_file.h>
24#include "md.h" 24#include "md.h"
25#include "dm-bio-list.h"
26#include "raid10.h" 25#include "raid10.h"
27#include "bitmap.h" 26#include "bitmap.h"
28 27
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-16 16:16:01 -0500