diff options
172 files changed, 1520 insertions, 2112 deletions
diff --git a/Documentation/block/biodoc.txt b/Documentation/block/biodoc.txt index b9a83dd24732..2a7b38c832c7 100644 --- a/Documentation/block/biodoc.txt +++ b/Documentation/block/biodoc.txt | |||
@@ -963,11 +963,6 @@ elevator_dispatch_fn* fills the dispatch queue with ready requests. | |||
963 | 963 | ||
964 | elevator_add_req_fn* called to add a new request into the scheduler | 964 | elevator_add_req_fn* called to add a new request into the scheduler |
965 | 965 | ||
966 | elevator_queue_empty_fn returns true if the merge queue is empty. | ||
967 | Drivers shouldn't use this, but rather check | ||
968 | if elv_next_request is NULL (without losing the | ||
969 | request if one exists!) | ||
970 | |||
971 | elevator_former_req_fn | 966 | elevator_former_req_fn |
972 | elevator_latter_req_fn These return the request before or after the | 967 | elevator_latter_req_fn These return the request before or after the |
973 | one specified in disk sort order. Used by the | 968 | one specified in disk sort order. Used by the |
diff --git a/Documentation/cgroups/blkio-controller.txt b/Documentation/cgroups/blkio-controller.txt index 4ed7b5ceeed2..465351d4cf85 100644 --- a/Documentation/cgroups/blkio-controller.txt +++ b/Documentation/cgroups/blkio-controller.txt | |||
@@ -140,7 +140,7 @@ Proportional weight policy files | |||
140 | - Specifies per cgroup weight. This is default weight of the group | 140 | - Specifies per cgroup weight. This is default weight of the group |
141 | on all the devices until and unless overridden by per device rule. | 141 | on all the devices until and unless overridden by per device rule. |
142 | (See blkio.weight_device). | 142 | (See blkio.weight_device). |
143 | Currently allowed range of weights is from 100 to 1000. | 143 | Currently allowed range of weights is from 10 to 1000. |
144 | 144 | ||
145 | - blkio.weight_device | 145 | - blkio.weight_device |
146 | - One can specify per cgroup per device rules using this interface. | 146 | - One can specify per cgroup per device rules using this interface. |
@@ -343,34 +343,6 @@ Common files among various policies | |||
343 | 343 | ||
344 | CFQ sysfs tunable | 344 | CFQ sysfs tunable |
345 | ================= | 345 | ================= |
346 | /sys/block/<disk>/queue/iosched/group_isolation | ||
347 | ----------------------------------------------- | ||
348 | |||
349 | If group_isolation=1, it provides stronger isolation between groups at the | ||
350 | expense of throughput. By default group_isolation is 0. In general that | ||
351 | means that if group_isolation=0, expect fairness for sequential workload | ||
352 | only. Set group_isolation=1 to see fairness for random IO workload also. | ||
353 | |||
354 | Generally CFQ will put random seeky workload in sync-noidle category. CFQ | ||
355 | will disable idling on these queues and it does a collective idling on group | ||
356 | of such queues. Generally these are slow moving queues and if there is a | ||
357 | sync-noidle service tree in each group, that group gets exclusive access to | ||
358 | disk for certain period. That means it will bring the throughput down if | ||
359 | group does not have enough IO to drive deeper queue depths and utilize disk | ||
360 | capacity to the fullest in the slice allocated to it. But the flip side is | ||
361 | that even a random reader should get better latencies and overall throughput | ||
362 | if there are lots of sequential readers/sync-idle workload running in the | ||
363 | system. | ||
364 | |||
365 | If group_isolation=0, then CFQ automatically moves all the random seeky queues | ||
366 | in the root group. That means there will be no service differentiation for | ||
367 | that kind of workload. This leads to better throughput as we do collective | ||
368 | idling on root sync-noidle tree. | ||
369 | |||
370 | By default one should run with group_isolation=0. If that is not sufficient | ||
371 | and one wants stronger isolation between groups, then set group_isolation=1 | ||
372 | but this will come at cost of reduced throughput. | ||
373 | |||
374 | /sys/block/<disk>/queue/iosched/slice_idle | 346 | /sys/block/<disk>/queue/iosched/slice_idle |
375 | ------------------------------------------ | 347 | ------------------------------------------ |
376 | On a faster hardware CFQ can be slow, especially with sequential workload. | 348 | On a faster hardware CFQ can be slow, especially with sequential workload. |
diff --git a/Documentation/iostats.txt b/Documentation/iostats.txt index f6dece5b7014..c76c21d87e85 100644 --- a/Documentation/iostats.txt +++ b/Documentation/iostats.txt | |||
@@ -1,8 +1,6 @@ | |||
1 | I/O statistics fields | 1 | I/O statistics fields |
2 | --------------- | 2 | --------------- |
3 | 3 | ||
4 | Last modified Sep 30, 2003 | ||
5 | |||
6 | Since 2.4.20 (and some versions before, with patches), and 2.5.45, | 4 | Since 2.4.20 (and some versions before, with patches), and 2.5.45, |
7 | more extensive disk statistics have been introduced to help measure disk | 5 | more extensive disk statistics have been introduced to help measure disk |
8 | activity. Tools such as sar and iostat typically interpret these and do | 6 | activity. Tools such as sar and iostat typically interpret these and do |
@@ -46,11 +44,12 @@ the above example, the first field of statistics would be 446216. | |||
46 | By contrast, in 2.6 if you look at /sys/block/hda/stat, you'll | 44 | By contrast, in 2.6 if you look at /sys/block/hda/stat, you'll |
47 | find just the eleven fields, beginning with 446216. If you look at | 45 | find just the eleven fields, beginning with 446216. If you look at |
48 | /proc/diskstats, the eleven fields will be preceded by the major and | 46 | /proc/diskstats, the eleven fields will be preceded by the major and |
49 | minor device numbers, and device name. Each of these formats provide | 47 | minor device numbers, and device name. Each of these formats provides |
50 | eleven fields of statistics, each meaning exactly the same things. | 48 | eleven fields of statistics, each meaning exactly the same things. |
51 | All fields except field 9 are cumulative since boot. Field 9 should | 49 | All fields except field 9 are cumulative since boot. Field 9 should |
52 | go to zero as I/Os complete; all others only increase. Yes, these are | 50 | go to zero as I/Os complete; all others only increase (unless they |
53 | 32 bit unsigned numbers, and on a very busy or long-lived system they | 51 | overflow and wrap). Yes, these are (32-bit or 64-bit) unsigned long |
52 | (native word size) numbers, and on a very busy or long-lived system they | ||
54 | may wrap. Applications should be prepared to deal with that; unless | 53 | may wrap. Applications should be prepared to deal with that; unless |
55 | your observations are measured in large numbers of minutes or hours, | 54 | your observations are measured in large numbers of minutes or hours, |
56 | they should not wrap twice before you notice them. | 55 | they should not wrap twice before you notice them. |
@@ -96,11 +95,11 @@ introduced when changes collide, so (for instance) adding up all the | |||
96 | read I/Os issued per partition should equal those made to the disks ... | 95 | read I/Os issued per partition should equal those made to the disks ... |
97 | but due to the lack of locking it may only be very close. | 96 | but due to the lack of locking it may only be very close. |
98 | 97 | ||
99 | In 2.6, there are counters for each cpu, which made the lack of locking | 98 | In 2.6, there are counters for each CPU, which make the lack of locking |
100 | almost a non-issue. When the statistics are read, the per-cpu counters | 99 | almost a non-issue. When the statistics are read, the per-CPU counters |
101 | are summed (possibly overflowing the unsigned 32-bit variable they are | 100 | are summed (possibly overflowing the unsigned long variable they are |
102 | summed to) and the result given to the user. There is no convenient | 101 | summed to) and the result given to the user. There is no convenient |
103 | user interface for accessing the per-cpu counters themselves. | 102 | user interface for accessing the per-CPU counters themselves. |
104 | 103 | ||
105 | Disks vs Partitions | 104 | Disks vs Partitions |
106 | ------------------- | 105 | ------------------- |
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 455768a3eb9e..2bef5705ce24 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c | |||
@@ -371,12 +371,14 @@ void blkiocg_update_io_remove_stats(struct blkio_group *blkg, | |||
371 | } | 371 | } |
372 | EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats); | 372 | EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats); |
373 | 373 | ||
374 | void blkiocg_update_timeslice_used(struct blkio_group *blkg, unsigned long time) | 374 | void blkiocg_update_timeslice_used(struct blkio_group *blkg, unsigned long time, |
375 | unsigned long unaccounted_time) | ||
375 | { | 376 | { |
376 | unsigned long flags; | 377 | unsigned long flags; |
377 | 378 | ||
378 | spin_lock_irqsave(&blkg->stats_lock, flags); | 379 | spin_lock_irqsave(&blkg->stats_lock, flags); |
379 | blkg->stats.time += time; | 380 | blkg->stats.time += time; |
381 | blkg->stats.unaccounted_time += unaccounted_time; | ||
380 | spin_unlock_irqrestore(&blkg->stats_lock, flags); | 382 | spin_unlock_irqrestore(&blkg->stats_lock, flags); |
381 | } | 383 | } |
382 | EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used); | 384 | EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used); |
@@ -604,6 +606,9 @@ static uint64_t blkio_get_stat(struct blkio_group *blkg, | |||
604 | return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, | 606 | return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, |
605 | blkg->stats.sectors, cb, dev); | 607 | blkg->stats.sectors, cb, dev); |
606 | #ifdef CONFIG_DEBUG_BLK_CGROUP | 608 | #ifdef CONFIG_DEBUG_BLK_CGROUP |
609 | if (type == BLKIO_STAT_UNACCOUNTED_TIME) | ||
610 | return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, | ||
611 | blkg->stats.unaccounted_time, cb, dev); | ||
607 | if (type == BLKIO_STAT_AVG_QUEUE_SIZE) { | 612 | if (type == BLKIO_STAT_AVG_QUEUE_SIZE) { |
608 | uint64_t sum = blkg->stats.avg_queue_size_sum; | 613 | uint64_t sum = blkg->stats.avg_queue_size_sum; |
609 | uint64_t samples = blkg->stats.avg_queue_size_samples; | 614 | uint64_t samples = blkg->stats.avg_queue_size_samples; |
@@ -1125,6 +1130,9 @@ static int blkiocg_file_read_map(struct cgroup *cgrp, struct cftype *cft, | |||
1125 | return blkio_read_blkg_stats(blkcg, cft, cb, | 1130 | return blkio_read_blkg_stats(blkcg, cft, cb, |
1126 | BLKIO_STAT_QUEUED, 1); | 1131 | BLKIO_STAT_QUEUED, 1); |
1127 | #ifdef CONFIG_DEBUG_BLK_CGROUP | 1132 | #ifdef CONFIG_DEBUG_BLK_CGROUP |
1133 | case BLKIO_PROP_unaccounted_time: | ||
1134 | return blkio_read_blkg_stats(blkcg, cft, cb, | ||
1135 | BLKIO_STAT_UNACCOUNTED_TIME, 0); | ||
1128 | case BLKIO_PROP_dequeue: | 1136 | case BLKIO_PROP_dequeue: |
1129 | return blkio_read_blkg_stats(blkcg, cft, cb, | 1137 | return blkio_read_blkg_stats(blkcg, cft, cb, |
1130 | BLKIO_STAT_DEQUEUE, 0); | 1138 | BLKIO_STAT_DEQUEUE, 0); |
@@ -1382,6 +1390,12 @@ struct cftype blkio_files[] = { | |||
1382 | BLKIO_PROP_dequeue), | 1390 | BLKIO_PROP_dequeue), |
1383 | .read_map = blkiocg_file_read_map, | 1391 | .read_map = blkiocg_file_read_map, |
1384 | }, | 1392 | }, |
1393 | { | ||
1394 | .name = "unaccounted_time", | ||
1395 | .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP, | ||
1396 | BLKIO_PROP_unaccounted_time), | ||
1397 | .read_map = blkiocg_file_read_map, | ||
1398 | }, | ||
1385 | #endif | 1399 | #endif |
1386 | }; | 1400 | }; |
1387 | 1401 | ||
diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index ea4861bdd549..10919fae2d3a 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h | |||
@@ -49,6 +49,8 @@ enum stat_type { | |||
49 | /* All the single valued stats go below this */ | 49 | /* All the single valued stats go below this */ |
50 | BLKIO_STAT_TIME, | 50 | BLKIO_STAT_TIME, |
51 | BLKIO_STAT_SECTORS, | 51 | BLKIO_STAT_SECTORS, |
52 | /* Time not charged to this cgroup */ | ||
53 | BLKIO_STAT_UNACCOUNTED_TIME, | ||
52 | #ifdef CONFIG_DEBUG_BLK_CGROUP | 54 | #ifdef CONFIG_DEBUG_BLK_CGROUP |
53 | BLKIO_STAT_AVG_QUEUE_SIZE, | 55 | BLKIO_STAT_AVG_QUEUE_SIZE, |
54 | BLKIO_STAT_IDLE_TIME, | 56 | BLKIO_STAT_IDLE_TIME, |
@@ -81,6 +83,7 @@ enum blkcg_file_name_prop { | |||
81 | BLKIO_PROP_io_serviced, | 83 | BLKIO_PROP_io_serviced, |
82 | BLKIO_PROP_time, | 84 | BLKIO_PROP_time, |
83 | BLKIO_PROP_sectors, | 85 | BLKIO_PROP_sectors, |
86 | BLKIO_PROP_unaccounted_time, | ||
84 | BLKIO_PROP_io_service_time, | 87 | BLKIO_PROP_io_service_time, |
85 | BLKIO_PROP_io_wait_time, | 88 | BLKIO_PROP_io_wait_time, |
86 | BLKIO_PROP_io_merged, | 89 | BLKIO_PROP_io_merged, |
@@ -114,6 +117,8 @@ struct blkio_group_stats { | |||
114 | /* total disk time and nr sectors dispatched by this group */ | 117 | /* total disk time and nr sectors dispatched by this group */ |
115 | uint64_t time; | 118 | uint64_t time; |
116 | uint64_t sectors; | 119 | uint64_t sectors; |
120 | /* Time not charged to this cgroup */ | ||
121 | uint64_t unaccounted_time; | ||
117 | uint64_t stat_arr[BLKIO_STAT_QUEUED + 1][BLKIO_STAT_TOTAL]; | 122 | uint64_t stat_arr[BLKIO_STAT_QUEUED + 1][BLKIO_STAT_TOTAL]; |
118 | #ifdef CONFIG_DEBUG_BLK_CGROUP | 123 | #ifdef CONFIG_DEBUG_BLK_CGROUP |
119 | /* Sum of number of IOs queued across all samples */ | 124 | /* Sum of number of IOs queued across all samples */ |
@@ -240,7 +245,7 @@ static inline char *blkg_path(struct blkio_group *blkg) { return NULL; } | |||
240 | 245 | ||
241 | #endif | 246 | #endif |
242 | 247 | ||
243 | #define BLKIO_WEIGHT_MIN 100 | 248 | #define BLKIO_WEIGHT_MIN 10 |
244 | #define BLKIO_WEIGHT_MAX 1000 | 249 | #define BLKIO_WEIGHT_MAX 1000 |
245 | #define BLKIO_WEIGHT_DEFAULT 500 | 250 | #define BLKIO_WEIGHT_DEFAULT 500 |
246 | 251 | ||
@@ -293,7 +298,8 @@ extern int blkiocg_del_blkio_group(struct blkio_group *blkg); | |||
293 | extern struct blkio_group *blkiocg_lookup_group(struct blkio_cgroup *blkcg, | 298 | extern struct blkio_group *blkiocg_lookup_group(struct blkio_cgroup *blkcg, |
294 | void *key); | 299 | void *key); |
295 | void blkiocg_update_timeslice_used(struct blkio_group *blkg, | 300 | void blkiocg_update_timeslice_used(struct blkio_group *blkg, |
296 | unsigned long time); | 301 | unsigned long time, |
302 | unsigned long unaccounted_time); | ||
297 | void blkiocg_update_dispatch_stats(struct blkio_group *blkg, uint64_t bytes, | 303 | void blkiocg_update_dispatch_stats(struct blkio_group *blkg, uint64_t bytes, |
298 | bool direction, bool sync); | 304 | bool direction, bool sync); |
299 | void blkiocg_update_completion_stats(struct blkio_group *blkg, | 305 | void blkiocg_update_completion_stats(struct blkio_group *blkg, |
@@ -319,7 +325,9 @@ blkiocg_del_blkio_group(struct blkio_group *blkg) { return 0; } | |||
319 | static inline struct blkio_group * | 325 | static inline struct blkio_group * |
320 | blkiocg_lookup_group(struct blkio_cgroup *blkcg, void *key) { return NULL; } | 326 | blkiocg_lookup_group(struct blkio_cgroup *blkcg, void *key) { return NULL; } |
321 | static inline void blkiocg_update_timeslice_used(struct blkio_group *blkg, | 327 | static inline void blkiocg_update_timeslice_used(struct blkio_group *blkg, |
322 | unsigned long time) {} | 328 | unsigned long time, |
329 | unsigned long unaccounted_time) | ||
330 | {} | ||
323 | static inline void blkiocg_update_dispatch_stats(struct blkio_group *blkg, | 331 | static inline void blkiocg_update_dispatch_stats(struct blkio_group *blkg, |
324 | uint64_t bytes, bool direction, bool sync) {} | 332 | uint64_t bytes, bool direction, bool sync) {} |
325 | static inline void blkiocg_update_completion_stats(struct blkio_group *blkg, | 333 | static inline void blkiocg_update_completion_stats(struct blkio_group *blkg, |
diff --git a/block/blk-core.c b/block/blk-core.c index a63336d49f30..59b5c00c0126 100644 --- a/block/blk-core.c +++ b/block/blk-core.c | |||
@@ -27,6 +27,7 @@ | |||
27 | #include <linux/writeback.h> | 27 | #include <linux/writeback.h> |
28 | #include <linux/task_io_accounting_ops.h> | 28 | #include <linux/task_io_accounting_ops.h> |
29 | #include <linux/fault-inject.h> | 29 | #include <linux/fault-inject.h> |
30 | #include <linux/list_sort.h> | ||
30 | 31 | ||
31 | #define CREATE_TRACE_POINTS | 32 | #define CREATE_TRACE_POINTS |
32 | #include <trace/events/block.h> | 33 | #include <trace/events/block.h> |
@@ -149,39 +150,29 @@ EXPORT_SYMBOL(blk_rq_init); | |||
149 | static void req_bio_endio(struct request *rq, struct bio *bio, | 150 | static void req_bio_endio(struct request *rq, struct bio *bio, |
150 | unsigned int nbytes, int error) | 151 | unsigned int nbytes, int error) |
151 | { | 152 | { |
152 | struct request_queue *q = rq->q; | 153 | if (error) |
153 | 154 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | |
154 | if (&q->flush_rq != rq) { | 155 | else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) |
155 | if (error) | 156 | error = -EIO; |
156 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | ||
157 | else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) | ||
158 | error = -EIO; | ||
159 | 157 | ||
160 | if (unlikely(nbytes > bio->bi_size)) { | 158 | if (unlikely(nbytes > bio->bi_size)) { |
161 | printk(KERN_ERR "%s: want %u bytes done, %u left\n", | 159 | printk(KERN_ERR "%s: want %u bytes done, %u left\n", |
162 | __func__, nbytes, bio->bi_size); | 160 | __func__, nbytes, bio->bi_size); |
163 | nbytes = bio->bi_size; | 161 | nbytes = bio->bi_size; |
164 | } | 162 | } |
165 | 163 | ||
166 | if (unlikely(rq->cmd_flags & REQ_QUIET)) | 164 | if (unlikely(rq->cmd_flags & REQ_QUIET)) |
167 | set_bit(BIO_QUIET, &bio->bi_flags); | 165 | set_bit(BIO_QUIET, &bio->bi_flags); |
168 | 166 | ||
169 | bio->bi_size -= nbytes; | 167 | bio->bi_size -= nbytes; |
170 | bio->bi_sector += (nbytes >> 9); | 168 | bio->bi_sector += (nbytes >> 9); |
171 | 169 | ||
172 | if (bio_integrity(bio)) | 170 | if (bio_integrity(bio)) |
173 | bio_integrity_advance(bio, nbytes); | 171 | bio_integrity_advance(bio, nbytes); |
174 | 172 | ||
175 | if (bio->bi_size == 0) | 173 | /* don't actually finish bio if it's part of flush sequence */ |
176 | bio_endio(bio, error); | 174 | if (bio->bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ)) |
177 | } else { | 175 | bio_endio(bio, error); |
178 | /* | ||
179 | * Okay, this is the sequenced flush request in | ||
180 | * progress, just record the error; | ||
181 | */ | ||
182 | if (error && !q->flush_err) | ||
183 | q->flush_err = error; | ||
184 | } | ||
185 | } | 176 | } |
186 | 177 | ||
187 | void blk_dump_rq_flags(struct request *rq, char *msg) | 178 | void blk_dump_rq_flags(struct request *rq, char *msg) |
@@ -208,135 +199,43 @@ void blk_dump_rq_flags(struct request *rq, char *msg) | |||
208 | EXPORT_SYMBOL(blk_dump_rq_flags); | 199 | EXPORT_SYMBOL(blk_dump_rq_flags); |
209 | 200 | ||
210 | /* | 201 | /* |
211 | * "plug" the device if there are no outstanding requests: this will | 202 | * Make sure that plugs that were pending when this function was entered, |
212 | * force the transfer to start only after we have put all the requests | 203 | * are now complete and requests pushed to the queue. |
213 | * on the list. | 204 | */ |
214 | * | 205 | static inline void queue_sync_plugs(struct request_queue *q) |
215 | * This is called with interrupts off and no requests on the queue and | ||
216 | * with the queue lock held. | ||
217 | */ | ||
218 | void blk_plug_device(struct request_queue *q) | ||
219 | { | 206 | { |
220 | WARN_ON(!irqs_disabled()); | ||
221 | |||
222 | /* | 207 | /* |
223 | * don't plug a stopped queue, it must be paired with blk_start_queue() | 208 | * If the current process is plugged and has barriers submitted, |
224 | * which will restart the queueing | 209 | * we will livelock if we don't unplug first. |
225 | */ | 210 | */ |
226 | if (blk_queue_stopped(q)) | 211 | blk_flush_plug(current); |
227 | return; | ||
228 | |||
229 | if (!queue_flag_test_and_set(QUEUE_FLAG_PLUGGED, q)) { | ||
230 | mod_timer(&q->unplug_timer, jiffies + q->unplug_delay); | ||
231 | trace_block_plug(q); | ||
232 | } | ||
233 | } | ||
234 | EXPORT_SYMBOL(blk_plug_device); | ||
235 | |||
236 | /** | ||
237 | * blk_plug_device_unlocked - plug a device without queue lock held | ||
238 | * @q: The &struct request_queue to plug | ||
239 | * | ||
240 | * Description: | ||
241 | * Like @blk_plug_device(), but grabs the queue lock and disables | ||
242 | * interrupts. | ||
243 | **/ | ||
244 | void blk_plug_device_unlocked(struct request_queue *q) | ||
245 | { | ||
246 | unsigned long flags; | ||
247 | |||
248 | spin_lock_irqsave(q->queue_lock, flags); | ||
249 | blk_plug_device(q); | ||
250 | spin_unlock_irqrestore(q->queue_lock, flags); | ||
251 | } | ||
252 | EXPORT_SYMBOL(blk_plug_device_unlocked); | ||
253 | |||
254 | /* | ||
255 | * remove the queue from the plugged list, if present. called with | ||
256 | * queue lock held and interrupts disabled. | ||
257 | */ | ||
258 | int blk_remove_plug(struct request_queue *q) | ||
259 | { | ||
260 | WARN_ON(!irqs_disabled()); | ||
261 | |||
262 | if (!queue_flag_test_and_clear(QUEUE_FLAG_PLUGGED, q)) | ||
263 | return 0; | ||
264 | |||
265 | del_timer(&q->unplug_timer); | ||
266 | return 1; | ||
267 | } | 212 | } |
268 | EXPORT_SYMBOL(blk_remove_plug); | ||
269 | 213 | ||
270 | /* | 214 | static void blk_delay_work(struct work_struct *work) |
271 | * remove the plug and let it rip.. | ||
272 | */ | ||
273 | void __generic_unplug_device(struct request_queue *q) | ||
274 | { | 215 | { |
275 | if (unlikely(blk_queue_stopped(q))) | 216 | struct request_queue *q; |
276 | return; | ||
277 | if (!blk_remove_plug(q) && !blk_queue_nonrot(q)) | ||
278 | return; | ||
279 | 217 | ||
280 | q->request_fn(q); | 218 | q = container_of(work, struct request_queue, delay_work.work); |
219 | spin_lock_irq(q->queue_lock); | ||
220 | __blk_run_queue(q, false); | ||
221 | spin_unlock_irq(q->queue_lock); | ||
281 | } | 222 | } |
282 | 223 | ||
283 | /** | 224 | /** |
284 | * generic_unplug_device - fire a request queue | 225 | * blk_delay_queue - restart queueing after defined interval |
285 | * @q: The &struct request_queue in question | 226 | * @q: The &struct request_queue in question |
227 | * @msecs: Delay in msecs | ||
286 | * | 228 | * |
287 | * Description: | 229 | * Description: |
288 | * Linux uses plugging to build bigger requests queues before letting | 230 | * Sometimes queueing needs to be postponed for a little while, to allow |
289 | * the device have at them. If a queue is plugged, the I/O scheduler | 231 | * resources to come back. This function will make sure that queueing is |
290 | * is still adding and merging requests on the queue. Once the queue | 232 | * restarted around the specified time. |
291 | * gets unplugged, the request_fn defined for the queue is invoked and | 233 | */ |
292 | * transfers started. | 234 | void blk_delay_queue(struct request_queue *q, unsigned long msecs) |
293 | **/ | ||
294 | void generic_unplug_device(struct request_queue *q) | ||
295 | { | ||
296 | if (blk_queue_plugged(q)) { | ||
297 | spin_lock_irq(q->queue_lock); | ||
298 | __generic_unplug_device(q); | ||
299 | spin_unlock_irq(q->queue_lock); | ||
300 | } | ||
301 | } | ||
302 | EXPORT_SYMBOL(generic_unplug_device); | ||
303 | |||
304 | static void blk_backing_dev_unplug(struct backing_dev_info *bdi, | ||
305 | struct page *page) | ||
306 | { | ||
307 | struct request_queue *q = bdi->unplug_io_data; | ||
308 | |||
309 | blk_unplug(q); | ||
310 | } | ||
311 | |||
312 | void blk_unplug_work(struct work_struct *work) | ||
313 | { | ||
314 | struct request_queue *q = | ||
315 | container_of(work, struct request_queue, unplug_work); | ||
316 | |||
317 | trace_block_unplug_io(q); | ||
318 | q->unplug_fn(q); | ||
319 | } | ||
320 | |||
321 | void blk_unplug_timeout(unsigned long data) | ||
322 | { | ||
323 | struct request_queue *q = (struct request_queue *)data; | ||
324 | |||
325 | trace_block_unplug_timer(q); | ||
326 | kblockd_schedule_work(q, &q->unplug_work); | ||
327 | } | ||
328 | |||
329 | void blk_unplug(struct request_queue *q) | ||
330 | { | 235 | { |
331 | /* | 236 | schedule_delayed_work(&q->delay_work, msecs_to_jiffies(msecs)); |
332 | * devices don't necessarily have an ->unplug_fn defined | ||
333 | */ | ||
334 | if (q->unplug_fn) { | ||
335 | trace_block_unplug_io(q); | ||
336 | q->unplug_fn(q); | ||
337 | } | ||
338 | } | 237 | } |
339 | EXPORT_SYMBOL(blk_unplug); | 238 | EXPORT_SYMBOL(blk_delay_queue); |
340 | 239 | ||
341 | /** | 240 | /** |
342 | * blk_start_queue - restart a previously stopped queue | 241 | * blk_start_queue - restart a previously stopped queue |
@@ -372,7 +271,7 @@ EXPORT_SYMBOL(blk_start_queue); | |||
372 | **/ | 271 | **/ |
373 | void blk_stop_queue(struct request_queue *q) | 272 | void blk_stop_queue(struct request_queue *q) |
374 | { | 273 | { |
375 | blk_remove_plug(q); | 274 | cancel_delayed_work(&q->delay_work); |
376 | queue_flag_set(QUEUE_FLAG_STOPPED, q); | 275 | queue_flag_set(QUEUE_FLAG_STOPPED, q); |
377 | } | 276 | } |
378 | EXPORT_SYMBOL(blk_stop_queue); | 277 | EXPORT_SYMBOL(blk_stop_queue); |
@@ -390,13 +289,16 @@ EXPORT_SYMBOL(blk_stop_queue); | |||
390 | * that its ->make_request_fn will not re-add plugging prior to calling | 289 | * that its ->make_request_fn will not re-add plugging prior to calling |
391 | * this function. | 290 | * this function. |
392 | * | 291 | * |
292 | * This function does not cancel any asynchronous activity arising | ||
293 | * out of elevator or throttling code. That would require elevaotor_exit() | ||
294 | * and blk_throtl_exit() to be called with queue lock initialized. | ||
295 | * | ||
393 | */ | 296 | */ |
394 | void blk_sync_queue(struct request_queue *q) | 297 | void blk_sync_queue(struct request_queue *q) |
395 | { | 298 | { |
396 | del_timer_sync(&q->unplug_timer); | ||
397 | del_timer_sync(&q->timeout); | 299 | del_timer_sync(&q->timeout); |
398 | cancel_work_sync(&q->unplug_work); | 300 | cancel_delayed_work_sync(&q->delay_work); |
399 | throtl_shutdown_timer_wq(q); | 301 | queue_sync_plugs(q); |
400 | } | 302 | } |
401 | EXPORT_SYMBOL(blk_sync_queue); | 303 | EXPORT_SYMBOL(blk_sync_queue); |
402 | 304 | ||
@@ -412,14 +314,9 @@ EXPORT_SYMBOL(blk_sync_queue); | |||
412 | */ | 314 | */ |
413 | void __blk_run_queue(struct request_queue *q, bool force_kblockd) | 315 | void __blk_run_queue(struct request_queue *q, bool force_kblockd) |
414 | { | 316 | { |
415 | blk_remove_plug(q); | ||
416 | |||
417 | if (unlikely(blk_queue_stopped(q))) | 317 | if (unlikely(blk_queue_stopped(q))) |
418 | return; | 318 | return; |
419 | 319 | ||
420 | if (elv_queue_empty(q)) | ||
421 | return; | ||
422 | |||
423 | /* | 320 | /* |
424 | * Only recurse once to avoid overrunning the stack, let the unplug | 321 | * Only recurse once to avoid overrunning the stack, let the unplug |
425 | * handling reinvoke the handler shortly if we already got there. | 322 | * handling reinvoke the handler shortly if we already got there. |
@@ -427,10 +324,8 @@ void __blk_run_queue(struct request_queue *q, bool force_kblockd) | |||
427 | if (!force_kblockd && !queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) { | 324 | if (!force_kblockd && !queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) { |
428 | q->request_fn(q); | 325 | q->request_fn(q); |
429 | queue_flag_clear(QUEUE_FLAG_REENTER, q); | 326 | queue_flag_clear(QUEUE_FLAG_REENTER, q); |
430 | } else { | 327 | } else |
431 | queue_flag_set(QUEUE_FLAG_PLUGGED, q); | 328 | queue_delayed_work(kblockd_workqueue, &q->delay_work, 0); |
432 | kblockd_schedule_work(q, &q->unplug_work); | ||
433 | } | ||
434 | } | 329 | } |
435 | EXPORT_SYMBOL(__blk_run_queue); | 330 | EXPORT_SYMBOL(__blk_run_queue); |
436 | 331 | ||
@@ -457,6 +352,11 @@ void blk_put_queue(struct request_queue *q) | |||
457 | kobject_put(&q->kobj); | 352 | kobject_put(&q->kobj); |
458 | } | 353 | } |
459 | 354 | ||
355 | /* | ||
356 | * Note: If a driver supplied the queue lock, it should not zap that lock | ||
357 | * unexpectedly as some queue cleanup components like elevator_exit() and | ||
358 | * blk_throtl_exit() need queue lock. | ||
359 | */ | ||
460 | void blk_cleanup_queue(struct request_queue *q) | 360 | void blk_cleanup_queue(struct request_queue *q) |
461 | { | 361 | { |
462 | /* | 362 | /* |
@@ -475,6 +375,8 @@ void blk_cleanup_queue(struct request_queue *q) | |||
475 | if (q->elevator) | 375 | if (q->elevator) |
476 | elevator_exit(q->elevator); | 376 | elevator_exit(q->elevator); |
477 | 377 | ||
378 | blk_throtl_exit(q); | ||
379 | |||
478 | blk_put_queue(q); | 380 | blk_put_queue(q); |
479 | } | 381 | } |
480 | EXPORT_SYMBOL(blk_cleanup_queue); | 382 | EXPORT_SYMBOL(blk_cleanup_queue); |
@@ -517,8 +419,6 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) | |||
517 | if (!q) | 419 | if (!q) |
518 | return NULL; | 420 | return NULL; |
519 | 421 | ||
520 | q->backing_dev_info.unplug_io_fn = blk_backing_dev_unplug; | ||
521 | q->backing_dev_info.unplug_io_data = q; | ||
522 | q->backing_dev_info.ra_pages = | 422 | q->backing_dev_info.ra_pages = |
523 | (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; | 423 | (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; |
524 | q->backing_dev_info.state = 0; | 424 | q->backing_dev_info.state = 0; |
@@ -538,17 +438,24 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) | |||
538 | 438 | ||
539 | setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, | 439 | setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, |
540 | laptop_mode_timer_fn, (unsigned long) q); | 440 | laptop_mode_timer_fn, (unsigned long) q); |
541 | init_timer(&q->unplug_timer); | ||
542 | setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q); | 441 | setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q); |
543 | INIT_LIST_HEAD(&q->timeout_list); | 442 | INIT_LIST_HEAD(&q->timeout_list); |
544 | INIT_LIST_HEAD(&q->pending_flushes); | 443 | INIT_LIST_HEAD(&q->flush_queue[0]); |
545 | INIT_WORK(&q->unplug_work, blk_unplug_work); | 444 | INIT_LIST_HEAD(&q->flush_queue[1]); |
445 | INIT_LIST_HEAD(&q->flush_data_in_flight); | ||
446 | INIT_DELAYED_WORK(&q->delay_work, blk_delay_work); | ||
546 | 447 | ||
547 | kobject_init(&q->kobj, &blk_queue_ktype); | 448 | kobject_init(&q->kobj, &blk_queue_ktype); |
548 | 449 | ||
549 | mutex_init(&q->sysfs_lock); | 450 | mutex_init(&q->sysfs_lock); |
550 | spin_lock_init(&q->__queue_lock); | 451 | spin_lock_init(&q->__queue_lock); |
551 | 452 | ||
453 | /* | ||
454 | * By default initialize queue_lock to internal lock and driver can | ||
455 | * override it later if need be. | ||
456 | */ | ||
457 | q->queue_lock = &q->__queue_lock; | ||
458 | |||
552 | return q; | 459 | return q; |
553 | } | 460 | } |
554 | EXPORT_SYMBOL(blk_alloc_queue_node); | 461 | EXPORT_SYMBOL(blk_alloc_queue_node); |
@@ -631,9 +538,11 @@ blk_init_allocated_queue_node(struct request_queue *q, request_fn_proc *rfn, | |||
631 | q->request_fn = rfn; | 538 | q->request_fn = rfn; |
632 | q->prep_rq_fn = NULL; | 539 | q->prep_rq_fn = NULL; |
633 | q->unprep_rq_fn = NULL; | 540 | q->unprep_rq_fn = NULL; |
634 | q->unplug_fn = generic_unplug_device; | ||
635 | q->queue_flags = QUEUE_FLAG_DEFAULT; | 541 | q->queue_flags = QUEUE_FLAG_DEFAULT; |
636 | q->queue_lock = lock; | 542 | |
543 | /* Override internal queue lock with supplied lock pointer */ | ||
544 | if (lock) | ||
545 | q->queue_lock = lock; | ||
637 | 546 | ||
638 | /* | 547 | /* |
639 | * This also sets hw/phys segments, boundary and size | 548 | * This also sets hw/phys segments, boundary and size |
@@ -666,6 +575,8 @@ int blk_get_queue(struct request_queue *q) | |||
666 | 575 | ||
667 | static inline void blk_free_request(struct request_queue *q, struct request *rq) | 576 | static inline void blk_free_request(struct request_queue *q, struct request *rq) |
668 | { | 577 | { |
578 | BUG_ON(rq->cmd_flags & REQ_ON_PLUG); | ||
579 | |||
669 | if (rq->cmd_flags & REQ_ELVPRIV) | 580 | if (rq->cmd_flags & REQ_ELVPRIV) |
670 | elv_put_request(q, rq); | 581 | elv_put_request(q, rq); |
671 | mempool_free(rq, q->rq.rq_pool); | 582 | mempool_free(rq, q->rq.rq_pool); |
@@ -762,6 +673,25 @@ static void freed_request(struct request_queue *q, int sync, int priv) | |||
762 | } | 673 | } |
763 | 674 | ||
764 | /* | 675 | /* |
676 | * Determine if elevator data should be initialized when allocating the | ||
677 | * request associated with @bio. | ||
678 | */ | ||
679 | static bool blk_rq_should_init_elevator(struct bio *bio) | ||
680 | { | ||
681 | if (!bio) | ||
682 | return true; | ||
683 | |||
684 | /* | ||
685 | * Flush requests do not use the elevator so skip initialization. | ||
686 | * This allows a request to share the flush and elevator data. | ||
687 | */ | ||
688 | if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) | ||
689 | return false; | ||
690 | |||
691 | return true; | ||
692 | } | ||
693 | |||
694 | /* | ||
765 | * Get a free request, queue_lock must be held. | 695 | * Get a free request, queue_lock must be held. |
766 | * Returns NULL on failure, with queue_lock held. | 696 | * Returns NULL on failure, with queue_lock held. |
767 | * Returns !NULL on success, with queue_lock *not held*. | 697 | * Returns !NULL on success, with queue_lock *not held*. |
@@ -773,7 +703,7 @@ static struct request *get_request(struct request_queue *q, int rw_flags, | |||
773 | struct request_list *rl = &q->rq; | 703 | struct request_list *rl = &q->rq; |
774 | struct io_context *ioc = NULL; | 704 | struct io_context *ioc = NULL; |
775 | const bool is_sync = rw_is_sync(rw_flags) != 0; | 705 | const bool is_sync = rw_is_sync(rw_flags) != 0; |
776 | int may_queue, priv; | 706 | int may_queue, priv = 0; |
777 | 707 | ||
778 | may_queue = elv_may_queue(q, rw_flags); | 708 | may_queue = elv_may_queue(q, rw_flags); |
779 | if (may_queue == ELV_MQUEUE_NO) | 709 | if (may_queue == ELV_MQUEUE_NO) |
@@ -817,9 +747,11 @@ static struct request *get_request(struct request_queue *q, int rw_flags, | |||
817 | rl->count[is_sync]++; | 747 | rl->count[is_sync]++; |
818 | rl->starved[is_sync] = 0; | 748 | rl->starved[is_sync] = 0; |
819 | 749 | ||
820 | priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags); | 750 | if (blk_rq_should_init_elevator(bio)) { |
821 | if (priv) | 751 | priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags); |
822 | rl->elvpriv++; | 752 | if (priv) |
753 | rl->elvpriv++; | ||
754 | } | ||
823 | 755 | ||
824 | if (blk_queue_io_stat(q)) | 756 | if (blk_queue_io_stat(q)) |
825 | rw_flags |= REQ_IO_STAT; | 757 | rw_flags |= REQ_IO_STAT; |
@@ -866,8 +798,8 @@ out: | |||
866 | } | 798 | } |
867 | 799 | ||
868 | /* | 800 | /* |
869 | * No available requests for this queue, unplug the device and wait for some | 801 | * No available requests for this queue, wait for some requests to become |
870 | * requests to become available. | 802 | * available. |
871 | * | 803 | * |
872 | * Called with q->queue_lock held, and returns with it unlocked. | 804 | * Called with q->queue_lock held, and returns with it unlocked. |
873 | */ | 805 | */ |
@@ -888,7 +820,6 @@ static struct request *get_request_wait(struct request_queue *q, int rw_flags, | |||
888 | 820 | ||
889 | trace_block_sleeprq(q, bio, rw_flags & 1); | 821 | trace_block_sleeprq(q, bio, rw_flags & 1); |
890 | 822 | ||
891 | __generic_unplug_device(q); | ||
892 | spin_unlock_irq(q->queue_lock); | 823 | spin_unlock_irq(q->queue_lock); |
893 | io_schedule(); | 824 | io_schedule(); |
894 | 825 | ||
@@ -1010,6 +941,13 @@ void blk_requeue_request(struct request_queue *q, struct request *rq) | |||
1010 | } | 941 | } |
1011 | EXPORT_SYMBOL(blk_requeue_request); | 942 | EXPORT_SYMBOL(blk_requeue_request); |
1012 | 943 | ||
944 | static void add_acct_request(struct request_queue *q, struct request *rq, | ||
945 | int where) | ||
946 | { | ||
947 | drive_stat_acct(rq, 1); | ||
948 | __elv_add_request(q, rq, where); | ||
949 | } | ||
950 | |||
1013 | /** | 951 | /** |
1014 | * blk_insert_request - insert a special request into a request queue | 952 | * blk_insert_request - insert a special request into a request queue |
1015 | * @q: request queue where request should be inserted | 953 | * @q: request queue where request should be inserted |
@@ -1052,8 +990,7 @@ void blk_insert_request(struct request_queue *q, struct request *rq, | |||
1052 | if (blk_rq_tagged(rq)) | 990 | if (blk_rq_tagged(rq)) |
1053 | blk_queue_end_tag(q, rq); | 991 | blk_queue_end_tag(q, rq); |
1054 | 992 | ||
1055 | drive_stat_acct(rq, 1); | 993 | add_acct_request(q, rq, where); |
1056 | __elv_add_request(q, rq, where, 0); | ||
1057 | __blk_run_queue(q, false); | 994 | __blk_run_queue(q, false); |
1058 | spin_unlock_irqrestore(q->queue_lock, flags); | 995 | spin_unlock_irqrestore(q->queue_lock, flags); |
1059 | } | 996 | } |
@@ -1174,6 +1111,113 @@ void blk_add_request_payload(struct request *rq, struct page *page, | |||
1174 | } | 1111 | } |
1175 | EXPORT_SYMBOL_GPL(blk_add_request_payload); | 1112 | EXPORT_SYMBOL_GPL(blk_add_request_payload); |
1176 | 1113 | ||
1114 | static bool bio_attempt_back_merge(struct request_queue *q, struct request *req, | ||
1115 | struct bio *bio) | ||
1116 | { | ||
1117 | const int ff = bio->bi_rw & REQ_FAILFAST_MASK; | ||
1118 | |||
1119 | /* | ||
1120 | * Debug stuff, kill later | ||
1121 | */ | ||
1122 | if (!rq_mergeable(req)) { | ||
1123 | blk_dump_rq_flags(req, "back"); | ||
1124 | return false; | ||
1125 | } | ||
1126 | |||
1127 | if (!ll_back_merge_fn(q, req, bio)) | ||
1128 | return false; | ||
1129 | |||
1130 | trace_block_bio_backmerge(q, bio); | ||
1131 | |||
1132 | if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) | ||
1133 | blk_rq_set_mixed_merge(req); | ||
1134 | |||
1135 | req->biotail->bi_next = bio; | ||
1136 | req->biotail = bio; | ||
1137 | req->__data_len += bio->bi_size; | ||
1138 | req->ioprio = ioprio_best(req->ioprio, bio_prio(bio)); | ||
1139 | |||
1140 | drive_stat_acct(req, 0); | ||
1141 | return true; | ||
1142 | } | ||
1143 | |||
1144 | static bool bio_attempt_front_merge(struct request_queue *q, | ||
1145 | struct request *req, struct bio *bio) | ||
1146 | { | ||
1147 | const int ff = bio->bi_rw & REQ_FAILFAST_MASK; | ||
1148 | sector_t sector; | ||
1149 | |||
1150 | /* | ||
1151 | * Debug stuff, kill later | ||
1152 | */ | ||
1153 | if (!rq_mergeable(req)) { | ||
1154 | blk_dump_rq_flags(req, "front"); | ||
1155 | return false; | ||
1156 | } | ||
1157 | |||
1158 | if (!ll_front_merge_fn(q, req, bio)) | ||
1159 | return false; | ||
1160 | |||
1161 | trace_block_bio_frontmerge(q, bio); | ||
1162 | |||
1163 | if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) | ||
1164 | blk_rq_set_mixed_merge(req); | ||
1165 | |||
1166 | sector = bio->bi_sector; | ||
1167 | |||
1168 | bio->bi_next = req->bio; | ||
1169 | req->bio = bio; | ||
1170 | |||
1171 | /* | ||
1172 | * may not be valid. if the low level driver said | ||
1173 | * it didn't need a bounce buffer then it better | ||
1174 | * not touch req->buffer either... | ||
1175 | */ | ||
1176 | req->buffer = bio_data(bio); | ||
1177 | req->__sector = bio->bi_sector; | ||
1178 | req->__data_len += bio->bi_size; | ||
1179 | req->ioprio = ioprio_best(req->ioprio, bio_prio(bio)); | ||
1180 | |||
1181 | drive_stat_acct(req, 0); | ||
1182 | return true; | ||
1183 | } | ||
1184 | |||
1185 | /* | ||
1186 | * Attempts to merge with the plugged list in the current process. Returns | ||
1187 | * true if merge was succesful, otherwise false. | ||
1188 | */ | ||
1189 | static bool attempt_plug_merge(struct task_struct *tsk, struct request_queue *q, | ||
1190 | struct bio *bio) | ||
1191 | { | ||
1192 | struct blk_plug *plug; | ||
1193 | struct request *rq; | ||
1194 | bool ret = false; | ||
1195 | |||
1196 | plug = tsk->plug; | ||
1197 | if (!plug) | ||
1198 | goto out; | ||
1199 | |||
1200 | list_for_each_entry_reverse(rq, &plug->list, queuelist) { | ||
1201 | int el_ret; | ||
1202 | |||
1203 | if (rq->q != q) | ||
1204 | continue; | ||
1205 | |||
1206 | el_ret = elv_try_merge(rq, bio); | ||
1207 | if (el_ret == ELEVATOR_BACK_MERGE) { | ||
1208 | ret = bio_attempt_back_merge(q, rq, bio); | ||
1209 | if (ret) | ||
1210 | break; | ||
1211 | } else if (el_ret == ELEVATOR_FRONT_MERGE) { | ||
1212 | ret = bio_attempt_front_merge(q, rq, bio); | ||
1213 | if (ret) | ||
1214 | break; | ||
1215 | } | ||
1216 | } | ||
1217 | out: | ||
1218 | return ret; | ||
1219 | } | ||
1220 | |||
1177 | void init_request_from_bio(struct request *req, struct bio *bio) | 1221 | void init_request_from_bio(struct request *req, struct bio *bio) |
1178 | { | 1222 | { |
1179 | req->cpu = bio->bi_comp_cpu; | 1223 | req->cpu = bio->bi_comp_cpu; |
@@ -1189,26 +1233,12 @@ void init_request_from_bio(struct request *req, struct bio *bio) | |||
1189 | blk_rq_bio_prep(req->q, req, bio); | 1233 | blk_rq_bio_prep(req->q, req, bio); |
1190 | } | 1234 | } |
1191 | 1235 | ||
1192 | /* | ||
1193 | * Only disabling plugging for non-rotational devices if it does tagging | ||
1194 | * as well, otherwise we do need the proper merging | ||
1195 | */ | ||
1196 | static inline bool queue_should_plug(struct request_queue *q) | ||
1197 | { | ||
1198 | return !(blk_queue_nonrot(q) && blk_queue_tagged(q)); | ||
1199 | } | ||
1200 | |||
1201 | static int __make_request(struct request_queue *q, struct bio *bio) | 1236 | static int __make_request(struct request_queue *q, struct bio *bio) |
1202 | { | 1237 | { |
1203 | struct request *req; | ||
1204 | int el_ret; | ||
1205 | unsigned int bytes = bio->bi_size; | ||
1206 | const unsigned short prio = bio_prio(bio); | ||
1207 | const bool sync = !!(bio->bi_rw & REQ_SYNC); | 1238 | const bool sync = !!(bio->bi_rw & REQ_SYNC); |
1208 | const bool unplug = !!(bio->bi_rw & REQ_UNPLUG); | 1239 | struct blk_plug *plug; |
1209 | const unsigned long ff = bio->bi_rw & REQ_FAILFAST_MASK; | 1240 | int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT; |
1210 | int where = ELEVATOR_INSERT_SORT; | 1241 | struct request *req; |
1211 | int rw_flags; | ||
1212 | 1242 | ||
1213 | /* | 1243 | /* |
1214 | * low level driver can indicate that it wants pages above a | 1244 | * low level driver can indicate that it wants pages above a |
@@ -1217,78 +1247,36 @@ static int __make_request(struct request_queue *q, struct bio *bio) | |||
1217 | */ | 1247 | */ |
1218 | blk_queue_bounce(q, &bio); | 1248 | blk_queue_bounce(q, &bio); |
1219 | 1249 | ||
1220 | spin_lock_irq(q->queue_lock); | ||
1221 | |||
1222 | if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) { | 1250 | if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) { |
1223 | where = ELEVATOR_INSERT_FRONT; | 1251 | spin_lock_irq(q->queue_lock); |
1252 | where = ELEVATOR_INSERT_FLUSH; | ||
1224 | goto get_rq; | 1253 | goto get_rq; |
1225 | } | 1254 | } |
1226 | 1255 | ||
1227 | if (elv_queue_empty(q)) | 1256 | /* |
1228 | goto get_rq; | 1257 | * Check if we can merge with the plugged list before grabbing |
1229 | 1258 | * any locks. | |
1230 | el_ret = elv_merge(q, &req, bio); | 1259 | */ |
1231 | switch (el_ret) { | 1260 | if (attempt_plug_merge(current, q, bio)) |
1232 | case ELEVATOR_BACK_MERGE: | ||
1233 | BUG_ON(!rq_mergeable(req)); | ||
1234 | |||
1235 | if (!ll_back_merge_fn(q, req, bio)) | ||
1236 | break; | ||
1237 | |||
1238 | trace_block_bio_backmerge(q, bio); | ||
1239 | |||
1240 | if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) | ||
1241 | blk_rq_set_mixed_merge(req); | ||
1242 | |||
1243 | req->biotail->bi_next = bio; | ||
1244 | req->biotail = bio; | ||
1245 | req->__data_len += bytes; | ||
1246 | req->ioprio = ioprio_best(req->ioprio, prio); | ||
1247 | if (!blk_rq_cpu_valid(req)) | ||
1248 | req->cpu = bio->bi_comp_cpu; | ||
1249 | drive_stat_acct(req, 0); | ||
1250 | elv_bio_merged(q, req, bio); | ||
1251 | if (!attempt_back_merge(q, req)) | ||
1252 | elv_merged_request(q, req, el_ret); | ||
1253 | goto out; | 1261 | goto out; |
1254 | 1262 | ||
1255 | case ELEVATOR_FRONT_MERGE: | 1263 | spin_lock_irq(q->queue_lock); |
1256 | BUG_ON(!rq_mergeable(req)); | ||
1257 | |||
1258 | if (!ll_front_merge_fn(q, req, bio)) | ||
1259 | break; | ||
1260 | |||
1261 | trace_block_bio_frontmerge(q, bio); | ||
1262 | 1264 | ||
1263 | if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) { | 1265 | el_ret = elv_merge(q, &req, bio); |
1264 | blk_rq_set_mixed_merge(req); | 1266 | if (el_ret == ELEVATOR_BACK_MERGE) { |
1265 | req->cmd_flags &= ~REQ_FAILFAST_MASK; | 1267 | BUG_ON(req->cmd_flags & REQ_ON_PLUG); |
1266 | req->cmd_flags |= ff; | 1268 | if (bio_attempt_back_merge(q, req, bio)) { |
1269 | if (!attempt_back_merge(q, req)) | ||
1270 | elv_merged_request(q, req, el_ret); | ||
1271 | goto out_unlock; | ||
1272 | } | ||
1273 | } else if (el_ret == ELEVATOR_FRONT_MERGE) { | ||
1274 | BUG_ON(req->cmd_flags & REQ_ON_PLUG); | ||
1275 | if (bio_attempt_front_merge(q, req, bio)) { | ||
1276 | if (!attempt_front_merge(q, req)) | ||
1277 | elv_merged_request(q, req, el_ret); | ||
1278 | goto out_unlock; | ||
1267 | } | 1279 | } |
1268 | |||
1269 | bio->bi_next = req->bio; | ||
1270 | req->bio = bio; | ||
1271 | |||
1272 | /* | ||
1273 | * may not be valid. if the low level driver said | ||
1274 | * it didn't need a bounce buffer then it better | ||
1275 | * not touch req->buffer either... | ||
1276 | */ | ||
1277 | req->buffer = bio_data(bio); | ||
1278 | req->__sector = bio->bi_sector; | ||
1279 | req->__data_len += bytes; | ||
1280 | req->ioprio = ioprio_best(req->ioprio, prio); | ||
1281 | if (!blk_rq_cpu_valid(req)) | ||
1282 | req->cpu = bio->bi_comp_cpu; | ||
1283 | drive_stat_acct(req, 0); | ||
1284 | elv_bio_merged(q, req, bio); | ||
1285 | if (!attempt_front_merge(q, req)) | ||
1286 | elv_merged_request(q, req, el_ret); | ||
1287 | goto out; | ||
1288 | |||
1289 | /* ELV_NO_MERGE: elevator says don't/can't merge. */ | ||
1290 | default: | ||
1291 | ; | ||
1292 | } | 1280 | } |
1293 | 1281 | ||
1294 | get_rq: | 1282 | get_rq: |
@@ -1315,20 +1303,35 @@ get_rq: | |||
1315 | */ | 1303 | */ |
1316 | init_request_from_bio(req, bio); | 1304 | init_request_from_bio(req, bio); |
1317 | 1305 | ||
1318 | spin_lock_irq(q->queue_lock); | ||
1319 | if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags) || | 1306 | if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags) || |
1320 | bio_flagged(bio, BIO_CPU_AFFINE)) | 1307 | bio_flagged(bio, BIO_CPU_AFFINE)) { |
1321 | req->cpu = blk_cpu_to_group(smp_processor_id()); | 1308 | req->cpu = blk_cpu_to_group(get_cpu()); |
1322 | if (queue_should_plug(q) && elv_queue_empty(q)) | 1309 | put_cpu(); |
1323 | blk_plug_device(q); | 1310 | } |
1324 | 1311 | ||
1325 | /* insert the request into the elevator */ | 1312 | plug = current->plug; |
1326 | drive_stat_acct(req, 1); | 1313 | if (plug) { |
1327 | __elv_add_request(q, req, where, 0); | 1314 | if (!plug->should_sort && !list_empty(&plug->list)) { |
1315 | struct request *__rq; | ||
1316 | |||
1317 | __rq = list_entry_rq(plug->list.prev); | ||
1318 | if (__rq->q != q) | ||
1319 | plug->should_sort = 1; | ||
1320 | } | ||
1321 | /* | ||
1322 | * Debug flag, kill later | ||
1323 | */ | ||
1324 | req->cmd_flags |= REQ_ON_PLUG; | ||
1325 | list_add_tail(&req->queuelist, &plug->list); | ||
1326 | drive_stat_acct(req, 1); | ||
1327 | } else { | ||
1328 | spin_lock_irq(q->queue_lock); | ||
1329 | add_acct_request(q, req, where); | ||
1330 | __blk_run_queue(q, false); | ||
1331 | out_unlock: | ||
1332 | spin_unlock_irq(q->queue_lock); | ||
1333 | } | ||
1328 | out: | 1334 | out: |
1329 | if (unplug || !queue_should_plug(q)) | ||
1330 | __generic_unplug_device(q); | ||
1331 | spin_unlock_irq(q->queue_lock); | ||
1332 | return 0; | 1335 | return 0; |
1333 | } | 1336 | } |
1334 | 1337 | ||
@@ -1731,9 +1734,7 @@ int blk_insert_cloned_request(struct request_queue *q, struct request *rq) | |||
1731 | */ | 1734 | */ |
1732 | BUG_ON(blk_queued_rq(rq)); | 1735 | BUG_ON(blk_queued_rq(rq)); |
1733 | 1736 | ||
1734 | drive_stat_acct(rq, 1); | 1737 | add_acct_request(q, rq, ELEVATOR_INSERT_BACK); |
1735 | __elv_add_request(q, rq, ELEVATOR_INSERT_BACK, 0); | ||
1736 | |||
1737 | spin_unlock_irqrestore(q->queue_lock, flags); | 1738 | spin_unlock_irqrestore(q->queue_lock, flags); |
1738 | 1739 | ||
1739 | return 0; | 1740 | return 0; |
@@ -1805,7 +1806,7 @@ static void blk_account_io_done(struct request *req) | |||
1805 | * normal IO on queueing nor completion. Accounting the | 1806 | * normal IO on queueing nor completion. Accounting the |
1806 | * containing request is enough. | 1807 | * containing request is enough. |
1807 | */ | 1808 | */ |
1808 | if (blk_do_io_stat(req) && req != &req->q->flush_rq) { | 1809 | if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) { |
1809 | unsigned long duration = jiffies - req->start_time; | 1810 | unsigned long duration = jiffies - req->start_time; |
1810 | const int rw = rq_data_dir(req); | 1811 | const int rw = rq_data_dir(req); |
1811 | struct hd_struct *part; | 1812 | struct hd_struct *part; |
@@ -2628,6 +2629,113 @@ int kblockd_schedule_work(struct request_queue *q, struct work_struct *work) | |||
2628 | } | 2629 | } |
2629 | EXPORT_SYMBOL(kblockd_schedule_work); | 2630 | EXPORT_SYMBOL(kblockd_schedule_work); |
2630 | 2631 | ||
2632 | int kblockd_schedule_delayed_work(struct request_queue *q, | ||
2633 | struct delayed_work *dwork, unsigned long delay) | ||
2634 | { | ||
2635 | return queue_delayed_work(kblockd_workqueue, dwork, delay); | ||
2636 | } | ||
2637 | EXPORT_SYMBOL(kblockd_schedule_delayed_work); | ||
2638 | |||
2639 | #define PLUG_MAGIC 0x91827364 | ||
2640 | |||
2641 | void blk_start_plug(struct blk_plug *plug) | ||
2642 | { | ||
2643 | struct task_struct *tsk = current; | ||
2644 | |||
2645 | plug->magic = PLUG_MAGIC; | ||
2646 | INIT_LIST_HEAD(&plug->list); | ||
2647 | plug->should_sort = 0; | ||
2648 | |||
2649 | /* | ||
2650 | * If this is a nested plug, don't actually assign it. It will be | ||
2651 | * flushed on its own. | ||
2652 | */ | ||
2653 | if (!tsk->plug) { | ||
2654 | /* | ||
2655 | * Store ordering should not be needed here, since a potential | ||
2656 | * preempt will imply a full memory barrier | ||
2657 | */ | ||
2658 | tsk->plug = plug; | ||
2659 | } | ||
2660 | } | ||
2661 | EXPORT_SYMBOL(blk_start_plug); | ||
2662 | |||
2663 | static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b) | ||
2664 | { | ||
2665 | struct request *rqa = container_of(a, struct request, queuelist); | ||
2666 | struct request *rqb = container_of(b, struct request, queuelist); | ||
2667 | |||
2668 | return !(rqa->q == rqb->q); | ||
2669 | } | ||
2670 | |||
2671 | static void flush_plug_list(struct blk_plug *plug) | ||
2672 | { | ||
2673 | struct request_queue *q; | ||
2674 | unsigned long flags; | ||
2675 | struct request *rq; | ||
2676 | |||
2677 | BUG_ON(plug->magic != PLUG_MAGIC); | ||
2678 | |||
2679 | if (list_empty(&plug->list)) | ||
2680 | return; | ||
2681 | |||
2682 | if (plug->should_sort) | ||
2683 | list_sort(NULL, &plug->list, plug_rq_cmp); | ||
2684 | |||
2685 | q = NULL; | ||
2686 | local_irq_save(flags); | ||
2687 | while (!list_empty(&plug->list)) { | ||
2688 | rq = list_entry_rq(plug->list.next); | ||
2689 | list_del_init(&rq->queuelist); | ||
2690 | BUG_ON(!(rq->cmd_flags & REQ_ON_PLUG)); | ||
2691 | BUG_ON(!rq->q); | ||
2692 | if (rq->q != q) { | ||
2693 | if (q) { | ||
2694 | __blk_run_queue(q, false); | ||
2695 | spin_unlock(q->queue_lock); | ||
2696 | } | ||
2697 | q = rq->q; | ||
2698 | spin_lock(q->queue_lock); | ||
2699 | } | ||
2700 | rq->cmd_flags &= ~REQ_ON_PLUG; | ||
2701 | |||
2702 | /* | ||
2703 | * rq is already accounted, so use raw insert | ||
2704 | */ | ||
2705 | __elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE); | ||
2706 | } | ||
2707 | |||
2708 | if (q) { | ||
2709 | __blk_run_queue(q, false); | ||
2710 | spin_unlock(q->queue_lock); | ||
2711 | } | ||
2712 | |||
2713 | BUG_ON(!list_empty(&plug->list)); | ||
2714 | local_irq_restore(flags); | ||
2715 | } | ||
2716 | |||
2717 | static void __blk_finish_plug(struct task_struct *tsk, struct blk_plug *plug) | ||
2718 | { | ||
2719 | flush_plug_list(plug); | ||
2720 | |||
2721 | if (plug == tsk->plug) | ||
2722 | tsk->plug = NULL; | ||
2723 | } | ||
2724 | |||
2725 | void blk_finish_plug(struct blk_plug *plug) | ||
2726 | { | ||
2727 | if (plug) | ||
2728 | __blk_finish_plug(current, plug); | ||
2729 | } | ||
2730 | EXPORT_SYMBOL(blk_finish_plug); | ||
2731 | |||
2732 | void __blk_flush_plug(struct task_struct *tsk, struct blk_plug *plug) | ||
2733 | { | ||
2734 | __blk_finish_plug(tsk, plug); | ||
2735 | tsk->plug = plug; | ||
2736 | } | ||
2737 | EXPORT_SYMBOL(__blk_flush_plug); | ||
2738 | |||
2631 | int __init blk_dev_init(void) | 2739 | int __init blk_dev_init(void) |
2632 | { | 2740 | { |
2633 | BUILD_BUG_ON(__REQ_NR_BITS > 8 * | 2741 | BUILD_BUG_ON(__REQ_NR_BITS > 8 * |
diff --git a/block/blk-exec.c b/block/blk-exec.c index cf1456a02acd..7482b7fa863b 100644 --- a/block/blk-exec.c +++ b/block/blk-exec.c | |||
@@ -54,8 +54,8 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk, | |||
54 | rq->end_io = done; | 54 | rq->end_io = done; |
55 | WARN_ON(irqs_disabled()); | 55 | WARN_ON(irqs_disabled()); |
56 | spin_lock_irq(q->queue_lock); | 56 | spin_lock_irq(q->queue_lock); |
57 | __elv_add_request(q, rq, where, 1); | 57 | __elv_add_request(q, rq, where); |
58 | __generic_unplug_device(q); | 58 | __blk_run_queue(q, false); |
59 | /* the queue is stopped so it won't be plugged+unplugged */ | 59 | /* the queue is stopped so it won't be plugged+unplugged */ |
60 | if (rq->cmd_type == REQ_TYPE_PM_RESUME) | 60 | if (rq->cmd_type == REQ_TYPE_PM_RESUME) |
61 | q->request_fn(q); | 61 | q->request_fn(q); |
diff --git a/block/blk-flush.c b/block/blk-flush.c index b27d0208611b..93d5fd8e51eb 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c | |||
@@ -1,6 +1,69 @@ | |||
1 | /* | 1 | /* |
2 | * Functions to sequence FLUSH and FUA writes. | 2 | * Functions to sequence FLUSH and FUA writes. |
3 | * | ||
4 | * Copyright (C) 2011 Max Planck Institute for Gravitational Physics | ||
5 | * Copyright (C) 2011 Tejun Heo <tj@kernel.org> | ||
6 | * | ||
7 | * This file is released under the GPLv2. | ||
8 | * | ||
9 | * REQ_{FLUSH|FUA} requests are decomposed to sequences consisted of three | ||
10 | * optional steps - PREFLUSH, DATA and POSTFLUSH - according to the request | ||
11 | * properties and hardware capability. | ||
12 | * | ||
13 | * If a request doesn't have data, only REQ_FLUSH makes sense, which | ||
14 | * indicates a simple flush request. If there is data, REQ_FLUSH indicates | ||
15 | * that the device cache should be flushed before the data is executed, and | ||
16 | * REQ_FUA means that the data must be on non-volatile media on request | ||
17 | * completion. | ||
18 | * | ||
19 | * If the device doesn't have writeback cache, FLUSH and FUA don't make any | ||
20 | * difference. The requests are either completed immediately if there's no | ||
21 | * data or executed as normal requests otherwise. | ||
22 | * | ||
23 | * If the device has writeback cache and supports FUA, REQ_FLUSH is | ||
24 | * translated to PREFLUSH but REQ_FUA is passed down directly with DATA. | ||
25 | * | ||
26 | * If the device has writeback cache and doesn't support FUA, REQ_FLUSH is | ||
27 | * translated to PREFLUSH and REQ_FUA to POSTFLUSH. | ||
28 | * | ||
29 | * The actual execution of flush is double buffered. Whenever a request | ||
30 | * needs to execute PRE or POSTFLUSH, it queues at | ||
31 | * q->flush_queue[q->flush_pending_idx]. Once certain criteria are met, a | ||
32 | * flush is issued and the pending_idx is toggled. When the flush | ||
33 | * completes, all the requests which were pending are proceeded to the next | ||
34 | * step. This allows arbitrary merging of different types of FLUSH/FUA | ||
35 | * requests. | ||
36 | * | ||
37 | * Currently, the following conditions are used to determine when to issue | ||
38 | * flush. | ||
39 | * | ||
40 | * C1. At any given time, only one flush shall be in progress. This makes | ||
41 | * double buffering sufficient. | ||
42 | * | ||
43 | * C2. Flush is deferred if any request is executing DATA of its sequence. | ||
44 | * This avoids issuing separate POSTFLUSHes for requests which shared | ||
45 | * PREFLUSH. | ||
46 | * | ||
47 | * C3. The second condition is ignored if there is a request which has | ||
48 | * waited longer than FLUSH_PENDING_TIMEOUT. This is to avoid | ||
49 | * starvation in the unlikely case where there are continuous stream of | ||
50 | * FUA (without FLUSH) requests. | ||
51 | * | ||
52 | * For devices which support FUA, it isn't clear whether C2 (and thus C3) | ||
53 | * is beneficial. | ||
54 | * | ||
55 | * Note that a sequenced FLUSH/FUA request with DATA is completed twice. | ||
56 | * Once while executing DATA and again after the whole sequence is | ||
57 | * complete. The first completion updates the contained bio but doesn't | ||
58 | * finish it so that the bio submitter is notified only after the whole | ||
59 | * sequence is complete. This is implemented by testing REQ_FLUSH_SEQ in | ||
60 | * req_bio_endio(). | ||
61 | * | ||
62 | * The above peculiarity requires that each FLUSH/FUA request has only one | ||
63 | * bio attached to it, which is guaranteed as they aren't allowed to be | ||
64 | * merged in the usual way. | ||
3 | */ | 65 | */ |
66 | |||
4 | #include <linux/kernel.h> | 67 | #include <linux/kernel.h> |
5 | #include <linux/module.h> | 68 | #include <linux/module.h> |
6 | #include <linux/bio.h> | 69 | #include <linux/bio.h> |
@@ -11,58 +74,142 @@ | |||
11 | 74 | ||
12 | /* FLUSH/FUA sequences */ | 75 | /* FLUSH/FUA sequences */ |
13 | enum { | 76 | enum { |
14 | QUEUE_FSEQ_STARTED = (1 << 0), /* flushing in progress */ | 77 | REQ_FSEQ_PREFLUSH = (1 << 0), /* pre-flushing in progress */ |
15 | QUEUE_FSEQ_PREFLUSH = (1 << 1), /* pre-flushing in progress */ | 78 | REQ_FSEQ_DATA = (1 << 1), /* data write in progress */ |
16 | QUEUE_FSEQ_DATA = (1 << 2), /* data write in progress */ | 79 | REQ_FSEQ_POSTFLUSH = (1 << 2), /* post-flushing in progress */ |
17 | QUEUE_FSEQ_POSTFLUSH = (1 << 3), /* post-flushing in progress */ | 80 | REQ_FSEQ_DONE = (1 << 3), |
18 | QUEUE_FSEQ_DONE = (1 << 4), | 81 | |
82 | REQ_FSEQ_ACTIONS = REQ_FSEQ_PREFLUSH | REQ_FSEQ_DATA | | ||
83 | REQ_FSEQ_POSTFLUSH, | ||
84 | |||
85 | /* | ||
86 | * If flush has been pending longer than the following timeout, | ||
87 | * it's issued even if flush_data requests are still in flight. | ||
88 | */ | ||
89 | FLUSH_PENDING_TIMEOUT = 5 * HZ, | ||
19 | }; | 90 | }; |
20 | 91 | ||
21 | static struct request *queue_next_fseq(struct request_queue *q); | 92 | static bool blk_kick_flush(struct request_queue *q); |
22 | 93 | ||
23 | unsigned blk_flush_cur_seq(struct request_queue *q) | 94 | static unsigned int blk_flush_policy(unsigned int fflags, struct request *rq) |
24 | { | 95 | { |
25 | if (!q->flush_seq) | 96 | unsigned int policy = 0; |
26 | return 0; | 97 | |
27 | return 1 << ffz(q->flush_seq); | 98 | if (fflags & REQ_FLUSH) { |
99 | if (rq->cmd_flags & REQ_FLUSH) | ||
100 | policy |= REQ_FSEQ_PREFLUSH; | ||
101 | if (blk_rq_sectors(rq)) | ||
102 | policy |= REQ_FSEQ_DATA; | ||
103 | if (!(fflags & REQ_FUA) && (rq->cmd_flags & REQ_FUA)) | ||
104 | policy |= REQ_FSEQ_POSTFLUSH; | ||
105 | } | ||
106 | return policy; | ||
28 | } | 107 | } |
29 | 108 | ||
30 | static struct request *blk_flush_complete_seq(struct request_queue *q, | 109 | static unsigned int blk_flush_cur_seq(struct request *rq) |
31 | unsigned seq, int error) | ||
32 | { | 110 | { |
33 | struct request *next_rq = NULL; | 111 | return 1 << ffz(rq->flush.seq); |
34 | 112 | } | |
35 | if (error && !q->flush_err) | 113 | |
36 | q->flush_err = error; | 114 | static void blk_flush_restore_request(struct request *rq) |
37 | 115 | { | |
38 | BUG_ON(q->flush_seq & seq); | 116 | /* |
39 | q->flush_seq |= seq; | 117 | * After flush data completion, @rq->bio is %NULL but we need to |
40 | 118 | * complete the bio again. @rq->biotail is guaranteed to equal the | |
41 | if (blk_flush_cur_seq(q) != QUEUE_FSEQ_DONE) { | 119 | * original @rq->bio. Restore it. |
42 | /* not complete yet, queue the next flush sequence */ | 120 | */ |
43 | next_rq = queue_next_fseq(q); | 121 | rq->bio = rq->biotail; |
44 | } else { | 122 | |
45 | /* complete this flush request */ | 123 | /* make @rq a normal request */ |
46 | __blk_end_request_all(q->orig_flush_rq, q->flush_err); | 124 | rq->cmd_flags &= ~REQ_FLUSH_SEQ; |
47 | q->orig_flush_rq = NULL; | 125 | rq->end_io = NULL; |
48 | q->flush_seq = 0; | 126 | } |
49 | 127 | ||
50 | /* dispatch the next flush if there's one */ | 128 | /** |
51 | if (!list_empty(&q->pending_flushes)) { | 129 | * blk_flush_complete_seq - complete flush sequence |
52 | next_rq = list_entry_rq(q->pending_flushes.next); | 130 | * @rq: FLUSH/FUA request being sequenced |
53 | list_move(&next_rq->queuelist, &q->queue_head); | 131 | * @seq: sequences to complete (mask of %REQ_FSEQ_*, can be zero) |
54 | } | 132 | * @error: whether an error occurred |
133 | * | ||
134 | * @rq just completed @seq part of its flush sequence, record the | ||
135 | * completion and trigger the next step. | ||
136 | * | ||
137 | * CONTEXT: | ||
138 | * spin_lock_irq(q->queue_lock) | ||
139 | * | ||
140 | * RETURNS: | ||
141 | * %true if requests were added to the dispatch queue, %false otherwise. | ||
142 | */ | ||
143 | static bool blk_flush_complete_seq(struct request *rq, unsigned int seq, | ||
144 | int error) | ||
145 | { | ||
146 | struct request_queue *q = rq->q; | ||
147 | struct list_head *pending = &q->flush_queue[q->flush_pending_idx]; | ||
148 | bool queued = false; | ||
149 | |||
150 | BUG_ON(rq->flush.seq & seq); | ||
151 | rq->flush.seq |= seq; | ||
152 | |||
153 | if (likely(!error)) | ||
154 | seq = blk_flush_cur_seq(rq); | ||
155 | else | ||
156 | seq = REQ_FSEQ_DONE; | ||
157 | |||
158 | switch (seq) { | ||
159 | case REQ_FSEQ_PREFLUSH: | ||
160 | case REQ_FSEQ_POSTFLUSH: | ||
161 | /* queue for flush */ | ||
162 | if (list_empty(pending)) | ||
163 | q->flush_pending_since = jiffies; | ||
164 | list_move_tail(&rq->flush.list, pending); | ||
165 | break; | ||
166 | |||
167 | case REQ_FSEQ_DATA: | ||
168 | list_move_tail(&rq->flush.list, &q->flush_data_in_flight); | ||
169 | list_add(&rq->queuelist, &q->queue_head); | ||
170 | queued = true; | ||
171 | break; | ||
172 | |||
173 | case REQ_FSEQ_DONE: | ||
174 | /* | ||
175 | * @rq was previously adjusted by blk_flush_issue() for | ||
176 | * flush sequencing and may already have gone through the | ||
177 | * flush data request completion path. Restore @rq for | ||
178 | * normal completion and end it. | ||
179 | */ | ||
180 | BUG_ON(!list_empty(&rq->queuelist)); | ||
181 | list_del_init(&rq->flush.list); | ||
182 | blk_flush_restore_request(rq); | ||
183 | __blk_end_request_all(rq, error); | ||
184 | break; | ||
185 | |||
186 | default: | ||
187 | BUG(); | ||
55 | } | 188 | } |
56 | return next_rq; | 189 | |
190 | return blk_kick_flush(q) | queued; | ||
57 | } | 191 | } |
58 | 192 | ||
59 | static void blk_flush_complete_seq_end_io(struct request_queue *q, | 193 | static void flush_end_io(struct request *flush_rq, int error) |
60 | unsigned seq, int error) | ||
61 | { | 194 | { |
62 | bool was_empty = elv_queue_empty(q); | 195 | struct request_queue *q = flush_rq->q; |
63 | struct request *next_rq; | 196 | struct list_head *running = &q->flush_queue[q->flush_running_idx]; |
197 | bool queued = false; | ||
198 | struct request *rq, *n; | ||
64 | 199 | ||
65 | next_rq = blk_flush_complete_seq(q, seq, error); | 200 | BUG_ON(q->flush_pending_idx == q->flush_running_idx); |
201 | |||
202 | /* account completion of the flush request */ | ||
203 | q->flush_running_idx ^= 1; | ||
204 | elv_completed_request(q, flush_rq); | ||
205 | |||
206 | /* and push the waiting requests to the next stage */ | ||
207 | list_for_each_entry_safe(rq, n, running, flush.list) { | ||
208 | unsigned int seq = blk_flush_cur_seq(rq); | ||
209 | |||
210 | BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH); | ||
211 | queued |= blk_flush_complete_seq(rq, seq, error); | ||
212 | } | ||
66 | 213 | ||
67 | /* | 214 | /* |
68 | * Moving a request silently to empty queue_head may stall the | 215 | * Moving a request silently to empty queue_head may stall the |
@@ -70,127 +217,153 @@ static void blk_flush_complete_seq_end_io(struct request_queue *q, | |||
70 | * from request completion path and calling directly into | 217 | * from request completion path and calling directly into |
71 | * request_fn may confuse the driver. Always use kblockd. | 218 | * request_fn may confuse the driver. Always use kblockd. |
72 | */ | 219 | */ |
73 | if (was_empty && next_rq) | 220 | if (queued) |
74 | __blk_run_queue(q, true); | 221 | __blk_run_queue(q, true); |
75 | } | 222 | } |
76 | 223 | ||
77 | static void pre_flush_end_io(struct request *rq, int error) | 224 | /** |
225 | * blk_kick_flush - consider issuing flush request | ||
226 | * @q: request_queue being kicked | ||
227 | * | ||
228 | * Flush related states of @q have changed, consider issuing flush request. | ||
229 | * Please read the comment at the top of this file for more info. | ||
230 | * | ||
231 | * CONTEXT: | ||
232 | * spin_lock_irq(q->queue_lock) | ||
233 | * | ||
234 | * RETURNS: | ||
235 | * %true if flush was issued, %false otherwise. | ||
236 | */ | ||
237 | static bool blk_kick_flush(struct request_queue *q) | ||
78 | { | 238 | { |
79 | elv_completed_request(rq->q, rq); | 239 | struct list_head *pending = &q->flush_queue[q->flush_pending_idx]; |
80 | blk_flush_complete_seq_end_io(rq->q, QUEUE_FSEQ_PREFLUSH, error); | 240 | struct request *first_rq = |
241 | list_first_entry(pending, struct request, flush.list); | ||
242 | |||
243 | /* C1 described at the top of this file */ | ||
244 | if (q->flush_pending_idx != q->flush_running_idx || list_empty(pending)) | ||
245 | return false; | ||
246 | |||
247 | /* C2 and C3 */ | ||
248 | if (!list_empty(&q->flush_data_in_flight) && | ||
249 | time_before(jiffies, | ||
250 | q->flush_pending_since + FLUSH_PENDING_TIMEOUT)) | ||
251 | return false; | ||
252 | |||
253 | /* | ||
254 | * Issue flush and toggle pending_idx. This makes pending_idx | ||
255 | * different from running_idx, which means flush is in flight. | ||
256 | */ | ||
257 | blk_rq_init(q, &q->flush_rq); | ||
258 | q->flush_rq.cmd_type = REQ_TYPE_FS; | ||
259 | q->flush_rq.cmd_flags = WRITE_FLUSH | REQ_FLUSH_SEQ; | ||
260 | q->flush_rq.rq_disk = first_rq->rq_disk; | ||
261 | q->flush_rq.end_io = flush_end_io; | ||
262 | |||
263 | q->flush_pending_idx ^= 1; | ||
264 | elv_insert(q, &q->flush_rq, ELEVATOR_INSERT_REQUEUE); | ||
265 | return true; | ||
81 | } | 266 | } |
82 | 267 | ||
83 | static void flush_data_end_io(struct request *rq, int error) | 268 | static void flush_data_end_io(struct request *rq, int error) |
84 | { | 269 | { |
85 | elv_completed_request(rq->q, rq); | 270 | struct request_queue *q = rq->q; |
86 | blk_flush_complete_seq_end_io(rq->q, QUEUE_FSEQ_DATA, error); | ||
87 | } | ||
88 | 271 | ||
89 | static void post_flush_end_io(struct request *rq, int error) | 272 | /* |
90 | { | 273 | * After populating an empty queue, kick it to avoid stall. Read |
91 | elv_completed_request(rq->q, rq); | 274 | * the comment in flush_end_io(). |
92 | blk_flush_complete_seq_end_io(rq->q, QUEUE_FSEQ_POSTFLUSH, error); | 275 | */ |
276 | if (blk_flush_complete_seq(rq, REQ_FSEQ_DATA, error)) | ||
277 | __blk_run_queue(q, true); | ||
93 | } | 278 | } |
94 | 279 | ||
95 | static void init_flush_request(struct request *rq, struct gendisk *disk) | 280 | /** |
281 | * blk_insert_flush - insert a new FLUSH/FUA request | ||
282 | * @rq: request to insert | ||
283 | * | ||
284 | * To be called from elv_insert() for %ELEVATOR_INSERT_FLUSH insertions. | ||
285 | * @rq is being submitted. Analyze what needs to be done and put it on the | ||
286 | * right queue. | ||
287 | * | ||
288 | * CONTEXT: | ||
289 | * spin_lock_irq(q->queue_lock) | ||
290 | */ | ||
291 | void blk_insert_flush(struct request *rq) | ||
96 | { | 292 | { |
97 | rq->cmd_type = REQ_TYPE_FS; | 293 | struct request_queue *q = rq->q; |
98 | rq->cmd_flags = WRITE_FLUSH; | 294 | unsigned int fflags = q->flush_flags; /* may change, cache */ |
99 | rq->rq_disk = disk; | 295 | unsigned int policy = blk_flush_policy(fflags, rq); |
100 | } | ||
101 | 296 | ||
102 | static struct request *queue_next_fseq(struct request_queue *q) | 297 | BUG_ON(rq->end_io); |
103 | { | 298 | BUG_ON(!rq->bio || rq->bio != rq->biotail); |
104 | struct request *orig_rq = q->orig_flush_rq; | ||
105 | struct request *rq = &q->flush_rq; | ||
106 | 299 | ||
107 | blk_rq_init(q, rq); | 300 | /* |
301 | * @policy now records what operations need to be done. Adjust | ||
302 | * REQ_FLUSH and FUA for the driver. | ||
303 | */ | ||
304 | rq->cmd_flags &= ~REQ_FLUSH; | ||
305 | if (!(fflags & REQ_FUA)) | ||
306 | rq->cmd_flags &= ~REQ_FUA; | ||
108 | 307 | ||
109 | switch (blk_flush_cur_seq(q)) { | 308 | /* |
110 | case QUEUE_FSEQ_PREFLUSH: | 309 | * If there's data but flush is not necessary, the request can be |
111 | init_flush_request(rq, orig_rq->rq_disk); | 310 | * processed directly without going through flush machinery. Queue |
112 | rq->end_io = pre_flush_end_io; | 311 | * for normal execution. |
113 | break; | 312 | */ |
114 | case QUEUE_FSEQ_DATA: | 313 | if ((policy & REQ_FSEQ_DATA) && |
115 | init_request_from_bio(rq, orig_rq->bio); | 314 | !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) { |
116 | /* | 315 | list_add(&rq->queuelist, &q->queue_head); |
117 | * orig_rq->rq_disk may be different from | 316 | return; |
118 | * bio->bi_bdev->bd_disk if orig_rq got here through | ||
119 | * remapping drivers. Make sure rq->rq_disk points | ||
120 | * to the same one as orig_rq. | ||
121 | */ | ||
122 | rq->rq_disk = orig_rq->rq_disk; | ||
123 | rq->cmd_flags &= ~(REQ_FLUSH | REQ_FUA); | ||
124 | rq->cmd_flags |= orig_rq->cmd_flags & (REQ_FLUSH | REQ_FUA); | ||
125 | rq->end_io = flush_data_end_io; | ||
126 | break; | ||
127 | case QUEUE_FSEQ_POSTFLUSH: | ||
128 | init_flush_request(rq, orig_rq->rq_disk); | ||
129 | rq->end_io = post_flush_end_io; | ||
130 | break; | ||
131 | default: | ||
132 | BUG(); | ||
133 | } | 317 | } |
134 | 318 | ||
135 | elv_insert(q, rq, ELEVATOR_INSERT_REQUEUE); | 319 | /* |
136 | return rq; | 320 | * @rq should go through flush machinery. Mark it part of flush |
321 | * sequence and submit for further processing. | ||
322 | */ | ||
323 | memset(&rq->flush, 0, sizeof(rq->flush)); | ||
324 | INIT_LIST_HEAD(&rq->flush.list); | ||
325 | rq->cmd_flags |= REQ_FLUSH_SEQ; | ||
326 | rq->end_io = flush_data_end_io; | ||
327 | |||
328 | blk_flush_complete_seq(rq, REQ_FSEQ_ACTIONS & ~policy, 0); | ||
137 | } | 329 | } |
138 | 330 | ||
139 | struct request *blk_do_flush(struct request_queue *q, struct request *rq) | 331 | /** |
332 | * blk_abort_flushes - @q is being aborted, abort flush requests | ||
333 | * @q: request_queue being aborted | ||
334 | * | ||
335 | * To be called from elv_abort_queue(). @q is being aborted. Prepare all | ||
336 | * FLUSH/FUA requests for abortion. | ||
337 | * | ||
338 | * CONTEXT: | ||
339 | * spin_lock_irq(q->queue_lock) | ||
340 | */ | ||
341 | void blk_abort_flushes(struct request_queue *q) | ||
140 | { | 342 | { |
141 | unsigned int fflags = q->flush_flags; /* may change, cache it */ | 343 | struct request *rq, *n; |
142 | bool has_flush = fflags & REQ_FLUSH, has_fua = fflags & REQ_FUA; | 344 | int i; |
143 | bool do_preflush = has_flush && (rq->cmd_flags & REQ_FLUSH); | ||
144 | bool do_postflush = has_flush && !has_fua && (rq->cmd_flags & REQ_FUA); | ||
145 | unsigned skip = 0; | ||
146 | 345 | ||
147 | /* | 346 | /* |
148 | * Special case. If there's data but flush is not necessary, | 347 | * Requests in flight for data are already owned by the dispatch |
149 | * the request can be issued directly. | 348 | * queue or the device driver. Just restore for normal completion. |
150 | * | ||
151 | * Flush w/o data should be able to be issued directly too but | ||
152 | * currently some drivers assume that rq->bio contains | ||
153 | * non-zero data if it isn't NULL and empty FLUSH requests | ||
154 | * getting here usually have bio's without data. | ||
155 | */ | 349 | */ |
156 | if (blk_rq_sectors(rq) && !do_preflush && !do_postflush) { | 350 | list_for_each_entry_safe(rq, n, &q->flush_data_in_flight, flush.list) { |
157 | rq->cmd_flags &= ~REQ_FLUSH; | 351 | list_del_init(&rq->flush.list); |
158 | if (!has_fua) | 352 | blk_flush_restore_request(rq); |
159 | rq->cmd_flags &= ~REQ_FUA; | ||
160 | return rq; | ||
161 | } | 353 | } |
162 | 354 | ||
163 | /* | 355 | /* |
164 | * Sequenced flushes can't be processed in parallel. If | 356 | * We need to give away requests on flush queues. Restore for |
165 | * another one is already in progress, queue for later | 357 | * normal completion and put them on the dispatch queue. |
166 | * processing. | ||
167 | */ | 358 | */ |
168 | if (q->flush_seq) { | 359 | for (i = 0; i < ARRAY_SIZE(q->flush_queue); i++) { |
169 | list_move_tail(&rq->queuelist, &q->pending_flushes); | 360 | list_for_each_entry_safe(rq, n, &q->flush_queue[i], |
170 | return NULL; | 361 | flush.list) { |
362 | list_del_init(&rq->flush.list); | ||
363 | blk_flush_restore_request(rq); | ||
364 | list_add_tail(&rq->queuelist, &q->queue_head); | ||
365 | } | ||
171 | } | 366 | } |
172 | |||
173 | /* | ||
174 | * Start a new flush sequence | ||
175 | */ | ||
176 | q->flush_err = 0; | ||
177 | q->flush_seq |= QUEUE_FSEQ_STARTED; | ||
178 | |||
179 | /* adjust FLUSH/FUA of the original request and stash it away */ | ||
180 | rq->cmd_flags &= ~REQ_FLUSH; | ||
181 | if (!has_fua) | ||
182 | rq->cmd_flags &= ~REQ_FUA; | ||
183 | blk_dequeue_request(rq); | ||
184 | q->orig_flush_rq = rq; | ||
185 | |||
186 | /* skip unneded sequences and return the first one */ | ||
187 | if (!do_preflush) | ||
188 | skip |= QUEUE_FSEQ_PREFLUSH; | ||
189 | if (!blk_rq_sectors(rq)) | ||
190 | skip |= QUEUE_FSEQ_DATA; | ||
191 | if (!do_postflush) | ||
192 | skip |= QUEUE_FSEQ_POSTFLUSH; | ||
193 | return blk_flush_complete_seq(q, skip, 0); | ||
194 | } | 367 | } |
195 | 368 | ||
196 | static void bio_end_flush(struct bio *bio, int err) | 369 | static void bio_end_flush(struct bio *bio, int err) |
diff --git a/block/blk-lib.c b/block/blk-lib.c index bd3e8df4d5e2..25de73e4759b 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c | |||
@@ -136,8 +136,6 @@ static void bio_batch_end_io(struct bio *bio, int err) | |||
136 | * | 136 | * |
137 | * Description: | 137 | * Description: |
138 | * Generate and issue number of bios with zerofiled pages. | 138 | * Generate and issue number of bios with zerofiled pages. |
139 | * Send barrier at the beginning and at the end if requested. This guarantie | ||
140 | * correct request ordering. Empty barrier allow us to avoid post queue flush. | ||
141 | */ | 139 | */ |
142 | 140 | ||
143 | int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, | 141 | int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, |
diff --git a/block/blk-merge.c b/block/blk-merge.c index ea85e20d5e94..cfcc37cb222b 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c | |||
@@ -465,3 +465,9 @@ int attempt_front_merge(struct request_queue *q, struct request *rq) | |||
465 | 465 | ||
466 | return 0; | 466 | return 0; |
467 | } | 467 | } |
468 | |||
469 | int blk_attempt_req_merge(struct request_queue *q, struct request *rq, | ||
470 | struct request *next) | ||
471 | { | ||
472 | return attempt_merge(q, rq, next); | ||
473 | } | ||
diff --git a/block/blk-settings.c b/block/blk-settings.c index 36c8c1f2af18..1fa769293597 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c | |||
@@ -164,25 +164,10 @@ void blk_queue_make_request(struct request_queue *q, make_request_fn *mfn) | |||
164 | blk_queue_congestion_threshold(q); | 164 | blk_queue_congestion_threshold(q); |
165 | q->nr_batching = BLK_BATCH_REQ; | 165 | q->nr_batching = BLK_BATCH_REQ; |
166 | 166 | ||
167 | q->unplug_thresh = 4; /* hmm */ | ||
168 | q->unplug_delay = msecs_to_jiffies(3); /* 3 milliseconds */ | ||
169 | if (q->unplug_delay == 0) | ||
170 | q->unplug_delay = 1; | ||
171 | |||
172 | q->unplug_timer.function = blk_unplug_timeout; | ||
173 | q->unplug_timer.data = (unsigned long)q; | ||
174 | |||
175 | blk_set_default_limits(&q->limits); | 167 | blk_set_default_limits(&q->limits); |
176 | blk_queue_max_hw_sectors(q, BLK_SAFE_MAX_SECTORS); | 168 | blk_queue_max_hw_sectors(q, BLK_SAFE_MAX_SECTORS); |
177 | 169 | ||
178 | /* | 170 | /* |
179 | * If the caller didn't supply a lock, fall back to our embedded | ||
180 | * per-queue locks | ||
181 | */ | ||
182 | if (!q->queue_lock) | ||
183 | q->queue_lock = &q->__queue_lock; | ||
184 | |||
185 | /* | ||
186 | * by default assume old behaviour and bounce for any highmem page | 171 | * by default assume old behaviour and bounce for any highmem page |
187 | */ | 172 | */ |
188 | blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH); | 173 | blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH); |
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 41fb69150b4d..261c75c665ae 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c | |||
@@ -471,8 +471,6 @@ static void blk_release_queue(struct kobject *kobj) | |||
471 | 471 | ||
472 | blk_sync_queue(q); | 472 | blk_sync_queue(q); |
473 | 473 | ||
474 | blk_throtl_exit(q); | ||
475 | |||
476 | if (rl->rq_pool) | 474 | if (rl->rq_pool) |
477 | mempool_destroy(rl->rq_pool); | 475 | mempool_destroy(rl->rq_pool); |
478 | 476 | ||
diff --git a/block/blk-throttle.c b/block/blk-throttle.c index e36cc10a346c..5352bdafbcf0 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c | |||
@@ -102,7 +102,7 @@ struct throtl_data | |||
102 | /* Work for dispatching throttled bios */ | 102 | /* Work for dispatching throttled bios */ |
103 | struct delayed_work throtl_work; | 103 | struct delayed_work throtl_work; |
104 | 104 | ||
105 | atomic_t limits_changed; | 105 | bool limits_changed; |
106 | }; | 106 | }; |
107 | 107 | ||
108 | enum tg_state_flags { | 108 | enum tg_state_flags { |
@@ -201,6 +201,7 @@ static struct throtl_grp * throtl_find_alloc_tg(struct throtl_data *td, | |||
201 | RB_CLEAR_NODE(&tg->rb_node); | 201 | RB_CLEAR_NODE(&tg->rb_node); |
202 | bio_list_init(&tg->bio_lists[0]); | 202 | bio_list_init(&tg->bio_lists[0]); |
203 | bio_list_init(&tg->bio_lists[1]); | 203 | bio_list_init(&tg->bio_lists[1]); |
204 | td->limits_changed = false; | ||
204 | 205 | ||
205 | /* | 206 | /* |
206 | * Take the initial reference that will be released on destroy | 207 | * Take the initial reference that will be released on destroy |
@@ -737,34 +738,36 @@ static void throtl_process_limit_change(struct throtl_data *td) | |||
737 | struct throtl_grp *tg; | 738 | struct throtl_grp *tg; |
738 | struct hlist_node *pos, *n; | 739 | struct hlist_node *pos, *n; |
739 | 740 | ||
740 | if (!atomic_read(&td->limits_changed)) | 741 | if (!td->limits_changed) |
741 | return; | 742 | return; |
742 | 743 | ||
743 | throtl_log(td, "limit changed =%d", atomic_read(&td->limits_changed)); | 744 | xchg(&td->limits_changed, false); |
744 | 745 | ||
745 | /* | 746 | throtl_log(td, "limits changed"); |
746 | * Make sure updates from throtl_update_blkio_group_read_bps() group | ||
747 | * of functions to tg->limits_changed are visible. We do not | ||
748 | * want update td->limits_changed to be visible but update to | ||
749 | * tg->limits_changed not being visible yet on this cpu. Hence | ||
750 | * the read barrier. | ||
751 | */ | ||
752 | smp_rmb(); | ||
753 | 747 | ||
754 | hlist_for_each_entry_safe(tg, pos, n, &td->tg_list, tg_node) { | 748 | hlist_for_each_entry_safe(tg, pos, n, &td->tg_list, tg_node) { |
755 | if (throtl_tg_on_rr(tg) && tg->limits_changed) { | 749 | if (!tg->limits_changed) |
756 | throtl_log_tg(td, tg, "limit change rbps=%llu wbps=%llu" | 750 | continue; |
757 | " riops=%u wiops=%u", tg->bps[READ], | 751 | |
758 | tg->bps[WRITE], tg->iops[READ], | 752 | if (!xchg(&tg->limits_changed, false)) |
759 | tg->iops[WRITE]); | 753 | continue; |
754 | |||
755 | throtl_log_tg(td, tg, "limit change rbps=%llu wbps=%llu" | ||
756 | " riops=%u wiops=%u", tg->bps[READ], tg->bps[WRITE], | ||
757 | tg->iops[READ], tg->iops[WRITE]); | ||
758 | |||
759 | /* | ||
760 | * Restart the slices for both READ and WRITES. It | ||
761 | * might happen that a group's limit are dropped | ||
762 | * suddenly and we don't want to account recently | ||
763 | * dispatched IO with new low rate | ||
764 | */ | ||
765 | throtl_start_new_slice(td, tg, 0); | ||
766 | throtl_start_new_slice(td, tg, 1); | ||
767 | |||
768 | if (throtl_tg_on_rr(tg)) | ||
760 | tg_update_disptime(td, tg); | 769 | tg_update_disptime(td, tg); |
761 | tg->limits_changed = false; | ||
762 | } | ||
763 | } | 770 | } |
764 | |||
765 | smp_mb__before_atomic_dec(); | ||
766 | atomic_dec(&td->limits_changed); | ||
767 | smp_mb__after_atomic_dec(); | ||
768 | } | 771 | } |
769 | 772 | ||
770 | /* Dispatch throttled bios. Should be called without queue lock held. */ | 773 | /* Dispatch throttled bios. Should be called without queue lock held. */ |
@@ -774,6 +777,7 @@ static int throtl_dispatch(struct request_queue *q) | |||
774 | unsigned int nr_disp = 0; | 777 | unsigned int nr_disp = 0; |
775 | struct bio_list bio_list_on_stack; | 778 | struct bio_list bio_list_on_stack; |
776 | struct bio *bio; | 779 | struct bio *bio; |
780 | struct blk_plug plug; | ||
777 | 781 | ||
778 | spin_lock_irq(q->queue_lock); | 782 | spin_lock_irq(q->queue_lock); |
779 | 783 | ||
@@ -802,9 +806,10 @@ out: | |||
802 | * immediate dispatch | 806 | * immediate dispatch |
803 | */ | 807 | */ |
804 | if (nr_disp) { | 808 | if (nr_disp) { |
809 | blk_start_plug(&plug); | ||
805 | while((bio = bio_list_pop(&bio_list_on_stack))) | 810 | while((bio = bio_list_pop(&bio_list_on_stack))) |
806 | generic_make_request(bio); | 811 | generic_make_request(bio); |
807 | blk_unplug(q); | 812 | blk_finish_plug(&plug); |
808 | } | 813 | } |
809 | return nr_disp; | 814 | return nr_disp; |
810 | } | 815 | } |
@@ -825,7 +830,8 @@ throtl_schedule_delayed_work(struct throtl_data *td, unsigned long delay) | |||
825 | 830 | ||
826 | struct delayed_work *dwork = &td->throtl_work; | 831 | struct delayed_work *dwork = &td->throtl_work; |
827 | 832 | ||
828 | if (total_nr_queued(td) > 0) { | 833 | /* schedule work if limits changed even if no bio is queued */ |
834 | if (total_nr_queued(td) > 0 || td->limits_changed) { | ||
829 | /* | 835 | /* |
830 | * We might have a work scheduled to be executed in future. | 836 | * We might have a work scheduled to be executed in future. |
831 | * Cancel that and schedule a new one. | 837 | * Cancel that and schedule a new one. |
@@ -898,6 +904,15 @@ void throtl_unlink_blkio_group(void *key, struct blkio_group *blkg) | |||
898 | spin_unlock_irqrestore(td->queue->queue_lock, flags); | 904 | spin_unlock_irqrestore(td->queue->queue_lock, flags); |
899 | } | 905 | } |
900 | 906 | ||
907 | static void throtl_update_blkio_group_common(struct throtl_data *td, | ||
908 | struct throtl_grp *tg) | ||
909 | { | ||
910 | xchg(&tg->limits_changed, true); | ||
911 | xchg(&td->limits_changed, true); | ||
912 | /* Schedule a work now to process the limit change */ | ||
913 | throtl_schedule_delayed_work(td, 0); | ||
914 | } | ||
915 | |||
901 | /* | 916 | /* |
902 | * For all update functions, key should be a valid pointer because these | 917 | * For all update functions, key should be a valid pointer because these |
903 | * update functions are called under blkcg_lock, that means, blkg is | 918 | * update functions are called under blkcg_lock, that means, blkg is |
@@ -911,64 +926,43 @@ static void throtl_update_blkio_group_read_bps(void *key, | |||
911 | struct blkio_group *blkg, u64 read_bps) | 926 | struct blkio_group *blkg, u64 read_bps) |
912 | { | 927 | { |
913 | struct throtl_data *td = key; | 928 | struct throtl_data *td = key; |
929 | struct throtl_grp *tg = tg_of_blkg(blkg); | ||
914 | 930 | ||
915 | tg_of_blkg(blkg)->bps[READ] = read_bps; | 931 | tg->bps[READ] = read_bps; |
916 | /* Make sure read_bps is updated before setting limits_changed */ | 932 | throtl_update_blkio_group_common(td, tg); |
917 | smp_wmb(); | ||
918 | tg_of_blkg(blkg)->limits_changed = true; | ||
919 | |||
920 | /* Make sure tg->limits_changed is updated before td->limits_changed */ | ||
921 | smp_mb__before_atomic_inc(); | ||
922 | atomic_inc(&td->limits_changed); | ||
923 | smp_mb__after_atomic_inc(); | ||
924 | |||
925 | /* Schedule a work now to process the limit change */ | ||
926 | throtl_schedule_delayed_work(td, 0); | ||
927 | } | 933 | } |
928 | 934 | ||
929 | static void throtl_update_blkio_group_write_bps(void *key, | 935 | static void throtl_update_blkio_group_write_bps(void *key, |
930 | struct blkio_group *blkg, u64 write_bps) | 936 | struct blkio_group *blkg, u64 write_bps) |
931 | { | 937 | { |
932 | struct throtl_data *td = key; | 938 | struct throtl_data *td = key; |
939 | struct throtl_grp *tg = tg_of_blkg(blkg); | ||
933 | 940 | ||
934 | tg_of_blkg(blkg)->bps[WRITE] = write_bps; | 941 | tg->bps[WRITE] = write_bps; |
935 | smp_wmb(); | 942 | throtl_update_blkio_group_common(td, tg); |
936 | tg_of_blkg(blkg)->limits_changed = true; | ||
937 | smp_mb__before_atomic_inc(); | ||
938 | atomic_inc(&td->limits_changed); | ||
939 | smp_mb__after_atomic_inc(); | ||
940 | throtl_schedule_delayed_work(td, 0); | ||
941 | } | 943 | } |
942 | 944 | ||
943 | static void throtl_update_blkio_group_read_iops(void *key, | 945 | static void throtl_update_blkio_group_read_iops(void *key, |
944 | struct blkio_group *blkg, unsigned int read_iops) | 946 | struct blkio_group *blkg, unsigned int read_iops) |
945 | { | 947 | { |
946 | struct throtl_data *td = key; | 948 | struct throtl_data *td = key; |
949 | struct throtl_grp *tg = tg_of_blkg(blkg); | ||
947 | 950 | ||
948 | tg_of_blkg(blkg)->iops[READ] = read_iops; | 951 | tg->iops[READ] = read_iops; |
949 | smp_wmb(); | 952 | throtl_update_blkio_group_common(td, tg); |
950 | tg_of_blkg(blkg)->limits_changed = true; | ||
951 | smp_mb__before_atomic_inc(); | ||
952 | atomic_inc(&td->limits_changed); | ||
953 | smp_mb__after_atomic_inc(); | ||
954 | throtl_schedule_delayed_work(td, 0); | ||
955 | } | 953 | } |
956 | 954 | ||
957 | static void throtl_update_blkio_group_write_iops(void *key, | 955 | static void throtl_update_blkio_group_write_iops(void *key, |
958 | struct blkio_group *blkg, unsigned int write_iops) | 956 | struct blkio_group *blkg, unsigned int write_iops) |
959 | { | 957 | { |
960 | struct throtl_data *td = key; | 958 | struct throtl_data *td = key; |
959 | struct throtl_grp *tg = tg_of_blkg(blkg); | ||
961 | 960 | ||
962 | tg_of_blkg(blkg)->iops[WRITE] = write_iops; | 961 | tg->iops[WRITE] = write_iops; |
963 | smp_wmb(); | 962 | throtl_update_blkio_group_common(td, tg); |
964 | tg_of_blkg(blkg)->limits_changed = true; | ||
965 | smp_mb__before_atomic_inc(); | ||
966 | atomic_inc(&td->limits_changed); | ||
967 | smp_mb__after_atomic_inc(); | ||
968 | throtl_schedule_delayed_work(td, 0); | ||
969 | } | 963 | } |
970 | 964 | ||
971 | void throtl_shutdown_timer_wq(struct request_queue *q) | 965 | static void throtl_shutdown_wq(struct request_queue *q) |
972 | { | 966 | { |
973 | struct throtl_data *td = q->td; | 967 | struct throtl_data *td = q->td; |
974 | 968 | ||
@@ -1009,20 +1003,28 @@ int blk_throtl_bio(struct request_queue *q, struct bio **biop) | |||
1009 | /* | 1003 | /* |
1010 | * There is already another bio queued in same dir. No | 1004 | * There is already another bio queued in same dir. No |
1011 | * need to update dispatch time. | 1005 | * need to update dispatch time. |
1012 | * Still update the disptime if rate limits on this group | ||
1013 | * were changed. | ||
1014 | */ | 1006 | */ |
1015 | if (!tg->limits_changed) | 1007 | update_disptime = false; |
1016 | update_disptime = false; | ||
1017 | else | ||
1018 | tg->limits_changed = false; | ||
1019 | |||
1020 | goto queue_bio; | 1008 | goto queue_bio; |
1009 | |||
1021 | } | 1010 | } |
1022 | 1011 | ||
1023 | /* Bio is with-in rate limit of group */ | 1012 | /* Bio is with-in rate limit of group */ |
1024 | if (tg_may_dispatch(td, tg, bio, NULL)) { | 1013 | if (tg_may_dispatch(td, tg, bio, NULL)) { |
1025 | throtl_charge_bio(tg, bio); | 1014 | throtl_charge_bio(tg, bio); |
1015 | |||
1016 | /* | ||
1017 | * We need to trim slice even when bios are not being queued | ||
1018 | * otherwise it might happen that a bio is not queued for | ||
1019 | * a long time and slice keeps on extending and trim is not | ||
1020 | * called for a long time. Now if limits are reduced suddenly | ||
1021 | * we take into account all the IO dispatched so far at new | ||
1022 | * low rate and * newly queued IO gets a really long dispatch | ||
1023 | * time. | ||
1024 | * | ||
1025 | * So keep on trimming slice even if bio is not queued. | ||
1026 | */ | ||
1027 | throtl_trim_slice(td, tg, rw); | ||
1026 | goto out; | 1028 | goto out; |
1027 | } | 1029 | } |
1028 | 1030 | ||
@@ -1058,7 +1060,7 @@ int blk_throtl_init(struct request_queue *q) | |||
1058 | 1060 | ||
1059 | INIT_HLIST_HEAD(&td->tg_list); | 1061 | INIT_HLIST_HEAD(&td->tg_list); |
1060 | td->tg_service_tree = THROTL_RB_ROOT; | 1062 | td->tg_service_tree = THROTL_RB_ROOT; |
1061 | atomic_set(&td->limits_changed, 0); | 1063 | td->limits_changed = false; |
1062 | 1064 | ||
1063 | /* Init root group */ | 1065 | /* Init root group */ |
1064 | tg = &td->root_tg; | 1066 | tg = &td->root_tg; |
@@ -1070,6 +1072,7 @@ int blk_throtl_init(struct request_queue *q) | |||
1070 | /* Practically unlimited BW */ | 1072 | /* Practically unlimited BW */ |
1071 | tg->bps[0] = tg->bps[1] = -1; | 1073 | tg->bps[0] = tg->bps[1] = -1; |
1072 | tg->iops[0] = tg->iops[1] = -1; | 1074 | tg->iops[0] = tg->iops[1] = -1; |
1075 | td->limits_changed = false; | ||
1073 | 1076 | ||
1074 | /* | 1077 | /* |
1075 | * Set root group reference to 2. One reference will be dropped when | 1078 | * Set root group reference to 2. One reference will be dropped when |
@@ -1102,7 +1105,7 @@ void blk_throtl_exit(struct request_queue *q) | |||
1102 | 1105 | ||
1103 | BUG_ON(!td); | 1106 | BUG_ON(!td); |
1104 | 1107 | ||
1105 | throtl_shutdown_timer_wq(q); | 1108 | throtl_shutdown_wq(q); |
1106 | 1109 | ||
1107 | spin_lock_irq(q->queue_lock); | 1110 | spin_lock_irq(q->queue_lock); |
1108 | throtl_release_tgs(td); | 1111 | throtl_release_tgs(td); |
@@ -1132,7 +1135,7 @@ void blk_throtl_exit(struct request_queue *q) | |||
1132 | * update limits through cgroup and another work got queued, cancel | 1135 | * update limits through cgroup and another work got queued, cancel |
1133 | * it. | 1136 | * it. |
1134 | */ | 1137 | */ |
1135 | throtl_shutdown_timer_wq(q); | 1138 | throtl_shutdown_wq(q); |
1136 | throtl_td_free(td); | 1139 | throtl_td_free(td); |
1137 | } | 1140 | } |
1138 | 1141 | ||
diff --git a/block/blk.h b/block/blk.h index 2db8f32838e7..c8db371a921d 100644 --- a/block/blk.h +++ b/block/blk.h | |||
@@ -18,8 +18,6 @@ int blk_rq_append_bio(struct request_queue *q, struct request *rq, | |||
18 | void blk_dequeue_request(struct request *rq); | 18 | void blk_dequeue_request(struct request *rq); |
19 | void __blk_queue_free_tags(struct request_queue *q); | 19 | void __blk_queue_free_tags(struct request_queue *q); |
20 | 20 | ||
21 | void blk_unplug_work(struct work_struct *work); | ||
22 | void blk_unplug_timeout(unsigned long data); | ||
23 | void blk_rq_timed_out_timer(unsigned long data); | 21 | void blk_rq_timed_out_timer(unsigned long data); |
24 | void blk_delete_timer(struct request *); | 22 | void blk_delete_timer(struct request *); |
25 | void blk_add_timer(struct request *); | 23 | void blk_add_timer(struct request *); |
@@ -51,21 +49,17 @@ static inline void blk_clear_rq_complete(struct request *rq) | |||
51 | */ | 49 | */ |
52 | #define ELV_ON_HASH(rq) (!hlist_unhashed(&(rq)->hash)) | 50 | #define ELV_ON_HASH(rq) (!hlist_unhashed(&(rq)->hash)) |
53 | 51 | ||
54 | struct request *blk_do_flush(struct request_queue *q, struct request *rq); | 52 | void blk_insert_flush(struct request *rq); |
53 | void blk_abort_flushes(struct request_queue *q); | ||
55 | 54 | ||
56 | static inline struct request *__elv_next_request(struct request_queue *q) | 55 | static inline struct request *__elv_next_request(struct request_queue *q) |
57 | { | 56 | { |
58 | struct request *rq; | 57 | struct request *rq; |
59 | 58 | ||
60 | while (1) { | 59 | while (1) { |
61 | while (!list_empty(&q->queue_head)) { | 60 | if (!list_empty(&q->queue_head)) { |
62 | rq = list_entry_rq(q->queue_head.next); | 61 | rq = list_entry_rq(q->queue_head.next); |
63 | if (!(rq->cmd_flags & (REQ_FLUSH | REQ_FUA)) || | 62 | return rq; |
64 | rq == &q->flush_rq) | ||
65 | return rq; | ||
66 | rq = blk_do_flush(q, rq); | ||
67 | if (rq) | ||
68 | return rq; | ||
69 | } | 63 | } |
70 | 64 | ||
71 | if (!q->elevator->ops->elevator_dispatch_fn(q, 0)) | 65 | if (!q->elevator->ops->elevator_dispatch_fn(q, 0)) |
@@ -109,6 +103,8 @@ int ll_front_merge_fn(struct request_queue *q, struct request *req, | |||
109 | struct bio *bio); | 103 | struct bio *bio); |
110 | int attempt_back_merge(struct request_queue *q, struct request *rq); | 104 | int attempt_back_merge(struct request_queue *q, struct request *rq); |
111 | int attempt_front_merge(struct request_queue *q, struct request *rq); | 105 | int attempt_front_merge(struct request_queue *q, struct request *rq); |
106 | int blk_attempt_req_merge(struct request_queue *q, struct request *rq, | ||
107 | struct request *next); | ||
112 | void blk_recalc_rq_segments(struct request *rq); | 108 | void blk_recalc_rq_segments(struct request *rq); |
113 | void blk_rq_set_mixed_merge(struct request *rq); | 109 | void blk_rq_set_mixed_merge(struct request *rq); |
114 | 110 | ||
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index ea83a4f0c27d..7785169f3c8f 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c | |||
@@ -54,9 +54,9 @@ static const int cfq_hist_divisor = 4; | |||
54 | #define CFQQ_SEEKY(cfqq) (hweight32(cfqq->seek_history) > 32/8) | 54 | #define CFQQ_SEEKY(cfqq) (hweight32(cfqq->seek_history) > 32/8) |
55 | 55 | ||
56 | #define RQ_CIC(rq) \ | 56 | #define RQ_CIC(rq) \ |
57 | ((struct cfq_io_context *) (rq)->elevator_private) | 57 | ((struct cfq_io_context *) (rq)->elevator_private[0]) |
58 | #define RQ_CFQQ(rq) (struct cfq_queue *) ((rq)->elevator_private2) | 58 | #define RQ_CFQQ(rq) (struct cfq_queue *) ((rq)->elevator_private[1]) |
59 | #define RQ_CFQG(rq) (struct cfq_group *) ((rq)->elevator_private3) | 59 | #define RQ_CFQG(rq) (struct cfq_group *) ((rq)->elevator_private[2]) |
60 | 60 | ||
61 | static struct kmem_cache *cfq_pool; | 61 | static struct kmem_cache *cfq_pool; |
62 | static struct kmem_cache *cfq_ioc_pool; | 62 | static struct kmem_cache *cfq_ioc_pool; |
@@ -146,7 +146,6 @@ struct cfq_queue { | |||
146 | struct cfq_rb_root *service_tree; | 146 | struct cfq_rb_root *service_tree; |
147 | struct cfq_queue *new_cfqq; | 147 | struct cfq_queue *new_cfqq; |
148 | struct cfq_group *cfqg; | 148 | struct cfq_group *cfqg; |
149 | struct cfq_group *orig_cfqg; | ||
150 | /* Number of sectors dispatched from queue in single dispatch round */ | 149 | /* Number of sectors dispatched from queue in single dispatch round */ |
151 | unsigned long nr_sectors; | 150 | unsigned long nr_sectors; |
152 | }; | 151 | }; |
@@ -179,6 +178,8 @@ struct cfq_group { | |||
179 | /* group service_tree key */ | 178 | /* group service_tree key */ |
180 | u64 vdisktime; | 179 | u64 vdisktime; |
181 | unsigned int weight; | 180 | unsigned int weight; |
181 | unsigned int new_weight; | ||
182 | bool needs_update; | ||
182 | 183 | ||
183 | /* number of cfqq currently on this group */ | 184 | /* number of cfqq currently on this group */ |
184 | int nr_cfqq; | 185 | int nr_cfqq; |
@@ -238,6 +239,7 @@ struct cfq_data { | |||
238 | struct rb_root prio_trees[CFQ_PRIO_LISTS]; | 239 | struct rb_root prio_trees[CFQ_PRIO_LISTS]; |
239 | 240 | ||
240 | unsigned int busy_queues; | 241 | unsigned int busy_queues; |
242 | unsigned int busy_sync_queues; | ||
241 | 243 | ||
242 | int rq_in_driver; | 244 | int rq_in_driver; |
243 | int rq_in_flight[2]; | 245 | int rq_in_flight[2]; |
@@ -285,7 +287,6 @@ struct cfq_data { | |||
285 | unsigned int cfq_slice_idle; | 287 | unsigned int cfq_slice_idle; |
286 | unsigned int cfq_group_idle; | 288 | unsigned int cfq_group_idle; |
287 | unsigned int cfq_latency; | 289 | unsigned int cfq_latency; |
288 | unsigned int cfq_group_isolation; | ||
289 | 290 | ||
290 | unsigned int cic_index; | 291 | unsigned int cic_index; |
291 | struct list_head cic_list; | 292 | struct list_head cic_list; |
@@ -501,13 +502,6 @@ static inline void cfq_schedule_dispatch(struct cfq_data *cfqd) | |||
501 | } | 502 | } |
502 | } | 503 | } |
503 | 504 | ||
504 | static int cfq_queue_empty(struct request_queue *q) | ||
505 | { | ||
506 | struct cfq_data *cfqd = q->elevator->elevator_data; | ||
507 | |||
508 | return !cfqd->rq_queued; | ||
509 | } | ||
510 | |||
511 | /* | 505 | /* |
512 | * Scale schedule slice based on io priority. Use the sync time slice only | 506 | * Scale schedule slice based on io priority. Use the sync time slice only |
513 | * if a queue is marked sync and has sync io queued. A sync queue with async | 507 | * if a queue is marked sync and has sync io queued. A sync queue with async |
@@ -558,15 +552,13 @@ static inline u64 min_vdisktime(u64 min_vdisktime, u64 vdisktime) | |||
558 | 552 | ||
559 | static void update_min_vdisktime(struct cfq_rb_root *st) | 553 | static void update_min_vdisktime(struct cfq_rb_root *st) |
560 | { | 554 | { |
561 | u64 vdisktime = st->min_vdisktime; | ||
562 | struct cfq_group *cfqg; | 555 | struct cfq_group *cfqg; |
563 | 556 | ||
564 | if (st->left) { | 557 | if (st->left) { |
565 | cfqg = rb_entry_cfqg(st->left); | 558 | cfqg = rb_entry_cfqg(st->left); |
566 | vdisktime = min_vdisktime(vdisktime, cfqg->vdisktime); | 559 | st->min_vdisktime = max_vdisktime(st->min_vdisktime, |
560 | cfqg->vdisktime); | ||
567 | } | 561 | } |
568 | |||
569 | st->min_vdisktime = max_vdisktime(st->min_vdisktime, vdisktime); | ||
570 | } | 562 | } |
571 | 563 | ||
572 | /* | 564 | /* |
@@ -863,7 +855,27 @@ __cfq_group_service_tree_add(struct cfq_rb_root *st, struct cfq_group *cfqg) | |||
863 | } | 855 | } |
864 | 856 | ||
865 | static void | 857 | static void |
866 | cfq_group_service_tree_add(struct cfq_data *cfqd, struct cfq_group *cfqg) | 858 | cfq_update_group_weight(struct cfq_group *cfqg) |
859 | { | ||
860 | BUG_ON(!RB_EMPTY_NODE(&cfqg->rb_node)); | ||
861 | if (cfqg->needs_update) { | ||
862 | cfqg->weight = cfqg->new_weight; | ||
863 | cfqg->needs_update = false; | ||
864 | } | ||
865 | } | ||
866 | |||
867 | static void | ||
868 | cfq_group_service_tree_add(struct cfq_rb_root *st, struct cfq_group *cfqg) | ||
869 | { | ||
870 | BUG_ON(!RB_EMPTY_NODE(&cfqg->rb_node)); | ||
871 | |||
872 | cfq_update_group_weight(cfqg); | ||
873 | __cfq_group_service_tree_add(st, cfqg); | ||
874 | st->total_weight += cfqg->weight; | ||
875 | } | ||
876 | |||
877 | static void | ||
878 | cfq_group_notify_queue_add(struct cfq_data *cfqd, struct cfq_group *cfqg) | ||
867 | { | 879 | { |
868 | struct cfq_rb_root *st = &cfqd->grp_service_tree; | 880 | struct cfq_rb_root *st = &cfqd->grp_service_tree; |
869 | struct cfq_group *__cfqg; | 881 | struct cfq_group *__cfqg; |
@@ -884,13 +896,19 @@ cfq_group_service_tree_add(struct cfq_data *cfqd, struct cfq_group *cfqg) | |||
884 | cfqg->vdisktime = __cfqg->vdisktime + CFQ_IDLE_DELAY; | 896 | cfqg->vdisktime = __cfqg->vdisktime + CFQ_IDLE_DELAY; |
885 | } else | 897 | } else |
886 | cfqg->vdisktime = st->min_vdisktime; | 898 | cfqg->vdisktime = st->min_vdisktime; |
899 | cfq_group_service_tree_add(st, cfqg); | ||
900 | } | ||
887 | 901 | ||
888 | __cfq_group_service_tree_add(st, cfqg); | 902 | static void |
889 | st->total_weight += cfqg->weight; | 903 | cfq_group_service_tree_del(struct cfq_rb_root *st, struct cfq_group *cfqg) |
904 | { | ||
905 | st->total_weight -= cfqg->weight; | ||
906 | if (!RB_EMPTY_NODE(&cfqg->rb_node)) | ||
907 | cfq_rb_erase(&cfqg->rb_node, st); | ||
890 | } | 908 | } |
891 | 909 | ||
892 | static void | 910 | static void |
893 | cfq_group_service_tree_del(struct cfq_data *cfqd, struct cfq_group *cfqg) | 911 | cfq_group_notify_queue_del(struct cfq_data *cfqd, struct cfq_group *cfqg) |
894 | { | 912 | { |
895 | struct cfq_rb_root *st = &cfqd->grp_service_tree; | 913 | struct cfq_rb_root *st = &cfqd->grp_service_tree; |
896 | 914 | ||
@@ -902,14 +920,13 @@ cfq_group_service_tree_del(struct cfq_data *cfqd, struct cfq_group *cfqg) | |||
902 | return; | 920 | return; |
903 | 921 | ||
904 | cfq_log_cfqg(cfqd, cfqg, "del_from_rr group"); | 922 | cfq_log_cfqg(cfqd, cfqg, "del_from_rr group"); |
905 | st->total_weight -= cfqg->weight; | 923 | cfq_group_service_tree_del(st, cfqg); |
906 | if (!RB_EMPTY_NODE(&cfqg->rb_node)) | ||
907 | cfq_rb_erase(&cfqg->rb_node, st); | ||
908 | cfqg->saved_workload_slice = 0; | 924 | cfqg->saved_workload_slice = 0; |
909 | cfq_blkiocg_update_dequeue_stats(&cfqg->blkg, 1); | 925 | cfq_blkiocg_update_dequeue_stats(&cfqg->blkg, 1); |
910 | } | 926 | } |
911 | 927 | ||
912 | static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq) | 928 | static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq, |
929 | unsigned int *unaccounted_time) | ||
913 | { | 930 | { |
914 | unsigned int slice_used; | 931 | unsigned int slice_used; |
915 | 932 | ||
@@ -928,8 +945,13 @@ static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq) | |||
928 | 1); | 945 | 1); |
929 | } else { | 946 | } else { |
930 | slice_used = jiffies - cfqq->slice_start; | 947 | slice_used = jiffies - cfqq->slice_start; |
931 | if (slice_used > cfqq->allocated_slice) | 948 | if (slice_used > cfqq->allocated_slice) { |
949 | *unaccounted_time = slice_used - cfqq->allocated_slice; | ||
932 | slice_used = cfqq->allocated_slice; | 950 | slice_used = cfqq->allocated_slice; |
951 | } | ||
952 | if (time_after(cfqq->slice_start, cfqq->dispatch_start)) | ||
953 | *unaccounted_time += cfqq->slice_start - | ||
954 | cfqq->dispatch_start; | ||
933 | } | 955 | } |
934 | 956 | ||
935 | return slice_used; | 957 | return slice_used; |
@@ -939,12 +961,12 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg, | |||
939 | struct cfq_queue *cfqq) | 961 | struct cfq_queue *cfqq) |
940 | { | 962 | { |
941 | struct cfq_rb_root *st = &cfqd->grp_service_tree; | 963 | struct cfq_rb_root *st = &cfqd->grp_service_tree; |
942 | unsigned int used_sl, charge; | 964 | unsigned int used_sl, charge, unaccounted_sl = 0; |
943 | int nr_sync = cfqg->nr_cfqq - cfqg_busy_async_queues(cfqd, cfqg) | 965 | int nr_sync = cfqg->nr_cfqq - cfqg_busy_async_queues(cfqd, cfqg) |
944 | - cfqg->service_tree_idle.count; | 966 | - cfqg->service_tree_idle.count; |
945 | 967 | ||
946 | BUG_ON(nr_sync < 0); | 968 | BUG_ON(nr_sync < 0); |
947 | used_sl = charge = cfq_cfqq_slice_usage(cfqq); | 969 | used_sl = charge = cfq_cfqq_slice_usage(cfqq, &unaccounted_sl); |
948 | 970 | ||
949 | if (iops_mode(cfqd)) | 971 | if (iops_mode(cfqd)) |
950 | charge = cfqq->slice_dispatch; | 972 | charge = cfqq->slice_dispatch; |
@@ -952,9 +974,10 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg, | |||
952 | charge = cfqq->allocated_slice; | 974 | charge = cfqq->allocated_slice; |
953 | 975 | ||
954 | /* Can't update vdisktime while group is on service tree */ | 976 | /* Can't update vdisktime while group is on service tree */ |
955 | cfq_rb_erase(&cfqg->rb_node, st); | 977 | cfq_group_service_tree_del(st, cfqg); |
956 | cfqg->vdisktime += cfq_scale_slice(charge, cfqg); | 978 | cfqg->vdisktime += cfq_scale_slice(charge, cfqg); |
957 | __cfq_group_service_tree_add(st, cfqg); | 979 | /* If a new weight was requested, update now, off tree */ |
980 | cfq_group_service_tree_add(st, cfqg); | ||
958 | 981 | ||
959 | /* This group is being expired. Save the context */ | 982 | /* This group is being expired. Save the context */ |
960 | if (time_after(cfqd->workload_expires, jiffies)) { | 983 | if (time_after(cfqd->workload_expires, jiffies)) { |
@@ -970,7 +993,8 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg, | |||
970 | cfq_log_cfqq(cfqq->cfqd, cfqq, "sl_used=%u disp=%u charge=%u iops=%u" | 993 | cfq_log_cfqq(cfqq->cfqd, cfqq, "sl_used=%u disp=%u charge=%u iops=%u" |
971 | " sect=%u", used_sl, cfqq->slice_dispatch, charge, | 994 | " sect=%u", used_sl, cfqq->slice_dispatch, charge, |
972 | iops_mode(cfqd), cfqq->nr_sectors); | 995 | iops_mode(cfqd), cfqq->nr_sectors); |
973 | cfq_blkiocg_update_timeslice_used(&cfqg->blkg, used_sl); | 996 | cfq_blkiocg_update_timeslice_used(&cfqg->blkg, used_sl, |
997 | unaccounted_sl); | ||
974 | cfq_blkiocg_set_start_empty_time(&cfqg->blkg); | 998 | cfq_blkiocg_set_start_empty_time(&cfqg->blkg); |
975 | } | 999 | } |
976 | 1000 | ||
@@ -985,7 +1009,9 @@ static inline struct cfq_group *cfqg_of_blkg(struct blkio_group *blkg) | |||
985 | void cfq_update_blkio_group_weight(void *key, struct blkio_group *blkg, | 1009 | void cfq_update_blkio_group_weight(void *key, struct blkio_group *blkg, |
986 | unsigned int weight) | 1010 | unsigned int weight) |
987 | { | 1011 | { |
988 | cfqg_of_blkg(blkg)->weight = weight; | 1012 | struct cfq_group *cfqg = cfqg_of_blkg(blkg); |
1013 | cfqg->new_weight = weight; | ||
1014 | cfqg->needs_update = true; | ||
989 | } | 1015 | } |
990 | 1016 | ||
991 | static struct cfq_group * | 1017 | static struct cfq_group * |
@@ -1187,32 +1213,6 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, | |||
1187 | int new_cfqq = 1; | 1213 | int new_cfqq = 1; |
1188 | int group_changed = 0; | 1214 | int group_changed = 0; |
1189 | 1215 | ||
1190 | #ifdef CONFIG_CFQ_GROUP_IOSCHED | ||
1191 | if (!cfqd->cfq_group_isolation | ||
1192 | && cfqq_type(cfqq) == SYNC_NOIDLE_WORKLOAD | ||
1193 | && cfqq->cfqg && cfqq->cfqg != &cfqd->root_group) { | ||
1194 | /* Move this cfq to root group */ | ||
1195 | cfq_log_cfqq(cfqd, cfqq, "moving to root group"); | ||
1196 | if (!RB_EMPTY_NODE(&cfqq->rb_node)) | ||
1197 | cfq_group_service_tree_del(cfqd, cfqq->cfqg); | ||
1198 | cfqq->orig_cfqg = cfqq->cfqg; | ||
1199 | cfqq->cfqg = &cfqd->root_group; | ||
1200 | cfqd->root_group.ref++; | ||
1201 | group_changed = 1; | ||
1202 | } else if (!cfqd->cfq_group_isolation | ||
1203 | && cfqq_type(cfqq) == SYNC_WORKLOAD && cfqq->orig_cfqg) { | ||
1204 | /* cfqq is sequential now needs to go to its original group */ | ||
1205 | BUG_ON(cfqq->cfqg != &cfqd->root_group); | ||
1206 | if (!RB_EMPTY_NODE(&cfqq->rb_node)) | ||
1207 | cfq_group_service_tree_del(cfqd, cfqq->cfqg); | ||
1208 | cfq_put_cfqg(cfqq->cfqg); | ||
1209 | cfqq->cfqg = cfqq->orig_cfqg; | ||
1210 | cfqq->orig_cfqg = NULL; | ||
1211 | group_changed = 1; | ||
1212 | cfq_log_cfqq(cfqd, cfqq, "moved to origin group"); | ||
1213 | } | ||
1214 | #endif | ||
1215 | |||
1216 | service_tree = service_tree_for(cfqq->cfqg, cfqq_prio(cfqq), | 1216 | service_tree = service_tree_for(cfqq->cfqg, cfqq_prio(cfqq), |
1217 | cfqq_type(cfqq)); | 1217 | cfqq_type(cfqq)); |
1218 | if (cfq_class_idle(cfqq)) { | 1218 | if (cfq_class_idle(cfqq)) { |
@@ -1284,7 +1284,7 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, | |||
1284 | service_tree->count++; | 1284 | service_tree->count++; |
1285 | if ((add_front || !new_cfqq) && !group_changed) | 1285 | if ((add_front || !new_cfqq) && !group_changed) |
1286 | return; | 1286 | return; |
1287 | cfq_group_service_tree_add(cfqd, cfqq->cfqg); | 1287 | cfq_group_notify_queue_add(cfqd, cfqq->cfqg); |
1288 | } | 1288 | } |
1289 | 1289 | ||
1290 | static struct cfq_queue * | 1290 | static struct cfq_queue * |
@@ -1372,6 +1372,8 @@ static void cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) | |||
1372 | BUG_ON(cfq_cfqq_on_rr(cfqq)); | 1372 | BUG_ON(cfq_cfqq_on_rr(cfqq)); |
1373 | cfq_mark_cfqq_on_rr(cfqq); | 1373 | cfq_mark_cfqq_on_rr(cfqq); |
1374 | cfqd->busy_queues++; | 1374 | cfqd->busy_queues++; |
1375 | if (cfq_cfqq_sync(cfqq)) | ||
1376 | cfqd->busy_sync_queues++; | ||
1375 | 1377 | ||
1376 | cfq_resort_rr_list(cfqd, cfqq); | 1378 | cfq_resort_rr_list(cfqd, cfqq); |
1377 | } | 1379 | } |
@@ -1395,9 +1397,11 @@ static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) | |||
1395 | cfqq->p_root = NULL; | 1397 | cfqq->p_root = NULL; |
1396 | } | 1398 | } |
1397 | 1399 | ||
1398 | cfq_group_service_tree_del(cfqd, cfqq->cfqg); | 1400 | cfq_group_notify_queue_del(cfqd, cfqq->cfqg); |
1399 | BUG_ON(!cfqd->busy_queues); | 1401 | BUG_ON(!cfqd->busy_queues); |
1400 | cfqd->busy_queues--; | 1402 | cfqd->busy_queues--; |
1403 | if (cfq_cfqq_sync(cfqq)) | ||
1404 | cfqd->busy_sync_queues--; | ||
1401 | } | 1405 | } |
1402 | 1406 | ||
1403 | /* | 1407 | /* |
@@ -2405,6 +2409,7 @@ static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq) | |||
2405 | * Does this cfqq already have too much IO in flight? | 2409 | * Does this cfqq already have too much IO in flight? |
2406 | */ | 2410 | */ |
2407 | if (cfqq->dispatched >= max_dispatch) { | 2411 | if (cfqq->dispatched >= max_dispatch) { |
2412 | bool promote_sync = false; | ||
2408 | /* | 2413 | /* |
2409 | * idle queue must always only have a single IO in flight | 2414 | * idle queue must always only have a single IO in flight |
2410 | */ | 2415 | */ |
@@ -2412,15 +2417,26 @@ static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq) | |||
2412 | return false; | 2417 | return false; |
2413 | 2418 | ||
2414 | /* | 2419 | /* |
2420 | * If there is only one sync queue | ||
2421 | * we can ignore async queue here and give the sync | ||
2422 | * queue no dispatch limit. The reason is a sync queue can | ||
2423 | * preempt async queue, limiting the sync queue doesn't make | ||
2424 | * sense. This is useful for aiostress test. | ||
2425 | */ | ||
2426 | if (cfq_cfqq_sync(cfqq) && cfqd->busy_sync_queues == 1) | ||
2427 | promote_sync = true; | ||
2428 | |||
2429 | /* | ||
2415 | * We have other queues, don't allow more IO from this one | 2430 | * We have other queues, don't allow more IO from this one |
2416 | */ | 2431 | */ |
2417 | if (cfqd->busy_queues > 1 && cfq_slice_used_soon(cfqd, cfqq)) | 2432 | if (cfqd->busy_queues > 1 && cfq_slice_used_soon(cfqd, cfqq) && |
2433 | !promote_sync) | ||
2418 | return false; | 2434 | return false; |
2419 | 2435 | ||
2420 | /* | 2436 | /* |
2421 | * Sole queue user, no limit | 2437 | * Sole queue user, no limit |
2422 | */ | 2438 | */ |
2423 | if (cfqd->busy_queues == 1) | 2439 | if (cfqd->busy_queues == 1 || promote_sync) |
2424 | max_dispatch = -1; | 2440 | max_dispatch = -1; |
2425 | else | 2441 | else |
2426 | /* | 2442 | /* |
@@ -2542,7 +2558,7 @@ static int cfq_dispatch_requests(struct request_queue *q, int force) | |||
2542 | static void cfq_put_queue(struct cfq_queue *cfqq) | 2558 | static void cfq_put_queue(struct cfq_queue *cfqq) |
2543 | { | 2559 | { |
2544 | struct cfq_data *cfqd = cfqq->cfqd; | 2560 | struct cfq_data *cfqd = cfqq->cfqd; |
2545 | struct cfq_group *cfqg, *orig_cfqg; | 2561 | struct cfq_group *cfqg; |
2546 | 2562 | ||
2547 | BUG_ON(cfqq->ref <= 0); | 2563 | BUG_ON(cfqq->ref <= 0); |
2548 | 2564 | ||
@@ -2554,7 +2570,6 @@ static void cfq_put_queue(struct cfq_queue *cfqq) | |||
2554 | BUG_ON(rb_first(&cfqq->sort_list)); | 2570 | BUG_ON(rb_first(&cfqq->sort_list)); |
2555 | BUG_ON(cfqq->allocated[READ] + cfqq->allocated[WRITE]); | 2571 | BUG_ON(cfqq->allocated[READ] + cfqq->allocated[WRITE]); |
2556 | cfqg = cfqq->cfqg; | 2572 | cfqg = cfqq->cfqg; |
2557 | orig_cfqg = cfqq->orig_cfqg; | ||
2558 | 2573 | ||
2559 | if (unlikely(cfqd->active_queue == cfqq)) { | 2574 | if (unlikely(cfqd->active_queue == cfqq)) { |
2560 | __cfq_slice_expired(cfqd, cfqq, 0); | 2575 | __cfq_slice_expired(cfqd, cfqq, 0); |
@@ -2564,8 +2579,6 @@ static void cfq_put_queue(struct cfq_queue *cfqq) | |||
2564 | BUG_ON(cfq_cfqq_on_rr(cfqq)); | 2579 | BUG_ON(cfq_cfqq_on_rr(cfqq)); |
2565 | kmem_cache_free(cfq_pool, cfqq); | 2580 | kmem_cache_free(cfq_pool, cfqq); |
2566 | cfq_put_cfqg(cfqg); | 2581 | cfq_put_cfqg(cfqg); |
2567 | if (orig_cfqg) | ||
2568 | cfq_put_cfqg(orig_cfqg); | ||
2569 | } | 2582 | } |
2570 | 2583 | ||
2571 | /* | 2584 | /* |
@@ -3613,12 +3626,12 @@ static void cfq_put_request(struct request *rq) | |||
3613 | 3626 | ||
3614 | put_io_context(RQ_CIC(rq)->ioc); | 3627 | put_io_context(RQ_CIC(rq)->ioc); |
3615 | 3628 | ||
3616 | rq->elevator_private = NULL; | 3629 | rq->elevator_private[0] = NULL; |
3617 | rq->elevator_private2 = NULL; | 3630 | rq->elevator_private[1] = NULL; |
3618 | 3631 | ||
3619 | /* Put down rq reference on cfqg */ | 3632 | /* Put down rq reference on cfqg */ |
3620 | cfq_put_cfqg(RQ_CFQG(rq)); | 3633 | cfq_put_cfqg(RQ_CFQG(rq)); |
3621 | rq->elevator_private3 = NULL; | 3634 | rq->elevator_private[2] = NULL; |
3622 | 3635 | ||
3623 | cfq_put_queue(cfqq); | 3636 | cfq_put_queue(cfqq); |
3624 | } | 3637 | } |
@@ -3705,13 +3718,12 @@ new_queue: | |||
3705 | } | 3718 | } |
3706 | 3719 | ||
3707 | cfqq->allocated[rw]++; | 3720 | cfqq->allocated[rw]++; |
3708 | cfqq->ref++; | ||
3709 | rq->elevator_private = cic; | ||
3710 | rq->elevator_private2 = cfqq; | ||
3711 | rq->elevator_private3 = cfq_ref_get_cfqg(cfqq->cfqg); | ||
3712 | 3721 | ||
3722 | cfqq->ref++; | ||
3723 | rq->elevator_private[0] = cic; | ||
3724 | rq->elevator_private[1] = cfqq; | ||
3725 | rq->elevator_private[2] = cfq_ref_get_cfqg(cfqq->cfqg); | ||
3713 | spin_unlock_irqrestore(q->queue_lock, flags); | 3726 | spin_unlock_irqrestore(q->queue_lock, flags); |
3714 | |||
3715 | return 0; | 3727 | return 0; |
3716 | 3728 | ||
3717 | queue_fail: | 3729 | queue_fail: |
@@ -3953,7 +3965,6 @@ static void *cfq_init_queue(struct request_queue *q) | |||
3953 | cfqd->cfq_slice_idle = cfq_slice_idle; | 3965 | cfqd->cfq_slice_idle = cfq_slice_idle; |
3954 | cfqd->cfq_group_idle = cfq_group_idle; | 3966 | cfqd->cfq_group_idle = cfq_group_idle; |
3955 | cfqd->cfq_latency = 1; | 3967 | cfqd->cfq_latency = 1; |
3956 | cfqd->cfq_group_isolation = 0; | ||
3957 | cfqd->hw_tag = -1; | 3968 | cfqd->hw_tag = -1; |
3958 | /* | 3969 | /* |
3959 | * we optimistically start assuming sync ops weren't delayed in last | 3970 | * we optimistically start assuming sync ops weren't delayed in last |
@@ -4029,7 +4040,6 @@ SHOW_FUNCTION(cfq_slice_sync_show, cfqd->cfq_slice[1], 1); | |||
4029 | SHOW_FUNCTION(cfq_slice_async_show, cfqd->cfq_slice[0], 1); | 4040 | SHOW_FUNCTION(cfq_slice_async_show, cfqd->cfq_slice[0], 1); |
4030 | SHOW_FUNCTION(cfq_slice_async_rq_show, cfqd->cfq_slice_async_rq, 0); | 4041 | SHOW_FUNCTION(cfq_slice_async_rq_show, cfqd->cfq_slice_async_rq, 0); |
4031 | SHOW_FUNCTION(cfq_low_latency_show, cfqd->cfq_latency, 0); | 4042 | SHOW_FUNCTION(cfq_low_latency_show, cfqd->cfq_latency, 0); |
4032 | SHOW_FUNCTION(cfq_group_isolation_show, cfqd->cfq_group_isolation, 0); | ||
4033 | #undef SHOW_FUNCTION | 4043 | #undef SHOW_FUNCTION |
4034 | 4044 | ||
4035 | #define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \ | 4045 | #define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \ |
@@ -4063,7 +4073,6 @@ STORE_FUNCTION(cfq_slice_async_store, &cfqd->cfq_slice[0], 1, UINT_MAX, 1); | |||
4063 | STORE_FUNCTION(cfq_slice_async_rq_store, &cfqd->cfq_slice_async_rq, 1, | 4073 | STORE_FUNCTION(cfq_slice_async_rq_store, &cfqd->cfq_slice_async_rq, 1, |
4064 | UINT_MAX, 0); | 4074 | UINT_MAX, 0); |
4065 | STORE_FUNCTION(cfq_low_latency_store, &cfqd->cfq_latency, 0, 1, 0); | 4075 | STORE_FUNCTION(cfq_low_latency_store, &cfqd->cfq_latency, 0, 1, 0); |
4066 | STORE_FUNCTION(cfq_group_isolation_store, &cfqd->cfq_group_isolation, 0, 1, 0); | ||
4067 | #undef STORE_FUNCTION | 4076 | #undef STORE_FUNCTION |
4068 | 4077 | ||
4069 | #define CFQ_ATTR(name) \ | 4078 | #define CFQ_ATTR(name) \ |
@@ -4081,7 +4090,6 @@ static struct elv_fs_entry cfq_attrs[] = { | |||
4081 | CFQ_ATTR(slice_idle), | 4090 | CFQ_ATTR(slice_idle), |
4082 | CFQ_ATTR(group_idle), | 4091 | CFQ_ATTR(group_idle), |
4083 | CFQ_ATTR(low_latency), | 4092 | CFQ_ATTR(low_latency), |
4084 | CFQ_ATTR(group_isolation), | ||
4085 | __ATTR_NULL | 4093 | __ATTR_NULL |
4086 | }; | 4094 | }; |
4087 | 4095 | ||
@@ -4096,7 +4104,6 @@ static struct elevator_type iosched_cfq = { | |||
4096 | .elevator_add_req_fn = cfq_insert_request, | 4104 | .elevator_add_req_fn = cfq_insert_request, |
4097 | .elevator_activate_req_fn = cfq_activate_request, | 4105 | .elevator_activate_req_fn = cfq_activate_request, |
4098 | .elevator_deactivate_req_fn = cfq_deactivate_request, | 4106 | .elevator_deactivate_req_fn = cfq_deactivate_request, |
4099 | .elevator_queue_empty_fn = cfq_queue_empty, | ||
4100 | .elevator_completed_req_fn = cfq_completed_request, | 4107 | .elevator_completed_req_fn = cfq_completed_request, |
4101 | .elevator_former_req_fn = elv_rb_former_request, | 4108 | .elevator_former_req_fn = elv_rb_former_request, |
4102 | .elevator_latter_req_fn = elv_rb_latter_request, | 4109 | .elevator_latter_req_fn = elv_rb_latter_request, |
diff --git a/block/cfq.h b/block/cfq.h index 54a6d90f8e8c..2a155927e37c 100644 --- a/block/cfq.h +++ b/block/cfq.h | |||
@@ -16,9 +16,9 @@ static inline void cfq_blkiocg_update_dequeue_stats(struct blkio_group *blkg, | |||
16 | } | 16 | } |
17 | 17 | ||
18 | static inline void cfq_blkiocg_update_timeslice_used(struct blkio_group *blkg, | 18 | static inline void cfq_blkiocg_update_timeslice_used(struct blkio_group *blkg, |
19 | unsigned long time) | 19 | unsigned long time, unsigned long unaccounted_time) |
20 | { | 20 | { |
21 | blkiocg_update_timeslice_used(blkg, time); | 21 | blkiocg_update_timeslice_used(blkg, time, unaccounted_time); |
22 | } | 22 | } |
23 | 23 | ||
24 | static inline void cfq_blkiocg_set_start_empty_time(struct blkio_group *blkg) | 24 | static inline void cfq_blkiocg_set_start_empty_time(struct blkio_group *blkg) |
@@ -85,7 +85,7 @@ static inline void cfq_blkiocg_update_dequeue_stats(struct blkio_group *blkg, | |||
85 | unsigned long dequeue) {} | 85 | unsigned long dequeue) {} |
86 | 86 | ||
87 | static inline void cfq_blkiocg_update_timeslice_used(struct blkio_group *blkg, | 87 | static inline void cfq_blkiocg_update_timeslice_used(struct blkio_group *blkg, |
88 | unsigned long time) {} | 88 | unsigned long time, unsigned long unaccounted_time) {} |
89 | static inline void cfq_blkiocg_set_start_empty_time(struct blkio_group *blkg) {} | 89 | static inline void cfq_blkiocg_set_start_empty_time(struct blkio_group *blkg) {} |
90 | static inline void cfq_blkiocg_update_io_remove_stats(struct blkio_group *blkg, | 90 | static inline void cfq_blkiocg_update_io_remove_stats(struct blkio_group *blkg, |
91 | bool direction, bool sync) {} | 91 | bool direction, bool sync) {} |
diff --git a/block/deadline-iosched.c b/block/deadline-iosched.c index b547cbca7b23..5139c0ea1864 100644 --- a/block/deadline-iosched.c +++ b/block/deadline-iosched.c | |||
@@ -326,14 +326,6 @@ dispatch_request: | |||
326 | return 1; | 326 | return 1; |
327 | } | 327 | } |
328 | 328 | ||
329 | static int deadline_queue_empty(struct request_queue *q) | ||
330 | { | ||
331 | struct deadline_data *dd = q->elevator->elevator_data; | ||
332 | |||
333 | return list_empty(&dd->fifo_list[WRITE]) | ||
334 | && list_empty(&dd->fifo_list[READ]); | ||
335 | } | ||
336 | |||
337 | static void deadline_exit_queue(struct elevator_queue *e) | 329 | static void deadline_exit_queue(struct elevator_queue *e) |
338 | { | 330 | { |
339 | struct deadline_data *dd = e->elevator_data; | 331 | struct deadline_data *dd = e->elevator_data; |
@@ -445,7 +437,6 @@ static struct elevator_type iosched_deadline = { | |||
445 | .elevator_merge_req_fn = deadline_merged_requests, | 437 | .elevator_merge_req_fn = deadline_merged_requests, |
446 | .elevator_dispatch_fn = deadline_dispatch_requests, | 438 | .elevator_dispatch_fn = deadline_dispatch_requests, |
447 | .elevator_add_req_fn = deadline_add_request, | 439 | .elevator_add_req_fn = deadline_add_request, |
448 | .elevator_queue_empty_fn = deadline_queue_empty, | ||
449 | .elevator_former_req_fn = elv_rb_former_request, | 440 | .elevator_former_req_fn = elv_rb_former_request, |
450 | .elevator_latter_req_fn = elv_rb_latter_request, | 441 | .elevator_latter_req_fn = elv_rb_latter_request, |
451 | .elevator_init_fn = deadline_init_queue, | 442 | .elevator_init_fn = deadline_init_queue, |
diff --git a/block/elevator.c b/block/elevator.c index 236e93c1f46c..c387d3168734 100644 --- a/block/elevator.c +++ b/block/elevator.c | |||
@@ -113,7 +113,7 @@ int elv_rq_merge_ok(struct request *rq, struct bio *bio) | |||
113 | } | 113 | } |
114 | EXPORT_SYMBOL(elv_rq_merge_ok); | 114 | EXPORT_SYMBOL(elv_rq_merge_ok); |
115 | 115 | ||
116 | static inline int elv_try_merge(struct request *__rq, struct bio *bio) | 116 | int elv_try_merge(struct request *__rq, struct bio *bio) |
117 | { | 117 | { |
118 | int ret = ELEVATOR_NO_MERGE; | 118 | int ret = ELEVATOR_NO_MERGE; |
119 | 119 | ||
@@ -421,6 +421,8 @@ void elv_dispatch_sort(struct request_queue *q, struct request *rq) | |||
421 | struct list_head *entry; | 421 | struct list_head *entry; |
422 | int stop_flags; | 422 | int stop_flags; |
423 | 423 | ||
424 | BUG_ON(rq->cmd_flags & REQ_ON_PLUG); | ||
425 | |||
424 | if (q->last_merge == rq) | 426 | if (q->last_merge == rq) |
425 | q->last_merge = NULL; | 427 | q->last_merge = NULL; |
426 | 428 | ||
@@ -519,6 +521,40 @@ int elv_merge(struct request_queue *q, struct request **req, struct bio *bio) | |||
519 | return ELEVATOR_NO_MERGE; | 521 | return ELEVATOR_NO_MERGE; |
520 | } | 522 | } |
521 | 523 | ||
524 | /* | ||
525 | * Attempt to do an insertion back merge. Only check for the case where | ||
526 | * we can append 'rq' to an existing request, so we can throw 'rq' away | ||
527 | * afterwards. | ||
528 | * | ||
529 | * Returns true if we merged, false otherwise | ||
530 | */ | ||
531 | static bool elv_attempt_insert_merge(struct request_queue *q, | ||
532 | struct request *rq) | ||
533 | { | ||
534 | struct request *__rq; | ||
535 | |||
536 | if (blk_queue_nomerges(q)) | ||
537 | return false; | ||
538 | |||
539 | /* | ||
540 | * First try one-hit cache. | ||
541 | */ | ||
542 | if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq)) | ||
543 | return true; | ||
544 | |||
545 | if (blk_queue_noxmerges(q)) | ||
546 | return false; | ||
547 | |||
548 | /* | ||
549 | * See if our hash lookup can find a potential backmerge. | ||
550 | */ | ||
551 | __rq = elv_rqhash_find(q, blk_rq_pos(rq)); | ||
552 | if (__rq && blk_attempt_req_merge(q, __rq, rq)) | ||
553 | return true; | ||
554 | |||
555 | return false; | ||
556 | } | ||
557 | |||
522 | void elv_merged_request(struct request_queue *q, struct request *rq, int type) | 558 | void elv_merged_request(struct request_queue *q, struct request *rq, int type) |
523 | { | 559 | { |
524 | struct elevator_queue *e = q->elevator; | 560 | struct elevator_queue *e = q->elevator; |
@@ -536,14 +572,18 @@ void elv_merge_requests(struct request_queue *q, struct request *rq, | |||
536 | struct request *next) | 572 | struct request *next) |
537 | { | 573 | { |
538 | struct elevator_queue *e = q->elevator; | 574 | struct elevator_queue *e = q->elevator; |
575 | const int next_sorted = next->cmd_flags & REQ_SORTED; | ||
539 | 576 | ||
540 | if (e->ops->elevator_merge_req_fn) | 577 | if (next_sorted && e->ops->elevator_merge_req_fn) |
541 | e->ops->elevator_merge_req_fn(q, rq, next); | 578 | e->ops->elevator_merge_req_fn(q, rq, next); |
542 | 579 | ||
543 | elv_rqhash_reposition(q, rq); | 580 | elv_rqhash_reposition(q, rq); |
544 | elv_rqhash_del(q, next); | ||
545 | 581 | ||
546 | q->nr_sorted--; | 582 | if (next_sorted) { |
583 | elv_rqhash_del(q, next); | ||
584 | q->nr_sorted--; | ||
585 | } | ||
586 | |||
547 | q->last_merge = rq; | 587 | q->last_merge = rq; |
548 | } | 588 | } |
549 | 589 | ||
@@ -617,21 +657,12 @@ void elv_quiesce_end(struct request_queue *q) | |||
617 | 657 | ||
618 | void elv_insert(struct request_queue *q, struct request *rq, int where) | 658 | void elv_insert(struct request_queue *q, struct request *rq, int where) |
619 | { | 659 | { |
620 | int unplug_it = 1; | ||
621 | |||
622 | trace_block_rq_insert(q, rq); | 660 | trace_block_rq_insert(q, rq); |
623 | 661 | ||
624 | rq->q = q; | 662 | rq->q = q; |
625 | 663 | ||
626 | switch (where) { | 664 | switch (where) { |
627 | case ELEVATOR_INSERT_REQUEUE: | 665 | case ELEVATOR_INSERT_REQUEUE: |
628 | /* | ||
629 | * Most requeues happen because of a busy condition, | ||
630 | * don't force unplug of the queue for that case. | ||
631 | * Clear unplug_it and fall through. | ||
632 | */ | ||
633 | unplug_it = 0; | ||
634 | |||
635 | case ELEVATOR_INSERT_FRONT: | 666 | case ELEVATOR_INSERT_FRONT: |
636 | rq->cmd_flags |= REQ_SOFTBARRIER; | 667 | rq->cmd_flags |= REQ_SOFTBARRIER; |
637 | list_add(&rq->queuelist, &q->queue_head); | 668 | list_add(&rq->queuelist, &q->queue_head); |
@@ -654,6 +685,14 @@ void elv_insert(struct request_queue *q, struct request *rq, int where) | |||
654 | __blk_run_queue(q, false); | 685 | __blk_run_queue(q, false); |
655 | break; | 686 | break; |
656 | 687 | ||
688 | case ELEVATOR_INSERT_SORT_MERGE: | ||
689 | /* | ||
690 | * If we succeed in merging this request with one in the | ||
691 | * queue already, we are done - rq has now been freed, | ||
692 | * so no need to do anything further. | ||
693 | */ | ||
694 | if (elv_attempt_insert_merge(q, rq)) | ||
695 | break; | ||
657 | case ELEVATOR_INSERT_SORT: | 696 | case ELEVATOR_INSERT_SORT: |
658 | BUG_ON(rq->cmd_type != REQ_TYPE_FS && | 697 | BUG_ON(rq->cmd_type != REQ_TYPE_FS && |
659 | !(rq->cmd_flags & REQ_DISCARD)); | 698 | !(rq->cmd_flags & REQ_DISCARD)); |
@@ -673,24 +712,21 @@ void elv_insert(struct request_queue *q, struct request *rq, int where) | |||
673 | q->elevator->ops->elevator_add_req_fn(q, rq); | 712 | q->elevator->ops->elevator_add_req_fn(q, rq); |
674 | break; | 713 | break; |
675 | 714 | ||
715 | case ELEVATOR_INSERT_FLUSH: | ||
716 | rq->cmd_flags |= REQ_SOFTBARRIER; | ||
717 | blk_insert_flush(rq); | ||
718 | break; | ||
676 | default: | 719 | default: |
677 | printk(KERN_ERR "%s: bad insertion point %d\n", | 720 | printk(KERN_ERR "%s: bad insertion point %d\n", |
678 | __func__, where); | 721 | __func__, where); |
679 | BUG(); | 722 | BUG(); |
680 | } | 723 | } |
681 | |||
682 | if (unplug_it && blk_queue_plugged(q)) { | ||
683 | int nrq = q->rq.count[BLK_RW_SYNC] + q->rq.count[BLK_RW_ASYNC] | ||
684 | - queue_in_flight(q); | ||
685 | |||
686 | if (nrq >= q->unplug_thresh) | ||
687 | __generic_unplug_device(q); | ||
688 | } | ||
689 | } | 724 | } |
690 | 725 | ||
691 | void __elv_add_request(struct request_queue *q, struct request *rq, int where, | 726 | void __elv_add_request(struct request_queue *q, struct request *rq, int where) |
692 | int plug) | ||
693 | { | 727 | { |
728 | BUG_ON(rq->cmd_flags & REQ_ON_PLUG); | ||
729 | |||
694 | if (rq->cmd_flags & REQ_SOFTBARRIER) { | 730 | if (rq->cmd_flags & REQ_SOFTBARRIER) { |
695 | /* barriers are scheduling boundary, update end_sector */ | 731 | /* barriers are scheduling boundary, update end_sector */ |
696 | if (rq->cmd_type == REQ_TYPE_FS || | 732 | if (rq->cmd_type == REQ_TYPE_FS || |
@@ -702,38 +738,20 @@ void __elv_add_request(struct request_queue *q, struct request *rq, int where, | |||
702 | where == ELEVATOR_INSERT_SORT) | 738 | where == ELEVATOR_INSERT_SORT) |
703 | where = ELEVATOR_INSERT_BACK; | 739 | where = ELEVATOR_INSERT_BACK; |
704 | 740 | ||
705 | if (plug) | ||
706 | blk_plug_device(q); | ||
707 | |||
708 | elv_insert(q, rq, where); | 741 | elv_insert(q, rq, where); |
709 | } | 742 | } |
710 | EXPORT_SYMBOL(__elv_add_request); | 743 | EXPORT_SYMBOL(__elv_add_request); |
711 | 744 | ||
712 | void elv_add_request(struct request_queue *q, struct request *rq, int where, | 745 | void elv_add_request(struct request_queue *q, struct request *rq, int where) |
713 | int plug) | ||
714 | { | 746 | { |
715 | unsigned long flags; | 747 | unsigned long flags; |
716 | 748 | ||
717 | spin_lock_irqsave(q->queue_lock, flags); | 749 | spin_lock_irqsave(q->queue_lock, flags); |
718 | __elv_add_request(q, rq, where, plug); | 750 | __elv_add_request(q, rq, where); |
719 | spin_unlock_irqrestore(q->queue_lock, flags); | 751 | spin_unlock_irqrestore(q->queue_lock, flags); |
720 | } | 752 | } |
721 | EXPORT_SYMBOL(elv_add_request); | 753 | EXPORT_SYMBOL(elv_add_request); |
722 | 754 | ||
723 | int elv_queue_empty(struct request_queue *q) | ||
724 | { | ||
725 | struct elevator_queue *e = q->elevator; | ||
726 | |||
727 | if (!list_empty(&q->queue_head)) | ||
728 | return 0; | ||
729 | |||
730 | if (e->ops->elevator_queue_empty_fn) | ||
731 | return e->ops->elevator_queue_empty_fn(q); | ||
732 | |||
733 | return 1; | ||
734 | } | ||
735 | EXPORT_SYMBOL(elv_queue_empty); | ||
736 | |||
737 | struct request *elv_latter_request(struct request_queue *q, struct request *rq) | 755 | struct request *elv_latter_request(struct request_queue *q, struct request *rq) |
738 | { | 756 | { |
739 | struct elevator_queue *e = q->elevator; | 757 | struct elevator_queue *e = q->elevator; |
@@ -759,7 +777,7 @@ int elv_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask) | |||
759 | if (e->ops->elevator_set_req_fn) | 777 | if (e->ops->elevator_set_req_fn) |
760 | return e->ops->elevator_set_req_fn(q, rq, gfp_mask); | 778 | return e->ops->elevator_set_req_fn(q, rq, gfp_mask); |
761 | 779 | ||
762 | rq->elevator_private = NULL; | 780 | rq->elevator_private[0] = NULL; |
763 | return 0; | 781 | return 0; |
764 | } | 782 | } |
765 | 783 | ||
@@ -785,6 +803,8 @@ void elv_abort_queue(struct request_queue *q) | |||
785 | { | 803 | { |
786 | struct request *rq; | 804 | struct request *rq; |
787 | 805 | ||
806 | blk_abort_flushes(q); | ||
807 | |||
788 | while (!list_empty(&q->queue_head)) { | 808 | while (!list_empty(&q->queue_head)) { |
789 | rq = list_entry_rq(q->queue_head.next); | 809 | rq = list_entry_rq(q->queue_head.next); |
790 | rq->cmd_flags |= REQ_QUIET; | 810 | rq->cmd_flags |= REQ_QUIET; |
diff --git a/block/genhd.c b/block/genhd.c index cbf1112a885c..c91a2dac6b6b 100644 --- a/block/genhd.c +++ b/block/genhd.c | |||
@@ -1158,14 +1158,14 @@ static int diskstats_show(struct seq_file *seqf, void *v) | |||
1158 | "%u %lu %lu %llu %u %u %u %u\n", | 1158 | "%u %lu %lu %llu %u %u %u %u\n", |
1159 | MAJOR(part_devt(hd)), MINOR(part_devt(hd)), | 1159 | MAJOR(part_devt(hd)), MINOR(part_devt(hd)), |
1160 | disk_name(gp, hd->partno, buf), | 1160 | disk_name(gp, hd->partno, buf), |
1161 | part_stat_read(hd, ios[0]), | 1161 | part_stat_read(hd, ios[READ]), |
1162 | part_stat_read(hd, merges[0]), | 1162 | part_stat_read(hd, merges[READ]), |
1163 | (unsigned long long)part_stat_read(hd, sectors[0]), | 1163 | (unsigned long long)part_stat_read(hd, sectors[READ]), |
1164 | jiffies_to_msecs(part_stat_read(hd, ticks[0])), | 1164 | jiffies_to_msecs(part_stat_read(hd, ticks[READ])), |
1165 | part_stat_read(hd, ios[1]), | 1165 | part_stat_read(hd, ios[WRITE]), |
1166 | part_stat_read(hd, merges[1]), | 1166 | part_stat_read(hd, merges[WRITE]), |
1167 | (unsigned long long)part_stat_read(hd, sectors[1]), | 1167 | (unsigned long long)part_stat_read(hd, sectors[WRITE]), |
1168 | jiffies_to_msecs(part_stat_read(hd, ticks[1])), | 1168 | jiffies_to_msecs(part_stat_read(hd, ticks[WRITE])), |
1169 | part_in_flight(hd), | 1169 | part_in_flight(hd), |
1170 | jiffies_to_msecs(part_stat_read(hd, io_ticks)), | 1170 | jiffies_to_msecs(part_stat_read(hd, io_ticks)), |
1171 | jiffies_to_msecs(part_stat_read(hd, time_in_queue)) | 1171 | jiffies_to_msecs(part_stat_read(hd, time_in_queue)) |
@@ -1494,7 +1494,7 @@ void disk_block_events(struct gendisk *disk) | |||
1494 | void disk_unblock_events(struct gendisk *disk) | 1494 | void disk_unblock_events(struct gendisk *disk) |
1495 | { | 1495 | { |
1496 | if (disk->ev) | 1496 | if (disk->ev) |
1497 | __disk_unblock_events(disk, true); | 1497 | __disk_unblock_events(disk, false); |
1498 | } | 1498 | } |
1499 | 1499 | ||
1500 | /** | 1500 | /** |
diff --git a/block/noop-iosched.c b/block/noop-iosched.c index 232c4b38cd37..06389e9ef96d 100644 --- a/block/noop-iosched.c +++ b/block/noop-iosched.c | |||
@@ -39,13 +39,6 @@ static void noop_add_request(struct request_queue *q, struct request *rq) | |||
39 | list_add_tail(&rq->queuelist, &nd->queue); | 39 | list_add_tail(&rq->queuelist, &nd->queue); |
40 | } | 40 | } |
41 | 41 | ||
42 | static int noop_queue_empty(struct request_queue *q) | ||
43 | { | ||
44 | struct noop_data *nd = q->elevator->elevator_data; | ||
45 | |||
46 | return list_empty(&nd->queue); | ||
47 | } | ||
48 | |||
49 | static struct request * | 42 | static struct request * |
50 | noop_former_request(struct request_queue *q, struct request *rq) | 43 | noop_former_request(struct request_queue *q, struct request *rq) |
51 | { | 44 | { |
@@ -90,7 +83,6 @@ static struct elevator_type elevator_noop = { | |||
90 | .elevator_merge_req_fn = noop_merged_requests, | 83 | .elevator_merge_req_fn = noop_merged_requests, |
91 | .elevator_dispatch_fn = noop_dispatch, | 84 | .elevator_dispatch_fn = noop_dispatch, |
92 | .elevator_add_req_fn = noop_add_request, | 85 | .elevator_add_req_fn = noop_add_request, |
93 | .elevator_queue_empty_fn = noop_queue_empty, | ||
94 | .elevator_former_req_fn = noop_former_request, | 86 | .elevator_former_req_fn = noop_former_request, |
95 | .elevator_latter_req_fn = noop_latter_request, | 87 | .elevator_latter_req_fn = noop_latter_request, |
96 | .elevator_init_fn = noop_init_queue, | 88 | .elevator_init_fn = noop_init_queue, |
diff --git a/drivers/block/DAC960.c b/drivers/block/DAC960.c index 1f286ab461d3..79882104e431 100644 --- a/drivers/block/DAC960.c +++ b/drivers/block/DAC960.c | |||
@@ -140,13 +140,14 @@ static int DAC960_getgeo(struct block_device *bdev, struct hd_geometry *geo) | |||
140 | return 0; | 140 | return 0; |
141 | } | 141 | } |
142 | 142 | ||
143 | static int DAC960_media_changed(struct gendisk *disk) | 143 | static unsigned int DAC960_check_events(struct gendisk *disk, |
144 | unsigned int clearing) | ||
144 | { | 145 | { |
145 | DAC960_Controller_T *p = disk->queue->queuedata; | 146 | DAC960_Controller_T *p = disk->queue->queuedata; |
146 | int drive_nr = (long)disk->private_data; | 147 | int drive_nr = (long)disk->private_data; |
147 | 148 | ||
148 | if (!p->LogicalDriveInitiallyAccessible[drive_nr]) | 149 | if (!p->LogicalDriveInitiallyAccessible[drive_nr]) |
149 | return 1; | 150 | return DISK_EVENT_MEDIA_CHANGE; |
150 | return 0; | 151 | return 0; |
151 | } | 152 | } |
152 | 153 | ||
@@ -163,7 +164,7 @@ static const struct block_device_operations DAC960_BlockDeviceOperations = { | |||
163 | .owner = THIS_MODULE, | 164 | .owner = THIS_MODULE, |
164 | .open = DAC960_open, | 165 | .open = DAC960_open, |
165 | .getgeo = DAC960_getgeo, | 166 | .getgeo = DAC960_getgeo, |
166 | .media_changed = DAC960_media_changed, | 167 | .check_events = DAC960_check_events, |
167 | .revalidate_disk = DAC960_revalidate_disk, | 168 | .revalidate_disk = DAC960_revalidate_disk, |
168 | }; | 169 | }; |
169 | 170 | ||
@@ -2546,6 +2547,7 @@ static bool DAC960_RegisterBlockDevice(DAC960_Controller_T *Controller) | |||
2546 | disk->major = MajorNumber; | 2547 | disk->major = MajorNumber; |
2547 | disk->first_minor = n << DAC960_MaxPartitionsBits; | 2548 | disk->first_minor = n << DAC960_MaxPartitionsBits; |
2548 | disk->fops = &DAC960_BlockDeviceOperations; | 2549 | disk->fops = &DAC960_BlockDeviceOperations; |
2550 | disk->events = DISK_EVENT_MEDIA_CHANGE; | ||
2549 | } | 2551 | } |
2550 | /* | 2552 | /* |
2551 | Indicate the Block Device Registration completed successfully, | 2553 | Indicate the Block Device Registration completed successfully, |
diff --git a/drivers/block/amiflop.c b/drivers/block/amiflop.c index 363855ca376e..456c0cc90dcf 100644 --- a/drivers/block/amiflop.c +++ b/drivers/block/amiflop.c | |||
@@ -1658,12 +1658,12 @@ static int floppy_release(struct gendisk *disk, fmode_t mode) | |||
1658 | } | 1658 | } |
1659 | 1659 | ||
1660 | /* | 1660 | /* |
1661 | * floppy-change is never called from an interrupt, so we can relax a bit | 1661 | * check_events is never called from an interrupt, so we can relax a bit |
1662 | * here, sleep etc. Note that floppy-on tries to set current_DOR to point | 1662 | * here, sleep etc. Note that floppy-on tries to set current_DOR to point |
1663 | * to the desired drive, but it will probably not survive the sleep if | 1663 | * to the desired drive, but it will probably not survive the sleep if |
1664 | * several floppies are used at the same time: thus the loop. | 1664 | * several floppies are used at the same time: thus the loop. |
1665 | */ | 1665 | */ |
1666 | static int amiga_floppy_change(struct gendisk *disk) | 1666 | static unsigned amiga_check_events(struct gendisk *disk, unsigned int clearing) |
1667 | { | 1667 | { |
1668 | struct amiga_floppy_struct *p = disk->private_data; | 1668 | struct amiga_floppy_struct *p = disk->private_data; |
1669 | int drive = p - unit; | 1669 | int drive = p - unit; |
@@ -1686,7 +1686,7 @@ static int amiga_floppy_change(struct gendisk *disk) | |||
1686 | p->dirty = 0; | 1686 | p->dirty = 0; |
1687 | writepending = 0; /* if this was true before, too bad! */ | 1687 | writepending = 0; /* if this was true before, too bad! */ |
1688 | writefromint = 0; | 1688 | writefromint = 0; |
1689 | return 1; | 1689 | return DISK_EVENT_MEDIA_CHANGE; |
1690 | } | 1690 | } |
1691 | return 0; | 1691 | return 0; |
1692 | } | 1692 | } |
@@ -1697,7 +1697,7 @@ static const struct block_device_operations floppy_fops = { | |||
1697 | .release = floppy_release, | 1697 | .release = floppy_release, |
1698 | .ioctl = fd_ioctl, | 1698 | .ioctl = fd_ioctl, |
1699 | .getgeo = fd_getgeo, | 1699 | .getgeo = fd_getgeo, |
1700 | .media_changed = amiga_floppy_change, | 1700 | .check_events = amiga_check_events, |
1701 | }; | 1701 | }; |
1702 | 1702 | ||
1703 | static int __init fd_probe_drives(void) | 1703 | static int __init fd_probe_drives(void) |
@@ -1736,6 +1736,7 @@ static int __init fd_probe_drives(void) | |||
1736 | disk->major = FLOPPY_MAJOR; | 1736 | disk->major = FLOPPY_MAJOR; |
1737 | disk->first_minor = drive; | 1737 | disk->first_minor = drive; |
1738 | disk->fops = &floppy_fops; | 1738 | disk->fops = &floppy_fops; |
1739 | disk->events = DISK_EVENT_MEDIA_CHANGE; | ||
1739 | sprintf(disk->disk_name, "fd%d", drive); | 1740 | sprintf(disk->disk_name, "fd%d", drive); |
1740 | disk->private_data = &unit[drive]; | 1741 | disk->private_data = &unit[drive]; |
1741 | set_capacity(disk, 880*2); | 1742 | set_capacity(disk, 880*2); |
diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c index 605a67e40bbf..c871eae14120 100644 --- a/drivers/block/ataflop.c +++ b/drivers/block/ataflop.c | |||
@@ -1324,23 +1324,24 @@ static void finish_fdc_done( int dummy ) | |||
1324 | * due to unrecognised disk changes. | 1324 | * due to unrecognised disk changes. |
1325 | */ | 1325 | */ |
1326 | 1326 | ||
1327 | static int check_floppy_change(struct gendisk *disk) | 1327 | static unsigned int floppy_check_events(struct gendisk *disk, |
1328 | unsigned int clearing) | ||
1328 | { | 1329 | { |
1329 | struct atari_floppy_struct *p = disk->private_data; | 1330 | struct atari_floppy_struct *p = disk->private_data; |
1330 | unsigned int drive = p - unit; | 1331 | unsigned int drive = p - unit; |
1331 | if (test_bit (drive, &fake_change)) { | 1332 | if (test_bit (drive, &fake_change)) { |
1332 | /* simulated change (e.g. after formatting) */ | 1333 | /* simulated change (e.g. after formatting) */ |
1333 | return 1; | 1334 | return DISK_EVENT_MEDIA_CHANGE; |
1334 | } | 1335 | } |
1335 | if (test_bit (drive, &changed_floppies)) { | 1336 | if (test_bit (drive, &changed_floppies)) { |
1336 | /* surely changed (the WP signal changed at least once) */ | 1337 | /* surely changed (the WP signal changed at least once) */ |
1337 | return 1; | 1338 | return DISK_EVENT_MEDIA_CHANGE; |
1338 | } | 1339 | } |
1339 | if (UD.wpstat) { | 1340 | if (UD.wpstat) { |
1340 | /* WP is on -> could be changed: to be sure, buffers should be | 1341 | /* WP is on -> could be changed: to be sure, buffers should be |
1341 | * invalidated... | 1342 | * invalidated... |
1342 | */ | 1343 | */ |
1343 | return 1; | 1344 | return DISK_EVENT_MEDIA_CHANGE; |
1344 | } | 1345 | } |
1345 | 1346 | ||
1346 | return 0; | 1347 | return 0; |
@@ -1570,7 +1571,7 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, | |||
1570 | * or the next access will revalidate - and clear UDT :-( | 1571 | * or the next access will revalidate - and clear UDT :-( |
1571 | */ | 1572 | */ |
1572 | 1573 | ||
1573 | if (check_floppy_change(disk)) | 1574 | if (floppy_check_events(disk, 0)) |
1574 | floppy_revalidate(disk); | 1575 | floppy_revalidate(disk); |
1575 | 1576 | ||
1576 | if (UD.flags & FTD_MSG) | 1577 | if (UD.flags & FTD_MSG) |
@@ -1904,7 +1905,7 @@ static const struct block_device_operations floppy_fops = { | |||
1904 | .open = floppy_unlocked_open, | 1905 | .open = floppy_unlocked_open, |
1905 | .release = floppy_release, | 1906 | .release = floppy_release, |
1906 | .ioctl = fd_ioctl, | 1907 | .ioctl = fd_ioctl, |
1907 | .media_changed = check_floppy_change, | 1908 | .check_events = floppy_check_events, |
1908 | .revalidate_disk= floppy_revalidate, | 1909 | .revalidate_disk= floppy_revalidate, |
1909 | }; | 1910 | }; |
1910 | 1911 | ||
@@ -1963,6 +1964,7 @@ static int __init atari_floppy_init (void) | |||
1963 | unit[i].disk->first_minor = i; | 1964 | unit[i].disk->first_minor = i; |
1964 | sprintf(unit[i].disk->disk_name, "fd%d", i); | 1965 | sprintf(unit[i].disk->disk_name, "fd%d", i); |
1965 | unit[i].disk->fops = &floppy_fops; | 1966 | unit[i].disk->fops = &floppy_fops; |
1967 | unit[i].disk->events = DISK_EVENT_MEDIA_CHANGE; | ||
1966 | unit[i].disk->private_data = &unit[i]; | 1968 | unit[i].disk->private_data = &unit[i]; |
1967 | unit[i].disk->queue = blk_init_queue(do_fd_request, | 1969 | unit[i].disk->queue = blk_init_queue(do_fd_request, |
1968 | &ataflop_lock); | 1970 | &ataflop_lock); |
diff --git a/drivers/block/cciss.c b/drivers/block/cciss.c index 9279272b3732..35658f445fca 100644 --- a/drivers/block/cciss.c +++ b/drivers/block/cciss.c | |||
@@ -3170,12 +3170,6 @@ static void do_cciss_request(struct request_queue *q) | |||
3170 | int sg_index = 0; | 3170 | int sg_index = 0; |
3171 | int chained = 0; | 3171 | int chained = 0; |
3172 | 3172 | ||
3173 | /* We call start_io here in case there is a command waiting on the | ||
3174 | * queue that has not been sent. | ||
3175 | */ | ||
3176 | if (blk_queue_plugged(q)) | ||
3177 | goto startio; | ||
3178 | |||
3179 | queue: | 3173 | queue: |
3180 | creq = blk_peek_request(q); | 3174 | creq = blk_peek_request(q); |
3181 | if (!creq) | 3175 | if (!creq) |
diff --git a/drivers/block/cpqarray.c b/drivers/block/cpqarray.c index 946dad4caef3..b2fceb53e809 100644 --- a/drivers/block/cpqarray.c +++ b/drivers/block/cpqarray.c | |||
@@ -911,9 +911,6 @@ static void do_ida_request(struct request_queue *q) | |||
911 | struct scatterlist tmp_sg[SG_MAX]; | 911 | struct scatterlist tmp_sg[SG_MAX]; |
912 | int i, dir, seg; | 912 | int i, dir, seg; |
913 | 913 | ||
914 | if (blk_queue_plugged(q)) | ||
915 | goto startio; | ||
916 | |||
917 | queue_next: | 914 | queue_next: |
918 | creq = blk_peek_request(q); | 915 | creq = blk_peek_request(q); |
919 | if (!creq) | 916 | if (!creq) |
diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c index ba95cba192be..aca302492ff2 100644 --- a/drivers/block/drbd/drbd_actlog.c +++ b/drivers/block/drbd/drbd_actlog.c | |||
@@ -80,7 +80,7 @@ static int _drbd_md_sync_page_io(struct drbd_conf *mdev, | |||
80 | 80 | ||
81 | if ((rw & WRITE) && !test_bit(MD_NO_FUA, &mdev->flags)) | 81 | if ((rw & WRITE) && !test_bit(MD_NO_FUA, &mdev->flags)) |
82 | rw |= REQ_FUA; | 82 | rw |= REQ_FUA; |
83 | rw |= REQ_UNPLUG | REQ_SYNC; | 83 | rw |= REQ_SYNC; |
84 | 84 | ||
85 | bio = bio_alloc(GFP_NOIO, 1); | 85 | bio = bio_alloc(GFP_NOIO, 1); |
86 | bio->bi_bdev = bdev->md_bdev; | 86 | bio->bi_bdev = bdev->md_bdev; |
@@ -689,8 +689,6 @@ void drbd_al_to_on_disk_bm(struct drbd_conf *mdev) | |||
689 | } | 689 | } |
690 | } | 690 | } |
691 | 691 | ||
692 | drbd_blk_run_queue(bdev_get_queue(mdev->ldev->md_bdev)); | ||
693 | |||
694 | /* always (try to) flush bitmap to stable storage */ | 692 | /* always (try to) flush bitmap to stable storage */ |
695 | drbd_md_flush(mdev); | 693 | drbd_md_flush(mdev); |
696 | 694 | ||
diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c index fd42832f785b..0645ca829a94 100644 --- a/drivers/block/drbd/drbd_bitmap.c +++ b/drivers/block/drbd/drbd_bitmap.c | |||
@@ -840,7 +840,6 @@ static int bm_rw(struct drbd_conf *mdev, int rw) __must_hold(local) | |||
840 | for (i = 0; i < num_pages; i++) | 840 | for (i = 0; i < num_pages; i++) |
841 | bm_page_io_async(mdev, b, i, rw); | 841 | bm_page_io_async(mdev, b, i, rw); |
842 | 842 | ||
843 | drbd_blk_run_queue(bdev_get_queue(mdev->ldev->md_bdev)); | ||
844 | wait_event(b->bm_io_wait, atomic_read(&b->bm_async_io) == 0); | 843 | wait_event(b->bm_io_wait, atomic_read(&b->bm_async_io) == 0); |
845 | 844 | ||
846 | if (test_bit(BM_MD_IO_ERROR, &b->bm_flags)) { | 845 | if (test_bit(BM_MD_IO_ERROR, &b->bm_flags)) { |
diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index 3803a0348937..b0bd27dfc1e8 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h | |||
@@ -377,7 +377,7 @@ union p_header { | |||
377 | #define DP_HARDBARRIER 1 /* depricated */ | 377 | #define DP_HARDBARRIER 1 /* depricated */ |
378 | #define DP_RW_SYNC 2 /* equals REQ_SYNC */ | 378 | #define DP_RW_SYNC 2 /* equals REQ_SYNC */ |
379 | #define DP_MAY_SET_IN_SYNC 4 | 379 | #define DP_MAY_SET_IN_SYNC 4 |
380 | #define DP_UNPLUG 8 /* equals REQ_UNPLUG */ | 380 | #define DP_UNPLUG 8 /* not used anymore */ |
381 | #define DP_FUA 16 /* equals REQ_FUA */ | 381 | #define DP_FUA 16 /* equals REQ_FUA */ |
382 | #define DP_FLUSH 32 /* equals REQ_FLUSH */ | 382 | #define DP_FLUSH 32 /* equals REQ_FLUSH */ |
383 | #define DP_DISCARD 64 /* equals REQ_DISCARD */ | 383 | #define DP_DISCARD 64 /* equals REQ_DISCARD */ |
@@ -2382,20 +2382,6 @@ static inline int drbd_queue_order_type(struct drbd_conf *mdev) | |||
2382 | return QUEUE_ORDERED_NONE; | 2382 | return QUEUE_ORDERED_NONE; |
2383 | } | 2383 | } |
2384 | 2384 | ||
2385 | static inline void drbd_blk_run_queue(struct request_queue *q) | ||
2386 | { | ||
2387 | if (q && q->unplug_fn) | ||
2388 | q->unplug_fn(q); | ||
2389 | } | ||
2390 | |||
2391 | static inline void drbd_kick_lo(struct drbd_conf *mdev) | ||
2392 | { | ||
2393 | if (get_ldev(mdev)) { | ||
2394 | drbd_blk_run_queue(bdev_get_queue(mdev->ldev->backing_bdev)); | ||
2395 | put_ldev(mdev); | ||
2396 | } | ||
2397 | } | ||
2398 | |||
2399 | static inline void drbd_md_flush(struct drbd_conf *mdev) | 2385 | static inline void drbd_md_flush(struct drbd_conf *mdev) |
2400 | { | 2386 | { |
2401 | int r; | 2387 | int r; |
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index 29cd0dc9fe4f..8a43ce0edeed 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c | |||
@@ -2477,12 +2477,11 @@ static u32 bio_flags_to_wire(struct drbd_conf *mdev, unsigned long bi_rw) | |||
2477 | { | 2477 | { |
2478 | if (mdev->agreed_pro_version >= 95) | 2478 | if (mdev->agreed_pro_version >= 95) |
2479 | return (bi_rw & REQ_SYNC ? DP_RW_SYNC : 0) | | 2479 | return (bi_rw & REQ_SYNC ? DP_RW_SYNC : 0) | |
2480 | (bi_rw & REQ_UNPLUG ? DP_UNPLUG : 0) | | ||
2481 | (bi_rw & REQ_FUA ? DP_FUA : 0) | | 2480 | (bi_rw & REQ_FUA ? DP_FUA : 0) | |
2482 | (bi_rw & REQ_FLUSH ? DP_FLUSH : 0) | | 2481 | (bi_rw & REQ_FLUSH ? DP_FLUSH : 0) | |
2483 | (bi_rw & REQ_DISCARD ? DP_DISCARD : 0); | 2482 | (bi_rw & REQ_DISCARD ? DP_DISCARD : 0); |
2484 | else | 2483 | else |
2485 | return bi_rw & (REQ_SYNC | REQ_UNPLUG) ? DP_RW_SYNC : 0; | 2484 | return bi_rw & REQ_SYNC ? DP_RW_SYNC : 0; |
2486 | } | 2485 | } |
2487 | 2486 | ||
2488 | /* Used to send write requests | 2487 | /* Used to send write requests |
@@ -2719,35 +2718,6 @@ static int drbd_release(struct gendisk *gd, fmode_t mode) | |||
2719 | return 0; | 2718 | return 0; |
2720 | } | 2719 | } |
2721 | 2720 | ||
2722 | static void drbd_unplug_fn(struct request_queue *q) | ||
2723 | { | ||
2724 | struct drbd_conf *mdev = q->queuedata; | ||
2725 | |||
2726 | /* unplug FIRST */ | ||
2727 | spin_lock_irq(q->queue_lock); | ||
2728 | blk_remove_plug(q); | ||
2729 | spin_unlock_irq(q->queue_lock); | ||
2730 | |||
2731 | /* only if connected */ | ||
2732 | spin_lock_irq(&mdev->req_lock); | ||
2733 | if (mdev->state.pdsk >= D_INCONSISTENT && mdev->state.conn >= C_CONNECTED) { | ||
2734 | D_ASSERT(mdev->state.role == R_PRIMARY); | ||
2735 | if (test_and_clear_bit(UNPLUG_REMOTE, &mdev->flags)) { | ||
2736 | /* add to the data.work queue, | ||
2737 | * unless already queued. | ||
2738 | * XXX this might be a good addition to drbd_queue_work | ||
2739 | * anyways, to detect "double queuing" ... */ | ||
2740 | if (list_empty(&mdev->unplug_work.list)) | ||
2741 | drbd_queue_work(&mdev->data.work, | ||
2742 | &mdev->unplug_work); | ||
2743 | } | ||
2744 | } | ||
2745 | spin_unlock_irq(&mdev->req_lock); | ||
2746 | |||
2747 | if (mdev->state.disk >= D_INCONSISTENT) | ||
2748 | drbd_kick_lo(mdev); | ||
2749 | } | ||
2750 | |||
2751 | static void drbd_set_defaults(struct drbd_conf *mdev) | 2721 | static void drbd_set_defaults(struct drbd_conf *mdev) |
2752 | { | 2722 | { |
2753 | /* This way we get a compile error when sync_conf grows, | 2723 | /* This way we get a compile error when sync_conf grows, |
@@ -3222,9 +3192,7 @@ struct drbd_conf *drbd_new_device(unsigned int minor) | |||
3222 | blk_queue_max_segment_size(q, DRBD_MAX_SEGMENT_SIZE); | 3192 | blk_queue_max_segment_size(q, DRBD_MAX_SEGMENT_SIZE); |
3223 | blk_queue_bounce_limit(q, BLK_BOUNCE_ANY); | 3193 | blk_queue_bounce_limit(q, BLK_BOUNCE_ANY); |
3224 | blk_queue_merge_bvec(q, drbd_merge_bvec); | 3194 | blk_queue_merge_bvec(q, drbd_merge_bvec); |
3225 | q->queue_lock = &mdev->req_lock; /* needed since we use */ | 3195 | q->queue_lock = &mdev->req_lock; |
3226 | /* plugging on a queue, that actually has no requests! */ | ||
3227 | q->unplug_fn = drbd_unplug_fn; | ||
3228 | 3196 | ||
3229 | mdev->md_io_page = alloc_page(GFP_KERNEL); | 3197 | mdev->md_io_page = alloc_page(GFP_KERNEL); |
3230 | if (!mdev->md_io_page) | 3198 | if (!mdev->md_io_page) |
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index 24487d4fb202..8e68be939deb 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c | |||
@@ -187,15 +187,6 @@ static struct page *drbd_pp_first_pages_or_try_alloc(struct drbd_conf *mdev, int | |||
187 | return NULL; | 187 | return NULL; |
188 | } | 188 | } |
189 | 189 | ||
190 | /* kick lower level device, if we have more than (arbitrary number) | ||
191 | * reference counts on it, which typically are locally submitted io | ||
192 | * requests. don't use unacked_cnt, so we speed up proto A and B, too. */ | ||
193 | static void maybe_kick_lo(struct drbd_conf *mdev) | ||
194 | { | ||
195 | if (atomic_read(&mdev->local_cnt) >= mdev->net_conf->unplug_watermark) | ||
196 | drbd_kick_lo(mdev); | ||
197 | } | ||
198 | |||
199 | static void reclaim_net_ee(struct drbd_conf *mdev, struct list_head *to_be_freed) | 190 | static void reclaim_net_ee(struct drbd_conf *mdev, struct list_head *to_be_freed) |
200 | { | 191 | { |
201 | struct drbd_epoch_entry *e; | 192 | struct drbd_epoch_entry *e; |
@@ -219,7 +210,6 @@ static void drbd_kick_lo_and_reclaim_net(struct drbd_conf *mdev) | |||
219 | LIST_HEAD(reclaimed); | 210 | LIST_HEAD(reclaimed); |
220 | struct drbd_epoch_entry *e, *t; | 211 | struct drbd_epoch_entry *e, *t; |
221 | 212 | ||
222 | maybe_kick_lo(mdev); | ||
223 | spin_lock_irq(&mdev->req_lock); | 213 | spin_lock_irq(&mdev->req_lock); |
224 | reclaim_net_ee(mdev, &reclaimed); | 214 | reclaim_net_ee(mdev, &reclaimed); |
225 | spin_unlock_irq(&mdev->req_lock); | 215 | spin_unlock_irq(&mdev->req_lock); |
@@ -436,8 +426,7 @@ void _drbd_wait_ee_list_empty(struct drbd_conf *mdev, struct list_head *head) | |||
436 | while (!list_empty(head)) { | 426 | while (!list_empty(head)) { |
437 | prepare_to_wait(&mdev->ee_wait, &wait, TASK_UNINTERRUPTIBLE); | 427 | prepare_to_wait(&mdev->ee_wait, &wait, TASK_UNINTERRUPTIBLE); |
438 | spin_unlock_irq(&mdev->req_lock); | 428 | spin_unlock_irq(&mdev->req_lock); |
439 | drbd_kick_lo(mdev); | 429 | io_schedule(); |
440 | schedule(); | ||
441 | finish_wait(&mdev->ee_wait, &wait); | 430 | finish_wait(&mdev->ee_wait, &wait); |
442 | spin_lock_irq(&mdev->req_lock); | 431 | spin_lock_irq(&mdev->req_lock); |
443 | } | 432 | } |
@@ -1111,8 +1100,6 @@ next_bio: | |||
1111 | /* > e->sector, unless this is the first bio */ | 1100 | /* > e->sector, unless this is the first bio */ |
1112 | bio->bi_sector = sector; | 1101 | bio->bi_sector = sector; |
1113 | bio->bi_bdev = mdev->ldev->backing_bdev; | 1102 | bio->bi_bdev = mdev->ldev->backing_bdev; |
1114 | /* we special case some flags in the multi-bio case, see below | ||
1115 | * (REQ_UNPLUG) */ | ||
1116 | bio->bi_rw = rw; | 1103 | bio->bi_rw = rw; |
1117 | bio->bi_private = e; | 1104 | bio->bi_private = e; |
1118 | bio->bi_end_io = drbd_endio_sec; | 1105 | bio->bi_end_io = drbd_endio_sec; |
@@ -1141,13 +1128,8 @@ next_bio: | |||
1141 | bios = bios->bi_next; | 1128 | bios = bios->bi_next; |
1142 | bio->bi_next = NULL; | 1129 | bio->bi_next = NULL; |
1143 | 1130 | ||
1144 | /* strip off REQ_UNPLUG unless it is the last bio */ | ||
1145 | if (bios) | ||
1146 | bio->bi_rw &= ~REQ_UNPLUG; | ||
1147 | |||
1148 | drbd_generic_make_request(mdev, fault_type, bio); | 1131 | drbd_generic_make_request(mdev, fault_type, bio); |
1149 | } while (bios); | 1132 | } while (bios); |
1150 | maybe_kick_lo(mdev); | ||
1151 | return 0; | 1133 | return 0; |
1152 | 1134 | ||
1153 | fail: | 1135 | fail: |
@@ -1167,9 +1149,6 @@ static int receive_Barrier(struct drbd_conf *mdev, enum drbd_packets cmd, unsign | |||
1167 | 1149 | ||
1168 | inc_unacked(mdev); | 1150 | inc_unacked(mdev); |
1169 | 1151 | ||
1170 | if (mdev->net_conf->wire_protocol != DRBD_PROT_C) | ||
1171 | drbd_kick_lo(mdev); | ||
1172 | |||
1173 | mdev->current_epoch->barrier_nr = p->barrier; | 1152 | mdev->current_epoch->barrier_nr = p->barrier; |
1174 | rv = drbd_may_finish_epoch(mdev, mdev->current_epoch, EV_GOT_BARRIER_NR); | 1153 | rv = drbd_may_finish_epoch(mdev, mdev->current_epoch, EV_GOT_BARRIER_NR); |
1175 | 1154 | ||
@@ -1636,12 +1615,11 @@ static unsigned long write_flags_to_bio(struct drbd_conf *mdev, u32 dpf) | |||
1636 | { | 1615 | { |
1637 | if (mdev->agreed_pro_version >= 95) | 1616 | if (mdev->agreed_pro_version >= 95) |
1638 | return (dpf & DP_RW_SYNC ? REQ_SYNC : 0) | | 1617 | return (dpf & DP_RW_SYNC ? REQ_SYNC : 0) | |
1639 | (dpf & DP_UNPLUG ? REQ_UNPLUG : 0) | | ||
1640 | (dpf & DP_FUA ? REQ_FUA : 0) | | 1618 | (dpf & DP_FUA ? REQ_FUA : 0) | |
1641 | (dpf & DP_FLUSH ? REQ_FUA : 0) | | 1619 | (dpf & DP_FLUSH ? REQ_FUA : 0) | |
1642 | (dpf & DP_DISCARD ? REQ_DISCARD : 0); | 1620 | (dpf & DP_DISCARD ? REQ_DISCARD : 0); |
1643 | else | 1621 | else |
1644 | return dpf & DP_RW_SYNC ? (REQ_SYNC | REQ_UNPLUG) : 0; | 1622 | return dpf & DP_RW_SYNC ? REQ_SYNC : 0; |
1645 | } | 1623 | } |
1646 | 1624 | ||
1647 | /* mirrored write */ | 1625 | /* mirrored write */ |
@@ -3556,9 +3534,6 @@ static int receive_skip(struct drbd_conf *mdev, enum drbd_packets cmd, unsigned | |||
3556 | 3534 | ||
3557 | static int receive_UnplugRemote(struct drbd_conf *mdev, enum drbd_packets cmd, unsigned int data_size) | 3535 | static int receive_UnplugRemote(struct drbd_conf *mdev, enum drbd_packets cmd, unsigned int data_size) |
3558 | { | 3536 | { |
3559 | if (mdev->state.disk >= D_INCONSISTENT) | ||
3560 | drbd_kick_lo(mdev); | ||
3561 | |||
3562 | /* Make sure we've acked all the TCP data associated | 3537 | /* Make sure we've acked all the TCP data associated |
3563 | * with the data requests being unplugged */ | 3538 | * with the data requests being unplugged */ |
3564 | drbd_tcp_quickack(mdev->data.socket); | 3539 | drbd_tcp_quickack(mdev->data.socket); |
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index 11a75d32a2e2..ad3fc6228f27 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c | |||
@@ -960,10 +960,6 @@ allocate_barrier: | |||
960 | bio_endio(req->private_bio, -EIO); | 960 | bio_endio(req->private_bio, -EIO); |
961 | } | 961 | } |
962 | 962 | ||
963 | /* we need to plug ALWAYS since we possibly need to kick lo_dev. | ||
964 | * we plug after submit, so we won't miss an unplug event */ | ||
965 | drbd_plug_device(mdev); | ||
966 | |||
967 | return 0; | 963 | return 0; |
968 | 964 | ||
969 | fail_conflicting: | 965 | fail_conflicting: |
diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c index 34f224b018b3..e027446590d3 100644 --- a/drivers/block/drbd/drbd_worker.c +++ b/drivers/block/drbd/drbd_worker.c | |||
@@ -792,7 +792,6 @@ int drbd_resync_finished(struct drbd_conf *mdev) | |||
792 | * queue (or even the read operations for those packets | 792 | * queue (or even the read operations for those packets |
793 | * is not finished by now). Retry in 100ms. */ | 793 | * is not finished by now). Retry in 100ms. */ |
794 | 794 | ||
795 | drbd_kick_lo(mdev); | ||
796 | __set_current_state(TASK_INTERRUPTIBLE); | 795 | __set_current_state(TASK_INTERRUPTIBLE); |
797 | schedule_timeout(HZ / 10); | 796 | schedule_timeout(HZ / 10); |
798 | w = kmalloc(sizeof(struct drbd_work), GFP_ATOMIC); | 797 | w = kmalloc(sizeof(struct drbd_work), GFP_ATOMIC); |
diff --git a/drivers/block/drbd/drbd_wrappers.h b/drivers/block/drbd/drbd_wrappers.h index defdb5013ea3..53586fa5ae1b 100644 --- a/drivers/block/drbd/drbd_wrappers.h +++ b/drivers/block/drbd/drbd_wrappers.h | |||
@@ -45,24 +45,6 @@ static inline void drbd_generic_make_request(struct drbd_conf *mdev, | |||
45 | generic_make_request(bio); | 45 | generic_make_request(bio); |
46 | } | 46 | } |
47 | 47 | ||
48 | static inline void drbd_plug_device(struct drbd_conf *mdev) | ||
49 | { | ||
50 | struct request_queue *q; | ||
51 | q = bdev_get_queue(mdev->this_bdev); | ||
52 | |||
53 | spin_lock_irq(q->queue_lock); | ||
54 | |||
55 | /* XXX the check on !blk_queue_plugged is redundant, | ||
56 | * implicitly checked in blk_plug_device */ | ||
57 | |||
58 | if (!blk_queue_plugged(q)) { | ||
59 | blk_plug_device(q); | ||
60 | del_timer(&q->unplug_timer); | ||
61 | /* unplugging should not happen automatically... */ | ||
62 | } | ||
63 | spin_unlock_irq(q->queue_lock); | ||
64 | } | ||
65 | |||
66 | static inline int drbd_crypto_is_hash(struct crypto_tfm *tfm) | 48 | static inline int drbd_crypto_is_hash(struct crypto_tfm *tfm) |
67 | { | 49 | { |
68 | return (crypto_tfm_alg_type(tfm) & CRYPTO_ALG_TYPE_HASH_MASK) | 50 | return (crypto_tfm_alg_type(tfm) & CRYPTO_ALG_TYPE_HASH_MASK) |
diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 77fc76f8aea9..301d7a9a41a6 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c | |||
@@ -3770,13 +3770,14 @@ out2: | |||
3770 | /* | 3770 | /* |
3771 | * Check if the disk has been changed or if a change has been faked. | 3771 | * Check if the disk has been changed or if a change has been faked. |
3772 | */ | 3772 | */ |
3773 | static int check_floppy_change(struct gendisk *disk) | 3773 | static unsigned int floppy_check_events(struct gendisk *disk, |
3774 | unsigned int clearing) | ||
3774 | { | 3775 | { |
3775 | int drive = (long)disk->private_data; | 3776 | int drive = (long)disk->private_data; |
3776 | 3777 | ||
3777 | if (test_bit(FD_DISK_CHANGED_BIT, &UDRS->flags) || | 3778 | if (test_bit(FD_DISK_CHANGED_BIT, &UDRS->flags) || |
3778 | test_bit(FD_VERIFY_BIT, &UDRS->flags)) | 3779 | test_bit(FD_VERIFY_BIT, &UDRS->flags)) |
3779 | return 1; | 3780 | return DISK_EVENT_MEDIA_CHANGE; |
3780 | 3781 | ||
3781 | if (time_after(jiffies, UDRS->last_checked + UDP->checkfreq)) { | 3782 | if (time_after(jiffies, UDRS->last_checked + UDP->checkfreq)) { |
3782 | lock_fdc(drive, false); | 3783 | lock_fdc(drive, false); |
@@ -3788,7 +3789,7 @@ static int check_floppy_change(struct gendisk *disk) | |||
3788 | test_bit(FD_VERIFY_BIT, &UDRS->flags) || | 3789 | test_bit(FD_VERIFY_BIT, &UDRS->flags) || |
3789 | test_bit(drive, &fake_change) || | 3790 | test_bit(drive, &fake_change) || |
3790 | drive_no_geom(drive)) | 3791 | drive_no_geom(drive)) |
3791 | return 1; | 3792 | return DISK_EVENT_MEDIA_CHANGE; |
3792 | return 0; | 3793 | return 0; |
3793 | } | 3794 | } |
3794 | 3795 | ||
@@ -3837,7 +3838,6 @@ static int __floppy_read_block_0(struct block_device *bdev) | |||
3837 | bio.bi_end_io = floppy_rb0_complete; | 3838 | bio.bi_end_io = floppy_rb0_complete; |
3838 | 3839 | ||
3839 | submit_bio(READ, &bio); | 3840 | submit_bio(READ, &bio); |
3840 | generic_unplug_device(bdev_get_queue(bdev)); | ||
3841 | process_fd_request(); | 3841 | process_fd_request(); |
3842 | wait_for_completion(&complete); | 3842 | wait_for_completion(&complete); |
3843 | 3843 | ||
@@ -3898,7 +3898,7 @@ static const struct block_device_operations floppy_fops = { | |||
3898 | .release = floppy_release, | 3898 | .release = floppy_release, |
3899 | .ioctl = fd_ioctl, | 3899 | .ioctl = fd_ioctl, |
3900 | .getgeo = fd_getgeo, | 3900 | .getgeo = fd_getgeo, |
3901 | .media_changed = check_floppy_change, | 3901 | .check_events = floppy_check_events, |
3902 | .revalidate_disk = floppy_revalidate, | 3902 | .revalidate_disk = floppy_revalidate, |
3903 | }; | 3903 | }; |
3904 | 3904 | ||
@@ -4205,6 +4205,7 @@ static int __init floppy_init(void) | |||
4205 | disks[dr]->major = FLOPPY_MAJOR; | 4205 | disks[dr]->major = FLOPPY_MAJOR; |
4206 | disks[dr]->first_minor = TOMINOR(dr); | 4206 | disks[dr]->first_minor = TOMINOR(dr); |
4207 | disks[dr]->fops = &floppy_fops; | 4207 | disks[dr]->fops = &floppy_fops; |
4208 | disks[dr]->events = DISK_EVENT_MEDIA_CHANGE; | ||
4208 | sprintf(disks[dr]->disk_name, "fd%d", dr); | 4209 | sprintf(disks[dr]->disk_name, "fd%d", dr); |
4209 | 4210 | ||
4210 | init_timer(&motor_off_timer[dr]); | 4211 | init_timer(&motor_off_timer[dr]); |
diff --git a/drivers/block/loop.c b/drivers/block/loop.c index dbf31ec9114d..a076a14ca72d 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c | |||
@@ -540,17 +540,6 @@ out: | |||
540 | return 0; | 540 | return 0; |
541 | } | 541 | } |
542 | 542 | ||
543 | /* | ||
544 | * kick off io on the underlying address space | ||
545 | */ | ||
546 | static void loop_unplug(struct request_queue *q) | ||
547 | { | ||
548 | struct loop_device *lo = q->queuedata; | ||
549 | |||
550 | queue_flag_clear_unlocked(QUEUE_FLAG_PLUGGED, q); | ||
551 | blk_run_address_space(lo->lo_backing_file->f_mapping); | ||
552 | } | ||
553 | |||
554 | struct switch_request { | 543 | struct switch_request { |
555 | struct file *file; | 544 | struct file *file; |
556 | struct completion wait; | 545 | struct completion wait; |
@@ -917,7 +906,6 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode, | |||
917 | */ | 906 | */ |
918 | blk_queue_make_request(lo->lo_queue, loop_make_request); | 907 | blk_queue_make_request(lo->lo_queue, loop_make_request); |
919 | lo->lo_queue->queuedata = lo; | 908 | lo->lo_queue->queuedata = lo; |
920 | lo->lo_queue->unplug_fn = loop_unplug; | ||
921 | 909 | ||
922 | if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync) | 910 | if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync) |
923 | blk_queue_flush(lo->lo_queue, REQ_FLUSH); | 911 | blk_queue_flush(lo->lo_queue, REQ_FLUSH); |
@@ -1019,7 +1007,6 @@ static int loop_clr_fd(struct loop_device *lo, struct block_device *bdev) | |||
1019 | 1007 | ||
1020 | kthread_stop(lo->lo_thread); | 1008 | kthread_stop(lo->lo_thread); |
1021 | 1009 | ||
1022 | lo->lo_queue->unplug_fn = NULL; | ||
1023 | lo->lo_backing_file = NULL; | 1010 | lo->lo_backing_file = NULL; |
1024 | 1011 | ||
1025 | loop_release_xfer(lo); | 1012 | loop_release_xfer(lo); |
@@ -1636,9 +1623,6 @@ out: | |||
1636 | 1623 | ||
1637 | static void loop_free(struct loop_device *lo) | 1624 | static void loop_free(struct loop_device *lo) |
1638 | { | 1625 | { |
1639 | if (!lo->lo_queue->queue_lock) | ||
1640 | lo->lo_queue->queue_lock = &lo->lo_queue->__queue_lock; | ||
1641 | |||
1642 | blk_cleanup_queue(lo->lo_queue); | 1626 | blk_cleanup_queue(lo->lo_queue); |
1643 | put_disk(lo->lo_disk); | 1627 | put_disk(lo->lo_disk); |
1644 | list_del(&lo->lo_list); | 1628 | list_del(&lo->lo_list); |
diff --git a/drivers/block/paride/pcd.c b/drivers/block/paride/pcd.c index 62cec6afd7ad..2f2ccf686251 100644 --- a/drivers/block/paride/pcd.c +++ b/drivers/block/paride/pcd.c | |||
@@ -172,7 +172,8 @@ module_param_array(drive3, int, NULL, 0); | |||
172 | static int pcd_open(struct cdrom_device_info *cdi, int purpose); | 172 | static int pcd_open(struct cdrom_device_info *cdi, int purpose); |
173 | static void pcd_release(struct cdrom_device_info *cdi); | 173 | static void pcd_release(struct cdrom_device_info *cdi); |
174 | static int pcd_drive_status(struct cdrom_device_info *cdi, int slot_nr); | 174 | static int pcd_drive_status(struct cdrom_device_info *cdi, int slot_nr); |
175 | static int pcd_media_changed(struct cdrom_device_info *cdi, int slot_nr); | 175 | static unsigned int pcd_check_events(struct cdrom_device_info *cdi, |
176 | unsigned int clearing, int slot_nr); | ||
176 | static int pcd_tray_move(struct cdrom_device_info *cdi, int position); | 177 | static int pcd_tray_move(struct cdrom_device_info *cdi, int position); |
177 | static int pcd_lock_door(struct cdrom_device_info *cdi, int lock); | 178 | static int pcd_lock_door(struct cdrom_device_info *cdi, int lock); |
178 | static int pcd_drive_reset(struct cdrom_device_info *cdi); | 179 | static int pcd_drive_reset(struct cdrom_device_info *cdi); |
@@ -257,10 +258,11 @@ static int pcd_block_ioctl(struct block_device *bdev, fmode_t mode, | |||
257 | return ret; | 258 | return ret; |
258 | } | 259 | } |
259 | 260 | ||
260 | static int pcd_block_media_changed(struct gendisk *disk) | 261 | static unsigned int pcd_block_check_events(struct gendisk *disk, |
262 | unsigned int clearing) | ||
261 | { | 263 | { |
262 | struct pcd_unit *cd = disk->private_data; | 264 | struct pcd_unit *cd = disk->private_data; |
263 | return cdrom_media_changed(&cd->info); | 265 | return cdrom_check_events(&cd->info, clearing); |
264 | } | 266 | } |
265 | 267 | ||
266 | static const struct block_device_operations pcd_bdops = { | 268 | static const struct block_device_operations pcd_bdops = { |
@@ -268,14 +270,14 @@ static const struct block_device_operations pcd_bdops = { | |||
268 | .open = pcd_block_open, | 270 | .open = pcd_block_open, |
269 | .release = pcd_block_release, | 271 | .release = pcd_block_release, |
270 | .ioctl = pcd_block_ioctl, | 272 | .ioctl = pcd_block_ioctl, |
271 | .media_changed = pcd_block_media_changed, | 273 | .check_events = pcd_block_check_events, |
272 | }; | 274 | }; |
273 | 275 | ||
274 | static struct cdrom_device_ops pcd_dops = { | 276 | static struct cdrom_device_ops pcd_dops = { |
275 | .open = pcd_open, | 277 | .open = pcd_open, |
276 | .release = pcd_release, | 278 | .release = pcd_release, |
277 | .drive_status = pcd_drive_status, | 279 | .drive_status = pcd_drive_status, |
278 | .media_changed = pcd_media_changed, | 280 | .check_events = pcd_check_events, |
279 | .tray_move = pcd_tray_move, | 281 | .tray_move = pcd_tray_move, |
280 | .lock_door = pcd_lock_door, | 282 | .lock_door = pcd_lock_door, |
281 | .get_mcn = pcd_get_mcn, | 283 | .get_mcn = pcd_get_mcn, |
@@ -318,6 +320,7 @@ static void pcd_init_units(void) | |||
318 | disk->first_minor = unit; | 320 | disk->first_minor = unit; |
319 | strcpy(disk->disk_name, cd->name); /* umm... */ | 321 | strcpy(disk->disk_name, cd->name); /* umm... */ |
320 | disk->fops = &pcd_bdops; | 322 | disk->fops = &pcd_bdops; |
323 | disk->events = DISK_EVENT_MEDIA_CHANGE; | ||
321 | } | 324 | } |
322 | } | 325 | } |
323 | 326 | ||
@@ -502,13 +505,14 @@ static int pcd_packet(struct cdrom_device_info *cdi, struct packet_command *cgc) | |||
502 | 505 | ||
503 | #define DBMSG(msg) ((verbose>1)?(msg):NULL) | 506 | #define DBMSG(msg) ((verbose>1)?(msg):NULL) |
504 | 507 | ||
505 | static int pcd_media_changed(struct cdrom_device_info *cdi, int slot_nr) | 508 | static unsigned int pcd_check_events(struct cdrom_device_info *cdi, |
509 | unsigned int clearing, int slot_nr) | ||
506 | { | 510 | { |
507 | struct pcd_unit *cd = cdi->handle; | 511 | struct pcd_unit *cd = cdi->handle; |
508 | int res = cd->changed; | 512 | int res = cd->changed; |
509 | if (res) | 513 | if (res) |
510 | cd->changed = 0; | 514 | cd->changed = 0; |
511 | return res; | 515 | return res ? DISK_EVENT_MEDIA_CHANGE : 0; |
512 | } | 516 | } |
513 | 517 | ||
514 | static int pcd_lock_door(struct cdrom_device_info *cdi, int lock) | 518 | static int pcd_lock_door(struct cdrom_device_info *cdi, int lock) |
diff --git a/drivers/block/paride/pd.c b/drivers/block/paride/pd.c index c0ee1558b9bb..21dfdb776869 100644 --- a/drivers/block/paride/pd.c +++ b/drivers/block/paride/pd.c | |||
@@ -794,7 +794,7 @@ static int pd_release(struct gendisk *p, fmode_t mode) | |||
794 | return 0; | 794 | return 0; |
795 | } | 795 | } |
796 | 796 | ||
797 | static int pd_check_media(struct gendisk *p) | 797 | static unsigned int pd_check_events(struct gendisk *p, unsigned int clearing) |
798 | { | 798 | { |
799 | struct pd_unit *disk = p->private_data; | 799 | struct pd_unit *disk = p->private_data; |
800 | int r; | 800 | int r; |
@@ -803,7 +803,7 @@ static int pd_check_media(struct gendisk *p) | |||
803 | pd_special_command(disk, pd_media_check); | 803 | pd_special_command(disk, pd_media_check); |
804 | r = disk->changed; | 804 | r = disk->changed; |
805 | disk->changed = 0; | 805 | disk->changed = 0; |
806 | return r; | 806 | return r ? DISK_EVENT_MEDIA_CHANGE : 0; |
807 | } | 807 | } |
808 | 808 | ||
809 | static int pd_revalidate(struct gendisk *p) | 809 | static int pd_revalidate(struct gendisk *p) |
@@ -822,7 +822,7 @@ static const struct block_device_operations pd_fops = { | |||
822 | .release = pd_release, | 822 | .release = pd_release, |
823 | .ioctl = pd_ioctl, | 823 | .ioctl = pd_ioctl, |
824 | .getgeo = pd_getgeo, | 824 | .getgeo = pd_getgeo, |
825 | .media_changed = pd_check_media, | 825 | .check_events = pd_check_events, |
826 | .revalidate_disk= pd_revalidate | 826 | .revalidate_disk= pd_revalidate |
827 | }; | 827 | }; |
828 | 828 | ||
@@ -837,6 +837,7 @@ static void pd_probe_drive(struct pd_unit *disk) | |||
837 | p->fops = &pd_fops; | 837 | p->fops = &pd_fops; |
838 | p->major = major; | 838 | p->major = major; |
839 | p->first_minor = (disk - pd) << PD_BITS; | 839 | p->first_minor = (disk - pd) << PD_BITS; |
840 | p->events = DISK_EVENT_MEDIA_CHANGE; | ||
840 | disk->gd = p; | 841 | disk->gd = p; |
841 | p->private_data = disk; | 842 | p->private_data = disk; |
842 | p->queue = pd_queue; | 843 | p->queue = pd_queue; |
diff --git a/drivers/block/paride/pf.c b/drivers/block/paride/pf.c index 635f25dd9e10..7adeb1edbf43 100644 --- a/drivers/block/paride/pf.c +++ b/drivers/block/paride/pf.c | |||
@@ -243,7 +243,8 @@ static struct pf_unit units[PF_UNITS]; | |||
243 | static int pf_identify(struct pf_unit *pf); | 243 | static int pf_identify(struct pf_unit *pf); |
244 | static void pf_lock(struct pf_unit *pf, int func); | 244 | static void pf_lock(struct pf_unit *pf, int func); |
245 | static void pf_eject(struct pf_unit *pf); | 245 | static void pf_eject(struct pf_unit *pf); |
246 | static int pf_check_media(struct gendisk *disk); | 246 | static unsigned int pf_check_events(struct gendisk *disk, |
247 | unsigned int clearing); | ||
247 | 248 | ||
248 | static char pf_scratch[512]; /* scratch block buffer */ | 249 | static char pf_scratch[512]; /* scratch block buffer */ |
249 | 250 | ||
@@ -270,7 +271,7 @@ static const struct block_device_operations pf_fops = { | |||
270 | .release = pf_release, | 271 | .release = pf_release, |
271 | .ioctl = pf_ioctl, | 272 | .ioctl = pf_ioctl, |
272 | .getgeo = pf_getgeo, | 273 | .getgeo = pf_getgeo, |
273 | .media_changed = pf_check_media, | 274 | .check_events = pf_check_events, |
274 | }; | 275 | }; |
275 | 276 | ||
276 | static void __init pf_init_units(void) | 277 | static void __init pf_init_units(void) |
@@ -293,6 +294,7 @@ static void __init pf_init_units(void) | |||
293 | disk->first_minor = unit; | 294 | disk->first_minor = unit; |
294 | strcpy(disk->disk_name, pf->name); | 295 | strcpy(disk->disk_name, pf->name); |
295 | disk->fops = &pf_fops; | 296 | disk->fops = &pf_fops; |
297 | disk->events = DISK_EVENT_MEDIA_CHANGE; | ||
296 | if (!(*drives[unit])[D_PRT]) | 298 | if (!(*drives[unit])[D_PRT]) |
297 | pf_drive_count++; | 299 | pf_drive_count++; |
298 | } | 300 | } |
@@ -377,9 +379,9 @@ static int pf_release(struct gendisk *disk, fmode_t mode) | |||
377 | 379 | ||
378 | } | 380 | } |
379 | 381 | ||
380 | static int pf_check_media(struct gendisk *disk) | 382 | static unsigned int pf_check_events(struct gendisk *disk, unsigned int clearing) |
381 | { | 383 | { |
382 | return 1; | 384 | return DISK_EVENT_MEDIA_CHANGE; |
383 | } | 385 | } |
384 | 386 | ||
385 | static inline int status_reg(struct pf_unit *pf) | 387 | static inline int status_reg(struct pf_unit *pf) |
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 77d70eebb6b2..07a382eaf0a8 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c | |||
@@ -1606,8 +1606,6 @@ static int kcdrwd(void *foobar) | |||
1606 | min_sleep_time = pkt->sleep_time; | 1606 | min_sleep_time = pkt->sleep_time; |
1607 | } | 1607 | } |
1608 | 1608 | ||
1609 | generic_unplug_device(bdev_get_queue(pd->bdev)); | ||
1610 | |||
1611 | VPRINTK("kcdrwd: sleeping\n"); | 1609 | VPRINTK("kcdrwd: sleeping\n"); |
1612 | residue = schedule_timeout(min_sleep_time); | 1610 | residue = schedule_timeout(min_sleep_time); |
1613 | VPRINTK("kcdrwd: wake up\n"); | 1611 | VPRINTK("kcdrwd: wake up\n"); |
@@ -2796,7 +2794,8 @@ static int pkt_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, | |||
2796 | return ret; | 2794 | return ret; |
2797 | } | 2795 | } |
2798 | 2796 | ||
2799 | static int pkt_media_changed(struct gendisk *disk) | 2797 | static unsigned int pkt_check_events(struct gendisk *disk, |
2798 | unsigned int clearing) | ||
2800 | { | 2799 | { |
2801 | struct pktcdvd_device *pd = disk->private_data; | 2800 | struct pktcdvd_device *pd = disk->private_data; |
2802 | struct gendisk *attached_disk; | 2801 | struct gendisk *attached_disk; |
@@ -2806,9 +2805,9 @@ static int pkt_media_changed(struct gendisk *disk) | |||
2806 | if (!pd->bdev) | 2805 | if (!pd->bdev) |
2807 | return 0; | 2806 | return 0; |
2808 | attached_disk = pd->bdev->bd_disk; | 2807 | attached_disk = pd->bdev->bd_disk; |
2809 | if (!attached_disk) | 2808 | if (!attached_disk || !attached_disk->fops->check_events) |
2810 | return 0; | 2809 | return 0; |
2811 | return attached_disk->fops->media_changed(attached_disk); | 2810 | return attached_disk->fops->check_events(attached_disk, clearing); |
2812 | } | 2811 | } |
2813 | 2812 | ||
2814 | static const struct block_device_operations pktcdvd_ops = { | 2813 | static const struct block_device_operations pktcdvd_ops = { |
@@ -2816,7 +2815,7 @@ static const struct block_device_operations pktcdvd_ops = { | |||
2816 | .open = pkt_open, | 2815 | .open = pkt_open, |
2817 | .release = pkt_close, | 2816 | .release = pkt_close, |
2818 | .ioctl = pkt_ioctl, | 2817 | .ioctl = pkt_ioctl, |
2819 | .media_changed = pkt_media_changed, | 2818 | .check_events = pkt_check_events, |
2820 | }; | 2819 | }; |
2821 | 2820 | ||
2822 | static char *pktcdvd_devnode(struct gendisk *gd, mode_t *mode) | 2821 | static char *pktcdvd_devnode(struct gendisk *gd, mode_t *mode) |
@@ -2889,6 +2888,10 @@ static int pkt_setup_dev(dev_t dev, dev_t* pkt_dev) | |||
2889 | if (ret) | 2888 | if (ret) |
2890 | goto out_new_dev; | 2889 | goto out_new_dev; |
2891 | 2890 | ||
2891 | /* inherit events of the host device */ | ||
2892 | disk->events = pd->bdev->bd_disk->events; | ||
2893 | disk->async_events = pd->bdev->bd_disk->async_events; | ||
2894 | |||
2892 | add_disk(disk); | 2895 | add_disk(disk); |
2893 | 2896 | ||
2894 | pkt_sysfs_dev_new(pd); | 2897 | pkt_sysfs_dev_new(pd); |
diff --git a/drivers/block/swim.c b/drivers/block/swim.c index 75333d0a3327..24a482f2fbd6 100644 --- a/drivers/block/swim.c +++ b/drivers/block/swim.c | |||
@@ -741,11 +741,12 @@ static int floppy_getgeo(struct block_device *bdev, struct hd_geometry *geo) | |||
741 | return 0; | 741 | return 0; |
742 | } | 742 | } |
743 | 743 | ||
744 | static int floppy_check_change(struct gendisk *disk) | 744 | static unsigned int floppy_check_events(struct gendisk *disk, |
745 | unsigned int clearing) | ||
745 | { | 746 | { |
746 | struct floppy_state *fs = disk->private_data; | 747 | struct floppy_state *fs = disk->private_data; |
747 | 748 | ||
748 | return fs->ejected; | 749 | return fs->ejected ? DISK_EVENT_MEDIA_CHANGE : 0; |
749 | } | 750 | } |
750 | 751 | ||
751 | static int floppy_revalidate(struct gendisk *disk) | 752 | static int floppy_revalidate(struct gendisk *disk) |
@@ -772,7 +773,7 @@ static const struct block_device_operations floppy_fops = { | |||
772 | .release = floppy_release, | 773 | .release = floppy_release, |
773 | .ioctl = floppy_ioctl, | 774 | .ioctl = floppy_ioctl, |
774 | .getgeo = floppy_getgeo, | 775 | .getgeo = floppy_getgeo, |
775 | .media_changed = floppy_check_change, | 776 | .check_events = floppy_check_events, |
776 | .revalidate_disk = floppy_revalidate, | 777 | .revalidate_disk = floppy_revalidate, |
777 | }; | 778 | }; |
778 | 779 | ||
@@ -857,6 +858,7 @@ static int __devinit swim_floppy_init(struct swim_priv *swd) | |||
857 | swd->unit[drive].disk->first_minor = drive; | 858 | swd->unit[drive].disk->first_minor = drive; |
858 | sprintf(swd->unit[drive].disk->disk_name, "fd%d", drive); | 859 | sprintf(swd->unit[drive].disk->disk_name, "fd%d", drive); |
859 | swd->unit[drive].disk->fops = &floppy_fops; | 860 | swd->unit[drive].disk->fops = &floppy_fops; |
861 | swd->unit[drive].disk->events = DISK_EVENT_MEDIA_CHANGE; | ||
860 | swd->unit[drive].disk->private_data = &swd->unit[drive]; | 862 | swd->unit[drive].disk->private_data = &swd->unit[drive]; |
861 | swd->unit[drive].disk->queue = swd->queue; | 863 | swd->unit[drive].disk->queue = swd->queue; |
862 | set_capacity(swd->unit[drive].disk, 2880); | 864 | set_capacity(swd->unit[drive].disk, 2880); |
diff --git a/drivers/block/swim3.c b/drivers/block/swim3.c index bf3a5b859299..4c10f56facbf 100644 --- a/drivers/block/swim3.c +++ b/drivers/block/swim3.c | |||
@@ -250,7 +250,8 @@ static int floppy_ioctl(struct block_device *bdev, fmode_t mode, | |||
250 | unsigned int cmd, unsigned long param); | 250 | unsigned int cmd, unsigned long param); |
251 | static int floppy_open(struct block_device *bdev, fmode_t mode); | 251 | static int floppy_open(struct block_device *bdev, fmode_t mode); |
252 | static int floppy_release(struct gendisk *disk, fmode_t mode); | 252 | static int floppy_release(struct gendisk *disk, fmode_t mode); |
253 | static int floppy_check_change(struct gendisk *disk); | 253 | static unsigned int floppy_check_events(struct gendisk *disk, |
254 | unsigned int clearing); | ||
254 | static int floppy_revalidate(struct gendisk *disk); | 255 | static int floppy_revalidate(struct gendisk *disk); |
255 | 256 | ||
256 | static bool swim3_end_request(int err, unsigned int nr_bytes) | 257 | static bool swim3_end_request(int err, unsigned int nr_bytes) |
@@ -975,10 +976,11 @@ static int floppy_release(struct gendisk *disk, fmode_t mode) | |||
975 | return 0; | 976 | return 0; |
976 | } | 977 | } |
977 | 978 | ||
978 | static int floppy_check_change(struct gendisk *disk) | 979 | static unsigned int floppy_check_events(struct gendisk *disk, |
980 | unsigned int clearing) | ||
979 | { | 981 | { |
980 | struct floppy_state *fs = disk->private_data; | 982 | struct floppy_state *fs = disk->private_data; |
981 | return fs->ejected; | 983 | return fs->ejected ? DISK_EVENT_MEDIA_CHANGE : 0; |
982 | } | 984 | } |
983 | 985 | ||
984 | static int floppy_revalidate(struct gendisk *disk) | 986 | static int floppy_revalidate(struct gendisk *disk) |
@@ -1025,7 +1027,7 @@ static const struct block_device_operations floppy_fops = { | |||
1025 | .open = floppy_unlocked_open, | 1027 | .open = floppy_unlocked_open, |
1026 | .release = floppy_release, | 1028 | .release = floppy_release, |
1027 | .ioctl = floppy_ioctl, | 1029 | .ioctl = floppy_ioctl, |
1028 | .media_changed = floppy_check_change, | 1030 | .check_events = floppy_check_events, |
1029 | .revalidate_disk= floppy_revalidate, | 1031 | .revalidate_disk= floppy_revalidate, |
1030 | }; | 1032 | }; |
1031 | 1033 | ||
@@ -1161,6 +1163,7 @@ static int __devinit swim3_attach(struct macio_dev *mdev, const struct of_device | |||
1161 | disk->major = FLOPPY_MAJOR; | 1163 | disk->major = FLOPPY_MAJOR; |
1162 | disk->first_minor = i; | 1164 | disk->first_minor = i; |
1163 | disk->fops = &floppy_fops; | 1165 | disk->fops = &floppy_fops; |
1166 | disk->events = DISK_EVENT_MEDIA_CHANGE; | ||
1164 | disk->private_data = &floppy_states[i]; | 1167 | disk->private_data = &floppy_states[i]; |
1165 | disk->queue = swim3_queue; | 1168 | disk->queue = swim3_queue; |
1166 | disk->flags |= GENHD_FL_REMOVABLE; | 1169 | disk->flags |= GENHD_FL_REMOVABLE; |
diff --git a/drivers/block/ub.c b/drivers/block/ub.c index 9ae3bb713286..68b9430c7cfe 100644 --- a/drivers/block/ub.c +++ b/drivers/block/ub.c | |||
@@ -1788,7 +1788,8 @@ static int ub_bd_revalidate(struct gendisk *disk) | |||
1788 | * | 1788 | * |
1789 | * The return code is bool! | 1789 | * The return code is bool! |
1790 | */ | 1790 | */ |
1791 | static int ub_bd_media_changed(struct gendisk *disk) | 1791 | static unsigned int ub_bd_check_events(struct gendisk *disk, |
1792 | unsigned int clearing) | ||
1792 | { | 1793 | { |
1793 | struct ub_lun *lun = disk->private_data; | 1794 | struct ub_lun *lun = disk->private_data; |
1794 | 1795 | ||
@@ -1806,10 +1807,10 @@ static int ub_bd_media_changed(struct gendisk *disk) | |||
1806 | */ | 1807 | */ |
1807 | if (ub_sync_tur(lun->udev, lun) != 0) { | 1808 | if (ub_sync_tur(lun->udev, lun) != 0) { |
1808 | lun->changed = 1; | 1809 | lun->changed = 1; |
1809 | return 1; | 1810 | return DISK_EVENT_MEDIA_CHANGE; |
1810 | } | 1811 | } |
1811 | 1812 | ||
1812 | return lun->changed; | 1813 | return lun->changed ? DISK_EVENT_MEDIA_CHANGE : 0; |
1813 | } | 1814 | } |
1814 | 1815 | ||
1815 | static const struct block_device_operations ub_bd_fops = { | 1816 | static const struct block_device_operations ub_bd_fops = { |
@@ -1817,7 +1818,7 @@ static const struct block_device_operations ub_bd_fops = { | |||
1817 | .open = ub_bd_unlocked_open, | 1818 | .open = ub_bd_unlocked_open, |
1818 | .release = ub_bd_release, | 1819 | .release = ub_bd_release, |
1819 | .ioctl = ub_bd_ioctl, | 1820 | .ioctl = ub_bd_ioctl, |
1820 | .media_changed = ub_bd_media_changed, | 1821 | .check_events = ub_bd_check_events, |
1821 | .revalidate_disk = ub_bd_revalidate, | 1822 | .revalidate_disk = ub_bd_revalidate, |
1822 | }; | 1823 | }; |
1823 | 1824 | ||
@@ -2333,6 +2334,7 @@ static int ub_probe_lun(struct ub_dev *sc, int lnum) | |||
2333 | disk->major = UB_MAJOR; | 2334 | disk->major = UB_MAJOR; |
2334 | disk->first_minor = lun->id * UB_PARTS_PER_LUN; | 2335 | disk->first_minor = lun->id * UB_PARTS_PER_LUN; |
2335 | disk->fops = &ub_bd_fops; | 2336 | disk->fops = &ub_bd_fops; |
2337 | disk->events = DISK_EVENT_MEDIA_CHANGE; | ||
2336 | disk->private_data = lun; | 2338 | disk->private_data = lun; |
2337 | disk->driverfs_dev = &sc->intf->dev; | 2339 | disk->driverfs_dev = &sc->intf->dev; |
2338 | 2340 | ||
diff --git a/drivers/block/umem.c b/drivers/block/umem.c index 8be57151f5d6..031ca720d926 100644 --- a/drivers/block/umem.c +++ b/drivers/block/umem.c | |||
@@ -241,8 +241,7 @@ static void dump_dmastat(struct cardinfo *card, unsigned int dmastat) | |||
241 | * | 241 | * |
242 | * Whenever IO on the active page completes, the Ready page is activated | 242 | * Whenever IO on the active page completes, the Ready page is activated |
243 | * and the ex-Active page is clean out and made Ready. | 243 | * and the ex-Active page is clean out and made Ready. |
244 | * Otherwise the Ready page is only activated when it becomes full, or | 244 | * Otherwise the Ready page is only activated when it becomes full. |
245 | * when mm_unplug_device is called via the unplug_io_fn. | ||
246 | * | 245 | * |
247 | * If a request arrives while both pages a full, it is queued, and b_rdev is | 246 | * If a request arrives while both pages a full, it is queued, and b_rdev is |
248 | * overloaded to record whether it was a read or a write. | 247 | * overloaded to record whether it was a read or a write. |
@@ -333,17 +332,6 @@ static inline void reset_page(struct mm_page *page) | |||
333 | page->biotail = &page->bio; | 332 | page->biotail = &page->bio; |
334 | } | 333 | } |
335 | 334 | ||
336 | static void mm_unplug_device(struct request_queue *q) | ||
337 | { | ||
338 | struct cardinfo *card = q->queuedata; | ||
339 | unsigned long flags; | ||
340 | |||
341 | spin_lock_irqsave(&card->lock, flags); | ||
342 | if (blk_remove_plug(q)) | ||
343 | activate(card); | ||
344 | spin_unlock_irqrestore(&card->lock, flags); | ||
345 | } | ||
346 | |||
347 | /* | 335 | /* |
348 | * If there is room on Ready page, take | 336 | * If there is room on Ready page, take |
349 | * one bh off list and add it. | 337 | * one bh off list and add it. |
@@ -535,7 +523,6 @@ static int mm_make_request(struct request_queue *q, struct bio *bio) | |||
535 | *card->biotail = bio; | 523 | *card->biotail = bio; |
536 | bio->bi_next = NULL; | 524 | bio->bi_next = NULL; |
537 | card->biotail = &bio->bi_next; | 525 | card->biotail = &bio->bi_next; |
538 | blk_plug_device(q); | ||
539 | spin_unlock_irq(&card->lock); | 526 | spin_unlock_irq(&card->lock); |
540 | 527 | ||
541 | return 0; | 528 | return 0; |
@@ -779,20 +766,10 @@ static int mm_getgeo(struct block_device *bdev, struct hd_geometry *geo) | |||
779 | return 0; | 766 | return 0; |
780 | } | 767 | } |
781 | 768 | ||
782 | /* | ||
783 | * Future support for removable devices | ||
784 | */ | ||
785 | static int mm_check_change(struct gendisk *disk) | ||
786 | { | ||
787 | /* struct cardinfo *dev = disk->private_data; */ | ||
788 | return 0; | ||
789 | } | ||
790 | |||
791 | static const struct block_device_operations mm_fops = { | 769 | static const struct block_device_operations mm_fops = { |
792 | .owner = THIS_MODULE, | 770 | .owner = THIS_MODULE, |
793 | .getgeo = mm_getgeo, | 771 | .getgeo = mm_getgeo, |
794 | .revalidate_disk = mm_revalidate, | 772 | .revalidate_disk = mm_revalidate, |
795 | .media_changed = mm_check_change, | ||
796 | }; | 773 | }; |
797 | 774 | ||
798 | static int __devinit mm_pci_probe(struct pci_dev *dev, | 775 | static int __devinit mm_pci_probe(struct pci_dev *dev, |
@@ -907,7 +884,6 @@ static int __devinit mm_pci_probe(struct pci_dev *dev, | |||
907 | blk_queue_make_request(card->queue, mm_make_request); | 884 | blk_queue_make_request(card->queue, mm_make_request); |
908 | card->queue->queue_lock = &card->lock; | 885 | card->queue->queue_lock = &card->lock; |
909 | card->queue->queuedata = card; | 886 | card->queue->queuedata = card; |
910 | card->queue->unplug_fn = mm_unplug_device; | ||
911 | 887 | ||
912 | tasklet_init(&card->tasklet, process_page, (unsigned long)card); | 888 | tasklet_init(&card->tasklet, process_page, (unsigned long)card); |
913 | 889 | ||
diff --git a/drivers/block/xsysace.c b/drivers/block/xsysace.c index 2c590a796aa1..73354b081ed3 100644 --- a/drivers/block/xsysace.c +++ b/drivers/block/xsysace.c | |||
@@ -867,12 +867,12 @@ static void ace_request(struct request_queue * q) | |||
867 | } | 867 | } |
868 | } | 868 | } |
869 | 869 | ||
870 | static int ace_media_changed(struct gendisk *gd) | 870 | static unsigned int ace_check_events(struct gendisk *gd, unsigned int clearing) |
871 | { | 871 | { |
872 | struct ace_device *ace = gd->private_data; | 872 | struct ace_device *ace = gd->private_data; |
873 | dev_dbg(ace->dev, "ace_media_changed(): %i\n", ace->media_change); | 873 | dev_dbg(ace->dev, "ace_check_events(): %i\n", ace->media_change); |
874 | 874 | ||
875 | return ace->media_change; | 875 | return ace->media_change ? DISK_EVENT_MEDIA_CHANGE : 0; |
876 | } | 876 | } |
877 | 877 | ||
878 | static int ace_revalidate_disk(struct gendisk *gd) | 878 | static int ace_revalidate_disk(struct gendisk *gd) |
@@ -953,7 +953,7 @@ static const struct block_device_operations ace_fops = { | |||
953 | .owner = THIS_MODULE, | 953 | .owner = THIS_MODULE, |
954 | .open = ace_open, | 954 | .open = ace_open, |
955 | .release = ace_release, | 955 | .release = ace_release, |
956 | .media_changed = ace_media_changed, | 956 | .check_events = ace_check_events, |
957 | .revalidate_disk = ace_revalidate_disk, | 957 | .revalidate_disk = ace_revalidate_disk, |
958 | .getgeo = ace_getgeo, | 958 | .getgeo = ace_getgeo, |
959 | }; | 959 | }; |
@@ -1005,6 +1005,7 @@ static int __devinit ace_setup(struct ace_device *ace) | |||
1005 | ace->gd->major = ace_major; | 1005 | ace->gd->major = ace_major; |
1006 | ace->gd->first_minor = ace->id * ACE_NUM_MINORS; | 1006 | ace->gd->first_minor = ace->id * ACE_NUM_MINORS; |
1007 | ace->gd->fops = &ace_fops; | 1007 | ace->gd->fops = &ace_fops; |
1008 | ace->gd->events = DISK_EVENT_MEDIA_CHANGE; | ||
1008 | ace->gd->queue = ace->queue; | 1009 | ace->gd->queue = ace->queue; |
1009 | ace->gd->private_data = ace; | 1010 | ace->gd->private_data = ace; |
1010 | snprintf(ace->gd->disk_name, 32, "xs%c", ace->id + 'a'); | 1011 | snprintf(ace->gd->disk_name, 32, "xs%c", ace->id + 'a'); |
diff --git a/drivers/cdrom/gdrom.c b/drivers/cdrom/gdrom.c index 64a21461c408..b2b034fea34e 100644 --- a/drivers/cdrom/gdrom.c +++ b/drivers/cdrom/gdrom.c | |||
@@ -395,10 +395,12 @@ static int gdrom_drivestatus(struct cdrom_device_info *cd_info, int ignore) | |||
395 | return CDS_NO_INFO; | 395 | return CDS_NO_INFO; |
396 | } | 396 | } |
397 | 397 | ||
398 | static int gdrom_mediachanged(struct cdrom_device_info *cd_info, int ignore) | 398 | static unsigned int gdrom_check_events(struct cdrom_device_info *cd_info, |
399 | unsigned int clearing, int ignore) | ||
399 | { | 400 | { |
400 | /* check the sense key */ | 401 | /* check the sense key */ |
401 | return (__raw_readb(GDROM_ERROR_REG) & 0xF0) == 0x60; | 402 | return (__raw_readb(GDROM_ERROR_REG) & 0xF0) == 0x60 ? |
403 | DISK_EVENT_MEDIA_CHANGE : 0; | ||
402 | } | 404 | } |
403 | 405 | ||
404 | /* reset the G1 bus */ | 406 | /* reset the G1 bus */ |
@@ -483,7 +485,7 @@ static struct cdrom_device_ops gdrom_ops = { | |||
483 | .open = gdrom_open, | 485 | .open = gdrom_open, |
484 | .release = gdrom_release, | 486 | .release = gdrom_release, |
485 | .drive_status = gdrom_drivestatus, | 487 | .drive_status = gdrom_drivestatus, |
486 | .media_changed = gdrom_mediachanged, | 488 | .check_events = gdrom_check_events, |
487 | .get_last_session = gdrom_get_last_session, | 489 | .get_last_session = gdrom_get_last_session, |
488 | .reset = gdrom_hardreset, | 490 | .reset = gdrom_hardreset, |
489 | .audio_ioctl = gdrom_audio_ioctl, | 491 | .audio_ioctl = gdrom_audio_ioctl, |
@@ -509,9 +511,10 @@ static int gdrom_bdops_release(struct gendisk *disk, fmode_t mode) | |||
509 | return 0; | 511 | return 0; |
510 | } | 512 | } |
511 | 513 | ||
512 | static int gdrom_bdops_mediachanged(struct gendisk *disk) | 514 | static unsigned int gdrom_bdops_check_events(struct gendisk *disk, |
515 | unsigned int clearing) | ||
513 | { | 516 | { |
514 | return cdrom_media_changed(gd.cd_info); | 517 | return cdrom_check_events(gd.cd_info, clearing); |
515 | } | 518 | } |
516 | 519 | ||
517 | static int gdrom_bdops_ioctl(struct block_device *bdev, fmode_t mode, | 520 | static int gdrom_bdops_ioctl(struct block_device *bdev, fmode_t mode, |
@@ -530,7 +533,7 @@ static const struct block_device_operations gdrom_bdops = { | |||
530 | .owner = THIS_MODULE, | 533 | .owner = THIS_MODULE, |
531 | .open = gdrom_bdops_open, | 534 | .open = gdrom_bdops_open, |
532 | .release = gdrom_bdops_release, | 535 | .release = gdrom_bdops_release, |
533 | .media_changed = gdrom_bdops_mediachanged, | 536 | .check_events = gdrom_bdops_check_events, |
534 | .ioctl = gdrom_bdops_ioctl, | 537 | .ioctl = gdrom_bdops_ioctl, |
535 | }; | 538 | }; |
536 | 539 | ||
@@ -800,6 +803,7 @@ static int __devinit probe_gdrom(struct platform_device *devptr) | |||
800 | goto probe_fail_cdrom_register; | 803 | goto probe_fail_cdrom_register; |
801 | } | 804 | } |
802 | gd.disk->fops = &gdrom_bdops; | 805 | gd.disk->fops = &gdrom_bdops; |
806 | gd.disk->events = DISK_EVENT_MEDIA_CHANGE; | ||
803 | /* latch on to the interrupt */ | 807 | /* latch on to the interrupt */ |
804 | err = gdrom_set_interrupt_handlers(); | 808 | err = gdrom_set_interrupt_handlers(); |
805 | if (err) | 809 | if (err) |
diff --git a/drivers/cdrom/viocd.c b/drivers/cdrom/viocd.c index be73a9b493a6..4e874c5fa605 100644 --- a/drivers/cdrom/viocd.c +++ b/drivers/cdrom/viocd.c | |||
@@ -186,10 +186,11 @@ static int viocd_blk_ioctl(struct block_device *bdev, fmode_t mode, | |||
186 | return ret; | 186 | return ret; |
187 | } | 187 | } |
188 | 188 | ||
189 | static int viocd_blk_media_changed(struct gendisk *disk) | 189 | static unsigned int viocd_blk_check_events(struct gendisk *disk, |
190 | unsigned int clearing) | ||
190 | { | 191 | { |
191 | struct disk_info *di = disk->private_data; | 192 | struct disk_info *di = disk->private_data; |
192 | return cdrom_media_changed(&di->viocd_info); | 193 | return cdrom_check_events(&di->viocd_info, clearing); |
193 | } | 194 | } |
194 | 195 | ||
195 | static const struct block_device_operations viocd_fops = { | 196 | static const struct block_device_operations viocd_fops = { |
@@ -197,7 +198,7 @@ static const struct block_device_operations viocd_fops = { | |||
197 | .open = viocd_blk_open, | 198 | .open = viocd_blk_open, |
198 | .release = viocd_blk_release, | 199 | .release = viocd_blk_release, |
199 | .ioctl = viocd_blk_ioctl, | 200 | .ioctl = viocd_blk_ioctl, |
200 | .media_changed = viocd_blk_media_changed, | 201 | .check_events = viocd_blk_check_events, |
201 | }; | 202 | }; |
202 | 203 | ||
203 | static int viocd_open(struct cdrom_device_info *cdi, int purpose) | 204 | static int viocd_open(struct cdrom_device_info *cdi, int purpose) |
@@ -320,7 +321,8 @@ static void do_viocd_request(struct request_queue *q) | |||
320 | } | 321 | } |
321 | } | 322 | } |
322 | 323 | ||
323 | static int viocd_media_changed(struct cdrom_device_info *cdi, int disc_nr) | 324 | static unsigned int viocd_check_events(struct cdrom_device_info *cdi, |
325 | unsigned int clearing, int disc_nr) | ||
324 | { | 326 | { |
325 | struct viocd_waitevent we; | 327 | struct viocd_waitevent we; |
326 | HvLpEvent_Rc hvrc; | 328 | HvLpEvent_Rc hvrc; |
@@ -340,7 +342,7 @@ static int viocd_media_changed(struct cdrom_device_info *cdi, int disc_nr) | |||
340 | if (hvrc != 0) { | 342 | if (hvrc != 0) { |
341 | pr_warning("bad rc on HvCallEvent_signalLpEventFast %d\n", | 343 | pr_warning("bad rc on HvCallEvent_signalLpEventFast %d\n", |
342 | (int)hvrc); | 344 | (int)hvrc); |
343 | return -EIO; | 345 | return 0; |
344 | } | 346 | } |
345 | 347 | ||
346 | wait_for_completion(&we.com); | 348 | wait_for_completion(&we.com); |
@@ -354,7 +356,7 @@ static int viocd_media_changed(struct cdrom_device_info *cdi, int disc_nr) | |||
354 | return 0; | 356 | return 0; |
355 | } | 357 | } |
356 | 358 | ||
357 | return we.changed; | 359 | return we.changed ? DISK_EVENT_MEDIA_CHANGE : 0; |
358 | } | 360 | } |
359 | 361 | ||
360 | static int viocd_lock_door(struct cdrom_device_info *cdi, int locking) | 362 | static int viocd_lock_door(struct cdrom_device_info *cdi, int locking) |
@@ -550,7 +552,7 @@ static int viocd_audio_ioctl(struct cdrom_device_info *cdi, unsigned int cmd, | |||
550 | static struct cdrom_device_ops viocd_dops = { | 552 | static struct cdrom_device_ops viocd_dops = { |
551 | .open = viocd_open, | 553 | .open = viocd_open, |
552 | .release = viocd_release, | 554 | .release = viocd_release, |
553 | .media_changed = viocd_media_changed, | 555 | .check_events = viocd_check_events, |
554 | .lock_door = viocd_lock_door, | 556 | .lock_door = viocd_lock_door, |
555 | .generic_packet = viocd_packet, | 557 | .generic_packet = viocd_packet, |
556 | .audio_ioctl = viocd_audio_ioctl, | 558 | .audio_ioctl = viocd_audio_ioctl, |
@@ -624,6 +626,7 @@ static int viocd_probe(struct vio_dev *vdev, const struct vio_device_id *id) | |||
624 | gendisk->queue = q; | 626 | gendisk->queue = q; |
625 | gendisk->fops = &viocd_fops; | 627 | gendisk->fops = &viocd_fops; |
626 | gendisk->flags = GENHD_FL_CD|GENHD_FL_REMOVABLE; | 628 | gendisk->flags = GENHD_FL_CD|GENHD_FL_REMOVABLE; |
629 | gendisk->events = DISK_EVENT_MEDIA_CHANGE; | ||
627 | set_capacity(gendisk, 0); | 630 | set_capacity(gendisk, 0); |
628 | gendisk->private_data = d; | 631 | gendisk->private_data = d; |
629 | d->viocd_disk = gendisk; | 632 | d->viocd_disk = gendisk; |
diff --git a/drivers/ide/ide-atapi.c b/drivers/ide/ide-atapi.c index e88a2cf17711..6f218e014e99 100644 --- a/drivers/ide/ide-atapi.c +++ b/drivers/ide/ide-atapi.c | |||
@@ -233,8 +233,7 @@ int ide_queue_sense_rq(ide_drive_t *drive, void *special) | |||
233 | 233 | ||
234 | drive->hwif->rq = NULL; | 234 | drive->hwif->rq = NULL; |
235 | 235 | ||
236 | elv_add_request(drive->queue, &drive->sense_rq, | 236 | elv_add_request(drive->queue, &drive->sense_rq, ELEVATOR_INSERT_FRONT); |
237 | ELEVATOR_INSERT_FRONT, 0); | ||
238 | return 0; | 237 | return 0; |
239 | } | 238 | } |
240 | EXPORT_SYMBOL_GPL(ide_queue_sense_rq); | 239 | EXPORT_SYMBOL_GPL(ide_queue_sense_rq); |
diff --git a/drivers/ide/ide-cd.c b/drivers/ide/ide-cd.c index 0c73fe39a236..fd1e11799137 100644 --- a/drivers/ide/ide-cd.c +++ b/drivers/ide/ide-cd.c | |||
@@ -258,17 +258,10 @@ static int ide_cd_breathe(ide_drive_t *drive, struct request *rq) | |||
258 | if (time_after(jiffies, info->write_timeout)) | 258 | if (time_after(jiffies, info->write_timeout)) |
259 | return 0; | 259 | return 0; |
260 | else { | 260 | else { |
261 | struct request_queue *q = drive->queue; | ||
262 | unsigned long flags; | ||
263 | |||
264 | /* | 261 | /* |
265 | * take a breather relying on the unplug timer to kick us again | 262 | * take a breather |
266 | */ | 263 | */ |
267 | 264 | blk_delay_queue(drive->queue, 1); | |
268 | spin_lock_irqsave(q->queue_lock, flags); | ||
269 | blk_plug_device(q); | ||
270 | spin_unlock_irqrestore(q->queue_lock, flags); | ||
271 | |||
272 | return 1; | 265 | return 1; |
273 | } | 266 | } |
274 | } | 267 | } |
@@ -1177,7 +1170,7 @@ static struct cdrom_device_ops ide_cdrom_dops = { | |||
1177 | .open = ide_cdrom_open_real, | 1170 | .open = ide_cdrom_open_real, |
1178 | .release = ide_cdrom_release_real, | 1171 | .release = ide_cdrom_release_real, |
1179 | .drive_status = ide_cdrom_drive_status, | 1172 | .drive_status = ide_cdrom_drive_status, |
1180 | .media_changed = ide_cdrom_check_media_change_real, | 1173 | .check_events = ide_cdrom_check_events_real, |
1181 | .tray_move = ide_cdrom_tray_move, | 1174 | .tray_move = ide_cdrom_tray_move, |
1182 | .lock_door = ide_cdrom_lock_door, | 1175 | .lock_door = ide_cdrom_lock_door, |
1183 | .select_speed = ide_cdrom_select_speed, | 1176 | .select_speed = ide_cdrom_select_speed, |
@@ -1514,8 +1507,6 @@ static int ide_cdrom_setup(ide_drive_t *drive) | |||
1514 | blk_queue_dma_alignment(q, 31); | 1507 | blk_queue_dma_alignment(q, 31); |
1515 | blk_queue_update_dma_pad(q, 15); | 1508 | blk_queue_update_dma_pad(q, 15); |
1516 | 1509 | ||
1517 | q->unplug_delay = max((1 * HZ) / 1000, 1); | ||
1518 | |||
1519 | drive->dev_flags |= IDE_DFLAG_MEDIA_CHANGED; | 1510 | drive->dev_flags |= IDE_DFLAG_MEDIA_CHANGED; |
1520 | drive->atapi_flags = IDE_AFLAG_NO_EJECT | ide_cd_flags(id); | 1511 | drive->atapi_flags = IDE_AFLAG_NO_EJECT | ide_cd_flags(id); |
1521 | 1512 | ||
@@ -1702,10 +1693,11 @@ static int idecd_ioctl(struct block_device *bdev, fmode_t mode, | |||
1702 | } | 1693 | } |
1703 | 1694 | ||
1704 | 1695 | ||
1705 | static int idecd_media_changed(struct gendisk *disk) | 1696 | static unsigned int idecd_check_events(struct gendisk *disk, |
1697 | unsigned int clearing) | ||
1706 | { | 1698 | { |
1707 | struct cdrom_info *info = ide_drv_g(disk, cdrom_info); | 1699 | struct cdrom_info *info = ide_drv_g(disk, cdrom_info); |
1708 | return cdrom_media_changed(&info->devinfo); | 1700 | return cdrom_check_events(&info->devinfo, clearing); |
1709 | } | 1701 | } |
1710 | 1702 | ||
1711 | static int idecd_revalidate_disk(struct gendisk *disk) | 1703 | static int idecd_revalidate_disk(struct gendisk *disk) |
@@ -1723,7 +1715,7 @@ static const struct block_device_operations idecd_ops = { | |||
1723 | .open = idecd_open, | 1715 | .open = idecd_open, |
1724 | .release = idecd_release, | 1716 | .release = idecd_release, |
1725 | .ioctl = idecd_ioctl, | 1717 | .ioctl = idecd_ioctl, |
1726 | .media_changed = idecd_media_changed, | 1718 | .check_events = idecd_check_events, |
1727 | .revalidate_disk = idecd_revalidate_disk | 1719 | .revalidate_disk = idecd_revalidate_disk |
1728 | }; | 1720 | }; |
1729 | 1721 | ||
@@ -1790,6 +1782,7 @@ static int ide_cd_probe(ide_drive_t *drive) | |||
1790 | ide_cd_read_toc(drive, &sense); | 1782 | ide_cd_read_toc(drive, &sense); |
1791 | g->fops = &idecd_ops; | 1783 | g->fops = &idecd_ops; |
1792 | g->flags |= GENHD_FL_REMOVABLE; | 1784 | g->flags |= GENHD_FL_REMOVABLE; |
1785 | g->events = DISK_EVENT_MEDIA_CHANGE; | ||
1793 | add_disk(g); | 1786 | add_disk(g); |
1794 | return 0; | 1787 | return 0; |
1795 | 1788 | ||
diff --git a/drivers/ide/ide-cd.h b/drivers/ide/ide-cd.h index 93a3cf1b0f3f..1efc936f5b66 100644 --- a/drivers/ide/ide-cd.h +++ b/drivers/ide/ide-cd.h | |||
@@ -111,7 +111,8 @@ int cdrom_check_status(ide_drive_t *, struct request_sense *); | |||
111 | int ide_cdrom_open_real(struct cdrom_device_info *, int); | 111 | int ide_cdrom_open_real(struct cdrom_device_info *, int); |
112 | void ide_cdrom_release_real(struct cdrom_device_info *); | 112 | void ide_cdrom_release_real(struct cdrom_device_info *); |
113 | int ide_cdrom_drive_status(struct cdrom_device_info *, int); | 113 | int ide_cdrom_drive_status(struct cdrom_device_info *, int); |
114 | int ide_cdrom_check_media_change_real(struct cdrom_device_info *, int); | 114 | unsigned int ide_cdrom_check_events_real(struct cdrom_device_info *, |
115 | unsigned int clearing, int slot_nr); | ||
115 | int ide_cdrom_tray_move(struct cdrom_device_info *, int); | 116 | int ide_cdrom_tray_move(struct cdrom_device_info *, int); |
116 | int ide_cdrom_lock_door(struct cdrom_device_info *, int); | 117 | int ide_cdrom_lock_door(struct cdrom_device_info *, int); |
117 | int ide_cdrom_select_speed(struct cdrom_device_info *, int); | 118 | int ide_cdrom_select_speed(struct cdrom_device_info *, int); |
diff --git a/drivers/ide/ide-cd_ioctl.c b/drivers/ide/ide-cd_ioctl.c index 766b3deeb23c..2a6bc50e8a41 100644 --- a/drivers/ide/ide-cd_ioctl.c +++ b/drivers/ide/ide-cd_ioctl.c | |||
@@ -79,8 +79,8 @@ int ide_cdrom_drive_status(struct cdrom_device_info *cdi, int slot_nr) | |||
79 | return CDS_DRIVE_NOT_READY; | 79 | return CDS_DRIVE_NOT_READY; |
80 | } | 80 | } |
81 | 81 | ||
82 | int ide_cdrom_check_media_change_real(struct cdrom_device_info *cdi, | 82 | unsigned int ide_cdrom_check_events_real(struct cdrom_device_info *cdi, |
83 | int slot_nr) | 83 | unsigned int clearing, int slot_nr) |
84 | { | 84 | { |
85 | ide_drive_t *drive = cdi->handle; | 85 | ide_drive_t *drive = cdi->handle; |
86 | int retval; | 86 | int retval; |
@@ -89,9 +89,9 @@ int ide_cdrom_check_media_change_real(struct cdrom_device_info *cdi, | |||
89 | (void) cdrom_check_status(drive, NULL); | 89 | (void) cdrom_check_status(drive, NULL); |
90 | retval = (drive->dev_flags & IDE_DFLAG_MEDIA_CHANGED) ? 1 : 0; | 90 | retval = (drive->dev_flags & IDE_DFLAG_MEDIA_CHANGED) ? 1 : 0; |
91 | drive->dev_flags &= ~IDE_DFLAG_MEDIA_CHANGED; | 91 | drive->dev_flags &= ~IDE_DFLAG_MEDIA_CHANGED; |
92 | return retval; | 92 | return retval ? DISK_EVENT_MEDIA_CHANGE : 0; |
93 | } else { | 93 | } else { |
94 | return -EINVAL; | 94 | return 0; |
95 | } | 95 | } |
96 | } | 96 | } |
97 | 97 | ||
diff --git a/drivers/ide/ide-gd.c b/drivers/ide/ide-gd.c index 35c4b43585e3..c4ffd4888939 100644 --- a/drivers/ide/ide-gd.c +++ b/drivers/ide/ide-gd.c | |||
@@ -285,11 +285,12 @@ static int ide_gd_getgeo(struct block_device *bdev, struct hd_geometry *geo) | |||
285 | return 0; | 285 | return 0; |
286 | } | 286 | } |
287 | 287 | ||
288 | static int ide_gd_media_changed(struct gendisk *disk) | 288 | static unsigned int ide_gd_check_events(struct gendisk *disk, |
289 | unsigned int clearing) | ||
289 | { | 290 | { |
290 | struct ide_disk_obj *idkp = ide_drv_g(disk, ide_disk_obj); | 291 | struct ide_disk_obj *idkp = ide_drv_g(disk, ide_disk_obj); |
291 | ide_drive_t *drive = idkp->drive; | 292 | ide_drive_t *drive = idkp->drive; |
292 | int ret; | 293 | bool ret; |
293 | 294 | ||
294 | /* do not scan partitions twice if this is a removable device */ | 295 | /* do not scan partitions twice if this is a removable device */ |
295 | if (drive->dev_flags & IDE_DFLAG_ATTACH) { | 296 | if (drive->dev_flags & IDE_DFLAG_ATTACH) { |
@@ -297,10 +298,10 @@ static int ide_gd_media_changed(struct gendisk *disk) | |||
297 | return 0; | 298 | return 0; |
298 | } | 299 | } |
299 | 300 | ||
300 | ret = !!(drive->dev_flags & IDE_DFLAG_MEDIA_CHANGED); | 301 | ret = drive->dev_flags & IDE_DFLAG_MEDIA_CHANGED; |
301 | drive->dev_flags &= ~IDE_DFLAG_MEDIA_CHANGED; | 302 | drive->dev_flags &= ~IDE_DFLAG_MEDIA_CHANGED; |
302 | 303 | ||
303 | return ret; | 304 | return ret ? DISK_EVENT_MEDIA_CHANGE : 0; |
304 | } | 305 | } |
305 | 306 | ||
306 | static void ide_gd_unlock_native_capacity(struct gendisk *disk) | 307 | static void ide_gd_unlock_native_capacity(struct gendisk *disk) |
@@ -318,7 +319,7 @@ static int ide_gd_revalidate_disk(struct gendisk *disk) | |||
318 | struct ide_disk_obj *idkp = ide_drv_g(disk, ide_disk_obj); | 319 | struct ide_disk_obj *idkp = ide_drv_g(disk, ide_disk_obj); |
319 | ide_drive_t *drive = idkp->drive; | 320 | ide_drive_t *drive = idkp->drive; |
320 | 321 | ||
321 | if (ide_gd_media_changed(disk)) | 322 | if (ide_gd_check_events(disk, 0)) |
322 | drive->disk_ops->get_capacity(drive); | 323 | drive->disk_ops->get_capacity(drive); |
323 | 324 | ||
324 | set_capacity(disk, ide_gd_capacity(drive)); | 325 | set_capacity(disk, ide_gd_capacity(drive)); |
@@ -340,7 +341,7 @@ static const struct block_device_operations ide_gd_ops = { | |||
340 | .release = ide_gd_release, | 341 | .release = ide_gd_release, |
341 | .ioctl = ide_gd_ioctl, | 342 | .ioctl = ide_gd_ioctl, |
342 | .getgeo = ide_gd_getgeo, | 343 | .getgeo = ide_gd_getgeo, |
343 | .media_changed = ide_gd_media_changed, | 344 | .check_events = ide_gd_check_events, |
344 | .unlock_native_capacity = ide_gd_unlock_native_capacity, | 345 | .unlock_native_capacity = ide_gd_unlock_native_capacity, |
345 | .revalidate_disk = ide_gd_revalidate_disk | 346 | .revalidate_disk = ide_gd_revalidate_disk |
346 | }; | 347 | }; |
@@ -412,6 +413,7 @@ static int ide_gd_probe(ide_drive_t *drive) | |||
412 | if (drive->dev_flags & IDE_DFLAG_REMOVABLE) | 413 | if (drive->dev_flags & IDE_DFLAG_REMOVABLE) |
413 | g->flags = GENHD_FL_REMOVABLE; | 414 | g->flags = GENHD_FL_REMOVABLE; |
414 | g->fops = &ide_gd_ops; | 415 | g->fops = &ide_gd_ops; |
416 | g->events = DISK_EVENT_MEDIA_CHANGE; | ||
415 | add_disk(g); | 417 | add_disk(g); |
416 | return 0; | 418 | return 0; |
417 | 419 | ||
diff --git a/drivers/ide/ide-io.c b/drivers/ide/ide-io.c index 999dac054bcc..f4077840d3ab 100644 --- a/drivers/ide/ide-io.c +++ b/drivers/ide/ide-io.c | |||
@@ -549,8 +549,6 @@ plug_device_2: | |||
549 | 549 | ||
550 | if (rq) | 550 | if (rq) |
551 | blk_requeue_request(q, rq); | 551 | blk_requeue_request(q, rq); |
552 | if (!elv_queue_empty(q)) | ||
553 | blk_plug_device(q); | ||
554 | } | 552 | } |
555 | 553 | ||
556 | void ide_requeue_and_plug(ide_drive_t *drive, struct request *rq) | 554 | void ide_requeue_and_plug(ide_drive_t *drive, struct request *rq) |
@@ -562,8 +560,6 @@ void ide_requeue_and_plug(ide_drive_t *drive, struct request *rq) | |||
562 | 560 | ||
563 | if (rq) | 561 | if (rq) |
564 | blk_requeue_request(q, rq); | 562 | blk_requeue_request(q, rq); |
565 | if (!elv_queue_empty(q)) | ||
566 | blk_plug_device(q); | ||
567 | 563 | ||
568 | spin_unlock_irqrestore(q->queue_lock, flags); | 564 | spin_unlock_irqrestore(q->queue_lock, flags); |
569 | } | 565 | } |
diff --git a/drivers/ide/ide-park.c b/drivers/ide/ide-park.c index 88a380c5a470..6ab9ab2a5081 100644 --- a/drivers/ide/ide-park.c +++ b/drivers/ide/ide-park.c | |||
@@ -52,7 +52,7 @@ static void issue_park_cmd(ide_drive_t *drive, unsigned long timeout) | |||
52 | rq->cmd[0] = REQ_UNPARK_HEADS; | 52 | rq->cmd[0] = REQ_UNPARK_HEADS; |
53 | rq->cmd_len = 1; | 53 | rq->cmd_len = 1; |
54 | rq->cmd_type = REQ_TYPE_SPECIAL; | 54 | rq->cmd_type = REQ_TYPE_SPECIAL; |
55 | elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 1); | 55 | elv_add_request(q, rq, ELEVATOR_INSERT_FRONT); |
56 | 56 | ||
57 | out: | 57 | out: |
58 | return; | 58 | return; |
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c index a2ce0b2da281..5c9362792f1d 100644 --- a/drivers/md/bitmap.c +++ b/drivers/md/bitmap.c | |||
@@ -347,7 +347,7 @@ static void write_page(struct bitmap *bitmap, struct page *page, int wait) | |||
347 | atomic_inc(&bitmap->pending_writes); | 347 | atomic_inc(&bitmap->pending_writes); |
348 | set_buffer_locked(bh); | 348 | set_buffer_locked(bh); |
349 | set_buffer_mapped(bh); | 349 | set_buffer_mapped(bh); |
350 | submit_bh(WRITE | REQ_UNPLUG | REQ_SYNC, bh); | 350 | submit_bh(WRITE | REQ_SYNC, bh); |
351 | bh = bh->b_this_page; | 351 | bh = bh->b_this_page; |
352 | } | 352 | } |
353 | 353 | ||
@@ -1339,8 +1339,7 @@ int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sect | |||
1339 | prepare_to_wait(&bitmap->overflow_wait, &__wait, | 1339 | prepare_to_wait(&bitmap->overflow_wait, &__wait, |
1340 | TASK_UNINTERRUPTIBLE); | 1340 | TASK_UNINTERRUPTIBLE); |
1341 | spin_unlock_irq(&bitmap->lock); | 1341 | spin_unlock_irq(&bitmap->lock); |
1342 | md_unplug(bitmap->mddev); | 1342 | io_schedule(); |
1343 | schedule(); | ||
1344 | finish_wait(&bitmap->overflow_wait, &__wait); | 1343 | finish_wait(&bitmap->overflow_wait, &__wait); |
1345 | continue; | 1344 | continue; |
1346 | } | 1345 | } |
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 4e054bd91664..2c62c1169f78 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c | |||
@@ -991,11 +991,6 @@ static void clone_init(struct dm_crypt_io *io, struct bio *clone) | |||
991 | clone->bi_destructor = dm_crypt_bio_destructor; | 991 | clone->bi_destructor = dm_crypt_bio_destructor; |
992 | } | 992 | } |
993 | 993 | ||
994 | static void kcryptd_unplug(struct crypt_config *cc) | ||
995 | { | ||
996 | blk_unplug(bdev_get_queue(cc->dev->bdev)); | ||
997 | } | ||
998 | |||
999 | static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) | 994 | static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) |
1000 | { | 995 | { |
1001 | struct crypt_config *cc = io->target->private; | 996 | struct crypt_config *cc = io->target->private; |
@@ -1008,10 +1003,8 @@ static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp) | |||
1008 | * one in order to decrypt the whole bio data *afterwards*. | 1003 | * one in order to decrypt the whole bio data *afterwards*. |
1009 | */ | 1004 | */ |
1010 | clone = bio_alloc_bioset(gfp, bio_segments(base_bio), cc->bs); | 1005 | clone = bio_alloc_bioset(gfp, bio_segments(base_bio), cc->bs); |
1011 | if (!clone) { | 1006 | if (!clone) |
1012 | kcryptd_unplug(cc); | ||
1013 | return 1; | 1007 | return 1; |
1014 | } | ||
1015 | 1008 | ||
1016 | crypt_inc_pending(io); | 1009 | crypt_inc_pending(io); |
1017 | 1010 | ||
diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c index 136d4f71a116..76a5af00a26b 100644 --- a/drivers/md/dm-io.c +++ b/drivers/md/dm-io.c | |||
@@ -352,7 +352,7 @@ static void dispatch_io(int rw, unsigned int num_regions, | |||
352 | BUG_ON(num_regions > DM_IO_MAX_REGIONS); | 352 | BUG_ON(num_regions > DM_IO_MAX_REGIONS); |
353 | 353 | ||
354 | if (sync) | 354 | if (sync) |
355 | rw |= REQ_SYNC | REQ_UNPLUG; | 355 | rw |= REQ_SYNC; |
356 | 356 | ||
357 | /* | 357 | /* |
358 | * For multiple regions we need to be careful to rewind | 358 | * For multiple regions we need to be careful to rewind |
diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index 924f5f0084c2..1bb73a13ca40 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c | |||
@@ -37,13 +37,6 @@ struct dm_kcopyd_client { | |||
37 | unsigned int nr_pages; | 37 | unsigned int nr_pages; |
38 | unsigned int nr_free_pages; | 38 | unsigned int nr_free_pages; |
39 | 39 | ||
40 | /* | ||
41 | * Block devices to unplug. | ||
42 | * Non-NULL pointer means that a block device has some pending requests | ||
43 | * and needs to be unplugged. | ||
44 | */ | ||
45 | struct block_device *unplug[2]; | ||
46 | |||
47 | struct dm_io_client *io_client; | 40 | struct dm_io_client *io_client; |
48 | 41 | ||
49 | wait_queue_head_t destroyq; | 42 | wait_queue_head_t destroyq; |
@@ -315,31 +308,6 @@ static int run_complete_job(struct kcopyd_job *job) | |||
315 | return 0; | 308 | return 0; |
316 | } | 309 | } |
317 | 310 | ||
318 | /* | ||
319 | * Unplug the block device at the specified index. | ||
320 | */ | ||
321 | static void unplug(struct dm_kcopyd_client *kc, int rw) | ||
322 | { | ||
323 | if (kc->unplug[rw] != NULL) { | ||
324 | blk_unplug(bdev_get_queue(kc->unplug[rw])); | ||
325 | kc->unplug[rw] = NULL; | ||
326 | } | ||
327 | } | ||
328 | |||
329 | /* | ||
330 | * Prepare block device unplug. If there's another device | ||
331 | * to be unplugged at the same array index, we unplug that | ||
332 | * device first. | ||
333 | */ | ||
334 | static void prepare_unplug(struct dm_kcopyd_client *kc, int rw, | ||
335 | struct block_device *bdev) | ||
336 | { | ||
337 | if (likely(kc->unplug[rw] == bdev)) | ||
338 | return; | ||
339 | unplug(kc, rw); | ||
340 | kc->unplug[rw] = bdev; | ||
341 | } | ||
342 | |||
343 | static void complete_io(unsigned long error, void *context) | 311 | static void complete_io(unsigned long error, void *context) |
344 | { | 312 | { |
345 | struct kcopyd_job *job = (struct kcopyd_job *) context; | 313 | struct kcopyd_job *job = (struct kcopyd_job *) context; |
@@ -386,16 +354,10 @@ static int run_io_job(struct kcopyd_job *job) | |||
386 | .client = job->kc->io_client, | 354 | .client = job->kc->io_client, |
387 | }; | 355 | }; |
388 | 356 | ||
389 | if (job->rw == READ) { | 357 | if (job->rw == READ) |
390 | r = dm_io(&io_req, 1, &job->source, NULL); | 358 | r = dm_io(&io_req, 1, &job->source, NULL); |
391 | prepare_unplug(job->kc, READ, job->source.bdev); | 359 | else |
392 | } else { | ||
393 | if (job->num_dests > 1) | ||
394 | io_req.bi_rw |= REQ_UNPLUG; | ||
395 | r = dm_io(&io_req, job->num_dests, job->dests, NULL); | 360 | r = dm_io(&io_req, job->num_dests, job->dests, NULL); |
396 | if (!(io_req.bi_rw & REQ_UNPLUG)) | ||
397 | prepare_unplug(job->kc, WRITE, job->dests[0].bdev); | ||
398 | } | ||
399 | 361 | ||
400 | return r; | 362 | return r; |
401 | } | 363 | } |
@@ -466,6 +428,7 @@ static void do_work(struct work_struct *work) | |||
466 | { | 428 | { |
467 | struct dm_kcopyd_client *kc = container_of(work, | 429 | struct dm_kcopyd_client *kc = container_of(work, |
468 | struct dm_kcopyd_client, kcopyd_work); | 430 | struct dm_kcopyd_client, kcopyd_work); |
431 | struct blk_plug plug; | ||
469 | 432 | ||
470 | /* | 433 | /* |
471 | * The order that these are called is *very* important. | 434 | * The order that these are called is *very* important. |
@@ -473,18 +436,12 @@ static void do_work(struct work_struct *work) | |||
473 | * Pages jobs when successful will jump onto the io jobs | 436 | * Pages jobs when successful will jump onto the io jobs |
474 | * list. io jobs call wake when they complete and it all | 437 | * list. io jobs call wake when they complete and it all |
475 | * starts again. | 438 | * starts again. |
476 | * | ||
477 | * Note that io_jobs add block devices to the unplug array, | ||
478 | * this array is cleared with "unplug" calls. It is thus | ||
479 | * forbidden to run complete_jobs after io_jobs and before | ||
480 | * unplug because the block device could be destroyed in | ||
481 | * job completion callback. | ||
482 | */ | 439 | */ |
440 | blk_start_plug(&plug); | ||
483 | process_jobs(&kc->complete_jobs, kc, run_complete_job); | 441 | process_jobs(&kc->complete_jobs, kc, run_complete_job); |
484 | process_jobs(&kc->pages_jobs, kc, run_pages_job); | 442 | process_jobs(&kc->pages_jobs, kc, run_pages_job); |
485 | process_jobs(&kc->io_jobs, kc, run_io_job); | 443 | process_jobs(&kc->io_jobs, kc, run_io_job); |
486 | unplug(kc, READ); | 444 | blk_finish_plug(&plug); |
487 | unplug(kc, WRITE); | ||
488 | } | 445 | } |
489 | 446 | ||
490 | /* | 447 | /* |
@@ -665,8 +622,6 @@ int dm_kcopyd_client_create(unsigned int nr_pages, | |||
665 | INIT_LIST_HEAD(&kc->io_jobs); | 622 | INIT_LIST_HEAD(&kc->io_jobs); |
666 | INIT_LIST_HEAD(&kc->pages_jobs); | 623 | INIT_LIST_HEAD(&kc->pages_jobs); |
667 | 624 | ||
668 | memset(kc->unplug, 0, sizeof(kc->unplug)); | ||
669 | |||
670 | kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache); | 625 | kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache); |
671 | if (!kc->job_pool) | 626 | if (!kc->job_pool) |
672 | goto bad_slab; | 627 | goto bad_slab; |
diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index b9e1e15ef11c..5ef136cdba91 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c | |||
@@ -394,7 +394,7 @@ static void raid_unplug(struct dm_target_callbacks *cb) | |||
394 | { | 394 | { |
395 | struct raid_set *rs = container_of(cb, struct raid_set, callbacks); | 395 | struct raid_set *rs = container_of(cb, struct raid_set, callbacks); |
396 | 396 | ||
397 | md_raid5_unplug_device(rs->md.private); | 397 | md_raid5_kick_device(rs->md.private); |
398 | } | 398 | } |
399 | 399 | ||
400 | /* | 400 | /* |
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index dee326775c60..976ad4688afc 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c | |||
@@ -842,8 +842,6 @@ static void do_mirror(struct work_struct *work) | |||
842 | do_reads(ms, &reads); | 842 | do_reads(ms, &reads); |
843 | do_writes(ms, &writes); | 843 | do_writes(ms, &writes); |
844 | do_failures(ms, &failures); | 844 | do_failures(ms, &failures); |
845 | |||
846 | dm_table_unplug_all(ms->ti->table); | ||
847 | } | 845 | } |
848 | 846 | ||
849 | /*----------------------------------------------------------------- | 847 | /*----------------------------------------------------------------- |
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 38e4eb1bb965..416d4e258df6 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c | |||
@@ -55,6 +55,7 @@ struct dm_table { | |||
55 | struct dm_target *targets; | 55 | struct dm_target *targets; |
56 | 56 | ||
57 | unsigned discards_supported:1; | 57 | unsigned discards_supported:1; |
58 | unsigned integrity_supported:1; | ||
58 | 59 | ||
59 | /* | 60 | /* |
60 | * Indicates the rw permissions for the new logical | 61 | * Indicates the rw permissions for the new logical |
@@ -859,7 +860,7 @@ int dm_table_alloc_md_mempools(struct dm_table *t) | |||
859 | return -EINVAL; | 860 | return -EINVAL; |
860 | } | 861 | } |
861 | 862 | ||
862 | t->mempools = dm_alloc_md_mempools(type); | 863 | t->mempools = dm_alloc_md_mempools(type, t->integrity_supported); |
863 | if (!t->mempools) | 864 | if (!t->mempools) |
864 | return -ENOMEM; | 865 | return -ENOMEM; |
865 | 866 | ||
@@ -935,8 +936,10 @@ static int dm_table_prealloc_integrity(struct dm_table *t, struct mapped_device | |||
935 | struct dm_dev_internal *dd; | 936 | struct dm_dev_internal *dd; |
936 | 937 | ||
937 | list_for_each_entry(dd, devices, list) | 938 | list_for_each_entry(dd, devices, list) |
938 | if (bdev_get_integrity(dd->dm_dev.bdev)) | 939 | if (bdev_get_integrity(dd->dm_dev.bdev)) { |
940 | t->integrity_supported = 1; | ||
939 | return blk_integrity_register(dm_disk(md), NULL); | 941 | return blk_integrity_register(dm_disk(md), NULL); |
942 | } | ||
940 | 943 | ||
941 | return 0; | 944 | return 0; |
942 | } | 945 | } |
@@ -1275,29 +1278,6 @@ int dm_table_any_busy_target(struct dm_table *t) | |||
1275 | return 0; | 1278 | return 0; |
1276 | } | 1279 | } |
1277 | 1280 | ||
1278 | void dm_table_unplug_all(struct dm_table *t) | ||
1279 | { | ||
1280 | struct dm_dev_internal *dd; | ||
1281 | struct list_head *devices = dm_table_get_devices(t); | ||
1282 | struct dm_target_callbacks *cb; | ||
1283 | |||
1284 | list_for_each_entry(dd, devices, list) { | ||
1285 | struct request_queue *q = bdev_get_queue(dd->dm_dev.bdev); | ||
1286 | char b[BDEVNAME_SIZE]; | ||
1287 | |||
1288 | if (likely(q)) | ||
1289 | blk_unplug(q); | ||
1290 | else | ||
1291 | DMWARN_LIMIT("%s: Cannot unplug nonexistent device %s", | ||
1292 | dm_device_name(t->md), | ||
1293 | bdevname(dd->dm_dev.bdev, b)); | ||
1294 | } | ||
1295 | |||
1296 | list_for_each_entry(cb, &t->target_callbacks, list) | ||
1297 | if (cb->unplug_fn) | ||
1298 | cb->unplug_fn(cb); | ||
1299 | } | ||
1300 | |||
1301 | struct mapped_device *dm_table_get_md(struct dm_table *t) | 1281 | struct mapped_device *dm_table_get_md(struct dm_table *t) |
1302 | { | 1282 | { |
1303 | return t->md; | 1283 | return t->md; |
@@ -1345,4 +1325,3 @@ EXPORT_SYMBOL(dm_table_get_mode); | |||
1345 | EXPORT_SYMBOL(dm_table_get_md); | 1325 | EXPORT_SYMBOL(dm_table_get_md); |
1346 | EXPORT_SYMBOL(dm_table_put); | 1326 | EXPORT_SYMBOL(dm_table_put); |
1347 | EXPORT_SYMBOL(dm_table_get); | 1327 | EXPORT_SYMBOL(dm_table_get); |
1348 | EXPORT_SYMBOL(dm_table_unplug_all); | ||
diff --git a/drivers/md/dm.c b/drivers/md/dm.c index eaa3af0e0632..0cf68b478878 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c | |||
@@ -477,7 +477,8 @@ static void start_io_acct(struct dm_io *io) | |||
477 | cpu = part_stat_lock(); | 477 | cpu = part_stat_lock(); |
478 | part_round_stats(cpu, &dm_disk(md)->part0); | 478 | part_round_stats(cpu, &dm_disk(md)->part0); |
479 | part_stat_unlock(); | 479 | part_stat_unlock(); |
480 | dm_disk(md)->part0.in_flight[rw] = atomic_inc_return(&md->pending[rw]); | 480 | atomic_set(&dm_disk(md)->part0.in_flight[rw], |
481 | atomic_inc_return(&md->pending[rw])); | ||
481 | } | 482 | } |
482 | 483 | ||
483 | static void end_io_acct(struct dm_io *io) | 484 | static void end_io_acct(struct dm_io *io) |
@@ -497,8 +498,8 @@ static void end_io_acct(struct dm_io *io) | |||
497 | * After this is decremented the bio must not be touched if it is | 498 | * After this is decremented the bio must not be touched if it is |
498 | * a flush. | 499 | * a flush. |
499 | */ | 500 | */ |
500 | dm_disk(md)->part0.in_flight[rw] = pending = | 501 | pending = atomic_dec_return(&md->pending[rw]); |
501 | atomic_dec_return(&md->pending[rw]); | 502 | atomic_set(&dm_disk(md)->part0.in_flight[rw], pending); |
502 | pending += atomic_read(&md->pending[rw^0x1]); | 503 | pending += atomic_read(&md->pending[rw^0x1]); |
503 | 504 | ||
504 | /* nudge anyone waiting on suspend queue */ | 505 | /* nudge anyone waiting on suspend queue */ |
@@ -807,8 +808,6 @@ void dm_requeue_unmapped_request(struct request *clone) | |||
807 | dm_unprep_request(rq); | 808 | dm_unprep_request(rq); |
808 | 809 | ||
809 | spin_lock_irqsave(q->queue_lock, flags); | 810 | spin_lock_irqsave(q->queue_lock, flags); |
810 | if (elv_queue_empty(q)) | ||
811 | blk_plug_device(q); | ||
812 | blk_requeue_request(q, rq); | 811 | blk_requeue_request(q, rq); |
813 | spin_unlock_irqrestore(q->queue_lock, flags); | 812 | spin_unlock_irqrestore(q->queue_lock, flags); |
814 | 813 | ||
@@ -1613,10 +1612,10 @@ static void dm_request_fn(struct request_queue *q) | |||
1613 | * number of in-flight I/Os after the queue is stopped in | 1612 | * number of in-flight I/Os after the queue is stopped in |
1614 | * dm_suspend(). | 1613 | * dm_suspend(). |
1615 | */ | 1614 | */ |
1616 | while (!blk_queue_plugged(q) && !blk_queue_stopped(q)) { | 1615 | while (!blk_queue_stopped(q)) { |
1617 | rq = blk_peek_request(q); | 1616 | rq = blk_peek_request(q); |
1618 | if (!rq) | 1617 | if (!rq) |
1619 | goto plug_and_out; | 1618 | goto delay_and_out; |
1620 | 1619 | ||
1621 | /* always use block 0 to find the target for flushes for now */ | 1620 | /* always use block 0 to find the target for flushes for now */ |
1622 | pos = 0; | 1621 | pos = 0; |
@@ -1627,7 +1626,7 @@ static void dm_request_fn(struct request_queue *q) | |||
1627 | BUG_ON(!dm_target_is_valid(ti)); | 1626 | BUG_ON(!dm_target_is_valid(ti)); |
1628 | 1627 | ||
1629 | if (ti->type->busy && ti->type->busy(ti)) | 1628 | if (ti->type->busy && ti->type->busy(ti)) |
1630 | goto plug_and_out; | 1629 | goto delay_and_out; |
1631 | 1630 | ||
1632 | blk_start_request(rq); | 1631 | blk_start_request(rq); |
1633 | clone = rq->special; | 1632 | clone = rq->special; |
@@ -1647,11 +1646,8 @@ requeued: | |||
1647 | BUG_ON(!irqs_disabled()); | 1646 | BUG_ON(!irqs_disabled()); |
1648 | spin_lock(q->queue_lock); | 1647 | spin_lock(q->queue_lock); |
1649 | 1648 | ||
1650 | plug_and_out: | 1649 | delay_and_out: |
1651 | if (!elv_queue_empty(q)) | 1650 | blk_delay_queue(q, HZ / 10); |
1652 | /* Some requests still remain, retry later */ | ||
1653 | blk_plug_device(q); | ||
1654 | |||
1655 | out: | 1651 | out: |
1656 | dm_table_put(map); | 1652 | dm_table_put(map); |
1657 | 1653 | ||
@@ -1680,20 +1676,6 @@ static int dm_lld_busy(struct request_queue *q) | |||
1680 | return r; | 1676 | return r; |
1681 | } | 1677 | } |
1682 | 1678 | ||
1683 | static void dm_unplug_all(struct request_queue *q) | ||
1684 | { | ||
1685 | struct mapped_device *md = q->queuedata; | ||
1686 | struct dm_table *map = dm_get_live_table(md); | ||
1687 | |||
1688 | if (map) { | ||
1689 | if (dm_request_based(md)) | ||
1690 | generic_unplug_device(q); | ||
1691 | |||
1692 | dm_table_unplug_all(map); | ||
1693 | dm_table_put(map); | ||
1694 | } | ||
1695 | } | ||
1696 | |||
1697 | static int dm_any_congested(void *congested_data, int bdi_bits) | 1679 | static int dm_any_congested(void *congested_data, int bdi_bits) |
1698 | { | 1680 | { |
1699 | int r = bdi_bits; | 1681 | int r = bdi_bits; |
@@ -1817,7 +1799,6 @@ static void dm_init_md_queue(struct mapped_device *md) | |||
1817 | md->queue->backing_dev_info.congested_data = md; | 1799 | md->queue->backing_dev_info.congested_data = md; |
1818 | blk_queue_make_request(md->queue, dm_request); | 1800 | blk_queue_make_request(md->queue, dm_request); |
1819 | blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY); | 1801 | blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY); |
1820 | md->queue->unplug_fn = dm_unplug_all; | ||
1821 | blk_queue_merge_bvec(md->queue, dm_merge_bvec); | 1802 | blk_queue_merge_bvec(md->queue, dm_merge_bvec); |
1822 | blk_queue_flush(md->queue, REQ_FLUSH | REQ_FUA); | 1803 | blk_queue_flush(md->queue, REQ_FLUSH | REQ_FUA); |
1823 | } | 1804 | } |
@@ -2263,8 +2244,6 @@ static int dm_wait_for_completion(struct mapped_device *md, int interruptible) | |||
2263 | int r = 0; | 2244 | int r = 0; |
2264 | DECLARE_WAITQUEUE(wait, current); | 2245 | DECLARE_WAITQUEUE(wait, current); |
2265 | 2246 | ||
2266 | dm_unplug_all(md->queue); | ||
2267 | |||
2268 | add_wait_queue(&md->wait, &wait); | 2247 | add_wait_queue(&md->wait, &wait); |
2269 | 2248 | ||
2270 | while (1) { | 2249 | while (1) { |
@@ -2539,7 +2518,6 @@ int dm_resume(struct mapped_device *md) | |||
2539 | 2518 | ||
2540 | clear_bit(DMF_SUSPENDED, &md->flags); | 2519 | clear_bit(DMF_SUSPENDED, &md->flags); |
2541 | 2520 | ||
2542 | dm_table_unplug_all(map); | ||
2543 | r = 0; | 2521 | r = 0; |
2544 | out: | 2522 | out: |
2545 | dm_table_put(map); | 2523 | dm_table_put(map); |
@@ -2643,9 +2621,10 @@ int dm_noflush_suspending(struct dm_target *ti) | |||
2643 | } | 2621 | } |
2644 | EXPORT_SYMBOL_GPL(dm_noflush_suspending); | 2622 | EXPORT_SYMBOL_GPL(dm_noflush_suspending); |
2645 | 2623 | ||
2646 | struct dm_md_mempools *dm_alloc_md_mempools(unsigned type) | 2624 | struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity) |
2647 | { | 2625 | { |
2648 | struct dm_md_mempools *pools = kmalloc(sizeof(*pools), GFP_KERNEL); | 2626 | struct dm_md_mempools *pools = kmalloc(sizeof(*pools), GFP_KERNEL); |
2627 | unsigned int pool_size = (type == DM_TYPE_BIO_BASED) ? 16 : MIN_IOS; | ||
2649 | 2628 | ||
2650 | if (!pools) | 2629 | if (!pools) |
2651 | return NULL; | 2630 | return NULL; |
@@ -2662,13 +2641,18 @@ struct dm_md_mempools *dm_alloc_md_mempools(unsigned type) | |||
2662 | if (!pools->tio_pool) | 2641 | if (!pools->tio_pool) |
2663 | goto free_io_pool_and_out; | 2642 | goto free_io_pool_and_out; |
2664 | 2643 | ||
2665 | pools->bs = (type == DM_TYPE_BIO_BASED) ? | 2644 | pools->bs = bioset_create(pool_size, 0); |
2666 | bioset_create(16, 0) : bioset_create(MIN_IOS, 0); | ||
2667 | if (!pools->bs) | 2645 | if (!pools->bs) |
2668 | goto free_tio_pool_and_out; | 2646 | goto free_tio_pool_and_out; |
2669 | 2647 | ||
2648 | if (integrity && bioset_integrity_create(pools->bs, pool_size)) | ||
2649 | goto free_bioset_and_out; | ||
2650 | |||
2670 | return pools; | 2651 | return pools; |
2671 | 2652 | ||
2653 | free_bioset_and_out: | ||
2654 | bioset_free(pools->bs); | ||
2655 | |||
2672 | free_tio_pool_and_out: | 2656 | free_tio_pool_and_out: |
2673 | mempool_destroy(pools->tio_pool); | 2657 | mempool_destroy(pools->tio_pool); |
2674 | 2658 | ||
diff --git a/drivers/md/dm.h b/drivers/md/dm.h index 0c2dd5f4af76..1aaf16746da8 100644 --- a/drivers/md/dm.h +++ b/drivers/md/dm.h | |||
@@ -149,7 +149,7 @@ void dm_kcopyd_exit(void); | |||
149 | /* | 149 | /* |
150 | * Mempool operations | 150 | * Mempool operations |
151 | */ | 151 | */ |
152 | struct dm_md_mempools *dm_alloc_md_mempools(unsigned type); | 152 | struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity); |
153 | void dm_free_md_mempools(struct dm_md_mempools *pools); | 153 | void dm_free_md_mempools(struct dm_md_mempools *pools); |
154 | 154 | ||
155 | #endif | 155 | #endif |
diff --git a/drivers/md/linear.c b/drivers/md/linear.c index 0ed7f6bc2a7f..abfb59a61ede 100644 --- a/drivers/md/linear.c +++ b/drivers/md/linear.c | |||
@@ -87,22 +87,6 @@ static int linear_mergeable_bvec(struct request_queue *q, | |||
87 | return maxsectors << 9; | 87 | return maxsectors << 9; |
88 | } | 88 | } |
89 | 89 | ||
90 | static void linear_unplug(struct request_queue *q) | ||
91 | { | ||
92 | mddev_t *mddev = q->queuedata; | ||
93 | linear_conf_t *conf; | ||
94 | int i; | ||
95 | |||
96 | rcu_read_lock(); | ||
97 | conf = rcu_dereference(mddev->private); | ||
98 | |||
99 | for (i=0; i < mddev->raid_disks; i++) { | ||
100 | struct request_queue *r_queue = bdev_get_queue(conf->disks[i].rdev->bdev); | ||
101 | blk_unplug(r_queue); | ||
102 | } | ||
103 | rcu_read_unlock(); | ||
104 | } | ||
105 | |||
106 | static int linear_congested(void *data, int bits) | 90 | static int linear_congested(void *data, int bits) |
107 | { | 91 | { |
108 | mddev_t *mddev = data; | 92 | mddev_t *mddev = data; |
@@ -224,11 +208,9 @@ static int linear_run (mddev_t *mddev) | |||
224 | md_set_array_sectors(mddev, linear_size(mddev, 0, 0)); | 208 | md_set_array_sectors(mddev, linear_size(mddev, 0, 0)); |
225 | 209 | ||
226 | blk_queue_merge_bvec(mddev->queue, linear_mergeable_bvec); | 210 | blk_queue_merge_bvec(mddev->queue, linear_mergeable_bvec); |
227 | mddev->queue->unplug_fn = linear_unplug; | ||
228 | mddev->queue->backing_dev_info.congested_fn = linear_congested; | 211 | mddev->queue->backing_dev_info.congested_fn = linear_congested; |
229 | mddev->queue->backing_dev_info.congested_data = mddev; | 212 | mddev->queue->backing_dev_info.congested_data = mddev; |
230 | md_integrity_register(mddev); | 213 | return md_integrity_register(mddev); |
231 | return 0; | ||
232 | } | 214 | } |
233 | 215 | ||
234 | static void free_conf(struct rcu_head *head) | 216 | static void free_conf(struct rcu_head *head) |
diff --git a/drivers/md/md.c b/drivers/md/md.c index d5ad7723b172..06ecea751a39 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c | |||
@@ -780,8 +780,7 @@ void md_super_write(mddev_t *mddev, mdk_rdev_t *rdev, | |||
780 | bio->bi_end_io = super_written; | 780 | bio->bi_end_io = super_written; |
781 | 781 | ||
782 | atomic_inc(&mddev->pending_writes); | 782 | atomic_inc(&mddev->pending_writes); |
783 | submit_bio(REQ_WRITE | REQ_SYNC | REQ_UNPLUG | REQ_FLUSH | REQ_FUA, | 783 | submit_bio(REQ_WRITE | REQ_SYNC | REQ_FLUSH | REQ_FUA, bio); |
784 | bio); | ||
785 | } | 784 | } |
786 | 785 | ||
787 | void md_super_wait(mddev_t *mddev) | 786 | void md_super_wait(mddev_t *mddev) |
@@ -809,7 +808,7 @@ int sync_page_io(mdk_rdev_t *rdev, sector_t sector, int size, | |||
809 | struct completion event; | 808 | struct completion event; |
810 | int ret; | 809 | int ret; |
811 | 810 | ||
812 | rw |= REQ_SYNC | REQ_UNPLUG; | 811 | rw |= REQ_SYNC; |
813 | 812 | ||
814 | bio->bi_bdev = (metadata_op && rdev->meta_bdev) ? | 813 | bio->bi_bdev = (metadata_op && rdev->meta_bdev) ? |
815 | rdev->meta_bdev : rdev->bdev; | 814 | rdev->meta_bdev : rdev->bdev; |
@@ -1804,8 +1803,12 @@ int md_integrity_register(mddev_t *mddev) | |||
1804 | mdname(mddev)); | 1803 | mdname(mddev)); |
1805 | return -EINVAL; | 1804 | return -EINVAL; |
1806 | } | 1805 | } |
1807 | printk(KERN_NOTICE "md: data integrity on %s enabled\n", | 1806 | printk(KERN_NOTICE "md: data integrity enabled on %s\n", mdname(mddev)); |
1808 | mdname(mddev)); | 1807 | if (bioset_integrity_create(mddev->bio_set, BIO_POOL_SIZE)) { |
1808 | printk(KERN_ERR "md: failed to create integrity pool for %s\n", | ||
1809 | mdname(mddev)); | ||
1810 | return -EINVAL; | ||
1811 | } | ||
1809 | return 0; | 1812 | return 0; |
1810 | } | 1813 | } |
1811 | EXPORT_SYMBOL(md_integrity_register); | 1814 | EXPORT_SYMBOL(md_integrity_register); |
@@ -4817,7 +4820,6 @@ static int do_md_stop(mddev_t * mddev, int mode, int is_open) | |||
4817 | __md_stop_writes(mddev); | 4820 | __md_stop_writes(mddev); |
4818 | md_stop(mddev); | 4821 | md_stop(mddev); |
4819 | mddev->queue->merge_bvec_fn = NULL; | 4822 | mddev->queue->merge_bvec_fn = NULL; |
4820 | mddev->queue->unplug_fn = NULL; | ||
4821 | mddev->queue->backing_dev_info.congested_fn = NULL; | 4823 | mddev->queue->backing_dev_info.congested_fn = NULL; |
4822 | 4824 | ||
4823 | /* tell userspace to handle 'inactive' */ | 4825 | /* tell userspace to handle 'inactive' */ |
@@ -6692,8 +6694,6 @@ EXPORT_SYMBOL_GPL(md_allow_write); | |||
6692 | 6694 | ||
6693 | void md_unplug(mddev_t *mddev) | 6695 | void md_unplug(mddev_t *mddev) |
6694 | { | 6696 | { |
6695 | if (mddev->queue) | ||
6696 | blk_unplug(mddev->queue); | ||
6697 | if (mddev->plug) | 6697 | if (mddev->plug) |
6698 | mddev->plug->unplug_fn(mddev->plug); | 6698 | mddev->plug->unplug_fn(mddev->plug); |
6699 | } | 6699 | } |
@@ -6876,7 +6876,6 @@ void md_do_sync(mddev_t *mddev) | |||
6876 | >= mddev->resync_max - mddev->curr_resync_completed | 6876 | >= mddev->resync_max - mddev->curr_resync_completed |
6877 | )) { | 6877 | )) { |
6878 | /* time to update curr_resync_completed */ | 6878 | /* time to update curr_resync_completed */ |
6879 | md_unplug(mddev); | ||
6880 | wait_event(mddev->recovery_wait, | 6879 | wait_event(mddev->recovery_wait, |
6881 | atomic_read(&mddev->recovery_active) == 0); | 6880 | atomic_read(&mddev->recovery_active) == 0); |
6882 | mddev->curr_resync_completed = j; | 6881 | mddev->curr_resync_completed = j; |
@@ -6952,7 +6951,6 @@ void md_do_sync(mddev_t *mddev) | |||
6952 | * about not overloading the IO subsystem. (things like an | 6951 | * about not overloading the IO subsystem. (things like an |
6953 | * e2fsck being done on the RAID array should execute fast) | 6952 | * e2fsck being done on the RAID array should execute fast) |
6954 | */ | 6953 | */ |
6955 | md_unplug(mddev); | ||
6956 | cond_resched(); | 6954 | cond_resched(); |
6957 | 6955 | ||
6958 | currspeed = ((unsigned long)(io_sectors-mddev->resync_mark_cnt))/2 | 6956 | currspeed = ((unsigned long)(io_sectors-mddev->resync_mark_cnt))/2 |
@@ -6971,8 +6969,6 @@ void md_do_sync(mddev_t *mddev) | |||
6971 | * this also signals 'finished resyncing' to md_stop | 6969 | * this also signals 'finished resyncing' to md_stop |
6972 | */ | 6970 | */ |
6973 | out: | 6971 | out: |
6974 | md_unplug(mddev); | ||
6975 | |||
6976 | wait_event(mddev->recovery_wait, !atomic_read(&mddev->recovery_active)); | 6972 | wait_event(mddev->recovery_wait, !atomic_read(&mddev->recovery_active)); |
6977 | 6973 | ||
6978 | /* tell personality that we are finished */ | 6974 | /* tell personality that we are finished */ |
diff --git a/drivers/md/multipath.c b/drivers/md/multipath.c index 3a62d440e27b..c35890990985 100644 --- a/drivers/md/multipath.c +++ b/drivers/md/multipath.c | |||
@@ -106,36 +106,6 @@ static void multipath_end_request(struct bio *bio, int error) | |||
106 | rdev_dec_pending(rdev, conf->mddev); | 106 | rdev_dec_pending(rdev, conf->mddev); |
107 | } | 107 | } |
108 | 108 | ||
109 | static void unplug_slaves(mddev_t *mddev) | ||
110 | { | ||
111 | multipath_conf_t *conf = mddev->private; | ||
112 | int i; | ||
113 | |||
114 | rcu_read_lock(); | ||
115 | for (i=0; i<mddev->raid_disks; i++) { | ||
116 | mdk_rdev_t *rdev = rcu_dereference(conf->multipaths[i].rdev); | ||
117 | if (rdev && !test_bit(Faulty, &rdev->flags) | ||
118 | && atomic_read(&rdev->nr_pending)) { | ||
119 | struct request_queue *r_queue = bdev_get_queue(rdev->bdev); | ||
120 | |||
121 | atomic_inc(&rdev->nr_pending); | ||
122 | rcu_read_unlock(); | ||
123 | |||
124 | blk_unplug(r_queue); | ||
125 | |||
126 | rdev_dec_pending(rdev, mddev); | ||
127 | rcu_read_lock(); | ||
128 | } | ||
129 | } | ||
130 | rcu_read_unlock(); | ||
131 | } | ||
132 | |||
133 | static void multipath_unplug(struct request_queue *q) | ||
134 | { | ||
135 | unplug_slaves(q->queuedata); | ||
136 | } | ||
137 | |||
138 | |||
139 | static int multipath_make_request(mddev_t *mddev, struct bio * bio) | 109 | static int multipath_make_request(mddev_t *mddev, struct bio * bio) |
140 | { | 110 | { |
141 | multipath_conf_t *conf = mddev->private; | 111 | multipath_conf_t *conf = mddev->private; |
@@ -345,7 +315,7 @@ static int multipath_remove_disk(mddev_t *mddev, int number) | |||
345 | p->rdev = rdev; | 315 | p->rdev = rdev; |
346 | goto abort; | 316 | goto abort; |
347 | } | 317 | } |
348 | md_integrity_register(mddev); | 318 | err = md_integrity_register(mddev); |
349 | } | 319 | } |
350 | abort: | 320 | abort: |
351 | 321 | ||
@@ -517,10 +487,12 @@ static int multipath_run (mddev_t *mddev) | |||
517 | */ | 487 | */ |
518 | md_set_array_sectors(mddev, multipath_size(mddev, 0, 0)); | 488 | md_set_array_sectors(mddev, multipath_size(mddev, 0, 0)); |
519 | 489 | ||
520 | mddev->queue->unplug_fn = multipath_unplug; | ||
521 | mddev->queue->backing_dev_info.congested_fn = multipath_congested; | 490 | mddev->queue->backing_dev_info.congested_fn = multipath_congested; |
522 | mddev->queue->backing_dev_info.congested_data = mddev; | 491 | mddev->queue->backing_dev_info.congested_data = mddev; |
523 | md_integrity_register(mddev); | 492 | |
493 | if (md_integrity_register(mddev)) | ||
494 | goto out_free_conf; | ||
495 | |||
524 | return 0; | 496 | return 0; |
525 | 497 | ||
526 | out_free_conf: | 498 | out_free_conf: |
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index c0ac457f1218..e86bf3682e1e 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c | |||
@@ -25,21 +25,6 @@ | |||
25 | #include "raid0.h" | 25 | #include "raid0.h" |
26 | #include "raid5.h" | 26 | #include "raid5.h" |
27 | 27 | ||
28 | static void raid0_unplug(struct request_queue *q) | ||
29 | { | ||
30 | mddev_t *mddev = q->queuedata; | ||
31 | raid0_conf_t *conf = mddev->private; | ||
32 | mdk_rdev_t **devlist = conf->devlist; | ||
33 | int raid_disks = conf->strip_zone[0].nb_dev; | ||
34 | int i; | ||
35 | |||
36 | for (i=0; i < raid_disks; i++) { | ||
37 | struct request_queue *r_queue = bdev_get_queue(devlist[i]->bdev); | ||
38 | |||
39 | blk_unplug(r_queue); | ||
40 | } | ||
41 | } | ||
42 | |||
43 | static int raid0_congested(void *data, int bits) | 28 | static int raid0_congested(void *data, int bits) |
44 | { | 29 | { |
45 | mddev_t *mddev = data; | 30 | mddev_t *mddev = data; |
@@ -272,7 +257,6 @@ static int create_strip_zones(mddev_t *mddev, raid0_conf_t **private_conf) | |||
272 | mdname(mddev), | 257 | mdname(mddev), |
273 | (unsigned long long)smallest->sectors); | 258 | (unsigned long long)smallest->sectors); |
274 | } | 259 | } |
275 | mddev->queue->unplug_fn = raid0_unplug; | ||
276 | mddev->queue->backing_dev_info.congested_fn = raid0_congested; | 260 | mddev->queue->backing_dev_info.congested_fn = raid0_congested; |
277 | mddev->queue->backing_dev_info.congested_data = mddev; | 261 | mddev->queue->backing_dev_info.congested_data = mddev; |
278 | 262 | ||
@@ -395,8 +379,7 @@ static int raid0_run(mddev_t *mddev) | |||
395 | 379 | ||
396 | blk_queue_merge_bvec(mddev->queue, raid0_mergeable_bvec); | 380 | blk_queue_merge_bvec(mddev->queue, raid0_mergeable_bvec); |
397 | dump_zones(mddev); | 381 | dump_zones(mddev); |
398 | md_integrity_register(mddev); | 382 | return md_integrity_register(mddev); |
399 | return 0; | ||
400 | } | 383 | } |
401 | 384 | ||
402 | static int raid0_stop(mddev_t *mddev) | 385 | static int raid0_stop(mddev_t *mddev) |
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 06cd712807d0..c2a21ae56d97 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c | |||
@@ -52,23 +52,16 @@ | |||
52 | #define NR_RAID1_BIOS 256 | 52 | #define NR_RAID1_BIOS 256 |
53 | 53 | ||
54 | 54 | ||
55 | static void unplug_slaves(mddev_t *mddev); | ||
56 | |||
57 | static void allow_barrier(conf_t *conf); | 55 | static void allow_barrier(conf_t *conf); |
58 | static void lower_barrier(conf_t *conf); | 56 | static void lower_barrier(conf_t *conf); |
59 | 57 | ||
60 | static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data) | 58 | static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data) |
61 | { | 59 | { |
62 | struct pool_info *pi = data; | 60 | struct pool_info *pi = data; |
63 | r1bio_t *r1_bio; | ||
64 | int size = offsetof(r1bio_t, bios[pi->raid_disks]); | 61 | int size = offsetof(r1bio_t, bios[pi->raid_disks]); |
65 | 62 | ||
66 | /* allocate a r1bio with room for raid_disks entries in the bios array */ | 63 | /* allocate a r1bio with room for raid_disks entries in the bios array */ |
67 | r1_bio = kzalloc(size, gfp_flags); | 64 | return kzalloc(size, gfp_flags); |
68 | if (!r1_bio && pi->mddev) | ||
69 | unplug_slaves(pi->mddev); | ||
70 | |||
71 | return r1_bio; | ||
72 | } | 65 | } |
73 | 66 | ||
74 | static void r1bio_pool_free(void *r1_bio, void *data) | 67 | static void r1bio_pool_free(void *r1_bio, void *data) |
@@ -91,10 +84,8 @@ static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data) | |||
91 | int i, j; | 84 | int i, j; |
92 | 85 | ||
93 | r1_bio = r1bio_pool_alloc(gfp_flags, pi); | 86 | r1_bio = r1bio_pool_alloc(gfp_flags, pi); |
94 | if (!r1_bio) { | 87 | if (!r1_bio) |
95 | unplug_slaves(pi->mddev); | ||
96 | return NULL; | 88 | return NULL; |
97 | } | ||
98 | 89 | ||
99 | /* | 90 | /* |
100 | * Allocate bios : 1 for reading, n-1 for writing | 91 | * Allocate bios : 1 for reading, n-1 for writing |
@@ -520,37 +511,6 @@ static int read_balance(conf_t *conf, r1bio_t *r1_bio) | |||
520 | return new_disk; | 511 | return new_disk; |
521 | } | 512 | } |
522 | 513 | ||
523 | static void unplug_slaves(mddev_t *mddev) | ||
524 | { | ||
525 | conf_t *conf = mddev->private; | ||
526 | int i; | ||
527 | |||
528 | rcu_read_lock(); | ||
529 | for (i=0; i<mddev->raid_disks; i++) { | ||
530 | mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev); | ||
531 | if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) { | ||
532 | struct request_queue *r_queue = bdev_get_queue(rdev->bdev); | ||
533 | |||
534 | atomic_inc(&rdev->nr_pending); | ||
535 | rcu_read_unlock(); | ||
536 | |||
537 | blk_unplug(r_queue); | ||
538 | |||
539 | rdev_dec_pending(rdev, mddev); | ||
540 | rcu_read_lock(); | ||
541 | } | ||
542 | } | ||
543 | rcu_read_unlock(); | ||
544 | } | ||
545 | |||
546 | static void raid1_unplug(struct request_queue *q) | ||
547 | { | ||
548 | mddev_t *mddev = q->queuedata; | ||
549 | |||
550 | unplug_slaves(mddev); | ||
551 | md_wakeup_thread(mddev->thread); | ||
552 | } | ||
553 | |||
554 | static int raid1_congested(void *data, int bits) | 514 | static int raid1_congested(void *data, int bits) |
555 | { | 515 | { |
556 | mddev_t *mddev = data; | 516 | mddev_t *mddev = data; |
@@ -580,23 +540,16 @@ static int raid1_congested(void *data, int bits) | |||
580 | } | 540 | } |
581 | 541 | ||
582 | 542 | ||
583 | static int flush_pending_writes(conf_t *conf) | 543 | static void flush_pending_writes(conf_t *conf) |
584 | { | 544 | { |
585 | /* Any writes that have been queued but are awaiting | 545 | /* Any writes that have been queued but are awaiting |
586 | * bitmap updates get flushed here. | 546 | * bitmap updates get flushed here. |
587 | * We return 1 if any requests were actually submitted. | ||
588 | */ | 547 | */ |
589 | int rv = 0; | ||
590 | |||
591 | spin_lock_irq(&conf->device_lock); | 548 | spin_lock_irq(&conf->device_lock); |
592 | 549 | ||
593 | if (conf->pending_bio_list.head) { | 550 | if (conf->pending_bio_list.head) { |
594 | struct bio *bio; | 551 | struct bio *bio; |
595 | bio = bio_list_get(&conf->pending_bio_list); | 552 | bio = bio_list_get(&conf->pending_bio_list); |
596 | /* Only take the spinlock to quiet a warning */ | ||
597 | spin_lock(conf->mddev->queue->queue_lock); | ||
598 | blk_remove_plug(conf->mddev->queue); | ||
599 | spin_unlock(conf->mddev->queue->queue_lock); | ||
600 | spin_unlock_irq(&conf->device_lock); | 553 | spin_unlock_irq(&conf->device_lock); |
601 | /* flush any pending bitmap writes to | 554 | /* flush any pending bitmap writes to |
602 | * disk before proceeding w/ I/O */ | 555 | * disk before proceeding w/ I/O */ |
@@ -608,10 +561,14 @@ static int flush_pending_writes(conf_t *conf) | |||
608 | generic_make_request(bio); | 561 | generic_make_request(bio); |
609 | bio = next; | 562 | bio = next; |
610 | } | 563 | } |
611 | rv = 1; | ||
612 | } else | 564 | } else |
613 | spin_unlock_irq(&conf->device_lock); | 565 | spin_unlock_irq(&conf->device_lock); |
614 | return rv; | 566 | } |
567 | |||
568 | static void md_kick_device(mddev_t *mddev) | ||
569 | { | ||
570 | blk_flush_plug(current); | ||
571 | md_wakeup_thread(mddev->thread); | ||
615 | } | 572 | } |
616 | 573 | ||
617 | /* Barriers.... | 574 | /* Barriers.... |
@@ -643,8 +600,7 @@ static void raise_barrier(conf_t *conf) | |||
643 | 600 | ||
644 | /* Wait until no block IO is waiting */ | 601 | /* Wait until no block IO is waiting */ |
645 | wait_event_lock_irq(conf->wait_barrier, !conf->nr_waiting, | 602 | wait_event_lock_irq(conf->wait_barrier, !conf->nr_waiting, |
646 | conf->resync_lock, | 603 | conf->resync_lock, md_kick_device(conf->mddev)); |
647 | raid1_unplug(conf->mddev->queue)); | ||
648 | 604 | ||
649 | /* block any new IO from starting */ | 605 | /* block any new IO from starting */ |
650 | conf->barrier++; | 606 | conf->barrier++; |
@@ -652,8 +608,7 @@ static void raise_barrier(conf_t *conf) | |||
652 | /* Now wait for all pending IO to complete */ | 608 | /* Now wait for all pending IO to complete */ |
653 | wait_event_lock_irq(conf->wait_barrier, | 609 | wait_event_lock_irq(conf->wait_barrier, |
654 | !conf->nr_pending && conf->barrier < RESYNC_DEPTH, | 610 | !conf->nr_pending && conf->barrier < RESYNC_DEPTH, |
655 | conf->resync_lock, | 611 | conf->resync_lock, md_kick_device(conf->mddev)); |
656 | raid1_unplug(conf->mddev->queue)); | ||
657 | 612 | ||
658 | spin_unlock_irq(&conf->resync_lock); | 613 | spin_unlock_irq(&conf->resync_lock); |
659 | } | 614 | } |
@@ -675,7 +630,7 @@ static void wait_barrier(conf_t *conf) | |||
675 | conf->nr_waiting++; | 630 | conf->nr_waiting++; |
676 | wait_event_lock_irq(conf->wait_barrier, !conf->barrier, | 631 | wait_event_lock_irq(conf->wait_barrier, !conf->barrier, |
677 | conf->resync_lock, | 632 | conf->resync_lock, |
678 | raid1_unplug(conf->mddev->queue)); | 633 | md_kick_device(conf->mddev)); |
679 | conf->nr_waiting--; | 634 | conf->nr_waiting--; |
680 | } | 635 | } |
681 | conf->nr_pending++; | 636 | conf->nr_pending++; |
@@ -712,7 +667,7 @@ static void freeze_array(conf_t *conf) | |||
712 | conf->nr_pending == conf->nr_queued+1, | 667 | conf->nr_pending == conf->nr_queued+1, |
713 | conf->resync_lock, | 668 | conf->resync_lock, |
714 | ({ flush_pending_writes(conf); | 669 | ({ flush_pending_writes(conf); |
715 | raid1_unplug(conf->mddev->queue); })); | 670 | md_kick_device(conf->mddev); })); |
716 | spin_unlock_irq(&conf->resync_lock); | 671 | spin_unlock_irq(&conf->resync_lock); |
717 | } | 672 | } |
718 | static void unfreeze_array(conf_t *conf) | 673 | static void unfreeze_array(conf_t *conf) |
@@ -962,7 +917,6 @@ static int make_request(mddev_t *mddev, struct bio * bio) | |||
962 | atomic_inc(&r1_bio->remaining); | 917 | atomic_inc(&r1_bio->remaining); |
963 | spin_lock_irqsave(&conf->device_lock, flags); | 918 | spin_lock_irqsave(&conf->device_lock, flags); |
964 | bio_list_add(&conf->pending_bio_list, mbio); | 919 | bio_list_add(&conf->pending_bio_list, mbio); |
965 | blk_plug_device_unlocked(mddev->queue); | ||
966 | spin_unlock_irqrestore(&conf->device_lock, flags); | 920 | spin_unlock_irqrestore(&conf->device_lock, flags); |
967 | } | 921 | } |
968 | r1_bio_write_done(r1_bio, bio->bi_vcnt, behind_pages, behind_pages != NULL); | 922 | r1_bio_write_done(r1_bio, bio->bi_vcnt, behind_pages, behind_pages != NULL); |
@@ -971,7 +925,7 @@ static int make_request(mddev_t *mddev, struct bio * bio) | |||
971 | /* In case raid1d snuck in to freeze_array */ | 925 | /* In case raid1d snuck in to freeze_array */ |
972 | wake_up(&conf->wait_barrier); | 926 | wake_up(&conf->wait_barrier); |
973 | 927 | ||
974 | if (do_sync) | 928 | if (do_sync || !bitmap) |
975 | md_wakeup_thread(mddev->thread); | 929 | md_wakeup_thread(mddev->thread); |
976 | 930 | ||
977 | return 0; | 931 | return 0; |
@@ -1178,7 +1132,7 @@ static int raid1_remove_disk(mddev_t *mddev, int number) | |||
1178 | p->rdev = rdev; | 1132 | p->rdev = rdev; |
1179 | goto abort; | 1133 | goto abort; |
1180 | } | 1134 | } |
1181 | md_integrity_register(mddev); | 1135 | err = md_integrity_register(mddev); |
1182 | } | 1136 | } |
1183 | abort: | 1137 | abort: |
1184 | 1138 | ||
@@ -1561,7 +1515,6 @@ static void raid1d(mddev_t *mddev) | |||
1561 | unsigned long flags; | 1515 | unsigned long flags; |
1562 | conf_t *conf = mddev->private; | 1516 | conf_t *conf = mddev->private; |
1563 | struct list_head *head = &conf->retry_list; | 1517 | struct list_head *head = &conf->retry_list; |
1564 | int unplug=0; | ||
1565 | mdk_rdev_t *rdev; | 1518 | mdk_rdev_t *rdev; |
1566 | 1519 | ||
1567 | md_check_recovery(mddev); | 1520 | md_check_recovery(mddev); |
@@ -1569,7 +1522,7 @@ static void raid1d(mddev_t *mddev) | |||
1569 | for (;;) { | 1522 | for (;;) { |
1570 | char b[BDEVNAME_SIZE]; | 1523 | char b[BDEVNAME_SIZE]; |
1571 | 1524 | ||
1572 | unplug += flush_pending_writes(conf); | 1525 | flush_pending_writes(conf); |
1573 | 1526 | ||
1574 | spin_lock_irqsave(&conf->device_lock, flags); | 1527 | spin_lock_irqsave(&conf->device_lock, flags); |
1575 | if (list_empty(head)) { | 1528 | if (list_empty(head)) { |
@@ -1583,10 +1536,9 @@ static void raid1d(mddev_t *mddev) | |||
1583 | 1536 | ||
1584 | mddev = r1_bio->mddev; | 1537 | mddev = r1_bio->mddev; |
1585 | conf = mddev->private; | 1538 | conf = mddev->private; |
1586 | if (test_bit(R1BIO_IsSync, &r1_bio->state)) { | 1539 | if (test_bit(R1BIO_IsSync, &r1_bio->state)) |
1587 | sync_request_write(mddev, r1_bio); | 1540 | sync_request_write(mddev, r1_bio); |
1588 | unplug = 1; | 1541 | else { |
1589 | } else { | ||
1590 | int disk; | 1542 | int disk; |
1591 | 1543 | ||
1592 | /* we got a read error. Maybe the drive is bad. Maybe just | 1544 | /* we got a read error. Maybe the drive is bad. Maybe just |
@@ -1636,14 +1588,11 @@ static void raid1d(mddev_t *mddev) | |||
1636 | bio->bi_end_io = raid1_end_read_request; | 1588 | bio->bi_end_io = raid1_end_read_request; |
1637 | bio->bi_rw = READ | do_sync; | 1589 | bio->bi_rw = READ | do_sync; |
1638 | bio->bi_private = r1_bio; | 1590 | bio->bi_private = r1_bio; |
1639 | unplug = 1; | ||
1640 | generic_make_request(bio); | 1591 | generic_make_request(bio); |
1641 | } | 1592 | } |
1642 | } | 1593 | } |
1643 | cond_resched(); | 1594 | cond_resched(); |
1644 | } | 1595 | } |
1645 | if (unplug) | ||
1646 | unplug_slaves(mddev); | ||
1647 | } | 1596 | } |
1648 | 1597 | ||
1649 | 1598 | ||
@@ -2066,11 +2015,9 @@ static int run(mddev_t *mddev) | |||
2066 | 2015 | ||
2067 | md_set_array_sectors(mddev, raid1_size(mddev, 0, 0)); | 2016 | md_set_array_sectors(mddev, raid1_size(mddev, 0, 0)); |
2068 | 2017 | ||
2069 | mddev->queue->unplug_fn = raid1_unplug; | ||
2070 | mddev->queue->backing_dev_info.congested_fn = raid1_congested; | 2018 | mddev->queue->backing_dev_info.congested_fn = raid1_congested; |
2071 | mddev->queue->backing_dev_info.congested_data = mddev; | 2019 | mddev->queue->backing_dev_info.congested_data = mddev; |
2072 | md_integrity_register(mddev); | 2020 | return md_integrity_register(mddev); |
2073 | return 0; | ||
2074 | } | 2021 | } |
2075 | 2022 | ||
2076 | static int stop(mddev_t *mddev) | 2023 | static int stop(mddev_t *mddev) |
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 747d061d8e05..f7b62370b374 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c | |||
@@ -57,23 +57,16 @@ | |||
57 | */ | 57 | */ |
58 | #define NR_RAID10_BIOS 256 | 58 | #define NR_RAID10_BIOS 256 |
59 | 59 | ||
60 | static void unplug_slaves(mddev_t *mddev); | ||
61 | |||
62 | static void allow_barrier(conf_t *conf); | 60 | static void allow_barrier(conf_t *conf); |
63 | static void lower_barrier(conf_t *conf); | 61 | static void lower_barrier(conf_t *conf); |
64 | 62 | ||
65 | static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) | 63 | static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) |
66 | { | 64 | { |
67 | conf_t *conf = data; | 65 | conf_t *conf = data; |
68 | r10bio_t *r10_bio; | ||
69 | int size = offsetof(struct r10bio_s, devs[conf->copies]); | 66 | int size = offsetof(struct r10bio_s, devs[conf->copies]); |
70 | 67 | ||
71 | /* allocate a r10bio with room for raid_disks entries in the bios array */ | 68 | /* allocate a r10bio with room for raid_disks entries in the bios array */ |
72 | r10_bio = kzalloc(size, gfp_flags); | 69 | return kzalloc(size, gfp_flags); |
73 | if (!r10_bio && conf->mddev) | ||
74 | unplug_slaves(conf->mddev); | ||
75 | |||
76 | return r10_bio; | ||
77 | } | 70 | } |
78 | 71 | ||
79 | static void r10bio_pool_free(void *r10_bio, void *data) | 72 | static void r10bio_pool_free(void *r10_bio, void *data) |
@@ -106,10 +99,8 @@ static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data) | |||
106 | int nalloc; | 99 | int nalloc; |
107 | 100 | ||
108 | r10_bio = r10bio_pool_alloc(gfp_flags, conf); | 101 | r10_bio = r10bio_pool_alloc(gfp_flags, conf); |
109 | if (!r10_bio) { | 102 | if (!r10_bio) |
110 | unplug_slaves(conf->mddev); | ||
111 | return NULL; | 103 | return NULL; |
112 | } | ||
113 | 104 | ||
114 | if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery)) | 105 | if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery)) |
115 | nalloc = conf->copies; /* resync */ | 106 | nalloc = conf->copies; /* resync */ |
@@ -597,37 +588,6 @@ rb_out: | |||
597 | return disk; | 588 | return disk; |
598 | } | 589 | } |
599 | 590 | ||
600 | static void unplug_slaves(mddev_t *mddev) | ||
601 | { | ||
602 | conf_t *conf = mddev->private; | ||
603 | int i; | ||
604 | |||
605 | rcu_read_lock(); | ||
606 | for (i=0; i < conf->raid_disks; i++) { | ||
607 | mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev); | ||
608 | if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) { | ||
609 | struct request_queue *r_queue = bdev_get_queue(rdev->bdev); | ||
610 | |||
611 | atomic_inc(&rdev->nr_pending); | ||
612 | rcu_read_unlock(); | ||
613 | |||
614 | blk_unplug(r_queue); | ||
615 | |||
616 | rdev_dec_pending(rdev, mddev); | ||
617 | rcu_read_lock(); | ||
618 | } | ||
619 | } | ||
620 | rcu_read_unlock(); | ||
621 | } | ||
622 | |||
623 | static void raid10_unplug(struct request_queue *q) | ||
624 | { | ||
625 | mddev_t *mddev = q->queuedata; | ||
626 | |||
627 | unplug_slaves(q->queuedata); | ||
628 | md_wakeup_thread(mddev->thread); | ||
629 | } | ||
630 | |||
631 | static int raid10_congested(void *data, int bits) | 591 | static int raid10_congested(void *data, int bits) |
632 | { | 592 | { |
633 | mddev_t *mddev = data; | 593 | mddev_t *mddev = data; |
@@ -649,23 +609,16 @@ static int raid10_congested(void *data, int bits) | |||
649 | return ret; | 609 | return ret; |
650 | } | 610 | } |
651 | 611 | ||
652 | static int flush_pending_writes(conf_t *conf) | 612 | static void flush_pending_writes(conf_t *conf) |
653 | { | 613 | { |
654 | /* Any writes that have been queued but are awaiting | 614 | /* Any writes that have been queued but are awaiting |
655 | * bitmap updates get flushed here. | 615 | * bitmap updates get flushed here. |
656 | * We return 1 if any requests were actually submitted. | ||
657 | */ | 616 | */ |
658 | int rv = 0; | ||
659 | |||
660 | spin_lock_irq(&conf->device_lock); | 617 | spin_lock_irq(&conf->device_lock); |
661 | 618 | ||
662 | if (conf->pending_bio_list.head) { | 619 | if (conf->pending_bio_list.head) { |
663 | struct bio *bio; | 620 | struct bio *bio; |
664 | bio = bio_list_get(&conf->pending_bio_list); | 621 | bio = bio_list_get(&conf->pending_bio_list); |
665 | /* Spinlock only taken to quiet a warning */ | ||
666 | spin_lock(conf->mddev->queue->queue_lock); | ||
667 | blk_remove_plug(conf->mddev->queue); | ||
668 | spin_unlock(conf->mddev->queue->queue_lock); | ||
669 | spin_unlock_irq(&conf->device_lock); | 622 | spin_unlock_irq(&conf->device_lock); |
670 | /* flush any pending bitmap writes to disk | 623 | /* flush any pending bitmap writes to disk |
671 | * before proceeding w/ I/O */ | 624 | * before proceeding w/ I/O */ |
@@ -677,11 +630,16 @@ static int flush_pending_writes(conf_t *conf) | |||
677 | generic_make_request(bio); | 630 | generic_make_request(bio); |
678 | bio = next; | 631 | bio = next; |
679 | } | 632 | } |
680 | rv = 1; | ||
681 | } else | 633 | } else |
682 | spin_unlock_irq(&conf->device_lock); | 634 | spin_unlock_irq(&conf->device_lock); |
683 | return rv; | ||
684 | } | 635 | } |
636 | |||
637 | static void md_kick_device(mddev_t *mddev) | ||
638 | { | ||
639 | blk_flush_plug(current); | ||
640 | md_wakeup_thread(mddev->thread); | ||
641 | } | ||
642 | |||
685 | /* Barriers.... | 643 | /* Barriers.... |
686 | * Sometimes we need to suspend IO while we do something else, | 644 | * Sometimes we need to suspend IO while we do something else, |
687 | * either some resync/recovery, or reconfigure the array. | 645 | * either some resync/recovery, or reconfigure the array. |
@@ -711,8 +669,7 @@ static void raise_barrier(conf_t *conf, int force) | |||
711 | 669 | ||
712 | /* Wait until no block IO is waiting (unless 'force') */ | 670 | /* Wait until no block IO is waiting (unless 'force') */ |
713 | wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting, | 671 | wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting, |
714 | conf->resync_lock, | 672 | conf->resync_lock, md_kick_device(conf->mddev)); |
715 | raid10_unplug(conf->mddev->queue)); | ||
716 | 673 | ||
717 | /* block any new IO from starting */ | 674 | /* block any new IO from starting */ |
718 | conf->barrier++; | 675 | conf->barrier++; |
@@ -720,8 +677,7 @@ static void raise_barrier(conf_t *conf, int force) | |||
720 | /* No wait for all pending IO to complete */ | 677 | /* No wait for all pending IO to complete */ |
721 | wait_event_lock_irq(conf->wait_barrier, | 678 | wait_event_lock_irq(conf->wait_barrier, |
722 | !conf->nr_pending && conf->barrier < RESYNC_DEPTH, | 679 | !conf->nr_pending && conf->barrier < RESYNC_DEPTH, |
723 | conf->resync_lock, | 680 | conf->resync_lock, md_kick_device(conf->mddev)); |
724 | raid10_unplug(conf->mddev->queue)); | ||
725 | 681 | ||
726 | spin_unlock_irq(&conf->resync_lock); | 682 | spin_unlock_irq(&conf->resync_lock); |
727 | } | 683 | } |
@@ -742,7 +698,7 @@ static void wait_barrier(conf_t *conf) | |||
742 | conf->nr_waiting++; | 698 | conf->nr_waiting++; |
743 | wait_event_lock_irq(conf->wait_barrier, !conf->barrier, | 699 | wait_event_lock_irq(conf->wait_barrier, !conf->barrier, |
744 | conf->resync_lock, | 700 | conf->resync_lock, |
745 | raid10_unplug(conf->mddev->queue)); | 701 | md_kick_device(conf->mddev)); |
746 | conf->nr_waiting--; | 702 | conf->nr_waiting--; |
747 | } | 703 | } |
748 | conf->nr_pending++; | 704 | conf->nr_pending++; |
@@ -779,7 +735,7 @@ static void freeze_array(conf_t *conf) | |||
779 | conf->nr_pending == conf->nr_queued+1, | 735 | conf->nr_pending == conf->nr_queued+1, |
780 | conf->resync_lock, | 736 | conf->resync_lock, |
781 | ({ flush_pending_writes(conf); | 737 | ({ flush_pending_writes(conf); |
782 | raid10_unplug(conf->mddev->queue); })); | 738 | md_kick_device(conf->mddev); })); |
783 | spin_unlock_irq(&conf->resync_lock); | 739 | spin_unlock_irq(&conf->resync_lock); |
784 | } | 740 | } |
785 | 741 | ||
@@ -974,7 +930,6 @@ static int make_request(mddev_t *mddev, struct bio * bio) | |||
974 | atomic_inc(&r10_bio->remaining); | 930 | atomic_inc(&r10_bio->remaining); |
975 | spin_lock_irqsave(&conf->device_lock, flags); | 931 | spin_lock_irqsave(&conf->device_lock, flags); |
976 | bio_list_add(&conf->pending_bio_list, mbio); | 932 | bio_list_add(&conf->pending_bio_list, mbio); |
977 | blk_plug_device_unlocked(mddev->queue); | ||
978 | spin_unlock_irqrestore(&conf->device_lock, flags); | 933 | spin_unlock_irqrestore(&conf->device_lock, flags); |
979 | } | 934 | } |
980 | 935 | ||
@@ -991,7 +946,7 @@ static int make_request(mddev_t *mddev, struct bio * bio) | |||
991 | /* In case raid10d snuck in to freeze_array */ | 946 | /* In case raid10d snuck in to freeze_array */ |
992 | wake_up(&conf->wait_barrier); | 947 | wake_up(&conf->wait_barrier); |
993 | 948 | ||
994 | if (do_sync) | 949 | if (do_sync || !mddev->bitmap) |
995 | md_wakeup_thread(mddev->thread); | 950 | md_wakeup_thread(mddev->thread); |
996 | 951 | ||
997 | return 0; | 952 | return 0; |
@@ -1233,7 +1188,7 @@ static int raid10_remove_disk(mddev_t *mddev, int number) | |||
1233 | p->rdev = rdev; | 1188 | p->rdev = rdev; |
1234 | goto abort; | 1189 | goto abort; |
1235 | } | 1190 | } |
1236 | md_integrity_register(mddev); | 1191 | err = md_integrity_register(mddev); |
1237 | } | 1192 | } |
1238 | abort: | 1193 | abort: |
1239 | 1194 | ||
@@ -1684,7 +1639,6 @@ static void raid10d(mddev_t *mddev) | |||
1684 | unsigned long flags; | 1639 | unsigned long flags; |
1685 | conf_t *conf = mddev->private; | 1640 | conf_t *conf = mddev->private; |
1686 | struct list_head *head = &conf->retry_list; | 1641 | struct list_head *head = &conf->retry_list; |
1687 | int unplug=0; | ||
1688 | mdk_rdev_t *rdev; | 1642 | mdk_rdev_t *rdev; |
1689 | 1643 | ||
1690 | md_check_recovery(mddev); | 1644 | md_check_recovery(mddev); |
@@ -1692,7 +1646,7 @@ static void raid10d(mddev_t *mddev) | |||
1692 | for (;;) { | 1646 | for (;;) { |
1693 | char b[BDEVNAME_SIZE]; | 1647 | char b[BDEVNAME_SIZE]; |
1694 | 1648 | ||
1695 | unplug += flush_pending_writes(conf); | 1649 | flush_pending_writes(conf); |
1696 | 1650 | ||
1697 | spin_lock_irqsave(&conf->device_lock, flags); | 1651 | spin_lock_irqsave(&conf->device_lock, flags); |
1698 | if (list_empty(head)) { | 1652 | if (list_empty(head)) { |
@@ -1706,13 +1660,11 @@ static void raid10d(mddev_t *mddev) | |||
1706 | 1660 | ||
1707 | mddev = r10_bio->mddev; | 1661 | mddev = r10_bio->mddev; |
1708 | conf = mddev->private; | 1662 | conf = mddev->private; |
1709 | if (test_bit(R10BIO_IsSync, &r10_bio->state)) { | 1663 | if (test_bit(R10BIO_IsSync, &r10_bio->state)) |
1710 | sync_request_write(mddev, r10_bio); | 1664 | sync_request_write(mddev, r10_bio); |
1711 | unplug = 1; | 1665 | else if (test_bit(R10BIO_IsRecover, &r10_bio->state)) |
1712 | } else if (test_bit(R10BIO_IsRecover, &r10_bio->state)) { | ||
1713 | recovery_request_write(mddev, r10_bio); | 1666 | recovery_request_write(mddev, r10_bio); |
1714 | unplug = 1; | 1667 | else { |
1715 | } else { | ||
1716 | int mirror; | 1668 | int mirror; |
1717 | /* we got a read error. Maybe the drive is bad. Maybe just | 1669 | /* we got a read error. Maybe the drive is bad. Maybe just |
1718 | * the block and we can fix it. | 1670 | * the block and we can fix it. |
@@ -1759,14 +1711,11 @@ static void raid10d(mddev_t *mddev) | |||
1759 | bio->bi_rw = READ | do_sync; | 1711 | bio->bi_rw = READ | do_sync; |
1760 | bio->bi_private = r10_bio; | 1712 | bio->bi_private = r10_bio; |
1761 | bio->bi_end_io = raid10_end_read_request; | 1713 | bio->bi_end_io = raid10_end_read_request; |
1762 | unplug = 1; | ||
1763 | generic_make_request(bio); | 1714 | generic_make_request(bio); |
1764 | } | 1715 | } |
1765 | } | 1716 | } |
1766 | cond_resched(); | 1717 | cond_resched(); |
1767 | } | 1718 | } |
1768 | if (unplug) | ||
1769 | unplug_slaves(mddev); | ||
1770 | } | 1719 | } |
1771 | 1720 | ||
1772 | 1721 | ||
@@ -2377,7 +2326,6 @@ static int run(mddev_t *mddev) | |||
2377 | md_set_array_sectors(mddev, size); | 2326 | md_set_array_sectors(mddev, size); |
2378 | mddev->resync_max_sectors = size; | 2327 | mddev->resync_max_sectors = size; |
2379 | 2328 | ||
2380 | mddev->queue->unplug_fn = raid10_unplug; | ||
2381 | mddev->queue->backing_dev_info.congested_fn = raid10_congested; | 2329 | mddev->queue->backing_dev_info.congested_fn = raid10_congested; |
2382 | mddev->queue->backing_dev_info.congested_data = mddev; | 2330 | mddev->queue->backing_dev_info.congested_data = mddev; |
2383 | 2331 | ||
@@ -2395,7 +2343,10 @@ static int run(mddev_t *mddev) | |||
2395 | 2343 | ||
2396 | if (conf->near_copies < conf->raid_disks) | 2344 | if (conf->near_copies < conf->raid_disks) |
2397 | blk_queue_merge_bvec(mddev->queue, raid10_mergeable_bvec); | 2345 | blk_queue_merge_bvec(mddev->queue, raid10_mergeable_bvec); |
2398 | md_integrity_register(mddev); | 2346 | |
2347 | if (md_integrity_register(mddev)) | ||
2348 | goto out_free_conf; | ||
2349 | |||
2399 | return 0; | 2350 | return 0; |
2400 | 2351 | ||
2401 | out_free_conf: | 2352 | out_free_conf: |
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 78536fdbd87f..e867ee42b152 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c | |||
@@ -433,8 +433,6 @@ static int has_failed(raid5_conf_t *conf) | |||
433 | return 0; | 433 | return 0; |
434 | } | 434 | } |
435 | 435 | ||
436 | static void unplug_slaves(mddev_t *mddev); | ||
437 | |||
438 | static struct stripe_head * | 436 | static struct stripe_head * |
439 | get_active_stripe(raid5_conf_t *conf, sector_t sector, | 437 | get_active_stripe(raid5_conf_t *conf, sector_t sector, |
440 | int previous, int noblock, int noquiesce) | 438 | int previous, int noblock, int noquiesce) |
@@ -463,8 +461,7 @@ get_active_stripe(raid5_conf_t *conf, sector_t sector, | |||
463 | < (conf->max_nr_stripes *3/4) | 461 | < (conf->max_nr_stripes *3/4) |
464 | || !conf->inactive_blocked), | 462 | || !conf->inactive_blocked), |
465 | conf->device_lock, | 463 | conf->device_lock, |
466 | md_raid5_unplug_device(conf) | 464 | md_raid5_kick_device(conf)); |
467 | ); | ||
468 | conf->inactive_blocked = 0; | 465 | conf->inactive_blocked = 0; |
469 | } else | 466 | } else |
470 | init_stripe(sh, sector, previous); | 467 | init_stripe(sh, sector, previous); |
@@ -1473,8 +1470,7 @@ static int resize_stripes(raid5_conf_t *conf, int newsize) | |||
1473 | wait_event_lock_irq(conf->wait_for_stripe, | 1470 | wait_event_lock_irq(conf->wait_for_stripe, |
1474 | !list_empty(&conf->inactive_list), | 1471 | !list_empty(&conf->inactive_list), |
1475 | conf->device_lock, | 1472 | conf->device_lock, |
1476 | unplug_slaves(conf->mddev) | 1473 | blk_flush_plug(current)); |
1477 | ); | ||
1478 | osh = get_free_stripe(conf); | 1474 | osh = get_free_stripe(conf); |
1479 | spin_unlock_irq(&conf->device_lock); | 1475 | spin_unlock_irq(&conf->device_lock); |
1480 | atomic_set(&nsh->count, 1); | 1476 | atomic_set(&nsh->count, 1); |
@@ -3645,58 +3641,19 @@ static void activate_bit_delay(raid5_conf_t *conf) | |||
3645 | } | 3641 | } |
3646 | } | 3642 | } |
3647 | 3643 | ||
3648 | static void unplug_slaves(mddev_t *mddev) | 3644 | void md_raid5_kick_device(raid5_conf_t *conf) |
3649 | { | 3645 | { |
3650 | raid5_conf_t *conf = mddev->private; | 3646 | blk_flush_plug(current); |
3651 | int i; | 3647 | raid5_activate_delayed(conf); |
3652 | int devs = max(conf->raid_disks, conf->previous_raid_disks); | ||
3653 | |||
3654 | rcu_read_lock(); | ||
3655 | for (i = 0; i < devs; i++) { | ||
3656 | mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev); | ||
3657 | if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) { | ||
3658 | struct request_queue *r_queue = bdev_get_queue(rdev->bdev); | ||
3659 | |||
3660 | atomic_inc(&rdev->nr_pending); | ||
3661 | rcu_read_unlock(); | ||
3662 | |||
3663 | blk_unplug(r_queue); | ||
3664 | |||
3665 | rdev_dec_pending(rdev, mddev); | ||
3666 | rcu_read_lock(); | ||
3667 | } | ||
3668 | } | ||
3669 | rcu_read_unlock(); | ||
3670 | } | ||
3671 | |||
3672 | void md_raid5_unplug_device(raid5_conf_t *conf) | ||
3673 | { | ||
3674 | unsigned long flags; | ||
3675 | |||
3676 | spin_lock_irqsave(&conf->device_lock, flags); | ||
3677 | |||
3678 | if (plugger_remove_plug(&conf->plug)) { | ||
3679 | conf->seq_flush++; | ||
3680 | raid5_activate_delayed(conf); | ||
3681 | } | ||
3682 | md_wakeup_thread(conf->mddev->thread); | 3648 | md_wakeup_thread(conf->mddev->thread); |
3683 | |||
3684 | spin_unlock_irqrestore(&conf->device_lock, flags); | ||
3685 | |||
3686 | unplug_slaves(conf->mddev); | ||
3687 | } | 3649 | } |
3688 | EXPORT_SYMBOL_GPL(md_raid5_unplug_device); | 3650 | EXPORT_SYMBOL_GPL(md_raid5_kick_device); |
3689 | 3651 | ||
3690 | static void raid5_unplug(struct plug_handle *plug) | 3652 | static void raid5_unplug(struct plug_handle *plug) |
3691 | { | 3653 | { |
3692 | raid5_conf_t *conf = container_of(plug, raid5_conf_t, plug); | 3654 | raid5_conf_t *conf = container_of(plug, raid5_conf_t, plug); |
3693 | md_raid5_unplug_device(conf); | ||
3694 | } | ||
3695 | 3655 | ||
3696 | static void raid5_unplug_queue(struct request_queue *q) | 3656 | md_raid5_kick_device(conf); |
3697 | { | ||
3698 | mddev_t *mddev = q->queuedata; | ||
3699 | md_raid5_unplug_device(mddev->private); | ||
3700 | } | 3657 | } |
3701 | 3658 | ||
3702 | int md_raid5_congested(mddev_t *mddev, int bits) | 3659 | int md_raid5_congested(mddev_t *mddev, int bits) |
@@ -4100,7 +4057,7 @@ static int make_request(mddev_t *mddev, struct bio * bi) | |||
4100 | * add failed due to overlap. Flush everything | 4057 | * add failed due to overlap. Flush everything |
4101 | * and wait a while | 4058 | * and wait a while |
4102 | */ | 4059 | */ |
4103 | md_raid5_unplug_device(conf); | 4060 | md_raid5_kick_device(conf); |
4104 | release_stripe(sh); | 4061 | release_stripe(sh); |
4105 | schedule(); | 4062 | schedule(); |
4106 | goto retry; | 4063 | goto retry; |
@@ -4365,7 +4322,6 @@ static inline sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *ski | |||
4365 | 4322 | ||
4366 | if (sector_nr >= max_sector) { | 4323 | if (sector_nr >= max_sector) { |
4367 | /* just being told to finish up .. nothing much to do */ | 4324 | /* just being told to finish up .. nothing much to do */ |
4368 | unplug_slaves(mddev); | ||
4369 | 4325 | ||
4370 | if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { | 4326 | if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { |
4371 | end_reshape(conf); | 4327 | end_reshape(conf); |
@@ -4569,7 +4525,6 @@ static void raid5d(mddev_t *mddev) | |||
4569 | spin_unlock_irq(&conf->device_lock); | 4525 | spin_unlock_irq(&conf->device_lock); |
4570 | 4526 | ||
4571 | async_tx_issue_pending_all(); | 4527 | async_tx_issue_pending_all(); |
4572 | unplug_slaves(mddev); | ||
4573 | 4528 | ||
4574 | pr_debug("--- raid5d inactive\n"); | 4529 | pr_debug("--- raid5d inactive\n"); |
4575 | } | 4530 | } |
@@ -5204,7 +5159,7 @@ static int run(mddev_t *mddev) | |||
5204 | 5159 | ||
5205 | mddev->queue->backing_dev_info.congested_data = mddev; | 5160 | mddev->queue->backing_dev_info.congested_data = mddev; |
5206 | mddev->queue->backing_dev_info.congested_fn = raid5_congested; | 5161 | mddev->queue->backing_dev_info.congested_fn = raid5_congested; |
5207 | mddev->queue->unplug_fn = raid5_unplug_queue; | 5162 | mddev->queue->queue_lock = &conf->device_lock; |
5208 | 5163 | ||
5209 | chunk_size = mddev->chunk_sectors << 9; | 5164 | chunk_size = mddev->chunk_sectors << 9; |
5210 | blk_queue_io_min(mddev->queue, chunk_size); | 5165 | blk_queue_io_min(mddev->queue, chunk_size); |
diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h index 2ace0582b409..8d563a4f022a 100644 --- a/drivers/md/raid5.h +++ b/drivers/md/raid5.h | |||
@@ -503,6 +503,6 @@ static inline int algorithm_is_DDF(int layout) | |||
503 | } | 503 | } |
504 | 504 | ||
505 | extern int md_raid5_congested(mddev_t *mddev, int bits); | 505 | extern int md_raid5_congested(mddev_t *mddev, int bits); |
506 | extern void md_raid5_unplug_device(raid5_conf_t *conf); | 506 | extern void md_raid5_kick_device(raid5_conf_t *conf); |
507 | extern int raid5_set_cache_size(mddev_t *mddev, int size); | 507 | extern int raid5_set_cache_size(mddev_t *mddev, int size); |
508 | #endif | 508 | #endif |
diff --git a/drivers/message/i2o/i2o_block.c b/drivers/message/i2o/i2o_block.c index ae7cad185898..47ec5bc0ed21 100644 --- a/drivers/message/i2o/i2o_block.c +++ b/drivers/message/i2o/i2o_block.c | |||
@@ -695,20 +695,22 @@ static int i2o_block_ioctl(struct block_device *bdev, fmode_t mode, | |||
695 | }; | 695 | }; |
696 | 696 | ||
697 | /** | 697 | /** |
698 | * i2o_block_media_changed - Have we seen a media change? | 698 | * i2o_block_check_events - Have we seen a media change? |
699 | * @disk: gendisk which should be verified | 699 | * @disk: gendisk which should be verified |
700 | * @clearing: events being cleared | ||
700 | * | 701 | * |
701 | * Verifies if the media has changed. | 702 | * Verifies if the media has changed. |
702 | * | 703 | * |
703 | * Returns 1 if the media was changed or 0 otherwise. | 704 | * Returns 1 if the media was changed or 0 otherwise. |
704 | */ | 705 | */ |
705 | static int i2o_block_media_changed(struct gendisk *disk) | 706 | static unsigned int i2o_block_check_events(struct gendisk *disk, |
707 | unsigned int clearing) | ||
706 | { | 708 | { |
707 | struct i2o_block_device *p = disk->private_data; | 709 | struct i2o_block_device *p = disk->private_data; |
708 | 710 | ||
709 | if (p->media_change_flag) { | 711 | if (p->media_change_flag) { |
710 | p->media_change_flag = 0; | 712 | p->media_change_flag = 0; |
711 | return 1; | 713 | return DISK_EVENT_MEDIA_CHANGE; |
712 | } | 714 | } |
713 | return 0; | 715 | return 0; |
714 | } | 716 | } |
@@ -895,11 +897,7 @@ static void i2o_block_request_fn(struct request_queue *q) | |||
895 | { | 897 | { |
896 | struct request *req; | 898 | struct request *req; |
897 | 899 | ||
898 | while (!blk_queue_plugged(q)) { | 900 | while ((req = blk_peek_request(q)) != NULL) { |
899 | req = blk_peek_request(q); | ||
900 | if (!req) | ||
901 | break; | ||
902 | |||
903 | if (req->cmd_type == REQ_TYPE_FS) { | 901 | if (req->cmd_type == REQ_TYPE_FS) { |
904 | struct i2o_block_delayed_request *dreq; | 902 | struct i2o_block_delayed_request *dreq; |
905 | struct i2o_block_request *ireq = req->special; | 903 | struct i2o_block_request *ireq = req->special; |
@@ -950,7 +948,7 @@ static const struct block_device_operations i2o_block_fops = { | |||
950 | .ioctl = i2o_block_ioctl, | 948 | .ioctl = i2o_block_ioctl, |
951 | .compat_ioctl = i2o_block_ioctl, | 949 | .compat_ioctl = i2o_block_ioctl, |
952 | .getgeo = i2o_block_getgeo, | 950 | .getgeo = i2o_block_getgeo, |
953 | .media_changed = i2o_block_media_changed | 951 | .check_events = i2o_block_check_events, |
954 | }; | 952 | }; |
955 | 953 | ||
956 | /** | 954 | /** |
@@ -1002,6 +1000,7 @@ static struct i2o_block_device *i2o_block_device_alloc(void) | |||
1002 | gd->major = I2O_MAJOR; | 1000 | gd->major = I2O_MAJOR; |
1003 | gd->queue = queue; | 1001 | gd->queue = queue; |
1004 | gd->fops = &i2o_block_fops; | 1002 | gd->fops = &i2o_block_fops; |
1003 | gd->events = DISK_EVENT_MEDIA_CHANGE; | ||
1005 | gd->private_data = dev; | 1004 | gd->private_data = dev; |
1006 | 1005 | ||
1007 | dev->gd = gd; | 1006 | dev->gd = gd; |
diff --git a/drivers/mmc/card/queue.c b/drivers/mmc/card/queue.c index 4e42d030e097..2ae727568df9 100644 --- a/drivers/mmc/card/queue.c +++ b/drivers/mmc/card/queue.c | |||
@@ -55,8 +55,7 @@ static int mmc_queue_thread(void *d) | |||
55 | 55 | ||
56 | spin_lock_irq(q->queue_lock); | 56 | spin_lock_irq(q->queue_lock); |
57 | set_current_state(TASK_INTERRUPTIBLE); | 57 | set_current_state(TASK_INTERRUPTIBLE); |
58 | if (!blk_queue_plugged(q)) | 58 | req = blk_fetch_request(q); |
59 | req = blk_fetch_request(q); | ||
60 | mq->req = req; | 59 | mq->req = req; |
61 | spin_unlock_irq(q->queue_lock); | 60 | spin_unlock_irq(q->queue_lock); |
62 | 61 | ||
diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index 794bfd962266..4d2df2f76ea0 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c | |||
@@ -1917,7 +1917,7 @@ static void __dasd_process_request_queue(struct dasd_block *block) | |||
1917 | return; | 1917 | return; |
1918 | } | 1918 | } |
1919 | /* Now we try to fetch requests from the request queue */ | 1919 | /* Now we try to fetch requests from the request queue */ |
1920 | while (!blk_queue_plugged(queue) && (req = blk_peek_request(queue))) { | 1920 | while ((req = blk_peek_request(queue))) { |
1921 | if (basedev->features & DASD_FEATURE_READONLY && | 1921 | if (basedev->features & DASD_FEATURE_READONLY && |
1922 | rq_data_dir(req) == WRITE) { | 1922 | rq_data_dir(req) == WRITE) { |
1923 | DBF_DEV_EVENT(DBF_ERR, basedev, | 1923 | DBF_DEV_EVENT(DBF_ERR, basedev, |
diff --git a/drivers/s390/char/tape_block.c b/drivers/s390/char/tape_block.c index 55d2d0f4eabc..83cea9a55e2f 100644 --- a/drivers/s390/char/tape_block.c +++ b/drivers/s390/char/tape_block.c | |||
@@ -48,14 +48,14 @@ | |||
48 | static DEFINE_MUTEX(tape_block_mutex); | 48 | static DEFINE_MUTEX(tape_block_mutex); |
49 | static int tapeblock_open(struct block_device *, fmode_t); | 49 | static int tapeblock_open(struct block_device *, fmode_t); |
50 | static int tapeblock_release(struct gendisk *, fmode_t); | 50 | static int tapeblock_release(struct gendisk *, fmode_t); |
51 | static int tapeblock_medium_changed(struct gendisk *); | 51 | static unsigned int tapeblock_check_events(struct gendisk *, unsigned int); |
52 | static int tapeblock_revalidate_disk(struct gendisk *); | 52 | static int tapeblock_revalidate_disk(struct gendisk *); |
53 | 53 | ||
54 | static const struct block_device_operations tapeblock_fops = { | 54 | static const struct block_device_operations tapeblock_fops = { |
55 | .owner = THIS_MODULE, | 55 | .owner = THIS_MODULE, |
56 | .open = tapeblock_open, | 56 | .open = tapeblock_open, |
57 | .release = tapeblock_release, | 57 | .release = tapeblock_release, |
58 | .media_changed = tapeblock_medium_changed, | 58 | .check_events = tapeblock_check_events, |
59 | .revalidate_disk = tapeblock_revalidate_disk, | 59 | .revalidate_disk = tapeblock_revalidate_disk, |
60 | }; | 60 | }; |
61 | 61 | ||
@@ -161,7 +161,6 @@ tapeblock_requeue(struct work_struct *work) { | |||
161 | 161 | ||
162 | spin_lock_irq(&device->blk_data.request_queue_lock); | 162 | spin_lock_irq(&device->blk_data.request_queue_lock); |
163 | while ( | 163 | while ( |
164 | !blk_queue_plugged(queue) && | ||
165 | blk_peek_request(queue) && | 164 | blk_peek_request(queue) && |
166 | nr_queued < TAPEBLOCK_MIN_REQUEUE | 165 | nr_queued < TAPEBLOCK_MIN_REQUEUE |
167 | ) { | 166 | ) { |
@@ -237,6 +236,7 @@ tapeblock_setup_device(struct tape_device * device) | |||
237 | disk->major = tapeblock_major; | 236 | disk->major = tapeblock_major; |
238 | disk->first_minor = device->first_minor; | 237 | disk->first_minor = device->first_minor; |
239 | disk->fops = &tapeblock_fops; | 238 | disk->fops = &tapeblock_fops; |
239 | disk->events = DISK_EVENT_MEDIA_CHANGE; | ||
240 | disk->private_data = tape_get_device(device); | 240 | disk->private_data = tape_get_device(device); |
241 | disk->queue = blkdat->request_queue; | 241 | disk->queue = blkdat->request_queue; |
242 | set_capacity(disk, 0); | 242 | set_capacity(disk, 0); |
@@ -340,8 +340,8 @@ tapeblock_revalidate_disk(struct gendisk *disk) | |||
340 | return 0; | 340 | return 0; |
341 | } | 341 | } |
342 | 342 | ||
343 | static int | 343 | static unsigned int |
344 | tapeblock_medium_changed(struct gendisk *disk) | 344 | tapeblock_check_events(struct gendisk *disk, unsigned int clearing) |
345 | { | 345 | { |
346 | struct tape_device *device; | 346 | struct tape_device *device; |
347 | 347 | ||
@@ -349,7 +349,7 @@ tapeblock_medium_changed(struct gendisk *disk) | |||
349 | DBF_LH(6, "tapeblock_medium_changed(%p) = %d\n", | 349 | DBF_LH(6, "tapeblock_medium_changed(%p) = %d\n", |
350 | device, device->blk_data.medium_changed); | 350 | device, device->blk_data.medium_changed); |
351 | 351 | ||
352 | return device->blk_data.medium_changed; | 352 | return device->blk_data.medium_changed ? DISK_EVENT_MEDIA_CHANGE : 0; |
353 | } | 353 | } |
354 | 354 | ||
355 | /* | 355 | /* |
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 2d63c8ad1442..6d5c7ff43f5b 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c | |||
@@ -67,6 +67,13 @@ static struct scsi_host_sg_pool scsi_sg_pools[] = { | |||
67 | 67 | ||
68 | struct kmem_cache *scsi_sdb_cache; | 68 | struct kmem_cache *scsi_sdb_cache; |
69 | 69 | ||
70 | /* | ||
71 | * When to reinvoke queueing after a resource shortage. It's 3 msecs to | ||
72 | * not change behaviour from the previous unplug mechanism, experimentation | ||
73 | * may prove this needs changing. | ||
74 | */ | ||
75 | #define SCSI_QUEUE_DELAY 3 | ||
76 | |||
70 | static void scsi_run_queue(struct request_queue *q); | 77 | static void scsi_run_queue(struct request_queue *q); |
71 | 78 | ||
72 | /* | 79 | /* |
@@ -149,14 +156,7 @@ static int __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy) | |||
149 | /* | 156 | /* |
150 | * Requeue this command. It will go before all other commands | 157 | * Requeue this command. It will go before all other commands |
151 | * that are already in the queue. | 158 | * that are already in the queue. |
152 | * | 159 | */ |
153 | * NOTE: there is magic here about the way the queue is plugged if | ||
154 | * we have no outstanding commands. | ||
155 | * | ||
156 | * Although we *don't* plug the queue, we call the request | ||
157 | * function. The SCSI request function detects the blocked condition | ||
158 | * and plugs the queue appropriately. | ||
159 | */ | ||
160 | spin_lock_irqsave(q->queue_lock, flags); | 160 | spin_lock_irqsave(q->queue_lock, flags); |
161 | blk_requeue_request(q, cmd->request); | 161 | blk_requeue_request(q, cmd->request); |
162 | spin_unlock_irqrestore(q->queue_lock, flags); | 162 | spin_unlock_irqrestore(q->queue_lock, flags); |
@@ -1226,11 +1226,11 @@ int scsi_prep_return(struct request_queue *q, struct request *req, int ret) | |||
1226 | case BLKPREP_DEFER: | 1226 | case BLKPREP_DEFER: |
1227 | /* | 1227 | /* |
1228 | * If we defer, the blk_peek_request() returns NULL, but the | 1228 | * If we defer, the blk_peek_request() returns NULL, but the |
1229 | * queue must be restarted, so we plug here if no returning | 1229 | * queue must be restarted, so we schedule a callback to happen |
1230 | * command will automatically do that. | 1230 | * shortly. |
1231 | */ | 1231 | */ |
1232 | if (sdev->device_busy == 0) | 1232 | if (sdev->device_busy == 0) |
1233 | blk_plug_device(q); | 1233 | blk_delay_queue(q, SCSI_QUEUE_DELAY); |
1234 | break; | 1234 | break; |
1235 | default: | 1235 | default: |
1236 | req->cmd_flags |= REQ_DONTPREP; | 1236 | req->cmd_flags |= REQ_DONTPREP; |
@@ -1269,7 +1269,7 @@ static inline int scsi_dev_queue_ready(struct request_queue *q, | |||
1269 | sdev_printk(KERN_INFO, sdev, | 1269 | sdev_printk(KERN_INFO, sdev, |
1270 | "unblocking device at zero depth\n")); | 1270 | "unblocking device at zero depth\n")); |
1271 | } else { | 1271 | } else { |
1272 | blk_plug_device(q); | 1272 | blk_delay_queue(q, SCSI_QUEUE_DELAY); |
1273 | return 0; | 1273 | return 0; |
1274 | } | 1274 | } |
1275 | } | 1275 | } |
@@ -1499,7 +1499,7 @@ static void scsi_request_fn(struct request_queue *q) | |||
1499 | * the host is no longer able to accept any more requests. | 1499 | * the host is no longer able to accept any more requests. |
1500 | */ | 1500 | */ |
1501 | shost = sdev->host; | 1501 | shost = sdev->host; |
1502 | while (!blk_queue_plugged(q)) { | 1502 | for (;;) { |
1503 | int rtn; | 1503 | int rtn; |
1504 | /* | 1504 | /* |
1505 | * get next queueable request. We do this early to make sure | 1505 | * get next queueable request. We do this early to make sure |
@@ -1578,15 +1578,8 @@ static void scsi_request_fn(struct request_queue *q) | |||
1578 | */ | 1578 | */ |
1579 | rtn = scsi_dispatch_cmd(cmd); | 1579 | rtn = scsi_dispatch_cmd(cmd); |
1580 | spin_lock_irq(q->queue_lock); | 1580 | spin_lock_irq(q->queue_lock); |
1581 | if(rtn) { | 1581 | if (rtn) |
1582 | /* we're refusing the command; because of | 1582 | goto out_delay; |
1583 | * the way locks get dropped, we need to | ||
1584 | * check here if plugging is required */ | ||
1585 | if(sdev->device_busy == 0) | ||
1586 | blk_plug_device(q); | ||
1587 | |||
1588 | break; | ||
1589 | } | ||
1590 | } | 1583 | } |
1591 | 1584 | ||
1592 | goto out; | 1585 | goto out; |
@@ -1605,9 +1598,10 @@ static void scsi_request_fn(struct request_queue *q) | |||
1605 | spin_lock_irq(q->queue_lock); | 1598 | spin_lock_irq(q->queue_lock); |
1606 | blk_requeue_request(q, req); | 1599 | blk_requeue_request(q, req); |
1607 | sdev->device_busy--; | 1600 | sdev->device_busy--; |
1608 | if(sdev->device_busy == 0) | 1601 | out_delay: |
1609 | blk_plug_device(q); | 1602 | if (sdev->device_busy == 0) |
1610 | out: | 1603 | blk_delay_queue(q, SCSI_QUEUE_DELAY); |
1604 | out: | ||
1611 | /* must be careful here...if we trigger the ->remove() function | 1605 | /* must be careful here...if we trigger the ->remove() function |
1612 | * we cannot be holding the q lock */ | 1606 | * we cannot be holding the q lock */ |
1613 | spin_unlock_irq(q->queue_lock); | 1607 | spin_unlock_irq(q->queue_lock); |
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c index 5c3ccfc6b622..2941d2d92c94 100644 --- a/drivers/scsi/scsi_transport_fc.c +++ b/drivers/scsi/scsi_transport_fc.c | |||
@@ -3913,7 +3913,7 @@ fc_bsg_request_handler(struct request_queue *q, struct Scsi_Host *shost, | |||
3913 | if (!get_device(dev)) | 3913 | if (!get_device(dev)) |
3914 | return; | 3914 | return; |
3915 | 3915 | ||
3916 | while (!blk_queue_plugged(q)) { | 3916 | while (1) { |
3917 | if (rport && (rport->port_state == FC_PORTSTATE_BLOCKED) && | 3917 | if (rport && (rport->port_state == FC_PORTSTATE_BLOCKED) && |
3918 | !(rport->flags & FC_RPORT_FAST_FAIL_TIMEDOUT)) | 3918 | !(rport->flags & FC_RPORT_FAST_FAIL_TIMEDOUT)) |
3919 | break; | 3919 | break; |
diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c index 927e99cb7225..c6fcf76cade5 100644 --- a/drivers/scsi/scsi_transport_sas.c +++ b/drivers/scsi/scsi_transport_sas.c | |||
@@ -173,11 +173,7 @@ static void sas_smp_request(struct request_queue *q, struct Scsi_Host *shost, | |||
173 | int ret; | 173 | int ret; |
174 | int (*handler)(struct Scsi_Host *, struct sas_rphy *, struct request *); | 174 | int (*handler)(struct Scsi_Host *, struct sas_rphy *, struct request *); |
175 | 175 | ||
176 | while (!blk_queue_plugged(q)) { | 176 | while ((req = blk_fetch_request(q)) != NULL) { |
177 | req = blk_fetch_request(q); | ||
178 | if (!req) | ||
179 | break; | ||
180 | |||
181 | spin_unlock_irq(q->queue_lock); | 177 | spin_unlock_irq(q->queue_lock); |
182 | 178 | ||
183 | handler = to_sas_internal(shost->transportt)->f->smp_handler; | 179 | handler = to_sas_internal(shost->transportt)->f->smp_handler; |
diff --git a/drivers/staging/hv/blkvsc_drv.c b/drivers/staging/hv/blkvsc_drv.c index 6e02f1b0c46f..af789937be4e 100644 --- a/drivers/staging/hv/blkvsc_drv.c +++ b/drivers/staging/hv/blkvsc_drv.c | |||
@@ -124,7 +124,8 @@ static void blkvsc_shutdown(struct device *device); | |||
124 | 124 | ||
125 | static int blkvsc_open(struct block_device *bdev, fmode_t mode); | 125 | static int blkvsc_open(struct block_device *bdev, fmode_t mode); |
126 | static int blkvsc_release(struct gendisk *disk, fmode_t mode); | 126 | static int blkvsc_release(struct gendisk *disk, fmode_t mode); |
127 | static int blkvsc_media_changed(struct gendisk *gd); | 127 | static unsigned int blkvsc_check_events(struct gendisk *gd, |
128 | unsigned int clearing); | ||
128 | static int blkvsc_revalidate_disk(struct gendisk *gd); | 129 | static int blkvsc_revalidate_disk(struct gendisk *gd); |
129 | static int blkvsc_getgeo(struct block_device *bd, struct hd_geometry *hg); | 130 | static int blkvsc_getgeo(struct block_device *bd, struct hd_geometry *hg); |
130 | static int blkvsc_ioctl(struct block_device *bd, fmode_t mode, | 131 | static int blkvsc_ioctl(struct block_device *bd, fmode_t mode, |
@@ -155,7 +156,7 @@ static const struct block_device_operations block_ops = { | |||
155 | .owner = THIS_MODULE, | 156 | .owner = THIS_MODULE, |
156 | .open = blkvsc_open, | 157 | .open = blkvsc_open, |
157 | .release = blkvsc_release, | 158 | .release = blkvsc_release, |
158 | .media_changed = blkvsc_media_changed, | 159 | .check_events = blkvsc_check_events, |
159 | .revalidate_disk = blkvsc_revalidate_disk, | 160 | .revalidate_disk = blkvsc_revalidate_disk, |
160 | .getgeo = blkvsc_getgeo, | 161 | .getgeo = blkvsc_getgeo, |
161 | .ioctl = blkvsc_ioctl, | 162 | .ioctl = blkvsc_ioctl, |
@@ -357,6 +358,7 @@ static int blkvsc_probe(struct device *device) | |||
357 | else | 358 | else |
358 | blkdev->gd->first_minor = 0; | 359 | blkdev->gd->first_minor = 0; |
359 | blkdev->gd->fops = &block_ops; | 360 | blkdev->gd->fops = &block_ops; |
361 | blkdev->gd->events = DISK_EVENT_MEDIA_CHANGE; | ||
360 | blkdev->gd->private_data = blkdev; | 362 | blkdev->gd->private_data = blkdev; |
361 | blkdev->gd->driverfs_dev = &(blkdev->device_ctx->device); | 363 | blkdev->gd->driverfs_dev = &(blkdev->device_ctx->device); |
362 | sprintf(blkdev->gd->disk_name, "hd%c", 'a' + devnum); | 364 | sprintf(blkdev->gd->disk_name, "hd%c", 'a' + devnum); |
@@ -1337,10 +1339,11 @@ static int blkvsc_release(struct gendisk *disk, fmode_t mode) | |||
1337 | return 0; | 1339 | return 0; |
1338 | } | 1340 | } |
1339 | 1341 | ||
1340 | static int blkvsc_media_changed(struct gendisk *gd) | 1342 | static unsigned int blkvsc_check_events(struct gendisk *gd, |
1343 | unsigned int clearing) | ||
1341 | { | 1344 | { |
1342 | DPRINT_DBG(BLKVSC_DRV, "- enter\n"); | 1345 | DPRINT_DBG(BLKVSC_DRV, "- enter\n"); |
1343 | return 1; | 1346 | return DISK_EVENT_MEDIA_CHANGE; |
1344 | } | 1347 | } |
1345 | 1348 | ||
1346 | static int blkvsc_revalidate_disk(struct gendisk *gd) | 1349 | static int blkvsc_revalidate_disk(struct gendisk *gd) |
diff --git a/drivers/staging/westbridge/astoria/block/cyasblkdev_block.c b/drivers/staging/westbridge/astoria/block/cyasblkdev_block.c index e1851f00be56..842cd9214a5e 100644 --- a/drivers/staging/westbridge/astoria/block/cyasblkdev_block.c +++ b/drivers/staging/westbridge/astoria/block/cyasblkdev_block.c | |||
@@ -381,10 +381,10 @@ static int cyasblkdev_blk_ioctl( | |||
381 | return -ENOTTY; | 381 | return -ENOTTY; |
382 | } | 382 | } |
383 | 383 | ||
384 | /* Media_changed block_device opp | 384 | /* check_events block_device opp |
385 | * this one is called by kernel to confirm if the media really changed | 385 | * this one is called by kernel to confirm if the media really changed |
386 | * as we indicated by issuing check_disk_change() call */ | 386 | * as we indicated by issuing check_disk_change() call */ |
387 | int cyasblkdev_media_changed(struct gendisk *gd) | 387 | unsigned int cyasblkdev_check_events(struct gendisk *gd, unsigned int clearing) |
388 | { | 388 | { |
389 | struct cyasblkdev_blk_data *bd; | 389 | struct cyasblkdev_blk_data *bd; |
390 | 390 | ||
@@ -402,7 +402,7 @@ int cyasblkdev_media_changed(struct gendisk *gd) | |||
402 | #endif | 402 | #endif |
403 | } | 403 | } |
404 | 404 | ||
405 | /* return media change state "1" yes, 0 no */ | 405 | /* return media change state - DISK_EVENT_MEDIA_CHANGE yes, 0 no */ |
406 | return 0; | 406 | return 0; |
407 | } | 407 | } |
408 | 408 | ||
@@ -432,7 +432,7 @@ static struct block_device_operations cyasblkdev_bdops = { | |||
432 | .ioctl = cyasblkdev_blk_ioctl, | 432 | .ioctl = cyasblkdev_blk_ioctl, |
433 | /* .getgeo = cyasblkdev_blk_getgeo, */ | 433 | /* .getgeo = cyasblkdev_blk_getgeo, */ |
434 | /* added to support media removal( real and simulated) media */ | 434 | /* added to support media removal( real and simulated) media */ |
435 | .media_changed = cyasblkdev_media_changed, | 435 | .check_events = cyasblkdev_check_events, |
436 | /* added to support media removal( real and simulated) media */ | 436 | /* added to support media removal( real and simulated) media */ |
437 | .revalidate_disk = cyasblkdev_revalidate_disk, | 437 | .revalidate_disk = cyasblkdev_revalidate_disk, |
438 | .owner = THIS_MODULE, | 438 | .owner = THIS_MODULE, |
@@ -1090,6 +1090,7 @@ static int cyasblkdev_add_disks(int bus_num, | |||
1090 | bd->user_disk_0->first_minor = devidx << CYASBLKDEV_SHIFT; | 1090 | bd->user_disk_0->first_minor = devidx << CYASBLKDEV_SHIFT; |
1091 | bd->user_disk_0->minors = 8; | 1091 | bd->user_disk_0->minors = 8; |
1092 | bd->user_disk_0->fops = &cyasblkdev_bdops; | 1092 | bd->user_disk_0->fops = &cyasblkdev_bdops; |
1093 | bd->user_disk_0->events = DISK_EVENT_MEDIA_CHANGE; | ||
1093 | bd->user_disk_0->private_data = bd; | 1094 | bd->user_disk_0->private_data = bd; |
1094 | bd->user_disk_0->queue = bd->queue.queue; | 1095 | bd->user_disk_0->queue = bd->queue.queue; |
1095 | bd->dbgprn_flags = DBGPRN_RD_RQ; | 1096 | bd->dbgprn_flags = DBGPRN_RD_RQ; |
@@ -1190,6 +1191,7 @@ static int cyasblkdev_add_disks(int bus_num, | |||
1190 | bd->user_disk_1->first_minor = (devidx + 1) << CYASBLKDEV_SHIFT; | 1191 | bd->user_disk_1->first_minor = (devidx + 1) << CYASBLKDEV_SHIFT; |
1191 | bd->user_disk_1->minors = 8; | 1192 | bd->user_disk_1->minors = 8; |
1192 | bd->user_disk_1->fops = &cyasblkdev_bdops; | 1193 | bd->user_disk_1->fops = &cyasblkdev_bdops; |
1194 | bd->user_disk_0->events = DISK_EVENT_MEDIA_CHANGE; | ||
1193 | bd->user_disk_1->private_data = bd; | 1195 | bd->user_disk_1->private_data = bd; |
1194 | bd->user_disk_1->queue = bd->queue.queue; | 1196 | bd->user_disk_1->queue = bd->queue.queue; |
1195 | bd->dbgprn_flags = DBGPRN_RD_RQ; | 1197 | bd->dbgprn_flags = DBGPRN_RD_RQ; |
@@ -1278,6 +1280,7 @@ static int cyasblkdev_add_disks(int bus_num, | |||
1278 | (devidx + 2) << CYASBLKDEV_SHIFT; | 1280 | (devidx + 2) << CYASBLKDEV_SHIFT; |
1279 | bd->system_disk->minors = 8; | 1281 | bd->system_disk->minors = 8; |
1280 | bd->system_disk->fops = &cyasblkdev_bdops; | 1282 | bd->system_disk->fops = &cyasblkdev_bdops; |
1283 | bd->system_disk->events = DISK_EVENT_MEDIA_CHANGE; | ||
1281 | bd->system_disk->private_data = bd; | 1284 | bd->system_disk->private_data = bd; |
1282 | bd->system_disk->queue = bd->queue.queue; | 1285 | bd->system_disk->queue = bd->queue.queue; |
1283 | /* don't search for vfat | 1286 | /* don't search for vfat |
diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c index 3df570db0e4f..eb0afec046e1 100644 --- a/drivers/target/target_core_iblock.c +++ b/drivers/target/target_core_iblock.c | |||
@@ -391,9 +391,8 @@ static int iblock_do_task(struct se_task *task) | |||
391 | { | 391 | { |
392 | struct se_device *dev = task->task_se_cmd->se_dev; | 392 | struct se_device *dev = task->task_se_cmd->se_dev; |
393 | struct iblock_req *req = IBLOCK_REQ(task); | 393 | struct iblock_req *req = IBLOCK_REQ(task); |
394 | struct iblock_dev *ibd = (struct iblock_dev *)req->ib_dev; | ||
395 | struct request_queue *q = bdev_get_queue(ibd->ibd_bd); | ||
396 | struct bio *bio = req->ib_bio, *nbio = NULL; | 394 | struct bio *bio = req->ib_bio, *nbio = NULL; |
395 | struct blk_plug plug; | ||
397 | int rw; | 396 | int rw; |
398 | 397 | ||
399 | if (task->task_data_direction == DMA_TO_DEVICE) { | 398 | if (task->task_data_direction == DMA_TO_DEVICE) { |
@@ -411,6 +410,7 @@ static int iblock_do_task(struct se_task *task) | |||
411 | rw = READ; | 410 | rw = READ; |
412 | } | 411 | } |
413 | 412 | ||
413 | blk_start_plug(&plug); | ||
414 | while (bio) { | 414 | while (bio) { |
415 | nbio = bio->bi_next; | 415 | nbio = bio->bi_next; |
416 | bio->bi_next = NULL; | 416 | bio->bi_next = NULL; |
@@ -420,9 +420,8 @@ static int iblock_do_task(struct se_task *task) | |||
420 | submit_bio(rw, bio); | 420 | submit_bio(rw, bio); |
421 | bio = nbio; | 421 | bio = nbio; |
422 | } | 422 | } |
423 | blk_finish_plug(&plug); | ||
423 | 424 | ||
424 | if (q->unplug_fn) | ||
425 | q->unplug_fn(q); | ||
426 | return PYX_TRANSPORT_SENT_TO_TRANSPORT; | 425 | return PYX_TRANSPORT_SENT_TO_TRANSPORT; |
427 | } | 426 | } |
428 | 427 | ||
diff --git a/fs/adfs/inode.c b/fs/adfs/inode.c index 92444e94f842..d5250c5aae21 100644 --- a/fs/adfs/inode.c +++ b/fs/adfs/inode.c | |||
@@ -72,7 +72,6 @@ static sector_t _adfs_bmap(struct address_space *mapping, sector_t block) | |||
72 | static const struct address_space_operations adfs_aops = { | 72 | static const struct address_space_operations adfs_aops = { |
73 | .readpage = adfs_readpage, | 73 | .readpage = adfs_readpage, |
74 | .writepage = adfs_writepage, | 74 | .writepage = adfs_writepage, |
75 | .sync_page = block_sync_page, | ||
76 | .write_begin = adfs_write_begin, | 75 | .write_begin = adfs_write_begin, |
77 | .write_end = generic_write_end, | 76 | .write_end = generic_write_end, |
78 | .bmap = _adfs_bmap | 77 | .bmap = _adfs_bmap |
diff --git a/fs/affs/file.c b/fs/affs/file.c index 0a90dcd46de2..acf321b70fcd 100644 --- a/fs/affs/file.c +++ b/fs/affs/file.c | |||
@@ -429,7 +429,6 @@ static sector_t _affs_bmap(struct address_space *mapping, sector_t block) | |||
429 | const struct address_space_operations affs_aops = { | 429 | const struct address_space_operations affs_aops = { |
430 | .readpage = affs_readpage, | 430 | .readpage = affs_readpage, |
431 | .writepage = affs_writepage, | 431 | .writepage = affs_writepage, |
432 | .sync_page = block_sync_page, | ||
433 | .write_begin = affs_write_begin, | 432 | .write_begin = affs_write_begin, |
434 | .write_end = generic_write_end, | 433 | .write_end = generic_write_end, |
435 | .bmap = _affs_bmap | 434 | .bmap = _affs_bmap |
@@ -786,7 +785,6 @@ out: | |||
786 | const struct address_space_operations affs_aops_ofs = { | 785 | const struct address_space_operations affs_aops_ofs = { |
787 | .readpage = affs_readpage_ofs, | 786 | .readpage = affs_readpage_ofs, |
788 | //.writepage = affs_writepage_ofs, | 787 | //.writepage = affs_writepage_ofs, |
789 | //.sync_page = affs_sync_page_ofs, | ||
790 | .write_begin = affs_write_begin_ofs, | 788 | .write_begin = affs_write_begin_ofs, |
791 | .write_end = affs_write_end_ofs | 789 | .write_end = affs_write_end_ofs |
792 | }; | 790 | }; |
@@ -34,8 +34,6 @@ | |||
34 | #include <linux/security.h> | 34 | #include <linux/security.h> |
35 | #include <linux/eventfd.h> | 35 | #include <linux/eventfd.h> |
36 | #include <linux/blkdev.h> | 36 | #include <linux/blkdev.h> |
37 | #include <linux/mempool.h> | ||
38 | #include <linux/hash.h> | ||
39 | #include <linux/compat.h> | 37 | #include <linux/compat.h> |
40 | 38 | ||
41 | #include <asm/kmap_types.h> | 39 | #include <asm/kmap_types.h> |
@@ -65,14 +63,6 @@ static DECLARE_WORK(fput_work, aio_fput_routine); | |||
65 | static DEFINE_SPINLOCK(fput_lock); | 63 | static DEFINE_SPINLOCK(fput_lock); |
66 | static LIST_HEAD(fput_head); | 64 | static LIST_HEAD(fput_head); |
67 | 65 | ||
68 | #define AIO_BATCH_HASH_BITS 3 /* allocated on-stack, so don't go crazy */ | ||
69 | #define AIO_BATCH_HASH_SIZE (1 << AIO_BATCH_HASH_BITS) | ||
70 | struct aio_batch_entry { | ||
71 | struct hlist_node list; | ||
72 | struct address_space *mapping; | ||
73 | }; | ||
74 | mempool_t *abe_pool; | ||
75 | |||
76 | static void aio_kick_handler(struct work_struct *); | 66 | static void aio_kick_handler(struct work_struct *); |
77 | static void aio_queue_work(struct kioctx *); | 67 | static void aio_queue_work(struct kioctx *); |
78 | 68 | ||
@@ -86,8 +76,7 @@ static int __init aio_setup(void) | |||
86 | kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); | 76 | kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); |
87 | 77 | ||
88 | aio_wq = alloc_workqueue("aio", 0, 1); /* used to limit concurrency */ | 78 | aio_wq = alloc_workqueue("aio", 0, 1); /* used to limit concurrency */ |
89 | abe_pool = mempool_create_kmalloc_pool(1, sizeof(struct aio_batch_entry)); | 79 | BUG_ON(!aio_wq); |
90 | BUG_ON(!aio_wq || !abe_pool); | ||
91 | 80 | ||
92 | pr_debug("aio_setup: sizeof(struct page) = %d\n", (int)sizeof(struct page)); | 81 | pr_debug("aio_setup: sizeof(struct page) = %d\n", (int)sizeof(struct page)); |
93 | 82 | ||
@@ -1525,57 +1514,8 @@ static ssize_t aio_setup_iocb(struct kiocb *kiocb, bool compat) | |||
1525 | return 0; | 1514 | return 0; |
1526 | } | 1515 | } |
1527 | 1516 | ||
1528 | static void aio_batch_add(struct address_space *mapping, | ||
1529 | struct hlist_head *batch_hash) | ||
1530 | { | ||
1531 | struct aio_batch_entry *abe; | ||
1532 | struct hlist_node *pos; | ||
1533 | unsigned bucket; | ||
1534 | |||
1535 | bucket = hash_ptr(mapping, AIO_BATCH_HASH_BITS); | ||
1536 | hlist_for_each_entry(abe, pos, &batch_hash[bucket], list) { | ||
1537 | if (abe->mapping == mapping) | ||
1538 | return; | ||
1539 | } | ||
1540 | |||
1541 | abe = mempool_alloc(abe_pool, GFP_KERNEL); | ||
1542 | |||
1543 | /* | ||
1544 | * we should be using igrab here, but | ||
1545 | * we don't want to hammer on the global | ||
1546 | * inode spinlock just to take an extra | ||
1547 | * reference on a file that we must already | ||
1548 | * have a reference to. | ||
1549 | * | ||
1550 | * When we're called, we always have a reference | ||
1551 | * on the file, so we must always have a reference | ||
1552 | * on the inode, so ihold() is safe here. | ||
1553 | */ | ||
1554 | ihold(mapping->host); | ||
1555 | abe->mapping = mapping; | ||
1556 | hlist_add_head(&abe->list, &batch_hash[bucket]); | ||
1557 | return; | ||
1558 | } | ||
1559 | |||
1560 | static void aio_batch_free(struct hlist_head *batch_hash) | ||
1561 | { | ||
1562 | struct aio_batch_entry *abe; | ||
1563 | struct hlist_node *pos, *n; | ||
1564 | int i; | ||
1565 | |||
1566 | for (i = 0; i < AIO_BATCH_HASH_SIZE; i++) { | ||
1567 | hlist_for_each_entry_safe(abe, pos, n, &batch_hash[i], list) { | ||
1568 | blk_run_address_space(abe->mapping); | ||
1569 | iput(abe->mapping->host); | ||
1570 | hlist_del(&abe->list); | ||
1571 | mempool_free(abe, abe_pool); | ||
1572 | } | ||
1573 | } | ||
1574 | } | ||
1575 | |||
1576 | static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, | 1517 | static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, |
1577 | struct iocb *iocb, struct hlist_head *batch_hash, | 1518 | struct iocb *iocb, bool compat) |
1578 | bool compat) | ||
1579 | { | 1519 | { |
1580 | struct kiocb *req; | 1520 | struct kiocb *req; |
1581 | struct file *file; | 1521 | struct file *file; |
@@ -1666,11 +1606,6 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, | |||
1666 | ; | 1606 | ; |
1667 | } | 1607 | } |
1668 | spin_unlock_irq(&ctx->ctx_lock); | 1608 | spin_unlock_irq(&ctx->ctx_lock); |
1669 | if (req->ki_opcode == IOCB_CMD_PREAD || | ||
1670 | req->ki_opcode == IOCB_CMD_PREADV || | ||
1671 | req->ki_opcode == IOCB_CMD_PWRITE || | ||
1672 | req->ki_opcode == IOCB_CMD_PWRITEV) | ||
1673 | aio_batch_add(file->f_mapping, batch_hash); | ||
1674 | 1609 | ||
1675 | aio_put_req(req); /* drop extra ref to req */ | 1610 | aio_put_req(req); /* drop extra ref to req */ |
1676 | return 0; | 1611 | return 0; |
@@ -1687,7 +1622,7 @@ long do_io_submit(aio_context_t ctx_id, long nr, | |||
1687 | struct kioctx *ctx; | 1622 | struct kioctx *ctx; |
1688 | long ret = 0; | 1623 | long ret = 0; |
1689 | int i; | 1624 | int i; |
1690 | struct hlist_head batch_hash[AIO_BATCH_HASH_SIZE] = { { 0, }, }; | 1625 | struct blk_plug plug; |
1691 | 1626 | ||
1692 | if (unlikely(nr < 0)) | 1627 | if (unlikely(nr < 0)) |
1693 | return -EINVAL; | 1628 | return -EINVAL; |
@@ -1704,6 +1639,8 @@ long do_io_submit(aio_context_t ctx_id, long nr, | |||
1704 | return -EINVAL; | 1639 | return -EINVAL; |
1705 | } | 1640 | } |
1706 | 1641 | ||
1642 | blk_start_plug(&plug); | ||
1643 | |||
1707 | /* | 1644 | /* |
1708 | * AKPM: should this return a partial result if some of the IOs were | 1645 | * AKPM: should this return a partial result if some of the IOs were |
1709 | * successfully submitted? | 1646 | * successfully submitted? |
@@ -1722,11 +1659,11 @@ long do_io_submit(aio_context_t ctx_id, long nr, | |||
1722 | break; | 1659 | break; |
1723 | } | 1660 | } |
1724 | 1661 | ||
1725 | ret = io_submit_one(ctx, user_iocb, &tmp, batch_hash, compat); | 1662 | ret = io_submit_one(ctx, user_iocb, &tmp, compat); |
1726 | if (ret) | 1663 | if (ret) |
1727 | break; | 1664 | break; |
1728 | } | 1665 | } |
1729 | aio_batch_free(batch_hash); | 1666 | blk_finish_plug(&plug); |
1730 | 1667 | ||
1731 | put_ioctx(ctx); | 1668 | put_ioctx(ctx); |
1732 | return i ? i : ret; | 1669 | return i ? i : ret; |
diff --git a/fs/befs/linuxvfs.c b/fs/befs/linuxvfs.c index b1d0c794747b..06457ed8f3e7 100644 --- a/fs/befs/linuxvfs.c +++ b/fs/befs/linuxvfs.c | |||
@@ -75,7 +75,6 @@ static const struct inode_operations befs_dir_inode_operations = { | |||
75 | 75 | ||
76 | static const struct address_space_operations befs_aops = { | 76 | static const struct address_space_operations befs_aops = { |
77 | .readpage = befs_readpage, | 77 | .readpage = befs_readpage, |
78 | .sync_page = block_sync_page, | ||
79 | .bmap = befs_bmap, | 78 | .bmap = befs_bmap, |
80 | }; | 79 | }; |
81 | 80 | ||
diff --git a/fs/bfs/file.c b/fs/bfs/file.c index eb67edd0f8ea..f20e8a71062f 100644 --- a/fs/bfs/file.c +++ b/fs/bfs/file.c | |||
@@ -186,7 +186,6 @@ static sector_t bfs_bmap(struct address_space *mapping, sector_t block) | |||
186 | const struct address_space_operations bfs_aops = { | 186 | const struct address_space_operations bfs_aops = { |
187 | .readpage = bfs_readpage, | 187 | .readpage = bfs_readpage, |
188 | .writepage = bfs_writepage, | 188 | .writepage = bfs_writepage, |
189 | .sync_page = block_sync_page, | ||
190 | .write_begin = bfs_write_begin, | 189 | .write_begin = bfs_write_begin, |
191 | .write_end = generic_write_end, | 190 | .write_end = generic_write_end, |
192 | .bmap = bfs_bmap, | 191 | .bmap = bfs_bmap, |
diff --git a/fs/bio-integrity.c b/fs/bio-integrity.c index e49cce234c65..9c5e6b2cd11a 100644 --- a/fs/bio-integrity.c +++ b/fs/bio-integrity.c | |||
@@ -761,6 +761,9 @@ int bioset_integrity_create(struct bio_set *bs, int pool_size) | |||
761 | { | 761 | { |
762 | unsigned int max_slab = vecs_to_idx(BIO_MAX_PAGES); | 762 | unsigned int max_slab = vecs_to_idx(BIO_MAX_PAGES); |
763 | 763 | ||
764 | if (bs->bio_integrity_pool) | ||
765 | return 0; | ||
766 | |||
764 | bs->bio_integrity_pool = | 767 | bs->bio_integrity_pool = |
765 | mempool_create_slab_pool(pool_size, bip_slab[max_slab].slab); | 768 | mempool_create_slab_pool(pool_size, bip_slab[max_slab].slab); |
766 | 769 | ||
@@ -43,7 +43,7 @@ static mempool_t *bio_split_pool __read_mostly; | |||
43 | * unsigned short | 43 | * unsigned short |
44 | */ | 44 | */ |
45 | #define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) } | 45 | #define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) } |
46 | struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = { | 46 | static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = { |
47 | BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES), | 47 | BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES), |
48 | }; | 48 | }; |
49 | #undef BV | 49 | #undef BV |
@@ -1636,9 +1636,6 @@ struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad) | |||
1636 | if (!bs->bio_pool) | 1636 | if (!bs->bio_pool) |
1637 | goto bad; | 1637 | goto bad; |
1638 | 1638 | ||
1639 | if (bioset_integrity_create(bs, pool_size)) | ||
1640 | goto bad; | ||
1641 | |||
1642 | if (!biovec_create_pools(bs, pool_size)) | 1639 | if (!biovec_create_pools(bs, pool_size)) |
1643 | return bs; | 1640 | return bs; |
1644 | 1641 | ||
@@ -1656,12 +1653,10 @@ static void __init biovec_init_slabs(void) | |||
1656 | int size; | 1653 | int size; |
1657 | struct biovec_slab *bvs = bvec_slabs + i; | 1654 | struct biovec_slab *bvs = bvec_slabs + i; |
1658 | 1655 | ||
1659 | #ifndef CONFIG_BLK_DEV_INTEGRITY | ||
1660 | if (bvs->nr_vecs <= BIO_INLINE_VECS) { | 1656 | if (bvs->nr_vecs <= BIO_INLINE_VECS) { |
1661 | bvs->slab = NULL; | 1657 | bvs->slab = NULL; |
1662 | continue; | 1658 | continue; |
1663 | } | 1659 | } |
1664 | #endif | ||
1665 | 1660 | ||
1666 | size = bvs->nr_vecs * sizeof(struct bio_vec); | 1661 | size = bvs->nr_vecs * sizeof(struct bio_vec); |
1667 | bvs->slab = kmem_cache_create(bvs->name, size, 0, | 1662 | bvs->slab = kmem_cache_create(bvs->name, size, 0, |
@@ -1684,6 +1679,9 @@ static int __init init_bio(void) | |||
1684 | if (!fs_bio_set) | 1679 | if (!fs_bio_set) |
1685 | panic("bio: can't allocate bios\n"); | 1680 | panic("bio: can't allocate bios\n"); |
1686 | 1681 | ||
1682 | if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE)) | ||
1683 | panic("bio: can't create integrity pool\n"); | ||
1684 | |||
1687 | bio_split_pool = mempool_create_kmalloc_pool(BIO_SPLIT_ENTRIES, | 1685 | bio_split_pool = mempool_create_kmalloc_pool(BIO_SPLIT_ENTRIES, |
1688 | sizeof(struct bio_pair)); | 1686 | sizeof(struct bio_pair)); |
1689 | if (!bio_split_pool) | 1687 | if (!bio_split_pool) |
diff --git a/fs/block_dev.c b/fs/block_dev.c index 889287019599..7d02afb2b7f4 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c | |||
@@ -1087,6 +1087,7 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) | |||
1087 | if (!disk) | 1087 | if (!disk) |
1088 | goto out; | 1088 | goto out; |
1089 | 1089 | ||
1090 | disk_block_events(disk); | ||
1090 | mutex_lock_nested(&bdev->bd_mutex, for_part); | 1091 | mutex_lock_nested(&bdev->bd_mutex, for_part); |
1091 | if (!bdev->bd_openers) { | 1092 | if (!bdev->bd_openers) { |
1092 | bdev->bd_disk = disk; | 1093 | bdev->bd_disk = disk; |
@@ -1108,10 +1109,11 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) | |||
1108 | */ | 1109 | */ |
1109 | disk_put_part(bdev->bd_part); | 1110 | disk_put_part(bdev->bd_part); |
1110 | bdev->bd_part = NULL; | 1111 | bdev->bd_part = NULL; |
1111 | module_put(disk->fops->owner); | ||
1112 | put_disk(disk); | ||
1113 | bdev->bd_disk = NULL; | 1112 | bdev->bd_disk = NULL; |
1114 | mutex_unlock(&bdev->bd_mutex); | 1113 | mutex_unlock(&bdev->bd_mutex); |
1114 | disk_unblock_events(disk); | ||
1115 | module_put(disk->fops->owner); | ||
1116 | put_disk(disk); | ||
1115 | goto restart; | 1117 | goto restart; |
1116 | } | 1118 | } |
1117 | if (ret) | 1119 | if (ret) |
@@ -1148,9 +1150,6 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) | |||
1148 | bd_set_size(bdev, (loff_t)bdev->bd_part->nr_sects << 9); | 1150 | bd_set_size(bdev, (loff_t)bdev->bd_part->nr_sects << 9); |
1149 | } | 1151 | } |
1150 | } else { | 1152 | } else { |
1151 | module_put(disk->fops->owner); | ||
1152 | put_disk(disk); | ||
1153 | disk = NULL; | ||
1154 | if (bdev->bd_contains == bdev) { | 1153 | if (bdev->bd_contains == bdev) { |
1155 | if (bdev->bd_disk->fops->open) { | 1154 | if (bdev->bd_disk->fops->open) { |
1156 | ret = bdev->bd_disk->fops->open(bdev, mode); | 1155 | ret = bdev->bd_disk->fops->open(bdev, mode); |
@@ -1160,11 +1159,15 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) | |||
1160 | if (bdev->bd_invalidated) | 1159 | if (bdev->bd_invalidated) |
1161 | rescan_partitions(bdev->bd_disk, bdev); | 1160 | rescan_partitions(bdev->bd_disk, bdev); |
1162 | } | 1161 | } |
1162 | /* only one opener holds refs to the module and disk */ | ||
1163 | module_put(disk->fops->owner); | ||
1164 | put_disk(disk); | ||
1163 | } | 1165 | } |
1164 | bdev->bd_openers++; | 1166 | bdev->bd_openers++; |
1165 | if (for_part) | 1167 | if (for_part) |
1166 | bdev->bd_part_count++; | 1168 | bdev->bd_part_count++; |
1167 | mutex_unlock(&bdev->bd_mutex); | 1169 | mutex_unlock(&bdev->bd_mutex); |
1170 | disk_unblock_events(disk); | ||
1168 | return 0; | 1171 | return 0; |
1169 | 1172 | ||
1170 | out_clear: | 1173 | out_clear: |
@@ -1177,10 +1180,10 @@ static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) | |||
1177 | bdev->bd_contains = NULL; | 1180 | bdev->bd_contains = NULL; |
1178 | out_unlock_bdev: | 1181 | out_unlock_bdev: |
1179 | mutex_unlock(&bdev->bd_mutex); | 1182 | mutex_unlock(&bdev->bd_mutex); |
1180 | out: | 1183 | disk_unblock_events(disk); |
1181 | if (disk) | 1184 | module_put(disk->fops->owner); |
1182 | module_put(disk->fops->owner); | ||
1183 | put_disk(disk); | 1185 | put_disk(disk); |
1186 | out: | ||
1184 | bdput(bdev); | 1187 | bdput(bdev); |
1185 | 1188 | ||
1186 | return ret; | 1189 | return ret; |
@@ -1446,14 +1449,13 @@ int blkdev_put(struct block_device *bdev, fmode_t mode) | |||
1446 | if (bdev_free) { | 1449 | if (bdev_free) { |
1447 | if (bdev->bd_write_holder) { | 1450 | if (bdev->bd_write_holder) { |
1448 | disk_unblock_events(bdev->bd_disk); | 1451 | disk_unblock_events(bdev->bd_disk); |
1449 | bdev->bd_write_holder = false; | ||
1450 | } else | ||
1451 | disk_check_events(bdev->bd_disk); | 1452 | disk_check_events(bdev->bd_disk); |
1453 | bdev->bd_write_holder = false; | ||
1454 | } | ||
1452 | } | 1455 | } |
1453 | 1456 | ||
1454 | mutex_unlock(&bdev->bd_mutex); | 1457 | mutex_unlock(&bdev->bd_mutex); |
1455 | } else | 1458 | } |
1456 | disk_check_events(bdev->bd_disk); | ||
1457 | 1459 | ||
1458 | return __blkdev_put(bdev, mode, 0); | 1460 | return __blkdev_put(bdev, mode, 0); |
1459 | } | 1461 | } |
@@ -1527,7 +1529,6 @@ static int blkdev_releasepage(struct page *page, gfp_t wait) | |||
1527 | static const struct address_space_operations def_blk_aops = { | 1529 | static const struct address_space_operations def_blk_aops = { |
1528 | .readpage = blkdev_readpage, | 1530 | .readpage = blkdev_readpage, |
1529 | .writepage = blkdev_writepage, | 1531 | .writepage = blkdev_writepage, |
1530 | .sync_page = block_sync_page, | ||
1531 | .write_begin = blkdev_write_begin, | 1532 | .write_begin = blkdev_write_begin, |
1532 | .write_end = blkdev_write_end, | 1533 | .write_end = blkdev_write_end, |
1533 | .writepages = generic_writepages, | 1534 | .writepages = generic_writepages, |
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 100b07f021b4..830d261d0e6b 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c | |||
@@ -847,7 +847,6 @@ static const struct address_space_operations btree_aops = { | |||
847 | .writepages = btree_writepages, | 847 | .writepages = btree_writepages, |
848 | .releasepage = btree_releasepage, | 848 | .releasepage = btree_releasepage, |
849 | .invalidatepage = btree_invalidatepage, | 849 | .invalidatepage = btree_invalidatepage, |
850 | .sync_page = block_sync_page, | ||
851 | #ifdef CONFIG_MIGRATION | 850 | #ifdef CONFIG_MIGRATION |
852 | .migratepage = btree_migratepage, | 851 | .migratepage = btree_migratepage, |
853 | #endif | 852 | #endif |
@@ -1331,82 +1330,6 @@ static int btrfs_congested_fn(void *congested_data, int bdi_bits) | |||
1331 | } | 1330 | } |
1332 | 1331 | ||
1333 | /* | 1332 | /* |
1334 | * this unplugs every device on the box, and it is only used when page | ||
1335 | * is null | ||
1336 | */ | ||
1337 | static void __unplug_io_fn(struct backing_dev_info *bdi, struct page *page) | ||
1338 | { | ||
1339 | struct btrfs_device *device; | ||
1340 | struct btrfs_fs_info *info; | ||
1341 | |||
1342 | info = (struct btrfs_fs_info *)bdi->unplug_io_data; | ||
1343 | list_for_each_entry(device, &info->fs_devices->devices, dev_list) { | ||
1344 | if (!device->bdev) | ||
1345 | continue; | ||
1346 | |||
1347 | bdi = blk_get_backing_dev_info(device->bdev); | ||
1348 | if (bdi->unplug_io_fn) | ||
1349 | bdi->unplug_io_fn(bdi, page); | ||
1350 | } | ||
1351 | } | ||
1352 | |||
1353 | static void btrfs_unplug_io_fn(struct backing_dev_info *bdi, struct page *page) | ||
1354 | { | ||
1355 | struct inode *inode; | ||
1356 | struct extent_map_tree *em_tree; | ||
1357 | struct extent_map *em; | ||
1358 | struct address_space *mapping; | ||
1359 | u64 offset; | ||
1360 | |||
1361 | /* the generic O_DIRECT read code does this */ | ||
1362 | if (1 || !page) { | ||
1363 | __unplug_io_fn(bdi, page); | ||
1364 | return; | ||
1365 | } | ||
1366 | |||
1367 | /* | ||
1368 | * page->mapping may change at any time. Get a consistent copy | ||
1369 | * and use that for everything below | ||
1370 | */ | ||
1371 | smp_mb(); | ||
1372 | mapping = page->mapping; | ||
1373 | if (!mapping) | ||
1374 | return; | ||
1375 | |||
1376 | inode = mapping->host; | ||
1377 | |||
1378 | /* | ||
1379 | * don't do the expensive searching for a small number of | ||
1380 | * devices | ||
1381 | */ | ||
1382 | if (BTRFS_I(inode)->root->fs_info->fs_devices->open_devices <= 2) { | ||
1383 | __unplug_io_fn(bdi, page); | ||
1384 | return; | ||
1385 | } | ||
1386 | |||
1387 | offset = page_offset(page); | ||
1388 | |||
1389 | em_tree = &BTRFS_I(inode)->extent_tree; | ||
1390 | read_lock(&em_tree->lock); | ||
1391 | em = lookup_extent_mapping(em_tree, offset, PAGE_CACHE_SIZE); | ||
1392 | read_unlock(&em_tree->lock); | ||
1393 | if (!em) { | ||
1394 | __unplug_io_fn(bdi, page); | ||
1395 | return; | ||
1396 | } | ||
1397 | |||
1398 | if (em->block_start >= EXTENT_MAP_LAST_BYTE) { | ||
1399 | free_extent_map(em); | ||
1400 | __unplug_io_fn(bdi, page); | ||
1401 | return; | ||
1402 | } | ||
1403 | offset = offset - em->start; | ||
1404 | btrfs_unplug_page(&BTRFS_I(inode)->root->fs_info->mapping_tree, | ||
1405 | em->block_start + offset, page); | ||
1406 | free_extent_map(em); | ||
1407 | } | ||
1408 | |||
1409 | /* | ||
1410 | * If this fails, caller must call bdi_destroy() to get rid of the | 1333 | * If this fails, caller must call bdi_destroy() to get rid of the |
1411 | * bdi again. | 1334 | * bdi again. |
1412 | */ | 1335 | */ |
@@ -1420,8 +1343,6 @@ static int setup_bdi(struct btrfs_fs_info *info, struct backing_dev_info *bdi) | |||
1420 | return err; | 1343 | return err; |
1421 | 1344 | ||
1422 | bdi->ra_pages = default_backing_dev_info.ra_pages; | 1345 | bdi->ra_pages = default_backing_dev_info.ra_pages; |
1423 | bdi->unplug_io_fn = btrfs_unplug_io_fn; | ||
1424 | bdi->unplug_io_data = info; | ||
1425 | bdi->congested_fn = btrfs_congested_fn; | 1346 | bdi->congested_fn = btrfs_congested_fn; |
1426 | bdi->congested_data = info; | 1347 | bdi->congested_data = info; |
1427 | return 0; | 1348 | return 0; |
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 714adc4ac4c2..b5b92824a271 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c | |||
@@ -2188,7 +2188,7 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc, | |||
2188 | unsigned long nr_written = 0; | 2188 | unsigned long nr_written = 0; |
2189 | 2189 | ||
2190 | if (wbc->sync_mode == WB_SYNC_ALL) | 2190 | if (wbc->sync_mode == WB_SYNC_ALL) |
2191 | write_flags = WRITE_SYNC_PLUG; | 2191 | write_flags = WRITE_SYNC; |
2192 | else | 2192 | else |
2193 | write_flags = WRITE; | 2193 | write_flags = WRITE; |
2194 | 2194 | ||
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 512c3d1da083..119520bdb9a5 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c | |||
@@ -7340,7 +7340,6 @@ static const struct address_space_operations btrfs_aops = { | |||
7340 | .writepage = btrfs_writepage, | 7340 | .writepage = btrfs_writepage, |
7341 | .writepages = btrfs_writepages, | 7341 | .writepages = btrfs_writepages, |
7342 | .readpages = btrfs_readpages, | 7342 | .readpages = btrfs_readpages, |
7343 | .sync_page = block_sync_page, | ||
7344 | .direct_IO = btrfs_direct_IO, | 7343 | .direct_IO = btrfs_direct_IO, |
7345 | .invalidatepage = btrfs_invalidatepage, | 7344 | .invalidatepage = btrfs_invalidatepage, |
7346 | .releasepage = btrfs_releasepage, | 7345 | .releasepage = btrfs_releasepage, |
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index dd13eb81ee40..9d554e8e6583 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c | |||
@@ -162,7 +162,6 @@ static noinline int run_scheduled_bios(struct btrfs_device *device) | |||
162 | struct bio *cur; | 162 | struct bio *cur; |
163 | int again = 0; | 163 | int again = 0; |
164 | unsigned long num_run; | 164 | unsigned long num_run; |
165 | unsigned long num_sync_run; | ||
166 | unsigned long batch_run = 0; | 165 | unsigned long batch_run = 0; |
167 | unsigned long limit; | 166 | unsigned long limit; |
168 | unsigned long last_waited = 0; | 167 | unsigned long last_waited = 0; |
@@ -173,11 +172,6 @@ static noinline int run_scheduled_bios(struct btrfs_device *device) | |||
173 | limit = btrfs_async_submit_limit(fs_info); | 172 | limit = btrfs_async_submit_limit(fs_info); |
174 | limit = limit * 2 / 3; | 173 | limit = limit * 2 / 3; |
175 | 174 | ||
176 | /* we want to make sure that every time we switch from the sync | ||
177 | * list to the normal list, we unplug | ||
178 | */ | ||
179 | num_sync_run = 0; | ||
180 | |||
181 | loop: | 175 | loop: |
182 | spin_lock(&device->io_lock); | 176 | spin_lock(&device->io_lock); |
183 | 177 | ||
@@ -223,15 +217,6 @@ loop_lock: | |||
223 | 217 | ||
224 | spin_unlock(&device->io_lock); | 218 | spin_unlock(&device->io_lock); |
225 | 219 | ||
226 | /* | ||
227 | * if we're doing the regular priority list, make sure we unplug | ||
228 | * for any high prio bios we've sent down | ||
229 | */ | ||
230 | if (pending_bios == &device->pending_bios && num_sync_run > 0) { | ||
231 | num_sync_run = 0; | ||
232 | blk_run_backing_dev(bdi, NULL); | ||
233 | } | ||
234 | |||
235 | while (pending) { | 220 | while (pending) { |
236 | 221 | ||
237 | rmb(); | 222 | rmb(); |
@@ -259,19 +244,11 @@ loop_lock: | |||
259 | 244 | ||
260 | BUG_ON(atomic_read(&cur->bi_cnt) == 0); | 245 | BUG_ON(atomic_read(&cur->bi_cnt) == 0); |
261 | 246 | ||
262 | if (cur->bi_rw & REQ_SYNC) | ||
263 | num_sync_run++; | ||
264 | |||
265 | submit_bio(cur->bi_rw, cur); | 247 | submit_bio(cur->bi_rw, cur); |
266 | num_run++; | 248 | num_run++; |
267 | batch_run++; | 249 | batch_run++; |
268 | if (need_resched()) { | 250 | if (need_resched()) |
269 | if (num_sync_run) { | ||
270 | blk_run_backing_dev(bdi, NULL); | ||
271 | num_sync_run = 0; | ||
272 | } | ||
273 | cond_resched(); | 251 | cond_resched(); |
274 | } | ||
275 | 252 | ||
276 | /* | 253 | /* |
277 | * we made progress, there is more work to do and the bdi | 254 | * we made progress, there is more work to do and the bdi |
@@ -304,13 +281,8 @@ loop_lock: | |||
304 | * against it before looping | 281 | * against it before looping |
305 | */ | 282 | */ |
306 | last_waited = ioc->last_waited; | 283 | last_waited = ioc->last_waited; |
307 | if (need_resched()) { | 284 | if (need_resched()) |
308 | if (num_sync_run) { | ||
309 | blk_run_backing_dev(bdi, NULL); | ||
310 | num_sync_run = 0; | ||
311 | } | ||
312 | cond_resched(); | 285 | cond_resched(); |
313 | } | ||
314 | continue; | 286 | continue; |
315 | } | 287 | } |
316 | spin_lock(&device->io_lock); | 288 | spin_lock(&device->io_lock); |
@@ -323,22 +295,6 @@ loop_lock: | |||
323 | } | 295 | } |
324 | } | 296 | } |
325 | 297 | ||
326 | if (num_sync_run) { | ||
327 | num_sync_run = 0; | ||
328 | blk_run_backing_dev(bdi, NULL); | ||
329 | } | ||
330 | /* | ||
331 | * IO has already been through a long path to get here. Checksumming, | ||
332 | * async helper threads, perhaps compression. We've done a pretty | ||
333 | * good job of collecting a batch of IO and should just unplug | ||
334 | * the device right away. | ||
335 | * | ||
336 | * This will help anyone who is waiting on the IO, they might have | ||
337 | * already unplugged, but managed to do so before the bio they | ||
338 | * cared about found its way down here. | ||
339 | */ | ||
340 | blk_run_backing_dev(bdi, NULL); | ||
341 | |||
342 | cond_resched(); | 298 | cond_resched(); |
343 | if (again) | 299 | if (again) |
344 | goto loop; | 300 | goto loop; |
@@ -2955,7 +2911,7 @@ static int find_live_mirror(struct map_lookup *map, int first, int num, | |||
2955 | static int __btrfs_map_block(struct btrfs_mapping_tree *map_tree, int rw, | 2911 | static int __btrfs_map_block(struct btrfs_mapping_tree *map_tree, int rw, |
2956 | u64 logical, u64 *length, | 2912 | u64 logical, u64 *length, |
2957 | struct btrfs_multi_bio **multi_ret, | 2913 | struct btrfs_multi_bio **multi_ret, |
2958 | int mirror_num, struct page *unplug_page) | 2914 | int mirror_num) |
2959 | { | 2915 | { |
2960 | struct extent_map *em; | 2916 | struct extent_map *em; |
2961 | struct map_lookup *map; | 2917 | struct map_lookup *map; |
@@ -2987,11 +2943,6 @@ again: | |||
2987 | em = lookup_extent_mapping(em_tree, logical, *length); | 2943 | em = lookup_extent_mapping(em_tree, logical, *length); |
2988 | read_unlock(&em_tree->lock); | 2944 | read_unlock(&em_tree->lock); |
2989 | 2945 | ||
2990 | if (!em && unplug_page) { | ||
2991 | kfree(multi); | ||
2992 | return 0; | ||
2993 | } | ||
2994 | |||
2995 | if (!em) { | 2946 | if (!em) { |
2996 | printk(KERN_CRIT "unable to find logical %llu len %llu\n", | 2947 | printk(KERN_CRIT "unable to find logical %llu len %llu\n", |
2997 | (unsigned long long)logical, | 2948 | (unsigned long long)logical, |
@@ -3047,13 +2998,13 @@ again: | |||
3047 | *length = em->len - offset; | 2998 | *length = em->len - offset; |
3048 | } | 2999 | } |
3049 | 3000 | ||
3050 | if (!multi_ret && !unplug_page) | 3001 | if (!multi_ret) |
3051 | goto out; | 3002 | goto out; |
3052 | 3003 | ||
3053 | num_stripes = 1; | 3004 | num_stripes = 1; |
3054 | stripe_index = 0; | 3005 | stripe_index = 0; |
3055 | if (map->type & BTRFS_BLOCK_GROUP_RAID1) { | 3006 | if (map->type & BTRFS_BLOCK_GROUP_RAID1) { |
3056 | if (unplug_page || (rw & REQ_WRITE)) | 3007 | if (rw & REQ_WRITE) |
3057 | num_stripes = map->num_stripes; | 3008 | num_stripes = map->num_stripes; |
3058 | else if (mirror_num) | 3009 | else if (mirror_num) |
3059 | stripe_index = mirror_num - 1; | 3010 | stripe_index = mirror_num - 1; |
@@ -3075,7 +3026,7 @@ again: | |||
3075 | stripe_index = do_div(stripe_nr, factor); | 3026 | stripe_index = do_div(stripe_nr, factor); |
3076 | stripe_index *= map->sub_stripes; | 3027 | stripe_index *= map->sub_stripes; |
3077 | 3028 | ||
3078 | if (unplug_page || (rw & REQ_WRITE)) | 3029 | if (rw & REQ_WRITE) |
3079 | num_stripes = map->sub_stripes; | 3030 | num_stripes = map->sub_stripes; |
3080 | else if (mirror_num) | 3031 | else if (mirror_num) |
3081 | stripe_index += mirror_num - 1; | 3032 | stripe_index += mirror_num - 1; |
@@ -3095,22 +3046,10 @@ again: | |||
3095 | BUG_ON(stripe_index >= map->num_stripes); | 3046 | BUG_ON(stripe_index >= map->num_stripes); |
3096 | 3047 | ||
3097 | for (i = 0; i < num_stripes; i++) { | 3048 | for (i = 0; i < num_stripes; i++) { |
3098 | if (unplug_page) { | 3049 | multi->stripes[i].physical = |
3099 | struct btrfs_device *device; | 3050 | map->stripes[stripe_index].physical + |
3100 | struct backing_dev_info *bdi; | 3051 | stripe_offset + stripe_nr * map->stripe_len; |
3101 | 3052 | multi->stripes[i].dev = map->stripes[stripe_index].dev; | |
3102 | device = map->stripes[stripe_index].dev; | ||
3103 | if (device->bdev) { | ||
3104 | bdi = blk_get_backing_dev_info(device->bdev); | ||
3105 | if (bdi->unplug_io_fn) | ||
3106 | bdi->unplug_io_fn(bdi, unplug_page); | ||
3107 | } | ||
3108 | } else { | ||
3109 | multi->stripes[i].physical = | ||
3110 | map->stripes[stripe_index].physical + | ||
3111 | stripe_offset + stripe_nr * map->stripe_len; | ||
3112 | multi->stripes[i].dev = map->stripes[stripe_index].dev; | ||
3113 | } | ||
3114 | stripe_index++; | 3053 | stripe_index++; |
3115 | } | 3054 | } |
3116 | if (multi_ret) { | 3055 | if (multi_ret) { |
@@ -3128,7 +3067,7 @@ int btrfs_map_block(struct btrfs_mapping_tree *map_tree, int rw, | |||
3128 | struct btrfs_multi_bio **multi_ret, int mirror_num) | 3067 | struct btrfs_multi_bio **multi_ret, int mirror_num) |
3129 | { | 3068 | { |
3130 | return __btrfs_map_block(map_tree, rw, logical, length, multi_ret, | 3069 | return __btrfs_map_block(map_tree, rw, logical, length, multi_ret, |
3131 | mirror_num, NULL); | 3070 | mirror_num); |
3132 | } | 3071 | } |
3133 | 3072 | ||
3134 | int btrfs_rmap_block(struct btrfs_mapping_tree *map_tree, | 3073 | int btrfs_rmap_block(struct btrfs_mapping_tree *map_tree, |
@@ -3196,14 +3135,6 @@ int btrfs_rmap_block(struct btrfs_mapping_tree *map_tree, | |||
3196 | return 0; | 3135 | return 0; |
3197 | } | 3136 | } |
3198 | 3137 | ||
3199 | int btrfs_unplug_page(struct btrfs_mapping_tree *map_tree, | ||
3200 | u64 logical, struct page *page) | ||
3201 | { | ||
3202 | u64 length = PAGE_CACHE_SIZE; | ||
3203 | return __btrfs_map_block(map_tree, READ, logical, &length, | ||
3204 | NULL, 0, page); | ||
3205 | } | ||
3206 | |||
3207 | static void end_bio_multi_stripe(struct bio *bio, int err) | 3138 | static void end_bio_multi_stripe(struct bio *bio, int err) |
3208 | { | 3139 | { |
3209 | struct btrfs_multi_bio *multi = bio->bi_private; | 3140 | struct btrfs_multi_bio *multi = bio->bi_private; |
diff --git a/fs/buffer.c b/fs/buffer.c index 2219a76e2caf..2e6b1a387b7e 100644 --- a/fs/buffer.c +++ b/fs/buffer.c | |||
@@ -54,23 +54,15 @@ init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private) | |||
54 | } | 54 | } |
55 | EXPORT_SYMBOL(init_buffer); | 55 | EXPORT_SYMBOL(init_buffer); |
56 | 56 | ||
57 | static int sync_buffer(void *word) | 57 | static int sleep_on_buffer(void *word) |
58 | { | 58 | { |
59 | struct block_device *bd; | ||
60 | struct buffer_head *bh | ||
61 | = container_of(word, struct buffer_head, b_state); | ||
62 | |||
63 | smp_mb(); | ||
64 | bd = bh->b_bdev; | ||
65 | if (bd) | ||
66 | blk_run_address_space(bd->bd_inode->i_mapping); | ||
67 | io_schedule(); | 59 | io_schedule(); |
68 | return 0; | 60 | return 0; |
69 | } | 61 | } |
70 | 62 | ||
71 | void __lock_buffer(struct buffer_head *bh) | 63 | void __lock_buffer(struct buffer_head *bh) |
72 | { | 64 | { |
73 | wait_on_bit_lock(&bh->b_state, BH_Lock, sync_buffer, | 65 | wait_on_bit_lock(&bh->b_state, BH_Lock, sleep_on_buffer, |
74 | TASK_UNINTERRUPTIBLE); | 66 | TASK_UNINTERRUPTIBLE); |
75 | } | 67 | } |
76 | EXPORT_SYMBOL(__lock_buffer); | 68 | EXPORT_SYMBOL(__lock_buffer); |
@@ -90,7 +82,7 @@ EXPORT_SYMBOL(unlock_buffer); | |||
90 | */ | 82 | */ |
91 | void __wait_on_buffer(struct buffer_head * bh) | 83 | void __wait_on_buffer(struct buffer_head * bh) |
92 | { | 84 | { |
93 | wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE); | 85 | wait_on_bit(&bh->b_state, BH_Lock, sleep_on_buffer, TASK_UNINTERRUPTIBLE); |
94 | } | 86 | } |
95 | EXPORT_SYMBOL(__wait_on_buffer); | 87 | EXPORT_SYMBOL(__wait_on_buffer); |
96 | 88 | ||
@@ -749,10 +741,12 @@ static int fsync_buffers_list(spinlock_t *lock, struct list_head *list) | |||
749 | { | 741 | { |
750 | struct buffer_head *bh; | 742 | struct buffer_head *bh; |
751 | struct list_head tmp; | 743 | struct list_head tmp; |
752 | struct address_space *mapping, *prev_mapping = NULL; | 744 | struct address_space *mapping; |
753 | int err = 0, err2; | 745 | int err = 0, err2; |
746 | struct blk_plug plug; | ||
754 | 747 | ||
755 | INIT_LIST_HEAD(&tmp); | 748 | INIT_LIST_HEAD(&tmp); |
749 | blk_start_plug(&plug); | ||
756 | 750 | ||
757 | spin_lock(lock); | 751 | spin_lock(lock); |
758 | while (!list_empty(list)) { | 752 | while (!list_empty(list)) { |
@@ -775,7 +769,7 @@ static int fsync_buffers_list(spinlock_t *lock, struct list_head *list) | |||
775 | * still in flight on potentially older | 769 | * still in flight on potentially older |
776 | * contents. | 770 | * contents. |
777 | */ | 771 | */ |
778 | write_dirty_buffer(bh, WRITE_SYNC_PLUG); | 772 | write_dirty_buffer(bh, WRITE_SYNC); |
779 | 773 | ||
780 | /* | 774 | /* |
781 | * Kick off IO for the previous mapping. Note | 775 | * Kick off IO for the previous mapping. Note |
@@ -783,16 +777,16 @@ static int fsync_buffers_list(spinlock_t *lock, struct list_head *list) | |||
783 | * wait_on_buffer() will do that for us | 777 | * wait_on_buffer() will do that for us |
784 | * through sync_buffer(). | 778 | * through sync_buffer(). |
785 | */ | 779 | */ |
786 | if (prev_mapping && prev_mapping != mapping) | ||
787 | blk_run_address_space(prev_mapping); | ||
788 | prev_mapping = mapping; | ||
789 | |||
790 | brelse(bh); | 780 | brelse(bh); |
791 | spin_lock(lock); | 781 | spin_lock(lock); |
792 | } | 782 | } |
793 | } | 783 | } |
794 | } | 784 | } |
795 | 785 | ||
786 | spin_unlock(lock); | ||
787 | blk_finish_plug(&plug); | ||
788 | spin_lock(lock); | ||
789 | |||
796 | while (!list_empty(&tmp)) { | 790 | while (!list_empty(&tmp)) { |
797 | bh = BH_ENTRY(tmp.prev); | 791 | bh = BH_ENTRY(tmp.prev); |
798 | get_bh(bh); | 792 | get_bh(bh); |
@@ -1614,14 +1608,8 @@ EXPORT_SYMBOL(unmap_underlying_metadata); | |||
1614 | * prevents this contention from occurring. | 1608 | * prevents this contention from occurring. |
1615 | * | 1609 | * |
1616 | * If block_write_full_page() is called with wbc->sync_mode == | 1610 | * If block_write_full_page() is called with wbc->sync_mode == |
1617 | * WB_SYNC_ALL, the writes are posted using WRITE_SYNC_PLUG; this | 1611 | * WB_SYNC_ALL, the writes are posted using WRITE_SYNC; this |
1618 | * causes the writes to be flagged as synchronous writes, but the | 1612 | * causes the writes to be flagged as synchronous writes. |
1619 | * block device queue will NOT be unplugged, since usually many pages | ||
1620 | * will be pushed to the out before the higher-level caller actually | ||
1621 | * waits for the writes to be completed. The various wait functions, | ||
1622 | * such as wait_on_writeback_range() will ultimately call sync_page() | ||
1623 | * which will ultimately call blk_run_backing_dev(), which will end up | ||
1624 | * unplugging the device queue. | ||
1625 | */ | 1613 | */ |
1626 | static int __block_write_full_page(struct inode *inode, struct page *page, | 1614 | static int __block_write_full_page(struct inode *inode, struct page *page, |
1627 | get_block_t *get_block, struct writeback_control *wbc, | 1615 | get_block_t *get_block, struct writeback_control *wbc, |
@@ -1634,7 +1622,7 @@ static int __block_write_full_page(struct inode *inode, struct page *page, | |||
1634 | const unsigned blocksize = 1 << inode->i_blkbits; | 1622 | const unsigned blocksize = 1 << inode->i_blkbits; |
1635 | int nr_underway = 0; | 1623 | int nr_underway = 0; |
1636 | int write_op = (wbc->sync_mode == WB_SYNC_ALL ? | 1624 | int write_op = (wbc->sync_mode == WB_SYNC_ALL ? |
1637 | WRITE_SYNC_PLUG : WRITE); | 1625 | WRITE_SYNC : WRITE); |
1638 | 1626 | ||
1639 | BUG_ON(!PageLocked(page)); | 1627 | BUG_ON(!PageLocked(page)); |
1640 | 1628 | ||
@@ -3138,17 +3126,6 @@ out: | |||
3138 | } | 3126 | } |
3139 | EXPORT_SYMBOL(try_to_free_buffers); | 3127 | EXPORT_SYMBOL(try_to_free_buffers); |
3140 | 3128 | ||
3141 | void block_sync_page(struct page *page) | ||
3142 | { | ||
3143 | struct address_space *mapping; | ||
3144 | |||
3145 | smp_mb(); | ||
3146 | mapping = page_mapping(page); | ||
3147 | if (mapping) | ||
3148 | blk_run_backing_dev(mapping->backing_dev_info, page); | ||
3149 | } | ||
3150 | EXPORT_SYMBOL(block_sync_page); | ||
3151 | |||
3152 | /* | 3129 | /* |
3153 | * There are no bdflush tunables left. But distributions are | 3130 | * There are no bdflush tunables left. But distributions are |
3154 | * still running obsolete flush daemons, so we terminate them here. | 3131 | * still running obsolete flush daemons, so we terminate them here. |
diff --git a/fs/cifs/file.c b/fs/cifs/file.c index e964b1cd5dd0..c27d236738fc 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c | |||
@@ -1569,34 +1569,6 @@ int cifs_fsync(struct file *file, int datasync) | |||
1569 | return rc; | 1569 | return rc; |
1570 | } | 1570 | } |
1571 | 1571 | ||
1572 | /* static void cifs_sync_page(struct page *page) | ||
1573 | { | ||
1574 | struct address_space *mapping; | ||
1575 | struct inode *inode; | ||
1576 | unsigned long index = page->index; | ||
1577 | unsigned int rpages = 0; | ||
1578 | int rc = 0; | ||
1579 | |||
1580 | cFYI(1, "sync page %p", page); | ||
1581 | mapping = page->mapping; | ||
1582 | if (!mapping) | ||
1583 | return 0; | ||
1584 | inode = mapping->host; | ||
1585 | if (!inode) | ||
1586 | return; */ | ||
1587 | |||
1588 | /* fill in rpages then | ||
1589 | result = cifs_pagein_inode(inode, index, rpages); */ /* BB finish */ | ||
1590 | |||
1591 | /* cFYI(1, "rpages is %d for sync page of Index %ld", rpages, index); | ||
1592 | |||
1593 | #if 0 | ||
1594 | if (rc < 0) | ||
1595 | return rc; | ||
1596 | return 0; | ||
1597 | #endif | ||
1598 | } */ | ||
1599 | |||
1600 | /* | 1572 | /* |
1601 | * As file closes, flush all cached write data for this inode checking | 1573 | * As file closes, flush all cached write data for this inode checking |
1602 | * for write behind errors. | 1574 | * for write behind errors. |
@@ -2510,7 +2482,6 @@ const struct address_space_operations cifs_addr_ops = { | |||
2510 | .set_page_dirty = __set_page_dirty_nobuffers, | 2482 | .set_page_dirty = __set_page_dirty_nobuffers, |
2511 | .releasepage = cifs_release_page, | 2483 | .releasepage = cifs_release_page, |
2512 | .invalidatepage = cifs_invalidate_page, | 2484 | .invalidatepage = cifs_invalidate_page, |
2513 | /* .sync_page = cifs_sync_page, */ | ||
2514 | /* .direct_IO = */ | 2485 | /* .direct_IO = */ |
2515 | }; | 2486 | }; |
2516 | 2487 | ||
@@ -2528,6 +2499,5 @@ const struct address_space_operations cifs_addr_ops_smallbuf = { | |||
2528 | .set_page_dirty = __set_page_dirty_nobuffers, | 2499 | .set_page_dirty = __set_page_dirty_nobuffers, |
2529 | .releasepage = cifs_release_page, | 2500 | .releasepage = cifs_release_page, |
2530 | .invalidatepage = cifs_invalidate_page, | 2501 | .invalidatepage = cifs_invalidate_page, |
2531 | /* .sync_page = cifs_sync_page, */ | ||
2532 | /* .direct_IO = */ | 2502 | /* .direct_IO = */ |
2533 | }; | 2503 | }; |
diff --git a/fs/direct-io.c b/fs/direct-io.c index dcb5577cde1d..ac5f164170e3 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c | |||
@@ -1110,11 +1110,8 @@ direct_io_worker(int rw, struct kiocb *iocb, struct inode *inode, | |||
1110 | ((rw & READ) || (dio->result == dio->size))) | 1110 | ((rw & READ) || (dio->result == dio->size))) |
1111 | ret = -EIOCBQUEUED; | 1111 | ret = -EIOCBQUEUED; |
1112 | 1112 | ||
1113 | if (ret != -EIOCBQUEUED) { | 1113 | if (ret != -EIOCBQUEUED) |
1114 | /* All IO is now issued, send it on its way */ | ||
1115 | blk_run_address_space(inode->i_mapping); | ||
1116 | dio_await_completion(dio); | 1114 | dio_await_completion(dio); |
1117 | } | ||
1118 | 1115 | ||
1119 | /* | 1116 | /* |
1120 | * Sync will always be dropping the final ref and completing the | 1117 | * Sync will always be dropping the final ref and completing the |
@@ -1176,7 +1173,7 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, | |||
1176 | struct dio *dio; | 1173 | struct dio *dio; |
1177 | 1174 | ||
1178 | if (rw & WRITE) | 1175 | if (rw & WRITE) |
1179 | rw = WRITE_ODIRECT_PLUG; | 1176 | rw = WRITE_ODIRECT; |
1180 | 1177 | ||
1181 | if (bdev) | 1178 | if (bdev) |
1182 | bdev_blkbits = blksize_bits(bdev_logical_block_size(bdev)); | 1179 | bdev_blkbits = blksize_bits(bdev_logical_block_size(bdev)); |
diff --git a/fs/efs/inode.c b/fs/efs/inode.c index a8e7797b9477..9c13412e6c99 100644 --- a/fs/efs/inode.c +++ b/fs/efs/inode.c | |||
@@ -23,7 +23,6 @@ static sector_t _efs_bmap(struct address_space *mapping, sector_t block) | |||
23 | } | 23 | } |
24 | static const struct address_space_operations efs_aops = { | 24 | static const struct address_space_operations efs_aops = { |
25 | .readpage = efs_readpage, | 25 | .readpage = efs_readpage, |
26 | .sync_page = block_sync_page, | ||
27 | .bmap = _efs_bmap | 26 | .bmap = _efs_bmap |
28 | }; | 27 | }; |
29 | 28 | ||
diff --git a/fs/exofs/inode.c b/fs/exofs/inode.c index 0c713cfbebf0..8472c098445d 100644 --- a/fs/exofs/inode.c +++ b/fs/exofs/inode.c | |||
@@ -823,7 +823,6 @@ const struct address_space_operations exofs_aops = { | |||
823 | .direct_IO = NULL, /* TODO: Should be trivial to do */ | 823 | .direct_IO = NULL, /* TODO: Should be trivial to do */ |
824 | 824 | ||
825 | /* With these NULL has special meaning or default is not exported */ | 825 | /* With these NULL has special meaning or default is not exported */ |
826 | .sync_page = NULL, | ||
827 | .get_xip_mem = NULL, | 826 | .get_xip_mem = NULL, |
828 | .migratepage = NULL, | 827 | .migratepage = NULL, |
829 | .launder_page = NULL, | 828 | .launder_page = NULL, |
diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 40ad210a5049..c47f706878b5 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c | |||
@@ -860,7 +860,6 @@ const struct address_space_operations ext2_aops = { | |||
860 | .readpage = ext2_readpage, | 860 | .readpage = ext2_readpage, |
861 | .readpages = ext2_readpages, | 861 | .readpages = ext2_readpages, |
862 | .writepage = ext2_writepage, | 862 | .writepage = ext2_writepage, |
863 | .sync_page = block_sync_page, | ||
864 | .write_begin = ext2_write_begin, | 863 | .write_begin = ext2_write_begin, |
865 | .write_end = ext2_write_end, | 864 | .write_end = ext2_write_end, |
866 | .bmap = ext2_bmap, | 865 | .bmap = ext2_bmap, |
@@ -880,7 +879,6 @@ const struct address_space_operations ext2_nobh_aops = { | |||
880 | .readpage = ext2_readpage, | 879 | .readpage = ext2_readpage, |
881 | .readpages = ext2_readpages, | 880 | .readpages = ext2_readpages, |
882 | .writepage = ext2_nobh_writepage, | 881 | .writepage = ext2_nobh_writepage, |
883 | .sync_page = block_sync_page, | ||
884 | .write_begin = ext2_nobh_write_begin, | 882 | .write_begin = ext2_nobh_write_begin, |
885 | .write_end = nobh_write_end, | 883 | .write_end = nobh_write_end, |
886 | .bmap = ext2_bmap, | 884 | .bmap = ext2_bmap, |
diff --git a/fs/ext3/inode.c b/fs/ext3/inode.c index ae94f6d949f5..fe2541d250e4 100644 --- a/fs/ext3/inode.c +++ b/fs/ext3/inode.c | |||
@@ -1894,7 +1894,6 @@ static const struct address_space_operations ext3_ordered_aops = { | |||
1894 | .readpage = ext3_readpage, | 1894 | .readpage = ext3_readpage, |
1895 | .readpages = ext3_readpages, | 1895 | .readpages = ext3_readpages, |
1896 | .writepage = ext3_ordered_writepage, | 1896 | .writepage = ext3_ordered_writepage, |
1897 | .sync_page = block_sync_page, | ||
1898 | .write_begin = ext3_write_begin, | 1897 | .write_begin = ext3_write_begin, |
1899 | .write_end = ext3_ordered_write_end, | 1898 | .write_end = ext3_ordered_write_end, |
1900 | .bmap = ext3_bmap, | 1899 | .bmap = ext3_bmap, |
@@ -1910,7 +1909,6 @@ static const struct address_space_operations ext3_writeback_aops = { | |||
1910 | .readpage = ext3_readpage, | 1909 | .readpage = ext3_readpage, |
1911 | .readpages = ext3_readpages, | 1910 | .readpages = ext3_readpages, |
1912 | .writepage = ext3_writeback_writepage, | 1911 | .writepage = ext3_writeback_writepage, |
1913 | .sync_page = block_sync_page, | ||
1914 | .write_begin = ext3_write_begin, | 1912 | .write_begin = ext3_write_begin, |
1915 | .write_end = ext3_writeback_write_end, | 1913 | .write_end = ext3_writeback_write_end, |
1916 | .bmap = ext3_bmap, | 1914 | .bmap = ext3_bmap, |
@@ -1926,7 +1924,6 @@ static const struct address_space_operations ext3_journalled_aops = { | |||
1926 | .readpage = ext3_readpage, | 1924 | .readpage = ext3_readpage, |
1927 | .readpages = ext3_readpages, | 1925 | .readpages = ext3_readpages, |
1928 | .writepage = ext3_journalled_writepage, | 1926 | .writepage = ext3_journalled_writepage, |
1929 | .sync_page = block_sync_page, | ||
1930 | .write_begin = ext3_write_begin, | 1927 | .write_begin = ext3_write_begin, |
1931 | .write_end = ext3_journalled_write_end, | 1928 | .write_end = ext3_journalled_write_end, |
1932 | .set_page_dirty = ext3_journalled_set_page_dirty, | 1929 | .set_page_dirty = ext3_journalled_set_page_dirty, |
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 9f7f9e49914f..9297ad46c465 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c | |||
@@ -3903,7 +3903,6 @@ static const struct address_space_operations ext4_ordered_aops = { | |||
3903 | .readpage = ext4_readpage, | 3903 | .readpage = ext4_readpage, |
3904 | .readpages = ext4_readpages, | 3904 | .readpages = ext4_readpages, |
3905 | .writepage = ext4_writepage, | 3905 | .writepage = ext4_writepage, |
3906 | .sync_page = block_sync_page, | ||
3907 | .write_begin = ext4_write_begin, | 3906 | .write_begin = ext4_write_begin, |
3908 | .write_end = ext4_ordered_write_end, | 3907 | .write_end = ext4_ordered_write_end, |
3909 | .bmap = ext4_bmap, | 3908 | .bmap = ext4_bmap, |
@@ -3919,7 +3918,6 @@ static const struct address_space_operations ext4_writeback_aops = { | |||
3919 | .readpage = ext4_readpage, | 3918 | .readpage = ext4_readpage, |
3920 | .readpages = ext4_readpages, | 3919 | .readpages = ext4_readpages, |
3921 | .writepage = ext4_writepage, | 3920 | .writepage = ext4_writepage, |
3922 | .sync_page = block_sync_page, | ||
3923 | .write_begin = ext4_write_begin, | 3921 | .write_begin = ext4_write_begin, |
3924 | .write_end = ext4_writeback_write_end, | 3922 | .write_end = ext4_writeback_write_end, |
3925 | .bmap = ext4_bmap, | 3923 | .bmap = ext4_bmap, |
@@ -3935,7 +3933,6 @@ static const struct address_space_operations ext4_journalled_aops = { | |||
3935 | .readpage = ext4_readpage, | 3933 | .readpage = ext4_readpage, |
3936 | .readpages = ext4_readpages, | 3934 | .readpages = ext4_readpages, |
3937 | .writepage = ext4_writepage, | 3935 | .writepage = ext4_writepage, |
3938 | .sync_page = block_sync_page, | ||
3939 | .write_begin = ext4_write_begin, | 3936 | .write_begin = ext4_write_begin, |
3940 | .write_end = ext4_journalled_write_end, | 3937 | .write_end = ext4_journalled_write_end, |
3941 | .set_page_dirty = ext4_journalled_set_page_dirty, | 3938 | .set_page_dirty = ext4_journalled_set_page_dirty, |
@@ -3951,7 +3948,6 @@ static const struct address_space_operations ext4_da_aops = { | |||
3951 | .readpages = ext4_readpages, | 3948 | .readpages = ext4_readpages, |
3952 | .writepage = ext4_writepage, | 3949 | .writepage = ext4_writepage, |
3953 | .writepages = ext4_da_writepages, | 3950 | .writepages = ext4_da_writepages, |
3954 | .sync_page = block_sync_page, | ||
3955 | .write_begin = ext4_da_write_begin, | 3951 | .write_begin = ext4_da_write_begin, |
3956 | .write_end = ext4_da_write_end, | 3952 | .write_end = ext4_da_write_end, |
3957 | .bmap = ext4_bmap, | 3953 | .bmap = ext4_bmap, |
diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c index 955cc309142f..e2cd90e4bb7c 100644 --- a/fs/ext4/page-io.c +++ b/fs/ext4/page-io.c | |||
@@ -310,8 +310,7 @@ static int io_submit_init(struct ext4_io_submit *io, | |||
310 | io_end->offset = (page->index << PAGE_CACHE_SHIFT) + bh_offset(bh); | 310 | io_end->offset = (page->index << PAGE_CACHE_SHIFT) + bh_offset(bh); |
311 | 311 | ||
312 | io->io_bio = bio; | 312 | io->io_bio = bio; |
313 | io->io_op = (wbc->sync_mode == WB_SYNC_ALL ? | 313 | io->io_op = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE); |
314 | WRITE_SYNC_PLUG : WRITE); | ||
315 | io->io_next_block = bh->b_blocknr; | 314 | io->io_next_block = bh->b_blocknr; |
316 | return 0; | 315 | return 0; |
317 | } | 316 | } |
diff --git a/fs/fat/inode.c b/fs/fat/inode.c index 0e277ec4b612..8d68690bdcf1 100644 --- a/fs/fat/inode.c +++ b/fs/fat/inode.c | |||
@@ -236,7 +236,6 @@ static const struct address_space_operations fat_aops = { | |||
236 | .readpages = fat_readpages, | 236 | .readpages = fat_readpages, |
237 | .writepage = fat_writepage, | 237 | .writepage = fat_writepage, |
238 | .writepages = fat_writepages, | 238 | .writepages = fat_writepages, |
239 | .sync_page = block_sync_page, | ||
240 | .write_begin = fat_write_begin, | 239 | .write_begin = fat_write_begin, |
241 | .write_end = fat_write_end, | 240 | .write_end = fat_write_end, |
242 | .direct_IO = fat_direct_IO, | 241 | .direct_IO = fat_direct_IO, |
diff --git a/fs/freevxfs/vxfs_subr.c b/fs/freevxfs/vxfs_subr.c index 1429f3ae1e86..5d318c44f855 100644 --- a/fs/freevxfs/vxfs_subr.c +++ b/fs/freevxfs/vxfs_subr.c | |||
@@ -44,7 +44,6 @@ static sector_t vxfs_bmap(struct address_space *, sector_t); | |||
44 | const struct address_space_operations vxfs_aops = { | 44 | const struct address_space_operations vxfs_aops = { |
45 | .readpage = vxfs_readpage, | 45 | .readpage = vxfs_readpage, |
46 | .bmap = vxfs_bmap, | 46 | .bmap = vxfs_bmap, |
47 | .sync_page = block_sync_page, | ||
48 | }; | 47 | }; |
49 | 48 | ||
50 | inline void | 49 | inline void |
diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 051b1a084528..cc6ec4b2f0ff 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c | |||
@@ -870,7 +870,6 @@ static int fuse_bdi_init(struct fuse_conn *fc, struct super_block *sb) | |||
870 | 870 | ||
871 | fc->bdi.name = "fuse"; | 871 | fc->bdi.name = "fuse"; |
872 | fc->bdi.ra_pages = (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; | 872 | fc->bdi.ra_pages = (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; |
873 | fc->bdi.unplug_io_fn = default_unplug_io_fn; | ||
874 | /* fuse does it's own writeback accounting */ | 873 | /* fuse does it's own writeback accounting */ |
875 | fc->bdi.capabilities = BDI_CAP_NO_ACCT_WB; | 874 | fc->bdi.capabilities = BDI_CAP_NO_ACCT_WB; |
876 | 875 | ||
diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c index aad77e4f61b5..c71995b111bf 100644 --- a/fs/gfs2/aops.c +++ b/fs/gfs2/aops.c | |||
@@ -1117,7 +1117,6 @@ static const struct address_space_operations gfs2_writeback_aops = { | |||
1117 | .writepages = gfs2_writeback_writepages, | 1117 | .writepages = gfs2_writeback_writepages, |
1118 | .readpage = gfs2_readpage, | 1118 | .readpage = gfs2_readpage, |
1119 | .readpages = gfs2_readpages, | 1119 | .readpages = gfs2_readpages, |
1120 | .sync_page = block_sync_page, | ||
1121 | .write_begin = gfs2_write_begin, | 1120 | .write_begin = gfs2_write_begin, |
1122 | .write_end = gfs2_write_end, | 1121 | .write_end = gfs2_write_end, |
1123 | .bmap = gfs2_bmap, | 1122 | .bmap = gfs2_bmap, |
@@ -1133,7 +1132,6 @@ static const struct address_space_operations gfs2_ordered_aops = { | |||
1133 | .writepage = gfs2_ordered_writepage, | 1132 | .writepage = gfs2_ordered_writepage, |
1134 | .readpage = gfs2_readpage, | 1133 | .readpage = gfs2_readpage, |
1135 | .readpages = gfs2_readpages, | 1134 | .readpages = gfs2_readpages, |
1136 | .sync_page = block_sync_page, | ||
1137 | .write_begin = gfs2_write_begin, | 1135 | .write_begin = gfs2_write_begin, |
1138 | .write_end = gfs2_write_end, | 1136 | .write_end = gfs2_write_end, |
1139 | .set_page_dirty = gfs2_set_page_dirty, | 1137 | .set_page_dirty = gfs2_set_page_dirty, |
@@ -1151,7 +1149,6 @@ static const struct address_space_operations gfs2_jdata_aops = { | |||
1151 | .writepages = gfs2_jdata_writepages, | 1149 | .writepages = gfs2_jdata_writepages, |
1152 | .readpage = gfs2_readpage, | 1150 | .readpage = gfs2_readpage, |
1153 | .readpages = gfs2_readpages, | 1151 | .readpages = gfs2_readpages, |
1154 | .sync_page = block_sync_page, | ||
1155 | .write_begin = gfs2_write_begin, | 1152 | .write_begin = gfs2_write_begin, |
1156 | .write_end = gfs2_write_end, | 1153 | .write_end = gfs2_write_end, |
1157 | .set_page_dirty = gfs2_set_page_dirty, | 1154 | .set_page_dirty = gfs2_set_page_dirty, |
diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index e7ed31f858dd..5b102c1887fd 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c | |||
@@ -121,7 +121,7 @@ __acquires(&sdp->sd_ail_lock) | |||
121 | lock_buffer(bh); | 121 | lock_buffer(bh); |
122 | if (test_clear_buffer_dirty(bh)) { | 122 | if (test_clear_buffer_dirty(bh)) { |
123 | bh->b_end_io = end_buffer_write_sync; | 123 | bh->b_end_io = end_buffer_write_sync; |
124 | submit_bh(WRITE_SYNC_PLUG, bh); | 124 | submit_bh(WRITE_SYNC, bh); |
125 | } else { | 125 | } else { |
126 | unlock_buffer(bh); | 126 | unlock_buffer(bh); |
127 | brelse(bh); | 127 | brelse(bh); |
@@ -647,7 +647,7 @@ static void gfs2_ordered_write(struct gfs2_sbd *sdp) | |||
647 | lock_buffer(bh); | 647 | lock_buffer(bh); |
648 | if (buffer_mapped(bh) && test_clear_buffer_dirty(bh)) { | 648 | if (buffer_mapped(bh) && test_clear_buffer_dirty(bh)) { |
649 | bh->b_end_io = end_buffer_write_sync; | 649 | bh->b_end_io = end_buffer_write_sync; |
650 | submit_bh(WRITE_SYNC_PLUG, bh); | 650 | submit_bh(WRITE_SYNC, bh); |
651 | } else { | 651 | } else { |
652 | unlock_buffer(bh); | 652 | unlock_buffer(bh); |
653 | brelse(bh); | 653 | brelse(bh); |
diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c index e919abf25ecd..51d27f00ebb4 100644 --- a/fs/gfs2/lops.c +++ b/fs/gfs2/lops.c | |||
@@ -204,7 +204,7 @@ static void buf_lo_before_commit(struct gfs2_sbd *sdp) | |||
204 | } | 204 | } |
205 | 205 | ||
206 | gfs2_log_unlock(sdp); | 206 | gfs2_log_unlock(sdp); |
207 | submit_bh(WRITE_SYNC_PLUG, bh); | 207 | submit_bh(WRITE_SYNC, bh); |
208 | gfs2_log_lock(sdp); | 208 | gfs2_log_lock(sdp); |
209 | 209 | ||
210 | n = 0; | 210 | n = 0; |
@@ -214,7 +214,7 @@ static void buf_lo_before_commit(struct gfs2_sbd *sdp) | |||
214 | gfs2_log_unlock(sdp); | 214 | gfs2_log_unlock(sdp); |
215 | lock_buffer(bd2->bd_bh); | 215 | lock_buffer(bd2->bd_bh); |
216 | bh = gfs2_log_fake_buf(sdp, bd2->bd_bh); | 216 | bh = gfs2_log_fake_buf(sdp, bd2->bd_bh); |
217 | submit_bh(WRITE_SYNC_PLUG, bh); | 217 | submit_bh(WRITE_SYNC, bh); |
218 | gfs2_log_lock(sdp); | 218 | gfs2_log_lock(sdp); |
219 | if (++n >= num) | 219 | if (++n >= num) |
220 | break; | 220 | break; |
@@ -356,7 +356,7 @@ static void revoke_lo_before_commit(struct gfs2_sbd *sdp) | |||
356 | sdp->sd_log_num_revoke--; | 356 | sdp->sd_log_num_revoke--; |
357 | 357 | ||
358 | if (offset + sizeof(u64) > sdp->sd_sb.sb_bsize) { | 358 | if (offset + sizeof(u64) > sdp->sd_sb.sb_bsize) { |
359 | submit_bh(WRITE_SYNC_PLUG, bh); | 359 | submit_bh(WRITE_SYNC, bh); |
360 | 360 | ||
361 | bh = gfs2_log_get_buf(sdp); | 361 | bh = gfs2_log_get_buf(sdp); |
362 | mh = (struct gfs2_meta_header *)bh->b_data; | 362 | mh = (struct gfs2_meta_header *)bh->b_data; |
@@ -373,7 +373,7 @@ static void revoke_lo_before_commit(struct gfs2_sbd *sdp) | |||
373 | } | 373 | } |
374 | gfs2_assert_withdraw(sdp, !sdp->sd_log_num_revoke); | 374 | gfs2_assert_withdraw(sdp, !sdp->sd_log_num_revoke); |
375 | 375 | ||
376 | submit_bh(WRITE_SYNC_PLUG, bh); | 376 | submit_bh(WRITE_SYNC, bh); |
377 | } | 377 | } |
378 | 378 | ||
379 | static void revoke_lo_before_scan(struct gfs2_jdesc *jd, | 379 | static void revoke_lo_before_scan(struct gfs2_jdesc *jd, |
@@ -575,7 +575,7 @@ static void gfs2_write_blocks(struct gfs2_sbd *sdp, struct buffer_head *bh, | |||
575 | ptr = bh_log_ptr(bh); | 575 | ptr = bh_log_ptr(bh); |
576 | 576 | ||
577 | get_bh(bh); | 577 | get_bh(bh); |
578 | submit_bh(WRITE_SYNC_PLUG, bh); | 578 | submit_bh(WRITE_SYNC, bh); |
579 | gfs2_log_lock(sdp); | 579 | gfs2_log_lock(sdp); |
580 | while(!list_empty(list)) { | 580 | while(!list_empty(list)) { |
581 | bd = list_entry(list->next, struct gfs2_bufdata, bd_le.le_list); | 581 | bd = list_entry(list->next, struct gfs2_bufdata, bd_le.le_list); |
@@ -601,7 +601,7 @@ static void gfs2_write_blocks(struct gfs2_sbd *sdp, struct buffer_head *bh, | |||
601 | } else { | 601 | } else { |
602 | bh1 = gfs2_log_fake_buf(sdp, bd->bd_bh); | 602 | bh1 = gfs2_log_fake_buf(sdp, bd->bd_bh); |
603 | } | 603 | } |
604 | submit_bh(WRITE_SYNC_PLUG, bh1); | 604 | submit_bh(WRITE_SYNC, bh1); |
605 | gfs2_log_lock(sdp); | 605 | gfs2_log_lock(sdp); |
606 | ptr += 2; | 606 | ptr += 2; |
607 | } | 607 | } |
diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c index 01d97f486553..675349b5a133 100644 --- a/fs/gfs2/meta_io.c +++ b/fs/gfs2/meta_io.c | |||
@@ -37,7 +37,7 @@ static int gfs2_aspace_writepage(struct page *page, struct writeback_control *wb | |||
37 | struct buffer_head *bh, *head; | 37 | struct buffer_head *bh, *head; |
38 | int nr_underway = 0; | 38 | int nr_underway = 0; |
39 | int write_op = REQ_META | | 39 | int write_op = REQ_META | |
40 | (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC_PLUG : WRITE); | 40 | (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE); |
41 | 41 | ||
42 | BUG_ON(!PageLocked(page)); | 42 | BUG_ON(!PageLocked(page)); |
43 | BUG_ON(!page_has_buffers(page)); | 43 | BUG_ON(!page_has_buffers(page)); |
@@ -94,7 +94,6 @@ static int gfs2_aspace_writepage(struct page *page, struct writeback_control *wb | |||
94 | const struct address_space_operations gfs2_meta_aops = { | 94 | const struct address_space_operations gfs2_meta_aops = { |
95 | .writepage = gfs2_aspace_writepage, | 95 | .writepage = gfs2_aspace_writepage, |
96 | .releasepage = gfs2_releasepage, | 96 | .releasepage = gfs2_releasepage, |
97 | .sync_page = block_sync_page, | ||
98 | }; | 97 | }; |
99 | 98 | ||
100 | /** | 99 | /** |
diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c index dffb4e996643..fff16c968e67 100644 --- a/fs/hfs/inode.c +++ b/fs/hfs/inode.c | |||
@@ -150,7 +150,6 @@ static int hfs_writepages(struct address_space *mapping, | |||
150 | const struct address_space_operations hfs_btree_aops = { | 150 | const struct address_space_operations hfs_btree_aops = { |
151 | .readpage = hfs_readpage, | 151 | .readpage = hfs_readpage, |
152 | .writepage = hfs_writepage, | 152 | .writepage = hfs_writepage, |
153 | .sync_page = block_sync_page, | ||
154 | .write_begin = hfs_write_begin, | 153 | .write_begin = hfs_write_begin, |
155 | .write_end = generic_write_end, | 154 | .write_end = generic_write_end, |
156 | .bmap = hfs_bmap, | 155 | .bmap = hfs_bmap, |
@@ -160,7 +159,6 @@ const struct address_space_operations hfs_btree_aops = { | |||
160 | const struct address_space_operations hfs_aops = { | 159 | const struct address_space_operations hfs_aops = { |
161 | .readpage = hfs_readpage, | 160 | .readpage = hfs_readpage, |
162 | .writepage = hfs_writepage, | 161 | .writepage = hfs_writepage, |
163 | .sync_page = block_sync_page, | ||
164 | .write_begin = hfs_write_begin, | 162 | .write_begin = hfs_write_begin, |
165 | .write_end = generic_write_end, | 163 | .write_end = generic_write_end, |
166 | .bmap = hfs_bmap, | 164 | .bmap = hfs_bmap, |
diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index a8df651747f0..b248a6cfcad9 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c | |||
@@ -146,7 +146,6 @@ static int hfsplus_writepages(struct address_space *mapping, | |||
146 | const struct address_space_operations hfsplus_btree_aops = { | 146 | const struct address_space_operations hfsplus_btree_aops = { |
147 | .readpage = hfsplus_readpage, | 147 | .readpage = hfsplus_readpage, |
148 | .writepage = hfsplus_writepage, | 148 | .writepage = hfsplus_writepage, |
149 | .sync_page = block_sync_page, | ||
150 | .write_begin = hfsplus_write_begin, | 149 | .write_begin = hfsplus_write_begin, |
151 | .write_end = generic_write_end, | 150 | .write_end = generic_write_end, |
152 | .bmap = hfsplus_bmap, | 151 | .bmap = hfsplus_bmap, |
@@ -156,7 +155,6 @@ const struct address_space_operations hfsplus_btree_aops = { | |||
156 | const struct address_space_operations hfsplus_aops = { | 155 | const struct address_space_operations hfsplus_aops = { |
157 | .readpage = hfsplus_readpage, | 156 | .readpage = hfsplus_readpage, |
158 | .writepage = hfsplus_writepage, | 157 | .writepage = hfsplus_writepage, |
159 | .sync_page = block_sync_page, | ||
160 | .write_begin = hfsplus_write_begin, | 158 | .write_begin = hfsplus_write_begin, |
161 | .write_end = generic_write_end, | 159 | .write_end = generic_write_end, |
162 | .bmap = hfsplus_bmap, | 160 | .bmap = hfsplus_bmap, |
diff --git a/fs/hpfs/file.c b/fs/hpfs/file.c index 2dbae20450f8..9b9eb6933e43 100644 --- a/fs/hpfs/file.c +++ b/fs/hpfs/file.c | |||
@@ -119,7 +119,6 @@ static sector_t _hpfs_bmap(struct address_space *mapping, sector_t block) | |||
119 | const struct address_space_operations hpfs_aops = { | 119 | const struct address_space_operations hpfs_aops = { |
120 | .readpage = hpfs_readpage, | 120 | .readpage = hpfs_readpage, |
121 | .writepage = hpfs_writepage, | 121 | .writepage = hpfs_writepage, |
122 | .sync_page = block_sync_page, | ||
123 | .write_begin = hpfs_write_begin, | 122 | .write_begin = hpfs_write_begin, |
124 | .write_end = generic_write_end, | 123 | .write_end = generic_write_end, |
125 | .bmap = _hpfs_bmap | 124 | .bmap = _hpfs_bmap |
diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c index a0f3833c0dbf..3db5ba4568fc 100644 --- a/fs/isofs/inode.c +++ b/fs/isofs/inode.c | |||
@@ -1158,7 +1158,6 @@ static sector_t _isofs_bmap(struct address_space *mapping, sector_t block) | |||
1158 | 1158 | ||
1159 | static const struct address_space_operations isofs_aops = { | 1159 | static const struct address_space_operations isofs_aops = { |
1160 | .readpage = isofs_readpage, | 1160 | .readpage = isofs_readpage, |
1161 | .sync_page = block_sync_page, | ||
1162 | .bmap = _isofs_bmap | 1161 | .bmap = _isofs_bmap |
1163 | }; | 1162 | }; |
1164 | 1163 | ||
diff --git a/fs/jbd/commit.c b/fs/jbd/commit.c index 34a4861c14b8..da871ee084d3 100644 --- a/fs/jbd/commit.c +++ b/fs/jbd/commit.c | |||
@@ -20,6 +20,7 @@ | |||
20 | #include <linux/mm.h> | 20 | #include <linux/mm.h> |
21 | #include <linux/pagemap.h> | 21 | #include <linux/pagemap.h> |
22 | #include <linux/bio.h> | 22 | #include <linux/bio.h> |
23 | #include <linux/blkdev.h> | ||
23 | 24 | ||
24 | /* | 25 | /* |
25 | * Default IO end handler for temporary BJ_IO buffer_heads. | 26 | * Default IO end handler for temporary BJ_IO buffer_heads. |
@@ -294,7 +295,7 @@ void journal_commit_transaction(journal_t *journal) | |||
294 | int first_tag = 0; | 295 | int first_tag = 0; |
295 | int tag_flag; | 296 | int tag_flag; |
296 | int i; | 297 | int i; |
297 | int write_op = WRITE_SYNC; | 298 | struct blk_plug plug; |
298 | 299 | ||
299 | /* | 300 | /* |
300 | * First job: lock down the current transaction and wait for | 301 | * First job: lock down the current transaction and wait for |
@@ -327,13 +328,6 @@ void journal_commit_transaction(journal_t *journal) | |||
327 | spin_lock(&journal->j_state_lock); | 328 | spin_lock(&journal->j_state_lock); |
328 | commit_transaction->t_state = T_LOCKED; | 329 | commit_transaction->t_state = T_LOCKED; |
329 | 330 | ||
330 | /* | ||
331 | * Use plugged writes here, since we want to submit several before | ||
332 | * we unplug the device. We don't do explicit unplugging in here, | ||
333 | * instead we rely on sync_buffer() doing the unplug for us. | ||
334 | */ | ||
335 | if (commit_transaction->t_synchronous_commit) | ||
336 | write_op = WRITE_SYNC_PLUG; | ||
337 | spin_lock(&commit_transaction->t_handle_lock); | 331 | spin_lock(&commit_transaction->t_handle_lock); |
338 | while (commit_transaction->t_updates) { | 332 | while (commit_transaction->t_updates) { |
339 | DEFINE_WAIT(wait); | 333 | DEFINE_WAIT(wait); |
@@ -418,8 +412,10 @@ void journal_commit_transaction(journal_t *journal) | |||
418 | * Now start flushing things to disk, in the order they appear | 412 | * Now start flushing things to disk, in the order they appear |
419 | * on the transaction lists. Data blocks go first. | 413 | * on the transaction lists. Data blocks go first. |
420 | */ | 414 | */ |
415 | blk_start_plug(&plug); | ||
421 | err = journal_submit_data_buffers(journal, commit_transaction, | 416 | err = journal_submit_data_buffers(journal, commit_transaction, |
422 | write_op); | 417 | WRITE_SYNC); |
418 | blk_finish_plug(&plug); | ||
423 | 419 | ||
424 | /* | 420 | /* |
425 | * Wait for all previously submitted IO to complete. | 421 | * Wait for all previously submitted IO to complete. |
@@ -480,7 +476,9 @@ void journal_commit_transaction(journal_t *journal) | |||
480 | err = 0; | 476 | err = 0; |
481 | } | 477 | } |
482 | 478 | ||
483 | journal_write_revoke_records(journal, commit_transaction, write_op); | 479 | blk_start_plug(&plug); |
480 | |||
481 | journal_write_revoke_records(journal, commit_transaction, WRITE_SYNC); | ||
484 | 482 | ||
485 | /* | 483 | /* |
486 | * If we found any dirty or locked buffers, then we should have | 484 | * If we found any dirty or locked buffers, then we should have |
@@ -650,7 +648,7 @@ start_journal_io: | |||
650 | clear_buffer_dirty(bh); | 648 | clear_buffer_dirty(bh); |
651 | set_buffer_uptodate(bh); | 649 | set_buffer_uptodate(bh); |
652 | bh->b_end_io = journal_end_buffer_io_sync; | 650 | bh->b_end_io = journal_end_buffer_io_sync; |
653 | submit_bh(write_op, bh); | 651 | submit_bh(WRITE_SYNC, bh); |
654 | } | 652 | } |
655 | cond_resched(); | 653 | cond_resched(); |
656 | 654 | ||
@@ -661,6 +659,8 @@ start_journal_io: | |||
661 | } | 659 | } |
662 | } | 660 | } |
663 | 661 | ||
662 | blk_finish_plug(&plug); | ||
663 | |||
664 | /* Lo and behold: we have just managed to send a transaction to | 664 | /* Lo and behold: we have just managed to send a transaction to |
665 | the log. Before we can commit it, wait for the IO so far to | 665 | the log. Before we can commit it, wait for the IO so far to |
666 | complete. Control buffers being written are on the | 666 | complete. Control buffers being written are on the |
diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c index f3ad1598b201..fa36d7662b21 100644 --- a/fs/jbd2/commit.c +++ b/fs/jbd2/commit.c | |||
@@ -137,9 +137,9 @@ static int journal_submit_commit_record(journal_t *journal, | |||
137 | if (journal->j_flags & JBD2_BARRIER && | 137 | if (journal->j_flags & JBD2_BARRIER && |
138 | !JBD2_HAS_INCOMPAT_FEATURE(journal, | 138 | !JBD2_HAS_INCOMPAT_FEATURE(journal, |
139 | JBD2_FEATURE_INCOMPAT_ASYNC_COMMIT)) | 139 | JBD2_FEATURE_INCOMPAT_ASYNC_COMMIT)) |
140 | ret = submit_bh(WRITE_SYNC_PLUG | WRITE_FLUSH_FUA, bh); | 140 | ret = submit_bh(WRITE_SYNC | WRITE_FLUSH_FUA, bh); |
141 | else | 141 | else |
142 | ret = submit_bh(WRITE_SYNC_PLUG, bh); | 142 | ret = submit_bh(WRITE_SYNC, bh); |
143 | 143 | ||
144 | *cbh = bh; | 144 | *cbh = bh; |
145 | return ret; | 145 | return ret; |
@@ -329,7 +329,7 @@ void jbd2_journal_commit_transaction(journal_t *journal) | |||
329 | int tag_bytes = journal_tag_bytes(journal); | 329 | int tag_bytes = journal_tag_bytes(journal); |
330 | struct buffer_head *cbh = NULL; /* For transactional checksums */ | 330 | struct buffer_head *cbh = NULL; /* For transactional checksums */ |
331 | __u32 crc32_sum = ~0; | 331 | __u32 crc32_sum = ~0; |
332 | int write_op = WRITE_SYNC; | 332 | struct blk_plug plug; |
333 | 333 | ||
334 | /* | 334 | /* |
335 | * First job: lock down the current transaction and wait for | 335 | * First job: lock down the current transaction and wait for |
@@ -363,13 +363,6 @@ void jbd2_journal_commit_transaction(journal_t *journal) | |||
363 | write_lock(&journal->j_state_lock); | 363 | write_lock(&journal->j_state_lock); |
364 | commit_transaction->t_state = T_LOCKED; | 364 | commit_transaction->t_state = T_LOCKED; |
365 | 365 | ||
366 | /* | ||
367 | * Use plugged writes here, since we want to submit several before | ||
368 | * we unplug the device. We don't do explicit unplugging in here, | ||
369 | * instead we rely on sync_buffer() doing the unplug for us. | ||
370 | */ | ||
371 | if (commit_transaction->t_synchronous_commit) | ||
372 | write_op = WRITE_SYNC_PLUG; | ||
373 | trace_jbd2_commit_locking(journal, commit_transaction); | 366 | trace_jbd2_commit_locking(journal, commit_transaction); |
374 | stats.run.rs_wait = commit_transaction->t_max_wait; | 367 | stats.run.rs_wait = commit_transaction->t_max_wait; |
375 | stats.run.rs_locked = jiffies; | 368 | stats.run.rs_locked = jiffies; |
@@ -469,8 +462,10 @@ void jbd2_journal_commit_transaction(journal_t *journal) | |||
469 | if (err) | 462 | if (err) |
470 | jbd2_journal_abort(journal, err); | 463 | jbd2_journal_abort(journal, err); |
471 | 464 | ||
465 | blk_start_plug(&plug); | ||
472 | jbd2_journal_write_revoke_records(journal, commit_transaction, | 466 | jbd2_journal_write_revoke_records(journal, commit_transaction, |
473 | write_op); | 467 | WRITE_SYNC); |
468 | blk_finish_plug(&plug); | ||
474 | 469 | ||
475 | jbd_debug(3, "JBD: commit phase 2\n"); | 470 | jbd_debug(3, "JBD: commit phase 2\n"); |
476 | 471 | ||
@@ -497,6 +492,7 @@ void jbd2_journal_commit_transaction(journal_t *journal) | |||
497 | err = 0; | 492 | err = 0; |
498 | descriptor = NULL; | 493 | descriptor = NULL; |
499 | bufs = 0; | 494 | bufs = 0; |
495 | blk_start_plug(&plug); | ||
500 | while (commit_transaction->t_buffers) { | 496 | while (commit_transaction->t_buffers) { |
501 | 497 | ||
502 | /* Find the next buffer to be journaled... */ | 498 | /* Find the next buffer to be journaled... */ |
@@ -658,7 +654,7 @@ start_journal_io: | |||
658 | clear_buffer_dirty(bh); | 654 | clear_buffer_dirty(bh); |
659 | set_buffer_uptodate(bh); | 655 | set_buffer_uptodate(bh); |
660 | bh->b_end_io = journal_end_buffer_io_sync; | 656 | bh->b_end_io = journal_end_buffer_io_sync; |
661 | submit_bh(write_op, bh); | 657 | submit_bh(WRITE_SYNC, bh); |
662 | } | 658 | } |
663 | cond_resched(); | 659 | cond_resched(); |
664 | stats.run.rs_blocks_logged += bufs; | 660 | stats.run.rs_blocks_logged += bufs; |
@@ -699,6 +695,8 @@ start_journal_io: | |||
699 | __jbd2_journal_abort_hard(journal); | 695 | __jbd2_journal_abort_hard(journal); |
700 | } | 696 | } |
701 | 697 | ||
698 | blk_finish_plug(&plug); | ||
699 | |||
702 | /* Lo and behold: we have just managed to send a transaction to | 700 | /* Lo and behold: we have just managed to send a transaction to |
703 | the log. Before we can commit it, wait for the IO so far to | 701 | the log. Before we can commit it, wait for the IO so far to |
704 | complete. Control buffers being written are on the | 702 | complete. Control buffers being written are on the |
diff --git a/fs/jfs/inode.c b/fs/jfs/inode.c index 9978803ceedc..eddbb373209e 100644 --- a/fs/jfs/inode.c +++ b/fs/jfs/inode.c | |||
@@ -352,7 +352,6 @@ const struct address_space_operations jfs_aops = { | |||
352 | .readpages = jfs_readpages, | 352 | .readpages = jfs_readpages, |
353 | .writepage = jfs_writepage, | 353 | .writepage = jfs_writepage, |
354 | .writepages = jfs_writepages, | 354 | .writepages = jfs_writepages, |
355 | .sync_page = block_sync_page, | ||
356 | .write_begin = jfs_write_begin, | 355 | .write_begin = jfs_write_begin, |
357 | .write_end = nobh_write_end, | 356 | .write_end = nobh_write_end, |
358 | .bmap = jfs_bmap, | 357 | .bmap = jfs_bmap, |
diff --git a/fs/jfs/jfs_metapage.c b/fs/jfs/jfs_metapage.c index 48b44bd8267b..6740d34cd82b 100644 --- a/fs/jfs/jfs_metapage.c +++ b/fs/jfs/jfs_metapage.c | |||
@@ -583,7 +583,6 @@ static void metapage_invalidatepage(struct page *page, unsigned long offset) | |||
583 | const struct address_space_operations jfs_metapage_aops = { | 583 | const struct address_space_operations jfs_metapage_aops = { |
584 | .readpage = metapage_readpage, | 584 | .readpage = metapage_readpage, |
585 | .writepage = metapage_writepage, | 585 | .writepage = metapage_writepage, |
586 | .sync_page = block_sync_page, | ||
587 | .releasepage = metapage_releasepage, | 586 | .releasepage = metapage_releasepage, |
588 | .invalidatepage = metapage_invalidatepage, | 587 | .invalidatepage = metapage_invalidatepage, |
589 | .set_page_dirty = __set_page_dirty_nobuffers, | 588 | .set_page_dirty = __set_page_dirty_nobuffers, |
diff --git a/fs/logfs/dev_bdev.c b/fs/logfs/dev_bdev.c index 723bc5bca09a..1adc8d455f0e 100644 --- a/fs/logfs/dev_bdev.c +++ b/fs/logfs/dev_bdev.c | |||
@@ -39,7 +39,6 @@ static int sync_request(struct page *page, struct block_device *bdev, int rw) | |||
39 | bio.bi_end_io = request_complete; | 39 | bio.bi_end_io = request_complete; |
40 | 40 | ||
41 | submit_bio(rw, &bio); | 41 | submit_bio(rw, &bio); |
42 | generic_unplug_device(bdev_get_queue(bdev)); | ||
43 | wait_for_completion(&complete); | 42 | wait_for_completion(&complete); |
44 | return test_bit(BIO_UPTODATE, &bio.bi_flags) ? 0 : -EIO; | 43 | return test_bit(BIO_UPTODATE, &bio.bi_flags) ? 0 : -EIO; |
45 | } | 44 | } |
@@ -168,7 +167,6 @@ static void bdev_writeseg(struct super_block *sb, u64 ofs, size_t len) | |||
168 | } | 167 | } |
169 | len = PAGE_ALIGN(len); | 168 | len = PAGE_ALIGN(len); |
170 | __bdev_writeseg(sb, ofs, ofs >> PAGE_SHIFT, len >> PAGE_SHIFT); | 169 | __bdev_writeseg(sb, ofs, ofs >> PAGE_SHIFT, len >> PAGE_SHIFT); |
171 | generic_unplug_device(bdev_get_queue(logfs_super(sb)->s_bdev)); | ||
172 | } | 170 | } |
173 | 171 | ||
174 | 172 | ||
diff --git a/fs/minix/inode.c b/fs/minix/inode.c index ae0b83f476a6..adcdc0a4e182 100644 --- a/fs/minix/inode.c +++ b/fs/minix/inode.c | |||
@@ -399,7 +399,6 @@ static sector_t minix_bmap(struct address_space *mapping, sector_t block) | |||
399 | static const struct address_space_operations minix_aops = { | 399 | static const struct address_space_operations minix_aops = { |
400 | .readpage = minix_readpage, | 400 | .readpage = minix_readpage, |
401 | .writepage = minix_writepage, | 401 | .writepage = minix_writepage, |
402 | .sync_page = block_sync_page, | ||
403 | .write_begin = minix_write_begin, | 402 | .write_begin = minix_write_begin, |
404 | .write_end = generic_write_end, | 403 | .write_end = generic_write_end, |
405 | .bmap = minix_bmap | 404 | .bmap = minix_bmap |
diff --git a/fs/mpage.c b/fs/mpage.c index d78455a81ec9..0afc809e46e0 100644 --- a/fs/mpage.c +++ b/fs/mpage.c | |||
@@ -364,6 +364,9 @@ mpage_readpages(struct address_space *mapping, struct list_head *pages, | |||
364 | sector_t last_block_in_bio = 0; | 364 | sector_t last_block_in_bio = 0; |
365 | struct buffer_head map_bh; | 365 | struct buffer_head map_bh; |
366 | unsigned long first_logical_block = 0; | 366 | unsigned long first_logical_block = 0; |
367 | struct blk_plug plug; | ||
368 | |||
369 | blk_start_plug(&plug); | ||
367 | 370 | ||
368 | map_bh.b_state = 0; | 371 | map_bh.b_state = 0; |
369 | map_bh.b_size = 0; | 372 | map_bh.b_size = 0; |
@@ -385,6 +388,7 @@ mpage_readpages(struct address_space *mapping, struct list_head *pages, | |||
385 | BUG_ON(!list_empty(pages)); | 388 | BUG_ON(!list_empty(pages)); |
386 | if (bio) | 389 | if (bio) |
387 | mpage_bio_submit(READ, bio); | 390 | mpage_bio_submit(READ, bio); |
391 | blk_finish_plug(&plug); | ||
388 | return 0; | 392 | return 0; |
389 | } | 393 | } |
390 | EXPORT_SYMBOL(mpage_readpages); | 394 | EXPORT_SYMBOL(mpage_readpages); |
@@ -666,8 +670,11 @@ int | |||
666 | mpage_writepages(struct address_space *mapping, | 670 | mpage_writepages(struct address_space *mapping, |
667 | struct writeback_control *wbc, get_block_t get_block) | 671 | struct writeback_control *wbc, get_block_t get_block) |
668 | { | 672 | { |
673 | struct blk_plug plug; | ||
669 | int ret; | 674 | int ret; |
670 | 675 | ||
676 | blk_start_plug(&plug); | ||
677 | |||
671 | if (!get_block) | 678 | if (!get_block) |
672 | ret = generic_writepages(mapping, wbc); | 679 | ret = generic_writepages(mapping, wbc); |
673 | else { | 680 | else { |
@@ -682,6 +689,7 @@ mpage_writepages(struct address_space *mapping, | |||
682 | if (mpd.bio) | 689 | if (mpd.bio) |
683 | mpage_bio_submit(WRITE, mpd.bio); | 690 | mpage_bio_submit(WRITE, mpd.bio); |
684 | } | 691 | } |
692 | blk_finish_plug(&plug); | ||
685 | return ret; | 693 | return ret; |
686 | } | 694 | } |
687 | EXPORT_SYMBOL(mpage_writepages); | 695 | EXPORT_SYMBOL(mpage_writepages); |
diff --git a/fs/nilfs2/btnode.c b/fs/nilfs2/btnode.c index 85f7baa15f5d..609cd223eea8 100644 --- a/fs/nilfs2/btnode.c +++ b/fs/nilfs2/btnode.c | |||
@@ -34,15 +34,10 @@ | |||
34 | #include "page.h" | 34 | #include "page.h" |
35 | #include "btnode.h" | 35 | #include "btnode.h" |
36 | 36 | ||
37 | |||
38 | static const struct address_space_operations def_btnode_aops = { | ||
39 | .sync_page = block_sync_page, | ||
40 | }; | ||
41 | |||
42 | void nilfs_btnode_cache_init(struct address_space *btnc, | 37 | void nilfs_btnode_cache_init(struct address_space *btnc, |
43 | struct backing_dev_info *bdi) | 38 | struct backing_dev_info *bdi) |
44 | { | 39 | { |
45 | nilfs_mapping_init(btnc, bdi, &def_btnode_aops); | 40 | nilfs_mapping_init(btnc, bdi); |
46 | } | 41 | } |
47 | 42 | ||
48 | void nilfs_btnode_cache_clear(struct address_space *btnc) | 43 | void nilfs_btnode_cache_clear(struct address_space *btnc) |
diff --git a/fs/nilfs2/gcinode.c b/fs/nilfs2/gcinode.c index caf9a6a3fb54..1c2a3e23f8b2 100644 --- a/fs/nilfs2/gcinode.c +++ b/fs/nilfs2/gcinode.c | |||
@@ -49,7 +49,6 @@ | |||
49 | #include "ifile.h" | 49 | #include "ifile.h" |
50 | 50 | ||
51 | static const struct address_space_operations def_gcinode_aops = { | 51 | static const struct address_space_operations def_gcinode_aops = { |
52 | .sync_page = block_sync_page, | ||
53 | }; | 52 | }; |
54 | 53 | ||
55 | /* | 54 | /* |
diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c index d5625be236a8..c0aa27490c02 100644 --- a/fs/nilfs2/inode.c +++ b/fs/nilfs2/inode.c | |||
@@ -280,7 +280,6 @@ nilfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, | |||
280 | const struct address_space_operations nilfs_aops = { | 280 | const struct address_space_operations nilfs_aops = { |
281 | .writepage = nilfs_writepage, | 281 | .writepage = nilfs_writepage, |
282 | .readpage = nilfs_readpage, | 282 | .readpage = nilfs_readpage, |
283 | .sync_page = block_sync_page, | ||
284 | .writepages = nilfs_writepages, | 283 | .writepages = nilfs_writepages, |
285 | .set_page_dirty = nilfs_set_page_dirty, | 284 | .set_page_dirty = nilfs_set_page_dirty, |
286 | .readpages = nilfs_readpages, | 285 | .readpages = nilfs_readpages, |
diff --git a/fs/nilfs2/mdt.c b/fs/nilfs2/mdt.c index a0babd2bff6a..a649b05f7069 100644 --- a/fs/nilfs2/mdt.c +++ b/fs/nilfs2/mdt.c | |||
@@ -399,7 +399,6 @@ nilfs_mdt_write_page(struct page *page, struct writeback_control *wbc) | |||
399 | 399 | ||
400 | static const struct address_space_operations def_mdt_aops = { | 400 | static const struct address_space_operations def_mdt_aops = { |
401 | .writepage = nilfs_mdt_write_page, | 401 | .writepage = nilfs_mdt_write_page, |
402 | .sync_page = block_sync_page, | ||
403 | }; | 402 | }; |
404 | 403 | ||
405 | static const struct inode_operations def_mdt_iops; | 404 | static const struct inode_operations def_mdt_iops; |
@@ -438,10 +437,6 @@ void nilfs_mdt_set_entry_size(struct inode *inode, unsigned entry_size, | |||
438 | mi->mi_first_entry_offset = DIV_ROUND_UP(header_size, entry_size); | 437 | mi->mi_first_entry_offset = DIV_ROUND_UP(header_size, entry_size); |
439 | } | 438 | } |
440 | 439 | ||
441 | static const struct address_space_operations shadow_map_aops = { | ||
442 | .sync_page = block_sync_page, | ||
443 | }; | ||
444 | |||
445 | /** | 440 | /** |
446 | * nilfs_mdt_setup_shadow_map - setup shadow map and bind it to metadata file | 441 | * nilfs_mdt_setup_shadow_map - setup shadow map and bind it to metadata file |
447 | * @inode: inode of the metadata file | 442 | * @inode: inode of the metadata file |
@@ -455,9 +450,9 @@ int nilfs_mdt_setup_shadow_map(struct inode *inode, | |||
455 | 450 | ||
456 | INIT_LIST_HEAD(&shadow->frozen_buffers); | 451 | INIT_LIST_HEAD(&shadow->frozen_buffers); |
457 | address_space_init_once(&shadow->frozen_data); | 452 | address_space_init_once(&shadow->frozen_data); |
458 | nilfs_mapping_init(&shadow->frozen_data, bdi, &shadow_map_aops); | 453 | nilfs_mapping_init(&shadow->frozen_data, bdi); |
459 | address_space_init_once(&shadow->frozen_btnodes); | 454 | address_space_init_once(&shadow->frozen_btnodes); |
460 | nilfs_mapping_init(&shadow->frozen_btnodes, bdi, &shadow_map_aops); | 455 | nilfs_mapping_init(&shadow->frozen_btnodes, bdi); |
461 | mi->mi_shadow = shadow; | 456 | mi->mi_shadow = shadow; |
462 | return 0; | 457 | return 0; |
463 | } | 458 | } |
diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c index a585b35fd6bc..4d2a1ee0eb47 100644 --- a/fs/nilfs2/page.c +++ b/fs/nilfs2/page.c | |||
@@ -493,15 +493,14 @@ unsigned nilfs_page_count_clean_buffers(struct page *page, | |||
493 | } | 493 | } |
494 | 494 | ||
495 | void nilfs_mapping_init(struct address_space *mapping, | 495 | void nilfs_mapping_init(struct address_space *mapping, |
496 | struct backing_dev_info *bdi, | 496 | struct backing_dev_info *bdi) |
497 | const struct address_space_operations *aops) | ||
498 | { | 497 | { |
499 | mapping->host = NULL; | 498 | mapping->host = NULL; |
500 | mapping->flags = 0; | 499 | mapping->flags = 0; |
501 | mapping_set_gfp_mask(mapping, GFP_NOFS); | 500 | mapping_set_gfp_mask(mapping, GFP_NOFS); |
502 | mapping->assoc_mapping = NULL; | 501 | mapping->assoc_mapping = NULL; |
503 | mapping->backing_dev_info = bdi; | 502 | mapping->backing_dev_info = bdi; |
504 | mapping->a_ops = aops; | 503 | mapping->a_ops = NULL; |
505 | } | 504 | } |
506 | 505 | ||
507 | /* | 506 | /* |
diff --git a/fs/nilfs2/page.h b/fs/nilfs2/page.h index 2a00953ebd5f..f06b79ad7493 100644 --- a/fs/nilfs2/page.h +++ b/fs/nilfs2/page.h | |||
@@ -62,8 +62,7 @@ int nilfs_copy_dirty_pages(struct address_space *, struct address_space *); | |||
62 | void nilfs_copy_back_pages(struct address_space *, struct address_space *); | 62 | void nilfs_copy_back_pages(struct address_space *, struct address_space *); |
63 | void nilfs_clear_dirty_pages(struct address_space *); | 63 | void nilfs_clear_dirty_pages(struct address_space *); |
64 | void nilfs_mapping_init(struct address_space *mapping, | 64 | void nilfs_mapping_init(struct address_space *mapping, |
65 | struct backing_dev_info *bdi, | 65 | struct backing_dev_info *bdi); |
66 | const struct address_space_operations *aops); | ||
67 | unsigned nilfs_page_count_clean_buffers(struct page *, unsigned, unsigned); | 66 | unsigned nilfs_page_count_clean_buffers(struct page *, unsigned, unsigned); |
68 | unsigned long nilfs_find_uncommitted_extent(struct inode *inode, | 67 | unsigned long nilfs_find_uncommitted_extent(struct inode *inode, |
69 | sector_t start_blk, | 68 | sector_t start_blk, |
diff --git a/fs/nilfs2/segbuf.c b/fs/nilfs2/segbuf.c index 0f83e93935b2..2853ff20f85a 100644 --- a/fs/nilfs2/segbuf.c +++ b/fs/nilfs2/segbuf.c | |||
@@ -509,7 +509,7 @@ static int nilfs_segbuf_write(struct nilfs_segment_buffer *segbuf, | |||
509 | * Last BIO is always sent through the following | 509 | * Last BIO is always sent through the following |
510 | * submission. | 510 | * submission. |
511 | */ | 511 | */ |
512 | rw |= REQ_SYNC | REQ_UNPLUG; | 512 | rw |= REQ_SYNC; |
513 | res = nilfs_segbuf_submit_bio(segbuf, &wi, rw); | 513 | res = nilfs_segbuf_submit_bio(segbuf, &wi, rw); |
514 | } | 514 | } |
515 | 515 | ||
diff --git a/fs/ntfs/aops.c b/fs/ntfs/aops.c index c3c2c7ac9020..0b1e885b8cf8 100644 --- a/fs/ntfs/aops.c +++ b/fs/ntfs/aops.c | |||
@@ -1543,8 +1543,6 @@ err_out: | |||
1543 | */ | 1543 | */ |
1544 | const struct address_space_operations ntfs_aops = { | 1544 | const struct address_space_operations ntfs_aops = { |
1545 | .readpage = ntfs_readpage, /* Fill page with data. */ | 1545 | .readpage = ntfs_readpage, /* Fill page with data. */ |
1546 | .sync_page = block_sync_page, /* Currently, just unplugs the | ||
1547 | disk request queue. */ | ||
1548 | #ifdef NTFS_RW | 1546 | #ifdef NTFS_RW |
1549 | .writepage = ntfs_writepage, /* Write dirty page to disk. */ | 1547 | .writepage = ntfs_writepage, /* Write dirty page to disk. */ |
1550 | #endif /* NTFS_RW */ | 1548 | #endif /* NTFS_RW */ |
@@ -1560,8 +1558,6 @@ const struct address_space_operations ntfs_aops = { | |||
1560 | */ | 1558 | */ |
1561 | const struct address_space_operations ntfs_mst_aops = { | 1559 | const struct address_space_operations ntfs_mst_aops = { |
1562 | .readpage = ntfs_readpage, /* Fill page with data. */ | 1560 | .readpage = ntfs_readpage, /* Fill page with data. */ |
1563 | .sync_page = block_sync_page, /* Currently, just unplugs the | ||
1564 | disk request queue. */ | ||
1565 | #ifdef NTFS_RW | 1561 | #ifdef NTFS_RW |
1566 | .writepage = ntfs_writepage, /* Write dirty page to disk. */ | 1562 | .writepage = ntfs_writepage, /* Write dirty page to disk. */ |
1567 | .set_page_dirty = __set_page_dirty_nobuffers, /* Set the page dirty | 1563 | .set_page_dirty = __set_page_dirty_nobuffers, /* Set the page dirty |
diff --git a/fs/ntfs/compress.c b/fs/ntfs/compress.c index 6551c7cbad92..ef9ed854255c 100644 --- a/fs/ntfs/compress.c +++ b/fs/ntfs/compress.c | |||
@@ -698,8 +698,7 @@ lock_retry_remap: | |||
698 | "uptodate! Unplugging the disk queue " | 698 | "uptodate! Unplugging the disk queue " |
699 | "and rescheduling."); | 699 | "and rescheduling."); |
700 | get_bh(tbh); | 700 | get_bh(tbh); |
701 | blk_run_address_space(mapping); | 701 | io_schedule(); |
702 | schedule(); | ||
703 | put_bh(tbh); | 702 | put_bh(tbh); |
704 | if (unlikely(!buffer_uptodate(tbh))) | 703 | if (unlikely(!buffer_uptodate(tbh))) |
705 | goto read_err; | 704 | goto read_err; |
diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c index 1fbb0e20131b..daea0359e974 100644 --- a/fs/ocfs2/aops.c +++ b/fs/ocfs2/aops.c | |||
@@ -2043,7 +2043,6 @@ const struct address_space_operations ocfs2_aops = { | |||
2043 | .write_begin = ocfs2_write_begin, | 2043 | .write_begin = ocfs2_write_begin, |
2044 | .write_end = ocfs2_write_end, | 2044 | .write_end = ocfs2_write_end, |
2045 | .bmap = ocfs2_bmap, | 2045 | .bmap = ocfs2_bmap, |
2046 | .sync_page = block_sync_page, | ||
2047 | .direct_IO = ocfs2_direct_IO, | 2046 | .direct_IO = ocfs2_direct_IO, |
2048 | .invalidatepage = ocfs2_invalidatepage, | 2047 | .invalidatepage = ocfs2_invalidatepage, |
2049 | .releasepage = ocfs2_releasepage, | 2048 | .releasepage = ocfs2_releasepage, |
diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c index b108e863d8f6..1adab287bd24 100644 --- a/fs/ocfs2/cluster/heartbeat.c +++ b/fs/ocfs2/cluster/heartbeat.c | |||
@@ -367,11 +367,7 @@ static inline void o2hb_bio_wait_dec(struct o2hb_bio_wait_ctxt *wc, | |||
367 | static void o2hb_wait_on_io(struct o2hb_region *reg, | 367 | static void o2hb_wait_on_io(struct o2hb_region *reg, |
368 | struct o2hb_bio_wait_ctxt *wc) | 368 | struct o2hb_bio_wait_ctxt *wc) |
369 | { | 369 | { |
370 | struct address_space *mapping = reg->hr_bdev->bd_inode->i_mapping; | ||
371 | |||
372 | blk_run_address_space(mapping); | ||
373 | o2hb_bio_wait_dec(wc, 1); | 370 | o2hb_bio_wait_dec(wc, 1); |
374 | |||
375 | wait_for_completion(&wc->wc_io_complete); | 371 | wait_for_completion(&wc->wc_io_complete); |
376 | } | 372 | } |
377 | 373 | ||
diff --git a/fs/omfs/file.c b/fs/omfs/file.c index 8a6d34fa668a..d738a7e493dd 100644 --- a/fs/omfs/file.c +++ b/fs/omfs/file.c | |||
@@ -372,7 +372,6 @@ const struct address_space_operations omfs_aops = { | |||
372 | .readpages = omfs_readpages, | 372 | .readpages = omfs_readpages, |
373 | .writepage = omfs_writepage, | 373 | .writepage = omfs_writepage, |
374 | .writepages = omfs_writepages, | 374 | .writepages = omfs_writepages, |
375 | .sync_page = block_sync_page, | ||
376 | .write_begin = omfs_write_begin, | 375 | .write_begin = omfs_write_begin, |
377 | .write_end = generic_write_end, | 376 | .write_end = generic_write_end, |
378 | .bmap = omfs_bmap, | 377 | .bmap = omfs_bmap, |
diff --git a/fs/partitions/check.c b/fs/partitions/check.c index 9c21119512b9..ac546975031f 100644 --- a/fs/partitions/check.c +++ b/fs/partitions/check.c | |||
@@ -290,7 +290,8 @@ ssize_t part_inflight_show(struct device *dev, | |||
290 | { | 290 | { |
291 | struct hd_struct *p = dev_to_part(dev); | 291 | struct hd_struct *p = dev_to_part(dev); |
292 | 292 | ||
293 | return sprintf(buf, "%8u %8u\n", p->in_flight[0], p->in_flight[1]); | 293 | return sprintf(buf, "%8u %8u\n", atomic_read(&p->in_flight[0]), |
294 | atomic_read(&p->in_flight[1])); | ||
294 | } | 295 | } |
295 | 296 | ||
296 | #ifdef CONFIG_FAIL_MAKE_REQUEST | 297 | #ifdef CONFIG_FAIL_MAKE_REQUEST |
diff --git a/fs/qnx4/inode.c b/fs/qnx4/inode.c index e63b4171d583..2b0646613f5a 100644 --- a/fs/qnx4/inode.c +++ b/fs/qnx4/inode.c | |||
@@ -335,7 +335,6 @@ static sector_t qnx4_bmap(struct address_space *mapping, sector_t block) | |||
335 | static const struct address_space_operations qnx4_aops = { | 335 | static const struct address_space_operations qnx4_aops = { |
336 | .readpage = qnx4_readpage, | 336 | .readpage = qnx4_readpage, |
337 | .writepage = qnx4_writepage, | 337 | .writepage = qnx4_writepage, |
338 | .sync_page = block_sync_page, | ||
339 | .write_begin = qnx4_write_begin, | 338 | .write_begin = qnx4_write_begin, |
340 | .write_end = generic_write_end, | 339 | .write_end = generic_write_end, |
341 | .bmap = qnx4_bmap | 340 | .bmap = qnx4_bmap |
diff --git a/fs/reiserfs/inode.c b/fs/reiserfs/inode.c index 1bba24bad820..4fd5bb33dbb5 100644 --- a/fs/reiserfs/inode.c +++ b/fs/reiserfs/inode.c | |||
@@ -3217,7 +3217,6 @@ const struct address_space_operations reiserfs_address_space_operations = { | |||
3217 | .readpages = reiserfs_readpages, | 3217 | .readpages = reiserfs_readpages, |
3218 | .releasepage = reiserfs_releasepage, | 3218 | .releasepage = reiserfs_releasepage, |
3219 | .invalidatepage = reiserfs_invalidatepage, | 3219 | .invalidatepage = reiserfs_invalidatepage, |
3220 | .sync_page = block_sync_page, | ||
3221 | .write_begin = reiserfs_write_begin, | 3220 | .write_begin = reiserfs_write_begin, |
3222 | .write_end = reiserfs_write_end, | 3221 | .write_end = reiserfs_write_end, |
3223 | .bmap = reiserfs_aop_bmap, | 3222 | .bmap = reiserfs_aop_bmap, |
diff --git a/fs/super.c b/fs/super.c index e84864908264..8a06881b1920 100644 --- a/fs/super.c +++ b/fs/super.c | |||
@@ -71,6 +71,7 @@ static struct super_block *alloc_super(struct file_system_type *type) | |||
71 | #else | 71 | #else |
72 | INIT_LIST_HEAD(&s->s_files); | 72 | INIT_LIST_HEAD(&s->s_files); |
73 | #endif | 73 | #endif |
74 | s->s_bdi = &default_backing_dev_info; | ||
74 | INIT_LIST_HEAD(&s->s_instances); | 75 | INIT_LIST_HEAD(&s->s_instances); |
75 | INIT_HLIST_BL_HEAD(&s->s_anon); | 76 | INIT_HLIST_BL_HEAD(&s->s_anon); |
76 | INIT_LIST_HEAD(&s->s_inodes); | 77 | INIT_LIST_HEAD(&s->s_inodes); |
@@ -936,6 +937,7 @@ mount_fs(struct file_system_type *type, int flags, const char *name, void *data) | |||
936 | sb = root->d_sb; | 937 | sb = root->d_sb; |
937 | BUG_ON(!sb); | 938 | BUG_ON(!sb); |
938 | WARN_ON(!sb->s_bdi); | 939 | WARN_ON(!sb->s_bdi); |
940 | WARN_ON(sb->s_bdi == &default_backing_dev_info); | ||
939 | sb->s_flags |= MS_BORN; | 941 | sb->s_flags |= MS_BORN; |
940 | 942 | ||
941 | error = security_sb_kern_mount(sb, flags, secdata); | 943 | error = security_sb_kern_mount(sb, flags, secdata); |
@@ -34,7 +34,7 @@ static int __sync_filesystem(struct super_block *sb, int wait) | |||
34 | * This should be safe, as we require bdi backing to actually | 34 | * This should be safe, as we require bdi backing to actually |
35 | * write out data in the first place | 35 | * write out data in the first place |
36 | */ | 36 | */ |
37 | if (!sb->s_bdi || sb->s_bdi == &noop_backing_dev_info) | 37 | if (sb->s_bdi == &noop_backing_dev_info) |
38 | return 0; | 38 | return 0; |
39 | 39 | ||
40 | if (sb->s_qcop && sb->s_qcop->quota_sync) | 40 | if (sb->s_qcop && sb->s_qcop->quota_sync) |
@@ -80,7 +80,7 @@ EXPORT_SYMBOL_GPL(sync_filesystem); | |||
80 | 80 | ||
81 | static void sync_one_sb(struct super_block *sb, void *arg) | 81 | static void sync_one_sb(struct super_block *sb, void *arg) |
82 | { | 82 | { |
83 | if (!(sb->s_flags & MS_RDONLY) && sb->s_bdi) | 83 | if (!(sb->s_flags & MS_RDONLY)) |
84 | __sync_filesystem(sb, *(int *)arg); | 84 | __sync_filesystem(sb, *(int *)arg); |
85 | } | 85 | } |
86 | /* | 86 | /* |
diff --git a/fs/sysv/itree.c b/fs/sysv/itree.c index 9ca66276315e..fa8d43c92bb8 100644 --- a/fs/sysv/itree.c +++ b/fs/sysv/itree.c | |||
@@ -488,7 +488,6 @@ static sector_t sysv_bmap(struct address_space *mapping, sector_t block) | |||
488 | const struct address_space_operations sysv_aops = { | 488 | const struct address_space_operations sysv_aops = { |
489 | .readpage = sysv_readpage, | 489 | .readpage = sysv_readpage, |
490 | .writepage = sysv_writepage, | 490 | .writepage = sysv_writepage, |
491 | .sync_page = block_sync_page, | ||
492 | .write_begin = sysv_write_begin, | 491 | .write_begin = sysv_write_begin, |
493 | .write_end = generic_write_end, | 492 | .write_end = generic_write_end, |
494 | .bmap = sysv_bmap | 493 | .bmap = sysv_bmap |
diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c index e5dc1e120e8d..6ddd9973e681 100644 --- a/fs/ubifs/super.c +++ b/fs/ubifs/super.c | |||
@@ -2011,7 +2011,6 @@ static int ubifs_fill_super(struct super_block *sb, void *data, int silent) | |||
2011 | */ | 2011 | */ |
2012 | c->bdi.name = "ubifs", | 2012 | c->bdi.name = "ubifs", |
2013 | c->bdi.capabilities = BDI_CAP_MAP_COPY; | 2013 | c->bdi.capabilities = BDI_CAP_MAP_COPY; |
2014 | c->bdi.unplug_io_fn = default_unplug_io_fn; | ||
2015 | err = bdi_init(&c->bdi); | 2014 | err = bdi_init(&c->bdi); |
2016 | if (err) | 2015 | if (err) |
2017 | goto out_close; | 2016 | goto out_close; |
diff --git a/fs/udf/file.c b/fs/udf/file.c index f391a2adc699..2a346bb1d9f5 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c | |||
@@ -98,7 +98,6 @@ static int udf_adinicb_write_end(struct file *file, | |||
98 | const struct address_space_operations udf_adinicb_aops = { | 98 | const struct address_space_operations udf_adinicb_aops = { |
99 | .readpage = udf_adinicb_readpage, | 99 | .readpage = udf_adinicb_readpage, |
100 | .writepage = udf_adinicb_writepage, | 100 | .writepage = udf_adinicb_writepage, |
101 | .sync_page = block_sync_page, | ||
102 | .write_begin = simple_write_begin, | 101 | .write_begin = simple_write_begin, |
103 | .write_end = udf_adinicb_write_end, | 102 | .write_end = udf_adinicb_write_end, |
104 | }; | 103 | }; |
diff --git a/fs/udf/inode.c b/fs/udf/inode.c index ccc814321414..1d1358ed80c1 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c | |||
@@ -140,7 +140,6 @@ static sector_t udf_bmap(struct address_space *mapping, sector_t block) | |||
140 | const struct address_space_operations udf_aops = { | 140 | const struct address_space_operations udf_aops = { |
141 | .readpage = udf_readpage, | 141 | .readpage = udf_readpage, |
142 | .writepage = udf_writepage, | 142 | .writepage = udf_writepage, |
143 | .sync_page = block_sync_page, | ||
144 | .write_begin = udf_write_begin, | 143 | .write_begin = udf_write_begin, |
145 | .write_end = generic_write_end, | 144 | .write_end = generic_write_end, |
146 | .bmap = udf_bmap, | 145 | .bmap = udf_bmap, |
diff --git a/fs/ufs/inode.c b/fs/ufs/inode.c index 03c255f12df5..27a4babe7df0 100644 --- a/fs/ufs/inode.c +++ b/fs/ufs/inode.c | |||
@@ -552,7 +552,6 @@ static sector_t ufs_bmap(struct address_space *mapping, sector_t block) | |||
552 | const struct address_space_operations ufs_aops = { | 552 | const struct address_space_operations ufs_aops = { |
553 | .readpage = ufs_readpage, | 553 | .readpage = ufs_readpage, |
554 | .writepage = ufs_writepage, | 554 | .writepage = ufs_writepage, |
555 | .sync_page = block_sync_page, | ||
556 | .write_begin = ufs_write_begin, | 555 | .write_begin = ufs_write_begin, |
557 | .write_end = generic_write_end, | 556 | .write_end = generic_write_end, |
558 | .bmap = ufs_bmap | 557 | .bmap = ufs_bmap |
diff --git a/fs/ufs/truncate.c b/fs/ufs/truncate.c index e56a4f567212..11014302c9ca 100644 --- a/fs/ufs/truncate.c +++ b/fs/ufs/truncate.c | |||
@@ -479,7 +479,7 @@ int ufs_truncate(struct inode *inode, loff_t old_i_size) | |||
479 | break; | 479 | break; |
480 | if (IS_SYNC(inode) && (inode->i_state & I_DIRTY)) | 480 | if (IS_SYNC(inode) && (inode->i_state & I_DIRTY)) |
481 | ufs_sync_inode (inode); | 481 | ufs_sync_inode (inode); |
482 | blk_run_address_space(inode->i_mapping); | 482 | blk_flush_plug(current); |
483 | yield(); | 483 | yield(); |
484 | } | 484 | } |
485 | 485 | ||
diff --git a/fs/xfs/linux-2.6/xfs_aops.c b/fs/xfs/linux-2.6/xfs_aops.c index 8c5c87277456..52dbd14260ba 100644 --- a/fs/xfs/linux-2.6/xfs_aops.c +++ b/fs/xfs/linux-2.6/xfs_aops.c | |||
@@ -413,8 +413,7 @@ xfs_submit_ioend_bio( | |||
413 | if (xfs_ioend_new_eof(ioend)) | 413 | if (xfs_ioend_new_eof(ioend)) |
414 | xfs_mark_inode_dirty(XFS_I(ioend->io_inode)); | 414 | xfs_mark_inode_dirty(XFS_I(ioend->io_inode)); |
415 | 415 | ||
416 | submit_bio(wbc->sync_mode == WB_SYNC_ALL ? | 416 | submit_bio(wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE, bio); |
417 | WRITE_SYNC_PLUG : WRITE, bio); | ||
418 | } | 417 | } |
419 | 418 | ||
420 | STATIC struct bio * | 419 | STATIC struct bio * |
@@ -1495,7 +1494,6 @@ const struct address_space_operations xfs_address_space_operations = { | |||
1495 | .readpages = xfs_vm_readpages, | 1494 | .readpages = xfs_vm_readpages, |
1496 | .writepage = xfs_vm_writepage, | 1495 | .writepage = xfs_vm_writepage, |
1497 | .writepages = xfs_vm_writepages, | 1496 | .writepages = xfs_vm_writepages, |
1498 | .sync_page = block_sync_page, | ||
1499 | .releasepage = xfs_vm_releasepage, | 1497 | .releasepage = xfs_vm_releasepage, |
1500 | .invalidatepage = xfs_vm_invalidatepage, | 1498 | .invalidatepage = xfs_vm_invalidatepage, |
1501 | .write_begin = xfs_vm_write_begin, | 1499 | .write_begin = xfs_vm_write_begin, |
diff --git a/fs/xfs/linux-2.6/xfs_buf.c b/fs/xfs/linux-2.6/xfs_buf.c index 5cb230f2cb4f..c05324d3282c 100644 --- a/fs/xfs/linux-2.6/xfs_buf.c +++ b/fs/xfs/linux-2.6/xfs_buf.c | |||
@@ -990,7 +990,7 @@ xfs_buf_lock( | |||
990 | if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE)) | 990 | if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE)) |
991 | xfs_log_force(bp->b_target->bt_mount, 0); | 991 | xfs_log_force(bp->b_target->bt_mount, 0); |
992 | if (atomic_read(&bp->b_io_remaining)) | 992 | if (atomic_read(&bp->b_io_remaining)) |
993 | blk_run_address_space(bp->b_target->bt_mapping); | 993 | blk_flush_plug(current); |
994 | down(&bp->b_sema); | 994 | down(&bp->b_sema); |
995 | XB_SET_OWNER(bp); | 995 | XB_SET_OWNER(bp); |
996 | 996 | ||
@@ -1034,9 +1034,7 @@ xfs_buf_wait_unpin( | |||
1034 | set_current_state(TASK_UNINTERRUPTIBLE); | 1034 | set_current_state(TASK_UNINTERRUPTIBLE); |
1035 | if (atomic_read(&bp->b_pin_count) == 0) | 1035 | if (atomic_read(&bp->b_pin_count) == 0) |
1036 | break; | 1036 | break; |
1037 | if (atomic_read(&bp->b_io_remaining)) | 1037 | io_schedule(); |
1038 | blk_run_address_space(bp->b_target->bt_mapping); | ||
1039 | schedule(); | ||
1040 | } | 1038 | } |
1041 | remove_wait_queue(&bp->b_waiters, &wait); | 1039 | remove_wait_queue(&bp->b_waiters, &wait); |
1042 | set_current_state(TASK_RUNNING); | 1040 | set_current_state(TASK_RUNNING); |
@@ -1442,7 +1440,7 @@ xfs_buf_iowait( | |||
1442 | trace_xfs_buf_iowait(bp, _RET_IP_); | 1440 | trace_xfs_buf_iowait(bp, _RET_IP_); |
1443 | 1441 | ||
1444 | if (atomic_read(&bp->b_io_remaining)) | 1442 | if (atomic_read(&bp->b_io_remaining)) |
1445 | blk_run_address_space(bp->b_target->bt_mapping); | 1443 | blk_flush_plug(current); |
1446 | wait_for_completion(&bp->b_iowait); | 1444 | wait_for_completion(&bp->b_iowait); |
1447 | 1445 | ||
1448 | trace_xfs_buf_iowait_done(bp, _RET_IP_); | 1446 | trace_xfs_buf_iowait_done(bp, _RET_IP_); |
@@ -1666,7 +1664,6 @@ xfs_mapping_buftarg( | |||
1666 | struct inode *inode; | 1664 | struct inode *inode; |
1667 | struct address_space *mapping; | 1665 | struct address_space *mapping; |
1668 | static const struct address_space_operations mapping_aops = { | 1666 | static const struct address_space_operations mapping_aops = { |
1669 | .sync_page = block_sync_page, | ||
1670 | .migratepage = fail_migrate_page, | 1667 | .migratepage = fail_migrate_page, |
1671 | }; | 1668 | }; |
1672 | 1669 | ||
@@ -1947,7 +1944,7 @@ xfsbufd( | |||
1947 | count++; | 1944 | count++; |
1948 | } | 1945 | } |
1949 | if (count) | 1946 | if (count) |
1950 | blk_run_address_space(target->bt_mapping); | 1947 | blk_flush_plug(current); |
1951 | 1948 | ||
1952 | } while (!kthread_should_stop()); | 1949 | } while (!kthread_should_stop()); |
1953 | 1950 | ||
@@ -1995,7 +1992,7 @@ xfs_flush_buftarg( | |||
1995 | 1992 | ||
1996 | if (wait) { | 1993 | if (wait) { |
1997 | /* Expedite and wait for IO to complete. */ | 1994 | /* Expedite and wait for IO to complete. */ |
1998 | blk_run_address_space(target->bt_mapping); | 1995 | blk_flush_plug(current); |
1999 | while (!list_empty(&wait_list)) { | 1996 | while (!list_empty(&wait_list)) { |
2000 | bp = list_first_entry(&wait_list, struct xfs_buf, b_list); | 1997 | bp = list_first_entry(&wait_list, struct xfs_buf, b_list); |
2001 | 1998 | ||
diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h index 4ce34fa937d4..96f4094b706d 100644 --- a/include/linux/backing-dev.h +++ b/include/linux/backing-dev.h | |||
@@ -66,8 +66,6 @@ struct backing_dev_info { | |||
66 | unsigned int capabilities; /* Device capabilities */ | 66 | unsigned int capabilities; /* Device capabilities */ |
67 | congested_fn *congested_fn; /* Function pointer if device is md/dm */ | 67 | congested_fn *congested_fn; /* Function pointer if device is md/dm */ |
68 | void *congested_data; /* Pointer to aux data for congested func */ | 68 | void *congested_data; /* Pointer to aux data for congested func */ |
69 | void (*unplug_io_fn)(struct backing_dev_info *, struct page *); | ||
70 | void *unplug_io_data; | ||
71 | 69 | ||
72 | char *name; | 70 | char *name; |
73 | 71 | ||
@@ -251,7 +249,6 @@ int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio); | |||
251 | 249 | ||
252 | extern struct backing_dev_info default_backing_dev_info; | 250 | extern struct backing_dev_info default_backing_dev_info; |
253 | extern struct backing_dev_info noop_backing_dev_info; | 251 | extern struct backing_dev_info noop_backing_dev_info; |
254 | void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page); | ||
255 | 252 | ||
256 | int writeback_in_progress(struct backing_dev_info *bdi); | 253 | int writeback_in_progress(struct backing_dev_info *bdi); |
257 | 254 | ||
@@ -336,17 +333,4 @@ static inline int bdi_sched_wait(void *word) | |||
336 | return 0; | 333 | return 0; |
337 | } | 334 | } |
338 | 335 | ||
339 | static inline void blk_run_backing_dev(struct backing_dev_info *bdi, | ||
340 | struct page *page) | ||
341 | { | ||
342 | if (bdi && bdi->unplug_io_fn) | ||
343 | bdi->unplug_io_fn(bdi, page); | ||
344 | } | ||
345 | |||
346 | static inline void blk_run_address_space(struct address_space *mapping) | ||
347 | { | ||
348 | if (mapping) | ||
349 | blk_run_backing_dev(mapping->backing_dev_info, NULL); | ||
350 | } | ||
351 | |||
352 | #endif /* _LINUX_BACKING_DEV_H */ | 336 | #endif /* _LINUX_BACKING_DEV_H */ |
diff --git a/include/linux/bio.h b/include/linux/bio.h index 35dcdb3589bc..ce33e6868a2f 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h | |||
@@ -304,7 +304,6 @@ struct biovec_slab { | |||
304 | }; | 304 | }; |
305 | 305 | ||
306 | extern struct bio_set *fs_bio_set; | 306 | extern struct bio_set *fs_bio_set; |
307 | extern struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly; | ||
308 | 307 | ||
309 | /* | 308 | /* |
310 | * a small number of entries is fine, not going to be performance critical. | 309 | * a small number of entries is fine, not going to be performance critical. |
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 46ad5197537a..be50d9e70a7d 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h | |||
@@ -128,7 +128,6 @@ enum rq_flag_bits { | |||
128 | __REQ_NOIDLE, /* don't anticipate more IO after this one */ | 128 | __REQ_NOIDLE, /* don't anticipate more IO after this one */ |
129 | 129 | ||
130 | /* bio only flags */ | 130 | /* bio only flags */ |
131 | __REQ_UNPLUG, /* unplug the immediately after submission */ | ||
132 | __REQ_RAHEAD, /* read ahead, can fail anytime */ | 131 | __REQ_RAHEAD, /* read ahead, can fail anytime */ |
133 | __REQ_THROTTLED, /* This bio has already been subjected to | 132 | __REQ_THROTTLED, /* This bio has already been subjected to |
134 | * throttling rules. Don't do it again. */ | 133 | * throttling rules. Don't do it again. */ |
@@ -148,9 +147,11 @@ enum rq_flag_bits { | |||
148 | __REQ_ALLOCED, /* request came from our alloc pool */ | 147 | __REQ_ALLOCED, /* request came from our alloc pool */ |
149 | __REQ_COPY_USER, /* contains copies of user pages */ | 148 | __REQ_COPY_USER, /* contains copies of user pages */ |
150 | __REQ_FLUSH, /* request for cache flush */ | 149 | __REQ_FLUSH, /* request for cache flush */ |
150 | __REQ_FLUSH_SEQ, /* request for flush sequence */ | ||
151 | __REQ_IO_STAT, /* account I/O stat */ | 151 | __REQ_IO_STAT, /* account I/O stat */ |
152 | __REQ_MIXED_MERGE, /* merge of different types, fail separately */ | 152 | __REQ_MIXED_MERGE, /* merge of different types, fail separately */ |
153 | __REQ_SECURE, /* secure discard (used with __REQ_DISCARD) */ | 153 | __REQ_SECURE, /* secure discard (used with __REQ_DISCARD) */ |
154 | __REQ_ON_PLUG, /* on plug list */ | ||
154 | __REQ_NR_BITS, /* stops here */ | 155 | __REQ_NR_BITS, /* stops here */ |
155 | }; | 156 | }; |
156 | 157 | ||
@@ -170,7 +171,6 @@ enum rq_flag_bits { | |||
170 | REQ_NOIDLE | REQ_FLUSH | REQ_FUA) | 171 | REQ_NOIDLE | REQ_FLUSH | REQ_FUA) |
171 | #define REQ_CLONE_MASK REQ_COMMON_MASK | 172 | #define REQ_CLONE_MASK REQ_COMMON_MASK |
172 | 173 | ||
173 | #define REQ_UNPLUG (1 << __REQ_UNPLUG) | ||
174 | #define REQ_RAHEAD (1 << __REQ_RAHEAD) | 174 | #define REQ_RAHEAD (1 << __REQ_RAHEAD) |
175 | #define REQ_THROTTLED (1 << __REQ_THROTTLED) | 175 | #define REQ_THROTTLED (1 << __REQ_THROTTLED) |
176 | 176 | ||
@@ -188,8 +188,10 @@ enum rq_flag_bits { | |||
188 | #define REQ_ALLOCED (1 << __REQ_ALLOCED) | 188 | #define REQ_ALLOCED (1 << __REQ_ALLOCED) |
189 | #define REQ_COPY_USER (1 << __REQ_COPY_USER) | 189 | #define REQ_COPY_USER (1 << __REQ_COPY_USER) |
190 | #define REQ_FLUSH (1 << __REQ_FLUSH) | 190 | #define REQ_FLUSH (1 << __REQ_FLUSH) |
191 | #define REQ_FLUSH_SEQ (1 << __REQ_FLUSH_SEQ) | ||
191 | #define REQ_IO_STAT (1 << __REQ_IO_STAT) | 192 | #define REQ_IO_STAT (1 << __REQ_IO_STAT) |
192 | #define REQ_MIXED_MERGE (1 << __REQ_MIXED_MERGE) | 193 | #define REQ_MIXED_MERGE (1 << __REQ_MIXED_MERGE) |
193 | #define REQ_SECURE (1 << __REQ_SECURE) | 194 | #define REQ_SECURE (1 << __REQ_SECURE) |
195 | #define REQ_ON_PLUG (1 << __REQ_ON_PLUG) | ||
194 | 196 | ||
195 | #endif /* __LINUX_BLK_TYPES_H */ | 197 | #endif /* __LINUX_BLK_TYPES_H */ |
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d5063e1b5555..16a902f099ac 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h | |||
@@ -108,11 +108,17 @@ struct request { | |||
108 | 108 | ||
109 | /* | 109 | /* |
110 | * Three pointers are available for the IO schedulers, if they need | 110 | * Three pointers are available for the IO schedulers, if they need |
111 | * more they have to dynamically allocate it. | 111 | * more they have to dynamically allocate it. Flush requests are |
112 | * never put on the IO scheduler. So let the flush fields share | ||
113 | * space with the three elevator_private pointers. | ||
112 | */ | 114 | */ |
113 | void *elevator_private; | 115 | union { |
114 | void *elevator_private2; | 116 | void *elevator_private[3]; |
115 | void *elevator_private3; | 117 | struct { |
118 | unsigned int seq; | ||
119 | struct list_head list; | ||
120 | } flush; | ||
121 | }; | ||
116 | 122 | ||
117 | struct gendisk *rq_disk; | 123 | struct gendisk *rq_disk; |
118 | struct hd_struct *part; | 124 | struct hd_struct *part; |
@@ -190,7 +196,6 @@ typedef void (request_fn_proc) (struct request_queue *q); | |||
190 | typedef int (make_request_fn) (struct request_queue *q, struct bio *bio); | 196 | typedef int (make_request_fn) (struct request_queue *q, struct bio *bio); |
191 | typedef int (prep_rq_fn) (struct request_queue *, struct request *); | 197 | typedef int (prep_rq_fn) (struct request_queue *, struct request *); |
192 | typedef void (unprep_rq_fn) (struct request_queue *, struct request *); | 198 | typedef void (unprep_rq_fn) (struct request_queue *, struct request *); |
193 | typedef void (unplug_fn) (struct request_queue *); | ||
194 | 199 | ||
195 | struct bio_vec; | 200 | struct bio_vec; |
196 | struct bvec_merge_data { | 201 | struct bvec_merge_data { |
@@ -273,7 +278,6 @@ struct request_queue | |||
273 | make_request_fn *make_request_fn; | 278 | make_request_fn *make_request_fn; |
274 | prep_rq_fn *prep_rq_fn; | 279 | prep_rq_fn *prep_rq_fn; |
275 | unprep_rq_fn *unprep_rq_fn; | 280 | unprep_rq_fn *unprep_rq_fn; |
276 | unplug_fn *unplug_fn; | ||
277 | merge_bvec_fn *merge_bvec_fn; | 281 | merge_bvec_fn *merge_bvec_fn; |
278 | softirq_done_fn *softirq_done_fn; | 282 | softirq_done_fn *softirq_done_fn; |
279 | rq_timed_out_fn *rq_timed_out_fn; | 283 | rq_timed_out_fn *rq_timed_out_fn; |
@@ -287,12 +291,9 @@ struct request_queue | |||
287 | struct request *boundary_rq; | 291 | struct request *boundary_rq; |
288 | 292 | ||
289 | /* | 293 | /* |
290 | * Auto-unplugging state | 294 | * Delayed queue handling |
291 | */ | 295 | */ |
292 | struct timer_list unplug_timer; | 296 | struct delayed_work delay_work; |
293 | int unplug_thresh; /* After this many requests */ | ||
294 | unsigned long unplug_delay; /* After this many jiffies */ | ||
295 | struct work_struct unplug_work; | ||
296 | 297 | ||
297 | struct backing_dev_info backing_dev_info; | 298 | struct backing_dev_info backing_dev_info; |
298 | 299 | ||
@@ -363,11 +364,12 @@ struct request_queue | |||
363 | * for flush operations | 364 | * for flush operations |
364 | */ | 365 | */ |
365 | unsigned int flush_flags; | 366 | unsigned int flush_flags; |
366 | unsigned int flush_seq; | 367 | unsigned int flush_pending_idx:1; |
367 | int flush_err; | 368 | unsigned int flush_running_idx:1; |
369 | unsigned long flush_pending_since; | ||
370 | struct list_head flush_queue[2]; | ||
371 | struct list_head flush_data_in_flight; | ||
368 | struct request flush_rq; | 372 | struct request flush_rq; |
369 | struct request *orig_flush_rq; | ||
370 | struct list_head pending_flushes; | ||
371 | 373 | ||
372 | struct mutex sysfs_lock; | 374 | struct mutex sysfs_lock; |
373 | 375 | ||
@@ -387,14 +389,13 @@ struct request_queue | |||
387 | #define QUEUE_FLAG_ASYNCFULL 4 /* write queue has been filled */ | 389 | #define QUEUE_FLAG_ASYNCFULL 4 /* write queue has been filled */ |
388 | #define QUEUE_FLAG_DEAD 5 /* queue being torn down */ | 390 | #define QUEUE_FLAG_DEAD 5 /* queue being torn down */ |
389 | #define QUEUE_FLAG_REENTER 6 /* Re-entrancy avoidance */ | 391 | #define QUEUE_FLAG_REENTER 6 /* Re-entrancy avoidance */ |
390 | #define QUEUE_FLAG_PLUGGED 7 /* queue is plugged */ | 392 | #define QUEUE_FLAG_ELVSWITCH 7 /* don't use elevator, just do FIFO */ |
391 | #define QUEUE_FLAG_ELVSWITCH 8 /* don't use elevator, just do FIFO */ | 393 | #define QUEUE_FLAG_BIDI 8 /* queue supports bidi requests */ |
392 | #define QUEUE_FLAG_BIDI 9 /* queue supports bidi requests */ | 394 | #define QUEUE_FLAG_NOMERGES 9 /* disable merge attempts */ |
393 | #define QUEUE_FLAG_NOMERGES 10 /* disable merge attempts */ | 395 | #define QUEUE_FLAG_SAME_COMP 10 /* force complete on same CPU */ |
394 | #define QUEUE_FLAG_SAME_COMP 11 /* force complete on same CPU */ | 396 | #define QUEUE_FLAG_FAIL_IO 11 /* fake timeout */ |
395 | #define QUEUE_FLAG_FAIL_IO 12 /* fake timeout */ | 397 | #define QUEUE_FLAG_STACKABLE 12 /* supports request stacking */ |
396 | #define QUEUE_FLAG_STACKABLE 13 /* supports request stacking */ | 398 | #define QUEUE_FLAG_NONROT 13 /* non-rotational device (SSD) */ |
397 | #define QUEUE_FLAG_NONROT 14 /* non-rotational device (SSD) */ | ||
398 | #define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */ | 399 | #define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */ |
399 | #define QUEUE_FLAG_IO_STAT 15 /* do IO stats */ | 400 | #define QUEUE_FLAG_IO_STAT 15 /* do IO stats */ |
400 | #define QUEUE_FLAG_DISCARD 16 /* supports DISCARD */ | 401 | #define QUEUE_FLAG_DISCARD 16 /* supports DISCARD */ |
@@ -472,7 +473,6 @@ static inline void queue_flag_clear(unsigned int flag, struct request_queue *q) | |||
472 | __clear_bit(flag, &q->queue_flags); | 473 | __clear_bit(flag, &q->queue_flags); |
473 | } | 474 | } |
474 | 475 | ||
475 | #define blk_queue_plugged(q) test_bit(QUEUE_FLAG_PLUGGED, &(q)->queue_flags) | ||
476 | #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags) | 476 | #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags) |
477 | #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) | 477 | #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) |
478 | #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) | 478 | #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) |
@@ -667,9 +667,7 @@ extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src, | |||
667 | extern void blk_rq_unprep_clone(struct request *rq); | 667 | extern void blk_rq_unprep_clone(struct request *rq); |
668 | extern int blk_insert_cloned_request(struct request_queue *q, | 668 | extern int blk_insert_cloned_request(struct request_queue *q, |
669 | struct request *rq); | 669 | struct request *rq); |
670 | extern void blk_plug_device(struct request_queue *); | 670 | extern void blk_delay_queue(struct request_queue *, unsigned long); |
671 | extern void blk_plug_device_unlocked(struct request_queue *); | ||
672 | extern int blk_remove_plug(struct request_queue *); | ||
673 | extern void blk_recount_segments(struct request_queue *, struct bio *); | 671 | extern void blk_recount_segments(struct request_queue *, struct bio *); |
674 | extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t, | 672 | extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t, |
675 | unsigned int, void __user *); | 673 | unsigned int, void __user *); |
@@ -713,7 +711,6 @@ extern int blk_execute_rq(struct request_queue *, struct gendisk *, | |||
713 | struct request *, int); | 711 | struct request *, int); |
714 | extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *, | 712 | extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *, |
715 | struct request *, int, rq_end_io_fn *); | 713 | struct request *, int, rq_end_io_fn *); |
716 | extern void blk_unplug(struct request_queue *q); | ||
717 | 714 | ||
718 | static inline struct request_queue *bdev_get_queue(struct block_device *bdev) | 715 | static inline struct request_queue *bdev_get_queue(struct block_device *bdev) |
719 | { | 716 | { |
@@ -850,7 +847,6 @@ extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bd | |||
850 | 847 | ||
851 | extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *); | 848 | extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *); |
852 | extern void blk_dump_rq_flags(struct request *, char *); | 849 | extern void blk_dump_rq_flags(struct request *, char *); |
853 | extern void generic_unplug_device(struct request_queue *); | ||
854 | extern long nr_blockdev_pages(void); | 850 | extern long nr_blockdev_pages(void); |
855 | 851 | ||
856 | int blk_get_queue(struct request_queue *); | 852 | int blk_get_queue(struct request_queue *); |
@@ -858,6 +854,31 @@ struct request_queue *blk_alloc_queue(gfp_t); | |||
858 | struct request_queue *blk_alloc_queue_node(gfp_t, int); | 854 | struct request_queue *blk_alloc_queue_node(gfp_t, int); |
859 | extern void blk_put_queue(struct request_queue *); | 855 | extern void blk_put_queue(struct request_queue *); |
860 | 856 | ||
857 | struct blk_plug { | ||
858 | unsigned long magic; | ||
859 | struct list_head list; | ||
860 | unsigned int should_sort; | ||
861 | }; | ||
862 | |||
863 | extern void blk_start_plug(struct blk_plug *); | ||
864 | extern void blk_finish_plug(struct blk_plug *); | ||
865 | extern void __blk_flush_plug(struct task_struct *, struct blk_plug *); | ||
866 | |||
867 | static inline void blk_flush_plug(struct task_struct *tsk) | ||
868 | { | ||
869 | struct blk_plug *plug = tsk->plug; | ||
870 | |||
871 | if (unlikely(plug)) | ||
872 | __blk_flush_plug(tsk, plug); | ||
873 | } | ||
874 | |||
875 | static inline bool blk_needs_flush_plug(struct task_struct *tsk) | ||
876 | { | ||
877 | struct blk_plug *plug = tsk->plug; | ||
878 | |||
879 | return plug && !list_empty(&plug->list); | ||
880 | } | ||
881 | |||
861 | /* | 882 | /* |
862 | * tag stuff | 883 | * tag stuff |
863 | */ | 884 | */ |
@@ -1135,7 +1156,6 @@ static inline uint64_t rq_io_start_time_ns(struct request *req) | |||
1135 | extern int blk_throtl_init(struct request_queue *q); | 1156 | extern int blk_throtl_init(struct request_queue *q); |
1136 | extern void blk_throtl_exit(struct request_queue *q); | 1157 | extern void blk_throtl_exit(struct request_queue *q); |
1137 | extern int blk_throtl_bio(struct request_queue *q, struct bio **bio); | 1158 | extern int blk_throtl_bio(struct request_queue *q, struct bio **bio); |
1138 | extern void throtl_shutdown_timer_wq(struct request_queue *q); | ||
1139 | #else /* CONFIG_BLK_DEV_THROTTLING */ | 1159 | #else /* CONFIG_BLK_DEV_THROTTLING */ |
1140 | static inline int blk_throtl_bio(struct request_queue *q, struct bio **bio) | 1160 | static inline int blk_throtl_bio(struct request_queue *q, struct bio **bio) |
1141 | { | 1161 | { |
@@ -1144,7 +1164,6 @@ static inline int blk_throtl_bio(struct request_queue *q, struct bio **bio) | |||
1144 | 1164 | ||
1145 | static inline int blk_throtl_init(struct request_queue *q) { return 0; } | 1165 | static inline int blk_throtl_init(struct request_queue *q) { return 0; } |
1146 | static inline int blk_throtl_exit(struct request_queue *q) { return 0; } | 1166 | static inline int blk_throtl_exit(struct request_queue *q) { return 0; } |
1147 | static inline void throtl_shutdown_timer_wq(struct request_queue *q) {} | ||
1148 | #endif /* CONFIG_BLK_DEV_THROTTLING */ | 1167 | #endif /* CONFIG_BLK_DEV_THROTTLING */ |
1149 | 1168 | ||
1150 | #define MODULE_ALIAS_BLOCKDEV(major,minor) \ | 1169 | #define MODULE_ALIAS_BLOCKDEV(major,minor) \ |
@@ -1278,6 +1297,26 @@ static inline long nr_blockdev_pages(void) | |||
1278 | return 0; | 1297 | return 0; |
1279 | } | 1298 | } |
1280 | 1299 | ||
1300 | struct blk_plug { | ||
1301 | }; | ||
1302 | |||
1303 | static inline void blk_start_plug(struct blk_plug *plug) | ||
1304 | { | ||
1305 | } | ||
1306 | |||
1307 | static inline void blk_finish_plug(struct blk_plug *plug) | ||
1308 | { | ||
1309 | } | ||
1310 | |||
1311 | static inline void blk_flush_plug(struct task_struct *task) | ||
1312 | { | ||
1313 | } | ||
1314 | |||
1315 | static inline bool blk_needs_flush_plug(struct task_struct *tsk) | ||
1316 | { | ||
1317 | return false; | ||
1318 | } | ||
1319 | |||
1281 | #endif /* CONFIG_BLOCK */ | 1320 | #endif /* CONFIG_BLOCK */ |
1282 | 1321 | ||
1283 | #endif | 1322 | #endif |
diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index 68d1fe7b877c..f5df23561b96 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h | |||
@@ -219,7 +219,6 @@ int generic_cont_expand_simple(struct inode *inode, loff_t size); | |||
219 | int block_commit_write(struct page *page, unsigned from, unsigned to); | 219 | int block_commit_write(struct page *page, unsigned from, unsigned to); |
220 | int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, | 220 | int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, |
221 | get_block_t get_block); | 221 | get_block_t get_block); |
222 | void block_sync_page(struct page *); | ||
223 | sector_t generic_block_bmap(struct address_space *, sector_t, get_block_t *); | 222 | sector_t generic_block_bmap(struct address_space *, sector_t, get_block_t *); |
224 | int block_truncate_page(struct address_space *, loff_t, get_block_t *); | 223 | int block_truncate_page(struct address_space *, loff_t, get_block_t *); |
225 | int nobh_write_begin(struct address_space *, loff_t, unsigned, unsigned, | 224 | int nobh_write_begin(struct address_space *, loff_t, unsigned, unsigned, |
diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 272496d1fae4..e2768834f397 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h | |||
@@ -286,11 +286,6 @@ void dm_table_add_target_callbacks(struct dm_table *t, struct dm_target_callback | |||
286 | int dm_table_complete(struct dm_table *t); | 286 | int dm_table_complete(struct dm_table *t); |
287 | 287 | ||
288 | /* | 288 | /* |
289 | * Unplug all devices in a table. | ||
290 | */ | ||
291 | void dm_table_unplug_all(struct dm_table *t); | ||
292 | |||
293 | /* | ||
294 | * Table reference counting. | 289 | * Table reference counting. |
295 | */ | 290 | */ |
296 | struct dm_table *dm_get_live_table(struct mapped_device *md); | 291 | struct dm_table *dm_get_live_table(struct mapped_device *md); |
diff --git a/include/linux/elevator.h b/include/linux/elevator.h index 4d857973d2c9..d93efcc44570 100644 --- a/include/linux/elevator.h +++ b/include/linux/elevator.h | |||
@@ -20,7 +20,6 @@ typedef void (elevator_bio_merged_fn) (struct request_queue *, | |||
20 | typedef int (elevator_dispatch_fn) (struct request_queue *, int); | 20 | typedef int (elevator_dispatch_fn) (struct request_queue *, int); |
21 | 21 | ||
22 | typedef void (elevator_add_req_fn) (struct request_queue *, struct request *); | 22 | typedef void (elevator_add_req_fn) (struct request_queue *, struct request *); |
23 | typedef int (elevator_queue_empty_fn) (struct request_queue *); | ||
24 | typedef struct request *(elevator_request_list_fn) (struct request_queue *, struct request *); | 23 | typedef struct request *(elevator_request_list_fn) (struct request_queue *, struct request *); |
25 | typedef void (elevator_completed_req_fn) (struct request_queue *, struct request *); | 24 | typedef void (elevator_completed_req_fn) (struct request_queue *, struct request *); |
26 | typedef int (elevator_may_queue_fn) (struct request_queue *, int); | 25 | typedef int (elevator_may_queue_fn) (struct request_queue *, int); |
@@ -46,7 +45,6 @@ struct elevator_ops | |||
46 | elevator_activate_req_fn *elevator_activate_req_fn; | 45 | elevator_activate_req_fn *elevator_activate_req_fn; |
47 | elevator_deactivate_req_fn *elevator_deactivate_req_fn; | 46 | elevator_deactivate_req_fn *elevator_deactivate_req_fn; |
48 | 47 | ||
49 | elevator_queue_empty_fn *elevator_queue_empty_fn; | ||
50 | elevator_completed_req_fn *elevator_completed_req_fn; | 48 | elevator_completed_req_fn *elevator_completed_req_fn; |
51 | 49 | ||
52 | elevator_request_list_fn *elevator_former_req_fn; | 50 | elevator_request_list_fn *elevator_former_req_fn; |
@@ -101,17 +99,17 @@ struct elevator_queue | |||
101 | */ | 99 | */ |
102 | extern void elv_dispatch_sort(struct request_queue *, struct request *); | 100 | extern void elv_dispatch_sort(struct request_queue *, struct request *); |
103 | extern void elv_dispatch_add_tail(struct request_queue *, struct request *); | 101 | extern void elv_dispatch_add_tail(struct request_queue *, struct request *); |
104 | extern void elv_add_request(struct request_queue *, struct request *, int, int); | 102 | extern void elv_add_request(struct request_queue *, struct request *, int); |
105 | extern void __elv_add_request(struct request_queue *, struct request *, int, int); | 103 | extern void __elv_add_request(struct request_queue *, struct request *, int); |
106 | extern void elv_insert(struct request_queue *, struct request *, int); | 104 | extern void elv_insert(struct request_queue *, struct request *, int); |
107 | extern int elv_merge(struct request_queue *, struct request **, struct bio *); | 105 | extern int elv_merge(struct request_queue *, struct request **, struct bio *); |
106 | extern int elv_try_merge(struct request *, struct bio *); | ||
108 | extern void elv_merge_requests(struct request_queue *, struct request *, | 107 | extern void elv_merge_requests(struct request_queue *, struct request *, |
109 | struct request *); | 108 | struct request *); |
110 | extern void elv_merged_request(struct request_queue *, struct request *, int); | 109 | extern void elv_merged_request(struct request_queue *, struct request *, int); |
111 | extern void elv_bio_merged(struct request_queue *q, struct request *, | 110 | extern void elv_bio_merged(struct request_queue *q, struct request *, |
112 | struct bio *); | 111 | struct bio *); |
113 | extern void elv_requeue_request(struct request_queue *, struct request *); | 112 | extern void elv_requeue_request(struct request_queue *, struct request *); |
114 | extern int elv_queue_empty(struct request_queue *); | ||
115 | extern struct request *elv_former_request(struct request_queue *, struct request *); | 113 | extern struct request *elv_former_request(struct request_queue *, struct request *); |
116 | extern struct request *elv_latter_request(struct request_queue *, struct request *); | 114 | extern struct request *elv_latter_request(struct request_queue *, struct request *); |
117 | extern int elv_register_queue(struct request_queue *q); | 115 | extern int elv_register_queue(struct request_queue *q); |
@@ -167,6 +165,8 @@ extern struct request *elv_rb_find(struct rb_root *, sector_t); | |||
167 | #define ELEVATOR_INSERT_BACK 2 | 165 | #define ELEVATOR_INSERT_BACK 2 |
168 | #define ELEVATOR_INSERT_SORT 3 | 166 | #define ELEVATOR_INSERT_SORT 3 |
169 | #define ELEVATOR_INSERT_REQUEUE 4 | 167 | #define ELEVATOR_INSERT_REQUEUE 4 |
168 | #define ELEVATOR_INSERT_FLUSH 5 | ||
169 | #define ELEVATOR_INSERT_SORT_MERGE 6 | ||
170 | 170 | ||
171 | /* | 171 | /* |
172 | * return values from elevator_may_queue_fn | 172 | * return values from elevator_may_queue_fn |
diff --git a/include/linux/fs.h b/include/linux/fs.h index 4dda076c24a1..ce7e18555197 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h | |||
@@ -138,16 +138,10 @@ struct inodes_stat_t { | |||
138 | * block layer could (in theory) choose to ignore this | 138 | * block layer could (in theory) choose to ignore this |
139 | * request if it runs into resource problems. | 139 | * request if it runs into resource problems. |
140 | * WRITE A normal async write. Device will be plugged. | 140 | * WRITE A normal async write. Device will be plugged. |
141 | * WRITE_SYNC_PLUG Synchronous write. Identical to WRITE, but passes down | 141 | * WRITE_SYNC Synchronous write. Identical to WRITE, but passes down |
142 | * the hint that someone will be waiting on this IO | 142 | * the hint that someone will be waiting on this IO |
143 | * shortly. The device must still be unplugged explicitly, | 143 | * shortly. The write equivalent of READ_SYNC. |
144 | * WRITE_SYNC_PLUG does not do this as we could be | 144 | * WRITE_ODIRECT Special case write for O_DIRECT only. |
145 | * submitting more writes before we actually wait on any | ||
146 | * of them. | ||
147 | * WRITE_SYNC Like WRITE_SYNC_PLUG, but also unplugs the device | ||
148 | * immediately after submission. The write equivalent | ||
149 | * of READ_SYNC. | ||
150 | * WRITE_ODIRECT_PLUG Special case write for O_DIRECT only. | ||
151 | * WRITE_FLUSH Like WRITE_SYNC but with preceding cache flush. | 145 | * WRITE_FLUSH Like WRITE_SYNC but with preceding cache flush. |
152 | * WRITE_FUA Like WRITE_SYNC but data is guaranteed to be on | 146 | * WRITE_FUA Like WRITE_SYNC but data is guaranteed to be on |
153 | * non-volatile media on completion. | 147 | * non-volatile media on completion. |
@@ -163,18 +157,14 @@ struct inodes_stat_t { | |||
163 | #define WRITE RW_MASK | 157 | #define WRITE RW_MASK |
164 | #define READA RWA_MASK | 158 | #define READA RWA_MASK |
165 | 159 | ||
166 | #define READ_SYNC (READ | REQ_SYNC | REQ_UNPLUG) | 160 | #define READ_SYNC (READ | REQ_SYNC) |
167 | #define READ_META (READ | REQ_META) | 161 | #define READ_META (READ | REQ_META) |
168 | #define WRITE_SYNC_PLUG (WRITE | REQ_SYNC | REQ_NOIDLE) | 162 | #define WRITE_SYNC (WRITE | REQ_SYNC | REQ_NOIDLE) |
169 | #define WRITE_SYNC (WRITE | REQ_SYNC | REQ_NOIDLE | REQ_UNPLUG) | 163 | #define WRITE_ODIRECT (WRITE | REQ_SYNC) |
170 | #define WRITE_ODIRECT_PLUG (WRITE | REQ_SYNC) | ||
171 | #define WRITE_META (WRITE | REQ_META) | 164 | #define WRITE_META (WRITE | REQ_META) |
172 | #define WRITE_FLUSH (WRITE | REQ_SYNC | REQ_NOIDLE | REQ_UNPLUG | \ | 165 | #define WRITE_FLUSH (WRITE | REQ_SYNC | REQ_NOIDLE | REQ_FLUSH) |
173 | REQ_FLUSH) | 166 | #define WRITE_FUA (WRITE | REQ_SYNC | REQ_NOIDLE | REQ_FUA) |
174 | #define WRITE_FUA (WRITE | REQ_SYNC | REQ_NOIDLE | REQ_UNPLUG | \ | 167 | #define WRITE_FLUSH_FUA (WRITE | REQ_SYNC | REQ_NOIDLE | REQ_FLUSH | REQ_FUA) |
175 | REQ_FUA) | ||
176 | #define WRITE_FLUSH_FUA (WRITE | REQ_SYNC | REQ_NOIDLE | REQ_UNPLUG | \ | ||
177 | REQ_FLUSH | REQ_FUA) | ||
178 | 168 | ||
179 | #define SEL_IN 1 | 169 | #define SEL_IN 1 |
180 | #define SEL_OUT 2 | 170 | #define SEL_OUT 2 |
@@ -586,7 +576,6 @@ typedef int (*read_actor_t)(read_descriptor_t *, struct page *, | |||
586 | struct address_space_operations { | 576 | struct address_space_operations { |
587 | int (*writepage)(struct page *page, struct writeback_control *wbc); | 577 | int (*writepage)(struct page *page, struct writeback_control *wbc); |
588 | int (*readpage)(struct file *, struct page *); | 578 | int (*readpage)(struct file *, struct page *); |
589 | void (*sync_page)(struct page *); | ||
590 | 579 | ||
591 | /* Write back some dirty pages from this mapping. */ | 580 | /* Write back some dirty pages from this mapping. */ |
592 | int (*writepages)(struct address_space *, struct writeback_control *); | 581 | int (*writepages)(struct address_space *, struct writeback_control *); |
diff --git a/include/linux/genhd.h b/include/linux/genhd.h index c0d5f6945c1e..d764a426e9fd 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h | |||
@@ -109,7 +109,7 @@ struct hd_struct { | |||
109 | int make_it_fail; | 109 | int make_it_fail; |
110 | #endif | 110 | #endif |
111 | unsigned long stamp; | 111 | unsigned long stamp; |
112 | int in_flight[2]; | 112 | atomic_t in_flight[2]; |
113 | #ifdef CONFIG_SMP | 113 | #ifdef CONFIG_SMP |
114 | struct disk_stats __percpu *dkstats; | 114 | struct disk_stats __percpu *dkstats; |
115 | #else | 115 | #else |
@@ -370,21 +370,21 @@ static inline void free_part_stats(struct hd_struct *part) | |||
370 | 370 | ||
371 | static inline void part_inc_in_flight(struct hd_struct *part, int rw) | 371 | static inline void part_inc_in_flight(struct hd_struct *part, int rw) |
372 | { | 372 | { |
373 | part->in_flight[rw]++; | 373 | atomic_inc(&part->in_flight[rw]); |
374 | if (part->partno) | 374 | if (part->partno) |
375 | part_to_disk(part)->part0.in_flight[rw]++; | 375 | atomic_inc(&part_to_disk(part)->part0.in_flight[rw]); |
376 | } | 376 | } |
377 | 377 | ||
378 | static inline void part_dec_in_flight(struct hd_struct *part, int rw) | 378 | static inline void part_dec_in_flight(struct hd_struct *part, int rw) |
379 | { | 379 | { |
380 | part->in_flight[rw]--; | 380 | atomic_dec(&part->in_flight[rw]); |
381 | if (part->partno) | 381 | if (part->partno) |
382 | part_to_disk(part)->part0.in_flight[rw]--; | 382 | atomic_dec(&part_to_disk(part)->part0.in_flight[rw]); |
383 | } | 383 | } |
384 | 384 | ||
385 | static inline int part_in_flight(struct hd_struct *part) | 385 | static inline int part_in_flight(struct hd_struct *part) |
386 | { | 386 | { |
387 | return part->in_flight[0] + part->in_flight[1]; | 387 | return atomic_read(&part->in_flight[0]) + atomic_read(&part->in_flight[1]); |
388 | } | 388 | } |
389 | 389 | ||
390 | static inline struct partition_meta_info *alloc_part_info(struct gendisk *disk) | 390 | static inline struct partition_meta_info *alloc_part_info(struct gendisk *disk) |
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 29ebba54c238..c11950652646 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h | |||
@@ -298,7 +298,6 @@ static inline pgoff_t linear_page_index(struct vm_area_struct *vma, | |||
298 | 298 | ||
299 | extern void __lock_page(struct page *page); | 299 | extern void __lock_page(struct page *page); |
300 | extern int __lock_page_killable(struct page *page); | 300 | extern int __lock_page_killable(struct page *page); |
301 | extern void __lock_page_nosync(struct page *page); | ||
302 | extern int __lock_page_or_retry(struct page *page, struct mm_struct *mm, | 301 | extern int __lock_page_or_retry(struct page *page, struct mm_struct *mm, |
303 | unsigned int flags); | 302 | unsigned int flags); |
304 | extern void unlock_page(struct page *page); | 303 | extern void unlock_page(struct page *page); |
@@ -342,17 +341,6 @@ static inline int lock_page_killable(struct page *page) | |||
342 | } | 341 | } |
343 | 342 | ||
344 | /* | 343 | /* |
345 | * lock_page_nosync should only be used if we can't pin the page's inode. | ||
346 | * Doesn't play quite so well with block device plugging. | ||
347 | */ | ||
348 | static inline void lock_page_nosync(struct page *page) | ||
349 | { | ||
350 | might_sleep(); | ||
351 | if (!trylock_page(page)) | ||
352 | __lock_page_nosync(page); | ||
353 | } | ||
354 | |||
355 | /* | ||
356 | * lock_page_or_retry - Lock the page, unless this would block and the | 344 | * lock_page_or_retry - Lock the page, unless this would block and the |
357 | * caller indicated that it can handle a retry. | 345 | * caller indicated that it can handle a retry. |
358 | */ | 346 | */ |
diff --git a/include/linux/sched.h b/include/linux/sched.h index 98fc7ed4b191..b8369d522bf8 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h | |||
@@ -99,6 +99,7 @@ struct robust_list_head; | |||
99 | struct bio_list; | 99 | struct bio_list; |
100 | struct fs_struct; | 100 | struct fs_struct; |
101 | struct perf_event_context; | 101 | struct perf_event_context; |
102 | struct blk_plug; | ||
102 | 103 | ||
103 | /* | 104 | /* |
104 | * List of flags we want to share for kernel threads, | 105 | * List of flags we want to share for kernel threads, |
@@ -1428,6 +1429,11 @@ struct task_struct { | |||
1428 | /* stacked block device info */ | 1429 | /* stacked block device info */ |
1429 | struct bio_list *bio_list; | 1430 | struct bio_list *bio_list; |
1430 | 1431 | ||
1432 | #ifdef CONFIG_BLOCK | ||
1433 | /* stack plugging */ | ||
1434 | struct blk_plug *plug; | ||
1435 | #endif | ||
1436 | |||
1431 | /* VM state */ | 1437 | /* VM state */ |
1432 | struct reclaim_state *reclaim_state; | 1438 | struct reclaim_state *reclaim_state; |
1433 | 1439 | ||
diff --git a/include/linux/swap.h b/include/linux/swap.h index ed6ebe690f4a..a5c6da5d8df8 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h | |||
@@ -309,8 +309,6 @@ extern void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff, | |||
309 | struct page **pagep, swp_entry_t *ent); | 309 | struct page **pagep, swp_entry_t *ent); |
310 | #endif | 310 | #endif |
311 | 311 | ||
312 | extern void swap_unplug_io_fn(struct backing_dev_info *, struct page *); | ||
313 | |||
314 | #ifdef CONFIG_SWAP | 312 | #ifdef CONFIG_SWAP |
315 | /* linux/mm/page_io.c */ | 313 | /* linux/mm/page_io.c */ |
316 | extern int swap_readpage(struct page *); | 314 | extern int swap_readpage(struct page *); |
diff --git a/kernel/exit.c b/kernel/exit.c index f9a45ebcc7b1..6a488ad2dce5 100644 --- a/kernel/exit.c +++ b/kernel/exit.c | |||
@@ -908,6 +908,7 @@ NORET_TYPE void do_exit(long code) | |||
908 | profile_task_exit(tsk); | 908 | profile_task_exit(tsk); |
909 | 909 | ||
910 | WARN_ON(atomic_read(&tsk->fs_excl)); | 910 | WARN_ON(atomic_read(&tsk->fs_excl)); |
911 | WARN_ON(blk_needs_flush_plug(tsk)); | ||
911 | 912 | ||
912 | if (unlikely(in_interrupt())) | 913 | if (unlikely(in_interrupt())) |
913 | panic("Aiee, killing interrupt handler!"); | 914 | panic("Aiee, killing interrupt handler!"); |
diff --git a/kernel/fork.c b/kernel/fork.c index 457fff2e17e0..e7548dee636b 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
@@ -1205,6 +1205,9 @@ static struct task_struct *copy_process(unsigned long clone_flags, | |||
1205 | * Clear TID on mm_release()? | 1205 | * Clear TID on mm_release()? |
1206 | */ | 1206 | */ |
1207 | p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr: NULL; | 1207 | p->clear_child_tid = (clone_flags & CLONE_CHILD_CLEARTID) ? child_tidptr: NULL; |
1208 | #ifdef CONFIG_BLOCK | ||
1209 | p->plug = NULL; | ||
1210 | #endif | ||
1208 | #ifdef CONFIG_FUTEX | 1211 | #ifdef CONFIG_FUTEX |
1209 | p->robust_list = NULL; | 1212 | p->robust_list = NULL; |
1210 | #ifdef CONFIG_COMPAT | 1213 | #ifdef CONFIG_COMPAT |
diff --git a/kernel/power/block_io.c b/kernel/power/block_io.c index 83bbc7c02df9..d09dd10c5a5e 100644 --- a/kernel/power/block_io.c +++ b/kernel/power/block_io.c | |||
@@ -28,7 +28,7 @@ | |||
28 | static int submit(int rw, struct block_device *bdev, sector_t sector, | 28 | static int submit(int rw, struct block_device *bdev, sector_t sector, |
29 | struct page *page, struct bio **bio_chain) | 29 | struct page *page, struct bio **bio_chain) |
30 | { | 30 | { |
31 | const int bio_rw = rw | REQ_SYNC | REQ_UNPLUG; | 31 | const int bio_rw = rw | REQ_SYNC; |
32 | struct bio *bio; | 32 | struct bio *bio; |
33 | 33 | ||
34 | bio = bio_alloc(__GFP_WAIT | __GFP_HIGH, 1); | 34 | bio = bio_alloc(__GFP_WAIT | __GFP_HIGH, 1); |
diff --git a/kernel/sched.c b/kernel/sched.c index 480adeb63f8f..ae659b99ce73 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -4115,6 +4115,16 @@ need_resched: | |||
4115 | switch_count = &prev->nvcsw; | 4115 | switch_count = &prev->nvcsw; |
4116 | } | 4116 | } |
4117 | 4117 | ||
4118 | /* | ||
4119 | * If we are going to sleep and we have plugged IO queued, make | ||
4120 | * sure to submit it to avoid deadlocks. | ||
4121 | */ | ||
4122 | if (prev->state != TASK_RUNNING && blk_needs_flush_plug(prev)) { | ||
4123 | raw_spin_unlock(&rq->lock); | ||
4124 | blk_flush_plug(prev); | ||
4125 | raw_spin_lock(&rq->lock); | ||
4126 | } | ||
4127 | |||
4118 | pre_schedule(rq, prev); | 4128 | pre_schedule(rq, prev); |
4119 | 4129 | ||
4120 | if (unlikely(!rq->nr_running)) | 4130 | if (unlikely(!rq->nr_running)) |
@@ -5528,6 +5538,7 @@ void __sched io_schedule(void) | |||
5528 | 5538 | ||
5529 | delayacct_blkio_start(); | 5539 | delayacct_blkio_start(); |
5530 | atomic_inc(&rq->nr_iowait); | 5540 | atomic_inc(&rq->nr_iowait); |
5541 | blk_flush_plug(current); | ||
5531 | current->in_iowait = 1; | 5542 | current->in_iowait = 1; |
5532 | schedule(); | 5543 | schedule(); |
5533 | current->in_iowait = 0; | 5544 | current->in_iowait = 0; |
@@ -5543,6 +5554,7 @@ long __sched io_schedule_timeout(long timeout) | |||
5543 | 5554 | ||
5544 | delayacct_blkio_start(); | 5555 | delayacct_blkio_start(); |
5545 | atomic_inc(&rq->nr_iowait); | 5556 | atomic_inc(&rq->nr_iowait); |
5557 | blk_flush_plug(current); | ||
5546 | current->in_iowait = 1; | 5558 | current->in_iowait = 1; |
5547 | ret = schedule_timeout(timeout); | 5559 | ret = schedule_timeout(timeout); |
5548 | current->in_iowait = 0; | 5560 | current->in_iowait = 0; |
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index cbafed7d4f38..7aa40f8e182d 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c | |||
@@ -703,28 +703,21 @@ void blk_trace_shutdown(struct request_queue *q) | |||
703 | * | 703 | * |
704 | **/ | 704 | **/ |
705 | static void blk_add_trace_rq(struct request_queue *q, struct request *rq, | 705 | static void blk_add_trace_rq(struct request_queue *q, struct request *rq, |
706 | u32 what) | 706 | u32 what) |
707 | { | 707 | { |
708 | struct blk_trace *bt = q->blk_trace; | 708 | struct blk_trace *bt = q->blk_trace; |
709 | int rw = rq->cmd_flags & 0x03; | ||
710 | 709 | ||
711 | if (likely(!bt)) | 710 | if (likely(!bt)) |
712 | return; | 711 | return; |
713 | 712 | ||
714 | if (rq->cmd_flags & REQ_DISCARD) | ||
715 | rw |= REQ_DISCARD; | ||
716 | |||
717 | if (rq->cmd_flags & REQ_SECURE) | ||
718 | rw |= REQ_SECURE; | ||
719 | |||
720 | if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { | 713 | if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { |
721 | what |= BLK_TC_ACT(BLK_TC_PC); | 714 | what |= BLK_TC_ACT(BLK_TC_PC); |
722 | __blk_add_trace(bt, 0, blk_rq_bytes(rq), rw, | 715 | __blk_add_trace(bt, 0, blk_rq_bytes(rq), rq->cmd_flags, |
723 | what, rq->errors, rq->cmd_len, rq->cmd); | 716 | what, rq->errors, rq->cmd_len, rq->cmd); |
724 | } else { | 717 | } else { |
725 | what |= BLK_TC_ACT(BLK_TC_FS); | 718 | what |= BLK_TC_ACT(BLK_TC_FS); |
726 | __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), rw, | 719 | __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), |
727 | what, rq->errors, 0, NULL); | 720 | rq->cmd_flags, what, rq->errors, 0, NULL); |
728 | } | 721 | } |
729 | } | 722 | } |
730 | 723 | ||
diff --git a/mm/backing-dev.c b/mm/backing-dev.c index 027100d30227..8fe9d3407921 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c | |||
@@ -14,17 +14,11 @@ | |||
14 | 14 | ||
15 | static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0); | 15 | static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0); |
16 | 16 | ||
17 | void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page) | ||
18 | { | ||
19 | } | ||
20 | EXPORT_SYMBOL(default_unplug_io_fn); | ||
21 | |||
22 | struct backing_dev_info default_backing_dev_info = { | 17 | struct backing_dev_info default_backing_dev_info = { |
23 | .name = "default", | 18 | .name = "default", |
24 | .ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_CACHE_SIZE, | 19 | .ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_CACHE_SIZE, |
25 | .state = 0, | 20 | .state = 0, |
26 | .capabilities = BDI_CAP_MAP_COPY, | 21 | .capabilities = BDI_CAP_MAP_COPY, |
27 | .unplug_io_fn = default_unplug_io_fn, | ||
28 | }; | 22 | }; |
29 | EXPORT_SYMBOL_GPL(default_backing_dev_info); | 23 | EXPORT_SYMBOL_GPL(default_backing_dev_info); |
30 | 24 | ||
@@ -604,7 +598,7 @@ static void bdi_prune_sb(struct backing_dev_info *bdi) | |||
604 | spin_lock(&sb_lock); | 598 | spin_lock(&sb_lock); |
605 | list_for_each_entry(sb, &super_blocks, s_list) { | 599 | list_for_each_entry(sb, &super_blocks, s_list) { |
606 | if (sb->s_bdi == bdi) | 600 | if (sb->s_bdi == bdi) |
607 | sb->s_bdi = NULL; | 601 | sb->s_bdi = &default_backing_dev_info; |
608 | } | 602 | } |
609 | spin_unlock(&sb_lock); | 603 | spin_unlock(&sb_lock); |
610 | } | 604 | } |
diff --git a/mm/filemap.c b/mm/filemap.c index f807afda86f2..04d1992fd86b 100644 --- a/mm/filemap.c +++ b/mm/filemap.c | |||
@@ -164,45 +164,15 @@ void delete_from_page_cache(struct page *page) | |||
164 | } | 164 | } |
165 | EXPORT_SYMBOL(delete_from_page_cache); | 165 | EXPORT_SYMBOL(delete_from_page_cache); |
166 | 166 | ||
167 | static int sync_page(void *word) | 167 | static int sleep_on_page(void *word) |
168 | { | 168 | { |
169 | struct address_space *mapping; | ||
170 | struct page *page; | ||
171 | |||
172 | page = container_of((unsigned long *)word, struct page, flags); | ||
173 | |||
174 | /* | ||
175 | * page_mapping() is being called without PG_locked held. | ||
176 | * Some knowledge of the state and use of the page is used to | ||
177 | * reduce the requirements down to a memory barrier. | ||
178 | * The danger here is of a stale page_mapping() return value | ||
179 | * indicating a struct address_space different from the one it's | ||
180 | * associated with when it is associated with one. | ||
181 | * After smp_mb(), it's either the correct page_mapping() for | ||
182 | * the page, or an old page_mapping() and the page's own | ||
183 | * page_mapping() has gone NULL. | ||
184 | * The ->sync_page() address_space operation must tolerate | ||
185 | * page_mapping() going NULL. By an amazing coincidence, | ||
186 | * this comes about because none of the users of the page | ||
187 | * in the ->sync_page() methods make essential use of the | ||
188 | * page_mapping(), merely passing the page down to the backing | ||
189 | * device's unplug functions when it's non-NULL, which in turn | ||
190 | * ignore it for all cases but swap, where only page_private(page) is | ||
191 | * of interest. When page_mapping() does go NULL, the entire | ||
192 | * call stack gracefully ignores the page and returns. | ||
193 | * -- wli | ||
194 | */ | ||
195 | smp_mb(); | ||
196 | mapping = page_mapping(page); | ||
197 | if (mapping && mapping->a_ops && mapping->a_ops->sync_page) | ||
198 | mapping->a_ops->sync_page(page); | ||
199 | io_schedule(); | 169 | io_schedule(); |
200 | return 0; | 170 | return 0; |
201 | } | 171 | } |
202 | 172 | ||
203 | static int sync_page_killable(void *word) | 173 | static int sleep_on_page_killable(void *word) |
204 | { | 174 | { |
205 | sync_page(word); | 175 | sleep_on_page(word); |
206 | return fatal_signal_pending(current) ? -EINTR : 0; | 176 | return fatal_signal_pending(current) ? -EINTR : 0; |
207 | } | 177 | } |
208 | 178 | ||
@@ -558,12 +528,6 @@ struct page *__page_cache_alloc(gfp_t gfp) | |||
558 | EXPORT_SYMBOL(__page_cache_alloc); | 528 | EXPORT_SYMBOL(__page_cache_alloc); |
559 | #endif | 529 | #endif |
560 | 530 | ||
561 | static int __sleep_on_page_lock(void *word) | ||
562 | { | ||
563 | io_schedule(); | ||
564 | return 0; | ||
565 | } | ||
566 | |||
567 | /* | 531 | /* |
568 | * In order to wait for pages to become available there must be | 532 | * In order to wait for pages to become available there must be |
569 | * waitqueues associated with pages. By using a hash table of | 533 | * waitqueues associated with pages. By using a hash table of |
@@ -591,7 +555,7 @@ void wait_on_page_bit(struct page *page, int bit_nr) | |||
591 | DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); | 555 | DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); |
592 | 556 | ||
593 | if (test_bit(bit_nr, &page->flags)) | 557 | if (test_bit(bit_nr, &page->flags)) |
594 | __wait_on_bit(page_waitqueue(page), &wait, sync_page, | 558 | __wait_on_bit(page_waitqueue(page), &wait, sleep_on_page, |
595 | TASK_UNINTERRUPTIBLE); | 559 | TASK_UNINTERRUPTIBLE); |
596 | } | 560 | } |
597 | EXPORT_SYMBOL(wait_on_page_bit); | 561 | EXPORT_SYMBOL(wait_on_page_bit); |
@@ -655,17 +619,12 @@ EXPORT_SYMBOL(end_page_writeback); | |||
655 | /** | 619 | /** |
656 | * __lock_page - get a lock on the page, assuming we need to sleep to get it | 620 | * __lock_page - get a lock on the page, assuming we need to sleep to get it |
657 | * @page: the page to lock | 621 | * @page: the page to lock |
658 | * | ||
659 | * Ugly. Running sync_page() in state TASK_UNINTERRUPTIBLE is scary. If some | ||
660 | * random driver's requestfn sets TASK_RUNNING, we could busywait. However | ||
661 | * chances are that on the second loop, the block layer's plug list is empty, | ||
662 | * so sync_page() will then return in state TASK_UNINTERRUPTIBLE. | ||
663 | */ | 622 | */ |
664 | void __lock_page(struct page *page) | 623 | void __lock_page(struct page *page) |
665 | { | 624 | { |
666 | DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); | 625 | DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); |
667 | 626 | ||
668 | __wait_on_bit_lock(page_waitqueue(page), &wait, sync_page, | 627 | __wait_on_bit_lock(page_waitqueue(page), &wait, sleep_on_page, |
669 | TASK_UNINTERRUPTIBLE); | 628 | TASK_UNINTERRUPTIBLE); |
670 | } | 629 | } |
671 | EXPORT_SYMBOL(__lock_page); | 630 | EXPORT_SYMBOL(__lock_page); |
@@ -675,24 +634,10 @@ int __lock_page_killable(struct page *page) | |||
675 | DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); | 634 | DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); |
676 | 635 | ||
677 | return __wait_on_bit_lock(page_waitqueue(page), &wait, | 636 | return __wait_on_bit_lock(page_waitqueue(page), &wait, |
678 | sync_page_killable, TASK_KILLABLE); | 637 | sleep_on_page_killable, TASK_KILLABLE); |
679 | } | 638 | } |
680 | EXPORT_SYMBOL_GPL(__lock_page_killable); | 639 | EXPORT_SYMBOL_GPL(__lock_page_killable); |
681 | 640 | ||
682 | /** | ||
683 | * __lock_page_nosync - get a lock on the page, without calling sync_page() | ||
684 | * @page: the page to lock | ||
685 | * | ||
686 | * Variant of lock_page that does not require the caller to hold a reference | ||
687 | * on the page's mapping. | ||
688 | */ | ||
689 | void __lock_page_nosync(struct page *page) | ||
690 | { | ||
691 | DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); | ||
692 | __wait_on_bit_lock(page_waitqueue(page), &wait, __sleep_on_page_lock, | ||
693 | TASK_UNINTERRUPTIBLE); | ||
694 | } | ||
695 | |||
696 | int __lock_page_or_retry(struct page *page, struct mm_struct *mm, | 641 | int __lock_page_or_retry(struct page *page, struct mm_struct *mm, |
697 | unsigned int flags) | 642 | unsigned int flags) |
698 | { | 643 | { |
@@ -1407,12 +1352,15 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, | |||
1407 | unsigned long seg = 0; | 1352 | unsigned long seg = 0; |
1408 | size_t count; | 1353 | size_t count; |
1409 | loff_t *ppos = &iocb->ki_pos; | 1354 | loff_t *ppos = &iocb->ki_pos; |
1355 | struct blk_plug plug; | ||
1410 | 1356 | ||
1411 | count = 0; | 1357 | count = 0; |
1412 | retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE); | 1358 | retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE); |
1413 | if (retval) | 1359 | if (retval) |
1414 | return retval; | 1360 | return retval; |
1415 | 1361 | ||
1362 | blk_start_plug(&plug); | ||
1363 | |||
1416 | /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ | 1364 | /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ |
1417 | if (filp->f_flags & O_DIRECT) { | 1365 | if (filp->f_flags & O_DIRECT) { |
1418 | loff_t size; | 1366 | loff_t size; |
@@ -1485,6 +1433,7 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, | |||
1485 | break; | 1433 | break; |
1486 | } | 1434 | } |
1487 | out: | 1435 | out: |
1436 | blk_finish_plug(&plug); | ||
1488 | return retval; | 1437 | return retval; |
1489 | } | 1438 | } |
1490 | EXPORT_SYMBOL(generic_file_aio_read); | 1439 | EXPORT_SYMBOL(generic_file_aio_read); |
@@ -2596,11 +2545,13 @@ ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, | |||
2596 | { | 2545 | { |
2597 | struct file *file = iocb->ki_filp; | 2546 | struct file *file = iocb->ki_filp; |
2598 | struct inode *inode = file->f_mapping->host; | 2547 | struct inode *inode = file->f_mapping->host; |
2548 | struct blk_plug plug; | ||
2599 | ssize_t ret; | 2549 | ssize_t ret; |
2600 | 2550 | ||
2601 | BUG_ON(iocb->ki_pos != pos); | 2551 | BUG_ON(iocb->ki_pos != pos); |
2602 | 2552 | ||
2603 | mutex_lock(&inode->i_mutex); | 2553 | mutex_lock(&inode->i_mutex); |
2554 | blk_start_plug(&plug); | ||
2604 | ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); | 2555 | ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); |
2605 | mutex_unlock(&inode->i_mutex); | 2556 | mutex_unlock(&inode->i_mutex); |
2606 | 2557 | ||
@@ -2611,6 +2562,7 @@ ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, | |||
2611 | if (err < 0 && ret > 0) | 2562 | if (err < 0 && ret > 0) |
2612 | ret = err; | 2563 | ret = err; |
2613 | } | 2564 | } |
2565 | blk_finish_plug(&plug); | ||
2614 | return ret; | 2566 | return ret; |
2615 | } | 2567 | } |
2616 | EXPORT_SYMBOL(generic_file_aio_write); | 2568 | EXPORT_SYMBOL(generic_file_aio_write); |
diff --git a/mm/memory-failure.c b/mm/memory-failure.c index e0af336530c6..37feb9fec228 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c | |||
@@ -945,7 +945,7 @@ static int hwpoison_user_mappings(struct page *p, unsigned long pfn, | |||
945 | collect_procs(ppage, &tokill); | 945 | collect_procs(ppage, &tokill); |
946 | 946 | ||
947 | if (hpage != ppage) | 947 | if (hpage != ppage) |
948 | lock_page_nosync(ppage); | 948 | lock_page(ppage); |
949 | 949 | ||
950 | ret = try_to_unmap(ppage, ttu); | 950 | ret = try_to_unmap(ppage, ttu); |
951 | if (ret != SWAP_SUCCESS) | 951 | if (ret != SWAP_SUCCESS) |
@@ -1038,7 +1038,7 @@ int __memory_failure(unsigned long pfn, int trapno, int flags) | |||
1038 | * Check "just unpoisoned", "filter hit", and | 1038 | * Check "just unpoisoned", "filter hit", and |
1039 | * "race with other subpage." | 1039 | * "race with other subpage." |
1040 | */ | 1040 | */ |
1041 | lock_page_nosync(hpage); | 1041 | lock_page(hpage); |
1042 | if (!PageHWPoison(hpage) | 1042 | if (!PageHWPoison(hpage) |
1043 | || (hwpoison_filter(p) && TestClearPageHWPoison(p)) | 1043 | || (hwpoison_filter(p) && TestClearPageHWPoison(p)) |
1044 | || (p != hpage && TestSetPageHWPoison(hpage))) { | 1044 | || (p != hpage && TestSetPageHWPoison(hpage))) { |
@@ -1088,7 +1088,7 @@ int __memory_failure(unsigned long pfn, int trapno, int flags) | |||
1088 | * It's very difficult to mess with pages currently under IO | 1088 | * It's very difficult to mess with pages currently under IO |
1089 | * and in many cases impossible, so we just avoid it here. | 1089 | * and in many cases impossible, so we just avoid it here. |
1090 | */ | 1090 | */ |
1091 | lock_page_nosync(hpage); | 1091 | lock_page(hpage); |
1092 | 1092 | ||
1093 | /* | 1093 | /* |
1094 | * unpoison always clear PG_hwpoison inside page lock | 1094 | * unpoison always clear PG_hwpoison inside page lock |
@@ -1231,7 +1231,7 @@ int unpoison_memory(unsigned long pfn) | |||
1231 | return 0; | 1231 | return 0; |
1232 | } | 1232 | } |
1233 | 1233 | ||
1234 | lock_page_nosync(page); | 1234 | lock_page(page); |
1235 | /* | 1235 | /* |
1236 | * This test is racy because PG_hwpoison is set outside of page lock. | 1236 | * This test is racy because PG_hwpoison is set outside of page lock. |
1237 | * That's acceptable because that won't trigger kernel panic. Instead, | 1237 | * That's acceptable because that won't trigger kernel panic. Instead, |
diff --git a/mm/nommu.c b/mm/nommu.c index e629143f9440..cb86e7d5e7f5 100644 --- a/mm/nommu.c +++ b/mm/nommu.c | |||
@@ -1842,10 +1842,6 @@ int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, | |||
1842 | } | 1842 | } |
1843 | EXPORT_SYMBOL(remap_vmalloc_range); | 1843 | EXPORT_SYMBOL(remap_vmalloc_range); |
1844 | 1844 | ||
1845 | void swap_unplug_io_fn(struct backing_dev_info *bdi, struct page *page) | ||
1846 | { | ||
1847 | } | ||
1848 | |||
1849 | unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr, | 1845 | unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr, |
1850 | unsigned long len, unsigned long pgoff, unsigned long flags) | 1846 | unsigned long len, unsigned long pgoff, unsigned long flags) |
1851 | { | 1847 | { |
diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 632b46479c94..31f698862420 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c | |||
@@ -1040,11 +1040,17 @@ static int __writepage(struct page *page, struct writeback_control *wbc, | |||
1040 | int generic_writepages(struct address_space *mapping, | 1040 | int generic_writepages(struct address_space *mapping, |
1041 | struct writeback_control *wbc) | 1041 | struct writeback_control *wbc) |
1042 | { | 1042 | { |
1043 | struct blk_plug plug; | ||
1044 | int ret; | ||
1045 | |||
1043 | /* deal with chardevs and other special file */ | 1046 | /* deal with chardevs and other special file */ |
1044 | if (!mapping->a_ops->writepage) | 1047 | if (!mapping->a_ops->writepage) |
1045 | return 0; | 1048 | return 0; |
1046 | 1049 | ||
1047 | return write_cache_pages(mapping, wbc, __writepage, mapping); | 1050 | blk_start_plug(&plug); |
1051 | ret = write_cache_pages(mapping, wbc, __writepage, mapping); | ||
1052 | blk_finish_plug(&plug); | ||
1053 | return ret; | ||
1048 | } | 1054 | } |
1049 | 1055 | ||
1050 | EXPORT_SYMBOL(generic_writepages); | 1056 | EXPORT_SYMBOL(generic_writepages); |
@@ -1251,7 +1257,7 @@ int set_page_dirty_lock(struct page *page) | |||
1251 | { | 1257 | { |
1252 | int ret; | 1258 | int ret; |
1253 | 1259 | ||
1254 | lock_page_nosync(page); | 1260 | lock_page(page); |
1255 | ret = set_page_dirty(page); | 1261 | ret = set_page_dirty(page); |
1256 | unlock_page(page); | 1262 | unlock_page(page); |
1257 | return ret; | 1263 | return ret; |
diff --git a/mm/page_io.c b/mm/page_io.c index 2dee975bf469..dc76b4d0611e 100644 --- a/mm/page_io.c +++ b/mm/page_io.c | |||
@@ -106,7 +106,7 @@ int swap_writepage(struct page *page, struct writeback_control *wbc) | |||
106 | goto out; | 106 | goto out; |
107 | } | 107 | } |
108 | if (wbc->sync_mode == WB_SYNC_ALL) | 108 | if (wbc->sync_mode == WB_SYNC_ALL) |
109 | rw |= REQ_SYNC | REQ_UNPLUG; | 109 | rw |= REQ_SYNC; |
110 | count_vm_event(PSWPOUT); | 110 | count_vm_event(PSWPOUT); |
111 | set_page_writeback(page); | 111 | set_page_writeback(page); |
112 | unlock_page(page); | 112 | unlock_page(page); |
diff --git a/mm/readahead.c b/mm/readahead.c index 77506a291a2d..2c0cc489e288 100644 --- a/mm/readahead.c +++ b/mm/readahead.c | |||
@@ -109,9 +109,12 @@ EXPORT_SYMBOL(read_cache_pages); | |||
109 | static int read_pages(struct address_space *mapping, struct file *filp, | 109 | static int read_pages(struct address_space *mapping, struct file *filp, |
110 | struct list_head *pages, unsigned nr_pages) | 110 | struct list_head *pages, unsigned nr_pages) |
111 | { | 111 | { |
112 | struct blk_plug plug; | ||
112 | unsigned page_idx; | 113 | unsigned page_idx; |
113 | int ret; | 114 | int ret; |
114 | 115 | ||
116 | blk_start_plug(&plug); | ||
117 | |||
115 | if (mapping->a_ops->readpages) { | 118 | if (mapping->a_ops->readpages) { |
116 | ret = mapping->a_ops->readpages(filp, mapping, pages, nr_pages); | 119 | ret = mapping->a_ops->readpages(filp, mapping, pages, nr_pages); |
117 | /* Clean up the remaining pages */ | 120 | /* Clean up the remaining pages */ |
@@ -129,7 +132,10 @@ static int read_pages(struct address_space *mapping, struct file *filp, | |||
129 | page_cache_release(page); | 132 | page_cache_release(page); |
130 | } | 133 | } |
131 | ret = 0; | 134 | ret = 0; |
135 | |||
132 | out: | 136 | out: |
137 | blk_finish_plug(&plug); | ||
138 | |||
133 | return ret; | 139 | return ret; |
134 | } | 140 | } |
135 | 141 | ||
@@ -554,17 +560,5 @@ page_cache_async_readahead(struct address_space *mapping, | |||
554 | 560 | ||
555 | /* do read-ahead */ | 561 | /* do read-ahead */ |
556 | ondemand_readahead(mapping, ra, filp, true, offset, req_size); | 562 | ondemand_readahead(mapping, ra, filp, true, offset, req_size); |
557 | |||
558 | #ifdef CONFIG_BLOCK | ||
559 | /* | ||
560 | * Normally the current page is !uptodate and lock_page() will be | ||
561 | * immediately called to implicitly unplug the device. However this | ||
562 | * is not always true for RAID conifgurations, where data arrives | ||
563 | * not strictly in their submission order. In this case we need to | ||
564 | * explicitly kick off the IO. | ||
565 | */ | ||
566 | if (PageUptodate(page)) | ||
567 | blk_run_backing_dev(mapping->backing_dev_info, NULL); | ||
568 | #endif | ||
569 | } | 563 | } |
570 | EXPORT_SYMBOL_GPL(page_cache_async_readahead); | 564 | EXPORT_SYMBOL_GPL(page_cache_async_readahead); |
diff --git a/mm/shmem.c b/mm/shmem.c index 91ce9a1024d7..58da7c150ba6 100644 --- a/mm/shmem.c +++ b/mm/shmem.c | |||
@@ -224,7 +224,6 @@ static const struct vm_operations_struct shmem_vm_ops; | |||
224 | static struct backing_dev_info shmem_backing_dev_info __read_mostly = { | 224 | static struct backing_dev_info shmem_backing_dev_info __read_mostly = { |
225 | .ra_pages = 0, /* No readahead */ | 225 | .ra_pages = 0, /* No readahead */ |
226 | .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED, | 226 | .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED, |
227 | .unplug_io_fn = default_unplug_io_fn, | ||
228 | }; | 227 | }; |
229 | 228 | ||
230 | static LIST_HEAD(shmem_swaplist); | 229 | static LIST_HEAD(shmem_swaplist); |
diff --git a/mm/swap_state.c b/mm/swap_state.c index 5c8cfabbc9bc..46680461785b 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c | |||
@@ -24,12 +24,10 @@ | |||
24 | 24 | ||
25 | /* | 25 | /* |
26 | * swapper_space is a fiction, retained to simplify the path through | 26 | * swapper_space is a fiction, retained to simplify the path through |
27 | * vmscan's shrink_page_list, to make sync_page look nicer, and to allow | 27 | * vmscan's shrink_page_list. |
28 | * future use of radix_tree tags in the swap cache. | ||
29 | */ | 28 | */ |
30 | static const struct address_space_operations swap_aops = { | 29 | static const struct address_space_operations swap_aops = { |
31 | .writepage = swap_writepage, | 30 | .writepage = swap_writepage, |
32 | .sync_page = block_sync_page, | ||
33 | .set_page_dirty = __set_page_dirty_nobuffers, | 31 | .set_page_dirty = __set_page_dirty_nobuffers, |
34 | .migratepage = migrate_page, | 32 | .migratepage = migrate_page, |
35 | }; | 33 | }; |
@@ -37,7 +35,6 @@ static const struct address_space_operations swap_aops = { | |||
37 | static struct backing_dev_info swap_backing_dev_info = { | 35 | static struct backing_dev_info swap_backing_dev_info = { |
38 | .name = "swap", | 36 | .name = "swap", |
39 | .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED, | 37 | .capabilities = BDI_CAP_NO_ACCT_AND_WRITEBACK | BDI_CAP_SWAP_BACKED, |
40 | .unplug_io_fn = swap_unplug_io_fn, | ||
41 | }; | 38 | }; |
42 | 39 | ||
43 | struct address_space swapper_space = { | 40 | struct address_space swapper_space = { |
diff --git a/mm/swapfile.c b/mm/swapfile.c index 039e61677635..8c6b3ce38f09 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c | |||
@@ -95,39 +95,6 @@ __try_to_reclaim_swap(struct swap_info_struct *si, unsigned long offset) | |||
95 | } | 95 | } |
96 | 96 | ||
97 | /* | 97 | /* |
98 | * We need this because the bdev->unplug_fn can sleep and we cannot | ||
99 | * hold swap_lock while calling the unplug_fn. And swap_lock | ||
100 | * cannot be turned into a mutex. | ||
101 | */ | ||
102 | static DECLARE_RWSEM(swap_unplug_sem); | ||
103 | |||
104 | void swap_unplug_io_fn(struct backing_dev_info *unused_bdi, struct page *page) | ||
105 | { | ||
106 | swp_entry_t entry; | ||
107 | |||
108 | down_read(&swap_unplug_sem); | ||
109 | entry.val = page_private(page); | ||
110 | if (PageSwapCache(page)) { | ||
111 | struct block_device *bdev = swap_info[swp_type(entry)]->bdev; | ||
112 | struct backing_dev_info *bdi; | ||
113 | |||
114 | /* | ||
115 | * If the page is removed from swapcache from under us (with a | ||
116 | * racy try_to_unuse/swapoff) we need an additional reference | ||
117 | * count to avoid reading garbage from page_private(page) above. | ||
118 | * If the WARN_ON triggers during a swapoff it maybe the race | ||
119 | * condition and it's harmless. However if it triggers without | ||
120 | * swapoff it signals a problem. | ||
121 | */ | ||
122 | WARN_ON(page_count(page) <= 1); | ||
123 | |||
124 | bdi = bdev->bd_inode->i_mapping->backing_dev_info; | ||
125 | blk_run_backing_dev(bdi, page); | ||
126 | } | ||
127 | up_read(&swap_unplug_sem); | ||
128 | } | ||
129 | |||
130 | /* | ||
131 | * swapon tell device that all the old swap contents can be discarded, | 98 | * swapon tell device that all the old swap contents can be discarded, |
132 | * to allow the swap device to optimize its wear-levelling. | 99 | * to allow the swap device to optimize its wear-levelling. |
133 | */ | 100 | */ |
@@ -1662,10 +1629,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) | |||
1662 | goto out_dput; | 1629 | goto out_dput; |
1663 | } | 1630 | } |
1664 | 1631 | ||
1665 | /* wait for any unplug function to finish */ | ||
1666 | down_write(&swap_unplug_sem); | ||
1667 | up_write(&swap_unplug_sem); | ||
1668 | |||
1669 | destroy_swap_extents(p); | 1632 | destroy_swap_extents(p); |
1670 | if (p->flags & SWP_CONTINUED) | 1633 | if (p->flags & SWP_CONTINUED) |
1671 | free_swap_count_continuations(p); | 1634 | free_swap_count_continuations(p); |
diff --git a/mm/vmscan.c b/mm/vmscan.c index 060e4c191403..f73b8657c2d0 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c | |||
@@ -358,7 +358,7 @@ static int may_write_to_queue(struct backing_dev_info *bdi, | |||
358 | static void handle_write_error(struct address_space *mapping, | 358 | static void handle_write_error(struct address_space *mapping, |
359 | struct page *page, int error) | 359 | struct page *page, int error) |
360 | { | 360 | { |
361 | lock_page_nosync(page); | 361 | lock_page(page); |
362 | if (page_mapping(page) == mapping) | 362 | if (page_mapping(page) == mapping) |
363 | mapping_set_error(mapping, error); | 363 | mapping_set_error(mapping, error); |
364 | unlock_page(page); | 364 | unlock_page(page); |