aboutsummaryrefslogtreecommitdiffstats
path: root/block
diff options
context:
space:
mode:
authorJens Axboe <jaxboe@fusionio.com>2011-03-07 03:40:21 -0500
committerJens Axboe <jaxboe@fusionio.com>2011-03-07 03:40:21 -0500
commitb873c5d692d4d5453cceed18bb06c62bb1a73ac0 (patch)
treef9d5816bc26ced37f187a141ee6c74c7b8e00fcf /block
parenta60327107b56573c305ecc78e471dbdbb4d2f426 (diff)
parente83a46bbb1d4c03defd733a64b727632a40059ad (diff)
Merge branch 'block-for-2.6.39-core' of ssh://master.kernel.org/pub/scm/linux/kernel/git/tj/misc into for-2.6.39/core
Diffstat (limited to 'block')
-rw-r--r--block/blk-core.c18
-rw-r--r--block/blk-flush.c18
-rw-r--r--block/blk-lib.c2
-rw-r--r--block/blk-throttle.c29
-rw-r--r--block/cfq-iosched.c6
-rw-r--r--block/elevator.c4
-rw-r--r--block/genhd.c2
-rw-r--r--block/ioctl.c8
8 files changed, 49 insertions, 38 deletions
diff --git a/block/blk-core.c b/block/blk-core.c
index accff29ad674..74d496ccf4d7 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -342,7 +342,7 @@ void blk_start_queue(struct request_queue *q)
342 WARN_ON(!irqs_disabled()); 342 WARN_ON(!irqs_disabled());
343 343
344 queue_flag_clear(QUEUE_FLAG_STOPPED, q); 344 queue_flag_clear(QUEUE_FLAG_STOPPED, q);
345 __blk_run_queue(q); 345 __blk_run_queue(q, false);
346} 346}
347EXPORT_SYMBOL(blk_start_queue); 347EXPORT_SYMBOL(blk_start_queue);
348 348
@@ -396,13 +396,14 @@ EXPORT_SYMBOL(blk_sync_queue);
396/** 396/**
397 * __blk_run_queue - run a single device queue 397 * __blk_run_queue - run a single device queue
398 * @q: The queue to run 398 * @q: The queue to run
399 * @force_kblockd: Don't run @q->request_fn directly. Use kblockd.
399 * 400 *
400 * Description: 401 * Description:
401 * See @blk_run_queue. This variant must be called with the queue lock 402 * See @blk_run_queue. This variant must be called with the queue lock
402 * held and interrupts disabled. 403 * held and interrupts disabled.
403 * 404 *
404 */ 405 */
405void __blk_run_queue(struct request_queue *q) 406void __blk_run_queue(struct request_queue *q, bool force_kblockd)
406{ 407{
407 blk_remove_plug(q); 408 blk_remove_plug(q);
408 409
@@ -416,7 +417,7 @@ void __blk_run_queue(struct request_queue *q)
416 * Only recurse once to avoid overrunning the stack, let the unplug 417 * Only recurse once to avoid overrunning the stack, let the unplug
417 * handling reinvoke the handler shortly if we already got there. 418 * handling reinvoke the handler shortly if we already got there.
418 */ 419 */
419 if (!queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) { 420 if (!force_kblockd && !queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
420 q->request_fn(q); 421 q->request_fn(q);
421 queue_flag_clear(QUEUE_FLAG_REENTER, q); 422 queue_flag_clear(QUEUE_FLAG_REENTER, q);
422 } else { 423 } else {
@@ -439,7 +440,7 @@ void blk_run_queue(struct request_queue *q)
439 unsigned long flags; 440 unsigned long flags;
440 441
441 spin_lock_irqsave(q->queue_lock, flags); 442 spin_lock_irqsave(q->queue_lock, flags);
442 __blk_run_queue(q); 443 __blk_run_queue(q, false);
443 spin_unlock_irqrestore(q->queue_lock, flags); 444 spin_unlock_irqrestore(q->queue_lock, flags);
444} 445}
445EXPORT_SYMBOL(blk_run_queue); 446EXPORT_SYMBOL(blk_run_queue);
@@ -1085,7 +1086,7 @@ void blk_insert_request(struct request_queue *q, struct request *rq,
1085 1086
1086 drive_stat_acct(rq, 1); 1087 drive_stat_acct(rq, 1);
1087 __elv_add_request(q, rq, where, 0); 1088 __elv_add_request(q, rq, where, 0);
1088 __blk_run_queue(q); 1089 __blk_run_queue(q, false);
1089 spin_unlock_irqrestore(q->queue_lock, flags); 1090 spin_unlock_irqrestore(q->queue_lock, flags);
1090} 1091}
1091EXPORT_SYMBOL(blk_insert_request); 1092EXPORT_SYMBOL(blk_insert_request);
@@ -2642,13 +2643,6 @@ int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
2642} 2643}
2643EXPORT_SYMBOL(kblockd_schedule_work); 2644EXPORT_SYMBOL(kblockd_schedule_work);
2644 2645
2645int kblockd_schedule_delayed_work(struct request_queue *q,
2646 struct delayed_work *dwork, unsigned long delay)
2647{
2648 return queue_delayed_work(kblockd_workqueue, dwork, delay);
2649}
2650EXPORT_SYMBOL(kblockd_schedule_delayed_work);
2651
2652int __init blk_dev_init(void) 2646int __init blk_dev_init(void)
2653{ 2647{
2654 BUILD_BUG_ON(__REQ_NR_BITS > 8 * 2648 BUILD_BUG_ON(__REQ_NR_BITS > 8 *
diff --git a/block/blk-flush.c b/block/blk-flush.c
index a867e3f524f3..0bd8c9c5d6e5 100644
--- a/block/blk-flush.c
+++ b/block/blk-flush.c
@@ -212,9 +212,14 @@ static void flush_end_io(struct request *flush_rq, int error)
212 queued |= blk_flush_complete_seq(rq, seq, error); 212 queued |= blk_flush_complete_seq(rq, seq, error);
213 } 213 }
214 214
215 /* after populating an empty queue, kick it to avoid stall */ 215 /*
216 * Moving a request silently to empty queue_head may stall the
217 * queue. Kick the queue in those cases. This function is called
218 * from request completion path and calling directly into
219 * request_fn may confuse the driver. Always use kblockd.
220 */
216 if (queued && was_empty) 221 if (queued && was_empty)
217 __blk_run_queue(q); 222 __blk_run_queue(q, true);
218} 223}
219 224
220/** 225/**
@@ -257,7 +262,7 @@ static bool blk_kick_flush(struct request_queue *q)
257 q->flush_rq.end_io = flush_end_io; 262 q->flush_rq.end_io = flush_end_io;
258 263
259 q->flush_pending_idx ^= 1; 264 q->flush_pending_idx ^= 1;
260 elv_insert(q, &q->flush_rq, ELEVATOR_INSERT_FRONT); 265 elv_insert(q, &q->flush_rq, ELEVATOR_INSERT_REQUEUE);
261 return true; 266 return true;
262} 267}
263 268
@@ -266,9 +271,12 @@ static void flush_data_end_io(struct request *rq, int error)
266 struct request_queue *q = rq->q; 271 struct request_queue *q = rq->q;
267 bool was_empty = elv_queue_empty(q); 272 bool was_empty = elv_queue_empty(q);
268 273
269 /* after populating an empty queue, kick it to avoid stall */ 274 /*
275 * After populating an empty queue, kick it to avoid stall. Read
276 * the comment in flush_end_io().
277 */
270 if (blk_flush_complete_seq(rq, REQ_FSEQ_DATA, error) && was_empty) 278 if (blk_flush_complete_seq(rq, REQ_FSEQ_DATA, error) && was_empty)
271 __blk_run_queue(q); 279 __blk_run_queue(q, true);
272} 280}
273 281
274/** 282/**
diff --git a/block/blk-lib.c b/block/blk-lib.c
index 1a320d2406b0..eec78becb355 100644
--- a/block/blk-lib.c
+++ b/block/blk-lib.c
@@ -132,7 +132,7 @@ static void bio_batch_end_io(struct bio *bio, int err)
132} 132}
133 133
134/** 134/**
135 * blkdev_issue_zeroout generate number of zero filed write bios 135 * blkdev_issue_zeroout - generate number of zero filed write bios
136 * @bdev: blockdev to issue 136 * @bdev: blockdev to issue
137 * @sector: start sector 137 * @sector: start sector
138 * @nr_sects: number of sectors to write 138 * @nr_sects: number of sectors to write
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index c0f623742165..061dee66e2a6 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -20,6 +20,11 @@ static int throtl_quantum = 32;
20/* Throttling is performed over 100ms slice and after that slice is renewed */ 20/* Throttling is performed over 100ms slice and after that slice is renewed */
21static unsigned long throtl_slice = HZ/10; /* 100 ms */ 21static unsigned long throtl_slice = HZ/10; /* 100 ms */
22 22
23/* A workqueue to queue throttle related work */
24static struct workqueue_struct *kthrotld_workqueue;
25static void throtl_schedule_delayed_work(struct throtl_data *td,
26 unsigned long delay);
27
23struct throtl_rb_root { 28struct throtl_rb_root {
24 struct rb_root rb; 29 struct rb_root rb;
25 struct rb_node *left; 30 struct rb_node *left;
@@ -345,10 +350,9 @@ static void throtl_schedule_next_dispatch(struct throtl_data *td)
345 update_min_dispatch_time(st); 350 update_min_dispatch_time(st);
346 351
347 if (time_before_eq(st->min_disptime, jiffies)) 352 if (time_before_eq(st->min_disptime, jiffies))
348 throtl_schedule_delayed_work(td->queue, 0); 353 throtl_schedule_delayed_work(td, 0);
349 else 354 else
350 throtl_schedule_delayed_work(td->queue, 355 throtl_schedule_delayed_work(td, (st->min_disptime - jiffies));
351 (st->min_disptime - jiffies));
352} 356}
353 357
354static inline void 358static inline void
@@ -815,10 +819,10 @@ void blk_throtl_work(struct work_struct *work)
815} 819}
816 820
817/* Call with queue lock held */ 821/* Call with queue lock held */
818void throtl_schedule_delayed_work(struct request_queue *q, unsigned long delay) 822static void
823throtl_schedule_delayed_work(struct throtl_data *td, unsigned long delay)
819{ 824{
820 825
821 struct throtl_data *td = q->td;
822 struct delayed_work *dwork = &td->throtl_work; 826 struct delayed_work *dwork = &td->throtl_work;
823 827
824 if (total_nr_queued(td) > 0) { 828 if (total_nr_queued(td) > 0) {
@@ -827,12 +831,11 @@ void throtl_schedule_delayed_work(struct request_queue *q, unsigned long delay)
827 * Cancel that and schedule a new one. 831 * Cancel that and schedule a new one.
828 */ 832 */
829 __cancel_delayed_work(dwork); 833 __cancel_delayed_work(dwork);
830 kblockd_schedule_delayed_work(q, dwork, delay); 834 queue_delayed_work(kthrotld_workqueue, dwork, delay);
831 throtl_log(td, "schedule work. delay=%lu jiffies=%lu", 835 throtl_log(td, "schedule work. delay=%lu jiffies=%lu",
832 delay, jiffies); 836 delay, jiffies);
833 } 837 }
834} 838}
835EXPORT_SYMBOL(throtl_schedule_delayed_work);
836 839
837static void 840static void
838throtl_destroy_tg(struct throtl_data *td, struct throtl_grp *tg) 841throtl_destroy_tg(struct throtl_data *td, struct throtl_grp *tg)
@@ -920,7 +923,7 @@ static void throtl_update_blkio_group_read_bps(void *key,
920 smp_mb__after_atomic_inc(); 923 smp_mb__after_atomic_inc();
921 924
922 /* Schedule a work now to process the limit change */ 925 /* Schedule a work now to process the limit change */
923 throtl_schedule_delayed_work(td->queue, 0); 926 throtl_schedule_delayed_work(td, 0);
924} 927}
925 928
926static void throtl_update_blkio_group_write_bps(void *key, 929static void throtl_update_blkio_group_write_bps(void *key,
@@ -934,7 +937,7 @@ static void throtl_update_blkio_group_write_bps(void *key,
934 smp_mb__before_atomic_inc(); 937 smp_mb__before_atomic_inc();
935 atomic_inc(&td->limits_changed); 938 atomic_inc(&td->limits_changed);
936 smp_mb__after_atomic_inc(); 939 smp_mb__after_atomic_inc();
937 throtl_schedule_delayed_work(td->queue, 0); 940 throtl_schedule_delayed_work(td, 0);
938} 941}
939 942
940static void throtl_update_blkio_group_read_iops(void *key, 943static void throtl_update_blkio_group_read_iops(void *key,
@@ -948,7 +951,7 @@ static void throtl_update_blkio_group_read_iops(void *key,
948 smp_mb__before_atomic_inc(); 951 smp_mb__before_atomic_inc();
949 atomic_inc(&td->limits_changed); 952 atomic_inc(&td->limits_changed);
950 smp_mb__after_atomic_inc(); 953 smp_mb__after_atomic_inc();
951 throtl_schedule_delayed_work(td->queue, 0); 954 throtl_schedule_delayed_work(td, 0);
952} 955}
953 956
954static void throtl_update_blkio_group_write_iops(void *key, 957static void throtl_update_blkio_group_write_iops(void *key,
@@ -962,7 +965,7 @@ static void throtl_update_blkio_group_write_iops(void *key,
962 smp_mb__before_atomic_inc(); 965 smp_mb__before_atomic_inc();
963 atomic_inc(&td->limits_changed); 966 atomic_inc(&td->limits_changed);
964 smp_mb__after_atomic_inc(); 967 smp_mb__after_atomic_inc();
965 throtl_schedule_delayed_work(td->queue, 0); 968 throtl_schedule_delayed_work(td, 0);
966} 969}
967 970
968static void throtl_shutdown_wq(struct request_queue *q) 971static void throtl_shutdown_wq(struct request_queue *q)
@@ -1135,6 +1138,10 @@ void blk_throtl_exit(struct request_queue *q)
1135 1138
1136static int __init throtl_init(void) 1139static int __init throtl_init(void)
1137{ 1140{
1141 kthrotld_workqueue = alloc_workqueue("kthrotld", WQ_MEM_RECLAIM, 0);
1142 if (!kthrotld_workqueue)
1143 panic("Failed to create kthrotld\n");
1144
1138 blkio_policy_register(&blkio_policy_throtl); 1145 blkio_policy_register(&blkio_policy_throtl);
1139 return 0; 1146 return 0;
1140} 1147}
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 938ae52aa927..9697053f80bc 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -3344,7 +3344,7 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
3344 cfqd->busy_queues > 1) { 3344 cfqd->busy_queues > 1) {
3345 cfq_del_timer(cfqd, cfqq); 3345 cfq_del_timer(cfqd, cfqq);
3346 cfq_clear_cfqq_wait_request(cfqq); 3346 cfq_clear_cfqq_wait_request(cfqq);
3347 __blk_run_queue(cfqd->queue); 3347 __blk_run_queue(cfqd->queue, false);
3348 } else { 3348 } else {
3349 cfq_blkiocg_update_idle_time_stats( 3349 cfq_blkiocg_update_idle_time_stats(
3350 &cfqq->cfqg->blkg); 3350 &cfqq->cfqg->blkg);
@@ -3359,7 +3359,7 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
3359 * this new queue is RT and the current one is BE 3359 * this new queue is RT and the current one is BE
3360 */ 3360 */
3361 cfq_preempt_queue(cfqd, cfqq); 3361 cfq_preempt_queue(cfqd, cfqq);
3362 __blk_run_queue(cfqd->queue); 3362 __blk_run_queue(cfqd->queue, false);
3363 } 3363 }
3364} 3364}
3365 3365
@@ -3719,7 +3719,7 @@ static void cfq_kick_queue(struct work_struct *work)
3719 struct request_queue *q = cfqd->queue; 3719 struct request_queue *q = cfqd->queue;
3720 3720
3721 spin_lock_irq(q->queue_lock); 3721 spin_lock_irq(q->queue_lock);
3722 __blk_run_queue(cfqd->queue); 3722 __blk_run_queue(cfqd->queue, false);
3723 spin_unlock_irq(q->queue_lock); 3723 spin_unlock_irq(q->queue_lock);
3724} 3724}
3725 3725
diff --git a/block/elevator.c b/block/elevator.c
index f98e92edc937..fabf3675c913 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -602,7 +602,7 @@ void elv_quiesce_start(struct request_queue *q)
602 */ 602 */
603 elv_drain_elevator(q); 603 elv_drain_elevator(q);
604 while (q->rq.elvpriv) { 604 while (q->rq.elvpriv) {
605 __blk_run_queue(q); 605 __blk_run_queue(q, false);
606 spin_unlock_irq(q->queue_lock); 606 spin_unlock_irq(q->queue_lock);
607 msleep(10); 607 msleep(10);
608 spin_lock_irq(q->queue_lock); 608 spin_lock_irq(q->queue_lock);
@@ -651,7 +651,7 @@ void elv_insert(struct request_queue *q, struct request *rq, int where)
651 * with anything. There's no point in delaying queue 651 * with anything. There's no point in delaying queue
652 * processing. 652 * processing.
653 */ 653 */
654 __blk_run_queue(q); 654 __blk_run_queue(q, false);
655 break; 655 break;
656 656
657 case ELEVATOR_INSERT_SORT: 657 case ELEVATOR_INSERT_SORT:
diff --git a/block/genhd.c b/block/genhd.c
index 73d85a8e3c85..3e2b57b55e38 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -1355,7 +1355,7 @@ int invalidate_partition(struct gendisk *disk, int partno)
1355 struct block_device *bdev = bdget_disk(disk, partno); 1355 struct block_device *bdev = bdget_disk(disk, partno);
1356 if (bdev) { 1356 if (bdev) {
1357 fsync_bdev(bdev); 1357 fsync_bdev(bdev);
1358 res = __invalidate_device(bdev); 1358 res = __invalidate_device(bdev, true);
1359 bdput(bdev); 1359 bdput(bdev);
1360 } 1360 }
1361 return res; 1361 return res;
diff --git a/block/ioctl.c b/block/ioctl.c
index 9049d460fa89..1124cd297263 100644
--- a/block/ioctl.c
+++ b/block/ioctl.c
@@ -294,9 +294,11 @@ int blkdev_ioctl(struct block_device *bdev, fmode_t mode, unsigned cmd,
294 return -EINVAL; 294 return -EINVAL;
295 if (get_user(n, (int __user *) arg)) 295 if (get_user(n, (int __user *) arg))
296 return -EFAULT; 296 return -EFAULT;
297 if (!(mode & FMODE_EXCL) && 297 if (!(mode & FMODE_EXCL)) {
298 blkdev_get(bdev, mode | FMODE_EXCL, &bdev) < 0) 298 bdgrab(bdev);
299 return -EBUSY; 299 if (blkdev_get(bdev, mode | FMODE_EXCL, &bdev) < 0)
300 return -EBUSY;
301 }
300 ret = set_blocksize(bdev, n); 302 ret = set_blocksize(bdev, n);
301 if (!(mode & FMODE_EXCL)) 303 if (!(mode & FMODE_EXCL))
302 blkdev_put(bdev, mode | FMODE_EXCL); 304 blkdev_put(bdev, mode | FMODE_EXCL);