aboutsummaryrefslogtreecommitdiffstats
path: root/block/blk-core.c
diff options
context:
space:
mode:
Diffstat (limited to 'block/blk-core.c')
-rw-r--r--block/blk-core.c134
1 files changed, 87 insertions, 47 deletions
diff --git a/block/blk-core.c b/block/blk-core.c
index 77411486b11..881fe44ec7d 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -64,13 +64,15 @@ static void drive_stat_acct(struct request *rq, int new_io)
64 return; 64 return;
65 65
66 cpu = part_stat_lock(); 66 cpu = part_stat_lock();
67 part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
68 67
69 if (!new_io) 68 if (!new_io) {
69 part = rq->part;
70 part_stat_inc(cpu, part, merges[rw]); 70 part_stat_inc(cpu, part, merges[rw]);
71 else { 71 } else {
72 part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
72 part_round_stats(cpu, part); 73 part_round_stats(cpu, part);
73 part_inc_in_flight(part, rw); 74 part_inc_in_flight(part, rw);
75 rq->part = part;
74 } 76 }
75 77
76 part_stat_unlock(); 78 part_stat_unlock();
@@ -128,6 +130,7 @@ void blk_rq_init(struct request_queue *q, struct request *rq)
128 rq->ref_count = 1; 130 rq->ref_count = 1;
129 rq->start_time = jiffies; 131 rq->start_time = jiffies;
130 set_start_time_ns(rq); 132 set_start_time_ns(rq);
133 rq->part = NULL;
131} 134}
132EXPORT_SYMBOL(blk_rq_init); 135EXPORT_SYMBOL(blk_rq_init);
133 136
@@ -136,7 +139,7 @@ static void req_bio_endio(struct request *rq, struct bio *bio,
136{ 139{
137 struct request_queue *q = rq->q; 140 struct request_queue *q = rq->q;
138 141
139 if (&q->bar_rq != rq) { 142 if (&q->flush_rq != rq) {
140 if (error) 143 if (error)
141 clear_bit(BIO_UPTODATE, &bio->bi_flags); 144 clear_bit(BIO_UPTODATE, &bio->bi_flags);
142 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) 145 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
@@ -160,13 +163,12 @@ static void req_bio_endio(struct request *rq, struct bio *bio,
160 if (bio->bi_size == 0) 163 if (bio->bi_size == 0)
161 bio_endio(bio, error); 164 bio_endio(bio, error);
162 } else { 165 } else {
163
164 /* 166 /*
165 * Okay, this is the barrier request in progress, just 167 * Okay, this is the sequenced flush request in
166 * record the error; 168 * progress, just record the error;
167 */ 169 */
168 if (error && !q->orderr) 170 if (error && !q->flush_err)
169 q->orderr = error; 171 q->flush_err = error;
170 } 172 }
171} 173}
172 174
@@ -382,6 +384,7 @@ void blk_sync_queue(struct request_queue *q)
382 del_timer_sync(&q->unplug_timer); 384 del_timer_sync(&q->unplug_timer);
383 del_timer_sync(&q->timeout); 385 del_timer_sync(&q->timeout);
384 cancel_work_sync(&q->unplug_work); 386 cancel_work_sync(&q->unplug_work);
387 throtl_shutdown_timer_wq(q);
385} 388}
386EXPORT_SYMBOL(blk_sync_queue); 389EXPORT_SYMBOL(blk_sync_queue);
387 390
@@ -515,11 +518,17 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
515 return NULL; 518 return NULL;
516 } 519 }
517 520
521 if (blk_throtl_init(q)) {
522 kmem_cache_free(blk_requestq_cachep, q);
523 return NULL;
524 }
525
518 setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, 526 setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
519 laptop_mode_timer_fn, (unsigned long) q); 527 laptop_mode_timer_fn, (unsigned long) q);
520 init_timer(&q->unplug_timer); 528 init_timer(&q->unplug_timer);
521 setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q); 529 setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
522 INIT_LIST_HEAD(&q->timeout_list); 530 INIT_LIST_HEAD(&q->timeout_list);
531 INIT_LIST_HEAD(&q->pending_flushes);
523 INIT_WORK(&q->unplug_work, blk_unplug_work); 532 INIT_WORK(&q->unplug_work, blk_unplug_work);
524 533
525 kobject_init(&q->kobj, &blk_queue_ktype); 534 kobject_init(&q->kobj, &blk_queue_ktype);
@@ -796,11 +805,16 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
796 rl->starved[is_sync] = 0; 805 rl->starved[is_sync] = 0;
797 806
798 priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags); 807 priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
799 if (priv) 808 if (priv) {
800 rl->elvpriv++; 809 rl->elvpriv++;
801 810
802 if (blk_queue_io_stat(q)) 811 /*
803 rw_flags |= REQ_IO_STAT; 812 * Don't do stats for non-priv requests
813 */
814 if (blk_queue_io_stat(q))
815 rw_flags |= REQ_IO_STAT;
816 }
817
804 spin_unlock_irq(q->queue_lock); 818 spin_unlock_irq(q->queue_lock);
805 819
806 rq = blk_alloc_request(q, rw_flags, priv, gfp_mask); 820 rq = blk_alloc_request(q, rw_flags, priv, gfp_mask);
@@ -1037,22 +1051,6 @@ void blk_insert_request(struct request_queue *q, struct request *rq,
1037} 1051}
1038EXPORT_SYMBOL(blk_insert_request); 1052EXPORT_SYMBOL(blk_insert_request);
1039 1053
1040/*
1041 * add-request adds a request to the linked list.
1042 * queue lock is held and interrupts disabled, as we muck with the
1043 * request queue list.
1044 */
1045static inline void add_request(struct request_queue *q, struct request *req)
1046{
1047 drive_stat_acct(req, 1);
1048
1049 /*
1050 * elevator indicated where it wants this request to be
1051 * inserted at elevator_merge time
1052 */
1053 __elv_add_request(q, req, ELEVATOR_INSERT_SORT, 0);
1054}
1055
1056static void part_round_stats_single(int cpu, struct hd_struct *part, 1054static void part_round_stats_single(int cpu, struct hd_struct *part,
1057 unsigned long now) 1055 unsigned long now)
1058{ 1056{
@@ -1198,16 +1196,19 @@ static int __make_request(struct request_queue *q, struct bio *bio)
1198 int el_ret; 1196 int el_ret;
1199 unsigned int bytes = bio->bi_size; 1197 unsigned int bytes = bio->bi_size;
1200 const unsigned short prio = bio_prio(bio); 1198 const unsigned short prio = bio_prio(bio);
1201 const bool sync = (bio->bi_rw & REQ_SYNC); 1199 const bool sync = !!(bio->bi_rw & REQ_SYNC);
1202 const bool unplug = (bio->bi_rw & REQ_UNPLUG); 1200 const bool unplug = !!(bio->bi_rw & REQ_UNPLUG);
1203 const unsigned int ff = bio->bi_rw & REQ_FAILFAST_MASK; 1201 const unsigned long ff = bio->bi_rw & REQ_FAILFAST_MASK;
1202 int where = ELEVATOR_INSERT_SORT;
1204 int rw_flags; 1203 int rw_flags;
1205 1204
1206 if ((bio->bi_rw & REQ_HARDBARRIER) && 1205 /* REQ_HARDBARRIER is no more */
1207 (q->next_ordered == QUEUE_ORDERED_NONE)) { 1206 if (WARN_ONCE(bio->bi_rw & REQ_HARDBARRIER,
1207 "block: HARDBARRIER is deprecated, use FLUSH/FUA instead\n")) {
1208 bio_endio(bio, -EOPNOTSUPP); 1208 bio_endio(bio, -EOPNOTSUPP);
1209 return 0; 1209 return 0;
1210 } 1210 }
1211
1211 /* 1212 /*
1212 * low level driver can indicate that it wants pages above a 1213 * low level driver can indicate that it wants pages above a
1213 * certain limit bounced to low memory (ie for highmem, or even 1214 * certain limit bounced to low memory (ie for highmem, or even
@@ -1217,7 +1218,12 @@ static int __make_request(struct request_queue *q, struct bio *bio)
1217 1218
1218 spin_lock_irq(q->queue_lock); 1219 spin_lock_irq(q->queue_lock);
1219 1220
1220 if (unlikely((bio->bi_rw & REQ_HARDBARRIER)) || elv_queue_empty(q)) 1221 if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
1222 where = ELEVATOR_INSERT_FRONT;
1223 goto get_rq;
1224 }
1225
1226 if (elv_queue_empty(q))
1221 goto get_rq; 1227 goto get_rq;
1222 1228
1223 el_ret = elv_merge(q, &req, bio); 1229 el_ret = elv_merge(q, &req, bio);
@@ -1314,7 +1320,10 @@ get_rq:
1314 req->cpu = blk_cpu_to_group(smp_processor_id()); 1320 req->cpu = blk_cpu_to_group(smp_processor_id());
1315 if (queue_should_plug(q) && elv_queue_empty(q)) 1321 if (queue_should_plug(q) && elv_queue_empty(q))
1316 blk_plug_device(q); 1322 blk_plug_device(q);
1317 add_request(q, req); 1323
1324 /* insert the request into the elevator */
1325 drive_stat_acct(req, 1);
1326 __elv_add_request(q, req, where, 0);
1318out: 1327out:
1319 if (unplug || !queue_should_plug(q)) 1328 if (unplug || !queue_should_plug(q))
1320 __generic_unplug_device(q); 1329 __generic_unplug_device(q);
@@ -1514,11 +1523,36 @@ static inline void __generic_make_request(struct bio *bio)
1514 if (bio_check_eod(bio, nr_sectors)) 1523 if (bio_check_eod(bio, nr_sectors))
1515 goto end_io; 1524 goto end_io;
1516 1525
1517 if ((bio->bi_rw & REQ_DISCARD) && !blk_queue_discard(q)) { 1526 /*
1527 * Filter flush bio's early so that make_request based
1528 * drivers without flush support don't have to worry
1529 * about them.
1530 */
1531 if ((bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && !q->flush_flags) {
1532 bio->bi_rw &= ~(REQ_FLUSH | REQ_FUA);
1533 if (!nr_sectors) {
1534 err = 0;
1535 goto end_io;
1536 }
1537 }
1538
1539 if ((bio->bi_rw & REQ_DISCARD) &&
1540 (!blk_queue_discard(q) ||
1541 ((bio->bi_rw & REQ_SECURE) &&
1542 !blk_queue_secdiscard(q)))) {
1518 err = -EOPNOTSUPP; 1543 err = -EOPNOTSUPP;
1519 goto end_io; 1544 goto end_io;
1520 } 1545 }
1521 1546
1547 blk_throtl_bio(q, &bio);
1548
1549 /*
1550 * If bio = NULL, bio has been throttled and will be submitted
1551 * later.
1552 */
1553 if (!bio)
1554 break;
1555
1522 trace_block_bio_queue(q, bio); 1556 trace_block_bio_queue(q, bio);
1523 1557
1524 ret = q->make_request_fn(q, bio); 1558 ret = q->make_request_fn(q, bio);
@@ -1609,11 +1643,12 @@ void submit_bio(int rw, struct bio *bio)
1609 1643
1610 if (unlikely(block_dump)) { 1644 if (unlikely(block_dump)) {
1611 char b[BDEVNAME_SIZE]; 1645 char b[BDEVNAME_SIZE];
1612 printk(KERN_DEBUG "%s(%d): %s block %Lu on %s\n", 1646 printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
1613 current->comm, task_pid_nr(current), 1647 current->comm, task_pid_nr(current),
1614 (rw & WRITE) ? "WRITE" : "READ", 1648 (rw & WRITE) ? "WRITE" : "READ",
1615 (unsigned long long)bio->bi_sector, 1649 (unsigned long long)bio->bi_sector,
1616 bdevname(bio->bi_bdev, b)); 1650 bdevname(bio->bi_bdev, b),
1651 count);
1617 } 1652 }
1618 } 1653 }
1619 1654
@@ -1756,7 +1791,7 @@ static void blk_account_io_completion(struct request *req, unsigned int bytes)
1756 int cpu; 1791 int cpu;
1757 1792
1758 cpu = part_stat_lock(); 1793 cpu = part_stat_lock();
1759 part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req)); 1794 part = req->part;
1760 part_stat_add(cpu, part, sectors[rw], bytes >> 9); 1795 part_stat_add(cpu, part, sectors[rw], bytes >> 9);
1761 part_stat_unlock(); 1796 part_stat_unlock();
1762 } 1797 }
@@ -1765,18 +1800,18 @@ static void blk_account_io_completion(struct request *req, unsigned int bytes)
1765static void blk_account_io_done(struct request *req) 1800static void blk_account_io_done(struct request *req)
1766{ 1801{
1767 /* 1802 /*
1768 * Account IO completion. bar_rq isn't accounted as a normal 1803 * Account IO completion. flush_rq isn't accounted as a
1769 * IO on queueing nor completion. Accounting the containing 1804 * normal IO on queueing nor completion. Accounting the
1770 * request is enough. 1805 * containing request is enough.
1771 */ 1806 */
1772 if (blk_do_io_stat(req) && req != &req->q->bar_rq) { 1807 if (blk_do_io_stat(req) && req != &req->q->flush_rq) {
1773 unsigned long duration = jiffies - req->start_time; 1808 unsigned long duration = jiffies - req->start_time;
1774 const int rw = rq_data_dir(req); 1809 const int rw = rq_data_dir(req);
1775 struct hd_struct *part; 1810 struct hd_struct *part;
1776 int cpu; 1811 int cpu;
1777 1812
1778 cpu = part_stat_lock(); 1813 cpu = part_stat_lock();
1779 part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req)); 1814 part = req->part;
1780 1815
1781 part_stat_inc(cpu, part, ios[rw]); 1816 part_stat_inc(cpu, part, ios[rw]);
1782 part_stat_add(cpu, part, ticks[rw], duration); 1817 part_stat_add(cpu, part, ticks[rw], duration);
@@ -2494,9 +2529,7 @@ EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
2494static void __blk_rq_prep_clone(struct request *dst, struct request *src) 2529static void __blk_rq_prep_clone(struct request *dst, struct request *src)
2495{ 2530{
2496 dst->cpu = src->cpu; 2531 dst->cpu = src->cpu;
2497 dst->cmd_flags = (rq_data_dir(src) | REQ_NOMERGE); 2532 dst->cmd_flags = (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE;
2498 if (src->cmd_flags & REQ_DISCARD)
2499 dst->cmd_flags |= REQ_DISCARD;
2500 dst->cmd_type = src->cmd_type; 2533 dst->cmd_type = src->cmd_type;
2501 dst->__sector = blk_rq_pos(src); 2534 dst->__sector = blk_rq_pos(src);
2502 dst->__data_len = blk_rq_bytes(src); 2535 dst->__data_len = blk_rq_bytes(src);
@@ -2576,6 +2609,13 @@ int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
2576} 2609}
2577EXPORT_SYMBOL(kblockd_schedule_work); 2610EXPORT_SYMBOL(kblockd_schedule_work);
2578 2611
2612int kblockd_schedule_delayed_work(struct request_queue *q,
2613 struct delayed_work *dwork, unsigned long delay)
2614{
2615 return queue_delayed_work(kblockd_workqueue, dwork, delay);
2616}
2617EXPORT_SYMBOL(kblockd_schedule_delayed_work);
2618
2579int __init blk_dev_init(void) 2619int __init blk_dev_init(void)
2580{ 2620{
2581 BUILD_BUG_ON(__REQ_NR_BITS > 8 * 2621 BUILD_BUG_ON(__REQ_NR_BITS > 8 *