diff options
Diffstat (limited to 'block/blk-core.c')
-rw-r--r-- | block/blk-core.c | 134 |
1 files changed, 87 insertions, 47 deletions
diff --git a/block/blk-core.c b/block/blk-core.c index 77411486b11..881fe44ec7d 100644 --- a/block/blk-core.c +++ b/block/blk-core.c | |||
@@ -64,13 +64,15 @@ static void drive_stat_acct(struct request *rq, int new_io) | |||
64 | return; | 64 | return; |
65 | 65 | ||
66 | cpu = part_stat_lock(); | 66 | cpu = part_stat_lock(); |
67 | part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq)); | ||
68 | 67 | ||
69 | if (!new_io) | 68 | if (!new_io) { |
69 | part = rq->part; | ||
70 | part_stat_inc(cpu, part, merges[rw]); | 70 | part_stat_inc(cpu, part, merges[rw]); |
71 | else { | 71 | } else { |
72 | part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq)); | ||
72 | part_round_stats(cpu, part); | 73 | part_round_stats(cpu, part); |
73 | part_inc_in_flight(part, rw); | 74 | part_inc_in_flight(part, rw); |
75 | rq->part = part; | ||
74 | } | 76 | } |
75 | 77 | ||
76 | part_stat_unlock(); | 78 | part_stat_unlock(); |
@@ -128,6 +130,7 @@ void blk_rq_init(struct request_queue *q, struct request *rq) | |||
128 | rq->ref_count = 1; | 130 | rq->ref_count = 1; |
129 | rq->start_time = jiffies; | 131 | rq->start_time = jiffies; |
130 | set_start_time_ns(rq); | 132 | set_start_time_ns(rq); |
133 | rq->part = NULL; | ||
131 | } | 134 | } |
132 | EXPORT_SYMBOL(blk_rq_init); | 135 | EXPORT_SYMBOL(blk_rq_init); |
133 | 136 | ||
@@ -136,7 +139,7 @@ static void req_bio_endio(struct request *rq, struct bio *bio, | |||
136 | { | 139 | { |
137 | struct request_queue *q = rq->q; | 140 | struct request_queue *q = rq->q; |
138 | 141 | ||
139 | if (&q->bar_rq != rq) { | 142 | if (&q->flush_rq != rq) { |
140 | if (error) | 143 | if (error) |
141 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | 144 | clear_bit(BIO_UPTODATE, &bio->bi_flags); |
142 | else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) | 145 | else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) |
@@ -160,13 +163,12 @@ static void req_bio_endio(struct request *rq, struct bio *bio, | |||
160 | if (bio->bi_size == 0) | 163 | if (bio->bi_size == 0) |
161 | bio_endio(bio, error); | 164 | bio_endio(bio, error); |
162 | } else { | 165 | } else { |
163 | |||
164 | /* | 166 | /* |
165 | * Okay, this is the barrier request in progress, just | 167 | * Okay, this is the sequenced flush request in |
166 | * record the error; | 168 | * progress, just record the error; |
167 | */ | 169 | */ |
168 | if (error && !q->orderr) | 170 | if (error && !q->flush_err) |
169 | q->orderr = error; | 171 | q->flush_err = error; |
170 | } | 172 | } |
171 | } | 173 | } |
172 | 174 | ||
@@ -382,6 +384,7 @@ void blk_sync_queue(struct request_queue *q) | |||
382 | del_timer_sync(&q->unplug_timer); | 384 | del_timer_sync(&q->unplug_timer); |
383 | del_timer_sync(&q->timeout); | 385 | del_timer_sync(&q->timeout); |
384 | cancel_work_sync(&q->unplug_work); | 386 | cancel_work_sync(&q->unplug_work); |
387 | throtl_shutdown_timer_wq(q); | ||
385 | } | 388 | } |
386 | EXPORT_SYMBOL(blk_sync_queue); | 389 | EXPORT_SYMBOL(blk_sync_queue); |
387 | 390 | ||
@@ -515,11 +518,17 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) | |||
515 | return NULL; | 518 | return NULL; |
516 | } | 519 | } |
517 | 520 | ||
521 | if (blk_throtl_init(q)) { | ||
522 | kmem_cache_free(blk_requestq_cachep, q); | ||
523 | return NULL; | ||
524 | } | ||
525 | |||
518 | setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, | 526 | setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, |
519 | laptop_mode_timer_fn, (unsigned long) q); | 527 | laptop_mode_timer_fn, (unsigned long) q); |
520 | init_timer(&q->unplug_timer); | 528 | init_timer(&q->unplug_timer); |
521 | setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q); | 529 | setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q); |
522 | INIT_LIST_HEAD(&q->timeout_list); | 530 | INIT_LIST_HEAD(&q->timeout_list); |
531 | INIT_LIST_HEAD(&q->pending_flushes); | ||
523 | INIT_WORK(&q->unplug_work, blk_unplug_work); | 532 | INIT_WORK(&q->unplug_work, blk_unplug_work); |
524 | 533 | ||
525 | kobject_init(&q->kobj, &blk_queue_ktype); | 534 | kobject_init(&q->kobj, &blk_queue_ktype); |
@@ -796,11 +805,16 @@ static struct request *get_request(struct request_queue *q, int rw_flags, | |||
796 | rl->starved[is_sync] = 0; | 805 | rl->starved[is_sync] = 0; |
797 | 806 | ||
798 | priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags); | 807 | priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags); |
799 | if (priv) | 808 | if (priv) { |
800 | rl->elvpriv++; | 809 | rl->elvpriv++; |
801 | 810 | ||
802 | if (blk_queue_io_stat(q)) | 811 | /* |
803 | rw_flags |= REQ_IO_STAT; | 812 | * Don't do stats for non-priv requests |
813 | */ | ||
814 | if (blk_queue_io_stat(q)) | ||
815 | rw_flags |= REQ_IO_STAT; | ||
816 | } | ||
817 | |||
804 | spin_unlock_irq(q->queue_lock); | 818 | spin_unlock_irq(q->queue_lock); |
805 | 819 | ||
806 | rq = blk_alloc_request(q, rw_flags, priv, gfp_mask); | 820 | rq = blk_alloc_request(q, rw_flags, priv, gfp_mask); |
@@ -1037,22 +1051,6 @@ void blk_insert_request(struct request_queue *q, struct request *rq, | |||
1037 | } | 1051 | } |
1038 | EXPORT_SYMBOL(blk_insert_request); | 1052 | EXPORT_SYMBOL(blk_insert_request); |
1039 | 1053 | ||
1040 | /* | ||
1041 | * add-request adds a request to the linked list. | ||
1042 | * queue lock is held and interrupts disabled, as we muck with the | ||
1043 | * request queue list. | ||
1044 | */ | ||
1045 | static inline void add_request(struct request_queue *q, struct request *req) | ||
1046 | { | ||
1047 | drive_stat_acct(req, 1); | ||
1048 | |||
1049 | /* | ||
1050 | * elevator indicated where it wants this request to be | ||
1051 | * inserted at elevator_merge time | ||
1052 | */ | ||
1053 | __elv_add_request(q, req, ELEVATOR_INSERT_SORT, 0); | ||
1054 | } | ||
1055 | |||
1056 | static void part_round_stats_single(int cpu, struct hd_struct *part, | 1054 | static void part_round_stats_single(int cpu, struct hd_struct *part, |
1057 | unsigned long now) | 1055 | unsigned long now) |
1058 | { | 1056 | { |
@@ -1198,16 +1196,19 @@ static int __make_request(struct request_queue *q, struct bio *bio) | |||
1198 | int el_ret; | 1196 | int el_ret; |
1199 | unsigned int bytes = bio->bi_size; | 1197 | unsigned int bytes = bio->bi_size; |
1200 | const unsigned short prio = bio_prio(bio); | 1198 | const unsigned short prio = bio_prio(bio); |
1201 | const bool sync = (bio->bi_rw & REQ_SYNC); | 1199 | const bool sync = !!(bio->bi_rw & REQ_SYNC); |
1202 | const bool unplug = (bio->bi_rw & REQ_UNPLUG); | 1200 | const bool unplug = !!(bio->bi_rw & REQ_UNPLUG); |
1203 | const unsigned int ff = bio->bi_rw & REQ_FAILFAST_MASK; | 1201 | const unsigned long ff = bio->bi_rw & REQ_FAILFAST_MASK; |
1202 | int where = ELEVATOR_INSERT_SORT; | ||
1204 | int rw_flags; | 1203 | int rw_flags; |
1205 | 1204 | ||
1206 | if ((bio->bi_rw & REQ_HARDBARRIER) && | 1205 | /* REQ_HARDBARRIER is no more */ |
1207 | (q->next_ordered == QUEUE_ORDERED_NONE)) { | 1206 | if (WARN_ONCE(bio->bi_rw & REQ_HARDBARRIER, |
1207 | "block: HARDBARRIER is deprecated, use FLUSH/FUA instead\n")) { | ||
1208 | bio_endio(bio, -EOPNOTSUPP); | 1208 | bio_endio(bio, -EOPNOTSUPP); |
1209 | return 0; | 1209 | return 0; |
1210 | } | 1210 | } |
1211 | |||
1211 | /* | 1212 | /* |
1212 | * low level driver can indicate that it wants pages above a | 1213 | * low level driver can indicate that it wants pages above a |
1213 | * certain limit bounced to low memory (ie for highmem, or even | 1214 | * certain limit bounced to low memory (ie for highmem, or even |
@@ -1217,7 +1218,12 @@ static int __make_request(struct request_queue *q, struct bio *bio) | |||
1217 | 1218 | ||
1218 | spin_lock_irq(q->queue_lock); | 1219 | spin_lock_irq(q->queue_lock); |
1219 | 1220 | ||
1220 | if (unlikely((bio->bi_rw & REQ_HARDBARRIER)) || elv_queue_empty(q)) | 1221 | if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) { |
1222 | where = ELEVATOR_INSERT_FRONT; | ||
1223 | goto get_rq; | ||
1224 | } | ||
1225 | |||
1226 | if (elv_queue_empty(q)) | ||
1221 | goto get_rq; | 1227 | goto get_rq; |
1222 | 1228 | ||
1223 | el_ret = elv_merge(q, &req, bio); | 1229 | el_ret = elv_merge(q, &req, bio); |
@@ -1314,7 +1320,10 @@ get_rq: | |||
1314 | req->cpu = blk_cpu_to_group(smp_processor_id()); | 1320 | req->cpu = blk_cpu_to_group(smp_processor_id()); |
1315 | if (queue_should_plug(q) && elv_queue_empty(q)) | 1321 | if (queue_should_plug(q) && elv_queue_empty(q)) |
1316 | blk_plug_device(q); | 1322 | blk_plug_device(q); |
1317 | add_request(q, req); | 1323 | |
1324 | /* insert the request into the elevator */ | ||
1325 | drive_stat_acct(req, 1); | ||
1326 | __elv_add_request(q, req, where, 0); | ||
1318 | out: | 1327 | out: |
1319 | if (unplug || !queue_should_plug(q)) | 1328 | if (unplug || !queue_should_plug(q)) |
1320 | __generic_unplug_device(q); | 1329 | __generic_unplug_device(q); |
@@ -1514,11 +1523,36 @@ static inline void __generic_make_request(struct bio *bio) | |||
1514 | if (bio_check_eod(bio, nr_sectors)) | 1523 | if (bio_check_eod(bio, nr_sectors)) |
1515 | goto end_io; | 1524 | goto end_io; |
1516 | 1525 | ||
1517 | if ((bio->bi_rw & REQ_DISCARD) && !blk_queue_discard(q)) { | 1526 | /* |
1527 | * Filter flush bio's early so that make_request based | ||
1528 | * drivers without flush support don't have to worry | ||
1529 | * about them. | ||
1530 | */ | ||
1531 | if ((bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && !q->flush_flags) { | ||
1532 | bio->bi_rw &= ~(REQ_FLUSH | REQ_FUA); | ||
1533 | if (!nr_sectors) { | ||
1534 | err = 0; | ||
1535 | goto end_io; | ||
1536 | } | ||
1537 | } | ||
1538 | |||
1539 | if ((bio->bi_rw & REQ_DISCARD) && | ||
1540 | (!blk_queue_discard(q) || | ||
1541 | ((bio->bi_rw & REQ_SECURE) && | ||
1542 | !blk_queue_secdiscard(q)))) { | ||
1518 | err = -EOPNOTSUPP; | 1543 | err = -EOPNOTSUPP; |
1519 | goto end_io; | 1544 | goto end_io; |
1520 | } | 1545 | } |
1521 | 1546 | ||
1547 | blk_throtl_bio(q, &bio); | ||
1548 | |||
1549 | /* | ||
1550 | * If bio = NULL, bio has been throttled and will be submitted | ||
1551 | * later. | ||
1552 | */ | ||
1553 | if (!bio) | ||
1554 | break; | ||
1555 | |||
1522 | trace_block_bio_queue(q, bio); | 1556 | trace_block_bio_queue(q, bio); |
1523 | 1557 | ||
1524 | ret = q->make_request_fn(q, bio); | 1558 | ret = q->make_request_fn(q, bio); |
@@ -1609,11 +1643,12 @@ void submit_bio(int rw, struct bio *bio) | |||
1609 | 1643 | ||
1610 | if (unlikely(block_dump)) { | 1644 | if (unlikely(block_dump)) { |
1611 | char b[BDEVNAME_SIZE]; | 1645 | char b[BDEVNAME_SIZE]; |
1612 | printk(KERN_DEBUG "%s(%d): %s block %Lu on %s\n", | 1646 | printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n", |
1613 | current->comm, task_pid_nr(current), | 1647 | current->comm, task_pid_nr(current), |
1614 | (rw & WRITE) ? "WRITE" : "READ", | 1648 | (rw & WRITE) ? "WRITE" : "READ", |
1615 | (unsigned long long)bio->bi_sector, | 1649 | (unsigned long long)bio->bi_sector, |
1616 | bdevname(bio->bi_bdev, b)); | 1650 | bdevname(bio->bi_bdev, b), |
1651 | count); | ||
1617 | } | 1652 | } |
1618 | } | 1653 | } |
1619 | 1654 | ||
@@ -1756,7 +1791,7 @@ static void blk_account_io_completion(struct request *req, unsigned int bytes) | |||
1756 | int cpu; | 1791 | int cpu; |
1757 | 1792 | ||
1758 | cpu = part_stat_lock(); | 1793 | cpu = part_stat_lock(); |
1759 | part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req)); | 1794 | part = req->part; |
1760 | part_stat_add(cpu, part, sectors[rw], bytes >> 9); | 1795 | part_stat_add(cpu, part, sectors[rw], bytes >> 9); |
1761 | part_stat_unlock(); | 1796 | part_stat_unlock(); |
1762 | } | 1797 | } |
@@ -1765,18 +1800,18 @@ static void blk_account_io_completion(struct request *req, unsigned int bytes) | |||
1765 | static void blk_account_io_done(struct request *req) | 1800 | static void blk_account_io_done(struct request *req) |
1766 | { | 1801 | { |
1767 | /* | 1802 | /* |
1768 | * Account IO completion. bar_rq isn't accounted as a normal | 1803 | * Account IO completion. flush_rq isn't accounted as a |
1769 | * IO on queueing nor completion. Accounting the containing | 1804 | * normal IO on queueing nor completion. Accounting the |
1770 | * request is enough. | 1805 | * containing request is enough. |
1771 | */ | 1806 | */ |
1772 | if (blk_do_io_stat(req) && req != &req->q->bar_rq) { | 1807 | if (blk_do_io_stat(req) && req != &req->q->flush_rq) { |
1773 | unsigned long duration = jiffies - req->start_time; | 1808 | unsigned long duration = jiffies - req->start_time; |
1774 | const int rw = rq_data_dir(req); | 1809 | const int rw = rq_data_dir(req); |
1775 | struct hd_struct *part; | 1810 | struct hd_struct *part; |
1776 | int cpu; | 1811 | int cpu; |
1777 | 1812 | ||
1778 | cpu = part_stat_lock(); | 1813 | cpu = part_stat_lock(); |
1779 | part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req)); | 1814 | part = req->part; |
1780 | 1815 | ||
1781 | part_stat_inc(cpu, part, ios[rw]); | 1816 | part_stat_inc(cpu, part, ios[rw]); |
1782 | part_stat_add(cpu, part, ticks[rw], duration); | 1817 | part_stat_add(cpu, part, ticks[rw], duration); |
@@ -2494,9 +2529,7 @@ EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); | |||
2494 | static void __blk_rq_prep_clone(struct request *dst, struct request *src) | 2529 | static void __blk_rq_prep_clone(struct request *dst, struct request *src) |
2495 | { | 2530 | { |
2496 | dst->cpu = src->cpu; | 2531 | dst->cpu = src->cpu; |
2497 | dst->cmd_flags = (rq_data_dir(src) | REQ_NOMERGE); | 2532 | dst->cmd_flags = (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE; |
2498 | if (src->cmd_flags & REQ_DISCARD) | ||
2499 | dst->cmd_flags |= REQ_DISCARD; | ||
2500 | dst->cmd_type = src->cmd_type; | 2533 | dst->cmd_type = src->cmd_type; |
2501 | dst->__sector = blk_rq_pos(src); | 2534 | dst->__sector = blk_rq_pos(src); |
2502 | dst->__data_len = blk_rq_bytes(src); | 2535 | dst->__data_len = blk_rq_bytes(src); |
@@ -2576,6 +2609,13 @@ int kblockd_schedule_work(struct request_queue *q, struct work_struct *work) | |||
2576 | } | 2609 | } |
2577 | EXPORT_SYMBOL(kblockd_schedule_work); | 2610 | EXPORT_SYMBOL(kblockd_schedule_work); |
2578 | 2611 | ||
2612 | int kblockd_schedule_delayed_work(struct request_queue *q, | ||
2613 | struct delayed_work *dwork, unsigned long delay) | ||
2614 | { | ||
2615 | return queue_delayed_work(kblockd_workqueue, dwork, delay); | ||
2616 | } | ||
2617 | EXPORT_SYMBOL(kblockd_schedule_delayed_work); | ||
2618 | |||
2579 | int __init blk_dev_init(void) | 2619 | int __init blk_dev_init(void) |
2580 | { | 2620 | { |
2581 | BUILD_BUG_ON(__REQ_NR_BITS > 8 * | 2621 | BUILD_BUG_ON(__REQ_NR_BITS > 8 * |