diff options
Diffstat (limited to 'block')
-rw-r--r-- | block/as-iosched.c | 2 | ||||
-rw-r--r-- | block/blk-core.c | 37 | ||||
-rw-r--r-- | block/blktrace.c | 29 | ||||
-rw-r--r-- | block/cfq-iosched.c | 36 | ||||
-rw-r--r-- | block/elevator.c | 2 | ||||
-rw-r--r-- | block/genhd.c | 2 |
6 files changed, 78 insertions, 30 deletions
diff --git a/block/as-iosched.c b/block/as-iosched.c index 8c3946787dbb..743f33a01a07 100644 --- a/block/as-iosched.c +++ b/block/as-iosched.c | |||
@@ -831,6 +831,8 @@ static void as_completed_request(struct request_queue *q, struct request *rq) | |||
831 | } | 831 | } |
832 | 832 | ||
833 | if (ad->changed_batch && ad->nr_dispatched == 1) { | 833 | if (ad->changed_batch && ad->nr_dispatched == 1) { |
834 | ad->current_batch_expires = jiffies + | ||
835 | ad->batch_expire[ad->batch_data_dir]; | ||
834 | kblockd_schedule_work(&ad->antic_work); | 836 | kblockd_schedule_work(&ad->antic_work); |
835 | ad->changed_batch = 0; | 837 | ad->changed_batch = 0; |
836 | 838 | ||
diff --git a/block/blk-core.c b/block/blk-core.c index 6a9cc0d22a61..1905aaba49fb 100644 --- a/block/blk-core.c +++ b/block/blk-core.c | |||
@@ -806,35 +806,32 @@ static struct request *get_request_wait(struct request_queue *q, int rw_flags, | |||
806 | rq = get_request(q, rw_flags, bio, GFP_NOIO); | 806 | rq = get_request(q, rw_flags, bio, GFP_NOIO); |
807 | while (!rq) { | 807 | while (!rq) { |
808 | DEFINE_WAIT(wait); | 808 | DEFINE_WAIT(wait); |
809 | struct io_context *ioc; | ||
809 | struct request_list *rl = &q->rq; | 810 | struct request_list *rl = &q->rq; |
810 | 811 | ||
811 | prepare_to_wait_exclusive(&rl->wait[rw], &wait, | 812 | prepare_to_wait_exclusive(&rl->wait[rw], &wait, |
812 | TASK_UNINTERRUPTIBLE); | 813 | TASK_UNINTERRUPTIBLE); |
813 | 814 | ||
814 | rq = get_request(q, rw_flags, bio, GFP_NOIO); | 815 | blk_add_trace_generic(q, bio, rw, BLK_TA_SLEEPRQ); |
815 | |||
816 | if (!rq) { | ||
817 | struct io_context *ioc; | ||
818 | 816 | ||
819 | blk_add_trace_generic(q, bio, rw, BLK_TA_SLEEPRQ); | 817 | __generic_unplug_device(q); |
820 | 818 | spin_unlock_irq(q->queue_lock); | |
821 | __generic_unplug_device(q); | 819 | io_schedule(); |
822 | spin_unlock_irq(q->queue_lock); | ||
823 | io_schedule(); | ||
824 | 820 | ||
825 | /* | 821 | /* |
826 | * After sleeping, we become a "batching" process and | 822 | * After sleeping, we become a "batching" process and |
827 | * will be able to allocate at least one request, and | 823 | * will be able to allocate at least one request, and |
828 | * up to a big batch of them for a small period time. | 824 | * up to a big batch of them for a small period time. |
829 | * See ioc_batching, ioc_set_batching | 825 | * See ioc_batching, ioc_set_batching |
830 | */ | 826 | */ |
831 | ioc = current_io_context(GFP_NOIO, q->node); | 827 | ioc = current_io_context(GFP_NOIO, q->node); |
832 | ioc_set_batching(q, ioc); | 828 | ioc_set_batching(q, ioc); |
833 | 829 | ||
834 | spin_lock_irq(q->queue_lock); | 830 | spin_lock_irq(q->queue_lock); |
835 | } | ||
836 | finish_wait(&rl->wait[rw], &wait); | 831 | finish_wait(&rl->wait[rw], &wait); |
837 | } | 832 | |
833 | rq = get_request(q, rw_flags, bio, GFP_NOIO); | ||
834 | }; | ||
838 | 835 | ||
839 | return rq; | 836 | return rq; |
840 | } | 837 | } |
diff --git a/block/blktrace.c b/block/blktrace.c index b2cbb4e5d767..8d3a27780260 100644 --- a/block/blktrace.c +++ b/block/blktrace.c | |||
@@ -75,6 +75,24 @@ static void trace_note_time(struct blk_trace *bt) | |||
75 | local_irq_restore(flags); | 75 | local_irq_restore(flags); |
76 | } | 76 | } |
77 | 77 | ||
78 | void __trace_note_message(struct blk_trace *bt, const char *fmt, ...) | ||
79 | { | ||
80 | int n; | ||
81 | va_list args; | ||
82 | unsigned long flags; | ||
83 | char *buf; | ||
84 | |||
85 | local_irq_save(flags); | ||
86 | buf = per_cpu_ptr(bt->msg_data, smp_processor_id()); | ||
87 | va_start(args, fmt); | ||
88 | n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args); | ||
89 | va_end(args); | ||
90 | |||
91 | trace_note(bt, 0, BLK_TN_MESSAGE, buf, n); | ||
92 | local_irq_restore(flags); | ||
93 | } | ||
94 | EXPORT_SYMBOL_GPL(__trace_note_message); | ||
95 | |||
78 | static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector, | 96 | static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector, |
79 | pid_t pid) | 97 | pid_t pid) |
80 | { | 98 | { |
@@ -141,10 +159,7 @@ void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, | |||
141 | /* | 159 | /* |
142 | * A word about the locking here - we disable interrupts to reserve | 160 | * A word about the locking here - we disable interrupts to reserve |
143 | * some space in the relay per-cpu buffer, to prevent an irq | 161 | * some space in the relay per-cpu buffer, to prevent an irq |
144 | * from coming in and stepping on our toes. Once reserved, it's | 162 | * from coming in and stepping on our toes. |
145 | * enough to get preemption disabled to prevent read of this data | ||
146 | * before we are through filling it. get_cpu()/put_cpu() does this | ||
147 | * for us | ||
148 | */ | 163 | */ |
149 | local_irq_save(flags); | 164 | local_irq_save(flags); |
150 | 165 | ||
@@ -232,6 +247,7 @@ static void blk_trace_cleanup(struct blk_trace *bt) | |||
232 | debugfs_remove(bt->dropped_file); | 247 | debugfs_remove(bt->dropped_file); |
233 | blk_remove_tree(bt->dir); | 248 | blk_remove_tree(bt->dir); |
234 | free_percpu(bt->sequence); | 249 | free_percpu(bt->sequence); |
250 | free_percpu(bt->msg_data); | ||
235 | kfree(bt); | 251 | kfree(bt); |
236 | } | 252 | } |
237 | 253 | ||
@@ -346,6 +362,10 @@ int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev, | |||
346 | if (!bt->sequence) | 362 | if (!bt->sequence) |
347 | goto err; | 363 | goto err; |
348 | 364 | ||
365 | bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG); | ||
366 | if (!bt->msg_data) | ||
367 | goto err; | ||
368 | |||
349 | ret = -ENOENT; | 369 | ret = -ENOENT; |
350 | dir = blk_create_tree(buts->name); | 370 | dir = blk_create_tree(buts->name); |
351 | if (!dir) | 371 | if (!dir) |
@@ -392,6 +412,7 @@ err: | |||
392 | if (bt->dropped_file) | 412 | if (bt->dropped_file) |
393 | debugfs_remove(bt->dropped_file); | 413 | debugfs_remove(bt->dropped_file); |
394 | free_percpu(bt->sequence); | 414 | free_percpu(bt->sequence); |
415 | free_percpu(bt->msg_data); | ||
395 | if (bt->rchan) | 416 | if (bt->rchan) |
396 | relay_close(bt->rchan); | 417 | relay_close(bt->rchan); |
397 | kfree(bt); | 418 | kfree(bt); |
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index b399c62936e0..d01b411c72f0 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c | |||
@@ -124,6 +124,8 @@ struct cfq_data { | |||
124 | struct cfq_queue { | 124 | struct cfq_queue { |
125 | /* reference count */ | 125 | /* reference count */ |
126 | atomic_t ref; | 126 | atomic_t ref; |
127 | /* various state flags, see below */ | ||
128 | unsigned int flags; | ||
127 | /* parent cfq_data */ | 129 | /* parent cfq_data */ |
128 | struct cfq_data *cfqd; | 130 | struct cfq_data *cfqd; |
129 | /* service_tree member */ | 131 | /* service_tree member */ |
@@ -138,14 +140,14 @@ struct cfq_queue { | |||
138 | int queued[2]; | 140 | int queued[2]; |
139 | /* currently allocated requests */ | 141 | /* currently allocated requests */ |
140 | int allocated[2]; | 142 | int allocated[2]; |
141 | /* pending metadata requests */ | ||
142 | int meta_pending; | ||
143 | /* fifo list of requests in sort_list */ | 143 | /* fifo list of requests in sort_list */ |
144 | struct list_head fifo; | 144 | struct list_head fifo; |
145 | 145 | ||
146 | unsigned long slice_end; | 146 | unsigned long slice_end; |
147 | long slice_resid; | 147 | long slice_resid; |
148 | 148 | ||
149 | /* pending metadata requests */ | ||
150 | int meta_pending; | ||
149 | /* number of requests that are on the dispatch list or inside driver */ | 151 | /* number of requests that are on the dispatch list or inside driver */ |
150 | int dispatched; | 152 | int dispatched; |
151 | 153 | ||
@@ -153,8 +155,6 @@ struct cfq_queue { | |||
153 | unsigned short ioprio, org_ioprio; | 155 | unsigned short ioprio, org_ioprio; |
154 | unsigned short ioprio_class, org_ioprio_class; | 156 | unsigned short ioprio_class, org_ioprio_class; |
155 | 157 | ||
156 | /* various state flags, see below */ | ||
157 | unsigned int flags; | ||
158 | }; | 158 | }; |
159 | 159 | ||
160 | enum cfqq_state_flags { | 160 | enum cfqq_state_flags { |
@@ -1142,6 +1142,9 @@ static void cfq_put_queue(struct cfq_queue *cfqq) | |||
1142 | kmem_cache_free(cfq_pool, cfqq); | 1142 | kmem_cache_free(cfq_pool, cfqq); |
1143 | } | 1143 | } |
1144 | 1144 | ||
1145 | /* | ||
1146 | * Must always be called with the rcu_read_lock() held | ||
1147 | */ | ||
1145 | static void | 1148 | static void |
1146 | __call_for_each_cic(struct io_context *ioc, | 1149 | __call_for_each_cic(struct io_context *ioc, |
1147 | void (*func)(struct io_context *, struct cfq_io_context *)) | 1150 | void (*func)(struct io_context *, struct cfq_io_context *)) |
@@ -1197,6 +1200,11 @@ static void cic_free_func(struct io_context *ioc, struct cfq_io_context *cic) | |||
1197 | cfq_cic_free(cic); | 1200 | cfq_cic_free(cic); |
1198 | } | 1201 | } |
1199 | 1202 | ||
1203 | /* | ||
1204 | * Must be called with rcu_read_lock() held or preemption otherwise disabled. | ||
1205 | * Only two callers of this - ->dtor() which is called with the rcu_read_lock(), | ||
1206 | * and ->trim() which is called with the task lock held | ||
1207 | */ | ||
1200 | static void cfq_free_io_context(struct io_context *ioc) | 1208 | static void cfq_free_io_context(struct io_context *ioc) |
1201 | { | 1209 | { |
1202 | /* | 1210 | /* |
@@ -1502,20 +1510,24 @@ static struct cfq_io_context * | |||
1502 | cfq_cic_lookup(struct cfq_data *cfqd, struct io_context *ioc) | 1510 | cfq_cic_lookup(struct cfq_data *cfqd, struct io_context *ioc) |
1503 | { | 1511 | { |
1504 | struct cfq_io_context *cic; | 1512 | struct cfq_io_context *cic; |
1513 | unsigned long flags; | ||
1505 | void *k; | 1514 | void *k; |
1506 | 1515 | ||
1507 | if (unlikely(!ioc)) | 1516 | if (unlikely(!ioc)) |
1508 | return NULL; | 1517 | return NULL; |
1509 | 1518 | ||
1519 | rcu_read_lock(); | ||
1520 | |||
1510 | /* | 1521 | /* |
1511 | * we maintain a last-hit cache, to avoid browsing over the tree | 1522 | * we maintain a last-hit cache, to avoid browsing over the tree |
1512 | */ | 1523 | */ |
1513 | cic = rcu_dereference(ioc->ioc_data); | 1524 | cic = rcu_dereference(ioc->ioc_data); |
1514 | if (cic && cic->key == cfqd) | 1525 | if (cic && cic->key == cfqd) { |
1526 | rcu_read_unlock(); | ||
1515 | return cic; | 1527 | return cic; |
1528 | } | ||
1516 | 1529 | ||
1517 | do { | 1530 | do { |
1518 | rcu_read_lock(); | ||
1519 | cic = radix_tree_lookup(&ioc->radix_root, (unsigned long) cfqd); | 1531 | cic = radix_tree_lookup(&ioc->radix_root, (unsigned long) cfqd); |
1520 | rcu_read_unlock(); | 1532 | rcu_read_unlock(); |
1521 | if (!cic) | 1533 | if (!cic) |
@@ -1524,10 +1536,13 @@ cfq_cic_lookup(struct cfq_data *cfqd, struct io_context *ioc) | |||
1524 | k = cic->key; | 1536 | k = cic->key; |
1525 | if (unlikely(!k)) { | 1537 | if (unlikely(!k)) { |
1526 | cfq_drop_dead_cic(cfqd, ioc, cic); | 1538 | cfq_drop_dead_cic(cfqd, ioc, cic); |
1539 | rcu_read_lock(); | ||
1527 | continue; | 1540 | continue; |
1528 | } | 1541 | } |
1529 | 1542 | ||
1543 | spin_lock_irqsave(&ioc->lock, flags); | ||
1530 | rcu_assign_pointer(ioc->ioc_data, cic); | 1544 | rcu_assign_pointer(ioc->ioc_data, cic); |
1545 | spin_unlock_irqrestore(&ioc->lock, flags); | ||
1531 | break; | 1546 | break; |
1532 | } while (1); | 1547 | } while (1); |
1533 | 1548 | ||
@@ -2134,6 +2149,10 @@ static void *cfq_init_queue(struct request_queue *q) | |||
2134 | 2149 | ||
2135 | static void cfq_slab_kill(void) | 2150 | static void cfq_slab_kill(void) |
2136 | { | 2151 | { |
2152 | /* | ||
2153 | * Caller already ensured that pending RCU callbacks are completed, | ||
2154 | * so we should have no busy allocations at this point. | ||
2155 | */ | ||
2137 | if (cfq_pool) | 2156 | if (cfq_pool) |
2138 | kmem_cache_destroy(cfq_pool); | 2157 | kmem_cache_destroy(cfq_pool); |
2139 | if (cfq_ioc_pool) | 2158 | if (cfq_ioc_pool) |
@@ -2292,6 +2311,11 @@ static void __exit cfq_exit(void) | |||
2292 | ioc_gone = &all_gone; | 2311 | ioc_gone = &all_gone; |
2293 | /* ioc_gone's update must be visible before reading ioc_count */ | 2312 | /* ioc_gone's update must be visible before reading ioc_count */ |
2294 | smp_wmb(); | 2313 | smp_wmb(); |
2314 | |||
2315 | /* | ||
2316 | * this also protects us from entering cfq_slab_kill() with | ||
2317 | * pending RCU callbacks | ||
2318 | */ | ||
2295 | if (elv_ioc_count_read(ioc_count)) | 2319 | if (elv_ioc_count_read(ioc_count)) |
2296 | wait_for_completion(ioc_gone); | 2320 | wait_for_completion(ioc_gone); |
2297 | cfq_slab_kill(); | 2321 | cfq_slab_kill(); |
diff --git a/block/elevator.c b/block/elevator.c index 980f8ae147b4..902dd1344d56 100644 --- a/block/elevator.c +++ b/block/elevator.c | |||
@@ -1110,6 +1110,8 @@ static int elevator_switch(struct request_queue *q, struct elevator_type *new_e) | |||
1110 | queue_flag_clear(QUEUE_FLAG_ELVSWITCH, q); | 1110 | queue_flag_clear(QUEUE_FLAG_ELVSWITCH, q); |
1111 | spin_unlock_irq(q->queue_lock); | 1111 | spin_unlock_irq(q->queue_lock); |
1112 | 1112 | ||
1113 | blk_add_trace_msg(q, "elv switch: %s", e->elevator_type->elevator_name); | ||
1114 | |||
1113 | return 1; | 1115 | return 1; |
1114 | 1116 | ||
1115 | fail_register: | 1117 | fail_register: |
diff --git a/block/genhd.c b/block/genhd.c index 129ad939f9dd..b922d4801c87 100644 --- a/block/genhd.c +++ b/block/genhd.c | |||
@@ -660,6 +660,8 @@ dev_t blk_lookup_devt(const char *name, int part) | |||
660 | 660 | ||
661 | mutex_lock(&block_class_lock); | 661 | mutex_lock(&block_class_lock); |
662 | list_for_each_entry(dev, &block_class.devices, node) { | 662 | list_for_each_entry(dev, &block_class.devices, node) { |
663 | if (dev->type != &disk_type) | ||
664 | continue; | ||
663 | if (strcmp(dev->bus_id, name) == 0) { | 665 | if (strcmp(dev->bus_id, name) == 0) { |
664 | struct gendisk *disk = dev_to_disk(dev); | 666 | struct gendisk *disk = dev_to_disk(dev); |
665 | 667 | ||