diff options
author | Greg KH <greg@press.(none)> | 2005-10-28 13:13:16 -0400 |
---|---|---|
committer | Greg Kroah-Hartman <gregkh@suse.de> | 2005-10-28 13:13:16 -0400 |
commit | 6fbfddcb52d8d9fa2cd209f5ac2a1c87497d55b5 (patch) | |
tree | c0414e89678fcef7ce3493e048d855bde781ae8d /drivers/block/elevator.c | |
parent | 1a222bca26ca691e83be1b08f5e96ae96d0d8cae (diff) | |
parent | 27d1097d39509494706eaa2620ef3b1e780a3224 (diff) |
Merge ../bleed-2.6
Diffstat (limited to 'drivers/block/elevator.c')
-rw-r--r-- | drivers/block/elevator.c | 345 |
1 files changed, 207 insertions, 138 deletions
diff --git a/drivers/block/elevator.c b/drivers/block/elevator.c index 98f0126a2deb..55621d5c5774 100644 --- a/drivers/block/elevator.c +++ b/drivers/block/elevator.c | |||
@@ -34,6 +34,7 @@ | |||
34 | #include <linux/slab.h> | 34 | #include <linux/slab.h> |
35 | #include <linux/init.h> | 35 | #include <linux/init.h> |
36 | #include <linux/compiler.h> | 36 | #include <linux/compiler.h> |
37 | #include <linux/delay.h> | ||
37 | 38 | ||
38 | #include <asm/uaccess.h> | 39 | #include <asm/uaccess.h> |
39 | 40 | ||
@@ -83,21 +84,11 @@ inline int elv_try_merge(struct request *__rq, struct bio *bio) | |||
83 | } | 84 | } |
84 | EXPORT_SYMBOL(elv_try_merge); | 85 | EXPORT_SYMBOL(elv_try_merge); |
85 | 86 | ||
86 | inline int elv_try_last_merge(request_queue_t *q, struct bio *bio) | ||
87 | { | ||
88 | if (q->last_merge) | ||
89 | return elv_try_merge(q->last_merge, bio); | ||
90 | |||
91 | return ELEVATOR_NO_MERGE; | ||
92 | } | ||
93 | EXPORT_SYMBOL(elv_try_last_merge); | ||
94 | |||
95 | static struct elevator_type *elevator_find(const char *name) | 87 | static struct elevator_type *elevator_find(const char *name) |
96 | { | 88 | { |
97 | struct elevator_type *e = NULL; | 89 | struct elevator_type *e = NULL; |
98 | struct list_head *entry; | 90 | struct list_head *entry; |
99 | 91 | ||
100 | spin_lock_irq(&elv_list_lock); | ||
101 | list_for_each(entry, &elv_list) { | 92 | list_for_each(entry, &elv_list) { |
102 | struct elevator_type *__e; | 93 | struct elevator_type *__e; |
103 | 94 | ||
@@ -108,7 +99,6 @@ static struct elevator_type *elevator_find(const char *name) | |||
108 | break; | 99 | break; |
109 | } | 100 | } |
110 | } | 101 | } |
111 | spin_unlock_irq(&elv_list_lock); | ||
112 | 102 | ||
113 | return e; | 103 | return e; |
114 | } | 104 | } |
@@ -120,12 +110,15 @@ static void elevator_put(struct elevator_type *e) | |||
120 | 110 | ||
121 | static struct elevator_type *elevator_get(const char *name) | 111 | static struct elevator_type *elevator_get(const char *name) |
122 | { | 112 | { |
123 | struct elevator_type *e = elevator_find(name); | 113 | struct elevator_type *e; |
124 | 114 | ||
125 | if (!e) | 115 | spin_lock_irq(&elv_list_lock); |
126 | return NULL; | 116 | |
127 | if (!try_module_get(e->elevator_owner)) | 117 | e = elevator_find(name); |
128 | return NULL; | 118 | if (e && !try_module_get(e->elevator_owner)) |
119 | e = NULL; | ||
120 | |||
121 | spin_unlock_irq(&elv_list_lock); | ||
129 | 122 | ||
130 | return e; | 123 | return e; |
131 | } | 124 | } |
@@ -139,8 +132,6 @@ static int elevator_attach(request_queue_t *q, struct elevator_type *e, | |||
139 | eq->ops = &e->ops; | 132 | eq->ops = &e->ops; |
140 | eq->elevator_type = e; | 133 | eq->elevator_type = e; |
141 | 134 | ||
142 | INIT_LIST_HEAD(&q->queue_head); | ||
143 | q->last_merge = NULL; | ||
144 | q->elevator = eq; | 135 | q->elevator = eq; |
145 | 136 | ||
146 | if (eq->ops->elevator_init_fn) | 137 | if (eq->ops->elevator_init_fn) |
@@ -153,11 +144,15 @@ static char chosen_elevator[16]; | |||
153 | 144 | ||
154 | static void elevator_setup_default(void) | 145 | static void elevator_setup_default(void) |
155 | { | 146 | { |
147 | struct elevator_type *e; | ||
148 | |||
156 | /* | 149 | /* |
157 | * check if default is set and exists | 150 | * check if default is set and exists |
158 | */ | 151 | */ |
159 | if (chosen_elevator[0] && elevator_find(chosen_elevator)) | 152 | if (chosen_elevator[0] && (e = elevator_get(chosen_elevator))) { |
153 | elevator_put(e); | ||
160 | return; | 154 | return; |
155 | } | ||
161 | 156 | ||
162 | #if defined(CONFIG_IOSCHED_AS) | 157 | #if defined(CONFIG_IOSCHED_AS) |
163 | strcpy(chosen_elevator, "anticipatory"); | 158 | strcpy(chosen_elevator, "anticipatory"); |
@@ -186,6 +181,11 @@ int elevator_init(request_queue_t *q, char *name) | |||
186 | struct elevator_queue *eq; | 181 | struct elevator_queue *eq; |
187 | int ret = 0; | 182 | int ret = 0; |
188 | 183 | ||
184 | INIT_LIST_HEAD(&q->queue_head); | ||
185 | q->last_merge = NULL; | ||
186 | q->end_sector = 0; | ||
187 | q->boundary_rq = NULL; | ||
188 | |||
189 | elevator_setup_default(); | 189 | elevator_setup_default(); |
190 | 190 | ||
191 | if (!name) | 191 | if (!name) |
@@ -220,9 +220,52 @@ void elevator_exit(elevator_t *e) | |||
220 | kfree(e); | 220 | kfree(e); |
221 | } | 221 | } |
222 | 222 | ||
223 | /* | ||
224 | * Insert rq into dispatch queue of q. Queue lock must be held on | ||
225 | * entry. If sort != 0, rq is sort-inserted; otherwise, rq will be | ||
226 | * appended to the dispatch queue. To be used by specific elevators. | ||
227 | */ | ||
228 | void elv_dispatch_sort(request_queue_t *q, struct request *rq) | ||
229 | { | ||
230 | sector_t boundary; | ||
231 | struct list_head *entry; | ||
232 | |||
233 | if (q->last_merge == rq) | ||
234 | q->last_merge = NULL; | ||
235 | |||
236 | boundary = q->end_sector; | ||
237 | |||
238 | list_for_each_prev(entry, &q->queue_head) { | ||
239 | struct request *pos = list_entry_rq(entry); | ||
240 | |||
241 | if (pos->flags & (REQ_SOFTBARRIER|REQ_HARDBARRIER|REQ_STARTED)) | ||
242 | break; | ||
243 | if (rq->sector >= boundary) { | ||
244 | if (pos->sector < boundary) | ||
245 | continue; | ||
246 | } else { | ||
247 | if (pos->sector >= boundary) | ||
248 | break; | ||
249 | } | ||
250 | if (rq->sector >= pos->sector) | ||
251 | break; | ||
252 | } | ||
253 | |||
254 | list_add(&rq->queuelist, entry); | ||
255 | } | ||
256 | |||
223 | int elv_merge(request_queue_t *q, struct request **req, struct bio *bio) | 257 | int elv_merge(request_queue_t *q, struct request **req, struct bio *bio) |
224 | { | 258 | { |
225 | elevator_t *e = q->elevator; | 259 | elevator_t *e = q->elevator; |
260 | int ret; | ||
261 | |||
262 | if (q->last_merge) { | ||
263 | ret = elv_try_merge(q->last_merge, bio); | ||
264 | if (ret != ELEVATOR_NO_MERGE) { | ||
265 | *req = q->last_merge; | ||
266 | return ret; | ||
267 | } | ||
268 | } | ||
226 | 269 | ||
227 | if (e->ops->elevator_merge_fn) | 270 | if (e->ops->elevator_merge_fn) |
228 | return e->ops->elevator_merge_fn(q, req, bio); | 271 | return e->ops->elevator_merge_fn(q, req, bio); |
@@ -236,6 +279,8 @@ void elv_merged_request(request_queue_t *q, struct request *rq) | |||
236 | 279 | ||
237 | if (e->ops->elevator_merged_fn) | 280 | if (e->ops->elevator_merged_fn) |
238 | e->ops->elevator_merged_fn(q, rq); | 281 | e->ops->elevator_merged_fn(q, rq); |
282 | |||
283 | q->last_merge = rq; | ||
239 | } | 284 | } |
240 | 285 | ||
241 | void elv_merge_requests(request_queue_t *q, struct request *rq, | 286 | void elv_merge_requests(request_queue_t *q, struct request *rq, |
@@ -243,20 +288,13 @@ void elv_merge_requests(request_queue_t *q, struct request *rq, | |||
243 | { | 288 | { |
244 | elevator_t *e = q->elevator; | 289 | elevator_t *e = q->elevator; |
245 | 290 | ||
246 | if (q->last_merge == next) | ||
247 | q->last_merge = NULL; | ||
248 | |||
249 | if (e->ops->elevator_merge_req_fn) | 291 | if (e->ops->elevator_merge_req_fn) |
250 | e->ops->elevator_merge_req_fn(q, rq, next); | 292 | e->ops->elevator_merge_req_fn(q, rq, next); |
293 | |||
294 | q->last_merge = rq; | ||
251 | } | 295 | } |
252 | 296 | ||
253 | /* | 297 | void elv_requeue_request(request_queue_t *q, struct request *rq) |
254 | * For careful internal use by the block layer. Essentially the same as | ||
255 | * a requeue in that it tells the io scheduler that this request is not | ||
256 | * active in the driver or hardware anymore, but we don't want the request | ||
257 | * added back to the scheduler. Function is not exported. | ||
258 | */ | ||
259 | void elv_deactivate_request(request_queue_t *q, struct request *rq) | ||
260 | { | 298 | { |
261 | elevator_t *e = q->elevator; | 299 | elevator_t *e = q->elevator; |
262 | 300 | ||
@@ -264,19 +302,14 @@ void elv_deactivate_request(request_queue_t *q, struct request *rq) | |||
264 | * it already went through dequeue, we need to decrement the | 302 | * it already went through dequeue, we need to decrement the |
265 | * in_flight count again | 303 | * in_flight count again |
266 | */ | 304 | */ |
267 | if (blk_account_rq(rq)) | 305 | if (blk_account_rq(rq)) { |
268 | q->in_flight--; | 306 | q->in_flight--; |
307 | if (blk_sorted_rq(rq) && e->ops->elevator_deactivate_req_fn) | ||
308 | e->ops->elevator_deactivate_req_fn(q, rq); | ||
309 | } | ||
269 | 310 | ||
270 | rq->flags &= ~REQ_STARTED; | 311 | rq->flags &= ~REQ_STARTED; |
271 | 312 | ||
272 | if (e->ops->elevator_deactivate_req_fn) | ||
273 | e->ops->elevator_deactivate_req_fn(q, rq); | ||
274 | } | ||
275 | |||
276 | void elv_requeue_request(request_queue_t *q, struct request *rq) | ||
277 | { | ||
278 | elv_deactivate_request(q, rq); | ||
279 | |||
280 | /* | 313 | /* |
281 | * if this is the flush, requeue the original instead and drop the flush | 314 | * if this is the flush, requeue the original instead and drop the flush |
282 | */ | 315 | */ |
@@ -285,31 +318,27 @@ void elv_requeue_request(request_queue_t *q, struct request *rq) | |||
285 | rq = rq->end_io_data; | 318 | rq = rq->end_io_data; |
286 | } | 319 | } |
287 | 320 | ||
288 | /* | 321 | __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0); |
289 | * the request is prepped and may have some resources allocated. | ||
290 | * allowing unprepped requests to pass this one may cause resource | ||
291 | * deadlock. turn on softbarrier. | ||
292 | */ | ||
293 | rq->flags |= REQ_SOFTBARRIER; | ||
294 | |||
295 | /* | ||
296 | * if iosched has an explicit requeue hook, then use that. otherwise | ||
297 | * just put the request at the front of the queue | ||
298 | */ | ||
299 | if (q->elevator->ops->elevator_requeue_req_fn) | ||
300 | q->elevator->ops->elevator_requeue_req_fn(q, rq); | ||
301 | else | ||
302 | __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0); | ||
303 | } | 322 | } |
304 | 323 | ||
305 | void __elv_add_request(request_queue_t *q, struct request *rq, int where, | 324 | void __elv_add_request(request_queue_t *q, struct request *rq, int where, |
306 | int plug) | 325 | int plug) |
307 | { | 326 | { |
308 | /* | 327 | if (rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER)) { |
309 | * barriers implicitly indicate back insertion | 328 | /* |
310 | */ | 329 | * barriers implicitly indicate back insertion |
311 | if (rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER) && | 330 | */ |
312 | where == ELEVATOR_INSERT_SORT) | 331 | if (where == ELEVATOR_INSERT_SORT) |
332 | where = ELEVATOR_INSERT_BACK; | ||
333 | |||
334 | /* | ||
335 | * this request is scheduling boundary, update end_sector | ||
336 | */ | ||
337 | if (blk_fs_request(rq)) { | ||
338 | q->end_sector = rq_end_sector(rq); | ||
339 | q->boundary_rq = rq; | ||
340 | } | ||
341 | } else if (!(rq->flags & REQ_ELVPRIV) && where == ELEVATOR_INSERT_SORT) | ||
313 | where = ELEVATOR_INSERT_BACK; | 342 | where = ELEVATOR_INSERT_BACK; |
314 | 343 | ||
315 | if (plug) | 344 | if (plug) |
@@ -317,23 +346,54 @@ void __elv_add_request(request_queue_t *q, struct request *rq, int where, | |||
317 | 346 | ||
318 | rq->q = q; | 347 | rq->q = q; |
319 | 348 | ||
320 | if (!test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags)) { | 349 | switch (where) { |
321 | q->elevator->ops->elevator_add_req_fn(q, rq, where); | 350 | case ELEVATOR_INSERT_FRONT: |
351 | rq->flags |= REQ_SOFTBARRIER; | ||
322 | 352 | ||
323 | if (blk_queue_plugged(q)) { | 353 | list_add(&rq->queuelist, &q->queue_head); |
324 | int nrq = q->rq.count[READ] + q->rq.count[WRITE] | 354 | break; |
325 | - q->in_flight; | ||
326 | 355 | ||
327 | if (nrq >= q->unplug_thresh) | 356 | case ELEVATOR_INSERT_BACK: |
328 | __generic_unplug_device(q); | 357 | rq->flags |= REQ_SOFTBARRIER; |
329 | } | 358 | |
330 | } else | 359 | while (q->elevator->ops->elevator_dispatch_fn(q, 1)) |
360 | ; | ||
361 | list_add_tail(&rq->queuelist, &q->queue_head); | ||
331 | /* | 362 | /* |
332 | * if drain is set, store the request "locally". when the drain | 363 | * We kick the queue here for the following reasons. |
333 | * is finished, the requests will be handed ordered to the io | 364 | * - The elevator might have returned NULL previously |
334 | * scheduler | 365 | * to delay requests and returned them now. As the |
366 | * queue wasn't empty before this request, ll_rw_blk | ||
367 | * won't run the queue on return, resulting in hang. | ||
368 | * - Usually, back inserted requests won't be merged | ||
369 | * with anything. There's no point in delaying queue | ||
370 | * processing. | ||
335 | */ | 371 | */ |
336 | list_add_tail(&rq->queuelist, &q->drain_list); | 372 | blk_remove_plug(q); |
373 | q->request_fn(q); | ||
374 | break; | ||
375 | |||
376 | case ELEVATOR_INSERT_SORT: | ||
377 | BUG_ON(!blk_fs_request(rq)); | ||
378 | rq->flags |= REQ_SORTED; | ||
379 | q->elevator->ops->elevator_add_req_fn(q, rq); | ||
380 | if (q->last_merge == NULL && rq_mergeable(rq)) | ||
381 | q->last_merge = rq; | ||
382 | break; | ||
383 | |||
384 | default: | ||
385 | printk(KERN_ERR "%s: bad insertion point %d\n", | ||
386 | __FUNCTION__, where); | ||
387 | BUG(); | ||
388 | } | ||
389 | |||
390 | if (blk_queue_plugged(q)) { | ||
391 | int nrq = q->rq.count[READ] + q->rq.count[WRITE] | ||
392 | - q->in_flight; | ||
393 | |||
394 | if (nrq >= q->unplug_thresh) | ||
395 | __generic_unplug_device(q); | ||
396 | } | ||
337 | } | 397 | } |
338 | 398 | ||
339 | void elv_add_request(request_queue_t *q, struct request *rq, int where, | 399 | void elv_add_request(request_queue_t *q, struct request *rq, int where, |
@@ -348,13 +408,19 @@ void elv_add_request(request_queue_t *q, struct request *rq, int where, | |||
348 | 408 | ||
349 | static inline struct request *__elv_next_request(request_queue_t *q) | 409 | static inline struct request *__elv_next_request(request_queue_t *q) |
350 | { | 410 | { |
351 | struct request *rq = q->elevator->ops->elevator_next_req_fn(q); | 411 | struct request *rq; |
412 | |||
413 | if (unlikely(list_empty(&q->queue_head) && | ||
414 | !q->elevator->ops->elevator_dispatch_fn(q, 0))) | ||
415 | return NULL; | ||
416 | |||
417 | rq = list_entry_rq(q->queue_head.next); | ||
352 | 418 | ||
353 | /* | 419 | /* |
354 | * if this is a barrier write and the device has to issue a | 420 | * if this is a barrier write and the device has to issue a |
355 | * flush sequence to support it, check how far we are | 421 | * flush sequence to support it, check how far we are |
356 | */ | 422 | */ |
357 | if (rq && blk_fs_request(rq) && blk_barrier_rq(rq)) { | 423 | if (blk_fs_request(rq) && blk_barrier_rq(rq)) { |
358 | BUG_ON(q->ordered == QUEUE_ORDERED_NONE); | 424 | BUG_ON(q->ordered == QUEUE_ORDERED_NONE); |
359 | 425 | ||
360 | if (q->ordered == QUEUE_ORDERED_FLUSH && | 426 | if (q->ordered == QUEUE_ORDERED_FLUSH && |
@@ -371,15 +437,30 @@ struct request *elv_next_request(request_queue_t *q) | |||
371 | int ret; | 437 | int ret; |
372 | 438 | ||
373 | while ((rq = __elv_next_request(q)) != NULL) { | 439 | while ((rq = __elv_next_request(q)) != NULL) { |
374 | /* | 440 | if (!(rq->flags & REQ_STARTED)) { |
375 | * just mark as started even if we don't start it, a request | 441 | elevator_t *e = q->elevator; |
376 | * that has been delayed should not be passed by new incoming | ||
377 | * requests | ||
378 | */ | ||
379 | rq->flags |= REQ_STARTED; | ||
380 | 442 | ||
381 | if (rq == q->last_merge) | 443 | /* |
382 | q->last_merge = NULL; | 444 | * This is the first time the device driver |
445 | * sees this request (possibly after | ||
446 | * requeueing). Notify IO scheduler. | ||
447 | */ | ||
448 | if (blk_sorted_rq(rq) && | ||
449 | e->ops->elevator_activate_req_fn) | ||
450 | e->ops->elevator_activate_req_fn(q, rq); | ||
451 | |||
452 | /* | ||
453 | * just mark as started even if we don't start | ||
454 | * it, a request that has been delayed should | ||
455 | * not be passed by new incoming requests | ||
456 | */ | ||
457 | rq->flags |= REQ_STARTED; | ||
458 | } | ||
459 | |||
460 | if (!q->boundary_rq || q->boundary_rq == rq) { | ||
461 | q->end_sector = rq_end_sector(rq); | ||
462 | q->boundary_rq = NULL; | ||
463 | } | ||
383 | 464 | ||
384 | if ((rq->flags & REQ_DONTPREP) || !q->prep_rq_fn) | 465 | if ((rq->flags & REQ_DONTPREP) || !q->prep_rq_fn) |
385 | break; | 466 | break; |
@@ -391,9 +472,9 @@ struct request *elv_next_request(request_queue_t *q) | |||
391 | /* | 472 | /* |
392 | * the request may have been (partially) prepped. | 473 | * the request may have been (partially) prepped. |
393 | * we need to keep this request in the front to | 474 | * we need to keep this request in the front to |
394 | * avoid resource deadlock. turn on softbarrier. | 475 | * avoid resource deadlock. REQ_STARTED will |
476 | * prevent other fs requests from passing this one. | ||
395 | */ | 477 | */ |
396 | rq->flags |= REQ_SOFTBARRIER; | ||
397 | rq = NULL; | 478 | rq = NULL; |
398 | break; | 479 | break; |
399 | } else if (ret == BLKPREP_KILL) { | 480 | } else if (ret == BLKPREP_KILL) { |
@@ -416,42 +497,32 @@ struct request *elv_next_request(request_queue_t *q) | |||
416 | return rq; | 497 | return rq; |
417 | } | 498 | } |
418 | 499 | ||
419 | void elv_remove_request(request_queue_t *q, struct request *rq) | 500 | void elv_dequeue_request(request_queue_t *q, struct request *rq) |
420 | { | 501 | { |
421 | elevator_t *e = q->elevator; | 502 | BUG_ON(list_empty(&rq->queuelist)); |
503 | |||
504 | list_del_init(&rq->queuelist); | ||
422 | 505 | ||
423 | /* | 506 | /* |
424 | * the time frame between a request being removed from the lists | 507 | * the time frame between a request being removed from the lists |
425 | * and to it is freed is accounted as io that is in progress at | 508 | * and to it is freed is accounted as io that is in progress at |
426 | * the driver side. note that we only account requests that the | 509 | * the driver side. |
427 | * driver has seen (REQ_STARTED set), to avoid false accounting | ||
428 | * for request-request merges | ||
429 | */ | 510 | */ |
430 | if (blk_account_rq(rq)) | 511 | if (blk_account_rq(rq)) |
431 | q->in_flight++; | 512 | q->in_flight++; |
432 | |||
433 | /* | ||
434 | * the main clearing point for q->last_merge is on retrieval of | ||
435 | * request by driver (it calls elv_next_request()), but it _can_ | ||
436 | * also happen here if a request is added to the queue but later | ||
437 | * deleted without ever being given to driver (merged with another | ||
438 | * request). | ||
439 | */ | ||
440 | if (rq == q->last_merge) | ||
441 | q->last_merge = NULL; | ||
442 | |||
443 | if (e->ops->elevator_remove_req_fn) | ||
444 | e->ops->elevator_remove_req_fn(q, rq); | ||
445 | } | 513 | } |
446 | 514 | ||
447 | int elv_queue_empty(request_queue_t *q) | 515 | int elv_queue_empty(request_queue_t *q) |
448 | { | 516 | { |
449 | elevator_t *e = q->elevator; | 517 | elevator_t *e = q->elevator; |
450 | 518 | ||
519 | if (!list_empty(&q->queue_head)) | ||
520 | return 0; | ||
521 | |||
451 | if (e->ops->elevator_queue_empty_fn) | 522 | if (e->ops->elevator_queue_empty_fn) |
452 | return e->ops->elevator_queue_empty_fn(q); | 523 | return e->ops->elevator_queue_empty_fn(q); |
453 | 524 | ||
454 | return list_empty(&q->queue_head); | 525 | return 1; |
455 | } | 526 | } |
456 | 527 | ||
457 | struct request *elv_latter_request(request_queue_t *q, struct request *rq) | 528 | struct request *elv_latter_request(request_queue_t *q, struct request *rq) |
@@ -487,7 +558,7 @@ struct request *elv_former_request(request_queue_t *q, struct request *rq) | |||
487 | } | 558 | } |
488 | 559 | ||
489 | int elv_set_request(request_queue_t *q, struct request *rq, struct bio *bio, | 560 | int elv_set_request(request_queue_t *q, struct request *rq, struct bio *bio, |
490 | int gfp_mask) | 561 | gfp_t gfp_mask) |
491 | { | 562 | { |
492 | elevator_t *e = q->elevator; | 563 | elevator_t *e = q->elevator; |
493 | 564 | ||
@@ -523,11 +594,11 @@ void elv_completed_request(request_queue_t *q, struct request *rq) | |||
523 | /* | 594 | /* |
524 | * request is released from the driver, io must be done | 595 | * request is released from the driver, io must be done |
525 | */ | 596 | */ |
526 | if (blk_account_rq(rq)) | 597 | if (blk_account_rq(rq)) { |
527 | q->in_flight--; | 598 | q->in_flight--; |
528 | 599 | if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn) | |
529 | if (e->ops->elevator_completed_req_fn) | 600 | e->ops->elevator_completed_req_fn(q, rq); |
530 | e->ops->elevator_completed_req_fn(q, rq); | 601 | } |
531 | } | 602 | } |
532 | 603 | ||
533 | int elv_register_queue(struct request_queue *q) | 604 | int elv_register_queue(struct request_queue *q) |
@@ -555,10 +626,9 @@ void elv_unregister_queue(struct request_queue *q) | |||
555 | 626 | ||
556 | int elv_register(struct elevator_type *e) | 627 | int elv_register(struct elevator_type *e) |
557 | { | 628 | { |
629 | spin_lock_irq(&elv_list_lock); | ||
558 | if (elevator_find(e->elevator_name)) | 630 | if (elevator_find(e->elevator_name)) |
559 | BUG(); | 631 | BUG(); |
560 | |||
561 | spin_lock_irq(&elv_list_lock); | ||
562 | list_add_tail(&e->list, &elv_list); | 632 | list_add_tail(&e->list, &elv_list); |
563 | spin_unlock_irq(&elv_list_lock); | 633 | spin_unlock_irq(&elv_list_lock); |
564 | 634 | ||
@@ -582,25 +652,36 @@ EXPORT_SYMBOL_GPL(elv_unregister); | |||
582 | * switch to new_e io scheduler. be careful not to introduce deadlocks - | 652 | * switch to new_e io scheduler. be careful not to introduce deadlocks - |
583 | * we don't free the old io scheduler, before we have allocated what we | 653 | * we don't free the old io scheduler, before we have allocated what we |
584 | * need for the new one. this way we have a chance of going back to the old | 654 | * need for the new one. this way we have a chance of going back to the old |
585 | * one, if the new one fails init for some reason. we also do an intermediate | 655 | * one, if the new one fails init for some reason. |
586 | * switch to noop to ensure safety with stack-allocated requests, since they | ||
587 | * don't originate from the block layer allocator. noop is safe here, because | ||
588 | * it never needs to touch the elevator itself for completion events. DRAIN | ||
589 | * flags will make sure we don't touch it for additions either. | ||
590 | */ | 656 | */ |
591 | static void elevator_switch(request_queue_t *q, struct elevator_type *new_e) | 657 | static void elevator_switch(request_queue_t *q, struct elevator_type *new_e) |
592 | { | 658 | { |
593 | elevator_t *e = kmalloc(sizeof(elevator_t), GFP_KERNEL); | 659 | elevator_t *old_elevator, *e; |
594 | struct elevator_type *noop_elevator = NULL; | ||
595 | elevator_t *old_elevator; | ||
596 | 660 | ||
661 | /* | ||
662 | * Allocate new elevator | ||
663 | */ | ||
664 | e = kmalloc(sizeof(elevator_t), GFP_KERNEL); | ||
597 | if (!e) | 665 | if (!e) |
598 | goto error; | 666 | goto error; |
599 | 667 | ||
600 | /* | 668 | /* |
601 | * first step, drain requests from the block freelist | 669 | * Turn on BYPASS and drain all requests w/ elevator private data |
602 | */ | 670 | */ |
603 | blk_wait_queue_drained(q, 0); | 671 | spin_lock_irq(q->queue_lock); |
672 | |||
673 | set_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags); | ||
674 | |||
675 | while (q->elevator->ops->elevator_dispatch_fn(q, 1)) | ||
676 | ; | ||
677 | |||
678 | while (q->rq.elvpriv) { | ||
679 | spin_unlock_irq(q->queue_lock); | ||
680 | msleep(10); | ||
681 | spin_lock_irq(q->queue_lock); | ||
682 | } | ||
683 | |||
684 | spin_unlock_irq(q->queue_lock); | ||
604 | 685 | ||
605 | /* | 686 | /* |
606 | * unregister old elevator data | 687 | * unregister old elevator data |
@@ -609,18 +690,6 @@ static void elevator_switch(request_queue_t *q, struct elevator_type *new_e) | |||
609 | old_elevator = q->elevator; | 690 | old_elevator = q->elevator; |
610 | 691 | ||
611 | /* | 692 | /* |
612 | * next step, switch to noop since it uses no private rq structures | ||
613 | * and doesn't allocate any memory for anything. then wait for any | ||
614 | * non-fs requests in-flight | ||
615 | */ | ||
616 | noop_elevator = elevator_get("noop"); | ||
617 | spin_lock_irq(q->queue_lock); | ||
618 | elevator_attach(q, noop_elevator, e); | ||
619 | spin_unlock_irq(q->queue_lock); | ||
620 | |||
621 | blk_wait_queue_drained(q, 1); | ||
622 | |||
623 | /* | ||
624 | * attach and start new elevator | 693 | * attach and start new elevator |
625 | */ | 694 | */ |
626 | if (elevator_attach(q, new_e, e)) | 695 | if (elevator_attach(q, new_e, e)) |
@@ -630,11 +699,10 @@ static void elevator_switch(request_queue_t *q, struct elevator_type *new_e) | |||
630 | goto fail_register; | 699 | goto fail_register; |
631 | 700 | ||
632 | /* | 701 | /* |
633 | * finally exit old elevator and start queue again | 702 | * finally exit old elevator and turn off BYPASS. |
634 | */ | 703 | */ |
635 | elevator_exit(old_elevator); | 704 | elevator_exit(old_elevator); |
636 | blk_finish_queue_drain(q); | 705 | clear_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags); |
637 | elevator_put(noop_elevator); | ||
638 | return; | 706 | return; |
639 | 707 | ||
640 | fail_register: | 708 | fail_register: |
@@ -643,13 +711,13 @@ fail_register: | |||
643 | * one again (along with re-adding the sysfs dir) | 711 | * one again (along with re-adding the sysfs dir) |
644 | */ | 712 | */ |
645 | elevator_exit(e); | 713 | elevator_exit(e); |
714 | e = NULL; | ||
646 | fail: | 715 | fail: |
647 | q->elevator = old_elevator; | 716 | q->elevator = old_elevator; |
648 | elv_register_queue(q); | 717 | elv_register_queue(q); |
649 | blk_finish_queue_drain(q); | 718 | clear_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags); |
719 | kfree(e); | ||
650 | error: | 720 | error: |
651 | if (noop_elevator) | ||
652 | elevator_put(noop_elevator); | ||
653 | elevator_put(new_e); | 721 | elevator_put(new_e); |
654 | printk(KERN_ERR "elevator: switch to %s failed\n",new_e->elevator_name); | 722 | printk(KERN_ERR "elevator: switch to %s failed\n",new_e->elevator_name); |
655 | } | 723 | } |
@@ -701,11 +769,12 @@ ssize_t elv_iosched_show(request_queue_t *q, char *name) | |||
701 | return len; | 769 | return len; |
702 | } | 770 | } |
703 | 771 | ||
772 | EXPORT_SYMBOL(elv_dispatch_sort); | ||
704 | EXPORT_SYMBOL(elv_add_request); | 773 | EXPORT_SYMBOL(elv_add_request); |
705 | EXPORT_SYMBOL(__elv_add_request); | 774 | EXPORT_SYMBOL(__elv_add_request); |
706 | EXPORT_SYMBOL(elv_requeue_request); | 775 | EXPORT_SYMBOL(elv_requeue_request); |
707 | EXPORT_SYMBOL(elv_next_request); | 776 | EXPORT_SYMBOL(elv_next_request); |
708 | EXPORT_SYMBOL(elv_remove_request); | 777 | EXPORT_SYMBOL(elv_dequeue_request); |
709 | EXPORT_SYMBOL(elv_queue_empty); | 778 | EXPORT_SYMBOL(elv_queue_empty); |
710 | EXPORT_SYMBOL(elv_completed_request); | 779 | EXPORT_SYMBOL(elv_completed_request); |
711 | EXPORT_SYMBOL(elevator_exit); | 780 | EXPORT_SYMBOL(elevator_exit); |