aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/block/elevator.c
diff options
context:
space:
mode:
authorGreg KH <greg@press.(none)>2005-10-28 13:13:16 -0400
committerGreg Kroah-Hartman <gregkh@suse.de>2005-10-28 13:13:16 -0400
commit6fbfddcb52d8d9fa2cd209f5ac2a1c87497d55b5 (patch)
treec0414e89678fcef7ce3493e048d855bde781ae8d /drivers/block/elevator.c
parent1a222bca26ca691e83be1b08f5e96ae96d0d8cae (diff)
parent27d1097d39509494706eaa2620ef3b1e780a3224 (diff)
Merge ../bleed-2.6
Diffstat (limited to 'drivers/block/elevator.c')
-rw-r--r--drivers/block/elevator.c345
1 files changed, 207 insertions, 138 deletions
diff --git a/drivers/block/elevator.c b/drivers/block/elevator.c
index 98f0126a2deb..55621d5c5774 100644
--- a/drivers/block/elevator.c
+++ b/drivers/block/elevator.c
@@ -34,6 +34,7 @@
34#include <linux/slab.h> 34#include <linux/slab.h>
35#include <linux/init.h> 35#include <linux/init.h>
36#include <linux/compiler.h> 36#include <linux/compiler.h>
37#include <linux/delay.h>
37 38
38#include <asm/uaccess.h> 39#include <asm/uaccess.h>
39 40
@@ -83,21 +84,11 @@ inline int elv_try_merge(struct request *__rq, struct bio *bio)
83} 84}
84EXPORT_SYMBOL(elv_try_merge); 85EXPORT_SYMBOL(elv_try_merge);
85 86
86inline int elv_try_last_merge(request_queue_t *q, struct bio *bio)
87{
88 if (q->last_merge)
89 return elv_try_merge(q->last_merge, bio);
90
91 return ELEVATOR_NO_MERGE;
92}
93EXPORT_SYMBOL(elv_try_last_merge);
94
95static struct elevator_type *elevator_find(const char *name) 87static struct elevator_type *elevator_find(const char *name)
96{ 88{
97 struct elevator_type *e = NULL; 89 struct elevator_type *e = NULL;
98 struct list_head *entry; 90 struct list_head *entry;
99 91
100 spin_lock_irq(&elv_list_lock);
101 list_for_each(entry, &elv_list) { 92 list_for_each(entry, &elv_list) {
102 struct elevator_type *__e; 93 struct elevator_type *__e;
103 94
@@ -108,7 +99,6 @@ static struct elevator_type *elevator_find(const char *name)
108 break; 99 break;
109 } 100 }
110 } 101 }
111 spin_unlock_irq(&elv_list_lock);
112 102
113 return e; 103 return e;
114} 104}
@@ -120,12 +110,15 @@ static void elevator_put(struct elevator_type *e)
120 110
121static struct elevator_type *elevator_get(const char *name) 111static struct elevator_type *elevator_get(const char *name)
122{ 112{
123 struct elevator_type *e = elevator_find(name); 113 struct elevator_type *e;
124 114
125 if (!e) 115 spin_lock_irq(&elv_list_lock);
126 return NULL; 116
127 if (!try_module_get(e->elevator_owner)) 117 e = elevator_find(name);
128 return NULL; 118 if (e && !try_module_get(e->elevator_owner))
119 e = NULL;
120
121 spin_unlock_irq(&elv_list_lock);
129 122
130 return e; 123 return e;
131} 124}
@@ -139,8 +132,6 @@ static int elevator_attach(request_queue_t *q, struct elevator_type *e,
139 eq->ops = &e->ops; 132 eq->ops = &e->ops;
140 eq->elevator_type = e; 133 eq->elevator_type = e;
141 134
142 INIT_LIST_HEAD(&q->queue_head);
143 q->last_merge = NULL;
144 q->elevator = eq; 135 q->elevator = eq;
145 136
146 if (eq->ops->elevator_init_fn) 137 if (eq->ops->elevator_init_fn)
@@ -153,11 +144,15 @@ static char chosen_elevator[16];
153 144
154static void elevator_setup_default(void) 145static void elevator_setup_default(void)
155{ 146{
147 struct elevator_type *e;
148
156 /* 149 /*
157 * check if default is set and exists 150 * check if default is set and exists
158 */ 151 */
159 if (chosen_elevator[0] && elevator_find(chosen_elevator)) 152 if (chosen_elevator[0] && (e = elevator_get(chosen_elevator))) {
153 elevator_put(e);
160 return; 154 return;
155 }
161 156
162#if defined(CONFIG_IOSCHED_AS) 157#if defined(CONFIG_IOSCHED_AS)
163 strcpy(chosen_elevator, "anticipatory"); 158 strcpy(chosen_elevator, "anticipatory");
@@ -186,6 +181,11 @@ int elevator_init(request_queue_t *q, char *name)
186 struct elevator_queue *eq; 181 struct elevator_queue *eq;
187 int ret = 0; 182 int ret = 0;
188 183
184 INIT_LIST_HEAD(&q->queue_head);
185 q->last_merge = NULL;
186 q->end_sector = 0;
187 q->boundary_rq = NULL;
188
189 elevator_setup_default(); 189 elevator_setup_default();
190 190
191 if (!name) 191 if (!name)
@@ -220,9 +220,52 @@ void elevator_exit(elevator_t *e)
220 kfree(e); 220 kfree(e);
221} 221}
222 222
223/*
224 * Insert rq into dispatch queue of q. Queue lock must be held on
225 * entry. If sort != 0, rq is sort-inserted; otherwise, rq will be
226 * appended to the dispatch queue. To be used by specific elevators.
227 */
228void elv_dispatch_sort(request_queue_t *q, struct request *rq)
229{
230 sector_t boundary;
231 struct list_head *entry;
232
233 if (q->last_merge == rq)
234 q->last_merge = NULL;
235
236 boundary = q->end_sector;
237
238 list_for_each_prev(entry, &q->queue_head) {
239 struct request *pos = list_entry_rq(entry);
240
241 if (pos->flags & (REQ_SOFTBARRIER|REQ_HARDBARRIER|REQ_STARTED))
242 break;
243 if (rq->sector >= boundary) {
244 if (pos->sector < boundary)
245 continue;
246 } else {
247 if (pos->sector >= boundary)
248 break;
249 }
250 if (rq->sector >= pos->sector)
251 break;
252 }
253
254 list_add(&rq->queuelist, entry);
255}
256
223int elv_merge(request_queue_t *q, struct request **req, struct bio *bio) 257int elv_merge(request_queue_t *q, struct request **req, struct bio *bio)
224{ 258{
225 elevator_t *e = q->elevator; 259 elevator_t *e = q->elevator;
260 int ret;
261
262 if (q->last_merge) {
263 ret = elv_try_merge(q->last_merge, bio);
264 if (ret != ELEVATOR_NO_MERGE) {
265 *req = q->last_merge;
266 return ret;
267 }
268 }
226 269
227 if (e->ops->elevator_merge_fn) 270 if (e->ops->elevator_merge_fn)
228 return e->ops->elevator_merge_fn(q, req, bio); 271 return e->ops->elevator_merge_fn(q, req, bio);
@@ -236,6 +279,8 @@ void elv_merged_request(request_queue_t *q, struct request *rq)
236 279
237 if (e->ops->elevator_merged_fn) 280 if (e->ops->elevator_merged_fn)
238 e->ops->elevator_merged_fn(q, rq); 281 e->ops->elevator_merged_fn(q, rq);
282
283 q->last_merge = rq;
239} 284}
240 285
241void elv_merge_requests(request_queue_t *q, struct request *rq, 286void elv_merge_requests(request_queue_t *q, struct request *rq,
@@ -243,20 +288,13 @@ void elv_merge_requests(request_queue_t *q, struct request *rq,
243{ 288{
244 elevator_t *e = q->elevator; 289 elevator_t *e = q->elevator;
245 290
246 if (q->last_merge == next)
247 q->last_merge = NULL;
248
249 if (e->ops->elevator_merge_req_fn) 291 if (e->ops->elevator_merge_req_fn)
250 e->ops->elevator_merge_req_fn(q, rq, next); 292 e->ops->elevator_merge_req_fn(q, rq, next);
293
294 q->last_merge = rq;
251} 295}
252 296
253/* 297void elv_requeue_request(request_queue_t *q, struct request *rq)
254 * For careful internal use by the block layer. Essentially the same as
255 * a requeue in that it tells the io scheduler that this request is not
256 * active in the driver or hardware anymore, but we don't want the request
257 * added back to the scheduler. Function is not exported.
258 */
259void elv_deactivate_request(request_queue_t *q, struct request *rq)
260{ 298{
261 elevator_t *e = q->elevator; 299 elevator_t *e = q->elevator;
262 300
@@ -264,19 +302,14 @@ void elv_deactivate_request(request_queue_t *q, struct request *rq)
264 * it already went through dequeue, we need to decrement the 302 * it already went through dequeue, we need to decrement the
265 * in_flight count again 303 * in_flight count again
266 */ 304 */
267 if (blk_account_rq(rq)) 305 if (blk_account_rq(rq)) {
268 q->in_flight--; 306 q->in_flight--;
307 if (blk_sorted_rq(rq) && e->ops->elevator_deactivate_req_fn)
308 e->ops->elevator_deactivate_req_fn(q, rq);
309 }
269 310
270 rq->flags &= ~REQ_STARTED; 311 rq->flags &= ~REQ_STARTED;
271 312
272 if (e->ops->elevator_deactivate_req_fn)
273 e->ops->elevator_deactivate_req_fn(q, rq);
274}
275
276void elv_requeue_request(request_queue_t *q, struct request *rq)
277{
278 elv_deactivate_request(q, rq);
279
280 /* 313 /*
281 * if this is the flush, requeue the original instead and drop the flush 314 * if this is the flush, requeue the original instead and drop the flush
282 */ 315 */
@@ -285,31 +318,27 @@ void elv_requeue_request(request_queue_t *q, struct request *rq)
285 rq = rq->end_io_data; 318 rq = rq->end_io_data;
286 } 319 }
287 320
288 /* 321 __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
289 * the request is prepped and may have some resources allocated.
290 * allowing unprepped requests to pass this one may cause resource
291 * deadlock. turn on softbarrier.
292 */
293 rq->flags |= REQ_SOFTBARRIER;
294
295 /*
296 * if iosched has an explicit requeue hook, then use that. otherwise
297 * just put the request at the front of the queue
298 */
299 if (q->elevator->ops->elevator_requeue_req_fn)
300 q->elevator->ops->elevator_requeue_req_fn(q, rq);
301 else
302 __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
303} 322}
304 323
305void __elv_add_request(request_queue_t *q, struct request *rq, int where, 324void __elv_add_request(request_queue_t *q, struct request *rq, int where,
306 int plug) 325 int plug)
307{ 326{
308 /* 327 if (rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER)) {
309 * barriers implicitly indicate back insertion 328 /*
310 */ 329 * barriers implicitly indicate back insertion
311 if (rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER) && 330 */
312 where == ELEVATOR_INSERT_SORT) 331 if (where == ELEVATOR_INSERT_SORT)
332 where = ELEVATOR_INSERT_BACK;
333
334 /*
335 * this request is scheduling boundary, update end_sector
336 */
337 if (blk_fs_request(rq)) {
338 q->end_sector = rq_end_sector(rq);
339 q->boundary_rq = rq;
340 }
341 } else if (!(rq->flags & REQ_ELVPRIV) && where == ELEVATOR_INSERT_SORT)
313 where = ELEVATOR_INSERT_BACK; 342 where = ELEVATOR_INSERT_BACK;
314 343
315 if (plug) 344 if (plug)
@@ -317,23 +346,54 @@ void __elv_add_request(request_queue_t *q, struct request *rq, int where,
317 346
318 rq->q = q; 347 rq->q = q;
319 348
320 if (!test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags)) { 349 switch (where) {
321 q->elevator->ops->elevator_add_req_fn(q, rq, where); 350 case ELEVATOR_INSERT_FRONT:
351 rq->flags |= REQ_SOFTBARRIER;
322 352
323 if (blk_queue_plugged(q)) { 353 list_add(&rq->queuelist, &q->queue_head);
324 int nrq = q->rq.count[READ] + q->rq.count[WRITE] 354 break;
325 - q->in_flight;
326 355
327 if (nrq >= q->unplug_thresh) 356 case ELEVATOR_INSERT_BACK:
328 __generic_unplug_device(q); 357 rq->flags |= REQ_SOFTBARRIER;
329 } 358
330 } else 359 while (q->elevator->ops->elevator_dispatch_fn(q, 1))
360 ;
361 list_add_tail(&rq->queuelist, &q->queue_head);
331 /* 362 /*
332 * if drain is set, store the request "locally". when the drain 363 * We kick the queue here for the following reasons.
333 * is finished, the requests will be handed ordered to the io 364 * - The elevator might have returned NULL previously
334 * scheduler 365 * to delay requests and returned them now. As the
366 * queue wasn't empty before this request, ll_rw_blk
367 * won't run the queue on return, resulting in hang.
368 * - Usually, back inserted requests won't be merged
369 * with anything. There's no point in delaying queue
370 * processing.
335 */ 371 */
336 list_add_tail(&rq->queuelist, &q->drain_list); 372 blk_remove_plug(q);
373 q->request_fn(q);
374 break;
375
376 case ELEVATOR_INSERT_SORT:
377 BUG_ON(!blk_fs_request(rq));
378 rq->flags |= REQ_SORTED;
379 q->elevator->ops->elevator_add_req_fn(q, rq);
380 if (q->last_merge == NULL && rq_mergeable(rq))
381 q->last_merge = rq;
382 break;
383
384 default:
385 printk(KERN_ERR "%s: bad insertion point %d\n",
386 __FUNCTION__, where);
387 BUG();
388 }
389
390 if (blk_queue_plugged(q)) {
391 int nrq = q->rq.count[READ] + q->rq.count[WRITE]
392 - q->in_flight;
393
394 if (nrq >= q->unplug_thresh)
395 __generic_unplug_device(q);
396 }
337} 397}
338 398
339void elv_add_request(request_queue_t *q, struct request *rq, int where, 399void elv_add_request(request_queue_t *q, struct request *rq, int where,
@@ -348,13 +408,19 @@ void elv_add_request(request_queue_t *q, struct request *rq, int where,
348 408
349static inline struct request *__elv_next_request(request_queue_t *q) 409static inline struct request *__elv_next_request(request_queue_t *q)
350{ 410{
351 struct request *rq = q->elevator->ops->elevator_next_req_fn(q); 411 struct request *rq;
412
413 if (unlikely(list_empty(&q->queue_head) &&
414 !q->elevator->ops->elevator_dispatch_fn(q, 0)))
415 return NULL;
416
417 rq = list_entry_rq(q->queue_head.next);
352 418
353 /* 419 /*
354 * if this is a barrier write and the device has to issue a 420 * if this is a barrier write and the device has to issue a
355 * flush sequence to support it, check how far we are 421 * flush sequence to support it, check how far we are
356 */ 422 */
357 if (rq && blk_fs_request(rq) && blk_barrier_rq(rq)) { 423 if (blk_fs_request(rq) && blk_barrier_rq(rq)) {
358 BUG_ON(q->ordered == QUEUE_ORDERED_NONE); 424 BUG_ON(q->ordered == QUEUE_ORDERED_NONE);
359 425
360 if (q->ordered == QUEUE_ORDERED_FLUSH && 426 if (q->ordered == QUEUE_ORDERED_FLUSH &&
@@ -371,15 +437,30 @@ struct request *elv_next_request(request_queue_t *q)
371 int ret; 437 int ret;
372 438
373 while ((rq = __elv_next_request(q)) != NULL) { 439 while ((rq = __elv_next_request(q)) != NULL) {
374 /* 440 if (!(rq->flags & REQ_STARTED)) {
375 * just mark as started even if we don't start it, a request 441 elevator_t *e = q->elevator;
376 * that has been delayed should not be passed by new incoming
377 * requests
378 */
379 rq->flags |= REQ_STARTED;
380 442
381 if (rq == q->last_merge) 443 /*
382 q->last_merge = NULL; 444 * This is the first time the device driver
445 * sees this request (possibly after
446 * requeueing). Notify IO scheduler.
447 */
448 if (blk_sorted_rq(rq) &&
449 e->ops->elevator_activate_req_fn)
450 e->ops->elevator_activate_req_fn(q, rq);
451
452 /*
453 * just mark as started even if we don't start
454 * it, a request that has been delayed should
455 * not be passed by new incoming requests
456 */
457 rq->flags |= REQ_STARTED;
458 }
459
460 if (!q->boundary_rq || q->boundary_rq == rq) {
461 q->end_sector = rq_end_sector(rq);
462 q->boundary_rq = NULL;
463 }
383 464
384 if ((rq->flags & REQ_DONTPREP) || !q->prep_rq_fn) 465 if ((rq->flags & REQ_DONTPREP) || !q->prep_rq_fn)
385 break; 466 break;
@@ -391,9 +472,9 @@ struct request *elv_next_request(request_queue_t *q)
391 /* 472 /*
392 * the request may have been (partially) prepped. 473 * the request may have been (partially) prepped.
393 * we need to keep this request in the front to 474 * we need to keep this request in the front to
394 * avoid resource deadlock. turn on softbarrier. 475 * avoid resource deadlock. REQ_STARTED will
476 * prevent other fs requests from passing this one.
395 */ 477 */
396 rq->flags |= REQ_SOFTBARRIER;
397 rq = NULL; 478 rq = NULL;
398 break; 479 break;
399 } else if (ret == BLKPREP_KILL) { 480 } else if (ret == BLKPREP_KILL) {
@@ -416,42 +497,32 @@ struct request *elv_next_request(request_queue_t *q)
416 return rq; 497 return rq;
417} 498}
418 499
419void elv_remove_request(request_queue_t *q, struct request *rq) 500void elv_dequeue_request(request_queue_t *q, struct request *rq)
420{ 501{
421 elevator_t *e = q->elevator; 502 BUG_ON(list_empty(&rq->queuelist));
503
504 list_del_init(&rq->queuelist);
422 505
423 /* 506 /*
424 * the time frame between a request being removed from the lists 507 * the time frame between a request being removed from the lists
425 * and to it is freed is accounted as io that is in progress at 508 * and to it is freed is accounted as io that is in progress at
426 * the driver side. note that we only account requests that the 509 * the driver side.
427 * driver has seen (REQ_STARTED set), to avoid false accounting
428 * for request-request merges
429 */ 510 */
430 if (blk_account_rq(rq)) 511 if (blk_account_rq(rq))
431 q->in_flight++; 512 q->in_flight++;
432
433 /*
434 * the main clearing point for q->last_merge is on retrieval of
435 * request by driver (it calls elv_next_request()), but it _can_
436 * also happen here if a request is added to the queue but later
437 * deleted without ever being given to driver (merged with another
438 * request).
439 */
440 if (rq == q->last_merge)
441 q->last_merge = NULL;
442
443 if (e->ops->elevator_remove_req_fn)
444 e->ops->elevator_remove_req_fn(q, rq);
445} 513}
446 514
447int elv_queue_empty(request_queue_t *q) 515int elv_queue_empty(request_queue_t *q)
448{ 516{
449 elevator_t *e = q->elevator; 517 elevator_t *e = q->elevator;
450 518
519 if (!list_empty(&q->queue_head))
520 return 0;
521
451 if (e->ops->elevator_queue_empty_fn) 522 if (e->ops->elevator_queue_empty_fn)
452 return e->ops->elevator_queue_empty_fn(q); 523 return e->ops->elevator_queue_empty_fn(q);
453 524
454 return list_empty(&q->queue_head); 525 return 1;
455} 526}
456 527
457struct request *elv_latter_request(request_queue_t *q, struct request *rq) 528struct request *elv_latter_request(request_queue_t *q, struct request *rq)
@@ -487,7 +558,7 @@ struct request *elv_former_request(request_queue_t *q, struct request *rq)
487} 558}
488 559
489int elv_set_request(request_queue_t *q, struct request *rq, struct bio *bio, 560int elv_set_request(request_queue_t *q, struct request *rq, struct bio *bio,
490 int gfp_mask) 561 gfp_t gfp_mask)
491{ 562{
492 elevator_t *e = q->elevator; 563 elevator_t *e = q->elevator;
493 564
@@ -523,11 +594,11 @@ void elv_completed_request(request_queue_t *q, struct request *rq)
523 /* 594 /*
524 * request is released from the driver, io must be done 595 * request is released from the driver, io must be done
525 */ 596 */
526 if (blk_account_rq(rq)) 597 if (blk_account_rq(rq)) {
527 q->in_flight--; 598 q->in_flight--;
528 599 if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn)
529 if (e->ops->elevator_completed_req_fn) 600 e->ops->elevator_completed_req_fn(q, rq);
530 e->ops->elevator_completed_req_fn(q, rq); 601 }
531} 602}
532 603
533int elv_register_queue(struct request_queue *q) 604int elv_register_queue(struct request_queue *q)
@@ -555,10 +626,9 @@ void elv_unregister_queue(struct request_queue *q)
555 626
556int elv_register(struct elevator_type *e) 627int elv_register(struct elevator_type *e)
557{ 628{
629 spin_lock_irq(&elv_list_lock);
558 if (elevator_find(e->elevator_name)) 630 if (elevator_find(e->elevator_name))
559 BUG(); 631 BUG();
560
561 spin_lock_irq(&elv_list_lock);
562 list_add_tail(&e->list, &elv_list); 632 list_add_tail(&e->list, &elv_list);
563 spin_unlock_irq(&elv_list_lock); 633 spin_unlock_irq(&elv_list_lock);
564 634
@@ -582,25 +652,36 @@ EXPORT_SYMBOL_GPL(elv_unregister);
582 * switch to new_e io scheduler. be careful not to introduce deadlocks - 652 * switch to new_e io scheduler. be careful not to introduce deadlocks -
583 * we don't free the old io scheduler, before we have allocated what we 653 * we don't free the old io scheduler, before we have allocated what we
584 * need for the new one. this way we have a chance of going back to the old 654 * need for the new one. this way we have a chance of going back to the old
585 * one, if the new one fails init for some reason. we also do an intermediate 655 * one, if the new one fails init for some reason.
586 * switch to noop to ensure safety with stack-allocated requests, since they
587 * don't originate from the block layer allocator. noop is safe here, because
588 * it never needs to touch the elevator itself for completion events. DRAIN
589 * flags will make sure we don't touch it for additions either.
590 */ 656 */
591static void elevator_switch(request_queue_t *q, struct elevator_type *new_e) 657static void elevator_switch(request_queue_t *q, struct elevator_type *new_e)
592{ 658{
593 elevator_t *e = kmalloc(sizeof(elevator_t), GFP_KERNEL); 659 elevator_t *old_elevator, *e;
594 struct elevator_type *noop_elevator = NULL;
595 elevator_t *old_elevator;
596 660
661 /*
662 * Allocate new elevator
663 */
664 e = kmalloc(sizeof(elevator_t), GFP_KERNEL);
597 if (!e) 665 if (!e)
598 goto error; 666 goto error;
599 667
600 /* 668 /*
601 * first step, drain requests from the block freelist 669 * Turn on BYPASS and drain all requests w/ elevator private data
602 */ 670 */
603 blk_wait_queue_drained(q, 0); 671 spin_lock_irq(q->queue_lock);
672
673 set_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
674
675 while (q->elevator->ops->elevator_dispatch_fn(q, 1))
676 ;
677
678 while (q->rq.elvpriv) {
679 spin_unlock_irq(q->queue_lock);
680 msleep(10);
681 spin_lock_irq(q->queue_lock);
682 }
683
684 spin_unlock_irq(q->queue_lock);
604 685
605 /* 686 /*
606 * unregister old elevator data 687 * unregister old elevator data
@@ -609,18 +690,6 @@ static void elevator_switch(request_queue_t *q, struct elevator_type *new_e)
609 old_elevator = q->elevator; 690 old_elevator = q->elevator;
610 691
611 /* 692 /*
612 * next step, switch to noop since it uses no private rq structures
613 * and doesn't allocate any memory for anything. then wait for any
614 * non-fs requests in-flight
615 */
616 noop_elevator = elevator_get("noop");
617 spin_lock_irq(q->queue_lock);
618 elevator_attach(q, noop_elevator, e);
619 spin_unlock_irq(q->queue_lock);
620
621 blk_wait_queue_drained(q, 1);
622
623 /*
624 * attach and start new elevator 693 * attach and start new elevator
625 */ 694 */
626 if (elevator_attach(q, new_e, e)) 695 if (elevator_attach(q, new_e, e))
@@ -630,11 +699,10 @@ static void elevator_switch(request_queue_t *q, struct elevator_type *new_e)
630 goto fail_register; 699 goto fail_register;
631 700
632 /* 701 /*
633 * finally exit old elevator and start queue again 702 * finally exit old elevator and turn off BYPASS.
634 */ 703 */
635 elevator_exit(old_elevator); 704 elevator_exit(old_elevator);
636 blk_finish_queue_drain(q); 705 clear_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
637 elevator_put(noop_elevator);
638 return; 706 return;
639 707
640fail_register: 708fail_register:
@@ -643,13 +711,13 @@ fail_register:
643 * one again (along with re-adding the sysfs dir) 711 * one again (along with re-adding the sysfs dir)
644 */ 712 */
645 elevator_exit(e); 713 elevator_exit(e);
714 e = NULL;
646fail: 715fail:
647 q->elevator = old_elevator; 716 q->elevator = old_elevator;
648 elv_register_queue(q); 717 elv_register_queue(q);
649 blk_finish_queue_drain(q); 718 clear_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
719 kfree(e);
650error: 720error:
651 if (noop_elevator)
652 elevator_put(noop_elevator);
653 elevator_put(new_e); 721 elevator_put(new_e);
654 printk(KERN_ERR "elevator: switch to %s failed\n",new_e->elevator_name); 722 printk(KERN_ERR "elevator: switch to %s failed\n",new_e->elevator_name);
655} 723}
@@ -701,11 +769,12 @@ ssize_t elv_iosched_show(request_queue_t *q, char *name)
701 return len; 769 return len;
702} 770}
703 771
772EXPORT_SYMBOL(elv_dispatch_sort);
704EXPORT_SYMBOL(elv_add_request); 773EXPORT_SYMBOL(elv_add_request);
705EXPORT_SYMBOL(__elv_add_request); 774EXPORT_SYMBOL(__elv_add_request);
706EXPORT_SYMBOL(elv_requeue_request); 775EXPORT_SYMBOL(elv_requeue_request);
707EXPORT_SYMBOL(elv_next_request); 776EXPORT_SYMBOL(elv_next_request);
708EXPORT_SYMBOL(elv_remove_request); 777EXPORT_SYMBOL(elv_dequeue_request);
709EXPORT_SYMBOL(elv_queue_empty); 778EXPORT_SYMBOL(elv_queue_empty);
710EXPORT_SYMBOL(elv_completed_request); 779EXPORT_SYMBOL(elv_completed_request);
711EXPORT_SYMBOL(elevator_exit); 780EXPORT_SYMBOL(elevator_exit);