summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJens Axboe <axboe@kernel.dk>2018-10-29 12:23:51 -0400
committerJens Axboe <axboe@kernel.dk>2018-11-07 15:42:32 -0500
commita1ce35fa49852db60fc6e268038530be533c5b15 (patch)
tree7a0bb9576a3f0e31e22f849463eee6cdda26bad5
parentf382fb0bcef4c37dc049e9f6963e3baf204d815c (diff)
block: remove dead elevator code
This removes a bunch of core and elevator related code. On the core front, we remove anything related to queue running, draining, initialization, plugging, and congestions. We also kill anything related to request allocation, merging, retrieval, and completion. Remove any checking for single queue IO schedulers, as they no longer exist. This means we can also delete a bunch of code related to request issue, adding, completion, etc - and all the SQ related ops and helpers. Also kill the load_default_modules(), as all that did was provide for a way to load the default single queue elevator. Tested-by: Ming Lei <ming.lei@redhat.com> Reviewed-by: Omar Sandoval <osandov@fb.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat
-rw-r--r--block/bfq-iosched.c1
-rw-r--r--block/blk-core.c1709
-rw-r--r--block/blk-exec.c20
-rw-r--r--block/blk-ioc.c33
-rw-r--r--block/blk-merge.c5
-rw-r--r--block/blk-settings.c36
-rw-r--r--block/blk-sysfs.c36
-rw-r--r--block/blk.h51
-rw-r--r--block/elevator.c377
-rw-r--r--block/kyber-iosched.c1
-rw-r--r--block/mq-deadline.c1
-rw-r--r--include/linux/blkdev.h93
-rw-r--r--include/linux/elevator.h90
-rw-r--r--include/linux/init.h1
-rw-r--r--init/do_mounts_initrd.c3
-rw-r--r--init/initramfs.c6
-rw-r--r--init/main.c12
17 files changed, 55 insertions, 2420 deletions
diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index 3a27d31fcda6..44c7e567aa25 100644
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -5745,7 +5745,6 @@ static struct elevator_type iosched_bfq_mq = {
5745 .exit_sched = bfq_exit_queue, 5745 .exit_sched = bfq_exit_queue,
5746 }, 5746 },
5747 5747
5748 .uses_mq = true,
5749 .icq_size = sizeof(struct bfq_io_cq), 5748 .icq_size = sizeof(struct bfq_io_cq),
5750 .icq_align = __alignof__(struct bfq_io_cq), 5749 .icq_align = __alignof__(struct bfq_io_cq),
5751 .elevator_attrs = bfq_attrs, 5750 .elevator_attrs = bfq_attrs,
diff --git a/block/blk-core.c b/block/blk-core.c
index daaed4dfa719..18538a41a532 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -144,46 +144,6 @@ bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q)
144} 144}
145EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_clear); 145EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_clear);
146 146
147static void blk_clear_congested(struct request_list *rl, int sync)
148{
149#ifdef CONFIG_CGROUP_WRITEBACK
150 clear_wb_congested(rl->blkg->wb_congested, sync);
151#else
152 /*
153 * If !CGROUP_WRITEBACK, all blkg's map to bdi->wb and we shouldn't
154 * flip its congestion state for events on other blkcgs.
155 */
156 if (rl == &rl->q->root_rl)
157 clear_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
158#endif
159}
160
161static void blk_set_congested(struct request_list *rl, int sync)
162{
163#ifdef CONFIG_CGROUP_WRITEBACK
164 set_wb_congested(rl->blkg->wb_congested, sync);
165#else
166 /* see blk_clear_congested() */
167 if (rl == &rl->q->root_rl)
168 set_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
169#endif
170}
171
172void blk_queue_congestion_threshold(struct request_queue *q)
173{
174 int nr;
175
176 nr = q->nr_requests - (q->nr_requests / 8) + 1;
177 if (nr > q->nr_requests)
178 nr = q->nr_requests;
179 q->nr_congestion_on = nr;
180
181 nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
182 if (nr < 1)
183 nr = 1;
184 q->nr_congestion_off = nr;
185}
186
187void blk_rq_init(struct request_queue *q, struct request *rq) 147void blk_rq_init(struct request_queue *q, struct request *rq)
188{ 148{
189 memset(rq, 0, sizeof(*rq)); 149 memset(rq, 0, sizeof(*rq));
@@ -292,99 +252,6 @@ void blk_dump_rq_flags(struct request *rq, char *msg)
292} 252}
293EXPORT_SYMBOL(blk_dump_rq_flags); 253EXPORT_SYMBOL(blk_dump_rq_flags);
294 254
295static void blk_delay_work(struct work_struct *work)
296{
297 struct request_queue *q;
298
299 q = container_of(work, struct request_queue, delay_work.work);
300 spin_lock_irq(q->queue_lock);
301 __blk_run_queue(q);
302 spin_unlock_irq(q->queue_lock);
303}
304
305/**
306 * blk_delay_queue - restart queueing after defined interval
307 * @q: The &struct request_queue in question
308 * @msecs: Delay in msecs
309 *
310 * Description:
311 * Sometimes queueing needs to be postponed for a little while, to allow
312 * resources to come back. This function will make sure that queueing is
313 * restarted around the specified time.
314 */
315void blk_delay_queue(struct request_queue *q, unsigned long msecs)
316{
317 lockdep_assert_held(q->queue_lock);
318 WARN_ON_ONCE(q->mq_ops);
319
320 if (likely(!blk_queue_dead(q)))
321 queue_delayed_work(kblockd_workqueue, &q->delay_work,
322 msecs_to_jiffies(msecs));
323}
324EXPORT_SYMBOL(blk_delay_queue);
325
326/**
327 * blk_start_queue_async - asynchronously restart a previously stopped queue
328 * @q: The &struct request_queue in question
329 *
330 * Description:
331 * blk_start_queue_async() will clear the stop flag on the queue, and
332 * ensure that the request_fn for the queue is run from an async
333 * context.
334 **/
335void blk_start_queue_async(struct request_queue *q)
336{
337 lockdep_assert_held(q->queue_lock);
338 WARN_ON_ONCE(q->mq_ops);
339
340 queue_flag_clear(QUEUE_FLAG_STOPPED, q);
341 blk_run_queue_async(q);
342}
343EXPORT_SYMBOL(blk_start_queue_async);
344
345/**
346 * blk_start_queue - restart a previously stopped queue
347 * @q: The &struct request_queue in question
348 *
349 * Description:
350 * blk_start_queue() will clear the stop flag on the queue, and call
351 * the request_fn for the queue if it was in a stopped state when
352 * entered. Also see blk_stop_queue().
353 **/
354void blk_start_queue(struct request_queue *q)
355{
356 lockdep_assert_held(q->queue_lock);
357 WARN_ON_ONCE(q->mq_ops);
358
359 queue_flag_clear(QUEUE_FLAG_STOPPED, q);
360 __blk_run_queue(q);
361}
362EXPORT_SYMBOL(blk_start_queue);
363
364/**
365 * blk_stop_queue - stop a queue
366 * @q: The &struct request_queue in question
367 *
368 * Description:
369 * The Linux block layer assumes that a block driver will consume all
370 * entries on the request queue when the request_fn strategy is called.
371 * Often this will not happen, because of hardware limitations (queue
372 * depth settings). If a device driver gets a 'queue full' response,
373 * or if it simply chooses not to queue more I/O at one point, it can
374 * call this function to prevent the request_fn from being called until
375 * the driver has signalled it's ready to go again. This happens by calling
376 * blk_start_queue() to restart queue operations.
377 **/
378void blk_stop_queue(struct request_queue *q)
379{
380 lockdep_assert_held(q->queue_lock);
381 WARN_ON_ONCE(q->mq_ops);
382
383 cancel_delayed_work(&q->delay_work);
384 queue_flag_set(QUEUE_FLAG_STOPPED, q);
385}
386EXPORT_SYMBOL(blk_stop_queue);
387
388/** 255/**
389 * blk_sync_queue - cancel any pending callbacks on a queue 256 * blk_sync_queue - cancel any pending callbacks on a queue
390 * @q: the queue 257 * @q: the queue
@@ -415,8 +282,6 @@ void blk_sync_queue(struct request_queue *q)
415 cancel_delayed_work_sync(&q->requeue_work); 282 cancel_delayed_work_sync(&q->requeue_work);
416 queue_for_each_hw_ctx(q, hctx, i) 283 queue_for_each_hw_ctx(q, hctx, i)
417 cancel_delayed_work_sync(&hctx->run_work); 284 cancel_delayed_work_sync(&hctx->run_work);
418 } else {
419 cancel_delayed_work_sync(&q->delay_work);
420 } 285 }
421} 286}
422EXPORT_SYMBOL(blk_sync_queue); 287EXPORT_SYMBOL(blk_sync_queue);
@@ -442,250 +307,12 @@ void blk_clear_pm_only(struct request_queue *q)
442} 307}
443EXPORT_SYMBOL_GPL(blk_clear_pm_only); 308EXPORT_SYMBOL_GPL(blk_clear_pm_only);
444 309
445/**
446 * __blk_run_queue_uncond - run a queue whether or not it has been stopped
447 * @q: The queue to run
448 *
449 * Description:
450 * Invoke request handling on a queue if there are any pending requests.
451 * May be used to restart request handling after a request has completed.
452 * This variant runs the queue whether or not the queue has been
453 * stopped. Must be called with the queue lock held and interrupts
454 * disabled. See also @blk_run_queue.
455 */
456inline void __blk_run_queue_uncond(struct request_queue *q)
457{
458 lockdep_assert_held(q->queue_lock);
459 WARN_ON_ONCE(q->mq_ops);
460
461 if (unlikely(blk_queue_dead(q)))
462 return;
463
464 /*
465 * Some request_fn implementations, e.g. scsi_request_fn(), unlock
466 * the queue lock internally. As a result multiple threads may be
467 * running such a request function concurrently. Keep track of the
468 * number of active request_fn invocations such that blk_drain_queue()
469 * can wait until all these request_fn calls have finished.
470 */
471 q->request_fn_active++;
472 q->request_fn(q);
473 q->request_fn_active--;
474}
475EXPORT_SYMBOL_GPL(__blk_run_queue_uncond);
476
477/**
478 * __blk_run_queue - run a single device queue
479 * @q: The queue to run
480 *
481 * Description:
482 * See @blk_run_queue.
483 */
484void __blk_run_queue(struct request_queue *q)
485{
486 lockdep_assert_held(q->queue_lock);
487 WARN_ON_ONCE(q->mq_ops);
488
489 if (unlikely(blk_queue_stopped(q)))
490 return;
491
492 __blk_run_queue_uncond(q);
493}
494EXPORT_SYMBOL(__blk_run_queue);
495
496/**
497 * blk_run_queue_async - run a single device queue in workqueue context
498 * @q: The queue to run
499 *
500 * Description:
501 * Tells kblockd to perform the equivalent of @blk_run_queue on behalf
502 * of us.
503 *
504 * Note:
505 * Since it is not allowed to run q->delay_work after blk_cleanup_queue()
506 * has canceled q->delay_work, callers must hold the queue lock to avoid
507 * race conditions between blk_cleanup_queue() and blk_run_queue_async().
508 */
509void blk_run_queue_async(struct request_queue *q)
510{
511 lockdep_assert_held(q->queue_lock);
512 WARN_ON_ONCE(q->mq_ops);
513
514 if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q)))
515 mod_delayed_work(kblockd_workqueue, &q->delay_work, 0);
516}
517EXPORT_SYMBOL(blk_run_queue_async);
518
519/**
520 * blk_run_queue - run a single device queue
521 * @q: The queue to run
522 *
523 * Description:
524 * Invoke request handling on this queue, if it has pending work to do.
525 * May be used to restart queueing when a request has completed.
526 */
527void blk_run_queue(struct request_queue *q)
528{
529 unsigned long flags;
530
531 WARN_ON_ONCE(q->mq_ops);
532
533 spin_lock_irqsave(q->queue_lock, flags);
534 __blk_run_queue(q);
535 spin_unlock_irqrestore(q->queue_lock, flags);
536}
537EXPORT_SYMBOL(blk_run_queue);
538
539void blk_put_queue(struct request_queue *q) 310void blk_put_queue(struct request_queue *q)
540{ 311{
541 kobject_put(&q->kobj); 312 kobject_put(&q->kobj);
542} 313}
543EXPORT_SYMBOL(blk_put_queue); 314EXPORT_SYMBOL(blk_put_queue);
544 315
545/**
546 * __blk_drain_queue - drain requests from request_queue
547 * @q: queue to drain
548 * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV
549 *
550 * Drain requests from @q. If @drain_all is set, all requests are drained.
551 * If not, only ELVPRIV requests are drained. The caller is responsible
552 * for ensuring that no new requests which need to be drained are queued.
553 */
554static void __blk_drain_queue(struct request_queue *q, bool drain_all)
555 __releases(q->queue_lock)
556 __acquires(q->queue_lock)
557{
558 int i;
559
560 lockdep_assert_held(q->queue_lock);
561 WARN_ON_ONCE(q->mq_ops);
562
563 while (true) {
564 bool drain = false;
565
566 /*
567 * The caller might be trying to drain @q before its
568 * elevator is initialized.
569 */
570 if (q->elevator)
571 elv_drain_elevator(q);
572
573 blkcg_drain_queue(q);
574
575 /*
576 * This function might be called on a queue which failed
577 * driver init after queue creation or is not yet fully
578 * active yet. Some drivers (e.g. fd and loop) get unhappy
579 * in such cases. Kick queue iff dispatch queue has
580 * something on it and @q has request_fn set.
581 */
582 if (!list_empty(&q->queue_head) && q->request_fn)
583 __blk_run_queue(q);
584
585 drain |= q->nr_rqs_elvpriv;
586 drain |= q->request_fn_active;
587
588 /*
589 * Unfortunately, requests are queued at and tracked from
590 * multiple places and there's no single counter which can
591 * be drained. Check all the queues and counters.
592 */
593 if (drain_all) {
594 struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
595 drain |= !list_empty(&q->queue_head);
596 for (i = 0; i < 2; i++) {
597 drain |= q->nr_rqs[i];
598 drain |= q->in_flight[i];
599 if (fq)
600 drain |= !list_empty(&fq->flush_queue[i]);
601 }
602 }
603
604 if (!drain)
605 break;
606
607 spin_unlock_irq(q->queue_lock);
608
609 msleep(10);
610
611 spin_lock_irq(q->queue_lock);
612 }
613
614 /*
615 * With queue marked dead, any woken up waiter will fail the
616 * allocation path, so the wakeup chaining is lost and we're
617 * left with hung waiters. We need to wake up those waiters.
618 */
619 if (q->request_fn) {
620 struct request_list *rl;
621
622 blk_queue_for_each_rl(rl, q)
623 for (i = 0; i < ARRAY_SIZE(rl->wait); i++)
624 wake_up_all(&rl->wait[i]);
625 }
626}
627
628void blk_drain_queue(struct request_queue *q)
629{
630 spin_lock_irq(q->queue_lock);
631 __blk_drain_queue(q, true);
632 spin_unlock_irq(q->queue_lock);
633}
634
635/**
636 * blk_queue_bypass_start - enter queue bypass mode
637 * @q: queue of interest
638 *
639 * In bypass mode, only the dispatch FIFO queue of @q is used. This
640 * function makes @q enter bypass mode and drains all requests which were
641 * throttled or issued before. On return, it's guaranteed that no request
642 * is being throttled or has ELVPRIV set and blk_queue_bypass() %true
643 * inside queue or RCU read lock.
644 */
645void blk_queue_bypass_start(struct request_queue *q)
646{
647 WARN_ON_ONCE(q->mq_ops);
648
649 spin_lock_irq(q->queue_lock);
650 q->bypass_depth++;
651 queue_flag_set(QUEUE_FLAG_BYPASS, q);
652 spin_unlock_irq(q->queue_lock);
653
654 /*
655 * Queues start drained. Skip actual draining till init is
656 * complete. This avoids lenghty delays during queue init which
657 * can happen many times during boot.
658 */
659 if (blk_queue_init_done(q)) {
660 spin_lock_irq(q->queue_lock);
661 __blk_drain_queue(q, false);
662 spin_unlock_irq(q->queue_lock);
663
664 /* ensure blk_queue_bypass() is %true inside RCU read lock */
665 synchronize_rcu();
666 }
667}
668EXPORT_SYMBOL_GPL(blk_queue_bypass_start);
669
670/**
671 * blk_queue_bypass_end - leave queue bypass mode
672 * @q: queue of interest
673 *
674 * Leave bypass mode and restore the normal queueing behavior.
675 *
676 * Note: although blk_queue_bypass_start() is only called for blk-sq queues,
677 * this function is called for both blk-sq and blk-mq queues.
678 */
679void blk_queue_bypass_end(struct request_queue *q)
680{
681 spin_lock_irq(q->queue_lock);
682 if (!--q->bypass_depth)
683 queue_flag_clear(QUEUE_FLAG_BYPASS, q);
684 WARN_ON_ONCE(q->bypass_depth < 0);
685 spin_unlock_irq(q->queue_lock);
686}
687EXPORT_SYMBOL_GPL(blk_queue_bypass_end);
688
689void blk_set_queue_dying(struct request_queue *q) 316void blk_set_queue_dying(struct request_queue *q)
690{ 317{
691 blk_queue_flag_set(QUEUE_FLAG_DYING, q); 318 blk_queue_flag_set(QUEUE_FLAG_DYING, q);
@@ -699,18 +326,6 @@ void blk_set_queue_dying(struct request_queue *q)
699 326
700 if (q->mq_ops) 327 if (q->mq_ops)
701 blk_mq_wake_waiters(q); 328 blk_mq_wake_waiters(q);
702 else {
703 struct request_list *rl;
704
705 spin_lock_irq(q->queue_lock);
706 blk_queue_for_each_rl(rl, q) {
707 if (rl->rq_pool) {
708 wake_up_all(&rl->wait[BLK_RW_SYNC]);
709 wake_up_all(&rl->wait[BLK_RW_ASYNC]);
710 }
711 }
712 spin_unlock_irq(q->queue_lock);
713 }
714 329
715 /* Make blk_queue_enter() reexamine the DYING flag. */ 330 /* Make blk_queue_enter() reexamine the DYING flag. */
716 wake_up_all(&q->mq_freeze_wq); 331 wake_up_all(&q->mq_freeze_wq);
@@ -822,6 +437,7 @@ void blk_cleanup_queue(struct request_queue *q)
822 437
823 if (q->mq_ops) 438 if (q->mq_ops)
824 blk_mq_free_queue(q); 439 blk_mq_free_queue(q);
440
825 percpu_ref_exit(&q->q_usage_counter); 441 percpu_ref_exit(&q->q_usage_counter);
826 442
827 spin_lock_irq(lock); 443 spin_lock_irq(lock);
@@ -1013,8 +629,6 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
1013 629
1014 INIT_LIST_HEAD(&q->queue_head); 630 INIT_LIST_HEAD(&q->queue_head);
1015 q->last_merge = NULL; 631 q->last_merge = NULL;
1016 q->end_sector = 0;
1017 q->boundary_rq = NULL;
1018 632
1019 q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask); 633 q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
1020 if (q->id < 0) 634 if (q->id < 0)
@@ -1047,7 +661,6 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
1047#ifdef CONFIG_BLK_CGROUP 661#ifdef CONFIG_BLK_CGROUP
1048 INIT_LIST_HEAD(&q->blkg_list); 662 INIT_LIST_HEAD(&q->blkg_list);
1049#endif 663#endif
1050 INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
1051 664
1052 kobject_init(&q->kobj, &blk_queue_ktype); 665 kobject_init(&q->kobj, &blk_queue_ktype);
1053 666
@@ -1100,105 +713,6 @@ fail_q:
1100} 713}
1101EXPORT_SYMBOL(blk_alloc_queue_node); 714EXPORT_SYMBOL(blk_alloc_queue_node);
1102 715
1103/**
1104 * blk_init_queue - prepare a request queue for use with a block device
1105 * @rfn: The function to be called to process requests that have been
1106 * placed on the queue.
1107 * @lock: Request queue spin lock
1108 *
1109 * Description:
1110 * If a block device wishes to use the standard request handling procedures,
1111 * which sorts requests and coalesces adjacent requests, then it must
1112 * call blk_init_queue(). The function @rfn will be called when there
1113 * are requests on the queue that need to be processed. If the device
1114 * supports plugging, then @rfn may not be called immediately when requests
1115 * are available on the queue, but may be called at some time later instead.
1116 * Plugged queues are generally unplugged when a buffer belonging to one
1117 * of the requests on the queue is needed, or due to memory pressure.
1118 *
1119 * @rfn is not required, or even expected, to remove all requests off the
1120 * queue, but only as many as it can handle at a time. If it does leave
1121 * requests on the queue, it is responsible for arranging that the requests
1122 * get dealt with eventually.
1123 *
1124 * The queue spin lock must be held while manipulating the requests on the
1125 * request queue; this lock will be taken also from interrupt context, so irq
1126 * disabling is needed for it.
1127 *
1128 * Function returns a pointer to the initialized request queue, or %NULL if
1129 * it didn't succeed.
1130 *
1131 * Note:
1132 * blk_init_queue() must be paired with a blk_cleanup_queue() call
1133 * when the block device is deactivated (such as at module unload).
1134 **/
1135
1136struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
1137{
1138 return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
1139}
1140EXPORT_SYMBOL(blk_init_queue);
1141
1142struct request_queue *
1143blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
1144{
1145 struct request_queue *q;
1146
1147 q = blk_alloc_queue_node(GFP_KERNEL, node_id, lock);
1148 if (!q)
1149 return NULL;
1150
1151 q->request_fn = rfn;
1152 if (blk_init_allocated_queue(q) < 0) {
1153 blk_cleanup_queue(q);
1154 return NULL;
1155 }
1156
1157 return q;
1158}
1159EXPORT_SYMBOL(blk_init_queue_node);
1160
1161static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio);
1162
1163
1164int blk_init_allocated_queue(struct request_queue *q)
1165{
1166 WARN_ON_ONCE(q->mq_ops);
1167
1168 q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, q->cmd_size, GFP_KERNEL);
1169 if (!q->fq)
1170 return -ENOMEM;
1171
1172 if (q->init_rq_fn && q->init_rq_fn(q, q->fq->flush_rq, GFP_KERNEL))
1173 goto out_free_flush_queue;
1174
1175 if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
1176 goto out_exit_flush_rq;
1177
1178 INIT_WORK(&q->timeout_work, blk_timeout_work);
1179 q->queue_flags |= QUEUE_FLAG_DEFAULT;
1180
1181 /*
1182 * This also sets hw/phys segments, boundary and size
1183 */
1184 blk_queue_make_request(q, blk_queue_bio);
1185
1186 q->sg_reserved_size = INT_MAX;
1187
1188 if (elevator_init(q))
1189 goto out_exit_flush_rq;
1190 return 0;
1191
1192out_exit_flush_rq:
1193 if (q->exit_rq_fn)
1194 q->exit_rq_fn(q, q->fq->flush_rq);
1195out_free_flush_queue:
1196 blk_free_flush_queue(q->fq);
1197 q->fq = NULL;
1198 return -ENOMEM;
1199}
1200EXPORT_SYMBOL(blk_init_allocated_queue);
1201
1202bool blk_get_queue(struct request_queue *q) 716bool blk_get_queue(struct request_queue *q)
1203{ 717{
1204 if (likely(!blk_queue_dying(q))) { 718 if (likely(!blk_queue_dying(q))) {
@@ -1210,406 +724,6 @@ bool blk_get_queue(struct request_queue *q)
1210} 724}
1211EXPORT_SYMBOL(blk_get_queue); 725EXPORT_SYMBOL(blk_get_queue);
1212 726
1213static inline void blk_free_request(struct request_list *rl, struct request *rq)
1214{
1215 if (rq->rq_flags & RQF_ELVPRIV) {
1216 elv_put_request(rl->q, rq);
1217 if (rq->elv.icq)
1218 put_io_context(rq->elv.icq->ioc);
1219 }
1220
1221 mempool_free(rq, rl->rq_pool);
1222}
1223
1224/*
1225 * ioc_batching returns true if the ioc is a valid batching request and
1226 * should be given priority access to a request.
1227 */
1228static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
1229{
1230 if (!ioc)
1231 return 0;
1232
1233 /*
1234 * Make sure the process is able to allocate at least 1 request
1235 * even if the batch times out, otherwise we could theoretically
1236 * lose wakeups.
1237 */
1238 return ioc->nr_batch_requests == q->nr_batching ||
1239 (ioc->nr_batch_requests > 0
1240 && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
1241}
1242
1243/*
1244 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
1245 * will cause the process to be a "batcher" on all queues in the system. This
1246 * is the behaviour we want though - once it gets a wakeup it should be given
1247 * a nice run.
1248 */
1249static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
1250{
1251 if (!ioc || ioc_batching(q, ioc))
1252 return;
1253
1254 ioc->nr_batch_requests = q->nr_batching;
1255 ioc->last_waited = jiffies;
1256}
1257
1258static void __freed_request(struct request_list *rl, int sync)
1259{
1260 struct request_queue *q = rl->q;
1261
1262 if (rl->count[sync] < queue_congestion_off_threshold(q))
1263 blk_clear_congested(rl, sync);
1264
1265 if (rl->count[sync] + 1 <= q->nr_requests) {
1266 if (waitqueue_active(&rl->wait[sync]))
1267 wake_up(&rl->wait[sync]);
1268
1269 blk_clear_rl_full(rl, sync);
1270 }
1271}
1272
1273/*
1274 * A request has just been released. Account for it, update the full and
1275 * congestion status, wake up any waiters. Called under q->queue_lock.
1276 */
1277static void freed_request(struct request_list *rl, bool sync,
1278 req_flags_t rq_flags)
1279{
1280 struct request_queue *q = rl->q;
1281
1282 q->nr_rqs[sync]--;
1283 rl->count[sync]--;
1284 if (rq_flags & RQF_ELVPRIV)
1285 q->nr_rqs_elvpriv--;
1286
1287 __freed_request(rl, sync);
1288
1289 if (unlikely(rl->starved[sync ^ 1]))
1290 __freed_request(rl, sync ^ 1);
1291}
1292
1293int blk_update_nr_requests(struct request_queue *q, unsigned int nr)
1294{
1295 struct request_list *rl;
1296 int on_thresh, off_thresh;
1297
1298 WARN_ON_ONCE(q->mq_ops);
1299
1300 spin_lock_irq(q->queue_lock);
1301 q->nr_requests = nr;
1302 blk_queue_congestion_threshold(q);
1303 on_thresh = queue_congestion_on_threshold(q);
1304 off_thresh = queue_congestion_off_threshold(q);
1305
1306 blk_queue_for_each_rl(rl, q) {
1307 if (rl->count[BLK_RW_SYNC] >= on_thresh)
1308 blk_set_congested(rl, BLK_RW_SYNC);
1309 else if (rl->count[BLK_RW_SYNC] < off_thresh)
1310 blk_clear_congested(rl, BLK_RW_SYNC);
1311
1312 if (rl->count[BLK_RW_ASYNC] >= on_thresh)
1313 blk_set_congested(rl, BLK_RW_ASYNC);
1314 else if (rl->count[BLK_RW_ASYNC] < off_thresh)
1315 blk_clear_congested(rl, BLK_RW_ASYNC);
1316
1317 if (rl->count[BLK_RW_SYNC] >= q->nr_requests) {
1318 blk_set_rl_full(rl, BLK_RW_SYNC);
1319 } else {
1320 blk_clear_rl_full(rl, BLK_RW_SYNC);
1321 wake_up(&rl->wait[BLK_RW_SYNC]);
1322 }
1323
1324 if (rl->count[BLK_RW_ASYNC] >= q->nr_requests) {
1325 blk_set_rl_full(rl, BLK_RW_ASYNC);
1326 } else {
1327 blk_clear_rl_full(rl, BLK_RW_ASYNC);
1328 wake_up(&rl->wait[BLK_RW_ASYNC]);
1329 }
1330 }
1331
1332 spin_unlock_irq(q->queue_lock);
1333 return 0;
1334}
1335
1336/**
1337 * __get_request - get a free request
1338 * @rl: request list to allocate from
1339 * @op: operation and flags
1340 * @bio: bio to allocate request for (can be %NULL)
1341 * @flags: BLQ_MQ_REQ_* flags
1342 * @gfp_mask: allocator flags
1343 *
1344 * Get a free request from @q. This function may fail under memory
1345 * pressure or if @q is dead.
1346 *
1347 * Must be called with @q->queue_lock held and,
1348 * Returns ERR_PTR on failure, with @q->queue_lock held.
1349 * Returns request pointer on success, with @q->queue_lock *not held*.
1350 */
1351static struct request *__get_request(struct request_list *rl, unsigned int op,
1352 struct bio *bio, blk_mq_req_flags_t flags, gfp_t gfp_mask)
1353{
1354 struct request_queue *q = rl->q;
1355 struct request *rq;
1356 struct elevator_type *et = q->elevator->type;
1357 struct io_context *ioc = rq_ioc(bio);
1358 struct io_cq *icq = NULL;
1359 const bool is_sync = op_is_sync(op);
1360 int may_queue;
1361 req_flags_t rq_flags = RQF_ALLOCED;
1362
1363 lockdep_assert_held(q->queue_lock);
1364
1365 if (unlikely(blk_queue_dying(q)))
1366 return ERR_PTR(-ENODEV);
1367
1368 may_queue = elv_may_queue(q, op);
1369 if (may_queue == ELV_MQUEUE_NO)
1370 goto rq_starved;
1371
1372 if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
1373 if (rl->count[is_sync]+1 >= q->nr_requests) {
1374 /*
1375 * The queue will fill after this allocation, so set
1376 * it as full, and mark this process as "batching".
1377 * This process will be allowed to complete a batch of
1378 * requests, others will be blocked.
1379 */
1380 if (!blk_rl_full(rl, is_sync)) {
1381 ioc_set_batching(q, ioc);
1382 blk_set_rl_full(rl, is_sync);
1383 } else {
1384 if (may_queue != ELV_MQUEUE_MUST
1385 && !ioc_batching(q, ioc)) {
1386 /*
1387 * The queue is full and the allocating
1388 * process is not a "batcher", and not
1389 * exempted by the IO scheduler
1390 */
1391 return ERR_PTR(-ENOMEM);
1392 }
1393 }
1394 }
1395 blk_set_congested(rl, is_sync);
1396 }
1397
1398 /*
1399 * Only allow batching queuers to allocate up to 50% over the defined
1400 * limit of requests, otherwise we could have thousands of requests
1401 * allocated with any setting of ->nr_requests
1402 */
1403 if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
1404 return ERR_PTR(-ENOMEM);
1405
1406 q->nr_rqs[is_sync]++;
1407 rl->count[is_sync]++;
1408 rl->starved[is_sync] = 0;
1409
1410 /*
1411 * Decide whether the new request will be managed by elevator. If
1412 * so, mark @rq_flags and increment elvpriv. Non-zero elvpriv will
1413 * prevent the current elevator from being destroyed until the new
1414 * request is freed. This guarantees icq's won't be destroyed and
1415 * makes creating new ones safe.
1416 *
1417 * Flush requests do not use the elevator so skip initialization.
1418 * This allows a request to share the flush and elevator data.
1419 *
1420 * Also, lookup icq while holding queue_lock. If it doesn't exist,
1421 * it will be created after releasing queue_lock.
1422 */
1423 if (!op_is_flush(op) && !blk_queue_bypass(q)) {
1424 rq_flags |= RQF_ELVPRIV;
1425 q->nr_rqs_elvpriv++;
1426 if (et->icq_cache && ioc)
1427 icq = ioc_lookup_icq(ioc, q);
1428 }
1429
1430 if (blk_queue_io_stat(q))
1431 rq_flags |= RQF_IO_STAT;
1432 spin_unlock_irq(q->queue_lock);
1433
1434 /* allocate and init request */
1435 rq = mempool_alloc(rl->rq_pool, gfp_mask);
1436 if (!rq)
1437 goto fail_alloc;
1438
1439 blk_rq_init(q, rq);
1440 blk_rq_set_rl(rq, rl);
1441 rq->cmd_flags = op;
1442 rq->rq_flags = rq_flags;
1443 if (flags & BLK_MQ_REQ_PREEMPT)
1444 rq->rq_flags |= RQF_PREEMPT;
1445
1446 /* init elvpriv */
1447 if (rq_flags & RQF_ELVPRIV) {
1448 if (unlikely(et->icq_cache && !icq)) {
1449 if (ioc)
1450 icq = ioc_create_icq(ioc, q, gfp_mask);
1451 if (!icq)
1452 goto fail_elvpriv;
1453 }
1454
1455 rq->elv.icq = icq;
1456 if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
1457 goto fail_elvpriv;
1458
1459 /* @rq->elv.icq holds io_context until @rq is freed */
1460 if (icq)
1461 get_io_context(icq->ioc);
1462 }
1463out:
1464 /*
1465 * ioc may be NULL here, and ioc_batching will be false. That's
1466 * OK, if the queue is under the request limit then requests need
1467 * not count toward the nr_batch_requests limit. There will always
1468 * be some limit enforced by BLK_BATCH_TIME.
1469 */
1470 if (ioc_batching(q, ioc))
1471 ioc->nr_batch_requests--;
1472
1473 trace_block_getrq(q, bio, op);
1474 return rq;
1475
1476fail_elvpriv:
1477 /*
1478 * elvpriv init failed. ioc, icq and elvpriv aren't mempool backed
1479 * and may fail indefinitely under memory pressure and thus
1480 * shouldn't stall IO. Treat this request as !elvpriv. This will
1481 * disturb iosched and blkcg but weird is bettern than dead.
1482 */
1483 printk_ratelimited(KERN_WARNING "%s: dev %s: request aux data allocation failed, iosched may be disturbed\n",
1484 __func__, dev_name(q->backing_dev_info->dev));
1485
1486 rq->rq_flags &= ~RQF_ELVPRIV;
1487 rq->elv.icq = NULL;
1488
1489 spin_lock_irq(q->queue_lock);
1490 q->nr_rqs_elvpriv--;
1491 spin_unlock_irq(q->queue_lock);
1492 goto out;
1493
1494fail_alloc:
1495 /*
1496 * Allocation failed presumably due to memory. Undo anything we
1497 * might have messed up.
1498 *
1499 * Allocating task should really be put onto the front of the wait
1500 * queue, but this is pretty rare.
1501 */
1502 spin_lock_irq(q->queue_lock);
1503 freed_request(rl, is_sync, rq_flags);
1504
1505 /*
1506 * in the very unlikely event that allocation failed and no
1507 * requests for this direction was pending, mark us starved so that
1508 * freeing of a request in the other direction will notice
1509 * us. another possible fix would be to split the rq mempool into
1510 * READ and WRITE
1511 */
1512rq_starved:
1513 if (unlikely(rl->count[is_sync] == 0))
1514 rl->starved[is_sync] = 1;
1515 return ERR_PTR(-ENOMEM);
1516}
1517
1518/**
1519 * get_request - get a free request
1520 * @q: request_queue to allocate request from
1521 * @op: operation and flags
1522 * @bio: bio to allocate request for (can be %NULL)
1523 * @flags: BLK_MQ_REQ_* flags.
1524 * @gfp: allocator flags
1525 *
1526 * Get a free request from @q. If %BLK_MQ_REQ_NOWAIT is set in @flags,
1527 * this function keeps retrying under memory pressure and fails iff @q is dead.
1528 *
1529 * Must be called with @q->queue_lock held and,
1530 * Returns ERR_PTR on failure, with @q->queue_lock held.
1531 * Returns request pointer on success, with @q->queue_lock *not held*.
1532 */
1533static struct request *get_request(struct request_queue *q, unsigned int op,
1534 struct bio *bio, blk_mq_req_flags_t flags, gfp_t gfp)
1535{
1536 const bool is_sync = op_is_sync(op);
1537 DEFINE_WAIT(wait);
1538 struct request_list *rl;
1539 struct request *rq;
1540
1541 lockdep_assert_held(q->queue_lock);
1542 WARN_ON_ONCE(q->mq_ops);
1543
1544 rl = blk_get_rl(q, bio); /* transferred to @rq on success */
1545retry:
1546 rq = __get_request(rl, op, bio, flags, gfp);
1547 if (!IS_ERR(rq))
1548 return rq;
1549
1550 if (op & REQ_NOWAIT) {
1551 blk_put_rl(rl);
1552 return ERR_PTR(-EAGAIN);
1553 }
1554
1555 if ((flags & BLK_MQ_REQ_NOWAIT) || unlikely(blk_queue_dying(q))) {
1556 blk_put_rl(rl);
1557 return rq;
1558 }
1559
1560 /* wait on @rl and retry */
1561 prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
1562 TASK_UNINTERRUPTIBLE);
1563
1564 trace_block_sleeprq(q, bio, op);
1565
1566 spin_unlock_irq(q->queue_lock);
1567 io_schedule();
1568
1569 /*
1570 * After sleeping, we become a "batching" process and will be able
1571 * to allocate at least one request, and up to a big batch of them
1572 * for a small period time. See ioc_batching, ioc_set_batching
1573 */
1574 ioc_set_batching(q, current->io_context);
1575
1576 spin_lock_irq(q->queue_lock);
1577 finish_wait(&rl->wait[is_sync], &wait);
1578
1579 goto retry;
1580}
1581
1582/* flags: BLK_MQ_REQ_PREEMPT and/or BLK_MQ_REQ_NOWAIT. */
1583static struct request *blk_old_get_request(struct request_queue *q,
1584 unsigned int op, blk_mq_req_flags_t flags)
1585{
1586 struct request *rq;
1587 gfp_t gfp_mask = flags & BLK_MQ_REQ_NOWAIT ? GFP_ATOMIC : GFP_NOIO;
1588 int ret = 0;
1589
1590 WARN_ON_ONCE(q->mq_ops);
1591
1592 /* create ioc upfront */
1593 create_io_context(gfp_mask, q->node);
1594
1595 ret = blk_queue_enter(q, flags);
1596 if (ret)
1597 return ERR_PTR(ret);
1598 spin_lock_irq(q->queue_lock);
1599 rq = get_request(q, op, NULL, flags, gfp_mask);
1600 if (IS_ERR(rq)) {
1601 spin_unlock_irq(q->queue_lock);
1602 blk_queue_exit(q);
1603 return rq;
1604 }
1605
1606 /* q->queue_lock is unlocked at this point */
1607 rq->__data_len = 0;
1608 rq->__sector = (sector_t) -1;
1609 rq->bio = rq->biotail = NULL;
1610 return rq;
1611}
1612
1613/** 727/**
1614 * blk_get_request - allocate a request 728 * blk_get_request - allocate a request
1615 * @q: request queue to allocate a request for 729 * @q: request queue to allocate a request for
@@ -1624,53 +738,14 @@ struct request *blk_get_request(struct request_queue *q, unsigned int op,
1624 WARN_ON_ONCE(op & REQ_NOWAIT); 738 WARN_ON_ONCE(op & REQ_NOWAIT);
1625 WARN_ON_ONCE(flags & ~(BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_PREEMPT)); 739 WARN_ON_ONCE(flags & ~(BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_PREEMPT));
1626 740
1627 if (q->mq_ops) { 741 req = blk_mq_alloc_request(q, op, flags);
1628 req = blk_mq_alloc_request(q, op, flags); 742 if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn)
1629 if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn) 743 q->mq_ops->initialize_rq_fn(req);
1630 q->mq_ops->initialize_rq_fn(req);
1631 } else {
1632 req = blk_old_get_request(q, op, flags);
1633 if (!IS_ERR(req) && q->initialize_rq_fn)
1634 q->initialize_rq_fn(req);
1635 }
1636 744
1637 return req; 745 return req;
1638} 746}
1639EXPORT_SYMBOL(blk_get_request); 747EXPORT_SYMBOL(blk_get_request);
1640 748
1641/**
1642 * blk_requeue_request - put a request back on queue
1643 * @q: request queue where request should be inserted
1644 * @rq: request to be inserted
1645 *
1646 * Description:
1647 * Drivers often keep queueing requests until the hardware cannot accept
1648 * more, when that condition happens we need to put the request back
1649 * on the queue. Must be called with queue lock held.
1650 */
1651void blk_requeue_request(struct request_queue *q, struct request *rq)
1652{
1653 lockdep_assert_held(q->queue_lock);
1654 WARN_ON_ONCE(q->mq_ops);
1655
1656 blk_delete_timer(rq);
1657 blk_clear_rq_complete(rq);
1658 trace_block_rq_requeue(q, rq);
1659 rq_qos_requeue(q, rq);
1660
1661 BUG_ON(blk_queued_rq(rq));
1662
1663 elv_requeue_request(q, rq);
1664}
1665EXPORT_SYMBOL(blk_requeue_request);
1666
1667static void add_acct_request(struct request_queue *q, struct request *rq,
1668 int where)
1669{
1670 blk_account_io_start(rq, true);
1671 __elv_add_request(q, rq, where);
1672}
1673
1674static void part_round_stats_single(struct request_queue *q, int cpu, 749static void part_round_stats_single(struct request_queue *q, int cpu,
1675 struct hd_struct *part, unsigned long now, 750 struct hd_struct *part, unsigned long now,
1676 unsigned int inflight) 751 unsigned int inflight)
@@ -1730,61 +805,16 @@ EXPORT_SYMBOL_GPL(part_round_stats);
1730 805
1731void __blk_put_request(struct request_queue *q, struct request *req) 806void __blk_put_request(struct request_queue *q, struct request *req)
1732{ 807{
1733 req_flags_t rq_flags = req->rq_flags;
1734
1735 if (unlikely(!q)) 808 if (unlikely(!q))
1736 return; 809 return;
1737 810
1738 if (q->mq_ops) { 811 blk_mq_free_request(req);
1739 blk_mq_free_request(req);
1740 return;
1741 }
1742
1743 lockdep_assert_held(q->queue_lock);
1744
1745 blk_req_zone_write_unlock(req);
1746 blk_pm_put_request(req);
1747 blk_pm_mark_last_busy(req);
1748
1749 elv_completed_request(q, req);
1750
1751 /* this is a bio leak */
1752 WARN_ON(req->bio != NULL);
1753
1754 rq_qos_done(q, req);
1755
1756 /*
1757 * Request may not have originated from ll_rw_blk. if not,
1758 * it didn't come out of our reserved rq pools
1759 */
1760 if (rq_flags & RQF_ALLOCED) {
1761 struct request_list *rl = blk_rq_rl(req);
1762 bool sync = op_is_sync(req->cmd_flags);
1763
1764 BUG_ON(!list_empty(&req->queuelist));
1765 BUG_ON(ELV_ON_HASH(req));
1766
1767 blk_free_request(rl, req);
1768 freed_request(rl, sync, rq_flags);
1769 blk_put_rl(rl);
1770 blk_queue_exit(q);
1771 }
1772} 812}
1773EXPORT_SYMBOL_GPL(__blk_put_request); 813EXPORT_SYMBOL_GPL(__blk_put_request);
1774 814
1775void blk_put_request(struct request *req) 815void blk_put_request(struct request *req)
1776{ 816{
1777 struct request_queue *q = req->q; 817 blk_mq_free_request(req);
1778
1779 if (q->mq_ops)
1780 blk_mq_free_request(req);
1781 else {
1782 unsigned long flags;
1783
1784 spin_lock_irqsave(q->queue_lock, flags);
1785 __blk_put_request(q, req);
1786 spin_unlock_irqrestore(q->queue_lock, flags);
1787 }
1788} 818}
1789EXPORT_SYMBOL(blk_put_request); 819EXPORT_SYMBOL(blk_put_request);
1790 820
@@ -1893,10 +923,7 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1893 return false; 923 return false;
1894 *request_count = 0; 924 *request_count = 0;
1895 925
1896 if (q->mq_ops) 926 plug_list = &plug->mq_list;
1897 plug_list = &plug->mq_list;
1898 else
1899 plug_list = &plug->list;
1900 927
1901 list_for_each_entry_reverse(rq, plug_list, queuelist) { 928 list_for_each_entry_reverse(rq, plug_list, queuelist) {
1902 bool merged = false; 929 bool merged = false;
@@ -1947,11 +974,7 @@ unsigned int blk_plug_queued_count(struct request_queue *q)
1947 if (!plug) 974 if (!plug)
1948 goto out; 975 goto out;
1949 976
1950 if (q->mq_ops) 977 plug_list = &plug->mq_list;
1951 plug_list = &plug->mq_list;
1952 else
1953 plug_list = &plug->list;
1954
1955 list_for_each_entry(rq, plug_list, queuelist) { 978 list_for_each_entry(rq, plug_list, queuelist) {
1956 if (rq->q == q) 979 if (rq->q == q)
1957 ret++; 980 ret++;
@@ -1979,133 +1002,6 @@ void blk_init_request_from_bio(struct request *req, struct bio *bio)
1979} 1002}
1980EXPORT_SYMBOL_GPL(blk_init_request_from_bio); 1003EXPORT_SYMBOL_GPL(blk_init_request_from_bio);
1981 1004
1982static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
1983{
1984 struct blk_plug *plug;
1985 int where = ELEVATOR_INSERT_SORT;
1986 struct request *req, *free;
1987 unsigned int request_count = 0;
1988
1989 /*
1990 * low level driver can indicate that it wants pages above a
1991 * certain limit bounced to low memory (ie for highmem, or even
1992 * ISA dma in theory)
1993 */
1994 blk_queue_bounce(q, &bio);
1995
1996 blk_queue_split(q, &bio);
1997
1998 if (!bio_integrity_prep(bio))
1999 return BLK_QC_T_NONE;
2000
2001 if (op_is_flush(bio->bi_opf)) {
2002 spin_lock_irq(q->queue_lock);
2003 where = ELEVATOR_INSERT_FLUSH;
2004 goto get_rq;
2005 }
2006
2007 /*
2008 * Check if we can merge with the plugged list before grabbing
2009 * any locks.
2010 */
2011 if (!blk_queue_nomerges(q)) {
2012 if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
2013 return BLK_QC_T_NONE;
2014 } else
2015 request_count = blk_plug_queued_count(q);
2016
2017 spin_lock_irq(q->queue_lock);
2018
2019 switch (elv_merge(q, &req, bio)) {
2020 case ELEVATOR_BACK_MERGE:
2021 if (!bio_attempt_back_merge(q, req, bio))
2022 break;
2023 elv_bio_merged(q, req, bio);
2024 free = attempt_back_merge(q, req);
2025 if (free)
2026 __blk_put_request(q, free);
2027 else
2028 elv_merged_request(q, req, ELEVATOR_BACK_MERGE);
2029 goto out_unlock;
2030 case ELEVATOR_FRONT_MERGE:
2031 if (!bio_attempt_front_merge(q, req, bio))
2032 break;
2033 elv_bio_merged(q, req, bio);
2034 free = attempt_front_merge(q, req);
2035 if (free)
2036 __blk_put_request(q, free);
2037 else
2038 elv_merged_request(q, req, ELEVATOR_FRONT_MERGE);
2039 goto out_unlock;
2040 default:
2041 break;
2042 }
2043
2044get_rq:
2045 rq_qos_throttle(q, bio, q->queue_lock);
2046
2047 /*
2048 * Grab a free request. This is might sleep but can not fail.
2049 * Returns with the queue unlocked.
2050 */
2051 blk_queue_enter_live(q);
2052 req = get_request(q, bio->bi_opf, bio, 0, GFP_NOIO);
2053 if (IS_ERR(req)) {
2054 blk_queue_exit(q);
2055 rq_qos_cleanup(q, bio);
2056 if (PTR_ERR(req) == -ENOMEM)
2057 bio->bi_status = BLK_STS_RESOURCE;
2058 else
2059 bio->bi_status = BLK_STS_IOERR;
2060 bio_endio(bio);
2061 goto out_unlock;
2062 }
2063
2064 rq_qos_track(q, req, bio);
2065
2066 /*
2067 * After dropping the lock and possibly sleeping here, our request
2068 * may now be mergeable after it had proven unmergeable (above).
2069 * We don't worry about that case for efficiency. It won't happen
2070 * often, and the elevators are able to handle it.
2071 */
2072 blk_init_request_from_bio(req, bio);
2073
2074 if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags))
2075 req->cpu = raw_smp_processor_id();
2076
2077 plug = current->plug;
2078 if (plug) {
2079 /*
2080 * If this is the first request added after a plug, fire
2081 * of a plug trace.
2082 *
2083 * @request_count may become stale because of schedule
2084 * out, so check plug list again.
2085 */
2086 if (!request_count || list_empty(&plug->list))
2087 trace_block_plug(q);
2088 else {
2089 struct request *last = list_entry_rq(plug->list.prev);
2090 if (request_count >= BLK_MAX_REQUEST_COUNT ||
2091 blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE) {
2092 blk_flush_plug_list(plug, false);
2093 trace_block_plug(q);
2094 }
2095 }
2096 list_add_tail(&req->queuelist, &plug->list);
2097 blk_account_io_start(req, true);
2098 } else {
2099 spin_lock_irq(q->queue_lock);
2100 add_acct_request(q, req, where);
2101 __blk_run_queue(q);
2102out_unlock:
2103 spin_unlock_irq(q->queue_lock);
2104 }
2105
2106 return BLK_QC_T_NONE;
2107}
2108
2109static void handle_bad_sector(struct bio *bio, sector_t maxsector) 1005static void handle_bad_sector(struct bio *bio, sector_t maxsector)
2110{ 1006{
2111 char b[BDEVNAME_SIZE]; 1007 char b[BDEVNAME_SIZE];
@@ -2617,9 +1513,6 @@ static int blk_cloned_rq_check_limits(struct request_queue *q,
2617 */ 1513 */
2618blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq) 1514blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
2619{ 1515{
2620 unsigned long flags;
2621 int where = ELEVATOR_INSERT_BACK;
2622
2623 if (blk_cloned_rq_check_limits(q, rq)) 1516 if (blk_cloned_rq_check_limits(q, rq))
2624 return BLK_STS_IOERR; 1517 return BLK_STS_IOERR;
2625 1518
@@ -2627,38 +1520,15 @@ blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *
2627 should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq))) 1520 should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
2628 return BLK_STS_IOERR; 1521 return BLK_STS_IOERR;
2629 1522
2630 if (q->mq_ops) { 1523 if (blk_queue_io_stat(q))
2631 if (blk_queue_io_stat(q)) 1524 blk_account_io_start(rq, true);
2632 blk_account_io_start(rq, true);
2633 /*
2634 * Since we have a scheduler attached on the top device,
2635 * bypass a potential scheduler on the bottom device for
2636 * insert.
2637 */
2638 return blk_mq_request_issue_directly(rq);
2639 }
2640
2641 spin_lock_irqsave(q->queue_lock, flags);
2642 if (unlikely(blk_queue_dying(q))) {
2643 spin_unlock_irqrestore(q->queue_lock, flags);
2644 return BLK_STS_IOERR;
2645 }
2646 1525
2647 /* 1526 /*
2648 * Submitting request must be dequeued before calling this function 1527 * Since we have a scheduler attached on the top device,
2649 * because it will be linked to another request_queue 1528 * bypass a potential scheduler on the bottom device for
1529 * insert.
2650 */ 1530 */
2651 BUG_ON(blk_queued_rq(rq)); 1531 return blk_mq_request_issue_directly(rq);
2652
2653 if (op_is_flush(rq->cmd_flags))
2654 where = ELEVATOR_INSERT_FLUSH;
2655
2656 add_acct_request(q, rq, where);
2657 if (where == ELEVATOR_INSERT_FLUSH)
2658 __blk_run_queue(q);
2659 spin_unlock_irqrestore(q->queue_lock, flags);
2660
2661 return BLK_STS_OK;
2662} 1532}
2663EXPORT_SYMBOL_GPL(blk_insert_cloned_request); 1533EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
2664 1534
@@ -2778,225 +1648,6 @@ void blk_account_io_start(struct request *rq, bool new_io)
2778 part_stat_unlock(); 1648 part_stat_unlock();
2779} 1649}
2780 1650
2781static struct request *elv_next_request(struct request_queue *q)
2782{
2783 struct request *rq;
2784 struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
2785
2786 WARN_ON_ONCE(q->mq_ops);
2787
2788 while (1) {
2789 list_for_each_entry(rq, &q->queue_head, queuelist) {
2790#ifdef CONFIG_PM
2791 /*
2792 * If a request gets queued in state RPM_SUSPENDED
2793 * then that's a kernel bug.
2794 */
2795 WARN_ON_ONCE(q->rpm_status == RPM_SUSPENDED);
2796#endif
2797 return rq;
2798 }
2799
2800 /*
2801 * Flush request is running and flush request isn't queueable
2802 * in the drive, we can hold the queue till flush request is
2803 * finished. Even we don't do this, driver can't dispatch next
2804 * requests and will requeue them. And this can improve
2805 * throughput too. For example, we have request flush1, write1,
2806 * flush 2. flush1 is dispatched, then queue is hold, write1
2807 * isn't inserted to queue. After flush1 is finished, flush2
2808 * will be dispatched. Since disk cache is already clean,
2809 * flush2 will be finished very soon, so looks like flush2 is
2810 * folded to flush1.
2811 * Since the queue is hold, a flag is set to indicate the queue
2812 * should be restarted later. Please see flush_end_io() for
2813 * details.
2814 */
2815 if (fq->flush_pending_idx != fq->flush_running_idx &&
2816 !queue_flush_queueable(q)) {
2817 fq->flush_queue_delayed = 1;
2818 return NULL;
2819 }
2820 if (unlikely(blk_queue_bypass(q)) ||
2821 !q->elevator->type->ops.sq.elevator_dispatch_fn(q, 0))
2822 return NULL;
2823 }
2824}
2825
2826/**
2827 * blk_peek_request - peek at the top of a request queue
2828 * @q: request queue to peek at
2829 *
2830 * Description:
2831 * Return the request at the top of @q. The returned request
2832 * should be started using blk_start_request() before LLD starts
2833 * processing it.
2834 *
2835 * Return:
2836 * Pointer to the request at the top of @q if available. Null
2837 * otherwise.
2838 */
2839struct request *blk_peek_request(struct request_queue *q)
2840{
2841 struct request *rq;
2842 int ret;
2843
2844 lockdep_assert_held(q->queue_lock);
2845 WARN_ON_ONCE(q->mq_ops);
2846
2847 while ((rq = elv_next_request(q)) != NULL) {
2848 if (!(rq->rq_flags & RQF_STARTED)) {
2849 /*
2850 * This is the first time the device driver
2851 * sees this request (possibly after
2852 * requeueing). Notify IO scheduler.
2853 */
2854 if (rq->rq_flags & RQF_SORTED)
2855 elv_activate_rq(q, rq);
2856
2857 /*
2858 * just mark as started even if we don't start
2859 * it, a request that has been delayed should
2860 * not be passed by new incoming requests
2861 */
2862 rq->rq_flags |= RQF_STARTED;
2863 trace_block_rq_issue(q, rq);
2864 }
2865
2866 if (!q->boundary_rq || q->boundary_rq == rq) {
2867 q->end_sector = rq_end_sector(rq);
2868 q->boundary_rq = NULL;
2869 }
2870
2871 if (rq->rq_flags & RQF_DONTPREP)
2872 break;
2873
2874 if (q->dma_drain_size && blk_rq_bytes(rq)) {
2875 /*
2876 * make sure space for the drain appears we
2877 * know we can do this because max_hw_segments
2878 * has been adjusted to be one fewer than the
2879 * device can handle
2880 */
2881 rq->nr_phys_segments++;
2882 }
2883
2884 if (!q->prep_rq_fn)
2885 break;
2886
2887 ret = q->prep_rq_fn(q, rq);
2888 if (ret == BLKPREP_OK) {
2889 break;
2890 } else if (ret == BLKPREP_DEFER) {
2891 /*
2892 * the request may have been (partially) prepped.
2893 * we need to keep this request in the front to
2894 * avoid resource deadlock. RQF_STARTED will
2895 * prevent other fs requests from passing this one.
2896 */
2897 if (q->dma_drain_size && blk_rq_bytes(rq) &&
2898 !(rq->rq_flags & RQF_DONTPREP)) {
2899 /*
2900 * remove the space for the drain we added
2901 * so that we don't add it again
2902 */
2903 --rq->nr_phys_segments;
2904 }
2905
2906 rq = NULL;
2907 break;
2908 } else if (ret == BLKPREP_KILL || ret == BLKPREP_INVALID) {
2909 rq->rq_flags |= RQF_QUIET;
2910 /*
2911 * Mark this request as started so we don't trigger
2912 * any debug logic in the end I/O path.
2913 */
2914 blk_start_request(rq);
2915 __blk_end_request_all(rq, ret == BLKPREP_INVALID ?
2916 BLK_STS_TARGET : BLK_STS_IOERR);
2917 } else {
2918 printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
2919 break;
2920 }
2921 }
2922
2923 return rq;
2924}
2925EXPORT_SYMBOL(blk_peek_request);
2926
2927static void blk_dequeue_request(struct request *rq)
2928{
2929 struct request_queue *q = rq->q;
2930
2931 BUG_ON(list_empty(&rq->queuelist));
2932 BUG_ON(ELV_ON_HASH(rq));
2933
2934 list_del_init(&rq->queuelist);
2935
2936 /*
2937 * the time frame between a request being removed from the lists
2938 * and to it is freed is accounted as io that is in progress at
2939 * the driver side.
2940 */
2941 if (blk_account_rq(rq))
2942 q->in_flight[rq_is_sync(rq)]++;
2943}
2944
2945/**
2946 * blk_start_request - start request processing on the driver
2947 * @req: request to dequeue
2948 *
2949 * Description:
2950 * Dequeue @req and start timeout timer on it. This hands off the
2951 * request to the driver.
2952 */
2953void blk_start_request(struct request *req)
2954{
2955 lockdep_assert_held(req->q->queue_lock);
2956 WARN_ON_ONCE(req->q->mq_ops);
2957
2958 blk_dequeue_request(req);
2959
2960 if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {
2961 req->io_start_time_ns = ktime_get_ns();
2962#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
2963 req->throtl_size = blk_rq_sectors(req);
2964#endif
2965 req->rq_flags |= RQF_STATS;
2966 rq_qos_issue(req->q, req);
2967 }
2968
2969 BUG_ON(blk_rq_is_complete(req));
2970 blk_add_timer(req);
2971}
2972EXPORT_SYMBOL(blk_start_request);
2973
2974/**
2975 * blk_fetch_request - fetch a request from a request queue
2976 * @q: request queue to fetch a request from
2977 *
2978 * Description:
2979 * Return the request at the top of @q. The request is started on
2980 * return and LLD can start processing it immediately.
2981 *
2982 * Return:
2983 * Pointer to the request at the top of @q if available. Null
2984 * otherwise.
2985 */
2986struct request *blk_fetch_request(struct request_queue *q)
2987{
2988 struct request *rq;
2989
2990 lockdep_assert_held(q->queue_lock);
2991 WARN_ON_ONCE(q->mq_ops);
2992
2993 rq = blk_peek_request(q);
2994 if (rq)
2995 blk_start_request(rq);
2996 return rq;
2997}
2998EXPORT_SYMBOL(blk_fetch_request);
2999
3000/* 1651/*
3001 * Steal bios from a request and add them to a bio list. 1652 * Steal bios from a request and add them to a bio list.
3002 * The request must not have been partially completed before. 1653 * The request must not have been partially completed before.
@@ -3122,252 +1773,6 @@ bool blk_update_request(struct request *req, blk_status_t error,
3122} 1773}
3123EXPORT_SYMBOL_GPL(blk_update_request); 1774EXPORT_SYMBOL_GPL(blk_update_request);
3124 1775
3125static bool blk_update_bidi_request(struct request *rq, blk_status_t error,
3126 unsigned int nr_bytes,
3127 unsigned int bidi_bytes)
3128{
3129 if (blk_update_request(rq, error, nr_bytes))
3130 return true;
3131
3132 /* Bidi request must be completed as a whole */
3133 if (unlikely(blk_bidi_rq(rq)) &&
3134 blk_update_request(rq->next_rq, error, bidi_bytes))
3135 return true;
3136
3137 if (blk_queue_add_random(rq->q))
3138 add_disk_randomness(rq->rq_disk);
3139
3140 return false;
3141}
3142
3143/**
3144 * blk_unprep_request - unprepare a request
3145 * @req: the request
3146 *
3147 * This function makes a request ready for complete resubmission (or
3148 * completion). It happens only after all error handling is complete,
3149 * so represents the appropriate moment to deallocate any resources
3150 * that were allocated to the request in the prep_rq_fn. The queue
3151 * lock is held when calling this.
3152 */
3153void blk_unprep_request(struct request *req)
3154{
3155 struct request_queue *q = req->q;
3156
3157 req->rq_flags &= ~RQF_DONTPREP;
3158 if (q->unprep_rq_fn)
3159 q->unprep_rq_fn(q, req);
3160}
3161EXPORT_SYMBOL_GPL(blk_unprep_request);
3162
3163void blk_finish_request(struct request *req, blk_status_t error)
3164{
3165 struct request_queue *q = req->q;
3166 u64 now = ktime_get_ns();
3167
3168 lockdep_assert_held(req->q->queue_lock);
3169 WARN_ON_ONCE(q->mq_ops);
3170
3171 if (req->rq_flags & RQF_STATS)
3172 blk_stat_add(req, now);
3173
3174 BUG_ON(blk_queued_rq(req));
3175
3176 if (unlikely(laptop_mode) && !blk_rq_is_passthrough(req))
3177 laptop_io_completion(req->q->backing_dev_info);
3178
3179 blk_delete_timer(req);
3180
3181 if (req->rq_flags & RQF_DONTPREP)
3182 blk_unprep_request(req);
3183
3184 blk_account_io_done(req, now);
3185
3186 if (req->end_io) {
3187 rq_qos_done(q, req);
3188 req->end_io(req, error);
3189 } else {
3190 if (blk_bidi_rq(req))
3191 __blk_put_request(req->next_rq->q, req->next_rq);
3192
3193 __blk_put_request(q, req);
3194 }
3195}
3196EXPORT_SYMBOL(blk_finish_request);
3197
3198/**
3199 * blk_end_bidi_request - Complete a bidi request
3200 * @rq: the request to complete
3201 * @error: block status code
3202 * @nr_bytes: number of bytes to complete @rq
3203 * @bidi_bytes: number of bytes to complete @rq->next_rq
3204 *
3205 * Description:
3206 * Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
3207 * Drivers that supports bidi can safely call this member for any
3208 * type of request, bidi or uni. In the later case @bidi_bytes is
3209 * just ignored.
3210 *
3211 * Return:
3212 * %false - we are done with this request
3213 * %true - still buffers pending for this request
3214 **/
3215static bool blk_end_bidi_request(struct request *rq, blk_status_t error,
3216 unsigned int nr_bytes, unsigned int bidi_bytes)
3217{
3218 struct request_queue *q = rq->q;
3219 unsigned long flags;
3220
3221 WARN_ON_ONCE(q->mq_ops);
3222
3223 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
3224 return true;
3225
3226 spin_lock_irqsave(q->queue_lock, flags);
3227 blk_finish_request(rq, error);
3228 spin_unlock_irqrestore(q->queue_lock, flags);
3229
3230 return false;
3231}
3232
3233/**
3234 * __blk_end_bidi_request - Complete a bidi request with queue lock held
3235 * @rq: the request to complete
3236 * @error: block status code
3237 * @nr_bytes: number of bytes to complete @rq
3238 * @bidi_bytes: number of bytes to complete @rq->next_rq
3239 *
3240 * Description:
3241 * Identical to blk_end_bidi_request() except that queue lock is
3242 * assumed to be locked on entry and remains so on return.
3243 *
3244 * Return:
3245 * %false - we are done with this request
3246 * %true - still buffers pending for this request
3247 **/
3248static bool __blk_end_bidi_request(struct request *rq, blk_status_t error,
3249 unsigned int nr_bytes, unsigned int bidi_bytes)
3250{
3251 lockdep_assert_held(rq->q->queue_lock);
3252 WARN_ON_ONCE(rq->q->mq_ops);
3253
3254 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
3255 return true;
3256
3257 blk_finish_request(rq, error);
3258
3259 return false;
3260}
3261
3262/**
3263 * blk_end_request - Helper function for drivers to complete the request.
3264 * @rq: the request being processed
3265 * @error: block status code
3266 * @nr_bytes: number of bytes to complete
3267 *
3268 * Description:
3269 * Ends I/O on a number of bytes attached to @rq.
3270 * If @rq has leftover, sets it up for the next range of segments.
3271 *
3272 * Return:
3273 * %false - we are done with this request
3274 * %true - still buffers pending for this request
3275 **/
3276bool blk_end_request(struct request *rq, blk_status_t error,
3277 unsigned int nr_bytes)
3278{
3279 WARN_ON_ONCE(rq->q->mq_ops);
3280 return blk_end_bidi_request(rq, error, nr_bytes, 0);
3281}
3282EXPORT_SYMBOL(blk_end_request);
3283
3284/**
3285 * blk_end_request_all - Helper function for drives to finish the request.
3286 * @rq: the request to finish
3287 * @error: block status code
3288 *
3289 * Description:
3290 * Completely finish @rq.
3291 */
3292void blk_end_request_all(struct request *rq, blk_status_t error)
3293{
3294 bool pending;
3295 unsigned int bidi_bytes = 0;
3296
3297 if (unlikely(blk_bidi_rq(rq)))
3298 bidi_bytes = blk_rq_bytes(rq->next_rq);
3299
3300 pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
3301 BUG_ON(pending);
3302}
3303EXPORT_SYMBOL(blk_end_request_all);
3304
3305/**
3306 * __blk_end_request - Helper function for drivers to complete the request.
3307 * @rq: the request being processed
3308 * @error: block status code
3309 * @nr_bytes: number of bytes to complete
3310 *
3311 * Description:
3312 * Must be called with queue lock held unlike blk_end_request().
3313 *
3314 * Return:
3315 * %false - we are done with this request
3316 * %true - still buffers pending for this request
3317 **/
3318bool __blk_end_request(struct request *rq, blk_status_t error,
3319 unsigned int nr_bytes)
3320{
3321 lockdep_assert_held(rq->q->queue_lock);
3322 WARN_ON_ONCE(rq->q->mq_ops);
3323
3324 return __blk_end_bidi_request(rq, error, nr_bytes, 0);
3325}
3326EXPORT_SYMBOL(__blk_end_request);
3327
3328/**
3329 * __blk_end_request_all - Helper function for drives to finish the request.
3330 * @rq: the request to finish
3331 * @error: block status code
3332 *
3333 * Description:
3334 * Completely finish @rq. Must be called with queue lock held.
3335 */
3336void __blk_end_request_all(struct request *rq, blk_status_t error)
3337{
3338 bool pending;
3339 unsigned int bidi_bytes = 0;
3340
3341 lockdep_assert_held(rq->q->queue_lock);
3342 WARN_ON_ONCE(rq->q->mq_ops);
3343
3344 if (unlikely(blk_bidi_rq(rq)))
3345 bidi_bytes = blk_rq_bytes(rq->next_rq);
3346
3347 pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
3348 BUG_ON(pending);
3349}
3350EXPORT_SYMBOL(__blk_end_request_all);
3351
3352/**
3353 * __blk_end_request_cur - Helper function to finish the current request chunk.
3354 * @rq: the request to finish the current chunk for
3355 * @error: block status code
3356 *
3357 * Description:
3358 * Complete the current consecutively mapped chunk from @rq. Must
3359 * be called with queue lock held.
3360 *
3361 * Return:
3362 * %false - we are done with this request
3363 * %true - still buffers pending for this request
3364 */
3365bool __blk_end_request_cur(struct request *rq, blk_status_t error)
3366{
3367 return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
3368}
3369EXPORT_SYMBOL(__blk_end_request_cur);
3370
3371void blk_rq_bio_prep(struct request_queue *q, struct request *rq, 1776void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
3372 struct bio *bio) 1777 struct bio *bio)
3373{ 1778{
@@ -3567,7 +1972,6 @@ void blk_start_plug(struct blk_plug *plug)
3567 if (tsk->plug) 1972 if (tsk->plug)
3568 return; 1973 return;
3569 1974
3570 INIT_LIST_HEAD(&plug->list);
3571 INIT_LIST_HEAD(&plug->mq_list); 1975 INIT_LIST_HEAD(&plug->mq_list);
3572 INIT_LIST_HEAD(&plug->cb_list); 1976 INIT_LIST_HEAD(&plug->cb_list);
3573 /* 1977 /*
@@ -3578,36 +1982,6 @@ void blk_start_plug(struct blk_plug *plug)
3578} 1982}
3579EXPORT_SYMBOL(blk_start_plug); 1983EXPORT_SYMBOL(blk_start_plug);
3580 1984
3581static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
3582{
3583 struct request *rqa = container_of(a, struct request, queuelist);
3584 struct request *rqb = container_of(b, struct request, queuelist);
3585
3586 return !(rqa->q < rqb->q ||
3587 (rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb)));
3588}
3589
3590/*
3591 * If 'from_schedule' is true, then postpone the dispatch of requests
3592 * until a safe kblockd context. We due this to avoid accidental big
3593 * additional stack usage in driver dispatch, in places where the originally
3594 * plugger did not intend it.
3595 */
3596static void queue_unplugged(struct request_queue *q, unsigned int depth,
3597 bool from_schedule)
3598 __releases(q->queue_lock)
3599{
3600 lockdep_assert_held(q->queue_lock);
3601
3602 trace_block_unplug(q, depth, !from_schedule);
3603
3604 if (from_schedule)
3605 blk_run_queue_async(q);
3606 else
3607 __blk_run_queue(q);
3608 spin_unlock_irq(q->queue_lock);
3609}
3610
3611static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule) 1985static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
3612{ 1986{
3613 LIST_HEAD(callbacks); 1987 LIST_HEAD(callbacks);
@@ -3652,65 +2026,10 @@ EXPORT_SYMBOL(blk_check_plugged);
3652 2026
3653void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) 2027void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
3654{ 2028{
3655 struct request_queue *q;
3656 struct request *rq;
3657 LIST_HEAD(list);
3658 unsigned int depth;
3659
3660 flush_plug_callbacks(plug, from_schedule); 2029 flush_plug_callbacks(plug, from_schedule);
3661 2030
3662 if (!list_empty(&plug->mq_list)) 2031 if (!list_empty(&plug->mq_list))
3663 blk_mq_flush_plug_list(plug, from_schedule); 2032 blk_mq_flush_plug_list(plug, from_schedule);
3664
3665 if (list_empty(&plug->list))
3666 return;
3667
3668 list_splice_init(&plug->list, &list);
3669
3670 list_sort(NULL, &list, plug_rq_cmp);
3671
3672 q = NULL;
3673 depth = 0;
3674
3675 while (!list_empty(&list)) {
3676 rq = list_entry_rq(list.next);
3677 list_del_init(&rq->queuelist);
3678 BUG_ON(!rq->q);
3679 if (rq->q != q) {
3680 /*
3681 * This drops the queue lock
3682 */
3683 if (q)
3684 queue_unplugged(q, depth, from_schedule);
3685 q = rq->q;
3686 depth = 0;
3687 spin_lock_irq(q->queue_lock);
3688 }
3689
3690 /*
3691 * Short-circuit if @q is dead
3692 */
3693 if (unlikely(blk_queue_dying(q))) {
3694 __blk_end_request_all(rq, BLK_STS_IOERR);
3695 continue;
3696 }
3697
3698 /*
3699 * rq is already accounted, so use raw insert
3700 */
3701 if (op_is_flush(rq->cmd_flags))
3702 __elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH);
3703 else
3704 __elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE);
3705
3706 depth++;
3707 }
3708
3709 /*
3710 * This drops the queue lock
3711 */
3712 if (q)
3713 queue_unplugged(q, depth, from_schedule);
3714} 2033}
3715 2034
3716void blk_finish_plug(struct blk_plug *plug) 2035void blk_finish_plug(struct blk_plug *plug)
diff --git a/block/blk-exec.c b/block/blk-exec.c
index f7b292f12449..a34b7d918742 100644
--- a/block/blk-exec.c
+++ b/block/blk-exec.c
@@ -48,8 +48,6 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
48 struct request *rq, int at_head, 48 struct request *rq, int at_head,
49 rq_end_io_fn *done) 49 rq_end_io_fn *done)
50{ 50{
51 int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;
52
53 WARN_ON(irqs_disabled()); 51 WARN_ON(irqs_disabled());
54 WARN_ON(!blk_rq_is_passthrough(rq)); 52 WARN_ON(!blk_rq_is_passthrough(rq));
55 53
@@ -60,23 +58,7 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
60 * don't check dying flag for MQ because the request won't 58 * don't check dying flag for MQ because the request won't
61 * be reused after dying flag is set 59 * be reused after dying flag is set
62 */ 60 */
63 if (q->mq_ops) { 61 blk_mq_sched_insert_request(rq, at_head, true, false);
64 blk_mq_sched_insert_request(rq, at_head, true, false);
65 return;
66 }
67
68 spin_lock_irq(q->queue_lock);
69
70 if (unlikely(blk_queue_dying(q))) {
71 rq->rq_flags |= RQF_QUIET;
72 __blk_end_request_all(rq, BLK_STS_IOERR);
73 spin_unlock_irq(q->queue_lock);
74 return;
75 }
76
77 __elv_add_request(q, rq, where);
78 __blk_run_queue(q);
79 spin_unlock_irq(q->queue_lock);
80} 62}
81EXPORT_SYMBOL_GPL(blk_execute_rq_nowait); 63EXPORT_SYMBOL_GPL(blk_execute_rq_nowait);
82 64
diff --git a/block/blk-ioc.c b/block/blk-ioc.c
index 01580f88fcb3..391128456aec 100644
--- a/block/blk-ioc.c
+++ b/block/blk-ioc.c
@@ -48,10 +48,8 @@ static void ioc_exit_icq(struct io_cq *icq)
48 if (icq->flags & ICQ_EXITED) 48 if (icq->flags & ICQ_EXITED)
49 return; 49 return;
50 50
51 if (et->uses_mq && et->ops.mq.exit_icq) 51 if (et->ops.mq.exit_icq)
52 et->ops.mq.exit_icq(icq); 52 et->ops.mq.exit_icq(icq);
53 else if (!et->uses_mq && et->ops.sq.elevator_exit_icq_fn)
54 et->ops.sq.elevator_exit_icq_fn(icq);
55 53
56 icq->flags |= ICQ_EXITED; 54 icq->flags |= ICQ_EXITED;
57} 55}
@@ -187,25 +185,13 @@ void put_io_context_active(struct io_context *ioc)
187 * reverse double locking. Read comment in ioc_release_fn() for 185 * reverse double locking. Read comment in ioc_release_fn() for
188 * explanation on the nested locking annotation. 186 * explanation on the nested locking annotation.
189 */ 187 */
190retry:
191 spin_lock_irqsave_nested(&ioc->lock, flags, 1); 188 spin_lock_irqsave_nested(&ioc->lock, flags, 1);
192 hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) { 189 hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) {
193 if (icq->flags & ICQ_EXITED) 190 if (icq->flags & ICQ_EXITED)
194 continue; 191 continue;
195 192
196 et = icq->q->elevator->type; 193 et = icq->q->elevator->type;
197 if (et->uses_mq) { 194 ioc_exit_icq(icq);
198 ioc_exit_icq(icq);
199 } else {
200 if (spin_trylock(icq->q->queue_lock)) {
201 ioc_exit_icq(icq);
202 spin_unlock(icq->q->queue_lock);
203 } else {
204 spin_unlock_irqrestore(&ioc->lock, flags);
205 cpu_relax();
206 goto retry;
207 }
208 }
209 } 195 }
210 spin_unlock_irqrestore(&ioc->lock, flags); 196 spin_unlock_irqrestore(&ioc->lock, flags);
211 197
@@ -232,7 +218,7 @@ static void __ioc_clear_queue(struct list_head *icq_list)
232 218
233 while (!list_empty(icq_list)) { 219 while (!list_empty(icq_list)) {
234 struct io_cq *icq = list_entry(icq_list->next, 220 struct io_cq *icq = list_entry(icq_list->next,
235 struct io_cq, q_node); 221 struct io_cq, q_node);
236 struct io_context *ioc = icq->ioc; 222 struct io_context *ioc = icq->ioc;
237 223
238 spin_lock_irqsave(&ioc->lock, flags); 224 spin_lock_irqsave(&ioc->lock, flags);
@@ -253,14 +239,9 @@ void ioc_clear_queue(struct request_queue *q)
253 239
254 spin_lock_irq(q->queue_lock); 240 spin_lock_irq(q->queue_lock);
255 list_splice_init(&q->icq_list, &icq_list); 241 list_splice_init(&q->icq_list, &icq_list);
242 spin_unlock_irq(q->queue_lock);
256 243
257 if (q->mq_ops) { 244 __ioc_clear_queue(&icq_list);
258 spin_unlock_irq(q->queue_lock);
259 __ioc_clear_queue(&icq_list);
260 } else {
261 __ioc_clear_queue(&icq_list);
262 spin_unlock_irq(q->queue_lock);
263 }
264} 245}
265 246
266int create_task_io_context(struct task_struct *task, gfp_t gfp_flags, int node) 247int create_task_io_context(struct task_struct *task, gfp_t gfp_flags, int node)
@@ -415,10 +396,8 @@ struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q,
415 if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) { 396 if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {
416 hlist_add_head(&icq->ioc_node, &ioc->icq_list); 397 hlist_add_head(&icq->ioc_node, &ioc->icq_list);
417 list_add(&icq->q_node, &q->icq_list); 398 list_add(&icq->q_node, &q->icq_list);
418 if (et->uses_mq && et->ops.mq.init_icq) 399 if (et->ops.mq.init_icq)
419 et->ops.mq.init_icq(icq); 400 et->ops.mq.init_icq(icq);
420 else if (!et->uses_mq && et->ops.sq.elevator_init_icq_fn)
421 et->ops.sq.elevator_init_icq_fn(icq);
422 } else { 401 } else {
423 kmem_cache_free(et->icq_cache, icq); 402 kmem_cache_free(et->icq_cache, icq);
424 icq = ioc_lookup_icq(ioc, q); 403 icq = ioc_lookup_icq(ioc, q);
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 6b5ad275ed56..c068c30b0c35 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -862,13 +862,8 @@ struct request *attempt_front_merge(struct request_queue *q, struct request *rq)
862int blk_attempt_req_merge(struct request_queue *q, struct request *rq, 862int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
863 struct request *next) 863 struct request *next)
864{ 864{
865 struct elevator_queue *e = q->elevator;
866 struct request *free; 865 struct request *free;
867 866
868 if (!e->uses_mq && e->type->ops.sq.elevator_allow_rq_merge_fn)
869 if (!e->type->ops.sq.elevator_allow_rq_merge_fn(q, rq, next))
870 return 0;
871
872 free = attempt_merge(q, rq, next); 867 free = attempt_merge(q, rq, next);
873 if (free) { 868 if (free) {
874 __blk_put_request(q, free); 869 __blk_put_request(q, free);
diff --git a/block/blk-settings.c b/block/blk-settings.c
index ac8b8ba4b126..39c3c301a687 100644
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -20,40 +20,6 @@ EXPORT_SYMBOL(blk_max_low_pfn);
20 20
21unsigned long blk_max_pfn; 21unsigned long blk_max_pfn;
22 22
23/**
24 * blk_queue_prep_rq - set a prepare_request function for queue
25 * @q: queue
26 * @pfn: prepare_request function
27 *
28 * It's possible for a queue to register a prepare_request callback which
29 * is invoked before the request is handed to the request_fn. The goal of
30 * the function is to prepare a request for I/O, it can be used to build a
31 * cdb from the request data for instance.
32 *
33 */
34void blk_queue_prep_rq(struct request_queue *q, prep_rq_fn *pfn)
35{
36 q->prep_rq_fn = pfn;
37}
38EXPORT_SYMBOL(blk_queue_prep_rq);
39
40/**
41 * blk_queue_unprep_rq - set an unprepare_request function for queue
42 * @q: queue
43 * @ufn: unprepare_request function
44 *
45 * It's possible for a queue to register an unprepare_request callback
46 * which is invoked before the request is finally completed. The goal
47 * of the function is to deallocate any data that was allocated in the
48 * prepare_request callback.
49 *
50 */
51void blk_queue_unprep_rq(struct request_queue *q, unprep_rq_fn *ufn)
52{
53 q->unprep_rq_fn = ufn;
54}
55EXPORT_SYMBOL(blk_queue_unprep_rq);
56
57void blk_queue_softirq_done(struct request_queue *q, softirq_done_fn *fn) 23void blk_queue_softirq_done(struct request_queue *q, softirq_done_fn *fn)
58{ 24{
59 q->softirq_done_fn = fn; 25 q->softirq_done_fn = fn;
@@ -163,8 +129,6 @@ void blk_queue_make_request(struct request_queue *q, make_request_fn *mfn)
163 129
164 q->make_request_fn = mfn; 130 q->make_request_fn = mfn;
165 blk_queue_dma_alignment(q, 511); 131 blk_queue_dma_alignment(q, 511);
166 blk_queue_congestion_threshold(q);
167 q->nr_batching = BLK_BATCH_REQ;
168 132
169 blk_set_default_limits(&q->limits); 133 blk_set_default_limits(&q->limits);
170} 134}
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 1b82ccfde3fe..d4b1b84ba8ca 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -68,7 +68,7 @@ queue_requests_store(struct request_queue *q, const char *page, size_t count)
68 unsigned long nr; 68 unsigned long nr;
69 int ret, err; 69 int ret, err;
70 70
71 if (!q->request_fn && !q->mq_ops) 71 if (!q->mq_ops)
72 return -EINVAL; 72 return -EINVAL;
73 73
74 ret = queue_var_store(&nr, page, count); 74 ret = queue_var_store(&nr, page, count);
@@ -78,11 +78,7 @@ queue_requests_store(struct request_queue *q, const char *page, size_t count)
78 if (nr < BLKDEV_MIN_RQ) 78 if (nr < BLKDEV_MIN_RQ)
79 nr = BLKDEV_MIN_RQ; 79 nr = BLKDEV_MIN_RQ;
80 80
81 if (q->request_fn) 81 err = blk_mq_update_nr_requests(q, nr);
82 err = blk_update_nr_requests(q, nr);
83 else
84 err = blk_mq_update_nr_requests(q, nr);
85
86 if (err) 82 if (err)
87 return err; 83 return err;
88 84
@@ -463,20 +459,14 @@ static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,
463 * ends up either enabling or disabling wbt completely. We can't 459 * ends up either enabling or disabling wbt completely. We can't
464 * have IO inflight if that happens. 460 * have IO inflight if that happens.
465 */ 461 */
466 if (q->mq_ops) { 462 blk_mq_freeze_queue(q);
467 blk_mq_freeze_queue(q); 463 blk_mq_quiesce_queue(q);
468 blk_mq_quiesce_queue(q);
469 } else
470 blk_queue_bypass_start(q);
471 464
472 wbt_set_min_lat(q, val); 465 wbt_set_min_lat(q, val);
473 wbt_update_limits(q); 466 wbt_update_limits(q);
474 467
475 if (q->mq_ops) { 468 blk_mq_unquiesce_queue(q);
476 blk_mq_unquiesce_queue(q); 469 blk_mq_unfreeze_queue(q);
477 blk_mq_unfreeze_queue(q);
478 } else
479 blk_queue_bypass_end(q);
480 470
481 return count; 471 return count;
482} 472}
@@ -847,17 +837,10 @@ static void __blk_release_queue(struct work_struct *work)
847 837
848 blk_free_queue_stats(q->stats); 838 blk_free_queue_stats(q->stats);
849 839
850 blk_exit_rl(q, &q->root_rl);
851
852 blk_queue_free_zone_bitmaps(q); 840 blk_queue_free_zone_bitmaps(q);
853 841
854 if (!q->mq_ops) { 842 if (q->mq_ops)
855 if (q->exit_rq_fn)
856 q->exit_rq_fn(q, q->fq->flush_rq);
857 blk_free_flush_queue(q->fq);
858 } else {
859 blk_mq_release(q); 843 blk_mq_release(q);
860 }
861 844
862 blk_trace_shutdown(q); 845 blk_trace_shutdown(q);
863 846
@@ -920,7 +903,6 @@ int blk_register_queue(struct gendisk *disk)
920 if (!blk_queue_init_done(q)) { 903 if (!blk_queue_init_done(q)) {
921 queue_flag_set_unlocked(QUEUE_FLAG_INIT_DONE, q); 904 queue_flag_set_unlocked(QUEUE_FLAG_INIT_DONE, q);
922 percpu_ref_switch_to_percpu(&q->q_usage_counter); 905 percpu_ref_switch_to_percpu(&q->q_usage_counter);
923 blk_queue_bypass_end(q);
924 } 906 }
925 907
926 ret = blk_trace_init_sysfs(dev); 908 ret = blk_trace_init_sysfs(dev);
@@ -947,7 +929,7 @@ int blk_register_queue(struct gendisk *disk)
947 929
948 blk_throtl_register_queue(q); 930 blk_throtl_register_queue(q);
949 931
950 if (q->request_fn || (q->mq_ops && q->elevator)) { 932 if ((q->mq_ops && q->elevator)) {
951 ret = elv_register_queue(q); 933 ret = elv_register_queue(q);
952 if (ret) { 934 if (ret) {
953 mutex_unlock(&q->sysfs_lock); 935 mutex_unlock(&q->sysfs_lock);
@@ -1005,7 +987,7 @@ void blk_unregister_queue(struct gendisk *disk)
1005 blk_trace_remove_sysfs(disk_to_dev(disk)); 987 blk_trace_remove_sysfs(disk_to_dev(disk));
1006 988
1007 mutex_lock(&q->sysfs_lock); 989 mutex_lock(&q->sysfs_lock);
1008 if (q->request_fn || (q->mq_ops && q->elevator)) 990 if (q->mq_ops && q->elevator)
1009 elv_unregister_queue(q); 991 elv_unregister_queue(q);
1010 mutex_unlock(&q->sysfs_lock); 992 mutex_unlock(&q->sysfs_lock);
1011 993
diff --git a/block/blk.h b/block/blk.h
index 57a302bf5a70..e2604ae7ddfa 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -7,12 +7,6 @@
7#include <xen/xen.h> 7#include <xen/xen.h>
8#include "blk-mq.h" 8#include "blk-mq.h"
9 9
10/* Amount of time in which a process may batch requests */
11#define BLK_BATCH_TIME (HZ/50UL)
12
13/* Number of requests a "batching" process may submit */
14#define BLK_BATCH_REQ 32
15
16/* Max future timer expiry for timeouts */ 10/* Max future timer expiry for timeouts */
17#define BLK_MAX_TIMEOUT (5 * HZ) 11#define BLK_MAX_TIMEOUT (5 * HZ)
18 12
@@ -132,9 +126,6 @@ void blk_exit_rl(struct request_queue *q, struct request_list *rl);
132void blk_exit_queue(struct request_queue *q); 126void blk_exit_queue(struct request_queue *q);
133void blk_rq_bio_prep(struct request_queue *q, struct request *rq, 127void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
134 struct bio *bio); 128 struct bio *bio);
135void blk_queue_bypass_start(struct request_queue *q);
136void blk_queue_bypass_end(struct request_queue *q);
137void __blk_queue_free_tags(struct request_queue *q);
138void blk_freeze_queue(struct request_queue *q); 129void blk_freeze_queue(struct request_queue *q);
139 130
140static inline void blk_queue_enter_live(struct request_queue *q) 131static inline void blk_queue_enter_live(struct request_queue *q)
@@ -281,23 +272,6 @@ static inline bool blk_rq_is_complete(struct request *rq)
281 272
282void blk_insert_flush(struct request *rq); 273void blk_insert_flush(struct request *rq);
283 274
284static inline void elv_activate_rq(struct request_queue *q, struct request *rq)
285{
286 struct elevator_queue *e = q->elevator;
287
288 if (e->type->ops.sq.elevator_activate_req_fn)
289 e->type->ops.sq.elevator_activate_req_fn(q, rq);
290}
291
292static inline void elv_deactivate_rq(struct request_queue *q, struct request *rq)
293{
294 struct elevator_queue *e = q->elevator;
295
296 if (e->type->ops.sq.elevator_deactivate_req_fn)
297 e->type->ops.sq.elevator_deactivate_req_fn(q, rq);
298}
299
300int elevator_init(struct request_queue *);
301int elevator_init_mq(struct request_queue *q); 275int elevator_init_mq(struct request_queue *q);
302int elevator_switch_mq(struct request_queue *q, 276int elevator_switch_mq(struct request_queue *q,
303 struct elevator_type *new_e); 277 struct elevator_type *new_e);
@@ -332,31 +306,8 @@ void blk_rq_set_mixed_merge(struct request *rq);
332bool blk_rq_merge_ok(struct request *rq, struct bio *bio); 306bool blk_rq_merge_ok(struct request *rq, struct bio *bio);
333enum elv_merge blk_try_merge(struct request *rq, struct bio *bio); 307enum elv_merge blk_try_merge(struct request *rq, struct bio *bio);
334 308
335void blk_queue_congestion_threshold(struct request_queue *q);
336
337int blk_dev_init(void); 309int blk_dev_init(void);
338 310
339
340/*
341 * Return the threshold (number of used requests) at which the queue is
342 * considered to be congested. It include a little hysteresis to keep the
343 * context switch rate down.
344 */
345static inline int queue_congestion_on_threshold(struct request_queue *q)
346{
347 return q->nr_congestion_on;
348}
349
350/*
351 * The threshold at which a queue is considered to be uncongested
352 */
353static inline int queue_congestion_off_threshold(struct request_queue *q)
354{
355 return q->nr_congestion_off;
356}
357
358extern int blk_update_nr_requests(struct request_queue *, unsigned int);
359
360/* 311/*
361 * Contribute to IO statistics IFF: 312 * Contribute to IO statistics IFF:
362 * 313 *
@@ -478,8 +429,6 @@ static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
478} 429}
479#endif /* CONFIG_BOUNCE */ 430#endif /* CONFIG_BOUNCE */
480 431
481extern void blk_drain_queue(struct request_queue *q);
482
483#ifdef CONFIG_BLK_CGROUP_IOLATENCY 432#ifdef CONFIG_BLK_CGROUP_IOLATENCY
484extern int blk_iolatency_init(struct request_queue *q); 433extern int blk_iolatency_init(struct request_queue *q);
485#else 434#else
diff --git a/block/elevator.c b/block/elevator.c
index 54e1adac26c5..334097c54b08 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -61,10 +61,8 @@ static int elv_iosched_allow_bio_merge(struct request *rq, struct bio *bio)
61 struct request_queue *q = rq->q; 61 struct request_queue *q = rq->q;
62 struct elevator_queue *e = q->elevator; 62 struct elevator_queue *e = q->elevator;
63 63
64 if (e->uses_mq && e->type->ops.mq.allow_merge) 64 if (e->type->ops.mq.allow_merge)
65 return e->type->ops.mq.allow_merge(q, rq, bio); 65 return e->type->ops.mq.allow_merge(q, rq, bio);
66 else if (!e->uses_mq && e->type->ops.sq.elevator_allow_bio_merge_fn)
67 return e->type->ops.sq.elevator_allow_bio_merge_fn(q, rq, bio);
68 66
69 return 1; 67 return 1;
70} 68}
@@ -95,14 +93,14 @@ static bool elevator_match(const struct elevator_type *e, const char *name)
95} 93}
96 94
97/* 95/*
98 * Return scheduler with name 'name' and with matching 'mq capability 96 * Return scheduler with name 'name'
99 */ 97 */
100static struct elevator_type *elevator_find(const char *name, bool mq) 98static struct elevator_type *elevator_find(const char *name)
101{ 99{
102 struct elevator_type *e; 100 struct elevator_type *e;
103 101
104 list_for_each_entry(e, &elv_list, list) { 102 list_for_each_entry(e, &elv_list, list) {
105 if (elevator_match(e, name) && (mq == e->uses_mq)) 103 if (elevator_match(e, name))
106 return e; 104 return e;
107 } 105 }
108 106
@@ -121,12 +119,12 @@ static struct elevator_type *elevator_get(struct request_queue *q,
121 119
122 spin_lock(&elv_list_lock); 120 spin_lock(&elv_list_lock);
123 121
124 e = elevator_find(name, q->mq_ops != NULL); 122 e = elevator_find(name);
125 if (!e && try_loading) { 123 if (!e && try_loading) {
126 spin_unlock(&elv_list_lock); 124 spin_unlock(&elv_list_lock);
127 request_module("%s-iosched", name); 125 request_module("%s-iosched", name);
128 spin_lock(&elv_list_lock); 126 spin_lock(&elv_list_lock);
129 e = elevator_find(name, q->mq_ops != NULL); 127 e = elevator_find(name);
130 } 128 }
131 129
132 if (e && !try_module_get(e->elevator_owner)) 130 if (e && !try_module_get(e->elevator_owner))
@@ -150,26 +148,6 @@ static int __init elevator_setup(char *str)
150 148
151__setup("elevator=", elevator_setup); 149__setup("elevator=", elevator_setup);
152 150
153/* called during boot to load the elevator chosen by the elevator param */
154void __init load_default_elevator_module(void)
155{
156 struct elevator_type *e;
157
158 if (!chosen_elevator[0])
159 return;
160
161 /*
162 * Boot parameter is deprecated, we haven't supported that for MQ.
163 * Only look for non-mq schedulers from here.
164 */
165 spin_lock(&elv_list_lock);
166 e = elevator_find(chosen_elevator, false);
167 spin_unlock(&elv_list_lock);
168
169 if (!e)
170 request_module("%s-iosched", chosen_elevator);
171}
172
173static struct kobj_type elv_ktype; 151static struct kobj_type elv_ktype;
174 152
175struct elevator_queue *elevator_alloc(struct request_queue *q, 153struct elevator_queue *elevator_alloc(struct request_queue *q,
@@ -185,7 +163,6 @@ struct elevator_queue *elevator_alloc(struct request_queue *q,
185 kobject_init(&eq->kobj, &elv_ktype); 163 kobject_init(&eq->kobj, &elv_ktype);
186 mutex_init(&eq->sysfs_lock); 164 mutex_init(&eq->sysfs_lock);
187 hash_init(eq->hash); 165 hash_init(eq->hash);
188 eq->uses_mq = e->uses_mq;
189 166
190 return eq; 167 return eq;
191} 168}
@@ -200,52 +177,11 @@ static void elevator_release(struct kobject *kobj)
200 kfree(e); 177 kfree(e);
201} 178}
202 179
203/*
204 * Use the default elevator specified by config boot param for non-mq devices,
205 * or by config option. Don't try to load modules as we could be running off
206 * async and request_module() isn't allowed from async.
207 */
208int elevator_init(struct request_queue *q)
209{
210 struct elevator_type *e = NULL;
211 int err = 0;
212
213 /*
214 * q->sysfs_lock must be held to provide mutual exclusion between
215 * elevator_switch() and here.
216 */
217 mutex_lock(&q->sysfs_lock);
218 if (unlikely(q->elevator))
219 goto out_unlock;
220
221 if (*chosen_elevator) {
222 e = elevator_get(q, chosen_elevator, false);
223 if (!e)
224 printk(KERN_ERR "I/O scheduler %s not found\n",
225 chosen_elevator);
226 }
227
228 if (!e) {
229 printk(KERN_ERR
230 "Default I/O scheduler not found. Using noop.\n");
231 e = elevator_get(q, "noop", false);
232 }
233
234 err = e->ops.sq.elevator_init_fn(q, e);
235 if (err)
236 elevator_put(e);
237out_unlock:
238 mutex_unlock(&q->sysfs_lock);
239 return err;
240}
241
242void elevator_exit(struct request_queue *q, struct elevator_queue *e) 180void elevator_exit(struct request_queue *q, struct elevator_queue *e)
243{ 181{
244 mutex_lock(&e->sysfs_lock); 182 mutex_lock(&e->sysfs_lock);
245 if (e->uses_mq && e->type->ops.mq.exit_sched) 183 if (e->type->ops.mq.exit_sched)
246 blk_mq_exit_sched(q, e); 184 blk_mq_exit_sched(q, e);
247 else if (!e->uses_mq && e->type->ops.sq.elevator_exit_fn)
248 e->type->ops.sq.elevator_exit_fn(e);
249 mutex_unlock(&e->sysfs_lock); 185 mutex_unlock(&e->sysfs_lock);
250 186
251 kobject_put(&e->kobj); 187 kobject_put(&e->kobj);
@@ -393,10 +329,8 @@ enum elv_merge elv_merge(struct request_queue *q, struct request **req,
393 return ELEVATOR_BACK_MERGE; 329 return ELEVATOR_BACK_MERGE;
394 } 330 }
395 331
396 if (e->uses_mq && e->type->ops.mq.request_merge) 332 if (e->type->ops.mq.request_merge)
397 return e->type->ops.mq.request_merge(q, req, bio); 333 return e->type->ops.mq.request_merge(q, req, bio);
398 else if (!e->uses_mq && e->type->ops.sq.elevator_merge_fn)
399 return e->type->ops.sq.elevator_merge_fn(q, req, bio);
400 334
401 return ELEVATOR_NO_MERGE; 335 return ELEVATOR_NO_MERGE;
402} 336}
@@ -447,10 +381,8 @@ void elv_merged_request(struct request_queue *q, struct request *rq,
447{ 381{
448 struct elevator_queue *e = q->elevator; 382 struct elevator_queue *e = q->elevator;
449 383
450 if (e->uses_mq && e->type->ops.mq.request_merged) 384 if (e->type->ops.mq.request_merged)
451 e->type->ops.mq.request_merged(q, rq, type); 385 e->type->ops.mq.request_merged(q, rq, type);
452 else if (!e->uses_mq && e->type->ops.sq.elevator_merged_fn)
453 e->type->ops.sq.elevator_merged_fn(q, rq, type);
454 386
455 if (type == ELEVATOR_BACK_MERGE) 387 if (type == ELEVATOR_BACK_MERGE)
456 elv_rqhash_reposition(q, rq); 388 elv_rqhash_reposition(q, rq);
@@ -464,13 +396,8 @@ void elv_merge_requests(struct request_queue *q, struct request *rq,
464 struct elevator_queue *e = q->elevator; 396 struct elevator_queue *e = q->elevator;
465 bool next_sorted = false; 397 bool next_sorted = false;
466 398
467 if (e->uses_mq && e->type->ops.mq.requests_merged) 399 if (e->type->ops.mq.requests_merged)
468 e->type->ops.mq.requests_merged(q, rq, next); 400 e->type->ops.mq.requests_merged(q, rq, next);
469 else if (e->type->ops.sq.elevator_merge_req_fn) {
470 next_sorted = (__force bool)(next->rq_flags & RQF_SORTED);
471 if (next_sorted)
472 e->type->ops.sq.elevator_merge_req_fn(q, rq, next);
473 }
474 401
475 elv_rqhash_reposition(q, rq); 402 elv_rqhash_reposition(q, rq);
476 403
@@ -482,156 +409,12 @@ void elv_merge_requests(struct request_queue *q, struct request *rq,
482 q->last_merge = rq; 409 q->last_merge = rq;
483} 410}
484 411
485void elv_bio_merged(struct request_queue *q, struct request *rq,
486 struct bio *bio)
487{
488 struct elevator_queue *e = q->elevator;
489
490 if (WARN_ON_ONCE(e->uses_mq))
491 return;
492
493 if (e->type->ops.sq.elevator_bio_merged_fn)
494 e->type->ops.sq.elevator_bio_merged_fn(q, rq, bio);
495}
496
497void elv_requeue_request(struct request_queue *q, struct request *rq)
498{
499 /*
500 * it already went through dequeue, we need to decrement the
501 * in_flight count again
502 */
503 if (blk_account_rq(rq)) {
504 q->in_flight[rq_is_sync(rq)]--;
505 if (rq->rq_flags & RQF_SORTED)
506 elv_deactivate_rq(q, rq);
507 }
508
509 rq->rq_flags &= ~RQF_STARTED;
510
511 blk_pm_requeue_request(rq);
512
513 __elv_add_request(q, rq, ELEVATOR_INSERT_REQUEUE);
514}
515
516void elv_drain_elevator(struct request_queue *q)
517{
518 struct elevator_queue *e = q->elevator;
519 static int printed;
520
521 if (WARN_ON_ONCE(e->uses_mq))
522 return;
523
524 lockdep_assert_held(q->queue_lock);
525
526 while (e->type->ops.sq.elevator_dispatch_fn(q, 1))
527 ;
528 if (q->nr_sorted && !blk_queue_is_zoned(q) && printed++ < 10 ) {
529 printk(KERN_ERR "%s: forced dispatching is broken "
530 "(nr_sorted=%u), please report this\n",
531 q->elevator->type->elevator_name, q->nr_sorted);
532 }
533}
534
535void __elv_add_request(struct request_queue *q, struct request *rq, int where)
536{
537 trace_block_rq_insert(q, rq);
538
539 blk_pm_add_request(q, rq);
540
541 rq->q = q;
542
543 if (rq->rq_flags & RQF_SOFTBARRIER) {
544 /* barriers are scheduling boundary, update end_sector */
545 if (!blk_rq_is_passthrough(rq)) {
546 q->end_sector = rq_end_sector(rq);
547 q->boundary_rq = rq;
548 }
549 } else if (!(rq->rq_flags & RQF_ELVPRIV) &&
550 (where == ELEVATOR_INSERT_SORT ||
551 where == ELEVATOR_INSERT_SORT_MERGE))
552 where = ELEVATOR_INSERT_BACK;
553
554 switch (where) {
555 case ELEVATOR_INSERT_REQUEUE:
556 case ELEVATOR_INSERT_FRONT:
557 rq->rq_flags |= RQF_SOFTBARRIER;
558 list_add(&rq->queuelist, &q->queue_head);
559 break;
560
561 case ELEVATOR_INSERT_BACK:
562 rq->rq_flags |= RQF_SOFTBARRIER;
563 elv_drain_elevator(q);
564 list_add_tail(&rq->queuelist, &q->queue_head);
565 /*
566 * We kick the queue here for the following reasons.
567 * - The elevator might have returned NULL previously
568 * to delay requests and returned them now. As the
569 * queue wasn't empty before this request, ll_rw_blk
570 * won't run the queue on return, resulting in hang.
571 * - Usually, back inserted requests won't be merged
572 * with anything. There's no point in delaying queue
573 * processing.
574 */
575 __blk_run_queue(q);
576 break;
577
578 case ELEVATOR_INSERT_SORT_MERGE:
579 /*
580 * If we succeed in merging this request with one in the
581 * queue already, we are done - rq has now been freed,
582 * so no need to do anything further.
583 */
584 if (elv_attempt_insert_merge(q, rq))
585 break;
586 /* fall through */
587 case ELEVATOR_INSERT_SORT:
588 BUG_ON(blk_rq_is_passthrough(rq));
589 rq->rq_flags |= RQF_SORTED;
590 q->nr_sorted++;
591 if (rq_mergeable(rq)) {
592 elv_rqhash_add(q, rq);
593 if (!q->last_merge)
594 q->last_merge = rq;
595 }
596
597 /*
598 * Some ioscheds (cfq) run q->request_fn directly, so
599 * rq cannot be accessed after calling
600 * elevator_add_req_fn.
601 */
602 q->elevator->type->ops.sq.elevator_add_req_fn(q, rq);
603 break;
604
605 case ELEVATOR_INSERT_FLUSH:
606 rq->rq_flags |= RQF_SOFTBARRIER;
607 blk_insert_flush(rq);
608 break;
609 default:
610 printk(KERN_ERR "%s: bad insertion point %d\n",
611 __func__, where);
612 BUG();
613 }
614}
615EXPORT_SYMBOL(__elv_add_request);
616
617void elv_add_request(struct request_queue *q, struct request *rq, int where)
618{
619 unsigned long flags;
620
621 spin_lock_irqsave(q->queue_lock, flags);
622 __elv_add_request(q, rq, where);
623 spin_unlock_irqrestore(q->queue_lock, flags);
624}
625EXPORT_SYMBOL(elv_add_request);
626
627struct request *elv_latter_request(struct request_queue *q, struct request *rq) 412struct request *elv_latter_request(struct request_queue *q, struct request *rq)
628{ 413{
629 struct elevator_queue *e = q->elevator; 414 struct elevator_queue *e = q->elevator;
630 415
631 if (e->uses_mq && e->type->ops.mq.next_request) 416 if (e->type->ops.mq.next_request)
632 return e->type->ops.mq.next_request(q, rq); 417 return e->type->ops.mq.next_request(q, rq);
633 else if (!e->uses_mq && e->type->ops.sq.elevator_latter_req_fn)
634 return e->type->ops.sq.elevator_latter_req_fn(q, rq);
635 418
636 return NULL; 419 return NULL;
637} 420}
@@ -640,66 +423,10 @@ struct request *elv_former_request(struct request_queue *q, struct request *rq)
640{ 423{
641 struct elevator_queue *e = q->elevator; 424 struct elevator_queue *e = q->elevator;
642 425
643 if (e->uses_mq && e->type->ops.mq.former_request) 426 if (e->type->ops.mq.former_request)
644 return e->type->ops.mq.former_request(q, rq); 427 return e->type->ops.mq.former_request(q, rq);
645 if (!e->uses_mq && e->type->ops.sq.elevator_former_req_fn)
646 return e->type->ops.sq.elevator_former_req_fn(q, rq);
647 return NULL;
648}
649
650int elv_set_request(struct request_queue *q, struct request *rq,
651 struct bio *bio, gfp_t gfp_mask)
652{
653 struct elevator_queue *e = q->elevator;
654
655 if (WARN_ON_ONCE(e->uses_mq))
656 return 0;
657 428
658 if (e->type->ops.sq.elevator_set_req_fn) 429 return NULL;
659 return e->type->ops.sq.elevator_set_req_fn(q, rq, bio, gfp_mask);
660 return 0;
661}
662
663void elv_put_request(struct request_queue *q, struct request *rq)
664{
665 struct elevator_queue *e = q->elevator;
666
667 if (WARN_ON_ONCE(e->uses_mq))
668 return;
669
670 if (e->type->ops.sq.elevator_put_req_fn)
671 e->type->ops.sq.elevator_put_req_fn(rq);
672}
673
674int elv_may_queue(struct request_queue *q, unsigned int op)
675{
676 struct elevator_queue *e = q->elevator;
677
678 if (WARN_ON_ONCE(e->uses_mq))
679 return 0;
680
681 if (e->type->ops.sq.elevator_may_queue_fn)
682 return e->type->ops.sq.elevator_may_queue_fn(q, op);
683
684 return ELV_MQUEUE_MAY;
685}
686
687void elv_completed_request(struct request_queue *q, struct request *rq)
688{
689 struct elevator_queue *e = q->elevator;
690
691 if (WARN_ON_ONCE(e->uses_mq))
692 return;
693
694 /*
695 * request is released from the driver, io must be done
696 */
697 if (blk_account_rq(rq)) {
698 q->in_flight[rq_is_sync(rq)]--;
699 if ((rq->rq_flags & RQF_SORTED) &&
700 e->type->ops.sq.elevator_completed_req_fn)
701 e->type->ops.sq.elevator_completed_req_fn(q, rq);
702 }
703} 430}
704 431
705#define to_elv(atr) container_of((atr), struct elv_fs_entry, attr) 432#define to_elv(atr) container_of((atr), struct elv_fs_entry, attr)
@@ -768,8 +495,6 @@ int elv_register_queue(struct request_queue *q)
768 } 495 }
769 kobject_uevent(&e->kobj, KOBJ_ADD); 496 kobject_uevent(&e->kobj, KOBJ_ADD);
770 e->registered = 1; 497 e->registered = 1;
771 if (!e->uses_mq && e->type->ops.sq.elevator_registered_fn)
772 e->type->ops.sq.elevator_registered_fn(q);
773 } 498 }
774 return error; 499 return error;
775} 500}
@@ -809,7 +534,7 @@ int elv_register(struct elevator_type *e)
809 534
810 /* register, don't allow duplicate names */ 535 /* register, don't allow duplicate names */
811 spin_lock(&elv_list_lock); 536 spin_lock(&elv_list_lock);
812 if (elevator_find(e->elevator_name, e->uses_mq)) { 537 if (elevator_find(e->elevator_name)) {
813 spin_unlock(&elv_list_lock); 538 spin_unlock(&elv_list_lock);
814 kmem_cache_destroy(e->icq_cache); 539 kmem_cache_destroy(e->icq_cache);
815 return -EBUSY; 540 return -EBUSY;
@@ -919,71 +644,17 @@ out_unlock:
919 */ 644 */
920static int elevator_switch(struct request_queue *q, struct elevator_type *new_e) 645static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
921{ 646{
922 struct elevator_queue *old = q->elevator;
923 bool old_registered = false;
924 int err; 647 int err;
925 648
926 lockdep_assert_held(&q->sysfs_lock); 649 lockdep_assert_held(&q->sysfs_lock);
927 650
928 if (q->mq_ops) { 651 blk_mq_freeze_queue(q);
929 blk_mq_freeze_queue(q); 652 blk_mq_quiesce_queue(q);
930 blk_mq_quiesce_queue(q);
931
932 err = elevator_switch_mq(q, new_e);
933
934 blk_mq_unquiesce_queue(q);
935 blk_mq_unfreeze_queue(q);
936
937 return err;
938 }
939
940 /*
941 * Turn on BYPASS and drain all requests w/ elevator private data.
942 * Block layer doesn't call into a quiesced elevator - all requests
943 * are directly put on the dispatch list without elevator data
944 * using INSERT_BACK. All requests have SOFTBARRIER set and no
945 * merge happens either.
946 */
947 if (old) {
948 old_registered = old->registered;
949
950 blk_queue_bypass_start(q);
951
952 /* unregister and clear all auxiliary data of the old elevator */
953 if (old_registered)
954 elv_unregister_queue(q);
955
956 ioc_clear_queue(q);
957 }
958
959 /* allocate, init and register new elevator */
960 err = new_e->ops.sq.elevator_init_fn(q, new_e);
961 if (err)
962 goto fail_init;
963
964 err = elv_register_queue(q);
965 if (err)
966 goto fail_register;
967
968 /* done, kill the old one and finish */
969 if (old) {
970 elevator_exit(q, old);
971 blk_queue_bypass_end(q);
972 }
973
974 blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);
975 653
976 return 0; 654 err = elevator_switch_mq(q, new_e);
977 655
978fail_register: 656 blk_mq_unquiesce_queue(q);
979 elevator_exit(q, q->elevator); 657 blk_mq_unfreeze_queue(q);
980fail_init:
981 /* switch failed, restore and re-register old elevator */
982 if (old) {
983 q->elevator = old;
984 elv_register_queue(q);
985 blk_queue_bypass_end(q);
986 }
987 658
988 return err; 659 return err;
989} 660}
@@ -1032,7 +703,7 @@ ssize_t elv_iosched_store(struct request_queue *q, const char *name,
1032{ 703{
1033 int ret; 704 int ret;
1034 705
1035 if (!(q->mq_ops || q->request_fn) || !elv_support_iosched(q)) 706 if (!q->mq_ops || !elv_support_iosched(q))
1036 return count; 707 return count;
1037 708
1038 ret = __elevator_change(q, name); 709 ret = __elevator_change(q, name);
@@ -1047,7 +718,6 @@ ssize_t elv_iosched_show(struct request_queue *q, char *name)
1047 struct elevator_queue *e = q->elevator; 718 struct elevator_queue *e = q->elevator;
1048 struct elevator_type *elv = NULL; 719 struct elevator_type *elv = NULL;
1049 struct elevator_type *__e; 720 struct elevator_type *__e;
1050 bool uses_mq = q->mq_ops != NULL;
1051 int len = 0; 721 int len = 0;
1052 722
1053 if (!queue_is_rq_based(q)) 723 if (!queue_is_rq_based(q))
@@ -1060,14 +730,11 @@ ssize_t elv_iosched_show(struct request_queue *q, char *name)
1060 730
1061 spin_lock(&elv_list_lock); 731 spin_lock(&elv_list_lock);
1062 list_for_each_entry(__e, &elv_list, list) { 732 list_for_each_entry(__e, &elv_list, list) {
1063 if (elv && elevator_match(elv, __e->elevator_name) && 733 if (elv && elevator_match(elv, __e->elevator_name)) {
1064 (__e->uses_mq == uses_mq)) {
1065 len += sprintf(name+len, "[%s] ", elv->elevator_name); 734 len += sprintf(name+len, "[%s] ", elv->elevator_name);
1066 continue; 735 continue;
1067 } 736 }
1068 if (__e->uses_mq && q->mq_ops && elv_support_iosched(q)) 737 if (elv_support_iosched(q))
1069 len += sprintf(name+len, "%s ", __e->elevator_name);
1070 else if (!__e->uses_mq && !q->mq_ops)
1071 len += sprintf(name+len, "%s ", __e->elevator_name); 738 len += sprintf(name+len, "%s ", __e->elevator_name);
1072 } 739 }
1073 spin_unlock(&elv_list_lock); 740 spin_unlock(&elv_list_lock);
diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c
index eccac01a10b6..728757a34fa0 100644
--- a/block/kyber-iosched.c
+++ b/block/kyber-iosched.c
@@ -1032,7 +1032,6 @@ static struct elevator_type kyber_sched = {
1032 .dispatch_request = kyber_dispatch_request, 1032 .dispatch_request = kyber_dispatch_request,
1033 .has_work = kyber_has_work, 1033 .has_work = kyber_has_work,
1034 }, 1034 },
1035 .uses_mq = true,
1036#ifdef CONFIG_BLK_DEBUG_FS 1035#ifdef CONFIG_BLK_DEBUG_FS
1037 .queue_debugfs_attrs = kyber_queue_debugfs_attrs, 1036 .queue_debugfs_attrs = kyber_queue_debugfs_attrs,
1038 .hctx_debugfs_attrs = kyber_hctx_debugfs_attrs, 1037 .hctx_debugfs_attrs = kyber_hctx_debugfs_attrs,
diff --git a/block/mq-deadline.c b/block/mq-deadline.c
index 099a9e05854c..513edefd10fd 100644
--- a/block/mq-deadline.c
+++ b/block/mq-deadline.c
@@ -777,7 +777,6 @@ static struct elevator_type mq_deadline = {
777 .exit_sched = dd_exit_queue, 777 .exit_sched = dd_exit_queue,
778 }, 778 },
779 779
780 .uses_mq = true,
781#ifdef CONFIG_BLK_DEBUG_FS 780#ifdef CONFIG_BLK_DEBUG_FS
782 .queue_debugfs_attrs = deadline_queue_debugfs_attrs, 781 .queue_debugfs_attrs = deadline_queue_debugfs_attrs,
783#endif 782#endif
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 8afe3331777e..a9f6db8abcda 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -58,9 +58,6 @@ struct blk_stat_callback;
58 58
59typedef void (rq_end_io_fn)(struct request *, blk_status_t); 59typedef void (rq_end_io_fn)(struct request *, blk_status_t);
60 60
61#define BLK_RL_SYNCFULL (1U << 0)
62#define BLK_RL_ASYNCFULL (1U << 1)
63
64struct request_list { 61struct request_list {
65 struct request_queue *q; /* the queue this rl belongs to */ 62 struct request_queue *q; /* the queue this rl belongs to */
66#ifdef CONFIG_BLK_CGROUP 63#ifdef CONFIG_BLK_CGROUP
@@ -309,11 +306,8 @@ static inline unsigned short req_get_ioprio(struct request *req)
309 306
310struct blk_queue_ctx; 307struct blk_queue_ctx;
311 308
312typedef void (request_fn_proc) (struct request_queue *q);
313typedef blk_qc_t (make_request_fn) (struct request_queue *q, struct bio *bio); 309typedef blk_qc_t (make_request_fn) (struct request_queue *q, struct bio *bio);
314typedef bool (poll_q_fn) (struct request_queue *q, blk_qc_t); 310typedef bool (poll_q_fn) (struct request_queue *q, blk_qc_t);
315typedef int (prep_rq_fn) (struct request_queue *, struct request *);
316typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
317 311
318struct bio_vec; 312struct bio_vec;
319typedef void (softirq_done_fn)(struct request *); 313typedef void (softirq_done_fn)(struct request *);
@@ -432,8 +426,6 @@ struct request_queue {
432 struct list_head queue_head; 426 struct list_head queue_head;
433 struct request *last_merge; 427 struct request *last_merge;
434 struct elevator_queue *elevator; 428 struct elevator_queue *elevator;
435 int nr_rqs[2]; /* # allocated [a]sync rqs */
436 int nr_rqs_elvpriv; /* # allocated rqs w/ elvpriv */
437 429
438 struct blk_queue_stats *stats; 430 struct blk_queue_stats *stats;
439 struct rq_qos *rq_qos; 431 struct rq_qos *rq_qos;
@@ -446,11 +438,8 @@ struct request_queue {
446 */ 438 */
447 struct request_list root_rl; 439 struct request_list root_rl;
448 440
449 request_fn_proc *request_fn;
450 make_request_fn *make_request_fn; 441 make_request_fn *make_request_fn;
451 poll_q_fn *poll_fn; 442 poll_q_fn *poll_fn;
452 prep_rq_fn *prep_rq_fn;
453 unprep_rq_fn *unprep_rq_fn;
454 softirq_done_fn *softirq_done_fn; 443 softirq_done_fn *softirq_done_fn;
455 rq_timed_out_fn *rq_timed_out_fn; 444 rq_timed_out_fn *rq_timed_out_fn;
456 dma_drain_needed_fn *dma_drain_needed; 445 dma_drain_needed_fn *dma_drain_needed;
@@ -458,8 +447,6 @@ struct request_queue {
458 init_rq_fn *init_rq_fn; 447 init_rq_fn *init_rq_fn;
459 /* Called just before a request is freed */ 448 /* Called just before a request is freed */
460 exit_rq_fn *exit_rq_fn; 449 exit_rq_fn *exit_rq_fn;
461 /* Called from inside blk_get_request() */
462 void (*initialize_rq_fn)(struct request *rq);
463 450
464 const struct blk_mq_ops *mq_ops; 451 const struct blk_mq_ops *mq_ops;
465 452
@@ -475,17 +462,6 @@ struct request_queue {
475 struct blk_mq_hw_ctx **queue_hw_ctx; 462 struct blk_mq_hw_ctx **queue_hw_ctx;
476 unsigned int nr_hw_queues; 463 unsigned int nr_hw_queues;
477 464
478 /*
479 * Dispatch queue sorting
480 */
481 sector_t end_sector;
482 struct request *boundary_rq;
483
484 /*
485 * Delayed queue handling
486 */
487 struct delayed_work delay_work;
488
489 struct backing_dev_info *backing_dev_info; 465 struct backing_dev_info *backing_dev_info;
490 466
491 /* 467 /*
@@ -548,9 +524,6 @@ struct request_queue {
548 * queue settings 524 * queue settings
549 */ 525 */
550 unsigned long nr_requests; /* Max # of requests */ 526 unsigned long nr_requests; /* Max # of requests */
551 unsigned int nr_congestion_on;
552 unsigned int nr_congestion_off;
553 unsigned int nr_batching;
554 527
555 unsigned int dma_drain_size; 528 unsigned int dma_drain_size;
556 void *dma_drain_buffer; 529 void *dma_drain_buffer;
@@ -560,13 +533,6 @@ struct request_queue {
560 unsigned int nr_sorted; 533 unsigned int nr_sorted;
561 unsigned int in_flight[2]; 534 unsigned int in_flight[2];
562 535
563 /*
564 * Number of active block driver functions for which blk_drain_queue()
565 * must wait. Must be incremented around functions that unlock the
566 * queue_lock internally, e.g. scsi_request_fn().
567 */
568 unsigned int request_fn_active;
569
570 unsigned int rq_timeout; 536 unsigned int rq_timeout;
571 int poll_nsec; 537 int poll_nsec;
572 538
@@ -740,11 +706,6 @@ bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q);
740extern void blk_set_pm_only(struct request_queue *q); 706extern void blk_set_pm_only(struct request_queue *q);
741extern void blk_clear_pm_only(struct request_queue *q); 707extern void blk_clear_pm_only(struct request_queue *q);
742 708
743static inline int queue_in_flight(struct request_queue *q)
744{
745 return q->in_flight[0] + q->in_flight[1];
746}
747
748static inline bool blk_account_rq(struct request *rq) 709static inline bool blk_account_rq(struct request *rq)
749{ 710{
750 return (rq->rq_flags & RQF_STARTED) && !blk_rq_is_passthrough(rq); 711 return (rq->rq_flags & RQF_STARTED) && !blk_rq_is_passthrough(rq);
@@ -765,7 +726,7 @@ static inline bool blk_account_rq(struct request *rq)
765 */ 726 */
766static inline bool queue_is_rq_based(struct request_queue *q) 727static inline bool queue_is_rq_based(struct request_queue *q)
767{ 728{
768 return q->request_fn || q->mq_ops; 729 return q->mq_ops;
769} 730}
770 731
771static inline unsigned int blk_queue_cluster(struct request_queue *q) 732static inline unsigned int blk_queue_cluster(struct request_queue *q)
@@ -828,27 +789,6 @@ static inline bool rq_is_sync(struct request *rq)
828 return op_is_sync(rq->cmd_flags); 789 return op_is_sync(rq->cmd_flags);
829} 790}
830 791
831static inline bool blk_rl_full(struct request_list *rl, bool sync)
832{
833 unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
834
835 return rl->flags & flag;
836}
837
838static inline void blk_set_rl_full(struct request_list *rl, bool sync)
839{
840 unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
841
842 rl->flags |= flag;
843}
844
845static inline void blk_clear_rl_full(struct request_list *rl, bool sync)
846{
847 unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
848
849 rl->flags &= ~flag;
850}
851
852static inline bool rq_mergeable(struct request *rq) 792static inline bool rq_mergeable(struct request *rq)
853{ 793{
854 if (blk_rq_is_passthrough(rq)) 794 if (blk_rq_is_passthrough(rq))
@@ -969,7 +909,6 @@ extern void blk_put_request(struct request *);
969extern void __blk_put_request(struct request_queue *, struct request *); 909extern void __blk_put_request(struct request_queue *, struct request *);
970extern struct request *blk_get_request(struct request_queue *, unsigned int op, 910extern struct request *blk_get_request(struct request_queue *, unsigned int op,
971 blk_mq_req_flags_t flags); 911 blk_mq_req_flags_t flags);
972extern void blk_requeue_request(struct request_queue *, struct request *);
973extern int blk_lld_busy(struct request_queue *q); 912extern int blk_lld_busy(struct request_queue *q);
974extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src, 913extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
975 struct bio_set *bs, gfp_t gfp_mask, 914 struct bio_set *bs, gfp_t gfp_mask,
@@ -979,7 +918,6 @@ extern void blk_rq_unprep_clone(struct request *rq);
979extern blk_status_t blk_insert_cloned_request(struct request_queue *q, 918extern blk_status_t blk_insert_cloned_request(struct request_queue *q,
980 struct request *rq); 919 struct request *rq);
981extern int blk_rq_append_bio(struct request *rq, struct bio **bio); 920extern int blk_rq_append_bio(struct request *rq, struct bio **bio);
982extern void blk_delay_queue(struct request_queue *, unsigned long);
983extern void blk_queue_split(struct request_queue *, struct bio **); 921extern void blk_queue_split(struct request_queue *, struct bio **);
984extern void blk_recount_segments(struct request_queue *, struct bio *); 922extern void blk_recount_segments(struct request_queue *, struct bio *);
985extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int); 923extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
@@ -992,15 +930,7 @@ extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
992 930
993extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags); 931extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
994extern void blk_queue_exit(struct request_queue *q); 932extern void blk_queue_exit(struct request_queue *q);
995extern void blk_start_queue(struct request_queue *q);
996extern void blk_start_queue_async(struct request_queue *q);
997extern void blk_stop_queue(struct request_queue *q);
998extern void blk_sync_queue(struct request_queue *q); 933extern void blk_sync_queue(struct request_queue *q);
999extern void __blk_stop_queue(struct request_queue *q);
1000extern void __blk_run_queue(struct request_queue *q);
1001extern void __blk_run_queue_uncond(struct request_queue *q);
1002extern void blk_run_queue(struct request_queue *);
1003extern void blk_run_queue_async(struct request_queue *q);
1004extern int blk_rq_map_user(struct request_queue *, struct request *, 934extern int blk_rq_map_user(struct request_queue *, struct request *,
1005 struct rq_map_data *, void __user *, unsigned long, 935 struct rq_map_data *, void __user *, unsigned long,
1006 gfp_t); 936 gfp_t);
@@ -1155,13 +1085,6 @@ static inline unsigned int blk_rq_count_bios(struct request *rq)
1155 return nr_bios; 1085 return nr_bios;
1156} 1086}
1157 1087
1158/*
1159 * Request issue related functions.
1160 */
1161extern struct request *blk_peek_request(struct request_queue *q);
1162extern void blk_start_request(struct request *rq);
1163extern struct request *blk_fetch_request(struct request_queue *q);
1164
1165void blk_steal_bios(struct bio_list *list, struct request *rq); 1088void blk_steal_bios(struct bio_list *list, struct request *rq);
1166 1089
1167/* 1090/*
@@ -1179,9 +1102,6 @@ void blk_steal_bios(struct bio_list *list, struct request *rq);
1179 */ 1102 */
1180extern bool blk_update_request(struct request *rq, blk_status_t error, 1103extern bool blk_update_request(struct request *rq, blk_status_t error,
1181 unsigned int nr_bytes); 1104 unsigned int nr_bytes);
1182extern void blk_finish_request(struct request *rq, blk_status_t error);
1183extern bool blk_end_request(struct request *rq, blk_status_t error,
1184 unsigned int nr_bytes);
1185extern void blk_end_request_all(struct request *rq, blk_status_t error); 1105extern void blk_end_request_all(struct request *rq, blk_status_t error);
1186extern bool __blk_end_request(struct request *rq, blk_status_t error, 1106extern bool __blk_end_request(struct request *rq, blk_status_t error,
1187 unsigned int nr_bytes); 1107 unsigned int nr_bytes);
@@ -1190,15 +1110,10 @@ extern bool __blk_end_request_cur(struct request *rq, blk_status_t error);
1190 1110
1191extern void __blk_complete_request(struct request *); 1111extern void __blk_complete_request(struct request *);
1192extern void blk_abort_request(struct request *); 1112extern void blk_abort_request(struct request *);
1193extern void blk_unprep_request(struct request *);
1194 1113
1195/* 1114/*
1196 * Access functions for manipulating queue properties 1115 * Access functions for manipulating queue properties
1197 */ 1116 */
1198extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn,
1199 spinlock_t *lock, int node_id);
1200extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *);
1201extern int blk_init_allocated_queue(struct request_queue *);
1202extern void blk_cleanup_queue(struct request_queue *); 1117extern void blk_cleanup_queue(struct request_queue *);
1203extern void blk_queue_make_request(struct request_queue *, make_request_fn *); 1118extern void blk_queue_make_request(struct request_queue *, make_request_fn *);
1204extern void blk_queue_bounce_limit(struct request_queue *, u64); 1119extern void blk_queue_bounce_limit(struct request_queue *, u64);
@@ -1239,8 +1154,6 @@ extern int blk_queue_dma_drain(struct request_queue *q,
1239 void *buf, unsigned int size); 1154 void *buf, unsigned int size);
1240extern void blk_queue_segment_boundary(struct request_queue *, unsigned long); 1155extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
1241extern void blk_queue_virt_boundary(struct request_queue *, unsigned long); 1156extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
1242extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn);
1243extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn);
1244extern void blk_queue_dma_alignment(struct request_queue *, int); 1157extern void blk_queue_dma_alignment(struct request_queue *, int);
1245extern void blk_queue_update_dma_alignment(struct request_queue *, int); 1158extern void blk_queue_update_dma_alignment(struct request_queue *, int);
1246extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *); 1159extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *);
@@ -1298,7 +1211,6 @@ extern void blk_set_queue_dying(struct request_queue *);
1298 * schedule() where blk_schedule_flush_plug() is called. 1211 * schedule() where blk_schedule_flush_plug() is called.
1299 */ 1212 */
1300struct blk_plug { 1213struct blk_plug {
1301 struct list_head list; /* requests */
1302 struct list_head mq_list; /* blk-mq requests */ 1214 struct list_head mq_list; /* blk-mq requests */
1303 struct list_head cb_list; /* md requires an unplug callback */ 1215 struct list_head cb_list; /* md requires an unplug callback */
1304}; 1216};
@@ -1339,8 +1251,7 @@ static inline bool blk_needs_flush_plug(struct task_struct *tsk)
1339 struct blk_plug *plug = tsk->plug; 1251 struct blk_plug *plug = tsk->plug;
1340 1252
1341 return plug && 1253 return plug &&
1342 (!list_empty(&plug->list) || 1254 (!list_empty(&plug->mq_list) ||
1343 !list_empty(&plug->mq_list) ||
1344 !list_empty(&plug->cb_list)); 1255 !list_empty(&plug->cb_list));
1345} 1256}
1346 1257
diff --git a/include/linux/elevator.h b/include/linux/elevator.h
index 015bb59c0331..158004f1754d 100644
--- a/include/linux/elevator.h
+++ b/include/linux/elevator.h
@@ -23,74 +23,6 @@ enum elv_merge {
23 ELEVATOR_DISCARD_MERGE = 3, 23 ELEVATOR_DISCARD_MERGE = 3,
24}; 24};
25 25
26typedef enum elv_merge (elevator_merge_fn) (struct request_queue *, struct request **,
27 struct bio *);
28
29typedef void (elevator_merge_req_fn) (struct request_queue *, struct request *, struct request *);
30
31typedef void (elevator_merged_fn) (struct request_queue *, struct request *, enum elv_merge);
32
33typedef int (elevator_allow_bio_merge_fn) (struct request_queue *,
34 struct request *, struct bio *);
35
36typedef int (elevator_allow_rq_merge_fn) (struct request_queue *,
37 struct request *, struct request *);
38
39typedef void (elevator_bio_merged_fn) (struct request_queue *,
40 struct request *, struct bio *);
41
42typedef int (elevator_dispatch_fn) (struct request_queue *, int);
43
44typedef void (elevator_add_req_fn) (struct request_queue *, struct request *);
45typedef struct request *(elevator_request_list_fn) (struct request_queue *, struct request *);
46typedef void (elevator_completed_req_fn) (struct request_queue *, struct request *);
47typedef int (elevator_may_queue_fn) (struct request_queue *, unsigned int);
48
49typedef void (elevator_init_icq_fn) (struct io_cq *);
50typedef void (elevator_exit_icq_fn) (struct io_cq *);
51typedef int (elevator_set_req_fn) (struct request_queue *, struct request *,
52 struct bio *, gfp_t);
53typedef void (elevator_put_req_fn) (struct request *);
54typedef void (elevator_activate_req_fn) (struct request_queue *, struct request *);
55typedef void (elevator_deactivate_req_fn) (struct request_queue *, struct request *);
56
57typedef int (elevator_init_fn) (struct request_queue *,
58 struct elevator_type *e);
59typedef void (elevator_exit_fn) (struct elevator_queue *);
60typedef void (elevator_registered_fn) (struct request_queue *);
61
62struct elevator_ops
63{
64 elevator_merge_fn *elevator_merge_fn;
65 elevator_merged_fn *elevator_merged_fn;
66 elevator_merge_req_fn *elevator_merge_req_fn;
67 elevator_allow_bio_merge_fn *elevator_allow_bio_merge_fn;
68 elevator_allow_rq_merge_fn *elevator_allow_rq_merge_fn;
69 elevator_bio_merged_fn *elevator_bio_merged_fn;
70
71 elevator_dispatch_fn *elevator_dispatch_fn;
72 elevator_add_req_fn *elevator_add_req_fn;
73 elevator_activate_req_fn *elevator_activate_req_fn;
74 elevator_deactivate_req_fn *elevator_deactivate_req_fn;
75
76 elevator_completed_req_fn *elevator_completed_req_fn;
77
78 elevator_request_list_fn *elevator_former_req_fn;
79 elevator_request_list_fn *elevator_latter_req_fn;
80
81 elevator_init_icq_fn *elevator_init_icq_fn; /* see iocontext.h */
82 elevator_exit_icq_fn *elevator_exit_icq_fn; /* ditto */
83
84 elevator_set_req_fn *elevator_set_req_fn;
85 elevator_put_req_fn *elevator_put_req_fn;
86
87 elevator_may_queue_fn *elevator_may_queue_fn;
88
89 elevator_init_fn *elevator_init_fn;
90 elevator_exit_fn *elevator_exit_fn;
91 elevator_registered_fn *elevator_registered_fn;
92};
93
94struct blk_mq_alloc_data; 26struct blk_mq_alloc_data;
95struct blk_mq_hw_ctx; 27struct blk_mq_hw_ctx;
96 28
@@ -138,16 +70,15 @@ struct elevator_type
138 70
139 /* fields provided by elevator implementation */ 71 /* fields provided by elevator implementation */
140 union { 72 union {
141 struct elevator_ops sq;
142 struct elevator_mq_ops mq; 73 struct elevator_mq_ops mq;
143 } ops; 74 } ops;
75
144 size_t icq_size; /* see iocontext.h */ 76 size_t icq_size; /* see iocontext.h */
145 size_t icq_align; /* ditto */ 77 size_t icq_align; /* ditto */
146 struct elv_fs_entry *elevator_attrs; 78 struct elv_fs_entry *elevator_attrs;
147 char elevator_name[ELV_NAME_MAX]; 79 char elevator_name[ELV_NAME_MAX];
148 const char *elevator_alias; 80 const char *elevator_alias;
149 struct module *elevator_owner; 81 struct module *elevator_owner;
150 bool uses_mq;
151#ifdef CONFIG_BLK_DEBUG_FS 82#ifdef CONFIG_BLK_DEBUG_FS
152 const struct blk_mq_debugfs_attr *queue_debugfs_attrs; 83 const struct blk_mq_debugfs_attr *queue_debugfs_attrs;
153 const struct blk_mq_debugfs_attr *hctx_debugfs_attrs; 84 const struct blk_mq_debugfs_attr *hctx_debugfs_attrs;
@@ -175,40 +106,25 @@ struct elevator_queue
175 struct kobject kobj; 106 struct kobject kobj;
176 struct mutex sysfs_lock; 107 struct mutex sysfs_lock;
177 unsigned int registered:1; 108 unsigned int registered:1;
178 unsigned int uses_mq:1;
179 DECLARE_HASHTABLE(hash, ELV_HASH_BITS); 109 DECLARE_HASHTABLE(hash, ELV_HASH_BITS);
180}; 110};
181 111
182/* 112/*
183 * block elevator interface 113 * block elevator interface
184 */ 114 */
185extern void elv_dispatch_sort(struct request_queue *, struct request *);
186extern void elv_dispatch_add_tail(struct request_queue *, struct request *);
187extern void elv_add_request(struct request_queue *, struct request *, int);
188extern void __elv_add_request(struct request_queue *, struct request *, int);
189extern enum elv_merge elv_merge(struct request_queue *, struct request **, 115extern enum elv_merge elv_merge(struct request_queue *, struct request **,
190 struct bio *); 116 struct bio *);
191extern void elv_merge_requests(struct request_queue *, struct request *, 117extern void elv_merge_requests(struct request_queue *, struct request *,
192 struct request *); 118 struct request *);
193extern void elv_merged_request(struct request_queue *, struct request *, 119extern void elv_merged_request(struct request_queue *, struct request *,
194 enum elv_merge); 120 enum elv_merge);
195extern void elv_bio_merged(struct request_queue *q, struct request *,
196 struct bio *);
197extern bool elv_attempt_insert_merge(struct request_queue *, struct request *); 121extern bool elv_attempt_insert_merge(struct request_queue *, struct request *);
198extern void elv_requeue_request(struct request_queue *, struct request *);
199extern struct request *elv_former_request(struct request_queue *, struct request *); 122extern struct request *elv_former_request(struct request_queue *, struct request *);
200extern struct request *elv_latter_request(struct request_queue *, struct request *); 123extern struct request *elv_latter_request(struct request_queue *, struct request *);
201extern int elv_may_queue(struct request_queue *, unsigned int);
202extern void elv_completed_request(struct request_queue *, struct request *);
203extern int elv_set_request(struct request_queue *q, struct request *rq,
204 struct bio *bio, gfp_t gfp_mask);
205extern void elv_put_request(struct request_queue *, struct request *);
206extern void elv_drain_elevator(struct request_queue *);
207 124
208/* 125/*
209 * io scheduler registration 126 * io scheduler registration
210 */ 127 */
211extern void __init load_default_elevator_module(void);
212extern int elv_register(struct elevator_type *); 128extern int elv_register(struct elevator_type *);
213extern void elv_unregister(struct elevator_type *); 129extern void elv_unregister(struct elevator_type *);
214 130
@@ -260,9 +176,5 @@ enum {
260#define rq_entry_fifo(ptr) list_entry((ptr), struct request, queuelist) 176#define rq_entry_fifo(ptr) list_entry((ptr), struct request, queuelist)
261#define rq_fifo_clear(rq) list_del_init(&(rq)->queuelist) 177#define rq_fifo_clear(rq) list_del_init(&(rq)->queuelist)
262 178
263#else /* CONFIG_BLOCK */
264
265static inline void load_default_elevator_module(void) { }
266
267#endif /* CONFIG_BLOCK */ 179#endif /* CONFIG_BLOCK */
268#endif 180#endif
diff --git a/include/linux/init.h b/include/linux/init.h
index 9c2aba1dbabf..5255069f5a9f 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -146,7 +146,6 @@ extern unsigned int reset_devices;
146/* used by init/main.c */ 146/* used by init/main.c */
147void setup_arch(char **); 147void setup_arch(char **);
148void prepare_namespace(void); 148void prepare_namespace(void);
149void __init load_default_modules(void);
150int __init init_rootfs(void); 149int __init init_rootfs(void);
151 150
152#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX) 151#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX)
diff --git a/init/do_mounts_initrd.c b/init/do_mounts_initrd.c
index d1a5d885ce13..73e02ea5d5d1 100644
--- a/init/do_mounts_initrd.c
+++ b/init/do_mounts_initrd.c
@@ -53,9 +53,6 @@ static void __init handle_initrd(void)
53 ksys_mkdir("/old", 0700); 53 ksys_mkdir("/old", 0700);
54 ksys_chdir("/old"); 54 ksys_chdir("/old");
55 55
56 /* try loading default modules from initrd */
57 load_default_modules();
58
59 /* 56 /*
60 * In case that a resume from disk is carried out by linuxrc or one of 57 * In case that a resume from disk is carried out by linuxrc or one of
61 * its children, we need to tell the freezer not to wait for us. 58 * its children, we need to tell the freezer not to wait for us.
diff --git a/init/initramfs.c b/init/initramfs.c
index 640557788026..96af18fec4d0 100644
--- a/init/initramfs.c
+++ b/init/initramfs.c
@@ -644,12 +644,6 @@ static int __init populate_rootfs(void)
644#endif 644#endif
645 } 645 }
646 flush_delayed_fput(); 646 flush_delayed_fput();
647 /*
648 * Try loading default modules from initramfs. This gives
649 * us a chance to load before device_initcalls.
650 */
651 load_default_modules();
652
653 return 0; 647 return 0;
654} 648}
655rootfs_initcall(populate_rootfs); 649rootfs_initcall(populate_rootfs);
diff --git a/init/main.c b/init/main.c
index ee147103ba1b..ca0cdb0c388b 100644
--- a/init/main.c
+++ b/init/main.c
@@ -996,17 +996,6 @@ static void __init do_pre_smp_initcalls(void)
996 do_one_initcall(initcall_from_entry(fn)); 996 do_one_initcall(initcall_from_entry(fn));
997} 997}
998 998
999/*
1000 * This function requests modules which should be loaded by default and is
1001 * called twice right after initrd is mounted and right before init is
1002 * exec'd. If such modules are on either initrd or rootfs, they will be
1003 * loaded before control is passed to userland.
1004 */
1005void __init load_default_modules(void)
1006{
1007 load_default_elevator_module();
1008}
1009
1010static int run_init_process(const char *init_filename) 999static int run_init_process(const char *init_filename)
1011{ 1000{
1012 argv_init[0] = init_filename; 1001 argv_init[0] = init_filename;
@@ -1180,5 +1169,4 @@ static noinline void __init kernel_init_freeable(void)
1180 */ 1169 */
1181 1170
1182 integrity_load_keys(); 1171 integrity_load_keys();
1183 load_default_modules();
1184} 1172}
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-12 03:30:41 -0400