diff options
Diffstat (limited to 'block/elevator.c')
-rw-r--r-- | block/elevator.c | 185 |
1 files changed, 36 insertions, 149 deletions
diff --git a/block/elevator.c b/block/elevator.c index e220f0c543e3..ca861927ba41 100644 --- a/block/elevator.c +++ b/block/elevator.c | |||
@@ -51,8 +51,7 @@ static const int elv_hash_shift = 6; | |||
51 | #define ELV_HASH_FN(sec) \ | 51 | #define ELV_HASH_FN(sec) \ |
52 | (hash_long(ELV_HASH_BLOCK((sec)), elv_hash_shift)) | 52 | (hash_long(ELV_HASH_BLOCK((sec)), elv_hash_shift)) |
53 | #define ELV_HASH_ENTRIES (1 << elv_hash_shift) | 53 | #define ELV_HASH_ENTRIES (1 << elv_hash_shift) |
54 | #define rq_hash_key(rq) ((rq)->sector + (rq)->nr_sectors) | 54 | #define rq_hash_key(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq)) |
55 | #define ELV_ON_HASH(rq) (!hlist_unhashed(&(rq)->hash)) | ||
56 | 55 | ||
57 | /* | 56 | /* |
58 | * Query io scheduler to see if the current process issuing bio may be | 57 | * Query io scheduler to see if the current process issuing bio may be |
@@ -116,9 +115,9 @@ static inline int elv_try_merge(struct request *__rq, struct bio *bio) | |||
116 | * we can merge and sequence is ok, check if it's possible | 115 | * we can merge and sequence is ok, check if it's possible |
117 | */ | 116 | */ |
118 | if (elv_rq_merge_ok(__rq, bio)) { | 117 | if (elv_rq_merge_ok(__rq, bio)) { |
119 | if (__rq->sector + __rq->nr_sectors == bio->bi_sector) | 118 | if (blk_rq_pos(__rq) + blk_rq_sectors(__rq) == bio->bi_sector) |
120 | ret = ELEVATOR_BACK_MERGE; | 119 | ret = ELEVATOR_BACK_MERGE; |
121 | else if (__rq->sector - bio_sectors(bio) == bio->bi_sector) | 120 | else if (blk_rq_pos(__rq) - bio_sectors(bio) == bio->bi_sector) |
122 | ret = ELEVATOR_FRONT_MERGE; | 121 | ret = ELEVATOR_FRONT_MERGE; |
123 | } | 122 | } |
124 | 123 | ||
@@ -306,22 +305,6 @@ void elevator_exit(struct elevator_queue *e) | |||
306 | } | 305 | } |
307 | EXPORT_SYMBOL(elevator_exit); | 306 | EXPORT_SYMBOL(elevator_exit); |
308 | 307 | ||
309 | static void elv_activate_rq(struct request_queue *q, struct request *rq) | ||
310 | { | ||
311 | struct elevator_queue *e = q->elevator; | ||
312 | |||
313 | if (e->ops->elevator_activate_req_fn) | ||
314 | e->ops->elevator_activate_req_fn(q, rq); | ||
315 | } | ||
316 | |||
317 | static void elv_deactivate_rq(struct request_queue *q, struct request *rq) | ||
318 | { | ||
319 | struct elevator_queue *e = q->elevator; | ||
320 | |||
321 | if (e->ops->elevator_deactivate_req_fn) | ||
322 | e->ops->elevator_deactivate_req_fn(q, rq); | ||
323 | } | ||
324 | |||
325 | static inline void __elv_rqhash_del(struct request *rq) | 308 | static inline void __elv_rqhash_del(struct request *rq) |
326 | { | 309 | { |
327 | hlist_del_init(&rq->hash); | 310 | hlist_del_init(&rq->hash); |
@@ -383,9 +366,9 @@ struct request *elv_rb_add(struct rb_root *root, struct request *rq) | |||
383 | parent = *p; | 366 | parent = *p; |
384 | __rq = rb_entry(parent, struct request, rb_node); | 367 | __rq = rb_entry(parent, struct request, rb_node); |
385 | 368 | ||
386 | if (rq->sector < __rq->sector) | 369 | if (blk_rq_pos(rq) < blk_rq_pos(__rq)) |
387 | p = &(*p)->rb_left; | 370 | p = &(*p)->rb_left; |
388 | else if (rq->sector > __rq->sector) | 371 | else if (blk_rq_pos(rq) > blk_rq_pos(__rq)) |
389 | p = &(*p)->rb_right; | 372 | p = &(*p)->rb_right; |
390 | else | 373 | else |
391 | return __rq; | 374 | return __rq; |
@@ -413,9 +396,9 @@ struct request *elv_rb_find(struct rb_root *root, sector_t sector) | |||
413 | while (n) { | 396 | while (n) { |
414 | rq = rb_entry(n, struct request, rb_node); | 397 | rq = rb_entry(n, struct request, rb_node); |
415 | 398 | ||
416 | if (sector < rq->sector) | 399 | if (sector < blk_rq_pos(rq)) |
417 | n = n->rb_left; | 400 | n = n->rb_left; |
418 | else if (sector > rq->sector) | 401 | else if (sector > blk_rq_pos(rq)) |
419 | n = n->rb_right; | 402 | n = n->rb_right; |
420 | else | 403 | else |
421 | return rq; | 404 | return rq; |
@@ -454,14 +437,14 @@ void elv_dispatch_sort(struct request_queue *q, struct request *rq) | |||
454 | break; | 437 | break; |
455 | if (pos->cmd_flags & stop_flags) | 438 | if (pos->cmd_flags & stop_flags) |
456 | break; | 439 | break; |
457 | if (rq->sector >= boundary) { | 440 | if (blk_rq_pos(rq) >= boundary) { |
458 | if (pos->sector < boundary) | 441 | if (blk_rq_pos(pos) < boundary) |
459 | continue; | 442 | continue; |
460 | } else { | 443 | } else { |
461 | if (pos->sector >= boundary) | 444 | if (blk_rq_pos(pos) >= boundary) |
462 | break; | 445 | break; |
463 | } | 446 | } |
464 | if (rq->sector >= pos->sector) | 447 | if (blk_rq_pos(rq) >= blk_rq_pos(pos)) |
465 | break; | 448 | break; |
466 | } | 449 | } |
467 | 450 | ||
@@ -559,7 +542,7 @@ void elv_requeue_request(struct request_queue *q, struct request *rq) | |||
559 | * in_flight count again | 542 | * in_flight count again |
560 | */ | 543 | */ |
561 | if (blk_account_rq(rq)) { | 544 | if (blk_account_rq(rq)) { |
562 | q->in_flight--; | 545 | q->in_flight[rq_is_sync(rq)]--; |
563 | if (blk_sorted_rq(rq)) | 546 | if (blk_sorted_rq(rq)) |
564 | elv_deactivate_rq(q, rq); | 547 | elv_deactivate_rq(q, rq); |
565 | } | 548 | } |
@@ -588,6 +571,9 @@ void elv_drain_elevator(struct request_queue *q) | |||
588 | */ | 571 | */ |
589 | void elv_quiesce_start(struct request_queue *q) | 572 | void elv_quiesce_start(struct request_queue *q) |
590 | { | 573 | { |
574 | if (!q->elevator) | ||
575 | return; | ||
576 | |||
591 | queue_flag_set(QUEUE_FLAG_ELVSWITCH, q); | 577 | queue_flag_set(QUEUE_FLAG_ELVSWITCH, q); |
592 | 578 | ||
593 | /* | 579 | /* |
@@ -595,7 +581,7 @@ void elv_quiesce_start(struct request_queue *q) | |||
595 | */ | 581 | */ |
596 | elv_drain_elevator(q); | 582 | elv_drain_elevator(q); |
597 | while (q->rq.elvpriv) { | 583 | while (q->rq.elvpriv) { |
598 | blk_start_queueing(q); | 584 | __blk_run_queue(q); |
599 | spin_unlock_irq(q->queue_lock); | 585 | spin_unlock_irq(q->queue_lock); |
600 | msleep(10); | 586 | msleep(10); |
601 | spin_lock_irq(q->queue_lock); | 587 | spin_lock_irq(q->queue_lock); |
@@ -639,8 +625,7 @@ void elv_insert(struct request_queue *q, struct request *rq, int where) | |||
639 | * with anything. There's no point in delaying queue | 625 | * with anything. There's no point in delaying queue |
640 | * processing. | 626 | * processing. |
641 | */ | 627 | */ |
642 | blk_remove_plug(q); | 628 | __blk_run_queue(q); |
643 | blk_start_queueing(q); | ||
644 | break; | 629 | break; |
645 | 630 | ||
646 | case ELEVATOR_INSERT_SORT: | 631 | case ELEVATOR_INSERT_SORT: |
@@ -699,7 +684,7 @@ void elv_insert(struct request_queue *q, struct request *rq, int where) | |||
699 | 684 | ||
700 | if (unplug_it && blk_queue_plugged(q)) { | 685 | if (unplug_it && blk_queue_plugged(q)) { |
701 | int nrq = q->rq.count[BLK_RW_SYNC] + q->rq.count[BLK_RW_ASYNC] | 686 | int nrq = q->rq.count[BLK_RW_SYNC] + q->rq.count[BLK_RW_ASYNC] |
702 | - q->in_flight; | 687 | - queue_in_flight(q); |
703 | 688 | ||
704 | if (nrq >= q->unplug_thresh) | 689 | if (nrq >= q->unplug_thresh) |
705 | __generic_unplug_device(q); | 690 | __generic_unplug_device(q); |
@@ -755,117 +740,6 @@ void elv_add_request(struct request_queue *q, struct request *rq, int where, | |||
755 | } | 740 | } |
756 | EXPORT_SYMBOL(elv_add_request); | 741 | EXPORT_SYMBOL(elv_add_request); |
757 | 742 | ||
758 | static inline struct request *__elv_next_request(struct request_queue *q) | ||
759 | { | ||
760 | struct request *rq; | ||
761 | |||
762 | while (1) { | ||
763 | while (!list_empty(&q->queue_head)) { | ||
764 | rq = list_entry_rq(q->queue_head.next); | ||
765 | if (blk_do_ordered(q, &rq)) | ||
766 | return rq; | ||
767 | } | ||
768 | |||
769 | if (!q->elevator->ops->elevator_dispatch_fn(q, 0)) | ||
770 | return NULL; | ||
771 | } | ||
772 | } | ||
773 | |||
774 | struct request *elv_next_request(struct request_queue *q) | ||
775 | { | ||
776 | struct request *rq; | ||
777 | int ret; | ||
778 | |||
779 | while ((rq = __elv_next_request(q)) != NULL) { | ||
780 | if (!(rq->cmd_flags & REQ_STARTED)) { | ||
781 | /* | ||
782 | * This is the first time the device driver | ||
783 | * sees this request (possibly after | ||
784 | * requeueing). Notify IO scheduler. | ||
785 | */ | ||
786 | if (blk_sorted_rq(rq)) | ||
787 | elv_activate_rq(q, rq); | ||
788 | |||
789 | /* | ||
790 | * just mark as started even if we don't start | ||
791 | * it, a request that has been delayed should | ||
792 | * not be passed by new incoming requests | ||
793 | */ | ||
794 | rq->cmd_flags |= REQ_STARTED; | ||
795 | trace_block_rq_issue(q, rq); | ||
796 | } | ||
797 | |||
798 | if (!q->boundary_rq || q->boundary_rq == rq) { | ||
799 | q->end_sector = rq_end_sector(rq); | ||
800 | q->boundary_rq = NULL; | ||
801 | } | ||
802 | |||
803 | if (rq->cmd_flags & REQ_DONTPREP) | ||
804 | break; | ||
805 | |||
806 | if (q->dma_drain_size && rq->data_len) { | ||
807 | /* | ||
808 | * make sure space for the drain appears we | ||
809 | * know we can do this because max_hw_segments | ||
810 | * has been adjusted to be one fewer than the | ||
811 | * device can handle | ||
812 | */ | ||
813 | rq->nr_phys_segments++; | ||
814 | } | ||
815 | |||
816 | if (!q->prep_rq_fn) | ||
817 | break; | ||
818 | |||
819 | ret = q->prep_rq_fn(q, rq); | ||
820 | if (ret == BLKPREP_OK) { | ||
821 | break; | ||
822 | } else if (ret == BLKPREP_DEFER) { | ||
823 | /* | ||
824 | * the request may have been (partially) prepped. | ||
825 | * we need to keep this request in the front to | ||
826 | * avoid resource deadlock. REQ_STARTED will | ||
827 | * prevent other fs requests from passing this one. | ||
828 | */ | ||
829 | if (q->dma_drain_size && rq->data_len && | ||
830 | !(rq->cmd_flags & REQ_DONTPREP)) { | ||
831 | /* | ||
832 | * remove the space for the drain we added | ||
833 | * so that we don't add it again | ||
834 | */ | ||
835 | --rq->nr_phys_segments; | ||
836 | } | ||
837 | |||
838 | rq = NULL; | ||
839 | break; | ||
840 | } else if (ret == BLKPREP_KILL) { | ||
841 | rq->cmd_flags |= REQ_QUIET; | ||
842 | __blk_end_request(rq, -EIO, blk_rq_bytes(rq)); | ||
843 | } else { | ||
844 | printk(KERN_ERR "%s: bad return=%d\n", __func__, ret); | ||
845 | break; | ||
846 | } | ||
847 | } | ||
848 | |||
849 | return rq; | ||
850 | } | ||
851 | EXPORT_SYMBOL(elv_next_request); | ||
852 | |||
853 | void elv_dequeue_request(struct request_queue *q, struct request *rq) | ||
854 | { | ||
855 | BUG_ON(list_empty(&rq->queuelist)); | ||
856 | BUG_ON(ELV_ON_HASH(rq)); | ||
857 | |||
858 | list_del_init(&rq->queuelist); | ||
859 | |||
860 | /* | ||
861 | * the time frame between a request being removed from the lists | ||
862 | * and to it is freed is accounted as io that is in progress at | ||
863 | * the driver side. | ||
864 | */ | ||
865 | if (blk_account_rq(rq)) | ||
866 | q->in_flight++; | ||
867 | } | ||
868 | |||
869 | int elv_queue_empty(struct request_queue *q) | 743 | int elv_queue_empty(struct request_queue *q) |
870 | { | 744 | { |
871 | struct elevator_queue *e = q->elevator; | 745 | struct elevator_queue *e = q->elevator; |
@@ -935,7 +809,12 @@ void elv_abort_queue(struct request_queue *q) | |||
935 | rq = list_entry_rq(q->queue_head.next); | 809 | rq = list_entry_rq(q->queue_head.next); |
936 | rq->cmd_flags |= REQ_QUIET; | 810 | rq->cmd_flags |= REQ_QUIET; |
937 | trace_block_rq_abort(q, rq); | 811 | trace_block_rq_abort(q, rq); |
938 | __blk_end_request(rq, -EIO, blk_rq_bytes(rq)); | 812 | /* |
813 | * Mark this request as started so we don't trigger | ||
814 | * any debug logic in the end I/O path. | ||
815 | */ | ||
816 | blk_start_request(rq); | ||
817 | __blk_end_request_all(rq, -EIO); | ||
939 | } | 818 | } |
940 | } | 819 | } |
941 | EXPORT_SYMBOL(elv_abort_queue); | 820 | EXPORT_SYMBOL(elv_abort_queue); |
@@ -948,7 +827,7 @@ void elv_completed_request(struct request_queue *q, struct request *rq) | |||
948 | * request is released from the driver, io must be done | 827 | * request is released from the driver, io must be done |
949 | */ | 828 | */ |
950 | if (blk_account_rq(rq)) { | 829 | if (blk_account_rq(rq)) { |
951 | q->in_flight--; | 830 | q->in_flight[rq_is_sync(rq)]--; |
952 | if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn) | 831 | if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn) |
953 | e->ops->elevator_completed_req_fn(q, rq); | 832 | e->ops->elevator_completed_req_fn(q, rq); |
954 | } | 833 | } |
@@ -963,11 +842,11 @@ void elv_completed_request(struct request_queue *q, struct request *rq) | |||
963 | if (!list_empty(&q->queue_head)) | 842 | if (!list_empty(&q->queue_head)) |
964 | next = list_entry_rq(q->queue_head.next); | 843 | next = list_entry_rq(q->queue_head.next); |
965 | 844 | ||
966 | if (!q->in_flight && | 845 | if (!queue_in_flight(q) && |
967 | blk_ordered_cur_seq(q) == QUEUE_ORDSEQ_DRAIN && | 846 | blk_ordered_cur_seq(q) == QUEUE_ORDSEQ_DRAIN && |
968 | (!next || blk_ordered_req_seq(next) > QUEUE_ORDSEQ_DRAIN)) { | 847 | (!next || blk_ordered_req_seq(next) > QUEUE_ORDSEQ_DRAIN)) { |
969 | blk_ordered_complete_seq(q, QUEUE_ORDSEQ_DRAIN, 0); | 848 | blk_ordered_complete_seq(q, QUEUE_ORDSEQ_DRAIN, 0); |
970 | blk_start_queueing(q); | 849 | __blk_run_queue(q); |
971 | } | 850 | } |
972 | } | 851 | } |
973 | } | 852 | } |
@@ -1175,6 +1054,9 @@ ssize_t elv_iosched_store(struct request_queue *q, const char *name, | |||
1175 | char elevator_name[ELV_NAME_MAX]; | 1054 | char elevator_name[ELV_NAME_MAX]; |
1176 | struct elevator_type *e; | 1055 | struct elevator_type *e; |
1177 | 1056 | ||
1057 | if (!q->elevator) | ||
1058 | return count; | ||
1059 | |||
1178 | strlcpy(elevator_name, name, sizeof(elevator_name)); | 1060 | strlcpy(elevator_name, name, sizeof(elevator_name)); |
1179 | strstrip(elevator_name); | 1061 | strstrip(elevator_name); |
1180 | 1062 | ||
@@ -1198,10 +1080,15 @@ ssize_t elv_iosched_store(struct request_queue *q, const char *name, | |||
1198 | ssize_t elv_iosched_show(struct request_queue *q, char *name) | 1080 | ssize_t elv_iosched_show(struct request_queue *q, char *name) |
1199 | { | 1081 | { |
1200 | struct elevator_queue *e = q->elevator; | 1082 | struct elevator_queue *e = q->elevator; |
1201 | struct elevator_type *elv = e->elevator_type; | 1083 | struct elevator_type *elv; |
1202 | struct elevator_type *__e; | 1084 | struct elevator_type *__e; |
1203 | int len = 0; | 1085 | int len = 0; |
1204 | 1086 | ||
1087 | if (!q->elevator) | ||
1088 | return sprintf(name, "none\n"); | ||
1089 | |||
1090 | elv = e->elevator_type; | ||
1091 | |||
1205 | spin_lock(&elv_list_lock); | 1092 | spin_lock(&elv_list_lock); |
1206 | list_for_each_entry(__e, &elv_list, list) { | 1093 | list_for_each_entry(__e, &elv_list, list) { |
1207 | if (!strcmp(elv->elevator_name, __e->elevator_name)) | 1094 | if (!strcmp(elv->elevator_name, __e->elevator_name)) |