aboutsummaryrefslogtreecommitdiffstats
path: root/block/blk-core.c
diff options
context:
space:
mode:
Diffstat (limited to 'block/blk-core.c')
-rw-r--r--block/blk-core.c70
1 files changed, 35 insertions, 35 deletions
diff --git a/block/blk-core.c b/block/blk-core.c
index 996ed906d8ca..a32b571aaaa2 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -484,11 +484,11 @@ static int blk_init_free_list(struct request_queue *q)
484{ 484{
485 struct request_list *rl = &q->rq; 485 struct request_list *rl = &q->rq;
486 486
487 rl->count[READ] = rl->count[WRITE] = 0; 487 rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
488 rl->starved[READ] = rl->starved[WRITE] = 0; 488 rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
489 rl->elvpriv = 0; 489 rl->elvpriv = 0;
490 init_waitqueue_head(&rl->wait[READ]); 490 init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
491 init_waitqueue_head(&rl->wait[WRITE]); 491 init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
492 492
493 rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab, 493 rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
494 mempool_free_slab, request_cachep, q->node); 494 mempool_free_slab, request_cachep, q->node);
@@ -699,18 +699,18 @@ static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
699 ioc->last_waited = jiffies; 699 ioc->last_waited = jiffies;
700} 700}
701 701
702static void __freed_request(struct request_queue *q, int rw) 702static void __freed_request(struct request_queue *q, int sync)
703{ 703{
704 struct request_list *rl = &q->rq; 704 struct request_list *rl = &q->rq;
705 705
706 if (rl->count[rw] < queue_congestion_off_threshold(q)) 706 if (rl->count[sync] < queue_congestion_off_threshold(q))
707 blk_clear_queue_congested(q, rw); 707 blk_clear_queue_congested(q, sync);
708 708
709 if (rl->count[rw] + 1 <= q->nr_requests) { 709 if (rl->count[sync] + 1 <= q->nr_requests) {
710 if (waitqueue_active(&rl->wait[rw])) 710 if (waitqueue_active(&rl->wait[sync]))
711 wake_up(&rl->wait[rw]); 711 wake_up(&rl->wait[sync]);
712 712
713 blk_clear_queue_full(q, rw); 713 blk_clear_queue_full(q, sync);
714 } 714 }
715} 715}
716 716
@@ -718,18 +718,18 @@ static void __freed_request(struct request_queue *q, int rw)
718 * A request has just been released. Account for it, update the full and 718 * A request has just been released. Account for it, update the full and
719 * congestion status, wake up any waiters. Called under q->queue_lock. 719 * congestion status, wake up any waiters. Called under q->queue_lock.
720 */ 720 */
721static void freed_request(struct request_queue *q, int rw, int priv) 721static void freed_request(struct request_queue *q, int sync, int priv)
722{ 722{
723 struct request_list *rl = &q->rq; 723 struct request_list *rl = &q->rq;
724 724
725 rl->count[rw]--; 725 rl->count[sync]--;
726 if (priv) 726 if (priv)
727 rl->elvpriv--; 727 rl->elvpriv--;
728 728
729 __freed_request(q, rw); 729 __freed_request(q, sync);
730 730
731 if (unlikely(rl->starved[rw ^ 1])) 731 if (unlikely(rl->starved[sync ^ 1]))
732 __freed_request(q, rw ^ 1); 732 __freed_request(q, sync ^ 1);
733} 733}
734 734
735/* 735/*
@@ -743,15 +743,15 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
743 struct request *rq = NULL; 743 struct request *rq = NULL;
744 struct request_list *rl = &q->rq; 744 struct request_list *rl = &q->rq;
745 struct io_context *ioc = NULL; 745 struct io_context *ioc = NULL;
746 const int rw = rw_flags & 0x01; 746 const bool is_sync = rw_is_sync(rw_flags) != 0;
747 int may_queue, priv; 747 int may_queue, priv;
748 748
749 may_queue = elv_may_queue(q, rw_flags); 749 may_queue = elv_may_queue(q, rw_flags);
750 if (may_queue == ELV_MQUEUE_NO) 750 if (may_queue == ELV_MQUEUE_NO)
751 goto rq_starved; 751 goto rq_starved;
752 752
753 if (rl->count[rw]+1 >= queue_congestion_on_threshold(q)) { 753 if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
754 if (rl->count[rw]+1 >= q->nr_requests) { 754 if (rl->count[is_sync]+1 >= q->nr_requests) {
755 ioc = current_io_context(GFP_ATOMIC, q->node); 755 ioc = current_io_context(GFP_ATOMIC, q->node);
756 /* 756 /*
757 * The queue will fill after this allocation, so set 757 * The queue will fill after this allocation, so set
@@ -759,9 +759,9 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
759 * This process will be allowed to complete a batch of 759 * This process will be allowed to complete a batch of
760 * requests, others will be blocked. 760 * requests, others will be blocked.
761 */ 761 */
762 if (!blk_queue_full(q, rw)) { 762 if (!blk_queue_full(q, is_sync)) {
763 ioc_set_batching(q, ioc); 763 ioc_set_batching(q, ioc);
764 blk_set_queue_full(q, rw); 764 blk_set_queue_full(q, is_sync);
765 } else { 765 } else {
766 if (may_queue != ELV_MQUEUE_MUST 766 if (may_queue != ELV_MQUEUE_MUST
767 && !ioc_batching(q, ioc)) { 767 && !ioc_batching(q, ioc)) {
@@ -774,7 +774,7 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
774 } 774 }
775 } 775 }
776 } 776 }
777 blk_set_queue_congested(q, rw); 777 blk_set_queue_congested(q, is_sync);
778 } 778 }
779 779
780 /* 780 /*
@@ -782,11 +782,11 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
782 * limit of requests, otherwise we could have thousands of requests 782 * limit of requests, otherwise we could have thousands of requests
783 * allocated with any setting of ->nr_requests 783 * allocated with any setting of ->nr_requests
784 */ 784 */
785 if (rl->count[rw] >= (3 * q->nr_requests / 2)) 785 if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
786 goto out; 786 goto out;
787 787
788 rl->count[rw]++; 788 rl->count[is_sync]++;
789 rl->starved[rw] = 0; 789 rl->starved[is_sync] = 0;
790 790
791 priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags); 791 priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
792 if (priv) 792 if (priv)
@@ -804,7 +804,7 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
804 * wait queue, but this is pretty rare. 804 * wait queue, but this is pretty rare.
805 */ 805 */
806 spin_lock_irq(q->queue_lock); 806 spin_lock_irq(q->queue_lock);
807 freed_request(q, rw, priv); 807 freed_request(q, is_sync, priv);
808 808
809 /* 809 /*
810 * in the very unlikely event that allocation failed and no 810 * in the very unlikely event that allocation failed and no
@@ -814,8 +814,8 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
814 * rq mempool into READ and WRITE 814 * rq mempool into READ and WRITE
815 */ 815 */
816rq_starved: 816rq_starved:
817 if (unlikely(rl->count[rw] == 0)) 817 if (unlikely(rl->count[is_sync] == 0))
818 rl->starved[rw] = 1; 818 rl->starved[is_sync] = 1;
819 819
820 goto out; 820 goto out;
821 } 821 }
@@ -829,7 +829,7 @@ rq_starved:
829 if (ioc_batching(q, ioc)) 829 if (ioc_batching(q, ioc))
830 ioc->nr_batch_requests--; 830 ioc->nr_batch_requests--;
831 831
832 trace_block_getrq(q, bio, rw); 832 trace_block_getrq(q, bio, rw_flags & 1);
833out: 833out:
834 return rq; 834 return rq;
835} 835}
@@ -843,7 +843,7 @@ out:
843static struct request *get_request_wait(struct request_queue *q, int rw_flags, 843static struct request *get_request_wait(struct request_queue *q, int rw_flags,
844 struct bio *bio) 844 struct bio *bio)
845{ 845{
846 const int rw = rw_flags & 0x01; 846 const bool is_sync = rw_is_sync(rw_flags) != 0;
847 struct request *rq; 847 struct request *rq;
848 848
849 rq = get_request(q, rw_flags, bio, GFP_NOIO); 849 rq = get_request(q, rw_flags, bio, GFP_NOIO);
@@ -852,10 +852,10 @@ static struct request *get_request_wait(struct request_queue *q, int rw_flags,
852 struct io_context *ioc; 852 struct io_context *ioc;
853 struct request_list *rl = &q->rq; 853 struct request_list *rl = &q->rq;
854 854
855 prepare_to_wait_exclusive(&rl->wait[rw], &wait, 855 prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
856 TASK_UNINTERRUPTIBLE); 856 TASK_UNINTERRUPTIBLE);
857 857
858 trace_block_sleeprq(q, bio, rw); 858 trace_block_sleeprq(q, bio, rw_flags & 1);
859 859
860 __generic_unplug_device(q); 860 __generic_unplug_device(q);
861 spin_unlock_irq(q->queue_lock); 861 spin_unlock_irq(q->queue_lock);
@@ -871,7 +871,7 @@ static struct request *get_request_wait(struct request_queue *q, int rw_flags,
871 ioc_set_batching(q, ioc); 871 ioc_set_batching(q, ioc);
872 872
873 spin_lock_irq(q->queue_lock); 873 spin_lock_irq(q->queue_lock);
874 finish_wait(&rl->wait[rw], &wait); 874 finish_wait(&rl->wait[is_sync], &wait);
875 875
876 rq = get_request(q, rw_flags, bio, GFP_NOIO); 876 rq = get_request(q, rw_flags, bio, GFP_NOIO);
877 }; 877 };
@@ -1070,14 +1070,14 @@ void __blk_put_request(struct request_queue *q, struct request *req)
1070 * it didn't come out of our reserved rq pools 1070 * it didn't come out of our reserved rq pools
1071 */ 1071 */
1072 if (req->cmd_flags & REQ_ALLOCED) { 1072 if (req->cmd_flags & REQ_ALLOCED) {
1073 int rw = rq_data_dir(req); 1073 int is_sync = rq_is_sync(req) != 0;
1074 int priv = req->cmd_flags & REQ_ELVPRIV; 1074 int priv = req->cmd_flags & REQ_ELVPRIV;
1075 1075
1076 BUG_ON(!list_empty(&req->queuelist)); 1076 BUG_ON(!list_empty(&req->queuelist));
1077 BUG_ON(!hlist_unhashed(&req->hash)); 1077 BUG_ON(!hlist_unhashed(&req->hash));
1078 1078
1079 blk_free_request(q, req); 1079 blk_free_request(q, req);
1080 freed_request(q, rw, priv); 1080 freed_request(q, is_sync, priv);
1081 } 1081 }
1082} 1082}
1083EXPORT_SYMBOL_GPL(__blk_put_request); 1083EXPORT_SYMBOL_GPL(__blk_put_request);