aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJens Axboe <jens.axboe@oracle.com>2007-01-18 19:51:58 -0500
committerJens Axboe <axboe@carl.home.kernel.dk>2007-02-11 17:14:45 -0500
commit44f7c16065c83060cbb9dd9b367141682a6e2b8e (patch)
tree5ed1af3ed6a98edf2d17eaa5047807764a04988c
parent99f9628aba4d8fb3b8d955c9efded0d0a1995fad (diff)
cfq-iosched: defer slice activation to first request being active
This better matches what time the queue is actually spending doing IO. Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
-rw-r--r--block/cfq-iosched.c91
1 files changed, 53 insertions, 38 deletions
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 7a8ef0f09699..d44402a4c5cd 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -171,6 +171,7 @@ enum cfqq_state_flags {
171 CFQ_CFQQ_FLAG_idle_window, /* slice idling enabled */ 171 CFQ_CFQQ_FLAG_idle_window, /* slice idling enabled */
172 CFQ_CFQQ_FLAG_prio_changed, /* task priority has changed */ 172 CFQ_CFQQ_FLAG_prio_changed, /* task priority has changed */
173 CFQ_CFQQ_FLAG_queue_new, /* queue never been serviced */ 173 CFQ_CFQQ_FLAG_queue_new, /* queue never been serviced */
174 CFQ_CFQQ_FLAG_slice_new, /* no requests dispatched in slice */
174}; 175};
175 176
176#define CFQ_CFQQ_FNS(name) \ 177#define CFQ_CFQQ_FNS(name) \
@@ -196,6 +197,7 @@ CFQ_CFQQ_FNS(fifo_expire);
196CFQ_CFQQ_FNS(idle_window); 197CFQ_CFQQ_FNS(idle_window);
197CFQ_CFQQ_FNS(prio_changed); 198CFQ_CFQQ_FNS(prio_changed);
198CFQ_CFQQ_FNS(queue_new); 199CFQ_CFQQ_FNS(queue_new);
200CFQ_CFQQ_FNS(slice_new);
199#undef CFQ_CFQQ_FNS 201#undef CFQ_CFQQ_FNS
200 202
201static struct cfq_queue *cfq_find_cfq_hash(struct cfq_data *, unsigned int, unsigned short); 203static struct cfq_queue *cfq_find_cfq_hash(struct cfq_data *, unsigned int, unsigned short);
@@ -231,6 +233,42 @@ static inline pid_t cfq_queue_pid(struct task_struct *task, int rw, int is_sync)
231} 233}
232 234
233/* 235/*
236 * Scale schedule slice based on io priority. Use the sync time slice only
237 * if a queue is marked sync and has sync io queued. A sync queue with async
238 * io only, should not get full sync slice length.
239 */
240static inline int
241cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
242{
243 const int base_slice = cfqd->cfq_slice[cfq_cfqq_sync(cfqq)];
244
245 WARN_ON(cfqq->ioprio >= IOPRIO_BE_NR);
246
247 return base_slice + (base_slice/CFQ_SLICE_SCALE * (4 - cfqq->ioprio));
248}
249
250static inline void
251cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
252{
253 cfqq->slice_end = cfq_prio_to_slice(cfqd, cfqq) + jiffies;
254}
255
256/*
257 * We need to wrap this check in cfq_cfqq_slice_new(), since ->slice_end
258 * isn't valid until the first request from the dispatch is activated
259 * and the slice time set.
260 */
261static inline int cfq_slice_used(struct cfq_queue *cfqq)
262{
263 if (cfq_cfqq_slice_new(cfqq))
264 return 0;
265 if (time_before(jiffies, cfqq->slice_end))
266 return 0;
267
268 return 1;
269}
270
271/*
234 * Lifted from AS - choose which of rq1 and rq2 that is best served now. 272 * Lifted from AS - choose which of rq1 and rq2 that is best served now.
235 * We choose the request that is closest to the head right now. Distance 273 * We choose the request that is closest to the head right now. Distance
236 * behind the head is penalized and only allowed to a certain extent. 274 * behind the head is penalized and only allowed to a certain extent.
@@ -632,6 +670,7 @@ __cfq_set_active_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq)
632 cfqq->slice_left = 0; 670 cfqq->slice_left = 0;
633 cfq_clear_cfqq_must_alloc_slice(cfqq); 671 cfq_clear_cfqq_must_alloc_slice(cfqq);
634 cfq_clear_cfqq_fifo_expire(cfqq); 672 cfq_clear_cfqq_fifo_expire(cfqq);
673 cfq_mark_cfqq_slice_new(cfqq);
635 } 674 }
636 675
637 cfqd->active_queue = cfqq; 676 cfqd->active_queue = cfqq;
@@ -660,7 +699,7 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
660 * store what was left of this slice, if the queue idled out 699 * store what was left of this slice, if the queue idled out
661 * or was preempted 700 * or was preempted
662 */ 701 */
663 if (time_after(cfqq->slice_end, now)) 702 if (cfq_slice_used(cfqq))
664 cfqq->slice_left = cfqq->slice_end - now; 703 cfqq->slice_left = cfqq->slice_end - now;
665 else 704 else
666 cfqq->slice_left = 0; 705 cfqq->slice_left = 0;
@@ -858,27 +897,6 @@ static inline struct request *cfq_check_fifo(struct cfq_queue *cfqq)
858 return NULL; 897 return NULL;
859} 898}
860 899
861/*
862 * Scale schedule slice based on io priority. Use the sync time slice only
863 * if a queue is marked sync and has sync io queued. A sync queue with async
864 * io only, should not get full sync slice length.
865 */
866static inline int
867cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
868{
869 const int base_slice = cfqd->cfq_slice[cfq_cfqq_sync(cfqq)];
870
871 WARN_ON(cfqq->ioprio >= IOPRIO_BE_NR);
872
873 return base_slice + (base_slice/CFQ_SLICE_SCALE * (4 - cfqq->ioprio));
874}
875
876static inline void
877cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
878{
879 cfqq->slice_end = cfq_prio_to_slice(cfqd, cfqq) + jiffies;
880}
881
882static inline int 900static inline int
883cfq_prio_to_maxrq(struct cfq_data *cfqd, struct cfq_queue *cfqq) 901cfq_prio_to_maxrq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
884{ 902{
@@ -894,7 +912,6 @@ cfq_prio_to_maxrq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
894 */ 912 */
895static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd) 913static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)
896{ 914{
897 unsigned long now = jiffies;
898 struct cfq_queue *cfqq; 915 struct cfq_queue *cfqq;
899 916
900 cfqq = cfqd->active_queue; 917 cfqq = cfqd->active_queue;
@@ -904,7 +921,7 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)
904 /* 921 /*
905 * slice has expired 922 * slice has expired
906 */ 923 */
907 if (!cfq_cfqq_must_dispatch(cfqq) && time_after(now, cfqq->slice_end)) 924 if (!cfq_cfqq_must_dispatch(cfqq) && cfq_slice_used(cfqq))
908 goto expire; 925 goto expire;
909 926
910 /* 927 /*
@@ -913,7 +930,7 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)
913 */ 930 */
914 if (!RB_EMPTY_ROOT(&cfqq->sort_list)) 931 if (!RB_EMPTY_ROOT(&cfqq->sort_list))
915 goto keep_queue; 932 goto keep_queue;
916 else if (cfq_cfqq_dispatched(cfqq)) { 933 else if (cfq_cfqq_slice_new(cfqq) || cfq_cfqq_dispatched(cfqq)) {
917 cfqq = NULL; 934 cfqq = NULL;
918 goto keep_queue; 935 goto keep_queue;
919 } else if (cfq_cfqq_class_sync(cfqq)) { 936 } else if (cfq_cfqq_class_sync(cfqq)) {
@@ -965,20 +982,15 @@ __cfq_dispatch_requests(struct cfq_data *cfqd, struct cfq_queue *cfqq,
965 } while (dispatched < max_dispatch); 982 } while (dispatched < max_dispatch);
966 983
967 /* 984 /*
968 * if slice end isn't set yet, set it.
969 */
970 if (!cfqq->slice_end)
971 cfq_set_prio_slice(cfqd, cfqq);
972
973 /*
974 * expire an async queue immediately if it has used up its slice. idle 985 * expire an async queue immediately if it has used up its slice. idle
975 * queue always expire after 1 dispatch round. 986 * queue always expire after 1 dispatch round.
976 */ 987 */
977 if ((!cfq_cfqq_sync(cfqq) && 988 if ((!cfq_cfqq_sync(cfqq) &&
978 cfqd->dispatch_slice >= cfq_prio_to_maxrq(cfqd, cfqq)) || 989 cfqd->dispatch_slice >= cfq_prio_to_maxrq(cfqd, cfqq)) ||
979 cfq_class_idle(cfqq) || 990 cfq_class_idle(cfqq)) {
980 !cfq_cfqq_idle_window(cfqq)) 991 cfqq->slice_end = jiffies + 1;
981 cfq_slice_expired(cfqd, 0); 992 cfq_slice_expired(cfqd, 0);
993 }
982 994
983 return dispatched; 995 return dispatched;
984} 996}
@@ -1612,7 +1624,8 @@ static void cfq_preempt_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq)
1612 BUG_ON(!cfq_cfqq_on_rr(cfqq)); 1624 BUG_ON(!cfq_cfqq_on_rr(cfqq));
1613 list_move(&cfqq->cfq_list, &cfqd->cur_rr); 1625 list_move(&cfqq->cfq_list, &cfqd->cur_rr);
1614 1626
1615 cfqq->slice_end = cfqq->slice_left + jiffies; 1627 cfqq->slice_end = 0;
1628 cfq_mark_cfqq_slice_new(cfqq);
1616} 1629}
1617 1630
1618/* 1631/*
@@ -1722,7 +1735,11 @@ static void cfq_completed_request(request_queue_t *q, struct request *rq)
1722 * or if we want to idle in case it has no pending requests. 1735 * or if we want to idle in case it has no pending requests.
1723 */ 1736 */
1724 if (cfqd->active_queue == cfqq) { 1737 if (cfqd->active_queue == cfqq) {
1725 if (time_after(now, cfqq->slice_end)) 1738 if (cfq_cfqq_slice_new(cfqq)) {
1739 cfq_set_prio_slice(cfqd, cfqq);
1740 cfq_clear_cfqq_slice_new(cfqq);
1741 }
1742 if (cfq_slice_used(cfqq))
1726 cfq_slice_expired(cfqd, 0); 1743 cfq_slice_expired(cfqd, 0);
1727 else if (sync && RB_EMPTY_ROOT(&cfqq->sort_list)) { 1744 else if (sync && RB_EMPTY_ROOT(&cfqq->sort_list)) {
1728 if (!cfq_arm_slice_timer(cfqd, cfqq)) 1745 if (!cfq_arm_slice_timer(cfqd, cfqq))
@@ -1901,12 +1918,10 @@ static void cfq_idle_slice_timer(unsigned long data)
1901 spin_lock_irqsave(cfqd->queue->queue_lock, flags); 1918 spin_lock_irqsave(cfqd->queue->queue_lock, flags);
1902 1919
1903 if ((cfqq = cfqd->active_queue) != NULL) { 1920 if ((cfqq = cfqd->active_queue) != NULL) {
1904 unsigned long now = jiffies;
1905
1906 /* 1921 /*
1907 * expired 1922 * expired
1908 */ 1923 */
1909 if (time_after(now, cfqq->slice_end)) 1924 if (cfq_slice_used(cfqq))
1910 goto expire; 1925 goto expire;
1911 1926
1912 /* 1927 /*