aboutsummaryrefslogtreecommitdiffstats
path: root/block/cfq-iosched.c
diff options
context:
space:
mode:
Diffstat (limited to 'block/cfq-iosched.c')
-rw-r--r--block/cfq-iosched.c41
1 files changed, 25 insertions, 16 deletions
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index a4d17265411e..fab2be0fa215 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -140,9 +140,9 @@ struct cfq_queue {
140 * IDLE is handled separately, so it has negative index 140 * IDLE is handled separately, so it has negative index
141 */ 141 */
142enum wl_prio_t { 142enum wl_prio_t {
143 IDLE_WORKLOAD = -1,
144 BE_WORKLOAD = 0, 143 BE_WORKLOAD = 0,
145 RT_WORKLOAD = 1 144 RT_WORKLOAD = 1,
145 IDLE_WORKLOAD = 2,
146}; 146};
147 147
148/* 148/*
@@ -303,6 +303,17 @@ CFQ_CFQQ_FNS(deep);
303#define cfq_log(cfqd, fmt, args...) \ 303#define cfq_log(cfqd, fmt, args...) \
304 blk_add_trace_msg((cfqd)->queue, "cfq " fmt, ##args) 304 blk_add_trace_msg((cfqd)->queue, "cfq " fmt, ##args)
305 305
306/* Traverses through cfq group service trees */
307#define for_each_cfqg_st(cfqg, i, j, st) \
308 for (i = 0; i <= IDLE_WORKLOAD; i++) \
309 for (j = 0, st = i < IDLE_WORKLOAD ? &cfqg->service_trees[i][j]\
310 : &cfqg->service_tree_idle; \
311 (i < IDLE_WORKLOAD && j <= SYNC_WORKLOAD) || \
312 (i == IDLE_WORKLOAD && j == 0); \
313 j++, st = i < IDLE_WORKLOAD ? \
314 &cfqg->service_trees[i][j]: NULL) \
315
316
306static inline enum wl_prio_t cfqq_prio(struct cfq_queue *cfqq) 317static inline enum wl_prio_t cfqq_prio(struct cfq_queue *cfqq)
307{ 318{
308 if (cfq_class_idle(cfqq)) 319 if (cfq_class_idle(cfqq))
@@ -565,6 +576,10 @@ cfq_choose_req(struct cfq_data *cfqd, struct request *rq1, struct request *rq2,
565 */ 576 */
566static struct cfq_queue *cfq_rb_first(struct cfq_rb_root *root) 577static struct cfq_queue *cfq_rb_first(struct cfq_rb_root *root)
567{ 578{
579 /* Service tree is empty */
580 if (!root->count)
581 return NULL;
582
568 if (!root->left) 583 if (!root->left)
569 root->left = rb_first(&root->rb); 584 root->left = rb_first(&root->rb);
570 585
@@ -1587,18 +1602,14 @@ static int cfq_forced_dispatch(struct cfq_data *cfqd)
1587 int dispatched = 0; 1602 int dispatched = 0;
1588 int i, j; 1603 int i, j;
1589 struct cfq_group *cfqg = &cfqd->root_group; 1604 struct cfq_group *cfqg = &cfqd->root_group;
1605 struct cfq_rb_root *st;
1590 1606
1591 for (i = 0; i < 2; ++i) 1607 for_each_cfqg_st(cfqg, i, j, st) {
1592 for (j = 0; j < 3; ++j) 1608 while ((cfqq = cfq_rb_first(st)) != NULL)
1593 while ((cfqq = cfq_rb_first(&cfqg->service_trees[i][j])) 1609 dispatched += __cfq_forced_dispatch_cfqq(cfqq);
1594 != NULL) 1610 }
1595 dispatched += __cfq_forced_dispatch_cfqq(cfqq);
1596
1597 while ((cfqq = cfq_rb_first(&cfqg->service_tree_idle)) != NULL)
1598 dispatched += __cfq_forced_dispatch_cfqq(cfqq);
1599 1611
1600 cfq_slice_expired(cfqd, 0); 1612 cfq_slice_expired(cfqd, 0);
1601
1602 BUG_ON(cfqd->busy_queues); 1613 BUG_ON(cfqd->busy_queues);
1603 1614
1604 cfq_log(cfqd, "forced_dispatch=%d", dispatched); 1615 cfq_log(cfqd, "forced_dispatch=%d", dispatched);
@@ -2969,6 +2980,7 @@ static void *cfq_init_queue(struct request_queue *q)
2969 struct cfq_data *cfqd; 2980 struct cfq_data *cfqd;
2970 int i, j; 2981 int i, j;
2971 struct cfq_group *cfqg; 2982 struct cfq_group *cfqg;
2983 struct cfq_rb_root *st;
2972 2984
2973 cfqd = kmalloc_node(sizeof(*cfqd), GFP_KERNEL | __GFP_ZERO, q->node); 2985 cfqd = kmalloc_node(sizeof(*cfqd), GFP_KERNEL | __GFP_ZERO, q->node);
2974 if (!cfqd) 2986 if (!cfqd)
@@ -2976,11 +2988,8 @@ static void *cfq_init_queue(struct request_queue *q)
2976 2988
2977 /* Init root group */ 2989 /* Init root group */
2978 cfqg = &cfqd->root_group; 2990 cfqg = &cfqd->root_group;
2979 2991 for_each_cfqg_st(cfqg, i, j, st)
2980 for (i = 0; i < 2; ++i) 2992 *st = CFQ_RB_ROOT;
2981 for (j = 0; j < 3; ++j)
2982 cfqg->service_trees[i][j] = CFQ_RB_ROOT;
2983 cfqg->service_tree_idle = CFQ_RB_ROOT;
2984 2993
2985 /* 2994 /*
2986 * Not strictly needed (since RB_ROOT just clears the node and we 2995 * Not strictly needed (since RB_ROOT just clears the node and we