aboutsummaryrefslogtreecommitdiffstats
path: root/block/cfq-iosched.c
diff options
context:
space:
mode:
Diffstat (limited to 'block/cfq-iosched.c')
-rw-r--r--block/cfq-iosched.c147
1 files changed, 68 insertions, 79 deletions
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 023f4e69a337..dee9d9378fee 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -19,7 +19,7 @@
19 * tunables 19 * tunables
20 */ 20 */
21/* max queue in one round of service */ 21/* max queue in one round of service */
22static const int cfq_quantum = 4; 22static const int cfq_quantum = 8;
23static const int cfq_fifo_expire[2] = { HZ / 4, HZ / 8 }; 23static const int cfq_fifo_expire[2] = { HZ / 4, HZ / 8 };
24/* maximum backwards seek, in KiB */ 24/* maximum backwards seek, in KiB */
25static const int cfq_back_max = 16 * 1024; 25static const int cfq_back_max = 16 * 1024;
@@ -46,8 +46,9 @@ static const int cfq_hist_divisor = 4;
46#define CFQ_HW_QUEUE_MIN (5) 46#define CFQ_HW_QUEUE_MIN (5)
47#define CFQ_SERVICE_SHIFT 12 47#define CFQ_SERVICE_SHIFT 12
48 48
49#define CFQQ_SEEK_THR 8 * 1024 49#define CFQQ_SEEK_THR (sector_t)(8 * 100)
50#define CFQQ_SEEKY(cfqq) ((cfqq)->seek_mean > CFQQ_SEEK_THR) 50#define CFQQ_SECT_THR_NONROT (sector_t)(2 * 32)
51#define CFQQ_SEEKY(cfqq) (hweight32(cfqq->seek_history) > 32/8)
51 52
52#define RQ_CIC(rq) \ 53#define RQ_CIC(rq) \
53 ((struct cfq_io_context *) (rq)->elevator_private) 54 ((struct cfq_io_context *) (rq)->elevator_private)
@@ -77,11 +78,12 @@ struct cfq_rb_root {
77 struct rb_root rb; 78 struct rb_root rb;
78 struct rb_node *left; 79 struct rb_node *left;
79 unsigned count; 80 unsigned count;
81 unsigned total_weight;
80 u64 min_vdisktime; 82 u64 min_vdisktime;
81 struct rb_node *active; 83 struct rb_node *active;
82 unsigned total_weight;
83}; 84};
84#define CFQ_RB_ROOT (struct cfq_rb_root) { RB_ROOT, NULL, 0, 0, } 85#define CFQ_RB_ROOT (struct cfq_rb_root) { .rb = RB_ROOT, .left = NULL, \
86 .count = 0, .min_vdisktime = 0, }
85 87
86/* 88/*
87 * Per process-grouping structure 89 * Per process-grouping structure
@@ -115,11 +117,11 @@ struct cfq_queue {
115 /* time when queue got scheduled in to dispatch first request. */ 117 /* time when queue got scheduled in to dispatch first request. */
116 unsigned long dispatch_start; 118 unsigned long dispatch_start;
117 unsigned int allocated_slice; 119 unsigned int allocated_slice;
120 unsigned int slice_dispatch;
118 /* time when first request from queue completed and slice started. */ 121 /* time when first request from queue completed and slice started. */
119 unsigned long slice_start; 122 unsigned long slice_start;
120 unsigned long slice_end; 123 unsigned long slice_end;
121 long slice_resid; 124 long slice_resid;
122 unsigned int slice_dispatch;
123 125
124 /* pending metadata requests */ 126 /* pending metadata requests */
125 int meta_pending; 127 int meta_pending;
@@ -130,13 +132,11 @@ struct cfq_queue {
130 unsigned short ioprio, org_ioprio; 132 unsigned short ioprio, org_ioprio;
131 unsigned short ioprio_class, org_ioprio_class; 133 unsigned short ioprio_class, org_ioprio_class;
132 134
133 unsigned int seek_samples;
134 u64 seek_total;
135 sector_t seek_mean;
136 sector_t last_request_pos;
137
138 pid_t pid; 135 pid_t pid;
139 136
137 u32 seek_history;
138 sector_t last_request_pos;
139
140 struct cfq_rb_root *service_tree; 140 struct cfq_rb_root *service_tree;
141 struct cfq_queue *new_cfqq; 141 struct cfq_queue *new_cfqq;
142 struct cfq_group *cfqg; 142 struct cfq_group *cfqg;
@@ -223,8 +223,8 @@ struct cfq_data {
223 223
224 unsigned int busy_queues; 224 unsigned int busy_queues;
225 225
226 int rq_in_driver[2]; 226 int rq_in_driver;
227 int sync_flight; 227 int rq_in_flight[2];
228 228
229 /* 229 /*
230 * queue-depth detection 230 * queue-depth detection
@@ -417,11 +417,6 @@ static struct cfq_queue *cfq_get_queue(struct cfq_data *, bool,
417static struct cfq_io_context *cfq_cic_lookup(struct cfq_data *, 417static struct cfq_io_context *cfq_cic_lookup(struct cfq_data *,
418 struct io_context *); 418 struct io_context *);
419 419
420static inline int rq_in_driver(struct cfq_data *cfqd)
421{
422 return cfqd->rq_in_driver[0] + cfqd->rq_in_driver[1];
423}
424
425static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_context *cic, 420static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_context *cic,
426 bool is_sync) 421 bool is_sync)
427{ 422{
@@ -951,10 +946,6 @@ cfq_find_alloc_cfqg(struct cfq_data *cfqd, struct cgroup *cgroup, int create)
951 struct backing_dev_info *bdi = &cfqd->queue->backing_dev_info; 946 struct backing_dev_info *bdi = &cfqd->queue->backing_dev_info;
952 unsigned int major, minor; 947 unsigned int major, minor;
953 948
954 /* Do we need to take this reference */
955 if (!blkiocg_css_tryget(blkcg))
956 return NULL;;
957
958 cfqg = cfqg_of_blkg(blkiocg_lookup_group(blkcg, key)); 949 cfqg = cfqg_of_blkg(blkiocg_lookup_group(blkcg, key));
959 if (cfqg || !create) 950 if (cfqg || !create)
960 goto done; 951 goto done;
@@ -985,7 +976,6 @@ cfq_find_alloc_cfqg(struct cfq_data *cfqd, struct cgroup *cgroup, int create)
985 hlist_add_head(&cfqg->cfqd_node, &cfqd->cfqg_list); 976 hlist_add_head(&cfqg->cfqd_node, &cfqd->cfqg_list);
986 977
987done: 978done:
988 blkiocg_css_put(blkcg);
989 return cfqg; 979 return cfqg;
990} 980}
991 981
@@ -1420,9 +1410,9 @@ static void cfq_activate_request(struct request_queue *q, struct request *rq)
1420{ 1410{
1421 struct cfq_data *cfqd = q->elevator->elevator_data; 1411 struct cfq_data *cfqd = q->elevator->elevator_data;
1422 1412
1423 cfqd->rq_in_driver[rq_is_sync(rq)]++; 1413 cfqd->rq_in_driver++;
1424 cfq_log_cfqq(cfqd, RQ_CFQQ(rq), "activate rq, drv=%d", 1414 cfq_log_cfqq(cfqd, RQ_CFQQ(rq), "activate rq, drv=%d",
1425 rq_in_driver(cfqd)); 1415 cfqd->rq_in_driver);
1426 1416
1427 cfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq); 1417 cfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq);
1428} 1418}
@@ -1430,12 +1420,11 @@ static void cfq_activate_request(struct request_queue *q, struct request *rq)
1430static void cfq_deactivate_request(struct request_queue *q, struct request *rq) 1420static void cfq_deactivate_request(struct request_queue *q, struct request *rq)
1431{ 1421{
1432 struct cfq_data *cfqd = q->elevator->elevator_data; 1422 struct cfq_data *cfqd = q->elevator->elevator_data;
1433 const int sync = rq_is_sync(rq);
1434 1423
1435 WARN_ON(!cfqd->rq_in_driver[sync]); 1424 WARN_ON(!cfqd->rq_in_driver);
1436 cfqd->rq_in_driver[sync]--; 1425 cfqd->rq_in_driver--;
1437 cfq_log_cfqq(cfqd, RQ_CFQQ(rq), "deactivate rq, drv=%d", 1426 cfq_log_cfqq(cfqd, RQ_CFQQ(rq), "deactivate rq, drv=%d",
1438 rq_in_driver(cfqd)); 1427 cfqd->rq_in_driver);
1439} 1428}
1440 1429
1441static void cfq_remove_request(struct request *rq) 1430static void cfq_remove_request(struct request *rq)
@@ -1673,16 +1662,7 @@ static inline sector_t cfq_dist_from_last(struct cfq_data *cfqd,
1673static inline int cfq_rq_close(struct cfq_data *cfqd, struct cfq_queue *cfqq, 1662static inline int cfq_rq_close(struct cfq_data *cfqd, struct cfq_queue *cfqq,
1674 struct request *rq, bool for_preempt) 1663 struct request *rq, bool for_preempt)
1675{ 1664{
1676 sector_t sdist = cfqq->seek_mean; 1665 return cfq_dist_from_last(cfqd, rq) <= CFQQ_SEEK_THR;
1677
1678 if (!sample_valid(cfqq->seek_samples))
1679 sdist = CFQQ_SEEK_THR;
1680
1681 /* if seek_mean is big, using it as close criteria is meaningless */
1682 if (sdist > CFQQ_SEEK_THR && !for_preempt)
1683 sdist = CFQQ_SEEK_THR;
1684
1685 return cfq_dist_from_last(cfqd, rq) <= sdist;
1686} 1666}
1687 1667
1688static struct cfq_queue *cfqq_close(struct cfq_data *cfqd, 1668static struct cfq_queue *cfqq_close(struct cfq_data *cfqd,
@@ -1878,8 +1858,7 @@ static void cfq_dispatch_insert(struct request_queue *q, struct request *rq)
1878 cfqq->dispatched++; 1858 cfqq->dispatched++;
1879 elv_dispatch_sort(q, rq); 1859 elv_dispatch_sort(q, rq);
1880 1860
1881 if (cfq_cfqq_sync(cfqq)) 1861 cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]++;
1882 cfqd->sync_flight++;
1883 cfqq->nr_sectors += blk_rq_sectors(rq); 1862 cfqq->nr_sectors += blk_rq_sectors(rq);
1884} 1863}
1885 1864
@@ -2219,6 +2198,19 @@ static int cfq_forced_dispatch(struct cfq_data *cfqd)
2219 return dispatched; 2198 return dispatched;
2220} 2199}
2221 2200
2201static inline bool cfq_slice_used_soon(struct cfq_data *cfqd,
2202 struct cfq_queue *cfqq)
2203{
2204 /* the queue hasn't finished any request, can't estimate */
2205 if (cfq_cfqq_slice_new(cfqq))
2206 return 1;
2207 if (time_after(jiffies + cfqd->cfq_slice_idle * cfqq->dispatched,
2208 cfqq->slice_end))
2209 return 1;
2210
2211 return 0;
2212}
2213
2222static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq) 2214static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq)
2223{ 2215{
2224 unsigned int max_dispatch; 2216 unsigned int max_dispatch;
@@ -2226,16 +2218,16 @@ static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq)
2226 /* 2218 /*
2227 * Drain async requests before we start sync IO 2219 * Drain async requests before we start sync IO
2228 */ 2220 */
2229 if (cfq_should_idle(cfqd, cfqq) && cfqd->rq_in_driver[BLK_RW_ASYNC]) 2221 if (cfq_should_idle(cfqd, cfqq) && cfqd->rq_in_flight[BLK_RW_ASYNC])
2230 return false; 2222 return false;
2231 2223
2232 /* 2224 /*
2233 * If this is an async queue and we have sync IO in flight, let it wait 2225 * If this is an async queue and we have sync IO in flight, let it wait
2234 */ 2226 */
2235 if (cfqd->sync_flight && !cfq_cfqq_sync(cfqq)) 2227 if (cfqd->rq_in_flight[BLK_RW_SYNC] && !cfq_cfqq_sync(cfqq))
2236 return false; 2228 return false;
2237 2229
2238 max_dispatch = cfqd->cfq_quantum; 2230 max_dispatch = max_t(unsigned int, cfqd->cfq_quantum / 2, 1);
2239 if (cfq_class_idle(cfqq)) 2231 if (cfq_class_idle(cfqq))
2240 max_dispatch = 1; 2232 max_dispatch = 1;
2241 2233
@@ -2252,13 +2244,22 @@ static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq)
2252 /* 2244 /*
2253 * We have other queues, don't allow more IO from this one 2245 * We have other queues, don't allow more IO from this one
2254 */ 2246 */
2255 if (cfqd->busy_queues > 1) 2247 if (cfqd->busy_queues > 1 && cfq_slice_used_soon(cfqd, cfqq))
2256 return false; 2248 return false;
2257 2249
2258 /* 2250 /*
2259 * Sole queue user, no limit 2251 * Sole queue user, no limit
2260 */ 2252 */
2261 max_dispatch = -1; 2253 if (cfqd->busy_queues == 1)
2254 max_dispatch = -1;
2255 else
2256 /*
2257 * Normally we start throttling cfqq when cfq_quantum/2
2258 * requests have been dispatched. But we can drive
2259 * deeper queue depths at the beginning of slice
2260 * subjected to upper limit of cfq_quantum.
2261 * */
2262 max_dispatch = cfqd->cfq_quantum;
2262 } 2263 }
2263 2264
2264 /* 2265 /*
@@ -2980,30 +2981,20 @@ static void
2980cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_queue *cfqq, 2981cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_queue *cfqq,
2981 struct request *rq) 2982 struct request *rq)
2982{ 2983{
2983 sector_t sdist; 2984 sector_t sdist = 0;
2984 u64 total; 2985 sector_t n_sec = blk_rq_sectors(rq);
2986 if (cfqq->last_request_pos) {
2987 if (cfqq->last_request_pos < blk_rq_pos(rq))
2988 sdist = blk_rq_pos(rq) - cfqq->last_request_pos;
2989 else
2990 sdist = cfqq->last_request_pos - blk_rq_pos(rq);
2991 }
2985 2992
2986 if (!cfqq->last_request_pos) 2993 cfqq->seek_history <<= 1;
2987 sdist = 0; 2994 if (blk_queue_nonrot(cfqd->queue))
2988 else if (cfqq->last_request_pos < blk_rq_pos(rq)) 2995 cfqq->seek_history |= (n_sec < CFQQ_SECT_THR_NONROT);
2989 sdist = blk_rq_pos(rq) - cfqq->last_request_pos;
2990 else 2996 else
2991 sdist = cfqq->last_request_pos - blk_rq_pos(rq); 2997 cfqq->seek_history |= (sdist > CFQQ_SEEK_THR);
2992
2993 /*
2994 * Don't allow the seek distance to get too large from the
2995 * odd fragment, pagein, etc
2996 */
2997 if (cfqq->seek_samples <= 60) /* second&third seek */
2998 sdist = min(sdist, (cfqq->seek_mean * 4) + 2*1024*1024);
2999 else
3000 sdist = min(sdist, (cfqq->seek_mean * 4) + 2*1024*64);
3001
3002 cfqq->seek_samples = (7*cfqq->seek_samples + 256) / 8;
3003 cfqq->seek_total = (7*cfqq->seek_total + (u64)256*sdist) / 8;
3004 total = cfqq->seek_total + (cfqq->seek_samples/2);
3005 do_div(total, cfqq->seek_samples);
3006 cfqq->seek_mean = (sector_t)total;
3007} 2998}
3008 2999
3009/* 3000/*
@@ -3028,8 +3019,7 @@ cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq,
3028 cfq_mark_cfqq_deep(cfqq); 3019 cfq_mark_cfqq_deep(cfqq);
3029 3020
3030 if (!atomic_read(&cic->ioc->nr_tasks) || !cfqd->cfq_slice_idle || 3021 if (!atomic_read(&cic->ioc->nr_tasks) || !cfqd->cfq_slice_idle ||
3031 (!cfq_cfqq_deep(cfqq) && sample_valid(cfqq->seek_samples) 3022 (!cfq_cfqq_deep(cfqq) && CFQQ_SEEKY(cfqq)))
3032 && CFQQ_SEEKY(cfqq)))
3033 enable_idle = 0; 3023 enable_idle = 0;
3034 else if (sample_valid(cic->ttime_samples)) { 3024 else if (sample_valid(cic->ttime_samples)) {
3035 if (cic->ttime_mean > cfqd->cfq_slice_idle) 3025 if (cic->ttime_mean > cfqd->cfq_slice_idle)
@@ -3215,14 +3205,14 @@ static void cfq_update_hw_tag(struct cfq_data *cfqd)
3215{ 3205{
3216 struct cfq_queue *cfqq = cfqd->active_queue; 3206 struct cfq_queue *cfqq = cfqd->active_queue;
3217 3207
3218 if (rq_in_driver(cfqd) > cfqd->hw_tag_est_depth) 3208 if (cfqd->rq_in_driver > cfqd->hw_tag_est_depth)
3219 cfqd->hw_tag_est_depth = rq_in_driver(cfqd); 3209 cfqd->hw_tag_est_depth = cfqd->rq_in_driver;
3220 3210
3221 if (cfqd->hw_tag == 1) 3211 if (cfqd->hw_tag == 1)
3222 return; 3212 return;
3223 3213
3224 if (cfqd->rq_queued <= CFQ_HW_QUEUE_MIN && 3214 if (cfqd->rq_queued <= CFQ_HW_QUEUE_MIN &&
3225 rq_in_driver(cfqd) <= CFQ_HW_QUEUE_MIN) 3215 cfqd->rq_in_driver <= CFQ_HW_QUEUE_MIN)
3226 return; 3216 return;
3227 3217
3228 /* 3218 /*
@@ -3232,7 +3222,7 @@ static void cfq_update_hw_tag(struct cfq_data *cfqd)
3232 */ 3222 */
3233 if (cfqq && cfq_cfqq_idle_window(cfqq) && 3223 if (cfqq && cfq_cfqq_idle_window(cfqq) &&
3234 cfqq->dispatched + cfqq->queued[0] + cfqq->queued[1] < 3224 cfqq->dispatched + cfqq->queued[0] + cfqq->queued[1] <
3235 CFQ_HW_QUEUE_MIN && rq_in_driver(cfqd) < CFQ_HW_QUEUE_MIN) 3225 CFQ_HW_QUEUE_MIN && cfqd->rq_in_driver < CFQ_HW_QUEUE_MIN)
3236 return; 3226 return;
3237 3227
3238 if (cfqd->hw_tag_samples++ < 50) 3228 if (cfqd->hw_tag_samples++ < 50)
@@ -3285,13 +3275,12 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
3285 3275
3286 cfq_update_hw_tag(cfqd); 3276 cfq_update_hw_tag(cfqd);
3287 3277
3288 WARN_ON(!cfqd->rq_in_driver[sync]); 3278 WARN_ON(!cfqd->rq_in_driver);
3289 WARN_ON(!cfqq->dispatched); 3279 WARN_ON(!cfqq->dispatched);
3290 cfqd->rq_in_driver[sync]--; 3280 cfqd->rq_in_driver--;
3291 cfqq->dispatched--; 3281 cfqq->dispatched--;
3292 3282
3293 if (cfq_cfqq_sync(cfqq)) 3283 cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--;
3294 cfqd->sync_flight--;
3295 3284
3296 if (sync) { 3285 if (sync) {
3297 RQ_CIC(rq)->last_end_request = now; 3286 RQ_CIC(rq)->last_end_request = now;
@@ -3345,7 +3334,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
3345 } 3334 }
3346 } 3335 }
3347 3336
3348 if (!rq_in_driver(cfqd)) 3337 if (!cfqd->rq_in_driver)
3349 cfq_schedule_dispatch(cfqd); 3338 cfq_schedule_dispatch(cfqd);
3350} 3339}
3351 3340