aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
authorPeter Zijlstra <a.p.zijlstra@chello.nl>2008-06-27 07:41:11 -0400
committerIngo Molnar <mingo@elte.hu>2008-06-27 08:31:27 -0400
commita7be37ac8e1565e00880531f4e2aff421a21c803 (patch)
tree82eb64ba5b221c3abea6d6f56c692841faea813a /kernel
parentbf647b62fdb948e757a7b4d18d4f16e3c763b1d1 (diff)
sched: revert the revert of: weight calculations
Try again.. initial commit: 8f1bc385cfbab474db6c27b5af1e439614f3025c revert: f9305d4a0968201b2818dbed0dc8cb0d4ee7aeb3 Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> Cc: Mike Galbraith <efault@gmx.de> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'kernel')
-rw-r--r--kernel/sched.c9
-rw-r--r--kernel/sched_fair.c105
-rw-r--r--kernel/sched_features.h1
3 files changed, 76 insertions, 39 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index c51d9fae8cd8..f653af684fb3 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -1342,6 +1342,9 @@ static void __resched_task(struct task_struct *p, int tif_bit)
1342 */ 1342 */
1343#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y)) 1343#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
1344 1344
1345/*
1346 * delta *= weight / lw
1347 */
1345static unsigned long 1348static unsigned long
1346calc_delta_mine(unsigned long delta_exec, unsigned long weight, 1349calc_delta_mine(unsigned long delta_exec, unsigned long weight,
1347 struct load_weight *lw) 1350 struct load_weight *lw)
@@ -1369,12 +1372,6 @@ calc_delta_mine(unsigned long delta_exec, unsigned long weight,
1369 return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX); 1372 return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
1370} 1373}
1371 1374
1372static inline unsigned long
1373calc_delta_fair(unsigned long delta_exec, struct load_weight *lw)
1374{
1375 return calc_delta_mine(delta_exec, NICE_0_LOAD, lw);
1376}
1377
1378static inline void update_load_add(struct load_weight *lw, unsigned long inc) 1375static inline void update_load_add(struct load_weight *lw, unsigned long inc)
1379{ 1376{
1380 lw->weight += inc; 1377 lw->weight += inc;
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 1fe4c65a8170..496500988ce5 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -334,6 +334,34 @@ int sched_nr_latency_handler(struct ctl_table *table, int write,
334#endif 334#endif
335 335
336/* 336/*
337 * delta *= w / rw
338 */
339static inline unsigned long
340calc_delta_weight(unsigned long delta, struct sched_entity *se)
341{
342 for_each_sched_entity(se) {
343 delta = calc_delta_mine(delta,
344 se->load.weight, &cfs_rq_of(se)->load);
345 }
346
347 return delta;
348}
349
350/*
351 * delta *= rw / w
352 */
353static inline unsigned long
354calc_delta_fair(unsigned long delta, struct sched_entity *se)
355{
356 for_each_sched_entity(se) {
357 delta = calc_delta_mine(delta,
358 cfs_rq_of(se)->load.weight, &se->load);
359 }
360
361 return delta;
362}
363
364/*
337 * The idea is to set a period in which each task runs once. 365 * The idea is to set a period in which each task runs once.
338 * 366 *
339 * When there are too many tasks (sysctl_sched_nr_latency) we have to stretch 367 * When there are too many tasks (sysctl_sched_nr_latency) we have to stretch
@@ -362,47 +390,54 @@ static u64 __sched_period(unsigned long nr_running)
362 */ 390 */
363static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 391static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
364{ 392{
365 u64 slice = __sched_period(cfs_rq->nr_running); 393 return calc_delta_weight(__sched_period(cfs_rq->nr_running), se);
366
367 for_each_sched_entity(se) {
368 cfs_rq = cfs_rq_of(se);
369
370 slice *= se->load.weight;
371 do_div(slice, cfs_rq->load.weight);
372 }
373
374
375 return slice;
376} 394}
377 395
378/* 396/*
379 * We calculate the vruntime slice of a to be inserted task 397 * We calculate the vruntime slice of a to be inserted task
380 * 398 *
381 * vs = s/w = p/rw 399 * vs = s*rw/w = p
382 */ 400 */
383static u64 sched_vslice_add(struct cfs_rq *cfs_rq, struct sched_entity *se) 401static u64 sched_vslice_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
384{ 402{
385 unsigned long nr_running = cfs_rq->nr_running; 403 unsigned long nr_running = cfs_rq->nr_running;
386 unsigned long weight;
387 u64 vslice;
388 404
389 if (!se->on_rq) 405 if (!se->on_rq)
390 nr_running++; 406 nr_running++;
391 407
392 vslice = __sched_period(nr_running); 408 return __sched_period(nr_running);
409}
410
411/*
412 * The goal of calc_delta_asym() is to be asymmetrically around NICE_0_LOAD, in
413 * that it favours >=0 over <0.
414 *
415 * -20 |
416 * |
417 * 0 --------+-------
418 * .'
419 * 19 .'
420 *
421 */
422static unsigned long
423calc_delta_asym(unsigned long delta, struct sched_entity *se)
424{
425 struct load_weight lw = {
426 .weight = NICE_0_LOAD,
427 .inv_weight = 1UL << (WMULT_SHIFT-NICE_0_SHIFT)
428 };
393 429
394 for_each_sched_entity(se) { 430 for_each_sched_entity(se) {
395 cfs_rq = cfs_rq_of(se); 431 struct load_weight *se_lw = &se->load;
396 432
397 weight = cfs_rq->load.weight; 433 if (se->load.weight < NICE_0_LOAD)
398 if (!se->on_rq) 434 se_lw = &lw;
399 weight += se->load.weight;
400 435
401 vslice *= NICE_0_LOAD; 436 delta = calc_delta_mine(delta,
402 do_div(vslice, weight); 437 cfs_rq_of(se)->load.weight, se_lw);
403 } 438 }
404 439
405 return vslice; 440 return delta;
406} 441}
407 442
408/* 443/*
@@ -419,11 +454,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr,
419 454
420 curr->sum_exec_runtime += delta_exec; 455 curr->sum_exec_runtime += delta_exec;
421 schedstat_add(cfs_rq, exec_clock, delta_exec); 456 schedstat_add(cfs_rq, exec_clock, delta_exec);
422 delta_exec_weighted = delta_exec; 457 delta_exec_weighted = calc_delta_fair(delta_exec, curr);
423 if (unlikely(curr->load.weight != NICE_0_LOAD)) {
424 delta_exec_weighted = calc_delta_fair(delta_exec_weighted,
425 &curr->load);
426 }
427 curr->vruntime += delta_exec_weighted; 458 curr->vruntime += delta_exec_weighted;
428} 459}
429 460
@@ -609,8 +640,17 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
609 640
610 if (!initial) { 641 if (!initial) {
611 /* sleeps upto a single latency don't count. */ 642 /* sleeps upto a single latency don't count. */
612 if (sched_feat(NEW_FAIR_SLEEPERS)) 643 if (sched_feat(NEW_FAIR_SLEEPERS)) {
613 vruntime -= sysctl_sched_latency; 644 unsigned long thresh = sysctl_sched_latency;
645
646 /*
647 * convert the sleeper threshold into virtual time
648 */
649 if (sched_feat(NORMALIZED_SLEEPER))
650 thresh = calc_delta_fair(thresh, se);
651
652 vruntime -= thresh;
653 }
614 654
615 /* ensure we never gain time by being placed backwards. */ 655 /* ensure we never gain time by being placed backwards. */
616 vruntime = max_vruntime(se->vruntime, vruntime); 656 vruntime = max_vruntime(se->vruntime, vruntime);
@@ -1111,11 +1151,10 @@ static unsigned long wakeup_gran(struct sched_entity *se)
1111 unsigned long gran = sysctl_sched_wakeup_granularity; 1151 unsigned long gran = sysctl_sched_wakeup_granularity;
1112 1152
1113 /* 1153 /*
1114 * More easily preempt - nice tasks, while not making 1154 * More easily preempt - nice tasks, while not making it harder for
1115 * it harder for + nice tasks. 1155 * + nice tasks.
1116 */ 1156 */
1117 if (unlikely(se->load.weight > NICE_0_LOAD)) 1157 gran = calc_delta_asym(sysctl_sched_wakeup_granularity, se);
1118 gran = calc_delta_fair(gran, &se->load);
1119 1158
1120 return gran; 1159 return gran;
1121} 1160}
diff --git a/kernel/sched_features.h b/kernel/sched_features.h
index 62b39ca92ebd..afa549166d8d 100644
--- a/kernel/sched_features.h
+++ b/kernel/sched_features.h
@@ -1,4 +1,5 @@
1SCHED_FEAT(NEW_FAIR_SLEEPERS, 1) 1SCHED_FEAT(NEW_FAIR_SLEEPERS, 1)
2SCHED_FEAT(NORMALIZED_SLEEPER, 1)
2SCHED_FEAT(WAKEUP_PREEMPT, 1) 3SCHED_FEAT(WAKEUP_PREEMPT, 1)
3SCHED_FEAT(START_DEBIT, 1) 4SCHED_FEAT(START_DEBIT, 1)
4SCHED_FEAT(AFFINE_WAKEUPS, 1) 5SCHED_FEAT(AFFINE_WAKEUPS, 1)