diff options
Diffstat (limited to 'kernel/sched_fair.c')
-rw-r--r-- | kernel/sched_fair.c | 45 |
1 files changed, 36 insertions, 9 deletions
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index ee3771850aaf..892616bf2c77 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c | |||
@@ -194,6 +194,8 @@ __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) | |||
194 | update_load_add(&cfs_rq->load, se->load.weight); | 194 | update_load_add(&cfs_rq->load, se->load.weight); |
195 | cfs_rq->nr_running++; | 195 | cfs_rq->nr_running++; |
196 | se->on_rq = 1; | 196 | se->on_rq = 1; |
197 | |||
198 | schedstat_add(cfs_rq, wait_runtime, se->wait_runtime); | ||
197 | } | 199 | } |
198 | 200 | ||
199 | static inline void | 201 | static inline void |
@@ -205,6 +207,8 @@ __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) | |||
205 | update_load_sub(&cfs_rq->load, se->load.weight); | 207 | update_load_sub(&cfs_rq->load, se->load.weight); |
206 | cfs_rq->nr_running--; | 208 | cfs_rq->nr_running--; |
207 | se->on_rq = 0; | 209 | se->on_rq = 0; |
210 | |||
211 | schedstat_add(cfs_rq, wait_runtime, -se->wait_runtime); | ||
208 | } | 212 | } |
209 | 213 | ||
210 | static inline struct rb_node *first_fair(struct cfs_rq *cfs_rq) | 214 | static inline struct rb_node *first_fair(struct cfs_rq *cfs_rq) |
@@ -291,7 +295,7 @@ niced_granularity(struct sched_entity *curr, unsigned long granularity) | |||
291 | /* | 295 | /* |
292 | * It will always fit into 'long': | 296 | * It will always fit into 'long': |
293 | */ | 297 | */ |
294 | return (long) (tmp >> WMULT_SHIFT); | 298 | return (long) (tmp >> (WMULT_SHIFT-NICE_0_SHIFT)); |
295 | } | 299 | } |
296 | 300 | ||
297 | static inline void | 301 | static inline void |
@@ -354,7 +358,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr) | |||
354 | delta_fair = calc_delta_fair(delta_exec, lw); | 358 | delta_fair = calc_delta_fair(delta_exec, lw); |
355 | delta_mine = calc_delta_mine(delta_exec, curr->load.weight, lw); | 359 | delta_mine = calc_delta_mine(delta_exec, curr->load.weight, lw); |
356 | 360 | ||
357 | if (cfs_rq->sleeper_bonus > sysctl_sched_latency) { | 361 | if (cfs_rq->sleeper_bonus > sysctl_sched_min_granularity) { |
358 | delta = min((u64)delta_mine, cfs_rq->sleeper_bonus); | 362 | delta = min((u64)delta_mine, cfs_rq->sleeper_bonus); |
359 | delta = min(delta, (unsigned long)( | 363 | delta = min(delta, (unsigned long)( |
360 | (long)sysctl_sched_runtime_limit - curr->wait_runtime)); | 364 | (long)sysctl_sched_runtime_limit - curr->wait_runtime)); |
@@ -489,6 +493,9 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) | |||
489 | { | 493 | { |
490 | unsigned long delta_fair; | 494 | unsigned long delta_fair; |
491 | 495 | ||
496 | if (unlikely(!se->wait_start_fair)) | ||
497 | return; | ||
498 | |||
492 | delta_fair = (unsigned long)min((u64)(2*sysctl_sched_runtime_limit), | 499 | delta_fair = (unsigned long)min((u64)(2*sysctl_sched_runtime_limit), |
493 | (u64)(cfs_rq->fair_clock - se->wait_start_fair)); | 500 | (u64)(cfs_rq->fair_clock - se->wait_start_fair)); |
494 | 501 | ||
@@ -571,7 +578,6 @@ static void __enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) | |||
571 | 578 | ||
572 | prev_runtime = se->wait_runtime; | 579 | prev_runtime = se->wait_runtime; |
573 | __add_wait_runtime(cfs_rq, se, delta_fair); | 580 | __add_wait_runtime(cfs_rq, se, delta_fair); |
574 | schedstat_add(cfs_rq, wait_runtime, se->wait_runtime); | ||
575 | delta_fair = se->wait_runtime - prev_runtime; | 581 | delta_fair = se->wait_runtime - prev_runtime; |
576 | 582 | ||
577 | /* | 583 | /* |
@@ -659,7 +665,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) | |||
659 | if (tsk->state & TASK_UNINTERRUPTIBLE) | 665 | if (tsk->state & TASK_UNINTERRUPTIBLE) |
660 | se->block_start = rq_of(cfs_rq)->clock; | 666 | se->block_start = rq_of(cfs_rq)->clock; |
661 | } | 667 | } |
662 | cfs_rq->wait_runtime -= se->wait_runtime; | ||
663 | #endif | 668 | #endif |
664 | } | 669 | } |
665 | __dequeue_entity(cfs_rq, se); | 670 | __dequeue_entity(cfs_rq, se); |
@@ -673,11 +678,31 @@ __check_preempt_curr_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, | |||
673 | struct sched_entity *curr, unsigned long granularity) | 678 | struct sched_entity *curr, unsigned long granularity) |
674 | { | 679 | { |
675 | s64 __delta = curr->fair_key - se->fair_key; | 680 | s64 __delta = curr->fair_key - se->fair_key; |
681 | unsigned long ideal_runtime, delta_exec; | ||
682 | |||
683 | /* | ||
684 | * ideal_runtime is compared against sum_exec_runtime, which is | ||
685 | * walltime, hence do not scale. | ||
686 | */ | ||
687 | ideal_runtime = max(sysctl_sched_latency / cfs_rq->nr_running, | ||
688 | (unsigned long)sysctl_sched_min_granularity); | ||
689 | |||
690 | /* | ||
691 | * If we executed more than what the latency constraint suggests, | ||
692 | * reduce the rescheduling granularity. This way the total latency | ||
693 | * of how much a task is not scheduled converges to | ||
694 | * sysctl_sched_latency: | ||
695 | */ | ||
696 | delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; | ||
697 | if (delta_exec > ideal_runtime) | ||
698 | granularity = 0; | ||
676 | 699 | ||
677 | /* | 700 | /* |
678 | * Take scheduling granularity into account - do not | 701 | * Take scheduling granularity into account - do not |
679 | * preempt the current task unless the best task has | 702 | * preempt the current task unless the best task has |
680 | * a larger than sched_granularity fairness advantage: | 703 | * a larger than sched_granularity fairness advantage: |
704 | * | ||
705 | * scale granularity as key space is in fair_clock. | ||
681 | */ | 706 | */ |
682 | if (__delta > niced_granularity(curr, granularity)) | 707 | if (__delta > niced_granularity(curr, granularity)) |
683 | resched_task(rq_of(cfs_rq)->curr); | 708 | resched_task(rq_of(cfs_rq)->curr); |
@@ -696,6 +721,7 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) | |||
696 | update_stats_wait_end(cfs_rq, se); | 721 | update_stats_wait_end(cfs_rq, se); |
697 | update_stats_curr_start(cfs_rq, se); | 722 | update_stats_curr_start(cfs_rq, se); |
698 | set_cfs_rq_curr(cfs_rq, se); | 723 | set_cfs_rq_curr(cfs_rq, se); |
724 | se->prev_sum_exec_runtime = se->sum_exec_runtime; | ||
699 | } | 725 | } |
700 | 726 | ||
701 | static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) | 727 | static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) |
@@ -1076,31 +1102,32 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr) | |||
1076 | static void task_new_fair(struct rq *rq, struct task_struct *p) | 1102 | static void task_new_fair(struct rq *rq, struct task_struct *p) |
1077 | { | 1103 | { |
1078 | struct cfs_rq *cfs_rq = task_cfs_rq(p); | 1104 | struct cfs_rq *cfs_rq = task_cfs_rq(p); |
1079 | struct sched_entity *se = &p->se; | 1105 | struct sched_entity *se = &p->se, *curr = cfs_rq_curr(cfs_rq); |
1080 | 1106 | ||
1081 | sched_info_queued(p); | 1107 | sched_info_queued(p); |
1082 | 1108 | ||
1109 | update_curr(cfs_rq); | ||
1083 | update_stats_enqueue(cfs_rq, se); | 1110 | update_stats_enqueue(cfs_rq, se); |
1084 | /* | 1111 | /* |
1085 | * Child runs first: we let it run before the parent | 1112 | * Child runs first: we let it run before the parent |
1086 | * until it reschedules once. We set up the key so that | 1113 | * until it reschedules once. We set up the key so that |
1087 | * it will preempt the parent: | 1114 | * it will preempt the parent: |
1088 | */ | 1115 | */ |
1089 | p->se.fair_key = current->se.fair_key - | 1116 | se->fair_key = curr->fair_key - |
1090 | niced_granularity(&rq->curr->se, sched_granularity(cfs_rq)) - 1; | 1117 | niced_granularity(curr, sched_granularity(cfs_rq)) - 1; |
1091 | /* | 1118 | /* |
1092 | * The first wait is dominated by the child-runs-first logic, | 1119 | * The first wait is dominated by the child-runs-first logic, |
1093 | * so do not credit it with that waiting time yet: | 1120 | * so do not credit it with that waiting time yet: |
1094 | */ | 1121 | */ |
1095 | if (sysctl_sched_features & SCHED_FEAT_SKIP_INITIAL) | 1122 | if (sysctl_sched_features & SCHED_FEAT_SKIP_INITIAL) |
1096 | p->se.wait_start_fair = 0; | 1123 | se->wait_start_fair = 0; |
1097 | 1124 | ||
1098 | /* | 1125 | /* |
1099 | * The statistical average of wait_runtime is about | 1126 | * The statistical average of wait_runtime is about |
1100 | * -granularity/2, so initialize the task with that: | 1127 | * -granularity/2, so initialize the task with that: |
1101 | */ | 1128 | */ |
1102 | if (sysctl_sched_features & SCHED_FEAT_START_DEBIT) | 1129 | if (sysctl_sched_features & SCHED_FEAT_START_DEBIT) |
1103 | p->se.wait_runtime = -(sched_granularity(cfs_rq) / 2); | 1130 | se->wait_runtime = -(sched_granularity(cfs_rq) / 2); |
1104 | 1131 | ||
1105 | __enqueue_entity(cfs_rq, se); | 1132 | __enqueue_entity(cfs_rq, se); |
1106 | } | 1133 | } |