aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched_fair.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched_fair.c')
-rw-r--r--kernel/sched_fair.c45
1 files changed, 36 insertions, 9 deletions
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index ee3771850aaf..892616bf2c77 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -194,6 +194,8 @@ __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
194 update_load_add(&cfs_rq->load, se->load.weight); 194 update_load_add(&cfs_rq->load, se->load.weight);
195 cfs_rq->nr_running++; 195 cfs_rq->nr_running++;
196 se->on_rq = 1; 196 se->on_rq = 1;
197
198 schedstat_add(cfs_rq, wait_runtime, se->wait_runtime);
197} 199}
198 200
199static inline void 201static inline void
@@ -205,6 +207,8 @@ __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
205 update_load_sub(&cfs_rq->load, se->load.weight); 207 update_load_sub(&cfs_rq->load, se->load.weight);
206 cfs_rq->nr_running--; 208 cfs_rq->nr_running--;
207 se->on_rq = 0; 209 se->on_rq = 0;
210
211 schedstat_add(cfs_rq, wait_runtime, -se->wait_runtime);
208} 212}
209 213
210static inline struct rb_node *first_fair(struct cfs_rq *cfs_rq) 214static inline struct rb_node *first_fair(struct cfs_rq *cfs_rq)
@@ -291,7 +295,7 @@ niced_granularity(struct sched_entity *curr, unsigned long granularity)
291 /* 295 /*
292 * It will always fit into 'long': 296 * It will always fit into 'long':
293 */ 297 */
294 return (long) (tmp >> WMULT_SHIFT); 298 return (long) (tmp >> (WMULT_SHIFT-NICE_0_SHIFT));
295} 299}
296 300
297static inline void 301static inline void
@@ -354,7 +358,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr)
354 delta_fair = calc_delta_fair(delta_exec, lw); 358 delta_fair = calc_delta_fair(delta_exec, lw);
355 delta_mine = calc_delta_mine(delta_exec, curr->load.weight, lw); 359 delta_mine = calc_delta_mine(delta_exec, curr->load.weight, lw);
356 360
357 if (cfs_rq->sleeper_bonus > sysctl_sched_latency) { 361 if (cfs_rq->sleeper_bonus > sysctl_sched_min_granularity) {
358 delta = min((u64)delta_mine, cfs_rq->sleeper_bonus); 362 delta = min((u64)delta_mine, cfs_rq->sleeper_bonus);
359 delta = min(delta, (unsigned long)( 363 delta = min(delta, (unsigned long)(
360 (long)sysctl_sched_runtime_limit - curr->wait_runtime)); 364 (long)sysctl_sched_runtime_limit - curr->wait_runtime));
@@ -489,6 +493,9 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
489{ 493{
490 unsigned long delta_fair; 494 unsigned long delta_fair;
491 495
496 if (unlikely(!se->wait_start_fair))
497 return;
498
492 delta_fair = (unsigned long)min((u64)(2*sysctl_sched_runtime_limit), 499 delta_fair = (unsigned long)min((u64)(2*sysctl_sched_runtime_limit),
493 (u64)(cfs_rq->fair_clock - se->wait_start_fair)); 500 (u64)(cfs_rq->fair_clock - se->wait_start_fair));
494 501
@@ -571,7 +578,6 @@ static void __enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
571 578
572 prev_runtime = se->wait_runtime; 579 prev_runtime = se->wait_runtime;
573 __add_wait_runtime(cfs_rq, se, delta_fair); 580 __add_wait_runtime(cfs_rq, se, delta_fair);
574 schedstat_add(cfs_rq, wait_runtime, se->wait_runtime);
575 delta_fair = se->wait_runtime - prev_runtime; 581 delta_fair = se->wait_runtime - prev_runtime;
576 582
577 /* 583 /*
@@ -659,7 +665,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
659 if (tsk->state & TASK_UNINTERRUPTIBLE) 665 if (tsk->state & TASK_UNINTERRUPTIBLE)
660 se->block_start = rq_of(cfs_rq)->clock; 666 se->block_start = rq_of(cfs_rq)->clock;
661 } 667 }
662 cfs_rq->wait_runtime -= se->wait_runtime;
663#endif 668#endif
664 } 669 }
665 __dequeue_entity(cfs_rq, se); 670 __dequeue_entity(cfs_rq, se);
@@ -673,11 +678,31 @@ __check_preempt_curr_fair(struct cfs_rq *cfs_rq, struct sched_entity *se,
673 struct sched_entity *curr, unsigned long granularity) 678 struct sched_entity *curr, unsigned long granularity)
674{ 679{
675 s64 __delta = curr->fair_key - se->fair_key; 680 s64 __delta = curr->fair_key - se->fair_key;
681 unsigned long ideal_runtime, delta_exec;
682
683 /*
684 * ideal_runtime is compared against sum_exec_runtime, which is
685 * walltime, hence do not scale.
686 */
687 ideal_runtime = max(sysctl_sched_latency / cfs_rq->nr_running,
688 (unsigned long)sysctl_sched_min_granularity);
689
690 /*
691 * If we executed more than what the latency constraint suggests,
692 * reduce the rescheduling granularity. This way the total latency
693 * of how much a task is not scheduled converges to
694 * sysctl_sched_latency:
695 */
696 delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
697 if (delta_exec > ideal_runtime)
698 granularity = 0;
676 699
677 /* 700 /*
678 * Take scheduling granularity into account - do not 701 * Take scheduling granularity into account - do not
679 * preempt the current task unless the best task has 702 * preempt the current task unless the best task has
680 * a larger than sched_granularity fairness advantage: 703 * a larger than sched_granularity fairness advantage:
704 *
705 * scale granularity as key space is in fair_clock.
681 */ 706 */
682 if (__delta > niced_granularity(curr, granularity)) 707 if (__delta > niced_granularity(curr, granularity))
683 resched_task(rq_of(cfs_rq)->curr); 708 resched_task(rq_of(cfs_rq)->curr);
@@ -696,6 +721,7 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
696 update_stats_wait_end(cfs_rq, se); 721 update_stats_wait_end(cfs_rq, se);
697 update_stats_curr_start(cfs_rq, se); 722 update_stats_curr_start(cfs_rq, se);
698 set_cfs_rq_curr(cfs_rq, se); 723 set_cfs_rq_curr(cfs_rq, se);
724 se->prev_sum_exec_runtime = se->sum_exec_runtime;
699} 725}
700 726
701static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) 727static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq)
@@ -1076,31 +1102,32 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr)
1076static void task_new_fair(struct rq *rq, struct task_struct *p) 1102static void task_new_fair(struct rq *rq, struct task_struct *p)
1077{ 1103{
1078 struct cfs_rq *cfs_rq = task_cfs_rq(p); 1104 struct cfs_rq *cfs_rq = task_cfs_rq(p);
1079 struct sched_entity *se = &p->se; 1105 struct sched_entity *se = &p->se, *curr = cfs_rq_curr(cfs_rq);
1080 1106
1081 sched_info_queued(p); 1107 sched_info_queued(p);
1082 1108
1109 update_curr(cfs_rq);
1083 update_stats_enqueue(cfs_rq, se); 1110 update_stats_enqueue(cfs_rq, se);
1084 /* 1111 /*
1085 * Child runs first: we let it run before the parent 1112 * Child runs first: we let it run before the parent
1086 * until it reschedules once. We set up the key so that 1113 * until it reschedules once. We set up the key so that
1087 * it will preempt the parent: 1114 * it will preempt the parent:
1088 */ 1115 */
1089 p->se.fair_key = current->se.fair_key - 1116 se->fair_key = curr->fair_key -
1090 niced_granularity(&rq->curr->se, sched_granularity(cfs_rq)) - 1; 1117 niced_granularity(curr, sched_granularity(cfs_rq)) - 1;
1091 /* 1118 /*
1092 * The first wait is dominated by the child-runs-first logic, 1119 * The first wait is dominated by the child-runs-first logic,
1093 * so do not credit it with that waiting time yet: 1120 * so do not credit it with that waiting time yet:
1094 */ 1121 */
1095 if (sysctl_sched_features & SCHED_FEAT_SKIP_INITIAL) 1122 if (sysctl_sched_features & SCHED_FEAT_SKIP_INITIAL)
1096 p->se.wait_start_fair = 0; 1123 se->wait_start_fair = 0;
1097 1124
1098 /* 1125 /*
1099 * The statistical average of wait_runtime is about 1126 * The statistical average of wait_runtime is about
1100 * -granularity/2, so initialize the task with that: 1127 * -granularity/2, so initialize the task with that:
1101 */ 1128 */
1102 if (sysctl_sched_features & SCHED_FEAT_START_DEBIT) 1129 if (sysctl_sched_features & SCHED_FEAT_START_DEBIT)
1103 p->se.wait_runtime = -(sched_granularity(cfs_rq) / 2); 1130 se->wait_runtime = -(sched_granularity(cfs_rq) / 2);
1104 1131
1105 __enqueue_entity(cfs_rq, se); 1132 __enqueue_entity(cfs_rq, se);
1106} 1133}