diff options
Diffstat (limited to 'kernel')
-rw-r--r-- | kernel/sched/core.c | 6 | ||||
-rw-r--r-- | kernel/sched/fair.c | 102 | ||||
-rw-r--r-- | kernel/sched/sched.h | 2 |
3 files changed, 55 insertions, 55 deletions
diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 2e1fb0902200..07bc78a50329 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c | |||
@@ -5764,7 +5764,7 @@ static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg) | |||
5764 | /* | 5764 | /* |
5765 | * build_sched_groups will build a circular linked list of the groups | 5765 | * build_sched_groups will build a circular linked list of the groups |
5766 | * covered by the given span, and will set each group's ->cpumask correctly, | 5766 | * covered by the given span, and will set each group's ->cpumask correctly, |
5767 | * and ->cpu_power to 0. | 5767 | * and ->cpu_capacity to 0. |
5768 | * | 5768 | * |
5769 | * Assumes the sched_domain tree is fully constructed | 5769 | * Assumes the sched_domain tree is fully constructed |
5770 | */ | 5770 | */ |
@@ -6471,7 +6471,7 @@ static int build_sched_domains(const struct cpumask *cpu_map, | |||
6471 | } | 6471 | } |
6472 | } | 6472 | } |
6473 | 6473 | ||
6474 | /* Calculate CPU power for physical packages and nodes */ | 6474 | /* Calculate CPU capacity for physical packages and nodes */ |
6475 | for (i = nr_cpumask_bits-1; i >= 0; i--) { | 6475 | for (i = nr_cpumask_bits-1; i >= 0; i--) { |
6476 | if (!cpumask_test_cpu(i, cpu_map)) | 6476 | if (!cpumask_test_cpu(i, cpu_map)) |
6477 | continue; | 6477 | continue; |
@@ -6921,7 +6921,7 @@ void __init sched_init(void) | |||
6921 | #ifdef CONFIG_SMP | 6921 | #ifdef CONFIG_SMP |
6922 | rq->sd = NULL; | 6922 | rq->sd = NULL; |
6923 | rq->rd = NULL; | 6923 | rq->rd = NULL; |
6924 | rq->cpu_power = SCHED_POWER_SCALE; | 6924 | rq->cpu_capacity = SCHED_POWER_SCALE; |
6925 | rq->post_schedule = 0; | 6925 | rq->post_schedule = 0; |
6926 | rq->active_balance = 0; | 6926 | rq->active_balance = 0; |
6927 | rq->next_balance = jiffies; | 6927 | rq->next_balance = jiffies; |
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 36bd4d23fca8..58684f684fa8 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c | |||
@@ -1017,7 +1017,7 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page * page, | |||
1017 | static unsigned long weighted_cpuload(const int cpu); | 1017 | static unsigned long weighted_cpuload(const int cpu); |
1018 | static unsigned long source_load(int cpu, int type); | 1018 | static unsigned long source_load(int cpu, int type); |
1019 | static unsigned long target_load(int cpu, int type); | 1019 | static unsigned long target_load(int cpu, int type); |
1020 | static unsigned long power_of(int cpu); | 1020 | static unsigned long capacity_of(int cpu); |
1021 | static long effective_load(struct task_group *tg, int cpu, long wl, long wg); | 1021 | static long effective_load(struct task_group *tg, int cpu, long wl, long wg); |
1022 | 1022 | ||
1023 | /* Cached statistics for all CPUs within a node */ | 1023 | /* Cached statistics for all CPUs within a node */ |
@@ -1046,7 +1046,7 @@ static void update_numa_stats(struct numa_stats *ns, int nid) | |||
1046 | 1046 | ||
1047 | ns->nr_running += rq->nr_running; | 1047 | ns->nr_running += rq->nr_running; |
1048 | ns->load += weighted_cpuload(cpu); | 1048 | ns->load += weighted_cpuload(cpu); |
1049 | ns->compute_capacity += power_of(cpu); | 1049 | ns->compute_capacity += capacity_of(cpu); |
1050 | 1050 | ||
1051 | cpus++; | 1051 | cpus++; |
1052 | } | 1052 | } |
@@ -1214,7 +1214,7 @@ balance: | |||
1214 | orig_dst_load = env->dst_stats.load; | 1214 | orig_dst_load = env->dst_stats.load; |
1215 | orig_src_load = env->src_stats.load; | 1215 | orig_src_load = env->src_stats.load; |
1216 | 1216 | ||
1217 | /* XXX missing power terms */ | 1217 | /* XXX missing capacity terms */ |
1218 | load = task_h_load(env->p); | 1218 | load = task_h_load(env->p); |
1219 | dst_load = orig_dst_load + load; | 1219 | dst_load = orig_dst_load + load; |
1220 | src_load = orig_src_load - load; | 1220 | src_load = orig_src_load - load; |
@@ -4043,9 +4043,9 @@ static unsigned long target_load(int cpu, int type) | |||
4043 | return max(rq->cpu_load[type-1], total); | 4043 | return max(rq->cpu_load[type-1], total); |
4044 | } | 4044 | } |
4045 | 4045 | ||
4046 | static unsigned long power_of(int cpu) | 4046 | static unsigned long capacity_of(int cpu) |
4047 | { | 4047 | { |
4048 | return cpu_rq(cpu)->cpu_power; | 4048 | return cpu_rq(cpu)->cpu_capacity; |
4049 | } | 4049 | } |
4050 | 4050 | ||
4051 | static unsigned long cpu_avg_load_per_task(int cpu) | 4051 | static unsigned long cpu_avg_load_per_task(int cpu) |
@@ -4288,12 +4288,12 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync) | |||
4288 | s64 this_eff_load, prev_eff_load; | 4288 | s64 this_eff_load, prev_eff_load; |
4289 | 4289 | ||
4290 | this_eff_load = 100; | 4290 | this_eff_load = 100; |
4291 | this_eff_load *= power_of(prev_cpu); | 4291 | this_eff_load *= capacity_of(prev_cpu); |
4292 | this_eff_load *= this_load + | 4292 | this_eff_load *= this_load + |
4293 | effective_load(tg, this_cpu, weight, weight); | 4293 | effective_load(tg, this_cpu, weight, weight); |
4294 | 4294 | ||
4295 | prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2; | 4295 | prev_eff_load = 100 + (sd->imbalance_pct - 100) / 2; |
4296 | prev_eff_load *= power_of(this_cpu); | 4296 | prev_eff_load *= capacity_of(this_cpu); |
4297 | prev_eff_load *= load + effective_load(tg, prev_cpu, 0, weight); | 4297 | prev_eff_load *= load + effective_load(tg, prev_cpu, 0, weight); |
4298 | 4298 | ||
4299 | balanced = this_eff_load <= prev_eff_load; | 4299 | balanced = this_eff_load <= prev_eff_load; |
@@ -4950,14 +4950,14 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p, bool preemp | |||
4950 | * | 4950 | * |
4951 | * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) | 4951 | * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) |
4952 | * | 4952 | * |
4953 | * P_i is the cpu power (or compute capacity) of cpu i, typically it is the | 4953 | * C_i is the compute capacity of cpu i, typically it is the |
4954 | * fraction of 'recent' time available for SCHED_OTHER task execution. But it | 4954 | * fraction of 'recent' time available for SCHED_OTHER task execution. But it |
4955 | * can also include other factors [XXX]. | 4955 | * can also include other factors [XXX]. |
4956 | * | 4956 | * |
4957 | * To achieve this balance we define a measure of imbalance which follows | 4957 | * To achieve this balance we define a measure of imbalance which follows |
4958 | * directly from (1): | 4958 | * directly from (1): |
4959 | * | 4959 | * |
4960 | * imb_i,j = max{ avg(W/P), W_i/P_i } - min{ avg(W/P), W_j/P_j } (4) | 4960 | * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) |
4961 | * | 4961 | * |
4962 | * We them move tasks around to minimize the imbalance. In the continuous | 4962 | * We them move tasks around to minimize the imbalance. In the continuous |
4963 | * function space it is obvious this converges, in the discrete case we get | 4963 | * function space it is obvious this converges, in the discrete case we get |
@@ -5607,17 +5607,17 @@ static inline int get_sd_load_idx(struct sched_domain *sd, | |||
5607 | return load_idx; | 5607 | return load_idx; |
5608 | } | 5608 | } |
5609 | 5609 | ||
5610 | static unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu) | 5610 | static unsigned long default_scale_capacity(struct sched_domain *sd, int cpu) |
5611 | { | 5611 | { |
5612 | return SCHED_POWER_SCALE; | 5612 | return SCHED_POWER_SCALE; |
5613 | } | 5613 | } |
5614 | 5614 | ||
5615 | unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu) | 5615 | unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu) |
5616 | { | 5616 | { |
5617 | return default_scale_freq_power(sd, cpu); | 5617 | return default_scale_capacity(sd, cpu); |
5618 | } | 5618 | } |
5619 | 5619 | ||
5620 | static unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu) | 5620 | static unsigned long default_scale_smt_capacity(struct sched_domain *sd, int cpu) |
5621 | { | 5621 | { |
5622 | unsigned long weight = sd->span_weight; | 5622 | unsigned long weight = sd->span_weight; |
5623 | unsigned long smt_gain = sd->smt_gain; | 5623 | unsigned long smt_gain = sd->smt_gain; |
@@ -5629,10 +5629,10 @@ static unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu) | |||
5629 | 5629 | ||
5630 | unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu) | 5630 | unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu) |
5631 | { | 5631 | { |
5632 | return default_scale_smt_power(sd, cpu); | 5632 | return default_scale_smt_capacity(sd, cpu); |
5633 | } | 5633 | } |
5634 | 5634 | ||
5635 | static unsigned long scale_rt_power(int cpu) | 5635 | static unsigned long scale_rt_capacity(int cpu) |
5636 | { | 5636 | { |
5637 | struct rq *rq = cpu_rq(cpu); | 5637 | struct rq *rq = cpu_rq(cpu); |
5638 | u64 total, available, age_stamp, avg; | 5638 | u64 total, available, age_stamp, avg; |
@@ -5652,7 +5652,7 @@ static unsigned long scale_rt_power(int cpu) | |||
5652 | total = sched_avg_period() + delta; | 5652 | total = sched_avg_period() + delta; |
5653 | 5653 | ||
5654 | if (unlikely(total < avg)) { | 5654 | if (unlikely(total < avg)) { |
5655 | /* Ensures that power won't end up being negative */ | 5655 | /* Ensures that capacity won't end up being negative */ |
5656 | available = 0; | 5656 | available = 0; |
5657 | } else { | 5657 | } else { |
5658 | available = total - avg; | 5658 | available = total - avg; |
@@ -5666,38 +5666,38 @@ static unsigned long scale_rt_power(int cpu) | |||
5666 | return div_u64(available, total); | 5666 | return div_u64(available, total); |
5667 | } | 5667 | } |
5668 | 5668 | ||
5669 | static void update_cpu_power(struct sched_domain *sd, int cpu) | 5669 | static void update_cpu_capacity(struct sched_domain *sd, int cpu) |
5670 | { | 5670 | { |
5671 | unsigned long weight = sd->span_weight; | 5671 | unsigned long weight = sd->span_weight; |
5672 | unsigned long power = SCHED_POWER_SCALE; | 5672 | unsigned long capacity = SCHED_POWER_SCALE; |
5673 | struct sched_group *sdg = sd->groups; | 5673 | struct sched_group *sdg = sd->groups; |
5674 | 5674 | ||
5675 | if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) { | 5675 | if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) { |
5676 | if (sched_feat(ARCH_POWER)) | 5676 | if (sched_feat(ARCH_POWER)) |
5677 | power *= arch_scale_smt_power(sd, cpu); | 5677 | capacity *= arch_scale_smt_power(sd, cpu); |
5678 | else | 5678 | else |
5679 | power *= default_scale_smt_power(sd, cpu); | 5679 | capacity *= default_scale_smt_capacity(sd, cpu); |
5680 | 5680 | ||
5681 | power >>= SCHED_POWER_SHIFT; | 5681 | capacity >>= SCHED_POWER_SHIFT; |
5682 | } | 5682 | } |
5683 | 5683 | ||
5684 | sdg->sgc->capacity_orig = power; | 5684 | sdg->sgc->capacity_orig = capacity; |
5685 | 5685 | ||
5686 | if (sched_feat(ARCH_POWER)) | 5686 | if (sched_feat(ARCH_POWER)) |
5687 | power *= arch_scale_freq_power(sd, cpu); | 5687 | capacity *= arch_scale_freq_power(sd, cpu); |
5688 | else | 5688 | else |
5689 | power *= default_scale_freq_power(sd, cpu); | 5689 | capacity *= default_scale_capacity(sd, cpu); |
5690 | 5690 | ||
5691 | power >>= SCHED_POWER_SHIFT; | 5691 | capacity >>= SCHED_POWER_SHIFT; |
5692 | 5692 | ||
5693 | power *= scale_rt_power(cpu); | 5693 | capacity *= scale_rt_capacity(cpu); |
5694 | power >>= SCHED_POWER_SHIFT; | 5694 | capacity >>= SCHED_POWER_SHIFT; |
5695 | 5695 | ||
5696 | if (!power) | 5696 | if (!capacity) |
5697 | power = 1; | 5697 | capacity = 1; |
5698 | 5698 | ||
5699 | cpu_rq(cpu)->cpu_power = power; | 5699 | cpu_rq(cpu)->cpu_capacity = capacity; |
5700 | sdg->sgc->capacity = power; | 5700 | sdg->sgc->capacity = capacity; |
5701 | } | 5701 | } |
5702 | 5702 | ||
5703 | void update_group_capacity(struct sched_domain *sd, int cpu) | 5703 | void update_group_capacity(struct sched_domain *sd, int cpu) |
@@ -5712,7 +5712,7 @@ void update_group_capacity(struct sched_domain *sd, int cpu) | |||
5712 | sdg->sgc->next_update = jiffies + interval; | 5712 | sdg->sgc->next_update = jiffies + interval; |
5713 | 5713 | ||
5714 | if (!child) { | 5714 | if (!child) { |
5715 | update_cpu_power(sd, cpu); | 5715 | update_cpu_capacity(sd, cpu); |
5716 | return; | 5716 | return; |
5717 | } | 5717 | } |
5718 | 5718 | ||
@@ -5733,8 +5733,8 @@ void update_group_capacity(struct sched_domain *sd, int cpu) | |||
5733 | * gets here before we've attached the domains to the | 5733 | * gets here before we've attached the domains to the |
5734 | * runqueues. | 5734 | * runqueues. |
5735 | * | 5735 | * |
5736 | * Use power_of(), which is set irrespective of domains | 5736 | * Use capacity_of(), which is set irrespective of domains |
5737 | * in update_cpu_power(). | 5737 | * in update_cpu_capacity(). |
5738 | * | 5738 | * |
5739 | * This avoids capacity/capacity_orig from being 0 and | 5739 | * This avoids capacity/capacity_orig from being 0 and |
5740 | * causing divide-by-zero issues on boot. | 5740 | * causing divide-by-zero issues on boot. |
@@ -5742,8 +5742,8 @@ void update_group_capacity(struct sched_domain *sd, int cpu) | |||
5742 | * Runtime updates will correct capacity_orig. | 5742 | * Runtime updates will correct capacity_orig. |
5743 | */ | 5743 | */ |
5744 | if (unlikely(!rq->sd)) { | 5744 | if (unlikely(!rq->sd)) { |
5745 | capacity_orig += power_of(cpu); | 5745 | capacity_orig += capacity_of(cpu); |
5746 | capacity += power_of(cpu); | 5746 | capacity += capacity_of(cpu); |
5747 | continue; | 5747 | continue; |
5748 | } | 5748 | } |
5749 | 5749 | ||
@@ -5831,7 +5831,7 @@ static inline int sg_imbalanced(struct sched_group *group) | |||
5831 | /* | 5831 | /* |
5832 | * Compute the group capacity factor. | 5832 | * Compute the group capacity factor. |
5833 | * | 5833 | * |
5834 | * Avoid the issue where N*frac(smt_power) >= 1 creates 'phantom' cores by | 5834 | * Avoid the issue where N*frac(smt_capacity) >= 1 creates 'phantom' cores by |
5835 | * first dividing out the smt factor and computing the actual number of cores | 5835 | * first dividing out the smt factor and computing the actual number of cores |
5836 | * and limit unit capacity with that. | 5836 | * and limit unit capacity with that. |
5837 | */ | 5837 | */ |
@@ -6129,7 +6129,7 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) | |||
6129 | 6129 | ||
6130 | /* | 6130 | /* |
6131 | * OK, we don't have enough imbalance to justify moving tasks, | 6131 | * OK, we don't have enough imbalance to justify moving tasks, |
6132 | * however we may be able to increase total CPU power used by | 6132 | * however we may be able to increase total CPU capacity used by |
6133 | * moving them. | 6133 | * moving them. |
6134 | */ | 6134 | */ |
6135 | 6135 | ||
@@ -6190,7 +6190,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s | |||
6190 | /* | 6190 | /* |
6191 | * In the presence of smp nice balancing, certain scenarios can have | 6191 | * In the presence of smp nice balancing, certain scenarios can have |
6192 | * max load less than avg load(as we skip the groups at or below | 6192 | * max load less than avg load(as we skip the groups at or below |
6193 | * its cpu_power, while calculating max_load..) | 6193 | * its cpu_capacity, while calculating max_load..) |
6194 | */ | 6194 | */ |
6195 | if (busiest->avg_load <= sds->avg_load || | 6195 | if (busiest->avg_load <= sds->avg_load || |
6196 | local->avg_load >= sds->avg_load) { | 6196 | local->avg_load >= sds->avg_load) { |
@@ -6345,11 +6345,11 @@ static struct rq *find_busiest_queue(struct lb_env *env, | |||
6345 | struct sched_group *group) | 6345 | struct sched_group *group) |
6346 | { | 6346 | { |
6347 | struct rq *busiest = NULL, *rq; | 6347 | struct rq *busiest = NULL, *rq; |
6348 | unsigned long busiest_load = 0, busiest_power = 1; | 6348 | unsigned long busiest_load = 0, busiest_capacity = 1; |
6349 | int i; | 6349 | int i; |
6350 | 6350 | ||
6351 | for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { | 6351 | for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { |
6352 | unsigned long power, capacity_factor, wl; | 6352 | unsigned long capacity, capacity_factor, wl; |
6353 | enum fbq_type rt; | 6353 | enum fbq_type rt; |
6354 | 6354 | ||
6355 | rq = cpu_rq(i); | 6355 | rq = cpu_rq(i); |
@@ -6377,8 +6377,8 @@ static struct rq *find_busiest_queue(struct lb_env *env, | |||
6377 | if (rt > env->fbq_type) | 6377 | if (rt > env->fbq_type) |
6378 | continue; | 6378 | continue; |
6379 | 6379 | ||
6380 | power = power_of(i); | 6380 | capacity = capacity_of(i); |
6381 | capacity_factor = DIV_ROUND_CLOSEST(power, SCHED_POWER_SCALE); | 6381 | capacity_factor = DIV_ROUND_CLOSEST(capacity, SCHED_POWER_SCALE); |
6382 | if (!capacity_factor) | 6382 | if (!capacity_factor) |
6383 | capacity_factor = fix_small_capacity(env->sd, group); | 6383 | capacity_factor = fix_small_capacity(env->sd, group); |
6384 | 6384 | ||
@@ -6386,25 +6386,25 @@ static struct rq *find_busiest_queue(struct lb_env *env, | |||
6386 | 6386 | ||
6387 | /* | 6387 | /* |
6388 | * When comparing with imbalance, use weighted_cpuload() | 6388 | * When comparing with imbalance, use weighted_cpuload() |
6389 | * which is not scaled with the cpu power. | 6389 | * which is not scaled with the cpu capacity. |
6390 | */ | 6390 | */ |
6391 | if (capacity_factor && rq->nr_running == 1 && wl > env->imbalance) | 6391 | if (capacity_factor && rq->nr_running == 1 && wl > env->imbalance) |
6392 | continue; | 6392 | continue; |
6393 | 6393 | ||
6394 | /* | 6394 | /* |
6395 | * For the load comparisons with the other cpu's, consider | 6395 | * For the load comparisons with the other cpu's, consider |
6396 | * the weighted_cpuload() scaled with the cpu power, so that | 6396 | * the weighted_cpuload() scaled with the cpu capacity, so |
6397 | * the load can be moved away from the cpu that is potentially | 6397 | * that the load can be moved away from the cpu that is |
6398 | * running at a lower capacity. | 6398 | * potentially running at a lower capacity. |
6399 | * | 6399 | * |
6400 | * Thus we're looking for max(wl_i / power_i), crosswise | 6400 | * Thus we're looking for max(wl_i / capacity_i), crosswise |
6401 | * multiplication to rid ourselves of the division works out | 6401 | * multiplication to rid ourselves of the division works out |
6402 | * to: wl_i * power_j > wl_j * power_i; where j is our | 6402 | * to: wl_i * capacity_j > wl_j * capacity_i; where j is |
6403 | * previous maximum. | 6403 | * our previous maximum. |
6404 | */ | 6404 | */ |
6405 | if (wl * busiest_power > busiest_load * power) { | 6405 | if (wl * busiest_capacity > busiest_load * capacity) { |
6406 | busiest_load = wl; | 6406 | busiest_load = wl; |
6407 | busiest_power = power; | 6407 | busiest_capacity = capacity; |
6408 | busiest = rq; | 6408 | busiest = rq; |
6409 | } | 6409 | } |
6410 | } | 6410 | } |
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index a5b957d53c92..956b8ca24893 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h | |||
@@ -567,7 +567,7 @@ struct rq { | |||
567 | struct root_domain *rd; | 567 | struct root_domain *rd; |
568 | struct sched_domain *sd; | 568 | struct sched_domain *sd; |
569 | 569 | ||
570 | unsigned long cpu_power; | 570 | unsigned long cpu_capacity; |
571 | 571 | ||
572 | unsigned char idle_balance; | 572 | unsigned char idle_balance; |
573 | /* For active balancing */ | 573 | /* For active balancing */ |