aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/sched/core.c6
-rw-r--r--kernel/sched/fair.c59
2 files changed, 33 insertions, 32 deletions
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 07bc78a50329..7ba4f5413a10 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5249,7 +5249,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
5249 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group)); 5249 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
5250 5250
5251 printk(KERN_CONT " %s", str); 5251 printk(KERN_CONT " %s", str);
5252 if (group->sgc->capacity != SCHED_POWER_SCALE) { 5252 if (group->sgc->capacity != SCHED_CAPACITY_SCALE) {
5253 printk(KERN_CONT " (cpu_capacity = %d)", 5253 printk(KERN_CONT " (cpu_capacity = %d)",
5254 group->sgc->capacity); 5254 group->sgc->capacity);
5255 } 5255 }
@@ -5715,7 +5715,7 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu)
5715 * domains and no possible iteration will get us here, we won't 5715 * domains and no possible iteration will get us here, we won't
5716 * die on a /0 trap. 5716 * die on a /0 trap.
5717 */ 5717 */
5718 sg->sgc->capacity = SCHED_POWER_SCALE * cpumask_weight(sg_span); 5718 sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);
5719 sg->sgc->capacity_orig = sg->sgc->capacity; 5719 sg->sgc->capacity_orig = sg->sgc->capacity;
5720 5720
5721 /* 5721 /*
@@ -6921,7 +6921,7 @@ void __init sched_init(void)
6921#ifdef CONFIG_SMP 6921#ifdef CONFIG_SMP
6922 rq->sd = NULL; 6922 rq->sd = NULL;
6923 rq->rd = NULL; 6923 rq->rd = NULL;
6924 rq->cpu_capacity = SCHED_POWER_SCALE; 6924 rq->cpu_capacity = SCHED_CAPACITY_SCALE;
6925 rq->post_schedule = 0; 6925 rq->post_schedule = 0;
6926 rq->active_balance = 0; 6926 rq->active_balance = 0;
6927 rq->next_balance = jiffies; 6927 rq->next_balance = jiffies;
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 58684f684fa8..dc7d6527a282 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1062,9 +1062,9 @@ static void update_numa_stats(struct numa_stats *ns, int nid)
1062 if (!cpus) 1062 if (!cpus)
1063 return; 1063 return;
1064 1064
1065 ns->load = (ns->load * SCHED_POWER_SCALE) / ns->compute_capacity; 1065 ns->load = (ns->load * SCHED_CAPACITY_SCALE) / ns->compute_capacity;
1066 ns->task_capacity = 1066 ns->task_capacity =
1067 DIV_ROUND_CLOSEST(ns->compute_capacity, SCHED_POWER_SCALE); 1067 DIV_ROUND_CLOSEST(ns->compute_capacity, SCHED_CAPACITY_SCALE);
1068 ns->has_free_capacity = (ns->nr_running < ns->task_capacity); 1068 ns->has_free_capacity = (ns->nr_running < ns->task_capacity);
1069} 1069}
1070 1070
@@ -4370,7 +4370,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p,
4370 } 4370 }
4371 4371
4372 /* Adjust by relative CPU capacity of the group */ 4372 /* Adjust by relative CPU capacity of the group */
4373 avg_load = (avg_load * SCHED_POWER_SCALE) / group->sgc->capacity; 4373 avg_load = (avg_load * SCHED_CAPACITY_SCALE) / group->sgc->capacity;
4374 4374
4375 if (local_group) { 4375 if (local_group) {
4376 this_load = avg_load; 4376 this_load = avg_load;
@@ -5609,10 +5609,10 @@ static inline int get_sd_load_idx(struct sched_domain *sd,
5609 5609
5610static unsigned long default_scale_capacity(struct sched_domain *sd, int cpu) 5610static unsigned long default_scale_capacity(struct sched_domain *sd, int cpu)
5611{ 5611{
5612 return SCHED_POWER_SCALE; 5612 return SCHED_CAPACITY_SCALE;
5613} 5613}
5614 5614
5615unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu) 5615unsigned long __weak arch_scale_freq_capacity(struct sched_domain *sd, int cpu)
5616{ 5616{
5617 return default_scale_capacity(sd, cpu); 5617 return default_scale_capacity(sd, cpu);
5618} 5618}
@@ -5627,7 +5627,7 @@ static unsigned long default_scale_smt_capacity(struct sched_domain *sd, int cpu
5627 return smt_gain; 5627 return smt_gain;
5628} 5628}
5629 5629
5630unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu) 5630unsigned long __weak arch_scale_smt_capacity(struct sched_domain *sd, int cpu)
5631{ 5631{
5632 return default_scale_smt_capacity(sd, cpu); 5632 return default_scale_smt_capacity(sd, cpu);
5633} 5633}
@@ -5658,10 +5658,10 @@ static unsigned long scale_rt_capacity(int cpu)
5658 available = total - avg; 5658 available = total - avg;
5659 } 5659 }
5660 5660
5661 if (unlikely((s64)total < SCHED_POWER_SCALE)) 5661 if (unlikely((s64)total < SCHED_CAPACITY_SCALE))
5662 total = SCHED_POWER_SCALE; 5662 total = SCHED_CAPACITY_SCALE;
5663 5663
5664 total >>= SCHED_POWER_SHIFT; 5664 total >>= SCHED_CAPACITY_SHIFT;
5665 5665
5666 return div_u64(available, total); 5666 return div_u64(available, total);
5667} 5667}
@@ -5669,29 +5669,29 @@ static unsigned long scale_rt_capacity(int cpu)
5669static void update_cpu_capacity(struct sched_domain *sd, int cpu) 5669static void update_cpu_capacity(struct sched_domain *sd, int cpu)
5670{ 5670{
5671 unsigned long weight = sd->span_weight; 5671 unsigned long weight = sd->span_weight;
5672 unsigned long capacity = SCHED_POWER_SCALE; 5672 unsigned long capacity = SCHED_CAPACITY_SCALE;
5673 struct sched_group *sdg = sd->groups; 5673 struct sched_group *sdg = sd->groups;
5674 5674
5675 if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) { 5675 if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
5676 if (sched_feat(ARCH_POWER)) 5676 if (sched_feat(ARCH_POWER))
5677 capacity *= arch_scale_smt_power(sd, cpu); 5677 capacity *= arch_scale_smt_capacity(sd, cpu);
5678 else 5678 else
5679 capacity *= default_scale_smt_capacity(sd, cpu); 5679 capacity *= default_scale_smt_capacity(sd, cpu);
5680 5680
5681 capacity >>= SCHED_POWER_SHIFT; 5681 capacity >>= SCHED_CAPACITY_SHIFT;
5682 } 5682 }
5683 5683
5684 sdg->sgc->capacity_orig = capacity; 5684 sdg->sgc->capacity_orig = capacity;
5685 5685
5686 if (sched_feat(ARCH_POWER)) 5686 if (sched_feat(ARCH_POWER))
5687 capacity *= arch_scale_freq_power(sd, cpu); 5687 capacity *= arch_scale_freq_capacity(sd, cpu);
5688 else 5688 else
5689 capacity *= default_scale_capacity(sd, cpu); 5689 capacity *= default_scale_capacity(sd, cpu);
5690 5690
5691 capacity >>= SCHED_POWER_SHIFT; 5691 capacity >>= SCHED_CAPACITY_SHIFT;
5692 5692
5693 capacity *= scale_rt_capacity(cpu); 5693 capacity *= scale_rt_capacity(cpu);
5694 capacity >>= SCHED_POWER_SHIFT; 5694 capacity >>= SCHED_CAPACITY_SHIFT;
5695 5695
5696 if (!capacity) 5696 if (!capacity)
5697 capacity = 1; 5697 capacity = 1;
@@ -5780,7 +5780,7 @@ static inline int
5780fix_small_capacity(struct sched_domain *sd, struct sched_group *group) 5780fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
5781{ 5781{
5782 /* 5782 /*
5783 * Only siblings can have significantly less than SCHED_POWER_SCALE 5783 * Only siblings can have significantly less than SCHED_CAPACITY_SCALE
5784 */ 5784 */
5785 if (!(sd->flags & SD_SHARE_CPUPOWER)) 5785 if (!(sd->flags & SD_SHARE_CPUPOWER))
5786 return 0; 5786 return 0;
@@ -5845,11 +5845,11 @@ static inline int sg_capacity_factor(struct lb_env *env, struct sched_group *gro
5845 cpus = group->group_weight; 5845 cpus = group->group_weight;
5846 5846
5847 /* smt := ceil(cpus / capacity), assumes: 1 < smt_capacity < 2 */ 5847 /* smt := ceil(cpus / capacity), assumes: 1 < smt_capacity < 2 */
5848 smt = DIV_ROUND_UP(SCHED_POWER_SCALE * cpus, capacity_orig); 5848 smt = DIV_ROUND_UP(SCHED_CAPACITY_SCALE * cpus, capacity_orig);
5849 capacity_factor = cpus / smt; /* cores */ 5849 capacity_factor = cpus / smt; /* cores */
5850 5850
5851 capacity_factor = min_t(unsigned, 5851 capacity_factor = min_t(unsigned,
5852 capacity_factor, DIV_ROUND_CLOSEST(capacity, SCHED_POWER_SCALE)); 5852 capacity_factor, DIV_ROUND_CLOSEST(capacity, SCHED_CAPACITY_SCALE));
5853 if (!capacity_factor) 5853 if (!capacity_factor)
5854 capacity_factor = fix_small_capacity(env->sd, group); 5854 capacity_factor = fix_small_capacity(env->sd, group);
5855 5855
@@ -5895,7 +5895,7 @@ static inline void update_sg_lb_stats(struct lb_env *env,
5895 5895
5896 /* Adjust by relative CPU capacity of the group */ 5896 /* Adjust by relative CPU capacity of the group */
5897 sgs->group_capacity = group->sgc->capacity; 5897 sgs->group_capacity = group->sgc->capacity;
5898 sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / sgs->group_capacity; 5898 sgs->avg_load = (sgs->group_load*SCHED_CAPACITY_SCALE) / sgs->group_capacity;
5899 5899
5900 if (sgs->sum_nr_running) 5900 if (sgs->sum_nr_running)
5901 sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running; 5901 sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running;
@@ -6089,7 +6089,7 @@ static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds)
6089 6089
6090 env->imbalance = DIV_ROUND_CLOSEST( 6090 env->imbalance = DIV_ROUND_CLOSEST(
6091 sds->busiest_stat.avg_load * sds->busiest_stat.group_capacity, 6091 sds->busiest_stat.avg_load * sds->busiest_stat.group_capacity,
6092 SCHED_POWER_SCALE); 6092 SCHED_CAPACITY_SCALE);
6093 6093
6094 return 1; 6094 return 1;
6095} 6095}
@@ -6118,7 +6118,7 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
6118 imbn = 1; 6118 imbn = 1;
6119 6119
6120 scaled_busy_load_per_task = 6120 scaled_busy_load_per_task =
6121 (busiest->load_per_task * SCHED_POWER_SCALE) / 6121 (busiest->load_per_task * SCHED_CAPACITY_SCALE) /
6122 busiest->group_capacity; 6122 busiest->group_capacity;
6123 6123
6124 if (busiest->avg_load + scaled_busy_load_per_task >= 6124 if (busiest->avg_load + scaled_busy_load_per_task >=
@@ -6137,7 +6137,7 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
6137 min(busiest->load_per_task, busiest->avg_load); 6137 min(busiest->load_per_task, busiest->avg_load);
6138 capa_now += local->group_capacity * 6138 capa_now += local->group_capacity *
6139 min(local->load_per_task, local->avg_load); 6139 min(local->load_per_task, local->avg_load);
6140 capa_now /= SCHED_POWER_SCALE; 6140 capa_now /= SCHED_CAPACITY_SCALE;
6141 6141
6142 /* Amount of load we'd subtract */ 6142 /* Amount of load we'd subtract */
6143 if (busiest->avg_load > scaled_busy_load_per_task) { 6143 if (busiest->avg_load > scaled_busy_load_per_task) {
@@ -6148,16 +6148,16 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
6148 6148
6149 /* Amount of load we'd add */ 6149 /* Amount of load we'd add */
6150 if (busiest->avg_load * busiest->group_capacity < 6150 if (busiest->avg_load * busiest->group_capacity <
6151 busiest->load_per_task * SCHED_POWER_SCALE) { 6151 busiest->load_per_task * SCHED_CAPACITY_SCALE) {
6152 tmp = (busiest->avg_load * busiest->group_capacity) / 6152 tmp = (busiest->avg_load * busiest->group_capacity) /
6153 local->group_capacity; 6153 local->group_capacity;
6154 } else { 6154 } else {
6155 tmp = (busiest->load_per_task * SCHED_POWER_SCALE) / 6155 tmp = (busiest->load_per_task * SCHED_CAPACITY_SCALE) /
6156 local->group_capacity; 6156 local->group_capacity;
6157 } 6157 }
6158 capa_move += local->group_capacity * 6158 capa_move += local->group_capacity *
6159 min(local->load_per_task, local->avg_load + tmp); 6159 min(local->load_per_task, local->avg_load + tmp);
6160 capa_move /= SCHED_POWER_SCALE; 6160 capa_move /= SCHED_CAPACITY_SCALE;
6161 6161
6162 /* Move if we gain throughput */ 6162 /* Move if we gain throughput */
6163 if (capa_move > capa_now) 6163 if (capa_move > capa_now)
@@ -6207,7 +6207,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
6207 load_above_capacity = 6207 load_above_capacity =
6208 (busiest->sum_nr_running - busiest->group_capacity_factor); 6208 (busiest->sum_nr_running - busiest->group_capacity_factor);
6209 6209
6210 load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_POWER_SCALE); 6210 load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_CAPACITY_SCALE);
6211 load_above_capacity /= busiest->group_capacity; 6211 load_above_capacity /= busiest->group_capacity;
6212 } 6212 }
6213 6213
@@ -6225,7 +6225,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
6225 env->imbalance = min( 6225 env->imbalance = min(
6226 max_pull * busiest->group_capacity, 6226 max_pull * busiest->group_capacity,
6227 (sds->avg_load - local->avg_load) * local->group_capacity 6227 (sds->avg_load - local->avg_load) * local->group_capacity
6228 ) / SCHED_POWER_SCALE; 6228 ) / SCHED_CAPACITY_SCALE;
6229 6229
6230 /* 6230 /*
6231 * if *imbalance is less than the average load per runnable task 6231 * if *imbalance is less than the average load per runnable task
@@ -6279,7 +6279,8 @@ static struct sched_group *find_busiest_group(struct lb_env *env)
6279 if (!sds.busiest || busiest->sum_nr_running == 0) 6279 if (!sds.busiest || busiest->sum_nr_running == 0)
6280 goto out_balanced; 6280 goto out_balanced;
6281 6281
6282 sds.avg_load = (SCHED_POWER_SCALE * sds.total_load) / sds.total_capacity; 6282 sds.avg_load = (SCHED_CAPACITY_SCALE * sds.total_load)
6283 / sds.total_capacity;
6283 6284
6284 /* 6285 /*
6285 * If the busiest group is imbalanced the below checks don't 6286 * If the busiest group is imbalanced the below checks don't
@@ -6378,7 +6379,7 @@ static struct rq *find_busiest_queue(struct lb_env *env,
6378 continue; 6379 continue;
6379 6380
6380 capacity = capacity_of(i); 6381 capacity = capacity_of(i);
6381 capacity_factor = DIV_ROUND_CLOSEST(capacity, SCHED_POWER_SCALE); 6382 capacity_factor = DIV_ROUND_CLOSEST(capacity, SCHED_CAPACITY_SCALE);
6382 if (!capacity_factor) 6383 if (!capacity_factor)
6383 capacity_factor = fix_small_capacity(env->sd, group); 6384 capacity_factor = fix_small_capacity(env->sd, group);
6384 6385