aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/sched.h2
-rw-r--r--kernel/sched/core.c81
-rw-r--r--kernel/sched/fair.c131
-rw-r--r--kernel/sched/sched.h16
4 files changed, 115 insertions, 115 deletions
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 6790c3b42072..a96f03598c61 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1013,7 +1013,7 @@ typedef const int (*sched_domain_flags_f)(void);
1013struct sd_data { 1013struct sd_data {
1014 struct sched_domain **__percpu sd; 1014 struct sched_domain **__percpu sd;
1015 struct sched_group **__percpu sg; 1015 struct sched_group **__percpu sg;
1016 struct sched_group_power **__percpu sgp; 1016 struct sched_group_capacity **__percpu sgc;
1017}; 1017};
1018 1018
1019struct sched_domain_topology_level { 1019struct sched_domain_topology_level {
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index afcc84234a3e..2e1fb0902200 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5221,14 +5221,13 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
5221 } 5221 }
5222 5222
5223 /* 5223 /*
5224 * Even though we initialize ->power to something semi-sane, 5224 * Even though we initialize ->capacity to something semi-sane,
5225 * we leave power_orig unset. This allows us to detect if 5225 * we leave capacity_orig unset. This allows us to detect if
5226 * domain iteration is still funny without causing /0 traps. 5226 * domain iteration is still funny without causing /0 traps.
5227 */ 5227 */
5228 if (!group->sgp->power_orig) { 5228 if (!group->sgc->capacity_orig) {
5229 printk(KERN_CONT "\n"); 5229 printk(KERN_CONT "\n");
5230 printk(KERN_ERR "ERROR: domain->cpu_power not " 5230 printk(KERN_ERR "ERROR: domain->cpu_capacity not set\n");
5231 "set\n");
5232 break; 5231 break;
5233 } 5232 }
5234 5233
@@ -5250,9 +5249,9 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
5250 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group)); 5249 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
5251 5250
5252 printk(KERN_CONT " %s", str); 5251 printk(KERN_CONT " %s", str);
5253 if (group->sgp->power != SCHED_POWER_SCALE) { 5252 if (group->sgc->capacity != SCHED_POWER_SCALE) {
5254 printk(KERN_CONT " (cpu_power = %d)", 5253 printk(KERN_CONT " (cpu_capacity = %d)",
5255 group->sgp->power); 5254 group->sgc->capacity);
5256 } 5255 }
5257 5256
5258 group = group->next; 5257 group = group->next;
@@ -5466,7 +5465,7 @@ static struct root_domain *alloc_rootdomain(void)
5466 return rd; 5465 return rd;
5467} 5466}
5468 5467
5469static void free_sched_groups(struct sched_group *sg, int free_sgp) 5468static void free_sched_groups(struct sched_group *sg, int free_sgc)
5470{ 5469{
5471 struct sched_group *tmp, *first; 5470 struct sched_group *tmp, *first;
5472 5471
@@ -5477,8 +5476,8 @@ static void free_sched_groups(struct sched_group *sg, int free_sgp)
5477 do { 5476 do {
5478 tmp = sg->next; 5477 tmp = sg->next;
5479 5478
5480 if (free_sgp && atomic_dec_and_test(&sg->sgp->ref)) 5479 if (free_sgc && atomic_dec_and_test(&sg->sgc->ref))
5481 kfree(sg->sgp); 5480 kfree(sg->sgc);
5482 5481
5483 kfree(sg); 5482 kfree(sg);
5484 sg = tmp; 5483 sg = tmp;
@@ -5496,7 +5495,7 @@ static void free_sched_domain(struct rcu_head *rcu)
5496 if (sd->flags & SD_OVERLAP) { 5495 if (sd->flags & SD_OVERLAP) {
5497 free_sched_groups(sd->groups, 1); 5496 free_sched_groups(sd->groups, 1);
5498 } else if (atomic_dec_and_test(&sd->groups->ref)) { 5497 } else if (atomic_dec_and_test(&sd->groups->ref)) {
5499 kfree(sd->groups->sgp); 5498 kfree(sd->groups->sgc);
5500 kfree(sd->groups); 5499 kfree(sd->groups);
5501 } 5500 }
5502 kfree(sd); 5501 kfree(sd);
@@ -5707,17 +5706,17 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu)
5707 5706
5708 cpumask_or(covered, covered, sg_span); 5707 cpumask_or(covered, covered, sg_span);
5709 5708
5710 sg->sgp = *per_cpu_ptr(sdd->sgp, i); 5709 sg->sgc = *per_cpu_ptr(sdd->sgc, i);
5711 if (atomic_inc_return(&sg->sgp->ref) == 1) 5710 if (atomic_inc_return(&sg->sgc->ref) == 1)
5712 build_group_mask(sd, sg); 5711 build_group_mask(sd, sg);
5713 5712
5714 /* 5713 /*
5715 * Initialize sgp->power such that even if we mess up the 5714 * Initialize sgc->capacity such that even if we mess up the
5716 * domains and no possible iteration will get us here, we won't 5715 * domains and no possible iteration will get us here, we won't
5717 * die on a /0 trap. 5716 * die on a /0 trap.
5718 */ 5717 */
5719 sg->sgp->power = SCHED_POWER_SCALE * cpumask_weight(sg_span); 5718 sg->sgc->capacity = SCHED_POWER_SCALE * cpumask_weight(sg_span);
5720 sg->sgp->power_orig = sg->sgp->power; 5719 sg->sgc->capacity_orig = sg->sgc->capacity;
5721 5720
5722 /* 5721 /*
5723 * Make sure the first group of this domain contains the 5722 * Make sure the first group of this domain contains the
@@ -5755,8 +5754,8 @@ static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg)
5755 5754
5756 if (sg) { 5755 if (sg) {
5757 *sg = *per_cpu_ptr(sdd->sg, cpu); 5756 *sg = *per_cpu_ptr(sdd->sg, cpu);
5758 (*sg)->sgp = *per_cpu_ptr(sdd->sgp, cpu); 5757 (*sg)->sgc = *per_cpu_ptr(sdd->sgc, cpu);
5759 atomic_set(&(*sg)->sgp->ref, 1); /* for claim_allocations */ 5758 atomic_set(&(*sg)->sgc->ref, 1); /* for claim_allocations */
5760 } 5759 }
5761 5760
5762 return cpu; 5761 return cpu;
@@ -5819,16 +5818,16 @@ build_sched_groups(struct sched_domain *sd, int cpu)
5819} 5818}
5820 5819
5821/* 5820/*
5822 * Initialize sched groups cpu_power. 5821 * Initialize sched groups cpu_capacity.
5823 * 5822 *
5824 * cpu_power indicates the capacity of sched group, which is used while 5823 * cpu_capacity indicates the capacity of sched group, which is used while
5825 * distributing the load between different sched groups in a sched domain. 5824 * distributing the load between different sched groups in a sched domain.
5826 * Typically cpu_power for all the groups in a sched domain will be same unless 5825 * Typically cpu_capacity for all the groups in a sched domain will be same
5827 * there are asymmetries in the topology. If there are asymmetries, group 5826 * unless there are asymmetries in the topology. If there are asymmetries,
5828 * having more cpu_power will pickup more load compared to the group having 5827 * group having more cpu_capacity will pickup more load compared to the
5829 * less cpu_power. 5828 * group having less cpu_capacity.
5830 */ 5829 */
5831static void init_sched_groups_power(int cpu, struct sched_domain *sd) 5830static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
5832{ 5831{
5833 struct sched_group *sg = sd->groups; 5832 struct sched_group *sg = sd->groups;
5834 5833
@@ -5842,8 +5841,8 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
5842 if (cpu != group_balance_cpu(sg)) 5841 if (cpu != group_balance_cpu(sg))
5843 return; 5842 return;
5844 5843
5845 update_group_power(sd, cpu); 5844 update_group_capacity(sd, cpu);
5846 atomic_set(&sg->sgp->nr_busy_cpus, sg->group_weight); 5845 atomic_set(&sg->sgc->nr_busy_cpus, sg->group_weight);
5847} 5846}
5848 5847
5849/* 5848/*
@@ -5934,8 +5933,8 @@ static void claim_allocations(int cpu, struct sched_domain *sd)
5934 if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref)) 5933 if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
5935 *per_cpu_ptr(sdd->sg, cpu) = NULL; 5934 *per_cpu_ptr(sdd->sg, cpu) = NULL;
5936 5935
5937 if (atomic_read(&(*per_cpu_ptr(sdd->sgp, cpu))->ref)) 5936 if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref))
5938 *per_cpu_ptr(sdd->sgp, cpu) = NULL; 5937 *per_cpu_ptr(sdd->sgc, cpu) = NULL;
5939} 5938}
5940 5939
5941#ifdef CONFIG_NUMA 5940#ifdef CONFIG_NUMA
@@ -6337,14 +6336,14 @@ static int __sdt_alloc(const struct cpumask *cpu_map)
6337 if (!sdd->sg) 6336 if (!sdd->sg)
6338 return -ENOMEM; 6337 return -ENOMEM;
6339 6338
6340 sdd->sgp = alloc_percpu(struct sched_group_power *); 6339 sdd->sgc = alloc_percpu(struct sched_group_capacity *);
6341 if (!sdd->sgp) 6340 if (!sdd->sgc)
6342 return -ENOMEM; 6341 return -ENOMEM;
6343 6342
6344 for_each_cpu(j, cpu_map) { 6343 for_each_cpu(j, cpu_map) {
6345 struct sched_domain *sd; 6344 struct sched_domain *sd;
6346 struct sched_group *sg; 6345 struct sched_group *sg;
6347 struct sched_group_power *sgp; 6346 struct sched_group_capacity *sgc;
6348 6347
6349 sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(), 6348 sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
6350 GFP_KERNEL, cpu_to_node(j)); 6349 GFP_KERNEL, cpu_to_node(j));
@@ -6362,12 +6361,12 @@ static int __sdt_alloc(const struct cpumask *cpu_map)
6362 6361
6363 *per_cpu_ptr(sdd->sg, j) = sg; 6362 *per_cpu_ptr(sdd->sg, j) = sg;
6364 6363
6365 sgp = kzalloc_node(sizeof(struct sched_group_power) + cpumask_size(), 6364 sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(),
6366 GFP_KERNEL, cpu_to_node(j)); 6365 GFP_KERNEL, cpu_to_node(j));
6367 if (!sgp) 6366 if (!sgc)
6368 return -ENOMEM; 6367 return -ENOMEM;
6369 6368
6370 *per_cpu_ptr(sdd->sgp, j) = sgp; 6369 *per_cpu_ptr(sdd->sgc, j) = sgc;
6371 } 6370 }
6372 } 6371 }
6373 6372
@@ -6394,15 +6393,15 @@ static void __sdt_free(const struct cpumask *cpu_map)
6394 6393
6395 if (sdd->sg) 6394 if (sdd->sg)
6396 kfree(*per_cpu_ptr(sdd->sg, j)); 6395 kfree(*per_cpu_ptr(sdd->sg, j));
6397 if (sdd->sgp) 6396 if (sdd->sgc)
6398 kfree(*per_cpu_ptr(sdd->sgp, j)); 6397 kfree(*per_cpu_ptr(sdd->sgc, j));
6399 } 6398 }
6400 free_percpu(sdd->sd); 6399 free_percpu(sdd->sd);
6401 sdd->sd = NULL; 6400 sdd->sd = NULL;
6402 free_percpu(sdd->sg); 6401 free_percpu(sdd->sg);
6403 sdd->sg = NULL; 6402 sdd->sg = NULL;
6404 free_percpu(sdd->sgp); 6403 free_percpu(sdd->sgc);
6405 sdd->sgp = NULL; 6404 sdd->sgc = NULL;
6406 } 6405 }
6407} 6406}
6408 6407
@@ -6479,7 +6478,7 @@ static int build_sched_domains(const struct cpumask *cpu_map,
6479 6478
6480 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { 6479 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
6481 claim_allocations(i, sd); 6480 claim_allocations(i, sd);
6482 init_sched_groups_power(i, sd); 6481 init_sched_groups_capacity(i, sd);
6483 } 6482 }
6484 } 6483 }
6485 6484
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index e401e446e87c..36bd4d23fca8 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -4369,8 +4369,8 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p,
4369 avg_load += load; 4369 avg_load += load;
4370 } 4370 }
4371 4371
4372 /* Adjust by relative CPU power of the group */ 4372 /* Adjust by relative CPU capacity of the group */
4373 avg_load = (avg_load * SCHED_POWER_SCALE) / group->sgp->power; 4373 avg_load = (avg_load * SCHED_POWER_SCALE) / group->sgc->capacity;
4374 4374
4375 if (local_group) { 4375 if (local_group) {
4376 this_load = avg_load; 4376 this_load = avg_load;
@@ -5532,7 +5532,7 @@ struct sg_lb_stats {
5532 unsigned long group_load; /* Total load over the CPUs of the group */ 5532 unsigned long group_load; /* Total load over the CPUs of the group */
5533 unsigned long sum_weighted_load; /* Weighted load of group's tasks */ 5533 unsigned long sum_weighted_load; /* Weighted load of group's tasks */
5534 unsigned long load_per_task; 5534 unsigned long load_per_task;
5535 unsigned long group_power; 5535 unsigned long group_capacity;
5536 unsigned int sum_nr_running; /* Nr tasks running in the group */ 5536 unsigned int sum_nr_running; /* Nr tasks running in the group */
5537 unsigned int group_capacity_factor; 5537 unsigned int group_capacity_factor;
5538 unsigned int idle_cpus; 5538 unsigned int idle_cpus;
@@ -5553,7 +5553,7 @@ struct sd_lb_stats {
5553 struct sched_group *busiest; /* Busiest group in this sd */ 5553 struct sched_group *busiest; /* Busiest group in this sd */
5554 struct sched_group *local; /* Local group in this sd */ 5554 struct sched_group *local; /* Local group in this sd */
5555 unsigned long total_load; /* Total load of all groups in sd */ 5555 unsigned long total_load; /* Total load of all groups in sd */
5556 unsigned long total_pwr; /* Total power of all groups in sd */ 5556 unsigned long total_capacity; /* Total capacity of all groups in sd */
5557 unsigned long avg_load; /* Average load across all groups in sd */ 5557 unsigned long avg_load; /* Average load across all groups in sd */
5558 5558
5559 struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */ 5559 struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */
@@ -5572,7 +5572,7 @@ static inline void init_sd_lb_stats(struct sd_lb_stats *sds)
5572 .busiest = NULL, 5572 .busiest = NULL,
5573 .local = NULL, 5573 .local = NULL,
5574 .total_load = 0UL, 5574 .total_load = 0UL,
5575 .total_pwr = 0UL, 5575 .total_capacity = 0UL,
5576 .busiest_stat = { 5576 .busiest_stat = {
5577 .avg_load = 0UL, 5577 .avg_load = 0UL,
5578 }, 5578 },
@@ -5681,7 +5681,7 @@ static void update_cpu_power(struct sched_domain *sd, int cpu)
5681 power >>= SCHED_POWER_SHIFT; 5681 power >>= SCHED_POWER_SHIFT;
5682 } 5682 }
5683 5683
5684 sdg->sgp->power_orig = power; 5684 sdg->sgc->capacity_orig = power;
5685 5685
5686 if (sched_feat(ARCH_POWER)) 5686 if (sched_feat(ARCH_POWER))
5687 power *= arch_scale_freq_power(sd, cpu); 5687 power *= arch_scale_freq_power(sd, cpu);
@@ -5697,26 +5697,26 @@ static void update_cpu_power(struct sched_domain *sd, int cpu)
5697 power = 1; 5697 power = 1;
5698 5698
5699 cpu_rq(cpu)->cpu_power = power; 5699 cpu_rq(cpu)->cpu_power = power;
5700 sdg->sgp->power = power; 5700 sdg->sgc->capacity = power;
5701} 5701}
5702 5702
5703void update_group_power(struct sched_domain *sd, int cpu) 5703void update_group_capacity(struct sched_domain *sd, int cpu)
5704{ 5704{
5705 struct sched_domain *child = sd->child; 5705 struct sched_domain *child = sd->child;
5706 struct sched_group *group, *sdg = sd->groups; 5706 struct sched_group *group, *sdg = sd->groups;
5707 unsigned long power, power_orig; 5707 unsigned long capacity, capacity_orig;
5708 unsigned long interval; 5708 unsigned long interval;
5709 5709
5710 interval = msecs_to_jiffies(sd->balance_interval); 5710 interval = msecs_to_jiffies(sd->balance_interval);
5711 interval = clamp(interval, 1UL, max_load_balance_interval); 5711 interval = clamp(interval, 1UL, max_load_balance_interval);
5712 sdg->sgp->next_update = jiffies + interval; 5712 sdg->sgc->next_update = jiffies + interval;
5713 5713
5714 if (!child) { 5714 if (!child) {
5715 update_cpu_power(sd, cpu); 5715 update_cpu_power(sd, cpu);
5716 return; 5716 return;
5717 } 5717 }
5718 5718
5719 power_orig = power = 0; 5719 capacity_orig = capacity = 0;
5720 5720
5721 if (child->flags & SD_OVERLAP) { 5721 if (child->flags & SD_OVERLAP) {
5722 /* 5722 /*
@@ -5725,31 +5725,31 @@ void update_group_power(struct sched_domain *sd, int cpu)
5725 */ 5725 */
5726 5726
5727 for_each_cpu(cpu, sched_group_cpus(sdg)) { 5727 for_each_cpu(cpu, sched_group_cpus(sdg)) {
5728 struct sched_group_power *sgp; 5728 struct sched_group_capacity *sgc;
5729 struct rq *rq = cpu_rq(cpu); 5729 struct rq *rq = cpu_rq(cpu);
5730 5730
5731 /* 5731 /*
5732 * build_sched_domains() -> init_sched_groups_power() 5732 * build_sched_domains() -> init_sched_groups_capacity()
5733 * gets here before we've attached the domains to the 5733 * gets here before we've attached the domains to the
5734 * runqueues. 5734 * runqueues.
5735 * 5735 *
5736 * Use power_of(), which is set irrespective of domains 5736 * Use power_of(), which is set irrespective of domains
5737 * in update_cpu_power(). 5737 * in update_cpu_power().
5738 * 5738 *
5739 * This avoids power/power_orig from being 0 and 5739 * This avoids capacity/capacity_orig from being 0 and
5740 * causing divide-by-zero issues on boot. 5740 * causing divide-by-zero issues on boot.
5741 * 5741 *
5742 * Runtime updates will correct power_orig. 5742 * Runtime updates will correct capacity_orig.
5743 */ 5743 */
5744 if (unlikely(!rq->sd)) { 5744 if (unlikely(!rq->sd)) {
5745 power_orig += power_of(cpu); 5745 capacity_orig += power_of(cpu);
5746 power += power_of(cpu); 5746 capacity += power_of(cpu);
5747 continue; 5747 continue;
5748 } 5748 }
5749 5749
5750 sgp = rq->sd->groups->sgp; 5750 sgc = rq->sd->groups->sgc;
5751 power_orig += sgp->power_orig; 5751 capacity_orig += sgc->capacity_orig;
5752 power += sgp->power; 5752 capacity += sgc->capacity;
5753 } 5753 }
5754 } else { 5754 } else {
5755 /* 5755 /*
@@ -5759,14 +5759,14 @@ void update_group_power(struct sched_domain *sd, int cpu)
5759 5759
5760 group = child->groups; 5760 group = child->groups;
5761 do { 5761 do {
5762 power_orig += group->sgp->power_orig; 5762 capacity_orig += group->sgc->capacity_orig;
5763 power += group->sgp->power; 5763 capacity += group->sgc->capacity;
5764 group = group->next; 5764 group = group->next;
5765 } while (group != child->groups); 5765 } while (group != child->groups);
5766 } 5766 }
5767 5767
5768 sdg->sgp->power_orig = power_orig; 5768 sdg->sgc->capacity_orig = capacity_orig;
5769 sdg->sgp->power = power; 5769 sdg->sgc->capacity = capacity;
5770} 5770}
5771 5771
5772/* 5772/*
@@ -5786,9 +5786,9 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
5786 return 0; 5786 return 0;
5787 5787
5788 /* 5788 /*
5789 * If ~90% of the cpu_power is still there, we're good. 5789 * If ~90% of the cpu_capacity is still there, we're good.
5790 */ 5790 */
5791 if (group->sgp->power * 32 > group->sgp->power_orig * 29) 5791 if (group->sgc->capacity * 32 > group->sgc->capacity_orig * 29)
5792 return 1; 5792 return 1;
5793 5793
5794 return 0; 5794 return 0;
@@ -5825,7 +5825,7 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
5825 5825
5826static inline int sg_imbalanced(struct sched_group *group) 5826static inline int sg_imbalanced(struct sched_group *group)
5827{ 5827{
5828 return group->sgp->imbalance; 5828 return group->sgc->imbalance;
5829} 5829}
5830 5830
5831/* 5831/*
@@ -5833,22 +5833,23 @@ static inline int sg_imbalanced(struct sched_group *group)
5833 * 5833 *
5834 * Avoid the issue where N*frac(smt_power) >= 1 creates 'phantom' cores by 5834 * Avoid the issue where N*frac(smt_power) >= 1 creates 'phantom' cores by
5835 * first dividing out the smt factor and computing the actual number of cores 5835 * first dividing out the smt factor and computing the actual number of cores
5836 * and limit power unit capacity with that. 5836 * and limit unit capacity with that.
5837 */ 5837 */
5838static inline int sg_capacity_factor(struct lb_env *env, struct sched_group *group) 5838static inline int sg_capacity_factor(struct lb_env *env, struct sched_group *group)
5839{ 5839{
5840 unsigned int capacity_factor, smt, cpus; 5840 unsigned int capacity_factor, smt, cpus;
5841 unsigned int power, power_orig; 5841 unsigned int capacity, capacity_orig;
5842 5842
5843 power = group->sgp->power; 5843 capacity = group->sgc->capacity;
5844 power_orig = group->sgp->power_orig; 5844 capacity_orig = group->sgc->capacity_orig;
5845 cpus = group->group_weight; 5845 cpus = group->group_weight;
5846 5846
5847 /* smt := ceil(cpus / power), assumes: 1 < smt_power < 2 */ 5847 /* smt := ceil(cpus / capacity), assumes: 1 < smt_capacity < 2 */
5848 smt = DIV_ROUND_UP(SCHED_POWER_SCALE * cpus, power_orig); 5848 smt = DIV_ROUND_UP(SCHED_POWER_SCALE * cpus, capacity_orig);
5849 capacity_factor = cpus / smt; /* cores */ 5849 capacity_factor = cpus / smt; /* cores */
5850 5850
5851 capacity_factor = min_t(unsigned, capacity_factor, DIV_ROUND_CLOSEST(power, SCHED_POWER_SCALE)); 5851 capacity_factor = min_t(unsigned,
5852 capacity_factor, DIV_ROUND_CLOSEST(capacity, SCHED_POWER_SCALE));
5852 if (!capacity_factor) 5853 if (!capacity_factor)
5853 capacity_factor = fix_small_capacity(env->sd, group); 5854 capacity_factor = fix_small_capacity(env->sd, group);
5854 5855
@@ -5892,9 +5893,9 @@ static inline void update_sg_lb_stats(struct lb_env *env,
5892 sgs->idle_cpus++; 5893 sgs->idle_cpus++;
5893 } 5894 }
5894 5895
5895 /* Adjust by relative CPU power of the group */ 5896 /* Adjust by relative CPU capacity of the group */
5896 sgs->group_power = group->sgp->power; 5897 sgs->group_capacity = group->sgc->capacity;
5897 sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / sgs->group_power; 5898 sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / sgs->group_capacity;
5898 5899
5899 if (sgs->sum_nr_running) 5900 if (sgs->sum_nr_running)
5900 sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running; 5901 sgs->load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running;
@@ -6009,8 +6010,8 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd
6009 sgs = &sds->local_stat; 6010 sgs = &sds->local_stat;
6010 6011
6011 if (env->idle != CPU_NEWLY_IDLE || 6012 if (env->idle != CPU_NEWLY_IDLE ||
6012 time_after_eq(jiffies, sg->sgp->next_update)) 6013 time_after_eq(jiffies, sg->sgc->next_update))
6013 update_group_power(env->sd, env->dst_cpu); 6014 update_group_capacity(env->sd, env->dst_cpu);
6014 } 6015 }
6015 6016
6016 update_sg_lb_stats(env, sg, load_idx, local_group, sgs); 6017 update_sg_lb_stats(env, sg, load_idx, local_group, sgs);
@@ -6040,7 +6041,7 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd
6040next_group: 6041next_group:
6041 /* Now, start updating sd_lb_stats */ 6042 /* Now, start updating sd_lb_stats */
6042 sds->total_load += sgs->group_load; 6043 sds->total_load += sgs->group_load;
6043 sds->total_pwr += sgs->group_power; 6044 sds->total_capacity += sgs->group_capacity;
6044 6045
6045 sg = sg->next; 6046 sg = sg->next;
6046 } while (sg != env->sd->groups); 6047 } while (sg != env->sd->groups);
@@ -6087,7 +6088,7 @@ static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds)
6087 return 0; 6088 return 0;
6088 6089
6089 env->imbalance = DIV_ROUND_CLOSEST( 6090 env->imbalance = DIV_ROUND_CLOSEST(
6090 sds->busiest_stat.avg_load * sds->busiest_stat.group_power, 6091 sds->busiest_stat.avg_load * sds->busiest_stat.group_capacity,
6091 SCHED_POWER_SCALE); 6092 SCHED_POWER_SCALE);
6092 6093
6093 return 1; 6094 return 1;
@@ -6103,7 +6104,7 @@ static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds)
6103static inline 6104static inline
6104void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) 6105void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
6105{ 6106{
6106 unsigned long tmp, pwr_now = 0, pwr_move = 0; 6107 unsigned long tmp, capa_now = 0, capa_move = 0;
6107 unsigned int imbn = 2; 6108 unsigned int imbn = 2;
6108 unsigned long scaled_busy_load_per_task; 6109 unsigned long scaled_busy_load_per_task;
6109 struct sg_lb_stats *local, *busiest; 6110 struct sg_lb_stats *local, *busiest;
@@ -6118,7 +6119,7 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
6118 6119
6119 scaled_busy_load_per_task = 6120 scaled_busy_load_per_task =
6120 (busiest->load_per_task * SCHED_POWER_SCALE) / 6121 (busiest->load_per_task * SCHED_POWER_SCALE) /
6121 busiest->group_power; 6122 busiest->group_capacity;
6122 6123
6123 if (busiest->avg_load + scaled_busy_load_per_task >= 6124 if (busiest->avg_load + scaled_busy_load_per_task >=
6124 local->avg_load + (scaled_busy_load_per_task * imbn)) { 6125 local->avg_load + (scaled_busy_load_per_task * imbn)) {
@@ -6132,34 +6133,34 @@ void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
6132 * moving them. 6133 * moving them.
6133 */ 6134 */
6134 6135
6135 pwr_now += busiest->group_power * 6136 capa_now += busiest->group_capacity *
6136 min(busiest->load_per_task, busiest->avg_load); 6137 min(busiest->load_per_task, busiest->avg_load);
6137 pwr_now += local->group_power * 6138 capa_now += local->group_capacity *
6138 min(local->load_per_task, local->avg_load); 6139 min(local->load_per_task, local->avg_load);
6139 pwr_now /= SCHED_POWER_SCALE; 6140 capa_now /= SCHED_POWER_SCALE;
6140 6141
6141 /* Amount of load we'd subtract */ 6142 /* Amount of load we'd subtract */
6142 if (busiest->avg_load > scaled_busy_load_per_task) { 6143 if (busiest->avg_load > scaled_busy_load_per_task) {
6143 pwr_move += busiest->group_power * 6144 capa_move += busiest->group_capacity *
6144 min(busiest->load_per_task, 6145 min(busiest->load_per_task,
6145 busiest->avg_load - scaled_busy_load_per_task); 6146 busiest->avg_load - scaled_busy_load_per_task);
6146 } 6147 }
6147 6148
6148 /* Amount of load we'd add */ 6149 /* Amount of load we'd add */
6149 if (busiest->avg_load * busiest->group_power < 6150 if (busiest->avg_load * busiest->group_capacity <
6150 busiest->load_per_task * SCHED_POWER_SCALE) { 6151 busiest->load_per_task * SCHED_POWER_SCALE) {
6151 tmp = (busiest->avg_load * busiest->group_power) / 6152 tmp = (busiest->avg_load * busiest->group_capacity) /
6152 local->group_power; 6153 local->group_capacity;
6153 } else { 6154 } else {
6154 tmp = (busiest->load_per_task * SCHED_POWER_SCALE) / 6155 tmp = (busiest->load_per_task * SCHED_POWER_SCALE) /
6155 local->group_power; 6156 local->group_capacity;
6156 } 6157 }
6157 pwr_move += local->group_power * 6158 capa_move += local->group_capacity *
6158 min(local->load_per_task, local->avg_load + tmp); 6159 min(local->load_per_task, local->avg_load + tmp);
6159 pwr_move /= SCHED_POWER_SCALE; 6160 capa_move /= SCHED_POWER_SCALE;
6160 6161
6161 /* Move if we gain throughput */ 6162 /* Move if we gain throughput */
6162 if (pwr_move > pwr_now) 6163 if (capa_move > capa_now)
6163 env->imbalance = busiest->load_per_task; 6164 env->imbalance = busiest->load_per_task;
6164} 6165}
6165 6166
@@ -6207,7 +6208,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
6207 (busiest->sum_nr_running - busiest->group_capacity_factor); 6208 (busiest->sum_nr_running - busiest->group_capacity_factor);
6208 6209
6209 load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_POWER_SCALE); 6210 load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_POWER_SCALE);
6210 load_above_capacity /= busiest->group_power; 6211 load_above_capacity /= busiest->group_capacity;
6211 } 6212 }
6212 6213
6213 /* 6214 /*
@@ -6222,8 +6223,8 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
6222 6223
6223 /* How much load to actually move to equalise the imbalance */ 6224 /* How much load to actually move to equalise the imbalance */
6224 env->imbalance = min( 6225 env->imbalance = min(
6225 max_pull * busiest->group_power, 6226 max_pull * busiest->group_capacity,
6226 (sds->avg_load - local->avg_load) * local->group_power 6227 (sds->avg_load - local->avg_load) * local->group_capacity
6227 ) / SCHED_POWER_SCALE; 6228 ) / SCHED_POWER_SCALE;
6228 6229
6229 /* 6230 /*
@@ -6278,7 +6279,7 @@ static struct sched_group *find_busiest_group(struct lb_env *env)
6278 if (!sds.busiest || busiest->sum_nr_running == 0) 6279 if (!sds.busiest || busiest->sum_nr_running == 0)
6279 goto out_balanced; 6280 goto out_balanced;
6280 6281
6281 sds.avg_load = (SCHED_POWER_SCALE * sds.total_load) / sds.total_pwr; 6282 sds.avg_load = (SCHED_POWER_SCALE * sds.total_load) / sds.total_capacity;
6282 6283
6283 /* 6284 /*
6284 * If the busiest group is imbalanced the below checks don't 6285 * If the busiest group is imbalanced the below checks don't
@@ -6611,7 +6612,7 @@ more_balance:
6611 * We failed to reach balance because of affinity. 6612 * We failed to reach balance because of affinity.
6612 */ 6613 */
6613 if (sd_parent) { 6614 if (sd_parent) {
6614 int *group_imbalance = &sd_parent->groups->sgp->imbalance; 6615 int *group_imbalance = &sd_parent->groups->sgc->imbalance;
6615 6616
6616 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) { 6617 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) {
6617 *group_imbalance = 1; 6618 *group_imbalance = 1;
@@ -6998,7 +6999,7 @@ static inline void set_cpu_sd_state_busy(void)
6998 goto unlock; 6999 goto unlock;
6999 sd->nohz_idle = 0; 7000 sd->nohz_idle = 0;
7000 7001
7001 atomic_inc(&sd->groups->sgp->nr_busy_cpus); 7002 atomic_inc(&sd->groups->sgc->nr_busy_cpus);
7002unlock: 7003unlock:
7003 rcu_read_unlock(); 7004 rcu_read_unlock();
7004} 7005}
@@ -7015,7 +7016,7 @@ void set_cpu_sd_state_idle(void)
7015 goto unlock; 7016 goto unlock;
7016 sd->nohz_idle = 1; 7017 sd->nohz_idle = 1;
7017 7018
7018 atomic_dec(&sd->groups->sgp->nr_busy_cpus); 7019 atomic_dec(&sd->groups->sgc->nr_busy_cpus);
7019unlock: 7020unlock:
7020 rcu_read_unlock(); 7021 rcu_read_unlock();
7021} 7022}
@@ -7219,7 +7220,7 @@ end:
7219 * of an idle cpu is the system. 7220 * of an idle cpu is the system.
7220 * - This rq has more than one task. 7221 * - This rq has more than one task.
7221 * - At any scheduler domain level, this cpu's scheduler group has multiple 7222 * - At any scheduler domain level, this cpu's scheduler group has multiple
7222 * busy cpu's exceeding the group's power. 7223 * busy cpu's exceeding the group's capacity.
7223 * - For SD_ASYM_PACKING, if the lower numbered cpu's in the scheduler 7224 * - For SD_ASYM_PACKING, if the lower numbered cpu's in the scheduler
7224 * domain span are idle. 7225 * domain span are idle.
7225 */ 7226 */
@@ -7227,7 +7228,7 @@ static inline int nohz_kick_needed(struct rq *rq)
7227{ 7228{
7228 unsigned long now = jiffies; 7229 unsigned long now = jiffies;
7229 struct sched_domain *sd; 7230 struct sched_domain *sd;
7230 struct sched_group_power *sgp; 7231 struct sched_group_capacity *sgc;
7231 int nr_busy, cpu = rq->cpu; 7232 int nr_busy, cpu = rq->cpu;
7232 7233
7233 if (unlikely(rq->idle_balance)) 7234 if (unlikely(rq->idle_balance))
@@ -7257,8 +7258,8 @@ static inline int nohz_kick_needed(struct rq *rq)
7257 sd = rcu_dereference(per_cpu(sd_busy, cpu)); 7258 sd = rcu_dereference(per_cpu(sd_busy, cpu));
7258 7259
7259 if (sd) { 7260 if (sd) {
7260 sgp = sd->groups->sgp; 7261 sgc = sd->groups->sgc;
7261 nr_busy = atomic_read(&sgp->nr_busy_cpus); 7262 nr_busy = atomic_read(&sgc->nr_busy_cpus);
7262 7263
7263 if (nr_busy > 1) 7264 if (nr_busy > 1)
7264 goto need_kick_unlock; 7265 goto need_kick_unlock;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 600e2291a75c..a5b957d53c92 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -728,15 +728,15 @@ DECLARE_PER_CPU(struct sched_domain *, sd_numa);
728DECLARE_PER_CPU(struct sched_domain *, sd_busy); 728DECLARE_PER_CPU(struct sched_domain *, sd_busy);
729DECLARE_PER_CPU(struct sched_domain *, sd_asym); 729DECLARE_PER_CPU(struct sched_domain *, sd_asym);
730 730
731struct sched_group_power { 731struct sched_group_capacity {
732 atomic_t ref; 732 atomic_t ref;
733 /* 733 /*
734 * CPU power of this group, SCHED_LOAD_SCALE being max power for a 734 * CPU capacity of this group, SCHED_LOAD_SCALE being max capacity
735 * single CPU. 735 * for a single CPU.
736 */ 736 */
737 unsigned int power, power_orig; 737 unsigned int capacity, capacity_orig;
738 unsigned long next_update; 738 unsigned long next_update;
739 int imbalance; /* XXX unrelated to power but shared group state */ 739 int imbalance; /* XXX unrelated to capacity but shared group state */
740 /* 740 /*
741 * Number of busy cpus in this group. 741 * Number of busy cpus in this group.
742 */ 742 */
@@ -750,7 +750,7 @@ struct sched_group {
750 atomic_t ref; 750 atomic_t ref;
751 751
752 unsigned int group_weight; 752 unsigned int group_weight;
753 struct sched_group_power *sgp; 753 struct sched_group_capacity *sgc;
754 754
755 /* 755 /*
756 * The CPUs this group covers. 756 * The CPUs this group covers.
@@ -773,7 +773,7 @@ static inline struct cpumask *sched_group_cpus(struct sched_group *sg)
773 */ 773 */
774static inline struct cpumask *sched_group_mask(struct sched_group *sg) 774static inline struct cpumask *sched_group_mask(struct sched_group *sg)
775{ 775{
776 return to_cpumask(sg->sgp->cpumask); 776 return to_cpumask(sg->sgc->cpumask);
777} 777}
778 778
779/** 779/**
@@ -1167,7 +1167,7 @@ extern const struct sched_class idle_sched_class;
1167 1167
1168#ifdef CONFIG_SMP 1168#ifdef CONFIG_SMP
1169 1169
1170extern void update_group_power(struct sched_domain *sd, int cpu); 1170extern void update_group_capacity(struct sched_domain *sd, int cpu);
1171 1171
1172extern void trigger_load_balance(struct rq *rq); 1172extern void trigger_load_balance(struct rq *rq);
1173 1173