aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched.c')
-rw-r--r--kernel/sched.c1105
1 files changed, 631 insertions, 474 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index fff1c4a20b65..deb5ac8c12f3 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -498,18 +498,26 @@ struct rt_rq {
498 */ 498 */
499struct root_domain { 499struct root_domain {
500 atomic_t refcount; 500 atomic_t refcount;
501 cpumask_t span; 501 cpumask_var_t span;
502 cpumask_t online; 502 cpumask_var_t online;
503 503
504 /* 504 /*
505 * The "RT overload" flag: it gets set if a CPU has more than 505 * The "RT overload" flag: it gets set if a CPU has more than
506 * one runnable RT task. 506 * one runnable RT task.
507 */ 507 */
508 cpumask_t rto_mask; 508 cpumask_var_t rto_mask;
509 atomic_t rto_count; 509 atomic_t rto_count;
510#ifdef CONFIG_SMP 510#ifdef CONFIG_SMP
511 struct cpupri cpupri; 511 struct cpupri cpupri;
512#endif 512#endif
513#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
514 /*
515 * Preferred wake up cpu nominated by sched_mc balance that will be
516 * used when most cpus are idle in the system indicating overall very
517 * low system utilisation. Triggered at POWERSAVINGS_BALANCE_WAKEUP(2)
518 */
519 unsigned int sched_mc_preferred_wakeup_cpu;
520#endif
513}; 521};
514 522
515/* 523/*
@@ -1514,7 +1522,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
1514 struct sched_domain *sd = data; 1522 struct sched_domain *sd = data;
1515 int i; 1523 int i;
1516 1524
1517 for_each_cpu_mask(i, sd->span) { 1525 for_each_cpu(i, sched_domain_span(sd)) {
1518 /* 1526 /*
1519 * If there are currently no tasks on the cpu pretend there 1527 * If there are currently no tasks on the cpu pretend there
1520 * is one of average load so that when a new task gets to 1528 * is one of average load so that when a new task gets to
@@ -1535,7 +1543,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
1535 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE)) 1543 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
1536 shares = tg->shares; 1544 shares = tg->shares;
1537 1545
1538 for_each_cpu_mask(i, sd->span) 1546 for_each_cpu(i, sched_domain_span(sd))
1539 update_group_shares_cpu(tg, i, shares, rq_weight); 1547 update_group_shares_cpu(tg, i, shares, rq_weight);
1540 1548
1541 return 0; 1549 return 0;
@@ -2101,15 +2109,17 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2101 int i; 2109 int i;
2102 2110
2103 /* Skip over this group if it has no CPUs allowed */ 2111 /* Skip over this group if it has no CPUs allowed */
2104 if (!cpus_intersects(group->cpumask, p->cpus_allowed)) 2112 if (!cpumask_intersects(sched_group_cpus(group),
2113 &p->cpus_allowed))
2105 continue; 2114 continue;
2106 2115
2107 local_group = cpu_isset(this_cpu, group->cpumask); 2116 local_group = cpumask_test_cpu(this_cpu,
2117 sched_group_cpus(group));
2108 2118
2109 /* Tally up the load of all CPUs in the group */ 2119 /* Tally up the load of all CPUs in the group */
2110 avg_load = 0; 2120 avg_load = 0;
2111 2121
2112 for_each_cpu_mask_nr(i, group->cpumask) { 2122 for_each_cpu(i, sched_group_cpus(group)) {
2113 /* Bias balancing toward cpus of our domain */ 2123 /* Bias balancing toward cpus of our domain */
2114 if (local_group) 2124 if (local_group)
2115 load = source_load(i, load_idx); 2125 load = source_load(i, load_idx);
@@ -2141,17 +2151,14 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2141 * find_idlest_cpu - find the idlest cpu among the cpus in group. 2151 * find_idlest_cpu - find the idlest cpu among the cpus in group.
2142 */ 2152 */
2143static int 2153static int
2144find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu, 2154find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
2145 cpumask_t *tmp)
2146{ 2155{
2147 unsigned long load, min_load = ULONG_MAX; 2156 unsigned long load, min_load = ULONG_MAX;
2148 int idlest = -1; 2157 int idlest = -1;
2149 int i; 2158 int i;
2150 2159
2151 /* Traverse only the allowed CPUs */ 2160 /* Traverse only the allowed CPUs */
2152 cpus_and(*tmp, group->cpumask, p->cpus_allowed); 2161 for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
2153
2154 for_each_cpu_mask_nr(i, *tmp) {
2155 load = weighted_cpuload(i); 2162 load = weighted_cpuload(i);
2156 2163
2157 if (load < min_load || (load == min_load && i == this_cpu)) { 2164 if (load < min_load || (load == min_load && i == this_cpu)) {
@@ -2193,7 +2200,6 @@ static int sched_balance_self(int cpu, int flag)
2193 update_shares(sd); 2200 update_shares(sd);
2194 2201
2195 while (sd) { 2202 while (sd) {
2196 cpumask_t span, tmpmask;
2197 struct sched_group *group; 2203 struct sched_group *group;
2198 int new_cpu, weight; 2204 int new_cpu, weight;
2199 2205
@@ -2202,14 +2208,13 @@ static int sched_balance_self(int cpu, int flag)
2202 continue; 2208 continue;
2203 } 2209 }
2204 2210
2205 span = sd->span;
2206 group = find_idlest_group(sd, t, cpu); 2211 group = find_idlest_group(sd, t, cpu);
2207 if (!group) { 2212 if (!group) {
2208 sd = sd->child; 2213 sd = sd->child;
2209 continue; 2214 continue;
2210 } 2215 }
2211 2216
2212 new_cpu = find_idlest_cpu(group, t, cpu, &tmpmask); 2217 new_cpu = find_idlest_cpu(group, t, cpu);
2213 if (new_cpu == -1 || new_cpu == cpu) { 2218 if (new_cpu == -1 || new_cpu == cpu) {
2214 /* Now try balancing at a lower domain level of cpu */ 2219 /* Now try balancing at a lower domain level of cpu */
2215 sd = sd->child; 2220 sd = sd->child;
@@ -2218,10 +2223,10 @@ static int sched_balance_self(int cpu, int flag)
2218 2223
2219 /* Now try balancing at a lower domain level of new_cpu */ 2224 /* Now try balancing at a lower domain level of new_cpu */
2220 cpu = new_cpu; 2225 cpu = new_cpu;
2226 weight = cpumask_weight(sched_domain_span(sd));
2221 sd = NULL; 2227 sd = NULL;
2222 weight = cpus_weight(span);
2223 for_each_domain(cpu, tmp) { 2228 for_each_domain(cpu, tmp) {
2224 if (weight <= cpus_weight(tmp->span)) 2229 if (weight <= cpumask_weight(sched_domain_span(tmp)))
2225 break; 2230 break;
2226 if (tmp->flags & flag) 2231 if (tmp->flags & flag)
2227 sd = tmp; 2232 sd = tmp;
@@ -2266,7 +2271,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2266 cpu = task_cpu(p); 2271 cpu = task_cpu(p);
2267 2272
2268 for_each_domain(this_cpu, sd) { 2273 for_each_domain(this_cpu, sd) {
2269 if (cpu_isset(cpu, sd->span)) { 2274 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2270 update_shares(sd); 2275 update_shares(sd);
2271 break; 2276 break;
2272 } 2277 }
@@ -2315,7 +2320,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2315 else { 2320 else {
2316 struct sched_domain *sd; 2321 struct sched_domain *sd;
2317 for_each_domain(this_cpu, sd) { 2322 for_each_domain(this_cpu, sd) {
2318 if (cpu_isset(cpu, sd->span)) { 2323 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2319 schedstat_inc(sd, ttwu_wake_remote); 2324 schedstat_inc(sd, ttwu_wake_remote);
2320 break; 2325 break;
2321 } 2326 }
@@ -2846,7 +2851,7 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu)
2846 struct rq *rq; 2851 struct rq *rq;
2847 2852
2848 rq = task_rq_lock(p, &flags); 2853 rq = task_rq_lock(p, &flags);
2849 if (!cpu_isset(dest_cpu, p->cpus_allowed) 2854 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
2850 || unlikely(!cpu_active(dest_cpu))) 2855 || unlikely(!cpu_active(dest_cpu)))
2851 goto out; 2856 goto out;
2852 2857
@@ -2911,7 +2916,7 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
2911 * 2) cannot be migrated to this CPU due to cpus_allowed, or 2916 * 2) cannot be migrated to this CPU due to cpus_allowed, or
2912 * 3) are cache-hot on their current CPU. 2917 * 3) are cache-hot on their current CPU.
2913 */ 2918 */
2914 if (!cpu_isset(this_cpu, p->cpus_allowed)) { 2919 if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
2915 schedstat_inc(p, se.nr_failed_migrations_affine); 2920 schedstat_inc(p, se.nr_failed_migrations_affine);
2916 return 0; 2921 return 0;
2917 } 2922 }
@@ -3086,7 +3091,7 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
3086static struct sched_group * 3091static struct sched_group *
3087find_busiest_group(struct sched_domain *sd, int this_cpu, 3092find_busiest_group(struct sched_domain *sd, int this_cpu,
3088 unsigned long *imbalance, enum cpu_idle_type idle, 3093 unsigned long *imbalance, enum cpu_idle_type idle,
3089 int *sd_idle, const cpumask_t *cpus, int *balance) 3094 int *sd_idle, const struct cpumask *cpus, int *balance)
3090{ 3095{
3091 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups; 3096 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
3092 unsigned long max_load, avg_load, total_load, this_load, total_pwr; 3097 unsigned long max_load, avg_load, total_load, this_load, total_pwr;
@@ -3122,10 +3127,11 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3122 unsigned long sum_avg_load_per_task; 3127 unsigned long sum_avg_load_per_task;
3123 unsigned long avg_load_per_task; 3128 unsigned long avg_load_per_task;
3124 3129
3125 local_group = cpu_isset(this_cpu, group->cpumask); 3130 local_group = cpumask_test_cpu(this_cpu,
3131 sched_group_cpus(group));
3126 3132
3127 if (local_group) 3133 if (local_group)
3128 balance_cpu = first_cpu(group->cpumask); 3134 balance_cpu = cpumask_first(sched_group_cpus(group));
3129 3135
3130 /* Tally up the load of all CPUs in the group */ 3136 /* Tally up the load of all CPUs in the group */
3131 sum_weighted_load = sum_nr_running = avg_load = 0; 3137 sum_weighted_load = sum_nr_running = avg_load = 0;
@@ -3134,13 +3140,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3134 max_cpu_load = 0; 3140 max_cpu_load = 0;
3135 min_cpu_load = ~0UL; 3141 min_cpu_load = ~0UL;
3136 3142
3137 for_each_cpu_mask_nr(i, group->cpumask) { 3143 for_each_cpu_and(i, sched_group_cpus(group), cpus) {
3138 struct rq *rq; 3144 struct rq *rq = cpu_rq(i);
3139
3140 if (!cpu_isset(i, *cpus))
3141 continue;
3142
3143 rq = cpu_rq(i);
3144 3145
3145 if (*sd_idle && rq->nr_running) 3146 if (*sd_idle && rq->nr_running)
3146 *sd_idle = 0; 3147 *sd_idle = 0;
@@ -3251,8 +3252,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3251 */ 3252 */
3252 if ((sum_nr_running < min_nr_running) || 3253 if ((sum_nr_running < min_nr_running) ||
3253 (sum_nr_running == min_nr_running && 3254 (sum_nr_running == min_nr_running &&
3254 first_cpu(group->cpumask) < 3255 cpumask_first(sched_group_cpus(group)) >
3255 first_cpu(group_min->cpumask))) { 3256 cpumask_first(sched_group_cpus(group_min)))) {
3256 group_min = group; 3257 group_min = group;
3257 min_nr_running = sum_nr_running; 3258 min_nr_running = sum_nr_running;
3258 min_load_per_task = sum_weighted_load / 3259 min_load_per_task = sum_weighted_load /
@@ -3267,8 +3268,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3267 if (sum_nr_running <= group_capacity - 1) { 3268 if (sum_nr_running <= group_capacity - 1) {
3268 if (sum_nr_running > leader_nr_running || 3269 if (sum_nr_running > leader_nr_running ||
3269 (sum_nr_running == leader_nr_running && 3270 (sum_nr_running == leader_nr_running &&
3270 first_cpu(group->cpumask) > 3271 cpumask_first(sched_group_cpus(group)) <
3271 first_cpu(group_leader->cpumask))) { 3272 cpumask_first(sched_group_cpus(group_leader)))) {
3272 group_leader = group; 3273 group_leader = group;
3273 leader_nr_running = sum_nr_running; 3274 leader_nr_running = sum_nr_running;
3274 } 3275 }
@@ -3394,6 +3395,10 @@ out_balanced:
3394 3395
3395 if (this == group_leader && group_leader != group_min) { 3396 if (this == group_leader && group_leader != group_min) {
3396 *imbalance = min_load_per_task; 3397 *imbalance = min_load_per_task;
3398 if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP) {
3399 cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu =
3400 cpumask_first(sched_group_cpus(group_leader));
3401 }
3397 return group_min; 3402 return group_min;
3398 } 3403 }
3399#endif 3404#endif
@@ -3407,16 +3412,16 @@ ret:
3407 */ 3412 */
3408static struct rq * 3413static struct rq *
3409find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle, 3414find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3410 unsigned long imbalance, const cpumask_t *cpus) 3415 unsigned long imbalance, const struct cpumask *cpus)
3411{ 3416{
3412 struct rq *busiest = NULL, *rq; 3417 struct rq *busiest = NULL, *rq;
3413 unsigned long max_load = 0; 3418 unsigned long max_load = 0;
3414 int i; 3419 int i;
3415 3420
3416 for_each_cpu_mask_nr(i, group->cpumask) { 3421 for_each_cpu(i, sched_group_cpus(group)) {
3417 unsigned long wl; 3422 unsigned long wl;
3418 3423
3419 if (!cpu_isset(i, *cpus)) 3424 if (!cpumask_test_cpu(i, cpus))
3420 continue; 3425 continue;
3421 3426
3422 rq = cpu_rq(i); 3427 rq = cpu_rq(i);
@@ -3446,7 +3451,7 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3446 */ 3451 */
3447static int load_balance(int this_cpu, struct rq *this_rq, 3452static int load_balance(int this_cpu, struct rq *this_rq,
3448 struct sched_domain *sd, enum cpu_idle_type idle, 3453 struct sched_domain *sd, enum cpu_idle_type idle,
3449 int *balance, cpumask_t *cpus) 3454 int *balance, struct cpumask *cpus)
3450{ 3455{
3451 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0; 3456 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
3452 struct sched_group *group; 3457 struct sched_group *group;
@@ -3454,7 +3459,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
3454 struct rq *busiest; 3459 struct rq *busiest;
3455 unsigned long flags; 3460 unsigned long flags;
3456 3461
3457 cpus_setall(*cpus); 3462 cpumask_setall(cpus);
3458 3463
3459 /* 3464 /*
3460 * When power savings policy is enabled for the parent domain, idle 3465 * When power savings policy is enabled for the parent domain, idle
@@ -3514,8 +3519,8 @@ redo:
3514 3519
3515 /* All tasks on this runqueue were pinned by CPU affinity */ 3520 /* All tasks on this runqueue were pinned by CPU affinity */
3516 if (unlikely(all_pinned)) { 3521 if (unlikely(all_pinned)) {
3517 cpu_clear(cpu_of(busiest), *cpus); 3522 cpumask_clear_cpu(cpu_of(busiest), cpus);
3518 if (!cpus_empty(*cpus)) 3523 if (!cpumask_empty(cpus))
3519 goto redo; 3524 goto redo;
3520 goto out_balanced; 3525 goto out_balanced;
3521 } 3526 }
@@ -3532,7 +3537,8 @@ redo:
3532 /* don't kick the migration_thread, if the curr 3537 /* don't kick the migration_thread, if the curr
3533 * task on busiest cpu can't be moved to this_cpu 3538 * task on busiest cpu can't be moved to this_cpu
3534 */ 3539 */
3535 if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) { 3540 if (!cpumask_test_cpu(this_cpu,
3541 &busiest->curr->cpus_allowed)) {
3536 spin_unlock_irqrestore(&busiest->lock, flags); 3542 spin_unlock_irqrestore(&busiest->lock, flags);
3537 all_pinned = 1; 3543 all_pinned = 1;
3538 goto out_one_pinned; 3544 goto out_one_pinned;
@@ -3607,7 +3613,7 @@ out:
3607 */ 3613 */
3608static int 3614static int
3609load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd, 3615load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3610 cpumask_t *cpus) 3616 struct cpumask *cpus)
3611{ 3617{
3612 struct sched_group *group; 3618 struct sched_group *group;
3613 struct rq *busiest = NULL; 3619 struct rq *busiest = NULL;
@@ -3616,7 +3622,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3616 int sd_idle = 0; 3622 int sd_idle = 0;
3617 int all_pinned = 0; 3623 int all_pinned = 0;
3618 3624
3619 cpus_setall(*cpus); 3625 cpumask_setall(cpus);
3620 3626
3621 /* 3627 /*
3622 * When power savings policy is enabled for the parent domain, idle 3628 * When power savings policy is enabled for the parent domain, idle
@@ -3660,17 +3666,76 @@ redo:
3660 double_unlock_balance(this_rq, busiest); 3666 double_unlock_balance(this_rq, busiest);
3661 3667
3662 if (unlikely(all_pinned)) { 3668 if (unlikely(all_pinned)) {
3663 cpu_clear(cpu_of(busiest), *cpus); 3669 cpumask_clear_cpu(cpu_of(busiest), cpus);
3664 if (!cpus_empty(*cpus)) 3670 if (!cpumask_empty(cpus))
3665 goto redo; 3671 goto redo;
3666 } 3672 }
3667 } 3673 }
3668 3674
3669 if (!ld_moved) { 3675 if (!ld_moved) {
3676 int active_balance = 0;
3677
3670 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]); 3678 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
3671 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER && 3679 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3672 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE)) 3680 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3673 return -1; 3681 return -1;
3682
3683 if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
3684 return -1;
3685
3686 if (sd->nr_balance_failed++ < 2)
3687 return -1;
3688
3689 /*
3690 * The only task running in a non-idle cpu can be moved to this
3691 * cpu in an attempt to completely freeup the other CPU
3692 * package. The same method used to move task in load_balance()
3693 * have been extended for load_balance_newidle() to speedup
3694 * consolidation at sched_mc=POWERSAVINGS_BALANCE_WAKEUP (2)
3695 *
3696 * The package power saving logic comes from
3697 * find_busiest_group(). If there are no imbalance, then
3698 * f_b_g() will return NULL. However when sched_mc={1,2} then
3699 * f_b_g() will select a group from which a running task may be
3700 * pulled to this cpu in order to make the other package idle.
3701 * If there is no opportunity to make a package idle and if
3702 * there are no imbalance, then f_b_g() will return NULL and no
3703 * action will be taken in load_balance_newidle().
3704 *
3705 * Under normal task pull operation due to imbalance, there
3706 * will be more than one task in the source run queue and
3707 * move_tasks() will succeed. ld_moved will be true and this
3708 * active balance code will not be triggered.
3709 */
3710
3711 /* Lock busiest in correct order while this_rq is held */
3712 double_lock_balance(this_rq, busiest);
3713
3714 /*
3715 * don't kick the migration_thread, if the curr
3716 * task on busiest cpu can't be moved to this_cpu
3717 */
3718 if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) {
3719 double_unlock_balance(this_rq, busiest);
3720 all_pinned = 1;
3721 return ld_moved;
3722 }
3723
3724 if (!busiest->active_balance) {
3725 busiest->active_balance = 1;
3726 busiest->push_cpu = this_cpu;
3727 active_balance = 1;
3728 }
3729
3730 double_unlock_balance(this_rq, busiest);
3731 /*
3732 * Should not call ttwu while holding a rq->lock
3733 */
3734 spin_unlock(&this_rq->lock);
3735 if (active_balance)
3736 wake_up_process(busiest->migration_thread);
3737 spin_lock(&this_rq->lock);
3738
3674 } else 3739 } else
3675 sd->nr_balance_failed = 0; 3740 sd->nr_balance_failed = 0;
3676 3741
@@ -3696,7 +3761,10 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3696 struct sched_domain *sd; 3761 struct sched_domain *sd;
3697 int pulled_task = 0; 3762 int pulled_task = 0;
3698 unsigned long next_balance = jiffies + HZ; 3763 unsigned long next_balance = jiffies + HZ;
3699 cpumask_t tmpmask; 3764 cpumask_var_t tmpmask;
3765
3766 if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
3767 return;
3700 3768
3701 for_each_domain(this_cpu, sd) { 3769 for_each_domain(this_cpu, sd) {
3702 unsigned long interval; 3770 unsigned long interval;
@@ -3707,7 +3775,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3707 if (sd->flags & SD_BALANCE_NEWIDLE) 3775 if (sd->flags & SD_BALANCE_NEWIDLE)
3708 /* If we've pulled tasks over stop searching: */ 3776 /* If we've pulled tasks over stop searching: */
3709 pulled_task = load_balance_newidle(this_cpu, this_rq, 3777 pulled_task = load_balance_newidle(this_cpu, this_rq,
3710 sd, &tmpmask); 3778 sd, tmpmask);
3711 3779
3712 interval = msecs_to_jiffies(sd->balance_interval); 3780 interval = msecs_to_jiffies(sd->balance_interval);
3713 if (time_after(next_balance, sd->last_balance + interval)) 3781 if (time_after(next_balance, sd->last_balance + interval))
@@ -3722,6 +3790,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3722 */ 3790 */
3723 this_rq->next_balance = next_balance; 3791 this_rq->next_balance = next_balance;
3724 } 3792 }
3793 free_cpumask_var(tmpmask);
3725} 3794}
3726 3795
3727/* 3796/*
@@ -3759,7 +3828,7 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3759 /* Search for an sd spanning us and the target CPU. */ 3828 /* Search for an sd spanning us and the target CPU. */
3760 for_each_domain(target_cpu, sd) { 3829 for_each_domain(target_cpu, sd) {
3761 if ((sd->flags & SD_LOAD_BALANCE) && 3830 if ((sd->flags & SD_LOAD_BALANCE) &&
3762 cpu_isset(busiest_cpu, sd->span)) 3831 cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
3763 break; 3832 break;
3764 } 3833 }
3765 3834
@@ -3778,10 +3847,9 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3778#ifdef CONFIG_NO_HZ 3847#ifdef CONFIG_NO_HZ
3779static struct { 3848static struct {
3780 atomic_t load_balancer; 3849 atomic_t load_balancer;
3781 cpumask_t cpu_mask; 3850 cpumask_var_t cpu_mask;
3782} nohz ____cacheline_aligned = { 3851} nohz ____cacheline_aligned = {
3783 .load_balancer = ATOMIC_INIT(-1), 3852 .load_balancer = ATOMIC_INIT(-1),
3784 .cpu_mask = CPU_MASK_NONE,
3785}; 3853};
3786 3854
3787/* 3855/*
@@ -3809,7 +3877,7 @@ int select_nohz_load_balancer(int stop_tick)
3809 int cpu = smp_processor_id(); 3877 int cpu = smp_processor_id();
3810 3878
3811 if (stop_tick) { 3879 if (stop_tick) {
3812 cpu_set(cpu, nohz.cpu_mask); 3880 cpumask_set_cpu(cpu, nohz.cpu_mask);
3813 cpu_rq(cpu)->in_nohz_recently = 1; 3881 cpu_rq(cpu)->in_nohz_recently = 1;
3814 3882
3815 /* 3883 /*
@@ -3823,7 +3891,7 @@ int select_nohz_load_balancer(int stop_tick)
3823 } 3891 }
3824 3892
3825 /* time for ilb owner also to sleep */ 3893 /* time for ilb owner also to sleep */
3826 if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 3894 if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
3827 if (atomic_read(&nohz.load_balancer) == cpu) 3895 if (atomic_read(&nohz.load_balancer) == cpu)
3828 atomic_set(&nohz.load_balancer, -1); 3896 atomic_set(&nohz.load_balancer, -1);
3829 return 0; 3897 return 0;
@@ -3836,10 +3904,10 @@ int select_nohz_load_balancer(int stop_tick)
3836 } else if (atomic_read(&nohz.load_balancer) == cpu) 3904 } else if (atomic_read(&nohz.load_balancer) == cpu)
3837 return 1; 3905 return 1;
3838 } else { 3906 } else {
3839 if (!cpu_isset(cpu, nohz.cpu_mask)) 3907 if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
3840 return 0; 3908 return 0;
3841 3909
3842 cpu_clear(cpu, nohz.cpu_mask); 3910 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3843 3911
3844 if (atomic_read(&nohz.load_balancer) == cpu) 3912 if (atomic_read(&nohz.load_balancer) == cpu)
3845 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu) 3913 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
@@ -3867,7 +3935,11 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3867 unsigned long next_balance = jiffies + 60*HZ; 3935 unsigned long next_balance = jiffies + 60*HZ;
3868 int update_next_balance = 0; 3936 int update_next_balance = 0;
3869 int need_serialize; 3937 int need_serialize;
3870 cpumask_t tmp; 3938 cpumask_var_t tmp;
3939
3940 /* Fails alloc? Rebalancing probably not a priority right now. */
3941 if (!alloc_cpumask_var(&tmp, GFP_ATOMIC))
3942 return;
3871 3943
3872 for_each_domain(cpu, sd) { 3944 for_each_domain(cpu, sd) {
3873 if (!(sd->flags & SD_LOAD_BALANCE)) 3945 if (!(sd->flags & SD_LOAD_BALANCE))
@@ -3892,7 +3964,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3892 } 3964 }
3893 3965
3894 if (time_after_eq(jiffies, sd->last_balance + interval)) { 3966 if (time_after_eq(jiffies, sd->last_balance + interval)) {
3895 if (load_balance(cpu, rq, sd, idle, &balance, &tmp)) { 3967 if (load_balance(cpu, rq, sd, idle, &balance, tmp)) {
3896 /* 3968 /*
3897 * We've pulled tasks over so either we're no 3969 * We've pulled tasks over so either we're no
3898 * longer idle, or one of our SMT siblings is 3970 * longer idle, or one of our SMT siblings is
@@ -3926,6 +3998,8 @@ out:
3926 */ 3998 */
3927 if (likely(update_next_balance)) 3999 if (likely(update_next_balance))
3928 rq->next_balance = next_balance; 4000 rq->next_balance = next_balance;
4001
4002 free_cpumask_var(tmp);
3929} 4003}
3930 4004
3931/* 4005/*
@@ -3950,12 +4024,13 @@ static void run_rebalance_domains(struct softirq_action *h)
3950 */ 4024 */
3951 if (this_rq->idle_at_tick && 4025 if (this_rq->idle_at_tick &&
3952 atomic_read(&nohz.load_balancer) == this_cpu) { 4026 atomic_read(&nohz.load_balancer) == this_cpu) {
3953 cpumask_t cpus = nohz.cpu_mask;
3954 struct rq *rq; 4027 struct rq *rq;
3955 int balance_cpu; 4028 int balance_cpu;
3956 4029
3957 cpu_clear(this_cpu, cpus); 4030 for_each_cpu(balance_cpu, nohz.cpu_mask) {
3958 for_each_cpu_mask_nr(balance_cpu, cpus) { 4031 if (balance_cpu == this_cpu)
4032 continue;
4033
3959 /* 4034 /*
3960 * If this cpu gets work to do, stop the load balancing 4035 * If this cpu gets work to do, stop the load balancing
3961 * work being done for other cpus. Next load 4036 * work being done for other cpus. Next load
@@ -3993,7 +4068,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
3993 rq->in_nohz_recently = 0; 4068 rq->in_nohz_recently = 0;
3994 4069
3995 if (atomic_read(&nohz.load_balancer) == cpu) { 4070 if (atomic_read(&nohz.load_balancer) == cpu) {
3996 cpu_clear(cpu, nohz.cpu_mask); 4071 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3997 atomic_set(&nohz.load_balancer, -1); 4072 atomic_set(&nohz.load_balancer, -1);
3998 } 4073 }
3999 4074
@@ -4006,7 +4081,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4006 * TBD: Traverse the sched domains and nominate 4081 * TBD: Traverse the sched domains and nominate
4007 * the nearest cpu in the nohz.cpu_mask. 4082 * the nearest cpu in the nohz.cpu_mask.
4008 */ 4083 */
4009 int ilb = first_cpu(nohz.cpu_mask); 4084 int ilb = cpumask_first(nohz.cpu_mask);
4010 4085
4011 if (ilb < nr_cpu_ids) 4086 if (ilb < nr_cpu_ids)
4012 resched_cpu(ilb); 4087 resched_cpu(ilb);
@@ -4018,7 +4093,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4018 * cpus with ticks stopped, is it time for that to stop? 4093 * cpus with ticks stopped, is it time for that to stop?
4019 */ 4094 */
4020 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu && 4095 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
4021 cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 4096 cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
4022 resched_cpu(cpu); 4097 resched_cpu(cpu);
4023 return; 4098 return;
4024 } 4099 }
@@ -4028,7 +4103,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4028 * someone else, then no need raise the SCHED_SOFTIRQ 4103 * someone else, then no need raise the SCHED_SOFTIRQ
4029 */ 4104 */
4030 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu && 4105 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
4031 cpu_isset(cpu, nohz.cpu_mask)) 4106 cpumask_test_cpu(cpu, nohz.cpu_mask))
4032 return; 4107 return;
4033#endif 4108#endif
4034 if (time_after_eq(jiffies, rq->next_balance)) 4109 if (time_after_eq(jiffies, rq->next_balance))
@@ -4080,13 +4155,17 @@ unsigned long long task_delta_exec(struct task_struct *p)
4080 * Account user cpu time to a process. 4155 * Account user cpu time to a process.
4081 * @p: the process that the cpu time gets accounted to 4156 * @p: the process that the cpu time gets accounted to
4082 * @cputime: the cpu time spent in user space since the last update 4157 * @cputime: the cpu time spent in user space since the last update
4158 * @cputime_scaled: cputime scaled by cpu frequency
4083 */ 4159 */
4084void account_user_time(struct task_struct *p, cputime_t cputime) 4160void account_user_time(struct task_struct *p, cputime_t cputime,
4161 cputime_t cputime_scaled)
4085{ 4162{
4086 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; 4163 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4087 cputime64_t tmp; 4164 cputime64_t tmp;
4088 4165
4166 /* Add user time to process. */
4089 p->utime = cputime_add(p->utime, cputime); 4167 p->utime = cputime_add(p->utime, cputime);
4168 p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
4090 account_group_user_time(p, cputime); 4169 account_group_user_time(p, cputime);
4091 4170
4092 /* Add user time to cpustat. */ 4171 /* Add user time to cpustat. */
@@ -4103,51 +4182,48 @@ void account_user_time(struct task_struct *p, cputime_t cputime)
4103 * Account guest cpu time to a process. 4182 * Account guest cpu time to a process.
4104 * @p: the process that the cpu time gets accounted to 4183 * @p: the process that the cpu time gets accounted to
4105 * @cputime: the cpu time spent in virtual machine since the last update 4184 * @cputime: the cpu time spent in virtual machine since the last update
4185 * @cputime_scaled: cputime scaled by cpu frequency
4106 */ 4186 */
4107static void account_guest_time(struct task_struct *p, cputime_t cputime) 4187static void account_guest_time(struct task_struct *p, cputime_t cputime,
4188 cputime_t cputime_scaled)
4108{ 4189{
4109 cputime64_t tmp; 4190 cputime64_t tmp;
4110 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; 4191 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4111 4192
4112 tmp = cputime_to_cputime64(cputime); 4193 tmp = cputime_to_cputime64(cputime);
4113 4194
4195 /* Add guest time to process. */
4114 p->utime = cputime_add(p->utime, cputime); 4196 p->utime = cputime_add(p->utime, cputime);
4197 p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
4115 account_group_user_time(p, cputime); 4198 account_group_user_time(p, cputime);
4116 p->gtime = cputime_add(p->gtime, cputime); 4199 p->gtime = cputime_add(p->gtime, cputime);
4117 4200
4201 /* Add guest time to cpustat. */
4118 cpustat->user = cputime64_add(cpustat->user, tmp); 4202 cpustat->user = cputime64_add(cpustat->user, tmp);
4119 cpustat->guest = cputime64_add(cpustat->guest, tmp); 4203 cpustat->guest = cputime64_add(cpustat->guest, tmp);
4120} 4204}
4121 4205
4122/* 4206/*
4123 * Account scaled user cpu time to a process.
4124 * @p: the process that the cpu time gets accounted to
4125 * @cputime: the cpu time spent in user space since the last update
4126 */
4127void account_user_time_scaled(struct task_struct *p, cputime_t cputime)
4128{
4129 p->utimescaled = cputime_add(p->utimescaled, cputime);
4130}
4131
4132/*
4133 * Account system cpu time to a process. 4207 * Account system cpu time to a process.
4134 * @p: the process that the cpu time gets accounted to 4208 * @p: the process that the cpu time gets accounted to
4135 * @hardirq_offset: the offset to subtract from hardirq_count() 4209 * @hardirq_offset: the offset to subtract from hardirq_count()
4136 * @cputime: the cpu time spent in kernel space since the last update 4210 * @cputime: the cpu time spent in kernel space since the last update
4211 * @cputime_scaled: cputime scaled by cpu frequency
4137 */ 4212 */
4138void account_system_time(struct task_struct *p, int hardirq_offset, 4213void account_system_time(struct task_struct *p, int hardirq_offset,
4139 cputime_t cputime) 4214 cputime_t cputime, cputime_t cputime_scaled)
4140{ 4215{
4141 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; 4216 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4142 struct rq *rq = this_rq();
4143 cputime64_t tmp; 4217 cputime64_t tmp;
4144 4218
4145 if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) { 4219 if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
4146 account_guest_time(p, cputime); 4220 account_guest_time(p, cputime, cputime_scaled);
4147 return; 4221 return;
4148 } 4222 }
4149 4223
4224 /* Add system time to process. */
4150 p->stime = cputime_add(p->stime, cputime); 4225 p->stime = cputime_add(p->stime, cputime);
4226 p->stimescaled = cputime_add(p->stimescaled, cputime_scaled);
4151 account_group_system_time(p, cputime); 4227 account_group_system_time(p, cputime);
4152 4228
4153 /* Add system time to cpustat. */ 4229 /* Add system time to cpustat. */
@@ -4156,48 +4232,84 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
4156 cpustat->irq = cputime64_add(cpustat->irq, tmp); 4232 cpustat->irq = cputime64_add(cpustat->irq, tmp);
4157 else if (softirq_count()) 4233 else if (softirq_count())
4158 cpustat->softirq = cputime64_add(cpustat->softirq, tmp); 4234 cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
4159 else if (p != rq->idle)
4160 cpustat->system = cputime64_add(cpustat->system, tmp);
4161 else if (atomic_read(&rq->nr_iowait) > 0)
4162 cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
4163 else 4235 else
4164 cpustat->idle = cputime64_add(cpustat->idle, tmp); 4236 cpustat->system = cputime64_add(cpustat->system, tmp);
4237
4165 /* Account for system time used */ 4238 /* Account for system time used */
4166 acct_update_integrals(p); 4239 acct_update_integrals(p);
4167} 4240}
4168 4241
4169/* 4242/*
4170 * Account scaled system cpu time to a process. 4243 * Account for involuntary wait time.
4171 * @p: the process that the cpu time gets accounted to 4244 * @steal: the cpu time spent in involuntary wait
4172 * @hardirq_offset: the offset to subtract from hardirq_count()
4173 * @cputime: the cpu time spent in kernel space since the last update
4174 */ 4245 */
4175void account_system_time_scaled(struct task_struct *p, cputime_t cputime) 4246void account_steal_time(cputime_t cputime)
4176{ 4247{
4177 p->stimescaled = cputime_add(p->stimescaled, cputime); 4248 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4249 cputime64_t cputime64 = cputime_to_cputime64(cputime);
4250
4251 cpustat->steal = cputime64_add(cpustat->steal, cputime64);
4178} 4252}
4179 4253
4180/* 4254/*
4181 * Account for involuntary wait time. 4255 * Account for idle time.
4182 * @p: the process from which the cpu time has been stolen 4256 * @cputime: the cpu time spent in idle wait
4183 * @steal: the cpu time spent in involuntary wait
4184 */ 4257 */
4185void account_steal_time(struct task_struct *p, cputime_t steal) 4258void account_idle_time(cputime_t cputime)
4186{ 4259{
4187 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; 4260 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4188 cputime64_t tmp = cputime_to_cputime64(steal); 4261 cputime64_t cputime64 = cputime_to_cputime64(cputime);
4189 struct rq *rq = this_rq(); 4262 struct rq *rq = this_rq();
4190 4263
4191 if (p == rq->idle) { 4264 if (atomic_read(&rq->nr_iowait) > 0)
4192 p->stime = cputime_add(p->stime, steal); 4265 cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
4193 if (atomic_read(&rq->nr_iowait) > 0) 4266 else
4194 cpustat->iowait = cputime64_add(cpustat->iowait, tmp); 4267 cpustat->idle = cputime64_add(cpustat->idle, cputime64);
4195 else
4196 cpustat->idle = cputime64_add(cpustat->idle, tmp);
4197 } else
4198 cpustat->steal = cputime64_add(cpustat->steal, tmp);
4199} 4268}
4200 4269
4270#ifndef CONFIG_VIRT_CPU_ACCOUNTING
4271
4272/*
4273 * Account a single tick of cpu time.
4274 * @p: the process that the cpu time gets accounted to
4275 * @user_tick: indicates if the tick is a user or a system tick
4276 */
4277void account_process_tick(struct task_struct *p, int user_tick)
4278{
4279 cputime_t one_jiffy = jiffies_to_cputime(1);
4280 cputime_t one_jiffy_scaled = cputime_to_scaled(one_jiffy);
4281 struct rq *rq = this_rq();
4282
4283 if (user_tick)
4284 account_user_time(p, one_jiffy, one_jiffy_scaled);
4285 else if (p != rq->idle)
4286 account_system_time(p, HARDIRQ_OFFSET, one_jiffy,
4287 one_jiffy_scaled);
4288 else
4289 account_idle_time(one_jiffy);
4290}
4291
4292/*
4293 * Account multiple ticks of steal time.
4294 * @p: the process from which the cpu time has been stolen
4295 * @ticks: number of stolen ticks
4296 */
4297void account_steal_ticks(unsigned long ticks)
4298{
4299 account_steal_time(jiffies_to_cputime(ticks));
4300}
4301
4302/*
4303 * Account multiple ticks of idle time.
4304 * @ticks: number of stolen ticks
4305 */
4306void account_idle_ticks(unsigned long ticks)
4307{
4308 account_idle_time(jiffies_to_cputime(ticks));
4309}
4310
4311#endif
4312
4201/* 4313/*
4202 * Use precise platform statistics if available: 4314 * Use precise platform statistics if available:
4203 */ 4315 */
@@ -5401,10 +5513,9 @@ out_unlock:
5401 return retval; 5513 return retval;
5402} 5514}
5403 5515
5404long sched_setaffinity(pid_t pid, const cpumask_t *in_mask) 5516long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
5405{ 5517{
5406 cpumask_t cpus_allowed; 5518 cpumask_var_t cpus_allowed, new_mask;
5407 cpumask_t new_mask = *in_mask;
5408 struct task_struct *p; 5519 struct task_struct *p;
5409 int retval; 5520 int retval;
5410 5521
@@ -5426,6 +5537,14 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5426 get_task_struct(p); 5537 get_task_struct(p);
5427 read_unlock(&tasklist_lock); 5538 read_unlock(&tasklist_lock);
5428 5539
5540 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
5541 retval = -ENOMEM;
5542 goto out_put_task;
5543 }
5544 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
5545 retval = -ENOMEM;
5546 goto out_free_cpus_allowed;
5547 }
5429 retval = -EPERM; 5548 retval = -EPERM;
5430 if (!check_same_owner(p) && !capable(CAP_SYS_NICE)) 5549 if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
5431 goto out_unlock; 5550 goto out_unlock;
@@ -5434,37 +5553,41 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5434 if (retval) 5553 if (retval)
5435 goto out_unlock; 5554 goto out_unlock;
5436 5555
5437 cpuset_cpus_allowed(p, &cpus_allowed); 5556 cpuset_cpus_allowed(p, cpus_allowed);
5438 cpus_and(new_mask, new_mask, cpus_allowed); 5557 cpumask_and(new_mask, in_mask, cpus_allowed);
5439 again: 5558 again:
5440 retval = set_cpus_allowed_ptr(p, &new_mask); 5559 retval = set_cpus_allowed_ptr(p, new_mask);
5441 5560
5442 if (!retval) { 5561 if (!retval) {
5443 cpuset_cpus_allowed(p, &cpus_allowed); 5562 cpuset_cpus_allowed(p, cpus_allowed);
5444 if (!cpus_subset(new_mask, cpus_allowed)) { 5563 if (!cpumask_subset(new_mask, cpus_allowed)) {
5445 /* 5564 /*
5446 * We must have raced with a concurrent cpuset 5565 * We must have raced with a concurrent cpuset
5447 * update. Just reset the cpus_allowed to the 5566 * update. Just reset the cpus_allowed to the
5448 * cpuset's cpus_allowed 5567 * cpuset's cpus_allowed
5449 */ 5568 */
5450 new_mask = cpus_allowed; 5569 cpumask_copy(new_mask, cpus_allowed);
5451 goto again; 5570 goto again;
5452 } 5571 }
5453 } 5572 }
5454out_unlock: 5573out_unlock:
5574 free_cpumask_var(new_mask);
5575out_free_cpus_allowed:
5576 free_cpumask_var(cpus_allowed);
5577out_put_task:
5455 put_task_struct(p); 5578 put_task_struct(p);
5456 put_online_cpus(); 5579 put_online_cpus();
5457 return retval; 5580 return retval;
5458} 5581}
5459 5582
5460static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 5583static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5461 cpumask_t *new_mask) 5584 struct cpumask *new_mask)
5462{ 5585{
5463 if (len < sizeof(cpumask_t)) { 5586 if (len < cpumask_size())
5464 memset(new_mask, 0, sizeof(cpumask_t)); 5587 cpumask_clear(new_mask);
5465 } else if (len > sizeof(cpumask_t)) { 5588 else if (len > cpumask_size())
5466 len = sizeof(cpumask_t); 5589 len = cpumask_size();
5467 } 5590
5468 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 5591 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
5469} 5592}
5470 5593
@@ -5477,17 +5600,20 @@ static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5477asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, 5600asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
5478 unsigned long __user *user_mask_ptr) 5601 unsigned long __user *user_mask_ptr)
5479{ 5602{
5480 cpumask_t new_mask; 5603 cpumask_var_t new_mask;
5481 int retval; 5604 int retval;
5482 5605
5483 retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask); 5606 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
5484 if (retval) 5607 return -ENOMEM;
5485 return retval;
5486 5608
5487 return sched_setaffinity(pid, &new_mask); 5609 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
5610 if (retval == 0)
5611 retval = sched_setaffinity(pid, new_mask);
5612 free_cpumask_var(new_mask);
5613 return retval;
5488} 5614}
5489 5615
5490long sched_getaffinity(pid_t pid, cpumask_t *mask) 5616long sched_getaffinity(pid_t pid, struct cpumask *mask)
5491{ 5617{
5492 struct task_struct *p; 5618 struct task_struct *p;
5493 int retval; 5619 int retval;
@@ -5504,7 +5630,7 @@ long sched_getaffinity(pid_t pid, cpumask_t *mask)
5504 if (retval) 5630 if (retval)
5505 goto out_unlock; 5631 goto out_unlock;
5506 5632
5507 cpus_and(*mask, p->cpus_allowed, cpu_online_map); 5633 cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
5508 5634
5509out_unlock: 5635out_unlock:
5510 read_unlock(&tasklist_lock); 5636 read_unlock(&tasklist_lock);
@@ -5523,19 +5649,24 @@ asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
5523 unsigned long __user *user_mask_ptr) 5649 unsigned long __user *user_mask_ptr)
5524{ 5650{
5525 int ret; 5651 int ret;
5526 cpumask_t mask; 5652 cpumask_var_t mask;
5527 5653
5528 if (len < sizeof(cpumask_t)) 5654 if (len < cpumask_size())
5529 return -EINVAL; 5655 return -EINVAL;
5530 5656
5531 ret = sched_getaffinity(pid, &mask); 5657 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
5532 if (ret < 0) 5658 return -ENOMEM;
5533 return ret;
5534 5659
5535 if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t))) 5660 ret = sched_getaffinity(pid, mask);
5536 return -EFAULT; 5661 if (ret == 0) {
5662 if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
5663 ret = -EFAULT;
5664 else
5665 ret = cpumask_size();
5666 }
5667 free_cpumask_var(mask);
5537 5668
5538 return sizeof(cpumask_t); 5669 return ret;
5539} 5670}
5540 5671
5541/** 5672/**
@@ -5877,7 +6008,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5877 idle->se.exec_start = sched_clock(); 6008 idle->se.exec_start = sched_clock();
5878 6009
5879 idle->prio = idle->normal_prio = MAX_PRIO; 6010 idle->prio = idle->normal_prio = MAX_PRIO;
5880 idle->cpus_allowed = cpumask_of_cpu(cpu); 6011 cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
5881 __set_task_cpu(idle, cpu); 6012 __set_task_cpu(idle, cpu);
5882 6013
5883 rq->curr = rq->idle = idle; 6014 rq->curr = rq->idle = idle;
@@ -5904,9 +6035,9 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5904 * indicates which cpus entered this state. This is used 6035 * indicates which cpus entered this state. This is used
5905 * in the rcu update to wait only for active cpus. For system 6036 * in the rcu update to wait only for active cpus. For system
5906 * which do not switch off the HZ timer nohz_cpu_mask should 6037 * which do not switch off the HZ timer nohz_cpu_mask should
5907 * always be CPU_MASK_NONE. 6038 * always be CPU_BITS_NONE.
5908 */ 6039 */
5909cpumask_t nohz_cpu_mask = CPU_MASK_NONE; 6040cpumask_var_t nohz_cpu_mask;
5910 6041
5911/* 6042/*
5912 * Increase the granularity value when there are more CPUs, 6043 * Increase the granularity value when there are more CPUs,
@@ -5961,7 +6092,7 @@ static inline void sched_init_granularity(void)
5961 * task must not exit() & deallocate itself prematurely. The 6092 * task must not exit() & deallocate itself prematurely. The
5962 * call is not atomic; no spinlocks may be held. 6093 * call is not atomic; no spinlocks may be held.
5963 */ 6094 */
5964int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask) 6095int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
5965{ 6096{
5966 struct migration_req req; 6097 struct migration_req req;
5967 unsigned long flags; 6098 unsigned long flags;
@@ -5969,13 +6100,13 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5969 int ret = 0; 6100 int ret = 0;
5970 6101
5971 rq = task_rq_lock(p, &flags); 6102 rq = task_rq_lock(p, &flags);
5972 if (!cpus_intersects(*new_mask, cpu_online_map)) { 6103 if (!cpumask_intersects(new_mask, cpu_online_mask)) {
5973 ret = -EINVAL; 6104 ret = -EINVAL;
5974 goto out; 6105 goto out;
5975 } 6106 }
5976 6107
5977 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current && 6108 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
5978 !cpus_equal(p->cpus_allowed, *new_mask))) { 6109 !cpumask_equal(&p->cpus_allowed, new_mask))) {
5979 ret = -EINVAL; 6110 ret = -EINVAL;
5980 goto out; 6111 goto out;
5981 } 6112 }
@@ -5983,15 +6114,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5983 if (p->sched_class->set_cpus_allowed) 6114 if (p->sched_class->set_cpus_allowed)
5984 p->sched_class->set_cpus_allowed(p, new_mask); 6115 p->sched_class->set_cpus_allowed(p, new_mask);
5985 else { 6116 else {
5986 p->cpus_allowed = *new_mask; 6117 cpumask_copy(&p->cpus_allowed, new_mask);
5987 p->rt.nr_cpus_allowed = cpus_weight(*new_mask); 6118 p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
5988 } 6119 }
5989 6120
5990 /* Can the task run on the task's current CPU? If so, we're done */ 6121 /* Can the task run on the task's current CPU? If so, we're done */
5991 if (cpu_isset(task_cpu(p), *new_mask)) 6122 if (cpumask_test_cpu(task_cpu(p), new_mask))
5992 goto out; 6123 goto out;
5993 6124
5994 if (migrate_task(p, any_online_cpu(*new_mask), &req)) { 6125 if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
5995 /* Need help from migration thread: drop lock and wait. */ 6126 /* Need help from migration thread: drop lock and wait. */
5996 task_rq_unlock(rq, &flags); 6127 task_rq_unlock(rq, &flags);
5997 wake_up_process(rq->migration_thread); 6128 wake_up_process(rq->migration_thread);
@@ -6033,7 +6164,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
6033 if (task_cpu(p) != src_cpu) 6164 if (task_cpu(p) != src_cpu)
6034 goto done; 6165 goto done;
6035 /* Affinity changed (again). */ 6166 /* Affinity changed (again). */
6036 if (!cpu_isset(dest_cpu, p->cpus_allowed)) 6167 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6037 goto fail; 6168 goto fail;
6038 6169
6039 on_rq = p->se.on_rq; 6170 on_rq = p->se.on_rq;
@@ -6130,50 +6261,41 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
6130 */ 6261 */
6131static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) 6262static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6132{ 6263{
6133 unsigned long flags;
6134 cpumask_t mask;
6135 struct rq *rq;
6136 int dest_cpu; 6264 int dest_cpu;
6265 const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(dead_cpu));
6137 6266
6138 do { 6267again:
6139 /* On same node? */ 6268 /* Look for allowed, online CPU in same node. */
6140 mask = node_to_cpumask(cpu_to_node(dead_cpu)); 6269 for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
6141 cpus_and(mask, mask, p->cpus_allowed); 6270 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6142 dest_cpu = any_online_cpu(mask); 6271 goto move;
6143 6272
6144 /* On any allowed CPU? */ 6273 /* Any allowed, online CPU? */
6145 if (dest_cpu >= nr_cpu_ids) 6274 dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
6146 dest_cpu = any_online_cpu(p->cpus_allowed); 6275 if (dest_cpu < nr_cpu_ids)
6276 goto move;
6147 6277
6148 /* No more Mr. Nice Guy. */ 6278 /* No more Mr. Nice Guy. */
6149 if (dest_cpu >= nr_cpu_ids) { 6279 if (dest_cpu >= nr_cpu_ids) {
6150 cpumask_t cpus_allowed; 6280 cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
6281 dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
6151 6282
6152 cpuset_cpus_allowed_locked(p, &cpus_allowed); 6283 /*
6153 /* 6284 * Don't tell them about moving exiting tasks or
6154 * Try to stay on the same cpuset, where the 6285 * kernel threads (both mm NULL), since they never
6155 * current cpuset may be a subset of all cpus. 6286 * leave kernel.
6156 * The cpuset_cpus_allowed_locked() variant of 6287 */
6157 * cpuset_cpus_allowed() will not block. It must be 6288 if (p->mm && printk_ratelimit()) {
6158 * called within calls to cpuset_lock/cpuset_unlock. 6289 printk(KERN_INFO "process %d (%s) no "
6159 */ 6290 "longer affine to cpu%d\n",
6160 rq = task_rq_lock(p, &flags); 6291 task_pid_nr(p), p->comm, dead_cpu);
6161 p->cpus_allowed = cpus_allowed;
6162 dest_cpu = any_online_cpu(p->cpus_allowed);
6163 task_rq_unlock(rq, &flags);
6164
6165 /*
6166 * Don't tell them about moving exiting tasks or
6167 * kernel threads (both mm NULL), since they never
6168 * leave kernel.
6169 */
6170 if (p->mm && printk_ratelimit()) {
6171 printk(KERN_INFO "process %d (%s) no "
6172 "longer affine to cpu%d\n",
6173 task_pid_nr(p), p->comm, dead_cpu);
6174 }
6175 } 6292 }
6176 } while (!__migrate_task_irq(p, dead_cpu, dest_cpu)); 6293 }
6294
6295move:
6296 /* It can have affinity changed while we were choosing. */
6297 if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
6298 goto again;
6177} 6299}
6178 6300
6179/* 6301/*
@@ -6185,7 +6307,7 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6185 */ 6307 */
6186static void migrate_nr_uninterruptible(struct rq *rq_src) 6308static void migrate_nr_uninterruptible(struct rq *rq_src)
6187{ 6309{
6188 struct rq *rq_dest = cpu_rq(any_online_cpu(*CPU_MASK_ALL_PTR)); 6310 struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
6189 unsigned long flags; 6311 unsigned long flags;
6190 6312
6191 local_irq_save(flags); 6313 local_irq_save(flags);
@@ -6475,7 +6597,7 @@ static void set_rq_online(struct rq *rq)
6475 if (!rq->online) { 6597 if (!rq->online) {
6476 const struct sched_class *class; 6598 const struct sched_class *class;
6477 6599
6478 cpu_set(rq->cpu, rq->rd->online); 6600 cpumask_set_cpu(rq->cpu, rq->rd->online);
6479 rq->online = 1; 6601 rq->online = 1;
6480 6602
6481 for_each_class(class) { 6603 for_each_class(class) {
@@ -6495,7 +6617,7 @@ static void set_rq_offline(struct rq *rq)
6495 class->rq_offline(rq); 6617 class->rq_offline(rq);
6496 } 6618 }
6497 6619
6498 cpu_clear(rq->cpu, rq->rd->online); 6620 cpumask_clear_cpu(rq->cpu, rq->rd->online);
6499 rq->online = 0; 6621 rq->online = 0;
6500 } 6622 }
6501} 6623}
@@ -6536,7 +6658,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6536 rq = cpu_rq(cpu); 6658 rq = cpu_rq(cpu);
6537 spin_lock_irqsave(&rq->lock, flags); 6659 spin_lock_irqsave(&rq->lock, flags);
6538 if (rq->rd) { 6660 if (rq->rd) {
6539 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6661 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6540 6662
6541 set_rq_online(rq); 6663 set_rq_online(rq);
6542 } 6664 }
@@ -6550,7 +6672,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6550 break; 6672 break;
6551 /* Unbind it from offline cpu so it can run. Fall thru. */ 6673 /* Unbind it from offline cpu so it can run. Fall thru. */
6552 kthread_bind(cpu_rq(cpu)->migration_thread, 6674 kthread_bind(cpu_rq(cpu)->migration_thread,
6553 any_online_cpu(cpu_online_map)); 6675 cpumask_any(cpu_online_mask));
6554 kthread_stop(cpu_rq(cpu)->migration_thread); 6676 kthread_stop(cpu_rq(cpu)->migration_thread);
6555 cpu_rq(cpu)->migration_thread = NULL; 6677 cpu_rq(cpu)->migration_thread = NULL;
6556 break; 6678 break;
@@ -6600,7 +6722,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6600 rq = cpu_rq(cpu); 6722 rq = cpu_rq(cpu);
6601 spin_lock_irqsave(&rq->lock, flags); 6723 spin_lock_irqsave(&rq->lock, flags);
6602 if (rq->rd) { 6724 if (rq->rd) {
6603 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6725 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6604 set_rq_offline(rq); 6726 set_rq_offline(rq);
6605 } 6727 }
6606 spin_unlock_irqrestore(&rq->lock, flags); 6728 spin_unlock_irqrestore(&rq->lock, flags);
@@ -6639,13 +6761,13 @@ early_initcall(migration_init);
6639#ifdef CONFIG_SCHED_DEBUG 6761#ifdef CONFIG_SCHED_DEBUG
6640 6762
6641static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, 6763static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6642 cpumask_t *groupmask) 6764 struct cpumask *groupmask)
6643{ 6765{
6644 struct sched_group *group = sd->groups; 6766 struct sched_group *group = sd->groups;
6645 char str[256]; 6767 char str[256];
6646 6768
6647 cpulist_scnprintf(str, sizeof(str), sd->span); 6769 cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
6648 cpus_clear(*groupmask); 6770 cpumask_clear(groupmask);
6649 6771
6650 printk(KERN_DEBUG "%*s domain %d: ", level, "", level); 6772 printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
6651 6773
@@ -6659,11 +6781,11 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6659 6781
6660 printk(KERN_CONT "span %s level %s\n", str, sd->name); 6782 printk(KERN_CONT "span %s level %s\n", str, sd->name);
6661 6783
6662 if (!cpu_isset(cpu, sd->span)) { 6784 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
6663 printk(KERN_ERR "ERROR: domain->span does not contain " 6785 printk(KERN_ERR "ERROR: domain->span does not contain "
6664 "CPU%d\n", cpu); 6786 "CPU%d\n", cpu);
6665 } 6787 }
6666 if (!cpu_isset(cpu, group->cpumask)) { 6788 if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
6667 printk(KERN_ERR "ERROR: domain->groups does not contain" 6789 printk(KERN_ERR "ERROR: domain->groups does not contain"
6668 " CPU%d\n", cpu); 6790 " CPU%d\n", cpu);
6669 } 6791 }
@@ -6683,31 +6805,32 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6683 break; 6805 break;
6684 } 6806 }
6685 6807
6686 if (!cpus_weight(group->cpumask)) { 6808 if (!cpumask_weight(sched_group_cpus(group))) {
6687 printk(KERN_CONT "\n"); 6809 printk(KERN_CONT "\n");
6688 printk(KERN_ERR "ERROR: empty group\n"); 6810 printk(KERN_ERR "ERROR: empty group\n");
6689 break; 6811 break;
6690 } 6812 }
6691 6813
6692 if (cpus_intersects(*groupmask, group->cpumask)) { 6814 if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
6693 printk(KERN_CONT "\n"); 6815 printk(KERN_CONT "\n");
6694 printk(KERN_ERR "ERROR: repeated CPUs\n"); 6816 printk(KERN_ERR "ERROR: repeated CPUs\n");
6695 break; 6817 break;
6696 } 6818 }
6697 6819
6698 cpus_or(*groupmask, *groupmask, group->cpumask); 6820 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
6699 6821
6700 cpulist_scnprintf(str, sizeof(str), group->cpumask); 6822 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
6701 printk(KERN_CONT " %s", str); 6823 printk(KERN_CONT " %s", str);
6702 6824
6703 group = group->next; 6825 group = group->next;
6704 } while (group != sd->groups); 6826 } while (group != sd->groups);
6705 printk(KERN_CONT "\n"); 6827 printk(KERN_CONT "\n");
6706 6828
6707 if (!cpus_equal(sd->span, *groupmask)) 6829 if (!cpumask_equal(sched_domain_span(sd), groupmask))
6708 printk(KERN_ERR "ERROR: groups don't span domain->span\n"); 6830 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
6709 6831
6710 if (sd->parent && !cpus_subset(*groupmask, sd->parent->span)) 6832 if (sd->parent &&
6833 !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
6711 printk(KERN_ERR "ERROR: parent span is not a superset " 6834 printk(KERN_ERR "ERROR: parent span is not a superset "
6712 "of domain->span\n"); 6835 "of domain->span\n");
6713 return 0; 6836 return 0;
@@ -6715,7 +6838,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6715 6838
6716static void sched_domain_debug(struct sched_domain *sd, int cpu) 6839static void sched_domain_debug(struct sched_domain *sd, int cpu)
6717{ 6840{
6718 cpumask_t *groupmask; 6841 cpumask_var_t groupmask;
6719 int level = 0; 6842 int level = 0;
6720 6843
6721 if (!sd) { 6844 if (!sd) {
@@ -6725,8 +6848,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6725 6848
6726 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); 6849 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
6727 6850
6728 groupmask = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 6851 if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
6729 if (!groupmask) {
6730 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n"); 6852 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
6731 return; 6853 return;
6732 } 6854 }
@@ -6739,7 +6861,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6739 if (!sd) 6861 if (!sd)
6740 break; 6862 break;
6741 } 6863 }
6742 kfree(groupmask); 6864 free_cpumask_var(groupmask);
6743} 6865}
6744#else /* !CONFIG_SCHED_DEBUG */ 6866#else /* !CONFIG_SCHED_DEBUG */
6745# define sched_domain_debug(sd, cpu) do { } while (0) 6867# define sched_domain_debug(sd, cpu) do { } while (0)
@@ -6747,7 +6869,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6747 6869
6748static int sd_degenerate(struct sched_domain *sd) 6870static int sd_degenerate(struct sched_domain *sd)
6749{ 6871{
6750 if (cpus_weight(sd->span) == 1) 6872 if (cpumask_weight(sched_domain_span(sd)) == 1)
6751 return 1; 6873 return 1;
6752 6874
6753 /* Following flags need at least 2 groups */ 6875 /* Following flags need at least 2 groups */
@@ -6778,7 +6900,7 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6778 if (sd_degenerate(parent)) 6900 if (sd_degenerate(parent))
6779 return 1; 6901 return 1;
6780 6902
6781 if (!cpus_equal(sd->span, parent->span)) 6903 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
6782 return 0; 6904 return 0;
6783 6905
6784 /* Does parent contain flags not in child? */ 6906 /* Does parent contain flags not in child? */
@@ -6802,6 +6924,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6802 return 1; 6924 return 1;
6803} 6925}
6804 6926
6927static void free_rootdomain(struct root_domain *rd)
6928{
6929 cpupri_cleanup(&rd->cpupri);
6930
6931 free_cpumask_var(rd->rto_mask);
6932 free_cpumask_var(rd->online);
6933 free_cpumask_var(rd->span);
6934 kfree(rd);
6935}
6936
6805static void rq_attach_root(struct rq *rq, struct root_domain *rd) 6937static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6806{ 6938{
6807 unsigned long flags; 6939 unsigned long flags;
@@ -6811,38 +6943,62 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6811 if (rq->rd) { 6943 if (rq->rd) {
6812 struct root_domain *old_rd = rq->rd; 6944 struct root_domain *old_rd = rq->rd;
6813 6945
6814 if (cpu_isset(rq->cpu, old_rd->online)) 6946 if (cpumask_test_cpu(rq->cpu, old_rd->online))
6815 set_rq_offline(rq); 6947 set_rq_offline(rq);
6816 6948
6817 cpu_clear(rq->cpu, old_rd->span); 6949 cpumask_clear_cpu(rq->cpu, old_rd->span);
6818 6950
6819 if (atomic_dec_and_test(&old_rd->refcount)) 6951 if (atomic_dec_and_test(&old_rd->refcount))
6820 kfree(old_rd); 6952 free_rootdomain(old_rd);
6821 } 6953 }
6822 6954
6823 atomic_inc(&rd->refcount); 6955 atomic_inc(&rd->refcount);
6824 rq->rd = rd; 6956 rq->rd = rd;
6825 6957
6826 cpu_set(rq->cpu, rd->span); 6958 cpumask_set_cpu(rq->cpu, rd->span);
6827 if (cpu_isset(rq->cpu, cpu_online_map)) 6959 if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
6828 set_rq_online(rq); 6960 set_rq_online(rq);
6829 6961
6830 spin_unlock_irqrestore(&rq->lock, flags); 6962 spin_unlock_irqrestore(&rq->lock, flags);
6831} 6963}
6832 6964
6833static void init_rootdomain(struct root_domain *rd) 6965static int __init_refok init_rootdomain(struct root_domain *rd, bool bootmem)
6834{ 6966{
6835 memset(rd, 0, sizeof(*rd)); 6967 memset(rd, 0, sizeof(*rd));
6836 6968
6837 cpus_clear(rd->span); 6969 if (bootmem) {
6838 cpus_clear(rd->online); 6970 alloc_bootmem_cpumask_var(&def_root_domain.span);
6971 alloc_bootmem_cpumask_var(&def_root_domain.online);
6972 alloc_bootmem_cpumask_var(&def_root_domain.rto_mask);
6973 cpupri_init(&rd->cpupri, true);
6974 return 0;
6975 }
6839 6976
6840 cpupri_init(&rd->cpupri); 6977 if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
6978 goto out;
6979 if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
6980 goto free_span;
6981 if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
6982 goto free_online;
6983
6984 if (cpupri_init(&rd->cpupri, false) != 0)
6985 goto free_rto_mask;
6986 return 0;
6987
6988free_rto_mask:
6989 free_cpumask_var(rd->rto_mask);
6990free_online:
6991 free_cpumask_var(rd->online);
6992free_span:
6993 free_cpumask_var(rd->span);
6994out:
6995 return -ENOMEM;
6841} 6996}
6842 6997
6843static void init_defrootdomain(void) 6998static void init_defrootdomain(void)
6844{ 6999{
6845 init_rootdomain(&def_root_domain); 7000 init_rootdomain(&def_root_domain, true);
7001
6846 atomic_set(&def_root_domain.refcount, 1); 7002 atomic_set(&def_root_domain.refcount, 1);
6847} 7003}
6848 7004
@@ -6854,7 +7010,10 @@ static struct root_domain *alloc_rootdomain(void)
6854 if (!rd) 7010 if (!rd)
6855 return NULL; 7011 return NULL;
6856 7012
6857 init_rootdomain(rd); 7013 if (init_rootdomain(rd, false) != 0) {
7014 kfree(rd);
7015 return NULL;
7016 }
6858 7017
6859 return rd; 7018 return rd;
6860} 7019}
@@ -6896,19 +7055,12 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
6896} 7055}
6897 7056
6898/* cpus with isolated domains */ 7057/* cpus with isolated domains */
6899static cpumask_t cpu_isolated_map = CPU_MASK_NONE; 7058static cpumask_var_t cpu_isolated_map;
6900 7059
6901/* Setup the mask of cpus configured for isolated domains */ 7060/* Setup the mask of cpus configured for isolated domains */
6902static int __init isolated_cpu_setup(char *str) 7061static int __init isolated_cpu_setup(char *str)
6903{ 7062{
6904 static int __initdata ints[NR_CPUS]; 7063 cpulist_parse(str, cpu_isolated_map);
6905 int i;
6906
6907 str = get_options(str, ARRAY_SIZE(ints), ints);
6908 cpus_clear(cpu_isolated_map);
6909 for (i = 1; i <= ints[0]; i++)
6910 if (ints[i] < NR_CPUS)
6911 cpu_set(ints[i], cpu_isolated_map);
6912 return 1; 7064 return 1;
6913} 7065}
6914 7066
@@ -6917,42 +7069,43 @@ __setup("isolcpus=", isolated_cpu_setup);
6917/* 7069/*
6918 * init_sched_build_groups takes the cpumask we wish to span, and a pointer 7070 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
6919 * to a function which identifies what group(along with sched group) a CPU 7071 * to a function which identifies what group(along with sched group) a CPU
6920 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS 7072 * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
6921 * (due to the fact that we keep track of groups covered with a cpumask_t). 7073 * (due to the fact that we keep track of groups covered with a struct cpumask).
6922 * 7074 *
6923 * init_sched_build_groups will build a circular linked list of the groups 7075 * init_sched_build_groups will build a circular linked list of the groups
6924 * covered by the given span, and will set each group's ->cpumask correctly, 7076 * covered by the given span, and will set each group's ->cpumask correctly,
6925 * and ->cpu_power to 0. 7077 * and ->cpu_power to 0.
6926 */ 7078 */
6927static void 7079static void
6928init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map, 7080init_sched_build_groups(const struct cpumask *span,
6929 int (*group_fn)(int cpu, const cpumask_t *cpu_map, 7081 const struct cpumask *cpu_map,
7082 int (*group_fn)(int cpu, const struct cpumask *cpu_map,
6930 struct sched_group **sg, 7083 struct sched_group **sg,
6931 cpumask_t *tmpmask), 7084 struct cpumask *tmpmask),
6932 cpumask_t *covered, cpumask_t *tmpmask) 7085 struct cpumask *covered, struct cpumask *tmpmask)
6933{ 7086{
6934 struct sched_group *first = NULL, *last = NULL; 7087 struct sched_group *first = NULL, *last = NULL;
6935 int i; 7088 int i;
6936 7089
6937 cpus_clear(*covered); 7090 cpumask_clear(covered);
6938 7091
6939 for_each_cpu_mask_nr(i, *span) { 7092 for_each_cpu(i, span) {
6940 struct sched_group *sg; 7093 struct sched_group *sg;
6941 int group = group_fn(i, cpu_map, &sg, tmpmask); 7094 int group = group_fn(i, cpu_map, &sg, tmpmask);
6942 int j; 7095 int j;
6943 7096
6944 if (cpu_isset(i, *covered)) 7097 if (cpumask_test_cpu(i, covered))
6945 continue; 7098 continue;
6946 7099
6947 cpus_clear(sg->cpumask); 7100 cpumask_clear(sched_group_cpus(sg));
6948 sg->__cpu_power = 0; 7101 sg->__cpu_power = 0;
6949 7102
6950 for_each_cpu_mask_nr(j, *span) { 7103 for_each_cpu(j, span) {
6951 if (group_fn(j, cpu_map, NULL, tmpmask) != group) 7104 if (group_fn(j, cpu_map, NULL, tmpmask) != group)
6952 continue; 7105 continue;
6953 7106
6954 cpu_set(j, *covered); 7107 cpumask_set_cpu(j, covered);
6955 cpu_set(j, sg->cpumask); 7108 cpumask_set_cpu(j, sched_group_cpus(sg));
6956 } 7109 }
6957 if (!first) 7110 if (!first)
6958 first = sg; 7111 first = sg;
@@ -7016,23 +7169,21 @@ static int find_next_best_node(int node, nodemask_t *used_nodes)
7016 * should be one that prevents unnecessary balancing, but also spreads tasks 7169 * should be one that prevents unnecessary balancing, but also spreads tasks
7017 * out optimally. 7170 * out optimally.
7018 */ 7171 */
7019static void sched_domain_node_span(int node, cpumask_t *span) 7172static void sched_domain_node_span(int node, struct cpumask *span)
7020{ 7173{
7021 nodemask_t used_nodes; 7174 nodemask_t used_nodes;
7022 node_to_cpumask_ptr(nodemask, node);
7023 int i; 7175 int i;
7024 7176
7025 cpus_clear(*span); 7177 cpumask_clear(span);
7026 nodes_clear(used_nodes); 7178 nodes_clear(used_nodes);
7027 7179
7028 cpus_or(*span, *span, *nodemask); 7180 cpumask_or(span, span, cpumask_of_node(node));
7029 node_set(node, used_nodes); 7181 node_set(node, used_nodes);
7030 7182
7031 for (i = 1; i < SD_NODES_PER_DOMAIN; i++) { 7183 for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
7032 int next_node = find_next_best_node(node, &used_nodes); 7184 int next_node = find_next_best_node(node, &used_nodes);
7033 7185
7034 node_to_cpumask_ptr_next(nodemask, next_node); 7186 cpumask_or(span, span, cpumask_of_node(next_node));
7035 cpus_or(*span, *span, *nodemask);
7036 } 7187 }
7037} 7188}
7038#endif /* CONFIG_NUMA */ 7189#endif /* CONFIG_NUMA */
@@ -7040,18 +7191,33 @@ static void sched_domain_node_span(int node, cpumask_t *span)
7040int sched_smt_power_savings = 0, sched_mc_power_savings = 0; 7191int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
7041 7192
7042/* 7193/*
7194 * The cpus mask in sched_group and sched_domain hangs off the end.
7195 * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space
7196 * for nr_cpu_ids < CONFIG_NR_CPUS.
7197 */
7198struct static_sched_group {
7199 struct sched_group sg;
7200 DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
7201};
7202
7203struct static_sched_domain {
7204 struct sched_domain sd;
7205 DECLARE_BITMAP(span, CONFIG_NR_CPUS);
7206};
7207
7208/*
7043 * SMT sched-domains: 7209 * SMT sched-domains:
7044 */ 7210 */
7045#ifdef CONFIG_SCHED_SMT 7211#ifdef CONFIG_SCHED_SMT
7046static DEFINE_PER_CPU(struct sched_domain, cpu_domains); 7212static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
7047static DEFINE_PER_CPU(struct sched_group, sched_group_cpus); 7213static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
7048 7214
7049static int 7215static int
7050cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7216cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
7051 cpumask_t *unused) 7217 struct sched_group **sg, struct cpumask *unused)
7052{ 7218{
7053 if (sg) 7219 if (sg)
7054 *sg = &per_cpu(sched_group_cpus, cpu); 7220 *sg = &per_cpu(sched_group_cpus, cpu).sg;
7055 return cpu; 7221 return cpu;
7056} 7222}
7057#endif /* CONFIG_SCHED_SMT */ 7223#endif /* CONFIG_SCHED_SMT */
@@ -7060,56 +7226,53 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
7060 * multi-core sched-domains: 7226 * multi-core sched-domains:
7061 */ 7227 */
7062#ifdef CONFIG_SCHED_MC 7228#ifdef CONFIG_SCHED_MC
7063static DEFINE_PER_CPU(struct sched_domain, core_domains); 7229static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
7064static DEFINE_PER_CPU(struct sched_group, sched_group_core); 7230static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
7065#endif /* CONFIG_SCHED_MC */ 7231#endif /* CONFIG_SCHED_MC */
7066 7232
7067#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) 7233#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
7068static int 7234static int
7069cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7235cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7070 cpumask_t *mask) 7236 struct sched_group **sg, struct cpumask *mask)
7071{ 7237{
7072 int group; 7238 int group;
7073 7239
7074 *mask = per_cpu(cpu_sibling_map, cpu); 7240 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7075 cpus_and(*mask, *mask, *cpu_map); 7241 group = cpumask_first(mask);
7076 group = first_cpu(*mask);
7077 if (sg) 7242 if (sg)
7078 *sg = &per_cpu(sched_group_core, group); 7243 *sg = &per_cpu(sched_group_core, group).sg;
7079 return group; 7244 return group;
7080} 7245}
7081#elif defined(CONFIG_SCHED_MC) 7246#elif defined(CONFIG_SCHED_MC)
7082static int 7247static int
7083cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7248cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7084 cpumask_t *unused) 7249 struct sched_group **sg, struct cpumask *unused)
7085{ 7250{
7086 if (sg) 7251 if (sg)
7087 *sg = &per_cpu(sched_group_core, cpu); 7252 *sg = &per_cpu(sched_group_core, cpu).sg;
7088 return cpu; 7253 return cpu;
7089} 7254}
7090#endif 7255#endif
7091 7256
7092static DEFINE_PER_CPU(struct sched_domain, phys_domains); 7257static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
7093static DEFINE_PER_CPU(struct sched_group, sched_group_phys); 7258static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
7094 7259
7095static int 7260static int
7096cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7261cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
7097 cpumask_t *mask) 7262 struct sched_group **sg, struct cpumask *mask)
7098{ 7263{
7099 int group; 7264 int group;
7100#ifdef CONFIG_SCHED_MC 7265#ifdef CONFIG_SCHED_MC
7101 *mask = cpu_coregroup_map(cpu); 7266 cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
7102 cpus_and(*mask, *mask, *cpu_map); 7267 group = cpumask_first(mask);
7103 group = first_cpu(*mask);
7104#elif defined(CONFIG_SCHED_SMT) 7268#elif defined(CONFIG_SCHED_SMT)
7105 *mask = per_cpu(cpu_sibling_map, cpu); 7269 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7106 cpus_and(*mask, *mask, *cpu_map); 7270 group = cpumask_first(mask);
7107 group = first_cpu(*mask);
7108#else 7271#else
7109 group = cpu; 7272 group = cpu;
7110#endif 7273#endif
7111 if (sg) 7274 if (sg)
7112 *sg = &per_cpu(sched_group_phys, group); 7275 *sg = &per_cpu(sched_group_phys, group).sg;
7113 return group; 7276 return group;
7114} 7277}
7115 7278
@@ -7123,19 +7286,19 @@ static DEFINE_PER_CPU(struct sched_domain, node_domains);
7123static struct sched_group ***sched_group_nodes_bycpu; 7286static struct sched_group ***sched_group_nodes_bycpu;
7124 7287
7125static DEFINE_PER_CPU(struct sched_domain, allnodes_domains); 7288static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
7126static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes); 7289static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
7127 7290
7128static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map, 7291static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
7129 struct sched_group **sg, cpumask_t *nodemask) 7292 struct sched_group **sg,
7293 struct cpumask *nodemask)
7130{ 7294{
7131 int group; 7295 int group;
7132 7296
7133 *nodemask = node_to_cpumask(cpu_to_node(cpu)); 7297 cpumask_and(nodemask, cpumask_of_node(cpu_to_node(cpu)), cpu_map);
7134 cpus_and(*nodemask, *nodemask, *cpu_map); 7298 group = cpumask_first(nodemask);
7135 group = first_cpu(*nodemask);
7136 7299
7137 if (sg) 7300 if (sg)
7138 *sg = &per_cpu(sched_group_allnodes, group); 7301 *sg = &per_cpu(sched_group_allnodes, group).sg;
7139 return group; 7302 return group;
7140} 7303}
7141 7304
@@ -7147,11 +7310,11 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7147 if (!sg) 7310 if (!sg)
7148 return; 7311 return;
7149 do { 7312 do {
7150 for_each_cpu_mask_nr(j, sg->cpumask) { 7313 for_each_cpu(j, sched_group_cpus(sg)) {
7151 struct sched_domain *sd; 7314 struct sched_domain *sd;
7152 7315
7153 sd = &per_cpu(phys_domains, j); 7316 sd = &per_cpu(phys_domains, j).sd;
7154 if (j != first_cpu(sd->groups->cpumask)) { 7317 if (j != cpumask_first(sched_group_cpus(sd->groups))) {
7155 /* 7318 /*
7156 * Only add "power" once for each 7319 * Only add "power" once for each
7157 * physical package. 7320 * physical package.
@@ -7168,11 +7331,12 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7168 7331
7169#ifdef CONFIG_NUMA 7332#ifdef CONFIG_NUMA
7170/* Free memory allocated for various sched_group structures */ 7333/* Free memory allocated for various sched_group structures */
7171static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7334static void free_sched_groups(const struct cpumask *cpu_map,
7335 struct cpumask *nodemask)
7172{ 7336{
7173 int cpu, i; 7337 int cpu, i;
7174 7338
7175 for_each_cpu_mask_nr(cpu, *cpu_map) { 7339 for_each_cpu(cpu, cpu_map) {
7176 struct sched_group **sched_group_nodes 7340 struct sched_group **sched_group_nodes
7177 = sched_group_nodes_bycpu[cpu]; 7341 = sched_group_nodes_bycpu[cpu];
7178 7342
@@ -7182,9 +7346,8 @@ static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
7182 for (i = 0; i < nr_node_ids; i++) { 7346 for (i = 0; i < nr_node_ids; i++) {
7183 struct sched_group *oldsg, *sg = sched_group_nodes[i]; 7347 struct sched_group *oldsg, *sg = sched_group_nodes[i];
7184 7348
7185 *nodemask = node_to_cpumask(i); 7349 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7186 cpus_and(*nodemask, *nodemask, *cpu_map); 7350 if (cpumask_empty(nodemask))
7187 if (cpus_empty(*nodemask))
7188 continue; 7351 continue;
7189 7352
7190 if (sg == NULL) 7353 if (sg == NULL)
@@ -7202,7 +7365,8 @@ next_sg:
7202 } 7365 }
7203} 7366}
7204#else /* !CONFIG_NUMA */ 7367#else /* !CONFIG_NUMA */
7205static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7368static void free_sched_groups(const struct cpumask *cpu_map,
7369 struct cpumask *nodemask)
7206{ 7370{
7207} 7371}
7208#endif /* CONFIG_NUMA */ 7372#endif /* CONFIG_NUMA */
@@ -7228,7 +7392,7 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
7228 7392
7229 WARN_ON(!sd || !sd->groups); 7393 WARN_ON(!sd || !sd->groups);
7230 7394
7231 if (cpu != first_cpu(sd->groups->cpumask)) 7395 if (cpu != cpumask_first(sched_group_cpus(sd->groups)))
7232 return; 7396 return;
7233 7397
7234 child = sd->child; 7398 child = sd->child;
@@ -7293,48 +7457,6 @@ SD_INIT_FUNC(CPU)
7293 SD_INIT_FUNC(MC) 7457 SD_INIT_FUNC(MC)
7294#endif 7458#endif
7295 7459
7296/*
7297 * To minimize stack usage kmalloc room for cpumasks and share the
7298 * space as the usage in build_sched_domains() dictates. Used only
7299 * if the amount of space is significant.
7300 */
7301struct allmasks {
7302 cpumask_t tmpmask; /* make this one first */
7303 union {
7304 cpumask_t nodemask;
7305 cpumask_t this_sibling_map;
7306 cpumask_t this_core_map;
7307 };
7308 cpumask_t send_covered;
7309
7310#ifdef CONFIG_NUMA
7311 cpumask_t domainspan;
7312 cpumask_t covered;
7313 cpumask_t notcovered;
7314#endif
7315};
7316
7317#if NR_CPUS > 128
7318#define SCHED_CPUMASK_DECLARE(v) struct allmasks *v
7319static inline void sched_cpumask_alloc(struct allmasks **masks)
7320{
7321 *masks = kmalloc(sizeof(**masks), GFP_KERNEL);
7322}
7323static inline void sched_cpumask_free(struct allmasks *masks)
7324{
7325 kfree(masks);
7326}
7327#else
7328#define SCHED_CPUMASK_DECLARE(v) struct allmasks _v, *v = &_v
7329static inline void sched_cpumask_alloc(struct allmasks **masks)
7330{ }
7331static inline void sched_cpumask_free(struct allmasks *masks)
7332{ }
7333#endif
7334
7335#define SCHED_CPUMASK_VAR(v, a) cpumask_t *v = (cpumask_t *) \
7336 ((unsigned long)(a) + offsetof(struct allmasks, v))
7337
7338static int default_relax_domain_level = -1; 7460static int default_relax_domain_level = -1;
7339 7461
7340static int __init setup_relax_domain_level(char *str) 7462static int __init setup_relax_domain_level(char *str)
@@ -7374,17 +7496,38 @@ static void set_domain_attribute(struct sched_domain *sd,
7374 * Build sched domains for a given set of cpus and attach the sched domains 7496 * Build sched domains for a given set of cpus and attach the sched domains
7375 * to the individual cpus 7497 * to the individual cpus
7376 */ 7498 */
7377static int __build_sched_domains(const cpumask_t *cpu_map, 7499static int __build_sched_domains(const struct cpumask *cpu_map,
7378 struct sched_domain_attr *attr) 7500 struct sched_domain_attr *attr)
7379{ 7501{
7380 int i; 7502 int i, err = -ENOMEM;
7381 struct root_domain *rd; 7503 struct root_domain *rd;
7382 SCHED_CPUMASK_DECLARE(allmasks); 7504 cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
7383 cpumask_t *tmpmask; 7505 tmpmask;
7384#ifdef CONFIG_NUMA 7506#ifdef CONFIG_NUMA
7507 cpumask_var_t domainspan, covered, notcovered;
7385 struct sched_group **sched_group_nodes = NULL; 7508 struct sched_group **sched_group_nodes = NULL;
7386 int sd_allnodes = 0; 7509 int sd_allnodes = 0;
7387 7510
7511 if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
7512 goto out;
7513 if (!alloc_cpumask_var(&covered, GFP_KERNEL))
7514 goto free_domainspan;
7515 if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
7516 goto free_covered;
7517#endif
7518
7519 if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
7520 goto free_notcovered;
7521 if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
7522 goto free_nodemask;
7523 if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
7524 goto free_this_sibling_map;
7525 if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
7526 goto free_this_core_map;
7527 if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
7528 goto free_send_covered;
7529
7530#ifdef CONFIG_NUMA
7388 /* 7531 /*
7389 * Allocate the per-node list of sched groups 7532 * Allocate the per-node list of sched groups
7390 */ 7533 */
@@ -7392,54 +7535,35 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7392 GFP_KERNEL); 7535 GFP_KERNEL);
7393 if (!sched_group_nodes) { 7536 if (!sched_group_nodes) {
7394 printk(KERN_WARNING "Can not alloc sched group node list\n"); 7537 printk(KERN_WARNING "Can not alloc sched group node list\n");
7395 return -ENOMEM; 7538 goto free_tmpmask;
7396 } 7539 }
7397#endif 7540#endif
7398 7541
7399 rd = alloc_rootdomain(); 7542 rd = alloc_rootdomain();
7400 if (!rd) { 7543 if (!rd) {
7401 printk(KERN_WARNING "Cannot alloc root domain\n"); 7544 printk(KERN_WARNING "Cannot alloc root domain\n");
7402#ifdef CONFIG_NUMA 7545 goto free_sched_groups;
7403 kfree(sched_group_nodes);
7404#endif
7405 return -ENOMEM;
7406 }
7407
7408 /* get space for all scratch cpumask variables */
7409 sched_cpumask_alloc(&allmasks);
7410 if (!allmasks) {
7411 printk(KERN_WARNING "Cannot alloc cpumask array\n");
7412 kfree(rd);
7413#ifdef CONFIG_NUMA
7414 kfree(sched_group_nodes);
7415#endif
7416 return -ENOMEM;
7417 } 7546 }
7418 7547
7419 tmpmask = (cpumask_t *)allmasks;
7420
7421
7422#ifdef CONFIG_NUMA 7548#ifdef CONFIG_NUMA
7423 sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes; 7549 sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
7424#endif 7550#endif
7425 7551
7426 /* 7552 /*
7427 * Set up domains for cpus specified by the cpu_map. 7553 * Set up domains for cpus specified by the cpu_map.
7428 */ 7554 */
7429 for_each_cpu_mask_nr(i, *cpu_map) { 7555 for_each_cpu(i, cpu_map) {
7430 struct sched_domain *sd = NULL, *p; 7556 struct sched_domain *sd = NULL, *p;
7431 SCHED_CPUMASK_VAR(nodemask, allmasks);
7432 7557
7433 *nodemask = node_to_cpumask(cpu_to_node(i)); 7558 cpumask_and(nodemask, cpumask_of_node(cpu_to_node(i)), cpu_map);
7434 cpus_and(*nodemask, *nodemask, *cpu_map);
7435 7559
7436#ifdef CONFIG_NUMA 7560#ifdef CONFIG_NUMA
7437 if (cpus_weight(*cpu_map) > 7561 if (cpumask_weight(cpu_map) >
7438 SD_NODES_PER_DOMAIN*cpus_weight(*nodemask)) { 7562 SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
7439 sd = &per_cpu(allnodes_domains, i); 7563 sd = &per_cpu(allnodes_domains, i);
7440 SD_INIT(sd, ALLNODES); 7564 SD_INIT(sd, ALLNODES);
7441 set_domain_attribute(sd, attr); 7565 set_domain_attribute(sd, attr);
7442 sd->span = *cpu_map; 7566 cpumask_copy(sched_domain_span(sd), cpu_map);
7443 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask); 7567 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
7444 p = sd; 7568 p = sd;
7445 sd_allnodes = 1; 7569 sd_allnodes = 1;
@@ -7449,18 +7573,19 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7449 sd = &per_cpu(node_domains, i); 7573 sd = &per_cpu(node_domains, i);
7450 SD_INIT(sd, NODE); 7574 SD_INIT(sd, NODE);
7451 set_domain_attribute(sd, attr); 7575 set_domain_attribute(sd, attr);
7452 sched_domain_node_span(cpu_to_node(i), &sd->span); 7576 sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
7453 sd->parent = p; 7577 sd->parent = p;
7454 if (p) 7578 if (p)
7455 p->child = sd; 7579 p->child = sd;
7456 cpus_and(sd->span, sd->span, *cpu_map); 7580 cpumask_and(sched_domain_span(sd),
7581 sched_domain_span(sd), cpu_map);
7457#endif 7582#endif
7458 7583
7459 p = sd; 7584 p = sd;
7460 sd = &per_cpu(phys_domains, i); 7585 sd = &per_cpu(phys_domains, i).sd;
7461 SD_INIT(sd, CPU); 7586 SD_INIT(sd, CPU);
7462 set_domain_attribute(sd, attr); 7587 set_domain_attribute(sd, attr);
7463 sd->span = *nodemask; 7588 cpumask_copy(sched_domain_span(sd), nodemask);
7464 sd->parent = p; 7589 sd->parent = p;
7465 if (p) 7590 if (p)
7466 p->child = sd; 7591 p->child = sd;
@@ -7468,11 +7593,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7468 7593
7469#ifdef CONFIG_SCHED_MC 7594#ifdef CONFIG_SCHED_MC
7470 p = sd; 7595 p = sd;
7471 sd = &per_cpu(core_domains, i); 7596 sd = &per_cpu(core_domains, i).sd;
7472 SD_INIT(sd, MC); 7597 SD_INIT(sd, MC);
7473 set_domain_attribute(sd, attr); 7598 set_domain_attribute(sd, attr);
7474 sd->span = cpu_coregroup_map(i); 7599 cpumask_and(sched_domain_span(sd), cpu_map,
7475 cpus_and(sd->span, sd->span, *cpu_map); 7600 cpu_coregroup_mask(i));
7476 sd->parent = p; 7601 sd->parent = p;
7477 p->child = sd; 7602 p->child = sd;
7478 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask); 7603 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7480,11 +7605,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7480 7605
7481#ifdef CONFIG_SCHED_SMT 7606#ifdef CONFIG_SCHED_SMT
7482 p = sd; 7607 p = sd;
7483 sd = &per_cpu(cpu_domains, i); 7608 sd = &per_cpu(cpu_domains, i).sd;
7484 SD_INIT(sd, SIBLING); 7609 SD_INIT(sd, SIBLING);
7485 set_domain_attribute(sd, attr); 7610 set_domain_attribute(sd, attr);
7486 sd->span = per_cpu(cpu_sibling_map, i); 7611 cpumask_and(sched_domain_span(sd),
7487 cpus_and(sd->span, sd->span, *cpu_map); 7612 &per_cpu(cpu_sibling_map, i), cpu_map);
7488 sd->parent = p; 7613 sd->parent = p;
7489 p->child = sd; 7614 p->child = sd;
7490 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask); 7615 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7493,13 +7618,10 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7493 7618
7494#ifdef CONFIG_SCHED_SMT 7619#ifdef CONFIG_SCHED_SMT
7495 /* Set up CPU (sibling) groups */ 7620 /* Set up CPU (sibling) groups */
7496 for_each_cpu_mask_nr(i, *cpu_map) { 7621 for_each_cpu(i, cpu_map) {
7497 SCHED_CPUMASK_VAR(this_sibling_map, allmasks); 7622 cpumask_and(this_sibling_map,
7498 SCHED_CPUMASK_VAR(send_covered, allmasks); 7623 &per_cpu(cpu_sibling_map, i), cpu_map);
7499 7624 if (i != cpumask_first(this_sibling_map))
7500 *this_sibling_map = per_cpu(cpu_sibling_map, i);
7501 cpus_and(*this_sibling_map, *this_sibling_map, *cpu_map);
7502 if (i != first_cpu(*this_sibling_map))
7503 continue; 7625 continue;
7504 7626
7505 init_sched_build_groups(this_sibling_map, cpu_map, 7627 init_sched_build_groups(this_sibling_map, cpu_map,
@@ -7510,13 +7632,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7510 7632
7511#ifdef CONFIG_SCHED_MC 7633#ifdef CONFIG_SCHED_MC
7512 /* Set up multi-core groups */ 7634 /* Set up multi-core groups */
7513 for_each_cpu_mask_nr(i, *cpu_map) { 7635 for_each_cpu(i, cpu_map) {
7514 SCHED_CPUMASK_VAR(this_core_map, allmasks); 7636 cpumask_and(this_core_map, cpu_coregroup_mask(i), cpu_map);
7515 SCHED_CPUMASK_VAR(send_covered, allmasks); 7637 if (i != cpumask_first(this_core_map))
7516
7517 *this_core_map = cpu_coregroup_map(i);
7518 cpus_and(*this_core_map, *this_core_map, *cpu_map);
7519 if (i != first_cpu(*this_core_map))
7520 continue; 7638 continue;
7521 7639
7522 init_sched_build_groups(this_core_map, cpu_map, 7640 init_sched_build_groups(this_core_map, cpu_map,
@@ -7527,12 +7645,8 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7527 7645
7528 /* Set up physical groups */ 7646 /* Set up physical groups */
7529 for (i = 0; i < nr_node_ids; i++) { 7647 for (i = 0; i < nr_node_ids; i++) {
7530 SCHED_CPUMASK_VAR(nodemask, allmasks); 7648 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7531 SCHED_CPUMASK_VAR(send_covered, allmasks); 7649 if (cpumask_empty(nodemask))
7532
7533 *nodemask = node_to_cpumask(i);
7534 cpus_and(*nodemask, *nodemask, *cpu_map);
7535 if (cpus_empty(*nodemask))
7536 continue; 7650 continue;
7537 7651
7538 init_sched_build_groups(nodemask, cpu_map, 7652 init_sched_build_groups(nodemask, cpu_map,
@@ -7543,8 +7657,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7543#ifdef CONFIG_NUMA 7657#ifdef CONFIG_NUMA
7544 /* Set up node groups */ 7658 /* Set up node groups */
7545 if (sd_allnodes) { 7659 if (sd_allnodes) {
7546 SCHED_CPUMASK_VAR(send_covered, allmasks);
7547
7548 init_sched_build_groups(cpu_map, cpu_map, 7660 init_sched_build_groups(cpu_map, cpu_map,
7549 &cpu_to_allnodes_group, 7661 &cpu_to_allnodes_group,
7550 send_covered, tmpmask); 7662 send_covered, tmpmask);
@@ -7553,58 +7665,53 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7553 for (i = 0; i < nr_node_ids; i++) { 7665 for (i = 0; i < nr_node_ids; i++) {
7554 /* Set up node groups */ 7666 /* Set up node groups */
7555 struct sched_group *sg, *prev; 7667 struct sched_group *sg, *prev;
7556 SCHED_CPUMASK_VAR(nodemask, allmasks);
7557 SCHED_CPUMASK_VAR(domainspan, allmasks);
7558 SCHED_CPUMASK_VAR(covered, allmasks);
7559 int j; 7668 int j;
7560 7669
7561 *nodemask = node_to_cpumask(i); 7670 cpumask_clear(covered);
7562 cpus_clear(*covered); 7671 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7563 7672 if (cpumask_empty(nodemask)) {
7564 cpus_and(*nodemask, *nodemask, *cpu_map);
7565 if (cpus_empty(*nodemask)) {
7566 sched_group_nodes[i] = NULL; 7673 sched_group_nodes[i] = NULL;
7567 continue; 7674 continue;
7568 } 7675 }
7569 7676
7570 sched_domain_node_span(i, domainspan); 7677 sched_domain_node_span(i, domainspan);
7571 cpus_and(*domainspan, *domainspan, *cpu_map); 7678 cpumask_and(domainspan, domainspan, cpu_map);
7572 7679
7573 sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i); 7680 sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
7681 GFP_KERNEL, i);
7574 if (!sg) { 7682 if (!sg) {
7575 printk(KERN_WARNING "Can not alloc domain group for " 7683 printk(KERN_WARNING "Can not alloc domain group for "
7576 "node %d\n", i); 7684 "node %d\n", i);
7577 goto error; 7685 goto error;
7578 } 7686 }
7579 sched_group_nodes[i] = sg; 7687 sched_group_nodes[i] = sg;
7580 for_each_cpu_mask_nr(j, *nodemask) { 7688 for_each_cpu(j, nodemask) {
7581 struct sched_domain *sd; 7689 struct sched_domain *sd;
7582 7690
7583 sd = &per_cpu(node_domains, j); 7691 sd = &per_cpu(node_domains, j);
7584 sd->groups = sg; 7692 sd->groups = sg;
7585 } 7693 }
7586 sg->__cpu_power = 0; 7694 sg->__cpu_power = 0;
7587 sg->cpumask = *nodemask; 7695 cpumask_copy(sched_group_cpus(sg), nodemask);
7588 sg->next = sg; 7696 sg->next = sg;
7589 cpus_or(*covered, *covered, *nodemask); 7697 cpumask_or(covered, covered, nodemask);
7590 prev = sg; 7698 prev = sg;
7591 7699
7592 for (j = 0; j < nr_node_ids; j++) { 7700 for (j = 0; j < nr_node_ids; j++) {
7593 SCHED_CPUMASK_VAR(notcovered, allmasks);
7594 int n = (i + j) % nr_node_ids; 7701 int n = (i + j) % nr_node_ids;
7595 node_to_cpumask_ptr(pnodemask, n);
7596 7702
7597 cpus_complement(*notcovered, *covered); 7703 cpumask_complement(notcovered, covered);
7598 cpus_and(*tmpmask, *notcovered, *cpu_map); 7704 cpumask_and(tmpmask, notcovered, cpu_map);
7599 cpus_and(*tmpmask, *tmpmask, *domainspan); 7705 cpumask_and(tmpmask, tmpmask, domainspan);
7600 if (cpus_empty(*tmpmask)) 7706 if (cpumask_empty(tmpmask))
7601 break; 7707 break;
7602 7708
7603 cpus_and(*tmpmask, *tmpmask, *pnodemask); 7709 cpumask_and(tmpmask, tmpmask, cpumask_of_node(n));
7604 if (cpus_empty(*tmpmask)) 7710 if (cpumask_empty(tmpmask))
7605 continue; 7711 continue;
7606 7712
7607 sg = kmalloc_node(sizeof(struct sched_group), 7713 sg = kmalloc_node(sizeof(struct sched_group) +
7714 cpumask_size(),
7608 GFP_KERNEL, i); 7715 GFP_KERNEL, i);
7609 if (!sg) { 7716 if (!sg) {
7610 printk(KERN_WARNING 7717 printk(KERN_WARNING
@@ -7612,9 +7719,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7612 goto error; 7719 goto error;
7613 } 7720 }
7614 sg->__cpu_power = 0; 7721 sg->__cpu_power = 0;
7615 sg->cpumask = *tmpmask; 7722 cpumask_copy(sched_group_cpus(sg), tmpmask);
7616 sg->next = prev->next; 7723 sg->next = prev->next;
7617 cpus_or(*covered, *covered, *tmpmask); 7724 cpumask_or(covered, covered, tmpmask);
7618 prev->next = sg; 7725 prev->next = sg;
7619 prev = sg; 7726 prev = sg;
7620 } 7727 }
@@ -7623,22 +7730,22 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7623 7730
7624 /* Calculate CPU power for physical packages and nodes */ 7731 /* Calculate CPU power for physical packages and nodes */
7625#ifdef CONFIG_SCHED_SMT 7732#ifdef CONFIG_SCHED_SMT
7626 for_each_cpu_mask_nr(i, *cpu_map) { 7733 for_each_cpu(i, cpu_map) {
7627 struct sched_domain *sd = &per_cpu(cpu_domains, i); 7734 struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
7628 7735
7629 init_sched_groups_power(i, sd); 7736 init_sched_groups_power(i, sd);
7630 } 7737 }
7631#endif 7738#endif
7632#ifdef CONFIG_SCHED_MC 7739#ifdef CONFIG_SCHED_MC
7633 for_each_cpu_mask_nr(i, *cpu_map) { 7740 for_each_cpu(i, cpu_map) {
7634 struct sched_domain *sd = &per_cpu(core_domains, i); 7741 struct sched_domain *sd = &per_cpu(core_domains, i).sd;
7635 7742
7636 init_sched_groups_power(i, sd); 7743 init_sched_groups_power(i, sd);
7637 } 7744 }
7638#endif 7745#endif
7639 7746
7640 for_each_cpu_mask_nr(i, *cpu_map) { 7747 for_each_cpu(i, cpu_map) {
7641 struct sched_domain *sd = &per_cpu(phys_domains, i); 7748 struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
7642 7749
7643 init_sched_groups_power(i, sd); 7750 init_sched_groups_power(i, sd);
7644 } 7751 }
@@ -7650,53 +7757,78 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7650 if (sd_allnodes) { 7757 if (sd_allnodes) {
7651 struct sched_group *sg; 7758 struct sched_group *sg;
7652 7759
7653 cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg, 7760 cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
7654 tmpmask); 7761 tmpmask);
7655 init_numa_sched_groups_power(sg); 7762 init_numa_sched_groups_power(sg);
7656 } 7763 }
7657#endif 7764#endif
7658 7765
7659 /* Attach the domains */ 7766 /* Attach the domains */
7660 for_each_cpu_mask_nr(i, *cpu_map) { 7767 for_each_cpu(i, cpu_map) {
7661 struct sched_domain *sd; 7768 struct sched_domain *sd;
7662#ifdef CONFIG_SCHED_SMT 7769#ifdef CONFIG_SCHED_SMT
7663 sd = &per_cpu(cpu_domains, i); 7770 sd = &per_cpu(cpu_domains, i).sd;
7664#elif defined(CONFIG_SCHED_MC) 7771#elif defined(CONFIG_SCHED_MC)
7665 sd = &per_cpu(core_domains, i); 7772 sd = &per_cpu(core_domains, i).sd;
7666#else 7773#else
7667 sd = &per_cpu(phys_domains, i); 7774 sd = &per_cpu(phys_domains, i).sd;
7668#endif 7775#endif
7669 cpu_attach_domain(sd, rd, i); 7776 cpu_attach_domain(sd, rd, i);
7670 } 7777 }
7671 7778
7672 sched_cpumask_free(allmasks); 7779 err = 0;
7673 return 0; 7780
7781free_tmpmask:
7782 free_cpumask_var(tmpmask);
7783free_send_covered:
7784 free_cpumask_var(send_covered);
7785free_this_core_map:
7786 free_cpumask_var(this_core_map);
7787free_this_sibling_map:
7788 free_cpumask_var(this_sibling_map);
7789free_nodemask:
7790 free_cpumask_var(nodemask);
7791free_notcovered:
7792#ifdef CONFIG_NUMA
7793 free_cpumask_var(notcovered);
7794free_covered:
7795 free_cpumask_var(covered);
7796free_domainspan:
7797 free_cpumask_var(domainspan);
7798out:
7799#endif
7800 return err;
7801
7802free_sched_groups:
7803#ifdef CONFIG_NUMA
7804 kfree(sched_group_nodes);
7805#endif
7806 goto free_tmpmask;
7674 7807
7675#ifdef CONFIG_NUMA 7808#ifdef CONFIG_NUMA
7676error: 7809error:
7677 free_sched_groups(cpu_map, tmpmask); 7810 free_sched_groups(cpu_map, tmpmask);
7678 sched_cpumask_free(allmasks); 7811 free_rootdomain(rd);
7679 kfree(rd); 7812 goto free_tmpmask;
7680 return -ENOMEM;
7681#endif 7813#endif
7682} 7814}
7683 7815
7684static int build_sched_domains(const cpumask_t *cpu_map) 7816static int build_sched_domains(const struct cpumask *cpu_map)
7685{ 7817{
7686 return __build_sched_domains(cpu_map, NULL); 7818 return __build_sched_domains(cpu_map, NULL);
7687} 7819}
7688 7820
7689static cpumask_t *doms_cur; /* current sched domains */ 7821static struct cpumask *doms_cur; /* current sched domains */
7690static int ndoms_cur; /* number of sched domains in 'doms_cur' */ 7822static int ndoms_cur; /* number of sched domains in 'doms_cur' */
7691static struct sched_domain_attr *dattr_cur; 7823static struct sched_domain_attr *dattr_cur;
7692 /* attribues of custom domains in 'doms_cur' */ 7824 /* attribues of custom domains in 'doms_cur' */
7693 7825
7694/* 7826/*
7695 * Special case: If a kmalloc of a doms_cur partition (array of 7827 * Special case: If a kmalloc of a doms_cur partition (array of
7696 * cpumask_t) fails, then fallback to a single sched domain, 7828 * cpumask) fails, then fallback to a single sched domain,
7697 * as determined by the single cpumask_t fallback_doms. 7829 * as determined by the single cpumask fallback_doms.
7698 */ 7830 */
7699static cpumask_t fallback_doms; 7831static cpumask_var_t fallback_doms;
7700 7832
7701/* 7833/*
7702 * arch_update_cpu_topology lets virtualized architectures update the 7834 * arch_update_cpu_topology lets virtualized architectures update the
@@ -7713,16 +7845,16 @@ int __attribute__((weak)) arch_update_cpu_topology(void)
7713 * For now this just excludes isolated cpus, but could be used to 7845 * For now this just excludes isolated cpus, but could be used to
7714 * exclude other special cases in the future. 7846 * exclude other special cases in the future.
7715 */ 7847 */
7716static int arch_init_sched_domains(const cpumask_t *cpu_map) 7848static int arch_init_sched_domains(const struct cpumask *cpu_map)
7717{ 7849{
7718 int err; 7850 int err;
7719 7851
7720 arch_update_cpu_topology(); 7852 arch_update_cpu_topology();
7721 ndoms_cur = 1; 7853 ndoms_cur = 1;
7722 doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 7854 doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
7723 if (!doms_cur) 7855 if (!doms_cur)
7724 doms_cur = &fallback_doms; 7856 doms_cur = fallback_doms;
7725 cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map); 7857 cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
7726 dattr_cur = NULL; 7858 dattr_cur = NULL;
7727 err = build_sched_domains(doms_cur); 7859 err = build_sched_domains(doms_cur);
7728 register_sched_domain_sysctl(); 7860 register_sched_domain_sysctl();
@@ -7730,8 +7862,8 @@ static int arch_init_sched_domains(const cpumask_t *cpu_map)
7730 return err; 7862 return err;
7731} 7863}
7732 7864
7733static void arch_destroy_sched_domains(const cpumask_t *cpu_map, 7865static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
7734 cpumask_t *tmpmask) 7866 struct cpumask *tmpmask)
7735{ 7867{
7736 free_sched_groups(cpu_map, tmpmask); 7868 free_sched_groups(cpu_map, tmpmask);
7737} 7869}
@@ -7740,15 +7872,16 @@ static void arch_destroy_sched_domains(const cpumask_t *cpu_map,
7740 * Detach sched domains from a group of cpus specified in cpu_map 7872 * Detach sched domains from a group of cpus specified in cpu_map
7741 * These cpus will now be attached to the NULL domain 7873 * These cpus will now be attached to the NULL domain
7742 */ 7874 */
7743static void detach_destroy_domains(const cpumask_t *cpu_map) 7875static void detach_destroy_domains(const struct cpumask *cpu_map)
7744{ 7876{
7745 cpumask_t tmpmask; 7877 /* Save because hotplug lock held. */
7878 static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
7746 int i; 7879 int i;
7747 7880
7748 for_each_cpu_mask_nr(i, *cpu_map) 7881 for_each_cpu(i, cpu_map)
7749 cpu_attach_domain(NULL, &def_root_domain, i); 7882 cpu_attach_domain(NULL, &def_root_domain, i);
7750 synchronize_sched(); 7883 synchronize_sched();
7751 arch_destroy_sched_domains(cpu_map, &tmpmask); 7884 arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
7752} 7885}
7753 7886
7754/* handle null as "default" */ 7887/* handle null as "default" */
@@ -7773,7 +7906,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7773 * doms_new[] to the current sched domain partitioning, doms_cur[]. 7906 * doms_new[] to the current sched domain partitioning, doms_cur[].
7774 * It destroys each deleted domain and builds each new domain. 7907 * It destroys each deleted domain and builds each new domain.
7775 * 7908 *
7776 * 'doms_new' is an array of cpumask_t's of length 'ndoms_new'. 7909 * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
7777 * The masks don't intersect (don't overlap.) We should setup one 7910 * The masks don't intersect (don't overlap.) We should setup one
7778 * sched domain for each mask. CPUs not in any of the cpumasks will 7911 * sched domain for each mask. CPUs not in any of the cpumasks will
7779 * not be load balanced. If the same cpumask appears both in the 7912 * not be load balanced. If the same cpumask appears both in the
@@ -7787,13 +7920,14 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7787 * the single partition 'fallback_doms', it also forces the domains 7920 * the single partition 'fallback_doms', it also forces the domains
7788 * to be rebuilt. 7921 * to be rebuilt.
7789 * 7922 *
7790 * If doms_new == NULL it will be replaced with cpu_online_map. 7923 * If doms_new == NULL it will be replaced with cpu_online_mask.
7791 * ndoms_new == 0 is a special case for destroying existing domains, 7924 * ndoms_new == 0 is a special case for destroying existing domains,
7792 * and it will not create the default domain. 7925 * and it will not create the default domain.
7793 * 7926 *
7794 * Call with hotplug lock held 7927 * Call with hotplug lock held
7795 */ 7928 */
7796void partition_sched_domains(int ndoms_new, cpumask_t *doms_new, 7929/* FIXME: Change to struct cpumask *doms_new[] */
7930void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
7797 struct sched_domain_attr *dattr_new) 7931 struct sched_domain_attr *dattr_new)
7798{ 7932{
7799 int i, j, n; 7933 int i, j, n;
@@ -7812,7 +7946,7 @@ void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
7812 /* Destroy deleted domains */ 7946 /* Destroy deleted domains */
7813 for (i = 0; i < ndoms_cur; i++) { 7947 for (i = 0; i < ndoms_cur; i++) {
7814 for (j = 0; j < n && !new_topology; j++) { 7948 for (j = 0; j < n && !new_topology; j++) {
7815 if (cpus_equal(doms_cur[i], doms_new[j]) 7949 if (cpumask_equal(&doms_cur[i], &doms_new[j])
7816 && dattrs_equal(dattr_cur, i, dattr_new, j)) 7950 && dattrs_equal(dattr_cur, i, dattr_new, j))
7817 goto match1; 7951 goto match1;
7818 } 7952 }
@@ -7824,15 +7958,15 @@ match1:
7824 7958
7825 if (doms_new == NULL) { 7959 if (doms_new == NULL) {
7826 ndoms_cur = 0; 7960 ndoms_cur = 0;
7827 doms_new = &fallback_doms; 7961 doms_new = fallback_doms;
7828 cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map); 7962 cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
7829 WARN_ON_ONCE(dattr_new); 7963 WARN_ON_ONCE(dattr_new);
7830 } 7964 }
7831 7965
7832 /* Build new domains */ 7966 /* Build new domains */
7833 for (i = 0; i < ndoms_new; i++) { 7967 for (i = 0; i < ndoms_new; i++) {
7834 for (j = 0; j < ndoms_cur && !new_topology; j++) { 7968 for (j = 0; j < ndoms_cur && !new_topology; j++) {
7835 if (cpus_equal(doms_new[i], doms_cur[j]) 7969 if (cpumask_equal(&doms_new[i], &doms_cur[j])
7836 && dattrs_equal(dattr_new, i, dattr_cur, j)) 7970 && dattrs_equal(dattr_new, i, dattr_cur, j))
7837 goto match2; 7971 goto match2;
7838 } 7972 }
@@ -7844,7 +7978,7 @@ match2:
7844 } 7978 }
7845 7979
7846 /* Remember the new sched domains */ 7980 /* Remember the new sched domains */
7847 if (doms_cur != &fallback_doms) 7981 if (doms_cur != fallback_doms)
7848 kfree(doms_cur); 7982 kfree(doms_cur);
7849 kfree(dattr_cur); /* kfree(NULL) is safe */ 7983 kfree(dattr_cur); /* kfree(NULL) is safe */
7850 doms_cur = doms_new; 7984 doms_cur = doms_new;
@@ -7857,7 +7991,7 @@ match2:
7857} 7991}
7858 7992
7859#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT) 7993#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
7860int arch_reinit_sched_domains(void) 7994static void arch_reinit_sched_domains(void)
7861{ 7995{
7862 get_online_cpus(); 7996 get_online_cpus();
7863 7997
@@ -7866,25 +8000,33 @@ int arch_reinit_sched_domains(void)
7866 8000
7867 rebuild_sched_domains(); 8001 rebuild_sched_domains();
7868 put_online_cpus(); 8002 put_online_cpus();
7869
7870 return 0;
7871} 8003}
7872 8004
7873static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt) 8005static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
7874{ 8006{
7875 int ret; 8007 unsigned int level = 0;
8008
8009 if (sscanf(buf, "%u", &level) != 1)
8010 return -EINVAL;
8011
8012 /*
8013 * level is always be positive so don't check for
8014 * level < POWERSAVINGS_BALANCE_NONE which is 0
8015 * What happens on 0 or 1 byte write,
8016 * need to check for count as well?
8017 */
7876 8018
7877 if (buf[0] != '0' && buf[0] != '1') 8019 if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
7878 return -EINVAL; 8020 return -EINVAL;
7879 8021
7880 if (smt) 8022 if (smt)
7881 sched_smt_power_savings = (buf[0] == '1'); 8023 sched_smt_power_savings = level;
7882 else 8024 else
7883 sched_mc_power_savings = (buf[0] == '1'); 8025 sched_mc_power_savings = level;
7884 8026
7885 ret = arch_reinit_sched_domains(); 8027 arch_reinit_sched_domains();
7886 8028
7887 return ret ? ret : count; 8029 return count;
7888} 8030}
7889 8031
7890#ifdef CONFIG_SCHED_MC 8032#ifdef CONFIG_SCHED_MC
@@ -7919,7 +8061,7 @@ static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
7919 sched_smt_power_savings_store); 8061 sched_smt_power_savings_store);
7920#endif 8062#endif
7921 8063
7922int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls) 8064int __init sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
7923{ 8065{
7924 int err = 0; 8066 int err = 0;
7925 8067
@@ -7984,7 +8126,9 @@ static int update_runtime(struct notifier_block *nfb,
7984 8126
7985void __init sched_init_smp(void) 8127void __init sched_init_smp(void)
7986{ 8128{
7987 cpumask_t non_isolated_cpus; 8129 cpumask_var_t non_isolated_cpus;
8130
8131 alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
7988 8132
7989#if defined(CONFIG_NUMA) 8133#if defined(CONFIG_NUMA)
7990 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **), 8134 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
@@ -7993,10 +8137,10 @@ void __init sched_init_smp(void)
7993#endif 8137#endif
7994 get_online_cpus(); 8138 get_online_cpus();
7995 mutex_lock(&sched_domains_mutex); 8139 mutex_lock(&sched_domains_mutex);
7996 arch_init_sched_domains(&cpu_online_map); 8140 arch_init_sched_domains(cpu_online_mask);
7997 cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map); 8141 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
7998 if (cpus_empty(non_isolated_cpus)) 8142 if (cpumask_empty(non_isolated_cpus))
7999 cpu_set(smp_processor_id(), non_isolated_cpus); 8143 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
8000 mutex_unlock(&sched_domains_mutex); 8144 mutex_unlock(&sched_domains_mutex);
8001 put_online_cpus(); 8145 put_online_cpus();
8002 8146
@@ -8011,9 +8155,13 @@ void __init sched_init_smp(void)
8011 init_hrtick(); 8155 init_hrtick();
8012 8156
8013 /* Move init over to a non-isolated CPU */ 8157 /* Move init over to a non-isolated CPU */
8014 if (set_cpus_allowed_ptr(current, &non_isolated_cpus) < 0) 8158 if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
8015 BUG(); 8159 BUG();
8016 sched_init_granularity(); 8160 sched_init_granularity();
8161 free_cpumask_var(non_isolated_cpus);
8162
8163 alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
8164 init_sched_rt_class();
8017} 8165}
8018#else 8166#else
8019void __init sched_init_smp(void) 8167void __init sched_init_smp(void)
@@ -8328,6 +8476,15 @@ void __init sched_init(void)
8328 */ 8476 */
8329 current->sched_class = &fair_sched_class; 8477 current->sched_class = &fair_sched_class;
8330 8478
8479 /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
8480 alloc_bootmem_cpumask_var(&nohz_cpu_mask);
8481#ifdef CONFIG_SMP
8482#ifdef CONFIG_NO_HZ
8483 alloc_bootmem_cpumask_var(&nohz.cpu_mask);
8484#endif
8485 alloc_bootmem_cpumask_var(&cpu_isolated_map);
8486#endif /* SMP */
8487
8331 scheduler_running = 1; 8488 scheduler_running = 1;
8332} 8489}
8333 8490
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-23 19:55:07 -0500