aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched.c
diff options
context:
space:
mode:
authorIngo Molnar <mingo@elte.hu>2009-01-02 16:41:36 -0500
committerIngo Molnar <mingo@elte.hu>2009-01-02 16:41:36 -0500
commit923a789b49c7269a0245d5af6afe486188d940df (patch)
treec3f168427372e64f7467a794f313416da5086ba0 /kernel/sched.c
parent103ceffb9501531f6931df6aebc11a05189201f0 (diff)
parentb840d79631c882786925303c2b0f4fefc31845ed (diff)
Merge branch 'linus' into x86/cleanups
Conflicts: arch/x86/kernel/reboot.c
Diffstat (limited to 'kernel/sched.c')
-rw-r--r--kernel/sched.c975
1 files changed, 557 insertions, 418 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index 748ff924a290..27ba1d642f0f 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -209,7 +209,6 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
209 hrtimer_init(&rt_b->rt_period_timer, 209 hrtimer_init(&rt_b->rt_period_timer,
210 CLOCK_MONOTONIC, HRTIMER_MODE_REL); 210 CLOCK_MONOTONIC, HRTIMER_MODE_REL);
211 rt_b->rt_period_timer.function = sched_rt_period_timer; 211 rt_b->rt_period_timer.function = sched_rt_period_timer;
212 rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;
213} 212}
214 213
215static inline int rt_bandwidth_enabled(void) 214static inline int rt_bandwidth_enabled(void)
@@ -499,18 +498,26 @@ struct rt_rq {
499 */ 498 */
500struct root_domain { 499struct root_domain {
501 atomic_t refcount; 500 atomic_t refcount;
502 cpumask_t span; 501 cpumask_var_t span;
503 cpumask_t online; 502 cpumask_var_t online;
504 503
505 /* 504 /*
506 * The "RT overload" flag: it gets set if a CPU has more than 505 * The "RT overload" flag: it gets set if a CPU has more than
507 * one runnable RT task. 506 * one runnable RT task.
508 */ 507 */
509 cpumask_t rto_mask; 508 cpumask_var_t rto_mask;
510 atomic_t rto_count; 509 atomic_t rto_count;
511#ifdef CONFIG_SMP 510#ifdef CONFIG_SMP
512 struct cpupri cpupri; 511 struct cpupri cpupri;
513#endif 512#endif
513#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
514 /*
515 * Preferred wake up cpu nominated by sched_mc balance that will be
516 * used when most cpus are idle in the system indicating overall very
517 * low system utilisation. Triggered at POWERSAVINGS_BALANCE_WAKEUP(2)
518 */
519 unsigned int sched_mc_preferred_wakeup_cpu;
520#endif
514}; 521};
515 522
516/* 523/*
@@ -1139,7 +1146,6 @@ static void init_rq_hrtick(struct rq *rq)
1139 1146
1140 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1147 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1141 rq->hrtick_timer.function = hrtick; 1148 rq->hrtick_timer.function = hrtick;
1142 rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
1143} 1149}
1144#else /* CONFIG_SCHED_HRTICK */ 1150#else /* CONFIG_SCHED_HRTICK */
1145static inline void hrtick_clear(struct rq *rq) 1151static inline void hrtick_clear(struct rq *rq)
@@ -1516,7 +1522,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
1516 struct sched_domain *sd = data; 1522 struct sched_domain *sd = data;
1517 int i; 1523 int i;
1518 1524
1519 for_each_cpu_mask(i, sd->span) { 1525 for_each_cpu(i, sched_domain_span(sd)) {
1520 /* 1526 /*
1521 * If there are currently no tasks on the cpu pretend there 1527 * If there are currently no tasks on the cpu pretend there
1522 * is one of average load so that when a new task gets to 1528 * is one of average load so that when a new task gets to
@@ -1537,7 +1543,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
1537 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE)) 1543 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
1538 shares = tg->shares; 1544 shares = tg->shares;
1539 1545
1540 for_each_cpu_mask(i, sd->span) 1546 for_each_cpu(i, sched_domain_span(sd))
1541 update_group_shares_cpu(tg, i, shares, rq_weight); 1547 update_group_shares_cpu(tg, i, shares, rq_weight);
1542 1548
1543 return 0; 1549 return 0;
@@ -2103,15 +2109,17 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2103 int i; 2109 int i;
2104 2110
2105 /* Skip over this group if it has no CPUs allowed */ 2111 /* Skip over this group if it has no CPUs allowed */
2106 if (!cpus_intersects(group->cpumask, p->cpus_allowed)) 2112 if (!cpumask_intersects(sched_group_cpus(group),
2113 &p->cpus_allowed))
2107 continue; 2114 continue;
2108 2115
2109 local_group = cpu_isset(this_cpu, group->cpumask); 2116 local_group = cpumask_test_cpu(this_cpu,
2117 sched_group_cpus(group));
2110 2118
2111 /* Tally up the load of all CPUs in the group */ 2119 /* Tally up the load of all CPUs in the group */
2112 avg_load = 0; 2120 avg_load = 0;
2113 2121
2114 for_each_cpu_mask_nr(i, group->cpumask) { 2122 for_each_cpu(i, sched_group_cpus(group)) {
2115 /* Bias balancing toward cpus of our domain */ 2123 /* Bias balancing toward cpus of our domain */
2116 if (local_group) 2124 if (local_group)
2117 load = source_load(i, load_idx); 2125 load = source_load(i, load_idx);
@@ -2143,17 +2151,14 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2143 * find_idlest_cpu - find the idlest cpu among the cpus in group. 2151 * find_idlest_cpu - find the idlest cpu among the cpus in group.
2144 */ 2152 */
2145static int 2153static int
2146find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu, 2154find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
2147 cpumask_t *tmp)
2148{ 2155{
2149 unsigned long load, min_load = ULONG_MAX; 2156 unsigned long load, min_load = ULONG_MAX;
2150 int idlest = -1; 2157 int idlest = -1;
2151 int i; 2158 int i;
2152 2159
2153 /* Traverse only the allowed CPUs */ 2160 /* Traverse only the allowed CPUs */
2154 cpus_and(*tmp, group->cpumask, p->cpus_allowed); 2161 for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
2155
2156 for_each_cpu_mask_nr(i, *tmp) {
2157 load = weighted_cpuload(i); 2162 load = weighted_cpuload(i);
2158 2163
2159 if (load < min_load || (load == min_load && i == this_cpu)) { 2164 if (load < min_load || (load == min_load && i == this_cpu)) {
@@ -2195,7 +2200,6 @@ static int sched_balance_self(int cpu, int flag)
2195 update_shares(sd); 2200 update_shares(sd);
2196 2201
2197 while (sd) { 2202 while (sd) {
2198 cpumask_t span, tmpmask;
2199 struct sched_group *group; 2203 struct sched_group *group;
2200 int new_cpu, weight; 2204 int new_cpu, weight;
2201 2205
@@ -2204,14 +2208,13 @@ static int sched_balance_self(int cpu, int flag)
2204 continue; 2208 continue;
2205 } 2209 }
2206 2210
2207 span = sd->span;
2208 group = find_idlest_group(sd, t, cpu); 2211 group = find_idlest_group(sd, t, cpu);
2209 if (!group) { 2212 if (!group) {
2210 sd = sd->child; 2213 sd = sd->child;
2211 continue; 2214 continue;
2212 } 2215 }
2213 2216
2214 new_cpu = find_idlest_cpu(group, t, cpu, &tmpmask); 2217 new_cpu = find_idlest_cpu(group, t, cpu);
2215 if (new_cpu == -1 || new_cpu == cpu) { 2218 if (new_cpu == -1 || new_cpu == cpu) {
2216 /* Now try balancing at a lower domain level of cpu */ 2219 /* Now try balancing at a lower domain level of cpu */
2217 sd = sd->child; 2220 sd = sd->child;
@@ -2220,10 +2223,10 @@ static int sched_balance_self(int cpu, int flag)
2220 2223
2221 /* Now try balancing at a lower domain level of new_cpu */ 2224 /* Now try balancing at a lower domain level of new_cpu */
2222 cpu = new_cpu; 2225 cpu = new_cpu;
2226 weight = cpumask_weight(sched_domain_span(sd));
2223 sd = NULL; 2227 sd = NULL;
2224 weight = cpus_weight(span);
2225 for_each_domain(cpu, tmp) { 2228 for_each_domain(cpu, tmp) {
2226 if (weight <= cpus_weight(tmp->span)) 2229 if (weight <= cpumask_weight(sched_domain_span(tmp)))
2227 break; 2230 break;
2228 if (tmp->flags & flag) 2231 if (tmp->flags & flag)
2229 sd = tmp; 2232 sd = tmp;
@@ -2268,7 +2271,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2268 cpu = task_cpu(p); 2271 cpu = task_cpu(p);
2269 2272
2270 for_each_domain(this_cpu, sd) { 2273 for_each_domain(this_cpu, sd) {
2271 if (cpu_isset(cpu, sd->span)) { 2274 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2272 update_shares(sd); 2275 update_shares(sd);
2273 break; 2276 break;
2274 } 2277 }
@@ -2317,7 +2320,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2317 else { 2320 else {
2318 struct sched_domain *sd; 2321 struct sched_domain *sd;
2319 for_each_domain(this_cpu, sd) { 2322 for_each_domain(this_cpu, sd) {
2320 if (cpu_isset(cpu, sd->span)) { 2323 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2321 schedstat_inc(sd, ttwu_wake_remote); 2324 schedstat_inc(sd, ttwu_wake_remote);
2322 break; 2325 break;
2323 } 2326 }
@@ -2848,7 +2851,7 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu)
2848 struct rq *rq; 2851 struct rq *rq;
2849 2852
2850 rq = task_rq_lock(p, &flags); 2853 rq = task_rq_lock(p, &flags);
2851 if (!cpu_isset(dest_cpu, p->cpus_allowed) 2854 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
2852 || unlikely(!cpu_active(dest_cpu))) 2855 || unlikely(!cpu_active(dest_cpu)))
2853 goto out; 2856 goto out;
2854 2857
@@ -2913,7 +2916,7 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
2913 * 2) cannot be migrated to this CPU due to cpus_allowed, or 2916 * 2) cannot be migrated to this CPU due to cpus_allowed, or
2914 * 3) are cache-hot on their current CPU. 2917 * 3) are cache-hot on their current CPU.
2915 */ 2918 */
2916 if (!cpu_isset(this_cpu, p->cpus_allowed)) { 2919 if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
2917 schedstat_inc(p, se.nr_failed_migrations_affine); 2920 schedstat_inc(p, se.nr_failed_migrations_affine);
2918 return 0; 2921 return 0;
2919 } 2922 }
@@ -3088,7 +3091,7 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
3088static struct sched_group * 3091static struct sched_group *
3089find_busiest_group(struct sched_domain *sd, int this_cpu, 3092find_busiest_group(struct sched_domain *sd, int this_cpu,
3090 unsigned long *imbalance, enum cpu_idle_type idle, 3093 unsigned long *imbalance, enum cpu_idle_type idle,
3091 int *sd_idle, const cpumask_t *cpus, int *balance) 3094 int *sd_idle, const struct cpumask *cpus, int *balance)
3092{ 3095{
3093 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups; 3096 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
3094 unsigned long max_load, avg_load, total_load, this_load, total_pwr; 3097 unsigned long max_load, avg_load, total_load, this_load, total_pwr;
@@ -3124,10 +3127,11 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3124 unsigned long sum_avg_load_per_task; 3127 unsigned long sum_avg_load_per_task;
3125 unsigned long avg_load_per_task; 3128 unsigned long avg_load_per_task;
3126 3129
3127 local_group = cpu_isset(this_cpu, group->cpumask); 3130 local_group = cpumask_test_cpu(this_cpu,
3131 sched_group_cpus(group));
3128 3132
3129 if (local_group) 3133 if (local_group)
3130 balance_cpu = first_cpu(group->cpumask); 3134 balance_cpu = cpumask_first(sched_group_cpus(group));
3131 3135
3132 /* Tally up the load of all CPUs in the group */ 3136 /* Tally up the load of all CPUs in the group */
3133 sum_weighted_load = sum_nr_running = avg_load = 0; 3137 sum_weighted_load = sum_nr_running = avg_load = 0;
@@ -3136,13 +3140,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3136 max_cpu_load = 0; 3140 max_cpu_load = 0;
3137 min_cpu_load = ~0UL; 3141 min_cpu_load = ~0UL;
3138 3142
3139 for_each_cpu_mask_nr(i, group->cpumask) { 3143 for_each_cpu_and(i, sched_group_cpus(group), cpus) {
3140 struct rq *rq; 3144 struct rq *rq = cpu_rq(i);
3141
3142 if (!cpu_isset(i, *cpus))
3143 continue;
3144
3145 rq = cpu_rq(i);
3146 3145
3147 if (*sd_idle && rq->nr_running) 3146 if (*sd_idle && rq->nr_running)
3148 *sd_idle = 0; 3147 *sd_idle = 0;
@@ -3253,8 +3252,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3253 */ 3252 */
3254 if ((sum_nr_running < min_nr_running) || 3253 if ((sum_nr_running < min_nr_running) ||
3255 (sum_nr_running == min_nr_running && 3254 (sum_nr_running == min_nr_running &&
3256 first_cpu(group->cpumask) < 3255 cpumask_first(sched_group_cpus(group)) >
3257 first_cpu(group_min->cpumask))) { 3256 cpumask_first(sched_group_cpus(group_min)))) {
3258 group_min = group; 3257 group_min = group;
3259 min_nr_running = sum_nr_running; 3258 min_nr_running = sum_nr_running;
3260 min_load_per_task = sum_weighted_load / 3259 min_load_per_task = sum_weighted_load /
@@ -3269,8 +3268,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3269 if (sum_nr_running <= group_capacity - 1) { 3268 if (sum_nr_running <= group_capacity - 1) {
3270 if (sum_nr_running > leader_nr_running || 3269 if (sum_nr_running > leader_nr_running ||
3271 (sum_nr_running == leader_nr_running && 3270 (sum_nr_running == leader_nr_running &&
3272 first_cpu(group->cpumask) > 3271 cpumask_first(sched_group_cpus(group)) <
3273 first_cpu(group_leader->cpumask))) { 3272 cpumask_first(sched_group_cpus(group_leader)))) {
3274 group_leader = group; 3273 group_leader = group;
3275 leader_nr_running = sum_nr_running; 3274 leader_nr_running = sum_nr_running;
3276 } 3275 }
@@ -3396,6 +3395,10 @@ out_balanced:
3396 3395
3397 if (this == group_leader && group_leader != group_min) { 3396 if (this == group_leader && group_leader != group_min) {
3398 *imbalance = min_load_per_task; 3397 *imbalance = min_load_per_task;
3398 if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP) {
3399 cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu =
3400 cpumask_first(sched_group_cpus(group_leader));
3401 }
3399 return group_min; 3402 return group_min;
3400 } 3403 }
3401#endif 3404#endif
@@ -3409,16 +3412,16 @@ ret:
3409 */ 3412 */
3410static struct rq * 3413static struct rq *
3411find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle, 3414find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3412 unsigned long imbalance, const cpumask_t *cpus) 3415 unsigned long imbalance, const struct cpumask *cpus)
3413{ 3416{
3414 struct rq *busiest = NULL, *rq; 3417 struct rq *busiest = NULL, *rq;
3415 unsigned long max_load = 0; 3418 unsigned long max_load = 0;
3416 int i; 3419 int i;
3417 3420
3418 for_each_cpu_mask_nr(i, group->cpumask) { 3421 for_each_cpu(i, sched_group_cpus(group)) {
3419 unsigned long wl; 3422 unsigned long wl;
3420 3423
3421 if (!cpu_isset(i, *cpus)) 3424 if (!cpumask_test_cpu(i, cpus))
3422 continue; 3425 continue;
3423 3426
3424 rq = cpu_rq(i); 3427 rq = cpu_rq(i);
@@ -3448,7 +3451,7 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3448 */ 3451 */
3449static int load_balance(int this_cpu, struct rq *this_rq, 3452static int load_balance(int this_cpu, struct rq *this_rq,
3450 struct sched_domain *sd, enum cpu_idle_type idle, 3453 struct sched_domain *sd, enum cpu_idle_type idle,
3451 int *balance, cpumask_t *cpus) 3454 int *balance, struct cpumask *cpus)
3452{ 3455{
3453 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0; 3456 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
3454 struct sched_group *group; 3457 struct sched_group *group;
@@ -3456,7 +3459,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
3456 struct rq *busiest; 3459 struct rq *busiest;
3457 unsigned long flags; 3460 unsigned long flags;
3458 3461
3459 cpus_setall(*cpus); 3462 cpumask_setall(cpus);
3460 3463
3461 /* 3464 /*
3462 * When power savings policy is enabled for the parent domain, idle 3465 * When power savings policy is enabled for the parent domain, idle
@@ -3516,8 +3519,8 @@ redo:
3516 3519
3517 /* All tasks on this runqueue were pinned by CPU affinity */ 3520 /* All tasks on this runqueue were pinned by CPU affinity */
3518 if (unlikely(all_pinned)) { 3521 if (unlikely(all_pinned)) {
3519 cpu_clear(cpu_of(busiest), *cpus); 3522 cpumask_clear_cpu(cpu_of(busiest), cpus);
3520 if (!cpus_empty(*cpus)) 3523 if (!cpumask_empty(cpus))
3521 goto redo; 3524 goto redo;
3522 goto out_balanced; 3525 goto out_balanced;
3523 } 3526 }
@@ -3534,7 +3537,8 @@ redo:
3534 /* don't kick the migration_thread, if the curr 3537 /* don't kick the migration_thread, if the curr
3535 * task on busiest cpu can't be moved to this_cpu 3538 * task on busiest cpu can't be moved to this_cpu
3536 */ 3539 */
3537 if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) { 3540 if (!cpumask_test_cpu(this_cpu,
3541 &busiest->curr->cpus_allowed)) {
3538 spin_unlock_irqrestore(&busiest->lock, flags); 3542 spin_unlock_irqrestore(&busiest->lock, flags);
3539 all_pinned = 1; 3543 all_pinned = 1;
3540 goto out_one_pinned; 3544 goto out_one_pinned;
@@ -3609,7 +3613,7 @@ out:
3609 */ 3613 */
3610static int 3614static int
3611load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd, 3615load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3612 cpumask_t *cpus) 3616 struct cpumask *cpus)
3613{ 3617{
3614 struct sched_group *group; 3618 struct sched_group *group;
3615 struct rq *busiest = NULL; 3619 struct rq *busiest = NULL;
@@ -3618,7 +3622,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3618 int sd_idle = 0; 3622 int sd_idle = 0;
3619 int all_pinned = 0; 3623 int all_pinned = 0;
3620 3624
3621 cpus_setall(*cpus); 3625 cpumask_setall(cpus);
3622 3626
3623 /* 3627 /*
3624 * When power savings policy is enabled for the parent domain, idle 3628 * When power savings policy is enabled for the parent domain, idle
@@ -3662,17 +3666,71 @@ redo:
3662 double_unlock_balance(this_rq, busiest); 3666 double_unlock_balance(this_rq, busiest);
3663 3667
3664 if (unlikely(all_pinned)) { 3668 if (unlikely(all_pinned)) {
3665 cpu_clear(cpu_of(busiest), *cpus); 3669 cpumask_clear_cpu(cpu_of(busiest), cpus);
3666 if (!cpus_empty(*cpus)) 3670 if (!cpumask_empty(cpus))
3667 goto redo; 3671 goto redo;
3668 } 3672 }
3669 } 3673 }
3670 3674
3671 if (!ld_moved) { 3675 if (!ld_moved) {
3676 int active_balance = 0;
3677
3672 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]); 3678 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
3673 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER && 3679 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3674 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE)) 3680 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3675 return -1; 3681 return -1;
3682
3683 if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
3684 return -1;
3685
3686 if (sd->nr_balance_failed++ < 2)
3687 return -1;
3688
3689 /*
3690 * The only task running in a non-idle cpu can be moved to this
3691 * cpu in an attempt to completely freeup the other CPU
3692 * package. The same method used to move task in load_balance()
3693 * have been extended for load_balance_newidle() to speedup
3694 * consolidation at sched_mc=POWERSAVINGS_BALANCE_WAKEUP (2)
3695 *
3696 * The package power saving logic comes from
3697 * find_busiest_group(). If there are no imbalance, then
3698 * f_b_g() will return NULL. However when sched_mc={1,2} then
3699 * f_b_g() will select a group from which a running task may be
3700 * pulled to this cpu in order to make the other package idle.
3701 * If there is no opportunity to make a package idle and if
3702 * there are no imbalance, then f_b_g() will return NULL and no
3703 * action will be taken in load_balance_newidle().
3704 *
3705 * Under normal task pull operation due to imbalance, there
3706 * will be more than one task in the source run queue and
3707 * move_tasks() will succeed. ld_moved will be true and this
3708 * active balance code will not be triggered.
3709 */
3710
3711 /* Lock busiest in correct order while this_rq is held */
3712 double_lock_balance(this_rq, busiest);
3713
3714 /*
3715 * don't kick the migration_thread, if the curr
3716 * task on busiest cpu can't be moved to this_cpu
3717 */
3718 if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) {
3719 double_unlock_balance(this_rq, busiest);
3720 all_pinned = 1;
3721 return ld_moved;
3722 }
3723
3724 if (!busiest->active_balance) {
3725 busiest->active_balance = 1;
3726 busiest->push_cpu = this_cpu;
3727 active_balance = 1;
3728 }
3729
3730 double_unlock_balance(this_rq, busiest);
3731 if (active_balance)
3732 wake_up_process(busiest->migration_thread);
3733
3676 } else 3734 } else
3677 sd->nr_balance_failed = 0; 3735 sd->nr_balance_failed = 0;
3678 3736
@@ -3698,7 +3756,10 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3698 struct sched_domain *sd; 3756 struct sched_domain *sd;
3699 int pulled_task = 0; 3757 int pulled_task = 0;
3700 unsigned long next_balance = jiffies + HZ; 3758 unsigned long next_balance = jiffies + HZ;
3701 cpumask_t tmpmask; 3759 cpumask_var_t tmpmask;
3760
3761 if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
3762 return;
3702 3763
3703 for_each_domain(this_cpu, sd) { 3764 for_each_domain(this_cpu, sd) {
3704 unsigned long interval; 3765 unsigned long interval;
@@ -3709,7 +3770,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3709 if (sd->flags & SD_BALANCE_NEWIDLE) 3770 if (sd->flags & SD_BALANCE_NEWIDLE)
3710 /* If we've pulled tasks over stop searching: */ 3771 /* If we've pulled tasks over stop searching: */
3711 pulled_task = load_balance_newidle(this_cpu, this_rq, 3772 pulled_task = load_balance_newidle(this_cpu, this_rq,
3712 sd, &tmpmask); 3773 sd, tmpmask);
3713 3774
3714 interval = msecs_to_jiffies(sd->balance_interval); 3775 interval = msecs_to_jiffies(sd->balance_interval);
3715 if (time_after(next_balance, sd->last_balance + interval)) 3776 if (time_after(next_balance, sd->last_balance + interval))
@@ -3724,6 +3785,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3724 */ 3785 */
3725 this_rq->next_balance = next_balance; 3786 this_rq->next_balance = next_balance;
3726 } 3787 }
3788 free_cpumask_var(tmpmask);
3727} 3789}
3728 3790
3729/* 3791/*
@@ -3761,7 +3823,7 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3761 /* Search for an sd spanning us and the target CPU. */ 3823 /* Search for an sd spanning us and the target CPU. */
3762 for_each_domain(target_cpu, sd) { 3824 for_each_domain(target_cpu, sd) {
3763 if ((sd->flags & SD_LOAD_BALANCE) && 3825 if ((sd->flags & SD_LOAD_BALANCE) &&
3764 cpu_isset(busiest_cpu, sd->span)) 3826 cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
3765 break; 3827 break;
3766 } 3828 }
3767 3829
@@ -3780,10 +3842,9 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3780#ifdef CONFIG_NO_HZ 3842#ifdef CONFIG_NO_HZ
3781static struct { 3843static struct {
3782 atomic_t load_balancer; 3844 atomic_t load_balancer;
3783 cpumask_t cpu_mask; 3845 cpumask_var_t cpu_mask;
3784} nohz ____cacheline_aligned = { 3846} nohz ____cacheline_aligned = {
3785 .load_balancer = ATOMIC_INIT(-1), 3847 .load_balancer = ATOMIC_INIT(-1),
3786 .cpu_mask = CPU_MASK_NONE,
3787}; 3848};
3788 3849
3789/* 3850/*
@@ -3811,7 +3872,7 @@ int select_nohz_load_balancer(int stop_tick)
3811 int cpu = smp_processor_id(); 3872 int cpu = smp_processor_id();
3812 3873
3813 if (stop_tick) { 3874 if (stop_tick) {
3814 cpu_set(cpu, nohz.cpu_mask); 3875 cpumask_set_cpu(cpu, nohz.cpu_mask);
3815 cpu_rq(cpu)->in_nohz_recently = 1; 3876 cpu_rq(cpu)->in_nohz_recently = 1;
3816 3877
3817 /* 3878 /*
@@ -3825,7 +3886,7 @@ int select_nohz_load_balancer(int stop_tick)
3825 } 3886 }
3826 3887
3827 /* time for ilb owner also to sleep */ 3888 /* time for ilb owner also to sleep */
3828 if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 3889 if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
3829 if (atomic_read(&nohz.load_balancer) == cpu) 3890 if (atomic_read(&nohz.load_balancer) == cpu)
3830 atomic_set(&nohz.load_balancer, -1); 3891 atomic_set(&nohz.load_balancer, -1);
3831 return 0; 3892 return 0;
@@ -3838,10 +3899,10 @@ int select_nohz_load_balancer(int stop_tick)
3838 } else if (atomic_read(&nohz.load_balancer) == cpu) 3899 } else if (atomic_read(&nohz.load_balancer) == cpu)
3839 return 1; 3900 return 1;
3840 } else { 3901 } else {
3841 if (!cpu_isset(cpu, nohz.cpu_mask)) 3902 if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
3842 return 0; 3903 return 0;
3843 3904
3844 cpu_clear(cpu, nohz.cpu_mask); 3905 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3845 3906
3846 if (atomic_read(&nohz.load_balancer) == cpu) 3907 if (atomic_read(&nohz.load_balancer) == cpu)
3847 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu) 3908 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
@@ -3869,7 +3930,11 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3869 unsigned long next_balance = jiffies + 60*HZ; 3930 unsigned long next_balance = jiffies + 60*HZ;
3870 int update_next_balance = 0; 3931 int update_next_balance = 0;
3871 int need_serialize; 3932 int need_serialize;
3872 cpumask_t tmp; 3933 cpumask_var_t tmp;
3934
3935 /* Fails alloc? Rebalancing probably not a priority right now. */
3936 if (!alloc_cpumask_var(&tmp, GFP_ATOMIC))
3937 return;
3873 3938
3874 for_each_domain(cpu, sd) { 3939 for_each_domain(cpu, sd) {
3875 if (!(sd->flags & SD_LOAD_BALANCE)) 3940 if (!(sd->flags & SD_LOAD_BALANCE))
@@ -3894,7 +3959,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3894 } 3959 }
3895 3960
3896 if (time_after_eq(jiffies, sd->last_balance + interval)) { 3961 if (time_after_eq(jiffies, sd->last_balance + interval)) {
3897 if (load_balance(cpu, rq, sd, idle, &balance, &tmp)) { 3962 if (load_balance(cpu, rq, sd, idle, &balance, tmp)) {
3898 /* 3963 /*
3899 * We've pulled tasks over so either we're no 3964 * We've pulled tasks over so either we're no
3900 * longer idle, or one of our SMT siblings is 3965 * longer idle, or one of our SMT siblings is
@@ -3928,6 +3993,8 @@ out:
3928 */ 3993 */
3929 if (likely(update_next_balance)) 3994 if (likely(update_next_balance))
3930 rq->next_balance = next_balance; 3995 rq->next_balance = next_balance;
3996
3997 free_cpumask_var(tmp);
3931} 3998}
3932 3999
3933/* 4000/*
@@ -3952,12 +4019,13 @@ static void run_rebalance_domains(struct softirq_action *h)
3952 */ 4019 */
3953 if (this_rq->idle_at_tick && 4020 if (this_rq->idle_at_tick &&
3954 atomic_read(&nohz.load_balancer) == this_cpu) { 4021 atomic_read(&nohz.load_balancer) == this_cpu) {
3955 cpumask_t cpus = nohz.cpu_mask;
3956 struct rq *rq; 4022 struct rq *rq;
3957 int balance_cpu; 4023 int balance_cpu;
3958 4024
3959 cpu_clear(this_cpu, cpus); 4025 for_each_cpu(balance_cpu, nohz.cpu_mask) {
3960 for_each_cpu_mask_nr(balance_cpu, cpus) { 4026 if (balance_cpu == this_cpu)
4027 continue;
4028
3961 /* 4029 /*
3962 * If this cpu gets work to do, stop the load balancing 4030 * If this cpu gets work to do, stop the load balancing
3963 * work being done for other cpus. Next load 4031 * work being done for other cpus. Next load
@@ -3995,7 +4063,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
3995 rq->in_nohz_recently = 0; 4063 rq->in_nohz_recently = 0;
3996 4064
3997 if (atomic_read(&nohz.load_balancer) == cpu) { 4065 if (atomic_read(&nohz.load_balancer) == cpu) {
3998 cpu_clear(cpu, nohz.cpu_mask); 4066 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3999 atomic_set(&nohz.load_balancer, -1); 4067 atomic_set(&nohz.load_balancer, -1);
4000 } 4068 }
4001 4069
@@ -4008,7 +4076,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4008 * TBD: Traverse the sched domains and nominate 4076 * TBD: Traverse the sched domains and nominate
4009 * the nearest cpu in the nohz.cpu_mask. 4077 * the nearest cpu in the nohz.cpu_mask.
4010 */ 4078 */
4011 int ilb = first_cpu(nohz.cpu_mask); 4079 int ilb = cpumask_first(nohz.cpu_mask);
4012 4080
4013 if (ilb < nr_cpu_ids) 4081 if (ilb < nr_cpu_ids)
4014 resched_cpu(ilb); 4082 resched_cpu(ilb);
@@ -4020,7 +4088,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4020 * cpus with ticks stopped, is it time for that to stop? 4088 * cpus with ticks stopped, is it time for that to stop?
4021 */ 4089 */
4022 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu && 4090 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
4023 cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 4091 cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
4024 resched_cpu(cpu); 4092 resched_cpu(cpu);
4025 return; 4093 return;
4026 } 4094 }
@@ -4030,7 +4098,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4030 * someone else, then no need raise the SCHED_SOFTIRQ 4098 * someone else, then no need raise the SCHED_SOFTIRQ
4031 */ 4099 */
4032 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu && 4100 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
4033 cpu_isset(cpu, nohz.cpu_mask)) 4101 cpumask_test_cpu(cpu, nohz.cpu_mask))
4034 return; 4102 return;
4035#endif 4103#endif
4036 if (time_after_eq(jiffies, rq->next_balance)) 4104 if (time_after_eq(jiffies, rq->next_balance))
@@ -4192,7 +4260,6 @@ void account_steal_time(struct task_struct *p, cputime_t steal)
4192 4260
4193 if (p == rq->idle) { 4261 if (p == rq->idle) {
4194 p->stime = cputime_add(p->stime, steal); 4262 p->stime = cputime_add(p->stime, steal);
4195 account_group_system_time(p, steal);
4196 if (atomic_read(&rq->nr_iowait) > 0) 4263 if (atomic_read(&rq->nr_iowait) > 0)
4197 cpustat->iowait = cputime64_add(cpustat->iowait, tmp); 4264 cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
4198 else 4265 else
@@ -4328,7 +4395,7 @@ void __kprobes sub_preempt_count(int val)
4328 /* 4395 /*
4329 * Underflow? 4396 * Underflow?
4330 */ 4397 */
4331 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 4398 if (DEBUG_LOCKS_WARN_ON(val > preempt_count() - (!!kernel_locked())))
4332 return; 4399 return;
4333 /* 4400 /*
4334 * Is the spinlock portion underflowing? 4401 * Is the spinlock portion underflowing?
@@ -5404,10 +5471,9 @@ out_unlock:
5404 return retval; 5471 return retval;
5405} 5472}
5406 5473
5407long sched_setaffinity(pid_t pid, const cpumask_t *in_mask) 5474long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
5408{ 5475{
5409 cpumask_t cpus_allowed; 5476 cpumask_var_t cpus_allowed, new_mask;
5410 cpumask_t new_mask = *in_mask;
5411 struct task_struct *p; 5477 struct task_struct *p;
5412 int retval; 5478 int retval;
5413 5479
@@ -5429,6 +5495,14 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5429 get_task_struct(p); 5495 get_task_struct(p);
5430 read_unlock(&tasklist_lock); 5496 read_unlock(&tasklist_lock);
5431 5497
5498 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
5499 retval = -ENOMEM;
5500 goto out_put_task;
5501 }
5502 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
5503 retval = -ENOMEM;
5504 goto out_free_cpus_allowed;
5505 }
5432 retval = -EPERM; 5506 retval = -EPERM;
5433 if (!check_same_owner(p) && !capable(CAP_SYS_NICE)) 5507 if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
5434 goto out_unlock; 5508 goto out_unlock;
@@ -5437,37 +5511,41 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5437 if (retval) 5511 if (retval)
5438 goto out_unlock; 5512 goto out_unlock;
5439 5513
5440 cpuset_cpus_allowed(p, &cpus_allowed); 5514 cpuset_cpus_allowed(p, cpus_allowed);
5441 cpus_and(new_mask, new_mask, cpus_allowed); 5515 cpumask_and(new_mask, in_mask, cpus_allowed);
5442 again: 5516 again:
5443 retval = set_cpus_allowed_ptr(p, &new_mask); 5517 retval = set_cpus_allowed_ptr(p, new_mask);
5444 5518
5445 if (!retval) { 5519 if (!retval) {
5446 cpuset_cpus_allowed(p, &cpus_allowed); 5520 cpuset_cpus_allowed(p, cpus_allowed);
5447 if (!cpus_subset(new_mask, cpus_allowed)) { 5521 if (!cpumask_subset(new_mask, cpus_allowed)) {
5448 /* 5522 /*
5449 * We must have raced with a concurrent cpuset 5523 * We must have raced with a concurrent cpuset
5450 * update. Just reset the cpus_allowed to the 5524 * update. Just reset the cpus_allowed to the
5451 * cpuset's cpus_allowed 5525 * cpuset's cpus_allowed
5452 */ 5526 */
5453 new_mask = cpus_allowed; 5527 cpumask_copy(new_mask, cpus_allowed);
5454 goto again; 5528 goto again;
5455 } 5529 }
5456 } 5530 }
5457out_unlock: 5531out_unlock:
5532 free_cpumask_var(new_mask);
5533out_free_cpus_allowed:
5534 free_cpumask_var(cpus_allowed);
5535out_put_task:
5458 put_task_struct(p); 5536 put_task_struct(p);
5459 put_online_cpus(); 5537 put_online_cpus();
5460 return retval; 5538 return retval;
5461} 5539}
5462 5540
5463static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 5541static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5464 cpumask_t *new_mask) 5542 struct cpumask *new_mask)
5465{ 5543{
5466 if (len < sizeof(cpumask_t)) { 5544 if (len < cpumask_size())
5467 memset(new_mask, 0, sizeof(cpumask_t)); 5545 cpumask_clear(new_mask);
5468 } else if (len > sizeof(cpumask_t)) { 5546 else if (len > cpumask_size())
5469 len = sizeof(cpumask_t); 5547 len = cpumask_size();
5470 } 5548
5471 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 5549 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
5472} 5550}
5473 5551
@@ -5480,17 +5558,20 @@ static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5480asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, 5558asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
5481 unsigned long __user *user_mask_ptr) 5559 unsigned long __user *user_mask_ptr)
5482{ 5560{
5483 cpumask_t new_mask; 5561 cpumask_var_t new_mask;
5484 int retval; 5562 int retval;
5485 5563
5486 retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask); 5564 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
5487 if (retval) 5565 return -ENOMEM;
5488 return retval;
5489 5566
5490 return sched_setaffinity(pid, &new_mask); 5567 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
5568 if (retval == 0)
5569 retval = sched_setaffinity(pid, new_mask);
5570 free_cpumask_var(new_mask);
5571 return retval;
5491} 5572}
5492 5573
5493long sched_getaffinity(pid_t pid, cpumask_t *mask) 5574long sched_getaffinity(pid_t pid, struct cpumask *mask)
5494{ 5575{
5495 struct task_struct *p; 5576 struct task_struct *p;
5496 int retval; 5577 int retval;
@@ -5507,7 +5588,7 @@ long sched_getaffinity(pid_t pid, cpumask_t *mask)
5507 if (retval) 5588 if (retval)
5508 goto out_unlock; 5589 goto out_unlock;
5509 5590
5510 cpus_and(*mask, p->cpus_allowed, cpu_online_map); 5591 cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
5511 5592
5512out_unlock: 5593out_unlock:
5513 read_unlock(&tasklist_lock); 5594 read_unlock(&tasklist_lock);
@@ -5526,19 +5607,24 @@ asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
5526 unsigned long __user *user_mask_ptr) 5607 unsigned long __user *user_mask_ptr)
5527{ 5608{
5528 int ret; 5609 int ret;
5529 cpumask_t mask; 5610 cpumask_var_t mask;
5530 5611
5531 if (len < sizeof(cpumask_t)) 5612 if (len < cpumask_size())
5532 return -EINVAL; 5613 return -EINVAL;
5533 5614
5534 ret = sched_getaffinity(pid, &mask); 5615 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
5535 if (ret < 0) 5616 return -ENOMEM;
5536 return ret;
5537 5617
5538 if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t))) 5618 ret = sched_getaffinity(pid, mask);
5539 return -EFAULT; 5619 if (ret == 0) {
5620 if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
5621 ret = -EFAULT;
5622 else
5623 ret = cpumask_size();
5624 }
5625 free_cpumask_var(mask);
5540 5626
5541 return sizeof(cpumask_t); 5627 return ret;
5542} 5628}
5543 5629
5544/** 5630/**
@@ -5880,7 +5966,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5880 idle->se.exec_start = sched_clock(); 5966 idle->se.exec_start = sched_clock();
5881 5967
5882 idle->prio = idle->normal_prio = MAX_PRIO; 5968 idle->prio = idle->normal_prio = MAX_PRIO;
5883 idle->cpus_allowed = cpumask_of_cpu(cpu); 5969 cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
5884 __set_task_cpu(idle, cpu); 5970 __set_task_cpu(idle, cpu);
5885 5971
5886 rq->curr = rq->idle = idle; 5972 rq->curr = rq->idle = idle;
@@ -5907,9 +5993,9 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5907 * indicates which cpus entered this state. This is used 5993 * indicates which cpus entered this state. This is used
5908 * in the rcu update to wait only for active cpus. For system 5994 * in the rcu update to wait only for active cpus. For system
5909 * which do not switch off the HZ timer nohz_cpu_mask should 5995 * which do not switch off the HZ timer nohz_cpu_mask should
5910 * always be CPU_MASK_NONE. 5996 * always be CPU_BITS_NONE.
5911 */ 5997 */
5912cpumask_t nohz_cpu_mask = CPU_MASK_NONE; 5998cpumask_var_t nohz_cpu_mask;
5913 5999
5914/* 6000/*
5915 * Increase the granularity value when there are more CPUs, 6001 * Increase the granularity value when there are more CPUs,
@@ -5964,7 +6050,7 @@ static inline void sched_init_granularity(void)
5964 * task must not exit() & deallocate itself prematurely. The 6050 * task must not exit() & deallocate itself prematurely. The
5965 * call is not atomic; no spinlocks may be held. 6051 * call is not atomic; no spinlocks may be held.
5966 */ 6052 */
5967int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask) 6053int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
5968{ 6054{
5969 struct migration_req req; 6055 struct migration_req req;
5970 unsigned long flags; 6056 unsigned long flags;
@@ -5972,13 +6058,13 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5972 int ret = 0; 6058 int ret = 0;
5973 6059
5974 rq = task_rq_lock(p, &flags); 6060 rq = task_rq_lock(p, &flags);
5975 if (!cpus_intersects(*new_mask, cpu_online_map)) { 6061 if (!cpumask_intersects(new_mask, cpu_online_mask)) {
5976 ret = -EINVAL; 6062 ret = -EINVAL;
5977 goto out; 6063 goto out;
5978 } 6064 }
5979 6065
5980 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current && 6066 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
5981 !cpus_equal(p->cpus_allowed, *new_mask))) { 6067 !cpumask_equal(&p->cpus_allowed, new_mask))) {
5982 ret = -EINVAL; 6068 ret = -EINVAL;
5983 goto out; 6069 goto out;
5984 } 6070 }
@@ -5986,15 +6072,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5986 if (p->sched_class->set_cpus_allowed) 6072 if (p->sched_class->set_cpus_allowed)
5987 p->sched_class->set_cpus_allowed(p, new_mask); 6073 p->sched_class->set_cpus_allowed(p, new_mask);
5988 else { 6074 else {
5989 p->cpus_allowed = *new_mask; 6075 cpumask_copy(&p->cpus_allowed, new_mask);
5990 p->rt.nr_cpus_allowed = cpus_weight(*new_mask); 6076 p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
5991 } 6077 }
5992 6078
5993 /* Can the task run on the task's current CPU? If so, we're done */ 6079 /* Can the task run on the task's current CPU? If so, we're done */
5994 if (cpu_isset(task_cpu(p), *new_mask)) 6080 if (cpumask_test_cpu(task_cpu(p), new_mask))
5995 goto out; 6081 goto out;
5996 6082
5997 if (migrate_task(p, any_online_cpu(*new_mask), &req)) { 6083 if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
5998 /* Need help from migration thread: drop lock and wait. */ 6084 /* Need help from migration thread: drop lock and wait. */
5999 task_rq_unlock(rq, &flags); 6085 task_rq_unlock(rq, &flags);
6000 wake_up_process(rq->migration_thread); 6086 wake_up_process(rq->migration_thread);
@@ -6036,7 +6122,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
6036 if (task_cpu(p) != src_cpu) 6122 if (task_cpu(p) != src_cpu)
6037 goto done; 6123 goto done;
6038 /* Affinity changed (again). */ 6124 /* Affinity changed (again). */
6039 if (!cpu_isset(dest_cpu, p->cpus_allowed)) 6125 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6040 goto fail; 6126 goto fail;
6041 6127
6042 on_rq = p->se.on_rq; 6128 on_rq = p->se.on_rq;
@@ -6133,50 +6219,43 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
6133 */ 6219 */
6134static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) 6220static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6135{ 6221{
6136 unsigned long flags;
6137 cpumask_t mask;
6138 struct rq *rq;
6139 int dest_cpu; 6222 int dest_cpu;
6223 /* FIXME: Use cpumask_of_node here. */
6224 cpumask_t _nodemask = node_to_cpumask(cpu_to_node(dead_cpu));
6225 const struct cpumask *nodemask = &_nodemask;
6226
6227again:
6228 /* Look for allowed, online CPU in same node. */
6229 for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
6230 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6231 goto move;
6232
6233 /* Any allowed, online CPU? */
6234 dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
6235 if (dest_cpu < nr_cpu_ids)
6236 goto move;
6237
6238 /* No more Mr. Nice Guy. */
6239 if (dest_cpu >= nr_cpu_ids) {
6240 cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
6241 dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
6140 6242
6141 do { 6243 /*
6142 /* On same node? */ 6244 * Don't tell them about moving exiting tasks or
6143 mask = node_to_cpumask(cpu_to_node(dead_cpu)); 6245 * kernel threads (both mm NULL), since they never
6144 cpus_and(mask, mask, p->cpus_allowed); 6246 * leave kernel.
6145 dest_cpu = any_online_cpu(mask); 6247 */
6146 6248 if (p->mm && printk_ratelimit()) {
6147 /* On any allowed CPU? */ 6249 printk(KERN_INFO "process %d (%s) no "
6148 if (dest_cpu >= nr_cpu_ids) 6250 "longer affine to cpu%d\n",
6149 dest_cpu = any_online_cpu(p->cpus_allowed); 6251 task_pid_nr(p), p->comm, dead_cpu);
6150
6151 /* No more Mr. Nice Guy. */
6152 if (dest_cpu >= nr_cpu_ids) {
6153 cpumask_t cpus_allowed;
6154
6155 cpuset_cpus_allowed_locked(p, &cpus_allowed);
6156 /*
6157 * Try to stay on the same cpuset, where the
6158 * current cpuset may be a subset of all cpus.
6159 * The cpuset_cpus_allowed_locked() variant of
6160 * cpuset_cpus_allowed() will not block. It must be
6161 * called within calls to cpuset_lock/cpuset_unlock.
6162 */
6163 rq = task_rq_lock(p, &flags);
6164 p->cpus_allowed = cpus_allowed;
6165 dest_cpu = any_online_cpu(p->cpus_allowed);
6166 task_rq_unlock(rq, &flags);
6167
6168 /*
6169 * Don't tell them about moving exiting tasks or
6170 * kernel threads (both mm NULL), since they never
6171 * leave kernel.
6172 */
6173 if (p->mm && printk_ratelimit()) {
6174 printk(KERN_INFO "process %d (%s) no "
6175 "longer affine to cpu%d\n",
6176 task_pid_nr(p), p->comm, dead_cpu);
6177 }
6178 } 6252 }
6179 } while (!__migrate_task_irq(p, dead_cpu, dest_cpu)); 6253 }
6254
6255move:
6256 /* It can have affinity changed while we were choosing. */
6257 if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
6258 goto again;
6180} 6259}
6181 6260
6182/* 6261/*
@@ -6188,7 +6267,7 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6188 */ 6267 */
6189static void migrate_nr_uninterruptible(struct rq *rq_src) 6268static void migrate_nr_uninterruptible(struct rq *rq_src)
6190{ 6269{
6191 struct rq *rq_dest = cpu_rq(any_online_cpu(*CPU_MASK_ALL_PTR)); 6270 struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
6192 unsigned long flags; 6271 unsigned long flags;
6193 6272
6194 local_irq_save(flags); 6273 local_irq_save(flags);
@@ -6478,7 +6557,7 @@ static void set_rq_online(struct rq *rq)
6478 if (!rq->online) { 6557 if (!rq->online) {
6479 const struct sched_class *class; 6558 const struct sched_class *class;
6480 6559
6481 cpu_set(rq->cpu, rq->rd->online); 6560 cpumask_set_cpu(rq->cpu, rq->rd->online);
6482 rq->online = 1; 6561 rq->online = 1;
6483 6562
6484 for_each_class(class) { 6563 for_each_class(class) {
@@ -6498,7 +6577,7 @@ static void set_rq_offline(struct rq *rq)
6498 class->rq_offline(rq); 6577 class->rq_offline(rq);
6499 } 6578 }
6500 6579
6501 cpu_clear(rq->cpu, rq->rd->online); 6580 cpumask_clear_cpu(rq->cpu, rq->rd->online);
6502 rq->online = 0; 6581 rq->online = 0;
6503 } 6582 }
6504} 6583}
@@ -6539,7 +6618,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6539 rq = cpu_rq(cpu); 6618 rq = cpu_rq(cpu);
6540 spin_lock_irqsave(&rq->lock, flags); 6619 spin_lock_irqsave(&rq->lock, flags);
6541 if (rq->rd) { 6620 if (rq->rd) {
6542 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6621 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6543 6622
6544 set_rq_online(rq); 6623 set_rq_online(rq);
6545 } 6624 }
@@ -6553,7 +6632,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6553 break; 6632 break;
6554 /* Unbind it from offline cpu so it can run. Fall thru. */ 6633 /* Unbind it from offline cpu so it can run. Fall thru. */
6555 kthread_bind(cpu_rq(cpu)->migration_thread, 6634 kthread_bind(cpu_rq(cpu)->migration_thread,
6556 any_online_cpu(cpu_online_map)); 6635 cpumask_any(cpu_online_mask));
6557 kthread_stop(cpu_rq(cpu)->migration_thread); 6636 kthread_stop(cpu_rq(cpu)->migration_thread);
6558 cpu_rq(cpu)->migration_thread = NULL; 6637 cpu_rq(cpu)->migration_thread = NULL;
6559 break; 6638 break;
@@ -6603,7 +6682,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6603 rq = cpu_rq(cpu); 6682 rq = cpu_rq(cpu);
6604 spin_lock_irqsave(&rq->lock, flags); 6683 spin_lock_irqsave(&rq->lock, flags);
6605 if (rq->rd) { 6684 if (rq->rd) {
6606 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6685 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6607 set_rq_offline(rq); 6686 set_rq_offline(rq);
6608 } 6687 }
6609 spin_unlock_irqrestore(&rq->lock, flags); 6688 spin_unlock_irqrestore(&rq->lock, flags);
@@ -6642,13 +6721,13 @@ early_initcall(migration_init);
6642#ifdef CONFIG_SCHED_DEBUG 6721#ifdef CONFIG_SCHED_DEBUG
6643 6722
6644static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, 6723static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6645 cpumask_t *groupmask) 6724 struct cpumask *groupmask)
6646{ 6725{
6647 struct sched_group *group = sd->groups; 6726 struct sched_group *group = sd->groups;
6648 char str[256]; 6727 char str[256];
6649 6728
6650 cpulist_scnprintf(str, sizeof(str), sd->span); 6729 cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
6651 cpus_clear(*groupmask); 6730 cpumask_clear(groupmask);
6652 6731
6653 printk(KERN_DEBUG "%*s domain %d: ", level, "", level); 6732 printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
6654 6733
@@ -6662,11 +6741,11 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6662 6741
6663 printk(KERN_CONT "span %s level %s\n", str, sd->name); 6742 printk(KERN_CONT "span %s level %s\n", str, sd->name);
6664 6743
6665 if (!cpu_isset(cpu, sd->span)) { 6744 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
6666 printk(KERN_ERR "ERROR: domain->span does not contain " 6745 printk(KERN_ERR "ERROR: domain->span does not contain "
6667 "CPU%d\n", cpu); 6746 "CPU%d\n", cpu);
6668 } 6747 }
6669 if (!cpu_isset(cpu, group->cpumask)) { 6748 if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
6670 printk(KERN_ERR "ERROR: domain->groups does not contain" 6749 printk(KERN_ERR "ERROR: domain->groups does not contain"
6671 " CPU%d\n", cpu); 6750 " CPU%d\n", cpu);
6672 } 6751 }
@@ -6686,31 +6765,32 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6686 break; 6765 break;
6687 } 6766 }
6688 6767
6689 if (!cpus_weight(group->cpumask)) { 6768 if (!cpumask_weight(sched_group_cpus(group))) {
6690 printk(KERN_CONT "\n"); 6769 printk(KERN_CONT "\n");
6691 printk(KERN_ERR "ERROR: empty group\n"); 6770 printk(KERN_ERR "ERROR: empty group\n");
6692 break; 6771 break;
6693 } 6772 }
6694 6773
6695 if (cpus_intersects(*groupmask, group->cpumask)) { 6774 if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
6696 printk(KERN_CONT "\n"); 6775 printk(KERN_CONT "\n");
6697 printk(KERN_ERR "ERROR: repeated CPUs\n"); 6776 printk(KERN_ERR "ERROR: repeated CPUs\n");
6698 break; 6777 break;
6699 } 6778 }
6700 6779
6701 cpus_or(*groupmask, *groupmask, group->cpumask); 6780 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
6702 6781
6703 cpulist_scnprintf(str, sizeof(str), group->cpumask); 6782 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
6704 printk(KERN_CONT " %s", str); 6783 printk(KERN_CONT " %s", str);
6705 6784
6706 group = group->next; 6785 group = group->next;
6707 } while (group != sd->groups); 6786 } while (group != sd->groups);
6708 printk(KERN_CONT "\n"); 6787 printk(KERN_CONT "\n");
6709 6788
6710 if (!cpus_equal(sd->span, *groupmask)) 6789 if (!cpumask_equal(sched_domain_span(sd), groupmask))
6711 printk(KERN_ERR "ERROR: groups don't span domain->span\n"); 6790 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
6712 6791
6713 if (sd->parent && !cpus_subset(*groupmask, sd->parent->span)) 6792 if (sd->parent &&
6793 !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
6714 printk(KERN_ERR "ERROR: parent span is not a superset " 6794 printk(KERN_ERR "ERROR: parent span is not a superset "
6715 "of domain->span\n"); 6795 "of domain->span\n");
6716 return 0; 6796 return 0;
@@ -6718,7 +6798,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6718 6798
6719static void sched_domain_debug(struct sched_domain *sd, int cpu) 6799static void sched_domain_debug(struct sched_domain *sd, int cpu)
6720{ 6800{
6721 cpumask_t *groupmask; 6801 cpumask_var_t groupmask;
6722 int level = 0; 6802 int level = 0;
6723 6803
6724 if (!sd) { 6804 if (!sd) {
@@ -6728,8 +6808,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6728 6808
6729 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); 6809 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
6730 6810
6731 groupmask = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 6811 if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
6732 if (!groupmask) {
6733 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n"); 6812 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
6734 return; 6813 return;
6735 } 6814 }
@@ -6742,7 +6821,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6742 if (!sd) 6821 if (!sd)
6743 break; 6822 break;
6744 } 6823 }
6745 kfree(groupmask); 6824 free_cpumask_var(groupmask);
6746} 6825}
6747#else /* !CONFIG_SCHED_DEBUG */ 6826#else /* !CONFIG_SCHED_DEBUG */
6748# define sched_domain_debug(sd, cpu) do { } while (0) 6827# define sched_domain_debug(sd, cpu) do { } while (0)
@@ -6750,7 +6829,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6750 6829
6751static int sd_degenerate(struct sched_domain *sd) 6830static int sd_degenerate(struct sched_domain *sd)
6752{ 6831{
6753 if (cpus_weight(sd->span) == 1) 6832 if (cpumask_weight(sched_domain_span(sd)) == 1)
6754 return 1; 6833 return 1;
6755 6834
6756 /* Following flags need at least 2 groups */ 6835 /* Following flags need at least 2 groups */
@@ -6781,7 +6860,7 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6781 if (sd_degenerate(parent)) 6860 if (sd_degenerate(parent))
6782 return 1; 6861 return 1;
6783 6862
6784 if (!cpus_equal(sd->span, parent->span)) 6863 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
6785 return 0; 6864 return 0;
6786 6865
6787 /* Does parent contain flags not in child? */ 6866 /* Does parent contain flags not in child? */
@@ -6805,6 +6884,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6805 return 1; 6884 return 1;
6806} 6885}
6807 6886
6887static void free_rootdomain(struct root_domain *rd)
6888{
6889 cpupri_cleanup(&rd->cpupri);
6890
6891 free_cpumask_var(rd->rto_mask);
6892 free_cpumask_var(rd->online);
6893 free_cpumask_var(rd->span);
6894 kfree(rd);
6895}
6896
6808static void rq_attach_root(struct rq *rq, struct root_domain *rd) 6897static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6809{ 6898{
6810 unsigned long flags; 6899 unsigned long flags;
@@ -6814,38 +6903,63 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6814 if (rq->rd) { 6903 if (rq->rd) {
6815 struct root_domain *old_rd = rq->rd; 6904 struct root_domain *old_rd = rq->rd;
6816 6905
6817 if (cpu_isset(rq->cpu, old_rd->online)) 6906 if (cpumask_test_cpu(rq->cpu, old_rd->online))
6818 set_rq_offline(rq); 6907 set_rq_offline(rq);
6819 6908
6820 cpu_clear(rq->cpu, old_rd->span); 6909 cpumask_clear_cpu(rq->cpu, old_rd->span);
6821 6910
6822 if (atomic_dec_and_test(&old_rd->refcount)) 6911 if (atomic_dec_and_test(&old_rd->refcount))
6823 kfree(old_rd); 6912 free_rootdomain(old_rd);
6824 } 6913 }
6825 6914
6826 atomic_inc(&rd->refcount); 6915 atomic_inc(&rd->refcount);
6827 rq->rd = rd; 6916 rq->rd = rd;
6828 6917
6829 cpu_set(rq->cpu, rd->span); 6918 cpumask_set_cpu(rq->cpu, rd->span);
6830 if (cpu_isset(rq->cpu, cpu_online_map)) 6919 if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
6831 set_rq_online(rq); 6920 set_rq_online(rq);
6832 6921
6833 spin_unlock_irqrestore(&rq->lock, flags); 6922 spin_unlock_irqrestore(&rq->lock, flags);
6834} 6923}
6835 6924
6836static void init_rootdomain(struct root_domain *rd) 6925static int init_rootdomain(struct root_domain *rd, bool bootmem)
6837{ 6926{
6838 memset(rd, 0, sizeof(*rd)); 6927 memset(rd, 0, sizeof(*rd));
6839 6928
6840 cpus_clear(rd->span); 6929 if (bootmem) {
6841 cpus_clear(rd->online); 6930 alloc_bootmem_cpumask_var(&def_root_domain.span);
6931 alloc_bootmem_cpumask_var(&def_root_domain.online);
6932 alloc_bootmem_cpumask_var(&def_root_domain.rto_mask);
6933 cpupri_init(&rd->cpupri, true);
6934 return 0;
6935 }
6936
6937 if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
6938 goto free_rd;
6939 if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
6940 goto free_span;
6941 if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
6942 goto free_online;
6943
6944 if (cpupri_init(&rd->cpupri, false) != 0)
6945 goto free_rto_mask;
6946 return 0;
6842 6947
6843 cpupri_init(&rd->cpupri); 6948free_rto_mask:
6949 free_cpumask_var(rd->rto_mask);
6950free_online:
6951 free_cpumask_var(rd->online);
6952free_span:
6953 free_cpumask_var(rd->span);
6954free_rd:
6955 kfree(rd);
6956 return -ENOMEM;
6844} 6957}
6845 6958
6846static void init_defrootdomain(void) 6959static void init_defrootdomain(void)
6847{ 6960{
6848 init_rootdomain(&def_root_domain); 6961 init_rootdomain(&def_root_domain, true);
6962
6849 atomic_set(&def_root_domain.refcount, 1); 6963 atomic_set(&def_root_domain.refcount, 1);
6850} 6964}
6851 6965
@@ -6857,7 +6971,10 @@ static struct root_domain *alloc_rootdomain(void)
6857 if (!rd) 6971 if (!rd)
6858 return NULL; 6972 return NULL;
6859 6973
6860 init_rootdomain(rd); 6974 if (init_rootdomain(rd, false) != 0) {
6975 kfree(rd);
6976 return NULL;
6977 }
6861 6978
6862 return rd; 6979 return rd;
6863} 6980}
@@ -6899,19 +7016,12 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
6899} 7016}
6900 7017
6901/* cpus with isolated domains */ 7018/* cpus with isolated domains */
6902static cpumask_t cpu_isolated_map = CPU_MASK_NONE; 7019static cpumask_var_t cpu_isolated_map;
6903 7020
6904/* Setup the mask of cpus configured for isolated domains */ 7021/* Setup the mask of cpus configured for isolated domains */
6905static int __init isolated_cpu_setup(char *str) 7022static int __init isolated_cpu_setup(char *str)
6906{ 7023{
6907 static int __initdata ints[NR_CPUS]; 7024 cpulist_parse(str, cpu_isolated_map);
6908 int i;
6909
6910 str = get_options(str, ARRAY_SIZE(ints), ints);
6911 cpus_clear(cpu_isolated_map);
6912 for (i = 1; i <= ints[0]; i++)
6913 if (ints[i] < NR_CPUS)
6914 cpu_set(ints[i], cpu_isolated_map);
6915 return 1; 7025 return 1;
6916} 7026}
6917 7027
@@ -6920,42 +7030,43 @@ __setup("isolcpus=", isolated_cpu_setup);
6920/* 7030/*
6921 * init_sched_build_groups takes the cpumask we wish to span, and a pointer 7031 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
6922 * to a function which identifies what group(along with sched group) a CPU 7032 * to a function which identifies what group(along with sched group) a CPU
6923 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS 7033 * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
6924 * (due to the fact that we keep track of groups covered with a cpumask_t). 7034 * (due to the fact that we keep track of groups covered with a struct cpumask).
6925 * 7035 *
6926 * init_sched_build_groups will build a circular linked list of the groups 7036 * init_sched_build_groups will build a circular linked list of the groups
6927 * covered by the given span, and will set each group's ->cpumask correctly, 7037 * covered by the given span, and will set each group's ->cpumask correctly,
6928 * and ->cpu_power to 0. 7038 * and ->cpu_power to 0.
6929 */ 7039 */
6930static void 7040static void
6931init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map, 7041init_sched_build_groups(const struct cpumask *span,
6932 int (*group_fn)(int cpu, const cpumask_t *cpu_map, 7042 const struct cpumask *cpu_map,
7043 int (*group_fn)(int cpu, const struct cpumask *cpu_map,
6933 struct sched_group **sg, 7044 struct sched_group **sg,
6934 cpumask_t *tmpmask), 7045 struct cpumask *tmpmask),
6935 cpumask_t *covered, cpumask_t *tmpmask) 7046 struct cpumask *covered, struct cpumask *tmpmask)
6936{ 7047{
6937 struct sched_group *first = NULL, *last = NULL; 7048 struct sched_group *first = NULL, *last = NULL;
6938 int i; 7049 int i;
6939 7050
6940 cpus_clear(*covered); 7051 cpumask_clear(covered);
6941 7052
6942 for_each_cpu_mask_nr(i, *span) { 7053 for_each_cpu(i, span) {
6943 struct sched_group *sg; 7054 struct sched_group *sg;
6944 int group = group_fn(i, cpu_map, &sg, tmpmask); 7055 int group = group_fn(i, cpu_map, &sg, tmpmask);
6945 int j; 7056 int j;
6946 7057
6947 if (cpu_isset(i, *covered)) 7058 if (cpumask_test_cpu(i, covered))
6948 continue; 7059 continue;
6949 7060
6950 cpus_clear(sg->cpumask); 7061 cpumask_clear(sched_group_cpus(sg));
6951 sg->__cpu_power = 0; 7062 sg->__cpu_power = 0;
6952 7063
6953 for_each_cpu_mask_nr(j, *span) { 7064 for_each_cpu(j, span) {
6954 if (group_fn(j, cpu_map, NULL, tmpmask) != group) 7065 if (group_fn(j, cpu_map, NULL, tmpmask) != group)
6955 continue; 7066 continue;
6956 7067
6957 cpu_set(j, *covered); 7068 cpumask_set_cpu(j, covered);
6958 cpu_set(j, sg->cpumask); 7069 cpumask_set_cpu(j, sched_group_cpus(sg));
6959 } 7070 }
6960 if (!first) 7071 if (!first)
6961 first = sg; 7072 first = sg;
@@ -7019,9 +7130,10 @@ static int find_next_best_node(int node, nodemask_t *used_nodes)
7019 * should be one that prevents unnecessary balancing, but also spreads tasks 7130 * should be one that prevents unnecessary balancing, but also spreads tasks
7020 * out optimally. 7131 * out optimally.
7021 */ 7132 */
7022static void sched_domain_node_span(int node, cpumask_t *span) 7133static void sched_domain_node_span(int node, struct cpumask *span)
7023{ 7134{
7024 nodemask_t used_nodes; 7135 nodemask_t used_nodes;
7136 /* FIXME: use cpumask_of_node() */
7025 node_to_cpumask_ptr(nodemask, node); 7137 node_to_cpumask_ptr(nodemask, node);
7026 int i; 7138 int i;
7027 7139
@@ -7043,18 +7155,33 @@ static void sched_domain_node_span(int node, cpumask_t *span)
7043int sched_smt_power_savings = 0, sched_mc_power_savings = 0; 7155int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
7044 7156
7045/* 7157/*
7158 * The cpus mask in sched_group and sched_domain hangs off the end.
7159 * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space
7160 * for nr_cpu_ids < CONFIG_NR_CPUS.
7161 */
7162struct static_sched_group {
7163 struct sched_group sg;
7164 DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
7165};
7166
7167struct static_sched_domain {
7168 struct sched_domain sd;
7169 DECLARE_BITMAP(span, CONFIG_NR_CPUS);
7170};
7171
7172/*
7046 * SMT sched-domains: 7173 * SMT sched-domains:
7047 */ 7174 */
7048#ifdef CONFIG_SCHED_SMT 7175#ifdef CONFIG_SCHED_SMT
7049static DEFINE_PER_CPU(struct sched_domain, cpu_domains); 7176static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
7050static DEFINE_PER_CPU(struct sched_group, sched_group_cpus); 7177static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
7051 7178
7052static int 7179static int
7053cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7180cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
7054 cpumask_t *unused) 7181 struct sched_group **sg, struct cpumask *unused)
7055{ 7182{
7056 if (sg) 7183 if (sg)
7057 *sg = &per_cpu(sched_group_cpus, cpu); 7184 *sg = &per_cpu(sched_group_cpus, cpu).sg;
7058 return cpu; 7185 return cpu;
7059} 7186}
7060#endif /* CONFIG_SCHED_SMT */ 7187#endif /* CONFIG_SCHED_SMT */
@@ -7063,56 +7190,55 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
7063 * multi-core sched-domains: 7190 * multi-core sched-domains:
7064 */ 7191 */
7065#ifdef CONFIG_SCHED_MC 7192#ifdef CONFIG_SCHED_MC
7066static DEFINE_PER_CPU(struct sched_domain, core_domains); 7193static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
7067static DEFINE_PER_CPU(struct sched_group, sched_group_core); 7194static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
7068#endif /* CONFIG_SCHED_MC */ 7195#endif /* CONFIG_SCHED_MC */
7069 7196
7070#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) 7197#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
7071static int 7198static int
7072cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7199cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7073 cpumask_t *mask) 7200 struct sched_group **sg, struct cpumask *mask)
7074{ 7201{
7075 int group; 7202 int group;
7076 7203
7077 *mask = per_cpu(cpu_sibling_map, cpu); 7204 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7078 cpus_and(*mask, *mask, *cpu_map); 7205 group = cpumask_first(mask);
7079 group = first_cpu(*mask);
7080 if (sg) 7206 if (sg)
7081 *sg = &per_cpu(sched_group_core, group); 7207 *sg = &per_cpu(sched_group_core, group).sg;
7082 return group; 7208 return group;
7083} 7209}
7084#elif defined(CONFIG_SCHED_MC) 7210#elif defined(CONFIG_SCHED_MC)
7085static int 7211static int
7086cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7212cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7087 cpumask_t *unused) 7213 struct sched_group **sg, struct cpumask *unused)
7088{ 7214{
7089 if (sg) 7215 if (sg)
7090 *sg = &per_cpu(sched_group_core, cpu); 7216 *sg = &per_cpu(sched_group_core, cpu).sg;
7091 return cpu; 7217 return cpu;
7092} 7218}
7093#endif 7219#endif
7094 7220
7095static DEFINE_PER_CPU(struct sched_domain, phys_domains); 7221static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
7096static DEFINE_PER_CPU(struct sched_group, sched_group_phys); 7222static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
7097 7223
7098static int 7224static int
7099cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7225cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
7100 cpumask_t *mask) 7226 struct sched_group **sg, struct cpumask *mask)
7101{ 7227{
7102 int group; 7228 int group;
7103#ifdef CONFIG_SCHED_MC 7229#ifdef CONFIG_SCHED_MC
7230 /* FIXME: Use cpu_coregroup_mask. */
7104 *mask = cpu_coregroup_map(cpu); 7231 *mask = cpu_coregroup_map(cpu);
7105 cpus_and(*mask, *mask, *cpu_map); 7232 cpus_and(*mask, *mask, *cpu_map);
7106 group = first_cpu(*mask); 7233 group = cpumask_first(mask);
7107#elif defined(CONFIG_SCHED_SMT) 7234#elif defined(CONFIG_SCHED_SMT)
7108 *mask = per_cpu(cpu_sibling_map, cpu); 7235 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7109 cpus_and(*mask, *mask, *cpu_map); 7236 group = cpumask_first(mask);
7110 group = first_cpu(*mask);
7111#else 7237#else
7112 group = cpu; 7238 group = cpu;
7113#endif 7239#endif
7114 if (sg) 7240 if (sg)
7115 *sg = &per_cpu(sched_group_phys, group); 7241 *sg = &per_cpu(sched_group_phys, group).sg;
7116 return group; 7242 return group;
7117} 7243}
7118 7244
@@ -7126,19 +7252,21 @@ static DEFINE_PER_CPU(struct sched_domain, node_domains);
7126static struct sched_group ***sched_group_nodes_bycpu; 7252static struct sched_group ***sched_group_nodes_bycpu;
7127 7253
7128static DEFINE_PER_CPU(struct sched_domain, allnodes_domains); 7254static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
7129static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes); 7255static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
7130 7256
7131static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map, 7257static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
7132 struct sched_group **sg, cpumask_t *nodemask) 7258 struct sched_group **sg,
7259 struct cpumask *nodemask)
7133{ 7260{
7134 int group; 7261 int group;
7262 /* FIXME: use cpumask_of_node */
7263 node_to_cpumask_ptr(pnodemask, cpu_to_node(cpu));
7135 7264
7136 *nodemask = node_to_cpumask(cpu_to_node(cpu)); 7265 cpumask_and(nodemask, pnodemask, cpu_map);
7137 cpus_and(*nodemask, *nodemask, *cpu_map); 7266 group = cpumask_first(nodemask);
7138 group = first_cpu(*nodemask);
7139 7267
7140 if (sg) 7268 if (sg)
7141 *sg = &per_cpu(sched_group_allnodes, group); 7269 *sg = &per_cpu(sched_group_allnodes, group).sg;
7142 return group; 7270 return group;
7143} 7271}
7144 7272
@@ -7150,11 +7278,11 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7150 if (!sg) 7278 if (!sg)
7151 return; 7279 return;
7152 do { 7280 do {
7153 for_each_cpu_mask_nr(j, sg->cpumask) { 7281 for_each_cpu(j, sched_group_cpus(sg)) {
7154 struct sched_domain *sd; 7282 struct sched_domain *sd;
7155 7283
7156 sd = &per_cpu(phys_domains, j); 7284 sd = &per_cpu(phys_domains, j).sd;
7157 if (j != first_cpu(sd->groups->cpumask)) { 7285 if (j != cpumask_first(sched_group_cpus(sd->groups))) {
7158 /* 7286 /*
7159 * Only add "power" once for each 7287 * Only add "power" once for each
7160 * physical package. 7288 * physical package.
@@ -7171,11 +7299,12 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7171 7299
7172#ifdef CONFIG_NUMA 7300#ifdef CONFIG_NUMA
7173/* Free memory allocated for various sched_group structures */ 7301/* Free memory allocated for various sched_group structures */
7174static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7302static void free_sched_groups(const struct cpumask *cpu_map,
7303 struct cpumask *nodemask)
7175{ 7304{
7176 int cpu, i; 7305 int cpu, i;
7177 7306
7178 for_each_cpu_mask_nr(cpu, *cpu_map) { 7307 for_each_cpu(cpu, cpu_map) {
7179 struct sched_group **sched_group_nodes 7308 struct sched_group **sched_group_nodes
7180 = sched_group_nodes_bycpu[cpu]; 7309 = sched_group_nodes_bycpu[cpu];
7181 7310
@@ -7184,10 +7313,11 @@ static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
7184 7313
7185 for (i = 0; i < nr_node_ids; i++) { 7314 for (i = 0; i < nr_node_ids; i++) {
7186 struct sched_group *oldsg, *sg = sched_group_nodes[i]; 7315 struct sched_group *oldsg, *sg = sched_group_nodes[i];
7316 /* FIXME: Use cpumask_of_node */
7317 node_to_cpumask_ptr(pnodemask, i);
7187 7318
7188 *nodemask = node_to_cpumask(i); 7319 cpus_and(*nodemask, *pnodemask, *cpu_map);
7189 cpus_and(*nodemask, *nodemask, *cpu_map); 7320 if (cpumask_empty(nodemask))
7190 if (cpus_empty(*nodemask))
7191 continue; 7321 continue;
7192 7322
7193 if (sg == NULL) 7323 if (sg == NULL)
@@ -7205,7 +7335,8 @@ next_sg:
7205 } 7335 }
7206} 7336}
7207#else /* !CONFIG_NUMA */ 7337#else /* !CONFIG_NUMA */
7208static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7338static void free_sched_groups(const struct cpumask *cpu_map,
7339 struct cpumask *nodemask)
7209{ 7340{
7210} 7341}
7211#endif /* CONFIG_NUMA */ 7342#endif /* CONFIG_NUMA */
@@ -7231,7 +7362,7 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
7231 7362
7232 WARN_ON(!sd || !sd->groups); 7363 WARN_ON(!sd || !sd->groups);
7233 7364
7234 if (cpu != first_cpu(sd->groups->cpumask)) 7365 if (cpu != cpumask_first(sched_group_cpus(sd->groups)))
7235 return; 7366 return;
7236 7367
7237 child = sd->child; 7368 child = sd->child;
@@ -7296,48 +7427,6 @@ SD_INIT_FUNC(CPU)
7296 SD_INIT_FUNC(MC) 7427 SD_INIT_FUNC(MC)
7297#endif 7428#endif
7298 7429
7299/*
7300 * To minimize stack usage kmalloc room for cpumasks and share the
7301 * space as the usage in build_sched_domains() dictates. Used only
7302 * if the amount of space is significant.
7303 */
7304struct allmasks {
7305 cpumask_t tmpmask; /* make this one first */
7306 union {
7307 cpumask_t nodemask;
7308 cpumask_t this_sibling_map;
7309 cpumask_t this_core_map;
7310 };
7311 cpumask_t send_covered;
7312
7313#ifdef CONFIG_NUMA
7314 cpumask_t domainspan;
7315 cpumask_t covered;
7316 cpumask_t notcovered;
7317#endif
7318};
7319
7320#if NR_CPUS > 128
7321#define SCHED_CPUMASK_DECLARE(v) struct allmasks *v
7322static inline void sched_cpumask_alloc(struct allmasks **masks)
7323{
7324 *masks = kmalloc(sizeof(**masks), GFP_KERNEL);
7325}
7326static inline void sched_cpumask_free(struct allmasks *masks)
7327{
7328 kfree(masks);
7329}
7330#else
7331#define SCHED_CPUMASK_DECLARE(v) struct allmasks _v, *v = &_v
7332static inline void sched_cpumask_alloc(struct allmasks **masks)
7333{ }
7334static inline void sched_cpumask_free(struct allmasks *masks)
7335{ }
7336#endif
7337
7338#define SCHED_CPUMASK_VAR(v, a) cpumask_t *v = (cpumask_t *) \
7339 ((unsigned long)(a) + offsetof(struct allmasks, v))
7340
7341static int default_relax_domain_level = -1; 7430static int default_relax_domain_level = -1;
7342 7431
7343static int __init setup_relax_domain_level(char *str) 7432static int __init setup_relax_domain_level(char *str)
@@ -7377,17 +7466,38 @@ static void set_domain_attribute(struct sched_domain *sd,
7377 * Build sched domains for a given set of cpus and attach the sched domains 7466 * Build sched domains for a given set of cpus and attach the sched domains
7378 * to the individual cpus 7467 * to the individual cpus
7379 */ 7468 */
7380static int __build_sched_domains(const cpumask_t *cpu_map, 7469static int __build_sched_domains(const struct cpumask *cpu_map,
7381 struct sched_domain_attr *attr) 7470 struct sched_domain_attr *attr)
7382{ 7471{
7383 int i; 7472 int i, err = -ENOMEM;
7384 struct root_domain *rd; 7473 struct root_domain *rd;
7385 SCHED_CPUMASK_DECLARE(allmasks); 7474 cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
7386 cpumask_t *tmpmask; 7475 tmpmask;
7387#ifdef CONFIG_NUMA 7476#ifdef CONFIG_NUMA
7477 cpumask_var_t domainspan, covered, notcovered;
7388 struct sched_group **sched_group_nodes = NULL; 7478 struct sched_group **sched_group_nodes = NULL;
7389 int sd_allnodes = 0; 7479 int sd_allnodes = 0;
7390 7480
7481 if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
7482 goto out;
7483 if (!alloc_cpumask_var(&covered, GFP_KERNEL))
7484 goto free_domainspan;
7485 if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
7486 goto free_covered;
7487#endif
7488
7489 if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
7490 goto free_notcovered;
7491 if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
7492 goto free_nodemask;
7493 if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
7494 goto free_this_sibling_map;
7495 if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
7496 goto free_this_core_map;
7497 if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
7498 goto free_send_covered;
7499
7500#ifdef CONFIG_NUMA
7391 /* 7501 /*
7392 * Allocate the per-node list of sched groups 7502 * Allocate the per-node list of sched groups
7393 */ 7503 */
@@ -7395,54 +7505,37 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7395 GFP_KERNEL); 7505 GFP_KERNEL);
7396 if (!sched_group_nodes) { 7506 if (!sched_group_nodes) {
7397 printk(KERN_WARNING "Can not alloc sched group node list\n"); 7507 printk(KERN_WARNING "Can not alloc sched group node list\n");
7398 return -ENOMEM; 7508 goto free_tmpmask;
7399 } 7509 }
7400#endif 7510#endif
7401 7511
7402 rd = alloc_rootdomain(); 7512 rd = alloc_rootdomain();
7403 if (!rd) { 7513 if (!rd) {
7404 printk(KERN_WARNING "Cannot alloc root domain\n"); 7514 printk(KERN_WARNING "Cannot alloc root domain\n");
7405#ifdef CONFIG_NUMA 7515 goto free_sched_groups;
7406 kfree(sched_group_nodes);
7407#endif
7408 return -ENOMEM;
7409 } 7516 }
7410 7517
7411 /* get space for all scratch cpumask variables */
7412 sched_cpumask_alloc(&allmasks);
7413 if (!allmasks) {
7414 printk(KERN_WARNING "Cannot alloc cpumask array\n");
7415 kfree(rd);
7416#ifdef CONFIG_NUMA 7518#ifdef CONFIG_NUMA
7417 kfree(sched_group_nodes); 7519 sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
7418#endif
7419 return -ENOMEM;
7420 }
7421
7422 tmpmask = (cpumask_t *)allmasks;
7423
7424
7425#ifdef CONFIG_NUMA
7426 sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes;
7427#endif 7520#endif
7428 7521
7429 /* 7522 /*
7430 * Set up domains for cpus specified by the cpu_map. 7523 * Set up domains for cpus specified by the cpu_map.
7431 */ 7524 */
7432 for_each_cpu_mask_nr(i, *cpu_map) { 7525 for_each_cpu(i, cpu_map) {
7433 struct sched_domain *sd = NULL, *p; 7526 struct sched_domain *sd = NULL, *p;
7434 SCHED_CPUMASK_VAR(nodemask, allmasks);
7435 7527
7528 /* FIXME: use cpumask_of_node */
7436 *nodemask = node_to_cpumask(cpu_to_node(i)); 7529 *nodemask = node_to_cpumask(cpu_to_node(i));
7437 cpus_and(*nodemask, *nodemask, *cpu_map); 7530 cpus_and(*nodemask, *nodemask, *cpu_map);
7438 7531
7439#ifdef CONFIG_NUMA 7532#ifdef CONFIG_NUMA
7440 if (cpus_weight(*cpu_map) > 7533 if (cpumask_weight(cpu_map) >
7441 SD_NODES_PER_DOMAIN*cpus_weight(*nodemask)) { 7534 SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
7442 sd = &per_cpu(allnodes_domains, i); 7535 sd = &per_cpu(allnodes_domains, i);
7443 SD_INIT(sd, ALLNODES); 7536 SD_INIT(sd, ALLNODES);
7444 set_domain_attribute(sd, attr); 7537 set_domain_attribute(sd, attr);
7445 sd->span = *cpu_map; 7538 cpumask_copy(sched_domain_span(sd), cpu_map);
7446 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask); 7539 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
7447 p = sd; 7540 p = sd;
7448 sd_allnodes = 1; 7541 sd_allnodes = 1;
@@ -7452,18 +7545,19 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7452 sd = &per_cpu(node_domains, i); 7545 sd = &per_cpu(node_domains, i);
7453 SD_INIT(sd, NODE); 7546 SD_INIT(sd, NODE);
7454 set_domain_attribute(sd, attr); 7547 set_domain_attribute(sd, attr);
7455 sched_domain_node_span(cpu_to_node(i), &sd->span); 7548 sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
7456 sd->parent = p; 7549 sd->parent = p;
7457 if (p) 7550 if (p)
7458 p->child = sd; 7551 p->child = sd;
7459 cpus_and(sd->span, sd->span, *cpu_map); 7552 cpumask_and(sched_domain_span(sd),
7553 sched_domain_span(sd), cpu_map);
7460#endif 7554#endif
7461 7555
7462 p = sd; 7556 p = sd;
7463 sd = &per_cpu(phys_domains, i); 7557 sd = &per_cpu(phys_domains, i).sd;
7464 SD_INIT(sd, CPU); 7558 SD_INIT(sd, CPU);
7465 set_domain_attribute(sd, attr); 7559 set_domain_attribute(sd, attr);
7466 sd->span = *nodemask; 7560 cpumask_copy(sched_domain_span(sd), nodemask);
7467 sd->parent = p; 7561 sd->parent = p;
7468 if (p) 7562 if (p)
7469 p->child = sd; 7563 p->child = sd;
@@ -7471,11 +7565,12 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7471 7565
7472#ifdef CONFIG_SCHED_MC 7566#ifdef CONFIG_SCHED_MC
7473 p = sd; 7567 p = sd;
7474 sd = &per_cpu(core_domains, i); 7568 sd = &per_cpu(core_domains, i).sd;
7475 SD_INIT(sd, MC); 7569 SD_INIT(sd, MC);
7476 set_domain_attribute(sd, attr); 7570 set_domain_attribute(sd, attr);
7477 sd->span = cpu_coregroup_map(i); 7571 *sched_domain_span(sd) = cpu_coregroup_map(i);
7478 cpus_and(sd->span, sd->span, *cpu_map); 7572 cpumask_and(sched_domain_span(sd),
7573 sched_domain_span(sd), cpu_map);
7479 sd->parent = p; 7574 sd->parent = p;
7480 p->child = sd; 7575 p->child = sd;
7481 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask); 7576 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7483,11 +7578,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7483 7578
7484#ifdef CONFIG_SCHED_SMT 7579#ifdef CONFIG_SCHED_SMT
7485 p = sd; 7580 p = sd;
7486 sd = &per_cpu(cpu_domains, i); 7581 sd = &per_cpu(cpu_domains, i).sd;
7487 SD_INIT(sd, SIBLING); 7582 SD_INIT(sd, SIBLING);
7488 set_domain_attribute(sd, attr); 7583 set_domain_attribute(sd, attr);
7489 sd->span = per_cpu(cpu_sibling_map, i); 7584 cpumask_and(sched_domain_span(sd),
7490 cpus_and(sd->span, sd->span, *cpu_map); 7585 &per_cpu(cpu_sibling_map, i), cpu_map);
7491 sd->parent = p; 7586 sd->parent = p;
7492 p->child = sd; 7587 p->child = sd;
7493 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask); 7588 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7496,13 +7591,10 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7496 7591
7497#ifdef CONFIG_SCHED_SMT 7592#ifdef CONFIG_SCHED_SMT
7498 /* Set up CPU (sibling) groups */ 7593 /* Set up CPU (sibling) groups */
7499 for_each_cpu_mask_nr(i, *cpu_map) { 7594 for_each_cpu(i, cpu_map) {
7500 SCHED_CPUMASK_VAR(this_sibling_map, allmasks); 7595 cpumask_and(this_sibling_map,
7501 SCHED_CPUMASK_VAR(send_covered, allmasks); 7596 &per_cpu(cpu_sibling_map, i), cpu_map);
7502 7597 if (i != cpumask_first(this_sibling_map))
7503 *this_sibling_map = per_cpu(cpu_sibling_map, i);
7504 cpus_and(*this_sibling_map, *this_sibling_map, *cpu_map);
7505 if (i != first_cpu(*this_sibling_map))
7506 continue; 7598 continue;
7507 7599
7508 init_sched_build_groups(this_sibling_map, cpu_map, 7600 init_sched_build_groups(this_sibling_map, cpu_map,
@@ -7513,13 +7605,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7513 7605
7514#ifdef CONFIG_SCHED_MC 7606#ifdef CONFIG_SCHED_MC
7515 /* Set up multi-core groups */ 7607 /* Set up multi-core groups */
7516 for_each_cpu_mask_nr(i, *cpu_map) { 7608 for_each_cpu(i, cpu_map) {
7517 SCHED_CPUMASK_VAR(this_core_map, allmasks); 7609 /* FIXME: Use cpu_coregroup_mask */
7518 SCHED_CPUMASK_VAR(send_covered, allmasks);
7519
7520 *this_core_map = cpu_coregroup_map(i); 7610 *this_core_map = cpu_coregroup_map(i);
7521 cpus_and(*this_core_map, *this_core_map, *cpu_map); 7611 cpus_and(*this_core_map, *this_core_map, *cpu_map);
7522 if (i != first_cpu(*this_core_map)) 7612 if (i != cpumask_first(this_core_map))
7523 continue; 7613 continue;
7524 7614
7525 init_sched_build_groups(this_core_map, cpu_map, 7615 init_sched_build_groups(this_core_map, cpu_map,
@@ -7530,12 +7620,10 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7530 7620
7531 /* Set up physical groups */ 7621 /* Set up physical groups */
7532 for (i = 0; i < nr_node_ids; i++) { 7622 for (i = 0; i < nr_node_ids; i++) {
7533 SCHED_CPUMASK_VAR(nodemask, allmasks); 7623 /* FIXME: Use cpumask_of_node */
7534 SCHED_CPUMASK_VAR(send_covered, allmasks);
7535
7536 *nodemask = node_to_cpumask(i); 7624 *nodemask = node_to_cpumask(i);
7537 cpus_and(*nodemask, *nodemask, *cpu_map); 7625 cpus_and(*nodemask, *nodemask, *cpu_map);
7538 if (cpus_empty(*nodemask)) 7626 if (cpumask_empty(nodemask))
7539 continue; 7627 continue;
7540 7628
7541 init_sched_build_groups(nodemask, cpu_map, 7629 init_sched_build_groups(nodemask, cpu_map,
@@ -7546,8 +7634,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7546#ifdef CONFIG_NUMA 7634#ifdef CONFIG_NUMA
7547 /* Set up node groups */ 7635 /* Set up node groups */
7548 if (sd_allnodes) { 7636 if (sd_allnodes) {
7549 SCHED_CPUMASK_VAR(send_covered, allmasks);
7550
7551 init_sched_build_groups(cpu_map, cpu_map, 7637 init_sched_build_groups(cpu_map, cpu_map,
7552 &cpu_to_allnodes_group, 7638 &cpu_to_allnodes_group,
7553 send_covered, tmpmask); 7639 send_covered, tmpmask);
@@ -7556,58 +7642,58 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7556 for (i = 0; i < nr_node_ids; i++) { 7642 for (i = 0; i < nr_node_ids; i++) {
7557 /* Set up node groups */ 7643 /* Set up node groups */
7558 struct sched_group *sg, *prev; 7644 struct sched_group *sg, *prev;
7559 SCHED_CPUMASK_VAR(nodemask, allmasks);
7560 SCHED_CPUMASK_VAR(domainspan, allmasks);
7561 SCHED_CPUMASK_VAR(covered, allmasks);
7562 int j; 7645 int j;
7563 7646
7647 /* FIXME: Use cpumask_of_node */
7564 *nodemask = node_to_cpumask(i); 7648 *nodemask = node_to_cpumask(i);
7565 cpus_clear(*covered); 7649 cpumask_clear(covered);
7566 7650
7567 cpus_and(*nodemask, *nodemask, *cpu_map); 7651 cpus_and(*nodemask, *nodemask, *cpu_map);
7568 if (cpus_empty(*nodemask)) { 7652 if (cpumask_empty(nodemask)) {
7569 sched_group_nodes[i] = NULL; 7653 sched_group_nodes[i] = NULL;
7570 continue; 7654 continue;
7571 } 7655 }
7572 7656
7573 sched_domain_node_span(i, domainspan); 7657 sched_domain_node_span(i, domainspan);
7574 cpus_and(*domainspan, *domainspan, *cpu_map); 7658 cpumask_and(domainspan, domainspan, cpu_map);
7575 7659
7576 sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i); 7660 sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
7661 GFP_KERNEL, i);
7577 if (!sg) { 7662 if (!sg) {
7578 printk(KERN_WARNING "Can not alloc domain group for " 7663 printk(KERN_WARNING "Can not alloc domain group for "
7579 "node %d\n", i); 7664 "node %d\n", i);
7580 goto error; 7665 goto error;
7581 } 7666 }
7582 sched_group_nodes[i] = sg; 7667 sched_group_nodes[i] = sg;
7583 for_each_cpu_mask_nr(j, *nodemask) { 7668 for_each_cpu(j, nodemask) {
7584 struct sched_domain *sd; 7669 struct sched_domain *sd;
7585 7670
7586 sd = &per_cpu(node_domains, j); 7671 sd = &per_cpu(node_domains, j);
7587 sd->groups = sg; 7672 sd->groups = sg;
7588 } 7673 }
7589 sg->__cpu_power = 0; 7674 sg->__cpu_power = 0;
7590 sg->cpumask = *nodemask; 7675 cpumask_copy(sched_group_cpus(sg), nodemask);
7591 sg->next = sg; 7676 sg->next = sg;
7592 cpus_or(*covered, *covered, *nodemask); 7677 cpumask_or(covered, covered, nodemask);
7593 prev = sg; 7678 prev = sg;
7594 7679
7595 for (j = 0; j < nr_node_ids; j++) { 7680 for (j = 0; j < nr_node_ids; j++) {
7596 SCHED_CPUMASK_VAR(notcovered, allmasks);
7597 int n = (i + j) % nr_node_ids; 7681 int n = (i + j) % nr_node_ids;
7682 /* FIXME: Use cpumask_of_node */
7598 node_to_cpumask_ptr(pnodemask, n); 7683 node_to_cpumask_ptr(pnodemask, n);
7599 7684
7600 cpus_complement(*notcovered, *covered); 7685 cpumask_complement(notcovered, covered);
7601 cpus_and(*tmpmask, *notcovered, *cpu_map); 7686 cpumask_and(tmpmask, notcovered, cpu_map);
7602 cpus_and(*tmpmask, *tmpmask, *domainspan); 7687 cpumask_and(tmpmask, tmpmask, domainspan);
7603 if (cpus_empty(*tmpmask)) 7688 if (cpumask_empty(tmpmask))
7604 break; 7689 break;
7605 7690
7606 cpus_and(*tmpmask, *tmpmask, *pnodemask); 7691 cpumask_and(tmpmask, tmpmask, pnodemask);
7607 if (cpus_empty(*tmpmask)) 7692 if (cpumask_empty(tmpmask))
7608 continue; 7693 continue;
7609 7694
7610 sg = kmalloc_node(sizeof(struct sched_group), 7695 sg = kmalloc_node(sizeof(struct sched_group) +
7696 cpumask_size(),
7611 GFP_KERNEL, i); 7697 GFP_KERNEL, i);
7612 if (!sg) { 7698 if (!sg) {
7613 printk(KERN_WARNING 7699 printk(KERN_WARNING
@@ -7615,9 +7701,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7615 goto error; 7701 goto error;
7616 } 7702 }
7617 sg->__cpu_power = 0; 7703 sg->__cpu_power = 0;
7618 sg->cpumask = *tmpmask; 7704 cpumask_copy(sched_group_cpus(sg), tmpmask);
7619 sg->next = prev->next; 7705 sg->next = prev->next;
7620 cpus_or(*covered, *covered, *tmpmask); 7706 cpumask_or(covered, covered, tmpmask);
7621 prev->next = sg; 7707 prev->next = sg;
7622 prev = sg; 7708 prev = sg;
7623 } 7709 }
@@ -7626,22 +7712,22 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7626 7712
7627 /* Calculate CPU power for physical packages and nodes */ 7713 /* Calculate CPU power for physical packages and nodes */
7628#ifdef CONFIG_SCHED_SMT 7714#ifdef CONFIG_SCHED_SMT
7629 for_each_cpu_mask_nr(i, *cpu_map) { 7715 for_each_cpu(i, cpu_map) {
7630 struct sched_domain *sd = &per_cpu(cpu_domains, i); 7716 struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
7631 7717
7632 init_sched_groups_power(i, sd); 7718 init_sched_groups_power(i, sd);
7633 } 7719 }
7634#endif 7720#endif
7635#ifdef CONFIG_SCHED_MC 7721#ifdef CONFIG_SCHED_MC
7636 for_each_cpu_mask_nr(i, *cpu_map) { 7722 for_each_cpu(i, cpu_map) {
7637 struct sched_domain *sd = &per_cpu(core_domains, i); 7723 struct sched_domain *sd = &per_cpu(core_domains, i).sd;
7638 7724
7639 init_sched_groups_power(i, sd); 7725 init_sched_groups_power(i, sd);
7640 } 7726 }
7641#endif 7727#endif
7642 7728
7643 for_each_cpu_mask_nr(i, *cpu_map) { 7729 for_each_cpu(i, cpu_map) {
7644 struct sched_domain *sd = &per_cpu(phys_domains, i); 7730 struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
7645 7731
7646 init_sched_groups_power(i, sd); 7732 init_sched_groups_power(i, sd);
7647 } 7733 }
@@ -7653,53 +7739,78 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7653 if (sd_allnodes) { 7739 if (sd_allnodes) {
7654 struct sched_group *sg; 7740 struct sched_group *sg;
7655 7741
7656 cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg, 7742 cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
7657 tmpmask); 7743 tmpmask);
7658 init_numa_sched_groups_power(sg); 7744 init_numa_sched_groups_power(sg);
7659 } 7745 }
7660#endif 7746#endif
7661 7747
7662 /* Attach the domains */ 7748 /* Attach the domains */
7663 for_each_cpu_mask_nr(i, *cpu_map) { 7749 for_each_cpu(i, cpu_map) {
7664 struct sched_domain *sd; 7750 struct sched_domain *sd;
7665#ifdef CONFIG_SCHED_SMT 7751#ifdef CONFIG_SCHED_SMT
7666 sd = &per_cpu(cpu_domains, i); 7752 sd = &per_cpu(cpu_domains, i).sd;
7667#elif defined(CONFIG_SCHED_MC) 7753#elif defined(CONFIG_SCHED_MC)
7668 sd = &per_cpu(core_domains, i); 7754 sd = &per_cpu(core_domains, i).sd;
7669#else 7755#else
7670 sd = &per_cpu(phys_domains, i); 7756 sd = &per_cpu(phys_domains, i).sd;
7671#endif 7757#endif
7672 cpu_attach_domain(sd, rd, i); 7758 cpu_attach_domain(sd, rd, i);
7673 } 7759 }
7674 7760
7675 sched_cpumask_free(allmasks); 7761 err = 0;
7676 return 0; 7762
7763free_tmpmask:
7764 free_cpumask_var(tmpmask);
7765free_send_covered:
7766 free_cpumask_var(send_covered);
7767free_this_core_map:
7768 free_cpumask_var(this_core_map);
7769free_this_sibling_map:
7770 free_cpumask_var(this_sibling_map);
7771free_nodemask:
7772 free_cpumask_var(nodemask);
7773free_notcovered:
7774#ifdef CONFIG_NUMA
7775 free_cpumask_var(notcovered);
7776free_covered:
7777 free_cpumask_var(covered);
7778free_domainspan:
7779 free_cpumask_var(domainspan);
7780out:
7781#endif
7782 return err;
7783
7784free_sched_groups:
7785#ifdef CONFIG_NUMA
7786 kfree(sched_group_nodes);
7787#endif
7788 goto free_tmpmask;
7677 7789
7678#ifdef CONFIG_NUMA 7790#ifdef CONFIG_NUMA
7679error: 7791error:
7680 free_sched_groups(cpu_map, tmpmask); 7792 free_sched_groups(cpu_map, tmpmask);
7681 sched_cpumask_free(allmasks); 7793 free_rootdomain(rd);
7682 kfree(rd); 7794 goto free_tmpmask;
7683 return -ENOMEM;
7684#endif 7795#endif
7685} 7796}
7686 7797
7687static int build_sched_domains(const cpumask_t *cpu_map) 7798static int build_sched_domains(const struct cpumask *cpu_map)
7688{ 7799{
7689 return __build_sched_domains(cpu_map, NULL); 7800 return __build_sched_domains(cpu_map, NULL);
7690} 7801}
7691 7802
7692static cpumask_t *doms_cur; /* current sched domains */ 7803static struct cpumask *doms_cur; /* current sched domains */
7693static int ndoms_cur; /* number of sched domains in 'doms_cur' */ 7804static int ndoms_cur; /* number of sched domains in 'doms_cur' */
7694static struct sched_domain_attr *dattr_cur; 7805static struct sched_domain_attr *dattr_cur;
7695 /* attribues of custom domains in 'doms_cur' */ 7806 /* attribues of custom domains in 'doms_cur' */
7696 7807
7697/* 7808/*
7698 * Special case: If a kmalloc of a doms_cur partition (array of 7809 * Special case: If a kmalloc of a doms_cur partition (array of
7699 * cpumask_t) fails, then fallback to a single sched domain, 7810 * cpumask) fails, then fallback to a single sched domain,
7700 * as determined by the single cpumask_t fallback_doms. 7811 * as determined by the single cpumask fallback_doms.
7701 */ 7812 */
7702static cpumask_t fallback_doms; 7813static cpumask_var_t fallback_doms;
7703 7814
7704/* 7815/*
7705 * arch_update_cpu_topology lets virtualized architectures update the 7816 * arch_update_cpu_topology lets virtualized architectures update the
@@ -7716,16 +7827,16 @@ int __attribute__((weak)) arch_update_cpu_topology(void)
7716 * For now this just excludes isolated cpus, but could be used to 7827 * For now this just excludes isolated cpus, but could be used to
7717 * exclude other special cases in the future. 7828 * exclude other special cases in the future.
7718 */ 7829 */
7719static int arch_init_sched_domains(const cpumask_t *cpu_map) 7830static int arch_init_sched_domains(const struct cpumask *cpu_map)
7720{ 7831{
7721 int err; 7832 int err;
7722 7833
7723 arch_update_cpu_topology(); 7834 arch_update_cpu_topology();
7724 ndoms_cur = 1; 7835 ndoms_cur = 1;
7725 doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 7836 doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
7726 if (!doms_cur) 7837 if (!doms_cur)
7727 doms_cur = &fallback_doms; 7838 doms_cur = fallback_doms;
7728 cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map); 7839 cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
7729 dattr_cur = NULL; 7840 dattr_cur = NULL;
7730 err = build_sched_domains(doms_cur); 7841 err = build_sched_domains(doms_cur);
7731 register_sched_domain_sysctl(); 7842 register_sched_domain_sysctl();
@@ -7733,8 +7844,8 @@ static int arch_init_sched_domains(const cpumask_t *cpu_map)
7733 return err; 7844 return err;
7734} 7845}
7735 7846
7736static void arch_destroy_sched_domains(const cpumask_t *cpu_map, 7847static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
7737 cpumask_t *tmpmask) 7848 struct cpumask *tmpmask)
7738{ 7849{
7739 free_sched_groups(cpu_map, tmpmask); 7850 free_sched_groups(cpu_map, tmpmask);
7740} 7851}
@@ -7743,15 +7854,16 @@ static void arch_destroy_sched_domains(const cpumask_t *cpu_map,
7743 * Detach sched domains from a group of cpus specified in cpu_map 7854 * Detach sched domains from a group of cpus specified in cpu_map
7744 * These cpus will now be attached to the NULL domain 7855 * These cpus will now be attached to the NULL domain
7745 */ 7856 */
7746static void detach_destroy_domains(const cpumask_t *cpu_map) 7857static void detach_destroy_domains(const struct cpumask *cpu_map)
7747{ 7858{
7748 cpumask_t tmpmask; 7859 /* Save because hotplug lock held. */
7860 static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
7749 int i; 7861 int i;
7750 7862
7751 for_each_cpu_mask_nr(i, *cpu_map) 7863 for_each_cpu(i, cpu_map)
7752 cpu_attach_domain(NULL, &def_root_domain, i); 7864 cpu_attach_domain(NULL, &def_root_domain, i);
7753 synchronize_sched(); 7865 synchronize_sched();
7754 arch_destroy_sched_domains(cpu_map, &tmpmask); 7866 arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
7755} 7867}
7756 7868
7757/* handle null as "default" */ 7869/* handle null as "default" */
@@ -7776,7 +7888,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7776 * doms_new[] to the current sched domain partitioning, doms_cur[]. 7888 * doms_new[] to the current sched domain partitioning, doms_cur[].
7777 * It destroys each deleted domain and builds each new domain. 7889 * It destroys each deleted domain and builds each new domain.
7778 * 7890 *
7779 * 'doms_new' is an array of cpumask_t's of length 'ndoms_new'. 7891 * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
7780 * The masks don't intersect (don't overlap.) We should setup one 7892 * The masks don't intersect (don't overlap.) We should setup one
7781 * sched domain for each mask. CPUs not in any of the cpumasks will 7893 * sched domain for each mask. CPUs not in any of the cpumasks will
7782 * not be load balanced. If the same cpumask appears both in the 7894 * not be load balanced. If the same cpumask appears both in the
@@ -7790,13 +7902,14 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7790 * the single partition 'fallback_doms', it also forces the domains 7902 * the single partition 'fallback_doms', it also forces the domains
7791 * to be rebuilt. 7903 * to be rebuilt.
7792 * 7904 *
7793 * If doms_new == NULL it will be replaced with cpu_online_map. 7905 * If doms_new == NULL it will be replaced with cpu_online_mask.
7794 * ndoms_new == 0 is a special case for destroying existing domains, 7906 * ndoms_new == 0 is a special case for destroying existing domains,
7795 * and it will not create the default domain. 7907 * and it will not create the default domain.
7796 * 7908 *
7797 * Call with hotplug lock held 7909 * Call with hotplug lock held
7798 */ 7910 */
7799void partition_sched_domains(int ndoms_new, cpumask_t *doms_new, 7911/* FIXME: Change to struct cpumask *doms_new[] */
7912void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
7800 struct sched_domain_attr *dattr_new) 7913 struct sched_domain_attr *dattr_new)
7801{ 7914{
7802 int i, j, n; 7915 int i, j, n;
@@ -7815,7 +7928,7 @@ void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
7815 /* Destroy deleted domains */ 7928 /* Destroy deleted domains */
7816 for (i = 0; i < ndoms_cur; i++) { 7929 for (i = 0; i < ndoms_cur; i++) {
7817 for (j = 0; j < n && !new_topology; j++) { 7930 for (j = 0; j < n && !new_topology; j++) {
7818 if (cpus_equal(doms_cur[i], doms_new[j]) 7931 if (cpumask_equal(&doms_cur[i], &doms_new[j])
7819 && dattrs_equal(dattr_cur, i, dattr_new, j)) 7932 && dattrs_equal(dattr_cur, i, dattr_new, j))
7820 goto match1; 7933 goto match1;
7821 } 7934 }
@@ -7827,15 +7940,15 @@ match1:
7827 7940
7828 if (doms_new == NULL) { 7941 if (doms_new == NULL) {
7829 ndoms_cur = 0; 7942 ndoms_cur = 0;
7830 doms_new = &fallback_doms; 7943 doms_new = fallback_doms;
7831 cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map); 7944 cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
7832 WARN_ON_ONCE(dattr_new); 7945 WARN_ON_ONCE(dattr_new);
7833 } 7946 }
7834 7947
7835 /* Build new domains */ 7948 /* Build new domains */
7836 for (i = 0; i < ndoms_new; i++) { 7949 for (i = 0; i < ndoms_new; i++) {
7837 for (j = 0; j < ndoms_cur && !new_topology; j++) { 7950 for (j = 0; j < ndoms_cur && !new_topology; j++) {
7838 if (cpus_equal(doms_new[i], doms_cur[j]) 7951 if (cpumask_equal(&doms_new[i], &doms_cur[j])
7839 && dattrs_equal(dattr_new, i, dattr_cur, j)) 7952 && dattrs_equal(dattr_new, i, dattr_cur, j))
7840 goto match2; 7953 goto match2;
7841 } 7954 }
@@ -7847,7 +7960,7 @@ match2:
7847 } 7960 }
7848 7961
7849 /* Remember the new sched domains */ 7962 /* Remember the new sched domains */
7850 if (doms_cur != &fallback_doms) 7963 if (doms_cur != fallback_doms)
7851 kfree(doms_cur); 7964 kfree(doms_cur);
7852 kfree(dattr_cur); /* kfree(NULL) is safe */ 7965 kfree(dattr_cur); /* kfree(NULL) is safe */
7853 doms_cur = doms_new; 7966 doms_cur = doms_new;
@@ -7876,14 +7989,25 @@ int arch_reinit_sched_domains(void)
7876static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt) 7989static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
7877{ 7990{
7878 int ret; 7991 int ret;
7992 unsigned int level = 0;
7879 7993
7880 if (buf[0] != '0' && buf[0] != '1') 7994 if (sscanf(buf, "%u", &level) != 1)
7995 return -EINVAL;
7996
7997 /*
7998 * level is always be positive so don't check for
7999 * level < POWERSAVINGS_BALANCE_NONE which is 0
8000 * What happens on 0 or 1 byte write,
8001 * need to check for count as well?
8002 */
8003
8004 if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
7881 return -EINVAL; 8005 return -EINVAL;
7882 8006
7883 if (smt) 8007 if (smt)
7884 sched_smt_power_savings = (buf[0] == '1'); 8008 sched_smt_power_savings = level;
7885 else 8009 else
7886 sched_mc_power_savings = (buf[0] == '1'); 8010 sched_mc_power_savings = level;
7887 8011
7888 ret = arch_reinit_sched_domains(); 8012 ret = arch_reinit_sched_domains();
7889 8013
@@ -7987,7 +8111,9 @@ static int update_runtime(struct notifier_block *nfb,
7987 8111
7988void __init sched_init_smp(void) 8112void __init sched_init_smp(void)
7989{ 8113{
7990 cpumask_t non_isolated_cpus; 8114 cpumask_var_t non_isolated_cpus;
8115
8116 alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
7991 8117
7992#if defined(CONFIG_NUMA) 8118#if defined(CONFIG_NUMA)
7993 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **), 8119 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
@@ -7996,10 +8122,10 @@ void __init sched_init_smp(void)
7996#endif 8122#endif
7997 get_online_cpus(); 8123 get_online_cpus();
7998 mutex_lock(&sched_domains_mutex); 8124 mutex_lock(&sched_domains_mutex);
7999 arch_init_sched_domains(&cpu_online_map); 8125 arch_init_sched_domains(cpu_online_mask);
8000 cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map); 8126 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
8001 if (cpus_empty(non_isolated_cpus)) 8127 if (cpumask_empty(non_isolated_cpus))
8002 cpu_set(smp_processor_id(), non_isolated_cpus); 8128 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
8003 mutex_unlock(&sched_domains_mutex); 8129 mutex_unlock(&sched_domains_mutex);
8004 put_online_cpus(); 8130 put_online_cpus();
8005 8131
@@ -8014,9 +8140,13 @@ void __init sched_init_smp(void)
8014 init_hrtick(); 8140 init_hrtick();
8015 8141
8016 /* Move init over to a non-isolated CPU */ 8142 /* Move init over to a non-isolated CPU */
8017 if (set_cpus_allowed_ptr(current, &non_isolated_cpus) < 0) 8143 if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
8018 BUG(); 8144 BUG();
8019 sched_init_granularity(); 8145 sched_init_granularity();
8146 free_cpumask_var(non_isolated_cpus);
8147
8148 alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
8149 init_sched_rt_class();
8020} 8150}
8021#else 8151#else
8022void __init sched_init_smp(void) 8152void __init sched_init_smp(void)
@@ -8331,6 +8461,15 @@ void __init sched_init(void)
8331 */ 8461 */
8332 current->sched_class = &fair_sched_class; 8462 current->sched_class = &fair_sched_class;
8333 8463
8464 /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
8465 alloc_bootmem_cpumask_var(&nohz_cpu_mask);
8466#ifdef CONFIG_SMP
8467#ifdef CONFIG_NO_HZ
8468 alloc_bootmem_cpumask_var(&nohz.cpu_mask);
8469#endif
8470 alloc_bootmem_cpumask_var(&cpu_isolated_map);
8471#endif /* SMP */
8472
8334 scheduler_running = 1; 8473 scheduler_running = 1;
8335} 8474}
8336 8475
generated by cgit v1.2.2 (git 2.25.0) at 2025-05-31 06:58:43 -0400