aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched.c')
-rw-r--r--kernel/sched.c1053
1 files changed, 539 insertions, 514 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index b7480fb5c3dc..8050a61a7adb 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -118,6 +118,12 @@
118 */ 118 */
119#define RUNTIME_INF ((u64)~0ULL) 119#define RUNTIME_INF ((u64)~0ULL)
120 120
121DEFINE_TRACE(sched_wait_task);
122DEFINE_TRACE(sched_wakeup);
123DEFINE_TRACE(sched_wakeup_new);
124DEFINE_TRACE(sched_switch);
125DEFINE_TRACE(sched_migrate_task);
126
121#ifdef CONFIG_SMP 127#ifdef CONFIG_SMP
122/* 128/*
123 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power) 129 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
@@ -481,14 +487,14 @@ struct rt_rq {
481 */ 487 */
482struct root_domain { 488struct root_domain {
483 atomic_t refcount; 489 atomic_t refcount;
484 cpumask_t span; 490 cpumask_var_t span;
485 cpumask_t online; 491 cpumask_var_t online;
486 492
487 /* 493 /*
488 * The "RT overload" flag: it gets set if a CPU has more than 494 * The "RT overload" flag: it gets set if a CPU has more than
489 * one runnable RT task. 495 * one runnable RT task.
490 */ 496 */
491 cpumask_t rto_mask; 497 cpumask_var_t rto_mask;
492 atomic_t rto_count; 498 atomic_t rto_count;
493#ifdef CONFIG_SMP 499#ifdef CONFIG_SMP
494 struct cpupri cpupri; 500 struct cpupri cpupri;
@@ -703,45 +709,18 @@ static __read_mostly char *sched_feat_names[] = {
703 709
704#undef SCHED_FEAT 710#undef SCHED_FEAT
705 711
706static int sched_feat_open(struct inode *inode, struct file *filp) 712static int sched_feat_show(struct seq_file *m, void *v)
707{ 713{
708 filp->private_data = inode->i_private;
709 return 0;
710}
711
712static ssize_t
713sched_feat_read(struct file *filp, char __user *ubuf,
714 size_t cnt, loff_t *ppos)
715{
716 char *buf;
717 int r = 0;
718 int len = 0;
719 int i; 714 int i;
720 715
721 for (i = 0; sched_feat_names[i]; i++) { 716 for (i = 0; sched_feat_names[i]; i++) {
722 len += strlen(sched_feat_names[i]); 717 if (!(sysctl_sched_features & (1UL << i)))
723 len += 4; 718 seq_puts(m, "NO_");
719 seq_printf(m, "%s ", sched_feat_names[i]);
724 } 720 }
721 seq_puts(m, "\n");
725 722
726 buf = kmalloc(len + 2, GFP_KERNEL); 723 return 0;
727 if (!buf)
728 return -ENOMEM;
729
730 for (i = 0; sched_feat_names[i]; i++) {
731 if (sysctl_sched_features & (1UL << i))
732 r += sprintf(buf + r, "%s ", sched_feat_names[i]);
733 else
734 r += sprintf(buf + r, "NO_%s ", sched_feat_names[i]);
735 }
736
737 r += sprintf(buf + r, "\n");
738 WARN_ON(r >= len + 2);
739
740 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
741
742 kfree(buf);
743
744 return r;
745} 724}
746 725
747static ssize_t 726static ssize_t
@@ -786,10 +765,17 @@ sched_feat_write(struct file *filp, const char __user *ubuf,
786 return cnt; 765 return cnt;
787} 766}
788 767
768static int sched_feat_open(struct inode *inode, struct file *filp)
769{
770 return single_open(filp, sched_feat_show, NULL);
771}
772
789static struct file_operations sched_feat_fops = { 773static struct file_operations sched_feat_fops = {
790 .open = sched_feat_open, 774 .open = sched_feat_open,
791 .read = sched_feat_read, 775 .write = sched_feat_write,
792 .write = sched_feat_write, 776 .read = seq_read,
777 .llseek = seq_lseek,
778 .release = single_release,
793}; 779};
794 780
795static __init int sched_init_debug(void) 781static __init int sched_init_debug(void)
@@ -1474,27 +1460,13 @@ static void
1474update_group_shares_cpu(struct task_group *tg, int cpu, 1460update_group_shares_cpu(struct task_group *tg, int cpu,
1475 unsigned long sd_shares, unsigned long sd_rq_weight) 1461 unsigned long sd_shares, unsigned long sd_rq_weight)
1476{ 1462{
1477 int boost = 0;
1478 unsigned long shares; 1463 unsigned long shares;
1479 unsigned long rq_weight; 1464 unsigned long rq_weight;
1480 1465
1481 if (!tg->se[cpu]) 1466 if (!tg->se[cpu])
1482 return; 1467 return;
1483 1468
1484 rq_weight = tg->cfs_rq[cpu]->load.weight; 1469 rq_weight = tg->cfs_rq[cpu]->rq_weight;
1485
1486 /*
1487 * If there are currently no tasks on the cpu pretend there is one of
1488 * average load so that when a new task gets to run here it will not
1489 * get delayed by group starvation.
1490 */
1491 if (!rq_weight) {
1492 boost = 1;
1493 rq_weight = NICE_0_LOAD;
1494 }
1495
1496 if (unlikely(rq_weight > sd_rq_weight))
1497 rq_weight = sd_rq_weight;
1498 1470
1499 /* 1471 /*
1500 * \Sum shares * rq_weight 1472 * \Sum shares * rq_weight
@@ -1502,7 +1474,7 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
1502 * \Sum rq_weight 1474 * \Sum rq_weight
1503 * 1475 *
1504 */ 1476 */
1505 shares = (sd_shares * rq_weight) / (sd_rq_weight + 1); 1477 shares = (sd_shares * rq_weight) / sd_rq_weight;
1506 shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES); 1478 shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
1507 1479
1508 if (abs(shares - tg->se[cpu]->load.weight) > 1480 if (abs(shares - tg->se[cpu]->load.weight) >
@@ -1511,11 +1483,7 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
1511 unsigned long flags; 1483 unsigned long flags;
1512 1484
1513 spin_lock_irqsave(&rq->lock, flags); 1485 spin_lock_irqsave(&rq->lock, flags);
1514 /* 1486 tg->cfs_rq[cpu]->shares = shares;
1515 * record the actual number of shares, not the boosted amount.
1516 */
1517 tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
1518 tg->cfs_rq[cpu]->rq_weight = rq_weight;
1519 1487
1520 __set_se_shares(tg->se[cpu], shares); 1488 __set_se_shares(tg->se[cpu], shares);
1521 spin_unlock_irqrestore(&rq->lock, flags); 1489 spin_unlock_irqrestore(&rq->lock, flags);
@@ -1529,13 +1497,23 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
1529 */ 1497 */
1530static int tg_shares_up(struct task_group *tg, void *data) 1498static int tg_shares_up(struct task_group *tg, void *data)
1531{ 1499{
1532 unsigned long rq_weight = 0; 1500 unsigned long weight, rq_weight = 0;
1533 unsigned long shares = 0; 1501 unsigned long shares = 0;
1534 struct sched_domain *sd = data; 1502 struct sched_domain *sd = data;
1535 int i; 1503 int i;
1536 1504
1537 for_each_cpu_mask(i, sd->span) { 1505 for_each_cpu(i, sched_domain_span(sd)) {
1538 rq_weight += tg->cfs_rq[i]->load.weight; 1506 /*
1507 * If there are currently no tasks on the cpu pretend there
1508 * is one of average load so that when a new task gets to
1509 * run here it will not get delayed by group starvation.
1510 */
1511 weight = tg->cfs_rq[i]->load.weight;
1512 if (!weight)
1513 weight = NICE_0_LOAD;
1514
1515 tg->cfs_rq[i]->rq_weight = weight;
1516 rq_weight += weight;
1539 shares += tg->cfs_rq[i]->shares; 1517 shares += tg->cfs_rq[i]->shares;
1540 } 1518 }
1541 1519
@@ -1545,10 +1523,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
1545 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE)) 1523 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
1546 shares = tg->shares; 1524 shares = tg->shares;
1547 1525
1548 if (!rq_weight) 1526 for_each_cpu(i, sched_domain_span(sd))
1549 rq_weight = cpus_weight(sd->span) * NICE_0_LOAD;
1550
1551 for_each_cpu_mask(i, sd->span)
1552 update_group_shares_cpu(tg, i, shares, rq_weight); 1527 update_group_shares_cpu(tg, i, shares, rq_weight);
1553 1528
1554 return 0; 1529 return 0;
@@ -2079,15 +2054,17 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2079 int i; 2054 int i;
2080 2055
2081 /* Skip over this group if it has no CPUs allowed */ 2056 /* Skip over this group if it has no CPUs allowed */
2082 if (!cpus_intersects(group->cpumask, p->cpus_allowed)) 2057 if (!cpumask_intersects(sched_group_cpus(group),
2058 &p->cpus_allowed))
2083 continue; 2059 continue;
2084 2060
2085 local_group = cpu_isset(this_cpu, group->cpumask); 2061 local_group = cpumask_test_cpu(this_cpu,
2062 sched_group_cpus(group));
2086 2063
2087 /* Tally up the load of all CPUs in the group */ 2064 /* Tally up the load of all CPUs in the group */
2088 avg_load = 0; 2065 avg_load = 0;
2089 2066
2090 for_each_cpu_mask_nr(i, group->cpumask) { 2067 for_each_cpu(i, sched_group_cpus(group)) {
2091 /* Bias balancing toward cpus of our domain */ 2068 /* Bias balancing toward cpus of our domain */
2092 if (local_group) 2069 if (local_group)
2093 load = source_load(i, load_idx); 2070 load = source_load(i, load_idx);
@@ -2119,17 +2096,14 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2119 * find_idlest_cpu - find the idlest cpu among the cpus in group. 2096 * find_idlest_cpu - find the idlest cpu among the cpus in group.
2120 */ 2097 */
2121static int 2098static int
2122find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu, 2099find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
2123 cpumask_t *tmp)
2124{ 2100{
2125 unsigned long load, min_load = ULONG_MAX; 2101 unsigned long load, min_load = ULONG_MAX;
2126 int idlest = -1; 2102 int idlest = -1;
2127 int i; 2103 int i;
2128 2104
2129 /* Traverse only the allowed CPUs */ 2105 /* Traverse only the allowed CPUs */
2130 cpus_and(*tmp, group->cpumask, p->cpus_allowed); 2106 for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
2131
2132 for_each_cpu_mask_nr(i, *tmp) {
2133 load = weighted_cpuload(i); 2107 load = weighted_cpuload(i);
2134 2108
2135 if (load < min_load || (load == min_load && i == this_cpu)) { 2109 if (load < min_load || (load == min_load && i == this_cpu)) {
@@ -2171,7 +2145,6 @@ static int sched_balance_self(int cpu, int flag)
2171 update_shares(sd); 2145 update_shares(sd);
2172 2146
2173 while (sd) { 2147 while (sd) {
2174 cpumask_t span, tmpmask;
2175 struct sched_group *group; 2148 struct sched_group *group;
2176 int new_cpu, weight; 2149 int new_cpu, weight;
2177 2150
@@ -2180,14 +2153,13 @@ static int sched_balance_self(int cpu, int flag)
2180 continue; 2153 continue;
2181 } 2154 }
2182 2155
2183 span = sd->span;
2184 group = find_idlest_group(sd, t, cpu); 2156 group = find_idlest_group(sd, t, cpu);
2185 if (!group) { 2157 if (!group) {
2186 sd = sd->child; 2158 sd = sd->child;
2187 continue; 2159 continue;
2188 } 2160 }
2189 2161
2190 new_cpu = find_idlest_cpu(group, t, cpu, &tmpmask); 2162 new_cpu = find_idlest_cpu(group, t, cpu);
2191 if (new_cpu == -1 || new_cpu == cpu) { 2163 if (new_cpu == -1 || new_cpu == cpu) {
2192 /* Now try balancing at a lower domain level of cpu */ 2164 /* Now try balancing at a lower domain level of cpu */
2193 sd = sd->child; 2165 sd = sd->child;
@@ -2196,10 +2168,10 @@ static int sched_balance_self(int cpu, int flag)
2196 2168
2197 /* Now try balancing at a lower domain level of new_cpu */ 2169 /* Now try balancing at a lower domain level of new_cpu */
2198 cpu = new_cpu; 2170 cpu = new_cpu;
2171 weight = cpumask_weight(sched_domain_span(sd));
2199 sd = NULL; 2172 sd = NULL;
2200 weight = cpus_weight(span);
2201 for_each_domain(cpu, tmp) { 2173 for_each_domain(cpu, tmp) {
2202 if (weight <= cpus_weight(tmp->span)) 2174 if (weight <= cpumask_weight(sched_domain_span(tmp)))
2203 break; 2175 break;
2204 if (tmp->flags & flag) 2176 if (tmp->flags & flag)
2205 sd = tmp; 2177 sd = tmp;
@@ -2244,7 +2216,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2244 cpu = task_cpu(p); 2216 cpu = task_cpu(p);
2245 2217
2246 for_each_domain(this_cpu, sd) { 2218 for_each_domain(this_cpu, sd) {
2247 if (cpu_isset(cpu, sd->span)) { 2219 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2248 update_shares(sd); 2220 update_shares(sd);
2249 break; 2221 break;
2250 } 2222 }
@@ -2292,7 +2264,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2292 else { 2264 else {
2293 struct sched_domain *sd; 2265 struct sched_domain *sd;
2294 for_each_domain(this_cpu, sd) { 2266 for_each_domain(this_cpu, sd) {
2295 if (cpu_isset(cpu, sd->span)) { 2267 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2296 schedstat_inc(sd, ttwu_wake_remote); 2268 schedstat_inc(sd, ttwu_wake_remote);
2297 break; 2269 break;
2298 } 2270 }
@@ -2838,7 +2810,7 @@ static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
2838 return ret; 2810 return ret;
2839} 2811}
2840 2812
2841static void double_unlock_balance(struct rq *this_rq, struct rq *busiest) 2813static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
2842 __releases(busiest->lock) 2814 __releases(busiest->lock)
2843{ 2815{
2844 spin_unlock(&busiest->lock); 2816 spin_unlock(&busiest->lock);
@@ -2858,7 +2830,7 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu)
2858 struct rq *rq; 2830 struct rq *rq;
2859 2831
2860 rq = task_rq_lock(p, &flags); 2832 rq = task_rq_lock(p, &flags);
2861 if (!cpu_isset(dest_cpu, p->cpus_allowed) 2833 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
2862 || unlikely(!cpu_active(dest_cpu))) 2834 || unlikely(!cpu_active(dest_cpu)))
2863 goto out; 2835 goto out;
2864 2836
@@ -2924,7 +2896,7 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
2924 * 2) cannot be migrated to this CPU due to cpus_allowed, or 2896 * 2) cannot be migrated to this CPU due to cpus_allowed, or
2925 * 3) are cache-hot on their current CPU. 2897 * 3) are cache-hot on their current CPU.
2926 */ 2898 */
2927 if (!cpu_isset(this_cpu, p->cpus_allowed)) { 2899 if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
2928 schedstat_inc(p, se.nr_failed_migrations_affine); 2900 schedstat_inc(p, se.nr_failed_migrations_affine);
2929 return 0; 2901 return 0;
2930 } 2902 }
@@ -3099,7 +3071,7 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
3099static struct sched_group * 3071static struct sched_group *
3100find_busiest_group(struct sched_domain *sd, int this_cpu, 3072find_busiest_group(struct sched_domain *sd, int this_cpu,
3101 unsigned long *imbalance, enum cpu_idle_type idle, 3073 unsigned long *imbalance, enum cpu_idle_type idle,
3102 int *sd_idle, const cpumask_t *cpus, int *balance) 3074 int *sd_idle, const struct cpumask *cpus, int *balance)
3103{ 3075{
3104 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups; 3076 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
3105 unsigned long max_load, avg_load, total_load, this_load, total_pwr; 3077 unsigned long max_load, avg_load, total_load, this_load, total_pwr;
@@ -3135,10 +3107,11 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3135 unsigned long sum_avg_load_per_task; 3107 unsigned long sum_avg_load_per_task;
3136 unsigned long avg_load_per_task; 3108 unsigned long avg_load_per_task;
3137 3109
3138 local_group = cpu_isset(this_cpu, group->cpumask); 3110 local_group = cpumask_test_cpu(this_cpu,
3111 sched_group_cpus(group));
3139 3112
3140 if (local_group) 3113 if (local_group)
3141 balance_cpu = first_cpu(group->cpumask); 3114 balance_cpu = cpumask_first(sched_group_cpus(group));
3142 3115
3143 /* Tally up the load of all CPUs in the group */ 3116 /* Tally up the load of all CPUs in the group */
3144 sum_weighted_load = sum_nr_running = avg_load = 0; 3117 sum_weighted_load = sum_nr_running = avg_load = 0;
@@ -3147,13 +3120,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3147 max_cpu_load = 0; 3120 max_cpu_load = 0;
3148 min_cpu_load = ~0UL; 3121 min_cpu_load = ~0UL;
3149 3122
3150 for_each_cpu_mask_nr(i, group->cpumask) { 3123 for_each_cpu_and(i, sched_group_cpus(group), cpus) {
3151 struct rq *rq; 3124 struct rq *rq = cpu_rq(i);
3152
3153 if (!cpu_isset(i, *cpus))
3154 continue;
3155
3156 rq = cpu_rq(i);
3157 3125
3158 if (*sd_idle && rq->nr_running) 3126 if (*sd_idle && rq->nr_running)
3159 *sd_idle = 0; 3127 *sd_idle = 0;
@@ -3264,8 +3232,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3264 */ 3232 */
3265 if ((sum_nr_running < min_nr_running) || 3233 if ((sum_nr_running < min_nr_running) ||
3266 (sum_nr_running == min_nr_running && 3234 (sum_nr_running == min_nr_running &&
3267 first_cpu(group->cpumask) < 3235 cpumask_first(sched_group_cpus(group)) <
3268 first_cpu(group_min->cpumask))) { 3236 cpumask_first(sched_group_cpus(group_min)))) {
3269 group_min = group; 3237 group_min = group;
3270 min_nr_running = sum_nr_running; 3238 min_nr_running = sum_nr_running;
3271 min_load_per_task = sum_weighted_load / 3239 min_load_per_task = sum_weighted_load /
@@ -3280,8 +3248,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3280 if (sum_nr_running <= group_capacity - 1) { 3248 if (sum_nr_running <= group_capacity - 1) {
3281 if (sum_nr_running > leader_nr_running || 3249 if (sum_nr_running > leader_nr_running ||
3282 (sum_nr_running == leader_nr_running && 3250 (sum_nr_running == leader_nr_running &&
3283 first_cpu(group->cpumask) > 3251 cpumask_first(sched_group_cpus(group)) >
3284 first_cpu(group_leader->cpumask))) { 3252 cpumask_first(sched_group_cpus(group_leader)))) {
3285 group_leader = group; 3253 group_leader = group;
3286 leader_nr_running = sum_nr_running; 3254 leader_nr_running = sum_nr_running;
3287 } 3255 }
@@ -3420,16 +3388,16 @@ ret:
3420 */ 3388 */
3421static struct rq * 3389static struct rq *
3422find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle, 3390find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3423 unsigned long imbalance, const cpumask_t *cpus) 3391 unsigned long imbalance, const struct cpumask *cpus)
3424{ 3392{
3425 struct rq *busiest = NULL, *rq; 3393 struct rq *busiest = NULL, *rq;
3426 unsigned long max_load = 0; 3394 unsigned long max_load = 0;
3427 int i; 3395 int i;
3428 3396
3429 for_each_cpu_mask_nr(i, group->cpumask) { 3397 for_each_cpu(i, sched_group_cpus(group)) {
3430 unsigned long wl; 3398 unsigned long wl;
3431 3399
3432 if (!cpu_isset(i, *cpus)) 3400 if (!cpumask_test_cpu(i, cpus))
3433 continue; 3401 continue;
3434 3402
3435 rq = cpu_rq(i); 3403 rq = cpu_rq(i);
@@ -3459,7 +3427,7 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3459 */ 3427 */
3460static int load_balance(int this_cpu, struct rq *this_rq, 3428static int load_balance(int this_cpu, struct rq *this_rq,
3461 struct sched_domain *sd, enum cpu_idle_type idle, 3429 struct sched_domain *sd, enum cpu_idle_type idle,
3462 int *balance, cpumask_t *cpus) 3430 int *balance, struct cpumask *cpus)
3463{ 3431{
3464 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0; 3432 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
3465 struct sched_group *group; 3433 struct sched_group *group;
@@ -3467,7 +3435,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
3467 struct rq *busiest; 3435 struct rq *busiest;
3468 unsigned long flags; 3436 unsigned long flags;
3469 3437
3470 cpus_setall(*cpus); 3438 cpumask_setall(cpus);
3471 3439
3472 /* 3440 /*
3473 * When power savings policy is enabled for the parent domain, idle 3441 * When power savings policy is enabled for the parent domain, idle
@@ -3527,8 +3495,8 @@ redo:
3527 3495
3528 /* All tasks on this runqueue were pinned by CPU affinity */ 3496 /* All tasks on this runqueue were pinned by CPU affinity */
3529 if (unlikely(all_pinned)) { 3497 if (unlikely(all_pinned)) {
3530 cpu_clear(cpu_of(busiest), *cpus); 3498 cpumask_clear_cpu(cpu_of(busiest), cpus);
3531 if (!cpus_empty(*cpus)) 3499 if (!cpumask_empty(cpus))
3532 goto redo; 3500 goto redo;
3533 goto out_balanced; 3501 goto out_balanced;
3534 } 3502 }
@@ -3545,7 +3513,8 @@ redo:
3545 /* don't kick the migration_thread, if the curr 3513 /* don't kick the migration_thread, if the curr
3546 * task on busiest cpu can't be moved to this_cpu 3514 * task on busiest cpu can't be moved to this_cpu
3547 */ 3515 */
3548 if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) { 3516 if (!cpumask_test_cpu(this_cpu,
3517 &busiest->curr->cpus_allowed)) {
3549 spin_unlock_irqrestore(&busiest->lock, flags); 3518 spin_unlock_irqrestore(&busiest->lock, flags);
3550 all_pinned = 1; 3519 all_pinned = 1;
3551 goto out_one_pinned; 3520 goto out_one_pinned;
@@ -3620,7 +3589,7 @@ out:
3620 */ 3589 */
3621static int 3590static int
3622load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd, 3591load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3623 cpumask_t *cpus) 3592 struct cpumask *cpus)
3624{ 3593{
3625 struct sched_group *group; 3594 struct sched_group *group;
3626 struct rq *busiest = NULL; 3595 struct rq *busiest = NULL;
@@ -3629,7 +3598,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3629 int sd_idle = 0; 3598 int sd_idle = 0;
3630 int all_pinned = 0; 3599 int all_pinned = 0;
3631 3600
3632 cpus_setall(*cpus); 3601 cpumask_setall(cpus);
3633 3602
3634 /* 3603 /*
3635 * When power savings policy is enabled for the parent domain, idle 3604 * When power savings policy is enabled for the parent domain, idle
@@ -3673,8 +3642,8 @@ redo:
3673 double_unlock_balance(this_rq, busiest); 3642 double_unlock_balance(this_rq, busiest);
3674 3643
3675 if (unlikely(all_pinned)) { 3644 if (unlikely(all_pinned)) {
3676 cpu_clear(cpu_of(busiest), *cpus); 3645 cpumask_clear_cpu(cpu_of(busiest), cpus);
3677 if (!cpus_empty(*cpus)) 3646 if (!cpumask_empty(cpus))
3678 goto redo; 3647 goto redo;
3679 } 3648 }
3680 } 3649 }
@@ -3709,7 +3678,10 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3709 struct sched_domain *sd; 3678 struct sched_domain *sd;
3710 int pulled_task = -1; 3679 int pulled_task = -1;
3711 unsigned long next_balance = jiffies + HZ; 3680 unsigned long next_balance = jiffies + HZ;
3712 cpumask_t tmpmask; 3681 cpumask_var_t tmpmask;
3682
3683 if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
3684 return;
3713 3685
3714 for_each_domain(this_cpu, sd) { 3686 for_each_domain(this_cpu, sd) {
3715 unsigned long interval; 3687 unsigned long interval;
@@ -3720,7 +3692,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3720 if (sd->flags & SD_BALANCE_NEWIDLE) 3692 if (sd->flags & SD_BALANCE_NEWIDLE)
3721 /* If we've pulled tasks over stop searching: */ 3693 /* If we've pulled tasks over stop searching: */
3722 pulled_task = load_balance_newidle(this_cpu, this_rq, 3694 pulled_task = load_balance_newidle(this_cpu, this_rq,
3723 sd, &tmpmask); 3695 sd, tmpmask);
3724 3696
3725 interval = msecs_to_jiffies(sd->balance_interval); 3697 interval = msecs_to_jiffies(sd->balance_interval);
3726 if (time_after(next_balance, sd->last_balance + interval)) 3698 if (time_after(next_balance, sd->last_balance + interval))
@@ -3735,6 +3707,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3735 */ 3707 */
3736 this_rq->next_balance = next_balance; 3708 this_rq->next_balance = next_balance;
3737 } 3709 }
3710 free_cpumask_var(tmpmask);
3738} 3711}
3739 3712
3740/* 3713/*
@@ -3772,7 +3745,7 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3772 /* Search for an sd spanning us and the target CPU. */ 3745 /* Search for an sd spanning us and the target CPU. */
3773 for_each_domain(target_cpu, sd) { 3746 for_each_domain(target_cpu, sd) {
3774 if ((sd->flags & SD_LOAD_BALANCE) && 3747 if ((sd->flags & SD_LOAD_BALANCE) &&
3775 cpu_isset(busiest_cpu, sd->span)) 3748 cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
3776 break; 3749 break;
3777 } 3750 }
3778 3751
@@ -3791,10 +3764,9 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3791#ifdef CONFIG_NO_HZ 3764#ifdef CONFIG_NO_HZ
3792static struct { 3765static struct {
3793 atomic_t load_balancer; 3766 atomic_t load_balancer;
3794 cpumask_t cpu_mask; 3767 cpumask_var_t cpu_mask;
3795} nohz ____cacheline_aligned = { 3768} nohz ____cacheline_aligned = {
3796 .load_balancer = ATOMIC_INIT(-1), 3769 .load_balancer = ATOMIC_INIT(-1),
3797 .cpu_mask = CPU_MASK_NONE,
3798}; 3770};
3799 3771
3800/* 3772/*
@@ -3822,7 +3794,7 @@ int select_nohz_load_balancer(int stop_tick)
3822 int cpu = smp_processor_id(); 3794 int cpu = smp_processor_id();
3823 3795
3824 if (stop_tick) { 3796 if (stop_tick) {
3825 cpu_set(cpu, nohz.cpu_mask); 3797 cpumask_set_cpu(cpu, nohz.cpu_mask);
3826 cpu_rq(cpu)->in_nohz_recently = 1; 3798 cpu_rq(cpu)->in_nohz_recently = 1;
3827 3799
3828 /* 3800 /*
@@ -3836,7 +3808,7 @@ int select_nohz_load_balancer(int stop_tick)
3836 } 3808 }
3837 3809
3838 /* time for ilb owner also to sleep */ 3810 /* time for ilb owner also to sleep */
3839 if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 3811 if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
3840 if (atomic_read(&nohz.load_balancer) == cpu) 3812 if (atomic_read(&nohz.load_balancer) == cpu)
3841 atomic_set(&nohz.load_balancer, -1); 3813 atomic_set(&nohz.load_balancer, -1);
3842 return 0; 3814 return 0;
@@ -3849,10 +3821,10 @@ int select_nohz_load_balancer(int stop_tick)
3849 } else if (atomic_read(&nohz.load_balancer) == cpu) 3821 } else if (atomic_read(&nohz.load_balancer) == cpu)
3850 return 1; 3822 return 1;
3851 } else { 3823 } else {
3852 if (!cpu_isset(cpu, nohz.cpu_mask)) 3824 if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
3853 return 0; 3825 return 0;
3854 3826
3855 cpu_clear(cpu, nohz.cpu_mask); 3827 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3856 3828
3857 if (atomic_read(&nohz.load_balancer) == cpu) 3829 if (atomic_read(&nohz.load_balancer) == cpu)
3858 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu) 3830 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
@@ -3880,7 +3852,11 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3880 unsigned long next_balance = jiffies + 60*HZ; 3852 unsigned long next_balance = jiffies + 60*HZ;
3881 int update_next_balance = 0; 3853 int update_next_balance = 0;
3882 int need_serialize; 3854 int need_serialize;
3883 cpumask_t tmp; 3855 cpumask_var_t tmp;
3856
3857 /* Fails alloc? Rebalancing probably not a priority right now. */
3858 if (!alloc_cpumask_var(&tmp, GFP_ATOMIC))
3859 return;
3884 3860
3885 for_each_domain(cpu, sd) { 3861 for_each_domain(cpu, sd) {
3886 if (!(sd->flags & SD_LOAD_BALANCE)) 3862 if (!(sd->flags & SD_LOAD_BALANCE))
@@ -3905,7 +3881,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3905 } 3881 }
3906 3882
3907 if (time_after_eq(jiffies, sd->last_balance + interval)) { 3883 if (time_after_eq(jiffies, sd->last_balance + interval)) {
3908 if (load_balance(cpu, rq, sd, idle, &balance, &tmp)) { 3884 if (load_balance(cpu, rq, sd, idle, &balance, tmp)) {
3909 /* 3885 /*
3910 * We've pulled tasks over so either we're no 3886 * We've pulled tasks over so either we're no
3911 * longer idle, or one of our SMT siblings is 3887 * longer idle, or one of our SMT siblings is
@@ -3939,6 +3915,8 @@ out:
3939 */ 3915 */
3940 if (likely(update_next_balance)) 3916 if (likely(update_next_balance))
3941 rq->next_balance = next_balance; 3917 rq->next_balance = next_balance;
3918
3919 free_cpumask_var(tmp);
3942} 3920}
3943 3921
3944/* 3922/*
@@ -3963,12 +3941,13 @@ static void run_rebalance_domains(struct softirq_action *h)
3963 */ 3941 */
3964 if (this_rq->idle_at_tick && 3942 if (this_rq->idle_at_tick &&
3965 atomic_read(&nohz.load_balancer) == this_cpu) { 3943 atomic_read(&nohz.load_balancer) == this_cpu) {
3966 cpumask_t cpus = nohz.cpu_mask;
3967 struct rq *rq; 3944 struct rq *rq;
3968 int balance_cpu; 3945 int balance_cpu;
3969 3946
3970 cpu_clear(this_cpu, cpus); 3947 for_each_cpu(balance_cpu, nohz.cpu_mask) {
3971 for_each_cpu_mask_nr(balance_cpu, cpus) { 3948 if (balance_cpu == this_cpu)
3949 continue;
3950
3972 /* 3951 /*
3973 * If this cpu gets work to do, stop the load balancing 3952 * If this cpu gets work to do, stop the load balancing
3974 * work being done for other cpus. Next load 3953 * work being done for other cpus. Next load
@@ -4006,7 +3985,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4006 rq->in_nohz_recently = 0; 3985 rq->in_nohz_recently = 0;
4007 3986
4008 if (atomic_read(&nohz.load_balancer) == cpu) { 3987 if (atomic_read(&nohz.load_balancer) == cpu) {
4009 cpu_clear(cpu, nohz.cpu_mask); 3988 cpumask_clear_cpu(cpu, nohz.cpu_mask);
4010 atomic_set(&nohz.load_balancer, -1); 3989 atomic_set(&nohz.load_balancer, -1);
4011 } 3990 }
4012 3991
@@ -4019,7 +3998,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4019 * TBD: Traverse the sched domains and nominate 3998 * TBD: Traverse the sched domains and nominate
4020 * the nearest cpu in the nohz.cpu_mask. 3999 * the nearest cpu in the nohz.cpu_mask.
4021 */ 4000 */
4022 int ilb = first_cpu(nohz.cpu_mask); 4001 int ilb = cpumask_first(nohz.cpu_mask);
4023 4002
4024 if (ilb < nr_cpu_ids) 4003 if (ilb < nr_cpu_ids)
4025 resched_cpu(ilb); 4004 resched_cpu(ilb);
@@ -4031,7 +4010,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4031 * cpus with ticks stopped, is it time for that to stop? 4010 * cpus with ticks stopped, is it time for that to stop?
4032 */ 4011 */
4033 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu && 4012 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
4034 cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 4013 cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
4035 resched_cpu(cpu); 4014 resched_cpu(cpu);
4036 return; 4015 return;
4037 } 4016 }
@@ -4041,7 +4020,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4041 * someone else, then no need raise the SCHED_SOFTIRQ 4020 * someone else, then no need raise the SCHED_SOFTIRQ
4042 */ 4021 */
4043 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu && 4022 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
4044 cpu_isset(cpu, nohz.cpu_mask)) 4023 cpumask_test_cpu(cpu, nohz.cpu_mask))
4045 return; 4024 return;
4046#endif 4025#endif
4047 if (time_after_eq(jiffies, rq->next_balance)) 4026 if (time_after_eq(jiffies, rq->next_balance))
@@ -4203,7 +4182,6 @@ void account_steal_time(struct task_struct *p, cputime_t steal)
4203 4182
4204 if (p == rq->idle) { 4183 if (p == rq->idle) {
4205 p->stime = cputime_add(p->stime, steal); 4184 p->stime = cputime_add(p->stime, steal);
4206 account_group_system_time(p, steal);
4207 if (atomic_read(&rq->nr_iowait) > 0) 4185 if (atomic_read(&rq->nr_iowait) > 0)
4208 cpustat->iowait = cputime64_add(cpustat->iowait, tmp); 4186 cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
4209 else 4187 else
@@ -4339,7 +4317,7 @@ void __kprobes sub_preempt_count(int val)
4339 /* 4317 /*
4340 * Underflow? 4318 * Underflow?
4341 */ 4319 */
4342 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 4320 if (DEBUG_LOCKS_WARN_ON(val > preempt_count() - (!!kernel_locked())))
4343 return; 4321 return;
4344 /* 4322 /*
4345 * Is the spinlock portion underflowing? 4323 * Is the spinlock portion underflowing?
@@ -5400,10 +5378,9 @@ out_unlock:
5400 return retval; 5378 return retval;
5401} 5379}
5402 5380
5403long sched_setaffinity(pid_t pid, const cpumask_t *in_mask) 5381long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
5404{ 5382{
5405 cpumask_t cpus_allowed; 5383 cpumask_var_t cpus_allowed, new_mask;
5406 cpumask_t new_mask = *in_mask;
5407 struct task_struct *p; 5384 struct task_struct *p;
5408 int retval; 5385 int retval;
5409 5386
@@ -5425,6 +5402,14 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5425 get_task_struct(p); 5402 get_task_struct(p);
5426 read_unlock(&tasklist_lock); 5403 read_unlock(&tasklist_lock);
5427 5404
5405 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
5406 retval = -ENOMEM;
5407 goto out_put_task;
5408 }
5409 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
5410 retval = -ENOMEM;
5411 goto out_free_cpus_allowed;
5412 }
5428 retval = -EPERM; 5413 retval = -EPERM;
5429 if ((current->euid != p->euid) && (current->euid != p->uid) && 5414 if ((current->euid != p->euid) && (current->euid != p->uid) &&
5430 !capable(CAP_SYS_NICE)) 5415 !capable(CAP_SYS_NICE))
@@ -5434,37 +5419,41 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5434 if (retval) 5419 if (retval)
5435 goto out_unlock; 5420 goto out_unlock;
5436 5421
5437 cpuset_cpus_allowed(p, &cpus_allowed); 5422 cpuset_cpus_allowed(p, cpus_allowed);
5438 cpus_and(new_mask, new_mask, cpus_allowed); 5423 cpumask_and(new_mask, in_mask, cpus_allowed);
5439 again: 5424 again:
5440 retval = set_cpus_allowed_ptr(p, &new_mask); 5425 retval = set_cpus_allowed_ptr(p, new_mask);
5441 5426
5442 if (!retval) { 5427 if (!retval) {
5443 cpuset_cpus_allowed(p, &cpus_allowed); 5428 cpuset_cpus_allowed(p, cpus_allowed);
5444 if (!cpus_subset(new_mask, cpus_allowed)) { 5429 if (!cpumask_subset(new_mask, cpus_allowed)) {
5445 /* 5430 /*
5446 * We must have raced with a concurrent cpuset 5431 * We must have raced with a concurrent cpuset
5447 * update. Just reset the cpus_allowed to the 5432 * update. Just reset the cpus_allowed to the
5448 * cpuset's cpus_allowed 5433 * cpuset's cpus_allowed
5449 */ 5434 */
5450 new_mask = cpus_allowed; 5435 cpumask_copy(new_mask, cpus_allowed);
5451 goto again; 5436 goto again;
5452 } 5437 }
5453 } 5438 }
5454out_unlock: 5439out_unlock:
5440 free_cpumask_var(new_mask);
5441out_free_cpus_allowed:
5442 free_cpumask_var(cpus_allowed);
5443out_put_task:
5455 put_task_struct(p); 5444 put_task_struct(p);
5456 put_online_cpus(); 5445 put_online_cpus();
5457 return retval; 5446 return retval;
5458} 5447}
5459 5448
5460static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 5449static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5461 cpumask_t *new_mask) 5450 struct cpumask *new_mask)
5462{ 5451{
5463 if (len < sizeof(cpumask_t)) { 5452 if (len < cpumask_size())
5464 memset(new_mask, 0, sizeof(cpumask_t)); 5453 cpumask_clear(new_mask);
5465 } else if (len > sizeof(cpumask_t)) { 5454 else if (len > cpumask_size())
5466 len = sizeof(cpumask_t); 5455 len = cpumask_size();
5467 } 5456
5468 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 5457 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
5469} 5458}
5470 5459
@@ -5477,17 +5466,20 @@ static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5477asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, 5466asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
5478 unsigned long __user *user_mask_ptr) 5467 unsigned long __user *user_mask_ptr)
5479{ 5468{
5480 cpumask_t new_mask; 5469 cpumask_var_t new_mask;
5481 int retval; 5470 int retval;
5482 5471
5483 retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask); 5472 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
5484 if (retval) 5473 return -ENOMEM;
5485 return retval;
5486 5474
5487 return sched_setaffinity(pid, &new_mask); 5475 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
5476 if (retval == 0)
5477 retval = sched_setaffinity(pid, new_mask);
5478 free_cpumask_var(new_mask);
5479 return retval;
5488} 5480}
5489 5481
5490long sched_getaffinity(pid_t pid, cpumask_t *mask) 5482long sched_getaffinity(pid_t pid, struct cpumask *mask)
5491{ 5483{
5492 struct task_struct *p; 5484 struct task_struct *p;
5493 int retval; 5485 int retval;
@@ -5504,7 +5496,7 @@ long sched_getaffinity(pid_t pid, cpumask_t *mask)
5504 if (retval) 5496 if (retval)
5505 goto out_unlock; 5497 goto out_unlock;
5506 5498
5507 cpus_and(*mask, p->cpus_allowed, cpu_online_map); 5499 cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
5508 5500
5509out_unlock: 5501out_unlock:
5510 read_unlock(&tasklist_lock); 5502 read_unlock(&tasklist_lock);
@@ -5523,19 +5515,24 @@ asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
5523 unsigned long __user *user_mask_ptr) 5515 unsigned long __user *user_mask_ptr)
5524{ 5516{
5525 int ret; 5517 int ret;
5526 cpumask_t mask; 5518 cpumask_var_t mask;
5527 5519
5528 if (len < sizeof(cpumask_t)) 5520 if (len < cpumask_size())
5529 return -EINVAL; 5521 return -EINVAL;
5530 5522
5531 ret = sched_getaffinity(pid, &mask); 5523 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
5532 if (ret < 0) 5524 return -ENOMEM;
5533 return ret;
5534 5525
5535 if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t))) 5526 ret = sched_getaffinity(pid, mask);
5536 return -EFAULT; 5527 if (ret == 0) {
5528 if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
5529 ret = -EFAULT;
5530 else
5531 ret = cpumask_size();
5532 }
5533 free_cpumask_var(mask);
5537 5534
5538 return sizeof(cpumask_t); 5535 return ret;
5539} 5536}
5540 5537
5541/** 5538/**
@@ -5877,7 +5874,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5877 idle->se.exec_start = sched_clock(); 5874 idle->se.exec_start = sched_clock();
5878 5875
5879 idle->prio = idle->normal_prio = MAX_PRIO; 5876 idle->prio = idle->normal_prio = MAX_PRIO;
5880 idle->cpus_allowed = cpumask_of_cpu(cpu); 5877 cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
5881 __set_task_cpu(idle, cpu); 5878 __set_task_cpu(idle, cpu);
5882 5879
5883 rq->curr = rq->idle = idle; 5880 rq->curr = rq->idle = idle;
@@ -5896,6 +5893,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5896 * The idle tasks have their own, simple scheduling class: 5893 * The idle tasks have their own, simple scheduling class:
5897 */ 5894 */
5898 idle->sched_class = &idle_sched_class; 5895 idle->sched_class = &idle_sched_class;
5896 ftrace_retfunc_init_task(idle);
5899} 5897}
5900 5898
5901/* 5899/*
@@ -5903,9 +5901,9 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5903 * indicates which cpus entered this state. This is used 5901 * indicates which cpus entered this state. This is used
5904 * in the rcu update to wait only for active cpus. For system 5902 * in the rcu update to wait only for active cpus. For system
5905 * which do not switch off the HZ timer nohz_cpu_mask should 5903 * which do not switch off the HZ timer nohz_cpu_mask should
5906 * always be CPU_MASK_NONE. 5904 * always be CPU_BITS_NONE.
5907 */ 5905 */
5908cpumask_t nohz_cpu_mask = CPU_MASK_NONE; 5906cpumask_var_t nohz_cpu_mask;
5909 5907
5910/* 5908/*
5911 * Increase the granularity value when there are more CPUs, 5909 * Increase the granularity value when there are more CPUs,
@@ -5960,7 +5958,7 @@ static inline void sched_init_granularity(void)
5960 * task must not exit() & deallocate itself prematurely. The 5958 * task must not exit() & deallocate itself prematurely. The
5961 * call is not atomic; no spinlocks may be held. 5959 * call is not atomic; no spinlocks may be held.
5962 */ 5960 */
5963int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask) 5961int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
5964{ 5962{
5965 struct migration_req req; 5963 struct migration_req req;
5966 unsigned long flags; 5964 unsigned long flags;
@@ -5968,13 +5966,13 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5968 int ret = 0; 5966 int ret = 0;
5969 5967
5970 rq = task_rq_lock(p, &flags); 5968 rq = task_rq_lock(p, &flags);
5971 if (!cpus_intersects(*new_mask, cpu_online_map)) { 5969 if (!cpumask_intersects(new_mask, cpu_online_mask)) {
5972 ret = -EINVAL; 5970 ret = -EINVAL;
5973 goto out; 5971 goto out;
5974 } 5972 }
5975 5973
5976 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current && 5974 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
5977 !cpus_equal(p->cpus_allowed, *new_mask))) { 5975 !cpumask_equal(&p->cpus_allowed, new_mask))) {
5978 ret = -EINVAL; 5976 ret = -EINVAL;
5979 goto out; 5977 goto out;
5980 } 5978 }
@@ -5982,15 +5980,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5982 if (p->sched_class->set_cpus_allowed) 5980 if (p->sched_class->set_cpus_allowed)
5983 p->sched_class->set_cpus_allowed(p, new_mask); 5981 p->sched_class->set_cpus_allowed(p, new_mask);
5984 else { 5982 else {
5985 p->cpus_allowed = *new_mask; 5983 cpumask_copy(&p->cpus_allowed, new_mask);
5986 p->rt.nr_cpus_allowed = cpus_weight(*new_mask); 5984 p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
5987 } 5985 }
5988 5986
5989 /* Can the task run on the task's current CPU? If so, we're done */ 5987 /* Can the task run on the task's current CPU? If so, we're done */
5990 if (cpu_isset(task_cpu(p), *new_mask)) 5988 if (cpumask_test_cpu(task_cpu(p), new_mask))
5991 goto out; 5989 goto out;
5992 5990
5993 if (migrate_task(p, any_online_cpu(*new_mask), &req)) { 5991 if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
5994 /* Need help from migration thread: drop lock and wait. */ 5992 /* Need help from migration thread: drop lock and wait. */
5995 task_rq_unlock(rq, &flags); 5993 task_rq_unlock(rq, &flags);
5996 wake_up_process(rq->migration_thread); 5994 wake_up_process(rq->migration_thread);
@@ -6032,7 +6030,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
6032 if (task_cpu(p) != src_cpu) 6030 if (task_cpu(p) != src_cpu)
6033 goto done; 6031 goto done;
6034 /* Affinity changed (again). */ 6032 /* Affinity changed (again). */
6035 if (!cpu_isset(dest_cpu, p->cpus_allowed)) 6033 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6036 goto fail; 6034 goto fail;
6037 6035
6038 on_rq = p->se.on_rq; 6036 on_rq = p->se.on_rq;
@@ -6126,54 +6124,46 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
6126 6124
6127/* 6125/*
6128 * Figure out where task on dead CPU should go, use force if necessary. 6126 * Figure out where task on dead CPU should go, use force if necessary.
6129 * NOTE: interrupts should be disabled by the caller
6130 */ 6127 */
6131static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) 6128static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6132{ 6129{
6133 unsigned long flags;
6134 cpumask_t mask;
6135 struct rq *rq;
6136 int dest_cpu; 6130 int dest_cpu;
6131 /* FIXME: Use cpumask_of_node here. */
6132 cpumask_t _nodemask = node_to_cpumask(cpu_to_node(dead_cpu));
6133 const struct cpumask *nodemask = &_nodemask;
6134
6135again:
6136 /* Look for allowed, online CPU in same node. */
6137 for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
6138 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6139 goto move;
6140
6141 /* Any allowed, online CPU? */
6142 dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
6143 if (dest_cpu < nr_cpu_ids)
6144 goto move;
6145
6146 /* No more Mr. Nice Guy. */
6147 if (dest_cpu >= nr_cpu_ids) {
6148 cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
6149 dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
6137 6150
6138 do { 6151 /*
6139 /* On same node? */ 6152 * Don't tell them about moving exiting tasks or
6140 mask = node_to_cpumask(cpu_to_node(dead_cpu)); 6153 * kernel threads (both mm NULL), since they never
6141 cpus_and(mask, mask, p->cpus_allowed); 6154 * leave kernel.
6142 dest_cpu = any_online_cpu(mask); 6155 */
6143 6156 if (p->mm && printk_ratelimit()) {
6144 /* On any allowed CPU? */ 6157 printk(KERN_INFO "process %d (%s) no "
6145 if (dest_cpu >= nr_cpu_ids) 6158 "longer affine to cpu%d\n",
6146 dest_cpu = any_online_cpu(p->cpus_allowed); 6159 task_pid_nr(p), p->comm, dead_cpu);
6147
6148 /* No more Mr. Nice Guy. */
6149 if (dest_cpu >= nr_cpu_ids) {
6150 cpumask_t cpus_allowed;
6151
6152 cpuset_cpus_allowed_locked(p, &cpus_allowed);
6153 /*
6154 * Try to stay on the same cpuset, where the
6155 * current cpuset may be a subset of all cpus.
6156 * The cpuset_cpus_allowed_locked() variant of
6157 * cpuset_cpus_allowed() will not block. It must be
6158 * called within calls to cpuset_lock/cpuset_unlock.
6159 */
6160 rq = task_rq_lock(p, &flags);
6161 p->cpus_allowed = cpus_allowed;
6162 dest_cpu = any_online_cpu(p->cpus_allowed);
6163 task_rq_unlock(rq, &flags);
6164
6165 /*
6166 * Don't tell them about moving exiting tasks or
6167 * kernel threads (both mm NULL), since they never
6168 * leave kernel.
6169 */
6170 if (p->mm && printk_ratelimit()) {
6171 printk(KERN_INFO "process %d (%s) no "
6172 "longer affine to cpu%d\n",
6173 task_pid_nr(p), p->comm, dead_cpu);
6174 }
6175 } 6160 }
6176 } while (!__migrate_task_irq(p, dead_cpu, dest_cpu)); 6161 }
6162
6163move:
6164 /* It can have affinity changed while we were choosing. */
6165 if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
6166 goto again;
6177} 6167}
6178 6168
6179/* 6169/*
@@ -6185,7 +6175,7 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6185 */ 6175 */
6186static void migrate_nr_uninterruptible(struct rq *rq_src) 6176static void migrate_nr_uninterruptible(struct rq *rq_src)
6187{ 6177{
6188 struct rq *rq_dest = cpu_rq(any_online_cpu(*CPU_MASK_ALL_PTR)); 6178 struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
6189 unsigned long flags; 6179 unsigned long flags;
6190 6180
6191 local_irq_save(flags); 6181 local_irq_save(flags);
@@ -6475,7 +6465,7 @@ static void set_rq_online(struct rq *rq)
6475 if (!rq->online) { 6465 if (!rq->online) {
6476 const struct sched_class *class; 6466 const struct sched_class *class;
6477 6467
6478 cpu_set(rq->cpu, rq->rd->online); 6468 cpumask_set_cpu(rq->cpu, rq->rd->online);
6479 rq->online = 1; 6469 rq->online = 1;
6480 6470
6481 for_each_class(class) { 6471 for_each_class(class) {
@@ -6495,7 +6485,7 @@ static void set_rq_offline(struct rq *rq)
6495 class->rq_offline(rq); 6485 class->rq_offline(rq);
6496 } 6486 }
6497 6487
6498 cpu_clear(rq->cpu, rq->rd->online); 6488 cpumask_clear_cpu(rq->cpu, rq->rd->online);
6499 rq->online = 0; 6489 rq->online = 0;
6500 } 6490 }
6501} 6491}
@@ -6536,7 +6526,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6536 rq = cpu_rq(cpu); 6526 rq = cpu_rq(cpu);
6537 spin_lock_irqsave(&rq->lock, flags); 6527 spin_lock_irqsave(&rq->lock, flags);
6538 if (rq->rd) { 6528 if (rq->rd) {
6539 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6529 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6540 6530
6541 set_rq_online(rq); 6531 set_rq_online(rq);
6542 } 6532 }
@@ -6550,7 +6540,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6550 break; 6540 break;
6551 /* Unbind it from offline cpu so it can run. Fall thru. */ 6541 /* Unbind it from offline cpu so it can run. Fall thru. */
6552 kthread_bind(cpu_rq(cpu)->migration_thread, 6542 kthread_bind(cpu_rq(cpu)->migration_thread,
6553 any_online_cpu(cpu_online_map)); 6543 cpumask_any(cpu_online_mask));
6554 kthread_stop(cpu_rq(cpu)->migration_thread); 6544 kthread_stop(cpu_rq(cpu)->migration_thread);
6555 cpu_rq(cpu)->migration_thread = NULL; 6545 cpu_rq(cpu)->migration_thread = NULL;
6556 break; 6546 break;
@@ -6598,7 +6588,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6598 rq = cpu_rq(cpu); 6588 rq = cpu_rq(cpu);
6599 spin_lock_irqsave(&rq->lock, flags); 6589 spin_lock_irqsave(&rq->lock, flags);
6600 if (rq->rd) { 6590 if (rq->rd) {
6601 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6591 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6602 set_rq_offline(rq); 6592 set_rq_offline(rq);
6603 } 6593 }
6604 spin_unlock_irqrestore(&rq->lock, flags); 6594 spin_unlock_irqrestore(&rq->lock, flags);
@@ -6636,36 +6626,14 @@ early_initcall(migration_init);
6636 6626
6637#ifdef CONFIG_SCHED_DEBUG 6627#ifdef CONFIG_SCHED_DEBUG
6638 6628
6639static inline const char *sd_level_to_string(enum sched_domain_level lvl)
6640{
6641 switch (lvl) {
6642 case SD_LV_NONE:
6643 return "NONE";
6644 case SD_LV_SIBLING:
6645 return "SIBLING";
6646 case SD_LV_MC:
6647 return "MC";
6648 case SD_LV_CPU:
6649 return "CPU";
6650 case SD_LV_NODE:
6651 return "NODE";
6652 case SD_LV_ALLNODES:
6653 return "ALLNODES";
6654 case SD_LV_MAX:
6655 return "MAX";
6656
6657 }
6658 return "MAX";
6659}
6660
6661static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, 6629static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6662 cpumask_t *groupmask) 6630 struct cpumask *groupmask)
6663{ 6631{
6664 struct sched_group *group = sd->groups; 6632 struct sched_group *group = sd->groups;
6665 char str[256]; 6633 char str[256];
6666 6634
6667 cpulist_scnprintf(str, sizeof(str), sd->span); 6635 cpulist_scnprintf(str, sizeof(str), *sched_domain_span(sd));
6668 cpus_clear(*groupmask); 6636 cpumask_clear(groupmask);
6669 6637
6670 printk(KERN_DEBUG "%*s domain %d: ", level, "", level); 6638 printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
6671 6639
@@ -6677,14 +6645,13 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6677 return -1; 6645 return -1;
6678 } 6646 }
6679 6647
6680 printk(KERN_CONT "span %s level %s\n", 6648 printk(KERN_CONT "span %s level %s\n", str, sd->name);
6681 str, sd_level_to_string(sd->level));
6682 6649
6683 if (!cpu_isset(cpu, sd->span)) { 6650 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
6684 printk(KERN_ERR "ERROR: domain->span does not contain " 6651 printk(KERN_ERR "ERROR: domain->span does not contain "
6685 "CPU%d\n", cpu); 6652 "CPU%d\n", cpu);
6686 } 6653 }
6687 if (!cpu_isset(cpu, group->cpumask)) { 6654 if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
6688 printk(KERN_ERR "ERROR: domain->groups does not contain" 6655 printk(KERN_ERR "ERROR: domain->groups does not contain"
6689 " CPU%d\n", cpu); 6656 " CPU%d\n", cpu);
6690 } 6657 }
@@ -6704,31 +6671,32 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6704 break; 6671 break;
6705 } 6672 }
6706 6673
6707 if (!cpus_weight(group->cpumask)) { 6674 if (!cpumask_weight(sched_group_cpus(group))) {
6708 printk(KERN_CONT "\n"); 6675 printk(KERN_CONT "\n");
6709 printk(KERN_ERR "ERROR: empty group\n"); 6676 printk(KERN_ERR "ERROR: empty group\n");
6710 break; 6677 break;
6711 } 6678 }
6712 6679
6713 if (cpus_intersects(*groupmask, group->cpumask)) { 6680 if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
6714 printk(KERN_CONT "\n"); 6681 printk(KERN_CONT "\n");
6715 printk(KERN_ERR "ERROR: repeated CPUs\n"); 6682 printk(KERN_ERR "ERROR: repeated CPUs\n");
6716 break; 6683 break;
6717 } 6684 }
6718 6685
6719 cpus_or(*groupmask, *groupmask, group->cpumask); 6686 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
6720 6687
6721 cpulist_scnprintf(str, sizeof(str), group->cpumask); 6688 cpulist_scnprintf(str, sizeof(str), *sched_group_cpus(group));
6722 printk(KERN_CONT " %s", str); 6689 printk(KERN_CONT " %s", str);
6723 6690
6724 group = group->next; 6691 group = group->next;
6725 } while (group != sd->groups); 6692 } while (group != sd->groups);
6726 printk(KERN_CONT "\n"); 6693 printk(KERN_CONT "\n");
6727 6694
6728 if (!cpus_equal(sd->span, *groupmask)) 6695 if (!cpumask_equal(sched_domain_span(sd), groupmask))
6729 printk(KERN_ERR "ERROR: groups don't span domain->span\n"); 6696 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
6730 6697
6731 if (sd->parent && !cpus_subset(*groupmask, sd->parent->span)) 6698 if (sd->parent &&
6699 !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
6732 printk(KERN_ERR "ERROR: parent span is not a superset " 6700 printk(KERN_ERR "ERROR: parent span is not a superset "
6733 "of domain->span\n"); 6701 "of domain->span\n");
6734 return 0; 6702 return 0;
@@ -6736,7 +6704,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6736 6704
6737static void sched_domain_debug(struct sched_domain *sd, int cpu) 6705static void sched_domain_debug(struct sched_domain *sd, int cpu)
6738{ 6706{
6739 cpumask_t *groupmask; 6707 cpumask_var_t groupmask;
6740 int level = 0; 6708 int level = 0;
6741 6709
6742 if (!sd) { 6710 if (!sd) {
@@ -6746,8 +6714,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6746 6714
6747 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); 6715 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
6748 6716
6749 groupmask = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 6717 if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
6750 if (!groupmask) {
6751 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n"); 6718 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
6752 return; 6719 return;
6753 } 6720 }
@@ -6760,7 +6727,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6760 if (!sd) 6727 if (!sd)
6761 break; 6728 break;
6762 } 6729 }
6763 kfree(groupmask); 6730 free_cpumask_var(groupmask);
6764} 6731}
6765#else /* !CONFIG_SCHED_DEBUG */ 6732#else /* !CONFIG_SCHED_DEBUG */
6766# define sched_domain_debug(sd, cpu) do { } while (0) 6733# define sched_domain_debug(sd, cpu) do { } while (0)
@@ -6768,7 +6735,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6768 6735
6769static int sd_degenerate(struct sched_domain *sd) 6736static int sd_degenerate(struct sched_domain *sd)
6770{ 6737{
6771 if (cpus_weight(sd->span) == 1) 6738 if (cpumask_weight(sched_domain_span(sd)) == 1)
6772 return 1; 6739 return 1;
6773 6740
6774 /* Following flags need at least 2 groups */ 6741 /* Following flags need at least 2 groups */
@@ -6799,7 +6766,7 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6799 if (sd_degenerate(parent)) 6766 if (sd_degenerate(parent))
6800 return 1; 6767 return 1;
6801 6768
6802 if (!cpus_equal(sd->span, parent->span)) 6769 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
6803 return 0; 6770 return 0;
6804 6771
6805 /* Does parent contain flags not in child? */ 6772 /* Does parent contain flags not in child? */
@@ -6821,6 +6788,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6821 return 1; 6788 return 1;
6822} 6789}
6823 6790
6791static void free_rootdomain(struct root_domain *rd)
6792{
6793 cpupri_cleanup(&rd->cpupri);
6794
6795 free_cpumask_var(rd->rto_mask);
6796 free_cpumask_var(rd->online);
6797 free_cpumask_var(rd->span);
6798 kfree(rd);
6799}
6800
6824static void rq_attach_root(struct rq *rq, struct root_domain *rd) 6801static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6825{ 6802{
6826 unsigned long flags; 6803 unsigned long flags;
@@ -6830,38 +6807,63 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6830 if (rq->rd) { 6807 if (rq->rd) {
6831 struct root_domain *old_rd = rq->rd; 6808 struct root_domain *old_rd = rq->rd;
6832 6809
6833 if (cpu_isset(rq->cpu, old_rd->online)) 6810 if (cpumask_test_cpu(rq->cpu, old_rd->online))
6834 set_rq_offline(rq); 6811 set_rq_offline(rq);
6835 6812
6836 cpu_clear(rq->cpu, old_rd->span); 6813 cpumask_clear_cpu(rq->cpu, old_rd->span);
6837 6814
6838 if (atomic_dec_and_test(&old_rd->refcount)) 6815 if (atomic_dec_and_test(&old_rd->refcount))
6839 kfree(old_rd); 6816 free_rootdomain(old_rd);
6840 } 6817 }
6841 6818
6842 atomic_inc(&rd->refcount); 6819 atomic_inc(&rd->refcount);
6843 rq->rd = rd; 6820 rq->rd = rd;
6844 6821
6845 cpu_set(rq->cpu, rd->span); 6822 cpumask_set_cpu(rq->cpu, rd->span);
6846 if (cpu_isset(rq->cpu, cpu_online_map)) 6823 if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
6847 set_rq_online(rq); 6824 set_rq_online(rq);
6848 6825
6849 spin_unlock_irqrestore(&rq->lock, flags); 6826 spin_unlock_irqrestore(&rq->lock, flags);
6850} 6827}
6851 6828
6852static void init_rootdomain(struct root_domain *rd) 6829static int init_rootdomain(struct root_domain *rd, bool bootmem)
6853{ 6830{
6854 memset(rd, 0, sizeof(*rd)); 6831 memset(rd, 0, sizeof(*rd));
6855 6832
6856 cpus_clear(rd->span); 6833 if (bootmem) {
6857 cpus_clear(rd->online); 6834 alloc_bootmem_cpumask_var(&def_root_domain.span);
6835 alloc_bootmem_cpumask_var(&def_root_domain.online);
6836 alloc_bootmem_cpumask_var(&def_root_domain.rto_mask);
6837 cpupri_init(&rd->cpupri, true);
6838 return 0;
6839 }
6840
6841 if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
6842 goto free_rd;
6843 if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
6844 goto free_span;
6845 if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
6846 goto free_online;
6847
6848 if (cpupri_init(&rd->cpupri, false) != 0)
6849 goto free_rto_mask;
6850 return 0;
6858 6851
6859 cpupri_init(&rd->cpupri); 6852free_rto_mask:
6853 free_cpumask_var(rd->rto_mask);
6854free_online:
6855 free_cpumask_var(rd->online);
6856free_span:
6857 free_cpumask_var(rd->span);
6858free_rd:
6859 kfree(rd);
6860 return -ENOMEM;
6860} 6861}
6861 6862
6862static void init_defrootdomain(void) 6863static void init_defrootdomain(void)
6863{ 6864{
6864 init_rootdomain(&def_root_domain); 6865 init_rootdomain(&def_root_domain, true);
6866
6865 atomic_set(&def_root_domain.refcount, 1); 6867 atomic_set(&def_root_domain.refcount, 1);
6866} 6868}
6867 6869
@@ -6873,7 +6875,10 @@ static struct root_domain *alloc_rootdomain(void)
6873 if (!rd) 6875 if (!rd)
6874 return NULL; 6876 return NULL;
6875 6877
6876 init_rootdomain(rd); 6878 if (init_rootdomain(rd, false) != 0) {
6879 kfree(rd);
6880 return NULL;
6881 }
6877 6882
6878 return rd; 6883 return rd;
6879} 6884}
@@ -6915,19 +6920,12 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
6915} 6920}
6916 6921
6917/* cpus with isolated domains */ 6922/* cpus with isolated domains */
6918static cpumask_t cpu_isolated_map = CPU_MASK_NONE; 6923static cpumask_var_t cpu_isolated_map;
6919 6924
6920/* Setup the mask of cpus configured for isolated domains */ 6925/* Setup the mask of cpus configured for isolated domains */
6921static int __init isolated_cpu_setup(char *str) 6926static int __init isolated_cpu_setup(char *str)
6922{ 6927{
6923 static int __initdata ints[NR_CPUS]; 6928 cpulist_parse(str, *cpu_isolated_map);
6924 int i;
6925
6926 str = get_options(str, ARRAY_SIZE(ints), ints);
6927 cpus_clear(cpu_isolated_map);
6928 for (i = 1; i <= ints[0]; i++)
6929 if (ints[i] < NR_CPUS)
6930 cpu_set(ints[i], cpu_isolated_map);
6931 return 1; 6929 return 1;
6932} 6930}
6933 6931
@@ -6936,42 +6934,43 @@ __setup("isolcpus=", isolated_cpu_setup);
6936/* 6934/*
6937 * init_sched_build_groups takes the cpumask we wish to span, and a pointer 6935 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
6938 * to a function which identifies what group(along with sched group) a CPU 6936 * to a function which identifies what group(along with sched group) a CPU
6939 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS 6937 * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
6940 * (due to the fact that we keep track of groups covered with a cpumask_t). 6938 * (due to the fact that we keep track of groups covered with a struct cpumask).
6941 * 6939 *
6942 * init_sched_build_groups will build a circular linked list of the groups 6940 * init_sched_build_groups will build a circular linked list of the groups
6943 * covered by the given span, and will set each group's ->cpumask correctly, 6941 * covered by the given span, and will set each group's ->cpumask correctly,
6944 * and ->cpu_power to 0. 6942 * and ->cpu_power to 0.
6945 */ 6943 */
6946static void 6944static void
6947init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map, 6945init_sched_build_groups(const struct cpumask *span,
6948 int (*group_fn)(int cpu, const cpumask_t *cpu_map, 6946 const struct cpumask *cpu_map,
6947 int (*group_fn)(int cpu, const struct cpumask *cpu_map,
6949 struct sched_group **sg, 6948 struct sched_group **sg,
6950 cpumask_t *tmpmask), 6949 struct cpumask *tmpmask),
6951 cpumask_t *covered, cpumask_t *tmpmask) 6950 struct cpumask *covered, struct cpumask *tmpmask)
6952{ 6951{
6953 struct sched_group *first = NULL, *last = NULL; 6952 struct sched_group *first = NULL, *last = NULL;
6954 int i; 6953 int i;
6955 6954
6956 cpus_clear(*covered); 6955 cpumask_clear(covered);
6957 6956
6958 for_each_cpu_mask_nr(i, *span) { 6957 for_each_cpu(i, span) {
6959 struct sched_group *sg; 6958 struct sched_group *sg;
6960 int group = group_fn(i, cpu_map, &sg, tmpmask); 6959 int group = group_fn(i, cpu_map, &sg, tmpmask);
6961 int j; 6960 int j;
6962 6961
6963 if (cpu_isset(i, *covered)) 6962 if (cpumask_test_cpu(i, covered))
6964 continue; 6963 continue;
6965 6964
6966 cpus_clear(sg->cpumask); 6965 cpumask_clear(sched_group_cpus(sg));
6967 sg->__cpu_power = 0; 6966 sg->__cpu_power = 0;
6968 6967
6969 for_each_cpu_mask_nr(j, *span) { 6968 for_each_cpu(j, span) {
6970 if (group_fn(j, cpu_map, NULL, tmpmask) != group) 6969 if (group_fn(j, cpu_map, NULL, tmpmask) != group)
6971 continue; 6970 continue;
6972 6971
6973 cpu_set(j, *covered); 6972 cpumask_set_cpu(j, covered);
6974 cpu_set(j, sg->cpumask); 6973 cpumask_set_cpu(j, sched_group_cpus(sg));
6975 } 6974 }
6976 if (!first) 6975 if (!first)
6977 first = sg; 6976 first = sg;
@@ -7035,9 +7034,10 @@ static int find_next_best_node(int node, nodemask_t *used_nodes)
7035 * should be one that prevents unnecessary balancing, but also spreads tasks 7034 * should be one that prevents unnecessary balancing, but also spreads tasks
7036 * out optimally. 7035 * out optimally.
7037 */ 7036 */
7038static void sched_domain_node_span(int node, cpumask_t *span) 7037static void sched_domain_node_span(int node, struct cpumask *span)
7039{ 7038{
7040 nodemask_t used_nodes; 7039 nodemask_t used_nodes;
7040 /* FIXME: use cpumask_of_node() */
7041 node_to_cpumask_ptr(nodemask, node); 7041 node_to_cpumask_ptr(nodemask, node);
7042 int i; 7042 int i;
7043 7043
@@ -7059,18 +7059,33 @@ static void sched_domain_node_span(int node, cpumask_t *span)
7059int sched_smt_power_savings = 0, sched_mc_power_savings = 0; 7059int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
7060 7060
7061/* 7061/*
7062 * The cpus mask in sched_group and sched_domain hangs off the end.
7063 * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space
7064 * for nr_cpu_ids < CONFIG_NR_CPUS.
7065 */
7066struct static_sched_group {
7067 struct sched_group sg;
7068 DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
7069};
7070
7071struct static_sched_domain {
7072 struct sched_domain sd;
7073 DECLARE_BITMAP(span, CONFIG_NR_CPUS);
7074};
7075
7076/*
7062 * SMT sched-domains: 7077 * SMT sched-domains:
7063 */ 7078 */
7064#ifdef CONFIG_SCHED_SMT 7079#ifdef CONFIG_SCHED_SMT
7065static DEFINE_PER_CPU(struct sched_domain, cpu_domains); 7080static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
7066static DEFINE_PER_CPU(struct sched_group, sched_group_cpus); 7081static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
7067 7082
7068static int 7083static int
7069cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7084cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
7070 cpumask_t *unused) 7085 struct sched_group **sg, struct cpumask *unused)
7071{ 7086{
7072 if (sg) 7087 if (sg)
7073 *sg = &per_cpu(sched_group_cpus, cpu); 7088 *sg = &per_cpu(sched_group_cpus, cpu).sg;
7074 return cpu; 7089 return cpu;
7075} 7090}
7076#endif /* CONFIG_SCHED_SMT */ 7091#endif /* CONFIG_SCHED_SMT */
@@ -7079,56 +7094,55 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
7079 * multi-core sched-domains: 7094 * multi-core sched-domains:
7080 */ 7095 */
7081#ifdef CONFIG_SCHED_MC 7096#ifdef CONFIG_SCHED_MC
7082static DEFINE_PER_CPU(struct sched_domain, core_domains); 7097static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
7083static DEFINE_PER_CPU(struct sched_group, sched_group_core); 7098static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
7084#endif /* CONFIG_SCHED_MC */ 7099#endif /* CONFIG_SCHED_MC */
7085 7100
7086#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) 7101#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
7087static int 7102static int
7088cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7103cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7089 cpumask_t *mask) 7104 struct sched_group **sg, struct cpumask *mask)
7090{ 7105{
7091 int group; 7106 int group;
7092 7107
7093 *mask = per_cpu(cpu_sibling_map, cpu); 7108 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7094 cpus_and(*mask, *mask, *cpu_map); 7109 group = cpumask_first(mask);
7095 group = first_cpu(*mask);
7096 if (sg) 7110 if (sg)
7097 *sg = &per_cpu(sched_group_core, group); 7111 *sg = &per_cpu(sched_group_core, group).sg;
7098 return group; 7112 return group;
7099} 7113}
7100#elif defined(CONFIG_SCHED_MC) 7114#elif defined(CONFIG_SCHED_MC)
7101static int 7115static int
7102cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7116cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7103 cpumask_t *unused) 7117 struct sched_group **sg, struct cpumask *unused)
7104{ 7118{
7105 if (sg) 7119 if (sg)
7106 *sg = &per_cpu(sched_group_core, cpu); 7120 *sg = &per_cpu(sched_group_core, cpu).sg;
7107 return cpu; 7121 return cpu;
7108} 7122}
7109#endif 7123#endif
7110 7124
7111static DEFINE_PER_CPU(struct sched_domain, phys_domains); 7125static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
7112static DEFINE_PER_CPU(struct sched_group, sched_group_phys); 7126static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
7113 7127
7114static int 7128static int
7115cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7129cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
7116 cpumask_t *mask) 7130 struct sched_group **sg, struct cpumask *mask)
7117{ 7131{
7118 int group; 7132 int group;
7119#ifdef CONFIG_SCHED_MC 7133#ifdef CONFIG_SCHED_MC
7134 /* FIXME: Use cpu_coregroup_mask. */
7120 *mask = cpu_coregroup_map(cpu); 7135 *mask = cpu_coregroup_map(cpu);
7121 cpus_and(*mask, *mask, *cpu_map); 7136 cpus_and(*mask, *mask, *cpu_map);
7122 group = first_cpu(*mask); 7137 group = cpumask_first(mask);
7123#elif defined(CONFIG_SCHED_SMT) 7138#elif defined(CONFIG_SCHED_SMT)
7124 *mask = per_cpu(cpu_sibling_map, cpu); 7139 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7125 cpus_and(*mask, *mask, *cpu_map); 7140 group = cpumask_first(mask);
7126 group = first_cpu(*mask);
7127#else 7141#else
7128 group = cpu; 7142 group = cpu;
7129#endif 7143#endif
7130 if (sg) 7144 if (sg)
7131 *sg = &per_cpu(sched_group_phys, group); 7145 *sg = &per_cpu(sched_group_phys, group).sg;
7132 return group; 7146 return group;
7133} 7147}
7134 7148
@@ -7142,19 +7156,21 @@ static DEFINE_PER_CPU(struct sched_domain, node_domains);
7142static struct sched_group ***sched_group_nodes_bycpu; 7156static struct sched_group ***sched_group_nodes_bycpu;
7143 7157
7144static DEFINE_PER_CPU(struct sched_domain, allnodes_domains); 7158static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
7145static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes); 7159static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
7146 7160
7147static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map, 7161static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
7148 struct sched_group **sg, cpumask_t *nodemask) 7162 struct sched_group **sg,
7163 struct cpumask *nodemask)
7149{ 7164{
7150 int group; 7165 int group;
7166 /* FIXME: use cpumask_of_node */
7167 node_to_cpumask_ptr(pnodemask, cpu_to_node(cpu));
7151 7168
7152 *nodemask = node_to_cpumask(cpu_to_node(cpu)); 7169 cpumask_and(nodemask, pnodemask, cpu_map);
7153 cpus_and(*nodemask, *nodemask, *cpu_map); 7170 group = cpumask_first(nodemask);
7154 group = first_cpu(*nodemask);
7155 7171
7156 if (sg) 7172 if (sg)
7157 *sg = &per_cpu(sched_group_allnodes, group); 7173 *sg = &per_cpu(sched_group_allnodes, group).sg;
7158 return group; 7174 return group;
7159} 7175}
7160 7176
@@ -7166,11 +7182,11 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7166 if (!sg) 7182 if (!sg)
7167 return; 7183 return;
7168 do { 7184 do {
7169 for_each_cpu_mask_nr(j, sg->cpumask) { 7185 for_each_cpu(j, sched_group_cpus(sg)) {
7170 struct sched_domain *sd; 7186 struct sched_domain *sd;
7171 7187
7172 sd = &per_cpu(phys_domains, j); 7188 sd = &per_cpu(phys_domains, j).sd;
7173 if (j != first_cpu(sd->groups->cpumask)) { 7189 if (j != cpumask_first(sched_group_cpus(sd->groups))) {
7174 /* 7190 /*
7175 * Only add "power" once for each 7191 * Only add "power" once for each
7176 * physical package. 7192 * physical package.
@@ -7187,11 +7203,12 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7187 7203
7188#ifdef CONFIG_NUMA 7204#ifdef CONFIG_NUMA
7189/* Free memory allocated for various sched_group structures */ 7205/* Free memory allocated for various sched_group structures */
7190static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7206static void free_sched_groups(const struct cpumask *cpu_map,
7207 struct cpumask *nodemask)
7191{ 7208{
7192 int cpu, i; 7209 int cpu, i;
7193 7210
7194 for_each_cpu_mask_nr(cpu, *cpu_map) { 7211 for_each_cpu(cpu, cpu_map) {
7195 struct sched_group **sched_group_nodes 7212 struct sched_group **sched_group_nodes
7196 = sched_group_nodes_bycpu[cpu]; 7213 = sched_group_nodes_bycpu[cpu];
7197 7214
@@ -7200,10 +7217,11 @@ static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
7200 7217
7201 for (i = 0; i < nr_node_ids; i++) { 7218 for (i = 0; i < nr_node_ids; i++) {
7202 struct sched_group *oldsg, *sg = sched_group_nodes[i]; 7219 struct sched_group *oldsg, *sg = sched_group_nodes[i];
7220 /* FIXME: Use cpumask_of_node */
7221 node_to_cpumask_ptr(pnodemask, i);
7203 7222
7204 *nodemask = node_to_cpumask(i); 7223 cpus_and(*nodemask, *pnodemask, *cpu_map);
7205 cpus_and(*nodemask, *nodemask, *cpu_map); 7224 if (cpumask_empty(nodemask))
7206 if (cpus_empty(*nodemask))
7207 continue; 7225 continue;
7208 7226
7209 if (sg == NULL) 7227 if (sg == NULL)
@@ -7221,7 +7239,8 @@ next_sg:
7221 } 7239 }
7222} 7240}
7223#else /* !CONFIG_NUMA */ 7241#else /* !CONFIG_NUMA */
7224static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7242static void free_sched_groups(const struct cpumask *cpu_map,
7243 struct cpumask *nodemask)
7225{ 7244{
7226} 7245}
7227#endif /* CONFIG_NUMA */ 7246#endif /* CONFIG_NUMA */
@@ -7247,7 +7266,7 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
7247 7266
7248 WARN_ON(!sd || !sd->groups); 7267 WARN_ON(!sd || !sd->groups);
7249 7268
7250 if (cpu != first_cpu(sd->groups->cpumask)) 7269 if (cpu != cpumask_first(sched_group_cpus(sd->groups)))
7251 return; 7270 return;
7252 7271
7253 child = sd->child; 7272 child = sd->child;
@@ -7312,40 +7331,6 @@ SD_INIT_FUNC(CPU)
7312 SD_INIT_FUNC(MC) 7331 SD_INIT_FUNC(MC)
7313#endif 7332#endif
7314 7333
7315/*
7316 * To minimize stack usage kmalloc room for cpumasks and share the
7317 * space as the usage in build_sched_domains() dictates. Used only
7318 * if the amount of space is significant.
7319 */
7320struct allmasks {
7321 cpumask_t tmpmask; /* make this one first */
7322 union {
7323 cpumask_t nodemask;
7324 cpumask_t this_sibling_map;
7325 cpumask_t this_core_map;
7326 };
7327 cpumask_t send_covered;
7328
7329#ifdef CONFIG_NUMA
7330 cpumask_t domainspan;
7331 cpumask_t covered;
7332 cpumask_t notcovered;
7333#endif
7334};
7335
7336#if NR_CPUS > 128
7337#define SCHED_CPUMASK_ALLOC 1
7338#define SCHED_CPUMASK_FREE(v) kfree(v)
7339#define SCHED_CPUMASK_DECLARE(v) struct allmasks *v
7340#else
7341#define SCHED_CPUMASK_ALLOC 0
7342#define SCHED_CPUMASK_FREE(v)
7343#define SCHED_CPUMASK_DECLARE(v) struct allmasks _v, *v = &_v
7344#endif
7345
7346#define SCHED_CPUMASK_VAR(v, a) cpumask_t *v = (cpumask_t *) \
7347 ((unsigned long)(a) + offsetof(struct allmasks, v))
7348
7349static int default_relax_domain_level = -1; 7334static int default_relax_domain_level = -1;
7350 7335
7351static int __init setup_relax_domain_level(char *str) 7336static int __init setup_relax_domain_level(char *str)
@@ -7385,17 +7370,38 @@ static void set_domain_attribute(struct sched_domain *sd,
7385 * Build sched domains for a given set of cpus and attach the sched domains 7370 * Build sched domains for a given set of cpus and attach the sched domains
7386 * to the individual cpus 7371 * to the individual cpus
7387 */ 7372 */
7388static int __build_sched_domains(const cpumask_t *cpu_map, 7373static int __build_sched_domains(const struct cpumask *cpu_map,
7389 struct sched_domain_attr *attr) 7374 struct sched_domain_attr *attr)
7390{ 7375{
7391 int i; 7376 int i, err = -ENOMEM;
7392 struct root_domain *rd; 7377 struct root_domain *rd;
7393 SCHED_CPUMASK_DECLARE(allmasks); 7378 cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
7394 cpumask_t *tmpmask; 7379 tmpmask;
7395#ifdef CONFIG_NUMA 7380#ifdef CONFIG_NUMA
7381 cpumask_var_t domainspan, covered, notcovered;
7396 struct sched_group **sched_group_nodes = NULL; 7382 struct sched_group **sched_group_nodes = NULL;
7397 int sd_allnodes = 0; 7383 int sd_allnodes = 0;
7398 7384
7385 if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
7386 goto out;
7387 if (!alloc_cpumask_var(&covered, GFP_KERNEL))
7388 goto free_domainspan;
7389 if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
7390 goto free_covered;
7391#endif
7392
7393 if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
7394 goto free_notcovered;
7395 if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
7396 goto free_nodemask;
7397 if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
7398 goto free_this_sibling_map;
7399 if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
7400 goto free_this_core_map;
7401 if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
7402 goto free_send_covered;
7403
7404#ifdef CONFIG_NUMA
7399 /* 7405 /*
7400 * Allocate the per-node list of sched groups 7406 * Allocate the per-node list of sched groups
7401 */ 7407 */
@@ -7403,55 +7409,37 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7403 GFP_KERNEL); 7409 GFP_KERNEL);
7404 if (!sched_group_nodes) { 7410 if (!sched_group_nodes) {
7405 printk(KERN_WARNING "Can not alloc sched group node list\n"); 7411 printk(KERN_WARNING "Can not alloc sched group node list\n");
7406 return -ENOMEM; 7412 goto free_tmpmask;
7407 } 7413 }
7408#endif 7414#endif
7409 7415
7410 rd = alloc_rootdomain(); 7416 rd = alloc_rootdomain();
7411 if (!rd) { 7417 if (!rd) {
7412 printk(KERN_WARNING "Cannot alloc root domain\n"); 7418 printk(KERN_WARNING "Cannot alloc root domain\n");
7413#ifdef CONFIG_NUMA 7419 goto free_sched_groups;
7414 kfree(sched_group_nodes);
7415#endif
7416 return -ENOMEM;
7417 }
7418
7419#if SCHED_CPUMASK_ALLOC
7420 /* get space for all scratch cpumask variables */
7421 allmasks = kmalloc(sizeof(*allmasks), GFP_KERNEL);
7422 if (!allmasks) {
7423 printk(KERN_WARNING "Cannot alloc cpumask array\n");
7424 kfree(rd);
7425#ifdef CONFIG_NUMA
7426 kfree(sched_group_nodes);
7427#endif
7428 return -ENOMEM;
7429 } 7420 }
7430#endif
7431 tmpmask = (cpumask_t *)allmasks;
7432
7433 7421
7434#ifdef CONFIG_NUMA 7422#ifdef CONFIG_NUMA
7435 sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes; 7423 sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
7436#endif 7424#endif
7437 7425
7438 /* 7426 /*
7439 * Set up domains for cpus specified by the cpu_map. 7427 * Set up domains for cpus specified by the cpu_map.
7440 */ 7428 */
7441 for_each_cpu_mask_nr(i, *cpu_map) { 7429 for_each_cpu(i, cpu_map) {
7442 struct sched_domain *sd = NULL, *p; 7430 struct sched_domain *sd = NULL, *p;
7443 SCHED_CPUMASK_VAR(nodemask, allmasks);
7444 7431
7432 /* FIXME: use cpumask_of_node */
7445 *nodemask = node_to_cpumask(cpu_to_node(i)); 7433 *nodemask = node_to_cpumask(cpu_to_node(i));
7446 cpus_and(*nodemask, *nodemask, *cpu_map); 7434 cpus_and(*nodemask, *nodemask, *cpu_map);
7447 7435
7448#ifdef CONFIG_NUMA 7436#ifdef CONFIG_NUMA
7449 if (cpus_weight(*cpu_map) > 7437 if (cpumask_weight(cpu_map) >
7450 SD_NODES_PER_DOMAIN*cpus_weight(*nodemask)) { 7438 SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
7451 sd = &per_cpu(allnodes_domains, i); 7439 sd = &per_cpu(allnodes_domains, i);
7452 SD_INIT(sd, ALLNODES); 7440 SD_INIT(sd, ALLNODES);
7453 set_domain_attribute(sd, attr); 7441 set_domain_attribute(sd, attr);
7454 sd->span = *cpu_map; 7442 cpumask_copy(sched_domain_span(sd), cpu_map);
7455 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask); 7443 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
7456 p = sd; 7444 p = sd;
7457 sd_allnodes = 1; 7445 sd_allnodes = 1;
@@ -7461,18 +7449,19 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7461 sd = &per_cpu(node_domains, i); 7449 sd = &per_cpu(node_domains, i);
7462 SD_INIT(sd, NODE); 7450 SD_INIT(sd, NODE);
7463 set_domain_attribute(sd, attr); 7451 set_domain_attribute(sd, attr);
7464 sched_domain_node_span(cpu_to_node(i), &sd->span); 7452 sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
7465 sd->parent = p; 7453 sd->parent = p;
7466 if (p) 7454 if (p)
7467 p->child = sd; 7455 p->child = sd;
7468 cpus_and(sd->span, sd->span, *cpu_map); 7456 cpumask_and(sched_domain_span(sd),
7457 sched_domain_span(sd), cpu_map);
7469#endif 7458#endif
7470 7459
7471 p = sd; 7460 p = sd;
7472 sd = &per_cpu(phys_domains, i); 7461 sd = &per_cpu(phys_domains, i).sd;
7473 SD_INIT(sd, CPU); 7462 SD_INIT(sd, CPU);
7474 set_domain_attribute(sd, attr); 7463 set_domain_attribute(sd, attr);
7475 sd->span = *nodemask; 7464 cpumask_copy(sched_domain_span(sd), nodemask);
7476 sd->parent = p; 7465 sd->parent = p;
7477 if (p) 7466 if (p)
7478 p->child = sd; 7467 p->child = sd;
@@ -7480,11 +7469,12 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7480 7469
7481#ifdef CONFIG_SCHED_MC 7470#ifdef CONFIG_SCHED_MC
7482 p = sd; 7471 p = sd;
7483 sd = &per_cpu(core_domains, i); 7472 sd = &per_cpu(core_domains, i).sd;
7484 SD_INIT(sd, MC); 7473 SD_INIT(sd, MC);
7485 set_domain_attribute(sd, attr); 7474 set_domain_attribute(sd, attr);
7486 sd->span = cpu_coregroup_map(i); 7475 *sched_domain_span(sd) = cpu_coregroup_map(i);
7487 cpus_and(sd->span, sd->span, *cpu_map); 7476 cpumask_and(sched_domain_span(sd),
7477 sched_domain_span(sd), cpu_map);
7488 sd->parent = p; 7478 sd->parent = p;
7489 p->child = sd; 7479 p->child = sd;
7490 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask); 7480 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7492,11 +7482,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7492 7482
7493#ifdef CONFIG_SCHED_SMT 7483#ifdef CONFIG_SCHED_SMT
7494 p = sd; 7484 p = sd;
7495 sd = &per_cpu(cpu_domains, i); 7485 sd = &per_cpu(cpu_domains, i).sd;
7496 SD_INIT(sd, SIBLING); 7486 SD_INIT(sd, SIBLING);
7497 set_domain_attribute(sd, attr); 7487 set_domain_attribute(sd, attr);
7498 sd->span = per_cpu(cpu_sibling_map, i); 7488 cpumask_and(sched_domain_span(sd),
7499 cpus_and(sd->span, sd->span, *cpu_map); 7489 &per_cpu(cpu_sibling_map, i), cpu_map);
7500 sd->parent = p; 7490 sd->parent = p;
7501 p->child = sd; 7491 p->child = sd;
7502 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask); 7492 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7505,13 +7495,10 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7505 7495
7506#ifdef CONFIG_SCHED_SMT 7496#ifdef CONFIG_SCHED_SMT
7507 /* Set up CPU (sibling) groups */ 7497 /* Set up CPU (sibling) groups */
7508 for_each_cpu_mask_nr(i, *cpu_map) { 7498 for_each_cpu(i, cpu_map) {
7509 SCHED_CPUMASK_VAR(this_sibling_map, allmasks); 7499 cpumask_and(this_sibling_map,
7510 SCHED_CPUMASK_VAR(send_covered, allmasks); 7500 &per_cpu(cpu_sibling_map, i), cpu_map);
7511 7501 if (i != cpumask_first(this_sibling_map))
7512 *this_sibling_map = per_cpu(cpu_sibling_map, i);
7513 cpus_and(*this_sibling_map, *this_sibling_map, *cpu_map);
7514 if (i != first_cpu(*this_sibling_map))
7515 continue; 7502 continue;
7516 7503
7517 init_sched_build_groups(this_sibling_map, cpu_map, 7504 init_sched_build_groups(this_sibling_map, cpu_map,
@@ -7522,13 +7509,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7522 7509
7523#ifdef CONFIG_SCHED_MC 7510#ifdef CONFIG_SCHED_MC
7524 /* Set up multi-core groups */ 7511 /* Set up multi-core groups */
7525 for_each_cpu_mask_nr(i, *cpu_map) { 7512 for_each_cpu(i, cpu_map) {
7526 SCHED_CPUMASK_VAR(this_core_map, allmasks); 7513 /* FIXME: Use cpu_coregroup_mask */
7527 SCHED_CPUMASK_VAR(send_covered, allmasks);
7528
7529 *this_core_map = cpu_coregroup_map(i); 7514 *this_core_map = cpu_coregroup_map(i);
7530 cpus_and(*this_core_map, *this_core_map, *cpu_map); 7515 cpus_and(*this_core_map, *this_core_map, *cpu_map);
7531 if (i != first_cpu(*this_core_map)) 7516 if (i != cpumask_first(this_core_map))
7532 continue; 7517 continue;
7533 7518
7534 init_sched_build_groups(this_core_map, cpu_map, 7519 init_sched_build_groups(this_core_map, cpu_map,
@@ -7539,12 +7524,10 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7539 7524
7540 /* Set up physical groups */ 7525 /* Set up physical groups */
7541 for (i = 0; i < nr_node_ids; i++) { 7526 for (i = 0; i < nr_node_ids; i++) {
7542 SCHED_CPUMASK_VAR(nodemask, allmasks); 7527 /* FIXME: Use cpumask_of_node */
7543 SCHED_CPUMASK_VAR(send_covered, allmasks);
7544
7545 *nodemask = node_to_cpumask(i); 7528 *nodemask = node_to_cpumask(i);
7546 cpus_and(*nodemask, *nodemask, *cpu_map); 7529 cpus_and(*nodemask, *nodemask, *cpu_map);
7547 if (cpus_empty(*nodemask)) 7530 if (cpumask_empty(nodemask))
7548 continue; 7531 continue;
7549 7532
7550 init_sched_build_groups(nodemask, cpu_map, 7533 init_sched_build_groups(nodemask, cpu_map,
@@ -7555,8 +7538,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7555#ifdef CONFIG_NUMA 7538#ifdef CONFIG_NUMA
7556 /* Set up node groups */ 7539 /* Set up node groups */
7557 if (sd_allnodes) { 7540 if (sd_allnodes) {
7558 SCHED_CPUMASK_VAR(send_covered, allmasks);
7559
7560 init_sched_build_groups(cpu_map, cpu_map, 7541 init_sched_build_groups(cpu_map, cpu_map,
7561 &cpu_to_allnodes_group, 7542 &cpu_to_allnodes_group,
7562 send_covered, tmpmask); 7543 send_covered, tmpmask);
@@ -7565,58 +7546,58 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7565 for (i = 0; i < nr_node_ids; i++) { 7546 for (i = 0; i < nr_node_ids; i++) {
7566 /* Set up node groups */ 7547 /* Set up node groups */
7567 struct sched_group *sg, *prev; 7548 struct sched_group *sg, *prev;
7568 SCHED_CPUMASK_VAR(nodemask, allmasks);
7569 SCHED_CPUMASK_VAR(domainspan, allmasks);
7570 SCHED_CPUMASK_VAR(covered, allmasks);
7571 int j; 7549 int j;
7572 7550
7551 /* FIXME: Use cpumask_of_node */
7573 *nodemask = node_to_cpumask(i); 7552 *nodemask = node_to_cpumask(i);
7574 cpus_clear(*covered); 7553 cpumask_clear(covered);
7575 7554
7576 cpus_and(*nodemask, *nodemask, *cpu_map); 7555 cpus_and(*nodemask, *nodemask, *cpu_map);
7577 if (cpus_empty(*nodemask)) { 7556 if (cpumask_empty(nodemask)) {
7578 sched_group_nodes[i] = NULL; 7557 sched_group_nodes[i] = NULL;
7579 continue; 7558 continue;
7580 } 7559 }
7581 7560
7582 sched_domain_node_span(i, domainspan); 7561 sched_domain_node_span(i, domainspan);
7583 cpus_and(*domainspan, *domainspan, *cpu_map); 7562 cpumask_and(domainspan, domainspan, cpu_map);
7584 7563
7585 sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i); 7564 sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
7565 GFP_KERNEL, i);
7586 if (!sg) { 7566 if (!sg) {
7587 printk(KERN_WARNING "Can not alloc domain group for " 7567 printk(KERN_WARNING "Can not alloc domain group for "
7588 "node %d\n", i); 7568 "node %d\n", i);
7589 goto error; 7569 goto error;
7590 } 7570 }
7591 sched_group_nodes[i] = sg; 7571 sched_group_nodes[i] = sg;
7592 for_each_cpu_mask_nr(j, *nodemask) { 7572 for_each_cpu(j, nodemask) {
7593 struct sched_domain *sd; 7573 struct sched_domain *sd;
7594 7574
7595 sd = &per_cpu(node_domains, j); 7575 sd = &per_cpu(node_domains, j);
7596 sd->groups = sg; 7576 sd->groups = sg;
7597 } 7577 }
7598 sg->__cpu_power = 0; 7578 sg->__cpu_power = 0;
7599 sg->cpumask = *nodemask; 7579 cpumask_copy(sched_group_cpus(sg), nodemask);
7600 sg->next = sg; 7580 sg->next = sg;
7601 cpus_or(*covered, *covered, *nodemask); 7581 cpumask_or(covered, covered, nodemask);
7602 prev = sg; 7582 prev = sg;
7603 7583
7604 for (j = 0; j < nr_node_ids; j++) { 7584 for (j = 0; j < nr_node_ids; j++) {
7605 SCHED_CPUMASK_VAR(notcovered, allmasks);
7606 int n = (i + j) % nr_node_ids; 7585 int n = (i + j) % nr_node_ids;
7586 /* FIXME: Use cpumask_of_node */
7607 node_to_cpumask_ptr(pnodemask, n); 7587 node_to_cpumask_ptr(pnodemask, n);
7608 7588
7609 cpus_complement(*notcovered, *covered); 7589 cpumask_complement(notcovered, covered);
7610 cpus_and(*tmpmask, *notcovered, *cpu_map); 7590 cpumask_and(tmpmask, notcovered, cpu_map);
7611 cpus_and(*tmpmask, *tmpmask, *domainspan); 7591 cpumask_and(tmpmask, tmpmask, domainspan);
7612 if (cpus_empty(*tmpmask)) 7592 if (cpumask_empty(tmpmask))
7613 break; 7593 break;
7614 7594
7615 cpus_and(*tmpmask, *tmpmask, *pnodemask); 7595 cpumask_and(tmpmask, tmpmask, pnodemask);
7616 if (cpus_empty(*tmpmask)) 7596 if (cpumask_empty(tmpmask))
7617 continue; 7597 continue;
7618 7598
7619 sg = kmalloc_node(sizeof(struct sched_group), 7599 sg = kmalloc_node(sizeof(struct sched_group) +
7600 cpumask_size(),
7620 GFP_KERNEL, i); 7601 GFP_KERNEL, i);
7621 if (!sg) { 7602 if (!sg) {
7622 printk(KERN_WARNING 7603 printk(KERN_WARNING
@@ -7624,9 +7605,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7624 goto error; 7605 goto error;
7625 } 7606 }
7626 sg->__cpu_power = 0; 7607 sg->__cpu_power = 0;
7627 sg->cpumask = *tmpmask; 7608 cpumask_copy(sched_group_cpus(sg), tmpmask);
7628 sg->next = prev->next; 7609 sg->next = prev->next;
7629 cpus_or(*covered, *covered, *tmpmask); 7610 cpumask_or(covered, covered, tmpmask);
7630 prev->next = sg; 7611 prev->next = sg;
7631 prev = sg; 7612 prev = sg;
7632 } 7613 }
@@ -7635,22 +7616,22 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7635 7616
7636 /* Calculate CPU power for physical packages and nodes */ 7617 /* Calculate CPU power for physical packages and nodes */
7637#ifdef CONFIG_SCHED_SMT 7618#ifdef CONFIG_SCHED_SMT
7638 for_each_cpu_mask_nr(i, *cpu_map) { 7619 for_each_cpu(i, cpu_map) {
7639 struct sched_domain *sd = &per_cpu(cpu_domains, i); 7620 struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
7640 7621
7641 init_sched_groups_power(i, sd); 7622 init_sched_groups_power(i, sd);
7642 } 7623 }
7643#endif 7624#endif
7644#ifdef CONFIG_SCHED_MC 7625#ifdef CONFIG_SCHED_MC
7645 for_each_cpu_mask_nr(i, *cpu_map) { 7626 for_each_cpu(i, cpu_map) {
7646 struct sched_domain *sd = &per_cpu(core_domains, i); 7627 struct sched_domain *sd = &per_cpu(core_domains, i).sd;
7647 7628
7648 init_sched_groups_power(i, sd); 7629 init_sched_groups_power(i, sd);
7649 } 7630 }
7650#endif 7631#endif
7651 7632
7652 for_each_cpu_mask_nr(i, *cpu_map) { 7633 for_each_cpu(i, cpu_map) {
7653 struct sched_domain *sd = &per_cpu(phys_domains, i); 7634 struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
7654 7635
7655 init_sched_groups_power(i, sd); 7636 init_sched_groups_power(i, sd);
7656 } 7637 }
@@ -7662,53 +7643,78 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7662 if (sd_allnodes) { 7643 if (sd_allnodes) {
7663 struct sched_group *sg; 7644 struct sched_group *sg;
7664 7645
7665 cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg, 7646 cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
7666 tmpmask); 7647 tmpmask);
7667 init_numa_sched_groups_power(sg); 7648 init_numa_sched_groups_power(sg);
7668 } 7649 }
7669#endif 7650#endif
7670 7651
7671 /* Attach the domains */ 7652 /* Attach the domains */
7672 for_each_cpu_mask_nr(i, *cpu_map) { 7653 for_each_cpu(i, cpu_map) {
7673 struct sched_domain *sd; 7654 struct sched_domain *sd;
7674#ifdef CONFIG_SCHED_SMT 7655#ifdef CONFIG_SCHED_SMT
7675 sd = &per_cpu(cpu_domains, i); 7656 sd = &per_cpu(cpu_domains, i).sd;
7676#elif defined(CONFIG_SCHED_MC) 7657#elif defined(CONFIG_SCHED_MC)
7677 sd = &per_cpu(core_domains, i); 7658 sd = &per_cpu(core_domains, i).sd;
7678#else 7659#else
7679 sd = &per_cpu(phys_domains, i); 7660 sd = &per_cpu(phys_domains, i).sd;
7680#endif 7661#endif
7681 cpu_attach_domain(sd, rd, i); 7662 cpu_attach_domain(sd, rd, i);
7682 } 7663 }
7683 7664
7684 SCHED_CPUMASK_FREE((void *)allmasks); 7665 err = 0;
7685 return 0; 7666
7667free_tmpmask:
7668 free_cpumask_var(tmpmask);
7669free_send_covered:
7670 free_cpumask_var(send_covered);
7671free_this_core_map:
7672 free_cpumask_var(this_core_map);
7673free_this_sibling_map:
7674 free_cpumask_var(this_sibling_map);
7675free_nodemask:
7676 free_cpumask_var(nodemask);
7677free_notcovered:
7678#ifdef CONFIG_NUMA
7679 free_cpumask_var(notcovered);
7680free_covered:
7681 free_cpumask_var(covered);
7682free_domainspan:
7683 free_cpumask_var(domainspan);
7684out:
7685#endif
7686 return err;
7687
7688free_sched_groups:
7689#ifdef CONFIG_NUMA
7690 kfree(sched_group_nodes);
7691#endif
7692 goto free_tmpmask;
7686 7693
7687#ifdef CONFIG_NUMA 7694#ifdef CONFIG_NUMA
7688error: 7695error:
7689 free_sched_groups(cpu_map, tmpmask); 7696 free_sched_groups(cpu_map, tmpmask);
7690 SCHED_CPUMASK_FREE((void *)allmasks); 7697 free_rootdomain(rd);
7691 kfree(rd); 7698 goto free_tmpmask;
7692 return -ENOMEM;
7693#endif 7699#endif
7694} 7700}
7695 7701
7696static int build_sched_domains(const cpumask_t *cpu_map) 7702static int build_sched_domains(const struct cpumask *cpu_map)
7697{ 7703{
7698 return __build_sched_domains(cpu_map, NULL); 7704 return __build_sched_domains(cpu_map, NULL);
7699} 7705}
7700 7706
7701static cpumask_t *doms_cur; /* current sched domains */ 7707static struct cpumask *doms_cur; /* current sched domains */
7702static int ndoms_cur; /* number of sched domains in 'doms_cur' */ 7708static int ndoms_cur; /* number of sched domains in 'doms_cur' */
7703static struct sched_domain_attr *dattr_cur; 7709static struct sched_domain_attr *dattr_cur;
7704 /* attribues of custom domains in 'doms_cur' */ 7710 /* attribues of custom domains in 'doms_cur' */
7705 7711
7706/* 7712/*
7707 * Special case: If a kmalloc of a doms_cur partition (array of 7713 * Special case: If a kmalloc of a doms_cur partition (array of
7708 * cpumask_t) fails, then fallback to a single sched domain, 7714 * cpumask) fails, then fallback to a single sched domain,
7709 * as determined by the single cpumask_t fallback_doms. 7715 * as determined by the single cpumask fallback_doms.
7710 */ 7716 */
7711static cpumask_t fallback_doms; 7717static cpumask_var_t fallback_doms;
7712 7718
7713void __attribute__((weak)) arch_update_cpu_topology(void) 7719void __attribute__((weak)) arch_update_cpu_topology(void)
7714{ 7720{
@@ -7719,16 +7725,16 @@ void __attribute__((weak)) arch_update_cpu_topology(void)
7719 * For now this just excludes isolated cpus, but could be used to 7725 * For now this just excludes isolated cpus, but could be used to
7720 * exclude other special cases in the future. 7726 * exclude other special cases in the future.
7721 */ 7727 */
7722static int arch_init_sched_domains(const cpumask_t *cpu_map) 7728static int arch_init_sched_domains(const struct cpumask *cpu_map)
7723{ 7729{
7724 int err; 7730 int err;
7725 7731
7726 arch_update_cpu_topology(); 7732 arch_update_cpu_topology();
7727 ndoms_cur = 1; 7733 ndoms_cur = 1;
7728 doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 7734 doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
7729 if (!doms_cur) 7735 if (!doms_cur)
7730 doms_cur = &fallback_doms; 7736 doms_cur = fallback_doms;
7731 cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map); 7737 cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
7732 dattr_cur = NULL; 7738 dattr_cur = NULL;
7733 err = build_sched_domains(doms_cur); 7739 err = build_sched_domains(doms_cur);
7734 register_sched_domain_sysctl(); 7740 register_sched_domain_sysctl();
@@ -7736,8 +7742,8 @@ static int arch_init_sched_domains(const cpumask_t *cpu_map)
7736 return err; 7742 return err;
7737} 7743}
7738 7744
7739static void arch_destroy_sched_domains(const cpumask_t *cpu_map, 7745static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
7740 cpumask_t *tmpmask) 7746 struct cpumask *tmpmask)
7741{ 7747{
7742 free_sched_groups(cpu_map, tmpmask); 7748 free_sched_groups(cpu_map, tmpmask);
7743} 7749}
@@ -7746,17 +7752,16 @@ static void arch_destroy_sched_domains(const cpumask_t *cpu_map,
7746 * Detach sched domains from a group of cpus specified in cpu_map 7752 * Detach sched domains from a group of cpus specified in cpu_map
7747 * These cpus will now be attached to the NULL domain 7753 * These cpus will now be attached to the NULL domain
7748 */ 7754 */
7749static void detach_destroy_domains(const cpumask_t *cpu_map) 7755static void detach_destroy_domains(const struct cpumask *cpu_map)
7750{ 7756{
7751 cpumask_t tmpmask; 7757 /* Save because hotplug lock held. */
7758 static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
7752 int i; 7759 int i;
7753 7760
7754 unregister_sched_domain_sysctl(); 7761 for_each_cpu(i, cpu_map)
7755
7756 for_each_cpu_mask_nr(i, *cpu_map)
7757 cpu_attach_domain(NULL, &def_root_domain, i); 7762 cpu_attach_domain(NULL, &def_root_domain, i);
7758 synchronize_sched(); 7763 synchronize_sched();
7759 arch_destroy_sched_domains(cpu_map, &tmpmask); 7764 arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
7760} 7765}
7761 7766
7762/* handle null as "default" */ 7767/* handle null as "default" */
@@ -7781,7 +7786,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7781 * doms_new[] to the current sched domain partitioning, doms_cur[]. 7786 * doms_new[] to the current sched domain partitioning, doms_cur[].
7782 * It destroys each deleted domain and builds each new domain. 7787 * It destroys each deleted domain and builds each new domain.
7783 * 7788 *
7784 * 'doms_new' is an array of cpumask_t's of length 'ndoms_new'. 7789 * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
7785 * The masks don't intersect (don't overlap.) We should setup one 7790 * The masks don't intersect (don't overlap.) We should setup one
7786 * sched domain for each mask. CPUs not in any of the cpumasks will 7791 * sched domain for each mask. CPUs not in any of the cpumasks will
7787 * not be load balanced. If the same cpumask appears both in the 7792 * not be load balanced. If the same cpumask appears both in the
@@ -7795,13 +7800,14 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7795 * the single partition 'fallback_doms', it also forces the domains 7800 * the single partition 'fallback_doms', it also forces the domains
7796 * to be rebuilt. 7801 * to be rebuilt.
7797 * 7802 *
7798 * If doms_new == NULL it will be replaced with cpu_online_map. 7803 * If doms_new == NULL it will be replaced with cpu_online_mask.
7799 * ndoms_new == 0 is a special case for destroying existing domains, 7804 * ndoms_new == 0 is a special case for destroying existing domains,
7800 * and it will not create the default domain. 7805 * and it will not create the default domain.
7801 * 7806 *
7802 * Call with hotplug lock held 7807 * Call with hotplug lock held
7803 */ 7808 */
7804void partition_sched_domains(int ndoms_new, cpumask_t *doms_new, 7809/* FIXME: Change to struct cpumask *doms_new[] */
7810void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
7805 struct sched_domain_attr *dattr_new) 7811 struct sched_domain_attr *dattr_new)
7806{ 7812{
7807 int i, j, n; 7813 int i, j, n;
@@ -7816,7 +7822,7 @@ void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
7816 /* Destroy deleted domains */ 7822 /* Destroy deleted domains */
7817 for (i = 0; i < ndoms_cur; i++) { 7823 for (i = 0; i < ndoms_cur; i++) {
7818 for (j = 0; j < n; j++) { 7824 for (j = 0; j < n; j++) {
7819 if (cpus_equal(doms_cur[i], doms_new[j]) 7825 if (cpumask_equal(&doms_cur[i], &doms_new[j])
7820 && dattrs_equal(dattr_cur, i, dattr_new, j)) 7826 && dattrs_equal(dattr_cur, i, dattr_new, j))
7821 goto match1; 7827 goto match1;
7822 } 7828 }
@@ -7828,15 +7834,15 @@ match1:
7828 7834
7829 if (doms_new == NULL) { 7835 if (doms_new == NULL) {
7830 ndoms_cur = 0; 7836 ndoms_cur = 0;
7831 doms_new = &fallback_doms; 7837 doms_new = fallback_doms;
7832 cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map); 7838 cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
7833 dattr_new = NULL; 7839 WARN_ON_ONCE(dattr_new);
7834 } 7840 }
7835 7841
7836 /* Build new domains */ 7842 /* Build new domains */
7837 for (i = 0; i < ndoms_new; i++) { 7843 for (i = 0; i < ndoms_new; i++) {
7838 for (j = 0; j < ndoms_cur; j++) { 7844 for (j = 0; j < ndoms_cur; j++) {
7839 if (cpus_equal(doms_new[i], doms_cur[j]) 7845 if (cpumask_equal(&doms_new[i], &doms_cur[j])
7840 && dattrs_equal(dattr_new, i, dattr_cur, j)) 7846 && dattrs_equal(dattr_new, i, dattr_cur, j))
7841 goto match2; 7847 goto match2;
7842 } 7848 }
@@ -7848,7 +7854,7 @@ match2:
7848 } 7854 }
7849 7855
7850 /* Remember the new sched domains */ 7856 /* Remember the new sched domains */
7851 if (doms_cur != &fallback_doms) 7857 if (doms_cur != fallback_doms)
7852 kfree(doms_cur); 7858 kfree(doms_cur);
7853 kfree(dattr_cur); /* kfree(NULL) is safe */ 7859 kfree(dattr_cur); /* kfree(NULL) is safe */
7854 doms_cur = doms_new; 7860 doms_cur = doms_new;
@@ -7988,7 +7994,9 @@ static int update_runtime(struct notifier_block *nfb,
7988 7994
7989void __init sched_init_smp(void) 7995void __init sched_init_smp(void)
7990{ 7996{
7991 cpumask_t non_isolated_cpus; 7997 cpumask_var_t non_isolated_cpus;
7998
7999 alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
7992 8000
7993#if defined(CONFIG_NUMA) 8001#if defined(CONFIG_NUMA)
7994 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **), 8002 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
@@ -7997,10 +8005,10 @@ void __init sched_init_smp(void)
7997#endif 8005#endif
7998 get_online_cpus(); 8006 get_online_cpus();
7999 mutex_lock(&sched_domains_mutex); 8007 mutex_lock(&sched_domains_mutex);
8000 arch_init_sched_domains(&cpu_online_map); 8008 arch_init_sched_domains(cpu_online_mask);
8001 cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map); 8009 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
8002 if (cpus_empty(non_isolated_cpus)) 8010 if (cpumask_empty(non_isolated_cpus))
8003 cpu_set(smp_processor_id(), non_isolated_cpus); 8011 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
8004 mutex_unlock(&sched_domains_mutex); 8012 mutex_unlock(&sched_domains_mutex);
8005 put_online_cpus(); 8013 put_online_cpus();
8006 8014
@@ -8015,9 +8023,13 @@ void __init sched_init_smp(void)
8015 init_hrtick(); 8023 init_hrtick();
8016 8024
8017 /* Move init over to a non-isolated CPU */ 8025 /* Move init over to a non-isolated CPU */
8018 if (set_cpus_allowed_ptr(current, &non_isolated_cpus) < 0) 8026 if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
8019 BUG(); 8027 BUG();
8020 sched_init_granularity(); 8028 sched_init_granularity();
8029 free_cpumask_var(non_isolated_cpus);
8030
8031 alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
8032 init_sched_rt_class();
8021} 8033}
8022#else 8034#else
8023void __init sched_init_smp(void) 8035void __init sched_init_smp(void)
@@ -8332,6 +8344,15 @@ void __init sched_init(void)
8332 */ 8344 */
8333 current->sched_class = &fair_sched_class; 8345 current->sched_class = &fair_sched_class;
8334 8346
8347 /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
8348 alloc_bootmem_cpumask_var(&nohz_cpu_mask);
8349#ifdef CONFIG_SMP
8350#ifdef CONFIG_NO_HZ
8351 alloc_bootmem_cpumask_var(&nohz.cpu_mask);
8352#endif
8353 alloc_bootmem_cpumask_var(&cpu_isolated_map);
8354#endif /* SMP */
8355
8335 scheduler_running = 1; 8356 scheduler_running = 1;
8336} 8357}
8337 8358
@@ -8490,7 +8511,7 @@ static
8490int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 8511int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
8491{ 8512{
8492 struct cfs_rq *cfs_rq; 8513 struct cfs_rq *cfs_rq;
8493 struct sched_entity *se, *parent_se; 8514 struct sched_entity *se;
8494 struct rq *rq; 8515 struct rq *rq;
8495 int i; 8516 int i;
8496 8517
@@ -8506,18 +8527,17 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
8506 for_each_possible_cpu(i) { 8527 for_each_possible_cpu(i) {
8507 rq = cpu_rq(i); 8528 rq = cpu_rq(i);
8508 8529
8509 cfs_rq = kmalloc_node(sizeof(struct cfs_rq), 8530 cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
8510 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8531 GFP_KERNEL, cpu_to_node(i));
8511 if (!cfs_rq) 8532 if (!cfs_rq)
8512 goto err; 8533 goto err;
8513 8534
8514 se = kmalloc_node(sizeof(struct sched_entity), 8535 se = kzalloc_node(sizeof(struct sched_entity),
8515 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8536 GFP_KERNEL, cpu_to_node(i));
8516 if (!se) 8537 if (!se)
8517 goto err; 8538 goto err;
8518 8539
8519 parent_se = parent ? parent->se[i] : NULL; 8540 init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
8520 init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent_se);
8521 } 8541 }
8522 8542
8523 return 1; 8543 return 1;
@@ -8578,7 +8598,7 @@ static
8578int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) 8598int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8579{ 8599{
8580 struct rt_rq *rt_rq; 8600 struct rt_rq *rt_rq;
8581 struct sched_rt_entity *rt_se, *parent_se; 8601 struct sched_rt_entity *rt_se;
8582 struct rq *rq; 8602 struct rq *rq;
8583 int i; 8603 int i;
8584 8604
@@ -8595,18 +8615,17 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8595 for_each_possible_cpu(i) { 8615 for_each_possible_cpu(i) {
8596 rq = cpu_rq(i); 8616 rq = cpu_rq(i);
8597 8617
8598 rt_rq = kmalloc_node(sizeof(struct rt_rq), 8618 rt_rq = kzalloc_node(sizeof(struct rt_rq),
8599 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8619 GFP_KERNEL, cpu_to_node(i));
8600 if (!rt_rq) 8620 if (!rt_rq)
8601 goto err; 8621 goto err;
8602 8622
8603 rt_se = kmalloc_node(sizeof(struct sched_rt_entity), 8623 rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
8604 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8624 GFP_KERNEL, cpu_to_node(i));
8605 if (!rt_se) 8625 if (!rt_se)
8606 goto err; 8626 goto err;
8607 8627
8608 parent_se = parent ? parent->rt_se[i] : NULL; 8628 init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
8609 init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent_se);
8610 } 8629 }
8611 8630
8612 return 1; 8631 return 1;
@@ -9249,11 +9268,12 @@ struct cgroup_subsys cpu_cgroup_subsys = {
9249 * (balbir@in.ibm.com). 9268 * (balbir@in.ibm.com).
9250 */ 9269 */
9251 9270
9252/* track cpu usage of a group of tasks */ 9271/* track cpu usage of a group of tasks and its child groups */
9253struct cpuacct { 9272struct cpuacct {
9254 struct cgroup_subsys_state css; 9273 struct cgroup_subsys_state css;
9255 /* cpuusage holds pointer to a u64-type object on every cpu */ 9274 /* cpuusage holds pointer to a u64-type object on every cpu */
9256 u64 *cpuusage; 9275 u64 *cpuusage;
9276 struct cpuacct *parent;
9257}; 9277};
9258 9278
9259struct cgroup_subsys cpuacct_subsys; 9279struct cgroup_subsys cpuacct_subsys;
@@ -9287,6 +9307,9 @@ static struct cgroup_subsys_state *cpuacct_create(
9287 return ERR_PTR(-ENOMEM); 9307 return ERR_PTR(-ENOMEM);
9288 } 9308 }
9289 9309
9310 if (cgrp->parent)
9311 ca->parent = cgroup_ca(cgrp->parent);
9312
9290 return &ca->css; 9313 return &ca->css;
9291} 9314}
9292 9315
@@ -9366,14 +9389,16 @@ static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
9366static void cpuacct_charge(struct task_struct *tsk, u64 cputime) 9389static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
9367{ 9390{
9368 struct cpuacct *ca; 9391 struct cpuacct *ca;
9392 int cpu;
9369 9393
9370 if (!cpuacct_subsys.active) 9394 if (!cpuacct_subsys.active)
9371 return; 9395 return;
9372 9396
9397 cpu = task_cpu(tsk);
9373 ca = task_ca(tsk); 9398 ca = task_ca(tsk);
9374 if (ca) {
9375 u64 *cpuusage = percpu_ptr(ca->cpuusage, task_cpu(tsk));
9376 9399
9400 for (; ca; ca = ca->parent) {
9401 u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
9377 *cpuusage += cputime; 9402 *cpuusage += cputime;
9378 } 9403 }
9379} 9404}
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-11 17:40:53 -0500