aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched.c')
-rw-r--r--kernel/sched.c1181
1 files changed, 621 insertions, 560 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index 57c933ffbee1..e00c92d22655 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -118,6 +118,12 @@
118 */ 118 */
119#define RUNTIME_INF ((u64)~0ULL) 119#define RUNTIME_INF ((u64)~0ULL)
120 120
121DEFINE_TRACE(sched_wait_task);
122DEFINE_TRACE(sched_wakeup);
123DEFINE_TRACE(sched_wakeup_new);
124DEFINE_TRACE(sched_switch);
125DEFINE_TRACE(sched_migrate_task);
126
121#ifdef CONFIG_SMP 127#ifdef CONFIG_SMP
122/* 128/*
123 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power) 129 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
@@ -261,6 +267,10 @@ struct task_group {
261 struct cgroup_subsys_state css; 267 struct cgroup_subsys_state css;
262#endif 268#endif
263 269
270#ifdef CONFIG_USER_SCHED
271 uid_t uid;
272#endif
273
264#ifdef CONFIG_FAIR_GROUP_SCHED 274#ifdef CONFIG_FAIR_GROUP_SCHED
265 /* schedulable entities of this group on each cpu */ 275 /* schedulable entities of this group on each cpu */
266 struct sched_entity **se; 276 struct sched_entity **se;
@@ -286,6 +296,12 @@ struct task_group {
286 296
287#ifdef CONFIG_USER_SCHED 297#ifdef CONFIG_USER_SCHED
288 298
299/* Helper function to pass uid information to create_sched_user() */
300void set_tg_uid(struct user_struct *user)
301{
302 user->tg->uid = user->uid;
303}
304
289/* 305/*
290 * Root task group. 306 * Root task group.
291 * Every UID task group (including init_task_group aka UID-0) will 307 * Every UID task group (including init_task_group aka UID-0) will
@@ -399,7 +415,7 @@ struct cfs_rq {
399 */ 415 */
400 struct sched_entity *curr, *next, *last; 416 struct sched_entity *curr, *next, *last;
401 417
402 unsigned long nr_spread_over; 418 unsigned int nr_spread_over;
403 419
404#ifdef CONFIG_FAIR_GROUP_SCHED 420#ifdef CONFIG_FAIR_GROUP_SCHED
405 struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */ 421 struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */
@@ -481,14 +497,14 @@ struct rt_rq {
481 */ 497 */
482struct root_domain { 498struct root_domain {
483 atomic_t refcount; 499 atomic_t refcount;
484 cpumask_t span; 500 cpumask_var_t span;
485 cpumask_t online; 501 cpumask_var_t online;
486 502
487 /* 503 /*
488 * The "RT overload" flag: it gets set if a CPU has more than 504 * The "RT overload" flag: it gets set if a CPU has more than
489 * one runnable RT task. 505 * one runnable RT task.
490 */ 506 */
491 cpumask_t rto_mask; 507 cpumask_var_t rto_mask;
492 atomic_t rto_count; 508 atomic_t rto_count;
493#ifdef CONFIG_SMP 509#ifdef CONFIG_SMP
494 struct cpupri cpupri; 510 struct cpupri cpupri;
@@ -703,45 +719,18 @@ static __read_mostly char *sched_feat_names[] = {
703 719
704#undef SCHED_FEAT 720#undef SCHED_FEAT
705 721
706static int sched_feat_open(struct inode *inode, struct file *filp) 722static int sched_feat_show(struct seq_file *m, void *v)
707{
708 filp->private_data = inode->i_private;
709 return 0;
710}
711
712static ssize_t
713sched_feat_read(struct file *filp, char __user *ubuf,
714 size_t cnt, loff_t *ppos)
715{ 723{
716 char *buf;
717 int r = 0;
718 int len = 0;
719 int i; 724 int i;
720 725
721 for (i = 0; sched_feat_names[i]; i++) { 726 for (i = 0; sched_feat_names[i]; i++) {
722 len += strlen(sched_feat_names[i]); 727 if (!(sysctl_sched_features & (1UL << i)))
723 len += 4; 728 seq_puts(m, "NO_");
729 seq_printf(m, "%s ", sched_feat_names[i]);
724 } 730 }
731 seq_puts(m, "\n");
725 732
726 buf = kmalloc(len + 2, GFP_KERNEL); 733 return 0;
727 if (!buf)
728 return -ENOMEM;
729
730 for (i = 0; sched_feat_names[i]; i++) {
731 if (sysctl_sched_features & (1UL << i))
732 r += sprintf(buf + r, "%s ", sched_feat_names[i]);
733 else
734 r += sprintf(buf + r, "NO_%s ", sched_feat_names[i]);
735 }
736
737 r += sprintf(buf + r, "\n");
738 WARN_ON(r >= len + 2);
739
740 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
741
742 kfree(buf);
743
744 return r;
745} 734}
746 735
747static ssize_t 736static ssize_t
@@ -786,10 +775,17 @@ sched_feat_write(struct file *filp, const char __user *ubuf,
786 return cnt; 775 return cnt;
787} 776}
788 777
778static int sched_feat_open(struct inode *inode, struct file *filp)
779{
780 return single_open(filp, sched_feat_show, NULL);
781}
782
789static struct file_operations sched_feat_fops = { 783static struct file_operations sched_feat_fops = {
790 .open = sched_feat_open, 784 .open = sched_feat_open,
791 .read = sched_feat_read, 785 .write = sched_feat_write,
792 .write = sched_feat_write, 786 .read = seq_read,
787 .llseek = seq_lseek,
788 .release = single_release,
793}; 789};
794 790
795static __init int sched_init_debug(void) 791static __init int sched_init_debug(void)
@@ -969,6 +965,14 @@ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
969 } 965 }
970} 966}
971 967
968void task_rq_unlock_wait(struct task_struct *p)
969{
970 struct rq *rq = task_rq(p);
971
972 smp_mb(); /* spin-unlock-wait is not a full memory barrier */
973 spin_unlock_wait(&rq->lock);
974}
975
972static void __task_rq_unlock(struct rq *rq) 976static void __task_rq_unlock(struct rq *rq)
973 __releases(rq->lock) 977 __releases(rq->lock)
974{ 978{
@@ -1445,9 +1449,12 @@ static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
1445static unsigned long cpu_avg_load_per_task(int cpu) 1449static unsigned long cpu_avg_load_per_task(int cpu)
1446{ 1450{
1447 struct rq *rq = cpu_rq(cpu); 1451 struct rq *rq = cpu_rq(cpu);
1452 unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
1448 1453
1449 if (rq->nr_running) 1454 if (nr_running)
1450 rq->avg_load_per_task = rq->load.weight / rq->nr_running; 1455 rq->avg_load_per_task = rq->load.weight / nr_running;
1456 else
1457 rq->avg_load_per_task = 0;
1451 1458
1452 return rq->avg_load_per_task; 1459 return rq->avg_load_per_task;
1453} 1460}
@@ -1463,27 +1470,13 @@ static void
1463update_group_shares_cpu(struct task_group *tg, int cpu, 1470update_group_shares_cpu(struct task_group *tg, int cpu,
1464 unsigned long sd_shares, unsigned long sd_rq_weight) 1471 unsigned long sd_shares, unsigned long sd_rq_weight)
1465{ 1472{
1466 int boost = 0;
1467 unsigned long shares; 1473 unsigned long shares;
1468 unsigned long rq_weight; 1474 unsigned long rq_weight;
1469 1475
1470 if (!tg->se[cpu]) 1476 if (!tg->se[cpu])
1471 return; 1477 return;
1472 1478
1473 rq_weight = tg->cfs_rq[cpu]->load.weight; 1479 rq_weight = tg->cfs_rq[cpu]->rq_weight;
1474
1475 /*
1476 * If there are currently no tasks on the cpu pretend there is one of
1477 * average load so that when a new task gets to run here it will not
1478 * get delayed by group starvation.
1479 */
1480 if (!rq_weight) {
1481 boost = 1;
1482 rq_weight = NICE_0_LOAD;
1483 }
1484
1485 if (unlikely(rq_weight > sd_rq_weight))
1486 rq_weight = sd_rq_weight;
1487 1480
1488 /* 1481 /*
1489 * \Sum shares * rq_weight 1482 * \Sum shares * rq_weight
@@ -1491,7 +1484,7 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
1491 * \Sum rq_weight 1484 * \Sum rq_weight
1492 * 1485 *
1493 */ 1486 */
1494 shares = (sd_shares * rq_weight) / (sd_rq_weight + 1); 1487 shares = (sd_shares * rq_weight) / sd_rq_weight;
1495 shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES); 1488 shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
1496 1489
1497 if (abs(shares - tg->se[cpu]->load.weight) > 1490 if (abs(shares - tg->se[cpu]->load.weight) >
@@ -1500,11 +1493,7 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
1500 unsigned long flags; 1493 unsigned long flags;
1501 1494
1502 spin_lock_irqsave(&rq->lock, flags); 1495 spin_lock_irqsave(&rq->lock, flags);
1503 /* 1496 tg->cfs_rq[cpu]->shares = shares;
1504 * record the actual number of shares, not the boosted amount.
1505 */
1506 tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
1507 tg->cfs_rq[cpu]->rq_weight = rq_weight;
1508 1497
1509 __set_se_shares(tg->se[cpu], shares); 1498 __set_se_shares(tg->se[cpu], shares);
1510 spin_unlock_irqrestore(&rq->lock, flags); 1499 spin_unlock_irqrestore(&rq->lock, flags);
@@ -1518,13 +1507,23 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
1518 */ 1507 */
1519static int tg_shares_up(struct task_group *tg, void *data) 1508static int tg_shares_up(struct task_group *tg, void *data)
1520{ 1509{
1521 unsigned long rq_weight = 0; 1510 unsigned long weight, rq_weight = 0;
1522 unsigned long shares = 0; 1511 unsigned long shares = 0;
1523 struct sched_domain *sd = data; 1512 struct sched_domain *sd = data;
1524 int i; 1513 int i;
1525 1514
1526 for_each_cpu_mask(i, sd->span) { 1515 for_each_cpu(i, sched_domain_span(sd)) {
1527 rq_weight += tg->cfs_rq[i]->load.weight; 1516 /*
1517 * If there are currently no tasks on the cpu pretend there
1518 * is one of average load so that when a new task gets to
1519 * run here it will not get delayed by group starvation.
1520 */
1521 weight = tg->cfs_rq[i]->load.weight;
1522 if (!weight)
1523 weight = NICE_0_LOAD;
1524
1525 tg->cfs_rq[i]->rq_weight = weight;
1526 rq_weight += weight;
1528 shares += tg->cfs_rq[i]->shares; 1527 shares += tg->cfs_rq[i]->shares;
1529 } 1528 }
1530 1529
@@ -1534,10 +1533,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
1534 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE)) 1533 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
1535 shares = tg->shares; 1534 shares = tg->shares;
1536 1535
1537 if (!rq_weight) 1536 for_each_cpu(i, sched_domain_span(sd))
1538 rq_weight = cpus_weight(sd->span) * NICE_0_LOAD;
1539
1540 for_each_cpu_mask(i, sd->span)
1541 update_group_shares_cpu(tg, i, shares, rq_weight); 1537 update_group_shares_cpu(tg, i, shares, rq_weight);
1542 1538
1543 return 0; 1539 return 0;
@@ -1601,6 +1597,39 @@ static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
1601 1597
1602#endif 1598#endif
1603 1599
1600/*
1601 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
1602 */
1603static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
1604 __releases(this_rq->lock)
1605 __acquires(busiest->lock)
1606 __acquires(this_rq->lock)
1607{
1608 int ret = 0;
1609
1610 if (unlikely(!irqs_disabled())) {
1611 /* printk() doesn't work good under rq->lock */
1612 spin_unlock(&this_rq->lock);
1613 BUG_ON(1);
1614 }
1615 if (unlikely(!spin_trylock(&busiest->lock))) {
1616 if (busiest < this_rq) {
1617 spin_unlock(&this_rq->lock);
1618 spin_lock(&busiest->lock);
1619 spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
1620 ret = 1;
1621 } else
1622 spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
1623 }
1624 return ret;
1625}
1626
1627static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
1628 __releases(busiest->lock)
1629{
1630 spin_unlock(&busiest->lock);
1631 lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
1632}
1604#endif 1633#endif
1605 1634
1606#ifdef CONFIG_FAIR_GROUP_SCHED 1635#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -2068,15 +2097,17 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2068 int i; 2097 int i;
2069 2098
2070 /* Skip over this group if it has no CPUs allowed */ 2099 /* Skip over this group if it has no CPUs allowed */
2071 if (!cpus_intersects(group->cpumask, p->cpus_allowed)) 2100 if (!cpumask_intersects(sched_group_cpus(group),
2101 &p->cpus_allowed))
2072 continue; 2102 continue;
2073 2103
2074 local_group = cpu_isset(this_cpu, group->cpumask); 2104 local_group = cpumask_test_cpu(this_cpu,
2105 sched_group_cpus(group));
2075 2106
2076 /* Tally up the load of all CPUs in the group */ 2107 /* Tally up the load of all CPUs in the group */
2077 avg_load = 0; 2108 avg_load = 0;
2078 2109
2079 for_each_cpu_mask_nr(i, group->cpumask) { 2110 for_each_cpu(i, sched_group_cpus(group)) {
2080 /* Bias balancing toward cpus of our domain */ 2111 /* Bias balancing toward cpus of our domain */
2081 if (local_group) 2112 if (local_group)
2082 load = source_load(i, load_idx); 2113 load = source_load(i, load_idx);
@@ -2108,17 +2139,14 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2108 * find_idlest_cpu - find the idlest cpu among the cpus in group. 2139 * find_idlest_cpu - find the idlest cpu among the cpus in group.
2109 */ 2140 */
2110static int 2141static int
2111find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu, 2142find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
2112 cpumask_t *tmp)
2113{ 2143{
2114 unsigned long load, min_load = ULONG_MAX; 2144 unsigned long load, min_load = ULONG_MAX;
2115 int idlest = -1; 2145 int idlest = -1;
2116 int i; 2146 int i;
2117 2147
2118 /* Traverse only the allowed CPUs */ 2148 /* Traverse only the allowed CPUs */
2119 cpus_and(*tmp, group->cpumask, p->cpus_allowed); 2149 for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
2120
2121 for_each_cpu_mask_nr(i, *tmp) {
2122 load = weighted_cpuload(i); 2150 load = weighted_cpuload(i);
2123 2151
2124 if (load < min_load || (load == min_load && i == this_cpu)) { 2152 if (load < min_load || (load == min_load && i == this_cpu)) {
@@ -2160,7 +2188,6 @@ static int sched_balance_self(int cpu, int flag)
2160 update_shares(sd); 2188 update_shares(sd);
2161 2189
2162 while (sd) { 2190 while (sd) {
2163 cpumask_t span, tmpmask;
2164 struct sched_group *group; 2191 struct sched_group *group;
2165 int new_cpu, weight; 2192 int new_cpu, weight;
2166 2193
@@ -2169,14 +2196,13 @@ static int sched_balance_self(int cpu, int flag)
2169 continue; 2196 continue;
2170 } 2197 }
2171 2198
2172 span = sd->span;
2173 group = find_idlest_group(sd, t, cpu); 2199 group = find_idlest_group(sd, t, cpu);
2174 if (!group) { 2200 if (!group) {
2175 sd = sd->child; 2201 sd = sd->child;
2176 continue; 2202 continue;
2177 } 2203 }
2178 2204
2179 new_cpu = find_idlest_cpu(group, t, cpu, &tmpmask); 2205 new_cpu = find_idlest_cpu(group, t, cpu);
2180 if (new_cpu == -1 || new_cpu == cpu) { 2206 if (new_cpu == -1 || new_cpu == cpu) {
2181 /* Now try balancing at a lower domain level of cpu */ 2207 /* Now try balancing at a lower domain level of cpu */
2182 sd = sd->child; 2208 sd = sd->child;
@@ -2185,10 +2211,10 @@ static int sched_balance_self(int cpu, int flag)
2185 2211
2186 /* Now try balancing at a lower domain level of new_cpu */ 2212 /* Now try balancing at a lower domain level of new_cpu */
2187 cpu = new_cpu; 2213 cpu = new_cpu;
2214 weight = cpumask_weight(sched_domain_span(sd));
2188 sd = NULL; 2215 sd = NULL;
2189 weight = cpus_weight(span);
2190 for_each_domain(cpu, tmp) { 2216 for_each_domain(cpu, tmp) {
2191 if (weight <= cpus_weight(tmp->span)) 2217 if (weight <= cpumask_weight(sched_domain_span(tmp)))
2192 break; 2218 break;
2193 if (tmp->flags & flag) 2219 if (tmp->flags & flag)
2194 sd = tmp; 2220 sd = tmp;
@@ -2233,7 +2259,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2233 cpu = task_cpu(p); 2259 cpu = task_cpu(p);
2234 2260
2235 for_each_domain(this_cpu, sd) { 2261 for_each_domain(this_cpu, sd) {
2236 if (cpu_isset(cpu, sd->span)) { 2262 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2237 update_shares(sd); 2263 update_shares(sd);
2238 break; 2264 break;
2239 } 2265 }
@@ -2281,7 +2307,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2281 else { 2307 else {
2282 struct sched_domain *sd; 2308 struct sched_domain *sd;
2283 for_each_domain(this_cpu, sd) { 2309 for_each_domain(this_cpu, sd) {
2284 if (cpu_isset(cpu, sd->span)) { 2310 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2285 schedstat_inc(sd, ttwu_wake_remote); 2311 schedstat_inc(sd, ttwu_wake_remote);
2286 break; 2312 break;
2287 } 2313 }
@@ -2801,40 +2827,6 @@ static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
2801} 2827}
2802 2828
2803/* 2829/*
2804 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
2805 */
2806static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
2807 __releases(this_rq->lock)
2808 __acquires(busiest->lock)
2809 __acquires(this_rq->lock)
2810{
2811 int ret = 0;
2812
2813 if (unlikely(!irqs_disabled())) {
2814 /* printk() doesn't work good under rq->lock */
2815 spin_unlock(&this_rq->lock);
2816 BUG_ON(1);
2817 }
2818 if (unlikely(!spin_trylock(&busiest->lock))) {
2819 if (busiest < this_rq) {
2820 spin_unlock(&this_rq->lock);
2821 spin_lock(&busiest->lock);
2822 spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
2823 ret = 1;
2824 } else
2825 spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
2826 }
2827 return ret;
2828}
2829
2830static void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
2831 __releases(busiest->lock)
2832{
2833 spin_unlock(&busiest->lock);
2834 lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
2835}
2836
2837/*
2838 * If dest_cpu is allowed for this process, migrate the task to it. 2830 * If dest_cpu is allowed for this process, migrate the task to it.
2839 * This is accomplished by forcing the cpu_allowed mask to only 2831 * This is accomplished by forcing the cpu_allowed mask to only
2840 * allow dest_cpu, which will force the cpu onto dest_cpu. Then 2832 * allow dest_cpu, which will force the cpu onto dest_cpu. Then
@@ -2847,7 +2839,7 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu)
2847 struct rq *rq; 2839 struct rq *rq;
2848 2840
2849 rq = task_rq_lock(p, &flags); 2841 rq = task_rq_lock(p, &flags);
2850 if (!cpu_isset(dest_cpu, p->cpus_allowed) 2842 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
2851 || unlikely(!cpu_active(dest_cpu))) 2843 || unlikely(!cpu_active(dest_cpu)))
2852 goto out; 2844 goto out;
2853 2845
@@ -2913,7 +2905,7 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
2913 * 2) cannot be migrated to this CPU due to cpus_allowed, or 2905 * 2) cannot be migrated to this CPU due to cpus_allowed, or
2914 * 3) are cache-hot on their current CPU. 2906 * 3) are cache-hot on their current CPU.
2915 */ 2907 */
2916 if (!cpu_isset(this_cpu, p->cpus_allowed)) { 2908 if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
2917 schedstat_inc(p, se.nr_failed_migrations_affine); 2909 schedstat_inc(p, se.nr_failed_migrations_affine);
2918 return 0; 2910 return 0;
2919 } 2911 }
@@ -3088,7 +3080,7 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
3088static struct sched_group * 3080static struct sched_group *
3089find_busiest_group(struct sched_domain *sd, int this_cpu, 3081find_busiest_group(struct sched_domain *sd, int this_cpu,
3090 unsigned long *imbalance, enum cpu_idle_type idle, 3082 unsigned long *imbalance, enum cpu_idle_type idle,
3091 int *sd_idle, const cpumask_t *cpus, int *balance) 3083 int *sd_idle, const struct cpumask *cpus, int *balance)
3092{ 3084{
3093 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups; 3085 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
3094 unsigned long max_load, avg_load, total_load, this_load, total_pwr; 3086 unsigned long max_load, avg_load, total_load, this_load, total_pwr;
@@ -3124,10 +3116,11 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3124 unsigned long sum_avg_load_per_task; 3116 unsigned long sum_avg_load_per_task;
3125 unsigned long avg_load_per_task; 3117 unsigned long avg_load_per_task;
3126 3118
3127 local_group = cpu_isset(this_cpu, group->cpumask); 3119 local_group = cpumask_test_cpu(this_cpu,
3120 sched_group_cpus(group));
3128 3121
3129 if (local_group) 3122 if (local_group)
3130 balance_cpu = first_cpu(group->cpumask); 3123 balance_cpu = cpumask_first(sched_group_cpus(group));
3131 3124
3132 /* Tally up the load of all CPUs in the group */ 3125 /* Tally up the load of all CPUs in the group */
3133 sum_weighted_load = sum_nr_running = avg_load = 0; 3126 sum_weighted_load = sum_nr_running = avg_load = 0;
@@ -3136,13 +3129,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3136 max_cpu_load = 0; 3129 max_cpu_load = 0;
3137 min_cpu_load = ~0UL; 3130 min_cpu_load = ~0UL;
3138 3131
3139 for_each_cpu_mask_nr(i, group->cpumask) { 3132 for_each_cpu_and(i, sched_group_cpus(group), cpus) {
3140 struct rq *rq; 3133 struct rq *rq = cpu_rq(i);
3141
3142 if (!cpu_isset(i, *cpus))
3143 continue;
3144
3145 rq = cpu_rq(i);
3146 3134
3147 if (*sd_idle && rq->nr_running) 3135 if (*sd_idle && rq->nr_running)
3148 *sd_idle = 0; 3136 *sd_idle = 0;
@@ -3253,8 +3241,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3253 */ 3241 */
3254 if ((sum_nr_running < min_nr_running) || 3242 if ((sum_nr_running < min_nr_running) ||
3255 (sum_nr_running == min_nr_running && 3243 (sum_nr_running == min_nr_running &&
3256 first_cpu(group->cpumask) < 3244 cpumask_first(sched_group_cpus(group)) <
3257 first_cpu(group_min->cpumask))) { 3245 cpumask_first(sched_group_cpus(group_min)))) {
3258 group_min = group; 3246 group_min = group;
3259 min_nr_running = sum_nr_running; 3247 min_nr_running = sum_nr_running;
3260 min_load_per_task = sum_weighted_load / 3248 min_load_per_task = sum_weighted_load /
@@ -3269,8 +3257,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3269 if (sum_nr_running <= group_capacity - 1) { 3257 if (sum_nr_running <= group_capacity - 1) {
3270 if (sum_nr_running > leader_nr_running || 3258 if (sum_nr_running > leader_nr_running ||
3271 (sum_nr_running == leader_nr_running && 3259 (sum_nr_running == leader_nr_running &&
3272 first_cpu(group->cpumask) > 3260 cpumask_first(sched_group_cpus(group)) >
3273 first_cpu(group_leader->cpumask))) { 3261 cpumask_first(sched_group_cpus(group_leader)))) {
3274 group_leader = group; 3262 group_leader = group;
3275 leader_nr_running = sum_nr_running; 3263 leader_nr_running = sum_nr_running;
3276 } 3264 }
@@ -3409,16 +3397,16 @@ ret:
3409 */ 3397 */
3410static struct rq * 3398static struct rq *
3411find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle, 3399find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3412 unsigned long imbalance, const cpumask_t *cpus) 3400 unsigned long imbalance, const struct cpumask *cpus)
3413{ 3401{
3414 struct rq *busiest = NULL, *rq; 3402 struct rq *busiest = NULL, *rq;
3415 unsigned long max_load = 0; 3403 unsigned long max_load = 0;
3416 int i; 3404 int i;
3417 3405
3418 for_each_cpu_mask_nr(i, group->cpumask) { 3406 for_each_cpu(i, sched_group_cpus(group)) {
3419 unsigned long wl; 3407 unsigned long wl;
3420 3408
3421 if (!cpu_isset(i, *cpus)) 3409 if (!cpumask_test_cpu(i, cpus))
3422 continue; 3410 continue;
3423 3411
3424 rq = cpu_rq(i); 3412 rq = cpu_rq(i);
@@ -3448,7 +3436,7 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3448 */ 3436 */
3449static int load_balance(int this_cpu, struct rq *this_rq, 3437static int load_balance(int this_cpu, struct rq *this_rq,
3450 struct sched_domain *sd, enum cpu_idle_type idle, 3438 struct sched_domain *sd, enum cpu_idle_type idle,
3451 int *balance, cpumask_t *cpus) 3439 int *balance, struct cpumask *cpus)
3452{ 3440{
3453 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0; 3441 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
3454 struct sched_group *group; 3442 struct sched_group *group;
@@ -3456,7 +3444,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
3456 struct rq *busiest; 3444 struct rq *busiest;
3457 unsigned long flags; 3445 unsigned long flags;
3458 3446
3459 cpus_setall(*cpus); 3447 cpumask_setall(cpus);
3460 3448
3461 /* 3449 /*
3462 * When power savings policy is enabled for the parent domain, idle 3450 * When power savings policy is enabled for the parent domain, idle
@@ -3516,8 +3504,8 @@ redo:
3516 3504
3517 /* All tasks on this runqueue were pinned by CPU affinity */ 3505 /* All tasks on this runqueue were pinned by CPU affinity */
3518 if (unlikely(all_pinned)) { 3506 if (unlikely(all_pinned)) {
3519 cpu_clear(cpu_of(busiest), *cpus); 3507 cpumask_clear_cpu(cpu_of(busiest), cpus);
3520 if (!cpus_empty(*cpus)) 3508 if (!cpumask_empty(cpus))
3521 goto redo; 3509 goto redo;
3522 goto out_balanced; 3510 goto out_balanced;
3523 } 3511 }
@@ -3534,7 +3522,8 @@ redo:
3534 /* don't kick the migration_thread, if the curr 3522 /* don't kick the migration_thread, if the curr
3535 * task on busiest cpu can't be moved to this_cpu 3523 * task on busiest cpu can't be moved to this_cpu
3536 */ 3524 */
3537 if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) { 3525 if (!cpumask_test_cpu(this_cpu,
3526 &busiest->curr->cpus_allowed)) {
3538 spin_unlock_irqrestore(&busiest->lock, flags); 3527 spin_unlock_irqrestore(&busiest->lock, flags);
3539 all_pinned = 1; 3528 all_pinned = 1;
3540 goto out_one_pinned; 3529 goto out_one_pinned;
@@ -3609,7 +3598,7 @@ out:
3609 */ 3598 */
3610static int 3599static int
3611load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd, 3600load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3612 cpumask_t *cpus) 3601 struct cpumask *cpus)
3613{ 3602{
3614 struct sched_group *group; 3603 struct sched_group *group;
3615 struct rq *busiest = NULL; 3604 struct rq *busiest = NULL;
@@ -3618,7 +3607,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3618 int sd_idle = 0; 3607 int sd_idle = 0;
3619 int all_pinned = 0; 3608 int all_pinned = 0;
3620 3609
3621 cpus_setall(*cpus); 3610 cpumask_setall(cpus);
3622 3611
3623 /* 3612 /*
3624 * When power savings policy is enabled for the parent domain, idle 3613 * When power savings policy is enabled for the parent domain, idle
@@ -3662,8 +3651,8 @@ redo:
3662 double_unlock_balance(this_rq, busiest); 3651 double_unlock_balance(this_rq, busiest);
3663 3652
3664 if (unlikely(all_pinned)) { 3653 if (unlikely(all_pinned)) {
3665 cpu_clear(cpu_of(busiest), *cpus); 3654 cpumask_clear_cpu(cpu_of(busiest), cpus);
3666 if (!cpus_empty(*cpus)) 3655 if (!cpumask_empty(cpus))
3667 goto redo; 3656 goto redo;
3668 } 3657 }
3669 } 3658 }
@@ -3696,9 +3685,12 @@ out_balanced:
3696static void idle_balance(int this_cpu, struct rq *this_rq) 3685static void idle_balance(int this_cpu, struct rq *this_rq)
3697{ 3686{
3698 struct sched_domain *sd; 3687 struct sched_domain *sd;
3699 int pulled_task = -1; 3688 int pulled_task = 0;
3700 unsigned long next_balance = jiffies + HZ; 3689 unsigned long next_balance = jiffies + HZ;
3701 cpumask_t tmpmask; 3690 cpumask_var_t tmpmask;
3691
3692 if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
3693 return;
3702 3694
3703 for_each_domain(this_cpu, sd) { 3695 for_each_domain(this_cpu, sd) {
3704 unsigned long interval; 3696 unsigned long interval;
@@ -3709,7 +3701,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3709 if (sd->flags & SD_BALANCE_NEWIDLE) 3701 if (sd->flags & SD_BALANCE_NEWIDLE)
3710 /* If we've pulled tasks over stop searching: */ 3702 /* If we've pulled tasks over stop searching: */
3711 pulled_task = load_balance_newidle(this_cpu, this_rq, 3703 pulled_task = load_balance_newidle(this_cpu, this_rq,
3712 sd, &tmpmask); 3704 sd, tmpmask);
3713 3705
3714 interval = msecs_to_jiffies(sd->balance_interval); 3706 interval = msecs_to_jiffies(sd->balance_interval);
3715 if (time_after(next_balance, sd->last_balance + interval)) 3707 if (time_after(next_balance, sd->last_balance + interval))
@@ -3724,6 +3716,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3724 */ 3716 */
3725 this_rq->next_balance = next_balance; 3717 this_rq->next_balance = next_balance;
3726 } 3718 }
3719 free_cpumask_var(tmpmask);
3727} 3720}
3728 3721
3729/* 3722/*
@@ -3761,7 +3754,7 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3761 /* Search for an sd spanning us and the target CPU. */ 3754 /* Search for an sd spanning us and the target CPU. */
3762 for_each_domain(target_cpu, sd) { 3755 for_each_domain(target_cpu, sd) {
3763 if ((sd->flags & SD_LOAD_BALANCE) && 3756 if ((sd->flags & SD_LOAD_BALANCE) &&
3764 cpu_isset(busiest_cpu, sd->span)) 3757 cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
3765 break; 3758 break;
3766 } 3759 }
3767 3760
@@ -3780,10 +3773,9 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3780#ifdef CONFIG_NO_HZ 3773#ifdef CONFIG_NO_HZ
3781static struct { 3774static struct {
3782 atomic_t load_balancer; 3775 atomic_t load_balancer;
3783 cpumask_t cpu_mask; 3776 cpumask_var_t cpu_mask;
3784} nohz ____cacheline_aligned = { 3777} nohz ____cacheline_aligned = {
3785 .load_balancer = ATOMIC_INIT(-1), 3778 .load_balancer = ATOMIC_INIT(-1),
3786 .cpu_mask = CPU_MASK_NONE,
3787}; 3779};
3788 3780
3789/* 3781/*
@@ -3811,7 +3803,7 @@ int select_nohz_load_balancer(int stop_tick)
3811 int cpu = smp_processor_id(); 3803 int cpu = smp_processor_id();
3812 3804
3813 if (stop_tick) { 3805 if (stop_tick) {
3814 cpu_set(cpu, nohz.cpu_mask); 3806 cpumask_set_cpu(cpu, nohz.cpu_mask);
3815 cpu_rq(cpu)->in_nohz_recently = 1; 3807 cpu_rq(cpu)->in_nohz_recently = 1;
3816 3808
3817 /* 3809 /*
@@ -3825,7 +3817,7 @@ int select_nohz_load_balancer(int stop_tick)
3825 } 3817 }
3826 3818
3827 /* time for ilb owner also to sleep */ 3819 /* time for ilb owner also to sleep */
3828 if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 3820 if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
3829 if (atomic_read(&nohz.load_balancer) == cpu) 3821 if (atomic_read(&nohz.load_balancer) == cpu)
3830 atomic_set(&nohz.load_balancer, -1); 3822 atomic_set(&nohz.load_balancer, -1);
3831 return 0; 3823 return 0;
@@ -3838,10 +3830,10 @@ int select_nohz_load_balancer(int stop_tick)
3838 } else if (atomic_read(&nohz.load_balancer) == cpu) 3830 } else if (atomic_read(&nohz.load_balancer) == cpu)
3839 return 1; 3831 return 1;
3840 } else { 3832 } else {
3841 if (!cpu_isset(cpu, nohz.cpu_mask)) 3833 if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
3842 return 0; 3834 return 0;
3843 3835
3844 cpu_clear(cpu, nohz.cpu_mask); 3836 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3845 3837
3846 if (atomic_read(&nohz.load_balancer) == cpu) 3838 if (atomic_read(&nohz.load_balancer) == cpu)
3847 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu) 3839 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
@@ -3869,7 +3861,11 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3869 unsigned long next_balance = jiffies + 60*HZ; 3861 unsigned long next_balance = jiffies + 60*HZ;
3870 int update_next_balance = 0; 3862 int update_next_balance = 0;
3871 int need_serialize; 3863 int need_serialize;
3872 cpumask_t tmp; 3864 cpumask_var_t tmp;
3865
3866 /* Fails alloc? Rebalancing probably not a priority right now. */
3867 if (!alloc_cpumask_var(&tmp, GFP_ATOMIC))
3868 return;
3873 3869
3874 for_each_domain(cpu, sd) { 3870 for_each_domain(cpu, sd) {
3875 if (!(sd->flags & SD_LOAD_BALANCE)) 3871 if (!(sd->flags & SD_LOAD_BALANCE))
@@ -3894,7 +3890,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3894 } 3890 }
3895 3891
3896 if (time_after_eq(jiffies, sd->last_balance + interval)) { 3892 if (time_after_eq(jiffies, sd->last_balance + interval)) {
3897 if (load_balance(cpu, rq, sd, idle, &balance, &tmp)) { 3893 if (load_balance(cpu, rq, sd, idle, &balance, tmp)) {
3898 /* 3894 /*
3899 * We've pulled tasks over so either we're no 3895 * We've pulled tasks over so either we're no
3900 * longer idle, or one of our SMT siblings is 3896 * longer idle, or one of our SMT siblings is
@@ -3928,6 +3924,8 @@ out:
3928 */ 3924 */
3929 if (likely(update_next_balance)) 3925 if (likely(update_next_balance))
3930 rq->next_balance = next_balance; 3926 rq->next_balance = next_balance;
3927
3928 free_cpumask_var(tmp);
3931} 3929}
3932 3930
3933/* 3931/*
@@ -3952,12 +3950,13 @@ static void run_rebalance_domains(struct softirq_action *h)
3952 */ 3950 */
3953 if (this_rq->idle_at_tick && 3951 if (this_rq->idle_at_tick &&
3954 atomic_read(&nohz.load_balancer) == this_cpu) { 3952 atomic_read(&nohz.load_balancer) == this_cpu) {
3955 cpumask_t cpus = nohz.cpu_mask;
3956 struct rq *rq; 3953 struct rq *rq;
3957 int balance_cpu; 3954 int balance_cpu;
3958 3955
3959 cpu_clear(this_cpu, cpus); 3956 for_each_cpu(balance_cpu, nohz.cpu_mask) {
3960 for_each_cpu_mask_nr(balance_cpu, cpus) { 3957 if (balance_cpu == this_cpu)
3958 continue;
3959
3961 /* 3960 /*
3962 * If this cpu gets work to do, stop the load balancing 3961 * If this cpu gets work to do, stop the load balancing
3963 * work being done for other cpus. Next load 3962 * work being done for other cpus. Next load
@@ -3995,7 +3994,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
3995 rq->in_nohz_recently = 0; 3994 rq->in_nohz_recently = 0;
3996 3995
3997 if (atomic_read(&nohz.load_balancer) == cpu) { 3996 if (atomic_read(&nohz.load_balancer) == cpu) {
3998 cpu_clear(cpu, nohz.cpu_mask); 3997 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3999 atomic_set(&nohz.load_balancer, -1); 3998 atomic_set(&nohz.load_balancer, -1);
4000 } 3999 }
4001 4000
@@ -4008,7 +4007,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4008 * TBD: Traverse the sched domains and nominate 4007 * TBD: Traverse the sched domains and nominate
4009 * the nearest cpu in the nohz.cpu_mask. 4008 * the nearest cpu in the nohz.cpu_mask.
4010 */ 4009 */
4011 int ilb = first_cpu(nohz.cpu_mask); 4010 int ilb = cpumask_first(nohz.cpu_mask);
4012 4011
4013 if (ilb < nr_cpu_ids) 4012 if (ilb < nr_cpu_ids)
4014 resched_cpu(ilb); 4013 resched_cpu(ilb);
@@ -4020,7 +4019,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4020 * cpus with ticks stopped, is it time for that to stop? 4019 * cpus with ticks stopped, is it time for that to stop?
4021 */ 4020 */
4022 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu && 4021 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
4023 cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 4022 cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
4024 resched_cpu(cpu); 4023 resched_cpu(cpu);
4025 return; 4024 return;
4026 } 4025 }
@@ -4030,7 +4029,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4030 * someone else, then no need raise the SCHED_SOFTIRQ 4029 * someone else, then no need raise the SCHED_SOFTIRQ
4031 */ 4030 */
4032 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu && 4031 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
4033 cpu_isset(cpu, nohz.cpu_mask)) 4032 cpumask_test_cpu(cpu, nohz.cpu_mask))
4034 return; 4033 return;
4035#endif 4034#endif
4036 if (time_after_eq(jiffies, rq->next_balance)) 4035 if (time_after_eq(jiffies, rq->next_balance))
@@ -4192,7 +4191,6 @@ void account_steal_time(struct task_struct *p, cputime_t steal)
4192 4191
4193 if (p == rq->idle) { 4192 if (p == rq->idle) {
4194 p->stime = cputime_add(p->stime, steal); 4193 p->stime = cputime_add(p->stime, steal);
4195 account_group_system_time(p, steal);
4196 if (atomic_read(&rq->nr_iowait) > 0) 4194 if (atomic_read(&rq->nr_iowait) > 0)
4197 cpustat->iowait = cputime64_add(cpustat->iowait, tmp); 4195 cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
4198 else 4196 else
@@ -4328,7 +4326,7 @@ void __kprobes sub_preempt_count(int val)
4328 /* 4326 /*
4329 * Underflow? 4327 * Underflow?
4330 */ 4328 */
4331 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 4329 if (DEBUG_LOCKS_WARN_ON(val > preempt_count() - (!!kernel_locked())))
4332 return; 4330 return;
4333 /* 4331 /*
4334 * Is the spinlock portion underflowing? 4332 * Is the spinlock portion underflowing?
@@ -5389,10 +5387,9 @@ out_unlock:
5389 return retval; 5387 return retval;
5390} 5388}
5391 5389
5392long sched_setaffinity(pid_t pid, const cpumask_t *in_mask) 5390long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
5393{ 5391{
5394 cpumask_t cpus_allowed; 5392 cpumask_var_t cpus_allowed, new_mask;
5395 cpumask_t new_mask = *in_mask;
5396 struct task_struct *p; 5393 struct task_struct *p;
5397 int retval; 5394 int retval;
5398 5395
@@ -5414,6 +5411,14 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5414 get_task_struct(p); 5411 get_task_struct(p);
5415 read_unlock(&tasklist_lock); 5412 read_unlock(&tasklist_lock);
5416 5413
5414 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
5415 retval = -ENOMEM;
5416 goto out_put_task;
5417 }
5418 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
5419 retval = -ENOMEM;
5420 goto out_free_cpus_allowed;
5421 }
5417 retval = -EPERM; 5422 retval = -EPERM;
5418 if ((current->euid != p->euid) && (current->euid != p->uid) && 5423 if ((current->euid != p->euid) && (current->euid != p->uid) &&
5419 !capable(CAP_SYS_NICE)) 5424 !capable(CAP_SYS_NICE))
@@ -5423,37 +5428,41 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5423 if (retval) 5428 if (retval)
5424 goto out_unlock; 5429 goto out_unlock;
5425 5430
5426 cpuset_cpus_allowed(p, &cpus_allowed); 5431 cpuset_cpus_allowed(p, cpus_allowed);
5427 cpus_and(new_mask, new_mask, cpus_allowed); 5432 cpumask_and(new_mask, in_mask, cpus_allowed);
5428 again: 5433 again:
5429 retval = set_cpus_allowed_ptr(p, &new_mask); 5434 retval = set_cpus_allowed_ptr(p, new_mask);
5430 5435
5431 if (!retval) { 5436 if (!retval) {
5432 cpuset_cpus_allowed(p, &cpus_allowed); 5437 cpuset_cpus_allowed(p, cpus_allowed);
5433 if (!cpus_subset(new_mask, cpus_allowed)) { 5438 if (!cpumask_subset(new_mask, cpus_allowed)) {
5434 /* 5439 /*
5435 * We must have raced with a concurrent cpuset 5440 * We must have raced with a concurrent cpuset
5436 * update. Just reset the cpus_allowed to the 5441 * update. Just reset the cpus_allowed to the
5437 * cpuset's cpus_allowed 5442 * cpuset's cpus_allowed
5438 */ 5443 */
5439 new_mask = cpus_allowed; 5444 cpumask_copy(new_mask, cpus_allowed);
5440 goto again; 5445 goto again;
5441 } 5446 }
5442 } 5447 }
5443out_unlock: 5448out_unlock:
5449 free_cpumask_var(new_mask);
5450out_free_cpus_allowed:
5451 free_cpumask_var(cpus_allowed);
5452out_put_task:
5444 put_task_struct(p); 5453 put_task_struct(p);
5445 put_online_cpus(); 5454 put_online_cpus();
5446 return retval; 5455 return retval;
5447} 5456}
5448 5457
5449static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 5458static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5450 cpumask_t *new_mask) 5459 struct cpumask *new_mask)
5451{ 5460{
5452 if (len < sizeof(cpumask_t)) { 5461 if (len < cpumask_size())
5453 memset(new_mask, 0, sizeof(cpumask_t)); 5462 cpumask_clear(new_mask);
5454 } else if (len > sizeof(cpumask_t)) { 5463 else if (len > cpumask_size())
5455 len = sizeof(cpumask_t); 5464 len = cpumask_size();
5456 } 5465
5457 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 5466 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
5458} 5467}
5459 5468
@@ -5466,17 +5475,20 @@ static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5466asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, 5475asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
5467 unsigned long __user *user_mask_ptr) 5476 unsigned long __user *user_mask_ptr)
5468{ 5477{
5469 cpumask_t new_mask; 5478 cpumask_var_t new_mask;
5470 int retval; 5479 int retval;
5471 5480
5472 retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask); 5481 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
5473 if (retval) 5482 return -ENOMEM;
5474 return retval;
5475 5483
5476 return sched_setaffinity(pid, &new_mask); 5484 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
5485 if (retval == 0)
5486 retval = sched_setaffinity(pid, new_mask);
5487 free_cpumask_var(new_mask);
5488 return retval;
5477} 5489}
5478 5490
5479long sched_getaffinity(pid_t pid, cpumask_t *mask) 5491long sched_getaffinity(pid_t pid, struct cpumask *mask)
5480{ 5492{
5481 struct task_struct *p; 5493 struct task_struct *p;
5482 int retval; 5494 int retval;
@@ -5493,7 +5505,7 @@ long sched_getaffinity(pid_t pid, cpumask_t *mask)
5493 if (retval) 5505 if (retval)
5494 goto out_unlock; 5506 goto out_unlock;
5495 5507
5496 cpus_and(*mask, p->cpus_allowed, cpu_online_map); 5508 cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
5497 5509
5498out_unlock: 5510out_unlock:
5499 read_unlock(&tasklist_lock); 5511 read_unlock(&tasklist_lock);
@@ -5512,19 +5524,24 @@ asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
5512 unsigned long __user *user_mask_ptr) 5524 unsigned long __user *user_mask_ptr)
5513{ 5525{
5514 int ret; 5526 int ret;
5515 cpumask_t mask; 5527 cpumask_var_t mask;
5516 5528
5517 if (len < sizeof(cpumask_t)) 5529 if (len < cpumask_size())
5518 return -EINVAL; 5530 return -EINVAL;
5519 5531
5520 ret = sched_getaffinity(pid, &mask); 5532 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
5521 if (ret < 0) 5533 return -ENOMEM;
5522 return ret;
5523 5534
5524 if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t))) 5535 ret = sched_getaffinity(pid, mask);
5525 return -EFAULT; 5536 if (ret == 0) {
5537 if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
5538 ret = -EFAULT;
5539 else
5540 ret = cpumask_size();
5541 }
5542 free_cpumask_var(mask);
5526 5543
5527 return sizeof(cpumask_t); 5544 return ret;
5528} 5545}
5529 5546
5530/** 5547/**
@@ -5860,14 +5877,15 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5860 struct rq *rq = cpu_rq(cpu); 5877 struct rq *rq = cpu_rq(cpu);
5861 unsigned long flags; 5878 unsigned long flags;
5862 5879
5880 spin_lock_irqsave(&rq->lock, flags);
5881
5863 __sched_fork(idle); 5882 __sched_fork(idle);
5864 idle->se.exec_start = sched_clock(); 5883 idle->se.exec_start = sched_clock();
5865 5884
5866 idle->prio = idle->normal_prio = MAX_PRIO; 5885 idle->prio = idle->normal_prio = MAX_PRIO;
5867 idle->cpus_allowed = cpumask_of_cpu(cpu); 5886 cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
5868 __set_task_cpu(idle, cpu); 5887 __set_task_cpu(idle, cpu);
5869 5888
5870 spin_lock_irqsave(&rq->lock, flags);
5871 rq->curr = rq->idle = idle; 5889 rq->curr = rq->idle = idle;
5872#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW) 5890#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
5873 idle->oncpu = 1; 5891 idle->oncpu = 1;
@@ -5884,6 +5902,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5884 * The idle tasks have their own, simple scheduling class: 5902 * The idle tasks have their own, simple scheduling class:
5885 */ 5903 */
5886 idle->sched_class = &idle_sched_class; 5904 idle->sched_class = &idle_sched_class;
5905 ftrace_graph_init_task(idle);
5887} 5906}
5888 5907
5889/* 5908/*
@@ -5891,9 +5910,9 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5891 * indicates which cpus entered this state. This is used 5910 * indicates which cpus entered this state. This is used
5892 * in the rcu update to wait only for active cpus. For system 5911 * in the rcu update to wait only for active cpus. For system
5893 * which do not switch off the HZ timer nohz_cpu_mask should 5912 * which do not switch off the HZ timer nohz_cpu_mask should
5894 * always be CPU_MASK_NONE. 5913 * always be CPU_BITS_NONE.
5895 */ 5914 */
5896cpumask_t nohz_cpu_mask = CPU_MASK_NONE; 5915cpumask_var_t nohz_cpu_mask;
5897 5916
5898/* 5917/*
5899 * Increase the granularity value when there are more CPUs, 5918 * Increase the granularity value when there are more CPUs,
@@ -5948,7 +5967,7 @@ static inline void sched_init_granularity(void)
5948 * task must not exit() & deallocate itself prematurely. The 5967 * task must not exit() & deallocate itself prematurely. The
5949 * call is not atomic; no spinlocks may be held. 5968 * call is not atomic; no spinlocks may be held.
5950 */ 5969 */
5951int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask) 5970int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
5952{ 5971{
5953 struct migration_req req; 5972 struct migration_req req;
5954 unsigned long flags; 5973 unsigned long flags;
@@ -5956,13 +5975,13 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5956 int ret = 0; 5975 int ret = 0;
5957 5976
5958 rq = task_rq_lock(p, &flags); 5977 rq = task_rq_lock(p, &flags);
5959 if (!cpus_intersects(*new_mask, cpu_online_map)) { 5978 if (!cpumask_intersects(new_mask, cpu_online_mask)) {
5960 ret = -EINVAL; 5979 ret = -EINVAL;
5961 goto out; 5980 goto out;
5962 } 5981 }
5963 5982
5964 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current && 5983 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
5965 !cpus_equal(p->cpus_allowed, *new_mask))) { 5984 !cpumask_equal(&p->cpus_allowed, new_mask))) {
5966 ret = -EINVAL; 5985 ret = -EINVAL;
5967 goto out; 5986 goto out;
5968 } 5987 }
@@ -5970,15 +5989,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5970 if (p->sched_class->set_cpus_allowed) 5989 if (p->sched_class->set_cpus_allowed)
5971 p->sched_class->set_cpus_allowed(p, new_mask); 5990 p->sched_class->set_cpus_allowed(p, new_mask);
5972 else { 5991 else {
5973 p->cpus_allowed = *new_mask; 5992 cpumask_copy(&p->cpus_allowed, new_mask);
5974 p->rt.nr_cpus_allowed = cpus_weight(*new_mask); 5993 p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
5975 } 5994 }
5976 5995
5977 /* Can the task run on the task's current CPU? If so, we're done */ 5996 /* Can the task run on the task's current CPU? If so, we're done */
5978 if (cpu_isset(task_cpu(p), *new_mask)) 5997 if (cpumask_test_cpu(task_cpu(p), new_mask))
5979 goto out; 5998 goto out;
5980 5999
5981 if (migrate_task(p, any_online_cpu(*new_mask), &req)) { 6000 if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
5982 /* Need help from migration thread: drop lock and wait. */ 6001 /* Need help from migration thread: drop lock and wait. */
5983 task_rq_unlock(rq, &flags); 6002 task_rq_unlock(rq, &flags);
5984 wake_up_process(rq->migration_thread); 6003 wake_up_process(rq->migration_thread);
@@ -6020,7 +6039,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
6020 if (task_cpu(p) != src_cpu) 6039 if (task_cpu(p) != src_cpu)
6021 goto done; 6040 goto done;
6022 /* Affinity changed (again). */ 6041 /* Affinity changed (again). */
6023 if (!cpu_isset(dest_cpu, p->cpus_allowed)) 6042 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6024 goto fail; 6043 goto fail;
6025 6044
6026 on_rq = p->se.on_rq; 6045 on_rq = p->se.on_rq;
@@ -6114,54 +6133,46 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
6114 6133
6115/* 6134/*
6116 * Figure out where task on dead CPU should go, use force if necessary. 6135 * Figure out where task on dead CPU should go, use force if necessary.
6117 * NOTE: interrupts should be disabled by the caller
6118 */ 6136 */
6119static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) 6137static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6120{ 6138{
6121 unsigned long flags;
6122 cpumask_t mask;
6123 struct rq *rq;
6124 int dest_cpu; 6139 int dest_cpu;
6140 /* FIXME: Use cpumask_of_node here. */
6141 cpumask_t _nodemask = node_to_cpumask(cpu_to_node(dead_cpu));
6142 const struct cpumask *nodemask = &_nodemask;
6143
6144again:
6145 /* Look for allowed, online CPU in same node. */
6146 for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
6147 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6148 goto move;
6149
6150 /* Any allowed, online CPU? */
6151 dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
6152 if (dest_cpu < nr_cpu_ids)
6153 goto move;
6154
6155 /* No more Mr. Nice Guy. */
6156 if (dest_cpu >= nr_cpu_ids) {
6157 cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
6158 dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
6125 6159
6126 do { 6160 /*
6127 /* On same node? */ 6161 * Don't tell them about moving exiting tasks or
6128 mask = node_to_cpumask(cpu_to_node(dead_cpu)); 6162 * kernel threads (both mm NULL), since they never
6129 cpus_and(mask, mask, p->cpus_allowed); 6163 * leave kernel.
6130 dest_cpu = any_online_cpu(mask); 6164 */
6131 6165 if (p->mm && printk_ratelimit()) {
6132 /* On any allowed CPU? */ 6166 printk(KERN_INFO "process %d (%s) no "
6133 if (dest_cpu >= nr_cpu_ids) 6167 "longer affine to cpu%d\n",
6134 dest_cpu = any_online_cpu(p->cpus_allowed); 6168 task_pid_nr(p), p->comm, dead_cpu);
6135
6136 /* No more Mr. Nice Guy. */
6137 if (dest_cpu >= nr_cpu_ids) {
6138 cpumask_t cpus_allowed;
6139
6140 cpuset_cpus_allowed_locked(p, &cpus_allowed);
6141 /*
6142 * Try to stay on the same cpuset, where the
6143 * current cpuset may be a subset of all cpus.
6144 * The cpuset_cpus_allowed_locked() variant of
6145 * cpuset_cpus_allowed() will not block. It must be
6146 * called within calls to cpuset_lock/cpuset_unlock.
6147 */
6148 rq = task_rq_lock(p, &flags);
6149 p->cpus_allowed = cpus_allowed;
6150 dest_cpu = any_online_cpu(p->cpus_allowed);
6151 task_rq_unlock(rq, &flags);
6152
6153 /*
6154 * Don't tell them about moving exiting tasks or
6155 * kernel threads (both mm NULL), since they never
6156 * leave kernel.
6157 */
6158 if (p->mm && printk_ratelimit()) {
6159 printk(KERN_INFO "process %d (%s) no "
6160 "longer affine to cpu%d\n",
6161 task_pid_nr(p), p->comm, dead_cpu);
6162 }
6163 } 6169 }
6164 } while (!__migrate_task_irq(p, dead_cpu, dest_cpu)); 6170 }
6171
6172move:
6173 /* It can have affinity changed while we were choosing. */
6174 if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
6175 goto again;
6165} 6176}
6166 6177
6167/* 6178/*
@@ -6173,7 +6184,7 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6173 */ 6184 */
6174static void migrate_nr_uninterruptible(struct rq *rq_src) 6185static void migrate_nr_uninterruptible(struct rq *rq_src)
6175{ 6186{
6176 struct rq *rq_dest = cpu_rq(any_online_cpu(*CPU_MASK_ALL_PTR)); 6187 struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
6177 unsigned long flags; 6188 unsigned long flags;
6178 6189
6179 local_irq_save(flags); 6190 local_irq_save(flags);
@@ -6463,7 +6474,7 @@ static void set_rq_online(struct rq *rq)
6463 if (!rq->online) { 6474 if (!rq->online) {
6464 const struct sched_class *class; 6475 const struct sched_class *class;
6465 6476
6466 cpu_set(rq->cpu, rq->rd->online); 6477 cpumask_set_cpu(rq->cpu, rq->rd->online);
6467 rq->online = 1; 6478 rq->online = 1;
6468 6479
6469 for_each_class(class) { 6480 for_each_class(class) {
@@ -6483,7 +6494,7 @@ static void set_rq_offline(struct rq *rq)
6483 class->rq_offline(rq); 6494 class->rq_offline(rq);
6484 } 6495 }
6485 6496
6486 cpu_clear(rq->cpu, rq->rd->online); 6497 cpumask_clear_cpu(rq->cpu, rq->rd->online);
6487 rq->online = 0; 6498 rq->online = 0;
6488 } 6499 }
6489} 6500}
@@ -6524,7 +6535,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6524 rq = cpu_rq(cpu); 6535 rq = cpu_rq(cpu);
6525 spin_lock_irqsave(&rq->lock, flags); 6536 spin_lock_irqsave(&rq->lock, flags);
6526 if (rq->rd) { 6537 if (rq->rd) {
6527 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6538 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6528 6539
6529 set_rq_online(rq); 6540 set_rq_online(rq);
6530 } 6541 }
@@ -6538,7 +6549,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6538 break; 6549 break;
6539 /* Unbind it from offline cpu so it can run. Fall thru. */ 6550 /* Unbind it from offline cpu so it can run. Fall thru. */
6540 kthread_bind(cpu_rq(cpu)->migration_thread, 6551 kthread_bind(cpu_rq(cpu)->migration_thread,
6541 any_online_cpu(cpu_online_map)); 6552 cpumask_any(cpu_online_mask));
6542 kthread_stop(cpu_rq(cpu)->migration_thread); 6553 kthread_stop(cpu_rq(cpu)->migration_thread);
6543 cpu_rq(cpu)->migration_thread = NULL; 6554 cpu_rq(cpu)->migration_thread = NULL;
6544 break; 6555 break;
@@ -6575,7 +6586,9 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6575 req = list_entry(rq->migration_queue.next, 6586 req = list_entry(rq->migration_queue.next,
6576 struct migration_req, list); 6587 struct migration_req, list);
6577 list_del_init(&req->list); 6588 list_del_init(&req->list);
6589 spin_unlock_irq(&rq->lock);
6578 complete(&req->done); 6590 complete(&req->done);
6591 spin_lock_irq(&rq->lock);
6579 } 6592 }
6580 spin_unlock_irq(&rq->lock); 6593 spin_unlock_irq(&rq->lock);
6581 break; 6594 break;
@@ -6586,7 +6599,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6586 rq = cpu_rq(cpu); 6599 rq = cpu_rq(cpu);
6587 spin_lock_irqsave(&rq->lock, flags); 6600 spin_lock_irqsave(&rq->lock, flags);
6588 if (rq->rd) { 6601 if (rq->rd) {
6589 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6602 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6590 set_rq_offline(rq); 6603 set_rq_offline(rq);
6591 } 6604 }
6592 spin_unlock_irqrestore(&rq->lock, flags); 6605 spin_unlock_irqrestore(&rq->lock, flags);
@@ -6624,36 +6637,14 @@ early_initcall(migration_init);
6624 6637
6625#ifdef CONFIG_SCHED_DEBUG 6638#ifdef CONFIG_SCHED_DEBUG
6626 6639
6627static inline const char *sd_level_to_string(enum sched_domain_level lvl)
6628{
6629 switch (lvl) {
6630 case SD_LV_NONE:
6631 return "NONE";
6632 case SD_LV_SIBLING:
6633 return "SIBLING";
6634 case SD_LV_MC:
6635 return "MC";
6636 case SD_LV_CPU:
6637 return "CPU";
6638 case SD_LV_NODE:
6639 return "NODE";
6640 case SD_LV_ALLNODES:
6641 return "ALLNODES";
6642 case SD_LV_MAX:
6643 return "MAX";
6644
6645 }
6646 return "MAX";
6647}
6648
6649static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, 6640static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6650 cpumask_t *groupmask) 6641 struct cpumask *groupmask)
6651{ 6642{
6652 struct sched_group *group = sd->groups; 6643 struct sched_group *group = sd->groups;
6653 char str[256]; 6644 char str[256];
6654 6645
6655 cpulist_scnprintf(str, sizeof(str), sd->span); 6646 cpulist_scnprintf(str, sizeof(str), *sched_domain_span(sd));
6656 cpus_clear(*groupmask); 6647 cpumask_clear(groupmask);
6657 6648
6658 printk(KERN_DEBUG "%*s domain %d: ", level, "", level); 6649 printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
6659 6650
@@ -6665,14 +6656,13 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6665 return -1; 6656 return -1;
6666 } 6657 }
6667 6658
6668 printk(KERN_CONT "span %s level %s\n", 6659 printk(KERN_CONT "span %s level %s\n", str, sd->name);
6669 str, sd_level_to_string(sd->level));
6670 6660
6671 if (!cpu_isset(cpu, sd->span)) { 6661 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
6672 printk(KERN_ERR "ERROR: domain->span does not contain " 6662 printk(KERN_ERR "ERROR: domain->span does not contain "
6673 "CPU%d\n", cpu); 6663 "CPU%d\n", cpu);
6674 } 6664 }
6675 if (!cpu_isset(cpu, group->cpumask)) { 6665 if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
6676 printk(KERN_ERR "ERROR: domain->groups does not contain" 6666 printk(KERN_ERR "ERROR: domain->groups does not contain"
6677 " CPU%d\n", cpu); 6667 " CPU%d\n", cpu);
6678 } 6668 }
@@ -6692,31 +6682,32 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6692 break; 6682 break;
6693 } 6683 }
6694 6684
6695 if (!cpus_weight(group->cpumask)) { 6685 if (!cpumask_weight(sched_group_cpus(group))) {
6696 printk(KERN_CONT "\n"); 6686 printk(KERN_CONT "\n");
6697 printk(KERN_ERR "ERROR: empty group\n"); 6687 printk(KERN_ERR "ERROR: empty group\n");
6698 break; 6688 break;
6699 } 6689 }
6700 6690
6701 if (cpus_intersects(*groupmask, group->cpumask)) { 6691 if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
6702 printk(KERN_CONT "\n"); 6692 printk(KERN_CONT "\n");
6703 printk(KERN_ERR "ERROR: repeated CPUs\n"); 6693 printk(KERN_ERR "ERROR: repeated CPUs\n");
6704 break; 6694 break;
6705 } 6695 }
6706 6696
6707 cpus_or(*groupmask, *groupmask, group->cpumask); 6697 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
6708 6698
6709 cpulist_scnprintf(str, sizeof(str), group->cpumask); 6699 cpulist_scnprintf(str, sizeof(str), *sched_group_cpus(group));
6710 printk(KERN_CONT " %s", str); 6700 printk(KERN_CONT " %s", str);
6711 6701
6712 group = group->next; 6702 group = group->next;
6713 } while (group != sd->groups); 6703 } while (group != sd->groups);
6714 printk(KERN_CONT "\n"); 6704 printk(KERN_CONT "\n");
6715 6705
6716 if (!cpus_equal(sd->span, *groupmask)) 6706 if (!cpumask_equal(sched_domain_span(sd), groupmask))
6717 printk(KERN_ERR "ERROR: groups don't span domain->span\n"); 6707 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
6718 6708
6719 if (sd->parent && !cpus_subset(*groupmask, sd->parent->span)) 6709 if (sd->parent &&
6710 !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
6720 printk(KERN_ERR "ERROR: parent span is not a superset " 6711 printk(KERN_ERR "ERROR: parent span is not a superset "
6721 "of domain->span\n"); 6712 "of domain->span\n");
6722 return 0; 6713 return 0;
@@ -6724,7 +6715,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6724 6715
6725static void sched_domain_debug(struct sched_domain *sd, int cpu) 6716static void sched_domain_debug(struct sched_domain *sd, int cpu)
6726{ 6717{
6727 cpumask_t *groupmask; 6718 cpumask_var_t groupmask;
6728 int level = 0; 6719 int level = 0;
6729 6720
6730 if (!sd) { 6721 if (!sd) {
@@ -6734,8 +6725,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6734 6725
6735 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); 6726 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
6736 6727
6737 groupmask = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 6728 if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
6738 if (!groupmask) {
6739 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n"); 6729 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
6740 return; 6730 return;
6741 } 6731 }
@@ -6748,7 +6738,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6748 if (!sd) 6738 if (!sd)
6749 break; 6739 break;
6750 } 6740 }
6751 kfree(groupmask); 6741 free_cpumask_var(groupmask);
6752} 6742}
6753#else /* !CONFIG_SCHED_DEBUG */ 6743#else /* !CONFIG_SCHED_DEBUG */
6754# define sched_domain_debug(sd, cpu) do { } while (0) 6744# define sched_domain_debug(sd, cpu) do { } while (0)
@@ -6756,7 +6746,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6756 6746
6757static int sd_degenerate(struct sched_domain *sd) 6747static int sd_degenerate(struct sched_domain *sd)
6758{ 6748{
6759 if (cpus_weight(sd->span) == 1) 6749 if (cpumask_weight(sched_domain_span(sd)) == 1)
6760 return 1; 6750 return 1;
6761 6751
6762 /* Following flags need at least 2 groups */ 6752 /* Following flags need at least 2 groups */
@@ -6787,7 +6777,7 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6787 if (sd_degenerate(parent)) 6777 if (sd_degenerate(parent))
6788 return 1; 6778 return 1;
6789 6779
6790 if (!cpus_equal(sd->span, parent->span)) 6780 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
6791 return 0; 6781 return 0;
6792 6782
6793 /* Does parent contain flags not in child? */ 6783 /* Does parent contain flags not in child? */
@@ -6802,6 +6792,8 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6802 SD_BALANCE_EXEC | 6792 SD_BALANCE_EXEC |
6803 SD_SHARE_CPUPOWER | 6793 SD_SHARE_CPUPOWER |
6804 SD_SHARE_PKG_RESOURCES); 6794 SD_SHARE_PKG_RESOURCES);
6795 if (nr_node_ids == 1)
6796 pflags &= ~SD_SERIALIZE;
6805 } 6797 }
6806 if (~cflags & pflags) 6798 if (~cflags & pflags)
6807 return 0; 6799 return 0;
@@ -6809,6 +6801,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6809 return 1; 6801 return 1;
6810} 6802}
6811 6803
6804static void free_rootdomain(struct root_domain *rd)
6805{
6806 cpupri_cleanup(&rd->cpupri);
6807
6808 free_cpumask_var(rd->rto_mask);
6809 free_cpumask_var(rd->online);
6810 free_cpumask_var(rd->span);
6811 kfree(rd);
6812}
6813
6812static void rq_attach_root(struct rq *rq, struct root_domain *rd) 6814static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6813{ 6815{
6814 unsigned long flags; 6816 unsigned long flags;
@@ -6818,38 +6820,63 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6818 if (rq->rd) { 6820 if (rq->rd) {
6819 struct root_domain *old_rd = rq->rd; 6821 struct root_domain *old_rd = rq->rd;
6820 6822
6821 if (cpu_isset(rq->cpu, old_rd->online)) 6823 if (cpumask_test_cpu(rq->cpu, old_rd->online))
6822 set_rq_offline(rq); 6824 set_rq_offline(rq);
6823 6825
6824 cpu_clear(rq->cpu, old_rd->span); 6826 cpumask_clear_cpu(rq->cpu, old_rd->span);
6825 6827
6826 if (atomic_dec_and_test(&old_rd->refcount)) 6828 if (atomic_dec_and_test(&old_rd->refcount))
6827 kfree(old_rd); 6829 free_rootdomain(old_rd);
6828 } 6830 }
6829 6831
6830 atomic_inc(&rd->refcount); 6832 atomic_inc(&rd->refcount);
6831 rq->rd = rd; 6833 rq->rd = rd;
6832 6834
6833 cpu_set(rq->cpu, rd->span); 6835 cpumask_set_cpu(rq->cpu, rd->span);
6834 if (cpu_isset(rq->cpu, cpu_online_map)) 6836 if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
6835 set_rq_online(rq); 6837 set_rq_online(rq);
6836 6838
6837 spin_unlock_irqrestore(&rq->lock, flags); 6839 spin_unlock_irqrestore(&rq->lock, flags);
6838} 6840}
6839 6841
6840static void init_rootdomain(struct root_domain *rd) 6842static int init_rootdomain(struct root_domain *rd, bool bootmem)
6841{ 6843{
6842 memset(rd, 0, sizeof(*rd)); 6844 memset(rd, 0, sizeof(*rd));
6843 6845
6844 cpus_clear(rd->span); 6846 if (bootmem) {
6845 cpus_clear(rd->online); 6847 alloc_bootmem_cpumask_var(&def_root_domain.span);
6848 alloc_bootmem_cpumask_var(&def_root_domain.online);
6849 alloc_bootmem_cpumask_var(&def_root_domain.rto_mask);
6850 cpupri_init(&rd->cpupri, true);
6851 return 0;
6852 }
6853
6854 if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
6855 goto free_rd;
6856 if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
6857 goto free_span;
6858 if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
6859 goto free_online;
6860
6861 if (cpupri_init(&rd->cpupri, false) != 0)
6862 goto free_rto_mask;
6863 return 0;
6846 6864
6847 cpupri_init(&rd->cpupri); 6865free_rto_mask:
6866 free_cpumask_var(rd->rto_mask);
6867free_online:
6868 free_cpumask_var(rd->online);
6869free_span:
6870 free_cpumask_var(rd->span);
6871free_rd:
6872 kfree(rd);
6873 return -ENOMEM;
6848} 6874}
6849 6875
6850static void init_defrootdomain(void) 6876static void init_defrootdomain(void)
6851{ 6877{
6852 init_rootdomain(&def_root_domain); 6878 init_rootdomain(&def_root_domain, true);
6879
6853 atomic_set(&def_root_domain.refcount, 1); 6880 atomic_set(&def_root_domain.refcount, 1);
6854} 6881}
6855 6882
@@ -6861,7 +6888,10 @@ static struct root_domain *alloc_rootdomain(void)
6861 if (!rd) 6888 if (!rd)
6862 return NULL; 6889 return NULL;
6863 6890
6864 init_rootdomain(rd); 6891 if (init_rootdomain(rd, false) != 0) {
6892 kfree(rd);
6893 return NULL;
6894 }
6865 6895
6866 return rd; 6896 return rd;
6867} 6897}
@@ -6903,19 +6933,12 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
6903} 6933}
6904 6934
6905/* cpus with isolated domains */ 6935/* cpus with isolated domains */
6906static cpumask_t cpu_isolated_map = CPU_MASK_NONE; 6936static cpumask_var_t cpu_isolated_map;
6907 6937
6908/* Setup the mask of cpus configured for isolated domains */ 6938/* Setup the mask of cpus configured for isolated domains */
6909static int __init isolated_cpu_setup(char *str) 6939static int __init isolated_cpu_setup(char *str)
6910{ 6940{
6911 static int __initdata ints[NR_CPUS]; 6941 cpulist_parse(str, *cpu_isolated_map);
6912 int i;
6913
6914 str = get_options(str, ARRAY_SIZE(ints), ints);
6915 cpus_clear(cpu_isolated_map);
6916 for (i = 1; i <= ints[0]; i++)
6917 if (ints[i] < NR_CPUS)
6918 cpu_set(ints[i], cpu_isolated_map);
6919 return 1; 6942 return 1;
6920} 6943}
6921 6944
@@ -6924,42 +6947,43 @@ __setup("isolcpus=", isolated_cpu_setup);
6924/* 6947/*
6925 * init_sched_build_groups takes the cpumask we wish to span, and a pointer 6948 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
6926 * to a function which identifies what group(along with sched group) a CPU 6949 * to a function which identifies what group(along with sched group) a CPU
6927 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS 6950 * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
6928 * (due to the fact that we keep track of groups covered with a cpumask_t). 6951 * (due to the fact that we keep track of groups covered with a struct cpumask).
6929 * 6952 *
6930 * init_sched_build_groups will build a circular linked list of the groups 6953 * init_sched_build_groups will build a circular linked list of the groups
6931 * covered by the given span, and will set each group's ->cpumask correctly, 6954 * covered by the given span, and will set each group's ->cpumask correctly,
6932 * and ->cpu_power to 0. 6955 * and ->cpu_power to 0.
6933 */ 6956 */
6934static void 6957static void
6935init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map, 6958init_sched_build_groups(const struct cpumask *span,
6936 int (*group_fn)(int cpu, const cpumask_t *cpu_map, 6959 const struct cpumask *cpu_map,
6960 int (*group_fn)(int cpu, const struct cpumask *cpu_map,
6937 struct sched_group **sg, 6961 struct sched_group **sg,
6938 cpumask_t *tmpmask), 6962 struct cpumask *tmpmask),
6939 cpumask_t *covered, cpumask_t *tmpmask) 6963 struct cpumask *covered, struct cpumask *tmpmask)
6940{ 6964{
6941 struct sched_group *first = NULL, *last = NULL; 6965 struct sched_group *first = NULL, *last = NULL;
6942 int i; 6966 int i;
6943 6967
6944 cpus_clear(*covered); 6968 cpumask_clear(covered);
6945 6969
6946 for_each_cpu_mask_nr(i, *span) { 6970 for_each_cpu(i, span) {
6947 struct sched_group *sg; 6971 struct sched_group *sg;
6948 int group = group_fn(i, cpu_map, &sg, tmpmask); 6972 int group = group_fn(i, cpu_map, &sg, tmpmask);
6949 int j; 6973 int j;
6950 6974
6951 if (cpu_isset(i, *covered)) 6975 if (cpumask_test_cpu(i, covered))
6952 continue; 6976 continue;
6953 6977
6954 cpus_clear(sg->cpumask); 6978 cpumask_clear(sched_group_cpus(sg));
6955 sg->__cpu_power = 0; 6979 sg->__cpu_power = 0;
6956 6980
6957 for_each_cpu_mask_nr(j, *span) { 6981 for_each_cpu(j, span) {
6958 if (group_fn(j, cpu_map, NULL, tmpmask) != group) 6982 if (group_fn(j, cpu_map, NULL, tmpmask) != group)
6959 continue; 6983 continue;
6960 6984
6961 cpu_set(j, *covered); 6985 cpumask_set_cpu(j, covered);
6962 cpu_set(j, sg->cpumask); 6986 cpumask_set_cpu(j, sched_group_cpus(sg));
6963 } 6987 }
6964 if (!first) 6988 if (!first)
6965 first = sg; 6989 first = sg;
@@ -7023,9 +7047,10 @@ static int find_next_best_node(int node, nodemask_t *used_nodes)
7023 * should be one that prevents unnecessary balancing, but also spreads tasks 7047 * should be one that prevents unnecessary balancing, but also spreads tasks
7024 * out optimally. 7048 * out optimally.
7025 */ 7049 */
7026static void sched_domain_node_span(int node, cpumask_t *span) 7050static void sched_domain_node_span(int node, struct cpumask *span)
7027{ 7051{
7028 nodemask_t used_nodes; 7052 nodemask_t used_nodes;
7053 /* FIXME: use cpumask_of_node() */
7029 node_to_cpumask_ptr(nodemask, node); 7054 node_to_cpumask_ptr(nodemask, node);
7030 int i; 7055 int i;
7031 7056
@@ -7047,18 +7072,33 @@ static void sched_domain_node_span(int node, cpumask_t *span)
7047int sched_smt_power_savings = 0, sched_mc_power_savings = 0; 7072int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
7048 7073
7049/* 7074/*
7075 * The cpus mask in sched_group and sched_domain hangs off the end.
7076 * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space
7077 * for nr_cpu_ids < CONFIG_NR_CPUS.
7078 */
7079struct static_sched_group {
7080 struct sched_group sg;
7081 DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
7082};
7083
7084struct static_sched_domain {
7085 struct sched_domain sd;
7086 DECLARE_BITMAP(span, CONFIG_NR_CPUS);
7087};
7088
7089/*
7050 * SMT sched-domains: 7090 * SMT sched-domains:
7051 */ 7091 */
7052#ifdef CONFIG_SCHED_SMT 7092#ifdef CONFIG_SCHED_SMT
7053static DEFINE_PER_CPU(struct sched_domain, cpu_domains); 7093static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
7054static DEFINE_PER_CPU(struct sched_group, sched_group_cpus); 7094static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
7055 7095
7056static int 7096static int
7057cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7097cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
7058 cpumask_t *unused) 7098 struct sched_group **sg, struct cpumask *unused)
7059{ 7099{
7060 if (sg) 7100 if (sg)
7061 *sg = &per_cpu(sched_group_cpus, cpu); 7101 *sg = &per_cpu(sched_group_cpus, cpu).sg;
7062 return cpu; 7102 return cpu;
7063} 7103}
7064#endif /* CONFIG_SCHED_SMT */ 7104#endif /* CONFIG_SCHED_SMT */
@@ -7067,56 +7107,55 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
7067 * multi-core sched-domains: 7107 * multi-core sched-domains:
7068 */ 7108 */
7069#ifdef CONFIG_SCHED_MC 7109#ifdef CONFIG_SCHED_MC
7070static DEFINE_PER_CPU(struct sched_domain, core_domains); 7110static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
7071static DEFINE_PER_CPU(struct sched_group, sched_group_core); 7111static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
7072#endif /* CONFIG_SCHED_MC */ 7112#endif /* CONFIG_SCHED_MC */
7073 7113
7074#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) 7114#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
7075static int 7115static int
7076cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7116cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7077 cpumask_t *mask) 7117 struct sched_group **sg, struct cpumask *mask)
7078{ 7118{
7079 int group; 7119 int group;
7080 7120
7081 *mask = per_cpu(cpu_sibling_map, cpu); 7121 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7082 cpus_and(*mask, *mask, *cpu_map); 7122 group = cpumask_first(mask);
7083 group = first_cpu(*mask);
7084 if (sg) 7123 if (sg)
7085 *sg = &per_cpu(sched_group_core, group); 7124 *sg = &per_cpu(sched_group_core, group).sg;
7086 return group; 7125 return group;
7087} 7126}
7088#elif defined(CONFIG_SCHED_MC) 7127#elif defined(CONFIG_SCHED_MC)
7089static int 7128static int
7090cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7129cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7091 cpumask_t *unused) 7130 struct sched_group **sg, struct cpumask *unused)
7092{ 7131{
7093 if (sg) 7132 if (sg)
7094 *sg = &per_cpu(sched_group_core, cpu); 7133 *sg = &per_cpu(sched_group_core, cpu).sg;
7095 return cpu; 7134 return cpu;
7096} 7135}
7097#endif 7136#endif
7098 7137
7099static DEFINE_PER_CPU(struct sched_domain, phys_domains); 7138static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
7100static DEFINE_PER_CPU(struct sched_group, sched_group_phys); 7139static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
7101 7140
7102static int 7141static int
7103cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7142cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
7104 cpumask_t *mask) 7143 struct sched_group **sg, struct cpumask *mask)
7105{ 7144{
7106 int group; 7145 int group;
7107#ifdef CONFIG_SCHED_MC 7146#ifdef CONFIG_SCHED_MC
7147 /* FIXME: Use cpu_coregroup_mask. */
7108 *mask = cpu_coregroup_map(cpu); 7148 *mask = cpu_coregroup_map(cpu);
7109 cpus_and(*mask, *mask, *cpu_map); 7149 cpus_and(*mask, *mask, *cpu_map);
7110 group = first_cpu(*mask); 7150 group = cpumask_first(mask);
7111#elif defined(CONFIG_SCHED_SMT) 7151#elif defined(CONFIG_SCHED_SMT)
7112 *mask = per_cpu(cpu_sibling_map, cpu); 7152 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7113 cpus_and(*mask, *mask, *cpu_map); 7153 group = cpumask_first(mask);
7114 group = first_cpu(*mask);
7115#else 7154#else
7116 group = cpu; 7155 group = cpu;
7117#endif 7156#endif
7118 if (sg) 7157 if (sg)
7119 *sg = &per_cpu(sched_group_phys, group); 7158 *sg = &per_cpu(sched_group_phys, group).sg;
7120 return group; 7159 return group;
7121} 7160}
7122 7161
@@ -7130,19 +7169,21 @@ static DEFINE_PER_CPU(struct sched_domain, node_domains);
7130static struct sched_group ***sched_group_nodes_bycpu; 7169static struct sched_group ***sched_group_nodes_bycpu;
7131 7170
7132static DEFINE_PER_CPU(struct sched_domain, allnodes_domains); 7171static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
7133static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes); 7172static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
7134 7173
7135static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map, 7174static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
7136 struct sched_group **sg, cpumask_t *nodemask) 7175 struct sched_group **sg,
7176 struct cpumask *nodemask)
7137{ 7177{
7138 int group; 7178 int group;
7179 /* FIXME: use cpumask_of_node */
7180 node_to_cpumask_ptr(pnodemask, cpu_to_node(cpu));
7139 7181
7140 *nodemask = node_to_cpumask(cpu_to_node(cpu)); 7182 cpumask_and(nodemask, pnodemask, cpu_map);
7141 cpus_and(*nodemask, *nodemask, *cpu_map); 7183 group = cpumask_first(nodemask);
7142 group = first_cpu(*nodemask);
7143 7184
7144 if (sg) 7185 if (sg)
7145 *sg = &per_cpu(sched_group_allnodes, group); 7186 *sg = &per_cpu(sched_group_allnodes, group).sg;
7146 return group; 7187 return group;
7147} 7188}
7148 7189
@@ -7154,11 +7195,11 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7154 if (!sg) 7195 if (!sg)
7155 return; 7196 return;
7156 do { 7197 do {
7157 for_each_cpu_mask_nr(j, sg->cpumask) { 7198 for_each_cpu(j, sched_group_cpus(sg)) {
7158 struct sched_domain *sd; 7199 struct sched_domain *sd;
7159 7200
7160 sd = &per_cpu(phys_domains, j); 7201 sd = &per_cpu(phys_domains, j).sd;
7161 if (j != first_cpu(sd->groups->cpumask)) { 7202 if (j != cpumask_first(sched_group_cpus(sd->groups))) {
7162 /* 7203 /*
7163 * Only add "power" once for each 7204 * Only add "power" once for each
7164 * physical package. 7205 * physical package.
@@ -7175,11 +7216,12 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7175 7216
7176#ifdef CONFIG_NUMA 7217#ifdef CONFIG_NUMA
7177/* Free memory allocated for various sched_group structures */ 7218/* Free memory allocated for various sched_group structures */
7178static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7219static void free_sched_groups(const struct cpumask *cpu_map,
7220 struct cpumask *nodemask)
7179{ 7221{
7180 int cpu, i; 7222 int cpu, i;
7181 7223
7182 for_each_cpu_mask_nr(cpu, *cpu_map) { 7224 for_each_cpu(cpu, cpu_map) {
7183 struct sched_group **sched_group_nodes 7225 struct sched_group **sched_group_nodes
7184 = sched_group_nodes_bycpu[cpu]; 7226 = sched_group_nodes_bycpu[cpu];
7185 7227
@@ -7188,10 +7230,11 @@ static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
7188 7230
7189 for (i = 0; i < nr_node_ids; i++) { 7231 for (i = 0; i < nr_node_ids; i++) {
7190 struct sched_group *oldsg, *sg = sched_group_nodes[i]; 7232 struct sched_group *oldsg, *sg = sched_group_nodes[i];
7233 /* FIXME: Use cpumask_of_node */
7234 node_to_cpumask_ptr(pnodemask, i);
7191 7235
7192 *nodemask = node_to_cpumask(i); 7236 cpus_and(*nodemask, *pnodemask, *cpu_map);
7193 cpus_and(*nodemask, *nodemask, *cpu_map); 7237 if (cpumask_empty(nodemask))
7194 if (cpus_empty(*nodemask))
7195 continue; 7238 continue;
7196 7239
7197 if (sg == NULL) 7240 if (sg == NULL)
@@ -7209,7 +7252,8 @@ next_sg:
7209 } 7252 }
7210} 7253}
7211#else /* !CONFIG_NUMA */ 7254#else /* !CONFIG_NUMA */
7212static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7255static void free_sched_groups(const struct cpumask *cpu_map,
7256 struct cpumask *nodemask)
7213{ 7257{
7214} 7258}
7215#endif /* CONFIG_NUMA */ 7259#endif /* CONFIG_NUMA */
@@ -7235,7 +7279,7 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
7235 7279
7236 WARN_ON(!sd || !sd->groups); 7280 WARN_ON(!sd || !sd->groups);
7237 7281
7238 if (cpu != first_cpu(sd->groups->cpumask)) 7282 if (cpu != cpumask_first(sched_group_cpus(sd->groups)))
7239 return; 7283 return;
7240 7284
7241 child = sd->child; 7285 child = sd->child;
@@ -7300,40 +7344,6 @@ SD_INIT_FUNC(CPU)
7300 SD_INIT_FUNC(MC) 7344 SD_INIT_FUNC(MC)
7301#endif 7345#endif
7302 7346
7303/*
7304 * To minimize stack usage kmalloc room for cpumasks and share the
7305 * space as the usage in build_sched_domains() dictates. Used only
7306 * if the amount of space is significant.
7307 */
7308struct allmasks {
7309 cpumask_t tmpmask; /* make this one first */
7310 union {
7311 cpumask_t nodemask;
7312 cpumask_t this_sibling_map;
7313 cpumask_t this_core_map;
7314 };
7315 cpumask_t send_covered;
7316
7317#ifdef CONFIG_NUMA
7318 cpumask_t domainspan;
7319 cpumask_t covered;
7320 cpumask_t notcovered;
7321#endif
7322};
7323
7324#if NR_CPUS > 128
7325#define SCHED_CPUMASK_ALLOC 1
7326#define SCHED_CPUMASK_FREE(v) kfree(v)
7327#define SCHED_CPUMASK_DECLARE(v) struct allmasks *v
7328#else
7329#define SCHED_CPUMASK_ALLOC 0
7330#define SCHED_CPUMASK_FREE(v)
7331#define SCHED_CPUMASK_DECLARE(v) struct allmasks _v, *v = &_v
7332#endif
7333
7334#define SCHED_CPUMASK_VAR(v, a) cpumask_t *v = (cpumask_t *) \
7335 ((unsigned long)(a) + offsetof(struct allmasks, v))
7336
7337static int default_relax_domain_level = -1; 7347static int default_relax_domain_level = -1;
7338 7348
7339static int __init setup_relax_domain_level(char *str) 7349static int __init setup_relax_domain_level(char *str)
@@ -7373,17 +7383,38 @@ static void set_domain_attribute(struct sched_domain *sd,
7373 * Build sched domains for a given set of cpus and attach the sched domains 7383 * Build sched domains for a given set of cpus and attach the sched domains
7374 * to the individual cpus 7384 * to the individual cpus
7375 */ 7385 */
7376static int __build_sched_domains(const cpumask_t *cpu_map, 7386static int __build_sched_domains(const struct cpumask *cpu_map,
7377 struct sched_domain_attr *attr) 7387 struct sched_domain_attr *attr)
7378{ 7388{
7379 int i; 7389 int i, err = -ENOMEM;
7380 struct root_domain *rd; 7390 struct root_domain *rd;
7381 SCHED_CPUMASK_DECLARE(allmasks); 7391 cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
7382 cpumask_t *tmpmask; 7392 tmpmask;
7383#ifdef CONFIG_NUMA 7393#ifdef CONFIG_NUMA
7394 cpumask_var_t domainspan, covered, notcovered;
7384 struct sched_group **sched_group_nodes = NULL; 7395 struct sched_group **sched_group_nodes = NULL;
7385 int sd_allnodes = 0; 7396 int sd_allnodes = 0;
7386 7397
7398 if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
7399 goto out;
7400 if (!alloc_cpumask_var(&covered, GFP_KERNEL))
7401 goto free_domainspan;
7402 if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
7403 goto free_covered;
7404#endif
7405
7406 if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
7407 goto free_notcovered;
7408 if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
7409 goto free_nodemask;
7410 if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
7411 goto free_this_sibling_map;
7412 if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
7413 goto free_this_core_map;
7414 if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
7415 goto free_send_covered;
7416
7417#ifdef CONFIG_NUMA
7387 /* 7418 /*
7388 * Allocate the per-node list of sched groups 7419 * Allocate the per-node list of sched groups
7389 */ 7420 */
@@ -7391,55 +7422,37 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7391 GFP_KERNEL); 7422 GFP_KERNEL);
7392 if (!sched_group_nodes) { 7423 if (!sched_group_nodes) {
7393 printk(KERN_WARNING "Can not alloc sched group node list\n"); 7424 printk(KERN_WARNING "Can not alloc sched group node list\n");
7394 return -ENOMEM; 7425 goto free_tmpmask;
7395 } 7426 }
7396#endif 7427#endif
7397 7428
7398 rd = alloc_rootdomain(); 7429 rd = alloc_rootdomain();
7399 if (!rd) { 7430 if (!rd) {
7400 printk(KERN_WARNING "Cannot alloc root domain\n"); 7431 printk(KERN_WARNING "Cannot alloc root domain\n");
7401#ifdef CONFIG_NUMA 7432 goto free_sched_groups;
7402 kfree(sched_group_nodes);
7403#endif
7404 return -ENOMEM;
7405 } 7433 }
7406 7434
7407#if SCHED_CPUMASK_ALLOC
7408 /* get space for all scratch cpumask variables */
7409 allmasks = kmalloc(sizeof(*allmasks), GFP_KERNEL);
7410 if (!allmasks) {
7411 printk(KERN_WARNING "Cannot alloc cpumask array\n");
7412 kfree(rd);
7413#ifdef CONFIG_NUMA
7414 kfree(sched_group_nodes);
7415#endif
7416 return -ENOMEM;
7417 }
7418#endif
7419 tmpmask = (cpumask_t *)allmasks;
7420
7421
7422#ifdef CONFIG_NUMA 7435#ifdef CONFIG_NUMA
7423 sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes; 7436 sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
7424#endif 7437#endif
7425 7438
7426 /* 7439 /*
7427 * Set up domains for cpus specified by the cpu_map. 7440 * Set up domains for cpus specified by the cpu_map.
7428 */ 7441 */
7429 for_each_cpu_mask_nr(i, *cpu_map) { 7442 for_each_cpu(i, cpu_map) {
7430 struct sched_domain *sd = NULL, *p; 7443 struct sched_domain *sd = NULL, *p;
7431 SCHED_CPUMASK_VAR(nodemask, allmasks);
7432 7444
7445 /* FIXME: use cpumask_of_node */
7433 *nodemask = node_to_cpumask(cpu_to_node(i)); 7446 *nodemask = node_to_cpumask(cpu_to_node(i));
7434 cpus_and(*nodemask, *nodemask, *cpu_map); 7447 cpus_and(*nodemask, *nodemask, *cpu_map);
7435 7448
7436#ifdef CONFIG_NUMA 7449#ifdef CONFIG_NUMA
7437 if (cpus_weight(*cpu_map) > 7450 if (cpumask_weight(cpu_map) >
7438 SD_NODES_PER_DOMAIN*cpus_weight(*nodemask)) { 7451 SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
7439 sd = &per_cpu(allnodes_domains, i); 7452 sd = &per_cpu(allnodes_domains, i);
7440 SD_INIT(sd, ALLNODES); 7453 SD_INIT(sd, ALLNODES);
7441 set_domain_attribute(sd, attr); 7454 set_domain_attribute(sd, attr);
7442 sd->span = *cpu_map; 7455 cpumask_copy(sched_domain_span(sd), cpu_map);
7443 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask); 7456 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
7444 p = sd; 7457 p = sd;
7445 sd_allnodes = 1; 7458 sd_allnodes = 1;
@@ -7449,18 +7462,19 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7449 sd = &per_cpu(node_domains, i); 7462 sd = &per_cpu(node_domains, i);
7450 SD_INIT(sd, NODE); 7463 SD_INIT(sd, NODE);
7451 set_domain_attribute(sd, attr); 7464 set_domain_attribute(sd, attr);
7452 sched_domain_node_span(cpu_to_node(i), &sd->span); 7465 sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
7453 sd->parent = p; 7466 sd->parent = p;
7454 if (p) 7467 if (p)
7455 p->child = sd; 7468 p->child = sd;
7456 cpus_and(sd->span, sd->span, *cpu_map); 7469 cpumask_and(sched_domain_span(sd),
7470 sched_domain_span(sd), cpu_map);
7457#endif 7471#endif
7458 7472
7459 p = sd; 7473 p = sd;
7460 sd = &per_cpu(phys_domains, i); 7474 sd = &per_cpu(phys_domains, i).sd;
7461 SD_INIT(sd, CPU); 7475 SD_INIT(sd, CPU);
7462 set_domain_attribute(sd, attr); 7476 set_domain_attribute(sd, attr);
7463 sd->span = *nodemask; 7477 cpumask_copy(sched_domain_span(sd), nodemask);
7464 sd->parent = p; 7478 sd->parent = p;
7465 if (p) 7479 if (p)
7466 p->child = sd; 7480 p->child = sd;
@@ -7468,11 +7482,12 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7468 7482
7469#ifdef CONFIG_SCHED_MC 7483#ifdef CONFIG_SCHED_MC
7470 p = sd; 7484 p = sd;
7471 sd = &per_cpu(core_domains, i); 7485 sd = &per_cpu(core_domains, i).sd;
7472 SD_INIT(sd, MC); 7486 SD_INIT(sd, MC);
7473 set_domain_attribute(sd, attr); 7487 set_domain_attribute(sd, attr);
7474 sd->span = cpu_coregroup_map(i); 7488 *sched_domain_span(sd) = cpu_coregroup_map(i);
7475 cpus_and(sd->span, sd->span, *cpu_map); 7489 cpumask_and(sched_domain_span(sd),
7490 sched_domain_span(sd), cpu_map);
7476 sd->parent = p; 7491 sd->parent = p;
7477 p->child = sd; 7492 p->child = sd;
7478 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask); 7493 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7480,11 +7495,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7480 7495
7481#ifdef CONFIG_SCHED_SMT 7496#ifdef CONFIG_SCHED_SMT
7482 p = sd; 7497 p = sd;
7483 sd = &per_cpu(cpu_domains, i); 7498 sd = &per_cpu(cpu_domains, i).sd;
7484 SD_INIT(sd, SIBLING); 7499 SD_INIT(sd, SIBLING);
7485 set_domain_attribute(sd, attr); 7500 set_domain_attribute(sd, attr);
7486 sd->span = per_cpu(cpu_sibling_map, i); 7501 cpumask_and(sched_domain_span(sd),
7487 cpus_and(sd->span, sd->span, *cpu_map); 7502 &per_cpu(cpu_sibling_map, i), cpu_map);
7488 sd->parent = p; 7503 sd->parent = p;
7489 p->child = sd; 7504 p->child = sd;
7490 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask); 7505 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7493,13 +7508,10 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7493 7508
7494#ifdef CONFIG_SCHED_SMT 7509#ifdef CONFIG_SCHED_SMT
7495 /* Set up CPU (sibling) groups */ 7510 /* Set up CPU (sibling) groups */
7496 for_each_cpu_mask_nr(i, *cpu_map) { 7511 for_each_cpu(i, cpu_map) {
7497 SCHED_CPUMASK_VAR(this_sibling_map, allmasks); 7512 cpumask_and(this_sibling_map,
7498 SCHED_CPUMASK_VAR(send_covered, allmasks); 7513 &per_cpu(cpu_sibling_map, i), cpu_map);
7499 7514 if (i != cpumask_first(this_sibling_map))
7500 *this_sibling_map = per_cpu(cpu_sibling_map, i);
7501 cpus_and(*this_sibling_map, *this_sibling_map, *cpu_map);
7502 if (i != first_cpu(*this_sibling_map))
7503 continue; 7515 continue;
7504 7516
7505 init_sched_build_groups(this_sibling_map, cpu_map, 7517 init_sched_build_groups(this_sibling_map, cpu_map,
@@ -7510,13 +7522,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7510 7522
7511#ifdef CONFIG_SCHED_MC 7523#ifdef CONFIG_SCHED_MC
7512 /* Set up multi-core groups */ 7524 /* Set up multi-core groups */
7513 for_each_cpu_mask_nr(i, *cpu_map) { 7525 for_each_cpu(i, cpu_map) {
7514 SCHED_CPUMASK_VAR(this_core_map, allmasks); 7526 /* FIXME: Use cpu_coregroup_mask */
7515 SCHED_CPUMASK_VAR(send_covered, allmasks);
7516
7517 *this_core_map = cpu_coregroup_map(i); 7527 *this_core_map = cpu_coregroup_map(i);
7518 cpus_and(*this_core_map, *this_core_map, *cpu_map); 7528 cpus_and(*this_core_map, *this_core_map, *cpu_map);
7519 if (i != first_cpu(*this_core_map)) 7529 if (i != cpumask_first(this_core_map))
7520 continue; 7530 continue;
7521 7531
7522 init_sched_build_groups(this_core_map, cpu_map, 7532 init_sched_build_groups(this_core_map, cpu_map,
@@ -7527,12 +7537,10 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7527 7537
7528 /* Set up physical groups */ 7538 /* Set up physical groups */
7529 for (i = 0; i < nr_node_ids; i++) { 7539 for (i = 0; i < nr_node_ids; i++) {
7530 SCHED_CPUMASK_VAR(nodemask, allmasks); 7540 /* FIXME: Use cpumask_of_node */
7531 SCHED_CPUMASK_VAR(send_covered, allmasks);
7532
7533 *nodemask = node_to_cpumask(i); 7541 *nodemask = node_to_cpumask(i);
7534 cpus_and(*nodemask, *nodemask, *cpu_map); 7542 cpus_and(*nodemask, *nodemask, *cpu_map);
7535 if (cpus_empty(*nodemask)) 7543 if (cpumask_empty(nodemask))
7536 continue; 7544 continue;
7537 7545
7538 init_sched_build_groups(nodemask, cpu_map, 7546 init_sched_build_groups(nodemask, cpu_map,
@@ -7543,8 +7551,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7543#ifdef CONFIG_NUMA 7551#ifdef CONFIG_NUMA
7544 /* Set up node groups */ 7552 /* Set up node groups */
7545 if (sd_allnodes) { 7553 if (sd_allnodes) {
7546 SCHED_CPUMASK_VAR(send_covered, allmasks);
7547
7548 init_sched_build_groups(cpu_map, cpu_map, 7554 init_sched_build_groups(cpu_map, cpu_map,
7549 &cpu_to_allnodes_group, 7555 &cpu_to_allnodes_group,
7550 send_covered, tmpmask); 7556 send_covered, tmpmask);
@@ -7553,58 +7559,58 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7553 for (i = 0; i < nr_node_ids; i++) { 7559 for (i = 0; i < nr_node_ids; i++) {
7554 /* Set up node groups */ 7560 /* Set up node groups */
7555 struct sched_group *sg, *prev; 7561 struct sched_group *sg, *prev;
7556 SCHED_CPUMASK_VAR(nodemask, allmasks);
7557 SCHED_CPUMASK_VAR(domainspan, allmasks);
7558 SCHED_CPUMASK_VAR(covered, allmasks);
7559 int j; 7562 int j;
7560 7563
7564 /* FIXME: Use cpumask_of_node */
7561 *nodemask = node_to_cpumask(i); 7565 *nodemask = node_to_cpumask(i);
7562 cpus_clear(*covered); 7566 cpumask_clear(covered);
7563 7567
7564 cpus_and(*nodemask, *nodemask, *cpu_map); 7568 cpus_and(*nodemask, *nodemask, *cpu_map);
7565 if (cpus_empty(*nodemask)) { 7569 if (cpumask_empty(nodemask)) {
7566 sched_group_nodes[i] = NULL; 7570 sched_group_nodes[i] = NULL;
7567 continue; 7571 continue;
7568 } 7572 }
7569 7573
7570 sched_domain_node_span(i, domainspan); 7574 sched_domain_node_span(i, domainspan);
7571 cpus_and(*domainspan, *domainspan, *cpu_map); 7575 cpumask_and(domainspan, domainspan, cpu_map);
7572 7576
7573 sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i); 7577 sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
7578 GFP_KERNEL, i);
7574 if (!sg) { 7579 if (!sg) {
7575 printk(KERN_WARNING "Can not alloc domain group for " 7580 printk(KERN_WARNING "Can not alloc domain group for "
7576 "node %d\n", i); 7581 "node %d\n", i);
7577 goto error; 7582 goto error;
7578 } 7583 }
7579 sched_group_nodes[i] = sg; 7584 sched_group_nodes[i] = sg;
7580 for_each_cpu_mask_nr(j, *nodemask) { 7585 for_each_cpu(j, nodemask) {
7581 struct sched_domain *sd; 7586 struct sched_domain *sd;
7582 7587
7583 sd = &per_cpu(node_domains, j); 7588 sd = &per_cpu(node_domains, j);
7584 sd->groups = sg; 7589 sd->groups = sg;
7585 } 7590 }
7586 sg->__cpu_power = 0; 7591 sg->__cpu_power = 0;
7587 sg->cpumask = *nodemask; 7592 cpumask_copy(sched_group_cpus(sg), nodemask);
7588 sg->next = sg; 7593 sg->next = sg;
7589 cpus_or(*covered, *covered, *nodemask); 7594 cpumask_or(covered, covered, nodemask);
7590 prev = sg; 7595 prev = sg;
7591 7596
7592 for (j = 0; j < nr_node_ids; j++) { 7597 for (j = 0; j < nr_node_ids; j++) {
7593 SCHED_CPUMASK_VAR(notcovered, allmasks);
7594 int n = (i + j) % nr_node_ids; 7598 int n = (i + j) % nr_node_ids;
7599 /* FIXME: Use cpumask_of_node */
7595 node_to_cpumask_ptr(pnodemask, n); 7600 node_to_cpumask_ptr(pnodemask, n);
7596 7601
7597 cpus_complement(*notcovered, *covered); 7602 cpumask_complement(notcovered, covered);
7598 cpus_and(*tmpmask, *notcovered, *cpu_map); 7603 cpumask_and(tmpmask, notcovered, cpu_map);
7599 cpus_and(*tmpmask, *tmpmask, *domainspan); 7604 cpumask_and(tmpmask, tmpmask, domainspan);
7600 if (cpus_empty(*tmpmask)) 7605 if (cpumask_empty(tmpmask))
7601 break; 7606 break;
7602 7607
7603 cpus_and(*tmpmask, *tmpmask, *pnodemask); 7608 cpumask_and(tmpmask, tmpmask, pnodemask);
7604 if (cpus_empty(*tmpmask)) 7609 if (cpumask_empty(tmpmask))
7605 continue; 7610 continue;
7606 7611
7607 sg = kmalloc_node(sizeof(struct sched_group), 7612 sg = kmalloc_node(sizeof(struct sched_group) +
7613 cpumask_size(),
7608 GFP_KERNEL, i); 7614 GFP_KERNEL, i);
7609 if (!sg) { 7615 if (!sg) {
7610 printk(KERN_WARNING 7616 printk(KERN_WARNING
@@ -7612,9 +7618,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7612 goto error; 7618 goto error;
7613 } 7619 }
7614 sg->__cpu_power = 0; 7620 sg->__cpu_power = 0;
7615 sg->cpumask = *tmpmask; 7621 cpumask_copy(sched_group_cpus(sg), tmpmask);
7616 sg->next = prev->next; 7622 sg->next = prev->next;
7617 cpus_or(*covered, *covered, *tmpmask); 7623 cpumask_or(covered, covered, tmpmask);
7618 prev->next = sg; 7624 prev->next = sg;
7619 prev = sg; 7625 prev = sg;
7620 } 7626 }
@@ -7623,22 +7629,22 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7623 7629
7624 /* Calculate CPU power for physical packages and nodes */ 7630 /* Calculate CPU power for physical packages and nodes */
7625#ifdef CONFIG_SCHED_SMT 7631#ifdef CONFIG_SCHED_SMT
7626 for_each_cpu_mask_nr(i, *cpu_map) { 7632 for_each_cpu(i, cpu_map) {
7627 struct sched_domain *sd = &per_cpu(cpu_domains, i); 7633 struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
7628 7634
7629 init_sched_groups_power(i, sd); 7635 init_sched_groups_power(i, sd);
7630 } 7636 }
7631#endif 7637#endif
7632#ifdef CONFIG_SCHED_MC 7638#ifdef CONFIG_SCHED_MC
7633 for_each_cpu_mask_nr(i, *cpu_map) { 7639 for_each_cpu(i, cpu_map) {
7634 struct sched_domain *sd = &per_cpu(core_domains, i); 7640 struct sched_domain *sd = &per_cpu(core_domains, i).sd;
7635 7641
7636 init_sched_groups_power(i, sd); 7642 init_sched_groups_power(i, sd);
7637 } 7643 }
7638#endif 7644#endif
7639 7645
7640 for_each_cpu_mask_nr(i, *cpu_map) { 7646 for_each_cpu(i, cpu_map) {
7641 struct sched_domain *sd = &per_cpu(phys_domains, i); 7647 struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
7642 7648
7643 init_sched_groups_power(i, sd); 7649 init_sched_groups_power(i, sd);
7644 } 7650 }
@@ -7650,56 +7656,87 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7650 if (sd_allnodes) { 7656 if (sd_allnodes) {
7651 struct sched_group *sg; 7657 struct sched_group *sg;
7652 7658
7653 cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg, 7659 cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
7654 tmpmask); 7660 tmpmask);
7655 init_numa_sched_groups_power(sg); 7661 init_numa_sched_groups_power(sg);
7656 } 7662 }
7657#endif 7663#endif
7658 7664
7659 /* Attach the domains */ 7665 /* Attach the domains */
7660 for_each_cpu_mask_nr(i, *cpu_map) { 7666 for_each_cpu(i, cpu_map) {
7661 struct sched_domain *sd; 7667 struct sched_domain *sd;
7662#ifdef CONFIG_SCHED_SMT 7668#ifdef CONFIG_SCHED_SMT
7663 sd = &per_cpu(cpu_domains, i); 7669 sd = &per_cpu(cpu_domains, i).sd;
7664#elif defined(CONFIG_SCHED_MC) 7670#elif defined(CONFIG_SCHED_MC)
7665 sd = &per_cpu(core_domains, i); 7671 sd = &per_cpu(core_domains, i).sd;
7666#else 7672#else
7667 sd = &per_cpu(phys_domains, i); 7673 sd = &per_cpu(phys_domains, i).sd;
7668#endif 7674#endif
7669 cpu_attach_domain(sd, rd, i); 7675 cpu_attach_domain(sd, rd, i);
7670 } 7676 }
7671 7677
7672 SCHED_CPUMASK_FREE((void *)allmasks); 7678 err = 0;
7673 return 0; 7679
7680free_tmpmask:
7681 free_cpumask_var(tmpmask);
7682free_send_covered:
7683 free_cpumask_var(send_covered);
7684free_this_core_map:
7685 free_cpumask_var(this_core_map);
7686free_this_sibling_map:
7687 free_cpumask_var(this_sibling_map);
7688free_nodemask:
7689 free_cpumask_var(nodemask);
7690free_notcovered:
7691#ifdef CONFIG_NUMA
7692 free_cpumask_var(notcovered);
7693free_covered:
7694 free_cpumask_var(covered);
7695free_domainspan:
7696 free_cpumask_var(domainspan);
7697out:
7698#endif
7699 return err;
7700
7701free_sched_groups:
7702#ifdef CONFIG_NUMA
7703 kfree(sched_group_nodes);
7704#endif
7705 goto free_tmpmask;
7674 7706
7675#ifdef CONFIG_NUMA 7707#ifdef CONFIG_NUMA
7676error: 7708error:
7677 free_sched_groups(cpu_map, tmpmask); 7709 free_sched_groups(cpu_map, tmpmask);
7678 SCHED_CPUMASK_FREE((void *)allmasks); 7710 free_rootdomain(rd);
7679 kfree(rd); 7711 goto free_tmpmask;
7680 return -ENOMEM;
7681#endif 7712#endif
7682} 7713}
7683 7714
7684static int build_sched_domains(const cpumask_t *cpu_map) 7715static int build_sched_domains(const struct cpumask *cpu_map)
7685{ 7716{
7686 return __build_sched_domains(cpu_map, NULL); 7717 return __build_sched_domains(cpu_map, NULL);
7687} 7718}
7688 7719
7689static cpumask_t *doms_cur; /* current sched domains */ 7720static struct cpumask *doms_cur; /* current sched domains */
7690static int ndoms_cur; /* number of sched domains in 'doms_cur' */ 7721static int ndoms_cur; /* number of sched domains in 'doms_cur' */
7691static struct sched_domain_attr *dattr_cur; 7722static struct sched_domain_attr *dattr_cur;
7692 /* attribues of custom domains in 'doms_cur' */ 7723 /* attribues of custom domains in 'doms_cur' */
7693 7724
7694/* 7725/*
7695 * Special case: If a kmalloc of a doms_cur partition (array of 7726 * Special case: If a kmalloc of a doms_cur partition (array of
7696 * cpumask_t) fails, then fallback to a single sched domain, 7727 * cpumask) fails, then fallback to a single sched domain,
7697 * as determined by the single cpumask_t fallback_doms. 7728 * as determined by the single cpumask fallback_doms.
7698 */ 7729 */
7699static cpumask_t fallback_doms; 7730static cpumask_var_t fallback_doms;
7700 7731
7701void __attribute__((weak)) arch_update_cpu_topology(void) 7732/*
7733 * arch_update_cpu_topology lets virtualized architectures update the
7734 * cpu core maps. It is supposed to return 1 if the topology changed
7735 * or 0 if it stayed the same.
7736 */
7737int __attribute__((weak)) arch_update_cpu_topology(void)
7702{ 7738{
7739 return 0;
7703} 7740}
7704 7741
7705/* 7742/*
@@ -7707,16 +7744,16 @@ void __attribute__((weak)) arch_update_cpu_topology(void)
7707 * For now this just excludes isolated cpus, but could be used to 7744 * For now this just excludes isolated cpus, but could be used to
7708 * exclude other special cases in the future. 7745 * exclude other special cases in the future.
7709 */ 7746 */
7710static int arch_init_sched_domains(const cpumask_t *cpu_map) 7747static int arch_init_sched_domains(const struct cpumask *cpu_map)
7711{ 7748{
7712 int err; 7749 int err;
7713 7750
7714 arch_update_cpu_topology(); 7751 arch_update_cpu_topology();
7715 ndoms_cur = 1; 7752 ndoms_cur = 1;
7716 doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 7753 doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
7717 if (!doms_cur) 7754 if (!doms_cur)
7718 doms_cur = &fallback_doms; 7755 doms_cur = fallback_doms;
7719 cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map); 7756 cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
7720 dattr_cur = NULL; 7757 dattr_cur = NULL;
7721 err = build_sched_domains(doms_cur); 7758 err = build_sched_domains(doms_cur);
7722 register_sched_domain_sysctl(); 7759 register_sched_domain_sysctl();
@@ -7724,8 +7761,8 @@ static int arch_init_sched_domains(const cpumask_t *cpu_map)
7724 return err; 7761 return err;
7725} 7762}
7726 7763
7727static void arch_destroy_sched_domains(const cpumask_t *cpu_map, 7764static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
7728 cpumask_t *tmpmask) 7765 struct cpumask *tmpmask)
7729{ 7766{
7730 free_sched_groups(cpu_map, tmpmask); 7767 free_sched_groups(cpu_map, tmpmask);
7731} 7768}
@@ -7734,17 +7771,16 @@ static void arch_destroy_sched_domains(const cpumask_t *cpu_map,
7734 * Detach sched domains from a group of cpus specified in cpu_map 7771 * Detach sched domains from a group of cpus specified in cpu_map
7735 * These cpus will now be attached to the NULL domain 7772 * These cpus will now be attached to the NULL domain
7736 */ 7773 */
7737static void detach_destroy_domains(const cpumask_t *cpu_map) 7774static void detach_destroy_domains(const struct cpumask *cpu_map)
7738{ 7775{
7739 cpumask_t tmpmask; 7776 /* Save because hotplug lock held. */
7777 static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
7740 int i; 7778 int i;
7741 7779
7742 unregister_sched_domain_sysctl(); 7780 for_each_cpu(i, cpu_map)
7743
7744 for_each_cpu_mask_nr(i, *cpu_map)
7745 cpu_attach_domain(NULL, &def_root_domain, i); 7781 cpu_attach_domain(NULL, &def_root_domain, i);
7746 synchronize_sched(); 7782 synchronize_sched();
7747 arch_destroy_sched_domains(cpu_map, &tmpmask); 7783 arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
7748} 7784}
7749 7785
7750/* handle null as "default" */ 7786/* handle null as "default" */
@@ -7769,7 +7805,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7769 * doms_new[] to the current sched domain partitioning, doms_cur[]. 7805 * doms_new[] to the current sched domain partitioning, doms_cur[].
7770 * It destroys each deleted domain and builds each new domain. 7806 * It destroys each deleted domain and builds each new domain.
7771 * 7807 *
7772 * 'doms_new' is an array of cpumask_t's of length 'ndoms_new'. 7808 * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
7773 * The masks don't intersect (don't overlap.) We should setup one 7809 * The masks don't intersect (don't overlap.) We should setup one
7774 * sched domain for each mask. CPUs not in any of the cpumasks will 7810 * sched domain for each mask. CPUs not in any of the cpumasks will
7775 * not be load balanced. If the same cpumask appears both in the 7811 * not be load balanced. If the same cpumask appears both in the
@@ -7778,32 +7814,38 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7778 * 7814 *
7779 * The passed in 'doms_new' should be kmalloc'd. This routine takes 7815 * The passed in 'doms_new' should be kmalloc'd. This routine takes
7780 * ownership of it and will kfree it when done with it. If the caller 7816 * ownership of it and will kfree it when done with it. If the caller
7781 * failed the kmalloc call, then it can pass in doms_new == NULL, 7817 * failed the kmalloc call, then it can pass in doms_new == NULL &&
7782 * and partition_sched_domains() will fallback to the single partition 7818 * ndoms_new == 1, and partition_sched_domains() will fallback to
7783 * 'fallback_doms', it also forces the domains to be rebuilt. 7819 * the single partition 'fallback_doms', it also forces the domains
7820 * to be rebuilt.
7784 * 7821 *
7785 * If doms_new==NULL it will be replaced with cpu_online_map. 7822 * If doms_new == NULL it will be replaced with cpu_online_mask.
7786 * ndoms_new==0 is a special case for destroying existing domains. 7823 * ndoms_new == 0 is a special case for destroying existing domains,
7787 * It will not create the default domain. 7824 * and it will not create the default domain.
7788 * 7825 *
7789 * Call with hotplug lock held 7826 * Call with hotplug lock held
7790 */ 7827 */
7791void partition_sched_domains(int ndoms_new, cpumask_t *doms_new, 7828/* FIXME: Change to struct cpumask *doms_new[] */
7829void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
7792 struct sched_domain_attr *dattr_new) 7830 struct sched_domain_attr *dattr_new)
7793{ 7831{
7794 int i, j, n; 7832 int i, j, n;
7833 int new_topology;
7795 7834
7796 mutex_lock(&sched_domains_mutex); 7835 mutex_lock(&sched_domains_mutex);
7797 7836
7798 /* always unregister in case we don't destroy any domains */ 7837 /* always unregister in case we don't destroy any domains */
7799 unregister_sched_domain_sysctl(); 7838 unregister_sched_domain_sysctl();
7800 7839
7840 /* Let architecture update cpu core mappings. */
7841 new_topology = arch_update_cpu_topology();
7842
7801 n = doms_new ? ndoms_new : 0; 7843 n = doms_new ? ndoms_new : 0;
7802 7844
7803 /* Destroy deleted domains */ 7845 /* Destroy deleted domains */
7804 for (i = 0; i < ndoms_cur; i++) { 7846 for (i = 0; i < ndoms_cur; i++) {
7805 for (j = 0; j < n; j++) { 7847 for (j = 0; j < n && !new_topology; j++) {
7806 if (cpus_equal(doms_cur[i], doms_new[j]) 7848 if (cpumask_equal(&doms_cur[i], &doms_new[j])
7807 && dattrs_equal(dattr_cur, i, dattr_new, j)) 7849 && dattrs_equal(dattr_cur, i, dattr_new, j))
7808 goto match1; 7850 goto match1;
7809 } 7851 }
@@ -7815,15 +7857,15 @@ match1:
7815 7857
7816 if (doms_new == NULL) { 7858 if (doms_new == NULL) {
7817 ndoms_cur = 0; 7859 ndoms_cur = 0;
7818 doms_new = &fallback_doms; 7860 doms_new = fallback_doms;
7819 cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map); 7861 cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
7820 dattr_new = NULL; 7862 WARN_ON_ONCE(dattr_new);
7821 } 7863 }
7822 7864
7823 /* Build new domains */ 7865 /* Build new domains */
7824 for (i = 0; i < ndoms_new; i++) { 7866 for (i = 0; i < ndoms_new; i++) {
7825 for (j = 0; j < ndoms_cur; j++) { 7867 for (j = 0; j < ndoms_cur && !new_topology; j++) {
7826 if (cpus_equal(doms_new[i], doms_cur[j]) 7868 if (cpumask_equal(&doms_new[i], &doms_cur[j])
7827 && dattrs_equal(dattr_new, i, dattr_cur, j)) 7869 && dattrs_equal(dattr_new, i, dattr_cur, j))
7828 goto match2; 7870 goto match2;
7829 } 7871 }
@@ -7835,7 +7877,7 @@ match2:
7835 } 7877 }
7836 7878
7837 /* Remember the new sched domains */ 7879 /* Remember the new sched domains */
7838 if (doms_cur != &fallback_doms) 7880 if (doms_cur != fallback_doms)
7839 kfree(doms_cur); 7881 kfree(doms_cur);
7840 kfree(dattr_cur); /* kfree(NULL) is safe */ 7882 kfree(dattr_cur); /* kfree(NULL) is safe */
7841 doms_cur = doms_new; 7883 doms_cur = doms_new;
@@ -7975,7 +8017,9 @@ static int update_runtime(struct notifier_block *nfb,
7975 8017
7976void __init sched_init_smp(void) 8018void __init sched_init_smp(void)
7977{ 8019{
7978 cpumask_t non_isolated_cpus; 8020 cpumask_var_t non_isolated_cpus;
8021
8022 alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
7979 8023
7980#if defined(CONFIG_NUMA) 8024#if defined(CONFIG_NUMA)
7981 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **), 8025 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
@@ -7984,10 +8028,10 @@ void __init sched_init_smp(void)
7984#endif 8028#endif
7985 get_online_cpus(); 8029 get_online_cpus();
7986 mutex_lock(&sched_domains_mutex); 8030 mutex_lock(&sched_domains_mutex);
7987 arch_init_sched_domains(&cpu_online_map); 8031 arch_init_sched_domains(cpu_online_mask);
7988 cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map); 8032 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
7989 if (cpus_empty(non_isolated_cpus)) 8033 if (cpumask_empty(non_isolated_cpus))
7990 cpu_set(smp_processor_id(), non_isolated_cpus); 8034 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
7991 mutex_unlock(&sched_domains_mutex); 8035 mutex_unlock(&sched_domains_mutex);
7992 put_online_cpus(); 8036 put_online_cpus();
7993 8037
@@ -8002,9 +8046,13 @@ void __init sched_init_smp(void)
8002 init_hrtick(); 8046 init_hrtick();
8003 8047
8004 /* Move init over to a non-isolated CPU */ 8048 /* Move init over to a non-isolated CPU */
8005 if (set_cpus_allowed_ptr(current, &non_isolated_cpus) < 0) 8049 if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
8006 BUG(); 8050 BUG();
8007 sched_init_granularity(); 8051 sched_init_granularity();
8052 free_cpumask_var(non_isolated_cpus);
8053
8054 alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
8055 init_sched_rt_class();
8008} 8056}
8009#else 8057#else
8010void __init sched_init_smp(void) 8058void __init sched_init_smp(void)
@@ -8319,6 +8367,15 @@ void __init sched_init(void)
8319 */ 8367 */
8320 current->sched_class = &fair_sched_class; 8368 current->sched_class = &fair_sched_class;
8321 8369
8370 /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
8371 alloc_bootmem_cpumask_var(&nohz_cpu_mask);
8372#ifdef CONFIG_SMP
8373#ifdef CONFIG_NO_HZ
8374 alloc_bootmem_cpumask_var(&nohz.cpu_mask);
8375#endif
8376 alloc_bootmem_cpumask_var(&cpu_isolated_map);
8377#endif /* SMP */
8378
8322 scheduler_running = 1; 8379 scheduler_running = 1;
8323} 8380}
8324 8381
@@ -8477,7 +8534,7 @@ static
8477int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 8534int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
8478{ 8535{
8479 struct cfs_rq *cfs_rq; 8536 struct cfs_rq *cfs_rq;
8480 struct sched_entity *se, *parent_se; 8537 struct sched_entity *se;
8481 struct rq *rq; 8538 struct rq *rq;
8482 int i; 8539 int i;
8483 8540
@@ -8493,18 +8550,17 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
8493 for_each_possible_cpu(i) { 8550 for_each_possible_cpu(i) {
8494 rq = cpu_rq(i); 8551 rq = cpu_rq(i);
8495 8552
8496 cfs_rq = kmalloc_node(sizeof(struct cfs_rq), 8553 cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
8497 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8554 GFP_KERNEL, cpu_to_node(i));
8498 if (!cfs_rq) 8555 if (!cfs_rq)
8499 goto err; 8556 goto err;
8500 8557
8501 se = kmalloc_node(sizeof(struct sched_entity), 8558 se = kzalloc_node(sizeof(struct sched_entity),
8502 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8559 GFP_KERNEL, cpu_to_node(i));
8503 if (!se) 8560 if (!se)
8504 goto err; 8561 goto err;
8505 8562
8506 parent_se = parent ? parent->se[i] : NULL; 8563 init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
8507 init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent_se);
8508 } 8564 }
8509 8565
8510 return 1; 8566 return 1;
@@ -8565,7 +8621,7 @@ static
8565int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) 8621int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8566{ 8622{
8567 struct rt_rq *rt_rq; 8623 struct rt_rq *rt_rq;
8568 struct sched_rt_entity *rt_se, *parent_se; 8624 struct sched_rt_entity *rt_se;
8569 struct rq *rq; 8625 struct rq *rq;
8570 int i; 8626 int i;
8571 8627
@@ -8582,18 +8638,17 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8582 for_each_possible_cpu(i) { 8638 for_each_possible_cpu(i) {
8583 rq = cpu_rq(i); 8639 rq = cpu_rq(i);
8584 8640
8585 rt_rq = kmalloc_node(sizeof(struct rt_rq), 8641 rt_rq = kzalloc_node(sizeof(struct rt_rq),
8586 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8642 GFP_KERNEL, cpu_to_node(i));
8587 if (!rt_rq) 8643 if (!rt_rq)
8588 goto err; 8644 goto err;
8589 8645
8590 rt_se = kmalloc_node(sizeof(struct sched_rt_entity), 8646 rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
8591 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8647 GFP_KERNEL, cpu_to_node(i));
8592 if (!rt_se) 8648 if (!rt_se)
8593 goto err; 8649 goto err;
8594 8650
8595 parent_se = parent ? parent->rt_se[i] : NULL; 8651 init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
8596 init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent_se);
8597 } 8652 }
8598 8653
8599 return 1; 8654 return 1;
@@ -9236,11 +9291,12 @@ struct cgroup_subsys cpu_cgroup_subsys = {
9236 * (balbir@in.ibm.com). 9291 * (balbir@in.ibm.com).
9237 */ 9292 */
9238 9293
9239/* track cpu usage of a group of tasks */ 9294/* track cpu usage of a group of tasks and its child groups */
9240struct cpuacct { 9295struct cpuacct {
9241 struct cgroup_subsys_state css; 9296 struct cgroup_subsys_state css;
9242 /* cpuusage holds pointer to a u64-type object on every cpu */ 9297 /* cpuusage holds pointer to a u64-type object on every cpu */
9243 u64 *cpuusage; 9298 u64 *cpuusage;
9299 struct cpuacct *parent;
9244}; 9300};
9245 9301
9246struct cgroup_subsys cpuacct_subsys; 9302struct cgroup_subsys cpuacct_subsys;
@@ -9274,6 +9330,9 @@ static struct cgroup_subsys_state *cpuacct_create(
9274 return ERR_PTR(-ENOMEM); 9330 return ERR_PTR(-ENOMEM);
9275 } 9331 }
9276 9332
9333 if (cgrp->parent)
9334 ca->parent = cgroup_ca(cgrp->parent);
9335
9277 return &ca->css; 9336 return &ca->css;
9278} 9337}
9279 9338
@@ -9353,14 +9412,16 @@ static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
9353static void cpuacct_charge(struct task_struct *tsk, u64 cputime) 9412static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
9354{ 9413{
9355 struct cpuacct *ca; 9414 struct cpuacct *ca;
9415 int cpu;
9356 9416
9357 if (!cpuacct_subsys.active) 9417 if (!cpuacct_subsys.active)
9358 return; 9418 return;
9359 9419
9420 cpu = task_cpu(tsk);
9360 ca = task_ca(tsk); 9421 ca = task_ca(tsk);
9361 if (ca) {
9362 u64 *cpuusage = percpu_ptr(ca->cpuusage, task_cpu(tsk));
9363 9422
9423 for (; ca; ca = ca->parent) {
9424 u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
9364 *cpuusage += cputime; 9425 *cpuusage += cputime;
9365 } 9426 }
9366} 9427}
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-29 08:19:55 -0500