aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched.c')
-rw-r--r--kernel/sched.c1533
1 files changed, 870 insertions, 663 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index e4bb1dd7b308..52bbf1c842a8 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -118,7 +118,16 @@
118 */ 118 */
119#define RUNTIME_INF ((u64)~0ULL) 119#define RUNTIME_INF ((u64)~0ULL)
120 120
121DEFINE_TRACE(sched_wait_task);
122DEFINE_TRACE(sched_wakeup);
123DEFINE_TRACE(sched_wakeup_new);
124DEFINE_TRACE(sched_switch);
125DEFINE_TRACE(sched_migrate_task);
126
121#ifdef CONFIG_SMP 127#ifdef CONFIG_SMP
128
129static void double_rq_lock(struct rq *rq1, struct rq *rq2);
130
122/* 131/*
123 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power) 132 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
124 * Since cpu_power is a 'constant', we can use a reciprocal divide. 133 * Since cpu_power is a 'constant', we can use a reciprocal divide.
@@ -203,7 +212,6 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
203 hrtimer_init(&rt_b->rt_period_timer, 212 hrtimer_init(&rt_b->rt_period_timer,
204 CLOCK_MONOTONIC, HRTIMER_MODE_REL); 213 CLOCK_MONOTONIC, HRTIMER_MODE_REL);
205 rt_b->rt_period_timer.function = sched_rt_period_timer; 214 rt_b->rt_period_timer.function = sched_rt_period_timer;
206 rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;
207} 215}
208 216
209static inline int rt_bandwidth_enabled(void) 217static inline int rt_bandwidth_enabled(void)
@@ -261,6 +269,10 @@ struct task_group {
261 struct cgroup_subsys_state css; 269 struct cgroup_subsys_state css;
262#endif 270#endif
263 271
272#ifdef CONFIG_USER_SCHED
273 uid_t uid;
274#endif
275
264#ifdef CONFIG_FAIR_GROUP_SCHED 276#ifdef CONFIG_FAIR_GROUP_SCHED
265 /* schedulable entities of this group on each cpu */ 277 /* schedulable entities of this group on each cpu */
266 struct sched_entity **se; 278 struct sched_entity **se;
@@ -286,6 +298,12 @@ struct task_group {
286 298
287#ifdef CONFIG_USER_SCHED 299#ifdef CONFIG_USER_SCHED
288 300
301/* Helper function to pass uid information to create_sched_user() */
302void set_tg_uid(struct user_struct *user)
303{
304 user->tg->uid = user->uid;
305}
306
289/* 307/*
290 * Root task group. 308 * Root task group.
291 * Every UID task group (including init_task_group aka UID-0) will 309 * Every UID task group (including init_task_group aka UID-0) will
@@ -345,7 +363,9 @@ static inline struct task_group *task_group(struct task_struct *p)
345 struct task_group *tg; 363 struct task_group *tg;
346 364
347#ifdef CONFIG_USER_SCHED 365#ifdef CONFIG_USER_SCHED
348 tg = p->user->tg; 366 rcu_read_lock();
367 tg = __task_cred(p)->user->tg;
368 rcu_read_unlock();
349#elif defined(CONFIG_CGROUP_SCHED) 369#elif defined(CONFIG_CGROUP_SCHED)
350 tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id), 370 tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
351 struct task_group, css); 371 struct task_group, css);
@@ -481,18 +501,26 @@ struct rt_rq {
481 */ 501 */
482struct root_domain { 502struct root_domain {
483 atomic_t refcount; 503 atomic_t refcount;
484 cpumask_t span; 504 cpumask_var_t span;
485 cpumask_t online; 505 cpumask_var_t online;
486 506
487 /* 507 /*
488 * The "RT overload" flag: it gets set if a CPU has more than 508 * The "RT overload" flag: it gets set if a CPU has more than
489 * one runnable RT task. 509 * one runnable RT task.
490 */ 510 */
491 cpumask_t rto_mask; 511 cpumask_var_t rto_mask;
492 atomic_t rto_count; 512 atomic_t rto_count;
493#ifdef CONFIG_SMP 513#ifdef CONFIG_SMP
494 struct cpupri cpupri; 514 struct cpupri cpupri;
495#endif 515#endif
516#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
517 /*
518 * Preferred wake up cpu nominated by sched_mc balance that will be
519 * used when most cpus are idle in the system indicating overall very
520 * low system utilisation. Triggered at POWERSAVINGS_BALANCE_WAKEUP(2)
521 */
522 unsigned int sched_mc_preferred_wakeup_cpu;
523#endif
496}; 524};
497 525
498/* 526/*
@@ -586,6 +614,8 @@ struct rq {
586#ifdef CONFIG_SCHEDSTATS 614#ifdef CONFIG_SCHEDSTATS
587 /* latency stats */ 615 /* latency stats */
588 struct sched_info rq_sched_info; 616 struct sched_info rq_sched_info;
617 unsigned long long rq_cpu_time;
618 /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
589 619
590 /* sys_sched_yield() stats */ 620 /* sys_sched_yield() stats */
591 unsigned int yld_exp_empty; 621 unsigned int yld_exp_empty;
@@ -703,45 +733,18 @@ static __read_mostly char *sched_feat_names[] = {
703 733
704#undef SCHED_FEAT 734#undef SCHED_FEAT
705 735
706static int sched_feat_open(struct inode *inode, struct file *filp) 736static int sched_feat_show(struct seq_file *m, void *v)
707{
708 filp->private_data = inode->i_private;
709 return 0;
710}
711
712static ssize_t
713sched_feat_read(struct file *filp, char __user *ubuf,
714 size_t cnt, loff_t *ppos)
715{ 737{
716 char *buf;
717 int r = 0;
718 int len = 0;
719 int i; 738 int i;
720 739
721 for (i = 0; sched_feat_names[i]; i++) { 740 for (i = 0; sched_feat_names[i]; i++) {
722 len += strlen(sched_feat_names[i]); 741 if (!(sysctl_sched_features & (1UL << i)))
723 len += 4; 742 seq_puts(m, "NO_");
743 seq_printf(m, "%s ", sched_feat_names[i]);
724 } 744 }
745 seq_puts(m, "\n");
725 746
726 buf = kmalloc(len + 2, GFP_KERNEL); 747 return 0;
727 if (!buf)
728 return -ENOMEM;
729
730 for (i = 0; sched_feat_names[i]; i++) {
731 if (sysctl_sched_features & (1UL << i))
732 r += sprintf(buf + r, "%s ", sched_feat_names[i]);
733 else
734 r += sprintf(buf + r, "NO_%s ", sched_feat_names[i]);
735 }
736
737 r += sprintf(buf + r, "\n");
738 WARN_ON(r >= len + 2);
739
740 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
741
742 kfree(buf);
743
744 return r;
745} 748}
746 749
747static ssize_t 750static ssize_t
@@ -786,10 +789,17 @@ sched_feat_write(struct file *filp, const char __user *ubuf,
786 return cnt; 789 return cnt;
787} 790}
788 791
792static int sched_feat_open(struct inode *inode, struct file *filp)
793{
794 return single_open(filp, sched_feat_show, NULL);
795}
796
789static struct file_operations sched_feat_fops = { 797static struct file_operations sched_feat_fops = {
790 .open = sched_feat_open, 798 .open = sched_feat_open,
791 .read = sched_feat_read, 799 .write = sched_feat_write,
792 .write = sched_feat_write, 800 .read = seq_read,
801 .llseek = seq_lseek,
802 .release = single_release,
793}; 803};
794 804
795static __init int sched_init_debug(void) 805static __init int sched_init_debug(void)
@@ -1139,7 +1149,6 @@ static void init_rq_hrtick(struct rq *rq)
1139 1149
1140 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1150 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1141 rq->hrtick_timer.function = hrtick; 1151 rq->hrtick_timer.function = hrtick;
1142 rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
1143} 1152}
1144#else /* CONFIG_SCHED_HRTICK */ 1153#else /* CONFIG_SCHED_HRTICK */
1145static inline void hrtick_clear(struct rq *rq) 1154static inline void hrtick_clear(struct rq *rq)
@@ -1314,8 +1323,8 @@ static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
1314 * slice expiry etc. 1323 * slice expiry etc.
1315 */ 1324 */
1316 1325
1317#define WEIGHT_IDLEPRIO 2 1326#define WEIGHT_IDLEPRIO 3
1318#define WMULT_IDLEPRIO (1 << 31) 1327#define WMULT_IDLEPRIO 1431655765
1319 1328
1320/* 1329/*
1321 * Nice levels are multiplicative, with a gentle 10% change for every 1330 * Nice levels are multiplicative, with a gentle 10% change for every
@@ -1474,27 +1483,13 @@ static void
1474update_group_shares_cpu(struct task_group *tg, int cpu, 1483update_group_shares_cpu(struct task_group *tg, int cpu,
1475 unsigned long sd_shares, unsigned long sd_rq_weight) 1484 unsigned long sd_shares, unsigned long sd_rq_weight)
1476{ 1485{
1477 int boost = 0;
1478 unsigned long shares; 1486 unsigned long shares;
1479 unsigned long rq_weight; 1487 unsigned long rq_weight;
1480 1488
1481 if (!tg->se[cpu]) 1489 if (!tg->se[cpu])
1482 return; 1490 return;
1483 1491
1484 rq_weight = tg->cfs_rq[cpu]->load.weight; 1492 rq_weight = tg->cfs_rq[cpu]->rq_weight;
1485
1486 /*
1487 * If there are currently no tasks on the cpu pretend there is one of
1488 * average load so that when a new task gets to run here it will not
1489 * get delayed by group starvation.
1490 */
1491 if (!rq_weight) {
1492 boost = 1;
1493 rq_weight = NICE_0_LOAD;
1494 }
1495
1496 if (unlikely(rq_weight > sd_rq_weight))
1497 rq_weight = sd_rq_weight;
1498 1493
1499 /* 1494 /*
1500 * \Sum shares * rq_weight 1495 * \Sum shares * rq_weight
@@ -1502,7 +1497,7 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
1502 * \Sum rq_weight 1497 * \Sum rq_weight
1503 * 1498 *
1504 */ 1499 */
1505 shares = (sd_shares * rq_weight) / (sd_rq_weight + 1); 1500 shares = (sd_shares * rq_weight) / sd_rq_weight;
1506 shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES); 1501 shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
1507 1502
1508 if (abs(shares - tg->se[cpu]->load.weight) > 1503 if (abs(shares - tg->se[cpu]->load.weight) >
@@ -1511,11 +1506,7 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
1511 unsigned long flags; 1506 unsigned long flags;
1512 1507
1513 spin_lock_irqsave(&rq->lock, flags); 1508 spin_lock_irqsave(&rq->lock, flags);
1514 /* 1509 tg->cfs_rq[cpu]->shares = shares;
1515 * record the actual number of shares, not the boosted amount.
1516 */
1517 tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
1518 tg->cfs_rq[cpu]->rq_weight = rq_weight;
1519 1510
1520 __set_se_shares(tg->se[cpu], shares); 1511 __set_se_shares(tg->se[cpu], shares);
1521 spin_unlock_irqrestore(&rq->lock, flags); 1512 spin_unlock_irqrestore(&rq->lock, flags);
@@ -1529,13 +1520,23 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
1529 */ 1520 */
1530static int tg_shares_up(struct task_group *tg, void *data) 1521static int tg_shares_up(struct task_group *tg, void *data)
1531{ 1522{
1532 unsigned long rq_weight = 0; 1523 unsigned long weight, rq_weight = 0;
1533 unsigned long shares = 0; 1524 unsigned long shares = 0;
1534 struct sched_domain *sd = data; 1525 struct sched_domain *sd = data;
1535 int i; 1526 int i;
1536 1527
1537 for_each_cpu_mask(i, sd->span) { 1528 for_each_cpu(i, sched_domain_span(sd)) {
1538 rq_weight += tg->cfs_rq[i]->load.weight; 1529 /*
1530 * If there are currently no tasks on the cpu pretend there
1531 * is one of average load so that when a new task gets to
1532 * run here it will not get delayed by group starvation.
1533 */
1534 weight = tg->cfs_rq[i]->load.weight;
1535 if (!weight)
1536 weight = NICE_0_LOAD;
1537
1538 tg->cfs_rq[i]->rq_weight = weight;
1539 rq_weight += weight;
1539 shares += tg->cfs_rq[i]->shares; 1540 shares += tg->cfs_rq[i]->shares;
1540 } 1541 }
1541 1542
@@ -1545,10 +1546,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
1545 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE)) 1546 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
1546 shares = tg->shares; 1547 shares = tg->shares;
1547 1548
1548 if (!rq_weight) 1549 for_each_cpu(i, sched_domain_span(sd))
1549 rq_weight = cpus_weight(sd->span) * NICE_0_LOAD;
1550
1551 for_each_cpu_mask(i, sd->span)
1552 update_group_shares_cpu(tg, i, shares, rq_weight); 1550 update_group_shares_cpu(tg, i, shares, rq_weight);
1553 1551
1554 return 0; 1552 return 0;
@@ -1612,6 +1610,39 @@ static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
1612 1610
1613#endif 1611#endif
1614 1612
1613/*
1614 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
1615 */
1616static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
1617 __releases(this_rq->lock)
1618 __acquires(busiest->lock)
1619 __acquires(this_rq->lock)
1620{
1621 int ret = 0;
1622
1623 if (unlikely(!irqs_disabled())) {
1624 /* printk() doesn't work good under rq->lock */
1625 spin_unlock(&this_rq->lock);
1626 BUG_ON(1);
1627 }
1628 if (unlikely(!spin_trylock(&busiest->lock))) {
1629 if (busiest < this_rq) {
1630 spin_unlock(&this_rq->lock);
1631 spin_lock(&busiest->lock);
1632 spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
1633 ret = 1;
1634 } else
1635 spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
1636 }
1637 return ret;
1638}
1639
1640static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
1641 __releases(busiest->lock)
1642{
1643 spin_unlock(&busiest->lock);
1644 lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
1645}
1615#endif 1646#endif
1616 1647
1617#ifdef CONFIG_FAIR_GROUP_SCHED 1648#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -1845,6 +1876,8 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
1845 1876
1846 clock_offset = old_rq->clock - new_rq->clock; 1877 clock_offset = old_rq->clock - new_rq->clock;
1847 1878
1879 trace_sched_migrate_task(p, task_cpu(p), new_cpu);
1880
1848#ifdef CONFIG_SCHEDSTATS 1881#ifdef CONFIG_SCHEDSTATS
1849 if (p->se.wait_start) 1882 if (p->se.wait_start)
1850 p->se.wait_start -= clock_offset; 1883 p->se.wait_start -= clock_offset;
@@ -2079,15 +2112,17 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2079 int i; 2112 int i;
2080 2113
2081 /* Skip over this group if it has no CPUs allowed */ 2114 /* Skip over this group if it has no CPUs allowed */
2082 if (!cpus_intersects(group->cpumask, p->cpus_allowed)) 2115 if (!cpumask_intersects(sched_group_cpus(group),
2116 &p->cpus_allowed))
2083 continue; 2117 continue;
2084 2118
2085 local_group = cpu_isset(this_cpu, group->cpumask); 2119 local_group = cpumask_test_cpu(this_cpu,
2120 sched_group_cpus(group));
2086 2121
2087 /* Tally up the load of all CPUs in the group */ 2122 /* Tally up the load of all CPUs in the group */
2088 avg_load = 0; 2123 avg_load = 0;
2089 2124
2090 for_each_cpu_mask_nr(i, group->cpumask) { 2125 for_each_cpu(i, sched_group_cpus(group)) {
2091 /* Bias balancing toward cpus of our domain */ 2126 /* Bias balancing toward cpus of our domain */
2092 if (local_group) 2127 if (local_group)
2093 load = source_load(i, load_idx); 2128 load = source_load(i, load_idx);
@@ -2119,17 +2154,14 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2119 * find_idlest_cpu - find the idlest cpu among the cpus in group. 2154 * find_idlest_cpu - find the idlest cpu among the cpus in group.
2120 */ 2155 */
2121static int 2156static int
2122find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu, 2157find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
2123 cpumask_t *tmp)
2124{ 2158{
2125 unsigned long load, min_load = ULONG_MAX; 2159 unsigned long load, min_load = ULONG_MAX;
2126 int idlest = -1; 2160 int idlest = -1;
2127 int i; 2161 int i;
2128 2162
2129 /* Traverse only the allowed CPUs */ 2163 /* Traverse only the allowed CPUs */
2130 cpus_and(*tmp, group->cpumask, p->cpus_allowed); 2164 for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
2131
2132 for_each_cpu_mask_nr(i, *tmp) {
2133 load = weighted_cpuload(i); 2165 load = weighted_cpuload(i);
2134 2166
2135 if (load < min_load || (load == min_load && i == this_cpu)) { 2167 if (load < min_load || (load == min_load && i == this_cpu)) {
@@ -2171,7 +2203,6 @@ static int sched_balance_self(int cpu, int flag)
2171 update_shares(sd); 2203 update_shares(sd);
2172 2204
2173 while (sd) { 2205 while (sd) {
2174 cpumask_t span, tmpmask;
2175 struct sched_group *group; 2206 struct sched_group *group;
2176 int new_cpu, weight; 2207 int new_cpu, weight;
2177 2208
@@ -2180,14 +2211,13 @@ static int sched_balance_self(int cpu, int flag)
2180 continue; 2211 continue;
2181 } 2212 }
2182 2213
2183 span = sd->span;
2184 group = find_idlest_group(sd, t, cpu); 2214 group = find_idlest_group(sd, t, cpu);
2185 if (!group) { 2215 if (!group) {
2186 sd = sd->child; 2216 sd = sd->child;
2187 continue; 2217 continue;
2188 } 2218 }
2189 2219
2190 new_cpu = find_idlest_cpu(group, t, cpu, &tmpmask); 2220 new_cpu = find_idlest_cpu(group, t, cpu);
2191 if (new_cpu == -1 || new_cpu == cpu) { 2221 if (new_cpu == -1 || new_cpu == cpu) {
2192 /* Now try balancing at a lower domain level of cpu */ 2222 /* Now try balancing at a lower domain level of cpu */
2193 sd = sd->child; 2223 sd = sd->child;
@@ -2196,10 +2226,10 @@ static int sched_balance_self(int cpu, int flag)
2196 2226
2197 /* Now try balancing at a lower domain level of new_cpu */ 2227 /* Now try balancing at a lower domain level of new_cpu */
2198 cpu = new_cpu; 2228 cpu = new_cpu;
2229 weight = cpumask_weight(sched_domain_span(sd));
2199 sd = NULL; 2230 sd = NULL;
2200 weight = cpus_weight(span);
2201 for_each_domain(cpu, tmp) { 2231 for_each_domain(cpu, tmp) {
2202 if (weight <= cpus_weight(tmp->span)) 2232 if (weight <= cpumask_weight(sched_domain_span(tmp)))
2203 break; 2233 break;
2204 if (tmp->flags & flag) 2234 if (tmp->flags & flag)
2205 sd = tmp; 2235 sd = tmp;
@@ -2244,7 +2274,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2244 cpu = task_cpu(p); 2274 cpu = task_cpu(p);
2245 2275
2246 for_each_domain(this_cpu, sd) { 2276 for_each_domain(this_cpu, sd) {
2247 if (cpu_isset(cpu, sd->span)) { 2277 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2248 update_shares(sd); 2278 update_shares(sd);
2249 break; 2279 break;
2250 } 2280 }
@@ -2254,6 +2284,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2254 2284
2255 smp_wmb(); 2285 smp_wmb();
2256 rq = task_rq_lock(p, &flags); 2286 rq = task_rq_lock(p, &flags);
2287 update_rq_clock(rq);
2257 old_state = p->state; 2288 old_state = p->state;
2258 if (!(old_state & state)) 2289 if (!(old_state & state))
2259 goto out; 2290 goto out;
@@ -2292,7 +2323,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2292 else { 2323 else {
2293 struct sched_domain *sd; 2324 struct sched_domain *sd;
2294 for_each_domain(this_cpu, sd) { 2325 for_each_domain(this_cpu, sd) {
2295 if (cpu_isset(cpu, sd->span)) { 2326 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2296 schedstat_inc(sd, ttwu_wake_remote); 2327 schedstat_inc(sd, ttwu_wake_remote);
2297 break; 2328 break;
2298 } 2329 }
@@ -2311,12 +2342,11 @@ out_activate:
2311 schedstat_inc(p, se.nr_wakeups_local); 2342 schedstat_inc(p, se.nr_wakeups_local);
2312 else 2343 else
2313 schedstat_inc(p, se.nr_wakeups_remote); 2344 schedstat_inc(p, se.nr_wakeups_remote);
2314 update_rq_clock(rq);
2315 activate_task(rq, p, 1); 2345 activate_task(rq, p, 1);
2316 success = 1; 2346 success = 1;
2317 2347
2318out_running: 2348out_running:
2319 trace_sched_wakeup(rq, p); 2349 trace_sched_wakeup(rq, p, success);
2320 check_preempt_curr(rq, p, sync); 2350 check_preempt_curr(rq, p, sync);
2321 2351
2322 p->state = TASK_RUNNING; 2352 p->state = TASK_RUNNING;
@@ -2449,7 +2479,7 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
2449 p->sched_class->task_new(rq, p); 2479 p->sched_class->task_new(rq, p);
2450 inc_nr_running(rq); 2480 inc_nr_running(rq);
2451 } 2481 }
2452 trace_sched_wakeup_new(rq, p); 2482 trace_sched_wakeup_new(rq, p, 1);
2453 check_preempt_curr(rq, p, 0); 2483 check_preempt_curr(rq, p, 0);
2454#ifdef CONFIG_SMP 2484#ifdef CONFIG_SMP
2455 if (p->sched_class->task_wake_up) 2485 if (p->sched_class->task_wake_up)
@@ -2812,40 +2842,6 @@ static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
2812} 2842}
2813 2843
2814/* 2844/*
2815 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
2816 */
2817static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
2818 __releases(this_rq->lock)
2819 __acquires(busiest->lock)
2820 __acquires(this_rq->lock)
2821{
2822 int ret = 0;
2823
2824 if (unlikely(!irqs_disabled())) {
2825 /* printk() doesn't work good under rq->lock */
2826 spin_unlock(&this_rq->lock);
2827 BUG_ON(1);
2828 }
2829 if (unlikely(!spin_trylock(&busiest->lock))) {
2830 if (busiest < this_rq) {
2831 spin_unlock(&this_rq->lock);
2832 spin_lock(&busiest->lock);
2833 spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
2834 ret = 1;
2835 } else
2836 spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
2837 }
2838 return ret;
2839}
2840
2841static void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
2842 __releases(busiest->lock)
2843{
2844 spin_unlock(&busiest->lock);
2845 lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
2846}
2847
2848/*
2849 * If dest_cpu is allowed for this process, migrate the task to it. 2845 * If dest_cpu is allowed for this process, migrate the task to it.
2850 * This is accomplished by forcing the cpu_allowed mask to only 2846 * This is accomplished by forcing the cpu_allowed mask to only
2851 * allow dest_cpu, which will force the cpu onto dest_cpu. Then 2847 * allow dest_cpu, which will force the cpu onto dest_cpu. Then
@@ -2858,11 +2854,10 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu)
2858 struct rq *rq; 2854 struct rq *rq;
2859 2855
2860 rq = task_rq_lock(p, &flags); 2856 rq = task_rq_lock(p, &flags);
2861 if (!cpu_isset(dest_cpu, p->cpus_allowed) 2857 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
2862 || unlikely(!cpu_active(dest_cpu))) 2858 || unlikely(!cpu_active(dest_cpu)))
2863 goto out; 2859 goto out;
2864 2860
2865 trace_sched_migrate_task(rq, p, dest_cpu);
2866 /* force the process onto the specified CPU */ 2861 /* force the process onto the specified CPU */
2867 if (migrate_task(p, dest_cpu, &req)) { 2862 if (migrate_task(p, dest_cpu, &req)) {
2868 /* Need to wait for migration thread (might exit: take ref). */ 2863 /* Need to wait for migration thread (might exit: take ref). */
@@ -2924,7 +2919,7 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
2924 * 2) cannot be migrated to this CPU due to cpus_allowed, or 2919 * 2) cannot be migrated to this CPU due to cpus_allowed, or
2925 * 3) are cache-hot on their current CPU. 2920 * 3) are cache-hot on their current CPU.
2926 */ 2921 */
2927 if (!cpu_isset(this_cpu, p->cpus_allowed)) { 2922 if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
2928 schedstat_inc(p, se.nr_failed_migrations_affine); 2923 schedstat_inc(p, se.nr_failed_migrations_affine);
2929 return 0; 2924 return 0;
2930 } 2925 }
@@ -3099,7 +3094,7 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
3099static struct sched_group * 3094static struct sched_group *
3100find_busiest_group(struct sched_domain *sd, int this_cpu, 3095find_busiest_group(struct sched_domain *sd, int this_cpu,
3101 unsigned long *imbalance, enum cpu_idle_type idle, 3096 unsigned long *imbalance, enum cpu_idle_type idle,
3102 int *sd_idle, const cpumask_t *cpus, int *balance) 3097 int *sd_idle, const struct cpumask *cpus, int *balance)
3103{ 3098{
3104 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups; 3099 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
3105 unsigned long max_load, avg_load, total_load, this_load, total_pwr; 3100 unsigned long max_load, avg_load, total_load, this_load, total_pwr;
@@ -3135,10 +3130,11 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3135 unsigned long sum_avg_load_per_task; 3130 unsigned long sum_avg_load_per_task;
3136 unsigned long avg_load_per_task; 3131 unsigned long avg_load_per_task;
3137 3132
3138 local_group = cpu_isset(this_cpu, group->cpumask); 3133 local_group = cpumask_test_cpu(this_cpu,
3134 sched_group_cpus(group));
3139 3135
3140 if (local_group) 3136 if (local_group)
3141 balance_cpu = first_cpu(group->cpumask); 3137 balance_cpu = cpumask_first(sched_group_cpus(group));
3142 3138
3143 /* Tally up the load of all CPUs in the group */ 3139 /* Tally up the load of all CPUs in the group */
3144 sum_weighted_load = sum_nr_running = avg_load = 0; 3140 sum_weighted_load = sum_nr_running = avg_load = 0;
@@ -3147,13 +3143,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3147 max_cpu_load = 0; 3143 max_cpu_load = 0;
3148 min_cpu_load = ~0UL; 3144 min_cpu_load = ~0UL;
3149 3145
3150 for_each_cpu_mask_nr(i, group->cpumask) { 3146 for_each_cpu_and(i, sched_group_cpus(group), cpus) {
3151 struct rq *rq; 3147 struct rq *rq = cpu_rq(i);
3152
3153 if (!cpu_isset(i, *cpus))
3154 continue;
3155
3156 rq = cpu_rq(i);
3157 3148
3158 if (*sd_idle && rq->nr_running) 3149 if (*sd_idle && rq->nr_running)
3159 *sd_idle = 0; 3150 *sd_idle = 0;
@@ -3264,8 +3255,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3264 */ 3255 */
3265 if ((sum_nr_running < min_nr_running) || 3256 if ((sum_nr_running < min_nr_running) ||
3266 (sum_nr_running == min_nr_running && 3257 (sum_nr_running == min_nr_running &&
3267 first_cpu(group->cpumask) < 3258 cpumask_first(sched_group_cpus(group)) >
3268 first_cpu(group_min->cpumask))) { 3259 cpumask_first(sched_group_cpus(group_min)))) {
3269 group_min = group; 3260 group_min = group;
3270 min_nr_running = sum_nr_running; 3261 min_nr_running = sum_nr_running;
3271 min_load_per_task = sum_weighted_load / 3262 min_load_per_task = sum_weighted_load /
@@ -3280,8 +3271,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3280 if (sum_nr_running <= group_capacity - 1) { 3271 if (sum_nr_running <= group_capacity - 1) {
3281 if (sum_nr_running > leader_nr_running || 3272 if (sum_nr_running > leader_nr_running ||
3282 (sum_nr_running == leader_nr_running && 3273 (sum_nr_running == leader_nr_running &&
3283 first_cpu(group->cpumask) > 3274 cpumask_first(sched_group_cpus(group)) <
3284 first_cpu(group_leader->cpumask))) { 3275 cpumask_first(sched_group_cpus(group_leader)))) {
3285 group_leader = group; 3276 group_leader = group;
3286 leader_nr_running = sum_nr_running; 3277 leader_nr_running = sum_nr_running;
3287 } 3278 }
@@ -3407,6 +3398,10 @@ out_balanced:
3407 3398
3408 if (this == group_leader && group_leader != group_min) { 3399 if (this == group_leader && group_leader != group_min) {
3409 *imbalance = min_load_per_task; 3400 *imbalance = min_load_per_task;
3401 if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP) {
3402 cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu =
3403 cpumask_first(sched_group_cpus(group_leader));
3404 }
3410 return group_min; 3405 return group_min;
3411 } 3406 }
3412#endif 3407#endif
@@ -3420,16 +3415,16 @@ ret:
3420 */ 3415 */
3421static struct rq * 3416static struct rq *
3422find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle, 3417find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3423 unsigned long imbalance, const cpumask_t *cpus) 3418 unsigned long imbalance, const struct cpumask *cpus)
3424{ 3419{
3425 struct rq *busiest = NULL, *rq; 3420 struct rq *busiest = NULL, *rq;
3426 unsigned long max_load = 0; 3421 unsigned long max_load = 0;
3427 int i; 3422 int i;
3428 3423
3429 for_each_cpu_mask_nr(i, group->cpumask) { 3424 for_each_cpu(i, sched_group_cpus(group)) {
3430 unsigned long wl; 3425 unsigned long wl;
3431 3426
3432 if (!cpu_isset(i, *cpus)) 3427 if (!cpumask_test_cpu(i, cpus))
3433 continue; 3428 continue;
3434 3429
3435 rq = cpu_rq(i); 3430 rq = cpu_rq(i);
@@ -3459,7 +3454,7 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3459 */ 3454 */
3460static int load_balance(int this_cpu, struct rq *this_rq, 3455static int load_balance(int this_cpu, struct rq *this_rq,
3461 struct sched_domain *sd, enum cpu_idle_type idle, 3456 struct sched_domain *sd, enum cpu_idle_type idle,
3462 int *balance, cpumask_t *cpus) 3457 int *balance, struct cpumask *cpus)
3463{ 3458{
3464 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0; 3459 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
3465 struct sched_group *group; 3460 struct sched_group *group;
@@ -3467,7 +3462,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
3467 struct rq *busiest; 3462 struct rq *busiest;
3468 unsigned long flags; 3463 unsigned long flags;
3469 3464
3470 cpus_setall(*cpus); 3465 cpumask_setall(cpus);
3471 3466
3472 /* 3467 /*
3473 * When power savings policy is enabled for the parent domain, idle 3468 * When power savings policy is enabled for the parent domain, idle
@@ -3527,8 +3522,8 @@ redo:
3527 3522
3528 /* All tasks on this runqueue were pinned by CPU affinity */ 3523 /* All tasks on this runqueue were pinned by CPU affinity */
3529 if (unlikely(all_pinned)) { 3524 if (unlikely(all_pinned)) {
3530 cpu_clear(cpu_of(busiest), *cpus); 3525 cpumask_clear_cpu(cpu_of(busiest), cpus);
3531 if (!cpus_empty(*cpus)) 3526 if (!cpumask_empty(cpus))
3532 goto redo; 3527 goto redo;
3533 goto out_balanced; 3528 goto out_balanced;
3534 } 3529 }
@@ -3545,7 +3540,8 @@ redo:
3545 /* don't kick the migration_thread, if the curr 3540 /* don't kick the migration_thread, if the curr
3546 * task on busiest cpu can't be moved to this_cpu 3541 * task on busiest cpu can't be moved to this_cpu
3547 */ 3542 */
3548 if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) { 3543 if (!cpumask_test_cpu(this_cpu,
3544 &busiest->curr->cpus_allowed)) {
3549 spin_unlock_irqrestore(&busiest->lock, flags); 3545 spin_unlock_irqrestore(&busiest->lock, flags);
3550 all_pinned = 1; 3546 all_pinned = 1;
3551 goto out_one_pinned; 3547 goto out_one_pinned;
@@ -3620,7 +3616,7 @@ out:
3620 */ 3616 */
3621static int 3617static int
3622load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd, 3618load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3623 cpumask_t *cpus) 3619 struct cpumask *cpus)
3624{ 3620{
3625 struct sched_group *group; 3621 struct sched_group *group;
3626 struct rq *busiest = NULL; 3622 struct rq *busiest = NULL;
@@ -3629,7 +3625,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3629 int sd_idle = 0; 3625 int sd_idle = 0;
3630 int all_pinned = 0; 3626 int all_pinned = 0;
3631 3627
3632 cpus_setall(*cpus); 3628 cpumask_setall(cpus);
3633 3629
3634 /* 3630 /*
3635 * When power savings policy is enabled for the parent domain, idle 3631 * When power savings policy is enabled for the parent domain, idle
@@ -3673,17 +3669,76 @@ redo:
3673 double_unlock_balance(this_rq, busiest); 3669 double_unlock_balance(this_rq, busiest);
3674 3670
3675 if (unlikely(all_pinned)) { 3671 if (unlikely(all_pinned)) {
3676 cpu_clear(cpu_of(busiest), *cpus); 3672 cpumask_clear_cpu(cpu_of(busiest), cpus);
3677 if (!cpus_empty(*cpus)) 3673 if (!cpumask_empty(cpus))
3678 goto redo; 3674 goto redo;
3679 } 3675 }
3680 } 3676 }
3681 3677
3682 if (!ld_moved) { 3678 if (!ld_moved) {
3679 int active_balance = 0;
3680
3683 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]); 3681 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
3684 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER && 3682 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3685 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE)) 3683 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3686 return -1; 3684 return -1;
3685
3686 if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
3687 return -1;
3688
3689 if (sd->nr_balance_failed++ < 2)
3690 return -1;
3691
3692 /*
3693 * The only task running in a non-idle cpu can be moved to this
3694 * cpu in an attempt to completely freeup the other CPU
3695 * package. The same method used to move task in load_balance()
3696 * have been extended for load_balance_newidle() to speedup
3697 * consolidation at sched_mc=POWERSAVINGS_BALANCE_WAKEUP (2)
3698 *
3699 * The package power saving logic comes from
3700 * find_busiest_group(). If there are no imbalance, then
3701 * f_b_g() will return NULL. However when sched_mc={1,2} then
3702 * f_b_g() will select a group from which a running task may be
3703 * pulled to this cpu in order to make the other package idle.
3704 * If there is no opportunity to make a package idle and if
3705 * there are no imbalance, then f_b_g() will return NULL and no
3706 * action will be taken in load_balance_newidle().
3707 *
3708 * Under normal task pull operation due to imbalance, there
3709 * will be more than one task in the source run queue and
3710 * move_tasks() will succeed. ld_moved will be true and this
3711 * active balance code will not be triggered.
3712 */
3713
3714 /* Lock busiest in correct order while this_rq is held */
3715 double_lock_balance(this_rq, busiest);
3716
3717 /*
3718 * don't kick the migration_thread, if the curr
3719 * task on busiest cpu can't be moved to this_cpu
3720 */
3721 if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) {
3722 double_unlock_balance(this_rq, busiest);
3723 all_pinned = 1;
3724 return ld_moved;
3725 }
3726
3727 if (!busiest->active_balance) {
3728 busiest->active_balance = 1;
3729 busiest->push_cpu = this_cpu;
3730 active_balance = 1;
3731 }
3732
3733 double_unlock_balance(this_rq, busiest);
3734 /*
3735 * Should not call ttwu while holding a rq->lock
3736 */
3737 spin_unlock(&this_rq->lock);
3738 if (active_balance)
3739 wake_up_process(busiest->migration_thread);
3740 spin_lock(&this_rq->lock);
3741
3687 } else 3742 } else
3688 sd->nr_balance_failed = 0; 3743 sd->nr_balance_failed = 0;
3689 3744
@@ -3707,9 +3762,12 @@ out_balanced:
3707static void idle_balance(int this_cpu, struct rq *this_rq) 3762static void idle_balance(int this_cpu, struct rq *this_rq)
3708{ 3763{
3709 struct sched_domain *sd; 3764 struct sched_domain *sd;
3710 int pulled_task = -1; 3765 int pulled_task = 0;
3711 unsigned long next_balance = jiffies + HZ; 3766 unsigned long next_balance = jiffies + HZ;
3712 cpumask_t tmpmask; 3767 cpumask_var_t tmpmask;
3768
3769 if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
3770 return;
3713 3771
3714 for_each_domain(this_cpu, sd) { 3772 for_each_domain(this_cpu, sd) {
3715 unsigned long interval; 3773 unsigned long interval;
@@ -3720,7 +3778,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3720 if (sd->flags & SD_BALANCE_NEWIDLE) 3778 if (sd->flags & SD_BALANCE_NEWIDLE)
3721 /* If we've pulled tasks over stop searching: */ 3779 /* If we've pulled tasks over stop searching: */
3722 pulled_task = load_balance_newidle(this_cpu, this_rq, 3780 pulled_task = load_balance_newidle(this_cpu, this_rq,
3723 sd, &tmpmask); 3781 sd, tmpmask);
3724 3782
3725 interval = msecs_to_jiffies(sd->balance_interval); 3783 interval = msecs_to_jiffies(sd->balance_interval);
3726 if (time_after(next_balance, sd->last_balance + interval)) 3784 if (time_after(next_balance, sd->last_balance + interval))
@@ -3735,6 +3793,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3735 */ 3793 */
3736 this_rq->next_balance = next_balance; 3794 this_rq->next_balance = next_balance;
3737 } 3795 }
3796 free_cpumask_var(tmpmask);
3738} 3797}
3739 3798
3740/* 3799/*
@@ -3772,7 +3831,7 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3772 /* Search for an sd spanning us and the target CPU. */ 3831 /* Search for an sd spanning us and the target CPU. */
3773 for_each_domain(target_cpu, sd) { 3832 for_each_domain(target_cpu, sd) {
3774 if ((sd->flags & SD_LOAD_BALANCE) && 3833 if ((sd->flags & SD_LOAD_BALANCE) &&
3775 cpu_isset(busiest_cpu, sd->span)) 3834 cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
3776 break; 3835 break;
3777 } 3836 }
3778 3837
@@ -3791,10 +3850,9 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3791#ifdef CONFIG_NO_HZ 3850#ifdef CONFIG_NO_HZ
3792static struct { 3851static struct {
3793 atomic_t load_balancer; 3852 atomic_t load_balancer;
3794 cpumask_t cpu_mask; 3853 cpumask_var_t cpu_mask;
3795} nohz ____cacheline_aligned = { 3854} nohz ____cacheline_aligned = {
3796 .load_balancer = ATOMIC_INIT(-1), 3855 .load_balancer = ATOMIC_INIT(-1),
3797 .cpu_mask = CPU_MASK_NONE,
3798}; 3856};
3799 3857
3800/* 3858/*
@@ -3822,7 +3880,7 @@ int select_nohz_load_balancer(int stop_tick)
3822 int cpu = smp_processor_id(); 3880 int cpu = smp_processor_id();
3823 3881
3824 if (stop_tick) { 3882 if (stop_tick) {
3825 cpu_set(cpu, nohz.cpu_mask); 3883 cpumask_set_cpu(cpu, nohz.cpu_mask);
3826 cpu_rq(cpu)->in_nohz_recently = 1; 3884 cpu_rq(cpu)->in_nohz_recently = 1;
3827 3885
3828 /* 3886 /*
@@ -3836,7 +3894,7 @@ int select_nohz_load_balancer(int stop_tick)
3836 } 3894 }
3837 3895
3838 /* time for ilb owner also to sleep */ 3896 /* time for ilb owner also to sleep */
3839 if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 3897 if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
3840 if (atomic_read(&nohz.load_balancer) == cpu) 3898 if (atomic_read(&nohz.load_balancer) == cpu)
3841 atomic_set(&nohz.load_balancer, -1); 3899 atomic_set(&nohz.load_balancer, -1);
3842 return 0; 3900 return 0;
@@ -3849,10 +3907,10 @@ int select_nohz_load_balancer(int stop_tick)
3849 } else if (atomic_read(&nohz.load_balancer) == cpu) 3907 } else if (atomic_read(&nohz.load_balancer) == cpu)
3850 return 1; 3908 return 1;
3851 } else { 3909 } else {
3852 if (!cpu_isset(cpu, nohz.cpu_mask)) 3910 if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
3853 return 0; 3911 return 0;
3854 3912
3855 cpu_clear(cpu, nohz.cpu_mask); 3913 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3856 3914
3857 if (atomic_read(&nohz.load_balancer) == cpu) 3915 if (atomic_read(&nohz.load_balancer) == cpu)
3858 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu) 3916 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
@@ -3880,7 +3938,11 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3880 unsigned long next_balance = jiffies + 60*HZ; 3938 unsigned long next_balance = jiffies + 60*HZ;
3881 int update_next_balance = 0; 3939 int update_next_balance = 0;
3882 int need_serialize; 3940 int need_serialize;
3883 cpumask_t tmp; 3941 cpumask_var_t tmp;
3942
3943 /* Fails alloc? Rebalancing probably not a priority right now. */
3944 if (!alloc_cpumask_var(&tmp, GFP_ATOMIC))
3945 return;
3884 3946
3885 for_each_domain(cpu, sd) { 3947 for_each_domain(cpu, sd) {
3886 if (!(sd->flags & SD_LOAD_BALANCE)) 3948 if (!(sd->flags & SD_LOAD_BALANCE))
@@ -3905,7 +3967,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3905 } 3967 }
3906 3968
3907 if (time_after_eq(jiffies, sd->last_balance + interval)) { 3969 if (time_after_eq(jiffies, sd->last_balance + interval)) {
3908 if (load_balance(cpu, rq, sd, idle, &balance, &tmp)) { 3970 if (load_balance(cpu, rq, sd, idle, &balance, tmp)) {
3909 /* 3971 /*
3910 * We've pulled tasks over so either we're no 3972 * We've pulled tasks over so either we're no
3911 * longer idle, or one of our SMT siblings is 3973 * longer idle, or one of our SMT siblings is
@@ -3939,6 +4001,8 @@ out:
3939 */ 4001 */
3940 if (likely(update_next_balance)) 4002 if (likely(update_next_balance))
3941 rq->next_balance = next_balance; 4003 rq->next_balance = next_balance;
4004
4005 free_cpumask_var(tmp);
3942} 4006}
3943 4007
3944/* 4008/*
@@ -3963,12 +4027,13 @@ static void run_rebalance_domains(struct softirq_action *h)
3963 */ 4027 */
3964 if (this_rq->idle_at_tick && 4028 if (this_rq->idle_at_tick &&
3965 atomic_read(&nohz.load_balancer) == this_cpu) { 4029 atomic_read(&nohz.load_balancer) == this_cpu) {
3966 cpumask_t cpus = nohz.cpu_mask;
3967 struct rq *rq; 4030 struct rq *rq;
3968 int balance_cpu; 4031 int balance_cpu;
3969 4032
3970 cpu_clear(this_cpu, cpus); 4033 for_each_cpu(balance_cpu, nohz.cpu_mask) {
3971 for_each_cpu_mask_nr(balance_cpu, cpus) { 4034 if (balance_cpu == this_cpu)
4035 continue;
4036
3972 /* 4037 /*
3973 * If this cpu gets work to do, stop the load balancing 4038 * If this cpu gets work to do, stop the load balancing
3974 * work being done for other cpus. Next load 4039 * work being done for other cpus. Next load
@@ -4006,7 +4071,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4006 rq->in_nohz_recently = 0; 4071 rq->in_nohz_recently = 0;
4007 4072
4008 if (atomic_read(&nohz.load_balancer) == cpu) { 4073 if (atomic_read(&nohz.load_balancer) == cpu) {
4009 cpu_clear(cpu, nohz.cpu_mask); 4074 cpumask_clear_cpu(cpu, nohz.cpu_mask);
4010 atomic_set(&nohz.load_balancer, -1); 4075 atomic_set(&nohz.load_balancer, -1);
4011 } 4076 }
4012 4077
@@ -4019,7 +4084,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4019 * TBD: Traverse the sched domains and nominate 4084 * TBD: Traverse the sched domains and nominate
4020 * the nearest cpu in the nohz.cpu_mask. 4085 * the nearest cpu in the nohz.cpu_mask.
4021 */ 4086 */
4022 int ilb = first_cpu(nohz.cpu_mask); 4087 int ilb = cpumask_first(nohz.cpu_mask);
4023 4088
4024 if (ilb < nr_cpu_ids) 4089 if (ilb < nr_cpu_ids)
4025 resched_cpu(ilb); 4090 resched_cpu(ilb);
@@ -4031,7 +4096,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4031 * cpus with ticks stopped, is it time for that to stop? 4096 * cpus with ticks stopped, is it time for that to stop?
4032 */ 4097 */
4033 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu && 4098 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
4034 cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 4099 cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
4035 resched_cpu(cpu); 4100 resched_cpu(cpu);
4036 return; 4101 return;
4037 } 4102 }
@@ -4041,7 +4106,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4041 * someone else, then no need raise the SCHED_SOFTIRQ 4106 * someone else, then no need raise the SCHED_SOFTIRQ
4042 */ 4107 */
4043 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu && 4108 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
4044 cpu_isset(cpu, nohz.cpu_mask)) 4109 cpumask_test_cpu(cpu, nohz.cpu_mask))
4045 return; 4110 return;
4046#endif 4111#endif
4047 if (time_after_eq(jiffies, rq->next_balance)) 4112 if (time_after_eq(jiffies, rq->next_balance))
@@ -4093,13 +4158,17 @@ unsigned long long task_delta_exec(struct task_struct *p)
4093 * Account user cpu time to a process. 4158 * Account user cpu time to a process.
4094 * @p: the process that the cpu time gets accounted to 4159 * @p: the process that the cpu time gets accounted to
4095 * @cputime: the cpu time spent in user space since the last update 4160 * @cputime: the cpu time spent in user space since the last update
4161 * @cputime_scaled: cputime scaled by cpu frequency
4096 */ 4162 */
4097void account_user_time(struct task_struct *p, cputime_t cputime) 4163void account_user_time(struct task_struct *p, cputime_t cputime,
4164 cputime_t cputime_scaled)
4098{ 4165{
4099 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; 4166 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4100 cputime64_t tmp; 4167 cputime64_t tmp;
4101 4168
4169 /* Add user time to process. */
4102 p->utime = cputime_add(p->utime, cputime); 4170 p->utime = cputime_add(p->utime, cputime);
4171 p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
4103 account_group_user_time(p, cputime); 4172 account_group_user_time(p, cputime);
4104 4173
4105 /* Add user time to cpustat. */ 4174 /* Add user time to cpustat. */
@@ -4116,51 +4185,48 @@ void account_user_time(struct task_struct *p, cputime_t cputime)
4116 * Account guest cpu time to a process. 4185 * Account guest cpu time to a process.
4117 * @p: the process that the cpu time gets accounted to 4186 * @p: the process that the cpu time gets accounted to
4118 * @cputime: the cpu time spent in virtual machine since the last update 4187 * @cputime: the cpu time spent in virtual machine since the last update
4188 * @cputime_scaled: cputime scaled by cpu frequency
4119 */ 4189 */
4120static void account_guest_time(struct task_struct *p, cputime_t cputime) 4190static void account_guest_time(struct task_struct *p, cputime_t cputime,
4191 cputime_t cputime_scaled)
4121{ 4192{
4122 cputime64_t tmp; 4193 cputime64_t tmp;
4123 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; 4194 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4124 4195
4125 tmp = cputime_to_cputime64(cputime); 4196 tmp = cputime_to_cputime64(cputime);
4126 4197
4198 /* Add guest time to process. */
4127 p->utime = cputime_add(p->utime, cputime); 4199 p->utime = cputime_add(p->utime, cputime);
4200 p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
4128 account_group_user_time(p, cputime); 4201 account_group_user_time(p, cputime);
4129 p->gtime = cputime_add(p->gtime, cputime); 4202 p->gtime = cputime_add(p->gtime, cputime);
4130 4203
4204 /* Add guest time to cpustat. */
4131 cpustat->user = cputime64_add(cpustat->user, tmp); 4205 cpustat->user = cputime64_add(cpustat->user, tmp);
4132 cpustat->guest = cputime64_add(cpustat->guest, tmp); 4206 cpustat->guest = cputime64_add(cpustat->guest, tmp);
4133} 4207}
4134 4208
4135/* 4209/*
4136 * Account scaled user cpu time to a process.
4137 * @p: the process that the cpu time gets accounted to
4138 * @cputime: the cpu time spent in user space since the last update
4139 */
4140void account_user_time_scaled(struct task_struct *p, cputime_t cputime)
4141{
4142 p->utimescaled = cputime_add(p->utimescaled, cputime);
4143}
4144
4145/*
4146 * Account system cpu time to a process. 4210 * Account system cpu time to a process.
4147 * @p: the process that the cpu time gets accounted to 4211 * @p: the process that the cpu time gets accounted to
4148 * @hardirq_offset: the offset to subtract from hardirq_count() 4212 * @hardirq_offset: the offset to subtract from hardirq_count()
4149 * @cputime: the cpu time spent in kernel space since the last update 4213 * @cputime: the cpu time spent in kernel space since the last update
4214 * @cputime_scaled: cputime scaled by cpu frequency
4150 */ 4215 */
4151void account_system_time(struct task_struct *p, int hardirq_offset, 4216void account_system_time(struct task_struct *p, int hardirq_offset,
4152 cputime_t cputime) 4217 cputime_t cputime, cputime_t cputime_scaled)
4153{ 4218{
4154 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; 4219 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4155 struct rq *rq = this_rq();
4156 cputime64_t tmp; 4220 cputime64_t tmp;
4157 4221
4158 if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) { 4222 if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
4159 account_guest_time(p, cputime); 4223 account_guest_time(p, cputime, cputime_scaled);
4160 return; 4224 return;
4161 } 4225 }
4162 4226
4227 /* Add system time to process. */
4163 p->stime = cputime_add(p->stime, cputime); 4228 p->stime = cputime_add(p->stime, cputime);
4229 p->stimescaled = cputime_add(p->stimescaled, cputime_scaled);
4164 account_group_system_time(p, cputime); 4230 account_group_system_time(p, cputime);
4165 4231
4166 /* Add system time to cpustat. */ 4232 /* Add system time to cpustat. */
@@ -4169,49 +4235,84 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
4169 cpustat->irq = cputime64_add(cpustat->irq, tmp); 4235 cpustat->irq = cputime64_add(cpustat->irq, tmp);
4170 else if (softirq_count()) 4236 else if (softirq_count())
4171 cpustat->softirq = cputime64_add(cpustat->softirq, tmp); 4237 cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
4172 else if (p != rq->idle)
4173 cpustat->system = cputime64_add(cpustat->system, tmp);
4174 else if (atomic_read(&rq->nr_iowait) > 0)
4175 cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
4176 else 4238 else
4177 cpustat->idle = cputime64_add(cpustat->idle, tmp); 4239 cpustat->system = cputime64_add(cpustat->system, tmp);
4240
4178 /* Account for system time used */ 4241 /* Account for system time used */
4179 acct_update_integrals(p); 4242 acct_update_integrals(p);
4180} 4243}
4181 4244
4182/* 4245/*
4183 * Account scaled system cpu time to a process. 4246 * Account for involuntary wait time.
4184 * @p: the process that the cpu time gets accounted to 4247 * @steal: the cpu time spent in involuntary wait
4185 * @hardirq_offset: the offset to subtract from hardirq_count()
4186 * @cputime: the cpu time spent in kernel space since the last update
4187 */ 4248 */
4188void account_system_time_scaled(struct task_struct *p, cputime_t cputime) 4249void account_steal_time(cputime_t cputime)
4189{ 4250{
4190 p->stimescaled = cputime_add(p->stimescaled, cputime); 4251 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4252 cputime64_t cputime64 = cputime_to_cputime64(cputime);
4253
4254 cpustat->steal = cputime64_add(cpustat->steal, cputime64);
4191} 4255}
4192 4256
4193/* 4257/*
4194 * Account for involuntary wait time. 4258 * Account for idle time.
4195 * @p: the process from which the cpu time has been stolen 4259 * @cputime: the cpu time spent in idle wait
4196 * @steal: the cpu time spent in involuntary wait
4197 */ 4260 */
4198void account_steal_time(struct task_struct *p, cputime_t steal) 4261void account_idle_time(cputime_t cputime)
4199{ 4262{
4200 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; 4263 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4201 cputime64_t tmp = cputime_to_cputime64(steal); 4264 cputime64_t cputime64 = cputime_to_cputime64(cputime);
4202 struct rq *rq = this_rq(); 4265 struct rq *rq = this_rq();
4203 4266
4204 if (p == rq->idle) { 4267 if (atomic_read(&rq->nr_iowait) > 0)
4205 p->stime = cputime_add(p->stime, steal); 4268 cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
4206 account_group_system_time(p, steal); 4269 else
4207 if (atomic_read(&rq->nr_iowait) > 0) 4270 cpustat->idle = cputime64_add(cpustat->idle, cputime64);
4208 cpustat->iowait = cputime64_add(cpustat->iowait, tmp); 4271}
4209 else 4272
4210 cpustat->idle = cputime64_add(cpustat->idle, tmp); 4273#ifndef CONFIG_VIRT_CPU_ACCOUNTING
4211 } else 4274
4212 cpustat->steal = cputime64_add(cpustat->steal, tmp); 4275/*
4276 * Account a single tick of cpu time.
4277 * @p: the process that the cpu time gets accounted to
4278 * @user_tick: indicates if the tick is a user or a system tick
4279 */
4280void account_process_tick(struct task_struct *p, int user_tick)
4281{
4282 cputime_t one_jiffy = jiffies_to_cputime(1);
4283 cputime_t one_jiffy_scaled = cputime_to_scaled(one_jiffy);
4284 struct rq *rq = this_rq();
4285
4286 if (user_tick)
4287 account_user_time(p, one_jiffy, one_jiffy_scaled);
4288 else if (p != rq->idle)
4289 account_system_time(p, HARDIRQ_OFFSET, one_jiffy,
4290 one_jiffy_scaled);
4291 else
4292 account_idle_time(one_jiffy);
4293}
4294
4295/*
4296 * Account multiple ticks of steal time.
4297 * @p: the process from which the cpu time has been stolen
4298 * @ticks: number of stolen ticks
4299 */
4300void account_steal_ticks(unsigned long ticks)
4301{
4302 account_steal_time(jiffies_to_cputime(ticks));
4303}
4304
4305/*
4306 * Account multiple ticks of idle time.
4307 * @ticks: number of stolen ticks
4308 */
4309void account_idle_ticks(unsigned long ticks)
4310{
4311 account_idle_time(jiffies_to_cputime(ticks));
4213} 4312}
4214 4313
4314#endif
4315
4215/* 4316/*
4216 * Use precise platform statistics if available: 4317 * Use precise platform statistics if available:
4217 */ 4318 */
@@ -5025,7 +5126,7 @@ int can_nice(const struct task_struct *p, const int nice)
5025 * sys_setpriority is a more generic, but much slower function that 5126 * sys_setpriority is a more generic, but much slower function that
5026 * does similar things. 5127 * does similar things.
5027 */ 5128 */
5028asmlinkage long sys_nice(int increment) 5129SYSCALL_DEFINE1(nice, int, increment)
5029{ 5130{
5030 long nice, retval; 5131 long nice, retval;
5031 5132
@@ -5134,6 +5235,22 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
5134 set_load_weight(p); 5235 set_load_weight(p);
5135} 5236}
5136 5237
5238/*
5239 * check the target process has a UID that matches the current process's
5240 */
5241static bool check_same_owner(struct task_struct *p)
5242{
5243 const struct cred *cred = current_cred(), *pcred;
5244 bool match;
5245
5246 rcu_read_lock();
5247 pcred = __task_cred(p);
5248 match = (cred->euid == pcred->euid ||
5249 cred->euid == pcred->uid);
5250 rcu_read_unlock();
5251 return match;
5252}
5253
5137static int __sched_setscheduler(struct task_struct *p, int policy, 5254static int __sched_setscheduler(struct task_struct *p, int policy,
5138 struct sched_param *param, bool user) 5255 struct sched_param *param, bool user)
5139{ 5256{
@@ -5193,8 +5310,7 @@ recheck:
5193 return -EPERM; 5310 return -EPERM;
5194 5311
5195 /* can't change other user's priorities */ 5312 /* can't change other user's priorities */
5196 if ((current->euid != p->euid) && 5313 if (!check_same_owner(p))
5197 (current->euid != p->uid))
5198 return -EPERM; 5314 return -EPERM;
5199 } 5315 }
5200 5316
@@ -5317,8 +5433,8 @@ do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
5317 * @policy: new policy. 5433 * @policy: new policy.
5318 * @param: structure containing the new RT priority. 5434 * @param: structure containing the new RT priority.
5319 */ 5435 */
5320asmlinkage long 5436SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
5321sys_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) 5437 struct sched_param __user *, param)
5322{ 5438{
5323 /* negative values for policy are not valid */ 5439 /* negative values for policy are not valid */
5324 if (policy < 0) 5440 if (policy < 0)
@@ -5332,7 +5448,7 @@ sys_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
5332 * @pid: the pid in question. 5448 * @pid: the pid in question.
5333 * @param: structure containing the new RT priority. 5449 * @param: structure containing the new RT priority.
5334 */ 5450 */
5335asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param) 5451SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
5336{ 5452{
5337 return do_sched_setscheduler(pid, -1, param); 5453 return do_sched_setscheduler(pid, -1, param);
5338} 5454}
@@ -5341,7 +5457,7 @@ asmlinkage long sys_sched_setparam(pid_t pid, struct sched_param __user *param)
5341 * sys_sched_getscheduler - get the policy (scheduling class) of a thread 5457 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
5342 * @pid: the pid in question. 5458 * @pid: the pid in question.
5343 */ 5459 */
5344asmlinkage long sys_sched_getscheduler(pid_t pid) 5460SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
5345{ 5461{
5346 struct task_struct *p; 5462 struct task_struct *p;
5347 int retval; 5463 int retval;
@@ -5366,7 +5482,7 @@ asmlinkage long sys_sched_getscheduler(pid_t pid)
5366 * @pid: the pid in question. 5482 * @pid: the pid in question.
5367 * @param: structure containing the RT priority. 5483 * @param: structure containing the RT priority.
5368 */ 5484 */
5369asmlinkage long sys_sched_getparam(pid_t pid, struct sched_param __user *param) 5485SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
5370{ 5486{
5371 struct sched_param lp; 5487 struct sched_param lp;
5372 struct task_struct *p; 5488 struct task_struct *p;
@@ -5400,10 +5516,9 @@ out_unlock:
5400 return retval; 5516 return retval;
5401} 5517}
5402 5518
5403long sched_setaffinity(pid_t pid, const cpumask_t *in_mask) 5519long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
5404{ 5520{
5405 cpumask_t cpus_allowed; 5521 cpumask_var_t cpus_allowed, new_mask;
5406 cpumask_t new_mask = *in_mask;
5407 struct task_struct *p; 5522 struct task_struct *p;
5408 int retval; 5523 int retval;
5409 5524
@@ -5425,46 +5540,57 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5425 get_task_struct(p); 5540 get_task_struct(p);
5426 read_unlock(&tasklist_lock); 5541 read_unlock(&tasklist_lock);
5427 5542
5543 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
5544 retval = -ENOMEM;
5545 goto out_put_task;
5546 }
5547 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
5548 retval = -ENOMEM;
5549 goto out_free_cpus_allowed;
5550 }
5428 retval = -EPERM; 5551 retval = -EPERM;
5429 if ((current->euid != p->euid) && (current->euid != p->uid) && 5552 if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
5430 !capable(CAP_SYS_NICE))
5431 goto out_unlock; 5553 goto out_unlock;
5432 5554
5433 retval = security_task_setscheduler(p, 0, NULL); 5555 retval = security_task_setscheduler(p, 0, NULL);
5434 if (retval) 5556 if (retval)
5435 goto out_unlock; 5557 goto out_unlock;
5436 5558
5437 cpuset_cpus_allowed(p, &cpus_allowed); 5559 cpuset_cpus_allowed(p, cpus_allowed);
5438 cpus_and(new_mask, new_mask, cpus_allowed); 5560 cpumask_and(new_mask, in_mask, cpus_allowed);
5439 again: 5561 again:
5440 retval = set_cpus_allowed_ptr(p, &new_mask); 5562 retval = set_cpus_allowed_ptr(p, new_mask);
5441 5563
5442 if (!retval) { 5564 if (!retval) {
5443 cpuset_cpus_allowed(p, &cpus_allowed); 5565 cpuset_cpus_allowed(p, cpus_allowed);
5444 if (!cpus_subset(new_mask, cpus_allowed)) { 5566 if (!cpumask_subset(new_mask, cpus_allowed)) {
5445 /* 5567 /*
5446 * We must have raced with a concurrent cpuset 5568 * We must have raced with a concurrent cpuset
5447 * update. Just reset the cpus_allowed to the 5569 * update. Just reset the cpus_allowed to the
5448 * cpuset's cpus_allowed 5570 * cpuset's cpus_allowed
5449 */ 5571 */
5450 new_mask = cpus_allowed; 5572 cpumask_copy(new_mask, cpus_allowed);
5451 goto again; 5573 goto again;
5452 } 5574 }
5453 } 5575 }
5454out_unlock: 5576out_unlock:
5577 free_cpumask_var(new_mask);
5578out_free_cpus_allowed:
5579 free_cpumask_var(cpus_allowed);
5580out_put_task:
5455 put_task_struct(p); 5581 put_task_struct(p);
5456 put_online_cpus(); 5582 put_online_cpus();
5457 return retval; 5583 return retval;
5458} 5584}
5459 5585
5460static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 5586static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5461 cpumask_t *new_mask) 5587 struct cpumask *new_mask)
5462{ 5588{
5463 if (len < sizeof(cpumask_t)) { 5589 if (len < cpumask_size())
5464 memset(new_mask, 0, sizeof(cpumask_t)); 5590 cpumask_clear(new_mask);
5465 } else if (len > sizeof(cpumask_t)) { 5591 else if (len > cpumask_size())
5466 len = sizeof(cpumask_t); 5592 len = cpumask_size();
5467 } 5593
5468 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 5594 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
5469} 5595}
5470 5596
@@ -5474,20 +5600,23 @@ static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5474 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 5600 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
5475 * @user_mask_ptr: user-space pointer to the new cpu mask 5601 * @user_mask_ptr: user-space pointer to the new cpu mask
5476 */ 5602 */
5477asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, 5603SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
5478 unsigned long __user *user_mask_ptr) 5604 unsigned long __user *, user_mask_ptr)
5479{ 5605{
5480 cpumask_t new_mask; 5606 cpumask_var_t new_mask;
5481 int retval; 5607 int retval;
5482 5608
5483 retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask); 5609 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
5484 if (retval) 5610 return -ENOMEM;
5485 return retval;
5486 5611
5487 return sched_setaffinity(pid, &new_mask); 5612 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
5613 if (retval == 0)
5614 retval = sched_setaffinity(pid, new_mask);
5615 free_cpumask_var(new_mask);
5616 return retval;
5488} 5617}
5489 5618
5490long sched_getaffinity(pid_t pid, cpumask_t *mask) 5619long sched_getaffinity(pid_t pid, struct cpumask *mask)
5491{ 5620{
5492 struct task_struct *p; 5621 struct task_struct *p;
5493 int retval; 5622 int retval;
@@ -5504,7 +5633,7 @@ long sched_getaffinity(pid_t pid, cpumask_t *mask)
5504 if (retval) 5633 if (retval)
5505 goto out_unlock; 5634 goto out_unlock;
5506 5635
5507 cpus_and(*mask, p->cpus_allowed, cpu_online_map); 5636 cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
5508 5637
5509out_unlock: 5638out_unlock:
5510 read_unlock(&tasklist_lock); 5639 read_unlock(&tasklist_lock);
@@ -5519,23 +5648,28 @@ out_unlock:
5519 * @len: length in bytes of the bitmask pointed to by user_mask_ptr 5648 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
5520 * @user_mask_ptr: user-space pointer to hold the current cpu mask 5649 * @user_mask_ptr: user-space pointer to hold the current cpu mask
5521 */ 5650 */
5522asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len, 5651SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
5523 unsigned long __user *user_mask_ptr) 5652 unsigned long __user *, user_mask_ptr)
5524{ 5653{
5525 int ret; 5654 int ret;
5526 cpumask_t mask; 5655 cpumask_var_t mask;
5527 5656
5528 if (len < sizeof(cpumask_t)) 5657 if (len < cpumask_size())
5529 return -EINVAL; 5658 return -EINVAL;
5530 5659
5531 ret = sched_getaffinity(pid, &mask); 5660 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
5532 if (ret < 0) 5661 return -ENOMEM;
5533 return ret;
5534 5662
5535 if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t))) 5663 ret = sched_getaffinity(pid, mask);
5536 return -EFAULT; 5664 if (ret == 0) {
5665 if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
5666 ret = -EFAULT;
5667 else
5668 ret = cpumask_size();
5669 }
5670 free_cpumask_var(mask);
5537 5671
5538 return sizeof(cpumask_t); 5672 return ret;
5539} 5673}
5540 5674
5541/** 5675/**
@@ -5544,7 +5678,7 @@ asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
5544 * This function yields the current CPU to other tasks. If there are no 5678 * This function yields the current CPU to other tasks. If there are no
5545 * other threads running on this CPU then this function will return. 5679 * other threads running on this CPU then this function will return.
5546 */ 5680 */
5547asmlinkage long sys_sched_yield(void) 5681SYSCALL_DEFINE0(sched_yield)
5548{ 5682{
5549 struct rq *rq = this_rq_lock(); 5683 struct rq *rq = this_rq_lock();
5550 5684
@@ -5685,7 +5819,7 @@ long __sched io_schedule_timeout(long timeout)
5685 * this syscall returns the maximum rt_priority that can be used 5819 * this syscall returns the maximum rt_priority that can be used
5686 * by a given scheduling class. 5820 * by a given scheduling class.
5687 */ 5821 */
5688asmlinkage long sys_sched_get_priority_max(int policy) 5822SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
5689{ 5823{
5690 int ret = -EINVAL; 5824 int ret = -EINVAL;
5691 5825
@@ -5710,7 +5844,7 @@ asmlinkage long sys_sched_get_priority_max(int policy)
5710 * this syscall returns the minimum rt_priority that can be used 5844 * this syscall returns the minimum rt_priority that can be used
5711 * by a given scheduling class. 5845 * by a given scheduling class.
5712 */ 5846 */
5713asmlinkage long sys_sched_get_priority_min(int policy) 5847SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
5714{ 5848{
5715 int ret = -EINVAL; 5849 int ret = -EINVAL;
5716 5850
@@ -5735,8 +5869,8 @@ asmlinkage long sys_sched_get_priority_min(int policy)
5735 * this syscall writes the default timeslice value of a given process 5869 * this syscall writes the default timeslice value of a given process
5736 * into the user-space timespec buffer. A value of '0' means infinity. 5870 * into the user-space timespec buffer. A value of '0' means infinity.
5737 */ 5871 */
5738asmlinkage 5872SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
5739long sys_sched_rr_get_interval(pid_t pid, struct timespec __user *interval) 5873 struct timespec __user *, interval)
5740{ 5874{
5741 struct task_struct *p; 5875 struct task_struct *p;
5742 unsigned int time_slice; 5876 unsigned int time_slice;
@@ -5877,7 +6011,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5877 idle->se.exec_start = sched_clock(); 6011 idle->se.exec_start = sched_clock();
5878 6012
5879 idle->prio = idle->normal_prio = MAX_PRIO; 6013 idle->prio = idle->normal_prio = MAX_PRIO;
5880 idle->cpus_allowed = cpumask_of_cpu(cpu); 6014 cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
5881 __set_task_cpu(idle, cpu); 6015 __set_task_cpu(idle, cpu);
5882 6016
5883 rq->curr = rq->idle = idle; 6017 rq->curr = rq->idle = idle;
@@ -5896,6 +6030,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5896 * The idle tasks have their own, simple scheduling class: 6030 * The idle tasks have their own, simple scheduling class:
5897 */ 6031 */
5898 idle->sched_class = &idle_sched_class; 6032 idle->sched_class = &idle_sched_class;
6033 ftrace_graph_init_task(idle);
5899} 6034}
5900 6035
5901/* 6036/*
@@ -5903,9 +6038,9 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5903 * indicates which cpus entered this state. This is used 6038 * indicates which cpus entered this state. This is used
5904 * in the rcu update to wait only for active cpus. For system 6039 * in the rcu update to wait only for active cpus. For system
5905 * which do not switch off the HZ timer nohz_cpu_mask should 6040 * which do not switch off the HZ timer nohz_cpu_mask should
5906 * always be CPU_MASK_NONE. 6041 * always be CPU_BITS_NONE.
5907 */ 6042 */
5908cpumask_t nohz_cpu_mask = CPU_MASK_NONE; 6043cpumask_var_t nohz_cpu_mask;
5909 6044
5910/* 6045/*
5911 * Increase the granularity value when there are more CPUs, 6046 * Increase the granularity value when there are more CPUs,
@@ -5960,7 +6095,7 @@ static inline void sched_init_granularity(void)
5960 * task must not exit() & deallocate itself prematurely. The 6095 * task must not exit() & deallocate itself prematurely. The
5961 * call is not atomic; no spinlocks may be held. 6096 * call is not atomic; no spinlocks may be held.
5962 */ 6097 */
5963int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask) 6098int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
5964{ 6099{
5965 struct migration_req req; 6100 struct migration_req req;
5966 unsigned long flags; 6101 unsigned long flags;
@@ -5968,13 +6103,13 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5968 int ret = 0; 6103 int ret = 0;
5969 6104
5970 rq = task_rq_lock(p, &flags); 6105 rq = task_rq_lock(p, &flags);
5971 if (!cpus_intersects(*new_mask, cpu_online_map)) { 6106 if (!cpumask_intersects(new_mask, cpu_online_mask)) {
5972 ret = -EINVAL; 6107 ret = -EINVAL;
5973 goto out; 6108 goto out;
5974 } 6109 }
5975 6110
5976 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current && 6111 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
5977 !cpus_equal(p->cpus_allowed, *new_mask))) { 6112 !cpumask_equal(&p->cpus_allowed, new_mask))) {
5978 ret = -EINVAL; 6113 ret = -EINVAL;
5979 goto out; 6114 goto out;
5980 } 6115 }
@@ -5982,15 +6117,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5982 if (p->sched_class->set_cpus_allowed) 6117 if (p->sched_class->set_cpus_allowed)
5983 p->sched_class->set_cpus_allowed(p, new_mask); 6118 p->sched_class->set_cpus_allowed(p, new_mask);
5984 else { 6119 else {
5985 p->cpus_allowed = *new_mask; 6120 cpumask_copy(&p->cpus_allowed, new_mask);
5986 p->rt.nr_cpus_allowed = cpus_weight(*new_mask); 6121 p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
5987 } 6122 }
5988 6123
5989 /* Can the task run on the task's current CPU? If so, we're done */ 6124 /* Can the task run on the task's current CPU? If so, we're done */
5990 if (cpu_isset(task_cpu(p), *new_mask)) 6125 if (cpumask_test_cpu(task_cpu(p), new_mask))
5991 goto out; 6126 goto out;
5992 6127
5993 if (migrate_task(p, any_online_cpu(*new_mask), &req)) { 6128 if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
5994 /* Need help from migration thread: drop lock and wait. */ 6129 /* Need help from migration thread: drop lock and wait. */
5995 task_rq_unlock(rq, &flags); 6130 task_rq_unlock(rq, &flags);
5996 wake_up_process(rq->migration_thread); 6131 wake_up_process(rq->migration_thread);
@@ -6032,7 +6167,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
6032 if (task_cpu(p) != src_cpu) 6167 if (task_cpu(p) != src_cpu)
6033 goto done; 6168 goto done;
6034 /* Affinity changed (again). */ 6169 /* Affinity changed (again). */
6035 if (!cpu_isset(dest_cpu, p->cpus_allowed)) 6170 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6036 goto fail; 6171 goto fail;
6037 6172
6038 on_rq = p->se.on_rq; 6173 on_rq = p->se.on_rq;
@@ -6126,54 +6261,44 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
6126 6261
6127/* 6262/*
6128 * Figure out where task on dead CPU should go, use force if necessary. 6263 * Figure out where task on dead CPU should go, use force if necessary.
6129 * NOTE: interrupts should be disabled by the caller
6130 */ 6264 */
6131static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) 6265static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6132{ 6266{
6133 unsigned long flags;
6134 cpumask_t mask;
6135 struct rq *rq;
6136 int dest_cpu; 6267 int dest_cpu;
6268 const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(dead_cpu));
6137 6269
6138 do { 6270again:
6139 /* On same node? */ 6271 /* Look for allowed, online CPU in same node. */
6140 mask = node_to_cpumask(cpu_to_node(dead_cpu)); 6272 for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
6141 cpus_and(mask, mask, p->cpus_allowed); 6273 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6142 dest_cpu = any_online_cpu(mask); 6274 goto move;
6143
6144 /* On any allowed CPU? */
6145 if (dest_cpu >= nr_cpu_ids)
6146 dest_cpu = any_online_cpu(p->cpus_allowed);
6147 6275
6148 /* No more Mr. Nice Guy. */ 6276 /* Any allowed, online CPU? */
6149 if (dest_cpu >= nr_cpu_ids) { 6277 dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
6150 cpumask_t cpus_allowed; 6278 if (dest_cpu < nr_cpu_ids)
6279 goto move;
6151 6280
6152 cpuset_cpus_allowed_locked(p, &cpus_allowed); 6281 /* No more Mr. Nice Guy. */
6153 /* 6282 if (dest_cpu >= nr_cpu_ids) {
6154 * Try to stay on the same cpuset, where the 6283 cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
6155 * current cpuset may be a subset of all cpus. 6284 dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
6156 * The cpuset_cpus_allowed_locked() variant of
6157 * cpuset_cpus_allowed() will not block. It must be
6158 * called within calls to cpuset_lock/cpuset_unlock.
6159 */
6160 rq = task_rq_lock(p, &flags);
6161 p->cpus_allowed = cpus_allowed;
6162 dest_cpu = any_online_cpu(p->cpus_allowed);
6163 task_rq_unlock(rq, &flags);
6164 6285
6165 /* 6286 /*
6166 * Don't tell them about moving exiting tasks or 6287 * Don't tell them about moving exiting tasks or
6167 * kernel threads (both mm NULL), since they never 6288 * kernel threads (both mm NULL), since they never
6168 * leave kernel. 6289 * leave kernel.
6169 */ 6290 */
6170 if (p->mm && printk_ratelimit()) { 6291 if (p->mm && printk_ratelimit()) {
6171 printk(KERN_INFO "process %d (%s) no " 6292 printk(KERN_INFO "process %d (%s) no "
6172 "longer affine to cpu%d\n", 6293 "longer affine to cpu%d\n",
6173 task_pid_nr(p), p->comm, dead_cpu); 6294 task_pid_nr(p), p->comm, dead_cpu);
6174 }
6175 } 6295 }
6176 } while (!__migrate_task_irq(p, dead_cpu, dest_cpu)); 6296 }
6297
6298move:
6299 /* It can have affinity changed while we were choosing. */
6300 if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
6301 goto again;
6177} 6302}
6178 6303
6179/* 6304/*
@@ -6185,7 +6310,7 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6185 */ 6310 */
6186static void migrate_nr_uninterruptible(struct rq *rq_src) 6311static void migrate_nr_uninterruptible(struct rq *rq_src)
6187{ 6312{
6188 struct rq *rq_dest = cpu_rq(any_online_cpu(*CPU_MASK_ALL_PTR)); 6313 struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
6189 unsigned long flags; 6314 unsigned long flags;
6190 6315
6191 local_irq_save(flags); 6316 local_irq_save(flags);
@@ -6475,7 +6600,7 @@ static void set_rq_online(struct rq *rq)
6475 if (!rq->online) { 6600 if (!rq->online) {
6476 const struct sched_class *class; 6601 const struct sched_class *class;
6477 6602
6478 cpu_set(rq->cpu, rq->rd->online); 6603 cpumask_set_cpu(rq->cpu, rq->rd->online);
6479 rq->online = 1; 6604 rq->online = 1;
6480 6605
6481 for_each_class(class) { 6606 for_each_class(class) {
@@ -6495,7 +6620,7 @@ static void set_rq_offline(struct rq *rq)
6495 class->rq_offline(rq); 6620 class->rq_offline(rq);
6496 } 6621 }
6497 6622
6498 cpu_clear(rq->cpu, rq->rd->online); 6623 cpumask_clear_cpu(rq->cpu, rq->rd->online);
6499 rq->online = 0; 6624 rq->online = 0;
6500 } 6625 }
6501} 6626}
@@ -6536,7 +6661,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6536 rq = cpu_rq(cpu); 6661 rq = cpu_rq(cpu);
6537 spin_lock_irqsave(&rq->lock, flags); 6662 spin_lock_irqsave(&rq->lock, flags);
6538 if (rq->rd) { 6663 if (rq->rd) {
6539 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6664 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6540 6665
6541 set_rq_online(rq); 6666 set_rq_online(rq);
6542 } 6667 }
@@ -6550,7 +6675,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6550 break; 6675 break;
6551 /* Unbind it from offline cpu so it can run. Fall thru. */ 6676 /* Unbind it from offline cpu so it can run. Fall thru. */
6552 kthread_bind(cpu_rq(cpu)->migration_thread, 6677 kthread_bind(cpu_rq(cpu)->migration_thread,
6553 any_online_cpu(cpu_online_map)); 6678 cpumask_any(cpu_online_mask));
6554 kthread_stop(cpu_rq(cpu)->migration_thread); 6679 kthread_stop(cpu_rq(cpu)->migration_thread);
6555 cpu_rq(cpu)->migration_thread = NULL; 6680 cpu_rq(cpu)->migration_thread = NULL;
6556 break; 6681 break;
@@ -6600,7 +6725,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6600 rq = cpu_rq(cpu); 6725 rq = cpu_rq(cpu);
6601 spin_lock_irqsave(&rq->lock, flags); 6726 spin_lock_irqsave(&rq->lock, flags);
6602 if (rq->rd) { 6727 if (rq->rd) {
6603 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6728 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6604 set_rq_offline(rq); 6729 set_rq_offline(rq);
6605 } 6730 }
6606 spin_unlock_irqrestore(&rq->lock, flags); 6731 spin_unlock_irqrestore(&rq->lock, flags);
@@ -6638,36 +6763,14 @@ early_initcall(migration_init);
6638 6763
6639#ifdef CONFIG_SCHED_DEBUG 6764#ifdef CONFIG_SCHED_DEBUG
6640 6765
6641static inline const char *sd_level_to_string(enum sched_domain_level lvl)
6642{
6643 switch (lvl) {
6644 case SD_LV_NONE:
6645 return "NONE";
6646 case SD_LV_SIBLING:
6647 return "SIBLING";
6648 case SD_LV_MC:
6649 return "MC";
6650 case SD_LV_CPU:
6651 return "CPU";
6652 case SD_LV_NODE:
6653 return "NODE";
6654 case SD_LV_ALLNODES:
6655 return "ALLNODES";
6656 case SD_LV_MAX:
6657 return "MAX";
6658
6659 }
6660 return "MAX";
6661}
6662
6663static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, 6766static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6664 cpumask_t *groupmask) 6767 struct cpumask *groupmask)
6665{ 6768{
6666 struct sched_group *group = sd->groups; 6769 struct sched_group *group = sd->groups;
6667 char str[256]; 6770 char str[256];
6668 6771
6669 cpulist_scnprintf(str, sizeof(str), sd->span); 6772 cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
6670 cpus_clear(*groupmask); 6773 cpumask_clear(groupmask);
6671 6774
6672 printk(KERN_DEBUG "%*s domain %d: ", level, "", level); 6775 printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
6673 6776
@@ -6679,14 +6782,13 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6679 return -1; 6782 return -1;
6680 } 6783 }
6681 6784
6682 printk(KERN_CONT "span %s level %s\n", 6785 printk(KERN_CONT "span %s level %s\n", str, sd->name);
6683 str, sd_level_to_string(sd->level));
6684 6786
6685 if (!cpu_isset(cpu, sd->span)) { 6787 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
6686 printk(KERN_ERR "ERROR: domain->span does not contain " 6788 printk(KERN_ERR "ERROR: domain->span does not contain "
6687 "CPU%d\n", cpu); 6789 "CPU%d\n", cpu);
6688 } 6790 }
6689 if (!cpu_isset(cpu, group->cpumask)) { 6791 if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
6690 printk(KERN_ERR "ERROR: domain->groups does not contain" 6792 printk(KERN_ERR "ERROR: domain->groups does not contain"
6691 " CPU%d\n", cpu); 6793 " CPU%d\n", cpu);
6692 } 6794 }
@@ -6706,31 +6808,32 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6706 break; 6808 break;
6707 } 6809 }
6708 6810
6709 if (!cpus_weight(group->cpumask)) { 6811 if (!cpumask_weight(sched_group_cpus(group))) {
6710 printk(KERN_CONT "\n"); 6812 printk(KERN_CONT "\n");
6711 printk(KERN_ERR "ERROR: empty group\n"); 6813 printk(KERN_ERR "ERROR: empty group\n");
6712 break; 6814 break;
6713 } 6815 }
6714 6816
6715 if (cpus_intersects(*groupmask, group->cpumask)) { 6817 if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
6716 printk(KERN_CONT "\n"); 6818 printk(KERN_CONT "\n");
6717 printk(KERN_ERR "ERROR: repeated CPUs\n"); 6819 printk(KERN_ERR "ERROR: repeated CPUs\n");
6718 break; 6820 break;
6719 } 6821 }
6720 6822
6721 cpus_or(*groupmask, *groupmask, group->cpumask); 6823 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
6722 6824
6723 cpulist_scnprintf(str, sizeof(str), group->cpumask); 6825 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
6724 printk(KERN_CONT " %s", str); 6826 printk(KERN_CONT " %s", str);
6725 6827
6726 group = group->next; 6828 group = group->next;
6727 } while (group != sd->groups); 6829 } while (group != sd->groups);
6728 printk(KERN_CONT "\n"); 6830 printk(KERN_CONT "\n");
6729 6831
6730 if (!cpus_equal(sd->span, *groupmask)) 6832 if (!cpumask_equal(sched_domain_span(sd), groupmask))
6731 printk(KERN_ERR "ERROR: groups don't span domain->span\n"); 6833 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
6732 6834
6733 if (sd->parent && !cpus_subset(*groupmask, sd->parent->span)) 6835 if (sd->parent &&
6836 !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
6734 printk(KERN_ERR "ERROR: parent span is not a superset " 6837 printk(KERN_ERR "ERROR: parent span is not a superset "
6735 "of domain->span\n"); 6838 "of domain->span\n");
6736 return 0; 6839 return 0;
@@ -6738,7 +6841,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6738 6841
6739static void sched_domain_debug(struct sched_domain *sd, int cpu) 6842static void sched_domain_debug(struct sched_domain *sd, int cpu)
6740{ 6843{
6741 cpumask_t *groupmask; 6844 cpumask_var_t groupmask;
6742 int level = 0; 6845 int level = 0;
6743 6846
6744 if (!sd) { 6847 if (!sd) {
@@ -6748,8 +6851,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6748 6851
6749 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); 6852 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
6750 6853
6751 groupmask = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 6854 if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
6752 if (!groupmask) {
6753 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n"); 6855 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
6754 return; 6856 return;
6755 } 6857 }
@@ -6762,7 +6864,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6762 if (!sd) 6864 if (!sd)
6763 break; 6865 break;
6764 } 6866 }
6765 kfree(groupmask); 6867 free_cpumask_var(groupmask);
6766} 6868}
6767#else /* !CONFIG_SCHED_DEBUG */ 6869#else /* !CONFIG_SCHED_DEBUG */
6768# define sched_domain_debug(sd, cpu) do { } while (0) 6870# define sched_domain_debug(sd, cpu) do { } while (0)
@@ -6770,7 +6872,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6770 6872
6771static int sd_degenerate(struct sched_domain *sd) 6873static int sd_degenerate(struct sched_domain *sd)
6772{ 6874{
6773 if (cpus_weight(sd->span) == 1) 6875 if (cpumask_weight(sched_domain_span(sd)) == 1)
6774 return 1; 6876 return 1;
6775 6877
6776 /* Following flags need at least 2 groups */ 6878 /* Following flags need at least 2 groups */
@@ -6801,7 +6903,7 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6801 if (sd_degenerate(parent)) 6903 if (sd_degenerate(parent))
6802 return 1; 6904 return 1;
6803 6905
6804 if (!cpus_equal(sd->span, parent->span)) 6906 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
6805 return 0; 6907 return 0;
6806 6908
6807 /* Does parent contain flags not in child? */ 6909 /* Does parent contain flags not in child? */
@@ -6816,6 +6918,8 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6816 SD_BALANCE_EXEC | 6918 SD_BALANCE_EXEC |
6817 SD_SHARE_CPUPOWER | 6919 SD_SHARE_CPUPOWER |
6818 SD_SHARE_PKG_RESOURCES); 6920 SD_SHARE_PKG_RESOURCES);
6921 if (nr_node_ids == 1)
6922 pflags &= ~SD_SERIALIZE;
6819 } 6923 }
6820 if (~cflags & pflags) 6924 if (~cflags & pflags)
6821 return 0; 6925 return 0;
@@ -6823,6 +6927,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6823 return 1; 6927 return 1;
6824} 6928}
6825 6929
6930static void free_rootdomain(struct root_domain *rd)
6931{
6932 cpupri_cleanup(&rd->cpupri);
6933
6934 free_cpumask_var(rd->rto_mask);
6935 free_cpumask_var(rd->online);
6936 free_cpumask_var(rd->span);
6937 kfree(rd);
6938}
6939
6826static void rq_attach_root(struct rq *rq, struct root_domain *rd) 6940static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6827{ 6941{
6828 unsigned long flags; 6942 unsigned long flags;
@@ -6832,38 +6946,62 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6832 if (rq->rd) { 6946 if (rq->rd) {
6833 struct root_domain *old_rd = rq->rd; 6947 struct root_domain *old_rd = rq->rd;
6834 6948
6835 if (cpu_isset(rq->cpu, old_rd->online)) 6949 if (cpumask_test_cpu(rq->cpu, old_rd->online))
6836 set_rq_offline(rq); 6950 set_rq_offline(rq);
6837 6951
6838 cpu_clear(rq->cpu, old_rd->span); 6952 cpumask_clear_cpu(rq->cpu, old_rd->span);
6839 6953
6840 if (atomic_dec_and_test(&old_rd->refcount)) 6954 if (atomic_dec_and_test(&old_rd->refcount))
6841 kfree(old_rd); 6955 free_rootdomain(old_rd);
6842 } 6956 }
6843 6957
6844 atomic_inc(&rd->refcount); 6958 atomic_inc(&rd->refcount);
6845 rq->rd = rd; 6959 rq->rd = rd;
6846 6960
6847 cpu_set(rq->cpu, rd->span); 6961 cpumask_set_cpu(rq->cpu, rd->span);
6848 if (cpu_isset(rq->cpu, cpu_online_map)) 6962 if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
6849 set_rq_online(rq); 6963 set_rq_online(rq);
6850 6964
6851 spin_unlock_irqrestore(&rq->lock, flags); 6965 spin_unlock_irqrestore(&rq->lock, flags);
6852} 6966}
6853 6967
6854static void init_rootdomain(struct root_domain *rd) 6968static int __init_refok init_rootdomain(struct root_domain *rd, bool bootmem)
6855{ 6969{
6856 memset(rd, 0, sizeof(*rd)); 6970 memset(rd, 0, sizeof(*rd));
6857 6971
6858 cpus_clear(rd->span); 6972 if (bootmem) {
6859 cpus_clear(rd->online); 6973 alloc_bootmem_cpumask_var(&def_root_domain.span);
6974 alloc_bootmem_cpumask_var(&def_root_domain.online);
6975 alloc_bootmem_cpumask_var(&def_root_domain.rto_mask);
6976 cpupri_init(&rd->cpupri, true);
6977 return 0;
6978 }
6979
6980 if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
6981 goto out;
6982 if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
6983 goto free_span;
6984 if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
6985 goto free_online;
6986
6987 if (cpupri_init(&rd->cpupri, false) != 0)
6988 goto free_rto_mask;
6989 return 0;
6860 6990
6861 cpupri_init(&rd->cpupri); 6991free_rto_mask:
6992 free_cpumask_var(rd->rto_mask);
6993free_online:
6994 free_cpumask_var(rd->online);
6995free_span:
6996 free_cpumask_var(rd->span);
6997out:
6998 return -ENOMEM;
6862} 6999}
6863 7000
6864static void init_defrootdomain(void) 7001static void init_defrootdomain(void)
6865{ 7002{
6866 init_rootdomain(&def_root_domain); 7003 init_rootdomain(&def_root_domain, true);
7004
6867 atomic_set(&def_root_domain.refcount, 1); 7005 atomic_set(&def_root_domain.refcount, 1);
6868} 7006}
6869 7007
@@ -6875,7 +7013,10 @@ static struct root_domain *alloc_rootdomain(void)
6875 if (!rd) 7013 if (!rd)
6876 return NULL; 7014 return NULL;
6877 7015
6878 init_rootdomain(rd); 7016 if (init_rootdomain(rd, false) != 0) {
7017 kfree(rd);
7018 return NULL;
7019 }
6879 7020
6880 return rd; 7021 return rd;
6881} 7022}
@@ -6917,19 +7058,12 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
6917} 7058}
6918 7059
6919/* cpus with isolated domains */ 7060/* cpus with isolated domains */
6920static cpumask_t cpu_isolated_map = CPU_MASK_NONE; 7061static cpumask_var_t cpu_isolated_map;
6921 7062
6922/* Setup the mask of cpus configured for isolated domains */ 7063/* Setup the mask of cpus configured for isolated domains */
6923static int __init isolated_cpu_setup(char *str) 7064static int __init isolated_cpu_setup(char *str)
6924{ 7065{
6925 static int __initdata ints[NR_CPUS]; 7066 cpulist_parse(str, cpu_isolated_map);
6926 int i;
6927
6928 str = get_options(str, ARRAY_SIZE(ints), ints);
6929 cpus_clear(cpu_isolated_map);
6930 for (i = 1; i <= ints[0]; i++)
6931 if (ints[i] < NR_CPUS)
6932 cpu_set(ints[i], cpu_isolated_map);
6933 return 1; 7067 return 1;
6934} 7068}
6935 7069
@@ -6938,42 +7072,43 @@ __setup("isolcpus=", isolated_cpu_setup);
6938/* 7072/*
6939 * init_sched_build_groups takes the cpumask we wish to span, and a pointer 7073 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
6940 * to a function which identifies what group(along with sched group) a CPU 7074 * to a function which identifies what group(along with sched group) a CPU
6941 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS 7075 * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
6942 * (due to the fact that we keep track of groups covered with a cpumask_t). 7076 * (due to the fact that we keep track of groups covered with a struct cpumask).
6943 * 7077 *
6944 * init_sched_build_groups will build a circular linked list of the groups 7078 * init_sched_build_groups will build a circular linked list of the groups
6945 * covered by the given span, and will set each group's ->cpumask correctly, 7079 * covered by the given span, and will set each group's ->cpumask correctly,
6946 * and ->cpu_power to 0. 7080 * and ->cpu_power to 0.
6947 */ 7081 */
6948static void 7082static void
6949init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map, 7083init_sched_build_groups(const struct cpumask *span,
6950 int (*group_fn)(int cpu, const cpumask_t *cpu_map, 7084 const struct cpumask *cpu_map,
7085 int (*group_fn)(int cpu, const struct cpumask *cpu_map,
6951 struct sched_group **sg, 7086 struct sched_group **sg,
6952 cpumask_t *tmpmask), 7087 struct cpumask *tmpmask),
6953 cpumask_t *covered, cpumask_t *tmpmask) 7088 struct cpumask *covered, struct cpumask *tmpmask)
6954{ 7089{
6955 struct sched_group *first = NULL, *last = NULL; 7090 struct sched_group *first = NULL, *last = NULL;
6956 int i; 7091 int i;
6957 7092
6958 cpus_clear(*covered); 7093 cpumask_clear(covered);
6959 7094
6960 for_each_cpu_mask_nr(i, *span) { 7095 for_each_cpu(i, span) {
6961 struct sched_group *sg; 7096 struct sched_group *sg;
6962 int group = group_fn(i, cpu_map, &sg, tmpmask); 7097 int group = group_fn(i, cpu_map, &sg, tmpmask);
6963 int j; 7098 int j;
6964 7099
6965 if (cpu_isset(i, *covered)) 7100 if (cpumask_test_cpu(i, covered))
6966 continue; 7101 continue;
6967 7102
6968 cpus_clear(sg->cpumask); 7103 cpumask_clear(sched_group_cpus(sg));
6969 sg->__cpu_power = 0; 7104 sg->__cpu_power = 0;
6970 7105
6971 for_each_cpu_mask_nr(j, *span) { 7106 for_each_cpu(j, span) {
6972 if (group_fn(j, cpu_map, NULL, tmpmask) != group) 7107 if (group_fn(j, cpu_map, NULL, tmpmask) != group)
6973 continue; 7108 continue;
6974 7109
6975 cpu_set(j, *covered); 7110 cpumask_set_cpu(j, covered);
6976 cpu_set(j, sg->cpumask); 7111 cpumask_set_cpu(j, sched_group_cpus(sg));
6977 } 7112 }
6978 if (!first) 7113 if (!first)
6979 first = sg; 7114 first = sg;
@@ -7037,23 +7172,21 @@ static int find_next_best_node(int node, nodemask_t *used_nodes)
7037 * should be one that prevents unnecessary balancing, but also spreads tasks 7172 * should be one that prevents unnecessary balancing, but also spreads tasks
7038 * out optimally. 7173 * out optimally.
7039 */ 7174 */
7040static void sched_domain_node_span(int node, cpumask_t *span) 7175static void sched_domain_node_span(int node, struct cpumask *span)
7041{ 7176{
7042 nodemask_t used_nodes; 7177 nodemask_t used_nodes;
7043 node_to_cpumask_ptr(nodemask, node);
7044 int i; 7178 int i;
7045 7179
7046 cpus_clear(*span); 7180 cpumask_clear(span);
7047 nodes_clear(used_nodes); 7181 nodes_clear(used_nodes);
7048 7182
7049 cpus_or(*span, *span, *nodemask); 7183 cpumask_or(span, span, cpumask_of_node(node));
7050 node_set(node, used_nodes); 7184 node_set(node, used_nodes);
7051 7185
7052 for (i = 1; i < SD_NODES_PER_DOMAIN; i++) { 7186 for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
7053 int next_node = find_next_best_node(node, &used_nodes); 7187 int next_node = find_next_best_node(node, &used_nodes);
7054 7188
7055 node_to_cpumask_ptr_next(nodemask, next_node); 7189 cpumask_or(span, span, cpumask_of_node(next_node));
7056 cpus_or(*span, *span, *nodemask);
7057 } 7190 }
7058} 7191}
7059#endif /* CONFIG_NUMA */ 7192#endif /* CONFIG_NUMA */
@@ -7061,18 +7194,33 @@ static void sched_domain_node_span(int node, cpumask_t *span)
7061int sched_smt_power_savings = 0, sched_mc_power_savings = 0; 7194int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
7062 7195
7063/* 7196/*
7197 * The cpus mask in sched_group and sched_domain hangs off the end.
7198 * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space
7199 * for nr_cpu_ids < CONFIG_NR_CPUS.
7200 */
7201struct static_sched_group {
7202 struct sched_group sg;
7203 DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
7204};
7205
7206struct static_sched_domain {
7207 struct sched_domain sd;
7208 DECLARE_BITMAP(span, CONFIG_NR_CPUS);
7209};
7210
7211/*
7064 * SMT sched-domains: 7212 * SMT sched-domains:
7065 */ 7213 */
7066#ifdef CONFIG_SCHED_SMT 7214#ifdef CONFIG_SCHED_SMT
7067static DEFINE_PER_CPU(struct sched_domain, cpu_domains); 7215static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
7068static DEFINE_PER_CPU(struct sched_group, sched_group_cpus); 7216static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
7069 7217
7070static int 7218static int
7071cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7219cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
7072 cpumask_t *unused) 7220 struct sched_group **sg, struct cpumask *unused)
7073{ 7221{
7074 if (sg) 7222 if (sg)
7075 *sg = &per_cpu(sched_group_cpus, cpu); 7223 *sg = &per_cpu(sched_group_cpus, cpu).sg;
7076 return cpu; 7224 return cpu;
7077} 7225}
7078#endif /* CONFIG_SCHED_SMT */ 7226#endif /* CONFIG_SCHED_SMT */
@@ -7081,56 +7229,53 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
7081 * multi-core sched-domains: 7229 * multi-core sched-domains:
7082 */ 7230 */
7083#ifdef CONFIG_SCHED_MC 7231#ifdef CONFIG_SCHED_MC
7084static DEFINE_PER_CPU(struct sched_domain, core_domains); 7232static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
7085static DEFINE_PER_CPU(struct sched_group, sched_group_core); 7233static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
7086#endif /* CONFIG_SCHED_MC */ 7234#endif /* CONFIG_SCHED_MC */
7087 7235
7088#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) 7236#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
7089static int 7237static int
7090cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7238cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7091 cpumask_t *mask) 7239 struct sched_group **sg, struct cpumask *mask)
7092{ 7240{
7093 int group; 7241 int group;
7094 7242
7095 *mask = per_cpu(cpu_sibling_map, cpu); 7243 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7096 cpus_and(*mask, *mask, *cpu_map); 7244 group = cpumask_first(mask);
7097 group = first_cpu(*mask);
7098 if (sg) 7245 if (sg)
7099 *sg = &per_cpu(sched_group_core, group); 7246 *sg = &per_cpu(sched_group_core, group).sg;
7100 return group; 7247 return group;
7101} 7248}
7102#elif defined(CONFIG_SCHED_MC) 7249#elif defined(CONFIG_SCHED_MC)
7103static int 7250static int
7104cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7251cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7105 cpumask_t *unused) 7252 struct sched_group **sg, struct cpumask *unused)
7106{ 7253{
7107 if (sg) 7254 if (sg)
7108 *sg = &per_cpu(sched_group_core, cpu); 7255 *sg = &per_cpu(sched_group_core, cpu).sg;
7109 return cpu; 7256 return cpu;
7110} 7257}
7111#endif 7258#endif
7112 7259
7113static DEFINE_PER_CPU(struct sched_domain, phys_domains); 7260static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
7114static DEFINE_PER_CPU(struct sched_group, sched_group_phys); 7261static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
7115 7262
7116static int 7263static int
7117cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7264cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
7118 cpumask_t *mask) 7265 struct sched_group **sg, struct cpumask *mask)
7119{ 7266{
7120 int group; 7267 int group;
7121#ifdef CONFIG_SCHED_MC 7268#ifdef CONFIG_SCHED_MC
7122 *mask = cpu_coregroup_map(cpu); 7269 cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
7123 cpus_and(*mask, *mask, *cpu_map); 7270 group = cpumask_first(mask);
7124 group = first_cpu(*mask);
7125#elif defined(CONFIG_SCHED_SMT) 7271#elif defined(CONFIG_SCHED_SMT)
7126 *mask = per_cpu(cpu_sibling_map, cpu); 7272 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7127 cpus_and(*mask, *mask, *cpu_map); 7273 group = cpumask_first(mask);
7128 group = first_cpu(*mask);
7129#else 7274#else
7130 group = cpu; 7275 group = cpu;
7131#endif 7276#endif
7132 if (sg) 7277 if (sg)
7133 *sg = &per_cpu(sched_group_phys, group); 7278 *sg = &per_cpu(sched_group_phys, group).sg;
7134 return group; 7279 return group;
7135} 7280}
7136 7281
@@ -7140,23 +7285,23 @@ cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
7140 * groups, so roll our own. Now each node has its own list of groups which 7285 * groups, so roll our own. Now each node has its own list of groups which
7141 * gets dynamically allocated. 7286 * gets dynamically allocated.
7142 */ 7287 */
7143static DEFINE_PER_CPU(struct sched_domain, node_domains); 7288static DEFINE_PER_CPU(struct static_sched_domain, node_domains);
7144static struct sched_group ***sched_group_nodes_bycpu; 7289static struct sched_group ***sched_group_nodes_bycpu;
7145 7290
7146static DEFINE_PER_CPU(struct sched_domain, allnodes_domains); 7291static DEFINE_PER_CPU(struct static_sched_domain, allnodes_domains);
7147static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes); 7292static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
7148 7293
7149static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map, 7294static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
7150 struct sched_group **sg, cpumask_t *nodemask) 7295 struct sched_group **sg,
7296 struct cpumask *nodemask)
7151{ 7297{
7152 int group; 7298 int group;
7153 7299
7154 *nodemask = node_to_cpumask(cpu_to_node(cpu)); 7300 cpumask_and(nodemask, cpumask_of_node(cpu_to_node(cpu)), cpu_map);
7155 cpus_and(*nodemask, *nodemask, *cpu_map); 7301 group = cpumask_first(nodemask);
7156 group = first_cpu(*nodemask);
7157 7302
7158 if (sg) 7303 if (sg)
7159 *sg = &per_cpu(sched_group_allnodes, group); 7304 *sg = &per_cpu(sched_group_allnodes, group).sg;
7160 return group; 7305 return group;
7161} 7306}
7162 7307
@@ -7168,11 +7313,11 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7168 if (!sg) 7313 if (!sg)
7169 return; 7314 return;
7170 do { 7315 do {
7171 for_each_cpu_mask_nr(j, sg->cpumask) { 7316 for_each_cpu(j, sched_group_cpus(sg)) {
7172 struct sched_domain *sd; 7317 struct sched_domain *sd;
7173 7318
7174 sd = &per_cpu(phys_domains, j); 7319 sd = &per_cpu(phys_domains, j).sd;
7175 if (j != first_cpu(sd->groups->cpumask)) { 7320 if (j != cpumask_first(sched_group_cpus(sd->groups))) {
7176 /* 7321 /*
7177 * Only add "power" once for each 7322 * Only add "power" once for each
7178 * physical package. 7323 * physical package.
@@ -7189,11 +7334,12 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7189 7334
7190#ifdef CONFIG_NUMA 7335#ifdef CONFIG_NUMA
7191/* Free memory allocated for various sched_group structures */ 7336/* Free memory allocated for various sched_group structures */
7192static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7337static void free_sched_groups(const struct cpumask *cpu_map,
7338 struct cpumask *nodemask)
7193{ 7339{
7194 int cpu, i; 7340 int cpu, i;
7195 7341
7196 for_each_cpu_mask_nr(cpu, *cpu_map) { 7342 for_each_cpu(cpu, cpu_map) {
7197 struct sched_group **sched_group_nodes 7343 struct sched_group **sched_group_nodes
7198 = sched_group_nodes_bycpu[cpu]; 7344 = sched_group_nodes_bycpu[cpu];
7199 7345
@@ -7203,9 +7349,8 @@ static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
7203 for (i = 0; i < nr_node_ids; i++) { 7349 for (i = 0; i < nr_node_ids; i++) {
7204 struct sched_group *oldsg, *sg = sched_group_nodes[i]; 7350 struct sched_group *oldsg, *sg = sched_group_nodes[i];
7205 7351
7206 *nodemask = node_to_cpumask(i); 7352 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7207 cpus_and(*nodemask, *nodemask, *cpu_map); 7353 if (cpumask_empty(nodemask))
7208 if (cpus_empty(*nodemask))
7209 continue; 7354 continue;
7210 7355
7211 if (sg == NULL) 7356 if (sg == NULL)
@@ -7223,7 +7368,8 @@ next_sg:
7223 } 7368 }
7224} 7369}
7225#else /* !CONFIG_NUMA */ 7370#else /* !CONFIG_NUMA */
7226static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7371static void free_sched_groups(const struct cpumask *cpu_map,
7372 struct cpumask *nodemask)
7227{ 7373{
7228} 7374}
7229#endif /* CONFIG_NUMA */ 7375#endif /* CONFIG_NUMA */
@@ -7249,7 +7395,7 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
7249 7395
7250 WARN_ON(!sd || !sd->groups); 7396 WARN_ON(!sd || !sd->groups);
7251 7397
7252 if (cpu != first_cpu(sd->groups->cpumask)) 7398 if (cpu != cpumask_first(sched_group_cpus(sd->groups)))
7253 return; 7399 return;
7254 7400
7255 child = sd->child; 7401 child = sd->child;
@@ -7314,40 +7460,6 @@ SD_INIT_FUNC(CPU)
7314 SD_INIT_FUNC(MC) 7460 SD_INIT_FUNC(MC)
7315#endif 7461#endif
7316 7462
7317/*
7318 * To minimize stack usage kmalloc room for cpumasks and share the
7319 * space as the usage in build_sched_domains() dictates. Used only
7320 * if the amount of space is significant.
7321 */
7322struct allmasks {
7323 cpumask_t tmpmask; /* make this one first */
7324 union {
7325 cpumask_t nodemask;
7326 cpumask_t this_sibling_map;
7327 cpumask_t this_core_map;
7328 };
7329 cpumask_t send_covered;
7330
7331#ifdef CONFIG_NUMA
7332 cpumask_t domainspan;
7333 cpumask_t covered;
7334 cpumask_t notcovered;
7335#endif
7336};
7337
7338#if NR_CPUS > 128
7339#define SCHED_CPUMASK_ALLOC 1
7340#define SCHED_CPUMASK_FREE(v) kfree(v)
7341#define SCHED_CPUMASK_DECLARE(v) struct allmasks *v
7342#else
7343#define SCHED_CPUMASK_ALLOC 0
7344#define SCHED_CPUMASK_FREE(v)
7345#define SCHED_CPUMASK_DECLARE(v) struct allmasks _v, *v = &_v
7346#endif
7347
7348#define SCHED_CPUMASK_VAR(v, a) cpumask_t *v = (cpumask_t *) \
7349 ((unsigned long)(a) + offsetof(struct allmasks, v))
7350
7351static int default_relax_domain_level = -1; 7463static int default_relax_domain_level = -1;
7352 7464
7353static int __init setup_relax_domain_level(char *str) 7465static int __init setup_relax_domain_level(char *str)
@@ -7387,17 +7499,38 @@ static void set_domain_attribute(struct sched_domain *sd,
7387 * Build sched domains for a given set of cpus and attach the sched domains 7499 * Build sched domains for a given set of cpus and attach the sched domains
7388 * to the individual cpus 7500 * to the individual cpus
7389 */ 7501 */
7390static int __build_sched_domains(const cpumask_t *cpu_map, 7502static int __build_sched_domains(const struct cpumask *cpu_map,
7391 struct sched_domain_attr *attr) 7503 struct sched_domain_attr *attr)
7392{ 7504{
7393 int i; 7505 int i, err = -ENOMEM;
7394 struct root_domain *rd; 7506 struct root_domain *rd;
7395 SCHED_CPUMASK_DECLARE(allmasks); 7507 cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
7396 cpumask_t *tmpmask; 7508 tmpmask;
7397#ifdef CONFIG_NUMA 7509#ifdef CONFIG_NUMA
7510 cpumask_var_t domainspan, covered, notcovered;
7398 struct sched_group **sched_group_nodes = NULL; 7511 struct sched_group **sched_group_nodes = NULL;
7399 int sd_allnodes = 0; 7512 int sd_allnodes = 0;
7400 7513
7514 if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
7515 goto out;
7516 if (!alloc_cpumask_var(&covered, GFP_KERNEL))
7517 goto free_domainspan;
7518 if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
7519 goto free_covered;
7520#endif
7521
7522 if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
7523 goto free_notcovered;
7524 if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
7525 goto free_nodemask;
7526 if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
7527 goto free_this_sibling_map;
7528 if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
7529 goto free_this_core_map;
7530 if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
7531 goto free_send_covered;
7532
7533#ifdef CONFIG_NUMA
7401 /* 7534 /*
7402 * Allocate the per-node list of sched groups 7535 * Allocate the per-node list of sched groups
7403 */ 7536 */
@@ -7405,76 +7538,57 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7405 GFP_KERNEL); 7538 GFP_KERNEL);
7406 if (!sched_group_nodes) { 7539 if (!sched_group_nodes) {
7407 printk(KERN_WARNING "Can not alloc sched group node list\n"); 7540 printk(KERN_WARNING "Can not alloc sched group node list\n");
7408 return -ENOMEM; 7541 goto free_tmpmask;
7409 } 7542 }
7410#endif 7543#endif
7411 7544
7412 rd = alloc_rootdomain(); 7545 rd = alloc_rootdomain();
7413 if (!rd) { 7546 if (!rd) {
7414 printk(KERN_WARNING "Cannot alloc root domain\n"); 7547 printk(KERN_WARNING "Cannot alloc root domain\n");
7415#ifdef CONFIG_NUMA 7548 goto free_sched_groups;
7416 kfree(sched_group_nodes);
7417#endif
7418 return -ENOMEM;
7419 }
7420
7421#if SCHED_CPUMASK_ALLOC
7422 /* get space for all scratch cpumask variables */
7423 allmasks = kmalloc(sizeof(*allmasks), GFP_KERNEL);
7424 if (!allmasks) {
7425 printk(KERN_WARNING "Cannot alloc cpumask array\n");
7426 kfree(rd);
7427#ifdef CONFIG_NUMA
7428 kfree(sched_group_nodes);
7429#endif
7430 return -ENOMEM;
7431 } 7549 }
7432#endif
7433 tmpmask = (cpumask_t *)allmasks;
7434
7435 7550
7436#ifdef CONFIG_NUMA 7551#ifdef CONFIG_NUMA
7437 sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes; 7552 sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
7438#endif 7553#endif
7439 7554
7440 /* 7555 /*
7441 * Set up domains for cpus specified by the cpu_map. 7556 * Set up domains for cpus specified by the cpu_map.
7442 */ 7557 */
7443 for_each_cpu_mask_nr(i, *cpu_map) { 7558 for_each_cpu(i, cpu_map) {
7444 struct sched_domain *sd = NULL, *p; 7559 struct sched_domain *sd = NULL, *p;
7445 SCHED_CPUMASK_VAR(nodemask, allmasks);
7446 7560
7447 *nodemask = node_to_cpumask(cpu_to_node(i)); 7561 cpumask_and(nodemask, cpumask_of_node(cpu_to_node(i)), cpu_map);
7448 cpus_and(*nodemask, *nodemask, *cpu_map);
7449 7562
7450#ifdef CONFIG_NUMA 7563#ifdef CONFIG_NUMA
7451 if (cpus_weight(*cpu_map) > 7564 if (cpumask_weight(cpu_map) >
7452 SD_NODES_PER_DOMAIN*cpus_weight(*nodemask)) { 7565 SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
7453 sd = &per_cpu(allnodes_domains, i); 7566 sd = &per_cpu(allnodes_domains, i).sd;
7454 SD_INIT(sd, ALLNODES); 7567 SD_INIT(sd, ALLNODES);
7455 set_domain_attribute(sd, attr); 7568 set_domain_attribute(sd, attr);
7456 sd->span = *cpu_map; 7569 cpumask_copy(sched_domain_span(sd), cpu_map);
7457 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask); 7570 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
7458 p = sd; 7571 p = sd;
7459 sd_allnodes = 1; 7572 sd_allnodes = 1;
7460 } else 7573 } else
7461 p = NULL; 7574 p = NULL;
7462 7575
7463 sd = &per_cpu(node_domains, i); 7576 sd = &per_cpu(node_domains, i).sd;
7464 SD_INIT(sd, NODE); 7577 SD_INIT(sd, NODE);
7465 set_domain_attribute(sd, attr); 7578 set_domain_attribute(sd, attr);
7466 sched_domain_node_span(cpu_to_node(i), &sd->span); 7579 sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
7467 sd->parent = p; 7580 sd->parent = p;
7468 if (p) 7581 if (p)
7469 p->child = sd; 7582 p->child = sd;
7470 cpus_and(sd->span, sd->span, *cpu_map); 7583 cpumask_and(sched_domain_span(sd),
7584 sched_domain_span(sd), cpu_map);
7471#endif 7585#endif
7472 7586
7473 p = sd; 7587 p = sd;
7474 sd = &per_cpu(phys_domains, i); 7588 sd = &per_cpu(phys_domains, i).sd;
7475 SD_INIT(sd, CPU); 7589 SD_INIT(sd, CPU);
7476 set_domain_attribute(sd, attr); 7590 set_domain_attribute(sd, attr);
7477 sd->span = *nodemask; 7591 cpumask_copy(sched_domain_span(sd), nodemask);
7478 sd->parent = p; 7592 sd->parent = p;
7479 if (p) 7593 if (p)
7480 p->child = sd; 7594 p->child = sd;
@@ -7482,11 +7596,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7482 7596
7483#ifdef CONFIG_SCHED_MC 7597#ifdef CONFIG_SCHED_MC
7484 p = sd; 7598 p = sd;
7485 sd = &per_cpu(core_domains, i); 7599 sd = &per_cpu(core_domains, i).sd;
7486 SD_INIT(sd, MC); 7600 SD_INIT(sd, MC);
7487 set_domain_attribute(sd, attr); 7601 set_domain_attribute(sd, attr);
7488 sd->span = cpu_coregroup_map(i); 7602 cpumask_and(sched_domain_span(sd), cpu_map,
7489 cpus_and(sd->span, sd->span, *cpu_map); 7603 cpu_coregroup_mask(i));
7490 sd->parent = p; 7604 sd->parent = p;
7491 p->child = sd; 7605 p->child = sd;
7492 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask); 7606 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7494,11 +7608,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7494 7608
7495#ifdef CONFIG_SCHED_SMT 7609#ifdef CONFIG_SCHED_SMT
7496 p = sd; 7610 p = sd;
7497 sd = &per_cpu(cpu_domains, i); 7611 sd = &per_cpu(cpu_domains, i).sd;
7498 SD_INIT(sd, SIBLING); 7612 SD_INIT(sd, SIBLING);
7499 set_domain_attribute(sd, attr); 7613 set_domain_attribute(sd, attr);
7500 sd->span = per_cpu(cpu_sibling_map, i); 7614 cpumask_and(sched_domain_span(sd),
7501 cpus_and(sd->span, sd->span, *cpu_map); 7615 &per_cpu(cpu_sibling_map, i), cpu_map);
7502 sd->parent = p; 7616 sd->parent = p;
7503 p->child = sd; 7617 p->child = sd;
7504 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask); 7618 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7507,13 +7621,10 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7507 7621
7508#ifdef CONFIG_SCHED_SMT 7622#ifdef CONFIG_SCHED_SMT
7509 /* Set up CPU (sibling) groups */ 7623 /* Set up CPU (sibling) groups */
7510 for_each_cpu_mask_nr(i, *cpu_map) { 7624 for_each_cpu(i, cpu_map) {
7511 SCHED_CPUMASK_VAR(this_sibling_map, allmasks); 7625 cpumask_and(this_sibling_map,
7512 SCHED_CPUMASK_VAR(send_covered, allmasks); 7626 &per_cpu(cpu_sibling_map, i), cpu_map);
7513 7627 if (i != cpumask_first(this_sibling_map))
7514 *this_sibling_map = per_cpu(cpu_sibling_map, i);
7515 cpus_and(*this_sibling_map, *this_sibling_map, *cpu_map);
7516 if (i != first_cpu(*this_sibling_map))
7517 continue; 7628 continue;
7518 7629
7519 init_sched_build_groups(this_sibling_map, cpu_map, 7630 init_sched_build_groups(this_sibling_map, cpu_map,
@@ -7524,13 +7635,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7524 7635
7525#ifdef CONFIG_SCHED_MC 7636#ifdef CONFIG_SCHED_MC
7526 /* Set up multi-core groups */ 7637 /* Set up multi-core groups */
7527 for_each_cpu_mask_nr(i, *cpu_map) { 7638 for_each_cpu(i, cpu_map) {
7528 SCHED_CPUMASK_VAR(this_core_map, allmasks); 7639 cpumask_and(this_core_map, cpu_coregroup_mask(i), cpu_map);
7529 SCHED_CPUMASK_VAR(send_covered, allmasks); 7640 if (i != cpumask_first(this_core_map))
7530
7531 *this_core_map = cpu_coregroup_map(i);
7532 cpus_and(*this_core_map, *this_core_map, *cpu_map);
7533 if (i != first_cpu(*this_core_map))
7534 continue; 7641 continue;
7535 7642
7536 init_sched_build_groups(this_core_map, cpu_map, 7643 init_sched_build_groups(this_core_map, cpu_map,
@@ -7541,12 +7648,8 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7541 7648
7542 /* Set up physical groups */ 7649 /* Set up physical groups */
7543 for (i = 0; i < nr_node_ids; i++) { 7650 for (i = 0; i < nr_node_ids; i++) {
7544 SCHED_CPUMASK_VAR(nodemask, allmasks); 7651 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7545 SCHED_CPUMASK_VAR(send_covered, allmasks); 7652 if (cpumask_empty(nodemask))
7546
7547 *nodemask = node_to_cpumask(i);
7548 cpus_and(*nodemask, *nodemask, *cpu_map);
7549 if (cpus_empty(*nodemask))
7550 continue; 7653 continue;
7551 7654
7552 init_sched_build_groups(nodemask, cpu_map, 7655 init_sched_build_groups(nodemask, cpu_map,
@@ -7557,8 +7660,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7557#ifdef CONFIG_NUMA 7660#ifdef CONFIG_NUMA
7558 /* Set up node groups */ 7661 /* Set up node groups */
7559 if (sd_allnodes) { 7662 if (sd_allnodes) {
7560 SCHED_CPUMASK_VAR(send_covered, allmasks);
7561
7562 init_sched_build_groups(cpu_map, cpu_map, 7663 init_sched_build_groups(cpu_map, cpu_map,
7563 &cpu_to_allnodes_group, 7664 &cpu_to_allnodes_group,
7564 send_covered, tmpmask); 7665 send_covered, tmpmask);
@@ -7567,58 +7668,53 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7567 for (i = 0; i < nr_node_ids; i++) { 7668 for (i = 0; i < nr_node_ids; i++) {
7568 /* Set up node groups */ 7669 /* Set up node groups */
7569 struct sched_group *sg, *prev; 7670 struct sched_group *sg, *prev;
7570 SCHED_CPUMASK_VAR(nodemask, allmasks);
7571 SCHED_CPUMASK_VAR(domainspan, allmasks);
7572 SCHED_CPUMASK_VAR(covered, allmasks);
7573 int j; 7671 int j;
7574 7672
7575 *nodemask = node_to_cpumask(i); 7673 cpumask_clear(covered);
7576 cpus_clear(*covered); 7674 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7577 7675 if (cpumask_empty(nodemask)) {
7578 cpus_and(*nodemask, *nodemask, *cpu_map);
7579 if (cpus_empty(*nodemask)) {
7580 sched_group_nodes[i] = NULL; 7676 sched_group_nodes[i] = NULL;
7581 continue; 7677 continue;
7582 } 7678 }
7583 7679
7584 sched_domain_node_span(i, domainspan); 7680 sched_domain_node_span(i, domainspan);
7585 cpus_and(*domainspan, *domainspan, *cpu_map); 7681 cpumask_and(domainspan, domainspan, cpu_map);
7586 7682
7587 sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i); 7683 sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
7684 GFP_KERNEL, i);
7588 if (!sg) { 7685 if (!sg) {
7589 printk(KERN_WARNING "Can not alloc domain group for " 7686 printk(KERN_WARNING "Can not alloc domain group for "
7590 "node %d\n", i); 7687 "node %d\n", i);
7591 goto error; 7688 goto error;
7592 } 7689 }
7593 sched_group_nodes[i] = sg; 7690 sched_group_nodes[i] = sg;
7594 for_each_cpu_mask_nr(j, *nodemask) { 7691 for_each_cpu(j, nodemask) {
7595 struct sched_domain *sd; 7692 struct sched_domain *sd;
7596 7693
7597 sd = &per_cpu(node_domains, j); 7694 sd = &per_cpu(node_domains, j).sd;
7598 sd->groups = sg; 7695 sd->groups = sg;
7599 } 7696 }
7600 sg->__cpu_power = 0; 7697 sg->__cpu_power = 0;
7601 sg->cpumask = *nodemask; 7698 cpumask_copy(sched_group_cpus(sg), nodemask);
7602 sg->next = sg; 7699 sg->next = sg;
7603 cpus_or(*covered, *covered, *nodemask); 7700 cpumask_or(covered, covered, nodemask);
7604 prev = sg; 7701 prev = sg;
7605 7702
7606 for (j = 0; j < nr_node_ids; j++) { 7703 for (j = 0; j < nr_node_ids; j++) {
7607 SCHED_CPUMASK_VAR(notcovered, allmasks);
7608 int n = (i + j) % nr_node_ids; 7704 int n = (i + j) % nr_node_ids;
7609 node_to_cpumask_ptr(pnodemask, n);
7610 7705
7611 cpus_complement(*notcovered, *covered); 7706 cpumask_complement(notcovered, covered);
7612 cpus_and(*tmpmask, *notcovered, *cpu_map); 7707 cpumask_and(tmpmask, notcovered, cpu_map);
7613 cpus_and(*tmpmask, *tmpmask, *domainspan); 7708 cpumask_and(tmpmask, tmpmask, domainspan);
7614 if (cpus_empty(*tmpmask)) 7709 if (cpumask_empty(tmpmask))
7615 break; 7710 break;
7616 7711
7617 cpus_and(*tmpmask, *tmpmask, *pnodemask); 7712 cpumask_and(tmpmask, tmpmask, cpumask_of_node(n));
7618 if (cpus_empty(*tmpmask)) 7713 if (cpumask_empty(tmpmask))
7619 continue; 7714 continue;
7620 7715
7621 sg = kmalloc_node(sizeof(struct sched_group), 7716 sg = kmalloc_node(sizeof(struct sched_group) +
7717 cpumask_size(),
7622 GFP_KERNEL, i); 7718 GFP_KERNEL, i);
7623 if (!sg) { 7719 if (!sg) {
7624 printk(KERN_WARNING 7720 printk(KERN_WARNING
@@ -7626,9 +7722,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7626 goto error; 7722 goto error;
7627 } 7723 }
7628 sg->__cpu_power = 0; 7724 sg->__cpu_power = 0;
7629 sg->cpumask = *tmpmask; 7725 cpumask_copy(sched_group_cpus(sg), tmpmask);
7630 sg->next = prev->next; 7726 sg->next = prev->next;
7631 cpus_or(*covered, *covered, *tmpmask); 7727 cpumask_or(covered, covered, tmpmask);
7632 prev->next = sg; 7728 prev->next = sg;
7633 prev = sg; 7729 prev = sg;
7634 } 7730 }
@@ -7637,22 +7733,22 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7637 7733
7638 /* Calculate CPU power for physical packages and nodes */ 7734 /* Calculate CPU power for physical packages and nodes */
7639#ifdef CONFIG_SCHED_SMT 7735#ifdef CONFIG_SCHED_SMT
7640 for_each_cpu_mask_nr(i, *cpu_map) { 7736 for_each_cpu(i, cpu_map) {
7641 struct sched_domain *sd = &per_cpu(cpu_domains, i); 7737 struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
7642 7738
7643 init_sched_groups_power(i, sd); 7739 init_sched_groups_power(i, sd);
7644 } 7740 }
7645#endif 7741#endif
7646#ifdef CONFIG_SCHED_MC 7742#ifdef CONFIG_SCHED_MC
7647 for_each_cpu_mask_nr(i, *cpu_map) { 7743 for_each_cpu(i, cpu_map) {
7648 struct sched_domain *sd = &per_cpu(core_domains, i); 7744 struct sched_domain *sd = &per_cpu(core_domains, i).sd;
7649 7745
7650 init_sched_groups_power(i, sd); 7746 init_sched_groups_power(i, sd);
7651 } 7747 }
7652#endif 7748#endif
7653 7749
7654 for_each_cpu_mask_nr(i, *cpu_map) { 7750 for_each_cpu(i, cpu_map) {
7655 struct sched_domain *sd = &per_cpu(phys_domains, i); 7751 struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
7656 7752
7657 init_sched_groups_power(i, sd); 7753 init_sched_groups_power(i, sd);
7658 } 7754 }
@@ -7664,56 +7760,87 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7664 if (sd_allnodes) { 7760 if (sd_allnodes) {
7665 struct sched_group *sg; 7761 struct sched_group *sg;
7666 7762
7667 cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg, 7763 cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
7668 tmpmask); 7764 tmpmask);
7669 init_numa_sched_groups_power(sg); 7765 init_numa_sched_groups_power(sg);
7670 } 7766 }
7671#endif 7767#endif
7672 7768
7673 /* Attach the domains */ 7769 /* Attach the domains */
7674 for_each_cpu_mask_nr(i, *cpu_map) { 7770 for_each_cpu(i, cpu_map) {
7675 struct sched_domain *sd; 7771 struct sched_domain *sd;
7676#ifdef CONFIG_SCHED_SMT 7772#ifdef CONFIG_SCHED_SMT
7677 sd = &per_cpu(cpu_domains, i); 7773 sd = &per_cpu(cpu_domains, i).sd;
7678#elif defined(CONFIG_SCHED_MC) 7774#elif defined(CONFIG_SCHED_MC)
7679 sd = &per_cpu(core_domains, i); 7775 sd = &per_cpu(core_domains, i).sd;
7680#else 7776#else
7681 sd = &per_cpu(phys_domains, i); 7777 sd = &per_cpu(phys_domains, i).sd;
7682#endif 7778#endif
7683 cpu_attach_domain(sd, rd, i); 7779 cpu_attach_domain(sd, rd, i);
7684 } 7780 }
7685 7781
7686 SCHED_CPUMASK_FREE((void *)allmasks); 7782 err = 0;
7687 return 0; 7783
7784free_tmpmask:
7785 free_cpumask_var(tmpmask);
7786free_send_covered:
7787 free_cpumask_var(send_covered);
7788free_this_core_map:
7789 free_cpumask_var(this_core_map);
7790free_this_sibling_map:
7791 free_cpumask_var(this_sibling_map);
7792free_nodemask:
7793 free_cpumask_var(nodemask);
7794free_notcovered:
7795#ifdef CONFIG_NUMA
7796 free_cpumask_var(notcovered);
7797free_covered:
7798 free_cpumask_var(covered);
7799free_domainspan:
7800 free_cpumask_var(domainspan);
7801out:
7802#endif
7803 return err;
7804
7805free_sched_groups:
7806#ifdef CONFIG_NUMA
7807 kfree(sched_group_nodes);
7808#endif
7809 goto free_tmpmask;
7688 7810
7689#ifdef CONFIG_NUMA 7811#ifdef CONFIG_NUMA
7690error: 7812error:
7691 free_sched_groups(cpu_map, tmpmask); 7813 free_sched_groups(cpu_map, tmpmask);
7692 SCHED_CPUMASK_FREE((void *)allmasks); 7814 free_rootdomain(rd);
7693 kfree(rd); 7815 goto free_tmpmask;
7694 return -ENOMEM;
7695#endif 7816#endif
7696} 7817}
7697 7818
7698static int build_sched_domains(const cpumask_t *cpu_map) 7819static int build_sched_domains(const struct cpumask *cpu_map)
7699{ 7820{
7700 return __build_sched_domains(cpu_map, NULL); 7821 return __build_sched_domains(cpu_map, NULL);
7701} 7822}
7702 7823
7703static cpumask_t *doms_cur; /* current sched domains */ 7824static struct cpumask *doms_cur; /* current sched domains */
7704static int ndoms_cur; /* number of sched domains in 'doms_cur' */ 7825static int ndoms_cur; /* number of sched domains in 'doms_cur' */
7705static struct sched_domain_attr *dattr_cur; 7826static struct sched_domain_attr *dattr_cur;
7706 /* attribues of custom domains in 'doms_cur' */ 7827 /* attribues of custom domains in 'doms_cur' */
7707 7828
7708/* 7829/*
7709 * Special case: If a kmalloc of a doms_cur partition (array of 7830 * Special case: If a kmalloc of a doms_cur partition (array of
7710 * cpumask_t) fails, then fallback to a single sched domain, 7831 * cpumask) fails, then fallback to a single sched domain,
7711 * as determined by the single cpumask_t fallback_doms. 7832 * as determined by the single cpumask fallback_doms.
7712 */ 7833 */
7713static cpumask_t fallback_doms; 7834static cpumask_var_t fallback_doms;
7714 7835
7715void __attribute__((weak)) arch_update_cpu_topology(void) 7836/*
7837 * arch_update_cpu_topology lets virtualized architectures update the
7838 * cpu core maps. It is supposed to return 1 if the topology changed
7839 * or 0 if it stayed the same.
7840 */
7841int __attribute__((weak)) arch_update_cpu_topology(void)
7716{ 7842{
7843 return 0;
7717} 7844}
7718 7845
7719/* 7846/*
@@ -7721,16 +7848,16 @@ void __attribute__((weak)) arch_update_cpu_topology(void)
7721 * For now this just excludes isolated cpus, but could be used to 7848 * For now this just excludes isolated cpus, but could be used to
7722 * exclude other special cases in the future. 7849 * exclude other special cases in the future.
7723 */ 7850 */
7724static int arch_init_sched_domains(const cpumask_t *cpu_map) 7851static int arch_init_sched_domains(const struct cpumask *cpu_map)
7725{ 7852{
7726 int err; 7853 int err;
7727 7854
7728 arch_update_cpu_topology(); 7855 arch_update_cpu_topology();
7729 ndoms_cur = 1; 7856 ndoms_cur = 1;
7730 doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 7857 doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
7731 if (!doms_cur) 7858 if (!doms_cur)
7732 doms_cur = &fallback_doms; 7859 doms_cur = fallback_doms;
7733 cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map); 7860 cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
7734 dattr_cur = NULL; 7861 dattr_cur = NULL;
7735 err = build_sched_domains(doms_cur); 7862 err = build_sched_domains(doms_cur);
7736 register_sched_domain_sysctl(); 7863 register_sched_domain_sysctl();
@@ -7738,8 +7865,8 @@ static int arch_init_sched_domains(const cpumask_t *cpu_map)
7738 return err; 7865 return err;
7739} 7866}
7740 7867
7741static void arch_destroy_sched_domains(const cpumask_t *cpu_map, 7868static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
7742 cpumask_t *tmpmask) 7869 struct cpumask *tmpmask)
7743{ 7870{
7744 free_sched_groups(cpu_map, tmpmask); 7871 free_sched_groups(cpu_map, tmpmask);
7745} 7872}
@@ -7748,17 +7875,16 @@ static void arch_destroy_sched_domains(const cpumask_t *cpu_map,
7748 * Detach sched domains from a group of cpus specified in cpu_map 7875 * Detach sched domains from a group of cpus specified in cpu_map
7749 * These cpus will now be attached to the NULL domain 7876 * These cpus will now be attached to the NULL domain
7750 */ 7877 */
7751static void detach_destroy_domains(const cpumask_t *cpu_map) 7878static void detach_destroy_domains(const struct cpumask *cpu_map)
7752{ 7879{
7753 cpumask_t tmpmask; 7880 /* Save because hotplug lock held. */
7881 static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
7754 int i; 7882 int i;
7755 7883
7756 unregister_sched_domain_sysctl(); 7884 for_each_cpu(i, cpu_map)
7757
7758 for_each_cpu_mask_nr(i, *cpu_map)
7759 cpu_attach_domain(NULL, &def_root_domain, i); 7885 cpu_attach_domain(NULL, &def_root_domain, i);
7760 synchronize_sched(); 7886 synchronize_sched();
7761 arch_destroy_sched_domains(cpu_map, &tmpmask); 7887 arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
7762} 7888}
7763 7889
7764/* handle null as "default" */ 7890/* handle null as "default" */
@@ -7783,7 +7909,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7783 * doms_new[] to the current sched domain partitioning, doms_cur[]. 7909 * doms_new[] to the current sched domain partitioning, doms_cur[].
7784 * It destroys each deleted domain and builds each new domain. 7910 * It destroys each deleted domain and builds each new domain.
7785 * 7911 *
7786 * 'doms_new' is an array of cpumask_t's of length 'ndoms_new'. 7912 * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
7787 * The masks don't intersect (don't overlap.) We should setup one 7913 * The masks don't intersect (don't overlap.) We should setup one
7788 * sched domain for each mask. CPUs not in any of the cpumasks will 7914 * sched domain for each mask. CPUs not in any of the cpumasks will
7789 * not be load balanced. If the same cpumask appears both in the 7915 * not be load balanced. If the same cpumask appears both in the
@@ -7797,28 +7923,33 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7797 * the single partition 'fallback_doms', it also forces the domains 7923 * the single partition 'fallback_doms', it also forces the domains
7798 * to be rebuilt. 7924 * to be rebuilt.
7799 * 7925 *
7800 * If doms_new == NULL it will be replaced with cpu_online_map. 7926 * If doms_new == NULL it will be replaced with cpu_online_mask.
7801 * ndoms_new == 0 is a special case for destroying existing domains, 7927 * ndoms_new == 0 is a special case for destroying existing domains,
7802 * and it will not create the default domain. 7928 * and it will not create the default domain.
7803 * 7929 *
7804 * Call with hotplug lock held 7930 * Call with hotplug lock held
7805 */ 7931 */
7806void partition_sched_domains(int ndoms_new, cpumask_t *doms_new, 7932/* FIXME: Change to struct cpumask *doms_new[] */
7933void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
7807 struct sched_domain_attr *dattr_new) 7934 struct sched_domain_attr *dattr_new)
7808{ 7935{
7809 int i, j, n; 7936 int i, j, n;
7937 int new_topology;
7810 7938
7811 mutex_lock(&sched_domains_mutex); 7939 mutex_lock(&sched_domains_mutex);
7812 7940
7813 /* always unregister in case we don't destroy any domains */ 7941 /* always unregister in case we don't destroy any domains */
7814 unregister_sched_domain_sysctl(); 7942 unregister_sched_domain_sysctl();
7815 7943
7944 /* Let architecture update cpu core mappings. */
7945 new_topology = arch_update_cpu_topology();
7946
7816 n = doms_new ? ndoms_new : 0; 7947 n = doms_new ? ndoms_new : 0;
7817 7948
7818 /* Destroy deleted domains */ 7949 /* Destroy deleted domains */
7819 for (i = 0; i < ndoms_cur; i++) { 7950 for (i = 0; i < ndoms_cur; i++) {
7820 for (j = 0; j < n; j++) { 7951 for (j = 0; j < n && !new_topology; j++) {
7821 if (cpus_equal(doms_cur[i], doms_new[j]) 7952 if (cpumask_equal(&doms_cur[i], &doms_new[j])
7822 && dattrs_equal(dattr_cur, i, dattr_new, j)) 7953 && dattrs_equal(dattr_cur, i, dattr_new, j))
7823 goto match1; 7954 goto match1;
7824 } 7955 }
@@ -7830,15 +7961,15 @@ match1:
7830 7961
7831 if (doms_new == NULL) { 7962 if (doms_new == NULL) {
7832 ndoms_cur = 0; 7963 ndoms_cur = 0;
7833 doms_new = &fallback_doms; 7964 doms_new = fallback_doms;
7834 cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map); 7965 cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
7835 dattr_new = NULL; 7966 WARN_ON_ONCE(dattr_new);
7836 } 7967 }
7837 7968
7838 /* Build new domains */ 7969 /* Build new domains */
7839 for (i = 0; i < ndoms_new; i++) { 7970 for (i = 0; i < ndoms_new; i++) {
7840 for (j = 0; j < ndoms_cur; j++) { 7971 for (j = 0; j < ndoms_cur && !new_topology; j++) {
7841 if (cpus_equal(doms_new[i], doms_cur[j]) 7972 if (cpumask_equal(&doms_new[i], &doms_cur[j])
7842 && dattrs_equal(dattr_new, i, dattr_cur, j)) 7973 && dattrs_equal(dattr_new, i, dattr_cur, j))
7843 goto match2; 7974 goto match2;
7844 } 7975 }
@@ -7850,7 +7981,7 @@ match2:
7850 } 7981 }
7851 7982
7852 /* Remember the new sched domains */ 7983 /* Remember the new sched domains */
7853 if (doms_cur != &fallback_doms) 7984 if (doms_cur != fallback_doms)
7854 kfree(doms_cur); 7985 kfree(doms_cur);
7855 kfree(dattr_cur); /* kfree(NULL) is safe */ 7986 kfree(dattr_cur); /* kfree(NULL) is safe */
7856 doms_cur = doms_new; 7987 doms_cur = doms_new;
@@ -7863,7 +7994,7 @@ match2:
7863} 7994}
7864 7995
7865#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT) 7996#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
7866int arch_reinit_sched_domains(void) 7997static void arch_reinit_sched_domains(void)
7867{ 7998{
7868 get_online_cpus(); 7999 get_online_cpus();
7869 8000
@@ -7872,25 +8003,33 @@ int arch_reinit_sched_domains(void)
7872 8003
7873 rebuild_sched_domains(); 8004 rebuild_sched_domains();
7874 put_online_cpus(); 8005 put_online_cpus();
7875
7876 return 0;
7877} 8006}
7878 8007
7879static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt) 8008static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
7880{ 8009{
7881 int ret; 8010 unsigned int level = 0;
7882 8011
7883 if (buf[0] != '0' && buf[0] != '1') 8012 if (sscanf(buf, "%u", &level) != 1)
8013 return -EINVAL;
8014
8015 /*
8016 * level is always be positive so don't check for
8017 * level < POWERSAVINGS_BALANCE_NONE which is 0
8018 * What happens on 0 or 1 byte write,
8019 * need to check for count as well?
8020 */
8021
8022 if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
7884 return -EINVAL; 8023 return -EINVAL;
7885 8024
7886 if (smt) 8025 if (smt)
7887 sched_smt_power_savings = (buf[0] == '1'); 8026 sched_smt_power_savings = level;
7888 else 8027 else
7889 sched_mc_power_savings = (buf[0] == '1'); 8028 sched_mc_power_savings = level;
7890 8029
7891 ret = arch_reinit_sched_domains(); 8030 arch_reinit_sched_domains();
7892 8031
7893 return ret ? ret : count; 8032 return count;
7894} 8033}
7895 8034
7896#ifdef CONFIG_SCHED_MC 8035#ifdef CONFIG_SCHED_MC
@@ -7925,7 +8064,7 @@ static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
7925 sched_smt_power_savings_store); 8064 sched_smt_power_savings_store);
7926#endif 8065#endif
7927 8066
7928int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls) 8067int __init sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
7929{ 8068{
7930 int err = 0; 8069 int err = 0;
7931 8070
@@ -7990,7 +8129,9 @@ static int update_runtime(struct notifier_block *nfb,
7990 8129
7991void __init sched_init_smp(void) 8130void __init sched_init_smp(void)
7992{ 8131{
7993 cpumask_t non_isolated_cpus; 8132 cpumask_var_t non_isolated_cpus;
8133
8134 alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
7994 8135
7995#if defined(CONFIG_NUMA) 8136#if defined(CONFIG_NUMA)
7996 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **), 8137 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
@@ -7999,10 +8140,10 @@ void __init sched_init_smp(void)
7999#endif 8140#endif
8000 get_online_cpus(); 8141 get_online_cpus();
8001 mutex_lock(&sched_domains_mutex); 8142 mutex_lock(&sched_domains_mutex);
8002 arch_init_sched_domains(&cpu_online_map); 8143 arch_init_sched_domains(cpu_online_mask);
8003 cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map); 8144 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
8004 if (cpus_empty(non_isolated_cpus)) 8145 if (cpumask_empty(non_isolated_cpus))
8005 cpu_set(smp_processor_id(), non_isolated_cpus); 8146 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
8006 mutex_unlock(&sched_domains_mutex); 8147 mutex_unlock(&sched_domains_mutex);
8007 put_online_cpus(); 8148 put_online_cpus();
8008 8149
@@ -8017,9 +8158,13 @@ void __init sched_init_smp(void)
8017 init_hrtick(); 8158 init_hrtick();
8018 8159
8019 /* Move init over to a non-isolated CPU */ 8160 /* Move init over to a non-isolated CPU */
8020 if (set_cpus_allowed_ptr(current, &non_isolated_cpus) < 0) 8161 if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
8021 BUG(); 8162 BUG();
8022 sched_init_granularity(); 8163 sched_init_granularity();
8164 free_cpumask_var(non_isolated_cpus);
8165
8166 alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
8167 init_sched_rt_class();
8023} 8168}
8024#else 8169#else
8025void __init sched_init_smp(void) 8170void __init sched_init_smp(void)
@@ -8334,6 +8479,15 @@ void __init sched_init(void)
8334 */ 8479 */
8335 current->sched_class = &fair_sched_class; 8480 current->sched_class = &fair_sched_class;
8336 8481
8482 /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
8483 alloc_bootmem_cpumask_var(&nohz_cpu_mask);
8484#ifdef CONFIG_SMP
8485#ifdef CONFIG_NO_HZ
8486 alloc_bootmem_cpumask_var(&nohz.cpu_mask);
8487#endif
8488 alloc_bootmem_cpumask_var(&cpu_isolated_map);
8489#endif /* SMP */
8490
8337 scheduler_running = 1; 8491 scheduler_running = 1;
8338} 8492}
8339 8493
@@ -8492,7 +8646,7 @@ static
8492int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 8646int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
8493{ 8647{
8494 struct cfs_rq *cfs_rq; 8648 struct cfs_rq *cfs_rq;
8495 struct sched_entity *se, *parent_se; 8649 struct sched_entity *se;
8496 struct rq *rq; 8650 struct rq *rq;
8497 int i; 8651 int i;
8498 8652
@@ -8508,18 +8662,17 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
8508 for_each_possible_cpu(i) { 8662 for_each_possible_cpu(i) {
8509 rq = cpu_rq(i); 8663 rq = cpu_rq(i);
8510 8664
8511 cfs_rq = kmalloc_node(sizeof(struct cfs_rq), 8665 cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
8512 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8666 GFP_KERNEL, cpu_to_node(i));
8513 if (!cfs_rq) 8667 if (!cfs_rq)
8514 goto err; 8668 goto err;
8515 8669
8516 se = kmalloc_node(sizeof(struct sched_entity), 8670 se = kzalloc_node(sizeof(struct sched_entity),
8517 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8671 GFP_KERNEL, cpu_to_node(i));
8518 if (!se) 8672 if (!se)
8519 goto err; 8673 goto err;
8520 8674
8521 parent_se = parent ? parent->se[i] : NULL; 8675 init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
8522 init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent_se);
8523 } 8676 }
8524 8677
8525 return 1; 8678 return 1;
@@ -8580,7 +8733,7 @@ static
8580int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) 8733int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8581{ 8734{
8582 struct rt_rq *rt_rq; 8735 struct rt_rq *rt_rq;
8583 struct sched_rt_entity *rt_se, *parent_se; 8736 struct sched_rt_entity *rt_se;
8584 struct rq *rq; 8737 struct rq *rq;
8585 int i; 8738 int i;
8586 8739
@@ -8597,18 +8750,17 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8597 for_each_possible_cpu(i) { 8750 for_each_possible_cpu(i) {
8598 rq = cpu_rq(i); 8751 rq = cpu_rq(i);
8599 8752
8600 rt_rq = kmalloc_node(sizeof(struct rt_rq), 8753 rt_rq = kzalloc_node(sizeof(struct rt_rq),
8601 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8754 GFP_KERNEL, cpu_to_node(i));
8602 if (!rt_rq) 8755 if (!rt_rq)
8603 goto err; 8756 goto err;
8604 8757
8605 rt_se = kmalloc_node(sizeof(struct sched_rt_entity), 8758 rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
8606 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8759 GFP_KERNEL, cpu_to_node(i));
8607 if (!rt_se) 8760 if (!rt_se)
8608 goto err; 8761 goto err;
8609 8762
8610 parent_se = parent ? parent->rt_se[i] : NULL; 8763 init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
8611 init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent_se);
8612 } 8764 }
8613 8765
8614 return 1; 8766 return 1;
@@ -8898,6 +9050,13 @@ static int tg_schedulable(struct task_group *tg, void *data)
8898 runtime = d->rt_runtime; 9050 runtime = d->rt_runtime;
8899 } 9051 }
8900 9052
9053#ifdef CONFIG_USER_SCHED
9054 if (tg == &root_task_group) {
9055 period = global_rt_period();
9056 runtime = global_rt_runtime();
9057 }
9058#endif
9059
8901 /* 9060 /*
8902 * Cannot have more runtime than the period. 9061 * Cannot have more runtime than the period.
8903 */ 9062 */
@@ -9251,11 +9410,12 @@ struct cgroup_subsys cpu_cgroup_subsys = {
9251 * (balbir@in.ibm.com). 9410 * (balbir@in.ibm.com).
9252 */ 9411 */
9253 9412
9254/* track cpu usage of a group of tasks */ 9413/* track cpu usage of a group of tasks and its child groups */
9255struct cpuacct { 9414struct cpuacct {
9256 struct cgroup_subsys_state css; 9415 struct cgroup_subsys_state css;
9257 /* cpuusage holds pointer to a u64-type object on every cpu */ 9416 /* cpuusage holds pointer to a u64-type object on every cpu */
9258 u64 *cpuusage; 9417 u64 *cpuusage;
9418 struct cpuacct *parent;
9259}; 9419};
9260 9420
9261struct cgroup_subsys cpuacct_subsys; 9421struct cgroup_subsys cpuacct_subsys;
@@ -9289,6 +9449,9 @@ static struct cgroup_subsys_state *cpuacct_create(
9289 return ERR_PTR(-ENOMEM); 9449 return ERR_PTR(-ENOMEM);
9290 } 9450 }
9291 9451
9452 if (cgrp->parent)
9453 ca->parent = cgroup_ca(cgrp->parent);
9454
9292 return &ca->css; 9455 return &ca->css;
9293} 9456}
9294 9457
@@ -9302,6 +9465,41 @@ cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
9302 kfree(ca); 9465 kfree(ca);
9303} 9466}
9304 9467
9468static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
9469{
9470 u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
9471 u64 data;
9472
9473#ifndef CONFIG_64BIT
9474 /*
9475 * Take rq->lock to make 64-bit read safe on 32-bit platforms.
9476 */
9477 spin_lock_irq(&cpu_rq(cpu)->lock);
9478 data = *cpuusage;
9479 spin_unlock_irq(&cpu_rq(cpu)->lock);
9480#else
9481 data = *cpuusage;
9482#endif
9483
9484 return data;
9485}
9486
9487static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
9488{
9489 u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
9490
9491#ifndef CONFIG_64BIT
9492 /*
9493 * Take rq->lock to make 64-bit write safe on 32-bit platforms.
9494 */
9495 spin_lock_irq(&cpu_rq(cpu)->lock);
9496 *cpuusage = val;
9497 spin_unlock_irq(&cpu_rq(cpu)->lock);
9498#else
9499 *cpuusage = val;
9500#endif
9501}
9502
9305/* return total cpu usage (in nanoseconds) of a group */ 9503/* return total cpu usage (in nanoseconds) of a group */
9306static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft) 9504static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
9307{ 9505{
@@ -9309,17 +9507,8 @@ static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
9309 u64 totalcpuusage = 0; 9507 u64 totalcpuusage = 0;
9310 int i; 9508 int i;
9311 9509
9312 for_each_possible_cpu(i) { 9510 for_each_present_cpu(i)
9313 u64 *cpuusage = percpu_ptr(ca->cpuusage, i); 9511 totalcpuusage += cpuacct_cpuusage_read(ca, i);
9314
9315 /*
9316 * Take rq->lock to make 64-bit addition safe on 32-bit
9317 * platforms.
9318 */
9319 spin_lock_irq(&cpu_rq(i)->lock);
9320 totalcpuusage += *cpuusage;
9321 spin_unlock_irq(&cpu_rq(i)->lock);
9322 }
9323 9512
9324 return totalcpuusage; 9513 return totalcpuusage;
9325} 9514}
@@ -9336,23 +9525,39 @@ static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
9336 goto out; 9525 goto out;
9337 } 9526 }
9338 9527
9339 for_each_possible_cpu(i) { 9528 for_each_present_cpu(i)
9340 u64 *cpuusage = percpu_ptr(ca->cpuusage, i); 9529 cpuacct_cpuusage_write(ca, i, 0);
9341 9530
9342 spin_lock_irq(&cpu_rq(i)->lock);
9343 *cpuusage = 0;
9344 spin_unlock_irq(&cpu_rq(i)->lock);
9345 }
9346out: 9531out:
9347 return err; 9532 return err;
9348} 9533}
9349 9534
9535static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
9536 struct seq_file *m)
9537{
9538 struct cpuacct *ca = cgroup_ca(cgroup);
9539 u64 percpu;
9540 int i;
9541
9542 for_each_present_cpu(i) {
9543 percpu = cpuacct_cpuusage_read(ca, i);
9544 seq_printf(m, "%llu ", (unsigned long long) percpu);
9545 }
9546 seq_printf(m, "\n");
9547 return 0;
9548}
9549
9350static struct cftype files[] = { 9550static struct cftype files[] = {
9351 { 9551 {
9352 .name = "usage", 9552 .name = "usage",
9353 .read_u64 = cpuusage_read, 9553 .read_u64 = cpuusage_read,
9354 .write_u64 = cpuusage_write, 9554 .write_u64 = cpuusage_write,
9355 }, 9555 },
9556 {
9557 .name = "usage_percpu",
9558 .read_seq_string = cpuacct_percpu_seq_read,
9559 },
9560
9356}; 9561};
9357 9562
9358static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp) 9563static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
@@ -9368,14 +9573,16 @@ static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
9368static void cpuacct_charge(struct task_struct *tsk, u64 cputime) 9573static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
9369{ 9574{
9370 struct cpuacct *ca; 9575 struct cpuacct *ca;
9576 int cpu;
9371 9577
9372 if (!cpuacct_subsys.active) 9578 if (!cpuacct_subsys.active)
9373 return; 9579 return;
9374 9580
9581 cpu = task_cpu(tsk);
9375 ca = task_ca(tsk); 9582 ca = task_ca(tsk);
9376 if (ca) {
9377 u64 *cpuusage = percpu_ptr(ca->cpuusage, task_cpu(tsk));
9378 9583
9584 for (; ca; ca = ca->parent) {
9585 u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
9379 *cpuusage += cputime; 9586 *cpuusage += cputime;
9380 } 9587 }
9381} 9588}
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-12 02:38:43 -0400