aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched.c')
-rw-r--r--kernel/sched.c258
1 files changed, 164 insertions, 94 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index ce375cb551e9..5b307da827ef 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -74,6 +74,8 @@
74#include <asm/tlb.h> 74#include <asm/tlb.h>
75#include <asm/irq_regs.h> 75#include <asm/irq_regs.h>
76 76
77#include "sched_cpupri.h"
78
77/* 79/*
78 * Convert user-nice values [ -20 ... 0 ... 19 ] 80 * Convert user-nice values [ -20 ... 0 ... 19 ]
79 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], 81 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
@@ -289,15 +291,15 @@ struct task_group root_task_group;
289static DEFINE_PER_CPU(struct sched_entity, init_sched_entity); 291static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
290/* Default task group's cfs_rq on each cpu */ 292/* Default task group's cfs_rq on each cpu */
291static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp; 293static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
292#endif 294#endif /* CONFIG_FAIR_GROUP_SCHED */
293 295
294#ifdef CONFIG_RT_GROUP_SCHED 296#ifdef CONFIG_RT_GROUP_SCHED
295static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity); 297static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
296static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp; 298static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
297#endif 299#endif /* CONFIG_RT_GROUP_SCHED */
298#else 300#else /* !CONFIG_FAIR_GROUP_SCHED */
299#define root_task_group init_task_group 301#define root_task_group init_task_group
300#endif 302#endif /* CONFIG_FAIR_GROUP_SCHED */
301 303
302/* task_group_lock serializes add/remove of task groups and also changes to 304/* task_group_lock serializes add/remove of task groups and also changes to
303 * a task group's cpu shares. 305 * a task group's cpu shares.
@@ -307,9 +309,9 @@ static DEFINE_SPINLOCK(task_group_lock);
307#ifdef CONFIG_FAIR_GROUP_SCHED 309#ifdef CONFIG_FAIR_GROUP_SCHED
308#ifdef CONFIG_USER_SCHED 310#ifdef CONFIG_USER_SCHED
309# define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD) 311# define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD)
310#else 312#else /* !CONFIG_USER_SCHED */
311# define INIT_TASK_GROUP_LOAD NICE_0_LOAD 313# define INIT_TASK_GROUP_LOAD NICE_0_LOAD
312#endif 314#endif /* CONFIG_USER_SCHED */
313 315
314/* 316/*
315 * A weight of 0 or 1 can cause arithmetics problems. 317 * A weight of 0 or 1 can cause arithmetics problems.
@@ -452,6 +454,9 @@ struct root_domain {
452 */ 454 */
453 cpumask_t rto_mask; 455 cpumask_t rto_mask;
454 atomic_t rto_count; 456 atomic_t rto_count;
457#ifdef CONFIG_SMP
458 struct cpupri cpupri;
459#endif
455}; 460};
456 461
457/* 462/*
@@ -526,6 +531,7 @@ struct rq {
526 int push_cpu; 531 int push_cpu;
527 /* cpu of this runqueue: */ 532 /* cpu of this runqueue: */
528 int cpu; 533 int cpu;
534 int online;
529 535
530 struct task_struct *migration_thread; 536 struct task_struct *migration_thread;
531 struct list_head migration_queue; 537 struct list_head migration_queue;
@@ -1313,15 +1319,15 @@ void wake_up_idle_cpu(int cpu)
1313 if (!tsk_is_polling(rq->idle)) 1319 if (!tsk_is_polling(rq->idle))
1314 smp_send_reschedule(cpu); 1320 smp_send_reschedule(cpu);
1315} 1321}
1316#endif 1322#endif /* CONFIG_NO_HZ */
1317 1323
1318#else 1324#else /* !CONFIG_SMP */
1319static void __resched_task(struct task_struct *p, int tif_bit) 1325static void __resched_task(struct task_struct *p, int tif_bit)
1320{ 1326{
1321 assert_spin_locked(&task_rq(p)->lock); 1327 assert_spin_locked(&task_rq(p)->lock);
1322 set_tsk_thread_flag(p, tif_bit); 1328 set_tsk_thread_flag(p, tif_bit);
1323} 1329}
1324#endif 1330#endif /* CONFIG_SMP */
1325 1331
1326#if BITS_PER_LONG == 32 1332#if BITS_PER_LONG == 32
1327# define WMULT_CONST (~0UL) 1333# define WMULT_CONST (~0UL)
@@ -1481,16 +1487,8 @@ static unsigned long source_load(int cpu, int type);
1481static unsigned long target_load(int cpu, int type); 1487static unsigned long target_load(int cpu, int type);
1482static unsigned long cpu_avg_load_per_task(int cpu); 1488static unsigned long cpu_avg_load_per_task(int cpu);
1483static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd); 1489static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
1484#else /* CONFIG_SMP */
1485
1486#ifdef CONFIG_FAIR_GROUP_SCHED
1487static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
1488{
1489}
1490#endif 1490#endif
1491 1491
1492#endif /* CONFIG_SMP */
1493
1494#include "sched_stats.h" 1492#include "sched_stats.h"
1495#include "sched_idletask.c" 1493#include "sched_idletask.c"
1496#include "sched_fair.c" 1494#include "sched_fair.c"
@@ -1500,6 +1498,8 @@ static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
1500#endif 1498#endif
1501 1499
1502#define sched_class_highest (&rt_sched_class) 1500#define sched_class_highest (&rt_sched_class)
1501#define for_each_class(class) \
1502 for (class = sched_class_highest; class; class = class->next)
1503 1503
1504static inline void inc_load(struct rq *rq, const struct task_struct *p) 1504static inline void inc_load(struct rq *rq, const struct task_struct *p)
1505{ 1505{
@@ -1636,12 +1636,6 @@ inline int task_curr(const struct task_struct *p)
1636 return cpu_curr(task_cpu(p)) == p; 1636 return cpu_curr(task_cpu(p)) == p;
1637} 1637}
1638 1638
1639/* Used instead of source_load when we know the type == 0 */
1640unsigned long weighted_cpuload(const int cpu)
1641{
1642 return cpu_rq(cpu)->load.weight;
1643}
1644
1645static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) 1639static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
1646{ 1640{
1647 set_task_rq(p, cpu); 1641 set_task_rq(p, cpu);
@@ -1670,6 +1664,12 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p,
1670 1664
1671#ifdef CONFIG_SMP 1665#ifdef CONFIG_SMP
1672 1666
1667/* Used instead of source_load when we know the type == 0 */
1668static unsigned long weighted_cpuload(const int cpu)
1669{
1670 return cpu_rq(cpu)->load.weight;
1671}
1672
1673/* 1673/*
1674 * Is this task likely cache-hot: 1674 * Is this task likely cache-hot:
1675 */ 1675 */
@@ -2131,7 +2131,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2131 } 2131 }
2132 } 2132 }
2133 } 2133 }
2134#endif 2134#endif /* CONFIG_SCHEDSTATS */
2135 2135
2136out_activate: 2136out_activate:
2137#endif /* CONFIG_SMP */ 2137#endif /* CONFIG_SMP */
@@ -2331,7 +2331,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr,
2331 notifier->ops->sched_out(notifier, next); 2331 notifier->ops->sched_out(notifier, next);
2332} 2332}
2333 2333
2334#else 2334#else /* !CONFIG_PREEMPT_NOTIFIERS */
2335 2335
2336static void fire_sched_in_preempt_notifiers(struct task_struct *curr) 2336static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2337{ 2337{
@@ -2343,7 +2343,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr,
2343{ 2343{
2344} 2344}
2345 2345
2346#endif 2346#endif /* CONFIG_PREEMPT_NOTIFIERS */
2347 2347
2348/** 2348/**
2349 * prepare_task_switch - prepare to switch tasks 2349 * prepare_task_switch - prepare to switch tasks
@@ -3672,6 +3672,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3672 /* Earliest time when we have to do rebalance again */ 3672 /* Earliest time when we have to do rebalance again */
3673 unsigned long next_balance = jiffies + 60*HZ; 3673 unsigned long next_balance = jiffies + 60*HZ;
3674 int update_next_balance = 0; 3674 int update_next_balance = 0;
3675 int need_serialize;
3675 cpumask_t tmp; 3676 cpumask_t tmp;
3676 3677
3677 for_each_domain(cpu, sd) { 3678 for_each_domain(cpu, sd) {
@@ -3689,8 +3690,9 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3689 if (interval > HZ*NR_CPUS/10) 3690 if (interval > HZ*NR_CPUS/10)
3690 interval = HZ*NR_CPUS/10; 3691 interval = HZ*NR_CPUS/10;
3691 3692
3693 need_serialize = sd->flags & SD_SERIALIZE;
3692 3694
3693 if (sd->flags & SD_SERIALIZE) { 3695 if (need_serialize) {
3694 if (!spin_trylock(&balancing)) 3696 if (!spin_trylock(&balancing))
3695 goto out; 3697 goto out;
3696 } 3698 }
@@ -3706,7 +3708,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3706 } 3708 }
3707 sd->last_balance = jiffies; 3709 sd->last_balance = jiffies;
3708 } 3710 }
3709 if (sd->flags & SD_SERIALIZE) 3711 if (need_serialize)
3710 spin_unlock(&balancing); 3712 spin_unlock(&balancing);
3711out: 3713out:
3712 if (time_after(next_balance, sd->last_balance + interval)) { 3714 if (time_after(next_balance, sd->last_balance + interval)) {
@@ -4070,6 +4072,7 @@ static noinline void __schedule_bug(struct task_struct *prev)
4070 prev->comm, prev->pid, preempt_count()); 4072 prev->comm, prev->pid, preempt_count());
4071 4073
4072 debug_show_held_locks(prev); 4074 debug_show_held_locks(prev);
4075 print_modules();
4073 if (irqs_disabled()) 4076 if (irqs_disabled())
4074 print_irqtrace_events(prev); 4077 print_irqtrace_events(prev);
4075 4078
@@ -4143,7 +4146,7 @@ asmlinkage void __sched schedule(void)
4143 struct task_struct *prev, *next; 4146 struct task_struct *prev, *next;
4144 unsigned long *switch_count; 4147 unsigned long *switch_count;
4145 struct rq *rq; 4148 struct rq *rq;
4146 int cpu; 4149 int cpu, hrtick = sched_feat(HRTICK);
4147 4150
4148need_resched: 4151need_resched:
4149 preempt_disable(); 4152 preempt_disable();
@@ -4158,7 +4161,8 @@ need_resched_nonpreemptible:
4158 4161
4159 schedule_debug(prev); 4162 schedule_debug(prev);
4160 4163
4161 hrtick_clear(rq); 4164 if (hrtick)
4165 hrtick_clear(rq);
4162 4166
4163 /* 4167 /*
4164 * Do the rq-clock update outside the rq lock: 4168 * Do the rq-clock update outside the rq lock:
@@ -4204,7 +4208,8 @@ need_resched_nonpreemptible:
4204 } else 4208 } else
4205 spin_unlock_irq(&rq->lock); 4209 spin_unlock_irq(&rq->lock);
4206 4210
4207 hrtick_set(rq); 4211 if (hrtick)
4212 hrtick_set(rq);
4208 4213
4209 if (unlikely(reacquire_kernel_lock(current) < 0)) 4214 if (unlikely(reacquire_kernel_lock(current) < 0))
4210 goto need_resched_nonpreemptible; 4215 goto need_resched_nonpreemptible;
@@ -5072,24 +5077,6 @@ asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
5072 return sched_setaffinity(pid, &new_mask); 5077 return sched_setaffinity(pid, &new_mask);
5073} 5078}
5074 5079
5075/*
5076 * Represents all cpu's present in the system
5077 * In systems capable of hotplug, this map could dynamically grow
5078 * as new cpu's are detected in the system via any platform specific
5079 * method, such as ACPI for e.g.
5080 */
5081
5082cpumask_t cpu_present_map __read_mostly;
5083EXPORT_SYMBOL(cpu_present_map);
5084
5085#ifndef CONFIG_SMP
5086cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
5087EXPORT_SYMBOL(cpu_online_map);
5088
5089cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
5090EXPORT_SYMBOL(cpu_possible_map);
5091#endif
5092
5093long sched_getaffinity(pid_t pid, cpumask_t *mask) 5080long sched_getaffinity(pid_t pid, cpumask_t *mask)
5094{ 5081{
5095 struct task_struct *p; 5082 struct task_struct *p;
@@ -5573,6 +5560,12 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5573 goto out; 5560 goto out;
5574 } 5561 }
5575 5562
5563 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
5564 !cpus_equal(p->cpus_allowed, *new_mask))) {
5565 ret = -EINVAL;
5566 goto out;
5567 }
5568
5576 if (p->sched_class->set_cpus_allowed) 5569 if (p->sched_class->set_cpus_allowed)
5577 p->sched_class->set_cpus_allowed(p, new_mask); 5570 p->sched_class->set_cpus_allowed(p, new_mask);
5578 else { 5571 else {
@@ -6060,6 +6053,36 @@ static void unregister_sched_domain_sysctl(void)
6060} 6053}
6061#endif 6054#endif
6062 6055
6056static void set_rq_online(struct rq *rq)
6057{
6058 if (!rq->online) {
6059 const struct sched_class *class;
6060
6061 cpu_set(rq->cpu, rq->rd->online);
6062 rq->online = 1;
6063
6064 for_each_class(class) {
6065 if (class->rq_online)
6066 class->rq_online(rq);
6067 }
6068 }
6069}
6070
6071static void set_rq_offline(struct rq *rq)
6072{
6073 if (rq->online) {
6074 const struct sched_class *class;
6075
6076 for_each_class(class) {
6077 if (class->rq_offline)
6078 class->rq_offline(rq);
6079 }
6080
6081 cpu_clear(rq->cpu, rq->rd->online);
6082 rq->online = 0;
6083 }
6084}
6085
6063/* 6086/*
6064 * migration_call - callback that gets triggered when a CPU is added. 6087 * migration_call - callback that gets triggered when a CPU is added.
6065 * Here we can start up the necessary migration thread for the new CPU. 6088 * Here we can start up the necessary migration thread for the new CPU.
@@ -6097,7 +6120,8 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6097 spin_lock_irqsave(&rq->lock, flags); 6120 spin_lock_irqsave(&rq->lock, flags);
6098 if (rq->rd) { 6121 if (rq->rd) {
6099 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6122 BUG_ON(!cpu_isset(cpu, rq->rd->span));
6100 cpu_set(cpu, rq->rd->online); 6123
6124 set_rq_online(rq);
6101 } 6125 }
6102 spin_unlock_irqrestore(&rq->lock, flags); 6126 spin_unlock_irqrestore(&rq->lock, flags);
6103 break; 6127 break;
@@ -6158,7 +6182,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6158 spin_lock_irqsave(&rq->lock, flags); 6182 spin_lock_irqsave(&rq->lock, flags);
6159 if (rq->rd) { 6183 if (rq->rd) {
6160 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6184 BUG_ON(!cpu_isset(cpu, rq->rd->span));
6161 cpu_clear(cpu, rq->rd->online); 6185 set_rq_offline(rq);
6162 } 6186 }
6163 spin_unlock_irqrestore(&rq->lock, flags); 6187 spin_unlock_irqrestore(&rq->lock, flags);
6164 break; 6188 break;
@@ -6192,6 +6216,28 @@ void __init migration_init(void)
6192 6216
6193#ifdef CONFIG_SCHED_DEBUG 6217#ifdef CONFIG_SCHED_DEBUG
6194 6218
6219static inline const char *sd_level_to_string(enum sched_domain_level lvl)
6220{
6221 switch (lvl) {
6222 case SD_LV_NONE:
6223 return "NONE";
6224 case SD_LV_SIBLING:
6225 return "SIBLING";
6226 case SD_LV_MC:
6227 return "MC";
6228 case SD_LV_CPU:
6229 return "CPU";
6230 case SD_LV_NODE:
6231 return "NODE";
6232 case SD_LV_ALLNODES:
6233 return "ALLNODES";
6234 case SD_LV_MAX:
6235 return "MAX";
6236
6237 }
6238 return "MAX";
6239}
6240
6195static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, 6241static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6196 cpumask_t *groupmask) 6242 cpumask_t *groupmask)
6197{ 6243{
@@ -6211,7 +6257,8 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6211 return -1; 6257 return -1;
6212 } 6258 }
6213 6259
6214 printk(KERN_CONT "span %s\n", str); 6260 printk(KERN_CONT "span %s level %s\n",
6261 str, sd_level_to_string(sd->level));
6215 6262
6216 if (!cpu_isset(cpu, sd->span)) { 6263 if (!cpu_isset(cpu, sd->span)) {
6217 printk(KERN_ERR "ERROR: domain->span does not contain " 6264 printk(KERN_ERR "ERROR: domain->span does not contain "
@@ -6295,9 +6342,9 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6295 } 6342 }
6296 kfree(groupmask); 6343 kfree(groupmask);
6297} 6344}
6298#else 6345#else /* !CONFIG_SCHED_DEBUG */
6299# define sched_domain_debug(sd, cpu) do { } while (0) 6346# define sched_domain_debug(sd, cpu) do { } while (0)
6300#endif 6347#endif /* CONFIG_SCHED_DEBUG */
6301 6348
6302static int sd_degenerate(struct sched_domain *sd) 6349static int sd_degenerate(struct sched_domain *sd)
6303{ 6350{
@@ -6357,20 +6404,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6357static void rq_attach_root(struct rq *rq, struct root_domain *rd) 6404static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6358{ 6405{
6359 unsigned long flags; 6406 unsigned long flags;
6360 const struct sched_class *class;
6361 6407
6362 spin_lock_irqsave(&rq->lock, flags); 6408 spin_lock_irqsave(&rq->lock, flags);
6363 6409
6364 if (rq->rd) { 6410 if (rq->rd) {
6365 struct root_domain *old_rd = rq->rd; 6411 struct root_domain *old_rd = rq->rd;
6366 6412
6367 for (class = sched_class_highest; class; class = class->next) { 6413 if (cpu_isset(rq->cpu, old_rd->online))
6368 if (class->leave_domain) 6414 set_rq_offline(rq);
6369 class->leave_domain(rq);
6370 }
6371 6415
6372 cpu_clear(rq->cpu, old_rd->span); 6416 cpu_clear(rq->cpu, old_rd->span);
6373 cpu_clear(rq->cpu, old_rd->online);
6374 6417
6375 if (atomic_dec_and_test(&old_rd->refcount)) 6418 if (atomic_dec_and_test(&old_rd->refcount))
6376 kfree(old_rd); 6419 kfree(old_rd);
@@ -6381,12 +6424,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6381 6424
6382 cpu_set(rq->cpu, rd->span); 6425 cpu_set(rq->cpu, rd->span);
6383 if (cpu_isset(rq->cpu, cpu_online_map)) 6426 if (cpu_isset(rq->cpu, cpu_online_map))
6384 cpu_set(rq->cpu, rd->online); 6427 set_rq_online(rq);
6385
6386 for (class = sched_class_highest; class; class = class->next) {
6387 if (class->join_domain)
6388 class->join_domain(rq);
6389 }
6390 6428
6391 spin_unlock_irqrestore(&rq->lock, flags); 6429 spin_unlock_irqrestore(&rq->lock, flags);
6392} 6430}
@@ -6397,6 +6435,8 @@ static void init_rootdomain(struct root_domain *rd)
6397 6435
6398 cpus_clear(rd->span); 6436 cpus_clear(rd->span);
6399 cpus_clear(rd->online); 6437 cpus_clear(rd->online);
6438
6439 cpupri_init(&rd->cpupri);
6400} 6440}
6401 6441
6402static void init_defrootdomain(void) 6442static void init_defrootdomain(void)
@@ -6591,7 +6631,7 @@ static void sched_domain_node_span(int node, cpumask_t *span)
6591 cpus_or(*span, *span, *nodemask); 6631 cpus_or(*span, *span, *nodemask);
6592 } 6632 }
6593} 6633}
6594#endif 6634#endif /* CONFIG_NUMA */
6595 6635
6596int sched_smt_power_savings = 0, sched_mc_power_savings = 0; 6636int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
6597 6637
@@ -6610,7 +6650,7 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
6610 *sg = &per_cpu(sched_group_cpus, cpu); 6650 *sg = &per_cpu(sched_group_cpus, cpu);
6611 return cpu; 6651 return cpu;
6612} 6652}
6613#endif 6653#endif /* CONFIG_SCHED_SMT */
6614 6654
6615/* 6655/*
6616 * multi-core sched-domains: 6656 * multi-core sched-domains:
@@ -6618,7 +6658,7 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
6618#ifdef CONFIG_SCHED_MC 6658#ifdef CONFIG_SCHED_MC
6619static DEFINE_PER_CPU(struct sched_domain, core_domains); 6659static DEFINE_PER_CPU(struct sched_domain, core_domains);
6620static DEFINE_PER_CPU(struct sched_group, sched_group_core); 6660static DEFINE_PER_CPU(struct sched_group, sched_group_core);
6621#endif 6661#endif /* CONFIG_SCHED_MC */
6622 6662
6623#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) 6663#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
6624static int 6664static int
@@ -6720,7 +6760,7 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
6720 sg = sg->next; 6760 sg = sg->next;
6721 } while (sg != group_head); 6761 } while (sg != group_head);
6722} 6762}
6723#endif 6763#endif /* CONFIG_NUMA */
6724 6764
6725#ifdef CONFIG_NUMA 6765#ifdef CONFIG_NUMA
6726/* Free memory allocated for various sched_group structures */ 6766/* Free memory allocated for various sched_group structures */
@@ -6757,11 +6797,11 @@ next_sg:
6757 sched_group_nodes_bycpu[cpu] = NULL; 6797 sched_group_nodes_bycpu[cpu] = NULL;
6758 } 6798 }
6759} 6799}
6760#else 6800#else /* !CONFIG_NUMA */
6761static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 6801static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
6762{ 6802{
6763} 6803}
6764#endif 6804#endif /* CONFIG_NUMA */
6765 6805
6766/* 6806/*
6767 * Initialize sched groups cpu_power. 6807 * Initialize sched groups cpu_power.
@@ -7238,6 +7278,18 @@ void __attribute__((weak)) arch_update_cpu_topology(void)
7238} 7278}
7239 7279
7240/* 7280/*
7281 * Free current domain masks.
7282 * Called after all cpus are attached to NULL domain.
7283 */
7284static void free_sched_domains(void)
7285{
7286 ndoms_cur = 0;
7287 if (doms_cur != &fallback_doms)
7288 kfree(doms_cur);
7289 doms_cur = &fallback_doms;
7290}
7291
7292/*
7241 * Set up scheduler domains and groups. Callers must hold the hotplug lock. 7293 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
7242 * For now this just excludes isolated cpus, but could be used to 7294 * For now this just excludes isolated cpus, but could be used to
7243 * exclude other special cases in the future. 7295 * exclude other special cases in the future.
@@ -7384,6 +7436,7 @@ int arch_reinit_sched_domains(void)
7384 get_online_cpus(); 7436 get_online_cpus();
7385 mutex_lock(&sched_domains_mutex); 7437 mutex_lock(&sched_domains_mutex);
7386 detach_destroy_domains(&cpu_online_map); 7438 detach_destroy_domains(&cpu_online_map);
7439 free_sched_domains();
7387 err = arch_init_sched_domains(&cpu_online_map); 7440 err = arch_init_sched_domains(&cpu_online_map);
7388 mutex_unlock(&sched_domains_mutex); 7441 mutex_unlock(&sched_domains_mutex);
7389 put_online_cpus(); 7442 put_online_cpus();
@@ -7452,7 +7505,7 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
7452#endif 7505#endif
7453 return err; 7506 return err;
7454} 7507}
7455#endif 7508#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
7456 7509
7457/* 7510/*
7458 * Force a reinitialization of the sched domains hierarchy. The domains 7511 * Force a reinitialization of the sched domains hierarchy. The domains
@@ -7463,20 +7516,28 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
7463static int update_sched_domains(struct notifier_block *nfb, 7516static int update_sched_domains(struct notifier_block *nfb,
7464 unsigned long action, void *hcpu) 7517 unsigned long action, void *hcpu)
7465{ 7518{
7519 int cpu = (int)(long)hcpu;
7520
7466 switch (action) { 7521 switch (action) {
7467 case CPU_UP_PREPARE:
7468 case CPU_UP_PREPARE_FROZEN:
7469 case CPU_DOWN_PREPARE: 7522 case CPU_DOWN_PREPARE:
7470 case CPU_DOWN_PREPARE_FROZEN: 7523 case CPU_DOWN_PREPARE_FROZEN:
7524 disable_runtime(cpu_rq(cpu));
7525 /* fall-through */
7526 case CPU_UP_PREPARE:
7527 case CPU_UP_PREPARE_FROZEN:
7471 detach_destroy_domains(&cpu_online_map); 7528 detach_destroy_domains(&cpu_online_map);
7529 free_sched_domains();
7472 return NOTIFY_OK; 7530 return NOTIFY_OK;
7473 7531
7474 case CPU_UP_CANCELED: 7532
7475 case CPU_UP_CANCELED_FROZEN:
7476 case CPU_DOWN_FAILED: 7533 case CPU_DOWN_FAILED:
7477 case CPU_DOWN_FAILED_FROZEN: 7534 case CPU_DOWN_FAILED_FROZEN:
7478 case CPU_ONLINE: 7535 case CPU_ONLINE:
7479 case CPU_ONLINE_FROZEN: 7536 case CPU_ONLINE_FROZEN:
7537 enable_runtime(cpu_rq(cpu));
7538 /* fall-through */
7539 case CPU_UP_CANCELED:
7540 case CPU_UP_CANCELED_FROZEN:
7480 case CPU_DEAD: 7541 case CPU_DEAD:
7481 case CPU_DEAD_FROZEN: 7542 case CPU_DEAD_FROZEN:
7482 /* 7543 /*
@@ -7487,8 +7548,16 @@ static int update_sched_domains(struct notifier_block *nfb,
7487 return NOTIFY_DONE; 7548 return NOTIFY_DONE;
7488 } 7549 }
7489 7550
7551#ifndef CONFIG_CPUSETS
7552 /*
7553 * Create default domain partitioning if cpusets are disabled.
7554 * Otherwise we let cpusets rebuild the domains based on the
7555 * current setup.
7556 */
7557
7490 /* The hotplug lock is already held by cpu_up/cpu_down */ 7558 /* The hotplug lock is already held by cpu_up/cpu_down */
7491 arch_init_sched_domains(&cpu_online_map); 7559 arch_init_sched_domains(&cpu_online_map);
7560#endif
7492 7561
7493 return NOTIFY_OK; 7562 return NOTIFY_OK;
7494} 7563}
@@ -7668,8 +7737,8 @@ void __init sched_init(void)
7668 7737
7669 root_task_group.cfs_rq = (struct cfs_rq **)ptr; 7738 root_task_group.cfs_rq = (struct cfs_rq **)ptr;
7670 ptr += nr_cpu_ids * sizeof(void **); 7739 ptr += nr_cpu_ids * sizeof(void **);
7671#endif 7740#endif /* CONFIG_USER_SCHED */
7672#endif 7741#endif /* CONFIG_FAIR_GROUP_SCHED */
7673#ifdef CONFIG_RT_GROUP_SCHED 7742#ifdef CONFIG_RT_GROUP_SCHED
7674 init_task_group.rt_se = (struct sched_rt_entity **)ptr; 7743 init_task_group.rt_se = (struct sched_rt_entity **)ptr;
7675 ptr += nr_cpu_ids * sizeof(void **); 7744 ptr += nr_cpu_ids * sizeof(void **);
@@ -7683,8 +7752,8 @@ void __init sched_init(void)
7683 7752
7684 root_task_group.rt_rq = (struct rt_rq **)ptr; 7753 root_task_group.rt_rq = (struct rt_rq **)ptr;
7685 ptr += nr_cpu_ids * sizeof(void **); 7754 ptr += nr_cpu_ids * sizeof(void **);
7686#endif 7755#endif /* CONFIG_USER_SCHED */
7687#endif 7756#endif /* CONFIG_RT_GROUP_SCHED */
7688 } 7757 }
7689 7758
7690#ifdef CONFIG_SMP 7759#ifdef CONFIG_SMP
@@ -7700,8 +7769,8 @@ void __init sched_init(void)
7700#ifdef CONFIG_USER_SCHED 7769#ifdef CONFIG_USER_SCHED
7701 init_rt_bandwidth(&root_task_group.rt_bandwidth, 7770 init_rt_bandwidth(&root_task_group.rt_bandwidth,
7702 global_rt_period(), RUNTIME_INF); 7771 global_rt_period(), RUNTIME_INF);
7703#endif 7772#endif /* CONFIG_USER_SCHED */
7704#endif 7773#endif /* CONFIG_RT_GROUP_SCHED */
7705 7774
7706#ifdef CONFIG_GROUP_SCHED 7775#ifdef CONFIG_GROUP_SCHED
7707 list_add(&init_task_group.list, &task_groups); 7776 list_add(&init_task_group.list, &task_groups);
@@ -7711,8 +7780,8 @@ void __init sched_init(void)
7711 INIT_LIST_HEAD(&root_task_group.children); 7780 INIT_LIST_HEAD(&root_task_group.children);
7712 init_task_group.parent = &root_task_group; 7781 init_task_group.parent = &root_task_group;
7713 list_add(&init_task_group.siblings, &root_task_group.children); 7782 list_add(&init_task_group.siblings, &root_task_group.children);
7714#endif 7783#endif /* CONFIG_USER_SCHED */
7715#endif 7784#endif /* CONFIG_GROUP_SCHED */
7716 7785
7717 for_each_possible_cpu(i) { 7786 for_each_possible_cpu(i) {
7718 struct rq *rq; 7787 struct rq *rq;
@@ -7792,6 +7861,7 @@ void __init sched_init(void)
7792 rq->next_balance = jiffies; 7861 rq->next_balance = jiffies;
7793 rq->push_cpu = 0; 7862 rq->push_cpu = 0;
7794 rq->cpu = i; 7863 rq->cpu = i;
7864 rq->online = 0;
7795 rq->migration_thread = NULL; 7865 rq->migration_thread = NULL;
7796 INIT_LIST_HEAD(&rq->migration_queue); 7866 INIT_LIST_HEAD(&rq->migration_queue);
7797 rq_attach_root(rq, &def_root_domain); 7867 rq_attach_root(rq, &def_root_domain);
@@ -8031,7 +8101,7 @@ static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
8031{ 8101{
8032 list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list); 8102 list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list);
8033} 8103}
8034#else 8104#else /* !CONFG_FAIR_GROUP_SCHED */
8035static inline void free_fair_sched_group(struct task_group *tg) 8105static inline void free_fair_sched_group(struct task_group *tg)
8036{ 8106{
8037} 8107}
@@ -8049,7 +8119,7 @@ static inline void register_fair_sched_group(struct task_group *tg, int cpu)
8049static inline void unregister_fair_sched_group(struct task_group *tg, int cpu) 8119static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
8050{ 8120{
8051} 8121}
8052#endif 8122#endif /* CONFIG_FAIR_GROUP_SCHED */
8053 8123
8054#ifdef CONFIG_RT_GROUP_SCHED 8124#ifdef CONFIG_RT_GROUP_SCHED
8055static void free_rt_sched_group(struct task_group *tg) 8125static void free_rt_sched_group(struct task_group *tg)
@@ -8120,7 +8190,7 @@ static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
8120{ 8190{
8121 list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list); 8191 list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list);
8122} 8192}
8123#else 8193#else /* !CONFIG_RT_GROUP_SCHED */
8124static inline void free_rt_sched_group(struct task_group *tg) 8194static inline void free_rt_sched_group(struct task_group *tg)
8125{ 8195{
8126} 8196}
@@ -8138,7 +8208,7 @@ static inline void register_rt_sched_group(struct task_group *tg, int cpu)
8138static inline void unregister_rt_sched_group(struct task_group *tg, int cpu) 8208static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
8139{ 8209{
8140} 8210}
8141#endif 8211#endif /* CONFIG_RT_GROUP_SCHED */
8142 8212
8143#ifdef CONFIG_GROUP_SCHED 8213#ifdef CONFIG_GROUP_SCHED
8144static void free_sched_group(struct task_group *tg) 8214static void free_sched_group(struct task_group *tg)
@@ -8249,7 +8319,7 @@ void sched_move_task(struct task_struct *tsk)
8249 8319
8250 task_rq_unlock(rq, &flags); 8320 task_rq_unlock(rq, &flags);
8251} 8321}
8252#endif 8322#endif /* CONFIG_GROUP_SCHED */
8253 8323
8254#ifdef CONFIG_FAIR_GROUP_SCHED 8324#ifdef CONFIG_FAIR_GROUP_SCHED
8255static void set_se_shares(struct sched_entity *se, unsigned long shares) 8325static void set_se_shares(struct sched_entity *se, unsigned long shares)
@@ -8499,7 +8569,7 @@ static int sched_rt_global_constraints(void)
8499 8569
8500 return ret; 8570 return ret;
8501} 8571}
8502#else 8572#else /* !CONFIG_RT_GROUP_SCHED */
8503static int sched_rt_global_constraints(void) 8573static int sched_rt_global_constraints(void)
8504{ 8574{
8505 unsigned long flags; 8575 unsigned long flags;
@@ -8517,7 +8587,7 @@ static int sched_rt_global_constraints(void)
8517 8587
8518 return 0; 8588 return 0;
8519} 8589}
8520#endif 8590#endif /* CONFIG_RT_GROUP_SCHED */
8521 8591
8522int sched_rt_handler(struct ctl_table *table, int write, 8592int sched_rt_handler(struct ctl_table *table, int write,
8523 struct file *filp, void __user *buffer, size_t *lenp, 8593 struct file *filp, void __user *buffer, size_t *lenp,
@@ -8625,7 +8695,7 @@ static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
8625 8695
8626 return (u64) tg->shares; 8696 return (u64) tg->shares;
8627} 8697}
8628#endif 8698#endif /* CONFIG_FAIR_GROUP_SCHED */
8629 8699
8630#ifdef CONFIG_RT_GROUP_SCHED 8700#ifdef CONFIG_RT_GROUP_SCHED
8631static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft, 8701static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
@@ -8649,7 +8719,7 @@ static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
8649{ 8719{
8650 return sched_group_rt_period(cgroup_tg(cgrp)); 8720 return sched_group_rt_period(cgroup_tg(cgrp));
8651} 8721}
8652#endif 8722#endif /* CONFIG_RT_GROUP_SCHED */
8653 8723
8654static struct cftype cpu_files[] = { 8724static struct cftype cpu_files[] = {
8655#ifdef CONFIG_FAIR_GROUP_SCHED 8725#ifdef CONFIG_FAIR_GROUP_SCHED