diff options
Diffstat (limited to 'kernel/sched.c')
-rw-r--r-- | kernel/sched.c | 258 |
1 files changed, 164 insertions, 94 deletions
diff --git a/kernel/sched.c b/kernel/sched.c index ce375cb551e9..5b307da827ef 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -74,6 +74,8 @@ | |||
74 | #include <asm/tlb.h> | 74 | #include <asm/tlb.h> |
75 | #include <asm/irq_regs.h> | 75 | #include <asm/irq_regs.h> |
76 | 76 | ||
77 | #include "sched_cpupri.h" | ||
78 | |||
77 | /* | 79 | /* |
78 | * Convert user-nice values [ -20 ... 0 ... 19 ] | 80 | * Convert user-nice values [ -20 ... 0 ... 19 ] |
79 | * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], | 81 | * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], |
@@ -289,15 +291,15 @@ struct task_group root_task_group; | |||
289 | static DEFINE_PER_CPU(struct sched_entity, init_sched_entity); | 291 | static DEFINE_PER_CPU(struct sched_entity, init_sched_entity); |
290 | /* Default task group's cfs_rq on each cpu */ | 292 | /* Default task group's cfs_rq on each cpu */ |
291 | static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp; | 293 | static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp; |
292 | #endif | 294 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
293 | 295 | ||
294 | #ifdef CONFIG_RT_GROUP_SCHED | 296 | #ifdef CONFIG_RT_GROUP_SCHED |
295 | static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity); | 297 | static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity); |
296 | static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp; | 298 | static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp; |
297 | #endif | 299 | #endif /* CONFIG_RT_GROUP_SCHED */ |
298 | #else | 300 | #else /* !CONFIG_FAIR_GROUP_SCHED */ |
299 | #define root_task_group init_task_group | 301 | #define root_task_group init_task_group |
300 | #endif | 302 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
301 | 303 | ||
302 | /* task_group_lock serializes add/remove of task groups and also changes to | 304 | /* task_group_lock serializes add/remove of task groups and also changes to |
303 | * a task group's cpu shares. | 305 | * a task group's cpu shares. |
@@ -307,9 +309,9 @@ static DEFINE_SPINLOCK(task_group_lock); | |||
307 | #ifdef CONFIG_FAIR_GROUP_SCHED | 309 | #ifdef CONFIG_FAIR_GROUP_SCHED |
308 | #ifdef CONFIG_USER_SCHED | 310 | #ifdef CONFIG_USER_SCHED |
309 | # define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD) | 311 | # define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD) |
310 | #else | 312 | #else /* !CONFIG_USER_SCHED */ |
311 | # define INIT_TASK_GROUP_LOAD NICE_0_LOAD | 313 | # define INIT_TASK_GROUP_LOAD NICE_0_LOAD |
312 | #endif | 314 | #endif /* CONFIG_USER_SCHED */ |
313 | 315 | ||
314 | /* | 316 | /* |
315 | * A weight of 0 or 1 can cause arithmetics problems. | 317 | * A weight of 0 or 1 can cause arithmetics problems. |
@@ -452,6 +454,9 @@ struct root_domain { | |||
452 | */ | 454 | */ |
453 | cpumask_t rto_mask; | 455 | cpumask_t rto_mask; |
454 | atomic_t rto_count; | 456 | atomic_t rto_count; |
457 | #ifdef CONFIG_SMP | ||
458 | struct cpupri cpupri; | ||
459 | #endif | ||
455 | }; | 460 | }; |
456 | 461 | ||
457 | /* | 462 | /* |
@@ -526,6 +531,7 @@ struct rq { | |||
526 | int push_cpu; | 531 | int push_cpu; |
527 | /* cpu of this runqueue: */ | 532 | /* cpu of this runqueue: */ |
528 | int cpu; | 533 | int cpu; |
534 | int online; | ||
529 | 535 | ||
530 | struct task_struct *migration_thread; | 536 | struct task_struct *migration_thread; |
531 | struct list_head migration_queue; | 537 | struct list_head migration_queue; |
@@ -1313,15 +1319,15 @@ void wake_up_idle_cpu(int cpu) | |||
1313 | if (!tsk_is_polling(rq->idle)) | 1319 | if (!tsk_is_polling(rq->idle)) |
1314 | smp_send_reschedule(cpu); | 1320 | smp_send_reschedule(cpu); |
1315 | } | 1321 | } |
1316 | #endif | 1322 | #endif /* CONFIG_NO_HZ */ |
1317 | 1323 | ||
1318 | #else | 1324 | #else /* !CONFIG_SMP */ |
1319 | static void __resched_task(struct task_struct *p, int tif_bit) | 1325 | static void __resched_task(struct task_struct *p, int tif_bit) |
1320 | { | 1326 | { |
1321 | assert_spin_locked(&task_rq(p)->lock); | 1327 | assert_spin_locked(&task_rq(p)->lock); |
1322 | set_tsk_thread_flag(p, tif_bit); | 1328 | set_tsk_thread_flag(p, tif_bit); |
1323 | } | 1329 | } |
1324 | #endif | 1330 | #endif /* CONFIG_SMP */ |
1325 | 1331 | ||
1326 | #if BITS_PER_LONG == 32 | 1332 | #if BITS_PER_LONG == 32 |
1327 | # define WMULT_CONST (~0UL) | 1333 | # define WMULT_CONST (~0UL) |
@@ -1481,16 +1487,8 @@ static unsigned long source_load(int cpu, int type); | |||
1481 | static unsigned long target_load(int cpu, int type); | 1487 | static unsigned long target_load(int cpu, int type); |
1482 | static unsigned long cpu_avg_load_per_task(int cpu); | 1488 | static unsigned long cpu_avg_load_per_task(int cpu); |
1483 | static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd); | 1489 | static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd); |
1484 | #else /* CONFIG_SMP */ | ||
1485 | |||
1486 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
1487 | static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares) | ||
1488 | { | ||
1489 | } | ||
1490 | #endif | 1490 | #endif |
1491 | 1491 | ||
1492 | #endif /* CONFIG_SMP */ | ||
1493 | |||
1494 | #include "sched_stats.h" | 1492 | #include "sched_stats.h" |
1495 | #include "sched_idletask.c" | 1493 | #include "sched_idletask.c" |
1496 | #include "sched_fair.c" | 1494 | #include "sched_fair.c" |
@@ -1500,6 +1498,8 @@ static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares) | |||
1500 | #endif | 1498 | #endif |
1501 | 1499 | ||
1502 | #define sched_class_highest (&rt_sched_class) | 1500 | #define sched_class_highest (&rt_sched_class) |
1501 | #define for_each_class(class) \ | ||
1502 | for (class = sched_class_highest; class; class = class->next) | ||
1503 | 1503 | ||
1504 | static inline void inc_load(struct rq *rq, const struct task_struct *p) | 1504 | static inline void inc_load(struct rq *rq, const struct task_struct *p) |
1505 | { | 1505 | { |
@@ -1636,12 +1636,6 @@ inline int task_curr(const struct task_struct *p) | |||
1636 | return cpu_curr(task_cpu(p)) == p; | 1636 | return cpu_curr(task_cpu(p)) == p; |
1637 | } | 1637 | } |
1638 | 1638 | ||
1639 | /* Used instead of source_load when we know the type == 0 */ | ||
1640 | unsigned long weighted_cpuload(const int cpu) | ||
1641 | { | ||
1642 | return cpu_rq(cpu)->load.weight; | ||
1643 | } | ||
1644 | |||
1645 | static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) | 1639 | static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) |
1646 | { | 1640 | { |
1647 | set_task_rq(p, cpu); | 1641 | set_task_rq(p, cpu); |
@@ -1670,6 +1664,12 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p, | |||
1670 | 1664 | ||
1671 | #ifdef CONFIG_SMP | 1665 | #ifdef CONFIG_SMP |
1672 | 1666 | ||
1667 | /* Used instead of source_load when we know the type == 0 */ | ||
1668 | static unsigned long weighted_cpuload(const int cpu) | ||
1669 | { | ||
1670 | return cpu_rq(cpu)->load.weight; | ||
1671 | } | ||
1672 | |||
1673 | /* | 1673 | /* |
1674 | * Is this task likely cache-hot: | 1674 | * Is this task likely cache-hot: |
1675 | */ | 1675 | */ |
@@ -2131,7 +2131,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync) | |||
2131 | } | 2131 | } |
2132 | } | 2132 | } |
2133 | } | 2133 | } |
2134 | #endif | 2134 | #endif /* CONFIG_SCHEDSTATS */ |
2135 | 2135 | ||
2136 | out_activate: | 2136 | out_activate: |
2137 | #endif /* CONFIG_SMP */ | 2137 | #endif /* CONFIG_SMP */ |
@@ -2331,7 +2331,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr, | |||
2331 | notifier->ops->sched_out(notifier, next); | 2331 | notifier->ops->sched_out(notifier, next); |
2332 | } | 2332 | } |
2333 | 2333 | ||
2334 | #else | 2334 | #else /* !CONFIG_PREEMPT_NOTIFIERS */ |
2335 | 2335 | ||
2336 | static void fire_sched_in_preempt_notifiers(struct task_struct *curr) | 2336 | static void fire_sched_in_preempt_notifiers(struct task_struct *curr) |
2337 | { | 2337 | { |
@@ -2343,7 +2343,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr, | |||
2343 | { | 2343 | { |
2344 | } | 2344 | } |
2345 | 2345 | ||
2346 | #endif | 2346 | #endif /* CONFIG_PREEMPT_NOTIFIERS */ |
2347 | 2347 | ||
2348 | /** | 2348 | /** |
2349 | * prepare_task_switch - prepare to switch tasks | 2349 | * prepare_task_switch - prepare to switch tasks |
@@ -3672,6 +3672,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle) | |||
3672 | /* Earliest time when we have to do rebalance again */ | 3672 | /* Earliest time when we have to do rebalance again */ |
3673 | unsigned long next_balance = jiffies + 60*HZ; | 3673 | unsigned long next_balance = jiffies + 60*HZ; |
3674 | int update_next_balance = 0; | 3674 | int update_next_balance = 0; |
3675 | int need_serialize; | ||
3675 | cpumask_t tmp; | 3676 | cpumask_t tmp; |
3676 | 3677 | ||
3677 | for_each_domain(cpu, sd) { | 3678 | for_each_domain(cpu, sd) { |
@@ -3689,8 +3690,9 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle) | |||
3689 | if (interval > HZ*NR_CPUS/10) | 3690 | if (interval > HZ*NR_CPUS/10) |
3690 | interval = HZ*NR_CPUS/10; | 3691 | interval = HZ*NR_CPUS/10; |
3691 | 3692 | ||
3693 | need_serialize = sd->flags & SD_SERIALIZE; | ||
3692 | 3694 | ||
3693 | if (sd->flags & SD_SERIALIZE) { | 3695 | if (need_serialize) { |
3694 | if (!spin_trylock(&balancing)) | 3696 | if (!spin_trylock(&balancing)) |
3695 | goto out; | 3697 | goto out; |
3696 | } | 3698 | } |
@@ -3706,7 +3708,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle) | |||
3706 | } | 3708 | } |
3707 | sd->last_balance = jiffies; | 3709 | sd->last_balance = jiffies; |
3708 | } | 3710 | } |
3709 | if (sd->flags & SD_SERIALIZE) | 3711 | if (need_serialize) |
3710 | spin_unlock(&balancing); | 3712 | spin_unlock(&balancing); |
3711 | out: | 3713 | out: |
3712 | if (time_after(next_balance, sd->last_balance + interval)) { | 3714 | if (time_after(next_balance, sd->last_balance + interval)) { |
@@ -4070,6 +4072,7 @@ static noinline void __schedule_bug(struct task_struct *prev) | |||
4070 | prev->comm, prev->pid, preempt_count()); | 4072 | prev->comm, prev->pid, preempt_count()); |
4071 | 4073 | ||
4072 | debug_show_held_locks(prev); | 4074 | debug_show_held_locks(prev); |
4075 | print_modules(); | ||
4073 | if (irqs_disabled()) | 4076 | if (irqs_disabled()) |
4074 | print_irqtrace_events(prev); | 4077 | print_irqtrace_events(prev); |
4075 | 4078 | ||
@@ -4143,7 +4146,7 @@ asmlinkage void __sched schedule(void) | |||
4143 | struct task_struct *prev, *next; | 4146 | struct task_struct *prev, *next; |
4144 | unsigned long *switch_count; | 4147 | unsigned long *switch_count; |
4145 | struct rq *rq; | 4148 | struct rq *rq; |
4146 | int cpu; | 4149 | int cpu, hrtick = sched_feat(HRTICK); |
4147 | 4150 | ||
4148 | need_resched: | 4151 | need_resched: |
4149 | preempt_disable(); | 4152 | preempt_disable(); |
@@ -4158,7 +4161,8 @@ need_resched_nonpreemptible: | |||
4158 | 4161 | ||
4159 | schedule_debug(prev); | 4162 | schedule_debug(prev); |
4160 | 4163 | ||
4161 | hrtick_clear(rq); | 4164 | if (hrtick) |
4165 | hrtick_clear(rq); | ||
4162 | 4166 | ||
4163 | /* | 4167 | /* |
4164 | * Do the rq-clock update outside the rq lock: | 4168 | * Do the rq-clock update outside the rq lock: |
@@ -4204,7 +4208,8 @@ need_resched_nonpreemptible: | |||
4204 | } else | 4208 | } else |
4205 | spin_unlock_irq(&rq->lock); | 4209 | spin_unlock_irq(&rq->lock); |
4206 | 4210 | ||
4207 | hrtick_set(rq); | 4211 | if (hrtick) |
4212 | hrtick_set(rq); | ||
4208 | 4213 | ||
4209 | if (unlikely(reacquire_kernel_lock(current) < 0)) | 4214 | if (unlikely(reacquire_kernel_lock(current) < 0)) |
4210 | goto need_resched_nonpreemptible; | 4215 | goto need_resched_nonpreemptible; |
@@ -5072,24 +5077,6 @@ asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, | |||
5072 | return sched_setaffinity(pid, &new_mask); | 5077 | return sched_setaffinity(pid, &new_mask); |
5073 | } | 5078 | } |
5074 | 5079 | ||
5075 | /* | ||
5076 | * Represents all cpu's present in the system | ||
5077 | * In systems capable of hotplug, this map could dynamically grow | ||
5078 | * as new cpu's are detected in the system via any platform specific | ||
5079 | * method, such as ACPI for e.g. | ||
5080 | */ | ||
5081 | |||
5082 | cpumask_t cpu_present_map __read_mostly; | ||
5083 | EXPORT_SYMBOL(cpu_present_map); | ||
5084 | |||
5085 | #ifndef CONFIG_SMP | ||
5086 | cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL; | ||
5087 | EXPORT_SYMBOL(cpu_online_map); | ||
5088 | |||
5089 | cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL; | ||
5090 | EXPORT_SYMBOL(cpu_possible_map); | ||
5091 | #endif | ||
5092 | |||
5093 | long sched_getaffinity(pid_t pid, cpumask_t *mask) | 5080 | long sched_getaffinity(pid_t pid, cpumask_t *mask) |
5094 | { | 5081 | { |
5095 | struct task_struct *p; | 5082 | struct task_struct *p; |
@@ -5573,6 +5560,12 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask) | |||
5573 | goto out; | 5560 | goto out; |
5574 | } | 5561 | } |
5575 | 5562 | ||
5563 | if (unlikely((p->flags & PF_THREAD_BOUND) && p != current && | ||
5564 | !cpus_equal(p->cpus_allowed, *new_mask))) { | ||
5565 | ret = -EINVAL; | ||
5566 | goto out; | ||
5567 | } | ||
5568 | |||
5576 | if (p->sched_class->set_cpus_allowed) | 5569 | if (p->sched_class->set_cpus_allowed) |
5577 | p->sched_class->set_cpus_allowed(p, new_mask); | 5570 | p->sched_class->set_cpus_allowed(p, new_mask); |
5578 | else { | 5571 | else { |
@@ -6060,6 +6053,36 @@ static void unregister_sched_domain_sysctl(void) | |||
6060 | } | 6053 | } |
6061 | #endif | 6054 | #endif |
6062 | 6055 | ||
6056 | static void set_rq_online(struct rq *rq) | ||
6057 | { | ||
6058 | if (!rq->online) { | ||
6059 | const struct sched_class *class; | ||
6060 | |||
6061 | cpu_set(rq->cpu, rq->rd->online); | ||
6062 | rq->online = 1; | ||
6063 | |||
6064 | for_each_class(class) { | ||
6065 | if (class->rq_online) | ||
6066 | class->rq_online(rq); | ||
6067 | } | ||
6068 | } | ||
6069 | } | ||
6070 | |||
6071 | static void set_rq_offline(struct rq *rq) | ||
6072 | { | ||
6073 | if (rq->online) { | ||
6074 | const struct sched_class *class; | ||
6075 | |||
6076 | for_each_class(class) { | ||
6077 | if (class->rq_offline) | ||
6078 | class->rq_offline(rq); | ||
6079 | } | ||
6080 | |||
6081 | cpu_clear(rq->cpu, rq->rd->online); | ||
6082 | rq->online = 0; | ||
6083 | } | ||
6084 | } | ||
6085 | |||
6063 | /* | 6086 | /* |
6064 | * migration_call - callback that gets triggered when a CPU is added. | 6087 | * migration_call - callback that gets triggered when a CPU is added. |
6065 | * Here we can start up the necessary migration thread for the new CPU. | 6088 | * Here we can start up the necessary migration thread for the new CPU. |
@@ -6097,7 +6120,8 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) | |||
6097 | spin_lock_irqsave(&rq->lock, flags); | 6120 | spin_lock_irqsave(&rq->lock, flags); |
6098 | if (rq->rd) { | 6121 | if (rq->rd) { |
6099 | BUG_ON(!cpu_isset(cpu, rq->rd->span)); | 6122 | BUG_ON(!cpu_isset(cpu, rq->rd->span)); |
6100 | cpu_set(cpu, rq->rd->online); | 6123 | |
6124 | set_rq_online(rq); | ||
6101 | } | 6125 | } |
6102 | spin_unlock_irqrestore(&rq->lock, flags); | 6126 | spin_unlock_irqrestore(&rq->lock, flags); |
6103 | break; | 6127 | break; |
@@ -6158,7 +6182,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) | |||
6158 | spin_lock_irqsave(&rq->lock, flags); | 6182 | spin_lock_irqsave(&rq->lock, flags); |
6159 | if (rq->rd) { | 6183 | if (rq->rd) { |
6160 | BUG_ON(!cpu_isset(cpu, rq->rd->span)); | 6184 | BUG_ON(!cpu_isset(cpu, rq->rd->span)); |
6161 | cpu_clear(cpu, rq->rd->online); | 6185 | set_rq_offline(rq); |
6162 | } | 6186 | } |
6163 | spin_unlock_irqrestore(&rq->lock, flags); | 6187 | spin_unlock_irqrestore(&rq->lock, flags); |
6164 | break; | 6188 | break; |
@@ -6192,6 +6216,28 @@ void __init migration_init(void) | |||
6192 | 6216 | ||
6193 | #ifdef CONFIG_SCHED_DEBUG | 6217 | #ifdef CONFIG_SCHED_DEBUG |
6194 | 6218 | ||
6219 | static inline const char *sd_level_to_string(enum sched_domain_level lvl) | ||
6220 | { | ||
6221 | switch (lvl) { | ||
6222 | case SD_LV_NONE: | ||
6223 | return "NONE"; | ||
6224 | case SD_LV_SIBLING: | ||
6225 | return "SIBLING"; | ||
6226 | case SD_LV_MC: | ||
6227 | return "MC"; | ||
6228 | case SD_LV_CPU: | ||
6229 | return "CPU"; | ||
6230 | case SD_LV_NODE: | ||
6231 | return "NODE"; | ||
6232 | case SD_LV_ALLNODES: | ||
6233 | return "ALLNODES"; | ||
6234 | case SD_LV_MAX: | ||
6235 | return "MAX"; | ||
6236 | |||
6237 | } | ||
6238 | return "MAX"; | ||
6239 | } | ||
6240 | |||
6195 | static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, | 6241 | static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, |
6196 | cpumask_t *groupmask) | 6242 | cpumask_t *groupmask) |
6197 | { | 6243 | { |
@@ -6211,7 +6257,8 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, | |||
6211 | return -1; | 6257 | return -1; |
6212 | } | 6258 | } |
6213 | 6259 | ||
6214 | printk(KERN_CONT "span %s\n", str); | 6260 | printk(KERN_CONT "span %s level %s\n", |
6261 | str, sd_level_to_string(sd->level)); | ||
6215 | 6262 | ||
6216 | if (!cpu_isset(cpu, sd->span)) { | 6263 | if (!cpu_isset(cpu, sd->span)) { |
6217 | printk(KERN_ERR "ERROR: domain->span does not contain " | 6264 | printk(KERN_ERR "ERROR: domain->span does not contain " |
@@ -6295,9 +6342,9 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu) | |||
6295 | } | 6342 | } |
6296 | kfree(groupmask); | 6343 | kfree(groupmask); |
6297 | } | 6344 | } |
6298 | #else | 6345 | #else /* !CONFIG_SCHED_DEBUG */ |
6299 | # define sched_domain_debug(sd, cpu) do { } while (0) | 6346 | # define sched_domain_debug(sd, cpu) do { } while (0) |
6300 | #endif | 6347 | #endif /* CONFIG_SCHED_DEBUG */ |
6301 | 6348 | ||
6302 | static int sd_degenerate(struct sched_domain *sd) | 6349 | static int sd_degenerate(struct sched_domain *sd) |
6303 | { | 6350 | { |
@@ -6357,20 +6404,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) | |||
6357 | static void rq_attach_root(struct rq *rq, struct root_domain *rd) | 6404 | static void rq_attach_root(struct rq *rq, struct root_domain *rd) |
6358 | { | 6405 | { |
6359 | unsigned long flags; | 6406 | unsigned long flags; |
6360 | const struct sched_class *class; | ||
6361 | 6407 | ||
6362 | spin_lock_irqsave(&rq->lock, flags); | 6408 | spin_lock_irqsave(&rq->lock, flags); |
6363 | 6409 | ||
6364 | if (rq->rd) { | 6410 | if (rq->rd) { |
6365 | struct root_domain *old_rd = rq->rd; | 6411 | struct root_domain *old_rd = rq->rd; |
6366 | 6412 | ||
6367 | for (class = sched_class_highest; class; class = class->next) { | 6413 | if (cpu_isset(rq->cpu, old_rd->online)) |
6368 | if (class->leave_domain) | 6414 | set_rq_offline(rq); |
6369 | class->leave_domain(rq); | ||
6370 | } | ||
6371 | 6415 | ||
6372 | cpu_clear(rq->cpu, old_rd->span); | 6416 | cpu_clear(rq->cpu, old_rd->span); |
6373 | cpu_clear(rq->cpu, old_rd->online); | ||
6374 | 6417 | ||
6375 | if (atomic_dec_and_test(&old_rd->refcount)) | 6418 | if (atomic_dec_and_test(&old_rd->refcount)) |
6376 | kfree(old_rd); | 6419 | kfree(old_rd); |
@@ -6381,12 +6424,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) | |||
6381 | 6424 | ||
6382 | cpu_set(rq->cpu, rd->span); | 6425 | cpu_set(rq->cpu, rd->span); |
6383 | if (cpu_isset(rq->cpu, cpu_online_map)) | 6426 | if (cpu_isset(rq->cpu, cpu_online_map)) |
6384 | cpu_set(rq->cpu, rd->online); | 6427 | set_rq_online(rq); |
6385 | |||
6386 | for (class = sched_class_highest; class; class = class->next) { | ||
6387 | if (class->join_domain) | ||
6388 | class->join_domain(rq); | ||
6389 | } | ||
6390 | 6428 | ||
6391 | spin_unlock_irqrestore(&rq->lock, flags); | 6429 | spin_unlock_irqrestore(&rq->lock, flags); |
6392 | } | 6430 | } |
@@ -6397,6 +6435,8 @@ static void init_rootdomain(struct root_domain *rd) | |||
6397 | 6435 | ||
6398 | cpus_clear(rd->span); | 6436 | cpus_clear(rd->span); |
6399 | cpus_clear(rd->online); | 6437 | cpus_clear(rd->online); |
6438 | |||
6439 | cpupri_init(&rd->cpupri); | ||
6400 | } | 6440 | } |
6401 | 6441 | ||
6402 | static void init_defrootdomain(void) | 6442 | static void init_defrootdomain(void) |
@@ -6591,7 +6631,7 @@ static void sched_domain_node_span(int node, cpumask_t *span) | |||
6591 | cpus_or(*span, *span, *nodemask); | 6631 | cpus_or(*span, *span, *nodemask); |
6592 | } | 6632 | } |
6593 | } | 6633 | } |
6594 | #endif | 6634 | #endif /* CONFIG_NUMA */ |
6595 | 6635 | ||
6596 | int sched_smt_power_savings = 0, sched_mc_power_savings = 0; | 6636 | int sched_smt_power_savings = 0, sched_mc_power_savings = 0; |
6597 | 6637 | ||
@@ -6610,7 +6650,7 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, | |||
6610 | *sg = &per_cpu(sched_group_cpus, cpu); | 6650 | *sg = &per_cpu(sched_group_cpus, cpu); |
6611 | return cpu; | 6651 | return cpu; |
6612 | } | 6652 | } |
6613 | #endif | 6653 | #endif /* CONFIG_SCHED_SMT */ |
6614 | 6654 | ||
6615 | /* | 6655 | /* |
6616 | * multi-core sched-domains: | 6656 | * multi-core sched-domains: |
@@ -6618,7 +6658,7 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, | |||
6618 | #ifdef CONFIG_SCHED_MC | 6658 | #ifdef CONFIG_SCHED_MC |
6619 | static DEFINE_PER_CPU(struct sched_domain, core_domains); | 6659 | static DEFINE_PER_CPU(struct sched_domain, core_domains); |
6620 | static DEFINE_PER_CPU(struct sched_group, sched_group_core); | 6660 | static DEFINE_PER_CPU(struct sched_group, sched_group_core); |
6621 | #endif | 6661 | #endif /* CONFIG_SCHED_MC */ |
6622 | 6662 | ||
6623 | #if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) | 6663 | #if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) |
6624 | static int | 6664 | static int |
@@ -6720,7 +6760,7 @@ static void init_numa_sched_groups_power(struct sched_group *group_head) | |||
6720 | sg = sg->next; | 6760 | sg = sg->next; |
6721 | } while (sg != group_head); | 6761 | } while (sg != group_head); |
6722 | } | 6762 | } |
6723 | #endif | 6763 | #endif /* CONFIG_NUMA */ |
6724 | 6764 | ||
6725 | #ifdef CONFIG_NUMA | 6765 | #ifdef CONFIG_NUMA |
6726 | /* Free memory allocated for various sched_group structures */ | 6766 | /* Free memory allocated for various sched_group structures */ |
@@ -6757,11 +6797,11 @@ next_sg: | |||
6757 | sched_group_nodes_bycpu[cpu] = NULL; | 6797 | sched_group_nodes_bycpu[cpu] = NULL; |
6758 | } | 6798 | } |
6759 | } | 6799 | } |
6760 | #else | 6800 | #else /* !CONFIG_NUMA */ |
6761 | static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) | 6801 | static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) |
6762 | { | 6802 | { |
6763 | } | 6803 | } |
6764 | #endif | 6804 | #endif /* CONFIG_NUMA */ |
6765 | 6805 | ||
6766 | /* | 6806 | /* |
6767 | * Initialize sched groups cpu_power. | 6807 | * Initialize sched groups cpu_power. |
@@ -7238,6 +7278,18 @@ void __attribute__((weak)) arch_update_cpu_topology(void) | |||
7238 | } | 7278 | } |
7239 | 7279 | ||
7240 | /* | 7280 | /* |
7281 | * Free current domain masks. | ||
7282 | * Called after all cpus are attached to NULL domain. | ||
7283 | */ | ||
7284 | static void free_sched_domains(void) | ||
7285 | { | ||
7286 | ndoms_cur = 0; | ||
7287 | if (doms_cur != &fallback_doms) | ||
7288 | kfree(doms_cur); | ||
7289 | doms_cur = &fallback_doms; | ||
7290 | } | ||
7291 | |||
7292 | /* | ||
7241 | * Set up scheduler domains and groups. Callers must hold the hotplug lock. | 7293 | * Set up scheduler domains and groups. Callers must hold the hotplug lock. |
7242 | * For now this just excludes isolated cpus, but could be used to | 7294 | * For now this just excludes isolated cpus, but could be used to |
7243 | * exclude other special cases in the future. | 7295 | * exclude other special cases in the future. |
@@ -7384,6 +7436,7 @@ int arch_reinit_sched_domains(void) | |||
7384 | get_online_cpus(); | 7436 | get_online_cpus(); |
7385 | mutex_lock(&sched_domains_mutex); | 7437 | mutex_lock(&sched_domains_mutex); |
7386 | detach_destroy_domains(&cpu_online_map); | 7438 | detach_destroy_domains(&cpu_online_map); |
7439 | free_sched_domains(); | ||
7387 | err = arch_init_sched_domains(&cpu_online_map); | 7440 | err = arch_init_sched_domains(&cpu_online_map); |
7388 | mutex_unlock(&sched_domains_mutex); | 7441 | mutex_unlock(&sched_domains_mutex); |
7389 | put_online_cpus(); | 7442 | put_online_cpus(); |
@@ -7452,7 +7505,7 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls) | |||
7452 | #endif | 7505 | #endif |
7453 | return err; | 7506 | return err; |
7454 | } | 7507 | } |
7455 | #endif | 7508 | #endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */ |
7456 | 7509 | ||
7457 | /* | 7510 | /* |
7458 | * Force a reinitialization of the sched domains hierarchy. The domains | 7511 | * Force a reinitialization of the sched domains hierarchy. The domains |
@@ -7463,20 +7516,28 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls) | |||
7463 | static int update_sched_domains(struct notifier_block *nfb, | 7516 | static int update_sched_domains(struct notifier_block *nfb, |
7464 | unsigned long action, void *hcpu) | 7517 | unsigned long action, void *hcpu) |
7465 | { | 7518 | { |
7519 | int cpu = (int)(long)hcpu; | ||
7520 | |||
7466 | switch (action) { | 7521 | switch (action) { |
7467 | case CPU_UP_PREPARE: | ||
7468 | case CPU_UP_PREPARE_FROZEN: | ||
7469 | case CPU_DOWN_PREPARE: | 7522 | case CPU_DOWN_PREPARE: |
7470 | case CPU_DOWN_PREPARE_FROZEN: | 7523 | case CPU_DOWN_PREPARE_FROZEN: |
7524 | disable_runtime(cpu_rq(cpu)); | ||
7525 | /* fall-through */ | ||
7526 | case CPU_UP_PREPARE: | ||
7527 | case CPU_UP_PREPARE_FROZEN: | ||
7471 | detach_destroy_domains(&cpu_online_map); | 7528 | detach_destroy_domains(&cpu_online_map); |
7529 | free_sched_domains(); | ||
7472 | return NOTIFY_OK; | 7530 | return NOTIFY_OK; |
7473 | 7531 | ||
7474 | case CPU_UP_CANCELED: | 7532 | |
7475 | case CPU_UP_CANCELED_FROZEN: | ||
7476 | case CPU_DOWN_FAILED: | 7533 | case CPU_DOWN_FAILED: |
7477 | case CPU_DOWN_FAILED_FROZEN: | 7534 | case CPU_DOWN_FAILED_FROZEN: |
7478 | case CPU_ONLINE: | 7535 | case CPU_ONLINE: |
7479 | case CPU_ONLINE_FROZEN: | 7536 | case CPU_ONLINE_FROZEN: |
7537 | enable_runtime(cpu_rq(cpu)); | ||
7538 | /* fall-through */ | ||
7539 | case CPU_UP_CANCELED: | ||
7540 | case CPU_UP_CANCELED_FROZEN: | ||
7480 | case CPU_DEAD: | 7541 | case CPU_DEAD: |
7481 | case CPU_DEAD_FROZEN: | 7542 | case CPU_DEAD_FROZEN: |
7482 | /* | 7543 | /* |
@@ -7487,8 +7548,16 @@ static int update_sched_domains(struct notifier_block *nfb, | |||
7487 | return NOTIFY_DONE; | 7548 | return NOTIFY_DONE; |
7488 | } | 7549 | } |
7489 | 7550 | ||
7551 | #ifndef CONFIG_CPUSETS | ||
7552 | /* | ||
7553 | * Create default domain partitioning if cpusets are disabled. | ||
7554 | * Otherwise we let cpusets rebuild the domains based on the | ||
7555 | * current setup. | ||
7556 | */ | ||
7557 | |||
7490 | /* The hotplug lock is already held by cpu_up/cpu_down */ | 7558 | /* The hotplug lock is already held by cpu_up/cpu_down */ |
7491 | arch_init_sched_domains(&cpu_online_map); | 7559 | arch_init_sched_domains(&cpu_online_map); |
7560 | #endif | ||
7492 | 7561 | ||
7493 | return NOTIFY_OK; | 7562 | return NOTIFY_OK; |
7494 | } | 7563 | } |
@@ -7668,8 +7737,8 @@ void __init sched_init(void) | |||
7668 | 7737 | ||
7669 | root_task_group.cfs_rq = (struct cfs_rq **)ptr; | 7738 | root_task_group.cfs_rq = (struct cfs_rq **)ptr; |
7670 | ptr += nr_cpu_ids * sizeof(void **); | 7739 | ptr += nr_cpu_ids * sizeof(void **); |
7671 | #endif | 7740 | #endif /* CONFIG_USER_SCHED */ |
7672 | #endif | 7741 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
7673 | #ifdef CONFIG_RT_GROUP_SCHED | 7742 | #ifdef CONFIG_RT_GROUP_SCHED |
7674 | init_task_group.rt_se = (struct sched_rt_entity **)ptr; | 7743 | init_task_group.rt_se = (struct sched_rt_entity **)ptr; |
7675 | ptr += nr_cpu_ids * sizeof(void **); | 7744 | ptr += nr_cpu_ids * sizeof(void **); |
@@ -7683,8 +7752,8 @@ void __init sched_init(void) | |||
7683 | 7752 | ||
7684 | root_task_group.rt_rq = (struct rt_rq **)ptr; | 7753 | root_task_group.rt_rq = (struct rt_rq **)ptr; |
7685 | ptr += nr_cpu_ids * sizeof(void **); | 7754 | ptr += nr_cpu_ids * sizeof(void **); |
7686 | #endif | 7755 | #endif /* CONFIG_USER_SCHED */ |
7687 | #endif | 7756 | #endif /* CONFIG_RT_GROUP_SCHED */ |
7688 | } | 7757 | } |
7689 | 7758 | ||
7690 | #ifdef CONFIG_SMP | 7759 | #ifdef CONFIG_SMP |
@@ -7700,8 +7769,8 @@ void __init sched_init(void) | |||
7700 | #ifdef CONFIG_USER_SCHED | 7769 | #ifdef CONFIG_USER_SCHED |
7701 | init_rt_bandwidth(&root_task_group.rt_bandwidth, | 7770 | init_rt_bandwidth(&root_task_group.rt_bandwidth, |
7702 | global_rt_period(), RUNTIME_INF); | 7771 | global_rt_period(), RUNTIME_INF); |
7703 | #endif | 7772 | #endif /* CONFIG_USER_SCHED */ |
7704 | #endif | 7773 | #endif /* CONFIG_RT_GROUP_SCHED */ |
7705 | 7774 | ||
7706 | #ifdef CONFIG_GROUP_SCHED | 7775 | #ifdef CONFIG_GROUP_SCHED |
7707 | list_add(&init_task_group.list, &task_groups); | 7776 | list_add(&init_task_group.list, &task_groups); |
@@ -7711,8 +7780,8 @@ void __init sched_init(void) | |||
7711 | INIT_LIST_HEAD(&root_task_group.children); | 7780 | INIT_LIST_HEAD(&root_task_group.children); |
7712 | init_task_group.parent = &root_task_group; | 7781 | init_task_group.parent = &root_task_group; |
7713 | list_add(&init_task_group.siblings, &root_task_group.children); | 7782 | list_add(&init_task_group.siblings, &root_task_group.children); |
7714 | #endif | 7783 | #endif /* CONFIG_USER_SCHED */ |
7715 | #endif | 7784 | #endif /* CONFIG_GROUP_SCHED */ |
7716 | 7785 | ||
7717 | for_each_possible_cpu(i) { | 7786 | for_each_possible_cpu(i) { |
7718 | struct rq *rq; | 7787 | struct rq *rq; |
@@ -7792,6 +7861,7 @@ void __init sched_init(void) | |||
7792 | rq->next_balance = jiffies; | 7861 | rq->next_balance = jiffies; |
7793 | rq->push_cpu = 0; | 7862 | rq->push_cpu = 0; |
7794 | rq->cpu = i; | 7863 | rq->cpu = i; |
7864 | rq->online = 0; | ||
7795 | rq->migration_thread = NULL; | 7865 | rq->migration_thread = NULL; |
7796 | INIT_LIST_HEAD(&rq->migration_queue); | 7866 | INIT_LIST_HEAD(&rq->migration_queue); |
7797 | rq_attach_root(rq, &def_root_domain); | 7867 | rq_attach_root(rq, &def_root_domain); |
@@ -8031,7 +8101,7 @@ static inline void unregister_fair_sched_group(struct task_group *tg, int cpu) | |||
8031 | { | 8101 | { |
8032 | list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list); | 8102 | list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list); |
8033 | } | 8103 | } |
8034 | #else | 8104 | #else /* !CONFG_FAIR_GROUP_SCHED */ |
8035 | static inline void free_fair_sched_group(struct task_group *tg) | 8105 | static inline void free_fair_sched_group(struct task_group *tg) |
8036 | { | 8106 | { |
8037 | } | 8107 | } |
@@ -8049,7 +8119,7 @@ static inline void register_fair_sched_group(struct task_group *tg, int cpu) | |||
8049 | static inline void unregister_fair_sched_group(struct task_group *tg, int cpu) | 8119 | static inline void unregister_fair_sched_group(struct task_group *tg, int cpu) |
8050 | { | 8120 | { |
8051 | } | 8121 | } |
8052 | #endif | 8122 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
8053 | 8123 | ||
8054 | #ifdef CONFIG_RT_GROUP_SCHED | 8124 | #ifdef CONFIG_RT_GROUP_SCHED |
8055 | static void free_rt_sched_group(struct task_group *tg) | 8125 | static void free_rt_sched_group(struct task_group *tg) |
@@ -8120,7 +8190,7 @@ static inline void unregister_rt_sched_group(struct task_group *tg, int cpu) | |||
8120 | { | 8190 | { |
8121 | list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list); | 8191 | list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list); |
8122 | } | 8192 | } |
8123 | #else | 8193 | #else /* !CONFIG_RT_GROUP_SCHED */ |
8124 | static inline void free_rt_sched_group(struct task_group *tg) | 8194 | static inline void free_rt_sched_group(struct task_group *tg) |
8125 | { | 8195 | { |
8126 | } | 8196 | } |
@@ -8138,7 +8208,7 @@ static inline void register_rt_sched_group(struct task_group *tg, int cpu) | |||
8138 | static inline void unregister_rt_sched_group(struct task_group *tg, int cpu) | 8208 | static inline void unregister_rt_sched_group(struct task_group *tg, int cpu) |
8139 | { | 8209 | { |
8140 | } | 8210 | } |
8141 | #endif | 8211 | #endif /* CONFIG_RT_GROUP_SCHED */ |
8142 | 8212 | ||
8143 | #ifdef CONFIG_GROUP_SCHED | 8213 | #ifdef CONFIG_GROUP_SCHED |
8144 | static void free_sched_group(struct task_group *tg) | 8214 | static void free_sched_group(struct task_group *tg) |
@@ -8249,7 +8319,7 @@ void sched_move_task(struct task_struct *tsk) | |||
8249 | 8319 | ||
8250 | task_rq_unlock(rq, &flags); | 8320 | task_rq_unlock(rq, &flags); |
8251 | } | 8321 | } |
8252 | #endif | 8322 | #endif /* CONFIG_GROUP_SCHED */ |
8253 | 8323 | ||
8254 | #ifdef CONFIG_FAIR_GROUP_SCHED | 8324 | #ifdef CONFIG_FAIR_GROUP_SCHED |
8255 | static void set_se_shares(struct sched_entity *se, unsigned long shares) | 8325 | static void set_se_shares(struct sched_entity *se, unsigned long shares) |
@@ -8499,7 +8569,7 @@ static int sched_rt_global_constraints(void) | |||
8499 | 8569 | ||
8500 | return ret; | 8570 | return ret; |
8501 | } | 8571 | } |
8502 | #else | 8572 | #else /* !CONFIG_RT_GROUP_SCHED */ |
8503 | static int sched_rt_global_constraints(void) | 8573 | static int sched_rt_global_constraints(void) |
8504 | { | 8574 | { |
8505 | unsigned long flags; | 8575 | unsigned long flags; |
@@ -8517,7 +8587,7 @@ static int sched_rt_global_constraints(void) | |||
8517 | 8587 | ||
8518 | return 0; | 8588 | return 0; |
8519 | } | 8589 | } |
8520 | #endif | 8590 | #endif /* CONFIG_RT_GROUP_SCHED */ |
8521 | 8591 | ||
8522 | int sched_rt_handler(struct ctl_table *table, int write, | 8592 | int sched_rt_handler(struct ctl_table *table, int write, |
8523 | struct file *filp, void __user *buffer, size_t *lenp, | 8593 | struct file *filp, void __user *buffer, size_t *lenp, |
@@ -8625,7 +8695,7 @@ static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft) | |||
8625 | 8695 | ||
8626 | return (u64) tg->shares; | 8696 | return (u64) tg->shares; |
8627 | } | 8697 | } |
8628 | #endif | 8698 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
8629 | 8699 | ||
8630 | #ifdef CONFIG_RT_GROUP_SCHED | 8700 | #ifdef CONFIG_RT_GROUP_SCHED |
8631 | static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft, | 8701 | static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft, |
@@ -8649,7 +8719,7 @@ static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft) | |||
8649 | { | 8719 | { |
8650 | return sched_group_rt_period(cgroup_tg(cgrp)); | 8720 | return sched_group_rt_period(cgroup_tg(cgrp)); |
8651 | } | 8721 | } |
8652 | #endif | 8722 | #endif /* CONFIG_RT_GROUP_SCHED */ |
8653 | 8723 | ||
8654 | static struct cftype cpu_files[] = { | 8724 | static struct cftype cpu_files[] = { |
8655 | #ifdef CONFIG_FAIR_GROUP_SCHED | 8725 | #ifdef CONFIG_FAIR_GROUP_SCHED |