diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/fork.c | 2 | ||||
| -rw-r--r-- | kernel/hrtimer.c | 2 | ||||
| -rw-r--r-- | kernel/profile.c | 5 | ||||
| -rw-r--r-- | kernel/sched.c | 396 | ||||
| -rw-r--r-- | kernel/sched_fair.c | 50 | ||||
| -rw-r--r-- | kernel/sched_idletask.c | 18 | ||||
| -rw-r--r-- | kernel/sched_rt.c | 32 | ||||
| -rw-r--r-- | kernel/signal.c | 4 | ||||
| -rw-r--r-- | kernel/sys_ni.c | 4 | ||||
| -rw-r--r-- | kernel/time/tick-sched.c | 2 | ||||
| -rw-r--r-- | kernel/time/timer_list.c | 6 | ||||
| -rw-r--r-- | kernel/user.c | 5 |
12 files changed, 321 insertions, 205 deletions
diff --git a/kernel/fork.c b/kernel/fork.c index ddafdfac9456..28a740151988 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
| @@ -1056,6 +1056,8 @@ static struct task_struct *copy_process(unsigned long clone_flags, | |||
| 1056 | p->gtime = cputime_zero; | 1056 | p->gtime = cputime_zero; |
| 1057 | p->utimescaled = cputime_zero; | 1057 | p->utimescaled = cputime_zero; |
| 1058 | p->stimescaled = cputime_zero; | 1058 | p->stimescaled = cputime_zero; |
| 1059 | p->prev_utime = cputime_zero; | ||
| 1060 | p->prev_stime = cputime_zero; | ||
| 1059 | 1061 | ||
| 1060 | #ifdef CONFIG_TASK_XACCT | 1062 | #ifdef CONFIG_TASK_XACCT |
| 1061 | p->rchar = 0; /* I/O counter: bytes read */ | 1063 | p->rchar = 0; /* I/O counter: bytes read */ |
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index b6d2ff7e37ee..22a25142e4cf 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
| @@ -602,7 +602,7 @@ static int hrtimer_switch_to_hres(void) | |||
| 602 | /* "Retrigger" the interrupt to get things going */ | 602 | /* "Retrigger" the interrupt to get things going */ |
| 603 | retrigger_next_event(NULL); | 603 | retrigger_next_event(NULL); |
| 604 | local_irq_restore(flags); | 604 | local_irq_restore(flags); |
| 605 | printk(KERN_INFO "Switched to high resolution mode on CPU %d\n", | 605 | printk(KERN_DEBUG "Switched to high resolution mode on CPU %d\n", |
| 606 | smp_processor_id()); | 606 | smp_processor_id()); |
| 607 | return 1; | 607 | return 1; |
| 608 | } | 608 | } |
diff --git a/kernel/profile.c b/kernel/profile.c index 631b75c25d7e..5e95330e5120 100644 --- a/kernel/profile.c +++ b/kernel/profile.c | |||
| @@ -60,6 +60,7 @@ static int __init profile_setup(char * str) | |||
| 60 | int par; | 60 | int par; |
| 61 | 61 | ||
| 62 | if (!strncmp(str, sleepstr, strlen(sleepstr))) { | 62 | if (!strncmp(str, sleepstr, strlen(sleepstr))) { |
| 63 | #ifdef CONFIG_SCHEDSTATS | ||
| 63 | prof_on = SLEEP_PROFILING; | 64 | prof_on = SLEEP_PROFILING; |
| 64 | if (str[strlen(sleepstr)] == ',') | 65 | if (str[strlen(sleepstr)] == ',') |
| 65 | str += strlen(sleepstr) + 1; | 66 | str += strlen(sleepstr) + 1; |
| @@ -68,6 +69,10 @@ static int __init profile_setup(char * str) | |||
| 68 | printk(KERN_INFO | 69 | printk(KERN_INFO |
| 69 | "kernel sleep profiling enabled (shift: %ld)\n", | 70 | "kernel sleep profiling enabled (shift: %ld)\n", |
| 70 | prof_shift); | 71 | prof_shift); |
| 72 | #else | ||
| 73 | printk(KERN_WARNING | ||
| 74 | "kernel sleep profiling requires CONFIG_SCHEDSTATS\n"); | ||
| 75 | #endif /* CONFIG_SCHEDSTATS */ | ||
| 71 | } else if (!strncmp(str, schedstr, strlen(schedstr))) { | 76 | } else if (!strncmp(str, schedstr, strlen(schedstr))) { |
| 72 | prof_on = SCHED_PROFILING; | 77 | prof_on = SCHED_PROFILING; |
| 73 | if (str[strlen(schedstr)] == ',') | 78 | if (str[strlen(schedstr)] == ',') |
diff --git a/kernel/sched.c b/kernel/sched.c index 2810e562a991..3f6bd1112900 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
| @@ -66,6 +66,7 @@ | |||
| 66 | #include <linux/pagemap.h> | 66 | #include <linux/pagemap.h> |
| 67 | 67 | ||
| 68 | #include <asm/tlb.h> | 68 | #include <asm/tlb.h> |
| 69 | #include <asm/irq_regs.h> | ||
| 69 | 70 | ||
| 70 | /* | 71 | /* |
| 71 | * Scheduler clock - returns current time in nanosec units. | 72 | * Scheduler clock - returns current time in nanosec units. |
| @@ -171,6 +172,7 @@ struct task_group { | |||
| 171 | unsigned long shares; | 172 | unsigned long shares; |
| 172 | /* spinlock to serialize modification to shares */ | 173 | /* spinlock to serialize modification to shares */ |
| 173 | spinlock_t lock; | 174 | spinlock_t lock; |
| 175 | struct rcu_head rcu; | ||
| 174 | }; | 176 | }; |
| 175 | 177 | ||
| 176 | /* Default task group's sched entity on each cpu */ | 178 | /* Default task group's sched entity on each cpu */ |
| @@ -257,7 +259,6 @@ struct cfs_rq { | |||
| 257 | */ | 259 | */ |
| 258 | struct list_head leaf_cfs_rq_list; /* Better name : task_cfs_rq_list? */ | 260 | struct list_head leaf_cfs_rq_list; /* Better name : task_cfs_rq_list? */ |
| 259 | struct task_group *tg; /* group that "owns" this runqueue */ | 261 | struct task_group *tg; /* group that "owns" this runqueue */ |
| 260 | struct rcu_head rcu; | ||
| 261 | #endif | 262 | #endif |
| 262 | }; | 263 | }; |
| 263 | 264 | ||
| @@ -837,11 +838,18 @@ struct rq_iterator { | |||
| 837 | struct task_struct *(*next)(void *); | 838 | struct task_struct *(*next)(void *); |
| 838 | }; | 839 | }; |
| 839 | 840 | ||
| 840 | static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, | 841 | #ifdef CONFIG_SMP |
| 841 | unsigned long max_nr_move, unsigned long max_load_move, | 842 | static unsigned long |
| 842 | struct sched_domain *sd, enum cpu_idle_type idle, | 843 | balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, |
| 843 | int *all_pinned, unsigned long *load_moved, | 844 | unsigned long max_load_move, struct sched_domain *sd, |
| 844 | int *this_best_prio, struct rq_iterator *iterator); | 845 | enum cpu_idle_type idle, int *all_pinned, |
| 846 | int *this_best_prio, struct rq_iterator *iterator); | ||
| 847 | |||
| 848 | static int | ||
| 849 | iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest, | ||
| 850 | struct sched_domain *sd, enum cpu_idle_type idle, | ||
| 851 | struct rq_iterator *iterator); | ||
| 852 | #endif | ||
| 845 | 853 | ||
| 846 | #include "sched_stats.h" | 854 | #include "sched_stats.h" |
| 847 | #include "sched_idletask.c" | 855 | #include "sched_idletask.c" |
| @@ -2223,17 +2231,17 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu, | |||
| 2223 | return 1; | 2231 | return 1; |
| 2224 | } | 2232 | } |
| 2225 | 2233 | ||
| 2226 | static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, | 2234 | static unsigned long |
| 2227 | unsigned long max_nr_move, unsigned long max_load_move, | 2235 | balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, |
| 2228 | struct sched_domain *sd, enum cpu_idle_type idle, | 2236 | unsigned long max_load_move, struct sched_domain *sd, |
| 2229 | int *all_pinned, unsigned long *load_moved, | 2237 | enum cpu_idle_type idle, int *all_pinned, |
| 2230 | int *this_best_prio, struct rq_iterator *iterator) | 2238 | int *this_best_prio, struct rq_iterator *iterator) |
| 2231 | { | 2239 | { |
| 2232 | int pulled = 0, pinned = 0, skip_for_load; | 2240 | int pulled = 0, pinned = 0, skip_for_load; |
| 2233 | struct task_struct *p; | 2241 | struct task_struct *p; |
| 2234 | long rem_load_move = max_load_move; | 2242 | long rem_load_move = max_load_move; |
| 2235 | 2243 | ||
| 2236 | if (max_nr_move == 0 || max_load_move == 0) | 2244 | if (max_load_move == 0) |
| 2237 | goto out; | 2245 | goto out; |
| 2238 | 2246 | ||
| 2239 | pinned = 1; | 2247 | pinned = 1; |
| @@ -2266,7 +2274,7 @@ next: | |||
| 2266 | * We only want to steal up to the prescribed number of tasks | 2274 | * We only want to steal up to the prescribed number of tasks |
| 2267 | * and the prescribed amount of weighted load. | 2275 | * and the prescribed amount of weighted load. |
| 2268 | */ | 2276 | */ |
| 2269 | if (pulled < max_nr_move && rem_load_move > 0) { | 2277 | if (rem_load_move > 0) { |
| 2270 | if (p->prio < *this_best_prio) | 2278 | if (p->prio < *this_best_prio) |
| 2271 | *this_best_prio = p->prio; | 2279 | *this_best_prio = p->prio; |
| 2272 | p = iterator->next(iterator->arg); | 2280 | p = iterator->next(iterator->arg); |
| @@ -2274,7 +2282,7 @@ next: | |||
| 2274 | } | 2282 | } |
| 2275 | out: | 2283 | out: |
| 2276 | /* | 2284 | /* |
| 2277 | * Right now, this is the only place pull_task() is called, | 2285 | * Right now, this is one of only two places pull_task() is called, |
| 2278 | * so we can safely collect pull_task() stats here rather than | 2286 | * so we can safely collect pull_task() stats here rather than |
| 2279 | * inside pull_task(). | 2287 | * inside pull_task(). |
| 2280 | */ | 2288 | */ |
| @@ -2282,8 +2290,8 @@ out: | |||
| 2282 | 2290 | ||
| 2283 | if (all_pinned) | 2291 | if (all_pinned) |
| 2284 | *all_pinned = pinned; | 2292 | *all_pinned = pinned; |
| 2285 | *load_moved = max_load_move - rem_load_move; | 2293 | |
| 2286 | return pulled; | 2294 | return max_load_move - rem_load_move; |
| 2287 | } | 2295 | } |
| 2288 | 2296 | ||
| 2289 | /* | 2297 | /* |
| @@ -2305,7 +2313,7 @@ static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, | |||
| 2305 | do { | 2313 | do { |
| 2306 | total_load_moved += | 2314 | total_load_moved += |
| 2307 | class->load_balance(this_rq, this_cpu, busiest, | 2315 | class->load_balance(this_rq, this_cpu, busiest, |
| 2308 | ULONG_MAX, max_load_move - total_load_moved, | 2316 | max_load_move - total_load_moved, |
| 2309 | sd, idle, all_pinned, &this_best_prio); | 2317 | sd, idle, all_pinned, &this_best_prio); |
| 2310 | class = class->next; | 2318 | class = class->next; |
| 2311 | } while (class && max_load_move > total_load_moved); | 2319 | } while (class && max_load_move > total_load_moved); |
| @@ -2313,6 +2321,32 @@ static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, | |||
| 2313 | return total_load_moved > 0; | 2321 | return total_load_moved > 0; |
| 2314 | } | 2322 | } |
| 2315 | 2323 | ||
| 2324 | static int | ||
| 2325 | iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest, | ||
| 2326 | struct sched_domain *sd, enum cpu_idle_type idle, | ||
| 2327 | struct rq_iterator *iterator) | ||
| 2328 | { | ||
| 2329 | struct task_struct *p = iterator->start(iterator->arg); | ||
| 2330 | int pinned = 0; | ||
| 2331 | |||
| 2332 | while (p) { | ||
| 2333 | if (can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) { | ||
| 2334 | pull_task(busiest, p, this_rq, this_cpu); | ||
| 2335 | /* | ||
| 2336 | * Right now, this is only the second place pull_task() | ||
| 2337 | * is called, so we can safely collect pull_task() | ||
| 2338 | * stats here rather than inside pull_task(). | ||
| 2339 | */ | ||
| 2340 | schedstat_inc(sd, lb_gained[idle]); | ||
| 2341 | |||
| 2342 | return 1; | ||
| 2343 | } | ||
| 2344 | p = iterator->next(iterator->arg); | ||
| 2345 | } | ||
| 2346 | |||
| 2347 | return 0; | ||
| 2348 | } | ||
| 2349 | |||
| 2316 | /* | 2350 | /* |
| 2317 | * move_one_task tries to move exactly one task from busiest to this_rq, as | 2351 | * move_one_task tries to move exactly one task from busiest to this_rq, as |
| 2318 | * part of active balancing operations within "domain". | 2352 | * part of active balancing operations within "domain". |
| @@ -2324,12 +2358,9 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest, | |||
| 2324 | struct sched_domain *sd, enum cpu_idle_type idle) | 2358 | struct sched_domain *sd, enum cpu_idle_type idle) |
| 2325 | { | 2359 | { |
| 2326 | const struct sched_class *class; | 2360 | const struct sched_class *class; |
| 2327 | int this_best_prio = MAX_PRIO; | ||
| 2328 | 2361 | ||
| 2329 | for (class = sched_class_highest; class; class = class->next) | 2362 | for (class = sched_class_highest; class; class = class->next) |
| 2330 | if (class->load_balance(this_rq, this_cpu, busiest, | 2363 | if (class->move_one_task(this_rq, this_cpu, busiest, sd, idle)) |
| 2331 | 1, ULONG_MAX, sd, idle, NULL, | ||
| 2332 | &this_best_prio)) | ||
| 2333 | return 1; | 2364 | return 1; |
| 2334 | 2365 | ||
| 2335 | return 0; | 2366 | return 0; |
| @@ -3266,18 +3297,6 @@ static inline void idle_balance(int cpu, struct rq *rq) | |||
| 3266 | { | 3297 | { |
| 3267 | } | 3298 | } |
| 3268 | 3299 | ||
| 3269 | /* Avoid "used but not defined" warning on UP */ | ||
| 3270 | static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, | ||
| 3271 | unsigned long max_nr_move, unsigned long max_load_move, | ||
| 3272 | struct sched_domain *sd, enum cpu_idle_type idle, | ||
| 3273 | int *all_pinned, unsigned long *load_moved, | ||
| 3274 | int *this_best_prio, struct rq_iterator *iterator) | ||
| 3275 | { | ||
| 3276 | *load_moved = 0; | ||
| 3277 | |||
| 3278 | return 0; | ||
| 3279 | } | ||
| 3280 | |||
| 3281 | #endif | 3300 | #endif |
| 3282 | 3301 | ||
| 3283 | DEFINE_PER_CPU(struct kernel_stat, kstat); | 3302 | DEFINE_PER_CPU(struct kernel_stat, kstat); |
| @@ -3336,7 +3355,7 @@ void account_user_time(struct task_struct *p, cputime_t cputime) | |||
| 3336 | * @p: the process that the cpu time gets accounted to | 3355 | * @p: the process that the cpu time gets accounted to |
| 3337 | * @cputime: the cpu time spent in virtual machine since the last update | 3356 | * @cputime: the cpu time spent in virtual machine since the last update |
| 3338 | */ | 3357 | */ |
| 3339 | void account_guest_time(struct task_struct *p, cputime_t cputime) | 3358 | static void account_guest_time(struct task_struct *p, cputime_t cputime) |
| 3340 | { | 3359 | { |
| 3341 | cputime64_t tmp; | 3360 | cputime64_t tmp; |
| 3342 | struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; | 3361 | struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; |
| @@ -3507,12 +3526,19 @@ EXPORT_SYMBOL(sub_preempt_count); | |||
| 3507 | */ | 3526 | */ |
| 3508 | static noinline void __schedule_bug(struct task_struct *prev) | 3527 | static noinline void __schedule_bug(struct task_struct *prev) |
| 3509 | { | 3528 | { |
| 3510 | printk(KERN_ERR "BUG: scheduling while atomic: %s/0x%08x/%d\n", | 3529 | struct pt_regs *regs = get_irq_regs(); |
| 3511 | prev->comm, preempt_count(), task_pid_nr(prev)); | 3530 | |
| 3531 | printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", | ||
| 3532 | prev->comm, prev->pid, preempt_count()); | ||
| 3533 | |||
| 3512 | debug_show_held_locks(prev); | 3534 | debug_show_held_locks(prev); |
| 3513 | if (irqs_disabled()) | 3535 | if (irqs_disabled()) |
| 3514 | print_irqtrace_events(prev); | 3536 | print_irqtrace_events(prev); |
| 3515 | dump_stack(); | 3537 | |
| 3538 | if (regs) | ||
| 3539 | show_regs(regs); | ||
| 3540 | else | ||
| 3541 | dump_stack(); | ||
| 3516 | } | 3542 | } |
| 3517 | 3543 | ||
| 3518 | /* | 3544 | /* |
| @@ -3820,7 +3846,7 @@ __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive) | |||
| 3820 | } | 3846 | } |
| 3821 | EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ | 3847 | EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ |
| 3822 | 3848 | ||
| 3823 | void fastcall complete(struct completion *x) | 3849 | void complete(struct completion *x) |
| 3824 | { | 3850 | { |
| 3825 | unsigned long flags; | 3851 | unsigned long flags; |
| 3826 | 3852 | ||
| @@ -3832,7 +3858,7 @@ void fastcall complete(struct completion *x) | |||
| 3832 | } | 3858 | } |
| 3833 | EXPORT_SYMBOL(complete); | 3859 | EXPORT_SYMBOL(complete); |
| 3834 | 3860 | ||
| 3835 | void fastcall complete_all(struct completion *x) | 3861 | void complete_all(struct completion *x) |
| 3836 | { | 3862 | { |
| 3837 | unsigned long flags; | 3863 | unsigned long flags; |
| 3838 | 3864 | ||
| @@ -3884,13 +3910,13 @@ wait_for_common(struct completion *x, long timeout, int state) | |||
| 3884 | return timeout; | 3910 | return timeout; |
| 3885 | } | 3911 | } |
| 3886 | 3912 | ||
| 3887 | void fastcall __sched wait_for_completion(struct completion *x) | 3913 | void __sched wait_for_completion(struct completion *x) |
| 3888 | { | 3914 | { |
| 3889 | wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); | 3915 | wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); |
| 3890 | } | 3916 | } |
| 3891 | EXPORT_SYMBOL(wait_for_completion); | 3917 | EXPORT_SYMBOL(wait_for_completion); |
| 3892 | 3918 | ||
| 3893 | unsigned long fastcall __sched | 3919 | unsigned long __sched |
| 3894 | wait_for_completion_timeout(struct completion *x, unsigned long timeout) | 3920 | wait_for_completion_timeout(struct completion *x, unsigned long timeout) |
| 3895 | { | 3921 | { |
| 3896 | return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); | 3922 | return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); |
| @@ -3906,7 +3932,7 @@ int __sched wait_for_completion_interruptible(struct completion *x) | |||
| 3906 | } | 3932 | } |
| 3907 | EXPORT_SYMBOL(wait_for_completion_interruptible); | 3933 | EXPORT_SYMBOL(wait_for_completion_interruptible); |
| 3908 | 3934 | ||
| 3909 | unsigned long fastcall __sched | 3935 | unsigned long __sched |
| 3910 | wait_for_completion_interruptible_timeout(struct completion *x, | 3936 | wait_for_completion_interruptible_timeout(struct completion *x, |
| 3911 | unsigned long timeout) | 3937 | unsigned long timeout) |
| 3912 | { | 3938 | { |
| @@ -5339,7 +5365,7 @@ static struct ctl_table sd_ctl_dir[] = { | |||
| 5339 | .procname = "sched_domain", | 5365 | .procname = "sched_domain", |
| 5340 | .mode = 0555, | 5366 | .mode = 0555, |
| 5341 | }, | 5367 | }, |
| 5342 | {0,}, | 5368 | {0, }, |
| 5343 | }; | 5369 | }; |
| 5344 | 5370 | ||
| 5345 | static struct ctl_table sd_ctl_root[] = { | 5371 | static struct ctl_table sd_ctl_root[] = { |
| @@ -5349,7 +5375,7 @@ static struct ctl_table sd_ctl_root[] = { | |||
| 5349 | .mode = 0555, | 5375 | .mode = 0555, |
| 5350 | .child = sd_ctl_dir, | 5376 | .child = sd_ctl_dir, |
| 5351 | }, | 5377 | }, |
| 5352 | {0,}, | 5378 | {0, }, |
| 5353 | }; | 5379 | }; |
| 5354 | 5380 | ||
| 5355 | static struct ctl_table *sd_alloc_ctl_entry(int n) | 5381 | static struct ctl_table *sd_alloc_ctl_entry(int n) |
| @@ -5461,11 +5487,12 @@ static void register_sched_domain_sysctl(void) | |||
| 5461 | struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1); | 5487 | struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1); |
| 5462 | char buf[32]; | 5488 | char buf[32]; |
| 5463 | 5489 | ||
| 5490 | WARN_ON(sd_ctl_dir[0].child); | ||
| 5491 | sd_ctl_dir[0].child = entry; | ||
| 5492 | |||
| 5464 | if (entry == NULL) | 5493 | if (entry == NULL) |
| 5465 | return; | 5494 | return; |
| 5466 | 5495 | ||
| 5467 | sd_ctl_dir[0].child = entry; | ||
| 5468 | |||
| 5469 | for_each_online_cpu(i) { | 5496 | for_each_online_cpu(i) { |
| 5470 | snprintf(buf, 32, "cpu%d", i); | 5497 | snprintf(buf, 32, "cpu%d", i); |
| 5471 | entry->procname = kstrdup(buf, GFP_KERNEL); | 5498 | entry->procname = kstrdup(buf, GFP_KERNEL); |
| @@ -5473,14 +5500,19 @@ static void register_sched_domain_sysctl(void) | |||
| 5473 | entry->child = sd_alloc_ctl_cpu_table(i); | 5500 | entry->child = sd_alloc_ctl_cpu_table(i); |
| 5474 | entry++; | 5501 | entry++; |
| 5475 | } | 5502 | } |
| 5503 | |||
| 5504 | WARN_ON(sd_sysctl_header); | ||
| 5476 | sd_sysctl_header = register_sysctl_table(sd_ctl_root); | 5505 | sd_sysctl_header = register_sysctl_table(sd_ctl_root); |
| 5477 | } | 5506 | } |
| 5478 | 5507 | ||
| 5508 | /* may be called multiple times per register */ | ||
| 5479 | static void unregister_sched_domain_sysctl(void) | 5509 | static void unregister_sched_domain_sysctl(void) |
| 5480 | { | 5510 | { |
| 5481 | unregister_sysctl_table(sd_sysctl_header); | 5511 | if (sd_sysctl_header) |
| 5512 | unregister_sysctl_table(sd_sysctl_header); | ||
| 5482 | sd_sysctl_header = NULL; | 5513 | sd_sysctl_header = NULL; |
| 5483 | sd_free_ctl_entry(&sd_ctl_dir[0].child); | 5514 | if (sd_ctl_dir[0].child) |
| 5515 | sd_free_ctl_entry(&sd_ctl_dir[0].child); | ||
| 5484 | } | 5516 | } |
| 5485 | #else | 5517 | #else |
| 5486 | static void register_sched_domain_sysctl(void) | 5518 | static void register_sched_domain_sysctl(void) |
| @@ -5611,101 +5643,101 @@ int nr_cpu_ids __read_mostly = NR_CPUS; | |||
| 5611 | EXPORT_SYMBOL(nr_cpu_ids); | 5643 | EXPORT_SYMBOL(nr_cpu_ids); |
| 5612 | 5644 | ||
| 5613 | #ifdef CONFIG_SCHED_DEBUG | 5645 | #ifdef CONFIG_SCHED_DEBUG |
| 5614 | static void sched_domain_debug(struct sched_domain *sd, int cpu) | 5646 | |
| 5647 | static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level) | ||
| 5615 | { | 5648 | { |
| 5616 | int level = 0; | 5649 | struct sched_group *group = sd->groups; |
| 5650 | cpumask_t groupmask; | ||
| 5651 | char str[NR_CPUS]; | ||
| 5617 | 5652 | ||
| 5618 | if (!sd) { | 5653 | cpumask_scnprintf(str, NR_CPUS, sd->span); |
| 5619 | printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu); | 5654 | cpus_clear(groupmask); |
| 5620 | return; | 5655 | |
| 5656 | printk(KERN_DEBUG "%*s domain %d: ", level, "", level); | ||
| 5657 | |||
| 5658 | if (!(sd->flags & SD_LOAD_BALANCE)) { | ||
| 5659 | printk("does not load-balance\n"); | ||
| 5660 | if (sd->parent) | ||
| 5661 | printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain" | ||
| 5662 | " has parent"); | ||
| 5663 | return -1; | ||
| 5621 | } | 5664 | } |
| 5622 | 5665 | ||
| 5623 | printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); | 5666 | printk(KERN_CONT "span %s\n", str); |
| 5624 | 5667 | ||
| 5668 | if (!cpu_isset(cpu, sd->span)) { | ||
| 5669 | printk(KERN_ERR "ERROR: domain->span does not contain " | ||
| 5670 | "CPU%d\n", cpu); | ||
| 5671 | } | ||
| 5672 | if (!cpu_isset(cpu, group->cpumask)) { | ||
| 5673 | printk(KERN_ERR "ERROR: domain->groups does not contain" | ||
| 5674 | " CPU%d\n", cpu); | ||
| 5675 | } | ||
| 5676 | |||
| 5677 | printk(KERN_DEBUG "%*s groups:", level + 1, ""); | ||
| 5625 | do { | 5678 | do { |
| 5626 | int i; | 5679 | if (!group) { |
| 5627 | char str[NR_CPUS]; | 5680 | printk("\n"); |
| 5628 | struct sched_group *group = sd->groups; | 5681 | printk(KERN_ERR "ERROR: group is NULL\n"); |
| 5629 | cpumask_t groupmask; | ||
| 5630 | |||
| 5631 | cpumask_scnprintf(str, NR_CPUS, sd->span); | ||
| 5632 | cpus_clear(groupmask); | ||
| 5633 | |||
| 5634 | printk(KERN_DEBUG); | ||
| 5635 | for (i = 0; i < level + 1; i++) | ||
| 5636 | printk(" "); | ||
| 5637 | printk("domain %d: ", level); | ||
| 5638 | |||
| 5639 | if (!(sd->flags & SD_LOAD_BALANCE)) { | ||
| 5640 | printk("does not load-balance\n"); | ||
| 5641 | if (sd->parent) | ||
| 5642 | printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain" | ||
| 5643 | " has parent"); | ||
| 5644 | break; | 5682 | break; |
| 5645 | } | 5683 | } |
| 5646 | 5684 | ||
| 5647 | printk("span %s\n", str); | 5685 | if (!group->__cpu_power) { |
| 5686 | printk(KERN_CONT "\n"); | ||
| 5687 | printk(KERN_ERR "ERROR: domain->cpu_power not " | ||
| 5688 | "set\n"); | ||
| 5689 | break; | ||
| 5690 | } | ||
| 5691 | |||
| 5692 | if (!cpus_weight(group->cpumask)) { | ||
| 5693 | printk(KERN_CONT "\n"); | ||
| 5694 | printk(KERN_ERR "ERROR: empty group\n"); | ||
| 5695 | break; | ||
| 5696 | } | ||
| 5648 | 5697 | ||
| 5649 | if (!cpu_isset(cpu, sd->span)) | 5698 | if (cpus_intersects(groupmask, group->cpumask)) { |
| 5650 | printk(KERN_ERR "ERROR: domain->span does not contain " | 5699 | printk(KERN_CONT "\n"); |
| 5651 | "CPU%d\n", cpu); | 5700 | printk(KERN_ERR "ERROR: repeated CPUs\n"); |
| 5652 | if (!cpu_isset(cpu, group->cpumask)) | 5701 | break; |
| 5653 | printk(KERN_ERR "ERROR: domain->groups does not contain" | 5702 | } |
| 5654 | " CPU%d\n", cpu); | ||
| 5655 | 5703 | ||
| 5656 | printk(KERN_DEBUG); | 5704 | cpus_or(groupmask, groupmask, group->cpumask); |
| 5657 | for (i = 0; i < level + 2; i++) | ||
| 5658 | printk(" "); | ||
| 5659 | printk("groups:"); | ||
| 5660 | do { | ||
| 5661 | if (!group) { | ||
| 5662 | printk("\n"); | ||
| 5663 | printk(KERN_ERR "ERROR: group is NULL\n"); | ||
| 5664 | break; | ||
| 5665 | } | ||
| 5666 | 5705 | ||
| 5667 | if (!group->__cpu_power) { | 5706 | cpumask_scnprintf(str, NR_CPUS, group->cpumask); |
| 5668 | printk(KERN_CONT "\n"); | 5707 | printk(KERN_CONT " %s", str); |
| 5669 | printk(KERN_ERR "ERROR: domain->cpu_power not " | ||
| 5670 | "set\n"); | ||
| 5671 | break; | ||
| 5672 | } | ||
| 5673 | 5708 | ||
| 5674 | if (!cpus_weight(group->cpumask)) { | 5709 | group = group->next; |
| 5675 | printk(KERN_CONT "\n"); | 5710 | } while (group != sd->groups); |
| 5676 | printk(KERN_ERR "ERROR: empty group\n"); | 5711 | printk(KERN_CONT "\n"); |
| 5677 | break; | ||
| 5678 | } | ||
| 5679 | 5712 | ||
| 5680 | if (cpus_intersects(groupmask, group->cpumask)) { | 5713 | if (!cpus_equal(sd->span, groupmask)) |
| 5681 | printk(KERN_CONT "\n"); | 5714 | printk(KERN_ERR "ERROR: groups don't span domain->span\n"); |
| 5682 | printk(KERN_ERR "ERROR: repeated CPUs\n"); | ||
| 5683 | break; | ||
| 5684 | } | ||
| 5685 | 5715 | ||
| 5686 | cpus_or(groupmask, groupmask, group->cpumask); | 5716 | if (sd->parent && !cpus_subset(groupmask, sd->parent->span)) |
| 5717 | printk(KERN_ERR "ERROR: parent span is not a superset " | ||
| 5718 | "of domain->span\n"); | ||
| 5719 | return 0; | ||
| 5720 | } | ||
| 5687 | 5721 | ||
| 5688 | cpumask_scnprintf(str, NR_CPUS, group->cpumask); | 5722 | static void sched_domain_debug(struct sched_domain *sd, int cpu) |
| 5689 | printk(KERN_CONT " %s", str); | 5723 | { |
| 5724 | int level = 0; | ||
| 5690 | 5725 | ||
| 5691 | group = group->next; | 5726 | if (!sd) { |
| 5692 | } while (group != sd->groups); | 5727 | printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu); |
| 5693 | printk(KERN_CONT "\n"); | 5728 | return; |
| 5729 | } | ||
| 5694 | 5730 | ||
| 5695 | if (!cpus_equal(sd->span, groupmask)) | 5731 | printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); |
| 5696 | printk(KERN_ERR "ERROR: groups don't span " | ||
| 5697 | "domain->span\n"); | ||
| 5698 | 5732 | ||
| 5733 | for (;;) { | ||
| 5734 | if (sched_domain_debug_one(sd, cpu, level)) | ||
| 5735 | break; | ||
| 5699 | level++; | 5736 | level++; |
| 5700 | sd = sd->parent; | 5737 | sd = sd->parent; |
| 5701 | if (!sd) | 5738 | if (!sd) |
| 5702 | continue; | 5739 | break; |
| 5703 | 5740 | } | |
| 5704 | if (!cpus_subset(groupmask, sd->span)) | ||
| 5705 | printk(KERN_ERR "ERROR: parent span is not a superset " | ||
| 5706 | "of domain->span\n"); | ||
| 5707 | |||
| 5708 | } while (sd); | ||
| 5709 | } | 5741 | } |
| 5710 | #else | 5742 | #else |
| 5711 | # define sched_domain_debug(sd, cpu) do { } while (0) | 5743 | # define sched_domain_debug(sd, cpu) do { } while (0) |
| @@ -6424,13 +6456,17 @@ static cpumask_t fallback_doms; | |||
| 6424 | */ | 6456 | */ |
| 6425 | static int arch_init_sched_domains(const cpumask_t *cpu_map) | 6457 | static int arch_init_sched_domains(const cpumask_t *cpu_map) |
| 6426 | { | 6458 | { |
| 6459 | int err; | ||
| 6460 | |||
| 6427 | ndoms_cur = 1; | 6461 | ndoms_cur = 1; |
| 6428 | doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL); | 6462 | doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL); |
| 6429 | if (!doms_cur) | 6463 | if (!doms_cur) |
| 6430 | doms_cur = &fallback_doms; | 6464 | doms_cur = &fallback_doms; |
| 6431 | cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map); | 6465 | cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map); |
| 6466 | err = build_sched_domains(doms_cur); | ||
| 6432 | register_sched_domain_sysctl(); | 6467 | register_sched_domain_sysctl(); |
| 6433 | return build_sched_domains(doms_cur); | 6468 | |
| 6469 | return err; | ||
| 6434 | } | 6470 | } |
| 6435 | 6471 | ||
| 6436 | static void arch_destroy_sched_domains(const cpumask_t *cpu_map) | 6472 | static void arch_destroy_sched_domains(const cpumask_t *cpu_map) |
| @@ -6479,6 +6515,9 @@ void partition_sched_domains(int ndoms_new, cpumask_t *doms_new) | |||
| 6479 | { | 6515 | { |
| 6480 | int i, j; | 6516 | int i, j; |
| 6481 | 6517 | ||
| 6518 | /* always unregister in case we don't destroy any domains */ | ||
| 6519 | unregister_sched_domain_sysctl(); | ||
| 6520 | |||
| 6482 | if (doms_new == NULL) { | 6521 | if (doms_new == NULL) { |
| 6483 | ndoms_new = 1; | 6522 | ndoms_new = 1; |
| 6484 | doms_new = &fallback_doms; | 6523 | doms_new = &fallback_doms; |
| @@ -6514,6 +6553,8 @@ match2: | |||
| 6514 | kfree(doms_cur); | 6553 | kfree(doms_cur); |
| 6515 | doms_cur = doms_new; | 6554 | doms_cur = doms_new; |
| 6516 | ndoms_cur = ndoms_new; | 6555 | ndoms_cur = ndoms_new; |
| 6556 | |||
| 6557 | register_sched_domain_sysctl(); | ||
| 6517 | } | 6558 | } |
| 6518 | 6559 | ||
| 6519 | #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT) | 6560 | #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT) |
| @@ -6978,8 +7019,8 @@ err: | |||
| 6978 | /* rcu callback to free various structures associated with a task group */ | 7019 | /* rcu callback to free various structures associated with a task group */ |
| 6979 | static void free_sched_group(struct rcu_head *rhp) | 7020 | static void free_sched_group(struct rcu_head *rhp) |
| 6980 | { | 7021 | { |
| 6981 | struct cfs_rq *cfs_rq = container_of(rhp, struct cfs_rq, rcu); | 7022 | struct task_group *tg = container_of(rhp, struct task_group, rcu); |
| 6982 | struct task_group *tg = cfs_rq->tg; | 7023 | struct cfs_rq *cfs_rq; |
| 6983 | struct sched_entity *se; | 7024 | struct sched_entity *se; |
| 6984 | int i; | 7025 | int i; |
| 6985 | 7026 | ||
| @@ -7000,7 +7041,7 @@ static void free_sched_group(struct rcu_head *rhp) | |||
| 7000 | /* Destroy runqueue etc associated with a task group */ | 7041 | /* Destroy runqueue etc associated with a task group */ |
| 7001 | void sched_destroy_group(struct task_group *tg) | 7042 | void sched_destroy_group(struct task_group *tg) |
| 7002 | { | 7043 | { |
| 7003 | struct cfs_rq *cfs_rq; | 7044 | struct cfs_rq *cfs_rq = NULL; |
| 7004 | int i; | 7045 | int i; |
| 7005 | 7046 | ||
| 7006 | for_each_possible_cpu(i) { | 7047 | for_each_possible_cpu(i) { |
| @@ -7008,10 +7049,10 @@ void sched_destroy_group(struct task_group *tg) | |||
| 7008 | list_del_rcu(&cfs_rq->leaf_cfs_rq_list); | 7049 | list_del_rcu(&cfs_rq->leaf_cfs_rq_list); |
| 7009 | } | 7050 | } |
| 7010 | 7051 | ||
| 7011 | cfs_rq = tg->cfs_rq[0]; | 7052 | BUG_ON(!cfs_rq); |
| 7012 | 7053 | ||
| 7013 | /* wait for possible concurrent references to cfs_rqs complete */ | 7054 | /* wait for possible concurrent references to cfs_rqs complete */ |
| 7014 | call_rcu(&cfs_rq->rcu, free_sched_group); | 7055 | call_rcu(&tg->rcu, free_sched_group); |
| 7015 | } | 7056 | } |
| 7016 | 7057 | ||
| 7017 | /* change task's runqueue when it moves between groups. | 7058 | /* change task's runqueue when it moves between groups. |
| @@ -7101,25 +7142,25 @@ unsigned long sched_group_shares(struct task_group *tg) | |||
| 7101 | #ifdef CONFIG_FAIR_CGROUP_SCHED | 7142 | #ifdef CONFIG_FAIR_CGROUP_SCHED |
| 7102 | 7143 | ||
| 7103 | /* return corresponding task_group object of a cgroup */ | 7144 | /* return corresponding task_group object of a cgroup */ |
| 7104 | static inline struct task_group *cgroup_tg(struct cgroup *cont) | 7145 | static inline struct task_group *cgroup_tg(struct cgroup *cgrp) |
| 7105 | { | 7146 | { |
| 7106 | return container_of(cgroup_subsys_state(cont, cpu_cgroup_subsys_id), | 7147 | return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id), |
| 7107 | struct task_group, css); | 7148 | struct task_group, css); |
| 7108 | } | 7149 | } |
| 7109 | 7150 | ||
| 7110 | static struct cgroup_subsys_state * | 7151 | static struct cgroup_subsys_state * |
| 7111 | cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) | 7152 | cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp) |
| 7112 | { | 7153 | { |
| 7113 | struct task_group *tg; | 7154 | struct task_group *tg; |
| 7114 | 7155 | ||
| 7115 | if (!cont->parent) { | 7156 | if (!cgrp->parent) { |
| 7116 | /* This is early initialization for the top cgroup */ | 7157 | /* This is early initialization for the top cgroup */ |
| 7117 | init_task_group.css.cgroup = cont; | 7158 | init_task_group.css.cgroup = cgrp; |
| 7118 | return &init_task_group.css; | 7159 | return &init_task_group.css; |
| 7119 | } | 7160 | } |
| 7120 | 7161 | ||
| 7121 | /* we support only 1-level deep hierarchical scheduler atm */ | 7162 | /* we support only 1-level deep hierarchical scheduler atm */ |
| 7122 | if (cont->parent->parent) | 7163 | if (cgrp->parent->parent) |
| 7123 | return ERR_PTR(-EINVAL); | 7164 | return ERR_PTR(-EINVAL); |
| 7124 | 7165 | ||
| 7125 | tg = sched_create_group(); | 7166 | tg = sched_create_group(); |
| @@ -7127,21 +7168,21 @@ cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) | |||
| 7127 | return ERR_PTR(-ENOMEM); | 7168 | return ERR_PTR(-ENOMEM); |
| 7128 | 7169 | ||
| 7129 | /* Bind the cgroup to task_group object we just created */ | 7170 | /* Bind the cgroup to task_group object we just created */ |
| 7130 | tg->css.cgroup = cont; | 7171 | tg->css.cgroup = cgrp; |
| 7131 | 7172 | ||
| 7132 | return &tg->css; | 7173 | return &tg->css; |
| 7133 | } | 7174 | } |
| 7134 | 7175 | ||
| 7135 | static void cpu_cgroup_destroy(struct cgroup_subsys *ss, | 7176 | static void cpu_cgroup_destroy(struct cgroup_subsys *ss, |
| 7136 | struct cgroup *cont) | 7177 | struct cgroup *cgrp) |
| 7137 | { | 7178 | { |
| 7138 | struct task_group *tg = cgroup_tg(cont); | 7179 | struct task_group *tg = cgroup_tg(cgrp); |
| 7139 | 7180 | ||
| 7140 | sched_destroy_group(tg); | 7181 | sched_destroy_group(tg); |
| 7141 | } | 7182 | } |
| 7142 | 7183 | ||
| 7143 | static int cpu_cgroup_can_attach(struct cgroup_subsys *ss, | 7184 | static int cpu_cgroup_can_attach(struct cgroup_subsys *ss, |
| 7144 | struct cgroup *cont, struct task_struct *tsk) | 7185 | struct cgroup *cgrp, struct task_struct *tsk) |
| 7145 | { | 7186 | { |
| 7146 | /* We don't support RT-tasks being in separate groups */ | 7187 | /* We don't support RT-tasks being in separate groups */ |
| 7147 | if (tsk->sched_class != &fair_sched_class) | 7188 | if (tsk->sched_class != &fair_sched_class) |
| @@ -7151,61 +7192,72 @@ static int cpu_cgroup_can_attach(struct cgroup_subsys *ss, | |||
| 7151 | } | 7192 | } |
| 7152 | 7193 | ||
| 7153 | static void | 7194 | static void |
| 7154 | cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cont, | 7195 | cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp, |
| 7155 | struct cgroup *old_cont, struct task_struct *tsk) | 7196 | struct cgroup *old_cont, struct task_struct *tsk) |
| 7156 | { | 7197 | { |
| 7157 | sched_move_task(tsk); | 7198 | sched_move_task(tsk); |
| 7158 | } | 7199 | } |
| 7159 | 7200 | ||
| 7160 | static ssize_t cpu_shares_write(struct cgroup *cont, struct cftype *cftype, | 7201 | static int cpu_shares_write_uint(struct cgroup *cgrp, struct cftype *cftype, |
| 7161 | struct file *file, const char __user *userbuf, | 7202 | u64 shareval) |
| 7162 | size_t nbytes, loff_t *ppos) | ||
| 7163 | { | 7203 | { |
| 7164 | unsigned long shareval; | 7204 | return sched_group_set_shares(cgroup_tg(cgrp), shareval); |
| 7165 | struct task_group *tg = cgroup_tg(cont); | 7205 | } |
| 7166 | char buffer[2*sizeof(unsigned long) + 1]; | ||
| 7167 | int rc; | ||
| 7168 | |||
| 7169 | if (nbytes > 2*sizeof(unsigned long)) /* safety check */ | ||
| 7170 | return -E2BIG; | ||
| 7171 | |||
| 7172 | if (copy_from_user(buffer, userbuf, nbytes)) | ||
| 7173 | return -EFAULT; | ||
| 7174 | |||
| 7175 | buffer[nbytes] = 0; /* nul-terminate */ | ||
| 7176 | shareval = simple_strtoul(buffer, NULL, 10); | ||
| 7177 | 7206 | ||
| 7178 | rc = sched_group_set_shares(tg, shareval); | 7207 | static u64 cpu_shares_read_uint(struct cgroup *cgrp, struct cftype *cft) |
| 7208 | { | ||
| 7209 | struct task_group *tg = cgroup_tg(cgrp); | ||
| 7179 | 7210 | ||
| 7180 | return (rc < 0 ? rc : nbytes); | 7211 | return (u64) tg->shares; |
| 7181 | } | 7212 | } |
| 7182 | 7213 | ||
| 7183 | static u64 cpu_shares_read_uint(struct cgroup *cont, struct cftype *cft) | 7214 | static u64 cpu_usage_read(struct cgroup *cgrp, struct cftype *cft) |
| 7184 | { | 7215 | { |
| 7185 | struct task_group *tg = cgroup_tg(cont); | 7216 | struct task_group *tg = cgroup_tg(cgrp); |
| 7217 | unsigned long flags; | ||
| 7218 | u64 res = 0; | ||
| 7219 | int i; | ||
| 7186 | 7220 | ||
| 7187 | return (u64) tg->shares; | 7221 | for_each_possible_cpu(i) { |
| 7222 | /* | ||
| 7223 | * Lock to prevent races with updating 64-bit counters | ||
| 7224 | * on 32-bit arches. | ||
| 7225 | */ | ||
| 7226 | spin_lock_irqsave(&cpu_rq(i)->lock, flags); | ||
| 7227 | res += tg->se[i]->sum_exec_runtime; | ||
| 7228 | spin_unlock_irqrestore(&cpu_rq(i)->lock, flags); | ||
| 7229 | } | ||
| 7230 | /* Convert from ns to ms */ | ||
| 7231 | do_div(res, 1000000); | ||
| 7232 | |||
| 7233 | return res; | ||
| 7188 | } | 7234 | } |
| 7189 | 7235 | ||
| 7190 | static struct cftype cpu_shares = { | 7236 | static struct cftype cpu_files[] = { |
| 7191 | .name = "shares", | 7237 | { |
| 7192 | .read_uint = cpu_shares_read_uint, | 7238 | .name = "shares", |
| 7193 | .write = cpu_shares_write, | 7239 | .read_uint = cpu_shares_read_uint, |
| 7240 | .write_uint = cpu_shares_write_uint, | ||
| 7241 | }, | ||
| 7242 | { | ||
| 7243 | .name = "usage", | ||
| 7244 | .read_uint = cpu_usage_read, | ||
| 7245 | }, | ||
| 7194 | }; | 7246 | }; |
| 7195 | 7247 | ||
| 7196 | static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont) | 7248 | static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont) |
| 7197 | { | 7249 | { |
| 7198 | return cgroup_add_file(cont, ss, &cpu_shares); | 7250 | return cgroup_add_files(cont, ss, cpu_files, ARRAY_SIZE(cpu_files)); |
| 7199 | } | 7251 | } |
| 7200 | 7252 | ||
| 7201 | struct cgroup_subsys cpu_cgroup_subsys = { | 7253 | struct cgroup_subsys cpu_cgroup_subsys = { |
| 7202 | .name = "cpu", | 7254 | .name = "cpu", |
| 7203 | .create = cpu_cgroup_create, | 7255 | .create = cpu_cgroup_create, |
| 7204 | .destroy = cpu_cgroup_destroy, | 7256 | .destroy = cpu_cgroup_destroy, |
| 7205 | .can_attach = cpu_cgroup_can_attach, | 7257 | .can_attach = cpu_cgroup_can_attach, |
| 7206 | .attach = cpu_cgroup_attach, | 7258 | .attach = cpu_cgroup_attach, |
| 7207 | .populate = cpu_cgroup_populate, | 7259 | .populate = cpu_cgroup_populate, |
| 7208 | .subsys_id = cpu_cgroup_subsys_id, | 7260 | .subsys_id = cpu_cgroup_subsys_id, |
| 7209 | .early_init = 1, | 7261 | .early_init = 1, |
| 7210 | }; | 7262 | }; |
| 7211 | 7263 | ||
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 166ed6db600b..01859f662ab7 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c | |||
| @@ -876,6 +876,7 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) | |||
| 876 | } | 876 | } |
| 877 | } | 877 | } |
| 878 | 878 | ||
| 879 | #ifdef CONFIG_SMP | ||
| 879 | /************************************************** | 880 | /************************************************** |
| 880 | * Fair scheduling class load-balancing methods: | 881 | * Fair scheduling class load-balancing methods: |
| 881 | */ | 882 | */ |
| @@ -936,12 +937,11 @@ static int cfs_rq_best_prio(struct cfs_rq *cfs_rq) | |||
| 936 | 937 | ||
| 937 | static unsigned long | 938 | static unsigned long |
| 938 | load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, | 939 | load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, |
| 939 | unsigned long max_nr_move, unsigned long max_load_move, | 940 | unsigned long max_load_move, |
| 940 | struct sched_domain *sd, enum cpu_idle_type idle, | 941 | struct sched_domain *sd, enum cpu_idle_type idle, |
| 941 | int *all_pinned, int *this_best_prio) | 942 | int *all_pinned, int *this_best_prio) |
| 942 | { | 943 | { |
| 943 | struct cfs_rq *busy_cfs_rq; | 944 | struct cfs_rq *busy_cfs_rq; |
| 944 | unsigned long load_moved, total_nr_moved = 0, nr_moved; | ||
| 945 | long rem_load_move = max_load_move; | 945 | long rem_load_move = max_load_move; |
| 946 | struct rq_iterator cfs_rq_iterator; | 946 | struct rq_iterator cfs_rq_iterator; |
| 947 | 947 | ||
| @@ -969,25 +969,48 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, | |||
| 969 | #else | 969 | #else |
| 970 | # define maxload rem_load_move | 970 | # define maxload rem_load_move |
| 971 | #endif | 971 | #endif |
| 972 | /* pass busy_cfs_rq argument into | 972 | /* |
| 973 | * pass busy_cfs_rq argument into | ||
| 973 | * load_balance_[start|next]_fair iterators | 974 | * load_balance_[start|next]_fair iterators |
| 974 | */ | 975 | */ |
| 975 | cfs_rq_iterator.arg = busy_cfs_rq; | 976 | cfs_rq_iterator.arg = busy_cfs_rq; |
| 976 | nr_moved = balance_tasks(this_rq, this_cpu, busiest, | 977 | rem_load_move -= balance_tasks(this_rq, this_cpu, busiest, |
| 977 | max_nr_move, maxload, sd, idle, all_pinned, | 978 | maxload, sd, idle, all_pinned, |
| 978 | &load_moved, this_best_prio, &cfs_rq_iterator); | 979 | this_best_prio, |
| 979 | 980 | &cfs_rq_iterator); | |
| 980 | total_nr_moved += nr_moved; | ||
| 981 | max_nr_move -= nr_moved; | ||
| 982 | rem_load_move -= load_moved; | ||
| 983 | 981 | ||
| 984 | if (max_nr_move <= 0 || rem_load_move <= 0) | 982 | if (rem_load_move <= 0) |
| 985 | break; | 983 | break; |
| 986 | } | 984 | } |
| 987 | 985 | ||
| 988 | return max_load_move - rem_load_move; | 986 | return max_load_move - rem_load_move; |
| 989 | } | 987 | } |
| 990 | 988 | ||
| 989 | static int | ||
| 990 | move_one_task_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, | ||
| 991 | struct sched_domain *sd, enum cpu_idle_type idle) | ||
| 992 | { | ||
| 993 | struct cfs_rq *busy_cfs_rq; | ||
| 994 | struct rq_iterator cfs_rq_iterator; | ||
| 995 | |||
| 996 | cfs_rq_iterator.start = load_balance_start_fair; | ||
| 997 | cfs_rq_iterator.next = load_balance_next_fair; | ||
| 998 | |||
| 999 | for_each_leaf_cfs_rq(busiest, busy_cfs_rq) { | ||
| 1000 | /* | ||
| 1001 | * pass busy_cfs_rq argument into | ||
| 1002 | * load_balance_[start|next]_fair iterators | ||
| 1003 | */ | ||
| 1004 | cfs_rq_iterator.arg = busy_cfs_rq; | ||
| 1005 | if (iter_move_one_task(this_rq, this_cpu, busiest, sd, idle, | ||
| 1006 | &cfs_rq_iterator)) | ||
| 1007 | return 1; | ||
| 1008 | } | ||
| 1009 | |||
| 1010 | return 0; | ||
| 1011 | } | ||
| 1012 | #endif | ||
| 1013 | |||
| 991 | /* | 1014 | /* |
| 992 | * scheduler tick hitting a task of our scheduling class: | 1015 | * scheduler tick hitting a task of our scheduling class: |
| 993 | */ | 1016 | */ |
| @@ -1002,7 +1025,7 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr) | |||
| 1002 | } | 1025 | } |
| 1003 | } | 1026 | } |
| 1004 | 1027 | ||
| 1005 | #define swap(a,b) do { typeof(a) tmp = (a); (a) = (b); (b) = tmp; } while (0) | 1028 | #define swap(a, b) do { typeof(a) tmp = (a); (a) = (b); (b) = tmp; } while (0) |
| 1006 | 1029 | ||
| 1007 | /* | 1030 | /* |
| 1008 | * Share the fairness runtime between parent and child, thus the | 1031 | * Share the fairness runtime between parent and child, thus the |
| @@ -1063,7 +1086,10 @@ static const struct sched_class fair_sched_class = { | |||
| 1063 | .pick_next_task = pick_next_task_fair, | 1086 | .pick_next_task = pick_next_task_fair, |
| 1064 | .put_prev_task = put_prev_task_fair, | 1087 | .put_prev_task = put_prev_task_fair, |
| 1065 | 1088 | ||
| 1089 | #ifdef CONFIG_SMP | ||
| 1066 | .load_balance = load_balance_fair, | 1090 | .load_balance = load_balance_fair, |
| 1091 | .move_one_task = move_one_task_fair, | ||
| 1092 | #endif | ||
| 1067 | 1093 | ||
| 1068 | .set_curr_task = set_curr_task_fair, | 1094 | .set_curr_task = set_curr_task_fair, |
| 1069 | .task_tick = task_tick_fair, | 1095 | .task_tick = task_tick_fair, |
diff --git a/kernel/sched_idletask.c b/kernel/sched_idletask.c index 6e2ead41516e..bf9c25c15b8b 100644 --- a/kernel/sched_idletask.c +++ b/kernel/sched_idletask.c | |||
| @@ -37,15 +37,24 @@ static void put_prev_task_idle(struct rq *rq, struct task_struct *prev) | |||
| 37 | { | 37 | { |
| 38 | } | 38 | } |
| 39 | 39 | ||
| 40 | #ifdef CONFIG_SMP | ||
| 40 | static unsigned long | 41 | static unsigned long |
| 41 | load_balance_idle(struct rq *this_rq, int this_cpu, struct rq *busiest, | 42 | load_balance_idle(struct rq *this_rq, int this_cpu, struct rq *busiest, |
| 42 | unsigned long max_nr_move, unsigned long max_load_move, | 43 | unsigned long max_load_move, |
| 43 | struct sched_domain *sd, enum cpu_idle_type idle, | 44 | struct sched_domain *sd, enum cpu_idle_type idle, |
| 44 | int *all_pinned, int *this_best_prio) | 45 | int *all_pinned, int *this_best_prio) |
| 45 | { | 46 | { |
| 46 | return 0; | 47 | return 0; |
| 47 | } | 48 | } |
| 48 | 49 | ||
| 50 | static int | ||
| 51 | move_one_task_idle(struct rq *this_rq, int this_cpu, struct rq *busiest, | ||
| 52 | struct sched_domain *sd, enum cpu_idle_type idle) | ||
| 53 | { | ||
| 54 | return 0; | ||
| 55 | } | ||
| 56 | #endif | ||
| 57 | |||
| 49 | static void task_tick_idle(struct rq *rq, struct task_struct *curr) | 58 | static void task_tick_idle(struct rq *rq, struct task_struct *curr) |
| 50 | { | 59 | { |
| 51 | } | 60 | } |
| @@ -69,7 +78,10 @@ const struct sched_class idle_sched_class = { | |||
| 69 | .pick_next_task = pick_next_task_idle, | 78 | .pick_next_task = pick_next_task_idle, |
| 70 | .put_prev_task = put_prev_task_idle, | 79 | .put_prev_task = put_prev_task_idle, |
| 71 | 80 | ||
| 81 | #ifdef CONFIG_SMP | ||
| 72 | .load_balance = load_balance_idle, | 82 | .load_balance = load_balance_idle, |
| 83 | .move_one_task = move_one_task_idle, | ||
| 84 | #endif | ||
| 73 | 85 | ||
| 74 | .set_curr_task = set_curr_task_idle, | 86 | .set_curr_task = set_curr_task_idle, |
| 75 | .task_tick = task_tick_idle, | 87 | .task_tick = task_tick_idle, |
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index d0097a0634e5..8abd752a0ebd 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c | |||
| @@ -98,6 +98,7 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p) | |||
| 98 | p->se.exec_start = 0; | 98 | p->se.exec_start = 0; |
| 99 | } | 99 | } |
| 100 | 100 | ||
| 101 | #ifdef CONFIG_SMP | ||
| 101 | /* | 102 | /* |
| 102 | * Load-balancing iterator. Note: while the runqueue stays locked | 103 | * Load-balancing iterator. Note: while the runqueue stays locked |
| 103 | * during the whole iteration, the current task might be | 104 | * during the whole iteration, the current task might be |
| @@ -172,13 +173,11 @@ static struct task_struct *load_balance_next_rt(void *arg) | |||
| 172 | 173 | ||
| 173 | static unsigned long | 174 | static unsigned long |
| 174 | load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, | 175 | load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, |
| 175 | unsigned long max_nr_move, unsigned long max_load_move, | 176 | unsigned long max_load_move, |
| 176 | struct sched_domain *sd, enum cpu_idle_type idle, | 177 | struct sched_domain *sd, enum cpu_idle_type idle, |
| 177 | int *all_pinned, int *this_best_prio) | 178 | int *all_pinned, int *this_best_prio) |
| 178 | { | 179 | { |
| 179 | int nr_moved; | ||
| 180 | struct rq_iterator rt_rq_iterator; | 180 | struct rq_iterator rt_rq_iterator; |
| 181 | unsigned long load_moved; | ||
| 182 | 181 | ||
| 183 | rt_rq_iterator.start = load_balance_start_rt; | 182 | rt_rq_iterator.start = load_balance_start_rt; |
| 184 | rt_rq_iterator.next = load_balance_next_rt; | 183 | rt_rq_iterator.next = load_balance_next_rt; |
| @@ -187,12 +186,24 @@ load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, | |||
| 187 | */ | 186 | */ |
| 188 | rt_rq_iterator.arg = busiest; | 187 | rt_rq_iterator.arg = busiest; |
| 189 | 188 | ||
| 190 | nr_moved = balance_tasks(this_rq, this_cpu, busiest, max_nr_move, | 189 | return balance_tasks(this_rq, this_cpu, busiest, max_load_move, sd, |
| 191 | max_load_move, sd, idle, all_pinned, &load_moved, | 190 | idle, all_pinned, this_best_prio, &rt_rq_iterator); |
| 192 | this_best_prio, &rt_rq_iterator); | 191 | } |
| 192 | |||
| 193 | static int | ||
| 194 | move_one_task_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, | ||
| 195 | struct sched_domain *sd, enum cpu_idle_type idle) | ||
| 196 | { | ||
| 197 | struct rq_iterator rt_rq_iterator; | ||
| 198 | |||
| 199 | rt_rq_iterator.start = load_balance_start_rt; | ||
| 200 | rt_rq_iterator.next = load_balance_next_rt; | ||
| 201 | rt_rq_iterator.arg = busiest; | ||
| 193 | 202 | ||
| 194 | return load_moved; | 203 | return iter_move_one_task(this_rq, this_cpu, busiest, sd, idle, |
| 204 | &rt_rq_iterator); | ||
| 195 | } | 205 | } |
| 206 | #endif | ||
| 196 | 207 | ||
| 197 | static void task_tick_rt(struct rq *rq, struct task_struct *p) | 208 | static void task_tick_rt(struct rq *rq, struct task_struct *p) |
| 198 | { | 209 | { |
| @@ -236,7 +247,10 @@ const struct sched_class rt_sched_class = { | |||
| 236 | .pick_next_task = pick_next_task_rt, | 247 | .pick_next_task = pick_next_task_rt, |
| 237 | .put_prev_task = put_prev_task_rt, | 248 | .put_prev_task = put_prev_task_rt, |
| 238 | 249 | ||
| 250 | #ifdef CONFIG_SMP | ||
| 239 | .load_balance = load_balance_rt, | 251 | .load_balance = load_balance_rt, |
| 252 | .move_one_task = move_one_task_rt, | ||
| 253 | #endif | ||
| 240 | 254 | ||
| 241 | .set_curr_task = set_curr_task_rt, | 255 | .set_curr_task = set_curr_task_rt, |
| 242 | .task_tick = task_tick_rt, | 256 | .task_tick = task_tick_rt, |
diff --git a/kernel/signal.c b/kernel/signal.c index 12006308c7eb..909a0cc6bc70 100644 --- a/kernel/signal.c +++ b/kernel/signal.c | |||
| @@ -124,7 +124,7 @@ void recalc_sigpending_and_wake(struct task_struct *t) | |||
| 124 | 124 | ||
| 125 | void recalc_sigpending(void) | 125 | void recalc_sigpending(void) |
| 126 | { | 126 | { |
| 127 | if (!recalc_sigpending_tsk(current)) | 127 | if (!recalc_sigpending_tsk(current) && !freezing(current)) |
| 128 | clear_thread_flag(TIF_SIGPENDING); | 128 | clear_thread_flag(TIF_SIGPENDING); |
| 129 | 129 | ||
| 130 | } | 130 | } |
| @@ -732,7 +732,7 @@ static void print_fatal_signal(struct pt_regs *regs, int signr) | |||
| 732 | printk("%s/%d: potentially unexpected fatal signal %d.\n", | 732 | printk("%s/%d: potentially unexpected fatal signal %d.\n", |
| 733 | current->comm, task_pid_nr(current), signr); | 733 | current->comm, task_pid_nr(current), signr); |
| 734 | 734 | ||
| 735 | #ifdef __i386__ | 735 | #if defined(__i386__) && !defined(__arch_um__) |
| 736 | printk("code at %08lx: ", regs->eip); | 736 | printk("code at %08lx: ", regs->eip); |
| 737 | { | 737 | { |
| 738 | int i; | 738 | int i; |
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index 52c7a151e298..56cb009a4b35 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c | |||
| @@ -40,10 +40,14 @@ cond_syscall(sys_recvfrom); | |||
| 40 | cond_syscall(sys_recv); | 40 | cond_syscall(sys_recv); |
| 41 | cond_syscall(sys_socket); | 41 | cond_syscall(sys_socket); |
| 42 | cond_syscall(sys_setsockopt); | 42 | cond_syscall(sys_setsockopt); |
| 43 | cond_syscall(compat_sys_setsockopt); | ||
| 43 | cond_syscall(sys_getsockopt); | 44 | cond_syscall(sys_getsockopt); |
| 45 | cond_syscall(compat_sys_getsockopt); | ||
| 44 | cond_syscall(sys_shutdown); | 46 | cond_syscall(sys_shutdown); |
| 45 | cond_syscall(sys_sendmsg); | 47 | cond_syscall(sys_sendmsg); |
| 48 | cond_syscall(compat_sys_sendmsg); | ||
| 46 | cond_syscall(sys_recvmsg); | 49 | cond_syscall(sys_recvmsg); |
| 50 | cond_syscall(compat_sys_recvmsg); | ||
| 47 | cond_syscall(sys_socketcall); | 51 | cond_syscall(sys_socketcall); |
| 48 | cond_syscall(sys_futex); | 52 | cond_syscall(sys_futex); |
| 49 | cond_syscall(compat_sys_futex); | 53 | cond_syscall(compat_sys_futex); |
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 10a1347597fd..5997456ebbc9 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
| @@ -320,8 +320,6 @@ ktime_t tick_nohz_get_sleep_length(void) | |||
| 320 | return ts->sleep_length; | 320 | return ts->sleep_length; |
| 321 | } | 321 | } |
| 322 | 322 | ||
| 323 | EXPORT_SYMBOL_GPL(tick_nohz_get_sleep_length); | ||
| 324 | |||
| 325 | /** | 323 | /** |
| 326 | * nohz_restart_sched_tick - restart the idle tick from the idle task | 324 | * nohz_restart_sched_tick - restart the idle tick from the idle task |
| 327 | * | 325 | * |
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index fdb2e03d4fe0..12c5f4cb6b8c 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c | |||
| @@ -129,7 +129,8 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now) | |||
| 129 | struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); | 129 | struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); |
| 130 | int i; | 130 | int i; |
| 131 | 131 | ||
| 132 | SEQ_printf(m, "\ncpu: %d\n", cpu); | 132 | SEQ_printf(m, "\n"); |
| 133 | SEQ_printf(m, "cpu: %d\n", cpu); | ||
| 133 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { | 134 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { |
| 134 | SEQ_printf(m, " clock %d:\n", i); | 135 | SEQ_printf(m, " clock %d:\n", i); |
| 135 | print_base(m, cpu_base->clock_base + i, now); | 136 | print_base(m, cpu_base->clock_base + i, now); |
| @@ -184,7 +185,8 @@ print_tickdevice(struct seq_file *m, struct tick_device *td) | |||
| 184 | { | 185 | { |
| 185 | struct clock_event_device *dev = td->evtdev; | 186 | struct clock_event_device *dev = td->evtdev; |
| 186 | 187 | ||
| 187 | SEQ_printf(m, "\nTick Device: mode: %d\n", td->mode); | 188 | SEQ_printf(m, "\n"); |
| 189 | SEQ_printf(m, "Tick Device: mode: %d\n", td->mode); | ||
| 188 | 190 | ||
| 189 | SEQ_printf(m, "Clock Event Device: "); | 191 | SEQ_printf(m, "Clock Event Device: "); |
| 190 | if (!dev) { | 192 | if (!dev) { |
diff --git a/kernel/user.c b/kernel/user.c index e91331c457e2..0f3aa0234107 100644 --- a/kernel/user.c +++ b/kernel/user.c | |||
| @@ -129,7 +129,7 @@ static inline void uids_mutex_unlock(void) | |||
| 129 | } | 129 | } |
| 130 | 130 | ||
| 131 | /* return cpu shares held by the user */ | 131 | /* return cpu shares held by the user */ |
| 132 | ssize_t cpu_shares_show(struct kset *kset, char *buffer) | 132 | static ssize_t cpu_shares_show(struct kset *kset, char *buffer) |
| 133 | { | 133 | { |
| 134 | struct user_struct *up = container_of(kset, struct user_struct, kset); | 134 | struct user_struct *up = container_of(kset, struct user_struct, kset); |
| 135 | 135 | ||
| @@ -137,7 +137,8 @@ ssize_t cpu_shares_show(struct kset *kset, char *buffer) | |||
| 137 | } | 137 | } |
| 138 | 138 | ||
| 139 | /* modify cpu shares held by the user */ | 139 | /* modify cpu shares held by the user */ |
| 140 | ssize_t cpu_shares_store(struct kset *kset, const char *buffer, size_t size) | 140 | static ssize_t cpu_shares_store(struct kset *kset, const char *buffer, |
| 141 | size_t size) | ||
| 141 | { | 142 | { |
| 142 | struct user_struct *up = container_of(kset, struct user_struct, kset); | 143 | struct user_struct *up = container_of(kset, struct user_struct, kset); |
| 143 | unsigned long shares; | 144 | unsigned long shares; |
