aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/kernel-parameters.txt3
-rw-r--r--include/linux/completion.h18
-rw-r--r--include/linux/sched.h37
-rw-r--r--init/Kconfig1
-rw-r--r--kernel/profile.c5
-rw-r--r--kernel/sched.c330
-rw-r--r--kernel/sched_fair.c48
-rw-r--r--kernel/sched_idletask.c18
-rw-r--r--kernel/sched_rt.c32
-rw-r--r--kernel/user.c5
10 files changed, 292 insertions, 205 deletions
diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt
index a13d69b2217d..8ae5fac08dfa 100644
--- a/Documentation/kernel-parameters.txt
+++ b/Documentation/kernel-parameters.txt
@@ -1444,7 +1444,8 @@ and is between 256 and 4096 characters. It is defined in the file
1444 Param: "schedule" - profile schedule points. 1444 Param: "schedule" - profile schedule points.
1445 Param: <number> - step/bucket size as a power of 2 for 1445 Param: <number> - step/bucket size as a power of 2 for
1446 statistical time based profiling. 1446 statistical time based profiling.
1447 Param: "sleep" - profile D-state sleeping (millisecs) 1447 Param: "sleep" - profile D-state sleeping (millisecs).
1448 Requires CONFIG_SCHEDSTATS
1448 Param: "kvm" - profile VM exits. 1449 Param: "kvm" - profile VM exits.
1449 1450
1450 processor.max_cstate= [HW,ACPI] 1451 processor.max_cstate= [HW,ACPI]
diff --git a/include/linux/completion.h b/include/linux/completion.h
index 268c5a4a2bd4..33d6aaf94447 100644
--- a/include/linux/completion.h
+++ b/include/linux/completion.h
@@ -42,15 +42,15 @@ static inline void init_completion(struct completion *x)
42 init_waitqueue_head(&x->wait); 42 init_waitqueue_head(&x->wait);
43} 43}
44 44
45extern void FASTCALL(wait_for_completion(struct completion *)); 45extern void wait_for_completion(struct completion *);
46extern int FASTCALL(wait_for_completion_interruptible(struct completion *x)); 46extern int wait_for_completion_interruptible(struct completion *x);
47extern unsigned long FASTCALL(wait_for_completion_timeout(struct completion *x, 47extern unsigned long wait_for_completion_timeout(struct completion *x,
48 unsigned long timeout)); 48 unsigned long timeout);
49extern unsigned long FASTCALL(wait_for_completion_interruptible_timeout( 49extern unsigned long wait_for_completion_interruptible_timeout(
50 struct completion *x, unsigned long timeout)); 50 struct completion *x, unsigned long timeout);
51 51
52extern void FASTCALL(complete(struct completion *)); 52extern void complete(struct completion *);
53extern void FASTCALL(complete_all(struct completion *)); 53extern void complete_all(struct completion *);
54 54
55#define INIT_COMPLETION(x) ((x).done = 0) 55#define INIT_COMPLETION(x) ((x).done = 0)
56 56
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 13df99fb2769..24e08d1d900d 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -828,12 +828,17 @@ struct sched_class {
828 struct task_struct * (*pick_next_task) (struct rq *rq); 828 struct task_struct * (*pick_next_task) (struct rq *rq);
829 void (*put_prev_task) (struct rq *rq, struct task_struct *p); 829 void (*put_prev_task) (struct rq *rq, struct task_struct *p);
830 830
831#ifdef CONFIG_SMP
831 unsigned long (*load_balance) (struct rq *this_rq, int this_cpu, 832 unsigned long (*load_balance) (struct rq *this_rq, int this_cpu,
832 struct rq *busiest, 833 struct rq *busiest, unsigned long max_load_move,
833 unsigned long max_nr_move, unsigned long max_load_move,
834 struct sched_domain *sd, enum cpu_idle_type idle, 834 struct sched_domain *sd, enum cpu_idle_type idle,
835 int *all_pinned, int *this_best_prio); 835 int *all_pinned, int *this_best_prio);
836 836
837 int (*move_one_task) (struct rq *this_rq, int this_cpu,
838 struct rq *busiest, struct sched_domain *sd,
839 enum cpu_idle_type idle);
840#endif
841
837 void (*set_curr_task) (struct rq *rq); 842 void (*set_curr_task) (struct rq *rq);
838 void (*task_tick) (struct rq *rq, struct task_struct *p); 843 void (*task_tick) (struct rq *rq, struct task_struct *p);
839 void (*task_new) (struct rq *rq, struct task_struct *p); 844 void (*task_new) (struct rq *rq, struct task_struct *p);
@@ -1196,7 +1201,7 @@ static inline int rt_prio(int prio)
1196 return 0; 1201 return 0;
1197} 1202}
1198 1203
1199static inline int rt_task(struct task_struct *p) 1204static inline int rt_task(const struct task_struct *p)
1200{ 1205{
1201 return rt_prio(p->prio); 1206 return rt_prio(p->prio);
1202} 1207}
@@ -1211,22 +1216,22 @@ static inline void set_task_pgrp(struct task_struct *tsk, pid_t pgrp)
1211 tsk->signal->__pgrp = pgrp; 1216 tsk->signal->__pgrp = pgrp;
1212} 1217}
1213 1218
1214static inline struct pid *task_pid(struct task_struct *task) 1219static inline struct pid *task_pid(const struct task_struct *task)
1215{ 1220{
1216 return task->pids[PIDTYPE_PID].pid; 1221 return task->pids[PIDTYPE_PID].pid;
1217} 1222}
1218 1223
1219static inline struct pid *task_tgid(struct task_struct *task) 1224static inline struct pid *task_tgid(const struct task_struct *task)
1220{ 1225{
1221 return task->group_leader->pids[PIDTYPE_PID].pid; 1226 return task->group_leader->pids[PIDTYPE_PID].pid;
1222} 1227}
1223 1228
1224static inline struct pid *task_pgrp(struct task_struct *task) 1229static inline struct pid *task_pgrp(const struct task_struct *task)
1225{ 1230{
1226 return task->group_leader->pids[PIDTYPE_PGID].pid; 1231 return task->group_leader->pids[PIDTYPE_PGID].pid;
1227} 1232}
1228 1233
1229static inline struct pid *task_session(struct task_struct *task) 1234static inline struct pid *task_session(const struct task_struct *task)
1230{ 1235{
1231 return task->group_leader->pids[PIDTYPE_SID].pid; 1236 return task->group_leader->pids[PIDTYPE_SID].pid;
1232} 1237}
@@ -1255,7 +1260,7 @@ struct pid_namespace;
1255 * see also pid_nr() etc in include/linux/pid.h 1260 * see also pid_nr() etc in include/linux/pid.h
1256 */ 1261 */
1257 1262
1258static inline pid_t task_pid_nr(struct task_struct *tsk) 1263static inline pid_t task_pid_nr(const struct task_struct *tsk)
1259{ 1264{
1260 return tsk->pid; 1265 return tsk->pid;
1261} 1266}
@@ -1268,7 +1273,7 @@ static inline pid_t task_pid_vnr(struct task_struct *tsk)
1268} 1273}
1269 1274
1270 1275
1271static inline pid_t task_tgid_nr(struct task_struct *tsk) 1276static inline pid_t task_tgid_nr(const struct task_struct *tsk)
1272{ 1277{
1273 return tsk->tgid; 1278 return tsk->tgid;
1274} 1279}
@@ -1281,7 +1286,7 @@ static inline pid_t task_tgid_vnr(struct task_struct *tsk)
1281} 1286}
1282 1287
1283 1288
1284static inline pid_t task_pgrp_nr(struct task_struct *tsk) 1289static inline pid_t task_pgrp_nr(const struct task_struct *tsk)
1285{ 1290{
1286 return tsk->signal->__pgrp; 1291 return tsk->signal->__pgrp;
1287} 1292}
@@ -1294,7 +1299,7 @@ static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
1294} 1299}
1295 1300
1296 1301
1297static inline pid_t task_session_nr(struct task_struct *tsk) 1302static inline pid_t task_session_nr(const struct task_struct *tsk)
1298{ 1303{
1299 return tsk->signal->__session; 1304 return tsk->signal->__session;
1300} 1305}
@@ -1321,7 +1326,7 @@ static inline pid_t task_ppid_nr_ns(struct task_struct *tsk,
1321 * If pid_alive fails, then pointers within the task structure 1326 * If pid_alive fails, then pointers within the task structure
1322 * can be stale and must not be dereferenced. 1327 * can be stale and must not be dereferenced.
1323 */ 1328 */
1324static inline int pid_alive(struct task_struct *p) 1329static inline int pid_alive(const struct task_struct *p)
1325{ 1330{
1326 return p->pids[PIDTYPE_PID].pid != NULL; 1331 return p->pids[PIDTYPE_PID].pid != NULL;
1327} 1332}
@@ -1332,7 +1337,7 @@ static inline int pid_alive(struct task_struct *p)
1332 * 1337 *
1333 * Check if a task structure is the first user space task the kernel created. 1338 * Check if a task structure is the first user space task the kernel created.
1334 */ 1339 */
1335static inline int is_global_init(struct task_struct *tsk) 1340static inline int is_global_init(const struct task_struct *tsk)
1336{ 1341{
1337 return tsk->pid == 1; 1342 return tsk->pid == 1;
1338} 1343}
@@ -1469,7 +1474,7 @@ extern int rt_mutex_getprio(struct task_struct *p);
1469extern void rt_mutex_setprio(struct task_struct *p, int prio); 1474extern void rt_mutex_setprio(struct task_struct *p, int prio);
1470extern void rt_mutex_adjust_pi(struct task_struct *p); 1475extern void rt_mutex_adjust_pi(struct task_struct *p);
1471#else 1476#else
1472static inline int rt_mutex_getprio(struct task_struct *p) 1477static inline int rt_mutex_getprio(const struct task_struct *p)
1473{ 1478{
1474 return p->normal_prio; 1479 return p->normal_prio;
1475} 1480}
@@ -1721,7 +1726,7 @@ extern void wait_task_inactive(struct task_struct * p);
1721 * all we care about is that we have a task with the appropriate 1726 * all we care about is that we have a task with the appropriate
1722 * pid, we don't actually care if we have the right task. 1727 * pid, we don't actually care if we have the right task.
1723 */ 1728 */
1724static inline int has_group_leader_pid(struct task_struct *p) 1729static inline int has_group_leader_pid(const struct task_struct *p)
1725{ 1730{
1726 return p->pid == p->tgid; 1731 return p->pid == p->tgid;
1727} 1732}
@@ -1738,7 +1743,7 @@ static inline struct task_struct *next_thread(const struct task_struct *p)
1738 struct task_struct, thread_group); 1743 struct task_struct, thread_group);
1739} 1744}
1740 1745
1741static inline int thread_group_empty(struct task_struct *p) 1746static inline int thread_group_empty(const struct task_struct *p)
1742{ 1747{
1743 return list_empty(&p->thread_group); 1748 return list_empty(&p->thread_group);
1744} 1749}
diff --git a/init/Kconfig b/init/Kconfig
index b7dffa837926..8b88d0bedcbd 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -322,7 +322,6 @@ config CPUSETS
322config FAIR_GROUP_SCHED 322config FAIR_GROUP_SCHED
323 bool "Fair group CPU scheduler" 323 bool "Fair group CPU scheduler"
324 default y 324 default y
325 depends on EXPERIMENTAL
326 help 325 help
327 This feature lets CPU scheduler recognize task groups and control CPU 326 This feature lets CPU scheduler recognize task groups and control CPU
328 bandwidth allocation to such task groups. 327 bandwidth allocation to such task groups.
diff --git a/kernel/profile.c b/kernel/profile.c
index 631b75c25d7e..5e95330e5120 100644
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -60,6 +60,7 @@ static int __init profile_setup(char * str)
60 int par; 60 int par;
61 61
62 if (!strncmp(str, sleepstr, strlen(sleepstr))) { 62 if (!strncmp(str, sleepstr, strlen(sleepstr))) {
63#ifdef CONFIG_SCHEDSTATS
63 prof_on = SLEEP_PROFILING; 64 prof_on = SLEEP_PROFILING;
64 if (str[strlen(sleepstr)] == ',') 65 if (str[strlen(sleepstr)] == ',')
65 str += strlen(sleepstr) + 1; 66 str += strlen(sleepstr) + 1;
@@ -68,6 +69,10 @@ static int __init profile_setup(char * str)
68 printk(KERN_INFO 69 printk(KERN_INFO
69 "kernel sleep profiling enabled (shift: %ld)\n", 70 "kernel sleep profiling enabled (shift: %ld)\n",
70 prof_shift); 71 prof_shift);
72#else
73 printk(KERN_WARNING
74 "kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
75#endif /* CONFIG_SCHEDSTATS */
71 } else if (!strncmp(str, schedstr, strlen(schedstr))) { 76 } else if (!strncmp(str, schedstr, strlen(schedstr))) {
72 prof_on = SCHED_PROFILING; 77 prof_on = SCHED_PROFILING;
73 if (str[strlen(schedstr)] == ',') 78 if (str[strlen(schedstr)] == ',')
diff --git a/kernel/sched.c b/kernel/sched.c
index 2810e562a991..b4fbbc440453 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -66,6 +66,7 @@
66#include <linux/pagemap.h> 66#include <linux/pagemap.h>
67 67
68#include <asm/tlb.h> 68#include <asm/tlb.h>
69#include <asm/irq_regs.h>
69 70
70/* 71/*
71 * Scheduler clock - returns current time in nanosec units. 72 * Scheduler clock - returns current time in nanosec units.
@@ -837,11 +838,18 @@ struct rq_iterator {
837 struct task_struct *(*next)(void *); 838 struct task_struct *(*next)(void *);
838}; 839};
839 840
840static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, 841#ifdef CONFIG_SMP
841 unsigned long max_nr_move, unsigned long max_load_move, 842static unsigned long
842 struct sched_domain *sd, enum cpu_idle_type idle, 843balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
843 int *all_pinned, unsigned long *load_moved, 844 unsigned long max_load_move, struct sched_domain *sd,
844 int *this_best_prio, struct rq_iterator *iterator); 845 enum cpu_idle_type idle, int *all_pinned,
846 int *this_best_prio, struct rq_iterator *iterator);
847
848static int
849iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
850 struct sched_domain *sd, enum cpu_idle_type idle,
851 struct rq_iterator *iterator);
852#endif
845 853
846#include "sched_stats.h" 854#include "sched_stats.h"
847#include "sched_idletask.c" 855#include "sched_idletask.c"
@@ -2223,17 +2231,17 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
2223 return 1; 2231 return 1;
2224} 2232}
2225 2233
2226static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, 2234static unsigned long
2227 unsigned long max_nr_move, unsigned long max_load_move, 2235balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2228 struct sched_domain *sd, enum cpu_idle_type idle, 2236 unsigned long max_load_move, struct sched_domain *sd,
2229 int *all_pinned, unsigned long *load_moved, 2237 enum cpu_idle_type idle, int *all_pinned,
2230 int *this_best_prio, struct rq_iterator *iterator) 2238 int *this_best_prio, struct rq_iterator *iterator)
2231{ 2239{
2232 int pulled = 0, pinned = 0, skip_for_load; 2240 int pulled = 0, pinned = 0, skip_for_load;
2233 struct task_struct *p; 2241 struct task_struct *p;
2234 long rem_load_move = max_load_move; 2242 long rem_load_move = max_load_move;
2235 2243
2236 if (max_nr_move == 0 || max_load_move == 0) 2244 if (max_load_move == 0)
2237 goto out; 2245 goto out;
2238 2246
2239 pinned = 1; 2247 pinned = 1;
@@ -2266,7 +2274,7 @@ next:
2266 * We only want to steal up to the prescribed number of tasks 2274 * We only want to steal up to the prescribed number of tasks
2267 * and the prescribed amount of weighted load. 2275 * and the prescribed amount of weighted load.
2268 */ 2276 */
2269 if (pulled < max_nr_move && rem_load_move > 0) { 2277 if (rem_load_move > 0) {
2270 if (p->prio < *this_best_prio) 2278 if (p->prio < *this_best_prio)
2271 *this_best_prio = p->prio; 2279 *this_best_prio = p->prio;
2272 p = iterator->next(iterator->arg); 2280 p = iterator->next(iterator->arg);
@@ -2274,7 +2282,7 @@ next:
2274 } 2282 }
2275out: 2283out:
2276 /* 2284 /*
2277 * Right now, this is the only place pull_task() is called, 2285 * Right now, this is one of only two places pull_task() is called,
2278 * so we can safely collect pull_task() stats here rather than 2286 * so we can safely collect pull_task() stats here rather than
2279 * inside pull_task(). 2287 * inside pull_task().
2280 */ 2288 */
@@ -2282,8 +2290,8 @@ out:
2282 2290
2283 if (all_pinned) 2291 if (all_pinned)
2284 *all_pinned = pinned; 2292 *all_pinned = pinned;
2285 *load_moved = max_load_move - rem_load_move; 2293
2286 return pulled; 2294 return max_load_move - rem_load_move;
2287} 2295}
2288 2296
2289/* 2297/*
@@ -2305,7 +2313,7 @@ static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2305 do { 2313 do {
2306 total_load_moved += 2314 total_load_moved +=
2307 class->load_balance(this_rq, this_cpu, busiest, 2315 class->load_balance(this_rq, this_cpu, busiest,
2308 ULONG_MAX, max_load_move - total_load_moved, 2316 max_load_move - total_load_moved,
2309 sd, idle, all_pinned, &this_best_prio); 2317 sd, idle, all_pinned, &this_best_prio);
2310 class = class->next; 2318 class = class->next;
2311 } while (class && max_load_move > total_load_moved); 2319 } while (class && max_load_move > total_load_moved);
@@ -2313,6 +2321,32 @@ static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2313 return total_load_moved > 0; 2321 return total_load_moved > 0;
2314} 2322}
2315 2323
2324static int
2325iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
2326 struct sched_domain *sd, enum cpu_idle_type idle,
2327 struct rq_iterator *iterator)
2328{
2329 struct task_struct *p = iterator->start(iterator->arg);
2330 int pinned = 0;
2331
2332 while (p) {
2333 if (can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
2334 pull_task(busiest, p, this_rq, this_cpu);
2335 /*
2336 * Right now, this is only the second place pull_task()
2337 * is called, so we can safely collect pull_task()
2338 * stats here rather than inside pull_task().
2339 */
2340 schedstat_inc(sd, lb_gained[idle]);
2341
2342 return 1;
2343 }
2344 p = iterator->next(iterator->arg);
2345 }
2346
2347 return 0;
2348}
2349
2316/* 2350/*
2317 * move_one_task tries to move exactly one task from busiest to this_rq, as 2351 * move_one_task tries to move exactly one task from busiest to this_rq, as
2318 * part of active balancing operations within "domain". 2352 * part of active balancing operations within "domain".
@@ -2324,12 +2358,9 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
2324 struct sched_domain *sd, enum cpu_idle_type idle) 2358 struct sched_domain *sd, enum cpu_idle_type idle)
2325{ 2359{
2326 const struct sched_class *class; 2360 const struct sched_class *class;
2327 int this_best_prio = MAX_PRIO;
2328 2361
2329 for (class = sched_class_highest; class; class = class->next) 2362 for (class = sched_class_highest; class; class = class->next)
2330 if (class->load_balance(this_rq, this_cpu, busiest, 2363 if (class->move_one_task(this_rq, this_cpu, busiest, sd, idle))
2331 1, ULONG_MAX, sd, idle, NULL,
2332 &this_best_prio))
2333 return 1; 2364 return 1;
2334 2365
2335 return 0; 2366 return 0;
@@ -3266,18 +3297,6 @@ static inline void idle_balance(int cpu, struct rq *rq)
3266{ 3297{
3267} 3298}
3268 3299
3269/* Avoid "used but not defined" warning on UP */
3270static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
3271 unsigned long max_nr_move, unsigned long max_load_move,
3272 struct sched_domain *sd, enum cpu_idle_type idle,
3273 int *all_pinned, unsigned long *load_moved,
3274 int *this_best_prio, struct rq_iterator *iterator)
3275{
3276 *load_moved = 0;
3277
3278 return 0;
3279}
3280
3281#endif 3300#endif
3282 3301
3283DEFINE_PER_CPU(struct kernel_stat, kstat); 3302DEFINE_PER_CPU(struct kernel_stat, kstat);
@@ -3507,12 +3526,19 @@ EXPORT_SYMBOL(sub_preempt_count);
3507 */ 3526 */
3508static noinline void __schedule_bug(struct task_struct *prev) 3527static noinline void __schedule_bug(struct task_struct *prev)
3509{ 3528{
3510 printk(KERN_ERR "BUG: scheduling while atomic: %s/0x%08x/%d\n", 3529 struct pt_regs *regs = get_irq_regs();
3511 prev->comm, preempt_count(), task_pid_nr(prev)); 3530
3531 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
3532 prev->comm, prev->pid, preempt_count());
3533
3512 debug_show_held_locks(prev); 3534 debug_show_held_locks(prev);
3513 if (irqs_disabled()) 3535 if (irqs_disabled())
3514 print_irqtrace_events(prev); 3536 print_irqtrace_events(prev);
3515 dump_stack(); 3537
3538 if (regs)
3539 show_regs(regs);
3540 else
3541 dump_stack();
3516} 3542}
3517 3543
3518/* 3544/*
@@ -3820,7 +3846,7 @@ __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
3820} 3846}
3821EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */ 3847EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */
3822 3848
3823void fastcall complete(struct completion *x) 3849void complete(struct completion *x)
3824{ 3850{
3825 unsigned long flags; 3851 unsigned long flags;
3826 3852
@@ -3832,7 +3858,7 @@ void fastcall complete(struct completion *x)
3832} 3858}
3833EXPORT_SYMBOL(complete); 3859EXPORT_SYMBOL(complete);
3834 3860
3835void fastcall complete_all(struct completion *x) 3861void complete_all(struct completion *x)
3836{ 3862{
3837 unsigned long flags; 3863 unsigned long flags;
3838 3864
@@ -3884,13 +3910,13 @@ wait_for_common(struct completion *x, long timeout, int state)
3884 return timeout; 3910 return timeout;
3885} 3911}
3886 3912
3887void fastcall __sched wait_for_completion(struct completion *x) 3913void __sched wait_for_completion(struct completion *x)
3888{ 3914{
3889 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); 3915 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
3890} 3916}
3891EXPORT_SYMBOL(wait_for_completion); 3917EXPORT_SYMBOL(wait_for_completion);
3892 3918
3893unsigned long fastcall __sched 3919unsigned long __sched
3894wait_for_completion_timeout(struct completion *x, unsigned long timeout) 3920wait_for_completion_timeout(struct completion *x, unsigned long timeout)
3895{ 3921{
3896 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); 3922 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
@@ -3906,7 +3932,7 @@ int __sched wait_for_completion_interruptible(struct completion *x)
3906} 3932}
3907EXPORT_SYMBOL(wait_for_completion_interruptible); 3933EXPORT_SYMBOL(wait_for_completion_interruptible);
3908 3934
3909unsigned long fastcall __sched 3935unsigned long __sched
3910wait_for_completion_interruptible_timeout(struct completion *x, 3936wait_for_completion_interruptible_timeout(struct completion *x,
3911 unsigned long timeout) 3937 unsigned long timeout)
3912{ 3938{
@@ -5461,11 +5487,12 @@ static void register_sched_domain_sysctl(void)
5461 struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1); 5487 struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
5462 char buf[32]; 5488 char buf[32];
5463 5489
5490 WARN_ON(sd_ctl_dir[0].child);
5491 sd_ctl_dir[0].child = entry;
5492
5464 if (entry == NULL) 5493 if (entry == NULL)
5465 return; 5494 return;
5466 5495
5467 sd_ctl_dir[0].child = entry;
5468
5469 for_each_online_cpu(i) { 5496 for_each_online_cpu(i) {
5470 snprintf(buf, 32, "cpu%d", i); 5497 snprintf(buf, 32, "cpu%d", i);
5471 entry->procname = kstrdup(buf, GFP_KERNEL); 5498 entry->procname = kstrdup(buf, GFP_KERNEL);
@@ -5473,14 +5500,19 @@ static void register_sched_domain_sysctl(void)
5473 entry->child = sd_alloc_ctl_cpu_table(i); 5500 entry->child = sd_alloc_ctl_cpu_table(i);
5474 entry++; 5501 entry++;
5475 } 5502 }
5503
5504 WARN_ON(sd_sysctl_header);
5476 sd_sysctl_header = register_sysctl_table(sd_ctl_root); 5505 sd_sysctl_header = register_sysctl_table(sd_ctl_root);
5477} 5506}
5478 5507
5508/* may be called multiple times per register */
5479static void unregister_sched_domain_sysctl(void) 5509static void unregister_sched_domain_sysctl(void)
5480{ 5510{
5481 unregister_sysctl_table(sd_sysctl_header); 5511 if (sd_sysctl_header)
5512 unregister_sysctl_table(sd_sysctl_header);
5482 sd_sysctl_header = NULL; 5513 sd_sysctl_header = NULL;
5483 sd_free_ctl_entry(&sd_ctl_dir[0].child); 5514 if (sd_ctl_dir[0].child)
5515 sd_free_ctl_entry(&sd_ctl_dir[0].child);
5484} 5516}
5485#else 5517#else
5486static void register_sched_domain_sysctl(void) 5518static void register_sched_domain_sysctl(void)
@@ -5611,101 +5643,101 @@ int nr_cpu_ids __read_mostly = NR_CPUS;
5611EXPORT_SYMBOL(nr_cpu_ids); 5643EXPORT_SYMBOL(nr_cpu_ids);
5612 5644
5613#ifdef CONFIG_SCHED_DEBUG 5645#ifdef CONFIG_SCHED_DEBUG
5614static void sched_domain_debug(struct sched_domain *sd, int cpu) 5646
5647static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level)
5615{ 5648{
5616 int level = 0; 5649 struct sched_group *group = sd->groups;
5650 cpumask_t groupmask;
5651 char str[NR_CPUS];
5617 5652
5618 if (!sd) { 5653 cpumask_scnprintf(str, NR_CPUS, sd->span);
5619 printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu); 5654 cpus_clear(groupmask);
5620 return; 5655
5656 printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
5657
5658 if (!(sd->flags & SD_LOAD_BALANCE)) {
5659 printk("does not load-balance\n");
5660 if (sd->parent)
5661 printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
5662 " has parent");
5663 return -1;
5621 } 5664 }
5622 5665
5623 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); 5666 printk(KERN_CONT "span %s\n", str);
5667
5668 if (!cpu_isset(cpu, sd->span)) {
5669 printk(KERN_ERR "ERROR: domain->span does not contain "
5670 "CPU%d\n", cpu);
5671 }
5672 if (!cpu_isset(cpu, group->cpumask)) {
5673 printk(KERN_ERR "ERROR: domain->groups does not contain"
5674 " CPU%d\n", cpu);
5675 }
5624 5676
5677 printk(KERN_DEBUG "%*s groups:", level + 1, "");
5625 do { 5678 do {
5626 int i; 5679 if (!group) {
5627 char str[NR_CPUS]; 5680 printk("\n");
5628 struct sched_group *group = sd->groups; 5681 printk(KERN_ERR "ERROR: group is NULL\n");
5629 cpumask_t groupmask;
5630
5631 cpumask_scnprintf(str, NR_CPUS, sd->span);
5632 cpus_clear(groupmask);
5633
5634 printk(KERN_DEBUG);
5635 for (i = 0; i < level + 1; i++)
5636 printk(" ");
5637 printk("domain %d: ", level);
5638
5639 if (!(sd->flags & SD_LOAD_BALANCE)) {
5640 printk("does not load-balance\n");
5641 if (sd->parent)
5642 printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
5643 " has parent");
5644 break; 5682 break;
5645 } 5683 }
5646 5684
5647 printk("span %s\n", str); 5685 if (!group->__cpu_power) {
5686 printk(KERN_CONT "\n");
5687 printk(KERN_ERR "ERROR: domain->cpu_power not "
5688 "set\n");
5689 break;
5690 }
5648 5691
5649 if (!cpu_isset(cpu, sd->span)) 5692 if (!cpus_weight(group->cpumask)) {
5650 printk(KERN_ERR "ERROR: domain->span does not contain " 5693 printk(KERN_CONT "\n");
5651 "CPU%d\n", cpu); 5694 printk(KERN_ERR "ERROR: empty group\n");
5652 if (!cpu_isset(cpu, group->cpumask)) 5695 break;
5653 printk(KERN_ERR "ERROR: domain->groups does not contain" 5696 }
5654 " CPU%d\n", cpu);
5655 5697
5656 printk(KERN_DEBUG); 5698 if (cpus_intersects(groupmask, group->cpumask)) {
5657 for (i = 0; i < level + 2; i++) 5699 printk(KERN_CONT "\n");
5658 printk(" "); 5700 printk(KERN_ERR "ERROR: repeated CPUs\n");
5659 printk("groups:"); 5701 break;
5660 do { 5702 }
5661 if (!group) {
5662 printk("\n");
5663 printk(KERN_ERR "ERROR: group is NULL\n");
5664 break;
5665 }
5666 5703
5667 if (!group->__cpu_power) { 5704 cpus_or(groupmask, groupmask, group->cpumask);
5668 printk(KERN_CONT "\n");
5669 printk(KERN_ERR "ERROR: domain->cpu_power not "
5670 "set\n");
5671 break;
5672 }
5673 5705
5674 if (!cpus_weight(group->cpumask)) { 5706 cpumask_scnprintf(str, NR_CPUS, group->cpumask);
5675 printk(KERN_CONT "\n"); 5707 printk(KERN_CONT " %s", str);
5676 printk(KERN_ERR "ERROR: empty group\n");
5677 break;
5678 }
5679 5708
5680 if (cpus_intersects(groupmask, group->cpumask)) { 5709 group = group->next;
5681 printk(KERN_CONT "\n"); 5710 } while (group != sd->groups);
5682 printk(KERN_ERR "ERROR: repeated CPUs\n"); 5711 printk(KERN_CONT "\n");
5683 break;
5684 }
5685 5712
5686 cpus_or(groupmask, groupmask, group->cpumask); 5713 if (!cpus_equal(sd->span, groupmask))
5714 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
5687 5715
5688 cpumask_scnprintf(str, NR_CPUS, group->cpumask); 5716 if (sd->parent && !cpus_subset(groupmask, sd->parent->span))
5689 printk(KERN_CONT " %s", str); 5717 printk(KERN_ERR "ERROR: parent span is not a superset "
5718 "of domain->span\n");
5719 return 0;
5720}
5690 5721
5691 group = group->next; 5722static void sched_domain_debug(struct sched_domain *sd, int cpu)
5692 } while (group != sd->groups); 5723{
5693 printk(KERN_CONT "\n"); 5724 int level = 0;
5694 5725
5695 if (!cpus_equal(sd->span, groupmask)) 5726 if (!sd) {
5696 printk(KERN_ERR "ERROR: groups don't span " 5727 printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
5697 "domain->span\n"); 5728 return;
5729 }
5730
5731 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
5698 5732
5733 for (;;) {
5734 if (sched_domain_debug_one(sd, cpu, level))
5735 break;
5699 level++; 5736 level++;
5700 sd = sd->parent; 5737 sd = sd->parent;
5701 if (!sd) 5738 if (!sd)
5702 continue; 5739 break;
5703 5740 }
5704 if (!cpus_subset(groupmask, sd->span))
5705 printk(KERN_ERR "ERROR: parent span is not a superset "
5706 "of domain->span\n");
5707
5708 } while (sd);
5709} 5741}
5710#else 5742#else
5711# define sched_domain_debug(sd, cpu) do { } while (0) 5743# define sched_domain_debug(sd, cpu) do { } while (0)
@@ -6424,13 +6456,17 @@ static cpumask_t fallback_doms;
6424 */ 6456 */
6425static int arch_init_sched_domains(const cpumask_t *cpu_map) 6457static int arch_init_sched_domains(const cpumask_t *cpu_map)
6426{ 6458{
6459 int err;
6460
6427 ndoms_cur = 1; 6461 ndoms_cur = 1;
6428 doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 6462 doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
6429 if (!doms_cur) 6463 if (!doms_cur)
6430 doms_cur = &fallback_doms; 6464 doms_cur = &fallback_doms;
6431 cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map); 6465 cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map);
6466 err = build_sched_domains(doms_cur);
6432 register_sched_domain_sysctl(); 6467 register_sched_domain_sysctl();
6433 return build_sched_domains(doms_cur); 6468
6469 return err;
6434} 6470}
6435 6471
6436static void arch_destroy_sched_domains(const cpumask_t *cpu_map) 6472static void arch_destroy_sched_domains(const cpumask_t *cpu_map)
@@ -6479,6 +6515,9 @@ void partition_sched_domains(int ndoms_new, cpumask_t *doms_new)
6479{ 6515{
6480 int i, j; 6516 int i, j;
6481 6517
6518 /* always unregister in case we don't destroy any domains */
6519 unregister_sched_domain_sysctl();
6520
6482 if (doms_new == NULL) { 6521 if (doms_new == NULL) {
6483 ndoms_new = 1; 6522 ndoms_new = 1;
6484 doms_new = &fallback_doms; 6523 doms_new = &fallback_doms;
@@ -6514,6 +6553,8 @@ match2:
6514 kfree(doms_cur); 6553 kfree(doms_cur);
6515 doms_cur = doms_new; 6554 doms_cur = doms_new;
6516 ndoms_cur = ndoms_new; 6555 ndoms_cur = ndoms_new;
6556
6557 register_sched_domain_sysctl();
6517} 6558}
6518 6559
6519#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT) 6560#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
@@ -7101,25 +7142,25 @@ unsigned long sched_group_shares(struct task_group *tg)
7101#ifdef CONFIG_FAIR_CGROUP_SCHED 7142#ifdef CONFIG_FAIR_CGROUP_SCHED
7102 7143
7103/* return corresponding task_group object of a cgroup */ 7144/* return corresponding task_group object of a cgroup */
7104static inline struct task_group *cgroup_tg(struct cgroup *cont) 7145static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
7105{ 7146{
7106 return container_of(cgroup_subsys_state(cont, cpu_cgroup_subsys_id), 7147 return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
7107 struct task_group, css); 7148 struct task_group, css);
7108} 7149}
7109 7150
7110static struct cgroup_subsys_state * 7151static struct cgroup_subsys_state *
7111cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) 7152cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
7112{ 7153{
7113 struct task_group *tg; 7154 struct task_group *tg;
7114 7155
7115 if (!cont->parent) { 7156 if (!cgrp->parent) {
7116 /* This is early initialization for the top cgroup */ 7157 /* This is early initialization for the top cgroup */
7117 init_task_group.css.cgroup = cont; 7158 init_task_group.css.cgroup = cgrp;
7118 return &init_task_group.css; 7159 return &init_task_group.css;
7119 } 7160 }
7120 7161
7121 /* we support only 1-level deep hierarchical scheduler atm */ 7162 /* we support only 1-level deep hierarchical scheduler atm */
7122 if (cont->parent->parent) 7163 if (cgrp->parent->parent)
7123 return ERR_PTR(-EINVAL); 7164 return ERR_PTR(-EINVAL);
7124 7165
7125 tg = sched_create_group(); 7166 tg = sched_create_group();
@@ -7127,21 +7168,21 @@ cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
7127 return ERR_PTR(-ENOMEM); 7168 return ERR_PTR(-ENOMEM);
7128 7169
7129 /* Bind the cgroup to task_group object we just created */ 7170 /* Bind the cgroup to task_group object we just created */
7130 tg->css.cgroup = cont; 7171 tg->css.cgroup = cgrp;
7131 7172
7132 return &tg->css; 7173 return &tg->css;
7133} 7174}
7134 7175
7135static void cpu_cgroup_destroy(struct cgroup_subsys *ss, 7176static void cpu_cgroup_destroy(struct cgroup_subsys *ss,
7136 struct cgroup *cont) 7177 struct cgroup *cgrp)
7137{ 7178{
7138 struct task_group *tg = cgroup_tg(cont); 7179 struct task_group *tg = cgroup_tg(cgrp);
7139 7180
7140 sched_destroy_group(tg); 7181 sched_destroy_group(tg);
7141} 7182}
7142 7183
7143static int cpu_cgroup_can_attach(struct cgroup_subsys *ss, 7184static int cpu_cgroup_can_attach(struct cgroup_subsys *ss,
7144 struct cgroup *cont, struct task_struct *tsk) 7185 struct cgroup *cgrp, struct task_struct *tsk)
7145{ 7186{
7146 /* We don't support RT-tasks being in separate groups */ 7187 /* We don't support RT-tasks being in separate groups */
7147 if (tsk->sched_class != &fair_sched_class) 7188 if (tsk->sched_class != &fair_sched_class)
@@ -7151,38 +7192,21 @@ static int cpu_cgroup_can_attach(struct cgroup_subsys *ss,
7151} 7192}
7152 7193
7153static void 7194static void
7154cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cont, 7195cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
7155 struct cgroup *old_cont, struct task_struct *tsk) 7196 struct cgroup *old_cont, struct task_struct *tsk)
7156{ 7197{
7157 sched_move_task(tsk); 7198 sched_move_task(tsk);
7158} 7199}
7159 7200
7160static ssize_t cpu_shares_write(struct cgroup *cont, struct cftype *cftype, 7201static int cpu_shares_write_uint(struct cgroup *cgrp, struct cftype *cftype,
7161 struct file *file, const char __user *userbuf, 7202 u64 shareval)
7162 size_t nbytes, loff_t *ppos)
7163{ 7203{
7164 unsigned long shareval; 7204 return sched_group_set_shares(cgroup_tg(cgrp), shareval);
7165 struct task_group *tg = cgroup_tg(cont);
7166 char buffer[2*sizeof(unsigned long) + 1];
7167 int rc;
7168
7169 if (nbytes > 2*sizeof(unsigned long)) /* safety check */
7170 return -E2BIG;
7171
7172 if (copy_from_user(buffer, userbuf, nbytes))
7173 return -EFAULT;
7174
7175 buffer[nbytes] = 0; /* nul-terminate */
7176 shareval = simple_strtoul(buffer, NULL, 10);
7177
7178 rc = sched_group_set_shares(tg, shareval);
7179
7180 return (rc < 0 ? rc : nbytes);
7181} 7205}
7182 7206
7183static u64 cpu_shares_read_uint(struct cgroup *cont, struct cftype *cft) 7207static u64 cpu_shares_read_uint(struct cgroup *cgrp, struct cftype *cft)
7184{ 7208{
7185 struct task_group *tg = cgroup_tg(cont); 7209 struct task_group *tg = cgroup_tg(cgrp);
7186 7210
7187 return (u64) tg->shares; 7211 return (u64) tg->shares;
7188} 7212}
@@ -7190,7 +7214,7 @@ static u64 cpu_shares_read_uint(struct cgroup *cont, struct cftype *cft)
7190static struct cftype cpu_shares = { 7214static struct cftype cpu_shares = {
7191 .name = "shares", 7215 .name = "shares",
7192 .read_uint = cpu_shares_read_uint, 7216 .read_uint = cpu_shares_read_uint,
7193 .write = cpu_shares_write, 7217 .write_uint = cpu_shares_write_uint,
7194}; 7218};
7195 7219
7196static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont) 7220static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont)
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 166ed6db600b..9971831b560e 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -876,6 +876,7 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
876 } 876 }
877} 877}
878 878
879#ifdef CONFIG_SMP
879/************************************************** 880/**************************************************
880 * Fair scheduling class load-balancing methods: 881 * Fair scheduling class load-balancing methods:
881 */ 882 */
@@ -936,12 +937,11 @@ static int cfs_rq_best_prio(struct cfs_rq *cfs_rq)
936 937
937static unsigned long 938static unsigned long
938load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, 939load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
939 unsigned long max_nr_move, unsigned long max_load_move, 940 unsigned long max_load_move,
940 struct sched_domain *sd, enum cpu_idle_type idle, 941 struct sched_domain *sd, enum cpu_idle_type idle,
941 int *all_pinned, int *this_best_prio) 942 int *all_pinned, int *this_best_prio)
942{ 943{
943 struct cfs_rq *busy_cfs_rq; 944 struct cfs_rq *busy_cfs_rq;
944 unsigned long load_moved, total_nr_moved = 0, nr_moved;
945 long rem_load_move = max_load_move; 945 long rem_load_move = max_load_move;
946 struct rq_iterator cfs_rq_iterator; 946 struct rq_iterator cfs_rq_iterator;
947 947
@@ -969,25 +969,48 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
969#else 969#else
970# define maxload rem_load_move 970# define maxload rem_load_move
971#endif 971#endif
972 /* pass busy_cfs_rq argument into 972 /*
973 * pass busy_cfs_rq argument into
973 * load_balance_[start|next]_fair iterators 974 * load_balance_[start|next]_fair iterators
974 */ 975 */
975 cfs_rq_iterator.arg = busy_cfs_rq; 976 cfs_rq_iterator.arg = busy_cfs_rq;
976 nr_moved = balance_tasks(this_rq, this_cpu, busiest, 977 rem_load_move -= balance_tasks(this_rq, this_cpu, busiest,
977 max_nr_move, maxload, sd, idle, all_pinned, 978 maxload, sd, idle, all_pinned,
978 &load_moved, this_best_prio, &cfs_rq_iterator); 979 this_best_prio,
979 980 &cfs_rq_iterator);
980 total_nr_moved += nr_moved;
981 max_nr_move -= nr_moved;
982 rem_load_move -= load_moved;
983 981
984 if (max_nr_move <= 0 || rem_load_move <= 0) 982 if (rem_load_move <= 0)
985 break; 983 break;
986 } 984 }
987 985
988 return max_load_move - rem_load_move; 986 return max_load_move - rem_load_move;
989} 987}
990 988
989static int
990move_one_task_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
991 struct sched_domain *sd, enum cpu_idle_type idle)
992{
993 struct cfs_rq *busy_cfs_rq;
994 struct rq_iterator cfs_rq_iterator;
995
996 cfs_rq_iterator.start = load_balance_start_fair;
997 cfs_rq_iterator.next = load_balance_next_fair;
998
999 for_each_leaf_cfs_rq(busiest, busy_cfs_rq) {
1000 /*
1001 * pass busy_cfs_rq argument into
1002 * load_balance_[start|next]_fair iterators
1003 */
1004 cfs_rq_iterator.arg = busy_cfs_rq;
1005 if (iter_move_one_task(this_rq, this_cpu, busiest, sd, idle,
1006 &cfs_rq_iterator))
1007 return 1;
1008 }
1009
1010 return 0;
1011}
1012#endif
1013
991/* 1014/*
992 * scheduler tick hitting a task of our scheduling class: 1015 * scheduler tick hitting a task of our scheduling class:
993 */ 1016 */
@@ -1063,7 +1086,10 @@ static const struct sched_class fair_sched_class = {
1063 .pick_next_task = pick_next_task_fair, 1086 .pick_next_task = pick_next_task_fair,
1064 .put_prev_task = put_prev_task_fair, 1087 .put_prev_task = put_prev_task_fair,
1065 1088
1089#ifdef CONFIG_SMP
1066 .load_balance = load_balance_fair, 1090 .load_balance = load_balance_fair,
1091 .move_one_task = move_one_task_fair,
1092#endif
1067 1093
1068 .set_curr_task = set_curr_task_fair, 1094 .set_curr_task = set_curr_task_fair,
1069 .task_tick = task_tick_fair, 1095 .task_tick = task_tick_fair,
diff --git a/kernel/sched_idletask.c b/kernel/sched_idletask.c
index 6e2ead41516e..bf9c25c15b8b 100644
--- a/kernel/sched_idletask.c
+++ b/kernel/sched_idletask.c
@@ -37,15 +37,24 @@ static void put_prev_task_idle(struct rq *rq, struct task_struct *prev)
37{ 37{
38} 38}
39 39
40#ifdef CONFIG_SMP
40static unsigned long 41static unsigned long
41load_balance_idle(struct rq *this_rq, int this_cpu, struct rq *busiest, 42load_balance_idle(struct rq *this_rq, int this_cpu, struct rq *busiest,
42 unsigned long max_nr_move, unsigned long max_load_move, 43 unsigned long max_load_move,
43 struct sched_domain *sd, enum cpu_idle_type idle, 44 struct sched_domain *sd, enum cpu_idle_type idle,
44 int *all_pinned, int *this_best_prio) 45 int *all_pinned, int *this_best_prio)
45{ 46{
46 return 0; 47 return 0;
47} 48}
48 49
50static int
51move_one_task_idle(struct rq *this_rq, int this_cpu, struct rq *busiest,
52 struct sched_domain *sd, enum cpu_idle_type idle)
53{
54 return 0;
55}
56#endif
57
49static void task_tick_idle(struct rq *rq, struct task_struct *curr) 58static void task_tick_idle(struct rq *rq, struct task_struct *curr)
50{ 59{
51} 60}
@@ -69,7 +78,10 @@ const struct sched_class idle_sched_class = {
69 .pick_next_task = pick_next_task_idle, 78 .pick_next_task = pick_next_task_idle,
70 .put_prev_task = put_prev_task_idle, 79 .put_prev_task = put_prev_task_idle,
71 80
81#ifdef CONFIG_SMP
72 .load_balance = load_balance_idle, 82 .load_balance = load_balance_idle,
83 .move_one_task = move_one_task_idle,
84#endif
73 85
74 .set_curr_task = set_curr_task_idle, 86 .set_curr_task = set_curr_task_idle,
75 .task_tick = task_tick_idle, 87 .task_tick = task_tick_idle,
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index d0097a0634e5..8abd752a0ebd 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -98,6 +98,7 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
98 p->se.exec_start = 0; 98 p->se.exec_start = 0;
99} 99}
100 100
101#ifdef CONFIG_SMP
101/* 102/*
102 * Load-balancing iterator. Note: while the runqueue stays locked 103 * Load-balancing iterator. Note: while the runqueue stays locked
103 * during the whole iteration, the current task might be 104 * during the whole iteration, the current task might be
@@ -172,13 +173,11 @@ static struct task_struct *load_balance_next_rt(void *arg)
172 173
173static unsigned long 174static unsigned long
174load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest, 175load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest,
175 unsigned long max_nr_move, unsigned long max_load_move, 176 unsigned long max_load_move,
176 struct sched_domain *sd, enum cpu_idle_type idle, 177 struct sched_domain *sd, enum cpu_idle_type idle,
177 int *all_pinned, int *this_best_prio) 178 int *all_pinned, int *this_best_prio)
178{ 179{
179 int nr_moved;
180 struct rq_iterator rt_rq_iterator; 180 struct rq_iterator rt_rq_iterator;
181 unsigned long load_moved;
182 181
183 rt_rq_iterator.start = load_balance_start_rt; 182 rt_rq_iterator.start = load_balance_start_rt;
184 rt_rq_iterator.next = load_balance_next_rt; 183 rt_rq_iterator.next = load_balance_next_rt;
@@ -187,12 +186,24 @@ load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest,
187 */ 186 */
188 rt_rq_iterator.arg = busiest; 187 rt_rq_iterator.arg = busiest;
189 188
190 nr_moved = balance_tasks(this_rq, this_cpu, busiest, max_nr_move, 189 return balance_tasks(this_rq, this_cpu, busiest, max_load_move, sd,
191 max_load_move, sd, idle, all_pinned, &load_moved, 190 idle, all_pinned, this_best_prio, &rt_rq_iterator);
192 this_best_prio, &rt_rq_iterator); 191}
192
193static int
194move_one_task_rt(struct rq *this_rq, int this_cpu, struct rq *busiest,
195 struct sched_domain *sd, enum cpu_idle_type idle)
196{
197 struct rq_iterator rt_rq_iterator;
198
199 rt_rq_iterator.start = load_balance_start_rt;
200 rt_rq_iterator.next = load_balance_next_rt;
201 rt_rq_iterator.arg = busiest;
193 202
194 return load_moved; 203 return iter_move_one_task(this_rq, this_cpu, busiest, sd, idle,
204 &rt_rq_iterator);
195} 205}
206#endif
196 207
197static void task_tick_rt(struct rq *rq, struct task_struct *p) 208static void task_tick_rt(struct rq *rq, struct task_struct *p)
198{ 209{
@@ -236,7 +247,10 @@ const struct sched_class rt_sched_class = {
236 .pick_next_task = pick_next_task_rt, 247 .pick_next_task = pick_next_task_rt,
237 .put_prev_task = put_prev_task_rt, 248 .put_prev_task = put_prev_task_rt,
238 249
250#ifdef CONFIG_SMP
239 .load_balance = load_balance_rt, 251 .load_balance = load_balance_rt,
252 .move_one_task = move_one_task_rt,
253#endif
240 254
241 .set_curr_task = set_curr_task_rt, 255 .set_curr_task = set_curr_task_rt,
242 .task_tick = task_tick_rt, 256 .task_tick = task_tick_rt,
diff --git a/kernel/user.c b/kernel/user.c
index e91331c457e2..0f3aa0234107 100644
--- a/kernel/user.c
+++ b/kernel/user.c
@@ -129,7 +129,7 @@ static inline void uids_mutex_unlock(void)
129} 129}
130 130
131/* return cpu shares held by the user */ 131/* return cpu shares held by the user */
132ssize_t cpu_shares_show(struct kset *kset, char *buffer) 132static ssize_t cpu_shares_show(struct kset *kset, char *buffer)
133{ 133{
134 struct user_struct *up = container_of(kset, struct user_struct, kset); 134 struct user_struct *up = container_of(kset, struct user_struct, kset);
135 135
@@ -137,7 +137,8 @@ ssize_t cpu_shares_show(struct kset *kset, char *buffer)
137} 137}
138 138
139/* modify cpu shares held by the user */ 139/* modify cpu shares held by the user */
140ssize_t cpu_shares_store(struct kset *kset, const char *buffer, size_t size) 140static ssize_t cpu_shares_store(struct kset *kset, const char *buffer,
141 size_t size)
141{ 142{
142 struct user_struct *up = container_of(kset, struct user_struct, kset); 143 struct user_struct *up = container_of(kset, struct user_struct, kset);
143 unsigned long shares; 144 unsigned long shares;
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-09 15:02:28 -0500