aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/cgroup_subsys.h2
-rw-r--r--include/linux/sched.h11
-rw-r--r--init/Kconfig23
-rw-r--r--kernel/sched.c148
-rw-r--r--kernel/sched_rt.c12
-rw-r--r--kernel/user.c22
6 files changed, 151 insertions, 67 deletions
diff --git a/include/linux/cgroup_subsys.h b/include/linux/cgroup_subsys.h
index 228235c5ae53..ac6aad98b607 100644
--- a/include/linux/cgroup_subsys.h
+++ b/include/linux/cgroup_subsys.h
@@ -25,7 +25,7 @@ SUBSYS(ns)
25 25
26/* */ 26/* */
27 27
28#ifdef CONFIG_FAIR_CGROUP_SCHED 28#ifdef CONFIG_CGROUP_SCHED
29SUBSYS(cpu_cgroup) 29SUBSYS(cpu_cgroup)
30#endif 30#endif
31 31
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 142eb293f9c4..b9bb313fe1ae 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -590,7 +590,7 @@ struct user_struct {
590 struct hlist_node uidhash_node; 590 struct hlist_node uidhash_node;
591 uid_t uid; 591 uid_t uid;
592 592
593#ifdef CONFIG_FAIR_USER_SCHED 593#ifdef CONFIG_USER_SCHED
594 struct task_group *tg; 594 struct task_group *tg;
595#ifdef CONFIG_SYSFS 595#ifdef CONFIG_SYSFS
596 struct kobject kobj; 596 struct kobject kobj;
@@ -973,7 +973,7 @@ struct sched_rt_entity {
973 unsigned long timeout; 973 unsigned long timeout;
974 int nr_cpus_allowed; 974 int nr_cpus_allowed;
975 975
976#ifdef CONFIG_FAIR_GROUP_SCHED 976#ifdef CONFIG_RT_GROUP_SCHED
977 struct sched_rt_entity *parent; 977 struct sched_rt_entity *parent;
978 /* rq on which this entity is (to be) queued: */ 978 /* rq on which this entity is (to be) queued: */
979 struct rt_rq *rt_rq; 979 struct rt_rq *rt_rq;
@@ -2027,19 +2027,22 @@ extern int sched_mc_power_savings, sched_smt_power_savings;
2027 2027
2028extern void normalize_rt_tasks(void); 2028extern void normalize_rt_tasks(void);
2029 2029
2030#ifdef CONFIG_FAIR_GROUP_SCHED 2030#ifdef CONFIG_GROUP_SCHED
2031 2031
2032extern struct task_group init_task_group; 2032extern struct task_group init_task_group;
2033 2033
2034extern struct task_group *sched_create_group(void); 2034extern struct task_group *sched_create_group(void);
2035extern void sched_destroy_group(struct task_group *tg); 2035extern void sched_destroy_group(struct task_group *tg);
2036extern void sched_move_task(struct task_struct *tsk); 2036extern void sched_move_task(struct task_struct *tsk);
2037#ifdef CONFIG_FAIR_GROUP_SCHED
2037extern int sched_group_set_shares(struct task_group *tg, unsigned long shares); 2038extern int sched_group_set_shares(struct task_group *tg, unsigned long shares);
2038extern unsigned long sched_group_shares(struct task_group *tg); 2039extern unsigned long sched_group_shares(struct task_group *tg);
2040#endif
2041#ifdef CONFIG_RT_GROUP_SCHED
2039extern int sched_group_set_rt_runtime(struct task_group *tg, 2042extern int sched_group_set_rt_runtime(struct task_group *tg,
2040 long rt_runtime_us); 2043 long rt_runtime_us);
2041extern long sched_group_rt_runtime(struct task_group *tg); 2044extern long sched_group_rt_runtime(struct task_group *tg);
2042 2045#endif
2043#endif 2046#endif
2044 2047
2045#ifdef CONFIG_TASK_XACCT 2048#ifdef CONFIG_TASK_XACCT
diff --git a/init/Kconfig b/init/Kconfig
index 824d48cb67bf..dcef8b55011a 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -311,25 +311,36 @@ config CPUSETS
311 311
312 Say N if unsure. 312 Say N if unsure.
313 313
314config FAIR_GROUP_SCHED 314config GROUP_SCHED
315 bool "Fair group CPU scheduler" 315 bool "Group CPU scheduler"
316 default y 316 default y
317 help 317 help
318 This feature lets CPU scheduler recognize task groups and control CPU 318 This feature lets CPU scheduler recognize task groups and control CPU
319 bandwidth allocation to such task groups. 319 bandwidth allocation to such task groups.
320 320
321config FAIR_GROUP_SCHED
322 bool "Group scheduling for SCHED_OTHER"
323 depends on GROUP_SCHED
324 default y
325
326config RT_GROUP_SCHED
327 bool "Group scheduling for SCHED_RR/FIFO"
328 depends on EXPERIMENTAL
329 depends on GROUP_SCHED
330 default n
331
321choice 332choice
322 depends on FAIR_GROUP_SCHED 333 depends on GROUP_SCHED
323 prompt "Basis for grouping tasks" 334 prompt "Basis for grouping tasks"
324 default FAIR_USER_SCHED 335 default USER_SCHED
325 336
326config FAIR_USER_SCHED 337config USER_SCHED
327 bool "user id" 338 bool "user id"
328 help 339 help
329 This option will choose userid as the basis for grouping 340 This option will choose userid as the basis for grouping
330 tasks, thus providing equal CPU bandwidth to each user. 341 tasks, thus providing equal CPU bandwidth to each user.
331 342
332config FAIR_CGROUP_SCHED 343config CGROUP_SCHED
333 bool "Control groups" 344 bool "Control groups"
334 depends on CGROUPS 345 depends on CGROUPS
335 help 346 help
diff --git a/kernel/sched.c b/kernel/sched.c
index 85a5fbff2b00..5edc549edae8 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -155,7 +155,7 @@ struct rt_prio_array {
155 struct list_head queue[MAX_RT_PRIO]; 155 struct list_head queue[MAX_RT_PRIO];
156}; 156};
157 157
158#ifdef CONFIG_FAIR_GROUP_SCHED 158#ifdef CONFIG_GROUP_SCHED
159 159
160#include <linux/cgroup.h> 160#include <linux/cgroup.h>
161 161
@@ -165,19 +165,16 @@ static LIST_HEAD(task_groups);
165 165
166/* task group related information */ 166/* task group related information */
167struct task_group { 167struct task_group {
168#ifdef CONFIG_FAIR_CGROUP_SCHED 168#ifdef CONFIG_CGROUP_SCHED
169 struct cgroup_subsys_state css; 169 struct cgroup_subsys_state css;
170#endif 170#endif
171
172#ifdef CONFIG_FAIR_GROUP_SCHED
171 /* schedulable entities of this group on each cpu */ 173 /* schedulable entities of this group on each cpu */
172 struct sched_entity **se; 174 struct sched_entity **se;
173 /* runqueue "owned" by this group on each cpu */ 175 /* runqueue "owned" by this group on each cpu */
174 struct cfs_rq **cfs_rq; 176 struct cfs_rq **cfs_rq;
175 177
176 struct sched_rt_entity **rt_se;
177 struct rt_rq **rt_rq;
178
179 u64 rt_runtime;
180
181 /* 178 /*
182 * shares assigned to a task group governs how much of cpu bandwidth 179 * shares assigned to a task group governs how much of cpu bandwidth
183 * is allocated to the group. The more shares a group has, the more is 180 * is allocated to the group. The more shares a group has, the more is
@@ -213,24 +210,36 @@ struct task_group {
213 * 210 *
214 */ 211 */
215 unsigned long shares; 212 unsigned long shares;
213#endif
214
215#ifdef CONFIG_RT_GROUP_SCHED
216 struct sched_rt_entity **rt_se;
217 struct rt_rq **rt_rq;
218
219 u64 rt_runtime;
220#endif
216 221
217 struct rcu_head rcu; 222 struct rcu_head rcu;
218 struct list_head list; 223 struct list_head list;
219}; 224};
220 225
226#ifdef CONFIG_FAIR_GROUP_SCHED
221/* Default task group's sched entity on each cpu */ 227/* Default task group's sched entity on each cpu */
222static DEFINE_PER_CPU(struct sched_entity, init_sched_entity); 228static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
223/* Default task group's cfs_rq on each cpu */ 229/* Default task group's cfs_rq on each cpu */
224static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp; 230static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
225 231
226static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
227static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
228
229static struct sched_entity *init_sched_entity_p[NR_CPUS]; 232static struct sched_entity *init_sched_entity_p[NR_CPUS];
230static struct cfs_rq *init_cfs_rq_p[NR_CPUS]; 233static struct cfs_rq *init_cfs_rq_p[NR_CPUS];
234#endif
235
236#ifdef CONFIG_RT_GROUP_SCHED
237static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
238static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
231 239
232static struct sched_rt_entity *init_sched_rt_entity_p[NR_CPUS]; 240static struct sched_rt_entity *init_sched_rt_entity_p[NR_CPUS];
233static struct rt_rq *init_rt_rq_p[NR_CPUS]; 241static struct rt_rq *init_rt_rq_p[NR_CPUS];
242#endif
234 243
235/* task_group_lock serializes add/remove of task groups and also changes to 244/* task_group_lock serializes add/remove of task groups and also changes to
236 * a task group's cpu shares. 245 * a task group's cpu shares.
@@ -240,6 +249,7 @@ static DEFINE_SPINLOCK(task_group_lock);
240/* doms_cur_mutex serializes access to doms_cur[] array */ 249/* doms_cur_mutex serializes access to doms_cur[] array */
241static DEFINE_MUTEX(doms_cur_mutex); 250static DEFINE_MUTEX(doms_cur_mutex);
242 251
252#ifdef CONFIG_FAIR_GROUP_SCHED
243#ifdef CONFIG_SMP 253#ifdef CONFIG_SMP
244/* kernel thread that runs rebalance_shares() periodically */ 254/* kernel thread that runs rebalance_shares() periodically */
245static struct task_struct *lb_monitor_task; 255static struct task_struct *lb_monitor_task;
@@ -248,35 +258,40 @@ static int load_balance_monitor(void *unused);
248 258
249static void set_se_shares(struct sched_entity *se, unsigned long shares); 259static void set_se_shares(struct sched_entity *se, unsigned long shares);
250 260
261#ifdef CONFIG_USER_SCHED
262# define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD)
263#else
264# define INIT_TASK_GROUP_LOAD NICE_0_LOAD
265#endif
266
267#define MIN_GROUP_SHARES 2
268
269static int init_task_group_load = INIT_TASK_GROUP_LOAD;
270#endif
271
251/* Default task group. 272/* Default task group.
252 * Every task in system belong to this group at bootup. 273 * Every task in system belong to this group at bootup.
253 */ 274 */
254struct task_group init_task_group = { 275struct task_group init_task_group = {
276#ifdef CONFIG_FAIR_GROUP_SCHED
255 .se = init_sched_entity_p, 277 .se = init_sched_entity_p,
256 .cfs_rq = init_cfs_rq_p, 278 .cfs_rq = init_cfs_rq_p,
279#endif
257 280
281#ifdef CONFIG_RT_GROUP_SCHED
258 .rt_se = init_sched_rt_entity_p, 282 .rt_se = init_sched_rt_entity_p,
259 .rt_rq = init_rt_rq_p, 283 .rt_rq = init_rt_rq_p,
260};
261
262#ifdef CONFIG_FAIR_USER_SCHED
263# define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD)
264#else
265# define INIT_TASK_GROUP_LOAD NICE_0_LOAD
266#endif 284#endif
267 285};
268#define MIN_GROUP_SHARES 2
269
270static int init_task_group_load = INIT_TASK_GROUP_LOAD;
271 286
272/* return group to which a task belongs */ 287/* return group to which a task belongs */
273static inline struct task_group *task_group(struct task_struct *p) 288static inline struct task_group *task_group(struct task_struct *p)
274{ 289{
275 struct task_group *tg; 290 struct task_group *tg;
276 291
277#ifdef CONFIG_FAIR_USER_SCHED 292#ifdef CONFIG_USER_SCHED
278 tg = p->user->tg; 293 tg = p->user->tg;
279#elif defined(CONFIG_FAIR_CGROUP_SCHED) 294#elif defined(CONFIG_CGROUP_SCHED)
280 tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id), 295 tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
281 struct task_group, css); 296 struct task_group, css);
282#else 297#else
@@ -288,11 +303,15 @@ static inline struct task_group *task_group(struct task_struct *p)
288/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ 303/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
289static inline void set_task_rq(struct task_struct *p, unsigned int cpu) 304static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
290{ 305{
306#ifdef CONFIG_FAIR_GROUP_SCHED
291 p->se.cfs_rq = task_group(p)->cfs_rq[cpu]; 307 p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
292 p->se.parent = task_group(p)->se[cpu]; 308 p->se.parent = task_group(p)->se[cpu];
309#endif
293 310
311#ifdef CONFIG_RT_GROUP_SCHED
294 p->rt.rt_rq = task_group(p)->rt_rq[cpu]; 312 p->rt.rt_rq = task_group(p)->rt_rq[cpu];
295 p->rt.parent = task_group(p)->rt_se[cpu]; 313 p->rt.parent = task_group(p)->rt_se[cpu];
314#endif
296} 315}
297 316
298static inline void lock_doms_cur(void) 317static inline void lock_doms_cur(void)
@@ -311,7 +330,7 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
311static inline void lock_doms_cur(void) { } 330static inline void lock_doms_cur(void) { }
312static inline void unlock_doms_cur(void) { } 331static inline void unlock_doms_cur(void) { }
313 332
314#endif /* CONFIG_FAIR_GROUP_SCHED */ 333#endif /* CONFIG_GROUP_SCHED */
315 334
316/* CFS-related fields in a runqueue */ 335/* CFS-related fields in a runqueue */
317struct cfs_rq { 336struct cfs_rq {
@@ -351,7 +370,7 @@ struct cfs_rq {
351struct rt_rq { 370struct rt_rq {
352 struct rt_prio_array active; 371 struct rt_prio_array active;
353 unsigned long rt_nr_running; 372 unsigned long rt_nr_running;
354#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED 373#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
355 int highest_prio; /* highest queued rt task prio */ 374 int highest_prio; /* highest queued rt task prio */
356#endif 375#endif
357#ifdef CONFIG_SMP 376#ifdef CONFIG_SMP
@@ -361,7 +380,7 @@ struct rt_rq {
361 int rt_throttled; 380 int rt_throttled;
362 u64 rt_time; 381 u64 rt_time;
363 382
364#ifdef CONFIG_FAIR_GROUP_SCHED 383#ifdef CONFIG_RT_GROUP_SCHED
365 unsigned long rt_nr_boosted; 384 unsigned long rt_nr_boosted;
366 385
367 struct rq *rq; 386 struct rq *rq;
@@ -437,6 +456,8 @@ struct rq {
437#ifdef CONFIG_FAIR_GROUP_SCHED 456#ifdef CONFIG_FAIR_GROUP_SCHED
438 /* list of leaf cfs_rq on this cpu: */ 457 /* list of leaf cfs_rq on this cpu: */
439 struct list_head leaf_cfs_rq_list; 458 struct list_head leaf_cfs_rq_list;
459#endif
460#ifdef CONFIG_RT_GROUP_SCHED
440 struct list_head leaf_rt_rq_list; 461 struct list_head leaf_rt_rq_list;
441#endif 462#endif
442 463
@@ -7104,7 +7125,7 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
7104 /* delimiter for bitsearch: */ 7125 /* delimiter for bitsearch: */
7105 __set_bit(MAX_RT_PRIO, array->bitmap); 7126 __set_bit(MAX_RT_PRIO, array->bitmap);
7106 7127
7107#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED 7128#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
7108 rt_rq->highest_prio = MAX_RT_PRIO; 7129 rt_rq->highest_prio = MAX_RT_PRIO;
7109#endif 7130#endif
7110#ifdef CONFIG_SMP 7131#ifdef CONFIG_SMP
@@ -7115,7 +7136,7 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
7115 rt_rq->rt_time = 0; 7136 rt_rq->rt_time = 0;
7116 rt_rq->rt_throttled = 0; 7137 rt_rq->rt_throttled = 0;
7117 7138
7118#ifdef CONFIG_FAIR_GROUP_SCHED 7139#ifdef CONFIG_RT_GROUP_SCHED
7119 rt_rq->rt_nr_boosted = 0; 7140 rt_rq->rt_nr_boosted = 0;
7120 rt_rq->rq = rq; 7141 rt_rq->rq = rq;
7121#endif 7142#endif
@@ -7139,7 +7160,9 @@ static void init_tg_cfs_entry(struct rq *rq, struct task_group *tg,
7139 se->load.inv_weight = div64_64(1ULL<<32, se->load.weight); 7160 se->load.inv_weight = div64_64(1ULL<<32, se->load.weight);
7140 se->parent = NULL; 7161 se->parent = NULL;
7141} 7162}
7163#endif
7142 7164
7165#ifdef CONFIG_RT_GROUP_SCHED
7143static void init_tg_rt_entry(struct rq *rq, struct task_group *tg, 7166static void init_tg_rt_entry(struct rq *rq, struct task_group *tg,
7144 struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, 7167 struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
7145 int cpu, int add) 7168 int cpu, int add)
@@ -7168,7 +7191,7 @@ void __init sched_init(void)
7168 init_defrootdomain(); 7191 init_defrootdomain();
7169#endif 7192#endif
7170 7193
7171#ifdef CONFIG_FAIR_GROUP_SCHED 7194#ifdef CONFIG_GROUP_SCHED
7172 list_add(&init_task_group.list, &task_groups); 7195 list_add(&init_task_group.list, &task_groups);
7173#endif 7196#endif
7174 7197
@@ -7189,6 +7212,8 @@ void __init sched_init(void)
7189 &per_cpu(init_cfs_rq, i), 7212 &per_cpu(init_cfs_rq, i),
7190 &per_cpu(init_sched_entity, i), i, 1); 7213 &per_cpu(init_sched_entity, i), i, 1);
7191 7214
7215#endif
7216#ifdef CONFIG_RT_GROUP_SCHED
7192 init_task_group.rt_runtime = 7217 init_task_group.rt_runtime =
7193 sysctl_sched_rt_runtime * NSEC_PER_USEC; 7218 sysctl_sched_rt_runtime * NSEC_PER_USEC;
7194 INIT_LIST_HEAD(&rq->leaf_rt_rq_list); 7219 INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
@@ -7381,9 +7406,9 @@ void set_curr_task(int cpu, struct task_struct *p)
7381 7406
7382#endif 7407#endif
7383 7408
7384#ifdef CONFIG_FAIR_GROUP_SCHED 7409#ifdef CONFIG_GROUP_SCHED
7385 7410
7386#ifdef CONFIG_SMP 7411#if defined CONFIG_FAIR_GROUP_SCHED && defined CONFIG_SMP
7387/* 7412/*
7388 * distribute shares of all task groups among their schedulable entities, 7413 * distribute shares of all task groups among their schedulable entities,
7389 * to reflect load distribution across cpus. 7414 * to reflect load distribution across cpus.
@@ -7539,20 +7564,28 @@ static void free_sched_group(struct task_group *tg)
7539 int i; 7564 int i;
7540 7565
7541 for_each_possible_cpu(i) { 7566 for_each_possible_cpu(i) {
7567#ifdef CONFIG_FAIR_GROUP_SCHED
7542 if (tg->cfs_rq) 7568 if (tg->cfs_rq)
7543 kfree(tg->cfs_rq[i]); 7569 kfree(tg->cfs_rq[i]);
7544 if (tg->se) 7570 if (tg->se)
7545 kfree(tg->se[i]); 7571 kfree(tg->se[i]);
7572#endif
7573#ifdef CONFIG_RT_GROUP_SCHED
7546 if (tg->rt_rq) 7574 if (tg->rt_rq)
7547 kfree(tg->rt_rq[i]); 7575 kfree(tg->rt_rq[i]);
7548 if (tg->rt_se) 7576 if (tg->rt_se)
7549 kfree(tg->rt_se[i]); 7577 kfree(tg->rt_se[i]);
7578#endif
7550 } 7579 }
7551 7580
7581#ifdef CONFIG_FAIR_GROUP_SCHED
7552 kfree(tg->cfs_rq); 7582 kfree(tg->cfs_rq);
7553 kfree(tg->se); 7583 kfree(tg->se);
7584#endif
7585#ifdef CONFIG_RT_GROUP_SCHED
7554 kfree(tg->rt_rq); 7586 kfree(tg->rt_rq);
7555 kfree(tg->rt_se); 7587 kfree(tg->rt_se);
7588#endif
7556 kfree(tg); 7589 kfree(tg);
7557} 7590}
7558 7591
@@ -7560,10 +7593,14 @@ static void free_sched_group(struct task_group *tg)
7560struct task_group *sched_create_group(void) 7593struct task_group *sched_create_group(void)
7561{ 7594{
7562 struct task_group *tg; 7595 struct task_group *tg;
7596#ifdef CONFIG_FAIR_GROUP_SCHED
7563 struct cfs_rq *cfs_rq; 7597 struct cfs_rq *cfs_rq;
7564 struct sched_entity *se; 7598 struct sched_entity *se;
7599#endif
7600#ifdef CONFIG_RT_GROUP_SCHED
7565 struct rt_rq *rt_rq; 7601 struct rt_rq *rt_rq;
7566 struct sched_rt_entity *rt_se; 7602 struct sched_rt_entity *rt_se;
7603#endif
7567 struct rq *rq; 7604 struct rq *rq;
7568 unsigned long flags; 7605 unsigned long flags;
7569 int i; 7606 int i;
@@ -7572,12 +7609,18 @@ struct task_group *sched_create_group(void)
7572 if (!tg) 7609 if (!tg)
7573 return ERR_PTR(-ENOMEM); 7610 return ERR_PTR(-ENOMEM);
7574 7611
7612#ifdef CONFIG_FAIR_GROUP_SCHED
7575 tg->cfs_rq = kzalloc(sizeof(cfs_rq) * NR_CPUS, GFP_KERNEL); 7613 tg->cfs_rq = kzalloc(sizeof(cfs_rq) * NR_CPUS, GFP_KERNEL);
7576 if (!tg->cfs_rq) 7614 if (!tg->cfs_rq)
7577 goto err; 7615 goto err;
7578 tg->se = kzalloc(sizeof(se) * NR_CPUS, GFP_KERNEL); 7616 tg->se = kzalloc(sizeof(se) * NR_CPUS, GFP_KERNEL);
7579 if (!tg->se) 7617 if (!tg->se)
7580 goto err; 7618 goto err;
7619
7620 tg->shares = NICE_0_LOAD;
7621#endif
7622
7623#ifdef CONFIG_RT_GROUP_SCHED
7581 tg->rt_rq = kzalloc(sizeof(rt_rq) * NR_CPUS, GFP_KERNEL); 7624 tg->rt_rq = kzalloc(sizeof(rt_rq) * NR_CPUS, GFP_KERNEL);
7582 if (!tg->rt_rq) 7625 if (!tg->rt_rq)
7583 goto err; 7626 goto err;
@@ -7585,12 +7628,13 @@ struct task_group *sched_create_group(void)
7585 if (!tg->rt_se) 7628 if (!tg->rt_se)
7586 goto err; 7629 goto err;
7587 7630
7588 tg->shares = NICE_0_LOAD;
7589 tg->rt_runtime = 0; 7631 tg->rt_runtime = 0;
7632#endif
7590 7633
7591 for_each_possible_cpu(i) { 7634 for_each_possible_cpu(i) {
7592 rq = cpu_rq(i); 7635 rq = cpu_rq(i);
7593 7636
7637#ifdef CONFIG_FAIR_GROUP_SCHED
7594 cfs_rq = kmalloc_node(sizeof(struct cfs_rq), 7638 cfs_rq = kmalloc_node(sizeof(struct cfs_rq),
7595 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 7639 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i));
7596 if (!cfs_rq) 7640 if (!cfs_rq)
@@ -7601,6 +7645,10 @@ struct task_group *sched_create_group(void)
7601 if (!se) 7645 if (!se)
7602 goto err; 7646 goto err;
7603 7647
7648 init_tg_cfs_entry(rq, tg, cfs_rq, se, i, 0);
7649#endif
7650
7651#ifdef CONFIG_RT_GROUP_SCHED
7604 rt_rq = kmalloc_node(sizeof(struct rt_rq), 7652 rt_rq = kmalloc_node(sizeof(struct rt_rq),
7605 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 7653 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i));
7606 if (!rt_rq) 7654 if (!rt_rq)
@@ -7611,17 +7659,21 @@ struct task_group *sched_create_group(void)
7611 if (!rt_se) 7659 if (!rt_se)
7612 goto err; 7660 goto err;
7613 7661
7614 init_tg_cfs_entry(rq, tg, cfs_rq, se, i, 0);
7615 init_tg_rt_entry(rq, tg, rt_rq, rt_se, i, 0); 7662 init_tg_rt_entry(rq, tg, rt_rq, rt_se, i, 0);
7663#endif
7616 } 7664 }
7617 7665
7618 spin_lock_irqsave(&task_group_lock, flags); 7666 spin_lock_irqsave(&task_group_lock, flags);
7619 for_each_possible_cpu(i) { 7667 for_each_possible_cpu(i) {
7620 rq = cpu_rq(i); 7668 rq = cpu_rq(i);
7669#ifdef CONFIG_FAIR_GROUP_SCHED
7621 cfs_rq = tg->cfs_rq[i]; 7670 cfs_rq = tg->cfs_rq[i];
7622 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list); 7671 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
7672#endif
7673#ifdef CONFIG_RT_GROUP_SCHED
7623 rt_rq = tg->rt_rq[i]; 7674 rt_rq = tg->rt_rq[i];
7624 list_add_rcu(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list); 7675 list_add_rcu(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list);
7676#endif
7625 } 7677 }
7626 list_add_rcu(&tg->list, &task_groups); 7678 list_add_rcu(&tg->list, &task_groups);
7627 spin_unlock_irqrestore(&task_group_lock, flags); 7679 spin_unlock_irqrestore(&task_group_lock, flags);
@@ -7643,23 +7695,21 @@ static void free_sched_group_rcu(struct rcu_head *rhp)
7643/* Destroy runqueue etc associated with a task group */ 7695/* Destroy runqueue etc associated with a task group */
7644void sched_destroy_group(struct task_group *tg) 7696void sched_destroy_group(struct task_group *tg)
7645{ 7697{
7646 struct cfs_rq *cfs_rq = NULL;
7647 struct rt_rq *rt_rq = NULL;
7648 unsigned long flags; 7698 unsigned long flags;
7649 int i; 7699 int i;
7650 7700
7651 spin_lock_irqsave(&task_group_lock, flags); 7701 spin_lock_irqsave(&task_group_lock, flags);
7652 for_each_possible_cpu(i) { 7702 for_each_possible_cpu(i) {
7653 cfs_rq = tg->cfs_rq[i]; 7703#ifdef CONFIG_FAIR_GROUP_SCHED
7654 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 7704 list_del_rcu(&tg->cfs_rq[i]->leaf_cfs_rq_list);
7655 rt_rq = tg->rt_rq[i]; 7705#endif
7656 list_del_rcu(&rt_rq->leaf_rt_rq_list); 7706#ifdef CONFIG_RT_GROUP_SCHED
7707 list_del_rcu(&tg->rt_rq[i]->leaf_rt_rq_list);
7708#endif
7657 } 7709 }
7658 list_del_rcu(&tg->list); 7710 list_del_rcu(&tg->list);
7659 spin_unlock_irqrestore(&task_group_lock, flags); 7711 spin_unlock_irqrestore(&task_group_lock, flags);
7660 7712
7661 BUG_ON(!cfs_rq);
7662
7663 /* wait for possible concurrent references to cfs_rqs complete */ 7713 /* wait for possible concurrent references to cfs_rqs complete */
7664 call_rcu(&tg->rcu, free_sched_group_rcu); 7714 call_rcu(&tg->rcu, free_sched_group_rcu);
7665} 7715}
@@ -7699,6 +7749,7 @@ void sched_move_task(struct task_struct *tsk)
7699 task_rq_unlock(rq, &flags); 7749 task_rq_unlock(rq, &flags);
7700} 7750}
7701 7751
7752#ifdef CONFIG_FAIR_GROUP_SCHED
7702/* rq->lock to be locked by caller */ 7753/* rq->lock to be locked by caller */
7703static void set_se_shares(struct sched_entity *se, unsigned long shares) 7754static void set_se_shares(struct sched_entity *se, unsigned long shares)
7704{ 7755{
@@ -7786,7 +7837,9 @@ unsigned long sched_group_shares(struct task_group *tg)
7786{ 7837{
7787 return tg->shares; 7838 return tg->shares;
7788} 7839}
7840#endif
7789 7841
7842#ifdef CONFIG_RT_GROUP_SCHED
7790/* 7843/*
7791 * Ensure that the real time constraints are schedulable. 7844 * Ensure that the real time constraints are schedulable.
7792 */ 7845 */
@@ -7858,9 +7911,10 @@ long sched_group_rt_runtime(struct task_group *tg)
7858 do_div(rt_runtime_us, NSEC_PER_USEC); 7911 do_div(rt_runtime_us, NSEC_PER_USEC);
7859 return rt_runtime_us; 7912 return rt_runtime_us;
7860} 7913}
7861#endif /* CONFIG_FAIR_GROUP_SCHED */ 7914#endif
7915#endif /* CONFIG_GROUP_SCHED */
7862 7916
7863#ifdef CONFIG_FAIR_CGROUP_SCHED 7917#ifdef CONFIG_CGROUP_SCHED
7864 7918
7865/* return corresponding task_group object of a cgroup */ 7919/* return corresponding task_group object of a cgroup */
7866static inline struct task_group *cgroup_tg(struct cgroup *cgrp) 7920static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
@@ -7920,6 +7974,7 @@ cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
7920 sched_move_task(tsk); 7974 sched_move_task(tsk);
7921} 7975}
7922 7976
7977#ifdef CONFIG_FAIR_GROUP_SCHED
7923static int cpu_shares_write_uint(struct cgroup *cgrp, struct cftype *cftype, 7978static int cpu_shares_write_uint(struct cgroup *cgrp, struct cftype *cftype,
7924 u64 shareval) 7979 u64 shareval)
7925{ 7980{
@@ -7932,7 +7987,9 @@ static u64 cpu_shares_read_uint(struct cgroup *cgrp, struct cftype *cft)
7932 7987
7933 return (u64) tg->shares; 7988 return (u64) tg->shares;
7934} 7989}
7990#endif
7935 7991
7992#ifdef CONFIG_RT_GROUP_SCHED
7936static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft, 7993static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
7937 struct file *file, 7994 struct file *file,
7938 const char __user *userbuf, 7995 const char __user *userbuf,
@@ -7977,18 +8034,23 @@ static ssize_t cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft,
7977 8034
7978 return simple_read_from_buffer(buf, nbytes, ppos, tmp, len); 8035 return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
7979} 8036}
8037#endif
7980 8038
7981static struct cftype cpu_files[] = { 8039static struct cftype cpu_files[] = {
8040#ifdef CONFIG_FAIR_GROUP_SCHED
7982 { 8041 {
7983 .name = "shares", 8042 .name = "shares",
7984 .read_uint = cpu_shares_read_uint, 8043 .read_uint = cpu_shares_read_uint,
7985 .write_uint = cpu_shares_write_uint, 8044 .write_uint = cpu_shares_write_uint,
7986 }, 8045 },
8046#endif
8047#ifdef CONFIG_RT_GROUP_SCHED
7987 { 8048 {
7988 .name = "rt_runtime_us", 8049 .name = "rt_runtime_us",
7989 .read = cpu_rt_runtime_read, 8050 .read = cpu_rt_runtime_read,
7990 .write = cpu_rt_runtime_write, 8051 .write = cpu_rt_runtime_write,
7991 }, 8052 },
8053#endif
7992}; 8054};
7993 8055
7994static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont) 8056static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont)
@@ -8007,7 +8069,7 @@ struct cgroup_subsys cpu_cgroup_subsys = {
8007 .early_init = 1, 8069 .early_init = 1,
8008}; 8070};
8009 8071
8010#endif /* CONFIG_FAIR_CGROUP_SCHED */ 8072#endif /* CONFIG_CGROUP_SCHED */
8011 8073
8012#ifdef CONFIG_CGROUP_CPUACCT 8074#ifdef CONFIG_CGROUP_CPUACCT
8013 8075
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index 35825b28e429..f54792b175b2 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -55,7 +55,7 @@ static inline int on_rt_rq(struct sched_rt_entity *rt_se)
55 return !list_empty(&rt_se->run_list); 55 return !list_empty(&rt_se->run_list);
56} 56}
57 57
58#ifdef CONFIG_FAIR_GROUP_SCHED 58#ifdef CONFIG_RT_GROUP_SCHED
59 59
60static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) 60static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
61{ 61{
@@ -177,7 +177,7 @@ static inline int rt_rq_throttled(struct rt_rq *rt_rq)
177 177
178static inline int rt_se_prio(struct sched_rt_entity *rt_se) 178static inline int rt_se_prio(struct sched_rt_entity *rt_se)
179{ 179{
180#ifdef CONFIG_FAIR_GROUP_SCHED 180#ifdef CONFIG_RT_GROUP_SCHED
181 struct rt_rq *rt_rq = group_rt_rq(rt_se); 181 struct rt_rq *rt_rq = group_rt_rq(rt_se);
182 182
183 if (rt_rq) 183 if (rt_rq)
@@ -269,7 +269,7 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
269{ 269{
270 WARN_ON(!rt_prio(rt_se_prio(rt_se))); 270 WARN_ON(!rt_prio(rt_se_prio(rt_se)));
271 rt_rq->rt_nr_running++; 271 rt_rq->rt_nr_running++;
272#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED 272#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
273 if (rt_se_prio(rt_se) < rt_rq->highest_prio) 273 if (rt_se_prio(rt_se) < rt_rq->highest_prio)
274 rt_rq->highest_prio = rt_se_prio(rt_se); 274 rt_rq->highest_prio = rt_se_prio(rt_se);
275#endif 275#endif
@@ -281,7 +281,7 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
281 281
282 update_rt_migration(rq_of_rt_rq(rt_rq)); 282 update_rt_migration(rq_of_rt_rq(rt_rq));
283#endif 283#endif
284#ifdef CONFIG_FAIR_GROUP_SCHED 284#ifdef CONFIG_RT_GROUP_SCHED
285 if (rt_se_boosted(rt_se)) 285 if (rt_se_boosted(rt_se))
286 rt_rq->rt_nr_boosted++; 286 rt_rq->rt_nr_boosted++;
287#endif 287#endif
@@ -293,7 +293,7 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
293 WARN_ON(!rt_prio(rt_se_prio(rt_se))); 293 WARN_ON(!rt_prio(rt_se_prio(rt_se)));
294 WARN_ON(!rt_rq->rt_nr_running); 294 WARN_ON(!rt_rq->rt_nr_running);
295 rt_rq->rt_nr_running--; 295 rt_rq->rt_nr_running--;
296#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED 296#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
297 if (rt_rq->rt_nr_running) { 297 if (rt_rq->rt_nr_running) {
298 struct rt_prio_array *array; 298 struct rt_prio_array *array;
299 299
@@ -315,7 +315,7 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
315 315
316 update_rt_migration(rq_of_rt_rq(rt_rq)); 316 update_rt_migration(rq_of_rt_rq(rt_rq));
317#endif /* CONFIG_SMP */ 317#endif /* CONFIG_SMP */
318#ifdef CONFIG_FAIR_GROUP_SCHED 318#ifdef CONFIG_RT_GROUP_SCHED
319 if (rt_se_boosted(rt_se)) 319 if (rt_se_boosted(rt_se))
320 rt_rq->rt_nr_boosted--; 320 rt_rq->rt_nr_boosted--;
321 321
diff --git a/kernel/user.c b/kernel/user.c
index 9f6d471bfd03..7132022a040c 100644
--- a/kernel/user.c
+++ b/kernel/user.c
@@ -57,7 +57,7 @@ struct user_struct root_user = {
57 .uid_keyring = &root_user_keyring, 57 .uid_keyring = &root_user_keyring,
58 .session_keyring = &root_session_keyring, 58 .session_keyring = &root_session_keyring,
59#endif 59#endif
60#ifdef CONFIG_FAIR_USER_SCHED 60#ifdef CONFIG_USER_SCHED
61 .tg = &init_task_group, 61 .tg = &init_task_group,
62#endif 62#endif
63}; 63};
@@ -90,7 +90,7 @@ static struct user_struct *uid_hash_find(uid_t uid, struct hlist_head *hashent)
90 return NULL; 90 return NULL;
91} 91}
92 92
93#ifdef CONFIG_FAIR_USER_SCHED 93#ifdef CONFIG_USER_SCHED
94 94
95static void sched_destroy_user(struct user_struct *up) 95static void sched_destroy_user(struct user_struct *up)
96{ 96{
@@ -113,15 +113,15 @@ static void sched_switch_user(struct task_struct *p)
113 sched_move_task(p); 113 sched_move_task(p);
114} 114}
115 115
116#else /* CONFIG_FAIR_USER_SCHED */ 116#else /* CONFIG_USER_SCHED */
117 117
118static void sched_destroy_user(struct user_struct *up) { } 118static void sched_destroy_user(struct user_struct *up) { }
119static int sched_create_user(struct user_struct *up) { return 0; } 119static int sched_create_user(struct user_struct *up) { return 0; }
120static void sched_switch_user(struct task_struct *p) { } 120static void sched_switch_user(struct task_struct *p) { }
121 121
122#endif /* CONFIG_FAIR_USER_SCHED */ 122#endif /* CONFIG_USER_SCHED */
123 123
124#if defined(CONFIG_FAIR_USER_SCHED) && defined(CONFIG_SYSFS) 124#if defined(CONFIG_USER_SCHED) && defined(CONFIG_SYSFS)
125 125
126static struct kset *uids_kset; /* represents the /sys/kernel/uids/ directory */ 126static struct kset *uids_kset; /* represents the /sys/kernel/uids/ directory */
127static DEFINE_MUTEX(uids_mutex); 127static DEFINE_MUTEX(uids_mutex);
@@ -137,6 +137,7 @@ static inline void uids_mutex_unlock(void)
137} 137}
138 138
139/* uid directory attributes */ 139/* uid directory attributes */
140#ifdef CONFIG_FAIR_GROUP_SCHED
140static ssize_t cpu_shares_show(struct kobject *kobj, 141static ssize_t cpu_shares_show(struct kobject *kobj,
141 struct kobj_attribute *attr, 142 struct kobj_attribute *attr,
142 char *buf) 143 char *buf)
@@ -163,7 +164,9 @@ static ssize_t cpu_shares_store(struct kobject *kobj,
163 164
164static struct kobj_attribute cpu_share_attr = 165static struct kobj_attribute cpu_share_attr =
165 __ATTR(cpu_share, 0644, cpu_shares_show, cpu_shares_store); 166 __ATTR(cpu_share, 0644, cpu_shares_show, cpu_shares_store);
167#endif
166 168
169#ifdef CONFIG_RT_GROUP_SCHED
167static ssize_t cpu_rt_runtime_show(struct kobject *kobj, 170static ssize_t cpu_rt_runtime_show(struct kobject *kobj,
168 struct kobj_attribute *attr, 171 struct kobj_attribute *attr,
169 char *buf) 172 char *buf)
@@ -190,11 +193,16 @@ static ssize_t cpu_rt_runtime_store(struct kobject *kobj,
190 193
191static struct kobj_attribute cpu_rt_runtime_attr = 194static struct kobj_attribute cpu_rt_runtime_attr =
192 __ATTR(cpu_rt_runtime, 0644, cpu_rt_runtime_show, cpu_rt_runtime_store); 195 __ATTR(cpu_rt_runtime, 0644, cpu_rt_runtime_show, cpu_rt_runtime_store);
196#endif
193 197
194/* default attributes per uid directory */ 198/* default attributes per uid directory */
195static struct attribute *uids_attributes[] = { 199static struct attribute *uids_attributes[] = {
200#ifdef CONFIG_FAIR_GROUP_SCHED
196 &cpu_share_attr.attr, 201 &cpu_share_attr.attr,
202#endif
203#ifdef CONFIG_RT_GROUP_SCHED
197 &cpu_rt_runtime_attr.attr, 204 &cpu_rt_runtime_attr.attr,
205#endif
198 NULL 206 NULL
199}; 207};
200 208
@@ -297,7 +305,7 @@ static inline void free_user(struct user_struct *up, unsigned long flags)
297 schedule_work(&up->work); 305 schedule_work(&up->work);
298} 306}
299 307
300#else /* CONFIG_FAIR_USER_SCHED && CONFIG_SYSFS */ 308#else /* CONFIG_USER_SCHED && CONFIG_SYSFS */
301 309
302int uids_sysfs_init(void) { return 0; } 310int uids_sysfs_init(void) { return 0; }
303static inline int uids_user_create(struct user_struct *up) { return 0; } 311static inline int uids_user_create(struct user_struct *up) { return 0; }
@@ -401,7 +409,7 @@ struct user_struct * alloc_uid(struct user_namespace *ns, uid_t uid)
401 spin_lock_irq(&uidhash_lock); 409 spin_lock_irq(&uidhash_lock);
402 up = uid_hash_find(uid, hashent); 410 up = uid_hash_find(uid, hashent);
403 if (up) { 411 if (up) {
404 /* This case is not possible when CONFIG_FAIR_USER_SCHED 412 /* This case is not possible when CONFIG_USER_SCHED
405 * is defined, since we serialize alloc_uid() using 413 * is defined, since we serialize alloc_uid() using
406 * uids_mutex. Hence no need to call 414 * uids_mutex. Hence no need to call
407 * sched_destroy_user() or remove_user_sysfs_dir(). 415 * sched_destroy_user() or remove_user_sysfs_dir().