aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
authorPeter Zijlstra <a.p.zijlstra@chello.nl>2008-02-13 09:45:40 -0500
committerIngo Molnar <mingo@elte.hu>2008-02-13 09:45:40 -0500
commit052f1dc7eb02300b05170ae341ccd03b76207778 (patch)
treef58630b7876ae9e4308c0577e36aa13318b7bcfc /kernel
parent9f0c1e560c43327b70998e6c702b2f01321130d9 (diff)
sched: rt-group: make rt groups scheduling configurable
Make the rt group scheduler compile time configurable. Keep it experimental for now. Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'kernel')
-rw-r--r--kernel/sched.c148
-rw-r--r--kernel/sched_rt.c12
-rw-r--r--kernel/user.c22
3 files changed, 126 insertions, 56 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index 85a5fbff2b00..5edc549edae8 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -155,7 +155,7 @@ struct rt_prio_array {
155 struct list_head queue[MAX_RT_PRIO]; 155 struct list_head queue[MAX_RT_PRIO];
156}; 156};
157 157
158#ifdef CONFIG_FAIR_GROUP_SCHED 158#ifdef CONFIG_GROUP_SCHED
159 159
160#include <linux/cgroup.h> 160#include <linux/cgroup.h>
161 161
@@ -165,19 +165,16 @@ static LIST_HEAD(task_groups);
165 165
166/* task group related information */ 166/* task group related information */
167struct task_group { 167struct task_group {
168#ifdef CONFIG_FAIR_CGROUP_SCHED 168#ifdef CONFIG_CGROUP_SCHED
169 struct cgroup_subsys_state css; 169 struct cgroup_subsys_state css;
170#endif 170#endif
171
172#ifdef CONFIG_FAIR_GROUP_SCHED
171 /* schedulable entities of this group on each cpu */ 173 /* schedulable entities of this group on each cpu */
172 struct sched_entity **se; 174 struct sched_entity **se;
173 /* runqueue "owned" by this group on each cpu */ 175 /* runqueue "owned" by this group on each cpu */
174 struct cfs_rq **cfs_rq; 176 struct cfs_rq **cfs_rq;
175 177
176 struct sched_rt_entity **rt_se;
177 struct rt_rq **rt_rq;
178
179 u64 rt_runtime;
180
181 /* 178 /*
182 * shares assigned to a task group governs how much of cpu bandwidth 179 * shares assigned to a task group governs how much of cpu bandwidth
183 * is allocated to the group. The more shares a group has, the more is 180 * is allocated to the group. The more shares a group has, the more is
@@ -213,24 +210,36 @@ struct task_group {
213 * 210 *
214 */ 211 */
215 unsigned long shares; 212 unsigned long shares;
213#endif
214
215#ifdef CONFIG_RT_GROUP_SCHED
216 struct sched_rt_entity **rt_se;
217 struct rt_rq **rt_rq;
218
219 u64 rt_runtime;
220#endif
216 221
217 struct rcu_head rcu; 222 struct rcu_head rcu;
218 struct list_head list; 223 struct list_head list;
219}; 224};
220 225
226#ifdef CONFIG_FAIR_GROUP_SCHED
221/* Default task group's sched entity on each cpu */ 227/* Default task group's sched entity on each cpu */
222static DEFINE_PER_CPU(struct sched_entity, init_sched_entity); 228static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
223/* Default task group's cfs_rq on each cpu */ 229/* Default task group's cfs_rq on each cpu */
224static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp; 230static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
225 231
226static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
227static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
228
229static struct sched_entity *init_sched_entity_p[NR_CPUS]; 232static struct sched_entity *init_sched_entity_p[NR_CPUS];
230static struct cfs_rq *init_cfs_rq_p[NR_CPUS]; 233static struct cfs_rq *init_cfs_rq_p[NR_CPUS];
234#endif
235
236#ifdef CONFIG_RT_GROUP_SCHED
237static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
238static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
231 239
232static struct sched_rt_entity *init_sched_rt_entity_p[NR_CPUS]; 240static struct sched_rt_entity *init_sched_rt_entity_p[NR_CPUS];
233static struct rt_rq *init_rt_rq_p[NR_CPUS]; 241static struct rt_rq *init_rt_rq_p[NR_CPUS];
242#endif
234 243
235/* task_group_lock serializes add/remove of task groups and also changes to 244/* task_group_lock serializes add/remove of task groups and also changes to
236 * a task group's cpu shares. 245 * a task group's cpu shares.
@@ -240,6 +249,7 @@ static DEFINE_SPINLOCK(task_group_lock);
240/* doms_cur_mutex serializes access to doms_cur[] array */ 249/* doms_cur_mutex serializes access to doms_cur[] array */
241static DEFINE_MUTEX(doms_cur_mutex); 250static DEFINE_MUTEX(doms_cur_mutex);
242 251
252#ifdef CONFIG_FAIR_GROUP_SCHED
243#ifdef CONFIG_SMP 253#ifdef CONFIG_SMP
244/* kernel thread that runs rebalance_shares() periodically */ 254/* kernel thread that runs rebalance_shares() periodically */
245static struct task_struct *lb_monitor_task; 255static struct task_struct *lb_monitor_task;
@@ -248,35 +258,40 @@ static int load_balance_monitor(void *unused);
248 258
249static void set_se_shares(struct sched_entity *se, unsigned long shares); 259static void set_se_shares(struct sched_entity *se, unsigned long shares);
250 260
261#ifdef CONFIG_USER_SCHED
262# define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD)
263#else
264# define INIT_TASK_GROUP_LOAD NICE_0_LOAD
265#endif
266
267#define MIN_GROUP_SHARES 2
268
269static int init_task_group_load = INIT_TASK_GROUP_LOAD;
270#endif
271
251/* Default task group. 272/* Default task group.
252 * Every task in system belong to this group at bootup. 273 * Every task in system belong to this group at bootup.
253 */ 274 */
254struct task_group init_task_group = { 275struct task_group init_task_group = {
276#ifdef CONFIG_FAIR_GROUP_SCHED
255 .se = init_sched_entity_p, 277 .se = init_sched_entity_p,
256 .cfs_rq = init_cfs_rq_p, 278 .cfs_rq = init_cfs_rq_p,
279#endif
257 280
281#ifdef CONFIG_RT_GROUP_SCHED
258 .rt_se = init_sched_rt_entity_p, 282 .rt_se = init_sched_rt_entity_p,
259 .rt_rq = init_rt_rq_p, 283 .rt_rq = init_rt_rq_p,
260};
261
262#ifdef CONFIG_FAIR_USER_SCHED
263# define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD)
264#else
265# define INIT_TASK_GROUP_LOAD NICE_0_LOAD
266#endif 284#endif
267 285};
268#define MIN_GROUP_SHARES 2
269
270static int init_task_group_load = INIT_TASK_GROUP_LOAD;
271 286
272/* return group to which a task belongs */ 287/* return group to which a task belongs */
273static inline struct task_group *task_group(struct task_struct *p) 288static inline struct task_group *task_group(struct task_struct *p)
274{ 289{
275 struct task_group *tg; 290 struct task_group *tg;
276 291
277#ifdef CONFIG_FAIR_USER_SCHED 292#ifdef CONFIG_USER_SCHED
278 tg = p->user->tg; 293 tg = p->user->tg;
279#elif defined(CONFIG_FAIR_CGROUP_SCHED) 294#elif defined(CONFIG_CGROUP_SCHED)
280 tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id), 295 tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
281 struct task_group, css); 296 struct task_group, css);
282#else 297#else
@@ -288,11 +303,15 @@ static inline struct task_group *task_group(struct task_struct *p)
288/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ 303/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
289static inline void set_task_rq(struct task_struct *p, unsigned int cpu) 304static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
290{ 305{
306#ifdef CONFIG_FAIR_GROUP_SCHED
291 p->se.cfs_rq = task_group(p)->cfs_rq[cpu]; 307 p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
292 p->se.parent = task_group(p)->se[cpu]; 308 p->se.parent = task_group(p)->se[cpu];
309#endif
293 310
311#ifdef CONFIG_RT_GROUP_SCHED
294 p->rt.rt_rq = task_group(p)->rt_rq[cpu]; 312 p->rt.rt_rq = task_group(p)->rt_rq[cpu];
295 p->rt.parent = task_group(p)->rt_se[cpu]; 313 p->rt.parent = task_group(p)->rt_se[cpu];
314#endif
296} 315}
297 316
298static inline void lock_doms_cur(void) 317static inline void lock_doms_cur(void)
@@ -311,7 +330,7 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
311static inline void lock_doms_cur(void) { } 330static inline void lock_doms_cur(void) { }
312static inline void unlock_doms_cur(void) { } 331static inline void unlock_doms_cur(void) { }
313 332
314#endif /* CONFIG_FAIR_GROUP_SCHED */ 333#endif /* CONFIG_GROUP_SCHED */
315 334
316/* CFS-related fields in a runqueue */ 335/* CFS-related fields in a runqueue */
317struct cfs_rq { 336struct cfs_rq {
@@ -351,7 +370,7 @@ struct cfs_rq {
351struct rt_rq { 370struct rt_rq {
352 struct rt_prio_array active; 371 struct rt_prio_array active;
353 unsigned long rt_nr_running; 372 unsigned long rt_nr_running;
354#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED 373#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
355 int highest_prio; /* highest queued rt task prio */ 374 int highest_prio; /* highest queued rt task prio */
356#endif 375#endif
357#ifdef CONFIG_SMP 376#ifdef CONFIG_SMP
@@ -361,7 +380,7 @@ struct rt_rq {
361 int rt_throttled; 380 int rt_throttled;
362 u64 rt_time; 381 u64 rt_time;
363 382
364#ifdef CONFIG_FAIR_GROUP_SCHED 383#ifdef CONFIG_RT_GROUP_SCHED
365 unsigned long rt_nr_boosted; 384 unsigned long rt_nr_boosted;
366 385
367 struct rq *rq; 386 struct rq *rq;
@@ -437,6 +456,8 @@ struct rq {
437#ifdef CONFIG_FAIR_GROUP_SCHED 456#ifdef CONFIG_FAIR_GROUP_SCHED
438 /* list of leaf cfs_rq on this cpu: */ 457 /* list of leaf cfs_rq on this cpu: */
439 struct list_head leaf_cfs_rq_list; 458 struct list_head leaf_cfs_rq_list;
459#endif
460#ifdef CONFIG_RT_GROUP_SCHED
440 struct list_head leaf_rt_rq_list; 461 struct list_head leaf_rt_rq_list;
441#endif 462#endif
442 463
@@ -7104,7 +7125,7 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
7104 /* delimiter for bitsearch: */ 7125 /* delimiter for bitsearch: */
7105 __set_bit(MAX_RT_PRIO, array->bitmap); 7126 __set_bit(MAX_RT_PRIO, array->bitmap);
7106 7127
7107#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED 7128#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
7108 rt_rq->highest_prio = MAX_RT_PRIO; 7129 rt_rq->highest_prio = MAX_RT_PRIO;
7109#endif 7130#endif
7110#ifdef CONFIG_SMP 7131#ifdef CONFIG_SMP
@@ -7115,7 +7136,7 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
7115 rt_rq->rt_time = 0; 7136 rt_rq->rt_time = 0;
7116 rt_rq->rt_throttled = 0; 7137 rt_rq->rt_throttled = 0;
7117 7138
7118#ifdef CONFIG_FAIR_GROUP_SCHED 7139#ifdef CONFIG_RT_GROUP_SCHED
7119 rt_rq->rt_nr_boosted = 0; 7140 rt_rq->rt_nr_boosted = 0;
7120 rt_rq->rq = rq; 7141 rt_rq->rq = rq;
7121#endif 7142#endif
@@ -7139,7 +7160,9 @@ static void init_tg_cfs_entry(struct rq *rq, struct task_group *tg,
7139 se->load.inv_weight = div64_64(1ULL<<32, se->load.weight); 7160 se->load.inv_weight = div64_64(1ULL<<32, se->load.weight);
7140 se->parent = NULL; 7161 se->parent = NULL;
7141} 7162}
7163#endif
7142 7164
7165#ifdef CONFIG_RT_GROUP_SCHED
7143static void init_tg_rt_entry(struct rq *rq, struct task_group *tg, 7166static void init_tg_rt_entry(struct rq *rq, struct task_group *tg,
7144 struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, 7167 struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
7145 int cpu, int add) 7168 int cpu, int add)
@@ -7168,7 +7191,7 @@ void __init sched_init(void)
7168 init_defrootdomain(); 7191 init_defrootdomain();
7169#endif 7192#endif
7170 7193
7171#ifdef CONFIG_FAIR_GROUP_SCHED 7194#ifdef CONFIG_GROUP_SCHED
7172 list_add(&init_task_group.list, &task_groups); 7195 list_add(&init_task_group.list, &task_groups);
7173#endif 7196#endif
7174 7197
@@ -7189,6 +7212,8 @@ void __init sched_init(void)
7189 &per_cpu(init_cfs_rq, i), 7212 &per_cpu(init_cfs_rq, i),
7190 &per_cpu(init_sched_entity, i), i, 1); 7213 &per_cpu(init_sched_entity, i), i, 1);
7191 7214
7215#endif
7216#ifdef CONFIG_RT_GROUP_SCHED
7192 init_task_group.rt_runtime = 7217 init_task_group.rt_runtime =
7193 sysctl_sched_rt_runtime * NSEC_PER_USEC; 7218 sysctl_sched_rt_runtime * NSEC_PER_USEC;
7194 INIT_LIST_HEAD(&rq->leaf_rt_rq_list); 7219 INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
@@ -7381,9 +7406,9 @@ void set_curr_task(int cpu, struct task_struct *p)
7381 7406
7382#endif 7407#endif
7383 7408
7384#ifdef CONFIG_FAIR_GROUP_SCHED 7409#ifdef CONFIG_GROUP_SCHED
7385 7410
7386#ifdef CONFIG_SMP 7411#if defined CONFIG_FAIR_GROUP_SCHED && defined CONFIG_SMP
7387/* 7412/*
7388 * distribute shares of all task groups among their schedulable entities, 7413 * distribute shares of all task groups among their schedulable entities,
7389 * to reflect load distribution across cpus. 7414 * to reflect load distribution across cpus.
@@ -7539,20 +7564,28 @@ static void free_sched_group(struct task_group *tg)
7539 int i; 7564 int i;
7540 7565
7541 for_each_possible_cpu(i) { 7566 for_each_possible_cpu(i) {
7567#ifdef CONFIG_FAIR_GROUP_SCHED
7542 if (tg->cfs_rq) 7568 if (tg->cfs_rq)
7543 kfree(tg->cfs_rq[i]); 7569 kfree(tg->cfs_rq[i]);
7544 if (tg->se) 7570 if (tg->se)
7545 kfree(tg->se[i]); 7571 kfree(tg->se[i]);
7572#endif
7573#ifdef CONFIG_RT_GROUP_SCHED
7546 if (tg->rt_rq) 7574 if (tg->rt_rq)
7547 kfree(tg->rt_rq[i]); 7575 kfree(tg->rt_rq[i]);
7548 if (tg->rt_se) 7576 if (tg->rt_se)
7549 kfree(tg->rt_se[i]); 7577 kfree(tg->rt_se[i]);
7578#endif
7550 } 7579 }
7551 7580
7581#ifdef CONFIG_FAIR_GROUP_SCHED
7552 kfree(tg->cfs_rq); 7582 kfree(tg->cfs_rq);
7553 kfree(tg->se); 7583 kfree(tg->se);
7584#endif
7585#ifdef CONFIG_RT_GROUP_SCHED
7554 kfree(tg->rt_rq); 7586 kfree(tg->rt_rq);
7555 kfree(tg->rt_se); 7587 kfree(tg->rt_se);
7588#endif
7556 kfree(tg); 7589 kfree(tg);
7557} 7590}
7558 7591
@@ -7560,10 +7593,14 @@ static void free_sched_group(struct task_group *tg)
7560struct task_group *sched_create_group(void) 7593struct task_group *sched_create_group(void)
7561{ 7594{
7562 struct task_group *tg; 7595 struct task_group *tg;
7596#ifdef CONFIG_FAIR_GROUP_SCHED
7563 struct cfs_rq *cfs_rq; 7597 struct cfs_rq *cfs_rq;
7564 struct sched_entity *se; 7598 struct sched_entity *se;
7599#endif
7600#ifdef CONFIG_RT_GROUP_SCHED
7565 struct rt_rq *rt_rq; 7601 struct rt_rq *rt_rq;
7566 struct sched_rt_entity *rt_se; 7602 struct sched_rt_entity *rt_se;
7603#endif
7567 struct rq *rq; 7604 struct rq *rq;
7568 unsigned long flags; 7605 unsigned long flags;
7569 int i; 7606 int i;
@@ -7572,12 +7609,18 @@ struct task_group *sched_create_group(void)
7572 if (!tg) 7609 if (!tg)
7573 return ERR_PTR(-ENOMEM); 7610 return ERR_PTR(-ENOMEM);
7574 7611
7612#ifdef CONFIG_FAIR_GROUP_SCHED
7575 tg->cfs_rq = kzalloc(sizeof(cfs_rq) * NR_CPUS, GFP_KERNEL); 7613 tg->cfs_rq = kzalloc(sizeof(cfs_rq) * NR_CPUS, GFP_KERNEL);
7576 if (!tg->cfs_rq) 7614 if (!tg->cfs_rq)
7577 goto err; 7615 goto err;
7578 tg->se = kzalloc(sizeof(se) * NR_CPUS, GFP_KERNEL); 7616 tg->se = kzalloc(sizeof(se) * NR_CPUS, GFP_KERNEL);
7579 if (!tg->se) 7617 if (!tg->se)
7580 goto err; 7618 goto err;
7619
7620 tg->shares = NICE_0_LOAD;
7621#endif
7622
7623#ifdef CONFIG_RT_GROUP_SCHED
7581 tg->rt_rq = kzalloc(sizeof(rt_rq) * NR_CPUS, GFP_KERNEL); 7624 tg->rt_rq = kzalloc(sizeof(rt_rq) * NR_CPUS, GFP_KERNEL);
7582 if (!tg->rt_rq) 7625 if (!tg->rt_rq)
7583 goto err; 7626 goto err;
@@ -7585,12 +7628,13 @@ struct task_group *sched_create_group(void)
7585 if (!tg->rt_se) 7628 if (!tg->rt_se)
7586 goto err; 7629 goto err;
7587 7630
7588 tg->shares = NICE_0_LOAD;
7589 tg->rt_runtime = 0; 7631 tg->rt_runtime = 0;
7632#endif
7590 7633
7591 for_each_possible_cpu(i) { 7634 for_each_possible_cpu(i) {
7592 rq = cpu_rq(i); 7635 rq = cpu_rq(i);
7593 7636
7637#ifdef CONFIG_FAIR_GROUP_SCHED
7594 cfs_rq = kmalloc_node(sizeof(struct cfs_rq), 7638 cfs_rq = kmalloc_node(sizeof(struct cfs_rq),
7595 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 7639 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i));
7596 if (!cfs_rq) 7640 if (!cfs_rq)
@@ -7601,6 +7645,10 @@ struct task_group *sched_create_group(void)
7601 if (!se) 7645 if (!se)
7602 goto err; 7646 goto err;
7603 7647
7648 init_tg_cfs_entry(rq, tg, cfs_rq, se, i, 0);
7649#endif
7650
7651#ifdef CONFIG_RT_GROUP_SCHED
7604 rt_rq = kmalloc_node(sizeof(struct rt_rq), 7652 rt_rq = kmalloc_node(sizeof(struct rt_rq),
7605 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 7653 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i));
7606 if (!rt_rq) 7654 if (!rt_rq)
@@ -7611,17 +7659,21 @@ struct task_group *sched_create_group(void)
7611 if (!rt_se) 7659 if (!rt_se)
7612 goto err; 7660 goto err;
7613 7661
7614 init_tg_cfs_entry(rq, tg, cfs_rq, se, i, 0);
7615 init_tg_rt_entry(rq, tg, rt_rq, rt_se, i, 0); 7662 init_tg_rt_entry(rq, tg, rt_rq, rt_se, i, 0);
7663#endif
7616 } 7664 }
7617 7665
7618 spin_lock_irqsave(&task_group_lock, flags); 7666 spin_lock_irqsave(&task_group_lock, flags);
7619 for_each_possible_cpu(i) { 7667 for_each_possible_cpu(i) {
7620 rq = cpu_rq(i); 7668 rq = cpu_rq(i);
7669#ifdef CONFIG_FAIR_GROUP_SCHED
7621 cfs_rq = tg->cfs_rq[i]; 7670 cfs_rq = tg->cfs_rq[i];
7622 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list); 7671 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
7672#endif
7673#ifdef CONFIG_RT_GROUP_SCHED
7623 rt_rq = tg->rt_rq[i]; 7674 rt_rq = tg->rt_rq[i];
7624 list_add_rcu(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list); 7675 list_add_rcu(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list);
7676#endif
7625 } 7677 }
7626 list_add_rcu(&tg->list, &task_groups); 7678 list_add_rcu(&tg->list, &task_groups);
7627 spin_unlock_irqrestore(&task_group_lock, flags); 7679 spin_unlock_irqrestore(&task_group_lock, flags);
@@ -7643,23 +7695,21 @@ static void free_sched_group_rcu(struct rcu_head *rhp)
7643/* Destroy runqueue etc associated with a task group */ 7695/* Destroy runqueue etc associated with a task group */
7644void sched_destroy_group(struct task_group *tg) 7696void sched_destroy_group(struct task_group *tg)
7645{ 7697{
7646 struct cfs_rq *cfs_rq = NULL;
7647 struct rt_rq *rt_rq = NULL;
7648 unsigned long flags; 7698 unsigned long flags;
7649 int i; 7699 int i;
7650 7700
7651 spin_lock_irqsave(&task_group_lock, flags); 7701 spin_lock_irqsave(&task_group_lock, flags);
7652 for_each_possible_cpu(i) { 7702 for_each_possible_cpu(i) {
7653 cfs_rq = tg->cfs_rq[i]; 7703#ifdef CONFIG_FAIR_GROUP_SCHED
7654 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 7704 list_del_rcu(&tg->cfs_rq[i]->leaf_cfs_rq_list);
7655 rt_rq = tg->rt_rq[i]; 7705#endif
7656 list_del_rcu(&rt_rq->leaf_rt_rq_list); 7706#ifdef CONFIG_RT_GROUP_SCHED
7707 list_del_rcu(&tg->rt_rq[i]->leaf_rt_rq_list);
7708#endif
7657 } 7709 }
7658 list_del_rcu(&tg->list); 7710 list_del_rcu(&tg->list);
7659 spin_unlock_irqrestore(&task_group_lock, flags); 7711 spin_unlock_irqrestore(&task_group_lock, flags);
7660 7712
7661 BUG_ON(!cfs_rq);
7662
7663 /* wait for possible concurrent references to cfs_rqs complete */ 7713 /* wait for possible concurrent references to cfs_rqs complete */
7664 call_rcu(&tg->rcu, free_sched_group_rcu); 7714 call_rcu(&tg->rcu, free_sched_group_rcu);
7665} 7715}
@@ -7699,6 +7749,7 @@ void sched_move_task(struct task_struct *tsk)
7699 task_rq_unlock(rq, &flags); 7749 task_rq_unlock(rq, &flags);
7700} 7750}
7701 7751
7752#ifdef CONFIG_FAIR_GROUP_SCHED
7702/* rq->lock to be locked by caller */ 7753/* rq->lock to be locked by caller */
7703static void set_se_shares(struct sched_entity *se, unsigned long shares) 7754static void set_se_shares(struct sched_entity *se, unsigned long shares)
7704{ 7755{
@@ -7786,7 +7837,9 @@ unsigned long sched_group_shares(struct task_group *tg)
7786{ 7837{
7787 return tg->shares; 7838 return tg->shares;
7788} 7839}
7840#endif
7789 7841
7842#ifdef CONFIG_RT_GROUP_SCHED
7790/* 7843/*
7791 * Ensure that the real time constraints are schedulable. 7844 * Ensure that the real time constraints are schedulable.
7792 */ 7845 */
@@ -7858,9 +7911,10 @@ long sched_group_rt_runtime(struct task_group *tg)
7858 do_div(rt_runtime_us, NSEC_PER_USEC); 7911 do_div(rt_runtime_us, NSEC_PER_USEC);
7859 return rt_runtime_us; 7912 return rt_runtime_us;
7860} 7913}
7861#endif /* CONFIG_FAIR_GROUP_SCHED */ 7914#endif
7915#endif /* CONFIG_GROUP_SCHED */
7862 7916
7863#ifdef CONFIG_FAIR_CGROUP_SCHED 7917#ifdef CONFIG_CGROUP_SCHED
7864 7918
7865/* return corresponding task_group object of a cgroup */ 7919/* return corresponding task_group object of a cgroup */
7866static inline struct task_group *cgroup_tg(struct cgroup *cgrp) 7920static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
@@ -7920,6 +7974,7 @@ cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
7920 sched_move_task(tsk); 7974 sched_move_task(tsk);
7921} 7975}
7922 7976
7977#ifdef CONFIG_FAIR_GROUP_SCHED
7923static int cpu_shares_write_uint(struct cgroup *cgrp, struct cftype *cftype, 7978static int cpu_shares_write_uint(struct cgroup *cgrp, struct cftype *cftype,
7924 u64 shareval) 7979 u64 shareval)
7925{ 7980{
@@ -7932,7 +7987,9 @@ static u64 cpu_shares_read_uint(struct cgroup *cgrp, struct cftype *cft)
7932 7987
7933 return (u64) tg->shares; 7988 return (u64) tg->shares;
7934} 7989}
7990#endif
7935 7991
7992#ifdef CONFIG_RT_GROUP_SCHED
7936static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft, 7993static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
7937 struct file *file, 7994 struct file *file,
7938 const char __user *userbuf, 7995 const char __user *userbuf,
@@ -7977,18 +8034,23 @@ static ssize_t cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft,
7977 8034
7978 return simple_read_from_buffer(buf, nbytes, ppos, tmp, len); 8035 return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
7979} 8036}
8037#endif
7980 8038
7981static struct cftype cpu_files[] = { 8039static struct cftype cpu_files[] = {
8040#ifdef CONFIG_FAIR_GROUP_SCHED
7982 { 8041 {
7983 .name = "shares", 8042 .name = "shares",
7984 .read_uint = cpu_shares_read_uint, 8043 .read_uint = cpu_shares_read_uint,
7985 .write_uint = cpu_shares_write_uint, 8044 .write_uint = cpu_shares_write_uint,
7986 }, 8045 },
8046#endif
8047#ifdef CONFIG_RT_GROUP_SCHED
7987 { 8048 {
7988 .name = "rt_runtime_us", 8049 .name = "rt_runtime_us",
7989 .read = cpu_rt_runtime_read, 8050 .read = cpu_rt_runtime_read,
7990 .write = cpu_rt_runtime_write, 8051 .write = cpu_rt_runtime_write,
7991 }, 8052 },
8053#endif
7992}; 8054};
7993 8055
7994static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont) 8056static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont)
@@ -8007,7 +8069,7 @@ struct cgroup_subsys cpu_cgroup_subsys = {
8007 .early_init = 1, 8069 .early_init = 1,
8008}; 8070};
8009 8071
8010#endif /* CONFIG_FAIR_CGROUP_SCHED */ 8072#endif /* CONFIG_CGROUP_SCHED */
8011 8073
8012#ifdef CONFIG_CGROUP_CPUACCT 8074#ifdef CONFIG_CGROUP_CPUACCT
8013 8075
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index 35825b28e429..f54792b175b2 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -55,7 +55,7 @@ static inline int on_rt_rq(struct sched_rt_entity *rt_se)
55 return !list_empty(&rt_se->run_list); 55 return !list_empty(&rt_se->run_list);
56} 56}
57 57
58#ifdef CONFIG_FAIR_GROUP_SCHED 58#ifdef CONFIG_RT_GROUP_SCHED
59 59
60static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) 60static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
61{ 61{
@@ -177,7 +177,7 @@ static inline int rt_rq_throttled(struct rt_rq *rt_rq)
177 177
178static inline int rt_se_prio(struct sched_rt_entity *rt_se) 178static inline int rt_se_prio(struct sched_rt_entity *rt_se)
179{ 179{
180#ifdef CONFIG_FAIR_GROUP_SCHED 180#ifdef CONFIG_RT_GROUP_SCHED
181 struct rt_rq *rt_rq = group_rt_rq(rt_se); 181 struct rt_rq *rt_rq = group_rt_rq(rt_se);
182 182
183 if (rt_rq) 183 if (rt_rq)
@@ -269,7 +269,7 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
269{ 269{
270 WARN_ON(!rt_prio(rt_se_prio(rt_se))); 270 WARN_ON(!rt_prio(rt_se_prio(rt_se)));
271 rt_rq->rt_nr_running++; 271 rt_rq->rt_nr_running++;
272#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED 272#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
273 if (rt_se_prio(rt_se) < rt_rq->highest_prio) 273 if (rt_se_prio(rt_se) < rt_rq->highest_prio)
274 rt_rq->highest_prio = rt_se_prio(rt_se); 274 rt_rq->highest_prio = rt_se_prio(rt_se);
275#endif 275#endif
@@ -281,7 +281,7 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
281 281
282 update_rt_migration(rq_of_rt_rq(rt_rq)); 282 update_rt_migration(rq_of_rt_rq(rt_rq));
283#endif 283#endif
284#ifdef CONFIG_FAIR_GROUP_SCHED 284#ifdef CONFIG_RT_GROUP_SCHED
285 if (rt_se_boosted(rt_se)) 285 if (rt_se_boosted(rt_se))
286 rt_rq->rt_nr_boosted++; 286 rt_rq->rt_nr_boosted++;
287#endif 287#endif
@@ -293,7 +293,7 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
293 WARN_ON(!rt_prio(rt_se_prio(rt_se))); 293 WARN_ON(!rt_prio(rt_se_prio(rt_se)));
294 WARN_ON(!rt_rq->rt_nr_running); 294 WARN_ON(!rt_rq->rt_nr_running);
295 rt_rq->rt_nr_running--; 295 rt_rq->rt_nr_running--;
296#if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED 296#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
297 if (rt_rq->rt_nr_running) { 297 if (rt_rq->rt_nr_running) {
298 struct rt_prio_array *array; 298 struct rt_prio_array *array;
299 299
@@ -315,7 +315,7 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
315 315
316 update_rt_migration(rq_of_rt_rq(rt_rq)); 316 update_rt_migration(rq_of_rt_rq(rt_rq));
317#endif /* CONFIG_SMP */ 317#endif /* CONFIG_SMP */
318#ifdef CONFIG_FAIR_GROUP_SCHED 318#ifdef CONFIG_RT_GROUP_SCHED
319 if (rt_se_boosted(rt_se)) 319 if (rt_se_boosted(rt_se))
320 rt_rq->rt_nr_boosted--; 320 rt_rq->rt_nr_boosted--;
321 321
diff --git a/kernel/user.c b/kernel/user.c
index 9f6d471bfd03..7132022a040c 100644
--- a/kernel/user.c
+++ b/kernel/user.c
@@ -57,7 +57,7 @@ struct user_struct root_user = {
57 .uid_keyring = &root_user_keyring, 57 .uid_keyring = &root_user_keyring,
58 .session_keyring = &root_session_keyring, 58 .session_keyring = &root_session_keyring,
59#endif 59#endif
60#ifdef CONFIG_FAIR_USER_SCHED 60#ifdef CONFIG_USER_SCHED
61 .tg = &init_task_group, 61 .tg = &init_task_group,
62#endif 62#endif
63}; 63};
@@ -90,7 +90,7 @@ static struct user_struct *uid_hash_find(uid_t uid, struct hlist_head *hashent)
90 return NULL; 90 return NULL;
91} 91}
92 92
93#ifdef CONFIG_FAIR_USER_SCHED 93#ifdef CONFIG_USER_SCHED
94 94
95static void sched_destroy_user(struct user_struct *up) 95static void sched_destroy_user(struct user_struct *up)
96{ 96{
@@ -113,15 +113,15 @@ static void sched_switch_user(struct task_struct *p)
113 sched_move_task(p); 113 sched_move_task(p);
114} 114}
115 115
116#else /* CONFIG_FAIR_USER_SCHED */ 116#else /* CONFIG_USER_SCHED */
117 117
118static void sched_destroy_user(struct user_struct *up) { } 118static void sched_destroy_user(struct user_struct *up) { }
119static int sched_create_user(struct user_struct *up) { return 0; } 119static int sched_create_user(struct user_struct *up) { return 0; }
120static void sched_switch_user(struct task_struct *p) { } 120static void sched_switch_user(struct task_struct *p) { }
121 121
122#endif /* CONFIG_FAIR_USER_SCHED */ 122#endif /* CONFIG_USER_SCHED */
123 123
124#if defined(CONFIG_FAIR_USER_SCHED) && defined(CONFIG_SYSFS) 124#if defined(CONFIG_USER_SCHED) && defined(CONFIG_SYSFS)
125 125
126static struct kset *uids_kset; /* represents the /sys/kernel/uids/ directory */ 126static struct kset *uids_kset; /* represents the /sys/kernel/uids/ directory */
127static DEFINE_MUTEX(uids_mutex); 127static DEFINE_MUTEX(uids_mutex);
@@ -137,6 +137,7 @@ static inline void uids_mutex_unlock(void)
137} 137}
138 138
139/* uid directory attributes */ 139/* uid directory attributes */
140#ifdef CONFIG_FAIR_GROUP_SCHED
140static ssize_t cpu_shares_show(struct kobject *kobj, 141static ssize_t cpu_shares_show(struct kobject *kobj,
141 struct kobj_attribute *attr, 142 struct kobj_attribute *attr,
142 char *buf) 143 char *buf)
@@ -163,7 +164,9 @@ static ssize_t cpu_shares_store(struct kobject *kobj,
163 164
164static struct kobj_attribute cpu_share_attr = 165static struct kobj_attribute cpu_share_attr =
165 __ATTR(cpu_share, 0644, cpu_shares_show, cpu_shares_store); 166 __ATTR(cpu_share, 0644, cpu_shares_show, cpu_shares_store);
167#endif
166 168
169#ifdef CONFIG_RT_GROUP_SCHED
167static ssize_t cpu_rt_runtime_show(struct kobject *kobj, 170static ssize_t cpu_rt_runtime_show(struct kobject *kobj,
168 struct kobj_attribute *attr, 171 struct kobj_attribute *attr,
169 char *buf) 172 char *buf)
@@ -190,11 +193,16 @@ static ssize_t cpu_rt_runtime_store(struct kobject *kobj,
190 193
191static struct kobj_attribute cpu_rt_runtime_attr = 194static struct kobj_attribute cpu_rt_runtime_attr =
192 __ATTR(cpu_rt_runtime, 0644, cpu_rt_runtime_show, cpu_rt_runtime_store); 195 __ATTR(cpu_rt_runtime, 0644, cpu_rt_runtime_show, cpu_rt_runtime_store);
196#endif
193 197
194/* default attributes per uid directory */ 198/* default attributes per uid directory */
195static struct attribute *uids_attributes[] = { 199static struct attribute *uids_attributes[] = {
200#ifdef CONFIG_FAIR_GROUP_SCHED
196 &cpu_share_attr.attr, 201 &cpu_share_attr.attr,
202#endif
203#ifdef CONFIG_RT_GROUP_SCHED
197 &cpu_rt_runtime_attr.attr, 204 &cpu_rt_runtime_attr.attr,
205#endif
198 NULL 206 NULL
199}; 207};
200 208
@@ -297,7 +305,7 @@ static inline void free_user(struct user_struct *up, unsigned long flags)
297 schedule_work(&up->work); 305 schedule_work(&up->work);
298} 306}
299 307
300#else /* CONFIG_FAIR_USER_SCHED && CONFIG_SYSFS */ 308#else /* CONFIG_USER_SCHED && CONFIG_SYSFS */
301 309
302int uids_sysfs_init(void) { return 0; } 310int uids_sysfs_init(void) { return 0; }
303static inline int uids_user_create(struct user_struct *up) { return 0; } 311static inline int uids_user_create(struct user_struct *up) { return 0; }
@@ -401,7 +409,7 @@ struct user_struct * alloc_uid(struct user_namespace *ns, uid_t uid)
401 spin_lock_irq(&uidhash_lock); 409 spin_lock_irq(&uidhash_lock);
402 up = uid_hash_find(uid, hashent); 410 up = uid_hash_find(uid, hashent);
403 if (up) { 411 if (up) {
404 /* This case is not possible when CONFIG_FAIR_USER_SCHED 412 /* This case is not possible when CONFIG_USER_SCHED
405 * is defined, since we serialize alloc_uid() using 413 * is defined, since we serialize alloc_uid() using
406 * uids_mutex. Hence no need to call 414 * uids_mutex. Hence no need to call
407 * sched_destroy_user() or remove_user_sysfs_dir(). 415 * sched_destroy_user() or remove_user_sysfs_dir().