diff options
author | Peter Zijlstra <a.p.zijlstra@chello.nl> | 2008-02-13 09:45:40 -0500 |
---|---|---|
committer | Ingo Molnar <mingo@elte.hu> | 2008-02-13 09:45:40 -0500 |
commit | 052f1dc7eb02300b05170ae341ccd03b76207778 (patch) | |
tree | f58630b7876ae9e4308c0577e36aa13318b7bcfc /kernel | |
parent | 9f0c1e560c43327b70998e6c702b2f01321130d9 (diff) |
sched: rt-group: make rt groups scheduling configurable
Make the rt group scheduler compile time configurable.
Keep it experimental for now.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'kernel')
-rw-r--r-- | kernel/sched.c | 148 | ||||
-rw-r--r-- | kernel/sched_rt.c | 12 | ||||
-rw-r--r-- | kernel/user.c | 22 |
3 files changed, 126 insertions, 56 deletions
diff --git a/kernel/sched.c b/kernel/sched.c index 85a5fbff2b00..5edc549edae8 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -155,7 +155,7 @@ struct rt_prio_array { | |||
155 | struct list_head queue[MAX_RT_PRIO]; | 155 | struct list_head queue[MAX_RT_PRIO]; |
156 | }; | 156 | }; |
157 | 157 | ||
158 | #ifdef CONFIG_FAIR_GROUP_SCHED | 158 | #ifdef CONFIG_GROUP_SCHED |
159 | 159 | ||
160 | #include <linux/cgroup.h> | 160 | #include <linux/cgroup.h> |
161 | 161 | ||
@@ -165,19 +165,16 @@ static LIST_HEAD(task_groups); | |||
165 | 165 | ||
166 | /* task group related information */ | 166 | /* task group related information */ |
167 | struct task_group { | 167 | struct task_group { |
168 | #ifdef CONFIG_FAIR_CGROUP_SCHED | 168 | #ifdef CONFIG_CGROUP_SCHED |
169 | struct cgroup_subsys_state css; | 169 | struct cgroup_subsys_state css; |
170 | #endif | 170 | #endif |
171 | |||
172 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
171 | /* schedulable entities of this group on each cpu */ | 173 | /* schedulable entities of this group on each cpu */ |
172 | struct sched_entity **se; | 174 | struct sched_entity **se; |
173 | /* runqueue "owned" by this group on each cpu */ | 175 | /* runqueue "owned" by this group on each cpu */ |
174 | struct cfs_rq **cfs_rq; | 176 | struct cfs_rq **cfs_rq; |
175 | 177 | ||
176 | struct sched_rt_entity **rt_se; | ||
177 | struct rt_rq **rt_rq; | ||
178 | |||
179 | u64 rt_runtime; | ||
180 | |||
181 | /* | 178 | /* |
182 | * shares assigned to a task group governs how much of cpu bandwidth | 179 | * shares assigned to a task group governs how much of cpu bandwidth |
183 | * is allocated to the group. The more shares a group has, the more is | 180 | * is allocated to the group. The more shares a group has, the more is |
@@ -213,24 +210,36 @@ struct task_group { | |||
213 | * | 210 | * |
214 | */ | 211 | */ |
215 | unsigned long shares; | 212 | unsigned long shares; |
213 | #endif | ||
214 | |||
215 | #ifdef CONFIG_RT_GROUP_SCHED | ||
216 | struct sched_rt_entity **rt_se; | ||
217 | struct rt_rq **rt_rq; | ||
218 | |||
219 | u64 rt_runtime; | ||
220 | #endif | ||
216 | 221 | ||
217 | struct rcu_head rcu; | 222 | struct rcu_head rcu; |
218 | struct list_head list; | 223 | struct list_head list; |
219 | }; | 224 | }; |
220 | 225 | ||
226 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
221 | /* Default task group's sched entity on each cpu */ | 227 | /* Default task group's sched entity on each cpu */ |
222 | static DEFINE_PER_CPU(struct sched_entity, init_sched_entity); | 228 | static DEFINE_PER_CPU(struct sched_entity, init_sched_entity); |
223 | /* Default task group's cfs_rq on each cpu */ | 229 | /* Default task group's cfs_rq on each cpu */ |
224 | static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp; | 230 | static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp; |
225 | 231 | ||
226 | static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity); | ||
227 | static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp; | ||
228 | |||
229 | static struct sched_entity *init_sched_entity_p[NR_CPUS]; | 232 | static struct sched_entity *init_sched_entity_p[NR_CPUS]; |
230 | static struct cfs_rq *init_cfs_rq_p[NR_CPUS]; | 233 | static struct cfs_rq *init_cfs_rq_p[NR_CPUS]; |
234 | #endif | ||
235 | |||
236 | #ifdef CONFIG_RT_GROUP_SCHED | ||
237 | static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity); | ||
238 | static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp; | ||
231 | 239 | ||
232 | static struct sched_rt_entity *init_sched_rt_entity_p[NR_CPUS]; | 240 | static struct sched_rt_entity *init_sched_rt_entity_p[NR_CPUS]; |
233 | static struct rt_rq *init_rt_rq_p[NR_CPUS]; | 241 | static struct rt_rq *init_rt_rq_p[NR_CPUS]; |
242 | #endif | ||
234 | 243 | ||
235 | /* task_group_lock serializes add/remove of task groups and also changes to | 244 | /* task_group_lock serializes add/remove of task groups and also changes to |
236 | * a task group's cpu shares. | 245 | * a task group's cpu shares. |
@@ -240,6 +249,7 @@ static DEFINE_SPINLOCK(task_group_lock); | |||
240 | /* doms_cur_mutex serializes access to doms_cur[] array */ | 249 | /* doms_cur_mutex serializes access to doms_cur[] array */ |
241 | static DEFINE_MUTEX(doms_cur_mutex); | 250 | static DEFINE_MUTEX(doms_cur_mutex); |
242 | 251 | ||
252 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
243 | #ifdef CONFIG_SMP | 253 | #ifdef CONFIG_SMP |
244 | /* kernel thread that runs rebalance_shares() periodically */ | 254 | /* kernel thread that runs rebalance_shares() periodically */ |
245 | static struct task_struct *lb_monitor_task; | 255 | static struct task_struct *lb_monitor_task; |
@@ -248,35 +258,40 @@ static int load_balance_monitor(void *unused); | |||
248 | 258 | ||
249 | static void set_se_shares(struct sched_entity *se, unsigned long shares); | 259 | static void set_se_shares(struct sched_entity *se, unsigned long shares); |
250 | 260 | ||
261 | #ifdef CONFIG_USER_SCHED | ||
262 | # define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD) | ||
263 | #else | ||
264 | # define INIT_TASK_GROUP_LOAD NICE_0_LOAD | ||
265 | #endif | ||
266 | |||
267 | #define MIN_GROUP_SHARES 2 | ||
268 | |||
269 | static int init_task_group_load = INIT_TASK_GROUP_LOAD; | ||
270 | #endif | ||
271 | |||
251 | /* Default task group. | 272 | /* Default task group. |
252 | * Every task in system belong to this group at bootup. | 273 | * Every task in system belong to this group at bootup. |
253 | */ | 274 | */ |
254 | struct task_group init_task_group = { | 275 | struct task_group init_task_group = { |
276 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
255 | .se = init_sched_entity_p, | 277 | .se = init_sched_entity_p, |
256 | .cfs_rq = init_cfs_rq_p, | 278 | .cfs_rq = init_cfs_rq_p, |
279 | #endif | ||
257 | 280 | ||
281 | #ifdef CONFIG_RT_GROUP_SCHED | ||
258 | .rt_se = init_sched_rt_entity_p, | 282 | .rt_se = init_sched_rt_entity_p, |
259 | .rt_rq = init_rt_rq_p, | 283 | .rt_rq = init_rt_rq_p, |
260 | }; | ||
261 | |||
262 | #ifdef CONFIG_FAIR_USER_SCHED | ||
263 | # define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD) | ||
264 | #else | ||
265 | # define INIT_TASK_GROUP_LOAD NICE_0_LOAD | ||
266 | #endif | 284 | #endif |
267 | 285 | }; | |
268 | #define MIN_GROUP_SHARES 2 | ||
269 | |||
270 | static int init_task_group_load = INIT_TASK_GROUP_LOAD; | ||
271 | 286 | ||
272 | /* return group to which a task belongs */ | 287 | /* return group to which a task belongs */ |
273 | static inline struct task_group *task_group(struct task_struct *p) | 288 | static inline struct task_group *task_group(struct task_struct *p) |
274 | { | 289 | { |
275 | struct task_group *tg; | 290 | struct task_group *tg; |
276 | 291 | ||
277 | #ifdef CONFIG_FAIR_USER_SCHED | 292 | #ifdef CONFIG_USER_SCHED |
278 | tg = p->user->tg; | 293 | tg = p->user->tg; |
279 | #elif defined(CONFIG_FAIR_CGROUP_SCHED) | 294 | #elif defined(CONFIG_CGROUP_SCHED) |
280 | tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id), | 295 | tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id), |
281 | struct task_group, css); | 296 | struct task_group, css); |
282 | #else | 297 | #else |
@@ -288,11 +303,15 @@ static inline struct task_group *task_group(struct task_struct *p) | |||
288 | /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ | 303 | /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ |
289 | static inline void set_task_rq(struct task_struct *p, unsigned int cpu) | 304 | static inline void set_task_rq(struct task_struct *p, unsigned int cpu) |
290 | { | 305 | { |
306 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
291 | p->se.cfs_rq = task_group(p)->cfs_rq[cpu]; | 307 | p->se.cfs_rq = task_group(p)->cfs_rq[cpu]; |
292 | p->se.parent = task_group(p)->se[cpu]; | 308 | p->se.parent = task_group(p)->se[cpu]; |
309 | #endif | ||
293 | 310 | ||
311 | #ifdef CONFIG_RT_GROUP_SCHED | ||
294 | p->rt.rt_rq = task_group(p)->rt_rq[cpu]; | 312 | p->rt.rt_rq = task_group(p)->rt_rq[cpu]; |
295 | p->rt.parent = task_group(p)->rt_se[cpu]; | 313 | p->rt.parent = task_group(p)->rt_se[cpu]; |
314 | #endif | ||
296 | } | 315 | } |
297 | 316 | ||
298 | static inline void lock_doms_cur(void) | 317 | static inline void lock_doms_cur(void) |
@@ -311,7 +330,7 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { } | |||
311 | static inline void lock_doms_cur(void) { } | 330 | static inline void lock_doms_cur(void) { } |
312 | static inline void unlock_doms_cur(void) { } | 331 | static inline void unlock_doms_cur(void) { } |
313 | 332 | ||
314 | #endif /* CONFIG_FAIR_GROUP_SCHED */ | 333 | #endif /* CONFIG_GROUP_SCHED */ |
315 | 334 | ||
316 | /* CFS-related fields in a runqueue */ | 335 | /* CFS-related fields in a runqueue */ |
317 | struct cfs_rq { | 336 | struct cfs_rq { |
@@ -351,7 +370,7 @@ struct cfs_rq { | |||
351 | struct rt_rq { | 370 | struct rt_rq { |
352 | struct rt_prio_array active; | 371 | struct rt_prio_array active; |
353 | unsigned long rt_nr_running; | 372 | unsigned long rt_nr_running; |
354 | #if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED | 373 | #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED |
355 | int highest_prio; /* highest queued rt task prio */ | 374 | int highest_prio; /* highest queued rt task prio */ |
356 | #endif | 375 | #endif |
357 | #ifdef CONFIG_SMP | 376 | #ifdef CONFIG_SMP |
@@ -361,7 +380,7 @@ struct rt_rq { | |||
361 | int rt_throttled; | 380 | int rt_throttled; |
362 | u64 rt_time; | 381 | u64 rt_time; |
363 | 382 | ||
364 | #ifdef CONFIG_FAIR_GROUP_SCHED | 383 | #ifdef CONFIG_RT_GROUP_SCHED |
365 | unsigned long rt_nr_boosted; | 384 | unsigned long rt_nr_boosted; |
366 | 385 | ||
367 | struct rq *rq; | 386 | struct rq *rq; |
@@ -437,6 +456,8 @@ struct rq { | |||
437 | #ifdef CONFIG_FAIR_GROUP_SCHED | 456 | #ifdef CONFIG_FAIR_GROUP_SCHED |
438 | /* list of leaf cfs_rq on this cpu: */ | 457 | /* list of leaf cfs_rq on this cpu: */ |
439 | struct list_head leaf_cfs_rq_list; | 458 | struct list_head leaf_cfs_rq_list; |
459 | #endif | ||
460 | #ifdef CONFIG_RT_GROUP_SCHED | ||
440 | struct list_head leaf_rt_rq_list; | 461 | struct list_head leaf_rt_rq_list; |
441 | #endif | 462 | #endif |
442 | 463 | ||
@@ -7104,7 +7125,7 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq) | |||
7104 | /* delimiter for bitsearch: */ | 7125 | /* delimiter for bitsearch: */ |
7105 | __set_bit(MAX_RT_PRIO, array->bitmap); | 7126 | __set_bit(MAX_RT_PRIO, array->bitmap); |
7106 | 7127 | ||
7107 | #if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED | 7128 | #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED |
7108 | rt_rq->highest_prio = MAX_RT_PRIO; | 7129 | rt_rq->highest_prio = MAX_RT_PRIO; |
7109 | #endif | 7130 | #endif |
7110 | #ifdef CONFIG_SMP | 7131 | #ifdef CONFIG_SMP |
@@ -7115,7 +7136,7 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq) | |||
7115 | rt_rq->rt_time = 0; | 7136 | rt_rq->rt_time = 0; |
7116 | rt_rq->rt_throttled = 0; | 7137 | rt_rq->rt_throttled = 0; |
7117 | 7138 | ||
7118 | #ifdef CONFIG_FAIR_GROUP_SCHED | 7139 | #ifdef CONFIG_RT_GROUP_SCHED |
7119 | rt_rq->rt_nr_boosted = 0; | 7140 | rt_rq->rt_nr_boosted = 0; |
7120 | rt_rq->rq = rq; | 7141 | rt_rq->rq = rq; |
7121 | #endif | 7142 | #endif |
@@ -7139,7 +7160,9 @@ static void init_tg_cfs_entry(struct rq *rq, struct task_group *tg, | |||
7139 | se->load.inv_weight = div64_64(1ULL<<32, se->load.weight); | 7160 | se->load.inv_weight = div64_64(1ULL<<32, se->load.weight); |
7140 | se->parent = NULL; | 7161 | se->parent = NULL; |
7141 | } | 7162 | } |
7163 | #endif | ||
7142 | 7164 | ||
7165 | #ifdef CONFIG_RT_GROUP_SCHED | ||
7143 | static void init_tg_rt_entry(struct rq *rq, struct task_group *tg, | 7166 | static void init_tg_rt_entry(struct rq *rq, struct task_group *tg, |
7144 | struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, | 7167 | struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, |
7145 | int cpu, int add) | 7168 | int cpu, int add) |
@@ -7168,7 +7191,7 @@ void __init sched_init(void) | |||
7168 | init_defrootdomain(); | 7191 | init_defrootdomain(); |
7169 | #endif | 7192 | #endif |
7170 | 7193 | ||
7171 | #ifdef CONFIG_FAIR_GROUP_SCHED | 7194 | #ifdef CONFIG_GROUP_SCHED |
7172 | list_add(&init_task_group.list, &task_groups); | 7195 | list_add(&init_task_group.list, &task_groups); |
7173 | #endif | 7196 | #endif |
7174 | 7197 | ||
@@ -7189,6 +7212,8 @@ void __init sched_init(void) | |||
7189 | &per_cpu(init_cfs_rq, i), | 7212 | &per_cpu(init_cfs_rq, i), |
7190 | &per_cpu(init_sched_entity, i), i, 1); | 7213 | &per_cpu(init_sched_entity, i), i, 1); |
7191 | 7214 | ||
7215 | #endif | ||
7216 | #ifdef CONFIG_RT_GROUP_SCHED | ||
7192 | init_task_group.rt_runtime = | 7217 | init_task_group.rt_runtime = |
7193 | sysctl_sched_rt_runtime * NSEC_PER_USEC; | 7218 | sysctl_sched_rt_runtime * NSEC_PER_USEC; |
7194 | INIT_LIST_HEAD(&rq->leaf_rt_rq_list); | 7219 | INIT_LIST_HEAD(&rq->leaf_rt_rq_list); |
@@ -7381,9 +7406,9 @@ void set_curr_task(int cpu, struct task_struct *p) | |||
7381 | 7406 | ||
7382 | #endif | 7407 | #endif |
7383 | 7408 | ||
7384 | #ifdef CONFIG_FAIR_GROUP_SCHED | 7409 | #ifdef CONFIG_GROUP_SCHED |
7385 | 7410 | ||
7386 | #ifdef CONFIG_SMP | 7411 | #if defined CONFIG_FAIR_GROUP_SCHED && defined CONFIG_SMP |
7387 | /* | 7412 | /* |
7388 | * distribute shares of all task groups among their schedulable entities, | 7413 | * distribute shares of all task groups among their schedulable entities, |
7389 | * to reflect load distribution across cpus. | 7414 | * to reflect load distribution across cpus. |
@@ -7539,20 +7564,28 @@ static void free_sched_group(struct task_group *tg) | |||
7539 | int i; | 7564 | int i; |
7540 | 7565 | ||
7541 | for_each_possible_cpu(i) { | 7566 | for_each_possible_cpu(i) { |
7567 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
7542 | if (tg->cfs_rq) | 7568 | if (tg->cfs_rq) |
7543 | kfree(tg->cfs_rq[i]); | 7569 | kfree(tg->cfs_rq[i]); |
7544 | if (tg->se) | 7570 | if (tg->se) |
7545 | kfree(tg->se[i]); | 7571 | kfree(tg->se[i]); |
7572 | #endif | ||
7573 | #ifdef CONFIG_RT_GROUP_SCHED | ||
7546 | if (tg->rt_rq) | 7574 | if (tg->rt_rq) |
7547 | kfree(tg->rt_rq[i]); | 7575 | kfree(tg->rt_rq[i]); |
7548 | if (tg->rt_se) | 7576 | if (tg->rt_se) |
7549 | kfree(tg->rt_se[i]); | 7577 | kfree(tg->rt_se[i]); |
7578 | #endif | ||
7550 | } | 7579 | } |
7551 | 7580 | ||
7581 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
7552 | kfree(tg->cfs_rq); | 7582 | kfree(tg->cfs_rq); |
7553 | kfree(tg->se); | 7583 | kfree(tg->se); |
7584 | #endif | ||
7585 | #ifdef CONFIG_RT_GROUP_SCHED | ||
7554 | kfree(tg->rt_rq); | 7586 | kfree(tg->rt_rq); |
7555 | kfree(tg->rt_se); | 7587 | kfree(tg->rt_se); |
7588 | #endif | ||
7556 | kfree(tg); | 7589 | kfree(tg); |
7557 | } | 7590 | } |
7558 | 7591 | ||
@@ -7560,10 +7593,14 @@ static void free_sched_group(struct task_group *tg) | |||
7560 | struct task_group *sched_create_group(void) | 7593 | struct task_group *sched_create_group(void) |
7561 | { | 7594 | { |
7562 | struct task_group *tg; | 7595 | struct task_group *tg; |
7596 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
7563 | struct cfs_rq *cfs_rq; | 7597 | struct cfs_rq *cfs_rq; |
7564 | struct sched_entity *se; | 7598 | struct sched_entity *se; |
7599 | #endif | ||
7600 | #ifdef CONFIG_RT_GROUP_SCHED | ||
7565 | struct rt_rq *rt_rq; | 7601 | struct rt_rq *rt_rq; |
7566 | struct sched_rt_entity *rt_se; | 7602 | struct sched_rt_entity *rt_se; |
7603 | #endif | ||
7567 | struct rq *rq; | 7604 | struct rq *rq; |
7568 | unsigned long flags; | 7605 | unsigned long flags; |
7569 | int i; | 7606 | int i; |
@@ -7572,12 +7609,18 @@ struct task_group *sched_create_group(void) | |||
7572 | if (!tg) | 7609 | if (!tg) |
7573 | return ERR_PTR(-ENOMEM); | 7610 | return ERR_PTR(-ENOMEM); |
7574 | 7611 | ||
7612 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
7575 | tg->cfs_rq = kzalloc(sizeof(cfs_rq) * NR_CPUS, GFP_KERNEL); | 7613 | tg->cfs_rq = kzalloc(sizeof(cfs_rq) * NR_CPUS, GFP_KERNEL); |
7576 | if (!tg->cfs_rq) | 7614 | if (!tg->cfs_rq) |
7577 | goto err; | 7615 | goto err; |
7578 | tg->se = kzalloc(sizeof(se) * NR_CPUS, GFP_KERNEL); | 7616 | tg->se = kzalloc(sizeof(se) * NR_CPUS, GFP_KERNEL); |
7579 | if (!tg->se) | 7617 | if (!tg->se) |
7580 | goto err; | 7618 | goto err; |
7619 | |||
7620 | tg->shares = NICE_0_LOAD; | ||
7621 | #endif | ||
7622 | |||
7623 | #ifdef CONFIG_RT_GROUP_SCHED | ||
7581 | tg->rt_rq = kzalloc(sizeof(rt_rq) * NR_CPUS, GFP_KERNEL); | 7624 | tg->rt_rq = kzalloc(sizeof(rt_rq) * NR_CPUS, GFP_KERNEL); |
7582 | if (!tg->rt_rq) | 7625 | if (!tg->rt_rq) |
7583 | goto err; | 7626 | goto err; |
@@ -7585,12 +7628,13 @@ struct task_group *sched_create_group(void) | |||
7585 | if (!tg->rt_se) | 7628 | if (!tg->rt_se) |
7586 | goto err; | 7629 | goto err; |
7587 | 7630 | ||
7588 | tg->shares = NICE_0_LOAD; | ||
7589 | tg->rt_runtime = 0; | 7631 | tg->rt_runtime = 0; |
7632 | #endif | ||
7590 | 7633 | ||
7591 | for_each_possible_cpu(i) { | 7634 | for_each_possible_cpu(i) { |
7592 | rq = cpu_rq(i); | 7635 | rq = cpu_rq(i); |
7593 | 7636 | ||
7637 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
7594 | cfs_rq = kmalloc_node(sizeof(struct cfs_rq), | 7638 | cfs_rq = kmalloc_node(sizeof(struct cfs_rq), |
7595 | GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); | 7639 | GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); |
7596 | if (!cfs_rq) | 7640 | if (!cfs_rq) |
@@ -7601,6 +7645,10 @@ struct task_group *sched_create_group(void) | |||
7601 | if (!se) | 7645 | if (!se) |
7602 | goto err; | 7646 | goto err; |
7603 | 7647 | ||
7648 | init_tg_cfs_entry(rq, tg, cfs_rq, se, i, 0); | ||
7649 | #endif | ||
7650 | |||
7651 | #ifdef CONFIG_RT_GROUP_SCHED | ||
7604 | rt_rq = kmalloc_node(sizeof(struct rt_rq), | 7652 | rt_rq = kmalloc_node(sizeof(struct rt_rq), |
7605 | GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); | 7653 | GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); |
7606 | if (!rt_rq) | 7654 | if (!rt_rq) |
@@ -7611,17 +7659,21 @@ struct task_group *sched_create_group(void) | |||
7611 | if (!rt_se) | 7659 | if (!rt_se) |
7612 | goto err; | 7660 | goto err; |
7613 | 7661 | ||
7614 | init_tg_cfs_entry(rq, tg, cfs_rq, se, i, 0); | ||
7615 | init_tg_rt_entry(rq, tg, rt_rq, rt_se, i, 0); | 7662 | init_tg_rt_entry(rq, tg, rt_rq, rt_se, i, 0); |
7663 | #endif | ||
7616 | } | 7664 | } |
7617 | 7665 | ||
7618 | spin_lock_irqsave(&task_group_lock, flags); | 7666 | spin_lock_irqsave(&task_group_lock, flags); |
7619 | for_each_possible_cpu(i) { | 7667 | for_each_possible_cpu(i) { |
7620 | rq = cpu_rq(i); | 7668 | rq = cpu_rq(i); |
7669 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
7621 | cfs_rq = tg->cfs_rq[i]; | 7670 | cfs_rq = tg->cfs_rq[i]; |
7622 | list_add_rcu(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list); | 7671 | list_add_rcu(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list); |
7672 | #endif | ||
7673 | #ifdef CONFIG_RT_GROUP_SCHED | ||
7623 | rt_rq = tg->rt_rq[i]; | 7674 | rt_rq = tg->rt_rq[i]; |
7624 | list_add_rcu(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list); | 7675 | list_add_rcu(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list); |
7676 | #endif | ||
7625 | } | 7677 | } |
7626 | list_add_rcu(&tg->list, &task_groups); | 7678 | list_add_rcu(&tg->list, &task_groups); |
7627 | spin_unlock_irqrestore(&task_group_lock, flags); | 7679 | spin_unlock_irqrestore(&task_group_lock, flags); |
@@ -7643,23 +7695,21 @@ static void free_sched_group_rcu(struct rcu_head *rhp) | |||
7643 | /* Destroy runqueue etc associated with a task group */ | 7695 | /* Destroy runqueue etc associated with a task group */ |
7644 | void sched_destroy_group(struct task_group *tg) | 7696 | void sched_destroy_group(struct task_group *tg) |
7645 | { | 7697 | { |
7646 | struct cfs_rq *cfs_rq = NULL; | ||
7647 | struct rt_rq *rt_rq = NULL; | ||
7648 | unsigned long flags; | 7698 | unsigned long flags; |
7649 | int i; | 7699 | int i; |
7650 | 7700 | ||
7651 | spin_lock_irqsave(&task_group_lock, flags); | 7701 | spin_lock_irqsave(&task_group_lock, flags); |
7652 | for_each_possible_cpu(i) { | 7702 | for_each_possible_cpu(i) { |
7653 | cfs_rq = tg->cfs_rq[i]; | 7703 | #ifdef CONFIG_FAIR_GROUP_SCHED |
7654 | list_del_rcu(&cfs_rq->leaf_cfs_rq_list); | 7704 | list_del_rcu(&tg->cfs_rq[i]->leaf_cfs_rq_list); |
7655 | rt_rq = tg->rt_rq[i]; | 7705 | #endif |
7656 | list_del_rcu(&rt_rq->leaf_rt_rq_list); | 7706 | #ifdef CONFIG_RT_GROUP_SCHED |
7707 | list_del_rcu(&tg->rt_rq[i]->leaf_rt_rq_list); | ||
7708 | #endif | ||
7657 | } | 7709 | } |
7658 | list_del_rcu(&tg->list); | 7710 | list_del_rcu(&tg->list); |
7659 | spin_unlock_irqrestore(&task_group_lock, flags); | 7711 | spin_unlock_irqrestore(&task_group_lock, flags); |
7660 | 7712 | ||
7661 | BUG_ON(!cfs_rq); | ||
7662 | |||
7663 | /* wait for possible concurrent references to cfs_rqs complete */ | 7713 | /* wait for possible concurrent references to cfs_rqs complete */ |
7664 | call_rcu(&tg->rcu, free_sched_group_rcu); | 7714 | call_rcu(&tg->rcu, free_sched_group_rcu); |
7665 | } | 7715 | } |
@@ -7699,6 +7749,7 @@ void sched_move_task(struct task_struct *tsk) | |||
7699 | task_rq_unlock(rq, &flags); | 7749 | task_rq_unlock(rq, &flags); |
7700 | } | 7750 | } |
7701 | 7751 | ||
7752 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
7702 | /* rq->lock to be locked by caller */ | 7753 | /* rq->lock to be locked by caller */ |
7703 | static void set_se_shares(struct sched_entity *se, unsigned long shares) | 7754 | static void set_se_shares(struct sched_entity *se, unsigned long shares) |
7704 | { | 7755 | { |
@@ -7786,7 +7837,9 @@ unsigned long sched_group_shares(struct task_group *tg) | |||
7786 | { | 7837 | { |
7787 | return tg->shares; | 7838 | return tg->shares; |
7788 | } | 7839 | } |
7840 | #endif | ||
7789 | 7841 | ||
7842 | #ifdef CONFIG_RT_GROUP_SCHED | ||
7790 | /* | 7843 | /* |
7791 | * Ensure that the real time constraints are schedulable. | 7844 | * Ensure that the real time constraints are schedulable. |
7792 | */ | 7845 | */ |
@@ -7858,9 +7911,10 @@ long sched_group_rt_runtime(struct task_group *tg) | |||
7858 | do_div(rt_runtime_us, NSEC_PER_USEC); | 7911 | do_div(rt_runtime_us, NSEC_PER_USEC); |
7859 | return rt_runtime_us; | 7912 | return rt_runtime_us; |
7860 | } | 7913 | } |
7861 | #endif /* CONFIG_FAIR_GROUP_SCHED */ | 7914 | #endif |
7915 | #endif /* CONFIG_GROUP_SCHED */ | ||
7862 | 7916 | ||
7863 | #ifdef CONFIG_FAIR_CGROUP_SCHED | 7917 | #ifdef CONFIG_CGROUP_SCHED |
7864 | 7918 | ||
7865 | /* return corresponding task_group object of a cgroup */ | 7919 | /* return corresponding task_group object of a cgroup */ |
7866 | static inline struct task_group *cgroup_tg(struct cgroup *cgrp) | 7920 | static inline struct task_group *cgroup_tg(struct cgroup *cgrp) |
@@ -7920,6 +7974,7 @@ cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp, | |||
7920 | sched_move_task(tsk); | 7974 | sched_move_task(tsk); |
7921 | } | 7975 | } |
7922 | 7976 | ||
7977 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
7923 | static int cpu_shares_write_uint(struct cgroup *cgrp, struct cftype *cftype, | 7978 | static int cpu_shares_write_uint(struct cgroup *cgrp, struct cftype *cftype, |
7924 | u64 shareval) | 7979 | u64 shareval) |
7925 | { | 7980 | { |
@@ -7932,7 +7987,9 @@ static u64 cpu_shares_read_uint(struct cgroup *cgrp, struct cftype *cft) | |||
7932 | 7987 | ||
7933 | return (u64) tg->shares; | 7988 | return (u64) tg->shares; |
7934 | } | 7989 | } |
7990 | #endif | ||
7935 | 7991 | ||
7992 | #ifdef CONFIG_RT_GROUP_SCHED | ||
7936 | static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft, | 7993 | static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft, |
7937 | struct file *file, | 7994 | struct file *file, |
7938 | const char __user *userbuf, | 7995 | const char __user *userbuf, |
@@ -7977,18 +8034,23 @@ static ssize_t cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft, | |||
7977 | 8034 | ||
7978 | return simple_read_from_buffer(buf, nbytes, ppos, tmp, len); | 8035 | return simple_read_from_buffer(buf, nbytes, ppos, tmp, len); |
7979 | } | 8036 | } |
8037 | #endif | ||
7980 | 8038 | ||
7981 | static struct cftype cpu_files[] = { | 8039 | static struct cftype cpu_files[] = { |
8040 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
7982 | { | 8041 | { |
7983 | .name = "shares", | 8042 | .name = "shares", |
7984 | .read_uint = cpu_shares_read_uint, | 8043 | .read_uint = cpu_shares_read_uint, |
7985 | .write_uint = cpu_shares_write_uint, | 8044 | .write_uint = cpu_shares_write_uint, |
7986 | }, | 8045 | }, |
8046 | #endif | ||
8047 | #ifdef CONFIG_RT_GROUP_SCHED | ||
7987 | { | 8048 | { |
7988 | .name = "rt_runtime_us", | 8049 | .name = "rt_runtime_us", |
7989 | .read = cpu_rt_runtime_read, | 8050 | .read = cpu_rt_runtime_read, |
7990 | .write = cpu_rt_runtime_write, | 8051 | .write = cpu_rt_runtime_write, |
7991 | }, | 8052 | }, |
8053 | #endif | ||
7992 | }; | 8054 | }; |
7993 | 8055 | ||
7994 | static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont) | 8056 | static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont) |
@@ -8007,7 +8069,7 @@ struct cgroup_subsys cpu_cgroup_subsys = { | |||
8007 | .early_init = 1, | 8069 | .early_init = 1, |
8008 | }; | 8070 | }; |
8009 | 8071 | ||
8010 | #endif /* CONFIG_FAIR_CGROUP_SCHED */ | 8072 | #endif /* CONFIG_CGROUP_SCHED */ |
8011 | 8073 | ||
8012 | #ifdef CONFIG_CGROUP_CPUACCT | 8074 | #ifdef CONFIG_CGROUP_CPUACCT |
8013 | 8075 | ||
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 35825b28e429..f54792b175b2 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c | |||
@@ -55,7 +55,7 @@ static inline int on_rt_rq(struct sched_rt_entity *rt_se) | |||
55 | return !list_empty(&rt_se->run_list); | 55 | return !list_empty(&rt_se->run_list); |
56 | } | 56 | } |
57 | 57 | ||
58 | #ifdef CONFIG_FAIR_GROUP_SCHED | 58 | #ifdef CONFIG_RT_GROUP_SCHED |
59 | 59 | ||
60 | static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) | 60 | static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) |
61 | { | 61 | { |
@@ -177,7 +177,7 @@ static inline int rt_rq_throttled(struct rt_rq *rt_rq) | |||
177 | 177 | ||
178 | static inline int rt_se_prio(struct sched_rt_entity *rt_se) | 178 | static inline int rt_se_prio(struct sched_rt_entity *rt_se) |
179 | { | 179 | { |
180 | #ifdef CONFIG_FAIR_GROUP_SCHED | 180 | #ifdef CONFIG_RT_GROUP_SCHED |
181 | struct rt_rq *rt_rq = group_rt_rq(rt_se); | 181 | struct rt_rq *rt_rq = group_rt_rq(rt_se); |
182 | 182 | ||
183 | if (rt_rq) | 183 | if (rt_rq) |
@@ -269,7 +269,7 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) | |||
269 | { | 269 | { |
270 | WARN_ON(!rt_prio(rt_se_prio(rt_se))); | 270 | WARN_ON(!rt_prio(rt_se_prio(rt_se))); |
271 | rt_rq->rt_nr_running++; | 271 | rt_rq->rt_nr_running++; |
272 | #if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED | 272 | #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED |
273 | if (rt_se_prio(rt_se) < rt_rq->highest_prio) | 273 | if (rt_se_prio(rt_se) < rt_rq->highest_prio) |
274 | rt_rq->highest_prio = rt_se_prio(rt_se); | 274 | rt_rq->highest_prio = rt_se_prio(rt_se); |
275 | #endif | 275 | #endif |
@@ -281,7 +281,7 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) | |||
281 | 281 | ||
282 | update_rt_migration(rq_of_rt_rq(rt_rq)); | 282 | update_rt_migration(rq_of_rt_rq(rt_rq)); |
283 | #endif | 283 | #endif |
284 | #ifdef CONFIG_FAIR_GROUP_SCHED | 284 | #ifdef CONFIG_RT_GROUP_SCHED |
285 | if (rt_se_boosted(rt_se)) | 285 | if (rt_se_boosted(rt_se)) |
286 | rt_rq->rt_nr_boosted++; | 286 | rt_rq->rt_nr_boosted++; |
287 | #endif | 287 | #endif |
@@ -293,7 +293,7 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) | |||
293 | WARN_ON(!rt_prio(rt_se_prio(rt_se))); | 293 | WARN_ON(!rt_prio(rt_se_prio(rt_se))); |
294 | WARN_ON(!rt_rq->rt_nr_running); | 294 | WARN_ON(!rt_rq->rt_nr_running); |
295 | rt_rq->rt_nr_running--; | 295 | rt_rq->rt_nr_running--; |
296 | #if defined CONFIG_SMP || defined CONFIG_FAIR_GROUP_SCHED | 296 | #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED |
297 | if (rt_rq->rt_nr_running) { | 297 | if (rt_rq->rt_nr_running) { |
298 | struct rt_prio_array *array; | 298 | struct rt_prio_array *array; |
299 | 299 | ||
@@ -315,7 +315,7 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) | |||
315 | 315 | ||
316 | update_rt_migration(rq_of_rt_rq(rt_rq)); | 316 | update_rt_migration(rq_of_rt_rq(rt_rq)); |
317 | #endif /* CONFIG_SMP */ | 317 | #endif /* CONFIG_SMP */ |
318 | #ifdef CONFIG_FAIR_GROUP_SCHED | 318 | #ifdef CONFIG_RT_GROUP_SCHED |
319 | if (rt_se_boosted(rt_se)) | 319 | if (rt_se_boosted(rt_se)) |
320 | rt_rq->rt_nr_boosted--; | 320 | rt_rq->rt_nr_boosted--; |
321 | 321 | ||
diff --git a/kernel/user.c b/kernel/user.c index 9f6d471bfd03..7132022a040c 100644 --- a/kernel/user.c +++ b/kernel/user.c | |||
@@ -57,7 +57,7 @@ struct user_struct root_user = { | |||
57 | .uid_keyring = &root_user_keyring, | 57 | .uid_keyring = &root_user_keyring, |
58 | .session_keyring = &root_session_keyring, | 58 | .session_keyring = &root_session_keyring, |
59 | #endif | 59 | #endif |
60 | #ifdef CONFIG_FAIR_USER_SCHED | 60 | #ifdef CONFIG_USER_SCHED |
61 | .tg = &init_task_group, | 61 | .tg = &init_task_group, |
62 | #endif | 62 | #endif |
63 | }; | 63 | }; |
@@ -90,7 +90,7 @@ static struct user_struct *uid_hash_find(uid_t uid, struct hlist_head *hashent) | |||
90 | return NULL; | 90 | return NULL; |
91 | } | 91 | } |
92 | 92 | ||
93 | #ifdef CONFIG_FAIR_USER_SCHED | 93 | #ifdef CONFIG_USER_SCHED |
94 | 94 | ||
95 | static void sched_destroy_user(struct user_struct *up) | 95 | static void sched_destroy_user(struct user_struct *up) |
96 | { | 96 | { |
@@ -113,15 +113,15 @@ static void sched_switch_user(struct task_struct *p) | |||
113 | sched_move_task(p); | 113 | sched_move_task(p); |
114 | } | 114 | } |
115 | 115 | ||
116 | #else /* CONFIG_FAIR_USER_SCHED */ | 116 | #else /* CONFIG_USER_SCHED */ |
117 | 117 | ||
118 | static void sched_destroy_user(struct user_struct *up) { } | 118 | static void sched_destroy_user(struct user_struct *up) { } |
119 | static int sched_create_user(struct user_struct *up) { return 0; } | 119 | static int sched_create_user(struct user_struct *up) { return 0; } |
120 | static void sched_switch_user(struct task_struct *p) { } | 120 | static void sched_switch_user(struct task_struct *p) { } |
121 | 121 | ||
122 | #endif /* CONFIG_FAIR_USER_SCHED */ | 122 | #endif /* CONFIG_USER_SCHED */ |
123 | 123 | ||
124 | #if defined(CONFIG_FAIR_USER_SCHED) && defined(CONFIG_SYSFS) | 124 | #if defined(CONFIG_USER_SCHED) && defined(CONFIG_SYSFS) |
125 | 125 | ||
126 | static struct kset *uids_kset; /* represents the /sys/kernel/uids/ directory */ | 126 | static struct kset *uids_kset; /* represents the /sys/kernel/uids/ directory */ |
127 | static DEFINE_MUTEX(uids_mutex); | 127 | static DEFINE_MUTEX(uids_mutex); |
@@ -137,6 +137,7 @@ static inline void uids_mutex_unlock(void) | |||
137 | } | 137 | } |
138 | 138 | ||
139 | /* uid directory attributes */ | 139 | /* uid directory attributes */ |
140 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
140 | static ssize_t cpu_shares_show(struct kobject *kobj, | 141 | static ssize_t cpu_shares_show(struct kobject *kobj, |
141 | struct kobj_attribute *attr, | 142 | struct kobj_attribute *attr, |
142 | char *buf) | 143 | char *buf) |
@@ -163,7 +164,9 @@ static ssize_t cpu_shares_store(struct kobject *kobj, | |||
163 | 164 | ||
164 | static struct kobj_attribute cpu_share_attr = | 165 | static struct kobj_attribute cpu_share_attr = |
165 | __ATTR(cpu_share, 0644, cpu_shares_show, cpu_shares_store); | 166 | __ATTR(cpu_share, 0644, cpu_shares_show, cpu_shares_store); |
167 | #endif | ||
166 | 168 | ||
169 | #ifdef CONFIG_RT_GROUP_SCHED | ||
167 | static ssize_t cpu_rt_runtime_show(struct kobject *kobj, | 170 | static ssize_t cpu_rt_runtime_show(struct kobject *kobj, |
168 | struct kobj_attribute *attr, | 171 | struct kobj_attribute *attr, |
169 | char *buf) | 172 | char *buf) |
@@ -190,11 +193,16 @@ static ssize_t cpu_rt_runtime_store(struct kobject *kobj, | |||
190 | 193 | ||
191 | static struct kobj_attribute cpu_rt_runtime_attr = | 194 | static struct kobj_attribute cpu_rt_runtime_attr = |
192 | __ATTR(cpu_rt_runtime, 0644, cpu_rt_runtime_show, cpu_rt_runtime_store); | 195 | __ATTR(cpu_rt_runtime, 0644, cpu_rt_runtime_show, cpu_rt_runtime_store); |
196 | #endif | ||
193 | 197 | ||
194 | /* default attributes per uid directory */ | 198 | /* default attributes per uid directory */ |
195 | static struct attribute *uids_attributes[] = { | 199 | static struct attribute *uids_attributes[] = { |
200 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
196 | &cpu_share_attr.attr, | 201 | &cpu_share_attr.attr, |
202 | #endif | ||
203 | #ifdef CONFIG_RT_GROUP_SCHED | ||
197 | &cpu_rt_runtime_attr.attr, | 204 | &cpu_rt_runtime_attr.attr, |
205 | #endif | ||
198 | NULL | 206 | NULL |
199 | }; | 207 | }; |
200 | 208 | ||
@@ -297,7 +305,7 @@ static inline void free_user(struct user_struct *up, unsigned long flags) | |||
297 | schedule_work(&up->work); | 305 | schedule_work(&up->work); |
298 | } | 306 | } |
299 | 307 | ||
300 | #else /* CONFIG_FAIR_USER_SCHED && CONFIG_SYSFS */ | 308 | #else /* CONFIG_USER_SCHED && CONFIG_SYSFS */ |
301 | 309 | ||
302 | int uids_sysfs_init(void) { return 0; } | 310 | int uids_sysfs_init(void) { return 0; } |
303 | static inline int uids_user_create(struct user_struct *up) { return 0; } | 311 | static inline int uids_user_create(struct user_struct *up) { return 0; } |
@@ -401,7 +409,7 @@ struct user_struct * alloc_uid(struct user_namespace *ns, uid_t uid) | |||
401 | spin_lock_irq(&uidhash_lock); | 409 | spin_lock_irq(&uidhash_lock); |
402 | up = uid_hash_find(uid, hashent); | 410 | up = uid_hash_find(uid, hashent); |
403 | if (up) { | 411 | if (up) { |
404 | /* This case is not possible when CONFIG_FAIR_USER_SCHED | 412 | /* This case is not possible when CONFIG_USER_SCHED |
405 | * is defined, since we serialize alloc_uid() using | 413 | * is defined, since we serialize alloc_uid() using |
406 | * uids_mutex. Hence no need to call | 414 | * uids_mutex. Hence no need to call |
407 | * sched_destroy_user() or remove_user_sysfs_dir(). | 415 | * sched_destroy_user() or remove_user_sysfs_dir(). |