aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSiddha, Suresh B <suresh.b.siddha@intel.com>2006-12-10 05:20:07 -0500
committerLinus Torvalds <torvalds@woody.osdl.org>2006-12-10 12:55:42 -0500
commit6711cab43ed5e60bf51e3dbbce6395e87d4e9805 (patch)
treede7c1b25add18dbfccf5fbccdaccca7a0a05bd33
parentcc2a73b5caf065f8612fcb5df5bd2f5e25881d99 (diff)
[PATCH] ched domain: move sched group allocations to percpu area
Move the sched group allocations to percpu area. This will minimize cross node memory references and also cleans up the sched groups allocation for allnodes sched domain. Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com> Acked-by: Ingo Molnar <mingo@elte.hu> Acked-by: Christoph Lameter <clameter@sgi.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
-rw-r--r--kernel/sched.c132
1 files changed, 64 insertions, 68 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index 479199ab14ab..a08387b5f7fa 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -5528,28 +5528,27 @@ static int __init isolated_cpu_setup(char *str)
5528__setup ("isolcpus=", isolated_cpu_setup); 5528__setup ("isolcpus=", isolated_cpu_setup);
5529 5529
5530/* 5530/*
5531 * init_sched_build_groups takes an array of groups, the cpumask we wish 5531 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
5532 * to span, and a pointer to a function which identifies what group a CPU 5532 * to a function which identifies what group(along with sched group) a CPU
5533 * belongs to. The return value of group_fn must be a valid index into the 5533 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS
5534 * groups[] array, and must be >= 0 and < NR_CPUS (due to the fact that we 5534 * (due to the fact that we keep track of groups covered with a cpumask_t).
5535 * keep track of groups covered with a cpumask_t).
5536 * 5535 *
5537 * init_sched_build_groups will build a circular linked list of the groups 5536 * init_sched_build_groups will build a circular linked list of the groups
5538 * covered by the given span, and will set each group's ->cpumask correctly, 5537 * covered by the given span, and will set each group's ->cpumask correctly,
5539 * and ->cpu_power to 0. 5538 * and ->cpu_power to 0.
5540 */ 5539 */
5541static void 5540static void
5542init_sched_build_groups(struct sched_group groups[], cpumask_t span, 5541init_sched_build_groups(cpumask_t span, const cpumask_t *cpu_map,
5543 const cpumask_t *cpu_map, 5542 int (*group_fn)(int cpu, const cpumask_t *cpu_map,
5544 int (*group_fn)(int cpu, const cpumask_t *cpu_map)) 5543 struct sched_group **sg))
5545{ 5544{
5546 struct sched_group *first = NULL, *last = NULL; 5545 struct sched_group *first = NULL, *last = NULL;
5547 cpumask_t covered = CPU_MASK_NONE; 5546 cpumask_t covered = CPU_MASK_NONE;
5548 int i; 5547 int i;
5549 5548
5550 for_each_cpu_mask(i, span) { 5549 for_each_cpu_mask(i, span) {
5551 int group = group_fn(i, cpu_map); 5550 struct sched_group *sg;
5552 struct sched_group *sg = &groups[group]; 5551 int group = group_fn(i, cpu_map, &sg);
5553 int j; 5552 int j;
5554 5553
5555 if (cpu_isset(i, covered)) 5554 if (cpu_isset(i, covered))
@@ -5559,7 +5558,7 @@ init_sched_build_groups(struct sched_group groups[], cpumask_t span,
5559 sg->cpu_power = 0; 5558 sg->cpu_power = 0;
5560 5559
5561 for_each_cpu_mask(j, span) { 5560 for_each_cpu_mask(j, span) {
5562 if (group_fn(j, cpu_map) != group) 5561 if (group_fn(j, cpu_map, NULL) != group)
5563 continue; 5562 continue;
5564 5563
5565 cpu_set(j, covered); 5564 cpu_set(j, covered);
@@ -6135,10 +6134,13 @@ int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
6135 */ 6134 */
6136#ifdef CONFIG_SCHED_SMT 6135#ifdef CONFIG_SCHED_SMT
6137static DEFINE_PER_CPU(struct sched_domain, cpu_domains); 6136static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
6138static struct sched_group sched_group_cpus[NR_CPUS]; 6137static DEFINE_PER_CPU(struct sched_group, sched_group_cpus);
6139 6138
6140static int cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map) 6139static int cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map,
6140 struct sched_group **sg)
6141{ 6141{
6142 if (sg)
6143 *sg = &per_cpu(sched_group_cpus, cpu);
6142 return cpu; 6144 return cpu;
6143} 6145}
6144#endif 6146#endif
@@ -6148,39 +6150,52 @@ static int cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map)
6148 */ 6150 */
6149#ifdef CONFIG_SCHED_MC 6151#ifdef CONFIG_SCHED_MC
6150static DEFINE_PER_CPU(struct sched_domain, core_domains); 6152static DEFINE_PER_CPU(struct sched_domain, core_domains);
6151static struct sched_group sched_group_core[NR_CPUS]; 6153static DEFINE_PER_CPU(struct sched_group, sched_group_core);
6152#endif 6154#endif
6153 6155
6154#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) 6156#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
6155static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map) 6157static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map,
6158 struct sched_group **sg)
6156{ 6159{
6160 int group;
6157 cpumask_t mask = cpu_sibling_map[cpu]; 6161 cpumask_t mask = cpu_sibling_map[cpu];
6158 cpus_and(mask, mask, *cpu_map); 6162 cpus_and(mask, mask, *cpu_map);
6159 return first_cpu(mask); 6163 group = first_cpu(mask);
6164 if (sg)
6165 *sg = &per_cpu(sched_group_core, group);
6166 return group;
6160} 6167}
6161#elif defined(CONFIG_SCHED_MC) 6168#elif defined(CONFIG_SCHED_MC)
6162static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map) 6169static int cpu_to_core_group(int cpu, const cpumask_t *cpu_map,
6170 struct sched_group **sg)
6163{ 6171{
6172 if (sg)
6173 *sg = &per_cpu(sched_group_core, cpu);
6164 return cpu; 6174 return cpu;
6165} 6175}
6166#endif 6176#endif
6167 6177
6168static DEFINE_PER_CPU(struct sched_domain, phys_domains); 6178static DEFINE_PER_CPU(struct sched_domain, phys_domains);
6169static struct sched_group sched_group_phys[NR_CPUS]; 6179static DEFINE_PER_CPU(struct sched_group, sched_group_phys);
6170 6180
6171static int cpu_to_phys_group(int cpu, const cpumask_t *cpu_map) 6181static int cpu_to_phys_group(int cpu, const cpumask_t *cpu_map,
6182 struct sched_group **sg)
6172{ 6183{
6184 int group;
6173#ifdef CONFIG_SCHED_MC 6185#ifdef CONFIG_SCHED_MC
6174 cpumask_t mask = cpu_coregroup_map(cpu); 6186 cpumask_t mask = cpu_coregroup_map(cpu);
6175 cpus_and(mask, mask, *cpu_map); 6187 cpus_and(mask, mask, *cpu_map);
6176 return first_cpu(mask); 6188 group = first_cpu(mask);
6177#elif defined(CONFIG_SCHED_SMT) 6189#elif defined(CONFIG_SCHED_SMT)
6178 cpumask_t mask = cpu_sibling_map[cpu]; 6190 cpumask_t mask = cpu_sibling_map[cpu];
6179 cpus_and(mask, mask, *cpu_map); 6191 cpus_and(mask, mask, *cpu_map);
6180 return first_cpu(mask); 6192 group = first_cpu(mask);
6181#else 6193#else
6182 return cpu; 6194 group = cpu;
6183#endif 6195#endif
6196 if (sg)
6197 *sg = &per_cpu(sched_group_phys, group);
6198 return group;
6184} 6199}
6185 6200
6186#ifdef CONFIG_NUMA 6201#ifdef CONFIG_NUMA
@@ -6193,12 +6208,22 @@ static DEFINE_PER_CPU(struct sched_domain, node_domains);
6193static struct sched_group **sched_group_nodes_bycpu[NR_CPUS]; 6208static struct sched_group **sched_group_nodes_bycpu[NR_CPUS];
6194 6209
6195static DEFINE_PER_CPU(struct sched_domain, allnodes_domains); 6210static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
6196static struct sched_group *sched_group_allnodes_bycpu[NR_CPUS]; 6211static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes);
6197 6212
6198static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map) 6213static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map,
6214 struct sched_group **sg)
6199{ 6215{
6200 return cpu_to_node(cpu); 6216 cpumask_t nodemask = node_to_cpumask(cpu_to_node(cpu));
6217 int group;
6218
6219 cpus_and(nodemask, nodemask, *cpu_map);
6220 group = first_cpu(nodemask);
6221
6222 if (sg)
6223 *sg = &per_cpu(sched_group_allnodes, group);
6224 return group;
6201} 6225}
6226
6202static void init_numa_sched_groups_power(struct sched_group *group_head) 6227static void init_numa_sched_groups_power(struct sched_group *group_head)
6203{ 6228{
6204 struct sched_group *sg = group_head; 6229 struct sched_group *sg = group_head;
@@ -6234,16 +6259,9 @@ static void free_sched_groups(const cpumask_t *cpu_map)
6234 int cpu, i; 6259 int cpu, i;
6235 6260
6236 for_each_cpu_mask(cpu, *cpu_map) { 6261 for_each_cpu_mask(cpu, *cpu_map) {
6237 struct sched_group *sched_group_allnodes
6238 = sched_group_allnodes_bycpu[cpu];
6239 struct sched_group **sched_group_nodes 6262 struct sched_group **sched_group_nodes
6240 = sched_group_nodes_bycpu[cpu]; 6263 = sched_group_nodes_bycpu[cpu];
6241 6264
6242 if (sched_group_allnodes) {
6243 kfree(sched_group_allnodes);
6244 sched_group_allnodes_bycpu[cpu] = NULL;
6245 }
6246
6247 if (!sched_group_nodes) 6265 if (!sched_group_nodes)
6248 continue; 6266 continue;
6249 6267
@@ -6337,7 +6355,7 @@ static int build_sched_domains(const cpumask_t *cpu_map)
6337 struct sched_domain *sd; 6355 struct sched_domain *sd;
6338#ifdef CONFIG_NUMA 6356#ifdef CONFIG_NUMA
6339 struct sched_group **sched_group_nodes = NULL; 6357 struct sched_group **sched_group_nodes = NULL;
6340 struct sched_group *sched_group_allnodes = NULL; 6358 int sd_allnodes = 0;
6341 6359
6342 /* 6360 /*
6343 * Allocate the per-node list of sched groups 6361 * Allocate the per-node list of sched groups
@@ -6355,7 +6373,6 @@ static int build_sched_domains(const cpumask_t *cpu_map)
6355 * Set up domains for cpus specified by the cpu_map. 6373 * Set up domains for cpus specified by the cpu_map.
6356 */ 6374 */
6357 for_each_cpu_mask(i, *cpu_map) { 6375 for_each_cpu_mask(i, *cpu_map) {
6358 int group;
6359 struct sched_domain *sd = NULL, *p; 6376 struct sched_domain *sd = NULL, *p;
6360 cpumask_t nodemask = node_to_cpumask(cpu_to_node(i)); 6377 cpumask_t nodemask = node_to_cpumask(cpu_to_node(i));
6361 6378
@@ -6364,26 +6381,12 @@ static int build_sched_domains(const cpumask_t *cpu_map)
6364#ifdef CONFIG_NUMA 6381#ifdef CONFIG_NUMA
6365 if (cpus_weight(*cpu_map) 6382 if (cpus_weight(*cpu_map)
6366 > SD_NODES_PER_DOMAIN*cpus_weight(nodemask)) { 6383 > SD_NODES_PER_DOMAIN*cpus_weight(nodemask)) {
6367 if (!sched_group_allnodes) {
6368 sched_group_allnodes
6369 = kmalloc_node(sizeof(struct sched_group)
6370 * MAX_NUMNODES,
6371 GFP_KERNEL,
6372 cpu_to_node(i));
6373 if (!sched_group_allnodes) {
6374 printk(KERN_WARNING
6375 "Can not alloc allnodes sched group\n");
6376 goto error;
6377 }
6378 sched_group_allnodes_bycpu[i]
6379 = sched_group_allnodes;
6380 }
6381 sd = &per_cpu(allnodes_domains, i); 6384 sd = &per_cpu(allnodes_domains, i);
6382 *sd = SD_ALLNODES_INIT; 6385 *sd = SD_ALLNODES_INIT;
6383 sd->span = *cpu_map; 6386 sd->span = *cpu_map;
6384 group = cpu_to_allnodes_group(i, cpu_map); 6387 cpu_to_allnodes_group(i, cpu_map, &sd->groups);
6385 sd->groups = &sched_group_allnodes[group];
6386 p = sd; 6388 p = sd;
6389 sd_allnodes = 1;
6387 } else 6390 } else
6388 p = NULL; 6391 p = NULL;
6389 6392
@@ -6398,36 +6401,33 @@ static int build_sched_domains(const cpumask_t *cpu_map)
6398 6401
6399 p = sd; 6402 p = sd;
6400 sd = &per_cpu(phys_domains, i); 6403 sd = &per_cpu(phys_domains, i);
6401 group = cpu_to_phys_group(i, cpu_map);
6402 *sd = SD_CPU_INIT; 6404 *sd = SD_CPU_INIT;
6403 sd->span = nodemask; 6405 sd->span = nodemask;
6404 sd->parent = p; 6406 sd->parent = p;
6405 if (p) 6407 if (p)
6406 p->child = sd; 6408 p->child = sd;
6407 sd->groups = &sched_group_phys[group]; 6409 cpu_to_phys_group(i, cpu_map, &sd->groups);
6408 6410
6409#ifdef CONFIG_SCHED_MC 6411#ifdef CONFIG_SCHED_MC
6410 p = sd; 6412 p = sd;
6411 sd = &per_cpu(core_domains, i); 6413 sd = &per_cpu(core_domains, i);
6412 group = cpu_to_core_group(i, cpu_map);
6413 *sd = SD_MC_INIT; 6414 *sd = SD_MC_INIT;
6414 sd->span = cpu_coregroup_map(i); 6415 sd->span = cpu_coregroup_map(i);
6415 cpus_and(sd->span, sd->span, *cpu_map); 6416 cpus_and(sd->span, sd->span, *cpu_map);
6416 sd->parent = p; 6417 sd->parent = p;
6417 p->child = sd; 6418 p->child = sd;
6418 sd->groups = &sched_group_core[group]; 6419 cpu_to_core_group(i, cpu_map, &sd->groups);
6419#endif 6420#endif
6420 6421
6421#ifdef CONFIG_SCHED_SMT 6422#ifdef CONFIG_SCHED_SMT
6422 p = sd; 6423 p = sd;
6423 sd = &per_cpu(cpu_domains, i); 6424 sd = &per_cpu(cpu_domains, i);
6424 group = cpu_to_cpu_group(i, cpu_map);
6425 *sd = SD_SIBLING_INIT; 6425 *sd = SD_SIBLING_INIT;
6426 sd->span = cpu_sibling_map[i]; 6426 sd->span = cpu_sibling_map[i];
6427 cpus_and(sd->span, sd->span, *cpu_map); 6427 cpus_and(sd->span, sd->span, *cpu_map);
6428 sd->parent = p; 6428 sd->parent = p;
6429 p->child = sd; 6429 p->child = sd;
6430 sd->groups = &sched_group_cpus[group]; 6430 cpu_to_cpu_group(i, cpu_map, &sd->groups);
6431#endif 6431#endif
6432 } 6432 }
6433 6433
@@ -6439,8 +6439,7 @@ static int build_sched_domains(const cpumask_t *cpu_map)
6439 if (i != first_cpu(this_sibling_map)) 6439 if (i != first_cpu(this_sibling_map))
6440 continue; 6440 continue;
6441 6441
6442 init_sched_build_groups(sched_group_cpus, this_sibling_map, 6442 init_sched_build_groups(this_sibling_map, cpu_map, &cpu_to_cpu_group);
6443 cpu_map, &cpu_to_cpu_group);
6444 } 6443 }
6445#endif 6444#endif
6446 6445
@@ -6451,8 +6450,7 @@ static int build_sched_domains(const cpumask_t *cpu_map)
6451 cpus_and(this_core_map, this_core_map, *cpu_map); 6450 cpus_and(this_core_map, this_core_map, *cpu_map);
6452 if (i != first_cpu(this_core_map)) 6451 if (i != first_cpu(this_core_map))
6453 continue; 6452 continue;
6454 init_sched_build_groups(sched_group_core, this_core_map, 6453 init_sched_build_groups(this_core_map, cpu_map, &cpu_to_core_group);
6455 cpu_map, &cpu_to_core_group);
6456 } 6454 }
6457#endif 6455#endif
6458 6456
@@ -6465,15 +6463,13 @@ static int build_sched_domains(const cpumask_t *cpu_map)
6465 if (cpus_empty(nodemask)) 6463 if (cpus_empty(nodemask))
6466 continue; 6464 continue;
6467 6465
6468 init_sched_build_groups(sched_group_phys, nodemask, 6466 init_sched_build_groups(nodemask, cpu_map, &cpu_to_phys_group);
6469 cpu_map, &cpu_to_phys_group);
6470 } 6467 }
6471 6468
6472#ifdef CONFIG_NUMA 6469#ifdef CONFIG_NUMA
6473 /* Set up node groups */ 6470 /* Set up node groups */
6474 if (sched_group_allnodes) 6471 if (sd_allnodes)
6475 init_sched_build_groups(sched_group_allnodes, *cpu_map, 6472 init_sched_build_groups(*cpu_map, cpu_map, &cpu_to_allnodes_group);
6476 cpu_map, &cpu_to_allnodes_group);
6477 6473
6478 for (i = 0; i < MAX_NUMNODES; i++) { 6474 for (i = 0; i < MAX_NUMNODES; i++) {
6479 /* Set up node groups */ 6475 /* Set up node groups */
@@ -6565,10 +6561,10 @@ static int build_sched_domains(const cpumask_t *cpu_map)
6565 for (i = 0; i < MAX_NUMNODES; i++) 6561 for (i = 0; i < MAX_NUMNODES; i++)
6566 init_numa_sched_groups_power(sched_group_nodes[i]); 6562 init_numa_sched_groups_power(sched_group_nodes[i]);
6567 6563
6568 if (sched_group_allnodes) { 6564 if (sd_allnodes) {
6569 int group = cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map); 6565 struct sched_group *sg;
6570 struct sched_group *sg = &sched_group_allnodes[group];
6571 6566
6567 cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg);
6572 init_numa_sched_groups_power(sg); 6568 init_numa_sched_groups_power(sg);
6573 } 6569 }
6574#endif 6570#endif