aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched.c
diff options
context:
space:
mode:
authorMike Travis <travis@sgi.com>2008-12-31 21:08:45 -0500
committerIngo Molnar <mingo@elte.hu>2009-01-03 13:00:09 -0500
commit6ca09dfc9f180d038dcef93c167a833f43a8246f (patch)
tree48cd4e6530a7523d7d14d2c18fa7d3bd2ef49a4e /kernel/sched.c
parent730cf27246225d56ca1603b2f3c4fdbf882d4e51 (diff)
sched: put back some stack hog changes that were undone in kernel/sched.c
Impact: prevents panic from stack overflow on numa-capable machines. Some of the "removal of stack hogs" changes in kernel/sched.c by using node_to_cpumask_ptr were undone by the early cpumask API updates, and causes a panic due to stack overflow. This patch undoes those changes by using cpumask_of_node() which returns a 'const struct cpumask *'. In addition, cpu_coregoup_map is replaced with cpu_coregroup_mask further reducing stack usage. (Both of these updates removed 9 FIXME's!) Also: Pick up some remaining changes from the old 'cpumask_t' functions to the new 'struct cpumask *' functions. Optimize memory traffic by allocating each percpu local_cpu_mask on the same node as the referring cpu. Signed-off-by: Mike Travis <travis@sgi.com> Acked-by: Rusty Russell <rusty@rustcorp.com.au> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'kernel/sched.c')
-rw-r--r--kernel/sched.c53
1 files changed, 15 insertions, 38 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index 27ba1d642f0f..dd862d70e715 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -3715,7 +3715,7 @@ redo:
3715 * don't kick the migration_thread, if the curr 3715 * don't kick the migration_thread, if the curr
3716 * task on busiest cpu can't be moved to this_cpu 3716 * task on busiest cpu can't be moved to this_cpu
3717 */ 3717 */
3718 if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) { 3718 if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) {
3719 double_unlock_balance(this_rq, busiest); 3719 double_unlock_balance(this_rq, busiest);
3720 all_pinned = 1; 3720 all_pinned = 1;
3721 return ld_moved; 3721 return ld_moved;
@@ -6220,9 +6220,7 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
6220static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) 6220static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6221{ 6221{
6222 int dest_cpu; 6222 int dest_cpu;
6223 /* FIXME: Use cpumask_of_node here. */ 6223 const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(dead_cpu));
6224 cpumask_t _nodemask = node_to_cpumask(cpu_to_node(dead_cpu));
6225 const struct cpumask *nodemask = &_nodemask;
6226 6224
6227again: 6225again:
6228 /* Look for allowed, online CPU in same node. */ 6226 /* Look for allowed, online CPU in same node. */
@@ -7133,21 +7131,18 @@ static int find_next_best_node(int node, nodemask_t *used_nodes)
7133static void sched_domain_node_span(int node, struct cpumask *span) 7131static void sched_domain_node_span(int node, struct cpumask *span)
7134{ 7132{
7135 nodemask_t used_nodes; 7133 nodemask_t used_nodes;
7136 /* FIXME: use cpumask_of_node() */
7137 node_to_cpumask_ptr(nodemask, node);
7138 int i; 7134 int i;
7139 7135
7140 cpus_clear(*span); 7136 cpumask_clear(span);
7141 nodes_clear(used_nodes); 7137 nodes_clear(used_nodes);
7142 7138
7143 cpus_or(*span, *span, *nodemask); 7139 cpumask_or(span, span, cpumask_of_node(node));
7144 node_set(node, used_nodes); 7140 node_set(node, used_nodes);
7145 7141
7146 for (i = 1; i < SD_NODES_PER_DOMAIN; i++) { 7142 for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
7147 int next_node = find_next_best_node(node, &used_nodes); 7143 int next_node = find_next_best_node(node, &used_nodes);
7148 7144
7149 node_to_cpumask_ptr_next(nodemask, next_node); 7145 cpumask_or(span, span, cpumask_of_node(next_node));
7150 cpus_or(*span, *span, *nodemask);
7151 } 7146 }
7152} 7147}
7153#endif /* CONFIG_NUMA */ 7148#endif /* CONFIG_NUMA */
@@ -7227,9 +7222,7 @@ cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
7227{ 7222{
7228 int group; 7223 int group;
7229#ifdef CONFIG_SCHED_MC 7224#ifdef CONFIG_SCHED_MC
7230 /* FIXME: Use cpu_coregroup_mask. */ 7225 cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
7231 *mask = cpu_coregroup_map(cpu);
7232 cpus_and(*mask, *mask, *cpu_map);
7233 group = cpumask_first(mask); 7226 group = cpumask_first(mask);
7234#elif defined(CONFIG_SCHED_SMT) 7227#elif defined(CONFIG_SCHED_SMT)
7235 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map); 7228 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
@@ -7259,10 +7252,8 @@ static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
7259 struct cpumask *nodemask) 7252 struct cpumask *nodemask)
7260{ 7253{
7261 int group; 7254 int group;
7262 /* FIXME: use cpumask_of_node */
7263 node_to_cpumask_ptr(pnodemask, cpu_to_node(cpu));
7264 7255
7265 cpumask_and(nodemask, pnodemask, cpu_map); 7256 cpumask_and(nodemask, cpumask_of_node(cpu_to_node(cpu)), cpu_map);
7266 group = cpumask_first(nodemask); 7257 group = cpumask_first(nodemask);
7267 7258
7268 if (sg) 7259 if (sg)
@@ -7313,10 +7304,8 @@ static void free_sched_groups(const struct cpumask *cpu_map,
7313 7304
7314 for (i = 0; i < nr_node_ids; i++) { 7305 for (i = 0; i < nr_node_ids; i++) {
7315 struct sched_group *oldsg, *sg = sched_group_nodes[i]; 7306 struct sched_group *oldsg, *sg = sched_group_nodes[i];
7316 /* FIXME: Use cpumask_of_node */
7317 node_to_cpumask_ptr(pnodemask, i);
7318 7307
7319 cpus_and(*nodemask, *pnodemask, *cpu_map); 7308 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7320 if (cpumask_empty(nodemask)) 7309 if (cpumask_empty(nodemask))
7321 continue; 7310 continue;
7322 7311
@@ -7525,9 +7514,7 @@ static int __build_sched_domains(const struct cpumask *cpu_map,
7525 for_each_cpu(i, cpu_map) { 7514 for_each_cpu(i, cpu_map) {
7526 struct sched_domain *sd = NULL, *p; 7515 struct sched_domain *sd = NULL, *p;
7527 7516
7528 /* FIXME: use cpumask_of_node */ 7517 cpumask_and(nodemask, cpumask_of_node(cpu_to_node(i)), cpu_map);
7529 *nodemask = node_to_cpumask(cpu_to_node(i));
7530 cpus_and(*nodemask, *nodemask, *cpu_map);
7531 7518
7532#ifdef CONFIG_NUMA 7519#ifdef CONFIG_NUMA
7533 if (cpumask_weight(cpu_map) > 7520 if (cpumask_weight(cpu_map) >
@@ -7568,9 +7555,8 @@ static int __build_sched_domains(const struct cpumask *cpu_map,
7568 sd = &per_cpu(core_domains, i).sd; 7555 sd = &per_cpu(core_domains, i).sd;
7569 SD_INIT(sd, MC); 7556 SD_INIT(sd, MC);
7570 set_domain_attribute(sd, attr); 7557 set_domain_attribute(sd, attr);
7571 *sched_domain_span(sd) = cpu_coregroup_map(i); 7558 cpumask_and(sched_domain_span(sd), cpu_map,
7572 cpumask_and(sched_domain_span(sd), 7559 cpu_coregroup_mask(i));
7573 sched_domain_span(sd), cpu_map);
7574 sd->parent = p; 7560 sd->parent = p;
7575 p->child = sd; 7561 p->child = sd;
7576 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask); 7562 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7606,9 +7592,7 @@ static int __build_sched_domains(const struct cpumask *cpu_map,
7606#ifdef CONFIG_SCHED_MC 7592#ifdef CONFIG_SCHED_MC
7607 /* Set up multi-core groups */ 7593 /* Set up multi-core groups */
7608 for_each_cpu(i, cpu_map) { 7594 for_each_cpu(i, cpu_map) {
7609 /* FIXME: Use cpu_coregroup_mask */ 7595 cpumask_and(this_core_map, cpu_coregroup_mask(i), cpu_map);
7610 *this_core_map = cpu_coregroup_map(i);
7611 cpus_and(*this_core_map, *this_core_map, *cpu_map);
7612 if (i != cpumask_first(this_core_map)) 7596 if (i != cpumask_first(this_core_map))
7613 continue; 7597 continue;
7614 7598
@@ -7620,9 +7604,7 @@ static int __build_sched_domains(const struct cpumask *cpu_map,
7620 7604
7621 /* Set up physical groups */ 7605 /* Set up physical groups */
7622 for (i = 0; i < nr_node_ids; i++) { 7606 for (i = 0; i < nr_node_ids; i++) {
7623 /* FIXME: Use cpumask_of_node */ 7607 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7624 *nodemask = node_to_cpumask(i);
7625 cpus_and(*nodemask, *nodemask, *cpu_map);
7626 if (cpumask_empty(nodemask)) 7608 if (cpumask_empty(nodemask))
7627 continue; 7609 continue;
7628 7610
@@ -7644,11 +7626,8 @@ static int __build_sched_domains(const struct cpumask *cpu_map,
7644 struct sched_group *sg, *prev; 7626 struct sched_group *sg, *prev;
7645 int j; 7627 int j;
7646 7628
7647 /* FIXME: Use cpumask_of_node */
7648 *nodemask = node_to_cpumask(i);
7649 cpumask_clear(covered); 7629 cpumask_clear(covered);
7650 7630 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7651 cpus_and(*nodemask, *nodemask, *cpu_map);
7652 if (cpumask_empty(nodemask)) { 7631 if (cpumask_empty(nodemask)) {
7653 sched_group_nodes[i] = NULL; 7632 sched_group_nodes[i] = NULL;
7654 continue; 7633 continue;
@@ -7679,8 +7658,6 @@ static int __build_sched_domains(const struct cpumask *cpu_map,
7679 7658
7680 for (j = 0; j < nr_node_ids; j++) { 7659 for (j = 0; j < nr_node_ids; j++) {
7681 int n = (i + j) % nr_node_ids; 7660 int n = (i + j) % nr_node_ids;
7682 /* FIXME: Use cpumask_of_node */
7683 node_to_cpumask_ptr(pnodemask, n);
7684 7661
7685 cpumask_complement(notcovered, covered); 7662 cpumask_complement(notcovered, covered);
7686 cpumask_and(tmpmask, notcovered, cpu_map); 7663 cpumask_and(tmpmask, notcovered, cpu_map);
@@ -7688,7 +7665,7 @@ static int __build_sched_domains(const struct cpumask *cpu_map,
7688 if (cpumask_empty(tmpmask)) 7665 if (cpumask_empty(tmpmask))
7689 break; 7666 break;
7690 7667
7691 cpumask_and(tmpmask, tmpmask, pnodemask); 7668 cpumask_and(tmpmask, tmpmask, cpumask_of_node(n));
7692 if (cpumask_empty(tmpmask)) 7669 if (cpumask_empty(tmpmask))
7693 continue; 7670 continue;
7694 7671