diff options
author | Andi Kleen <ak@suse.de> | 2005-11-05 11:25:53 -0500 |
---|---|---|
committer | Linus Torvalds <torvalds@g5.osdl.org> | 2005-11-14 22:55:14 -0500 |
commit | 69d81fcde7797342417591ba7affb372b9c86eae (patch) | |
tree | 97fbc73fd20f8ebc313b2bc41367a8efe36c2c3d | |
parent | 50895c5d76e15d8af480eff1aaab5770cabbc2c2 (diff) |
[PATCH] x86_64: Speed up numa_node_id by putting it directly into the PDA
Not go from the CPU number to an mapping array.
Mode number is often used now in fast paths.
This also adds a generic numa_node_id to all the topology includes
Suggested by Eric Dumazet
Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
-rw-r--r-- | arch/x86_64/kernel/setup.c | 4 | ||||
-rw-r--r-- | arch/x86_64/mm/numa.c | 10 | ||||
-rw-r--r-- | arch/x86_64/mm/srat.c | 2 | ||||
-rw-r--r-- | include/asm-x86_64/numa.h | 2 | ||||
-rw-r--r-- | include/asm-x86_64/pda.h | 1 | ||||
-rw-r--r-- | include/asm-x86_64/topology.h | 2 | ||||
-rw-r--r-- | include/linux/mmzone.h | 2 |
7 files changed, 18 insertions, 5 deletions
diff --git a/arch/x86_64/kernel/setup.c b/arch/x86_64/kernel/setup.c index f27731ac95c..99cfa751949 100644 --- a/arch/x86_64/kernel/setup.c +++ b/arch/x86_64/kernel/setup.c | |||
@@ -823,7 +823,7 @@ static void __init amd_detect_cmp(struct cpuinfo_x86 *c) | |||
823 | if (!node_online(node)) | 823 | if (!node_online(node)) |
824 | node = nearby_node(apicid); | 824 | node = nearby_node(apicid); |
825 | } | 825 | } |
826 | cpu_to_node[cpu] = node; | 826 | numa_set_node(cpu, node); |
827 | 827 | ||
828 | printk(KERN_INFO "CPU %d(%d) -> Node %d -> Core %d\n", | 828 | printk(KERN_INFO "CPU %d(%d) -> Node %d -> Core %d\n", |
829 | cpu, c->x86_num_cores, node, cpu_core_id[cpu]); | 829 | cpu, c->x86_num_cores, node, cpu_core_id[cpu]); |
@@ -975,7 +975,7 @@ static void srat_detect_node(void) | |||
975 | node = apicid_to_node[hard_smp_processor_id()]; | 975 | node = apicid_to_node[hard_smp_processor_id()]; |
976 | if (node == NUMA_NO_NODE) | 976 | if (node == NUMA_NO_NODE) |
977 | node = 0; | 977 | node = 0; |
978 | cpu_to_node[cpu] = node; | 978 | numa_set_node(cpu, node); |
979 | 979 | ||
980 | if (acpi_numa > 0) | 980 | if (acpi_numa > 0) |
981 | printk(KERN_INFO "CPU %d -> Node %d\n", cpu, node); | 981 | printk(KERN_INFO "CPU %d -> Node %d\n", cpu, node); |
diff --git a/arch/x86_64/mm/numa.c b/arch/x86_64/mm/numa.c index 18e86e2eac2..4bf64583ba3 100644 --- a/arch/x86_64/mm/numa.c +++ b/arch/x86_64/mm/numa.c | |||
@@ -156,7 +156,7 @@ void __init numa_init_array(void) | |||
156 | for (i = 0; i < NR_CPUS; i++) { | 156 | for (i = 0; i < NR_CPUS; i++) { |
157 | if (cpu_to_node[i] != NUMA_NO_NODE) | 157 | if (cpu_to_node[i] != NUMA_NO_NODE) |
158 | continue; | 158 | continue; |
159 | cpu_to_node[i] = rr; | 159 | numa_set_node(i, rr); |
160 | rr = next_node(rr, node_online_map); | 160 | rr = next_node(rr, node_online_map); |
161 | if (rr == MAX_NUMNODES) | 161 | if (rr == MAX_NUMNODES) |
162 | rr = first_node(node_online_map); | 162 | rr = first_node(node_online_map); |
@@ -242,7 +242,7 @@ void __init numa_initmem_init(unsigned long start_pfn, unsigned long end_pfn) | |||
242 | nodes_clear(node_online_map); | 242 | nodes_clear(node_online_map); |
243 | node_set_online(0); | 243 | node_set_online(0); |
244 | for (i = 0; i < NR_CPUS; i++) | 244 | for (i = 0; i < NR_CPUS; i++) |
245 | cpu_to_node[i] = 0; | 245 | numa_set_node(i, 0); |
246 | node_to_cpumask[0] = cpumask_of_cpu(0); | 246 | node_to_cpumask[0] = cpumask_of_cpu(0); |
247 | setup_node_bootmem(0, start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT); | 247 | setup_node_bootmem(0, start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT); |
248 | } | 248 | } |
@@ -252,6 +252,12 @@ __cpuinit void numa_add_cpu(int cpu) | |||
252 | set_bit(cpu, &node_to_cpumask[cpu_to_node(cpu)]); | 252 | set_bit(cpu, &node_to_cpumask[cpu_to_node(cpu)]); |
253 | } | 253 | } |
254 | 254 | ||
255 | void __cpuinit numa_set_node(int cpu, int node) | ||
256 | { | ||
257 | cpu_pda[cpu].nodenumber = node; | ||
258 | cpu_to_node[cpu] = node; | ||
259 | } | ||
260 | |||
255 | unsigned long __init numa_free_all_bootmem(void) | 261 | unsigned long __init numa_free_all_bootmem(void) |
256 | { | 262 | { |
257 | int i; | 263 | int i; |
diff --git a/arch/x86_64/mm/srat.c b/arch/x86_64/mm/srat.c index 4b2e844c15a..c7aa08a5804 100644 --- a/arch/x86_64/mm/srat.c +++ b/arch/x86_64/mm/srat.c | |||
@@ -203,7 +203,7 @@ int __init acpi_scan_nodes(unsigned long start, unsigned long end) | |||
203 | if (cpu_to_node[i] == NUMA_NO_NODE) | 203 | if (cpu_to_node[i] == NUMA_NO_NODE) |
204 | continue; | 204 | continue; |
205 | if (!node_isset(cpu_to_node[i], nodes_parsed)) | 205 | if (!node_isset(cpu_to_node[i], nodes_parsed)) |
206 | cpu_to_node[i] = NUMA_NO_NODE; | 206 | numa_set_node(i, NUMA_NO_NODE); |
207 | } | 207 | } |
208 | numa_init_array(); | 208 | numa_init_array(); |
209 | return 0; | 209 | return 0; |
diff --git a/include/asm-x86_64/numa.h b/include/asm-x86_64/numa.h index bcf55c3f7f7..d51e56fdc3d 100644 --- a/include/asm-x86_64/numa.h +++ b/include/asm-x86_64/numa.h | |||
@@ -17,6 +17,8 @@ extern void numa_add_cpu(int cpu); | |||
17 | extern void numa_init_array(void); | 17 | extern void numa_init_array(void); |
18 | extern int numa_off; | 18 | extern int numa_off; |
19 | 19 | ||
20 | extern void numa_set_node(int cpu, int node); | ||
21 | |||
20 | extern unsigned char apicid_to_node[256]; | 22 | extern unsigned char apicid_to_node[256]; |
21 | 23 | ||
22 | #define NUMA_NO_NODE 0xff | 24 | #define NUMA_NO_NODE 0xff |
diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h index bbf89aa8a1a..8733ccfa442 100644 --- a/include/asm-x86_64/pda.h +++ b/include/asm-x86_64/pda.h | |||
@@ -15,6 +15,7 @@ struct x8664_pda { | |||
15 | int irqcount; /* Irq nesting counter. Starts with -1 */ | 15 | int irqcount; /* Irq nesting counter. Starts with -1 */ |
16 | int cpunumber; /* Logical CPU number */ | 16 | int cpunumber; /* Logical CPU number */ |
17 | char *irqstackptr; /* top of irqstack */ | 17 | char *irqstackptr; /* top of irqstack */ |
18 | int nodenumber; /* number of current node */ | ||
18 | unsigned int __softirq_pending; | 19 | unsigned int __softirq_pending; |
19 | unsigned int __nmi_count; /* number of NMI on this CPUs */ | 20 | unsigned int __nmi_count; /* number of NMI on this CPUs */ |
20 | struct mm_struct *active_mm; | 21 | struct mm_struct *active_mm; |
diff --git a/include/asm-x86_64/topology.h b/include/asm-x86_64/topology.h index 1c603cd7e4d..d39ebd5263e 100644 --- a/include/asm-x86_64/topology.h +++ b/include/asm-x86_64/topology.h | |||
@@ -28,6 +28,8 @@ extern int __node_distance(int, int); | |||
28 | #define pcibus_to_node(bus) ((long)(bus->sysdata)) | 28 | #define pcibus_to_node(bus) ((long)(bus->sysdata)) |
29 | #define pcibus_to_cpumask(bus) node_to_cpumask(pcibus_to_node(bus)); | 29 | #define pcibus_to_cpumask(bus) node_to_cpumask(pcibus_to_node(bus)); |
30 | 30 | ||
31 | #define numa_node_id() read_pda(nodenumber) | ||
32 | |||
31 | /* sched_domains SD_NODE_INIT for x86_64 machines */ | 33 | /* sched_domains SD_NODE_INIT for x86_64 machines */ |
32 | #define SD_NODE_INIT (struct sched_domain) { \ | 34 | #define SD_NODE_INIT (struct sched_domain) { \ |
33 | .span = CPU_MASK_NONE, \ | 35 | .span = CPU_MASK_NONE, \ |
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 57fc99c67c3..f3cffc354de 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h | |||
@@ -435,7 +435,9 @@ int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int, struct file *, | |||
435 | 435 | ||
436 | #include <linux/topology.h> | 436 | #include <linux/topology.h> |
437 | /* Returns the number of the current Node. */ | 437 | /* Returns the number of the current Node. */ |
438 | #ifndef numa_node_id | ||
438 | #define numa_node_id() (cpu_to_node(raw_smp_processor_id())) | 439 | #define numa_node_id() (cpu_to_node(raw_smp_processor_id())) |
440 | #endif | ||
439 | 441 | ||
440 | #ifndef CONFIG_NEED_MULTIPLE_NODES | 442 | #ifndef CONFIG_NEED_MULTIPLE_NODES |
441 | 443 | ||