aboutsummaryrefslogtreecommitdiffstats
path: root/include/asm-x86
diff options
context:
space:
mode:
authorMike Travis <travis@sgi.com>2008-05-12 15:21:12 -0400
committerIngo Molnar <mingo@elte.hu>2008-07-08 05:31:20 -0400
commit23ca4bba3e20c6c3cb11c1bb0ab4770b724d39ac (patch)
tree39ba5f7705e48717d7a6f2621b8ca7e7015c9802 /include/asm-x86
parent1184dc2ffe2c8fb9afb766d870850f2c3165ef25 (diff)
x86: cleanup early per cpu variables/accesses v4
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is used by some per_cpu variables that are initialized and accessed before there are per_cpu areas allocated. ["Early" in respect to per_cpu variables is "earlier than the per_cpu areas have been setup".] This patchset adds these new macros: DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) EXPORT_EARLY_PER_CPU_SYMBOL(_name) DECLARE_EARLY_PER_CPU(_type, _name) early_per_cpu_ptr(_name) early_per_cpu_map(_name, _idx) early_per_cpu(_name, _cpu) The DEFINE macro defines the per_cpu variable as well as the early map and pointer. It also initializes the per_cpu variable and map elements to "_initvalue". The early_* macros provide access to the initial map (usually setup during system init) and the early pointer. This pointer is initialized to point to the early map but is then NULL'ed when the actual per_cpu areas are setup. After that the per_cpu variable is the correct access to the variable. The early_per_cpu() macro is not very efficient but does show how to access the variable if you have a function that can be called both "early" and "late". It tests the early ptr to be NULL, and if not then it's still valid. Otherwise, the per_cpu variable is used instead: #define early_per_cpu(_name, _cpu) \ (early_per_cpu_ptr(_name) ? \ early_per_cpu_ptr(_name)[_cpu] : \ per_cpu(_name, _cpu)) A better method is to actually check the pointer manually. In the case below, numa_set_node can be called both "early" and "late": void __cpuinit numa_set_node(int cpu, int node) { int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map); if (cpu_to_node_map) cpu_to_node_map[cpu] = node; else per_cpu(x86_cpu_to_node_map, cpu) = node; } * Add a flag "arch_provides_topology_pointers" that indicates pointers to topology cpumask_t maps are available. Otherwise, use the function returning the cpumask_t value. This is useful if cpumask_t set size is very large to avoid copying data on to/off of the stack. * The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while the non-debug case has been optimized a bit. * Remove an unreferenced compiler warning in drivers/base/topology.c * Clean up #ifdef in setup.c For inclusion into sched-devel/latest tree. Based on: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git + sched-devel/latest .../mingo/linux-2.6-sched-devel.git Signed-off-by: Mike Travis <travis@sgi.com> Signed-off-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Diffstat (limited to 'include/asm-x86')
-rw-r--r--include/asm-x86/numa_64.h19
-rw-r--r--include/asm-x86/percpu.h46
-rw-r--r--include/asm-x86/smp.h15
-rw-r--r--include/asm-x86/topology.h143
4 files changed, 140 insertions, 83 deletions
diff --git a/include/asm-x86/numa_64.h b/include/asm-x86/numa_64.h
index 22e87c9f6a8..b510daf4f4d 100644
--- a/include/asm-x86/numa_64.h
+++ b/include/asm-x86/numa_64.h
@@ -14,11 +14,9 @@ extern int compute_hash_shift(struct bootnode *nodes, int numblks,
14 14
15#define ZONE_ALIGN (1UL << (MAX_ORDER+PAGE_SHIFT)) 15#define ZONE_ALIGN (1UL << (MAX_ORDER+PAGE_SHIFT))
16 16
17extern void numa_add_cpu(int cpu);
18extern void numa_init_array(void); 17extern void numa_init_array(void);
19extern int numa_off; 18extern int numa_off;
20 19
21extern void numa_set_node(int cpu, int node);
22extern void srat_reserve_add_area(int nodeid); 20extern void srat_reserve_add_area(int nodeid);
23extern int hotadd_percent; 21extern int hotadd_percent;
24 22
@@ -31,15 +29,16 @@ extern void setup_node_bootmem(int nodeid, unsigned long start,
31 29
32#ifdef CONFIG_NUMA 30#ifdef CONFIG_NUMA
33extern void __init init_cpu_to_node(void); 31extern void __init init_cpu_to_node(void);
34 32extern void __cpuinit numa_set_node(int cpu, int node);
35static inline void clear_node_cpumask(int cpu) 33extern void __cpuinit numa_clear_node(int cpu);
36{ 34extern void __cpuinit numa_add_cpu(int cpu);
37 clear_bit(cpu, (unsigned long *)&node_to_cpumask_map[cpu_to_node(cpu)]); 35extern void __cpuinit numa_remove_cpu(int cpu);
38}
39
40#else 36#else
41#define init_cpu_to_node() do {} while (0) 37static inline void init_cpu_to_node(void) { }
42#define clear_node_cpumask(cpu) do {} while (0) 38static inline void numa_set_node(int cpu, int node) { }
39static inline void numa_clear_node(int cpu) { }
40static inline void numa_add_cpu(int cpu, int node) { }
41static inline void numa_remove_cpu(int cpu) { }
43#endif 42#endif
44 43
45#endif 44#endif
diff --git a/include/asm-x86/percpu.h b/include/asm-x86/percpu.h
index 736fc3bb8e1..912a3a17b9d 100644
--- a/include/asm-x86/percpu.h
+++ b/include/asm-x86/percpu.h
@@ -143,4 +143,50 @@ do { \
143#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val) 143#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val)
144#endif /* !__ASSEMBLY__ */ 144#endif /* !__ASSEMBLY__ */
145#endif /* !CONFIG_X86_64 */ 145#endif /* !CONFIG_X86_64 */
146
147#ifdef CONFIG_SMP
148
149/*
150 * Define the "EARLY_PER_CPU" macros. These are used for some per_cpu
151 * variables that are initialized and accessed before there are per_cpu
152 * areas allocated.
153 */
154
155#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
156 DEFINE_PER_CPU(_type, _name) = _initvalue; \
157 __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \
158 { [0 ... NR_CPUS-1] = _initvalue }; \
159 __typeof__(_type) *_name##_early_ptr = _name##_early_map
160
161#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
162 EXPORT_PER_CPU_SYMBOL(_name)
163
164#define DECLARE_EARLY_PER_CPU(_type, _name) \
165 DECLARE_PER_CPU(_type, _name); \
166 extern __typeof__(_type) *_name##_early_ptr; \
167 extern __typeof__(_type) _name##_early_map[]
168
169#define early_per_cpu_ptr(_name) (_name##_early_ptr)
170#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
171#define early_per_cpu(_name, _cpu) \
172 (early_per_cpu_ptr(_name) ? \
173 early_per_cpu_ptr(_name)[_cpu] : \
174 per_cpu(_name, _cpu))
175
176#else /* !CONFIG_SMP */
177#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
178 DEFINE_PER_CPU(_type, _name) = _initvalue
179
180#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
181 EXPORT_PER_CPU_SYMBOL(_name)
182
183#define DECLARE_EARLY_PER_CPU(_type, _name) \
184 DECLARE_PER_CPU(_type, _name)
185
186#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu)
187#define early_per_cpu_ptr(_name) NULL
188/* no early_per_cpu_map() */
189
190#endif /* !CONFIG_SMP */
191
146#endif /* _ASM_X86_PERCPU_H_ */ 192#endif /* _ASM_X86_PERCPU_H_ */
diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h
index 1ebaa5cd311..ec841639fb4 100644
--- a/include/asm-x86/smp.h
+++ b/include/asm-x86/smp.h
@@ -29,21 +29,12 @@ extern int smp_num_siblings;
29extern unsigned int num_processors; 29extern unsigned int num_processors;
30extern cpumask_t cpu_initialized; 30extern cpumask_t cpu_initialized;
31 31
32#ifdef CONFIG_SMP
33extern u16 x86_cpu_to_apicid_init[];
34extern u16 x86_bios_cpu_apicid_init[];
35extern void *x86_cpu_to_apicid_early_ptr;
36extern void *x86_bios_cpu_apicid_early_ptr;
37#else
38#define x86_cpu_to_apicid_early_ptr NULL
39#define x86_bios_cpu_apicid_early_ptr NULL
40#endif
41
42DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); 32DECLARE_PER_CPU(cpumask_t, cpu_sibling_map);
43DECLARE_PER_CPU(cpumask_t, cpu_core_map); 33DECLARE_PER_CPU(cpumask_t, cpu_core_map);
44DECLARE_PER_CPU(u16, cpu_llc_id); 34DECLARE_PER_CPU(u16, cpu_llc_id);
45DECLARE_PER_CPU(u16, x86_cpu_to_apicid); 35
46DECLARE_PER_CPU(u16, x86_bios_cpu_apicid); 36DECLARE_EARLY_PER_CPU(u16, x86_cpu_to_apicid);
37DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid);
47 38
48/* Static state in head.S used to set up a CPU */ 39/* Static state in head.S used to set up a CPU */
49extern struct { 40extern struct {
diff --git a/include/asm-x86/topology.h b/include/asm-x86/topology.h
index dcf3f8131d6..dac09cb66dc 100644
--- a/include/asm-x86/topology.h
+++ b/include/asm-x86/topology.h
@@ -35,87 +35,67 @@
35# endif 35# endif
36#endif 36#endif
37 37
38/* Node not present */
39#define NUMA_NO_NODE (-1)
40
38#ifdef CONFIG_NUMA 41#ifdef CONFIG_NUMA
39#include <linux/cpumask.h> 42#include <linux/cpumask.h>
40#include <asm/mpspec.h> 43#include <asm/mpspec.h>
41 44
42/* Mappings between logical cpu number and node number */
43#ifdef CONFIG_X86_32 45#ifdef CONFIG_X86_32
44extern int cpu_to_node_map[];
45#else
46/* Returns the number of the current Node. */
47#define numa_node_id() (early_cpu_to_node(raw_smp_processor_id()))
48#endif
49
50DECLARE_PER_CPU(int, x86_cpu_to_node_map);
51
52#ifdef CONFIG_SMP
53extern int x86_cpu_to_node_map_init[];
54extern void *x86_cpu_to_node_map_early_ptr;
55#else
56#define x86_cpu_to_node_map_early_ptr NULL
57#endif
58 46
47/* Mappings between node number and cpus on that node. */
59extern cpumask_t node_to_cpumask_map[]; 48extern cpumask_t node_to_cpumask_map[];
60 49
61#define NUMA_NO_NODE (-1) 50/* Mappings between logical cpu number and node number */
51extern int cpu_to_node_map[];
62 52
63/* Returns the number of the node containing CPU 'cpu' */ 53/* Returns the number of the node containing CPU 'cpu' */
64#ifdef CONFIG_X86_32
65#define early_cpu_to_node(cpu) cpu_to_node(cpu)
66static inline int cpu_to_node(int cpu) 54static inline int cpu_to_node(int cpu)
67{ 55{
68 return cpu_to_node_map[cpu]; 56 return cpu_to_node_map[cpu];
69} 57}
58#define early_cpu_to_node(cpu) cpu_to_node(cpu)
70 59
71#else /* CONFIG_X86_64 */ 60#else /* CONFIG_X86_64 */
72 61
73#ifdef CONFIG_SMP 62/* Mappings between node number and cpus on that node. */
74static inline int early_cpu_to_node(int cpu) 63extern cpumask_t node_to_cpumask_map[];
75{ 64
76 int *cpu_to_node_map = x86_cpu_to_node_map_early_ptr; 65/* Mappings between logical cpu number and node number */
77 66DECLARE_EARLY_PER_CPU(int, x86_cpu_to_node_map);
78 if (cpu_to_node_map) 67
79 return cpu_to_node_map[cpu]; 68/* Returns the number of the current Node. */
80 else if (per_cpu_offset(cpu)) 69#define numa_node_id() (per_cpu(x86_cpu_to_node_map, raw_smp_processor_id()))
81 return per_cpu(x86_cpu_to_node_map, cpu); 70
82 else 71#ifdef CONFIG_DEBUG_PER_CPU_MAPS
83 return NUMA_NO_NODE; 72extern int cpu_to_node(int cpu);
84} 73extern int early_cpu_to_node(int cpu);
85#else 74extern cpumask_t *_node_to_cpumask_ptr(int node);
86#define early_cpu_to_node(cpu) cpu_to_node(cpu) 75extern cpumask_t node_to_cpumask(int node);
87#endif
88 76
77#else /* !CONFIG_DEBUG_PER_CPU_MAPS */
78
79/* Returns the number of the node containing CPU 'cpu' */
89static inline int cpu_to_node(int cpu) 80static inline int cpu_to_node(int cpu)
90{ 81{
91#ifdef CONFIG_DEBUG_PER_CPU_MAPS
92 if (x86_cpu_to_node_map_early_ptr) {
93 printk("KERN_NOTICE cpu_to_node(%d): usage too early!\n",
94 (int)cpu);
95 dump_stack();
96 return ((int *)x86_cpu_to_node_map_early_ptr)[cpu];
97 }
98#endif
99 return per_cpu(x86_cpu_to_node_map, cpu); 82 return per_cpu(x86_cpu_to_node_map, cpu);
100} 83}
101 84
102#ifdef CONFIG_NUMA 85/* Same function but used if called before per_cpu areas are setup */
103 86static inline int early_cpu_to_node(int cpu)
104/* Returns a pointer to the cpumask of CPUs on Node 'node'. */ 87{
105#define node_to_cpumask_ptr(v, node) \ 88 if (early_per_cpu_ptr(x86_cpu_to_node_map))
106 cpumask_t *v = &(node_to_cpumask_map[node]) 89 return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
107
108#define node_to_cpumask_ptr_next(v, node) \
109 v = &(node_to_cpumask_map[node])
110#endif
111 90
112#endif /* CONFIG_X86_64 */ 91 return per_cpu(x86_cpu_to_node_map, cpu);
92}
113 93
114/* 94/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
115 * Returns the number of the node containing Node 'node'. This 95static inline cpumask_t *_node_to_cpumask_ptr(int node)
116 * architecture is flat, so it is a pretty simple function! 96{
117 */ 97 return &node_to_cpumask_map[node];
118#define parent_node(node) (node) 98}
119 99
120/* Returns a bitmask of CPUs on Node 'node'. */ 100/* Returns a bitmask of CPUs on Node 'node'. */
121static inline cpumask_t node_to_cpumask(int node) 101static inline cpumask_t node_to_cpumask(int node)
@@ -123,14 +103,29 @@ static inline cpumask_t node_to_cpumask(int node)
123 return node_to_cpumask_map[node]; 103 return node_to_cpumask_map[node];
124} 104}
125 105
106#endif /* !CONFIG_DEBUG_PER_CPU_MAPS */
107#endif /* CONFIG_X86_64 */
108
109/* Replace default node_to_cpumask_ptr with optimized version */
110#define node_to_cpumask_ptr(v, node) \
111 cpumask_t *v = _node_to_cpumask_ptr(node)
112
113#define node_to_cpumask_ptr_next(v, node) \
114 v = _node_to_cpumask_ptr(node)
115
126/* Returns the number of the first CPU on Node 'node'. */ 116/* Returns the number of the first CPU on Node 'node'. */
127static inline int node_to_first_cpu(int node) 117static inline int node_to_first_cpu(int node)
128{ 118{
129 cpumask_t mask = node_to_cpumask(node); 119 node_to_cpumask_ptr(mask, node);
130 120 return first_cpu(*mask);
131 return first_cpu(mask);
132} 121}
133 122
123/*
124 * Returns the number of the node containing Node 'node'. This
125 * architecture is flat, so it is a pretty simple function!
126 */
127#define parent_node(node) (node)
128
134#define pcibus_to_node(bus) __pcibus_to_node(bus) 129#define pcibus_to_node(bus) __pcibus_to_node(bus)
135#define pcibus_to_cpumask(bus) __pcibus_to_cpumask(bus) 130#define pcibus_to_cpumask(bus) __pcibus_to_cpumask(bus)
136 131
@@ -180,8 +175,31 @@ extern int __node_distance(int, int);
180#define node_distance(a, b) __node_distance(a, b) 175#define node_distance(a, b) __node_distance(a, b)
181#endif 176#endif
182 177
183#else /* CONFIG_NUMA */ 178#else /* !CONFIG_NUMA */
179
180#define numa_node_id() 0
181#define cpu_to_node(cpu) 0
182#define early_cpu_to_node(cpu) 0
183
184static inline cpumask_t *_node_to_cpumask_ptr(int node)
185{
186 return &cpu_online_map;
187}
188static inline cpumask_t node_to_cpumask(int node)
189{
190 return cpu_online_map;
191}
192static inline int node_to_first_cpu(int node)
193{
194 return first_cpu(cpu_online_map);
195}
196
197/* Replace default node_to_cpumask_ptr with optimized version */
198#define node_to_cpumask_ptr(v, node) \
199 cpumask_t *v = _node_to_cpumask_ptr(node)
184 200
201#define node_to_cpumask_ptr_next(v, node) \
202 v = _node_to_cpumask_ptr(node)
185#endif 203#endif
186 204
187#include <asm-generic/topology.h> 205#include <asm-generic/topology.h>
@@ -193,6 +211,9 @@ extern cpumask_t cpu_coregroup_map(int cpu);
193#define topology_core_id(cpu) (cpu_data(cpu).cpu_core_id) 211#define topology_core_id(cpu) (cpu_data(cpu).cpu_core_id)
194#define topology_core_siblings(cpu) (per_cpu(cpu_core_map, cpu)) 212#define topology_core_siblings(cpu) (per_cpu(cpu_core_map, cpu))
195#define topology_thread_siblings(cpu) (per_cpu(cpu_sibling_map, cpu)) 213#define topology_thread_siblings(cpu) (per_cpu(cpu_sibling_map, cpu))
214
215/* indicates that pointers to the topology cpumask_t maps are valid */
216#define arch_provides_topology_pointers yes
196#endif 217#endif
197 218
198static inline void arch_fix_phys_package_id(int num, u32 slot) 219static inline void arch_fix_phys_package_id(int num, u32 slot)
@@ -220,4 +241,4 @@ static inline void set_mp_bus_to_node(int busnum, int node)
220} 241}
221#endif 242#endif
222 243
223#endif 244#endif /* _ASM_X86_TOPOLOGY_H */