aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorIngo Molnar <mingo@elte.hu>2009-03-10 05:16:17 -0400
committerIngo Molnar <mingo@elte.hu>2009-03-10 05:17:48 -0400
commit8293dd6f86e759068ce918aa10ca9c5d6d711cd0 (patch)
treede366d69078bf2b98c6765fa4ec1cc652f3d3173
parent631595fbf4aeac260e664a8a002897e4db6a50dd (diff)
parent467c88fee51e2ae862e9485245687da0730e29aa (diff)
Merge branch 'x86/core' into tracing/ftrace
Semantic merge: kernel/trace/trace_functions_graph.c Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat
-rw-r--r--arch/blackfin/include/asm/percpu.h10
-rw-r--r--arch/x86/include/asm/linkage.h16
-rw-r--r--arch/x86/kernel/cpu/amd.c52
-rw-r--r--arch/x86/kernel/cpu/intel.c25
-rw-r--r--arch/x86/kernel/setup_percpu.c73
-rw-r--r--arch/x86/kernel/smpboot.c78
-rw-r--r--arch/x86/kernel/tlb_uv.c2
-rw-r--r--arch/x86/mm/init.c2
-rw-r--r--arch/x86/mm/init_32.c23
-rw-r--r--arch/x86/mm/init_64.c8
-rw-r--r--arch/x86/mm/ioremap.c21
-rw-r--r--arch/x86/mm/kmmio.c15
-rw-r--r--arch/x86/mm/memtest.c3
-rw-r--r--include/linux/percpu.h60
-rw-r--r--kernel/module.c2
-rw-r--r--kernel/trace/trace_functions_graph.c2
-rw-r--r--mm/percpu.c593
17 files changed, 631 insertions, 354 deletions
diff --git a/arch/blackfin/include/asm/percpu.h b/arch/blackfin/include/asm/percpu.h
index 797c0c165069..c94c7bc88c71 100644
--- a/arch/blackfin/include/asm/percpu.h
+++ b/arch/blackfin/include/asm/percpu.h
@@ -3,14 +3,4 @@
3 3
4#include <asm-generic/percpu.h> 4#include <asm-generic/percpu.h>
5 5
6#ifdef CONFIG_MODULES
7#define PERCPU_MODULE_RESERVE 8192
8#else
9#define PERCPU_MODULE_RESERVE 0
10#endif
11
12#define PERCPU_ENOUGH_ROOM \
13 (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \
14 PERCPU_MODULE_RESERVE)
15
16#endif /* __ARCH_BLACKFIN_PERCPU__ */ 6#endif /* __ARCH_BLACKFIN_PERCPU__ */
diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h
index 9320e2a8a26a..a0d70b46c27c 100644
--- a/arch/x86/include/asm/linkage.h
+++ b/arch/x86/include/asm/linkage.h
@@ -4,11 +4,6 @@
4#undef notrace 4#undef notrace
5#define notrace __attribute__((no_instrument_function)) 5#define notrace __attribute__((no_instrument_function))
6 6
7#ifdef CONFIG_X86_64
8#define __ALIGN .p2align 4,,15
9#define __ALIGN_STR ".p2align 4,,15"
10#endif
11
12#ifdef CONFIG_X86_32 7#ifdef CONFIG_X86_32
13#define asmlinkage CPP_ASMLINKAGE __attribute__((regparm(0))) 8#define asmlinkage CPP_ASMLINKAGE __attribute__((regparm(0)))
14/* 9/*
@@ -50,16 +45,25 @@
50 __asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3), \ 45 __asmlinkage_protect_n(ret, "g" (arg1), "g" (arg2), "g" (arg3), \
51 "g" (arg4), "g" (arg5), "g" (arg6)) 46 "g" (arg4), "g" (arg5), "g" (arg6))
52 47
53#endif 48#endif /* CONFIG_X86_32 */
49
50#ifdef __ASSEMBLY__
54 51
55#define GLOBAL(name) \ 52#define GLOBAL(name) \
56 .globl name; \ 53 .globl name; \
57 name: 54 name:
58 55
56#ifdef CONFIG_X86_64
57#define __ALIGN .p2align 4,,15
58#define __ALIGN_STR ".p2align 4,,15"
59#endif
60
59#ifdef CONFIG_X86_ALIGNMENT_16 61#ifdef CONFIG_X86_ALIGNMENT_16
60#define __ALIGN .align 16,0x90 62#define __ALIGN .align 16,0x90
61#define __ALIGN_STR ".align 16,0x90" 63#define __ALIGN_STR ".align 16,0x90"
62#endif 64#endif
63 65
66#endif /* __ASSEMBLY__ */
67
64#endif /* _ASM_X86_LINKAGE_H */ 68#endif /* _ASM_X86_LINKAGE_H */
65 69
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 25423a5b80ed..f47df59016c5 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -5,6 +5,7 @@
5#include <asm/io.h> 5#include <asm/io.h>
6#include <asm/processor.h> 6#include <asm/processor.h>
7#include <asm/apic.h> 7#include <asm/apic.h>
8#include <asm/cpu.h>
8 9
9#ifdef CONFIG_X86_64 10#ifdef CONFIG_X86_64
10# include <asm/numa_64.h> 11# include <asm/numa_64.h>
@@ -141,6 +142,55 @@ static void __cpuinit init_amd_k6(struct cpuinfo_x86 *c)
141 } 142 }
142} 143}
143 144
145static void __cpuinit amd_k7_smp_check(struct cpuinfo_x86 *c)
146{
147#ifdef CONFIG_SMP
148 /* calling is from identify_secondary_cpu() ? */
149 if (c->cpu_index == boot_cpu_id)
150 return;
151
152 /*
153 * Certain Athlons might work (for various values of 'work') in SMP
154 * but they are not certified as MP capable.
155 */
156 /* Athlon 660/661 is valid. */
157 if ((c->x86_model == 6) && ((c->x86_mask == 0) ||
158 (c->x86_mask == 1)))
159 goto valid_k7;
160
161 /* Duron 670 is valid */
162 if ((c->x86_model == 7) && (c->x86_mask == 0))
163 goto valid_k7;
164
165 /*
166 * Athlon 662, Duron 671, and Athlon >model 7 have capability
167 * bit. It's worth noting that the A5 stepping (662) of some
168 * Athlon XP's have the MP bit set.
169 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for
170 * more.
171 */
172 if (((c->x86_model == 6) && (c->x86_mask >= 2)) ||
173 ((c->x86_model == 7) && (c->x86_mask >= 1)) ||
174 (c->x86_model > 7))
175 if (cpu_has_mp)
176 goto valid_k7;
177
178 /* If we get here, not a certified SMP capable AMD system. */
179
180 /*
181 * Don't taint if we are running SMP kernel on a single non-MP
182 * approved Athlon
183 */
184 WARN_ONCE(1, "WARNING: This combination of AMD"
185 "processors is not suitable for SMP.\n");
186 if (!test_taint(TAINT_UNSAFE_SMP))
187 add_taint(TAINT_UNSAFE_SMP);
188
189valid_k7:
190 ;
191#endif
192}
193
144static void __cpuinit init_amd_k7(struct cpuinfo_x86 *c) 194static void __cpuinit init_amd_k7(struct cpuinfo_x86 *c)
145{ 195{
146 u32 l, h; 196 u32 l, h;
@@ -175,6 +225,8 @@ static void __cpuinit init_amd_k7(struct cpuinfo_x86 *c)
175 } 225 }
176 226
177 set_cpu_cap(c, X86_FEATURE_K7); 227 set_cpu_cap(c, X86_FEATURE_K7);
228
229 amd_k7_smp_check(c);
178} 230}
179#endif 231#endif
180 232
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 1a89a2b68d15..c1c04bf0df77 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -14,6 +14,7 @@
14#include <asm/uaccess.h> 14#include <asm/uaccess.h>
15#include <asm/ds.h> 15#include <asm/ds.h>
16#include <asm/bugs.h> 16#include <asm/bugs.h>
17#include <asm/cpu.h>
17 18
18#ifdef CONFIG_X86_64 19#ifdef CONFIG_X86_64
19#include <asm/topology.h> 20#include <asm/topology.h>
@@ -116,6 +117,28 @@ static void __cpuinit trap_init_f00f_bug(void)
116} 117}
117#endif 118#endif
118 119
120static void __cpuinit intel_smp_check(struct cpuinfo_x86 *c)
121{
122#ifdef CONFIG_SMP
123 /* calling is from identify_secondary_cpu() ? */
124 if (c->cpu_index == boot_cpu_id)
125 return;
126
127 /*
128 * Mask B, Pentium, but not Pentium MMX
129 */
130 if (c->x86 == 5 &&
131 c->x86_mask >= 1 && c->x86_mask <= 4 &&
132 c->x86_model <= 3) {
133 /*
134 * Remember we have B step Pentia with bugs
135 */
136 WARN_ONCE(1, "WARNING: SMP operation may be unreliable"
137 "with B stepping processors.\n");
138 }
139#endif
140}
141
119static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c) 142static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c)
120{ 143{
121 unsigned long lo, hi; 144 unsigned long lo, hi;
@@ -192,6 +215,8 @@ static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c)
192#ifdef CONFIG_X86_NUMAQ 215#ifdef CONFIG_X86_NUMAQ
193 numaq_tsc_disable(); 216 numaq_tsc_disable();
194#endif 217#endif
218
219 intel_smp_check(c);
195} 220}
196#else 221#else
197static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c) 222static void __cpuinit intel_workarounds(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index c29f301d3885..efa615f2bf43 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -42,6 +42,19 @@ unsigned long __per_cpu_offset[NR_CPUS] __read_mostly = {
42}; 42};
43EXPORT_SYMBOL(__per_cpu_offset); 43EXPORT_SYMBOL(__per_cpu_offset);
44 44
45/*
46 * On x86_64 symbols referenced from code should be reachable using
47 * 32bit relocations. Reserve space for static percpu variables in
48 * modules so that they are always served from the first chunk which
49 * is located at the percpu segment base. On x86_32, anything can
50 * address anywhere. No need to reserve space in the first chunk.
51 */
52#ifdef CONFIG_X86_64
53#define PERCPU_FIRST_CHUNK_RESERVE PERCPU_MODULE_RESERVE
54#else
55#define PERCPU_FIRST_CHUNK_RESERVE 0
56#endif
57
45/** 58/**
46 * pcpu_need_numa - determine percpu allocation needs to consider NUMA 59 * pcpu_need_numa - determine percpu allocation needs to consider NUMA
47 * 60 *
@@ -141,7 +154,7 @@ static ssize_t __init setup_pcpu_remap(size_t static_size)
141{ 154{
142 static struct vm_struct vm; 155 static struct vm_struct vm;
143 pg_data_t *last; 156 pg_data_t *last;
144 size_t ptrs_size; 157 size_t ptrs_size, dyn_size;
145 unsigned int cpu; 158 unsigned int cpu;
146 ssize_t ret; 159 ssize_t ret;
147 160
@@ -169,12 +182,14 @@ proceed:
169 * Currently supports only single page. Supporting multiple 182 * Currently supports only single page. Supporting multiple
170 * pages won't be too difficult if it ever becomes necessary. 183 * pages won't be too difficult if it ever becomes necessary.
171 */ 184 */
172 pcpur_size = PFN_ALIGN(static_size + PERCPU_DYNAMIC_RESERVE); 185 pcpur_size = PFN_ALIGN(static_size + PERCPU_MODULE_RESERVE +
186 PERCPU_DYNAMIC_RESERVE);
173 if (pcpur_size > PMD_SIZE) { 187 if (pcpur_size > PMD_SIZE) {
174 pr_warning("PERCPU: static data is larger than large page, " 188 pr_warning("PERCPU: static data is larger than large page, "
175 "can't use large page\n"); 189 "can't use large page\n");
176 return -EINVAL; 190 return -EINVAL;
177 } 191 }
192 dyn_size = pcpur_size - static_size - PERCPU_FIRST_CHUNK_RESERVE;
178 193
179 /* allocate pointer array and alloc large pages */ 194 /* allocate pointer array and alloc large pages */
180 ptrs_size = PFN_ALIGN(num_possible_cpus() * sizeof(pcpur_ptrs[0])); 195 ptrs_size = PFN_ALIGN(num_possible_cpus() * sizeof(pcpur_ptrs[0]));
@@ -217,8 +232,9 @@ proceed:
217 pr_info("PERCPU: Remapped at %p with large pages, static data " 232 pr_info("PERCPU: Remapped at %p with large pages, static data "
218 "%zu bytes\n", vm.addr, static_size); 233 "%zu bytes\n", vm.addr, static_size);
219 234
220 ret = pcpu_setup_first_chunk(pcpur_get_page, static_size, PMD_SIZE, 235 ret = pcpu_setup_first_chunk(pcpur_get_page, static_size,
221 pcpur_size - static_size, vm.addr, NULL); 236 PERCPU_FIRST_CHUNK_RESERVE,
237 PMD_SIZE, dyn_size, vm.addr, NULL);
222 goto out_free_ar; 238 goto out_free_ar;
223 239
224enomem: 240enomem:
@@ -241,24 +257,31 @@ static ssize_t __init setup_pcpu_remap(size_t static_size)
241 * Embedding allocator 257 * Embedding allocator
242 * 258 *
243 * The first chunk is sized to just contain the static area plus 259 * The first chunk is sized to just contain the static area plus
244 * PERCPU_DYNAMIC_RESERVE and allocated as a contiguous area using 260 * module and dynamic reserves, and allocated as a contiguous area
245 * bootmem allocator and used as-is without being mapped into vmalloc 261 * using bootmem allocator and used as-is without being mapped into
246 * area. This enables the first chunk to piggy back on the linear 262 * vmalloc area. This enables the first chunk to piggy back on the
247 * physical PMD mapping and doesn't add any additional pressure to 263 * linear physical PMD mapping and doesn't add any additional pressure
248 * TLB. 264 * to TLB. Note that if the needed size is smaller than the minimum
265 * unit size, the leftover is returned to the bootmem allocator.
249 */ 266 */
250static void *pcpue_ptr __initdata; 267static void *pcpue_ptr __initdata;
268static size_t pcpue_size __initdata;
251static size_t pcpue_unit_size __initdata; 269static size_t pcpue_unit_size __initdata;
252 270
253static struct page * __init pcpue_get_page(unsigned int cpu, int pageno) 271static struct page * __init pcpue_get_page(unsigned int cpu, int pageno)
254{ 272{
255 return virt_to_page(pcpue_ptr + cpu * pcpue_unit_size 273 size_t off = (size_t)pageno << PAGE_SHIFT;
256 + ((size_t)pageno << PAGE_SHIFT)); 274
275 if (off >= pcpue_size)
276 return NULL;
277
278 return virt_to_page(pcpue_ptr + cpu * pcpue_unit_size + off);
257} 279}
258 280
259static ssize_t __init setup_pcpu_embed(size_t static_size) 281static ssize_t __init setup_pcpu_embed(size_t static_size)
260{ 282{
261 unsigned int cpu; 283 unsigned int cpu;
284 size_t dyn_size;
262 285
263 /* 286 /*
264 * If large page isn't supported, there's no benefit in doing 287 * If large page isn't supported, there's no benefit in doing
@@ -269,25 +292,32 @@ static ssize_t __init setup_pcpu_embed(size_t static_size)
269 return -EINVAL; 292 return -EINVAL;
270 293
271 /* allocate and copy */ 294 /* allocate and copy */
272 pcpue_unit_size = PFN_ALIGN(static_size + PERCPU_DYNAMIC_RESERVE); 295 pcpue_size = PFN_ALIGN(static_size + PERCPU_MODULE_RESERVE +
273 pcpue_unit_size = max_t(size_t, pcpue_unit_size, PCPU_MIN_UNIT_SIZE); 296 PERCPU_DYNAMIC_RESERVE);
297 pcpue_unit_size = max_t(size_t, pcpue_size, PCPU_MIN_UNIT_SIZE);
298 dyn_size = pcpue_size - static_size - PERCPU_FIRST_CHUNK_RESERVE;
299
274 pcpue_ptr = pcpu_alloc_bootmem(0, num_possible_cpus() * pcpue_unit_size, 300 pcpue_ptr = pcpu_alloc_bootmem(0, num_possible_cpus() * pcpue_unit_size,
275 PAGE_SIZE); 301 PAGE_SIZE);
276 if (!pcpue_ptr) 302 if (!pcpue_ptr)
277 return -ENOMEM; 303 return -ENOMEM;
278 304
279 for_each_possible_cpu(cpu) 305 for_each_possible_cpu(cpu) {
280 memcpy(pcpue_ptr + cpu * pcpue_unit_size, __per_cpu_load, 306 void *ptr = pcpue_ptr + cpu * pcpue_unit_size;
281 static_size); 307
308 free_bootmem(__pa(ptr + pcpue_size),
309 pcpue_unit_size - pcpue_size);
310 memcpy(ptr, __per_cpu_load, static_size);
311 }
282 312
283 /* we're ready, commit */ 313 /* we're ready, commit */
284 pr_info("PERCPU: Embedded %zu pages at %p, static data %zu bytes\n", 314 pr_info("PERCPU: Embedded %zu pages at %p, static data %zu bytes\n",
285 pcpue_unit_size >> PAGE_SHIFT, pcpue_ptr, static_size); 315 pcpue_size >> PAGE_SHIFT, pcpue_ptr, static_size);
286 316
287 return pcpu_setup_first_chunk(pcpue_get_page, static_size, 317 return pcpu_setup_first_chunk(pcpue_get_page, static_size,
288 pcpue_unit_size, 318 PERCPU_FIRST_CHUNK_RESERVE,
289 pcpue_unit_size - static_size, pcpue_ptr, 319 pcpue_unit_size, dyn_size,
290 NULL); 320 pcpue_ptr, NULL);
291} 321}
292 322
293/* 323/*
@@ -344,7 +374,8 @@ static ssize_t __init setup_pcpu_4k(size_t static_size)
344 pr_info("PERCPU: Allocated %d 4k pages, static data %zu bytes\n", 374 pr_info("PERCPU: Allocated %d 4k pages, static data %zu bytes\n",
345 pcpu4k_nr_static_pages, static_size); 375 pcpu4k_nr_static_pages, static_size);
346 376
347 ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, 0, 0, NULL, 377 ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size,
378 PERCPU_FIRST_CHUNK_RESERVE, -1, -1, NULL,
348 pcpu4k_populate_pte); 379 pcpu4k_populate_pte);
349 goto out_free_ar; 380 goto out_free_ar;
350 381
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 249334f5080a..ef7d10170c30 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -114,10 +114,6 @@ EXPORT_PER_CPU_SYMBOL(cpu_info);
114 114
115atomic_t init_deasserted; 115atomic_t init_deasserted;
116 116
117
118/* Set if we find a B stepping CPU */
119static int __cpuinitdata smp_b_stepping;
120
121#if defined(CONFIG_NUMA) && defined(CONFIG_X86_32) 117#if defined(CONFIG_NUMA) && defined(CONFIG_X86_32)
122 118
123/* which logical CPUs are on which nodes */ 119/* which logical CPUs are on which nodes */
@@ -271,8 +267,6 @@ static void __cpuinit smp_callin(void)
271 cpumask_set_cpu(cpuid, cpu_callin_mask); 267 cpumask_set_cpu(cpuid, cpu_callin_mask);
272} 268}
273 269
274static int __cpuinitdata unsafe_smp;
275
276/* 270/*
277 * Activate a secondary processor. 271 * Activate a secondary processor.
278 */ 272 */
@@ -340,76 +334,6 @@ notrace static void __cpuinit start_secondary(void *unused)
340 cpu_idle(); 334 cpu_idle();
341} 335}
342 336
343static void __cpuinit smp_apply_quirks(struct cpuinfo_x86 *c)
344{
345 /*
346 * Mask B, Pentium, but not Pentium MMX
347 */
348 if (c->x86_vendor == X86_VENDOR_INTEL &&
349 c->x86 == 5 &&
350 c->x86_mask >= 1 && c->x86_mask <= 4 &&
351 c->x86_model <= 3)
352 /*
353 * Remember we have B step Pentia with bugs
354 */
355 smp_b_stepping = 1;
356
357 /*
358 * Certain Athlons might work (for various values of 'work') in SMP
359 * but they are not certified as MP capable.
360 */
361 if ((c->x86_vendor == X86_VENDOR_AMD) && (c->x86 == 6)) {
362
363 if (num_possible_cpus() == 1)
364 goto valid_k7;
365
366 /* Athlon 660/661 is valid. */
367 if ((c->x86_model == 6) && ((c->x86_mask == 0) ||
368 (c->x86_mask == 1)))
369 goto valid_k7;
370
371 /* Duron 670 is valid */
372 if ((c->x86_model == 7) && (c->x86_mask == 0))
373 goto valid_k7;
374
375 /*
376 * Athlon 662, Duron 671, and Athlon >model 7 have capability
377 * bit. It's worth noting that the A5 stepping (662) of some
378 * Athlon XP's have the MP bit set.
379 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for
380 * more.
381 */
382 if (((c->x86_model == 6) && (c->x86_mask >= 2)) ||
383 ((c->x86_model == 7) && (c->x86_mask >= 1)) ||
384 (c->x86_model > 7))
385 if (cpu_has_mp)
386 goto valid_k7;
387
388 /* If we get here, not a certified SMP capable AMD system. */
389 unsafe_smp = 1;
390 }
391
392valid_k7:
393 ;
394}
395
396static void __cpuinit smp_checks(void)
397{
398 if (smp_b_stepping)
399 printk(KERN_WARNING "WARNING: SMP operation may be unreliable"
400 "with B stepping processors.\n");
401
402 /*
403 * Don't taint if we are running SMP kernel on a single non-MP
404 * approved Athlon
405 */
406 if (unsafe_smp && num_online_cpus() > 1) {
407 printk(KERN_INFO "WARNING: This combination of AMD"
408 "processors is not suitable for SMP.\n");
409 add_taint(TAINT_UNSAFE_SMP);
410 }
411}
412
413/* 337/*
414 * The bootstrap kernel entry code has set these up. Save them for 338 * The bootstrap kernel entry code has set these up. Save them for
415 * a given CPU 339 * a given CPU
@@ -423,7 +347,6 @@ void __cpuinit smp_store_cpu_info(int id)
423 c->cpu_index = id; 347 c->cpu_index = id;
424 if (id != 0) 348 if (id != 0)
425 identify_secondary_cpu(c); 349 identify_secondary_cpu(c);
426 smp_apply_quirks(c);
427} 350}
428 351
429 352
@@ -1193,7 +1116,6 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
1193 pr_debug("Boot done.\n"); 1116 pr_debug("Boot done.\n");
1194 1117
1195 impress_friends(); 1118 impress_friends();
1196 smp_checks();
1197#ifdef CONFIG_X86_IO_APIC 1119#ifdef CONFIG_X86_IO_APIC
1198 setup_ioapic_dest(); 1120 setup_ioapic_dest();
1199#endif 1121#endif
diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c
index f04549afcfe9..d038b9c45cf8 100644
--- a/arch/x86/kernel/tlb_uv.c
+++ b/arch/x86/kernel/tlb_uv.c
@@ -314,8 +314,6 @@ const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
314 int locals = 0; 314 int locals = 0;
315 struct bau_desc *bau_desc; 315 struct bau_desc *bau_desc;
316 316
317 WARN_ON(!in_atomic());
318
319 cpumask_andnot(flush_mask, cpumask, cpumask_of(cpu)); 317 cpumask_andnot(flush_mask, cpumask, cpumask_of(cpu));
320 318
321 uv_cpu = uv_blade_processor_id(); 319 uv_cpu = uv_blade_processor_id();
diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index 6d63e3d1253d..15219e0d1243 100644
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -134,8 +134,8 @@ unsigned long __init_refok init_memory_mapping(unsigned long start,
134{ 134{
135 unsigned long page_size_mask = 0; 135 unsigned long page_size_mask = 0;
136 unsigned long start_pfn, end_pfn; 136 unsigned long start_pfn, end_pfn;
137 unsigned long ret = 0;
137 unsigned long pos; 138 unsigned long pos;
138 unsigned long ret;
139 139
140 struct map_range mr[NR_RANGE_MR]; 140 struct map_range mr[NR_RANGE_MR];
141 int nr_range, i; 141 int nr_range, i;
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index d7f5060ab21c..749559ed80f5 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -806,11 +806,6 @@ static unsigned long __init setup_node_bootmem(int nodeid,
806{ 806{
807 unsigned long bootmap_size; 807 unsigned long bootmap_size;
808 808
809 if (start_pfn > max_low_pfn)
810 return bootmap;
811 if (end_pfn > max_low_pfn)
812 end_pfn = max_low_pfn;
813
814 /* don't touch min_low_pfn */ 809 /* don't touch min_low_pfn */
815 bootmap_size = init_bootmem_node(NODE_DATA(nodeid), 810 bootmap_size = init_bootmem_node(NODE_DATA(nodeid),
816 bootmap >> PAGE_SHIFT, 811 bootmap >> PAGE_SHIFT,
@@ -843,13 +838,23 @@ void __init setup_bootmem_allocator(void)
843 max_pfn_mapped<<PAGE_SHIFT); 838 max_pfn_mapped<<PAGE_SHIFT);
844 printk(KERN_INFO " low ram: 0 - %08lx\n", max_low_pfn<<PAGE_SHIFT); 839 printk(KERN_INFO " low ram: 0 - %08lx\n", max_low_pfn<<PAGE_SHIFT);
845 840
841 for_each_online_node(nodeid) {
842 unsigned long start_pfn, end_pfn;
843
846#ifdef CONFIG_NEED_MULTIPLE_NODES 844#ifdef CONFIG_NEED_MULTIPLE_NODES
847 for_each_online_node(nodeid) 845 start_pfn = node_start_pfn[nodeid];
848 bootmap = setup_node_bootmem(nodeid, node_start_pfn[nodeid], 846 end_pfn = node_end_pfn[nodeid];
849 node_end_pfn[nodeid], bootmap); 847 if (start_pfn > max_low_pfn)
848 continue;
849 if (end_pfn > max_low_pfn)
850 end_pfn = max_low_pfn;
850#else 851#else
851 bootmap = setup_node_bootmem(0, 0, max_low_pfn, bootmap); 852 start_pfn = 0;
853 end_pfn = max_low_pfn;
852#endif 854#endif
855 bootmap = setup_node_bootmem(nodeid, start_pfn, end_pfn,
856 bootmap);
857 }
853 858
854 after_bootmem = 1; 859 after_bootmem = 1;
855} 860}
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 66d6be85df82..1753e8020df6 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -85,7 +85,7 @@ early_param("gbpages", parse_direct_gbpages_on);
85pteval_t __supported_pte_mask __read_mostly = ~_PAGE_IOMAP; 85pteval_t __supported_pte_mask __read_mostly = ~_PAGE_IOMAP;
86EXPORT_SYMBOL_GPL(__supported_pte_mask); 86EXPORT_SYMBOL_GPL(__supported_pte_mask);
87 87
88static int do_not_nx __cpuinitdata; 88static int disable_nx __cpuinitdata;
89 89
90/* 90/*
91 * noexec=on|off 91 * noexec=on|off
@@ -100,9 +100,9 @@ static int __init nonx_setup(char *str)
100 return -EINVAL; 100 return -EINVAL;
101 if (!strncmp(str, "on", 2)) { 101 if (!strncmp(str, "on", 2)) {
102 __supported_pte_mask |= _PAGE_NX; 102 __supported_pte_mask |= _PAGE_NX;
103 do_not_nx = 0; 103 disable_nx = 0;
104 } else if (!strncmp(str, "off", 3)) { 104 } else if (!strncmp(str, "off", 3)) {
105 do_not_nx = 1; 105 disable_nx = 1;
106 __supported_pte_mask &= ~_PAGE_NX; 106 __supported_pte_mask &= ~_PAGE_NX;
107 } 107 }
108 return 0; 108 return 0;
@@ -114,7 +114,7 @@ void __cpuinit check_efer(void)
114 unsigned long efer; 114 unsigned long efer;
115 115
116 rdmsrl(MSR_EFER, efer); 116 rdmsrl(MSR_EFER, efer);
117 if (!(efer & EFER_NX) || do_not_nx) 117 if (!(efer & EFER_NX) || disable_nx)
118 __supported_pte_mask &= ~_PAGE_NX; 118 __supported_pte_mask &= ~_PAGE_NX;
119} 119}
120 120
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index 62773abdf088..aca924a30ee6 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -87,6 +87,8 @@ bool __virt_addr_valid(unsigned long x)
87 return false; 87 return false;
88 if (__vmalloc_start_set && is_vmalloc_addr((void *) x)) 88 if (__vmalloc_start_set && is_vmalloc_addr((void *) x))
89 return false; 89 return false;
90 if (x >= FIXADDR_START)
91 return false;
90 return pfn_valid((x - PAGE_OFFSET) >> PAGE_SHIFT); 92 return pfn_valid((x - PAGE_OFFSET) >> PAGE_SHIFT);
91} 93}
92EXPORT_SYMBOL(__virt_addr_valid); 94EXPORT_SYMBOL(__virt_addr_valid);
@@ -504,13 +506,19 @@ static inline pte_t * __init early_ioremap_pte(unsigned long addr)
504 return &bm_pte[pte_index(addr)]; 506 return &bm_pte[pte_index(addr)];
505} 507}
506 508
509static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata;
510
507void __init early_ioremap_init(void) 511void __init early_ioremap_init(void)
508{ 512{
509 pmd_t *pmd; 513 pmd_t *pmd;
514 int i;
510 515
511 if (early_ioremap_debug) 516 if (early_ioremap_debug)
512 printk(KERN_INFO "early_ioremap_init()\n"); 517 printk(KERN_INFO "early_ioremap_init()\n");
513 518
519 for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
520 slot_virt[i] = fix_to_virt(FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*i);
521
514 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)); 522 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
515 memset(bm_pte, 0, sizeof(bm_pte)); 523 memset(bm_pte, 0, sizeof(bm_pte));
516 pmd_populate_kernel(&init_mm, pmd, bm_pte); 524 pmd_populate_kernel(&init_mm, pmd, bm_pte);
@@ -577,6 +585,7 @@ static inline void __init early_clear_fixmap(enum fixed_addresses idx)
577 585
578static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata; 586static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata;
579static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata; 587static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata;
588
580static int __init check_early_ioremap_leak(void) 589static int __init check_early_ioremap_leak(void)
581{ 590{
582 int count = 0; 591 int count = 0;
@@ -598,7 +607,8 @@ static int __init check_early_ioremap_leak(void)
598} 607}
599late_initcall(check_early_ioremap_leak); 608late_initcall(check_early_ioremap_leak);
600 609
601static void __init __iomem *__early_ioremap(unsigned long phys_addr, unsigned long size, pgprot_t prot) 610static void __init __iomem *
611__early_ioremap(unsigned long phys_addr, unsigned long size, pgprot_t prot)
602{ 612{
603 unsigned long offset, last_addr; 613 unsigned long offset, last_addr;
604 unsigned int nrpages; 614 unsigned int nrpages;
@@ -664,9 +674,9 @@ static void __init __iomem *__early_ioremap(unsigned long phys_addr, unsigned lo
664 --nrpages; 674 --nrpages;
665 } 675 }
666 if (early_ioremap_debug) 676 if (early_ioremap_debug)
667 printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0)); 677 printk(KERN_CONT "%08lx + %08lx\n", offset, slot_virt[slot]);
668 678
669 prev_map[slot] = (void __iomem *)(offset + fix_to_virt(idx0)); 679 prev_map[slot] = (void __iomem *)(offset + slot_virt[slot]);
670 return prev_map[slot]; 680 return prev_map[slot];
671} 681}
672 682
@@ -734,8 +744,3 @@ void __init early_iounmap(void __iomem *addr, unsigned long size)
734 } 744 }
735 prev_map[slot] = NULL; 745 prev_map[slot] = NULL;
736} 746}
737
738void __this_fixmap_does_not_exist(void)
739{
740 WARN_ON(1);
741}
diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c
index 9f205030d9aa..6a518dd08a36 100644
--- a/arch/x86/mm/kmmio.c
+++ b/arch/x86/mm/kmmio.c
@@ -451,23 +451,24 @@ static void rcu_free_kmmio_fault_pages(struct rcu_head *head)
451 451
452static void remove_kmmio_fault_pages(struct rcu_head *head) 452static void remove_kmmio_fault_pages(struct rcu_head *head)
453{ 453{
454 struct kmmio_delayed_release *dr = container_of( 454 struct kmmio_delayed_release *dr =
455 head, 455 container_of(head, struct kmmio_delayed_release, rcu);
456 struct kmmio_delayed_release,
457 rcu);
458 struct kmmio_fault_page *p = dr->release_list; 456 struct kmmio_fault_page *p = dr->release_list;
459 struct kmmio_fault_page **prevp = &dr->release_list; 457 struct kmmio_fault_page **prevp = &dr->release_list;
460 unsigned long flags; 458 unsigned long flags;
459
461 spin_lock_irqsave(&kmmio_lock, flags); 460 spin_lock_irqsave(&kmmio_lock, flags);
462 while (p) { 461 while (p) {
463 if (!p->count) 462 if (!p->count) {
464 list_del_rcu(&p->list); 463 list_del_rcu(&p->list);
465 else 464 prevp = &p->release_next;
465 } else {
466 *prevp = p->release_next; 466 *prevp = p->release_next;
467 prevp = &p->release_next; 467 }
468 p = p->release_next; 468 p = p->release_next;
469 } 469 }
470 spin_unlock_irqrestore(&kmmio_lock, flags); 470 spin_unlock_irqrestore(&kmmio_lock, flags);
471
471 /* This is the real RCU destroy call. */ 472 /* This is the real RCU destroy call. */
472 call_rcu(&dr->rcu, rcu_free_kmmio_fault_pages); 473 call_rcu(&dr->rcu, rcu_free_kmmio_fault_pages);
473} 474}
diff --git a/arch/x86/mm/memtest.c b/arch/x86/mm/memtest.c
index 0bcd7883d036..605c8be06217 100644
--- a/arch/x86/mm/memtest.c
+++ b/arch/x86/mm/memtest.c
@@ -100,6 +100,9 @@ static int __init parse_memtest(char *arg)
100{ 100{
101 if (arg) 101 if (arg)
102 memtest_pattern = simple_strtoul(arg, NULL, 0); 102 memtest_pattern = simple_strtoul(arg, NULL, 0);
103 else
104 memtest_pattern = ARRAY_SIZE(patterns);
105
103 return 0; 106 return 0;
104} 107}
105 108
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 545b068bcb70..54a968b4b924 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -5,6 +5,7 @@
5#include <linux/slab.h> /* For kmalloc() */ 5#include <linux/slab.h> /* For kmalloc() */
6#include <linux/smp.h> 6#include <linux/smp.h>
7#include <linux/cpumask.h> 7#include <linux/cpumask.h>
8#include <linux/pfn.h>
8 9
9#include <asm/percpu.h> 10#include <asm/percpu.h>
10 11
@@ -52,17 +53,18 @@
52#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) 53#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
53#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) 54#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
54 55
55/* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */ 56/* enough to cover all DEFINE_PER_CPUs in modules */
56#ifndef PERCPU_ENOUGH_ROOM
57#ifdef CONFIG_MODULES 57#ifdef CONFIG_MODULES
58#define PERCPU_MODULE_RESERVE 8192 58#define PERCPU_MODULE_RESERVE (8 << 10)
59#else 59#else
60#define PERCPU_MODULE_RESERVE 0 60#define PERCPU_MODULE_RESERVE 0
61#endif 61#endif
62 62
63#ifndef PERCPU_ENOUGH_ROOM
63#define PERCPU_ENOUGH_ROOM \ 64#define PERCPU_ENOUGH_ROOM \
64 (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE) 65 (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \
65#endif /* PERCPU_ENOUGH_ROOM */ 66 PERCPU_MODULE_RESERVE)
67#endif
66 68
67/* 69/*
68 * Must be an lvalue. Since @var must be a simple identifier, 70 * Must be an lvalue. Since @var must be a simple identifier,
@@ -79,35 +81,24 @@
79#ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA 81#ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
80 82
81/* minimum unit size, also is the maximum supported allocation size */ 83/* minimum unit size, also is the maximum supported allocation size */
82#define PCPU_MIN_UNIT_SIZE (16UL << PAGE_SHIFT) 84#define PCPU_MIN_UNIT_SIZE PFN_ALIGN(64 << 10)
83 85
84/* 86/*
85 * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy 87 * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy
86 * back on the first chunk if arch is manually allocating and mapping 88 * back on the first chunk for dynamic percpu allocation if arch is
87 * it for faster access (as a part of large page mapping for example). 89 * manually allocating and mapping it for faster access (as a part of
88 * Note that dynamic percpu allocator covers both static and dynamic 90 * large page mapping for example).
89 * areas, so these values are bigger than PERCPU_MODULE_RESERVE.
90 * 91 *
91 * On typical configuration with modules, the following values leave 92 * The following values give between one and two pages of free space
92 * about 8k of free space on the first chunk after boot on both x86_32 93 * after typical minimal boot (2-way SMP, single disk and NIC) with
93 * and 64 when module support is enabled. When module support is 94 * both defconfig and a distro config on x86_64 and 32. More
94 * disabled, it's much tighter. 95 * intelligent way to determine this would be nice.
95 */ 96 */
96#ifndef PERCPU_DYNAMIC_RESERVE 97#if BITS_PER_LONG > 32
97# if BITS_PER_LONG > 32 98#define PERCPU_DYNAMIC_RESERVE (20 << 10)
98# ifdef CONFIG_MODULES 99#else
99# define PERCPU_DYNAMIC_RESERVE (6 << PAGE_SHIFT) 100#define PERCPU_DYNAMIC_RESERVE (12 << 10)
100# else 101#endif
101# define PERCPU_DYNAMIC_RESERVE (4 << PAGE_SHIFT)
102# endif
103# else
104# ifdef CONFIG_MODULES
105# define PERCPU_DYNAMIC_RESERVE (4 << PAGE_SHIFT)
106# else
107# define PERCPU_DYNAMIC_RESERVE (2 << PAGE_SHIFT)
108# endif
109# endif
110#endif /* PERCPU_DYNAMIC_RESERVE */
111 102
112extern void *pcpu_base_addr; 103extern void *pcpu_base_addr;
113 104
@@ -115,9 +106,10 @@ typedef struct page * (*pcpu_get_page_fn_t)(unsigned int cpu, int pageno);
115typedef void (*pcpu_populate_pte_fn_t)(unsigned long addr); 106typedef void (*pcpu_populate_pte_fn_t)(unsigned long addr);
116 107
117extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, 108extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
118 size_t static_size, size_t unit_size, 109 size_t static_size, size_t reserved_size,
119 size_t free_size, void *base_addr, 110 ssize_t unit_size, ssize_t dyn_size,
120 pcpu_populate_pte_fn_t populate_pte_fn); 111 void *base_addr,
112 pcpu_populate_pte_fn_t populate_pte_fn);
121 113
122/* 114/*
123 * Use this to get to a cpu's version of the per-cpu object 115 * Use this to get to a cpu's version of the per-cpu object
@@ -126,6 +118,8 @@ extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
126 */ 118 */
127#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) 119#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
128 120
121extern void *__alloc_reserved_percpu(size_t size, size_t align);
122
129#else /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */ 123#else /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
130 124
131struct percpu_data { 125struct percpu_data {
diff --git a/kernel/module.c b/kernel/module.c
index 90a6d63d9211..8b742f2b3845 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -381,7 +381,7 @@ static void *percpu_modalloc(unsigned long size, unsigned long align,
381 align = PAGE_SIZE; 381 align = PAGE_SIZE;
382 } 382 }
383 383
384 ptr = __alloc_percpu(size, align); 384 ptr = __alloc_reserved_percpu(size, align);
385 if (!ptr) 385 if (!ptr)
386 printk(KERN_WARNING 386 printk(KERN_WARNING
387 "Could not allocate %lu bytes percpu data\n", size); 387 "Could not allocate %lu bytes percpu data\n", size);
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 453ebd3b636e..35257be6a9d6 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -837,7 +837,7 @@ static void graph_trace_open(struct trace_iterator *iter)
837 837
838static void graph_trace_close(struct trace_iterator *iter) 838static void graph_trace_close(struct trace_iterator *iter)
839{ 839{
840 percpu_free(iter->private); 840 free_percpu(iter->private);
841} 841}
842 842
843static struct tracer graph_trace __read_mostly = { 843static struct tracer graph_trace __read_mostly = {
diff --git a/mm/percpu.c b/mm/percpu.c
index 3d0f5456827c..bfe6a3afaf45 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -62,7 +62,9 @@
62#include <linux/pfn.h> 62#include <linux/pfn.h>
63#include <linux/rbtree.h> 63#include <linux/rbtree.h>
64#include <linux/slab.h> 64#include <linux/slab.h>
65#include <linux/spinlock.h>
65#include <linux/vmalloc.h> 66#include <linux/vmalloc.h>
67#include <linux/workqueue.h>
66 68
67#include <asm/cacheflush.h> 69#include <asm/cacheflush.h>
68#include <asm/tlbflush.h> 70#include <asm/tlbflush.h>
@@ -80,7 +82,8 @@ struct pcpu_chunk {
80 int map_alloc; /* # of map entries allocated */ 82 int map_alloc; /* # of map entries allocated */
81 int *map; /* allocation map */ 83 int *map; /* allocation map */
82 bool immutable; /* no [de]population allowed */ 84 bool immutable; /* no [de]population allowed */
83 struct page *page[]; /* #cpus * UNIT_PAGES */ 85 struct page **page; /* points to page array */
86 struct page *page_ar[]; /* #cpus * UNIT_PAGES */
84}; 87};
85 88
86static int pcpu_unit_pages __read_mostly; 89static int pcpu_unit_pages __read_mostly;
@@ -93,28 +96,42 @@ static size_t pcpu_chunk_struct_size __read_mostly;
93void *pcpu_base_addr __read_mostly; 96void *pcpu_base_addr __read_mostly;
94EXPORT_SYMBOL_GPL(pcpu_base_addr); 97EXPORT_SYMBOL_GPL(pcpu_base_addr);
95 98
96/* the size of kernel static area */ 99/* optional reserved chunk, only accessible for reserved allocations */
97static int pcpu_static_size __read_mostly; 100static struct pcpu_chunk *pcpu_reserved_chunk;
101/* offset limit of the reserved chunk */
102static int pcpu_reserved_chunk_limit;
98 103
99/* 104/*
100 * One mutex to rule them all. 105 * Synchronization rules.
101 * 106 *
102 * The following mutex is grabbed in the outermost public alloc/free 107 * There are two locks - pcpu_alloc_mutex and pcpu_lock. The former
103 * interface functions and released only when the operation is 108 * protects allocation/reclaim paths, chunks and chunk->page arrays.
104 * complete. As such, every function in this file other than the 109 * The latter is a spinlock and protects the index data structures -
105 * outermost functions are called under pcpu_mutex. 110 * chunk slots, rbtree, chunks and area maps in chunks.
106 * 111 *
107 * It can easily be switched to use spinlock such that only the area 112 * During allocation, pcpu_alloc_mutex is kept locked all the time and
108 * allocation and page population commit are protected with it doing 113 * pcpu_lock is grabbed and released as necessary. All actual memory
109 * actual [de]allocation without holding any lock. However, given 114 * allocations are done using GFP_KERNEL with pcpu_lock released.
110 * what this allocator does, I think it's better to let them run 115 *
111 * sequentially. 116 * Free path accesses and alters only the index data structures, so it
117 * can be safely called from atomic context. When memory needs to be
118 * returned to the system, free path schedules reclaim_work which
119 * grabs both pcpu_alloc_mutex and pcpu_lock, unlinks chunks to be
120 * reclaimed, release both locks and frees the chunks. Note that it's
121 * necessary to grab both locks to remove a chunk from circulation as
122 * allocation path might be referencing the chunk with only
123 * pcpu_alloc_mutex locked.
112 */ 124 */
113static DEFINE_MUTEX(pcpu_mutex); 125static DEFINE_MUTEX(pcpu_alloc_mutex); /* protects whole alloc and reclaim */
126static DEFINE_SPINLOCK(pcpu_lock); /* protects index data structures */
114 127
115static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */ 128static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */
116static struct rb_root pcpu_addr_root = RB_ROOT; /* chunks by address */ 129static struct rb_root pcpu_addr_root = RB_ROOT; /* chunks by address */
117 130
131/* reclaim work to release fully free chunks, scheduled from free path */
132static void pcpu_reclaim(struct work_struct *work);
133static DECLARE_WORK(pcpu_reclaim_work, pcpu_reclaim);
134
118static int __pcpu_size_to_slot(int size) 135static int __pcpu_size_to_slot(int size)
119{ 136{
120 int highbit = fls(size); /* size is in bytes */ 137 int highbit = fls(size); /* size is in bytes */
@@ -161,39 +178,44 @@ static bool pcpu_chunk_page_occupied(struct pcpu_chunk *chunk,
161} 178}
162 179
163/** 180/**
164 * pcpu_realloc - versatile realloc 181 * pcpu_mem_alloc - allocate memory
165 * @p: the current pointer (can be NULL for new allocations) 182 * @size: bytes to allocate
166 * @size: the current size in bytes (can be 0 for new allocations) 183 *
167 * @new_size: the wanted new size in bytes (can be 0 for free) 184 * Allocate @size bytes. If @size is smaller than PAGE_SIZE,
185 * kzalloc() is used; otherwise, vmalloc() is used. The returned
186 * memory is always zeroed.
168 * 187 *
169 * More robust realloc which can be used to allocate, resize or free a 188 * CONTEXT:
170 * memory area of arbitrary size. If the needed size goes over 189 * Does GFP_KERNEL allocation.
171 * PAGE_SIZE, kernel VM is used.
172 * 190 *
173 * RETURNS: 191 * RETURNS:
174 * The new pointer on success, NULL on failure. 192 * Pointer to the allocated area on success, NULL on failure.
175 */ 193 */
176static void *pcpu_realloc(void *p, size_t size, size_t new_size) 194static void *pcpu_mem_alloc(size_t size)
177{ 195{
178 void *new; 196 if (size <= PAGE_SIZE)
179 197 return kzalloc(size, GFP_KERNEL);
180 if (new_size <= PAGE_SIZE) 198 else {
181 new = kmalloc(new_size, GFP_KERNEL); 199 void *ptr = vmalloc(size);
182 else 200 if (ptr)
183 new = vmalloc(new_size); 201 memset(ptr, 0, size);
184 if (new_size && !new) 202 return ptr;
185 return NULL; 203 }
186 204}
187 memcpy(new, p, min(size, new_size));
188 if (new_size > size)
189 memset(new + size, 0, new_size - size);
190 205
206/**
207 * pcpu_mem_free - free memory
208 * @ptr: memory to free
209 * @size: size of the area
210 *
211 * Free @ptr. @ptr should have been allocated using pcpu_mem_alloc().
212 */
213static void pcpu_mem_free(void *ptr, size_t size)
214{
191 if (size <= PAGE_SIZE) 215 if (size <= PAGE_SIZE)
192 kfree(p); 216 kfree(ptr);
193 else 217 else
194 vfree(p); 218 vfree(ptr);
195
196 return new;
197} 219}
198 220
199/** 221/**
@@ -203,13 +225,17 @@ static void *pcpu_realloc(void *p, size_t size, size_t new_size)
203 * 225 *
204 * This function is called after an allocation or free changed @chunk. 226 * This function is called after an allocation or free changed @chunk.
205 * New slot according to the changed state is determined and @chunk is 227 * New slot according to the changed state is determined and @chunk is
206 * moved to the slot. 228 * moved to the slot. Note that the reserved chunk is never put on
229 * chunk slots.
230 *
231 * CONTEXT:
232 * pcpu_lock.
207 */ 233 */
208static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot) 234static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
209{ 235{
210 int nslot = pcpu_chunk_slot(chunk); 236 int nslot = pcpu_chunk_slot(chunk);
211 237
212 if (oslot != nslot) { 238 if (chunk != pcpu_reserved_chunk && oslot != nslot) {
213 if (oslot < nslot) 239 if (oslot < nslot)
214 list_move(&chunk->list, &pcpu_slot[nslot]); 240 list_move(&chunk->list, &pcpu_slot[nslot]);
215 else 241 else
@@ -249,6 +275,9 @@ static struct rb_node **pcpu_chunk_rb_search(void *addr,
249 * searchs for the chunk with the highest start address which isn't 275 * searchs for the chunk with the highest start address which isn't
250 * beyond @addr. 276 * beyond @addr.
251 * 277 *
278 * CONTEXT:
279 * pcpu_lock.
280 *
252 * RETURNS: 281 * RETURNS:
253 * The address of the found chunk. 282 * The address of the found chunk.
254 */ 283 */
@@ -257,6 +286,15 @@ static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
257 struct rb_node *n, *parent; 286 struct rb_node *n, *parent;
258 struct pcpu_chunk *chunk; 287 struct pcpu_chunk *chunk;
259 288
289 /* is it in the reserved chunk? */
290 if (pcpu_reserved_chunk) {
291 void *start = pcpu_reserved_chunk->vm->addr;
292
293 if (addr >= start && addr < start + pcpu_reserved_chunk_limit)
294 return pcpu_reserved_chunk;
295 }
296
297 /* nah... search the regular ones */
260 n = *pcpu_chunk_rb_search(addr, &parent); 298 n = *pcpu_chunk_rb_search(addr, &parent);
261 if (!n) { 299 if (!n) {
262 /* no exactly matching chunk, the parent is the closest */ 300 /* no exactly matching chunk, the parent is the closest */
@@ -280,6 +318,9 @@ static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
280 * @new: chunk to insert 318 * @new: chunk to insert
281 * 319 *
282 * Insert @new into address rb tree. 320 * Insert @new into address rb tree.
321 *
322 * CONTEXT:
323 * pcpu_lock.
283 */ 324 */
284static void pcpu_chunk_addr_insert(struct pcpu_chunk *new) 325static void pcpu_chunk_addr_insert(struct pcpu_chunk *new)
285{ 326{
@@ -292,6 +333,66 @@ static void pcpu_chunk_addr_insert(struct pcpu_chunk *new)
292} 333}
293 334
294/** 335/**
336 * pcpu_extend_area_map - extend area map for allocation
337 * @chunk: target chunk
338 *
339 * Extend area map of @chunk so that it can accomodate an allocation.
340 * A single allocation can split an area into three areas, so this
341 * function makes sure that @chunk->map has at least two extra slots.
342 *
343 * CONTEXT:
344 * pcpu_alloc_mutex, pcpu_lock. pcpu_lock is released and reacquired
345 * if area map is extended.
346 *
347 * RETURNS:
348 * 0 if noop, 1 if successfully extended, -errno on failure.
349 */
350static int pcpu_extend_area_map(struct pcpu_chunk *chunk)
351{
352 int new_alloc;
353 int *new;
354 size_t size;
355
356 /* has enough? */
357 if (chunk->map_alloc >= chunk->map_used + 2)
358 return 0;
359
360 spin_unlock_irq(&pcpu_lock);
361
362 new_alloc = PCPU_DFL_MAP_ALLOC;
363 while (new_alloc < chunk->map_used + 2)
364 new_alloc *= 2;
365
366 new = pcpu_mem_alloc(new_alloc * sizeof(new[0]));
367 if (!new) {
368 spin_lock_irq(&pcpu_lock);
369 return -ENOMEM;
370 }
371
372 /*
373 * Acquire pcpu_lock and switch to new area map. Only free
374 * could have happened inbetween, so map_used couldn't have
375 * grown.
376 */
377 spin_lock_irq(&pcpu_lock);
378 BUG_ON(new_alloc < chunk->map_used + 2);
379
380 size = chunk->map_alloc * sizeof(chunk->map[0]);
381 memcpy(new, chunk->map, size);
382
383 /*
384 * map_alloc < PCPU_DFL_MAP_ALLOC indicates that the chunk is
385 * one of the first chunks and still using static map.
386 */
387 if (chunk->map_alloc >= PCPU_DFL_MAP_ALLOC)
388 pcpu_mem_free(chunk->map, size);
389
390 chunk->map_alloc = new_alloc;
391 chunk->map = new;
392 return 0;
393}
394
395/**
295 * pcpu_split_block - split a map block 396 * pcpu_split_block - split a map block
296 * @chunk: chunk of interest 397 * @chunk: chunk of interest
297 * @i: index of map block to split 398 * @i: index of map block to split
@@ -306,33 +407,19 @@ static void pcpu_chunk_addr_insert(struct pcpu_chunk *new)
306 * depending on @head, is reduced by @tail bytes and @tail byte block 407 * depending on @head, is reduced by @tail bytes and @tail byte block
307 * is inserted after the target block. 408 * is inserted after the target block.
308 * 409 *
309 * RETURNS: 410 * @chunk->map must have enough free slots to accomodate the split.
310 * 0 on success, -errno on failure. 411 *
412 * CONTEXT:
413 * pcpu_lock.
311 */ 414 */
312static int pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail) 415static void pcpu_split_block(struct pcpu_chunk *chunk, int i,
416 int head, int tail)
313{ 417{
314 int nr_extra = !!head + !!tail; 418 int nr_extra = !!head + !!tail;
315 int target = chunk->map_used + nr_extra;
316
317 /* reallocation required? */
318 if (chunk->map_alloc < target) {
319 int new_alloc = chunk->map_alloc;
320 int *new;
321 419
322 while (new_alloc < target) 420 BUG_ON(chunk->map_alloc < chunk->map_used + nr_extra);
323 new_alloc *= 2;
324 421
325 new = pcpu_realloc(chunk->map, 422 /* insert new subblocks */
326 chunk->map_alloc * sizeof(new[0]),
327 new_alloc * sizeof(new[0]));
328 if (!new)
329 return -ENOMEM;
330
331 chunk->map_alloc = new_alloc;
332 chunk->map = new;
333 }
334
335 /* insert a new subblock */
336 memmove(&chunk->map[i + nr_extra], &chunk->map[i], 423 memmove(&chunk->map[i + nr_extra], &chunk->map[i],
337 sizeof(chunk->map[0]) * (chunk->map_used - i)); 424 sizeof(chunk->map[0]) * (chunk->map_used - i));
338 chunk->map_used += nr_extra; 425 chunk->map_used += nr_extra;
@@ -345,7 +432,6 @@ static int pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail)
345 chunk->map[i++] -= tail; 432 chunk->map[i++] -= tail;
346 chunk->map[i] = tail; 433 chunk->map[i] = tail;
347 } 434 }
348 return 0;
349} 435}
350 436
351/** 437/**
@@ -358,8 +444,14 @@ static int pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail)
358 * Note that this function only allocates the offset. It doesn't 444 * Note that this function only allocates the offset. It doesn't
359 * populate or map the area. 445 * populate or map the area.
360 * 446 *
447 * @chunk->map must have at least two free slots.
448 *
449 * CONTEXT:
450 * pcpu_lock.
451 *
361 * RETURNS: 452 * RETURNS:
362 * Allocated offset in @chunk on success, -errno on failure. 453 * Allocated offset in @chunk on success, -1 if no matching area is
454 * found.
363 */ 455 */
364static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align) 456static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
365{ 457{
@@ -367,22 +459,6 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
367 int max_contig = 0; 459 int max_contig = 0;
368 int i, off; 460 int i, off;
369 461
370 /*
371 * The static chunk initially doesn't have map attached
372 * because kmalloc wasn't available during init. Give it one.
373 */
374 if (unlikely(!chunk->map)) {
375 chunk->map = pcpu_realloc(NULL, 0,
376 PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
377 if (!chunk->map)
378 return -ENOMEM;
379
380 chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
381 chunk->map[chunk->map_used++] = -pcpu_static_size;
382 if (chunk->free_size)
383 chunk->map[chunk->map_used++] = chunk->free_size;
384 }
385
386 for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) { 462 for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) {
387 bool is_last = i + 1 == chunk->map_used; 463 bool is_last = i + 1 == chunk->map_used;
388 int head, tail; 464 int head, tail;
@@ -423,8 +499,7 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
423 499
424 /* split if warranted */ 500 /* split if warranted */
425 if (head || tail) { 501 if (head || tail) {
426 if (pcpu_split_block(chunk, i, head, tail)) 502 pcpu_split_block(chunk, i, head, tail);
427 return -ENOMEM;
428 if (head) { 503 if (head) {
429 i++; 504 i++;
430 off += head; 505 off += head;
@@ -451,14 +526,8 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
451 chunk->contig_hint = max_contig; /* fully scanned */ 526 chunk->contig_hint = max_contig; /* fully scanned */
452 pcpu_chunk_relocate(chunk, oslot); 527 pcpu_chunk_relocate(chunk, oslot);
453 528
454 /* 529 /* tell the upper layer that this chunk has no matching area */
455 * Tell the upper layer that this chunk has no area left. 530 return -1;
456 * Note that this is not an error condition but a notification
457 * to upper layer that it needs to look at other chunks.
458 * -ENOSPC is chosen as it isn't used in memory subsystem and
459 * matches the meaning in a way.
460 */
461 return -ENOSPC;
462} 531}
463 532
464/** 533/**
@@ -469,6 +538,9 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
469 * Free area starting from @freeme to @chunk. Note that this function 538 * Free area starting from @freeme to @chunk. Note that this function
470 * only modifies the allocation map. It doesn't depopulate or unmap 539 * only modifies the allocation map. It doesn't depopulate or unmap
471 * the area. 540 * the area.
541 *
542 * CONTEXT:
543 * pcpu_lock.
472 */ 544 */
473static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme) 545static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
474{ 546{
@@ -554,6 +626,9 @@ static void pcpu_unmap(struct pcpu_chunk *chunk, int page_start, int page_end,
554 * For each cpu, depopulate and unmap pages [@page_start,@page_end) 626 * For each cpu, depopulate and unmap pages [@page_start,@page_end)
555 * from @chunk. If @flush is true, vcache is flushed before unmapping 627 * from @chunk. If @flush is true, vcache is flushed before unmapping
556 * and tlb after. 628 * and tlb after.
629 *
630 * CONTEXT:
631 * pcpu_alloc_mutex.
557 */ 632 */
558static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size, 633static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size,
559 bool flush) 634 bool flush)
@@ -632,6 +707,9 @@ static int pcpu_map(struct pcpu_chunk *chunk, int page_start, int page_end)
632 * 707 *
633 * For each cpu, populate and map pages [@page_start,@page_end) into 708 * For each cpu, populate and map pages [@page_start,@page_end) into
634 * @chunk. The area is cleared on return. 709 * @chunk. The area is cleared on return.
710 *
711 * CONTEXT:
712 * pcpu_alloc_mutex, does GFP_KERNEL allocation.
635 */ 713 */
636static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size) 714static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
637{ 715{
@@ -686,7 +764,7 @@ static void free_pcpu_chunk(struct pcpu_chunk *chunk)
686 return; 764 return;
687 if (chunk->vm) 765 if (chunk->vm)
688 free_vm_area(chunk->vm); 766 free_vm_area(chunk->vm);
689 pcpu_realloc(chunk->map, chunk->map_alloc * sizeof(chunk->map[0]), 0); 767 pcpu_mem_free(chunk->map, chunk->map_alloc * sizeof(chunk->map[0]));
690 kfree(chunk); 768 kfree(chunk);
691} 769}
692 770
@@ -698,10 +776,10 @@ static struct pcpu_chunk *alloc_pcpu_chunk(void)
698 if (!chunk) 776 if (!chunk)
699 return NULL; 777 return NULL;
700 778
701 chunk->map = pcpu_realloc(NULL, 0, 779 chunk->map = pcpu_mem_alloc(PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
702 PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
703 chunk->map_alloc = PCPU_DFL_MAP_ALLOC; 780 chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
704 chunk->map[chunk->map_used++] = pcpu_unit_size; 781 chunk->map[chunk->map_used++] = pcpu_unit_size;
782 chunk->page = chunk->page_ar;
705 783
706 chunk->vm = get_vm_area(pcpu_chunk_size, GFP_KERNEL); 784 chunk->vm = get_vm_area(pcpu_chunk_size, GFP_KERNEL);
707 if (!chunk->vm) { 785 if (!chunk->vm) {
@@ -717,19 +795,21 @@ static struct pcpu_chunk *alloc_pcpu_chunk(void)
717} 795}
718 796
719/** 797/**
720 * __alloc_percpu - allocate percpu area 798 * pcpu_alloc - the percpu allocator
721 * @size: size of area to allocate in bytes 799 * @size: size of area to allocate in bytes
722 * @align: alignment of area (max PAGE_SIZE) 800 * @align: alignment of area (max PAGE_SIZE)
801 * @reserved: allocate from the reserved chunk if available
723 * 802 *
724 * Allocate percpu area of @size bytes aligned at @align. Might 803 * Allocate percpu area of @size bytes aligned at @align.
725 * sleep. Might trigger writeouts. 804 *
805 * CONTEXT:
806 * Does GFP_KERNEL allocation.
726 * 807 *
727 * RETURNS: 808 * RETURNS:
728 * Percpu pointer to the allocated area on success, NULL on failure. 809 * Percpu pointer to the allocated area on success, NULL on failure.
729 */ 810 */
730void *__alloc_percpu(size_t size, size_t align) 811static void *pcpu_alloc(size_t size, size_t align, bool reserved)
731{ 812{
732 void *ptr = NULL;
733 struct pcpu_chunk *chunk; 813 struct pcpu_chunk *chunk;
734 int slot, off; 814 int slot, off;
735 815
@@ -739,90 +819,192 @@ void *__alloc_percpu(size_t size, size_t align)
739 return NULL; 819 return NULL;
740 } 820 }
741 821
742 mutex_lock(&pcpu_mutex); 822 mutex_lock(&pcpu_alloc_mutex);
823 spin_lock_irq(&pcpu_lock);
824
825 /* serve reserved allocations from the reserved chunk if available */
826 if (reserved && pcpu_reserved_chunk) {
827 chunk = pcpu_reserved_chunk;
828 if (size > chunk->contig_hint ||
829 pcpu_extend_area_map(chunk) < 0)
830 goto fail_unlock;
831 off = pcpu_alloc_area(chunk, size, align);
832 if (off >= 0)
833 goto area_found;
834 goto fail_unlock;
835 }
743 836
744 /* allocate area */ 837restart:
838 /* search through normal chunks */
745 for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) { 839 for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) {
746 list_for_each_entry(chunk, &pcpu_slot[slot], list) { 840 list_for_each_entry(chunk, &pcpu_slot[slot], list) {
747 if (size > chunk->contig_hint) 841 if (size > chunk->contig_hint)
748 continue; 842 continue;
843
844 switch (pcpu_extend_area_map(chunk)) {
845 case 0:
846 break;
847 case 1:
848 goto restart; /* pcpu_lock dropped, restart */
849 default:
850 goto fail_unlock;
851 }
852
749 off = pcpu_alloc_area(chunk, size, align); 853 off = pcpu_alloc_area(chunk, size, align);
750 if (off >= 0) 854 if (off >= 0)
751 goto area_found; 855 goto area_found;
752 if (off != -ENOSPC)
753 goto out_unlock;
754 } 856 }
755 } 857 }
756 858
757 /* hmmm... no space left, create a new chunk */ 859 /* hmmm... no space left, create a new chunk */
860 spin_unlock_irq(&pcpu_lock);
861
758 chunk = alloc_pcpu_chunk(); 862 chunk = alloc_pcpu_chunk();
759 if (!chunk) 863 if (!chunk)
760 goto out_unlock; 864 goto fail_unlock_mutex;
865
866 spin_lock_irq(&pcpu_lock);
761 pcpu_chunk_relocate(chunk, -1); 867 pcpu_chunk_relocate(chunk, -1);
762 pcpu_chunk_addr_insert(chunk); 868 pcpu_chunk_addr_insert(chunk);
763 869 goto restart;
764 off = pcpu_alloc_area(chunk, size, align);
765 if (off < 0)
766 goto out_unlock;
767 870
768area_found: 871area_found:
872 spin_unlock_irq(&pcpu_lock);
873
769 /* populate, map and clear the area */ 874 /* populate, map and clear the area */
770 if (pcpu_populate_chunk(chunk, off, size)) { 875 if (pcpu_populate_chunk(chunk, off, size)) {
876 spin_lock_irq(&pcpu_lock);
771 pcpu_free_area(chunk, off); 877 pcpu_free_area(chunk, off);
772 goto out_unlock; 878 goto fail_unlock;
773 } 879 }
774 880
775 ptr = __addr_to_pcpu_ptr(chunk->vm->addr + off); 881 mutex_unlock(&pcpu_alloc_mutex);
776out_unlock: 882
777 mutex_unlock(&pcpu_mutex); 883 return __addr_to_pcpu_ptr(chunk->vm->addr + off);
778 return ptr; 884
885fail_unlock:
886 spin_unlock_irq(&pcpu_lock);
887fail_unlock_mutex:
888 mutex_unlock(&pcpu_alloc_mutex);
889 return NULL;
890}
891
892/**
893 * __alloc_percpu - allocate dynamic percpu area
894 * @size: size of area to allocate in bytes
895 * @align: alignment of area (max PAGE_SIZE)
896 *
897 * Allocate percpu area of @size bytes aligned at @align. Might
898 * sleep. Might trigger writeouts.
899 *
900 * CONTEXT:
901 * Does GFP_KERNEL allocation.
902 *
903 * RETURNS:
904 * Percpu pointer to the allocated area on success, NULL on failure.
905 */
906void *__alloc_percpu(size_t size, size_t align)
907{
908 return pcpu_alloc(size, align, false);
779} 909}
780EXPORT_SYMBOL_GPL(__alloc_percpu); 910EXPORT_SYMBOL_GPL(__alloc_percpu);
781 911
782static void pcpu_kill_chunk(struct pcpu_chunk *chunk) 912/**
913 * __alloc_reserved_percpu - allocate reserved percpu area
914 * @size: size of area to allocate in bytes
915 * @align: alignment of area (max PAGE_SIZE)
916 *
917 * Allocate percpu area of @size bytes aligned at @align from reserved
918 * percpu area if arch has set it up; otherwise, allocation is served
919 * from the same dynamic area. Might sleep. Might trigger writeouts.
920 *
921 * CONTEXT:
922 * Does GFP_KERNEL allocation.
923 *
924 * RETURNS:
925 * Percpu pointer to the allocated area on success, NULL on failure.
926 */
927void *__alloc_reserved_percpu(size_t size, size_t align)
928{
929 return pcpu_alloc(size, align, true);
930}
931
932/**
933 * pcpu_reclaim - reclaim fully free chunks, workqueue function
934 * @work: unused
935 *
936 * Reclaim all fully free chunks except for the first one.
937 *
938 * CONTEXT:
939 * workqueue context.
940 */
941static void pcpu_reclaim(struct work_struct *work)
783{ 942{
784 WARN_ON(chunk->immutable); 943 LIST_HEAD(todo);
785 pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size, false); 944 struct list_head *head = &pcpu_slot[pcpu_nr_slots - 1];
786 list_del(&chunk->list); 945 struct pcpu_chunk *chunk, *next;
787 rb_erase(&chunk->rb_node, &pcpu_addr_root); 946
788 free_pcpu_chunk(chunk); 947 mutex_lock(&pcpu_alloc_mutex);
948 spin_lock_irq(&pcpu_lock);
949
950 list_for_each_entry_safe(chunk, next, head, list) {
951 WARN_ON(chunk->immutable);
952
953 /* spare the first one */
954 if (chunk == list_first_entry(head, struct pcpu_chunk, list))
955 continue;
956
957 rb_erase(&chunk->rb_node, &pcpu_addr_root);
958 list_move(&chunk->list, &todo);
959 }
960
961 spin_unlock_irq(&pcpu_lock);
962 mutex_unlock(&pcpu_alloc_mutex);
963
964 list_for_each_entry_safe(chunk, next, &todo, list) {
965 pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size, false);
966 free_pcpu_chunk(chunk);
967 }
789} 968}
790 969
791/** 970/**
792 * free_percpu - free percpu area 971 * free_percpu - free percpu area
793 * @ptr: pointer to area to free 972 * @ptr: pointer to area to free
794 * 973 *
795 * Free percpu area @ptr. Might sleep. 974 * Free percpu area @ptr.
975 *
976 * CONTEXT:
977 * Can be called from atomic context.
796 */ 978 */
797void free_percpu(void *ptr) 979void free_percpu(void *ptr)
798{ 980{
799 void *addr = __pcpu_ptr_to_addr(ptr); 981 void *addr = __pcpu_ptr_to_addr(ptr);
800 struct pcpu_chunk *chunk; 982 struct pcpu_chunk *chunk;
983 unsigned long flags;
801 int off; 984 int off;
802 985
803 if (!ptr) 986 if (!ptr)
804 return; 987 return;
805 988
806 mutex_lock(&pcpu_mutex); 989 spin_lock_irqsave(&pcpu_lock, flags);
807 990
808 chunk = pcpu_chunk_addr_search(addr); 991 chunk = pcpu_chunk_addr_search(addr);
809 off = addr - chunk->vm->addr; 992 off = addr - chunk->vm->addr;
810 993
811 pcpu_free_area(chunk, off); 994 pcpu_free_area(chunk, off);
812 995
813 /* the chunk became fully free, kill one if there are other free ones */ 996 /* if there are more than one fully free chunks, wake up grim reaper */
814 if (chunk->free_size == pcpu_unit_size) { 997 if (chunk->free_size == pcpu_unit_size) {
815 struct pcpu_chunk *pos; 998 struct pcpu_chunk *pos;
816 999
817 list_for_each_entry(pos, 1000 list_for_each_entry(pos, &pcpu_slot[pcpu_nr_slots - 1], list)
818 &pcpu_slot[pcpu_chunk_slot(chunk)], list)
819 if (pos != chunk) { 1001 if (pos != chunk) {
820 pcpu_kill_chunk(pos); 1002 schedule_work(&pcpu_reclaim_work);
821 break; 1003 break;
822 } 1004 }
823 } 1005 }
824 1006
825 mutex_unlock(&pcpu_mutex); 1007 spin_unlock_irqrestore(&pcpu_lock, flags);
826} 1008}
827EXPORT_SYMBOL_GPL(free_percpu); 1009EXPORT_SYMBOL_GPL(free_percpu);
828 1010
@@ -830,8 +1012,9 @@ EXPORT_SYMBOL_GPL(free_percpu);
830 * pcpu_setup_first_chunk - initialize the first percpu chunk 1012 * pcpu_setup_first_chunk - initialize the first percpu chunk
831 * @get_page_fn: callback to fetch page pointer 1013 * @get_page_fn: callback to fetch page pointer
832 * @static_size: the size of static percpu area in bytes 1014 * @static_size: the size of static percpu area in bytes
833 * @unit_size: unit size in bytes, must be multiple of PAGE_SIZE, 0 for auto 1015 * @reserved_size: the size of reserved percpu area in bytes
834 * @free_size: free size in bytes, 0 for auto 1016 * @unit_size: unit size in bytes, must be multiple of PAGE_SIZE, -1 for auto
1017 * @dyn_size: free size for dynamic allocation in bytes, -1 for auto
835 * @base_addr: mapped address, NULL for auto 1018 * @base_addr: mapped address, NULL for auto
836 * @populate_pte_fn: callback to allocate pagetable, NULL if unnecessary 1019 * @populate_pte_fn: callback to allocate pagetable, NULL if unnecessary
837 * 1020 *
@@ -848,13 +1031,22 @@ EXPORT_SYMBOL_GPL(free_percpu);
848 * indicates end of pages for the cpu. Note that @get_page_fn() must 1031 * indicates end of pages for the cpu. Note that @get_page_fn() must
849 * return the same number of pages for all cpus. 1032 * return the same number of pages for all cpus.
850 * 1033 *
851 * @unit_size, if non-zero, determines unit size and must be aligned 1034 * @reserved_size, if non-zero, specifies the amount of bytes to
852 * to PAGE_SIZE and equal to or larger than @static_size + @free_size. 1035 * reserve after the static area in the first chunk. This reserves
1036 * the first chunk such that it's available only through reserved
1037 * percpu allocation. This is primarily used to serve module percpu
1038 * static areas on architectures where the addressing model has
1039 * limited offset range for symbol relocations to guarantee module
1040 * percpu symbols fall inside the relocatable range.
853 * 1041 *
854 * @free_size determines the number of free bytes after the static 1042 * @unit_size, if non-negative, specifies unit size and must be
855 * area in the first chunk. If zero, whatever left is available. 1043 * aligned to PAGE_SIZE and equal to or larger than @static_size +
856 * Specifying non-zero value make percpu leave the area after 1044 * @reserved_size + @dyn_size.
857 * @static_size + @free_size alone. 1045 *
1046 * @dyn_size, if non-negative, limits the number of bytes available
1047 * for dynamic allocation in the first chunk. Specifying non-negative
1048 * value make percpu leave alone the area beyond @static_size +
1049 * @reserved_size + @dyn_size.
858 * 1050 *
859 * Non-null @base_addr means that the caller already allocated virtual 1051 * Non-null @base_addr means that the caller already allocated virtual
860 * region for the first chunk and mapped it. percpu must not mess 1052 * region for the first chunk and mapped it. percpu must not mess
@@ -864,41 +1056,58 @@ EXPORT_SYMBOL_GPL(free_percpu);
864 * @populate_pte_fn is used to populate the pagetable. NULL means the 1056 * @populate_pte_fn is used to populate the pagetable. NULL means the
865 * caller already populated the pagetable. 1057 * caller already populated the pagetable.
866 * 1058 *
1059 * If the first chunk ends up with both reserved and dynamic areas, it
1060 * is served by two chunks - one to serve the core static and reserved
1061 * areas and the other for the dynamic area. They share the same vm
1062 * and page map but uses different area allocation map to stay away
1063 * from each other. The latter chunk is circulated in the chunk slots
1064 * and available for dynamic allocation like any other chunks.
1065 *
867 * RETURNS: 1066 * RETURNS:
868 * The determined pcpu_unit_size which can be used to initialize 1067 * The determined pcpu_unit_size which can be used to initialize
869 * percpu access. 1068 * percpu access.
870 */ 1069 */
871size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, 1070size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
872 size_t static_size, size_t unit_size, 1071 size_t static_size, size_t reserved_size,
873 size_t free_size, void *base_addr, 1072 ssize_t unit_size, ssize_t dyn_size,
1073 void *base_addr,
874 pcpu_populate_pte_fn_t populate_pte_fn) 1074 pcpu_populate_pte_fn_t populate_pte_fn)
875{ 1075{
876 static struct vm_struct static_vm; 1076 static struct vm_struct first_vm;
877 struct pcpu_chunk *static_chunk; 1077 static int smap[2], dmap[2];
1078 struct pcpu_chunk *schunk, *dchunk = NULL;
878 unsigned int cpu; 1079 unsigned int cpu;
879 int nr_pages; 1080 int nr_pages;
880 int err, i; 1081 int err, i;
881 1082
882 /* santiy checks */ 1083 /* santiy checks */
1084 BUILD_BUG_ON(ARRAY_SIZE(smap) >= PCPU_DFL_MAP_ALLOC ||
1085 ARRAY_SIZE(dmap) >= PCPU_DFL_MAP_ALLOC);
883 BUG_ON(!static_size); 1086 BUG_ON(!static_size);
884 BUG_ON(!unit_size && free_size); 1087 if (unit_size >= 0) {
885 BUG_ON(unit_size && unit_size < static_size + free_size); 1088 BUG_ON(unit_size < static_size + reserved_size +
886 BUG_ON(unit_size & ~PAGE_MASK); 1089 (dyn_size >= 0 ? dyn_size : 0));
887 BUG_ON(base_addr && !unit_size); 1090 BUG_ON(unit_size & ~PAGE_MASK);
1091 } else {
1092 BUG_ON(dyn_size >= 0);
1093 BUG_ON(base_addr);
1094 }
888 BUG_ON(base_addr && populate_pte_fn); 1095 BUG_ON(base_addr && populate_pte_fn);
889 1096
890 if (unit_size) 1097 if (unit_size >= 0)
891 pcpu_unit_pages = unit_size >> PAGE_SHIFT; 1098 pcpu_unit_pages = unit_size >> PAGE_SHIFT;
892 else 1099 else
893 pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_SIZE >> PAGE_SHIFT, 1100 pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_SIZE >> PAGE_SHIFT,
894 PFN_UP(static_size)); 1101 PFN_UP(static_size + reserved_size));
895 1102
896 pcpu_static_size = static_size;
897 pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT; 1103 pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT;
898 pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size; 1104 pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size;
899 pcpu_chunk_struct_size = sizeof(struct pcpu_chunk) 1105 pcpu_chunk_struct_size = sizeof(struct pcpu_chunk)
900 + num_possible_cpus() * pcpu_unit_pages * sizeof(struct page *); 1106 + num_possible_cpus() * pcpu_unit_pages * sizeof(struct page *);
901 1107
1108 if (dyn_size < 0)
1109 dyn_size = pcpu_unit_size - static_size - reserved_size;
1110
902 /* 1111 /*
903 * Allocate chunk slots. The additional last slot is for 1112 * Allocate chunk slots. The additional last slot is for
904 * empty chunks. 1113 * empty chunks.
@@ -908,33 +1117,66 @@ size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
908 for (i = 0; i < pcpu_nr_slots; i++) 1117 for (i = 0; i < pcpu_nr_slots; i++)
909 INIT_LIST_HEAD(&pcpu_slot[i]); 1118 INIT_LIST_HEAD(&pcpu_slot[i]);
910 1119
911 /* init static_chunk */ 1120 /*
912 static_chunk = alloc_bootmem(pcpu_chunk_struct_size); 1121 * Initialize static chunk. If reserved_size is zero, the
913 INIT_LIST_HEAD(&static_chunk->list); 1122 * static chunk covers static area + dynamic allocation area
914 static_chunk->vm = &static_vm; 1123 * in the first chunk. If reserved_size is not zero, it
915 1124 * covers static area + reserved area (mostly used for module
916 if (free_size) 1125 * static percpu allocation).
917 static_chunk->free_size = free_size; 1126 */
918 else 1127 schunk = alloc_bootmem(pcpu_chunk_struct_size);
919 static_chunk->free_size = pcpu_unit_size - pcpu_static_size; 1128 INIT_LIST_HEAD(&schunk->list);
920 1129 schunk->vm = &first_vm;
921 static_chunk->contig_hint = static_chunk->free_size; 1130 schunk->map = smap;
1131 schunk->map_alloc = ARRAY_SIZE(smap);
1132 schunk->page = schunk->page_ar;
1133
1134 if (reserved_size) {
1135 schunk->free_size = reserved_size;
1136 pcpu_reserved_chunk = schunk; /* not for dynamic alloc */
1137 } else {
1138 schunk->free_size = dyn_size;
1139 dyn_size = 0; /* dynamic area covered */
1140 }
1141 schunk->contig_hint = schunk->free_size;
1142
1143 schunk->map[schunk->map_used++] = -static_size;
1144 if (schunk->free_size)
1145 schunk->map[schunk->map_used++] = schunk->free_size;
1146
1147 pcpu_reserved_chunk_limit = static_size + schunk->free_size;
1148
1149 /* init dynamic chunk if necessary */
1150 if (dyn_size) {
1151 dchunk = alloc_bootmem(sizeof(struct pcpu_chunk));
1152 INIT_LIST_HEAD(&dchunk->list);
1153 dchunk->vm = &first_vm;
1154 dchunk->map = dmap;
1155 dchunk->map_alloc = ARRAY_SIZE(dmap);
1156 dchunk->page = schunk->page_ar; /* share page map with schunk */
1157
1158 dchunk->contig_hint = dchunk->free_size = dyn_size;
1159 dchunk->map[dchunk->map_used++] = -pcpu_reserved_chunk_limit;
1160 dchunk->map[dchunk->map_used++] = dchunk->free_size;
1161 }
922 1162
923 /* allocate vm address */ 1163 /* allocate vm address */
924 static_vm.flags = VM_ALLOC; 1164 first_vm.flags = VM_ALLOC;
925 static_vm.size = pcpu_chunk_size; 1165 first_vm.size = pcpu_chunk_size;
926 1166
927 if (!base_addr) 1167 if (!base_addr)
928 vm_area_register_early(&static_vm, PAGE_SIZE); 1168 vm_area_register_early(&first_vm, PAGE_SIZE);
929 else { 1169 else {
930 /* 1170 /*
931 * Pages already mapped. No need to remap into 1171 * Pages already mapped. No need to remap into
932 * vmalloc area. In this case the static chunk can't 1172 * vmalloc area. In this case the first chunks can't
933 * be mapped or unmapped by percpu and is marked 1173 * be mapped or unmapped by percpu and are marked
934 * immutable. 1174 * immutable.
935 */ 1175 */
936 static_vm.addr = base_addr; 1176 first_vm.addr = base_addr;
937 static_chunk->immutable = true; 1177 schunk->immutable = true;
1178 if (dchunk)
1179 dchunk->immutable = true;
938 } 1180 }
939 1181
940 /* assign pages */ 1182 /* assign pages */
@@ -945,10 +1187,10 @@ size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
945 1187
946 if (!page) 1188 if (!page)
947 break; 1189 break;
948 *pcpu_chunk_pagep(static_chunk, cpu, i) = page; 1190 *pcpu_chunk_pagep(schunk, cpu, i) = page;
949 } 1191 }
950 1192
951 BUG_ON(i < PFN_UP(pcpu_static_size)); 1193 BUG_ON(i < PFN_UP(static_size));
952 1194
953 if (nr_pages < 0) 1195 if (nr_pages < 0)
954 nr_pages = i; 1196 nr_pages = i;
@@ -960,20 +1202,25 @@ size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
960 if (populate_pte_fn) { 1202 if (populate_pte_fn) {
961 for_each_possible_cpu(cpu) 1203 for_each_possible_cpu(cpu)
962 for (i = 0; i < nr_pages; i++) 1204 for (i = 0; i < nr_pages; i++)
963 populate_pte_fn(pcpu_chunk_addr(static_chunk, 1205 populate_pte_fn(pcpu_chunk_addr(schunk,
964 cpu, i)); 1206 cpu, i));
965 1207
966 err = pcpu_map(static_chunk, 0, nr_pages); 1208 err = pcpu_map(schunk, 0, nr_pages);
967 if (err) 1209 if (err)
968 panic("failed to setup static percpu area, err=%d\n", 1210 panic("failed to setup static percpu area, err=%d\n",
969 err); 1211 err);
970 } 1212 }
971 1213
972 /* link static_chunk in */ 1214 /* link the first chunk in */
973 pcpu_chunk_relocate(static_chunk, -1); 1215 if (!dchunk) {
974 pcpu_chunk_addr_insert(static_chunk); 1216 pcpu_chunk_relocate(schunk, -1);
1217 pcpu_chunk_addr_insert(schunk);
1218 } else {
1219 pcpu_chunk_relocate(dchunk, -1);
1220 pcpu_chunk_addr_insert(dchunk);
1221 }
975 1222
976 /* we're done */ 1223 /* we're done */
977 pcpu_base_addr = (void *)pcpu_chunk_addr(static_chunk, 0, 0); 1224 pcpu_base_addr = (void *)pcpu_chunk_addr(schunk, 0, 0);
978 return pcpu_unit_size; 1225 return pcpu_unit_size;
979} 1226}
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-08 13:01:01 -0500