diff options
author | Ingo Molnar <mingo@elte.hu> | 2009-03-14 04:50:10 -0400 |
---|---|---|
committer | Ingo Molnar <mingo@elte.hu> | 2009-03-14 04:50:10 -0400 |
commit | c550033ced484d8d333bc1edc0a482728680e689 (patch) | |
tree | f5e10a2879a2a446be8d4aefb385cd6845c19c72 /arch/x86 | |
parent | a98fe7f3425c6b4e90de16f8da63b0429a8fed08 (diff) | |
parent | 7a46c594bf7f1f2eeb1e12d4b857d5f581957a92 (diff) |
Merge branch 'core/percpu' into x86/core
Diffstat (limited to 'arch/x86')
-rw-r--r-- | arch/x86/include/asm/percpu.h | 8 | ||||
-rw-r--r-- | arch/x86/kernel/setup_percpu.c | 63 |
2 files changed, 10 insertions, 61 deletions
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 8f1d2fbec1d4..aee103b26d01 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h | |||
@@ -43,14 +43,6 @@ | |||
43 | #else /* ...!ASSEMBLY */ | 43 | #else /* ...!ASSEMBLY */ |
44 | 44 | ||
45 | #include <linux/stringify.h> | 45 | #include <linux/stringify.h> |
46 | #include <asm/sections.h> | ||
47 | |||
48 | #define __addr_to_pcpu_ptr(addr) \ | ||
49 | (void *)((unsigned long)(addr) - (unsigned long)pcpu_base_addr \ | ||
50 | + (unsigned long)__per_cpu_start) | ||
51 | #define __pcpu_ptr_to_addr(ptr) \ | ||
52 | (void *)((unsigned long)(ptr) + (unsigned long)pcpu_base_addr \ | ||
53 | - (unsigned long)__per_cpu_start) | ||
54 | 46 | ||
55 | #ifdef CONFIG_SMP | 47 | #ifdef CONFIG_SMP |
56 | #define __percpu_arg(x) "%%"__stringify(__percpu_seg)":%P" #x | 48 | #define __percpu_arg(x) "%%"__stringify(__percpu_seg)":%P" #x |
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index efa615f2bf43..400331b50a53 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c | |||
@@ -233,8 +233,8 @@ proceed: | |||
233 | "%zu bytes\n", vm.addr, static_size); | 233 | "%zu bytes\n", vm.addr, static_size); |
234 | 234 | ||
235 | ret = pcpu_setup_first_chunk(pcpur_get_page, static_size, | 235 | ret = pcpu_setup_first_chunk(pcpur_get_page, static_size, |
236 | PERCPU_FIRST_CHUNK_RESERVE, | 236 | PERCPU_FIRST_CHUNK_RESERVE, dyn_size, |
237 | PMD_SIZE, dyn_size, vm.addr, NULL); | 237 | PMD_SIZE, vm.addr, NULL); |
238 | goto out_free_ar; | 238 | goto out_free_ar; |
239 | 239 | ||
240 | enomem: | 240 | enomem: |
@@ -257,31 +257,13 @@ static ssize_t __init setup_pcpu_remap(size_t static_size) | |||
257 | * Embedding allocator | 257 | * Embedding allocator |
258 | * | 258 | * |
259 | * The first chunk is sized to just contain the static area plus | 259 | * The first chunk is sized to just contain the static area plus |
260 | * module and dynamic reserves, and allocated as a contiguous area | 260 | * module and dynamic reserves and embedded into linear physical |
261 | * using bootmem allocator and used as-is without being mapped into | 261 | * mapping so that it can use PMD mapping without additional TLB |
262 | * vmalloc area. This enables the first chunk to piggy back on the | 262 | * pressure. |
263 | * linear physical PMD mapping and doesn't add any additional pressure | ||
264 | * to TLB. Note that if the needed size is smaller than the minimum | ||
265 | * unit size, the leftover is returned to the bootmem allocator. | ||
266 | */ | 263 | */ |
267 | static void *pcpue_ptr __initdata; | ||
268 | static size_t pcpue_size __initdata; | ||
269 | static size_t pcpue_unit_size __initdata; | ||
270 | |||
271 | static struct page * __init pcpue_get_page(unsigned int cpu, int pageno) | ||
272 | { | ||
273 | size_t off = (size_t)pageno << PAGE_SHIFT; | ||
274 | |||
275 | if (off >= pcpue_size) | ||
276 | return NULL; | ||
277 | |||
278 | return virt_to_page(pcpue_ptr + cpu * pcpue_unit_size + off); | ||
279 | } | ||
280 | |||
281 | static ssize_t __init setup_pcpu_embed(size_t static_size) | 264 | static ssize_t __init setup_pcpu_embed(size_t static_size) |
282 | { | 265 | { |
283 | unsigned int cpu; | 266 | size_t reserve = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE; |
284 | size_t dyn_size; | ||
285 | 267 | ||
286 | /* | 268 | /* |
287 | * If large page isn't supported, there's no benefit in doing | 269 | * If large page isn't supported, there's no benefit in doing |
@@ -291,33 +273,8 @@ static ssize_t __init setup_pcpu_embed(size_t static_size) | |||
291 | if (!cpu_has_pse || pcpu_need_numa()) | 273 | if (!cpu_has_pse || pcpu_need_numa()) |
292 | return -EINVAL; | 274 | return -EINVAL; |
293 | 275 | ||
294 | /* allocate and copy */ | 276 | return pcpu_embed_first_chunk(static_size, PERCPU_FIRST_CHUNK_RESERVE, |
295 | pcpue_size = PFN_ALIGN(static_size + PERCPU_MODULE_RESERVE + | 277 | reserve - PERCPU_FIRST_CHUNK_RESERVE, -1); |
296 | PERCPU_DYNAMIC_RESERVE); | ||
297 | pcpue_unit_size = max_t(size_t, pcpue_size, PCPU_MIN_UNIT_SIZE); | ||
298 | dyn_size = pcpue_size - static_size - PERCPU_FIRST_CHUNK_RESERVE; | ||
299 | |||
300 | pcpue_ptr = pcpu_alloc_bootmem(0, num_possible_cpus() * pcpue_unit_size, | ||
301 | PAGE_SIZE); | ||
302 | if (!pcpue_ptr) | ||
303 | return -ENOMEM; | ||
304 | |||
305 | for_each_possible_cpu(cpu) { | ||
306 | void *ptr = pcpue_ptr + cpu * pcpue_unit_size; | ||
307 | |||
308 | free_bootmem(__pa(ptr + pcpue_size), | ||
309 | pcpue_unit_size - pcpue_size); | ||
310 | memcpy(ptr, __per_cpu_load, static_size); | ||
311 | } | ||
312 | |||
313 | /* we're ready, commit */ | ||
314 | pr_info("PERCPU: Embedded %zu pages at %p, static data %zu bytes\n", | ||
315 | pcpue_size >> PAGE_SHIFT, pcpue_ptr, static_size); | ||
316 | |||
317 | return pcpu_setup_first_chunk(pcpue_get_page, static_size, | ||
318 | PERCPU_FIRST_CHUNK_RESERVE, | ||
319 | pcpue_unit_size, dyn_size, | ||
320 | pcpue_ptr, NULL); | ||
321 | } | 278 | } |
322 | 279 | ||
323 | /* | 280 | /* |
@@ -375,8 +332,8 @@ static ssize_t __init setup_pcpu_4k(size_t static_size) | |||
375 | pcpu4k_nr_static_pages, static_size); | 332 | pcpu4k_nr_static_pages, static_size); |
376 | 333 | ||
377 | ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, | 334 | ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, |
378 | PERCPU_FIRST_CHUNK_RESERVE, -1, -1, NULL, | 335 | PERCPU_FIRST_CHUNK_RESERVE, -1, |
379 | pcpu4k_populate_pte); | 336 | -1, NULL, pcpu4k_populate_pte); |
380 | goto out_free_ar; | 337 | goto out_free_ar; |
381 | 338 | ||
382 | enomem: | 339 | enomem: |