diff options
Diffstat (limited to 'arch/x86/kernel/setup_percpu.c')
-rw-r--r-- | arch/x86/kernel/setup_percpu.c | 63 |
1 files changed, 10 insertions, 53 deletions
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index efa615f2bf43..400331b50a53 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c | |||
@@ -233,8 +233,8 @@ proceed: | |||
233 | "%zu bytes\n", vm.addr, static_size); | 233 | "%zu bytes\n", vm.addr, static_size); |
234 | 234 | ||
235 | ret = pcpu_setup_first_chunk(pcpur_get_page, static_size, | 235 | ret = pcpu_setup_first_chunk(pcpur_get_page, static_size, |
236 | PERCPU_FIRST_CHUNK_RESERVE, | 236 | PERCPU_FIRST_CHUNK_RESERVE, dyn_size, |
237 | PMD_SIZE, dyn_size, vm.addr, NULL); | 237 | PMD_SIZE, vm.addr, NULL); |
238 | goto out_free_ar; | 238 | goto out_free_ar; |
239 | 239 | ||
240 | enomem: | 240 | enomem: |
@@ -257,31 +257,13 @@ static ssize_t __init setup_pcpu_remap(size_t static_size) | |||
257 | * Embedding allocator | 257 | * Embedding allocator |
258 | * | 258 | * |
259 | * The first chunk is sized to just contain the static area plus | 259 | * The first chunk is sized to just contain the static area plus |
260 | * module and dynamic reserves, and allocated as a contiguous area | 260 | * module and dynamic reserves and embedded into linear physical |
261 | * using bootmem allocator and used as-is without being mapped into | 261 | * mapping so that it can use PMD mapping without additional TLB |
262 | * vmalloc area. This enables the first chunk to piggy back on the | 262 | * pressure. |
263 | * linear physical PMD mapping and doesn't add any additional pressure | ||
264 | * to TLB. Note that if the needed size is smaller than the minimum | ||
265 | * unit size, the leftover is returned to the bootmem allocator. | ||
266 | */ | 263 | */ |
267 | static void *pcpue_ptr __initdata; | ||
268 | static size_t pcpue_size __initdata; | ||
269 | static size_t pcpue_unit_size __initdata; | ||
270 | |||
271 | static struct page * __init pcpue_get_page(unsigned int cpu, int pageno) | ||
272 | { | ||
273 | size_t off = (size_t)pageno << PAGE_SHIFT; | ||
274 | |||
275 | if (off >= pcpue_size) | ||
276 | return NULL; | ||
277 | |||
278 | return virt_to_page(pcpue_ptr + cpu * pcpue_unit_size + off); | ||
279 | } | ||
280 | |||
281 | static ssize_t __init setup_pcpu_embed(size_t static_size) | 264 | static ssize_t __init setup_pcpu_embed(size_t static_size) |
282 | { | 265 | { |
283 | unsigned int cpu; | 266 | size_t reserve = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE; |
284 | size_t dyn_size; | ||
285 | 267 | ||
286 | /* | 268 | /* |
287 | * If large page isn't supported, there's no benefit in doing | 269 | * If large page isn't supported, there's no benefit in doing |
@@ -291,33 +273,8 @@ static ssize_t __init setup_pcpu_embed(size_t static_size) | |||
291 | if (!cpu_has_pse || pcpu_need_numa()) | 273 | if (!cpu_has_pse || pcpu_need_numa()) |
292 | return -EINVAL; | 274 | return -EINVAL; |
293 | 275 | ||
294 | /* allocate and copy */ | 276 | return pcpu_embed_first_chunk(static_size, PERCPU_FIRST_CHUNK_RESERVE, |
295 | pcpue_size = PFN_ALIGN(static_size + PERCPU_MODULE_RESERVE + | 277 | reserve - PERCPU_FIRST_CHUNK_RESERVE, -1); |
296 | PERCPU_DYNAMIC_RESERVE); | ||
297 | pcpue_unit_size = max_t(size_t, pcpue_size, PCPU_MIN_UNIT_SIZE); | ||
298 | dyn_size = pcpue_size - static_size - PERCPU_FIRST_CHUNK_RESERVE; | ||
299 | |||
300 | pcpue_ptr = pcpu_alloc_bootmem(0, num_possible_cpus() * pcpue_unit_size, | ||
301 | PAGE_SIZE); | ||
302 | if (!pcpue_ptr) | ||
303 | return -ENOMEM; | ||
304 | |||
305 | for_each_possible_cpu(cpu) { | ||
306 | void *ptr = pcpue_ptr + cpu * pcpue_unit_size; | ||
307 | |||
308 | free_bootmem(__pa(ptr + pcpue_size), | ||
309 | pcpue_unit_size - pcpue_size); | ||
310 | memcpy(ptr, __per_cpu_load, static_size); | ||
311 | } | ||
312 | |||
313 | /* we're ready, commit */ | ||
314 | pr_info("PERCPU: Embedded %zu pages at %p, static data %zu bytes\n", | ||
315 | pcpue_size >> PAGE_SHIFT, pcpue_ptr, static_size); | ||
316 | |||
317 | return pcpu_setup_first_chunk(pcpue_get_page, static_size, | ||
318 | PERCPU_FIRST_CHUNK_RESERVE, | ||
319 | pcpue_unit_size, dyn_size, | ||
320 | pcpue_ptr, NULL); | ||
321 | } | 278 | } |
322 | 279 | ||
323 | /* | 280 | /* |
@@ -375,8 +332,8 @@ static ssize_t __init setup_pcpu_4k(size_t static_size) | |||
375 | pcpu4k_nr_static_pages, static_size); | 332 | pcpu4k_nr_static_pages, static_size); |
376 | 333 | ||
377 | ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, | 334 | ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, |
378 | PERCPU_FIRST_CHUNK_RESERVE, -1, -1, NULL, | 335 | PERCPU_FIRST_CHUNK_RESERVE, -1, |
379 | pcpu4k_populate_pte); | 336 | -1, NULL, pcpu4k_populate_pte); |
380 | goto out_free_ar; | 337 | goto out_free_ar; |
381 | 338 | ||
382 | enomem: | 339 | enomem: |