diff options
Diffstat (limited to 'arch/x86/kernel/setup_percpu.c')
| -rw-r--r-- | arch/x86/kernel/setup_percpu.c | 63 |
1 files changed, 10 insertions, 53 deletions
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index efa615f2bf43..400331b50a53 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c | |||
| @@ -233,8 +233,8 @@ proceed: | |||
| 233 | "%zu bytes\n", vm.addr, static_size); | 233 | "%zu bytes\n", vm.addr, static_size); |
| 234 | 234 | ||
| 235 | ret = pcpu_setup_first_chunk(pcpur_get_page, static_size, | 235 | ret = pcpu_setup_first_chunk(pcpur_get_page, static_size, |
| 236 | PERCPU_FIRST_CHUNK_RESERVE, | 236 | PERCPU_FIRST_CHUNK_RESERVE, dyn_size, |
| 237 | PMD_SIZE, dyn_size, vm.addr, NULL); | 237 | PMD_SIZE, vm.addr, NULL); |
| 238 | goto out_free_ar; | 238 | goto out_free_ar; |
| 239 | 239 | ||
| 240 | enomem: | 240 | enomem: |
| @@ -257,31 +257,13 @@ static ssize_t __init setup_pcpu_remap(size_t static_size) | |||
| 257 | * Embedding allocator | 257 | * Embedding allocator |
| 258 | * | 258 | * |
| 259 | * The first chunk is sized to just contain the static area plus | 259 | * The first chunk is sized to just contain the static area plus |
| 260 | * module and dynamic reserves, and allocated as a contiguous area | 260 | * module and dynamic reserves and embedded into linear physical |
| 261 | * using bootmem allocator and used as-is without being mapped into | 261 | * mapping so that it can use PMD mapping without additional TLB |
| 262 | * vmalloc area. This enables the first chunk to piggy back on the | 262 | * pressure. |
| 263 | * linear physical PMD mapping and doesn't add any additional pressure | ||
| 264 | * to TLB. Note that if the needed size is smaller than the minimum | ||
| 265 | * unit size, the leftover is returned to the bootmem allocator. | ||
| 266 | */ | 263 | */ |
| 267 | static void *pcpue_ptr __initdata; | ||
| 268 | static size_t pcpue_size __initdata; | ||
| 269 | static size_t pcpue_unit_size __initdata; | ||
| 270 | |||
| 271 | static struct page * __init pcpue_get_page(unsigned int cpu, int pageno) | ||
| 272 | { | ||
| 273 | size_t off = (size_t)pageno << PAGE_SHIFT; | ||
| 274 | |||
| 275 | if (off >= pcpue_size) | ||
| 276 | return NULL; | ||
| 277 | |||
| 278 | return virt_to_page(pcpue_ptr + cpu * pcpue_unit_size + off); | ||
| 279 | } | ||
| 280 | |||
| 281 | static ssize_t __init setup_pcpu_embed(size_t static_size) | 264 | static ssize_t __init setup_pcpu_embed(size_t static_size) |
| 282 | { | 265 | { |
| 283 | unsigned int cpu; | 266 | size_t reserve = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE; |
| 284 | size_t dyn_size; | ||
| 285 | 267 | ||
| 286 | /* | 268 | /* |
| 287 | * If large page isn't supported, there's no benefit in doing | 269 | * If large page isn't supported, there's no benefit in doing |
| @@ -291,33 +273,8 @@ static ssize_t __init setup_pcpu_embed(size_t static_size) | |||
| 291 | if (!cpu_has_pse || pcpu_need_numa()) | 273 | if (!cpu_has_pse || pcpu_need_numa()) |
| 292 | return -EINVAL; | 274 | return -EINVAL; |
| 293 | 275 | ||
| 294 | /* allocate and copy */ | 276 | return pcpu_embed_first_chunk(static_size, PERCPU_FIRST_CHUNK_RESERVE, |
| 295 | pcpue_size = PFN_ALIGN(static_size + PERCPU_MODULE_RESERVE + | 277 | reserve - PERCPU_FIRST_CHUNK_RESERVE, -1); |
| 296 | PERCPU_DYNAMIC_RESERVE); | ||
| 297 | pcpue_unit_size = max_t(size_t, pcpue_size, PCPU_MIN_UNIT_SIZE); | ||
| 298 | dyn_size = pcpue_size - static_size - PERCPU_FIRST_CHUNK_RESERVE; | ||
| 299 | |||
| 300 | pcpue_ptr = pcpu_alloc_bootmem(0, num_possible_cpus() * pcpue_unit_size, | ||
| 301 | PAGE_SIZE); | ||
| 302 | if (!pcpue_ptr) | ||
| 303 | return -ENOMEM; | ||
| 304 | |||
| 305 | for_each_possible_cpu(cpu) { | ||
| 306 | void *ptr = pcpue_ptr + cpu * pcpue_unit_size; | ||
| 307 | |||
| 308 | free_bootmem(__pa(ptr + pcpue_size), | ||
| 309 | pcpue_unit_size - pcpue_size); | ||
| 310 | memcpy(ptr, __per_cpu_load, static_size); | ||
| 311 | } | ||
| 312 | |||
| 313 | /* we're ready, commit */ | ||
| 314 | pr_info("PERCPU: Embedded %zu pages at %p, static data %zu bytes\n", | ||
| 315 | pcpue_size >> PAGE_SHIFT, pcpue_ptr, static_size); | ||
| 316 | |||
| 317 | return pcpu_setup_first_chunk(pcpue_get_page, static_size, | ||
| 318 | PERCPU_FIRST_CHUNK_RESERVE, | ||
| 319 | pcpue_unit_size, dyn_size, | ||
| 320 | pcpue_ptr, NULL); | ||
| 321 | } | 278 | } |
| 322 | 279 | ||
| 323 | /* | 280 | /* |
| @@ -375,8 +332,8 @@ static ssize_t __init setup_pcpu_4k(size_t static_size) | |||
| 375 | pcpu4k_nr_static_pages, static_size); | 332 | pcpu4k_nr_static_pages, static_size); |
| 376 | 333 | ||
| 377 | ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, | 334 | ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, |
| 378 | PERCPU_FIRST_CHUNK_RESERVE, -1, -1, NULL, | 335 | PERCPU_FIRST_CHUNK_RESERVE, -1, |
| 379 | pcpu4k_populate_pte); | 336 | -1, NULL, pcpu4k_populate_pte); |
| 380 | goto out_free_ar; | 337 | goto out_free_ar; |
| 381 | 338 | ||
| 382 | enomem: | 339 | enomem: |
