diff options
-rw-r--r-- | arch/x86/kernel/setup_percpu.c | 13 | ||||
-rw-r--r-- | include/linux/percpu.h | 2 | ||||
-rw-r--r-- | mm/percpu.c | 28 |
3 files changed, 21 insertions, 22 deletions
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index efa615f2bf43..e41c51f6ada1 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c | |||
@@ -233,8 +233,8 @@ proceed: | |||
233 | "%zu bytes\n", vm.addr, static_size); | 233 | "%zu bytes\n", vm.addr, static_size); |
234 | 234 | ||
235 | ret = pcpu_setup_first_chunk(pcpur_get_page, static_size, | 235 | ret = pcpu_setup_first_chunk(pcpur_get_page, static_size, |
236 | PERCPU_FIRST_CHUNK_RESERVE, | 236 | PERCPU_FIRST_CHUNK_RESERVE, dyn_size, |
237 | PMD_SIZE, dyn_size, vm.addr, NULL); | 237 | PMD_SIZE, vm.addr, NULL); |
238 | goto out_free_ar; | 238 | goto out_free_ar; |
239 | 239 | ||
240 | enomem: | 240 | enomem: |
@@ -315,9 +315,8 @@ static ssize_t __init setup_pcpu_embed(size_t static_size) | |||
315 | pcpue_size >> PAGE_SHIFT, pcpue_ptr, static_size); | 315 | pcpue_size >> PAGE_SHIFT, pcpue_ptr, static_size); |
316 | 316 | ||
317 | return pcpu_setup_first_chunk(pcpue_get_page, static_size, | 317 | return pcpu_setup_first_chunk(pcpue_get_page, static_size, |
318 | PERCPU_FIRST_CHUNK_RESERVE, | 318 | PERCPU_FIRST_CHUNK_RESERVE, dyn_size, |
319 | pcpue_unit_size, dyn_size, | 319 | pcpue_unit_size, pcpue_ptr, NULL); |
320 | pcpue_ptr, NULL); | ||
321 | } | 320 | } |
322 | 321 | ||
323 | /* | 322 | /* |
@@ -375,8 +374,8 @@ static ssize_t __init setup_pcpu_4k(size_t static_size) | |||
375 | pcpu4k_nr_static_pages, static_size); | 374 | pcpu4k_nr_static_pages, static_size); |
376 | 375 | ||
377 | ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, | 376 | ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, |
378 | PERCPU_FIRST_CHUNK_RESERVE, -1, -1, NULL, | 377 | PERCPU_FIRST_CHUNK_RESERVE, -1, |
379 | pcpu4k_populate_pte); | 378 | -1, NULL, pcpu4k_populate_pte); |
380 | goto out_free_ar; | 379 | goto out_free_ar; |
381 | 380 | ||
382 | enomem: | 381 | enomem: |
diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 54a968b4b924..fb455dcc59c7 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h | |||
@@ -107,7 +107,7 @@ typedef void (*pcpu_populate_pte_fn_t)(unsigned long addr); | |||
107 | 107 | ||
108 | extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, | 108 | extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, |
109 | size_t static_size, size_t reserved_size, | 109 | size_t static_size, size_t reserved_size, |
110 | ssize_t unit_size, ssize_t dyn_size, | 110 | ssize_t dyn_size, ssize_t unit_size, |
111 | void *base_addr, | 111 | void *base_addr, |
112 | pcpu_populate_pte_fn_t populate_pte_fn); | 112 | pcpu_populate_pte_fn_t populate_pte_fn); |
113 | 113 | ||
diff --git a/mm/percpu.c b/mm/percpu.c index c6f38a2aface..2f94661d3e36 100644 --- a/mm/percpu.c +++ b/mm/percpu.c | |||
@@ -1027,8 +1027,8 @@ EXPORT_SYMBOL_GPL(free_percpu); | |||
1027 | * @get_page_fn: callback to fetch page pointer | 1027 | * @get_page_fn: callback to fetch page pointer |
1028 | * @static_size: the size of static percpu area in bytes | 1028 | * @static_size: the size of static percpu area in bytes |
1029 | * @reserved_size: the size of reserved percpu area in bytes | 1029 | * @reserved_size: the size of reserved percpu area in bytes |
1030 | * @unit_size: unit size in bytes, must be multiple of PAGE_SIZE, -1 for auto | ||
1031 | * @dyn_size: free size for dynamic allocation in bytes, -1 for auto | 1030 | * @dyn_size: free size for dynamic allocation in bytes, -1 for auto |
1031 | * @unit_size: unit size in bytes, must be multiple of PAGE_SIZE, -1 for auto | ||
1032 | * @base_addr: mapped address, NULL for auto | 1032 | * @base_addr: mapped address, NULL for auto |
1033 | * @populate_pte_fn: callback to allocate pagetable, NULL if unnecessary | 1033 | * @populate_pte_fn: callback to allocate pagetable, NULL if unnecessary |
1034 | * | 1034 | * |
@@ -1053,14 +1053,14 @@ EXPORT_SYMBOL_GPL(free_percpu); | |||
1053 | * limited offset range for symbol relocations to guarantee module | 1053 | * limited offset range for symbol relocations to guarantee module |
1054 | * percpu symbols fall inside the relocatable range. | 1054 | * percpu symbols fall inside the relocatable range. |
1055 | * | 1055 | * |
1056 | * @dyn_size, if non-negative, determines the number of bytes | ||
1057 | * available for dynamic allocation in the first chunk. Specifying | ||
1058 | * non-negative value makes percpu leave alone the area beyond | ||
1059 | * @static_size + @reserved_size + @dyn_size. | ||
1060 | * | ||
1056 | * @unit_size, if non-negative, specifies unit size and must be | 1061 | * @unit_size, if non-negative, specifies unit size and must be |
1057 | * aligned to PAGE_SIZE and equal to or larger than @static_size + | 1062 | * aligned to PAGE_SIZE and equal to or larger than @static_size + |
1058 | * @reserved_size + @dyn_size. | 1063 | * @reserved_size + if non-negative, @dyn_size. |
1059 | * | ||
1060 | * @dyn_size, if non-negative, limits the number of bytes available | ||
1061 | * for dynamic allocation in the first chunk. Specifying non-negative | ||
1062 | * value make percpu leave alone the area beyond @static_size + | ||
1063 | * @reserved_size + @dyn_size. | ||
1064 | * | 1064 | * |
1065 | * Non-null @base_addr means that the caller already allocated virtual | 1065 | * Non-null @base_addr means that the caller already allocated virtual |
1066 | * region for the first chunk and mapped it. percpu must not mess | 1066 | * region for the first chunk and mapped it. percpu must not mess |
@@ -1083,12 +1083,14 @@ EXPORT_SYMBOL_GPL(free_percpu); | |||
1083 | */ | 1083 | */ |
1084 | size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, | 1084 | size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, |
1085 | size_t static_size, size_t reserved_size, | 1085 | size_t static_size, size_t reserved_size, |
1086 | ssize_t unit_size, ssize_t dyn_size, | 1086 | ssize_t dyn_size, ssize_t unit_size, |
1087 | void *base_addr, | 1087 | void *base_addr, |
1088 | pcpu_populate_pte_fn_t populate_pte_fn) | 1088 | pcpu_populate_pte_fn_t populate_pte_fn) |
1089 | { | 1089 | { |
1090 | static struct vm_struct first_vm; | 1090 | static struct vm_struct first_vm; |
1091 | static int smap[2], dmap[2]; | 1091 | static int smap[2], dmap[2]; |
1092 | size_t size_sum = static_size + reserved_size + | ||
1093 | (dyn_size >= 0 ? dyn_size : 0); | ||
1092 | struct pcpu_chunk *schunk, *dchunk = NULL; | 1094 | struct pcpu_chunk *schunk, *dchunk = NULL; |
1093 | unsigned int cpu; | 1095 | unsigned int cpu; |
1094 | int nr_pages; | 1096 | int nr_pages; |
@@ -1099,20 +1101,18 @@ size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, | |||
1099 | ARRAY_SIZE(dmap) >= PCPU_DFL_MAP_ALLOC); | 1101 | ARRAY_SIZE(dmap) >= PCPU_DFL_MAP_ALLOC); |
1100 | BUG_ON(!static_size); | 1102 | BUG_ON(!static_size); |
1101 | if (unit_size >= 0) { | 1103 | if (unit_size >= 0) { |
1102 | BUG_ON(unit_size < static_size + reserved_size + | 1104 | BUG_ON(unit_size < size_sum); |
1103 | (dyn_size >= 0 ? dyn_size : 0)); | ||
1104 | BUG_ON(unit_size & ~PAGE_MASK); | 1105 | BUG_ON(unit_size & ~PAGE_MASK); |
1105 | } else { | 1106 | BUG_ON(unit_size < PCPU_MIN_UNIT_SIZE); |
1106 | BUG_ON(dyn_size >= 0); | 1107 | } else |
1107 | BUG_ON(base_addr); | 1108 | BUG_ON(base_addr); |
1108 | } | ||
1109 | BUG_ON(base_addr && populate_pte_fn); | 1109 | BUG_ON(base_addr && populate_pte_fn); |
1110 | 1110 | ||
1111 | if (unit_size >= 0) | 1111 | if (unit_size >= 0) |
1112 | pcpu_unit_pages = unit_size >> PAGE_SHIFT; | 1112 | pcpu_unit_pages = unit_size >> PAGE_SHIFT; |
1113 | else | 1113 | else |
1114 | pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_SIZE >> PAGE_SHIFT, | 1114 | pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_SIZE >> PAGE_SHIFT, |
1115 | PFN_UP(static_size + reserved_size)); | 1115 | PFN_UP(size_sum)); |
1116 | 1116 | ||
1117 | pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT; | 1117 | pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT; |
1118 | pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size; | 1118 | pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size; |