aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/x86/kernel/setup_percpu.c13
-rw-r--r--include/linux/percpu.h2
-rw-r--r--mm/percpu.c28
3 files changed, 21 insertions, 22 deletions
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index efa615f2bf43..e41c51f6ada1 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -233,8 +233,8 @@ proceed:
233 "%zu bytes\n", vm.addr, static_size); 233 "%zu bytes\n", vm.addr, static_size);
234 234
235 ret = pcpu_setup_first_chunk(pcpur_get_page, static_size, 235 ret = pcpu_setup_first_chunk(pcpur_get_page, static_size,
236 PERCPU_FIRST_CHUNK_RESERVE, 236 PERCPU_FIRST_CHUNK_RESERVE, dyn_size,
237 PMD_SIZE, dyn_size, vm.addr, NULL); 237 PMD_SIZE, vm.addr, NULL);
238 goto out_free_ar; 238 goto out_free_ar;
239 239
240enomem: 240enomem:
@@ -315,9 +315,8 @@ static ssize_t __init setup_pcpu_embed(size_t static_size)
315 pcpue_size >> PAGE_SHIFT, pcpue_ptr, static_size); 315 pcpue_size >> PAGE_SHIFT, pcpue_ptr, static_size);
316 316
317 return pcpu_setup_first_chunk(pcpue_get_page, static_size, 317 return pcpu_setup_first_chunk(pcpue_get_page, static_size,
318 PERCPU_FIRST_CHUNK_RESERVE, 318 PERCPU_FIRST_CHUNK_RESERVE, dyn_size,
319 pcpue_unit_size, dyn_size, 319 pcpue_unit_size, pcpue_ptr, NULL);
320 pcpue_ptr, NULL);
321} 320}
322 321
323/* 322/*
@@ -375,8 +374,8 @@ static ssize_t __init setup_pcpu_4k(size_t static_size)
375 pcpu4k_nr_static_pages, static_size); 374 pcpu4k_nr_static_pages, static_size);
376 375
377 ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size, 376 ret = pcpu_setup_first_chunk(pcpu4k_get_page, static_size,
378 PERCPU_FIRST_CHUNK_RESERVE, -1, -1, NULL, 377 PERCPU_FIRST_CHUNK_RESERVE, -1,
379 pcpu4k_populate_pte); 378 -1, NULL, pcpu4k_populate_pte);
380 goto out_free_ar; 379 goto out_free_ar;
381 380
382enomem: 381enomem:
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 54a968b4b924..fb455dcc59c7 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -107,7 +107,7 @@ typedef void (*pcpu_populate_pte_fn_t)(unsigned long addr);
107 107
108extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, 108extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
109 size_t static_size, size_t reserved_size, 109 size_t static_size, size_t reserved_size,
110 ssize_t unit_size, ssize_t dyn_size, 110 ssize_t dyn_size, ssize_t unit_size,
111 void *base_addr, 111 void *base_addr,
112 pcpu_populate_pte_fn_t populate_pte_fn); 112 pcpu_populate_pte_fn_t populate_pte_fn);
113 113
diff --git a/mm/percpu.c b/mm/percpu.c
index c6f38a2aface..2f94661d3e36 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1027,8 +1027,8 @@ EXPORT_SYMBOL_GPL(free_percpu);
1027 * @get_page_fn: callback to fetch page pointer 1027 * @get_page_fn: callback to fetch page pointer
1028 * @static_size: the size of static percpu area in bytes 1028 * @static_size: the size of static percpu area in bytes
1029 * @reserved_size: the size of reserved percpu area in bytes 1029 * @reserved_size: the size of reserved percpu area in bytes
1030 * @unit_size: unit size in bytes, must be multiple of PAGE_SIZE, -1 for auto
1031 * @dyn_size: free size for dynamic allocation in bytes, -1 for auto 1030 * @dyn_size: free size for dynamic allocation in bytes, -1 for auto
1031 * @unit_size: unit size in bytes, must be multiple of PAGE_SIZE, -1 for auto
1032 * @base_addr: mapped address, NULL for auto 1032 * @base_addr: mapped address, NULL for auto
1033 * @populate_pte_fn: callback to allocate pagetable, NULL if unnecessary 1033 * @populate_pte_fn: callback to allocate pagetable, NULL if unnecessary
1034 * 1034 *
@@ -1053,14 +1053,14 @@ EXPORT_SYMBOL_GPL(free_percpu);
1053 * limited offset range for symbol relocations to guarantee module 1053 * limited offset range for symbol relocations to guarantee module
1054 * percpu symbols fall inside the relocatable range. 1054 * percpu symbols fall inside the relocatable range.
1055 * 1055 *
1056 * @dyn_size, if non-negative, determines the number of bytes
1057 * available for dynamic allocation in the first chunk. Specifying
1058 * non-negative value makes percpu leave alone the area beyond
1059 * @static_size + @reserved_size + @dyn_size.
1060 *
1056 * @unit_size, if non-negative, specifies unit size and must be 1061 * @unit_size, if non-negative, specifies unit size and must be
1057 * aligned to PAGE_SIZE and equal to or larger than @static_size + 1062 * aligned to PAGE_SIZE and equal to or larger than @static_size +
1058 * @reserved_size + @dyn_size. 1063 * @reserved_size + if non-negative, @dyn_size.
1059 *
1060 * @dyn_size, if non-negative, limits the number of bytes available
1061 * for dynamic allocation in the first chunk. Specifying non-negative
1062 * value make percpu leave alone the area beyond @static_size +
1063 * @reserved_size + @dyn_size.
1064 * 1064 *
1065 * Non-null @base_addr means that the caller already allocated virtual 1065 * Non-null @base_addr means that the caller already allocated virtual
1066 * region for the first chunk and mapped it. percpu must not mess 1066 * region for the first chunk and mapped it. percpu must not mess
@@ -1083,12 +1083,14 @@ EXPORT_SYMBOL_GPL(free_percpu);
1083 */ 1083 */
1084size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, 1084size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
1085 size_t static_size, size_t reserved_size, 1085 size_t static_size, size_t reserved_size,
1086 ssize_t unit_size, ssize_t dyn_size, 1086 ssize_t dyn_size, ssize_t unit_size,
1087 void *base_addr, 1087 void *base_addr,
1088 pcpu_populate_pte_fn_t populate_pte_fn) 1088 pcpu_populate_pte_fn_t populate_pte_fn)
1089{ 1089{
1090 static struct vm_struct first_vm; 1090 static struct vm_struct first_vm;
1091 static int smap[2], dmap[2]; 1091 static int smap[2], dmap[2];
1092 size_t size_sum = static_size + reserved_size +
1093 (dyn_size >= 0 ? dyn_size : 0);
1092 struct pcpu_chunk *schunk, *dchunk = NULL; 1094 struct pcpu_chunk *schunk, *dchunk = NULL;
1093 unsigned int cpu; 1095 unsigned int cpu;
1094 int nr_pages; 1096 int nr_pages;
@@ -1099,20 +1101,18 @@ size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
1099 ARRAY_SIZE(dmap) >= PCPU_DFL_MAP_ALLOC); 1101 ARRAY_SIZE(dmap) >= PCPU_DFL_MAP_ALLOC);
1100 BUG_ON(!static_size); 1102 BUG_ON(!static_size);
1101 if (unit_size >= 0) { 1103 if (unit_size >= 0) {
1102 BUG_ON(unit_size < static_size + reserved_size + 1104 BUG_ON(unit_size < size_sum);
1103 (dyn_size >= 0 ? dyn_size : 0));
1104 BUG_ON(unit_size & ~PAGE_MASK); 1105 BUG_ON(unit_size & ~PAGE_MASK);
1105 } else { 1106 BUG_ON(unit_size < PCPU_MIN_UNIT_SIZE);
1106 BUG_ON(dyn_size >= 0); 1107 } else
1107 BUG_ON(base_addr); 1108 BUG_ON(base_addr);
1108 }
1109 BUG_ON(base_addr && populate_pte_fn); 1109 BUG_ON(base_addr && populate_pte_fn);
1110 1110
1111 if (unit_size >= 0) 1111 if (unit_size >= 0)
1112 pcpu_unit_pages = unit_size >> PAGE_SHIFT; 1112 pcpu_unit_pages = unit_size >> PAGE_SHIFT;
1113 else 1113 else
1114 pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_SIZE >> PAGE_SHIFT, 1114 pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_SIZE >> PAGE_SHIFT,
1115 PFN_UP(static_size + reserved_size)); 1115 PFN_UP(size_sum));
1116 1116
1117 pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT; 1117 pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT;
1118 pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size; 1118 pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size;