aboutsummaryrefslogtreecommitdiffstats
path: root/include
diff options
context:
space:
mode:
Diffstat (limited to 'include')
-rw-r--r--include/linux/percpu.h60
1 files changed, 27 insertions, 33 deletions
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 545b068bcb70..54a968b4b924 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -5,6 +5,7 @@
5#include <linux/slab.h> /* For kmalloc() */ 5#include <linux/slab.h> /* For kmalloc() */
6#include <linux/smp.h> 6#include <linux/smp.h>
7#include <linux/cpumask.h> 7#include <linux/cpumask.h>
8#include <linux/pfn.h>
8 9
9#include <asm/percpu.h> 10#include <asm/percpu.h>
10 11
@@ -52,17 +53,18 @@
52#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) 53#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
53#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) 54#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
54 55
55/* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */ 56/* enough to cover all DEFINE_PER_CPUs in modules */
56#ifndef PERCPU_ENOUGH_ROOM
57#ifdef CONFIG_MODULES 57#ifdef CONFIG_MODULES
58#define PERCPU_MODULE_RESERVE 8192 58#define PERCPU_MODULE_RESERVE (8 << 10)
59#else 59#else
60#define PERCPU_MODULE_RESERVE 0 60#define PERCPU_MODULE_RESERVE 0
61#endif 61#endif
62 62
63#ifndef PERCPU_ENOUGH_ROOM
63#define PERCPU_ENOUGH_ROOM \ 64#define PERCPU_ENOUGH_ROOM \
64 (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE) 65 (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \
65#endif /* PERCPU_ENOUGH_ROOM */ 66 PERCPU_MODULE_RESERVE)
67#endif
66 68
67/* 69/*
68 * Must be an lvalue. Since @var must be a simple identifier, 70 * Must be an lvalue. Since @var must be a simple identifier,
@@ -79,35 +81,24 @@
79#ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA 81#ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
80 82
81/* minimum unit size, also is the maximum supported allocation size */ 83/* minimum unit size, also is the maximum supported allocation size */
82#define PCPU_MIN_UNIT_SIZE (16UL << PAGE_SHIFT) 84#define PCPU_MIN_UNIT_SIZE PFN_ALIGN(64 << 10)
83 85
84/* 86/*
85 * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy 87 * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy
86 * back on the first chunk if arch is manually allocating and mapping 88 * back on the first chunk for dynamic percpu allocation if arch is
87 * it for faster access (as a part of large page mapping for example). 89 * manually allocating and mapping it for faster access (as a part of
88 * Note that dynamic percpu allocator covers both static and dynamic 90 * large page mapping for example).
89 * areas, so these values are bigger than PERCPU_MODULE_RESERVE.
90 * 91 *
91 * On typical configuration with modules, the following values leave 92 * The following values give between one and two pages of free space
92 * about 8k of free space on the first chunk after boot on both x86_32 93 * after typical minimal boot (2-way SMP, single disk and NIC) with
93 * and 64 when module support is enabled. When module support is 94 * both defconfig and a distro config on x86_64 and 32. More
94 * disabled, it's much tighter. 95 * intelligent way to determine this would be nice.
95 */ 96 */
96#ifndef PERCPU_DYNAMIC_RESERVE 97#if BITS_PER_LONG > 32
97# if BITS_PER_LONG > 32 98#define PERCPU_DYNAMIC_RESERVE (20 << 10)
98# ifdef CONFIG_MODULES 99#else
99# define PERCPU_DYNAMIC_RESERVE (6 << PAGE_SHIFT) 100#define PERCPU_DYNAMIC_RESERVE (12 << 10)
100# else 101#endif
101# define PERCPU_DYNAMIC_RESERVE (4 << PAGE_SHIFT)
102# endif
103# else
104# ifdef CONFIG_MODULES
105# define PERCPU_DYNAMIC_RESERVE (4 << PAGE_SHIFT)
106# else
107# define PERCPU_DYNAMIC_RESERVE (2 << PAGE_SHIFT)
108# endif
109# endif
110#endif /* PERCPU_DYNAMIC_RESERVE */
111 102
112extern void *pcpu_base_addr; 103extern void *pcpu_base_addr;
113 104
@@ -115,9 +106,10 @@ typedef struct page * (*pcpu_get_page_fn_t)(unsigned int cpu, int pageno);
115typedef void (*pcpu_populate_pte_fn_t)(unsigned long addr); 106typedef void (*pcpu_populate_pte_fn_t)(unsigned long addr);
116 107
117extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, 108extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
118 size_t static_size, size_t unit_size, 109 size_t static_size, size_t reserved_size,
119 size_t free_size, void *base_addr, 110 ssize_t unit_size, ssize_t dyn_size,
120 pcpu_populate_pte_fn_t populate_pte_fn); 111 void *base_addr,
112 pcpu_populate_pte_fn_t populate_pte_fn);
121 113
122/* 114/*
123 * Use this to get to a cpu's version of the per-cpu object 115 * Use this to get to a cpu's version of the per-cpu object
@@ -126,6 +118,8 @@ extern size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn,
126 */ 118 */
127#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))) 119#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
128 120
121extern void *__alloc_reserved_percpu(size_t size, size_t align);
122
129#else /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */ 123#else /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
130 124
131struct percpu_data { 125struct percpu_data {