diff options
-rw-r--r-- | mm/percpu-internal.h | 33 | ||||
-rw-r--r-- | mm/percpu.c | 30 |
2 files changed, 40 insertions, 23 deletions
diff --git a/mm/percpu-internal.h b/mm/percpu-internal.h new file mode 100644 index 000000000000..68bf18522a6d --- /dev/null +++ b/mm/percpu-internal.h | |||
@@ -0,0 +1,33 @@ | |||
1 | #ifndef _MM_PERCPU_INTERNAL_H | ||
2 | #define _MM_PERCPU_INTERNAL_H | ||
3 | |||
4 | #include <linux/types.h> | ||
5 | #include <linux/percpu.h> | ||
6 | |||
7 | struct pcpu_chunk { | ||
8 | struct list_head list; /* linked to pcpu_slot lists */ | ||
9 | int free_size; /* free bytes in the chunk */ | ||
10 | int contig_hint; /* max contiguous size hint */ | ||
11 | void *base_addr; /* base address of this chunk */ | ||
12 | |||
13 | int map_used; /* # of map entries used before the sentry */ | ||
14 | int map_alloc; /* # of map entries allocated */ | ||
15 | int *map; /* allocation map */ | ||
16 | struct list_head map_extend_list;/* on pcpu_map_extend_chunks */ | ||
17 | |||
18 | void *data; /* chunk data */ | ||
19 | int first_free; /* no free below this */ | ||
20 | bool immutable; /* no [de]population allowed */ | ||
21 | int nr_populated; /* # of populated pages */ | ||
22 | unsigned long populated[]; /* populated bitmap */ | ||
23 | }; | ||
24 | |||
25 | extern spinlock_t pcpu_lock; | ||
26 | |||
27 | extern struct list_head *pcpu_slot; | ||
28 | extern int pcpu_nr_slots; | ||
29 | |||
30 | extern struct pcpu_chunk *pcpu_first_chunk; | ||
31 | extern struct pcpu_chunk *pcpu_reserved_chunk; | ||
32 | |||
33 | #endif | ||
diff --git a/mm/percpu.c b/mm/percpu.c index be95d3192acb..75ac982c19df 100644 --- a/mm/percpu.c +++ b/mm/percpu.c | |||
@@ -76,6 +76,8 @@ | |||
76 | #include <asm/tlbflush.h> | 76 | #include <asm/tlbflush.h> |
77 | #include <asm/io.h> | 77 | #include <asm/io.h> |
78 | 78 | ||
79 | #include "percpu-internal.h" | ||
80 | |||
79 | #define PCPU_SLOT_BASE_SHIFT 5 /* 1-31 shares the same slot */ | 81 | #define PCPU_SLOT_BASE_SHIFT 5 /* 1-31 shares the same slot */ |
80 | #define PCPU_DFL_MAP_ALLOC 16 /* start a map with 16 ents */ | 82 | #define PCPU_DFL_MAP_ALLOC 16 /* start a map with 16 ents */ |
81 | #define PCPU_ATOMIC_MAP_MARGIN_LOW 32 | 83 | #define PCPU_ATOMIC_MAP_MARGIN_LOW 32 |
@@ -103,29 +105,11 @@ | |||
103 | #define __pcpu_ptr_to_addr(ptr) (void __force *)(ptr) | 105 | #define __pcpu_ptr_to_addr(ptr) (void __force *)(ptr) |
104 | #endif /* CONFIG_SMP */ | 106 | #endif /* CONFIG_SMP */ |
105 | 107 | ||
106 | struct pcpu_chunk { | ||
107 | struct list_head list; /* linked to pcpu_slot lists */ | ||
108 | int free_size; /* free bytes in the chunk */ | ||
109 | int contig_hint; /* max contiguous size hint */ | ||
110 | void *base_addr; /* base address of this chunk */ | ||
111 | |||
112 | int map_used; /* # of map entries used before the sentry */ | ||
113 | int map_alloc; /* # of map entries allocated */ | ||
114 | int *map; /* allocation map */ | ||
115 | struct list_head map_extend_list;/* on pcpu_map_extend_chunks */ | ||
116 | |||
117 | void *data; /* chunk data */ | ||
118 | int first_free; /* no free below this */ | ||
119 | bool immutable; /* no [de]population allowed */ | ||
120 | int nr_populated; /* # of populated pages */ | ||
121 | unsigned long populated[]; /* populated bitmap */ | ||
122 | }; | ||
123 | |||
124 | static int pcpu_unit_pages __ro_after_init; | 108 | static int pcpu_unit_pages __ro_after_init; |
125 | static int pcpu_unit_size __ro_after_init; | 109 | static int pcpu_unit_size __ro_after_init; |
126 | static int pcpu_nr_units __ro_after_init; | 110 | static int pcpu_nr_units __ro_after_init; |
127 | static int pcpu_atom_size __ro_after_init; | 111 | static int pcpu_atom_size __ro_after_init; |
128 | static int pcpu_nr_slots __ro_after_init; | 112 | int pcpu_nr_slots __ro_after_init; |
129 | static size_t pcpu_chunk_struct_size __ro_after_init; | 113 | static size_t pcpu_chunk_struct_size __ro_after_init; |
130 | 114 | ||
131 | /* cpus with the lowest and highest unit addresses */ | 115 | /* cpus with the lowest and highest unit addresses */ |
@@ -149,7 +133,7 @@ static const size_t *pcpu_group_sizes __ro_after_init; | |||
149 | * chunks, this one can be allocated and mapped in several different | 133 | * chunks, this one can be allocated and mapped in several different |
150 | * ways and thus often doesn't live in the vmalloc area. | 134 | * ways and thus often doesn't live in the vmalloc area. |
151 | */ | 135 | */ |
152 | static struct pcpu_chunk *pcpu_first_chunk __ro_after_init; | 136 | struct pcpu_chunk *pcpu_first_chunk __ro_after_init; |
153 | 137 | ||
154 | /* | 138 | /* |
155 | * Optional reserved chunk. This chunk reserves part of the first | 139 | * Optional reserved chunk. This chunk reserves part of the first |
@@ -158,13 +142,13 @@ static struct pcpu_chunk *pcpu_first_chunk __ro_after_init; | |||
158 | * area doesn't exist, the following variables contain NULL and 0 | 142 | * area doesn't exist, the following variables contain NULL and 0 |
159 | * respectively. | 143 | * respectively. |
160 | */ | 144 | */ |
161 | static struct pcpu_chunk *pcpu_reserved_chunk __ro_after_init; | 145 | struct pcpu_chunk *pcpu_reserved_chunk __ro_after_init; |
162 | static int pcpu_reserved_chunk_limit __ro_after_init; | 146 | static int pcpu_reserved_chunk_limit __ro_after_init; |
163 | 147 | ||
164 | static DEFINE_SPINLOCK(pcpu_lock); /* all internal data structures */ | 148 | DEFINE_SPINLOCK(pcpu_lock); /* all internal data structures */ |
165 | static DEFINE_MUTEX(pcpu_alloc_mutex); /* chunk create/destroy, [de]pop, map ext */ | 149 | static DEFINE_MUTEX(pcpu_alloc_mutex); /* chunk create/destroy, [de]pop, map ext */ |
166 | 150 | ||
167 | static struct list_head *pcpu_slot __ro_after_init; /* chunk list slots */ | 151 | struct list_head *pcpu_slot __ro_after_init; /* chunk list slots */ |
168 | 152 | ||
169 | /* chunks which need their map areas extended, protected by pcpu_lock */ | 153 | /* chunks which need their map areas extended, protected by pcpu_lock */ |
170 | static LIST_HEAD(pcpu_map_extend_chunks); | 154 | static LIST_HEAD(pcpu_map_extend_chunks); |