aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--mm/percpu-internal.h33
-rw-r--r--mm/percpu.c30
2 files changed, 40 insertions, 23 deletions
diff --git a/mm/percpu-internal.h b/mm/percpu-internal.h
new file mode 100644
index 000000000000..68bf18522a6d
--- /dev/null
+++ b/mm/percpu-internal.h
@@ -0,0 +1,33 @@
1#ifndef _MM_PERCPU_INTERNAL_H
2#define _MM_PERCPU_INTERNAL_H
3
4#include <linux/types.h>
5#include <linux/percpu.h>
6
7struct pcpu_chunk {
8 struct list_head list; /* linked to pcpu_slot lists */
9 int free_size; /* free bytes in the chunk */
10 int contig_hint; /* max contiguous size hint */
11 void *base_addr; /* base address of this chunk */
12
13 int map_used; /* # of map entries used before the sentry */
14 int map_alloc; /* # of map entries allocated */
15 int *map; /* allocation map */
16 struct list_head map_extend_list;/* on pcpu_map_extend_chunks */
17
18 void *data; /* chunk data */
19 int first_free; /* no free below this */
20 bool immutable; /* no [de]population allowed */
21 int nr_populated; /* # of populated pages */
22 unsigned long populated[]; /* populated bitmap */
23};
24
25extern spinlock_t pcpu_lock;
26
27extern struct list_head *pcpu_slot;
28extern int pcpu_nr_slots;
29
30extern struct pcpu_chunk *pcpu_first_chunk;
31extern struct pcpu_chunk *pcpu_reserved_chunk;
32
33#endif
diff --git a/mm/percpu.c b/mm/percpu.c
index be95d3192acb..75ac982c19df 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -76,6 +76,8 @@
76#include <asm/tlbflush.h> 76#include <asm/tlbflush.h>
77#include <asm/io.h> 77#include <asm/io.h>
78 78
79#include "percpu-internal.h"
80
79#define PCPU_SLOT_BASE_SHIFT 5 /* 1-31 shares the same slot */ 81#define PCPU_SLOT_BASE_SHIFT 5 /* 1-31 shares the same slot */
80#define PCPU_DFL_MAP_ALLOC 16 /* start a map with 16 ents */ 82#define PCPU_DFL_MAP_ALLOC 16 /* start a map with 16 ents */
81#define PCPU_ATOMIC_MAP_MARGIN_LOW 32 83#define PCPU_ATOMIC_MAP_MARGIN_LOW 32
@@ -103,29 +105,11 @@
103#define __pcpu_ptr_to_addr(ptr) (void __force *)(ptr) 105#define __pcpu_ptr_to_addr(ptr) (void __force *)(ptr)
104#endif /* CONFIG_SMP */ 106#endif /* CONFIG_SMP */
105 107
106struct pcpu_chunk {
107 struct list_head list; /* linked to pcpu_slot lists */
108 int free_size; /* free bytes in the chunk */
109 int contig_hint; /* max contiguous size hint */
110 void *base_addr; /* base address of this chunk */
111
112 int map_used; /* # of map entries used before the sentry */
113 int map_alloc; /* # of map entries allocated */
114 int *map; /* allocation map */
115 struct list_head map_extend_list;/* on pcpu_map_extend_chunks */
116
117 void *data; /* chunk data */
118 int first_free; /* no free below this */
119 bool immutable; /* no [de]population allowed */
120 int nr_populated; /* # of populated pages */
121 unsigned long populated[]; /* populated bitmap */
122};
123
124static int pcpu_unit_pages __ro_after_init; 108static int pcpu_unit_pages __ro_after_init;
125static int pcpu_unit_size __ro_after_init; 109static int pcpu_unit_size __ro_after_init;
126static int pcpu_nr_units __ro_after_init; 110static int pcpu_nr_units __ro_after_init;
127static int pcpu_atom_size __ro_after_init; 111static int pcpu_atom_size __ro_after_init;
128static int pcpu_nr_slots __ro_after_init; 112int pcpu_nr_slots __ro_after_init;
129static size_t pcpu_chunk_struct_size __ro_after_init; 113static size_t pcpu_chunk_struct_size __ro_after_init;
130 114
131/* cpus with the lowest and highest unit addresses */ 115/* cpus with the lowest and highest unit addresses */
@@ -149,7 +133,7 @@ static const size_t *pcpu_group_sizes __ro_after_init;
149 * chunks, this one can be allocated and mapped in several different 133 * chunks, this one can be allocated and mapped in several different
150 * ways and thus often doesn't live in the vmalloc area. 134 * ways and thus often doesn't live in the vmalloc area.
151 */ 135 */
152static struct pcpu_chunk *pcpu_first_chunk __ro_after_init; 136struct pcpu_chunk *pcpu_first_chunk __ro_after_init;
153 137
154/* 138/*
155 * Optional reserved chunk. This chunk reserves part of the first 139 * Optional reserved chunk. This chunk reserves part of the first
@@ -158,13 +142,13 @@ static struct pcpu_chunk *pcpu_first_chunk __ro_after_init;
158 * area doesn't exist, the following variables contain NULL and 0 142 * area doesn't exist, the following variables contain NULL and 0
159 * respectively. 143 * respectively.
160 */ 144 */
161static struct pcpu_chunk *pcpu_reserved_chunk __ro_after_init; 145struct pcpu_chunk *pcpu_reserved_chunk __ro_after_init;
162static int pcpu_reserved_chunk_limit __ro_after_init; 146static int pcpu_reserved_chunk_limit __ro_after_init;
163 147
164static DEFINE_SPINLOCK(pcpu_lock); /* all internal data structures */ 148DEFINE_SPINLOCK(pcpu_lock); /* all internal data structures */
165static DEFINE_MUTEX(pcpu_alloc_mutex); /* chunk create/destroy, [de]pop, map ext */ 149static DEFINE_MUTEX(pcpu_alloc_mutex); /* chunk create/destroy, [de]pop, map ext */
166 150
167static struct list_head *pcpu_slot __ro_after_init; /* chunk list slots */ 151struct list_head *pcpu_slot __ro_after_init; /* chunk list slots */
168 152
169/* chunks which need their map areas extended, protected by pcpu_lock */ 153/* chunks which need their map areas extended, protected by pcpu_lock */
170static LIST_HEAD(pcpu_map_extend_chunks); 154static LIST_HEAD(pcpu_map_extend_chunks);