diff options
Diffstat (limited to 'mm')
-rw-r--r-- | mm/Makefile | 4 | ||||
-rw-r--r-- | mm/allocpercpu.c | 32 | ||||
-rw-r--r-- | mm/bootmem.c | 35 | ||||
-rw-r--r-- | mm/filemap.c | 7 | ||||
-rw-r--r-- | mm/percpu.c | 979 | ||||
-rw-r--r-- | mm/vmalloc.c | 97 |
6 files changed, 1128 insertions, 26 deletions
diff --git a/mm/Makefile b/mm/Makefile index 72255be57f89..818569b68f46 100644 --- a/mm/Makefile +++ b/mm/Makefile | |||
@@ -30,6 +30,10 @@ obj-$(CONFIG_FAILSLAB) += failslab.o | |||
30 | obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o | 30 | obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o |
31 | obj-$(CONFIG_FS_XIP) += filemap_xip.o | 31 | obj-$(CONFIG_FS_XIP) += filemap_xip.o |
32 | obj-$(CONFIG_MIGRATION) += migrate.o | 32 | obj-$(CONFIG_MIGRATION) += migrate.o |
33 | ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA | ||
34 | obj-$(CONFIG_SMP) += percpu.o | ||
35 | else | ||
33 | obj-$(CONFIG_SMP) += allocpercpu.o | 36 | obj-$(CONFIG_SMP) += allocpercpu.o |
37 | endif | ||
34 | obj-$(CONFIG_QUICKLIST) += quicklist.o | 38 | obj-$(CONFIG_QUICKLIST) += quicklist.o |
35 | obj-$(CONFIG_CGROUP_MEM_RES_CTLR) += memcontrol.o page_cgroup.o | 39 | obj-$(CONFIG_CGROUP_MEM_RES_CTLR) += memcontrol.o page_cgroup.o |
diff --git a/mm/allocpercpu.c b/mm/allocpercpu.c index 4297bc41bfd2..3653c570232b 100644 --- a/mm/allocpercpu.c +++ b/mm/allocpercpu.c | |||
@@ -99,45 +99,51 @@ static int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp, | |||
99 | __percpu_populate_mask((__pdata), (size), (gfp), &(mask)) | 99 | __percpu_populate_mask((__pdata), (size), (gfp), &(mask)) |
100 | 100 | ||
101 | /** | 101 | /** |
102 | * percpu_alloc_mask - initial setup of per-cpu data | 102 | * alloc_percpu - initial setup of per-cpu data |
103 | * @size: size of per-cpu object | 103 | * @size: size of per-cpu object |
104 | * @gfp: may sleep or not etc. | 104 | * @align: alignment |
105 | * @mask: populate per-data for cpu's selected through mask bits | ||
106 | * | 105 | * |
107 | * Populating per-cpu data for all online cpu's would be a typical use case, | 106 | * Allocate dynamic percpu area. Percpu objects are populated with |
108 | * which is simplified by the percpu_alloc() wrapper. | 107 | * zeroed buffers. |
109 | * Per-cpu objects are populated with zeroed buffers. | ||
110 | */ | 108 | */ |
111 | void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask) | 109 | void *__alloc_percpu(size_t size, size_t align) |
112 | { | 110 | { |
113 | /* | 111 | /* |
114 | * We allocate whole cache lines to avoid false sharing | 112 | * We allocate whole cache lines to avoid false sharing |
115 | */ | 113 | */ |
116 | size_t sz = roundup(nr_cpu_ids * sizeof(void *), cache_line_size()); | 114 | size_t sz = roundup(nr_cpu_ids * sizeof(void *), cache_line_size()); |
117 | void *pdata = kzalloc(sz, gfp); | 115 | void *pdata = kzalloc(sz, GFP_KERNEL); |
118 | void *__pdata = __percpu_disguise(pdata); | 116 | void *__pdata = __percpu_disguise(pdata); |
119 | 117 | ||
118 | /* | ||
119 | * Can't easily make larger alignment work with kmalloc. WARN | ||
120 | * on it. Larger alignment should only be used for module | ||
121 | * percpu sections on SMP for which this path isn't used. | ||
122 | */ | ||
123 | WARN_ON_ONCE(align > __alignof__(unsigned long long)); | ||
124 | |||
120 | if (unlikely(!pdata)) | 125 | if (unlikely(!pdata)) |
121 | return NULL; | 126 | return NULL; |
122 | if (likely(!__percpu_populate_mask(__pdata, size, gfp, mask))) | 127 | if (likely(!__percpu_populate_mask(__pdata, size, GFP_KERNEL, |
128 | &cpu_possible_map))) | ||
123 | return __pdata; | 129 | return __pdata; |
124 | kfree(pdata); | 130 | kfree(pdata); |
125 | return NULL; | 131 | return NULL; |
126 | } | 132 | } |
127 | EXPORT_SYMBOL_GPL(__percpu_alloc_mask); | 133 | EXPORT_SYMBOL_GPL(__alloc_percpu); |
128 | 134 | ||
129 | /** | 135 | /** |
130 | * percpu_free - final cleanup of per-cpu data | 136 | * free_percpu - final cleanup of per-cpu data |
131 | * @__pdata: object to clean up | 137 | * @__pdata: object to clean up |
132 | * | 138 | * |
133 | * We simply clean up any per-cpu object left. No need for the client to | 139 | * We simply clean up any per-cpu object left. No need for the client to |
134 | * track and specify through a bis mask which per-cpu objects are to free. | 140 | * track and specify through a bis mask which per-cpu objects are to free. |
135 | */ | 141 | */ |
136 | void percpu_free(void *__pdata) | 142 | void free_percpu(void *__pdata) |
137 | { | 143 | { |
138 | if (unlikely(!__pdata)) | 144 | if (unlikely(!__pdata)) |
139 | return; | 145 | return; |
140 | __percpu_depopulate_mask(__pdata, &cpu_possible_map); | 146 | __percpu_depopulate_mask(__pdata, &cpu_possible_map); |
141 | kfree(__percpu_disguise(__pdata)); | 147 | kfree(__percpu_disguise(__pdata)); |
142 | } | 148 | } |
143 | EXPORT_SYMBOL_GPL(percpu_free); | 149 | EXPORT_SYMBOL_GPL(free_percpu); |
diff --git a/mm/bootmem.c b/mm/bootmem.c index 51a0ccf61e0e..daf92713f7de 100644 --- a/mm/bootmem.c +++ b/mm/bootmem.c | |||
@@ -382,7 +382,6 @@ int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, | |||
382 | return mark_bootmem_node(pgdat->bdata, start, end, 1, flags); | 382 | return mark_bootmem_node(pgdat->bdata, start, end, 1, flags); |
383 | } | 383 | } |
384 | 384 | ||
385 | #ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE | ||
386 | /** | 385 | /** |
387 | * reserve_bootmem - mark a page range as usable | 386 | * reserve_bootmem - mark a page range as usable |
388 | * @addr: starting address of the range | 387 | * @addr: starting address of the range |
@@ -403,7 +402,6 @@ int __init reserve_bootmem(unsigned long addr, unsigned long size, | |||
403 | 402 | ||
404 | return mark_bootmem(start, end, 1, flags); | 403 | return mark_bootmem(start, end, 1, flags); |
405 | } | 404 | } |
406 | #endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */ | ||
407 | 405 | ||
408 | static unsigned long align_idx(struct bootmem_data *bdata, unsigned long idx, | 406 | static unsigned long align_idx(struct bootmem_data *bdata, unsigned long idx, |
409 | unsigned long step) | 407 | unsigned long step) |
@@ -429,8 +427,8 @@ static unsigned long align_off(struct bootmem_data *bdata, unsigned long off, | |||
429 | } | 427 | } |
430 | 428 | ||
431 | static void * __init alloc_bootmem_core(struct bootmem_data *bdata, | 429 | static void * __init alloc_bootmem_core(struct bootmem_data *bdata, |
432 | unsigned long size, unsigned long align, | 430 | unsigned long size, unsigned long align, |
433 | unsigned long goal, unsigned long limit) | 431 | unsigned long goal, unsigned long limit) |
434 | { | 432 | { |
435 | unsigned long fallback = 0; | 433 | unsigned long fallback = 0; |
436 | unsigned long min, max, start, sidx, midx, step; | 434 | unsigned long min, max, start, sidx, midx, step; |
@@ -530,17 +528,34 @@ find_block: | |||
530 | return NULL; | 528 | return NULL; |
531 | } | 529 | } |
532 | 530 | ||
531 | static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata, | ||
532 | unsigned long size, unsigned long align, | ||
533 | unsigned long goal, unsigned long limit) | ||
534 | { | ||
535 | #ifdef CONFIG_HAVE_ARCH_BOOTMEM | ||
536 | bootmem_data_t *p_bdata; | ||
537 | |||
538 | p_bdata = bootmem_arch_preferred_node(bdata, size, align, goal, limit); | ||
539 | if (p_bdata) | ||
540 | return alloc_bootmem_core(p_bdata, size, align, goal, limit); | ||
541 | #endif | ||
542 | return NULL; | ||
543 | } | ||
544 | |||
533 | static void * __init ___alloc_bootmem_nopanic(unsigned long size, | 545 | static void * __init ___alloc_bootmem_nopanic(unsigned long size, |
534 | unsigned long align, | 546 | unsigned long align, |
535 | unsigned long goal, | 547 | unsigned long goal, |
536 | unsigned long limit) | 548 | unsigned long limit) |
537 | { | 549 | { |
538 | bootmem_data_t *bdata; | 550 | bootmem_data_t *bdata; |
551 | void *region; | ||
539 | 552 | ||
540 | restart: | 553 | restart: |
541 | list_for_each_entry(bdata, &bdata_list, list) { | 554 | region = alloc_arch_preferred_bootmem(NULL, size, align, goal, limit); |
542 | void *region; | 555 | if (region) |
556 | return region; | ||
543 | 557 | ||
558 | list_for_each_entry(bdata, &bdata_list, list) { | ||
544 | if (goal && bdata->node_low_pfn <= PFN_DOWN(goal)) | 559 | if (goal && bdata->node_low_pfn <= PFN_DOWN(goal)) |
545 | continue; | 560 | continue; |
546 | if (limit && bdata->node_min_pfn >= PFN_DOWN(limit)) | 561 | if (limit && bdata->node_min_pfn >= PFN_DOWN(limit)) |
@@ -618,6 +633,10 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata, | |||
618 | { | 633 | { |
619 | void *ptr; | 634 | void *ptr; |
620 | 635 | ||
636 | ptr = alloc_arch_preferred_bootmem(bdata, size, align, goal, limit); | ||
637 | if (ptr) | ||
638 | return ptr; | ||
639 | |||
621 | ptr = alloc_bootmem_core(bdata, size, align, goal, limit); | 640 | ptr = alloc_bootmem_core(bdata, size, align, goal, limit); |
622 | if (ptr) | 641 | if (ptr) |
623 | return ptr; | 642 | return ptr; |
@@ -674,6 +693,10 @@ void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size, | |||
674 | { | 693 | { |
675 | void *ptr; | 694 | void *ptr; |
676 | 695 | ||
696 | ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0); | ||
697 | if (ptr) | ||
698 | return ptr; | ||
699 | |||
677 | ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0); | 700 | ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0); |
678 | if (ptr) | 701 | if (ptr) |
679 | return ptr; | 702 | return ptr; |
diff --git a/mm/filemap.c b/mm/filemap.c index 23acefe51808..126d3973b3d1 100644 --- a/mm/filemap.c +++ b/mm/filemap.c | |||
@@ -1823,7 +1823,7 @@ static size_t __iovec_copy_from_user_inatomic(char *vaddr, | |||
1823 | int copy = min(bytes, iov->iov_len - base); | 1823 | int copy = min(bytes, iov->iov_len - base); |
1824 | 1824 | ||
1825 | base = 0; | 1825 | base = 0; |
1826 | left = __copy_from_user_inatomic_nocache(vaddr, buf, copy); | 1826 | left = __copy_from_user_inatomic(vaddr, buf, copy); |
1827 | copied += copy; | 1827 | copied += copy; |
1828 | bytes -= copy; | 1828 | bytes -= copy; |
1829 | vaddr += copy; | 1829 | vaddr += copy; |
@@ -1851,8 +1851,7 @@ size_t iov_iter_copy_from_user_atomic(struct page *page, | |||
1851 | if (likely(i->nr_segs == 1)) { | 1851 | if (likely(i->nr_segs == 1)) { |
1852 | int left; | 1852 | int left; |
1853 | char __user *buf = i->iov->iov_base + i->iov_offset; | 1853 | char __user *buf = i->iov->iov_base + i->iov_offset; |
1854 | left = __copy_from_user_inatomic_nocache(kaddr + offset, | 1854 | left = __copy_from_user_inatomic(kaddr + offset, buf, bytes); |
1855 | buf, bytes); | ||
1856 | copied = bytes - left; | 1855 | copied = bytes - left; |
1857 | } else { | 1856 | } else { |
1858 | copied = __iovec_copy_from_user_inatomic(kaddr + offset, | 1857 | copied = __iovec_copy_from_user_inatomic(kaddr + offset, |
@@ -1880,7 +1879,7 @@ size_t iov_iter_copy_from_user(struct page *page, | |||
1880 | if (likely(i->nr_segs == 1)) { | 1879 | if (likely(i->nr_segs == 1)) { |
1881 | int left; | 1880 | int left; |
1882 | char __user *buf = i->iov->iov_base + i->iov_offset; | 1881 | char __user *buf = i->iov->iov_base + i->iov_offset; |
1883 | left = __copy_from_user_nocache(kaddr + offset, buf, bytes); | 1882 | left = __copy_from_user(kaddr + offset, buf, bytes); |
1884 | copied = bytes - left; | 1883 | copied = bytes - left; |
1885 | } else { | 1884 | } else { |
1886 | copied = __iovec_copy_from_user_inatomic(kaddr + offset, | 1885 | copied = __iovec_copy_from_user_inatomic(kaddr + offset, |
diff --git a/mm/percpu.c b/mm/percpu.c new file mode 100644 index 000000000000..3d0f5456827c --- /dev/null +++ b/mm/percpu.c | |||
@@ -0,0 +1,979 @@ | |||
1 | /* | ||
2 | * linux/mm/percpu.c - percpu memory allocator | ||
3 | * | ||
4 | * Copyright (C) 2009 SUSE Linux Products GmbH | ||
5 | * Copyright (C) 2009 Tejun Heo <tj@kernel.org> | ||
6 | * | ||
7 | * This file is released under the GPLv2. | ||
8 | * | ||
9 | * This is percpu allocator which can handle both static and dynamic | ||
10 | * areas. Percpu areas are allocated in chunks in vmalloc area. Each | ||
11 | * chunk is consisted of num_possible_cpus() units and the first chunk | ||
12 | * is used for static percpu variables in the kernel image (special | ||
13 | * boot time alloc/init handling necessary as these areas need to be | ||
14 | * brought up before allocation services are running). Unit grows as | ||
15 | * necessary and all units grow or shrink in unison. When a chunk is | ||
16 | * filled up, another chunk is allocated. ie. in vmalloc area | ||
17 | * | ||
18 | * c0 c1 c2 | ||
19 | * ------------------- ------------------- ------------ | ||
20 | * | u0 | u1 | u2 | u3 | | u0 | u1 | u2 | u3 | | u0 | u1 | u | ||
21 | * ------------------- ...... ------------------- .... ------------ | ||
22 | * | ||
23 | * Allocation is done in offset-size areas of single unit space. Ie, | ||
24 | * an area of 512 bytes at 6k in c1 occupies 512 bytes at 6k of c1:u0, | ||
25 | * c1:u1, c1:u2 and c1:u3. Percpu access can be done by configuring | ||
26 | * percpu base registers UNIT_SIZE apart. | ||
27 | * | ||
28 | * There are usually many small percpu allocations many of them as | ||
29 | * small as 4 bytes. The allocator organizes chunks into lists | ||
30 | * according to free size and tries to allocate from the fullest one. | ||
31 | * Each chunk keeps the maximum contiguous area size hint which is | ||
32 | * guaranteed to be eqaul to or larger than the maximum contiguous | ||
33 | * area in the chunk. This helps the allocator not to iterate the | ||
34 | * chunk maps unnecessarily. | ||
35 | * | ||
36 | * Allocation state in each chunk is kept using an array of integers | ||
37 | * on chunk->map. A positive value in the map represents a free | ||
38 | * region and negative allocated. Allocation inside a chunk is done | ||
39 | * by scanning this map sequentially and serving the first matching | ||
40 | * entry. This is mostly copied from the percpu_modalloc() allocator. | ||
41 | * Chunks are also linked into a rb tree to ease address to chunk | ||
42 | * mapping during free. | ||
43 | * | ||
44 | * To use this allocator, arch code should do the followings. | ||
45 | * | ||
46 | * - define CONFIG_HAVE_DYNAMIC_PER_CPU_AREA | ||
47 | * | ||
48 | * - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate | ||
49 | * regular address to percpu pointer and back | ||
50 | * | ||
51 | * - use pcpu_setup_first_chunk() during percpu area initialization to | ||
52 | * setup the first chunk containing the kernel static percpu area | ||
53 | */ | ||
54 | |||
55 | #include <linux/bitmap.h> | ||
56 | #include <linux/bootmem.h> | ||
57 | #include <linux/list.h> | ||
58 | #include <linux/mm.h> | ||
59 | #include <linux/module.h> | ||
60 | #include <linux/mutex.h> | ||
61 | #include <linux/percpu.h> | ||
62 | #include <linux/pfn.h> | ||
63 | #include <linux/rbtree.h> | ||
64 | #include <linux/slab.h> | ||
65 | #include <linux/vmalloc.h> | ||
66 | |||
67 | #include <asm/cacheflush.h> | ||
68 | #include <asm/tlbflush.h> | ||
69 | |||
70 | #define PCPU_SLOT_BASE_SHIFT 5 /* 1-31 shares the same slot */ | ||
71 | #define PCPU_DFL_MAP_ALLOC 16 /* start a map with 16 ents */ | ||
72 | |||
73 | struct pcpu_chunk { | ||
74 | struct list_head list; /* linked to pcpu_slot lists */ | ||
75 | struct rb_node rb_node; /* key is chunk->vm->addr */ | ||
76 | int free_size; /* free bytes in the chunk */ | ||
77 | int contig_hint; /* max contiguous size hint */ | ||
78 | struct vm_struct *vm; /* mapped vmalloc region */ | ||
79 | int map_used; /* # of map entries used */ | ||
80 | int map_alloc; /* # of map entries allocated */ | ||
81 | int *map; /* allocation map */ | ||
82 | bool immutable; /* no [de]population allowed */ | ||
83 | struct page *page[]; /* #cpus * UNIT_PAGES */ | ||
84 | }; | ||
85 | |||
86 | static int pcpu_unit_pages __read_mostly; | ||
87 | static int pcpu_unit_size __read_mostly; | ||
88 | static int pcpu_chunk_size __read_mostly; | ||
89 | static int pcpu_nr_slots __read_mostly; | ||
90 | static size_t pcpu_chunk_struct_size __read_mostly; | ||
91 | |||
92 | /* the address of the first chunk which starts with the kernel static area */ | ||
93 | void *pcpu_base_addr __read_mostly; | ||
94 | EXPORT_SYMBOL_GPL(pcpu_base_addr); | ||
95 | |||
96 | /* the size of kernel static area */ | ||
97 | static int pcpu_static_size __read_mostly; | ||
98 | |||
99 | /* | ||
100 | * One mutex to rule them all. | ||
101 | * | ||
102 | * The following mutex is grabbed in the outermost public alloc/free | ||
103 | * interface functions and released only when the operation is | ||
104 | * complete. As such, every function in this file other than the | ||
105 | * outermost functions are called under pcpu_mutex. | ||
106 | * | ||
107 | * It can easily be switched to use spinlock such that only the area | ||
108 | * allocation and page population commit are protected with it doing | ||
109 | * actual [de]allocation without holding any lock. However, given | ||
110 | * what this allocator does, I think it's better to let them run | ||
111 | * sequentially. | ||
112 | */ | ||
113 | static DEFINE_MUTEX(pcpu_mutex); | ||
114 | |||
115 | static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */ | ||
116 | static struct rb_root pcpu_addr_root = RB_ROOT; /* chunks by address */ | ||
117 | |||
118 | static int __pcpu_size_to_slot(int size) | ||
119 | { | ||
120 | int highbit = fls(size); /* size is in bytes */ | ||
121 | return max(highbit - PCPU_SLOT_BASE_SHIFT + 2, 1); | ||
122 | } | ||
123 | |||
124 | static int pcpu_size_to_slot(int size) | ||
125 | { | ||
126 | if (size == pcpu_unit_size) | ||
127 | return pcpu_nr_slots - 1; | ||
128 | return __pcpu_size_to_slot(size); | ||
129 | } | ||
130 | |||
131 | static int pcpu_chunk_slot(const struct pcpu_chunk *chunk) | ||
132 | { | ||
133 | if (chunk->free_size < sizeof(int) || chunk->contig_hint < sizeof(int)) | ||
134 | return 0; | ||
135 | |||
136 | return pcpu_size_to_slot(chunk->free_size); | ||
137 | } | ||
138 | |||
139 | static int pcpu_page_idx(unsigned int cpu, int page_idx) | ||
140 | { | ||
141 | return cpu * pcpu_unit_pages + page_idx; | ||
142 | } | ||
143 | |||
144 | static struct page **pcpu_chunk_pagep(struct pcpu_chunk *chunk, | ||
145 | unsigned int cpu, int page_idx) | ||
146 | { | ||
147 | return &chunk->page[pcpu_page_idx(cpu, page_idx)]; | ||
148 | } | ||
149 | |||
150 | static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk, | ||
151 | unsigned int cpu, int page_idx) | ||
152 | { | ||
153 | return (unsigned long)chunk->vm->addr + | ||
154 | (pcpu_page_idx(cpu, page_idx) << PAGE_SHIFT); | ||
155 | } | ||
156 | |||
157 | static bool pcpu_chunk_page_occupied(struct pcpu_chunk *chunk, | ||
158 | int page_idx) | ||
159 | { | ||
160 | return *pcpu_chunk_pagep(chunk, 0, page_idx) != NULL; | ||
161 | } | ||
162 | |||
163 | /** | ||
164 | * pcpu_realloc - versatile realloc | ||
165 | * @p: the current pointer (can be NULL for new allocations) | ||
166 | * @size: the current size in bytes (can be 0 for new allocations) | ||
167 | * @new_size: the wanted new size in bytes (can be 0 for free) | ||
168 | * | ||
169 | * More robust realloc which can be used to allocate, resize or free a | ||
170 | * memory area of arbitrary size. If the needed size goes over | ||
171 | * PAGE_SIZE, kernel VM is used. | ||
172 | * | ||
173 | * RETURNS: | ||
174 | * The new pointer on success, NULL on failure. | ||
175 | */ | ||
176 | static void *pcpu_realloc(void *p, size_t size, size_t new_size) | ||
177 | { | ||
178 | void *new; | ||
179 | |||
180 | if (new_size <= PAGE_SIZE) | ||
181 | new = kmalloc(new_size, GFP_KERNEL); | ||
182 | else | ||
183 | new = vmalloc(new_size); | ||
184 | if (new_size && !new) | ||
185 | return NULL; | ||
186 | |||
187 | memcpy(new, p, min(size, new_size)); | ||
188 | if (new_size > size) | ||
189 | memset(new + size, 0, new_size - size); | ||
190 | |||
191 | if (size <= PAGE_SIZE) | ||
192 | kfree(p); | ||
193 | else | ||
194 | vfree(p); | ||
195 | |||
196 | return new; | ||
197 | } | ||
198 | |||
199 | /** | ||
200 | * pcpu_chunk_relocate - put chunk in the appropriate chunk slot | ||
201 | * @chunk: chunk of interest | ||
202 | * @oslot: the previous slot it was on | ||
203 | * | ||
204 | * This function is called after an allocation or free changed @chunk. | ||
205 | * New slot according to the changed state is determined and @chunk is | ||
206 | * moved to the slot. | ||
207 | */ | ||
208 | static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot) | ||
209 | { | ||
210 | int nslot = pcpu_chunk_slot(chunk); | ||
211 | |||
212 | if (oslot != nslot) { | ||
213 | if (oslot < nslot) | ||
214 | list_move(&chunk->list, &pcpu_slot[nslot]); | ||
215 | else | ||
216 | list_move_tail(&chunk->list, &pcpu_slot[nslot]); | ||
217 | } | ||
218 | } | ||
219 | |||
220 | static struct rb_node **pcpu_chunk_rb_search(void *addr, | ||
221 | struct rb_node **parentp) | ||
222 | { | ||
223 | struct rb_node **p = &pcpu_addr_root.rb_node; | ||
224 | struct rb_node *parent = NULL; | ||
225 | struct pcpu_chunk *chunk; | ||
226 | |||
227 | while (*p) { | ||
228 | parent = *p; | ||
229 | chunk = rb_entry(parent, struct pcpu_chunk, rb_node); | ||
230 | |||
231 | if (addr < chunk->vm->addr) | ||
232 | p = &(*p)->rb_left; | ||
233 | else if (addr > chunk->vm->addr) | ||
234 | p = &(*p)->rb_right; | ||
235 | else | ||
236 | break; | ||
237 | } | ||
238 | |||
239 | if (parentp) | ||
240 | *parentp = parent; | ||
241 | return p; | ||
242 | } | ||
243 | |||
244 | /** | ||
245 | * pcpu_chunk_addr_search - search for chunk containing specified address | ||
246 | * @addr: address to search for | ||
247 | * | ||
248 | * Look for chunk which might contain @addr. More specifically, it | ||
249 | * searchs for the chunk with the highest start address which isn't | ||
250 | * beyond @addr. | ||
251 | * | ||
252 | * RETURNS: | ||
253 | * The address of the found chunk. | ||
254 | */ | ||
255 | static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr) | ||
256 | { | ||
257 | struct rb_node *n, *parent; | ||
258 | struct pcpu_chunk *chunk; | ||
259 | |||
260 | n = *pcpu_chunk_rb_search(addr, &parent); | ||
261 | if (!n) { | ||
262 | /* no exactly matching chunk, the parent is the closest */ | ||
263 | n = parent; | ||
264 | BUG_ON(!n); | ||
265 | } | ||
266 | chunk = rb_entry(n, struct pcpu_chunk, rb_node); | ||
267 | |||
268 | if (addr < chunk->vm->addr) { | ||
269 | /* the parent was the next one, look for the previous one */ | ||
270 | n = rb_prev(n); | ||
271 | BUG_ON(!n); | ||
272 | chunk = rb_entry(n, struct pcpu_chunk, rb_node); | ||
273 | } | ||
274 | |||
275 | return chunk; | ||
276 | } | ||
277 | |||
278 | /** | ||
279 | * pcpu_chunk_addr_insert - insert chunk into address rb tree | ||
280 | * @new: chunk to insert | ||
281 | * | ||
282 | * Insert @new into address rb tree. | ||
283 | */ | ||
284 | static void pcpu_chunk_addr_insert(struct pcpu_chunk *new) | ||
285 | { | ||
286 | struct rb_node **p, *parent; | ||
287 | |||
288 | p = pcpu_chunk_rb_search(new->vm->addr, &parent); | ||
289 | BUG_ON(*p); | ||
290 | rb_link_node(&new->rb_node, parent, p); | ||
291 | rb_insert_color(&new->rb_node, &pcpu_addr_root); | ||
292 | } | ||
293 | |||
294 | /** | ||
295 | * pcpu_split_block - split a map block | ||
296 | * @chunk: chunk of interest | ||
297 | * @i: index of map block to split | ||
298 | * @head: head size in bytes (can be 0) | ||
299 | * @tail: tail size in bytes (can be 0) | ||
300 | * | ||
301 | * Split the @i'th map block into two or three blocks. If @head is | ||
302 | * non-zero, @head bytes block is inserted before block @i moving it | ||
303 | * to @i+1 and reducing its size by @head bytes. | ||
304 | * | ||
305 | * If @tail is non-zero, the target block, which can be @i or @i+1 | ||
306 | * depending on @head, is reduced by @tail bytes and @tail byte block | ||
307 | * is inserted after the target block. | ||
308 | * | ||
309 | * RETURNS: | ||
310 | * 0 on success, -errno on failure. | ||
311 | */ | ||
312 | static int pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail) | ||
313 | { | ||
314 | int nr_extra = !!head + !!tail; | ||
315 | int target = chunk->map_used + nr_extra; | ||
316 | |||
317 | /* reallocation required? */ | ||
318 | if (chunk->map_alloc < target) { | ||
319 | int new_alloc = chunk->map_alloc; | ||
320 | int *new; | ||
321 | |||
322 | while (new_alloc < target) | ||
323 | new_alloc *= 2; | ||
324 | |||
325 | new = pcpu_realloc(chunk->map, | ||
326 | chunk->map_alloc * sizeof(new[0]), | ||
327 | new_alloc * sizeof(new[0])); | ||
328 | if (!new) | ||
329 | return -ENOMEM; | ||
330 | |||
331 | chunk->map_alloc = new_alloc; | ||
332 | chunk->map = new; | ||
333 | } | ||
334 | |||
335 | /* insert a new subblock */ | ||
336 | memmove(&chunk->map[i + nr_extra], &chunk->map[i], | ||
337 | sizeof(chunk->map[0]) * (chunk->map_used - i)); | ||
338 | chunk->map_used += nr_extra; | ||
339 | |||
340 | if (head) { | ||
341 | chunk->map[i + 1] = chunk->map[i] - head; | ||
342 | chunk->map[i++] = head; | ||
343 | } | ||
344 | if (tail) { | ||
345 | chunk->map[i++] -= tail; | ||
346 | chunk->map[i] = tail; | ||
347 | } | ||
348 | return 0; | ||
349 | } | ||
350 | |||
351 | /** | ||
352 | * pcpu_alloc_area - allocate area from a pcpu_chunk | ||
353 | * @chunk: chunk of interest | ||
354 | * @size: wanted size in bytes | ||
355 | * @align: wanted align | ||
356 | * | ||
357 | * Try to allocate @size bytes area aligned at @align from @chunk. | ||
358 | * Note that this function only allocates the offset. It doesn't | ||
359 | * populate or map the area. | ||
360 | * | ||
361 | * RETURNS: | ||
362 | * Allocated offset in @chunk on success, -errno on failure. | ||
363 | */ | ||
364 | static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align) | ||
365 | { | ||
366 | int oslot = pcpu_chunk_slot(chunk); | ||
367 | int max_contig = 0; | ||
368 | int i, off; | ||
369 | |||
370 | /* | ||
371 | * The static chunk initially doesn't have map attached | ||
372 | * because kmalloc wasn't available during init. Give it one. | ||
373 | */ | ||
374 | if (unlikely(!chunk->map)) { | ||
375 | chunk->map = pcpu_realloc(NULL, 0, | ||
376 | PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0])); | ||
377 | if (!chunk->map) | ||
378 | return -ENOMEM; | ||
379 | |||
380 | chunk->map_alloc = PCPU_DFL_MAP_ALLOC; | ||
381 | chunk->map[chunk->map_used++] = -pcpu_static_size; | ||
382 | if (chunk->free_size) | ||
383 | chunk->map[chunk->map_used++] = chunk->free_size; | ||
384 | } | ||
385 | |||
386 | for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) { | ||
387 | bool is_last = i + 1 == chunk->map_used; | ||
388 | int head, tail; | ||
389 | |||
390 | /* extra for alignment requirement */ | ||
391 | head = ALIGN(off, align) - off; | ||
392 | BUG_ON(i == 0 && head != 0); | ||
393 | |||
394 | if (chunk->map[i] < 0) | ||
395 | continue; | ||
396 | if (chunk->map[i] < head + size) { | ||
397 | max_contig = max(chunk->map[i], max_contig); | ||
398 | continue; | ||
399 | } | ||
400 | |||
401 | /* | ||
402 | * If head is small or the previous block is free, | ||
403 | * merge'em. Note that 'small' is defined as smaller | ||
404 | * than sizeof(int), which is very small but isn't too | ||
405 | * uncommon for percpu allocations. | ||
406 | */ | ||
407 | if (head && (head < sizeof(int) || chunk->map[i - 1] > 0)) { | ||
408 | if (chunk->map[i - 1] > 0) | ||
409 | chunk->map[i - 1] += head; | ||
410 | else { | ||
411 | chunk->map[i - 1] -= head; | ||
412 | chunk->free_size -= head; | ||
413 | } | ||
414 | chunk->map[i] -= head; | ||
415 | off += head; | ||
416 | head = 0; | ||
417 | } | ||
418 | |||
419 | /* if tail is small, just keep it around */ | ||
420 | tail = chunk->map[i] - head - size; | ||
421 | if (tail < sizeof(int)) | ||
422 | tail = 0; | ||
423 | |||
424 | /* split if warranted */ | ||
425 | if (head || tail) { | ||
426 | if (pcpu_split_block(chunk, i, head, tail)) | ||
427 | return -ENOMEM; | ||
428 | if (head) { | ||
429 | i++; | ||
430 | off += head; | ||
431 | max_contig = max(chunk->map[i - 1], max_contig); | ||
432 | } | ||
433 | if (tail) | ||
434 | max_contig = max(chunk->map[i + 1], max_contig); | ||
435 | } | ||
436 | |||
437 | /* update hint and mark allocated */ | ||
438 | if (is_last) | ||
439 | chunk->contig_hint = max_contig; /* fully scanned */ | ||
440 | else | ||
441 | chunk->contig_hint = max(chunk->contig_hint, | ||
442 | max_contig); | ||
443 | |||
444 | chunk->free_size -= chunk->map[i]; | ||
445 | chunk->map[i] = -chunk->map[i]; | ||
446 | |||
447 | pcpu_chunk_relocate(chunk, oslot); | ||
448 | return off; | ||
449 | } | ||
450 | |||
451 | chunk->contig_hint = max_contig; /* fully scanned */ | ||
452 | pcpu_chunk_relocate(chunk, oslot); | ||
453 | |||
454 | /* | ||
455 | * Tell the upper layer that this chunk has no area left. | ||
456 | * Note that this is not an error condition but a notification | ||
457 | * to upper layer that it needs to look at other chunks. | ||
458 | * -ENOSPC is chosen as it isn't used in memory subsystem and | ||
459 | * matches the meaning in a way. | ||
460 | */ | ||
461 | return -ENOSPC; | ||
462 | } | ||
463 | |||
464 | /** | ||
465 | * pcpu_free_area - free area to a pcpu_chunk | ||
466 | * @chunk: chunk of interest | ||
467 | * @freeme: offset of area to free | ||
468 | * | ||
469 | * Free area starting from @freeme to @chunk. Note that this function | ||
470 | * only modifies the allocation map. It doesn't depopulate or unmap | ||
471 | * the area. | ||
472 | */ | ||
473 | static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme) | ||
474 | { | ||
475 | int oslot = pcpu_chunk_slot(chunk); | ||
476 | int i, off; | ||
477 | |||
478 | for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) | ||
479 | if (off == freeme) | ||
480 | break; | ||
481 | BUG_ON(off != freeme); | ||
482 | BUG_ON(chunk->map[i] > 0); | ||
483 | |||
484 | chunk->map[i] = -chunk->map[i]; | ||
485 | chunk->free_size += chunk->map[i]; | ||
486 | |||
487 | /* merge with previous? */ | ||
488 | if (i > 0 && chunk->map[i - 1] >= 0) { | ||
489 | chunk->map[i - 1] += chunk->map[i]; | ||
490 | chunk->map_used--; | ||
491 | memmove(&chunk->map[i], &chunk->map[i + 1], | ||
492 | (chunk->map_used - i) * sizeof(chunk->map[0])); | ||
493 | i--; | ||
494 | } | ||
495 | /* merge with next? */ | ||
496 | if (i + 1 < chunk->map_used && chunk->map[i + 1] >= 0) { | ||
497 | chunk->map[i] += chunk->map[i + 1]; | ||
498 | chunk->map_used--; | ||
499 | memmove(&chunk->map[i + 1], &chunk->map[i + 2], | ||
500 | (chunk->map_used - (i + 1)) * sizeof(chunk->map[0])); | ||
501 | } | ||
502 | |||
503 | chunk->contig_hint = max(chunk->map[i], chunk->contig_hint); | ||
504 | pcpu_chunk_relocate(chunk, oslot); | ||
505 | } | ||
506 | |||
507 | /** | ||
508 | * pcpu_unmap - unmap pages out of a pcpu_chunk | ||
509 | * @chunk: chunk of interest | ||
510 | * @page_start: page index of the first page to unmap | ||
511 | * @page_end: page index of the last page to unmap + 1 | ||
512 | * @flush: whether to flush cache and tlb or not | ||
513 | * | ||
514 | * For each cpu, unmap pages [@page_start,@page_end) out of @chunk. | ||
515 | * If @flush is true, vcache is flushed before unmapping and tlb | ||
516 | * after. | ||
517 | */ | ||
518 | static void pcpu_unmap(struct pcpu_chunk *chunk, int page_start, int page_end, | ||
519 | bool flush) | ||
520 | { | ||
521 | unsigned int last = num_possible_cpus() - 1; | ||
522 | unsigned int cpu; | ||
523 | |||
524 | /* unmap must not be done on immutable chunk */ | ||
525 | WARN_ON(chunk->immutable); | ||
526 | |||
527 | /* | ||
528 | * Each flushing trial can be very expensive, issue flush on | ||
529 | * the whole region at once rather than doing it for each cpu. | ||
530 | * This could be an overkill but is more scalable. | ||
531 | */ | ||
532 | if (flush) | ||
533 | flush_cache_vunmap(pcpu_chunk_addr(chunk, 0, page_start), | ||
534 | pcpu_chunk_addr(chunk, last, page_end)); | ||
535 | |||
536 | for_each_possible_cpu(cpu) | ||
537 | unmap_kernel_range_noflush( | ||
538 | pcpu_chunk_addr(chunk, cpu, page_start), | ||
539 | (page_end - page_start) << PAGE_SHIFT); | ||
540 | |||
541 | /* ditto as flush_cache_vunmap() */ | ||
542 | if (flush) | ||
543 | flush_tlb_kernel_range(pcpu_chunk_addr(chunk, 0, page_start), | ||
544 | pcpu_chunk_addr(chunk, last, page_end)); | ||
545 | } | ||
546 | |||
547 | /** | ||
548 | * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk | ||
549 | * @chunk: chunk to depopulate | ||
550 | * @off: offset to the area to depopulate | ||
551 | * @size: size of the area to depopulate in bytes | ||
552 | * @flush: whether to flush cache and tlb or not | ||
553 | * | ||
554 | * For each cpu, depopulate and unmap pages [@page_start,@page_end) | ||
555 | * from @chunk. If @flush is true, vcache is flushed before unmapping | ||
556 | * and tlb after. | ||
557 | */ | ||
558 | static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size, | ||
559 | bool flush) | ||
560 | { | ||
561 | int page_start = PFN_DOWN(off); | ||
562 | int page_end = PFN_UP(off + size); | ||
563 | int unmap_start = -1; | ||
564 | int uninitialized_var(unmap_end); | ||
565 | unsigned int cpu; | ||
566 | int i; | ||
567 | |||
568 | for (i = page_start; i < page_end; i++) { | ||
569 | for_each_possible_cpu(cpu) { | ||
570 | struct page **pagep = pcpu_chunk_pagep(chunk, cpu, i); | ||
571 | |||
572 | if (!*pagep) | ||
573 | continue; | ||
574 | |||
575 | __free_page(*pagep); | ||
576 | |||
577 | /* | ||
578 | * If it's partial depopulation, it might get | ||
579 | * populated or depopulated again. Mark the | ||
580 | * page gone. | ||
581 | */ | ||
582 | *pagep = NULL; | ||
583 | |||
584 | unmap_start = unmap_start < 0 ? i : unmap_start; | ||
585 | unmap_end = i + 1; | ||
586 | } | ||
587 | } | ||
588 | |||
589 | if (unmap_start >= 0) | ||
590 | pcpu_unmap(chunk, unmap_start, unmap_end, flush); | ||
591 | } | ||
592 | |||
593 | /** | ||
594 | * pcpu_map - map pages into a pcpu_chunk | ||
595 | * @chunk: chunk of interest | ||
596 | * @page_start: page index of the first page to map | ||
597 | * @page_end: page index of the last page to map + 1 | ||
598 | * | ||
599 | * For each cpu, map pages [@page_start,@page_end) into @chunk. | ||
600 | * vcache is flushed afterwards. | ||
601 | */ | ||
602 | static int pcpu_map(struct pcpu_chunk *chunk, int page_start, int page_end) | ||
603 | { | ||
604 | unsigned int last = num_possible_cpus() - 1; | ||
605 | unsigned int cpu; | ||
606 | int err; | ||
607 | |||
608 | /* map must not be done on immutable chunk */ | ||
609 | WARN_ON(chunk->immutable); | ||
610 | |||
611 | for_each_possible_cpu(cpu) { | ||
612 | err = map_kernel_range_noflush( | ||
613 | pcpu_chunk_addr(chunk, cpu, page_start), | ||
614 | (page_end - page_start) << PAGE_SHIFT, | ||
615 | PAGE_KERNEL, | ||
616 | pcpu_chunk_pagep(chunk, cpu, page_start)); | ||
617 | if (err < 0) | ||
618 | return err; | ||
619 | } | ||
620 | |||
621 | /* flush at once, please read comments in pcpu_unmap() */ | ||
622 | flush_cache_vmap(pcpu_chunk_addr(chunk, 0, page_start), | ||
623 | pcpu_chunk_addr(chunk, last, page_end)); | ||
624 | return 0; | ||
625 | } | ||
626 | |||
627 | /** | ||
628 | * pcpu_populate_chunk - populate and map an area of a pcpu_chunk | ||
629 | * @chunk: chunk of interest | ||
630 | * @off: offset to the area to populate | ||
631 | * @size: size of the area to populate in bytes | ||
632 | * | ||
633 | * For each cpu, populate and map pages [@page_start,@page_end) into | ||
634 | * @chunk. The area is cleared on return. | ||
635 | */ | ||
636 | static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size) | ||
637 | { | ||
638 | const gfp_t alloc_mask = GFP_KERNEL | __GFP_HIGHMEM | __GFP_COLD; | ||
639 | int page_start = PFN_DOWN(off); | ||
640 | int page_end = PFN_UP(off + size); | ||
641 | int map_start = -1; | ||
642 | int uninitialized_var(map_end); | ||
643 | unsigned int cpu; | ||
644 | int i; | ||
645 | |||
646 | for (i = page_start; i < page_end; i++) { | ||
647 | if (pcpu_chunk_page_occupied(chunk, i)) { | ||
648 | if (map_start >= 0) { | ||
649 | if (pcpu_map(chunk, map_start, map_end)) | ||
650 | goto err; | ||
651 | map_start = -1; | ||
652 | } | ||
653 | continue; | ||
654 | } | ||
655 | |||
656 | map_start = map_start < 0 ? i : map_start; | ||
657 | map_end = i + 1; | ||
658 | |||
659 | for_each_possible_cpu(cpu) { | ||
660 | struct page **pagep = pcpu_chunk_pagep(chunk, cpu, i); | ||
661 | |||
662 | *pagep = alloc_pages_node(cpu_to_node(cpu), | ||
663 | alloc_mask, 0); | ||
664 | if (!*pagep) | ||
665 | goto err; | ||
666 | } | ||
667 | } | ||
668 | |||
669 | if (map_start >= 0 && pcpu_map(chunk, map_start, map_end)) | ||
670 | goto err; | ||
671 | |||
672 | for_each_possible_cpu(cpu) | ||
673 | memset(chunk->vm->addr + cpu * pcpu_unit_size + off, 0, | ||
674 | size); | ||
675 | |||
676 | return 0; | ||
677 | err: | ||
678 | /* likely under heavy memory pressure, give memory back */ | ||
679 | pcpu_depopulate_chunk(chunk, off, size, true); | ||
680 | return -ENOMEM; | ||
681 | } | ||
682 | |||
683 | static void free_pcpu_chunk(struct pcpu_chunk *chunk) | ||
684 | { | ||
685 | if (!chunk) | ||
686 | return; | ||
687 | if (chunk->vm) | ||
688 | free_vm_area(chunk->vm); | ||
689 | pcpu_realloc(chunk->map, chunk->map_alloc * sizeof(chunk->map[0]), 0); | ||
690 | kfree(chunk); | ||
691 | } | ||
692 | |||
693 | static struct pcpu_chunk *alloc_pcpu_chunk(void) | ||
694 | { | ||
695 | struct pcpu_chunk *chunk; | ||
696 | |||
697 | chunk = kzalloc(pcpu_chunk_struct_size, GFP_KERNEL); | ||
698 | if (!chunk) | ||
699 | return NULL; | ||
700 | |||
701 | chunk->map = pcpu_realloc(NULL, 0, | ||
702 | PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0])); | ||
703 | chunk->map_alloc = PCPU_DFL_MAP_ALLOC; | ||
704 | chunk->map[chunk->map_used++] = pcpu_unit_size; | ||
705 | |||
706 | chunk->vm = get_vm_area(pcpu_chunk_size, GFP_KERNEL); | ||
707 | if (!chunk->vm) { | ||
708 | free_pcpu_chunk(chunk); | ||
709 | return NULL; | ||
710 | } | ||
711 | |||
712 | INIT_LIST_HEAD(&chunk->list); | ||
713 | chunk->free_size = pcpu_unit_size; | ||
714 | chunk->contig_hint = pcpu_unit_size; | ||
715 | |||
716 | return chunk; | ||
717 | } | ||
718 | |||
719 | /** | ||
720 | * __alloc_percpu - allocate percpu area | ||
721 | * @size: size of area to allocate in bytes | ||
722 | * @align: alignment of area (max PAGE_SIZE) | ||
723 | * | ||
724 | * Allocate percpu area of @size bytes aligned at @align. Might | ||
725 | * sleep. Might trigger writeouts. | ||
726 | * | ||
727 | * RETURNS: | ||
728 | * Percpu pointer to the allocated area on success, NULL on failure. | ||
729 | */ | ||
730 | void *__alloc_percpu(size_t size, size_t align) | ||
731 | { | ||
732 | void *ptr = NULL; | ||
733 | struct pcpu_chunk *chunk; | ||
734 | int slot, off; | ||
735 | |||
736 | if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) { | ||
737 | WARN(true, "illegal size (%zu) or align (%zu) for " | ||
738 | "percpu allocation\n", size, align); | ||
739 | return NULL; | ||
740 | } | ||
741 | |||
742 | mutex_lock(&pcpu_mutex); | ||
743 | |||
744 | /* allocate area */ | ||
745 | for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) { | ||
746 | list_for_each_entry(chunk, &pcpu_slot[slot], list) { | ||
747 | if (size > chunk->contig_hint) | ||
748 | continue; | ||
749 | off = pcpu_alloc_area(chunk, size, align); | ||
750 | if (off >= 0) | ||
751 | goto area_found; | ||
752 | if (off != -ENOSPC) | ||
753 | goto out_unlock; | ||
754 | } | ||
755 | } | ||
756 | |||
757 | /* hmmm... no space left, create a new chunk */ | ||
758 | chunk = alloc_pcpu_chunk(); | ||
759 | if (!chunk) | ||
760 | goto out_unlock; | ||
761 | pcpu_chunk_relocate(chunk, -1); | ||
762 | pcpu_chunk_addr_insert(chunk); | ||
763 | |||
764 | off = pcpu_alloc_area(chunk, size, align); | ||
765 | if (off < 0) | ||
766 | goto out_unlock; | ||
767 | |||
768 | area_found: | ||
769 | /* populate, map and clear the area */ | ||
770 | if (pcpu_populate_chunk(chunk, off, size)) { | ||
771 | pcpu_free_area(chunk, off); | ||
772 | goto out_unlock; | ||
773 | } | ||
774 | |||
775 | ptr = __addr_to_pcpu_ptr(chunk->vm->addr + off); | ||
776 | out_unlock: | ||
777 | mutex_unlock(&pcpu_mutex); | ||
778 | return ptr; | ||
779 | } | ||
780 | EXPORT_SYMBOL_GPL(__alloc_percpu); | ||
781 | |||
782 | static void pcpu_kill_chunk(struct pcpu_chunk *chunk) | ||
783 | { | ||
784 | WARN_ON(chunk->immutable); | ||
785 | pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size, false); | ||
786 | list_del(&chunk->list); | ||
787 | rb_erase(&chunk->rb_node, &pcpu_addr_root); | ||
788 | free_pcpu_chunk(chunk); | ||
789 | } | ||
790 | |||
791 | /** | ||
792 | * free_percpu - free percpu area | ||
793 | * @ptr: pointer to area to free | ||
794 | * | ||
795 | * Free percpu area @ptr. Might sleep. | ||
796 | */ | ||
797 | void free_percpu(void *ptr) | ||
798 | { | ||
799 | void *addr = __pcpu_ptr_to_addr(ptr); | ||
800 | struct pcpu_chunk *chunk; | ||
801 | int off; | ||
802 | |||
803 | if (!ptr) | ||
804 | return; | ||
805 | |||
806 | mutex_lock(&pcpu_mutex); | ||
807 | |||
808 | chunk = pcpu_chunk_addr_search(addr); | ||
809 | off = addr - chunk->vm->addr; | ||
810 | |||
811 | pcpu_free_area(chunk, off); | ||
812 | |||
813 | /* the chunk became fully free, kill one if there are other free ones */ | ||
814 | if (chunk->free_size == pcpu_unit_size) { | ||
815 | struct pcpu_chunk *pos; | ||
816 | |||
817 | list_for_each_entry(pos, | ||
818 | &pcpu_slot[pcpu_chunk_slot(chunk)], list) | ||
819 | if (pos != chunk) { | ||
820 | pcpu_kill_chunk(pos); | ||
821 | break; | ||
822 | } | ||
823 | } | ||
824 | |||
825 | mutex_unlock(&pcpu_mutex); | ||
826 | } | ||
827 | EXPORT_SYMBOL_GPL(free_percpu); | ||
828 | |||
829 | /** | ||
830 | * pcpu_setup_first_chunk - initialize the first percpu chunk | ||
831 | * @get_page_fn: callback to fetch page pointer | ||
832 | * @static_size: the size of static percpu area in bytes | ||
833 | * @unit_size: unit size in bytes, must be multiple of PAGE_SIZE, 0 for auto | ||
834 | * @free_size: free size in bytes, 0 for auto | ||
835 | * @base_addr: mapped address, NULL for auto | ||
836 | * @populate_pte_fn: callback to allocate pagetable, NULL if unnecessary | ||
837 | * | ||
838 | * Initialize the first percpu chunk which contains the kernel static | ||
839 | * perpcu area. This function is to be called from arch percpu area | ||
840 | * setup path. The first two parameters are mandatory. The rest are | ||
841 | * optional. | ||
842 | * | ||
843 | * @get_page_fn() should return pointer to percpu page given cpu | ||
844 | * number and page number. It should at least return enough pages to | ||
845 | * cover the static area. The returned pages for static area should | ||
846 | * have been initialized with valid data. If @unit_size is specified, | ||
847 | * it can also return pages after the static area. NULL return | ||
848 | * indicates end of pages for the cpu. Note that @get_page_fn() must | ||
849 | * return the same number of pages for all cpus. | ||
850 | * | ||
851 | * @unit_size, if non-zero, determines unit size and must be aligned | ||
852 | * to PAGE_SIZE and equal to or larger than @static_size + @free_size. | ||
853 | * | ||
854 | * @free_size determines the number of free bytes after the static | ||
855 | * area in the first chunk. If zero, whatever left is available. | ||
856 | * Specifying non-zero value make percpu leave the area after | ||
857 | * @static_size + @free_size alone. | ||
858 | * | ||
859 | * Non-null @base_addr means that the caller already allocated virtual | ||
860 | * region for the first chunk and mapped it. percpu must not mess | ||
861 | * with the chunk. Note that @base_addr with 0 @unit_size or non-NULL | ||
862 | * @populate_pte_fn doesn't make any sense. | ||
863 | * | ||
864 | * @populate_pte_fn is used to populate the pagetable. NULL means the | ||
865 | * caller already populated the pagetable. | ||
866 | * | ||
867 | * RETURNS: | ||
868 | * The determined pcpu_unit_size which can be used to initialize | ||
869 | * percpu access. | ||
870 | */ | ||
871 | size_t __init pcpu_setup_first_chunk(pcpu_get_page_fn_t get_page_fn, | ||
872 | size_t static_size, size_t unit_size, | ||
873 | size_t free_size, void *base_addr, | ||
874 | pcpu_populate_pte_fn_t populate_pte_fn) | ||
875 | { | ||
876 | static struct vm_struct static_vm; | ||
877 | struct pcpu_chunk *static_chunk; | ||
878 | unsigned int cpu; | ||
879 | int nr_pages; | ||
880 | int err, i; | ||
881 | |||
882 | /* santiy checks */ | ||
883 | BUG_ON(!static_size); | ||
884 | BUG_ON(!unit_size && free_size); | ||
885 | BUG_ON(unit_size && unit_size < static_size + free_size); | ||
886 | BUG_ON(unit_size & ~PAGE_MASK); | ||
887 | BUG_ON(base_addr && !unit_size); | ||
888 | BUG_ON(base_addr && populate_pte_fn); | ||
889 | |||
890 | if (unit_size) | ||
891 | pcpu_unit_pages = unit_size >> PAGE_SHIFT; | ||
892 | else | ||
893 | pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_SIZE >> PAGE_SHIFT, | ||
894 | PFN_UP(static_size)); | ||
895 | |||
896 | pcpu_static_size = static_size; | ||
897 | pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT; | ||
898 | pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size; | ||
899 | pcpu_chunk_struct_size = sizeof(struct pcpu_chunk) | ||
900 | + num_possible_cpus() * pcpu_unit_pages * sizeof(struct page *); | ||
901 | |||
902 | /* | ||
903 | * Allocate chunk slots. The additional last slot is for | ||
904 | * empty chunks. | ||
905 | */ | ||
906 | pcpu_nr_slots = __pcpu_size_to_slot(pcpu_unit_size) + 2; | ||
907 | pcpu_slot = alloc_bootmem(pcpu_nr_slots * sizeof(pcpu_slot[0])); | ||
908 | for (i = 0; i < pcpu_nr_slots; i++) | ||
909 | INIT_LIST_HEAD(&pcpu_slot[i]); | ||
910 | |||
911 | /* init static_chunk */ | ||
912 | static_chunk = alloc_bootmem(pcpu_chunk_struct_size); | ||
913 | INIT_LIST_HEAD(&static_chunk->list); | ||
914 | static_chunk->vm = &static_vm; | ||
915 | |||
916 | if (free_size) | ||
917 | static_chunk->free_size = free_size; | ||
918 | else | ||
919 | static_chunk->free_size = pcpu_unit_size - pcpu_static_size; | ||
920 | |||
921 | static_chunk->contig_hint = static_chunk->free_size; | ||
922 | |||
923 | /* allocate vm address */ | ||
924 | static_vm.flags = VM_ALLOC; | ||
925 | static_vm.size = pcpu_chunk_size; | ||
926 | |||
927 | if (!base_addr) | ||
928 | vm_area_register_early(&static_vm, PAGE_SIZE); | ||
929 | else { | ||
930 | /* | ||
931 | * Pages already mapped. No need to remap into | ||
932 | * vmalloc area. In this case the static chunk can't | ||
933 | * be mapped or unmapped by percpu and is marked | ||
934 | * immutable. | ||
935 | */ | ||
936 | static_vm.addr = base_addr; | ||
937 | static_chunk->immutable = true; | ||
938 | } | ||
939 | |||
940 | /* assign pages */ | ||
941 | nr_pages = -1; | ||
942 | for_each_possible_cpu(cpu) { | ||
943 | for (i = 0; i < pcpu_unit_pages; i++) { | ||
944 | struct page *page = get_page_fn(cpu, i); | ||
945 | |||
946 | if (!page) | ||
947 | break; | ||
948 | *pcpu_chunk_pagep(static_chunk, cpu, i) = page; | ||
949 | } | ||
950 | |||
951 | BUG_ON(i < PFN_UP(pcpu_static_size)); | ||
952 | |||
953 | if (nr_pages < 0) | ||
954 | nr_pages = i; | ||
955 | else | ||
956 | BUG_ON(nr_pages != i); | ||
957 | } | ||
958 | |||
959 | /* map them */ | ||
960 | if (populate_pte_fn) { | ||
961 | for_each_possible_cpu(cpu) | ||
962 | for (i = 0; i < nr_pages; i++) | ||
963 | populate_pte_fn(pcpu_chunk_addr(static_chunk, | ||
964 | cpu, i)); | ||
965 | |||
966 | err = pcpu_map(static_chunk, 0, nr_pages); | ||
967 | if (err) | ||
968 | panic("failed to setup static percpu area, err=%d\n", | ||
969 | err); | ||
970 | } | ||
971 | |||
972 | /* link static_chunk in */ | ||
973 | pcpu_chunk_relocate(static_chunk, -1); | ||
974 | pcpu_chunk_addr_insert(static_chunk); | ||
975 | |||
976 | /* we're done */ | ||
977 | pcpu_base_addr = (void *)pcpu_chunk_addr(static_chunk, 0, 0); | ||
978 | return pcpu_unit_size; | ||
979 | } | ||
diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 520a75980269..af58324c361a 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c | |||
@@ -24,6 +24,7 @@ | |||
24 | #include <linux/radix-tree.h> | 24 | #include <linux/radix-tree.h> |
25 | #include <linux/rcupdate.h> | 25 | #include <linux/rcupdate.h> |
26 | #include <linux/bootmem.h> | 26 | #include <linux/bootmem.h> |
27 | #include <linux/pfn.h> | ||
27 | 28 | ||
28 | #include <asm/atomic.h> | 29 | #include <asm/atomic.h> |
29 | #include <asm/uaccess.h> | 30 | #include <asm/uaccess.h> |
@@ -152,8 +153,8 @@ static int vmap_pud_range(pgd_t *pgd, unsigned long addr, | |||
152 | * | 153 | * |
153 | * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N] | 154 | * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N] |
154 | */ | 155 | */ |
155 | static int vmap_page_range(unsigned long start, unsigned long end, | 156 | static int vmap_page_range_noflush(unsigned long start, unsigned long end, |
156 | pgprot_t prot, struct page **pages) | 157 | pgprot_t prot, struct page **pages) |
157 | { | 158 | { |
158 | pgd_t *pgd; | 159 | pgd_t *pgd; |
159 | unsigned long next; | 160 | unsigned long next; |
@@ -169,13 +170,22 @@ static int vmap_page_range(unsigned long start, unsigned long end, | |||
169 | if (err) | 170 | if (err) |
170 | break; | 171 | break; |
171 | } while (pgd++, addr = next, addr != end); | 172 | } while (pgd++, addr = next, addr != end); |
172 | flush_cache_vmap(start, end); | ||
173 | 173 | ||
174 | if (unlikely(err)) | 174 | if (unlikely(err)) |
175 | return err; | 175 | return err; |
176 | return nr; | 176 | return nr; |
177 | } | 177 | } |
178 | 178 | ||
179 | static int vmap_page_range(unsigned long start, unsigned long end, | ||
180 | pgprot_t prot, struct page **pages) | ||
181 | { | ||
182 | int ret; | ||
183 | |||
184 | ret = vmap_page_range_noflush(start, end, prot, pages); | ||
185 | flush_cache_vmap(start, end); | ||
186 | return ret; | ||
187 | } | ||
188 | |||
179 | static inline int is_vmalloc_or_module_addr(const void *x) | 189 | static inline int is_vmalloc_or_module_addr(const void *x) |
180 | { | 190 | { |
181 | /* | 191 | /* |
@@ -990,6 +1000,32 @@ void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t pro | |||
990 | } | 1000 | } |
991 | EXPORT_SYMBOL(vm_map_ram); | 1001 | EXPORT_SYMBOL(vm_map_ram); |
992 | 1002 | ||
1003 | /** | ||
1004 | * vm_area_register_early - register vmap area early during boot | ||
1005 | * @vm: vm_struct to register | ||
1006 | * @align: requested alignment | ||
1007 | * | ||
1008 | * This function is used to register kernel vm area before | ||
1009 | * vmalloc_init() is called. @vm->size and @vm->flags should contain | ||
1010 | * proper values on entry and other fields should be zero. On return, | ||
1011 | * vm->addr contains the allocated address. | ||
1012 | * | ||
1013 | * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. | ||
1014 | */ | ||
1015 | void __init vm_area_register_early(struct vm_struct *vm, size_t align) | ||
1016 | { | ||
1017 | static size_t vm_init_off __initdata; | ||
1018 | unsigned long addr; | ||
1019 | |||
1020 | addr = ALIGN(VMALLOC_START + vm_init_off, align); | ||
1021 | vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START; | ||
1022 | |||
1023 | vm->addr = (void *)addr; | ||
1024 | |||
1025 | vm->next = vmlist; | ||
1026 | vmlist = vm; | ||
1027 | } | ||
1028 | |||
993 | void __init vmalloc_init(void) | 1029 | void __init vmalloc_init(void) |
994 | { | 1030 | { |
995 | struct vmap_area *va; | 1031 | struct vmap_area *va; |
@@ -1017,6 +1053,58 @@ void __init vmalloc_init(void) | |||
1017 | vmap_initialized = true; | 1053 | vmap_initialized = true; |
1018 | } | 1054 | } |
1019 | 1055 | ||
1056 | /** | ||
1057 | * map_kernel_range_noflush - map kernel VM area with the specified pages | ||
1058 | * @addr: start of the VM area to map | ||
1059 | * @size: size of the VM area to map | ||
1060 | * @prot: page protection flags to use | ||
1061 | * @pages: pages to map | ||
1062 | * | ||
1063 | * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size | ||
1064 | * specify should have been allocated using get_vm_area() and its | ||
1065 | * friends. | ||
1066 | * | ||
1067 | * NOTE: | ||
1068 | * This function does NOT do any cache flushing. The caller is | ||
1069 | * responsible for calling flush_cache_vmap() on to-be-mapped areas | ||
1070 | * before calling this function. | ||
1071 | * | ||
1072 | * RETURNS: | ||
1073 | * The number of pages mapped on success, -errno on failure. | ||
1074 | */ | ||
1075 | int map_kernel_range_noflush(unsigned long addr, unsigned long size, | ||
1076 | pgprot_t prot, struct page **pages) | ||
1077 | { | ||
1078 | return vmap_page_range_noflush(addr, addr + size, prot, pages); | ||
1079 | } | ||
1080 | |||
1081 | /** | ||
1082 | * unmap_kernel_range_noflush - unmap kernel VM area | ||
1083 | * @addr: start of the VM area to unmap | ||
1084 | * @size: size of the VM area to unmap | ||
1085 | * | ||
1086 | * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size | ||
1087 | * specify should have been allocated using get_vm_area() and its | ||
1088 | * friends. | ||
1089 | * | ||
1090 | * NOTE: | ||
1091 | * This function does NOT do any cache flushing. The caller is | ||
1092 | * responsible for calling flush_cache_vunmap() on to-be-mapped areas | ||
1093 | * before calling this function and flush_tlb_kernel_range() after. | ||
1094 | */ | ||
1095 | void unmap_kernel_range_noflush(unsigned long addr, unsigned long size) | ||
1096 | { | ||
1097 | vunmap_page_range(addr, addr + size); | ||
1098 | } | ||
1099 | |||
1100 | /** | ||
1101 | * unmap_kernel_range - unmap kernel VM area and flush cache and TLB | ||
1102 | * @addr: start of the VM area to unmap | ||
1103 | * @size: size of the VM area to unmap | ||
1104 | * | ||
1105 | * Similar to unmap_kernel_range_noflush() but flushes vcache before | ||
1106 | * the unmapping and tlb after. | ||
1107 | */ | ||
1020 | void unmap_kernel_range(unsigned long addr, unsigned long size) | 1108 | void unmap_kernel_range(unsigned long addr, unsigned long size) |
1021 | { | 1109 | { |
1022 | unsigned long end = addr + size; | 1110 | unsigned long end = addr + size; |
@@ -1267,6 +1355,7 @@ EXPORT_SYMBOL(vfree); | |||
1267 | void vunmap(const void *addr) | 1355 | void vunmap(const void *addr) |
1268 | { | 1356 | { |
1269 | BUG_ON(in_interrupt()); | 1357 | BUG_ON(in_interrupt()); |
1358 | might_sleep(); | ||
1270 | __vunmap(addr, 0); | 1359 | __vunmap(addr, 0); |
1271 | } | 1360 | } |
1272 | EXPORT_SYMBOL(vunmap); | 1361 | EXPORT_SYMBOL(vunmap); |
@@ -1286,6 +1375,8 @@ void *vmap(struct page **pages, unsigned int count, | |||
1286 | { | 1375 | { |
1287 | struct vm_struct *area; | 1376 | struct vm_struct *area; |
1288 | 1377 | ||
1378 | might_sleep(); | ||
1379 | |||
1289 | if (count > num_physpages) | 1380 | if (count > num_physpages) |
1290 | return NULL; | 1381 | return NULL; |
1291 | 1382 | ||