aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/hugetlb.c2
-rw-r--r--mm/memcontrol.c4
-rw-r--r--mm/mlock.c41
-rw-r--r--mm/page_alloc.c2
-rw-r--r--mm/percpu-km.c104
-rw-r--r--mm/percpu-vm.c451
-rw-r--r--mm/percpu.c585
-rw-r--r--mm/rmap.c9
-rw-r--r--mm/slub.c2
9 files changed, 675 insertions, 525 deletions
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index ffbdfc86aedf..4c9e6bbf3772 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1039,7 +1039,7 @@ static struct page *alloc_huge_page(struct vm_area_struct *vma,
1039 page = alloc_buddy_huge_page(h, vma, addr); 1039 page = alloc_buddy_huge_page(h, vma, addr);
1040 if (!page) { 1040 if (!page) {
1041 hugetlb_put_quota(inode->i_mapping, chg); 1041 hugetlb_put_quota(inode->i_mapping, chg);
1042 return ERR_PTR(-VM_FAULT_OOM); 1042 return ERR_PTR(-VM_FAULT_SIGBUS);
1043 } 1043 }
1044 } 1044 }
1045 1045
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 6c755de385f7..c8569bc298ff 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1438,7 +1438,7 @@ static void drain_local_stock(struct work_struct *dummy)
1438 1438
1439/* 1439/*
1440 * Cache charges(val) which is from res_counter, to local per_cpu area. 1440 * Cache charges(val) which is from res_counter, to local per_cpu area.
1441 * This will be consumed by consumt_stock() function, later. 1441 * This will be consumed by consume_stock() function, later.
1442 */ 1442 */
1443static void refill_stock(struct mem_cgroup *mem, int val) 1443static void refill_stock(struct mem_cgroup *mem, int val)
1444{ 1444{
@@ -1601,7 +1601,6 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
1601 * There is a small race that "from" or "to" can be 1601 * There is a small race that "from" or "to" can be
1602 * freed by rmdir, so we use css_tryget(). 1602 * freed by rmdir, so we use css_tryget().
1603 */ 1603 */
1604 rcu_read_lock();
1605 from = mc.from; 1604 from = mc.from;
1606 to = mc.to; 1605 to = mc.to;
1607 if (from && css_tryget(&from->css)) { 1606 if (from && css_tryget(&from->css)) {
@@ -1622,7 +1621,6 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
1622 do_continue = (to == mem_over_limit); 1621 do_continue = (to == mem_over_limit);
1623 css_put(&to->css); 1622 css_put(&to->css);
1624 } 1623 }
1625 rcu_read_unlock();
1626 if (do_continue) { 1624 if (do_continue) {
1627 DEFINE_WAIT(wait); 1625 DEFINE_WAIT(wait);
1628 prepare_to_wait(&mc.waitq, &wait, 1626 prepare_to_wait(&mc.waitq, &wait,
diff --git a/mm/mlock.c b/mm/mlock.c
index 8f4e2dfceec1..3f82720e0515 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -607,44 +607,3 @@ void user_shm_unlock(size_t size, struct user_struct *user)
607 spin_unlock(&shmlock_user_lock); 607 spin_unlock(&shmlock_user_lock);
608 free_uid(user); 608 free_uid(user);
609} 609}
610
611int account_locked_memory(struct mm_struct *mm, struct rlimit *rlim,
612 size_t size)
613{
614 unsigned long lim, vm, pgsz;
615 int error = -ENOMEM;
616
617 pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT;
618
619 down_write(&mm->mmap_sem);
620
621 lim = ACCESS_ONCE(rlim[RLIMIT_AS].rlim_cur) >> PAGE_SHIFT;
622 vm = mm->total_vm + pgsz;
623 if (lim < vm)
624 goto out;
625
626 lim = ACCESS_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur) >> PAGE_SHIFT;
627 vm = mm->locked_vm + pgsz;
628 if (lim < vm)
629 goto out;
630
631 mm->total_vm += pgsz;
632 mm->locked_vm += pgsz;
633
634 error = 0;
635 out:
636 up_write(&mm->mmap_sem);
637 return error;
638}
639
640void refund_locked_memory(struct mm_struct *mm, size_t size)
641{
642 unsigned long pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT;
643
644 down_write(&mm->mmap_sem);
645
646 mm->total_vm -= pgsz;
647 mm->locked_vm -= pgsz;
648
649 up_write(&mm->mmap_sem);
650}
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index d03c946d5566..a6326c71b663 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2579,7 +2579,7 @@ static int default_zonelist_order(void)
2579 struct zone *z; 2579 struct zone *z;
2580 int average_size; 2580 int average_size;
2581 /* 2581 /*
2582 * ZONE_DMA and ZONE_DMA32 can be very small area in the sytem. 2582 * ZONE_DMA and ZONE_DMA32 can be very small area in the system.
2583 * If they are really small and used heavily, the system can fall 2583 * If they are really small and used heavily, the system can fall
2584 * into OOM very easily. 2584 * into OOM very easily.
2585 * This function detect ZONE_DMA/DMA32 size and confgigures zone order. 2585 * This function detect ZONE_DMA/DMA32 size and confgigures zone order.
diff --git a/mm/percpu-km.c b/mm/percpu-km.c
new file mode 100644
index 000000000000..df680855540a
--- /dev/null
+++ b/mm/percpu-km.c
@@ -0,0 +1,104 @@
1/*
2 * mm/percpu-km.c - kernel memory based chunk allocation
3 *
4 * Copyright (C) 2010 SUSE Linux Products GmbH
5 * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
6 *
7 * This file is released under the GPLv2.
8 *
9 * Chunks are allocated as a contiguous kernel memory using gfp
10 * allocation. This is to be used on nommu architectures.
11 *
12 * To use percpu-km,
13 *
14 * - define CONFIG_NEED_PER_CPU_KM from the arch Kconfig.
15 *
16 * - CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK must not be defined. It's
17 * not compatible with PER_CPU_KM. EMBED_FIRST_CHUNK should work
18 * fine.
19 *
20 * - NUMA is not supported. When setting up the first chunk,
21 * @cpu_distance_fn should be NULL or report all CPUs to be nearer
22 * than or at LOCAL_DISTANCE.
23 *
24 * - It's best if the chunk size is power of two multiple of
25 * PAGE_SIZE. Because each chunk is allocated as a contiguous
26 * kernel memory block using alloc_pages(), memory will be wasted if
27 * chunk size is not aligned. percpu-km code will whine about it.
28 */
29
30#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
31#error "contiguous percpu allocation is incompatible with paged first chunk"
32#endif
33
34#include <linux/log2.h>
35
36static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
37{
38 /* noop */
39 return 0;
40}
41
42static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size)
43{
44 /* nada */
45}
46
47static struct pcpu_chunk *pcpu_create_chunk(void)
48{
49 const int nr_pages = pcpu_group_sizes[0] >> PAGE_SHIFT;
50 struct pcpu_chunk *chunk;
51 struct page *pages;
52 int i;
53
54 chunk = pcpu_alloc_chunk();
55 if (!chunk)
56 return NULL;
57
58 pages = alloc_pages(GFP_KERNEL, order_base_2(nr_pages));
59 if (!pages) {
60 pcpu_free_chunk(chunk);
61 return NULL;
62 }
63
64 for (i = 0; i < nr_pages; i++)
65 pcpu_set_page_chunk(nth_page(pages, i), chunk);
66
67 chunk->data = pages;
68 chunk->base_addr = page_address(pages) - pcpu_group_offsets[0];
69 return chunk;
70}
71
72static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)
73{
74 const int nr_pages = pcpu_group_sizes[0] >> PAGE_SHIFT;
75
76 if (chunk && chunk->data)
77 __free_pages(chunk->data, order_base_2(nr_pages));
78 pcpu_free_chunk(chunk);
79}
80
81static struct page *pcpu_addr_to_page(void *addr)
82{
83 return virt_to_page(addr);
84}
85
86static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai)
87{
88 size_t nr_pages, alloc_pages;
89
90 /* all units must be in a single group */
91 if (ai->nr_groups != 1) {
92 printk(KERN_CRIT "percpu: can't handle more than one groups\n");
93 return -EINVAL;
94 }
95
96 nr_pages = (ai->groups[0].nr_units * ai->unit_size) >> PAGE_SHIFT;
97 alloc_pages = roundup_pow_of_two(nr_pages);
98
99 if (alloc_pages > nr_pages)
100 printk(KERN_WARNING "percpu: wasting %zu pages per chunk\n",
101 alloc_pages - nr_pages);
102
103 return 0;
104}
diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c
new file mode 100644
index 000000000000..7d9c1d0ebd3f
--- /dev/null
+++ b/mm/percpu-vm.c
@@ -0,0 +1,451 @@
1/*
2 * mm/percpu-vm.c - vmalloc area based chunk allocation
3 *
4 * Copyright (C) 2010 SUSE Linux Products GmbH
5 * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
6 *
7 * This file is released under the GPLv2.
8 *
9 * Chunks are mapped into vmalloc areas and populated page by page.
10 * This is the default chunk allocator.
11 */
12
13static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
14 unsigned int cpu, int page_idx)
15{
16 /* must not be used on pre-mapped chunk */
17 WARN_ON(chunk->immutable);
18
19 return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));
20}
21
22/**
23 * pcpu_get_pages_and_bitmap - get temp pages array and bitmap
24 * @chunk: chunk of interest
25 * @bitmapp: output parameter for bitmap
26 * @may_alloc: may allocate the array
27 *
28 * Returns pointer to array of pointers to struct page and bitmap,
29 * both of which can be indexed with pcpu_page_idx(). The returned
30 * array is cleared to zero and *@bitmapp is copied from
31 * @chunk->populated. Note that there is only one array and bitmap
32 * and access exclusion is the caller's responsibility.
33 *
34 * CONTEXT:
35 * pcpu_alloc_mutex and does GFP_KERNEL allocation if @may_alloc.
36 * Otherwise, don't care.
37 *
38 * RETURNS:
39 * Pointer to temp pages array on success, NULL on failure.
40 */
41static struct page **pcpu_get_pages_and_bitmap(struct pcpu_chunk *chunk,
42 unsigned long **bitmapp,
43 bool may_alloc)
44{
45 static struct page **pages;
46 static unsigned long *bitmap;
47 size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]);
48 size_t bitmap_size = BITS_TO_LONGS(pcpu_unit_pages) *
49 sizeof(unsigned long);
50
51 if (!pages || !bitmap) {
52 if (may_alloc && !pages)
53 pages = pcpu_mem_alloc(pages_size);
54 if (may_alloc && !bitmap)
55 bitmap = pcpu_mem_alloc(bitmap_size);
56 if (!pages || !bitmap)
57 return NULL;
58 }
59
60 memset(pages, 0, pages_size);
61 bitmap_copy(bitmap, chunk->populated, pcpu_unit_pages);
62
63 *bitmapp = bitmap;
64 return pages;
65}
66
67/**
68 * pcpu_free_pages - free pages which were allocated for @chunk
69 * @chunk: chunk pages were allocated for
70 * @pages: array of pages to be freed, indexed by pcpu_page_idx()
71 * @populated: populated bitmap
72 * @page_start: page index of the first page to be freed
73 * @page_end: page index of the last page to be freed + 1
74 *
75 * Free pages [@page_start and @page_end) in @pages for all units.
76 * The pages were allocated for @chunk.
77 */
78static void pcpu_free_pages(struct pcpu_chunk *chunk,
79 struct page **pages, unsigned long *populated,
80 int page_start, int page_end)
81{
82 unsigned int cpu;
83 int i;
84
85 for_each_possible_cpu(cpu) {
86 for (i = page_start; i < page_end; i++) {
87 struct page *page = pages[pcpu_page_idx(cpu, i)];
88
89 if (page)
90 __free_page(page);
91 }
92 }
93}
94
95/**
96 * pcpu_alloc_pages - allocates pages for @chunk
97 * @chunk: target chunk
98 * @pages: array to put the allocated pages into, indexed by pcpu_page_idx()
99 * @populated: populated bitmap
100 * @page_start: page index of the first page to be allocated
101 * @page_end: page index of the last page to be allocated + 1
102 *
103 * Allocate pages [@page_start,@page_end) into @pages for all units.
104 * The allocation is for @chunk. Percpu core doesn't care about the
105 * content of @pages and will pass it verbatim to pcpu_map_pages().
106 */
107static int pcpu_alloc_pages(struct pcpu_chunk *chunk,
108 struct page **pages, unsigned long *populated,
109 int page_start, int page_end)
110{
111 const gfp_t gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_COLD;
112 unsigned int cpu;
113 int i;
114
115 for_each_possible_cpu(cpu) {
116 for (i = page_start; i < page_end; i++) {
117 struct page **pagep = &pages[pcpu_page_idx(cpu, i)];
118
119 *pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0);
120 if (!*pagep) {
121 pcpu_free_pages(chunk, pages, populated,
122 page_start, page_end);
123 return -ENOMEM;
124 }
125 }
126 }
127 return 0;
128}
129
130/**
131 * pcpu_pre_unmap_flush - flush cache prior to unmapping
132 * @chunk: chunk the regions to be flushed belongs to
133 * @page_start: page index of the first page to be flushed
134 * @page_end: page index of the last page to be flushed + 1
135 *
136 * Pages in [@page_start,@page_end) of @chunk are about to be
137 * unmapped. Flush cache. As each flushing trial can be very
138 * expensive, issue flush on the whole region at once rather than
139 * doing it for each cpu. This could be an overkill but is more
140 * scalable.
141 */
142static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,
143 int page_start, int page_end)
144{
145 flush_cache_vunmap(
146 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
147 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
148}
149
150static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)
151{
152 unmap_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT);
153}
154
155/**
156 * pcpu_unmap_pages - unmap pages out of a pcpu_chunk
157 * @chunk: chunk of interest
158 * @pages: pages array which can be used to pass information to free
159 * @populated: populated bitmap
160 * @page_start: page index of the first page to unmap
161 * @page_end: page index of the last page to unmap + 1
162 *
163 * For each cpu, unmap pages [@page_start,@page_end) out of @chunk.
164 * Corresponding elements in @pages were cleared by the caller and can
165 * be used to carry information to pcpu_free_pages() which will be
166 * called after all unmaps are finished. The caller should call
167 * proper pre/post flush functions.
168 */
169static void pcpu_unmap_pages(struct pcpu_chunk *chunk,
170 struct page **pages, unsigned long *populated,
171 int page_start, int page_end)
172{
173 unsigned int cpu;
174 int i;
175
176 for_each_possible_cpu(cpu) {
177 for (i = page_start; i < page_end; i++) {
178 struct page *page;
179
180 page = pcpu_chunk_page(chunk, cpu, i);
181 WARN_ON(!page);
182 pages[pcpu_page_idx(cpu, i)] = page;
183 }
184 __pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),
185 page_end - page_start);
186 }
187
188 for (i = page_start; i < page_end; i++)
189 __clear_bit(i, populated);
190}
191
192/**
193 * pcpu_post_unmap_tlb_flush - flush TLB after unmapping
194 * @chunk: pcpu_chunk the regions to be flushed belong to
195 * @page_start: page index of the first page to be flushed
196 * @page_end: page index of the last page to be flushed + 1
197 *
198 * Pages [@page_start,@page_end) of @chunk have been unmapped. Flush
199 * TLB for the regions. This can be skipped if the area is to be
200 * returned to vmalloc as vmalloc will handle TLB flushing lazily.
201 *
202 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
203 * for the whole region.
204 */
205static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
206 int page_start, int page_end)
207{
208 flush_tlb_kernel_range(
209 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
210 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
211}
212
213static int __pcpu_map_pages(unsigned long addr, struct page **pages,
214 int nr_pages)
215{
216 return map_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT,
217 PAGE_KERNEL, pages);
218}
219
220/**
221 * pcpu_map_pages - map pages into a pcpu_chunk
222 * @chunk: chunk of interest
223 * @pages: pages array containing pages to be mapped
224 * @populated: populated bitmap
225 * @page_start: page index of the first page to map
226 * @page_end: page index of the last page to map + 1
227 *
228 * For each cpu, map pages [@page_start,@page_end) into @chunk. The
229 * caller is responsible for calling pcpu_post_map_flush() after all
230 * mappings are complete.
231 *
232 * This function is responsible for setting corresponding bits in
233 * @chunk->populated bitmap and whatever is necessary for reverse
234 * lookup (addr -> chunk).
235 */
236static int pcpu_map_pages(struct pcpu_chunk *chunk,
237 struct page **pages, unsigned long *populated,
238 int page_start, int page_end)
239{
240 unsigned int cpu, tcpu;
241 int i, err;
242
243 for_each_possible_cpu(cpu) {
244 err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start),
245 &pages[pcpu_page_idx(cpu, page_start)],
246 page_end - page_start);
247 if (err < 0)
248 goto err;
249 }
250
251 /* mapping successful, link chunk and mark populated */
252 for (i = page_start; i < page_end; i++) {
253 for_each_possible_cpu(cpu)
254 pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)],
255 chunk);
256 __set_bit(i, populated);
257 }
258
259 return 0;
260
261err:
262 for_each_possible_cpu(tcpu) {
263 if (tcpu == cpu)
264 break;
265 __pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),
266 page_end - page_start);
267 }
268 return err;
269}
270
271/**
272 * pcpu_post_map_flush - flush cache after mapping
273 * @chunk: pcpu_chunk the regions to be flushed belong to
274 * @page_start: page index of the first page to be flushed
275 * @page_end: page index of the last page to be flushed + 1
276 *
277 * Pages [@page_start,@page_end) of @chunk have been mapped. Flush
278 * cache.
279 *
280 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
281 * for the whole region.
282 */
283static void pcpu_post_map_flush(struct pcpu_chunk *chunk,
284 int page_start, int page_end)
285{
286 flush_cache_vmap(
287 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
288 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
289}
290
291/**
292 * pcpu_populate_chunk - populate and map an area of a pcpu_chunk
293 * @chunk: chunk of interest
294 * @off: offset to the area to populate
295 * @size: size of the area to populate in bytes
296 *
297 * For each cpu, populate and map pages [@page_start,@page_end) into
298 * @chunk. The area is cleared on return.
299 *
300 * CONTEXT:
301 * pcpu_alloc_mutex, does GFP_KERNEL allocation.
302 */
303static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
304{
305 int page_start = PFN_DOWN(off);
306 int page_end = PFN_UP(off + size);
307 int free_end = page_start, unmap_end = page_start;
308 struct page **pages;
309 unsigned long *populated;
310 unsigned int cpu;
311 int rs, re, rc;
312
313 /* quick path, check whether all pages are already there */
314 rs = page_start;
315 pcpu_next_pop(chunk, &rs, &re, page_end);
316 if (rs == page_start && re == page_end)
317 goto clear;
318
319 /* need to allocate and map pages, this chunk can't be immutable */
320 WARN_ON(chunk->immutable);
321
322 pages = pcpu_get_pages_and_bitmap(chunk, &populated, true);
323 if (!pages)
324 return -ENOMEM;
325
326 /* alloc and map */
327 pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
328 rc = pcpu_alloc_pages(chunk, pages, populated, rs, re);
329 if (rc)
330 goto err_free;
331 free_end = re;
332 }
333
334 pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
335 rc = pcpu_map_pages(chunk, pages, populated, rs, re);
336 if (rc)
337 goto err_unmap;
338 unmap_end = re;
339 }
340 pcpu_post_map_flush(chunk, page_start, page_end);
341
342 /* commit new bitmap */
343 bitmap_copy(chunk->populated, populated, pcpu_unit_pages);
344clear:
345 for_each_possible_cpu(cpu)
346 memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size);
347 return 0;
348
349err_unmap:
350 pcpu_pre_unmap_flush(chunk, page_start, unmap_end);
351 pcpu_for_each_unpop_region(chunk, rs, re, page_start, unmap_end)
352 pcpu_unmap_pages(chunk, pages, populated, rs, re);
353 pcpu_post_unmap_tlb_flush(chunk, page_start, unmap_end);
354err_free:
355 pcpu_for_each_unpop_region(chunk, rs, re, page_start, free_end)
356 pcpu_free_pages(chunk, pages, populated, rs, re);
357 return rc;
358}
359
360/**
361 * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
362 * @chunk: chunk to depopulate
363 * @off: offset to the area to depopulate
364 * @size: size of the area to depopulate in bytes
365 * @flush: whether to flush cache and tlb or not
366 *
367 * For each cpu, depopulate and unmap pages [@page_start,@page_end)
368 * from @chunk. If @flush is true, vcache is flushed before unmapping
369 * and tlb after.
370 *
371 * CONTEXT:
372 * pcpu_alloc_mutex.
373 */
374static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size)
375{
376 int page_start = PFN_DOWN(off);
377 int page_end = PFN_UP(off + size);
378 struct page **pages;
379 unsigned long *populated;
380 int rs, re;
381
382 /* quick path, check whether it's empty already */
383 rs = page_start;
384 pcpu_next_unpop(chunk, &rs, &re, page_end);
385 if (rs == page_start && re == page_end)
386 return;
387
388 /* immutable chunks can't be depopulated */
389 WARN_ON(chunk->immutable);
390
391 /*
392 * If control reaches here, there must have been at least one
393 * successful population attempt so the temp pages array must
394 * be available now.
395 */
396 pages = pcpu_get_pages_and_bitmap(chunk, &populated, false);
397 BUG_ON(!pages);
398
399 /* unmap and free */
400 pcpu_pre_unmap_flush(chunk, page_start, page_end);
401
402 pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end)
403 pcpu_unmap_pages(chunk, pages, populated, rs, re);
404
405 /* no need to flush tlb, vmalloc will handle it lazily */
406
407 pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end)
408 pcpu_free_pages(chunk, pages, populated, rs, re);
409
410 /* commit new bitmap */
411 bitmap_copy(chunk->populated, populated, pcpu_unit_pages);
412}
413
414static struct pcpu_chunk *pcpu_create_chunk(void)
415{
416 struct pcpu_chunk *chunk;
417 struct vm_struct **vms;
418
419 chunk = pcpu_alloc_chunk();
420 if (!chunk)
421 return NULL;
422
423 vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes,
424 pcpu_nr_groups, pcpu_atom_size, GFP_KERNEL);
425 if (!vms) {
426 pcpu_free_chunk(chunk);
427 return NULL;
428 }
429
430 chunk->data = vms;
431 chunk->base_addr = vms[0]->addr - pcpu_group_offsets[0];
432 return chunk;
433}
434
435static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)
436{
437 if (chunk && chunk->data)
438 pcpu_free_vm_areas(chunk->data, pcpu_nr_groups);
439 pcpu_free_chunk(chunk);
440}
441
442static struct page *pcpu_addr_to_page(void *addr)
443{
444 return vmalloc_to_page(addr);
445}
446
447static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai)
448{
449 /* no extra restriction */
450 return 0;
451}
diff --git a/mm/percpu.c b/mm/percpu.c
index 6e09741ddc62..39f7dfd59585 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * linux/mm/percpu.c - percpu memory allocator 2 * mm/percpu.c - percpu memory allocator
3 * 3 *
4 * Copyright (C) 2009 SUSE Linux Products GmbH 4 * Copyright (C) 2009 SUSE Linux Products GmbH
5 * Copyright (C) 2009 Tejun Heo <tj@kernel.org> 5 * Copyright (C) 2009 Tejun Heo <tj@kernel.org>
@@ -7,14 +7,13 @@
7 * This file is released under the GPLv2. 7 * This file is released under the GPLv2.
8 * 8 *
9 * This is percpu allocator which can handle both static and dynamic 9 * This is percpu allocator which can handle both static and dynamic
10 * areas. Percpu areas are allocated in chunks in vmalloc area. Each 10 * areas. Percpu areas are allocated in chunks. Each chunk is
11 * chunk is consisted of boot-time determined number of units and the 11 * consisted of boot-time determined number of units and the first
12 * first chunk is used for static percpu variables in the kernel image 12 * chunk is used for static percpu variables in the kernel image
13 * (special boot time alloc/init handling necessary as these areas 13 * (special boot time alloc/init handling necessary as these areas
14 * need to be brought up before allocation services are running). 14 * need to be brought up before allocation services are running).
15 * Unit grows as necessary and all units grow or shrink in unison. 15 * Unit grows as necessary and all units grow or shrink in unison.
16 * When a chunk is filled up, another chunk is allocated. ie. in 16 * When a chunk is filled up, another chunk is allocated.
17 * vmalloc area
18 * 17 *
19 * c0 c1 c2 18 * c0 c1 c2
20 * ------------------- ------------------- ------------ 19 * ------------------- ------------------- ------------
@@ -99,7 +98,7 @@ struct pcpu_chunk {
99 int map_used; /* # of map entries used */ 98 int map_used; /* # of map entries used */
100 int map_alloc; /* # of map entries allocated */ 99 int map_alloc; /* # of map entries allocated */
101 int *map; /* allocation map */ 100 int *map; /* allocation map */
102 struct vm_struct **vms; /* mapped vmalloc regions */ 101 void *data; /* chunk data */
103 bool immutable; /* no [de]population allowed */ 102 bool immutable; /* no [de]population allowed */
104 unsigned long populated[]; /* populated bitmap */ 103 unsigned long populated[]; /* populated bitmap */
105}; 104};
@@ -177,6 +176,21 @@ static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */
177static void pcpu_reclaim(struct work_struct *work); 176static void pcpu_reclaim(struct work_struct *work);
178static DECLARE_WORK(pcpu_reclaim_work, pcpu_reclaim); 177static DECLARE_WORK(pcpu_reclaim_work, pcpu_reclaim);
179 178
179static bool pcpu_addr_in_first_chunk(void *addr)
180{
181 void *first_start = pcpu_first_chunk->base_addr;
182
183 return addr >= first_start && addr < first_start + pcpu_unit_size;
184}
185
186static bool pcpu_addr_in_reserved_chunk(void *addr)
187{
188 void *first_start = pcpu_first_chunk->base_addr;
189
190 return addr >= first_start &&
191 addr < first_start + pcpu_reserved_chunk_limit;
192}
193
180static int __pcpu_size_to_slot(int size) 194static int __pcpu_size_to_slot(int size)
181{ 195{
182 int highbit = fls(size); /* size is in bytes */ 196 int highbit = fls(size); /* size is in bytes */
@@ -198,27 +212,6 @@ static int pcpu_chunk_slot(const struct pcpu_chunk *chunk)
198 return pcpu_size_to_slot(chunk->free_size); 212 return pcpu_size_to_slot(chunk->free_size);
199} 213}
200 214
201static int pcpu_page_idx(unsigned int cpu, int page_idx)
202{
203 return pcpu_unit_map[cpu] * pcpu_unit_pages + page_idx;
204}
205
206static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk,
207 unsigned int cpu, int page_idx)
208{
209 return (unsigned long)chunk->base_addr + pcpu_unit_offsets[cpu] +
210 (page_idx << PAGE_SHIFT);
211}
212
213static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
214 unsigned int cpu, int page_idx)
215{
216 /* must not be used on pre-mapped chunk */
217 WARN_ON(chunk->immutable);
218
219 return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));
220}
221
222/* set the pointer to a chunk in a page struct */ 215/* set the pointer to a chunk in a page struct */
223static void pcpu_set_page_chunk(struct page *page, struct pcpu_chunk *pcpu) 216static void pcpu_set_page_chunk(struct page *page, struct pcpu_chunk *pcpu)
224{ 217{
@@ -231,13 +224,27 @@ static struct pcpu_chunk *pcpu_get_page_chunk(struct page *page)
231 return (struct pcpu_chunk *)page->index; 224 return (struct pcpu_chunk *)page->index;
232} 225}
233 226
234static void pcpu_next_unpop(struct pcpu_chunk *chunk, int *rs, int *re, int end) 227static int __maybe_unused pcpu_page_idx(unsigned int cpu, int page_idx)
228{
229 return pcpu_unit_map[cpu] * pcpu_unit_pages + page_idx;
230}
231
232static unsigned long __maybe_unused pcpu_chunk_addr(struct pcpu_chunk *chunk,
233 unsigned int cpu, int page_idx)
234{
235 return (unsigned long)chunk->base_addr + pcpu_unit_offsets[cpu] +
236 (page_idx << PAGE_SHIFT);
237}
238
239static void __maybe_unused pcpu_next_unpop(struct pcpu_chunk *chunk,
240 int *rs, int *re, int end)
235{ 241{
236 *rs = find_next_zero_bit(chunk->populated, end, *rs); 242 *rs = find_next_zero_bit(chunk->populated, end, *rs);
237 *re = find_next_bit(chunk->populated, end, *rs + 1); 243 *re = find_next_bit(chunk->populated, end, *rs + 1);
238} 244}
239 245
240static void pcpu_next_pop(struct pcpu_chunk *chunk, int *rs, int *re, int end) 246static void __maybe_unused pcpu_next_pop(struct pcpu_chunk *chunk,
247 int *rs, int *re, int end)
241{ 248{
242 *rs = find_next_bit(chunk->populated, end, *rs); 249 *rs = find_next_bit(chunk->populated, end, *rs);
243 *re = find_next_zero_bit(chunk->populated, end, *rs + 1); 250 *re = find_next_zero_bit(chunk->populated, end, *rs + 1);
@@ -326,36 +333,6 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
326} 333}
327 334
328/** 335/**
329 * pcpu_chunk_addr_search - determine chunk containing specified address
330 * @addr: address for which the chunk needs to be determined.
331 *
332 * RETURNS:
333 * The address of the found chunk.
334 */
335static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
336{
337 void *first_start = pcpu_first_chunk->base_addr;
338
339 /* is it in the first chunk? */
340 if (addr >= first_start && addr < first_start + pcpu_unit_size) {
341 /* is it in the reserved area? */
342 if (addr < first_start + pcpu_reserved_chunk_limit)
343 return pcpu_reserved_chunk;
344 return pcpu_first_chunk;
345 }
346
347 /*
348 * The address is relative to unit0 which might be unused and
349 * thus unmapped. Offset the address to the unit space of the
350 * current processor before looking it up in the vmalloc
351 * space. Note that any possible cpu id can be used here, so
352 * there's no need to worry about preemption or cpu hotplug.
353 */
354 addr += pcpu_unit_offsets[raw_smp_processor_id()];
355 return pcpu_get_page_chunk(vmalloc_to_page(addr));
356}
357
358/**
359 * pcpu_need_to_extend - determine whether chunk area map needs to be extended 336 * pcpu_need_to_extend - determine whether chunk area map needs to be extended
360 * @chunk: chunk of interest 337 * @chunk: chunk of interest
361 * 338 *
@@ -623,434 +600,92 @@ static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
623 pcpu_chunk_relocate(chunk, oslot); 600 pcpu_chunk_relocate(chunk, oslot);
624} 601}
625 602
626/** 603static struct pcpu_chunk *pcpu_alloc_chunk(void)
627 * pcpu_get_pages_and_bitmap - get temp pages array and bitmap
628 * @chunk: chunk of interest
629 * @bitmapp: output parameter for bitmap
630 * @may_alloc: may allocate the array
631 *
632 * Returns pointer to array of pointers to struct page and bitmap,
633 * both of which can be indexed with pcpu_page_idx(). The returned
634 * array is cleared to zero and *@bitmapp is copied from
635 * @chunk->populated. Note that there is only one array and bitmap
636 * and access exclusion is the caller's responsibility.
637 *
638 * CONTEXT:
639 * pcpu_alloc_mutex and does GFP_KERNEL allocation if @may_alloc.
640 * Otherwise, don't care.
641 *
642 * RETURNS:
643 * Pointer to temp pages array on success, NULL on failure.
644 */
645static struct page **pcpu_get_pages_and_bitmap(struct pcpu_chunk *chunk,
646 unsigned long **bitmapp,
647 bool may_alloc)
648{
649 static struct page **pages;
650 static unsigned long *bitmap;
651 size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]);
652 size_t bitmap_size = BITS_TO_LONGS(pcpu_unit_pages) *
653 sizeof(unsigned long);
654
655 if (!pages || !bitmap) {
656 if (may_alloc && !pages)
657 pages = pcpu_mem_alloc(pages_size);
658 if (may_alloc && !bitmap)
659 bitmap = pcpu_mem_alloc(bitmap_size);
660 if (!pages || !bitmap)
661 return NULL;
662 }
663
664 memset(pages, 0, pages_size);
665 bitmap_copy(bitmap, chunk->populated, pcpu_unit_pages);
666
667 *bitmapp = bitmap;
668 return pages;
669}
670
671/**
672 * pcpu_free_pages - free pages which were allocated for @chunk
673 * @chunk: chunk pages were allocated for
674 * @pages: array of pages to be freed, indexed by pcpu_page_idx()
675 * @populated: populated bitmap
676 * @page_start: page index of the first page to be freed
677 * @page_end: page index of the last page to be freed + 1
678 *
679 * Free pages [@page_start and @page_end) in @pages for all units.
680 * The pages were allocated for @chunk.
681 */
682static void pcpu_free_pages(struct pcpu_chunk *chunk,
683 struct page **pages, unsigned long *populated,
684 int page_start, int page_end)
685{ 604{
686 unsigned int cpu; 605 struct pcpu_chunk *chunk;
687 int i;
688 606
689 for_each_possible_cpu(cpu) { 607 chunk = kzalloc(pcpu_chunk_struct_size, GFP_KERNEL);
690 for (i = page_start; i < page_end; i++) { 608 if (!chunk)
691 struct page *page = pages[pcpu_page_idx(cpu, i)]; 609 return NULL;
692 610
693 if (page) 611 chunk->map = pcpu_mem_alloc(PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
694 __free_page(page); 612 if (!chunk->map) {
695 } 613 kfree(chunk);
614 return NULL;
696 } 615 }
697}
698 616
699/** 617 chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
700 * pcpu_alloc_pages - allocates pages for @chunk 618 chunk->map[chunk->map_used++] = pcpu_unit_size;
701 * @chunk: target chunk
702 * @pages: array to put the allocated pages into, indexed by pcpu_page_idx()
703 * @populated: populated bitmap
704 * @page_start: page index of the first page to be allocated
705 * @page_end: page index of the last page to be allocated + 1
706 *
707 * Allocate pages [@page_start,@page_end) into @pages for all units.
708 * The allocation is for @chunk. Percpu core doesn't care about the
709 * content of @pages and will pass it verbatim to pcpu_map_pages().
710 */
711static int pcpu_alloc_pages(struct pcpu_chunk *chunk,
712 struct page **pages, unsigned long *populated,
713 int page_start, int page_end)
714{
715 const gfp_t gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_COLD;
716 unsigned int cpu;
717 int i;
718 619
719 for_each_possible_cpu(cpu) { 620 INIT_LIST_HEAD(&chunk->list);
720 for (i = page_start; i < page_end; i++) { 621 chunk->free_size = pcpu_unit_size;
721 struct page **pagep = &pages[pcpu_page_idx(cpu, i)]; 622 chunk->contig_hint = pcpu_unit_size;
722
723 *pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0);
724 if (!*pagep) {
725 pcpu_free_pages(chunk, pages, populated,
726 page_start, page_end);
727 return -ENOMEM;
728 }
729 }
730 }
731 return 0;
732}
733 623
734/** 624 return chunk;
735 * pcpu_pre_unmap_flush - flush cache prior to unmapping
736 * @chunk: chunk the regions to be flushed belongs to
737 * @page_start: page index of the first page to be flushed
738 * @page_end: page index of the last page to be flushed + 1
739 *
740 * Pages in [@page_start,@page_end) of @chunk are about to be
741 * unmapped. Flush cache. As each flushing trial can be very
742 * expensive, issue flush on the whole region at once rather than
743 * doing it for each cpu. This could be an overkill but is more
744 * scalable.
745 */
746static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,
747 int page_start, int page_end)
748{
749 flush_cache_vunmap(
750 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
751 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
752} 625}
753 626
754static void __pcpu_unmap_pages(unsigned long addr, int nr_pages) 627static void pcpu_free_chunk(struct pcpu_chunk *chunk)
755{ 628{
756 unmap_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT); 629 if (!chunk)
630 return;
631 pcpu_mem_free(chunk->map, chunk->map_alloc * sizeof(chunk->map[0]));
632 kfree(chunk);
757} 633}
758 634
759/** 635/*
760 * pcpu_unmap_pages - unmap pages out of a pcpu_chunk 636 * Chunk management implementation.
761 * @chunk: chunk of interest 637 *
762 * @pages: pages array which can be used to pass information to free 638 * To allow different implementations, chunk alloc/free and
763 * @populated: populated bitmap 639 * [de]population are implemented in a separate file which is pulled
764 * @page_start: page index of the first page to unmap 640 * into this file and compiled together. The following functions
765 * @page_end: page index of the last page to unmap + 1 641 * should be implemented.
766 * 642 *
767 * For each cpu, unmap pages [@page_start,@page_end) out of @chunk. 643 * pcpu_populate_chunk - populate the specified range of a chunk
768 * Corresponding elements in @pages were cleared by the caller and can 644 * pcpu_depopulate_chunk - depopulate the specified range of a chunk
769 * be used to carry information to pcpu_free_pages() which will be 645 * pcpu_create_chunk - create a new chunk
770 * called after all unmaps are finished. The caller should call 646 * pcpu_destroy_chunk - destroy a chunk, always preceded by full depop
771 * proper pre/post flush functions. 647 * pcpu_addr_to_page - translate address to physical address
648 * pcpu_verify_alloc_info - check alloc_info is acceptable during init
772 */ 649 */
773static void pcpu_unmap_pages(struct pcpu_chunk *chunk, 650static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size);
774 struct page **pages, unsigned long *populated, 651static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size);
775 int page_start, int page_end) 652static struct pcpu_chunk *pcpu_create_chunk(void);
776{ 653static void pcpu_destroy_chunk(struct pcpu_chunk *chunk);
777 unsigned int cpu; 654static struct page *pcpu_addr_to_page(void *addr);
778 int i; 655static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai);
779 656
780 for_each_possible_cpu(cpu) { 657#ifdef CONFIG_NEED_PER_CPU_KM
781 for (i = page_start; i < page_end; i++) { 658#include "percpu-km.c"
782 struct page *page; 659#else
783 660#include "percpu-vm.c"
784 page = pcpu_chunk_page(chunk, cpu, i); 661#endif
785 WARN_ON(!page);
786 pages[pcpu_page_idx(cpu, i)] = page;
787 }
788 __pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),
789 page_end - page_start);
790 }
791
792 for (i = page_start; i < page_end; i++)
793 __clear_bit(i, populated);
794}
795 662
796/** 663/**
797 * pcpu_post_unmap_tlb_flush - flush TLB after unmapping 664 * pcpu_chunk_addr_search - determine chunk containing specified address
798 * @chunk: pcpu_chunk the regions to be flushed belong to 665 * @addr: address for which the chunk needs to be determined.
799 * @page_start: page index of the first page to be flushed
800 * @page_end: page index of the last page to be flushed + 1
801 *
802 * Pages [@page_start,@page_end) of @chunk have been unmapped. Flush
803 * TLB for the regions. This can be skipped if the area is to be
804 * returned to vmalloc as vmalloc will handle TLB flushing lazily.
805 * 666 *
806 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once 667 * RETURNS:
807 * for the whole region. 668 * The address of the found chunk.
808 */
809static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
810 int page_start, int page_end)
811{
812 flush_tlb_kernel_range(
813 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
814 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
815}
816
817static int __pcpu_map_pages(unsigned long addr, struct page **pages,
818 int nr_pages)
819{
820 return map_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT,
821 PAGE_KERNEL, pages);
822}
823
824/**
825 * pcpu_map_pages - map pages into a pcpu_chunk
826 * @chunk: chunk of interest
827 * @pages: pages array containing pages to be mapped
828 * @populated: populated bitmap
829 * @page_start: page index of the first page to map
830 * @page_end: page index of the last page to map + 1
831 *
832 * For each cpu, map pages [@page_start,@page_end) into @chunk. The
833 * caller is responsible for calling pcpu_post_map_flush() after all
834 * mappings are complete.
835 *
836 * This function is responsible for setting corresponding bits in
837 * @chunk->populated bitmap and whatever is necessary for reverse
838 * lookup (addr -> chunk).
839 */ 669 */
840static int pcpu_map_pages(struct pcpu_chunk *chunk, 670static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
841 struct page **pages, unsigned long *populated,
842 int page_start, int page_end)
843{ 671{
844 unsigned int cpu, tcpu; 672 /* is it in the first chunk? */
845 int i, err; 673 if (pcpu_addr_in_first_chunk(addr)) {
846 674 /* is it in the reserved area? */
847 for_each_possible_cpu(cpu) { 675 if (pcpu_addr_in_reserved_chunk(addr))
848 err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start), 676 return pcpu_reserved_chunk;
849 &pages[pcpu_page_idx(cpu, page_start)], 677 return pcpu_first_chunk;
850 page_end - page_start);
851 if (err < 0)
852 goto err;
853 }
854
855 /* mapping successful, link chunk and mark populated */
856 for (i = page_start; i < page_end; i++) {
857 for_each_possible_cpu(cpu)
858 pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)],
859 chunk);
860 __set_bit(i, populated);
861 }
862
863 return 0;
864
865err:
866 for_each_possible_cpu(tcpu) {
867 if (tcpu == cpu)
868 break;
869 __pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),
870 page_end - page_start);
871 } 678 }
872 return err;
873}
874
875/**
876 * pcpu_post_map_flush - flush cache after mapping
877 * @chunk: pcpu_chunk the regions to be flushed belong to
878 * @page_start: page index of the first page to be flushed
879 * @page_end: page index of the last page to be flushed + 1
880 *
881 * Pages [@page_start,@page_end) of @chunk have been mapped. Flush
882 * cache.
883 *
884 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
885 * for the whole region.
886 */
887static void pcpu_post_map_flush(struct pcpu_chunk *chunk,
888 int page_start, int page_end)
889{
890 flush_cache_vmap(
891 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
892 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
893}
894
895/**
896 * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
897 * @chunk: chunk to depopulate
898 * @off: offset to the area to depopulate
899 * @size: size of the area to depopulate in bytes
900 * @flush: whether to flush cache and tlb or not
901 *
902 * For each cpu, depopulate and unmap pages [@page_start,@page_end)
903 * from @chunk. If @flush is true, vcache is flushed before unmapping
904 * and tlb after.
905 *
906 * CONTEXT:
907 * pcpu_alloc_mutex.
908 */
909static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size)
910{
911 int page_start = PFN_DOWN(off);
912 int page_end = PFN_UP(off + size);
913 struct page **pages;
914 unsigned long *populated;
915 int rs, re;
916
917 /* quick path, check whether it's empty already */
918 rs = page_start;
919 pcpu_next_unpop(chunk, &rs, &re, page_end);
920 if (rs == page_start && re == page_end)
921 return;
922
923 /* immutable chunks can't be depopulated */
924 WARN_ON(chunk->immutable);
925 679
926 /* 680 /*
927 * If control reaches here, there must have been at least one 681 * The address is relative to unit0 which might be unused and
928 * successful population attempt so the temp pages array must 682 * thus unmapped. Offset the address to the unit space of the
929 * be available now. 683 * current processor before looking it up in the vmalloc
684 * space. Note that any possible cpu id can be used here, so
685 * there's no need to worry about preemption or cpu hotplug.
930 */ 686 */
931 pages = pcpu_get_pages_and_bitmap(chunk, &populated, false); 687 addr += pcpu_unit_offsets[raw_smp_processor_id()];
932 BUG_ON(!pages); 688 return pcpu_get_page_chunk(pcpu_addr_to_page(addr));
933
934 /* unmap and free */
935 pcpu_pre_unmap_flush(chunk, page_start, page_end);
936
937 pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end)
938 pcpu_unmap_pages(chunk, pages, populated, rs, re);
939
940 /* no need to flush tlb, vmalloc will handle it lazily */
941
942 pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end)
943 pcpu_free_pages(chunk, pages, populated, rs, re);
944
945 /* commit new bitmap */
946 bitmap_copy(chunk->populated, populated, pcpu_unit_pages);
947}
948
949/**
950 * pcpu_populate_chunk - populate and map an area of a pcpu_chunk
951 * @chunk: chunk of interest
952 * @off: offset to the area to populate
953 * @size: size of the area to populate in bytes
954 *
955 * For each cpu, populate and map pages [@page_start,@page_end) into
956 * @chunk. The area is cleared on return.
957 *
958 * CONTEXT:
959 * pcpu_alloc_mutex, does GFP_KERNEL allocation.
960 */
961static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
962{
963 int page_start = PFN_DOWN(off);
964 int page_end = PFN_UP(off + size);
965 int free_end = page_start, unmap_end = page_start;
966 struct page **pages;
967 unsigned long *populated;
968 unsigned int cpu;
969 int rs, re, rc;
970
971 /* quick path, check whether all pages are already there */
972 rs = page_start;
973 pcpu_next_pop(chunk, &rs, &re, page_end);
974 if (rs == page_start && re == page_end)
975 goto clear;
976
977 /* need to allocate and map pages, this chunk can't be immutable */
978 WARN_ON(chunk->immutable);
979
980 pages = pcpu_get_pages_and_bitmap(chunk, &populated, true);
981 if (!pages)
982 return -ENOMEM;
983
984 /* alloc and map */
985 pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
986 rc = pcpu_alloc_pages(chunk, pages, populated, rs, re);
987 if (rc)
988 goto err_free;
989 free_end = re;
990 }
991
992 pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
993 rc = pcpu_map_pages(chunk, pages, populated, rs, re);
994 if (rc)
995 goto err_unmap;
996 unmap_end = re;
997 }
998 pcpu_post_map_flush(chunk, page_start, page_end);
999
1000 /* commit new bitmap */
1001 bitmap_copy(chunk->populated, populated, pcpu_unit_pages);
1002clear:
1003 for_each_possible_cpu(cpu)
1004 memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size);
1005 return 0;
1006
1007err_unmap:
1008 pcpu_pre_unmap_flush(chunk, page_start, unmap_end);
1009 pcpu_for_each_unpop_region(chunk, rs, re, page_start, unmap_end)
1010 pcpu_unmap_pages(chunk, pages, populated, rs, re);
1011 pcpu_post_unmap_tlb_flush(chunk, page_start, unmap_end);
1012err_free:
1013 pcpu_for_each_unpop_region(chunk, rs, re, page_start, free_end)
1014 pcpu_free_pages(chunk, pages, populated, rs, re);
1015 return rc;
1016}
1017
1018static void free_pcpu_chunk(struct pcpu_chunk *chunk)
1019{
1020 if (!chunk)
1021 return;
1022 if (chunk->vms)
1023 pcpu_free_vm_areas(chunk->vms, pcpu_nr_groups);
1024 pcpu_mem_free(chunk->map, chunk->map_alloc * sizeof(chunk->map[0]));
1025 kfree(chunk);
1026}
1027
1028static struct pcpu_chunk *alloc_pcpu_chunk(void)
1029{
1030 struct pcpu_chunk *chunk;
1031
1032 chunk = kzalloc(pcpu_chunk_struct_size, GFP_KERNEL);
1033 if (!chunk)
1034 return NULL;
1035
1036 chunk->map = pcpu_mem_alloc(PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
1037 chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
1038 chunk->map[chunk->map_used++] = pcpu_unit_size;
1039
1040 chunk->vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes,
1041 pcpu_nr_groups, pcpu_atom_size,
1042 GFP_KERNEL);
1043 if (!chunk->vms) {
1044 free_pcpu_chunk(chunk);
1045 return NULL;
1046 }
1047
1048 INIT_LIST_HEAD(&chunk->list);
1049 chunk->free_size = pcpu_unit_size;
1050 chunk->contig_hint = pcpu_unit_size;
1051 chunk->base_addr = chunk->vms[0]->addr - pcpu_group_offsets[0];
1052
1053 return chunk;
1054} 689}
1055 690
1056/** 691/**
@@ -1142,7 +777,7 @@ restart:
1142 /* hmmm... no space left, create a new chunk */ 777 /* hmmm... no space left, create a new chunk */
1143 spin_unlock_irqrestore(&pcpu_lock, flags); 778 spin_unlock_irqrestore(&pcpu_lock, flags);
1144 779
1145 chunk = alloc_pcpu_chunk(); 780 chunk = pcpu_create_chunk();
1146 if (!chunk) { 781 if (!chunk) {
1147 err = "failed to allocate new chunk"; 782 err = "failed to allocate new chunk";
1148 goto fail_unlock_mutex; 783 goto fail_unlock_mutex;
@@ -1254,7 +889,7 @@ static void pcpu_reclaim(struct work_struct *work)
1254 889
1255 list_for_each_entry_safe(chunk, next, &todo, list) { 890 list_for_each_entry_safe(chunk, next, &todo, list) {
1256 pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size); 891 pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size);
1257 free_pcpu_chunk(chunk); 892 pcpu_destroy_chunk(chunk);
1258 } 893 }
1259 894
1260 mutex_unlock(&pcpu_alloc_mutex); 895 mutex_unlock(&pcpu_alloc_mutex);
@@ -1343,11 +978,14 @@ bool is_kernel_percpu_address(unsigned long addr)
1343 */ 978 */
1344phys_addr_t per_cpu_ptr_to_phys(void *addr) 979phys_addr_t per_cpu_ptr_to_phys(void *addr)
1345{ 980{
1346 if ((unsigned long)addr < VMALLOC_START || 981 if (pcpu_addr_in_first_chunk(addr)) {
1347 (unsigned long)addr >= VMALLOC_END) 982 if ((unsigned long)addr < VMALLOC_START ||
1348 return __pa(addr); 983 (unsigned long)addr >= VMALLOC_END)
1349 else 984 return __pa(addr);
1350 return page_to_phys(vmalloc_to_page(addr)); 985 else
986 return page_to_phys(vmalloc_to_page(addr));
987 } else
988 return page_to_phys(pcpu_addr_to_page(addr));
1351} 989}
1352 990
1353static inline size_t pcpu_calc_fc_sizes(size_t static_size, 991static inline size_t pcpu_calc_fc_sizes(size_t static_size,
@@ -1719,6 +1357,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
1719 PCPU_SETUP_BUG_ON(ai->unit_size < size_sum); 1357 PCPU_SETUP_BUG_ON(ai->unit_size < size_sum);
1720 PCPU_SETUP_BUG_ON(ai->unit_size & ~PAGE_MASK); 1358 PCPU_SETUP_BUG_ON(ai->unit_size & ~PAGE_MASK);
1721 PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE); 1359 PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE);
1360 PCPU_SETUP_BUG_ON(pcpu_verify_alloc_info(ai) < 0);
1722 1361
1723 /* process group information and build config tables accordingly */ 1362 /* process group information and build config tables accordingly */
1724 group_offsets = alloc_bootmem(ai->nr_groups * sizeof(group_offsets[0])); 1363 group_offsets = alloc_bootmem(ai->nr_groups * sizeof(group_offsets[0]));
diff --git a/mm/rmap.c b/mm/rmap.c
index 07fc94758799..0feeef860a8f 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -336,14 +336,13 @@ vma_address(struct page *page, struct vm_area_struct *vma)
336 336
337/* 337/*
338 * At what user virtual address is page expected in vma? 338 * At what user virtual address is page expected in vma?
339 * checking that the page matches the vma. 339 * Caller should check the page is actually part of the vma.
340 */ 340 */
341unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) 341unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
342{ 342{
343 if (PageAnon(page)) { 343 if (PageAnon(page))
344 if (vma->anon_vma != page_anon_vma(page)) 344 ;
345 return -EFAULT; 345 else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) {
346 } else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) {
347 if (!vma->vm_file || 346 if (!vma->vm_file ||
348 vma->vm_file->f_mapping != page->mapping) 347 vma->vm_file->f_mapping != page->mapping)
349 return -EFAULT; 348 return -EFAULT;
diff --git a/mm/slub.c b/mm/slub.c
index 7d6c8b1ccf63..d2a54fe71ea2 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2153,7 +2153,7 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
2153 int local_node; 2153 int local_node;
2154 2154
2155 if (slab_state >= UP && (s < kmalloc_caches || 2155 if (slab_state >= UP && (s < kmalloc_caches ||
2156 s > kmalloc_caches + KMALLOC_CACHES)) 2156 s >= kmalloc_caches + KMALLOC_CACHES))
2157 local_node = page_to_nid(virt_to_page(s)); 2157 local_node = page_to_nid(virt_to_page(s));
2158 else 2158 else
2159 local_node = 0; 2159 local_node = 0;
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-18 10:25:00 -0500