aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/backing-dev.c49
-rw-r--r--mm/hugetlb.c7
-rw-r--r--mm/ksm.c12
-rw-r--r--mm/memcontrol.c8
-rw-r--r--mm/mlock.c41
-rw-r--r--mm/mmap.c3
-rw-r--r--mm/msync.c2
-rw-r--r--mm/page-writeback.c44
-rw-r--r--mm/page_alloc.c2
-rw-r--r--mm/percpu-km.c104
-rw-r--r--mm/percpu-vm.c451
-rw-r--r--mm/percpu.c585
-rw-r--r--mm/rmap.c45
-rw-r--r--mm/shmem.c29
-rw-r--r--mm/slab.c190
-rw-r--r--mm/slob.c8
-rw-r--r--mm/slub.c48
-rw-r--r--mm/swapfile.c14
18 files changed, 956 insertions, 686 deletions
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index f13e067e146..660a87a2251 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -11,6 +11,8 @@
11#include <linux/writeback.h> 11#include <linux/writeback.h>
12#include <linux/device.h> 12#include <linux/device.h>
13 13
14static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0);
15
14void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page) 16void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page)
15{ 17{
16} 18}
@@ -25,6 +27,11 @@ struct backing_dev_info default_backing_dev_info = {
25}; 27};
26EXPORT_SYMBOL_GPL(default_backing_dev_info); 28EXPORT_SYMBOL_GPL(default_backing_dev_info);
27 29
30struct backing_dev_info noop_backing_dev_info = {
31 .name = "noop",
32};
33EXPORT_SYMBOL_GPL(noop_backing_dev_info);
34
28static struct class *bdi_class; 35static struct class *bdi_class;
29 36
30/* 37/*
@@ -41,7 +48,6 @@ static struct timer_list sync_supers_timer;
41 48
42static int bdi_sync_supers(void *); 49static int bdi_sync_supers(void *);
43static void sync_supers_timer_fn(unsigned long); 50static void sync_supers_timer_fn(unsigned long);
44static void arm_supers_timer(void);
45 51
46static void bdi_add_default_flusher_task(struct backing_dev_info *bdi); 52static void bdi_add_default_flusher_task(struct backing_dev_info *bdi);
47 53
@@ -245,7 +251,7 @@ static int __init default_bdi_init(void)
245 251
246 init_timer(&sync_supers_timer); 252 init_timer(&sync_supers_timer);
247 setup_timer(&sync_supers_timer, sync_supers_timer_fn, 0); 253 setup_timer(&sync_supers_timer, sync_supers_timer_fn, 0);
248 arm_supers_timer(); 254 bdi_arm_supers_timer();
249 255
250 err = bdi_init(&default_backing_dev_info); 256 err = bdi_init(&default_backing_dev_info);
251 if (!err) 257 if (!err)
@@ -367,10 +373,13 @@ static int bdi_sync_supers(void *unused)
367 return 0; 373 return 0;
368} 374}
369 375
370static void arm_supers_timer(void) 376void bdi_arm_supers_timer(void)
371{ 377{
372 unsigned long next; 378 unsigned long next;
373 379
380 if (!dirty_writeback_interval)
381 return;
382
374 next = msecs_to_jiffies(dirty_writeback_interval * 10) + jiffies; 383 next = msecs_to_jiffies(dirty_writeback_interval * 10) + jiffies;
375 mod_timer(&sync_supers_timer, round_jiffies_up(next)); 384 mod_timer(&sync_supers_timer, round_jiffies_up(next));
376} 385}
@@ -378,7 +387,7 @@ static void arm_supers_timer(void)
378static void sync_supers_timer_fn(unsigned long unused) 387static void sync_supers_timer_fn(unsigned long unused)
379{ 388{
380 wake_up_process(sync_supers_tsk); 389 wake_up_process(sync_supers_tsk);
381 arm_supers_timer(); 390 bdi_arm_supers_timer();
382} 391}
383 392
384static int bdi_forker_task(void *ptr) 393static int bdi_forker_task(void *ptr)
@@ -421,7 +430,10 @@ static int bdi_forker_task(void *ptr)
421 430
422 spin_unlock_bh(&bdi_lock); 431 spin_unlock_bh(&bdi_lock);
423 wait = msecs_to_jiffies(dirty_writeback_interval * 10); 432 wait = msecs_to_jiffies(dirty_writeback_interval * 10);
424 schedule_timeout(wait); 433 if (wait)
434 schedule_timeout(wait);
435 else
436 schedule();
425 try_to_freeze(); 437 try_to_freeze();
426 continue; 438 continue;
427 } 439 }
@@ -715,6 +727,33 @@ void bdi_destroy(struct backing_dev_info *bdi)
715} 727}
716EXPORT_SYMBOL(bdi_destroy); 728EXPORT_SYMBOL(bdi_destroy);
717 729
730/*
731 * For use from filesystems to quickly init and register a bdi associated
732 * with dirty writeback
733 */
734int bdi_setup_and_register(struct backing_dev_info *bdi, char *name,
735 unsigned int cap)
736{
737 char tmp[32];
738 int err;
739
740 bdi->name = name;
741 bdi->capabilities = cap;
742 err = bdi_init(bdi);
743 if (err)
744 return err;
745
746 sprintf(tmp, "%.28s%s", name, "-%d");
747 err = bdi_register(bdi, NULL, tmp, atomic_long_inc_return(&bdi_seq));
748 if (err) {
749 bdi_destroy(bdi);
750 return err;
751 }
752
753 return 0;
754}
755EXPORT_SYMBOL(bdi_setup_and_register);
756
718static wait_queue_head_t congestion_wqh[2] = { 757static wait_queue_head_t congestion_wqh[2] = {
719 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]), 758 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[0]),
720 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1]) 759 __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1])
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 6034dc9e979..4c9e6bbf377 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -546,6 +546,7 @@ static void free_huge_page(struct page *page)
546 546
547 mapping = (struct address_space *) page_private(page); 547 mapping = (struct address_space *) page_private(page);
548 set_page_private(page, 0); 548 set_page_private(page, 0);
549 page->mapping = NULL;
549 BUG_ON(page_count(page)); 550 BUG_ON(page_count(page));
550 INIT_LIST_HEAD(&page->lru); 551 INIT_LIST_HEAD(&page->lru);
551 552
@@ -1038,7 +1039,7 @@ static struct page *alloc_huge_page(struct vm_area_struct *vma,
1038 page = alloc_buddy_huge_page(h, vma, addr); 1039 page = alloc_buddy_huge_page(h, vma, addr);
1039 if (!page) { 1040 if (!page) {
1040 hugetlb_put_quota(inode->i_mapping, chg); 1041 hugetlb_put_quota(inode->i_mapping, chg);
1041 return ERR_PTR(-VM_FAULT_OOM); 1042 return ERR_PTR(-VM_FAULT_SIGBUS);
1042 } 1043 }
1043 } 1044 }
1044 1045
@@ -2447,8 +2448,10 @@ retry:
2447 spin_lock(&inode->i_lock); 2448 spin_lock(&inode->i_lock);
2448 inode->i_blocks += blocks_per_huge_page(h); 2449 inode->i_blocks += blocks_per_huge_page(h);
2449 spin_unlock(&inode->i_lock); 2450 spin_unlock(&inode->i_lock);
2450 } else 2451 } else {
2451 lock_page(page); 2452 lock_page(page);
2453 page->mapping = HUGETLB_POISON;
2454 }
2452 } 2455 }
2453 2456
2454 /* 2457 /*
diff --git a/mm/ksm.c b/mm/ksm.c
index 8cdfc2a1e8b..956880f2ff4 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -365,7 +365,7 @@ static int break_ksm(struct vm_area_struct *vma, unsigned long addr)
365 do { 365 do {
366 cond_resched(); 366 cond_resched();
367 page = follow_page(vma, addr, FOLL_GET); 367 page = follow_page(vma, addr, FOLL_GET);
368 if (!page) 368 if (IS_ERR_OR_NULL(page))
369 break; 369 break;
370 if (PageKsm(page)) 370 if (PageKsm(page))
371 ret = handle_mm_fault(vma->vm_mm, vma, addr, 371 ret = handle_mm_fault(vma->vm_mm, vma, addr,
@@ -447,7 +447,7 @@ static struct page *get_mergeable_page(struct rmap_item *rmap_item)
447 goto out; 447 goto out;
448 448
449 page = follow_page(vma, addr, FOLL_GET); 449 page = follow_page(vma, addr, FOLL_GET);
450 if (!page) 450 if (IS_ERR_OR_NULL(page))
451 goto out; 451 goto out;
452 if (PageAnon(page)) { 452 if (PageAnon(page)) {
453 flush_anon_page(vma, page, addr); 453 flush_anon_page(vma, page, addr);
@@ -1086,7 +1086,7 @@ struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item,
1086 cond_resched(); 1086 cond_resched();
1087 tree_rmap_item = rb_entry(*new, struct rmap_item, node); 1087 tree_rmap_item = rb_entry(*new, struct rmap_item, node);
1088 tree_page = get_mergeable_page(tree_rmap_item); 1088 tree_page = get_mergeable_page(tree_rmap_item);
1089 if (!tree_page) 1089 if (IS_ERR_OR_NULL(tree_page))
1090 return NULL; 1090 return NULL;
1091 1091
1092 /* 1092 /*
@@ -1294,7 +1294,7 @@ next_mm:
1294 if (ksm_test_exit(mm)) 1294 if (ksm_test_exit(mm))
1295 break; 1295 break;
1296 *page = follow_page(vma, ksm_scan.address, FOLL_GET); 1296 *page = follow_page(vma, ksm_scan.address, FOLL_GET);
1297 if (*page && PageAnon(*page)) { 1297 if (!IS_ERR_OR_NULL(*page) && PageAnon(*page)) {
1298 flush_anon_page(vma, *page, ksm_scan.address); 1298 flush_anon_page(vma, *page, ksm_scan.address);
1299 flush_dcache_page(*page); 1299 flush_dcache_page(*page);
1300 rmap_item = get_next_rmap_item(slot, 1300 rmap_item = get_next_rmap_item(slot,
@@ -1308,7 +1308,7 @@ next_mm:
1308 up_read(&mm->mmap_sem); 1308 up_read(&mm->mmap_sem);
1309 return rmap_item; 1309 return rmap_item;
1310 } 1310 }
1311 if (*page) 1311 if (!IS_ERR_OR_NULL(*page))
1312 put_page(*page); 1312 put_page(*page);
1313 ksm_scan.address += PAGE_SIZE; 1313 ksm_scan.address += PAGE_SIZE;
1314 cond_resched(); 1314 cond_resched();
@@ -1367,7 +1367,7 @@ next_mm:
1367static void ksm_do_scan(unsigned int scan_npages) 1367static void ksm_do_scan(unsigned int scan_npages)
1368{ 1368{
1369 struct rmap_item *rmap_item; 1369 struct rmap_item *rmap_item;
1370 struct page *page; 1370 struct page *uninitialized_var(page);
1371 1371
1372 while (scan_npages--) { 1372 while (scan_npages--) {
1373 cond_resched(); 1373 cond_resched();
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index f4ede99c8b9..c8569bc298f 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1438,7 +1438,7 @@ static void drain_local_stock(struct work_struct *dummy)
1438 1438
1439/* 1439/*
1440 * Cache charges(val) which is from res_counter, to local per_cpu area. 1440 * Cache charges(val) which is from res_counter, to local per_cpu area.
1441 * This will be consumed by consumt_stock() function, later. 1441 * This will be consumed by consume_stock() function, later.
1442 */ 1442 */
1443static void refill_stock(struct mem_cgroup *mem, int val) 1443static void refill_stock(struct mem_cgroup *mem, int val)
1444{ 1444{
@@ -1601,7 +1601,6 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
1601 * There is a small race that "from" or "to" can be 1601 * There is a small race that "from" or "to" can be
1602 * freed by rmdir, so we use css_tryget(). 1602 * freed by rmdir, so we use css_tryget().
1603 */ 1603 */
1604 rcu_read_lock();
1605 from = mc.from; 1604 from = mc.from;
1606 to = mc.to; 1605 to = mc.to;
1607 if (from && css_tryget(&from->css)) { 1606 if (from && css_tryget(&from->css)) {
@@ -1622,7 +1621,6 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
1622 do_continue = (to == mem_over_limit); 1621 do_continue = (to == mem_over_limit);
1623 css_put(&to->css); 1622 css_put(&to->css);
1624 } 1623 }
1625 rcu_read_unlock();
1626 if (do_continue) { 1624 if (do_continue) {
1627 DEFINE_WAIT(wait); 1625 DEFINE_WAIT(wait);
1628 prepare_to_wait(&mc.waitq, &wait, 1626 prepare_to_wait(&mc.waitq, &wait,
@@ -2429,11 +2427,11 @@ int mem_cgroup_prepare_migration(struct page *page, struct mem_cgroup **ptr)
2429 } 2427 }
2430 unlock_page_cgroup(pc); 2428 unlock_page_cgroup(pc);
2431 2429
2430 *ptr = mem;
2432 if (mem) { 2431 if (mem) {
2433 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false); 2432 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, ptr, false);
2434 css_put(&mem->css); 2433 css_put(&mem->css);
2435 } 2434 }
2436 *ptr = mem;
2437 return ret; 2435 return ret;
2438} 2436}
2439 2437
diff --git a/mm/mlock.c b/mm/mlock.c
index 8f4e2dfceec..3f82720e051 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -607,44 +607,3 @@ void user_shm_unlock(size_t size, struct user_struct *user)
607 spin_unlock(&shmlock_user_lock); 607 spin_unlock(&shmlock_user_lock);
608 free_uid(user); 608 free_uid(user);
609} 609}
610
611int account_locked_memory(struct mm_struct *mm, struct rlimit *rlim,
612 size_t size)
613{
614 unsigned long lim, vm, pgsz;
615 int error = -ENOMEM;
616
617 pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT;
618
619 down_write(&mm->mmap_sem);
620
621 lim = ACCESS_ONCE(rlim[RLIMIT_AS].rlim_cur) >> PAGE_SHIFT;
622 vm = mm->total_vm + pgsz;
623 if (lim < vm)
624 goto out;
625
626 lim = ACCESS_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur) >> PAGE_SHIFT;
627 vm = mm->locked_vm + pgsz;
628 if (lim < vm)
629 goto out;
630
631 mm->total_vm += pgsz;
632 mm->locked_vm += pgsz;
633
634 error = 0;
635 out:
636 up_write(&mm->mmap_sem);
637 return error;
638}
639
640void refund_locked_memory(struct mm_struct *mm, size_t size)
641{
642 unsigned long pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT;
643
644 down_write(&mm->mmap_sem);
645
646 mm->total_vm -= pgsz;
647 mm->locked_vm -= pgsz;
648
649 up_write(&mm->mmap_sem);
650}
diff --git a/mm/mmap.c b/mm/mmap.c
index f90ea92f755..456ec6f2788 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -1977,7 +1977,8 @@ static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
1977 return 0; 1977 return 0;
1978 1978
1979 /* Clean everything up if vma_adjust failed. */ 1979 /* Clean everything up if vma_adjust failed. */
1980 new->vm_ops->close(new); 1980 if (new->vm_ops && new->vm_ops->close)
1981 new->vm_ops->close(new);
1981 if (new->vm_file) { 1982 if (new->vm_file) {
1982 if (vma->vm_flags & VM_EXECUTABLE) 1983 if (vma->vm_flags & VM_EXECUTABLE)
1983 removed_exe_file_vma(mm); 1984 removed_exe_file_vma(mm);
diff --git a/mm/msync.c b/mm/msync.c
index 4083209b7f0..632df4527c0 100644
--- a/mm/msync.c
+++ b/mm/msync.c
@@ -82,7 +82,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags)
82 (vma->vm_flags & VM_SHARED)) { 82 (vma->vm_flags & VM_SHARED)) {
83 get_file(file); 83 get_file(file);
84 up_read(&mm->mmap_sem); 84 up_read(&mm->mmap_sem);
85 error = vfs_fsync(file, file->f_path.dentry, 0); 85 error = vfs_fsync(file, 0);
86 fput(file); 86 fput(file);
87 if (error || start >= end) 87 if (error || start >= end)
88 goto out; 88 goto out;
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 0b19943ecf8..b289310e2c8 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -597,7 +597,7 @@ static void balance_dirty_pages(struct address_space *mapping,
597 (!laptop_mode && ((global_page_state(NR_FILE_DIRTY) 597 (!laptop_mode && ((global_page_state(NR_FILE_DIRTY)
598 + global_page_state(NR_UNSTABLE_NFS)) 598 + global_page_state(NR_UNSTABLE_NFS))
599 > background_thresh))) 599 > background_thresh)))
600 bdi_start_writeback(bdi, NULL, 0); 600 bdi_start_writeback(bdi, NULL, 0, 0);
601} 601}
602 602
603void set_page_dirty_balance(struct page *page, int page_mkwrite) 603void set_page_dirty_balance(struct page *page, int page_mkwrite)
@@ -683,10 +683,6 @@ void throttle_vm_writeout(gfp_t gfp_mask)
683 } 683 }
684} 684}
685 685
686static void laptop_timer_fn(unsigned long unused);
687
688static DEFINE_TIMER(laptop_mode_wb_timer, laptop_timer_fn, 0, 0);
689
690/* 686/*
691 * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs 687 * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs
692 */ 688 */
@@ -694,24 +690,24 @@ int dirty_writeback_centisecs_handler(ctl_table *table, int write,
694 void __user *buffer, size_t *length, loff_t *ppos) 690 void __user *buffer, size_t *length, loff_t *ppos)
695{ 691{
696 proc_dointvec(table, write, buffer, length, ppos); 692 proc_dointvec(table, write, buffer, length, ppos);
693 bdi_arm_supers_timer();
697 return 0; 694 return 0;
698} 695}
699 696
700static void do_laptop_sync(struct work_struct *work) 697#ifdef CONFIG_BLOCK
698void laptop_mode_timer_fn(unsigned long data)
701{ 699{
702 wakeup_flusher_threads(0); 700 struct request_queue *q = (struct request_queue *)data;
703 kfree(work); 701 int nr_pages = global_page_state(NR_FILE_DIRTY) +
704} 702 global_page_state(NR_UNSTABLE_NFS);
705 703
706static void laptop_timer_fn(unsigned long unused) 704 /*
707{ 705 * We want to write everything out, not just down to the dirty
708 struct work_struct *work; 706 * threshold
707 */
709 708
710 work = kmalloc(sizeof(*work), GFP_ATOMIC); 709 if (bdi_has_dirty_io(&q->backing_dev_info))
711 if (work) { 710 bdi_start_writeback(&q->backing_dev_info, NULL, nr_pages, 0);
712 INIT_WORK(work, do_laptop_sync);
713 schedule_work(work);
714 }
715} 711}
716 712
717/* 713/*
@@ -719,9 +715,9 @@ static void laptop_timer_fn(unsigned long unused)
719 * of all dirty data a few seconds from now. If the flush is already scheduled 715 * of all dirty data a few seconds from now. If the flush is already scheduled
720 * then push it back - the user is still using the disk. 716 * then push it back - the user is still using the disk.
721 */ 717 */
722void laptop_io_completion(void) 718void laptop_io_completion(struct backing_dev_info *info)
723{ 719{
724 mod_timer(&laptop_mode_wb_timer, jiffies + laptop_mode); 720 mod_timer(&info->laptop_mode_wb_timer, jiffies + laptop_mode);
725} 721}
726 722
727/* 723/*
@@ -731,8 +727,16 @@ void laptop_io_completion(void)
731 */ 727 */
732void laptop_sync_completion(void) 728void laptop_sync_completion(void)
733{ 729{
734 del_timer(&laptop_mode_wb_timer); 730 struct backing_dev_info *bdi;
731
732 rcu_read_lock();
733
734 list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
735 del_timer(&bdi->laptop_mode_wb_timer);
736
737 rcu_read_unlock();
735} 738}
739#endif
736 740
737/* 741/*
738 * If ratelimit_pages is too high then we can get into dirty-data overload 742 * If ratelimit_pages is too high then we can get into dirty-data overload
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index d03c946d556..a6326c71b66 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2579,7 +2579,7 @@ static int default_zonelist_order(void)
2579 struct zone *z; 2579 struct zone *z;
2580 int average_size; 2580 int average_size;
2581 /* 2581 /*
2582 * ZONE_DMA and ZONE_DMA32 can be very small area in the sytem. 2582 * ZONE_DMA and ZONE_DMA32 can be very small area in the system.
2583 * If they are really small and used heavily, the system can fall 2583 * If they are really small and used heavily, the system can fall
2584 * into OOM very easily. 2584 * into OOM very easily.
2585 * This function detect ZONE_DMA/DMA32 size and confgigures zone order. 2585 * This function detect ZONE_DMA/DMA32 size and confgigures zone order.
diff --git a/mm/percpu-km.c b/mm/percpu-km.c
new file mode 100644
index 00000000000..df680855540
--- /dev/null
+++ b/mm/percpu-km.c
@@ -0,0 +1,104 @@
1/*
2 * mm/percpu-km.c - kernel memory based chunk allocation
3 *
4 * Copyright (C) 2010 SUSE Linux Products GmbH
5 * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
6 *
7 * This file is released under the GPLv2.
8 *
9 * Chunks are allocated as a contiguous kernel memory using gfp
10 * allocation. This is to be used on nommu architectures.
11 *
12 * To use percpu-km,
13 *
14 * - define CONFIG_NEED_PER_CPU_KM from the arch Kconfig.
15 *
16 * - CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK must not be defined. It's
17 * not compatible with PER_CPU_KM. EMBED_FIRST_CHUNK should work
18 * fine.
19 *
20 * - NUMA is not supported. When setting up the first chunk,
21 * @cpu_distance_fn should be NULL or report all CPUs to be nearer
22 * than or at LOCAL_DISTANCE.
23 *
24 * - It's best if the chunk size is power of two multiple of
25 * PAGE_SIZE. Because each chunk is allocated as a contiguous
26 * kernel memory block using alloc_pages(), memory will be wasted if
27 * chunk size is not aligned. percpu-km code will whine about it.
28 */
29
30#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
31#error "contiguous percpu allocation is incompatible with paged first chunk"
32#endif
33
34#include <linux/log2.h>
35
36static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
37{
38 /* noop */
39 return 0;
40}
41
42static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size)
43{
44 /* nada */
45}
46
47static struct pcpu_chunk *pcpu_create_chunk(void)
48{
49 const int nr_pages = pcpu_group_sizes[0] >> PAGE_SHIFT;
50 struct pcpu_chunk *chunk;
51 struct page *pages;
52 int i;
53
54 chunk = pcpu_alloc_chunk();
55 if (!chunk)
56 return NULL;
57
58 pages = alloc_pages(GFP_KERNEL, order_base_2(nr_pages));
59 if (!pages) {
60 pcpu_free_chunk(chunk);
61 return NULL;
62 }
63
64 for (i = 0; i < nr_pages; i++)
65 pcpu_set_page_chunk(nth_page(pages, i), chunk);
66
67 chunk->data = pages;
68 chunk->base_addr = page_address(pages) - pcpu_group_offsets[0];
69 return chunk;
70}
71
72static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)
73{
74 const int nr_pages = pcpu_group_sizes[0] >> PAGE_SHIFT;
75
76 if (chunk && chunk->data)
77 __free_pages(chunk->data, order_base_2(nr_pages));
78 pcpu_free_chunk(chunk);
79}
80
81static struct page *pcpu_addr_to_page(void *addr)
82{
83 return virt_to_page(addr);
84}
85
86static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai)
87{
88 size_t nr_pages, alloc_pages;
89
90 /* all units must be in a single group */
91 if (ai->nr_groups != 1) {
92 printk(KERN_CRIT "percpu: can't handle more than one groups\n");
93 return -EINVAL;
94 }
95
96 nr_pages = (ai->groups[0].nr_units * ai->unit_size) >> PAGE_SHIFT;
97 alloc_pages = roundup_pow_of_two(nr_pages);
98
99 if (alloc_pages > nr_pages)
100 printk(KERN_WARNING "percpu: wasting %zu pages per chunk\n",
101 alloc_pages - nr_pages);
102
103 return 0;
104}
diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c
new file mode 100644
index 00000000000..7d9c1d0ebd3
--- /dev/null
+++ b/mm/percpu-vm.c
@@ -0,0 +1,451 @@
1/*
2 * mm/percpu-vm.c - vmalloc area based chunk allocation
3 *
4 * Copyright (C) 2010 SUSE Linux Products GmbH
5 * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
6 *
7 * This file is released under the GPLv2.
8 *
9 * Chunks are mapped into vmalloc areas and populated page by page.
10 * This is the default chunk allocator.
11 */
12
13static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
14 unsigned int cpu, int page_idx)
15{
16 /* must not be used on pre-mapped chunk */
17 WARN_ON(chunk->immutable);
18
19 return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));
20}
21
22/**
23 * pcpu_get_pages_and_bitmap - get temp pages array and bitmap
24 * @chunk: chunk of interest
25 * @bitmapp: output parameter for bitmap
26 * @may_alloc: may allocate the array
27 *
28 * Returns pointer to array of pointers to struct page and bitmap,
29 * both of which can be indexed with pcpu_page_idx(). The returned
30 * array is cleared to zero and *@bitmapp is copied from
31 * @chunk->populated. Note that there is only one array and bitmap
32 * and access exclusion is the caller's responsibility.
33 *
34 * CONTEXT:
35 * pcpu_alloc_mutex and does GFP_KERNEL allocation if @may_alloc.
36 * Otherwise, don't care.
37 *
38 * RETURNS:
39 * Pointer to temp pages array on success, NULL on failure.
40 */
41static struct page **pcpu_get_pages_and_bitmap(struct pcpu_chunk *chunk,
42 unsigned long **bitmapp,
43 bool may_alloc)
44{
45 static struct page **pages;
46 static unsigned long *bitmap;
47 size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]);
48 size_t bitmap_size = BITS_TO_LONGS(pcpu_unit_pages) *
49 sizeof(unsigned long);
50
51 if (!pages || !bitmap) {
52 if (may_alloc && !pages)
53 pages = pcpu_mem_alloc(pages_size);
54 if (may_alloc && !bitmap)
55 bitmap = pcpu_mem_alloc(bitmap_size);
56 if (!pages || !bitmap)
57 return NULL;
58 }
59
60 memset(pages, 0, pages_size);
61 bitmap_copy(bitmap, chunk->populated, pcpu_unit_pages);
62
63 *bitmapp = bitmap;
64 return pages;
65}
66
67/**
68 * pcpu_free_pages - free pages which were allocated for @chunk
69 * @chunk: chunk pages were allocated for
70 * @pages: array of pages to be freed, indexed by pcpu_page_idx()
71 * @populated: populated bitmap
72 * @page_start: page index of the first page to be freed
73 * @page_end: page index of the last page to be freed + 1
74 *
75 * Free pages [@page_start and @page_end) in @pages for all units.
76 * The pages were allocated for @chunk.
77 */
78static void pcpu_free_pages(struct pcpu_chunk *chunk,
79 struct page **pages, unsigned long *populated,
80 int page_start, int page_end)
81{
82 unsigned int cpu;
83 int i;
84
85 for_each_possible_cpu(cpu) {
86 for (i = page_start; i < page_end; i++) {
87 struct page *page = pages[pcpu_page_idx(cpu, i)];
88
89 if (page)
90 __free_page(page);
91 }
92 }
93}
94
95/**
96 * pcpu_alloc_pages - allocates pages for @chunk
97 * @chunk: target chunk
98 * @pages: array to put the allocated pages into, indexed by pcpu_page_idx()
99 * @populated: populated bitmap
100 * @page_start: page index of the first page to be allocated
101 * @page_end: page index of the last page to be allocated + 1
102 *
103 * Allocate pages [@page_start,@page_end) into @pages for all units.
104 * The allocation is for @chunk. Percpu core doesn't care about the
105 * content of @pages and will pass it verbatim to pcpu_map_pages().
106 */
107static int pcpu_alloc_pages(struct pcpu_chunk *chunk,
108 struct page **pages, unsigned long *populated,
109 int page_start, int page_end)
110{
111 const gfp_t gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_COLD;
112 unsigned int cpu;
113 int i;
114
115 for_each_possible_cpu(cpu) {
116 for (i = page_start; i < page_end; i++) {
117 struct page **pagep = &pages[pcpu_page_idx(cpu, i)];
118
119 *pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0);
120 if (!*pagep) {
121 pcpu_free_pages(chunk, pages, populated,
122 page_start, page_end);
123 return -ENOMEM;
124 }
125 }
126 }
127 return 0;
128}
129
130/**
131 * pcpu_pre_unmap_flush - flush cache prior to unmapping
132 * @chunk: chunk the regions to be flushed belongs to
133 * @page_start: page index of the first page to be flushed
134 * @page_end: page index of the last page to be flushed + 1
135 *
136 * Pages in [@page_start,@page_end) of @chunk are about to be
137 * unmapped. Flush cache. As each flushing trial can be very
138 * expensive, issue flush on the whole region at once rather than
139 * doing it for each cpu. This could be an overkill but is more
140 * scalable.
141 */
142static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,
143 int page_start, int page_end)
144{
145 flush_cache_vunmap(
146 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
147 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
148}
149
150static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)
151{
152 unmap_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT);
153}
154
155/**
156 * pcpu_unmap_pages - unmap pages out of a pcpu_chunk
157 * @chunk: chunk of interest
158 * @pages: pages array which can be used to pass information to free
159 * @populated: populated bitmap
160 * @page_start: page index of the first page to unmap
161 * @page_end: page index of the last page to unmap + 1
162 *
163 * For each cpu, unmap pages [@page_start,@page_end) out of @chunk.
164 * Corresponding elements in @pages were cleared by the caller and can
165 * be used to carry information to pcpu_free_pages() which will be
166 * called after all unmaps are finished. The caller should call
167 * proper pre/post flush functions.
168 */
169static void pcpu_unmap_pages(struct pcpu_chunk *chunk,
170 struct page **pages, unsigned long *populated,
171 int page_start, int page_end)
172{
173 unsigned int cpu;
174 int i;
175
176 for_each_possible_cpu(cpu) {
177 for (i = page_start; i < page_end; i++) {
178 struct page *page;
179
180 page = pcpu_chunk_page(chunk, cpu, i);
181 WARN_ON(!page);
182 pages[pcpu_page_idx(cpu, i)] = page;
183 }
184 __pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),
185 page_end - page_start);
186 }
187
188 for (i = page_start; i < page_end; i++)
189 __clear_bit(i, populated);
190}
191
192/**
193 * pcpu_post_unmap_tlb_flush - flush TLB after unmapping
194 * @chunk: pcpu_chunk the regions to be flushed belong to
195 * @page_start: page index of the first page to be flushed
196 * @page_end: page index of the last page to be flushed + 1
197 *
198 * Pages [@page_start,@page_end) of @chunk have been unmapped. Flush
199 * TLB for the regions. This can be skipped if the area is to be
200 * returned to vmalloc as vmalloc will handle TLB flushing lazily.
201 *
202 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
203 * for the whole region.
204 */
205static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
206 int page_start, int page_end)
207{
208 flush_tlb_kernel_range(
209 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
210 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
211}
212
213static int __pcpu_map_pages(unsigned long addr, struct page **pages,
214 int nr_pages)
215{
216 return map_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT,
217 PAGE_KERNEL, pages);
218}
219
220/**
221 * pcpu_map_pages - map pages into a pcpu_chunk
222 * @chunk: chunk of interest
223 * @pages: pages array containing pages to be mapped
224 * @populated: populated bitmap
225 * @page_start: page index of the first page to map
226 * @page_end: page index of the last page to map + 1
227 *
228 * For each cpu, map pages [@page_start,@page_end) into @chunk. The
229 * caller is responsible for calling pcpu_post_map_flush() after all
230 * mappings are complete.
231 *
232 * This function is responsible for setting corresponding bits in
233 * @chunk->populated bitmap and whatever is necessary for reverse
234 * lookup (addr -> chunk).
235 */
236static int pcpu_map_pages(struct pcpu_chunk *chunk,
237 struct page **pages, unsigned long *populated,
238 int page_start, int page_end)
239{
240 unsigned int cpu, tcpu;
241 int i, err;
242
243 for_each_possible_cpu(cpu) {
244 err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start),
245 &pages[pcpu_page_idx(cpu, page_start)],
246 page_end - page_start);
247 if (err < 0)
248 goto err;
249 }
250
251 /* mapping successful, link chunk and mark populated */
252 for (i = page_start; i < page_end; i++) {
253 for_each_possible_cpu(cpu)
254 pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)],
255 chunk);
256 __set_bit(i, populated);
257 }
258
259 return 0;
260
261err:
262 for_each_possible_cpu(tcpu) {
263 if (tcpu == cpu)
264 break;
265 __pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),
266 page_end - page_start);
267 }
268 return err;
269}
270
271/**
272 * pcpu_post_map_flush - flush cache after mapping
273 * @chunk: pcpu_chunk the regions to be flushed belong to
274 * @page_start: page index of the first page to be flushed
275 * @page_end: page index of the last page to be flushed + 1
276 *
277 * Pages [@page_start,@page_end) of @chunk have been mapped. Flush
278 * cache.
279 *
280 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
281 * for the whole region.
282 */
283static void pcpu_post_map_flush(struct pcpu_chunk *chunk,
284 int page_start, int page_end)
285{
286 flush_cache_vmap(
287 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
288 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
289}
290
291/**
292 * pcpu_populate_chunk - populate and map an area of a pcpu_chunk
293 * @chunk: chunk of interest
294 * @off: offset to the area to populate
295 * @size: size of the area to populate in bytes
296 *
297 * For each cpu, populate and map pages [@page_start,@page_end) into
298 * @chunk. The area is cleared on return.
299 *
300 * CONTEXT:
301 * pcpu_alloc_mutex, does GFP_KERNEL allocation.
302 */
303static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
304{
305 int page_start = PFN_DOWN(off);
306 int page_end = PFN_UP(off + size);
307 int free_end = page_start, unmap_end = page_start;
308 struct page **pages;
309 unsigned long *populated;
310 unsigned int cpu;
311 int rs, re, rc;
312
313 /* quick path, check whether all pages are already there */
314 rs = page_start;
315 pcpu_next_pop(chunk, &rs, &re, page_end);
316 if (rs == page_start && re == page_end)
317 goto clear;
318
319 /* need to allocate and map pages, this chunk can't be immutable */
320 WARN_ON(chunk->immutable);
321
322 pages = pcpu_get_pages_and_bitmap(chunk, &populated, true);
323 if (!pages)
324 return -ENOMEM;
325
326 /* alloc and map */
327 pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
328 rc = pcpu_alloc_pages(chunk, pages, populated, rs, re);
329 if (rc)
330 goto err_free;
331 free_end = re;
332 }
333
334 pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
335 rc = pcpu_map_pages(chunk, pages, populated, rs, re);
336 if (rc)
337 goto err_unmap;
338 unmap_end = re;
339 }
340 pcpu_post_map_flush(chunk, page_start, page_end);
341
342 /* commit new bitmap */
343 bitmap_copy(chunk->populated, populated, pcpu_unit_pages);
344clear:
345 for_each_possible_cpu(cpu)
346 memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size);
347 return 0;
348
349err_unmap:
350 pcpu_pre_unmap_flush(chunk, page_start, unmap_end);
351 pcpu_for_each_unpop_region(chunk, rs, re, page_start, unmap_end)
352 pcpu_unmap_pages(chunk, pages, populated, rs, re);
353 pcpu_post_unmap_tlb_flush(chunk, page_start, unmap_end);
354err_free:
355 pcpu_for_each_unpop_region(chunk, rs, re, page_start, free_end)
356 pcpu_free_pages(chunk, pages, populated, rs, re);
357 return rc;
358}
359
360/**
361 * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
362 * @chunk: chunk to depopulate
363 * @off: offset to the area to depopulate
364 * @size: size of the area to depopulate in bytes
365 * @flush: whether to flush cache and tlb or not
366 *
367 * For each cpu, depopulate and unmap pages [@page_start,@page_end)
368 * from @chunk. If @flush is true, vcache is flushed before unmapping
369 * and tlb after.
370 *
371 * CONTEXT:
372 * pcpu_alloc_mutex.
373 */
374static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size)
375{
376 int page_start = PFN_DOWN(off);
377 int page_end = PFN_UP(off + size);
378 struct page **pages;
379 unsigned long *populated;
380 int rs, re;
381
382 /* quick path, check whether it's empty already */
383 rs = page_start;
384 pcpu_next_unpop(chunk, &rs, &re, page_end);
385 if (rs == page_start && re == page_end)
386 return;
387
388 /* immutable chunks can't be depopulated */
389 WARN_ON(chunk->immutable);
390
391 /*
392 * If control reaches here, there must have been at least one
393 * successful population attempt so the temp pages array must
394 * be available now.
395 */
396 pages = pcpu_get_pages_and_bitmap(chunk, &populated, false);
397 BUG_ON(!pages);
398
399 /* unmap and free */
400 pcpu_pre_unmap_flush(chunk, page_start, page_end);
401
402 pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end)
403 pcpu_unmap_pages(chunk, pages, populated, rs, re);
404
405 /* no need to flush tlb, vmalloc will handle it lazily */
406
407 pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end)
408 pcpu_free_pages(chunk, pages, populated, rs, re);
409
410 /* commit new bitmap */
411 bitmap_copy(chunk->populated, populated, pcpu_unit_pages);
412}
413
414static struct pcpu_chunk *pcpu_create_chunk(void)
415{
416 struct pcpu_chunk *chunk;
417 struct vm_struct **vms;
418
419 chunk = pcpu_alloc_chunk();
420 if (!chunk)
421 return NULL;
422
423 vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes,
424 pcpu_nr_groups, pcpu_atom_size, GFP_KERNEL);
425 if (!vms) {
426 pcpu_free_chunk(chunk);
427 return NULL;
428 }
429
430 chunk->data = vms;
431 chunk->base_addr = vms[0]->addr - pcpu_group_offsets[0];
432 return chunk;
433}
434
435static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)
436{
437 if (chunk && chunk->data)
438 pcpu_free_vm_areas(chunk->data, pcpu_nr_groups);
439 pcpu_free_chunk(chunk);
440}
441
442static struct page *pcpu_addr_to_page(void *addr)
443{
444 return vmalloc_to_page(addr);
445}
446
447static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai)
448{
449 /* no extra restriction */
450 return 0;
451}
diff --git a/mm/percpu.c b/mm/percpu.c
index 6e09741ddc6..39f7dfd5958 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * linux/mm/percpu.c - percpu memory allocator 2 * mm/percpu.c - percpu memory allocator
3 * 3 *
4 * Copyright (C) 2009 SUSE Linux Products GmbH 4 * Copyright (C) 2009 SUSE Linux Products GmbH
5 * Copyright (C) 2009 Tejun Heo <tj@kernel.org> 5 * Copyright (C) 2009 Tejun Heo <tj@kernel.org>
@@ -7,14 +7,13 @@
7 * This file is released under the GPLv2. 7 * This file is released under the GPLv2.
8 * 8 *
9 * This is percpu allocator which can handle both static and dynamic 9 * This is percpu allocator which can handle both static and dynamic
10 * areas. Percpu areas are allocated in chunks in vmalloc area. Each 10 * areas. Percpu areas are allocated in chunks. Each chunk is
11 * chunk is consisted of boot-time determined number of units and the 11 * consisted of boot-time determined number of units and the first
12 * first chunk is used for static percpu variables in the kernel image 12 * chunk is used for static percpu variables in the kernel image
13 * (special boot time alloc/init handling necessary as these areas 13 * (special boot time alloc/init handling necessary as these areas
14 * need to be brought up before allocation services are running). 14 * need to be brought up before allocation services are running).
15 * Unit grows as necessary and all units grow or shrink in unison. 15 * Unit grows as necessary and all units grow or shrink in unison.
16 * When a chunk is filled up, another chunk is allocated. ie. in 16 * When a chunk is filled up, another chunk is allocated.
17 * vmalloc area
18 * 17 *
19 * c0 c1 c2 18 * c0 c1 c2
20 * ------------------- ------------------- ------------ 19 * ------------------- ------------------- ------------
@@ -99,7 +98,7 @@ struct pcpu_chunk {
99 int map_used; /* # of map entries used */ 98 int map_used; /* # of map entries used */
100 int map_alloc; /* # of map entries allocated */ 99 int map_alloc; /* # of map entries allocated */
101 int *map; /* allocation map */ 100 int *map; /* allocation map */
102 struct vm_struct **vms; /* mapped vmalloc regions */ 101 void *data; /* chunk data */
103 bool immutable; /* no [de]population allowed */ 102 bool immutable; /* no [de]population allowed */
104 unsigned long populated[]; /* populated bitmap */ 103 unsigned long populated[]; /* populated bitmap */
105}; 104};
@@ -177,6 +176,21 @@ static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */
177static void pcpu_reclaim(struct work_struct *work); 176static void pcpu_reclaim(struct work_struct *work);
178static DECLARE_WORK(pcpu_reclaim_work, pcpu_reclaim); 177static DECLARE_WORK(pcpu_reclaim_work, pcpu_reclaim);
179 178
179static bool pcpu_addr_in_first_chunk(void *addr)
180{
181 void *first_start = pcpu_first_chunk->base_addr;
182
183 return addr >= first_start && addr < first_start + pcpu_unit_size;
184}
185
186static bool pcpu_addr_in_reserved_chunk(void *addr)
187{
188 void *first_start = pcpu_first_chunk->base_addr;
189
190 return addr >= first_start &&
191 addr < first_start + pcpu_reserved_chunk_limit;
192}
193
180static int __pcpu_size_to_slot(int size) 194static int __pcpu_size_to_slot(int size)
181{ 195{
182 int highbit = fls(size); /* size is in bytes */ 196 int highbit = fls(size); /* size is in bytes */
@@ -198,27 +212,6 @@ static int pcpu_chunk_slot(const struct pcpu_chunk *chunk)
198 return pcpu_size_to_slot(chunk->free_size); 212 return pcpu_size_to_slot(chunk->free_size);
199} 213}
200 214
201static int pcpu_page_idx(unsigned int cpu, int page_idx)
202{
203 return pcpu_unit_map[cpu] * pcpu_unit_pages + page_idx;
204}
205
206static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk,
207 unsigned int cpu, int page_idx)
208{
209 return (unsigned long)chunk->base_addr + pcpu_unit_offsets[cpu] +
210 (page_idx << PAGE_SHIFT);
211}
212
213static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
214 unsigned int cpu, int page_idx)
215{
216 /* must not be used on pre-mapped chunk */
217 WARN_ON(chunk->immutable);
218
219 return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));
220}
221
222/* set the pointer to a chunk in a page struct */ 215/* set the pointer to a chunk in a page struct */
223static void pcpu_set_page_chunk(struct page *page, struct pcpu_chunk *pcpu) 216static void pcpu_set_page_chunk(struct page *page, struct pcpu_chunk *pcpu)
224{ 217{
@@ -231,13 +224,27 @@ static struct pcpu_chunk *pcpu_get_page_chunk(struct page *page)
231 return (struct pcpu_chunk *)page->index; 224 return (struct pcpu_chunk *)page->index;
232} 225}
233 226
234static void pcpu_next_unpop(struct pcpu_chunk *chunk, int *rs, int *re, int end) 227static int __maybe_unused pcpu_page_idx(unsigned int cpu, int page_idx)
228{
229 return pcpu_unit_map[cpu] * pcpu_unit_pages + page_idx;
230}
231
232static unsigned long __maybe_unused pcpu_chunk_addr(struct pcpu_chunk *chunk,
233 unsigned int cpu, int page_idx)
234{
235 return (unsigned long)chunk->base_addr + pcpu_unit_offsets[cpu] +
236 (page_idx << PAGE_SHIFT);
237}
238
239static void __maybe_unused pcpu_next_unpop(struct pcpu_chunk *chunk,
240 int *rs, int *re, int end)
235{ 241{
236 *rs = find_next_zero_bit(chunk->populated, end, *rs); 242 *rs = find_next_zero_bit(chunk->populated, end, *rs);
237 *re = find_next_bit(chunk->populated, end, *rs + 1); 243 *re = find_next_bit(chunk->populated, end, *rs + 1);
238} 244}
239 245
240static void pcpu_next_pop(struct pcpu_chunk *chunk, int *rs, int *re, int end) 246static void __maybe_unused pcpu_next_pop(struct pcpu_chunk *chunk,
247 int *rs, int *re, int end)
241{ 248{
242 *rs = find_next_bit(chunk->populated, end, *rs); 249 *rs = find_next_bit(chunk->populated, end, *rs);
243 *re = find_next_zero_bit(chunk->populated, end, *rs + 1); 250 *re = find_next_zero_bit(chunk->populated, end, *rs + 1);
@@ -326,36 +333,6 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
326} 333}
327 334
328/** 335/**
329 * pcpu_chunk_addr_search - determine chunk containing specified address
330 * @addr: address for which the chunk needs to be determined.
331 *
332 * RETURNS:
333 * The address of the found chunk.
334 */
335static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
336{
337 void *first_start = pcpu_first_chunk->base_addr;
338
339 /* is it in the first chunk? */
340 if (addr >= first_start && addr < first_start + pcpu_unit_size) {
341 /* is it in the reserved area? */
342 if (addr < first_start + pcpu_reserved_chunk_limit)
343 return pcpu_reserved_chunk;
344 return pcpu_first_chunk;
345 }
346
347 /*
348 * The address is relative to unit0 which might be unused and
349 * thus unmapped. Offset the address to the unit space of the
350 * current processor before looking it up in the vmalloc
351 * space. Note that any possible cpu id can be used here, so
352 * there's no need to worry about preemption or cpu hotplug.
353 */
354 addr += pcpu_unit_offsets[raw_smp_processor_id()];
355 return pcpu_get_page_chunk(vmalloc_to_page(addr));
356}
357
358/**
359 * pcpu_need_to_extend - determine whether chunk area map needs to be extended 336 * pcpu_need_to_extend - determine whether chunk area map needs to be extended
360 * @chunk: chunk of interest 337 * @chunk: chunk of interest
361 * 338 *
@@ -623,434 +600,92 @@ static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
623 pcpu_chunk_relocate(chunk, oslot); 600 pcpu_chunk_relocate(chunk, oslot);
624} 601}
625 602
626/** 603static struct pcpu_chunk *pcpu_alloc_chunk(void)
627 * pcpu_get_pages_and_bitmap - get temp pages array and bitmap
628 * @chunk: chunk of interest
629 * @bitmapp: output parameter for bitmap
630 * @may_alloc: may allocate the array
631 *
632 * Returns pointer to array of pointers to struct page and bitmap,
633 * both of which can be indexed with pcpu_page_idx(). The returned
634 * array is cleared to zero and *@bitmapp is copied from
635 * @chunk->populated. Note that there is only one array and bitmap
636 * and access exclusion is the caller's responsibility.
637 *
638 * CONTEXT:
639 * pcpu_alloc_mutex and does GFP_KERNEL allocation if @may_alloc.
640 * Otherwise, don't care.
641 *
642 * RETURNS:
643 * Pointer to temp pages array on success, NULL on failure.
644 */
645static struct page **pcpu_get_pages_and_bitmap(struct pcpu_chunk *chunk,
646 unsigned long **bitmapp,
647 bool may_alloc)
648{
649 static struct page **pages;
650 static unsigned long *bitmap;
651 size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]);
652 size_t bitmap_size = BITS_TO_LONGS(pcpu_unit_pages) *
653 sizeof(unsigned long);
654
655 if (!pages || !bitmap) {
656 if (may_alloc && !pages)
657 pages = pcpu_mem_alloc(pages_size);
658 if (may_alloc && !bitmap)
659 bitmap = pcpu_mem_alloc(bitmap_size);
660 if (!pages || !bitmap)
661 return NULL;
662 }
663
664 memset(pages, 0, pages_size);
665 bitmap_copy(bitmap, chunk->populated, pcpu_unit_pages);
666
667 *bitmapp = bitmap;
668 return pages;
669}
670
671/**
672 * pcpu_free_pages - free pages which were allocated for @chunk
673 * @chunk: chunk pages were allocated for
674 * @pages: array of pages to be freed, indexed by pcpu_page_idx()
675 * @populated: populated bitmap
676 * @page_start: page index of the first page to be freed
677 * @page_end: page index of the last page to be freed + 1
678 *
679 * Free pages [@page_start and @page_end) in @pages for all units.
680 * The pages were allocated for @chunk.
681 */
682static void pcpu_free_pages(struct pcpu_chunk *chunk,
683 struct page **pages, unsigned long *populated,
684 int page_start, int page_end)
685{ 604{
686 unsigned int cpu; 605 struct pcpu_chunk *chunk;
687 int i;
688 606
689 for_each_possible_cpu(cpu) { 607 chunk = kzalloc(pcpu_chunk_struct_size, GFP_KERNEL);
690 for (i = page_start; i < page_end; i++) { 608 if (!chunk)
691 struct page *page = pages[pcpu_page_idx(cpu, i)]; 609 return NULL;
692 610
693 if (page) 611 chunk->map = pcpu_mem_alloc(PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
694 __free_page(page); 612 if (!chunk->map) {
695 } 613 kfree(chunk);
614 return NULL;
696 } 615 }
697}
698 616
699/** 617 chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
700 * pcpu_alloc_pages - allocates pages for @chunk 618 chunk->map[chunk->map_used++] = pcpu_unit_size;
701 * @chunk: target chunk
702 * @pages: array to put the allocated pages into, indexed by pcpu_page_idx()
703 * @populated: populated bitmap
704 * @page_start: page index of the first page to be allocated
705 * @page_end: page index of the last page to be allocated + 1
706 *
707 * Allocate pages [@page_start,@page_end) into @pages for all units.
708 * The allocation is for @chunk. Percpu core doesn't care about the
709 * content of @pages and will pass it verbatim to pcpu_map_pages().
710 */
711static int pcpu_alloc_pages(struct pcpu_chunk *chunk,
712 struct page **pages, unsigned long *populated,
713 int page_start, int page_end)
714{
715 const gfp_t gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_COLD;
716 unsigned int cpu;
717 int i;
718 619
719 for_each_possible_cpu(cpu) { 620 INIT_LIST_HEAD(&chunk->list);
720 for (i = page_start; i < page_end; i++) { 621 chunk->free_size = pcpu_unit_size;
721 struct page **pagep = &pages[pcpu_page_idx(cpu, i)]; 622 chunk->contig_hint = pcpu_unit_size;
722
723 *pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0);
724 if (!*pagep) {
725 pcpu_free_pages(chunk, pages, populated,
726 page_start, page_end);
727 return -ENOMEM;
728 }
729 }
730 }
731 return 0;
732}
733 623
734/** 624 return chunk;
735 * pcpu_pre_unmap_flush - flush cache prior to unmapping
736 * @chunk: chunk the regions to be flushed belongs to
737 * @page_start: page index of the first page to be flushed
738 * @page_end: page index of the last page to be flushed + 1
739 *
740 * Pages in [@page_start,@page_end) of @chunk are about to be
741 * unmapped. Flush cache. As each flushing trial can be very
742 * expensive, issue flush on the whole region at once rather than
743 * doing it for each cpu. This could be an overkill but is more
744 * scalable.
745 */
746static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,
747 int page_start, int page_end)
748{
749 flush_cache_vunmap(
750 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
751 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
752} 625}
753 626
754static void __pcpu_unmap_pages(unsigned long addr, int nr_pages) 627static void pcpu_free_chunk(struct pcpu_chunk *chunk)
755{ 628{
756 unmap_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT); 629 if (!chunk)
630 return;
631 pcpu_mem_free(chunk->map, chunk->map_alloc * sizeof(chunk->map[0]));
632 kfree(chunk);
757} 633}
758 634
759/** 635/*
760 * pcpu_unmap_pages - unmap pages out of a pcpu_chunk 636 * Chunk management implementation.
761 * @chunk: chunk of interest 637 *
762 * @pages: pages array which can be used to pass information to free 638 * To allow different implementations, chunk alloc/free and
763 * @populated: populated bitmap 639 * [de]population are implemented in a separate file which is pulled
764 * @page_start: page index of the first page to unmap 640 * into this file and compiled together. The following functions
765 * @page_end: page index of the last page to unmap + 1 641 * should be implemented.
766 * 642 *
767 * For each cpu, unmap pages [@page_start,@page_end) out of @chunk. 643 * pcpu_populate_chunk - populate the specified range of a chunk
768 * Corresponding elements in @pages were cleared by the caller and can 644 * pcpu_depopulate_chunk - depopulate the specified range of a chunk
769 * be used to carry information to pcpu_free_pages() which will be 645 * pcpu_create_chunk - create a new chunk
770 * called after all unmaps are finished. The caller should call 646 * pcpu_destroy_chunk - destroy a chunk, always preceded by full depop
771 * proper pre/post flush functions. 647 * pcpu_addr_to_page - translate address to physical address
648 * pcpu_verify_alloc_info - check alloc_info is acceptable during init
772 */ 649 */
773static void pcpu_unmap_pages(struct pcpu_chunk *chunk, 650static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size);
774 struct page **pages, unsigned long *populated, 651static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size);
775 int page_start, int page_end) 652static struct pcpu_chunk *pcpu_create_chunk(void);
776{ 653static void pcpu_destroy_chunk(struct pcpu_chunk *chunk);
777 unsigned int cpu; 654static struct page *pcpu_addr_to_page(void *addr);
778 int i; 655static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai);
779 656
780 for_each_possible_cpu(cpu) { 657#ifdef CONFIG_NEED_PER_CPU_KM
781 for (i = page_start; i < page_end; i++) { 658#include "percpu-km.c"
782 struct page *page; 659#else
783 660#include "percpu-vm.c"
784 page = pcpu_chunk_page(chunk, cpu, i); 661#endif
785 WARN_ON(!page);
786 pages[pcpu_page_idx(cpu, i)] = page;
787 }
788 __pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),
789 page_end - page_start);
790 }
791
792 for (i = page_start; i < page_end; i++)
793 __clear_bit(i, populated);
794}
795 662
796/** 663/**
797 * pcpu_post_unmap_tlb_flush - flush TLB after unmapping 664 * pcpu_chunk_addr_search - determine chunk containing specified address
798 * @chunk: pcpu_chunk the regions to be flushed belong to 665 * @addr: address for which the chunk needs to be determined.
799 * @page_start: page index of the first page to be flushed
800 * @page_end: page index of the last page to be flushed + 1
801 *
802 * Pages [@page_start,@page_end) of @chunk have been unmapped. Flush
803 * TLB for the regions. This can be skipped if the area is to be
804 * returned to vmalloc as vmalloc will handle TLB flushing lazily.
805 * 666 *
806 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once 667 * RETURNS:
807 * for the whole region. 668 * The address of the found chunk.
808 */
809static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
810 int page_start, int page_end)
811{
812 flush_tlb_kernel_range(
813 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
814 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
815}
816
817static int __pcpu_map_pages(unsigned long addr, struct page **pages,
818 int nr_pages)
819{
820 return map_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT,
821 PAGE_KERNEL, pages);
822}
823
824/**
825 * pcpu_map_pages - map pages into a pcpu_chunk
826 * @chunk: chunk of interest
827 * @pages: pages array containing pages to be mapped
828 * @populated: populated bitmap
829 * @page_start: page index of the first page to map
830 * @page_end: page index of the last page to map + 1
831 *
832 * For each cpu, map pages [@page_start,@page_end) into @chunk. The
833 * caller is responsible for calling pcpu_post_map_flush() after all
834 * mappings are complete.
835 *
836 * This function is responsible for setting corresponding bits in
837 * @chunk->populated bitmap and whatever is necessary for reverse
838 * lookup (addr -> chunk).
839 */ 669 */
840static int pcpu_map_pages(struct pcpu_chunk *chunk, 670static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
841 struct page **pages, unsigned long *populated,
842 int page_start, int page_end)
843{ 671{
844 unsigned int cpu, tcpu; 672 /* is it in the first chunk? */
845 int i, err; 673 if (pcpu_addr_in_first_chunk(addr)) {
846 674 /* is it in the reserved area? */
847 for_each_possible_cpu(cpu) { 675 if (pcpu_addr_in_reserved_chunk(addr))
848 err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start), 676 return pcpu_reserved_chunk;
849 &pages[pcpu_page_idx(cpu, page_start)], 677 return pcpu_first_chunk;
850 page_end - page_start);
851 if (err < 0)
852 goto err;
853 }
854
855 /* mapping successful, link chunk and mark populated */
856 for (i = page_start; i < page_end; i++) {
857 for_each_possible_cpu(cpu)
858 pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)],
859 chunk);
860 __set_bit(i, populated);
861 }
862
863 return 0;
864
865err:
866 for_each_possible_cpu(tcpu) {
867 if (tcpu == cpu)
868 break;
869 __pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),
870 page_end - page_start);
871 } 678 }
872 return err;
873}
874
875/**
876 * pcpu_post_map_flush - flush cache after mapping
877 * @chunk: pcpu_chunk the regions to be flushed belong to
878 * @page_start: page index of the first page to be flushed
879 * @page_end: page index of the last page to be flushed + 1
880 *
881 * Pages [@page_start,@page_end) of @chunk have been mapped. Flush
882 * cache.
883 *
884 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
885 * for the whole region.
886 */
887static void pcpu_post_map_flush(struct pcpu_chunk *chunk,
888 int page_start, int page_end)
889{
890 flush_cache_vmap(
891 pcpu_chunk_addr(chunk, pcpu_first_unit_cpu, page_start),
892 pcpu_chunk_addr(chunk, pcpu_last_unit_cpu, page_end));
893}
894
895/**
896 * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
897 * @chunk: chunk to depopulate
898 * @off: offset to the area to depopulate
899 * @size: size of the area to depopulate in bytes
900 * @flush: whether to flush cache and tlb or not
901 *
902 * For each cpu, depopulate and unmap pages [@page_start,@page_end)
903 * from @chunk. If @flush is true, vcache is flushed before unmapping
904 * and tlb after.
905 *
906 * CONTEXT:
907 * pcpu_alloc_mutex.
908 */
909static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size)
910{
911 int page_start = PFN_DOWN(off);
912 int page_end = PFN_UP(off + size);
913 struct page **pages;
914 unsigned long *populated;
915 int rs, re;
916
917 /* quick path, check whether it's empty already */
918 rs = page_start;
919 pcpu_next_unpop(chunk, &rs, &re, page_end);
920 if (rs == page_start && re == page_end)
921 return;
922
923 /* immutable chunks can't be depopulated */
924 WARN_ON(chunk->immutable);
925 679
926 /* 680 /*
927 * If control reaches here, there must have been at least one 681 * The address is relative to unit0 which might be unused and
928 * successful population attempt so the temp pages array must 682 * thus unmapped. Offset the address to the unit space of the
929 * be available now. 683 * current processor before looking it up in the vmalloc
684 * space. Note that any possible cpu id can be used here, so
685 * there's no need to worry about preemption or cpu hotplug.
930 */ 686 */
931 pages = pcpu_get_pages_and_bitmap(chunk, &populated, false); 687 addr += pcpu_unit_offsets[raw_smp_processor_id()];
932 BUG_ON(!pages); 688 return pcpu_get_page_chunk(pcpu_addr_to_page(addr));
933
934 /* unmap and free */
935 pcpu_pre_unmap_flush(chunk, page_start, page_end);
936
937 pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end)
938 pcpu_unmap_pages(chunk, pages, populated, rs, re);
939
940 /* no need to flush tlb, vmalloc will handle it lazily */
941
942 pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end)
943 pcpu_free_pages(chunk, pages, populated, rs, re);
944
945 /* commit new bitmap */
946 bitmap_copy(chunk->populated, populated, pcpu_unit_pages);
947}
948
949/**
950 * pcpu_populate_chunk - populate and map an area of a pcpu_chunk
951 * @chunk: chunk of interest
952 * @off: offset to the area to populate
953 * @size: size of the area to populate in bytes
954 *
955 * For each cpu, populate and map pages [@page_start,@page_end) into
956 * @chunk. The area is cleared on return.
957 *
958 * CONTEXT:
959 * pcpu_alloc_mutex, does GFP_KERNEL allocation.
960 */
961static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
962{
963 int page_start = PFN_DOWN(off);
964 int page_end = PFN_UP(off + size);
965 int free_end = page_start, unmap_end = page_start;
966 struct page **pages;
967 unsigned long *populated;
968 unsigned int cpu;
969 int rs, re, rc;
970
971 /* quick path, check whether all pages are already there */
972 rs = page_start;
973 pcpu_next_pop(chunk, &rs, &re, page_end);
974 if (rs == page_start && re == page_end)
975 goto clear;
976
977 /* need to allocate and map pages, this chunk can't be immutable */
978 WARN_ON(chunk->immutable);
979
980 pages = pcpu_get_pages_and_bitmap(chunk, &populated, true);
981 if (!pages)
982 return -ENOMEM;
983
984 /* alloc and map */
985 pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
986 rc = pcpu_alloc_pages(chunk, pages, populated, rs, re);
987 if (rc)
988 goto err_free;
989 free_end = re;
990 }
991
992 pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
993 rc = pcpu_map_pages(chunk, pages, populated, rs, re);
994 if (rc)
995 goto err_unmap;
996 unmap_end = re;
997 }
998 pcpu_post_map_flush(chunk, page_start, page_end);
999
1000 /* commit new bitmap */
1001 bitmap_copy(chunk->populated, populated, pcpu_unit_pages);
1002clear:
1003 for_each_possible_cpu(cpu)
1004 memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size);
1005 return 0;
1006
1007err_unmap:
1008 pcpu_pre_unmap_flush(chunk, page_start, unmap_end);
1009 pcpu_for_each_unpop_region(chunk, rs, re, page_start, unmap_end)
1010 pcpu_unmap_pages(chunk, pages, populated, rs, re);
1011 pcpu_post_unmap_tlb_flush(chunk, page_start, unmap_end);
1012err_free:
1013 pcpu_for_each_unpop_region(chunk, rs, re, page_start, free_end)
1014 pcpu_free_pages(chunk, pages, populated, rs, re);
1015 return rc;
1016}
1017
1018static void free_pcpu_chunk(struct pcpu_chunk *chunk)
1019{
1020 if (!chunk)
1021 return;
1022 if (chunk->vms)
1023 pcpu_free_vm_areas(chunk->vms, pcpu_nr_groups);
1024 pcpu_mem_free(chunk->map, chunk->map_alloc * sizeof(chunk->map[0]));
1025 kfree(chunk);
1026}
1027
1028static struct pcpu_chunk *alloc_pcpu_chunk(void)
1029{
1030 struct pcpu_chunk *chunk;
1031
1032 chunk = kzalloc(pcpu_chunk_struct_size, GFP_KERNEL);
1033 if (!chunk)
1034 return NULL;
1035
1036 chunk->map = pcpu_mem_alloc(PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
1037 chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
1038 chunk->map[chunk->map_used++] = pcpu_unit_size;
1039
1040 chunk->vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes,
1041 pcpu_nr_groups, pcpu_atom_size,
1042 GFP_KERNEL);
1043 if (!chunk->vms) {
1044 free_pcpu_chunk(chunk);
1045 return NULL;
1046 }
1047
1048 INIT_LIST_HEAD(&chunk->list);
1049 chunk->free_size = pcpu_unit_size;
1050 chunk->contig_hint = pcpu_unit_size;
1051 chunk->base_addr = chunk->vms[0]->addr - pcpu_group_offsets[0];
1052
1053 return chunk;
1054} 689}
1055 690
1056/** 691/**
@@ -1142,7 +777,7 @@ restart:
1142 /* hmmm... no space left, create a new chunk */ 777 /* hmmm... no space left, create a new chunk */
1143 spin_unlock_irqrestore(&pcpu_lock, flags); 778 spin_unlock_irqrestore(&pcpu_lock, flags);
1144 779
1145 chunk = alloc_pcpu_chunk(); 780 chunk = pcpu_create_chunk();
1146 if (!chunk) { 781 if (!chunk) {
1147 err = "failed to allocate new chunk"; 782 err = "failed to allocate new chunk";
1148 goto fail_unlock_mutex; 783 goto fail_unlock_mutex;
@@ -1254,7 +889,7 @@ static void pcpu_reclaim(struct work_struct *work)
1254 889
1255 list_for_each_entry_safe(chunk, next, &todo, list) { 890 list_for_each_entry_safe(chunk, next, &todo, list) {
1256 pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size); 891 pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size);
1257 free_pcpu_chunk(chunk); 892 pcpu_destroy_chunk(chunk);
1258 } 893 }
1259 894
1260 mutex_unlock(&pcpu_alloc_mutex); 895 mutex_unlock(&pcpu_alloc_mutex);
@@ -1343,11 +978,14 @@ bool is_kernel_percpu_address(unsigned long addr)
1343 */ 978 */
1344phys_addr_t per_cpu_ptr_to_phys(void *addr) 979phys_addr_t per_cpu_ptr_to_phys(void *addr)
1345{ 980{
1346 if ((unsigned long)addr < VMALLOC_START || 981 if (pcpu_addr_in_first_chunk(addr)) {
1347 (unsigned long)addr >= VMALLOC_END) 982 if ((unsigned long)addr < VMALLOC_START ||
1348 return __pa(addr); 983 (unsigned long)addr >= VMALLOC_END)
1349 else 984 return __pa(addr);
1350 return page_to_phys(vmalloc_to_page(addr)); 985 else
986 return page_to_phys(vmalloc_to_page(addr));
987 } else
988 return page_to_phys(pcpu_addr_to_page(addr));
1351} 989}
1352 990
1353static inline size_t pcpu_calc_fc_sizes(size_t static_size, 991static inline size_t pcpu_calc_fc_sizes(size_t static_size,
@@ -1719,6 +1357,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
1719 PCPU_SETUP_BUG_ON(ai->unit_size < size_sum); 1357 PCPU_SETUP_BUG_ON(ai->unit_size < size_sum);
1720 PCPU_SETUP_BUG_ON(ai->unit_size & ~PAGE_MASK); 1358 PCPU_SETUP_BUG_ON(ai->unit_size & ~PAGE_MASK);
1721 PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE); 1359 PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE);
1360 PCPU_SETUP_BUG_ON(pcpu_verify_alloc_info(ai) < 0);
1722 1361
1723 /* process group information and build config tables accordingly */ 1362 /* process group information and build config tables accordingly */
1724 group_offsets = alloc_bootmem(ai->nr_groups * sizeof(group_offsets[0])); 1363 group_offsets = alloc_bootmem(ai->nr_groups * sizeof(group_offsets[0]));
diff --git a/mm/rmap.c b/mm/rmap.c
index 4bad3267537..0feeef860a8 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -133,8 +133,8 @@ int anon_vma_prepare(struct vm_area_struct *vma)
133 goto out_enomem_free_avc; 133 goto out_enomem_free_avc;
134 allocated = anon_vma; 134 allocated = anon_vma;
135 } 135 }
136 spin_lock(&anon_vma->lock);
137 136
137 spin_lock(&anon_vma->lock);
138 /* page_table_lock to protect against threads */ 138 /* page_table_lock to protect against threads */
139 spin_lock(&mm->page_table_lock); 139 spin_lock(&mm->page_table_lock);
140 if (likely(!vma->anon_vma)) { 140 if (likely(!vma->anon_vma)) {
@@ -144,14 +144,15 @@ int anon_vma_prepare(struct vm_area_struct *vma)
144 list_add(&avc->same_vma, &vma->anon_vma_chain); 144 list_add(&avc->same_vma, &vma->anon_vma_chain);
145 list_add(&avc->same_anon_vma, &anon_vma->head); 145 list_add(&avc->same_anon_vma, &anon_vma->head);
146 allocated = NULL; 146 allocated = NULL;
147 avc = NULL;
147 } 148 }
148 spin_unlock(&mm->page_table_lock); 149 spin_unlock(&mm->page_table_lock);
149
150 spin_unlock(&anon_vma->lock); 150 spin_unlock(&anon_vma->lock);
151 if (unlikely(allocated)) { 151
152 if (unlikely(allocated))
152 anon_vma_free(allocated); 153 anon_vma_free(allocated);
154 if (unlikely(avc))
153 anon_vma_chain_free(avc); 155 anon_vma_chain_free(avc);
154 }
155 } 156 }
156 return 0; 157 return 0;
157 158
@@ -335,14 +336,13 @@ vma_address(struct page *page, struct vm_area_struct *vma)
335 336
336/* 337/*
337 * At what user virtual address is page expected in vma? 338 * At what user virtual address is page expected in vma?
338 * checking that the page matches the vma. 339 * Caller should check the page is actually part of the vma.
339 */ 340 */
340unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) 341unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
341{ 342{
342 if (PageAnon(page)) { 343 if (PageAnon(page))
343 if (vma->anon_vma != page_anon_vma(page)) 344 ;
344 return -EFAULT; 345 else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) {
345 } else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) {
346 if (!vma->vm_file || 346 if (!vma->vm_file ||
347 vma->vm_file->f_mapping != page->mapping) 347 vma->vm_file->f_mapping != page->mapping)
348 return -EFAULT; 348 return -EFAULT;
@@ -730,23 +730,28 @@ void page_move_anon_rmap(struct page *page,
730 * @page: the page to add the mapping to 730 * @page: the page to add the mapping to
731 * @vma: the vm area in which the mapping is added 731 * @vma: the vm area in which the mapping is added
732 * @address: the user virtual address mapped 732 * @address: the user virtual address mapped
733 * @exclusive: the page is exclusively owned by the current process
733 */ 734 */
734static void __page_set_anon_rmap(struct page *page, 735static void __page_set_anon_rmap(struct page *page,
735 struct vm_area_struct *vma, unsigned long address) 736 struct vm_area_struct *vma, unsigned long address, int exclusive)
736{ 737{
737 struct anon_vma_chain *avc; 738 struct anon_vma *anon_vma = vma->anon_vma;
738 struct anon_vma *anon_vma;
739 739
740 BUG_ON(!vma->anon_vma); 740 BUG_ON(!anon_vma);
741 741
742 /* 742 /*
743 * We must use the _oldest_ possible anon_vma for the page mapping! 743 * If the page isn't exclusively mapped into this vma,
744 * we must use the _oldest_ possible anon_vma for the
745 * page mapping!
744 * 746 *
745 * So take the last AVC chain entry in the vma, which is the deepest 747 * So take the last AVC chain entry in the vma, which is
746 * ancestor, and use the anon_vma from that. 748 * the deepest ancestor, and use the anon_vma from that.
747 */ 749 */
748 avc = list_entry(vma->anon_vma_chain.prev, struct anon_vma_chain, same_vma); 750 if (!exclusive) {
749 anon_vma = avc->anon_vma; 751 struct anon_vma_chain *avc;
752 avc = list_entry(vma->anon_vma_chain.prev, struct anon_vma_chain, same_vma);
753 anon_vma = avc->anon_vma;
754 }
750 755
751 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; 756 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
752 page->mapping = (struct address_space *) anon_vma; 757 page->mapping = (struct address_space *) anon_vma;
@@ -802,7 +807,7 @@ void page_add_anon_rmap(struct page *page,
802 VM_BUG_ON(!PageLocked(page)); 807 VM_BUG_ON(!PageLocked(page));
803 VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end); 808 VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end);
804 if (first) 809 if (first)
805 __page_set_anon_rmap(page, vma, address); 810 __page_set_anon_rmap(page, vma, address, 0);
806 else 811 else
807 __page_check_anon_rmap(page, vma, address); 812 __page_check_anon_rmap(page, vma, address);
808} 813}
@@ -824,7 +829,7 @@ void page_add_new_anon_rmap(struct page *page,
824 SetPageSwapBacked(page); 829 SetPageSwapBacked(page);
825 atomic_set(&page->_mapcount, 0); /* increment count (starts at -1) */ 830 atomic_set(&page->_mapcount, 0); /* increment count (starts at -1) */
826 __inc_zone_page_state(page, NR_ANON_PAGES); 831 __inc_zone_page_state(page, NR_ANON_PAGES);
827 __page_set_anon_rmap(page, vma, address); 832 __page_set_anon_rmap(page, vma, address, 1);
828 if (page_evictable(page, vma)) 833 if (page_evictable(page, vma))
829 lru_cache_add_lru(page, LRU_ACTIVE_ANON); 834 lru_cache_add_lru(page, LRU_ACTIVE_ANON);
830 else 835 else
diff --git a/mm/shmem.c b/mm/shmem.c
index eef4ebea515..0cd7f66f1c6 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1545,8 +1545,8 @@ static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1545 return 0; 1545 return 0;
1546} 1546}
1547 1547
1548static struct inode *shmem_get_inode(struct super_block *sb, int mode, 1548static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
1549 dev_t dev, unsigned long flags) 1549 int mode, dev_t dev, unsigned long flags)
1550{ 1550{
1551 struct inode *inode; 1551 struct inode *inode;
1552 struct shmem_inode_info *info; 1552 struct shmem_inode_info *info;
@@ -1557,9 +1557,7 @@ static struct inode *shmem_get_inode(struct super_block *sb, int mode,
1557 1557
1558 inode = new_inode(sb); 1558 inode = new_inode(sb);
1559 if (inode) { 1559 if (inode) {
1560 inode->i_mode = mode; 1560 inode_init_owner(inode, dir, mode);
1561 inode->i_uid = current_fsuid();
1562 inode->i_gid = current_fsgid();
1563 inode->i_blocks = 0; 1561 inode->i_blocks = 0;
1564 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info; 1562 inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1565 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; 1563 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
@@ -1814,7 +1812,7 @@ shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
1814 struct inode *inode; 1812 struct inode *inode;
1815 int error = -ENOSPC; 1813 int error = -ENOSPC;
1816 1814
1817 inode = shmem_get_inode(dir->i_sb, mode, dev, VM_NORESERVE); 1815 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
1818 if (inode) { 1816 if (inode) {
1819 error = security_inode_init_security(inode, dir, NULL, NULL, 1817 error = security_inode_init_security(inode, dir, NULL, NULL,
1820 NULL); 1818 NULL);
@@ -1833,11 +1831,6 @@ shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
1833#else 1831#else
1834 error = 0; 1832 error = 0;
1835#endif 1833#endif
1836 if (dir->i_mode & S_ISGID) {
1837 inode->i_gid = dir->i_gid;
1838 if (S_ISDIR(mode))
1839 inode->i_mode |= S_ISGID;
1840 }
1841 dir->i_size += BOGO_DIRENT_SIZE; 1834 dir->i_size += BOGO_DIRENT_SIZE;
1842 dir->i_ctime = dir->i_mtime = CURRENT_TIME; 1835 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1843 d_instantiate(dentry, inode); 1836 d_instantiate(dentry, inode);
@@ -1957,7 +1950,7 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s
1957 if (len > PAGE_CACHE_SIZE) 1950 if (len > PAGE_CACHE_SIZE)
1958 return -ENAMETOOLONG; 1951 return -ENAMETOOLONG;
1959 1952
1960 inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE); 1953 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
1961 if (!inode) 1954 if (!inode)
1962 return -ENOSPC; 1955 return -ENOSPC;
1963 1956
@@ -1992,8 +1985,6 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s
1992 unlock_page(page); 1985 unlock_page(page);
1993 page_cache_release(page); 1986 page_cache_release(page);
1994 } 1987 }
1995 if (dir->i_mode & S_ISGID)
1996 inode->i_gid = dir->i_gid;
1997 dir->i_size += BOGO_DIRENT_SIZE; 1988 dir->i_size += BOGO_DIRENT_SIZE;
1998 dir->i_ctime = dir->i_mtime = CURRENT_TIME; 1989 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1999 d_instantiate(dentry, inode); 1990 d_instantiate(dentry, inode);
@@ -2071,14 +2062,14 @@ static int shmem_xattr_security_set(struct dentry *dentry, const char *name,
2071 size, flags); 2062 size, flags);
2072} 2063}
2073 2064
2074static struct xattr_handler shmem_xattr_security_handler = { 2065static const struct xattr_handler shmem_xattr_security_handler = {
2075 .prefix = XATTR_SECURITY_PREFIX, 2066 .prefix = XATTR_SECURITY_PREFIX,
2076 .list = shmem_xattr_security_list, 2067 .list = shmem_xattr_security_list,
2077 .get = shmem_xattr_security_get, 2068 .get = shmem_xattr_security_get,
2078 .set = shmem_xattr_security_set, 2069 .set = shmem_xattr_security_set,
2079}; 2070};
2080 2071
2081static struct xattr_handler *shmem_xattr_handlers[] = { 2072static const struct xattr_handler *shmem_xattr_handlers[] = {
2082 &generic_acl_access_handler, 2073 &generic_acl_access_handler,
2083 &generic_acl_default_handler, 2074 &generic_acl_default_handler,
2084 &shmem_xattr_security_handler, 2075 &shmem_xattr_security_handler,
@@ -2366,7 +2357,7 @@ int shmem_fill_super(struct super_block *sb, void *data, int silent)
2366 sb->s_flags |= MS_POSIXACL; 2357 sb->s_flags |= MS_POSIXACL;
2367#endif 2358#endif
2368 2359
2369 inode = shmem_get_inode(sb, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 2360 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
2370 if (!inode) 2361 if (!inode)
2371 goto failed; 2362 goto failed;
2372 inode->i_uid = sbinfo->uid; 2363 inode->i_uid = sbinfo->uid;
@@ -2611,7 +2602,7 @@ int shmem_lock(struct file *file, int lock, struct user_struct *user)
2611 2602
2612#define shmem_vm_ops generic_file_vm_ops 2603#define shmem_vm_ops generic_file_vm_ops
2613#define shmem_file_operations ramfs_file_operations 2604#define shmem_file_operations ramfs_file_operations
2614#define shmem_get_inode(sb, mode, dev, flags) ramfs_get_inode(sb, mode, dev) 2605#define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev)
2615#define shmem_acct_size(flags, size) 0 2606#define shmem_acct_size(flags, size) 0
2616#define shmem_unacct_size(flags, size) do {} while (0) 2607#define shmem_unacct_size(flags, size) do {} while (0)
2617#define SHMEM_MAX_BYTES MAX_LFS_FILESIZE 2608#define SHMEM_MAX_BYTES MAX_LFS_FILESIZE
@@ -2655,7 +2646,7 @@ struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags
2655 path.mnt = mntget(shm_mnt); 2646 path.mnt = mntget(shm_mnt);
2656 2647
2657 error = -ENOSPC; 2648 error = -ENOSPC;
2658 inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0, flags); 2649 inode = shmem_get_inode(root->d_sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
2659 if (!inode) 2650 if (!inode)
2660 goto put_dentry; 2651 goto put_dentry;
2661 2652
diff --git a/mm/slab.c b/mm/slab.c
index 525c6646646..50a73fca19c 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -115,6 +115,7 @@
115#include <linux/reciprocal_div.h> 115#include <linux/reciprocal_div.h>
116#include <linux/debugobjects.h> 116#include <linux/debugobjects.h>
117#include <linux/kmemcheck.h> 117#include <linux/kmemcheck.h>
118#include <linux/memory.h>
118 119
119#include <asm/cacheflush.h> 120#include <asm/cacheflush.h>
120#include <asm/tlbflush.h> 121#include <asm/tlbflush.h>
@@ -144,30 +145,6 @@
144#define BYTES_PER_WORD sizeof(void *) 145#define BYTES_PER_WORD sizeof(void *)
145#define REDZONE_ALIGN max(BYTES_PER_WORD, __alignof__(unsigned long long)) 146#define REDZONE_ALIGN max(BYTES_PER_WORD, __alignof__(unsigned long long))
146 147
147#ifndef ARCH_KMALLOC_MINALIGN
148/*
149 * Enforce a minimum alignment for the kmalloc caches.
150 * Usually, the kmalloc caches are cache_line_size() aligned, except when
151 * DEBUG and FORCED_DEBUG are enabled, then they are BYTES_PER_WORD aligned.
152 * Some archs want to perform DMA into kmalloc caches and need a guaranteed
153 * alignment larger than the alignment of a 64-bit integer.
154 * ARCH_KMALLOC_MINALIGN allows that.
155 * Note that increasing this value may disable some debug features.
156 */
157#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
158#endif
159
160#ifndef ARCH_SLAB_MINALIGN
161/*
162 * Enforce a minimum alignment for all caches.
163 * Intended for archs that get misalignment faults even for BYTES_PER_WORD
164 * aligned buffers. Includes ARCH_KMALLOC_MINALIGN.
165 * If possible: Do not enable this flag for CONFIG_DEBUG_SLAB, it disables
166 * some debug features.
167 */
168#define ARCH_SLAB_MINALIGN 0
169#endif
170
171#ifndef ARCH_KMALLOC_FLAGS 148#ifndef ARCH_KMALLOC_FLAGS
172#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN 149#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
173#endif 150#endif
@@ -1102,6 +1079,52 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
1102} 1079}
1103#endif 1080#endif
1104 1081
1082/*
1083 * Allocates and initializes nodelists for a node on each slab cache, used for
1084 * either memory or cpu hotplug. If memory is being hot-added, the kmem_list3
1085 * will be allocated off-node since memory is not yet online for the new node.
1086 * When hotplugging memory or a cpu, existing nodelists are not replaced if
1087 * already in use.
1088 *
1089 * Must hold cache_chain_mutex.
1090 */
1091static int init_cache_nodelists_node(int node)
1092{
1093 struct kmem_cache *cachep;
1094 struct kmem_list3 *l3;
1095 const int memsize = sizeof(struct kmem_list3);
1096
1097 list_for_each_entry(cachep, &cache_chain, next) {
1098 /*
1099 * Set up the size64 kmemlist for cpu before we can
1100 * begin anything. Make sure some other cpu on this
1101 * node has not already allocated this
1102 */
1103 if (!cachep->nodelists[node]) {
1104 l3 = kmalloc_node(memsize, GFP_KERNEL, node);
1105 if (!l3)
1106 return -ENOMEM;
1107 kmem_list3_init(l3);
1108 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 +
1109 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
1110
1111 /*
1112 * The l3s don't come and go as CPUs come and
1113 * go. cache_chain_mutex is sufficient
1114 * protection here.
1115 */
1116 cachep->nodelists[node] = l3;
1117 }
1118
1119 spin_lock_irq(&cachep->nodelists[node]->list_lock);
1120 cachep->nodelists[node]->free_limit =
1121 (1 + nr_cpus_node(node)) *
1122 cachep->batchcount + cachep->num;
1123 spin_unlock_irq(&cachep->nodelists[node]->list_lock);
1124 }
1125 return 0;
1126}
1127
1105static void __cpuinit cpuup_canceled(long cpu) 1128static void __cpuinit cpuup_canceled(long cpu)
1106{ 1129{
1107 struct kmem_cache *cachep; 1130 struct kmem_cache *cachep;
@@ -1172,7 +1195,7 @@ static int __cpuinit cpuup_prepare(long cpu)
1172 struct kmem_cache *cachep; 1195 struct kmem_cache *cachep;
1173 struct kmem_list3 *l3 = NULL; 1196 struct kmem_list3 *l3 = NULL;
1174 int node = cpu_to_node(cpu); 1197 int node = cpu_to_node(cpu);
1175 const int memsize = sizeof(struct kmem_list3); 1198 int err;
1176 1199
1177 /* 1200 /*
1178 * We need to do this right in the beginning since 1201 * We need to do this right in the beginning since
@@ -1180,35 +1203,9 @@ static int __cpuinit cpuup_prepare(long cpu)
1180 * kmalloc_node allows us to add the slab to the right 1203 * kmalloc_node allows us to add the slab to the right
1181 * kmem_list3 and not this cpu's kmem_list3 1204 * kmem_list3 and not this cpu's kmem_list3
1182 */ 1205 */
1183 1206 err = init_cache_nodelists_node(node);
1184 list_for_each_entry(cachep, &cache_chain, next) { 1207 if (err < 0)
1185 /* 1208 goto bad;
1186 * Set up the size64 kmemlist for cpu before we can
1187 * begin anything. Make sure some other cpu on this
1188 * node has not already allocated this
1189 */
1190 if (!cachep->nodelists[node]) {
1191 l3 = kmalloc_node(memsize, GFP_KERNEL, node);
1192 if (!l3)
1193 goto bad;
1194 kmem_list3_init(l3);
1195 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 +
1196 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
1197
1198 /*
1199 * The l3s don't come and go as CPUs come and
1200 * go. cache_chain_mutex is sufficient
1201 * protection here.
1202 */
1203 cachep->nodelists[node] = l3;
1204 }
1205
1206 spin_lock_irq(&cachep->nodelists[node]->list_lock);
1207 cachep->nodelists[node]->free_limit =
1208 (1 + nr_cpus_node(node)) *
1209 cachep->batchcount + cachep->num;
1210 spin_unlock_irq(&cachep->nodelists[node]->list_lock);
1211 }
1212 1209
1213 /* 1210 /*
1214 * Now we can go ahead with allocating the shared arrays and 1211 * Now we can go ahead with allocating the shared arrays and
@@ -1331,11 +1328,75 @@ static struct notifier_block __cpuinitdata cpucache_notifier = {
1331 &cpuup_callback, NULL, 0 1328 &cpuup_callback, NULL, 0
1332}; 1329};
1333 1330
1331#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
1332/*
1333 * Drains freelist for a node on each slab cache, used for memory hot-remove.
1334 * Returns -EBUSY if all objects cannot be drained so that the node is not
1335 * removed.
1336 *
1337 * Must hold cache_chain_mutex.
1338 */
1339static int __meminit drain_cache_nodelists_node(int node)
1340{
1341 struct kmem_cache *cachep;
1342 int ret = 0;
1343
1344 list_for_each_entry(cachep, &cache_chain, next) {
1345 struct kmem_list3 *l3;
1346
1347 l3 = cachep->nodelists[node];
1348 if (!l3)
1349 continue;
1350
1351 drain_freelist(cachep, l3, l3->free_objects);
1352
1353 if (!list_empty(&l3->slabs_full) ||
1354 !list_empty(&l3->slabs_partial)) {
1355 ret = -EBUSY;
1356 break;
1357 }
1358 }
1359 return ret;
1360}
1361
1362static int __meminit slab_memory_callback(struct notifier_block *self,
1363 unsigned long action, void *arg)
1364{
1365 struct memory_notify *mnb = arg;
1366 int ret = 0;
1367 int nid;
1368
1369 nid = mnb->status_change_nid;
1370 if (nid < 0)
1371 goto out;
1372
1373 switch (action) {
1374 case MEM_GOING_ONLINE:
1375 mutex_lock(&cache_chain_mutex);
1376 ret = init_cache_nodelists_node(nid);
1377 mutex_unlock(&cache_chain_mutex);
1378 break;
1379 case MEM_GOING_OFFLINE:
1380 mutex_lock(&cache_chain_mutex);
1381 ret = drain_cache_nodelists_node(nid);
1382 mutex_unlock(&cache_chain_mutex);
1383 break;
1384 case MEM_ONLINE:
1385 case MEM_OFFLINE:
1386 case MEM_CANCEL_ONLINE:
1387 case MEM_CANCEL_OFFLINE:
1388 break;
1389 }
1390out:
1391 return ret ? notifier_from_errno(ret) : NOTIFY_OK;
1392}
1393#endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */
1394
1334/* 1395/*
1335 * swap the static kmem_list3 with kmalloced memory 1396 * swap the static kmem_list3 with kmalloced memory
1336 */ 1397 */
1337static void init_list(struct kmem_cache *cachep, struct kmem_list3 *list, 1398static void __init init_list(struct kmem_cache *cachep, struct kmem_list3 *list,
1338 int nodeid) 1399 int nodeid)
1339{ 1400{
1340 struct kmem_list3 *ptr; 1401 struct kmem_list3 *ptr;
1341 1402
@@ -1580,6 +1641,14 @@ void __init kmem_cache_init_late(void)
1580 */ 1641 */
1581 register_cpu_notifier(&cpucache_notifier); 1642 register_cpu_notifier(&cpucache_notifier);
1582 1643
1644#ifdef CONFIG_NUMA
1645 /*
1646 * Register a memory hotplug callback that initializes and frees
1647 * nodelists.
1648 */
1649 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
1650#endif
1651
1583 /* 1652 /*
1584 * The reap timers are started later, with a module init call: That part 1653 * The reap timers are started later, with a module init call: That part
1585 * of the kernel is not yet operational. 1654 * of the kernel is not yet operational.
@@ -4216,10 +4285,11 @@ static int s_show(struct seq_file *m, void *p)
4216 unsigned long node_frees = cachep->node_frees; 4285 unsigned long node_frees = cachep->node_frees;
4217 unsigned long overflows = cachep->node_overflow; 4286 unsigned long overflows = cachep->node_overflow;
4218 4287
4219 seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu \ 4288 seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu "
4220 %4lu %4lu %4lu %4lu %4lu", allocs, high, grown, 4289 "%4lu %4lu %4lu %4lu %4lu",
4221 reaped, errors, max_freeable, node_allocs, 4290 allocs, high, grown,
4222 node_frees, overflows); 4291 reaped, errors, max_freeable, node_allocs,
4292 node_frees, overflows);
4223 } 4293 }
4224 /* cpu stats */ 4294 /* cpu stats */
4225 { 4295 {
diff --git a/mm/slob.c b/mm/slob.c
index 837ebd64cc3..23631e2bb57 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -467,14 +467,6 @@ out:
467 * End of slob allocator proper. Begin kmem_cache_alloc and kmalloc frontend. 467 * End of slob allocator proper. Begin kmem_cache_alloc and kmalloc frontend.
468 */ 468 */
469 469
470#ifndef ARCH_KMALLOC_MINALIGN
471#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long)
472#endif
473
474#ifndef ARCH_SLAB_MINALIGN
475#define ARCH_SLAB_MINALIGN __alignof__(unsigned long)
476#endif
477
478void *__kmalloc_node(size_t size, gfp_t gfp, int node) 470void *__kmalloc_node(size_t size, gfp_t gfp, int node)
479{ 471{
480 unsigned int *m; 472 unsigned int *m;
diff --git a/mm/slub.c b/mm/slub.c
index 7d6c8b1ccf6..e46e3129697 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -157,14 +157,6 @@
157#define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \ 157#define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
158 SLAB_CACHE_DMA | SLAB_NOTRACK) 158 SLAB_CACHE_DMA | SLAB_NOTRACK)
159 159
160#ifndef ARCH_KMALLOC_MINALIGN
161#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
162#endif
163
164#ifndef ARCH_SLAB_MINALIGN
165#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
166#endif
167
168#define OO_SHIFT 16 160#define OO_SHIFT 16
169#define OO_MASK ((1 << OO_SHIFT) - 1) 161#define OO_MASK ((1 << OO_SHIFT) - 1)
170#define MAX_OBJS_PER_PAGE 65535 /* since page.objects is u16 */ 162#define MAX_OBJS_PER_PAGE 65535 /* since page.objects is u16 */
@@ -1084,7 +1076,7 @@ static inline struct page *alloc_slab_page(gfp_t flags, int node,
1084 if (node == -1) 1076 if (node == -1)
1085 return alloc_pages(flags, order); 1077 return alloc_pages(flags, order);
1086 else 1078 else
1087 return alloc_pages_node(node, flags, order); 1079 return alloc_pages_exact_node(node, flags, order);
1088} 1080}
1089 1081
1090static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) 1082static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
@@ -2153,7 +2145,7 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
2153 int local_node; 2145 int local_node;
2154 2146
2155 if (slab_state >= UP && (s < kmalloc_caches || 2147 if (slab_state >= UP && (s < kmalloc_caches ||
2156 s > kmalloc_caches + KMALLOC_CACHES)) 2148 s >= kmalloc_caches + KMALLOC_CACHES))
2157 local_node = page_to_nid(virt_to_page(s)); 2149 local_node = page_to_nid(virt_to_page(s));
2158 else 2150 else
2159 local_node = 0; 2151 local_node = 0;
@@ -2429,9 +2421,11 @@ static void list_slab_objects(struct kmem_cache *s, struct page *page,
2429#ifdef CONFIG_SLUB_DEBUG 2421#ifdef CONFIG_SLUB_DEBUG
2430 void *addr = page_address(page); 2422 void *addr = page_address(page);
2431 void *p; 2423 void *p;
2432 DECLARE_BITMAP(map, page->objects); 2424 long *map = kzalloc(BITS_TO_LONGS(page->objects) * sizeof(long),
2425 GFP_ATOMIC);
2433 2426
2434 bitmap_zero(map, page->objects); 2427 if (!map)
2428 return;
2435 slab_err(s, page, "%s", text); 2429 slab_err(s, page, "%s", text);
2436 slab_lock(page); 2430 slab_lock(page);
2437 for_each_free_object(p, s, page->freelist) 2431 for_each_free_object(p, s, page->freelist)
@@ -2446,6 +2440,7 @@ static void list_slab_objects(struct kmem_cache *s, struct page *page,
2446 } 2440 }
2447 } 2441 }
2448 slab_unlock(page); 2442 slab_unlock(page);
2443 kfree(map);
2449#endif 2444#endif
2450} 2445}
2451 2446
@@ -3338,8 +3333,15 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
3338 struct kmem_cache *s; 3333 struct kmem_cache *s;
3339 void *ret; 3334 void *ret;
3340 3335
3341 if (unlikely(size > SLUB_MAX_SIZE)) 3336 if (unlikely(size > SLUB_MAX_SIZE)) {
3342 return kmalloc_large_node(size, gfpflags, node); 3337 ret = kmalloc_large_node(size, gfpflags, node);
3338
3339 trace_kmalloc_node(caller, ret,
3340 size, PAGE_SIZE << get_order(size),
3341 gfpflags, node);
3342
3343 return ret;
3344 }
3343 3345
3344 s = get_slab(size, gfpflags); 3346 s = get_slab(size, gfpflags);
3345 3347
@@ -3651,10 +3653,10 @@ static int add_location(struct loc_track *t, struct kmem_cache *s,
3651} 3653}
3652 3654
3653static void process_slab(struct loc_track *t, struct kmem_cache *s, 3655static void process_slab(struct loc_track *t, struct kmem_cache *s,
3654 struct page *page, enum track_item alloc) 3656 struct page *page, enum track_item alloc,
3657 long *map)
3655{ 3658{
3656 void *addr = page_address(page); 3659 void *addr = page_address(page);
3657 DECLARE_BITMAP(map, page->objects);
3658 void *p; 3660 void *p;
3659 3661
3660 bitmap_zero(map, page->objects); 3662 bitmap_zero(map, page->objects);
@@ -3673,11 +3675,14 @@ static int list_locations(struct kmem_cache *s, char *buf,
3673 unsigned long i; 3675 unsigned long i;
3674 struct loc_track t = { 0, 0, NULL }; 3676 struct loc_track t = { 0, 0, NULL };
3675 int node; 3677 int node;
3678 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
3679 sizeof(unsigned long), GFP_KERNEL);
3676 3680
3677 if (!alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location), 3681 if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
3678 GFP_TEMPORARY)) 3682 GFP_TEMPORARY)) {
3683 kfree(map);
3679 return sprintf(buf, "Out of memory\n"); 3684 return sprintf(buf, "Out of memory\n");
3680 3685 }
3681 /* Push back cpu slabs */ 3686 /* Push back cpu slabs */
3682 flush_all(s); 3687 flush_all(s);
3683 3688
@@ -3691,9 +3696,9 @@ static int list_locations(struct kmem_cache *s, char *buf,
3691 3696
3692 spin_lock_irqsave(&n->list_lock, flags); 3697 spin_lock_irqsave(&n->list_lock, flags);
3693 list_for_each_entry(page, &n->partial, lru) 3698 list_for_each_entry(page, &n->partial, lru)
3694 process_slab(&t, s, page, alloc); 3699 process_slab(&t, s, page, alloc, map);
3695 list_for_each_entry(page, &n->full, lru) 3700 list_for_each_entry(page, &n->full, lru)
3696 process_slab(&t, s, page, alloc); 3701 process_slab(&t, s, page, alloc, map);
3697 spin_unlock_irqrestore(&n->list_lock, flags); 3702 spin_unlock_irqrestore(&n->list_lock, flags);
3698 } 3703 }
3699 3704
@@ -3744,6 +3749,7 @@ static int list_locations(struct kmem_cache *s, char *buf,
3744 } 3749 }
3745 3750
3746 free_loc_track(&t); 3751 free_loc_track(&t);
3752 kfree(map);
3747 if (!t.count) 3753 if (!t.count)
3748 len += sprintf(buf, "No data\n"); 3754 len += sprintf(buf, "No data\n");
3749 return len; 3755 return len;
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 6cd0a8f90dc..03aa2d55f1a 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -139,7 +139,8 @@ static int discard_swap(struct swap_info_struct *si)
139 nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9); 139 nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
140 if (nr_blocks) { 140 if (nr_blocks) {
141 err = blkdev_issue_discard(si->bdev, start_block, 141 err = blkdev_issue_discard(si->bdev, start_block,
142 nr_blocks, GFP_KERNEL, DISCARD_FL_BARRIER); 142 nr_blocks, GFP_KERNEL,
143 BLKDEV_IFL_WAIT | BLKDEV_IFL_BARRIER);
143 if (err) 144 if (err)
144 return err; 145 return err;
145 cond_resched(); 146 cond_resched();
@@ -150,7 +151,8 @@ static int discard_swap(struct swap_info_struct *si)
150 nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9); 151 nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
151 152
152 err = blkdev_issue_discard(si->bdev, start_block, 153 err = blkdev_issue_discard(si->bdev, start_block,
153 nr_blocks, GFP_KERNEL, DISCARD_FL_BARRIER); 154 nr_blocks, GFP_KERNEL,
155 BLKDEV_IFL_WAIT | BLKDEV_IFL_BARRIER);
154 if (err) 156 if (err)
155 break; 157 break;
156 158
@@ -189,7 +191,8 @@ static void discard_swap_cluster(struct swap_info_struct *si,
189 start_block <<= PAGE_SHIFT - 9; 191 start_block <<= PAGE_SHIFT - 9;
190 nr_blocks <<= PAGE_SHIFT - 9; 192 nr_blocks <<= PAGE_SHIFT - 9;
191 if (blkdev_issue_discard(si->bdev, start_block, 193 if (blkdev_issue_discard(si->bdev, start_block,
192 nr_blocks, GFP_NOIO, DISCARD_FL_BARRIER)) 194 nr_blocks, GFP_NOIO, BLKDEV_IFL_WAIT |
195 BLKDEV_IFL_BARRIER))
193 break; 196 break;
194 } 197 }
195 198
@@ -574,6 +577,7 @@ static unsigned char swap_entry_free(struct swap_info_struct *p,
574 577
575 /* free if no reference */ 578 /* free if no reference */
576 if (!usage) { 579 if (!usage) {
580 struct gendisk *disk = p->bdev->bd_disk;
577 if (offset < p->lowest_bit) 581 if (offset < p->lowest_bit)
578 p->lowest_bit = offset; 582 p->lowest_bit = offset;
579 if (offset > p->highest_bit) 583 if (offset > p->highest_bit)
@@ -583,6 +587,9 @@ static unsigned char swap_entry_free(struct swap_info_struct *p,
583 swap_list.next = p->type; 587 swap_list.next = p->type;
584 nr_swap_pages++; 588 nr_swap_pages++;
585 p->inuse_pages--; 589 p->inuse_pages--;
590 if ((p->flags & SWP_BLKDEV) &&
591 disk->fops->swap_slot_free_notify)
592 disk->fops->swap_slot_free_notify(p->bdev, offset);
586 } 593 }
587 594
588 return usage; 595 return usage;
@@ -1884,6 +1891,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
1884 if (error < 0) 1891 if (error < 0)
1885 goto bad_swap; 1892 goto bad_swap;
1886 p->bdev = bdev; 1893 p->bdev = bdev;
1894 p->flags |= SWP_BLKDEV;
1887 } else if (S_ISREG(inode->i_mode)) { 1895 } else if (S_ISREG(inode->i_mode)) {
1888 p->bdev = inode->i_sb->s_bdev; 1896 p->bdev = inode->i_sb->s_bdev;
1889 mutex_lock(&inode->i_mutex); 1897 mutex_lock(&inode->i_mutex);
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-17 19:41:05 -0400