diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/memory_hotplug.c | 48 | ||||
| -rw-r--r-- | mm/shmem.c | 37 | ||||
| -rw-r--r-- | mm/slub.c | 118 |
3 files changed, 177 insertions, 26 deletions
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 1833879f8438..3a47871a29d9 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c | |||
| @@ -187,7 +187,24 @@ int online_pages(unsigned long pfn, unsigned long nr_pages) | |||
| 187 | unsigned long onlined_pages = 0; | 187 | unsigned long onlined_pages = 0; |
| 188 | struct zone *zone; | 188 | struct zone *zone; |
| 189 | int need_zonelists_rebuild = 0; | 189 | int need_zonelists_rebuild = 0; |
| 190 | int nid; | ||
| 191 | int ret; | ||
| 192 | struct memory_notify arg; | ||
| 193 | |||
| 194 | arg.start_pfn = pfn; | ||
| 195 | arg.nr_pages = nr_pages; | ||
| 196 | arg.status_change_nid = -1; | ||
| 197 | |||
| 198 | nid = page_to_nid(pfn_to_page(pfn)); | ||
| 199 | if (node_present_pages(nid) == 0) | ||
| 200 | arg.status_change_nid = nid; | ||
| 190 | 201 | ||
| 202 | ret = memory_notify(MEM_GOING_ONLINE, &arg); | ||
| 203 | ret = notifier_to_errno(ret); | ||
| 204 | if (ret) { | ||
| 205 | memory_notify(MEM_CANCEL_ONLINE, &arg); | ||
| 206 | return ret; | ||
| 207 | } | ||
| 191 | /* | 208 | /* |
| 192 | * This doesn't need a lock to do pfn_to_page(). | 209 | * This doesn't need a lock to do pfn_to_page(). |
| 193 | * The section can't be removed here because of the | 210 | * The section can't be removed here because of the |
| @@ -222,6 +239,10 @@ int online_pages(unsigned long pfn, unsigned long nr_pages) | |||
| 222 | build_all_zonelists(); | 239 | build_all_zonelists(); |
| 223 | vm_total_pages = nr_free_pagecache_pages(); | 240 | vm_total_pages = nr_free_pagecache_pages(); |
| 224 | writeback_set_ratelimit(); | 241 | writeback_set_ratelimit(); |
| 242 | |||
| 243 | if (onlined_pages) | ||
| 244 | memory_notify(MEM_ONLINE, &arg); | ||
| 245 | |||
| 225 | return 0; | 246 | return 0; |
| 226 | } | 247 | } |
| 227 | #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ | 248 | #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ |
| @@ -467,8 +488,9 @@ int offline_pages(unsigned long start_pfn, | |||
| 467 | { | 488 | { |
| 468 | unsigned long pfn, nr_pages, expire; | 489 | unsigned long pfn, nr_pages, expire; |
| 469 | long offlined_pages; | 490 | long offlined_pages; |
| 470 | int ret, drain, retry_max; | 491 | int ret, drain, retry_max, node; |
| 471 | struct zone *zone; | 492 | struct zone *zone; |
| 493 | struct memory_notify arg; | ||
| 472 | 494 | ||
| 473 | BUG_ON(start_pfn >= end_pfn); | 495 | BUG_ON(start_pfn >= end_pfn); |
| 474 | /* at least, alignment against pageblock is necessary */ | 496 | /* at least, alignment against pageblock is necessary */ |
| @@ -480,11 +502,27 @@ int offline_pages(unsigned long start_pfn, | |||
| 480 | we assume this for now. .*/ | 502 | we assume this for now. .*/ |
| 481 | if (!test_pages_in_a_zone(start_pfn, end_pfn)) | 503 | if (!test_pages_in_a_zone(start_pfn, end_pfn)) |
| 482 | return -EINVAL; | 504 | return -EINVAL; |
| 505 | |||
| 506 | zone = page_zone(pfn_to_page(start_pfn)); | ||
| 507 | node = zone_to_nid(zone); | ||
| 508 | nr_pages = end_pfn - start_pfn; | ||
| 509 | |||
| 483 | /* set above range as isolated */ | 510 | /* set above range as isolated */ |
| 484 | ret = start_isolate_page_range(start_pfn, end_pfn); | 511 | ret = start_isolate_page_range(start_pfn, end_pfn); |
| 485 | if (ret) | 512 | if (ret) |
| 486 | return ret; | 513 | return ret; |
| 487 | nr_pages = end_pfn - start_pfn; | 514 | |
| 515 | arg.start_pfn = start_pfn; | ||
| 516 | arg.nr_pages = nr_pages; | ||
| 517 | arg.status_change_nid = -1; | ||
| 518 | if (nr_pages >= node_present_pages(node)) | ||
| 519 | arg.status_change_nid = node; | ||
| 520 | |||
| 521 | ret = memory_notify(MEM_GOING_OFFLINE, &arg); | ||
| 522 | ret = notifier_to_errno(ret); | ||
| 523 | if (ret) | ||
| 524 | goto failed_removal; | ||
| 525 | |||
| 488 | pfn = start_pfn; | 526 | pfn = start_pfn; |
| 489 | expire = jiffies + timeout; | 527 | expire = jiffies + timeout; |
| 490 | drain = 0; | 528 | drain = 0; |
| @@ -539,20 +577,24 @@ repeat: | |||
| 539 | /* reset pagetype flags */ | 577 | /* reset pagetype flags */ |
| 540 | start_isolate_page_range(start_pfn, end_pfn); | 578 | start_isolate_page_range(start_pfn, end_pfn); |
| 541 | /* removal success */ | 579 | /* removal success */ |
| 542 | zone = page_zone(pfn_to_page(start_pfn)); | ||
| 543 | zone->present_pages -= offlined_pages; | 580 | zone->present_pages -= offlined_pages; |
| 544 | zone->zone_pgdat->node_present_pages -= offlined_pages; | 581 | zone->zone_pgdat->node_present_pages -= offlined_pages; |
| 545 | totalram_pages -= offlined_pages; | 582 | totalram_pages -= offlined_pages; |
| 546 | num_physpages -= offlined_pages; | 583 | num_physpages -= offlined_pages; |
| 584 | |||
| 547 | vm_total_pages = nr_free_pagecache_pages(); | 585 | vm_total_pages = nr_free_pagecache_pages(); |
| 548 | writeback_set_ratelimit(); | 586 | writeback_set_ratelimit(); |
| 587 | |||
| 588 | memory_notify(MEM_OFFLINE, &arg); | ||
| 549 | return 0; | 589 | return 0; |
| 550 | 590 | ||
| 551 | failed_removal: | 591 | failed_removal: |
| 552 | printk(KERN_INFO "memory offlining %lx to %lx failed\n", | 592 | printk(KERN_INFO "memory offlining %lx to %lx failed\n", |
| 553 | start_pfn, end_pfn); | 593 | start_pfn, end_pfn); |
| 594 | memory_notify(MEM_CANCEL_OFFLINE, &arg); | ||
| 554 | /* pushback to free area */ | 595 | /* pushback to free area */ |
| 555 | undo_isolate_page_range(start_pfn, end_pfn); | 596 | undo_isolate_page_range(start_pfn, end_pfn); |
| 597 | |||
| 556 | return ret; | 598 | return ret; |
| 557 | } | 599 | } |
| 558 | #else | 600 | #else |
diff --git a/mm/shmem.c b/mm/shmem.c index 289dbb0a6fd6..404e53bb2127 100644 --- a/mm/shmem.c +++ b/mm/shmem.c | |||
| @@ -2020,33 +2020,25 @@ static int shmem_match(struct inode *ino, void *vfh) | |||
| 2020 | return ino->i_ino == inum && fh[0] == ino->i_generation; | 2020 | return ino->i_ino == inum && fh[0] == ino->i_generation; |
| 2021 | } | 2021 | } |
| 2022 | 2022 | ||
| 2023 | static struct dentry *shmem_get_dentry(struct super_block *sb, void *vfh) | 2023 | static struct dentry *shmem_fh_to_dentry(struct super_block *sb, |
| 2024 | struct fid *fid, int fh_len, int fh_type) | ||
| 2024 | { | 2025 | { |
| 2025 | struct dentry *de = NULL; | ||
| 2026 | struct inode *inode; | 2026 | struct inode *inode; |
| 2027 | __u32 *fh = vfh; | 2027 | struct dentry *dentry = NULL; |
| 2028 | __u64 inum = fh[2]; | 2028 | u64 inum = fid->raw[2]; |
| 2029 | inum = (inum << 32) | fh[1]; | 2029 | inum = (inum << 32) | fid->raw[1]; |
| 2030 | |||
| 2031 | if (fh_len < 3) | ||
| 2032 | return NULL; | ||
| 2030 | 2033 | ||
| 2031 | inode = ilookup5(sb, (unsigned long)(inum+fh[0]), shmem_match, vfh); | 2034 | inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), |
| 2035 | shmem_match, fid->raw); | ||
| 2032 | if (inode) { | 2036 | if (inode) { |
| 2033 | de = d_find_alias(inode); | 2037 | dentry = d_find_alias(inode); |
| 2034 | iput(inode); | 2038 | iput(inode); |
| 2035 | } | 2039 | } |
| 2036 | 2040 | ||
| 2037 | return de? de: ERR_PTR(-ESTALE); | 2041 | return dentry; |
| 2038 | } | ||
| 2039 | |||
| 2040 | static struct dentry *shmem_decode_fh(struct super_block *sb, __u32 *fh, | ||
| 2041 | int len, int type, | ||
| 2042 | int (*acceptable)(void *context, struct dentry *de), | ||
| 2043 | void *context) | ||
| 2044 | { | ||
| 2045 | if (len < 3) | ||
| 2046 | return ERR_PTR(-ESTALE); | ||
| 2047 | |||
| 2048 | return sb->s_export_op->find_exported_dentry(sb, fh, NULL, acceptable, | ||
| 2049 | context); | ||
| 2050 | } | 2042 | } |
| 2051 | 2043 | ||
| 2052 | static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len, | 2044 | static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len, |
| @@ -2079,11 +2071,10 @@ static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len, | |||
| 2079 | return 1; | 2071 | return 1; |
| 2080 | } | 2072 | } |
| 2081 | 2073 | ||
| 2082 | static struct export_operations shmem_export_ops = { | 2074 | static const struct export_operations shmem_export_ops = { |
| 2083 | .get_parent = shmem_get_parent, | 2075 | .get_parent = shmem_get_parent, |
| 2084 | .get_dentry = shmem_get_dentry, | ||
| 2085 | .encode_fh = shmem_encode_fh, | 2076 | .encode_fh = shmem_encode_fh, |
| 2086 | .decode_fh = shmem_decode_fh, | 2077 | .fh_to_dentry = shmem_fh_to_dentry, |
| 2087 | }; | 2078 | }; |
| 2088 | 2079 | ||
| 2089 | static int shmem_parse_options(char *options, int *mode, uid_t *uid, | 2080 | static int shmem_parse_options(char *options, int *mode, uid_t *uid, |
| @@ -20,6 +20,7 @@ | |||
| 20 | #include <linux/mempolicy.h> | 20 | #include <linux/mempolicy.h> |
| 21 | #include <linux/ctype.h> | 21 | #include <linux/ctype.h> |
| 22 | #include <linux/kallsyms.h> | 22 | #include <linux/kallsyms.h> |
| 23 | #include <linux/memory.h> | ||
| 23 | 24 | ||
| 24 | /* | 25 | /* |
| 25 | * Lock order: | 26 | * Lock order: |
| @@ -2694,6 +2695,121 @@ int kmem_cache_shrink(struct kmem_cache *s) | |||
| 2694 | } | 2695 | } |
| 2695 | EXPORT_SYMBOL(kmem_cache_shrink); | 2696 | EXPORT_SYMBOL(kmem_cache_shrink); |
| 2696 | 2697 | ||
| 2698 | #if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG) | ||
| 2699 | static int slab_mem_going_offline_callback(void *arg) | ||
| 2700 | { | ||
| 2701 | struct kmem_cache *s; | ||
| 2702 | |||
| 2703 | down_read(&slub_lock); | ||
| 2704 | list_for_each_entry(s, &slab_caches, list) | ||
| 2705 | kmem_cache_shrink(s); | ||
| 2706 | up_read(&slub_lock); | ||
| 2707 | |||
| 2708 | return 0; | ||
| 2709 | } | ||
| 2710 | |||
| 2711 | static void slab_mem_offline_callback(void *arg) | ||
| 2712 | { | ||
| 2713 | struct kmem_cache_node *n; | ||
| 2714 | struct kmem_cache *s; | ||
| 2715 | struct memory_notify *marg = arg; | ||
| 2716 | int offline_node; | ||
| 2717 | |||
| 2718 | offline_node = marg->status_change_nid; | ||
| 2719 | |||
| 2720 | /* | ||
| 2721 | * If the node still has available memory. we need kmem_cache_node | ||
| 2722 | * for it yet. | ||
| 2723 | */ | ||
| 2724 | if (offline_node < 0) | ||
| 2725 | return; | ||
| 2726 | |||
| 2727 | down_read(&slub_lock); | ||
| 2728 | list_for_each_entry(s, &slab_caches, list) { | ||
| 2729 | n = get_node(s, offline_node); | ||
| 2730 | if (n) { | ||
| 2731 | /* | ||
| 2732 | * if n->nr_slabs > 0, slabs still exist on the node | ||
| 2733 | * that is going down. We were unable to free them, | ||
| 2734 | * and offline_pages() function shoudn't call this | ||
| 2735 | * callback. So, we must fail. | ||
| 2736 | */ | ||
| 2737 | BUG_ON(atomic_read(&n->nr_slabs)); | ||
| 2738 | |||
| 2739 | s->node[offline_node] = NULL; | ||
| 2740 | kmem_cache_free(kmalloc_caches, n); | ||
| 2741 | } | ||
| 2742 | } | ||
| 2743 | up_read(&slub_lock); | ||
| 2744 | } | ||
| 2745 | |||
| 2746 | static int slab_mem_going_online_callback(void *arg) | ||
| 2747 | { | ||
| 2748 | struct kmem_cache_node *n; | ||
| 2749 | struct kmem_cache *s; | ||
| 2750 | struct memory_notify *marg = arg; | ||
| 2751 | int nid = marg->status_change_nid; | ||
| 2752 | int ret = 0; | ||
| 2753 | |||
| 2754 | /* | ||
| 2755 | * If the node's memory is already available, then kmem_cache_node is | ||
| 2756 | * already created. Nothing to do. | ||
| 2757 | */ | ||
| 2758 | if (nid < 0) | ||
| 2759 | return 0; | ||
| 2760 | |||
| 2761 | /* | ||
| 2762 | * We are bringing a node online. No memory is availabe yet. We must | ||
| 2763 | * allocate a kmem_cache_node structure in order to bring the node | ||
| 2764 | * online. | ||
| 2765 | */ | ||
| 2766 | down_read(&slub_lock); | ||
| 2767 | list_for_each_entry(s, &slab_caches, list) { | ||
| 2768 | /* | ||
| 2769 | * XXX: kmem_cache_alloc_node will fallback to other nodes | ||
| 2770 | * since memory is not yet available from the node that | ||
| 2771 | * is brought up. | ||
| 2772 | */ | ||
| 2773 | n = kmem_cache_alloc(kmalloc_caches, GFP_KERNEL); | ||
| 2774 | if (!n) { | ||
| 2775 | ret = -ENOMEM; | ||
| 2776 | goto out; | ||
| 2777 | } | ||
| 2778 | init_kmem_cache_node(n); | ||
| 2779 | s->node[nid] = n; | ||
| 2780 | } | ||
| 2781 | out: | ||
| 2782 | up_read(&slub_lock); | ||
| 2783 | return ret; | ||
| 2784 | } | ||
| 2785 | |||
| 2786 | static int slab_memory_callback(struct notifier_block *self, | ||
| 2787 | unsigned long action, void *arg) | ||
| 2788 | { | ||
| 2789 | int ret = 0; | ||
| 2790 | |||
| 2791 | switch (action) { | ||
| 2792 | case MEM_GOING_ONLINE: | ||
| 2793 | ret = slab_mem_going_online_callback(arg); | ||
| 2794 | break; | ||
| 2795 | case MEM_GOING_OFFLINE: | ||
| 2796 | ret = slab_mem_going_offline_callback(arg); | ||
| 2797 | break; | ||
| 2798 | case MEM_OFFLINE: | ||
| 2799 | case MEM_CANCEL_ONLINE: | ||
| 2800 | slab_mem_offline_callback(arg); | ||
| 2801 | break; | ||
| 2802 | case MEM_ONLINE: | ||
| 2803 | case MEM_CANCEL_OFFLINE: | ||
| 2804 | break; | ||
| 2805 | } | ||
| 2806 | |||
| 2807 | ret = notifier_from_errno(ret); | ||
| 2808 | return ret; | ||
| 2809 | } | ||
| 2810 | |||
| 2811 | #endif /* CONFIG_MEMORY_HOTPLUG */ | ||
| 2812 | |||
| 2697 | /******************************************************************** | 2813 | /******************************************************************** |
| 2698 | * Basic setup of slabs | 2814 | * Basic setup of slabs |
| 2699 | *******************************************************************/ | 2815 | *******************************************************************/ |
| @@ -2715,6 +2831,8 @@ void __init kmem_cache_init(void) | |||
| 2715 | sizeof(struct kmem_cache_node), GFP_KERNEL); | 2831 | sizeof(struct kmem_cache_node), GFP_KERNEL); |
| 2716 | kmalloc_caches[0].refcount = -1; | 2832 | kmalloc_caches[0].refcount = -1; |
| 2717 | caches++; | 2833 | caches++; |
| 2834 | |||
| 2835 | hotplug_memory_notifier(slab_memory_callback, 1); | ||
| 2718 | #endif | 2836 | #endif |
| 2719 | 2837 | ||
| 2720 | /* Able to allocate the per node structures */ | 2838 | /* Able to allocate the per node structures */ |
