diff options
Diffstat (limited to 'mm/page_alloc.c')
| -rw-r--r-- | mm/page_alloc.c | 410 |
1 files changed, 255 insertions, 155 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 4e9f5cc5fb59..d03c946d5566 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c | |||
| @@ -50,6 +50,7 @@ | |||
| 50 | #include <linux/kmemleak.h> | 50 | #include <linux/kmemleak.h> |
| 51 | #include <linux/memory.h> | 51 | #include <linux/memory.h> |
| 52 | #include <trace/events/kmem.h> | 52 | #include <trace/events/kmem.h> |
| 53 | #include <linux/ftrace_event.h> | ||
| 53 | 54 | ||
| 54 | #include <asm/tlbflush.h> | 55 | #include <asm/tlbflush.h> |
| 55 | #include <asm/div64.h> | 56 | #include <asm/div64.h> |
| @@ -76,6 +77,31 @@ unsigned long totalreserve_pages __read_mostly; | |||
| 76 | int percpu_pagelist_fraction; | 77 | int percpu_pagelist_fraction; |
| 77 | gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; | 78 | gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; |
| 78 | 79 | ||
| 80 | #ifdef CONFIG_PM_SLEEP | ||
| 81 | /* | ||
| 82 | * The following functions are used by the suspend/hibernate code to temporarily | ||
| 83 | * change gfp_allowed_mask in order to avoid using I/O during memory allocations | ||
| 84 | * while devices are suspended. To avoid races with the suspend/hibernate code, | ||
| 85 | * they should always be called with pm_mutex held (gfp_allowed_mask also should | ||
| 86 | * only be modified with pm_mutex held, unless the suspend/hibernate code is | ||
| 87 | * guaranteed not to run in parallel with that modification). | ||
| 88 | */ | ||
| 89 | void set_gfp_allowed_mask(gfp_t mask) | ||
| 90 | { | ||
| 91 | WARN_ON(!mutex_is_locked(&pm_mutex)); | ||
| 92 | gfp_allowed_mask = mask; | ||
| 93 | } | ||
| 94 | |||
| 95 | gfp_t clear_gfp_allowed_mask(gfp_t mask) | ||
| 96 | { | ||
| 97 | gfp_t ret = gfp_allowed_mask; | ||
| 98 | |||
| 99 | WARN_ON(!mutex_is_locked(&pm_mutex)); | ||
| 100 | gfp_allowed_mask &= ~mask; | ||
| 101 | return ret; | ||
| 102 | } | ||
| 103 | #endif /* CONFIG_PM_SLEEP */ | ||
| 104 | |||
| 79 | #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE | 105 | #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE |
| 80 | int pageblock_order __read_mostly; | 106 | int pageblock_order __read_mostly; |
| 81 | #endif | 107 | #endif |
| @@ -263,10 +289,7 @@ static void bad_page(struct page *page) | |||
| 263 | 289 | ||
| 264 | printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n", | 290 | printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n", |
| 265 | current->comm, page_to_pfn(page)); | 291 | current->comm, page_to_pfn(page)); |
| 266 | printk(KERN_ALERT | 292 | dump_page(page); |
| 267 | "page:%p flags:%p count:%d mapcount:%d mapping:%p index:%lx\n", | ||
| 268 | page, (void *)page->flags, page_count(page), | ||
| 269 | page_mapcount(page), page->mapping, page->index); | ||
| 270 | 293 | ||
| 271 | dump_stack(); | 294 | dump_stack(); |
| 272 | out: | 295 | out: |
| @@ -530,7 +553,7 @@ static void free_pcppages_bulk(struct zone *zone, int count, | |||
| 530 | int batch_free = 0; | 553 | int batch_free = 0; |
| 531 | 554 | ||
| 532 | spin_lock(&zone->lock); | 555 | spin_lock(&zone->lock); |
| 533 | zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE); | 556 | zone->all_unreclaimable = 0; |
| 534 | zone->pages_scanned = 0; | 557 | zone->pages_scanned = 0; |
| 535 | 558 | ||
| 536 | __mod_zone_page_state(zone, NR_FREE_PAGES, count); | 559 | __mod_zone_page_state(zone, NR_FREE_PAGES, count); |
| @@ -556,8 +579,9 @@ static void free_pcppages_bulk(struct zone *zone, int count, | |||
| 556 | page = list_entry(list->prev, struct page, lru); | 579 | page = list_entry(list->prev, struct page, lru); |
| 557 | /* must delete as __free_one_page list manipulates */ | 580 | /* must delete as __free_one_page list manipulates */ |
| 558 | list_del(&page->lru); | 581 | list_del(&page->lru); |
| 559 | __free_one_page(page, zone, 0, migratetype); | 582 | /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ |
| 560 | trace_mm_page_pcpu_drain(page, 0, migratetype); | 583 | __free_one_page(page, zone, 0, page_private(page)); |
| 584 | trace_mm_page_pcpu_drain(page, 0, page_private(page)); | ||
| 561 | } while (--count && --batch_free && !list_empty(list)); | 585 | } while (--count && --batch_free && !list_empty(list)); |
| 562 | } | 586 | } |
| 563 | spin_unlock(&zone->lock); | 587 | spin_unlock(&zone->lock); |
| @@ -567,7 +591,7 @@ static void free_one_page(struct zone *zone, struct page *page, int order, | |||
| 567 | int migratetype) | 591 | int migratetype) |
| 568 | { | 592 | { |
| 569 | spin_lock(&zone->lock); | 593 | spin_lock(&zone->lock); |
| 570 | zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE); | 594 | zone->all_unreclaimable = 0; |
| 571 | zone->pages_scanned = 0; | 595 | zone->pages_scanned = 0; |
| 572 | 596 | ||
| 573 | __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order); | 597 | __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order); |
| @@ -582,6 +606,7 @@ static void __free_pages_ok(struct page *page, unsigned int order) | |||
| 582 | int bad = 0; | 606 | int bad = 0; |
| 583 | int wasMlocked = __TestClearPageMlocked(page); | 607 | int wasMlocked = __TestClearPageMlocked(page); |
| 584 | 608 | ||
| 609 | trace_mm_page_free_direct(page, order); | ||
| 585 | kmemcheck_free_shadow(page, order); | 610 | kmemcheck_free_shadow(page, order); |
| 586 | 611 | ||
| 587 | for (i = 0 ; i < (1 << order) ; ++i) | 612 | for (i = 0 ; i < (1 << order) ; ++i) |
| @@ -1008,10 +1033,10 @@ static void drain_pages(unsigned int cpu) | |||
| 1008 | struct per_cpu_pageset *pset; | 1033 | struct per_cpu_pageset *pset; |
| 1009 | struct per_cpu_pages *pcp; | 1034 | struct per_cpu_pages *pcp; |
| 1010 | 1035 | ||
| 1011 | pset = zone_pcp(zone, cpu); | 1036 | local_irq_save(flags); |
| 1037 | pset = per_cpu_ptr(zone->pageset, cpu); | ||
| 1012 | 1038 | ||
| 1013 | pcp = &pset->pcp; | 1039 | pcp = &pset->pcp; |
| 1014 | local_irq_save(flags); | ||
| 1015 | free_pcppages_bulk(zone, pcp->count, pcp); | 1040 | free_pcppages_bulk(zone, pcp->count, pcp); |
| 1016 | pcp->count = 0; | 1041 | pcp->count = 0; |
| 1017 | local_irq_restore(flags); | 1042 | local_irq_restore(flags); |
| @@ -1072,8 +1097,9 @@ void mark_free_pages(struct zone *zone) | |||
| 1072 | 1097 | ||
| 1073 | /* | 1098 | /* |
| 1074 | * Free a 0-order page | 1099 | * Free a 0-order page |
| 1100 | * cold == 1 ? free a cold page : free a hot page | ||
| 1075 | */ | 1101 | */ |
| 1076 | static void free_hot_cold_page(struct page *page, int cold) | 1102 | void free_hot_cold_page(struct page *page, int cold) |
| 1077 | { | 1103 | { |
| 1078 | struct zone *zone = page_zone(page); | 1104 | struct zone *zone = page_zone(page); |
| 1079 | struct per_cpu_pages *pcp; | 1105 | struct per_cpu_pages *pcp; |
| @@ -1081,6 +1107,7 @@ static void free_hot_cold_page(struct page *page, int cold) | |||
| 1081 | int migratetype; | 1107 | int migratetype; |
| 1082 | int wasMlocked = __TestClearPageMlocked(page); | 1108 | int wasMlocked = __TestClearPageMlocked(page); |
| 1083 | 1109 | ||
| 1110 | trace_mm_page_free_direct(page, 0); | ||
| 1084 | kmemcheck_free_shadow(page, 0); | 1111 | kmemcheck_free_shadow(page, 0); |
| 1085 | 1112 | ||
| 1086 | if (PageAnon(page)) | 1113 | if (PageAnon(page)) |
| @@ -1095,7 +1122,6 @@ static void free_hot_cold_page(struct page *page, int cold) | |||
| 1095 | arch_free_page(page, 0); | 1122 | arch_free_page(page, 0); |
| 1096 | kernel_map_pages(page, 1, 0); | 1123 | kernel_map_pages(page, 1, 0); |
| 1097 | 1124 | ||
| 1098 | pcp = &zone_pcp(zone, get_cpu())->pcp; | ||
| 1099 | migratetype = get_pageblock_migratetype(page); | 1125 | migratetype = get_pageblock_migratetype(page); |
| 1100 | set_page_private(page, migratetype); | 1126 | set_page_private(page, migratetype); |
| 1101 | local_irq_save(flags); | 1127 | local_irq_save(flags); |
| @@ -1118,6 +1144,7 @@ static void free_hot_cold_page(struct page *page, int cold) | |||
| 1118 | migratetype = MIGRATE_MOVABLE; | 1144 | migratetype = MIGRATE_MOVABLE; |
| 1119 | } | 1145 | } |
| 1120 | 1146 | ||
| 1147 | pcp = &this_cpu_ptr(zone->pageset)->pcp; | ||
| 1121 | if (cold) | 1148 | if (cold) |
| 1122 | list_add_tail(&page->lru, &pcp->lists[migratetype]); | 1149 | list_add_tail(&page->lru, &pcp->lists[migratetype]); |
| 1123 | else | 1150 | else |
| @@ -1130,15 +1157,8 @@ static void free_hot_cold_page(struct page *page, int cold) | |||
| 1130 | 1157 | ||
| 1131 | out: | 1158 | out: |
| 1132 | local_irq_restore(flags); | 1159 | local_irq_restore(flags); |
| 1133 | put_cpu(); | ||
| 1134 | } | 1160 | } |
| 1135 | 1161 | ||
| 1136 | void free_hot_page(struct page *page) | ||
| 1137 | { | ||
| 1138 | trace_mm_page_free_direct(page, 0); | ||
| 1139 | free_hot_cold_page(page, 0); | ||
| 1140 | } | ||
| 1141 | |||
| 1142 | /* | 1162 | /* |
| 1143 | * split_page takes a non-compound higher-order page, and splits it into | 1163 | * split_page takes a non-compound higher-order page, and splits it into |
| 1144 | * n (1<<order) sub-pages: page[0..n] | 1164 | * n (1<<order) sub-pages: page[0..n] |
| @@ -1180,17 +1200,15 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, | |||
| 1180 | unsigned long flags; | 1200 | unsigned long flags; |
| 1181 | struct page *page; | 1201 | struct page *page; |
| 1182 | int cold = !!(gfp_flags & __GFP_COLD); | 1202 | int cold = !!(gfp_flags & __GFP_COLD); |
| 1183 | int cpu; | ||
| 1184 | 1203 | ||
| 1185 | again: | 1204 | again: |
| 1186 | cpu = get_cpu(); | ||
| 1187 | if (likely(order == 0)) { | 1205 | if (likely(order == 0)) { |
| 1188 | struct per_cpu_pages *pcp; | 1206 | struct per_cpu_pages *pcp; |
| 1189 | struct list_head *list; | 1207 | struct list_head *list; |
| 1190 | 1208 | ||
| 1191 | pcp = &zone_pcp(zone, cpu)->pcp; | ||
| 1192 | list = &pcp->lists[migratetype]; | ||
| 1193 | local_irq_save(flags); | 1209 | local_irq_save(flags); |
| 1210 | pcp = &this_cpu_ptr(zone->pageset)->pcp; | ||
| 1211 | list = &pcp->lists[migratetype]; | ||
| 1194 | if (list_empty(list)) { | 1212 | if (list_empty(list)) { |
| 1195 | pcp->count += rmqueue_bulk(zone, 0, | 1213 | pcp->count += rmqueue_bulk(zone, 0, |
| 1196 | pcp->batch, list, | 1214 | pcp->batch, list, |
| @@ -1222,16 +1240,15 @@ again: | |||
| 1222 | } | 1240 | } |
| 1223 | spin_lock_irqsave(&zone->lock, flags); | 1241 | spin_lock_irqsave(&zone->lock, flags); |
| 1224 | page = __rmqueue(zone, order, migratetype); | 1242 | page = __rmqueue(zone, order, migratetype); |
| 1225 | __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order)); | ||
| 1226 | spin_unlock(&zone->lock); | 1243 | spin_unlock(&zone->lock); |
| 1227 | if (!page) | 1244 | if (!page) |
| 1228 | goto failed; | 1245 | goto failed; |
| 1246 | __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order)); | ||
| 1229 | } | 1247 | } |
| 1230 | 1248 | ||
| 1231 | __count_zone_vm_events(PGALLOC, zone, 1 << order); | 1249 | __count_zone_vm_events(PGALLOC, zone, 1 << order); |
| 1232 | zone_statistics(preferred_zone, zone); | 1250 | zone_statistics(preferred_zone, zone); |
| 1233 | local_irq_restore(flags); | 1251 | local_irq_restore(flags); |
| 1234 | put_cpu(); | ||
| 1235 | 1252 | ||
| 1236 | VM_BUG_ON(bad_range(zone, page)); | 1253 | VM_BUG_ON(bad_range(zone, page)); |
| 1237 | if (prep_new_page(page, order, gfp_flags)) | 1254 | if (prep_new_page(page, order, gfp_flags)) |
| @@ -1240,7 +1257,6 @@ again: | |||
| 1240 | 1257 | ||
| 1241 | failed: | 1258 | failed: |
| 1242 | local_irq_restore(flags); | 1259 | local_irq_restore(flags); |
| 1243 | put_cpu(); | ||
| 1244 | return NULL; | 1260 | return NULL; |
| 1245 | } | 1261 | } |
| 1246 | 1262 | ||
| @@ -2012,9 +2028,8 @@ void __pagevec_free(struct pagevec *pvec) | |||
| 2012 | void __free_pages(struct page *page, unsigned int order) | 2028 | void __free_pages(struct page *page, unsigned int order) |
| 2013 | { | 2029 | { |
| 2014 | if (put_page_testzero(page)) { | 2030 | if (put_page_testzero(page)) { |
| 2015 | trace_mm_page_free_direct(page, order); | ||
| 2016 | if (order == 0) | 2031 | if (order == 0) |
| 2017 | free_hot_page(page); | 2032 | free_hot_cold_page(page, 0); |
| 2018 | else | 2033 | else |
| 2019 | __free_pages_ok(page, order); | 2034 | __free_pages_ok(page, order); |
| 2020 | } | 2035 | } |
| @@ -2179,7 +2194,7 @@ void show_free_areas(void) | |||
| 2179 | for_each_online_cpu(cpu) { | 2194 | for_each_online_cpu(cpu) { |
| 2180 | struct per_cpu_pageset *pageset; | 2195 | struct per_cpu_pageset *pageset; |
| 2181 | 2196 | ||
| 2182 | pageset = zone_pcp(zone, cpu); | 2197 | pageset = per_cpu_ptr(zone->pageset, cpu); |
| 2183 | 2198 | ||
| 2184 | printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n", | 2199 | printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n", |
| 2185 | cpu, pageset->pcp.high, | 2200 | cpu, pageset->pcp.high, |
| @@ -2270,7 +2285,7 @@ void show_free_areas(void) | |||
| 2270 | K(zone_page_state(zone, NR_BOUNCE)), | 2285 | K(zone_page_state(zone, NR_BOUNCE)), |
| 2271 | K(zone_page_state(zone, NR_WRITEBACK_TEMP)), | 2286 | K(zone_page_state(zone, NR_WRITEBACK_TEMP)), |
| 2272 | zone->pages_scanned, | 2287 | zone->pages_scanned, |
| 2273 | (zone_is_all_unreclaimable(zone) ? "yes" : "no") | 2288 | (zone->all_unreclaimable ? "yes" : "no") |
| 2274 | ); | 2289 | ); |
| 2275 | printk("lowmem_reserve[]:"); | 2290 | printk("lowmem_reserve[]:"); |
| 2276 | for (i = 0; i < MAX_NR_ZONES; i++) | 2291 | for (i = 0; i < MAX_NR_ZONES; i++) |
| @@ -2744,10 +2759,29 @@ static void build_zonelist_cache(pg_data_t *pgdat) | |||
| 2744 | 2759 | ||
| 2745 | #endif /* CONFIG_NUMA */ | 2760 | #endif /* CONFIG_NUMA */ |
| 2746 | 2761 | ||
| 2762 | /* | ||
| 2763 | * Boot pageset table. One per cpu which is going to be used for all | ||
| 2764 | * zones and all nodes. The parameters will be set in such a way | ||
| 2765 | * that an item put on a list will immediately be handed over to | ||
| 2766 | * the buddy list. This is safe since pageset manipulation is done | ||
| 2767 | * with interrupts disabled. | ||
| 2768 | * | ||
| 2769 | * The boot_pagesets must be kept even after bootup is complete for | ||
| 2770 | * unused processors and/or zones. They do play a role for bootstrapping | ||
| 2771 | * hotplugged processors. | ||
| 2772 | * | ||
| 2773 | * zoneinfo_show() and maybe other functions do | ||
| 2774 | * not check if the processor is online before following the pageset pointer. | ||
| 2775 | * Other parts of the kernel may not check if the zone is available. | ||
| 2776 | */ | ||
| 2777 | static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); | ||
| 2778 | static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); | ||
| 2779 | |||
| 2747 | /* return values int ....just for stop_machine() */ | 2780 | /* return values int ....just for stop_machine() */ |
| 2748 | static int __build_all_zonelists(void *dummy) | 2781 | static int __build_all_zonelists(void *dummy) |
| 2749 | { | 2782 | { |
| 2750 | int nid; | 2783 | int nid; |
| 2784 | int cpu; | ||
| 2751 | 2785 | ||
| 2752 | #ifdef CONFIG_NUMA | 2786 | #ifdef CONFIG_NUMA |
| 2753 | memset(node_load, 0, sizeof(node_load)); | 2787 | memset(node_load, 0, sizeof(node_load)); |
| @@ -2758,6 +2792,23 @@ static int __build_all_zonelists(void *dummy) | |||
| 2758 | build_zonelists(pgdat); | 2792 | build_zonelists(pgdat); |
| 2759 | build_zonelist_cache(pgdat); | 2793 | build_zonelist_cache(pgdat); |
| 2760 | } | 2794 | } |
| 2795 | |||
| 2796 | /* | ||
| 2797 | * Initialize the boot_pagesets that are going to be used | ||
| 2798 | * for bootstrapping processors. The real pagesets for | ||
| 2799 | * each zone will be allocated later when the per cpu | ||
| 2800 | * allocator is available. | ||
| 2801 | * | ||
| 2802 | * boot_pagesets are used also for bootstrapping offline | ||
| 2803 | * cpus if the system is already booted because the pagesets | ||
| 2804 | * are needed to initialize allocators on a specific cpu too. | ||
| 2805 | * F.e. the percpu allocator needs the page allocator which | ||
| 2806 | * needs the percpu allocator in order to allocate its pagesets | ||
| 2807 | * (a chicken-egg dilemma). | ||
| 2808 | */ | ||
| 2809 | for_each_possible_cpu(cpu) | ||
| 2810 | setup_pageset(&per_cpu(boot_pageset, cpu), 0); | ||
| 2811 | |||
| 2761 | return 0; | 2812 | return 0; |
| 2762 | } | 2813 | } |
| 2763 | 2814 | ||
| @@ -3095,121 +3146,33 @@ static void setup_pagelist_highmark(struct per_cpu_pageset *p, | |||
| 3095 | pcp->batch = PAGE_SHIFT * 8; | 3146 | pcp->batch = PAGE_SHIFT * 8; |
| 3096 | } | 3147 | } |
| 3097 | 3148 | ||
| 3098 | |||
| 3099 | #ifdef CONFIG_NUMA | ||
| 3100 | /* | 3149 | /* |
| 3101 | * Boot pageset table. One per cpu which is going to be used for all | 3150 | * Allocate per cpu pagesets and initialize them. |
| 3102 | * zones and all nodes. The parameters will be set in such a way | 3151 | * Before this call only boot pagesets were available. |
| 3103 | * that an item put on a list will immediately be handed over to | 3152 | * Boot pagesets will no longer be used by this processorr |
| 3104 | * the buddy list. This is safe since pageset manipulation is done | 3153 | * after setup_per_cpu_pageset(). |
| 3105 | * with interrupts disabled. | ||
| 3106 | * | ||
| 3107 | * Some NUMA counter updates may also be caught by the boot pagesets. | ||
| 3108 | * | ||
| 3109 | * The boot_pagesets must be kept even after bootup is complete for | ||
| 3110 | * unused processors and/or zones. They do play a role for bootstrapping | ||
| 3111 | * hotplugged processors. | ||
| 3112 | * | ||
| 3113 | * zoneinfo_show() and maybe other functions do | ||
| 3114 | * not check if the processor is online before following the pageset pointer. | ||
| 3115 | * Other parts of the kernel may not check if the zone is available. | ||
| 3116 | */ | ||
| 3117 | static struct per_cpu_pageset boot_pageset[NR_CPUS]; | ||
| 3118 | |||
| 3119 | /* | ||
| 3120 | * Dynamically allocate memory for the | ||
| 3121 | * per cpu pageset array in struct zone. | ||
| 3122 | */ | 3154 | */ |
| 3123 | static int __cpuinit process_zones(int cpu) | 3155 | void __init setup_per_cpu_pageset(void) |
| 3124 | { | 3156 | { |
| 3125 | struct zone *zone, *dzone; | 3157 | struct zone *zone; |
| 3126 | int node = cpu_to_node(cpu); | 3158 | int cpu; |
| 3127 | |||
| 3128 | node_set_state(node, N_CPU); /* this node has a cpu */ | ||
| 3129 | 3159 | ||
| 3130 | for_each_populated_zone(zone) { | 3160 | for_each_populated_zone(zone) { |
| 3131 | zone_pcp(zone, cpu) = kmalloc_node(sizeof(struct per_cpu_pageset), | 3161 | zone->pageset = alloc_percpu(struct per_cpu_pageset); |
| 3132 | GFP_KERNEL, node); | ||
| 3133 | if (!zone_pcp(zone, cpu)) | ||
| 3134 | goto bad; | ||
| 3135 | 3162 | ||
| 3136 | setup_pageset(zone_pcp(zone, cpu), zone_batchsize(zone)); | 3163 | for_each_possible_cpu(cpu) { |
| 3164 | struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); | ||
| 3137 | 3165 | ||
| 3138 | if (percpu_pagelist_fraction) | 3166 | setup_pageset(pcp, zone_batchsize(zone)); |
| 3139 | setup_pagelist_highmark(zone_pcp(zone, cpu), | ||
| 3140 | (zone->present_pages / percpu_pagelist_fraction)); | ||
| 3141 | } | ||
| 3142 | |||
| 3143 | return 0; | ||
| 3144 | bad: | ||
| 3145 | for_each_zone(dzone) { | ||
| 3146 | if (!populated_zone(dzone)) | ||
| 3147 | continue; | ||
| 3148 | if (dzone == zone) | ||
| 3149 | break; | ||
| 3150 | kfree(zone_pcp(dzone, cpu)); | ||
| 3151 | zone_pcp(dzone, cpu) = &boot_pageset[cpu]; | ||
| 3152 | } | ||
| 3153 | return -ENOMEM; | ||
| 3154 | } | ||
| 3155 | |||
| 3156 | static inline void free_zone_pagesets(int cpu) | ||
| 3157 | { | ||
| 3158 | struct zone *zone; | ||
| 3159 | 3167 | ||
| 3160 | for_each_zone(zone) { | 3168 | if (percpu_pagelist_fraction) |
| 3161 | struct per_cpu_pageset *pset = zone_pcp(zone, cpu); | 3169 | setup_pagelist_highmark(pcp, |
| 3162 | 3170 | (zone->present_pages / | |
| 3163 | /* Free per_cpu_pageset if it is slab allocated */ | 3171 | percpu_pagelist_fraction)); |
| 3164 | if (pset != &boot_pageset[cpu]) | 3172 | } |
| 3165 | kfree(pset); | ||
| 3166 | zone_pcp(zone, cpu) = &boot_pageset[cpu]; | ||
| 3167 | } | ||
| 3168 | } | ||
| 3169 | |||
| 3170 | static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb, | ||
| 3171 | unsigned long action, | ||
| 3172 | void *hcpu) | ||
| 3173 | { | ||
| 3174 | int cpu = (long)hcpu; | ||
| 3175 | int ret = NOTIFY_OK; | ||
| 3176 | |||
| 3177 | switch (action) { | ||
| 3178 | case CPU_UP_PREPARE: | ||
| 3179 | case CPU_UP_PREPARE_FROZEN: | ||
| 3180 | if (process_zones(cpu)) | ||
| 3181 | ret = NOTIFY_BAD; | ||
| 3182 | break; | ||
| 3183 | case CPU_UP_CANCELED: | ||
| 3184 | case CPU_UP_CANCELED_FROZEN: | ||
| 3185 | case CPU_DEAD: | ||
| 3186 | case CPU_DEAD_FROZEN: | ||
| 3187 | free_zone_pagesets(cpu); | ||
| 3188 | break; | ||
| 3189 | default: | ||
| 3190 | break; | ||
| 3191 | } | 3173 | } |
| 3192 | return ret; | ||
| 3193 | } | ||
| 3194 | |||
| 3195 | static struct notifier_block __cpuinitdata pageset_notifier = | ||
| 3196 | { &pageset_cpuup_callback, NULL, 0 }; | ||
| 3197 | |||
| 3198 | void __init setup_per_cpu_pageset(void) | ||
| 3199 | { | ||
| 3200 | int err; | ||
| 3201 | |||
| 3202 | /* Initialize per_cpu_pageset for cpu 0. | ||
| 3203 | * A cpuup callback will do this for every cpu | ||
| 3204 | * as it comes online | ||
| 3205 | */ | ||
| 3206 | err = process_zones(smp_processor_id()); | ||
| 3207 | BUG_ON(err); | ||
| 3208 | register_cpu_notifier(&pageset_notifier); | ||
| 3209 | } | 3174 | } |
| 3210 | 3175 | ||
| 3211 | #endif | ||
| 3212 | |||
| 3213 | static noinline __init_refok | 3176 | static noinline __init_refok |
| 3214 | int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) | 3177 | int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) |
| 3215 | { | 3178 | { |
| @@ -3259,11 +3222,11 @@ static int __zone_pcp_update(void *data) | |||
| 3259 | int cpu; | 3222 | int cpu; |
| 3260 | unsigned long batch = zone_batchsize(zone), flags; | 3223 | unsigned long batch = zone_batchsize(zone), flags; |
| 3261 | 3224 | ||
| 3262 | for (cpu = 0; cpu < NR_CPUS; cpu++) { | 3225 | for_each_possible_cpu(cpu) { |
| 3263 | struct per_cpu_pageset *pset; | 3226 | struct per_cpu_pageset *pset; |
| 3264 | struct per_cpu_pages *pcp; | 3227 | struct per_cpu_pages *pcp; |
| 3265 | 3228 | ||
| 3266 | pset = zone_pcp(zone, cpu); | 3229 | pset = per_cpu_ptr(zone->pageset, cpu); |
| 3267 | pcp = &pset->pcp; | 3230 | pcp = &pset->pcp; |
| 3268 | 3231 | ||
| 3269 | local_irq_save(flags); | 3232 | local_irq_save(flags); |
| @@ -3281,21 +3244,17 @@ void zone_pcp_update(struct zone *zone) | |||
| 3281 | 3244 | ||
| 3282 | static __meminit void zone_pcp_init(struct zone *zone) | 3245 | static __meminit void zone_pcp_init(struct zone *zone) |
| 3283 | { | 3246 | { |
| 3284 | int cpu; | 3247 | /* |
| 3285 | unsigned long batch = zone_batchsize(zone); | 3248 | * per cpu subsystem is not up at this point. The following code |
| 3249 | * relies on the ability of the linker to provide the | ||
| 3250 | * offset of a (static) per cpu variable into the per cpu area. | ||
| 3251 | */ | ||
| 3252 | zone->pageset = &boot_pageset; | ||
| 3286 | 3253 | ||
| 3287 | for (cpu = 0; cpu < NR_CPUS; cpu++) { | ||
| 3288 | #ifdef CONFIG_NUMA | ||
| 3289 | /* Early boot. Slab allocator not functional yet */ | ||
| 3290 | zone_pcp(zone, cpu) = &boot_pageset[cpu]; | ||
| 3291 | setup_pageset(&boot_pageset[cpu],0); | ||
| 3292 | #else | ||
| 3293 | setup_pageset(zone_pcp(zone,cpu), batch); | ||
| 3294 | #endif | ||
| 3295 | } | ||
| 3296 | if (zone->present_pages) | 3254 | if (zone->present_pages) |
| 3297 | printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n", | 3255 | printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", |
| 3298 | zone->name, zone->present_pages, batch); | 3256 | zone->name, zone->present_pages, |
| 3257 | zone_batchsize(zone)); | ||
| 3299 | } | 3258 | } |
| 3300 | 3259 | ||
| 3301 | __meminit int init_currently_empty_zone(struct zone *zone, | 3260 | __meminit int init_currently_empty_zone(struct zone *zone, |
| @@ -3434,6 +3393,61 @@ void __init free_bootmem_with_active_regions(int nid, | |||
| 3434 | } | 3393 | } |
| 3435 | } | 3394 | } |
| 3436 | 3395 | ||
| 3396 | int __init add_from_early_node_map(struct range *range, int az, | ||
| 3397 | int nr_range, int nid) | ||
| 3398 | { | ||
| 3399 | int i; | ||
| 3400 | u64 start, end; | ||
| 3401 | |||
| 3402 | /* need to go over early_node_map to find out good range for node */ | ||
| 3403 | for_each_active_range_index_in_nid(i, nid) { | ||
| 3404 | start = early_node_map[i].start_pfn; | ||
| 3405 | end = early_node_map[i].end_pfn; | ||
| 3406 | nr_range = add_range(range, az, nr_range, start, end); | ||
| 3407 | } | ||
| 3408 | return nr_range; | ||
| 3409 | } | ||
| 3410 | |||
| 3411 | #ifdef CONFIG_NO_BOOTMEM | ||
| 3412 | void * __init __alloc_memory_core_early(int nid, u64 size, u64 align, | ||
| 3413 | u64 goal, u64 limit) | ||
| 3414 | { | ||
| 3415 | int i; | ||
| 3416 | void *ptr; | ||
| 3417 | |||
| 3418 | /* need to go over early_node_map to find out good range for node */ | ||
| 3419 | for_each_active_range_index_in_nid(i, nid) { | ||
| 3420 | u64 addr; | ||
| 3421 | u64 ei_start, ei_last; | ||
| 3422 | |||
| 3423 | ei_last = early_node_map[i].end_pfn; | ||
| 3424 | ei_last <<= PAGE_SHIFT; | ||
| 3425 | ei_start = early_node_map[i].start_pfn; | ||
| 3426 | ei_start <<= PAGE_SHIFT; | ||
| 3427 | addr = find_early_area(ei_start, ei_last, | ||
| 3428 | goal, limit, size, align); | ||
| 3429 | |||
| 3430 | if (addr == -1ULL) | ||
| 3431 | continue; | ||
| 3432 | |||
| 3433 | #if 0 | ||
| 3434 | printk(KERN_DEBUG "alloc (nid=%d %llx - %llx) (%llx - %llx) %llx %llx => %llx\n", | ||
| 3435 | nid, | ||
| 3436 | ei_start, ei_last, goal, limit, size, | ||
| 3437 | align, addr); | ||
| 3438 | #endif | ||
| 3439 | |||
| 3440 | ptr = phys_to_virt(addr); | ||
| 3441 | memset(ptr, 0, size); | ||
| 3442 | reserve_early_without_check(addr, addr + size, "BOOTMEM"); | ||
| 3443 | return ptr; | ||
| 3444 | } | ||
| 3445 | |||
| 3446 | return NULL; | ||
| 3447 | } | ||
| 3448 | #endif | ||
| 3449 | |||
| 3450 | |||
| 3437 | void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data) | 3451 | void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data) |
| 3438 | { | 3452 | { |
| 3439 | int i; | 3453 | int i; |
| @@ -3998,7 +4012,7 @@ void __init add_active_range(unsigned int nid, unsigned long start_pfn, | |||
| 3998 | } | 4012 | } |
| 3999 | 4013 | ||
| 4000 | /* Merge backward if suitable */ | 4014 | /* Merge backward if suitable */ |
| 4001 | if (start_pfn < early_node_map[i].end_pfn && | 4015 | if (start_pfn < early_node_map[i].start_pfn && |
| 4002 | end_pfn >= early_node_map[i].start_pfn) { | 4016 | end_pfn >= early_node_map[i].start_pfn) { |
| 4003 | early_node_map[i].start_pfn = start_pfn; | 4017 | early_node_map[i].start_pfn = start_pfn; |
| 4004 | return; | 4018 | return; |
| @@ -4376,8 +4390,12 @@ void __init free_area_init_nodes(unsigned long *max_zone_pfn) | |||
| 4376 | for (i = 0; i < MAX_NR_ZONES; i++) { | 4390 | for (i = 0; i < MAX_NR_ZONES; i++) { |
| 4377 | if (i == ZONE_MOVABLE) | 4391 | if (i == ZONE_MOVABLE) |
| 4378 | continue; | 4392 | continue; |
| 4379 | printk(" %-8s %0#10lx -> %0#10lx\n", | 4393 | printk(" %-8s ", zone_names[i]); |
| 4380 | zone_names[i], | 4394 | if (arch_zone_lowest_possible_pfn[i] == |
| 4395 | arch_zone_highest_possible_pfn[i]) | ||
| 4396 | printk("empty\n"); | ||
| 4397 | else | ||
| 4398 | printk("%0#10lx -> %0#10lx\n", | ||
| 4381 | arch_zone_lowest_possible_pfn[i], | 4399 | arch_zone_lowest_possible_pfn[i], |
| 4382 | arch_zone_highest_possible_pfn[i]); | 4400 | arch_zone_highest_possible_pfn[i]); |
| 4383 | } | 4401 | } |
| @@ -4466,7 +4484,11 @@ void __init set_dma_reserve(unsigned long new_dma_reserve) | |||
| 4466 | } | 4484 | } |
| 4467 | 4485 | ||
| 4468 | #ifndef CONFIG_NEED_MULTIPLE_NODES | 4486 | #ifndef CONFIG_NEED_MULTIPLE_NODES |
| 4469 | struct pglist_data __refdata contig_page_data = { .bdata = &bootmem_node_data[0] }; | 4487 | struct pglist_data __refdata contig_page_data = { |
| 4488 | #ifndef CONFIG_NO_BOOTMEM | ||
| 4489 | .bdata = &bootmem_node_data[0] | ||
| 4490 | #endif | ||
| 4491 | }; | ||
| 4470 | EXPORT_SYMBOL(contig_page_data); | 4492 | EXPORT_SYMBOL(contig_page_data); |
| 4471 | #endif | 4493 | #endif |
| 4472 | 4494 | ||
| @@ -4809,10 +4831,11 @@ int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write, | |||
| 4809 | if (!write || (ret == -EINVAL)) | 4831 | if (!write || (ret == -EINVAL)) |
| 4810 | return ret; | 4832 | return ret; |
| 4811 | for_each_populated_zone(zone) { | 4833 | for_each_populated_zone(zone) { |
| 4812 | for_each_online_cpu(cpu) { | 4834 | for_each_possible_cpu(cpu) { |
| 4813 | unsigned long high; | 4835 | unsigned long high; |
| 4814 | high = zone->present_pages / percpu_pagelist_fraction; | 4836 | high = zone->present_pages / percpu_pagelist_fraction; |
| 4815 | setup_pagelist_highmark(zone_pcp(zone, cpu), high); | 4837 | setup_pagelist_highmark( |
| 4838 | per_cpu_ptr(zone->pageset, cpu), high); | ||
| 4816 | } | 4839 | } |
| 4817 | } | 4840 | } |
| 4818 | return 0; | 4841 | return 0; |
| @@ -5158,3 +5181,80 @@ bool is_free_buddy_page(struct page *page) | |||
| 5158 | return order < MAX_ORDER; | 5181 | return order < MAX_ORDER; |
| 5159 | } | 5182 | } |
| 5160 | #endif | 5183 | #endif |
| 5184 | |||
| 5185 | static struct trace_print_flags pageflag_names[] = { | ||
| 5186 | {1UL << PG_locked, "locked" }, | ||
| 5187 | {1UL << PG_error, "error" }, | ||
| 5188 | {1UL << PG_referenced, "referenced" }, | ||
| 5189 | {1UL << PG_uptodate, "uptodate" }, | ||
| 5190 | {1UL << PG_dirty, "dirty" }, | ||
| 5191 | {1UL << PG_lru, "lru" }, | ||
| 5192 | {1UL << PG_active, "active" }, | ||
| 5193 | {1UL << PG_slab, "slab" }, | ||
| 5194 | {1UL << PG_owner_priv_1, "owner_priv_1" }, | ||
| 5195 | {1UL << PG_arch_1, "arch_1" }, | ||
| 5196 | {1UL << PG_reserved, "reserved" }, | ||
| 5197 | {1UL << PG_private, "private" }, | ||
| 5198 | {1UL << PG_private_2, "private_2" }, | ||
| 5199 | {1UL << PG_writeback, "writeback" }, | ||
| 5200 | #ifdef CONFIG_PAGEFLAGS_EXTENDED | ||
| 5201 | {1UL << PG_head, "head" }, | ||
| 5202 | {1UL << PG_tail, "tail" }, | ||
| 5203 | #else | ||
| 5204 | {1UL << PG_compound, "compound" }, | ||
| 5205 | #endif | ||
| 5206 | {1UL << PG_swapcache, "swapcache" }, | ||
| 5207 | {1UL << PG_mappedtodisk, "mappedtodisk" }, | ||
| 5208 | {1UL << PG_reclaim, "reclaim" }, | ||
| 5209 | {1UL << PG_buddy, "buddy" }, | ||
| 5210 | {1UL << PG_swapbacked, "swapbacked" }, | ||
| 5211 | {1UL << PG_unevictable, "unevictable" }, | ||
| 5212 | #ifdef CONFIG_MMU | ||
| 5213 | {1UL << PG_mlocked, "mlocked" }, | ||
| 5214 | #endif | ||
| 5215 | #ifdef CONFIG_ARCH_USES_PG_UNCACHED | ||
| 5216 | {1UL << PG_uncached, "uncached" }, | ||
| 5217 | #endif | ||
| 5218 | #ifdef CONFIG_MEMORY_FAILURE | ||
| 5219 | {1UL << PG_hwpoison, "hwpoison" }, | ||
| 5220 | #endif | ||
| 5221 | {-1UL, NULL }, | ||
| 5222 | }; | ||
| 5223 | |||
| 5224 | static void dump_page_flags(unsigned long flags) | ||
| 5225 | { | ||
| 5226 | const char *delim = ""; | ||
| 5227 | unsigned long mask; | ||
| 5228 | int i; | ||
| 5229 | |||
| 5230 | printk(KERN_ALERT "page flags: %#lx(", flags); | ||
| 5231 | |||
| 5232 | /* remove zone id */ | ||
| 5233 | flags &= (1UL << NR_PAGEFLAGS) - 1; | ||
| 5234 | |||
| 5235 | for (i = 0; pageflag_names[i].name && flags; i++) { | ||
| 5236 | |||
| 5237 | mask = pageflag_names[i].mask; | ||
| 5238 | if ((flags & mask) != mask) | ||
| 5239 | continue; | ||
| 5240 | |||
| 5241 | flags &= ~mask; | ||
| 5242 | printk("%s%s", delim, pageflag_names[i].name); | ||
| 5243 | delim = "|"; | ||
| 5244 | } | ||
| 5245 | |||
| 5246 | /* check for left over flags */ | ||
| 5247 | if (flags) | ||
| 5248 | printk("%s%#lx", delim, flags); | ||
| 5249 | |||
| 5250 | printk(")\n"); | ||
| 5251 | } | ||
| 5252 | |||
| 5253 | void dump_page(struct page *page) | ||
| 5254 | { | ||
| 5255 | printk(KERN_ALERT | ||
| 5256 | "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n", | ||
| 5257 | page, page_count(page), page_mapcount(page), | ||
| 5258 | page->mapping, page->index); | ||
| 5259 | dump_page_flags(page->flags); | ||
| 5260 | } | ||
