aboutsummaryrefslogtreecommitdiffstats
path: root/mm/page_alloc.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/page_alloc.c')
-rw-r--r--mm/page_alloc.c410
1 files changed, 255 insertions, 155 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 4e9f5cc5fb59..d03c946d5566 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -50,6 +50,7 @@
50#include <linux/kmemleak.h> 50#include <linux/kmemleak.h>
51#include <linux/memory.h> 51#include <linux/memory.h>
52#include <trace/events/kmem.h> 52#include <trace/events/kmem.h>
53#include <linux/ftrace_event.h>
53 54
54#include <asm/tlbflush.h> 55#include <asm/tlbflush.h>
55#include <asm/div64.h> 56#include <asm/div64.h>
@@ -76,6 +77,31 @@ unsigned long totalreserve_pages __read_mostly;
76int percpu_pagelist_fraction; 77int percpu_pagelist_fraction;
77gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; 78gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
78 79
80#ifdef CONFIG_PM_SLEEP
81/*
82 * The following functions are used by the suspend/hibernate code to temporarily
83 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
84 * while devices are suspended. To avoid races with the suspend/hibernate code,
85 * they should always be called with pm_mutex held (gfp_allowed_mask also should
86 * only be modified with pm_mutex held, unless the suspend/hibernate code is
87 * guaranteed not to run in parallel with that modification).
88 */
89void set_gfp_allowed_mask(gfp_t mask)
90{
91 WARN_ON(!mutex_is_locked(&pm_mutex));
92 gfp_allowed_mask = mask;
93}
94
95gfp_t clear_gfp_allowed_mask(gfp_t mask)
96{
97 gfp_t ret = gfp_allowed_mask;
98
99 WARN_ON(!mutex_is_locked(&pm_mutex));
100 gfp_allowed_mask &= ~mask;
101 return ret;
102}
103#endif /* CONFIG_PM_SLEEP */
104
79#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE 105#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
80int pageblock_order __read_mostly; 106int pageblock_order __read_mostly;
81#endif 107#endif
@@ -263,10 +289,7 @@ static void bad_page(struct page *page)
263 289
264 printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n", 290 printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n",
265 current->comm, page_to_pfn(page)); 291 current->comm, page_to_pfn(page));
266 printk(KERN_ALERT 292 dump_page(page);
267 "page:%p flags:%p count:%d mapcount:%d mapping:%p index:%lx\n",
268 page, (void *)page->flags, page_count(page),
269 page_mapcount(page), page->mapping, page->index);
270 293
271 dump_stack(); 294 dump_stack();
272out: 295out:
@@ -530,7 +553,7 @@ static void free_pcppages_bulk(struct zone *zone, int count,
530 int batch_free = 0; 553 int batch_free = 0;
531 554
532 spin_lock(&zone->lock); 555 spin_lock(&zone->lock);
533 zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE); 556 zone->all_unreclaimable = 0;
534 zone->pages_scanned = 0; 557 zone->pages_scanned = 0;
535 558
536 __mod_zone_page_state(zone, NR_FREE_PAGES, count); 559 __mod_zone_page_state(zone, NR_FREE_PAGES, count);
@@ -556,8 +579,9 @@ static void free_pcppages_bulk(struct zone *zone, int count,
556 page = list_entry(list->prev, struct page, lru); 579 page = list_entry(list->prev, struct page, lru);
557 /* must delete as __free_one_page list manipulates */ 580 /* must delete as __free_one_page list manipulates */
558 list_del(&page->lru); 581 list_del(&page->lru);
559 __free_one_page(page, zone, 0, migratetype); 582 /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
560 trace_mm_page_pcpu_drain(page, 0, migratetype); 583 __free_one_page(page, zone, 0, page_private(page));
584 trace_mm_page_pcpu_drain(page, 0, page_private(page));
561 } while (--count && --batch_free && !list_empty(list)); 585 } while (--count && --batch_free && !list_empty(list));
562 } 586 }
563 spin_unlock(&zone->lock); 587 spin_unlock(&zone->lock);
@@ -567,7 +591,7 @@ static void free_one_page(struct zone *zone, struct page *page, int order,
567 int migratetype) 591 int migratetype)
568{ 592{
569 spin_lock(&zone->lock); 593 spin_lock(&zone->lock);
570 zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE); 594 zone->all_unreclaimable = 0;
571 zone->pages_scanned = 0; 595 zone->pages_scanned = 0;
572 596
573 __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order); 597 __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order);
@@ -582,6 +606,7 @@ static void __free_pages_ok(struct page *page, unsigned int order)
582 int bad = 0; 606 int bad = 0;
583 int wasMlocked = __TestClearPageMlocked(page); 607 int wasMlocked = __TestClearPageMlocked(page);
584 608
609 trace_mm_page_free_direct(page, order);
585 kmemcheck_free_shadow(page, order); 610 kmemcheck_free_shadow(page, order);
586 611
587 for (i = 0 ; i < (1 << order) ; ++i) 612 for (i = 0 ; i < (1 << order) ; ++i)
@@ -1008,10 +1033,10 @@ static void drain_pages(unsigned int cpu)
1008 struct per_cpu_pageset *pset; 1033 struct per_cpu_pageset *pset;
1009 struct per_cpu_pages *pcp; 1034 struct per_cpu_pages *pcp;
1010 1035
1011 pset = zone_pcp(zone, cpu); 1036 local_irq_save(flags);
1037 pset = per_cpu_ptr(zone->pageset, cpu);
1012 1038
1013 pcp = &pset->pcp; 1039 pcp = &pset->pcp;
1014 local_irq_save(flags);
1015 free_pcppages_bulk(zone, pcp->count, pcp); 1040 free_pcppages_bulk(zone, pcp->count, pcp);
1016 pcp->count = 0; 1041 pcp->count = 0;
1017 local_irq_restore(flags); 1042 local_irq_restore(flags);
@@ -1072,8 +1097,9 @@ void mark_free_pages(struct zone *zone)
1072 1097
1073/* 1098/*
1074 * Free a 0-order page 1099 * Free a 0-order page
1100 * cold == 1 ? free a cold page : free a hot page
1075 */ 1101 */
1076static void free_hot_cold_page(struct page *page, int cold) 1102void free_hot_cold_page(struct page *page, int cold)
1077{ 1103{
1078 struct zone *zone = page_zone(page); 1104 struct zone *zone = page_zone(page);
1079 struct per_cpu_pages *pcp; 1105 struct per_cpu_pages *pcp;
@@ -1081,6 +1107,7 @@ static void free_hot_cold_page(struct page *page, int cold)
1081 int migratetype; 1107 int migratetype;
1082 int wasMlocked = __TestClearPageMlocked(page); 1108 int wasMlocked = __TestClearPageMlocked(page);
1083 1109
1110 trace_mm_page_free_direct(page, 0);
1084 kmemcheck_free_shadow(page, 0); 1111 kmemcheck_free_shadow(page, 0);
1085 1112
1086 if (PageAnon(page)) 1113 if (PageAnon(page))
@@ -1095,7 +1122,6 @@ static void free_hot_cold_page(struct page *page, int cold)
1095 arch_free_page(page, 0); 1122 arch_free_page(page, 0);
1096 kernel_map_pages(page, 1, 0); 1123 kernel_map_pages(page, 1, 0);
1097 1124
1098 pcp = &zone_pcp(zone, get_cpu())->pcp;
1099 migratetype = get_pageblock_migratetype(page); 1125 migratetype = get_pageblock_migratetype(page);
1100 set_page_private(page, migratetype); 1126 set_page_private(page, migratetype);
1101 local_irq_save(flags); 1127 local_irq_save(flags);
@@ -1118,6 +1144,7 @@ static void free_hot_cold_page(struct page *page, int cold)
1118 migratetype = MIGRATE_MOVABLE; 1144 migratetype = MIGRATE_MOVABLE;
1119 } 1145 }
1120 1146
1147 pcp = &this_cpu_ptr(zone->pageset)->pcp;
1121 if (cold) 1148 if (cold)
1122 list_add_tail(&page->lru, &pcp->lists[migratetype]); 1149 list_add_tail(&page->lru, &pcp->lists[migratetype]);
1123 else 1150 else
@@ -1130,15 +1157,8 @@ static void free_hot_cold_page(struct page *page, int cold)
1130 1157
1131out: 1158out:
1132 local_irq_restore(flags); 1159 local_irq_restore(flags);
1133 put_cpu();
1134} 1160}
1135 1161
1136void free_hot_page(struct page *page)
1137{
1138 trace_mm_page_free_direct(page, 0);
1139 free_hot_cold_page(page, 0);
1140}
1141
1142/* 1162/*
1143 * split_page takes a non-compound higher-order page, and splits it into 1163 * split_page takes a non-compound higher-order page, and splits it into
1144 * n (1<<order) sub-pages: page[0..n] 1164 * n (1<<order) sub-pages: page[0..n]
@@ -1180,17 +1200,15 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
1180 unsigned long flags; 1200 unsigned long flags;
1181 struct page *page; 1201 struct page *page;
1182 int cold = !!(gfp_flags & __GFP_COLD); 1202 int cold = !!(gfp_flags & __GFP_COLD);
1183 int cpu;
1184 1203
1185again: 1204again:
1186 cpu = get_cpu();
1187 if (likely(order == 0)) { 1205 if (likely(order == 0)) {
1188 struct per_cpu_pages *pcp; 1206 struct per_cpu_pages *pcp;
1189 struct list_head *list; 1207 struct list_head *list;
1190 1208
1191 pcp = &zone_pcp(zone, cpu)->pcp;
1192 list = &pcp->lists[migratetype];
1193 local_irq_save(flags); 1209 local_irq_save(flags);
1210 pcp = &this_cpu_ptr(zone->pageset)->pcp;
1211 list = &pcp->lists[migratetype];
1194 if (list_empty(list)) { 1212 if (list_empty(list)) {
1195 pcp->count += rmqueue_bulk(zone, 0, 1213 pcp->count += rmqueue_bulk(zone, 0,
1196 pcp->batch, list, 1214 pcp->batch, list,
@@ -1222,16 +1240,15 @@ again:
1222 } 1240 }
1223 spin_lock_irqsave(&zone->lock, flags); 1241 spin_lock_irqsave(&zone->lock, flags);
1224 page = __rmqueue(zone, order, migratetype); 1242 page = __rmqueue(zone, order, migratetype);
1225 __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
1226 spin_unlock(&zone->lock); 1243 spin_unlock(&zone->lock);
1227 if (!page) 1244 if (!page)
1228 goto failed; 1245 goto failed;
1246 __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order));
1229 } 1247 }
1230 1248
1231 __count_zone_vm_events(PGALLOC, zone, 1 << order); 1249 __count_zone_vm_events(PGALLOC, zone, 1 << order);
1232 zone_statistics(preferred_zone, zone); 1250 zone_statistics(preferred_zone, zone);
1233 local_irq_restore(flags); 1251 local_irq_restore(flags);
1234 put_cpu();
1235 1252
1236 VM_BUG_ON(bad_range(zone, page)); 1253 VM_BUG_ON(bad_range(zone, page));
1237 if (prep_new_page(page, order, gfp_flags)) 1254 if (prep_new_page(page, order, gfp_flags))
@@ -1240,7 +1257,6 @@ again:
1240 1257
1241failed: 1258failed:
1242 local_irq_restore(flags); 1259 local_irq_restore(flags);
1243 put_cpu();
1244 return NULL; 1260 return NULL;
1245} 1261}
1246 1262
@@ -2012,9 +2028,8 @@ void __pagevec_free(struct pagevec *pvec)
2012void __free_pages(struct page *page, unsigned int order) 2028void __free_pages(struct page *page, unsigned int order)
2013{ 2029{
2014 if (put_page_testzero(page)) { 2030 if (put_page_testzero(page)) {
2015 trace_mm_page_free_direct(page, order);
2016 if (order == 0) 2031 if (order == 0)
2017 free_hot_page(page); 2032 free_hot_cold_page(page, 0);
2018 else 2033 else
2019 __free_pages_ok(page, order); 2034 __free_pages_ok(page, order);
2020 } 2035 }
@@ -2179,7 +2194,7 @@ void show_free_areas(void)
2179 for_each_online_cpu(cpu) { 2194 for_each_online_cpu(cpu) {
2180 struct per_cpu_pageset *pageset; 2195 struct per_cpu_pageset *pageset;
2181 2196
2182 pageset = zone_pcp(zone, cpu); 2197 pageset = per_cpu_ptr(zone->pageset, cpu);
2183 2198
2184 printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n", 2199 printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
2185 cpu, pageset->pcp.high, 2200 cpu, pageset->pcp.high,
@@ -2270,7 +2285,7 @@ void show_free_areas(void)
2270 K(zone_page_state(zone, NR_BOUNCE)), 2285 K(zone_page_state(zone, NR_BOUNCE)),
2271 K(zone_page_state(zone, NR_WRITEBACK_TEMP)), 2286 K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
2272 zone->pages_scanned, 2287 zone->pages_scanned,
2273 (zone_is_all_unreclaimable(zone) ? "yes" : "no") 2288 (zone->all_unreclaimable ? "yes" : "no")
2274 ); 2289 );
2275 printk("lowmem_reserve[]:"); 2290 printk("lowmem_reserve[]:");
2276 for (i = 0; i < MAX_NR_ZONES; i++) 2291 for (i = 0; i < MAX_NR_ZONES; i++)
@@ -2744,10 +2759,29 @@ static void build_zonelist_cache(pg_data_t *pgdat)
2744 2759
2745#endif /* CONFIG_NUMA */ 2760#endif /* CONFIG_NUMA */
2746 2761
2762/*
2763 * Boot pageset table. One per cpu which is going to be used for all
2764 * zones and all nodes. The parameters will be set in such a way
2765 * that an item put on a list will immediately be handed over to
2766 * the buddy list. This is safe since pageset manipulation is done
2767 * with interrupts disabled.
2768 *
2769 * The boot_pagesets must be kept even after bootup is complete for
2770 * unused processors and/or zones. They do play a role for bootstrapping
2771 * hotplugged processors.
2772 *
2773 * zoneinfo_show() and maybe other functions do
2774 * not check if the processor is online before following the pageset pointer.
2775 * Other parts of the kernel may not check if the zone is available.
2776 */
2777static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
2778static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
2779
2747/* return values int ....just for stop_machine() */ 2780/* return values int ....just for stop_machine() */
2748static int __build_all_zonelists(void *dummy) 2781static int __build_all_zonelists(void *dummy)
2749{ 2782{
2750 int nid; 2783 int nid;
2784 int cpu;
2751 2785
2752#ifdef CONFIG_NUMA 2786#ifdef CONFIG_NUMA
2753 memset(node_load, 0, sizeof(node_load)); 2787 memset(node_load, 0, sizeof(node_load));
@@ -2758,6 +2792,23 @@ static int __build_all_zonelists(void *dummy)
2758 build_zonelists(pgdat); 2792 build_zonelists(pgdat);
2759 build_zonelist_cache(pgdat); 2793 build_zonelist_cache(pgdat);
2760 } 2794 }
2795
2796 /*
2797 * Initialize the boot_pagesets that are going to be used
2798 * for bootstrapping processors. The real pagesets for
2799 * each zone will be allocated later when the per cpu
2800 * allocator is available.
2801 *
2802 * boot_pagesets are used also for bootstrapping offline
2803 * cpus if the system is already booted because the pagesets
2804 * are needed to initialize allocators on a specific cpu too.
2805 * F.e. the percpu allocator needs the page allocator which
2806 * needs the percpu allocator in order to allocate its pagesets
2807 * (a chicken-egg dilemma).
2808 */
2809 for_each_possible_cpu(cpu)
2810 setup_pageset(&per_cpu(boot_pageset, cpu), 0);
2811
2761 return 0; 2812 return 0;
2762} 2813}
2763 2814
@@ -3095,121 +3146,33 @@ static void setup_pagelist_highmark(struct per_cpu_pageset *p,
3095 pcp->batch = PAGE_SHIFT * 8; 3146 pcp->batch = PAGE_SHIFT * 8;
3096} 3147}
3097 3148
3098
3099#ifdef CONFIG_NUMA
3100/* 3149/*
3101 * Boot pageset table. One per cpu which is going to be used for all 3150 * Allocate per cpu pagesets and initialize them.
3102 * zones and all nodes. The parameters will be set in such a way 3151 * Before this call only boot pagesets were available.
3103 * that an item put on a list will immediately be handed over to 3152 * Boot pagesets will no longer be used by this processorr
3104 * the buddy list. This is safe since pageset manipulation is done 3153 * after setup_per_cpu_pageset().
3105 * with interrupts disabled.
3106 *
3107 * Some NUMA counter updates may also be caught by the boot pagesets.
3108 *
3109 * The boot_pagesets must be kept even after bootup is complete for
3110 * unused processors and/or zones. They do play a role for bootstrapping
3111 * hotplugged processors.
3112 *
3113 * zoneinfo_show() and maybe other functions do
3114 * not check if the processor is online before following the pageset pointer.
3115 * Other parts of the kernel may not check if the zone is available.
3116 */
3117static struct per_cpu_pageset boot_pageset[NR_CPUS];
3118
3119/*
3120 * Dynamically allocate memory for the
3121 * per cpu pageset array in struct zone.
3122 */ 3154 */
3123static int __cpuinit process_zones(int cpu) 3155void __init setup_per_cpu_pageset(void)
3124{ 3156{
3125 struct zone *zone, *dzone; 3157 struct zone *zone;
3126 int node = cpu_to_node(cpu); 3158 int cpu;
3127
3128 node_set_state(node, N_CPU); /* this node has a cpu */
3129 3159
3130 for_each_populated_zone(zone) { 3160 for_each_populated_zone(zone) {
3131 zone_pcp(zone, cpu) = kmalloc_node(sizeof(struct per_cpu_pageset), 3161 zone->pageset = alloc_percpu(struct per_cpu_pageset);
3132 GFP_KERNEL, node);
3133 if (!zone_pcp(zone, cpu))
3134 goto bad;
3135 3162
3136 setup_pageset(zone_pcp(zone, cpu), zone_batchsize(zone)); 3163 for_each_possible_cpu(cpu) {
3164 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
3137 3165
3138 if (percpu_pagelist_fraction) 3166 setup_pageset(pcp, zone_batchsize(zone));
3139 setup_pagelist_highmark(zone_pcp(zone, cpu),
3140 (zone->present_pages / percpu_pagelist_fraction));
3141 }
3142
3143 return 0;
3144bad:
3145 for_each_zone(dzone) {
3146 if (!populated_zone(dzone))
3147 continue;
3148 if (dzone == zone)
3149 break;
3150 kfree(zone_pcp(dzone, cpu));
3151 zone_pcp(dzone, cpu) = &boot_pageset[cpu];
3152 }
3153 return -ENOMEM;
3154}
3155
3156static inline void free_zone_pagesets(int cpu)
3157{
3158 struct zone *zone;
3159 3167
3160 for_each_zone(zone) { 3168 if (percpu_pagelist_fraction)
3161 struct per_cpu_pageset *pset = zone_pcp(zone, cpu); 3169 setup_pagelist_highmark(pcp,
3162 3170 (zone->present_pages /
3163 /* Free per_cpu_pageset if it is slab allocated */ 3171 percpu_pagelist_fraction));
3164 if (pset != &boot_pageset[cpu]) 3172 }
3165 kfree(pset);
3166 zone_pcp(zone, cpu) = &boot_pageset[cpu];
3167 }
3168}
3169
3170static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb,
3171 unsigned long action,
3172 void *hcpu)
3173{
3174 int cpu = (long)hcpu;
3175 int ret = NOTIFY_OK;
3176
3177 switch (action) {
3178 case CPU_UP_PREPARE:
3179 case CPU_UP_PREPARE_FROZEN:
3180 if (process_zones(cpu))
3181 ret = NOTIFY_BAD;
3182 break;
3183 case CPU_UP_CANCELED:
3184 case CPU_UP_CANCELED_FROZEN:
3185 case CPU_DEAD:
3186 case CPU_DEAD_FROZEN:
3187 free_zone_pagesets(cpu);
3188 break;
3189 default:
3190 break;
3191 } 3173 }
3192 return ret;
3193}
3194
3195static struct notifier_block __cpuinitdata pageset_notifier =
3196 { &pageset_cpuup_callback, NULL, 0 };
3197
3198void __init setup_per_cpu_pageset(void)
3199{
3200 int err;
3201
3202 /* Initialize per_cpu_pageset for cpu 0.
3203 * A cpuup callback will do this for every cpu
3204 * as it comes online
3205 */
3206 err = process_zones(smp_processor_id());
3207 BUG_ON(err);
3208 register_cpu_notifier(&pageset_notifier);
3209} 3174}
3210 3175
3211#endif
3212
3213static noinline __init_refok 3176static noinline __init_refok
3214int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) 3177int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
3215{ 3178{
@@ -3259,11 +3222,11 @@ static int __zone_pcp_update(void *data)
3259 int cpu; 3222 int cpu;
3260 unsigned long batch = zone_batchsize(zone), flags; 3223 unsigned long batch = zone_batchsize(zone), flags;
3261 3224
3262 for (cpu = 0; cpu < NR_CPUS; cpu++) { 3225 for_each_possible_cpu(cpu) {
3263 struct per_cpu_pageset *pset; 3226 struct per_cpu_pageset *pset;
3264 struct per_cpu_pages *pcp; 3227 struct per_cpu_pages *pcp;
3265 3228
3266 pset = zone_pcp(zone, cpu); 3229 pset = per_cpu_ptr(zone->pageset, cpu);
3267 pcp = &pset->pcp; 3230 pcp = &pset->pcp;
3268 3231
3269 local_irq_save(flags); 3232 local_irq_save(flags);
@@ -3281,21 +3244,17 @@ void zone_pcp_update(struct zone *zone)
3281 3244
3282static __meminit void zone_pcp_init(struct zone *zone) 3245static __meminit void zone_pcp_init(struct zone *zone)
3283{ 3246{
3284 int cpu; 3247 /*
3285 unsigned long batch = zone_batchsize(zone); 3248 * per cpu subsystem is not up at this point. The following code
3249 * relies on the ability of the linker to provide the
3250 * offset of a (static) per cpu variable into the per cpu area.
3251 */
3252 zone->pageset = &boot_pageset;
3286 3253
3287 for (cpu = 0; cpu < NR_CPUS; cpu++) {
3288#ifdef CONFIG_NUMA
3289 /* Early boot. Slab allocator not functional yet */
3290 zone_pcp(zone, cpu) = &boot_pageset[cpu];
3291 setup_pageset(&boot_pageset[cpu],0);
3292#else
3293 setup_pageset(zone_pcp(zone,cpu), batch);
3294#endif
3295 }
3296 if (zone->present_pages) 3254 if (zone->present_pages)
3297 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n", 3255 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
3298 zone->name, zone->present_pages, batch); 3256 zone->name, zone->present_pages,
3257 zone_batchsize(zone));
3299} 3258}
3300 3259
3301__meminit int init_currently_empty_zone(struct zone *zone, 3260__meminit int init_currently_empty_zone(struct zone *zone,
@@ -3434,6 +3393,61 @@ void __init free_bootmem_with_active_regions(int nid,
3434 } 3393 }
3435} 3394}
3436 3395
3396int __init add_from_early_node_map(struct range *range, int az,
3397 int nr_range, int nid)
3398{
3399 int i;
3400 u64 start, end;
3401
3402 /* need to go over early_node_map to find out good range for node */
3403 for_each_active_range_index_in_nid(i, nid) {
3404 start = early_node_map[i].start_pfn;
3405 end = early_node_map[i].end_pfn;
3406 nr_range = add_range(range, az, nr_range, start, end);
3407 }
3408 return nr_range;
3409}
3410
3411#ifdef CONFIG_NO_BOOTMEM
3412void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
3413 u64 goal, u64 limit)
3414{
3415 int i;
3416 void *ptr;
3417
3418 /* need to go over early_node_map to find out good range for node */
3419 for_each_active_range_index_in_nid(i, nid) {
3420 u64 addr;
3421 u64 ei_start, ei_last;
3422
3423 ei_last = early_node_map[i].end_pfn;
3424 ei_last <<= PAGE_SHIFT;
3425 ei_start = early_node_map[i].start_pfn;
3426 ei_start <<= PAGE_SHIFT;
3427 addr = find_early_area(ei_start, ei_last,
3428 goal, limit, size, align);
3429
3430 if (addr == -1ULL)
3431 continue;
3432
3433#if 0
3434 printk(KERN_DEBUG "alloc (nid=%d %llx - %llx) (%llx - %llx) %llx %llx => %llx\n",
3435 nid,
3436 ei_start, ei_last, goal, limit, size,
3437 align, addr);
3438#endif
3439
3440 ptr = phys_to_virt(addr);
3441 memset(ptr, 0, size);
3442 reserve_early_without_check(addr, addr + size, "BOOTMEM");
3443 return ptr;
3444 }
3445
3446 return NULL;
3447}
3448#endif
3449
3450
3437void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data) 3451void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data)
3438{ 3452{
3439 int i; 3453 int i;
@@ -3998,7 +4012,7 @@ void __init add_active_range(unsigned int nid, unsigned long start_pfn,
3998 } 4012 }
3999 4013
4000 /* Merge backward if suitable */ 4014 /* Merge backward if suitable */
4001 if (start_pfn < early_node_map[i].end_pfn && 4015 if (start_pfn < early_node_map[i].start_pfn &&
4002 end_pfn >= early_node_map[i].start_pfn) { 4016 end_pfn >= early_node_map[i].start_pfn) {
4003 early_node_map[i].start_pfn = start_pfn; 4017 early_node_map[i].start_pfn = start_pfn;
4004 return; 4018 return;
@@ -4376,8 +4390,12 @@ void __init free_area_init_nodes(unsigned long *max_zone_pfn)
4376 for (i = 0; i < MAX_NR_ZONES; i++) { 4390 for (i = 0; i < MAX_NR_ZONES; i++) {
4377 if (i == ZONE_MOVABLE) 4391 if (i == ZONE_MOVABLE)
4378 continue; 4392 continue;
4379 printk(" %-8s %0#10lx -> %0#10lx\n", 4393 printk(" %-8s ", zone_names[i]);
4380 zone_names[i], 4394 if (arch_zone_lowest_possible_pfn[i] ==
4395 arch_zone_highest_possible_pfn[i])
4396 printk("empty\n");
4397 else
4398 printk("%0#10lx -> %0#10lx\n",
4381 arch_zone_lowest_possible_pfn[i], 4399 arch_zone_lowest_possible_pfn[i],
4382 arch_zone_highest_possible_pfn[i]); 4400 arch_zone_highest_possible_pfn[i]);
4383 } 4401 }
@@ -4466,7 +4484,11 @@ void __init set_dma_reserve(unsigned long new_dma_reserve)
4466} 4484}
4467 4485
4468#ifndef CONFIG_NEED_MULTIPLE_NODES 4486#ifndef CONFIG_NEED_MULTIPLE_NODES
4469struct pglist_data __refdata contig_page_data = { .bdata = &bootmem_node_data[0] }; 4487struct pglist_data __refdata contig_page_data = {
4488#ifndef CONFIG_NO_BOOTMEM
4489 .bdata = &bootmem_node_data[0]
4490#endif
4491 };
4470EXPORT_SYMBOL(contig_page_data); 4492EXPORT_SYMBOL(contig_page_data);
4471#endif 4493#endif
4472 4494
@@ -4809,10 +4831,11 @@ int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
4809 if (!write || (ret == -EINVAL)) 4831 if (!write || (ret == -EINVAL))
4810 return ret; 4832 return ret;
4811 for_each_populated_zone(zone) { 4833 for_each_populated_zone(zone) {
4812 for_each_online_cpu(cpu) { 4834 for_each_possible_cpu(cpu) {
4813 unsigned long high; 4835 unsigned long high;
4814 high = zone->present_pages / percpu_pagelist_fraction; 4836 high = zone->present_pages / percpu_pagelist_fraction;
4815 setup_pagelist_highmark(zone_pcp(zone, cpu), high); 4837 setup_pagelist_highmark(
4838 per_cpu_ptr(zone->pageset, cpu), high);
4816 } 4839 }
4817 } 4840 }
4818 return 0; 4841 return 0;
@@ -5158,3 +5181,80 @@ bool is_free_buddy_page(struct page *page)
5158 return order < MAX_ORDER; 5181 return order < MAX_ORDER;
5159} 5182}
5160#endif 5183#endif
5184
5185static struct trace_print_flags pageflag_names[] = {
5186 {1UL << PG_locked, "locked" },
5187 {1UL << PG_error, "error" },
5188 {1UL << PG_referenced, "referenced" },
5189 {1UL << PG_uptodate, "uptodate" },
5190 {1UL << PG_dirty, "dirty" },
5191 {1UL << PG_lru, "lru" },
5192 {1UL << PG_active, "active" },
5193 {1UL << PG_slab, "slab" },
5194 {1UL << PG_owner_priv_1, "owner_priv_1" },
5195 {1UL << PG_arch_1, "arch_1" },
5196 {1UL << PG_reserved, "reserved" },
5197 {1UL << PG_private, "private" },
5198 {1UL << PG_private_2, "private_2" },
5199 {1UL << PG_writeback, "writeback" },
5200#ifdef CONFIG_PAGEFLAGS_EXTENDED
5201 {1UL << PG_head, "head" },
5202 {1UL << PG_tail, "tail" },
5203#else
5204 {1UL << PG_compound, "compound" },
5205#endif
5206 {1UL << PG_swapcache, "swapcache" },
5207 {1UL << PG_mappedtodisk, "mappedtodisk" },
5208 {1UL << PG_reclaim, "reclaim" },
5209 {1UL << PG_buddy, "buddy" },
5210 {1UL << PG_swapbacked, "swapbacked" },
5211 {1UL << PG_unevictable, "unevictable" },
5212#ifdef CONFIG_MMU
5213 {1UL << PG_mlocked, "mlocked" },
5214#endif
5215#ifdef CONFIG_ARCH_USES_PG_UNCACHED
5216 {1UL << PG_uncached, "uncached" },
5217#endif
5218#ifdef CONFIG_MEMORY_FAILURE
5219 {1UL << PG_hwpoison, "hwpoison" },
5220#endif
5221 {-1UL, NULL },
5222};
5223
5224static void dump_page_flags(unsigned long flags)
5225{
5226 const char *delim = "";
5227 unsigned long mask;
5228 int i;
5229
5230 printk(KERN_ALERT "page flags: %#lx(", flags);
5231
5232 /* remove zone id */
5233 flags &= (1UL << NR_PAGEFLAGS) - 1;
5234
5235 for (i = 0; pageflag_names[i].name && flags; i++) {
5236
5237 mask = pageflag_names[i].mask;
5238 if ((flags & mask) != mask)
5239 continue;
5240
5241 flags &= ~mask;
5242 printk("%s%s", delim, pageflag_names[i].name);
5243 delim = "|";
5244 }
5245
5246 /* check for left over flags */
5247 if (flags)
5248 printk("%s%#lx", delim, flags);
5249
5250 printk(")\n");
5251}
5252
5253void dump_page(struct page *page)
5254{
5255 printk(KERN_ALERT
5256 "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n",
5257 page, page_count(page), page_mapcount(page),
5258 page->mapping, page->index);
5259 dump_page_flags(page->flags);
5260}