diff options
Diffstat (limited to 'mm/page_alloc.c')
-rw-r--r-- | mm/page_alloc.c | 410 |
1 files changed, 255 insertions, 155 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 4e9f5cc5fb59..d03c946d5566 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c | |||
@@ -50,6 +50,7 @@ | |||
50 | #include <linux/kmemleak.h> | 50 | #include <linux/kmemleak.h> |
51 | #include <linux/memory.h> | 51 | #include <linux/memory.h> |
52 | #include <trace/events/kmem.h> | 52 | #include <trace/events/kmem.h> |
53 | #include <linux/ftrace_event.h> | ||
53 | 54 | ||
54 | #include <asm/tlbflush.h> | 55 | #include <asm/tlbflush.h> |
55 | #include <asm/div64.h> | 56 | #include <asm/div64.h> |
@@ -76,6 +77,31 @@ unsigned long totalreserve_pages __read_mostly; | |||
76 | int percpu_pagelist_fraction; | 77 | int percpu_pagelist_fraction; |
77 | gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; | 78 | gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; |
78 | 79 | ||
80 | #ifdef CONFIG_PM_SLEEP | ||
81 | /* | ||
82 | * The following functions are used by the suspend/hibernate code to temporarily | ||
83 | * change gfp_allowed_mask in order to avoid using I/O during memory allocations | ||
84 | * while devices are suspended. To avoid races with the suspend/hibernate code, | ||
85 | * they should always be called with pm_mutex held (gfp_allowed_mask also should | ||
86 | * only be modified with pm_mutex held, unless the suspend/hibernate code is | ||
87 | * guaranteed not to run in parallel with that modification). | ||
88 | */ | ||
89 | void set_gfp_allowed_mask(gfp_t mask) | ||
90 | { | ||
91 | WARN_ON(!mutex_is_locked(&pm_mutex)); | ||
92 | gfp_allowed_mask = mask; | ||
93 | } | ||
94 | |||
95 | gfp_t clear_gfp_allowed_mask(gfp_t mask) | ||
96 | { | ||
97 | gfp_t ret = gfp_allowed_mask; | ||
98 | |||
99 | WARN_ON(!mutex_is_locked(&pm_mutex)); | ||
100 | gfp_allowed_mask &= ~mask; | ||
101 | return ret; | ||
102 | } | ||
103 | #endif /* CONFIG_PM_SLEEP */ | ||
104 | |||
79 | #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE | 105 | #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE |
80 | int pageblock_order __read_mostly; | 106 | int pageblock_order __read_mostly; |
81 | #endif | 107 | #endif |
@@ -263,10 +289,7 @@ static void bad_page(struct page *page) | |||
263 | 289 | ||
264 | printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n", | 290 | printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n", |
265 | current->comm, page_to_pfn(page)); | 291 | current->comm, page_to_pfn(page)); |
266 | printk(KERN_ALERT | 292 | dump_page(page); |
267 | "page:%p flags:%p count:%d mapcount:%d mapping:%p index:%lx\n", | ||
268 | page, (void *)page->flags, page_count(page), | ||
269 | page_mapcount(page), page->mapping, page->index); | ||
270 | 293 | ||
271 | dump_stack(); | 294 | dump_stack(); |
272 | out: | 295 | out: |
@@ -530,7 +553,7 @@ static void free_pcppages_bulk(struct zone *zone, int count, | |||
530 | int batch_free = 0; | 553 | int batch_free = 0; |
531 | 554 | ||
532 | spin_lock(&zone->lock); | 555 | spin_lock(&zone->lock); |
533 | zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE); | 556 | zone->all_unreclaimable = 0; |
534 | zone->pages_scanned = 0; | 557 | zone->pages_scanned = 0; |
535 | 558 | ||
536 | __mod_zone_page_state(zone, NR_FREE_PAGES, count); | 559 | __mod_zone_page_state(zone, NR_FREE_PAGES, count); |
@@ -556,8 +579,9 @@ static void free_pcppages_bulk(struct zone *zone, int count, | |||
556 | page = list_entry(list->prev, struct page, lru); | 579 | page = list_entry(list->prev, struct page, lru); |
557 | /* must delete as __free_one_page list manipulates */ | 580 | /* must delete as __free_one_page list manipulates */ |
558 | list_del(&page->lru); | 581 | list_del(&page->lru); |
559 | __free_one_page(page, zone, 0, migratetype); | 582 | /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ |
560 | trace_mm_page_pcpu_drain(page, 0, migratetype); | 583 | __free_one_page(page, zone, 0, page_private(page)); |
584 | trace_mm_page_pcpu_drain(page, 0, page_private(page)); | ||
561 | } while (--count && --batch_free && !list_empty(list)); | 585 | } while (--count && --batch_free && !list_empty(list)); |
562 | } | 586 | } |
563 | spin_unlock(&zone->lock); | 587 | spin_unlock(&zone->lock); |
@@ -567,7 +591,7 @@ static void free_one_page(struct zone *zone, struct page *page, int order, | |||
567 | int migratetype) | 591 | int migratetype) |
568 | { | 592 | { |
569 | spin_lock(&zone->lock); | 593 | spin_lock(&zone->lock); |
570 | zone_clear_flag(zone, ZONE_ALL_UNRECLAIMABLE); | 594 | zone->all_unreclaimable = 0; |
571 | zone->pages_scanned = 0; | 595 | zone->pages_scanned = 0; |
572 | 596 | ||
573 | __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order); | 597 | __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order); |
@@ -582,6 +606,7 @@ static void __free_pages_ok(struct page *page, unsigned int order) | |||
582 | int bad = 0; | 606 | int bad = 0; |
583 | int wasMlocked = __TestClearPageMlocked(page); | 607 | int wasMlocked = __TestClearPageMlocked(page); |
584 | 608 | ||
609 | trace_mm_page_free_direct(page, order); | ||
585 | kmemcheck_free_shadow(page, order); | 610 | kmemcheck_free_shadow(page, order); |
586 | 611 | ||
587 | for (i = 0 ; i < (1 << order) ; ++i) | 612 | for (i = 0 ; i < (1 << order) ; ++i) |
@@ -1008,10 +1033,10 @@ static void drain_pages(unsigned int cpu) | |||
1008 | struct per_cpu_pageset *pset; | 1033 | struct per_cpu_pageset *pset; |
1009 | struct per_cpu_pages *pcp; | 1034 | struct per_cpu_pages *pcp; |
1010 | 1035 | ||
1011 | pset = zone_pcp(zone, cpu); | 1036 | local_irq_save(flags); |
1037 | pset = per_cpu_ptr(zone->pageset, cpu); | ||
1012 | 1038 | ||
1013 | pcp = &pset->pcp; | 1039 | pcp = &pset->pcp; |
1014 | local_irq_save(flags); | ||
1015 | free_pcppages_bulk(zone, pcp->count, pcp); | 1040 | free_pcppages_bulk(zone, pcp->count, pcp); |
1016 | pcp->count = 0; | 1041 | pcp->count = 0; |
1017 | local_irq_restore(flags); | 1042 | local_irq_restore(flags); |
@@ -1072,8 +1097,9 @@ void mark_free_pages(struct zone *zone) | |||
1072 | 1097 | ||
1073 | /* | 1098 | /* |
1074 | * Free a 0-order page | 1099 | * Free a 0-order page |
1100 | * cold == 1 ? free a cold page : free a hot page | ||
1075 | */ | 1101 | */ |
1076 | static void free_hot_cold_page(struct page *page, int cold) | 1102 | void free_hot_cold_page(struct page *page, int cold) |
1077 | { | 1103 | { |
1078 | struct zone *zone = page_zone(page); | 1104 | struct zone *zone = page_zone(page); |
1079 | struct per_cpu_pages *pcp; | 1105 | struct per_cpu_pages *pcp; |
@@ -1081,6 +1107,7 @@ static void free_hot_cold_page(struct page *page, int cold) | |||
1081 | int migratetype; | 1107 | int migratetype; |
1082 | int wasMlocked = __TestClearPageMlocked(page); | 1108 | int wasMlocked = __TestClearPageMlocked(page); |
1083 | 1109 | ||
1110 | trace_mm_page_free_direct(page, 0); | ||
1084 | kmemcheck_free_shadow(page, 0); | 1111 | kmemcheck_free_shadow(page, 0); |
1085 | 1112 | ||
1086 | if (PageAnon(page)) | 1113 | if (PageAnon(page)) |
@@ -1095,7 +1122,6 @@ static void free_hot_cold_page(struct page *page, int cold) | |||
1095 | arch_free_page(page, 0); | 1122 | arch_free_page(page, 0); |
1096 | kernel_map_pages(page, 1, 0); | 1123 | kernel_map_pages(page, 1, 0); |
1097 | 1124 | ||
1098 | pcp = &zone_pcp(zone, get_cpu())->pcp; | ||
1099 | migratetype = get_pageblock_migratetype(page); | 1125 | migratetype = get_pageblock_migratetype(page); |
1100 | set_page_private(page, migratetype); | 1126 | set_page_private(page, migratetype); |
1101 | local_irq_save(flags); | 1127 | local_irq_save(flags); |
@@ -1118,6 +1144,7 @@ static void free_hot_cold_page(struct page *page, int cold) | |||
1118 | migratetype = MIGRATE_MOVABLE; | 1144 | migratetype = MIGRATE_MOVABLE; |
1119 | } | 1145 | } |
1120 | 1146 | ||
1147 | pcp = &this_cpu_ptr(zone->pageset)->pcp; | ||
1121 | if (cold) | 1148 | if (cold) |
1122 | list_add_tail(&page->lru, &pcp->lists[migratetype]); | 1149 | list_add_tail(&page->lru, &pcp->lists[migratetype]); |
1123 | else | 1150 | else |
@@ -1130,15 +1157,8 @@ static void free_hot_cold_page(struct page *page, int cold) | |||
1130 | 1157 | ||
1131 | out: | 1158 | out: |
1132 | local_irq_restore(flags); | 1159 | local_irq_restore(flags); |
1133 | put_cpu(); | ||
1134 | } | 1160 | } |
1135 | 1161 | ||
1136 | void free_hot_page(struct page *page) | ||
1137 | { | ||
1138 | trace_mm_page_free_direct(page, 0); | ||
1139 | free_hot_cold_page(page, 0); | ||
1140 | } | ||
1141 | |||
1142 | /* | 1162 | /* |
1143 | * split_page takes a non-compound higher-order page, and splits it into | 1163 | * split_page takes a non-compound higher-order page, and splits it into |
1144 | * n (1<<order) sub-pages: page[0..n] | 1164 | * n (1<<order) sub-pages: page[0..n] |
@@ -1180,17 +1200,15 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, | |||
1180 | unsigned long flags; | 1200 | unsigned long flags; |
1181 | struct page *page; | 1201 | struct page *page; |
1182 | int cold = !!(gfp_flags & __GFP_COLD); | 1202 | int cold = !!(gfp_flags & __GFP_COLD); |
1183 | int cpu; | ||
1184 | 1203 | ||
1185 | again: | 1204 | again: |
1186 | cpu = get_cpu(); | ||
1187 | if (likely(order == 0)) { | 1205 | if (likely(order == 0)) { |
1188 | struct per_cpu_pages *pcp; | 1206 | struct per_cpu_pages *pcp; |
1189 | struct list_head *list; | 1207 | struct list_head *list; |
1190 | 1208 | ||
1191 | pcp = &zone_pcp(zone, cpu)->pcp; | ||
1192 | list = &pcp->lists[migratetype]; | ||
1193 | local_irq_save(flags); | 1209 | local_irq_save(flags); |
1210 | pcp = &this_cpu_ptr(zone->pageset)->pcp; | ||
1211 | list = &pcp->lists[migratetype]; | ||
1194 | if (list_empty(list)) { | 1212 | if (list_empty(list)) { |
1195 | pcp->count += rmqueue_bulk(zone, 0, | 1213 | pcp->count += rmqueue_bulk(zone, 0, |
1196 | pcp->batch, list, | 1214 | pcp->batch, list, |
@@ -1222,16 +1240,15 @@ again: | |||
1222 | } | 1240 | } |
1223 | spin_lock_irqsave(&zone->lock, flags); | 1241 | spin_lock_irqsave(&zone->lock, flags); |
1224 | page = __rmqueue(zone, order, migratetype); | 1242 | page = __rmqueue(zone, order, migratetype); |
1225 | __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order)); | ||
1226 | spin_unlock(&zone->lock); | 1243 | spin_unlock(&zone->lock); |
1227 | if (!page) | 1244 | if (!page) |
1228 | goto failed; | 1245 | goto failed; |
1246 | __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order)); | ||
1229 | } | 1247 | } |
1230 | 1248 | ||
1231 | __count_zone_vm_events(PGALLOC, zone, 1 << order); | 1249 | __count_zone_vm_events(PGALLOC, zone, 1 << order); |
1232 | zone_statistics(preferred_zone, zone); | 1250 | zone_statistics(preferred_zone, zone); |
1233 | local_irq_restore(flags); | 1251 | local_irq_restore(flags); |
1234 | put_cpu(); | ||
1235 | 1252 | ||
1236 | VM_BUG_ON(bad_range(zone, page)); | 1253 | VM_BUG_ON(bad_range(zone, page)); |
1237 | if (prep_new_page(page, order, gfp_flags)) | 1254 | if (prep_new_page(page, order, gfp_flags)) |
@@ -1240,7 +1257,6 @@ again: | |||
1240 | 1257 | ||
1241 | failed: | 1258 | failed: |
1242 | local_irq_restore(flags); | 1259 | local_irq_restore(flags); |
1243 | put_cpu(); | ||
1244 | return NULL; | 1260 | return NULL; |
1245 | } | 1261 | } |
1246 | 1262 | ||
@@ -2012,9 +2028,8 @@ void __pagevec_free(struct pagevec *pvec) | |||
2012 | void __free_pages(struct page *page, unsigned int order) | 2028 | void __free_pages(struct page *page, unsigned int order) |
2013 | { | 2029 | { |
2014 | if (put_page_testzero(page)) { | 2030 | if (put_page_testzero(page)) { |
2015 | trace_mm_page_free_direct(page, order); | ||
2016 | if (order == 0) | 2031 | if (order == 0) |
2017 | free_hot_page(page); | 2032 | free_hot_cold_page(page, 0); |
2018 | else | 2033 | else |
2019 | __free_pages_ok(page, order); | 2034 | __free_pages_ok(page, order); |
2020 | } | 2035 | } |
@@ -2179,7 +2194,7 @@ void show_free_areas(void) | |||
2179 | for_each_online_cpu(cpu) { | 2194 | for_each_online_cpu(cpu) { |
2180 | struct per_cpu_pageset *pageset; | 2195 | struct per_cpu_pageset *pageset; |
2181 | 2196 | ||
2182 | pageset = zone_pcp(zone, cpu); | 2197 | pageset = per_cpu_ptr(zone->pageset, cpu); |
2183 | 2198 | ||
2184 | printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n", | 2199 | printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n", |
2185 | cpu, pageset->pcp.high, | 2200 | cpu, pageset->pcp.high, |
@@ -2270,7 +2285,7 @@ void show_free_areas(void) | |||
2270 | K(zone_page_state(zone, NR_BOUNCE)), | 2285 | K(zone_page_state(zone, NR_BOUNCE)), |
2271 | K(zone_page_state(zone, NR_WRITEBACK_TEMP)), | 2286 | K(zone_page_state(zone, NR_WRITEBACK_TEMP)), |
2272 | zone->pages_scanned, | 2287 | zone->pages_scanned, |
2273 | (zone_is_all_unreclaimable(zone) ? "yes" : "no") | 2288 | (zone->all_unreclaimable ? "yes" : "no") |
2274 | ); | 2289 | ); |
2275 | printk("lowmem_reserve[]:"); | 2290 | printk("lowmem_reserve[]:"); |
2276 | for (i = 0; i < MAX_NR_ZONES; i++) | 2291 | for (i = 0; i < MAX_NR_ZONES; i++) |
@@ -2744,10 +2759,29 @@ static void build_zonelist_cache(pg_data_t *pgdat) | |||
2744 | 2759 | ||
2745 | #endif /* CONFIG_NUMA */ | 2760 | #endif /* CONFIG_NUMA */ |
2746 | 2761 | ||
2762 | /* | ||
2763 | * Boot pageset table. One per cpu which is going to be used for all | ||
2764 | * zones and all nodes. The parameters will be set in such a way | ||
2765 | * that an item put on a list will immediately be handed over to | ||
2766 | * the buddy list. This is safe since pageset manipulation is done | ||
2767 | * with interrupts disabled. | ||
2768 | * | ||
2769 | * The boot_pagesets must be kept even after bootup is complete for | ||
2770 | * unused processors and/or zones. They do play a role for bootstrapping | ||
2771 | * hotplugged processors. | ||
2772 | * | ||
2773 | * zoneinfo_show() and maybe other functions do | ||
2774 | * not check if the processor is online before following the pageset pointer. | ||
2775 | * Other parts of the kernel may not check if the zone is available. | ||
2776 | */ | ||
2777 | static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); | ||
2778 | static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); | ||
2779 | |||
2747 | /* return values int ....just for stop_machine() */ | 2780 | /* return values int ....just for stop_machine() */ |
2748 | static int __build_all_zonelists(void *dummy) | 2781 | static int __build_all_zonelists(void *dummy) |
2749 | { | 2782 | { |
2750 | int nid; | 2783 | int nid; |
2784 | int cpu; | ||
2751 | 2785 | ||
2752 | #ifdef CONFIG_NUMA | 2786 | #ifdef CONFIG_NUMA |
2753 | memset(node_load, 0, sizeof(node_load)); | 2787 | memset(node_load, 0, sizeof(node_load)); |
@@ -2758,6 +2792,23 @@ static int __build_all_zonelists(void *dummy) | |||
2758 | build_zonelists(pgdat); | 2792 | build_zonelists(pgdat); |
2759 | build_zonelist_cache(pgdat); | 2793 | build_zonelist_cache(pgdat); |
2760 | } | 2794 | } |
2795 | |||
2796 | /* | ||
2797 | * Initialize the boot_pagesets that are going to be used | ||
2798 | * for bootstrapping processors. The real pagesets for | ||
2799 | * each zone will be allocated later when the per cpu | ||
2800 | * allocator is available. | ||
2801 | * | ||
2802 | * boot_pagesets are used also for bootstrapping offline | ||
2803 | * cpus if the system is already booted because the pagesets | ||
2804 | * are needed to initialize allocators on a specific cpu too. | ||
2805 | * F.e. the percpu allocator needs the page allocator which | ||
2806 | * needs the percpu allocator in order to allocate its pagesets | ||
2807 | * (a chicken-egg dilemma). | ||
2808 | */ | ||
2809 | for_each_possible_cpu(cpu) | ||
2810 | setup_pageset(&per_cpu(boot_pageset, cpu), 0); | ||
2811 | |||
2761 | return 0; | 2812 | return 0; |
2762 | } | 2813 | } |
2763 | 2814 | ||
@@ -3095,121 +3146,33 @@ static void setup_pagelist_highmark(struct per_cpu_pageset *p, | |||
3095 | pcp->batch = PAGE_SHIFT * 8; | 3146 | pcp->batch = PAGE_SHIFT * 8; |
3096 | } | 3147 | } |
3097 | 3148 | ||
3098 | |||
3099 | #ifdef CONFIG_NUMA | ||
3100 | /* | 3149 | /* |
3101 | * Boot pageset table. One per cpu which is going to be used for all | 3150 | * Allocate per cpu pagesets and initialize them. |
3102 | * zones and all nodes. The parameters will be set in such a way | 3151 | * Before this call only boot pagesets were available. |
3103 | * that an item put on a list will immediately be handed over to | 3152 | * Boot pagesets will no longer be used by this processorr |
3104 | * the buddy list. This is safe since pageset manipulation is done | 3153 | * after setup_per_cpu_pageset(). |
3105 | * with interrupts disabled. | ||
3106 | * | ||
3107 | * Some NUMA counter updates may also be caught by the boot pagesets. | ||
3108 | * | ||
3109 | * The boot_pagesets must be kept even after bootup is complete for | ||
3110 | * unused processors and/or zones. They do play a role for bootstrapping | ||
3111 | * hotplugged processors. | ||
3112 | * | ||
3113 | * zoneinfo_show() and maybe other functions do | ||
3114 | * not check if the processor is online before following the pageset pointer. | ||
3115 | * Other parts of the kernel may not check if the zone is available. | ||
3116 | */ | ||
3117 | static struct per_cpu_pageset boot_pageset[NR_CPUS]; | ||
3118 | |||
3119 | /* | ||
3120 | * Dynamically allocate memory for the | ||
3121 | * per cpu pageset array in struct zone. | ||
3122 | */ | 3154 | */ |
3123 | static int __cpuinit process_zones(int cpu) | 3155 | void __init setup_per_cpu_pageset(void) |
3124 | { | 3156 | { |
3125 | struct zone *zone, *dzone; | 3157 | struct zone *zone; |
3126 | int node = cpu_to_node(cpu); | 3158 | int cpu; |
3127 | |||
3128 | node_set_state(node, N_CPU); /* this node has a cpu */ | ||
3129 | 3159 | ||
3130 | for_each_populated_zone(zone) { | 3160 | for_each_populated_zone(zone) { |
3131 | zone_pcp(zone, cpu) = kmalloc_node(sizeof(struct per_cpu_pageset), | 3161 | zone->pageset = alloc_percpu(struct per_cpu_pageset); |
3132 | GFP_KERNEL, node); | ||
3133 | if (!zone_pcp(zone, cpu)) | ||
3134 | goto bad; | ||
3135 | 3162 | ||
3136 | setup_pageset(zone_pcp(zone, cpu), zone_batchsize(zone)); | 3163 | for_each_possible_cpu(cpu) { |
3164 | struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); | ||
3137 | 3165 | ||
3138 | if (percpu_pagelist_fraction) | 3166 | setup_pageset(pcp, zone_batchsize(zone)); |
3139 | setup_pagelist_highmark(zone_pcp(zone, cpu), | ||
3140 | (zone->present_pages / percpu_pagelist_fraction)); | ||
3141 | } | ||
3142 | |||
3143 | return 0; | ||
3144 | bad: | ||
3145 | for_each_zone(dzone) { | ||
3146 | if (!populated_zone(dzone)) | ||
3147 | continue; | ||
3148 | if (dzone == zone) | ||
3149 | break; | ||
3150 | kfree(zone_pcp(dzone, cpu)); | ||
3151 | zone_pcp(dzone, cpu) = &boot_pageset[cpu]; | ||
3152 | } | ||
3153 | return -ENOMEM; | ||
3154 | } | ||
3155 | |||
3156 | static inline void free_zone_pagesets(int cpu) | ||
3157 | { | ||
3158 | struct zone *zone; | ||
3159 | 3167 | ||
3160 | for_each_zone(zone) { | 3168 | if (percpu_pagelist_fraction) |
3161 | struct per_cpu_pageset *pset = zone_pcp(zone, cpu); | 3169 | setup_pagelist_highmark(pcp, |
3162 | 3170 | (zone->present_pages / | |
3163 | /* Free per_cpu_pageset if it is slab allocated */ | 3171 | percpu_pagelist_fraction)); |
3164 | if (pset != &boot_pageset[cpu]) | 3172 | } |
3165 | kfree(pset); | ||
3166 | zone_pcp(zone, cpu) = &boot_pageset[cpu]; | ||
3167 | } | ||
3168 | } | ||
3169 | |||
3170 | static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb, | ||
3171 | unsigned long action, | ||
3172 | void *hcpu) | ||
3173 | { | ||
3174 | int cpu = (long)hcpu; | ||
3175 | int ret = NOTIFY_OK; | ||
3176 | |||
3177 | switch (action) { | ||
3178 | case CPU_UP_PREPARE: | ||
3179 | case CPU_UP_PREPARE_FROZEN: | ||
3180 | if (process_zones(cpu)) | ||
3181 | ret = NOTIFY_BAD; | ||
3182 | break; | ||
3183 | case CPU_UP_CANCELED: | ||
3184 | case CPU_UP_CANCELED_FROZEN: | ||
3185 | case CPU_DEAD: | ||
3186 | case CPU_DEAD_FROZEN: | ||
3187 | free_zone_pagesets(cpu); | ||
3188 | break; | ||
3189 | default: | ||
3190 | break; | ||
3191 | } | 3173 | } |
3192 | return ret; | ||
3193 | } | ||
3194 | |||
3195 | static struct notifier_block __cpuinitdata pageset_notifier = | ||
3196 | { &pageset_cpuup_callback, NULL, 0 }; | ||
3197 | |||
3198 | void __init setup_per_cpu_pageset(void) | ||
3199 | { | ||
3200 | int err; | ||
3201 | |||
3202 | /* Initialize per_cpu_pageset for cpu 0. | ||
3203 | * A cpuup callback will do this for every cpu | ||
3204 | * as it comes online | ||
3205 | */ | ||
3206 | err = process_zones(smp_processor_id()); | ||
3207 | BUG_ON(err); | ||
3208 | register_cpu_notifier(&pageset_notifier); | ||
3209 | } | 3174 | } |
3210 | 3175 | ||
3211 | #endif | ||
3212 | |||
3213 | static noinline __init_refok | 3176 | static noinline __init_refok |
3214 | int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) | 3177 | int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) |
3215 | { | 3178 | { |
@@ -3259,11 +3222,11 @@ static int __zone_pcp_update(void *data) | |||
3259 | int cpu; | 3222 | int cpu; |
3260 | unsigned long batch = zone_batchsize(zone), flags; | 3223 | unsigned long batch = zone_batchsize(zone), flags; |
3261 | 3224 | ||
3262 | for (cpu = 0; cpu < NR_CPUS; cpu++) { | 3225 | for_each_possible_cpu(cpu) { |
3263 | struct per_cpu_pageset *pset; | 3226 | struct per_cpu_pageset *pset; |
3264 | struct per_cpu_pages *pcp; | 3227 | struct per_cpu_pages *pcp; |
3265 | 3228 | ||
3266 | pset = zone_pcp(zone, cpu); | 3229 | pset = per_cpu_ptr(zone->pageset, cpu); |
3267 | pcp = &pset->pcp; | 3230 | pcp = &pset->pcp; |
3268 | 3231 | ||
3269 | local_irq_save(flags); | 3232 | local_irq_save(flags); |
@@ -3281,21 +3244,17 @@ void zone_pcp_update(struct zone *zone) | |||
3281 | 3244 | ||
3282 | static __meminit void zone_pcp_init(struct zone *zone) | 3245 | static __meminit void zone_pcp_init(struct zone *zone) |
3283 | { | 3246 | { |
3284 | int cpu; | 3247 | /* |
3285 | unsigned long batch = zone_batchsize(zone); | 3248 | * per cpu subsystem is not up at this point. The following code |
3249 | * relies on the ability of the linker to provide the | ||
3250 | * offset of a (static) per cpu variable into the per cpu area. | ||
3251 | */ | ||
3252 | zone->pageset = &boot_pageset; | ||
3286 | 3253 | ||
3287 | for (cpu = 0; cpu < NR_CPUS; cpu++) { | ||
3288 | #ifdef CONFIG_NUMA | ||
3289 | /* Early boot. Slab allocator not functional yet */ | ||
3290 | zone_pcp(zone, cpu) = &boot_pageset[cpu]; | ||
3291 | setup_pageset(&boot_pageset[cpu],0); | ||
3292 | #else | ||
3293 | setup_pageset(zone_pcp(zone,cpu), batch); | ||
3294 | #endif | ||
3295 | } | ||
3296 | if (zone->present_pages) | 3254 | if (zone->present_pages) |
3297 | printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%lu\n", | 3255 | printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", |
3298 | zone->name, zone->present_pages, batch); | 3256 | zone->name, zone->present_pages, |
3257 | zone_batchsize(zone)); | ||
3299 | } | 3258 | } |
3300 | 3259 | ||
3301 | __meminit int init_currently_empty_zone(struct zone *zone, | 3260 | __meminit int init_currently_empty_zone(struct zone *zone, |
@@ -3434,6 +3393,61 @@ void __init free_bootmem_with_active_regions(int nid, | |||
3434 | } | 3393 | } |
3435 | } | 3394 | } |
3436 | 3395 | ||
3396 | int __init add_from_early_node_map(struct range *range, int az, | ||
3397 | int nr_range, int nid) | ||
3398 | { | ||
3399 | int i; | ||
3400 | u64 start, end; | ||
3401 | |||
3402 | /* need to go over early_node_map to find out good range for node */ | ||
3403 | for_each_active_range_index_in_nid(i, nid) { | ||
3404 | start = early_node_map[i].start_pfn; | ||
3405 | end = early_node_map[i].end_pfn; | ||
3406 | nr_range = add_range(range, az, nr_range, start, end); | ||
3407 | } | ||
3408 | return nr_range; | ||
3409 | } | ||
3410 | |||
3411 | #ifdef CONFIG_NO_BOOTMEM | ||
3412 | void * __init __alloc_memory_core_early(int nid, u64 size, u64 align, | ||
3413 | u64 goal, u64 limit) | ||
3414 | { | ||
3415 | int i; | ||
3416 | void *ptr; | ||
3417 | |||
3418 | /* need to go over early_node_map to find out good range for node */ | ||
3419 | for_each_active_range_index_in_nid(i, nid) { | ||
3420 | u64 addr; | ||
3421 | u64 ei_start, ei_last; | ||
3422 | |||
3423 | ei_last = early_node_map[i].end_pfn; | ||
3424 | ei_last <<= PAGE_SHIFT; | ||
3425 | ei_start = early_node_map[i].start_pfn; | ||
3426 | ei_start <<= PAGE_SHIFT; | ||
3427 | addr = find_early_area(ei_start, ei_last, | ||
3428 | goal, limit, size, align); | ||
3429 | |||
3430 | if (addr == -1ULL) | ||
3431 | continue; | ||
3432 | |||
3433 | #if 0 | ||
3434 | printk(KERN_DEBUG "alloc (nid=%d %llx - %llx) (%llx - %llx) %llx %llx => %llx\n", | ||
3435 | nid, | ||
3436 | ei_start, ei_last, goal, limit, size, | ||
3437 | align, addr); | ||
3438 | #endif | ||
3439 | |||
3440 | ptr = phys_to_virt(addr); | ||
3441 | memset(ptr, 0, size); | ||
3442 | reserve_early_without_check(addr, addr + size, "BOOTMEM"); | ||
3443 | return ptr; | ||
3444 | } | ||
3445 | |||
3446 | return NULL; | ||
3447 | } | ||
3448 | #endif | ||
3449 | |||
3450 | |||
3437 | void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data) | 3451 | void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data) |
3438 | { | 3452 | { |
3439 | int i; | 3453 | int i; |
@@ -3998,7 +4012,7 @@ void __init add_active_range(unsigned int nid, unsigned long start_pfn, | |||
3998 | } | 4012 | } |
3999 | 4013 | ||
4000 | /* Merge backward if suitable */ | 4014 | /* Merge backward if suitable */ |
4001 | if (start_pfn < early_node_map[i].end_pfn && | 4015 | if (start_pfn < early_node_map[i].start_pfn && |
4002 | end_pfn >= early_node_map[i].start_pfn) { | 4016 | end_pfn >= early_node_map[i].start_pfn) { |
4003 | early_node_map[i].start_pfn = start_pfn; | 4017 | early_node_map[i].start_pfn = start_pfn; |
4004 | return; | 4018 | return; |
@@ -4376,8 +4390,12 @@ void __init free_area_init_nodes(unsigned long *max_zone_pfn) | |||
4376 | for (i = 0; i < MAX_NR_ZONES; i++) { | 4390 | for (i = 0; i < MAX_NR_ZONES; i++) { |
4377 | if (i == ZONE_MOVABLE) | 4391 | if (i == ZONE_MOVABLE) |
4378 | continue; | 4392 | continue; |
4379 | printk(" %-8s %0#10lx -> %0#10lx\n", | 4393 | printk(" %-8s ", zone_names[i]); |
4380 | zone_names[i], | 4394 | if (arch_zone_lowest_possible_pfn[i] == |
4395 | arch_zone_highest_possible_pfn[i]) | ||
4396 | printk("empty\n"); | ||
4397 | else | ||
4398 | printk("%0#10lx -> %0#10lx\n", | ||
4381 | arch_zone_lowest_possible_pfn[i], | 4399 | arch_zone_lowest_possible_pfn[i], |
4382 | arch_zone_highest_possible_pfn[i]); | 4400 | arch_zone_highest_possible_pfn[i]); |
4383 | } | 4401 | } |
@@ -4466,7 +4484,11 @@ void __init set_dma_reserve(unsigned long new_dma_reserve) | |||
4466 | } | 4484 | } |
4467 | 4485 | ||
4468 | #ifndef CONFIG_NEED_MULTIPLE_NODES | 4486 | #ifndef CONFIG_NEED_MULTIPLE_NODES |
4469 | struct pglist_data __refdata contig_page_data = { .bdata = &bootmem_node_data[0] }; | 4487 | struct pglist_data __refdata contig_page_data = { |
4488 | #ifndef CONFIG_NO_BOOTMEM | ||
4489 | .bdata = &bootmem_node_data[0] | ||
4490 | #endif | ||
4491 | }; | ||
4470 | EXPORT_SYMBOL(contig_page_data); | 4492 | EXPORT_SYMBOL(contig_page_data); |
4471 | #endif | 4493 | #endif |
4472 | 4494 | ||
@@ -4809,10 +4831,11 @@ int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write, | |||
4809 | if (!write || (ret == -EINVAL)) | 4831 | if (!write || (ret == -EINVAL)) |
4810 | return ret; | 4832 | return ret; |
4811 | for_each_populated_zone(zone) { | 4833 | for_each_populated_zone(zone) { |
4812 | for_each_online_cpu(cpu) { | 4834 | for_each_possible_cpu(cpu) { |
4813 | unsigned long high; | 4835 | unsigned long high; |
4814 | high = zone->present_pages / percpu_pagelist_fraction; | 4836 | high = zone->present_pages / percpu_pagelist_fraction; |
4815 | setup_pagelist_highmark(zone_pcp(zone, cpu), high); | 4837 | setup_pagelist_highmark( |
4838 | per_cpu_ptr(zone->pageset, cpu), high); | ||
4816 | } | 4839 | } |
4817 | } | 4840 | } |
4818 | return 0; | 4841 | return 0; |
@@ -5158,3 +5181,80 @@ bool is_free_buddy_page(struct page *page) | |||
5158 | return order < MAX_ORDER; | 5181 | return order < MAX_ORDER; |
5159 | } | 5182 | } |
5160 | #endif | 5183 | #endif |
5184 | |||
5185 | static struct trace_print_flags pageflag_names[] = { | ||
5186 | {1UL << PG_locked, "locked" }, | ||
5187 | {1UL << PG_error, "error" }, | ||
5188 | {1UL << PG_referenced, "referenced" }, | ||
5189 | {1UL << PG_uptodate, "uptodate" }, | ||
5190 | {1UL << PG_dirty, "dirty" }, | ||
5191 | {1UL << PG_lru, "lru" }, | ||
5192 | {1UL << PG_active, "active" }, | ||
5193 | {1UL << PG_slab, "slab" }, | ||
5194 | {1UL << PG_owner_priv_1, "owner_priv_1" }, | ||
5195 | {1UL << PG_arch_1, "arch_1" }, | ||
5196 | {1UL << PG_reserved, "reserved" }, | ||
5197 | {1UL << PG_private, "private" }, | ||
5198 | {1UL << PG_private_2, "private_2" }, | ||
5199 | {1UL << PG_writeback, "writeback" }, | ||
5200 | #ifdef CONFIG_PAGEFLAGS_EXTENDED | ||
5201 | {1UL << PG_head, "head" }, | ||
5202 | {1UL << PG_tail, "tail" }, | ||
5203 | #else | ||
5204 | {1UL << PG_compound, "compound" }, | ||
5205 | #endif | ||
5206 | {1UL << PG_swapcache, "swapcache" }, | ||
5207 | {1UL << PG_mappedtodisk, "mappedtodisk" }, | ||
5208 | {1UL << PG_reclaim, "reclaim" }, | ||
5209 | {1UL << PG_buddy, "buddy" }, | ||
5210 | {1UL << PG_swapbacked, "swapbacked" }, | ||
5211 | {1UL << PG_unevictable, "unevictable" }, | ||
5212 | #ifdef CONFIG_MMU | ||
5213 | {1UL << PG_mlocked, "mlocked" }, | ||
5214 | #endif | ||
5215 | #ifdef CONFIG_ARCH_USES_PG_UNCACHED | ||
5216 | {1UL << PG_uncached, "uncached" }, | ||
5217 | #endif | ||
5218 | #ifdef CONFIG_MEMORY_FAILURE | ||
5219 | {1UL << PG_hwpoison, "hwpoison" }, | ||
5220 | #endif | ||
5221 | {-1UL, NULL }, | ||
5222 | }; | ||
5223 | |||
5224 | static void dump_page_flags(unsigned long flags) | ||
5225 | { | ||
5226 | const char *delim = ""; | ||
5227 | unsigned long mask; | ||
5228 | int i; | ||
5229 | |||
5230 | printk(KERN_ALERT "page flags: %#lx(", flags); | ||
5231 | |||
5232 | /* remove zone id */ | ||
5233 | flags &= (1UL << NR_PAGEFLAGS) - 1; | ||
5234 | |||
5235 | for (i = 0; pageflag_names[i].name && flags; i++) { | ||
5236 | |||
5237 | mask = pageflag_names[i].mask; | ||
5238 | if ((flags & mask) != mask) | ||
5239 | continue; | ||
5240 | |||
5241 | flags &= ~mask; | ||
5242 | printk("%s%s", delim, pageflag_names[i].name); | ||
5243 | delim = "|"; | ||
5244 | } | ||
5245 | |||
5246 | /* check for left over flags */ | ||
5247 | if (flags) | ||
5248 | printk("%s%#lx", delim, flags); | ||
5249 | |||
5250 | printk(")\n"); | ||
5251 | } | ||
5252 | |||
5253 | void dump_page(struct page *page) | ||
5254 | { | ||
5255 | printk(KERN_ALERT | ||
5256 | "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n", | ||
5257 | page, page_count(page), page_mapcount(page), | ||
5258 | page->mapping, page->index); | ||
5259 | dump_page_flags(page->flags); | ||
5260 | } | ||