aboutsummaryrefslogtreecommitdiffstats
path: root/mm/bootmem.c
diff options
context:
space:
mode:
authorGlenn Elliott <gelliott@cs.unc.edu>2012-03-04 19:47:13 -0500
committerGlenn Elliott <gelliott@cs.unc.edu>2012-03-04 19:47:13 -0500
commitc71c03bda1e86c9d5198c5d83f712e695c4f2a1e (patch)
treeecb166cb3e2b7e2adb3b5e292245fefd23381ac8 /mm/bootmem.c
parentea53c912f8a86a8567697115b6a0d8152beee5c8 (diff)
parent6a00f206debf8a5c8899055726ad127dbeeed098 (diff)
Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp
Conflicts: litmus/sched_cedf.c
Diffstat (limited to 'mm/bootmem.c')
-rw-r--r--mm/bootmem.c193
1 files changed, 17 insertions, 176 deletions
diff --git a/mm/bootmem.c b/mm/bootmem.c
index 142c84a54993..01d5a4b3dd0c 100644
--- a/mm/bootmem.c
+++ b/mm/bootmem.c
@@ -15,6 +15,7 @@
15#include <linux/module.h> 15#include <linux/module.h>
16#include <linux/kmemleak.h> 16#include <linux/kmemleak.h>
17#include <linux/range.h> 17#include <linux/range.h>
18#include <linux/memblock.h>
18 19
19#include <asm/bug.h> 20#include <asm/bug.h>
20#include <asm/io.h> 21#include <asm/io.h>
@@ -22,19 +23,17 @@
22 23
23#include "internal.h" 24#include "internal.h"
24 25
26#ifndef CONFIG_NEED_MULTIPLE_NODES
27struct pglist_data __refdata contig_page_data = {
28 .bdata = &bootmem_node_data[0]
29};
30EXPORT_SYMBOL(contig_page_data);
31#endif
32
25unsigned long max_low_pfn; 33unsigned long max_low_pfn;
26unsigned long min_low_pfn; 34unsigned long min_low_pfn;
27unsigned long max_pfn; 35unsigned long max_pfn;
28 36
29#ifdef CONFIG_CRASH_DUMP
30/*
31 * If we have booted due to a crash, max_pfn will be a very low value. We need
32 * to know the amount of memory that the previous kernel used.
33 */
34unsigned long saved_max_pfn;
35#endif
36
37#ifndef CONFIG_NO_BOOTMEM
38bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata; 37bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
39 38
40static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list); 39static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
@@ -145,7 +144,7 @@ unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
145 min_low_pfn = start; 144 min_low_pfn = start;
146 return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages); 145 return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
147} 146}
148#endif 147
149/* 148/*
150 * free_bootmem_late - free bootmem pages directly to page allocator 149 * free_bootmem_late - free bootmem pages directly to page allocator
151 * @addr: starting address of the range 150 * @addr: starting address of the range
@@ -170,53 +169,6 @@ void __init free_bootmem_late(unsigned long addr, unsigned long size)
170 } 169 }
171} 170}
172 171
173#ifdef CONFIG_NO_BOOTMEM
174static void __init __free_pages_memory(unsigned long start, unsigned long end)
175{
176 int i;
177 unsigned long start_aligned, end_aligned;
178 int order = ilog2(BITS_PER_LONG);
179
180 start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
181 end_aligned = end & ~(BITS_PER_LONG - 1);
182
183 if (end_aligned <= start_aligned) {
184 for (i = start; i < end; i++)
185 __free_pages_bootmem(pfn_to_page(i), 0);
186
187 return;
188 }
189
190 for (i = start; i < start_aligned; i++)
191 __free_pages_bootmem(pfn_to_page(i), 0);
192
193 for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
194 __free_pages_bootmem(pfn_to_page(i), order);
195
196 for (i = end_aligned; i < end; i++)
197 __free_pages_bootmem(pfn_to_page(i), 0);
198}
199
200unsigned long __init free_all_memory_core_early(int nodeid)
201{
202 int i;
203 u64 start, end;
204 unsigned long count = 0;
205 struct range *range = NULL;
206 int nr_range;
207
208 nr_range = get_free_all_memory_range(&range, nodeid);
209
210 for (i = 0; i < nr_range; i++) {
211 start = range[i].start;
212 end = range[i].end;
213 count += end - start;
214 __free_pages_memory(start, end);
215 }
216
217 return count;
218}
219#else
220static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata) 172static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
221{ 173{
222 int aligned; 174 int aligned;
@@ -277,7 +229,6 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
277 229
278 return count; 230 return count;
279} 231}
280#endif
281 232
282/** 233/**
283 * free_all_bootmem_node - release a node's free pages to the buddy allocator 234 * free_all_bootmem_node - release a node's free pages to the buddy allocator
@@ -288,12 +239,7 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
288unsigned long __init free_all_bootmem_node(pg_data_t *pgdat) 239unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
289{ 240{
290 register_page_bootmem_info_node(pgdat); 241 register_page_bootmem_info_node(pgdat);
291#ifdef CONFIG_NO_BOOTMEM
292 /* free_all_memory_core_early(MAX_NUMNODES) will be called later */
293 return 0;
294#else
295 return free_all_bootmem_core(pgdat->bdata); 242 return free_all_bootmem_core(pgdat->bdata);
296#endif
297} 243}
298 244
299/** 245/**
@@ -303,16 +249,6 @@ unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
303 */ 249 */
304unsigned long __init free_all_bootmem(void) 250unsigned long __init free_all_bootmem(void)
305{ 251{
306#ifdef CONFIG_NO_BOOTMEM
307 /*
308 * We need to use MAX_NUMNODES instead of NODE_DATA(0)->node_id
309 * because in some case like Node0 doesnt have RAM installed
310 * low ram will be on Node1
311 * Use MAX_NUMNODES will make sure all ranges in early_node_map[]
312 * will be used instead of only Node0 related
313 */
314 return free_all_memory_core_early(MAX_NUMNODES);
315#else
316 unsigned long total_pages = 0; 252 unsigned long total_pages = 0;
317 bootmem_data_t *bdata; 253 bootmem_data_t *bdata;
318 254
@@ -320,10 +256,8 @@ unsigned long __init free_all_bootmem(void)
320 total_pages += free_all_bootmem_core(bdata); 256 total_pages += free_all_bootmem_core(bdata);
321 257
322 return total_pages; 258 return total_pages;
323#endif
324} 259}
325 260
326#ifndef CONFIG_NO_BOOTMEM
327static void __init __free(bootmem_data_t *bdata, 261static void __init __free(bootmem_data_t *bdata,
328 unsigned long sidx, unsigned long eidx) 262 unsigned long sidx, unsigned long eidx)
329{ 263{
@@ -418,7 +352,6 @@ static int __init mark_bootmem(unsigned long start, unsigned long end,
418 } 352 }
419 BUG(); 353 BUG();
420} 354}
421#endif
422 355
423/** 356/**
424 * free_bootmem_node - mark a page range as usable 357 * free_bootmem_node - mark a page range as usable
@@ -433,9 +366,6 @@ static int __init mark_bootmem(unsigned long start, unsigned long end,
433void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, 366void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
434 unsigned long size) 367 unsigned long size)
435{ 368{
436#ifdef CONFIG_NO_BOOTMEM
437 free_early(physaddr, physaddr + size);
438#else
439 unsigned long start, end; 369 unsigned long start, end;
440 370
441 kmemleak_free_part(__va(physaddr), size); 371 kmemleak_free_part(__va(physaddr), size);
@@ -444,7 +374,6 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
444 end = PFN_DOWN(physaddr + size); 374 end = PFN_DOWN(physaddr + size);
445 375
446 mark_bootmem_node(pgdat->bdata, start, end, 0, 0); 376 mark_bootmem_node(pgdat->bdata, start, end, 0, 0);
447#endif
448} 377}
449 378
450/** 379/**
@@ -458,9 +387,6 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
458 */ 387 */
459void __init free_bootmem(unsigned long addr, unsigned long size) 388void __init free_bootmem(unsigned long addr, unsigned long size)
460{ 389{
461#ifdef CONFIG_NO_BOOTMEM
462 free_early(addr, addr + size);
463#else
464 unsigned long start, end; 390 unsigned long start, end;
465 391
466 kmemleak_free_part(__va(addr), size); 392 kmemleak_free_part(__va(addr), size);
@@ -469,7 +395,6 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
469 end = PFN_DOWN(addr + size); 395 end = PFN_DOWN(addr + size);
470 396
471 mark_bootmem(start, end, 0, 0); 397 mark_bootmem(start, end, 0, 0);
472#endif
473} 398}
474 399
475/** 400/**
@@ -486,17 +411,12 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
486int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, 411int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
487 unsigned long size, int flags) 412 unsigned long size, int flags)
488{ 413{
489#ifdef CONFIG_NO_BOOTMEM
490 panic("no bootmem");
491 return 0;
492#else
493 unsigned long start, end; 414 unsigned long start, end;
494 415
495 start = PFN_DOWN(physaddr); 416 start = PFN_DOWN(physaddr);
496 end = PFN_UP(physaddr + size); 417 end = PFN_UP(physaddr + size);
497 418
498 return mark_bootmem_node(pgdat->bdata, start, end, 1, flags); 419 return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);
499#endif
500} 420}
501 421
502/** 422/**
@@ -512,20 +432,20 @@ int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
512int __init reserve_bootmem(unsigned long addr, unsigned long size, 432int __init reserve_bootmem(unsigned long addr, unsigned long size,
513 int flags) 433 int flags)
514{ 434{
515#ifdef CONFIG_NO_BOOTMEM
516 panic("no bootmem");
517 return 0;
518#else
519 unsigned long start, end; 435 unsigned long start, end;
520 436
521 start = PFN_DOWN(addr); 437 start = PFN_DOWN(addr);
522 end = PFN_UP(addr + size); 438 end = PFN_UP(addr + size);
523 439
524 return mark_bootmem(start, end, 1, flags); 440 return mark_bootmem(start, end, 1, flags);
525#endif
526} 441}
527 442
528#ifndef CONFIG_NO_BOOTMEM 443int __weak __init reserve_bootmem_generic(unsigned long phys, unsigned long len,
444 int flags)
445{
446 return reserve_bootmem(phys, len, flags);
447}
448
529static unsigned long __init align_idx(struct bootmem_data *bdata, 449static unsigned long __init align_idx(struct bootmem_data *bdata,
530 unsigned long idx, unsigned long step) 450 unsigned long idx, unsigned long step)
531{ 451{
@@ -676,33 +596,12 @@ static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata,
676#endif 596#endif
677 return NULL; 597 return NULL;
678} 598}
679#endif
680 599
681static void * __init ___alloc_bootmem_nopanic(unsigned long size, 600static void * __init ___alloc_bootmem_nopanic(unsigned long size,
682 unsigned long align, 601 unsigned long align,
683 unsigned long goal, 602 unsigned long goal,
684 unsigned long limit) 603 unsigned long limit)
685{ 604{
686#ifdef CONFIG_NO_BOOTMEM
687 void *ptr;
688
689 if (WARN_ON_ONCE(slab_is_available()))
690 return kzalloc(size, GFP_NOWAIT);
691
692restart:
693
694 ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit);
695
696 if (ptr)
697 return ptr;
698
699 if (goal != 0) {
700 goal = 0;
701 goto restart;
702 }
703
704 return NULL;
705#else
706 bootmem_data_t *bdata; 605 bootmem_data_t *bdata;
707 void *region; 606 void *region;
708 607
@@ -728,7 +627,6 @@ restart:
728 } 627 }
729 628
730 return NULL; 629 return NULL;
731#endif
732} 630}
733 631
734/** 632/**
@@ -749,10 +647,6 @@ void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
749{ 647{
750 unsigned long limit = 0; 648 unsigned long limit = 0;
751 649
752#ifdef CONFIG_NO_BOOTMEM
753 limit = -1UL;
754#endif
755
756 return ___alloc_bootmem_nopanic(size, align, goal, limit); 650 return ___alloc_bootmem_nopanic(size, align, goal, limit);
757} 651}
758 652
@@ -789,14 +683,9 @@ void * __init __alloc_bootmem(unsigned long size, unsigned long align,
789{ 683{
790 unsigned long limit = 0; 684 unsigned long limit = 0;
791 685
792#ifdef CONFIG_NO_BOOTMEM
793 limit = -1UL;
794#endif
795
796 return ___alloc_bootmem(size, align, goal, limit); 686 return ___alloc_bootmem(size, align, goal, limit);
797} 687}
798 688
799#ifndef CONFIG_NO_BOOTMEM
800static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata, 689static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
801 unsigned long size, unsigned long align, 690 unsigned long size, unsigned long align,
802 unsigned long goal, unsigned long limit) 691 unsigned long goal, unsigned long limit)
@@ -813,7 +702,6 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
813 702
814 return ___alloc_bootmem(size, align, goal, limit); 703 return ___alloc_bootmem(size, align, goal, limit);
815} 704}
816#endif
817 705
818/** 706/**
819 * __alloc_bootmem_node - allocate boot memory from a specific node 707 * __alloc_bootmem_node - allocate boot memory from a specific node
@@ -833,24 +721,10 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
833void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, 721void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
834 unsigned long align, unsigned long goal) 722 unsigned long align, unsigned long goal)
835{ 723{
836 void *ptr;
837
838 if (WARN_ON_ONCE(slab_is_available())) 724 if (WARN_ON_ONCE(slab_is_available()))
839 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 725 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
840 726
841#ifdef CONFIG_NO_BOOTMEM 727 return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
842 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
843 goal, -1ULL);
844 if (ptr)
845 return ptr;
846
847 ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align,
848 goal, -1ULL);
849#else
850 ptr = ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
851#endif
852
853 return ptr;
854} 728}
855 729
856void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size, 730void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
@@ -871,13 +745,8 @@ void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
871 unsigned long new_goal; 745 unsigned long new_goal;
872 746
873 new_goal = MAX_DMA32_PFN << PAGE_SHIFT; 747 new_goal = MAX_DMA32_PFN << PAGE_SHIFT;
874#ifdef CONFIG_NO_BOOTMEM
875 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
876 new_goal, -1ULL);
877#else
878 ptr = alloc_bootmem_core(pgdat->bdata, size, align, 748 ptr = alloc_bootmem_core(pgdat->bdata, size, align,
879 new_goal, 0); 749 new_goal, 0);
880#endif
881 if (ptr) 750 if (ptr)
882 return ptr; 751 return ptr;
883 } 752 }
@@ -898,16 +767,6 @@ void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
898void * __init alloc_bootmem_section(unsigned long size, 767void * __init alloc_bootmem_section(unsigned long size,
899 unsigned long section_nr) 768 unsigned long section_nr)
900{ 769{
901#ifdef CONFIG_NO_BOOTMEM
902 unsigned long pfn, goal, limit;
903
904 pfn = section_nr_to_pfn(section_nr);
905 goal = pfn << PAGE_SHIFT;
906 limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT;
907
908 return __alloc_memory_core_early(early_pfn_to_nid(pfn), size,
909 SMP_CACHE_BYTES, goal, limit);
910#else
911 bootmem_data_t *bdata; 770 bootmem_data_t *bdata;
912 unsigned long pfn, goal, limit; 771 unsigned long pfn, goal, limit;
913 772
@@ -917,7 +776,6 @@ void * __init alloc_bootmem_section(unsigned long size,
917 bdata = &bootmem_node_data[early_pfn_to_nid(pfn)]; 776 bdata = &bootmem_node_data[early_pfn_to_nid(pfn)];
918 777
919 return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit); 778 return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit);
920#endif
921} 779}
922#endif 780#endif
923 781
@@ -929,16 +787,11 @@ void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
929 if (WARN_ON_ONCE(slab_is_available())) 787 if (WARN_ON_ONCE(slab_is_available()))
930 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 788 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
931 789
932#ifdef CONFIG_NO_BOOTMEM
933 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
934 goal, -1ULL);
935#else
936 ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0); 790 ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);
937 if (ptr) 791 if (ptr)
938 return ptr; 792 return ptr;
939 793
940 ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0); 794 ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
941#endif
942 if (ptr) 795 if (ptr)
943 return ptr; 796 return ptr;
944 797
@@ -986,21 +839,9 @@ void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
986void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size, 839void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
987 unsigned long align, unsigned long goal) 840 unsigned long align, unsigned long goal)
988{ 841{
989 void *ptr;
990
991 if (WARN_ON_ONCE(slab_is_available())) 842 if (WARN_ON_ONCE(slab_is_available()))
992 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 843 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
993 844
994#ifdef CONFIG_NO_BOOTMEM 845 return ___alloc_bootmem_node(pgdat->bdata, size, align,
995 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
996 goal, ARCH_LOW_ADDRESS_LIMIT); 846 goal, ARCH_LOW_ADDRESS_LIMIT);
997 if (ptr)
998 return ptr;
999 ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align,
1000 goal, ARCH_LOW_ADDRESS_LIMIT);
1001#else
1002 ptr = ___alloc_bootmem_node(pgdat->bdata, size, align,
1003 goal, ARCH_LOW_ADDRESS_LIMIT);
1004#endif
1005 return ptr;
1006} 847}