aboutsummaryrefslogtreecommitdiffstats
path: root/mm/bootmem.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/bootmem.c')
-rw-r--r--mm/bootmem.c180
1 files changed, 10 insertions, 170 deletions
diff --git a/mm/bootmem.c b/mm/bootmem.c
index 13b0caa9793c..07aeb89e396e 100644
--- a/mm/bootmem.c
+++ b/mm/bootmem.c
@@ -23,6 +23,13 @@
23 23
24#include "internal.h" 24#include "internal.h"
25 25
26#ifndef CONFIG_NEED_MULTIPLE_NODES
27struct pglist_data __refdata contig_page_data = {
28 .bdata = &bootmem_node_data[0]
29};
30EXPORT_SYMBOL(contig_page_data);
31#endif
32
26unsigned long max_low_pfn; 33unsigned long max_low_pfn;
27unsigned long min_low_pfn; 34unsigned long min_low_pfn;
28unsigned long max_pfn; 35unsigned long max_pfn;
@@ -35,7 +42,6 @@ unsigned long max_pfn;
35unsigned long saved_max_pfn; 42unsigned long saved_max_pfn;
36#endif 43#endif
37 44
38#ifndef CONFIG_NO_BOOTMEM
39bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata; 45bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
40 46
41static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list); 47static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
@@ -146,7 +152,7 @@ unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
146 min_low_pfn = start; 152 min_low_pfn = start;
147 return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages); 153 return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
148} 154}
149#endif 155
150/* 156/*
151 * free_bootmem_late - free bootmem pages directly to page allocator 157 * free_bootmem_late - free bootmem pages directly to page allocator
152 * @addr: starting address of the range 158 * @addr: starting address of the range
@@ -171,53 +177,6 @@ void __init free_bootmem_late(unsigned long addr, unsigned long size)
171 } 177 }
172} 178}
173 179
174#ifdef CONFIG_NO_BOOTMEM
175static void __init __free_pages_memory(unsigned long start, unsigned long end)
176{
177 int i;
178 unsigned long start_aligned, end_aligned;
179 int order = ilog2(BITS_PER_LONG);
180
181 start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
182 end_aligned = end & ~(BITS_PER_LONG - 1);
183
184 if (end_aligned <= start_aligned) {
185 for (i = start; i < end; i++)
186 __free_pages_bootmem(pfn_to_page(i), 0);
187
188 return;
189 }
190
191 for (i = start; i < start_aligned; i++)
192 __free_pages_bootmem(pfn_to_page(i), 0);
193
194 for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
195 __free_pages_bootmem(pfn_to_page(i), order);
196
197 for (i = end_aligned; i < end; i++)
198 __free_pages_bootmem(pfn_to_page(i), 0);
199}
200
201unsigned long __init free_all_memory_core_early(int nodeid)
202{
203 int i;
204 u64 start, end;
205 unsigned long count = 0;
206 struct range *range = NULL;
207 int nr_range;
208
209 nr_range = get_free_all_memory_range(&range, nodeid);
210
211 for (i = 0; i < nr_range; i++) {
212 start = range[i].start;
213 end = range[i].end;
214 count += end - start;
215 __free_pages_memory(start, end);
216 }
217
218 return count;
219}
220#else
221static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata) 180static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
222{ 181{
223 int aligned; 182 int aligned;
@@ -278,7 +237,6 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
278 237
279 return count; 238 return count;
280} 239}
281#endif
282 240
283/** 241/**
284 * free_all_bootmem_node - release a node's free pages to the buddy allocator 242 * free_all_bootmem_node - release a node's free pages to the buddy allocator
@@ -289,12 +247,7 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
289unsigned long __init free_all_bootmem_node(pg_data_t *pgdat) 247unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
290{ 248{
291 register_page_bootmem_info_node(pgdat); 249 register_page_bootmem_info_node(pgdat);
292#ifdef CONFIG_NO_BOOTMEM
293 /* free_all_memory_core_early(MAX_NUMNODES) will be called later */
294 return 0;
295#else
296 return free_all_bootmem_core(pgdat->bdata); 250 return free_all_bootmem_core(pgdat->bdata);
297#endif
298} 251}
299 252
300/** 253/**
@@ -304,16 +257,6 @@ unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
304 */ 257 */
305unsigned long __init free_all_bootmem(void) 258unsigned long __init free_all_bootmem(void)
306{ 259{
307#ifdef CONFIG_NO_BOOTMEM
308 /*
309 * We need to use MAX_NUMNODES instead of NODE_DATA(0)->node_id
310 * because in some case like Node0 doesnt have RAM installed
311 * low ram will be on Node1
312 * Use MAX_NUMNODES will make sure all ranges in early_node_map[]
313 * will be used instead of only Node0 related
314 */
315 return free_all_memory_core_early(MAX_NUMNODES);
316#else
317 unsigned long total_pages = 0; 260 unsigned long total_pages = 0;
318 bootmem_data_t *bdata; 261 bootmem_data_t *bdata;
319 262
@@ -321,10 +264,8 @@ unsigned long __init free_all_bootmem(void)
321 total_pages += free_all_bootmem_core(bdata); 264 total_pages += free_all_bootmem_core(bdata);
322 265
323 return total_pages; 266 return total_pages;
324#endif
325} 267}
326 268
327#ifndef CONFIG_NO_BOOTMEM
328static void __init __free(bootmem_data_t *bdata, 269static void __init __free(bootmem_data_t *bdata,
329 unsigned long sidx, unsigned long eidx) 270 unsigned long sidx, unsigned long eidx)
330{ 271{
@@ -419,7 +360,6 @@ static int __init mark_bootmem(unsigned long start, unsigned long end,
419 } 360 }
420 BUG(); 361 BUG();
421} 362}
422#endif
423 363
424/** 364/**
425 * free_bootmem_node - mark a page range as usable 365 * free_bootmem_node - mark a page range as usable
@@ -434,10 +374,6 @@ static int __init mark_bootmem(unsigned long start, unsigned long end,
434void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, 374void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
435 unsigned long size) 375 unsigned long size)
436{ 376{
437#ifdef CONFIG_NO_BOOTMEM
438 kmemleak_free_part(__va(physaddr), size);
439 memblock_x86_free_range(physaddr, physaddr + size);
440#else
441 unsigned long start, end; 377 unsigned long start, end;
442 378
443 kmemleak_free_part(__va(physaddr), size); 379 kmemleak_free_part(__va(physaddr), size);
@@ -446,7 +382,6 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
446 end = PFN_DOWN(physaddr + size); 382 end = PFN_DOWN(physaddr + size);
447 383
448 mark_bootmem_node(pgdat->bdata, start, end, 0, 0); 384 mark_bootmem_node(pgdat->bdata, start, end, 0, 0);
449#endif
450} 385}
451 386
452/** 387/**
@@ -460,10 +395,6 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
460 */ 395 */
461void __init free_bootmem(unsigned long addr, unsigned long size) 396void __init free_bootmem(unsigned long addr, unsigned long size)
462{ 397{
463#ifdef CONFIG_NO_BOOTMEM
464 kmemleak_free_part(__va(addr), size);
465 memblock_x86_free_range(addr, addr + size);
466#else
467 unsigned long start, end; 398 unsigned long start, end;
468 399
469 kmemleak_free_part(__va(addr), size); 400 kmemleak_free_part(__va(addr), size);
@@ -472,7 +403,6 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
472 end = PFN_DOWN(addr + size); 403 end = PFN_DOWN(addr + size);
473 404
474 mark_bootmem(start, end, 0, 0); 405 mark_bootmem(start, end, 0, 0);
475#endif
476} 406}
477 407
478/** 408/**
@@ -489,17 +419,12 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
489int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, 419int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
490 unsigned long size, int flags) 420 unsigned long size, int flags)
491{ 421{
492#ifdef CONFIG_NO_BOOTMEM
493 panic("no bootmem");
494 return 0;
495#else
496 unsigned long start, end; 422 unsigned long start, end;
497 423
498 start = PFN_DOWN(physaddr); 424 start = PFN_DOWN(physaddr);
499 end = PFN_UP(physaddr + size); 425 end = PFN_UP(physaddr + size);
500 426
501 return mark_bootmem_node(pgdat->bdata, start, end, 1, flags); 427 return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);
502#endif
503} 428}
504 429
505/** 430/**
@@ -515,20 +440,14 @@ int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
515int __init reserve_bootmem(unsigned long addr, unsigned long size, 440int __init reserve_bootmem(unsigned long addr, unsigned long size,
516 int flags) 441 int flags)
517{ 442{
518#ifdef CONFIG_NO_BOOTMEM
519 panic("no bootmem");
520 return 0;
521#else
522 unsigned long start, end; 443 unsigned long start, end;
523 444
524 start = PFN_DOWN(addr); 445 start = PFN_DOWN(addr);
525 end = PFN_UP(addr + size); 446 end = PFN_UP(addr + size);
526 447
527 return mark_bootmem(start, end, 1, flags); 448 return mark_bootmem(start, end, 1, flags);
528#endif
529} 449}
530 450
531#ifndef CONFIG_NO_BOOTMEM
532int __weak __init reserve_bootmem_generic(unsigned long phys, unsigned long len, 451int __weak __init reserve_bootmem_generic(unsigned long phys, unsigned long len,
533 int flags) 452 int flags)
534{ 453{
@@ -685,33 +604,12 @@ static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata,
685#endif 604#endif
686 return NULL; 605 return NULL;
687} 606}
688#endif
689 607
690static void * __init ___alloc_bootmem_nopanic(unsigned long size, 608static void * __init ___alloc_bootmem_nopanic(unsigned long size,
691 unsigned long align, 609 unsigned long align,
692 unsigned long goal, 610 unsigned long goal,
693 unsigned long limit) 611 unsigned long limit)
694{ 612{
695#ifdef CONFIG_NO_BOOTMEM
696 void *ptr;
697
698 if (WARN_ON_ONCE(slab_is_available()))
699 return kzalloc(size, GFP_NOWAIT);
700
701restart:
702
703 ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit);
704
705 if (ptr)
706 return ptr;
707
708 if (goal != 0) {
709 goal = 0;
710 goto restart;
711 }
712
713 return NULL;
714#else
715 bootmem_data_t *bdata; 613 bootmem_data_t *bdata;
716 void *region; 614 void *region;
717 615
@@ -737,7 +635,6 @@ restart:
737 } 635 }
738 636
739 return NULL; 637 return NULL;
740#endif
741} 638}
742 639
743/** 640/**
@@ -758,10 +655,6 @@ void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
758{ 655{
759 unsigned long limit = 0; 656 unsigned long limit = 0;
760 657
761#ifdef CONFIG_NO_BOOTMEM
762 limit = -1UL;
763#endif
764
765 return ___alloc_bootmem_nopanic(size, align, goal, limit); 658 return ___alloc_bootmem_nopanic(size, align, goal, limit);
766} 659}
767 660
@@ -798,14 +691,9 @@ void * __init __alloc_bootmem(unsigned long size, unsigned long align,
798{ 691{
799 unsigned long limit = 0; 692 unsigned long limit = 0;
800 693
801#ifdef CONFIG_NO_BOOTMEM
802 limit = -1UL;
803#endif
804
805 return ___alloc_bootmem(size, align, goal, limit); 694 return ___alloc_bootmem(size, align, goal, limit);
806} 695}
807 696
808#ifndef CONFIG_NO_BOOTMEM
809static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata, 697static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
810 unsigned long size, unsigned long align, 698 unsigned long size, unsigned long align,
811 unsigned long goal, unsigned long limit) 699 unsigned long goal, unsigned long limit)
@@ -822,7 +710,6 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
822 710
823 return ___alloc_bootmem(size, align, goal, limit); 711 return ___alloc_bootmem(size, align, goal, limit);
824} 712}
825#endif
826 713
827/** 714/**
828 * __alloc_bootmem_node - allocate boot memory from a specific node 715 * __alloc_bootmem_node - allocate boot memory from a specific node
@@ -842,24 +729,10 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
842void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, 729void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
843 unsigned long align, unsigned long goal) 730 unsigned long align, unsigned long goal)
844{ 731{
845 void *ptr;
846
847 if (WARN_ON_ONCE(slab_is_available())) 732 if (WARN_ON_ONCE(slab_is_available()))
848 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 733 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
849 734
850#ifdef CONFIG_NO_BOOTMEM 735 return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
851 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
852 goal, -1ULL);
853 if (ptr)
854 return ptr;
855
856 ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align,
857 goal, -1ULL);
858#else
859 ptr = ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
860#endif
861
862 return ptr;
863} 736}
864 737
865void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size, 738void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
@@ -880,13 +753,8 @@ void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
880 unsigned long new_goal; 753 unsigned long new_goal;
881 754
882 new_goal = MAX_DMA32_PFN << PAGE_SHIFT; 755 new_goal = MAX_DMA32_PFN << PAGE_SHIFT;
883#ifdef CONFIG_NO_BOOTMEM
884 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
885 new_goal, -1ULL);
886#else
887 ptr = alloc_bootmem_core(pgdat->bdata, size, align, 756 ptr = alloc_bootmem_core(pgdat->bdata, size, align,
888 new_goal, 0); 757 new_goal, 0);
889#endif
890 if (ptr) 758 if (ptr)
891 return ptr; 759 return ptr;
892 } 760 }
@@ -907,16 +775,6 @@ void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
907void * __init alloc_bootmem_section(unsigned long size, 775void * __init alloc_bootmem_section(unsigned long size,
908 unsigned long section_nr) 776 unsigned long section_nr)
909{ 777{
910#ifdef CONFIG_NO_BOOTMEM
911 unsigned long pfn, goal, limit;
912
913 pfn = section_nr_to_pfn(section_nr);
914 goal = pfn << PAGE_SHIFT;
915 limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT;
916
917 return __alloc_memory_core_early(early_pfn_to_nid(pfn), size,
918 SMP_CACHE_BYTES, goal, limit);
919#else
920 bootmem_data_t *bdata; 778 bootmem_data_t *bdata;
921 unsigned long pfn, goal, limit; 779 unsigned long pfn, goal, limit;
922 780
@@ -926,7 +784,6 @@ void * __init alloc_bootmem_section(unsigned long size,
926 bdata = &bootmem_node_data[early_pfn_to_nid(pfn)]; 784 bdata = &bootmem_node_data[early_pfn_to_nid(pfn)];
927 785
928 return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit); 786 return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit);
929#endif
930} 787}
931#endif 788#endif
932 789
@@ -938,16 +795,11 @@ void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
938 if (WARN_ON_ONCE(slab_is_available())) 795 if (WARN_ON_ONCE(slab_is_available()))
939 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 796 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
940 797
941#ifdef CONFIG_NO_BOOTMEM
942 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
943 goal, -1ULL);
944#else
945 ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0); 798 ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);
946 if (ptr) 799 if (ptr)
947 return ptr; 800 return ptr;
948 801
949 ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0); 802 ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
950#endif
951 if (ptr) 803 if (ptr)
952 return ptr; 804 return ptr;
953 805
@@ -995,21 +847,9 @@ void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
995void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size, 847void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
996 unsigned long align, unsigned long goal) 848 unsigned long align, unsigned long goal)
997{ 849{
998 void *ptr;
999
1000 if (WARN_ON_ONCE(slab_is_available())) 850 if (WARN_ON_ONCE(slab_is_available()))
1001 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 851 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
1002 852
1003#ifdef CONFIG_NO_BOOTMEM 853 return ___alloc_bootmem_node(pgdat->bdata, size, align,
1004 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
1005 goal, ARCH_LOW_ADDRESS_LIMIT); 854 goal, ARCH_LOW_ADDRESS_LIMIT);
1006 if (ptr)
1007 return ptr;
1008 ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align,
1009 goal, ARCH_LOW_ADDRESS_LIMIT);
1010#else
1011 ptr = ___alloc_bootmem_node(pgdat->bdata, size, align,
1012 goal, ARCH_LOW_ADDRESS_LIMIT);
1013#endif
1014 return ptr;
1015} 855}