aboutsummaryrefslogtreecommitdiffstats
path: root/mm/bootmem.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/bootmem.c')
-rw-r--r--mm/bootmem.c228
1 files changed, 221 insertions, 7 deletions
diff --git a/mm/bootmem.c b/mm/bootmem.c
index 555d5d2731c6..58c66cc5056a 100644
--- a/mm/bootmem.c
+++ b/mm/bootmem.c
@@ -10,9 +10,11 @@
10 */ 10 */
11#include <linux/init.h> 11#include <linux/init.h>
12#include <linux/pfn.h> 12#include <linux/pfn.h>
13#include <linux/slab.h>
13#include <linux/bootmem.h> 14#include <linux/bootmem.h>
14#include <linux/module.h> 15#include <linux/module.h>
15#include <linux/kmemleak.h> 16#include <linux/kmemleak.h>
17#include <linux/range.h>
16 18
17#include <asm/bug.h> 19#include <asm/bug.h>
18#include <asm/io.h> 20#include <asm/io.h>
@@ -32,6 +34,7 @@ unsigned long max_pfn;
32unsigned long saved_max_pfn; 34unsigned long saved_max_pfn;
33#endif 35#endif
34 36
37#ifndef CONFIG_NO_BOOTMEM
35bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata; 38bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
36 39
37static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list); 40static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
@@ -142,7 +145,78 @@ unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
142 min_low_pfn = start; 145 min_low_pfn = start;
143 return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages); 146 return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
144} 147}
148#endif
149/*
150 * free_bootmem_late - free bootmem pages directly to page allocator
151 * @addr: starting address of the range
152 * @size: size of the range in bytes
153 *
154 * This is only useful when the bootmem allocator has already been torn
155 * down, but we are still initializing the system. Pages are given directly
156 * to the page allocator, no bootmem metadata is updated because it is gone.
157 */
158void __init free_bootmem_late(unsigned long addr, unsigned long size)
159{
160 unsigned long cursor, end;
145 161
162 kmemleak_free_part(__va(addr), size);
163
164 cursor = PFN_UP(addr);
165 end = PFN_DOWN(addr + size);
166
167 for (; cursor < end; cursor++) {
168 __free_pages_bootmem(pfn_to_page(cursor), 0);
169 totalram_pages++;
170 }
171}
172
173#ifdef CONFIG_NO_BOOTMEM
174static void __init __free_pages_memory(unsigned long start, unsigned long end)
175{
176 int i;
177 unsigned long start_aligned, end_aligned;
178 int order = ilog2(BITS_PER_LONG);
179
180 start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
181 end_aligned = end & ~(BITS_PER_LONG - 1);
182
183 if (end_aligned <= start_aligned) {
184 for (i = start; i < end; i++)
185 __free_pages_bootmem(pfn_to_page(i), 0);
186
187 return;
188 }
189
190 for (i = start; i < start_aligned; i++)
191 __free_pages_bootmem(pfn_to_page(i), 0);
192
193 for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
194 __free_pages_bootmem(pfn_to_page(i), order);
195
196 for (i = end_aligned; i < end; i++)
197 __free_pages_bootmem(pfn_to_page(i), 0);
198}
199
200unsigned long __init free_all_memory_core_early(int nodeid)
201{
202 int i;
203 u64 start, end;
204 unsigned long count = 0;
205 struct range *range = NULL;
206 int nr_range;
207
208 nr_range = get_free_all_memory_range(&range, nodeid);
209
210 for (i = 0; i < nr_range; i++) {
211 start = range[i].start;
212 end = range[i].end;
213 count += end - start;
214 __free_pages_memory(start, end);
215 }
216
217 return count;
218}
219#else
146static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata) 220static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
147{ 221{
148 int aligned; 222 int aligned;
@@ -203,6 +277,7 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
203 277
204 return count; 278 return count;
205} 279}
280#endif
206 281
207/** 282/**
208 * free_all_bootmem_node - release a node's free pages to the buddy allocator 283 * free_all_bootmem_node - release a node's free pages to the buddy allocator
@@ -213,7 +288,12 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
213unsigned long __init free_all_bootmem_node(pg_data_t *pgdat) 288unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
214{ 289{
215 register_page_bootmem_info_node(pgdat); 290 register_page_bootmem_info_node(pgdat);
291#ifdef CONFIG_NO_BOOTMEM
292 /* free_all_memory_core_early(MAX_NUMNODES) will be called later */
293 return 0;
294#else
216 return free_all_bootmem_core(pgdat->bdata); 295 return free_all_bootmem_core(pgdat->bdata);
296#endif
217} 297}
218 298
219/** 299/**
@@ -223,9 +303,27 @@ unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
223 */ 303 */
224unsigned long __init free_all_bootmem(void) 304unsigned long __init free_all_bootmem(void)
225{ 305{
226 return free_all_bootmem_core(NODE_DATA(0)->bdata); 306#ifdef CONFIG_NO_BOOTMEM
307 /*
308 * We need to use MAX_NUMNODES instead of NODE_DATA(0)->node_id
309 * because in some case like Node0 doesnt have RAM installed
310 * low ram will be on Node1
311 * Use MAX_NUMNODES will make sure all ranges in early_node_map[]
312 * will be used instead of only Node0 related
313 */
314 return free_all_memory_core_early(MAX_NUMNODES);
315#else
316 unsigned long total_pages = 0;
317 bootmem_data_t *bdata;
318
319 list_for_each_entry(bdata, &bdata_list, list)
320 total_pages += free_all_bootmem_core(bdata);
321
322 return total_pages;
323#endif
227} 324}
228 325
326#ifndef CONFIG_NO_BOOTMEM
229static void __init __free(bootmem_data_t *bdata, 327static void __init __free(bootmem_data_t *bdata,
230 unsigned long sidx, unsigned long eidx) 328 unsigned long sidx, unsigned long eidx)
231{ 329{
@@ -320,6 +418,7 @@ static int __init mark_bootmem(unsigned long start, unsigned long end,
320 } 418 }
321 BUG(); 419 BUG();
322} 420}
421#endif
323 422
324/** 423/**
325 * free_bootmem_node - mark a page range as usable 424 * free_bootmem_node - mark a page range as usable
@@ -334,6 +433,9 @@ static int __init mark_bootmem(unsigned long start, unsigned long end,
334void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, 433void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
335 unsigned long size) 434 unsigned long size)
336{ 435{
436#ifdef CONFIG_NO_BOOTMEM
437 free_early(physaddr, physaddr + size);
438#else
337 unsigned long start, end; 439 unsigned long start, end;
338 440
339 kmemleak_free_part(__va(physaddr), size); 441 kmemleak_free_part(__va(physaddr), size);
@@ -342,6 +444,7 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
342 end = PFN_DOWN(physaddr + size); 444 end = PFN_DOWN(physaddr + size);
343 445
344 mark_bootmem_node(pgdat->bdata, start, end, 0, 0); 446 mark_bootmem_node(pgdat->bdata, start, end, 0, 0);
447#endif
345} 448}
346 449
347/** 450/**
@@ -355,6 +458,9 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
355 */ 458 */
356void __init free_bootmem(unsigned long addr, unsigned long size) 459void __init free_bootmem(unsigned long addr, unsigned long size)
357{ 460{
461#ifdef CONFIG_NO_BOOTMEM
462 free_early(addr, addr + size);
463#else
358 unsigned long start, end; 464 unsigned long start, end;
359 465
360 kmemleak_free_part(__va(addr), size); 466 kmemleak_free_part(__va(addr), size);
@@ -363,6 +469,7 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
363 end = PFN_DOWN(addr + size); 469 end = PFN_DOWN(addr + size);
364 470
365 mark_bootmem(start, end, 0, 0); 471 mark_bootmem(start, end, 0, 0);
472#endif
366} 473}
367 474
368/** 475/**
@@ -379,12 +486,17 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
379int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, 486int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
380 unsigned long size, int flags) 487 unsigned long size, int flags)
381{ 488{
489#ifdef CONFIG_NO_BOOTMEM
490 panic("no bootmem");
491 return 0;
492#else
382 unsigned long start, end; 493 unsigned long start, end;
383 494
384 start = PFN_DOWN(physaddr); 495 start = PFN_DOWN(physaddr);
385 end = PFN_UP(physaddr + size); 496 end = PFN_UP(physaddr + size);
386 497
387 return mark_bootmem_node(pgdat->bdata, start, end, 1, flags); 498 return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);
499#endif
388} 500}
389 501
390/** 502/**
@@ -400,16 +512,22 @@ int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
400int __init reserve_bootmem(unsigned long addr, unsigned long size, 512int __init reserve_bootmem(unsigned long addr, unsigned long size,
401 int flags) 513 int flags)
402{ 514{
515#ifdef CONFIG_NO_BOOTMEM
516 panic("no bootmem");
517 return 0;
518#else
403 unsigned long start, end; 519 unsigned long start, end;
404 520
405 start = PFN_DOWN(addr); 521 start = PFN_DOWN(addr);
406 end = PFN_UP(addr + size); 522 end = PFN_UP(addr + size);
407 523
408 return mark_bootmem(start, end, 1, flags); 524 return mark_bootmem(start, end, 1, flags);
525#endif
409} 526}
410 527
411static unsigned long align_idx(struct bootmem_data *bdata, unsigned long idx, 528#ifndef CONFIG_NO_BOOTMEM
412 unsigned long step) 529static unsigned long __init align_idx(struct bootmem_data *bdata,
530 unsigned long idx, unsigned long step)
413{ 531{
414 unsigned long base = bdata->node_min_pfn; 532 unsigned long base = bdata->node_min_pfn;
415 533
@@ -421,8 +539,8 @@ static unsigned long align_idx(struct bootmem_data *bdata, unsigned long idx,
421 return ALIGN(base + idx, step) - base; 539 return ALIGN(base + idx, step) - base;
422} 540}
423 541
424static unsigned long align_off(struct bootmem_data *bdata, unsigned long off, 542static unsigned long __init align_off(struct bootmem_data *bdata,
425 unsigned long align) 543 unsigned long off, unsigned long align)
426{ 544{
427 unsigned long base = PFN_PHYS(bdata->node_min_pfn); 545 unsigned long base = PFN_PHYS(bdata->node_min_pfn);
428 546
@@ -558,12 +676,33 @@ static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata,
558#endif 676#endif
559 return NULL; 677 return NULL;
560} 678}
679#endif
561 680
562static void * __init ___alloc_bootmem_nopanic(unsigned long size, 681static void * __init ___alloc_bootmem_nopanic(unsigned long size,
563 unsigned long align, 682 unsigned long align,
564 unsigned long goal, 683 unsigned long goal,
565 unsigned long limit) 684 unsigned long limit)
566{ 685{
686#ifdef CONFIG_NO_BOOTMEM
687 void *ptr;
688
689 if (WARN_ON_ONCE(slab_is_available()))
690 return kzalloc(size, GFP_NOWAIT);
691
692restart:
693
694 ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit);
695
696 if (ptr)
697 return ptr;
698
699 if (goal != 0) {
700 goal = 0;
701 goto restart;
702 }
703
704 return NULL;
705#else
567 bootmem_data_t *bdata; 706 bootmem_data_t *bdata;
568 void *region; 707 void *region;
569 708
@@ -589,6 +728,7 @@ restart:
589 } 728 }
590 729
591 return NULL; 730 return NULL;
731#endif
592} 732}
593 733
594/** 734/**
@@ -607,7 +747,13 @@ restart:
607void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align, 747void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
608 unsigned long goal) 748 unsigned long goal)
609{ 749{
610 return ___alloc_bootmem_nopanic(size, align, goal, 0); 750 unsigned long limit = 0;
751
752#ifdef CONFIG_NO_BOOTMEM
753 limit = -1UL;
754#endif
755
756 return ___alloc_bootmem_nopanic(size, align, goal, limit);
611} 757}
612 758
613static void * __init ___alloc_bootmem(unsigned long size, unsigned long align, 759static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
@@ -641,9 +787,16 @@ static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
641void * __init __alloc_bootmem(unsigned long size, unsigned long align, 787void * __init __alloc_bootmem(unsigned long size, unsigned long align,
642 unsigned long goal) 788 unsigned long goal)
643{ 789{
644 return ___alloc_bootmem(size, align, goal, 0); 790 unsigned long limit = 0;
791
792#ifdef CONFIG_NO_BOOTMEM
793 limit = -1UL;
794#endif
795
796 return ___alloc_bootmem(size, align, goal, limit);
645} 797}
646 798
799#ifndef CONFIG_NO_BOOTMEM
647static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata, 800static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
648 unsigned long size, unsigned long align, 801 unsigned long size, unsigned long align,
649 unsigned long goal, unsigned long limit) 802 unsigned long goal, unsigned long limit)
@@ -660,6 +813,7 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
660 813
661 return ___alloc_bootmem(size, align, goal, limit); 814 return ___alloc_bootmem(size, align, goal, limit);
662} 815}
816#endif
663 817
664/** 818/**
665 * __alloc_bootmem_node - allocate boot memory from a specific node 819 * __alloc_bootmem_node - allocate boot memory from a specific node
@@ -682,7 +836,46 @@ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
682 if (WARN_ON_ONCE(slab_is_available())) 836 if (WARN_ON_ONCE(slab_is_available()))
683 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 837 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
684 838
839#ifdef CONFIG_NO_BOOTMEM
840 return __alloc_memory_core_early(pgdat->node_id, size, align,
841 goal, -1ULL);
842#else
685 return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0); 843 return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
844#endif
845}
846
847void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
848 unsigned long align, unsigned long goal)
849{
850#ifdef MAX_DMA32_PFN
851 unsigned long end_pfn;
852
853 if (WARN_ON_ONCE(slab_is_available()))
854 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
855
856 /* update goal according ...MAX_DMA32_PFN */
857 end_pfn = pgdat->node_start_pfn + pgdat->node_spanned_pages;
858
859 if (end_pfn > MAX_DMA32_PFN + (128 >> (20 - PAGE_SHIFT)) &&
860 (goal >> PAGE_SHIFT) < MAX_DMA32_PFN) {
861 void *ptr;
862 unsigned long new_goal;
863
864 new_goal = MAX_DMA32_PFN << PAGE_SHIFT;
865#ifdef CONFIG_NO_BOOTMEM
866 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
867 new_goal, -1ULL);
868#else
869 ptr = alloc_bootmem_core(pgdat->bdata, size, align,
870 new_goal, 0);
871#endif
872 if (ptr)
873 return ptr;
874 }
875#endif
876
877 return __alloc_bootmem_node(pgdat, size, align, goal);
878
686} 879}
687 880
688#ifdef CONFIG_SPARSEMEM 881#ifdef CONFIG_SPARSEMEM
@@ -696,6 +889,16 @@ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
696void * __init alloc_bootmem_section(unsigned long size, 889void * __init alloc_bootmem_section(unsigned long size,
697 unsigned long section_nr) 890 unsigned long section_nr)
698{ 891{
892#ifdef CONFIG_NO_BOOTMEM
893 unsigned long pfn, goal, limit;
894
895 pfn = section_nr_to_pfn(section_nr);
896 goal = pfn << PAGE_SHIFT;
897 limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT;
898
899 return __alloc_memory_core_early(early_pfn_to_nid(pfn), size,
900 SMP_CACHE_BYTES, goal, limit);
901#else
699 bootmem_data_t *bdata; 902 bootmem_data_t *bdata;
700 unsigned long pfn, goal, limit; 903 unsigned long pfn, goal, limit;
701 904
@@ -705,6 +908,7 @@ void * __init alloc_bootmem_section(unsigned long size,
705 bdata = &bootmem_node_data[early_pfn_to_nid(pfn)]; 908 bdata = &bootmem_node_data[early_pfn_to_nid(pfn)];
706 909
707 return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit); 910 return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit);
911#endif
708} 912}
709#endif 913#endif
710 914
@@ -716,11 +920,16 @@ void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
716 if (WARN_ON_ONCE(slab_is_available())) 920 if (WARN_ON_ONCE(slab_is_available()))
717 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 921 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
718 922
923#ifdef CONFIG_NO_BOOTMEM
924 ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
925 goal, -1ULL);
926#else
719 ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0); 927 ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);
720 if (ptr) 928 if (ptr)
721 return ptr; 929 return ptr;
722 930
723 ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0); 931 ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
932#endif
724 if (ptr) 933 if (ptr)
725 return ptr; 934 return ptr;
726 935
@@ -771,6 +980,11 @@ void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
771 if (WARN_ON_ONCE(slab_is_available())) 980 if (WARN_ON_ONCE(slab_is_available()))
772 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); 981 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
773 982
983#ifdef CONFIG_NO_BOOTMEM
984 return __alloc_memory_core_early(pgdat->node_id, size, align,
985 goal, ARCH_LOW_ADDRESS_LIMIT);
986#else
774 return ___alloc_bootmem_node(pgdat->bdata, size, align, 987 return ___alloc_bootmem_node(pgdat->bdata, size, align,
775 goal, ARCH_LOW_ADDRESS_LIMIT); 988 goal, ARCH_LOW_ADDRESS_LIMIT);
989#endif
776} 990}