1 files changed, 179 insertions, 3 deletions
diff --git a/mm/bootmem.c b/mm/bootmem.c
index 7d1486875e1c..9b134460b016 100644
--- a/mm/bootmem.c
+++ b/mm/bootmem.c
@@ -13,6 +13,7 @@
 #include <linux/bootmem.h>
 #include <linux/module.h>
 #include <linux/kmemleak.h>
+#include <linux/range.h>
 #include <asm/bug.h>
 #include <asm/io.h>
@@ -32,6 +33,7 @@ unsigned long max_pfn;
 unsigned long saved_max_pfn;
 #endif
+#ifndef CONFIG_NO_BOOTMEM
 bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
 static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
@@ -142,7 +144,7 @@ unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
        min_low_pfn = start;
        return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
 }
+#endif
 /*
 * free_bootmem_late - free bootmem pages directly to page allocator
 * @addr: starting address of the range
@@ -167,6 +169,53 @@ void __init free_bootmem_late(unsigned long addr, unsigned long size)
        }
 }
+#ifdef CONFIG_NO_BOOTMEM
+static void __init __free_pages_memory(unsigned long start, unsigned long end)
+{
+        int i;
+        unsigned long start_aligned, end_aligned;
+        int order = ilog2(BITS_PER_LONG);
+        start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
+        end_aligned = end & ~(BITS_PER_LONG - 1);
+        if (end_aligned <= start_aligned) {
+                for (i = start; i < end; i++)
+                        __free_pages_bootmem(pfn_to_page(i), 0);
+                return;
+        }
+        for (i = start; i < start_aligned; i++)
+                __free_pages_bootmem(pfn_to_page(i), 0);
+        for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
+                __free_pages_bootmem(pfn_to_page(i), order);
+        for (i = end_aligned; i < end; i++)
+                __free_pages_bootmem(pfn_to_page(i), 0);
+}
+unsigned long __init free_all_memory_core_early(int nodeid)
+{
+        int i;
+        u64 start, end;
+        unsigned long count = 0;
+        struct range *range = NULL;
+        int nr_range;
+        nr_range = get_free_all_memory_range(&range, nodeid);
+        for (i = 0; i < nr_range; i++) {
+                start = range[i].start;
+                end = range[i].end;
+                count += end - start;
+                __free_pages_memory(start, end);
+        }
+        return count;
+}
+#else
 static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
 {
        int aligned;
@@ -227,6 +276,7 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
        return count;
 }
+#endif
 /**
 * free_all_bootmem_node - release a node's free pages to the buddy allocator
@@ -237,7 +287,12 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
 unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
 {
        register_page_bootmem_info_node(pgdat);
+#ifdef CONFIG_NO_BOOTMEM
+        /* free_all_memory_core_early(MAX_NUMNODES) will be called later */
+        return 0;
+#else
        return free_all_bootmem_core(pgdat->bdata);
+#endif
 }
 /**
@@ -247,9 +302,14 @@ unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
 */
 unsigned long __init free_all_bootmem(void)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        return free_all_memory_core_early(NODE_DATA(0)->node_id);
+#else
        return free_all_bootmem_core(NODE_DATA(0)->bdata);
+#endif
 }
+#ifndef CONFIG_NO_BOOTMEM
 static void __init __free(bootmem_data_t *bdata,
                        unsigned long sidx, unsigned long eidx)
 {
@@ -344,6 +404,7 @@ static int __init mark_bootmem(unsigned long start, unsigned long end,
        }
        BUG();
 }
+#endif
 /**
 * free_bootmem_node - mark a page range as usable
@@ -358,6 +419,9 @@ static int __init mark_bootmem(unsigned long start, unsigned long end,
 void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
                              unsigned long size)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        free_early(physaddr, physaddr + size);
+#else
        unsigned long start, end;
        kmemleak_free_part(__va(physaddr), size);
@@ -366,6 +430,7 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
        end = PFN_DOWN(physaddr + size);
        mark_bootmem_node(pgdat->bdata, start, end, 0, 0);
+#endif
 }
 /**
@@ -379,6 +444,9 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
 */
 void __init free_bootmem(unsigned long addr, unsigned long size)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        free_early(addr, addr + size);
+#else
        unsigned long start, end;
        kmemleak_free_part(__va(addr), size);
@@ -387,6 +455,7 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
        end = PFN_DOWN(addr + size);
        mark_bootmem(start, end, 0, 0);
+#endif
 }
 /**
@@ -403,12 +472,17 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
 int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
                                 unsigned long size, int flags)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        panic("no bootmem");
+        return 0;
+#else
        unsigned long start, end;
        start = PFN_DOWN(physaddr);
        end = PFN_UP(physaddr + size);
        return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);
+#endif
 }
 /**
@@ -424,14 +498,20 @@ int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
 int __init reserve_bootmem(unsigned long addr, unsigned long size,
                            int flags)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        panic("no bootmem");
+        return 0;
+#else
        unsigned long start, end;
        start = PFN_DOWN(addr);
        end = PFN_UP(addr + size);
        return mark_bootmem(start, end, 1, flags);
+#endif
 }
+#ifndef CONFIG_NO_BOOTMEM
 static unsigned long __init align_idx(struct bootmem_data *bdata,
                                      unsigned long idx, unsigned long step)
 {
@@ -582,12 +662,33 @@ static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata,
 #endif
        return NULL;
 }
+#endif
 static void * __init ___alloc_bootmem_nopanic(unsigned long size,
                                        unsigned long align,
                                        unsigned long goal,
                                        unsigned long limit)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        void *ptr;
+        if (WARN_ON_ONCE(slab_is_available()))
+                return kzalloc(size, GFP_NOWAIT);
+restart:
+        ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit);
+        if (ptr)
+                return ptr;
+        if (goal != 0) {
+                goal = 0;
+                goto restart;
+        }
+        return NULL;
+#else
        bootmem_data_t *bdata;
        void *region;
@@ -613,6 +714,7 @@ restart:
        }
        return NULL;
+#endif
 }
 /**
@@ -631,7 +733,13 @@ restart:
 void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
                                        unsigned long goal)
 {
-        return ___alloc_bootmem_nopanic(size, align, goal, 0);
+        unsigned long limit = 0;
+#ifdef CONFIG_NO_BOOTMEM
+        limit = -1UL;
+#endif
+        return ___alloc_bootmem_nopanic(size, align, goal, limit);
 }
 static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
@@ -665,9 +773,16 @@ static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
 void * __init __alloc_bootmem(unsigned long size, unsigned long align,
                              unsigned long goal)
 {
-        return ___alloc_bootmem(size, align, goal, 0);
+        unsigned long limit = 0;
+#ifdef CONFIG_NO_BOOTMEM
+        limit = -1UL;
+#endif
+        return ___alloc_bootmem(size, align, goal, limit);
 }
+#ifndef CONFIG_NO_BOOTMEM
 static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
                                unsigned long size, unsigned long align,
                                unsigned long goal, unsigned long limit)
@@ -684,6 +799,7 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
        return ___alloc_bootmem(size, align, goal, limit);
 }
+#endif
 /**
 * __alloc_bootmem_node - allocate boot memory from a specific node
@@ -706,7 +822,46 @@ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
        if (WARN_ON_ONCE(slab_is_available()))
                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+#ifdef CONFIG_NO_BOOTMEM
+        return __alloc_memory_core_early(pgdat->node_id, size, align,
+                                         goal, -1ULL);
+#else
        return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
+#endif
+}
+void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
+                                   unsigned long align, unsigned long goal)
+{
+#ifdef MAX_DMA32_PFN
+        unsigned long end_pfn;
+        if (WARN_ON_ONCE(slab_is_available()))
+                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+        /* update goal according ...MAX_DMA32_PFN */
+        end_pfn = pgdat->node_start_pfn + pgdat->node_spanned_pages;
+        if (end_pfn > MAX_DMA32_PFN + (128 >> (20 - PAGE_SHIFT)) &&
+            (goal >> PAGE_SHIFT) < MAX_DMA32_PFN) {
+                void *ptr;
+                unsigned long new_goal;
+                new_goal = MAX_DMA32_PFN << PAGE_SHIFT;
+#ifdef CONFIG_NO_BOOTMEM
+                ptr =  __alloc_memory_core_early(pgdat->node_id, size, align,
+                                                 new_goal, -1ULL);
+#else
+                ptr = alloc_bootmem_core(pgdat->bdata, size, align,
+                                                 new_goal, 0);
+#endif
+                if (ptr)
+                        return ptr;
+        }
+#endif
+        return __alloc_bootmem_node(pgdat, size, align, goal);
 }
 #ifdef CONFIG_SPARSEMEM
@@ -720,6 +875,16 @@ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
 void * __init alloc_bootmem_section(unsigned long size,
                                    unsigned long section_nr)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        unsigned long pfn, goal, limit;
+        pfn = section_nr_to_pfn(section_nr);
+        goal = pfn << PAGE_SHIFT;
+        limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT;
+        return __alloc_memory_core_early(early_pfn_to_nid(pfn), size,
+                                         SMP_CACHE_BYTES, goal, limit);
+#else
        bootmem_data_t *bdata;
        unsigned long pfn, goal, limit;
@@ -729,6 +894,7 @@ void * __init alloc_bootmem_section(unsigned long size,
        bdata = &bootmem_node_data[early_pfn_to_nid(pfn)];
        return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit);
+#endif
 }
 #endif
@@ -740,11 +906,16 @@ void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
        if (WARN_ON_ONCE(slab_is_available()))
                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+#ifdef CONFIG_NO_BOOTMEM
+        ptr =  __alloc_memory_core_early(pgdat->node_id, size, align,
+                                                 goal, -1ULL);
+#else
        ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);
        if (ptr)
                return ptr;
        ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
+#endif
        if (ptr)
                return ptr;
@@ -795,6 +966,11 @@ void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
        if (WARN_ON_ONCE(slab_is_available()))
                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+#ifdef CONFIG_NO_BOOTMEM
+        return __alloc_memory_core_early(pgdat->node_id, size, align,
+                                goal, ARCH_LOW_ADDRESS_LIMIT);
+#else
        return ___alloc_bootmem_node(pgdat->bdata, size, align,
                                goal, ARCH_LOW_ADDRESS_LIMIT);
+#endif
 }

diff --git a/mm/bootmem.c b/mm/bootmem.c index 7d1486875e1c..9b134460b016 100644 --- a/mm/bootmem.c +++ b/mm/bootmem.c
@@ -13,6 +13,7 @@
13	#include <linux/bootmem.h>	13	#include <linux/bootmem.h>
14	#include <linux/module.h>	14	#include <linux/module.h>
15	#include <linux/kmemleak.h>	15	#include <linux/kmemleak.h>
		16	#include <linux/range.h>
16		17
17	#include <asm/bug.h>	18	#include <asm/bug.h>
18	#include <asm/io.h>	19	#include <asm/io.h>
@@ -32,6 +33,7 @@ unsigned long max_pfn;
32	unsigned long saved_max_pfn;	33	unsigned long saved_max_pfn;
33	#endif	34	#endif
34		35
		36	#ifndef CONFIG_NO_BOOTMEM
35	bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;	37	bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
36		38
37	static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);	39	static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
@@ -142,7 +144,7 @@ unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
142	min_low_pfn = start;	144	min_low_pfn = start;
143	return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);	145	return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
144	}	146	}
145		147	#endif
146	/*	148	/*
147	* free_bootmem_late - free bootmem pages directly to page allocator	149	* free_bootmem_late - free bootmem pages directly to page allocator
148	* @addr: starting address of the range	150	* @addr: starting address of the range
@@ -167,6 +169,53 @@ void __init free_bootmem_late(unsigned long addr, unsigned long size)
167	}	169	}
168	}	170	}
169		171
		172	#ifdef CONFIG_NO_BOOTMEM
		173	static void __init __free_pages_memory(unsigned long start, unsigned long end)
		174	{
		175	int i;
		176	unsigned long start_aligned, end_aligned;
		177	int order = ilog2(BITS_PER_LONG);
		178
		179	start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
		180	end_aligned = end & ~(BITS_PER_LONG - 1);
		181
		182	if (end_aligned <= start_aligned) {
		183	for (i = start; i < end; i++)
		184	__free_pages_bootmem(pfn_to_page(i), 0);
		185
		186	return;
		187	}
		188
		189	for (i = start; i < start_aligned; i++)
		190	__free_pages_bootmem(pfn_to_page(i), 0);
		191
		192	for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
		193	__free_pages_bootmem(pfn_to_page(i), order);
		194
		195	for (i = end_aligned; i < end; i++)
		196	__free_pages_bootmem(pfn_to_page(i), 0);
		197	}
		198
		199	unsigned long __init free_all_memory_core_early(int nodeid)
		200	{
		201	int i;
		202	u64 start, end;
		203	unsigned long count = 0;
		204	struct range *range = NULL;
		205	int nr_range;
		206
		207	nr_range = get_free_all_memory_range(&range, nodeid);
		208
		209	for (i = 0; i < nr_range; i++) {
		210	start = range[i].start;
		211	end = range[i].end;
		212	count += end - start;
		213	__free_pages_memory(start, end);
		214	}
		215
		216	return count;
		217	}
		218	#else
170	static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)	219	static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
171	{	220	{
172	int aligned;	221	int aligned;
@@ -227,6 +276,7 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
227		276
228	return count;	277	return count;
229	}	278	}
		279	#endif
230		280
231	/**	281	/**
232	* free_all_bootmem_node - release a node's free pages to the buddy allocator	282	* free_all_bootmem_node - release a node's free pages to the buddy allocator
@@ -237,7 +287,12 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
237	unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)	287	unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
238	{	288	{
239	register_page_bootmem_info_node(pgdat);	289	register_page_bootmem_info_node(pgdat);
		290	#ifdef CONFIG_NO_BOOTMEM
		291	/* free_all_memory_core_early(MAX_NUMNODES) will be called later */
		292	return 0;
		293	#else
240	return free_all_bootmem_core(pgdat->bdata);	294	return free_all_bootmem_core(pgdat->bdata);
		295	#endif
241	}	296	}
242		297
243	/**	298	/**
@@ -247,9 +302,14 @@ unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
247	*/	302	*/
248	unsigned long __init free_all_bootmem(void)	303	unsigned long __init free_all_bootmem(void)
249	{	304	{
		305	#ifdef CONFIG_NO_BOOTMEM
		306	return free_all_memory_core_early(NODE_DATA(0)->node_id);
		307	#else
250	return free_all_bootmem_core(NODE_DATA(0)->bdata);	308	return free_all_bootmem_core(NODE_DATA(0)->bdata);
		309	#endif
251	}	310	}
252		311
		312	#ifndef CONFIG_NO_BOOTMEM
253	static void __init __free(bootmem_data_t *bdata,	313	static void __init __free(bootmem_data_t *bdata,
254	unsigned long sidx, unsigned long eidx)	314	unsigned long sidx, unsigned long eidx)
255	{	315	{
@@ -344,6 +404,7 @@ static int __init mark_bootmem(unsigned long start, unsigned long end,
344	}	404	}
345	BUG();	405	BUG();
346	}	406	}
		407	#endif
347		408
348	/**	409	/**
349	* free_bootmem_node - mark a page range as usable	410	* free_bootmem_node - mark a page range as usable
@@ -358,6 +419,9 @@ static int __init mark_bootmem(unsigned long start, unsigned long end,
358	void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,	419	void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
359	unsigned long size)	420	unsigned long size)
360	{	421	{
		422	#ifdef CONFIG_NO_BOOTMEM
		423	free_early(physaddr, physaddr + size);
		424	#else
361	unsigned long start, end;	425	unsigned long start, end;
362		426
363	kmemleak_free_part(__va(physaddr), size);	427	kmemleak_free_part(__va(physaddr), size);
@@ -366,6 +430,7 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
366	end = PFN_DOWN(physaddr + size);	430	end = PFN_DOWN(physaddr + size);
367		431
368	mark_bootmem_node(pgdat->bdata, start, end, 0, 0);	432	mark_bootmem_node(pgdat->bdata, start, end, 0, 0);
		433	#endif
369	}	434	}
370		435
371	/**	436	/**
@@ -379,6 +444,9 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
379	*/	444	*/
380	void __init free_bootmem(unsigned long addr, unsigned long size)	445	void __init free_bootmem(unsigned long addr, unsigned long size)
381	{	446	{
		447	#ifdef CONFIG_NO_BOOTMEM
		448	free_early(addr, addr + size);
		449	#else
382	unsigned long start, end;	450	unsigned long start, end;
383		451
384	kmemleak_free_part(__va(addr), size);	452	kmemleak_free_part(__va(addr), size);
@@ -387,6 +455,7 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
387	end = PFN_DOWN(addr + size);	455	end = PFN_DOWN(addr + size);
388		456
389	mark_bootmem(start, end, 0, 0);	457	mark_bootmem(start, end, 0, 0);
		458	#endif
390	}	459	}
391		460
392	/**	461	/**
@@ -403,12 +472,17 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
403	int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,	472	int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
404	unsigned long size, int flags)	473	unsigned long size, int flags)
405	{	474	{
		475	#ifdef CONFIG_NO_BOOTMEM
		476	panic("no bootmem");
		477	return 0;
		478	#else
406	unsigned long start, end;	479	unsigned long start, end;
407		480
408	start = PFN_DOWN(physaddr);	481	start = PFN_DOWN(physaddr);
409	end = PFN_UP(physaddr + size);	482	end = PFN_UP(physaddr + size);
410		483
411	return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);	484	return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);
		485	#endif
412	}	486	}
413		487
414	/**	488	/**
@@ -424,14 +498,20 @@ int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
424	int __init reserve_bootmem(unsigned long addr, unsigned long size,	498	int __init reserve_bootmem(unsigned long addr, unsigned long size,
425	int flags)	499	int flags)
426	{	500	{
		501	#ifdef CONFIG_NO_BOOTMEM
		502	panic("no bootmem");
		503	return 0;
		504	#else
427	unsigned long start, end;	505	unsigned long start, end;
428		506
429	start = PFN_DOWN(addr);	507	start = PFN_DOWN(addr);
430	end = PFN_UP(addr + size);	508	end = PFN_UP(addr + size);
431		509
432	return mark_bootmem(start, end, 1, flags);	510	return mark_bootmem(start, end, 1, flags);
		511	#endif
433	}	512	}
434		513
		514	#ifndef CONFIG_NO_BOOTMEM
435	static unsigned long __init align_idx(struct bootmem_data *bdata,	515	static unsigned long __init align_idx(struct bootmem_data *bdata,
436	unsigned long idx, unsigned long step)	516	unsigned long idx, unsigned long step)
437	{	517	{
@@ -582,12 +662,33 @@ static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata,
582	#endif	662	#endif
583	return NULL;	663	return NULL;
584	}	664	}
		665	#endif
585		666
586	static void * __init ___alloc_bootmem_nopanic(unsigned long size,	667	static void * __init ___alloc_bootmem_nopanic(unsigned long size,
587	unsigned long align,	668	unsigned long align,
588	unsigned long goal,	669	unsigned long goal,
589	unsigned long limit)	670	unsigned long limit)
590	{	671	{
		672	#ifdef CONFIG_NO_BOOTMEM
		673	void *ptr;
		674
		675	if (WARN_ON_ONCE(slab_is_available()))
		676	return kzalloc(size, GFP_NOWAIT);
		677
		678	restart:
		679
		680	ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit);
		681
		682	if (ptr)
		683	return ptr;
		684
		685	if (goal != 0) {
		686	goal = 0;
		687	goto restart;
		688	}
		689
		690	return NULL;
		691	#else
591	bootmem_data_t *bdata;	692	bootmem_data_t *bdata;
592	void *region;	693	void *region;
593		694
@@ -613,6 +714,7 @@ restart:
613	}	714	}
614		715
615	return NULL;	716	return NULL;
		717	#endif
616	}	718	}
617		719
618	/**	720	/**
@@ -631,7 +733,13 @@ restart:
631	void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,	733	void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
632	unsigned long goal)	734	unsigned long goal)
633	{	735	{
634	return ___alloc_bootmem_nopanic(size, align, goal, 0);	736	unsigned long limit = 0;
		737
		738	#ifdef CONFIG_NO_BOOTMEM
		739	limit = -1UL;
		740	#endif
		741
		742	return ___alloc_bootmem_nopanic(size, align, goal, limit);
635	}	743	}
636		744
637	static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,	745	static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
@@ -665,9 +773,16 @@ static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
665	void * __init __alloc_bootmem(unsigned long size, unsigned long align,	773	void * __init __alloc_bootmem(unsigned long size, unsigned long align,
666	unsigned long goal)	774	unsigned long goal)
667	{	775	{
668	return ___alloc_bootmem(size, align, goal, 0);	776	unsigned long limit = 0;
		777
		778	#ifdef CONFIG_NO_BOOTMEM
		779	limit = -1UL;
		780	#endif
		781
		782	return ___alloc_bootmem(size, align, goal, limit);
669	}	783	}
670		784
		785	#ifndef CONFIG_NO_BOOTMEM
671	static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,	786	static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
672	unsigned long size, unsigned long align,	787	unsigned long size, unsigned long align,
673	unsigned long goal, unsigned long limit)	788	unsigned long goal, unsigned long limit)
@@ -684,6 +799,7 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
684		799
685	return ___alloc_bootmem(size, align, goal, limit);	800	return ___alloc_bootmem(size, align, goal, limit);
686	}	801	}
		802	#endif
687		803
688	/**	804	/**
689	* __alloc_bootmem_node - allocate boot memory from a specific node	805	* __alloc_bootmem_node - allocate boot memory from a specific node
@@ -706,7 +822,46 @@ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
706	if (WARN_ON_ONCE(slab_is_available()))	822	if (WARN_ON_ONCE(slab_is_available()))
707	return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);	823	return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
708		824
		825	#ifdef CONFIG_NO_BOOTMEM
		826	return __alloc_memory_core_early(pgdat->node_id, size, align,
		827	goal, -1ULL);
		828	#else
709	return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);	829	return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
		830	#endif
		831	}
		832
		833	void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
		834	unsigned long align, unsigned long goal)
		835	{
		836	#ifdef MAX_DMA32_PFN
		837	unsigned long end_pfn;
		838
		839	if (WARN_ON_ONCE(slab_is_available()))
		840	return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
		841
		842	/* update goal according ...MAX_DMA32_PFN */
		843	end_pfn = pgdat->node_start_pfn + pgdat->node_spanned_pages;
		844
		845	if (end_pfn > MAX_DMA32_PFN + (128 >> (20 - PAGE_SHIFT)) &&
		846	(goal >> PAGE_SHIFT) < MAX_DMA32_PFN) {
		847	void *ptr;
		848	unsigned long new_goal;
		849
		850	new_goal = MAX_DMA32_PFN << PAGE_SHIFT;
		851	#ifdef CONFIG_NO_BOOTMEM
		852	ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
		853	new_goal, -1ULL);
		854	#else
		855	ptr = alloc_bootmem_core(pgdat->bdata, size, align,
		856	new_goal, 0);
		857	#endif
		858	if (ptr)
		859	return ptr;
		860	}
		861	#endif
		862
		863	return __alloc_bootmem_node(pgdat, size, align, goal);
		864
710	}	865	}
711		866
712	#ifdef CONFIG_SPARSEMEM	867	#ifdef CONFIG_SPARSEMEM
@@ -720,6 +875,16 @@ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
720	void * __init alloc_bootmem_section(unsigned long size,	875	void * __init alloc_bootmem_section(unsigned long size,
721	unsigned long section_nr)	876	unsigned long section_nr)
722	{	877	{
		878	#ifdef CONFIG_NO_BOOTMEM
		879	unsigned long pfn, goal, limit;
		880
		881	pfn = section_nr_to_pfn(section_nr);
		882	goal = pfn << PAGE_SHIFT;
		883	limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT;
		884
		885	return __alloc_memory_core_early(early_pfn_to_nid(pfn), size,
		886	SMP_CACHE_BYTES, goal, limit);
		887	#else
723	bootmem_data_t *bdata;	888	bootmem_data_t *bdata;
724	unsigned long pfn, goal, limit;	889	unsigned long pfn, goal, limit;
725		890
@@ -729,6 +894,7 @@ void * __init alloc_bootmem_section(unsigned long size,
729	bdata = &bootmem_node_data[early_pfn_to_nid(pfn)];	894	bdata = &bootmem_node_data[early_pfn_to_nid(pfn)];
730		895
731	return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit);	896	return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit);
		897	#endif
732	}	898	}
733	#endif	899	#endif
734		900
@@ -740,11 +906,16 @@ void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
740	if (WARN_ON_ONCE(slab_is_available()))	906	if (WARN_ON_ONCE(slab_is_available()))
741	return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);	907	return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
742		908
		909	#ifdef CONFIG_NO_BOOTMEM
		910	ptr = __alloc_memory_core_early(pgdat->node_id, size, align,
		911	goal, -1ULL);
		912	#else
743	ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);	913	ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);
744	if (ptr)	914	if (ptr)
745	return ptr;	915	return ptr;
746		916
747	ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);	917	ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
		918	#endif
748	if (ptr)	919	if (ptr)
749	return ptr;	920	return ptr;
750		921
@@ -795,6 +966,11 @@ void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
795	if (WARN_ON_ONCE(slab_is_available()))	966	if (WARN_ON_ONCE(slab_is_available()))
796	return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);	967	return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
797		968
		969	#ifdef CONFIG_NO_BOOTMEM
		970	return __alloc_memory_core_early(pgdat->node_id, size, align,
		971	goal, ARCH_LOW_ADDRESS_LIMIT);
		972	#else
798	return ___alloc_bootmem_node(pgdat->bdata, size, align,	973	return ___alloc_bootmem_node(pgdat->bdata, size, align,
799	goal, ARCH_LOW_ADDRESS_LIMIT);	974	goal, ARCH_LOW_ADDRESS_LIMIT);
		975	#endif
800	}	976	}