aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm/mm/init.c37
-rw-r--r--arch/arm/plat-omap/fb.c6
-rw-r--r--arch/microblaze/include/asm/memblock.h3
-rw-r--r--arch/microblaze/mm/init.c30
-rw-r--r--arch/powerpc/include/asm/memblock.h7
-rw-r--r--arch/powerpc/include/asm/mmu.h12
-rw-r--r--arch/powerpc/kernel/head_40x.S6
-rw-r--r--arch/powerpc/kernel/paca.c2
-rw-r--r--arch/powerpc/kernel/prom.c15
-rw-r--r--arch/powerpc/kernel/rtas.c2
-rw-r--r--arch/powerpc/kernel/setup_32.c2
-rw-r--r--arch/powerpc/kernel/setup_64.c2
-rw-r--r--arch/powerpc/mm/40x_mmu.c17
-rw-r--r--arch/powerpc/mm/44x_mmu.c14
-rw-r--r--arch/powerpc/mm/fsl_booke_mmu.c12
-rw-r--r--arch/powerpc/mm/hash_utils_64.c35
-rw-r--r--arch/powerpc/mm/init_32.c43
-rw-r--r--arch/powerpc/mm/init_64.c1
-rw-r--r--arch/powerpc/mm/mem.c94
-rw-r--r--arch/powerpc/mm/numa.c17
-rw-r--r--arch/powerpc/mm/ppc_mmu_32.c18
-rw-r--r--arch/powerpc/mm/tlb_nohash.c16
-rw-r--r--arch/powerpc/platforms/embedded6xx/wii.c2
-rw-r--r--arch/sh/include/asm/memblock.h2
-rw-r--r--arch/sh/mm/init.c17
-rw-r--r--arch/sparc/include/asm/memblock.h2
-rw-r--r--arch/sparc/mm/init_64.c46
-rw-r--r--arch/x86/Kconfig15
-rw-r--r--arch/x86/include/asm/e820.h20
-rw-r--r--arch/x86/include/asm/efi.h2
-rw-r--r--arch/x86/include/asm/io.h1
-rw-r--r--arch/x86/include/asm/memblock.h23
-rw-r--r--arch/x86/kernel/acpi/sleep.c9
-rw-r--r--arch/x86/kernel/apic/numaq_32.c3
-rw-r--r--arch/x86/kernel/check.c16
-rw-r--r--arch/x86/kernel/e820.c191
-rw-r--r--arch/x86/kernel/efi.c5
-rw-r--r--arch/x86/kernel/head.c3
-rw-r--r--arch/x86/kernel/head32.c10
-rw-r--r--arch/x86/kernel/head64.c9
-rw-r--r--arch/x86/kernel/mpparse.c5
-rw-r--r--arch/x86/kernel/setup.c88
-rw-r--r--arch/x86/kernel/setup_percpu.c6
-rw-r--r--arch/x86/kernel/trampoline.c10
-rw-r--r--arch/x86/mm/Makefile2
-rw-r--r--arch/x86/mm/init.c10
-rw-r--r--arch/x86/mm/init_32.c119
-rw-r--r--arch/x86/mm/init_64.c67
-rw-r--r--arch/x86/mm/ioremap.c5
-rw-r--r--arch/x86/mm/k8topology_64.c4
-rw-r--r--arch/x86/mm/memblock.c348
-rw-r--r--arch/x86/mm/memtest.c7
-rw-r--r--arch/x86/mm/numa_32.c30
-rw-r--r--arch/x86/mm/numa_64.c84
-rw-r--r--arch/x86/mm/srat_32.c3
-rw-r--r--arch/x86/mm/srat_64.c11
-rw-r--r--arch/x86/xen/enlighten.c3
-rw-r--r--arch/x86/xen/mmu.c31
-rw-r--r--arch/x86/xen/setup.c3
-rw-r--r--drivers/video/omap2/vram.c8
-rw-r--r--include/linux/early_res.h23
-rw-r--r--include/linux/memblock.h168
-rw-r--r--include/linux/mm.h2
-rw-r--r--kernel/Makefile1
-rw-r--r--kernel/early_res.c590
-rw-r--r--mm/bootmem.c13
-rw-r--r--mm/memblock.c837
-rw-r--r--mm/page_alloc.c86
-rw-r--r--mm/sparse-vmemmap.c11
69 files changed, 1639 insertions, 1703 deletions
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 36c4553ffcce..7fd9b5eb177f 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -150,6 +150,7 @@ static void __init find_limits(struct meminfo *mi,
150static void __init arm_bootmem_init(struct meminfo *mi, 150static void __init arm_bootmem_init(struct meminfo *mi,
151 unsigned long start_pfn, unsigned long end_pfn) 151 unsigned long start_pfn, unsigned long end_pfn)
152{ 152{
153 struct memblock_region *reg;
153 unsigned int boot_pages; 154 unsigned int boot_pages;
154 phys_addr_t bitmap; 155 phys_addr_t bitmap;
155 pg_data_t *pgdat; 156 pg_data_t *pgdat;
@@ -180,13 +181,13 @@ static void __init arm_bootmem_init(struct meminfo *mi,
180 /* 181 /*
181 * Reserve the memblock reserved regions in bootmem. 182 * Reserve the memblock reserved regions in bootmem.
182 */ 183 */
183 for (i = 0; i < memblock.reserved.cnt; i++) { 184 for_each_memblock(reserved, reg) {
184 phys_addr_t start = memblock_start_pfn(&memblock.reserved, i); 185 phys_addr_t start = memblock_region_reserved_base_pfn(reg);
185 if (start >= start_pfn && 186 phys_addr_t end = memblock_region_reserved_end_pfn(reg);
186 memblock_end_pfn(&memblock.reserved, i) <= end_pfn) 187 if (start >= start_pfn && end <= end_pfn)
187 reserve_bootmem_node(pgdat, __pfn_to_phys(start), 188 reserve_bootmem_node(pgdat, __pfn_to_phys(start),
188 memblock_size_bytes(&memblock.reserved, i), 189 (end - start) << PAGE_SHIFT,
189 BOOTMEM_DEFAULT); 190 BOOTMEM_DEFAULT);
190 } 191 }
191} 192}
192 193
@@ -237,20 +238,7 @@ static void __init arm_bootmem_free(struct meminfo *mi, unsigned long min,
237#ifndef CONFIG_SPARSEMEM 238#ifndef CONFIG_SPARSEMEM
238int pfn_valid(unsigned long pfn) 239int pfn_valid(unsigned long pfn)
239{ 240{
240 struct memblock_region *mem = &memblock.memory; 241 return memblock_is_memory(pfn << PAGE_SHIFT);
241 unsigned int left = 0, right = mem->cnt;
242
243 do {
244 unsigned int mid = (right + left) / 2;
245
246 if (pfn < memblock_start_pfn(mem, mid))
247 right = mid;
248 else if (pfn >= memblock_end_pfn(mem, mid))
249 left = mid + 1;
250 else
251 return 1;
252 } while (left < right);
253 return 0;
254} 242}
255EXPORT_SYMBOL(pfn_valid); 243EXPORT_SYMBOL(pfn_valid);
256 244
@@ -260,10 +248,11 @@ static void arm_memory_present(void)
260#else 248#else
261static void arm_memory_present(void) 249static void arm_memory_present(void)
262{ 250{
263 int i; 251 struct memblock_region *reg;
264 for (i = 0; i < memblock.memory.cnt; i++) 252
265 memory_present(0, memblock_start_pfn(&memblock.memory, i), 253 for_each_memblock(memory, reg)
266 memblock_end_pfn(&memblock.memory, i)); 254 memory_present(0, memblock_region_memory_base_pfn(reg),
255 memblock_region_memory_end_pfn(reg));
267} 256}
268#endif 257#endif
269 258
diff --git a/arch/arm/plat-omap/fb.c b/arch/arm/plat-omap/fb.c
index 0054b9501a53..71934817e172 100644
--- a/arch/arm/plat-omap/fb.c
+++ b/arch/arm/plat-omap/fb.c
@@ -173,11 +173,7 @@ static int check_fbmem_region(int region_idx, struct omapfb_mem_region *rg,
173 173
174static int valid_sdram(unsigned long addr, unsigned long size) 174static int valid_sdram(unsigned long addr, unsigned long size)
175{ 175{
176 struct memblock_property res; 176 return memblock_is_region_memory(addr, size);
177
178 res.base = addr;
179 res.size = size;
180 return !memblock_find(&res) && res.base == addr && res.size == size;
181} 177}
182 178
183static int reserve_sdram(unsigned long addr, unsigned long size) 179static int reserve_sdram(unsigned long addr, unsigned long size)
diff --git a/arch/microblaze/include/asm/memblock.h b/arch/microblaze/include/asm/memblock.h
index f9c2fa331d2a..20a8e257c77f 100644
--- a/arch/microblaze/include/asm/memblock.h
+++ b/arch/microblaze/include/asm/memblock.h
@@ -9,9 +9,6 @@
9#ifndef _ASM_MICROBLAZE_MEMBLOCK_H 9#ifndef _ASM_MICROBLAZE_MEMBLOCK_H
10#define _ASM_MICROBLAZE_MEMBLOCK_H 10#define _ASM_MICROBLAZE_MEMBLOCK_H
11 11
12/* MEMBLOCK limit is OFF */
13#define MEMBLOCK_REAL_LIMIT 0xFFFFFFFF
14
15#endif /* _ASM_MICROBLAZE_MEMBLOCK_H */ 12#endif /* _ASM_MICROBLAZE_MEMBLOCK_H */
16 13
17 14
diff --git a/arch/microblaze/mm/init.c b/arch/microblaze/mm/init.c
index 65eb00419d19..c8437866d3b7 100644
--- a/arch/microblaze/mm/init.c
+++ b/arch/microblaze/mm/init.c
@@ -70,16 +70,16 @@ static void __init paging_init(void)
70 70
71void __init setup_memory(void) 71void __init setup_memory(void)
72{ 72{
73 int i;
74 unsigned long map_size; 73 unsigned long map_size;
74 struct memblock_region *reg;
75
75#ifndef CONFIG_MMU 76#ifndef CONFIG_MMU
76 u32 kernel_align_start, kernel_align_size; 77 u32 kernel_align_start, kernel_align_size;
77 78
78 /* Find main memory where is the kernel */ 79 /* Find main memory where is the kernel */
79 for (i = 0; i < memblock.memory.cnt; i++) { 80 for_each_memblock(memory, reg) {
80 memory_start = (u32) memblock.memory.region[i].base; 81 memory_start = (u32)reg->base;
81 memory_end = (u32) memblock.memory.region[i].base 82 memory_end = (u32) reg->base + reg->size;
82 + (u32) memblock.memory.region[i].size;
83 if ((memory_start <= (u32)_text) && 83 if ((memory_start <= (u32)_text) &&
84 ((u32)_text <= memory_end)) { 84 ((u32)_text <= memory_end)) {
85 memory_size = memory_end - memory_start; 85 memory_size = memory_end - memory_start;
@@ -142,12 +142,10 @@ void __init setup_memory(void)
142 free_bootmem(memory_start, memory_size); 142 free_bootmem(memory_start, memory_size);
143 143
144 /* reserve allocate blocks */ 144 /* reserve allocate blocks */
145 for (i = 0; i < memblock.reserved.cnt; i++) { 145 for_each_memblock(reserved, reg) {
146 pr_debug("reserved %d - 0x%08x-0x%08x\n", i, 146 pr_debug("reserved - 0x%08x-0x%08x\n",
147 (u32) memblock.reserved.region[i].base, 147 (u32) reg->base, (u32) reg->size);
148 (u32) memblock_size_bytes(&memblock.reserved, i)); 148 reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
149 reserve_bootmem(memblock.reserved.region[i].base,
150 memblock_size_bytes(&memblock.reserved, i) - 1, BOOTMEM_DEFAULT);
151 } 149 }
152#ifdef CONFIG_MMU 150#ifdef CONFIG_MMU
153 init_bootmem_done = 1; 151 init_bootmem_done = 1;
@@ -230,7 +228,7 @@ static void mm_cmdline_setup(void)
230 if (maxmem && memory_size > maxmem) { 228 if (maxmem && memory_size > maxmem) {
231 memory_size = maxmem; 229 memory_size = maxmem;
232 memory_end = memory_start + memory_size; 230 memory_end = memory_start + memory_size;
233 memblock.memory.region[0].size = memory_size; 231 memblock.memory.regions[0].size = memory_size;
234 } 232 }
235 } 233 }
236} 234}
@@ -273,14 +271,14 @@ asmlinkage void __init mmu_init(void)
273 machine_restart(NULL); 271 machine_restart(NULL);
274 } 272 }
275 273
276 if ((u32) memblock.memory.region[0].size < 0x1000000) { 274 if ((u32) memblock.memory.regions[0].size < 0x1000000) {
277 printk(KERN_EMERG "Memory must be greater than 16MB\n"); 275 printk(KERN_EMERG "Memory must be greater than 16MB\n");
278 machine_restart(NULL); 276 machine_restart(NULL);
279 } 277 }
280 /* Find main memory where the kernel is */ 278 /* Find main memory where the kernel is */
281 memory_start = (u32) memblock.memory.region[0].base; 279 memory_start = (u32) memblock.memory.regions[0].base;
282 memory_end = (u32) memblock.memory.region[0].base + 280 memory_end = (u32) memblock.memory.regions[0].base +
283 (u32) memblock.memory.region[0].size; 281 (u32) memblock.memory.regions[0].size;
284 memory_size = memory_end - memory_start; 282 memory_size = memory_end - memory_start;
285 283
286 mm_cmdline_setup(); /* FIXME parse args from command line - not used */ 284 mm_cmdline_setup(); /* FIXME parse args from command line - not used */
diff --git a/arch/powerpc/include/asm/memblock.h b/arch/powerpc/include/asm/memblock.h
index 3c29728b56b1..43efc345065e 100644
--- a/arch/powerpc/include/asm/memblock.h
+++ b/arch/powerpc/include/asm/memblock.h
@@ -5,11 +5,4 @@
5 5
6#define MEMBLOCK_DBG(fmt...) udbg_printf(fmt) 6#define MEMBLOCK_DBG(fmt...) udbg_printf(fmt)
7 7
8#ifdef CONFIG_PPC32
9extern phys_addr_t lowmem_end_addr;
10#define MEMBLOCK_REAL_LIMIT lowmem_end_addr
11#else
12#define MEMBLOCK_REAL_LIMIT 0
13#endif
14
15#endif /* _ASM_POWERPC_MEMBLOCK_H */ 8#endif /* _ASM_POWERPC_MEMBLOCK_H */
diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h
index 7ebf42ed84a2..bb40a06d3b77 100644
--- a/arch/powerpc/include/asm/mmu.h
+++ b/arch/powerpc/include/asm/mmu.h
@@ -2,6 +2,8 @@
2#define _ASM_POWERPC_MMU_H_ 2#define _ASM_POWERPC_MMU_H_
3#ifdef __KERNEL__ 3#ifdef __KERNEL__
4 4
5#include <linux/types.h>
6
5#include <asm/asm-compat.h> 7#include <asm/asm-compat.h>
6#include <asm/feature-fixups.h> 8#include <asm/feature-fixups.h>
7 9
@@ -82,6 +84,16 @@ extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup;
82extern void early_init_mmu(void); 84extern void early_init_mmu(void);
83extern void early_init_mmu_secondary(void); 85extern void early_init_mmu_secondary(void);
84 86
87extern void setup_initial_memory_limit(phys_addr_t first_memblock_base,
88 phys_addr_t first_memblock_size);
89
90#ifdef CONFIG_PPC64
91/* This is our real memory area size on ppc64 server, on embedded, we
92 * make it match the size our of bolted TLB area
93 */
94extern u64 ppc64_rma_size;
95#endif /* CONFIG_PPC64 */
96
85#endif /* !__ASSEMBLY__ */ 97#endif /* !__ASSEMBLY__ */
86 98
87/* The kernel use the constants below to index in the page sizes array. 99/* The kernel use the constants below to index in the page sizes array.
diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S
index a90625f9b485..8278e8bad5a0 100644
--- a/arch/powerpc/kernel/head_40x.S
+++ b/arch/powerpc/kernel/head_40x.S
@@ -923,11 +923,7 @@ initial_mmu:
923 mtspr SPRN_PID,r0 923 mtspr SPRN_PID,r0
924 sync 924 sync
925 925
926 /* Configure and load two entries into TLB slots 62 and 63. 926 /* Configure and load one entry into TLB slots 63 */
927 * In case we are pinning TLBs, these are reserved in by the
928 * other TLB functions. If not reserving, then it doesn't
929 * matter where they are loaded.
930 */
931 clrrwi r4,r4,10 /* Mask off the real page number */ 927 clrrwi r4,r4,10 /* Mask off the real page number */
932 ori r4,r4,(TLB_WR | TLB_EX) /* Set the write and execute bits */ 928 ori r4,r4,(TLB_WR | TLB_EX) /* Set the write and execute bits */
933 929
diff --git a/arch/powerpc/kernel/paca.c b/arch/powerpc/kernel/paca.c
index d0a26f1770fe..a4e72159234f 100644
--- a/arch/powerpc/kernel/paca.c
+++ b/arch/powerpc/kernel/paca.c
@@ -127,7 +127,7 @@ void __init allocate_pacas(void)
127 * the first segment. On iSeries they must be within the area mapped 127 * the first segment. On iSeries they must be within the area mapped
128 * by the HV, which is HvPagesToMap * HVPAGESIZE bytes. 128 * by the HV, which is HvPagesToMap * HVPAGESIZE bytes.
129 */ 129 */
130 limit = min(0x10000000ULL, memblock.rmo_size); 130 limit = min(0x10000000ULL, ppc64_rma_size);
131 if (firmware_has_feature(FW_FEATURE_ISERIES)) 131 if (firmware_has_feature(FW_FEATURE_ISERIES))
132 limit = min(limit, HvPagesToMap * HVPAGESIZE); 132 limit = min(limit, HvPagesToMap * HVPAGESIZE);
133 133
diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c
index fed9bf6187d1..c3c6a8857544 100644
--- a/arch/powerpc/kernel/prom.c
+++ b/arch/powerpc/kernel/prom.c
@@ -66,6 +66,7 @@
66int __initdata iommu_is_off; 66int __initdata iommu_is_off;
67int __initdata iommu_force_on; 67int __initdata iommu_force_on;
68unsigned long tce_alloc_start, tce_alloc_end; 68unsigned long tce_alloc_start, tce_alloc_end;
69u64 ppc64_rma_size;
69#endif 70#endif
70 71
71static int __init early_parse_mem(char *p) 72static int __init early_parse_mem(char *p)
@@ -98,7 +99,7 @@ static void __init move_device_tree(void)
98 99
99 if ((memory_limit && (start + size) > memory_limit) || 100 if ((memory_limit && (start + size) > memory_limit) ||
100 overlaps_crashkernel(start, size)) { 101 overlaps_crashkernel(start, size)) {
101 p = __va(memblock_alloc_base(size, PAGE_SIZE, memblock.rmo_size)); 102 p = __va(memblock_alloc(size, PAGE_SIZE));
102 memcpy(p, initial_boot_params, size); 103 memcpy(p, initial_boot_params, size);
103 initial_boot_params = (struct boot_param_header *)p; 104 initial_boot_params = (struct boot_param_header *)p;
104 DBG("Moved device tree to 0x%p\n", p); 105 DBG("Moved device tree to 0x%p\n", p);
@@ -492,7 +493,7 @@ static int __init early_init_dt_scan_memory_ppc(unsigned long node,
492 493
493void __init early_init_dt_add_memory_arch(u64 base, u64 size) 494void __init early_init_dt_add_memory_arch(u64 base, u64 size)
494{ 495{
495#if defined(CONFIG_PPC64) 496#ifdef CONFIG_PPC64
496 if (iommu_is_off) { 497 if (iommu_is_off) {
497 if (base >= 0x80000000ul) 498 if (base >= 0x80000000ul)
498 return; 499 return;
@@ -501,9 +502,13 @@ void __init early_init_dt_add_memory_arch(u64 base, u64 size)
501 } 502 }
502#endif 503#endif
503 504
504 memblock_add(base, size); 505 /* First MEMBLOCK added, do some special initializations */
505 506 if (memstart_addr == ~(phys_addr_t)0)
507 setup_initial_memory_limit(base, size);
506 memstart_addr = min((u64)memstart_addr, base); 508 memstart_addr = min((u64)memstart_addr, base);
509
510 /* Add the chunk to the MEMBLOCK list */
511 memblock_add(base, size);
507} 512}
508 513
509u64 __init early_init_dt_alloc_memory_arch(u64 size, u64 align) 514u64 __init early_init_dt_alloc_memory_arch(u64 size, u64 align)
@@ -655,7 +660,6 @@ static void __init phyp_dump_reserve_mem(void)
655static inline void __init phyp_dump_reserve_mem(void) {} 660static inline void __init phyp_dump_reserve_mem(void) {}
656#endif /* CONFIG_PHYP_DUMP && CONFIG_PPC_RTAS */ 661#endif /* CONFIG_PHYP_DUMP && CONFIG_PPC_RTAS */
657 662
658
659void __init early_init_devtree(void *params) 663void __init early_init_devtree(void *params)
660{ 664{
661 phys_addr_t limit; 665 phys_addr_t limit;
@@ -683,6 +687,7 @@ void __init early_init_devtree(void *params)
683 687
684 /* Scan memory nodes and rebuild MEMBLOCKs */ 688 /* Scan memory nodes and rebuild MEMBLOCKs */
685 memblock_init(); 689 memblock_init();
690
686 of_scan_flat_dt(early_init_dt_scan_root, NULL); 691 of_scan_flat_dt(early_init_dt_scan_root, NULL);
687 of_scan_flat_dt(early_init_dt_scan_memory_ppc, NULL); 692 of_scan_flat_dt(early_init_dt_scan_memory_ppc, NULL);
688 693
diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c
index 41048de3c6c3..7333fdbf857b 100644
--- a/arch/powerpc/kernel/rtas.c
+++ b/arch/powerpc/kernel/rtas.c
@@ -969,7 +969,7 @@ void __init rtas_initialize(void)
969 */ 969 */
970#ifdef CONFIG_PPC64 970#ifdef CONFIG_PPC64
971 if (machine_is(pseries) && firmware_has_feature(FW_FEATURE_LPAR)) { 971 if (machine_is(pseries) && firmware_has_feature(FW_FEATURE_LPAR)) {
972 rtas_region = min(memblock.rmo_size, RTAS_INSTANTIATE_MAX); 972 rtas_region = min(ppc64_rma_size, RTAS_INSTANTIATE_MAX);
973 ibm_suspend_me_token = rtas_token("ibm,suspend-me"); 973 ibm_suspend_me_token = rtas_token("ibm,suspend-me");
974 } 974 }
975#endif 975#endif
diff --git a/arch/powerpc/kernel/setup_32.c b/arch/powerpc/kernel/setup_32.c
index 93666f9cabf1..b86111fe9257 100644
--- a/arch/powerpc/kernel/setup_32.c
+++ b/arch/powerpc/kernel/setup_32.c
@@ -246,7 +246,7 @@ static void __init irqstack_early_init(void)
246 unsigned int i; 246 unsigned int i;
247 247
248 /* interrupt stacks must be in lowmem, we get that for free on ppc32 248 /* interrupt stacks must be in lowmem, we get that for free on ppc32
249 * as the memblock is limited to lowmem by MEMBLOCK_REAL_LIMIT */ 249 * as the memblock is limited to lowmem by default */
250 for_each_possible_cpu(i) { 250 for_each_possible_cpu(i) {
251 softirq_ctx[i] = (struct thread_info *) 251 softirq_ctx[i] = (struct thread_info *)
252 __va(memblock_alloc(THREAD_SIZE, THREAD_SIZE)); 252 __va(memblock_alloc(THREAD_SIZE, THREAD_SIZE));
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index e72690ec9b87..2a178b0ebcdf 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -486,7 +486,7 @@ static void __init emergency_stack_init(void)
486 * bringup, we need to get at them in real mode. This means they 486 * bringup, we need to get at them in real mode. This means they
487 * must also be within the RMO region. 487 * must also be within the RMO region.
488 */ 488 */
489 limit = min(slb0_limit(), memblock.rmo_size); 489 limit = min(slb0_limit(), ppc64_rma_size);
490 490
491 for_each_possible_cpu(i) { 491 for_each_possible_cpu(i) {
492 unsigned long sp; 492 unsigned long sp;
diff --git a/arch/powerpc/mm/40x_mmu.c b/arch/powerpc/mm/40x_mmu.c
index 1dc2fa5ce1bd..5810967511d4 100644
--- a/arch/powerpc/mm/40x_mmu.c
+++ b/arch/powerpc/mm/40x_mmu.c
@@ -35,6 +35,7 @@
35#include <linux/init.h> 35#include <linux/init.h>
36#include <linux/delay.h> 36#include <linux/delay.h>
37#include <linux/highmem.h> 37#include <linux/highmem.h>
38#include <linux/memblock.h>
38 39
39#include <asm/pgalloc.h> 40#include <asm/pgalloc.h>
40#include <asm/prom.h> 41#include <asm/prom.h>
@@ -47,6 +48,7 @@
47#include <asm/bootx.h> 48#include <asm/bootx.h>
48#include <asm/machdep.h> 49#include <asm/machdep.h>
49#include <asm/setup.h> 50#include <asm/setup.h>
51
50#include "mmu_decl.h" 52#include "mmu_decl.h"
51 53
52extern int __map_without_ltlbs; 54extern int __map_without_ltlbs;
@@ -139,8 +141,19 @@ unsigned long __init mmu_mapin_ram(unsigned long top)
139 * coverage with normal-sized pages (or other reasons) do not 141 * coverage with normal-sized pages (or other reasons) do not
140 * attempt to allocate outside the allowed range. 142 * attempt to allocate outside the allowed range.
141 */ 143 */
142 144 memblock_set_current_limit(mapped);
143 __initial_memory_limit_addr = memstart_addr + mapped;
144 145
145 return mapped; 146 return mapped;
146} 147}
148
149void setup_initial_memory_limit(phys_addr_t first_memblock_base,
150 phys_addr_t first_memblock_size)
151{
152 /* We don't currently support the first MEMBLOCK not mapping 0
153 * physical on those processors
154 */
155 BUG_ON(first_memblock_base != 0);
156
157 /* 40x can only access 16MB at the moment (see head_40x.S) */
158 memblock_set_current_limit(min_t(u64, first_memblock_size, 0x00800000));
159}
diff --git a/arch/powerpc/mm/44x_mmu.c b/arch/powerpc/mm/44x_mmu.c
index d8c6efb32bc6..024acab588fd 100644
--- a/arch/powerpc/mm/44x_mmu.c
+++ b/arch/powerpc/mm/44x_mmu.c
@@ -24,6 +24,8 @@
24 */ 24 */
25 25
26#include <linux/init.h> 26#include <linux/init.h>
27#include <linux/memblock.h>
28
27#include <asm/mmu.h> 29#include <asm/mmu.h>
28#include <asm/system.h> 30#include <asm/system.h>
29#include <asm/page.h> 31#include <asm/page.h>
@@ -213,6 +215,18 @@ unsigned long __init mmu_mapin_ram(unsigned long top)
213 return total_lowmem; 215 return total_lowmem;
214} 216}
215 217
218void setup_initial_memory_limit(phys_addr_t first_memblock_base,
219 phys_addr_t first_memblock_size)
220{
221 /* We don't currently support the first MEMBLOCK not mapping 0
222 * physical on those processors
223 */
224 BUG_ON(first_memblock_base != 0);
225
226 /* 44x has a 256M TLB entry pinned at boot */
227 memblock_set_current_limit(min_t(u64, first_memblock_size, PPC_PIN_SIZE));
228}
229
216#ifdef CONFIG_SMP 230#ifdef CONFIG_SMP
217void __cpuinit mmu_init_secondary(int cpu) 231void __cpuinit mmu_init_secondary(int cpu)
218{ 232{
diff --git a/arch/powerpc/mm/fsl_booke_mmu.c b/arch/powerpc/mm/fsl_booke_mmu.c
index 4b66a1ece6d8..cde270847e7c 100644
--- a/arch/powerpc/mm/fsl_booke_mmu.c
+++ b/arch/powerpc/mm/fsl_booke_mmu.c
@@ -40,6 +40,7 @@
40#include <linux/init.h> 40#include <linux/init.h>
41#include <linux/delay.h> 41#include <linux/delay.h>
42#include <linux/highmem.h> 42#include <linux/highmem.h>
43#include <linux/memblock.h>
43 44
44#include <asm/pgalloc.h> 45#include <asm/pgalloc.h>
45#include <asm/prom.h> 46#include <asm/prom.h>
@@ -213,5 +214,14 @@ void __init adjust_total_lowmem(void)
213 pr_cont("%lu Mb, residual: %dMb\n", tlbcam_sz(tlbcam_index - 1) >> 20, 214 pr_cont("%lu Mb, residual: %dMb\n", tlbcam_sz(tlbcam_index - 1) >> 20,
214 (unsigned int)((total_lowmem - __max_low_memory) >> 20)); 215 (unsigned int)((total_lowmem - __max_low_memory) >> 20));
215 216
216 __initial_memory_limit_addr = memstart_addr + __max_low_memory; 217 memblock_set_current_limit(memstart_addr + __max_low_memory);
218}
219
220void setup_initial_memory_limit(phys_addr_t first_memblock_base,
221 phys_addr_t first_memblock_size)
222{
223 phys_addr_t limit = first_memblock_base + first_memblock_size;
224
225 /* 64M mapped initially according to head_fsl_booke.S */
226 memblock_set_current_limit(min_t(u64, limit, 0x04000000));
217} 227}
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 09dffe6efa46..83f534d862db 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -588,7 +588,7 @@ static void __init htab_initialize(void)
588 unsigned long pteg_count; 588 unsigned long pteg_count;
589 unsigned long prot; 589 unsigned long prot;
590 unsigned long base = 0, size = 0, limit; 590 unsigned long base = 0, size = 0, limit;
591 int i; 591 struct memblock_region *reg;
592 592
593 DBG(" -> htab_initialize()\n"); 593 DBG(" -> htab_initialize()\n");
594 594
@@ -625,7 +625,7 @@ static void __init htab_initialize(void)
625 if (machine_is(cell)) 625 if (machine_is(cell))
626 limit = 0x80000000; 626 limit = 0x80000000;
627 else 627 else
628 limit = 0; 628 limit = MEMBLOCK_ALLOC_ANYWHERE;
629 629
630 table = memblock_alloc_base(htab_size_bytes, htab_size_bytes, limit); 630 table = memblock_alloc_base(htab_size_bytes, htab_size_bytes, limit);
631 631
@@ -649,7 +649,7 @@ static void __init htab_initialize(void)
649#ifdef CONFIG_DEBUG_PAGEALLOC 649#ifdef CONFIG_DEBUG_PAGEALLOC
650 linear_map_hash_count = memblock_end_of_DRAM() >> PAGE_SHIFT; 650 linear_map_hash_count = memblock_end_of_DRAM() >> PAGE_SHIFT;
651 linear_map_hash_slots = __va(memblock_alloc_base(linear_map_hash_count, 651 linear_map_hash_slots = __va(memblock_alloc_base(linear_map_hash_count,
652 1, memblock.rmo_size)); 652 1, ppc64_rma_size));
653 memset(linear_map_hash_slots, 0, linear_map_hash_count); 653 memset(linear_map_hash_slots, 0, linear_map_hash_count);
654#endif /* CONFIG_DEBUG_PAGEALLOC */ 654#endif /* CONFIG_DEBUG_PAGEALLOC */
655 655
@@ -659,9 +659,9 @@ static void __init htab_initialize(void)
659 */ 659 */
660 660
661 /* create bolted the linear mapping in the hash table */ 661 /* create bolted the linear mapping in the hash table */
662 for (i=0; i < memblock.memory.cnt; i++) { 662 for_each_memblock(memory, reg) {
663 base = (unsigned long)__va(memblock.memory.region[i].base); 663 base = (unsigned long)__va(reg->base);
664 size = memblock.memory.region[i].size; 664 size = reg->size;
665 665
666 DBG("creating mapping for region: %lx..%lx (prot: %lx)\n", 666 DBG("creating mapping for region: %lx..%lx (prot: %lx)\n",
667 base, size, prot); 667 base, size, prot);
@@ -696,7 +696,8 @@ static void __init htab_initialize(void)
696#endif /* CONFIG_U3_DART */ 696#endif /* CONFIG_U3_DART */
697 BUG_ON(htab_bolt_mapping(base, base + size, __pa(base), 697 BUG_ON(htab_bolt_mapping(base, base + size, __pa(base),
698 prot, mmu_linear_psize, mmu_kernel_ssize)); 698 prot, mmu_linear_psize, mmu_kernel_ssize));
699 } 699 }
700 memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
700 701
701 /* 702 /*
702 * If we have a memory_limit and we've allocated TCEs then we need to 703 * If we have a memory_limit and we've allocated TCEs then we need to
@@ -1247,3 +1248,23 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
1247 local_irq_restore(flags); 1248 local_irq_restore(flags);
1248} 1249}
1249#endif /* CONFIG_DEBUG_PAGEALLOC */ 1250#endif /* CONFIG_DEBUG_PAGEALLOC */
1251
1252void setup_initial_memory_limit(phys_addr_t first_memblock_base,
1253 phys_addr_t first_memblock_size)
1254{
1255 /* We don't currently support the first MEMBLOCK not mapping 0
1256 * physical on those processors
1257 */
1258 BUG_ON(first_memblock_base != 0);
1259
1260 /* On LPAR systems, the first entry is our RMA region,
1261 * non-LPAR 64-bit hash MMU systems don't have a limitation
1262 * on real mode access, but using the first entry works well
1263 * enough. We also clamp it to 1G to avoid some funky things
1264 * such as RTAS bugs etc...
1265 */
1266 ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);
1267
1268 /* Finally limit subsequent allocations */
1269 memblock_set_current_limit(ppc64_rma_size);
1270}
diff --git a/arch/powerpc/mm/init_32.c b/arch/powerpc/mm/init_32.c
index 6a6975dc2654..742da43b4ab6 100644
--- a/arch/powerpc/mm/init_32.c
+++ b/arch/powerpc/mm/init_32.c
@@ -92,12 +92,6 @@ int __allow_ioremap_reserved;
92unsigned long __max_low_memory = MAX_LOW_MEM; 92unsigned long __max_low_memory = MAX_LOW_MEM;
93 93
94/* 94/*
95 * address of the limit of what is accessible with initial MMU setup -
96 * 256MB usually, but only 16MB on 601.
97 */
98phys_addr_t __initial_memory_limit_addr = (phys_addr_t)0x10000000;
99
100/*
101 * Check for command-line options that affect what MMU_init will do. 95 * Check for command-line options that affect what MMU_init will do.
102 */ 96 */
103void MMU_setup(void) 97void MMU_setup(void)
@@ -126,13 +120,6 @@ void __init MMU_init(void)
126 if (ppc_md.progress) 120 if (ppc_md.progress)
127 ppc_md.progress("MMU:enter", 0x111); 121 ppc_md.progress("MMU:enter", 0x111);
128 122
129 /* 601 can only access 16MB at the moment */
130 if (PVR_VER(mfspr(SPRN_PVR)) == 1)
131 __initial_memory_limit_addr = 0x01000000;
132 /* 8xx can only access 8MB at the moment */
133 if (PVR_VER(mfspr(SPRN_PVR)) == 0x50)
134 __initial_memory_limit_addr = 0x00800000;
135
136 /* parse args from command line */ 123 /* parse args from command line */
137 MMU_setup(); 124 MMU_setup();
138 125
@@ -190,20 +177,18 @@ void __init MMU_init(void)
190#ifdef CONFIG_BOOTX_TEXT 177#ifdef CONFIG_BOOTX_TEXT
191 btext_unmap(); 178 btext_unmap();
192#endif 179#endif
180
181 /* Shortly after that, the entire linear mapping will be available */
182 memblock_set_current_limit(lowmem_end_addr);
193} 183}
194 184
195/* This is only called until mem_init is done. */ 185/* This is only called until mem_init is done. */
196void __init *early_get_page(void) 186void __init *early_get_page(void)
197{ 187{
198 void *p; 188 if (init_bootmem_done)
199 189 return alloc_bootmem_pages(PAGE_SIZE);
200 if (init_bootmem_done) { 190 else
201 p = alloc_bootmem_pages(PAGE_SIZE); 191 return __va(memblock_alloc(PAGE_SIZE, PAGE_SIZE));
202 } else {
203 p = __va(memblock_alloc_base(PAGE_SIZE, PAGE_SIZE,
204 __initial_memory_limit_addr));
205 }
206 return p;
207} 192}
208 193
209/* Free up now-unused memory */ 194/* Free up now-unused memory */
@@ -252,3 +237,17 @@ void free_initrd_mem(unsigned long start, unsigned long end)
252} 237}
253#endif 238#endif
254 239
240
241#ifdef CONFIG_8xx /* No 8xx specific .c file to put that in ... */
242void setup_initial_memory_limit(phys_addr_t first_memblock_base,
243 phys_addr_t first_memblock_size)
244{
245 /* We don't currently support the first MEMBLOCK not mapping 0
246 * physical on those processors
247 */
248 BUG_ON(first_memblock_base != 0);
249
250 /* 8xx can only access 8MB at the moment */
251 memblock_set_current_limit(min_t(u64, first_memblock_size, 0x00800000));
252}
253#endif /* CONFIG_8xx */
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index ace85fa74b29..6374b2196a17 100644
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -330,3 +330,4 @@ int __meminit vmemmap_populate(struct page *start_page,
330 return 0; 330 return 0;
331} 331}
332#endif /* CONFIG_SPARSEMEM_VMEMMAP */ 332#endif /* CONFIG_SPARSEMEM_VMEMMAP */
333
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 1a84a8d00005..a66499650909 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -82,18 +82,11 @@ int page_is_ram(unsigned long pfn)
82 return pfn < max_pfn; 82 return pfn < max_pfn;
83#else 83#else
84 unsigned long paddr = (pfn << PAGE_SHIFT); 84 unsigned long paddr = (pfn << PAGE_SHIFT);
85 int i; 85 struct memblock_region *reg;
86 for (i=0; i < memblock.memory.cnt; i++) {
87 unsigned long base;
88 86
89 base = memblock.memory.region[i].base; 87 for_each_memblock(memory, reg)
90 88 if (paddr >= reg->base && paddr < (reg->base + reg->size))
91 if ((paddr >= base) &&
92 (paddr < (base + memblock.memory.region[i].size))) {
93 return 1; 89 return 1;
94 }
95 }
96
97 return 0; 90 return 0;
98#endif 91#endif
99} 92}
@@ -149,23 +142,19 @@ int
149walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages, 142walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages,
150 void *arg, int (*func)(unsigned long, unsigned long, void *)) 143 void *arg, int (*func)(unsigned long, unsigned long, void *))
151{ 144{
152 struct memblock_property res; 145 struct memblock_region *reg;
153 unsigned long pfn, len; 146 unsigned long end_pfn = start_pfn + nr_pages;
154 u64 end; 147 unsigned long tstart, tend;
155 int ret = -1; 148 int ret = -1;
156 149
157 res.base = (u64) start_pfn << PAGE_SHIFT; 150 for_each_memblock(memory, reg) {
158 res.size = (u64) nr_pages << PAGE_SHIFT; 151 tstart = max(start_pfn, memblock_region_memory_base_pfn(reg));
159 152 tend = min(end_pfn, memblock_region_memory_end_pfn(reg));
160 end = res.base + res.size - 1; 153 if (tstart >= tend)
161 while ((res.base < end) && (memblock_find(&res) >= 0)) { 154 continue;
162 pfn = (unsigned long)(res.base >> PAGE_SHIFT); 155 ret = (*func)(tstart, tend - tstart, arg);
163 len = (unsigned long)(res.size >> PAGE_SHIFT);
164 ret = (*func)(pfn, len, arg);
165 if (ret) 156 if (ret)
166 break; 157 break;
167 res.base += (res.size + 1);
168 res.size = (end - res.base + 1);
169 } 158 }
170 return ret; 159 return ret;
171} 160}
@@ -179,9 +168,9 @@ EXPORT_SYMBOL_GPL(walk_system_ram_range);
179#ifndef CONFIG_NEED_MULTIPLE_NODES 168#ifndef CONFIG_NEED_MULTIPLE_NODES
180void __init do_init_bootmem(void) 169void __init do_init_bootmem(void)
181{ 170{
182 unsigned long i;
183 unsigned long start, bootmap_pages; 171 unsigned long start, bootmap_pages;
184 unsigned long total_pages; 172 unsigned long total_pages;
173 struct memblock_region *reg;
185 int boot_mapsize; 174 int boot_mapsize;
186 175
187 max_low_pfn = max_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT; 176 max_low_pfn = max_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
@@ -204,10 +193,10 @@ void __init do_init_bootmem(void)
204 boot_mapsize = init_bootmem_node(NODE_DATA(0), start >> PAGE_SHIFT, min_low_pfn, max_low_pfn); 193 boot_mapsize = init_bootmem_node(NODE_DATA(0), start >> PAGE_SHIFT, min_low_pfn, max_low_pfn);
205 194
206 /* Add active regions with valid PFNs */ 195 /* Add active regions with valid PFNs */
207 for (i = 0; i < memblock.memory.cnt; i++) { 196 for_each_memblock(memory, reg) {
208 unsigned long start_pfn, end_pfn; 197 unsigned long start_pfn, end_pfn;
209 start_pfn = memblock.memory.region[i].base >> PAGE_SHIFT; 198 start_pfn = memblock_region_memory_base_pfn(reg);
210 end_pfn = start_pfn + memblock_size_pages(&memblock.memory, i); 199 end_pfn = memblock_region_memory_end_pfn(reg);
211 add_active_range(0, start_pfn, end_pfn); 200 add_active_range(0, start_pfn, end_pfn);
212 } 201 }
213 202
@@ -218,29 +207,21 @@ void __init do_init_bootmem(void)
218 free_bootmem_with_active_regions(0, lowmem_end_addr >> PAGE_SHIFT); 207 free_bootmem_with_active_regions(0, lowmem_end_addr >> PAGE_SHIFT);
219 208
220 /* reserve the sections we're already using */ 209 /* reserve the sections we're already using */
221 for (i = 0; i < memblock.reserved.cnt; i++) { 210 for_each_memblock(reserved, reg) {
222 unsigned long addr = memblock.reserved.region[i].base + 211 unsigned long top = reg->base + reg->size - 1;
223 memblock_size_bytes(&memblock.reserved, i) - 1; 212 if (top < lowmem_end_addr)
224 if (addr < lowmem_end_addr) 213 reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
225 reserve_bootmem(memblock.reserved.region[i].base, 214 else if (reg->base < lowmem_end_addr) {
226 memblock_size_bytes(&memblock.reserved, i), 215 unsigned long trunc_size = lowmem_end_addr - reg->base;
227 BOOTMEM_DEFAULT); 216 reserve_bootmem(reg->base, trunc_size, BOOTMEM_DEFAULT);
228 else if (memblock.reserved.region[i].base < lowmem_end_addr) {
229 unsigned long adjusted_size = lowmem_end_addr -
230 memblock.reserved.region[i].base;
231 reserve_bootmem(memblock.reserved.region[i].base,
232 adjusted_size, BOOTMEM_DEFAULT);
233 } 217 }
234 } 218 }
235#else 219#else
236 free_bootmem_with_active_regions(0, max_pfn); 220 free_bootmem_with_active_regions(0, max_pfn);
237 221
238 /* reserve the sections we're already using */ 222 /* reserve the sections we're already using */
239 for (i = 0; i < memblock.reserved.cnt; i++) 223 for_each_memblock(reserved, reg)
240 reserve_bootmem(memblock.reserved.region[i].base, 224 reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
241 memblock_size_bytes(&memblock.reserved, i),
242 BOOTMEM_DEFAULT);
243
244#endif 225#endif
245 /* XXX need to clip this if using highmem? */ 226 /* XXX need to clip this if using highmem? */
246 sparse_memory_present_with_active_regions(0); 227 sparse_memory_present_with_active_regions(0);
@@ -251,22 +232,15 @@ void __init do_init_bootmem(void)
251/* mark pages that don't exist as nosave */ 232/* mark pages that don't exist as nosave */
252static int __init mark_nonram_nosave(void) 233static int __init mark_nonram_nosave(void)
253{ 234{
254 unsigned long memblock_next_region_start_pfn, 235 struct memblock_region *reg, *prev = NULL;
255 memblock_region_max_pfn; 236
256 int i; 237 for_each_memblock(memory, reg) {
257 238 if (prev &&
258 for (i = 0; i < memblock.memory.cnt - 1; i++) { 239 memblock_region_memory_end_pfn(prev) < memblock_region_memory_base_pfn(reg))
259 memblock_region_max_pfn = 240 register_nosave_region(memblock_region_memory_end_pfn(prev),
260 (memblock.memory.region[i].base >> PAGE_SHIFT) + 241 memblock_region_memory_base_pfn(reg));
261 (memblock.memory.region[i].size >> PAGE_SHIFT); 242 prev = reg;
262 memblock_next_region_start_pfn =
263 memblock.memory.region[i+1].base >> PAGE_SHIFT;
264
265 if (memblock_region_max_pfn < memblock_next_region_start_pfn)
266 register_nosave_region(memblock_region_max_pfn,
267 memblock_next_region_start_pfn);
268 } 243 }
269
270 return 0; 244 return 0;
271} 245}
272 246
@@ -327,7 +301,7 @@ void __init mem_init(void)
327 swiotlb_init(1); 301 swiotlb_init(1);
328#endif 302#endif
329 303
330 num_physpages = memblock.memory.size >> PAGE_SHIFT; 304 num_physpages = memblock_phys_mem_size() >> PAGE_SHIFT;
331 high_memory = (void *) __va(max_low_pfn * PAGE_SIZE); 305 high_memory = (void *) __va(max_low_pfn * PAGE_SIZE);
332 306
333#ifdef CONFIG_NEED_MULTIPLE_NODES 307#ifdef CONFIG_NEED_MULTIPLE_NODES
diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
index 002878ccf90b..74505b245374 100644
--- a/arch/powerpc/mm/numa.c
+++ b/arch/powerpc/mm/numa.c
@@ -802,16 +802,17 @@ static void __init setup_nonnuma(void)
802 unsigned long top_of_ram = memblock_end_of_DRAM(); 802 unsigned long top_of_ram = memblock_end_of_DRAM();
803 unsigned long total_ram = memblock_phys_mem_size(); 803 unsigned long total_ram = memblock_phys_mem_size();
804 unsigned long start_pfn, end_pfn; 804 unsigned long start_pfn, end_pfn;
805 unsigned int i, nid = 0; 805 unsigned int nid = 0;
806 struct memblock_region *reg;
806 807
807 printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n", 808 printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
808 top_of_ram, total_ram); 809 top_of_ram, total_ram);
809 printk(KERN_DEBUG "Memory hole size: %ldMB\n", 810 printk(KERN_DEBUG "Memory hole size: %ldMB\n",
810 (top_of_ram - total_ram) >> 20); 811 (top_of_ram - total_ram) >> 20);
811 812
812 for (i = 0; i < memblock.memory.cnt; ++i) { 813 for_each_memblock(memory, reg) {
813 start_pfn = memblock.memory.region[i].base >> PAGE_SHIFT; 814 start_pfn = memblock_region_memory_base_pfn(reg);
814 end_pfn = start_pfn + memblock_size_pages(&memblock.memory, i); 815 end_pfn = memblock_region_memory_end_pfn(reg);
815 816
816 fake_numa_create_new_node(end_pfn, &nid); 817 fake_numa_create_new_node(end_pfn, &nid);
817 add_active_range(nid, start_pfn, end_pfn); 818 add_active_range(nid, start_pfn, end_pfn);
@@ -947,11 +948,11 @@ static struct notifier_block __cpuinitdata ppc64_numa_nb = {
947static void mark_reserved_regions_for_nid(int nid) 948static void mark_reserved_regions_for_nid(int nid)
948{ 949{
949 struct pglist_data *node = NODE_DATA(nid); 950 struct pglist_data *node = NODE_DATA(nid);
950 int i; 951 struct memblock_region *reg;
951 952
952 for (i = 0; i < memblock.reserved.cnt; i++) { 953 for_each_memblock(reserved, reg) {
953 unsigned long physbase = memblock.reserved.region[i].base; 954 unsigned long physbase = reg->base;
954 unsigned long size = memblock.reserved.region[i].size; 955 unsigned long size = reg->size;
955 unsigned long start_pfn = physbase >> PAGE_SHIFT; 956 unsigned long start_pfn = physbase >> PAGE_SHIFT;
956 unsigned long end_pfn = PFN_UP(physbase + size); 957 unsigned long end_pfn = PFN_UP(physbase + size);
957 struct node_active_region node_ar; 958 struct node_active_region node_ar;
diff --git a/arch/powerpc/mm/ppc_mmu_32.c b/arch/powerpc/mm/ppc_mmu_32.c
index f8a01829d64f..11571e118831 100644
--- a/arch/powerpc/mm/ppc_mmu_32.c
+++ b/arch/powerpc/mm/ppc_mmu_32.c
@@ -223,8 +223,7 @@ void __init MMU_init_hw(void)
223 * Find some memory for the hash table. 223 * Find some memory for the hash table.
224 */ 224 */
225 if ( ppc_md.progress ) ppc_md.progress("hash:find piece", 0x322); 225 if ( ppc_md.progress ) ppc_md.progress("hash:find piece", 0x322);
226 Hash = __va(memblock_alloc_base(Hash_size, Hash_size, 226 Hash = __va(memblock_alloc(Hash_size, Hash_size));
227 __initial_memory_limit_addr));
228 cacheable_memzero(Hash, Hash_size); 227 cacheable_memzero(Hash, Hash_size);
229 _SDR1 = __pa(Hash) | SDR1_LOW_BITS; 228 _SDR1 = __pa(Hash) | SDR1_LOW_BITS;
230 229
@@ -272,3 +271,18 @@ void __init MMU_init_hw(void)
272 271
273 if ( ppc_md.progress ) ppc_md.progress("hash:done", 0x205); 272 if ( ppc_md.progress ) ppc_md.progress("hash:done", 0x205);
274} 273}
274
275void setup_initial_memory_limit(phys_addr_t first_memblock_base,
276 phys_addr_t first_memblock_size)
277{
278 /* We don't currently support the first MEMBLOCK not mapping 0
279 * physical on those processors
280 */
281 BUG_ON(first_memblock_base != 0);
282
283 /* 601 can only access 16MB at the moment */
284 if (PVR_VER(mfspr(SPRN_PVR)) == 1)
285 memblock_set_current_limit(min_t(u64, first_memblock_size, 0x01000000));
286 else /* Anything else has 256M mapped */
287 memblock_set_current_limit(min_t(u64, first_memblock_size, 0x10000000));
288}
diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c
index fe391e942521..6a0f20c25469 100644
--- a/arch/powerpc/mm/tlb_nohash.c
+++ b/arch/powerpc/mm/tlb_nohash.c
@@ -509,6 +509,8 @@ static void __early_init_mmu(int boot_cpu)
509 * the MMU configuration 509 * the MMU configuration
510 */ 510 */
511 mb(); 511 mb();
512
513 memblock_set_current_limit(linear_map_top);
512} 514}
513 515
514void __init early_init_mmu(void) 516void __init early_init_mmu(void)
@@ -521,4 +523,18 @@ void __cpuinit early_init_mmu_secondary(void)
521 __early_init_mmu(0); 523 __early_init_mmu(0);
522} 524}
523 525
526void setup_initial_memory_limit(phys_addr_t first_memblock_base,
527 phys_addr_t first_memblock_size)
528{
529 /* On Embedded 64-bit, we adjust the RMA size to match
530 * the bolted TLB entry. We know for now that only 1G
531 * entries are supported though that may eventually
532 * change. We crop it to the size of the first MEMBLOCK to
533 * avoid going over total available memory just in case...
534 */
535 ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);
536
537 /* Finally limit subsequent allocations */
538 memblock_set_current_limit(ppc64_memblock_base + ppc64_rma_size);
539}
524#endif /* CONFIG_PPC64 */ 540#endif /* CONFIG_PPC64 */
diff --git a/arch/powerpc/platforms/embedded6xx/wii.c b/arch/powerpc/platforms/embedded6xx/wii.c
index 5cdcc7c8d973..649473a729b8 100644
--- a/arch/powerpc/platforms/embedded6xx/wii.c
+++ b/arch/powerpc/platforms/embedded6xx/wii.c
@@ -65,7 +65,7 @@ static int __init page_aligned(unsigned long x)
65 65
66void __init wii_memory_fixups(void) 66void __init wii_memory_fixups(void)
67{ 67{
68 struct memblock_property *p = memblock.memory.region; 68 struct memblock_region *p = memblock.memory.regions;
69 69
70 /* 70 /*
71 * This is part of a workaround to allow the use of two 71 * This is part of a workaround to allow the use of two
diff --git a/arch/sh/include/asm/memblock.h b/arch/sh/include/asm/memblock.h
index dfe683b88075..e87063fad2ea 100644
--- a/arch/sh/include/asm/memblock.h
+++ b/arch/sh/include/asm/memblock.h
@@ -1,6 +1,4 @@
1#ifndef __ASM_SH_MEMBLOCK_H 1#ifndef __ASM_SH_MEMBLOCK_H
2#define __ASM_SH_MEMBLOCK_H 2#define __ASM_SH_MEMBLOCK_H
3 3
4#define MEMBLOCK_REAL_LIMIT 0
5
6#endif /* __ASM_SH_MEMBLOCK_H */ 4#endif /* __ASM_SH_MEMBLOCK_H */
diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c
index d0e249100e98..552bea5113f5 100644
--- a/arch/sh/mm/init.c
+++ b/arch/sh/mm/init.c
@@ -200,7 +200,6 @@ static void __init bootmem_init_one_node(unsigned int nid)
200 unsigned long total_pages, paddr; 200 unsigned long total_pages, paddr;
201 unsigned long end_pfn; 201 unsigned long end_pfn;
202 struct pglist_data *p; 202 struct pglist_data *p;
203 int i;
204 203
205 p = NODE_DATA(nid); 204 p = NODE_DATA(nid);
206 205
@@ -226,11 +225,12 @@ static void __init bootmem_init_one_node(unsigned int nid)
226 * reservations in other nodes. 225 * reservations in other nodes.
227 */ 226 */
228 if (nid == 0) { 227 if (nid == 0) {
228 struct memblock_region *reg;
229
229 /* Reserve the sections we're already using. */ 230 /* Reserve the sections we're already using. */
230 for (i = 0; i < memblock.reserved.cnt; i++) 231 for_each_memblock(reserved, reg) {
231 reserve_bootmem(memblock.reserved.region[i].base, 232 reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
232 memblock_size_bytes(&memblock.reserved, i), 233 }
233 BOOTMEM_DEFAULT);
234 } 234 }
235 235
236 sparse_memory_present_with_active_regions(nid); 236 sparse_memory_present_with_active_regions(nid);
@@ -238,13 +238,14 @@ static void __init bootmem_init_one_node(unsigned int nid)
238 238
239static void __init do_init_bootmem(void) 239static void __init do_init_bootmem(void)
240{ 240{
241 struct memblock_region *reg;
241 int i; 242 int i;
242 243
243 /* Add active regions with valid PFNs. */ 244 /* Add active regions with valid PFNs. */
244 for (i = 0; i < memblock.memory.cnt; i++) { 245 for_each_memblock(memory, reg) {
245 unsigned long start_pfn, end_pfn; 246 unsigned long start_pfn, end_pfn;
246 start_pfn = memblock.memory.region[i].base >> PAGE_SHIFT; 247 start_pfn = memblock_region_memory_base_pfn(reg);
247 end_pfn = start_pfn + memblock_size_pages(&memblock.memory, i); 248 end_pfn = memblock_region_memory_end_pfn(reg);
248 __add_active_range(0, start_pfn, end_pfn); 249 __add_active_range(0, start_pfn, end_pfn);
249 } 250 }
250 251
diff --git a/arch/sparc/include/asm/memblock.h b/arch/sparc/include/asm/memblock.h
index f12af880649b..c67b047ef85e 100644
--- a/arch/sparc/include/asm/memblock.h
+++ b/arch/sparc/include/asm/memblock.h
@@ -5,6 +5,4 @@
5 5
6#define MEMBLOCK_DBG(fmt...) prom_printf(fmt) 6#define MEMBLOCK_DBG(fmt...) prom_printf(fmt)
7 7
8#define MEMBLOCK_REAL_LIMIT 0
9
10#endif /* !(_SPARC64_MEMBLOCK_H) */ 8#endif /* !(_SPARC64_MEMBLOCK_H) */
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index f0434513df15..4c2572773b55 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -785,8 +785,7 @@ static int find_node(unsigned long addr)
785 return -1; 785 return -1;
786} 786}
787 787
788static unsigned long long nid_range(unsigned long long start, 788u64 memblock_nid_range(u64 start, u64 end, int *nid)
789 unsigned long long end, int *nid)
790{ 789{
791 *nid = find_node(start); 790 *nid = find_node(start);
792 start += PAGE_SIZE; 791 start += PAGE_SIZE;
@@ -804,8 +803,7 @@ static unsigned long long nid_range(unsigned long long start,
804 return start; 803 return start;
805} 804}
806#else 805#else
807static unsigned long long nid_range(unsigned long long start, 806u64 memblock_nid_range(u64 start, u64 end, int *nid)
808 unsigned long long end, int *nid)
809{ 807{
810 *nid = 0; 808 *nid = 0;
811 return end; 809 return end;
@@ -822,8 +820,7 @@ static void __init allocate_node_data(int nid)
822 struct pglist_data *p; 820 struct pglist_data *p;
823 821
824#ifdef CONFIG_NEED_MULTIPLE_NODES 822#ifdef CONFIG_NEED_MULTIPLE_NODES
825 paddr = memblock_alloc_nid(sizeof(struct pglist_data), 823 paddr = memblock_alloc_try_nid(sizeof(struct pglist_data), SMP_CACHE_BYTES, nid);
826 SMP_CACHE_BYTES, nid, nid_range);
827 if (!paddr) { 824 if (!paddr) {
828 prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid); 825 prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid);
829 prom_halt(); 826 prom_halt();
@@ -843,8 +840,7 @@ static void __init allocate_node_data(int nid)
843 if (p->node_spanned_pages) { 840 if (p->node_spanned_pages) {
844 num_pages = bootmem_bootmap_pages(p->node_spanned_pages); 841 num_pages = bootmem_bootmap_pages(p->node_spanned_pages);
845 842
846 paddr = memblock_alloc_nid(num_pages << PAGE_SHIFT, PAGE_SIZE, nid, 843 paddr = memblock_alloc_try_nid(num_pages << PAGE_SHIFT, PAGE_SIZE, nid);
847 nid_range);
848 if (!paddr) { 844 if (!paddr) {
849 prom_printf("Cannot allocate bootmap for nid[%d]\n", 845 prom_printf("Cannot allocate bootmap for nid[%d]\n",
850 nid); 846 nid);
@@ -972,19 +968,19 @@ int of_node_to_nid(struct device_node *dp)
972 968
973static void __init add_node_ranges(void) 969static void __init add_node_ranges(void)
974{ 970{
975 int i; 971 struct memblock_region *reg;
976 972
977 for (i = 0; i < memblock.memory.cnt; i++) { 973 for_each_memblock(memory, reg) {
978 unsigned long size = memblock_size_bytes(&memblock.memory, i); 974 unsigned long size = reg->size;
979 unsigned long start, end; 975 unsigned long start, end;
980 976
981 start = memblock.memory.region[i].base; 977 start = reg->base;
982 end = start + size; 978 end = start + size;
983 while (start < end) { 979 while (start < end) {
984 unsigned long this_end; 980 unsigned long this_end;
985 int nid; 981 int nid;
986 982
987 this_end = nid_range(start, end, &nid); 983 this_end = memblock_nid_range(start, end, &nid);
988 984
989 numadbg("Adding active range nid[%d] " 985 numadbg("Adding active range nid[%d] "
990 "start[%lx] end[%lx]\n", 986 "start[%lx] end[%lx]\n",
@@ -1281,7 +1277,7 @@ static void __init bootmem_init_nonnuma(void)
1281{ 1277{
1282 unsigned long top_of_ram = memblock_end_of_DRAM(); 1278 unsigned long top_of_ram = memblock_end_of_DRAM();
1283 unsigned long total_ram = memblock_phys_mem_size(); 1279 unsigned long total_ram = memblock_phys_mem_size();
1284 unsigned int i; 1280 struct memblock_region *reg;
1285 1281
1286 numadbg("bootmem_init_nonnuma()\n"); 1282 numadbg("bootmem_init_nonnuma()\n");
1287 1283
@@ -1292,15 +1288,14 @@ static void __init bootmem_init_nonnuma(void)
1292 1288
1293 init_node_masks_nonnuma(); 1289 init_node_masks_nonnuma();
1294 1290
1295 for (i = 0; i < memblock.memory.cnt; i++) { 1291 for_each_memblock(memory, reg) {
1296 unsigned long size = memblock_size_bytes(&memblock.memory, i);
1297 unsigned long start_pfn, end_pfn; 1292 unsigned long start_pfn, end_pfn;
1298 1293
1299 if (!size) 1294 if (!reg->size)
1300 continue; 1295 continue;
1301 1296
1302 start_pfn = memblock.memory.region[i].base >> PAGE_SHIFT; 1297 start_pfn = memblock_region_memory_base_pfn(reg);
1303 end_pfn = start_pfn + memblock_size_pages(&memblock.memory, i); 1298 end_pfn = memblock_region_memory_end_pfn(reg);
1304 add_active_range(0, start_pfn, end_pfn); 1299 add_active_range(0, start_pfn, end_pfn);
1305 } 1300 }
1306 1301
@@ -1318,7 +1313,7 @@ static void __init reserve_range_in_node(int nid, unsigned long start,
1318 unsigned long this_end; 1313 unsigned long this_end;
1319 int n; 1314 int n;
1320 1315
1321 this_end = nid_range(start, end, &n); 1316 this_end = memblock_nid_range(start, end, &n);
1322 if (n == nid) { 1317 if (n == nid) {
1323 numadbg(" MATCH reserving range [%lx:%lx]\n", 1318 numadbg(" MATCH reserving range [%lx:%lx]\n",
1324 start, this_end); 1319 start, this_end);
@@ -1334,17 +1329,12 @@ static void __init reserve_range_in_node(int nid, unsigned long start,
1334 1329
1335static void __init trim_reserved_in_node(int nid) 1330static void __init trim_reserved_in_node(int nid)
1336{ 1331{
1337 int i; 1332 struct memblock_region *reg;
1338 1333
1339 numadbg(" trim_reserved_in_node(%d)\n", nid); 1334 numadbg(" trim_reserved_in_node(%d)\n", nid);
1340 1335
1341 for (i = 0; i < memblock.reserved.cnt; i++) { 1336 for_each_memblock(reserved, reg)
1342 unsigned long start = memblock.reserved.region[i].base; 1337 reserve_range_in_node(nid, reg->base, reg->base + reg->size);
1343 unsigned long size = memblock_size_bytes(&memblock.reserved, i);
1344 unsigned long end = start + size;
1345
1346 reserve_range_in_node(nid, start, end);
1347 }
1348} 1338}
1349 1339
1350static void __init bootmem_init_one_node(int nid) 1340static void __init bootmem_init_one_node(int nid)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 7ab9db88ab6a..dfabfefc21c4 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -28,6 +28,7 @@ config X86
28 select HAVE_IRQ_WORK 28 select HAVE_IRQ_WORK
29 select HAVE_IOREMAP_PROT 29 select HAVE_IOREMAP_PROT
30 select HAVE_KPROBES 30 select HAVE_KPROBES
31 select HAVE_MEMBLOCK
31 select ARCH_WANT_OPTIONAL_GPIOLIB 32 select ARCH_WANT_OPTIONAL_GPIOLIB
32 select ARCH_WANT_FRAME_POINTERS 33 select ARCH_WANT_FRAME_POINTERS
33 select HAVE_DMA_ATTRS 34 select HAVE_DMA_ATTRS
@@ -201,9 +202,6 @@ config ARCH_SUPPORTS_OPTIMIZED_INLINING
201config ARCH_SUPPORTS_DEBUG_PAGEALLOC 202config ARCH_SUPPORTS_DEBUG_PAGEALLOC
202 def_bool y 203 def_bool y
203 204
204config HAVE_EARLY_RES
205 def_bool y
206
207config HAVE_INTEL_TXT 205config HAVE_INTEL_TXT
208 def_bool y 206 def_bool y
209 depends on EXPERIMENTAL && DMAR && ACPI 207 depends on EXPERIMENTAL && DMAR && ACPI
@@ -548,16 +546,7 @@ config PARAVIRT_DEBUG
548 a paravirt_op is missing when it is called. 546 a paravirt_op is missing when it is called.
549 547
550config NO_BOOTMEM 548config NO_BOOTMEM
551 default y 549 def_bool y
552 bool "Disable Bootmem code"
553 ---help---
554 Use early_res directly instead of bootmem before slab is ready.
555 - allocator (buddy) [generic]
556 - early allocator (bootmem) [generic]
557 - very early allocator (reserve_early*()) [x86]
558 - very very early allocator (early brk model) [x86]
559 So reduce one layer between early allocator to final allocator
560
561 550
562config MEMTEST 551config MEMTEST
563 bool "Memtest" 552 bool "Memtest"
diff --git a/arch/x86/include/asm/e820.h b/arch/x86/include/asm/e820.h
index ec8a52d14ab1..5be1542fbfaf 100644
--- a/arch/x86/include/asm/e820.h
+++ b/arch/x86/include/asm/e820.h
@@ -112,23 +112,13 @@ static inline void early_memtest(unsigned long start, unsigned long end)
112} 112}
113#endif 113#endif
114 114
115extern unsigned long end_user_pfn;
116
117extern u64 find_e820_area(u64 start, u64 end, u64 size, u64 align);
118extern u64 find_e820_area_size(u64 start, u64 *sizep, u64 align);
119extern u64 early_reserve_e820(u64 startt, u64 sizet, u64 align);
120#include <linux/early_res.h>
121
122extern unsigned long e820_end_of_ram_pfn(void); 115extern unsigned long e820_end_of_ram_pfn(void);
123extern unsigned long e820_end_of_low_ram_pfn(void); 116extern unsigned long e820_end_of_low_ram_pfn(void);
124extern int e820_find_active_region(const struct e820entry *ei, 117extern u64 early_reserve_e820(u64 startt, u64 sizet, u64 align);
125 unsigned long start_pfn, 118
126 unsigned long last_pfn, 119void memblock_x86_fill(void);
127 unsigned long *ei_startpfn, 120void memblock_find_dma_reserve(void);
128 unsigned long *ei_endpfn); 121
129extern void e820_register_active_regions(int nid, unsigned long start_pfn,
130 unsigned long end_pfn);
131extern u64 e820_hole_size(u64 start, u64 end);
132extern void finish_e820_parsing(void); 122extern void finish_e820_parsing(void);
133extern void e820_reserve_resources(void); 123extern void e820_reserve_resources(void);
134extern void e820_reserve_resources_late(void); 124extern void e820_reserve_resources_late(void);
diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index 8406ed7f9926..8e4a16508d4e 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -90,7 +90,7 @@ extern void __iomem *efi_ioremap(unsigned long addr, unsigned long size,
90#endif /* CONFIG_X86_32 */ 90#endif /* CONFIG_X86_32 */
91 91
92extern int add_efi_memmap; 92extern int add_efi_memmap;
93extern void efi_reserve_early(void); 93extern void efi_memblock_x86_reserve_range(void);
94extern void efi_call_phys_prelog(void); 94extern void efi_call_phys_prelog(void);
95extern void efi_call_phys_epilog(void); 95extern void efi_call_phys_epilog(void);
96 96
diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index 6a45ec41ec26..f0203f4791a8 100644
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -349,6 +349,7 @@ extern void __iomem *early_memremap(resource_size_t phys_addr,
349 unsigned long size); 349 unsigned long size);
350extern void early_iounmap(void __iomem *addr, unsigned long size); 350extern void early_iounmap(void __iomem *addr, unsigned long size);
351extern void fixup_early_ioremap(void); 351extern void fixup_early_ioremap(void);
352extern bool is_early_ioremap_ptep(pte_t *ptep);
352 353
353#define IO_SPACE_LIMIT 0xffff 354#define IO_SPACE_LIMIT 0xffff
354 355
diff --git a/arch/x86/include/asm/memblock.h b/arch/x86/include/asm/memblock.h
new file mode 100644
index 000000000000..19ae14ba6978
--- /dev/null
+++ b/arch/x86/include/asm/memblock.h
@@ -0,0 +1,23 @@
1#ifndef _X86_MEMBLOCK_H
2#define _X86_MEMBLOCK_H
3
4#define ARCH_DISCARD_MEMBLOCK
5
6u64 memblock_x86_find_in_range_size(u64 start, u64 *sizep, u64 align);
7void memblock_x86_to_bootmem(u64 start, u64 end);
8
9void memblock_x86_reserve_range(u64 start, u64 end, char *name);
10void memblock_x86_free_range(u64 start, u64 end);
11struct range;
12int __get_free_all_memory_range(struct range **range, int nodeid,
13 unsigned long start_pfn, unsigned long end_pfn);
14int get_free_all_memory_range(struct range **rangep, int nodeid);
15
16void memblock_x86_register_active_regions(int nid, unsigned long start_pfn,
17 unsigned long last_pfn);
18u64 memblock_x86_hole_size(u64 start, u64 end);
19u64 memblock_x86_find_in_range_node(int nid, u64 start, u64 end, u64 size, u64 align);
20u64 memblock_x86_free_memory_in_range(u64 addr, u64 limit);
21u64 memblock_x86_memory_in_range(u64 addr, u64 limit);
22
23#endif
diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index 33cec152070d..e1252074ea40 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -7,6 +7,7 @@
7 7
8#include <linux/acpi.h> 8#include <linux/acpi.h>
9#include <linux/bootmem.h> 9#include <linux/bootmem.h>
10#include <linux/memblock.h>
10#include <linux/dmi.h> 11#include <linux/dmi.h>
11#include <linux/cpumask.h> 12#include <linux/cpumask.h>
12#include <asm/segment.h> 13#include <asm/segment.h>
@@ -125,7 +126,7 @@ void acpi_restore_state_mem(void)
125 */ 126 */
126void __init acpi_reserve_wakeup_memory(void) 127void __init acpi_reserve_wakeup_memory(void)
127{ 128{
128 unsigned long mem; 129 phys_addr_t mem;
129 130
130 if ((&wakeup_code_end - &wakeup_code_start) > WAKEUP_SIZE) { 131 if ((&wakeup_code_end - &wakeup_code_start) > WAKEUP_SIZE) {
131 printk(KERN_ERR 132 printk(KERN_ERR
@@ -133,15 +134,15 @@ void __init acpi_reserve_wakeup_memory(void)
133 return; 134 return;
134 } 135 }
135 136
136 mem = find_e820_area(0, 1<<20, WAKEUP_SIZE, PAGE_SIZE); 137 mem = memblock_find_in_range(0, 1<<20, WAKEUP_SIZE, PAGE_SIZE);
137 138
138 if (mem == -1L) { 139 if (mem == MEMBLOCK_ERROR) {
139 printk(KERN_ERR "ACPI: Cannot allocate lowmem, S3 disabled.\n"); 140 printk(KERN_ERR "ACPI: Cannot allocate lowmem, S3 disabled.\n");
140 return; 141 return;
141 } 142 }
142 acpi_realmode = (unsigned long) phys_to_virt(mem); 143 acpi_realmode = (unsigned long) phys_to_virt(mem);
143 acpi_wakeup_address = mem; 144 acpi_wakeup_address = mem;
144 reserve_early(mem, mem + WAKEUP_SIZE, "ACPI WAKEUP"); 145 memblock_x86_reserve_range(mem, mem + WAKEUP_SIZE, "ACPI WAKEUP");
145} 146}
146 147
147 148
diff --git a/arch/x86/kernel/apic/numaq_32.c b/arch/x86/kernel/apic/numaq_32.c
index 3e28401f161c..960f26ab5c9f 100644
--- a/arch/x86/kernel/apic/numaq_32.c
+++ b/arch/x86/kernel/apic/numaq_32.c
@@ -26,6 +26,7 @@
26#include <linux/nodemask.h> 26#include <linux/nodemask.h>
27#include <linux/topology.h> 27#include <linux/topology.h>
28#include <linux/bootmem.h> 28#include <linux/bootmem.h>
29#include <linux/memblock.h>
29#include <linux/threads.h> 30#include <linux/threads.h>
30#include <linux/cpumask.h> 31#include <linux/cpumask.h>
31#include <linux/kernel.h> 32#include <linux/kernel.h>
@@ -88,7 +89,7 @@ static inline void numaq_register_node(int node, struct sys_cfg_data *scd)
88 node_end_pfn[node] = 89 node_end_pfn[node] =
89 MB_TO_PAGES(eq->hi_shrd_mem_start + eq->hi_shrd_mem_size); 90 MB_TO_PAGES(eq->hi_shrd_mem_start + eq->hi_shrd_mem_size);
90 91
91 e820_register_active_regions(node, node_start_pfn[node], 92 memblock_x86_register_active_regions(node, node_start_pfn[node],
92 node_end_pfn[node]); 93 node_end_pfn[node]);
93 94
94 memory_present(node, node_start_pfn[node], node_end_pfn[node]); 95 memory_present(node, node_start_pfn[node], node_end_pfn[node]);
diff --git a/arch/x86/kernel/check.c b/arch/x86/kernel/check.c
index fc999e6fc46a..13a389179514 100644
--- a/arch/x86/kernel/check.c
+++ b/arch/x86/kernel/check.c
@@ -2,7 +2,8 @@
2#include <linux/sched.h> 2#include <linux/sched.h>
3#include <linux/kthread.h> 3#include <linux/kthread.h>
4#include <linux/workqueue.h> 4#include <linux/workqueue.h>
5#include <asm/e820.h> 5#include <linux/memblock.h>
6
6#include <asm/proto.h> 7#include <asm/proto.h>
7 8
8/* 9/*
@@ -18,10 +19,12 @@ static int __read_mostly memory_corruption_check = -1;
18static unsigned __read_mostly corruption_check_size = 64*1024; 19static unsigned __read_mostly corruption_check_size = 64*1024;
19static unsigned __read_mostly corruption_check_period = 60; /* seconds */ 20static unsigned __read_mostly corruption_check_period = 60; /* seconds */
20 21
21static struct e820entry scan_areas[MAX_SCAN_AREAS]; 22static struct scan_area {
23 u64 addr;
24 u64 size;
25} scan_areas[MAX_SCAN_AREAS];
22static int num_scan_areas; 26static int num_scan_areas;
23 27
24
25static __init int set_corruption_check(char *arg) 28static __init int set_corruption_check(char *arg)
26{ 29{
27 char *end; 30 char *end;
@@ -81,9 +84,9 @@ void __init setup_bios_corruption_check(void)
81 84
82 while (addr < corruption_check_size && num_scan_areas < MAX_SCAN_AREAS) { 85 while (addr < corruption_check_size && num_scan_areas < MAX_SCAN_AREAS) {
83 u64 size; 86 u64 size;
84 addr = find_e820_area_size(addr, &size, PAGE_SIZE); 87 addr = memblock_x86_find_in_range_size(addr, &size, PAGE_SIZE);
85 88
86 if (!(addr + 1)) 89 if (addr == MEMBLOCK_ERROR)
87 break; 90 break;
88 91
89 if (addr >= corruption_check_size) 92 if (addr >= corruption_check_size)
@@ -92,7 +95,7 @@ void __init setup_bios_corruption_check(void)
92 if ((addr + size) > corruption_check_size) 95 if ((addr + size) > corruption_check_size)
93 size = corruption_check_size - addr; 96 size = corruption_check_size - addr;
94 97
95 e820_update_range(addr, size, E820_RAM, E820_RESERVED); 98 memblock_x86_reserve_range(addr, addr + size, "SCAN RAM");
96 scan_areas[num_scan_areas].addr = addr; 99 scan_areas[num_scan_areas].addr = addr;
97 scan_areas[num_scan_areas].size = size; 100 scan_areas[num_scan_areas].size = size;
98 num_scan_areas++; 101 num_scan_areas++;
@@ -105,7 +108,6 @@ void __init setup_bios_corruption_check(void)
105 108
106 printk(KERN_INFO "Scanning %d areas for low memory corruption\n", 109 printk(KERN_INFO "Scanning %d areas for low memory corruption\n",
107 num_scan_areas); 110 num_scan_areas);
108 update_e820();
109} 111}
110 112
111 113
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
index 0d6fc71bedb1..0c2b7ef7a34d 100644
--- a/arch/x86/kernel/e820.c
+++ b/arch/x86/kernel/e820.c
@@ -15,6 +15,7 @@
15#include <linux/pfn.h> 15#include <linux/pfn.h>
16#include <linux/suspend.h> 16#include <linux/suspend.h>
17#include <linux/firmware-map.h> 17#include <linux/firmware-map.h>
18#include <linux/memblock.h>
18 19
19#include <asm/e820.h> 20#include <asm/e820.h>
20#include <asm/proto.h> 21#include <asm/proto.h>
@@ -738,73 +739,7 @@ core_initcall(e820_mark_nvs_memory);
738#endif 739#endif
739 740
740/* 741/*
741 * Find a free area with specified alignment in a specific range. 742 * pre allocated 4k and reserved it in memblock and e820_saved
742 */
743u64 __init find_e820_area(u64 start, u64 end, u64 size, u64 align)
744{
745 int i;
746
747 for (i = 0; i < e820.nr_map; i++) {
748 struct e820entry *ei = &e820.map[i];
749 u64 addr;
750 u64 ei_start, ei_last;
751
752 if (ei->type != E820_RAM)
753 continue;
754
755 ei_last = ei->addr + ei->size;
756 ei_start = ei->addr;
757 addr = find_early_area(ei_start, ei_last, start, end,
758 size, align);
759
760 if (addr != -1ULL)
761 return addr;
762 }
763 return -1ULL;
764}
765
766u64 __init find_fw_memmap_area(u64 start, u64 end, u64 size, u64 align)
767{
768 return find_e820_area(start, end, size, align);
769}
770
771u64 __init get_max_mapped(void)
772{
773 u64 end = max_pfn_mapped;
774
775 end <<= PAGE_SHIFT;
776
777 return end;
778}
779/*
780 * Find next free range after *start
781 */
782u64 __init find_e820_area_size(u64 start, u64 *sizep, u64 align)
783{
784 int i;
785
786 for (i = 0; i < e820.nr_map; i++) {
787 struct e820entry *ei = &e820.map[i];
788 u64 addr;
789 u64 ei_start, ei_last;
790
791 if (ei->type != E820_RAM)
792 continue;
793
794 ei_last = ei->addr + ei->size;
795 ei_start = ei->addr;
796 addr = find_early_area_size(ei_start, ei_last, start,
797 sizep, align);
798
799 if (addr != -1ULL)
800 return addr;
801 }
802
803 return -1ULL;
804}
805
806/*
807 * pre allocated 4k and reserved it in e820
808 */ 743 */
809u64 __init early_reserve_e820(u64 startt, u64 sizet, u64 align) 744u64 __init early_reserve_e820(u64 startt, u64 sizet, u64 align)
810{ 745{
@@ -813,8 +748,8 @@ u64 __init early_reserve_e820(u64 startt, u64 sizet, u64 align)
813 u64 start; 748 u64 start;
814 749
815 for (start = startt; ; start += size) { 750 for (start = startt; ; start += size) {
816 start = find_e820_area_size(start, &size, align); 751 start = memblock_x86_find_in_range_size(start, &size, align);
817 if (!(start + 1)) 752 if (start == MEMBLOCK_ERROR)
818 return 0; 753 return 0;
819 if (size >= sizet) 754 if (size >= sizet)
820 break; 755 break;
@@ -830,10 +765,9 @@ u64 __init early_reserve_e820(u64 startt, u64 sizet, u64 align)
830 addr = round_down(start + size - sizet, align); 765 addr = round_down(start + size - sizet, align);
831 if (addr < start) 766 if (addr < start)
832 return 0; 767 return 0;
833 e820_update_range(addr, sizet, E820_RAM, E820_RESERVED); 768 memblock_x86_reserve_range(addr, addr + sizet, "new next");
834 e820_update_range_saved(addr, sizet, E820_RAM, E820_RESERVED); 769 e820_update_range_saved(addr, sizet, E820_RAM, E820_RESERVED);
835 printk(KERN_INFO "update e820 for early_reserve_e820\n"); 770 printk(KERN_INFO "update e820_saved for early_reserve_e820\n");
836 update_e820();
837 update_e820_saved(); 771 update_e820_saved();
838 772
839 return addr; 773 return addr;
@@ -895,74 +829,6 @@ unsigned long __init e820_end_of_low_ram_pfn(void)
895{ 829{
896 return e820_end_pfn(1UL<<(32 - PAGE_SHIFT), E820_RAM); 830 return e820_end_pfn(1UL<<(32 - PAGE_SHIFT), E820_RAM);
897} 831}
898/*
899 * Finds an active region in the address range from start_pfn to last_pfn and
900 * returns its range in ei_startpfn and ei_endpfn for the e820 entry.
901 */
902int __init e820_find_active_region(const struct e820entry *ei,
903 unsigned long start_pfn,
904 unsigned long last_pfn,
905 unsigned long *ei_startpfn,
906 unsigned long *ei_endpfn)
907{
908 u64 align = PAGE_SIZE;
909
910 *ei_startpfn = round_up(ei->addr, align) >> PAGE_SHIFT;
911 *ei_endpfn = round_down(ei->addr + ei->size, align) >> PAGE_SHIFT;
912
913 /* Skip map entries smaller than a page */
914 if (*ei_startpfn >= *ei_endpfn)
915 return 0;
916
917 /* Skip if map is outside the node */
918 if (ei->type != E820_RAM || *ei_endpfn <= start_pfn ||
919 *ei_startpfn >= last_pfn)
920 return 0;
921
922 /* Check for overlaps */
923 if (*ei_startpfn < start_pfn)
924 *ei_startpfn = start_pfn;
925 if (*ei_endpfn > last_pfn)
926 *ei_endpfn = last_pfn;
927
928 return 1;
929}
930
931/* Walk the e820 map and register active regions within a node */
932void __init e820_register_active_regions(int nid, unsigned long start_pfn,
933 unsigned long last_pfn)
934{
935 unsigned long ei_startpfn;
936 unsigned long ei_endpfn;
937 int i;
938
939 for (i = 0; i < e820.nr_map; i++)
940 if (e820_find_active_region(&e820.map[i],
941 start_pfn, last_pfn,
942 &ei_startpfn, &ei_endpfn))
943 add_active_range(nid, ei_startpfn, ei_endpfn);
944}
945
946/*
947 * Find the hole size (in bytes) in the memory range.
948 * @start: starting address of the memory range to scan
949 * @end: ending address of the memory range to scan
950 */
951u64 __init e820_hole_size(u64 start, u64 end)
952{
953 unsigned long start_pfn = start >> PAGE_SHIFT;
954 unsigned long last_pfn = end >> PAGE_SHIFT;
955 unsigned long ei_startpfn, ei_endpfn, ram = 0;
956 int i;
957
958 for (i = 0; i < e820.nr_map; i++) {
959 if (e820_find_active_region(&e820.map[i],
960 start_pfn, last_pfn,
961 &ei_startpfn, &ei_endpfn))
962 ram += ei_endpfn - ei_startpfn;
963 }
964 return end - start - ((u64)ram << PAGE_SHIFT);
965}
966 832
967static void early_panic(char *msg) 833static void early_panic(char *msg)
968{ 834{
@@ -1210,3 +1076,48 @@ void __init setup_memory_map(void)
1210 printk(KERN_INFO "BIOS-provided physical RAM map:\n"); 1076 printk(KERN_INFO "BIOS-provided physical RAM map:\n");
1211 e820_print_map(who); 1077 e820_print_map(who);
1212} 1078}
1079
1080void __init memblock_x86_fill(void)
1081{
1082 int i;
1083 u64 end;
1084
1085 /*
1086 * EFI may have more than 128 entries
1087 * We are safe to enable resizing, beause memblock_x86_fill()
1088 * is rather later for x86
1089 */
1090 memblock_can_resize = 1;
1091
1092 for (i = 0; i < e820.nr_map; i++) {
1093 struct e820entry *ei = &e820.map[i];
1094
1095 end = ei->addr + ei->size;
1096 if (end != (resource_size_t)end)
1097 continue;
1098
1099 if (ei->type != E820_RAM && ei->type != E820_RESERVED_KERN)
1100 continue;
1101
1102 memblock_add(ei->addr, ei->size);
1103 }
1104
1105 memblock_analyze();
1106 memblock_dump_all();
1107}
1108
1109void __init memblock_find_dma_reserve(void)
1110{
1111#ifdef CONFIG_X86_64
1112 u64 free_size_pfn;
1113 u64 mem_size_pfn;
1114 /*
1115 * need to find out used area below MAX_DMA_PFN
1116 * need to use memblock to get free size in [0, MAX_DMA_PFN]
1117 * at first, and assume boot_mem will not take below MAX_DMA_PFN
1118 */
1119 mem_size_pfn = memblock_x86_memory_in_range(0, MAX_DMA_PFN << PAGE_SHIFT) >> PAGE_SHIFT;
1120 free_size_pfn = memblock_x86_free_memory_in_range(0, MAX_DMA_PFN << PAGE_SHIFT) >> PAGE_SHIFT;
1121 set_dma_reserve(mem_size_pfn - free_size_pfn);
1122#endif
1123}
diff --git a/arch/x86/kernel/efi.c b/arch/x86/kernel/efi.c
index c2fa9b8b497e..0fe27d7c6258 100644
--- a/arch/x86/kernel/efi.c
+++ b/arch/x86/kernel/efi.c
@@ -30,6 +30,7 @@
30#include <linux/init.h> 30#include <linux/init.h>
31#include <linux/efi.h> 31#include <linux/efi.h>
32#include <linux/bootmem.h> 32#include <linux/bootmem.h>
33#include <linux/memblock.h>
33#include <linux/spinlock.h> 34#include <linux/spinlock.h>
34#include <linux/uaccess.h> 35#include <linux/uaccess.h>
35#include <linux/time.h> 36#include <linux/time.h>
@@ -275,7 +276,7 @@ static void __init do_add_efi_memmap(void)
275 sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map); 276 sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
276} 277}
277 278
278void __init efi_reserve_early(void) 279void __init efi_memblock_x86_reserve_range(void)
279{ 280{
280 unsigned long pmap; 281 unsigned long pmap;
281 282
@@ -290,7 +291,7 @@ void __init efi_reserve_early(void)
290 boot_params.efi_info.efi_memdesc_size; 291 boot_params.efi_info.efi_memdesc_size;
291 memmap.desc_version = boot_params.efi_info.efi_memdesc_version; 292 memmap.desc_version = boot_params.efi_info.efi_memdesc_version;
292 memmap.desc_size = boot_params.efi_info.efi_memdesc_size; 293 memmap.desc_size = boot_params.efi_info.efi_memdesc_size;
293 reserve_early(pmap, pmap + memmap.nr_map * memmap.desc_size, 294 memblock_x86_reserve_range(pmap, pmap + memmap.nr_map * memmap.desc_size,
294 "EFI memmap"); 295 "EFI memmap");
295} 296}
296 297
diff --git a/arch/x86/kernel/head.c b/arch/x86/kernel/head.c
index 3e66bd364a9d..af0699ba48cf 100644
--- a/arch/x86/kernel/head.c
+++ b/arch/x86/kernel/head.c
@@ -1,5 +1,6 @@
1#include <linux/kernel.h> 1#include <linux/kernel.h>
2#include <linux/init.h> 2#include <linux/init.h>
3#include <linux/memblock.h>
3 4
4#include <asm/setup.h> 5#include <asm/setup.h>
5#include <asm/bios_ebda.h> 6#include <asm/bios_ebda.h>
@@ -51,5 +52,5 @@ void __init reserve_ebda_region(void)
51 lowmem = 0x9f000; 52 lowmem = 0x9f000;
52 53
53 /* reserve all memory between lowmem and the 1MB mark */ 54 /* reserve all memory between lowmem and the 1MB mark */
54 reserve_early_overlap_ok(lowmem, 0x100000, "BIOS reserved"); 55 memblock_x86_reserve_range(lowmem, 0x100000, "* BIOS reserved");
55} 56}
diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c
index 784360c0625c..9a6ca2392170 100644
--- a/arch/x86/kernel/head32.c
+++ b/arch/x86/kernel/head32.c
@@ -8,6 +8,7 @@
8#include <linux/init.h> 8#include <linux/init.h>
9#include <linux/start_kernel.h> 9#include <linux/start_kernel.h>
10#include <linux/mm.h> 10#include <linux/mm.h>
11#include <linux/memblock.h>
11 12
12#include <asm/setup.h> 13#include <asm/setup.h>
13#include <asm/sections.h> 14#include <asm/sections.h>
@@ -30,17 +31,18 @@ static void __init i386_default_early_setup(void)
30 31
31void __init i386_start_kernel(void) 32void __init i386_start_kernel(void)
32{ 33{
34 memblock_init();
35
33#ifdef CONFIG_X86_TRAMPOLINE 36#ifdef CONFIG_X86_TRAMPOLINE
34 /* 37 /*
35 * But first pinch a few for the stack/trampoline stuff 38 * But first pinch a few for the stack/trampoline stuff
36 * FIXME: Don't need the extra page at 4K, but need to fix 39 * FIXME: Don't need the extra page at 4K, but need to fix
37 * trampoline before removing it. (see the GDT stuff) 40 * trampoline before removing it. (see the GDT stuff)
38 */ 41 */
39 reserve_early_overlap_ok(PAGE_SIZE, PAGE_SIZE + PAGE_SIZE, 42 memblock_x86_reserve_range(PAGE_SIZE, PAGE_SIZE + PAGE_SIZE, "EX TRAMPOLINE");
40 "EX TRAMPOLINE");
41#endif 43#endif
42 44
43 reserve_early(__pa_symbol(&_text), __pa_symbol(&__bss_stop), "TEXT DATA BSS"); 45 memblock_x86_reserve_range(__pa_symbol(&_text), __pa_symbol(&__bss_stop), "TEXT DATA BSS");
44 46
45#ifdef CONFIG_BLK_DEV_INITRD 47#ifdef CONFIG_BLK_DEV_INITRD
46 /* Reserve INITRD */ 48 /* Reserve INITRD */
@@ -49,7 +51,7 @@ void __init i386_start_kernel(void)
49 u64 ramdisk_image = boot_params.hdr.ramdisk_image; 51 u64 ramdisk_image = boot_params.hdr.ramdisk_image;
50 u64 ramdisk_size = boot_params.hdr.ramdisk_size; 52 u64 ramdisk_size = boot_params.hdr.ramdisk_size;
51 u64 ramdisk_end = PAGE_ALIGN(ramdisk_image + ramdisk_size); 53 u64 ramdisk_end = PAGE_ALIGN(ramdisk_image + ramdisk_size);
52 reserve_early(ramdisk_image, ramdisk_end, "RAMDISK"); 54 memblock_x86_reserve_range(ramdisk_image, ramdisk_end, "RAMDISK");
53 } 55 }
54#endif 56#endif
55 57
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 7147143fd614..2d2673c28aff 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -12,6 +12,7 @@
12#include <linux/percpu.h> 12#include <linux/percpu.h>
13#include <linux/start_kernel.h> 13#include <linux/start_kernel.h>
14#include <linux/io.h> 14#include <linux/io.h>
15#include <linux/memblock.h>
15 16
16#include <asm/processor.h> 17#include <asm/processor.h>
17#include <asm/proto.h> 18#include <asm/proto.h>
@@ -79,6 +80,8 @@ void __init x86_64_start_kernel(char * real_mode_data)
79 /* Cleanup the over mapped high alias */ 80 /* Cleanup the over mapped high alias */
80 cleanup_highmap(); 81 cleanup_highmap();
81 82
83 max_pfn_mapped = KERNEL_IMAGE_SIZE >> PAGE_SHIFT;
84
82 for (i = 0; i < NUM_EXCEPTION_VECTORS; i++) { 85 for (i = 0; i < NUM_EXCEPTION_VECTORS; i++) {
83#ifdef CONFIG_EARLY_PRINTK 86#ifdef CONFIG_EARLY_PRINTK
84 set_intr_gate(i, &early_idt_handlers[i]); 87 set_intr_gate(i, &early_idt_handlers[i]);
@@ -98,7 +101,9 @@ void __init x86_64_start_reservations(char *real_mode_data)
98{ 101{
99 copy_bootdata(__va(real_mode_data)); 102 copy_bootdata(__va(real_mode_data));
100 103
101 reserve_early(__pa_symbol(&_text), __pa_symbol(&__bss_stop), "TEXT DATA BSS"); 104 memblock_init();
105
106 memblock_x86_reserve_range(__pa_symbol(&_text), __pa_symbol(&__bss_stop), "TEXT DATA BSS");
102 107
103#ifdef CONFIG_BLK_DEV_INITRD 108#ifdef CONFIG_BLK_DEV_INITRD
104 /* Reserve INITRD */ 109 /* Reserve INITRD */
@@ -107,7 +112,7 @@ void __init x86_64_start_reservations(char *real_mode_data)
107 unsigned long ramdisk_image = boot_params.hdr.ramdisk_image; 112 unsigned long ramdisk_image = boot_params.hdr.ramdisk_image;
108 unsigned long ramdisk_size = boot_params.hdr.ramdisk_size; 113 unsigned long ramdisk_size = boot_params.hdr.ramdisk_size;
109 unsigned long ramdisk_end = PAGE_ALIGN(ramdisk_image + ramdisk_size); 114 unsigned long ramdisk_end = PAGE_ALIGN(ramdisk_image + ramdisk_size);
110 reserve_early(ramdisk_image, ramdisk_end, "RAMDISK"); 115 memblock_x86_reserve_range(ramdisk_image, ramdisk_end, "RAMDISK");
111 } 116 }
112#endif 117#endif
113 118
diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c
index d7b6f7fb4fec..9af64d9c4b67 100644
--- a/arch/x86/kernel/mpparse.c
+++ b/arch/x86/kernel/mpparse.c
@@ -11,6 +11,7 @@
11#include <linux/init.h> 11#include <linux/init.h>
12#include <linux/delay.h> 12#include <linux/delay.h>
13#include <linux/bootmem.h> 13#include <linux/bootmem.h>
14#include <linux/memblock.h>
14#include <linux/kernel_stat.h> 15#include <linux/kernel_stat.h>
15#include <linux/mc146818rtc.h> 16#include <linux/mc146818rtc.h>
16#include <linux/bitops.h> 17#include <linux/bitops.h>
@@ -657,7 +658,7 @@ static void __init smp_reserve_memory(struct mpf_intel *mpf)
657{ 658{
658 unsigned long size = get_mpc_size(mpf->physptr); 659 unsigned long size = get_mpc_size(mpf->physptr);
659 660
660 reserve_early_overlap_ok(mpf->physptr, mpf->physptr+size, "MP-table mpc"); 661 memblock_x86_reserve_range(mpf->physptr, mpf->physptr+size, "* MP-table mpc");
661} 662}
662 663
663static int __init smp_scan_config(unsigned long base, unsigned long length) 664static int __init smp_scan_config(unsigned long base, unsigned long length)
@@ -686,7 +687,7 @@ static int __init smp_scan_config(unsigned long base, unsigned long length)
686 mpf, (u64)virt_to_phys(mpf)); 687 mpf, (u64)virt_to_phys(mpf));
687 688
688 mem = virt_to_phys(mpf); 689 mem = virt_to_phys(mpf);
689 reserve_early_overlap_ok(mem, mem + sizeof(*mpf), "MP-table mpf"); 690 memblock_x86_reserve_range(mem, mem + sizeof(*mpf), "* MP-table mpf");
690 if (mpf->physptr) 691 if (mpf->physptr)
691 smp_reserve_memory(mpf); 692 smp_reserve_memory(mpf);
692 693
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index a59f6a6df5e2..420e64197850 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -31,6 +31,7 @@
31#include <linux/apm_bios.h> 31#include <linux/apm_bios.h>
32#include <linux/initrd.h> 32#include <linux/initrd.h>
33#include <linux/bootmem.h> 33#include <linux/bootmem.h>
34#include <linux/memblock.h>
34#include <linux/seq_file.h> 35#include <linux/seq_file.h>
35#include <linux/console.h> 36#include <linux/console.h>
36#include <linux/mca.h> 37#include <linux/mca.h>
@@ -301,7 +302,7 @@ static inline void init_gbpages(void)
301static void __init reserve_brk(void) 302static void __init reserve_brk(void)
302{ 303{
303 if (_brk_end > _brk_start) 304 if (_brk_end > _brk_start)
304 reserve_early(__pa(_brk_start), __pa(_brk_end), "BRK"); 305 memblock_x86_reserve_range(__pa(_brk_start), __pa(_brk_end), "BRK");
305 306
306 /* Mark brk area as locked down and no longer taking any 307 /* Mark brk area as locked down and no longer taking any
307 new allocations */ 308 new allocations */
@@ -323,17 +324,16 @@ static void __init relocate_initrd(void)
323 char *p, *q; 324 char *p, *q;
324 325
325 /* We need to move the initrd down into lowmem */ 326 /* We need to move the initrd down into lowmem */
326 ramdisk_here = find_e820_area(0, end_of_lowmem, area_size, 327 ramdisk_here = memblock_find_in_range(0, end_of_lowmem, area_size,
327 PAGE_SIZE); 328 PAGE_SIZE);
328 329
329 if (ramdisk_here == -1ULL) 330 if (ramdisk_here == MEMBLOCK_ERROR)
330 panic("Cannot find place for new RAMDISK of size %lld\n", 331 panic("Cannot find place for new RAMDISK of size %lld\n",
331 ramdisk_size); 332 ramdisk_size);
332 333
333 /* Note: this includes all the lowmem currently occupied by 334 /* Note: this includes all the lowmem currently occupied by
334 the initrd, we rely on that fact to keep the data intact. */ 335 the initrd, we rely on that fact to keep the data intact. */
335 reserve_early(ramdisk_here, ramdisk_here + area_size, 336 memblock_x86_reserve_range(ramdisk_here, ramdisk_here + area_size, "NEW RAMDISK");
336 "NEW RAMDISK");
337 initrd_start = ramdisk_here + PAGE_OFFSET; 337 initrd_start = ramdisk_here + PAGE_OFFSET;
338 initrd_end = initrd_start + ramdisk_size; 338 initrd_end = initrd_start + ramdisk_size;
339 printk(KERN_INFO "Allocated new RAMDISK: %08llx - %08llx\n", 339 printk(KERN_INFO "Allocated new RAMDISK: %08llx - %08llx\n",
@@ -389,7 +389,7 @@ static void __init reserve_initrd(void)
389 initrd_start = 0; 389 initrd_start = 0;
390 390
391 if (ramdisk_size >= (end_of_lowmem>>1)) { 391 if (ramdisk_size >= (end_of_lowmem>>1)) {
392 free_early(ramdisk_image, ramdisk_end); 392 memblock_x86_free_range(ramdisk_image, ramdisk_end);
393 printk(KERN_ERR "initrd too large to handle, " 393 printk(KERN_ERR "initrd too large to handle, "
394 "disabling initrd\n"); 394 "disabling initrd\n");
395 return; 395 return;
@@ -412,7 +412,7 @@ static void __init reserve_initrd(void)
412 412
413 relocate_initrd(); 413 relocate_initrd();
414 414
415 free_early(ramdisk_image, ramdisk_end); 415 memblock_x86_free_range(ramdisk_image, ramdisk_end);
416} 416}
417#else 417#else
418static void __init reserve_initrd(void) 418static void __init reserve_initrd(void)
@@ -468,7 +468,7 @@ static void __init e820_reserve_setup_data(void)
468 e820_print_map("reserve setup_data"); 468 e820_print_map("reserve setup_data");
469} 469}
470 470
471static void __init reserve_early_setup_data(void) 471static void __init memblock_x86_reserve_range_setup_data(void)
472{ 472{
473 struct setup_data *data; 473 struct setup_data *data;
474 u64 pa_data; 474 u64 pa_data;
@@ -480,7 +480,7 @@ static void __init reserve_early_setup_data(void)
480 while (pa_data) { 480 while (pa_data) {
481 data = early_memremap(pa_data, sizeof(*data)); 481 data = early_memremap(pa_data, sizeof(*data));
482 sprintf(buf, "setup data %x", data->type); 482 sprintf(buf, "setup data %x", data->type);
483 reserve_early(pa_data, pa_data+sizeof(*data)+data->len, buf); 483 memblock_x86_reserve_range(pa_data, pa_data+sizeof(*data)+data->len, buf);
484 pa_data = data->next; 484 pa_data = data->next;
485 early_iounmap(data, sizeof(*data)); 485 early_iounmap(data, sizeof(*data));
486 } 486 }
@@ -501,6 +501,7 @@ static inline unsigned long long get_total_mem(void)
501 return total << PAGE_SHIFT; 501 return total << PAGE_SHIFT;
502} 502}
503 503
504#define DEFAULT_BZIMAGE_ADDR_MAX 0x37FFFFFF
504static void __init reserve_crashkernel(void) 505static void __init reserve_crashkernel(void)
505{ 506{
506 unsigned long long total_mem; 507 unsigned long long total_mem;
@@ -518,23 +519,27 @@ static void __init reserve_crashkernel(void)
518 if (crash_base <= 0) { 519 if (crash_base <= 0) {
519 const unsigned long long alignment = 16<<20; /* 16M */ 520 const unsigned long long alignment = 16<<20; /* 16M */
520 521
521 crash_base = find_e820_area(alignment, ULONG_MAX, crash_size, 522 /*
522 alignment); 523 * kexec want bzImage is below DEFAULT_BZIMAGE_ADDR_MAX
523 if (crash_base == -1ULL) { 524 */
525 crash_base = memblock_find_in_range(alignment,
526 DEFAULT_BZIMAGE_ADDR_MAX, crash_size, alignment);
527
528 if (crash_base == MEMBLOCK_ERROR) {
524 pr_info("crashkernel reservation failed - No suitable area found.\n"); 529 pr_info("crashkernel reservation failed - No suitable area found.\n");
525 return; 530 return;
526 } 531 }
527 } else { 532 } else {
528 unsigned long long start; 533 unsigned long long start;
529 534
530 start = find_e820_area(crash_base, ULONG_MAX, crash_size, 535 start = memblock_find_in_range(crash_base,
531 1<<20); 536 crash_base + crash_size, crash_size, 1<<20);
532 if (start != crash_base) { 537 if (start != crash_base) {
533 pr_info("crashkernel reservation failed - memory is in use.\n"); 538 pr_info("crashkernel reservation failed - memory is in use.\n");
534 return; 539 return;
535 } 540 }
536 } 541 }
537 reserve_early(crash_base, crash_base + crash_size, "CRASH KERNEL"); 542 memblock_x86_reserve_range(crash_base, crash_base + crash_size, "CRASH KERNEL");
538 543
539 printk(KERN_INFO "Reserving %ldMB of memory at %ldMB " 544 printk(KERN_INFO "Reserving %ldMB of memory at %ldMB "
540 "for crashkernel (System RAM: %ldMB)\n", 545 "for crashkernel (System RAM: %ldMB)\n",
@@ -614,7 +619,7 @@ static __init void reserve_ibft_region(void)
614 addr = find_ibft_region(&size); 619 addr = find_ibft_region(&size);
615 620
616 if (size) 621 if (size)
617 reserve_early_overlap_ok(addr, addr + size, "ibft"); 622 memblock_x86_reserve_range(addr, addr + size, "* ibft");
618} 623}
619 624
620static unsigned reserve_low = CONFIG_X86_RESERVE_LOW << 10; 625static unsigned reserve_low = CONFIG_X86_RESERVE_LOW << 10;
@@ -664,6 +669,15 @@ static int __init parse_reservelow(char *p)
664 669
665early_param("reservelow", parse_reservelow); 670early_param("reservelow", parse_reservelow);
666 671
672static u64 __init get_max_mapped(void)
673{
674 u64 end = max_pfn_mapped;
675
676 end <<= PAGE_SHIFT;
677
678 return end;
679}
680
667/* 681/*
668 * Determine if we were loaded by an EFI loader. If so, then we have also been 682 * Determine if we were loaded by an EFI loader. If so, then we have also been
669 * passed the efi memmap, systab, etc., so we should use these data structures 683 * passed the efi memmap, systab, etc., so we should use these data structures
@@ -738,7 +752,7 @@ void __init setup_arch(char **cmdline_p)
738#endif 752#endif
739 4)) { 753 4)) {
740 efi_enabled = 1; 754 efi_enabled = 1;
741 efi_reserve_early(); 755 efi_memblock_x86_reserve_range();
742 } 756 }
743#endif 757#endif
744 758
@@ -795,7 +809,7 @@ void __init setup_arch(char **cmdline_p)
795 x86_report_nx(); 809 x86_report_nx();
796 810
797 /* after early param, so could get panic from serial */ 811 /* after early param, so could get panic from serial */
798 reserve_early_setup_data(); 812 memblock_x86_reserve_range_setup_data();
799 813
800 if (acpi_mps_check()) { 814 if (acpi_mps_check()) {
801#ifdef CONFIG_X86_LOCAL_APIC 815#ifdef CONFIG_X86_LOCAL_APIC
@@ -848,8 +862,6 @@ void __init setup_arch(char **cmdline_p)
848 */ 862 */
849 max_pfn = e820_end_of_ram_pfn(); 863 max_pfn = e820_end_of_ram_pfn();
850 864
851 /* preallocate 4k for mptable mpc */
852 early_reserve_e820_mpc_new();
853 /* update e820 for memory not covered by WB MTRRs */ 865 /* update e820 for memory not covered by WB MTRRs */
854 mtrr_bp_init(); 866 mtrr_bp_init();
855 if (mtrr_trim_uncached_memory(max_pfn)) 867 if (mtrr_trim_uncached_memory(max_pfn))
@@ -871,18 +883,8 @@ void __init setup_arch(char **cmdline_p)
871 max_low_pfn = max_pfn; 883 max_low_pfn = max_pfn;
872 884
873 high_memory = (void *)__va(max_pfn * PAGE_SIZE - 1) + 1; 885 high_memory = (void *)__va(max_pfn * PAGE_SIZE - 1) + 1;
874 max_pfn_mapped = KERNEL_IMAGE_SIZE >> PAGE_SHIFT;
875#endif
876
877#ifdef CONFIG_X86_CHECK_BIOS_CORRUPTION
878 setup_bios_corruption_check();
879#endif 886#endif
880 887
881 printk(KERN_DEBUG "initial memory mapped : 0 - %08lx\n",
882 max_pfn_mapped<<PAGE_SHIFT);
883
884 reserve_brk();
885
886 /* 888 /*
887 * Find and reserve possible boot-time SMP configuration: 889 * Find and reserve possible boot-time SMP configuration:
888 */ 890 */
@@ -890,6 +892,26 @@ void __init setup_arch(char **cmdline_p)
890 892
891 reserve_ibft_region(); 893 reserve_ibft_region();
892 894
895 /*
896 * Need to conclude brk, before memblock_x86_fill()
897 * it could use memblock_find_in_range, could overlap with
898 * brk area.
899 */
900 reserve_brk();
901
902 memblock.current_limit = get_max_mapped();
903 memblock_x86_fill();
904
905 /* preallocate 4k for mptable mpc */
906 early_reserve_e820_mpc_new();
907
908#ifdef CONFIG_X86_CHECK_BIOS_CORRUPTION
909 setup_bios_corruption_check();
910#endif
911
912 printk(KERN_DEBUG "initial memory mapped : 0 - %08lx\n",
913 max_pfn_mapped<<PAGE_SHIFT);
914
893 reserve_trampoline_memory(); 915 reserve_trampoline_memory();
894 916
895#ifdef CONFIG_ACPI_SLEEP 917#ifdef CONFIG_ACPI_SLEEP
@@ -913,6 +935,7 @@ void __init setup_arch(char **cmdline_p)
913 max_low_pfn = max_pfn; 935 max_low_pfn = max_pfn;
914 } 936 }
915#endif 937#endif
938 memblock.current_limit = get_max_mapped();
916 939
917 /* 940 /*
918 * NOTE: On x86-32, only from this point on, fixmaps are ready for use. 941 * NOTE: On x86-32, only from this point on, fixmaps are ready for use.
@@ -951,10 +974,7 @@ void __init setup_arch(char **cmdline_p)
951#endif 974#endif
952 975
953 initmem_init(0, max_pfn, acpi, k8); 976 initmem_init(0, max_pfn, acpi, k8);
954#ifndef CONFIG_NO_BOOTMEM 977 memblock_find_dma_reserve();
955 early_res_to_bootmem(0, max_low_pfn<<PAGE_SHIFT);
956#endif
957
958 dma32_reserve_bootmem(); 978 dma32_reserve_bootmem();
959 979
960#ifdef CONFIG_KVM_CLOCK 980#ifdef CONFIG_KVM_CLOCK
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index 2335c15c93a4..002b79685f73 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -131,13 +131,7 @@ static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align)
131 131
132static void __init pcpu_fc_free(void *ptr, size_t size) 132static void __init pcpu_fc_free(void *ptr, size_t size)
133{ 133{
134#ifdef CONFIG_NO_BOOTMEM
135 u64 start = __pa(ptr);
136 u64 end = start + size;
137 free_early_partial(start, end);
138#else
139 free_bootmem(__pa(ptr), size); 134 free_bootmem(__pa(ptr), size);
140#endif
141} 135}
142 136
143static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) 137static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
diff --git a/arch/x86/kernel/trampoline.c b/arch/x86/kernel/trampoline.c
index e2a595257390..4c3da5674e67 100644
--- a/arch/x86/kernel/trampoline.c
+++ b/arch/x86/kernel/trampoline.c
@@ -1,8 +1,8 @@
1#include <linux/io.h> 1#include <linux/io.h>
2#include <linux/memblock.h>
2 3
3#include <asm/trampoline.h> 4#include <asm/trampoline.h>
4#include <asm/pgtable.h> 5#include <asm/pgtable.h>
5#include <asm/e820.h>
6 6
7#if defined(CONFIG_X86_64) && defined(CONFIG_ACPI_SLEEP) 7#if defined(CONFIG_X86_64) && defined(CONFIG_ACPI_SLEEP)
8#define __trampinit 8#define __trampinit
@@ -17,15 +17,15 @@ unsigned char *__trampinitdata trampoline_base;
17 17
18void __init reserve_trampoline_memory(void) 18void __init reserve_trampoline_memory(void)
19{ 19{
20 unsigned long mem; 20 phys_addr_t mem;
21 21
22 /* Has to be in very low memory so we can execute real-mode AP code. */ 22 /* Has to be in very low memory so we can execute real-mode AP code. */
23 mem = find_e820_area(0, 1<<20, TRAMPOLINE_SIZE, PAGE_SIZE); 23 mem = memblock_find_in_range(0, 1<<20, TRAMPOLINE_SIZE, PAGE_SIZE);
24 if (mem == -1L) 24 if (mem == MEMBLOCK_ERROR)
25 panic("Cannot allocate trampoline\n"); 25 panic("Cannot allocate trampoline\n");
26 26
27 trampoline_base = __va(mem); 27 trampoline_base = __va(mem);
28 reserve_early(mem, mem + TRAMPOLINE_SIZE, "TRAMPOLINE"); 28 memblock_x86_reserve_range(mem, mem + TRAMPOLINE_SIZE, "TRAMPOLINE");
29} 29}
30 30
31/* 31/*
diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
index a4c768397baa..55543397a8a7 100644
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -26,4 +26,6 @@ obj-$(CONFIG_NUMA) += numa.o numa_$(BITS).o
26obj-$(CONFIG_K8_NUMA) += k8topology_64.o 26obj-$(CONFIG_K8_NUMA) += k8topology_64.o
27obj-$(CONFIG_ACPI_NUMA) += srat_$(BITS).o 27obj-$(CONFIG_ACPI_NUMA) += srat_$(BITS).o
28 28
29obj-$(CONFIG_HAVE_MEMBLOCK) += memblock.o
30
29obj-$(CONFIG_MEMTEST) += memtest.o 31obj-$(CONFIG_MEMTEST) += memtest.o
diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index b278535b14aa..c0e28a13de7d 100644
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -2,6 +2,7 @@
2#include <linux/initrd.h> 2#include <linux/initrd.h>
3#include <linux/ioport.h> 3#include <linux/ioport.h>
4#include <linux/swap.h> 4#include <linux/swap.h>
5#include <linux/memblock.h>
5 6
6#include <asm/cacheflush.h> 7#include <asm/cacheflush.h>
7#include <asm/e820.h> 8#include <asm/e820.h>
@@ -33,6 +34,7 @@ static void __init find_early_table_space(unsigned long end, int use_pse,
33 int use_gbpages) 34 int use_gbpages)
34{ 35{
35 unsigned long puds, pmds, ptes, tables, start; 36 unsigned long puds, pmds, ptes, tables, start;
37 phys_addr_t base;
36 38
37 puds = (end + PUD_SIZE - 1) >> PUD_SHIFT; 39 puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
38 tables = roundup(puds * sizeof(pud_t), PAGE_SIZE); 40 tables = roundup(puds * sizeof(pud_t), PAGE_SIZE);
@@ -75,12 +77,12 @@ static void __init find_early_table_space(unsigned long end, int use_pse,
75#else 77#else
76 start = 0x8000; 78 start = 0x8000;
77#endif 79#endif
78 e820_table_start = find_e820_area(start, max_pfn_mapped<<PAGE_SHIFT, 80 base = memblock_find_in_range(start, max_pfn_mapped<<PAGE_SHIFT,
79 tables, PAGE_SIZE); 81 tables, PAGE_SIZE);
80 if (e820_table_start == -1UL) 82 if (base == MEMBLOCK_ERROR)
81 panic("Cannot find space for the kernel page tables"); 83 panic("Cannot find space for the kernel page tables");
82 84
83 e820_table_start >>= PAGE_SHIFT; 85 e820_table_start = base >> PAGE_SHIFT;
84 e820_table_end = e820_table_start; 86 e820_table_end = e820_table_start;
85 e820_table_top = e820_table_start + (tables >> PAGE_SHIFT); 87 e820_table_top = e820_table_start + (tables >> PAGE_SHIFT);
86 88
@@ -299,7 +301,7 @@ unsigned long __init_refok init_memory_mapping(unsigned long start,
299 __flush_tlb_all(); 301 __flush_tlb_all();
300 302
301 if (!after_bootmem && e820_table_end > e820_table_start) 303 if (!after_bootmem && e820_table_end > e820_table_start)
302 reserve_early(e820_table_start << PAGE_SHIFT, 304 memblock_x86_reserve_range(e820_table_start << PAGE_SHIFT,
303 e820_table_end << PAGE_SHIFT, "PGTABLE"); 305 e820_table_end << PAGE_SHIFT, "PGTABLE");
304 306
305 if (!after_bootmem) 307 if (!after_bootmem)
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 558f2d332076..5d0a6711c282 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -25,6 +25,7 @@
25#include <linux/pfn.h> 25#include <linux/pfn.h>
26#include <linux/poison.h> 26#include <linux/poison.h>
27#include <linux/bootmem.h> 27#include <linux/bootmem.h>
28#include <linux/memblock.h>
28#include <linux/proc_fs.h> 29#include <linux/proc_fs.h>
29#include <linux/memory_hotplug.h> 30#include <linux/memory_hotplug.h>
30#include <linux/initrd.h> 31#include <linux/initrd.h>
@@ -422,49 +423,28 @@ static void __init add_one_highpage_init(struct page *page)
422 totalhigh_pages++; 423 totalhigh_pages++;
423} 424}
424 425
425struct add_highpages_data { 426void __init add_highpages_with_active_regions(int nid,
426 unsigned long start_pfn; 427 unsigned long start_pfn, unsigned long end_pfn)
427 unsigned long end_pfn;
428};
429
430static int __init add_highpages_work_fn(unsigned long start_pfn,
431 unsigned long end_pfn, void *datax)
432{ 428{
433 int node_pfn; 429 struct range *range;
434 struct page *page; 430 int nr_range;
435 unsigned long final_start_pfn, final_end_pfn; 431 int i;
436 struct add_highpages_data *data;
437 432
438 data = (struct add_highpages_data *)datax; 433 nr_range = __get_free_all_memory_range(&range, nid, start_pfn, end_pfn);
439 434
440 final_start_pfn = max(start_pfn, data->start_pfn); 435 for (i = 0; i < nr_range; i++) {
441 final_end_pfn = min(end_pfn, data->end_pfn); 436 struct page *page;
442 if (final_start_pfn >= final_end_pfn) 437 int node_pfn;
443 return 0;
444 438
445 for (node_pfn = final_start_pfn; node_pfn < final_end_pfn; 439 for (node_pfn = range[i].start; node_pfn < range[i].end;
446 node_pfn++) { 440 node_pfn++) {
447 if (!pfn_valid(node_pfn)) 441 if (!pfn_valid(node_pfn))
448 continue; 442 continue;
449 page = pfn_to_page(node_pfn); 443 page = pfn_to_page(node_pfn);
450 add_one_highpage_init(page); 444 add_one_highpage_init(page);
445 }
451 } 446 }
452
453 return 0;
454
455} 447}
456
457void __init add_highpages_with_active_regions(int nid, unsigned long start_pfn,
458 unsigned long end_pfn)
459{
460 struct add_highpages_data data;
461
462 data.start_pfn = start_pfn;
463 data.end_pfn = end_pfn;
464
465 work_with_active_regions(nid, add_highpages_work_fn, &data);
466}
467
468#else 448#else
469static inline void permanent_kmaps_init(pgd_t *pgd_base) 449static inline void permanent_kmaps_init(pgd_t *pgd_base)
470{ 450{
@@ -712,14 +692,14 @@ void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn,
712 highstart_pfn = highend_pfn = max_pfn; 692 highstart_pfn = highend_pfn = max_pfn;
713 if (max_pfn > max_low_pfn) 693 if (max_pfn > max_low_pfn)
714 highstart_pfn = max_low_pfn; 694 highstart_pfn = max_low_pfn;
715 e820_register_active_regions(0, 0, highend_pfn); 695 memblock_x86_register_active_regions(0, 0, highend_pfn);
716 sparse_memory_present_with_active_regions(0); 696 sparse_memory_present_with_active_regions(0);
717 printk(KERN_NOTICE "%ldMB HIGHMEM available.\n", 697 printk(KERN_NOTICE "%ldMB HIGHMEM available.\n",
718 pages_to_mb(highend_pfn - highstart_pfn)); 698 pages_to_mb(highend_pfn - highstart_pfn));
719 num_physpages = highend_pfn; 699 num_physpages = highend_pfn;
720 high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1; 700 high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1;
721#else 701#else
722 e820_register_active_regions(0, 0, max_low_pfn); 702 memblock_x86_register_active_regions(0, 0, max_low_pfn);
723 sparse_memory_present_with_active_regions(0); 703 sparse_memory_present_with_active_regions(0);
724 num_physpages = max_low_pfn; 704 num_physpages = max_low_pfn;
725 high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1; 705 high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1;
@@ -750,68 +730,12 @@ static void __init zone_sizes_init(void)
750 free_area_init_nodes(max_zone_pfns); 730 free_area_init_nodes(max_zone_pfns);
751} 731}
752 732
753#ifndef CONFIG_NO_BOOTMEM
754static unsigned long __init setup_node_bootmem(int nodeid,
755 unsigned long start_pfn,
756 unsigned long end_pfn,
757 unsigned long bootmap)
758{
759 unsigned long bootmap_size;
760
761 /* don't touch min_low_pfn */
762 bootmap_size = init_bootmem_node(NODE_DATA(nodeid),
763 bootmap >> PAGE_SHIFT,
764 start_pfn, end_pfn);
765 printk(KERN_INFO " node %d low ram: %08lx - %08lx\n",
766 nodeid, start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
767 printk(KERN_INFO " node %d bootmap %08lx - %08lx\n",
768 nodeid, bootmap, bootmap + bootmap_size);
769 free_bootmem_with_active_regions(nodeid, end_pfn);
770
771 return bootmap + bootmap_size;
772}
773#endif
774
775void __init setup_bootmem_allocator(void) 733void __init setup_bootmem_allocator(void)
776{ 734{
777#ifndef CONFIG_NO_BOOTMEM
778 int nodeid;
779 unsigned long bootmap_size, bootmap;
780 /*
781 * Initialize the boot-time allocator (with low memory only):
782 */
783 bootmap_size = bootmem_bootmap_pages(max_low_pfn)<<PAGE_SHIFT;
784 bootmap = find_e820_area(0, max_pfn_mapped<<PAGE_SHIFT, bootmap_size,
785 PAGE_SIZE);
786 if (bootmap == -1L)
787 panic("Cannot find bootmem map of size %ld\n", bootmap_size);
788 reserve_early(bootmap, bootmap + bootmap_size, "BOOTMAP");
789#endif
790
791 printk(KERN_INFO " mapped low ram: 0 - %08lx\n", 735 printk(KERN_INFO " mapped low ram: 0 - %08lx\n",
792 max_pfn_mapped<<PAGE_SHIFT); 736 max_pfn_mapped<<PAGE_SHIFT);
793 printk(KERN_INFO " low ram: 0 - %08lx\n", max_low_pfn<<PAGE_SHIFT); 737 printk(KERN_INFO " low ram: 0 - %08lx\n", max_low_pfn<<PAGE_SHIFT);
794 738
795#ifndef CONFIG_NO_BOOTMEM
796 for_each_online_node(nodeid) {
797 unsigned long start_pfn, end_pfn;
798
799#ifdef CONFIG_NEED_MULTIPLE_NODES
800 start_pfn = node_start_pfn[nodeid];
801 end_pfn = node_end_pfn[nodeid];
802 if (start_pfn > max_low_pfn)
803 continue;
804 if (end_pfn > max_low_pfn)
805 end_pfn = max_low_pfn;
806#else
807 start_pfn = 0;
808 end_pfn = max_low_pfn;
809#endif
810 bootmap = setup_node_bootmem(nodeid, start_pfn, end_pfn,
811 bootmap);
812 }
813#endif
814
815 after_bootmem = 1; 739 after_bootmem = 1;
816} 740}
817 741
@@ -1070,8 +994,3 @@ void mark_rodata_ro(void)
1070} 994}
1071#endif 995#endif
1072 996
1073int __init reserve_bootmem_generic(unsigned long phys, unsigned long len,
1074 int flags)
1075{
1076 return reserve_bootmem(phys, len, flags);
1077}
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index c55f900fbf89..84346200e783 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -21,6 +21,7 @@
21#include <linux/initrd.h> 21#include <linux/initrd.h>
22#include <linux/pagemap.h> 22#include <linux/pagemap.h>
23#include <linux/bootmem.h> 23#include <linux/bootmem.h>
24#include <linux/memblock.h>
24#include <linux/proc_fs.h> 25#include <linux/proc_fs.h>
25#include <linux/pci.h> 26#include <linux/pci.h>
26#include <linux/pfn.h> 27#include <linux/pfn.h>
@@ -52,8 +53,6 @@
52#include <asm/init.h> 53#include <asm/init.h>
53#include <linux/bootmem.h> 54#include <linux/bootmem.h>
54 55
55static unsigned long dma_reserve __initdata;
56
57static int __init parse_direct_gbpages_off(char *arg) 56static int __init parse_direct_gbpages_off(char *arg)
58{ 57{
59 direct_gbpages = 0; 58 direct_gbpages = 0;
@@ -617,23 +616,7 @@ kernel_physical_mapping_init(unsigned long start,
617void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn, 616void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn,
618 int acpi, int k8) 617 int acpi, int k8)
619{ 618{
620#ifndef CONFIG_NO_BOOTMEM 619 memblock_x86_register_active_regions(0, start_pfn, end_pfn);
621 unsigned long bootmap_size, bootmap;
622
623 bootmap_size = bootmem_bootmap_pages(end_pfn)<<PAGE_SHIFT;
624 bootmap = find_e820_area(0, end_pfn<<PAGE_SHIFT, bootmap_size,
625 PAGE_SIZE);
626 if (bootmap == -1L)
627 panic("Cannot find bootmem map of size %ld\n", bootmap_size);
628 reserve_early(bootmap, bootmap + bootmap_size, "BOOTMAP");
629 /* don't touch min_low_pfn */
630 bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap >> PAGE_SHIFT,
631 0, end_pfn);
632 e820_register_active_regions(0, start_pfn, end_pfn);
633 free_bootmem_with_active_regions(0, end_pfn);
634#else
635 e820_register_active_regions(0, start_pfn, end_pfn);
636#endif
637} 620}
638#endif 621#endif
639 622
@@ -843,52 +826,6 @@ void mark_rodata_ro(void)
843 826
844#endif 827#endif
845 828
846int __init reserve_bootmem_generic(unsigned long phys, unsigned long len,
847 int flags)
848{
849#ifdef CONFIG_NUMA
850 int nid, next_nid;
851 int ret;
852#endif
853 unsigned long pfn = phys >> PAGE_SHIFT;
854
855 if (pfn >= max_pfn) {
856 /*
857 * This can happen with kdump kernels when accessing
858 * firmware tables:
859 */
860 if (pfn < max_pfn_mapped)
861 return -EFAULT;
862
863 printk(KERN_ERR "reserve_bootmem: illegal reserve %lx %lu\n",
864 phys, len);
865 return -EFAULT;
866 }
867
868 /* Should check here against the e820 map to avoid double free */
869#ifdef CONFIG_NUMA
870 nid = phys_to_nid(phys);
871 next_nid = phys_to_nid(phys + len - 1);
872 if (nid == next_nid)
873 ret = reserve_bootmem_node(NODE_DATA(nid), phys, len, flags);
874 else
875 ret = reserve_bootmem(phys, len, flags);
876
877 if (ret != 0)
878 return ret;
879
880#else
881 reserve_bootmem(phys, len, flags);
882#endif
883
884 if (phys+len <= MAX_DMA_PFN*PAGE_SIZE) {
885 dma_reserve += len / PAGE_SIZE;
886 set_dma_reserve(dma_reserve);
887 }
888
889 return 0;
890}
891
892int kern_addr_valid(unsigned long addr) 829int kern_addr_valid(unsigned long addr)
893{ 830{
894 unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT; 831 unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT;
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index 3ba6e0608c55..0369843511dc 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -362,6 +362,11 @@ static inline pte_t * __init early_ioremap_pte(unsigned long addr)
362 return &bm_pte[pte_index(addr)]; 362 return &bm_pte[pte_index(addr)];
363} 363}
364 364
365bool __init is_early_ioremap_ptep(pte_t *ptep)
366{
367 return ptep >= &bm_pte[0] && ptep < &bm_pte[PAGE_SIZE/sizeof(pte_t)];
368}
369
365static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata; 370static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata;
366 371
367void __init early_ioremap_init(void) 372void __init early_ioremap_init(void)
diff --git a/arch/x86/mm/k8topology_64.c b/arch/x86/mm/k8topology_64.c
index 52d54bfc1ebb..804a3b6c6e14 100644
--- a/arch/x86/mm/k8topology_64.c
+++ b/arch/x86/mm/k8topology_64.c
@@ -11,6 +11,8 @@
11#include <linux/string.h> 11#include <linux/string.h>
12#include <linux/module.h> 12#include <linux/module.h>
13#include <linux/nodemask.h> 13#include <linux/nodemask.h>
14#include <linux/memblock.h>
15
14#include <asm/io.h> 16#include <asm/io.h>
15#include <linux/pci_ids.h> 17#include <linux/pci_ids.h>
16#include <linux/acpi.h> 18#include <linux/acpi.h>
@@ -222,7 +224,7 @@ int __init k8_scan_nodes(void)
222 for_each_node_mask(i, node_possible_map) { 224 for_each_node_mask(i, node_possible_map) {
223 int j; 225 int j;
224 226
225 e820_register_active_regions(i, 227 memblock_x86_register_active_regions(i,
226 nodes[i].start >> PAGE_SHIFT, 228 nodes[i].start >> PAGE_SHIFT,
227 nodes[i].end >> PAGE_SHIFT); 229 nodes[i].end >> PAGE_SHIFT);
228 for (j = apicid_base; j < cores + apicid_base; j++) 230 for (j = apicid_base; j < cores + apicid_base; j++)
diff --git a/arch/x86/mm/memblock.c b/arch/x86/mm/memblock.c
new file mode 100644
index 000000000000..aa1169392b83
--- /dev/null
+++ b/arch/x86/mm/memblock.c
@@ -0,0 +1,348 @@
1#include <linux/kernel.h>
2#include <linux/types.h>
3#include <linux/init.h>
4#include <linux/bitops.h>
5#include <linux/memblock.h>
6#include <linux/bootmem.h>
7#include <linux/mm.h>
8#include <linux/range.h>
9
10/* Check for already reserved areas */
11static bool __init check_with_memblock_reserved_size(u64 *addrp, u64 *sizep, u64 align)
12{
13 struct memblock_region *r;
14 u64 addr = *addrp, last;
15 u64 size = *sizep;
16 bool changed = false;
17
18again:
19 last = addr + size;
20 for_each_memblock(reserved, r) {
21 if (last > r->base && addr < r->base) {
22 size = r->base - addr;
23 changed = true;
24 goto again;
25 }
26 if (last > (r->base + r->size) && addr < (r->base + r->size)) {
27 addr = round_up(r->base + r->size, align);
28 size = last - addr;
29 changed = true;
30 goto again;
31 }
32 if (last <= (r->base + r->size) && addr >= r->base) {
33 *sizep = 0;
34 return false;
35 }
36 }
37 if (changed) {
38 *addrp = addr;
39 *sizep = size;
40 }
41 return changed;
42}
43
44/*
45 * Find next free range after start, and size is returned in *sizep
46 */
47u64 __init memblock_x86_find_in_range_size(u64 start, u64 *sizep, u64 align)
48{
49 struct memblock_region *r;
50
51 for_each_memblock(memory, r) {
52 u64 ei_start = r->base;
53 u64 ei_last = ei_start + r->size;
54 u64 addr;
55
56 addr = round_up(ei_start, align);
57 if (addr < start)
58 addr = round_up(start, align);
59 if (addr >= ei_last)
60 continue;
61 *sizep = ei_last - addr;
62 while (check_with_memblock_reserved_size(&addr, sizep, align))
63 ;
64
65 if (*sizep)
66 return addr;
67 }
68
69 return MEMBLOCK_ERROR;
70}
71
72static __init struct range *find_range_array(int count)
73{
74 u64 end, size, mem;
75 struct range *range;
76
77 size = sizeof(struct range) * count;
78 end = memblock.current_limit;
79
80 mem = memblock_find_in_range(0, end, size, sizeof(struct range));
81 if (mem == MEMBLOCK_ERROR)
82 panic("can not find more space for range array");
83
84 /*
85 * This range is tempoaray, so don't reserve it, it will not be
86 * overlapped because We will not alloccate new buffer before
87 * We discard this one
88 */
89 range = __va(mem);
90 memset(range, 0, size);
91
92 return range;
93}
94
95static void __init memblock_x86_subtract_reserved(struct range *range, int az)
96{
97 u64 final_start, final_end;
98 struct memblock_region *r;
99
100 /* Take out region array itself at first*/
101 memblock_free_reserved_regions();
102
103 memblock_dbg("Subtract (%ld early reservations)\n", memblock.reserved.cnt);
104
105 for_each_memblock(reserved, r) {
106 memblock_dbg(" [%010llx-%010llx]\n", (u64)r->base, (u64)r->base + r->size - 1);
107 final_start = PFN_DOWN(r->base);
108 final_end = PFN_UP(r->base + r->size);
109 if (final_start >= final_end)
110 continue;
111 subtract_range(range, az, final_start, final_end);
112 }
113
114 /* Put region array back ? */
115 memblock_reserve_reserved_regions();
116}
117
118struct count_data {
119 int nr;
120};
121
122static int __init count_work_fn(unsigned long start_pfn,
123 unsigned long end_pfn, void *datax)
124{
125 struct count_data *data = datax;
126
127 data->nr++;
128
129 return 0;
130}
131
132static int __init count_early_node_map(int nodeid)
133{
134 struct count_data data;
135
136 data.nr = 0;
137 work_with_active_regions(nodeid, count_work_fn, &data);
138
139 return data.nr;
140}
141
142int __init __get_free_all_memory_range(struct range **rangep, int nodeid,
143 unsigned long start_pfn, unsigned long end_pfn)
144{
145 int count;
146 struct range *range;
147 int nr_range;
148
149 count = (memblock.reserved.cnt + count_early_node_map(nodeid)) * 2;
150
151 range = find_range_array(count);
152 nr_range = 0;
153
154 /*
155 * Use early_node_map[] and memblock.reserved.region to get range array
156 * at first
157 */
158 nr_range = add_from_early_node_map(range, count, nr_range, nodeid);
159 subtract_range(range, count, 0, start_pfn);
160 subtract_range(range, count, end_pfn, -1ULL);
161
162 memblock_x86_subtract_reserved(range, count);
163 nr_range = clean_sort_range(range, count);
164
165 *rangep = range;
166 return nr_range;
167}
168
169int __init get_free_all_memory_range(struct range **rangep, int nodeid)
170{
171 unsigned long end_pfn = -1UL;
172
173#ifdef CONFIG_X86_32
174 end_pfn = max_low_pfn;
175#endif
176 return __get_free_all_memory_range(rangep, nodeid, 0, end_pfn);
177}
178
179static u64 __init __memblock_x86_memory_in_range(u64 addr, u64 limit, bool get_free)
180{
181 int i, count;
182 struct range *range;
183 int nr_range;
184 u64 final_start, final_end;
185 u64 free_size;
186 struct memblock_region *r;
187
188 count = (memblock.reserved.cnt + memblock.memory.cnt) * 2;
189
190 range = find_range_array(count);
191 nr_range = 0;
192
193 addr = PFN_UP(addr);
194 limit = PFN_DOWN(limit);
195
196 for_each_memblock(memory, r) {
197 final_start = PFN_UP(r->base);
198 final_end = PFN_DOWN(r->base + r->size);
199 if (final_start >= final_end)
200 continue;
201 if (final_start >= limit || final_end <= addr)
202 continue;
203
204 nr_range = add_range(range, count, nr_range, final_start, final_end);
205 }
206 subtract_range(range, count, 0, addr);
207 subtract_range(range, count, limit, -1ULL);
208
209 /* Subtract memblock.reserved.region in range ? */
210 if (!get_free)
211 goto sort_and_count_them;
212 for_each_memblock(reserved, r) {
213 final_start = PFN_DOWN(r->base);
214 final_end = PFN_UP(r->base + r->size);
215 if (final_start >= final_end)
216 continue;
217 if (final_start >= limit || final_end <= addr)
218 continue;
219
220 subtract_range(range, count, final_start, final_end);
221 }
222
223sort_and_count_them:
224 nr_range = clean_sort_range(range, count);
225
226 free_size = 0;
227 for (i = 0; i < nr_range; i++)
228 free_size += range[i].end - range[i].start;
229
230 return free_size << PAGE_SHIFT;
231}
232
233u64 __init memblock_x86_free_memory_in_range(u64 addr, u64 limit)
234{
235 return __memblock_x86_memory_in_range(addr, limit, true);
236}
237
238u64 __init memblock_x86_memory_in_range(u64 addr, u64 limit)
239{
240 return __memblock_x86_memory_in_range(addr, limit, false);
241}
242
243void __init memblock_x86_reserve_range(u64 start, u64 end, char *name)
244{
245 if (start == end)
246 return;
247
248 if (WARN_ONCE(start > end, "memblock_x86_reserve_range: wrong range [%#llx, %#llx)\n", start, end))
249 return;
250
251 memblock_dbg(" memblock_x86_reserve_range: [%#010llx-%#010llx] %16s\n", start, end - 1, name);
252
253 memblock_reserve(start, end - start);
254}
255
256void __init memblock_x86_free_range(u64 start, u64 end)
257{
258 if (start == end)
259 return;
260
261 if (WARN_ONCE(start > end, "memblock_x86_free_range: wrong range [%#llx, %#llx)\n", start, end))
262 return;
263
264 memblock_dbg(" memblock_x86_free_range: [%#010llx-%#010llx]\n", start, end - 1);
265
266 memblock_free(start, end - start);
267}
268
269/*
270 * Need to call this function after memblock_x86_register_active_regions,
271 * so early_node_map[] is filled already.
272 */
273u64 __init memblock_x86_find_in_range_node(int nid, u64 start, u64 end, u64 size, u64 align)
274{
275 u64 addr;
276 addr = find_memory_core_early(nid, size, align, start, end);
277 if (addr != MEMBLOCK_ERROR)
278 return addr;
279
280 /* Fallback, should already have start end within node range */
281 return memblock_find_in_range(start, end, size, align);
282}
283
284/*
285 * Finds an active region in the address range from start_pfn to last_pfn and
286 * returns its range in ei_startpfn and ei_endpfn for the memblock entry.
287 */
288static int __init memblock_x86_find_active_region(const struct memblock_region *ei,
289 unsigned long start_pfn,
290 unsigned long last_pfn,
291 unsigned long *ei_startpfn,
292 unsigned long *ei_endpfn)
293{
294 u64 align = PAGE_SIZE;
295
296 *ei_startpfn = round_up(ei->base, align) >> PAGE_SHIFT;
297 *ei_endpfn = round_down(ei->base + ei->size, align) >> PAGE_SHIFT;
298
299 /* Skip map entries smaller than a page */
300 if (*ei_startpfn >= *ei_endpfn)
301 return 0;
302
303 /* Skip if map is outside the node */
304 if (*ei_endpfn <= start_pfn || *ei_startpfn >= last_pfn)
305 return 0;
306
307 /* Check for overlaps */
308 if (*ei_startpfn < start_pfn)
309 *ei_startpfn = start_pfn;
310 if (*ei_endpfn > last_pfn)
311 *ei_endpfn = last_pfn;
312
313 return 1;
314}
315
316/* Walk the memblock.memory map and register active regions within a node */
317void __init memblock_x86_register_active_regions(int nid, unsigned long start_pfn,
318 unsigned long last_pfn)
319{
320 unsigned long ei_startpfn;
321 unsigned long ei_endpfn;
322 struct memblock_region *r;
323
324 for_each_memblock(memory, r)
325 if (memblock_x86_find_active_region(r, start_pfn, last_pfn,
326 &ei_startpfn, &ei_endpfn))
327 add_active_range(nid, ei_startpfn, ei_endpfn);
328}
329
330/*
331 * Find the hole size (in bytes) in the memory range.
332 * @start: starting address of the memory range to scan
333 * @end: ending address of the memory range to scan
334 */
335u64 __init memblock_x86_hole_size(u64 start, u64 end)
336{
337 unsigned long start_pfn = start >> PAGE_SHIFT;
338 unsigned long last_pfn = end >> PAGE_SHIFT;
339 unsigned long ei_startpfn, ei_endpfn, ram = 0;
340 struct memblock_region *r;
341
342 for_each_memblock(memory, r)
343 if (memblock_x86_find_active_region(r, start_pfn, last_pfn,
344 &ei_startpfn, &ei_endpfn))
345 ram += ei_endpfn - ei_startpfn;
346
347 return end - start - ((u64)ram << PAGE_SHIFT);
348}
diff --git a/arch/x86/mm/memtest.c b/arch/x86/mm/memtest.c
index 18d244f70205..92faf3a1c53e 100644
--- a/arch/x86/mm/memtest.c
+++ b/arch/x86/mm/memtest.c
@@ -6,8 +6,7 @@
6#include <linux/smp.h> 6#include <linux/smp.h>
7#include <linux/init.h> 7#include <linux/init.h>
8#include <linux/pfn.h> 8#include <linux/pfn.h>
9 9#include <linux/memblock.h>
10#include <asm/e820.h>
11 10
12static u64 patterns[] __initdata = { 11static u64 patterns[] __initdata = {
13 0, 12 0,
@@ -35,7 +34,7 @@ static void __init reserve_bad_mem(u64 pattern, u64 start_bad, u64 end_bad)
35 (unsigned long long) pattern, 34 (unsigned long long) pattern,
36 (unsigned long long) start_bad, 35 (unsigned long long) start_bad,
37 (unsigned long long) end_bad); 36 (unsigned long long) end_bad);
38 reserve_early(start_bad, end_bad, "BAD RAM"); 37 memblock_x86_reserve_range(start_bad, end_bad, "BAD RAM");
39} 38}
40 39
41static void __init memtest(u64 pattern, u64 start_phys, u64 size) 40static void __init memtest(u64 pattern, u64 start_phys, u64 size)
@@ -74,7 +73,7 @@ static void __init do_one_pass(u64 pattern, u64 start, u64 end)
74 u64 size = 0; 73 u64 size = 0;
75 74
76 while (start < end) { 75 while (start < end) {
77 start = find_e820_area_size(start, &size, 1); 76 start = memblock_x86_find_in_range_size(start, &size, 1);
78 77
79 /* done ? */ 78 /* done ? */
80 if (start >= end) 79 if (start >= end)
diff --git a/arch/x86/mm/numa_32.c b/arch/x86/mm/numa_32.c
index 809baaaf48b1..84a3e4c9f277 100644
--- a/arch/x86/mm/numa_32.c
+++ b/arch/x86/mm/numa_32.c
@@ -24,6 +24,7 @@
24 24
25#include <linux/mm.h> 25#include <linux/mm.h>
26#include <linux/bootmem.h> 26#include <linux/bootmem.h>
27#include <linux/memblock.h>
27#include <linux/mmzone.h> 28#include <linux/mmzone.h>
28#include <linux/highmem.h> 29#include <linux/highmem.h>
29#include <linux/initrd.h> 30#include <linux/initrd.h>
@@ -120,7 +121,7 @@ int __init get_memcfg_numa_flat(void)
120 121
121 node_start_pfn[0] = 0; 122 node_start_pfn[0] = 0;
122 node_end_pfn[0] = max_pfn; 123 node_end_pfn[0] = max_pfn;
123 e820_register_active_regions(0, 0, max_pfn); 124 memblock_x86_register_active_regions(0, 0, max_pfn);
124 memory_present(0, 0, max_pfn); 125 memory_present(0, 0, max_pfn);
125 node_remap_size[0] = node_memmap_size_bytes(0, 0, max_pfn); 126 node_remap_size[0] = node_memmap_size_bytes(0, 0, max_pfn);
126 127
@@ -161,14 +162,14 @@ static void __init allocate_pgdat(int nid)
161 NODE_DATA(nid) = (pg_data_t *)node_remap_start_vaddr[nid]; 162 NODE_DATA(nid) = (pg_data_t *)node_remap_start_vaddr[nid];
162 else { 163 else {
163 unsigned long pgdat_phys; 164 unsigned long pgdat_phys;
164 pgdat_phys = find_e820_area(min_low_pfn<<PAGE_SHIFT, 165 pgdat_phys = memblock_find_in_range(min_low_pfn<<PAGE_SHIFT,
165 max_pfn_mapped<<PAGE_SHIFT, 166 max_pfn_mapped<<PAGE_SHIFT,
166 sizeof(pg_data_t), 167 sizeof(pg_data_t),
167 PAGE_SIZE); 168 PAGE_SIZE);
168 NODE_DATA(nid) = (pg_data_t *)(pfn_to_kaddr(pgdat_phys>>PAGE_SHIFT)); 169 NODE_DATA(nid) = (pg_data_t *)(pfn_to_kaddr(pgdat_phys>>PAGE_SHIFT));
169 memset(buf, 0, sizeof(buf)); 170 memset(buf, 0, sizeof(buf));
170 sprintf(buf, "NODE_DATA %d", nid); 171 sprintf(buf, "NODE_DATA %d", nid);
171 reserve_early(pgdat_phys, pgdat_phys + sizeof(pg_data_t), buf); 172 memblock_x86_reserve_range(pgdat_phys, pgdat_phys + sizeof(pg_data_t), buf);
172 } 173 }
173 printk(KERN_DEBUG "allocate_pgdat: node %d NODE_DATA %08lx\n", 174 printk(KERN_DEBUG "allocate_pgdat: node %d NODE_DATA %08lx\n",
174 nid, (unsigned long)NODE_DATA(nid)); 175 nid, (unsigned long)NODE_DATA(nid));
@@ -291,15 +292,15 @@ static __init unsigned long calculate_numa_remap_pages(void)
291 PTRS_PER_PTE); 292 PTRS_PER_PTE);
292 node_kva_target <<= PAGE_SHIFT; 293 node_kva_target <<= PAGE_SHIFT;
293 do { 294 do {
294 node_kva_final = find_e820_area(node_kva_target, 295 node_kva_final = memblock_find_in_range(node_kva_target,
295 ((u64)node_end_pfn[nid])<<PAGE_SHIFT, 296 ((u64)node_end_pfn[nid])<<PAGE_SHIFT,
296 ((u64)size)<<PAGE_SHIFT, 297 ((u64)size)<<PAGE_SHIFT,
297 LARGE_PAGE_BYTES); 298 LARGE_PAGE_BYTES);
298 node_kva_target -= LARGE_PAGE_BYTES; 299 node_kva_target -= LARGE_PAGE_BYTES;
299 } while (node_kva_final == -1ULL && 300 } while (node_kva_final == MEMBLOCK_ERROR &&
300 (node_kva_target>>PAGE_SHIFT) > (node_start_pfn[nid])); 301 (node_kva_target>>PAGE_SHIFT) > (node_start_pfn[nid]));
301 302
302 if (node_kva_final == -1ULL) 303 if (node_kva_final == MEMBLOCK_ERROR)
303 panic("Can not get kva ram\n"); 304 panic("Can not get kva ram\n");
304 305
305 node_remap_size[nid] = size; 306 node_remap_size[nid] = size;
@@ -318,15 +319,13 @@ static __init unsigned long calculate_numa_remap_pages(void)
318 * but we could have some hole in high memory, and it will only 319 * but we could have some hole in high memory, and it will only
319 * check page_is_ram(pfn) && !page_is_reserved_early(pfn) to decide 320 * check page_is_ram(pfn) && !page_is_reserved_early(pfn) to decide
320 * to use it as free. 321 * to use it as free.
321 * So reserve_early here, hope we don't run out of that array 322 * So memblock_x86_reserve_range here, hope we don't run out of that array
322 */ 323 */
323 reserve_early(node_kva_final, 324 memblock_x86_reserve_range(node_kva_final,
324 node_kva_final+(((u64)size)<<PAGE_SHIFT), 325 node_kva_final+(((u64)size)<<PAGE_SHIFT),
325 "KVA RAM"); 326 "KVA RAM");
326 327
327 node_remap_start_pfn[nid] = node_kva_final>>PAGE_SHIFT; 328 node_remap_start_pfn[nid] = node_kva_final>>PAGE_SHIFT;
328 remove_active_range(nid, node_remap_start_pfn[nid],
329 node_remap_start_pfn[nid] + size);
330 } 329 }
331 printk(KERN_INFO "Reserving total of %lx pages for numa KVA remap\n", 330 printk(KERN_INFO "Reserving total of %lx pages for numa KVA remap\n",
332 reserve_pages); 331 reserve_pages);
@@ -367,14 +366,14 @@ void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn,
367 366
368 kva_target_pfn = round_down(max_low_pfn - kva_pages, PTRS_PER_PTE); 367 kva_target_pfn = round_down(max_low_pfn - kva_pages, PTRS_PER_PTE);
369 do { 368 do {
370 kva_start_pfn = find_e820_area(kva_target_pfn<<PAGE_SHIFT, 369 kva_start_pfn = memblock_find_in_range(kva_target_pfn<<PAGE_SHIFT,
371 max_low_pfn<<PAGE_SHIFT, 370 max_low_pfn<<PAGE_SHIFT,
372 kva_pages<<PAGE_SHIFT, 371 kva_pages<<PAGE_SHIFT,
373 PTRS_PER_PTE<<PAGE_SHIFT) >> PAGE_SHIFT; 372 PTRS_PER_PTE<<PAGE_SHIFT) >> PAGE_SHIFT;
374 kva_target_pfn -= PTRS_PER_PTE; 373 kva_target_pfn -= PTRS_PER_PTE;
375 } while (kva_start_pfn == -1UL && kva_target_pfn > min_low_pfn); 374 } while (kva_start_pfn == MEMBLOCK_ERROR && kva_target_pfn > min_low_pfn);
376 375
377 if (kva_start_pfn == -1UL) 376 if (kva_start_pfn == MEMBLOCK_ERROR)
378 panic("Can not get kva space\n"); 377 panic("Can not get kva space\n");
379 378
380 printk(KERN_INFO "kva_start_pfn ~ %lx max_low_pfn ~ %lx\n", 379 printk(KERN_INFO "kva_start_pfn ~ %lx max_low_pfn ~ %lx\n",
@@ -382,7 +381,7 @@ void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn,
382 printk(KERN_INFO "max_pfn = %lx\n", max_pfn); 381 printk(KERN_INFO "max_pfn = %lx\n", max_pfn);
383 382
384 /* avoid clash with initrd */ 383 /* avoid clash with initrd */
385 reserve_early(kva_start_pfn<<PAGE_SHIFT, 384 memblock_x86_reserve_range(kva_start_pfn<<PAGE_SHIFT,
386 (kva_start_pfn + kva_pages)<<PAGE_SHIFT, 385 (kva_start_pfn + kva_pages)<<PAGE_SHIFT,
387 "KVA PG"); 386 "KVA PG");
388#ifdef CONFIG_HIGHMEM 387#ifdef CONFIG_HIGHMEM
@@ -419,9 +418,6 @@ void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn,
419 for_each_online_node(nid) { 418 for_each_online_node(nid) {
420 memset(NODE_DATA(nid), 0, sizeof(struct pglist_data)); 419 memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));
421 NODE_DATA(nid)->node_id = nid; 420 NODE_DATA(nid)->node_id = nid;
422#ifndef CONFIG_NO_BOOTMEM
423 NODE_DATA(nid)->bdata = &bootmem_node_data[nid];
424#endif
425 } 421 }
426 422
427 setup_bootmem_allocator(); 423 setup_bootmem_allocator();
diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c
index 4962f1aeda6f..60f498511dd6 100644
--- a/arch/x86/mm/numa_64.c
+++ b/arch/x86/mm/numa_64.c
@@ -7,6 +7,7 @@
7#include <linux/string.h> 7#include <linux/string.h>
8#include <linux/init.h> 8#include <linux/init.h>
9#include <linux/bootmem.h> 9#include <linux/bootmem.h>
10#include <linux/memblock.h>
10#include <linux/mmzone.h> 11#include <linux/mmzone.h>
11#include <linux/ctype.h> 12#include <linux/ctype.h>
12#include <linux/module.h> 13#include <linux/module.h>
@@ -86,16 +87,16 @@ static int __init allocate_cachealigned_memnodemap(void)
86 87
87 addr = 0x8000; 88 addr = 0x8000;
88 nodemap_size = roundup(sizeof(s16) * memnodemapsize, L1_CACHE_BYTES); 89 nodemap_size = roundup(sizeof(s16) * memnodemapsize, L1_CACHE_BYTES);
89 nodemap_addr = find_e820_area(addr, max_pfn<<PAGE_SHIFT, 90 nodemap_addr = memblock_find_in_range(addr, max_pfn<<PAGE_SHIFT,
90 nodemap_size, L1_CACHE_BYTES); 91 nodemap_size, L1_CACHE_BYTES);
91 if (nodemap_addr == -1UL) { 92 if (nodemap_addr == MEMBLOCK_ERROR) {
92 printk(KERN_ERR 93 printk(KERN_ERR
93 "NUMA: Unable to allocate Memory to Node hash map\n"); 94 "NUMA: Unable to allocate Memory to Node hash map\n");
94 nodemap_addr = nodemap_size = 0; 95 nodemap_addr = nodemap_size = 0;
95 return -1; 96 return -1;
96 } 97 }
97 memnodemap = phys_to_virt(nodemap_addr); 98 memnodemap = phys_to_virt(nodemap_addr);
98 reserve_early(nodemap_addr, nodemap_addr + nodemap_size, "MEMNODEMAP"); 99 memblock_x86_reserve_range(nodemap_addr, nodemap_addr + nodemap_size, "MEMNODEMAP");
99 100
100 printk(KERN_DEBUG "NUMA: Allocated memnodemap from %lx - %lx\n", 101 printk(KERN_DEBUG "NUMA: Allocated memnodemap from %lx - %lx\n",
101 nodemap_addr, nodemap_addr + nodemap_size); 102 nodemap_addr, nodemap_addr + nodemap_size);
@@ -171,8 +172,8 @@ static void * __init early_node_mem(int nodeid, unsigned long start,
171 if (start < (MAX_DMA32_PFN<<PAGE_SHIFT) && 172 if (start < (MAX_DMA32_PFN<<PAGE_SHIFT) &&
172 end > (MAX_DMA32_PFN<<PAGE_SHIFT)) 173 end > (MAX_DMA32_PFN<<PAGE_SHIFT))
173 start = MAX_DMA32_PFN<<PAGE_SHIFT; 174 start = MAX_DMA32_PFN<<PAGE_SHIFT;
174 mem = find_e820_area(start, end, size, align); 175 mem = memblock_x86_find_in_range_node(nodeid, start, end, size, align);
175 if (mem != -1L) 176 if (mem != MEMBLOCK_ERROR)
176 return __va(mem); 177 return __va(mem);
177 178
178 /* extend the search scope */ 179 /* extend the search scope */
@@ -181,8 +182,8 @@ static void * __init early_node_mem(int nodeid, unsigned long start,
181 start = MAX_DMA32_PFN<<PAGE_SHIFT; 182 start = MAX_DMA32_PFN<<PAGE_SHIFT;
182 else 183 else
183 start = MAX_DMA_PFN<<PAGE_SHIFT; 184 start = MAX_DMA_PFN<<PAGE_SHIFT;
184 mem = find_e820_area(start, end, size, align); 185 mem = memblock_x86_find_in_range_node(nodeid, start, end, size, align);
185 if (mem != -1L) 186 if (mem != MEMBLOCK_ERROR)
186 return __va(mem); 187 return __va(mem);
187 188
188 printk(KERN_ERR "Cannot find %lu bytes in node %d\n", 189 printk(KERN_ERR "Cannot find %lu bytes in node %d\n",
@@ -198,10 +199,6 @@ setup_node_bootmem(int nodeid, unsigned long start, unsigned long end)
198 unsigned long start_pfn, last_pfn, nodedata_phys; 199 unsigned long start_pfn, last_pfn, nodedata_phys;
199 const int pgdat_size = roundup(sizeof(pg_data_t), PAGE_SIZE); 200 const int pgdat_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
200 int nid; 201 int nid;
201#ifndef CONFIG_NO_BOOTMEM
202 unsigned long bootmap_start, bootmap_pages, bootmap_size;
203 void *bootmap;
204#endif
205 202
206 if (!end) 203 if (!end)
207 return; 204 return;
@@ -226,7 +223,7 @@ setup_node_bootmem(int nodeid, unsigned long start, unsigned long end)
226 if (node_data[nodeid] == NULL) 223 if (node_data[nodeid] == NULL)
227 return; 224 return;
228 nodedata_phys = __pa(node_data[nodeid]); 225 nodedata_phys = __pa(node_data[nodeid]);
229 reserve_early(nodedata_phys, nodedata_phys + pgdat_size, "NODE_DATA"); 226 memblock_x86_reserve_range(nodedata_phys, nodedata_phys + pgdat_size, "NODE_DATA");
230 printk(KERN_INFO " NODE_DATA [%016lx - %016lx]\n", nodedata_phys, 227 printk(KERN_INFO " NODE_DATA [%016lx - %016lx]\n", nodedata_phys,
231 nodedata_phys + pgdat_size - 1); 228 nodedata_phys + pgdat_size - 1);
232 nid = phys_to_nid(nodedata_phys); 229 nid = phys_to_nid(nodedata_phys);
@@ -238,47 +235,6 @@ setup_node_bootmem(int nodeid, unsigned long start, unsigned long end)
238 NODE_DATA(nodeid)->node_start_pfn = start_pfn; 235 NODE_DATA(nodeid)->node_start_pfn = start_pfn;
239 NODE_DATA(nodeid)->node_spanned_pages = last_pfn - start_pfn; 236 NODE_DATA(nodeid)->node_spanned_pages = last_pfn - start_pfn;
240 237
241#ifndef CONFIG_NO_BOOTMEM
242 NODE_DATA(nodeid)->bdata = &bootmem_node_data[nodeid];
243
244 /*
245 * Find a place for the bootmem map
246 * nodedata_phys could be on other nodes by alloc_bootmem,
247 * so need to sure bootmap_start not to be small, otherwise
248 * early_node_mem will get that with find_e820_area instead
249 * of alloc_bootmem, that could clash with reserved range
250 */
251 bootmap_pages = bootmem_bootmap_pages(last_pfn - start_pfn);
252 bootmap_start = roundup(nodedata_phys + pgdat_size, PAGE_SIZE);
253 /*
254 * SMP_CACHE_BYTES could be enough, but init_bootmem_node like
255 * to use that to align to PAGE_SIZE
256 */
257 bootmap = early_node_mem(nodeid, bootmap_start, end,
258 bootmap_pages<<PAGE_SHIFT, PAGE_SIZE);
259 if (bootmap == NULL) {
260 free_early(nodedata_phys, nodedata_phys + pgdat_size);
261 node_data[nodeid] = NULL;
262 return;
263 }
264 bootmap_start = __pa(bootmap);
265 reserve_early(bootmap_start, bootmap_start+(bootmap_pages<<PAGE_SHIFT),
266 "BOOTMAP");
267
268 bootmap_size = init_bootmem_node(NODE_DATA(nodeid),
269 bootmap_start >> PAGE_SHIFT,
270 start_pfn, last_pfn);
271
272 printk(KERN_INFO " bootmap [%016lx - %016lx] pages %lx\n",
273 bootmap_start, bootmap_start + bootmap_size - 1,
274 bootmap_pages);
275 nid = phys_to_nid(bootmap_start);
276 if (nid != nodeid)
277 printk(KERN_INFO " bootmap(%d) on node %d\n", nodeid, nid);
278
279 free_bootmem_with_active_regions(nodeid, end);
280#endif
281
282 node_set_online(nodeid); 238 node_set_online(nodeid);
283} 239}
284 240
@@ -416,7 +372,7 @@ static int __init split_nodes_interleave(u64 addr, u64 max_addr,
416 nr_nodes = MAX_NUMNODES; 372 nr_nodes = MAX_NUMNODES;
417 } 373 }
418 374
419 size = (max_addr - addr - e820_hole_size(addr, max_addr)) / nr_nodes; 375 size = (max_addr - addr - memblock_x86_hole_size(addr, max_addr)) / nr_nodes;
420 /* 376 /*
421 * Calculate the number of big nodes that can be allocated as a result 377 * Calculate the number of big nodes that can be allocated as a result
422 * of consolidating the remainder. 378 * of consolidating the remainder.
@@ -452,7 +408,7 @@ static int __init split_nodes_interleave(u64 addr, u64 max_addr,
452 * non-reserved memory is less than the per-node size. 408 * non-reserved memory is less than the per-node size.
453 */ 409 */
454 while (end - physnodes[i].start - 410 while (end - physnodes[i].start -
455 e820_hole_size(physnodes[i].start, end) < size) { 411 memblock_x86_hole_size(physnodes[i].start, end) < size) {
456 end += FAKE_NODE_MIN_SIZE; 412 end += FAKE_NODE_MIN_SIZE;
457 if (end > physnodes[i].end) { 413 if (end > physnodes[i].end) {
458 end = physnodes[i].end; 414 end = physnodes[i].end;
@@ -466,7 +422,7 @@ static int __init split_nodes_interleave(u64 addr, u64 max_addr,
466 * this one must extend to the boundary. 422 * this one must extend to the boundary.
467 */ 423 */
468 if (end < dma32_end && dma32_end - end - 424 if (end < dma32_end && dma32_end - end -
469 e820_hole_size(end, dma32_end) < FAKE_NODE_MIN_SIZE) 425 memblock_x86_hole_size(end, dma32_end) < FAKE_NODE_MIN_SIZE)
470 end = dma32_end; 426 end = dma32_end;
471 427
472 /* 428 /*
@@ -475,7 +431,7 @@ static int __init split_nodes_interleave(u64 addr, u64 max_addr,
475 * physical node. 431 * physical node.
476 */ 432 */
477 if (physnodes[i].end - end - 433 if (physnodes[i].end - end -
478 e820_hole_size(end, physnodes[i].end) < size) 434 memblock_x86_hole_size(end, physnodes[i].end) < size)
479 end = physnodes[i].end; 435 end = physnodes[i].end;
480 436
481 /* 437 /*
@@ -503,7 +459,7 @@ static u64 __init find_end_of_node(u64 start, u64 max_addr, u64 size)
503{ 459{
504 u64 end = start + size; 460 u64 end = start + size;
505 461
506 while (end - start - e820_hole_size(start, end) < size) { 462 while (end - start - memblock_x86_hole_size(start, end) < size) {
507 end += FAKE_NODE_MIN_SIZE; 463 end += FAKE_NODE_MIN_SIZE;
508 if (end > max_addr) { 464 if (end > max_addr) {
509 end = max_addr; 465 end = max_addr;
@@ -532,7 +488,7 @@ static int __init split_nodes_size_interleave(u64 addr, u64 max_addr, u64 size)
532 * creates a uniform distribution of node sizes across the entire 488 * creates a uniform distribution of node sizes across the entire
533 * machine (but not necessarily over physical nodes). 489 * machine (but not necessarily over physical nodes).
534 */ 490 */
535 min_size = (max_addr - addr - e820_hole_size(addr, max_addr)) / 491 min_size = (max_addr - addr - memblock_x86_hole_size(addr, max_addr)) /
536 MAX_NUMNODES; 492 MAX_NUMNODES;
537 min_size = max(min_size, FAKE_NODE_MIN_SIZE); 493 min_size = max(min_size, FAKE_NODE_MIN_SIZE);
538 if ((min_size & FAKE_NODE_MIN_HASH_MASK) < min_size) 494 if ((min_size & FAKE_NODE_MIN_HASH_MASK) < min_size)
@@ -565,7 +521,7 @@ static int __init split_nodes_size_interleave(u64 addr, u64 max_addr, u64 size)
565 * this one must extend to the boundary. 521 * this one must extend to the boundary.
566 */ 522 */
567 if (end < dma32_end && dma32_end - end - 523 if (end < dma32_end && dma32_end - end -
568 e820_hole_size(end, dma32_end) < FAKE_NODE_MIN_SIZE) 524 memblock_x86_hole_size(end, dma32_end) < FAKE_NODE_MIN_SIZE)
569 end = dma32_end; 525 end = dma32_end;
570 526
571 /* 527 /*
@@ -574,7 +530,7 @@ static int __init split_nodes_size_interleave(u64 addr, u64 max_addr, u64 size)
574 * physical node. 530 * physical node.
575 */ 531 */
576 if (physnodes[i].end - end - 532 if (physnodes[i].end - end -
577 e820_hole_size(end, physnodes[i].end) < size) 533 memblock_x86_hole_size(end, physnodes[i].end) < size)
578 end = physnodes[i].end; 534 end = physnodes[i].end;
579 535
580 /* 536 /*
@@ -638,7 +594,7 @@ static int __init numa_emulation(unsigned long start_pfn,
638 */ 594 */
639 remove_all_active_ranges(); 595 remove_all_active_ranges();
640 for_each_node_mask(i, node_possible_map) { 596 for_each_node_mask(i, node_possible_map) {
641 e820_register_active_regions(i, nodes[i].start >> PAGE_SHIFT, 597 memblock_x86_register_active_regions(i, nodes[i].start >> PAGE_SHIFT,
642 nodes[i].end >> PAGE_SHIFT); 598 nodes[i].end >> PAGE_SHIFT);
643 setup_node_bootmem(i, nodes[i].start, nodes[i].end); 599 setup_node_bootmem(i, nodes[i].start, nodes[i].end);
644 } 600 }
@@ -691,7 +647,7 @@ void __init initmem_init(unsigned long start_pfn, unsigned long last_pfn,
691 node_set(0, node_possible_map); 647 node_set(0, node_possible_map);
692 for (i = 0; i < nr_cpu_ids; i++) 648 for (i = 0; i < nr_cpu_ids; i++)
693 numa_set_node(i, 0); 649 numa_set_node(i, 0);
694 e820_register_active_regions(0, start_pfn, last_pfn); 650 memblock_x86_register_active_regions(0, start_pfn, last_pfn);
695 setup_node_bootmem(0, start_pfn << PAGE_SHIFT, last_pfn << PAGE_SHIFT); 651 setup_node_bootmem(0, start_pfn << PAGE_SHIFT, last_pfn << PAGE_SHIFT);
696} 652}
697 653
@@ -703,9 +659,7 @@ unsigned long __init numa_free_all_bootmem(void)
703 for_each_online_node(i) 659 for_each_online_node(i)
704 pages += free_all_bootmem_node(NODE_DATA(i)); 660 pages += free_all_bootmem_node(NODE_DATA(i));
705 661
706#ifdef CONFIG_NO_BOOTMEM
707 pages += free_all_memory_core_early(MAX_NUMNODES); 662 pages += free_all_memory_core_early(MAX_NUMNODES);
708#endif
709 663
710 return pages; 664 return pages;
711} 665}
diff --git a/arch/x86/mm/srat_32.c b/arch/x86/mm/srat_32.c
index 9324f13492d5..a17dffd136c1 100644
--- a/arch/x86/mm/srat_32.c
+++ b/arch/x86/mm/srat_32.c
@@ -25,6 +25,7 @@
25 */ 25 */
26#include <linux/mm.h> 26#include <linux/mm.h>
27#include <linux/bootmem.h> 27#include <linux/bootmem.h>
28#include <linux/memblock.h>
28#include <linux/mmzone.h> 29#include <linux/mmzone.h>
29#include <linux/acpi.h> 30#include <linux/acpi.h>
30#include <linux/nodemask.h> 31#include <linux/nodemask.h>
@@ -264,7 +265,7 @@ int __init get_memcfg_from_srat(void)
264 if (node_read_chunk(chunk->nid, chunk)) 265 if (node_read_chunk(chunk->nid, chunk))
265 continue; 266 continue;
266 267
267 e820_register_active_regions(chunk->nid, chunk->start_pfn, 268 memblock_x86_register_active_regions(chunk->nid, chunk->start_pfn,
268 min(chunk->end_pfn, max_pfn)); 269 min(chunk->end_pfn, max_pfn));
269 } 270 }
270 /* for out of order entries in SRAT */ 271 /* for out of order entries in SRAT */
diff --git a/arch/x86/mm/srat_64.c b/arch/x86/mm/srat_64.c
index 9c0d0d399c30..a35cb9d8b060 100644
--- a/arch/x86/mm/srat_64.c
+++ b/arch/x86/mm/srat_64.c
@@ -16,6 +16,7 @@
16#include <linux/module.h> 16#include <linux/module.h>
17#include <linux/topology.h> 17#include <linux/topology.h>
18#include <linux/bootmem.h> 18#include <linux/bootmem.h>
19#include <linux/memblock.h>
19#include <linux/mm.h> 20#include <linux/mm.h>
20#include <asm/proto.h> 21#include <asm/proto.h>
21#include <asm/numa.h> 22#include <asm/numa.h>
@@ -98,15 +99,15 @@ void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
98 unsigned long phys; 99 unsigned long phys;
99 100
100 length = slit->header.length; 101 length = slit->header.length;
101 phys = find_e820_area(0, max_pfn_mapped<<PAGE_SHIFT, length, 102 phys = memblock_find_in_range(0, max_pfn_mapped<<PAGE_SHIFT, length,
102 PAGE_SIZE); 103 PAGE_SIZE);
103 104
104 if (phys == -1L) 105 if (phys == MEMBLOCK_ERROR)
105 panic(" Can not save slit!\n"); 106 panic(" Can not save slit!\n");
106 107
107 acpi_slit = __va(phys); 108 acpi_slit = __va(phys);
108 memcpy(acpi_slit, slit, length); 109 memcpy(acpi_slit, slit, length);
109 reserve_early(phys, phys + length, "ACPI SLIT"); 110 memblock_x86_reserve_range(phys, phys + length, "ACPI SLIT");
110} 111}
111 112
112/* Callback for Proximity Domain -> x2APIC mapping */ 113/* Callback for Proximity Domain -> x2APIC mapping */
@@ -324,7 +325,7 @@ static int __init nodes_cover_memory(const struct bootnode *nodes)
324 pxmram = 0; 325 pxmram = 0;
325 } 326 }
326 327
327 e820ram = max_pfn - (e820_hole_size(0, max_pfn<<PAGE_SHIFT)>>PAGE_SHIFT); 328 e820ram = max_pfn - (memblock_x86_hole_size(0, max_pfn<<PAGE_SHIFT)>>PAGE_SHIFT);
328 /* We seem to lose 3 pages somewhere. Allow 1M of slack. */ 329 /* We seem to lose 3 pages somewhere. Allow 1M of slack. */
329 if ((long)(e820ram - pxmram) >= (1<<(20 - PAGE_SHIFT))) { 330 if ((long)(e820ram - pxmram) >= (1<<(20 - PAGE_SHIFT))) {
330 printk(KERN_ERR 331 printk(KERN_ERR
@@ -421,7 +422,7 @@ int __init acpi_scan_nodes(unsigned long start, unsigned long end)
421 } 422 }
422 423
423 for (i = 0; i < num_node_memblks; i++) 424 for (i = 0; i < num_node_memblks; i++)
424 e820_register_active_regions(memblk_nodeid[i], 425 memblock_x86_register_active_regions(memblk_nodeid[i],
425 node_memblk_range[i].start >> PAGE_SHIFT, 426 node_memblk_range[i].start >> PAGE_SHIFT,
426 node_memblk_range[i].end >> PAGE_SHIFT); 427 node_memblk_range[i].end >> PAGE_SHIFT);
427 428
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 7d46c8441418..63b83ceebd1a 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -30,6 +30,7 @@
30#include <linux/console.h> 30#include <linux/console.h>
31#include <linux/pci.h> 31#include <linux/pci.h>
32#include <linux/gfp.h> 32#include <linux/gfp.h>
33#include <linux/memblock.h>
33 34
34#include <xen/xen.h> 35#include <xen/xen.h>
35#include <xen/interface/xen.h> 36#include <xen/interface/xen.h>
@@ -1183,6 +1184,8 @@ asmlinkage void __init xen_start_kernel(void)
1183 local_irq_disable(); 1184 local_irq_disable();
1184 early_boot_irqs_off(); 1185 early_boot_irqs_off();
1185 1186
1187 memblock_init();
1188
1186 xen_raw_console_write("mapping kernel into physical memory\n"); 1189 xen_raw_console_write("mapping kernel into physical memory\n");
1187 pgd = xen_setup_kernel_pagetable(pgd, xen_start_info->nr_pages); 1190 pgd = xen_setup_kernel_pagetable(pgd, xen_start_info->nr_pages);
1188 1191
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index b2363fcbcd0f..f72d18c69221 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -45,6 +45,7 @@
45#include <linux/vmalloc.h> 45#include <linux/vmalloc.h>
46#include <linux/module.h> 46#include <linux/module.h>
47#include <linux/gfp.h> 47#include <linux/gfp.h>
48#include <linux/memblock.h>
48 49
49#include <asm/pgtable.h> 50#include <asm/pgtable.h>
50#include <asm/tlbflush.h> 51#include <asm/tlbflush.h>
@@ -55,6 +56,7 @@
55#include <asm/e820.h> 56#include <asm/e820.h>
56#include <asm/linkage.h> 57#include <asm/linkage.h>
57#include <asm/page.h> 58#include <asm/page.h>
59#include <asm/init.h>
58 60
59#include <asm/xen/hypercall.h> 61#include <asm/xen/hypercall.h>
60#include <asm/xen/hypervisor.h> 62#include <asm/xen/hypervisor.h>
@@ -359,7 +361,8 @@ void make_lowmem_page_readonly(void *vaddr)
359 unsigned int level; 361 unsigned int level;
360 362
361 pte = lookup_address(address, &level); 363 pte = lookup_address(address, &level);
362 BUG_ON(pte == NULL); 364 if (pte == NULL)
365 return; /* vaddr missing */
363 366
364 ptev = pte_wrprotect(*pte); 367 ptev = pte_wrprotect(*pte);
365 368
@@ -374,7 +377,8 @@ void make_lowmem_page_readwrite(void *vaddr)
374 unsigned int level; 377 unsigned int level;
375 378
376 pte = lookup_address(address, &level); 379 pte = lookup_address(address, &level);
377 BUG_ON(pte == NULL); 380 if (pte == NULL)
381 return; /* vaddr missing */
378 382
379 ptev = pte_mkwrite(*pte); 383 ptev = pte_mkwrite(*pte);
380 384
@@ -1508,13 +1512,25 @@ static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)
1508#endif 1512#endif
1509} 1513}
1510 1514
1511#ifdef CONFIG_X86_32
1512static __init pte_t mask_rw_pte(pte_t *ptep, pte_t pte) 1515static __init pte_t mask_rw_pte(pte_t *ptep, pte_t pte)
1513{ 1516{
1517 unsigned long pfn = pte_pfn(pte);
1518
1519#ifdef CONFIG_X86_32
1514 /* If there's an existing pte, then don't allow _PAGE_RW to be set */ 1520 /* If there's an existing pte, then don't allow _PAGE_RW to be set */
1515 if (pte_val_ma(*ptep) & _PAGE_PRESENT) 1521 if (pte_val_ma(*ptep) & _PAGE_PRESENT)
1516 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) & 1522 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
1517 pte_val_ma(pte)); 1523 pte_val_ma(pte));
1524#endif
1525
1526 /*
1527 * If the new pfn is within the range of the newly allocated
1528 * kernel pagetable, and it isn't being mapped into an
1529 * early_ioremap fixmap slot, make sure it is RO.
1530 */
1531 if (!is_early_ioremap_ptep(ptep) &&
1532 pfn >= e820_table_start && pfn < e820_table_end)
1533 pte = pte_wrprotect(pte);
1518 1534
1519 return pte; 1535 return pte;
1520} 1536}
@@ -1527,7 +1543,6 @@ static __init void xen_set_pte_init(pte_t *ptep, pte_t pte)
1527 1543
1528 xen_set_pte(ptep, pte); 1544 xen_set_pte(ptep, pte);
1529} 1545}
1530#endif
1531 1546
1532static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn) 1547static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1533{ 1548{
@@ -1814,7 +1829,7 @@ __init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd,
1814 __xen_write_cr3(true, __pa(pgd)); 1829 __xen_write_cr3(true, __pa(pgd));
1815 xen_mc_issue(PARAVIRT_LAZY_CPU); 1830 xen_mc_issue(PARAVIRT_LAZY_CPU);
1816 1831
1817 reserve_early(__pa(xen_start_info->pt_base), 1832 memblock_x86_reserve_range(__pa(xen_start_info->pt_base),
1818 __pa(xen_start_info->pt_base + 1833 __pa(xen_start_info->pt_base +
1819 xen_start_info->nr_pt_frames * PAGE_SIZE), 1834 xen_start_info->nr_pt_frames * PAGE_SIZE),
1820 "XEN PAGETABLES"); 1835 "XEN PAGETABLES");
@@ -1852,7 +1867,7 @@ __init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd,
1852 1867
1853 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(swapper_pg_dir))); 1868 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(swapper_pg_dir)));
1854 1869
1855 reserve_early(__pa(xen_start_info->pt_base), 1870 memblock_x86_reserve_range(__pa(xen_start_info->pt_base),
1856 __pa(xen_start_info->pt_base + 1871 __pa(xen_start_info->pt_base +
1857 xen_start_info->nr_pt_frames * PAGE_SIZE), 1872 xen_start_info->nr_pt_frames * PAGE_SIZE),
1858 "XEN PAGETABLES"); 1873 "XEN PAGETABLES");
@@ -1971,11 +1986,7 @@ static const struct pv_mmu_ops xen_mmu_ops __initdata = {
1971 .alloc_pmd = xen_alloc_pmd_init, 1986 .alloc_pmd = xen_alloc_pmd_init,
1972 .release_pmd = xen_release_pmd_init, 1987 .release_pmd = xen_release_pmd_init,
1973 1988
1974#ifdef CONFIG_X86_64
1975 .set_pte = xen_set_pte,
1976#else
1977 .set_pte = xen_set_pte_init, 1989 .set_pte = xen_set_pte_init,
1978#endif
1979 .set_pte_at = xen_set_pte_at, 1990 .set_pte_at = xen_set_pte_at,
1980 .set_pmd = xen_set_pmd_hyper, 1991 .set_pmd = xen_set_pmd_hyper,
1981 1992
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index 328b00305426..9729c903404b 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -8,6 +8,7 @@
8#include <linux/sched.h> 8#include <linux/sched.h>
9#include <linux/mm.h> 9#include <linux/mm.h>
10#include <linux/pm.h> 10#include <linux/pm.h>
11#include <linux/memblock.h>
11 12
12#include <asm/elf.h> 13#include <asm/elf.h>
13#include <asm/vdso.h> 14#include <asm/vdso.h>
@@ -129,7 +130,7 @@ char * __init xen_memory_setup(void)
129 * - xen_start_info 130 * - xen_start_info
130 * See comment above "struct start_info" in <xen/interface/xen.h> 131 * See comment above "struct start_info" in <xen/interface/xen.h>
131 */ 132 */
132 reserve_early(__pa(xen_start_info->mfn_list), 133 memblock_x86_reserve_range(__pa(xen_start_info->mfn_list),
133 __pa(xen_start_info->pt_base), 134 __pa(xen_start_info->pt_base),
134 "XEN START INFO"); 135 "XEN START INFO");
135 136
diff --git a/drivers/video/omap2/vram.c b/drivers/video/omap2/vram.c
index f6fdc2085f3e..fed2a72bc6b6 100644
--- a/drivers/video/omap2/vram.c
+++ b/drivers/video/omap2/vram.c
@@ -554,12 +554,8 @@ void __init omap_vram_reserve_sdram_memblock(void)
554 size = PAGE_ALIGN(size); 554 size = PAGE_ALIGN(size);
555 555
556 if (paddr) { 556 if (paddr) {
557 struct memblock_property res; 557 if ((paddr & ~PAGE_MASK) ||
558 558 !memblock_is_region_memory(paddr, size)) {
559 res.base = paddr;
560 res.size = size;
561 if ((paddr & ~PAGE_MASK) || memblock_find(&res) ||
562 res.base != paddr || res.size != size) {
563 pr_err("Illegal SDRAM region for VRAM\n"); 559 pr_err("Illegal SDRAM region for VRAM\n");
564 return; 560 return;
565 } 561 }
diff --git a/include/linux/early_res.h b/include/linux/early_res.h
deleted file mode 100644
index 29c09f57a13c..000000000000
--- a/include/linux/early_res.h
+++ /dev/null
@@ -1,23 +0,0 @@
1#ifndef _LINUX_EARLY_RES_H
2#define _LINUX_EARLY_RES_H
3#ifdef __KERNEL__
4
5extern void reserve_early(u64 start, u64 end, char *name);
6extern void reserve_early_overlap_ok(u64 start, u64 end, char *name);
7extern void free_early(u64 start, u64 end);
8void free_early_partial(u64 start, u64 end);
9extern void early_res_to_bootmem(u64 start, u64 end);
10
11void reserve_early_without_check(u64 start, u64 end, char *name);
12u64 find_early_area(u64 ei_start, u64 ei_last, u64 start, u64 end,
13 u64 size, u64 align);
14u64 find_early_area_size(u64 ei_start, u64 ei_last, u64 start,
15 u64 *sizep, u64 align);
16u64 find_fw_memmap_area(u64 start, u64 end, u64 size, u64 align);
17u64 get_max_mapped(void);
18#include <linux/range.h>
19int get_free_all_memory_range(struct range **rangep, int nodeid);
20
21#endif /* __KERNEL__ */
22
23#endif /* _LINUX_EARLY_RES_H */
diff --git a/include/linux/memblock.h b/include/linux/memblock.h
index a59faf2b5edd..62a10c2a11f2 100644
--- a/include/linux/memblock.h
+++ b/include/linux/memblock.h
@@ -2,6 +2,7 @@
2#define _LINUX_MEMBLOCK_H 2#define _LINUX_MEMBLOCK_H
3#ifdef __KERNEL__ 3#ifdef __KERNEL__
4 4
5#ifdef CONFIG_HAVE_MEMBLOCK
5/* 6/*
6 * Logical memory blocks. 7 * Logical memory blocks.
7 * 8 *
@@ -16,73 +17,150 @@
16#include <linux/init.h> 17#include <linux/init.h>
17#include <linux/mm.h> 18#include <linux/mm.h>
18 19
19#define MAX_MEMBLOCK_REGIONS 128 20#include <asm/memblock.h>
20 21
21struct memblock_property { 22#define INIT_MEMBLOCK_REGIONS 128
22 u64 base; 23#define MEMBLOCK_ERROR 0
23 u64 size;
24};
25 24
26struct memblock_region { 25struct memblock_region {
27 unsigned long cnt; 26 phys_addr_t base;
28 u64 size; 27 phys_addr_t size;
29 struct memblock_property region[MAX_MEMBLOCK_REGIONS+1]; 28};
29
30struct memblock_type {
31 unsigned long cnt; /* number of regions */
32 unsigned long max; /* size of the allocated array */
33 struct memblock_region *regions;
30}; 34};
31 35
32struct memblock { 36struct memblock {
33 unsigned long debug; 37 phys_addr_t current_limit;
34 u64 rmo_size; 38 phys_addr_t memory_size; /* Updated by memblock_analyze() */
35 struct memblock_region memory; 39 struct memblock_type memory;
36 struct memblock_region reserved; 40 struct memblock_type reserved;
37}; 41};
38 42
39extern struct memblock memblock; 43extern struct memblock memblock;
44extern int memblock_debug;
45extern int memblock_can_resize;
40 46
41extern void __init memblock_init(void); 47#define memblock_dbg(fmt, ...) \
42extern void __init memblock_analyze(void); 48 if (memblock_debug) printk(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__)
43extern long memblock_add(u64 base, u64 size); 49
44extern long memblock_remove(u64 base, u64 size); 50u64 memblock_find_in_range(u64 start, u64 end, u64 size, u64 align);
45extern long __init memblock_free(u64 base, u64 size); 51int memblock_free_reserved_regions(void);
46extern long __init memblock_reserve(u64 base, u64 size); 52int memblock_reserve_reserved_regions(void);
47extern u64 __init memblock_alloc_nid(u64 size, u64 align, int nid, 53
48 u64 (*nid_range)(u64, u64, int *)); 54extern void memblock_init(void);
49extern u64 __init memblock_alloc(u64 size, u64 align); 55extern void memblock_analyze(void);
50extern u64 __init memblock_alloc_base(u64 size, 56extern long memblock_add(phys_addr_t base, phys_addr_t size);
51 u64, u64 max_addr); 57extern long memblock_remove(phys_addr_t base, phys_addr_t size);
52extern u64 __init __memblock_alloc_base(u64 size, 58extern long memblock_free(phys_addr_t base, phys_addr_t size);
53 u64 align, u64 max_addr); 59extern long memblock_reserve(phys_addr_t base, phys_addr_t size);
54extern u64 __init memblock_phys_mem_size(void); 60
55extern u64 memblock_end_of_DRAM(void); 61/* The numa aware allocator is only available if
56extern void __init memblock_enforce_memory_limit(u64 memory_limit); 62 * CONFIG_ARCH_POPULATES_NODE_MAP is set
57extern int __init memblock_is_reserved(u64 addr); 63 */
58extern int memblock_is_region_reserved(u64 base, u64 size); 64extern phys_addr_t memblock_alloc_nid(phys_addr_t size, phys_addr_t align,
59extern int memblock_find(struct memblock_property *res); 65 int nid);
66extern phys_addr_t memblock_alloc_try_nid(phys_addr_t size, phys_addr_t align,
67 int nid);
68
69extern phys_addr_t memblock_alloc(phys_addr_t size, phys_addr_t align);
70
71/* Flags for memblock_alloc_base() amd __memblock_alloc_base() */
72#define MEMBLOCK_ALLOC_ANYWHERE (~(phys_addr_t)0)
73#define MEMBLOCK_ALLOC_ACCESSIBLE 0
74
75extern phys_addr_t memblock_alloc_base(phys_addr_t size,
76 phys_addr_t align,
77 phys_addr_t max_addr);
78extern phys_addr_t __memblock_alloc_base(phys_addr_t size,
79 phys_addr_t align,
80 phys_addr_t max_addr);
81extern phys_addr_t memblock_phys_mem_size(void);
82extern phys_addr_t memblock_end_of_DRAM(void);
83extern void memblock_enforce_memory_limit(phys_addr_t memory_limit);
84extern int memblock_is_memory(phys_addr_t addr);
85extern int memblock_is_region_memory(phys_addr_t base, phys_addr_t size);
86extern int memblock_is_reserved(phys_addr_t addr);
87extern int memblock_is_region_reserved(phys_addr_t base, phys_addr_t size);
60 88
61extern void memblock_dump_all(void); 89extern void memblock_dump_all(void);
62 90
63static inline u64 91/* Provided by the architecture */
64memblock_size_bytes(struct memblock_region *type, unsigned long region_nr) 92extern phys_addr_t memblock_nid_range(phys_addr_t start, phys_addr_t end, int *nid);
93extern int memblock_memory_can_coalesce(phys_addr_t addr1, phys_addr_t size1,
94 phys_addr_t addr2, phys_addr_t size2);
95
96/**
97 * memblock_set_current_limit - Set the current allocation limit to allow
98 * limiting allocations to what is currently
99 * accessible during boot
100 * @limit: New limit value (physical address)
101 */
102extern void memblock_set_current_limit(phys_addr_t limit);
103
104
105/*
106 * pfn conversion functions
107 *
108 * While the memory MEMBLOCKs should always be page aligned, the reserved
109 * MEMBLOCKs may not be. This accessor attempt to provide a very clear
110 * idea of what they return for such non aligned MEMBLOCKs.
111 */
112
113/**
114 * memblock_region_memory_base_pfn - Return the lowest pfn intersecting with the memory region
115 * @reg: memblock_region structure
116 */
117static inline unsigned long memblock_region_memory_base_pfn(const struct memblock_region *reg)
65{ 118{
66 return type->region[region_nr].size; 119 return PFN_UP(reg->base);
67} 120}
68static inline u64 121
69memblock_size_pages(struct memblock_region *type, unsigned long region_nr) 122/**
123 * memblock_region_memory_end_pfn - Return the end_pfn this region
124 * @reg: memblock_region structure
125 */
126static inline unsigned long memblock_region_memory_end_pfn(const struct memblock_region *reg)
70{ 127{
71 return memblock_size_bytes(type, region_nr) >> PAGE_SHIFT; 128 return PFN_DOWN(reg->base + reg->size);
72} 129}
73static inline u64 130
74memblock_start_pfn(struct memblock_region *type, unsigned long region_nr) 131/**
132 * memblock_region_reserved_base_pfn - Return the lowest pfn intersecting with the reserved region
133 * @reg: memblock_region structure
134 */
135static inline unsigned long memblock_region_reserved_base_pfn(const struct memblock_region *reg)
75{ 136{
76 return type->region[region_nr].base >> PAGE_SHIFT; 137 return PFN_DOWN(reg->base);
77} 138}
78static inline u64 139
79memblock_end_pfn(struct memblock_region *type, unsigned long region_nr) 140/**
141 * memblock_region_reserved_end_pfn - Return the end_pfn this region
142 * @reg: memblock_region structure
143 */
144static inline unsigned long memblock_region_reserved_end_pfn(const struct memblock_region *reg)
80{ 145{
81 return memblock_start_pfn(type, region_nr) + 146 return PFN_UP(reg->base + reg->size);
82 memblock_size_pages(type, region_nr);
83} 147}
84 148
85#include <asm/memblock.h> 149#define for_each_memblock(memblock_type, region) \
150 for (region = memblock.memblock_type.regions; \
151 region < (memblock.memblock_type.regions + memblock.memblock_type.cnt); \
152 region++)
153
154
155#ifdef ARCH_DISCARD_MEMBLOCK
156#define __init_memblock __init
157#define __initdata_memblock __initdata
158#else
159#define __init_memblock
160#define __initdata_memblock
161#endif
162
163#endif /* CONFIG_HAVE_MEMBLOCK */
86 164
87#endif /* __KERNEL__ */ 165#endif /* __KERNEL__ */
88 166
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 74949fbef8c6..7687228dd3b7 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1175,6 +1175,8 @@ extern void free_bootmem_with_active_regions(int nid,
1175 unsigned long max_low_pfn); 1175 unsigned long max_low_pfn);
1176int add_from_early_node_map(struct range *range, int az, 1176int add_from_early_node_map(struct range *range, int az,
1177 int nr_range, int nid); 1177 int nr_range, int nid);
1178u64 __init find_memory_core_early(int nid, u64 size, u64 align,
1179 u64 goal, u64 limit);
1178void *__alloc_memory_core_early(int nodeid, u64 size, u64 align, 1180void *__alloc_memory_core_early(int nodeid, u64 size, u64 align,
1179 u64 goal, u64 limit); 1181 u64 goal, u64 limit);
1180typedef int (*work_fn_t)(unsigned long, unsigned long, void *); 1182typedef int (*work_fn_t)(unsigned long, unsigned long, void *);
diff --git a/kernel/Makefile b/kernel/Makefile
index e2c9d52cfe9e..0b5ff083fa22 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -11,7 +11,6 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o \
11 hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \ 11 hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
12 notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \ 12 notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \
13 async.o range.o jump_label.o 13 async.o range.o jump_label.o
14obj-$(CONFIG_HAVE_EARLY_RES) += early_res.o
15obj-y += groups.o 14obj-y += groups.o
16 15
17ifdef CONFIG_FUNCTION_TRACER 16ifdef CONFIG_FUNCTION_TRACER
diff --git a/kernel/early_res.c b/kernel/early_res.c
deleted file mode 100644
index 7bfae887f211..000000000000
--- a/kernel/early_res.c
+++ /dev/null
@@ -1,590 +0,0 @@
1/*
2 * early_res, could be used to replace bootmem
3 */
4#include <linux/kernel.h>
5#include <linux/types.h>
6#include <linux/init.h>
7#include <linux/bootmem.h>
8#include <linux/mm.h>
9#include <linux/early_res.h>
10#include <linux/slab.h>
11#include <linux/kmemleak.h>
12
13/*
14 * Early reserved memory areas.
15 */
16/*
17 * need to make sure this one is bigger enough before
18 * find_fw_memmap_area could be used
19 */
20#define MAX_EARLY_RES_X 32
21
22struct early_res {
23 u64 start, end;
24 char name[15];
25 char overlap_ok;
26};
27static struct early_res early_res_x[MAX_EARLY_RES_X] __initdata;
28
29static int max_early_res __initdata = MAX_EARLY_RES_X;
30static struct early_res *early_res __initdata = &early_res_x[0];
31static int early_res_count __initdata;
32
33static int __init find_overlapped_early(u64 start, u64 end)
34{
35 int i;
36 struct early_res *r;
37
38 for (i = 0; i < max_early_res && early_res[i].end; i++) {
39 r = &early_res[i];
40 if (end > r->start && start < r->end)
41 break;
42 }
43
44 return i;
45}
46
47/*
48 * Drop the i-th range from the early reservation map,
49 * by copying any higher ranges down one over it, and
50 * clearing what had been the last slot.
51 */
52static void __init drop_range(int i)
53{
54 int j;
55
56 for (j = i + 1; j < max_early_res && early_res[j].end; j++)
57 ;
58
59 memmove(&early_res[i], &early_res[i + 1],
60 (j - 1 - i) * sizeof(struct early_res));
61
62 early_res[j - 1].end = 0;
63 early_res_count--;
64}
65
66static void __init drop_range_partial(int i, u64 start, u64 end)
67{
68 u64 common_start, common_end;
69 u64 old_start, old_end;
70
71 old_start = early_res[i].start;
72 old_end = early_res[i].end;
73 common_start = max(old_start, start);
74 common_end = min(old_end, end);
75
76 /* no overlap ? */
77 if (common_start >= common_end)
78 return;
79
80 if (old_start < common_start) {
81 /* make head segment */
82 early_res[i].end = common_start;
83 if (old_end > common_end) {
84 char name[15];
85
86 /*
87 * Save a local copy of the name, since the
88 * early_res array could get resized inside
89 * reserve_early_without_check() ->
90 * __check_and_double_early_res(), which would
91 * make the current name pointer invalid.
92 */
93 strncpy(name, early_res[i].name,
94 sizeof(early_res[i].name) - 1);
95 /* add another for left over on tail */
96 reserve_early_without_check(common_end, old_end, name);
97 }
98 return;
99 } else {
100 if (old_end > common_end) {
101 /* reuse the entry for tail left */
102 early_res[i].start = common_end;
103 return;
104 }
105 /* all covered */
106 drop_range(i);
107 }
108}
109
110/*
111 * Split any existing ranges that:
112 * 1) are marked 'overlap_ok', and
113 * 2) overlap with the stated range [start, end)
114 * into whatever portion (if any) of the existing range is entirely
115 * below or entirely above the stated range. Drop the portion
116 * of the existing range that overlaps with the stated range,
117 * which will allow the caller of this routine to then add that
118 * stated range without conflicting with any existing range.
119 */
120static void __init drop_overlaps_that_are_ok(u64 start, u64 end)
121{
122 int i;
123 struct early_res *r;
124 u64 lower_start, lower_end;
125 u64 upper_start, upper_end;
126 char name[15];
127
128 for (i = 0; i < max_early_res && early_res[i].end; i++) {
129 r = &early_res[i];
130
131 /* Continue past non-overlapping ranges */
132 if (end <= r->start || start >= r->end)
133 continue;
134
135 /*
136 * Leave non-ok overlaps as is; let caller
137 * panic "Overlapping early reservations"
138 * when it hits this overlap.
139 */
140 if (!r->overlap_ok)
141 return;
142
143 /*
144 * We have an ok overlap. We will drop it from the early
145 * reservation map, and add back in any non-overlapping
146 * portions (lower or upper) as separate, overlap_ok,
147 * non-overlapping ranges.
148 */
149
150 /* 1. Note any non-overlapping (lower or upper) ranges. */
151 strncpy(name, r->name, sizeof(name) - 1);
152
153 lower_start = lower_end = 0;
154 upper_start = upper_end = 0;
155 if (r->start < start) {
156 lower_start = r->start;
157 lower_end = start;
158 }
159 if (r->end > end) {
160 upper_start = end;
161 upper_end = r->end;
162 }
163
164 /* 2. Drop the original ok overlapping range */
165 drop_range(i);
166
167 i--; /* resume for-loop on copied down entry */
168
169 /* 3. Add back in any non-overlapping ranges. */
170 if (lower_end)
171 reserve_early_overlap_ok(lower_start, lower_end, name);
172 if (upper_end)
173 reserve_early_overlap_ok(upper_start, upper_end, name);
174 }
175}
176
177static void __init __reserve_early(u64 start, u64 end, char *name,
178 int overlap_ok)
179{
180 int i;
181 struct early_res *r;
182
183 i = find_overlapped_early(start, end);
184 if (i >= max_early_res)
185 panic("Too many early reservations");
186 r = &early_res[i];
187 if (r->end)
188 panic("Overlapping early reservations "
189 "%llx-%llx %s to %llx-%llx %s\n",
190 start, end - 1, name ? name : "", r->start,
191 r->end - 1, r->name);
192 r->start = start;
193 r->end = end;
194 r->overlap_ok = overlap_ok;
195 if (name)
196 strncpy(r->name, name, sizeof(r->name) - 1);
197 early_res_count++;
198}
199
200/*
201 * A few early reservtations come here.
202 *
203 * The 'overlap_ok' in the name of this routine does -not- mean it
204 * is ok for these reservations to overlap an earlier reservation.
205 * Rather it means that it is ok for subsequent reservations to
206 * overlap this one.
207 *
208 * Use this entry point to reserve early ranges when you are doing
209 * so out of "Paranoia", reserving perhaps more memory than you need,
210 * just in case, and don't mind a subsequent overlapping reservation
211 * that is known to be needed.
212 *
213 * The drop_overlaps_that_are_ok() call here isn't really needed.
214 * It would be needed if we had two colliding 'overlap_ok'
215 * reservations, so that the second such would not panic on the
216 * overlap with the first. We don't have any such as of this
217 * writing, but might as well tolerate such if it happens in
218 * the future.
219 */
220void __init reserve_early_overlap_ok(u64 start, u64 end, char *name)
221{
222 drop_overlaps_that_are_ok(start, end);
223 __reserve_early(start, end, name, 1);
224}
225
226static void __init __check_and_double_early_res(u64 ex_start, u64 ex_end)
227{
228 u64 start, end, size, mem;
229 struct early_res *new;
230
231 /* do we have enough slots left ? */
232 if ((max_early_res - early_res_count) > max(max_early_res/8, 2))
233 return;
234
235 /* double it */
236 mem = -1ULL;
237 size = sizeof(struct early_res) * max_early_res * 2;
238 if (early_res == early_res_x)
239 start = 0;
240 else
241 start = early_res[0].end;
242 end = ex_start;
243 if (start + size < end)
244 mem = find_fw_memmap_area(start, end, size,
245 sizeof(struct early_res));
246 if (mem == -1ULL) {
247 start = ex_end;
248 end = get_max_mapped();
249 if (start + size < end)
250 mem = find_fw_memmap_area(start, end, size,
251 sizeof(struct early_res));
252 }
253 if (mem == -1ULL)
254 panic("can not find more space for early_res array");
255
256 new = __va(mem);
257 /* save the first one for own */
258 new[0].start = mem;
259 new[0].end = mem + size;
260 new[0].overlap_ok = 0;
261 /* copy old to new */
262 if (early_res == early_res_x) {
263 memcpy(&new[1], &early_res[0],
264 sizeof(struct early_res) * max_early_res);
265 memset(&new[max_early_res+1], 0,
266 sizeof(struct early_res) * (max_early_res - 1));
267 early_res_count++;
268 } else {
269 memcpy(&new[1], &early_res[1],
270 sizeof(struct early_res) * (max_early_res - 1));
271 memset(&new[max_early_res], 0,
272 sizeof(struct early_res) * max_early_res);
273 }
274 memset(&early_res[0], 0, sizeof(struct early_res) * max_early_res);
275 early_res = new;
276 max_early_res *= 2;
277 printk(KERN_DEBUG "early_res array is doubled to %d at [%llx - %llx]\n",
278 max_early_res, mem, mem + size - 1);
279}
280
281/*
282 * Most early reservations come here.
283 *
284 * We first have drop_overlaps_that_are_ok() drop any pre-existing
285 * 'overlap_ok' ranges, so that we can then reserve this memory
286 * range without risk of panic'ing on an overlapping overlap_ok
287 * early reservation.
288 */
289void __init reserve_early(u64 start, u64 end, char *name)
290{
291 if (start >= end)
292 return;
293
294 __check_and_double_early_res(start, end);
295
296 drop_overlaps_that_are_ok(start, end);
297 __reserve_early(start, end, name, 0);
298}
299
300void __init reserve_early_without_check(u64 start, u64 end, char *name)
301{
302 struct early_res *r;
303
304 if (start >= end)
305 return;
306
307 __check_and_double_early_res(start, end);
308
309 r = &early_res[early_res_count];
310
311 r->start = start;
312 r->end = end;
313 r->overlap_ok = 0;
314 if (name)
315 strncpy(r->name, name, sizeof(r->name) - 1);
316 early_res_count++;
317}
318
319void __init free_early(u64 start, u64 end)
320{
321 struct early_res *r;
322 int i;
323
324 kmemleak_free_part(__va(start), end - start);
325
326 i = find_overlapped_early(start, end);
327 r = &early_res[i];
328 if (i >= max_early_res || r->end != end || r->start != start)
329 panic("free_early on not reserved area: %llx-%llx!",
330 start, end - 1);
331
332 drop_range(i);
333}
334
335void __init free_early_partial(u64 start, u64 end)
336{
337 struct early_res *r;
338 int i;
339
340 kmemleak_free_part(__va(start), end - start);
341
342 if (start == end)
343 return;
344
345 if (WARN_ONCE(start > end, " wrong range [%#llx, %#llx]\n", start, end))
346 return;
347
348try_next:
349 i = find_overlapped_early(start, end);
350 if (i >= max_early_res)
351 return;
352
353 r = &early_res[i];
354 /* hole ? */
355 if (r->end >= end && r->start <= start) {
356 drop_range_partial(i, start, end);
357 return;
358 }
359
360 drop_range_partial(i, start, end);
361 goto try_next;
362}
363
364#ifdef CONFIG_NO_BOOTMEM
365static void __init subtract_early_res(struct range *range, int az)
366{
367 int i, count;
368 u64 final_start, final_end;
369 int idx = 0;
370
371 count = 0;
372 for (i = 0; i < max_early_res && early_res[i].end; i++)
373 count++;
374
375 /* need to skip first one ?*/
376 if (early_res != early_res_x)
377 idx = 1;
378
379#define DEBUG_PRINT_EARLY_RES 1
380
381#if DEBUG_PRINT_EARLY_RES
382 printk(KERN_INFO "Subtract (%d early reservations)\n", count);
383#endif
384 for (i = idx; i < count; i++) {
385 struct early_res *r = &early_res[i];
386#if DEBUG_PRINT_EARLY_RES
387 printk(KERN_INFO " #%d [%010llx - %010llx] %15s\n", i,
388 r->start, r->end, r->name);
389#endif
390 final_start = PFN_DOWN(r->start);
391 final_end = PFN_UP(r->end);
392 if (final_start >= final_end)
393 continue;
394 subtract_range(range, az, final_start, final_end);
395 }
396
397}
398
399int __init get_free_all_memory_range(struct range **rangep, int nodeid)
400{
401 int i, count;
402 u64 start = 0, end;
403 u64 size;
404 u64 mem;
405 struct range *range;
406 int nr_range;
407
408 count = 0;
409 for (i = 0; i < max_early_res && early_res[i].end; i++)
410 count++;
411
412 count *= 2;
413
414 size = sizeof(struct range) * count;
415 end = get_max_mapped();
416#ifdef MAX_DMA32_PFN
417 if (end > (MAX_DMA32_PFN << PAGE_SHIFT))
418 start = MAX_DMA32_PFN << PAGE_SHIFT;
419#endif
420 mem = find_fw_memmap_area(start, end, size, sizeof(struct range));
421 if (mem == -1ULL)
422 panic("can not find more space for range free");
423
424 range = __va(mem);
425 /* use early_node_map[] and early_res to get range array at first */
426 memset(range, 0, size);
427 nr_range = 0;
428
429 /* need to go over early_node_map to find out good range for node */
430 nr_range = add_from_early_node_map(range, count, nr_range, nodeid);
431#ifdef CONFIG_X86_32
432 subtract_range(range, count, max_low_pfn, -1ULL);
433#endif
434 subtract_early_res(range, count);
435 nr_range = clean_sort_range(range, count);
436
437 /* need to clear it ? */
438 if (nodeid == MAX_NUMNODES) {
439 memset(&early_res[0], 0,
440 sizeof(struct early_res) * max_early_res);
441 early_res = NULL;
442 max_early_res = 0;
443 }
444
445 *rangep = range;
446 return nr_range;
447}
448#else
449void __init early_res_to_bootmem(u64 start, u64 end)
450{
451 int i, count;
452 u64 final_start, final_end;
453 int idx = 0;
454
455 count = 0;
456 for (i = 0; i < max_early_res && early_res[i].end; i++)
457 count++;
458
459 /* need to skip first one ?*/
460 if (early_res != early_res_x)
461 idx = 1;
462
463 printk(KERN_INFO "(%d/%d early reservations) ==> bootmem [%010llx - %010llx]\n",
464 count - idx, max_early_res, start, end);
465 for (i = idx; i < count; i++) {
466 struct early_res *r = &early_res[i];
467 printk(KERN_INFO " #%d [%010llx - %010llx] %16s", i,
468 r->start, r->end, r->name);
469 final_start = max(start, r->start);
470 final_end = min(end, r->end);
471 if (final_start >= final_end) {
472 printk(KERN_CONT "\n");
473 continue;
474 }
475 printk(KERN_CONT " ==> [%010llx - %010llx]\n",
476 final_start, final_end);
477 reserve_bootmem_generic(final_start, final_end - final_start,
478 BOOTMEM_DEFAULT);
479 }
480 /* clear them */
481 memset(&early_res[0], 0, sizeof(struct early_res) * max_early_res);
482 early_res = NULL;
483 max_early_res = 0;
484 early_res_count = 0;
485}
486#endif
487
488/* Check for already reserved areas */
489static inline int __init bad_addr(u64 *addrp, u64 size, u64 align)
490{
491 int i;
492 u64 addr = *addrp;
493 int changed = 0;
494 struct early_res *r;
495again:
496 i = find_overlapped_early(addr, addr + size);
497 r = &early_res[i];
498 if (i < max_early_res && r->end) {
499 *addrp = addr = round_up(r->end, align);
500 changed = 1;
501 goto again;
502 }
503 return changed;
504}
505
506/* Check for already reserved areas */
507static inline int __init bad_addr_size(u64 *addrp, u64 *sizep, u64 align)
508{
509 int i;
510 u64 addr = *addrp, last;
511 u64 size = *sizep;
512 int changed = 0;
513again:
514 last = addr + size;
515 for (i = 0; i < max_early_res && early_res[i].end; i++) {
516 struct early_res *r = &early_res[i];
517 if (last > r->start && addr < r->start) {
518 size = r->start - addr;
519 changed = 1;
520 goto again;
521 }
522 if (last > r->end && addr < r->end) {
523 addr = round_up(r->end, align);
524 size = last - addr;
525 changed = 1;
526 goto again;
527 }
528 if (last <= r->end && addr >= r->start) {
529 (*sizep)++;
530 return 0;
531 }
532 }
533 if (changed) {
534 *addrp = addr;
535 *sizep = size;
536 }
537 return changed;
538}
539
540/*
541 * Find a free area with specified alignment in a specific range.
542 * only with the area.between start to end is active range from early_node_map
543 * so they are good as RAM
544 */
545u64 __init find_early_area(u64 ei_start, u64 ei_last, u64 start, u64 end,
546 u64 size, u64 align)
547{
548 u64 addr, last;
549
550 addr = round_up(ei_start, align);
551 if (addr < start)
552 addr = round_up(start, align);
553 if (addr >= ei_last)
554 goto out;
555 while (bad_addr(&addr, size, align) && addr+size <= ei_last)
556 ;
557 last = addr + size;
558 if (last > ei_last)
559 goto out;
560 if (last > end)
561 goto out;
562
563 return addr;
564
565out:
566 return -1ULL;
567}
568
569u64 __init find_early_area_size(u64 ei_start, u64 ei_last, u64 start,
570 u64 *sizep, u64 align)
571{
572 u64 addr, last;
573
574 addr = round_up(ei_start, align);
575 if (addr < start)
576 addr = round_up(start, align);
577 if (addr >= ei_last)
578 goto out;
579 *sizep = ei_last - addr;
580 while (bad_addr_size(&addr, sizep, align) && addr + *sizep <= ei_last)
581 ;
582 last = addr + *sizep;
583 if (last > ei_last)
584 goto out;
585
586 return addr;
587
588out:
589 return -1ULL;
590}
diff --git a/mm/bootmem.c b/mm/bootmem.c
index 142c84a54993..13b0caa9793c 100644
--- a/mm/bootmem.c
+++ b/mm/bootmem.c
@@ -15,6 +15,7 @@
15#include <linux/module.h> 15#include <linux/module.h>
16#include <linux/kmemleak.h> 16#include <linux/kmemleak.h>
17#include <linux/range.h> 17#include <linux/range.h>
18#include <linux/memblock.h>
18 19
19#include <asm/bug.h> 20#include <asm/bug.h>
20#include <asm/io.h> 21#include <asm/io.h>
@@ -434,7 +435,8 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
434 unsigned long size) 435 unsigned long size)
435{ 436{
436#ifdef CONFIG_NO_BOOTMEM 437#ifdef CONFIG_NO_BOOTMEM
437 free_early(physaddr, physaddr + size); 438 kmemleak_free_part(__va(physaddr), size);
439 memblock_x86_free_range(physaddr, physaddr + size);
438#else 440#else
439 unsigned long start, end; 441 unsigned long start, end;
440 442
@@ -459,7 +461,8 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
459void __init free_bootmem(unsigned long addr, unsigned long size) 461void __init free_bootmem(unsigned long addr, unsigned long size)
460{ 462{
461#ifdef CONFIG_NO_BOOTMEM 463#ifdef CONFIG_NO_BOOTMEM
462 free_early(addr, addr + size); 464 kmemleak_free_part(__va(addr), size);
465 memblock_x86_free_range(addr, addr + size);
463#else 466#else
464 unsigned long start, end; 467 unsigned long start, end;
465 468
@@ -526,6 +529,12 @@ int __init reserve_bootmem(unsigned long addr, unsigned long size,
526} 529}
527 530
528#ifndef CONFIG_NO_BOOTMEM 531#ifndef CONFIG_NO_BOOTMEM
532int __weak __init reserve_bootmem_generic(unsigned long phys, unsigned long len,
533 int flags)
534{
535 return reserve_bootmem(phys, len, flags);
536}
537
529static unsigned long __init align_idx(struct bootmem_data *bdata, 538static unsigned long __init align_idx(struct bootmem_data *bdata,
530 unsigned long idx, unsigned long step) 539 unsigned long idx, unsigned long step)
531{ 540{
diff --git a/mm/memblock.c b/mm/memblock.c
index 43840b305ecb..400dc62697d7 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -11,237 +11,423 @@
11 */ 11 */
12 12
13#include <linux/kernel.h> 13#include <linux/kernel.h>
14#include <linux/slab.h>
14#include <linux/init.h> 15#include <linux/init.h>
15#include <linux/bitops.h> 16#include <linux/bitops.h>
17#include <linux/poison.h>
18#include <linux/pfn.h>
19#include <linux/debugfs.h>
20#include <linux/seq_file.h>
16#include <linux/memblock.h> 21#include <linux/memblock.h>
17 22
18#define MEMBLOCK_ALLOC_ANYWHERE 0 23struct memblock memblock __initdata_memblock;
19 24
20struct memblock memblock; 25int memblock_debug __initdata_memblock;
26int memblock_can_resize __initdata_memblock;
27static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_REGIONS + 1] __initdata_memblock;
28static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_REGIONS + 1] __initdata_memblock;
21 29
22static int memblock_debug; 30/* inline so we don't get a warning when pr_debug is compiled out */
31static inline const char *memblock_type_name(struct memblock_type *type)
32{
33 if (type == &memblock.memory)
34 return "memory";
35 else if (type == &memblock.reserved)
36 return "reserved";
37 else
38 return "unknown";
39}
23 40
24static int __init early_memblock(char *p) 41/*
42 * Address comparison utilities
43 */
44
45static phys_addr_t __init_memblock memblock_align_down(phys_addr_t addr, phys_addr_t size)
25{ 46{
26 if (p && strstr(p, "debug")) 47 return addr & ~(size - 1);
27 memblock_debug = 1; 48}
49
50static phys_addr_t __init_memblock memblock_align_up(phys_addr_t addr, phys_addr_t size)
51{
52 return (addr + (size - 1)) & ~(size - 1);
53}
54
55static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1,
56 phys_addr_t base2, phys_addr_t size2)
57{
58 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1)));
59}
60
61static long __init_memblock memblock_addrs_adjacent(phys_addr_t base1, phys_addr_t size1,
62 phys_addr_t base2, phys_addr_t size2)
63{
64 if (base2 == base1 + size1)
65 return 1;
66 else if (base1 == base2 + size2)
67 return -1;
68
28 return 0; 69 return 0;
29} 70}
30early_param("memblock", early_memblock);
31 71
32static void memblock_dump(struct memblock_region *region, char *name) 72static long __init_memblock memblock_regions_adjacent(struct memblock_type *type,
73 unsigned long r1, unsigned long r2)
33{ 74{
34 unsigned long long base, size; 75 phys_addr_t base1 = type->regions[r1].base;
35 int i; 76 phys_addr_t size1 = type->regions[r1].size;
77 phys_addr_t base2 = type->regions[r2].base;
78 phys_addr_t size2 = type->regions[r2].size;
36 79
37 pr_info(" %s.cnt = 0x%lx\n", name, region->cnt); 80 return memblock_addrs_adjacent(base1, size1, base2, size2);
81}
38 82
39 for (i = 0; i < region->cnt; i++) { 83long __init_memblock memblock_overlaps_region(struct memblock_type *type, phys_addr_t base, phys_addr_t size)
40 base = region->region[i].base; 84{
41 size = region->region[i].size; 85 unsigned long i;
42 86
43 pr_info(" %s[0x%x]\t0x%016llx - 0x%016llx, 0x%llx bytes\n", 87 for (i = 0; i < type->cnt; i++) {
44 name, i, base, base + size - 1, size); 88 phys_addr_t rgnbase = type->regions[i].base;
89 phys_addr_t rgnsize = type->regions[i].size;
90 if (memblock_addrs_overlap(base, size, rgnbase, rgnsize))
91 break;
45 } 92 }
93
94 return (i < type->cnt) ? i : -1;
46} 95}
47 96
48void memblock_dump_all(void) 97/*
98 * Find, allocate, deallocate or reserve unreserved regions. All allocations
99 * are top-down.
100 */
101
102static phys_addr_t __init_memblock memblock_find_region(phys_addr_t start, phys_addr_t end,
103 phys_addr_t size, phys_addr_t align)
49{ 104{
50 if (!memblock_debug) 105 phys_addr_t base, res_base;
51 return; 106 long j;
52 107
53 pr_info("MEMBLOCK configuration:\n"); 108 /* In case, huge size is requested */
54 pr_info(" rmo_size = 0x%llx\n", (unsigned long long)memblock.rmo_size); 109 if (end < size)
55 pr_info(" memory.size = 0x%llx\n", (unsigned long long)memblock.memory.size); 110 return MEMBLOCK_ERROR;
56 111
57 memblock_dump(&memblock.memory, "memory"); 112 base = memblock_align_down((end - size), align);
58 memblock_dump(&memblock.reserved, "reserved"); 113
114 /* Prevent allocations returning 0 as it's also used to
115 * indicate an allocation failure
116 */
117 if (start == 0)
118 start = PAGE_SIZE;
119
120 while (start <= base) {
121 j = memblock_overlaps_region(&memblock.reserved, base, size);
122 if (j < 0)
123 return base;
124 res_base = memblock.reserved.regions[j].base;
125 if (res_base < size)
126 break;
127 base = memblock_align_down(res_base - size, align);
128 }
129
130 return MEMBLOCK_ERROR;
59} 131}
60 132
61static unsigned long memblock_addrs_overlap(u64 base1, u64 size1, u64 base2, 133static phys_addr_t __init_memblock memblock_find_base(phys_addr_t size,
62 u64 size2) 134 phys_addr_t align, phys_addr_t start, phys_addr_t end)
63{ 135{
64 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); 136 long i;
137
138 BUG_ON(0 == size);
139
140 size = memblock_align_up(size, align);
141
142 /* Pump up max_addr */
143 if (end == MEMBLOCK_ALLOC_ACCESSIBLE)
144 end = memblock.current_limit;
145
146 /* We do a top-down search, this tends to limit memory
147 * fragmentation by keeping early boot allocs near the
148 * top of memory
149 */
150 for (i = memblock.memory.cnt - 1; i >= 0; i--) {
151 phys_addr_t memblockbase = memblock.memory.regions[i].base;
152 phys_addr_t memblocksize = memblock.memory.regions[i].size;
153 phys_addr_t bottom, top, found;
154
155 if (memblocksize < size)
156 continue;
157 if ((memblockbase + memblocksize) <= start)
158 break;
159 bottom = max(memblockbase, start);
160 top = min(memblockbase + memblocksize, end);
161 if (bottom >= top)
162 continue;
163 found = memblock_find_region(bottom, top, size, align);
164 if (found != MEMBLOCK_ERROR)
165 return found;
166 }
167 return MEMBLOCK_ERROR;
65} 168}
66 169
67static long memblock_addrs_adjacent(u64 base1, u64 size1, u64 base2, u64 size2) 170/*
171 * Find a free area with specified alignment in a specific range.
172 */
173u64 __init_memblock memblock_find_in_range(u64 start, u64 end, u64 size, u64 align)
68{ 174{
69 if (base2 == base1 + size1) 175 return memblock_find_base(size, align, start, end);
70 return 1; 176}
71 else if (base1 == base2 + size2)
72 return -1;
73 177
74 return 0; 178/*
179 * Free memblock.reserved.regions
180 */
181int __init_memblock memblock_free_reserved_regions(void)
182{
183 if (memblock.reserved.regions == memblock_reserved_init_regions)
184 return 0;
185
186 return memblock_free(__pa(memblock.reserved.regions),
187 sizeof(struct memblock_region) * memblock.reserved.max);
75} 188}
76 189
77static long memblock_regions_adjacent(struct memblock_region *rgn, 190/*
78 unsigned long r1, unsigned long r2) 191 * Reserve memblock.reserved.regions
192 */
193int __init_memblock memblock_reserve_reserved_regions(void)
79{ 194{
80 u64 base1 = rgn->region[r1].base; 195 if (memblock.reserved.regions == memblock_reserved_init_regions)
81 u64 size1 = rgn->region[r1].size; 196 return 0;
82 u64 base2 = rgn->region[r2].base;
83 u64 size2 = rgn->region[r2].size;
84 197
85 return memblock_addrs_adjacent(base1, size1, base2, size2); 198 return memblock_reserve(__pa(memblock.reserved.regions),
199 sizeof(struct memblock_region) * memblock.reserved.max);
86} 200}
87 201
88static void memblock_remove_region(struct memblock_region *rgn, unsigned long r) 202static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r)
89{ 203{
90 unsigned long i; 204 unsigned long i;
91 205
92 for (i = r; i < rgn->cnt - 1; i++) { 206 for (i = r; i < type->cnt - 1; i++) {
93 rgn->region[i].base = rgn->region[i + 1].base; 207 type->regions[i].base = type->regions[i + 1].base;
94 rgn->region[i].size = rgn->region[i + 1].size; 208 type->regions[i].size = type->regions[i + 1].size;
95 } 209 }
96 rgn->cnt--; 210 type->cnt--;
97} 211}
98 212
99/* Assumption: base addr of region 1 < base addr of region 2 */ 213/* Assumption: base addr of region 1 < base addr of region 2 */
100static void memblock_coalesce_regions(struct memblock_region *rgn, 214static void __init_memblock memblock_coalesce_regions(struct memblock_type *type,
101 unsigned long r1, unsigned long r2) 215 unsigned long r1, unsigned long r2)
102{ 216{
103 rgn->region[r1].size += rgn->region[r2].size; 217 type->regions[r1].size += type->regions[r2].size;
104 memblock_remove_region(rgn, r2); 218 memblock_remove_region(type, r2);
105} 219}
106 220
107void __init memblock_init(void) 221/* Defined below but needed now */
222static long memblock_add_region(struct memblock_type *type, phys_addr_t base, phys_addr_t size);
223
224static int __init_memblock memblock_double_array(struct memblock_type *type)
108{ 225{
109 /* Create a dummy zero size MEMBLOCK which will get coalesced away later. 226 struct memblock_region *new_array, *old_array;
110 * This simplifies the memblock_add() code below... 227 phys_addr_t old_size, new_size, addr;
228 int use_slab = slab_is_available();
229
230 /* We don't allow resizing until we know about the reserved regions
231 * of memory that aren't suitable for allocation
111 */ 232 */
112 memblock.memory.region[0].base = 0; 233 if (!memblock_can_resize)
113 memblock.memory.region[0].size = 0; 234 return -1;
114 memblock.memory.cnt = 1;
115 235
116 /* Ditto. */ 236 /* Calculate new doubled size */
117 memblock.reserved.region[0].base = 0; 237 old_size = type->max * sizeof(struct memblock_region);
118 memblock.reserved.region[0].size = 0; 238 new_size = old_size << 1;
119 memblock.reserved.cnt = 1; 239
120} 240 /* Try to find some space for it.
241 *
242 * WARNING: We assume that either slab_is_available() and we use it or
243 * we use MEMBLOCK for allocations. That means that this is unsafe to use
244 * when bootmem is currently active (unless bootmem itself is implemented
245 * on top of MEMBLOCK which isn't the case yet)
246 *
247 * This should however not be an issue for now, as we currently only
248 * call into MEMBLOCK while it's still active, or much later when slab is
249 * active for memory hotplug operations
250 */
251 if (use_slab) {
252 new_array = kmalloc(new_size, GFP_KERNEL);
253 addr = new_array == NULL ? MEMBLOCK_ERROR : __pa(new_array);
254 } else
255 addr = memblock_find_base(new_size, sizeof(phys_addr_t), 0, MEMBLOCK_ALLOC_ACCESSIBLE);
256 if (addr == MEMBLOCK_ERROR) {
257 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n",
258 memblock_type_name(type), type->max, type->max * 2);
259 return -1;
260 }
261 new_array = __va(addr);
121 262
122void __init memblock_analyze(void) 263 memblock_dbg("memblock: %s array is doubled to %ld at [%#010llx-%#010llx]",
123{ 264 memblock_type_name(type), type->max * 2, (u64)addr, (u64)addr + new_size - 1);
124 int i;
125 265
126 memblock.memory.size = 0; 266 /* Found space, we now need to move the array over before
267 * we add the reserved region since it may be our reserved
268 * array itself that is full.
269 */
270 memcpy(new_array, type->regions, old_size);
271 memset(new_array + type->max, 0, old_size);
272 old_array = type->regions;
273 type->regions = new_array;
274 type->max <<= 1;
275
276 /* If we use SLAB that's it, we are done */
277 if (use_slab)
278 return 0;
127 279
128 for (i = 0; i < memblock.memory.cnt; i++) 280 /* Add the new reserved region now. Should not fail ! */
129 memblock.memory.size += memblock.memory.region[i].size; 281 BUG_ON(memblock_add_region(&memblock.reserved, addr, new_size) < 0);
282
283 /* If the array wasn't our static init one, then free it. We only do
284 * that before SLAB is available as later on, we don't know whether
285 * to use kfree or free_bootmem_pages(). Shouldn't be a big deal
286 * anyways
287 */
288 if (old_array != memblock_memory_init_regions &&
289 old_array != memblock_reserved_init_regions)
290 memblock_free(__pa(old_array), old_size);
291
292 return 0;
130} 293}
131 294
132static long memblock_add_region(struct memblock_region *rgn, u64 base, u64 size) 295extern int __init_memblock __weak memblock_memory_can_coalesce(phys_addr_t addr1, phys_addr_t size1,
296 phys_addr_t addr2, phys_addr_t size2)
297{
298 return 1;
299}
300
301static long __init_memblock memblock_add_region(struct memblock_type *type, phys_addr_t base, phys_addr_t size)
133{ 302{
134 unsigned long coalesced = 0; 303 unsigned long coalesced = 0;
135 long adjacent, i; 304 long adjacent, i;
136 305
137 if ((rgn->cnt == 1) && (rgn->region[0].size == 0)) { 306 if ((type->cnt == 1) && (type->regions[0].size == 0)) {
138 rgn->region[0].base = base; 307 type->regions[0].base = base;
139 rgn->region[0].size = size; 308 type->regions[0].size = size;
140 return 0; 309 return 0;
141 } 310 }
142 311
143 /* First try and coalesce this MEMBLOCK with another. */ 312 /* First try and coalesce this MEMBLOCK with another. */
144 for (i = 0; i < rgn->cnt; i++) { 313 for (i = 0; i < type->cnt; i++) {
145 u64 rgnbase = rgn->region[i].base; 314 phys_addr_t rgnbase = type->regions[i].base;
146 u64 rgnsize = rgn->region[i].size; 315 phys_addr_t rgnsize = type->regions[i].size;
147 316
148 if ((rgnbase == base) && (rgnsize == size)) 317 if ((rgnbase == base) && (rgnsize == size))
149 /* Already have this region, so we're done */ 318 /* Already have this region, so we're done */
150 return 0; 319 return 0;
151 320
152 adjacent = memblock_addrs_adjacent(base, size, rgnbase, rgnsize); 321 adjacent = memblock_addrs_adjacent(base, size, rgnbase, rgnsize);
322 /* Check if arch allows coalescing */
323 if (adjacent != 0 && type == &memblock.memory &&
324 !memblock_memory_can_coalesce(base, size, rgnbase, rgnsize))
325 break;
153 if (adjacent > 0) { 326 if (adjacent > 0) {
154 rgn->region[i].base -= size; 327 type->regions[i].base -= size;
155 rgn->region[i].size += size; 328 type->regions[i].size += size;
156 coalesced++; 329 coalesced++;
157 break; 330 break;
158 } else if (adjacent < 0) { 331 } else if (adjacent < 0) {
159 rgn->region[i].size += size; 332 type->regions[i].size += size;
160 coalesced++; 333 coalesced++;
161 break; 334 break;
162 } 335 }
163 } 336 }
164 337
165 if ((i < rgn->cnt - 1) && memblock_regions_adjacent(rgn, i, i+1)) { 338 /* If we plugged a hole, we may want to also coalesce with the
166 memblock_coalesce_regions(rgn, i, i+1); 339 * next region
340 */
341 if ((i < type->cnt - 1) && memblock_regions_adjacent(type, i, i+1) &&
342 ((type != &memblock.memory || memblock_memory_can_coalesce(type->regions[i].base,
343 type->regions[i].size,
344 type->regions[i+1].base,
345 type->regions[i+1].size)))) {
346 memblock_coalesce_regions(type, i, i+1);
167 coalesced++; 347 coalesced++;
168 } 348 }
169 349
170 if (coalesced) 350 if (coalesced)
171 return coalesced; 351 return coalesced;
172 if (rgn->cnt >= MAX_MEMBLOCK_REGIONS) 352
353 /* If we are out of space, we fail. It's too late to resize the array
354 * but then this shouldn't have happened in the first place.
355 */
356 if (WARN_ON(type->cnt >= type->max))
173 return -1; 357 return -1;
174 358
175 /* Couldn't coalesce the MEMBLOCK, so add it to the sorted table. */ 359 /* Couldn't coalesce the MEMBLOCK, so add it to the sorted table. */
176 for (i = rgn->cnt - 1; i >= 0; i--) { 360 for (i = type->cnt - 1; i >= 0; i--) {
177 if (base < rgn->region[i].base) { 361 if (base < type->regions[i].base) {
178 rgn->region[i+1].base = rgn->region[i].base; 362 type->regions[i+1].base = type->regions[i].base;
179 rgn->region[i+1].size = rgn->region[i].size; 363 type->regions[i+1].size = type->regions[i].size;
180 } else { 364 } else {
181 rgn->region[i+1].base = base; 365 type->regions[i+1].base = base;
182 rgn->region[i+1].size = size; 366 type->regions[i+1].size = size;
183 break; 367 break;
184 } 368 }
185 } 369 }
186 370
187 if (base < rgn->region[0].base) { 371 if (base < type->regions[0].base) {
188 rgn->region[0].base = base; 372 type->regions[0].base = base;
189 rgn->region[0].size = size; 373 type->regions[0].size = size;
374 }
375 type->cnt++;
376
377 /* The array is full ? Try to resize it. If that fails, we undo
378 * our allocation and return an error
379 */
380 if (type->cnt == type->max && memblock_double_array(type)) {
381 type->cnt--;
382 return -1;
190 } 383 }
191 rgn->cnt++;
192 384
193 return 0; 385 return 0;
194} 386}
195 387
196long memblock_add(u64 base, u64 size) 388long __init_memblock memblock_add(phys_addr_t base, phys_addr_t size)
197{ 389{
198 struct memblock_region *_rgn = &memblock.memory; 390 return memblock_add_region(&memblock.memory, base, size);
199
200 /* On pSeries LPAR systems, the first MEMBLOCK is our RMO region. */
201 if (base == 0)
202 memblock.rmo_size = size;
203
204 return memblock_add_region(_rgn, base, size);
205 391
206} 392}
207 393
208static long __memblock_remove(struct memblock_region *rgn, u64 base, u64 size) 394static long __init_memblock __memblock_remove(struct memblock_type *type, phys_addr_t base, phys_addr_t size)
209{ 395{
210 u64 rgnbegin, rgnend; 396 phys_addr_t rgnbegin, rgnend;
211 u64 end = base + size; 397 phys_addr_t end = base + size;
212 int i; 398 int i;
213 399
214 rgnbegin = rgnend = 0; /* supress gcc warnings */ 400 rgnbegin = rgnend = 0; /* supress gcc warnings */
215 401
216 /* Find the region where (base, size) belongs to */ 402 /* Find the region where (base, size) belongs to */
217 for (i=0; i < rgn->cnt; i++) { 403 for (i=0; i < type->cnt; i++) {
218 rgnbegin = rgn->region[i].base; 404 rgnbegin = type->regions[i].base;
219 rgnend = rgnbegin + rgn->region[i].size; 405 rgnend = rgnbegin + type->regions[i].size;
220 406
221 if ((rgnbegin <= base) && (end <= rgnend)) 407 if ((rgnbegin <= base) && (end <= rgnend))
222 break; 408 break;
223 } 409 }
224 410
225 /* Didn't find the region */ 411 /* Didn't find the region */
226 if (i == rgn->cnt) 412 if (i == type->cnt)
227 return -1; 413 return -1;
228 414
229 /* Check to see if we are removing entire region */ 415 /* Check to see if we are removing entire region */
230 if ((rgnbegin == base) && (rgnend == end)) { 416 if ((rgnbegin == base) && (rgnend == end)) {
231 memblock_remove_region(rgn, i); 417 memblock_remove_region(type, i);
232 return 0; 418 return 0;
233 } 419 }
234 420
235 /* Check to see if region is matching at the front */ 421 /* Check to see if region is matching at the front */
236 if (rgnbegin == base) { 422 if (rgnbegin == base) {
237 rgn->region[i].base = end; 423 type->regions[i].base = end;
238 rgn->region[i].size -= size; 424 type->regions[i].size -= size;
239 return 0; 425 return 0;
240 } 426 }
241 427
242 /* Check to see if the region is matching at the end */ 428 /* Check to see if the region is matching at the end */
243 if (rgnend == end) { 429 if (rgnend == end) {
244 rgn->region[i].size -= size; 430 type->regions[i].size -= size;
245 return 0; 431 return 0;
246 } 432 }
247 433
@@ -249,208 +435,189 @@ static long __memblock_remove(struct memblock_region *rgn, u64 base, u64 size)
249 * We need to split the entry - adjust the current one to the 435 * We need to split the entry - adjust the current one to the
250 * beginging of the hole and add the region after hole. 436 * beginging of the hole and add the region after hole.
251 */ 437 */
252 rgn->region[i].size = base - rgn->region[i].base; 438 type->regions[i].size = base - type->regions[i].base;
253 return memblock_add_region(rgn, end, rgnend - end); 439 return memblock_add_region(type, end, rgnend - end);
254} 440}
255 441
256long memblock_remove(u64 base, u64 size) 442long __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size)
257{ 443{
258 return __memblock_remove(&memblock.memory, base, size); 444 return __memblock_remove(&memblock.memory, base, size);
259} 445}
260 446
261long __init memblock_free(u64 base, u64 size) 447long __init_memblock memblock_free(phys_addr_t base, phys_addr_t size)
262{ 448{
263 return __memblock_remove(&memblock.reserved, base, size); 449 return __memblock_remove(&memblock.reserved, base, size);
264} 450}
265 451
266long __init memblock_reserve(u64 base, u64 size) 452long __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
267{ 453{
268 struct memblock_region *_rgn = &memblock.reserved; 454 struct memblock_type *_rgn = &memblock.reserved;
269 455
270 BUG_ON(0 == size); 456 BUG_ON(0 == size);
271 457
272 return memblock_add_region(_rgn, base, size); 458 return memblock_add_region(_rgn, base, size);
273} 459}
274 460
275long memblock_overlaps_region(struct memblock_region *rgn, u64 base, u64 size) 461phys_addr_t __init __memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr)
276{ 462{
277 unsigned long i; 463 phys_addr_t found;
278 464
279 for (i = 0; i < rgn->cnt; i++) { 465 /* We align the size to limit fragmentation. Without this, a lot of
280 u64 rgnbase = rgn->region[i].base; 466 * small allocs quickly eat up the whole reserve array on sparc
281 u64 rgnsize = rgn->region[i].size; 467 */
282 if (memblock_addrs_overlap(base, size, rgnbase, rgnsize)) 468 size = memblock_align_up(size, align);
283 break;
284 }
285 469
286 return (i < rgn->cnt) ? i : -1; 470 found = memblock_find_base(size, align, 0, max_addr);
471 if (found != MEMBLOCK_ERROR &&
472 memblock_add_region(&memblock.reserved, found, size) >= 0)
473 return found;
474
475 return 0;
287} 476}
288 477
289static u64 memblock_align_down(u64 addr, u64 size) 478phys_addr_t __init memblock_alloc_base(phys_addr_t size, phys_addr_t align, phys_addr_t max_addr)
290{ 479{
291 return addr & ~(size - 1); 480 phys_addr_t alloc;
481
482 alloc = __memblock_alloc_base(size, align, max_addr);
483
484 if (alloc == 0)
485 panic("ERROR: Failed to allocate 0x%llx bytes below 0x%llx.\n",
486 (unsigned long long) size, (unsigned long long) max_addr);
487
488 return alloc;
292} 489}
293 490
294static u64 memblock_align_up(u64 addr, u64 size) 491phys_addr_t __init memblock_alloc(phys_addr_t size, phys_addr_t align)
295{ 492{
296 return (addr + (size - 1)) & ~(size - 1); 493 return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ACCESSIBLE);
297} 494}
298 495
299static u64 __init memblock_alloc_nid_unreserved(u64 start, u64 end, 496
300 u64 size, u64 align) 497/*
498 * Additional node-local allocators. Search for node memory is bottom up
499 * and walks memblock regions within that node bottom-up as well, but allocation
500 * within an memblock region is top-down. XXX I plan to fix that at some stage
501 *
502 * WARNING: Only available after early_node_map[] has been populated,
503 * on some architectures, that is after all the calls to add_active_range()
504 * have been done to populate it.
505 */
506
507phys_addr_t __weak __init memblock_nid_range(phys_addr_t start, phys_addr_t end, int *nid)
301{ 508{
302 u64 base, res_base; 509#ifdef CONFIG_ARCH_POPULATES_NODE_MAP
303 long j; 510 /*
511 * This code originates from sparc which really wants use to walk by addresses
512 * and returns the nid. This is not very convenient for early_pfn_map[] users
513 * as the map isn't sorted yet, and it really wants to be walked by nid.
514 *
515 * For now, I implement the inefficient method below which walks the early
516 * map multiple times. Eventually we may want to use an ARCH config option
517 * to implement a completely different method for both case.
518 */
519 unsigned long start_pfn, end_pfn;
520 int i;
304 521
305 base = memblock_align_down((end - size), align); 522 for (i = 0; i < MAX_NUMNODES; i++) {
306 while (start <= base) { 523 get_pfn_range_for_nid(i, &start_pfn, &end_pfn);
307 j = memblock_overlaps_region(&memblock.reserved, base, size); 524 if (start < PFN_PHYS(start_pfn) || start >= PFN_PHYS(end_pfn))
308 if (j < 0) { 525 continue;
309 /* this area isn't reserved, take it */ 526 *nid = i;
310 if (memblock_add_region(&memblock.reserved, base, size) < 0) 527 return min(end, PFN_PHYS(end_pfn));
311 base = ~(u64)0;
312 return base;
313 }
314 res_base = memblock.reserved.region[j].base;
315 if (res_base < size)
316 break;
317 base = memblock_align_down(res_base - size, align);
318 } 528 }
529#endif
530 *nid = 0;
319 531
320 return ~(u64)0; 532 return end;
321} 533}
322 534
323static u64 __init memblock_alloc_nid_region(struct memblock_property *mp, 535static phys_addr_t __init memblock_alloc_nid_region(struct memblock_region *mp,
324 u64 (*nid_range)(u64, u64, int *), 536 phys_addr_t size,
325 u64 size, u64 align, int nid) 537 phys_addr_t align, int nid)
326{ 538{
327 u64 start, end; 539 phys_addr_t start, end;
328 540
329 start = mp->base; 541 start = mp->base;
330 end = start + mp->size; 542 end = start + mp->size;
331 543
332 start = memblock_align_up(start, align); 544 start = memblock_align_up(start, align);
333 while (start < end) { 545 while (start < end) {
334 u64 this_end; 546 phys_addr_t this_end;
335 int this_nid; 547 int this_nid;
336 548
337 this_end = nid_range(start, end, &this_nid); 549 this_end = memblock_nid_range(start, end, &this_nid);
338 if (this_nid == nid) { 550 if (this_nid == nid) {
339 u64 ret = memblock_alloc_nid_unreserved(start, this_end, 551 phys_addr_t ret = memblock_find_region(start, this_end, size, align);
340 size, align); 552 if (ret != MEMBLOCK_ERROR &&
341 if (ret != ~(u64)0) 553 memblock_add_region(&memblock.reserved, ret, size) >= 0)
342 return ret; 554 return ret;
343 } 555 }
344 start = this_end; 556 start = this_end;
345 } 557 }
346 558
347 return ~(u64)0; 559 return MEMBLOCK_ERROR;
348} 560}
349 561
350u64 __init memblock_alloc_nid(u64 size, u64 align, int nid, 562phys_addr_t __init memblock_alloc_nid(phys_addr_t size, phys_addr_t align, int nid)
351 u64 (*nid_range)(u64 start, u64 end, int *nid))
352{ 563{
353 struct memblock_region *mem = &memblock.memory; 564 struct memblock_type *mem = &memblock.memory;
354 int i; 565 int i;
355 566
356 BUG_ON(0 == size); 567 BUG_ON(0 == size);
357 568
569 /* We align the size to limit fragmentation. Without this, a lot of
570 * small allocs quickly eat up the whole reserve array on sparc
571 */
358 size = memblock_align_up(size, align); 572 size = memblock_align_up(size, align);
359 573
574 /* We do a bottom-up search for a region with the right
575 * nid since that's easier considering how memblock_nid_range()
576 * works
577 */
360 for (i = 0; i < mem->cnt; i++) { 578 for (i = 0; i < mem->cnt; i++) {
361 u64 ret = memblock_alloc_nid_region(&mem->region[i], 579 phys_addr_t ret = memblock_alloc_nid_region(&mem->regions[i],
362 nid_range,
363 size, align, nid); 580 size, align, nid);
364 if (ret != ~(u64)0) 581 if (ret != MEMBLOCK_ERROR)
365 return ret; 582 return ret;
366 } 583 }
367 584
368 return memblock_alloc(size, align); 585 return 0;
369}
370
371u64 __init memblock_alloc(u64 size, u64 align)
372{
373 return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ANYWHERE);
374} 586}
375 587
376u64 __init memblock_alloc_base(u64 size, u64 align, u64 max_addr) 588phys_addr_t __init memblock_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
377{ 589{
378 u64 alloc; 590 phys_addr_t res = memblock_alloc_nid(size, align, nid);
379
380 alloc = __memblock_alloc_base(size, align, max_addr);
381 591
382 if (alloc == 0) 592 if (res)
383 panic("ERROR: Failed to allocate 0x%llx bytes below 0x%llx.\n", 593 return res;
384 (unsigned long long) size, (unsigned long long) max_addr); 594 return memblock_alloc_base(size, align, MEMBLOCK_ALLOC_ANYWHERE);
385
386 return alloc;
387} 595}
388 596
389u64 __init __memblock_alloc_base(u64 size, u64 align, u64 max_addr)
390{
391 long i, j;
392 u64 base = 0;
393 u64 res_base;
394
395 BUG_ON(0 == size);
396 597
397 size = memblock_align_up(size, align); 598/*
398 599 * Remaining API functions
399 /* On some platforms, make sure we allocate lowmem */ 600 */
400 /* Note that MEMBLOCK_REAL_LIMIT may be MEMBLOCK_ALLOC_ANYWHERE */
401 if (max_addr == MEMBLOCK_ALLOC_ANYWHERE)
402 max_addr = MEMBLOCK_REAL_LIMIT;
403
404 for (i = memblock.memory.cnt - 1; i >= 0; i--) {
405 u64 memblockbase = memblock.memory.region[i].base;
406 u64 memblocksize = memblock.memory.region[i].size;
407
408 if (memblocksize < size)
409 continue;
410 if (max_addr == MEMBLOCK_ALLOC_ANYWHERE)
411 base = memblock_align_down(memblockbase + memblocksize - size, align);
412 else if (memblockbase < max_addr) {
413 base = min(memblockbase + memblocksize, max_addr);
414 base = memblock_align_down(base - size, align);
415 } else
416 continue;
417
418 while (base && memblockbase <= base) {
419 j = memblock_overlaps_region(&memblock.reserved, base, size);
420 if (j < 0) {
421 /* this area isn't reserved, take it */
422 if (memblock_add_region(&memblock.reserved, base, size) < 0)
423 return 0;
424 return base;
425 }
426 res_base = memblock.reserved.region[j].base;
427 if (res_base < size)
428 break;
429 base = memblock_align_down(res_base - size, align);
430 }
431 }
432 return 0;
433}
434 601
435/* You must call memblock_analyze() before this. */ 602/* You must call memblock_analyze() before this. */
436u64 __init memblock_phys_mem_size(void) 603phys_addr_t __init memblock_phys_mem_size(void)
437{ 604{
438 return memblock.memory.size; 605 return memblock.memory_size;
439} 606}
440 607
441u64 memblock_end_of_DRAM(void) 608phys_addr_t __init_memblock memblock_end_of_DRAM(void)
442{ 609{
443 int idx = memblock.memory.cnt - 1; 610 int idx = memblock.memory.cnt - 1;
444 611
445 return (memblock.memory.region[idx].base + memblock.memory.region[idx].size); 612 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size);
446} 613}
447 614
448/* You must call memblock_analyze() after this. */ 615/* You must call memblock_analyze() after this. */
449void __init memblock_enforce_memory_limit(u64 memory_limit) 616void __init memblock_enforce_memory_limit(phys_addr_t memory_limit)
450{ 617{
451 unsigned long i; 618 unsigned long i;
452 u64 limit; 619 phys_addr_t limit;
453 struct memblock_property *p; 620 struct memblock_region *p;
454 621
455 if (!memory_limit) 622 if (!memory_limit)
456 return; 623 return;
@@ -458,24 +625,21 @@ void __init memblock_enforce_memory_limit(u64 memory_limit)
458 /* Truncate the memblock regions to satisfy the memory limit. */ 625 /* Truncate the memblock regions to satisfy the memory limit. */
459 limit = memory_limit; 626 limit = memory_limit;
460 for (i = 0; i < memblock.memory.cnt; i++) { 627 for (i = 0; i < memblock.memory.cnt; i++) {
461 if (limit > memblock.memory.region[i].size) { 628 if (limit > memblock.memory.regions[i].size) {
462 limit -= memblock.memory.region[i].size; 629 limit -= memblock.memory.regions[i].size;
463 continue; 630 continue;
464 } 631 }
465 632
466 memblock.memory.region[i].size = limit; 633 memblock.memory.regions[i].size = limit;
467 memblock.memory.cnt = i + 1; 634 memblock.memory.cnt = i + 1;
468 break; 635 break;
469 } 636 }
470 637
471 if (memblock.memory.region[0].size < memblock.rmo_size)
472 memblock.rmo_size = memblock.memory.region[0].size;
473
474 memory_limit = memblock_end_of_DRAM(); 638 memory_limit = memblock_end_of_DRAM();
475 639
476 /* And truncate any reserves above the limit also. */ 640 /* And truncate any reserves above the limit also. */
477 for (i = 0; i < memblock.reserved.cnt; i++) { 641 for (i = 0; i < memblock.reserved.cnt; i++) {
478 p = &memblock.reserved.region[i]; 642 p = &memblock.reserved.regions[i];
479 643
480 if (p->base > memory_limit) 644 if (p->base > memory_limit)
481 p->size = 0; 645 p->size = 0;
@@ -489,53 +653,190 @@ void __init memblock_enforce_memory_limit(u64 memory_limit)
489 } 653 }
490} 654}
491 655
492int __init memblock_is_reserved(u64 addr) 656static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr)
657{
658 unsigned int left = 0, right = type->cnt;
659
660 do {
661 unsigned int mid = (right + left) / 2;
662
663 if (addr < type->regions[mid].base)
664 right = mid;
665 else if (addr >= (type->regions[mid].base +
666 type->regions[mid].size))
667 left = mid + 1;
668 else
669 return mid;
670 } while (left < right);
671 return -1;
672}
673
674int __init memblock_is_reserved(phys_addr_t addr)
675{
676 return memblock_search(&memblock.reserved, addr) != -1;
677}
678
679int __init_memblock memblock_is_memory(phys_addr_t addr)
680{
681 return memblock_search(&memblock.memory, addr) != -1;
682}
683
684int __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size)
685{
686 int idx = memblock_search(&memblock.reserved, base);
687
688 if (idx == -1)
689 return 0;
690 return memblock.reserved.regions[idx].base <= base &&
691 (memblock.reserved.regions[idx].base +
692 memblock.reserved.regions[idx].size) >= (base + size);
693}
694
695int __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size)
696{
697 return memblock_overlaps_region(&memblock.reserved, base, size) >= 0;
698}
699
700
701void __init_memblock memblock_set_current_limit(phys_addr_t limit)
493{ 702{
703 memblock.current_limit = limit;
704}
705
706static void __init_memblock memblock_dump(struct memblock_type *region, char *name)
707{
708 unsigned long long base, size;
494 int i; 709 int i;
495 710
496 for (i = 0; i < memblock.reserved.cnt; i++) { 711 pr_info(" %s.cnt = 0x%lx\n", name, region->cnt);
497 u64 upper = memblock.reserved.region[i].base + 712
498 memblock.reserved.region[i].size - 1; 713 for (i = 0; i < region->cnt; i++) {
499 if ((addr >= memblock.reserved.region[i].base) && (addr <= upper)) 714 base = region->regions[i].base;
500 return 1; 715 size = region->regions[i].size;
716
717 pr_info(" %s[%#x]\t[%#016llx-%#016llx], %#llx bytes\n",
718 name, i, base, base + size - 1, size);
501 } 719 }
502 return 0;
503} 720}
504 721
505int memblock_is_region_reserved(u64 base, u64 size) 722void __init_memblock memblock_dump_all(void)
506{ 723{
507 return memblock_overlaps_region(&memblock.reserved, base, size) >= 0; 724 if (!memblock_debug)
725 return;
726
727 pr_info("MEMBLOCK configuration:\n");
728 pr_info(" memory size = 0x%llx\n", (unsigned long long)memblock.memory_size);
729
730 memblock_dump(&memblock.memory, "memory");
731 memblock_dump(&memblock.reserved, "reserved");
508} 732}
509 733
510/* 734void __init memblock_analyze(void)
511 * Given a <base, len>, find which memory regions belong to this range.
512 * Adjust the request and return a contiguous chunk.
513 */
514int memblock_find(struct memblock_property *res)
515{ 735{
516 int i; 736 int i;
517 u64 rstart, rend;
518 737
519 rstart = res->base; 738 /* Check marker in the unused last array entry */
520 rend = rstart + res->size - 1; 739 WARN_ON(memblock_memory_init_regions[INIT_MEMBLOCK_REGIONS].base
740 != (phys_addr_t)RED_INACTIVE);
741 WARN_ON(memblock_reserved_init_regions[INIT_MEMBLOCK_REGIONS].base
742 != (phys_addr_t)RED_INACTIVE);
743
744 memblock.memory_size = 0;
745
746 for (i = 0; i < memblock.memory.cnt; i++)
747 memblock.memory_size += memblock.memory.regions[i].size;
748
749 /* We allow resizing from there */
750 memblock_can_resize = 1;
751}
752
753void __init memblock_init(void)
754{
755 static int init_done __initdata = 0;
756
757 if (init_done)
758 return;
759 init_done = 1;
760
761 /* Hookup the initial arrays */
762 memblock.memory.regions = memblock_memory_init_regions;
763 memblock.memory.max = INIT_MEMBLOCK_REGIONS;
764 memblock.reserved.regions = memblock_reserved_init_regions;
765 memblock.reserved.max = INIT_MEMBLOCK_REGIONS;
766
767 /* Write a marker in the unused last array entry */
768 memblock.memory.regions[INIT_MEMBLOCK_REGIONS].base = (phys_addr_t)RED_INACTIVE;
769 memblock.reserved.regions[INIT_MEMBLOCK_REGIONS].base = (phys_addr_t)RED_INACTIVE;
770
771 /* Create a dummy zero size MEMBLOCK which will get coalesced away later.
772 * This simplifies the memblock_add() code below...
773 */
774 memblock.memory.regions[0].base = 0;
775 memblock.memory.regions[0].size = 0;
776 memblock.memory.cnt = 1;
777
778 /* Ditto. */
779 memblock.reserved.regions[0].base = 0;
780 memblock.reserved.regions[0].size = 0;
781 memblock.reserved.cnt = 1;
782
783 memblock.current_limit = MEMBLOCK_ALLOC_ANYWHERE;
784}
785
786static int __init early_memblock(char *p)
787{
788 if (p && strstr(p, "debug"))
789 memblock_debug = 1;
790 return 0;
791}
792early_param("memblock", early_memblock);
793
794#if defined(CONFIG_DEBUG_FS) && !defined(ARCH_DISCARD_MEMBLOCK)
795
796static int memblock_debug_show(struct seq_file *m, void *private)
797{
798 struct memblock_type *type = m->private;
799 struct memblock_region *reg;
800 int i;
801
802 for (i = 0; i < type->cnt; i++) {
803 reg = &type->regions[i];
804 seq_printf(m, "%4d: ", i);
805 if (sizeof(phys_addr_t) == 4)
806 seq_printf(m, "0x%08lx..0x%08lx\n",
807 (unsigned long)reg->base,
808 (unsigned long)(reg->base + reg->size - 1));
809 else
810 seq_printf(m, "0x%016llx..0x%016llx\n",
811 (unsigned long long)reg->base,
812 (unsigned long long)(reg->base + reg->size - 1));
521 813
522 for (i = 0; i < memblock.memory.cnt; i++) {
523 u64 start = memblock.memory.region[i].base;
524 u64 end = start + memblock.memory.region[i].size - 1;
525
526 if (start > rend)
527 return -1;
528
529 if ((end >= rstart) && (start < rend)) {
530 /* adjust the request */
531 if (rstart < start)
532 rstart = start;
533 if (rend > end)
534 rend = end;
535 res->base = rstart;
536 res->size = rend - rstart + 1;
537 return 0;
538 }
539 } 814 }
540 return -1; 815 return 0;
816}
817
818static int memblock_debug_open(struct inode *inode, struct file *file)
819{
820 return single_open(file, memblock_debug_show, inode->i_private);
541} 821}
822
823static const struct file_operations memblock_debug_fops = {
824 .open = memblock_debug_open,
825 .read = seq_read,
826 .llseek = seq_lseek,
827 .release = single_release,
828};
829
830static int __init memblock_init_debugfs(void)
831{
832 struct dentry *root = debugfs_create_dir("memblock", NULL);
833 if (!root)
834 return -ENXIO;
835 debugfs_create_file("memory", S_IRUGO, root, &memblock.memory, &memblock_debug_fops);
836 debugfs_create_file("reserved", S_IRUGO, root, &memblock.reserved, &memblock_debug_fops);
837
838 return 0;
839}
840__initcall(memblock_init_debugfs);
841
842#endif /* CONFIG_DEBUG_FS */
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index f12ad1836abe..2a362c52fdf4 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -21,6 +21,7 @@
21#include <linux/pagemap.h> 21#include <linux/pagemap.h>
22#include <linux/jiffies.h> 22#include <linux/jiffies.h>
23#include <linux/bootmem.h> 23#include <linux/bootmem.h>
24#include <linux/memblock.h>
24#include <linux/compiler.h> 25#include <linux/compiler.h>
25#include <linux/kernel.h> 26#include <linux/kernel.h>
26#include <linux/kmemcheck.h> 27#include <linux/kmemcheck.h>
@@ -3636,6 +3637,41 @@ void __init free_bootmem_with_active_regions(int nid,
3636 } 3637 }
3637} 3638}
3638 3639
3640#ifdef CONFIG_HAVE_MEMBLOCK
3641u64 __init find_memory_core_early(int nid, u64 size, u64 align,
3642 u64 goal, u64 limit)
3643{
3644 int i;
3645
3646 /* Need to go over early_node_map to find out good range for node */
3647 for_each_active_range_index_in_nid(i, nid) {
3648 u64 addr;
3649 u64 ei_start, ei_last;
3650 u64 final_start, final_end;
3651
3652 ei_last = early_node_map[i].end_pfn;
3653 ei_last <<= PAGE_SHIFT;
3654 ei_start = early_node_map[i].start_pfn;
3655 ei_start <<= PAGE_SHIFT;
3656
3657 final_start = max(ei_start, goal);
3658 final_end = min(ei_last, limit);
3659
3660 if (final_start >= final_end)
3661 continue;
3662
3663 addr = memblock_find_in_range(final_start, final_end, size, align);
3664
3665 if (addr == MEMBLOCK_ERROR)
3666 continue;
3667
3668 return addr;
3669 }
3670
3671 return MEMBLOCK_ERROR;
3672}
3673#endif
3674
3639int __init add_from_early_node_map(struct range *range, int az, 3675int __init add_from_early_node_map(struct range *range, int az,
3640 int nr_range, int nid) 3676 int nr_range, int nid)
3641{ 3677{
@@ -3655,46 +3691,26 @@ int __init add_from_early_node_map(struct range *range, int az,
3655void * __init __alloc_memory_core_early(int nid, u64 size, u64 align, 3691void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
3656 u64 goal, u64 limit) 3692 u64 goal, u64 limit)
3657{ 3693{
3658 int i;
3659 void *ptr; 3694 void *ptr;
3695 u64 addr;
3660 3696
3661 if (limit > get_max_mapped()) 3697 if (limit > memblock.current_limit)
3662 limit = get_max_mapped(); 3698 limit = memblock.current_limit;
3663
3664 /* need to go over early_node_map to find out good range for node */
3665 for_each_active_range_index_in_nid(i, nid) {
3666 u64 addr;
3667 u64 ei_start, ei_last;
3668
3669 ei_last = early_node_map[i].end_pfn;
3670 ei_last <<= PAGE_SHIFT;
3671 ei_start = early_node_map[i].start_pfn;
3672 ei_start <<= PAGE_SHIFT;
3673 addr = find_early_area(ei_start, ei_last,
3674 goal, limit, size, align);
3675
3676 if (addr == -1ULL)
3677 continue;
3678 3699
3679#if 0 3700 addr = find_memory_core_early(nid, size, align, goal, limit);
3680 printk(KERN_DEBUG "alloc (nid=%d %llx - %llx) (%llx - %llx) %llx %llx => %llx\n",
3681 nid,
3682 ei_start, ei_last, goal, limit, size,
3683 align, addr);
3684#endif
3685 3701
3686 ptr = phys_to_virt(addr); 3702 if (addr == MEMBLOCK_ERROR)
3687 memset(ptr, 0, size); 3703 return NULL;
3688 reserve_early_without_check(addr, addr + size, "BOOTMEM");
3689 /*
3690 * The min_count is set to 0 so that bootmem allocated blocks
3691 * are never reported as leaks.
3692 */
3693 kmemleak_alloc(ptr, size, 0, 0);
3694 return ptr;
3695 }
3696 3704
3697 return NULL; 3705 ptr = phys_to_virt(addr);
3706 memset(ptr, 0, size);
3707 memblock_x86_reserve_range(addr, addr + size, "BOOTMEM");
3708 /*
3709 * The min_count is set to 0 so that bootmem allocated blocks
3710 * are never reported as leaks.
3711 */
3712 kmemleak_alloc(ptr, size, 0, 0);
3713 return ptr;
3698} 3714}
3699#endif 3715#endif
3700 3716
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index aa33fd67fa41..29d6cbffb283 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -220,18 +220,7 @@ void __init sparse_mem_maps_populate_node(struct page **map_map,
220 220
221 if (vmemmap_buf_start) { 221 if (vmemmap_buf_start) {
222 /* need to free left buf */ 222 /* need to free left buf */
223#ifdef CONFIG_NO_BOOTMEM
224 free_early(__pa(vmemmap_buf_start), __pa(vmemmap_buf_end));
225 if (vmemmap_buf_start < vmemmap_buf) {
226 char name[15];
227
228 snprintf(name, sizeof(name), "MEMMAP %d", nodeid);
229 reserve_early_without_check(__pa(vmemmap_buf_start),
230 __pa(vmemmap_buf), name);
231 }
232#else
233 free_bootmem(__pa(vmemmap_buf), vmemmap_buf_end - vmemmap_buf); 223 free_bootmem(__pa(vmemmap_buf), vmemmap_buf_end - vmemmap_buf);
234#endif
235 vmemmap_buf = NULL; 224 vmemmap_buf = NULL;
236 vmemmap_buf_end = NULL; 225 vmemmap_buf_end = NULL;
237 } 226 }
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-03 00:42:11 -0500