aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/arm64/mm/mmu.c13
-rw-r--r--arch/ia64/mm/discontig.c7
-rw-r--r--arch/powerpc/mm/init_64.c11
-rw-r--r--arch/s390/mm/vmem.c15
-rw-r--r--arch/sparc/mm/init_64.c7
-rw-r--r--arch/x86/mm/init_64.c15
-rw-r--r--include/linux/mm.h8
-rw-r--r--mm/sparse-vmemmap.c19
-rw-r--r--mm/sparse.c10
9 files changed, 49 insertions, 56 deletions
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 70b8cd4021c4..eeecc9c8ed68 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -391,17 +391,14 @@ int kern_addr_valid(unsigned long addr)
391} 391}
392#ifdef CONFIG_SPARSEMEM_VMEMMAP 392#ifdef CONFIG_SPARSEMEM_VMEMMAP
393#ifdef CONFIG_ARM64_64K_PAGES 393#ifdef CONFIG_ARM64_64K_PAGES
394int __meminit vmemmap_populate(struct page *start_page, 394int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
395 unsigned long size, int node)
396{ 395{
397 return vmemmap_populate_basepages(start_page, size, node); 396 return vmemmap_populate_basepages(start, end, node);
398} 397}
399#else /* !CONFIG_ARM64_64K_PAGES */ 398#else /* !CONFIG_ARM64_64K_PAGES */
400int __meminit vmemmap_populate(struct page *start_page, 399int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
401 unsigned long size, int node)
402{ 400{
403 unsigned long addr = (unsigned long)start_page; 401 unsigned long addr = start;
404 unsigned long end = (unsigned long)(start_page + size);
405 unsigned long next; 402 unsigned long next;
406 pgd_t *pgd; 403 pgd_t *pgd;
407 pud_t *pud; 404 pud_t *pud;
@@ -434,7 +431,7 @@ int __meminit vmemmap_populate(struct page *start_page,
434 return 0; 431 return 0;
435} 432}
436#endif /* CONFIG_ARM64_64K_PAGES */ 433#endif /* CONFIG_ARM64_64K_PAGES */
437void vmemmap_free(struct page *memmap, unsigned long nr_pages) 434void vmemmap_free(unsigned long start, unsigned long end)
438{ 435{
439} 436}
440#endif /* CONFIG_SPARSEMEM_VMEMMAP */ 437#endif /* CONFIG_SPARSEMEM_VMEMMAP */
diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c
index a57436e5d405..ae4db4bd6d97 100644
--- a/arch/ia64/mm/discontig.c
+++ b/arch/ia64/mm/discontig.c
@@ -819,13 +819,12 @@ void arch_refresh_nodedata(int update_node, pg_data_t *update_pgdat)
819#endif 819#endif
820 820
821#ifdef CONFIG_SPARSEMEM_VMEMMAP 821#ifdef CONFIG_SPARSEMEM_VMEMMAP
822int __meminit vmemmap_populate(struct page *start_page, 822int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
823 unsigned long size, int node)
824{ 823{
825 return vmemmap_populate_basepages(start_page, size, node); 824 return vmemmap_populate_basepages(start, end, node);
826} 825}
827 826
828void vmemmap_free(struct page *memmap, unsigned long nr_pages) 827void vmemmap_free(unsigned long start, unsigned long end)
829{ 828{
830} 829}
831#endif 830#endif
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index 7e2246fb2f31..5a535b73ea18 100644
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -263,19 +263,14 @@ static __meminit void vmemmap_list_populate(unsigned long phys,
263 vmemmap_list = vmem_back; 263 vmemmap_list = vmem_back;
264} 264}
265 265
266int __meminit vmemmap_populate(struct page *start_page, 266int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
267 unsigned long nr_pages, int node)
268{ 267{
269 unsigned long start = (unsigned long)start_page;
270 unsigned long end = (unsigned long)(start_page + nr_pages);
271 unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift; 268 unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
272 269
273 /* Align to the page size of the linear mapping. */ 270 /* Align to the page size of the linear mapping. */
274 start = _ALIGN_DOWN(start, page_size); 271 start = _ALIGN_DOWN(start, page_size);
275 272
276 pr_debug("vmemmap_populate page %p, %ld pages, node %d\n", 273 pr_debug("vmemmap_populate %lx..%lx, node %d\n", start, end, node);
277 start_page, nr_pages, node);
278 pr_debug(" -> map %lx..%lx\n", start, end);
279 274
280 for (; start < end; start += page_size) { 275 for (; start < end; start += page_size) {
281 void *p; 276 void *p;
@@ -298,7 +293,7 @@ int __meminit vmemmap_populate(struct page *start_page,
298 return 0; 293 return 0;
299} 294}
300 295
301void vmemmap_free(struct page *memmap, unsigned long nr_pages) 296void vmemmap_free(unsigned long start, unsigned long end)
302{ 297{
303} 298}
304 299
diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index ffab84db6907..35837054f734 100644
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -191,19 +191,16 @@ static void vmem_remove_range(unsigned long start, unsigned long size)
191/* 191/*
192 * Add a backed mem_map array to the virtual mem_map array. 192 * Add a backed mem_map array to the virtual mem_map array.
193 */ 193 */
194int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) 194int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
195{ 195{
196 unsigned long address, start_addr, end_addr; 196 unsigned long address = start;
197 pgd_t *pg_dir; 197 pgd_t *pg_dir;
198 pud_t *pu_dir; 198 pud_t *pu_dir;
199 pmd_t *pm_dir; 199 pmd_t *pm_dir;
200 pte_t *pt_dir; 200 pte_t *pt_dir;
201 int ret = -ENOMEM; 201 int ret = -ENOMEM;
202 202
203 start_addr = (unsigned long) start; 203 for (address = start; address < end;) {
204 end_addr = (unsigned long) (start + nr);
205
206 for (address = start_addr; address < end_addr;) {
207 pg_dir = pgd_offset_k(address); 204 pg_dir = pgd_offset_k(address);
208 if (pgd_none(*pg_dir)) { 205 if (pgd_none(*pg_dir)) {
209 pu_dir = vmem_pud_alloc(); 206 pu_dir = vmem_pud_alloc();
@@ -262,14 +259,14 @@ int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node)
262 } 259 }
263 address += PAGE_SIZE; 260 address += PAGE_SIZE;
264 } 261 }
265 memset(start, 0, nr * sizeof(struct page)); 262 memset((void *)start, 0, end - start);
266 ret = 0; 263 ret = 0;
267out: 264out:
268 flush_tlb_kernel_range(start_addr, end_addr); 265 flush_tlb_kernel_range(start, end);
269 return ret; 266 return ret;
270} 267}
271 268
272void vmemmap_free(struct page *memmap, unsigned long nr_pages) 269void vmemmap_free(unsigned long start, unsigned long end)
273{ 270{
274} 271}
275 272
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 1588d33d5492..6ac99d64a13c 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -2181,10 +2181,9 @@ unsigned long vmemmap_table[VMEMMAP_SIZE];
2181static long __meminitdata addr_start, addr_end; 2181static long __meminitdata addr_start, addr_end;
2182static int __meminitdata node_start; 2182static int __meminitdata node_start;
2183 2183
2184int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) 2184int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
2185 int node)
2185{ 2186{
2186 unsigned long vstart = (unsigned long) start;
2187 unsigned long vend = (unsigned long) (start + nr);
2188 unsigned long phys_start = (vstart - VMEMMAP_BASE); 2187 unsigned long phys_start = (vstart - VMEMMAP_BASE);
2189 unsigned long phys_end = (vend - VMEMMAP_BASE); 2188 unsigned long phys_end = (vend - VMEMMAP_BASE);
2190 unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK; 2189 unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK;
@@ -2236,7 +2235,7 @@ void __meminit vmemmap_populate_print_last(void)
2236 } 2235 }
2237} 2236}
2238 2237
2239void vmemmap_free(struct page *memmap, unsigned long nr_pages) 2238void vmemmap_free(unsigned long start, unsigned long end)
2240{ 2239{
2241} 2240}
2242 2241
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 2ef81f19bd6c..528c143f467c 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1011,11 +1011,8 @@ remove_pagetable(unsigned long start, unsigned long end, bool direct)
1011 flush_tlb_all(); 1011 flush_tlb_all();
1012} 1012}
1013 1013
1014void __ref vmemmap_free(struct page *memmap, unsigned long nr_pages) 1014void __ref vmemmap_free(unsigned long start, unsigned long end)
1015{ 1015{
1016 unsigned long start = (unsigned long)memmap;
1017 unsigned long end = (unsigned long)(memmap + nr_pages);
1018
1019 remove_pagetable(start, end, false); 1016 remove_pagetable(start, end, false);
1020} 1017}
1021 1018
@@ -1284,17 +1281,15 @@ static long __meminitdata addr_start, addr_end;
1284static void __meminitdata *p_start, *p_end; 1281static void __meminitdata *p_start, *p_end;
1285static int __meminitdata node_start; 1282static int __meminitdata node_start;
1286 1283
1287int __meminit 1284int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
1288vmemmap_populate(struct page *start_page, unsigned long size, int node)
1289{ 1285{
1290 unsigned long addr = (unsigned long)start_page; 1286 unsigned long addr;
1291 unsigned long end = (unsigned long)(start_page + size);
1292 unsigned long next; 1287 unsigned long next;
1293 pgd_t *pgd; 1288 pgd_t *pgd;
1294 pud_t *pud; 1289 pud_t *pud;
1295 pmd_t *pmd; 1290 pmd_t *pmd;
1296 1291
1297 for (; addr < end; addr = next) { 1292 for (addr = start; addr < end; addr = next) {
1298 void *p = NULL; 1293 void *p = NULL;
1299 1294
1300 pgd = vmemmap_pgd_populate(addr, node); 1295 pgd = vmemmap_pgd_populate(addr, node);
@@ -1351,7 +1346,7 @@ vmemmap_populate(struct page *start_page, unsigned long size, int node)
1351 } 1346 }
1352 1347
1353 } 1348 }
1354 sync_global_pgds((unsigned long)start_page, end - 1); 1349 sync_global_pgds(start, end - 1);
1355 return 0; 1350 return 0;
1356} 1351}
1357 1352
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 98a44376e4f3..6d7266842abd 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1764,12 +1764,12 @@ pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
1764void *vmemmap_alloc_block(unsigned long size, int node); 1764void *vmemmap_alloc_block(unsigned long size, int node);
1765void *vmemmap_alloc_block_buf(unsigned long size, int node); 1765void *vmemmap_alloc_block_buf(unsigned long size, int node);
1766void vmemmap_verify(pte_t *, int, unsigned long, unsigned long); 1766void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
1767int vmemmap_populate_basepages(struct page *start_page, 1767int vmemmap_populate_basepages(unsigned long start, unsigned long end,
1768 unsigned long pages, int node); 1768 int node);
1769int vmemmap_populate(struct page *start_page, unsigned long pages, int node); 1769int vmemmap_populate(unsigned long start, unsigned long end, int node);
1770void vmemmap_populate_print_last(void); 1770void vmemmap_populate_print_last(void);
1771#ifdef CONFIG_MEMORY_HOTPLUG 1771#ifdef CONFIG_MEMORY_HOTPLUG
1772void vmemmap_free(struct page *memmap, unsigned long nr_pages); 1772void vmemmap_free(unsigned long start, unsigned long end);
1773#endif 1773#endif
1774void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, 1774void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
1775 unsigned long size); 1775 unsigned long size);
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 22b7e18e9dea..27eeab3be757 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -147,11 +147,10 @@ pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
147 return pgd; 147 return pgd;
148} 148}
149 149
150int __meminit vmemmap_populate_basepages(struct page *start_page, 150int __meminit vmemmap_populate_basepages(unsigned long start,
151 unsigned long size, int node) 151 unsigned long end, int node)
152{ 152{
153 unsigned long addr = (unsigned long)start_page; 153 unsigned long addr = start;
154 unsigned long end = (unsigned long)(start_page + size);
155 pgd_t *pgd; 154 pgd_t *pgd;
156 pud_t *pud; 155 pud_t *pud;
157 pmd_t *pmd; 156 pmd_t *pmd;
@@ -178,9 +177,15 @@ int __meminit vmemmap_populate_basepages(struct page *start_page,
178 177
179struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid) 178struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid)
180{ 179{
181 struct page *map = pfn_to_page(pnum * PAGES_PER_SECTION); 180 unsigned long start;
182 int error = vmemmap_populate(map, PAGES_PER_SECTION, nid); 181 unsigned long end;
183 if (error) 182 struct page *map;
183
184 map = pfn_to_page(pnum * PAGES_PER_SECTION);
185 start = (unsigned long)map;
186 end = (unsigned long)(map + PAGES_PER_SECTION);
187
188 if (vmemmap_populate(start, end, nid))
184 return NULL; 189 return NULL;
185 190
186 return map; 191 return map;
diff --git a/mm/sparse.c b/mm/sparse.c
index 7ca6dc847947..a37be5f9050d 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -615,11 +615,17 @@ static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
615} 615}
616static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 616static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages)
617{ 617{
618 vmemmap_free(memmap, nr_pages); 618 unsigned long start = (unsigned long)memmap;
619 unsigned long end = (unsigned long)(memmap + nr_pages);
620
621 vmemmap_free(start, end);
619} 622}
620static void free_map_bootmem(struct page *memmap, unsigned long nr_pages) 623static void free_map_bootmem(struct page *memmap, unsigned long nr_pages)
621{ 624{
622 vmemmap_free(memmap, nr_pages); 625 unsigned long start = (unsigned long)memmap;
626 unsigned long end = (unsigned long)(memmap + nr_pages);
627
628 vmemmap_free(start, end);
623} 629}
624#else 630#else
625static struct page *__kmalloc_section_memmap(unsigned long nr_pages) 631static struct page *__kmalloc_section_memmap(unsigned long nr_pages)