diff options
-rw-r--r-- | arch/arm64/mm/mmu.c | 13 | ||||
-rw-r--r-- | arch/ia64/mm/discontig.c | 7 | ||||
-rw-r--r-- | arch/powerpc/mm/init_64.c | 11 | ||||
-rw-r--r-- | arch/s390/mm/vmem.c | 15 | ||||
-rw-r--r-- | arch/sparc/mm/init_64.c | 7 | ||||
-rw-r--r-- | arch/x86/mm/init_64.c | 15 | ||||
-rw-r--r-- | include/linux/mm.h | 8 | ||||
-rw-r--r-- | mm/sparse-vmemmap.c | 19 | ||||
-rw-r--r-- | mm/sparse.c | 10 |
9 files changed, 49 insertions, 56 deletions
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 70b8cd4021c4..eeecc9c8ed68 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c | |||
@@ -391,17 +391,14 @@ int kern_addr_valid(unsigned long addr) | |||
391 | } | 391 | } |
392 | #ifdef CONFIG_SPARSEMEM_VMEMMAP | 392 | #ifdef CONFIG_SPARSEMEM_VMEMMAP |
393 | #ifdef CONFIG_ARM64_64K_PAGES | 393 | #ifdef CONFIG_ARM64_64K_PAGES |
394 | int __meminit vmemmap_populate(struct page *start_page, | 394 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) |
395 | unsigned long size, int node) | ||
396 | { | 395 | { |
397 | return vmemmap_populate_basepages(start_page, size, node); | 396 | return vmemmap_populate_basepages(start, end, node); |
398 | } | 397 | } |
399 | #else /* !CONFIG_ARM64_64K_PAGES */ | 398 | #else /* !CONFIG_ARM64_64K_PAGES */ |
400 | int __meminit vmemmap_populate(struct page *start_page, | 399 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) |
401 | unsigned long size, int node) | ||
402 | { | 400 | { |
403 | unsigned long addr = (unsigned long)start_page; | 401 | unsigned long addr = start; |
404 | unsigned long end = (unsigned long)(start_page + size); | ||
405 | unsigned long next; | 402 | unsigned long next; |
406 | pgd_t *pgd; | 403 | pgd_t *pgd; |
407 | pud_t *pud; | 404 | pud_t *pud; |
@@ -434,7 +431,7 @@ int __meminit vmemmap_populate(struct page *start_page, | |||
434 | return 0; | 431 | return 0; |
435 | } | 432 | } |
436 | #endif /* CONFIG_ARM64_64K_PAGES */ | 433 | #endif /* CONFIG_ARM64_64K_PAGES */ |
437 | void vmemmap_free(struct page *memmap, unsigned long nr_pages) | 434 | void vmemmap_free(unsigned long start, unsigned long end) |
438 | { | 435 | { |
439 | } | 436 | } |
440 | #endif /* CONFIG_SPARSEMEM_VMEMMAP */ | 437 | #endif /* CONFIG_SPARSEMEM_VMEMMAP */ |
diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c index a57436e5d405..ae4db4bd6d97 100644 --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c | |||
@@ -819,13 +819,12 @@ void arch_refresh_nodedata(int update_node, pg_data_t *update_pgdat) | |||
819 | #endif | 819 | #endif |
820 | 820 | ||
821 | #ifdef CONFIG_SPARSEMEM_VMEMMAP | 821 | #ifdef CONFIG_SPARSEMEM_VMEMMAP |
822 | int __meminit vmemmap_populate(struct page *start_page, | 822 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) |
823 | unsigned long size, int node) | ||
824 | { | 823 | { |
825 | return vmemmap_populate_basepages(start_page, size, node); | 824 | return vmemmap_populate_basepages(start, end, node); |
826 | } | 825 | } |
827 | 826 | ||
828 | void vmemmap_free(struct page *memmap, unsigned long nr_pages) | 827 | void vmemmap_free(unsigned long start, unsigned long end) |
829 | { | 828 | { |
830 | } | 829 | } |
831 | #endif | 830 | #endif |
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c index 7e2246fb2f31..5a535b73ea18 100644 --- a/arch/powerpc/mm/init_64.c +++ b/arch/powerpc/mm/init_64.c | |||
@@ -263,19 +263,14 @@ static __meminit void vmemmap_list_populate(unsigned long phys, | |||
263 | vmemmap_list = vmem_back; | 263 | vmemmap_list = vmem_back; |
264 | } | 264 | } |
265 | 265 | ||
266 | int __meminit vmemmap_populate(struct page *start_page, | 266 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) |
267 | unsigned long nr_pages, int node) | ||
268 | { | 267 | { |
269 | unsigned long start = (unsigned long)start_page; | ||
270 | unsigned long end = (unsigned long)(start_page + nr_pages); | ||
271 | unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift; | 268 | unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift; |
272 | 269 | ||
273 | /* Align to the page size of the linear mapping. */ | 270 | /* Align to the page size of the linear mapping. */ |
274 | start = _ALIGN_DOWN(start, page_size); | 271 | start = _ALIGN_DOWN(start, page_size); |
275 | 272 | ||
276 | pr_debug("vmemmap_populate page %p, %ld pages, node %d\n", | 273 | pr_debug("vmemmap_populate %lx..%lx, node %d\n", start, end, node); |
277 | start_page, nr_pages, node); | ||
278 | pr_debug(" -> map %lx..%lx\n", start, end); | ||
279 | 274 | ||
280 | for (; start < end; start += page_size) { | 275 | for (; start < end; start += page_size) { |
281 | void *p; | 276 | void *p; |
@@ -298,7 +293,7 @@ int __meminit vmemmap_populate(struct page *start_page, | |||
298 | return 0; | 293 | return 0; |
299 | } | 294 | } |
300 | 295 | ||
301 | void vmemmap_free(struct page *memmap, unsigned long nr_pages) | 296 | void vmemmap_free(unsigned long start, unsigned long end) |
302 | { | 297 | { |
303 | } | 298 | } |
304 | 299 | ||
diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index ffab84db6907..35837054f734 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c | |||
@@ -191,19 +191,16 @@ static void vmem_remove_range(unsigned long start, unsigned long size) | |||
191 | /* | 191 | /* |
192 | * Add a backed mem_map array to the virtual mem_map array. | 192 | * Add a backed mem_map array to the virtual mem_map array. |
193 | */ | 193 | */ |
194 | int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) | 194 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) |
195 | { | 195 | { |
196 | unsigned long address, start_addr, end_addr; | 196 | unsigned long address = start; |
197 | pgd_t *pg_dir; | 197 | pgd_t *pg_dir; |
198 | pud_t *pu_dir; | 198 | pud_t *pu_dir; |
199 | pmd_t *pm_dir; | 199 | pmd_t *pm_dir; |
200 | pte_t *pt_dir; | 200 | pte_t *pt_dir; |
201 | int ret = -ENOMEM; | 201 | int ret = -ENOMEM; |
202 | 202 | ||
203 | start_addr = (unsigned long) start; | 203 | for (address = start; address < end;) { |
204 | end_addr = (unsigned long) (start + nr); | ||
205 | |||
206 | for (address = start_addr; address < end_addr;) { | ||
207 | pg_dir = pgd_offset_k(address); | 204 | pg_dir = pgd_offset_k(address); |
208 | if (pgd_none(*pg_dir)) { | 205 | if (pgd_none(*pg_dir)) { |
209 | pu_dir = vmem_pud_alloc(); | 206 | pu_dir = vmem_pud_alloc(); |
@@ -262,14 +259,14 @@ int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) | |||
262 | } | 259 | } |
263 | address += PAGE_SIZE; | 260 | address += PAGE_SIZE; |
264 | } | 261 | } |
265 | memset(start, 0, nr * sizeof(struct page)); | 262 | memset((void *)start, 0, end - start); |
266 | ret = 0; | 263 | ret = 0; |
267 | out: | 264 | out: |
268 | flush_tlb_kernel_range(start_addr, end_addr); | 265 | flush_tlb_kernel_range(start, end); |
269 | return ret; | 266 | return ret; |
270 | } | 267 | } |
271 | 268 | ||
272 | void vmemmap_free(struct page *memmap, unsigned long nr_pages) | 269 | void vmemmap_free(unsigned long start, unsigned long end) |
273 | { | 270 | { |
274 | } | 271 | } |
275 | 272 | ||
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 1588d33d5492..6ac99d64a13c 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c | |||
@@ -2181,10 +2181,9 @@ unsigned long vmemmap_table[VMEMMAP_SIZE]; | |||
2181 | static long __meminitdata addr_start, addr_end; | 2181 | static long __meminitdata addr_start, addr_end; |
2182 | static int __meminitdata node_start; | 2182 | static int __meminitdata node_start; |
2183 | 2183 | ||
2184 | int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) | 2184 | int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend, |
2185 | int node) | ||
2185 | { | 2186 | { |
2186 | unsigned long vstart = (unsigned long) start; | ||
2187 | unsigned long vend = (unsigned long) (start + nr); | ||
2188 | unsigned long phys_start = (vstart - VMEMMAP_BASE); | 2187 | unsigned long phys_start = (vstart - VMEMMAP_BASE); |
2189 | unsigned long phys_end = (vend - VMEMMAP_BASE); | 2188 | unsigned long phys_end = (vend - VMEMMAP_BASE); |
2190 | unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK; | 2189 | unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK; |
@@ -2236,7 +2235,7 @@ void __meminit vmemmap_populate_print_last(void) | |||
2236 | } | 2235 | } |
2237 | } | 2236 | } |
2238 | 2237 | ||
2239 | void vmemmap_free(struct page *memmap, unsigned long nr_pages) | 2238 | void vmemmap_free(unsigned long start, unsigned long end) |
2240 | { | 2239 | { |
2241 | } | 2240 | } |
2242 | 2241 | ||
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 2ef81f19bd6c..528c143f467c 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c | |||
@@ -1011,11 +1011,8 @@ remove_pagetable(unsigned long start, unsigned long end, bool direct) | |||
1011 | flush_tlb_all(); | 1011 | flush_tlb_all(); |
1012 | } | 1012 | } |
1013 | 1013 | ||
1014 | void __ref vmemmap_free(struct page *memmap, unsigned long nr_pages) | 1014 | void __ref vmemmap_free(unsigned long start, unsigned long end) |
1015 | { | 1015 | { |
1016 | unsigned long start = (unsigned long)memmap; | ||
1017 | unsigned long end = (unsigned long)(memmap + nr_pages); | ||
1018 | |||
1019 | remove_pagetable(start, end, false); | 1016 | remove_pagetable(start, end, false); |
1020 | } | 1017 | } |
1021 | 1018 | ||
@@ -1284,17 +1281,15 @@ static long __meminitdata addr_start, addr_end; | |||
1284 | static void __meminitdata *p_start, *p_end; | 1281 | static void __meminitdata *p_start, *p_end; |
1285 | static int __meminitdata node_start; | 1282 | static int __meminitdata node_start; |
1286 | 1283 | ||
1287 | int __meminit | 1284 | int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) |
1288 | vmemmap_populate(struct page *start_page, unsigned long size, int node) | ||
1289 | { | 1285 | { |
1290 | unsigned long addr = (unsigned long)start_page; | 1286 | unsigned long addr; |
1291 | unsigned long end = (unsigned long)(start_page + size); | ||
1292 | unsigned long next; | 1287 | unsigned long next; |
1293 | pgd_t *pgd; | 1288 | pgd_t *pgd; |
1294 | pud_t *pud; | 1289 | pud_t *pud; |
1295 | pmd_t *pmd; | 1290 | pmd_t *pmd; |
1296 | 1291 | ||
1297 | for (; addr < end; addr = next) { | 1292 | for (addr = start; addr < end; addr = next) { |
1298 | void *p = NULL; | 1293 | void *p = NULL; |
1299 | 1294 | ||
1300 | pgd = vmemmap_pgd_populate(addr, node); | 1295 | pgd = vmemmap_pgd_populate(addr, node); |
@@ -1351,7 +1346,7 @@ vmemmap_populate(struct page *start_page, unsigned long size, int node) | |||
1351 | } | 1346 | } |
1352 | 1347 | ||
1353 | } | 1348 | } |
1354 | sync_global_pgds((unsigned long)start_page, end - 1); | 1349 | sync_global_pgds(start, end - 1); |
1355 | return 0; | 1350 | return 0; |
1356 | } | 1351 | } |
1357 | 1352 | ||
diff --git a/include/linux/mm.h b/include/linux/mm.h index 98a44376e4f3..6d7266842abd 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h | |||
@@ -1764,12 +1764,12 @@ pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node); | |||
1764 | void *vmemmap_alloc_block(unsigned long size, int node); | 1764 | void *vmemmap_alloc_block(unsigned long size, int node); |
1765 | void *vmemmap_alloc_block_buf(unsigned long size, int node); | 1765 | void *vmemmap_alloc_block_buf(unsigned long size, int node); |
1766 | void vmemmap_verify(pte_t *, int, unsigned long, unsigned long); | 1766 | void vmemmap_verify(pte_t *, int, unsigned long, unsigned long); |
1767 | int vmemmap_populate_basepages(struct page *start_page, | 1767 | int vmemmap_populate_basepages(unsigned long start, unsigned long end, |
1768 | unsigned long pages, int node); | 1768 | int node); |
1769 | int vmemmap_populate(struct page *start_page, unsigned long pages, int node); | 1769 | int vmemmap_populate(unsigned long start, unsigned long end, int node); |
1770 | void vmemmap_populate_print_last(void); | 1770 | void vmemmap_populate_print_last(void); |
1771 | #ifdef CONFIG_MEMORY_HOTPLUG | 1771 | #ifdef CONFIG_MEMORY_HOTPLUG |
1772 | void vmemmap_free(struct page *memmap, unsigned long nr_pages); | 1772 | void vmemmap_free(unsigned long start, unsigned long end); |
1773 | #endif | 1773 | #endif |
1774 | void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, | 1774 | void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, |
1775 | unsigned long size); | 1775 | unsigned long size); |
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 22b7e18e9dea..27eeab3be757 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c | |||
@@ -147,11 +147,10 @@ pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node) | |||
147 | return pgd; | 147 | return pgd; |
148 | } | 148 | } |
149 | 149 | ||
150 | int __meminit vmemmap_populate_basepages(struct page *start_page, | 150 | int __meminit vmemmap_populate_basepages(unsigned long start, |
151 | unsigned long size, int node) | 151 | unsigned long end, int node) |
152 | { | 152 | { |
153 | unsigned long addr = (unsigned long)start_page; | 153 | unsigned long addr = start; |
154 | unsigned long end = (unsigned long)(start_page + size); | ||
155 | pgd_t *pgd; | 154 | pgd_t *pgd; |
156 | pud_t *pud; | 155 | pud_t *pud; |
157 | pmd_t *pmd; | 156 | pmd_t *pmd; |
@@ -178,9 +177,15 @@ int __meminit vmemmap_populate_basepages(struct page *start_page, | |||
178 | 177 | ||
179 | struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid) | 178 | struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid) |
180 | { | 179 | { |
181 | struct page *map = pfn_to_page(pnum * PAGES_PER_SECTION); | 180 | unsigned long start; |
182 | int error = vmemmap_populate(map, PAGES_PER_SECTION, nid); | 181 | unsigned long end; |
183 | if (error) | 182 | struct page *map; |
183 | |||
184 | map = pfn_to_page(pnum * PAGES_PER_SECTION); | ||
185 | start = (unsigned long)map; | ||
186 | end = (unsigned long)(map + PAGES_PER_SECTION); | ||
187 | |||
188 | if (vmemmap_populate(start, end, nid)) | ||
184 | return NULL; | 189 | return NULL; |
185 | 190 | ||
186 | return map; | 191 | return map; |
diff --git a/mm/sparse.c b/mm/sparse.c index 7ca6dc847947..a37be5f9050d 100644 --- a/mm/sparse.c +++ b/mm/sparse.c | |||
@@ -615,11 +615,17 @@ static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, | |||
615 | } | 615 | } |
616 | static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) | 616 | static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) |
617 | { | 617 | { |
618 | vmemmap_free(memmap, nr_pages); | 618 | unsigned long start = (unsigned long)memmap; |
619 | unsigned long end = (unsigned long)(memmap + nr_pages); | ||
620 | |||
621 | vmemmap_free(start, end); | ||
619 | } | 622 | } |
620 | static void free_map_bootmem(struct page *memmap, unsigned long nr_pages) | 623 | static void free_map_bootmem(struct page *memmap, unsigned long nr_pages) |
621 | { | 624 | { |
622 | vmemmap_free(memmap, nr_pages); | 625 | unsigned long start = (unsigned long)memmap; |
626 | unsigned long end = (unsigned long)(memmap + nr_pages); | ||
627 | |||
628 | vmemmap_free(start, end); | ||
623 | } | 629 | } |
624 | #else | 630 | #else |
625 | static struct page *__kmalloc_section_memmap(unsigned long nr_pages) | 631 | static struct page *__kmalloc_section_memmap(unsigned long nr_pages) |