summaryrefslogtreecommitdiffstats
path: root/mm/sparse.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/sparse.c')
-rw-r--r--mm/sparse.c237
1 files changed, 1 insertions, 236 deletions
diff --git a/mm/sparse.c b/mm/sparse.c
index 248d5d7bbf55..10b07eea9a6e 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -205,12 +205,6 @@ static inline unsigned long first_present_section_nr(void)
205 return next_present_section_nr(-1); 205 return next_present_section_nr(-1);
206} 206}
207 207
208/*
209 * Record how many memory sections are marked as present
210 * during system bootup.
211 */
212static int __initdata nr_present_sections;
213
214/* Record a memory area against a node. */ 208/* Record a memory area against a node. */
215void __init memory_present(int nid, unsigned long start, unsigned long end) 209void __init memory_present(int nid, unsigned long start, unsigned long end)
216{ 210{
@@ -240,7 +234,6 @@ void __init memory_present(int nid, unsigned long start, unsigned long end)
240 ms->section_mem_map = sparse_encode_early_nid(nid) | 234 ms->section_mem_map = sparse_encode_early_nid(nid) |
241 SECTION_IS_ONLINE; 235 SECTION_IS_ONLINE;
242 section_mark_present(ms); 236 section_mark_present(ms);
243 nr_present_sections++;
244 } 237 }
245 } 238 }
246} 239}
@@ -377,37 +370,8 @@ static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
377} 370}
378#endif /* CONFIG_MEMORY_HOTREMOVE */ 371#endif /* CONFIG_MEMORY_HOTREMOVE */
379 372
380static void __init sparse_early_usemaps_alloc_node(void *data,
381 unsigned long pnum_begin,
382 unsigned long pnum_end,
383 unsigned long usemap_count, int nodeid)
384{
385 void *usemap;
386 unsigned long pnum;
387 unsigned long **usemap_map = (unsigned long **)data;
388 int size = usemap_size();
389 int nr_consumed_maps = 0;
390
391 usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid),
392 size * usemap_count);
393 if (!usemap) {
394 pr_warn("%s: allocation failed\n", __func__);
395 return;
396 }
397
398 for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
399 if (!present_section_nr(pnum))
400 continue;
401 usemap_map[nr_consumed_maps] = usemap;
402 usemap += size;
403 check_usemap_section_nr(nodeid, usemap_map[nr_consumed_maps]);
404 nr_consumed_maps++;
405 }
406}
407
408#ifdef CONFIG_SPARSEMEM_VMEMMAP 373#ifdef CONFIG_SPARSEMEM_VMEMMAP
409static unsigned long __init section_map_size(void) 374static unsigned long __init section_map_size(void)
410
411{ 375{
412 return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE); 376 return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE);
413} 377}
@@ -432,25 +396,6 @@ struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid,
432 BOOTMEM_ALLOC_ACCESSIBLE, nid); 396 BOOTMEM_ALLOC_ACCESSIBLE, nid);
433 return map; 397 return map;
434} 398}
435void __init sparse_mem_maps_populate_node(struct page **map_map,
436 unsigned long pnum_begin,
437 unsigned long pnum_end,
438 unsigned long map_count, int nodeid)
439{
440 unsigned long pnum;
441 int nr_consumed_maps = 0;
442
443 for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
444 if (!present_section_nr(pnum))
445 continue;
446 map_map[nr_consumed_maps] =
447 sparse_mem_map_populate(pnum, nodeid, NULL);
448 if (map_map[nr_consumed_maps++])
449 continue;
450 pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
451 __func__);
452 }
453}
454#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 399#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
455 400
456static void *sparsemap_buf __meminitdata; 401static void *sparsemap_buf __meminitdata;
@@ -489,190 +434,10 @@ void * __meminit sparse_buffer_alloc(unsigned long size)
489 return ptr; 434 return ptr;
490} 435}
491 436
492#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
493static void __init sparse_early_mem_maps_alloc_node(void *data,
494 unsigned long pnum_begin,
495 unsigned long pnum_end,
496 unsigned long map_count, int nodeid)
497{
498 struct page **map_map = (struct page **)data;
499
500 sparse_buffer_init(section_map_size() * map_count, nodeid);
501 sparse_mem_maps_populate_node(map_map, pnum_begin, pnum_end,
502 map_count, nodeid);
503 sparse_buffer_fini();
504}
505#else
506static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum)
507{
508 struct page *map;
509 struct mem_section *ms = __nr_to_section(pnum);
510 int nid = sparse_early_nid(ms);
511
512 map = sparse_mem_map_populate(pnum, nid, NULL);
513 if (map)
514 return map;
515
516 pr_err("%s: sparsemem memory map backing failed some memory will not be available\n",
517 __func__);
518 return NULL;
519}
520#endif
521
522void __weak __meminit vmemmap_populate_print_last(void) 437void __weak __meminit vmemmap_populate_print_last(void)
523{ 438{
524} 439}
525 440
526/**
527 * alloc_usemap_and_memmap - memory alloction for pageblock flags and vmemmap
528 * @map: usemap_map for pageblock flags or mmap_map for vmemmap
529 * @unit_size: size of map unit
530 */
531static void __init alloc_usemap_and_memmap(void (*alloc_func)
532 (void *, unsigned long, unsigned long,
533 unsigned long, int), void *data,
534 int data_unit_size)
535{
536 unsigned long pnum;
537 unsigned long map_count;
538 int nodeid_begin = 0;
539 unsigned long pnum_begin = 0;
540
541 for_each_present_section_nr(0, pnum) {
542 struct mem_section *ms;
543
544 ms = __nr_to_section(pnum);
545 nodeid_begin = sparse_early_nid(ms);
546 pnum_begin = pnum;
547 break;
548 }
549 map_count = 1;
550 for_each_present_section_nr(pnum_begin + 1, pnum) {
551 struct mem_section *ms;
552 int nodeid;
553
554 ms = __nr_to_section(pnum);
555 nodeid = sparse_early_nid(ms);
556 if (nodeid == nodeid_begin) {
557 map_count++;
558 continue;
559 }
560 /* ok, we need to take cake of from pnum_begin to pnum - 1*/
561 alloc_func(data, pnum_begin, pnum,
562 map_count, nodeid_begin);
563 /* new start, update count etc*/
564 nodeid_begin = nodeid;
565 pnum_begin = pnum;
566 data += map_count * data_unit_size;
567 map_count = 1;
568 }
569 /* ok, last chunk */
570 alloc_func(data, pnum_begin, __highest_present_section_nr+1,
571 map_count, nodeid_begin);
572}
573
574/*
575 * Allocate the accumulated non-linear sections, allocate a mem_map
576 * for each and record the physical to section mapping.
577 */
578void __init sparse_init(void)
579{
580 unsigned long pnum;
581 struct page *map;
582 unsigned long *usemap;
583 unsigned long **usemap_map;
584 int size;
585 int nr_consumed_maps = 0;
586#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
587 int size2;
588 struct page **map_map;
589#endif
590
591 /* see include/linux/mmzone.h 'struct mem_section' definition */
592 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section)));
593
594 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
595 set_pageblock_order();
596
597 /*
598 * map is using big page (aka 2M in x86 64 bit)
599 * usemap is less one page (aka 24 bytes)
600 * so alloc 2M (with 2M align) and 24 bytes in turn will
601 * make next 2M slip to one more 2M later.
602 * then in big system, the memory will have a lot of holes...
603 * here try to allocate 2M pages continuously.
604 *
605 * powerpc need to call sparse_init_one_section right after each
606 * sparse_early_mem_map_alloc, so allocate usemap_map at first.
607 */
608 size = sizeof(unsigned long *) * nr_present_sections;
609 usemap_map = memblock_virt_alloc(size, 0);
610 if (!usemap_map)
611 panic("can not allocate usemap_map\n");
612 alloc_usemap_and_memmap(sparse_early_usemaps_alloc_node,
613 (void *)usemap_map,
614 sizeof(usemap_map[0]));
615
616#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
617 size2 = sizeof(struct page *) * nr_present_sections;
618 map_map = memblock_virt_alloc(size2, 0);
619 if (!map_map)
620 panic("can not allocate map_map\n");
621 alloc_usemap_and_memmap(sparse_early_mem_maps_alloc_node,
622 (void *)map_map,
623 sizeof(map_map[0]));
624#endif
625
626 /*
627 * The number of present sections stored in nr_present_sections
628 * are kept the same since mem sections are marked as present in
629 * memory_present(). In this for loop, we need check which sections
630 * failed to allocate memmap or usemap, then clear its
631 * ->section_mem_map accordingly. During this process, we need
632 * increase 'nr_consumed_maps' whether its allocation of memmap
633 * or usemap failed or not, so that after we handle the i-th
634 * memory section, can get memmap and usemap of (i+1)-th section
635 * correctly.
636 */
637 for_each_present_section_nr(0, pnum) {
638 struct mem_section *ms;
639
640 if (nr_consumed_maps >= nr_present_sections) {
641 pr_err("nr_consumed_maps goes beyond nr_present_sections\n");
642 break;
643 }
644 ms = __nr_to_section(pnum);
645 usemap = usemap_map[nr_consumed_maps];
646 if (!usemap) {
647 ms->section_mem_map = 0;
648 nr_consumed_maps++;
649 continue;
650 }
651
652#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
653 map = map_map[nr_consumed_maps];
654#else
655 map = sparse_early_mem_map_alloc(pnum);
656#endif
657 if (!map) {
658 ms->section_mem_map = 0;
659 nr_consumed_maps++;
660 continue;
661 }
662
663 sparse_init_one_section(__nr_to_section(pnum), pnum, map,
664 usemap);
665 nr_consumed_maps++;
666 }
667
668 vmemmap_populate_print_last();
669
670#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
671 memblock_free_early(__pa(map_map), size2);
672#endif
673 memblock_free_early(__pa(usemap_map), size);
674}
675
676/* 441/*
677 * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end) 442 * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end)
678 * And number of present sections in this node is map_count. 443 * And number of present sections in this node is map_count.
@@ -726,7 +491,7 @@ failed:
726 * Allocate the accumulated non-linear sections, allocate a mem_map 491 * Allocate the accumulated non-linear sections, allocate a mem_map
727 * for each and record the physical to section mapping. 492 * for each and record the physical to section mapping.
728 */ 493 */
729void __init new_sparse_init(void) 494void __init sparse_init(void)
730{ 495{
731 unsigned long pnum_begin = first_present_section_nr(); 496 unsigned long pnum_begin = first_present_section_nr();
732 int nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); 497 int nid_begin = sparse_early_nid(__nr_to_section(pnum_begin));