diff options
author | Ingo Molnar <mingo@elte.hu> | 2009-03-05 15:49:35 -0500 |
---|---|---|
committer | Ingo Molnar <mingo@elte.hu> | 2009-03-05 15:49:35 -0500 |
commit | 28e93a005b65cc5b4f569642e9c7903618ea5fe1 (patch) | |
tree | 3c98b8dc457b05b290ad640c413e453a264739bf /arch/x86/mm/init_64.c | |
parent | caab36b593b44c97e3c7707c6a8054b320f8d622 (diff) | |
parent | ed26dbe5ae045e5bf95c6dc27497397a3fde52e1 (diff) |
Merge branch 'x86/mm' into x86/core
Diffstat (limited to 'arch/x86/mm/init_64.c')
-rw-r--r-- | arch/x86/mm/init_64.c | 272 |
1 files changed, 7 insertions, 265 deletions
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 07f44d491df1..8a853bc3b287 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c | |||
@@ -48,6 +48,7 @@ | |||
48 | #include <asm/kdebug.h> | 48 | #include <asm/kdebug.h> |
49 | #include <asm/numa.h> | 49 | #include <asm/numa.h> |
50 | #include <asm/cacheflush.h> | 50 | #include <asm/cacheflush.h> |
51 | #include <asm/init.h> | ||
51 | 52 | ||
52 | /* | 53 | /* |
53 | * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries. | 54 | * end_pfn only includes RAM, while max_pfn_mapped includes all e820 entries. |
@@ -61,12 +62,6 @@ static unsigned long dma_reserve __initdata; | |||
61 | 62 | ||
62 | DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); | 63 | DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); |
63 | 64 | ||
64 | int direct_gbpages | ||
65 | #ifdef CONFIG_DIRECT_GBPAGES | ||
66 | = 1 | ||
67 | #endif | ||
68 | ; | ||
69 | |||
70 | static int __init parse_direct_gbpages_off(char *arg) | 65 | static int __init parse_direct_gbpages_off(char *arg) |
71 | { | 66 | { |
72 | direct_gbpages = 0; | 67 | direct_gbpages = 0; |
@@ -87,8 +82,6 @@ early_param("gbpages", parse_direct_gbpages_on); | |||
87 | * around without checking the pgd every time. | 82 | * around without checking the pgd every time. |
88 | */ | 83 | */ |
89 | 84 | ||
90 | int after_bootmem; | ||
91 | |||
92 | pteval_t __supported_pte_mask __read_mostly = ~_PAGE_IOMAP; | 85 | pteval_t __supported_pte_mask __read_mostly = ~_PAGE_IOMAP; |
93 | EXPORT_SYMBOL_GPL(__supported_pte_mask); | 86 | EXPORT_SYMBOL_GPL(__supported_pte_mask); |
94 | 87 | ||
@@ -325,13 +318,9 @@ void __init cleanup_highmap(void) | |||
325 | } | 318 | } |
326 | } | 319 | } |
327 | 320 | ||
328 | static unsigned long __initdata table_start; | ||
329 | static unsigned long __meminitdata table_end; | ||
330 | static unsigned long __meminitdata table_top; | ||
331 | |||
332 | static __ref void *alloc_low_page(unsigned long *phys) | 321 | static __ref void *alloc_low_page(unsigned long *phys) |
333 | { | 322 | { |
334 | unsigned long pfn = table_end++; | 323 | unsigned long pfn = e820_table_end++; |
335 | void *adr; | 324 | void *adr; |
336 | 325 | ||
337 | if (after_bootmem) { | 326 | if (after_bootmem) { |
@@ -341,7 +330,7 @@ static __ref void *alloc_low_page(unsigned long *phys) | |||
341 | return adr; | 330 | return adr; |
342 | } | 331 | } |
343 | 332 | ||
344 | if (pfn >= table_top) | 333 | if (pfn >= e820_table_top) |
345 | panic("alloc_low_page: ran out of memory"); | 334 | panic("alloc_low_page: ran out of memory"); |
346 | 335 | ||
347 | adr = early_memremap(pfn * PAGE_SIZE, PAGE_SIZE); | 336 | adr = early_memremap(pfn * PAGE_SIZE, PAGE_SIZE); |
@@ -581,58 +570,10 @@ phys_pud_update(pgd_t *pgd, unsigned long addr, unsigned long end, | |||
581 | return phys_pud_init(pud, addr, end, page_size_mask); | 570 | return phys_pud_init(pud, addr, end, page_size_mask); |
582 | } | 571 | } |
583 | 572 | ||
584 | static void __init find_early_table_space(unsigned long end, int use_pse, | 573 | unsigned long __init |
585 | int use_gbpages) | 574 | kernel_physical_mapping_init(unsigned long start, |
586 | { | 575 | unsigned long end, |
587 | unsigned long puds, pmds, ptes, tables, start; | 576 | unsigned long page_size_mask) |
588 | |||
589 | puds = (end + PUD_SIZE - 1) >> PUD_SHIFT; | ||
590 | tables = roundup(puds * sizeof(pud_t), PAGE_SIZE); | ||
591 | if (use_gbpages) { | ||
592 | unsigned long extra; | ||
593 | extra = end - ((end>>PUD_SHIFT) << PUD_SHIFT); | ||
594 | pmds = (extra + PMD_SIZE - 1) >> PMD_SHIFT; | ||
595 | } else | ||
596 | pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT; | ||
597 | tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE); | ||
598 | |||
599 | if (use_pse) { | ||
600 | unsigned long extra; | ||
601 | extra = end - ((end>>PMD_SHIFT) << PMD_SHIFT); | ||
602 | ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT; | ||
603 | } else | ||
604 | ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT; | ||
605 | tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE); | ||
606 | |||
607 | /* | ||
608 | * RED-PEN putting page tables only on node 0 could | ||
609 | * cause a hotspot and fill up ZONE_DMA. The page tables | ||
610 | * need roughly 0.5KB per GB. | ||
611 | */ | ||
612 | start = 0x8000; | ||
613 | table_start = find_e820_area(start, end, tables, PAGE_SIZE); | ||
614 | if (table_start == -1UL) | ||
615 | panic("Cannot find space for the kernel page tables"); | ||
616 | |||
617 | table_start >>= PAGE_SHIFT; | ||
618 | table_end = table_start; | ||
619 | table_top = table_start + (tables >> PAGE_SHIFT); | ||
620 | |||
621 | printk(KERN_DEBUG "kernel direct mapping tables up to %lx @ %lx-%lx\n", | ||
622 | end, table_start << PAGE_SHIFT, table_top << PAGE_SHIFT); | ||
623 | } | ||
624 | |||
625 | static void __init init_gbpages(void) | ||
626 | { | ||
627 | if (direct_gbpages && cpu_has_gbpages) | ||
628 | printk(KERN_INFO "Using GB pages for direct mapping\n"); | ||
629 | else | ||
630 | direct_gbpages = 0; | ||
631 | } | ||
632 | |||
633 | static unsigned long __meminit kernel_physical_mapping_init(unsigned long start, | ||
634 | unsigned long end, | ||
635 | unsigned long page_size_mask) | ||
636 | { | 577 | { |
637 | 578 | ||
638 | unsigned long next, last_map_addr = end; | 579 | unsigned long next, last_map_addr = end; |
@@ -669,176 +610,6 @@ static unsigned long __meminit kernel_physical_mapping_init(unsigned long start, | |||
669 | return last_map_addr; | 610 | return last_map_addr; |
670 | } | 611 | } |
671 | 612 | ||
672 | struct map_range { | ||
673 | unsigned long start; | ||
674 | unsigned long end; | ||
675 | unsigned page_size_mask; | ||
676 | }; | ||
677 | |||
678 | #define NR_RANGE_MR 5 | ||
679 | |||
680 | static int save_mr(struct map_range *mr, int nr_range, | ||
681 | unsigned long start_pfn, unsigned long end_pfn, | ||
682 | unsigned long page_size_mask) | ||
683 | { | ||
684 | |||
685 | if (start_pfn < end_pfn) { | ||
686 | if (nr_range >= NR_RANGE_MR) | ||
687 | panic("run out of range for init_memory_mapping\n"); | ||
688 | mr[nr_range].start = start_pfn<<PAGE_SHIFT; | ||
689 | mr[nr_range].end = end_pfn<<PAGE_SHIFT; | ||
690 | mr[nr_range].page_size_mask = page_size_mask; | ||
691 | nr_range++; | ||
692 | } | ||
693 | |||
694 | return nr_range; | ||
695 | } | ||
696 | |||
697 | /* | ||
698 | * Setup the direct mapping of the physical memory at PAGE_OFFSET. | ||
699 | * This runs before bootmem is initialized and gets pages directly from | ||
700 | * the physical memory. To access them they are temporarily mapped. | ||
701 | */ | ||
702 | unsigned long __init_refok init_memory_mapping(unsigned long start, | ||
703 | unsigned long end) | ||
704 | { | ||
705 | unsigned long last_map_addr = 0; | ||
706 | unsigned long page_size_mask = 0; | ||
707 | unsigned long start_pfn, end_pfn; | ||
708 | unsigned long pos; | ||
709 | |||
710 | struct map_range mr[NR_RANGE_MR]; | ||
711 | int nr_range, i; | ||
712 | int use_pse, use_gbpages; | ||
713 | |||
714 | printk(KERN_INFO "init_memory_mapping: %016lx-%016lx\n", start, end); | ||
715 | |||
716 | /* | ||
717 | * Find space for the kernel direct mapping tables. | ||
718 | * | ||
719 | * Later we should allocate these tables in the local node of the | ||
720 | * memory mapped. Unfortunately this is done currently before the | ||
721 | * nodes are discovered. | ||
722 | */ | ||
723 | if (!after_bootmem) | ||
724 | init_gbpages(); | ||
725 | |||
726 | #ifdef CONFIG_DEBUG_PAGEALLOC | ||
727 | /* | ||
728 | * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages. | ||
729 | * This will simplify cpa(), which otherwise needs to support splitting | ||
730 | * large pages into small in interrupt context, etc. | ||
731 | */ | ||
732 | use_pse = use_gbpages = 0; | ||
733 | #else | ||
734 | use_pse = cpu_has_pse; | ||
735 | use_gbpages = direct_gbpages; | ||
736 | #endif | ||
737 | |||
738 | if (use_gbpages) | ||
739 | page_size_mask |= 1 << PG_LEVEL_1G; | ||
740 | if (use_pse) | ||
741 | page_size_mask |= 1 << PG_LEVEL_2M; | ||
742 | |||
743 | memset(mr, 0, sizeof(mr)); | ||
744 | nr_range = 0; | ||
745 | |||
746 | /* head if not big page alignment ?*/ | ||
747 | start_pfn = start >> PAGE_SHIFT; | ||
748 | pos = start_pfn << PAGE_SHIFT; | ||
749 | end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT) | ||
750 | << (PMD_SHIFT - PAGE_SHIFT); | ||
751 | if (end_pfn > (end >> PAGE_SHIFT)) | ||
752 | end_pfn = end >> PAGE_SHIFT; | ||
753 | if (start_pfn < end_pfn) { | ||
754 | nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); | ||
755 | pos = end_pfn << PAGE_SHIFT; | ||
756 | } | ||
757 | |||
758 | /* big page (2M) range*/ | ||
759 | start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) | ||
760 | << (PMD_SHIFT - PAGE_SHIFT); | ||
761 | end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) | ||
762 | << (PUD_SHIFT - PAGE_SHIFT); | ||
763 | if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT))) | ||
764 | end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)); | ||
765 | if (start_pfn < end_pfn) { | ||
766 | nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, | ||
767 | page_size_mask & (1<<PG_LEVEL_2M)); | ||
768 | pos = end_pfn << PAGE_SHIFT; | ||
769 | } | ||
770 | |||
771 | /* big page (1G) range */ | ||
772 | start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT) | ||
773 | << (PUD_SHIFT - PAGE_SHIFT); | ||
774 | end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT); | ||
775 | if (start_pfn < end_pfn) { | ||
776 | nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, | ||
777 | page_size_mask & | ||
778 | ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G))); | ||
779 | pos = end_pfn << PAGE_SHIFT; | ||
780 | } | ||
781 | |||
782 | /* tail is not big page (1G) alignment */ | ||
783 | start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT) | ||
784 | << (PMD_SHIFT - PAGE_SHIFT); | ||
785 | end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT); | ||
786 | if (start_pfn < end_pfn) { | ||
787 | nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, | ||
788 | page_size_mask & (1<<PG_LEVEL_2M)); | ||
789 | pos = end_pfn << PAGE_SHIFT; | ||
790 | } | ||
791 | |||
792 | /* tail is not big page (2M) alignment */ | ||
793 | start_pfn = pos>>PAGE_SHIFT; | ||
794 | end_pfn = end>>PAGE_SHIFT; | ||
795 | nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); | ||
796 | |||
797 | /* try to merge same page size and continuous */ | ||
798 | for (i = 0; nr_range > 1 && i < nr_range - 1; i++) { | ||
799 | unsigned long old_start; | ||
800 | if (mr[i].end != mr[i+1].start || | ||
801 | mr[i].page_size_mask != mr[i+1].page_size_mask) | ||
802 | continue; | ||
803 | /* move it */ | ||
804 | old_start = mr[i].start; | ||
805 | memmove(&mr[i], &mr[i+1], | ||
806 | (nr_range - 1 - i) * sizeof (struct map_range)); | ||
807 | mr[i--].start = old_start; | ||
808 | nr_range--; | ||
809 | } | ||
810 | |||
811 | for (i = 0; i < nr_range; i++) | ||
812 | printk(KERN_DEBUG " %010lx - %010lx page %s\n", | ||
813 | mr[i].start, mr[i].end, | ||
814 | (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":( | ||
815 | (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k")); | ||
816 | |||
817 | if (!after_bootmem) | ||
818 | find_early_table_space(end, use_pse, use_gbpages); | ||
819 | |||
820 | for (i = 0; i < nr_range; i++) | ||
821 | last_map_addr = kernel_physical_mapping_init( | ||
822 | mr[i].start, mr[i].end, | ||
823 | mr[i].page_size_mask); | ||
824 | |||
825 | if (!after_bootmem) | ||
826 | mmu_cr4_features = read_cr4(); | ||
827 | __flush_tlb_all(); | ||
828 | |||
829 | if (!after_bootmem && table_end > table_start) | ||
830 | reserve_early(table_start << PAGE_SHIFT, | ||
831 | table_end << PAGE_SHIFT, "PGTABLE"); | ||
832 | |||
833 | printk(KERN_INFO "last_map_addr: %lx end: %lx\n", | ||
834 | last_map_addr, end); | ||
835 | |||
836 | if (!after_bootmem) | ||
837 | early_memtest(start, end); | ||
838 | |||
839 | return last_map_addr >> PAGE_SHIFT; | ||
840 | } | ||
841 | |||
842 | #ifndef CONFIG_NUMA | 613 | #ifndef CONFIG_NUMA |
843 | void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn) | 614 | void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn) |
844 | { | 615 | { |
@@ -910,28 +681,6 @@ EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid); | |||
910 | 681 | ||
911 | #endif /* CONFIG_MEMORY_HOTPLUG */ | 682 | #endif /* CONFIG_MEMORY_HOTPLUG */ |
912 | 683 | ||
913 | /* | ||
914 | * devmem_is_allowed() checks to see if /dev/mem access to a certain address | ||
915 | * is valid. The argument is a physical page number. | ||
916 | * | ||
917 | * | ||
918 | * On x86, access has to be given to the first megabyte of ram because that area | ||
919 | * contains bios code and data regions used by X and dosemu and similar apps. | ||
920 | * Access has to be given to non-kernel-ram areas as well, these contain the PCI | ||
921 | * mmio resources as well as potential bios/acpi data regions. | ||
922 | */ | ||
923 | int devmem_is_allowed(unsigned long pagenr) | ||
924 | { | ||
925 | if (pagenr <= 256) | ||
926 | return 1; | ||
927 | if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) | ||
928 | return 0; | ||
929 | if (!page_is_ram(pagenr)) | ||
930 | return 1; | ||
931 | return 0; | ||
932 | } | ||
933 | |||
934 | |||
935 | static struct kcore_list kcore_mem, kcore_vmalloc, kcore_kernel, | 684 | static struct kcore_list kcore_mem, kcore_vmalloc, kcore_kernel, |
936 | kcore_modules, kcore_vsyscall; | 685 | kcore_modules, kcore_vsyscall; |
937 | 686 | ||
@@ -1019,13 +768,6 @@ void mark_rodata_ro(void) | |||
1019 | 768 | ||
1020 | #endif | 769 | #endif |
1021 | 770 | ||
1022 | #ifdef CONFIG_BLK_DEV_INITRD | ||
1023 | void free_initrd_mem(unsigned long start, unsigned long end) | ||
1024 | { | ||
1025 | free_init_pages("initrd memory", start, end); | ||
1026 | } | ||
1027 | #endif | ||
1028 | |||
1029 | int __init reserve_bootmem_generic(unsigned long phys, unsigned long len, | 771 | int __init reserve_bootmem_generic(unsigned long phys, unsigned long len, |
1030 | int flags) | 772 | int flags) |
1031 | { | 773 | { |