Merge branch 'x86/bootmem' into x86/mm

Merge reason: the topic is ready - consolidate it into the more generic x86/mm tree
              and prevent conflicts.

Signed-off-by: Ingo Molnar <mingo@elte.hu>

5 files changed, 97 insertions, 74 deletions

diff --git a/arch/x86/mm/amdtopology_64.c b/arch/x86/mm/amdtopology_64.c
index f21962c435ed..49b334cdd64c 100644
--- a/arch/x86/mm/amdtopology_64.c
+++ b/arch/x86/mm/amdtopology_64.c
@@ -278,12 +278,14 @@ int __init amd_scan_nodes(void)
                 apicid_base = boot_cpu_physical_apicid;
         }
 
-        for_each_node_mask(i, node_possible_map) {
-                int j;
-
+        for_each_node_mask(i, node_possible_map)
                 memblock_x86_register_active_regions(i,
                                 nodes[i].start >> PAGE_SHIFT,
                                 nodes[i].end >> PAGE_SHIFT);
+        init_memory_mapping_high();
+        for_each_node_mask(i, node_possible_map) {
+                int j;
+
                 for (j = apicid_base; j < cores + apicid_base; j++)
                         apicid_to_node[(i << bits) + j] = i;
                 setup_node_bootmem(i, nodes[i].start, nodes[i].end);
diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index 947f42abe820..b8054e087ead 100644
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -33,7 +33,7 @@ int direct_gbpages
 static void __init find_early_table_space(unsigned long end, int use_pse,
                                           int use_gbpages)
 {
-        unsigned long puds, pmds, ptes, tables, start;
+        unsigned long puds, pmds, ptes, tables, start = 0, good_end = end;
         phys_addr_t base;
 
         puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
@@ -65,20 +65,11 @@ static void __init find_early_table_space(unsigned long end, int use_pse,
 #ifdef CONFIG_X86_32
         /* for fixmap */
         tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE);
-#endif
 
-        /*
-         * RED-PEN putting page tables only on node 0 could
-         * cause a hotspot and fill up ZONE_DMA. The page tables
-         * need roughly 0.5KB per GB.
-         */
-#ifdef CONFIG_X86_32
-        start = 0x7000;
-#else
-        start = 0x8000;
+        good_end = max_pfn_mapped << PAGE_SHIFT;
 #endif
-        base = memblock_find_in_range(start, max_pfn_mapped<<PAGE_SHIFT,
-                                        tables, PAGE_SIZE);
+
+        base = memblock_find_in_range(start, good_end, tables, PAGE_SIZE);
         if (base == MEMBLOCK_ERROR)
                 panic("Cannot find space for the kernel page tables");
 
@@ -279,25 +270,6 @@ unsigned long __init_refok init_memory_mapping(unsigned long start,
         load_cr3(swapper_pg_dir);
 #endif
 
-#ifdef CONFIG_X86_64
-        if (!after_bootmem && !start) {
-                pud_t *pud;
-                pmd_t *pmd;
-
-                mmu_cr4_features = read_cr4();
-
-                /*
-                 * _brk_end cannot change anymore, but it and _end may be
-                 * located on different 2M pages. cleanup_highmap(), however,
-                 * can only consider _end when it runs, so destroy any
-                 * mappings beyond _brk_end here.
-                 */
-                pud = pud_offset(pgd_offset_k(_brk_end), _brk_end);
-                pmd = pmd_offset(pud, _brk_end - 1);
-                while (++pmd <= pmd_offset(pud, (unsigned long)_end - 1))
-                        pmd_clear(pmd);
-        }
-#endif
         __flush_tlb_all();
 
         if (!after_bootmem && e820_table_end > e820_table_start)
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 71a59296af80..194f2732ab77 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -333,12 +333,28 @@ static __ref void *alloc_low_page(unsigned long *phys)
         return adr;
 }
 
+static __ref void *map_low_page(void *virt)
+{
+        void *adr;
+        unsigned long phys, left;
+
+        if (after_bootmem)
+                return virt;
+
+        phys = __pa(virt);
+        left = phys & (PAGE_SIZE - 1);
+        adr = early_memremap(phys & PAGE_MASK, PAGE_SIZE);
+        adr = (void *)(((unsigned long)adr) | left);
+
+        return adr;
+}
+
 static __ref void unmap_low_page(void *adr)
 {
         if (after_bootmem)
                 return;
 
-        early_iounmap(adr, PAGE_SIZE);
+        early_iounmap((void *)((unsigned long)adr & PAGE_MASK), PAGE_SIZE);
 }
 
 static unsigned long __meminit
@@ -386,15 +402,6 @@ phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end,
 }
 
 static unsigned long __meminit
-phys_pte_update(pmd_t *pmd, unsigned long address, unsigned long end,
-                pgprot_t prot)
-{
-        pte_t *pte = (pte_t *)pmd_page_vaddr(*pmd);
-
-        return phys_pte_init(pte, address, end, prot);
-}
-
-static unsigned long __meminit
 phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
               unsigned long page_size_mask, pgprot_t prot)
 {
@@ -420,8 +427,10 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
                 if (pmd_val(*pmd)) {
                         if (!pmd_large(*pmd)) {
                                 spin_lock(&init_mm.page_table_lock);
-                                last_map_addr = phys_pte_update(pmd, address,
+                                pte = map_low_page((pte_t *)pmd_page_vaddr(*pmd));
+                                last_map_addr = phys_pte_init(pte, address,
                                                                 end, prot);
+                                unmap_low_page(pte);
                                 spin_unlock(&init_mm.page_table_lock);
                                 continue;
                         }
@@ -468,18 +477,6 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
 }
 
 static unsigned long __meminit
-phys_pmd_update(pud_t *pud, unsigned long address, unsigned long end,
-                unsigned long page_size_mask, pgprot_t prot)
-{
-        pmd_t *pmd = pmd_offset(pud, 0);
-        unsigned long last_map_addr;
-
-        last_map_addr = phys_pmd_init(pmd, address, end, page_size_mask, prot);
-        __flush_tlb_all();
-        return last_map_addr;
-}
-
-static unsigned long __meminit
 phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
                          unsigned long page_size_mask)
 {
@@ -504,8 +501,11 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
 
                 if (pud_val(*pud)) {
                         if (!pud_large(*pud)) {
-                                last_map_addr = phys_pmd_update(pud, addr, end,
+                                pmd = map_low_page(pmd_offset(pud, 0));
+                                last_map_addr = phys_pmd_init(pmd, addr, end,
                                                          page_size_mask, prot);
+                                unmap_low_page(pmd);
+                                __flush_tlb_all();
                                 continue;
                         }
                         /*
@@ -553,17 +553,6 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
         return last_map_addr;
 }
 
-static unsigned long __meminit
-phys_pud_update(pgd_t *pgd, unsigned long addr, unsigned long end,
-                 unsigned long page_size_mask)
-{
-        pud_t *pud;
-
-        pud = (pud_t *)pgd_page_vaddr(*pgd);
-
-        return phys_pud_init(pud, addr, end, page_size_mask);
-}
-
 unsigned long __meminit
 kernel_physical_mapping_init(unsigned long start,
                              unsigned long end,
@@ -587,8 +576,10 @@ kernel_physical_mapping_init(unsigned long start,
                         next = end;
 
                 if (pgd_val(*pgd)) {
-                        last_map_addr = phys_pud_update(pgd, __pa(start),
+                        pud = map_low_page((pud_t *)pgd_page_vaddr(*pgd));
+                        last_map_addr = phys_pud_init(pud, __pa(start),
                                                  __pa(end), page_size_mask);
+                        unmap_low_page(pud);
                         continue;
                 }
 
@@ -616,9 +607,63 @@ void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn,
                                 int acpi, int k8)
 {
         memblock_x86_register_active_regions(0, start_pfn, end_pfn);
+        init_memory_mapping_high();
 }
 #endif
 
+struct mapping_work_data {
+        unsigned long start;
+        unsigned long end;
+        unsigned long pfn_mapped;
+};
+
+static int __init_refok
+mapping_work_fn(unsigned long start_pfn, unsigned long end_pfn, void *datax)
+{
+        struct mapping_work_data *data = datax;
+        unsigned long pfn_mapped;
+        unsigned long final_start, final_end;
+
+        final_start = max_t(unsigned long, start_pfn<<PAGE_SHIFT, data->start);
+        final_end = min_t(unsigned long, end_pfn<<PAGE_SHIFT, data->end);
+
+        if (final_end <= final_start)
+                return 0;
+
+        pfn_mapped = init_memory_mapping(final_start, final_end);
+
+        if (pfn_mapped > data->pfn_mapped)
+                data->pfn_mapped = pfn_mapped;
+
+        return 0;
+}
+
+static unsigned long __init_refok
+init_memory_mapping_active_regions(unsigned long start, unsigned long end)
+{
+        struct mapping_work_data data;
+
+        data.start = start;
+        data.end = end;
+        data.pfn_mapped = 0;
+
+        work_with_active_regions(MAX_NUMNODES, mapping_work_fn, &data);
+
+        return data.pfn_mapped;
+}
+
+void __init_refok init_memory_mapping_high(void)
+{
+        if (max_pfn > max_low_pfn) {
+                max_pfn_mapped = init_memory_mapping_active_regions(1UL<<32,
+                                                         max_pfn<<PAGE_SHIFT);
+                /* can we preserve max_low_pfn ? */
+                max_low_pfn = max_pfn;
+
+                memblock.current_limit = get_max_mapped();
+        }
+}
+
 void __init paging_init(void)
 {
         unsigned long max_zone_pfns[MAX_NR_ZONES];
diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c
index 95ea1551eebc..62cb634b5cf8 100644
--- a/arch/x86/mm/numa_64.c
+++ b/arch/x86/mm/numa_64.c
@@ -86,7 +86,7 @@ static int __init allocate_cachealigned_memnodemap(void)
 
         addr = 0x8000;
         nodemap_size = roundup(sizeof(s16) * memnodemapsize, L1_CACHE_BYTES);
-        nodemap_addr = memblock_find_in_range(addr, max_pfn<<PAGE_SHIFT,
+        nodemap_addr = memblock_find_in_range(addr, get_max_mapped(),
                                       nodemap_size, L1_CACHE_BYTES);
         if (nodemap_addr == MEMBLOCK_ERROR) {
                 printk(KERN_ERR
@@ -598,11 +598,12 @@ static int __init numa_emulation(unsigned long start_pfn,
          * the e820 memory map.
          */
         remove_all_active_ranges();
-        for_each_node_mask(i, node_possible_map) {
+        for_each_node_mask(i, node_possible_map)
                 memblock_x86_register_active_regions(i, nodes[i].start >> PAGE_SHIFT,
                                                 nodes[i].end >> PAGE_SHIFT);
+        init_memory_mapping_high();
+        for_each_node_mask(i, node_possible_map)
                 setup_node_bootmem(i, nodes[i].start, nodes[i].end);
-        }
         setup_physnodes(addr, max_addr, acpi, amd);
         fake_physnodes(acpi, amd, num_nodes);
         numa_init_array();
@@ -658,6 +659,7 @@ void __init initmem_init(unsigned long start_pfn, unsigned long last_pfn,
         for (i = 0; i < nr_cpu_ids; i++)
                 numa_set_node(i, 0);
         memblock_x86_register_active_regions(0, start_pfn, last_pfn);
+        init_memory_mapping_high();
         setup_node_bootmem(0, start_pfn << PAGE_SHIFT, last_pfn << PAGE_SHIFT);
 }
 
diff --git a/arch/x86/mm/srat_64.c b/arch/x86/mm/srat_64.c
index 603d285d1daa..4c03e13da138 100644
--- a/arch/x86/mm/srat_64.c
+++ b/arch/x86/mm/srat_64.c
@@ -444,6 +444,8 @@ int __init acpi_scan_nodes(unsigned long start, unsigned long end)
                 return -1;
         }
 
+        init_memory_mapping_high();
+
         /* Account for nodes with cpus and no memory */
         nodes_or(node_possible_map, nodes_parsed, cpu_nodes_parsed);