6 files changed, 524 insertions, 326 deletions

diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index d7aea41563b3..d41815265a0b 100644
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -17,86 +17,132 @@
 #include <asm/proto.h>
 #include <asm/dma.h>            /* for MAX_DMA_PFN */
 
-unsigned long __initdata pgt_buf_start;
-unsigned long __meminitdata pgt_buf_end;
-unsigned long __meminitdata pgt_buf_top;
+#include "mm_internal.h"
 
-int after_bootmem;
+static unsigned long __initdata pgt_buf_start;
+static unsigned long __initdata pgt_buf_end;
+static unsigned long __initdata pgt_buf_top;
 
-int direct_gbpages
-#ifdef CONFIG_DIRECT_GBPAGES
-                                = 1
-#endif
-;
+static unsigned long min_pfn_mapped;
 
-struct map_range {
-        unsigned long start;
-        unsigned long end;
-        unsigned page_size_mask;
-};
+static bool __initdata can_use_brk_pgt = true;
 
 /*
- * First calculate space needed for kernel direct mapping page tables to cover
- * mr[0].start to mr[nr_range - 1].end, while accounting for possible 2M and 1GB
- * pages. Then find enough contiguous space for those page tables.
+ * Pages returned are already directly mapped.
+ *
+ * Changing that is likely to break Xen, see commit:
+ *
+ *    279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve
+ *
+ * for detailed information.
  */
-static void __init find_early_table_space(struct map_range *mr, int nr_range)
+__ref void *alloc_low_pages(unsigned int num)
 {
+        unsigned long pfn;
         int i;
-        unsigned long puds = 0, pmds = 0, ptes = 0, tables;
-        unsigned long start = 0, good_end;
-        phys_addr_t base;
 
-        for (i = 0; i < nr_range; i++) {
-                unsigned long range, extra;
+        if (after_bootmem) {
+                unsigned int order;
 
-                range = mr[i].end - mr[i].start;
-                puds += (range + PUD_SIZE - 1) >> PUD_SHIFT;
+                order = get_order((unsigned long)num << PAGE_SHIFT);
+                return (void *)__get_free_pages(GFP_ATOMIC | __GFP_NOTRACK |
+                                                __GFP_ZERO, order);
+        }
 
-                if (mr[i].page_size_mask & (1 << PG_LEVEL_1G)) {
-                        extra = range - ((range >> PUD_SHIFT) << PUD_SHIFT);
-                        pmds += (extra + PMD_SIZE - 1) >> PMD_SHIFT;
-                } else {
-                        pmds += (range + PMD_SIZE - 1) >> PMD_SHIFT;
-                }
+        if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) {
+                unsigned long ret;
+                if (min_pfn_mapped >= max_pfn_mapped)
+                        panic("alloc_low_page: ran out of memory");
+                ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT,
+                                        max_pfn_mapped << PAGE_SHIFT,
+                                        PAGE_SIZE * num , PAGE_SIZE);
+                if (!ret)
+                        panic("alloc_low_page: can not alloc memory");
+                memblock_reserve(ret, PAGE_SIZE * num);
+                pfn = ret >> PAGE_SHIFT;
+        } else {
+                pfn = pgt_buf_end;
+                pgt_buf_end += num;
+                printk(KERN_DEBUG "BRK [%#010lx, %#010lx] PGTABLE\n",
+                        pfn << PAGE_SHIFT, (pgt_buf_end << PAGE_SHIFT) - 1);
+        }
 
-                if (mr[i].page_size_mask & (1 << PG_LEVEL_2M)) {
-                        extra = range - ((range >> PMD_SHIFT) << PMD_SHIFT);
-#ifdef CONFIG_X86_32
-                        extra += PMD_SIZE;
-#endif
-                        ptes += (extra + PAGE_SIZE - 1) >> PAGE_SHIFT;
-                } else {
-                        ptes += (range + PAGE_SIZE - 1) >> PAGE_SHIFT;
-                }
+        for (i = 0; i < num; i++) {
+                void *adr;
+
+                adr = __va((pfn + i) << PAGE_SHIFT);
+                clear_page(adr);
         }
 
-        tables = roundup(puds * sizeof(pud_t), PAGE_SIZE);
-        tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE);
-        tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE);
+        return __va(pfn << PAGE_SHIFT);
+}
 
-#ifdef CONFIG_X86_32
-        /* for fixmap */
-        tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE);
-#endif
-        good_end = max_pfn_mapped << PAGE_SHIFT;
+/* need 4 4k for initial PMD_SIZE, 4k for 0-ISA_END_ADDRESS */
+#define INIT_PGT_BUF_SIZE       (5 * PAGE_SIZE)
+RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE);
+void  __init early_alloc_pgt_buf(void)
+{
+        unsigned long tables = INIT_PGT_BUF_SIZE;
+        phys_addr_t base;
 
-        base = memblock_find_in_range(start, good_end, tables, PAGE_SIZE);
-        if (!base)
-                panic("Cannot find space for the kernel page tables");
+        base = __pa(extend_brk(tables, PAGE_SIZE));
 
         pgt_buf_start = base >> PAGE_SHIFT;
         pgt_buf_end = pgt_buf_start;
         pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT);
+}
+
+int after_bootmem;
+
+int direct_gbpages
+#ifdef CONFIG_DIRECT_GBPAGES
+                                = 1
+#endif
+;
 
-        printk(KERN_DEBUG "kernel direct mapping tables up to %#lx @ [mem %#010lx-%#010lx]\n",
-                mr[nr_range - 1].end - 1, pgt_buf_start << PAGE_SHIFT,
-                (pgt_buf_top << PAGE_SHIFT) - 1);
+static void __init init_gbpages(void)
+{
+#ifdef CONFIG_X86_64
+        if (direct_gbpages && cpu_has_gbpages)
+                printk(KERN_INFO "Using GB pages for direct mapping\n");
+        else
+                direct_gbpages = 0;
+#endif
 }
 
-void __init native_pagetable_reserve(u64 start, u64 end)
+struct map_range {
+        unsigned long start;
+        unsigned long end;
+        unsigned page_size_mask;
+};
+
+static int page_size_mask;
+
+static void __init probe_page_size_mask(void)
 {
-        memblock_reserve(start, end - start);
+        init_gbpages();
+
+#if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK)
+        /*
+         * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
+         * This will simplify cpa(), which otherwise needs to support splitting
+         * large pages into small in interrupt context, etc.
+         */
+        if (direct_gbpages)
+                page_size_mask |= 1 << PG_LEVEL_1G;
+        if (cpu_has_pse)
+                page_size_mask |= 1 << PG_LEVEL_2M;
+#endif
+
+        /* Enable PSE if available */
+        if (cpu_has_pse)
+                set_in_cr4(X86_CR4_PSE);
+
+        /* Enable PGE if available */
+        if (cpu_has_pge) {
+                set_in_cr4(X86_CR4_PGE);
+                __supported_pte_mask |= _PAGE_GLOBAL;
+        }
 }
 
 #ifdef CONFIG_X86_32
@@ -122,58 +168,51 @@ static int __meminit save_mr(struct map_range *mr, int nr_range,
 }
 
 /*
- * Setup the direct mapping of the physical memory at PAGE_OFFSET.
- * This runs before bootmem is initialized and gets pages directly from
- * the physical memory. To access them they are temporarily mapped.
+ * adjust the page_size_mask for small range to go with
+ *      big page size instead small one if nearby are ram too.
  */
-unsigned long __init_refok init_memory_mapping(unsigned long start,
-                                               unsigned long end)
+static void __init_refok adjust_range_page_size_mask(struct map_range *mr,
+                                                         int nr_range)
 {
-        unsigned long page_size_mask = 0;
-        unsigned long start_pfn, end_pfn;
-        unsigned long ret = 0;
-        unsigned long pos;
-
-        struct map_range mr[NR_RANGE_MR];
-        int nr_range, i;
-        int use_pse, use_gbpages;
+        int i;
 
-        printk(KERN_INFO "init_memory_mapping: [mem %#010lx-%#010lx]\n",
-               start, end - 1);
+        for (i = 0; i < nr_range; i++) {
+                if ((page_size_mask & (1<<PG_LEVEL_2M)) &&
+                    !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) {
+                        unsigned long start = round_down(mr[i].start, PMD_SIZE);
+                        unsigned long end = round_up(mr[i].end, PMD_SIZE);
 
-#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK)
-        /*
-         * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
-         * This will simplify cpa(), which otherwise needs to support splitting
-         * large pages into small in interrupt context, etc.
-         */
-        use_pse = use_gbpages = 0;
-#else
-        use_pse = cpu_has_pse;
-        use_gbpages = direct_gbpages;
+#ifdef CONFIG_X86_32
+                        if ((end >> PAGE_SHIFT) > max_low_pfn)
+                                continue;
 #endif
 
-        /* Enable PSE if available */
-        if (cpu_has_pse)
-                set_in_cr4(X86_CR4_PSE);
+                        if (memblock_is_region_memory(start, end - start))
+                                mr[i].page_size_mask |= 1<<PG_LEVEL_2M;
+                }
+                if ((page_size_mask & (1<<PG_LEVEL_1G)) &&
+                    !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) {
+                        unsigned long start = round_down(mr[i].start, PUD_SIZE);
+                        unsigned long end = round_up(mr[i].end, PUD_SIZE);
 
-        /* Enable PGE if available */
-        if (cpu_has_pge) {
-                set_in_cr4(X86_CR4_PGE);
-                __supported_pte_mask |= _PAGE_GLOBAL;
+                        if (memblock_is_region_memory(start, end - start))
+                                mr[i].page_size_mask |= 1<<PG_LEVEL_1G;
+                }
         }
+}
 
-        if (use_gbpages)
-                page_size_mask |= 1 << PG_LEVEL_1G;
-        if (use_pse)
-                page_size_mask |= 1 << PG_LEVEL_2M;
+static int __meminit split_mem_range(struct map_range *mr, int nr_range,
+                                     unsigned long start,
+                                     unsigned long end)
+{
+        unsigned long start_pfn, end_pfn, limit_pfn;
+        unsigned long pfn;
+        int i;
 
-        memset(mr, 0, sizeof(mr));
-        nr_range = 0;
+        limit_pfn = PFN_DOWN(end);
 
         /* head if not big page alignment ? */
-        start_pfn = start >> PAGE_SHIFT;
-        pos = start_pfn << PAGE_SHIFT;
+        pfn = start_pfn = PFN_DOWN(start);
 #ifdef CONFIG_X86_32
         /*
          * Don't use a large page for the first 2/4MB of memory
@@ -181,66 +220,60 @@ unsigned long __init_refok init_memory_mapping(unsigned long start,
          * and overlapping MTRRs into large pages can cause
          * slowdowns.
          */
-        if (pos == 0)
-                end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT);
+        if (pfn == 0)
+                end_pfn = PFN_DOWN(PMD_SIZE);
         else
-                end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
-                                 << (PMD_SHIFT - PAGE_SHIFT);
+                end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 #else /* CONFIG_X86_64 */
-        end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT)
-                        << (PMD_SHIFT - PAGE_SHIFT);
+        end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 #endif
-        if (end_pfn > (end >> PAGE_SHIFT))
-                end_pfn = end >> PAGE_SHIFT;
+        if (end_pfn > limit_pfn)
+                end_pfn = limit_pfn;
         if (start_pfn < end_pfn) {
                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
-                pos = end_pfn << PAGE_SHIFT;
+                pfn = end_pfn;
         }
 
         /* big page (2M) range */
-        start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
-                         << (PMD_SHIFT - PAGE_SHIFT);
+        start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 #ifdef CONFIG_X86_32
-        end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
+        end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
 #else /* CONFIG_X86_64 */
-        end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
-                         << (PUD_SHIFT - PAGE_SHIFT);
-        if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)))
-                end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT));
+        end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
+        if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE)))
+                end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
 #endif
 
         if (start_pfn < end_pfn) {
                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
                                 page_size_mask & (1<<PG_LEVEL_2M));
-                pos = end_pfn << PAGE_SHIFT;
+                pfn = end_pfn;
         }
 
 #ifdef CONFIG_X86_64
         /* big page (1G) range */
-        start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
-                         << (PUD_SHIFT - PAGE_SHIFT);
-        end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT);
+        start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
+        end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE));
         if (start_pfn < end_pfn) {
                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
                                 page_size_mask &
                                  ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
-                pos = end_pfn << PAGE_SHIFT;
+                pfn = end_pfn;
         }
 
         /* tail is not big page (1G) alignment */
-        start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
-                         << (PMD_SHIFT - PAGE_SHIFT);
-        end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
+        start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
+        end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
         if (start_pfn < end_pfn) {
                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
                                 page_size_mask & (1<<PG_LEVEL_2M));
-                pos = end_pfn << PAGE_SHIFT;
+                pfn = end_pfn;
         }
 #endif
 
         /* tail is not big page (2M) alignment */
-        start_pfn = pos>>PAGE_SHIFT;
-        end_pfn = end>>PAGE_SHIFT;
+        start_pfn = pfn;
+        end_pfn = limit_pfn;
         nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
 
         /* try to merge same page size and continuous */
@@ -257,59 +290,169 @@ unsigned long __init_refok init_memory_mapping(unsigned long start,
                 nr_range--;
         }
 
+        if (!after_bootmem)
+                adjust_range_page_size_mask(mr, nr_range);
+
         for (i = 0; i < nr_range; i++)
                 printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n",
                                 mr[i].start, mr[i].end - 1,
                         (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":(
                          (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k"));
 
-        /*
-         * Find space for the kernel direct mapping tables.
-         *
-         * Later we should allocate these tables in the local node of the
-         * memory mapped. Unfortunately this is done currently before the
-         * nodes are discovered.
-         */
-        if (!after_bootmem)
-                find_early_table_space(mr, nr_range);
+        return nr_range;
+}
+
+struct range pfn_mapped[E820_X_MAX];
+int nr_pfn_mapped;
+
+static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn)
+{
+        nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_X_MAX,
+                                             nr_pfn_mapped, start_pfn, end_pfn);
+        nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_X_MAX);
+
+        max_pfn_mapped = max(max_pfn_mapped, end_pfn);
+
+        if (start_pfn < (1UL<<(32-PAGE_SHIFT)))
+                max_low_pfn_mapped = max(max_low_pfn_mapped,
+                                         min(end_pfn, 1UL<<(32-PAGE_SHIFT)));
+}
+
+bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn)
+{
+        int i;
+
+        for (i = 0; i < nr_pfn_mapped; i++)
+                if ((start_pfn >= pfn_mapped[i].start) &&
+                    (end_pfn <= pfn_mapped[i].end))
+                        return true;
+
+        return false;
+}
+
+/*
+ * Setup the direct mapping of the physical memory at PAGE_OFFSET.
+ * This runs before bootmem is initialized and gets pages directly from
+ * the physical memory. To access them they are temporarily mapped.
+ */
+unsigned long __init_refok init_memory_mapping(unsigned long start,
+                                               unsigned long end)
+{
+        struct map_range mr[NR_RANGE_MR];
+        unsigned long ret = 0;
+        int nr_range, i;
+
+        pr_info("init_memory_mapping: [mem %#010lx-%#010lx]\n",
+               start, end - 1);
+
+        memset(mr, 0, sizeof(mr));
+        nr_range = split_mem_range(mr, 0, start, end);
 
         for (i = 0; i < nr_range; i++)
                 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
                                                    mr[i].page_size_mask);
 
-#ifdef CONFIG_X86_32
-        early_ioremap_page_table_range_init();
+        add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT);
 
-        load_cr3(swapper_pg_dir);
-#endif
+        return ret >> PAGE_SHIFT;
+}
 
-        __flush_tlb_all();
+/*
+ * would have hole in the middle or ends, and only ram parts will be mapped.
+ */
+static unsigned long __init init_range_memory_mapping(
+                                           unsigned long r_start,
+                                           unsigned long r_end)
+{
+        unsigned long start_pfn, end_pfn;
+        unsigned long mapped_ram_size = 0;
+        int i;
 
-        /*
-         * Reserve the kernel pagetable pages we used (pgt_buf_start -
-         * pgt_buf_end) and free the other ones (pgt_buf_end - pgt_buf_top)
-         * so that they can be reused for other purposes.
-         *
-         * On native it just means calling memblock_reserve, on Xen it also
-         * means marking RW the pagetable pages that we allocated before
-         * but that haven't been used.
-         *
-         * In fact on xen we mark RO the whole range pgt_buf_start -
-         * pgt_buf_top, because we have to make sure that when
-         * init_memory_mapping reaches the pagetable pages area, it maps
-         * RO all the pagetable pages, including the ones that are beyond
-         * pgt_buf_end at that time.
-         */
-        if (!after_bootmem && pgt_buf_end > pgt_buf_start)
-                x86_init.mapping.pagetable_reserve(PFN_PHYS(pgt_buf_start),
-                                PFN_PHYS(pgt_buf_end));
+        for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
+                u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end);
+                u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end);
+                if (start >= end)
+                        continue;
 
-        if (!after_bootmem)
-                early_memtest(start, end);
+                /*
+                 * if it is overlapping with brk pgt, we need to
+                 * alloc pgt buf from memblock instead.
+                 */
+                can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >=
+                                    min(end, (u64)pgt_buf_top<<PAGE_SHIFT);
+                init_memory_mapping(start, end);
+                mapped_ram_size += end - start;
+                can_use_brk_pgt = true;
+        }
 
-        return ret >> PAGE_SHIFT;
+        return mapped_ram_size;
 }
 
+/* (PUD_SHIFT-PMD_SHIFT)/2 */
+#define STEP_SIZE_SHIFT 5
+void __init init_mem_mapping(void)
+{
+        unsigned long end, real_end, start, last_start;
+        unsigned long step_size;
+        unsigned long addr;
+        unsigned long mapped_ram_size = 0;
+        unsigned long new_mapped_ram_size;
+
+        probe_page_size_mask();
+
+#ifdef CONFIG_X86_64
+        end = max_pfn << PAGE_SHIFT;
+#else
+        end = max_low_pfn << PAGE_SHIFT;
+#endif
+
+        /* the ISA range is always mapped regardless of memory holes */
+        init_memory_mapping(0, ISA_END_ADDRESS);
+
+        /* xen has big range in reserved near end of ram, skip it at first */
+        addr = memblock_find_in_range(ISA_END_ADDRESS, end, PMD_SIZE,
+                         PAGE_SIZE);
+        real_end = addr + PMD_SIZE;
+
+        /* step_size need to be small so pgt_buf from BRK could cover it */
+        step_size = PMD_SIZE;
+        max_pfn_mapped = 0; /* will get exact value next */
+        min_pfn_mapped = real_end >> PAGE_SHIFT;
+        last_start = start = real_end;
+        while (last_start > ISA_END_ADDRESS) {
+                if (last_start > step_size) {
+                        start = round_down(last_start - 1, step_size);
+                        if (start < ISA_END_ADDRESS)
+                                start = ISA_END_ADDRESS;
+                } else
+                        start = ISA_END_ADDRESS;
+                new_mapped_ram_size = init_range_memory_mapping(start,
+                                                        last_start);
+                last_start = start;
+                min_pfn_mapped = last_start >> PAGE_SHIFT;
+                /* only increase step_size after big range get mapped */
+                if (new_mapped_ram_size > mapped_ram_size)
+                        step_size <<= STEP_SIZE_SHIFT;
+                mapped_ram_size += new_mapped_ram_size;
+        }
+
+        if (real_end < end)
+                init_range_memory_mapping(real_end, end);
+
+#ifdef CONFIG_X86_64
+        if (max_pfn > max_low_pfn) {
+                /* can we preseve max_low_pfn ?*/
+                max_low_pfn = max_pfn;
+        }
+#else
+        early_ioremap_page_table_range_init();
+#endif
+
+        load_cr3(swapper_pg_dir);
+        __flush_tlb_all();
+
+        early_memtest(0, max_pfn_mapped << PAGE_SHIFT);
+}
 
 /*
  * devmem_is_allowed() checks to see if /dev/mem access to a certain address
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 745d66b843c8..b299724f6e34 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -53,25 +53,14 @@
 #include <asm/page_types.h>
 #include <asm/init.h>
 
+#include "mm_internal.h"
+
 unsigned long highstart_pfn, highend_pfn;
 
 static noinline int do_test_wp_bit(void);
 
 bool __read_mostly __vmalloc_start_set = false;
 
-static __init void *alloc_low_page(void)
-{
-        unsigned long pfn = pgt_buf_end++;
-        void *adr;
-
-        if (pfn >= pgt_buf_top)
-                panic("alloc_low_page: ran out of memory");
-
-        adr = __va(pfn * PAGE_SIZE);
-        clear_page(adr);
-        return adr;
-}
-
 /*
  * Creates a middle page table and puts a pointer to it in the
  * given global directory entry. This only returns the gd entry
@@ -84,10 +73,7 @@ static pmd_t * __init one_md_table_init(pgd_t *pgd)
 
 #ifdef CONFIG_X86_PAE
         if (!(pgd_val(*pgd) & _PAGE_PRESENT)) {
-                if (after_bootmem)
-                        pmd_table = (pmd_t *)alloc_bootmem_pages(PAGE_SIZE);
-                else
-                        pmd_table = (pmd_t *)alloc_low_page();
+                pmd_table = (pmd_t *)alloc_low_page();
                 paravirt_alloc_pmd(&init_mm, __pa(pmd_table) >> PAGE_SHIFT);
                 set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
                 pud = pud_offset(pgd, 0);
@@ -109,17 +95,7 @@ static pmd_t * __init one_md_table_init(pgd_t *pgd)
 static pte_t * __init one_page_table_init(pmd_t *pmd)
 {
         if (!(pmd_val(*pmd) & _PAGE_PRESENT)) {
-                pte_t *page_table = NULL;
-
-                if (after_bootmem) {
-#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK)
-                        page_table = (pte_t *) alloc_bootmem_pages(PAGE_SIZE);
-#endif
-                        if (!page_table)
-                                page_table =
-                                (pte_t *)alloc_bootmem_pages(PAGE_SIZE);
-                } else
-                        page_table = (pte_t *)alloc_low_page();
+                pte_t *page_table = (pte_t *)alloc_low_page();
 
                 paravirt_alloc_pte(&init_mm, __pa(page_table) >> PAGE_SHIFT);
                 set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE));
@@ -146,8 +122,39 @@ pte_t * __init populate_extra_pte(unsigned long vaddr)
         return one_page_table_init(pmd) + pte_idx;
 }
 
+static unsigned long __init
+page_table_range_init_count(unsigned long start, unsigned long end)
+{
+        unsigned long count = 0;
+#ifdef CONFIG_HIGHMEM
+        int pmd_idx_kmap_begin = fix_to_virt(FIX_KMAP_END) >> PMD_SHIFT;
+        int pmd_idx_kmap_end = fix_to_virt(FIX_KMAP_BEGIN) >> PMD_SHIFT;
+        int pgd_idx, pmd_idx;
+        unsigned long vaddr;
+
+        if (pmd_idx_kmap_begin == pmd_idx_kmap_end)
+                return 0;
+
+        vaddr = start;
+        pgd_idx = pgd_index(vaddr);
+
+        for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd_idx++) {
+                for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
+                                                        pmd_idx++) {
+                        if ((vaddr >> PMD_SHIFT) >= pmd_idx_kmap_begin &&
+                            (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end)
+                                count++;
+                        vaddr += PMD_SIZE;
+                }
+                pmd_idx = 0;
+        }
+#endif
+        return count;
+}
+
 static pte_t *__init page_table_kmap_check(pte_t *pte, pmd_t *pmd,
-                                           unsigned long vaddr, pte_t *lastpte)
+                                           unsigned long vaddr, pte_t *lastpte,
+                                           void **adr)
 {
 #ifdef CONFIG_HIGHMEM
         /*
@@ -161,16 +168,15 @@ static pte_t *__init page_table_kmap_check(pte_t *pte, pmd_t *pmd,
 
         if (pmd_idx_kmap_begin != pmd_idx_kmap_end
             && (vaddr >> PMD_SHIFT) >= pmd_idx_kmap_begin
-            && (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end
-            && ((__pa(pte) >> PAGE_SHIFT) < pgt_buf_start
-                || (__pa(pte) >> PAGE_SHIFT) >= pgt_buf_end)) {
+            && (vaddr >> PMD_SHIFT) <= pmd_idx_kmap_end) {
                 pte_t *newpte;
                 int i;
 
                 BUG_ON(after_bootmem);
-                newpte = alloc_low_page();
+                newpte = *adr;
                 for (i = 0; i < PTRS_PER_PTE; i++)
                         set_pte(newpte + i, pte[i]);
+                *adr = (void *)(((unsigned long)(*adr)) + PAGE_SIZE);
 
                 paravirt_alloc_pte(&init_mm, __pa(newpte) >> PAGE_SHIFT);
                 set_pmd(pmd, __pmd(__pa(newpte)|_PAGE_TABLE));
@@ -204,6 +210,11 @@ page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
         pgd_t *pgd;
         pmd_t *pmd;
         pte_t *pte = NULL;
+        unsigned long count = page_table_range_init_count(start, end);
+        void *adr = NULL;
+
+        if (count)
+                adr = alloc_low_pages(count);
 
         vaddr = start;
         pgd_idx = pgd_index(vaddr);
@@ -216,7 +227,7 @@ page_table_range_init(unsigned long start, unsigned long end, pgd_t *pgd_base)
                 for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end);
                                                         pmd++, pmd_idx++) {
                         pte = page_table_kmap_check(one_page_table_init(pmd),
-                                                    pmd, vaddr, pte);
+                                                    pmd, vaddr, pte, &adr);
 
                         vaddr += PMD_SIZE;
                 }
@@ -310,6 +321,7 @@ repeat:
                                         __pgprot(PTE_IDENT_ATTR |
                                                  _PAGE_PSE);
 
+                                pfn &= PMD_MASK >> PAGE_SHIFT;
                                 addr2 = (pfn + PTRS_PER_PTE-1) * PAGE_SIZE +
                                         PAGE_OFFSET + PAGE_SIZE-1;
 
@@ -455,9 +467,14 @@ void __init native_pagetable_init(void)
 
         /*
          * Remove any mappings which extend past the end of physical
-         * memory from the boot time page table:
+         * memory from the boot time page table.
+         * In virtual address space, we should have at least two pages
+         * from VMALLOC_END to pkmap or fixmap according to VMALLOC_END
+         * definition. And max_low_pfn is set to VMALLOC_END physical
+         * address. If initial memory mapping is doing right job, we
+         * should have pte used near max_low_pfn or one pmd is not present.
          */
-        for (pfn = max_low_pfn + 1; pfn < 1<<(32-PAGE_SHIFT); pfn++) {
+        for (pfn = max_low_pfn; pfn < 1<<(32-PAGE_SHIFT); pfn++) {
                 va = PAGE_OFFSET + (pfn<<PAGE_SHIFT);
                 pgd = base + pgd_index(va);
                 if (!pgd_present(*pgd))
@@ -468,10 +485,19 @@ void __init native_pagetable_init(void)
                 if (!pmd_present(*pmd))
                         break;
 
+                /* should not be large page here */
+                if (pmd_large(*pmd)) {
+                        pr_warn("try to clear pte for ram above max_low_pfn: pfn: %lx pmd: %p pmd phys: %lx, but pmd is big page and is not using pte !\n",
+                                pfn, pmd, __pa(pmd));
+                        BUG_ON(1);
+                }
+
                 pte = pte_offset_kernel(pmd, va);
                 if (!pte_present(*pte))
                         break;
 
+                printk(KERN_DEBUG "clearing pte for ram above max_low_pfn: pfn: %lx pmd: %p pmd phys: %lx pte: %p pte phys: %lx\n",
+                                pfn, pmd, __pa(pmd), pte, __pa(pte));
                 pte_clear(NULL, va, pte);
         }
         paravirt_alloc_pmd(&init_mm, __pa(base) >> PAGE_SHIFT);
@@ -550,7 +576,7 @@ early_param("highmem", parse_highmem);
  * artificially via the highmem=x boot parameter then create
  * it:
  */
-void __init lowmem_pfn_init(void)
+static void __init lowmem_pfn_init(void)
 {
         /* max_low_pfn is 0, we already have early_res support */
         max_low_pfn = max_pfn;
@@ -586,7 +612,7 @@ void __init lowmem_pfn_init(void)
  * We have more RAM than fits into lowmem - we try to put it into
  * highmem, also taking the highmem=x boot parameter into account:
  */
-void __init highmem_pfn_init(void)
+static void __init highmem_pfn_init(void)
 {
         max_low_pfn = MAXMEM_PFN;
 
@@ -669,8 +695,6 @@ void __init setup_bootmem_allocator(void)
         printk(KERN_INFO "  mapped low ram: 0 - %08lx\n",
                  max_pfn_mapped<<PAGE_SHIFT);
         printk(KERN_INFO "  low ram: 0 - %08lx\n", max_low_pfn<<PAGE_SHIFT);
-
-        after_bootmem = 1;
 }
 
 /*
@@ -753,6 +777,8 @@ void __init mem_init(void)
                 if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp)))
                         reservedpages++;
 
+        after_bootmem = 1;
+
         codesize =  (unsigned long) &_etext - (unsigned long) &_text;
         datasize =  (unsigned long) &_edata - (unsigned long) &_etext;
         initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 287c6d6a9ef1..edaa2daf4b37 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -54,6 +54,82 @@
 #include <asm/uv/uv.h>
 #include <asm/setup.h>
 
+#include "mm_internal.h"
+
+static void ident_pmd_init(unsigned long pmd_flag, pmd_t *pmd_page,
+                           unsigned long addr, unsigned long end)
+{
+        addr &= PMD_MASK;
+        for (; addr < end; addr += PMD_SIZE) {
+                pmd_t *pmd = pmd_page + pmd_index(addr);
+
+                if (!pmd_present(*pmd))
+                        set_pmd(pmd, __pmd(addr | pmd_flag));
+        }
+}
+static int ident_pud_init(struct x86_mapping_info *info, pud_t *pud_page,
+                          unsigned long addr, unsigned long end)
+{
+        unsigned long next;
+
+        for (; addr < end; addr = next) {
+                pud_t *pud = pud_page + pud_index(addr);
+                pmd_t *pmd;
+
+                next = (addr & PUD_MASK) + PUD_SIZE;
+                if (next > end)
+                        next = end;
+
+                if (pud_present(*pud)) {
+                        pmd = pmd_offset(pud, 0);
+                        ident_pmd_init(info->pmd_flag, pmd, addr, next);
+                        continue;
+                }
+                pmd = (pmd_t *)info->alloc_pgt_page(info->context);
+                if (!pmd)
+                        return -ENOMEM;
+                ident_pmd_init(info->pmd_flag, pmd, addr, next);
+                set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
+        }
+
+        return 0;
+}
+
+int kernel_ident_mapping_init(struct x86_mapping_info *info, pgd_t *pgd_page,
+                              unsigned long addr, unsigned long end)
+{
+        unsigned long next;
+        int result;
+        int off = info->kernel_mapping ? pgd_index(__PAGE_OFFSET) : 0;
+
+        for (; addr < end; addr = next) {
+                pgd_t *pgd = pgd_page + pgd_index(addr) + off;
+                pud_t *pud;
+
+                next = (addr & PGDIR_MASK) + PGDIR_SIZE;
+                if (next > end)
+                        next = end;
+
+                if (pgd_present(*pgd)) {
+                        pud = pud_offset(pgd, 0);
+                        result = ident_pud_init(info, pud, addr, next);
+                        if (result)
+                                return result;
+                        continue;
+                }
+
+                pud = (pud_t *)info->alloc_pgt_page(info->context);
+                if (!pud)
+                        return -ENOMEM;
+                result = ident_pud_init(info, pud, addr, next);
+                if (result)
+                        return result;
+                set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE));
+        }
+
+        return 0;
+}
+
 static int __init parse_direct_gbpages_off(char *arg)
 {
         direct_gbpages = 0;
@@ -302,10 +378,18 @@ void __init init_extra_mapping_uc(unsigned long phys, unsigned long size)
 void __init cleanup_highmap(void)
 {
         unsigned long vaddr = __START_KERNEL_map;
-        unsigned long vaddr_end = __START_KERNEL_map + (max_pfn_mapped << PAGE_SHIFT);
+        unsigned long vaddr_end = __START_KERNEL_map + KERNEL_IMAGE_SIZE;
         unsigned long end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1;
         pmd_t *pmd = level2_kernel_pgt;
 
+        /*
+         * Native path, max_pfn_mapped is not set yet.
+         * Xen has valid max_pfn_mapped set in
+         *      arch/x86/xen/mmu.c:xen_setup_kernel_pagetable().
+         */
+        if (max_pfn_mapped)
+                vaddr_end = __START_KERNEL_map + (max_pfn_mapped << PAGE_SHIFT);
+
         for (; vaddr + PMD_SIZE - 1 < vaddr_end; pmd++, vaddr += PMD_SIZE) {
                 if (pmd_none(*pmd))
                         continue;
@@ -314,69 +398,24 @@ void __init cleanup_highmap(void)
         }
 }
 
-static __ref void *alloc_low_page(unsigned long *phys)
-{
-        unsigned long pfn = pgt_buf_end++;
-        void *adr;
-
-        if (after_bootmem) {
-                adr = (void *)get_zeroed_page(GFP_ATOMIC | __GFP_NOTRACK);
-                *phys = __pa(adr);
-
-                return adr;
-        }
-
-        if (pfn >= pgt_buf_top)
-                panic("alloc_low_page: ran out of memory");
-
-        adr = early_memremap(pfn * PAGE_SIZE, PAGE_SIZE);
-        clear_page(adr);
-        *phys  = pfn * PAGE_SIZE;
-        return adr;
-}
-
-static __ref void *map_low_page(void *virt)
-{
-        void *adr;
-        unsigned long phys, left;
-
-        if (after_bootmem)
-                return virt;
-
-        phys = __pa(virt);
-        left = phys & (PAGE_SIZE - 1);
-        adr = early_memremap(phys & PAGE_MASK, PAGE_SIZE);
-        adr = (void *)(((unsigned long)adr) | left);
-
-        return adr;
-}
-
-static __ref void unmap_low_page(void *adr)
-{
-        if (after_bootmem)
-                return;
-
-        early_iounmap((void *)((unsigned long)adr & PAGE_MASK), PAGE_SIZE);
-}
-
 static unsigned long __meminit
 phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end,
               pgprot_t prot)
 {
-        unsigned pages = 0;
+        unsigned long pages = 0, next;
         unsigned long last_map_addr = end;
         int i;
 
         pte_t *pte = pte_page + pte_index(addr);
 
-        for(i = pte_index(addr); i < PTRS_PER_PTE; i++, addr += PAGE_SIZE, pte++) {
-
+        for (i = pte_index(addr); i < PTRS_PER_PTE; i++, addr = next, pte++) {
+                next = (addr & PAGE_MASK) + PAGE_SIZE;
                 if (addr >= end) {
-                        if (!after_bootmem) {
-                                for(; i < PTRS_PER_PTE; i++, pte++)
-                                        set_pte(pte, __pte(0));
-                        }
-                        break;
+                        if (!after_bootmem &&
+                            !e820_any_mapped(addr & PAGE_MASK, next, E820_RAM) &&
+                            !e820_any_mapped(addr & PAGE_MASK, next, E820_RESERVED_KERN))
+                                set_pte(pte, __pte(0));
+                        continue;
                 }
 
                 /*
@@ -414,28 +453,25 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
         int i = pmd_index(address);
 
         for (; i < PTRS_PER_PMD; i++, address = next) {
-                unsigned long pte_phys;
                 pmd_t *pmd = pmd_page + pmd_index(address);
                 pte_t *pte;
                 pgprot_t new_prot = prot;
 
+                next = (address & PMD_MASK) + PMD_SIZE;
                 if (address >= end) {
-                        if (!after_bootmem) {
-                                for (; i < PTRS_PER_PMD; i++, pmd++)
-                                        set_pmd(pmd, __pmd(0));
-                        }
-                        break;
+                        if (!after_bootmem &&
+                            !e820_any_mapped(address & PMD_MASK, next, E820_RAM) &&
+                            !e820_any_mapped(address & PMD_MASK, next, E820_RESERVED_KERN))
+                                set_pmd(pmd, __pmd(0));
+                        continue;
                 }
 
-                next = (address & PMD_MASK) + PMD_SIZE;
-
                 if (pmd_val(*pmd)) {
                         if (!pmd_large(*pmd)) {
                                 spin_lock(&init_mm.page_table_lock);
-                                pte = map_low_page((pte_t *)pmd_page_vaddr(*pmd));
+                                pte = (pte_t *)pmd_page_vaddr(*pmd);
                                 last_map_addr = phys_pte_init(pte, address,
                                                                 end, prot);
-                                unmap_low_page(pte);
                                 spin_unlock(&init_mm.page_table_lock);
                                 continue;
                         }
@@ -464,19 +500,18 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
                         pages++;
                         spin_lock(&init_mm.page_table_lock);
                         set_pte((pte_t *)pmd,
-                                pfn_pte(address >> PAGE_SHIFT,
+                                pfn_pte((address & PMD_MASK) >> PAGE_SHIFT,
                                         __pgprot(pgprot_val(prot) | _PAGE_PSE)));
                         spin_unlock(&init_mm.page_table_lock);
                         last_map_addr = next;
                         continue;
                 }
 
-                pte = alloc_low_page(&pte_phys);
+                pte = alloc_low_page();
                 last_map_addr = phys_pte_init(pte, address, end, new_prot);
-                unmap_low_page(pte);
 
                 spin_lock(&init_mm.page_table_lock);
-                pmd_populate_kernel(&init_mm, pmd, __va(pte_phys));
+                pmd_populate_kernel(&init_mm, pmd, pte);
                 spin_unlock(&init_mm.page_table_lock);
         }
         update_page_count(PG_LEVEL_2M, pages);
@@ -492,27 +527,24 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
         int i = pud_index(addr);
 
         for (; i < PTRS_PER_PUD; i++, addr = next) {
-                unsigned long pmd_phys;
                 pud_t *pud = pud_page + pud_index(addr);
                 pmd_t *pmd;
                 pgprot_t prot = PAGE_KERNEL;
 
-                if (addr >= end)
-                        break;
-
                 next = (addr & PUD_MASK) + PUD_SIZE;
-
-                if (!after_bootmem && !e820_any_mapped(addr, next, 0)) {
-                        set_pud(pud, __pud(0));
+                if (addr >= end) {
+                        if (!after_bootmem &&
+                            !e820_any_mapped(addr & PUD_MASK, next, E820_RAM) &&
+                            !e820_any_mapped(addr & PUD_MASK, next, E820_RESERVED_KERN))
+                                set_pud(pud, __pud(0));
                         continue;
                 }
 
                 if (pud_val(*pud)) {
                         if (!pud_large(*pud)) {
-                                pmd = map_low_page(pmd_offset(pud, 0));
+                                pmd = pmd_offset(pud, 0);
                                 last_map_addr = phys_pmd_init(pmd, addr, end,
                                                          page_size_mask, prot);
-                                unmap_low_page(pmd);
                                 __flush_tlb_all();
                                 continue;
                         }
@@ -541,19 +573,19 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
                         pages++;
                         spin_lock(&init_mm.page_table_lock);
                         set_pte((pte_t *)pud,
-                                pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL_LARGE));
+                                pfn_pte((addr & PUD_MASK) >> PAGE_SHIFT,
+                                        PAGE_KERNEL_LARGE));
                         spin_unlock(&init_mm.page_table_lock);
                         last_map_addr = next;
                         continue;
                 }
 
-                pmd = alloc_low_page(&pmd_phys);
+                pmd = alloc_low_page();
                 last_map_addr = phys_pmd_init(pmd, addr, end, page_size_mask,
                                               prot);
-                unmap_low_page(pmd);
 
                 spin_lock(&init_mm.page_table_lock);
-                pud_populate(&init_mm, pud, __va(pmd_phys));
+                pud_populate(&init_mm, pud, pmd);
                 spin_unlock(&init_mm.page_table_lock);
         }
         __flush_tlb_all();
@@ -578,28 +610,23 @@ kernel_physical_mapping_init(unsigned long start,
 
         for (; start < end; start = next) {
                 pgd_t *pgd = pgd_offset_k(start);
-                unsigned long pud_phys;
                 pud_t *pud;
 
-                next = (start + PGDIR_SIZE) & PGDIR_MASK;
-                if (next > end)
-                        next = end;
+                next = (start & PGDIR_MASK) + PGDIR_SIZE;
 
                 if (pgd_val(*pgd)) {
-                        pud = map_low_page((pud_t *)pgd_page_vaddr(*pgd));
+                        pud = (pud_t *)pgd_page_vaddr(*pgd);
                         last_map_addr = phys_pud_init(pud, __pa(start),
                                                  __pa(end), page_size_mask);
-                        unmap_low_page(pud);
                         continue;
                 }
 
-                pud = alloc_low_page(&pud_phys);
-                last_map_addr = phys_pud_init(pud, __pa(start), __pa(next),
+                pud = alloc_low_page();
+                last_map_addr = phys_pud_init(pud, __pa(start), __pa(end),
                                                  page_size_mask);
-                unmap_low_page(pud);
 
                 spin_lock(&init_mm.page_table_lock);
-                pgd_populate(&init_mm, pgd, __va(pud_phys));
+                pgd_populate(&init_mm, pgd, pud);
                 spin_unlock(&init_mm.page_table_lock);
                 pgd_changed = true;
         }
@@ -664,13 +691,11 @@ int arch_add_memory(int nid, u64 start, u64 size)
 {
         struct pglist_data *pgdat = NODE_DATA(nid);
         struct zone *zone = pgdat->node_zones + ZONE_NORMAL;
-        unsigned long last_mapped_pfn, start_pfn = start >> PAGE_SHIFT;
+        unsigned long start_pfn = start >> PAGE_SHIFT;
         unsigned long nr_pages = size >> PAGE_SHIFT;
         int ret;
 
-        last_mapped_pfn = init_memory_mapping(start, start + size);
-        if (last_mapped_pfn > max_pfn_mapped)
-                max_pfn_mapped = last_mapped_pfn;
+        init_memory_mapping(start, start + size);
 
         ret = __add_pages(nid, zone, start_pfn, nr_pages);
         WARN_ON_ONCE(ret);
@@ -686,6 +711,16 @@ EXPORT_SYMBOL_GPL(arch_add_memory);
 
 static struct kcore_list kcore_vsyscall;
 
+static void __init register_page_bootmem_info(void)
+{
+#ifdef CONFIG_NUMA
+        int i;
+
+        for_each_online_node(i)
+                register_page_bootmem_info_node(NODE_DATA(i));
+#endif
+}
+
 void __init mem_init(void)
 {
         long codesize, reservedpages, datasize, initsize;
@@ -698,11 +733,8 @@ void __init mem_init(void)
         reservedpages = 0;
 
         /* this will put all low memory onto the freelists */
-#ifdef CONFIG_NUMA
-        totalram_pages = numa_free_all_bootmem();
-#else
+        register_page_bootmem_info();
         totalram_pages = free_all_bootmem();
-#endif
 
         absent_pages = absent_pages_in_range(0, max_pfn);
         reservedpages = max_pfn - totalram_pages - absent_pages;
@@ -776,6 +808,7 @@ void mark_rodata_ro(void)
         unsigned long end = (unsigned long) &__end_rodata_hpage_align;
         unsigned long text_end = PFN_ALIGN(&__stop___ex_table);
         unsigned long rodata_end = PFN_ALIGN(&__end_rodata);
+        unsigned long all_end = PFN_ALIGN(&_end);
 
         printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n",
                (end - start) >> 10);
@@ -784,10 +817,10 @@ void mark_rodata_ro(void)
         kernel_set_to_readonly = 1;
 
         /*
-         * The rodata section (but not the kernel text!) should also be
-         * not-executable.
+         * The rodata/data/bss/brk section (but not the kernel text!)
+         * should also be not-executable.
          */
-        set_memory_nx(rodata_start, (end - rodata_start) >> PAGE_SHIFT);
+        set_memory_nx(rodata_start, (all_end - rodata_start) >> PAGE_SHIFT);
 
         rodata_test();
 
diff --git a/arch/x86/mm/mm_internal.h b/arch/x86/mm/mm_internal.h
new file mode 100644
index 000000000000..6b563a118891
--- /dev/null
+++ b/arch/x86/mm/mm_internal.h
@@ -0,0 +1,19 @@
+#ifndef __X86_MM_INTERNAL_H
+#define __X86_MM_INTERNAL_H
+
+void *alloc_low_pages(unsigned int num);
+static inline void *alloc_low_page(void)
+{
+        return alloc_low_pages(1);
+}
+
+void early_ioremap_page_table_range_init(void);
+
+unsigned long kernel_physical_mapping_init(unsigned long start,
+                                             unsigned long end,
+                                             unsigned long page_size_mask);
+void zone_sizes_init(void);
+
+extern int after_bootmem;
+
+#endif  /* __X86_MM_INTERNAL_H */
diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c
index 92e27119ee1a..9405ffc91502 100644
--- a/arch/x86/mm/numa_64.c
+++ b/arch/x86/mm/numa_64.c
@@ -10,16 +10,3 @@ void __init initmem_init(void)
 {
         x86_numa_init();
 }
-
-unsigned long __init numa_free_all_bootmem(void)
-{
-        unsigned long pages = 0;
-        int i;
-
-        for_each_online_node(i)
-                pages += free_all_bootmem_node(NODE_DATA(i));
-
-        pages += free_low_memory_core_early(MAX_NUMNODES);
-
-        return pages;
-}
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 6d13d2a3f825..a1b1c88f9caf 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -579,16 +579,10 @@ static int split_large_page(pte_t *kpte, unsigned long address)
         for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc)
                 set_pte(&pbase[i], pfn_pte(pfn, ref_prot));
 
-        if (address >= (unsigned long)__va(0) &&
-                address < (unsigned long)__va(max_low_pfn_mapped << PAGE_SHIFT))
+        if (pfn_range_is_mapped(PFN_DOWN(__pa(address)),
+                                PFN_DOWN(__pa(address)) + 1))
                 split_page_count(level);
 
-#ifdef CONFIG_X86_64
-        if (address >= (unsigned long)__va(1UL<<32) &&
-                address < (unsigned long)__va(max_pfn_mapped << PAGE_SHIFT))
-                split_page_count(level);
-#endif
-
         /*
          * Install the new, split up pagetable.
          *
@@ -757,13 +751,9 @@ static int cpa_process_alias(struct cpa_data *cpa)
         unsigned long vaddr;
         int ret;
 
-        if (cpa->pfn >= max_pfn_mapped)
+        if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1))
                 return 0;
 
-#ifdef CONFIG_X86_64
-        if (cpa->pfn >= max_low_pfn_mapped && cpa->pfn < (1UL<<(32-PAGE_SHIFT)))
-                return 0;
-#endif
         /*
          * No need to redo, when the primary call touched the direct
          * mapping already: