1 files changed, 301 insertions, 158 deletions

diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index d7aea41563b3..d41815265a0b 100644
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -17,86 +17,132 @@
 #include <asm/proto.h>
 #include <asm/dma.h>            /* for MAX_DMA_PFN */
 
-unsigned long __initdata pgt_buf_start;
-unsigned long __meminitdata pgt_buf_end;
-unsigned long __meminitdata pgt_buf_top;
+#include "mm_internal.h"
 
-int after_bootmem;
+static unsigned long __initdata pgt_buf_start;
+static unsigned long __initdata pgt_buf_end;
+static unsigned long __initdata pgt_buf_top;
 
-int direct_gbpages
-#ifdef CONFIG_DIRECT_GBPAGES
-                                = 1
-#endif
-;
+static unsigned long min_pfn_mapped;
 
-struct map_range {
-        unsigned long start;
-        unsigned long end;
-        unsigned page_size_mask;
-};
+static bool __initdata can_use_brk_pgt = true;
 
 /*
- * First calculate space needed for kernel direct mapping page tables to cover
- * mr[0].start to mr[nr_range - 1].end, while accounting for possible 2M and 1GB
- * pages. Then find enough contiguous space for those page tables.
+ * Pages returned are already directly mapped.
+ *
+ * Changing that is likely to break Xen, see commit:
+ *
+ *    279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve
+ *
+ * for detailed information.
  */
-static void __init find_early_table_space(struct map_range *mr, int nr_range)
+__ref void *alloc_low_pages(unsigned int num)
 {
+        unsigned long pfn;
         int i;
-        unsigned long puds = 0, pmds = 0, ptes = 0, tables;
-        unsigned long start = 0, good_end;
-        phys_addr_t base;
 
-        for (i = 0; i < nr_range; i++) {
-                unsigned long range, extra;
+        if (after_bootmem) {
+                unsigned int order;
 
-                range = mr[i].end - mr[i].start;
-                puds += (range + PUD_SIZE - 1) >> PUD_SHIFT;
+                order = get_order((unsigned long)num << PAGE_SHIFT);
+                return (void *)__get_free_pages(GFP_ATOMIC | __GFP_NOTRACK |
+                                                __GFP_ZERO, order);
+        }
 
-                if (mr[i].page_size_mask & (1 << PG_LEVEL_1G)) {
-                        extra = range - ((range >> PUD_SHIFT) << PUD_SHIFT);
-                        pmds += (extra + PMD_SIZE - 1) >> PMD_SHIFT;
-                } else {
-                        pmds += (range + PMD_SIZE - 1) >> PMD_SHIFT;
-                }
+        if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) {
+                unsigned long ret;
+                if (min_pfn_mapped >= max_pfn_mapped)
+                        panic("alloc_low_page: ran out of memory");
+                ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT,
+                                        max_pfn_mapped << PAGE_SHIFT,
+                                        PAGE_SIZE * num , PAGE_SIZE);
+                if (!ret)
+                        panic("alloc_low_page: can not alloc memory");
+                memblock_reserve(ret, PAGE_SIZE * num);
+                pfn = ret >> PAGE_SHIFT;
+        } else {
+                pfn = pgt_buf_end;
+                pgt_buf_end += num;
+                printk(KERN_DEBUG "BRK [%#010lx, %#010lx] PGTABLE\n",
+                        pfn << PAGE_SHIFT, (pgt_buf_end << PAGE_SHIFT) - 1);
+        }
 
-                if (mr[i].page_size_mask & (1 << PG_LEVEL_2M)) {
-                        extra = range - ((range >> PMD_SHIFT) << PMD_SHIFT);
-#ifdef CONFIG_X86_32
-                        extra += PMD_SIZE;
-#endif
-                        ptes += (extra + PAGE_SIZE - 1) >> PAGE_SHIFT;
-                } else {
-                        ptes += (range + PAGE_SIZE - 1) >> PAGE_SHIFT;
-                }
+        for (i = 0; i < num; i++) {
+                void *adr;
+
+                adr = __va((pfn + i) << PAGE_SHIFT);
+                clear_page(adr);
         }
 
-        tables = roundup(puds * sizeof(pud_t), PAGE_SIZE);
-        tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE);
-        tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE);
+        return __va(pfn << PAGE_SHIFT);
+}
 
-#ifdef CONFIG_X86_32
-        /* for fixmap */
-        tables += roundup(__end_of_fixed_addresses * sizeof(pte_t), PAGE_SIZE);
-#endif
-        good_end = max_pfn_mapped << PAGE_SHIFT;
+/* need 4 4k for initial PMD_SIZE, 4k for 0-ISA_END_ADDRESS */
+#define INIT_PGT_BUF_SIZE       (5 * PAGE_SIZE)
+RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE);
+void  __init early_alloc_pgt_buf(void)
+{
+        unsigned long tables = INIT_PGT_BUF_SIZE;
+        phys_addr_t base;
 
-        base = memblock_find_in_range(start, good_end, tables, PAGE_SIZE);
-        if (!base)
-                panic("Cannot find space for the kernel page tables");
+        base = __pa(extend_brk(tables, PAGE_SIZE));
 
         pgt_buf_start = base >> PAGE_SHIFT;
         pgt_buf_end = pgt_buf_start;
         pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT);
+}
+
+int after_bootmem;
+
+int direct_gbpages
+#ifdef CONFIG_DIRECT_GBPAGES
+                                = 1
+#endif
+;
 
-        printk(KERN_DEBUG "kernel direct mapping tables up to %#lx @ [mem %#010lx-%#010lx]\n",
-                mr[nr_range - 1].end - 1, pgt_buf_start << PAGE_SHIFT,
-                (pgt_buf_top << PAGE_SHIFT) - 1);
+static void __init init_gbpages(void)
+{
+#ifdef CONFIG_X86_64
+        if (direct_gbpages && cpu_has_gbpages)
+                printk(KERN_INFO "Using GB pages for direct mapping\n");
+        else
+                direct_gbpages = 0;
+#endif
 }
 
-void __init native_pagetable_reserve(u64 start, u64 end)
+struct map_range {
+        unsigned long start;
+        unsigned long end;
+        unsigned page_size_mask;
+};
+
+static int page_size_mask;
+
+static void __init probe_page_size_mask(void)
 {
-        memblock_reserve(start, end - start);
+        init_gbpages();
+
+#if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK)
+        /*
+         * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
+         * This will simplify cpa(), which otherwise needs to support splitting
+         * large pages into small in interrupt context, etc.
+         */
+        if (direct_gbpages)
+                page_size_mask |= 1 << PG_LEVEL_1G;
+        if (cpu_has_pse)
+                page_size_mask |= 1 << PG_LEVEL_2M;
+#endif
+
+        /* Enable PSE if available */
+        if (cpu_has_pse)
+                set_in_cr4(X86_CR4_PSE);
+
+        /* Enable PGE if available */
+        if (cpu_has_pge) {
+                set_in_cr4(X86_CR4_PGE);
+                __supported_pte_mask |= _PAGE_GLOBAL;
+        }
 }
 
 #ifdef CONFIG_X86_32
@@ -122,58 +168,51 @@ static int __meminit save_mr(struct map_range *mr, int nr_range,
 }
 
 /*
- * Setup the direct mapping of the physical memory at PAGE_OFFSET.
- * This runs before bootmem is initialized and gets pages directly from
- * the physical memory. To access them they are temporarily mapped.
+ * adjust the page_size_mask for small range to go with
+ *      big page size instead small one if nearby are ram too.
  */
-unsigned long __init_refok init_memory_mapping(unsigned long start,
-                                               unsigned long end)
+static void __init_refok adjust_range_page_size_mask(struct map_range *mr,
+                                                         int nr_range)
 {
-        unsigned long page_size_mask = 0;
-        unsigned long start_pfn, end_pfn;
-        unsigned long ret = 0;
-        unsigned long pos;
-
-        struct map_range mr[NR_RANGE_MR];
-        int nr_range, i;
-        int use_pse, use_gbpages;
+        int i;
 
-        printk(KERN_INFO "init_memory_mapping: [mem %#010lx-%#010lx]\n",
-               start, end - 1);
+        for (i = 0; i < nr_range; i++) {
+                if ((page_size_mask & (1<<PG_LEVEL_2M)) &&
+                    !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) {
+                        unsigned long start = round_down(mr[i].start, PMD_SIZE);
+                        unsigned long end = round_up(mr[i].end, PMD_SIZE);
 
-#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KMEMCHECK)
-        /*
-         * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
-         * This will simplify cpa(), which otherwise needs to support splitting
-         * large pages into small in interrupt context, etc.
-         */
-        use_pse = use_gbpages = 0;
-#else
-        use_pse = cpu_has_pse;
-        use_gbpages = direct_gbpages;
+#ifdef CONFIG_X86_32
+                        if ((end >> PAGE_SHIFT) > max_low_pfn)
+                                continue;
 #endif
 
-        /* Enable PSE if available */
-        if (cpu_has_pse)
-                set_in_cr4(X86_CR4_PSE);
+                        if (memblock_is_region_memory(start, end - start))
+                                mr[i].page_size_mask |= 1<<PG_LEVEL_2M;
+                }
+                if ((page_size_mask & (1<<PG_LEVEL_1G)) &&
+                    !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) {
+                        unsigned long start = round_down(mr[i].start, PUD_SIZE);
+                        unsigned long end = round_up(mr[i].end, PUD_SIZE);
 
-        /* Enable PGE if available */
-        if (cpu_has_pge) {
-                set_in_cr4(X86_CR4_PGE);
-                __supported_pte_mask |= _PAGE_GLOBAL;
+                        if (memblock_is_region_memory(start, end - start))
+                                mr[i].page_size_mask |= 1<<PG_LEVEL_1G;
+                }
         }
+}
 
-        if (use_gbpages)
-                page_size_mask |= 1 << PG_LEVEL_1G;
-        if (use_pse)
-                page_size_mask |= 1 << PG_LEVEL_2M;
+static int __meminit split_mem_range(struct map_range *mr, int nr_range,
+                                     unsigned long start,
+                                     unsigned long end)
+{
+        unsigned long start_pfn, end_pfn, limit_pfn;
+        unsigned long pfn;
+        int i;
 
-        memset(mr, 0, sizeof(mr));
-        nr_range = 0;
+        limit_pfn = PFN_DOWN(end);
 
         /* head if not big page alignment ? */
-        start_pfn = start >> PAGE_SHIFT;
-        pos = start_pfn << PAGE_SHIFT;
+        pfn = start_pfn = PFN_DOWN(start);
 #ifdef CONFIG_X86_32
         /*
          * Don't use a large page for the first 2/4MB of memory
@@ -181,66 +220,60 @@ unsigned long __init_refok init_memory_mapping(unsigned long start,
          * and overlapping MTRRs into large pages can cause
          * slowdowns.
          */
-        if (pos == 0)
-                end_pfn = 1<<(PMD_SHIFT - PAGE_SHIFT);
+        if (pfn == 0)
+                end_pfn = PFN_DOWN(PMD_SIZE);
         else
-                end_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
-                                 << (PMD_SHIFT - PAGE_SHIFT);
+                end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 #else /* CONFIG_X86_64 */
-        end_pfn = ((pos + (PMD_SIZE - 1)) >> PMD_SHIFT)
-                        << (PMD_SHIFT - PAGE_SHIFT);
+        end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 #endif
-        if (end_pfn > (end >> PAGE_SHIFT))
-                end_pfn = end >> PAGE_SHIFT;
+        if (end_pfn > limit_pfn)
+                end_pfn = limit_pfn;
         if (start_pfn < end_pfn) {
                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
-                pos = end_pfn << PAGE_SHIFT;
+                pfn = end_pfn;
         }
 
         /* big page (2M) range */
-        start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
-                         << (PMD_SHIFT - PAGE_SHIFT);
+        start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 #ifdef CONFIG_X86_32
-        end_pfn = (end>>PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
+        end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
 #else /* CONFIG_X86_64 */
-        end_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
-                         << (PUD_SHIFT - PAGE_SHIFT);
-        if (end_pfn > ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT)))
-                end_pfn = ((end>>PMD_SHIFT)<<(PMD_SHIFT - PAGE_SHIFT));
+        end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
+        if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE)))
+                end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
 #endif
 
         if (start_pfn < end_pfn) {
                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
                                 page_size_mask & (1<<PG_LEVEL_2M));
-                pos = end_pfn << PAGE_SHIFT;
+                pfn = end_pfn;
         }
 
 #ifdef CONFIG_X86_64
         /* big page (1G) range */
-        start_pfn = ((pos + (PUD_SIZE - 1))>>PUD_SHIFT)
-                         << (PUD_SHIFT - PAGE_SHIFT);
-        end_pfn = (end >> PUD_SHIFT) << (PUD_SHIFT - PAGE_SHIFT);
+        start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
+        end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE));
         if (start_pfn < end_pfn) {
                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
                                 page_size_mask &
                                  ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
-                pos = end_pfn << PAGE_SHIFT;
+                pfn = end_pfn;
         }
 
         /* tail is not big page (1G) alignment */
-        start_pfn = ((pos + (PMD_SIZE - 1))>>PMD_SHIFT)
-                         << (PMD_SHIFT - PAGE_SHIFT);
-        end_pfn = (end >> PMD_SHIFT) << (PMD_SHIFT - PAGE_SHIFT);
+        start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
+        end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
         if (start_pfn < end_pfn) {
                 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
                                 page_size_mask & (1<<PG_LEVEL_2M));
-                pos = end_pfn << PAGE_SHIFT;
+                pfn = end_pfn;
         }
 #endif
 
         /* tail is not big page (2M) alignment */
-        start_pfn = pos>>PAGE_SHIFT;
-        end_pfn = end>>PAGE_SHIFT;
+        start_pfn = pfn;
+        end_pfn = limit_pfn;
         nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
 
         /* try to merge same page size and continuous */
@@ -257,59 +290,169 @@ unsigned long __init_refok init_memory_mapping(unsigned long start,
                 nr_range--;
         }
 
+        if (!after_bootmem)
+                adjust_range_page_size_mask(mr, nr_range);
+
         for (i = 0; i < nr_range; i++)
                 printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n",
                                 mr[i].start, mr[i].end - 1,
                         (mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":(
                          (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k"));
 
-        /*
-         * Find space for the kernel direct mapping tables.
-         *
-         * Later we should allocate these tables in the local node of the
-         * memory mapped. Unfortunately this is done currently before the
-         * nodes are discovered.
-         */
-        if (!after_bootmem)
-                find_early_table_space(mr, nr_range);
+        return nr_range;
+}
+
+struct range pfn_mapped[E820_X_MAX];
+int nr_pfn_mapped;
+
+static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn)
+{
+        nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_X_MAX,
+                                             nr_pfn_mapped, start_pfn, end_pfn);
+        nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_X_MAX);
+
+        max_pfn_mapped = max(max_pfn_mapped, end_pfn);
+
+        if (start_pfn < (1UL<<(32-PAGE_SHIFT)))
+                max_low_pfn_mapped = max(max_low_pfn_mapped,
+                                         min(end_pfn, 1UL<<(32-PAGE_SHIFT)));
+}
+
+bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn)
+{
+        int i;
+
+        for (i = 0; i < nr_pfn_mapped; i++)
+                if ((start_pfn >= pfn_mapped[i].start) &&
+                    (end_pfn <= pfn_mapped[i].end))
+                        return true;
+
+        return false;
+}
+
+/*
+ * Setup the direct mapping of the physical memory at PAGE_OFFSET.
+ * This runs before bootmem is initialized and gets pages directly from
+ * the physical memory. To access them they are temporarily mapped.
+ */
+unsigned long __init_refok init_memory_mapping(unsigned long start,
+                                               unsigned long end)
+{
+        struct map_range mr[NR_RANGE_MR];
+        unsigned long ret = 0;
+        int nr_range, i;
+
+        pr_info("init_memory_mapping: [mem %#010lx-%#010lx]\n",
+               start, end - 1);
+
+        memset(mr, 0, sizeof(mr));
+        nr_range = split_mem_range(mr, 0, start, end);
 
         for (i = 0; i < nr_range; i++)
                 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
                                                    mr[i].page_size_mask);
 
-#ifdef CONFIG_X86_32
-        early_ioremap_page_table_range_init();
+        add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT);
 
-        load_cr3(swapper_pg_dir);
-#endif
+        return ret >> PAGE_SHIFT;
+}
 
-        __flush_tlb_all();
+/*
+ * would have hole in the middle or ends, and only ram parts will be mapped.
+ */
+static unsigned long __init init_range_memory_mapping(
+                                           unsigned long r_start,
+                                           unsigned long r_end)
+{
+        unsigned long start_pfn, end_pfn;
+        unsigned long mapped_ram_size = 0;
+        int i;
 
-        /*
-         * Reserve the kernel pagetable pages we used (pgt_buf_start -
-         * pgt_buf_end) and free the other ones (pgt_buf_end - pgt_buf_top)
-         * so that they can be reused for other purposes.
-         *
-         * On native it just means calling memblock_reserve, on Xen it also
-         * means marking RW the pagetable pages that we allocated before
-         * but that haven't been used.
-         *
-         * In fact on xen we mark RO the whole range pgt_buf_start -
-         * pgt_buf_top, because we have to make sure that when
-         * init_memory_mapping reaches the pagetable pages area, it maps
-         * RO all the pagetable pages, including the ones that are beyond
-         * pgt_buf_end at that time.
-         */
-        if (!after_bootmem && pgt_buf_end > pgt_buf_start)
-                x86_init.mapping.pagetable_reserve(PFN_PHYS(pgt_buf_start),
-                                PFN_PHYS(pgt_buf_end));
+        for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
+                u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end);
+                u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end);
+                if (start >= end)
+                        continue;
 
-        if (!after_bootmem)
-                early_memtest(start, end);
+                /*
+                 * if it is overlapping with brk pgt, we need to
+                 * alloc pgt buf from memblock instead.
+                 */
+                can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >=
+                                    min(end, (u64)pgt_buf_top<<PAGE_SHIFT);
+                init_memory_mapping(start, end);
+                mapped_ram_size += end - start;
+                can_use_brk_pgt = true;
+        }
 
-        return ret >> PAGE_SHIFT;
+        return mapped_ram_size;
 }
 
+/* (PUD_SHIFT-PMD_SHIFT)/2 */
+#define STEP_SIZE_SHIFT 5
+void __init init_mem_mapping(void)
+{
+        unsigned long end, real_end, start, last_start;
+        unsigned long step_size;
+        unsigned long addr;
+        unsigned long mapped_ram_size = 0;
+        unsigned long new_mapped_ram_size;
+
+        probe_page_size_mask();
+
+#ifdef CONFIG_X86_64
+        end = max_pfn << PAGE_SHIFT;
+#else
+        end = max_low_pfn << PAGE_SHIFT;
+#endif
+
+        /* the ISA range is always mapped regardless of memory holes */
+        init_memory_mapping(0, ISA_END_ADDRESS);
+
+        /* xen has big range in reserved near end of ram, skip it at first */
+        addr = memblock_find_in_range(ISA_END_ADDRESS, end, PMD_SIZE,
+                         PAGE_SIZE);
+        real_end = addr + PMD_SIZE;
+
+        /* step_size need to be small so pgt_buf from BRK could cover it */
+        step_size = PMD_SIZE;
+        max_pfn_mapped = 0; /* will get exact value next */
+        min_pfn_mapped = real_end >> PAGE_SHIFT;
+        last_start = start = real_end;
+        while (last_start > ISA_END_ADDRESS) {
+                if (last_start > step_size) {
+                        start = round_down(last_start - 1, step_size);
+                        if (start < ISA_END_ADDRESS)
+                                start = ISA_END_ADDRESS;
+                } else
+                        start = ISA_END_ADDRESS;
+                new_mapped_ram_size = init_range_memory_mapping(start,
+                                                        last_start);
+                last_start = start;
+                min_pfn_mapped = last_start >> PAGE_SHIFT;
+                /* only increase step_size after big range get mapped */
+                if (new_mapped_ram_size > mapped_ram_size)
+                        step_size <<= STEP_SIZE_SHIFT;
+                mapped_ram_size += new_mapped_ram_size;
+        }
+
+        if (real_end < end)
+                init_range_memory_mapping(real_end, end);
+
+#ifdef CONFIG_X86_64
+        if (max_pfn > max_low_pfn) {
+                /* can we preseve max_low_pfn ?*/
+                max_low_pfn = max_pfn;
+        }
+#else
+        early_ioremap_page_table_range_init();
+#endif
+
+        load_cr3(swapper_pg_dir);
+        __flush_tlb_all();
+
+        early_memtest(0, max_pfn_mapped << PAGE_SHIFT);
+}
 
 /*
  * devmem_is_allowed() checks to see if /dev/mem access to a certain address