1 files changed, 123 insertions, 25 deletions

diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index 04664cdb7fda..f97130618113 100644
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -53,12 +53,12 @@ __ref void *alloc_low_pages(unsigned int num)
         if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) {
                 unsigned long ret;
                 if (min_pfn_mapped >= max_pfn_mapped)
-                        panic("alloc_low_page: ran out of memory");
+                        panic("alloc_low_pages: ran out of memory");
                 ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT,
                                         max_pfn_mapped << PAGE_SHIFT,
                                         PAGE_SIZE * num , PAGE_SIZE);
                 if (!ret)
-                        panic("alloc_low_page: can not alloc memory");
+                        panic("alloc_low_pages: can not alloc memory");
                 memblock_reserve(ret, PAGE_SIZE * num);
                 pfn = ret >> PAGE_SHIFT;
         } else {
@@ -399,29 +399,46 @@ static unsigned long __init init_range_memory_mapping(
         return mapped_ram_size;
 }
 
-/* (PUD_SHIFT-PMD_SHIFT)/2 */
-#define STEP_SIZE_SHIFT 5
-void __init init_mem_mapping(void)
+static unsigned long __init get_new_step_size(unsigned long step_size)
+{
+        /*
+         * Explain why we shift by 5 and why we don't have to worry about
+         * 'step_size << 5' overflowing:
+         *
+         * initial mapped size is PMD_SIZE (2M).
+         * We can not set step_size to be PUD_SIZE (1G) yet.
+         * In worse case, when we cross the 1G boundary, and
+         * PG_LEVEL_2M is not set, we will need 1+1+512 pages (2M + 8k)
+         * to map 1G range with PTE. Use 5 as shift for now.
+         *
+         * Don't need to worry about overflow, on 32bit, when step_size
+         * is 0, round_down() returns 0 for start, and that turns it
+         * into 0x100000000ULL.
+         */
+        return step_size << 5;
+}
+
+/**
+ * memory_map_top_down - Map [map_start, map_end) top down
+ * @map_start: start address of the target memory range
+ * @map_end: end address of the target memory range
+ *
+ * This function will setup direct mapping for memory range
+ * [map_start, map_end) in top-down. That said, the page tables
+ * will be allocated at the end of the memory, and we map the
+ * memory in top-down.
+ */
+static void __init memory_map_top_down(unsigned long map_start,
+                                       unsigned long map_end)
 {
-        unsigned long end, real_end, start, last_start;
+        unsigned long real_end, start, last_start;
         unsigned long step_size;
         unsigned long addr;
         unsigned long mapped_ram_size = 0;
         unsigned long new_mapped_ram_size;
 
-        probe_page_size_mask();
-
-#ifdef CONFIG_X86_64
-        end = max_pfn << PAGE_SHIFT;
-#else
-        end = max_low_pfn << PAGE_SHIFT;
-#endif
-
-        /* the ISA range is always mapped regardless of memory holes */
-        init_memory_mapping(0, ISA_END_ADDRESS);
-
         /* xen has big range in reserved near end of ram, skip it at first.*/
-        addr = memblock_find_in_range(ISA_END_ADDRESS, end, PMD_SIZE, PMD_SIZE);
+        addr = memblock_find_in_range(map_start, map_end, PMD_SIZE, PMD_SIZE);
         real_end = addr + PMD_SIZE;
 
         /* step_size need to be small so pgt_buf from BRK could cover it */
@@ -436,25 +453,106 @@ void __init init_mem_mapping(void)
          * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
          * for page table.
          */
-        while (last_start > ISA_END_ADDRESS) {
+        while (last_start > map_start) {
                 if (last_start > step_size) {
                         start = round_down(last_start - 1, step_size);
-                        if (start < ISA_END_ADDRESS)
-                                start = ISA_END_ADDRESS;
+                        if (start < map_start)
+                                start = map_start;
                 } else
-                        start = ISA_END_ADDRESS;
+                        start = map_start;
                 new_mapped_ram_size = init_range_memory_mapping(start,
                                                         last_start);
                 last_start = start;
                 min_pfn_mapped = last_start >> PAGE_SHIFT;
                 /* only increase step_size after big range get mapped */
                 if (new_mapped_ram_size > mapped_ram_size)
-                        step_size <<= STEP_SIZE_SHIFT;
+                        step_size = get_new_step_size(step_size);
                 mapped_ram_size += new_mapped_ram_size;
         }
 
-        if (real_end < end)
-                init_range_memory_mapping(real_end, end);
+        if (real_end < map_end)
+                init_range_memory_mapping(real_end, map_end);
+}
+
+/**
+ * memory_map_bottom_up - Map [map_start, map_end) bottom up
+ * @map_start: start address of the target memory range
+ * @map_end: end address of the target memory range
+ *
+ * This function will setup direct mapping for memory range
+ * [map_start, map_end) in bottom-up. Since we have limited the
+ * bottom-up allocation above the kernel, the page tables will
+ * be allocated just above the kernel and we map the memory
+ * in [map_start, map_end) in bottom-up.
+ */
+static void __init memory_map_bottom_up(unsigned long map_start,
+                                        unsigned long map_end)
+{
+        unsigned long next, new_mapped_ram_size, start;
+        unsigned long mapped_ram_size = 0;
+        /* step_size need to be small so pgt_buf from BRK could cover it */
+        unsigned long step_size = PMD_SIZE;
+
+        start = map_start;
+        min_pfn_mapped = start >> PAGE_SHIFT;
+
+        /*
+         * We start from the bottom (@map_start) and go to the top (@map_end).
+         * The memblock_find_in_range() gets us a block of RAM from the
+         * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
+         * for page table.
+         */
+        while (start < map_end) {
+                if (map_end - start > step_size) {
+                        next = round_up(start + 1, step_size);
+                        if (next > map_end)
+                                next = map_end;
+                } else
+                        next = map_end;
+
+                new_mapped_ram_size = init_range_memory_mapping(start, next);
+                start = next;
+
+                if (new_mapped_ram_size > mapped_ram_size)
+                        step_size = get_new_step_size(step_size);
+                mapped_ram_size += new_mapped_ram_size;
+        }
+}
+
+void __init init_mem_mapping(void)
+{
+        unsigned long end;
+
+        probe_page_size_mask();
+
+#ifdef CONFIG_X86_64
+        end = max_pfn << PAGE_SHIFT;
+#else
+        end = max_low_pfn << PAGE_SHIFT;
+#endif
+
+        /* the ISA range is always mapped regardless of memory holes */
+        init_memory_mapping(0, ISA_END_ADDRESS);
+
+        /*
+         * If the allocation is in bottom-up direction, we setup direct mapping
+         * in bottom-up, otherwise we setup direct mapping in top-down.
+         */
+        if (memblock_bottom_up()) {
+                unsigned long kernel_end = __pa_symbol(_end);
+
+                /*
+                 * we need two separate calls here. This is because we want to
+                 * allocate page tables above the kernel. So we first map
+                 * [kernel_end, end) to make memory above the kernel be mapped
+                 * as soon as possible. And then use page tables allocated above
+                 * the kernel to map [ISA_END_ADDRESS, kernel_end).
+                 */
+                memory_map_bottom_up(kernel_end, end);
+                memory_map_bottom_up(ISA_END_ADDRESS, kernel_end);
+        } else {
+                memory_map_top_down(ISA_END_ADDRESS, end);
+        }
 
 #ifdef CONFIG_X86_64
         if (max_pfn > max_low_pfn) {