1 files changed, 82 insertions, 12 deletions

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index d5d4b04d48a..363751dc3fb 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -253,6 +253,43 @@ static __meminit void unmap_low_page(void *adr)
         early_iounmap(adr, PAGE_SIZE);
 }
 
+static void __meminit
+phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end)
+{
+        unsigned pages = 0;
+        int i;
+        pte_t *pte = pte_page + pte_index(addr);
+
+        for(i = pte_index(addr); i < PTRS_PER_PTE; i++, addr += PAGE_SIZE, pte++) {
+
+                if (addr >= end) {
+                        if (!after_bootmem) {
+                                for(; i < PTRS_PER_PTE; i++, pte++)
+                                        set_pte(pte, __pte(0));
+                        }
+                        break;
+                }
+
+                if (pte_val(*pte))
+                        continue;
+
+                if (0)
+                        printk("   pte=%p addr=%lx pte=%016lx\n",
+                               pte, addr, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL).pte);
+                set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL));
+                pages++;
+        }
+        update_page_count(PG_LEVEL_4K, pages);
+}
+
+static void __meminit
+phys_pte_update(pmd_t *pmd, unsigned long address, unsigned long end)
+{
+        pte_t *pte = (pte_t *)pmd_page_vaddr(*pmd);
+
+        phys_pte_init(pte, address, end);
+}
+
 static unsigned long __meminit
 phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end)
 {
@@ -261,7 +298,9 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end)
         int i = pmd_index(address);
 
         for (; i < PTRS_PER_PMD; i++, address += PMD_SIZE) {
+                unsigned long pte_phys;
                 pmd_t *pmd = pmd_page + pmd_index(address);
+                pte_t *pte;
 
                 if (address >= end) {
                         if (!after_bootmem) {
@@ -271,12 +310,23 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end)
                         break;
                 }
 
-                if (pmd_val(*pmd))
+                if (pmd_val(*pmd)) {
+                        phys_pte_update(pmd, address, end);
+                        continue;
+                }
+
+                if (cpu_has_pse) {
+                        pages++;
+                        set_pte((pte_t *)pmd,
+                                pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL_LARGE));
                         continue;
+                }
 
-                pages++;
-                set_pte((pte_t *)pmd,
-                        pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL_LARGE));
+                pte = alloc_low_page(&pte_phys);
+                phys_pte_init(pte, address, end);
+                unmap_low_page(pte);
+
+                pmd_populate_kernel(&init_mm, pmd, __va(pte_phys));
         }
         update_page_count(PG_LEVEL_2M, pages);
         return address;
@@ -333,11 +383,11 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end)
                 pmd = alloc_low_page(&pmd_phys);
 
                 spin_lock(&init_mm.page_table_lock);
-                pud_populate(&init_mm, pud, __va(pmd_phys));
                 last_map_addr = phys_pmd_init(pmd, addr, end);
+                unmap_low_page(pmd);
+                pud_populate(&init_mm, pud, __va(pmd_phys));
                 spin_unlock(&init_mm.page_table_lock);
 
-                unmap_low_page(pmd);
         }
         __flush_tlb_all();
         update_page_count(PG_LEVEL_1G, pages);
@@ -345,16 +395,30 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end)
         return last_map_addr;
 }
 
+static unsigned long __meminit
+phys_pud_update(pgd_t *pgd, unsigned long addr, unsigned long end)
+{
+        pud_t *pud;
+
+        pud = (pud_t *)pgd_page_vaddr(*pgd);
+
+        return phys_pud_init(pud, addr, end);
+}
+
 static void __init find_early_table_space(unsigned long end)
 {
-        unsigned long puds, pmds, tables, start;
+        unsigned long puds, tables, start;
 
         puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
         tables = round_up(puds * sizeof(pud_t), PAGE_SIZE);
         if (!direct_gbpages) {
-                pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
+                unsigned long pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
                 tables += round_up(pmds * sizeof(pmd_t), PAGE_SIZE);
         }
+        if (!cpu_has_pse) {
+                unsigned long ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;
+                tables += round_up(ptes * sizeof(pte_t), PAGE_SIZE);
+        }
 
         /*
          * RED-PEN putting page tables only on node 0 could
@@ -526,19 +590,25 @@ unsigned long __init_refok init_memory_mapping(unsigned long start, unsigned lon
                 unsigned long pud_phys;
                 pud_t *pud;
 
+                next = start + PGDIR_SIZE;
+                if (next > end)
+                        next = end;
+
+                if (pgd_val(*pgd)) {
+                        last_map_addr = phys_pud_update(pgd, __pa(start), __pa(end));
+                        continue;
+                }
+
                 if (after_bootmem)
                         pud = pud_offset(pgd, start & PGDIR_MASK);
                 else
                         pud = alloc_low_page(&pud_phys);
 
-                next = start + PGDIR_SIZE;
-                if (next > end)
-                        next = end;
                 last_map_addr = phys_pud_init(pud, __pa(start), __pa(next));
+                unmap_low_page(pud);
                 if (!after_bootmem)
                         pgd_populate(&init_mm, pgd_offset_k(start),
                                      __va(pud_phys));
-                unmap_low_page(pud);
         }
 
         if (!after_bootmem)