9 files changed, 181 insertions, 113 deletions

diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index 6e8fdf5ad11..fd13815fe85 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -69,23 +69,37 @@ extern u64 efi_call6(void *fp, u64 arg1, u64 arg2, u64 arg3,
         efi_call6((void *)(f), (u64)(a1), (u64)(a2), (u64)(a3),         \
                   (u64)(a4), (u64)(a5), (u64)(a6))
 
+extern unsigned long efi_call_virt_prelog(void);
+extern void efi_call_virt_epilog(unsigned long);
+
+#define efi_callx(x, func, ...)                                 \
+        ({                                                      \
+                efi_status_t __status;                          \
+                unsigned long __pgd;                            \
+                                                                \
+                __pgd = efi_call_virt_prelog();                 \
+                __status = efi_call##x(func, __VA_ARGS__);      \
+                efi_call_virt_epilog(__pgd);                    \
+                __status;                                       \
+        })
+
 #define efi_call_virt0(f)                               \
-        efi_call0((void *)(efi.systab->runtime->f))
+        efi_callx(0, (void *)(efi.systab->runtime->f))
 #define efi_call_virt1(f, a1)                                   \
-        efi_call1((void *)(efi.systab->runtime->f), (u64)(a1))
+        efi_callx(1, (void *)(efi.systab->runtime->f), (u64)(a1))
 #define efi_call_virt2(f, a1, a2)                                       \
-        efi_call2((void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2))
+        efi_callx(2, (void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2))
 #define efi_call_virt3(f, a1, a2, a3)                                   \
-        efi_call3((void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
+        efi_callx(3, (void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
                   (u64)(a3))
 #define efi_call_virt4(f, a1, a2, a3, a4)                               \
-        efi_call4((void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
+        efi_callx(4, (void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
                   (u64)(a3), (u64)(a4))
 #define efi_call_virt5(f, a1, a2, a3, a4, a5)                           \
-        efi_call5((void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
+        efi_callx(5, (void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
                   (u64)(a3), (u64)(a4), (u64)(a5))
 #define efi_call_virt6(f, a1, a2, a3, a4, a5, a6)                       \
-        efi_call6((void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
+        efi_callx(6, (void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
                   (u64)(a3), (u64)(a4), (u64)(a5), (u64)(a6))
 
 extern void __iomem *efi_ioremap(unsigned long addr, unsigned long size,
diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c
index f84fe00fad4..d4f460f962e 100644
--- a/arch/x86/kernel/tboot.c
+++ b/arch/x86/kernel/tboot.c
@@ -103,71 +103,13 @@ void __init tboot_probe(void)
         pr_debug("tboot_size: 0x%x\n", tboot->tboot_size);
 }
 
-static pgd_t *tboot_pg_dir;
-static struct mm_struct tboot_mm = {
-        .mm_rb          = RB_ROOT,
-        .pgd            = swapper_pg_dir,
-        .mm_users       = ATOMIC_INIT(2),
-        .mm_count       = ATOMIC_INIT(1),
-        .mmap_sem       = __RWSEM_INITIALIZER(init_mm.mmap_sem),
-        .page_table_lock =  __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
-        .mmlist         = LIST_HEAD_INIT(init_mm.mmlist),
-};
-
 static inline void switch_to_tboot_pt(void)
 {
-        write_cr3(virt_to_phys(tboot_pg_dir));
-}
-
-static int map_tboot_page(unsigned long vaddr, unsigned long pfn,
-                          pgprot_t prot)
-{
-        pgd_t *pgd;
-        pud_t *pud;
-        pmd_t *pmd;
-        pte_t *pte;
-
-        pgd = pgd_offset(&tboot_mm, vaddr);
-        pud = pud_alloc(&tboot_mm, pgd, vaddr);
-        if (!pud)
-                return -1;
-        pmd = pmd_alloc(&tboot_mm, pud, vaddr);
-        if (!pmd)
-                return -1;
-        pte = pte_alloc_map(&tboot_mm, NULL, pmd, vaddr);
-        if (!pte)
-                return -1;
-        set_pte_at(&tboot_mm, vaddr, pte, pfn_pte(pfn, prot));
-        pte_unmap(pte);
-        return 0;
-}
-
-static int map_tboot_pages(unsigned long vaddr, unsigned long start_pfn,
-                           unsigned long nr)
-{
-        /* Reuse the original kernel mapping */
-        tboot_pg_dir = pgd_alloc(&tboot_mm);
-        if (!tboot_pg_dir)
-                return -1;
-
-        for (; nr > 0; nr--, vaddr += PAGE_SIZE, start_pfn++) {
-                if (map_tboot_page(vaddr, start_pfn, PAGE_KERNEL_EXEC))
-                        return -1;
-        }
-
-        return 0;
-}
-
-static void tboot_create_trampoline(void)
-{
-        u32 map_base, map_size;
-
-        /* Create identity map for tboot shutdown code. */
-        map_base = PFN_DOWN(tboot->tboot_base);
-        map_size = PFN_UP(tboot->tboot_size);
-        if (map_tboot_pages(map_base << PAGE_SHIFT, map_base, map_size))
-                panic("tboot: Error mapping tboot pages (mfns) @ 0x%x, 0x%x\n",
-                      map_base, map_size);
+#ifdef CONFIG_X86_32
+        load_cr3(initial_page_table);
+#else
+        write_cr3(real_mode_header->trampoline_pgd);
+#endif
 }
 
 #ifdef CONFIG_ACPI_SLEEP
@@ -225,14 +167,6 @@ void tboot_shutdown(u32 shutdown_type)
         if (!tboot_enabled())
                 return;
 
-        /*
-         * if we're being called before the 1:1 mapping is set up then just
-         * return and let the normal shutdown happen; this should only be
-         * due to very early panic()
-         */
-        if (!tboot_pg_dir)
-                return;
-
         /* if this is S3 then set regions to MAC */
         if (shutdown_type == TB_SHUTDOWN_S3)
                 if (tboot_setup_sleep())
@@ -343,8 +277,6 @@ static __init int tboot_late_init(void)
         if (!tboot_enabled())
                 return 0;
 
-        tboot_create_trampoline();
-
         atomic_set(&ap_wfs_count, 0);
         register_hotcpu_notifier(&tboot_cpu_notifier);
 
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 2ead3c8a4c8..07519a12044 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -108,13 +108,13 @@ void sync_global_pgds(unsigned long start, unsigned long end)
         for (address = start; address <= end; address += PGDIR_SIZE) {
                 const pgd_t *pgd_ref = pgd_offset_k(address);
                 struct page *page;
+                pgd_t *pgd;
 
                 if (pgd_none(*pgd_ref))
                         continue;
 
                 spin_lock(&pgd_lock);
                 list_for_each_entry(page, &pgd_list, lru) {
-                        pgd_t *pgd;
                         spinlock_t *pgt_lock;
 
                         pgd = (pgd_t *)page_address(page) + pgd_index(address);
@@ -130,6 +130,13 @@ void sync_global_pgds(unsigned long start, unsigned long end)
 
                         spin_unlock(pgt_lock);
                 }
+
+                pgd = __va(real_mode_header->trampoline_pgd);
+                pgd += pgd_index(address);
+
+                if (pgd_none(*pgd))
+                        set_pgd(pgd, *pgd_ref);
+
                 spin_unlock(&pgd_lock);
         }
 }
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index 78fe3f1ac49..e190f7b5665 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -50,6 +50,107 @@ int ioremap_change_attr(unsigned long vaddr, unsigned long size,
         return err;
 }
 
+#ifdef CONFIG_X86_64
+static void ident_pte_range(unsigned long paddr, unsigned long vaddr,
+                            pmd_t *ppmd, pmd_t *vpmd, unsigned long end)
+{
+        pte_t *ppte = pte_offset_kernel(ppmd, paddr);
+        pte_t *vpte = pte_offset_kernel(vpmd, vaddr);
+
+        do {
+                set_pte(ppte, *vpte);
+        } while (ppte++, vpte++, vaddr += PAGE_SIZE, vaddr != end);
+}
+
+static int ident_pmd_range(unsigned long paddr, unsigned long vaddr,
+                            pud_t *ppud, pud_t *vpud, unsigned long end)
+{
+        pmd_t *ppmd = pmd_offset(ppud, paddr);
+        pmd_t *vpmd = pmd_offset(vpud, vaddr);
+        unsigned long next;
+
+        do {
+                next = pmd_addr_end(vaddr, end);
+
+                if (!pmd_present(*ppmd)) {
+                        pte_t *ppte = (pte_t *)get_zeroed_page(GFP_KERNEL);
+                        if (!ppte)
+                                return 1;
+
+                        set_pmd(ppmd, __pmd(_KERNPG_TABLE | __pa(ppte)));
+                }
+
+                ident_pte_range(paddr, vaddr, ppmd, vpmd, next);
+        } while (ppmd++, vpmd++, vaddr = next, vaddr != end);
+
+        return 0;
+}
+
+static int ident_pud_range(unsigned long paddr, unsigned long vaddr,
+                            pgd_t *ppgd, pgd_t *vpgd, unsigned long end)
+{
+        pud_t *ppud = pud_offset(ppgd, paddr);
+        pud_t *vpud = pud_offset(vpgd, vaddr);
+        unsigned long next;
+
+        do {
+                next = pud_addr_end(vaddr, end);
+
+                if (!pud_present(*ppud)) {
+                        pmd_t *ppmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
+                        if (!ppmd)
+                                return 1;
+
+                        set_pud(ppud, __pud(_KERNPG_TABLE | __pa(ppmd)));
+                }
+
+                if (ident_pmd_range(paddr, vaddr, ppud, vpud, next))
+                        return 1;
+        } while (ppud++, vpud++, vaddr = next, vaddr != end);
+
+        return 0;
+}
+
+static int insert_identity_mapping(resource_size_t paddr, unsigned long vaddr,
+                                    unsigned long size)
+{
+        unsigned long end = vaddr + size;
+        unsigned long next;
+        pgd_t *vpgd, *ppgd;
+
+        /* Don't map over the guard hole. */
+        if (paddr >= 0x800000000000 || paddr + size > 0x800000000000)
+                return 1;
+
+        ppgd = __va(real_mode_header->trampoline_pgd) + pgd_index(paddr);
+
+        vpgd = pgd_offset_k(vaddr);
+        do {
+                next = pgd_addr_end(vaddr, end);
+
+                if (!pgd_present(*ppgd)) {
+                        pud_t *ppud = (pud_t *)get_zeroed_page(GFP_KERNEL);
+                        if (!ppud)
+                                return 1;
+
+                        set_pgd(ppgd, __pgd(_KERNPG_TABLE | __pa(ppud)));
+                }
+
+                if (ident_pud_range(paddr, vaddr, ppgd, vpgd, next))
+                        return 1;
+        } while (ppgd++, vpgd++, vaddr = next, vaddr != end);
+
+        return 0;
+}
+#else
+static inline int insert_identity_mapping(resource_size_t paddr,
+                                          unsigned long vaddr,
+                                          unsigned long size)
+{
+        return 0;
+}
+#endif /* CONFIG_X86_64 */
+
 /*
  * Remap an arbitrary physical address space into the kernel virtual
  * address space. Needed when the kernel wants to access high addresses
@@ -163,6 +264,10 @@ static void __iomem *__ioremap_caller(resource_size_t phys_addr,
         ret_addr = (void __iomem *) (vaddr + offset);
         mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
 
+        if (insert_identity_mapping(phys_addr, vaddr, size))
+                printk(KERN_WARNING "ioremap: unable to map 0x%llx in identity pagetable\n",
+                                        (unsigned long long)phys_addr);
+
         /*
          * Check if the request spans more than any BAR in the iomem resource
          * tree.
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index a718e0d2350..931930a9616 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -919,11 +919,13 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
 
         /*
          * On success we use clflush, when the CPU supports it to
-         * avoid the wbindv. If the CPU does not support it and in the
-         * error case we fall back to cpa_flush_all (which uses
-         * wbindv):
+         * avoid the wbindv. If the CPU does not support it, in the
+         * error case, and during early boot (for EFI) we fall back
+         * to cpa_flush_all (which uses wbinvd):
          */
-        if (!ret && cpu_has_clflush) {
+        if (early_boot_irqs_disabled)
+                __cpa_flush_all((void *)(long)cache);
+        else if (!ret && cpu_has_clflush) {
                 if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
                         cpa_flush_array(addr, numpages, cache,
                                         cpa.flags, pages);
diff --git a/arch/x86/platform/efi/efi-bgrt.c b/arch/x86/platform/efi/efi-bgrt.c
index f6a0c1b8e51..d9c1b95af17 100644
--- a/arch/x86/platform/efi/efi-bgrt.c
+++ b/arch/x86/platform/efi/efi-bgrt.c
@@ -39,6 +39,8 @@ void efi_bgrt_init(void)
         if (ACPI_FAILURE(status))
                 return;
 
+        if (bgrt_tab->header.length < sizeof(*bgrt_tab))
+                return;
         if (bgrt_tab->version != 1)
                 return;
         if (bgrt_tab->image_type != 0 || !bgrt_tab->image_address)
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index ad4439145f8..0a34d9e9c26 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -239,22 +239,7 @@ static efi_status_t __init phys_efi_set_virtual_address_map(
         return status;
 }
 
-static efi_status_t __init phys_efi_get_time(efi_time_t *tm,
-                                             efi_time_cap_t *tc)
-{
-        unsigned long flags;
-        efi_status_t status;
-
-        spin_lock_irqsave(&rtc_lock, flags);
-        efi_call_phys_prelog();
-        status = efi_call_phys2(efi_phys.get_time, virt_to_phys(tm),
-                                virt_to_phys(tc));
-        efi_call_phys_epilog();
-        spin_unlock_irqrestore(&rtc_lock, flags);
-        return status;
-}
-
-int efi_set_rtc_mmss(unsigned long nowtime)
+static int efi_set_rtc_mmss(unsigned long nowtime)
 {
         int real_seconds, real_minutes;
         efi_status_t    status;
@@ -283,7 +268,7 @@ int efi_set_rtc_mmss(unsigned long nowtime)
         return 0;
 }
 
-unsigned long efi_get_time(void)
+static unsigned long efi_get_time(void)
 {
         efi_status_t status;
         efi_time_t eft;
@@ -639,18 +624,13 @@ static int __init efi_runtime_init(void)
         }
         /*
          * We will only need *early* access to the following
-         * two EFI runtime services before set_virtual_address_map
+         * EFI runtime service before set_virtual_address_map
          * is invoked.
          */
-        efi_phys.get_time = (efi_get_time_t *)runtime->get_time;
         efi_phys.set_virtual_address_map =
                 (efi_set_virtual_address_map_t *)
                 runtime->set_virtual_address_map;
-        /*
-         * Make efi_get_time can be called before entering
-         * virtual mode.
-         */
-        efi.get_time = phys_efi_get_time;
+
         early_iounmap(runtime, sizeof(efi_runtime_services_t));
 
         return 0;
@@ -736,12 +716,10 @@ void __init efi_init(void)
                 efi_enabled = 0;
                 return;
         }
-#ifdef CONFIG_X86_32
         if (efi_is_native()) {
                 x86_platform.get_wallclock = efi_get_time;
                 x86_platform.set_wallclock = efi_set_rtc_mmss;
         }
-#endif
 
 #if EFI_DEBUG
         print_efi_memmap();
diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c
index 95fd505dfeb..06c8b2e662a 100644
--- a/arch/x86/platform/efi/efi_64.c
+++ b/arch/x86/platform/efi/efi_64.c
@@ -58,6 +58,21 @@ static void __init early_code_mapping_set_exec(int executable)
         }
 }
 
+unsigned long efi_call_virt_prelog(void)
+{
+        unsigned long saved;
+
+        saved = read_cr3();
+        write_cr3(real_mode_header->trampoline_pgd);
+
+        return saved;
+}
+
+void efi_call_virt_epilog(unsigned long saved)
+{
+        write_cr3(saved);
+}
+
 void __init efi_call_phys_prelog(void)
 {
         unsigned long vaddress;
diff --git a/arch/x86/realmode/init.c b/arch/x86/realmode/init.c
index cbca565af5b..8e6ab613785 100644
--- a/arch/x86/realmode/init.c
+++ b/arch/x86/realmode/init.c
@@ -78,8 +78,21 @@ void __init setup_real_mode(void)
         *trampoline_cr4_features = read_cr4();
 
         trampoline_pgd = (u64 *) __va(real_mode_header->trampoline_pgd);
-        trampoline_pgd[0] = __pa(level3_ident_pgt) + _KERNPG_TABLE;
-        trampoline_pgd[511] = __pa(level3_kernel_pgt) + _KERNPG_TABLE;
+
+        /*
+         * Create an identity mapping for all of physical memory.
+         */
+        for (i = 0; i <= pgd_index(max_pfn << PAGE_SHIFT); i++) {
+                int index = pgd_index(PAGE_OFFSET) + i;
+
+                trampoline_pgd[i] = (u64)pgd_val(swapper_pg_dir[index]);
+        }
+
+        /*
+         * Copy the upper-half of the kernel pages tables.
+         */
+        for (i = pgd_index(PAGE_OFFSET); i < PTRS_PER_PGD; i++)
+                trampoline_pgd[i] = (u64)pgd_val(swapper_pg_dir[i]);
 #endif
 }