Merge remote-tracking branch 'tip/x86/efi-mixed' into efi-for-mingo

Conflicts:
	arch/x86/kernel/setup.c
	arch/x86/platform/efi/efi.c
	arch/x86/platform/efi/efi_64.c

5 files changed, 651 insertions, 57 deletions

diff --git a/arch/x86/platform/efi/Makefile b/arch/x86/platform/efi/Makefile
index b7b0b35c1981..d51045afcaaf 100644
--- a/arch/x86/platform/efi/Makefile
+++ b/arch/x86/platform/efi/Makefile
@@ -1,3 +1,4 @@
 obj-$(CONFIG_EFI)               += efi.o efi_$(BITS).o efi_stub_$(BITS).o
 obj-$(CONFIG_ACPI_BGRT) += efi-bgrt.o
 obj-$(CONFIG_EARLY_PRINTK_EFI)  += early_printk.o
+obj-$(CONFIG_EFI_MIXED)         += efi_thunk_$(BITS).o
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index 45d4f7674678..43e7cf6c6111 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -453,9 +453,6 @@ void __init efi_free_boot_services(void)
 {
         void *p;
 
-        if (!efi_is_native())
-                return;
-
         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
                 efi_memory_desc_t *md = p;
                 unsigned long long start = md->phys_addr;
@@ -579,37 +576,85 @@ static int __init efi_systab_init(void *phys)
         return 0;
 }
 
-static int __init efi_runtime_init(void)
+static int __init efi_runtime_init32(void)
 {
-        efi_runtime_services_t *runtime;
+        efi_runtime_services_32_t *runtime;
+
+        runtime = early_ioremap((unsigned long)efi.systab->runtime,
+                        sizeof(efi_runtime_services_32_t));
+        if (!runtime) {
+                pr_err("Could not map the runtime service table!\n");
+                return -ENOMEM;
+        }
 
         /*
-         * Check out the runtime services table. We need to map
-         * the runtime services table so that we can grab the physical
-         * address of several of the EFI runtime functions, needed to
-         * set the firmware into virtual mode.
+         * We will only need *early* access to the following two
+         * EFI runtime services before set_virtual_address_map
+         * is invoked.
+         */
+        efi_phys.get_time = (efi_get_time_t *)
+                        (unsigned long)runtime->get_time;
+        efi_phys.set_virtual_address_map =
+                        (efi_set_virtual_address_map_t *)
+                        (unsigned long)runtime->set_virtual_address_map;
+        /*
+         * Make efi_get_time can be called before entering
+         * virtual mode.
          */
+        efi.get_time = phys_efi_get_time;
+        early_iounmap(runtime, sizeof(efi_runtime_services_32_t));
+
+        return 0;
+}
+
+static int __init efi_runtime_init64(void)
+{
+        efi_runtime_services_64_t *runtime;
+
         runtime = early_ioremap((unsigned long)efi.systab->runtime,
-                                sizeof(efi_runtime_services_t));
+                        sizeof(efi_runtime_services_64_t));
         if (!runtime) {
                 pr_err("Could not map the runtime service table!\n");
                 return -ENOMEM;
         }
+
         /*
-         * We will only need *early* access to the following
-         * two EFI runtime services before set_virtual_address_map
+         * We will only need *early* access to the following two
+         * EFI runtime services before set_virtual_address_map
          * is invoked.
          */
-        efi_phys.get_time = (efi_get_time_t *)runtime->get_time;
+        efi_phys.get_time = (efi_get_time_t *)
+                        (unsigned long)runtime->get_time;
         efi_phys.set_virtual_address_map =
-                (efi_set_virtual_address_map_t *)
-                runtime->set_virtual_address_map;
+                        (efi_set_virtual_address_map_t *)
+                        (unsigned long)runtime->set_virtual_address_map;
         /*
          * Make efi_get_time can be called before entering
          * virtual mode.
          */
         efi.get_time = phys_efi_get_time;
-        early_iounmap(runtime, sizeof(efi_runtime_services_t));
+        early_iounmap(runtime, sizeof(efi_runtime_services_64_t));
+
+        return 0;
+}
+
+static int __init efi_runtime_init(void)
+{
+        int rv;
+
+        /*
+         * Check out the runtime services table. We need to map
+         * the runtime services table so that we can grab the physical
+         * address of several of the EFI runtime functions, needed to
+         * set the firmware into virtual mode.
+         */
+        if (efi_enabled(EFI_64BIT))
+                rv = efi_runtime_init64();
+        else
+                rv = efi_runtime_init32();
+
+        if (rv)
+                return rv;
 
         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 
@@ -747,7 +792,7 @@ void __init efi_init(void)
          * that doesn't match the kernel 32/64-bit mode.
          */
 
-        if (!efi_is_native())
+        if (!efi_runtime_supported())
                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
         else {
                 if (disable_runtime || efi_runtime_init())
@@ -833,6 +878,22 @@ void __init old_map_region(efi_memory_desc_t *md)
                        (unsigned long long)md->phys_addr);
 }
 
+static void native_runtime_setup(void)
+{
+        efi.get_time = virt_efi_get_time;
+        efi.set_time = virt_efi_set_time;
+        efi.get_wakeup_time = virt_efi_get_wakeup_time;
+        efi.set_wakeup_time = virt_efi_set_wakeup_time;
+        efi.get_variable = virt_efi_get_variable;
+        efi.get_next_variable = virt_efi_get_next_variable;
+        efi.set_variable = virt_efi_set_variable;
+        efi.get_next_high_mono_count = virt_efi_get_next_high_mono_count;
+        efi.reset_system = virt_efi_reset_system;
+        efi.query_variable_info = virt_efi_query_variable_info;
+        efi.update_capsule = virt_efi_update_capsule;
+        efi.query_capsule_caps = virt_efi_query_capsule_caps;
+}
+
 /* Merge contiguous regions of the same type and attribute */
 static void __init efi_merge_regions(void)
 {
@@ -1015,19 +1076,10 @@ static void __init kexec_enter_virtual_mode(void)
          * Call EFI services through wrapper functions.
          */
         efi.runtime_version = efi_systab.hdr.revision;
-        efi.get_time = virt_efi_get_time;
-        efi.set_time = virt_efi_set_time;
-        efi.get_wakeup_time = virt_efi_get_wakeup_time;
-        efi.set_wakeup_time = virt_efi_set_wakeup_time;
-        efi.get_variable = virt_efi_get_variable;
-        efi.get_next_variable = virt_efi_get_next_variable;
-        efi.set_variable = virt_efi_set_variable;
-        efi.get_next_high_mono_count = virt_efi_get_next_high_mono_count;
-        efi.reset_system = virt_efi_reset_system;
+
+        native_runtime_setup();
+
         efi.set_virtual_address_map = NULL;
-        efi.query_variable_info = virt_efi_query_variable_info;
-        efi.update_capsule = virt_efi_update_capsule;
-        efi.query_capsule_caps = virt_efi_query_capsule_caps;
 
         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
                 runtime_code_page_mkexec();
@@ -1071,15 +1123,6 @@ static void __init __efi_enter_virtual_mode(void)
 
         efi.systab = NULL;
 
-        /*
-         * We don't do virtual mode, since we don't do runtime services, on
-         * non-native EFI
-         */
-        if (!efi_is_native()) {
-                efi_unmap_memmap();
-                return;
-        }
-
         efi_merge_regions();
         new_memmap = efi_map_regions(&count, &pg_shift);
         if (!new_memmap) {
@@ -1097,11 +1140,20 @@ static void __init __efi_enter_virtual_mode(void)
         efi_sync_low_kernel_mappings();
         efi_dump_pagetable();
 
-        status = phys_efi_set_virtual_address_map(
-                        memmap.desc_size * count,
-                        memmap.desc_size,
-                        memmap.desc_version,
-                        (efi_memory_desc_t *)__pa(new_memmap));
+        if (efi_is_native()) {
+                status = phys_efi_set_virtual_address_map(
+                                memmap.desc_size * count,
+                                memmap.desc_size,
+                                memmap.desc_version,
+                                (efi_memory_desc_t *)__pa(new_memmap));
+        } else {
+                status = efi_thunk_set_virtual_address_map(
+                                efi_phys.set_virtual_address_map,
+                                memmap.desc_size * count,
+                                memmap.desc_size,
+                                memmap.desc_version,
+                                (efi_memory_desc_t *)__pa(new_memmap));
+        }
 
         if (status != EFI_SUCCESS) {
                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
@@ -1116,19 +1168,13 @@ static void __init __efi_enter_virtual_mode(void)
          * Call EFI services through wrapper functions.
          */
         efi.runtime_version = efi_systab.hdr.revision;
-        efi.get_time = virt_efi_get_time;
-        efi.set_time = virt_efi_set_time;
-        efi.get_wakeup_time = virt_efi_get_wakeup_time;
-        efi.set_wakeup_time = virt_efi_set_wakeup_time;
-        efi.get_variable = virt_efi_get_variable;
-        efi.get_next_variable = virt_efi_get_next_variable;
-        efi.set_variable = virt_efi_set_variable;
-        efi.get_next_high_mono_count = virt_efi_get_next_high_mono_count;
-        efi.reset_system = virt_efi_reset_system;
+
+        if (efi_is_native())
+                native_runtime_setup();
+        else
+                efi_thunk_runtime_setup();
+
         efi.set_virtual_address_map = NULL;
-        efi.query_variable_info = virt_efi_query_variable_info;
-        efi.update_capsule = virt_efi_update_capsule;
-        efi.query_capsule_caps = virt_efi_query_capsule_caps;
 
         efi_runtime_mkexec();
 
@@ -1311,7 +1357,7 @@ void __init efi_apply_memmap_quirks(void)
          * firmware/kernel architectures since there is no support for runtime
          * services.
          */
-        if (!efi_is_native()) {
+        if (!efi_runtime_supported()) {
                 pr_info("efi: Setup done, disabling due to 32/64-bit mismatch\n");
                 efi_unmap_memmap();
         }
diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c
index 19280900ec25..7e7f195aa5cf 100644
--- a/arch/x86/platform/efi/efi_64.c
+++ b/arch/x86/platform/efi/efi_64.c
@@ -39,6 +39,7 @@
 #include <asm/cacheflush.h>
 #include <asm/fixmap.h>
 #include <asm/realmode.h>
+#include <asm/time.h>
 
 static pgd_t *save_pgd __initdata;
 static unsigned long efi_flags __initdata;
@@ -58,7 +59,8 @@ struct efi_scratch {
         u64 prev_cr3;
         pgd_t *efi_pgt;
         bool use_pgd;
-};
+        u64 phys_stack;
+} __packed;
 
 static void __init early_code_mapping_set_exec(int executable)
 {
@@ -139,6 +141,9 @@ void efi_sync_low_kernel_mappings(void)
 
 int efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages)
 {
+        unsigned long text;
+        struct page *page;
+        unsigned npages;
         pgd_t *pgd;
 
         if (efi_enabled(EFI_OLD_MEMMAP))
@@ -160,6 +165,29 @@ int efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages)
 
         efi_scratch.use_pgd = true;
 
+        /*
+         * When making calls to the firmware everything needs to be 1:1
+         * mapped and addressable with 32-bit pointers. Map the kernel
+         * text and allocate a new stack because we can't rely on the
+         * stack pointer being < 4GB.
+         */
+        if (!IS_ENABLED(CONFIG_EFI_MIXED))
+                return 0;
+
+        page = alloc_page(GFP_KERNEL|__GFP_DMA32);
+        if (!page)
+                panic("Unable to allocate EFI runtime stack < 4GB\n");
+
+        efi_scratch.phys_stack = virt_to_phys(page_address(page));
+        efi_scratch.phys_stack += PAGE_SIZE; /* stack grows down */
+
+        npages = (_end - _text) >> PAGE_SHIFT;
+        text = __pa(_text);
+
+        if (kernel_map_pages_in_pgd(pgd, text >> PAGE_SHIFT, text, npages, 0)) {
+                pr_err("Failed to map kernel text 1:1\n");
+                return 1;
+        }
 
         return 0;
 }
@@ -199,6 +227,16 @@ void __init efi_map_region(efi_memory_desc_t *md)
          */
         __map_region(md, md->phys_addr);
 
+        /*
+         * Enforce the 1:1 mapping as the default virtual address when
+         * booting in EFI mixed mode, because even though we may be
+         * running a 64-bit kernel, the firmware may only be 32-bit.
+         */
+        if (!efi_is_native () && IS_ENABLED(CONFIG_EFI_MIXED)) {
+                md->virt_addr = md->phys_addr;
+                return;
+        }
+
         efi_va -= size;
 
         /* Is PA 2M-aligned? */
@@ -277,3 +315,290 @@ void __init efi_dump_pagetable(void)
         ptdump_walk_pgd_level(NULL, pgd);
 #endif
 }
+
+#ifdef CONFIG_EFI_MIXED
+extern efi_status_t efi64_thunk(u32, ...);
+
+#define runtime_service32(func)                                          \
+({                                                                       \
+        u32 table = (u32)(unsigned long)efi.systab;                      \
+        u32 *rt, *___f;                                                  \
+                                                                         \
+        rt = (u32 *)(table + offsetof(efi_system_table_32_t, runtime));  \
+        ___f = (u32 *)(*rt + offsetof(efi_runtime_services_32_t, func)); \
+        *___f;                                                           \
+})
+
+/*
+ * Switch to the EFI page tables early so that we can access the 1:1
+ * runtime services mappings which are not mapped in any other page
+ * tables. This function must be called before runtime_service32().
+ *
+ * Also, disable interrupts because the IDT points to 64-bit handlers,
+ * which aren't going to function correctly when we switch to 32-bit.
+ */
+#define efi_thunk(f, ...)                                               \
+({                                                                      \
+        efi_status_t __s;                                               \
+        unsigned long flags;                                            \
+        u32 func;                                                       \
+                                                                        \
+        efi_sync_low_kernel_mappings();                                 \
+        local_irq_save(flags);                                          \
+                                                                        \
+        efi_scratch.prev_cr3 = read_cr3();                              \
+        write_cr3((unsigned long)efi_scratch.efi_pgt);                  \
+        __flush_tlb_all();                                              \
+                                                                        \
+        func = runtime_service32(f);                                    \
+        __s = efi64_thunk(func, __VA_ARGS__);                   \
+                                                                        \
+        write_cr3(efi_scratch.prev_cr3);                                \
+        __flush_tlb_all();                                              \
+        local_irq_restore(flags);                                       \
+                                                                        \
+        __s;                                                            \
+})
+
+efi_status_t efi_thunk_set_virtual_address_map(
+        void *phys_set_virtual_address_map,
+        unsigned long memory_map_size,
+        unsigned long descriptor_size,
+        u32 descriptor_version,
+        efi_memory_desc_t *virtual_map)
+{
+        efi_status_t status;
+        unsigned long flags;
+        u32 func;
+
+        efi_sync_low_kernel_mappings();
+        local_irq_save(flags);
+
+        efi_scratch.prev_cr3 = read_cr3();
+        write_cr3((unsigned long)efi_scratch.efi_pgt);
+        __flush_tlb_all();
+
+        func = (u32)(unsigned long)phys_set_virtual_address_map;
+        status = efi64_thunk(func, memory_map_size, descriptor_size,
+                             descriptor_version, virtual_map);
+
+        write_cr3(efi_scratch.prev_cr3);
+        __flush_tlb_all();
+        local_irq_restore(flags);
+
+        return status;
+}
+
+static efi_status_t efi_thunk_get_time(efi_time_t *tm, efi_time_cap_t *tc)
+{
+        efi_status_t status;
+        u32 phys_tm, phys_tc;
+
+        spin_lock(&rtc_lock);
+
+        phys_tm = virt_to_phys(tm);
+        phys_tc = virt_to_phys(tc);
+
+        status = efi_thunk(get_time, phys_tm, phys_tc);
+
+        spin_unlock(&rtc_lock);
+
+        return status;
+}
+
+static efi_status_t efi_thunk_set_time(efi_time_t *tm)
+{
+        efi_status_t status;
+        u32 phys_tm;
+
+        spin_lock(&rtc_lock);
+
+        phys_tm = virt_to_phys(tm);
+
+        status = efi_thunk(set_time, phys_tm);
+
+        spin_unlock(&rtc_lock);
+
+        return status;
+}
+
+static efi_status_t
+efi_thunk_get_wakeup_time(efi_bool_t *enabled, efi_bool_t *pending,
+                          efi_time_t *tm)
+{
+        efi_status_t status;
+        u32 phys_enabled, phys_pending, phys_tm;
+
+        spin_lock(&rtc_lock);
+
+        phys_enabled = virt_to_phys(enabled);
+        phys_pending = virt_to_phys(pending);
+        phys_tm = virt_to_phys(tm);
+
+        status = efi_thunk(get_wakeup_time, phys_enabled,
+                             phys_pending, phys_tm);
+
+        spin_unlock(&rtc_lock);
+
+        return status;
+}
+
+static efi_status_t
+efi_thunk_set_wakeup_time(efi_bool_t enabled, efi_time_t *tm)
+{
+        efi_status_t status;
+        u32 phys_tm;
+
+        spin_lock(&rtc_lock);
+
+        phys_tm = virt_to_phys(tm);
+
+        status = efi_thunk(set_wakeup_time, enabled, phys_tm);
+
+        spin_unlock(&rtc_lock);
+
+        return status;
+}
+
+
+static efi_status_t
+efi_thunk_get_variable(efi_char16_t *name, efi_guid_t *vendor,
+                       u32 *attr, unsigned long *data_size, void *data)
+{
+        efi_status_t status;
+        u32 phys_name, phys_vendor, phys_attr;
+        u32 phys_data_size, phys_data;
+
+        phys_data_size = virt_to_phys(data_size);
+        phys_vendor = virt_to_phys(vendor);
+        phys_name = virt_to_phys(name);
+        phys_attr = virt_to_phys(attr);
+        phys_data = virt_to_phys(data);
+
+        status = efi_thunk(get_variable, phys_name, phys_vendor,
+                           phys_attr, phys_data_size, phys_data);
+
+        return status;
+}
+
+static efi_status_t
+efi_thunk_set_variable(efi_char16_t *name, efi_guid_t *vendor,
+                       u32 attr, unsigned long data_size, void *data)
+{
+        u32 phys_name, phys_vendor, phys_data;
+        efi_status_t status;
+
+        phys_name = virt_to_phys(name);
+        phys_vendor = virt_to_phys(vendor);
+        phys_data = virt_to_phys(data);
+
+        /* If data_size is > sizeof(u32) we've got problems */
+        status = efi_thunk(set_variable, phys_name, phys_vendor,
+                           attr, data_size, phys_data);
+
+        return status;
+}
+
+static efi_status_t
+efi_thunk_get_next_variable(unsigned long *name_size,
+                            efi_char16_t *name,
+                            efi_guid_t *vendor)
+{
+        efi_status_t status;
+        u32 phys_name_size, phys_name, phys_vendor;
+
+        phys_name_size = virt_to_phys(name_size);
+        phys_vendor = virt_to_phys(vendor);
+        phys_name = virt_to_phys(name);
+
+        status = efi_thunk(get_next_variable, phys_name_size,
+                           phys_name, phys_vendor);
+
+        return status;
+}
+
+static efi_status_t
+efi_thunk_get_next_high_mono_count(u32 *count)
+{
+        efi_status_t status;
+        u32 phys_count;
+
+        phys_count = virt_to_phys(count);
+        status = efi_thunk(get_next_high_mono_count, phys_count);
+
+        return status;
+}
+
+static void
+efi_thunk_reset_system(int reset_type, efi_status_t status,
+                       unsigned long data_size, efi_char16_t *data)
+{
+        u32 phys_data;
+
+        phys_data = virt_to_phys(data);
+
+        efi_thunk(reset_system, reset_type, status, data_size, phys_data);
+}
+
+static efi_status_t
+efi_thunk_update_capsule(efi_capsule_header_t **capsules,
+                         unsigned long count, unsigned long sg_list)
+{
+        /*
+         * To properly support this function we would need to repackage
+         * 'capsules' because the firmware doesn't understand 64-bit
+         * pointers.
+         */
+        return EFI_UNSUPPORTED;
+}
+
+static efi_status_t
+efi_thunk_query_variable_info(u32 attr, u64 *storage_space,
+                              u64 *remaining_space,
+                              u64 *max_variable_size)
+{
+        efi_status_t status;
+        u32 phys_storage, phys_remaining, phys_max;
+
+        if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION)
+                return EFI_UNSUPPORTED;
+
+        phys_storage = virt_to_phys(storage_space);
+        phys_remaining = virt_to_phys(remaining_space);
+        phys_max = virt_to_phys(max_variable_size);
+
+        status = efi_thunk(query_variable_info, phys_storage,
+                           phys_remaining, phys_max);
+
+        return status;
+}
+
+static efi_status_t
+efi_thunk_query_capsule_caps(efi_capsule_header_t **capsules,
+                             unsigned long count, u64 *max_size,
+                             int *reset_type)
+{
+        /*
+         * To properly support this function we would need to repackage
+         * 'capsules' because the firmware doesn't understand 64-bit
+         * pointers.
+         */
+        return EFI_UNSUPPORTED;
+}
+
+void efi_thunk_runtime_setup(void)
+{
+        efi.get_time = efi_thunk_get_time;
+        efi.set_time = efi_thunk_set_time;
+        efi.get_wakeup_time = efi_thunk_get_wakeup_time;
+        efi.set_wakeup_time = efi_thunk_set_wakeup_time;
+        efi.get_variable = efi_thunk_get_variable;
+        efi.get_next_variable = efi_thunk_get_next_variable;
+        efi.set_variable = efi_thunk_set_variable;
+        efi.get_next_high_mono_count = efi_thunk_get_next_high_mono_count;
+        efi.reset_system = efi_thunk_reset_system;
+        efi.query_variable_info = efi_thunk_query_variable_info;
+        efi.update_capsule = efi_thunk_update_capsule;
+        efi.query_capsule_caps = efi_thunk_query_capsule_caps;
+}
+#endif /* CONFIG_EFI_MIXED */
diff --git a/arch/x86/platform/efi/efi_stub_64.S b/arch/x86/platform/efi/efi_stub_64.S
index 88073b140298..65b787a9fc4e 100644
--- a/arch/x86/platform/efi/efi_stub_64.S
+++ b/arch/x86/platform/efi/efi_stub_64.S
@@ -7,6 +7,10 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/segment.h>
+#include <asm/msr.h>
+#include <asm/processor-flags.h>
+#include <asm/page_types.h>
 
 #define SAVE_XMM                        \
         mov %rsp, %rax;                 \
@@ -164,7 +168,160 @@ ENTRY(efi_call6)
         ret
 ENDPROC(efi_call6)
 
+#ifdef CONFIG_EFI_MIXED
+
+/*
+ * We run this function from the 1:1 mapping.
+ *
+ * This function must be invoked with a 1:1 mapped stack.
+ */
+ENTRY(__efi64_thunk)
+        subq    $32, %rsp
+        movl    %esi, 0x0(%rsp)
+        movl    %edx, 0x4(%rsp)
+        movl    %ecx, 0x8(%rsp)
+        movq    %r8, %rsi
+        movl    %esi, 0xc(%rsp)
+        movq    %r9, %rsi
+        movl    %esi,  0x10(%rsp)
+
+        sgdt    save_gdt(%rip)
+
+        leaq    1f(%rip), %rbx
+        movq    %rbx, func_rt_ptr(%rip)
+
+        /* Switch to gdt with 32-bit segments */
+        movl    40(%rsp), %eax
+        lgdt    (%rax)
+
+        leaq    efi_enter32(%rip), %rax
+        pushq   $__KERNEL_CS
+        pushq   %rax
+        lretq
+
+1:      addq    $32, %rsp
+
+        lgdt    save_gdt(%rip)
+
+        /*
+         * Convert 32-bit status code into 64-bit.
+         */
+        test    %rax, %rax
+        jz      1f
+        movl    %eax, %ecx
+        andl    $0x0fffffff, %ecx
+        andl    $0xf0000000, %eax
+        shl     $32, %rax
+        or      %rcx, %rax
+1:
+        ret
+ENDPROC(__efi64_thunk)
+
+ENTRY(efi_exit32)
+        xorq    %rax, %rax
+        movl    %eax, %ds
+        movl    %eax, %es
+        movl    %eax, %ss
+
+        movq    func_rt_ptr(%rip), %rax
+        push    %rax
+        mov     %rdi, %rax
+        ret
+ENDPROC(efi_exit32)
+
+        .code32
+/*
+ * EFI service pointer must be in %edi.
+ *
+ * The stack should represent the 32-bit calling convention.
+ */
+ENTRY(efi_enter32)
+        movl    $__KERNEL_DS, %eax
+        movl    %eax, %ds
+        movl    %eax, %es
+        movl    %eax, %ss
+
+        /* Reload pgtables */
+        movl    %cr3, %eax
+        movl    %eax, %cr3
+
+        /* Disable paging */
+        movl    %cr0, %eax
+        btrl    $X86_CR0_PG_BIT, %eax
+        movl    %eax, %cr0
+
+        /* Disable long mode via EFER */
+        movl    $MSR_EFER, %ecx
+        rdmsr
+        btrl    $_EFER_LME, %eax
+        wrmsr
+
+        call    *%edi
+
+        /* We must preserve return value */
+        movl    %eax, %edi
+
+        /*
+         * Some firmware will return with interrupts enabled. Be sure to
+         * disable them before we switch GDTs.
+         */
+        cli
+
+        movl    44(%esp), %eax
+        movl    %eax, 2(%eax)
+        lgdtl   (%eax)
+
+        movl    %cr4, %eax
+        btsl    $(X86_CR4_PAE_BIT), %eax
+        movl    %eax, %cr4
+
+        movl    %cr3, %eax
+        movl    %eax, %cr3
+
+        movl    $MSR_EFER, %ecx
+        rdmsr
+        btsl    $_EFER_LME, %eax
+        wrmsr
+
+        xorl    %eax, %eax
+        lldt    %ax
+
+        movl    48(%esp), %eax
+        pushl   $__KERNEL_CS
+        pushl   %eax
+
+        /* Enable paging */
+        movl    %cr0, %eax
+        btsl    $X86_CR0_PG_BIT, %eax
+        movl    %eax, %cr0
+        lret
+ENDPROC(efi_enter32)
+
+        .data
+        .balign 8
+        .global efi32_boot_gdt
+efi32_boot_gdt: .word   0
+                .quad   0
+
+save_gdt:       .word   0
+                .quad   0
+func_rt_ptr:    .quad   0
+
+        .global efi_gdt64
+efi_gdt64:
+        .word   efi_gdt64_end - efi_gdt64
+        .long   0                       /* Filled out by user */
+        .word   0
+        .quad   0x0000000000000000      /* NULL descriptor */
+        .quad   0x00af9a000000ffff      /* __KERNEL_CS */
+        .quad   0x00cf92000000ffff      /* __KERNEL_DS */
+        .quad   0x0080890000000000      /* TS descriptor */
+        .quad   0x0000000000000000      /* TS continued */
+efi_gdt64_end:
+#endif /* CONFIG_EFI_MIXED */
+
         .data
 ENTRY(efi_scratch)
         .fill 3,8,0
         .byte 0
+        .quad 0
diff --git a/arch/x86/platform/efi/efi_thunk_64.S b/arch/x86/platform/efi/efi_thunk_64.S
new file mode 100644
index 000000000000..8806fa73e6e6
--- /dev/null
+++ b/arch/x86/platform/efi/efi_thunk_64.S
@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2014 Intel Corporation; author Matt Fleming
+ */
+
+#include <linux/linkage.h>
+#include <asm/page_types.h>
+
+        .text
+        .code64
+ENTRY(efi64_thunk)
+        push    %rbp
+        push    %rbx
+
+        /*
+         * Switch to 1:1 mapped 32-bit stack pointer.
+         */
+        movq    %rsp, efi_saved_sp(%rip)
+        movq    efi_scratch+25(%rip), %rsp
+
+        /*
+         * Calculate the physical address of the kernel text.
+         */
+        movq    $__START_KERNEL_map, %rax
+        subq    phys_base(%rip), %rax
+
+        /*
+         * Push some physical addresses onto the stack. This is easier
+         * to do now in a code64 section while the assembler can address
+         * 64-bit values. Note that all the addresses on the stack are
+         * 32-bit.
+         */
+        subq    $16, %rsp
+        leaq    efi_exit32(%rip), %rbx
+        subq    %rax, %rbx
+        movl    %ebx, 8(%rsp)
+        leaq    efi_gdt64(%rip), %rbx
+        subq    %rax, %rbx
+        movl    %ebx, 2(%ebx)
+        movl    %ebx, 4(%rsp)
+        leaq    efi_gdt32(%rip), %rbx
+        subq    %rax, %rbx
+        movl    %ebx, 2(%ebx)
+        movl    %ebx, (%rsp)
+
+        leaq    __efi64_thunk(%rip), %rbx
+        subq    %rax, %rbx
+        call    *%rbx
+
+        movq    efi_saved_sp(%rip), %rsp
+        pop     %rbx
+        pop     %rbp
+        retq
+ENDPROC(efi64_thunk)
+
+        .data
+efi_gdt32:
+        .word   efi_gdt32_end - efi_gdt32
+        .long   0                       /* Filled out above */
+        .word   0
+        .quad   0x0000000000000000      /* NULL descriptor */
+        .quad   0x00cf9a000000ffff      /* __KERNEL_CS */
+        .quad   0x00cf93000000ffff      /* __KERNEL_DS */
+efi_gdt32_end:
+
+efi_saved_sp:           .quad 0