x86/efi: Add mixed runtime services support

Setup the runtime services based on whether we're booting in EFI native
mode or not. For non-native mode we need to thunk from 64-bit into
32-bit mode before invoking the EFI runtime services.

Using the runtime services after SetVirtualAddressMap() is slightly more
complicated because we need to ensure that all the addresses we pass to
the firmware are below the 4GB boundary so that they can be addressed
with 32-bit pointers, see efi_setup_page_tables().

Signed-off-by: Matt Fleming <matt.fleming@intel.com>

6 files changed, 517 insertions, 33 deletions

diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index 85856868507b..85935e6f69f3 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -154,6 +154,27 @@ static inline bool efi_is_native(void)
 
 extern struct console early_efi_console;
 extern void parse_efi_setup(u64 phys_addr, u32 data_len);
+
+#ifdef CONFIG_EFI_MIXED
+extern void efi_thunk_runtime_setup(void);
+extern efi_status_t efi_thunk_set_virtual_address_map(
+        void *phys_set_virtual_address_map,
+        unsigned long memory_map_size,
+        unsigned long descriptor_size,
+        u32 descriptor_version,
+        efi_memory_desc_t *virtual_map);
+#else
+static inline void efi_thunk_runtime_setup(void) {}
+static inline efi_status_t efi_thunk_set_virtual_address_map(
+        void *phys_set_virtual_address_map,
+        unsigned long memory_map_size,
+        unsigned long descriptor_size,
+        u32 descriptor_version,
+        efi_memory_desc_t *virtual_map)
+{
+        return EFI_SUCCESS;
+}
+#endif /* CONFIG_EFI_MIXED */
 #else
 /*
  * IF EFI is not configured, have the EFI calls return -ENOSYS.
diff --git a/arch/x86/platform/efi/Makefile b/arch/x86/platform/efi/Makefile
index b7b0b35c1981..d51045afcaaf 100644
--- a/arch/x86/platform/efi/Makefile
+++ b/arch/x86/platform/efi/Makefile
@@ -1,3 +1,4 @@
 obj-$(CONFIG_EFI)               += efi.o efi_$(BITS).o efi_stub_$(BITS).o
 obj-$(CONFIG_ACPI_BGRT) += efi-bgrt.o
 obj-$(CONFIG_EARLY_PRINTK_EFI)  += early_printk.o
+obj-$(CONFIG_EFI_MIXED)         += efi_thunk_$(BITS).o
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index b96ae7918a16..39f5b7bba695 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -588,37 +588,85 @@ static int __init efi_systab_init(void *phys)
         return 0;
 }
 
-static int __init efi_runtime_init(void)
+static int __init efi_runtime_init32(void)
 {
-        efi_runtime_services_t *runtime;
+        efi_runtime_services_32_t *runtime;
+
+        runtime = early_ioremap((unsigned long)efi.systab->runtime,
+                        sizeof(efi_runtime_services_32_t));
+        if (!runtime) {
+                pr_err("Could not map the runtime service table!\n");
+                return -ENOMEM;
+        }
 
         /*
-         * Check out the runtime services table. We need to map
-         * the runtime services table so that we can grab the physical
-         * address of several of the EFI runtime functions, needed to
-         * set the firmware into virtual mode.
+         * We will only need *early* access to the following two
+         * EFI runtime services before set_virtual_address_map
+         * is invoked.
          */
+        efi_phys.get_time = (efi_get_time_t *)
+                        (unsigned long)runtime->get_time;
+        efi_phys.set_virtual_address_map =
+                        (efi_set_virtual_address_map_t *)
+                        (unsigned long)runtime->set_virtual_address_map;
+        /*
+         * Make efi_get_time can be called before entering
+         * virtual mode.
+         */
+        efi.get_time = phys_efi_get_time;
+        early_iounmap(runtime, sizeof(efi_runtime_services_32_t));
+
+        return 0;
+}
+
+static int __init efi_runtime_init64(void)
+{
+        efi_runtime_services_64_t *runtime;
+
         runtime = early_ioremap((unsigned long)efi.systab->runtime,
-                                sizeof(efi_runtime_services_t));
+                        sizeof(efi_runtime_services_64_t));
         if (!runtime) {
                 pr_err("Could not map the runtime service table!\n");
                 return -ENOMEM;
         }
+
         /*
-         * We will only need *early* access to the following
-         * two EFI runtime services before set_virtual_address_map
+         * We will only need *early* access to the following two
+         * EFI runtime services before set_virtual_address_map
          * is invoked.
          */
-        efi_phys.get_time = (efi_get_time_t *)runtime->get_time;
+        efi_phys.get_time = (efi_get_time_t *)
+                        (unsigned long)runtime->get_time;
         efi_phys.set_virtual_address_map =
-                (efi_set_virtual_address_map_t *)
-                runtime->set_virtual_address_map;
+                        (efi_set_virtual_address_map_t *)
+                        (unsigned long)runtime->set_virtual_address_map;
         /*
          * Make efi_get_time can be called before entering
          * virtual mode.
          */
         efi.get_time = phys_efi_get_time;
-        early_iounmap(runtime, sizeof(efi_runtime_services_t));
+        early_iounmap(runtime, sizeof(efi_runtime_services_64_t));
+
+        return 0;
+}
+
+static int __init efi_runtime_init(void)
+{
+        int rv;
+
+        /*
+         * Check out the runtime services table. We need to map
+         * the runtime services table so that we can grab the physical
+         * address of several of the EFI runtime functions, needed to
+         * set the firmware into virtual mode.
+         */
+        if (efi_enabled(EFI_64BIT))
+                rv = efi_runtime_init64();
+        else
+                rv = efi_runtime_init32();
+
+        if (rv)
+                return rv;
 
         return 0;
 }
@@ -841,6 +889,22 @@ void __init old_map_region(efi_memory_desc_t *md)
                        (unsigned long long)md->phys_addr);
 }
 
+static void native_runtime_setup(void)
+{
+        efi.get_time = virt_efi_get_time;
+        efi.set_time = virt_efi_set_time;
+        efi.get_wakeup_time = virt_efi_get_wakeup_time;
+        efi.set_wakeup_time = virt_efi_set_wakeup_time;
+        efi.get_variable = virt_efi_get_variable;
+        efi.get_next_variable = virt_efi_get_next_variable;
+        efi.set_variable = virt_efi_set_variable;
+        efi.get_next_high_mono_count = virt_efi_get_next_high_mono_count;
+        efi.reset_system = virt_efi_reset_system;
+        efi.query_variable_info = virt_efi_query_variable_info;
+        efi.update_capsule = virt_efi_update_capsule;
+        efi.query_capsule_caps = virt_efi_query_capsule_caps;
+}
+
 /* Merge contiguous regions of the same type and attribute */
 static void __init efi_merge_regions(void)
 {
@@ -1023,11 +1087,20 @@ void __init efi_enter_virtual_mode(void)
         efi_sync_low_kernel_mappings();
 
         if (!efi_setup) {
-                status = phys_efi_set_virtual_address_map(
-                        memmap.desc_size * count,
-                        memmap.desc_size,
-                        memmap.desc_version,
-                        (efi_memory_desc_t *)__pa(new_memmap));
+                if (efi_is_native()) {
+                        status = phys_efi_set_virtual_address_map(
+                                        memmap.desc_size * count,
+                                        memmap.desc_size,
+                                        memmap.desc_version,
+                                        (efi_memory_desc_t *)__pa(new_memmap));
+                } else {
+                        status = efi_thunk_set_virtual_address_map(
+                                        efi_phys.set_virtual_address_map,
+                                        memmap.desc_size * count,
+                                        memmap.desc_size,
+                                        memmap.desc_version,
+                                        (efi_memory_desc_t *)__pa(new_memmap));
+                }
 
                 if (status != EFI_SUCCESS) {
                         pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
@@ -1043,19 +1116,13 @@ void __init efi_enter_virtual_mode(void)
          * Call EFI services through wrapper functions.
          */
         efi.runtime_version = efi_systab.hdr.revision;
-        efi.get_time = virt_efi_get_time;
-        efi.set_time = virt_efi_set_time;
-        efi.get_wakeup_time = virt_efi_get_wakeup_time;
-        efi.set_wakeup_time = virt_efi_set_wakeup_time;
-        efi.get_variable = virt_efi_get_variable;
-        efi.get_next_variable = virt_efi_get_next_variable;
-        efi.set_variable = virt_efi_set_variable;
-        efi.get_next_high_mono_count = virt_efi_get_next_high_mono_count;
-        efi.reset_system = virt_efi_reset_system;
+
+        if (efi_is_native())
+                native_runtime_setup();
+        else
+                efi_thunk_runtime_setup();
+
         efi.set_virtual_address_map = NULL;
-        efi.query_variable_info = virt_efi_query_variable_info;
-        efi.update_capsule = virt_efi_update_capsule;
-        efi.query_capsule_caps = virt_efi_query_capsule_caps;
 
         efi_runtime_mkexec();
 
diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c
index 0c2a234fef1e..12112ab4fd40 100644
--- a/arch/x86/platform/efi/efi_64.c
+++ b/arch/x86/platform/efi/efi_64.c
@@ -39,6 +39,7 @@
 #include <asm/cacheflush.h>
 #include <asm/fixmap.h>
 #include <asm/realmode.h>
+#include <asm/time.h>
 
 static pgd_t *save_pgd __initdata;
 static unsigned long efi_flags __initdata;
@@ -58,7 +59,8 @@ struct efi_scratch {
         u64 prev_cr3;
         pgd_t *efi_pgt;
         bool use_pgd;
-};
+        u64 phys_stack;
+} __packed;
 
 static void __init early_code_mapping_set_exec(int executable)
 {
@@ -139,10 +141,40 @@ void efi_sync_low_kernel_mappings(void)
 
 void efi_setup_page_tables(void)
 {
+        unsigned long text;
+        unsigned npages;
+        struct page *page;
+
         efi_scratch.efi_pgt = (pgd_t *)(unsigned long)real_mode_header->trampoline_pgd;
 
-        if (!efi_enabled(EFI_OLD_MEMMAP))
-                efi_scratch.use_pgd = true;
+        if (efi_enabled(EFI_OLD_MEMMAP))
+                return;
+
+        efi_scratch.use_pgd = true;
+
+        /*
+         * When making calls to the firmware everything needs to be 1:1
+         * mapped and addressable with 32-bit pointers. Map the kernel
+         * text and allocate a new stack because we can't rely on the
+         * stack pointer being < 4GB.
+         */
+        if (!IS_ENABLED(CONFIG_EFI_MIXED))
+                return;
+
+        page = alloc_page(GFP_KERNEL|__GFP_DMA32);
+        if (!page)
+                panic("Unable to allocate EFI runtime stack < 4GB\n");
+
+        efi_scratch.phys_stack = virt_to_phys(page_address(page));
+        efi_scratch.phys_stack += PAGE_SIZE; /* stack grows down */
+
+        npages = (_end - _text) >> PAGE_SHIFT;
+        text = __pa(_text);
+
+        if (kernel_map_pages_in_pgd(__va(efi_scratch.efi_pgt),
+                                    text >> PAGE_SHIFT, text, npages, 0)) {
+                pr_err("Failed to map kernel text 1:1\n");
+        }
 }
 
 static void __init __map_region(efi_memory_desc_t *md, u64 va)
@@ -173,6 +205,16 @@ void __init efi_map_region(efi_memory_desc_t *md)
          */
         __map_region(md, md->phys_addr);
 
+        /*
+         * Enforce the 1:1 mapping as the default virtual address when
+         * booting in EFI mixed mode, because even though we may be
+         * running a 64-bit kernel, the firmware may only be 32-bit.
+         */
+        if (!efi_is_native () && IS_ENABLED(CONFIG_EFI_MIXED)) {
+                md->virt_addr = md->phys_addr;
+                return;
+        }
+
         efi_va -= size;
 
         /* Is PA 2M-aligned? */
@@ -242,3 +284,290 @@ void __init efi_runtime_mkexec(void)
         if (__supported_pte_mask & _PAGE_NX)
                 runtime_code_page_mkexec();
 }
+
+#ifdef CONFIG_EFI_MIXED
+extern efi_status_t efi64_thunk(u32, ...);
+
+#define runtime_service32(func)                                          \
+({                                                                       \
+        u32 table = (u32)(unsigned long)efi.systab;                      \
+        u32 *rt, *___f;                                                  \
+                                                                         \
+        rt = (u32 *)(table + offsetof(efi_system_table_32_t, runtime));  \
+        ___f = (u32 *)(*rt + offsetof(efi_runtime_services_32_t, func)); \
+        *___f;                                                           \
+})
+
+/*
+ * Switch to the EFI page tables early so that we can access the 1:1
+ * runtime services mappings which are not mapped in any other page
+ * tables. This function must be called before runtime_service32().
+ *
+ * Also, disable interrupts because the IDT points to 64-bit handlers,
+ * which aren't going to function correctly when we switch to 32-bit.
+ */
+#define efi_thunk(f, ...)                                               \
+({                                                                      \
+        efi_status_t __s;                                               \
+        unsigned long flags;                                            \
+        u32 func;                                                       \
+                                                                        \
+        efi_sync_low_kernel_mappings();                                 \
+        local_irq_save(flags);                                          \
+                                                                        \
+        efi_scratch.prev_cr3 = read_cr3();                              \
+        write_cr3((unsigned long)efi_scratch.efi_pgt);                  \
+        __flush_tlb_all();                                              \
+                                                                        \
+        func = runtime_service32(f);                                    \
+        __s = efi64_thunk(func, __VA_ARGS__);                   \
+                                                                        \
+        write_cr3(efi_scratch.prev_cr3);                                \
+        __flush_tlb_all();                                              \
+        local_irq_restore(flags);                                       \
+                                                                        \
+        __s;                                                            \
+})
+
+efi_status_t efi_thunk_set_virtual_address_map(
+        void *phys_set_virtual_address_map,
+        unsigned long memory_map_size,
+        unsigned long descriptor_size,
+        u32 descriptor_version,
+        efi_memory_desc_t *virtual_map)
+{
+        efi_status_t status;
+        unsigned long flags;
+        u32 func;
+
+        efi_sync_low_kernel_mappings();
+        local_irq_save(flags);
+
+        efi_scratch.prev_cr3 = read_cr3();
+        write_cr3((unsigned long)efi_scratch.efi_pgt);
+        __flush_tlb_all();
+
+        func = (u32)(unsigned long)phys_set_virtual_address_map;
+        status = efi64_thunk(func, memory_map_size, descriptor_size,
+                             descriptor_version, virtual_map);
+
+        write_cr3(efi_scratch.prev_cr3);
+        __flush_tlb_all();
+        local_irq_restore(flags);
+
+        return status;
+}
+
+static efi_status_t efi_thunk_get_time(efi_time_t *tm, efi_time_cap_t *tc)
+{
+        efi_status_t status;
+        u32 phys_tm, phys_tc;
+
+        spin_lock(&rtc_lock);
+
+        phys_tm = virt_to_phys(tm);
+        phys_tc = virt_to_phys(tc);
+
+        status = efi_thunk(get_time, phys_tm, phys_tc);
+
+        spin_unlock(&rtc_lock);
+
+        return status;
+}
+
+static efi_status_t efi_thunk_set_time(efi_time_t *tm)
+{
+        efi_status_t status;
+        u32 phys_tm;
+
+        spin_lock(&rtc_lock);
+
+        phys_tm = virt_to_phys(tm);
+
+        status = efi_thunk(set_time, phys_tm);
+
+        spin_unlock(&rtc_lock);
+
+        return status;
+}
+
+static efi_status_t
+efi_thunk_get_wakeup_time(efi_bool_t *enabled, efi_bool_t *pending,
+                          efi_time_t *tm)
+{
+        efi_status_t status;
+        u32 phys_enabled, phys_pending, phys_tm;
+
+        spin_lock(&rtc_lock);
+
+        phys_enabled = virt_to_phys(enabled);
+        phys_pending = virt_to_phys(pending);
+        phys_tm = virt_to_phys(tm);
+
+        status = efi_thunk(get_wakeup_time, phys_enabled,
+                             phys_pending, phys_tm);
+
+        spin_unlock(&rtc_lock);
+
+        return status;
+}
+
+static efi_status_t
+efi_thunk_set_wakeup_time(efi_bool_t enabled, efi_time_t *tm)
+{
+        efi_status_t status;
+        u32 phys_tm;
+
+        spin_lock(&rtc_lock);
+
+        phys_tm = virt_to_phys(tm);
+
+        status = efi_thunk(set_wakeup_time, enabled, phys_tm);
+
+        spin_unlock(&rtc_lock);
+
+        return status;
+}
+
+
+static efi_status_t
+efi_thunk_get_variable(efi_char16_t *name, efi_guid_t *vendor,
+                       u32 *attr, unsigned long *data_size, void *data)
+{
+        efi_status_t status;
+        u32 phys_name, phys_vendor, phys_attr;
+        u32 phys_data_size, phys_data;
+
+        phys_data_size = virt_to_phys(data_size);
+        phys_vendor = virt_to_phys(vendor);
+        phys_name = virt_to_phys(name);
+        phys_attr = virt_to_phys(attr);
+        phys_data = virt_to_phys(data);
+
+        status = efi_thunk(get_variable, phys_name, phys_vendor,
+                           phys_attr, phys_data_size, phys_data);
+
+        return status;
+}
+
+static efi_status_t
+efi_thunk_set_variable(efi_char16_t *name, efi_guid_t *vendor,
+                       u32 attr, unsigned long data_size, void *data)
+{
+        u32 phys_name, phys_vendor, phys_data;
+        efi_status_t status;
+
+        phys_name = virt_to_phys(name);
+        phys_vendor = virt_to_phys(vendor);
+        phys_data = virt_to_phys(data);
+
+        /* If data_size is > sizeof(u32) we've got problems */
+        status = efi_thunk(set_variable, phys_name, phys_vendor,
+                           attr, data_size, phys_data);
+
+        return status;
+}
+
+static efi_status_t
+efi_thunk_get_next_variable(unsigned long *name_size,
+                            efi_char16_t *name,
+                            efi_guid_t *vendor)
+{
+        efi_status_t status;
+        u32 phys_name_size, phys_name, phys_vendor;
+
+        phys_name_size = virt_to_phys(name_size);
+        phys_vendor = virt_to_phys(vendor);
+        phys_name = virt_to_phys(name);
+
+        status = efi_thunk(get_next_variable, phys_name_size,
+                           phys_name, phys_vendor);
+
+        return status;
+}
+
+static efi_status_t
+efi_thunk_get_next_high_mono_count(u32 *count)
+{
+        efi_status_t status;
+        u32 phys_count;
+
+        phys_count = virt_to_phys(count);
+        status = efi_thunk(get_next_high_mono_count, phys_count);
+
+        return status;
+}
+
+static void
+efi_thunk_reset_system(int reset_type, efi_status_t status,
+                       unsigned long data_size, efi_char16_t *data)
+{
+        u32 phys_data;
+
+        phys_data = virt_to_phys(data);
+
+        efi_thunk(reset_system, reset_type, status, data_size, phys_data);
+}
+
+static efi_status_t
+efi_thunk_update_capsule(efi_capsule_header_t **capsules,
+                         unsigned long count, unsigned long sg_list)
+{
+        /*
+         * To properly support this function we would need to repackage
+         * 'capsules' because the firmware doesn't understand 64-bit
+         * pointers.
+         */
+        return EFI_UNSUPPORTED;
+}
+
+static efi_status_t
+efi_thunk_query_variable_info(u32 attr, u64 *storage_space,
+                              u64 *remaining_space,
+                              u64 *max_variable_size)
+{
+        efi_status_t status;
+        u32 phys_storage, phys_remaining, phys_max;
+
+        if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION)
+                return EFI_UNSUPPORTED;
+
+        phys_storage = virt_to_phys(storage_space);
+        phys_remaining = virt_to_phys(remaining_space);
+        phys_max = virt_to_phys(max_variable_size);
+
+        status = efi_thunk(query_variable_info, phys_storage,
+                           phys_remaining, phys_max);
+
+        return status;
+}
+
+static efi_status_t
+efi_thunk_query_capsule_caps(efi_capsule_header_t **capsules,
+                             unsigned long count, u64 *max_size,
+                             int *reset_type)
+{
+        /*
+         * To properly support this function we would need to repackage
+         * 'capsules' because the firmware doesn't understand 64-bit
+         * pointers.
+         */
+        return EFI_UNSUPPORTED;
+}
+
+void efi_thunk_runtime_setup(void)
+{
+        efi.get_time = efi_thunk_get_time;
+        efi.set_time = efi_thunk_set_time;
+        efi.get_wakeup_time = efi_thunk_get_wakeup_time;
+        efi.set_wakeup_time = efi_thunk_set_wakeup_time;
+        efi.get_variable = efi_thunk_get_variable;
+        efi.get_next_variable = efi_thunk_get_next_variable;
+        efi.set_variable = efi_thunk_set_variable;
+        efi.get_next_high_mono_count = efi_thunk_get_next_high_mono_count;
+        efi.reset_system = efi_thunk_reset_system;
+        efi.query_variable_info = efi_thunk_query_variable_info;
+        efi.update_capsule = efi_thunk_update_capsule;
+        efi.query_capsule_caps = efi_thunk_query_capsule_caps;
+}
+#endif /* CONFIG_EFI_MIXED */
diff --git a/arch/x86/platform/efi/efi_stub_64.S b/arch/x86/platform/efi/efi_stub_64.S
index a790d69cc85e..e811514beeac 100644
--- a/arch/x86/platform/efi/efi_stub_64.S
+++ b/arch/x86/platform/efi/efi_stub_64.S
@@ -318,3 +318,4 @@ efi_gdt64_end:
 ENTRY(efi_scratch)
         .fill 3,8,0
         .byte 0
+        .quad 0
diff --git a/arch/x86/platform/efi/efi_thunk_64.S b/arch/x86/platform/efi/efi_thunk_64.S
new file mode 100644
index 000000000000..8806fa73e6e6
--- /dev/null
+++ b/arch/x86/platform/efi/efi_thunk_64.S
@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2014 Intel Corporation; author Matt Fleming
+ */
+
+#include <linux/linkage.h>
+#include <asm/page_types.h>
+
+        .text
+        .code64
+ENTRY(efi64_thunk)
+        push    %rbp
+        push    %rbx
+
+        /*
+         * Switch to 1:1 mapped 32-bit stack pointer.
+         */
+        movq    %rsp, efi_saved_sp(%rip)
+        movq    efi_scratch+25(%rip), %rsp
+
+        /*
+         * Calculate the physical address of the kernel text.
+         */
+        movq    $__START_KERNEL_map, %rax
+        subq    phys_base(%rip), %rax
+
+        /*
+         * Push some physical addresses onto the stack. This is easier
+         * to do now in a code64 section while the assembler can address
+         * 64-bit values. Note that all the addresses on the stack are
+         * 32-bit.
+         */
+        subq    $16, %rsp
+        leaq    efi_exit32(%rip), %rbx
+        subq    %rax, %rbx
+        movl    %ebx, 8(%rsp)
+        leaq    efi_gdt64(%rip), %rbx
+        subq    %rax, %rbx
+        movl    %ebx, 2(%ebx)
+        movl    %ebx, 4(%rsp)
+        leaq    efi_gdt32(%rip), %rbx
+        subq    %rax, %rbx
+        movl    %ebx, 2(%ebx)
+        movl    %ebx, (%rsp)
+
+        leaq    __efi64_thunk(%rip), %rbx
+        subq    %rax, %rbx
+        call    *%rbx
+
+        movq    efi_saved_sp(%rip), %rsp
+        pop     %rbx
+        pop     %rbp
+        retq
+ENDPROC(efi64_thunk)
+
+        .data
+efi_gdt32:
+        .word   efi_gdt32_end - efi_gdt32
+        .long   0                       /* Filled out above */
+        .word   0
+        .quad   0x0000000000000000      /* NULL descriptor */
+        .quad   0x00cf9a000000ffff      /* __KERNEL_CS */
+        .quad   0x00cf93000000ffff      /* __KERNEL_DS */
+efi_gdt32_end:
+
+efi_saved_sp:           .quad 0