arm64/efi: move SetVirtualAddressMap() to UEFI stub

In order to support kexec, the kernel needs to be able to deal with the
state of the UEFI firmware after SetVirtualAddressMap() has been called.
To avoid having separate code paths for non-kexec and kexec, let's move
the call to SetVirtualAddressMap() to the stub: this will guarantee us
that it will only be called once (since the stub is not executed during
kexec), and ensures that the UEFI state is identical between kexec and
normal boot.

This implies that the layout of the virtual mapping needs to be created
by the stub as well. All regions are rounded up to a naturally aligned
multiple of 64 KB (for compatibility with 64k pages kernels) and recorded
in the UEFI memory map. The kernel proper reads those values and installs
the mappings in a dedicated set of page tables that are swapped in during
UEFI Runtime Services calls.

Acked-by: Leif Lindholm <leif.lindholm@linaro.org>
Acked-by: Matt Fleming <matt.fleming@intel.com>
Tested-by: Leif Lindholm <leif.lindholm@linaro.org>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>

3 files changed, 122 insertions, 3 deletions

diff --git a/drivers/firmware/efi/libstub/arm-stub.c b/drivers/firmware/efi/libstub/arm-stub.c
index eb48a1a1a576..e2432b39b6df 100644
--- a/drivers/firmware/efi/libstub/arm-stub.c
+++ b/drivers/firmware/efi/libstub/arm-stub.c
@@ -295,3 +295,62 @@ fail_free_image:
 fail:
         return EFI_ERROR;
 }
+
+/*
+ * This is the base address at which to start allocating virtual memory ranges
+ * for UEFI Runtime Services. This is in the low TTBR0 range so that we can use
+ * any allocation we choose, and eliminate the risk of a conflict after kexec.
+ * The value chosen is the largest non-zero power of 2 suitable for this purpose
+ * both on 32-bit and 64-bit ARM CPUs, to maximize the likelihood that it can
+ * be mapped efficiently.
+ */
+#define EFI_RT_VIRTUAL_BASE     0x40000000
+
+/*
+ * efi_get_virtmap() - create a virtual mapping for the EFI memory map
+ *
+ * This function populates the virt_addr fields of all memory region descriptors
+ * in @memory_map whose EFI_MEMORY_RUNTIME attribute is set. Those descriptors
+ * are also copied to @runtime_map, and their total count is returned in @count.
+ */
+void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size,
+                     unsigned long desc_size, efi_memory_desc_t *runtime_map,
+                     int *count)
+{
+        u64 efi_virt_base = EFI_RT_VIRTUAL_BASE;
+        efi_memory_desc_t *out = runtime_map;
+        int l;
+
+        for (l = 0; l < map_size; l += desc_size) {
+                efi_memory_desc_t *in = (void *)memory_map + l;
+                u64 paddr, size;
+
+                if (!(in->attribute & EFI_MEMORY_RUNTIME))
+                        continue;
+
+                /*
+                 * Make the mapping compatible with 64k pages: this allows
+                 * a 4k page size kernel to kexec a 64k page size kernel and
+                 * vice versa.
+                 */
+                paddr = round_down(in->phys_addr, SZ_64K);
+                size = round_up(in->num_pages * EFI_PAGE_SIZE +
+                                in->phys_addr - paddr, SZ_64K);
+
+                /*
+                 * Avoid wasting memory on PTEs by choosing a virtual base that
+                 * is compatible with section mappings if this region has the
+                 * appropriate size and physical alignment. (Sections are 2 MB
+                 * on 4k granule kernels)
+                 */
+                if (IS_ALIGNED(in->phys_addr, SZ_2M) && size >= SZ_2M)
+                        efi_virt_base = round_up(efi_virt_base, SZ_2M);
+
+                in->virt_addr = efi_virt_base + in->phys_addr - paddr;
+                efi_virt_base += size;
+
+                memcpy(out, in, desc_size);
+                out = (void *)out + desc_size;
+                ++*count;
+        }
+}
diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h
index 304ab295ca1a..2be10984a67a 100644
--- a/drivers/firmware/efi/libstub/efistub.h
+++ b/drivers/firmware/efi/libstub/efistub.h
@@ -39,4 +39,8 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
 
 void *get_fdt(efi_system_table_t *sys_table);
 
+void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size,
+                     unsigned long desc_size, efi_memory_desc_t *runtime_map,
+                     int *count);
+
 #endif
diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c
index c846a9608cbd..91da56c4fd54 100644
--- a/drivers/firmware/efi/libstub/fdt.c
+++ b/drivers/firmware/efi/libstub/fdt.c
@@ -14,6 +14,8 @@
 #include <linux/libfdt.h>
 #include <asm/efi.h>
 
+#include "efistub.h"
+
 efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt,
                         unsigned long orig_fdt_size,
                         void *fdt, int new_fdt_size, char *cmdline_ptr,
@@ -193,9 +195,26 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
         unsigned long map_size, desc_size;
         u32 desc_ver;
         unsigned long mmap_key;
-        efi_memory_desc_t *memory_map;
+        efi_memory_desc_t *memory_map, *runtime_map;
         unsigned long new_fdt_size;
         efi_status_t status;
+        int runtime_entry_count = 0;
+
+        /*
+         * Get a copy of the current memory map that we will use to prepare
+         * the input for SetVirtualAddressMap(). We don't have to worry about
+         * subsequent allocations adding entries, since they could not affect
+         * the number of EFI_MEMORY_RUNTIME regions.
+         */
+        status = efi_get_memory_map(sys_table, &runtime_map, &map_size,
+                                    &desc_size, &desc_ver, &mmap_key);
+        if (status != EFI_SUCCESS) {
+                pr_efi_err(sys_table, "Unable to retrieve UEFI memory map.\n");
+                return status;
+        }
+
+        pr_efi(sys_table,
+               "Exiting boot services and installing virtual address map...\n");
 
         /*
          * Estimate size of new FDT, and allocate memory for it. We
@@ -248,12 +267,48 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
                 }
         }
 
+        /*
+         * Update the memory map with virtual addresses. The function will also
+         * populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME
+         * entries so that we can pass it straight into SetVirtualAddressMap()
+         */
+        efi_get_virtmap(memory_map, map_size, desc_size, runtime_map,
+                        &runtime_entry_count);
+
         /* Now we are ready to exit_boot_services.*/
         status = sys_table->boottime->exit_boot_services(handle, mmap_key);
 
+        if (status == EFI_SUCCESS) {
+                efi_set_virtual_address_map_t *svam;
 
-        if (status == EFI_SUCCESS)
-                return status;
+                /* Install the new virtual address map */
+                svam = sys_table->runtime->set_virtual_address_map;
+                status = svam(runtime_entry_count * desc_size, desc_size,
+                              desc_ver, runtime_map);
+
+                /*
+                 * We are beyond the point of no return here, so if the call to
+                 * SetVirtualAddressMap() failed, we need to signal that to the
+                 * incoming kernel but proceed normally otherwise.
+                 */
+                if (status != EFI_SUCCESS) {
+                        int l;
+
+                        /*
+                         * Set the virtual address field of all
+                         * EFI_MEMORY_RUNTIME entries to 0. This will signal
+                         * the incoming kernel that no virtual translation has
+                         * been installed.
+                         */
+                        for (l = 0; l < map_size; l += desc_size) {
+                                efi_memory_desc_t *p = (void *)memory_map + l;
+
+                                if (p->attribute & EFI_MEMORY_RUNTIME)
+                                        p->virt_addr = 0;
+                        }
+                }
+                return EFI_SUCCESS;
+        }
 
         pr_efi_err(sys_table, "Exit boot services failed.\n");
 
@@ -264,6 +319,7 @@ fail_free_new_fdt:
         efi_free(sys_table, new_fdt_size, *new_fdt_addr);
 
 fail:
+        sys_table->boottime->free_pool(runtime_map);
         return EFI_LOAD_ERROR;
 }