4 files changed, 208 insertions, 153 deletions

diff --git a/arch/x86/platform/efi/efi-bgrt.c b/arch/x86/platform/efi/efi-bgrt.c
index 6a2f5691b1ab..6aad870e8962 100644
--- a/arch/x86/platform/efi/efi-bgrt.c
+++ b/arch/x86/platform/efi/efi-bgrt.c
@@ -82,21 +82,12 @@ void __init efi_bgrt_init(void)
         }
         bgrt_image_size = bmp_header.size;
 
-        bgrt_image = kmalloc(bgrt_image_size, GFP_KERNEL | __GFP_NOWARN);
+        bgrt_image = memremap(bgrt_tab->image_address, bmp_header.size, MEMREMAP_WB);
         if (!bgrt_image) {
-                pr_notice("Ignoring BGRT: failed to allocate memory for image (wanted %zu bytes)\n",
-                       bgrt_image_size);
-                return;
-        }
-
-        image = memremap(bgrt_tab->image_address, bmp_header.size, MEMREMAP_WB);
-        if (!image) {
                 pr_notice("Ignoring BGRT: failed to map image memory\n");
-                kfree(bgrt_image);
                 bgrt_image = NULL;
                 return;
         }
 
-        memcpy(bgrt_image, image, bgrt_image_size);
-        memunmap(image);
+        efi_mem_reserve(bgrt_tab->image_address, bgrt_image_size);
 }
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index 1fbb408e2e72..0955c70897ae 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -172,7 +172,9 @@ static void __init do_add_efi_memmap(void)
 int __init efi_memblock_x86_reserve_range(void)
 {
         struct efi_info *e = &boot_params.efi_info;
+        struct efi_memory_map_data data;
         phys_addr_t pmap;
+        int rv;
 
         if (efi_enabled(EFI_PARAVIRT))
                 return 0;
@@ -187,11 +189,17 @@ int __init efi_memblock_x86_reserve_range(void)
 #else
         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 #endif
-        efi.memmap.phys_map     = pmap;
-        efi.memmap.nr_map       = e->efi_memmap_size /
-                                  e->efi_memdesc_size;
-        efi.memmap.desc_size    = e->efi_memdesc_size;
-        efi.memmap.desc_version = e->efi_memdesc_version;
+        data.phys_map           = pmap;
+        data.size               = e->efi_memmap_size;
+        data.desc_size          = e->efi_memdesc_size;
+        data.desc_version       = e->efi_memdesc_version;
+
+        rv = efi_memmap_init_early(&data);
+        if (rv)
+                return rv;
+
+        if (add_efi_memmap)
+                do_add_efi_memmap();
 
         WARN(efi.memmap.desc_version != 1,
              "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
@@ -218,19 +226,6 @@ void __init efi_print_memmap(void)
         }
 }
 
-void __init efi_unmap_memmap(void)
-{
-        unsigned long size;
-
-        clear_bit(EFI_MEMMAP, &efi.flags);
-
-        size = efi.memmap.nr_map * efi.memmap.desc_size;
-        if (efi.memmap.map) {
-                early_memunmap(efi.memmap.map, size);
-                efi.memmap.map = NULL;
-        }
-}
-
 static int __init efi_systab_init(void *phys)
 {
         if (efi_enabled(EFI_64BIT)) {
@@ -414,33 +409,6 @@ static int __init efi_runtime_init(void)
         return 0;
 }
 
-static int __init efi_memmap_init(void)
-{
-        unsigned long addr, size;
-
-        if (efi_enabled(EFI_PARAVIRT))
-                return 0;
-
-        /* Map the EFI memory map */
-        size = efi.memmap.nr_map * efi.memmap.desc_size;
-        addr = (unsigned long)efi.memmap.phys_map;
-
-        efi.memmap.map = early_memremap(addr, size);
-        if (efi.memmap.map == NULL) {
-                pr_err("Could not map the memory map!\n");
-                return -ENOMEM;
-        }
-
-        efi.memmap.map_end = efi.memmap.map + size;
-
-        if (add_efi_memmap)
-                do_add_efi_memmap();
-
-        set_bit(EFI_MEMMAP, &efi.flags);
-
-        return 0;
-}
-
 void __init efi_init(void)
 {
         efi_char16_t *c16;
@@ -498,16 +466,14 @@ void __init efi_init(void)
         if (!efi_runtime_supported())
                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
         else {
-                if (efi_runtime_disabled() || efi_runtime_init())
+                if (efi_runtime_disabled() || efi_runtime_init()) {
+                        efi_memmap_unmap();
                         return;
+                }
         }
-        if (efi_memmap_init())
-                return;
 
         if (efi_enabled(EFI_DBG))
                 efi_print_memmap();
-
-        efi_esrt_init();
 }
 
 void __init efi_late_init(void)
@@ -624,42 +590,6 @@ static void __init get_systab_virt_addr(efi_memory_desc_t *md)
         }
 }
 
-static void __init save_runtime_map(void)
-{
-#ifdef CONFIG_KEXEC_CORE
-        unsigned long desc_size;
-        efi_memory_desc_t *md;
-        void *tmp, *q = NULL;
-        int count = 0;
-
-        if (efi_enabled(EFI_OLD_MEMMAP))
-                return;
-
-        desc_size = efi.memmap.desc_size;
-
-        for_each_efi_memory_desc(md) {
-                if (!(md->attribute & EFI_MEMORY_RUNTIME) ||
-                    (md->type == EFI_BOOT_SERVICES_CODE) ||
-                    (md->type == EFI_BOOT_SERVICES_DATA))
-                        continue;
-                tmp = krealloc(q, (count + 1) * desc_size, GFP_KERNEL);
-                if (!tmp)
-                        goto out;
-                q = tmp;
-
-                memcpy(q + count * desc_size, md, desc_size);
-                count++;
-        }
-
-        efi_runtime_map_setup(q, count, desc_size);
-        return;
-
-out:
-        kfree(q);
-        pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n");
-#endif
-}
-
 static void *realloc_pages(void *old_memmap, int old_shift)
 {
         void *ret;
@@ -745,6 +675,46 @@ static void *efi_map_next_entry(void *entry)
         return entry;
 }
 
+static bool should_map_region(efi_memory_desc_t *md)
+{
+        /*
+         * Runtime regions always require runtime mappings (obviously).
+         */
+        if (md->attribute & EFI_MEMORY_RUNTIME)
+                return true;
+
+        /*
+         * 32-bit EFI doesn't suffer from the bug that requires us to
+         * reserve boot services regions, and mixed mode support
+         * doesn't exist for 32-bit kernels.
+         */
+        if (IS_ENABLED(CONFIG_X86_32))
+                return false;
+
+        /*
+         * Map all of RAM so that we can access arguments in the 1:1
+         * mapping when making EFI runtime calls.
+         */
+        if (IS_ENABLED(CONFIG_EFI_MIXED) && !efi_is_native()) {
+                if (md->type == EFI_CONVENTIONAL_MEMORY ||
+                    md->type == EFI_LOADER_DATA ||
+                    md->type == EFI_LOADER_CODE)
+                        return true;
+        }
+
+        /*
+         * Map boot services regions as a workaround for buggy
+         * firmware that accesses them even when they shouldn't.
+         *
+         * See efi_{reserve,free}_boot_services().
+         */
+        if (md->type == EFI_BOOT_SERVICES_CODE ||
+            md->type == EFI_BOOT_SERVICES_DATA)
+                return true;
+
+        return false;
+}
+
 /*
  * Map the efi memory ranges of the runtime services and update new_mmap with
  * virtual addresses.
@@ -761,13 +731,9 @@ static void * __init efi_map_regions(int *count, int *pg_shift)
         p = NULL;
         while ((p = efi_map_next_entry(p))) {
                 md = p;
-                if (!(md->attribute & EFI_MEMORY_RUNTIME)) {
-#ifdef CONFIG_X86_64
-                        if (md->type != EFI_BOOT_SERVICES_CODE &&
-                            md->type != EFI_BOOT_SERVICES_DATA)
-#endif
-                                continue;
-                }
+
+                if (!should_map_region(md))
+                        continue;
 
                 efi_map_region(md);
                 get_systab_virt_addr(md);
@@ -803,7 +769,7 @@ static void __init kexec_enter_virtual_mode(void)
          * non-native EFI
          */
         if (!efi_is_native()) {
-                efi_unmap_memmap();
+                efi_memmap_unmap();
                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
                 return;
         }
@@ -823,7 +789,18 @@ static void __init kexec_enter_virtual_mode(void)
                 get_systab_virt_addr(md);
         }
 
-        save_runtime_map();
+        /*
+         * Unregister the early EFI memmap from efi_init() and install
+         * the new EFI memory map.
+         */
+        efi_memmap_unmap();
+
+        if (efi_memmap_init_late(efi.memmap.phys_map,
+                                 efi.memmap.desc_size * efi.memmap.nr_map)) {
+                pr_err("Failed to remap late EFI memory map\n");
+                clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
+                return;
+        }
 
         BUG_ON(!efi.systab);
 
@@ -884,6 +861,7 @@ static void __init __efi_enter_virtual_mode(void)
         int count = 0, pg_shift = 0;
         void *new_memmap = NULL;
         efi_status_t status;
+        phys_addr_t pa;
 
         efi.systab = NULL;
 
@@ -901,11 +879,24 @@ static void __init __efi_enter_virtual_mode(void)
                 return;
         }
 
-        save_runtime_map();
+        pa = __pa(new_memmap);
+
+        /*
+         * Unregister the early EFI memmap from efi_init() and install
+         * the new EFI memory map that we are about to pass to the
+         * firmware via SetVirtualAddressMap().
+         */
+        efi_memmap_unmap();
+
+        if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) {
+                pr_err("Failed to remap late EFI memory map\n");
+                clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
+                return;
+        }
 
         BUG_ON(!efi.systab);
 
-        if (efi_setup_page_tables(__pa(new_memmap), 1 << pg_shift)) {
+        if (efi_setup_page_tables(pa, 1 << pg_shift)) {
                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
                 return;
         }
@@ -917,14 +908,14 @@ static void __init __efi_enter_virtual_mode(void)
                                 efi.memmap.desc_size * count,
                                 efi.memmap.desc_size,
                                 efi.memmap.desc_version,
-                                (efi_memory_desc_t *)__pa(new_memmap));
+                                (efi_memory_desc_t *)pa);
         } else {
                 status = efi_thunk_set_virtual_address_map(
                                 efi_phys.set_virtual_address_map,
                                 efi.memmap.desc_size * count,
                                 efi.memmap.desc_size,
                                 efi.memmap.desc_version,
-                                (efi_memory_desc_t *)__pa(new_memmap));
+                                (efi_memory_desc_t *)pa);
         }
 
         if (status != EFI_SUCCESS) {
@@ -956,15 +947,6 @@ static void __init __efi_enter_virtual_mode(void)
         efi_runtime_update_mappings();
         efi_dump_pagetable();
 
-        /*
-         * We mapped the descriptor array into the EFI pagetable above
-         * but we're not unmapping it here because if we're running in
-         * EFI mixed mode we need all of memory to be accessible when
-         * we pass parameters to the EFI runtime services in the
-         * thunking code.
-         */
-        free_pages((unsigned long)new_memmap, pg_shift);
-
         /* clean DUMMY object */
         efi_delete_dummy_variable();
 }
diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c
index 8dd3784eb075..58b0f801f66f 100644
--- a/arch/x86/platform/efi/efi_64.c
+++ b/arch/x86/platform/efi/efi_64.c
@@ -85,7 +85,7 @@ pgd_t * __init efi_call_phys_prolog(void)
         early_code_mapping_set_exec(1);
 
         n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT), PGDIR_SIZE);
-        save_pgd = kmalloc(n_pgds * sizeof(pgd_t), GFP_KERNEL);
+        save_pgd = kmalloc_array(n_pgds, sizeof(*save_pgd), GFP_KERNEL);
 
         for (pgd = 0; pgd < n_pgds; pgd++) {
                 save_pgd[pgd] = *pgd_offset_k(pgd * PGDIR_SIZE);
@@ -214,7 +214,6 @@ void efi_sync_low_kernel_mappings(void)
 int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages)
 {
         unsigned long pfn, text;
-        efi_memory_desc_t *md;
         struct page *page;
         unsigned npages;
         pgd_t *pgd;
@@ -248,25 +247,6 @@ int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages)
         if (!IS_ENABLED(CONFIG_EFI_MIXED) || efi_is_native())
                 return 0;
 
-        /*
-         * Map all of RAM so that we can access arguments in the 1:1
-         * mapping when making EFI runtime calls.
-         */
-        for_each_efi_memory_desc(md) {
-                if (md->type != EFI_CONVENTIONAL_MEMORY &&
-                    md->type != EFI_LOADER_DATA &&
-                    md->type != EFI_LOADER_CODE)
-                        continue;
-
-                pfn = md->phys_addr >> PAGE_SHIFT;
-                npages = md->num_pages;
-
-                if (kernel_map_pages_in_pgd(pgd, pfn, md->phys_addr, npages, _PAGE_RW)) {
-                        pr_err("Failed to map 1:1 memory\n");
-                        return 1;
-                }
-        }
-
         page = alloc_page(GFP_KERNEL|__GFP_DMA32);
         if (!page)
                 panic("Unable to allocate EFI runtime stack < 4GB\n");
@@ -359,6 +339,7 @@ void __init efi_map_region(efi_memory_desc_t *md)
  */
 void __init efi_map_region_fixed(efi_memory_desc_t *md)
 {
+        __map_region(md, md->phys_addr);
         __map_region(md, md->virt_addr);
 }
 
diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c
index 89d1146f5a6f..10aca63a50d7 100644
--- a/arch/x86/platform/efi/quirks.c
+++ b/arch/x86/platform/efi/quirks.c
@@ -164,6 +164,75 @@ efi_status_t efi_query_variable_store(u32 attributes, unsigned long size,
 EXPORT_SYMBOL_GPL(efi_query_variable_store);
 
 /*
+ * The UEFI specification makes it clear that the operating system is
+ * free to do whatever it wants with boot services code after
+ * ExitBootServices() has been called. Ignoring this recommendation a
+ * significant bunch of EFI implementations continue calling into boot
+ * services code (SetVirtualAddressMap). In order to work around such
+ * buggy implementations we reserve boot services region during EFI
+ * init and make sure it stays executable. Then, after
+ * SetVirtualAddressMap(), it is discarded.
+ *
+ * However, some boot services regions contain data that is required
+ * by drivers, so we need to track which memory ranges can never be
+ * freed. This is done by tagging those regions with the
+ * EFI_MEMORY_RUNTIME attribute.
+ *
+ * Any driver that wants to mark a region as reserved must use
+ * efi_mem_reserve() which will insert a new EFI memory descriptor
+ * into efi.memmap (splitting existing regions if necessary) and tag
+ * it with EFI_MEMORY_RUNTIME.
+ */
+void __init efi_arch_mem_reserve(phys_addr_t addr, u64 size)
+{
+        phys_addr_t new_phys, new_size;
+        struct efi_mem_range mr;
+        efi_memory_desc_t md;
+        int num_entries;
+        void *new;
+
+        if (efi_mem_desc_lookup(addr, &md)) {
+                pr_err("Failed to lookup EFI memory descriptor for %pa\n", &addr);
+                return;
+        }
+
+        if (addr + size > md.phys_addr + (md.num_pages << EFI_PAGE_SHIFT)) {
+                pr_err("Region spans EFI memory descriptors, %pa\n", &addr);
+                return;
+        }
+
+        size += addr % EFI_PAGE_SIZE;
+        size = round_up(size, EFI_PAGE_SIZE);
+        addr = round_down(addr, EFI_PAGE_SIZE);
+
+        mr.range.start = addr;
+        mr.range.end = addr + size - 1;
+        mr.attribute = md.attribute | EFI_MEMORY_RUNTIME;
+
+        num_entries = efi_memmap_split_count(&md, &mr.range);
+        num_entries += efi.memmap.nr_map;
+
+        new_size = efi.memmap.desc_size * num_entries;
+
+        new_phys = memblock_alloc(new_size, 0);
+        if (!new_phys) {
+                pr_err("Could not allocate boot services memmap\n");
+                return;
+        }
+
+        new = early_memremap(new_phys, new_size);
+        if (!new) {
+                pr_err("Failed to map new boot services memmap\n");
+                return;
+        }
+
+        efi_memmap_insert(&efi.memmap, new, &mr);
+        early_memunmap(new, new_size);
+
+        efi_memmap_install(new_phys, num_entries);
+}
+
+/*
  * Helper function for efi_reserve_boot_services() to figure out if we
  * can free regions in efi_free_boot_services().
  *
@@ -184,15 +253,6 @@ static bool can_free_region(u64 start, u64 size)
         return true;
 }
 
-/*
- * The UEFI specification makes it clear that the operating system is free to do
- * whatever it wants with boot services code after ExitBootServices() has been
- * called. Ignoring this recommendation a significant bunch of EFI implementations 
- * continue calling into boot services code (SetVirtualAddressMap). In order to 
- * work around such buggy implementations we reserve boot services region during 
- * EFI init and make sure it stays executable. Then, after SetVirtualAddressMap(), it
-* is discarded.
-*/
 void __init efi_reserve_boot_services(void)
 {
         efi_memory_desc_t *md;
@@ -249,7 +309,10 @@ void __init efi_reserve_boot_services(void)
 
 void __init efi_free_boot_services(void)
 {
+        phys_addr_t new_phys, new_size;
         efi_memory_desc_t *md;
+        int num_entries = 0;
+        void *new, *new_md;
 
         for_each_efi_memory_desc(md) {
                 unsigned long long start = md->phys_addr;
@@ -257,12 +320,16 @@ void __init efi_free_boot_services(void)
                 size_t rm_size;
 
                 if (md->type != EFI_BOOT_SERVICES_CODE &&
-                    md->type != EFI_BOOT_SERVICES_DATA)
+                    md->type != EFI_BOOT_SERVICES_DATA) {
+                        num_entries++;
                         continue;
+                }
 
                 /* Do not free, someone else owns it: */
-                if (md->attribute & EFI_MEMORY_RUNTIME)
+                if (md->attribute & EFI_MEMORY_RUNTIME) {
+                        num_entries++;
                         continue;
+                }
 
                 /*
                  * Nasty quirk: if all sub-1MB memory is used for boot
@@ -287,7 +354,41 @@ void __init efi_free_boot_services(void)
                 free_bootmem_late(start, size);
         }
 
-        efi_unmap_memmap();
+        new_size = efi.memmap.desc_size * num_entries;
+        new_phys = memblock_alloc(new_size, 0);
+        if (!new_phys) {
+                pr_err("Failed to allocate new EFI memmap\n");
+                return;
+        }
+
+        new = memremap(new_phys, new_size, MEMREMAP_WB);
+        if (!new) {
+                pr_err("Failed to map new EFI memmap\n");
+                return;
+        }
+
+        /*
+         * Build a new EFI memmap that excludes any boot services
+         * regions that are not tagged EFI_MEMORY_RUNTIME, since those
+         * regions have now been freed.
+         */
+        new_md = new;
+        for_each_efi_memory_desc(md) {
+                if (!(md->attribute & EFI_MEMORY_RUNTIME) &&
+                    (md->type == EFI_BOOT_SERVICES_CODE ||
+                     md->type == EFI_BOOT_SERVICES_DATA))
+                        continue;
+
+                memcpy(new_md, md, efi.memmap.desc_size);
+                new_md += efi.memmap.desc_size;
+        }
+
+        memunmap(new);
+
+        if (efi_memmap_install(new_phys, num_entries)) {
+                pr_err("Could not install new EFI memmap\n");
+                return;
+        }
 }
 
 /*
@@ -365,7 +466,7 @@ void __init efi_apply_memmap_quirks(void)
          */
         if (!efi_runtime_supported()) {
                 pr_info("Setup done, disabling due to 32/64-bit mismatch\n");
-                efi_unmap_memmap();
+                efi_memmap_unmap();
         }
 
         /* UV2+ BIOS has a fix for this issue.  UV1 still needs the quirk. */