6 files changed, 42 insertions, 8 deletions

diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h
index e203169931c7..6390bd8c141b 100644
--- a/arch/x86/include/asm/fixmap.h
+++ b/arch/x86/include/asm/fixmap.h
@@ -14,6 +14,16 @@
 #ifndef _ASM_X86_FIXMAP_H
 #define _ASM_X86_FIXMAP_H
 
+/*
+ * Exposed to assembly code for setting up initial page tables. Cannot be
+ * calculated in assembly code (fixmap entries are an enum), but is sanity
+ * checked in the actual fixmap C code to make sure that the fixmap is
+ * covered fully.
+ */
+#define FIXMAP_PMD_NUM  2
+/* fixmap starts downwards from the 507th entry in level2_fixmap_pgt */
+#define FIXMAP_PMD_TOP  507
+
 #ifndef __ASSEMBLY__
 #include <linux/kernel.h>
 #include <asm/acpi.h>
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index ce2b59047cb8..9c85b54bf03c 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -14,6 +14,7 @@
 #include <asm/processor.h>
 #include <linux/bitops.h>
 #include <linux/threads.h>
+#include <asm/fixmap.h>
 
 extern p4d_t level4_kernel_pgt[512];
 extern p4d_t level4_ident_pgt[512];
@@ -22,7 +23,7 @@ extern pud_t level3_ident_pgt[512];
 extern pmd_t level2_kernel_pgt[512];
 extern pmd_t level2_fixmap_pgt[512];
 extern pmd_t level2_ident_pgt[512];
-extern pte_t level1_fixmap_pgt[512];
+extern pte_t level1_fixmap_pgt[512 * FIXMAP_PMD_NUM];
 extern pgd_t init_top_pgt[];
 
 #define swapper_pg_dir init_top_pgt
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index c16af27eb23f..ddee1f0870c4 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -35,6 +35,7 @@
 #include <asm/bootparam_utils.h>
 #include <asm/microcode.h>
 #include <asm/kasan.h>
+#include <asm/fixmap.h>
 
 /*
  * Manage page tables very early on.
@@ -166,7 +167,8 @@ unsigned long __head __startup_64(unsigned long physaddr,
         pud[511] += load_delta;
 
         pmd = fixup_pointer(level2_fixmap_pgt, physaddr);
-        pmd[506] += load_delta;
+        for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
+                pmd[i] += load_delta;
 
         /*
          * Set up the identity mapping for the switchover.  These
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 15ebc2fc166e..a3618cf04cf6 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -24,6 +24,7 @@
 #include "../entry/calling.h"
 #include <asm/export.h>
 #include <asm/nospec-branch.h>
+#include <asm/fixmap.h>
 
 #ifdef CONFIG_PARAVIRT
 #include <asm/asm-offsets.h>
@@ -445,13 +446,20 @@ NEXT_PAGE(level2_kernel_pgt)
                 KERNEL_IMAGE_SIZE/PMD_SIZE)
 
 NEXT_PAGE(level2_fixmap_pgt)
-        .fill   506,8,0
-        .quad   level1_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC
-        /* 8MB reserved for vsyscalls + a 2MB hole = 4 + 1 entries */
-        .fill   5,8,0
+        .fill   (512 - 4 - FIXMAP_PMD_NUM),8,0
+        pgtno = 0
+        .rept (FIXMAP_PMD_NUM)
+        .quad level1_fixmap_pgt + (pgtno << PAGE_SHIFT) - __START_KERNEL_map \
+                + _PAGE_TABLE_NOENC;
+        pgtno = pgtno + 1
+        .endr
+        /* 6 MB reserved space + a 2MB hole */
+        .fill   4,8,0
 
 NEXT_PAGE(level1_fixmap_pgt)
+        .rept (FIXMAP_PMD_NUM)
         .fill   512,8,0
+        .endr
 
 #undef PMDS
 
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index ae394552fb94..089e78c4effd 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -637,6 +637,15 @@ void __native_set_fixmap(enum fixed_addresses idx, pte_t pte)
 {
         unsigned long address = __fix_to_virt(idx);
 
+#ifdef CONFIG_X86_64
+       /*
+        * Ensure that the static initial page tables are covering the
+        * fixmap completely.
+        */
+        BUILD_BUG_ON(__end_of_permanent_fixed_addresses >
+                     (FIXMAP_PMD_NUM * PTRS_PER_PTE));
+#endif
+
         if (idx >= __end_of_fixed_addresses) {
                 BUG();
                 return;
diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c
index 2fe5c9b1816b..dd461c0167ef 100644
--- a/arch/x86/xen/mmu_pv.c
+++ b/arch/x86/xen/mmu_pv.c
@@ -1907,7 +1907,7 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
         /* L3_k[511] -> level2_fixmap_pgt */
         convert_pfn_mfn(level3_kernel_pgt);
 
-        /* L3_k[511][506] -> level1_fixmap_pgt */
+        /* L3_k[511][508-FIXMAP_PMD_NUM ... 507] -> level1_fixmap_pgt */
         convert_pfn_mfn(level2_fixmap_pgt);
 
         /* We get [511][511] and have Xen's version of level2_kernel_pgt */
@@ -1952,7 +1952,11 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
         set_page_prot(level2_ident_pgt, PAGE_KERNEL_RO);
         set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
         set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
-        set_page_prot(level1_fixmap_pgt, PAGE_KERNEL_RO);
+
+        for (i = 0; i < FIXMAP_PMD_NUM; i++) {
+                set_page_prot(level1_fixmap_pgt + i * PTRS_PER_PTE,
+                              PAGE_KERNEL_RO);
+        }
 
         /* Pin down new L4 */
         pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,