8 files changed, 59 insertions, 4 deletions

diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index 7dede2e34b70..ccf00a8b98c6 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -876,6 +876,23 @@ static inline int pmd_present(pmd_t pmd)
         return false;
 }
 
+static inline int pmd_is_serializing(pmd_t pmd)
+{
+        /*
+         * If the pmd is undergoing a split, the _PAGE_PRESENT bit is clear
+         * and _PAGE_INVALID is set (see pmd_present, pmdp_invalidate).
+         *
+         * This condition may also occur when flushing a pmd while flushing
+         * it (see ptep_modify_prot_start), so callers must ensure this
+         * case is fine as well.
+         */
+        if ((pmd_raw(pmd) & cpu_to_be64(_PAGE_PRESENT | _PAGE_INVALID)) ==
+                                                cpu_to_be64(_PAGE_INVALID))
+                return true;
+
+        return false;
+}
+
 static inline int pmd_bad(pmd_t pmd)
 {
         if (radix_enabled())
@@ -1092,6 +1109,19 @@ static inline int pmd_protnone(pmd_t pmd)
 #define pmd_access_permitted pmd_access_permitted
 static inline bool pmd_access_permitted(pmd_t pmd, bool write)
 {
+        /*
+         * pmdp_invalidate sets this combination (which is not caught by
+         * !pte_present() check in pte_access_permitted), to prevent
+         * lock-free lookups, as part of the serialize_against_pte_lookup()
+         * synchronisation.
+         *
+         * This also catches the case where the PTE's hardware PRESENT bit is
+         * cleared while TLB is flushed, which is suboptimal but should not
+         * be frequent.
+         */
+        if (pmd_is_serializing(pmd))
+                return false;
+
         return pte_access_permitted(pmd_pte(pmd), write);
 }
 
diff --git a/arch/powerpc/include/asm/btext.h b/arch/powerpc/include/asm/btext.h
index 3ffad030393c..461b0f193864 100644
--- a/arch/powerpc/include/asm/btext.h
+++ b/arch/powerpc/include/asm/btext.h
@@ -13,7 +13,11 @@ extern void btext_update_display(unsigned long phys, int width, int height,
                                  int depth, int pitch);
 extern void btext_setup_display(int width, int height, int depth, int pitch,
                                 unsigned long address);
+#ifdef CONFIG_PPC32
 extern void btext_prepare_BAT(void);
+#else
+static inline void btext_prepare_BAT(void) { }
+#endif
 extern void btext_map(void);
 extern void btext_unmap(void);
 
diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h
index 4a585cba1787..c68476818753 100644
--- a/arch/powerpc/include/asm/kexec.h
+++ b/arch/powerpc/include/asm/kexec.h
@@ -94,6 +94,9 @@ static inline bool kdump_in_progress(void)
         return crashing_cpu >= 0;
 }
 
+void relocate_new_kernel(unsigned long indirection_page, unsigned long reboot_code_buffer,
+                         unsigned long start_address) __noreturn;
+
 #ifdef CONFIG_KEXEC_FILE
 extern const struct kexec_file_ops kexec_elf64_ops;
 
diff --git a/arch/powerpc/kernel/machine_kexec_32.c b/arch/powerpc/kernel/machine_kexec_32.c
index affe5dcce7f4..2b160d68db49 100644
--- a/arch/powerpc/kernel/machine_kexec_32.c
+++ b/arch/powerpc/kernel/machine_kexec_32.c
@@ -30,7 +30,6 @@ typedef void (*relocate_new_kernel_t)(
  */
 void default_machine_kexec(struct kimage *image)
 {
-        extern const unsigned char relocate_new_kernel[];
         extern const unsigned int relocate_new_kernel_size;
         unsigned long page_list;
         unsigned long reboot_code_buffer, reboot_code_buffer_phys;
@@ -58,6 +57,9 @@ void default_machine_kexec(struct kimage *image)
                                 reboot_code_buffer + KEXEC_CONTROL_PAGE_SIZE);
         printk(KERN_INFO "Bye!\n");
 
+        if (!IS_ENABLED(CONFIG_FSL_BOOKE) && !IS_ENABLED(CONFIG_44x))
+                relocate_new_kernel(page_list, reboot_code_buffer_phys, image->start);
+
         /* now call it */
         rnk = (relocate_new_kernel_t) reboot_code_buffer;
         (*rnk)(page_list, reboot_code_buffer_phys, image->start);
diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c
index 3555cad7bdde..ed446b7ea164 100644
--- a/arch/powerpc/kernel/prom_init.c
+++ b/arch/powerpc/kernel/prom_init.c
@@ -2336,6 +2336,7 @@ static void __init prom_check_displays(void)
                         prom_printf("W=%d H=%d LB=%d addr=0x%x\n",
                                     width, height, pitch, addr);
                         btext_setup_display(width, height, 8, pitch, addr);
+                        btext_prepare_BAT();
                 }
 #endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */
         }
diff --git a/arch/powerpc/kernel/prom_init_check.sh b/arch/powerpc/kernel/prom_init_check.sh
index 518d416971c1..160bef0d553d 100644
--- a/arch/powerpc/kernel/prom_init_check.sh
+++ b/arch/powerpc/kernel/prom_init_check.sh
@@ -24,7 +24,7 @@ fi
 WHITELIST="add_reloc_offset __bss_start __bss_stop copy_and_flush
 _end enter_prom $MEM_FUNCS reloc_offset __secondary_hold
 __secondary_hold_acknowledge __secondary_hold_spinloop __start
-logo_linux_clut224
+logo_linux_clut224 btext_prepare_BAT
 reloc_got2 kernstart_addr memstart_addr linux_banner _stext
 __prom_init_toc_start __prom_init_toc_end btext_setup_display TOC."
 
diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c
index a255707e4aee..01bc9663360d 100644
--- a/arch/powerpc/mm/book3s64/pgtable.c
+++ b/arch/powerpc/mm/book3s64/pgtable.c
@@ -112,6 +112,9 @@ pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
         /*
          * This ensures that generic code that rely on IRQ disabling
          * to prevent a parallel THP split work as expected.
+         *
+         * Marking the entry with _PAGE_INVALID && ~_PAGE_PRESENT requires
+         * a special case check in pmd_access_permitted.
          */
         serialize_against_pte_lookup(vma->vm_mm);
         return __pmd(old_pmd);
diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c
index 39d2f8012386..fc10c0c24f51 100644
--- a/arch/powerpc/mm/pgtable.c
+++ b/arch/powerpc/mm/pgtable.c
@@ -368,13 +368,25 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,
         pdshift = PMD_SHIFT;
         pmdp = pmd_offset(&pud, ea);
         pmd  = READ_ONCE(*pmdp);
+
         /*
-         * A hugepage collapse is captured by pmd_none, because
-         * it mark the pmd none and do a hpte invalidate.
+         * A hugepage collapse is captured by this condition, see
+         * pmdp_collapse_flush.
          */
         if (pmd_none(pmd))
                 return NULL;
 
+#ifdef CONFIG_PPC_BOOK3S_64
+        /*
+         * A hugepage split is captured by this condition, see
+         * pmdp_invalidate.
+         *
+         * Huge page modification can be caught here too.
+         */
+        if (pmd_is_serializing(pmd))
+                return NULL;
+#endif
+
         if (pmd_trans_huge(pmd) || pmd_devmap(pmd)) {
                 if (is_thp)
                         *is_thp = true;