12 files changed, 32 insertions, 21 deletions

diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h
index 8719ce122a38..4128bec033f6 100644
--- a/arch/arm64/include/asm/asm-uaccess.h
+++ b/arch/arm64/include/asm/asm-uaccess.h
@@ -14,11 +14,11 @@
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
         .macro  __uaccess_ttbr0_disable, tmp1
         mrs     \tmp1, ttbr1_el1                        // swapper_pg_dir
+        bic     \tmp1, \tmp1, #TTBR_ASID_MASK
         sub     \tmp1, \tmp1, #RESERVED_TTBR0_SIZE      // reserved_ttbr0 just before swapper_pg_dir
         msr     ttbr0_el1, \tmp1                        // set reserved TTBR0_EL1
         isb
         add     \tmp1, \tmp1, #RESERVED_TTBR0_SIZE
-        bic     \tmp1, \tmp1, #TTBR_ASID_MASK
         msr     ttbr1_el1, \tmp1                // set reserved ASID
         isb
         .endm
@@ -35,9 +35,11 @@
         isb
         .endm
 
-        .macro  uaccess_ttbr0_disable, tmp1
+        .macro  uaccess_ttbr0_disable, tmp1, tmp2
 alternative_if_not ARM64_HAS_PAN
+        save_and_disable_irq \tmp2              // avoid preemption
         __uaccess_ttbr0_disable \tmp1
+        restore_irq \tmp2
 alternative_else_nop_endif
         .endm
 
@@ -49,7 +51,7 @@ alternative_if_not ARM64_HAS_PAN
 alternative_else_nop_endif
         .endm
 #else
-        .macro  uaccess_ttbr0_disable, tmp1
+        .macro  uaccess_ttbr0_disable, tmp1, tmp2
         .endm
 
         .macro  uaccess_ttbr0_enable, tmp1, tmp2, tmp3
@@ -59,8 +61,8 @@ alternative_else_nop_endif
 /*
  * These macros are no-ops when UAO is present.
  */
-        .macro  uaccess_disable_not_uao, tmp1
-        uaccess_ttbr0_disable \tmp1
+        .macro  uaccess_disable_not_uao, tmp1, tmp2
+        uaccess_ttbr0_disable \tmp1, \tmp2
 alternative_if ARM64_ALT_PAN_NOT_UAO
         SET_PSTATE_PAN(1)
 alternative_else_nop_endif
diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h
index c4cd5081d78b..8389050328bb 100644
--- a/arch/arm64/include/asm/efi.h
+++ b/arch/arm64/include/asm/efi.h
@@ -121,19 +121,21 @@ static inline void efi_set_pgd(struct mm_struct *mm)
                 if (mm != current->active_mm) {
                         /*
                          * Update the current thread's saved ttbr0 since it is
-                         * restored as part of a return from exception. Set
-                         * the hardware TTBR0_EL1 using cpu_switch_mm()
-                         * directly to enable potential errata workarounds.
+                         * restored as part of a return from exception. Enable
+                         * access to the valid TTBR0_EL1 and invoke the errata
+                         * workaround directly since there is no return from
+                         * exception when invoking the EFI run-time services.
                          */
                         update_saved_ttbr0(current, mm);
-                        cpu_switch_mm(mm->pgd, mm);
+                        uaccess_ttbr0_enable();
+                        post_ttbr_update_workaround();
                 } else {
                         /*
                          * Defer the switch to the current thread's TTBR0_EL1
                          * until uaccess_enable(). Restore the current
                          * thread's saved ttbr0 corresponding to its active_mm
                          */
-                        cpu_set_reserved_ttbr0();
+                        uaccess_ttbr0_disable();
                         update_saved_ttbr0(current, current->active_mm);
                 }
         }
diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index 5aee19905556..8d3331985d2e 100644
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -184,7 +184,7 @@ static inline void update_saved_ttbr0(struct task_struct *tsk,
         else
                 ttbr = virt_to_phys(mm->pgd) | ASID(mm) << 48;
 
-        task_thread_info(tsk)->ttbr0 = ttbr;
+        WRITE_ONCE(task_thread_info(tsk)->ttbr0, ttbr);
 }
 #else
 static inline void update_saved_ttbr0(struct task_struct *tsk,
@@ -239,6 +239,7 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,
 #define activate_mm(prev,next)  switch_mm(prev, next, current)
 
 void verify_cpu_asid_bits(void);
+void post_ttbr_update_workaround(void);
 
 #endif /* !__ASSEMBLY__ */
 
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 01ade20d5456..59fda5292936 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -105,16 +105,18 @@ static inline void set_fs(mm_segment_t fs)
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 static inline void __uaccess_ttbr0_disable(void)
 {
-        unsigned long ttbr;
+        unsigned long flags, ttbr;
 
+        local_irq_save(flags);
         ttbr = read_sysreg(ttbr1_el1);
+        ttbr &= ~TTBR_ASID_MASK;
         /* reserved_ttbr0 placed before swapper_pg_dir */
         write_sysreg(ttbr - RESERVED_TTBR0_SIZE, ttbr0_el1);
         isb();
         /* Set reserved ASID */
-        ttbr &= ~TTBR_ASID_MASK;
         write_sysreg(ttbr, ttbr1_el1);
         isb();
+        local_irq_restore(flags);
 }
 
 static inline void __uaccess_ttbr0_enable(void)
@@ -127,10 +129,11 @@ static inline void __uaccess_ttbr0_enable(void)
          * roll-over and an update of 'ttbr0'.
          */
         local_irq_save(flags);
-        ttbr0 = current_thread_info()->ttbr0;
+        ttbr0 = READ_ONCE(current_thread_info()->ttbr0);
 
         /* Restore active ASID */
         ttbr1 = read_sysreg(ttbr1_el1);
+        ttbr1 &= ~TTBR_ASID_MASK;               /* safety measure */
         ttbr1 |= ttbr0 & TTBR_ASID_MASK;
         write_sysreg(ttbr1, ttbr1_el1);
         isb();
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 1e1d1842888d..b34e717d7597 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -204,7 +204,7 @@ alternative_if ARM64_HAS_PAN
 alternative_else_nop_endif
 
         .if     \el != 0
-        mrs     x21, ttbr1_el1
+        mrs     x21, ttbr0_el1
         tst     x21, #TTBR_ASID_MASK            // Check for the reserved ASID
         orr     x23, x23, #PSR_PAN_BIT          // Set the emulated PAN in the saved SPSR
         b.eq    1f                              // TTBR0 access already disabled
diff --git a/arch/arm64/lib/clear_user.S b/arch/arm64/lib/clear_user.S
index 8f9c4641e706..3d69a8d41fa5 100644
--- a/arch/arm64/lib/clear_user.S
+++ b/arch/arm64/lib/clear_user.S
@@ -50,7 +50,7 @@ uao_user_alternative 9f, strh, sttrh, wzr, x0, 2
         b.mi    5f
 uao_user_alternative 9f, strb, sttrb, wzr, x0, 0
 5:      mov     x0, #0
-        uaccess_disable_not_uao x2
+        uaccess_disable_not_uao x2, x3
         ret
 ENDPROC(__clear_user)
 
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 69d86a80f3e2..20305d485046 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -67,7 +67,7 @@ ENTRY(__arch_copy_from_user)
         uaccess_enable_not_uao x3, x4, x5
         add     end, x0, x2
 #include "copy_template.S"
-        uaccess_disable_not_uao x3
+        uaccess_disable_not_uao x3, x4
         mov     x0, #0                          // Nothing to copy
         ret
 ENDPROC(__arch_copy_from_user)
diff --git a/arch/arm64/lib/copy_in_user.S b/arch/arm64/lib/copy_in_user.S
index e442b531252a..fbb090f431a5 100644
--- a/arch/arm64/lib/copy_in_user.S
+++ b/arch/arm64/lib/copy_in_user.S
@@ -68,7 +68,7 @@ ENTRY(raw_copy_in_user)
         uaccess_enable_not_uao x3, x4, x5
         add     end, x0, x2
 #include "copy_template.S"
-        uaccess_disable_not_uao x3
+        uaccess_disable_not_uao x3, x4
         mov     x0, #0
         ret
 ENDPROC(raw_copy_in_user)
diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
index 318f15d5c336..fda6172d6b88 100644
--- a/arch/arm64/lib/copy_to_user.S
+++ b/arch/arm64/lib/copy_to_user.S
@@ -66,7 +66,7 @@ ENTRY(__arch_copy_to_user)
         uaccess_enable_not_uao x3, x4, x5
         add     end, x0, x2
 #include "copy_template.S"
-        uaccess_disable_not_uao x3
+        uaccess_disable_not_uao x3, x4
         mov     x0, #0
         ret
 ENDPROC(__arch_copy_to_user)
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index 6cd20a8c0952..91464e7f77cc 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -72,7 +72,7 @@ USER(9f, ic	ivau, x4	)		// invalidate I line PoU
         isb
         mov     x0, #0
 1:
-        uaccess_ttbr0_disable x1
+        uaccess_ttbr0_disable x1, x2
         ret
 9:
         mov     x0, #-EFAULT
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 05e4ae934b23..c6a12073ef46 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -153,6 +153,9 @@ ENDPROC(cpu_do_resume)
 ENTRY(cpu_do_switch_mm)
         mrs     x2, ttbr1_el1
         mmid    x1, x1                          // get mm->context.id
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+        bfi     x0, x1, #48, #16                // set the ASID field in TTBR0
+#endif
         bfi     x2, x1, #48, #16                // set the ASID
         msr     ttbr1_el1, x2                   // in TTBR1 (since TCR.A1 is set)
         isb
diff --git a/arch/arm64/xen/hypercall.S b/arch/arm64/xen/hypercall.S
index acdbd2c9e899..c5f05c4a4d00 100644
--- a/arch/arm64/xen/hypercall.S
+++ b/arch/arm64/xen/hypercall.S
@@ -107,6 +107,6 @@ ENTRY(privcmd_call)
         /*
          * Disable userspace access from kernel once the hyp call completed.
          */
-        uaccess_ttbr0_disable x6
+        uaccess_ttbr0_disable x6, x7
         ret
 ENDPROC(privcmd_call);