1 files changed, 46 insertions, 62 deletions

diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index e2baeaa053a5..febf6980e653 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *  Copyright (C) 1995  Linus Torvalds
  *  Copyright (C) 2001, 2002 Andi Kleen, SuSE Labs.
@@ -19,7 +20,6 @@
 #include <asm/cpufeature.h>             /* boot_cpu_has, ...            */
 #include <asm/traps.h>                  /* dotraplinkage, ...           */
 #include <asm/pgalloc.h>                /* pgd_*(), ...                 */
-#include <asm/kmemcheck.h>              /* kmemcheck_*(), ...           */
 #include <asm/fixmap.h>                 /* VSYSCALL_ADDR                */
 #include <asm/vsyscall.h>               /* emulate_vsyscall             */
 #include <asm/vm86.h>                   /* struct vm86                  */
@@ -29,26 +29,6 @@
 #include <asm/trace/exceptions.h>
 
 /*
- * Page fault error code bits:
- *
- *   bit 0 ==    0: no page found       1: protection fault
- *   bit 1 ==    0: read access         1: write access
- *   bit 2 ==    0: kernel-mode access  1: user-mode access
- *   bit 3 ==                           1: use of reserved bit detected
- *   bit 4 ==                           1: fault was an instruction fetch
- *   bit 5 ==                           1: protection keys block access
- */
-enum x86_pf_error_code {
-
-        PF_PROT         =               1 << 0,
-        PF_WRITE        =               1 << 1,
-        PF_USER         =               1 << 2,
-        PF_RSVD         =               1 << 3,
-        PF_INSTR        =               1 << 4,
-        PF_PK           =               1 << 5,
-};
-
-/*
  * Returns 0 if mmiotrace is disabled, or if the fault is not
  * handled by mmiotrace:
  */
@@ -149,7 +129,7 @@ is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
          * If it was a exec (instruction fetch) fault on NX page, then
          * do not ignore the fault:
          */
-        if (error_code & PF_INSTR)
+        if (error_code & X86_PF_INSTR)
                 return 0;
 
         instr = (void *)convert_ip_to_linear(current, regs);
@@ -179,7 +159,7 @@ is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
  * siginfo so userspace can discover which protection key was set
  * on the PTE.
  *
- * If we get here, we know that the hardware signaled a PF_PK
+ * If we get here, we know that the hardware signaled a X86_PF_PK
  * fault and that there was a VMA once we got in the fault
  * handler.  It does *not* guarantee that the VMA we find here
  * was the one that we faulted on.
@@ -204,7 +184,7 @@ static void fill_sig_info_pkey(int si_code, siginfo_t *info, u32 *pkey)
         /*
          * force_sig_info_fault() is called from a number of
          * contexts, some of which have a VMA and some of which
-         * do not.  The PF_PK handing happens after we have a
+         * do not.  The X86_PF_PK handing happens after we have a
          * valid VMA, so we should never reach this without a
          * valid VMA.
          */
@@ -697,7 +677,7 @@ show_fault_oops(struct pt_regs *regs, unsigned long error_code,
         if (!oops_may_print())
                 return;
 
-        if (error_code & PF_INSTR) {
+        if (error_code & X86_PF_INSTR) {
                 unsigned int level;
                 pgd_t *pgd;
                 pte_t *pte;
@@ -721,7 +701,7 @@ show_fault_oops(struct pt_regs *regs, unsigned long error_code,
         else
                 printk(KERN_CONT "paging request");
 
-        printk(KERN_CONT " at %p\n", (void *) address);
+        printk(KERN_CONT " at %px\n", (void *) address);
         printk(KERN_ALERT "IP: %pS\n", (void *)regs->ip);
 
         dump_pagetable(address);
@@ -779,7 +759,7 @@ no_context(struct pt_regs *regs, unsigned long error_code,
                  */
                 if (current->thread.sig_on_uaccess_err && signal) {
                         tsk->thread.trap_nr = X86_TRAP_PF;
-                        tsk->thread.error_code = error_code | PF_USER;
+                        tsk->thread.error_code = error_code | X86_PF_USER;
                         tsk->thread.cr2 = address;
 
                         /* XXX: hwpoison faults will set the wrong code. */
@@ -897,7 +877,7 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
         struct task_struct *tsk = current;
 
         /* User mode accesses just cause a SIGSEGV */
-        if (error_code & PF_USER) {
+        if (error_code & X86_PF_USER) {
                 /*
                  * It's possible to have interrupts off here:
                  */
@@ -918,7 +898,7 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
                  * Instruction fetch faults in the vsyscall page might need
                  * emulation.
                  */
-                if (unlikely((error_code & PF_INSTR) &&
+                if (unlikely((error_code & X86_PF_INSTR) &&
                              ((address & ~0xfff) == VSYSCALL_ADDR))) {
                         if (emulate_vsyscall(regs, address))
                                 return;
@@ -931,7 +911,7 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
                  * are always protection faults.
                  */
                 if (address >= TASK_SIZE_MAX)
-                        error_code |= PF_PROT;
+                        error_code |= X86_PF_PROT;
 
                 if (likely(show_unhandled_signals))
                         show_signal_msg(regs, error_code, address, tsk);
@@ -992,11 +972,11 @@ static inline bool bad_area_access_from_pkeys(unsigned long error_code,
 
         if (!boot_cpu_has(X86_FEATURE_OSPKE))
                 return false;
-        if (error_code & PF_PK)
+        if (error_code & X86_PF_PK)
                 return true;
         /* this checks permission keys on the VMA: */
-        if (!arch_vma_access_permitted(vma, (error_code & PF_WRITE),
-                                (error_code & PF_INSTR), foreign))
+        if (!arch_vma_access_permitted(vma, (error_code & X86_PF_WRITE),
+                                       (error_code & X86_PF_INSTR), foreign))
                 return true;
         return false;
 }
@@ -1024,7 +1004,7 @@ do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
         int code = BUS_ADRERR;
 
         /* Kernel mode? Handle exceptions or die: */
-        if (!(error_code & PF_USER)) {
+        if (!(error_code & X86_PF_USER)) {
                 no_context(regs, error_code, address, SIGBUS, BUS_ADRERR);
                 return;
         }
@@ -1052,14 +1032,14 @@ static noinline void
 mm_fault_error(struct pt_regs *regs, unsigned long error_code,
                unsigned long address, u32 *pkey, unsigned int fault)
 {
-        if (fatal_signal_pending(current) && !(error_code & PF_USER)) {
+        if (fatal_signal_pending(current) && !(error_code & X86_PF_USER)) {
                 no_context(regs, error_code, address, 0, 0);
                 return;
         }
 
         if (fault & VM_FAULT_OOM) {
                 /* Kernel mode? Handle exceptions or die: */
-                if (!(error_code & PF_USER)) {
+                if (!(error_code & X86_PF_USER)) {
                         no_context(regs, error_code, address,
                                    SIGSEGV, SEGV_MAPERR);
                         return;
@@ -1084,16 +1064,16 @@ mm_fault_error(struct pt_regs *regs, unsigned long error_code,
 
 static int spurious_fault_check(unsigned long error_code, pte_t *pte)
 {
-        if ((error_code & PF_WRITE) && !pte_write(*pte))
+        if ((error_code & X86_PF_WRITE) && !pte_write(*pte))
                 return 0;
 
-        if ((error_code & PF_INSTR) && !pte_exec(*pte))
+        if ((error_code & X86_PF_INSTR) && !pte_exec(*pte))
                 return 0;
         /*
          * Note: We do not do lazy flushing on protection key
-         * changes, so no spurious fault will ever set PF_PK.
+         * changes, so no spurious fault will ever set X86_PF_PK.
          */
-        if ((error_code & PF_PK))
+        if ((error_code & X86_PF_PK))
                 return 1;
 
         return 1;
@@ -1139,8 +1119,8 @@ spurious_fault(unsigned long error_code, unsigned long address)
          * change, so user accesses are not expected to cause spurious
          * faults.
          */
-        if (error_code != (PF_WRITE | PF_PROT)
-            && error_code != (PF_INSTR | PF_PROT))
+        if (error_code != (X86_PF_WRITE | X86_PF_PROT) &&
+            error_code != (X86_PF_INSTR | X86_PF_PROT))
                 return 0;
 
         pgd = init_mm.pgd + pgd_index(address);
@@ -1200,19 +1180,19 @@ access_error(unsigned long error_code, struct vm_area_struct *vma)
          * always an unconditional error and can never result in
          * a follow-up action to resolve the fault, like a COW.
          */
-        if (error_code & PF_PK)
+        if (error_code & X86_PF_PK)
                 return 1;
 
         /*
          * Make sure to check the VMA so that we do not perform
-         * faults just to hit a PF_PK as soon as we fill in a
+         * faults just to hit a X86_PF_PK as soon as we fill in a
          * page.
          */
-        if (!arch_vma_access_permitted(vma, (error_code & PF_WRITE),
-                                (error_code & PF_INSTR), foreign))
+        if (!arch_vma_access_permitted(vma, (error_code & X86_PF_WRITE),
+                                       (error_code & X86_PF_INSTR), foreign))
                 return 1;
 
-        if (error_code & PF_WRITE) {
+        if (error_code & X86_PF_WRITE) {
                 /* write, present and write, not present: */
                 if (unlikely(!(vma->vm_flags & VM_WRITE)))
                         return 1;
@@ -1220,7 +1200,7 @@ access_error(unsigned long error_code, struct vm_area_struct *vma)
         }
 
         /* read, present: */
-        if (unlikely(error_code & PF_PROT))
+        if (unlikely(error_code & X86_PF_PROT))
                 return 1;
 
         /* read, not present: */
@@ -1243,7 +1223,7 @@ static inline bool smap_violation(int error_code, struct pt_regs *regs)
         if (!static_cpu_has(X86_FEATURE_SMAP))
                 return false;
 
-        if (error_code & PF_USER)
+        if (error_code & X86_PF_USER)
                 return false;
 
         if (!user_mode(regs) && (regs->flags & X86_EFLAGS_AC))
@@ -1275,8 +1255,6 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
          * Detect and handle instructions that would cause a page fault for
          * both a tracked kernel page and a userspace page.
          */
-        if (kmemcheck_active(regs))
-                kmemcheck_hide(regs);
         prefetchw(&mm->mmap_sem);
 
         if (unlikely(kmmio_fault(regs, address)))
@@ -1296,12 +1274,9 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
          * protection error (error_code & 9) == 0.
          */
         if (unlikely(fault_in_kernel_space(address))) {
-                if (!(error_code & (PF_RSVD | PF_USER | PF_PROT))) {
+                if (!(error_code & (X86_PF_RSVD | X86_PF_USER | X86_PF_PROT))) {
                         if (vmalloc_fault(address) >= 0)
                                 return;
-
-                        if (kmemcheck_fault(regs, address, error_code))
-                                return;
                 }
 
                 /* Can handle a stale RO->RW TLB: */
@@ -1324,7 +1299,7 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
         if (unlikely(kprobes_fault(regs)))
                 return;
 
-        if (unlikely(error_code & PF_RSVD))
+        if (unlikely(error_code & X86_PF_RSVD))
                 pgtable_bad(regs, error_code, address);
 
         if (unlikely(smap_violation(error_code, regs))) {
@@ -1350,7 +1325,7 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
          */
         if (user_mode(regs)) {
                 local_irq_enable();
-                error_code |= PF_USER;
+                error_code |= X86_PF_USER;
                 flags |= FAULT_FLAG_USER;
         } else {
                 if (regs->flags & X86_EFLAGS_IF)
@@ -1359,9 +1334,9 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
 
         perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
 
-        if (error_code & PF_WRITE)
+        if (error_code & X86_PF_WRITE)
                 flags |= FAULT_FLAG_WRITE;
-        if (error_code & PF_INSTR)
+        if (error_code & X86_PF_INSTR)
                 flags |= FAULT_FLAG_INSTRUCTION;
 
         /*
@@ -1381,7 +1356,7 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
          * space check, thus avoiding the deadlock:
          */
         if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
-                if ((error_code & PF_USER) == 0 &&
+                if (!(error_code & X86_PF_USER) &&
                     !search_exception_tables(regs->ip)) {
                         bad_area_nosemaphore(regs, error_code, address, NULL);
                         return;
@@ -1408,7 +1383,7 @@ retry:
                 bad_area(regs, error_code, address);
                 return;
         }
-        if (error_code & PF_USER) {
+        if (error_code & X86_PF_USER) {
                 /*
                  * Accessing the stack below %sp is always a bug.
                  * The large cushion allows instructions like enter
@@ -1440,7 +1415,17 @@ good_area:
          * make sure we exit gracefully rather than endlessly redo
          * the fault.  Since we never set FAULT_FLAG_RETRY_NOWAIT, if
          * we get VM_FAULT_RETRY back, the mmap_sem has been unlocked.
+         *
+         * Note that handle_userfault() may also release and reacquire mmap_sem
+         * (and not return with VM_FAULT_RETRY), when returning to userland to
+         * repeat the page fault later with a VM_FAULT_NOPAGE retval
+         * (potentially after handling any pending signal during the return to
+         * userland). The return to userland is identified whenever
+         * FAULT_FLAG_USER|FAULT_FLAG_KILLABLE are both set in flags.
+         * Thus we have to be careful about not touching vma after handling the
+         * fault, so we read the pkey beforehand.
          */
+        pkey = vma_pkey(vma);
         fault = handle_mm_fault(vma, address, flags);
         major |= fault & VM_FAULT_MAJOR;
 
@@ -1467,7 +1452,6 @@ good_area:
                 return;
         }
 
-        pkey = vma_pkey(vma);
         up_read(&mm->mmap_sem);
         if (unlikely(fault & VM_FAULT_ERROR)) {
                 mm_fault_error(regs, error_code, address, &pkey, fault);