1 files changed, 62 insertions, 39 deletions

diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index cd08f4fef836..c7e32f453852 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -1032,7 +1032,7 @@ mm_fault_error(struct pt_regs *regs, unsigned long error_code,
         }
 }
 
-static int spurious_fault_check(unsigned long error_code, pte_t *pte)
+static int spurious_kernel_fault_check(unsigned long error_code, pte_t *pte)
 {
         if ((error_code & X86_PF_WRITE) && !pte_write(*pte))
                 return 0;
@@ -1071,7 +1071,7 @@ static int spurious_fault_check(unsigned long error_code, pte_t *pte)
  * (Optional Invalidation).
  */
 static noinline int
-spurious_fault(unsigned long error_code, unsigned long address)
+spurious_kernel_fault(unsigned long error_code, unsigned long address)
 {
         pgd_t *pgd;
         p4d_t *p4d;
@@ -1102,27 +1102,27 @@ spurious_fault(unsigned long error_code, unsigned long address)
                 return 0;
 
         if (p4d_large(*p4d))
-                return spurious_fault_check(error_code, (pte_t *) p4d);
+                return spurious_kernel_fault_check(error_code, (pte_t *) p4d);
 
         pud = pud_offset(p4d, address);
         if (!pud_present(*pud))
                 return 0;
 
         if (pud_large(*pud))
-                return spurious_fault_check(error_code, (pte_t *) pud);
+                return spurious_kernel_fault_check(error_code, (pte_t *) pud);
 
         pmd = pmd_offset(pud, address);
         if (!pmd_present(*pmd))
                 return 0;
 
         if (pmd_large(*pmd))
-                return spurious_fault_check(error_code, (pte_t *) pmd);
+                return spurious_kernel_fault_check(error_code, (pte_t *) pmd);
 
         pte = pte_offset_kernel(pmd, address);
         if (!pte_present(*pte))
                 return 0;
 
-        ret = spurious_fault_check(error_code, pte);
+        ret = spurious_kernel_fault_check(error_code, pte);
         if (!ret)
                 return 0;
 
@@ -1130,12 +1130,12 @@ spurious_fault(unsigned long error_code, unsigned long address)
          * Make sure we have permissions in PMD.
          * If not, then there's a bug in the page tables:
          */
-        ret = spurious_fault_check(error_code, (pte_t *) pmd);
+        ret = spurious_kernel_fault_check(error_code, (pte_t *) pmd);
         WARN_ONCE(!ret, "PMD has incorrect permission bits\n");
 
         return ret;
 }
-NOKPROBE_SYMBOL(spurious_fault);
+NOKPROBE_SYMBOL(spurious_kernel_fault);
 
 int show_unhandled_signals = 1;
 
@@ -1203,6 +1203,58 @@ static inline bool smap_violation(int error_code, struct pt_regs *regs)
 }
 
 /*
+ * Called for all faults where 'address' is part of the kernel address
+ * space.  Might get called for faults that originate from *code* that
+ * ran in userspace or the kernel.
+ */
+static void
+do_kern_addr_fault(struct pt_regs *regs, unsigned long hw_error_code,
+                   unsigned long address)
+{
+        /*
+         * We can fault-in kernel-space virtual memory on-demand. The
+         * 'reference' page table is init_mm.pgd.
+         *
+         * NOTE! We MUST NOT take any locks for this case. We may
+         * be in an interrupt or a critical region, and should
+         * only copy the information from the master page table,
+         * nothing more.
+         *
+         * Before doing this on-demand faulting, ensure that the
+         * fault is not any of the following:
+         * 1. A fault on a PTE with a reserved bit set.
+         * 2. A fault caused by a user-mode access.  (Do not demand-
+         *    fault kernel memory due to user-mode accesses).
+         * 3. A fault caused by a page-level protection violation.
+         *    (A demand fault would be on a non-present page which
+         *     would have X86_PF_PROT==0).
+         */
+        if (!(hw_error_code & (X86_PF_RSVD | X86_PF_USER | X86_PF_PROT))) {
+                if (vmalloc_fault(address) >= 0)
+                        return;
+        }
+
+        /* Was the fault spurious, caused by lazy TLB invalidation? */
+        if (spurious_kernel_fault(hw_error_code, address))
+                return;
+
+        /* kprobes don't want to hook the spurious faults: */
+        if (kprobes_fault(regs))
+                return;
+
+        /*
+         * Note, despite being a "bad area", there are quite a few
+         * acceptable reasons to get here, such as erratum fixups
+         * and handling kernel code that can fault, like get_user().
+         *
+         * Don't take the mm semaphore here. If we fixup a prefetch
+         * fault we could otherwise deadlock:
+         */
+        bad_area_nosemaphore(regs, hw_error_code, address, NULL);
+}
+NOKPROBE_SYMBOL(do_kern_addr_fault);
+
+/*
  * This routine handles page faults.  It determines the address,
  * and the problem, and then passes it off to one of the appropriate
  * routines.
@@ -1227,38 +1279,9 @@ __do_page_fault(struct pt_regs *regs, unsigned long hw_error_code,
         if (unlikely(kmmio_fault(regs, address)))
                 return;
 
-        /*
-         * We fault-in kernel-space virtual memory on-demand. The
-         * 'reference' page table is init_mm.pgd.
-         *
-         * NOTE! We MUST NOT take any locks for this case. We may
-         * be in an interrupt or a critical region, and should
-         * only copy the information from the master page table,
-         * nothing more.
-         *
-         * This verifies that the fault happens in kernel space
-         * (hw_error_code & 4) == 0, and that the fault was not a
-         * protection error (hw_error_code & 9) == 0.
-         */
+        /* Was the fault on kernel-controlled part of the address space? */
         if (unlikely(fault_in_kernel_space(address))) {
-                if (!(hw_error_code & (X86_PF_RSVD | X86_PF_USER | X86_PF_PROT))) {
-                        if (vmalloc_fault(address) >= 0)
-                                return;
-                }
-
-                /* Can handle a stale RO->RW TLB: */
-                if (spurious_fault(hw_error_code, address))
-                        return;
-
-                /* kprobes don't want to hook the spurious faults: */
-                if (kprobes_fault(regs))
-                        return;
-                /*
-                 * Don't take the mm semaphore here. If we fixup a prefetch
-                 * fault we could otherwise deadlock:
-                 */
-                bad_area_nosemaphore(regs, hw_error_code, address, NULL);
-
+                do_kern_addr_fault(regs, hw_error_code, address);
                 return;
         }