x86: Move do_page_fault()'s error path under unlikely()

Ingo suggested SIGKILL check should be moved into slowpath
function. This will reduce the page fault fastpath impact
of this recent commit:

  37b23e0525d3: x86,mm: make pagefault killable

Suggested-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: kamezawa.hiroyu@jp.fujitsu.com
Cc: minchan.kim@gmail.com
Cc: willy@linux.intel.com
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/4DDE0B5C.9050907@jp.fujitsu.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 20 insertions, 15 deletions

diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index f7a2a054a3c..2dbf6bf4c7e 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -823,16 +823,30 @@ do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address,
         force_sig_info_fault(SIGBUS, code, address, tsk, fault);
 }
 
-static noinline void
+static noinline int
 mm_fault_error(struct pt_regs *regs, unsigned long error_code,
                unsigned long address, unsigned int fault)
 {
+        /*
+         * Pagefault was interrupted by SIGKILL. We have no reason to
+         * continue pagefault.
+         */
+        if (fatal_signal_pending(current)) {
+                if (!(fault & VM_FAULT_RETRY))
+                        up_read(&current->mm->mmap_sem);
+                if (!(error_code & PF_USER))
+                        no_context(regs, error_code, address);
+                return 1;
+        }
+        if (!(fault & VM_FAULT_ERROR))
+                return 0;
+
         if (fault & VM_FAULT_OOM) {
                 /* Kernel mode? Handle exceptions or die: */
                 if (!(error_code & PF_USER)) {
                         up_read(&current->mm->mmap_sem);
                         no_context(regs, error_code, address);
-                        return;
+                        return 1;
                 }
 
                 out_of_memory(regs, error_code, address);
@@ -843,6 +857,7 @@ mm_fault_error(struct pt_regs *regs, unsigned long error_code,
                 else
                         BUG();
         }
+        return 1;
 }
 
 static int spurious_fault_check(unsigned long error_code, pte_t *pte)
@@ -1133,19 +1148,9 @@ good_area:
          */
         fault = handle_mm_fault(mm, vma, address, flags);
 
-        if (unlikely(fault & VM_FAULT_ERROR)) {
-                mm_fault_error(regs, error_code, address, fault);
-                return;
-        }
-
-        /*
-         * Pagefault was interrupted by SIGKILL. We have no reason to
-         * continue pagefault.
-         */
-        if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
-                if (!(error_code & PF_USER))
-                        no_context(regs, error_code, address);
-                return;
+        if (unlikely(fault & (VM_FAULT_RETRY|VM_FAULT_ERROR))) {
+                if (mm_fault_error(regs, error_code, address, fault))
+                        return;
         }
 
         /*