2 files changed, 15 insertions, 20 deletions

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 3eeb1694210c..d6ffdc9afcbb 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -249,12 +249,14 @@ return_from_SYSCALL_64:
 
         /*
          * Try to use SYSRET instead of IRET if we're returning to
-         * a completely clean 64-bit userspace context.
+         * a completely clean 64-bit userspace context.  If we're not,
+         * go to the slow exit path.
          */
         movq    RCX(%rsp), %rcx
         movq    RIP(%rsp), %r11
-        cmpq    %rcx, %r11                      /* RCX == RIP */
-        jne     opportunistic_sysret_failed
+
+        cmpq    %rcx, %r11      /* SYSRET requires RCX == RIP */
+        jne     swapgs_restore_regs_and_return_to_usermode
 
         /*
          * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
@@ -272,14 +274,14 @@ return_from_SYSCALL_64:
 
         /* If this changed %rcx, it was not canonical */
         cmpq    %rcx, %r11
-        jne     opportunistic_sysret_failed
+        jne     swapgs_restore_regs_and_return_to_usermode
 
         cmpq    $__USER_CS, CS(%rsp)            /* CS must match SYSRET */
-        jne     opportunistic_sysret_failed
+        jne     swapgs_restore_regs_and_return_to_usermode
 
         movq    R11(%rsp), %r11
         cmpq    %r11, EFLAGS(%rsp)              /* R11 == RFLAGS */
-        jne     opportunistic_sysret_failed
+        jne     swapgs_restore_regs_and_return_to_usermode
 
         /*
          * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
@@ -300,12 +302,12 @@ return_from_SYSCALL_64:
          * would never get past 'stuck_here'.
          */
         testq   $(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
-        jnz     opportunistic_sysret_failed
+        jnz     swapgs_restore_regs_and_return_to_usermode
 
         /* nothing to check for RSP */
 
         cmpq    $__USER_DS, SS(%rsp)            /* SS must match SYSRET */
-        jne     opportunistic_sysret_failed
+        jne     swapgs_restore_regs_and_return_to_usermode
 
         /*
          * We win! This label is here just for ease of understanding
@@ -318,10 +320,6 @@ syscall_return_via_sysret:
         movq    RSP(%rsp), %rsp
         UNWIND_HINT_EMPTY
         USERGS_SYSRET64
-
-opportunistic_sysret_failed:
-        SWAPGS
-        jmp     restore_regs_and_return_to_usermode
 END(entry_SYSCALL_64)
 
 ENTRY(stub_ptregs_64)
@@ -422,8 +420,7 @@ ENTRY(ret_from_fork)
         movq    %rsp, %rdi
         call    syscall_return_slowpath /* returns with IRQs disabled */
         TRACE_IRQS_ON                   /* user mode is traced as IRQS on */
-        SWAPGS
-        jmp     restore_regs_and_return_to_usermode
+        jmp     swapgs_restore_regs_and_return_to_usermode
 
 1:
         /* kernel thread */
@@ -611,9 +608,8 @@ GLOBAL(retint_user)
         mov     %rsp,%rdi
         call    prepare_exit_to_usermode
         TRACE_IRQS_IRETQ
-        SWAPGS
 
-GLOBAL(restore_regs_and_return_to_usermode)
+GLOBAL(swapgs_restore_regs_and_return_to_usermode)
 #ifdef CONFIG_DEBUG_ENTRY
         /* Assert that pt_regs indicates user mode. */
         testl   $3, CS(%rsp)
@@ -621,6 +617,7 @@ GLOBAL(restore_regs_and_return_to_usermode)
         ud2
 1:
 #endif
+        SWAPGS
         RESTORE_EXTRA_REGS
         RESTORE_C_REGS
         REMOVE_PT_GPREGS_FROM_STACK 8
@@ -1342,8 +1339,7 @@ ENTRY(nmi)
          * Return back to user mode.  We must *not* do the normal exit
          * work, because we don't want to enable interrupts.
          */
-        SWAPGS
-        jmp     restore_regs_and_return_to_usermode
+        jmp     swapgs_restore_regs_and_return_to_usermode
 
 .Lnmi_from_kernel:
         /*
diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S
index 9ca014a99968..932b96ce1b06 100644
--- a/arch/x86/entry/entry_64_compat.S
+++ b/arch/x86/entry/entry_64_compat.S
@@ -336,8 +336,7 @@ ENTRY(entry_INT80_compat)
 
         /* Go back to user mode. */
         TRACE_IRQS_ON
-        SWAPGS
-        jmp     restore_regs_and_return_to_usermode
+        jmp     swapgs_restore_regs_and_return_to_usermode
 END(entry_INT80_compat)
 
 ENTRY(stub32_clone)