1 files changed, 54 insertions, 0 deletions

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 501212f14c87..eeab4cf8b2c9 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -794,6 +794,60 @@ retint_swapgs:		/* return to user-space */
          */
         DISABLE_INTERRUPTS(CLBR_ANY)
         TRACE_IRQS_IRETQ
+
+        /*
+         * Try to use SYSRET instead of IRET if we're returning to
+         * a completely clean 64-bit userspace context.
+         */
+        movq (RCX-R11)(%rsp), %rcx
+        cmpq %rcx,(RIP-R11)(%rsp)               /* RCX == RIP */
+        jne opportunistic_sysret_failed
+
+        /*
+         * On Intel CPUs, sysret with non-canonical RCX/RIP will #GP
+         * in kernel space.  This essentially lets the user take over
+         * the kernel, since userspace controls RSP.  It's not worth
+         * testing for canonicalness exactly -- this check detects any
+         * of the 17 high bits set, which is true for non-canonical
+         * or kernel addresses.  (This will pessimize vsyscall=native.
+         * Big deal.)
+         *
+         * If virtual addresses ever become wider, this will need
+         * to be updated to remain correct on both old and new CPUs.
+         */
+        .ifne __VIRTUAL_MASK_SHIFT - 47
+        .error "virtual address width changed -- sysret checks need update"
+        .endif
+        shr $__VIRTUAL_MASK_SHIFT, %rcx
+        jnz opportunistic_sysret_failed
+
+        cmpq $__USER_CS,(CS-R11)(%rsp)          /* CS must match SYSRET */
+        jne opportunistic_sysret_failed
+
+        movq (R11-ARGOFFSET)(%rsp), %r11
+        cmpq %r11,(EFLAGS-ARGOFFSET)(%rsp)      /* R11 == RFLAGS */
+        jne opportunistic_sysret_failed
+
+        testq $X86_EFLAGS_RF,%r11               /* sysret can't restore RF */
+        jnz opportunistic_sysret_failed
+
+        /* nothing to check for RSP */
+
+        cmpq $__USER_DS,(SS-ARGOFFSET)(%rsp)    /* SS must match SYSRET */
+        jne opportunistic_sysret_failed
+
+        /*
+         * We win!  This label is here just for ease of understanding
+         * perf profiles.  Nothing jumps here.
+         */
+irq_return_via_sysret:
+        CFI_REMEMBER_STATE
+        RESTORE_ARGS 1,8,1
+        movq (RSP-RIP)(%rsp),%rsp
+        USERGS_SYSRET64
+        CFI_RESTORE_STATE
+
+opportunistic_sysret_failed:
         SWAPGS
         jmp restore_args