2 files changed, 48 insertions, 26 deletions

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 49a0c1781253..c0226ab54106 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -830,8 +830,13 @@ ENTRY(native_iret)
 
 .global native_irq_return_iret
 native_irq_return_iret:
+        /*
+         * This may fault.  Non-paranoid faults on return to userspace are
+         * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
+         * Double-faults due to espfix64 are handled in do_double_fault.
+         * Other faults here are fatal.
+         */
         iretq
-        _ASM_EXTABLE(native_irq_return_iret, bad_iret)
 
 #ifdef CONFIG_X86_ESPFIX64
 native_irq_return_ldt:
@@ -859,25 +864,6 @@ native_irq_return_ldt:
         jmp native_irq_return_iret
 #endif
 
-        .section .fixup,"ax"
-bad_iret:
-        /*
-         * The iret traps when the %cs or %ss being restored is bogus.
-         * We've lost the original trap vector and error code.
-         * #GPF is the most likely one to get for an invalid selector.
-         * So pretend we completed the iret and took the #GPF in user mode.
-         *
-         * We are now running with the kernel GS after exception recovery.
-         * But error_entry expects us to have user GS to match the user %cs,
-         * so swap back.
-         */
-        pushq $0
-
-        SWAPGS
-        jmp general_protection
-
-        .previous
-
         /* edi: workmask, edx: work */
 retint_careful:
         CFI_RESTORE_STATE
@@ -1369,17 +1355,16 @@ error_sti:
 
 /*
  * There are two places in the kernel that can potentially fault with
- * usergs. Handle them here. The exception handlers after iret run with
- * kernel gs again, so don't set the user space flag. B stepping K8s
- * sometimes report an truncated RIP for IRET exceptions returning to
- * compat mode. Check for these here too.
+ * usergs. Handle them here.  B stepping K8s sometimes report a
+ * truncated RIP for IRET exceptions returning to compat mode. Check
+ * for these here too.
  */
 error_kernelspace:
         CFI_REL_OFFSET rcx, RCX+8
         incl %ebx
         leaq native_irq_return_iret(%rip),%rcx
         cmpq %rcx,RIP+8(%rsp)
-        je error_swapgs
+        je error_bad_iret
         movl %ecx,%eax  /* zero extend */
         cmpq %rax,RIP+8(%rsp)
         je bstep_iret
@@ -1390,7 +1375,15 @@ error_kernelspace:
 bstep_iret:
         /* Fix truncated RIP */
         movq %rcx,RIP+8(%rsp)
-        jmp error_swapgs
+        /* fall through */
+
+error_bad_iret:
+        SWAPGS
+        mov %rsp,%rdi
+        call fixup_bad_iret
+        mov %rax,%rsp
+        decl %ebx       /* Return to usergs */
+        jmp error_sti
         CFI_ENDPROC
 END(error_entry)
 
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 48035e9cdde9..de801f22128a 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -407,6 +407,35 @@ asmlinkage __visible struct pt_regs *sync_regs(struct pt_regs *eregs)
         return regs;
 }
 NOKPROBE_SYMBOL(sync_regs);
+
+struct bad_iret_stack {
+        void *error_entry_ret;
+        struct pt_regs regs;
+};
+
+asmlinkage __visible
+struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s)
+{
+        /*
+         * This is called from entry_64.S early in handling a fault
+         * caused by a bad iret to user mode.  To handle the fault
+         * correctly, we want move our stack frame to task_pt_regs
+         * and we want to pretend that the exception came from the
+         * iret target.
+         */
+        struct bad_iret_stack *new_stack =
+                container_of(task_pt_regs(current),
+                             struct bad_iret_stack, regs);
+
+        /* Copy the IRET target to the new stack. */
+        memmove(&new_stack->regs.ip, (void *)s->regs.sp, 5*8);
+
+        /* Copy the remainder of the stack from the current stack. */
+        memmove(new_stack, s, offsetof(struct bad_iret_stack, regs.ip));
+
+        BUG_ON(!user_mode_vm(&new_stack->regs));
+        return new_stack;
+}
 #endif
 
 /*