x86: fixup more paravirt fallout

Use a common irq_return entry point for all the iret places, which
need the paravirt INTERRUPT return wrapper.

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

2 files changed, 19 insertions, 14 deletions

diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index be5c31d04884..824e21b80aad 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -409,7 +409,8 @@ restore_nocheck_notrace:
         RESTORE_REGS
         addl $4, %esp                   # skip orig_eax/error_code
         CFI_ADJUST_CFA_OFFSET -4
-1:      INTERRUPT_RETURN
+ENTRY(irq_return)
+        INTERRUPT_RETURN
 .section .fixup,"ax"
 iret_exc:
         pushl $0                        # no error code
@@ -418,7 +419,7 @@ iret_exc:
 .previous
 .section __ex_table,"a"
         .align 4
-        .long 1b,iret_exc
+        .long irq_return,iret_exc
 .previous
 
         CFI_RESTORE_STATE
@@ -865,20 +866,16 @@ nmi_espfix_stack:
         RESTORE_REGS
         lss 12+4(%esp), %esp            # back to espfix stack
         CFI_ADJUST_CFA_OFFSET -24
-1:      INTERRUPT_RETURN
+        jmp irq_return
         CFI_ENDPROC
-.section __ex_table,"a"
-        .align 4
-        .long 1b,iret_exc
-.previous
 KPROBE_END(nmi)
 
 #ifdef CONFIG_PARAVIRT
 ENTRY(native_iret)
-1:      iret
+        iret
 .section __ex_table,"a"
         .align 4
-        .long 1b,iret_exc
+        .long native_iret, iret_exc
 .previous
 END(native_iret)
 
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index c7341e81941c..6be39a387c5a 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -581,16 +581,24 @@ retint_restore_args:	/* return to kernel space */
          */
         TRACE_IRQS_IRETQ
 restore_args:
-        RESTORE_ARGS 0,8,0                                              
-#ifdef CONFIG_PARAVIRT
+        RESTORE_ARGS 0,8,0
+
+ENTRY(irq_return)
         INTERRUPT_RETURN
-#endif
+
+        .section __ex_table, "a"
+        .quad irq_return, bad_iret
+        .previous
+
+#ifdef CONFIG_PARAVIRT
 ENTRY(native_iret)
         iretq
 
         .section __ex_table,"a"
         .quad native_iret, bad_iret
         .previous
+#endif
+
         .section .fixup,"ax"
 bad_iret:
         /*
@@ -804,7 +812,7 @@ paranoid_swapgs\trace:
         SWAPGS_UNSAFE_STACK
 paranoid_restore\trace:
         RESTORE_ALL 8
-        INTERRUPT_RETURN
+        jmp irq_return
 paranoid_userspace\trace:
         GET_THREAD_INFO(%rcx)
         movl threadinfo_flags(%rcx),%ebx
@@ -919,7 +927,7 @@ error_kernelspace:
            iret run with kernel gs again, so don't set the user space flag.
            B stepping K8s sometimes report an truncated RIP for IRET 
            exceptions returning to compat mode. Check for these here too. */
-        leaq native_iret(%rip),%rbp
+        leaq irq_return(%rip),%rbp
         cmpq %rbp,RIP(%rsp) 
         je   error_swapgs
         movl %ebp,%ebp  /* zero extend */