x86: replace privileged instructions with paravirt macros

The assembly code in entry_64.S issues a bunch of privileged instructions,
like cli, sti, swapgs, and others. Paravirt guests are forbidden to do so,
and we then replace them with macros that will do the right thing.

Signed-off-by: Glauber de Oliveira Costa <gcosta@redhat.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

1 files changed, 59 insertions, 42 deletions

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index e70f3881d7e4..bea8474744ff 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -50,6 +50,7 @@
 #include <asm/hw_irq.h>
 #include <asm/page.h>
 #include <asm/irqflags.h>
+#include <asm/paravirt.h>
 
         .code64
 
@@ -57,6 +58,13 @@
 #define retint_kernel retint_restore_args
 #endif  
 
+#ifdef CONFIG_PARAVIRT
+ENTRY(native_irq_enable_syscall_ret)
+        movq    %gs:pda_oldrsp,%rsp
+        swapgs
+        sysretq
+#endif /* CONFIG_PARAVIRT */
+
 
 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
 #ifdef CONFIG_TRACE_IRQFLAGS
@@ -216,14 +224,21 @@ ENTRY(system_call)
         CFI_DEF_CFA     rsp,PDA_STACKOFFSET
         CFI_REGISTER    rip,rcx
         /*CFI_REGISTER  rflags,r11*/
-        swapgs
+        SWAPGS_UNSAFE_STACK
+        /*
+         * A hypervisor implementation might want to use a label
+         * after the swapgs, so that it can do the swapgs
+         * for the guest and jump here on syscall.
+         */
+ENTRY(system_call_after_swapgs)
+
         movq    %rsp,%gs:pda_oldrsp 
         movq    %gs:pda_kernelstack,%rsp
         /*
          * No need to follow this irqs off/on section - it's straight
          * and short:
          */
-        sti                                     
+        ENABLE_INTERRUPTS(CLBR_NONE)
         SAVE_ARGS 8,1
         movq  %rax,ORIG_RAX-ARGOFFSET(%rsp) 
         movq  %rcx,RIP-ARGOFFSET(%rsp)
@@ -246,7 +261,7 @@ ret_from_sys_call:
 sysret_check:           
         LOCKDEP_SYS_EXIT
         GET_THREAD_INFO(%rcx)
-        cli
+        DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         movl threadinfo_flags(%rcx),%edx
         andl %edi,%edx
@@ -260,9 +275,7 @@ sysret_check:
         CFI_REGISTER    rip,rcx
         RESTORE_ARGS 0,-ARG_SKIP,1
         /*CFI_REGISTER  rflags,r11*/
-        movq    %gs:pda_oldrsp,%rsp
-        swapgs
-        sysretq
+        ENABLE_INTERRUPTS_SYSCALL_RET
 
         CFI_RESTORE_STATE
         /* Handle reschedules */
@@ -271,7 +284,7 @@ sysret_careful:
         bt $TIF_NEED_RESCHED,%edx
         jnc sysret_signal
         TRACE_IRQS_ON
-        sti
+        ENABLE_INTERRUPTS(CLBR_NONE)
         pushq %rdi
         CFI_ADJUST_CFA_OFFSET 8
         call schedule
@@ -282,7 +295,7 @@ sysret_careful:
         /* Handle a signal */ 
 sysret_signal:
         TRACE_IRQS_ON
-        sti
+        ENABLE_INTERRUPTS(CLBR_NONE)
         testl $_TIF_DO_NOTIFY_MASK,%edx
         jz    1f
 
@@ -295,7 +308,7 @@ sysret_signal:
 1:      movl $_TIF_NEED_RESCHED,%edi
         /* Use IRET because user could have changed frame. This
            works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
-        cli
+        DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         jmp int_with_check
         
@@ -327,7 +340,7 @@ tracesys:
  */
         .globl int_ret_from_sys_call
 int_ret_from_sys_call:
-        cli
+        DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         testl $3,CS-ARGOFFSET(%rsp)
         je retint_restore_args
@@ -349,20 +362,20 @@ int_careful:
         bt $TIF_NEED_RESCHED,%edx
         jnc  int_very_careful
         TRACE_IRQS_ON
-        sti
+        ENABLE_INTERRUPTS(CLBR_NONE)
         pushq %rdi
         CFI_ADJUST_CFA_OFFSET 8
         call schedule
         popq %rdi
         CFI_ADJUST_CFA_OFFSET -8
-        cli
+        DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         jmp int_with_check
 
         /* handle signals and tracing -- both require a full stack frame */
 int_very_careful:
         TRACE_IRQS_ON
-        sti
+        ENABLE_INTERRUPTS(CLBR_NONE)
         SAVE_REST
         /* Check for syscall exit trace */      
         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
@@ -385,7 +398,7 @@ int_signal:
 1:      movl $_TIF_NEED_RESCHED,%edi    
 int_restore_rest:
         RESTORE_REST
-        cli
+        DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         jmp int_with_check
         CFI_ENDPROC
@@ -506,7 +519,7 @@ END(stub_rt_sigreturn)
         CFI_DEF_CFA_REGISTER    rbp
         testl $3,CS(%rdi)
         je 1f
-        swapgs  
+        SWAPGS
         /* irqcount is used to check if a CPU is already on an interrupt
            stack or not. While this is essentially redundant with preempt_count
            it is a little cheaper to use a separate counter in the PDA
@@ -527,7 +540,7 @@ ENTRY(common_interrupt)
         interrupt do_IRQ
         /* 0(%rsp): oldrsp-ARGOFFSET */
 ret_from_intr:
-        cli     
+        DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         decl %gs:pda_irqcount
         leaveq
@@ -556,13 +569,13 @@ retint_swapgs:		/* return to user-space */
         /*
          * The iretq could re-enable interrupts:
          */
-        cli
+        DISABLE_INTERRUPTS(CLBR_ANY)
         TRACE_IRQS_IRETQ
-        swapgs 
+        SWAPGS
         jmp restore_args
 
 retint_restore_args:    /* return to kernel space */
-        cli
+        DISABLE_INTERRUPTS(CLBR_ANY)
         /*
          * The iretq could re-enable interrupts:
          */
@@ -570,10 +583,14 @@ retint_restore_args:	/* return to kernel space */
 restore_args:
         RESTORE_ARGS 0,8,0                                              
 iret_label:     
+#ifdef CONFIG_PARAVIRT
+        INTERRUPT_RETURN
+#endif
+ENTRY(native_iret)
         iretq
 
         .section __ex_table,"a"
-        .quad iret_label,bad_iret       
+        .quad native_iret, bad_iret
         .previous
         .section .fixup,"ax"
         /* force a signal here? this matches i386 behaviour */
@@ -581,24 +598,24 @@ iret_label:
 bad_iret:
         movq $11,%rdi   /* SIGSEGV */
         TRACE_IRQS_ON
-        sti
-        jmp do_exit                     
-        .previous       
-        
+        ENABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
+        jmp do_exit
+        .previous
+
         /* edi: workmask, edx: work */
 retint_careful:
         CFI_RESTORE_STATE
         bt    $TIF_NEED_RESCHED,%edx
         jnc   retint_signal
         TRACE_IRQS_ON
-        sti
+        ENABLE_INTERRUPTS(CLBR_NONE)
         pushq %rdi
         CFI_ADJUST_CFA_OFFSET   8
         call  schedule
         popq %rdi               
         CFI_ADJUST_CFA_OFFSET   -8
         GET_THREAD_INFO(%rcx)
-        cli
+        DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         jmp retint_check
         
@@ -606,14 +623,14 @@ retint_signal:
         testl $_TIF_DO_NOTIFY_MASK,%edx
         jz    retint_swapgs
         TRACE_IRQS_ON
-        sti
+        ENABLE_INTERRUPTS(CLBR_NONE)
         SAVE_REST
         movq $-1,ORIG_RAX(%rsp)                         
         xorl %esi,%esi          # oldset
         movq %rsp,%rdi          # &pt_regs
         call do_notify_resume
         RESTORE_REST
-        cli
+        DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         movl $_TIF_NEED_RESCHED,%edi
         GET_THREAD_INFO(%rcx)
@@ -731,7 +748,7 @@ END(spurious_interrupt)
         rdmsr
         testl %edx,%edx
         js    1f
-        swapgs
+        SWAPGS
         xorl  %ebx,%ebx
 1:
         .if \ist
@@ -747,7 +764,7 @@ END(spurious_interrupt)
         .if \ist
         addq    $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
         .endif
-        cli
+        DISABLE_INTERRUPTS(CLBR_NONE)
         .if \irqtrace
         TRACE_IRQS_OFF
         .endif
@@ -776,10 +793,10 @@ paranoid_swapgs\trace:
         .if \trace
         TRACE_IRQS_IRETQ 0
         .endif
-        swapgs
+        SWAPGS_UNSAFE_STACK
 paranoid_restore\trace:
         RESTORE_ALL 8
-        iretq
+        INTERRUPT_RETURN
 paranoid_userspace\trace:
         GET_THREAD_INFO(%rcx)
         movl threadinfo_flags(%rcx),%ebx
@@ -794,11 +811,11 @@ paranoid_userspace\trace:
         .if \trace
         TRACE_IRQS_ON
         .endif
-        sti
+        ENABLE_INTERRUPTS(CLBR_NONE)
         xorl %esi,%esi                  /* arg2: oldset */
         movq %rsp,%rdi                  /* arg1: &pt_regs */
         call do_notify_resume
-        cli
+        DISABLE_INTERRUPTS(CLBR_NONE)
         .if \trace
         TRACE_IRQS_OFF
         .endif
@@ -807,9 +824,9 @@ paranoid_schedule\trace:
         .if \trace
         TRACE_IRQS_ON
         .endif
-        sti
+        ENABLE_INTERRUPTS(CLBR_ANY)
         call schedule
-        cli
+        DISABLE_INTERRUPTS(CLBR_ANY)
         .if \trace
         TRACE_IRQS_OFF
         .endif
@@ -862,7 +879,7 @@ KPROBE_ENTRY(error_entry)
         testl $3,CS(%rsp)
         je  error_kernelspace
 error_swapgs:   
-        swapgs
+        SWAPGS
 error_sti:      
         movq %rdi,RDI(%rsp)     
         CFI_REL_OFFSET  rdi,RDI
@@ -874,7 +891,7 @@ error_sti:
 error_exit:
         movl %ebx,%eax
         RESTORE_REST
-        cli
+        DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         GET_THREAD_INFO(%rcx)   
         testl %eax,%eax
@@ -911,12 +928,12 @@ ENTRY(load_gs_index)
         CFI_STARTPROC
         pushf
         CFI_ADJUST_CFA_OFFSET 8
-        cli
-        swapgs
+        DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
+        SWAPGS
 gs_change:     
         movl %edi,%gs   
 2:      mfence          /* workaround */
-        swapgs
+        SWAPGS
         popf
         CFI_ADJUST_CFA_OFFSET -8
         ret
@@ -930,7 +947,7 @@ ENDPROC(load_gs_index)
         .section .fixup,"ax"
         /* running with kernelgs */
 bad_gs: 
-        swapgs                  /* switch back to user gs */
+        SWAPGS                  /* switch back to user gs */
         xorl %eax,%eax
         movl %eax,%gs
         jmp  2b