aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGlauber de Oliveira Costa <gcosta@redhat.com>2008-01-30 07:32:08 -0500
committerIngo Molnar <mingo@elte.hu>2008-01-30 07:32:08 -0500
commit72fe4858544292ad64600765cb78bc02298c6b1c (patch)
treea3876c02ee67a98f0d0f9606fcb943af627acea6
parente801f864ec7e5b149bd05337800e419f408523bb (diff)
x86: replace privileged instructions with paravirt macros
The assembly code in entry_64.S issues a bunch of privileged instructions, like cli, sti, swapgs, and others. Paravirt guests are forbidden to do so, and we then replace them with macros that will do the right thing. Signed-off-by: Glauber de Oliveira Costa <gcosta@redhat.com> Signed-off-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
-rw-r--r--arch/x86/kernel/entry_64.S101
1 files changed, 59 insertions, 42 deletions
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index e70f3881d7e4..bea8474744ff 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -50,6 +50,7 @@
50#include <asm/hw_irq.h> 50#include <asm/hw_irq.h>
51#include <asm/page.h> 51#include <asm/page.h>
52#include <asm/irqflags.h> 52#include <asm/irqflags.h>
53#include <asm/paravirt.h>
53 54
54 .code64 55 .code64
55 56
@@ -57,6 +58,13 @@
57#define retint_kernel retint_restore_args 58#define retint_kernel retint_restore_args
58#endif 59#endif
59 60
61#ifdef CONFIG_PARAVIRT
62ENTRY(native_irq_enable_syscall_ret)
63 movq %gs:pda_oldrsp,%rsp
64 swapgs
65 sysretq
66#endif /* CONFIG_PARAVIRT */
67
60 68
61.macro TRACE_IRQS_IRETQ offset=ARGOFFSET 69.macro TRACE_IRQS_IRETQ offset=ARGOFFSET
62#ifdef CONFIG_TRACE_IRQFLAGS 70#ifdef CONFIG_TRACE_IRQFLAGS
@@ -216,14 +224,21 @@ ENTRY(system_call)
216 CFI_DEF_CFA rsp,PDA_STACKOFFSET 224 CFI_DEF_CFA rsp,PDA_STACKOFFSET
217 CFI_REGISTER rip,rcx 225 CFI_REGISTER rip,rcx
218 /*CFI_REGISTER rflags,r11*/ 226 /*CFI_REGISTER rflags,r11*/
219 swapgs 227 SWAPGS_UNSAFE_STACK
228 /*
229 * A hypervisor implementation might want to use a label
230 * after the swapgs, so that it can do the swapgs
231 * for the guest and jump here on syscall.
232 */
233ENTRY(system_call_after_swapgs)
234
220 movq %rsp,%gs:pda_oldrsp 235 movq %rsp,%gs:pda_oldrsp
221 movq %gs:pda_kernelstack,%rsp 236 movq %gs:pda_kernelstack,%rsp
222 /* 237 /*
223 * No need to follow this irqs off/on section - it's straight 238 * No need to follow this irqs off/on section - it's straight
224 * and short: 239 * and short:
225 */ 240 */
226 sti 241 ENABLE_INTERRUPTS(CLBR_NONE)
227 SAVE_ARGS 8,1 242 SAVE_ARGS 8,1
228 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) 243 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
229 movq %rcx,RIP-ARGOFFSET(%rsp) 244 movq %rcx,RIP-ARGOFFSET(%rsp)
@@ -246,7 +261,7 @@ ret_from_sys_call:
246sysret_check: 261sysret_check:
247 LOCKDEP_SYS_EXIT 262 LOCKDEP_SYS_EXIT
248 GET_THREAD_INFO(%rcx) 263 GET_THREAD_INFO(%rcx)
249 cli 264 DISABLE_INTERRUPTS(CLBR_NONE)
250 TRACE_IRQS_OFF 265 TRACE_IRQS_OFF
251 movl threadinfo_flags(%rcx),%edx 266 movl threadinfo_flags(%rcx),%edx
252 andl %edi,%edx 267 andl %edi,%edx
@@ -260,9 +275,7 @@ sysret_check:
260 CFI_REGISTER rip,rcx 275 CFI_REGISTER rip,rcx
261 RESTORE_ARGS 0,-ARG_SKIP,1 276 RESTORE_ARGS 0,-ARG_SKIP,1
262 /*CFI_REGISTER rflags,r11*/ 277 /*CFI_REGISTER rflags,r11*/
263 movq %gs:pda_oldrsp,%rsp 278 ENABLE_INTERRUPTS_SYSCALL_RET
264 swapgs
265 sysretq
266 279
267 CFI_RESTORE_STATE 280 CFI_RESTORE_STATE
268 /* Handle reschedules */ 281 /* Handle reschedules */
@@ -271,7 +284,7 @@ sysret_careful:
271 bt $TIF_NEED_RESCHED,%edx 284 bt $TIF_NEED_RESCHED,%edx
272 jnc sysret_signal 285 jnc sysret_signal
273 TRACE_IRQS_ON 286 TRACE_IRQS_ON
274 sti 287 ENABLE_INTERRUPTS(CLBR_NONE)
275 pushq %rdi 288 pushq %rdi
276 CFI_ADJUST_CFA_OFFSET 8 289 CFI_ADJUST_CFA_OFFSET 8
277 call schedule 290 call schedule
@@ -282,7 +295,7 @@ sysret_careful:
282 /* Handle a signal */ 295 /* Handle a signal */
283sysret_signal: 296sysret_signal:
284 TRACE_IRQS_ON 297 TRACE_IRQS_ON
285 sti 298 ENABLE_INTERRUPTS(CLBR_NONE)
286 testl $_TIF_DO_NOTIFY_MASK,%edx 299 testl $_TIF_DO_NOTIFY_MASK,%edx
287 jz 1f 300 jz 1f
288 301
@@ -295,7 +308,7 @@ sysret_signal:
2951: movl $_TIF_NEED_RESCHED,%edi 3081: movl $_TIF_NEED_RESCHED,%edi
296 /* Use IRET because user could have changed frame. This 309 /* Use IRET because user could have changed frame. This
297 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */ 310 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
298 cli 311 DISABLE_INTERRUPTS(CLBR_NONE)
299 TRACE_IRQS_OFF 312 TRACE_IRQS_OFF
300 jmp int_with_check 313 jmp int_with_check
301 314
@@ -327,7 +340,7 @@ tracesys:
327 */ 340 */
328 .globl int_ret_from_sys_call 341 .globl int_ret_from_sys_call
329int_ret_from_sys_call: 342int_ret_from_sys_call:
330 cli 343 DISABLE_INTERRUPTS(CLBR_NONE)
331 TRACE_IRQS_OFF 344 TRACE_IRQS_OFF
332 testl $3,CS-ARGOFFSET(%rsp) 345 testl $3,CS-ARGOFFSET(%rsp)
333 je retint_restore_args 346 je retint_restore_args
@@ -349,20 +362,20 @@ int_careful:
349 bt $TIF_NEED_RESCHED,%edx 362 bt $TIF_NEED_RESCHED,%edx
350 jnc int_very_careful 363 jnc int_very_careful
351 TRACE_IRQS_ON 364 TRACE_IRQS_ON
352 sti 365 ENABLE_INTERRUPTS(CLBR_NONE)
353 pushq %rdi 366 pushq %rdi
354 CFI_ADJUST_CFA_OFFSET 8 367 CFI_ADJUST_CFA_OFFSET 8
355 call schedule 368 call schedule
356 popq %rdi 369 popq %rdi
357 CFI_ADJUST_CFA_OFFSET -8 370 CFI_ADJUST_CFA_OFFSET -8
358 cli 371 DISABLE_INTERRUPTS(CLBR_NONE)
359 TRACE_IRQS_OFF 372 TRACE_IRQS_OFF
360 jmp int_with_check 373 jmp int_with_check
361 374
362 /* handle signals and tracing -- both require a full stack frame */ 375 /* handle signals and tracing -- both require a full stack frame */
363int_very_careful: 376int_very_careful:
364 TRACE_IRQS_ON 377 TRACE_IRQS_ON
365 sti 378 ENABLE_INTERRUPTS(CLBR_NONE)
366 SAVE_REST 379 SAVE_REST
367 /* Check for syscall exit trace */ 380 /* Check for syscall exit trace */
368 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx 381 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
@@ -385,7 +398,7 @@ int_signal:
3851: movl $_TIF_NEED_RESCHED,%edi 3981: movl $_TIF_NEED_RESCHED,%edi
386int_restore_rest: 399int_restore_rest:
387 RESTORE_REST 400 RESTORE_REST
388 cli 401 DISABLE_INTERRUPTS(CLBR_NONE)
389 TRACE_IRQS_OFF 402 TRACE_IRQS_OFF
390 jmp int_with_check 403 jmp int_with_check
391 CFI_ENDPROC 404 CFI_ENDPROC
@@ -506,7 +519,7 @@ END(stub_rt_sigreturn)
506 CFI_DEF_CFA_REGISTER rbp 519 CFI_DEF_CFA_REGISTER rbp
507 testl $3,CS(%rdi) 520 testl $3,CS(%rdi)
508 je 1f 521 je 1f
509 swapgs 522 SWAPGS
510 /* irqcount is used to check if a CPU is already on an interrupt 523 /* irqcount is used to check if a CPU is already on an interrupt
511 stack or not. While this is essentially redundant with preempt_count 524 stack or not. While this is essentially redundant with preempt_count
512 it is a little cheaper to use a separate counter in the PDA 525 it is a little cheaper to use a separate counter in the PDA
@@ -527,7 +540,7 @@ ENTRY(common_interrupt)
527 interrupt do_IRQ 540 interrupt do_IRQ
528 /* 0(%rsp): oldrsp-ARGOFFSET */ 541 /* 0(%rsp): oldrsp-ARGOFFSET */
529ret_from_intr: 542ret_from_intr:
530 cli 543 DISABLE_INTERRUPTS(CLBR_NONE)
531 TRACE_IRQS_OFF 544 TRACE_IRQS_OFF
532 decl %gs:pda_irqcount 545 decl %gs:pda_irqcount
533 leaveq 546 leaveq
@@ -556,13 +569,13 @@ retint_swapgs: /* return to user-space */
556 /* 569 /*
557 * The iretq could re-enable interrupts: 570 * The iretq could re-enable interrupts:
558 */ 571 */
559 cli 572 DISABLE_INTERRUPTS(CLBR_ANY)
560 TRACE_IRQS_IRETQ 573 TRACE_IRQS_IRETQ
561 swapgs 574 SWAPGS
562 jmp restore_args 575 jmp restore_args
563 576
564retint_restore_args: /* return to kernel space */ 577retint_restore_args: /* return to kernel space */
565 cli 578 DISABLE_INTERRUPTS(CLBR_ANY)
566 /* 579 /*
567 * The iretq could re-enable interrupts: 580 * The iretq could re-enable interrupts:
568 */ 581 */
@@ -570,10 +583,14 @@ retint_restore_args: /* return to kernel space */
570restore_args: 583restore_args:
571 RESTORE_ARGS 0,8,0 584 RESTORE_ARGS 0,8,0
572iret_label: 585iret_label:
586#ifdef CONFIG_PARAVIRT
587 INTERRUPT_RETURN
588#endif
589ENTRY(native_iret)
573 iretq 590 iretq
574 591
575 .section __ex_table,"a" 592 .section __ex_table,"a"
576 .quad iret_label,bad_iret 593 .quad native_iret, bad_iret
577 .previous 594 .previous
578 .section .fixup,"ax" 595 .section .fixup,"ax"
579 /* force a signal here? this matches i386 behaviour */ 596 /* force a signal here? this matches i386 behaviour */
@@ -581,24 +598,24 @@ iret_label:
581bad_iret: 598bad_iret:
582 movq $11,%rdi /* SIGSEGV */ 599 movq $11,%rdi /* SIGSEGV */
583 TRACE_IRQS_ON 600 TRACE_IRQS_ON
584 sti 601 ENABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
585 jmp do_exit 602 jmp do_exit
586 .previous 603 .previous
587 604
588 /* edi: workmask, edx: work */ 605 /* edi: workmask, edx: work */
589retint_careful: 606retint_careful:
590 CFI_RESTORE_STATE 607 CFI_RESTORE_STATE
591 bt $TIF_NEED_RESCHED,%edx 608 bt $TIF_NEED_RESCHED,%edx
592 jnc retint_signal 609 jnc retint_signal
593 TRACE_IRQS_ON 610 TRACE_IRQS_ON
594 sti 611 ENABLE_INTERRUPTS(CLBR_NONE)
595 pushq %rdi 612 pushq %rdi
596 CFI_ADJUST_CFA_OFFSET 8 613 CFI_ADJUST_CFA_OFFSET 8
597 call schedule 614 call schedule
598 popq %rdi 615 popq %rdi
599 CFI_ADJUST_CFA_OFFSET -8 616 CFI_ADJUST_CFA_OFFSET -8
600 GET_THREAD_INFO(%rcx) 617 GET_THREAD_INFO(%rcx)
601 cli 618 DISABLE_INTERRUPTS(CLBR_NONE)
602 TRACE_IRQS_OFF 619 TRACE_IRQS_OFF
603 jmp retint_check 620 jmp retint_check
604 621
@@ -606,14 +623,14 @@ retint_signal:
606 testl $_TIF_DO_NOTIFY_MASK,%edx 623 testl $_TIF_DO_NOTIFY_MASK,%edx
607 jz retint_swapgs 624 jz retint_swapgs
608 TRACE_IRQS_ON 625 TRACE_IRQS_ON
609 sti 626 ENABLE_INTERRUPTS(CLBR_NONE)
610 SAVE_REST 627 SAVE_REST
611 movq $-1,ORIG_RAX(%rsp) 628 movq $-1,ORIG_RAX(%rsp)
612 xorl %esi,%esi # oldset 629 xorl %esi,%esi # oldset
613 movq %rsp,%rdi # &pt_regs 630 movq %rsp,%rdi # &pt_regs
614 call do_notify_resume 631 call do_notify_resume
615 RESTORE_REST 632 RESTORE_REST
616 cli 633 DISABLE_INTERRUPTS(CLBR_NONE)
617 TRACE_IRQS_OFF 634 TRACE_IRQS_OFF
618 movl $_TIF_NEED_RESCHED,%edi 635 movl $_TIF_NEED_RESCHED,%edi
619 GET_THREAD_INFO(%rcx) 636 GET_THREAD_INFO(%rcx)
@@ -731,7 +748,7 @@ END(spurious_interrupt)
731 rdmsr 748 rdmsr
732 testl %edx,%edx 749 testl %edx,%edx
733 js 1f 750 js 1f
734 swapgs 751 SWAPGS
735 xorl %ebx,%ebx 752 xorl %ebx,%ebx
7361: 7531:
737 .if \ist 754 .if \ist
@@ -747,7 +764,7 @@ END(spurious_interrupt)
747 .if \ist 764 .if \ist
748 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp) 765 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
749 .endif 766 .endif
750 cli 767 DISABLE_INTERRUPTS(CLBR_NONE)
751 .if \irqtrace 768 .if \irqtrace
752 TRACE_IRQS_OFF 769 TRACE_IRQS_OFF
753 .endif 770 .endif
@@ -776,10 +793,10 @@ paranoid_swapgs\trace:
776 .if \trace 793 .if \trace
777 TRACE_IRQS_IRETQ 0 794 TRACE_IRQS_IRETQ 0
778 .endif 795 .endif
779 swapgs 796 SWAPGS_UNSAFE_STACK
780paranoid_restore\trace: 797paranoid_restore\trace:
781 RESTORE_ALL 8 798 RESTORE_ALL 8
782 iretq 799 INTERRUPT_RETURN
783paranoid_userspace\trace: 800paranoid_userspace\trace:
784 GET_THREAD_INFO(%rcx) 801 GET_THREAD_INFO(%rcx)
785 movl threadinfo_flags(%rcx),%ebx 802 movl threadinfo_flags(%rcx),%ebx
@@ -794,11 +811,11 @@ paranoid_userspace\trace:
794 .if \trace 811 .if \trace
795 TRACE_IRQS_ON 812 TRACE_IRQS_ON
796 .endif 813 .endif
797 sti 814 ENABLE_INTERRUPTS(CLBR_NONE)
798 xorl %esi,%esi /* arg2: oldset */ 815 xorl %esi,%esi /* arg2: oldset */
799 movq %rsp,%rdi /* arg1: &pt_regs */ 816 movq %rsp,%rdi /* arg1: &pt_regs */
800 call do_notify_resume 817 call do_notify_resume
801 cli 818 DISABLE_INTERRUPTS(CLBR_NONE)
802 .if \trace 819 .if \trace
803 TRACE_IRQS_OFF 820 TRACE_IRQS_OFF
804 .endif 821 .endif
@@ -807,9 +824,9 @@ paranoid_schedule\trace:
807 .if \trace 824 .if \trace
808 TRACE_IRQS_ON 825 TRACE_IRQS_ON
809 .endif 826 .endif
810 sti 827 ENABLE_INTERRUPTS(CLBR_ANY)
811 call schedule 828 call schedule
812 cli 829 DISABLE_INTERRUPTS(CLBR_ANY)
813 .if \trace 830 .if \trace
814 TRACE_IRQS_OFF 831 TRACE_IRQS_OFF
815 .endif 832 .endif
@@ -862,7 +879,7 @@ KPROBE_ENTRY(error_entry)
862 testl $3,CS(%rsp) 879 testl $3,CS(%rsp)
863 je error_kernelspace 880 je error_kernelspace
864error_swapgs: 881error_swapgs:
865 swapgs 882 SWAPGS
866error_sti: 883error_sti:
867 movq %rdi,RDI(%rsp) 884 movq %rdi,RDI(%rsp)
868 CFI_REL_OFFSET rdi,RDI 885 CFI_REL_OFFSET rdi,RDI
@@ -874,7 +891,7 @@ error_sti:
874error_exit: 891error_exit:
875 movl %ebx,%eax 892 movl %ebx,%eax
876 RESTORE_REST 893 RESTORE_REST
877 cli 894 DISABLE_INTERRUPTS(CLBR_NONE)
878 TRACE_IRQS_OFF 895 TRACE_IRQS_OFF
879 GET_THREAD_INFO(%rcx) 896 GET_THREAD_INFO(%rcx)
880 testl %eax,%eax 897 testl %eax,%eax
@@ -911,12 +928,12 @@ ENTRY(load_gs_index)
911 CFI_STARTPROC 928 CFI_STARTPROC
912 pushf 929 pushf
913 CFI_ADJUST_CFA_OFFSET 8 930 CFI_ADJUST_CFA_OFFSET 8
914 cli 931 DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
915 swapgs 932 SWAPGS
916gs_change: 933gs_change:
917 movl %edi,%gs 934 movl %edi,%gs
9182: mfence /* workaround */ 9352: mfence /* workaround */
919 swapgs 936 SWAPGS
920 popf 937 popf
921 CFI_ADJUST_CFA_OFFSET -8 938 CFI_ADJUST_CFA_OFFSET -8
922 ret 939 ret
@@ -930,7 +947,7 @@ ENDPROC(load_gs_index)
930 .section .fixup,"ax" 947 .section .fixup,"ax"
931 /* running with kernelgs */ 948 /* running with kernelgs */
932bad_gs: 949bad_gs:
933 swapgs /* switch back to user gs */ 950 SWAPGS /* switch back to user gs */
934 xorl %eax,%eax 951 xorl %eax,%eax
935 movl %eax,%gs 952 movl %eax,%gs
936 jmp 2b 953 jmp 2b