aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel/entry_64.S
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2010-10-21 16:20:32 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2010-10-21 16:20:32 -0400
commit214515b5787a1035b2c8807abe8be569de63b2f6 (patch)
treef169eb1702e7e1369fa6d25bcec2b79359bad94a /arch/x86/kernel/entry_64.S
parentbf70030dc0b031f000c74721f2e9c88686b7da6d (diff)
parentd0ed0c32662e756e7daf85e70a5a27a9c1111331 (diff)
Merge branch 'x86-debug-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'x86-debug-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: x86: Remove pr_<level> uses of KERN_<level> therm_throt.c: Trivial printk message fix for a unsuitable abbreviation of 'thermal' x86: Use {push,pop}{l,q}_cfi in more places i386: Add unwind directives to syscall ptregs stubs x86-64: Use symbolics instead of raw numbers in entry_64.S x86-64: Adjust frame type at paranoid_exit: x86-64: Fix unwind annotations in syscall stubs
Diffstat (limited to 'arch/x86/kernel/entry_64.S')
-rw-r--r--arch/x86/kernel/entry_64.S108
1 files changed, 44 insertions, 64 deletions
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index c375c79065f8..a7ae7fd1010f 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -213,23 +213,17 @@ ENDPROC(native_usergs_sysret64)
213 .macro FAKE_STACK_FRAME child_rip 213 .macro FAKE_STACK_FRAME child_rip
214 /* push in order ss, rsp, eflags, cs, rip */ 214 /* push in order ss, rsp, eflags, cs, rip */
215 xorl %eax, %eax 215 xorl %eax, %eax
216 pushq $__KERNEL_DS /* ss */ 216 pushq_cfi $__KERNEL_DS /* ss */
217 CFI_ADJUST_CFA_OFFSET 8
218 /*CFI_REL_OFFSET ss,0*/ 217 /*CFI_REL_OFFSET ss,0*/
219 pushq %rax /* rsp */ 218 pushq_cfi %rax /* rsp */
220 CFI_ADJUST_CFA_OFFSET 8
221 CFI_REL_OFFSET rsp,0 219 CFI_REL_OFFSET rsp,0
222 pushq $X86_EFLAGS_IF /* eflags - interrupts on */ 220 pushq_cfi $X86_EFLAGS_IF /* eflags - interrupts on */
223 CFI_ADJUST_CFA_OFFSET 8
224 /*CFI_REL_OFFSET rflags,0*/ 221 /*CFI_REL_OFFSET rflags,0*/
225 pushq $__KERNEL_CS /* cs */ 222 pushq_cfi $__KERNEL_CS /* cs */
226 CFI_ADJUST_CFA_OFFSET 8
227 /*CFI_REL_OFFSET cs,0*/ 223 /*CFI_REL_OFFSET cs,0*/
228 pushq \child_rip /* rip */ 224 pushq_cfi \child_rip /* rip */
229 CFI_ADJUST_CFA_OFFSET 8
230 CFI_REL_OFFSET rip,0 225 CFI_REL_OFFSET rip,0
231 pushq %rax /* orig rax */ 226 pushq_cfi %rax /* orig rax */
232 CFI_ADJUST_CFA_OFFSET 8
233 .endm 227 .endm
234 228
235 .macro UNFAKE_STACK_FRAME 229 .macro UNFAKE_STACK_FRAME
@@ -398,10 +392,8 @@ ENTRY(ret_from_fork)
398 392
399 LOCK ; btr $TIF_FORK,TI_flags(%r8) 393 LOCK ; btr $TIF_FORK,TI_flags(%r8)
400 394
401 push kernel_eflags(%rip) 395 pushq_cfi kernel_eflags(%rip)
402 CFI_ADJUST_CFA_OFFSET 8 396 popfq_cfi # reset kernel eflags
403 popf # reset kernel eflags
404 CFI_ADJUST_CFA_OFFSET -8
405 397
406 call schedule_tail # rdi: 'prev' task parameter 398 call schedule_tail # rdi: 'prev' task parameter
407 399
@@ -521,11 +513,9 @@ sysret_careful:
521 jnc sysret_signal 513 jnc sysret_signal
522 TRACE_IRQS_ON 514 TRACE_IRQS_ON
523 ENABLE_INTERRUPTS(CLBR_NONE) 515 ENABLE_INTERRUPTS(CLBR_NONE)
524 pushq %rdi 516 pushq_cfi %rdi
525 CFI_ADJUST_CFA_OFFSET 8
526 call schedule 517 call schedule
527 popq %rdi 518 popq_cfi %rdi
528 CFI_ADJUST_CFA_OFFSET -8
529 jmp sysret_check 519 jmp sysret_check
530 520
531 /* Handle a signal */ 521 /* Handle a signal */
@@ -634,11 +624,9 @@ int_careful:
634 jnc int_very_careful 624 jnc int_very_careful
635 TRACE_IRQS_ON 625 TRACE_IRQS_ON
636 ENABLE_INTERRUPTS(CLBR_NONE) 626 ENABLE_INTERRUPTS(CLBR_NONE)
637 pushq %rdi 627 pushq_cfi %rdi
638 CFI_ADJUST_CFA_OFFSET 8
639 call schedule 628 call schedule
640 popq %rdi 629 popq_cfi %rdi
641 CFI_ADJUST_CFA_OFFSET -8
642 DISABLE_INTERRUPTS(CLBR_NONE) 630 DISABLE_INTERRUPTS(CLBR_NONE)
643 TRACE_IRQS_OFF 631 TRACE_IRQS_OFF
644 jmp int_with_check 632 jmp int_with_check
@@ -652,12 +640,10 @@ int_check_syscall_exit_work:
652 /* Check for syscall exit trace */ 640 /* Check for syscall exit trace */
653 testl $_TIF_WORK_SYSCALL_EXIT,%edx 641 testl $_TIF_WORK_SYSCALL_EXIT,%edx
654 jz int_signal 642 jz int_signal
655 pushq %rdi 643 pushq_cfi %rdi
656 CFI_ADJUST_CFA_OFFSET 8
657 leaq 8(%rsp),%rdi # &ptregs -> arg1 644 leaq 8(%rsp),%rdi # &ptregs -> arg1
658 call syscall_trace_leave 645 call syscall_trace_leave
659 popq %rdi 646 popq_cfi %rdi
660 CFI_ADJUST_CFA_OFFSET -8
661 andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi 647 andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
662 jmp int_restore_rest 648 jmp int_restore_rest
663 649
@@ -714,9 +700,8 @@ END(ptregscall_common)
714 700
715ENTRY(stub_execve) 701ENTRY(stub_execve)
716 CFI_STARTPROC 702 CFI_STARTPROC
717 popq %r11 703 addq $8, %rsp
718 CFI_ADJUST_CFA_OFFSET -8 704 PARTIAL_FRAME 0
719 CFI_REGISTER rip, r11
720 SAVE_REST 705 SAVE_REST
721 FIXUP_TOP_OF_STACK %r11 706 FIXUP_TOP_OF_STACK %r11
722 movq %rsp, %rcx 707 movq %rsp, %rcx
@@ -735,7 +720,7 @@ END(stub_execve)
735ENTRY(stub_rt_sigreturn) 720ENTRY(stub_rt_sigreturn)
736 CFI_STARTPROC 721 CFI_STARTPROC
737 addq $8, %rsp 722 addq $8, %rsp
738 CFI_ADJUST_CFA_OFFSET -8 723 PARTIAL_FRAME 0
739 SAVE_REST 724 SAVE_REST
740 movq %rsp,%rdi 725 movq %rsp,%rdi
741 FIXUP_TOP_OF_STACK %r11 726 FIXUP_TOP_OF_STACK %r11
@@ -766,8 +751,7 @@ vector=FIRST_EXTERNAL_VECTOR
766 .if vector <> FIRST_EXTERNAL_VECTOR 751 .if vector <> FIRST_EXTERNAL_VECTOR
767 CFI_ADJUST_CFA_OFFSET -8 752 CFI_ADJUST_CFA_OFFSET -8
768 .endif 753 .endif
7691: pushq $(~vector+0x80) /* Note: always in signed byte range */ 7541: pushq_cfi $(~vector+0x80) /* Note: always in signed byte range */
770 CFI_ADJUST_CFA_OFFSET 8
771 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6 755 .if ((vector-FIRST_EXTERNAL_VECTOR)%7) <> 6
772 jmp 2f 756 jmp 2f
773 .endif 757 .endif
@@ -796,8 +780,8 @@ END(interrupt)
796 780
797/* 0(%rsp): ~(interrupt number) */ 781/* 0(%rsp): ~(interrupt number) */
798 .macro interrupt func 782 .macro interrupt func
799 subq $10*8, %rsp 783 subq $ORIG_RAX-ARGOFFSET+8, %rsp
800 CFI_ADJUST_CFA_OFFSET 10*8 784 CFI_ADJUST_CFA_OFFSET ORIG_RAX-ARGOFFSET+8
801 call save_args 785 call save_args
802 PARTIAL_FRAME 0 786 PARTIAL_FRAME 0
803 call \func 787 call \func
@@ -822,6 +806,7 @@ ret_from_intr:
822 TRACE_IRQS_OFF 806 TRACE_IRQS_OFF
823 decl PER_CPU_VAR(irq_count) 807 decl PER_CPU_VAR(irq_count)
824 leaveq 808 leaveq
809 CFI_RESTORE rbp
825 CFI_DEF_CFA_REGISTER rsp 810 CFI_DEF_CFA_REGISTER rsp
826 CFI_ADJUST_CFA_OFFSET -8 811 CFI_ADJUST_CFA_OFFSET -8
827exit_intr: 812exit_intr:
@@ -903,11 +888,9 @@ retint_careful:
903 jnc retint_signal 888 jnc retint_signal
904 TRACE_IRQS_ON 889 TRACE_IRQS_ON
905 ENABLE_INTERRUPTS(CLBR_NONE) 890 ENABLE_INTERRUPTS(CLBR_NONE)
906 pushq %rdi 891 pushq_cfi %rdi
907 CFI_ADJUST_CFA_OFFSET 8
908 call schedule 892 call schedule
909 popq %rdi 893 popq_cfi %rdi
910 CFI_ADJUST_CFA_OFFSET -8
911 GET_THREAD_INFO(%rcx) 894 GET_THREAD_INFO(%rcx)
912 DISABLE_INTERRUPTS(CLBR_NONE) 895 DISABLE_INTERRUPTS(CLBR_NONE)
913 TRACE_IRQS_OFF 896 TRACE_IRQS_OFF
@@ -956,8 +939,7 @@ END(common_interrupt)
956.macro apicinterrupt num sym do_sym 939.macro apicinterrupt num sym do_sym
957ENTRY(\sym) 940ENTRY(\sym)
958 INTR_FRAME 941 INTR_FRAME
959 pushq $~(\num) 942 pushq_cfi $~(\num)
960 CFI_ADJUST_CFA_OFFSET 8
961 interrupt \do_sym 943 interrupt \do_sym
962 jmp ret_from_intr 944 jmp ret_from_intr
963 CFI_ENDPROC 945 CFI_ENDPROC
@@ -1036,8 +1018,8 @@ ENTRY(\sym)
1036 INTR_FRAME 1018 INTR_FRAME
1037 PARAVIRT_ADJUST_EXCEPTION_FRAME 1019 PARAVIRT_ADJUST_EXCEPTION_FRAME
1038 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */ 1020 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
1039 subq $15*8,%rsp 1021 subq $ORIG_RAX-R15, %rsp
1040 CFI_ADJUST_CFA_OFFSET 15*8 1022 CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
1041 call error_entry 1023 call error_entry
1042 DEFAULT_FRAME 0 1024 DEFAULT_FRAME 0
1043 movq %rsp,%rdi /* pt_regs pointer */ 1025 movq %rsp,%rdi /* pt_regs pointer */
@@ -1052,9 +1034,9 @@ END(\sym)
1052ENTRY(\sym) 1034ENTRY(\sym)
1053 INTR_FRAME 1035 INTR_FRAME
1054 PARAVIRT_ADJUST_EXCEPTION_FRAME 1036 PARAVIRT_ADJUST_EXCEPTION_FRAME
1055 pushq $-1 /* ORIG_RAX: no syscall to restart */ 1037 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
1056 CFI_ADJUST_CFA_OFFSET 8 1038 subq $ORIG_RAX-R15, %rsp
1057 subq $15*8, %rsp 1039 CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
1058 call save_paranoid 1040 call save_paranoid
1059 TRACE_IRQS_OFF 1041 TRACE_IRQS_OFF
1060 movq %rsp,%rdi /* pt_regs pointer */ 1042 movq %rsp,%rdi /* pt_regs pointer */
@@ -1070,9 +1052,9 @@ END(\sym)
1070ENTRY(\sym) 1052ENTRY(\sym)
1071 INTR_FRAME 1053 INTR_FRAME
1072 PARAVIRT_ADJUST_EXCEPTION_FRAME 1054 PARAVIRT_ADJUST_EXCEPTION_FRAME
1073 pushq $-1 /* ORIG_RAX: no syscall to restart */ 1055 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
1074 CFI_ADJUST_CFA_OFFSET 8 1056 subq $ORIG_RAX-R15, %rsp
1075 subq $15*8, %rsp 1057 CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
1076 call save_paranoid 1058 call save_paranoid
1077 TRACE_IRQS_OFF 1059 TRACE_IRQS_OFF
1078 movq %rsp,%rdi /* pt_regs pointer */ 1060 movq %rsp,%rdi /* pt_regs pointer */
@@ -1089,8 +1071,8 @@ END(\sym)
1089ENTRY(\sym) 1071ENTRY(\sym)
1090 XCPT_FRAME 1072 XCPT_FRAME
1091 PARAVIRT_ADJUST_EXCEPTION_FRAME 1073 PARAVIRT_ADJUST_EXCEPTION_FRAME
1092 subq $15*8,%rsp 1074 subq $ORIG_RAX-R15, %rsp
1093 CFI_ADJUST_CFA_OFFSET 15*8 1075 CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
1094 call error_entry 1076 call error_entry
1095 DEFAULT_FRAME 0 1077 DEFAULT_FRAME 0
1096 movq %rsp,%rdi /* pt_regs pointer */ 1078 movq %rsp,%rdi /* pt_regs pointer */
@@ -1107,8 +1089,8 @@ END(\sym)
1107ENTRY(\sym) 1089ENTRY(\sym)
1108 XCPT_FRAME 1090 XCPT_FRAME
1109 PARAVIRT_ADJUST_EXCEPTION_FRAME 1091 PARAVIRT_ADJUST_EXCEPTION_FRAME
1110 subq $15*8,%rsp 1092 subq $ORIG_RAX-R15, %rsp
1111 CFI_ADJUST_CFA_OFFSET 15*8 1093 CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
1112 call save_paranoid 1094 call save_paranoid
1113 DEFAULT_FRAME 0 1095 DEFAULT_FRAME 0
1114 TRACE_IRQS_OFF 1096 TRACE_IRQS_OFF
@@ -1139,16 +1121,14 @@ zeroentry simd_coprocessor_error do_simd_coprocessor_error
1139 /* edi: new selector */ 1121 /* edi: new selector */
1140ENTRY(native_load_gs_index) 1122ENTRY(native_load_gs_index)
1141 CFI_STARTPROC 1123 CFI_STARTPROC
1142 pushf 1124 pushfq_cfi
1143 CFI_ADJUST_CFA_OFFSET 8
1144 DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI) 1125 DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
1145 SWAPGS 1126 SWAPGS
1146gs_change: 1127gs_change:
1147 movl %edi,%gs 1128 movl %edi,%gs
11482: mfence /* workaround */ 11292: mfence /* workaround */
1149 SWAPGS 1130 SWAPGS
1150 popf 1131 popfq_cfi
1151 CFI_ADJUST_CFA_OFFSET -8
1152 ret 1132 ret
1153 CFI_ENDPROC 1133 CFI_ENDPROC
1154END(native_load_gs_index) 1134END(native_load_gs_index)
@@ -1215,8 +1195,7 @@ END(kernel_execve)
1215/* Call softirq on interrupt stack. Interrupts are off. */ 1195/* Call softirq on interrupt stack. Interrupts are off. */
1216ENTRY(call_softirq) 1196ENTRY(call_softirq)
1217 CFI_STARTPROC 1197 CFI_STARTPROC
1218 push %rbp 1198 pushq_cfi %rbp
1219 CFI_ADJUST_CFA_OFFSET 8
1220 CFI_REL_OFFSET rbp,0 1199 CFI_REL_OFFSET rbp,0
1221 mov %rsp,%rbp 1200 mov %rsp,%rbp
1222 CFI_DEF_CFA_REGISTER rbp 1201 CFI_DEF_CFA_REGISTER rbp
@@ -1225,6 +1204,7 @@ ENTRY(call_softirq)
1225 push %rbp # backlink for old unwinder 1204 push %rbp # backlink for old unwinder
1226 call __do_softirq 1205 call __do_softirq
1227 leaveq 1206 leaveq
1207 CFI_RESTORE rbp
1228 CFI_DEF_CFA_REGISTER rsp 1208 CFI_DEF_CFA_REGISTER rsp
1229 CFI_ADJUST_CFA_OFFSET -8 1209 CFI_ADJUST_CFA_OFFSET -8
1230 decl PER_CPU_VAR(irq_count) 1210 decl PER_CPU_VAR(irq_count)
@@ -1368,7 +1348,7 @@ paranoidzeroentry machine_check *machine_check_vector(%rip)
1368 1348
1369 /* ebx: no swapgs flag */ 1349 /* ebx: no swapgs flag */
1370ENTRY(paranoid_exit) 1350ENTRY(paranoid_exit)
1371 INTR_FRAME 1351 DEFAULT_FRAME
1372 DISABLE_INTERRUPTS(CLBR_NONE) 1352 DISABLE_INTERRUPTS(CLBR_NONE)
1373 TRACE_IRQS_OFF 1353 TRACE_IRQS_OFF
1374 testl %ebx,%ebx /* swapgs needed? */ 1354 testl %ebx,%ebx /* swapgs needed? */
@@ -1445,7 +1425,6 @@ error_swapgs:
1445error_sti: 1425error_sti:
1446 TRACE_IRQS_OFF 1426 TRACE_IRQS_OFF
1447 ret 1427 ret
1448 CFI_ENDPROC
1449 1428
1450/* 1429/*
1451 * There are two places in the kernel that can potentially fault with 1430 * There are two places in the kernel that can potentially fault with
@@ -1470,6 +1449,7 @@ bstep_iret:
1470 /* Fix truncated RIP */ 1449 /* Fix truncated RIP */
1471 movq %rcx,RIP+8(%rsp) 1450 movq %rcx,RIP+8(%rsp)
1472 jmp error_swapgs 1451 jmp error_swapgs
1452 CFI_ENDPROC
1473END(error_entry) 1453END(error_entry)
1474 1454
1475 1455
@@ -1498,8 +1478,8 @@ ENTRY(nmi)
1498 INTR_FRAME 1478 INTR_FRAME
1499 PARAVIRT_ADJUST_EXCEPTION_FRAME 1479 PARAVIRT_ADJUST_EXCEPTION_FRAME
1500 pushq_cfi $-1 1480 pushq_cfi $-1
1501 subq $15*8, %rsp 1481 subq $ORIG_RAX-R15, %rsp
1502 CFI_ADJUST_CFA_OFFSET 15*8 1482 CFI_ADJUST_CFA_OFFSET ORIG_RAX-R15
1503 call save_paranoid 1483 call save_paranoid
1504 DEFAULT_FRAME 0 1484 DEFAULT_FRAME 0
1505 /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ 1485 /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */