aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexander van Heukelum <heukelum@mailshack.com>2008-11-21 10:43:18 -0500
committerIngo Molnar <mingo@elte.hu>2008-11-21 13:02:55 -0500
commite2f6bc25b98dbb10d809ee50262b43fcae67840a (patch)
tree20cb125af2bd27ed255a91436af7fe2da33388ff
parentc002a1e6b6b6f07ae04e68987054bf1f2150ae48 (diff)
x86: entry_64.S: factor out save_paranoid and paranoid_exit
Impact: cleanup, shrink kernel image size Also expand the paranoid_exit0 macro into nmi_exit inside the nmi stub in the case of enabled irq-tracing. This gives a few hundred bytes code size reduction. Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm> Signed-off-by: Ingo Molnar <mingo@elte.hu>
-rw-r--r--arch/x86/kernel/entry_64.S151
1 files changed, 102 insertions, 49 deletions
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index ef95c45b9269..fad777b11366 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -353,6 +353,36 @@ ENTRY(save_rest)
353 CFI_ENDPROC 353 CFI_ENDPROC
354END(save_rest) 354END(save_rest)
355 355
356/* save complete stack frame */
357ENTRY(save_paranoid)
358 XCPT_FRAME 1 RDI+8
359 cld
360 movq_cfi rdi, RDI+8
361 movq_cfi rsi, RSI+8
362 movq_cfi rdx, RDX+8
363 movq_cfi rcx, RCX+8
364 movq_cfi rax, RAX+8
365 movq_cfi r8, R8+8
366 movq_cfi r9, R9+8
367 movq_cfi r10, R10+8
368 movq_cfi r11, R11+8
369 movq_cfi rbx, RBX+8
370 movq_cfi rbp, RBP+8
371 movq_cfi r12, R12+8
372 movq_cfi r13, R13+8
373 movq_cfi r14, R14+8
374 movq_cfi r15, R15+8
375 movl $1,%ebx
376 movl $MSR_GS_BASE,%ecx
377 rdmsr
378 testl %edx,%edx
379 js 1f /* negative -> in kernel */
380 SWAPGS
381 xorl %ebx,%ebx
3821: ret
383 CFI_ENDPROC
384END(save_paranoid)
385
356/* 386/*
357 * A newly forked process directly context switches into this. 387 * A newly forked process directly context switches into this.
358 */ 388 */
@@ -1012,24 +1042,15 @@ END(spurious_interrupt)
1012 .endm 1042 .endm
1013 1043
1014 /* error code is on the stack already */ 1044 /* error code is on the stack already */
1015 /* handle NMI like exceptions that can happen everywhere */ 1045 .macro paranoidentry sym ist=0
1016 .macro paranoidentry sym, ist=0, irqtrace=1 1046 subq $15*8, %rsp
1017 SAVE_ALL 1047 CFI_ADJUST_CFA_OFFSET 15*8
1018 cld 1048 call save_paranoid
1019 movl $1,%ebx 1049 DEFAULT_FRAME 0
1020 movl $MSR_GS_BASE,%ecx
1021 rdmsr
1022 testl %edx,%edx
1023 js 1f
1024 SWAPGS
1025 xorl %ebx,%ebx
10261:
1027 .if \ist 1050 .if \ist
1028 movq %gs:pda_data_offset, %rbp 1051 movq %gs:pda_data_offset, %rbp
1029 .endif 1052 .endif
1030 .if \irqtrace
1031 TRACE_IRQS_OFF 1053 TRACE_IRQS_OFF
1032 .endif
1033 movq %rsp,%rdi 1054 movq %rsp,%rdi
1034 movq ORIG_RAX(%rsp),%rsi 1055 movq ORIG_RAX(%rsp),%rsi
1035 movq $-1,ORIG_RAX(%rsp) 1056 movq $-1,ORIG_RAX(%rsp)
@@ -1041,9 +1062,7 @@ END(spurious_interrupt)
1041 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp) 1062 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
1042 .endif 1063 .endif
1043 DISABLE_INTERRUPTS(CLBR_NONE) 1064 DISABLE_INTERRUPTS(CLBR_NONE)
1044 .if \irqtrace
1045 TRACE_IRQS_OFF 1065 TRACE_IRQS_OFF
1046 .endif
1047 .endm 1066 .endm
1048 1067
1049 /* 1068 /*
@@ -1058,57 +1077,48 @@ END(spurious_interrupt)
1058 * is fundamentally NMI-unsafe. (we cannot change the soft and 1077 * is fundamentally NMI-unsafe. (we cannot change the soft and
1059 * hard flags at once, atomically) 1078 * hard flags at once, atomically)
1060 */ 1079 */
1061 .macro paranoidexit trace=1 1080
1062 /* ebx: no swapgs flag */ 1081 /* ebx: no swapgs flag */
1063paranoid_exit\trace: 1082KPROBE_ENTRY(paranoid_exit)
1083 INTR_FRAME
1064 testl %ebx,%ebx /* swapgs needed? */ 1084 testl %ebx,%ebx /* swapgs needed? */
1065 jnz paranoid_restore\trace 1085 jnz paranoid_restore
1066 testl $3,CS(%rsp) 1086 testl $3,CS(%rsp)
1067 jnz paranoid_userspace\trace 1087 jnz paranoid_userspace
1068paranoid_swapgs\trace: 1088paranoid_swapgs:
1069 .if \trace
1070 TRACE_IRQS_IRETQ 0 1089 TRACE_IRQS_IRETQ 0
1071 .endif
1072 SWAPGS_UNSAFE_STACK 1090 SWAPGS_UNSAFE_STACK
1073paranoid_restore\trace: 1091paranoid_restore:
1074 RESTORE_ALL 8 1092 RESTORE_ALL 8
1075 jmp irq_return 1093 jmp irq_return
1076paranoid_userspace\trace: 1094paranoid_userspace:
1077 GET_THREAD_INFO(%rcx) 1095 GET_THREAD_INFO(%rcx)
1078 movl TI_flags(%rcx),%ebx 1096 movl TI_flags(%rcx),%ebx
1079 andl $_TIF_WORK_MASK,%ebx 1097 andl $_TIF_WORK_MASK,%ebx
1080 jz paranoid_swapgs\trace 1098 jz paranoid_swapgs
1081 movq %rsp,%rdi /* &pt_regs */ 1099 movq %rsp,%rdi /* &pt_regs */
1082 call sync_regs 1100 call sync_regs
1083 movq %rax,%rsp /* switch stack for scheduling */ 1101 movq %rax,%rsp /* switch stack for scheduling */
1084 testl $_TIF_NEED_RESCHED,%ebx 1102 testl $_TIF_NEED_RESCHED,%ebx
1085 jnz paranoid_schedule\trace 1103 jnz paranoid_schedule
1086 movl %ebx,%edx /* arg3: thread flags */ 1104 movl %ebx,%edx /* arg3: thread flags */
1087 .if \trace
1088 TRACE_IRQS_ON 1105 TRACE_IRQS_ON
1089 .endif
1090 ENABLE_INTERRUPTS(CLBR_NONE) 1106 ENABLE_INTERRUPTS(CLBR_NONE)
1091 xorl %esi,%esi /* arg2: oldset */ 1107 xorl %esi,%esi /* arg2: oldset */
1092 movq %rsp,%rdi /* arg1: &pt_regs */ 1108 movq %rsp,%rdi /* arg1: &pt_regs */
1093 call do_notify_resume 1109 call do_notify_resume
1094 DISABLE_INTERRUPTS(CLBR_NONE) 1110 DISABLE_INTERRUPTS(CLBR_NONE)
1095 .if \trace
1096 TRACE_IRQS_OFF 1111 TRACE_IRQS_OFF
1097 .endif 1112 jmp paranoid_userspace
1098 jmp paranoid_userspace\trace 1113paranoid_schedule:
1099paranoid_schedule\trace:
1100 .if \trace
1101 TRACE_IRQS_ON 1114 TRACE_IRQS_ON
1102 .endif
1103 ENABLE_INTERRUPTS(CLBR_ANY) 1115 ENABLE_INTERRUPTS(CLBR_ANY)
1104 call schedule 1116 call schedule
1105 DISABLE_INTERRUPTS(CLBR_ANY) 1117 DISABLE_INTERRUPTS(CLBR_ANY)
1106 .if \trace
1107 TRACE_IRQS_OFF 1118 TRACE_IRQS_OFF
1108 .endif 1119 jmp paranoid_userspace
1109 jmp paranoid_userspace\trace
1110 CFI_ENDPROC 1120 CFI_ENDPROC
1111 .endm 1121END(paranoid_exit)
1112 1122
1113/* 1123/*
1114 * Exception entry point. This expects an error code/orig_rax on the stack. 1124 * Exception entry point. This expects an error code/orig_rax on the stack.
@@ -1326,20 +1336,63 @@ KPROBE_ENTRY(debug)
1326 pushq $0 1336 pushq $0
1327 CFI_ADJUST_CFA_OFFSET 8 1337 CFI_ADJUST_CFA_OFFSET 8
1328 paranoidentry do_debug, DEBUG_STACK 1338 paranoidentry do_debug, DEBUG_STACK
1329 paranoidexit 1339 jmp paranoid_exit
1340 CFI_ENDPROC
1330KPROBE_END(debug) 1341KPROBE_END(debug)
1331 1342
1332 /* runs on exception stack */ 1343 /* runs on exception stack */
1333KPROBE_ENTRY(nmi) 1344KPROBE_ENTRY(nmi)
1334 INTR_FRAME 1345 INTR_FRAME
1335 PARAVIRT_ADJUST_EXCEPTION_FRAME 1346 PARAVIRT_ADJUST_EXCEPTION_FRAME
1336 pushq $-1 1347 pushq_cfi $-1
1337 CFI_ADJUST_CFA_OFFSET 8 1348 subq $15*8, %rsp
1338 paranoidentry do_nmi, 0, 0 1349 CFI_ADJUST_CFA_OFFSET 15*8
1350 call save_paranoid
1351 DEFAULT_FRAME 0
1352 /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1353 movq %rsp,%rdi
1354 movq ORIG_RAX(%rsp),%rsi
1355 movq $-1,ORIG_RAX(%rsp)
1356 call do_nmi
1357 DISABLE_INTERRUPTS(CLBR_NONE)
1339#ifdef CONFIG_TRACE_IRQFLAGS 1358#ifdef CONFIG_TRACE_IRQFLAGS
1340 paranoidexit 0 1359 /* paranoidexit; without TRACE_IRQS_OFF */
1360 /* ebx: no swapgs flag */
1361nmi_exit:
1362 testl %ebx,%ebx /* swapgs needed? */
1363 jnz nmi_restore
1364 testl $3,CS(%rsp)
1365 jnz nmi_userspace
1366nmi_swapgs:
1367 SWAPGS_UNSAFE_STACK
1368nmi_restore:
1369 RESTORE_ALL 8
1370 jmp irq_return
1371nmi_userspace:
1372 GET_THREAD_INFO(%rcx)
1373 movl TI_flags(%rcx),%ebx
1374 andl $_TIF_WORK_MASK,%ebx
1375 jz nmi_swapgs
1376 movq %rsp,%rdi /* &pt_regs */
1377 call sync_regs
1378 movq %rax,%rsp /* switch stack for scheduling */
1379 testl $_TIF_NEED_RESCHED,%ebx
1380 jnz nmi_schedule
1381 movl %ebx,%edx /* arg3: thread flags */
1382 ENABLE_INTERRUPTS(CLBR_NONE)
1383 xorl %esi,%esi /* arg2: oldset */
1384 movq %rsp,%rdi /* arg1: &pt_regs */
1385 call do_notify_resume
1386 DISABLE_INTERRUPTS(CLBR_NONE)
1387 jmp nmi_userspace
1388nmi_schedule:
1389 ENABLE_INTERRUPTS(CLBR_ANY)
1390 call schedule
1391 DISABLE_INTERRUPTS(CLBR_ANY)
1392 jmp nmi_userspace
1393 CFI_ENDPROC
1341#else 1394#else
1342 jmp paranoid_exit1 1395 jmp paranoid_exit
1343 CFI_ENDPROC 1396 CFI_ENDPROC
1344#endif 1397#endif
1345KPROBE_END(nmi) 1398KPROBE_END(nmi)
@@ -1350,7 +1403,7 @@ KPROBE_ENTRY(int3)
1350 pushq $0 1403 pushq $0
1351 CFI_ADJUST_CFA_OFFSET 8 1404 CFI_ADJUST_CFA_OFFSET 8
1352 paranoidentry do_int3, DEBUG_STACK 1405 paranoidentry do_int3, DEBUG_STACK
1353 jmp paranoid_exit1 1406 jmp paranoid_exit
1354 CFI_ENDPROC 1407 CFI_ENDPROC
1355KPROBE_END(int3) 1408KPROBE_END(int3)
1356 1409
@@ -1375,7 +1428,7 @@ ENTRY(double_fault)
1375 XCPT_FRAME 1428 XCPT_FRAME
1376 PARAVIRT_ADJUST_EXCEPTION_FRAME 1429 PARAVIRT_ADJUST_EXCEPTION_FRAME
1377 paranoidentry do_double_fault 1430 paranoidentry do_double_fault
1378 jmp paranoid_exit1 1431 jmp paranoid_exit
1379 CFI_ENDPROC 1432 CFI_ENDPROC
1380END(double_fault) 1433END(double_fault)
1381 1434
@@ -1392,7 +1445,7 @@ ENTRY(stack_segment)
1392 XCPT_FRAME 1445 XCPT_FRAME
1393 PARAVIRT_ADJUST_EXCEPTION_FRAME 1446 PARAVIRT_ADJUST_EXCEPTION_FRAME
1394 paranoidentry do_stack_segment 1447 paranoidentry do_stack_segment
1395 jmp paranoid_exit1 1448 jmp paranoid_exit
1396 CFI_ENDPROC 1449 CFI_ENDPROC
1397END(stack_segment) 1450END(stack_segment)
1398 1451
@@ -1420,7 +1473,7 @@ ENTRY(machine_check)
1420 pushq $0 1473 pushq $0
1421 CFI_ADJUST_CFA_OFFSET 8 1474 CFI_ADJUST_CFA_OFFSET 8
1422 paranoidentry do_machine_check 1475 paranoidentry do_machine_check
1423 jmp paranoid_exit1 1476 jmp paranoid_exit
1424 CFI_ENDPROC 1477 CFI_ENDPROC
1425END(machine_check) 1478END(machine_check)
1426#endif 1479#endif