diff options
author | Alexander van Heukelum <heukelum@mailshack.com> | 2008-11-21 10:43:18 -0500 |
---|---|---|
committer | Ingo Molnar <mingo@elte.hu> | 2008-11-21 13:02:55 -0500 |
commit | e2f6bc25b98dbb10d809ee50262b43fcae67840a (patch) | |
tree | 20cb125af2bd27ed255a91436af7fe2da33388ff /arch/x86/kernel | |
parent | c002a1e6b6b6f07ae04e68987054bf1f2150ae48 (diff) |
x86: entry_64.S: factor out save_paranoid and paranoid_exit
Impact: cleanup, shrink kernel image size
Also expand the paranoid_exit0 macro into nmi_exit inside the
nmi stub in the case of enabled irq-tracing.
This gives a few hundred bytes code size reduction.
Signed-off-by: Alexander van Heukelum <heukelum@fastmail.fm>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'arch/x86/kernel')
-rw-r--r-- | arch/x86/kernel/entry_64.S | 151 |
1 files changed, 102 insertions, 49 deletions
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index ef95c45b9269..fad777b11366 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S | |||
@@ -353,6 +353,36 @@ ENTRY(save_rest) | |||
353 | CFI_ENDPROC | 353 | CFI_ENDPROC |
354 | END(save_rest) | 354 | END(save_rest) |
355 | 355 | ||
356 | /* save complete stack frame */ | ||
357 | ENTRY(save_paranoid) | ||
358 | XCPT_FRAME 1 RDI+8 | ||
359 | cld | ||
360 | movq_cfi rdi, RDI+8 | ||
361 | movq_cfi rsi, RSI+8 | ||
362 | movq_cfi rdx, RDX+8 | ||
363 | movq_cfi rcx, RCX+8 | ||
364 | movq_cfi rax, RAX+8 | ||
365 | movq_cfi r8, R8+8 | ||
366 | movq_cfi r9, R9+8 | ||
367 | movq_cfi r10, R10+8 | ||
368 | movq_cfi r11, R11+8 | ||
369 | movq_cfi rbx, RBX+8 | ||
370 | movq_cfi rbp, RBP+8 | ||
371 | movq_cfi r12, R12+8 | ||
372 | movq_cfi r13, R13+8 | ||
373 | movq_cfi r14, R14+8 | ||
374 | movq_cfi r15, R15+8 | ||
375 | movl $1,%ebx | ||
376 | movl $MSR_GS_BASE,%ecx | ||
377 | rdmsr | ||
378 | testl %edx,%edx | ||
379 | js 1f /* negative -> in kernel */ | ||
380 | SWAPGS | ||
381 | xorl %ebx,%ebx | ||
382 | 1: ret | ||
383 | CFI_ENDPROC | ||
384 | END(save_paranoid) | ||
385 | |||
356 | /* | 386 | /* |
357 | * A newly forked process directly context switches into this. | 387 | * A newly forked process directly context switches into this. |
358 | */ | 388 | */ |
@@ -1012,24 +1042,15 @@ END(spurious_interrupt) | |||
1012 | .endm | 1042 | .endm |
1013 | 1043 | ||
1014 | /* error code is on the stack already */ | 1044 | /* error code is on the stack already */ |
1015 | /* handle NMI like exceptions that can happen everywhere */ | 1045 | .macro paranoidentry sym ist=0 |
1016 | .macro paranoidentry sym, ist=0, irqtrace=1 | 1046 | subq $15*8, %rsp |
1017 | SAVE_ALL | 1047 | CFI_ADJUST_CFA_OFFSET 15*8 |
1018 | cld | 1048 | call save_paranoid |
1019 | movl $1,%ebx | 1049 | DEFAULT_FRAME 0 |
1020 | movl $MSR_GS_BASE,%ecx | ||
1021 | rdmsr | ||
1022 | testl %edx,%edx | ||
1023 | js 1f | ||
1024 | SWAPGS | ||
1025 | xorl %ebx,%ebx | ||
1026 | 1: | ||
1027 | .if \ist | 1050 | .if \ist |
1028 | movq %gs:pda_data_offset, %rbp | 1051 | movq %gs:pda_data_offset, %rbp |
1029 | .endif | 1052 | .endif |
1030 | .if \irqtrace | ||
1031 | TRACE_IRQS_OFF | 1053 | TRACE_IRQS_OFF |
1032 | .endif | ||
1033 | movq %rsp,%rdi | 1054 | movq %rsp,%rdi |
1034 | movq ORIG_RAX(%rsp),%rsi | 1055 | movq ORIG_RAX(%rsp),%rsi |
1035 | movq $-1,ORIG_RAX(%rsp) | 1056 | movq $-1,ORIG_RAX(%rsp) |
@@ -1041,9 +1062,7 @@ END(spurious_interrupt) | |||
1041 | addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp) | 1062 | addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp) |
1042 | .endif | 1063 | .endif |
1043 | DISABLE_INTERRUPTS(CLBR_NONE) | 1064 | DISABLE_INTERRUPTS(CLBR_NONE) |
1044 | .if \irqtrace | ||
1045 | TRACE_IRQS_OFF | 1065 | TRACE_IRQS_OFF |
1046 | .endif | ||
1047 | .endm | 1066 | .endm |
1048 | 1067 | ||
1049 | /* | 1068 | /* |
@@ -1058,57 +1077,48 @@ END(spurious_interrupt) | |||
1058 | * is fundamentally NMI-unsafe. (we cannot change the soft and | 1077 | * is fundamentally NMI-unsafe. (we cannot change the soft and |
1059 | * hard flags at once, atomically) | 1078 | * hard flags at once, atomically) |
1060 | */ | 1079 | */ |
1061 | .macro paranoidexit trace=1 | 1080 | |
1062 | /* ebx: no swapgs flag */ | 1081 | /* ebx: no swapgs flag */ |
1063 | paranoid_exit\trace: | 1082 | KPROBE_ENTRY(paranoid_exit) |
1083 | INTR_FRAME | ||
1064 | testl %ebx,%ebx /* swapgs needed? */ | 1084 | testl %ebx,%ebx /* swapgs needed? */ |
1065 | jnz paranoid_restore\trace | 1085 | jnz paranoid_restore |
1066 | testl $3,CS(%rsp) | 1086 | testl $3,CS(%rsp) |
1067 | jnz paranoid_userspace\trace | 1087 | jnz paranoid_userspace |
1068 | paranoid_swapgs\trace: | 1088 | paranoid_swapgs: |
1069 | .if \trace | ||
1070 | TRACE_IRQS_IRETQ 0 | 1089 | TRACE_IRQS_IRETQ 0 |
1071 | .endif | ||
1072 | SWAPGS_UNSAFE_STACK | 1090 | SWAPGS_UNSAFE_STACK |
1073 | paranoid_restore\trace: | 1091 | paranoid_restore: |
1074 | RESTORE_ALL 8 | 1092 | RESTORE_ALL 8 |
1075 | jmp irq_return | 1093 | jmp irq_return |
1076 | paranoid_userspace\trace: | 1094 | paranoid_userspace: |
1077 | GET_THREAD_INFO(%rcx) | 1095 | GET_THREAD_INFO(%rcx) |
1078 | movl TI_flags(%rcx),%ebx | 1096 | movl TI_flags(%rcx),%ebx |
1079 | andl $_TIF_WORK_MASK,%ebx | 1097 | andl $_TIF_WORK_MASK,%ebx |
1080 | jz paranoid_swapgs\trace | 1098 | jz paranoid_swapgs |
1081 | movq %rsp,%rdi /* &pt_regs */ | 1099 | movq %rsp,%rdi /* &pt_regs */ |
1082 | call sync_regs | 1100 | call sync_regs |
1083 | movq %rax,%rsp /* switch stack for scheduling */ | 1101 | movq %rax,%rsp /* switch stack for scheduling */ |
1084 | testl $_TIF_NEED_RESCHED,%ebx | 1102 | testl $_TIF_NEED_RESCHED,%ebx |
1085 | jnz paranoid_schedule\trace | 1103 | jnz paranoid_schedule |
1086 | movl %ebx,%edx /* arg3: thread flags */ | 1104 | movl %ebx,%edx /* arg3: thread flags */ |
1087 | .if \trace | ||
1088 | TRACE_IRQS_ON | 1105 | TRACE_IRQS_ON |
1089 | .endif | ||
1090 | ENABLE_INTERRUPTS(CLBR_NONE) | 1106 | ENABLE_INTERRUPTS(CLBR_NONE) |
1091 | xorl %esi,%esi /* arg2: oldset */ | 1107 | xorl %esi,%esi /* arg2: oldset */ |
1092 | movq %rsp,%rdi /* arg1: &pt_regs */ | 1108 | movq %rsp,%rdi /* arg1: &pt_regs */ |
1093 | call do_notify_resume | 1109 | call do_notify_resume |
1094 | DISABLE_INTERRUPTS(CLBR_NONE) | 1110 | DISABLE_INTERRUPTS(CLBR_NONE) |
1095 | .if \trace | ||
1096 | TRACE_IRQS_OFF | 1111 | TRACE_IRQS_OFF |
1097 | .endif | 1112 | jmp paranoid_userspace |
1098 | jmp paranoid_userspace\trace | 1113 | paranoid_schedule: |
1099 | paranoid_schedule\trace: | ||
1100 | .if \trace | ||
1101 | TRACE_IRQS_ON | 1114 | TRACE_IRQS_ON |
1102 | .endif | ||
1103 | ENABLE_INTERRUPTS(CLBR_ANY) | 1115 | ENABLE_INTERRUPTS(CLBR_ANY) |
1104 | call schedule | 1116 | call schedule |
1105 | DISABLE_INTERRUPTS(CLBR_ANY) | 1117 | DISABLE_INTERRUPTS(CLBR_ANY) |
1106 | .if \trace | ||
1107 | TRACE_IRQS_OFF | 1118 | TRACE_IRQS_OFF |
1108 | .endif | 1119 | jmp paranoid_userspace |
1109 | jmp paranoid_userspace\trace | ||
1110 | CFI_ENDPROC | 1120 | CFI_ENDPROC |
1111 | .endm | 1121 | END(paranoid_exit) |
1112 | 1122 | ||
1113 | /* | 1123 | /* |
1114 | * Exception entry point. This expects an error code/orig_rax on the stack. | 1124 | * Exception entry point. This expects an error code/orig_rax on the stack. |
@@ -1326,20 +1336,63 @@ KPROBE_ENTRY(debug) | |||
1326 | pushq $0 | 1336 | pushq $0 |
1327 | CFI_ADJUST_CFA_OFFSET 8 | 1337 | CFI_ADJUST_CFA_OFFSET 8 |
1328 | paranoidentry do_debug, DEBUG_STACK | 1338 | paranoidentry do_debug, DEBUG_STACK |
1329 | paranoidexit | 1339 | jmp paranoid_exit |
1340 | CFI_ENDPROC | ||
1330 | KPROBE_END(debug) | 1341 | KPROBE_END(debug) |
1331 | 1342 | ||
1332 | /* runs on exception stack */ | 1343 | /* runs on exception stack */ |
1333 | KPROBE_ENTRY(nmi) | 1344 | KPROBE_ENTRY(nmi) |
1334 | INTR_FRAME | 1345 | INTR_FRAME |
1335 | PARAVIRT_ADJUST_EXCEPTION_FRAME | 1346 | PARAVIRT_ADJUST_EXCEPTION_FRAME |
1336 | pushq $-1 | 1347 | pushq_cfi $-1 |
1337 | CFI_ADJUST_CFA_OFFSET 8 | 1348 | subq $15*8, %rsp |
1338 | paranoidentry do_nmi, 0, 0 | 1349 | CFI_ADJUST_CFA_OFFSET 15*8 |
1350 | call save_paranoid | ||
1351 | DEFAULT_FRAME 0 | ||
1352 | /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ | ||
1353 | movq %rsp,%rdi | ||
1354 | movq ORIG_RAX(%rsp),%rsi | ||
1355 | movq $-1,ORIG_RAX(%rsp) | ||
1356 | call do_nmi | ||
1357 | DISABLE_INTERRUPTS(CLBR_NONE) | ||
1339 | #ifdef CONFIG_TRACE_IRQFLAGS | 1358 | #ifdef CONFIG_TRACE_IRQFLAGS |
1340 | paranoidexit 0 | 1359 | /* paranoidexit; without TRACE_IRQS_OFF */ |
1360 | /* ebx: no swapgs flag */ | ||
1361 | nmi_exit: | ||
1362 | testl %ebx,%ebx /* swapgs needed? */ | ||
1363 | jnz nmi_restore | ||
1364 | testl $3,CS(%rsp) | ||
1365 | jnz nmi_userspace | ||
1366 | nmi_swapgs: | ||
1367 | SWAPGS_UNSAFE_STACK | ||
1368 | nmi_restore: | ||
1369 | RESTORE_ALL 8 | ||
1370 | jmp irq_return | ||
1371 | nmi_userspace: | ||
1372 | GET_THREAD_INFO(%rcx) | ||
1373 | movl TI_flags(%rcx),%ebx | ||
1374 | andl $_TIF_WORK_MASK,%ebx | ||
1375 | jz nmi_swapgs | ||
1376 | movq %rsp,%rdi /* &pt_regs */ | ||
1377 | call sync_regs | ||
1378 | movq %rax,%rsp /* switch stack for scheduling */ | ||
1379 | testl $_TIF_NEED_RESCHED,%ebx | ||
1380 | jnz nmi_schedule | ||
1381 | movl %ebx,%edx /* arg3: thread flags */ | ||
1382 | ENABLE_INTERRUPTS(CLBR_NONE) | ||
1383 | xorl %esi,%esi /* arg2: oldset */ | ||
1384 | movq %rsp,%rdi /* arg1: &pt_regs */ | ||
1385 | call do_notify_resume | ||
1386 | DISABLE_INTERRUPTS(CLBR_NONE) | ||
1387 | jmp nmi_userspace | ||
1388 | nmi_schedule: | ||
1389 | ENABLE_INTERRUPTS(CLBR_ANY) | ||
1390 | call schedule | ||
1391 | DISABLE_INTERRUPTS(CLBR_ANY) | ||
1392 | jmp nmi_userspace | ||
1393 | CFI_ENDPROC | ||
1341 | #else | 1394 | #else |
1342 | jmp paranoid_exit1 | 1395 | jmp paranoid_exit |
1343 | CFI_ENDPROC | 1396 | CFI_ENDPROC |
1344 | #endif | 1397 | #endif |
1345 | KPROBE_END(nmi) | 1398 | KPROBE_END(nmi) |
@@ -1350,7 +1403,7 @@ KPROBE_ENTRY(int3) | |||
1350 | pushq $0 | 1403 | pushq $0 |
1351 | CFI_ADJUST_CFA_OFFSET 8 | 1404 | CFI_ADJUST_CFA_OFFSET 8 |
1352 | paranoidentry do_int3, DEBUG_STACK | 1405 | paranoidentry do_int3, DEBUG_STACK |
1353 | jmp paranoid_exit1 | 1406 | jmp paranoid_exit |
1354 | CFI_ENDPROC | 1407 | CFI_ENDPROC |
1355 | KPROBE_END(int3) | 1408 | KPROBE_END(int3) |
1356 | 1409 | ||
@@ -1375,7 +1428,7 @@ ENTRY(double_fault) | |||
1375 | XCPT_FRAME | 1428 | XCPT_FRAME |
1376 | PARAVIRT_ADJUST_EXCEPTION_FRAME | 1429 | PARAVIRT_ADJUST_EXCEPTION_FRAME |
1377 | paranoidentry do_double_fault | 1430 | paranoidentry do_double_fault |
1378 | jmp paranoid_exit1 | 1431 | jmp paranoid_exit |
1379 | CFI_ENDPROC | 1432 | CFI_ENDPROC |
1380 | END(double_fault) | 1433 | END(double_fault) |
1381 | 1434 | ||
@@ -1392,7 +1445,7 @@ ENTRY(stack_segment) | |||
1392 | XCPT_FRAME | 1445 | XCPT_FRAME |
1393 | PARAVIRT_ADJUST_EXCEPTION_FRAME | 1446 | PARAVIRT_ADJUST_EXCEPTION_FRAME |
1394 | paranoidentry do_stack_segment | 1447 | paranoidentry do_stack_segment |
1395 | jmp paranoid_exit1 | 1448 | jmp paranoid_exit |
1396 | CFI_ENDPROC | 1449 | CFI_ENDPROC |
1397 | END(stack_segment) | 1450 | END(stack_segment) |
1398 | 1451 | ||
@@ -1420,7 +1473,7 @@ ENTRY(machine_check) | |||
1420 | pushq $0 | 1473 | pushq $0 |
1421 | CFI_ADJUST_CFA_OFFSET 8 | 1474 | CFI_ADJUST_CFA_OFFSET 8 |
1422 | paranoidentry do_machine_check | 1475 | paranoidentry do_machine_check |
1423 | jmp paranoid_exit1 | 1476 | jmp paranoid_exit |
1424 | CFI_ENDPROC | 1477 | CFI_ENDPROC |
1425 | END(machine_check) | 1478 | END(machine_check) |
1426 | #endif | 1479 | #endif |