aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86/kernel/entry_64.S444
1 files changed, 220 insertions, 224 deletions
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index f2d546e16354..38fcd0517c31 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1002,7 +1002,7 @@ END(\sym)
1002.endm 1002.endm
1003 1003
1004.macro paranoidzeroentry sym do_sym 1004.macro paranoidzeroentry sym do_sym
1005KPROBE_ENTRY(\sym) 1005ENTRY(\sym)
1006 INTR_FRAME 1006 INTR_FRAME
1007 PARAVIRT_ADJUST_EXCEPTION_FRAME 1007 PARAVIRT_ADJUST_EXCEPTION_FRAME
1008 pushq $-1 /* ORIG_RAX: no syscall to restart */ 1008 pushq $-1 /* ORIG_RAX: no syscall to restart */
@@ -1015,11 +1015,11 @@ KPROBE_ENTRY(\sym)
1015 call \do_sym 1015 call \do_sym
1016 jmp paranoid_exit /* %ebx: no swapgs flag */ 1016 jmp paranoid_exit /* %ebx: no swapgs flag */
1017 CFI_ENDPROC 1017 CFI_ENDPROC
1018KPROBE_END(\sym) 1018END(\sym)
1019.endm 1019.endm
1020 1020
1021.macro paranoidzeroentry_ist sym do_sym ist 1021.macro paranoidzeroentry_ist sym do_sym ist
1022KPROBE_ENTRY(\sym) 1022ENTRY(\sym)
1023 INTR_FRAME 1023 INTR_FRAME
1024 PARAVIRT_ADJUST_EXCEPTION_FRAME 1024 PARAVIRT_ADJUST_EXCEPTION_FRAME
1025 pushq $-1 /* ORIG_RAX: no syscall to restart */ 1025 pushq $-1 /* ORIG_RAX: no syscall to restart */
@@ -1035,15 +1035,11 @@ KPROBE_ENTRY(\sym)
1035 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp) 1035 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
1036 jmp paranoid_exit /* %ebx: no swapgs flag */ 1036 jmp paranoid_exit /* %ebx: no swapgs flag */
1037 CFI_ENDPROC 1037 CFI_ENDPROC
1038KPROBE_END(\sym) 1038END(\sym)
1039.endm 1039.endm
1040 1040
1041.macro errorentry sym do_sym entry=0 1041.macro errorentry sym do_sym
1042.if \entry
1043KPROBE_ENTRY(\sym)
1044.else
1045ENTRY(\sym) 1042ENTRY(\sym)
1046.endif
1047 XCPT_FRAME 1043 XCPT_FRAME
1048 PARAVIRT_ADJUST_EXCEPTION_FRAME 1044 PARAVIRT_ADJUST_EXCEPTION_FRAME
1049 subq $15*8,%rsp 1045 subq $15*8,%rsp
@@ -1056,20 +1052,12 @@ ENTRY(\sym)
1056 call \do_sym 1052 call \do_sym
1057 jmp error_exit /* %ebx: no swapgs flag */ 1053 jmp error_exit /* %ebx: no swapgs flag */
1058 CFI_ENDPROC 1054 CFI_ENDPROC
1059.if \entry
1060KPROBE_END(\sym)
1061.else
1062END(\sym) 1055END(\sym)
1063.endif
1064.endm 1056.endm
1065 1057
1066 /* error code is on the stack already */ 1058 /* error code is on the stack already */
1067.macro paranoiderrorentry sym do_sym entry=1 1059.macro paranoiderrorentry sym do_sym
1068.if \entry
1069KPROBE_ENTRY(\sym)
1070.else
1071ENTRY(\sym) 1060ENTRY(\sym)
1072.endif
1073 XCPT_FRAME 1061 XCPT_FRAME
1074 PARAVIRT_ADJUST_EXCEPTION_FRAME 1062 PARAVIRT_ADJUST_EXCEPTION_FRAME
1075 subq $15*8,%rsp 1063 subq $15*8,%rsp
@@ -1083,166 +1071,23 @@ ENTRY(\sym)
1083 call \do_sym 1071 call \do_sym
1084 jmp paranoid_exit /* %ebx: no swapgs flag */ 1072 jmp paranoid_exit /* %ebx: no swapgs flag */
1085 CFI_ENDPROC 1073 CFI_ENDPROC
1086.if \entry
1087KPROBE_END(\sym)
1088.else
1089END(\sym) 1074END(\sym)
1090.endif
1091.endm 1075.endm
1092 1076
1093zeroentry divide_error do_divide_error 1077zeroentry divide_error do_divide_error
1094paranoidzeroentry_ist debug do_debug DEBUG_STACK
1095paranoidzeroentry_ist int3 do_int3 DEBUG_STACK
1096zeroentry overflow do_overflow 1078zeroentry overflow do_overflow
1097zeroentry bounds do_bounds 1079zeroentry bounds do_bounds
1098zeroentry invalid_op do_invalid_op 1080zeroentry invalid_op do_invalid_op
1099zeroentry device_not_available do_device_not_available 1081zeroentry device_not_available do_device_not_available
1100paranoiderrorentry double_fault do_double_fault 0 1082paranoiderrorentry double_fault do_double_fault
1101zeroentry coprocessor_segment_overrun do_coprocessor_segment_overrun 1083zeroentry coprocessor_segment_overrun do_coprocessor_segment_overrun
1102errorentry invalid_TSS do_invalid_TSS 1084errorentry invalid_TSS do_invalid_TSS
1103errorentry segment_not_present do_segment_not_present 1085errorentry segment_not_present do_segment_not_present
1104paranoiderrorentry stack_segment do_stack_segment
1105errorentry general_protection do_general_protection 1
1106errorentry page_fault do_page_fault 1
1107zeroentry spurious_interrupt_bug do_spurious_interrupt_bug 1086zeroentry spurious_interrupt_bug do_spurious_interrupt_bug
1108zeroentry coprocessor_error do_coprocessor_error 1087zeroentry coprocessor_error do_coprocessor_error
1109errorentry alignment_check do_alignment_check 1088errorentry alignment_check do_alignment_check
1110#ifdef CONFIG_X86_MCE
1111paranoidzeroentry machine_check do_machine_check
1112#endif
1113zeroentry simd_coprocessor_error do_simd_coprocessor_error 1089zeroentry simd_coprocessor_error do_simd_coprocessor_error
1114 1090
1115 /*
1116 * "Paranoid" exit path from exception stack.
1117 * Paranoid because this is used by NMIs and cannot take
1118 * any kernel state for granted.
1119 * We don't do kernel preemption checks here, because only
1120 * NMI should be common and it does not enable IRQs and
1121 * cannot get reschedule ticks.
1122 *
1123 * "trace" is 0 for the NMI handler only, because irq-tracing
1124 * is fundamentally NMI-unsafe. (we cannot change the soft and
1125 * hard flags at once, atomically)
1126 */
1127
1128 /* ebx: no swapgs flag */
1129KPROBE_ENTRY(paranoid_exit)
1130 INTR_FRAME
1131 DISABLE_INTERRUPTS(CLBR_NONE)
1132 TRACE_IRQS_OFF
1133 testl %ebx,%ebx /* swapgs needed? */
1134 jnz paranoid_restore
1135 testl $3,CS(%rsp)
1136 jnz paranoid_userspace
1137paranoid_swapgs:
1138 TRACE_IRQS_IRETQ 0
1139 SWAPGS_UNSAFE_STACK
1140paranoid_restore:
1141 RESTORE_ALL 8
1142 jmp irq_return
1143paranoid_userspace:
1144 GET_THREAD_INFO(%rcx)
1145 movl TI_flags(%rcx),%ebx
1146 andl $_TIF_WORK_MASK,%ebx
1147 jz paranoid_swapgs
1148 movq %rsp,%rdi /* &pt_regs */
1149 call sync_regs
1150 movq %rax,%rsp /* switch stack for scheduling */
1151 testl $_TIF_NEED_RESCHED,%ebx
1152 jnz paranoid_schedule
1153 movl %ebx,%edx /* arg3: thread flags */
1154 TRACE_IRQS_ON
1155 ENABLE_INTERRUPTS(CLBR_NONE)
1156 xorl %esi,%esi /* arg2: oldset */
1157 movq %rsp,%rdi /* arg1: &pt_regs */
1158 call do_notify_resume
1159 DISABLE_INTERRUPTS(CLBR_NONE)
1160 TRACE_IRQS_OFF
1161 jmp paranoid_userspace
1162paranoid_schedule:
1163 TRACE_IRQS_ON
1164 ENABLE_INTERRUPTS(CLBR_ANY)
1165 call schedule
1166 DISABLE_INTERRUPTS(CLBR_ANY)
1167 TRACE_IRQS_OFF
1168 jmp paranoid_userspace
1169 CFI_ENDPROC
1170KPROBE_END(paranoid_exit)
1171
1172/*
1173 * Exception entry point. This expects an error code/orig_rax on the stack.
1174 * returns in "no swapgs flag" in %ebx.
1175 */
1176KPROBE_ENTRY(error_entry)
1177 XCPT_FRAME
1178 CFI_ADJUST_CFA_OFFSET 15*8
1179 /* oldrax contains error code */
1180 cld
1181 movq_cfi rdi, RDI+8
1182 movq_cfi rsi, RSI+8
1183 movq_cfi rdx, RDX+8
1184 movq_cfi rcx, RCX+8
1185 movq_cfi rax, RAX+8
1186 movq_cfi r8, R8+8
1187 movq_cfi r9, R9+8
1188 movq_cfi r10, R10+8
1189 movq_cfi r11, R11+8
1190 movq_cfi rbx, RBX+8
1191 movq_cfi rbp, RBP+8
1192 movq_cfi r12, R12+8
1193 movq_cfi r13, R13+8
1194 movq_cfi r14, R14+8
1195 movq_cfi r15, R15+8
1196 xorl %ebx,%ebx
1197 testl $3,CS+8(%rsp)
1198 je error_kernelspace
1199error_swapgs:
1200 SWAPGS
1201error_sti:
1202 TRACE_IRQS_OFF
1203 ret
1204 CFI_ENDPROC
1205
1206/*
1207 * There are two places in the kernel that can potentially fault with
1208 * usergs. Handle them here. The exception handlers after iret run with
1209 * kernel gs again, so don't set the user space flag. B stepping K8s
1210 * sometimes report an truncated RIP for IRET exceptions returning to
1211 * compat mode. Check for these here too.
1212 */
1213error_kernelspace:
1214 incl %ebx
1215 leaq irq_return(%rip),%rcx
1216 cmpq %rcx,RIP+8(%rsp)
1217 je error_swapgs
1218 movl %ecx,%ecx /* zero extend */
1219 cmpq %rcx,RIP+8(%rsp)
1220 je error_swapgs
1221 cmpq $gs_change,RIP+8(%rsp)
1222 je error_swapgs
1223 jmp error_sti
1224KPROBE_END(error_entry)
1225
1226
1227/* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
1228KPROBE_ENTRY(error_exit)
1229 DEFAULT_FRAME
1230 movl %ebx,%eax
1231 RESTORE_REST
1232 DISABLE_INTERRUPTS(CLBR_NONE)
1233 TRACE_IRQS_OFF
1234 GET_THREAD_INFO(%rcx)
1235 testl %eax,%eax
1236 jne retint_kernel
1237 LOCKDEP_SYS_EXIT_IRQ
1238 movl TI_flags(%rcx),%edx
1239 movl $_TIF_WORK_MASK,%edi
1240 andl %edi,%edx
1241 jnz retint_careful
1242 jmp retint_swapgs
1243 CFI_ENDPROC
1244KPROBE_END(error_exit)
1245
1246 /* Reload gs selector with exception handling */ 1091 /* Reload gs selector with exception handling */
1247 /* edi: new selector */ 1092 /* edi: new selector */
1248ENTRY(native_load_gs_index) 1093ENTRY(native_load_gs_index)
@@ -1362,61 +1207,6 @@ ENTRY(kernel_execve)
1362 CFI_ENDPROC 1207 CFI_ENDPROC
1363END(kernel_execve) 1208END(kernel_execve)
1364 1209
1365 /* runs on exception stack */
1366KPROBE_ENTRY(nmi)
1367 INTR_FRAME
1368 PARAVIRT_ADJUST_EXCEPTION_FRAME
1369 pushq_cfi $-1
1370 subq $15*8, %rsp
1371 CFI_ADJUST_CFA_OFFSET 15*8
1372 call save_paranoid
1373 DEFAULT_FRAME 0
1374 /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1375 movq %rsp,%rdi
1376 movq $-1,%rsi
1377 call do_nmi
1378#ifdef CONFIG_TRACE_IRQFLAGS
1379 /* paranoidexit; without TRACE_IRQS_OFF */
1380 /* ebx: no swapgs flag */
1381 DISABLE_INTERRUPTS(CLBR_NONE)
1382 testl %ebx,%ebx /* swapgs needed? */
1383 jnz nmi_restore
1384 testl $3,CS(%rsp)
1385 jnz nmi_userspace
1386nmi_swapgs:
1387 SWAPGS_UNSAFE_STACK
1388nmi_restore:
1389 RESTORE_ALL 8
1390 jmp irq_return
1391nmi_userspace:
1392 GET_THREAD_INFO(%rcx)
1393 movl TI_flags(%rcx),%ebx
1394 andl $_TIF_WORK_MASK,%ebx
1395 jz nmi_swapgs
1396 movq %rsp,%rdi /* &pt_regs */
1397 call sync_regs
1398 movq %rax,%rsp /* switch stack for scheduling */
1399 testl $_TIF_NEED_RESCHED,%ebx
1400 jnz nmi_schedule
1401 movl %ebx,%edx /* arg3: thread flags */
1402 ENABLE_INTERRUPTS(CLBR_NONE)
1403 xorl %esi,%esi /* arg2: oldset */
1404 movq %rsp,%rdi /* arg1: &pt_regs */
1405 call do_notify_resume
1406 DISABLE_INTERRUPTS(CLBR_NONE)
1407 jmp nmi_userspace
1408nmi_schedule:
1409 ENABLE_INTERRUPTS(CLBR_ANY)
1410 call schedule
1411 DISABLE_INTERRUPTS(CLBR_ANY)
1412 jmp nmi_userspace
1413 CFI_ENDPROC
1414#else
1415 jmp paranoid_exit
1416 CFI_ENDPROC
1417#endif
1418KPROBE_END(nmi)
1419
1420/* Call softirq on interrupt stack. Interrupts are off. */ 1210/* Call softirq on interrupt stack. Interrupts are off. */
1421ENTRY(call_softirq) 1211ENTRY(call_softirq)
1422 CFI_STARTPROC 1212 CFI_STARTPROC
@@ -1437,13 +1227,6 @@ ENTRY(call_softirq)
1437 CFI_ENDPROC 1227 CFI_ENDPROC
1438END(call_softirq) 1228END(call_softirq)
1439 1229
1440KPROBE_ENTRY(ignore_sysret)
1441 CFI_STARTPROC
1442 mov $-ENOSYS,%eax
1443 sysret
1444 CFI_ENDPROC
1445KPROBE_END(ignore_sysret)
1446
1447#ifdef CONFIG_XEN 1230#ifdef CONFIG_XEN
1448zeroentry xen_hypervisor_callback xen_do_hypervisor_callback 1231zeroentry xen_hypervisor_callback xen_do_hypervisor_callback
1449 1232
@@ -1540,3 +1323,216 @@ ENTRY(xen_failsafe_callback)
1540END(xen_failsafe_callback) 1323END(xen_failsafe_callback)
1541 1324
1542#endif /* CONFIG_XEN */ 1325#endif /* CONFIG_XEN */
1326
1327/*
1328 * Some functions should be protected against kprobes
1329 */
1330 .pushsection .kprobes.text, "ax"
1331
1332paranoidzeroentry_ist debug do_debug DEBUG_STACK
1333paranoidzeroentry_ist int3 do_int3 DEBUG_STACK
1334paranoiderrorentry stack_segment do_stack_segment
1335errorentry general_protection do_general_protection
1336errorentry page_fault do_page_fault
1337#ifdef CONFIG_X86_MCE
1338paranoidzeroentry machine_check do_machine_check
1339#endif
1340
1341 /*
1342 * "Paranoid" exit path from exception stack.
1343 * Paranoid because this is used by NMIs and cannot take
1344 * any kernel state for granted.
1345 * We don't do kernel preemption checks here, because only
1346 * NMI should be common and it does not enable IRQs and
1347 * cannot get reschedule ticks.
1348 *
1349 * "trace" is 0 for the NMI handler only, because irq-tracing
1350 * is fundamentally NMI-unsafe. (we cannot change the soft and
1351 * hard flags at once, atomically)
1352 */
1353
1354 /* ebx: no swapgs flag */
1355ENTRY(paranoid_exit)
1356 INTR_FRAME
1357 DISABLE_INTERRUPTS(CLBR_NONE)
1358 TRACE_IRQS_OFF
1359 testl %ebx,%ebx /* swapgs needed? */
1360 jnz paranoid_restore
1361 testl $3,CS(%rsp)
1362 jnz paranoid_userspace
1363paranoid_swapgs:
1364 TRACE_IRQS_IRETQ 0
1365 SWAPGS_UNSAFE_STACK
1366paranoid_restore:
1367 RESTORE_ALL 8
1368 jmp irq_return
1369paranoid_userspace:
1370 GET_THREAD_INFO(%rcx)
1371 movl TI_flags(%rcx),%ebx
1372 andl $_TIF_WORK_MASK,%ebx
1373 jz paranoid_swapgs
1374 movq %rsp,%rdi /* &pt_regs */
1375 call sync_regs
1376 movq %rax,%rsp /* switch stack for scheduling */
1377 testl $_TIF_NEED_RESCHED,%ebx
1378 jnz paranoid_schedule
1379 movl %ebx,%edx /* arg3: thread flags */
1380 TRACE_IRQS_ON
1381 ENABLE_INTERRUPTS(CLBR_NONE)
1382 xorl %esi,%esi /* arg2: oldset */
1383 movq %rsp,%rdi /* arg1: &pt_regs */
1384 call do_notify_resume
1385 DISABLE_INTERRUPTS(CLBR_NONE)
1386 TRACE_IRQS_OFF
1387 jmp paranoid_userspace
1388paranoid_schedule:
1389 TRACE_IRQS_ON
1390 ENABLE_INTERRUPTS(CLBR_ANY)
1391 call schedule
1392 DISABLE_INTERRUPTS(CLBR_ANY)
1393 TRACE_IRQS_OFF
1394 jmp paranoid_userspace
1395 CFI_ENDPROC
1396END(paranoid_exit)
1397
1398/*
1399 * Exception entry point. This expects an error code/orig_rax on the stack.
1400 * returns in "no swapgs flag" in %ebx.
1401 */
1402ENTRY(error_entry)
1403 XCPT_FRAME
1404 CFI_ADJUST_CFA_OFFSET 15*8
1405 /* oldrax contains error code */
1406 cld
1407 movq_cfi rdi, RDI+8
1408 movq_cfi rsi, RSI+8
1409 movq_cfi rdx, RDX+8
1410 movq_cfi rcx, RCX+8
1411 movq_cfi rax, RAX+8
1412 movq_cfi r8, R8+8
1413 movq_cfi r9, R9+8
1414 movq_cfi r10, R10+8
1415 movq_cfi r11, R11+8
1416 movq_cfi rbx, RBX+8
1417 movq_cfi rbp, RBP+8
1418 movq_cfi r12, R12+8
1419 movq_cfi r13, R13+8
1420 movq_cfi r14, R14+8
1421 movq_cfi r15, R15+8
1422 xorl %ebx,%ebx
1423 testl $3,CS+8(%rsp)
1424 je error_kernelspace
1425error_swapgs:
1426 SWAPGS
1427error_sti:
1428 TRACE_IRQS_OFF
1429 ret
1430 CFI_ENDPROC
1431
1432/*
1433 * There are two places in the kernel that can potentially fault with
1434 * usergs. Handle them here. The exception handlers after iret run with
1435 * kernel gs again, so don't set the user space flag. B stepping K8s
1436 * sometimes report an truncated RIP for IRET exceptions returning to
1437 * compat mode. Check for these here too.
1438 */
1439error_kernelspace:
1440 incl %ebx
1441 leaq irq_return(%rip),%rcx
1442 cmpq %rcx,RIP+8(%rsp)
1443 je error_swapgs
1444 movl %ecx,%ecx /* zero extend */
1445 cmpq %rcx,RIP+8(%rsp)
1446 je error_swapgs
1447 cmpq $gs_change,RIP+8(%rsp)
1448 je error_swapgs
1449 jmp error_sti
1450END(error_entry)
1451
1452
1453/* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
1454ENTRY(error_exit)
1455 DEFAULT_FRAME
1456 movl %ebx,%eax
1457 RESTORE_REST
1458 DISABLE_INTERRUPTS(CLBR_NONE)
1459 TRACE_IRQS_OFF
1460 GET_THREAD_INFO(%rcx)
1461 testl %eax,%eax
1462 jne retint_kernel
1463 LOCKDEP_SYS_EXIT_IRQ
1464 movl TI_flags(%rcx),%edx
1465 movl $_TIF_WORK_MASK,%edi
1466 andl %edi,%edx
1467 jnz retint_careful
1468 jmp retint_swapgs
1469 CFI_ENDPROC
1470END(error_exit)
1471
1472
1473 /* runs on exception stack */
1474ENTRY(nmi)
1475 INTR_FRAME
1476 PARAVIRT_ADJUST_EXCEPTION_FRAME
1477 pushq_cfi $-1
1478 subq $15*8, %rsp
1479 CFI_ADJUST_CFA_OFFSET 15*8
1480 call save_paranoid
1481 DEFAULT_FRAME 0
1482 /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1483 movq %rsp,%rdi
1484 movq $-1,%rsi
1485 call do_nmi
1486#ifdef CONFIG_TRACE_IRQFLAGS
1487 /* paranoidexit; without TRACE_IRQS_OFF */
1488 /* ebx: no swapgs flag */
1489 DISABLE_INTERRUPTS(CLBR_NONE)
1490 testl %ebx,%ebx /* swapgs needed? */
1491 jnz nmi_restore
1492 testl $3,CS(%rsp)
1493 jnz nmi_userspace
1494nmi_swapgs:
1495 SWAPGS_UNSAFE_STACK
1496nmi_restore:
1497 RESTORE_ALL 8
1498 jmp irq_return
1499nmi_userspace:
1500 GET_THREAD_INFO(%rcx)
1501 movl TI_flags(%rcx),%ebx
1502 andl $_TIF_WORK_MASK,%ebx
1503 jz nmi_swapgs
1504 movq %rsp,%rdi /* &pt_regs */
1505 call sync_regs
1506 movq %rax,%rsp /* switch stack for scheduling */
1507 testl $_TIF_NEED_RESCHED,%ebx
1508 jnz nmi_schedule
1509 movl %ebx,%edx /* arg3: thread flags */
1510 ENABLE_INTERRUPTS(CLBR_NONE)
1511 xorl %esi,%esi /* arg2: oldset */
1512 movq %rsp,%rdi /* arg1: &pt_regs */
1513 call do_notify_resume
1514 DISABLE_INTERRUPTS(CLBR_NONE)
1515 jmp nmi_userspace
1516nmi_schedule:
1517 ENABLE_INTERRUPTS(CLBR_ANY)
1518 call schedule
1519 DISABLE_INTERRUPTS(CLBR_ANY)
1520 jmp nmi_userspace
1521 CFI_ENDPROC
1522#else
1523 jmp paranoid_exit
1524 CFI_ENDPROC
1525#endif
1526END(nmi)
1527
1528ENTRY(ignore_sysret)
1529 CFI_STARTPROC
1530 mov $-ENOSYS,%eax
1531 sysret
1532 CFI_ENDPROC
1533END(ignore_sysret)
1534
1535/*
1536 * End of kprobes section
1537 */
1538 .popsection
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-03 02:16:44 -0500