aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorIngo Molnar <mingo@kernel.org>2017-08-18 04:29:54 -0400
committerIngo Molnar <mingo@kernel.org>2017-08-18 04:29:54 -0400
commit0c2364791343e4b04cd1f097ff2abc2799062448 (patch)
treebeb9d94252d42d35b2066bf23383576b0beeebeb
parente26f34a407aec9c65bce2bc0c838fabe4f051fc6 (diff)
parentaa5d1b81500e6059190f18fe25a7617682321910 (diff)
Merge branch 'x86/asm' into locking/core
We need the ASM_UNREACHABLE() macro for a dependent patch. Signed-off-by: Ingo Molnar <mingo@kernel.org>
Diffstat
-rw-r--r--arch/x86/Kconfig.debug2
-rw-r--r--arch/x86/entry/Makefile1
-rw-r--r--arch/x86/entry/calling.h5
-rw-r--r--arch/x86/entry/entry_64.S170
-rw-r--r--arch/x86/include/asm/io.h98
-rw-r--r--arch/x86/include/asm/orc_types.h107
-rw-r--r--arch/x86/include/asm/processor.h3
-rw-r--r--arch/x86/include/asm/rmwcc.h37
-rw-r--r--arch/x86/include/asm/unwind_hints.h103
-rw-r--r--arch/x86/kernel/dumpstack.c12
-rw-r--r--arch/x86/kernel/dumpstack_32.c4
-rw-r--r--arch/x86/kernel/dumpstack_64.c4
-rw-r--r--arch/x86/kernel/process_64.c3
-rw-r--r--include/asm-generic/io.h27
-rw-r--r--include/linux/compiler-gcc.h13
-rw-r--r--include/linux/compiler.h3
-rw-r--r--scripts/Makefile.build3
-rw-r--r--tools/objtool/Build3
-rw-r--r--tools/objtool/Documentation/stack-validation.txt56
-rw-r--r--tools/objtool/Makefile3
-rw-r--r--tools/objtool/builtin-check.c7
-rw-r--r--tools/objtool/builtin-orc.c70
-rw-r--r--tools/objtool/builtin.h1
-rw-r--r--tools/objtool/check.c281
-rw-r--r--tools/objtool/check.h19
-rw-r--r--tools/objtool/elf.c212
-rw-r--r--tools/objtool/elf.h15
-rw-r--r--tools/objtool/objtool.c3
-rw-r--r--tools/objtool/orc.h30
-rw-r--r--tools/objtool/orc_dump.c212
-rw-r--r--tools/objtool/orc_gen.c214
-rw-r--r--tools/objtool/orc_types.h107
32 files changed, 1616 insertions, 212 deletions
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index cd20ca0b4043..1fc519f3c49e 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -305,8 +305,6 @@ config DEBUG_ENTRY
305 Some of these sanity checks may slow down kernel entries and 305 Some of these sanity checks may slow down kernel entries and
306 exits or otherwise impact performance. 306 exits or otherwise impact performance.
307 307
308 This is currently used to help test NMI code.
309
310 If unsure, say N. 308 If unsure, say N.
311 309
312config DEBUG_NMI_SELFTEST 310config DEBUG_NMI_SELFTEST
diff --git a/arch/x86/entry/Makefile b/arch/x86/entry/Makefile
index 9976fcecd17e..af28a8a24366 100644
--- a/arch/x86/entry/Makefile
+++ b/arch/x86/entry/Makefile
@@ -2,7 +2,6 @@
2# Makefile for the x86 low level entry code 2# Makefile for the x86 low level entry code
3# 3#
4 4
5OBJECT_FILES_NON_STANDARD_entry_$(BITS).o := y
6OBJECT_FILES_NON_STANDARD_entry_64_compat.o := y 5OBJECT_FILES_NON_STANDARD_entry_64_compat.o := y
7 6
8CFLAGS_syscall_64.o += $(call cc-option,-Wno-override-init,) 7CFLAGS_syscall_64.o += $(call cc-option,-Wno-override-init,)
diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index 05ed3d393da7..640aafebdc00 100644
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -1,4 +1,5 @@
1#include <linux/jump_label.h> 1#include <linux/jump_label.h>
2#include <asm/unwind_hints.h>
2 3
3/* 4/*
4 5
@@ -112,6 +113,7 @@ For 32-bit we have the following conventions - kernel is built with
112 movq %rdx, 12*8+\offset(%rsp) 113 movq %rdx, 12*8+\offset(%rsp)
113 movq %rsi, 13*8+\offset(%rsp) 114 movq %rsi, 13*8+\offset(%rsp)
114 movq %rdi, 14*8+\offset(%rsp) 115 movq %rdi, 14*8+\offset(%rsp)
116 UNWIND_HINT_REGS offset=\offset extra=0
115 .endm 117 .endm
116 .macro SAVE_C_REGS offset=0 118 .macro SAVE_C_REGS offset=0
117 SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1 119 SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1
@@ -136,6 +138,7 @@ For 32-bit we have the following conventions - kernel is built with
136 movq %r12, 3*8+\offset(%rsp) 138 movq %r12, 3*8+\offset(%rsp)
137 movq %rbp, 4*8+\offset(%rsp) 139 movq %rbp, 4*8+\offset(%rsp)
138 movq %rbx, 5*8+\offset(%rsp) 140 movq %rbx, 5*8+\offset(%rsp)
141 UNWIND_HINT_REGS offset=\offset
139 .endm 142 .endm
140 143
141 .macro RESTORE_EXTRA_REGS offset=0 144 .macro RESTORE_EXTRA_REGS offset=0
@@ -145,6 +148,7 @@ For 32-bit we have the following conventions - kernel is built with
145 movq 3*8+\offset(%rsp), %r12 148 movq 3*8+\offset(%rsp), %r12
146 movq 4*8+\offset(%rsp), %rbp 149 movq 4*8+\offset(%rsp), %rbp
147 movq 5*8+\offset(%rsp), %rbx 150 movq 5*8+\offset(%rsp), %rbx
151 UNWIND_HINT_REGS offset=\offset extra=0
148 .endm 152 .endm
149 153
150 .macro RESTORE_C_REGS_HELPER rstor_rax=1, rstor_rcx=1, rstor_r11=1, rstor_r8910=1, rstor_rdx=1 154 .macro RESTORE_C_REGS_HELPER rstor_rax=1, rstor_rcx=1, rstor_r11=1, rstor_r8910=1, rstor_rdx=1
@@ -167,6 +171,7 @@ For 32-bit we have the following conventions - kernel is built with
167 .endif 171 .endif
168 movq 13*8(%rsp), %rsi 172 movq 13*8(%rsp), %rsi
169 movq 14*8(%rsp), %rdi 173 movq 14*8(%rsp), %rdi
174 UNWIND_HINT_IRET_REGS offset=16*8
170 .endm 175 .endm
171 .macro RESTORE_C_REGS 176 .macro RESTORE_C_REGS
172 RESTORE_C_REGS_HELPER 1,1,1,1,1 177 RESTORE_C_REGS_HELPER 1,1,1,1,1
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index d271fb79248f..daf8936d0628 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -36,6 +36,7 @@
36#include <asm/smap.h> 36#include <asm/smap.h>
37#include <asm/pgtable_types.h> 37#include <asm/pgtable_types.h>
38#include <asm/export.h> 38#include <asm/export.h>
39#include <asm/frame.h>
39#include <linux/err.h> 40#include <linux/err.h>
40 41
41.code64 42.code64
@@ -43,9 +44,10 @@
43 44
44#ifdef CONFIG_PARAVIRT 45#ifdef CONFIG_PARAVIRT
45ENTRY(native_usergs_sysret64) 46ENTRY(native_usergs_sysret64)
47 UNWIND_HINT_EMPTY
46 swapgs 48 swapgs
47 sysretq 49 sysretq
48ENDPROC(native_usergs_sysret64) 50END(native_usergs_sysret64)
49#endif /* CONFIG_PARAVIRT */ 51#endif /* CONFIG_PARAVIRT */
50 52
51.macro TRACE_IRQS_IRETQ 53.macro TRACE_IRQS_IRETQ
@@ -134,6 +136,7 @@ ENDPROC(native_usergs_sysret64)
134 */ 136 */
135 137
136ENTRY(entry_SYSCALL_64) 138ENTRY(entry_SYSCALL_64)
139 UNWIND_HINT_EMPTY
137 /* 140 /*
138 * Interrupts are off on entry. 141 * Interrupts are off on entry.
139 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON, 142 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
@@ -169,6 +172,7 @@ GLOBAL(entry_SYSCALL_64_after_swapgs)
169 pushq %r10 /* pt_regs->r10 */ 172 pushq %r10 /* pt_regs->r10 */
170 pushq %r11 /* pt_regs->r11 */ 173 pushq %r11 /* pt_regs->r11 */
171 sub $(6*8), %rsp /* pt_regs->bp, bx, r12-15 not saved */ 174 sub $(6*8), %rsp /* pt_regs->bp, bx, r12-15 not saved */
175 UNWIND_HINT_REGS extra=0
172 176
173 /* 177 /*
174 * If we need to do entry work or if we guess we'll need to do 178 * If we need to do entry work or if we guess we'll need to do
@@ -223,6 +227,7 @@ entry_SYSCALL_64_fastpath:
223 movq EFLAGS(%rsp), %r11 227 movq EFLAGS(%rsp), %r11
224 RESTORE_C_REGS_EXCEPT_RCX_R11 228 RESTORE_C_REGS_EXCEPT_RCX_R11
225 movq RSP(%rsp), %rsp 229 movq RSP(%rsp), %rsp
230 UNWIND_HINT_EMPTY
226 USERGS_SYSRET64 231 USERGS_SYSRET64
227 232
2281: 2331:
@@ -316,6 +321,7 @@ syscall_return_via_sysret:
316 /* rcx and r11 are already restored (see code above) */ 321 /* rcx and r11 are already restored (see code above) */
317 RESTORE_C_REGS_EXCEPT_RCX_R11 322 RESTORE_C_REGS_EXCEPT_RCX_R11
318 movq RSP(%rsp), %rsp 323 movq RSP(%rsp), %rsp
324 UNWIND_HINT_EMPTY
319 USERGS_SYSRET64 325 USERGS_SYSRET64
320 326
321opportunistic_sysret_failed: 327opportunistic_sysret_failed:
@@ -343,6 +349,7 @@ ENTRY(stub_ptregs_64)
343 DISABLE_INTERRUPTS(CLBR_ANY) 349 DISABLE_INTERRUPTS(CLBR_ANY)
344 TRACE_IRQS_OFF 350 TRACE_IRQS_OFF
345 popq %rax 351 popq %rax
352 UNWIND_HINT_REGS extra=0
346 jmp entry_SYSCALL64_slow_path 353 jmp entry_SYSCALL64_slow_path
347 354
3481: 3551:
@@ -351,6 +358,7 @@ END(stub_ptregs_64)
351 358
352.macro ptregs_stub func 359.macro ptregs_stub func
353ENTRY(ptregs_\func) 360ENTRY(ptregs_\func)
361 UNWIND_HINT_FUNC
354 leaq \func(%rip), %rax 362 leaq \func(%rip), %rax
355 jmp stub_ptregs_64 363 jmp stub_ptregs_64
356END(ptregs_\func) 364END(ptregs_\func)
@@ -367,6 +375,7 @@ END(ptregs_\func)
367 * %rsi: next task 375 * %rsi: next task
368 */ 376 */
369ENTRY(__switch_to_asm) 377ENTRY(__switch_to_asm)
378 UNWIND_HINT_FUNC
370 /* 379 /*
371 * Save callee-saved registers 380 * Save callee-saved registers
372 * This must match the order in inactive_task_frame 381 * This must match the order in inactive_task_frame
@@ -406,6 +415,7 @@ END(__switch_to_asm)
406 * r12: kernel thread arg 415 * r12: kernel thread arg
407 */ 416 */
408ENTRY(ret_from_fork) 417ENTRY(ret_from_fork)
418 UNWIND_HINT_EMPTY
409 movq %rax, %rdi 419 movq %rax, %rdi
410 call schedule_tail /* rdi: 'prev' task parameter */ 420 call schedule_tail /* rdi: 'prev' task parameter */
411 421
@@ -413,6 +423,7 @@ ENTRY(ret_from_fork)
413 jnz 1f /* kernel threads are uncommon */ 423 jnz 1f /* kernel threads are uncommon */
414 424
4152: 4252:
426 UNWIND_HINT_REGS
416 movq %rsp, %rdi 427 movq %rsp, %rdi
417 call syscall_return_slowpath /* returns with IRQs disabled */ 428 call syscall_return_slowpath /* returns with IRQs disabled */
418 TRACE_IRQS_ON /* user mode is traced as IRQS on */ 429 TRACE_IRQS_ON /* user mode is traced as IRQS on */
@@ -440,13 +451,102 @@ END(ret_from_fork)
440ENTRY(irq_entries_start) 451ENTRY(irq_entries_start)
441 vector=FIRST_EXTERNAL_VECTOR 452 vector=FIRST_EXTERNAL_VECTOR
442 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) 453 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
454 UNWIND_HINT_IRET_REGS
443 pushq $(~vector+0x80) /* Note: always in signed byte range */ 455 pushq $(~vector+0x80) /* Note: always in signed byte range */
444 vector=vector+1
445 jmp common_interrupt 456 jmp common_interrupt
446 .align 8 457 .align 8
458 vector=vector+1
447 .endr 459 .endr
448END(irq_entries_start) 460END(irq_entries_start)
449 461
462.macro DEBUG_ENTRY_ASSERT_IRQS_OFF
463#ifdef CONFIG_DEBUG_ENTRY
464 pushfq
465 testl $X86_EFLAGS_IF, (%rsp)
466 jz .Lokay_\@
467 ud2
468.Lokay_\@:
469 addq $8, %rsp
470#endif
471.endm
472
473/*
474 * Enters the IRQ stack if we're not already using it. NMI-safe. Clobbers
475 * flags and puts old RSP into old_rsp, and leaves all other GPRs alone.
476 * Requires kernel GSBASE.
477 *
478 * The invariant is that, if irq_count != -1, then the IRQ stack is in use.
479 */
480.macro ENTER_IRQ_STACK regs=1 old_rsp
481 DEBUG_ENTRY_ASSERT_IRQS_OFF
482 movq %rsp, \old_rsp
483
484 .if \regs
485 UNWIND_HINT_REGS base=\old_rsp
486 .endif
487
488 incl PER_CPU_VAR(irq_count)
489 jnz .Lirq_stack_push_old_rsp_\@
490
491 /*
492 * Right now, if we just incremented irq_count to zero, we've
493 * claimed the IRQ stack but we haven't switched to it yet.
494 *
495 * If anything is added that can interrupt us here without using IST,
496 * it must be *extremely* careful to limit its stack usage. This
497 * could include kprobes and a hypothetical future IST-less #DB
498 * handler.
499 *
500 * The OOPS unwinder relies on the word at the top of the IRQ
501 * stack linking back to the previous RSP for the entire time we're
502 * on the IRQ stack. For this to work reliably, we need to write
503 * it before we actually move ourselves to the IRQ stack.
504 */
505
506 movq \old_rsp, PER_CPU_VAR(irq_stack_union + IRQ_STACK_SIZE - 8)
507 movq PER_CPU_VAR(irq_stack_ptr), %rsp
508
509#ifdef CONFIG_DEBUG_ENTRY
510 /*
511 * If the first movq above becomes wrong due to IRQ stack layout
512 * changes, the only way we'll notice is if we try to unwind right
513 * here. Assert that we set up the stack right to catch this type
514 * of bug quickly.
515 */
516 cmpq -8(%rsp), \old_rsp
517 je .Lirq_stack_okay\@
518 ud2
519 .Lirq_stack_okay\@:
520#endif
521
522.Lirq_stack_push_old_rsp_\@:
523 pushq \old_rsp
524
525 .if \regs
526 UNWIND_HINT_REGS indirect=1
527 .endif
528.endm
529
530/*
531 * Undoes ENTER_IRQ_STACK.
532 */
533.macro LEAVE_IRQ_STACK regs=1
534 DEBUG_ENTRY_ASSERT_IRQS_OFF
535 /* We need to be off the IRQ stack before decrementing irq_count. */
536 popq %rsp
537
538 .if \regs
539 UNWIND_HINT_REGS
540 .endif
541
542 /*
543 * As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming
544 * the irq stack but we're not on it.
545 */
546
547 decl PER_CPU_VAR(irq_count)
548.endm
549
450/* 550/*
451 * Interrupt entry/exit. 551 * Interrupt entry/exit.
452 * 552 *
@@ -485,17 +585,7 @@ END(irq_entries_start)
485 CALL_enter_from_user_mode 585 CALL_enter_from_user_mode
486 586
4871: 5871:
488 /* 588 ENTER_IRQ_STACK old_rsp=%rdi
489 * Save previous stack pointer, optionally switch to interrupt stack.
490 * irq_count is used to check if a CPU is already on an interrupt stack
491 * or not. While this is essentially redundant with preempt_count it is
492 * a little cheaper to use a separate counter in the PDA (short of
493 * moving irq_enter into assembly, which would be too much work)
494 */
495 movq %rsp, %rdi
496 incl PER_CPU_VAR(irq_count)
497 cmovzq PER_CPU_VAR(irq_stack_ptr), %rsp
498 pushq %rdi
499 /* We entered an interrupt context - irqs are off: */ 589 /* We entered an interrupt context - irqs are off: */
500 TRACE_IRQS_OFF 590 TRACE_IRQS_OFF
501 591
@@ -515,10 +605,8 @@ common_interrupt:
515ret_from_intr: 605ret_from_intr:
516 DISABLE_INTERRUPTS(CLBR_ANY) 606 DISABLE_INTERRUPTS(CLBR_ANY)
517 TRACE_IRQS_OFF 607 TRACE_IRQS_OFF
518 decl PER_CPU_VAR(irq_count)
519 608
520 /* Restore saved previous stack */ 609 LEAVE_IRQ_STACK
521 popq %rsp
522 610
523 testb $3, CS(%rsp) 611 testb $3, CS(%rsp)
524 jz retint_kernel 612 jz retint_kernel
@@ -561,6 +649,7 @@ restore_c_regs_and_iret:
561 INTERRUPT_RETURN 649 INTERRUPT_RETURN
562 650
563ENTRY(native_iret) 651ENTRY(native_iret)
652 UNWIND_HINT_IRET_REGS
564 /* 653 /*
565 * Are we returning to a stack segment from the LDT? Note: in 654 * Are we returning to a stack segment from the LDT? Note: in
566 * 64-bit mode SS:RSP on the exception stack is always valid. 655 * 64-bit mode SS:RSP on the exception stack is always valid.
@@ -633,6 +722,7 @@ native_irq_return_ldt:
633 orq PER_CPU_VAR(espfix_stack), %rax 722 orq PER_CPU_VAR(espfix_stack), %rax
634 SWAPGS 723 SWAPGS
635 movq %rax, %rsp 724 movq %rax, %rsp
725 UNWIND_HINT_IRET_REGS offset=8
636 726
637 /* 727 /*
638 * At this point, we cannot write to the stack any more, but we can 728 * At this point, we cannot write to the stack any more, but we can
@@ -654,6 +744,7 @@ END(common_interrupt)
654 */ 744 */
655.macro apicinterrupt3 num sym do_sym 745.macro apicinterrupt3 num sym do_sym
656ENTRY(\sym) 746ENTRY(\sym)
747 UNWIND_HINT_IRET_REGS
657 ASM_CLAC 748 ASM_CLAC
658 pushq $~(\num) 749 pushq $~(\num)
659.Lcommon_\sym: 750.Lcommon_\sym:
@@ -740,6 +831,8 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt
740 831
741.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 832.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
742ENTRY(\sym) 833ENTRY(\sym)
834 UNWIND_HINT_IRET_REGS offset=8
835
743 /* Sanity check */ 836 /* Sanity check */
744 .if \shift_ist != -1 && \paranoid == 0 837 .if \shift_ist != -1 && \paranoid == 0
745 .error "using shift_ist requires paranoid=1" 838 .error "using shift_ist requires paranoid=1"
@@ -763,6 +856,7 @@ ENTRY(\sym)
763 .else 856 .else
764 call error_entry 857 call error_entry
765 .endif 858 .endif
859 UNWIND_HINT_REGS
766 /* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */ 860 /* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */
767 861
768 .if \paranoid 862 .if \paranoid
@@ -860,6 +954,7 @@ idtentry simd_coprocessor_error do_simd_coprocessor_error has_error_code=0
860 * edi: new selector 954 * edi: new selector
861 */ 955 */
862ENTRY(native_load_gs_index) 956ENTRY(native_load_gs_index)
957 FRAME_BEGIN
863 pushfq 958 pushfq
864 DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI) 959 DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
865 SWAPGS 960 SWAPGS
@@ -868,8 +963,9 @@ ENTRY(native_load_gs_index)
8682: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE 9632: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
869 SWAPGS 964 SWAPGS
870 popfq 965 popfq
966 FRAME_END
871 ret 967 ret
872END(native_load_gs_index) 968ENDPROC(native_load_gs_index)
873EXPORT_SYMBOL(native_load_gs_index) 969EXPORT_SYMBOL(native_load_gs_index)
874 970
875 _ASM_EXTABLE(.Lgs_change, bad_gs) 971 _ASM_EXTABLE(.Lgs_change, bad_gs)
@@ -892,14 +988,12 @@ bad_gs:
892ENTRY(do_softirq_own_stack) 988ENTRY(do_softirq_own_stack)
893 pushq %rbp 989 pushq %rbp
894 mov %rsp, %rbp 990 mov %rsp, %rbp
895 incl PER_CPU_VAR(irq_count) 991 ENTER_IRQ_STACK regs=0 old_rsp=%r11
896 cmove PER_CPU_VAR(irq_stack_ptr), %rsp
897 push %rbp /* frame pointer backlink */
898 call __do_softirq 992 call __do_softirq
993 LEAVE_IRQ_STACK regs=0
899 leaveq 994 leaveq
900 decl PER_CPU_VAR(irq_count)
901 ret 995 ret
902END(do_softirq_own_stack) 996ENDPROC(do_softirq_own_stack)
903 997
904#ifdef CONFIG_XEN 998#ifdef CONFIG_XEN
905idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0 999idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0
@@ -923,14 +1017,14 @@ ENTRY(xen_do_hypervisor_callback) /* do_hypervisor_callback(struct *pt_regs) */
923 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will 1017 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
924 * see the correct pointer to the pt_regs 1018 * see the correct pointer to the pt_regs
925 */ 1019 */
1020 UNWIND_HINT_FUNC
926 movq %rdi, %rsp /* we don't return, adjust the stack frame */ 1021 movq %rdi, %rsp /* we don't return, adjust the stack frame */
92711: incl PER_CPU_VAR(irq_count) 1022 UNWIND_HINT_REGS
928 movq %rsp, %rbp 1023
929 cmovzq PER_CPU_VAR(irq_stack_ptr), %rsp 1024 ENTER_IRQ_STACK old_rsp=%r10
930 pushq %rbp /* frame pointer backlink */
931 call xen_evtchn_do_upcall 1025 call xen_evtchn_do_upcall
932 popq %rsp 1026 LEAVE_IRQ_STACK
933 decl PER_CPU_VAR(irq_count) 1027
934#ifndef CONFIG_PREEMPT 1028#ifndef CONFIG_PREEMPT
935 call xen_maybe_preempt_hcall 1029 call xen_maybe_preempt_hcall
936#endif 1030#endif
@@ -951,6 +1045,7 @@ END(xen_do_hypervisor_callback)
951 * with its current contents: any discrepancy means we in category 1. 1045 * with its current contents: any discrepancy means we in category 1.
952 */ 1046 */
953ENTRY(xen_failsafe_callback) 1047ENTRY(xen_failsafe_callback)
1048 UNWIND_HINT_EMPTY
954 movl %ds, %ecx 1049 movl %ds, %ecx
955 cmpw %cx, 0x10(%rsp) 1050 cmpw %cx, 0x10(%rsp)
956 jne 1f 1051 jne 1f
@@ -970,11 +1065,13 @@ ENTRY(xen_failsafe_callback)
970 pushq $0 /* RIP */ 1065 pushq $0 /* RIP */
971 pushq %r11 1066 pushq %r11
972 pushq %rcx 1067 pushq %rcx
1068 UNWIND_HINT_IRET_REGS offset=8
973 jmp general_protection 1069 jmp general_protection
9741: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */ 10701: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
975 movq (%rsp), %rcx 1071 movq (%rsp), %rcx
976 movq 8(%rsp), %r11 1072 movq 8(%rsp), %r11
977 addq $0x30, %rsp 1073 addq $0x30, %rsp
1074 UNWIND_HINT_IRET_REGS
978 pushq $-1 /* orig_ax = -1 => not a system call */ 1075 pushq $-1 /* orig_ax = -1 => not a system call */
979 ALLOC_PT_GPREGS_ON_STACK 1076 ALLOC_PT_GPREGS_ON_STACK
980 SAVE_C_REGS 1077 SAVE_C_REGS
@@ -1020,6 +1117,7 @@ idtentry machine_check has_error_code=0 paranoid=1 do_sym=*machine_check_vec
1020 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise 1117 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise
1021 */ 1118 */
1022ENTRY(paranoid_entry) 1119ENTRY(paranoid_entry)
1120 UNWIND_HINT_FUNC
1023 cld 1121 cld
1024 SAVE_C_REGS 8 1122 SAVE_C_REGS 8
1025 SAVE_EXTRA_REGS 8 1123 SAVE_EXTRA_REGS 8
@@ -1047,6 +1145,7 @@ END(paranoid_entry)
1047 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it) 1145 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
1048 */ 1146 */
1049ENTRY(paranoid_exit) 1147ENTRY(paranoid_exit)
1148 UNWIND_HINT_REGS
1050 DISABLE_INTERRUPTS(CLBR_ANY) 1149 DISABLE_INTERRUPTS(CLBR_ANY)
1051 TRACE_IRQS_OFF_DEBUG 1150 TRACE_IRQS_OFF_DEBUG
1052 testl %ebx, %ebx /* swapgs needed? */ 1151 testl %ebx, %ebx /* swapgs needed? */
@@ -1068,6 +1167,7 @@ END(paranoid_exit)
1068 * Return: EBX=0: came from user mode; EBX=1: otherwise 1167 * Return: EBX=0: came from user mode; EBX=1: otherwise
1069 */ 1168 */
1070ENTRY(error_entry) 1169ENTRY(error_entry)
1170 UNWIND_HINT_FUNC
1071 cld 1171 cld
1072 SAVE_C_REGS 8 1172 SAVE_C_REGS 8
1073 SAVE_EXTRA_REGS 8 1173 SAVE_EXTRA_REGS 8
@@ -1152,6 +1252,7 @@ END(error_entry)
1152 * 0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode 1252 * 0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode
1153 */ 1253 */
1154ENTRY(error_exit) 1254ENTRY(error_exit)
1255 UNWIND_HINT_REGS
1155 DISABLE_INTERRUPTS(CLBR_ANY) 1256 DISABLE_INTERRUPTS(CLBR_ANY)
1156 TRACE_IRQS_OFF 1257 TRACE_IRQS_OFF
1157 testl %ebx, %ebx 1258 testl %ebx, %ebx
@@ -1161,6 +1262,7 @@ END(error_exit)
1161 1262
1162/* Runs on exception stack */ 1263/* Runs on exception stack */
1163ENTRY(nmi) 1264ENTRY(nmi)
1265 UNWIND_HINT_IRET_REGS
1164 /* 1266 /*
1165 * Fix up the exception frame if we're on Xen. 1267 * Fix up the exception frame if we're on Xen.
1166 * PARAVIRT_ADJUST_EXCEPTION_FRAME is guaranteed to push at most 1268 * PARAVIRT_ADJUST_EXCEPTION_FRAME is guaranteed to push at most
@@ -1232,11 +1334,13 @@ ENTRY(nmi)
1232 cld 1334 cld
1233 movq %rsp, %rdx 1335 movq %rsp, %rdx
1234 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp 1336 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
1337 UNWIND_HINT_IRET_REGS base=%rdx offset=8
1235 pushq 5*8(%rdx) /* pt_regs->ss */ 1338 pushq 5*8(%rdx) /* pt_regs->ss */
1236 pushq 4*8(%rdx) /* pt_regs->rsp */ 1339 pushq 4*8(%rdx) /* pt_regs->rsp */
1237 pushq 3*8(%rdx) /* pt_regs->flags */ 1340 pushq 3*8(%rdx) /* pt_regs->flags */
1238 pushq 2*8(%rdx) /* pt_regs->cs */ 1341 pushq 2*8(%rdx) /* pt_regs->cs */
1239 pushq 1*8(%rdx) /* pt_regs->rip */ 1342 pushq 1*8(%rdx) /* pt_regs->rip */
1343 UNWIND_HINT_IRET_REGS
1240 pushq $-1 /* pt_regs->orig_ax */ 1344 pushq $-1 /* pt_regs->orig_ax */
1241 pushq %rdi /* pt_regs->di */ 1345 pushq %rdi /* pt_regs->di */
1242 pushq %rsi /* pt_regs->si */ 1346 pushq %rsi /* pt_regs->si */
@@ -1253,6 +1357,7 @@ ENTRY(nmi)
1253 pushq %r13 /* pt_regs->r13 */ 1357 pushq %r13 /* pt_regs->r13 */
1254 pushq %r14 /* pt_regs->r14 */ 1358 pushq %r14 /* pt_regs->r14 */
1255 pushq %r15 /* pt_regs->r15 */ 1359 pushq %r15 /* pt_regs->r15 */
1360 UNWIND_HINT_REGS
1256 ENCODE_FRAME_POINTER 1361 ENCODE_FRAME_POINTER
1257 1362
1258 /* 1363 /*
@@ -1407,6 +1512,7 @@ first_nmi:
1407 .rept 5 1512 .rept 5
1408 pushq 11*8(%rsp) 1513 pushq 11*8(%rsp)
1409 .endr 1514 .endr
1515 UNWIND_HINT_IRET_REGS
1410 1516
1411 /* Everything up to here is safe from nested NMIs */ 1517 /* Everything up to here is safe from nested NMIs */
1412 1518
@@ -1422,6 +1528,7 @@ first_nmi:
1422 pushq $__KERNEL_CS /* CS */ 1528 pushq $__KERNEL_CS /* CS */
1423 pushq $1f /* RIP */ 1529 pushq $1f /* RIP */
1424 INTERRUPT_RETURN /* continues at repeat_nmi below */ 1530 INTERRUPT_RETURN /* continues at repeat_nmi below */
1531 UNWIND_HINT_IRET_REGS
14251: 15321:
1426#endif 1533#endif
1427 1534
@@ -1471,6 +1578,7 @@ end_repeat_nmi:
1471 * exceptions might do. 1578 * exceptions might do.
1472 */ 1579 */
1473 call paranoid_entry 1580 call paranoid_entry
1581 UNWIND_HINT_REGS
1474 1582
1475 /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ 1583 /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
1476 movq %rsp, %rdi 1584 movq %rsp, %rdi
@@ -1508,17 +1616,19 @@ nmi_restore:
1508END(nmi) 1616END(nmi)
1509 1617
1510ENTRY(ignore_sysret) 1618ENTRY(ignore_sysret)
1619 UNWIND_HINT_EMPTY
1511 mov $-ENOSYS, %eax 1620 mov $-ENOSYS, %eax
1512 sysret 1621 sysret
1513END(ignore_sysret) 1622END(ignore_sysret)
1514 1623
1515ENTRY(rewind_stack_do_exit) 1624ENTRY(rewind_stack_do_exit)
1625 UNWIND_HINT_FUNC
1516 /* Prevent any naive code from trying to unwind to our caller. */ 1626 /* Prevent any naive code from trying to unwind to our caller. */
1517 xorl %ebp, %ebp 1627 xorl %ebp, %ebp
1518 1628
1519 movq PER_CPU_VAR(cpu_current_top_of_stack), %rax 1629 movq PER_CPU_VAR(cpu_current_top_of_stack), %rax
1520 leaq -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%rax), %rsp 1630 leaq -PTREGS_SIZE(%rax), %rsp
1631 UNWIND_HINT_FUNC sp_offset=PTREGS_SIZE
1521 1632
1522 call do_exit 1633 call do_exit
15231: jmp 1b
1524END(rewind_stack_do_exit) 1634END(rewind_stack_do_exit)
diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index 48febf07e828..1310e1f1cd65 100644
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -69,6 +69,9 @@ build_mmio_write(__writeb, "b", unsigned char, "q", )
69build_mmio_write(__writew, "w", unsigned short, "r", ) 69build_mmio_write(__writew, "w", unsigned short, "r", )
70build_mmio_write(__writel, "l", unsigned int, "r", ) 70build_mmio_write(__writel, "l", unsigned int, "r", )
71 71
72#define readb readb
73#define readw readw
74#define readl readl
72#define readb_relaxed(a) __readb(a) 75#define readb_relaxed(a) __readb(a)
73#define readw_relaxed(a) __readw(a) 76#define readw_relaxed(a) __readw(a)
74#define readl_relaxed(a) __readl(a) 77#define readl_relaxed(a) __readl(a)
@@ -76,6 +79,9 @@ build_mmio_write(__writel, "l", unsigned int, "r", )
76#define __raw_readw __readw 79#define __raw_readw __readw
77#define __raw_readl __readl 80#define __raw_readl __readl
78 81
82#define writeb writeb
83#define writew writew
84#define writel writel
79#define writeb_relaxed(v, a) __writeb(v, a) 85#define writeb_relaxed(v, a) __writeb(v, a)
80#define writew_relaxed(v, a) __writew(v, a) 86#define writew_relaxed(v, a) __writew(v, a)
81#define writel_relaxed(v, a) __writel(v, a) 87#define writel_relaxed(v, a) __writel(v, a)
@@ -88,13 +94,15 @@ build_mmio_write(__writel, "l", unsigned int, "r", )
88#ifdef CONFIG_X86_64 94#ifdef CONFIG_X86_64
89 95
90build_mmio_read(readq, "q", unsigned long, "=r", :"memory") 96build_mmio_read(readq, "q", unsigned long, "=r", :"memory")
97build_mmio_read(__readq, "q", unsigned long, "=r", )
91build_mmio_write(writeq, "q", unsigned long, "r", :"memory") 98build_mmio_write(writeq, "q", unsigned long, "r", :"memory")
99build_mmio_write(__writeq, "q", unsigned long, "r", )
92 100
93#define readq_relaxed(a) readq(a) 101#define readq_relaxed(a) __readq(a)
94#define writeq_relaxed(v, a) writeq(v, a) 102#define writeq_relaxed(v, a) __writeq(v, a)
95 103
96#define __raw_readq(a) readq(a) 104#define __raw_readq __readq
97#define __raw_writeq(val, addr) writeq(val, addr) 105#define __raw_writeq __writeq
98 106
99/* Let people know that we have them */ 107/* Let people know that we have them */
100#define readq readq 108#define readq readq
@@ -119,6 +127,7 @@ static inline phys_addr_t virt_to_phys(volatile void *address)
119{ 127{
120 return __pa(address); 128 return __pa(address);
121} 129}
130#define virt_to_phys virt_to_phys
122 131
123/** 132/**
124 * phys_to_virt - map physical address to virtual 133 * phys_to_virt - map physical address to virtual
@@ -137,6 +146,7 @@ static inline void *phys_to_virt(phys_addr_t address)
137{ 146{
138 return __va(address); 147 return __va(address);
139} 148}
149#define phys_to_virt phys_to_virt
140 150
141/* 151/*
142 * Change "struct page" to physical address. 152 * Change "struct page" to physical address.
@@ -169,11 +179,14 @@ static inline unsigned int isa_virt_to_bus(volatile void *address)
169 * else, you probably want one of the following. 179 * else, you probably want one of the following.
170 */ 180 */
171extern void __iomem *ioremap_nocache(resource_size_t offset, unsigned long size); 181extern void __iomem *ioremap_nocache(resource_size_t offset, unsigned long size);
182#define ioremap_nocache ioremap_nocache
172extern void __iomem *ioremap_uc(resource_size_t offset, unsigned long size); 183extern void __iomem *ioremap_uc(resource_size_t offset, unsigned long size);
173#define ioremap_uc ioremap_uc 184#define ioremap_uc ioremap_uc
174 185
175extern void __iomem *ioremap_cache(resource_size_t offset, unsigned long size); 186extern void __iomem *ioremap_cache(resource_size_t offset, unsigned long size);
187#define ioremap_cache ioremap_cache
176extern void __iomem *ioremap_prot(resource_size_t offset, unsigned long size, unsigned long prot_val); 188extern void __iomem *ioremap_prot(resource_size_t offset, unsigned long size, unsigned long prot_val);
189#define ioremap_prot ioremap_prot
177 190
178/** 191/**
179 * ioremap - map bus memory into CPU space 192 * ioremap - map bus memory into CPU space
@@ -193,8 +206,10 @@ static inline void __iomem *ioremap(resource_size_t offset, unsigned long size)
193{ 206{
194 return ioremap_nocache(offset, size); 207 return ioremap_nocache(offset, size);
195} 208}
209#define ioremap ioremap
196 210
197extern void iounmap(volatile void __iomem *addr); 211extern void iounmap(volatile void __iomem *addr);
212#define iounmap iounmap
198 213
199extern void set_iounmap_nonlazy(void); 214extern void set_iounmap_nonlazy(void);
200 215
@@ -203,53 +218,6 @@ extern void set_iounmap_nonlazy(void);
203#include <asm-generic/iomap.h> 218#include <asm-generic/iomap.h>
204 219
205/* 220/*
206 * Convert a virtual cached pointer to an uncached pointer
207 */
208#define xlate_dev_kmem_ptr(p) p
209
210/**
211 * memset_io Set a range of I/O memory to a constant value
212 * @addr: The beginning of the I/O-memory range to set
213 * @val: The value to set the memory to
214 * @count: The number of bytes to set
215 *
216 * Set a range of I/O memory to a given value.
217 */
218static inline void
219memset_io(volatile void __iomem *addr, unsigned char val, size_t count)
220{
221 memset((void __force *)addr, val, count);
222}
223
224/**
225 * memcpy_fromio Copy a block of data from I/O memory
226 * @dst: The (RAM) destination for the copy
227 * @src: The (I/O memory) source for the data
228 * @count: The number of bytes to copy
229 *
230 * Copy a block of data from I/O memory.
231 */
232static inline void
233memcpy_fromio(void *dst, const volatile void __iomem *src, size_t count)
234{
235 memcpy(dst, (const void __force *)src, count);
236}
237
238/**
239 * memcpy_toio Copy a block of data into I/O memory
240 * @dst: The (I/O memory) destination for the copy
241 * @src: The (RAM) source for the data
242 * @count: The number of bytes to copy
243 *
244 * Copy a block of data to I/O memory.
245 */
246static inline void
247memcpy_toio(volatile void __iomem *dst, const void *src, size_t count)
248{
249 memcpy((void __force *)dst, src, count);
250}
251
252/*
253 * ISA space is 'always mapped' on a typical x86 system, no need to 221 * ISA space is 'always mapped' on a typical x86 system, no need to
254 * explicitly ioremap() it. The fact that the ISA IO space is mapped 222 * explicitly ioremap() it. The fact that the ISA IO space is mapped
255 * to PAGE_OFFSET is pure coincidence - it does not mean ISA values 223 * to PAGE_OFFSET is pure coincidence - it does not mean ISA values
@@ -341,13 +309,38 @@ BUILDIO(b, b, char)
341BUILDIO(w, w, short) 309BUILDIO(w, w, short)
342BUILDIO(l, , int) 310BUILDIO(l, , int)
343 311
312#define inb inb
313#define inw inw
314#define inl inl
315#define inb_p inb_p
316#define inw_p inw_p
317#define inl_p inl_p
318#define insb insb
319#define insw insw
320#define insl insl
321
322#define outb outb
323#define outw outw
324#define outl outl
325#define outb_p outb_p
326#define outw_p outw_p
327#define outl_p outl_p
328#define outsb outsb
329#define outsw outsw
330#define outsl outsl
331
344extern void *xlate_dev_mem_ptr(phys_addr_t phys); 332extern void *xlate_dev_mem_ptr(phys_addr_t phys);
345extern void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr); 333extern void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr);
346 334
335#define xlate_dev_mem_ptr xlate_dev_mem_ptr
336#define unxlate_dev_mem_ptr unxlate_dev_mem_ptr
337
347extern int ioremap_change_attr(unsigned long vaddr, unsigned long size, 338extern int ioremap_change_attr(unsigned long vaddr, unsigned long size,
348 enum page_cache_mode pcm); 339 enum page_cache_mode pcm);
349extern void __iomem *ioremap_wc(resource_size_t offset, unsigned long size); 340extern void __iomem *ioremap_wc(resource_size_t offset, unsigned long size);
341#define ioremap_wc ioremap_wc
350extern void __iomem *ioremap_wt(resource_size_t offset, unsigned long size); 342extern void __iomem *ioremap_wt(resource_size_t offset, unsigned long size);
343#define ioremap_wt ioremap_wt
351 344
352extern bool is_early_ioremap_ptep(pte_t *ptep); 345extern bool is_early_ioremap_ptep(pte_t *ptep);
353 346
@@ -365,6 +358,9 @@ extern bool xen_biovec_phys_mergeable(const struct bio_vec *vec1,
365 358
366#define IO_SPACE_LIMIT 0xffff 359#define IO_SPACE_LIMIT 0xffff
367 360
361#include <asm-generic/io.h>
362#undef PCI_IOBASE
363
368#ifdef CONFIG_MTRR 364#ifdef CONFIG_MTRR
369extern int __must_check arch_phys_wc_index(int handle); 365extern int __must_check arch_phys_wc_index(int handle);
370#define arch_phys_wc_index arch_phys_wc_index 366#define arch_phys_wc_index arch_phys_wc_index
diff --git a/arch/x86/include/asm/orc_types.h b/arch/x86/include/asm/orc_types.h
new file mode 100644
index 000000000000..7dc777a6cb40
--- /dev/null
+++ b/arch/x86/include/asm/orc_types.h
@@ -0,0 +1,107 @@
1/*
2 * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 2
7 * of the License, or (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, see <http://www.gnu.org/licenses/>.
16 */
17
18#ifndef _ORC_TYPES_H
19#define _ORC_TYPES_H
20
21#include <linux/types.h>
22#include <linux/compiler.h>
23
24/*
25 * The ORC_REG_* registers are base registers which are used to find other
26 * registers on the stack.
27 *
28 * ORC_REG_PREV_SP, also known as DWARF Call Frame Address (CFA), is the
29 * address of the previous frame: the caller's SP before it called the current
30 * function.
31 *
32 * ORC_REG_UNDEFINED means the corresponding register's value didn't change in
33 * the current frame.
34 *
35 * The most commonly used base registers are SP and BP -- which the previous SP
36 * is usually based on -- and PREV_SP and UNDEFINED -- which the previous BP is
37 * usually based on.
38 *
39 * The rest of the base registers are needed for special cases like entry code
40 * and GCC realigned stacks.
41 */
42#define ORC_REG_UNDEFINED 0
43#define ORC_REG_PREV_SP 1
44#define ORC_REG_DX 2
45#define ORC_REG_DI 3
46#define ORC_REG_BP 4
47#define ORC_REG_SP 5
48#define ORC_REG_R10 6
49#define ORC_REG_R13 7
50#define ORC_REG_BP_INDIRECT 8
51#define ORC_REG_SP_INDIRECT 9
52#define ORC_REG_MAX 15
53
54/*
55 * ORC_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP (the
56 * caller's SP right before it made the call). Used for all callable
57 * functions, i.e. all C code and all callable asm functions.
58 *
59 * ORC_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset points
60 * to a fully populated pt_regs from a syscall, interrupt, or exception.
61 *
62 * ORC_TYPE_REGS_IRET: Used in entry code to indicate that sp_reg+sp_offset
63 * points to the iret return frame.
64 *
65 * The UNWIND_HINT macros are used only for the unwind_hint struct. They
66 * aren't used in struct orc_entry due to size and complexity constraints.
67 * Objtool converts them to real types when it converts the hints to orc
68 * entries.
69 */
70#define ORC_TYPE_CALL 0
71#define ORC_TYPE_REGS 1
72#define ORC_TYPE_REGS_IRET 2
73#define UNWIND_HINT_TYPE_SAVE 3
74#define UNWIND_HINT_TYPE_RESTORE 4
75
76#ifndef __ASSEMBLY__
77/*
78 * This struct is more or less a vastly simplified version of the DWARF Call
79 * Frame Information standard. It contains only the necessary parts of DWARF
80 * CFI, simplified for ease of access by the in-kernel unwinder. It tells the
81 * unwinder how to find the previous SP and BP (and sometimes entry regs) on
82 * the stack for a given code address. Each instance of the struct corresponds
83 * to one or more code locations.
84 */
85struct orc_entry {
86 s16 sp_offset;
87 s16 bp_offset;
88 unsigned sp_reg:4;
89 unsigned bp_reg:4;
90 unsigned type:2;
91};
92
93/*
94 * This struct is used by asm and inline asm code to manually annotate the
95 * location of registers on the stack for the ORC unwinder.
96 *
97 * Type can be either ORC_TYPE_* or UNWIND_HINT_TYPE_*.
98 */
99struct unwind_hint {
100 u32 ip;
101 s16 sp_offset;
102 u8 sp_reg;
103 u8 type;
104};
105#endif /* __ASSEMBLY__ */
106
107#endif /* _ORC_TYPES_H */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 028245e1c42b..0b03d655db7c 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -22,6 +22,7 @@ struct vm86;
22#include <asm/nops.h> 22#include <asm/nops.h>
23#include <asm/special_insns.h> 23#include <asm/special_insns.h>
24#include <asm/fpu/types.h> 24#include <asm/fpu/types.h>
25#include <asm/unwind_hints.h>
25 26
26#include <linux/personality.h> 27#include <linux/personality.h>
27#include <linux/cache.h> 28#include <linux/cache.h>
@@ -684,6 +685,7 @@ static inline void sync_core(void)
684 unsigned int tmp; 685 unsigned int tmp;
685 686
686 asm volatile ( 687 asm volatile (
688 UNWIND_HINT_SAVE
687 "mov %%ss, %0\n\t" 689 "mov %%ss, %0\n\t"
688 "pushq %q0\n\t" 690 "pushq %q0\n\t"
689 "pushq %%rsp\n\t" 691 "pushq %%rsp\n\t"
@@ -693,6 +695,7 @@ static inline void sync_core(void)
693 "pushq %q0\n\t" 695 "pushq %q0\n\t"
694 "pushq $1f\n\t" 696 "pushq $1f\n\t"
695 "iretq\n\t" 697 "iretq\n\t"
698 UNWIND_HINT_RESTORE
696 "1:" 699 "1:"
697 : "=&r" (tmp), "+r" (__sp) : : "cc", "memory"); 700 : "=&r" (tmp), "+r" (__sp) : : "cc", "memory");
698#endif 701#endif
diff --git a/arch/x86/include/asm/rmwcc.h b/arch/x86/include/asm/rmwcc.h
index 661dd305694a..045f99211a99 100644
--- a/arch/x86/include/asm/rmwcc.h
+++ b/arch/x86/include/asm/rmwcc.h
@@ -1,45 +1,56 @@
1#ifndef _ASM_X86_RMWcc 1#ifndef _ASM_X86_RMWcc
2#define _ASM_X86_RMWcc 2#define _ASM_X86_RMWcc
3 3
4#define __CLOBBERS_MEM "memory"
5#define __CLOBBERS_MEM_CC_CX "memory", "cc", "cx"
6
4#if !defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(CC_HAVE_ASM_GOTO) 7#if !defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(CC_HAVE_ASM_GOTO)
5 8
6/* Use asm goto */ 9/* Use asm goto */
7 10
8#define __GEN_RMWcc(fullop, var, cc, ...) \ 11#define __GEN_RMWcc(fullop, var, cc, clobbers, ...) \
9do { \ 12do { \
10 asm_volatile_goto (fullop "; j" #cc " %l[cc_label]" \ 13 asm_volatile_goto (fullop "; j" #cc " %l[cc_label]" \
11 : : "m" (var), ## __VA_ARGS__ \ 14 : : [counter] "m" (var), ## __VA_ARGS__ \
12 : "memory" : cc_label); \ 15 : clobbers : cc_label); \
13 return 0; \ 16 return 0; \
14cc_label: \ 17cc_label: \
15 return 1; \ 18 return 1; \
16} while (0) 19} while (0)
17 20
18#define GEN_UNARY_RMWcc(op, var, arg0, cc) \ 21#define __BINARY_RMWcc_ARG " %1, "
19 __GEN_RMWcc(op " " arg0, var, cc)
20 22
21#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \
22 __GEN_RMWcc(op " %1, " arg0, var, cc, vcon (val))
23 23
24#else /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */ 24#else /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */
25 25
26/* Use flags output or a set instruction */ 26/* Use flags output or a set instruction */
27 27
28#define __GEN_RMWcc(fullop, var, cc, ...) \ 28#define __GEN_RMWcc(fullop, var, cc, clobbers, ...) \
29do { \ 29do { \
30 bool c; \ 30 bool c; \
31 asm volatile (fullop ";" CC_SET(cc) \ 31 asm volatile (fullop ";" CC_SET(cc) \
32 : "+m" (var), CC_OUT(cc) (c) \ 32 : [counter] "+m" (var), CC_OUT(cc) (c) \
33 : __VA_ARGS__ : "memory"); \ 33 : __VA_ARGS__ : clobbers); \
34 return c; \ 34 return c; \
35} while (0) 35} while (0)
36 36
37#define __BINARY_RMWcc_ARG " %2, "
38
39#endif /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */
40
37#define GEN_UNARY_RMWcc(op, var, arg0, cc) \ 41#define GEN_UNARY_RMWcc(op, var, arg0, cc) \
38 __GEN_RMWcc(op " " arg0, var, cc) 42 __GEN_RMWcc(op " " arg0, var, cc, __CLOBBERS_MEM)
43
44#define GEN_UNARY_SUFFIXED_RMWcc(op, suffix, var, arg0, cc) \
45 __GEN_RMWcc(op " " arg0 "\n\t" suffix, var, cc, \
46 __CLOBBERS_MEM_CC_CX)
39 47
40#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \ 48#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \
41 __GEN_RMWcc(op " %2, " arg0, var, cc, vcon (val)) 49 __GEN_RMWcc(op __BINARY_RMWcc_ARG arg0, var, cc, \
50 __CLOBBERS_MEM, vcon (val))
42 51
43#endif /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */ 52#define GEN_BINARY_SUFFIXED_RMWcc(op, suffix, var, vcon, val, arg0, cc) \
53 __GEN_RMWcc(op __BINARY_RMWcc_ARG arg0 "\n\t" suffix, var, cc, \
54 __CLOBBERS_MEM_CC_CX, vcon (val))
44 55
45#endif /* _ASM_X86_RMWcc */ 56#endif /* _ASM_X86_RMWcc */
diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h
new file mode 100644
index 000000000000..5e02b11c9b86
--- /dev/null
+++ b/arch/x86/include/asm/unwind_hints.h
@@ -0,0 +1,103 @@
1#ifndef _ASM_X86_UNWIND_HINTS_H
2#define _ASM_X86_UNWIND_HINTS_H
3
4#include "orc_types.h"
5
6#ifdef __ASSEMBLY__
7
8/*
9 * In asm, there are two kinds of code: normal C-type callable functions and
10 * the rest. The normal callable functions can be called by other code, and
11 * don't do anything unusual with the stack. Such normal callable functions
12 * are annotated with the ENTRY/ENDPROC macros. Most asm code falls in this
13 * category. In this case, no special debugging annotations are needed because
14 * objtool can automatically generate the ORC data for the ORC unwinder to read
15 * at runtime.
16 *
17 * Anything which doesn't fall into the above category, such as syscall and
18 * interrupt handlers, tends to not be called directly by other functions, and
19 * often does unusual non-C-function-type things with the stack pointer. Such
20 * code needs to be annotated such that objtool can understand it. The
21 * following CFI hint macros are for this type of code.
22 *
23 * These macros provide hints to objtool about the state of the stack at each
24 * instruction. Objtool starts from the hints and follows the code flow,
25 * making automatic CFI adjustments when it sees pushes and pops, filling out
26 * the debuginfo as necessary. It will also warn if it sees any
27 * inconsistencies.
28 */
29.macro UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=0 type=ORC_TYPE_CALL
30#ifdef CONFIG_STACK_VALIDATION
31.Lunwind_hint_ip_\@:
32 .pushsection .discard.unwind_hints
33 /* struct unwind_hint */
34 .long .Lunwind_hint_ip_\@ - .
35 .short \sp_offset
36 .byte \sp_reg
37 .byte \type
38 .popsection
39#endif
40.endm
41
42.macro UNWIND_HINT_EMPTY
43 UNWIND_HINT sp_reg=ORC_REG_UNDEFINED
44.endm
45
46.macro UNWIND_HINT_REGS base=%rsp offset=0 indirect=0 extra=1 iret=0
47 .if \base == %rsp && \indirect
48 .set sp_reg, ORC_REG_SP_INDIRECT
49 .elseif \base == %rsp
50 .set sp_reg, ORC_REG_SP
51 .elseif \base == %rbp
52 .set sp_reg, ORC_REG_BP
53 .elseif \base == %rdi
54 .set sp_reg, ORC_REG_DI
55 .elseif \base == %rdx
56 .set sp_reg, ORC_REG_DX
57 .elseif \base == %r10
58 .set sp_reg, ORC_REG_R10
59 .else
60 .error "UNWIND_HINT_REGS: bad base register"
61 .endif
62
63 .set sp_offset, \offset
64
65 .if \iret
66 .set type, ORC_TYPE_REGS_IRET
67 .elseif \extra == 0
68 .set type, ORC_TYPE_REGS_IRET
69 .set sp_offset, \offset + (16*8)
70 .else
71 .set type, ORC_TYPE_REGS
72 .endif
73
74 UNWIND_HINT sp_reg=sp_reg sp_offset=sp_offset type=type
75.endm
76
77.macro UNWIND_HINT_IRET_REGS base=%rsp offset=0
78 UNWIND_HINT_REGS base=\base offset=\offset iret=1
79.endm
80
81.macro UNWIND_HINT_FUNC sp_offset=8
82 UNWIND_HINT sp_offset=\sp_offset
83.endm
84
85#else /* !__ASSEMBLY__ */
86
87#define UNWIND_HINT(sp_reg, sp_offset, type) \
88 "987: \n\t" \
89 ".pushsection .discard.unwind_hints\n\t" \
90 /* struct unwind_hint */ \
91 ".long 987b - .\n\t" \
92 ".short " __stringify(sp_offset) "\n\t" \
93 ".byte " __stringify(sp_reg) "\n\t" \
94 ".byte " __stringify(type) "\n\t" \
95 ".popsection\n\t"
96
97#define UNWIND_HINT_SAVE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_SAVE)
98
99#define UNWIND_HINT_RESTORE UNWIND_HINT(0, 0, UNWIND_HINT_TYPE_RESTORE)
100
101#endif /* __ASSEMBLY__ */
102
103#endif /* _ASM_X86_UNWIND_HINTS_H */
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index dbce3cca94cb..bd265a4cf108 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -94,6 +94,9 @@ void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
94 if (stack_name) 94 if (stack_name)
95 printk("%s <%s>\n", log_lvl, stack_name); 95 printk("%s <%s>\n", log_lvl, stack_name);
96 96
97 if (regs && on_stack(&stack_info, regs, sizeof(*regs)))
98 __show_regs(regs, 0);
99
97 /* 100 /*
98 * Scan the stack, printing any text addresses we find. At the 101 * Scan the stack, printing any text addresses we find. At the
99 * same time, follow proper stack frames with the unwinder. 102 * same time, follow proper stack frames with the unwinder.
@@ -118,10 +121,8 @@ void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
118 * Don't print regs->ip again if it was already printed 121 * Don't print regs->ip again if it was already printed
119 * by __show_regs() below. 122 * by __show_regs() below.
120 */ 123 */
121 if (regs && stack == &regs->ip) { 124 if (regs && stack == &regs->ip)
122 unwind_next_frame(&state); 125 goto next;
123 continue;
124 }
125 126
126 if (stack == ret_addr_p) 127 if (stack == ret_addr_p)
127 reliable = 1; 128 reliable = 1;
@@ -144,6 +145,7 @@ void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
144 if (!reliable) 145 if (!reliable)
145 continue; 146 continue;
146 147
148next:
147 /* 149 /*
148 * Get the next frame from the unwinder. No need to 150 * Get the next frame from the unwinder. No need to
149 * check for an error: if anything goes wrong, the rest 151 * check for an error: if anything goes wrong, the rest
@@ -153,7 +155,7 @@ void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
153 155
154 /* if the frame has entry regs, print them */ 156 /* if the frame has entry regs, print them */
155 regs = unwind_get_entry_regs(&state); 157 regs = unwind_get_entry_regs(&state);
156 if (regs) 158 if (regs && on_stack(&stack_info, regs, sizeof(*regs)))
157 __show_regs(regs, 0); 159 __show_regs(regs, 0);
158 } 160 }
159 161
diff --git a/arch/x86/kernel/dumpstack_32.c b/arch/x86/kernel/dumpstack_32.c
index e5f0b40e66d2..4f0481474903 100644
--- a/arch/x86/kernel/dumpstack_32.c
+++ b/arch/x86/kernel/dumpstack_32.c
@@ -37,7 +37,7 @@ static bool in_hardirq_stack(unsigned long *stack, struct stack_info *info)
37 * This is a software stack, so 'end' can be a valid stack pointer. 37 * This is a software stack, so 'end' can be a valid stack pointer.
38 * It just means the stack is empty. 38 * It just means the stack is empty.
39 */ 39 */
40 if (stack < begin || stack > end) 40 if (stack <= begin || stack > end)
41 return false; 41 return false;
42 42
43 info->type = STACK_TYPE_IRQ; 43 info->type = STACK_TYPE_IRQ;
@@ -62,7 +62,7 @@ static bool in_softirq_stack(unsigned long *stack, struct stack_info *info)
62 * This is a software stack, so 'end' can be a valid stack pointer. 62 * This is a software stack, so 'end' can be a valid stack pointer.
63 * It just means the stack is empty. 63 * It just means the stack is empty.
64 */ 64 */
65 if (stack < begin || stack > end) 65 if (stack <= begin || stack > end)
66 return false; 66 return false;
67 67
68 info->type = STACK_TYPE_SOFTIRQ; 68 info->type = STACK_TYPE_SOFTIRQ;
diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
index 3e1471d57487..225af4184f06 100644
--- a/arch/x86/kernel/dumpstack_64.c
+++ b/arch/x86/kernel/dumpstack_64.c
@@ -55,7 +55,7 @@ static bool in_exception_stack(unsigned long *stack, struct stack_info *info)
55 begin = end - (exception_stack_sizes[k] / sizeof(long)); 55 begin = end - (exception_stack_sizes[k] / sizeof(long));
56 regs = (struct pt_regs *)end - 1; 56 regs = (struct pt_regs *)end - 1;
57 57
58 if (stack < begin || stack >= end) 58 if (stack <= begin || stack >= end)
59 continue; 59 continue;
60 60
61 info->type = STACK_TYPE_EXCEPTION + k; 61 info->type = STACK_TYPE_EXCEPTION + k;
@@ -78,7 +78,7 @@ static bool in_irq_stack(unsigned long *stack, struct stack_info *info)
78 * This is a software stack, so 'end' can be a valid stack pointer. 78 * This is a software stack, so 'end' can be a valid stack pointer.
79 * It just means the stack is empty. 79 * It just means the stack is empty.
80 */ 80 */
81 if (stack < begin || stack > end) 81 if (stack <= begin || stack > end)
82 return false; 82 return false;
83 83
84 info->type = STACK_TYPE_IRQ; 84 info->type = STACK_TYPE_IRQ;
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index c3169be4c596..2987e3991c2b 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -279,6 +279,9 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
279 struct tss_struct *tss = &per_cpu(cpu_tss, cpu); 279 struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
280 unsigned prev_fsindex, prev_gsindex; 280 unsigned prev_fsindex, prev_gsindex;
281 281
282 WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
283 this_cpu_read(irq_count) != -1);
284
282 switch_fpu_prepare(prev_fpu, cpu); 285 switch_fpu_prepare(prev_fpu, cpu);
283 286
284 /* We must save %fs and %gs before load_TLS() because 287 /* We must save %fs and %gs before load_TLS() because
diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h
index 7ef015eb3403..b4531e3b2120 100644
--- a/include/asm-generic/io.h
+++ b/include/asm-generic/io.h
@@ -915,6 +915,9 @@ extern void ioport_unmap(void __iomem *p);
915#endif /* CONFIG_GENERIC_IOMAP */ 915#endif /* CONFIG_GENERIC_IOMAP */
916#endif /* CONFIG_HAS_IOPORT_MAP */ 916#endif /* CONFIG_HAS_IOPORT_MAP */
917 917
918/*
919 * Convert a virtual cached pointer to an uncached pointer
920 */
918#ifndef xlate_dev_kmem_ptr 921#ifndef xlate_dev_kmem_ptr
919#define xlate_dev_kmem_ptr xlate_dev_kmem_ptr 922#define xlate_dev_kmem_ptr xlate_dev_kmem_ptr
920static inline void *xlate_dev_kmem_ptr(void *addr) 923static inline void *xlate_dev_kmem_ptr(void *addr)
@@ -954,6 +957,14 @@ static inline void *bus_to_virt(unsigned long address)
954 957
955#ifndef memset_io 958#ifndef memset_io
956#define memset_io memset_io 959#define memset_io memset_io
960/**
961 * memset_io Set a range of I/O memory to a constant value
962 * @addr: The beginning of the I/O-memory range to set
963 * @val: The value to set the memory to
964 * @count: The number of bytes to set
965 *
966 * Set a range of I/O memory to a given value.
967 */
957static inline void memset_io(volatile void __iomem *addr, int value, 968static inline void memset_io(volatile void __iomem *addr, int value,
958 size_t size) 969 size_t size)
959{ 970{
@@ -963,6 +974,14 @@ static inline void memset_io(volatile void __iomem *addr, int value,
963 974
964#ifndef memcpy_fromio 975#ifndef memcpy_fromio
965#define memcpy_fromio memcpy_fromio 976#define memcpy_fromio memcpy_fromio
977/**
978 * memcpy_fromio Copy a block of data from I/O memory
979 * @dst: The (RAM) destination for the copy
980 * @src: The (I/O memory) source for the data
981 * @count: The number of bytes to copy
982 *
983 * Copy a block of data from I/O memory.
984 */
966static inline void memcpy_fromio(void *buffer, 985static inline void memcpy_fromio(void *buffer,
967 const volatile void __iomem *addr, 986 const volatile void __iomem *addr,
968 size_t size) 987 size_t size)
@@ -973,6 +992,14 @@ static inline void memcpy_fromio(void *buffer,
973 992
974#ifndef memcpy_toio 993#ifndef memcpy_toio
975#define memcpy_toio memcpy_toio 994#define memcpy_toio memcpy_toio
995/**
996 * memcpy_toio Copy a block of data into I/O memory
997 * @dst: The (I/O memory) destination for the copy
998 * @src: The (RAM) source for the data
999 * @count: The number of bytes to copy
1000 *
1001 * Copy a block of data to I/O memory.
1002 */
976static inline void memcpy_toio(volatile void __iomem *addr, const void *buffer, 1003static inline void memcpy_toio(volatile void __iomem *addr, const void *buffer,
977 size_t size) 1004 size_t size)
978{ 1005{
diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h
index bdb80c4aef6e..10825052b03f 100644
--- a/include/linux/compiler-gcc.h
+++ b/include/linux/compiler-gcc.h
@@ -203,11 +203,16 @@
203 203
204#ifdef CONFIG_STACK_VALIDATION 204#ifdef CONFIG_STACK_VALIDATION
205#define annotate_unreachable() ({ \ 205#define annotate_unreachable() ({ \
206 asm("%c0:\t\n" \ 206 asm("%c0:\n\t" \
207 ".pushsection .discard.unreachable\t\n" \ 207 ".pushsection .discard.unreachable\n\t" \
208 ".long %c0b - .\t\n" \ 208 ".long %c0b - .\n\t" \
209 ".popsection\t\n" : : "i" (__LINE__)); \ 209 ".popsection\n\t" : : "i" (__LINE__)); \
210}) 210})
211#define ASM_UNREACHABLE \
212 "999:\n\t" \
213 ".pushsection .discard.unreachable\n\t" \
214 ".long 999b - .\n\t" \
215 ".popsection\n\t"
211#else 216#else
212#define annotate_unreachable() 217#define annotate_unreachable()
213#endif 218#endif
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index eca8ad75e28b..e25746d88697 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -185,6 +185,9 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
185#endif 185#endif
186 186
187/* Unreachable code */ 187/* Unreachable code */
188#ifndef ASM_UNREACHABLE
189# define ASM_UNREACHABLE
190#endif
188#ifndef unreachable 191#ifndef unreachable
189# define unreachable() do { } while (1) 192# define unreachable() do { } while (1)
190#endif 193#endif
diff --git a/scripts/Makefile.build b/scripts/Makefile.build
index 4a9a2cec0a1b..854608d42e85 100644
--- a/scripts/Makefile.build
+++ b/scripts/Makefile.build
@@ -262,6 +262,9 @@ objtool_args = check
262ifndef CONFIG_FRAME_POINTER 262ifndef CONFIG_FRAME_POINTER
263objtool_args += --no-fp 263objtool_args += --no-fp
264endif 264endif
265ifdef CONFIG_GCOV_KERNEL
266objtool_args += --no-unreachable
267endif
265 268
266# 'OBJECT_FILES_NON_STANDARD := y': skip objtool checking for a directory 269# 'OBJECT_FILES_NON_STANDARD := y': skip objtool checking for a directory
267# 'OBJECT_FILES_NON_STANDARD_foo.o := 'y': skip objtool checking for a file 270# 'OBJECT_FILES_NON_STANDARD_foo.o := 'y': skip objtool checking for a file
diff --git a/tools/objtool/Build b/tools/objtool/Build
index 6f2e1987c4d9..749becdf5b90 100644
--- a/tools/objtool/Build
+++ b/tools/objtool/Build
@@ -1,6 +1,9 @@
1objtool-y += arch/$(SRCARCH)/ 1objtool-y += arch/$(SRCARCH)/
2objtool-y += builtin-check.o 2objtool-y += builtin-check.o
3objtool-y += builtin-orc.o
3objtool-y += check.o 4objtool-y += check.o
5objtool-y += orc_gen.o
6objtool-y += orc_dump.o
4objtool-y += elf.o 7objtool-y += elf.o
5objtool-y += special.o 8objtool-y += special.o
6objtool-y += objtool.o 9objtool-y += objtool.o
diff --git a/tools/objtool/Documentation/stack-validation.txt b/tools/objtool/Documentation/stack-validation.txt
index 17c1195f11f4..6a1af43862df 100644
--- a/tools/objtool/Documentation/stack-validation.txt
+++ b/tools/objtool/Documentation/stack-validation.txt
@@ -11,9 +11,6 @@ analyzes every .o file and ensures the validity of its stack metadata.
11It enforces a set of rules on asm code and C inline assembly code so 11It enforces a set of rules on asm code and C inline assembly code so
12that stack traces can be reliable. 12that stack traces can be reliable.
13 13
14Currently it only checks frame pointer usage, but there are plans to add
15CFI validation for C files and CFI generation for asm files.
16
17For each function, it recursively follows all possible code paths and 14For each function, it recursively follows all possible code paths and
18validates the correct frame pointer state at each instruction. 15validates the correct frame pointer state at each instruction.
19 16
@@ -23,6 +20,10 @@ alternative execution paths to a given instruction (or set of
23instructions). Similarly, it knows how to follow switch statements, for 20instructions). Similarly, it knows how to follow switch statements, for
24which gcc sometimes uses jump tables. 21which gcc sometimes uses jump tables.
25 22
23(Objtool also has an 'orc generate' subcommand which generates debuginfo
24for the ORC unwinder. See Documentation/x86/orc-unwinder.txt in the
25kernel tree for more details.)
26
26 27
27Why do we need stack metadata validation? 28Why do we need stack metadata validation?
28----------------------------------------- 29-----------------------------------------
@@ -93,37 +94,14 @@ a) More reliable stack traces for frame pointer enabled kernels
93 or at the very end of the function after the stack frame has been 94 or at the very end of the function after the stack frame has been
94 destroyed. This is an inherent limitation of frame pointers. 95 destroyed. This is an inherent limitation of frame pointers.
95 96
96b) 100% reliable stack traces for DWARF enabled kernels 97b) ORC (Oops Rewind Capability) unwind table generation
97
98 (NOTE: This is not yet implemented)
99
100 As an alternative to frame pointers, DWARF Call Frame Information
101 (CFI) metadata can be used to walk the stack. Unlike frame pointers,
102 CFI metadata is out of band. So it doesn't affect runtime
103 performance and it can be reliable even when interrupts or exceptions
104 are involved.
105
106 For C code, gcc automatically generates DWARF CFI metadata. But for
107 asm code, generating CFI is a tedious manual approach which requires
108 manually placed .cfi assembler macros to be scattered throughout the
109 code. It's clumsy and very easy to get wrong, and it makes the real
110 code harder to read.
111
112 Stacktool will improve this situation in several ways. For code
113 which already has CFI annotations, it will validate them. For code
114 which doesn't have CFI annotations, it will generate them. So an
115 architecture can opt to strip out all the manual .cfi annotations
116 from their asm code and have objtool generate them instead.
117 98
118 We might also add a runtime stack validation debug option where we 99 An alternative to frame pointers and DWARF, ORC unwind data can be
119 periodically walk the stack from schedule() and/or an NMI to ensure 100 used to walk the stack. Unlike frame pointers, ORC data is out of
120 that the stack metadata is sane and that we reach the bottom of the 101 band. So it doesn't affect runtime performance and it can be
121 stack. 102 reliable even when interrupts or exceptions are involved.
122 103
123 So the benefit of objtool here will be that external tooling should 104 For more details, see Documentation/x86/orc-unwinder.txt.
124 always show perfect stack traces. And the same will be true for
125 kernel warning/oops traces if the architecture has a runtime DWARF
126 unwinder.
127 105
128c) Higher live patching compatibility rate 106c) Higher live patching compatibility rate
129 107
@@ -211,7 +189,7 @@ they mean, and suggestions for how to fix them.
211 function, add proper frame pointer logic using the FRAME_BEGIN and 189 function, add proper frame pointer logic using the FRAME_BEGIN and
212 FRAME_END macros. Otherwise, if it's not a callable function, remove 190 FRAME_END macros. Otherwise, if it's not a callable function, remove
213 its ELF function annotation by changing ENDPROC to END, and instead 191 its ELF function annotation by changing ENDPROC to END, and instead
214 use the manual CFI hint macros in asm/undwarf.h. 192 use the manual unwind hint macros in asm/unwind_hints.h.
215 193
216 If it's a GCC-compiled .c file, the error may be because the function 194 If it's a GCC-compiled .c file, the error may be because the function
217 uses an inline asm() statement which has a "call" instruction. An 195 uses an inline asm() statement which has a "call" instruction. An
@@ -231,8 +209,8 @@ they mean, and suggestions for how to fix them.
231 If the error is for an asm file, and the instruction is inside (or 209 If the error is for an asm file, and the instruction is inside (or
232 reachable from) a callable function, the function should be annotated 210 reachable from) a callable function, the function should be annotated
233 with the ENTRY/ENDPROC macros (ENDPROC is the important one). 211 with the ENTRY/ENDPROC macros (ENDPROC is the important one).
234 Otherwise, the code should probably be annotated with the CFI hint 212 Otherwise, the code should probably be annotated with the unwind hint
235 macros in asm/undwarf.h so objtool and the unwinder can know the 213 macros in asm/unwind_hints.h so objtool and the unwinder can know the
236 stack state associated with the code. 214 stack state associated with the code.
237 215
238 If you're 100% sure the code won't affect stack traces, or if you're 216 If you're 100% sure the code won't affect stack traces, or if you're
@@ -258,7 +236,7 @@ they mean, and suggestions for how to fix them.
258 instructions aren't allowed in a callable function, and are most 236 instructions aren't allowed in a callable function, and are most
259 likely part of the kernel entry code. They should usually not have 237 likely part of the kernel entry code. They should usually not have
260 the callable function annotation (ENDPROC) and should always be 238 the callable function annotation (ENDPROC) and should always be
261 annotated with the CFI hint macros in asm/undwarf.h. 239 annotated with the unwind hint macros in asm/unwind_hints.h.
262 240
263 241
2646. file.o: warning: objtool: func()+0x26: sibling call from callable instruction with modified stack frame 2426. file.o: warning: objtool: func()+0x26: sibling call from callable instruction with modified stack frame
@@ -272,7 +250,7 @@ they mean, and suggestions for how to fix them.
272 250
273 If the instruction is not actually in a callable function (e.g. 251 If the instruction is not actually in a callable function (e.g.
274 kernel entry code), change ENDPROC to END and annotate manually with 252 kernel entry code), change ENDPROC to END and annotate manually with
275 the CFI hint macros in asm/undwarf.h. 253 the unwind hint macros in asm/unwind_hints.h.
276 254
277 255
2787. file: warning: objtool: func()+0x5c: stack state mismatch 2567. file: warning: objtool: func()+0x5c: stack state mismatch
@@ -288,8 +266,8 @@ they mean, and suggestions for how to fix them.
288 266
289 Another possibility is that the code has some asm or inline asm which 267 Another possibility is that the code has some asm or inline asm which
290 does some unusual things to the stack or the frame pointer. In such 268 does some unusual things to the stack or the frame pointer. In such
291 cases it's probably appropriate to use the CFI hint macros in 269 cases it's probably appropriate to use the unwind hint macros in
292 asm/undwarf.h. 270 asm/unwind_hints.h.
293 271
294 272
2958. file.o: warning: objtool: funcA() falls through to next function funcB() 2738. file.o: warning: objtool: funcA() falls through to next function funcB()
diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile
index 0e2765e243c0..3a6425fefc43 100644
--- a/tools/objtool/Makefile
+++ b/tools/objtool/Makefile
@@ -52,6 +52,9 @@ $(OBJTOOL): $(LIBSUBCMD) $(OBJTOOL_IN)
52 diff -I'^#include' arch/x86/insn/inat.h ../../arch/x86/include/asm/inat.h >/dev/null && \ 52 diff -I'^#include' arch/x86/insn/inat.h ../../arch/x86/include/asm/inat.h >/dev/null && \
53 diff -I'^#include' arch/x86/insn/inat_types.h ../../arch/x86/include/asm/inat_types.h >/dev/null) \ 53 diff -I'^#include' arch/x86/insn/inat_types.h ../../arch/x86/include/asm/inat_types.h >/dev/null) \
54 || echo "warning: objtool: x86 instruction decoder differs from kernel" >&2 )) || true 54 || echo "warning: objtool: x86 instruction decoder differs from kernel" >&2 )) || true
55 @(test -d ../../kernel -a -d ../../tools -a -d ../objtool && (( \
56 diff ../../arch/x86/include/asm/orc_types.h orc_types.h >/dev/null) \
57 || echo "warning: objtool: orc_types.h differs from kernel" >&2 )) || true
55 $(QUIET_LINK)$(CC) $(OBJTOOL_IN) $(LDFLAGS) -o $@ 58 $(QUIET_LINK)$(CC) $(OBJTOOL_IN) $(LDFLAGS) -o $@
56 59
57 60
diff --git a/tools/objtool/builtin-check.c b/tools/objtool/builtin-check.c
index 365c34ecab26..57254f5b2779 100644
--- a/tools/objtool/builtin-check.c
+++ b/tools/objtool/builtin-check.c
@@ -29,7 +29,7 @@
29#include "builtin.h" 29#include "builtin.h"
30#include "check.h" 30#include "check.h"
31 31
32bool nofp; 32bool no_fp, no_unreachable;
33 33
34static const char * const check_usage[] = { 34static const char * const check_usage[] = {
35 "objtool check [<options>] file.o", 35 "objtool check [<options>] file.o",
@@ -37,7 +37,8 @@ static const char * const check_usage[] = {
37}; 37};
38 38
39const struct option check_options[] = { 39const struct option check_options[] = {
40 OPT_BOOLEAN('f', "no-fp", &nofp, "Skip frame pointer validation"), 40 OPT_BOOLEAN('f', "no-fp", &no_fp, "Skip frame pointer validation"),
41 OPT_BOOLEAN('u', "no-unreachable", &no_unreachable, "Skip 'unreachable instruction' warnings"),
41 OPT_END(), 42 OPT_END(),
42}; 43};
43 44
@@ -52,5 +53,5 @@ int cmd_check(int argc, const char **argv)
52 53
53 objname = argv[0]; 54 objname = argv[0];
54 55
55 return check(objname, nofp); 56 return check(objname, no_fp, no_unreachable, false);
56} 57}
diff --git a/tools/objtool/builtin-orc.c b/tools/objtool/builtin-orc.c
new file mode 100644
index 000000000000..4c6b5c9ef073
--- /dev/null
+++ b/tools/objtool/builtin-orc.c
@@ -0,0 +1,70 @@
1/*
2 * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 2
7 * of the License, or (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, see <http://www.gnu.org/licenses/>.
16 */
17
18/*
19 * objtool orc:
20 *
21 * This command analyzes a .o file and adds .orc_unwind and .orc_unwind_ip
22 * sections to it, which is used by the in-kernel ORC unwinder.
23 *
24 * This command is a superset of "objtool check".
25 */
26
27#include <string.h>
28#include <subcmd/parse-options.h>
29#include "builtin.h"
30#include "check.h"
31
32
33static const char *orc_usage[] = {
34 "objtool orc generate [<options>] file.o",
35 "objtool orc dump file.o",
36 NULL,
37};
38
39extern const struct option check_options[];
40extern bool no_fp, no_unreachable;
41
42int cmd_orc(int argc, const char **argv)
43{
44 const char *objname;
45
46 argc--; argv++;
47 if (!strncmp(argv[0], "gen", 3)) {
48 argc = parse_options(argc, argv, check_options, orc_usage, 0);
49 if (argc != 1)
50 usage_with_options(orc_usage, check_options);
51
52 objname = argv[0];
53
54 return check(objname, no_fp, no_unreachable, true);
55
56 }
57
58 if (!strcmp(argv[0], "dump")) {
59 if (argc != 2)
60 usage_with_options(orc_usage, check_options);
61
62 objname = argv[1];
63
64 return orc_dump(objname);
65 }
66
67 usage_with_options(orc_usage, check_options);
68
69 return 0;
70}
diff --git a/tools/objtool/builtin.h b/tools/objtool/builtin.h
index 34d2ba78a616..dd526067fed5 100644
--- a/tools/objtool/builtin.h
+++ b/tools/objtool/builtin.h
@@ -18,5 +18,6 @@
18#define _BUILTIN_H 18#define _BUILTIN_H
19 19
20extern int cmd_check(int argc, const char **argv); 20extern int cmd_check(int argc, const char **argv);
21extern int cmd_orc(int argc, const char **argv);
21 22
22#endif /* _BUILTIN_H */ 23#endif /* _BUILTIN_H */
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index 2c6d74880403..3436a942b606 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -33,11 +33,11 @@ struct alternative {
33}; 33};
34 34
35const char *objname; 35const char *objname;
36static bool nofp; 36static bool no_fp;
37struct cfi_state initial_func_cfi; 37struct cfi_state initial_func_cfi;
38 38
39static struct instruction *find_insn(struct objtool_file *file, 39struct instruction *find_insn(struct objtool_file *file,
40 struct section *sec, unsigned long offset) 40 struct section *sec, unsigned long offset)
41{ 41{
42 struct instruction *insn; 42 struct instruction *insn;
43 43
@@ -59,19 +59,6 @@ static struct instruction *next_insn_same_sec(struct objtool_file *file,
59 return next; 59 return next;
60} 60}
61 61
62static bool gcov_enabled(struct objtool_file *file)
63{
64 struct section *sec;
65 struct symbol *sym;
66
67 for_each_sec(file, sec)
68 list_for_each_entry(sym, &sec->symbol_list, list)
69 if (!strncmp(sym->name, "__gcov_.", 8))
70 return true;
71
72 return false;
73}
74
75#define func_for_each_insn(file, func, insn) \ 62#define func_for_each_insn(file, func, insn) \
76 for (insn = find_insn(file, func->sec, func->offset); \ 63 for (insn = find_insn(file, func->sec, func->offset); \
77 insn && &insn->list != &file->insn_list && \ 64 insn && &insn->list != &file->insn_list && \
@@ -100,7 +87,6 @@ static bool gcov_enabled(struct objtool_file *file)
100static bool ignore_func(struct objtool_file *file, struct symbol *func) 87static bool ignore_func(struct objtool_file *file, struct symbol *func)
101{ 88{
102 struct rela *rela; 89 struct rela *rela;
103 struct instruction *insn;
104 90
105 /* check for STACK_FRAME_NON_STANDARD */ 91 /* check for STACK_FRAME_NON_STANDARD */
106 if (file->whitelist && file->whitelist->rela) 92 if (file->whitelist && file->whitelist->rela)
@@ -113,11 +99,6 @@ static bool ignore_func(struct objtool_file *file, struct symbol *func)
113 return true; 99 return true;
114 } 100 }
115 101
116 /* check if it has a context switching instruction */
117 func_for_each_insn(file, func, insn)
118 if (insn->type == INSN_CONTEXT_SWITCH)
119 return true;
120
121 return false; 102 return false;
122} 103}
123 104
@@ -259,6 +240,11 @@ static int decode_instructions(struct objtool_file *file)
259 if (!(sec->sh.sh_flags & SHF_EXECINSTR)) 240 if (!(sec->sh.sh_flags & SHF_EXECINSTR))
260 continue; 241 continue;
261 242
243 if (strcmp(sec->name, ".altinstr_replacement") &&
244 strcmp(sec->name, ".altinstr_aux") &&
245 strncmp(sec->name, ".discard.", 9))
246 sec->text = true;
247
262 for (offset = 0; offset < sec->len; offset += insn->len) { 248 for (offset = 0; offset < sec->len; offset += insn->len) {
263 insn = malloc(sizeof(*insn)); 249 insn = malloc(sizeof(*insn));
264 if (!insn) { 250 if (!insn) {
@@ -874,6 +860,99 @@ static int add_switch_table_alts(struct objtool_file *file)
874 return 0; 860 return 0;
875} 861}
876 862
863static int read_unwind_hints(struct objtool_file *file)
864{
865 struct section *sec, *relasec;
866 struct rela *rela;
867 struct unwind_hint *hint;
868 struct instruction *insn;
869 struct cfi_reg *cfa;
870 int i;
871
872 sec = find_section_by_name(file->elf, ".discard.unwind_hints");
873 if (!sec)
874 return 0;
875
876 relasec = sec->rela;
877 if (!relasec) {
878 WARN("missing .rela.discard.unwind_hints section");
879 return -1;
880 }
881
882 if (sec->len % sizeof(struct unwind_hint)) {
883 WARN("struct unwind_hint size mismatch");
884 return -1;
885 }
886
887 file->hints = true;
888
889 for (i = 0; i < sec->len / sizeof(struct unwind_hint); i++) {
890 hint = (struct unwind_hint *)sec->data->d_buf + i;
891
892 rela = find_rela_by_dest(sec, i * sizeof(*hint));
893 if (!rela) {
894 WARN("can't find rela for unwind_hints[%d]", i);
895 return -1;
896 }
897
898 insn = find_insn(file, rela->sym->sec, rela->addend);
899 if (!insn) {
900 WARN("can't find insn for unwind_hints[%d]", i);
901 return -1;
902 }
903
904 cfa = &insn->state.cfa;
905
906 if (hint->type == UNWIND_HINT_TYPE_SAVE) {
907 insn->save = true;
908 continue;
909
910 } else if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
911 insn->restore = true;
912 insn->hint = true;
913 continue;
914 }
915
916 insn->hint = true;
917
918 switch (hint->sp_reg) {
919 case ORC_REG_UNDEFINED:
920 cfa->base = CFI_UNDEFINED;
921 break;
922 case ORC_REG_SP:
923 cfa->base = CFI_SP;
924 break;
925 case ORC_REG_BP:
926 cfa->base = CFI_BP;
927 break;
928 case ORC_REG_SP_INDIRECT:
929 cfa->base = CFI_SP_INDIRECT;
930 break;
931 case ORC_REG_R10:
932 cfa->base = CFI_R10;
933 break;
934 case ORC_REG_R13:
935 cfa->base = CFI_R13;
936 break;
937 case ORC_REG_DI:
938 cfa->base = CFI_DI;
939 break;
940 case ORC_REG_DX:
941 cfa->base = CFI_DX;
942 break;
943 default:
944 WARN_FUNC("unsupported unwind_hint sp base reg %d",
945 insn->sec, insn->offset, hint->sp_reg);
946 return -1;
947 }
948
949 cfa->offset = hint->sp_offset;
950 insn->state.type = hint->type;
951 }
952
953 return 0;
954}
955
877static int decode_sections(struct objtool_file *file) 956static int decode_sections(struct objtool_file *file)
878{ 957{
879 int ret; 958 int ret;
@@ -904,6 +983,10 @@ static int decode_sections(struct objtool_file *file)
904 if (ret) 983 if (ret)
905 return ret; 984 return ret;
906 985
986 ret = read_unwind_hints(file);
987 if (ret)
988 return ret;
989
907 return 0; 990 return 0;
908} 991}
909 992
@@ -947,6 +1030,30 @@ static bool has_valid_stack_frame(struct insn_state *state)
947 return false; 1030 return false;
948} 1031}
949 1032
1033static int update_insn_state_regs(struct instruction *insn, struct insn_state *state)
1034{
1035 struct cfi_reg *cfa = &state->cfa;
1036 struct stack_op *op = &insn->stack_op;
1037
1038 if (cfa->base != CFI_SP)
1039 return 0;
1040
1041 /* push */
1042 if (op->dest.type == OP_DEST_PUSH)
1043 cfa->offset += 8;
1044
1045 /* pop */
1046 if (op->src.type == OP_SRC_POP)
1047 cfa->offset -= 8;
1048
1049 /* add immediate to sp */
1050 if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
1051 op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
1052 cfa->offset -= op->src.offset;
1053
1054 return 0;
1055}
1056
950static void save_reg(struct insn_state *state, unsigned char reg, int base, 1057static void save_reg(struct insn_state *state, unsigned char reg, int base,
951 int offset) 1058 int offset)
952{ 1059{
@@ -1032,6 +1139,9 @@ static int update_insn_state(struct instruction *insn, struct insn_state *state)
1032 return 0; 1139 return 0;
1033 } 1140 }
1034 1141
1142 if (state->type == ORC_TYPE_REGS || state->type == ORC_TYPE_REGS_IRET)
1143 return update_insn_state_regs(insn, state);
1144
1035 switch (op->dest.type) { 1145 switch (op->dest.type) {
1036 1146
1037 case OP_DEST_REG: 1147 case OP_DEST_REG:
@@ -1051,7 +1161,7 @@ static int update_insn_state(struct instruction *insn, struct insn_state *state)
1051 regs[CFI_BP].base = CFI_BP; 1161 regs[CFI_BP].base = CFI_BP;
1052 regs[CFI_BP].offset = -state->stack_size; 1162 regs[CFI_BP].offset = -state->stack_size;
1053 state->bp_scratch = false; 1163 state->bp_scratch = false;
1054 } else if (!nofp) { 1164 } else if (!no_fp) {
1055 1165
1056 WARN_FUNC("unknown stack-related register move", 1166 WARN_FUNC("unknown stack-related register move",
1057 insn->sec, insn->offset); 1167 insn->sec, insn->offset);
@@ -1222,7 +1332,7 @@ static int update_insn_state(struct instruction *insn, struct insn_state *state)
1222 } 1332 }
1223 1333
1224 /* detect when asm code uses rbp as a scratch register */ 1334 /* detect when asm code uses rbp as a scratch register */
1225 if (!nofp && insn->func && op->src.reg == CFI_BP && 1335 if (!no_fp && insn->func && op->src.reg == CFI_BP &&
1226 cfa->base != CFI_BP) 1336 cfa->base != CFI_BP)
1227 state->bp_scratch = true; 1337 state->bp_scratch = true;
1228 break; 1338 break;
@@ -1323,6 +1433,10 @@ static bool insn_state_match(struct instruction *insn, struct insn_state *state)
1323 break; 1433 break;
1324 } 1434 }
1325 1435
1436 } else if (state1->type != state2->type) {
1437 WARN_FUNC("stack state mismatch: type1=%d type2=%d",
1438 insn->sec, insn->offset, state1->type, state2->type);
1439
1326 } else if (state1->drap != state2->drap || 1440 } else if (state1->drap != state2->drap ||
1327 (state1->drap && state1->drap_reg != state2->drap_reg)) { 1441 (state1->drap && state1->drap_reg != state2->drap_reg)) {
1328 WARN_FUNC("stack state mismatch: drap1=%d(%d) drap2=%d(%d)", 1442 WARN_FUNC("stack state mismatch: drap1=%d(%d) drap2=%d(%d)",
@@ -1346,7 +1460,7 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
1346 struct insn_state state) 1460 struct insn_state state)
1347{ 1461{
1348 struct alternative *alt; 1462 struct alternative *alt;
1349 struct instruction *insn; 1463 struct instruction *insn, *next_insn;
1350 struct section *sec; 1464 struct section *sec;
1351 struct symbol *func = NULL; 1465 struct symbol *func = NULL;
1352 int ret; 1466 int ret;
@@ -1361,6 +1475,8 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
1361 } 1475 }
1362 1476
1363 while (1) { 1477 while (1) {
1478 next_insn = next_insn_same_sec(file, insn);
1479
1364 if (file->c_file && insn->func) { 1480 if (file->c_file && insn->func) {
1365 if (func && func != insn->func) { 1481 if (func && func != insn->func) {
1366 WARN("%s() falls through to next function %s()", 1482 WARN("%s() falls through to next function %s()",
@@ -1378,13 +1494,54 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
1378 } 1494 }
1379 1495
1380 if (insn->visited) { 1496 if (insn->visited) {
1381 if (!!insn_state_match(insn, &state)) 1497 if (!insn->hint && !insn_state_match(insn, &state))
1382 return 1; 1498 return 1;
1383 1499
1384 return 0; 1500 return 0;
1385 } 1501 }
1386 1502
1387 insn->state = state; 1503 if (insn->hint) {
1504 if (insn->restore) {
1505 struct instruction *save_insn, *i;
1506
1507 i = insn;
1508 save_insn = NULL;
1509 func_for_each_insn_continue_reverse(file, func, i) {
1510 if (i->save) {
1511 save_insn = i;
1512 break;
1513 }
1514 }
1515
1516 if (!save_insn) {
1517 WARN_FUNC("no corresponding CFI save for CFI restore",
1518 sec, insn->offset);
1519 return 1;
1520 }
1521
1522 if (!save_insn->visited) {
1523 /*
1524 * Oops, no state to copy yet.
1525 * Hopefully we can reach this
1526 * instruction from another branch
1527 * after the save insn has been
1528 * visited.
1529 */
1530 if (insn == first)
1531 return 0;
1532
1533 WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo",
1534 sec, insn->offset);
1535 return 1;
1536 }
1537
1538 insn->state = save_insn->state;
1539 }
1540
1541 state = insn->state;
1542
1543 } else
1544 insn->state = state;
1388 1545
1389 insn->visited = true; 1546 insn->visited = true;
1390 1547
@@ -1423,7 +1580,7 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
1423 1580
1424 /* fallthrough */ 1581 /* fallthrough */
1425 case INSN_CALL_DYNAMIC: 1582 case INSN_CALL_DYNAMIC:
1426 if (!nofp && func && !has_valid_stack_frame(&state)) { 1583 if (!no_fp && func && !has_valid_stack_frame(&state)) {
1427 WARN_FUNC("call without frame pointer save/setup", 1584 WARN_FUNC("call without frame pointer save/setup",
1428 sec, insn->offset); 1585 sec, insn->offset);
1429 return 1; 1586 return 1;
@@ -1461,6 +1618,14 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
1461 1618
1462 return 0; 1619 return 0;
1463 1620
1621 case INSN_CONTEXT_SWITCH:
1622 if (func && (!next_insn || !next_insn->hint)) {
1623 WARN_FUNC("unsupported instruction in callable function",
1624 sec, insn->offset);
1625 return 1;
1626 }
1627 return 0;
1628
1464 case INSN_STACK: 1629 case INSN_STACK:
1465 if (update_insn_state(insn, &state)) 1630 if (update_insn_state(insn, &state))
1466 return -1; 1631 return -1;
@@ -1474,7 +1639,7 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
1474 if (insn->dead_end) 1639 if (insn->dead_end)
1475 return 0; 1640 return 0;
1476 1641
1477 insn = next_insn_same_sec(file, insn); 1642 insn = next_insn;
1478 if (!insn) { 1643 if (!insn) {
1479 WARN("%s: unexpected end of section", sec->name); 1644 WARN("%s: unexpected end of section", sec->name);
1480 return 1; 1645 return 1;
@@ -1484,6 +1649,27 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
1484 return 0; 1649 return 0;
1485} 1650}
1486 1651
1652static int validate_unwind_hints(struct objtool_file *file)
1653{
1654 struct instruction *insn;
1655 int ret, warnings = 0;
1656 struct insn_state state;
1657
1658 if (!file->hints)
1659 return 0;
1660
1661 clear_insn_state(&state);
1662
1663 for_each_insn(file, insn) {
1664 if (insn->hint && !insn->visited) {
1665 ret = validate_branch(file, insn, state);
1666 warnings += ret;
1667 }
1668 }
1669
1670 return warnings;
1671}
1672
1487static bool is_kasan_insn(struct instruction *insn) 1673static bool is_kasan_insn(struct instruction *insn)
1488{ 1674{
1489 return (insn->type == INSN_CALL && 1675 return (insn->type == INSN_CALL &&
@@ -1580,15 +1766,6 @@ static int validate_reachable_instructions(struct objtool_file *file)
1580 if (insn->visited || ignore_unreachable_insn(insn)) 1766 if (insn->visited || ignore_unreachable_insn(insn))
1581 continue; 1767 continue;
1582 1768
1583 /*
1584 * gcov produces a lot of unreachable instructions. If we get
1585 * an unreachable warning and the file has gcov enabled, just
1586 * ignore it, and all other such warnings for the file. Do
1587 * this here because this is an expensive function.
1588 */
1589 if (gcov_enabled(file))
1590 return 0;
1591
1592 WARN_FUNC("unreachable instruction", insn->sec, insn->offset); 1769 WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
1593 return 1; 1770 return 1;
1594 } 1771 }
@@ -1613,15 +1790,15 @@ static void cleanup(struct objtool_file *file)
1613 elf_close(file->elf); 1790 elf_close(file->elf);
1614} 1791}
1615 1792
1616int check(const char *_objname, bool _nofp) 1793int check(const char *_objname, bool _no_fp, bool no_unreachable, bool orc)
1617{ 1794{
1618 struct objtool_file file; 1795 struct objtool_file file;
1619 int ret, warnings = 0; 1796 int ret, warnings = 0;
1620 1797
1621 objname = _objname; 1798 objname = _objname;
1622 nofp = _nofp; 1799 no_fp = _no_fp;
1623 1800
1624 file.elf = elf_open(objname); 1801 file.elf = elf_open(objname, orc ? O_RDWR : O_RDONLY);
1625 if (!file.elf) 1802 if (!file.elf)
1626 return 1; 1803 return 1;
1627 1804
@@ -1629,8 +1806,9 @@ int check(const char *_objname, bool _nofp)
1629 hash_init(file.insn_hash); 1806 hash_init(file.insn_hash);
1630 file.whitelist = find_section_by_name(file.elf, ".discard.func_stack_frame_non_standard"); 1807 file.whitelist = find_section_by_name(file.elf, ".discard.func_stack_frame_non_standard");
1631 file.rodata = find_section_by_name(file.elf, ".rodata"); 1808 file.rodata = find_section_by_name(file.elf, ".rodata");
1632 file.ignore_unreachables = false;
1633 file.c_file = find_section_by_name(file.elf, ".comment"); 1809 file.c_file = find_section_by_name(file.elf, ".comment");
1810 file.ignore_unreachables = no_unreachable;
1811 file.hints = false;
1634 1812
1635 arch_initial_func_cfi_state(&initial_func_cfi); 1813 arch_initial_func_cfi_state(&initial_func_cfi);
1636 1814
@@ -1647,6 +1825,11 @@ int check(const char *_objname, bool _nofp)
1647 goto out; 1825 goto out;
1648 warnings += ret; 1826 warnings += ret;
1649 1827
1828 ret = validate_unwind_hints(&file);
1829 if (ret < 0)
1830 goto out;
1831 warnings += ret;
1832
1650 if (!warnings) { 1833 if (!warnings) {
1651 ret = validate_reachable_instructions(&file); 1834 ret = validate_reachable_instructions(&file);
1652 if (ret < 0) 1835 if (ret < 0)
@@ -1654,6 +1837,20 @@ int check(const char *_objname, bool _nofp)
1654 warnings += ret; 1837 warnings += ret;
1655 } 1838 }
1656 1839
1840 if (orc) {
1841 ret = create_orc(&file);
1842 if (ret < 0)
1843 goto out;
1844
1845 ret = create_orc_sections(&file);
1846 if (ret < 0)
1847 goto out;
1848
1849 ret = elf_write(file.elf);
1850 if (ret < 0)
1851 goto out;
1852 }
1853
1657out: 1854out:
1658 cleanup(&file); 1855 cleanup(&file);
1659 1856
diff --git a/tools/objtool/check.h b/tools/objtool/check.h
index da85f5b00ec6..c9af11f0c8af 100644
--- a/tools/objtool/check.h
+++ b/tools/objtool/check.h
@@ -22,12 +22,14 @@
22#include "elf.h" 22#include "elf.h"
23#include "cfi.h" 23#include "cfi.h"
24#include "arch.h" 24#include "arch.h"
25#include "orc.h"
25#include <linux/hashtable.h> 26#include <linux/hashtable.h>
26 27
27struct insn_state { 28struct insn_state {
28 struct cfi_reg cfa; 29 struct cfi_reg cfa;
29 struct cfi_reg regs[CFI_NUM_REGS]; 30 struct cfi_reg regs[CFI_NUM_REGS];
30 int stack_size; 31 int stack_size;
32 unsigned char type;
31 bool bp_scratch; 33 bool bp_scratch;
32 bool drap; 34 bool drap;
33 int drap_reg; 35 int drap_reg;
@@ -41,13 +43,14 @@ struct instruction {
41 unsigned int len; 43 unsigned int len;
42 unsigned char type; 44 unsigned char type;
43 unsigned long immediate; 45 unsigned long immediate;
44 bool alt_group, visited, dead_end, ignore; 46 bool alt_group, visited, dead_end, ignore, hint, save, restore;
45 struct symbol *call_dest; 47 struct symbol *call_dest;
46 struct instruction *jump_dest; 48 struct instruction *jump_dest;
47 struct list_head alts; 49 struct list_head alts;
48 struct symbol *func; 50 struct symbol *func;
49 struct stack_op stack_op; 51 struct stack_op stack_op;
50 struct insn_state state; 52 struct insn_state state;
53 struct orc_entry orc;
51}; 54};
52 55
53struct objtool_file { 56struct objtool_file {
@@ -55,12 +58,22 @@ struct objtool_file {
55 struct list_head insn_list; 58 struct list_head insn_list;
56 DECLARE_HASHTABLE(insn_hash, 16); 59 DECLARE_HASHTABLE(insn_hash, 16);
57 struct section *rodata, *whitelist; 60 struct section *rodata, *whitelist;
58 bool ignore_unreachables, c_file; 61 bool ignore_unreachables, c_file, hints;
59}; 62};
60 63
61int check(const char *objname, bool nofp); 64int check(const char *objname, bool no_fp, bool no_unreachable, bool orc);
65
66struct instruction *find_insn(struct objtool_file *file,
67 struct section *sec, unsigned long offset);
62 68
63#define for_each_insn(file, insn) \ 69#define for_each_insn(file, insn) \
64 list_for_each_entry(insn, &file->insn_list, list) 70 list_for_each_entry(insn, &file->insn_list, list)
65 71
72#define sec_for_each_insn(file, sec, insn) \
73 for (insn = find_insn(file, sec, 0); \
74 insn && &insn->list != &file->insn_list && \
75 insn->sec == sec; \
76 insn = list_next_entry(insn, list))
77
78
66#endif /* _CHECK_H */ 79#endif /* _CHECK_H */
diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c
index 1a7e8aa2af58..6e9f980a7d26 100644
--- a/tools/objtool/elf.c
+++ b/tools/objtool/elf.c
@@ -30,16 +30,6 @@
30#include "elf.h" 30#include "elf.h"
31#include "warn.h" 31#include "warn.h"
32 32
33/*
34 * Fallback for systems without this "read, mmaping if possible" cmd.
35 */
36#ifndef ELF_C_READ_MMAP
37#define ELF_C_READ_MMAP ELF_C_READ
38#endif
39
40#define WARN_ELF(format, ...) \
41 WARN(format ": %s", ##__VA_ARGS__, elf_errmsg(-1))
42
43struct section *find_section_by_name(struct elf *elf, const char *name) 33struct section *find_section_by_name(struct elf *elf, const char *name)
44{ 34{
45 struct section *sec; 35 struct section *sec;
@@ -349,9 +339,10 @@ static int read_relas(struct elf *elf)
349 return 0; 339 return 0;
350} 340}
351 341
352struct elf *elf_open(const char *name) 342struct elf *elf_open(const char *name, int flags)
353{ 343{
354 struct elf *elf; 344 struct elf *elf;
345 Elf_Cmd cmd;
355 346
356 elf_version(EV_CURRENT); 347 elf_version(EV_CURRENT);
357 348
@@ -364,13 +355,20 @@ struct elf *elf_open(const char *name)
364 355
365 INIT_LIST_HEAD(&elf->sections); 356 INIT_LIST_HEAD(&elf->sections);
366 357
367 elf->fd = open(name, O_RDONLY); 358 elf->fd = open(name, flags);
368 if (elf->fd == -1) { 359 if (elf->fd == -1) {
369 perror("open"); 360 perror("open");
370 goto err; 361 goto err;
371 } 362 }
372 363
373 elf->elf = elf_begin(elf->fd, ELF_C_READ_MMAP, NULL); 364 if ((flags & O_ACCMODE) == O_RDONLY)
365 cmd = ELF_C_READ_MMAP;
366 else if ((flags & O_ACCMODE) == O_RDWR)
367 cmd = ELF_C_RDWR;
368 else /* O_WRONLY */
369 cmd = ELF_C_WRITE;
370
371 elf->elf = elf_begin(elf->fd, cmd, NULL);
374 if (!elf->elf) { 372 if (!elf->elf) {
375 WARN_ELF("elf_begin"); 373 WARN_ELF("elf_begin");
376 goto err; 374 goto err;
@@ -397,6 +395,194 @@ err:
397 return NULL; 395 return NULL;
398} 396}
399 397
398struct section *elf_create_section(struct elf *elf, const char *name,
399 size_t entsize, int nr)
400{
401 struct section *sec, *shstrtab;
402 size_t size = entsize * nr;
403 struct Elf_Scn *s;
404 Elf_Data *data;
405
406 sec = malloc(sizeof(*sec));
407 if (!sec) {
408 perror("malloc");
409 return NULL;
410 }
411 memset(sec, 0, sizeof(*sec));
412
413 INIT_LIST_HEAD(&sec->symbol_list);
414 INIT_LIST_HEAD(&sec->rela_list);
415 hash_init(sec->rela_hash);
416 hash_init(sec->symbol_hash);
417
418 list_add_tail(&sec->list, &elf->sections);
419
420 s = elf_newscn(elf->elf);
421 if (!s) {
422 WARN_ELF("elf_newscn");
423 return NULL;
424 }
425
426 sec->name = strdup(name);
427 if (!sec->name) {
428 perror("strdup");
429 return NULL;
430 }
431
432 sec->idx = elf_ndxscn(s);
433 sec->len = size;
434 sec->changed = true;
435
436 sec->data = elf_newdata(s);
437 if (!sec->data) {
438 WARN_ELF("elf_newdata");
439 return NULL;
440 }
441
442 sec->data->d_size = size;
443 sec->data->d_align = 1;
444
445 if (size) {
446 sec->data->d_buf = malloc(size);
447 if (!sec->data->d_buf) {
448 perror("malloc");
449 return NULL;
450 }
451 memset(sec->data->d_buf, 0, size);
452 }
453
454 if (!gelf_getshdr(s, &sec->sh)) {
455 WARN_ELF("gelf_getshdr");
456 return NULL;
457 }
458
459 sec->sh.sh_size = size;
460 sec->sh.sh_entsize = entsize;
461 sec->sh.sh_type = SHT_PROGBITS;
462 sec->sh.sh_addralign = 1;
463 sec->sh.sh_flags = SHF_ALLOC;
464
465
466 /* Add section name to .shstrtab */
467 shstrtab = find_section_by_name(elf, ".shstrtab");
468 if (!shstrtab) {
469 WARN("can't find .shstrtab section");
470 return NULL;
471 }
472
473 s = elf_getscn(elf->elf, shstrtab->idx);
474 if (!s) {
475 WARN_ELF("elf_getscn");
476 return NULL;
477 }
478
479 data = elf_newdata(s);
480 if (!data) {
481 WARN_ELF("elf_newdata");
482 return NULL;
483 }
484
485 data->d_buf = sec->name;
486 data->d_size = strlen(name) + 1;
487 data->d_align = 1;
488
489 sec->sh.sh_name = shstrtab->len;
490
491 shstrtab->len += strlen(name) + 1;
492 shstrtab->changed = true;
493
494 return sec;
495}
496
497struct section *elf_create_rela_section(struct elf *elf, struct section *base)
498{
499 char *relaname;
500 struct section *sec;
501
502 relaname = malloc(strlen(base->name) + strlen(".rela") + 1);
503 if (!relaname) {
504 perror("malloc");
505 return NULL;
506 }
507 strcpy(relaname, ".rela");
508 strcat(relaname, base->name);
509
510 sec = elf_create_section(elf, relaname, sizeof(GElf_Rela), 0);
511 if (!sec)
512 return NULL;
513
514 base->rela = sec;
515 sec->base = base;
516
517 sec->sh.sh_type = SHT_RELA;
518 sec->sh.sh_addralign = 8;
519 sec->sh.sh_link = find_section_by_name(elf, ".symtab")->idx;
520 sec->sh.sh_info = base->idx;
521 sec->sh.sh_flags = SHF_INFO_LINK;
522
523 return sec;
524}
525
526int elf_rebuild_rela_section(struct section *sec)
527{
528 struct rela *rela;
529 int nr, idx = 0, size;
530 GElf_Rela *relas;
531
532 nr = 0;
533 list_for_each_entry(rela, &sec->rela_list, list)
534 nr++;
535
536 size = nr * sizeof(*relas);
537 relas = malloc(size);
538 if (!relas) {
539 perror("malloc");
540 return -1;
541 }
542
543 sec->data->d_buf = relas;
544 sec->data->d_size = size;
545
546 sec->sh.sh_size = size;
547
548 idx = 0;
549 list_for_each_entry(rela, &sec->rela_list, list) {
550 relas[idx].r_offset = rela->offset;
551 relas[idx].r_addend = rela->addend;
552 relas[idx].r_info = GELF_R_INFO(rela->sym->idx, rela->type);
553 idx++;
554 }
555
556 return 0;
557}
558
559int elf_write(struct elf *elf)
560{
561 struct section *sec;
562 Elf_Scn *s;
563
564 list_for_each_entry(sec, &elf->sections, list) {
565 if (sec->changed) {
566 s = elf_getscn(elf->elf, sec->idx);
567 if (!s) {
568 WARN_ELF("elf_getscn");
569 return -1;
570 }
571 if (!gelf_update_shdr (s, &sec->sh)) {
572 WARN_ELF("gelf_update_shdr");
573 return -1;
574 }
575 }
576 }
577
578 if (elf_update(elf->elf, ELF_C_WRITE) < 0) {
579 WARN_ELF("elf_update");
580 return -1;
581 }
582
583 return 0;
584}
585
400void elf_close(struct elf *elf) 586void elf_close(struct elf *elf)
401{ 587{
402 struct section *sec, *tmpsec; 588 struct section *sec, *tmpsec;
diff --git a/tools/objtool/elf.h b/tools/objtool/elf.h
index 343968b778cb..d86e2ff14466 100644
--- a/tools/objtool/elf.h
+++ b/tools/objtool/elf.h
@@ -28,6 +28,13 @@
28# define elf_getshdrstrndx elf_getshstrndx 28# define elf_getshdrstrndx elf_getshstrndx
29#endif 29#endif
30 30
31/*
32 * Fallback for systems without this "read, mmaping if possible" cmd.
33 */
34#ifndef ELF_C_READ_MMAP
35#define ELF_C_READ_MMAP ELF_C_READ
36#endif
37
31struct section { 38struct section {
32 struct list_head list; 39 struct list_head list;
33 GElf_Shdr sh; 40 GElf_Shdr sh;
@@ -41,6 +48,7 @@ struct section {
41 char *name; 48 char *name;
42 int idx; 49 int idx;
43 unsigned int len; 50 unsigned int len;
51 bool changed, text;
44}; 52};
45 53
46struct symbol { 54struct symbol {
@@ -75,7 +83,7 @@ struct elf {
75}; 83};
76 84
77 85
78struct elf *elf_open(const char *name); 86struct elf *elf_open(const char *name, int flags);
79struct section *find_section_by_name(struct elf *elf, const char *name); 87struct section *find_section_by_name(struct elf *elf, const char *name);
80struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset); 88struct symbol *find_symbol_by_offset(struct section *sec, unsigned long offset);
81struct symbol *find_symbol_containing(struct section *sec, unsigned long offset); 89struct symbol *find_symbol_containing(struct section *sec, unsigned long offset);
@@ -83,6 +91,11 @@ struct rela *find_rela_by_dest(struct section *sec, unsigned long offset);
83struct rela *find_rela_by_dest_range(struct section *sec, unsigned long offset, 91struct rela *find_rela_by_dest_range(struct section *sec, unsigned long offset,
84 unsigned int len); 92 unsigned int len);
85struct symbol *find_containing_func(struct section *sec, unsigned long offset); 93struct symbol *find_containing_func(struct section *sec, unsigned long offset);
94struct section *elf_create_section(struct elf *elf, const char *name, size_t
95 entsize, int nr);
96struct section *elf_create_rela_section(struct elf *elf, struct section *base);
97int elf_rebuild_rela_section(struct section *sec);
98int elf_write(struct elf *elf);
86void elf_close(struct elf *elf); 99void elf_close(struct elf *elf);
87 100
88#define for_each_sec(file, sec) \ 101#define for_each_sec(file, sec) \
diff --git a/tools/objtool/objtool.c b/tools/objtool/objtool.c
index ecc5b1b5d15d..31e0f9143840 100644
--- a/tools/objtool/objtool.c
+++ b/tools/objtool/objtool.c
@@ -42,10 +42,11 @@ struct cmd_struct {
42}; 42};
43 43
44static const char objtool_usage_string[] = 44static const char objtool_usage_string[] =
45 "objtool [OPTIONS] COMMAND [ARGS]"; 45 "objtool COMMAND [ARGS]";
46 46
47static struct cmd_struct objtool_cmds[] = { 47static struct cmd_struct objtool_cmds[] = {
48 {"check", cmd_check, "Perform stack metadata validation on an object file" }, 48 {"check", cmd_check, "Perform stack metadata validation on an object file" },
49 {"orc", cmd_orc, "Generate in-place ORC unwind tables for an object file" },
49}; 50};
50 51
51bool help; 52bool help;
diff --git a/tools/objtool/orc.h b/tools/objtool/orc.h
new file mode 100644
index 000000000000..a4139e386ef3
--- /dev/null
+++ b/tools/objtool/orc.h
@@ -0,0 +1,30 @@
1/*
2 * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 2
7 * of the License, or (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, see <http://www.gnu.org/licenses/>.
16 */
17
18#ifndef _ORC_H
19#define _ORC_H
20
21#include "orc_types.h"
22
23struct objtool_file;
24
25int create_orc(struct objtool_file *file);
26int create_orc_sections(struct objtool_file *file);
27
28int orc_dump(const char *objname);
29
30#endif /* _ORC_H */
diff --git a/tools/objtool/orc_dump.c b/tools/objtool/orc_dump.c
new file mode 100644
index 000000000000..36c5bf6a2675
--- /dev/null
+++ b/tools/objtool/orc_dump.c
@@ -0,0 +1,212 @@
1/*
2 * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 2
7 * of the License, or (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include <unistd.h>
19#include "orc.h"
20#include "warn.h"
21
22static const char *reg_name(unsigned int reg)
23{
24 switch (reg) {
25 case ORC_REG_PREV_SP:
26 return "prevsp";
27 case ORC_REG_DX:
28 return "dx";
29 case ORC_REG_DI:
30 return "di";
31 case ORC_REG_BP:
32 return "bp";
33 case ORC_REG_SP:
34 return "sp";
35 case ORC_REG_R10:
36 return "r10";
37 case ORC_REG_R13:
38 return "r13";
39 case ORC_REG_BP_INDIRECT:
40 return "bp(ind)";
41 case ORC_REG_SP_INDIRECT:
42 return "sp(ind)";
43 default:
44 return "?";
45 }
46}
47
48static const char *orc_type_name(unsigned int type)
49{
50 switch (type) {
51 case ORC_TYPE_CALL:
52 return "call";
53 case ORC_TYPE_REGS:
54 return "regs";
55 case ORC_TYPE_REGS_IRET:
56 return "iret";
57 default:
58 return "?";
59 }
60}
61
62static void print_reg(unsigned int reg, int offset)
63{
64 if (reg == ORC_REG_BP_INDIRECT)
65 printf("(bp%+d)", offset);
66 else if (reg == ORC_REG_SP_INDIRECT)
67 printf("(sp%+d)", offset);
68 else if (reg == ORC_REG_UNDEFINED)
69 printf("(und)");
70 else
71 printf("%s%+d", reg_name(reg), offset);
72}
73
74int orc_dump(const char *_objname)
75{
76 int fd, nr_entries, i, *orc_ip = NULL, orc_size = 0;
77 struct orc_entry *orc = NULL;
78 char *name;
79 unsigned long nr_sections, orc_ip_addr = 0;
80 size_t shstrtab_idx;
81 Elf *elf;
82 Elf_Scn *scn;
83 GElf_Shdr sh;
84 GElf_Rela rela;
85 GElf_Sym sym;
86 Elf_Data *data, *symtab = NULL, *rela_orc_ip = NULL;
87
88
89 objname = _objname;
90
91 elf_version(EV_CURRENT);
92
93 fd = open(objname, O_RDONLY);
94 if (fd == -1) {
95 perror("open");
96 return -1;
97 }
98
99 elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
100 if (!elf) {
101 WARN_ELF("elf_begin");
102 return -1;
103 }
104
105 if (elf_getshdrnum(elf, &nr_sections)) {
106 WARN_ELF("elf_getshdrnum");
107 return -1;
108 }
109
110 if (elf_getshdrstrndx(elf, &shstrtab_idx)) {
111 WARN_ELF("elf_getshdrstrndx");
112 return -1;
113 }
114
115 for (i = 0; i < nr_sections; i++) {
116 scn = elf_getscn(elf, i);
117 if (!scn) {
118 WARN_ELF("elf_getscn");
119 return -1;
120 }
121
122 if (!gelf_getshdr(scn, &sh)) {
123 WARN_ELF("gelf_getshdr");
124 return -1;
125 }
126
127 name = elf_strptr(elf, shstrtab_idx, sh.sh_name);
128 if (!name) {
129 WARN_ELF("elf_strptr");
130 return -1;
131 }
132
133 data = elf_getdata(scn, NULL);
134 if (!data) {
135 WARN_ELF("elf_getdata");
136 return -1;
137 }
138
139 if (!strcmp(name, ".symtab")) {
140 symtab = data;
141 } else if (!strcmp(name, ".orc_unwind")) {
142 orc = data->d_buf;
143 orc_size = sh.sh_size;
144 } else if (!strcmp(name, ".orc_unwind_ip")) {
145 orc_ip = data->d_buf;
146 orc_ip_addr = sh.sh_addr;
147 } else if (!strcmp(name, ".rela.orc_unwind_ip")) {
148 rela_orc_ip = data;
149 }
150 }
151
152 if (!symtab || !orc || !orc_ip)
153 return 0;
154
155 if (orc_size % sizeof(*orc) != 0) {
156 WARN("bad .orc_unwind section size");
157 return -1;
158 }
159
160 nr_entries = orc_size / sizeof(*orc);
161 for (i = 0; i < nr_entries; i++) {
162 if (rela_orc_ip) {
163 if (!gelf_getrela(rela_orc_ip, i, &rela)) {
164 WARN_ELF("gelf_getrela");
165 return -1;
166 }
167
168 if (!gelf_getsym(symtab, GELF_R_SYM(rela.r_info), &sym)) {
169 WARN_ELF("gelf_getsym");
170 return -1;
171 }
172
173 scn = elf_getscn(elf, sym.st_shndx);
174 if (!scn) {
175 WARN_ELF("elf_getscn");
176 return -1;
177 }
178
179 if (!gelf_getshdr(scn, &sh)) {
180 WARN_ELF("gelf_getshdr");
181 return -1;
182 }
183
184 name = elf_strptr(elf, shstrtab_idx, sh.sh_name);
185 if (!name || !*name) {
186 WARN_ELF("elf_strptr");
187 return -1;
188 }
189
190 printf("%s+%lx:", name, rela.r_addend);
191
192 } else {
193 printf("%lx:", orc_ip_addr + (i * sizeof(int)) + orc_ip[i]);
194 }
195
196
197 printf(" sp:");
198
199 print_reg(orc[i].sp_reg, orc[i].sp_offset);
200
201 printf(" bp:");
202
203 print_reg(orc[i].bp_reg, orc[i].bp_offset);
204
205 printf(" type:%s\n", orc_type_name(orc[i].type));
206 }
207
208 elf_end(elf);
209 close(fd);
210
211 return 0;
212}
diff --git a/tools/objtool/orc_gen.c b/tools/objtool/orc_gen.c
new file mode 100644
index 000000000000..e5ca31429c9b
--- /dev/null
+++ b/tools/objtool/orc_gen.c
@@ -0,0 +1,214 @@
1/*
2 * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 2
7 * of the License, or (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, see <http://www.gnu.org/licenses/>.
16 */
17
18#include <stdlib.h>
19#include <string.h>
20
21#include "orc.h"
22#include "check.h"
23#include "warn.h"
24
25int create_orc(struct objtool_file *file)
26{
27 struct instruction *insn;
28
29 for_each_insn(file, insn) {
30 struct orc_entry *orc = &insn->orc;
31 struct cfi_reg *cfa = &insn->state.cfa;
32 struct cfi_reg *bp = &insn->state.regs[CFI_BP];
33
34 if (cfa->base == CFI_UNDEFINED) {
35 orc->sp_reg = ORC_REG_UNDEFINED;
36 continue;
37 }
38
39 switch (cfa->base) {
40 case CFI_SP:
41 orc->sp_reg = ORC_REG_SP;
42 break;
43 case CFI_SP_INDIRECT:
44 orc->sp_reg = ORC_REG_SP_INDIRECT;
45 break;
46 case CFI_BP:
47 orc->sp_reg = ORC_REG_BP;
48 break;
49 case CFI_BP_INDIRECT:
50 orc->sp_reg = ORC_REG_BP_INDIRECT;
51 break;
52 case CFI_R10:
53 orc->sp_reg = ORC_REG_R10;
54 break;
55 case CFI_R13:
56 orc->sp_reg = ORC_REG_R13;
57 break;
58 case CFI_DI:
59 orc->sp_reg = ORC_REG_DI;
60 break;
61 case CFI_DX:
62 orc->sp_reg = ORC_REG_DX;
63 break;
64 default:
65 WARN_FUNC("unknown CFA base reg %d",
66 insn->sec, insn->offset, cfa->base);
67 return -1;
68 }
69
70 switch(bp->base) {
71 case CFI_UNDEFINED:
72 orc->bp_reg = ORC_REG_UNDEFINED;
73 break;
74 case CFI_CFA:
75 orc->bp_reg = ORC_REG_PREV_SP;
76 break;
77 case CFI_BP:
78 orc->bp_reg = ORC_REG_BP;
79 break;
80 default:
81 WARN_FUNC("unknown BP base reg %d",
82 insn->sec, insn->offset, bp->base);
83 return -1;
84 }
85
86 orc->sp_offset = cfa->offset;
87 orc->bp_offset = bp->offset;
88 orc->type = insn->state.type;
89 }
90
91 return 0;
92}
93
94static int create_orc_entry(struct section *u_sec, struct section *ip_relasec,
95 unsigned int idx, struct section *insn_sec,
96 unsigned long insn_off, struct orc_entry *o)
97{
98 struct orc_entry *orc;
99 struct rela *rela;
100
101 /* populate ORC data */
102 orc = (struct orc_entry *)u_sec->data->d_buf + idx;
103 memcpy(orc, o, sizeof(*orc));
104
105 /* populate rela for ip */
106 rela = malloc(sizeof(*rela));
107 if (!rela) {
108 perror("malloc");
109 return -1;
110 }
111 memset(rela, 0, sizeof(*rela));
112
113 rela->sym = insn_sec->sym;
114 rela->addend = insn_off;
115 rela->type = R_X86_64_PC32;
116 rela->offset = idx * sizeof(int);
117
118 list_add_tail(&rela->list, &ip_relasec->rela_list);
119 hash_add(ip_relasec->rela_hash, &rela->hash, rela->offset);
120
121 return 0;
122}
123
124int create_orc_sections(struct objtool_file *file)
125{
126 struct instruction *insn, *prev_insn;
127 struct section *sec, *u_sec, *ip_relasec;
128 unsigned int idx;
129
130 struct orc_entry empty = {
131 .sp_reg = ORC_REG_UNDEFINED,
132 .bp_reg = ORC_REG_UNDEFINED,
133 .type = ORC_TYPE_CALL,
134 };
135
136 sec = find_section_by_name(file->elf, ".orc_unwind");
137 if (sec) {
138 WARN("file already has .orc_unwind section, skipping");
139 return -1;
140 }
141
142 /* count the number of needed orcs */
143 idx = 0;
144 for_each_sec(file, sec) {
145 if (!sec->text)
146 continue;
147
148 prev_insn = NULL;
149 sec_for_each_insn(file, sec, insn) {
150 if (!prev_insn ||
151 memcmp(&insn->orc, &prev_insn->orc,
152 sizeof(struct orc_entry))) {
153 idx++;
154 }
155 prev_insn = insn;
156 }
157
158 /* section terminator */
159 if (prev_insn)
160 idx++;
161 }
162 if (!idx)
163 return -1;
164
165
166 /* create .orc_unwind_ip and .rela.orc_unwind_ip sections */
167 sec = elf_create_section(file->elf, ".orc_unwind_ip", sizeof(int), idx);
168
169 ip_relasec = elf_create_rela_section(file->elf, sec);
170 if (!ip_relasec)
171 return -1;
172
173 /* create .orc_unwind section */
174 u_sec = elf_create_section(file->elf, ".orc_unwind",
175 sizeof(struct orc_entry), idx);
176
177 /* populate sections */
178 idx = 0;
179 for_each_sec(file, sec) {
180 if (!sec->text)
181 continue;
182
183 prev_insn = NULL;
184 sec_for_each_insn(file, sec, insn) {
185 if (!prev_insn || memcmp(&insn->orc, &prev_insn->orc,
186 sizeof(struct orc_entry))) {
187
188 if (create_orc_entry(u_sec, ip_relasec, idx,
189 insn->sec, insn->offset,
190 &insn->orc))
191 return -1;
192
193 idx++;
194 }
195 prev_insn = insn;
196 }
197
198 /* section terminator */
199 if (prev_insn) {
200 if (create_orc_entry(u_sec, ip_relasec, idx,
201 prev_insn->sec,
202 prev_insn->offset + prev_insn->len,
203 &empty))
204 return -1;
205
206 idx++;
207 }
208 }
209
210 if (elf_rebuild_rela_section(ip_relasec))
211 return -1;
212
213 return 0;
214}
diff --git a/tools/objtool/orc_types.h b/tools/objtool/orc_types.h
new file mode 100644
index 000000000000..9c9dc579bd7d
--- /dev/null
+++ b/tools/objtool/orc_types.h
@@ -0,0 +1,107 @@
1/*
2 * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com>
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 2
7 * of the License, or (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, see <http://www.gnu.org/licenses/>.
16 */
17
18#ifndef _ORC_TYPES_H
19#define _ORC_TYPES_H
20
21#include <linux/types.h>
22#include <linux/compiler.h>
23
24/*
25 * The ORC_REG_* registers are base registers which are used to find other
26 * registers on the stack.
27 *
28 * ORC_REG_PREV_SP, also known as DWARF Call Frame Address (CFA), is the
29 * address of the previous frame: the caller's SP before it called the current
30 * function.
31 *
32 * ORC_REG_UNDEFINED means the corresponding register's value didn't change in
33 * the current frame.
34 *
35 * The most commonly used base registers are SP and BP -- which the previous SP
36 * is usually based on -- and PREV_SP and UNDEFINED -- which the previous BP is
37 * usually based on.
38 *
39 * The rest of the base registers are needed for special cases like entry code
40 * and GCC realigned stacks.
41 */
42#define ORC_REG_UNDEFINED 0
43#define ORC_REG_PREV_SP 1
44#define ORC_REG_DX 2
45#define ORC_REG_DI 3
46#define ORC_REG_BP 4
47#define ORC_REG_SP 5
48#define ORC_REG_R10 6
49#define ORC_REG_R13 7
50#define ORC_REG_BP_INDIRECT 8
51#define ORC_REG_SP_INDIRECT 9
52#define ORC_REG_MAX 15
53
54/*
55 * ORC_TYPE_CALL: Indicates that sp_reg+sp_offset resolves to PREV_SP (the
56 * caller's SP right before it made the call). Used for all callable
57 * functions, i.e. all C code and all callable asm functions.
58 *
59 * ORC_TYPE_REGS: Used in entry code to indicate that sp_reg+sp_offset points
60 * to a fully populated pt_regs from a syscall, interrupt, or exception.
61 *
62 * ORC_TYPE_REGS_IRET: Used in entry code to indicate that sp_reg+sp_offset
63 * points to the iret return frame.
64 *
65 * The UNWIND_HINT macros are used only for the unwind_hint struct. They
66 * aren't used in struct orc_entry due to size and complexity constraints.
67 * Objtool converts them to real types when it converts the hints to orc
68 * entries.
69 */
70#define ORC_TYPE_CALL 0
71#define ORC_TYPE_REGS 1
72#define ORC_TYPE_REGS_IRET 2
73#define UNWIND_HINT_TYPE_SAVE 3
74#define UNWIND_HINT_TYPE_RESTORE 4
75
76#ifndef __ASSEMBLY__
77/*
78 * This struct is more or less a vastly simplified version of the DWARF Call
79 * Frame Information standard. It contains only the necessary parts of DWARF
80 * CFI, simplified for ease of access by the in-kernel unwinder. It tells the
81 * unwinder how to find the previous SP and BP (and sometimes entry regs) on
82 * the stack for a given code address. Each instance of the struct corresponds
83 * to one or more code locations.
84 */
85struct orc_entry {
86 s16 sp_offset;
87 s16 bp_offset;
88 unsigned sp_reg:4;
89 unsigned bp_reg:4;
90 unsigned type:2;
91} __packed;
92
93/*
94 * This struct is used by asm and inline asm code to manually annotate the
95 * location of registers on the stack for the ORC unwinder.
96 *
97 * Type can be either ORC_TYPE_* or UNWIND_HINT_TYPE_*.
98 */
99struct unwind_hint {
100 u32 ip;
101 s16 sp_offset;
102 u8 sp_reg;
103 u8 type;
104};
105#endif /* __ASSEMBLY__ */
106
107#endif /* _ORC_TYPES_H */
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-05 04:27:08 -0500