[PATCH] x86-64: Fix CFI information

Being the foundation for reliable stack unwinding, this fixes CFI unwind
annotations in many low-level x86_64 routines, plus a config option
(available to all architectures, and also present in the previously sent
patch adding such annotations to i386 code) to enable them separatly
rather than only along with adding full debug information.

Signed-off-by: Jan Beulich <jbeulich@novell.com>
Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

1 files changed, 150 insertions, 57 deletions

diff --git a/arch/x86_64/kernel/entry.S b/arch/x86_64/kernel/entry.S
index 873c39d8f818..7937971d1853 100644
--- a/arch/x86_64/kernel/entry.S
+++ b/arch/x86_64/kernel/entry.S
@@ -79,16 +79,19 @@
         xorl %eax, %eax
         pushq %rax /* ss */
         CFI_ADJUST_CFA_OFFSET   8
+        /*CFI_REL_OFFSET        ss,0*/
         pushq %rax /* rsp */
         CFI_ADJUST_CFA_OFFSET   8
-        CFI_OFFSET      rip,0
+        CFI_REL_OFFSET  rsp,0
         pushq $(1<<9) /* eflags - interrupts on */
         CFI_ADJUST_CFA_OFFSET   8
+        /*CFI_REL_OFFSET        rflags,0*/
         pushq $__KERNEL_CS /* cs */
         CFI_ADJUST_CFA_OFFSET   8
+        /*CFI_REL_OFFSET        cs,0*/
         pushq \child_rip /* rip */
         CFI_ADJUST_CFA_OFFSET   8
-        CFI_OFFSET      rip,0
+        CFI_REL_OFFSET  rip,0
         pushq   %rax /* orig rax */
         CFI_ADJUST_CFA_OFFSET   8
         .endm
@@ -98,32 +101,39 @@
         CFI_ADJUST_CFA_OFFSET   -(6*8)
         .endm
 
-        .macro  CFI_DEFAULT_STACK
-        CFI_ADJUST_CFA_OFFSET  (SS)
-        CFI_OFFSET      r15,R15-SS
-        CFI_OFFSET      r14,R14-SS
-        CFI_OFFSET      r13,R13-SS
-        CFI_OFFSET      r12,R12-SS
-        CFI_OFFSET      rbp,RBP-SS
-        CFI_OFFSET      rbx,RBX-SS
-        CFI_OFFSET      r11,R11-SS
-        CFI_OFFSET      r10,R10-SS
-        CFI_OFFSET      r9,R9-SS
-        CFI_OFFSET      r8,R8-SS
-        CFI_OFFSET      rax,RAX-SS
-        CFI_OFFSET      rcx,RCX-SS
-        CFI_OFFSET      rdx,RDX-SS
-        CFI_OFFSET      rsi,RSI-SS
-        CFI_OFFSET      rdi,RDI-SS
-        CFI_OFFSET      rsp,RSP-SS
-        CFI_OFFSET      rip,RIP-SS
+        .macro  CFI_DEFAULT_STACK start=1
+        .if \start
+        CFI_STARTPROC   simple
+        CFI_DEF_CFA     rsp,SS+8
+        .else
+        CFI_DEF_CFA_OFFSET SS+8
+        .endif
+        CFI_REL_OFFSET  r15,R15
+        CFI_REL_OFFSET  r14,R14
+        CFI_REL_OFFSET  r13,R13
+        CFI_REL_OFFSET  r12,R12
+        CFI_REL_OFFSET  rbp,RBP
+        CFI_REL_OFFSET  rbx,RBX
+        CFI_REL_OFFSET  r11,R11
+        CFI_REL_OFFSET  r10,R10
+        CFI_REL_OFFSET  r9,R9
+        CFI_REL_OFFSET  r8,R8
+        CFI_REL_OFFSET  rax,RAX
+        CFI_REL_OFFSET  rcx,RCX
+        CFI_REL_OFFSET  rdx,RDX
+        CFI_REL_OFFSET  rsi,RSI
+        CFI_REL_OFFSET  rdi,RDI
+        CFI_REL_OFFSET  rip,RIP
+        /*CFI_REL_OFFSET        cs,CS*/
+        /*CFI_REL_OFFSET        rflags,EFLAGS*/
+        CFI_REL_OFFSET  rsp,RSP
+        /*CFI_REL_OFFSET        ss,SS*/
         .endm
 /*
  * A newly forked process directly context switches into this.
  */     
 /* rdi: prev */ 
 ENTRY(ret_from_fork)
-        CFI_STARTPROC
         CFI_DEFAULT_STACK
         call schedule_tail
         GET_THREAD_INFO(%rcx)
@@ -172,16 +182,21 @@ rff_trace:
  */                                     
 
 ENTRY(system_call)
-        CFI_STARTPROC
+        CFI_STARTPROC   simple
+        CFI_DEF_CFA     rsp,0
+        CFI_REGISTER    rip,rcx
+        /*CFI_REGISTER  rflags,r11*/
         swapgs
         movq    %rsp,%gs:pda_oldrsp 
         movq    %gs:pda_kernelstack,%rsp
         sti                                     
         SAVE_ARGS 8,1
         movq  %rax,ORIG_RAX-ARGOFFSET(%rsp) 
-        movq  %rcx,RIP-ARGOFFSET(%rsp)  
+        movq  %rcx,RIP-ARGOFFSET(%rsp)
+        CFI_REL_OFFSET rip,RIP-ARGOFFSET
         GET_THREAD_INFO(%rcx)
         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
+        CFI_REMEMBER_STATE
         jnz tracesys
         cmpq $__NR_syscall_max,%rax
         ja badsys
@@ -201,9 +216,12 @@ sysret_check:
         cli
         movl threadinfo_flags(%rcx),%edx
         andl %edi,%edx
+        CFI_REMEMBER_STATE
         jnz  sysret_careful 
         movq RIP-ARGOFFSET(%rsp),%rcx
+        CFI_REGISTER    rip,rcx
         RESTORE_ARGS 0,-ARG_SKIP,1
+        /*CFI_REGISTER  rflags,r11*/
         movq    %gs:pda_oldrsp,%rsp
         swapgs
         sysretq
@@ -211,12 +229,15 @@ sysret_check:
         /* Handle reschedules */
         /* edx: work, edi: workmask */  
 sysret_careful:
+        CFI_RESTORE_STATE
         bt $TIF_NEED_RESCHED,%edx
         jnc sysret_signal
         sti
         pushq %rdi
+        CFI_ADJUST_CFA_OFFSET 8
         call schedule
         popq  %rdi
+        CFI_ADJUST_CFA_OFFSET -8
         jmp sysret_check
 
         /* Handle a signal */ 
@@ -234,8 +255,13 @@ sysret_signal:
 1:      movl $_TIF_NEED_RESCHED,%edi
         jmp sysret_check
         
+badsys:
+        movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
+        jmp ret_from_sys_call
+
         /* Do syscall tracing */
 tracesys:                        
+        CFI_RESTORE_STATE
         SAVE_REST
         movq $-ENOSYS,RAX(%rsp)
         FIXUP_TOP_OF_STACK %rdi
@@ -254,16 +280,29 @@ tracesys:
         RESTORE_TOP_OF_STACK %rbx
         RESTORE_REST
         jmp ret_from_sys_call
+        CFI_ENDPROC
                 
-badsys:
-        movq $-ENOSYS,RAX-ARGOFFSET(%rsp)       
-        jmp ret_from_sys_call
-
 /* 
  * Syscall return path ending with IRET.
  * Has correct top of stack, but partial stack frame.
  */     
-ENTRY(int_ret_from_sys_call)    
+ENTRY(int_ret_from_sys_call)
+        CFI_STARTPROC   simple
+        CFI_DEF_CFA     rsp,SS+8-ARGOFFSET
+        /*CFI_REL_OFFSET        ss,SS-ARGOFFSET*/
+        CFI_REL_OFFSET  rsp,RSP-ARGOFFSET
+        /*CFI_REL_OFFSET        rflags,EFLAGS-ARGOFFSET*/
+        /*CFI_REL_OFFSET        cs,CS-ARGOFFSET*/
+        CFI_REL_OFFSET  rip,RIP-ARGOFFSET
+        CFI_REL_OFFSET  rdx,RDX-ARGOFFSET
+        CFI_REL_OFFSET  rcx,RCX-ARGOFFSET
+        CFI_REL_OFFSET  rax,RAX-ARGOFFSET
+        CFI_REL_OFFSET  rdi,RDI-ARGOFFSET
+        CFI_REL_OFFSET  rsi,RSI-ARGOFFSET
+        CFI_REL_OFFSET  r8,R8-ARGOFFSET
+        CFI_REL_OFFSET  r9,R9-ARGOFFSET
+        CFI_REL_OFFSET  r10,R10-ARGOFFSET
+        CFI_REL_OFFSET  r11,R11-ARGOFFSET
         cli
         testl $3,CS-ARGOFFSET(%rsp)
         je retint_restore_args
@@ -284,8 +323,10 @@ int_careful:
         jnc  int_very_careful
         sti
         pushq %rdi
+        CFI_ADJUST_CFA_OFFSET 8
         call schedule
         popq %rdi
+        CFI_ADJUST_CFA_OFFSET -8
         cli
         jmp int_with_check
 
@@ -297,9 +338,11 @@ int_very_careful:
         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
         jz int_signal
         pushq %rdi
+        CFI_ADJUST_CFA_OFFSET 8
         leaq 8(%rsp),%rdi       # &ptregs -> arg1       
         call syscall_trace_leave
         popq %rdi
+        CFI_ADJUST_CFA_OFFSET -8
         andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
         cli
         jmp int_restore_rest
@@ -329,6 +372,8 @@ int_restore_rest:
         jmp     ptregscall_common
         .endm
 
+        CFI_STARTPROC
+
         PTREGSCALL stub_clone, sys_clone, %r8
         PTREGSCALL stub_fork, sys_fork, %rdi
         PTREGSCALL stub_vfork, sys_vfork, %rdi
@@ -337,40 +382,49 @@ int_restore_rest:
         PTREGSCALL stub_iopl, sys_iopl, %rsi
 
 ENTRY(ptregscall_common)
-        CFI_STARTPROC
         popq %r11
-        CFI_ADJUST_CFA_OFFSET   -8
+        CFI_ADJUST_CFA_OFFSET -8
+        CFI_REGISTER rip, r11
         SAVE_REST
         movq %r11, %r15
+        CFI_REGISTER rip, r15
         FIXUP_TOP_OF_STACK %r11
         call *%rax
         RESTORE_TOP_OF_STACK %r11
         movq %r15, %r11
+        CFI_REGISTER rip, r11
         RESTORE_REST
         pushq %r11
-        CFI_ADJUST_CFA_OFFSET   8
+        CFI_ADJUST_CFA_OFFSET 8
+        CFI_REL_OFFSET rip, 0
         ret
         CFI_ENDPROC
         
 ENTRY(stub_execve)
         CFI_STARTPROC
         popq %r11
-        CFI_ADJUST_CFA_OFFSET   -8
+        CFI_ADJUST_CFA_OFFSET -8
+        CFI_REGISTER rip, r11
         SAVE_REST
         movq %r11, %r15
+        CFI_REGISTER rip, r15
         FIXUP_TOP_OF_STACK %r11
         call sys_execve
         GET_THREAD_INFO(%rcx)
         bt $TIF_IA32,threadinfo_flags(%rcx)
+        CFI_REMEMBER_STATE
         jc exec_32bit
         RESTORE_TOP_OF_STACK %r11
         movq %r15, %r11
+        CFI_REGISTER rip, r11
         RESTORE_REST
-        push %r11
+        pushq %r11
+        CFI_ADJUST_CFA_OFFSET 8
+        CFI_REL_OFFSET rip, 0
         ret
 
 exec_32bit:
-        CFI_ADJUST_CFA_OFFSET   REST_SKIP
+        CFI_RESTORE_STATE
         movq %rax,RAX(%rsp)
         RESTORE_REST
         jmp int_ret_from_sys_call
@@ -382,7 +436,8 @@ exec_32bit:
  */                
 ENTRY(stub_rt_sigreturn)
         CFI_STARTPROC
-        addq $8, %rsp           
+        addq $8, %rsp
+        CFI_ADJUST_CFA_OFFSET   -8
         SAVE_REST
         movq %rsp,%rdi
         FIXUP_TOP_OF_STACK %r11
@@ -392,6 +447,25 @@ ENTRY(stub_rt_sigreturn)
         jmp int_ret_from_sys_call
         CFI_ENDPROC
 
+/*
+ * initial frame state for interrupts and exceptions
+ */
+        .macro _frame ref
+        CFI_STARTPROC simple
+        CFI_DEF_CFA rsp,SS+8-\ref
+        /*CFI_REL_OFFSET ss,SS-\ref*/
+        CFI_REL_OFFSET rsp,RSP-\ref
+        /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
+        /*CFI_REL_OFFSET cs,CS-\ref*/
+        CFI_REL_OFFSET rip,RIP-\ref
+        .endm
+
+/* initial frame state for interrupts (and exceptions without error code) */
+#define INTR_FRAME _frame RIP
+/* initial frame state for exceptions with error code (and interrupts with
+   vector already pushed) */
+#define XCPT_FRAME _frame ORIG_RAX
+
 /* 
  * Interrupt entry/exit.
  *
@@ -402,10 +476,6 @@ ENTRY(stub_rt_sigreturn)
 
 /* 0(%rsp): interrupt number */ 
         .macro interrupt func
-        CFI_STARTPROC   simple
-        CFI_DEF_CFA     rsp,(SS-RDI)
-        CFI_REL_OFFSET  rsp,(RSP-ORIG_RAX)
-        CFI_REL_OFFSET  rip,(RIP-ORIG_RAX)
         cld
 #ifdef CONFIG_DEBUG_INFO
         SAVE_ALL        
@@ -425,23 +495,27 @@ ENTRY(stub_rt_sigreturn)
         swapgs  
 1:      incl    %gs:pda_irqcount        # RED-PEN should check preempt count
         movq %gs:pda_irqstackptr,%rax
-        cmoveq %rax,%rsp                                                        
+        cmoveq %rax,%rsp /*todo This needs CFI annotation! */
         pushq %rdi                      # save old stack        
+        CFI_ADJUST_CFA_OFFSET   8
         call \func
         .endm
 
 ENTRY(common_interrupt)
+        XCPT_FRAME
         interrupt do_IRQ
         /* 0(%rsp): oldrsp-ARGOFFSET */
-ret_from_intr:          
+ret_from_intr:
         popq  %rdi
+        CFI_ADJUST_CFA_OFFSET   -8
         cli     
         decl %gs:pda_irqcount
 #ifdef CONFIG_DEBUG_INFO
         movq RBP(%rdi),%rbp
+        CFI_DEF_CFA_REGISTER    rsp
 #endif
-        leaq ARGOFFSET(%rdi),%rsp
-exit_intr:              
+        leaq ARGOFFSET(%rdi),%rsp /*todo This needs CFI annotation! */
+exit_intr:
         GET_THREAD_INFO(%rcx)
         testl $3,CS-ARGOFFSET(%rsp)
         je retint_kernel
@@ -453,9 +527,10 @@ exit_intr:
          */             
 retint_with_reschedule:
         movl $_TIF_WORK_MASK,%edi
-retint_check:                   
+retint_check:
         movl threadinfo_flags(%rcx),%edx
         andl %edi,%edx
+        CFI_REMEMBER_STATE
         jnz  retint_careful
 retint_swapgs:          
         swapgs 
@@ -476,14 +551,17 @@ bad_iret:
         jmp do_exit                     
         .previous       
         
-        /* edi: workmask, edx: work */  
+        /* edi: workmask, edx: work */
 retint_careful:
+        CFI_RESTORE_STATE
         bt    $TIF_NEED_RESCHED,%edx
         jnc   retint_signal
         sti
         pushq %rdi
+        CFI_ADJUST_CFA_OFFSET   8
         call  schedule
         popq %rdi               
+        CFI_ADJUST_CFA_OFFSET   -8
         GET_THREAD_INFO(%rcx)
         cli
         jmp retint_check
@@ -523,7 +601,9 @@ retint_kernel:
  * APIC interrupts.
  */             
         .macro apicinterrupt num,func
+        INTR_FRAME
         pushq $\num-256
+        CFI_ADJUST_CFA_OFFSET 8
         interrupt \func
         jmp ret_from_intr
         CFI_ENDPROC
@@ -569,16 +649,23 @@ ENTRY(spurious_interrupt)
  * Exception entry points.
  */             
         .macro zeroentry sym
+        INTR_FRAME
         pushq $0        /* push error code/oldrax */ 
+        CFI_ADJUST_CFA_OFFSET 8
         pushq %rax      /* push real oldrax to the rdi slot */ 
+        CFI_ADJUST_CFA_OFFSET 8
         leaq  \sym(%rip),%rax
         jmp error_entry
+        CFI_ENDPROC
         .endm   
 
         .macro errorentry sym
+        XCPT_FRAME
         pushq %rax
+        CFI_ADJUST_CFA_OFFSET 8
         leaq  \sym(%rip),%rax
         jmp error_entry
+        CFI_ENDPROC
         .endm
 
         /* error code is on the stack already */
@@ -605,10 +692,7 @@ ENTRY(spurious_interrupt)
  * and the exception handler in %rax.   
  */                                             
 ENTRY(error_entry)
-        CFI_STARTPROC   simple
-        CFI_DEF_CFA     rsp,(SS-RDI)
-        CFI_REL_OFFSET  rsp,(RSP-RDI)
-        CFI_REL_OFFSET  rip,(RIP-RDI)
+        _frame RDI
         /* rdi slot contains rax, oldrax contains error code */
         cld     
         subq  $14*8,%rsp
@@ -690,7 +774,9 @@ error_kernelspace:
        /* Reload gs selector with exception handling */
        /* edi:  new selector */ 
 ENTRY(load_gs_index)
+        CFI_STARTPROC
         pushf
+        CFI_ADJUST_CFA_OFFSET 8
         cli
         swapgs
 gs_change:     
@@ -698,7 +784,9 @@ gs_change:
 2:      mfence          /* workaround */
         swapgs
         popf
+        CFI_ADJUST_CFA_OFFSET -8
         ret
+        CFI_ENDPROC
        
         .section __ex_table,"a"
         .align 8
@@ -810,7 +898,7 @@ ENTRY(device_not_available)
 
         /* runs on exception stack */
 KPROBE_ENTRY(debug)
-        CFI_STARTPROC
+        INTR_FRAME
         pushq $0
         CFI_ADJUST_CFA_OFFSET 8         
         paranoidentry do_debug
@@ -820,9 +908,9 @@ KPROBE_ENTRY(debug)
 
         /* runs on exception stack */   
 ENTRY(nmi)
-        CFI_STARTPROC
+        INTR_FRAME
         pushq $-1
-        CFI_ADJUST_CFA_OFFSET 8         
+        CFI_ADJUST_CFA_OFFSET 8
         paranoidentry do_nmi
         /*
          * "Paranoid" exit path from exception stack.
@@ -888,7 +976,7 @@ ENTRY(reserved)
 
         /* runs on exception stack */
 ENTRY(double_fault)
-        CFI_STARTPROC
+        XCPT_FRAME
         paranoidentry do_double_fault
         jmp paranoid_exit
         CFI_ENDPROC
@@ -901,7 +989,7 @@ ENTRY(segment_not_present)
 
         /* runs on exception stack */
 ENTRY(stack_segment)
-        CFI_STARTPROC
+        XCPT_FRAME
         paranoidentry do_stack_segment
         jmp paranoid_exit
         CFI_ENDPROC
@@ -922,7 +1010,7 @@ ENTRY(spurious_interrupt_bug)
 #ifdef CONFIG_X86_MCE
         /* runs on exception stack */
 ENTRY(machine_check)
-        CFI_STARTPROC
+        INTR_FRAME
         pushq $0
         CFI_ADJUST_CFA_OFFSET 8 
         paranoidentry do_machine_check
@@ -934,14 +1022,19 @@ ENTRY(call_debug)
        zeroentry do_call_debug
 
 ENTRY(call_softirq)
+        CFI_STARTPROC
         movq %gs:pda_irqstackptr,%rax
         pushq %r15
+        CFI_ADJUST_CFA_OFFSET 8
         movq %rsp,%r15
+        CFI_DEF_CFA_REGISTER    r15
         incl %gs:pda_irqcount
         cmove %rax,%rsp
         call __do_softirq
         movq %r15,%rsp
+        CFI_DEF_CFA_REGISTER    rsp
         decl %gs:pda_irqcount
         popq %r15
+        CFI_ADJUST_CFA_OFFSET -8
         ret
-
+        CFI_ENDPROC