1 files changed, 150 insertions, 57 deletions

diff --git a/arch/x86_64/kernel/entry.S b/arch/x86_64/kernel/entry.S
index 873c39d8f818..7937971d1853 100644
--- a/arch/x86_64/kernel/entry.S
+++ b/arch/x86_64/kernel/entry.S
@@ -79,16 +79,19 @@
         xorl %eax, %eax
         pushq %rax /* ss */
         CFI_ADJUST_CFA_OFFSET   8
+        /*CFI_REL_OFFSET        ss,0*/
         pushq %rax /* rsp */
         CFI_ADJUST_CFA_OFFSET   8
-        CFI_OFFSET      rip,0
+        CFI_REL_OFFSET  rsp,0
         pushq $(1<<9) /* eflags - interrupts on */
         CFI_ADJUST_CFA_OFFSET   8
+        /*CFI_REL_OFFSET        rflags,0*/
         pushq $__KERNEL_CS /* cs */
         CFI_ADJUST_CFA_OFFSET   8
+        /*CFI_REL_OFFSET        cs,0*/
         pushq \child_rip /* rip */
         CFI_ADJUST_CFA_OFFSET   8
-        CFI_OFFSET      rip,0
+        CFI_REL_OFFSET  rip,0
         pushq   %rax /* orig rax */
         CFI_ADJUST_CFA_OFFSET   8
         .endm
@@ -98,32 +101,39 @@
         CFI_ADJUST_CFA_OFFSET   -(6*8)
         .endm
 
-        .macro  CFI_DEFAULT_STACK
-        CFI_ADJUST_CFA_OFFSET  (SS)
-        CFI_OFFSET      r15,R15-SS
-        CFI_OFFSET      r14,R14-SS
-        CFI_OFFSET      r13,R13-SS
-        CFI_OFFSET      r12,R12-SS
-        CFI_OFFSET      rbp,RBP-SS
-        CFI_OFFSET      rbx,RBX-SS
-        CFI_OFFSET      r11,R11-SS
-        CFI_OFFSET      r10,R10-SS
-        CFI_OFFSET      r9,R9-SS
-        CFI_OFFSET      r8,R8-SS
-        CFI_OFFSET      rax,RAX-SS
-        CFI_OFFSET      rcx,RCX-SS
-        CFI_OFFSET      rdx,RDX-SS
-        CFI_OFFSET      rsi,RSI-SS
-        CFI_OFFSET      rdi,RDI-SS
-        CFI_OFFSET      rsp,RSP-SS
-        CFI_OFFSET      rip,RIP-SS
+        .macro  CFI_DEFAULT_STACK start=1
+        .if \start
+        CFI_STARTPROC   simple
+        CFI_DEF_CFA     rsp,SS+8
+        .else
+        CFI_DEF_CFA_OFFSET SS+8
+        .endif
+        CFI_REL_OFFSET  r15,R15
+        CFI_REL_OFFSET  r14,R14
+        CFI_REL_OFFSET  r13,R13
+        CFI_REL_OFFSET  r12,R12
+        CFI_REL_OFFSET  rbp,RBP
+        CFI_REL_OFFSET  rbx,RBX
+        CFI_REL_OFFSET  r11,R11
+        CFI_REL_OFFSET  r10,R10
+        CFI_REL_OFFSET  r9,R9
+        CFI_REL_OFFSET  r8,R8
+        CFI_REL_OFFSET  rax,RAX
+        CFI_REL_OFFSET  rcx,RCX
+        CFI_REL_OFFSET  rdx,RDX
+        CFI_REL_OFFSET  rsi,RSI
+        CFI_REL_OFFSET  rdi,RDI
+        CFI_REL_OFFSET  rip,RIP
+        /*CFI_REL_OFFSET        cs,CS*/
+        /*CFI_REL_OFFSET        rflags,EFLAGS*/
+        CFI_REL_OFFSET  rsp,RSP
+        /*CFI_REL_OFFSET        ss,SS*/
         .endm
 /*
  * A newly forked process directly context switches into this.
  */     
 /* rdi: prev */ 
 ENTRY(ret_from_fork)
-        CFI_STARTPROC
         CFI_DEFAULT_STACK
         call schedule_tail
         GET_THREAD_INFO(%rcx)
@@ -172,16 +182,21 @@ rff_trace:
  */                                     
 
 ENTRY(system_call)
-        CFI_STARTPROC
+        CFI_STARTPROC   simple
+        CFI_DEF_CFA     rsp,0
+        CFI_REGISTER    rip,rcx
+        /*CFI_REGISTER  rflags,r11*/
         swapgs
         movq    %rsp,%gs:pda_oldrsp 
         movq    %gs:pda_kernelstack,%rsp
         sti                                     
         SAVE_ARGS 8,1
         movq  %rax,ORIG_RAX-ARGOFFSET(%rsp) 
-        movq  %rcx,RIP-ARGOFFSET(%rsp)  
+        movq  %rcx,RIP-ARGOFFSET(%rsp)
+        CFI_REL_OFFSET rip,RIP-ARGOFFSET
         GET_THREAD_INFO(%rcx)
         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
+        CFI_REMEMBER_STATE
         jnz tracesys
         cmpq $__NR_syscall_max,%rax
         ja badsys
@@ -201,9 +216,12 @@ sysret_check:
         cli
         movl threadinfo_flags(%rcx),%edx
         andl %edi,%edx
+        CFI_REMEMBER_STATE
         jnz  sysret_careful 
         movq RIP-ARGOFFSET(%rsp),%rcx
+        CFI_REGISTER    rip,rcx
         RESTORE_ARGS 0,-ARG_SKIP,1
+        /*CFI_REGISTER  rflags,r11*/
         movq    %gs:pda_oldrsp,%rsp
         swapgs
         sysretq
@@ -211,12 +229,15 @@ sysret_check:
         /* Handle reschedules */
         /* edx: work, edi: workmask */  
 sysret_careful:
+        CFI_RESTORE_STATE
         bt $TIF_NEED_RESCHED,%edx
         jnc sysret_signal
         sti
         pushq %rdi
+        CFI_ADJUST_CFA_OFFSET 8
         call schedule
         popq  %rdi
+        CFI_ADJUST_CFA_OFFSET -8
         jmp sysret_check
 
         /* Handle a signal */ 
@@ -234,8 +255,13 @@ sysret_signal:
 1:      movl $_TIF_NEED_RESCHED,%edi
         jmp sysret_check
         
+badsys:
+        movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
+        jmp ret_from_sys_call
+
         /* Do syscall tracing */
 tracesys:                        
+        CFI_RESTORE_STATE
         SAVE_REST
         movq $-ENOSYS,RAX(%rsp)
         FIXUP_TOP_OF_STACK %rdi
@@ -254,16 +280,29 @@ tracesys:
         RESTORE_TOP_OF_STACK %rbx
         RESTORE_REST
         jmp ret_from_sys_call
+        CFI_ENDPROC
                 
-badsys:
-        movq $-ENOSYS,RAX-ARGOFFSET(%rsp)       
-        jmp ret_from_sys_call
-
 /* 
  * Syscall return path ending with IRET.
  * Has correct top of stack, but partial stack frame.
  */     
-ENTRY(int_ret_from_sys_call)    
+ENTRY(int_ret_from_sys_call)
+        CFI_STARTPROC   simple
+        CFI_DEF_CFA     rsp,SS+8-ARGOFFSET
+        /*CFI_REL_OFFSET        ss,SS-ARGOFFSET*/
+        CFI_REL_OFFSET  rsp,RSP-ARGOFFSET
+        /*CFI_REL_OFFSET        rflags,EFLAGS-ARGOFFSET*/
+        /*CFI_REL_OFFSET        cs,CS-ARGOFFSET*/
+        CFI_REL_OFFSET  rip,RIP-ARGOFFSET
+        CFI_REL_OFFSET  rdx,RDX-ARGOFFSET
+        CFI_REL_OFFSET  rcx,RCX-ARGOFFSET
+        CFI_REL_OFFSET  rax,RAX-ARGOFFSET
+        CFI_REL_OFFSET  rdi,RDI-ARGOFFSET
+        CFI_REL_OFFSET  rsi,RSI-ARGOFFSET
+        CFI_REL_OFFSET  r8,R8-ARGOFFSET
+        CFI_REL_OFFSET  r9,R9-ARGOFFSET
+        CFI_REL_OFFSET  r10,R10-ARGOFFSET
+        CFI_REL_OFFSET  r11,R11-ARGOFFSET
         cli
         testl $3,CS-ARGOFFSET(%rsp)
         je retint_restore_args
@@ -284,8 +323,10 @@ int_careful:
         jnc  int_very_careful
         sti
         pushq %rdi
+        CFI_ADJUST_CFA_OFFSET 8
         call schedule
         popq %rdi
+        CFI_ADJUST_CFA_OFFSET -8
         cli
         jmp int_with_check
 
@@ -297,9 +338,11 @@ int_very_careful:
         testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
         jz int_signal
         pushq %rdi
+        CFI_ADJUST_CFA_OFFSET 8
         leaq 8(%rsp),%rdi       # &ptregs -> arg1       
         call syscall_trace_leave
         popq %rdi
+        CFI_ADJUST_CFA_OFFSET -8
         andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
         cli
         jmp int_restore_rest
@@ -329,6 +372,8 @@ int_restore_rest:
         jmp     ptregscall_common
         .endm
 
+        CFI_STARTPROC
+
         PTREGSCALL stub_clone, sys_clone, %r8
         PTREGSCALL stub_fork, sys_fork, %rdi
         PTREGSCALL stub_vfork, sys_vfork, %rdi
@@ -337,40 +382,49 @@ int_restore_rest:
         PTREGSCALL stub_iopl, sys_iopl, %rsi
 
 ENTRY(ptregscall_common)
-        CFI_STARTPROC
         popq %r11
-        CFI_ADJUST_CFA_OFFSET   -8
+        CFI_ADJUST_CFA_OFFSET -8
+        CFI_REGISTER rip, r11
         SAVE_REST
         movq %r11, %r15
+        CFI_REGISTER rip, r15
         FIXUP_TOP_OF_STACK %r11
         call *%rax
         RESTORE_TOP_OF_STACK %r11
         movq %r15, %r11
+        CFI_REGISTER rip, r11
         RESTORE_REST
         pushq %r11
-        CFI_ADJUST_CFA_OFFSET   8
+        CFI_ADJUST_CFA_OFFSET 8
+        CFI_REL_OFFSET rip, 0
         ret
         CFI_ENDPROC
         
 ENTRY(stub_execve)
         CFI_STARTPROC
         popq %r11
-        CFI_ADJUST_CFA_OFFSET   -8
+        CFI_ADJUST_CFA_OFFSET -8
+        CFI_REGISTER rip, r11
         SAVE_REST
         movq %r11, %r15
+        CFI_REGISTER rip, r15
         FIXUP_TOP_OF_STACK %r11
         call sys_execve
         GET_THREAD_INFO(%rcx)
         bt $TIF_IA32,threadinfo_flags(%rcx)
+        CFI_REMEMBER_STATE
         jc exec_32bit
         RESTORE_TOP_OF_STACK %r11
         movq %r15, %r11
+        CFI_REGISTER rip, r11
         RESTORE_REST
-        push %r11
+        pushq %r11
+        CFI_ADJUST_CFA_OFFSET 8
+        CFI_REL_OFFSET rip, 0
         ret
 
 exec_32bit:
-        CFI_ADJUST_CFA_OFFSET   REST_SKIP
+        CFI_RESTORE_STATE
         movq %rax,RAX(%rsp)
         RESTORE_REST
         jmp int_ret_from_sys_call
@@ -382,7 +436,8 @@ exec_32bit:
  */                
 ENTRY(stub_rt_sigreturn)
         CFI_STARTPROC
-        addq $8, %rsp           
+        addq $8, %rsp
+        CFI_ADJUST_CFA_OFFSET   -8
         SAVE_REST
         movq %rsp,%rdi
         FIXUP_TOP_OF_STACK %r11
@@ -392,6 +447,25 @@ ENTRY(stub_rt_sigreturn)
         jmp int_ret_from_sys_call
         CFI_ENDPROC
 
+/*
+ * initial frame state for interrupts and exceptions
+ */
+        .macro _frame ref
+        CFI_STARTPROC simple
+        CFI_DEF_CFA rsp,SS+8-\ref
+        /*CFI_REL_OFFSET ss,SS-\ref*/
+        CFI_REL_OFFSET rsp,RSP-\ref
+        /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
+        /*CFI_REL_OFFSET cs,CS-\ref*/
+        CFI_REL_OFFSET rip,RIP-\ref
+        .endm
+
+/* initial frame state for interrupts (and exceptions without error code) */
+#define INTR_FRAME _frame RIP
+/* initial frame state for exceptions with error code (and interrupts with
+   vector already pushed) */
+#define XCPT_FRAME _frame ORIG_RAX
+
 /* 
  * Interrupt entry/exit.
  *
@@ -402,10 +476,6 @@ ENTRY(stub_rt_sigreturn)
 
 /* 0(%rsp): interrupt number */ 
         .macro interrupt func
-        CFI_STARTPROC   simple
-        CFI_DEF_CFA     rsp,(SS-RDI)
-        CFI_REL_OFFSET  rsp,(RSP-ORIG_RAX)
-        CFI_REL_OFFSET  rip,(RIP-ORIG_RAX)
         cld
 #ifdef CONFIG_DEBUG_INFO
         SAVE_ALL        
@@ -425,23 +495,27 @@ ENTRY(stub_rt_sigreturn)
         swapgs  
 1:      incl    %gs:pda_irqcount        # RED-PEN should check preempt count
         movq %gs:pda_irqstackptr,%rax
-        cmoveq %rax,%rsp                                                        
+        cmoveq %rax,%rsp /*todo This needs CFI annotation! */
         pushq %rdi                      # save old stack        
+        CFI_ADJUST_CFA_OFFSET   8
         call \func
         .endm
 
 ENTRY(common_interrupt)
+        XCPT_FRAME
         interrupt do_IRQ
         /* 0(%rsp): oldrsp-ARGOFFSET */
-ret_from_intr:          
+ret_from_intr:
         popq  %rdi
+        CFI_ADJUST_CFA_OFFSET   -8
         cli     
         decl %gs:pda_irqcount
 #ifdef CONFIG_DEBUG_INFO
         movq RBP(%rdi),%rbp
+        CFI_DEF_CFA_REGISTER    rsp
 #endif
-        leaq ARGOFFSET(%rdi),%rsp
-exit_intr:              
+        leaq ARGOFFSET(%rdi),%rsp /*todo This needs CFI annotation! */
+exit_intr:
         GET_THREAD_INFO(%rcx)
         testl $3,CS-ARGOFFSET(%rsp)
         je retint_kernel
@@ -453,9 +527,10 @@ exit_intr:
          */             
 retint_with_reschedule:
         movl $_TIF_WORK_MASK,%edi
-retint_check:                   
+retint_check:
         movl threadinfo_flags(%rcx),%edx
         andl %edi,%edx
+        CFI_REMEMBER_STATE
         jnz  retint_careful
 retint_swapgs:          
         swapgs 
@@ -476,14 +551,17 @@ bad_iret:
         jmp do_exit                     
         .previous       
         
-        /* edi: workmask, edx: work */  
+        /* edi: workmask, edx: work */
 retint_careful:
+        CFI_RESTORE_STATE
         bt    $TIF_NEED_RESCHED,%edx
         jnc   retint_signal
         sti
         pushq %rdi
+        CFI_ADJUST_CFA_OFFSET   8
         call  schedule
         popq %rdi               
+        CFI_ADJUST_CFA_OFFSET   -8
         GET_THREAD_INFO(%rcx)
         cli
         jmp retint_check
@@ -523,7 +601,9 @@ retint_kernel:
  * APIC interrupts.
  */             
         .macro apicinterrupt num,func
+        INTR_FRAME
         pushq $\num-256
+        CFI_ADJUST_CFA_OFFSET 8
         interrupt \func
         jmp ret_from_intr
         CFI_ENDPROC
@@ -569,16 +649,23 @@ ENTRY(spurious_interrupt)
  * Exception entry points.
  */             
         .macro zeroentry sym
+        INTR_FRAME
         pushq $0        /* push error code/oldrax */ 
+        CFI_ADJUST_CFA_OFFSET 8
         pushq %rax      /* push real oldrax to the rdi slot */ 
+        CFI_ADJUST_CFA_OFFSET 8
         leaq  \sym(%rip),%rax
         jmp error_entry
+        CFI_ENDPROC
         .endm   
 
         .macro errorentry sym
+        XCPT_FRAME
         pushq %rax
+        CFI_ADJUST_CFA_OFFSET 8
         leaq  \sym(%rip),%rax
         jmp error_entry
+        CFI_ENDPROC
         .endm
 
         /* error code is on the stack already */
@@ -605,10 +692,7 @@ ENTRY(spurious_interrupt)
  * and the exception handler in %rax.   
  */                                             
 ENTRY(error_entry)
-        CFI_STARTPROC   simple
-        CFI_DEF_CFA     rsp,(SS-RDI)
-        CFI_REL_OFFSET  rsp,(RSP-RDI)
-        CFI_REL_OFFSET  rip,(RIP-RDI)
+        _frame RDI
         /* rdi slot contains rax, oldrax contains error code */
         cld     
         subq  $14*8,%rsp
@@ -690,7 +774,9 @@ error_kernelspace:
        /* Reload gs selector with exception handling */
        /* edi:  new selector */ 
 ENTRY(load_gs_index)
+        CFI_STARTPROC
         pushf
+        CFI_ADJUST_CFA_OFFSET 8
         cli
         swapgs
 gs_change:     
@@ -698,7 +784,9 @@ gs_change:
 2:      mfence          /* workaround */
         swapgs
         popf
+        CFI_ADJUST_CFA_OFFSET -8
         ret
+        CFI_ENDPROC
        
         .section __ex_table,"a"
         .align 8
@@ -810,7 +898,7 @@ ENTRY(device_not_available)
 
         /* runs on exception stack */
 KPROBE_ENTRY(debug)
-        CFI_STARTPROC
+        INTR_FRAME
         pushq $0
         CFI_ADJUST_CFA_OFFSET 8         
         paranoidentry do_debug
@@ -820,9 +908,9 @@ KPROBE_ENTRY(debug)
 
         /* runs on exception stack */   
 ENTRY(nmi)
-        CFI_STARTPROC
+        INTR_FRAME
         pushq $-1
-        CFI_ADJUST_CFA_OFFSET 8         
+        CFI_ADJUST_CFA_OFFSET 8
         paranoidentry do_nmi
         /*
          * "Paranoid" exit path from exception stack.
@@ -888,7 +976,7 @@ ENTRY(reserved)
 
         /* runs on exception stack */
 ENTRY(double_fault)
-        CFI_STARTPROC
+        XCPT_FRAME
         paranoidentry do_double_fault
         jmp paranoid_exit
         CFI_ENDPROC
@@ -901,7 +989,7 @@ ENTRY(segment_not_present)
 
         /* runs on exception stack */
 ENTRY(stack_segment)
-        CFI_STARTPROC
+        XCPT_FRAME
         paranoidentry do_stack_segment
         jmp paranoid_exit
         CFI_ENDPROC
@@ -922,7 +1010,7 @@ ENTRY(spurious_interrupt_bug)
 #ifdef CONFIG_X86_MCE
         /* runs on exception stack */
 ENTRY(machine_check)
-        CFI_STARTPROC
+        INTR_FRAME
         pushq $0
         CFI_ADJUST_CFA_OFFSET 8 
         paranoidentry do_machine_check
@@ -934,14 +1022,19 @@ ENTRY(call_debug)
        zeroentry do_call_debug
 
 ENTRY(call_softirq)
+        CFI_STARTPROC
         movq %gs:pda_irqstackptr,%rax
         pushq %r15
+        CFI_ADJUST_CFA_OFFSET 8
         movq %rsp,%r15
+        CFI_DEF_CFA_REGISTER    r15
         incl %gs:pda_irqcount
         cmove %rax,%rsp
         call __do_softirq
         movq %r15,%rsp
+        CFI_DEF_CFA_REGISTER    rsp
         decl %gs:pda_irqcount
         popq %r15
+        CFI_ADJUST_CFA_OFFSET -8
         ret
-
+        CFI_ENDPROC