lockdep: x86_64: connect the sysexit hook

Run the lockdep_sys_exit hook after all other C code on the syscall
return path.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

3 files changed, 29 insertions, 12 deletions

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 1d232e5f5658..f1cacd4897f7 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -244,6 +244,7 @@ ret_from_sys_call:
         movl $_TIF_ALLWORK_MASK,%edi
         /* edi: flagmask */
 sysret_check:           
+        LOCKDEP_SYS_EXIT
         GET_THREAD_INFO(%rcx)
         cli
         TRACE_IRQS_OFF
@@ -333,6 +334,7 @@ int_ret_from_sys_call:
         movl $_TIF_ALLWORK_MASK,%edi
         /* edi: mask to check */
 int_with_check:
+        LOCKDEP_SYS_EXIT_IRQ
         GET_THREAD_INFO(%rcx)
         movl threadinfo_flags(%rcx),%edx
         andl %edi,%edx
@@ -544,11 +546,13 @@ exit_intr:
 retint_with_reschedule:
         movl $_TIF_WORK_MASK,%edi
 retint_check:
+        LOCKDEP_SYS_EXIT_IRQ
         movl threadinfo_flags(%rcx),%edx
         andl %edi,%edx
         CFI_REMEMBER_STATE
         jnz  retint_careful
-retint_swapgs:          
+
+retint_swapgs:          /* return to user-space */
         /*
          * The iretq could re-enable interrupts:
          */
@@ -557,7 +561,7 @@ retint_swapgs:
         swapgs 
         jmp restore_args
 
-retint_restore_args:                            
+retint_restore_args:    /* return to kernel space */
         cli
         /*
          * The iretq could re-enable interrupts:
@@ -866,26 +870,21 @@ error_sti:
         movq ORIG_RAX(%rsp),%rsi        /* get error code */ 
         movq $-1,ORIG_RAX(%rsp)
         call *%rax
-        /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */     
-error_exit:             
-        movl %ebx,%eax          
+        /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
+error_exit:
+        movl %ebx,%eax
         RESTORE_REST
         cli
         TRACE_IRQS_OFF
         GET_THREAD_INFO(%rcx)   
         testl %eax,%eax
         jne  retint_kernel
+        LOCKDEP_SYS_EXIT_IRQ
         movl  threadinfo_flags(%rcx),%edx
         movl  $_TIF_WORK_MASK,%edi
         andl  %edi,%edx
         jnz  retint_careful
-        /*
-         * The iret might restore flags:
-         */
-        TRACE_IRQS_IRETQ
-        swapgs 
-        RESTORE_ARGS 0,8,0                                              
-        jmp iret_label
+        jmp retint_swapgs
         CFI_ENDPROC
 
 error_kernelspace:
diff --git a/arch/x86/lib/thunk_64.S b/arch/x86/lib/thunk_64.S
index 55e586d352d3..6ea73f3de567 100644
--- a/arch/x86/lib/thunk_64.S
+++ b/arch/x86/lib/thunk_64.S
@@ -50,6 +50,10 @@
         thunk trace_hardirqs_on_thunk,trace_hardirqs_on
         thunk trace_hardirqs_off_thunk,trace_hardirqs_off
 #endif
+
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+        thunk lockdep_sys_exit_thunk,lockdep_sys_exit
+#endif
         
         /* SAVE_ARGS below is used only for the .cfi directives it contains. */
         CFI_STARTPROC
diff --git a/include/asm-x86/irqflags_64.h b/include/asm-x86/irqflags_64.h
index 86e70fe23659..5341ea1f815a 100644
--- a/include/asm-x86/irqflags_64.h
+++ b/include/asm-x86/irqflags_64.h
@@ -137,6 +137,20 @@ static inline void halt(void)
 #  define TRACE_IRQS_ON
 #  define TRACE_IRQS_OFF
 # endif
+# ifdef CONFIG_DEBUG_LOCK_ALLOC
+#  define LOCKDEP_SYS_EXIT      call lockdep_sys_exit_thunk
+#  define LOCKDEP_SYS_EXIT_IRQ  \
+        TRACE_IRQS_ON; \
+        sti; \
+        SAVE_REST; \
+        LOCKDEP_SYS_EXIT; \
+        RESTORE_REST; \
+        cli; \
+        TRACE_IRQS_OFF;
+# else
+#  define LOCKDEP_SYS_EXIT
+#  define LOCKDEP_SYS_EXIT_IRQ
+# endif
 #endif
 
 #endif