x86: Speed up ___preempt_schedule*() by using THUNK helpers

___preempt_schedule() does SAVE_ALL/RESTORE_ALL but this is
suboptimal, we do not need to save/restore the callee-saved
register. And we already have arch/x86/lib/thunk_*.S which
implements the similar asm wrappers, so it makes sense to
redefine ___preempt_schedule() as "THUNK ..." and remove
preempt.S altogether.

Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Reviewed-by: Andy Lutomirski <luto@amacapital.net>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20140921184153.GA23727@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

2 files changed, 23 insertions, 4 deletions

diff --git a/arch/x86/lib/thunk_32.S b/arch/x86/lib/thunk_32.S
index 28f85c916712..e9acf5f4fc92 100644
--- a/arch/x86/lib/thunk_32.S
+++ b/arch/x86/lib/thunk_32.S
@@ -7,16 +7,19 @@
         #include <linux/linkage.h>
         #include <asm/asm.h>
 
-#ifdef CONFIG_TRACE_IRQFLAGS
         /* put return address in eax (arg1) */
-        .macro thunk_ra name,func
+        .macro THUNK name, func, put_ret_addr_in_eax=0
         .globl \name
 \name:
         pushl %eax
         pushl %ecx
         pushl %edx
+
+        .if \put_ret_addr_in_eax
         /* Place EIP in the arg1 */
         movl 3*4(%esp), %eax
+        .endif
+
         call \func
         popl %edx
         popl %ecx
@@ -25,6 +28,15 @@
         _ASM_NOKPROBE(\name)
         .endm
 
-        thunk_ra trace_hardirqs_on_thunk,trace_hardirqs_on_caller
-        thunk_ra trace_hardirqs_off_thunk,trace_hardirqs_off_caller
+#ifdef CONFIG_TRACE_IRQFLAGS
+        THUNK trace_hardirqs_on_thunk,trace_hardirqs_on_caller,1
+        THUNK trace_hardirqs_off_thunk,trace_hardirqs_off_caller,1
+#endif
+
+#ifdef CONFIG_PREEMPT
+        THUNK ___preempt_schedule, preempt_schedule
+#ifdef CONFIG_CONTEXT_TRACKING
+        THUNK ___preempt_schedule_context, preempt_schedule_context
 #endif
+#endif
+
diff --git a/arch/x86/lib/thunk_64.S b/arch/x86/lib/thunk_64.S
index 92d9feaff42b..b30b5ebd614a 100644
--- a/arch/x86/lib/thunk_64.S
+++ b/arch/x86/lib/thunk_64.S
@@ -38,6 +38,13 @@
         THUNK lockdep_sys_exit_thunk,lockdep_sys_exit
 #endif
 
+#ifdef CONFIG_PREEMPT
+        THUNK ___preempt_schedule, preempt_schedule
+#ifdef CONFIG_CONTEXT_TRACKING
+        THUNK ___preempt_schedule_context, preempt_schedule_context
+#endif
+#endif
+
         /* SAVE_ARGS below is used only for the .cfi directives it contains. */
         CFI_STARTPROC
         SAVE_ARGS