sched/x86: Rewrite the switch_to() code

Move the low-level context switch code to an out-of-line asm stub instead of
using complex inline asm.  This allows constructing a new stack frame for the
child process to make it seamlessly flow to ret_from_fork without an extra
test and branch in __switch_to().  It also improves code generation for
__schedule() by using the C calling convention instead of clobbering all
registers.

Signed-off-by: Brian Gerst <brgerst@gmail.com>
Reviewed-by: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/1471106302-10159-5-git-send-email-brgerst@gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

6 files changed, 29 insertions, 6 deletions

diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 2bd5c6ff7ee7..db3a0af9b9ec 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -29,6 +29,12 @@
 
 void common(void) {
         BLANK();
+        OFFSET(TASK_threadsp, task_struct, thread.sp);
+#ifdef CONFIG_CC_STACKPROTECTOR
+        OFFSET(TASK_stack_canary, task_struct, stack_canary);
+#endif
+
+        BLANK();
         OFFSET(TI_flags, thread_info, flags);
         OFFSET(TI_status, thread_info, status);
 
diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
index ecdc1d217dc0..880aa093268d 100644
--- a/arch/x86/kernel/asm-offsets_32.c
+++ b/arch/x86/kernel/asm-offsets_32.c
@@ -57,6 +57,11 @@ void foo(void)
         /* Size of SYSENTER_stack */
         DEFINE(SIZEOF_SYSENTER_stack, sizeof(((struct tss_struct *)0)->SYSENTER_stack));
 
+#ifdef CONFIG_CC_STACKPROTECTOR
+        BLANK();
+        OFFSET(stack_canary_offset, stack_canary, canary);
+#endif
+
 #if defined(CONFIG_LGUEST) || defined(CONFIG_LGUEST_GUEST) || defined(CONFIG_LGUEST_MODULE)
         BLANK();
         OFFSET(LGUEST_DATA_irq_enabled, lguest_data, irq_enabled);
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index d875f97d4e0b..210927ee2e74 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -56,6 +56,11 @@ int main(void)
         OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
         BLANK();
 
+#ifdef CONFIG_CC_STACKPROTECTOR
+        DEFINE(stack_canary_offset, offsetof(union irq_stack_union, stack_canary));
+        BLANK();
+#endif
+
         DEFINE(__NR_syscall_max, sizeof(syscalls_64) - 1);
         DEFINE(NR_syscalls, sizeof(syscalls_64));
 
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index d86be29c38c7..4bedbc08e53c 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -133,17 +133,20 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp,
         unsigned long arg, struct task_struct *p, unsigned long tls)
 {
         struct pt_regs *childregs = task_pt_regs(p);
+        struct fork_frame *fork_frame = container_of(childregs, struct fork_frame, regs);
+        struct inactive_task_frame *frame = &fork_frame->frame;
         struct task_struct *tsk;
         int err;
 
-        p->thread.sp = (unsigned long) childregs;
+        frame->bp = 0;
+        p->thread.sp = (unsigned long) fork_frame;
         p->thread.sp0 = (unsigned long) (childregs+1);
         memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
 
         if (unlikely(p->flags & PF_KTHREAD)) {
                 /* kernel thread */
                 memset(childregs, 0, sizeof(struct pt_regs));
-                p->thread.ip = (unsigned long) ret_from_kernel_thread;
+                frame->ret_addr = (unsigned long) ret_from_kernel_thread;
                 task_user_gs(p) = __KERNEL_STACK_CANARY;
                 childregs->ds = __USER_DS;
                 childregs->es = __USER_DS;
@@ -161,7 +164,7 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp,
         if (sp)
                 childregs->sp = sp;
 
-        p->thread.ip = (unsigned long) ret_from_fork;
+        frame->ret_addr = (unsigned long) ret_from_fork;
         task_user_gs(p) = get_user_gs(current_pt_regs());
 
         p->thread.io_bitmap_ptr = NULL;
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 63236d8f84bf..827eeed03e16 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -141,12 +141,17 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long sp,
 {
         int err;
         struct pt_regs *childregs;
+        struct fork_frame *fork_frame;
+        struct inactive_task_frame *frame;
         struct task_struct *me = current;
 
         p->thread.sp0 = (unsigned long)task_stack_page(p) + THREAD_SIZE;
         childregs = task_pt_regs(p);
-        p->thread.sp = (unsigned long) childregs;
-        set_tsk_thread_flag(p, TIF_FORK);
+        fork_frame = container_of(childregs, struct fork_frame, regs);
+        frame = &fork_frame->frame;
+        frame->bp = 0;
+        frame->ret_addr = (unsigned long) ret_from_fork;
+        p->thread.sp = (unsigned long) fork_frame;
         p->thread.io_bitmap_ptr = NULL;
 
         savesegment(gs, p->thread.gsindex);
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index c85d2c636092..7e52f83d3a4b 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -942,7 +942,6 @@ void common_cpu_up(unsigned int cpu, struct task_struct *idle)
         per_cpu(cpu_current_top_of_stack, cpu) =
                 (unsigned long)task_stack_page(idle) + THREAD_SIZE;
 #else
-        clear_tsk_thread_flag(idle, TIF_FORK);
         initial_gs = per_cpu_offset(cpu);
 #endif
 }