Merge branch 'sched/core' into core/locking, to prepare the kernel/locking/ file move

Conflicts:
	kernel/Makefile

There are conflicts in kernel/Makefile due to file moving in the
scheduler tree - resolve them.

Signed-off-by: Ingo Molnar <mingo@kernel.org>

13 files changed, 70 insertions, 18 deletions

diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index a5408b965c9d..9b0a34e2cd79 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -36,6 +36,8 @@ obj-y			+= tsc.o io_delay.o rtc.o
 obj-y                   += pci-iommu_table.o
 obj-y                   += resource.o
 
+obj-$(CONFIG_PREEMPT)   += preempt.o
+
 obj-y                           += process.o
 obj-y                           += i387.o xsave.o
 obj-y                           += ptrace.o
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 28610822fb3c..9f6b9341950f 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -32,7 +32,6 @@ void common(void) {
         OFFSET(TI_flags, thread_info, flags);
         OFFSET(TI_status, thread_info, status);
         OFFSET(TI_addr_limit, thread_info, addr_limit);
-        OFFSET(TI_preempt_count, thread_info, preempt_count);
 
         BLANK();
         OFFSET(crypto_tfm_ctx_offset, crypto_tfm, __crt_ctx);
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 2793d1f095a2..5223fe6dec7b 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1095,6 +1095,9 @@ DEFINE_PER_CPU(char *, irq_stack_ptr) =
 
 DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1;
 
+DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
+EXPORT_PER_CPU_SYMBOL(__preempt_count);
+
 DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
 
 /*
@@ -1169,6 +1172,8 @@ void debug_stack_reset(void)
 
 DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
 EXPORT_PER_CPU_SYMBOL(current_task);
+DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
+EXPORT_PER_CPU_SYMBOL(__preempt_count);
 DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
 
 #ifdef CONFIG_CC_STACKPROTECTOR
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index f0dcb0ceb6a2..fd1bc1b15e6d 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -362,12 +362,9 @@ END(ret_from_exception)
 #ifdef CONFIG_PREEMPT
 ENTRY(resume_kernel)
         DISABLE_INTERRUPTS(CLBR_ANY)
-        cmpl $0,TI_preempt_count(%ebp)  # non-zero preempt_count ?
-        jnz restore_all
 need_resched:
-        movl TI_flags(%ebp), %ecx       # need_resched set ?
-        testb $_TIF_NEED_RESCHED, %cl
-        jz restore_all
+        cmpl $0,PER_CPU_VAR(__preempt_count)
+        jnz restore_all
         testl $X86_EFLAGS_IF,PT_EFLAGS(%esp)    # interrupts off (exception path) ?
         jz restore_all
         call preempt_schedule_irq
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index b077f4cc225a..1a2cc64abcd7 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1103,10 +1103,8 @@ retint_signal:
         /* Returning to kernel space. Check if we need preemption */
         /* rcx:  threadinfo. interrupts off. */
 ENTRY(retint_kernel)
-        cmpl $0,TI_preempt_count(%rcx)
+        cmpl $0,PER_CPU_VAR(__preempt_count)
         jnz  retint_restore_args
-        bt  $TIF_NEED_RESCHED,TI_flags(%rcx)
-        jnc  retint_restore_args
         bt   $9,EFLAGS-ARGOFFSET(%rsp)  /* interrupts off? */
         jnc  retint_restore_args
         call preempt_schedule_irq
diff --git a/arch/x86/kernel/i386_ksyms_32.c b/arch/x86/kernel/i386_ksyms_32.c
index 0fa69127209a..05fd74f537d6 100644
--- a/arch/x86/kernel/i386_ksyms_32.c
+++ b/arch/x86/kernel/i386_ksyms_32.c
@@ -37,3 +37,10 @@ EXPORT_SYMBOL(strstr);
 
 EXPORT_SYMBOL(csum_partial);
 EXPORT_SYMBOL(empty_zero_page);
+
+#ifdef CONFIG_PREEMPT
+EXPORT_SYMBOL(___preempt_schedule);
+#ifdef CONFIG_CONTEXT_TRACKING
+EXPORT_SYMBOL(___preempt_schedule_context);
+#endif
+#endif
diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index 4186755f1d7c..3fe066359ac0 100644
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -100,9 +100,6 @@ execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
         irqctx->tinfo.task = curctx->tinfo.task;
         irqctx->tinfo.previous_esp = current_stack_pointer;
 
-        /* Copy the preempt_count so that the [soft]irq checks work. */
-        irqctx->tinfo.preempt_count = curctx->tinfo.preempt_count;
-
         if (unlikely(overflow))
                 call_on_stack(print_stack_overflow, isp);
 
@@ -131,7 +128,6 @@ void irq_ctx_init(int cpu)
                                                THREAD_SIZE_ORDER));
         memset(&irqctx->tinfo, 0, sizeof(struct thread_info));
         irqctx->tinfo.cpu               = cpu;
-        irqctx->tinfo.preempt_count     = HARDIRQ_OFFSET;
         irqctx->tinfo.addr_limit        = MAKE_MM_SEG(0);
 
         per_cpu(hardirq_ctx, cpu) = irqctx;
diff --git a/arch/x86/kernel/preempt.S b/arch/x86/kernel/preempt.S
new file mode 100644
index 000000000000..ca7f0d58a87d
--- /dev/null
+++ b/arch/x86/kernel/preempt.S
@@ -0,0 +1,25 @@
+
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/asm.h>
+#include <asm/calling.h>
+
+ENTRY(___preempt_schedule)
+        CFI_STARTPROC
+        SAVE_ALL
+        call preempt_schedule
+        RESTORE_ALL
+        ret
+        CFI_ENDPROC
+
+#ifdef CONFIG_CONTEXT_TRACKING
+
+ENTRY(___preempt_schedule_context)
+        CFI_STARTPROC
+        SAVE_ALL
+        call preempt_schedule_context
+        RESTORE_ALL
+        ret
+        CFI_ENDPROC
+
+#endif
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index c83516be1052..3fb8d95ab8b5 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -391,9 +391,9 @@ static void amd_e400_idle(void)
                  * The switch back from broadcast mode needs to be
                  * called with interrupts disabled.
                  */
-                 local_irq_disable();
-                 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
-                 local_irq_enable();
+                local_irq_disable();
+                clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
+                local_irq_enable();
         } else
                 default_idle();
 }
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 884f98f69354..c2ec1aa6d454 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -292,6 +292,14 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
                 set_iopl_mask(next->iopl);
 
         /*
+         * If it were not for PREEMPT_ACTIVE we could guarantee that the
+         * preempt_count of all tasks was equal here and this would not be
+         * needed.
+         */
+        task_thread_info(prev_p)->saved_preempt_count = this_cpu_read(__preempt_count);
+        this_cpu_write(__preempt_count, task_thread_info(next_p)->saved_preempt_count);
+
+        /*
          * Now maybe handle debug registers and/or IO bitmaps
          */
         if (unlikely(task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV ||
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index bb1dc51bab05..45ab4d6fc8a7 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -363,6 +363,14 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
         this_cpu_write(old_rsp, next->usersp);
         this_cpu_write(current_task, next_p);
 
+        /*
+         * If it were not for PREEMPT_ACTIVE we could guarantee that the
+         * preempt_count of all tasks was equal here and this would not be
+         * needed.
+         */
+        task_thread_info(prev_p)->saved_preempt_count = this_cpu_read(__preempt_count);
+        this_cpu_write(__preempt_count, task_thread_info(next_p)->saved_preempt_count);
+
         this_cpu_write(kernel_stack,
                   (unsigned long)task_stack_page(next_p) +
                   THREAD_SIZE - KERNEL_STACK_OFFSET);
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 8c8093b146ca..729aa779ff75 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -88,7 +88,7 @@ static inline void conditional_sti(struct pt_regs *regs)
 
 static inline void preempt_conditional_sti(struct pt_regs *regs)
 {
-        inc_preempt_count();
+        preempt_count_inc();
         if (regs->flags & X86_EFLAGS_IF)
                 local_irq_enable();
 }
@@ -103,7 +103,7 @@ static inline void preempt_conditional_cli(struct pt_regs *regs)
 {
         if (regs->flags & X86_EFLAGS_IF)
                 local_irq_disable();
-        dec_preempt_count();
+        preempt_count_dec();
 }
 
 static int __kprobes
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c
index b014d9414d08..040681928e9d 100644
--- a/arch/x86/kernel/x8664_ksyms_64.c
+++ b/arch/x86/kernel/x8664_ksyms_64.c
@@ -66,3 +66,10 @@ EXPORT_SYMBOL(empty_zero_page);
 #ifndef CONFIG_PARAVIRT
 EXPORT_SYMBOL(native_load_gs_index);
 #endif
+
+#ifdef CONFIG_PREEMPT
+EXPORT_SYMBOL(___preempt_schedule);
+#ifdef CONFIG_CONTEXT_TRACKING
+EXPORT_SYMBOL(___preempt_schedule_context);
+#endif
+#endif