1 files changed, 126 insertions, 81 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f5c6635b806c..ae365aaa8181 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1745,8 +1745,10 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
         p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
         p->numa_scan_period = sysctl_numa_balancing_scan_delay;
         p->numa_work.next = &p->numa_work;
-        p->numa_faults = NULL;
-        p->numa_faults_buffer = NULL;
+        p->numa_faults_memory = NULL;
+        p->numa_faults_buffer_memory = NULL;
+        p->last_task_numa_placement = 0;
+        p->last_sum_exec_runtime = 0;
 
         INIT_LIST_HEAD(&p->numa_entry);
         p->numa_group = NULL;
@@ -2149,8 +2151,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
         if (mm)
                 mmdrop(mm);
         if (unlikely(prev_state == TASK_DEAD)) {
-                task_numa_free(prev);
-
                 if (prev->sched_class->task_dead)
                         prev->sched_class->task_dead(prev);
 
@@ -2167,13 +2167,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
 
 #ifdef CONFIG_SMP
 
-/* assumes rq->lock is held */
-static inline void pre_schedule(struct rq *rq, struct task_struct *prev)
-{
-        if (prev->sched_class->pre_schedule)
-                prev->sched_class->pre_schedule(rq, prev);
-}
-
 /* rq->lock is NOT held, but preemption is disabled */
 static inline void post_schedule(struct rq *rq)
 {
@@ -2191,10 +2184,6 @@ static inline void post_schedule(struct rq *rq)
 
 #else
 
-static inline void pre_schedule(struct rq *rq, struct task_struct *p)
-{
-}
-
 static inline void post_schedule(struct rq *rq)
 {
 }
@@ -2510,8 +2499,13 @@ void __kprobes preempt_count_add(int val)
         DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
                                 PREEMPT_MASK - 10);
 #endif
-        if (preempt_count() == val)
-                trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
+        if (preempt_count() == val) {
+                unsigned long ip = get_parent_ip(CALLER_ADDR1);
+#ifdef CONFIG_DEBUG_PREEMPT
+                current->preempt_disable_ip = ip;
+#endif
+                trace_preempt_off(CALLER_ADDR0, ip);
+        }
 }
 EXPORT_SYMBOL(preempt_count_add);
 
@@ -2554,6 +2548,13 @@ static noinline void __schedule_bug(struct task_struct *prev)
         print_modules();
         if (irqs_disabled())
                 print_irqtrace_events(prev);
+#ifdef CONFIG_DEBUG_PREEMPT
+        if (in_atomic_preempt_off()) {
+                pr_err("Preemption disabled at:");
+                print_ip_sym(current->preempt_disable_ip);
+                pr_cont("\n");
+        }
+#endif
         dump_stack();
         add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
 }
@@ -2577,36 +2578,34 @@ static inline void schedule_debug(struct task_struct *prev)
         schedstat_inc(this_rq(), sched_count);
 }
 
-static void put_prev_task(struct rq *rq, struct task_struct *prev)
-{
-        if (prev->on_rq || rq->skip_clock_update < 0)
-                update_rq_clock(rq);
-        prev->sched_class->put_prev_task(rq, prev);
-}
-
 /*
  * Pick up the highest-prio task:
  */
 static inline struct task_struct *
-pick_next_task(struct rq *rq)
+pick_next_task(struct rq *rq, struct task_struct *prev)
 {
-        const struct sched_class *class;
+        const struct sched_class *class = &fair_sched_class;
         struct task_struct *p;
 
         /*
          * Optimization: we know that if all tasks are in
          * the fair class we can call that function directly:
          */
-        if (likely(rq->nr_running == rq->cfs.h_nr_running)) {
-                p = fair_sched_class.pick_next_task(rq);
-                if (likely(p))
+        if (likely(prev->sched_class == class &&
+                   rq->nr_running == rq->cfs.h_nr_running)) {
+                p = fair_sched_class.pick_next_task(rq, prev);
+                if (likely(p && p != RETRY_TASK))
                         return p;
         }
 
+again:
         for_each_class(class) {
-                p = class->pick_next_task(rq);
-                if (p)
+                p = class->pick_next_task(rq, prev);
+                if (p) {
+                        if (unlikely(p == RETRY_TASK))
+                                goto again;
                         return p;
+                }
         }
 
         BUG(); /* the idle class will always have a runnable task */
@@ -2700,13 +2699,10 @@ need_resched:
                 switch_count = &prev->nvcsw;
         }
 
-        pre_schedule(rq, prev);
-
-        if (unlikely(!rq->nr_running))
-                idle_balance(cpu, rq);
+        if (prev->on_rq || rq->skip_clock_update < 0)
+                update_rq_clock(rq);
 
-        put_prev_task(rq, prev);
-        next = pick_next_task(rq);
+        next = pick_next_task(rq, prev);
         clear_tsk_need_resched(prev);
         clear_preempt_need_resched();
         rq->skip_clock_update = 0;
@@ -2908,7 +2904,8 @@ EXPORT_SYMBOL(sleep_on_timeout);
  * This function changes the 'effective' priority of a task. It does
  * not touch ->normal_prio like __setscheduler().
  *
- * Used by the rt_mutex code to implement priority inheritance logic.
+ * Used by the rt_mutex code to implement priority inheritance
+ * logic. Call site only calls if the priority of the task changed.
  */
 void rt_mutex_setprio(struct task_struct *p, int prio)
 {
@@ -2998,7 +2995,7 @@ void set_user_nice(struct task_struct *p, long nice)
         unsigned long flags;
         struct rq *rq;
 
-        if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
+        if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
                 return;
         /*
          * We have to be careful, if called from sys_setpriority(),
@@ -3076,11 +3073,11 @@ SYSCALL_DEFINE1(nice, int, increment)
         if (increment > 40)
                 increment = 40;
 
-        nice = TASK_NICE(current) + increment;
-        if (nice < -20)
-                nice = -20;
+        nice = task_nice(current) + increment;
+        if (nice < MIN_NICE)
+                nice = MIN_NICE;