3 files changed, 115 insertions, 87 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index c1808606ee5f..a88f4a485c5e 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2660,6 +2660,7 @@ asmlinkage void __sched notrace preempt_schedule(void)
         } while (need_resched());
 }
 EXPORT_SYMBOL(preempt_schedule);
+#endif /* CONFIG_PREEMPT */
 
 /*
  * this is the entry point to schedule() from kernel preemption
@@ -2693,8 +2694,6 @@ asmlinkage void __sched preempt_schedule_irq(void)
         exception_exit(prev_state);
 }
 
-#endif /* CONFIG_PREEMPT */
-
 int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
                           void *key)
 {
@@ -4762,7 +4761,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
                 cpumask_clear_cpu(rq->cpu, old_rd->span);
 
                 /*
-                 * If we dont want to free the old_rt yet then
+                 * If we dont want to free the old_rd yet then
                  * set old_rd to NULL to skip the freeing later
                  * in this function:
                  */
@@ -4903,6 +4902,7 @@ DEFINE_PER_CPU(struct sched_domain *, sd_asym);
 static void update_top_cache_domain(int cpu)
 {
         struct sched_domain *sd;
+        struct sched_domain *busy_sd = NULL;
         int id = cpu;
         int size = 1;
 
@@ -4910,8 +4910,9 @@ static void update_top_cache_domain(int cpu)
         if (sd) {
                 id = cpumask_first(sched_domain_span(sd));
                 size = cpumask_weight(sched_domain_span(sd));
-                rcu_assign_pointer(per_cpu(sd_busy, cpu), sd->parent);
+                busy_sd = sd->parent; /* sd_busy */
         }
+        rcu_assign_pointer(per_cpu(sd_busy, cpu), busy_sd);
 
         rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
         per_cpu(sd_llc_size, cpu) = size;
@@ -5112,6 +5113,7 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu)
                  * die on a /0 trap.
                  */
                 sg->sgp->power = SCHED_POWER_SCALE * cpumask_weight(sg_span);
+                sg->sgp->power_orig = sg->sgp->power;
 
                 /*
                  * Make sure the first group of this domain contains the
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index e8b652ebe027..c7395d97e4cb 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -178,59 +178,61 @@ void sched_init_granularity(void)
         update_sysctl();
 }
 
-#if BITS_PER_LONG == 32
-# define WMULT_CONST    (~0UL)
-#else
-# define WMULT_CONST    (1UL << 32)
-#endif
-
+#define WMULT_CONST     (~0U)
 #define WMULT_SHIFT     32
 
-/*
- * Shift right and round:
- */
-#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
+static void __update_inv_weight(struct load_weight *lw)
+{
+        unsigned long w;
+
+        if (likely(lw->inv_weight))
+                return;
+
+        w = scale_load_down(lw->weight);
+
+        if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
+                lw->inv_weight = 1;
+        else if (unlikely(!w))
+                lw->inv_weight = WMULT_CONST;
+        else
+                lw->inv_weight = WMULT_CONST / w;
+}
 
 /*
- * delta *= weight / lw
+ * delta_exec * weight / lw.weight
+ *   OR
+ * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT
+ *
+ * Either weight := NICE_0_LOAD and lw \e prio_to_wmult[], in which case
+ * we're guaranteed shift stays positive because inv_weight is guaranteed to
+ * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22.
+ *
+ * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus
+ * weight/lw.weight <= 1, and therefore our shift will also be positive.
  */
-static unsigned long
-calc_delta_mine(unsigned long delta_exec, unsigned long weight,
-                struct load_weight *lw)
+static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
 {
-        u64 tmp;
+        u64 fact = scale_load_down(weight);
+        int shift = WMULT_SHIFT;
 
-        /*
-         * weight can be less than 2^SCHED_LOAD_RESOLUTION for task group sched
-         * entities since MIN_SHARES = 2. Treat weight as 1 if less than
-         * 2^SCHED_LOAD_RESOLUTION.
-         */
-        if (likely(weight > (1UL << SCHED_LOAD_RESOLUTION)))
-                tmp = (u64)delta_exec * scale_load_down(weight);
-        else
-                tmp = (u64)delta_exec;
+        __update_inv_weight(lw);
 
-        if (!lw->inv_weight) {
-                unsigned long w = scale_load_down(lw->weight);
-
-                if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
-                        lw->inv_weight = 1;
-                else if (unlikely(!w))
-                        lw->inv_weight = WMULT_CONST;
-                else
-                        lw->inv_weight = WMULT_CONST / w;
+        if (unlikely(fact >> 32)) {
+                while (fact >> 32) {
+                        fact >>= 1;
+                        shift--;
+                }
         }
 
-        /*
-         * Check whether we'd overflow the 64-bit multiplication:
-         */
-        if (unlikely(tmp > WMULT_CONST))
-                tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
-                        WMULT_SHIFT/2);
-        else
-                tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
+        /* hint to use a 32x32->64 mul */
+        fact = (u64)(u32)fact * lw->inv_weight;
 
-        return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
+        while (fact >> 32) {
+                fact >>= 1;
+                shift--;
+        }
+
+        return mul_u64_u32_shr(delta_exec, fact, shift);
 }
 
 
@@ -443,7 +445,7 @@ find_matching_se(struct sched_entity **se, struct sched_entity **pse)
 #endif  /* CONFIG_FAIR_GROUP_SCHED */
 
 static __always_inline
-void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, unsigned long delta_exec);
+void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
 
 /**************************************************************
  * Scheduling class tree data structure manipulation methods:
@@ -612,11 +614,10 @@ int sched_proc_update_handler(struct ctl_table *table, int write,
 /*
  * delta /= w
  */
-static inline unsigned long
-calc_delta_fair(unsigned long delta, struct sched_entity *se)
+static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se)
 {
         if (unlikely(se->load.weight != NICE_0_LOAD))
-                delta = calc_delta_mine(delta, NICE_0_LOAD, &se->load);
+                delta = __calc_delta(delta, NICE_0_LOAD, &se->load);
 
         return delta;
 }
@@ -665,7 +666,7 @@ static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
                         update_load_add(&lw, se->load.weight);
                         load = &lw;
                 }
-                slice = calc_delta_mine(slice, se->load.weight, load);
+                slice = __calc_delta(slice, se->load.weight, load);
         }
         return slice;
 }
@@ -703,47 +704,32 @@ void init_task_runnable_average(struct task_struct *p)
 #endif
 
 /*
- * Update the current task's runtime statistics. Skip current tasks that
- * are not in our scheduling class.
+ * Update the current task's runtime statistics.
  */
-static inline void
-__update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr,
-              unsigned long delta_exec)
-{
-        unsigned long delta_exec_weighted;
-
-        schedstat_set(curr->statistics.exec_max,
-                      max((u64)delta_exec, curr->statistics.exec_max));
-
-        curr->sum_exec_runtime += delta_exec;
-        schedstat_add(cfs_rq, exec_clock, delta_exec);
-        delta_exec_weighted = calc_delta_fair(delta_exec, curr);
-
-        curr->vruntime += delta_exec_weighted;
-        update_min_vruntime(cfs_rq);
-}
-
 static void update_curr(struct cfs_rq *cfs_rq)
 {
         struct sched_entity *curr = cfs_rq->curr;
         u64 now = rq_clock_task(rq_of(cfs_rq));
-        unsigned long delta_exec;
+        u64 delta_exec;
 
         if (unlikely(!curr))
                 return;
 
-        /*
-         * Get the amount of time the current task was running
-         * since the last time we changed load (this cannot
-         * overflow on 32 bits):
-         */
-        delta_exec = (unsigned long)(now - curr->exec_start);
-        if (!delta_exec)
+        delta_exec = now - curr->exec_start;
+        if (unlikely((s64)delta_exec <= 0))
                 return;
 
-        __update_curr(cfs_rq, curr, delta_exec);
         curr->exec_start = now;
 
+        schedstat_set(curr->statistics.exec_max,
+                      max(delta_exec, curr->statistics.exec_max));
+
+        curr->sum_exec_runtime += delta_exec;
+        schedstat_add(cfs_rq, exec_clock, delta_exec);
+
+        curr->vruntime += calc_delta_fair(delta_exec, curr);
+        update_min_vruntime(cfs_rq);
+
         if (entity_is_task(curr)) {
                 struct task_struct *curtask = task_of(curr);
 
@@ -1752,6 +1738,13 @@ void task_numa_work(struct callback_head *work)
                     (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ)))
                         continue;
 
+                /*
+                 * Skip inaccessible VMAs to avoid any confusion between
+                 * PROT_NONE and NUMA hinting ptes
+                 */
+                if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
+                        continue;
+
                 do {
                         start = max(start, vma->vm_start);
                         end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
@@ -3015,8 +3008,7 @@ static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq)
         }
 }
 
-static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
-                                     unsigned long delta_exec)
+static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
 {
         /* dock delta_exec before expiring quota (as it could span periods) */
         cfs_rq->runtime_remaining -= delta_exec;
@@ -3034,7 +3026,7 @@ static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
 }
 
 static __always_inline
-void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, unsigned long delta_exec)
+void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
 {
         if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled)
                 return;
@@ -3574,8 +3566,7 @@ static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq)
         return rq_clock_task(rq_of(cfs_rq));
 }
 
-static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
-                                     unsigned long delta_exec) {}
+static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {}
 static void check_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
@@ -5379,10 +5370,31 @@ void update_group_power(struct sched_domain *sd, int cpu)
                  */
 
                 for_each_cpu(cpu, sched_group_cpus(sdg)) {
-                        struct sched_group *sg = cpu_rq(cpu)->sd->groups;
+                        struct sched_group_power *sgp;
+                        struct rq *rq = cpu_rq(cpu);
+
+                        /*
+                         * build_sched_domains() -> init_sched_groups_power()
+                         * gets here before we've attached the domains to the
+                         * runqueues.
+                         *
+                         * Use power_of(), which is set irrespective of domains
+                         * in update_cpu_power().
+                         *
+                         * This avoids power/power_orig from being 0 and
+                         * causing divide-by-zero issues on boot.
+                         *
+                         * Runtime updates will correct power_orig.
+                         */
+                        if (unlikely(!rq->sd)) {
+                                power_orig += power_of(cpu);
+                                power += power_of(cpu);
+                                continue;
+                        }
 
-                        power_orig += sg->sgp->power_orig;
-                        power += sg->sgp->power;
+                        sgp = rq->sd->groups->sgp;
+                        power_orig += sgp->power_orig;
+                        power += sgp->power;
                 }
         } else  {
                 /*
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 7d57275fc396..1c4065575fa2 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -901,6 +901,13 @@ inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
 {
         struct rq *rq = rq_of_rt_rq(rt_rq);
 
+#ifdef CONFIG_RT_GROUP_SCHED
+        /*
+         * Change rq's cpupri only if rt_rq is the top queue.
+         */
+        if (&rq->rt != rt_rq)
+                return;
+#endif
         if (rq->online && prio < prev_prio)
                 cpupri_set(&rq->rd->cpupri, rq->cpu, prio);
 }
@@ -910,6 +917,13 @@ dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
 {
         struct rq *rq = rq_of_rt_rq(rt_rq);
 
+#ifdef CONFIG_RT_GROUP_SCHED
+        /*
+         * Change rq's cpupri only if rt_rq is the top queue.
+         */
+        if (&rq->rt != rt_rq)
+                return;
+#endif
         if (rq->online && rt_rq->highest_prio.curr != prev_prio)
                 cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);
 }