Merge master.kernel.org:/pub/scm/linux/kernel/git/torvalds/linux-2.6 into next

1 files changed, 489 insertions, 483 deletions

diff --git a/kernel/sched.c b/kernel/sched.c
index cfa222a91539..99e6d850ecab 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -70,10 +70,13 @@
 #include <linux/bootmem.h>
 #include <linux/debugfs.h>
 #include <linux/ctype.h>
+#include <linux/ftrace.h>
 
 #include <asm/tlb.h>
 #include <asm/irq_regs.h>
 
+#include "sched_cpupri.h"
+
 /*
  * Convert user-nice values [ -20 ... 0 ... 19 ]
  * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
@@ -136,7 +139,7 @@ static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
 
 static inline int rt_policy(int policy)
 {
-        if (unlikely(policy == SCHED_FIFO) || unlikely(policy == SCHED_RR))
+        if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
                 return 1;
         return 0;
 }
@@ -289,15 +292,15 @@ struct task_group root_task_group;
 static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
 /* Default task group's cfs_rq on each cpu */
 static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
-#endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
 static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
-#endif
-#else
+#endif /* CONFIG_RT_GROUP_SCHED */
+#else /* !CONFIG_FAIR_GROUP_SCHED */
 #define root_task_group init_task_group
-#endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 
 /* task_group_lock serializes add/remove of task groups and also changes to
  * a task group's cpu shares.
@@ -307,17 +310,20 @@ static DEFINE_SPINLOCK(task_group_lock);
 #ifdef CONFIG_FAIR_GROUP_SCHED
 #ifdef CONFIG_USER_SCHED
 # define INIT_TASK_GROUP_LOAD   (2*NICE_0_LOAD)
-#else
+#else /* !CONFIG_USER_SCHED */
 # define INIT_TASK_GROUP_LOAD   NICE_0_LOAD
-#endif
+#endif /* CONFIG_USER_SCHED */
 
 /*
- * A weight of 0, 1 or ULONG_MAX can cause arithmetics problems.
+ * A weight of 0 or 1 can cause arithmetics problems.
+ * A weight of a cfs_rq is the sum of weights of which entities
+ * are queued on this cfs_rq, so a weight of a entity should not be
+ * too large, so as the shares value of a task group.
  * (The default weight is 1024 - so there's no practical
  *  limitation from this.)
  */
 #define MIN_SHARES      2
-#define MAX_SHARES      (ULONG_MAX - 1)
+#define MAX_SHARES      (1UL << 18)
 
 static int init_task_group_load = INIT_TASK_GROUP_LOAD;
 #endif
@@ -360,6 +366,10 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
 #else
 
 static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
+static inline struct task_group *task_group(struct task_struct *p)
+{
+        return NULL;
+}
 
 #endif  /* CONFIG_GROUP_SCHED */
 
@@ -370,6 +380,7 @@ struct cfs_rq {
 
         u64 exec_clock;
         u64 min_vruntime;
+        u64 pair_start;
 
         struct rb_root tasks_timeline;
         struct rb_node *rb_leftmost;
@@ -400,40 +411,28 @@ struct cfs_rq {
         struct task_group *tg;  /* group that "owns" this runqueue */
 
 #ifdef CONFIG_SMP
-        unsigned long task_weight;
-        unsigned long shares;
         /*
-         * We need space to build a sched_domain wide view of the full task
-         * group tree, in order to avoid depending on dynamic memory allocation
-         * during the load balancing we place this in the per cpu task group
-         * hierarchy. This limits the load balancing to one instance per cpu,
-         * but more should not be needed anyway.
+         * the part of load.weight contributed by tasks
          */
-        struct aggregate_struct {
-                /*
-                 *   load = weight(cpus) * f(tg)
-                 *
-                 * Where f(tg) is the recursive weight fraction assigned to
-                 * this group.
-                 */
-                unsigned long load;
+        unsigned long task_weight;
 
-                /*
-                 * part of the group weight distributed to this span.
-                 */
-                unsigned long shares;
+        /*
+         *   h_load = weight * f(tg)
+         *
+         * Where f(tg) is the recursive weight fraction assigned to
+         * this group.
+         */
+        unsigned long h_load;
 
-                /*
-                 * The sum of all runqueue weights within this span.
-                 */
-                unsigned long rq_weight;
+        /*
+         * this cpu's part of tg->shares
+         */
+        unsigned long shares;
 
-                /*
-                 * Weight contributed by tasks; this is the part we can
-                 * influence by moving tasks around.
-                 */
-                unsigned long task_weight;
-        } aggregate;
+        /*
+         * load.weight at the time we set shares
+         */
+        unsigned long rq_weight;
 #endif
 #endif
 };
@@ -486,6 +485,9 @@ struct root_domain {
          */
         cpumask_t rto_mask;
         atomic_t rto_count;
+#ifdef CONFIG_SMP
+        struct cpupri cpupri;
+#endif
 };
 
 /*
@@ -560,6 +562,9 @@ struct rq {
         int push_cpu;
         /* cpu of this runqueue: */
         int cpu;
+        int online;
+
+        unsigned long avg_load_per_task;
 
         struct task_struct *migration_thread;
         struct list_head migration_queue;
@@ -641,6 +646,24 @@ static inline void update_rq_clock(struct rq *rq)
 # define const_debug static const
 #endif
 
+/**
+ * runqueue_is_locked
+ *
+ * Returns true if the current cpu runqueue is locked.
+ * This interface allows printk to be called with the runqueue lock
+ * held and know whether or not it is OK to wake up the klogd.
+ */
+int runqueue_is_locked(void)
+{
+        int cpu = get_cpu();
+        struct rq *rq = cpu_rq(cpu);
+        int ret;
+
+        ret = spin_is_locked(&rq->lock);
+        put_cpu();
+        return ret;
+}
+
 /*
  * Debugging: various feature bits
  */
@@ -783,6 +806,12 @@ late_initcall(sched_init_debug);
 const_debug unsigned int sysctl_sched_nr_migrate = 32;
 
 /*
+ * ratelimit for updating the group shares.
+ * default: 0.5ms
+ */
+const_debug unsigned int sysctl_sched_shares_ratelimit = 500000;
+
+/*
  * period over which we measure -rt task cpu usage in us.
  * default: 1s
  */
@@ -809,82 +838,6 @@ static inline u64 global_rt_runtime(void)
         return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
 }
 
-unsigned long long time_sync_thresh = 100000;
-
-static DEFINE_PER_CPU(unsigned long long, time_offset);
-static DEFINE_PER_CPU(unsigned long long, prev_cpu_time);
-
-/*
- * Global lock which we take every now and then to synchronize
- * the CPUs time. This method is not warp-safe, but it's good
- * enough to synchronize slowly diverging time sources and thus
- * it's good enough for tracing:
- */
-static DEFINE_SPINLOCK(time_sync_lock);
-static unsigned long long prev_global_time;
-
-static unsigned long long __sync_cpu_clock(unsigned long long time, int cpu)
-{
-        /*
-         * We want this inlined, to not get tracer function calls
-         * in this critical section:
-         */
-        spin_acquire(&time_sync_lock.dep_map, 0, 0, _THIS_IP_);
-        __raw_spin_lock(&time_sync_lock.raw_lock);
-
-        if (time < prev_global_time) {
-                per_cpu(time_offset, cpu) += prev_global_time - time;
-                time = prev_global_time;
-        } else {
-                prev_global_time = time;
-        }
-
-        __raw_spin_unlock(&time_sync_lock.raw_lock);
-        spin_release(&time_sync_lock.dep_map, 1, _THIS_IP_);
-
-        return time;
-}
-
-static unsigned long long __cpu_clock(int cpu)
-{
-        unsigned long long now;
-
-        /*
-         * Only call sched_clock() if the scheduler has already been
-         * initialized (some code might call cpu_clock() very early):
-         */
-        if (unlikely(!scheduler_running))
-                return 0;
-
-        now = sched_clock_cpu(cpu);
-
-        return now;
-}
-
-/*
- * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
- * clock constructed from sched_clock():
- */
-unsigned long long cpu_clock(int cpu)
-{
-        unsigned long long prev_cpu_time, time, delta_time;
-        unsigned long flags;
-
-        local_irq_save(flags);
-        prev_cpu_time = per_cpu(prev_cpu_time, cpu);
-        time = __cpu_clock(cpu) + per_cpu(time_offset, cpu);
-        delta_time = time-prev_cpu_time;
-
-        if (unlikely(delta_time > time_sync_thresh)) {
-                time = __sync_cpu_clock(time, cpu);
-                per_cpu(prev_cpu_time, cpu) = time;
-        }
-        local_irq_restore(flags);
-
-        return time;
-}
-EXPORT_SYMBOL_GPL(cpu_clock);
-
 #ifndef prepare_arch_switch
 # define prepare_arch_switch(next)      do { } while (0)
 #endif
@@ -1161,6 +1114,7 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer)
         return HRTIMER_NORESTART;
 }
 
+#ifdef CONFIG_SMP
 static void hotplug_hrtick_disable(int cpu)
 {
         struct rq *rq = cpu_rq(cpu);
@@ -1216,6 +1170,7 @@ static void init_hrtick(void)
 {
         hotcpu_notifier(hotplug_hrtick, 0);
 }
+#endif /* CONFIG_SMP */
 
 static void init_rq_hrtick(struct rq *rq)
 {
@@ -1345,15 +1300,15 @@ void wake_up_idle_cpu(int cpu)
         if (!tsk_is_polling(rq->idle))
                 smp_send_reschedule(cpu);
 }
-#endif
+#endif /* CONFIG_NO_HZ */
 
-#else
+#else /* !CONFIG_SMP */
 static void __resched_task(struct task_struct *p, int tif_bit)
 {
         assert_spin_locked(&task_rq(p)->lock);
         set_tsk_thread_flag(p, tif_bit);
 }
-#endif
+#endif /* CONFIG_SMP */
 
 #if BITS_PER_LONG == 32
 # define WMULT_CONST    (~0UL)
@@ -1377,8 +1332,13 @@ calc_delta_mine(unsigned long delta_exec, unsigned long weight,
 {
         u64 tmp;
 
-        if (!lw->inv_weight)
-                lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)/(lw->weight+1);
+        if (!lw->inv_weight) {
+                if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
+                        lw->inv_weight = 1;
+                else
+                        lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
+                                / (lw->weight+1);
+        }
 
         tmp = (u64)delta_exec * weight;
         /*
@@ -1503,63 +1463,35 @@ static inline void dec_cpu_load(struct rq *rq, unsigned long load)
 #ifdef CONFIG_SMP
 static unsigned long source_load(int cpu, int type);
 static unsigned long target_load(int cpu, int type);
-static unsigned long cpu_avg_load_per_task(int cpu);
 static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
 
-#ifdef CONFIG_FAIR_GROUP_SCHED
+static unsigned long cpu_avg_load_per_task(int cpu)
+{
+        struct rq *rq = cpu_rq(cpu);
 
-/*
- * Group load balancing.
- *
- * We calculate a few balance domain wide aggregate numbers; load and weight.
- * Given the pictures below, and assuming each item has equal weight:
- *
- *         root          1 - thread
- *         / | \         A - group
- *        A  1  B
- *       /|\   / \
- *      C 2 D 3   4
- *      |   |
- *      5   6
- *
- * load:
- *    A and B get 1/3-rd of the total load. C and D get 1/3-rd of A's 1/3-rd,
- *    which equals 1/9-th of the total load.
- *
- * shares:
- *    The weight of this group on the selected cpus.
- *
- * rq_weight:
- *    Direct sum of all the cpu's their rq weight, e.g. A would get 3 while
- *    B would get 2.
- *
- * task_weight:
- *    Part of the rq_weight contributed by tasks; all groups except B would
- *    get 1, B gets 2.
- */
+        if (rq->nr_running)
+                rq->avg_load_per_task = rq->load.weight / rq->nr_running;
 
-static inline struct aggregate_struct *
-aggregate(struct task_group *tg, struct sched_domain *sd)
-{
-        return &tg->cfs_rq[sd->first_cpu]->aggregate;
+        return rq->avg_load_per_task;
 }
 
-typedef void (*aggregate_func)(struct task_group *, struct sched_domain *);
+#ifdef CONFIG_FAIR_GROUP_SCHED
+
+typedef void (*tg_visitor)(struct task_group *, int, struct sched_domain *);
 
 /*
  * Iterate the full tree, calling @down when first entering a node and @up when
  * leaving it for the final time.
  */
-static
-void aggregate_walk_tree(aggregate_func down, aggregate_func up,
-                         struct sched_domain *sd)
+static void
+walk_tg_tree(tg_visitor down, tg_visitor up, int cpu, struct sched_domain *sd)
 {
         struct task_group *parent, *child;
 
         rcu_read_lock();
         parent = &root_task_group;
 down:
-        (*down)(parent, sd);
+        (*down)(parent, cpu, sd);
         list_for_each_entry_rcu(child, &parent->children, siblings) {
                 parent = child;
                 goto down;
@@ -1567,7 +1499,7 @@ down:
 up:
                 continue;
         }
-        (*up)(parent, sd);
+        (*up)(parent, cpu, sd);
 
         child = parent;
         parent = parent->parent;
@@ -1576,90 +1508,23 @@ up:
         rcu_read_unlock();
 }
 
-/*
- * Calculate the aggregate runqueue weight.
- */
-static
-void aggregate_group_weight(struct task_group *tg, struct sched_domain *sd)
-{
-        unsigned long rq_weight = 0;
-        unsigned long task_weight = 0;
-        int i;
-
-        for_each_cpu_mask(i, sd->span) {
-                rq_weight += tg->cfs_rq[i]->load.weight;
-                task_weight += tg->cfs_rq[i]->task_weight;
-        }
-
-        aggregate(tg, sd)->rq_weight = rq_weight;
-        aggregate(tg, sd)->task_weight = task_weight;
-}
-
-/*
- * Compute the weight of this group on the given cpus.
- */
-static
-void aggregate_group_shares(struct task_group *tg, struct sched_domain *sd)
-{
-        unsigned long shares = 0;
-        int i;
-
-        for_each_cpu_mask(i, sd->span)
-                shares += tg->cfs_rq[i]->shares;
-
-        if ((!shares && aggregate(tg, sd)->rq_weight) || shares > tg->shares)
-                shares = tg->shares;
-
-        aggregate(tg, sd)->shares = shares;
-}
-
-/*
- * Compute the load fraction assigned to this group, relies on the aggregate
- * weight and this group's parent's load, i.e. top-down.
- */
-static
-void aggregate_group_load(struct task_group *tg, struct sched_domain *sd)
-{
-        unsigned long load;
-
-        if (!tg->parent) {
-                int i;
-
-                load = 0;
-                for_each_cpu_mask(i, sd->span)
-                        load += cpu_rq(i)->load.weight;
-
-        } else {
-                load = aggregate(tg->parent, sd)->load;
-
-                /*
-                 * shares is our weight in the parent's rq so
-                 * shares/parent->rq_weight gives our fraction of the load
-                 */
-                load *= aggregate(tg, sd)->shares;
-                load /= aggregate(tg->parent, sd)->rq_weight + 1;
-        }
-
-        aggregate(tg, sd)->load = load;
-}
-
 static void __set_se_shares(struct sched_entity *se, unsigned long shares);
 
 /*
  * Calculate and set the cpu's group shares.
  */
 static void
-__update_group_shares_cpu(struct task_group *tg, struct sched_domain *sd,
-                          int tcpu)
+__update_group_shares_cpu(struct task_group *tg, int cpu,
+                          unsigned long sd_shares, unsigned long sd_rq_weight)
 {
         int boost = 0;
         unsigned long shares;
         unsigned long rq_weight;
 
-        if (!tg->se[tcpu])
+        if (!tg->se[cpu])
                 return;
 
-        rq_weight = tg->cfs_rq[tcpu]->load.weight;
+        rq_weight = tg->cfs_rq[cpu]->load.weight;
 
         /*
          * If there are currently no tasks on the cpu pretend there is one of
@@ -1671,170 +1536,139 @@ __update_group_shares_cpu(struct task_group *tg, struct sched_domain *sd,
                 rq_weight = NICE_0_LOAD;
         }
 
+        if (unlikely(rq_weight > sd_rq_weight))
+                rq_weight = sd_rq_weight;
+
         /*
          *           \Sum shares * rq_weight
          * shares =  -----------------------
          *               \Sum rq_weight
          *
          */
-        shares = aggregate(tg, sd)->shares * rq_weight;
-        shares /= aggregate(tg, sd)->rq_weight + 1;
+        shares = (sd_shares * rq_weight) / (sd_rq_weight + 1);
 
         /*
          * record the actual number of shares, not the boosted amount.
          */
-        tg->cfs_rq[tcpu]->shares = boost ? 0 : shares;
+        tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
+        tg->cfs_rq[cpu]->rq_weight = rq_weight;
 
         if (shares < MIN_SHARES)
                 shares = MIN_SHARES;
         else if (shares > MAX_SHARES)
                 shares = MAX_SHARES;
 
-        __set_se_shares(tg->se[tcpu], shares);
+        __set_se_shares(tg->se[cpu], shares);
 }
 
 /*
- * Re-adjust the weights on the cpu the task came from and on the cpu the
- * task went to.
+ * Re-compute the task group their per cpu shares over the given domain.
+ * This needs to be done in a bottom-up fashion because the rq weight of a
+ * parent group depends on the shares of its child groups.
  */
 static void
-__move_group_shares(struct task_group *tg, struct sched_domain *sd,
-                    int scpu, int dcpu)
+tg_shares_up(struct task_group *tg, int cpu, struct sched_domain *sd)
 {
-        unsigned long shares;
-
-        shares = tg->cfs_rq[scpu]->shares + tg->cfs_rq[dcpu]->shares;
+        unsigned long rq_weight = 0;
+        unsigned long shares = 0;
+        int i;
 
-        __update_group_shares_cpu(tg, sd, scpu);
-        __update_group_shares_cpu(tg, sd, dcpu);
+        for_each_cpu_mask(i, sd->span) {
+                rq_weight += tg->cfs_rq[i]->load.weight;
+                shares += tg->cfs_rq[i]->shares;
+        }
 
-        /*
-         * ensure we never loose shares due to rounding errors in the
-         * above redistribution.
-         */
-        shares -= tg->cfs_rq[scpu]->shares + tg->cfs_rq[dcpu]->shares;
-        if (shares)
-                tg->cfs_rq[dcpu]->shares += shares;
-}
+        if ((!shares && rq_weight) || shares > tg->shares)
+                shares = tg->shares;
 
-/*
- * Because changing a group's shares changes the weight of the super-group
- * we need to walk up the tree and change all shares until we hit the root.
- */
-static void
-move_group_shares(struct task_group *tg, struct sched_domain *sd,
-                  int scpu, int dcpu)
-{
-        while (tg) {
-                __move_group_shares(tg, sd, scpu, dcpu);
-                tg = tg->parent;
-        }
-}
+        if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
+                shares = tg->shares;
 
-static
-void aggregate_group_set_shares(struct task_group *tg, struct sched_domain *sd)
-{
-        unsigned long shares = aggregate(tg, sd)->shares;
-        int i;
+        if (!rq_weight)
+                rq_weight = cpus_weight(sd->span) * NICE_0_LOAD;
 
         for_each_cpu_mask(i, sd->span) {
                 struct rq *rq = cpu_rq(i);
                 unsigned long flags;
 
                 spin_lock_irqsave(&rq->lock, flags);
-                __update_group_shares_cpu(tg, sd, i);
+                __update_group_shares_cpu(tg, i, shares, rq_weight);
                 spin_unlock_irqrestore(&rq->lock, flags);
         }
-
-        aggregate_group_shares(tg, sd);
-
-        /*
-         * ensure we never loose shares due to rounding errors in the
-         * above redistribution.
-         */
-        shares -= aggregate(tg, sd)->shares;
-        if (shares) {
-                tg->cfs_rq[sd->first_cpu]->shares += shares;
-                aggregate(tg, sd)->shares += shares;
-        }
 }
 
 /*
- * Calculate the accumulative weight and recursive load of each task group
- * while walking down the tree.
+ * Compute the cpu's hierarchical load factor for each task group.
+ * This needs to be done in a top-down fashion because the load of a child
+ * group is a fraction of its parents load.
  */
-static
-void aggregate_get_down(struct task_group *tg, struct sched_domain *sd)
+static void
+tg_load_down(struct task_group *tg, int cpu, struct sched_domain *sd)
 {
-        aggregate_group_weight(tg, sd);
-        aggregate_group_shares(tg, sd);
-        aggregate_group_load(tg, sd);
-}
+        unsigned long load;
 
-/*
- * Rebalance the cpu shares while walking back up the tree.
- */
-static
-void aggregate_get_up(struct task_group *tg, struct sched_domain *sd)
-{
-        aggregate_group_set_shares(tg, sd);
-}
+        if (!tg->parent) {
+                load = cpu_rq(cpu)->load.weight;
+        } else {
+                load = tg->parent->cfs_rq[cpu]->h_load;
+                load *= tg->cfs_rq[cpu]->shares;
+                load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
+        }
 
-static DEFINE_PER_CPU(spinlock_t, aggregate_lock);
+        tg->cfs_rq[cpu]->h_load = load;
+}
 
-static void __init init_aggregate(void)
+static void
+tg_nop(struct task_group *tg, int cpu, struct sched_domain *sd)
 {
-        int i;
-
-        for_each_possible_cpu(i)
-                spin_lock_init(&per_cpu(aggregate_lock, i));
 }
 
-static int get_aggregate(struct sched_domain *sd)
+static void update_shares(struct sched_domain *sd)
 {
-        if (!spin_trylock(&per_cpu(aggregate_lock, sd->first_cpu)))
-                return 0;
+        u64 now = cpu_clock(raw_smp_processor_id());
+        s64 elapsed = now - sd->last_update;
 
-        aggregate_walk_tree(aggregate_get_down, aggregate_get_up, sd);
-        return 1;
+        if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
+                sd->last_update = now;
+                walk_tg_tree(tg_nop, tg_shares_up, 0, sd);
+        }
 }
 
-static void put_aggregate(struct sched_domain *sd)
+static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
 {
-        spin_unlock(&per_cpu(aggregate_lock, sd->first_cpu));
+        spin_unlock(&rq->lock);
+        update_shares(sd);
+        spin_lock(&rq->lock);
 }
 
-static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
+static void update_h_load(int cpu)
 {
-        cfs_rq->shares = shares;
+        walk_tg_tree(tg_load_down, tg_nop, cpu, NULL);
 }
 
 #else
 
-static inline void init_aggregate(void)
+static inline void update_shares(struct sched_domain *sd)
 {
 }
 
-static inline int get_aggregate(struct sched_domain *sd)
+static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
 {
-        return 0;
 }
 
-static inline void put_aggregate(struct sched_domain *sd)
-{
-}
 #endif
 
-#else /* CONFIG_SMP */
+#endif
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
 static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
 {
+#ifdef CONFIG_SMP
+        cfs_rq->shares = shares;
+#endif
 }
 #endif
 
-#endif /* CONFIG_SMP */
-
 #include "sched_stats.h"
 #include "sched_idletask.c"
 #include "sched_fair.c"
@@ -1844,6 +1678,8 @@ static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
 #endif
 
 #define sched_class_highest (&rt_sched_class)
+#define for_each_class(class) \
+   for (class = sched_class_highest; class; class = class->next)
 
 static void inc_nr_running(struct rq *rq)
 {
@@ -1876,6 +1712,12 @@ static void set_load_weight(struct task_struct *p)
         p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
 }
 
+static void update_avg(u64 *avg, u64 sample)
+{
+        s64 diff = sample - *avg;
+        *avg += diff >> 3;
+}
+
 static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
 {
         sched_info_queued(p);
@@ -1885,6 +1727,13 @@ static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
 
 static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
 {
+        if (sleep && p->se.last_wakeup) {
+                update_avg(&p->se.avg_overlap,
+                           p->se.sum_exec_runtime - p->se.last_wakeup);
+                p->se.last_wakeup = 0;
+        }
+
+        sched_info_dequeued(p);
         p->sched_class->dequeue_task(rq, p, sleep);
         p->se.on_rq = 0;
 }
@@ -1968,12 +1817,6 @@ inline int task_curr(const struct task_struct *p)
         return cpu_curr(task_cpu(p)) == p;
 }
 
-/* Used instead of source_load when we know the type == 0 */
-unsigned long weighted_cpuload(const int cpu)
-{
-        return cpu_rq(cpu)->load.weight;
-}
-
 static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
 {
         set_task_rq(p, cpu);
@@ -2002,6 +1845,12 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p,
 
 #ifdef CONFIG_SMP
 
+/* Used instead of source_load when we know the type == 0 */
+static unsigned long weighted_cpuload(const int cpu)
+{
+        return cpu_rq(cpu)->load.weight;
+}
+
 /*
  * Is this task likely cache-hot:
  */
@@ -2212,7 +2061,7 @@ static unsigned long source_load(int cpu, int type)
         struct rq *rq = cpu_rq(cpu);
         unsigned long total = weighted_cpuload(cpu);
 
-        if (type == 0)
+        if (type == 0 || !sched_feat(LB_BIAS))
                 return total;
 
         return min(rq->cpu_load[type-1], total);
@@ -2227,25 +2076,13 @@ static unsigned long target_load(int cpu, int type)
         struct rq *rq = cpu_rq(cpu);
         unsigned long total = weighted_cpuload(cpu);
 
-        if (type == 0)
+        if (type == 0 || !sched_feat(LB_BIAS))
                 return total;
 
         return max(rq->cpu_load[type-1], total);
 }
 
 /*
- * Return the average load per task on the cpu's run queue
- */
-static unsigned long cpu_avg_load_per_task(int cpu)
-{
-        struct rq *rq = cpu_rq(cpu);
-        unsigned long total = weighted_cpuload(cpu);
-        unsigned long n = rq->nr_running;
-
-        return n ? total / n : SCHED_LOAD_SCALE;
-}
-
-/*
  * find_idlest_group finds and returns the least busy CPU group within the
  * domain.
  */
@@ -2351,6 +2188,9 @@ static int sched_balance_self(int cpu, int flag)
                         sd = tmp;
         }
 
+        if (sd)
+                update_shares(sd);
+
         while (sd) {
                 cpumask_t span, tmpmask;
                 struct sched_group *group;
@@ -2417,6 +2257,22 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
         if (!sched_feat(SYNC_WAKEUPS))
                 sync = 0;
 
+#ifdef CONFIG_SMP
+        if (sched_feat(LB_WAKEUP_UPDATE)) {
+                struct sched_domain *sd;
+
+                this_cpu = raw_smp_processor_id();
+                cpu = task_cpu(p);
+
+                for_each_domain(this_cpu, sd) {
+                        if (cpu_isset(cpu, sd->span)) {
+                                update_shares(sd);
+                                break;
+                        }
+                }
+        }
+#endif
+
         smp_wmb();
         rq = task_rq_lock(p, &flags);
         old_state = p->state;
@@ -2463,7 +2319,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
                         }
                 }
         }
-#endif
+#endif /* CONFIG_SCHEDSTATS */
 
 out_activate:
 #endif /* CONFIG_SMP */
@@ -2481,6 +2337,9 @@ out_activate:
         success = 1;
 
 out_running:
+        trace_mark(kernel_sched_wakeup,
+                "pid %d state %ld ## rq %p task %p rq->curr %p",
+                p->pid, p->state, rq, p, rq->curr);
         check_preempt_curr(rq, p);
 
         p->state = TASK_RUNNING;
@@ -2489,6 +2348,8 @@ out_running:
                 p->sched_class->task_wake_up(rq, p);
 #endif
 out:
+        current->se.last_wakeup = current->se.sum_exec_runtime;
+
         task_rq_unlock(rq, &flags);
 
         return success;
@@ -2611,6 +2472,9 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
                 p->sched_class->task_new(rq, p);
                 inc_nr_running(rq);
         }
+        trace_mark(kernel_sched_wakeup_new,
+                "pid %d state %ld ## rq %p task %p rq->curr %p",
+                p->pid, p->state, rq, p, rq->curr);
         check_preempt_curr(rq, p);
 #ifdef CONFIG_SMP
         if (p->sched_class->task_wake_up)
@@ -2663,7 +2527,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr,
                 notifier->ops->sched_out(notifier, next);
 }
 
-#else
+#else /* !CONFIG_PREEMPT_NOTIFIERS */
 
 static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
 {
@@ -2675,7 +2539,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr,
 {
 }
 
-#endif
+#endif /* CONFIG_PREEMPT_NOTIFIERS */
 
 /**
  * prepare_task_switch - prepare to switch tasks
@@ -2783,6 +2647,11 @@ context_switch(struct rq *rq, struct task_struct *prev,
         struct mm_struct *mm, *oldmm;
 
         prepare_task_switch(rq, prev, next);
+        trace_mark(kernel_sched_schedule,
+                "prev_pid %d next_pid %d prev_state %ld "
+                "## rq %p prev %p next %p",
+                prev->pid, next->pid, prev->state,
+                rq, prev, next);
         mm = next->mm;
         oldmm = prev->active_mm;
         /*
@@ -3117,7 +2986,7 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
               enum cpu_idle_type idle, int *all_pinned,
               int *this_best_prio, struct rq_iterator *iterator)
 {
-        int loops = 0, pulled = 0, pinned = 0, skip_for_load;
+        int loops = 0, pulled = 0, pinned = 0;
         struct task_struct *p;
         long rem_load_move = max_load_move;
 
@@ -3133,14 +3002,8 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
 next:
         if (!p || loops++ > sysctl_sched_nr_migrate)
                 goto out;
-        /*
-         * To help distribute high priority tasks across CPUs we don't
-         * skip a task if it will be the highest priority task (i.e. smallest
-         * prio value) on its new queue regardless of its load weight
-         */
-        skip_for_load = (p->se.load.weight >> 1) > rem_load_move +
-                                                         SCHED_LOAD_SCALE_FUZZ;
-        if ((skip_for_load && p->prio >= *this_best_prio) ||
+
+        if ((p->se.load.weight >> 1) > rem_load_move ||
             !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
                 p = iterator->next(iterator->arg);
                 goto next;
@@ -3195,6 +3058,10 @@ static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
                                 max_load_move - total_load_moved,
                                 sd, idle, all_pinned, &this_best_prio);
                 class = class->next;
+
+                if (idle == CPU_NEWLY_IDLE && this_rq->nr_running)
+                        break;
+
         } while (class && max_load_move > total_load_moved);
 
         return total_load_moved > 0;
@@ -3271,6 +3138,7 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
         max_load = this_load = total_load = total_pwr = 0;
         busiest_load_per_task = busiest_nr_running = 0;
         this_load_per_task = this_nr_running = 0;
+
         if (idle == CPU_NOT_IDLE)
                 load_idx = sd->busy_idx;
         else if (idle == CPU_NEWLY_IDLE)
@@ -3285,6 +3153,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                 int __group_imb = 0;
                 unsigned int balance_cpu = -1, first_idle_cpu = 0;
                 unsigned long sum_nr_running, sum_weighted_load;
+                unsigned long sum_avg_load_per_task;
+                unsigned long avg_load_per_task;
 
                 local_group = cpu_isset(this_cpu, group->cpumask);
 
@@ -3293,6 +3163,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
 
                 /* Tally up the load of all CPUs in the group */
                 sum_weighted_load = sum_nr_running = avg_load = 0;
+                sum_avg_load_per_task = avg_load_per_task = 0;
+
                 max_cpu_load = 0;
                 min_cpu_load = ~0UL;
 
@@ -3326,6 +3198,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                         avg_load += load;
                         sum_nr_running += rq->nr_running;
                         sum_weighted_load += weighted_cpuload(i);
+
+                        sum_avg_load_per_task += cpu_avg_load_per_task(i);
                 }
 
                 /*
@@ -3347,7 +3221,20 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                 avg_load = sg_div_cpu_power(group,
                                 avg_load * SCHED_LOAD_SCALE);
 
-                if ((max_cpu_load - min_cpu_load) > SCHED_LOAD_SCALE)
+
+                /*
+                 * Consider the group unbalanced when the imbalance is larger
+                 * than the average weight of two tasks.
+                 *
+                 * APZ: with cgroup the avg task weight can vary wildly and
+                 *      might not be a suitable number - should we keep a
+                 *      normalized nr_running number somewhere that negates
+                 *      the hierarchy?
+                 */
+                avg_load_per_task = sg_div_cpu_power(group,
+                                sum_avg_load_per_task * SCHED_LOAD_SCALE);
+
+                if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
                         __group_imb = 1;
 
                 group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
@@ -3488,9 +3375,9 @@ small_imbalance:
                         if (busiest_load_per_task > this_load_per_task)
                                 imbn = 1;
                 } else
-                        this_load_per_task = SCHED_LOAD_SCALE;
+                        this_load_per_task = cpu_avg_load_per_task(this_cpu);
 
-                if (max_load - this_load + SCHED_LOAD_SCALE_FUZZ >=
+                if (max_load - this_load + 2*busiest_load_per_task >=
                                         busiest_load_per_task * imbn) {
                         *imbalance = busiest_load_per_task;
                         return busiest;
@@ -3600,12 +3487,9 @@ static int load_balance(int this_cpu, struct rq *this_rq,
         unsigned long imbalance;
         struct rq *busiest;
         unsigned long flags;
-        int unlock_aggregate;
 
         cpus_setall(*cpus);
 
-        unlock_aggregate = get_aggregate(sd);
-
         /*
          * When power savings policy is enabled for the parent domain, idle
          * sibling can pick up load irrespective of busy siblings. In this case,
@@ -3619,6 +3503,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
         schedstat_inc(sd, lb_count[idle]);
 
 redo:
+        update_shares(sd);
         group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
                                    cpus, balance);
 
@@ -3742,8 +3627,8 @@ out_one_pinned:
         else
                 ld_moved = 0;
 out:
-        if (unlock_aggregate)
-                put_aggregate(sd);
+        if (ld_moved)
+                update_shares(sd);
         return ld_moved;
 }
 
@@ -3779,6 +3664,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
 
         schedstat_inc(sd, lb_count[CPU_NEWLY_IDLE]);
 redo:
+        update_shares_locked(this_rq, sd);
         group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
                                    &sd_idle, cpus, NULL);
         if (!group) {
@@ -3822,6 +3708,7 @@ redo:
         } else
                 sd->nr_balance_failed = 0;
 
+        update_shares_locked(this_rq, sd);
         return ld_moved;
 
 out_balanced:
@@ -4013,6 +3900,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
         /* Earliest time when we have to do rebalance again */
         unsigned long next_balance = jiffies + 60*HZ;
         int update_next_balance = 0;
+        int need_serialize;
         cpumask_t tmp;
 
         for_each_domain(cpu, sd) {
@@ -4030,8 +3918,9 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
                 if (interval > HZ*NR_CPUS/10)
                         interval = HZ*NR_CPUS/10;
 
+                need_serialize = sd->flags & SD_SERIALIZE;
 
-                if (sd->flags & SD_SERIALIZE) {
+                if (need_serialize) {
                         if (!spin_trylock(&balancing))
                                 goto out;
                 }
@@ -4047,7 +3936,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
                         }
                         sd->last_balance = jiffies;
                 }
-                if (sd->flags & SD_SERIALIZE)
+                if (need_serialize)
                         spin_unlock(&balancing);
 out:
                 if (time_after(next_balance, sd->last_balance + interval)) {
@@ -4362,26 +4251,44 @@ void scheduler_tick(void)
 #endif
 }
 
-#if defined(CONFIG_PREEMPT) && defined(CONFIG_DEBUG_PREEMPT)
+#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
+                                defined(CONFIG_PREEMPT_TRACER))
+
+static inline unsigned long get_parent_ip(unsigned long addr)
+{
+        if (in_lock_functions(addr)) {
+                addr = CALLER_ADDR2;
+                if (in_lock_functions(addr))
+                        addr = CALLER_ADDR3;
+        }
+        return addr;
+}
 
 void __kprobes add_preempt_count(int val)
 {
+#ifdef CONFIG_DEBUG_PREEMPT
         /*
          * Underflow?
          */
         if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
                 return;
+#endif
         preempt_count() += val;
+#ifdef CONFIG_DEBUG_PREEMPT
         /*
          * Spinlock count overflowing soon?
          */
         DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
                                 PREEMPT_MASK - 10);
+#endif
+        if (preempt_count() == val)
+                trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 }
 EXPORT_SYMBOL(add_preempt_count);
 
 void __kprobes sub_preempt_count(int val)
 {
+#ifdef CONFIG_DEBUG_PREEMPT
         /*
          * Underflow?
          */
@@ -4393,7 +4300,10 @@ void __kprobes sub_preempt_count(int val)
         if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
                         !(preempt_count() & PREEMPT_MASK)))
                 return;
+#endif
 
+        if (preempt_count() == val)
+                trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
         preempt_count() -= val;
 }
 EXPORT_SYMBOL(sub_preempt_count);
@@ -4411,6 +4321,7 @@ static noinline void __schedule_bug(struct task_struct *prev)
                 prev->comm, prev->pid, preempt_count());
 
         debug_show_held_locks(prev);
+        print_modules();
         if (irqs_disabled())
                 print_irqtrace_events(prev);
 
@@ -4430,7 +4341,7 @@ static inline void schedule_debug(struct task_struct *prev)
          * schedule() atomically, we ignore that path for now.
          * Otherwise, whine if we are scheduling when we should not be.
          */
-        if (unlikely(in_atomic_preempt_off()) && unlikely(!prev->exit_state))
+        if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
                 __schedule_bug(prev);
 
         profile_hit(SCHED_PROFILING, __builtin_return_address(0));
@@ -4484,7 +4395,7 @@ asmlinkage void __sched schedule(void)
         struct task_struct *prev, *next;
         unsigned long *switch_count;
         struct rq *rq;
-        int cpu;
+        int cpu, hrtick = sched_feat(HRTICK);
 
 need_resched:
         preempt_disable();
@@ -4499,7 +4410,8 @@ need_resched_nonpreemptible:
 
         schedule_debug(prev);
 
-        hrtick_clear(rq);
+        if (hrtick)
+                hrtick_clear(rq);
 
         /*
          * Do the rq-clock update outside the rq lock:
@@ -4510,12 +4422,10 @@ need_resched_nonpreemptible:
         clear_tsk_need_resched(prev);
 
         if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
-                if (unlikely((prev->state & TASK_INTERRUPTIBLE) &&
-                                signal_pending(prev))) {
+                if (unlikely(signal_pending_state(prev->state, prev)))
                         prev->state = TASK_RUNNING;
-                } else {
+                else
                         deactivate_task(rq, prev, 1);
-                }
                 switch_count = &prev->nvcsw;
         }
 
@@ -4547,7 +4457,8 @@ need_resched_nonpreemptible:
         } else
                 spin_unlock_irq(&rq->lock);
 
-        hrtick_set(rq);
+        if (hrtick)
+                hrtick_set(rq);
 
         if (unlikely(reacquire_kernel_lock(current) < 0))
                 goto need_resched_nonpreemptible;
@@ -4741,22 +4652,20 @@ do_wait_for_common(struct completion *x, long timeout, int state)
                              signal_pending(current)) ||
                             (state == TASK_KILLABLE &&
                              fatal_signal_pending(current))) {
-                                __remove_wait_queue(&x->wait, &wait);
-                                return -ERESTARTSYS;
+                                timeout = -ERESTARTSYS;
+                                break;
                         }
                         __set_current_state(state);
                         spin_unlock_irq(&x->wait.lock);
                         timeout = schedule_timeout(timeout);
                         spin_lock_irq(&x->wait.lock);
-                        if (!timeout) {
-                                __remove_wait_queue(&x->wait, &wait);
-                                return timeout;
-                        }
-                } while (!x->done);
+                } while (!x->done && timeout);
                 __remove_wait_queue(&x->wait, &wait);
+                if (!x->done)
+                        return timeout;
         }
         x->done--;
-        return timeout;
+        return timeout ?: 1;
 }
 
 static long __sched
@@ -5086,16 +4995,8 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
         set_load_weight(p);
 }
 
-/**
- * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
- * @p: the task in question.
- * @policy: new policy.
- * @param: structure containing the new RT priority.
- *
- * NOTE that the task may be already dead.
- */
-int sched_setscheduler(struct task_struct *p, int policy,
-                       struct sched_param *param)
+static int __sched_setscheduler(struct task_struct *p, int policy,
+                                struct sched_param *param, bool user)
 {
         int retval, oldprio, oldpolicy = -1, on_rq, running;
         unsigned long flags;
@@ -5127,7 +5028,7 @@ recheck:
         /*
          * Allow unprivileged RT tasks to decrease priority:
          */
-        if (!capable(CAP_SYS_NICE)) {
+        if (user && !capable(CAP_SYS_NICE)) {
                 if (rt_policy(policy)) {
                         unsigned long rlim_rtprio;
 
@@ -5163,7 +5064,8 @@ recheck:
          * Do not allow realtime tasks into groups that have no runtime
          * assigned.
          */
-        if (rt_policy(policy) && task_group(p)->rt_bandwidth.rt_runtime == 0)
+        if (user
+            && rt_policy(policy) && task_group(p)->rt_bandwidth.rt_runtime == 0)
                 return -EPERM;
 #endif
 
@@ -5212,8 +5114,39 @@ recheck:
 
         return 0;
 }
+
+/**
+ * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
+ * @p: the task in question.
+ * @policy: new policy.
+ * @param: structure containing the new RT priority.
+ *
+ * NOTE that the task may be already dead.
+ */
+int sched_setscheduler(struct task_struct *p, int policy,
+                       struct sched_param *param)
+{
+        return __sched_setscheduler(p, policy, param, true);
+}
 EXPORT_SYMBOL_GPL(sched_setscheduler);
 
+/**
+ * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
+ * @p: the task in question.
+ * @policy: new policy.
+ * @param: structure containing the new RT priority.
+ *
+ * Just like sched_setscheduler, only don't bother checking if the
+ * current context has permission.  For example, this is needed in
+ * stop_machine(): we create temporary high priority worker threads,
+ * but our caller might not have that capability.
+ */
+int sched_setscheduler_nocheck(struct task_struct *p, int policy,
+                               struct sched_param *param)
+{
+        return __sched_setscheduler(p, policy, param, false);
+}
+
 static int
 do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
 {
@@ -5412,24 +5345,6 @@ asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
         return sched_setaffinity(pid, &new_mask);
 }
 
-/*
- * Represents all cpu's present in the system
- * In systems capable of hotplug, this map could dynamically grow
- * as new cpu's are detected in the system via any platform specific
- * method, such as ACPI for e.g.
- */
-
-cpumask_t cpu_present_map __read_mostly;
-EXPORT_SYMBOL(cpu_present_map);
-
-#ifndef CONFIG_SMP
-cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
-EXPORT_SYMBOL(cpu_online_map);
-
-cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
-EXPORT_SYMBOL(cpu_possible_map);
-#endif
-
 long sched_getaffinity(pid_t pid, cpumask_t *mask)
 {
         struct task_struct *p;
@@ -5726,7 +5641,7 @@ out_unlock:
         return retval;
 }
 
-static const char stat_nam[] = "RSDTtZX";
+static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
 
 void sched_show_task(struct task_struct *p)
 {
@@ -5913,6 +5828,12 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
                 goto out;
         }
 
+        if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
+                     !cpus_equal(p->cpus_allowed, *new_mask))) {
+                ret = -EINVAL;
+                goto out;
+        }
+
         if (p->sched_class->set_cpus_allowed)
                 p->sched_class->set_cpus_allowed(p, new_mask);
         else {
@@ -5964,10 +5885,10 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
         double_rq_lock(rq_src, rq_dest);
         /* Already moved. */
         if (task_cpu(p) != src_cpu)
-                goto out;
+                goto done;
         /* Affinity changed (again). */
         if (!cpu_isset(dest_cpu, p->cpus_allowed))
-                goto out;
+                goto fail;
 
         on_rq = p->se.on_rq;
         if (on_rq)
@@ -5978,8 +5899,9 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
                 activate_task(rq_dest, p, 0);
                 check_preempt_curr(rq_dest, p);
         }
+done:
         ret = 1;
-out:
+fail:
         double_rq_unlock(rq_src, rq_dest);
         return ret;
 }
@@ -6229,6 +6151,7 @@ static void migrate_dead_tasks(unsigned int dead_cpu)
                 next = pick_next_task(rq, rq->curr);
                 if (!next)
                         break;
+                next->sched_class->put_prev_task(rq, next);
                 migrate_dead(dead_cpu, next);
 
         }
@@ -6400,6 +6323,36 @@ static void unregister_sched_domain_sysctl(void)
 }
 #endif
 
+static void set_rq_online(struct rq *rq)
+{
+        if (!rq->online) {
+                const struct sched_class *class;
+
+                cpu_set(rq->cpu, rq->rd->online);
+                rq->online = 1;
+
+                for_each_class(class) {
+                        if (class->rq_online)
+                                class->rq_online(rq);
+                }
+        }
+}
+
+static void set_rq_offline(struct rq *rq)
+{
+        if (rq->online) {
+                const struct sched_class *class;
+
+                for_each_class(class) {
+                        if (class->rq_offline)
+                                class->rq_offline(rq);
+                }
+
+                cpu_clear(rq->cpu, rq->rd->online);
+                rq->online = 0;
+        }
+}
+
 /*
  * migration_call - callback that gets triggered when a CPU is added.
  * Here we can start up the necessary migration thread for the new CPU.
@@ -6437,7 +6390,8 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 spin_lock_irqsave(&rq->lock, flags);
                 if (rq->rd) {
                         BUG_ON(!cpu_isset(cpu, rq->rd->span));
-                        cpu_set(cpu, rq->rd->online);
+
+                        set_rq_online(rq);
                 }
                 spin_unlock_irqrestore(&rq->lock, flags);
                 break;
@@ -6498,7 +6452,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 spin_lock_irqsave(&rq->lock, flags);
                 if (rq->rd) {
                         BUG_ON(!cpu_isset(cpu, rq->rd->span));
-                        cpu_clear(cpu, rq->rd->online);
+                        set_rq_offline(rq);
                 }
                 spin_unlock_irqrestore(&rq->lock, flags);
                 break;
@@ -6532,6 +6486,28 @@ void __init migration_init(void)
 
 #ifdef CONFIG_SCHED_DEBUG
 
+static inline const char *sd_level_to_string(enum sched_domain_level lvl)
+{
+        switch (lvl) {
+        case SD_LV_NONE:
+                        return "NONE";
+        case SD_LV_SIBLING:
+                        return "SIBLING";
+        case SD_LV_MC:
+                        return "MC";
+        case SD_LV_CPU:
+                        return "CPU";
+        case SD_LV_NODE:
+                        return "NODE";
+        case SD_LV_ALLNODES:
+                        return "ALLNODES";
+        case SD_LV_MAX:
+                        return "MAX";
+
+        }
+        return "MAX";
+}
+
 static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                                   cpumask_t *groupmask)
 {
@@ -6551,7 +6527,8 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                 return -1;
         }
 
-        printk(KERN_CONT "span %s\n", str);
+        printk(KERN_CONT "span %s level %s\n",
+                str, sd_level_to_string(sd->level));
 
         if (!cpu_isset(cpu, sd->span)) {
                 printk(KERN_ERR "ERROR: domain->span does not contain "
@@ -6635,9 +6612,9 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
         }
         kfree(groupmask);
 }
-#else
+#else /* !CONFIG_SCHED_DEBUG */
 # define sched_domain_debug(sd, cpu) do { } while (0)
-#endif
+#endif /* CONFIG_SCHED_DEBUG */
 
 static int sd_degenerate(struct sched_domain *sd)
 {
@@ -6697,20 +6674,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
 static void rq_attach_root(struct rq *rq, struct root_domain *rd)
 {
         unsigned long flags;
-        const struct sched_class *class;
 
         spin_lock_irqsave(&rq->lock, flags);
 
         if (rq->rd) {
                 struct root_domain *old_rd = rq->rd;
 
-                for (class = sched_class_highest; class; class = class->next) {
-                        if (class->leave_domain)
-                                class->leave_domain(rq);
-                }
+                if (cpu_isset(rq->cpu, old_rd->online))
+                        set_rq_offline(rq);
 
                 cpu_clear(rq->cpu, old_rd->span);
-                cpu_clear(rq->cpu, old_rd->online);
 
                 if (atomic_dec_and_test(&old_rd->refcount))
                         kfree(old_rd);
@@ -6721,12 +6694,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
 
         cpu_set(rq->cpu, rd->span);
         if (cpu_isset(rq->cpu, cpu_online_map))
-                cpu_set(rq->cpu, rd->online);
-
-        for (class = sched_class_highest; class; class = class->next) {
-                if (class->join_domain)
-                        class->join_domain(rq);
-        }
+                set_rq_online(rq);
 
         spin_unlock_irqrestore(&rq->lock, flags);
 }
@@ -6737,6 +6705,8 @@ static void init_rootdomain(struct root_domain *rd)
 
         cpus_clear(rd->span);
         cpus_clear(rd->online);
+
+        cpupri_init(&rd->cpupri);
 }
 
 static void init_defrootdomain(void)
@@ -6879,9 +6849,9 @@ static int find_next_best_node(int node, nodemask_t *used_nodes)
 
         min_val = INT_MAX;
 
-        for (i = 0; i < MAX_NUMNODES; i++) {
+        for (i = 0; i < nr_node_ids; i++) {
                 /* Start at @node */
-                n = (node + i) % MAX_NUMNODES;
+                n = (node + i) % nr_node_ids;
 
                 if (!nr_cpus_node(n))
                         continue;
@@ -6931,7 +6901,7 @@ static void sched_domain_node_span(int node, cpumask_t *span)
                 cpus_or(*span, *span, *nodemask);
         }
 }
-#endif
+#endif /* CONFIG_NUMA */
 
 int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
 
@@ -6950,7 +6920,7 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
                 *sg = &per_cpu(sched_group_cpus, cpu);
         return cpu;
 }
-#endif
+#endif /* CONFIG_SCHED_SMT */
 
 /*
  * multi-core sched-domains:
@@ -6958,7 +6928,7 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
 #ifdef CONFIG_SCHED_MC
 static DEFINE_PER_CPU(struct sched_domain, core_domains);
 static DEFINE_PER_CPU(struct sched_group, sched_group_core);
-#endif
+#endif /* CONFIG_SCHED_MC */
 
 #if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
 static int
@@ -7060,7 +7030,7 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
                 sg = sg->next;
         } while (sg != group_head);
 }
-#endif
+#endif /* CONFIG_NUMA */
 
 #ifdef CONFIG_NUMA
 /* Free memory allocated for various sched_group structures */
@@ -7075,7 +7045,7 @@ static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
                 if (!sched_group_nodes)
                         continue;
 
-                for (i = 0; i < MAX_NUMNODES; i++) {
+                for (i = 0; i < nr_node_ids; i++) {
                         struct sched_group *oldsg, *sg = sched_group_nodes[i];
 
                         *nodemask = node_to_cpumask(i);
@@ -7097,11 +7067,11 @@ next_sg:
                 sched_group_nodes_bycpu[cpu] = NULL;
         }
 }
-#else
+#else /* !CONFIG_NUMA */
 static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
 {
 }
-#endif
+#endif /* CONFIG_NUMA */
 
 /*
  * Initialize sched groups cpu_power.
@@ -7219,7 +7189,12 @@ static int default_relax_domain_level = -1;
 
 static int __init setup_relax_domain_level(char *str)
 {
-        default_relax_domain_level = simple_strtoul(str, NULL, 0);
+        unsigned long val;
+
+        val = simple_strtoul(str, NULL, 0);
+        if (val < SD_LV_MAX)
+                default_relax_domain_level = val;
+
         return 1;
 }
 __setup("relax_domain_level=", setup_relax_domain_level);
@@ -7263,7 +7238,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
         /*
          * Allocate the per-node list of sched groups
          */
-        sched_group_nodes = kcalloc(MAX_NUMNODES, sizeof(struct sched_group *),
+        sched_group_nodes = kcalloc(nr_node_ids, sizeof(struct sched_group *),
                                     GFP_KERNEL);
         if (!sched_group_nodes) {
                 printk(KERN_WARNING "Can not alloc sched group node list\n");
@@ -7316,7 +7291,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
                         SD_INIT(sd, ALLNODES);
                         set_domain_attribute(sd, attr);
                         sd->span = *cpu_map;
-                        sd->first_cpu = first_cpu(sd->span);
                         cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
                         p = sd;
                         sd_allnodes = 1;
@@ -7327,7 +7301,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
                 SD_INIT(sd, NODE);
                 set_domain_attribute(sd, attr);
                 sched_domain_node_span(cpu_to_node(i), &sd->span);
-                sd->first_cpu = first_cpu(sd->span);
                 sd->parent = p;
                 if (p)
                         p->child = sd;
@@ -7339,7 +7312,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
                 SD_INIT(sd, CPU);
                 set_domain_attribute(sd, attr);
                 sd->span = *nodemask;
-                sd->first_cpu = first_cpu(sd->span);
                 sd->parent = p;
                 if (p)
                         p->child = sd;
@@ -7351,7 +7323,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
                 SD_INIT(sd, MC);
                 set_domain_attribute(sd, attr);
                 sd->span = cpu_coregroup_map(i);
-                sd->first_cpu = first_cpu(sd->span);
                 cpus_and(sd->span, sd->span, *cpu_map);
                 sd->parent = p;
                 p->child = sd;
@@ -7364,7 +7335,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
                 SD_INIT(sd, SIBLING);
                 set_domain_attribute(sd, attr);
                 sd->span = per_cpu(cpu_sibling_map, i);
-                sd->first_cpu = first_cpu(sd->span);
                 cpus_and(sd->span, sd->span, *cpu_map);
                 sd->parent = p;
                 p->child = sd;
@@ -7407,7 +7377,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
 #endif
 
         /* Set up physical groups */
-        for (i = 0; i < MAX_NUMNODES; i++) {
+        for (i = 0; i < nr_node_ids; i++) {
                 SCHED_CPUMASK_VAR(nodemask, allmasks);
                 SCHED_CPUMASK_VAR(send_covered, allmasks);
 
@@ -7431,7 +7401,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
                                         send_covered, tmpmask);
         }
 
-        for (i = 0; i < MAX_NUMNODES; i++) {
+        for (i = 0; i < nr_node_ids; i++) {
                 /* Set up node groups */
                 struct sched_group *sg, *prev;
                 SCHED_CPUMASK_VAR(nodemask, allmasks);
@@ -7470,9 +7440,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
                 cpus_or(*covered, *covered, *nodemask);
                 prev = sg;
 
-                for (j = 0; j < MAX_NUMNODES; j++) {
+                for (j = 0; j < nr_node_ids; j++) {
                         SCHED_CPUMASK_VAR(notcovered, allmasks);
-                        int n = (i + j) % MAX_NUMNODES;
+                        int n = (i + j) % nr_node_ids;
                         node_to_cpumask_ptr(pnodemask, n);
 
                         cpus_complement(*notcovered, *covered);
@@ -7525,7 +7495,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
         }
 
 #ifdef CONFIG_NUMA
-        for (i = 0; i < MAX_NUMNODES; i++)
+        for (i = 0; i < nr_node_ids; i++)
                 init_numa_sched_groups_power(sched_group_nodes[i]);
 
         if (sd_allnodes) {
@@ -7568,8 +7538,8 @@ static int build_sched_domains(const cpumask_t *cpu_map)
 
 static cpumask_t *doms_cur;     /* current sched domains */
 static int ndoms_cur;           /* number of sched domains in 'doms_cur' */
-static struct sched_domain_attr *dattr_cur;     /* attribues of custom domains
-                                                   in 'doms_cur' */
+static struct sched_domain_attr *dattr_cur;
+                                /* attribues of custom domains in 'doms_cur' */
 
 /*
  * Special case: If a kmalloc of a doms_cur partition (array of
@@ -7583,6 +7553,18 @@ void __attribute__((weak)) arch_update_cpu_topology(void)
 }
 
 /*
+ * Free current domain masks.
+ * Called after all cpus are attached to NULL domain.
+ */
+static void free_sched_domains(void)
+{
+        ndoms_cur = 0;
+        if (doms_cur != &fallback_doms)
+                kfree(doms_cur);
+        doms_cur = &fallback_doms;
+}
+
+/*
  * Set up scheduler domains and groups. Callers must hold the hotplug lock.
  * For now this just excludes isolated cpus, but could be used to
  * exclude other special cases in the future.
@@ -7729,6 +7711,7 @@ int arch_reinit_sched_domains(void)
         get_online_cpus();
         mutex_lock(&sched_domains_mutex);
         detach_destroy_domains(&cpu_online_map);
+        free_sched_domains();
         err = arch_init_sched_domains(&cpu_online_map);
         mutex_unlock(&sched_domains_mutex);
         put_online_cpus();
@@ -7797,7 +7780,7 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
 #endif
         return err;
 }
-#endif
+#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
 
 /*
  * Force a reinitialization of the sched domains hierarchy. The domains
@@ -7808,20 +7791,28 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
 static int update_sched_domains(struct notifier_block *nfb,
                                 unsigned long action, void *hcpu)
 {
+        int cpu = (int)(long)hcpu;
+
         switch (action) {
-        case CPU_UP_PREPARE:
-        case CPU_UP_PREPARE_FROZEN:
         case CPU_DOWN_PREPARE:
         case CPU_DOWN_PREPARE_FROZEN:
+                disable_runtime(cpu_rq(cpu));
+                /* fall-through */
+        case CPU_UP_PREPARE:
+        case CPU_UP_PREPARE_FROZEN:
                 detach_destroy_domains(&cpu_online_map);
+                free_sched_domains();
                 return NOTIFY_OK;
 
-        case CPU_UP_CANCELED:
-        case CPU_UP_CANCELED_FROZEN:
+
         case CPU_DOWN_FAILED:
         case CPU_DOWN_FAILED_FROZEN:
         case CPU_ONLINE:
         case CPU_ONLINE_FROZEN:
+                enable_runtime(cpu_rq(cpu));
+                /* fall-through */
+        case CPU_UP_CANCELED:
+        case CPU_UP_CANCELED_FROZEN:
         case CPU_DEAD:
         case CPU_DEAD_FROZEN:
                 /*
@@ -7832,8 +7823,16 @@ static int update_sched_domains(struct notifier_block *nfb,
                 return NOTIFY_DONE;
         }
 
+#ifndef CONFIG_CPUSETS
+        /*
+         * Create default domain partitioning if cpusets are disabled.
+         * Otherwise we let cpusets rebuild the domains based on the
+         * current setup.
+         */
+
         /* The hotplug lock is already held by cpu_up/cpu_down */
         arch_init_sched_domains(&cpu_online_map);
+#endif
 
         return NOTIFY_OK;
 }
@@ -7973,7 +7972,6 @@ static void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
         else
                 rt_se->rt_rq = parent->my_q;
 
-        rt_se->rt_rq = &rq->rt;
         rt_se->my_q = rt_rq;
         rt_se->parent = parent;
         INIT_LIST_HEAD(&rt_se->run_list);
@@ -8014,8 +8012,8 @@ void __init sched_init(void)
 
                 root_task_group.cfs_rq = (struct cfs_rq **)ptr;
                 ptr += nr_cpu_ids * sizeof(void **);
-#endif
-#endif
+#endif /* CONFIG_USER_SCHED */
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 #ifdef CONFIG_RT_GROUP_SCHED
                 init_task_group.rt_se = (struct sched_rt_entity **)ptr;
                 ptr += nr_cpu_ids * sizeof(void **);
@@ -8029,12 +8027,11 @@ void __init sched_init(void)
 
                 root_task_group.rt_rq = (struct rt_rq **)ptr;
                 ptr += nr_cpu_ids * sizeof(void **);
-#endif
-#endif
+#endif /* CONFIG_USER_SCHED */
+#endif /* CONFIG_RT_GROUP_SCHED */
         }
 
 #ifdef CONFIG_SMP
-        init_aggregate();
         init_defrootdomain();
 #endif
 
@@ -8047,8 +8044,8 @@ void __init sched_init(void)
 #ifdef CONFIG_USER_SCHED
         init_rt_bandwidth(&root_task_group.rt_bandwidth,
                         global_rt_period(), RUNTIME_INF);
-#endif
-#endif
+#endif /* CONFIG_USER_SCHED */
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 #ifdef CONFIG_GROUP_SCHED
         list_add(&init_task_group.list, &task_groups);
@@ -8058,8 +8055,8 @@ void __init sched_init(void)
         INIT_LIST_HEAD(&root_task_group.children);
         init_task_group.parent = &root_task_group;
         list_add(&init_task_group.siblings, &root_task_group.children);
-#endif
-#endif
+#endif /* CONFIG_USER_SCHED */
+#endif /* CONFIG_GROUP_SCHED */
 
         for_each_possible_cpu(i) {
                 struct rq *rq;
@@ -8139,6 +8136,7 @@ void __init sched_init(void)
                 rq->next_balance = jiffies;
                 rq->push_cpu = 0;
                 rq->cpu = i;
+                rq->online = 0;
                 rq->migration_thread = NULL;
                 INIT_LIST_HEAD(&rq->migration_queue);
                 rq_attach_root(rq, &def_root_domain);
@@ -8154,7 +8152,7 @@ void __init sched_init(void)
 #endif
 
 #ifdef CONFIG_SMP
-        open_softirq(SCHED_SOFTIRQ, run_rebalance_domains, NULL);
+        open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
 #endif
 
 #ifdef CONFIG_RT_MUTEXES
@@ -8378,7 +8376,7 @@ static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
 {
         list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list);
 }
-#else
+#else /* !CONFG_FAIR_GROUP_SCHED */
 static inline void free_fair_sched_group(struct task_group *tg)
 {
 }
@@ -8396,7 +8394,7 @@ static inline void register_fair_sched_group(struct task_group *tg, int cpu)
 static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
 {
 }
-#endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static void free_rt_sched_group(struct task_group *tg)
@@ -8467,7 +8465,7 @@ static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
 {
         list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list);
 }
-#else
+#else /* !CONFIG_RT_GROUP_SCHED */
 static inline void free_rt_sched_group(struct task_group *tg)
 {
 }
@@ -8485,7 +8483,7 @@ static inline void register_rt_sched_group(struct task_group *tg, int cpu)
 static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
 {
 }
-#endif
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 #ifdef CONFIG_GROUP_SCHED
 static void free_sched_group(struct task_group *tg)
@@ -8596,7 +8594,7 @@ void sched_move_task(struct task_struct *tsk)
 
         task_rq_unlock(rq, &flags);
 }
-#endif
+#endif /* CONFIG_GROUP_SCHED */
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
 static void __set_se_shares(struct sched_entity *se, unsigned long shares)
@@ -8731,7 +8729,7 @@ static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
         }
         rcu_read_unlock();
 
-        return total + to_ratio(period, runtime) <
+        return total + to_ratio(period, runtime) <=
                 to_ratio(ktime_to_ns(parent->rt_bandwidth.rt_period),
                                 parent->rt_bandwidth.rt_runtime);
 }
@@ -8834,6 +8832,9 @@ int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
         rt_period = (u64)rt_period_us * NSEC_PER_USEC;
         rt_runtime = tg->rt_bandwidth.rt_runtime;
 
+        if (rt_period == 0)
+                return -EINVAL;
+
         return tg_set_bandwidth(tg, rt_period, rt_runtime);
 }
 
@@ -8848,16 +8849,21 @@ long sched_group_rt_period(struct task_group *tg)
 
 static int sched_rt_global_constraints(void)
 {
+        struct task_group *tg = &root_task_group;
+        u64 rt_runtime, rt_period;
         int ret = 0;
 
+        rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
+        rt_runtime = tg->rt_bandwidth.rt_runtime;
+
         mutex_lock(&rt_constraints_mutex);
-        if (!__rt_schedulable(NULL, 1, 0))
+        if (!__rt_schedulable(tg, rt_period, rt_runtime))
                 ret = -EINVAL;
         mutex_unlock(&rt_constraints_mutex);
 
         return ret;
 }
-#else
+#else /* !CONFIG_RT_GROUP_SCHED */
 static int sched_rt_global_constraints(void)
 {
         unsigned long flags;
@@ -8875,7 +8881,7 @@ static int sched_rt_global_constraints(void)
 
         return 0;
 }
-#endif
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 int sched_rt_handler(struct ctl_table *table, int write,
                 struct file *filp, void __user *buffer, size_t *lenp,
@@ -8983,7 +8989,7 @@ static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
 
         return (u64) tg->shares;
 }
-#endif
+#endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
@@ -9007,7 +9013,7 @@ static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
 {
         return sched_group_rt_period(cgroup_tg(cgrp));
 }
-#endif
+#endif /* CONFIG_RT_GROUP_SCHED */
 
 static struct cftype cpu_files[] = {
 #ifdef CONFIG_FAIR_GROUP_SCHED