1 files changed, 839 insertions, 642 deletions

diff --git a/kernel/sched.c b/kernel/sched.c
index e4bb1dd7b308..deb5ac8c12f3 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -118,6 +118,12 @@
  */
 #define RUNTIME_INF     ((u64)~0ULL)
 
+DEFINE_TRACE(sched_wait_task);
+DEFINE_TRACE(sched_wakeup);
+DEFINE_TRACE(sched_wakeup_new);
+DEFINE_TRACE(sched_switch);
+DEFINE_TRACE(sched_migrate_task);
+
 #ifdef CONFIG_SMP
 /*
  * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
@@ -203,7 +209,6 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
         hrtimer_init(&rt_b->rt_period_timer,
                         CLOCK_MONOTONIC, HRTIMER_MODE_REL);
         rt_b->rt_period_timer.function = sched_rt_period_timer;
-        rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;
 }
 
 static inline int rt_bandwidth_enabled(void)
@@ -261,6 +266,10 @@ struct task_group {
         struct cgroup_subsys_state css;
 #endif
 
+#ifdef CONFIG_USER_SCHED
+        uid_t uid;
+#endif
+
 #ifdef CONFIG_FAIR_GROUP_SCHED
         /* schedulable entities of this group on each cpu */
         struct sched_entity **se;
@@ -286,6 +295,12 @@ struct task_group {
 
 #ifdef CONFIG_USER_SCHED
 
+/* Helper function to pass uid information to create_sched_user() */
+void set_tg_uid(struct user_struct *user)
+{
+        user->tg->uid = user->uid;
+}
+
 /*
  * Root task group.
  *      Every UID task group (including init_task_group aka UID-0) will
@@ -345,7 +360,9 @@ static inline struct task_group *task_group(struct task_struct *p)
         struct task_group *tg;
 
 #ifdef CONFIG_USER_SCHED
-        tg = p->user->tg;
+        rcu_read_lock();
+        tg = __task_cred(p)->user->tg;
+        rcu_read_unlock();
 #elif defined(CONFIG_CGROUP_SCHED)
         tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
                                 struct task_group, css);
@@ -481,18 +498,26 @@ struct rt_rq {
  */
 struct root_domain {
         atomic_t refcount;
-        cpumask_t span;
-        cpumask_t online;
+        cpumask_var_t span;
+        cpumask_var_t online;
 
         /*
          * The "RT overload" flag: it gets set if a CPU has more than
          * one runnable RT task.
          */
-        cpumask_t rto_mask;
+        cpumask_var_t rto_mask;
         atomic_t rto_count;
 #ifdef CONFIG_SMP
         struct cpupri cpupri;
 #endif
+#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
+        /*
+         * Preferred wake up cpu nominated by sched_mc balance that will be
+         * used when most cpus are idle in the system indicating overall very
+         * low system utilisation. Triggered at POWERSAVINGS_BALANCE_WAKEUP(2)
+         */
+        unsigned int sched_mc_preferred_wakeup_cpu;
+#endif
 };
 
 /*
@@ -586,6 +611,8 @@ struct rq {
 #ifdef CONFIG_SCHEDSTATS
         /* latency stats */
         struct sched_info rq_sched_info;
+        unsigned long long rq_cpu_time;
+        /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
 
         /* sys_sched_yield() stats */
         unsigned int yld_exp_empty;
@@ -703,45 +730,18 @@ static __read_mostly char *sched_feat_names[] = {
 
 #undef SCHED_FEAT
 
-static int sched_feat_open(struct inode *inode, struct file *filp)
+static int sched_feat_show(struct seq_file *m, void *v)
 {
-        filp->private_data = inode->i_private;
-        return 0;
-}
-
-static ssize_t
-sched_feat_read(struct file *filp, char __user *ubuf,
-                size_t cnt, loff_t *ppos)
-{
-        char *buf;
-        int r = 0;
-        int len = 0;
         int i;
 
         for (i = 0; sched_feat_names[i]; i++) {
-                len += strlen(sched_feat_names[i]);
-                len += 4;
-        }
-
-        buf = kmalloc(len + 2, GFP_KERNEL);
-        if (!buf)
-                return -ENOMEM;
-
-        for (i = 0; sched_feat_names[i]; i++) {
-                if (sysctl_sched_features & (1UL << i))
-                        r += sprintf(buf + r, "%s ", sched_feat_names[i]);
-                else
-                        r += sprintf(buf + r, "NO_%s ", sched_feat_names[i]);
+                if (!(sysctl_sched_features & (1UL << i)))
+                        seq_puts(m, "NO_");
+                seq_printf(m, "%s ", sched_feat_names[i]);
         }
+        seq_puts(m, "\n");
 
-        r += sprintf(buf + r, "\n");
-        WARN_ON(r >= len + 2);
-
-        r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
-
-        kfree(buf);
-
-        return r;
+        return 0;
 }
 
 static ssize_t
@@ -786,10 +786,17 @@ sched_feat_write(struct file *filp, const char __user *ubuf,
         return cnt;
 }
 
+static int sched_feat_open(struct inode *inode, struct file *filp)
+{
+        return single_open(filp, sched_feat_show, NULL);
+}
+
 static struct file_operations sched_feat_fops = {
-        .open   = sched_feat_open,
-        .read   = sched_feat_read,
-        .write  = sched_feat_write,
+        .open           = sched_feat_open,
+        .write          = sched_feat_write,
+        .read           = seq_read,
+        .llseek         = seq_lseek,
+        .release        = single_release,
 };
 
 static __init int sched_init_debug(void)
@@ -1139,7 +1146,6 @@ static void init_rq_hrtick(struct rq *rq)
 
         hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
         rq->hrtick_timer.function = hrtick;
-        rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
 }
 #else   /* CONFIG_SCHED_HRTICK */
 static inline void hrtick_clear(struct rq *rq)
@@ -1474,27 +1480,13 @@ static void
 update_group_shares_cpu(struct task_group *tg, int cpu,
                         unsigned long sd_shares, unsigned long sd_rq_weight)
 {
-        int boost = 0;
         unsigned long shares;
         unsigned long rq_weight;
 
         if (!tg->se[cpu])
                 return;
 
-        rq_weight = tg->cfs_rq[cpu]->load.weight;
-
-        /*
-         * If there are currently no tasks on the cpu pretend there is one of
-         * average load so that when a new task gets to run here it will not
-         * get delayed by group starvation.
-         */
-        if (!rq_weight) {
-                boost = 1;
-                rq_weight = NICE_0_LOAD;
-        }
-
-        if (unlikely(rq_weight > sd_rq_weight))
-                rq_weight = sd_rq_weight;
+        rq_weight = tg->cfs_rq[cpu]->rq_weight;
 
         /*
          *           \Sum shares * rq_weight
@@ -1502,7 +1494,7 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
          *               \Sum rq_weight
          *
          */
-        shares = (sd_shares * rq_weight) / (sd_rq_weight + 1);
+        shares = (sd_shares * rq_weight) / sd_rq_weight;
         shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
 
         if (abs(shares - tg->se[cpu]->load.weight) >
@@ -1511,11 +1503,7 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
                 unsigned long flags;
 
                 spin_lock_irqsave(&rq->lock, flags);
-                /*
-                 * record the actual number of shares, not the boosted amount.
-                 */
-                tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
-                tg->cfs_rq[cpu]->rq_weight = rq_weight;
+                tg->cfs_rq[cpu]->shares = shares;
 
                 __set_se_shares(tg->se[cpu], shares);
                 spin_unlock_irqrestore(&rq->lock, flags);
@@ -1529,13 +1517,23 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
  */
 static int tg_shares_up(struct task_group *tg, void *data)
 {
-        unsigned long rq_weight = 0;
+        unsigned long weight, rq_weight = 0;
         unsigned long shares = 0;
         struct sched_domain *sd = data;
         int i;
 
-        for_each_cpu_mask(i, sd->span) {
-                rq_weight += tg->cfs_rq[i]->load.weight;
+        for_each_cpu(i, sched_domain_span(sd)) {
+                /*
+                 * If there are currently no tasks on the cpu pretend there
+                 * is one of average load so that when a new task gets to
+                 * run here it will not get delayed by group starvation.
+                 */
+                weight = tg->cfs_rq[i]->load.weight;
+                if (!weight)
+                        weight = NICE_0_LOAD;
+
+                tg->cfs_rq[i]->rq_weight = weight;
+                rq_weight += weight;
                 shares += tg->cfs_rq[i]->shares;
         }
 
@@ -1545,10 +1543,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
         if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
                 shares = tg->shares;
 
-        if (!rq_weight)
-                rq_weight = cpus_weight(sd->span) * NICE_0_LOAD;
-
-        for_each_cpu_mask(i, sd->span)
+        for_each_cpu(i, sched_domain_span(sd))
                 update_group_shares_cpu(tg, i, shares, rq_weight);
 
         return 0;
@@ -1612,6 +1607,39 @@ static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
 
 #endif
 
+/*
+ * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
+ */
+static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
+        __releases(this_rq->lock)
+        __acquires(busiest->lock)
+        __acquires(this_rq->lock)
+{
+        int ret = 0;
+
+        if (unlikely(!irqs_disabled())) {
+                /* printk() doesn't work good under rq->lock */
+                spin_unlock(&this_rq->lock);
+                BUG_ON(1);
+        }
+        if (unlikely(!spin_trylock(&busiest->lock))) {
+                if (busiest < this_rq) {
+                        spin_unlock(&this_rq->lock);
+                        spin_lock(&busiest->lock);
+                        spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
+                        ret = 1;
+                } else
+                        spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
+        }
+        return ret;
+}
+
+static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
+        __releases(busiest->lock)
+{
+        spin_unlock(&busiest->lock);
+        lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
+}
 #endif
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
@@ -1845,6 +1873,8 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
 
         clock_offset = old_rq->clock - new_rq->clock;
 
+        trace_sched_migrate_task(p, task_cpu(p), new_cpu);
+
 #ifdef CONFIG_SCHEDSTATS
         if (p->se.wait_start)
                 p->se.wait_start -= clock_offset;
@@ -2079,15 +2109,17 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
                 int i;
 
                 /* Skip over this group if it has no CPUs allowed */
-                if (!cpus_intersects(group->cpumask, p->cpus_allowed))
+                if (!cpumask_intersects(sched_group_cpus(group),
+                                        &p->cpus_allowed))
                         continue;
 
-                local_group = cpu_isset(this_cpu, group->cpumask);
+                local_group = cpumask_test_cpu(this_cpu,
+                                               sched_group_cpus(group));
 
                 /* Tally up the load of all CPUs in the group */
                 avg_load = 0;
 
-                for_each_cpu_mask_nr(i, group->cpumask) {
+                for_each_cpu(i, sched_group_cpus(group)) {
                         /* Bias balancing toward cpus of our domain */
                         if (local_group)
                                 load = source_load(i, load_idx);
@@ -2119,17 +2151,14 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
  * find_idlest_cpu - find the idlest cpu among the cpus in group.
  */
 static int
-find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu,
-                cpumask_t *tmp)
+find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
 {
         unsigned long load, min_load = ULONG_MAX;
         int idlest = -1;
         int i;
 
         /* Traverse only the allowed CPUs */
-        cpus_and(*tmp, group->cpumask, p->cpus_allowed);
-
-        for_each_cpu_mask_nr(i, *tmp) {
+        for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
                 load = weighted_cpuload(i);
 
                 if (load < min_load || (load == min_load && i == this_cpu)) {
@@ -2171,7 +2200,6 @@ static int sched_balance_self(int cpu, int flag)
                 update_shares(sd);
 
         while (sd) {
-                cpumask_t span, tmpmask;
                 struct sched_group *group;
                 int new_cpu, weight;
 
@@ -2180,14 +2208,13 @@ static int sched_balance_self(int cpu, int flag)
                         continue;
                 }
 
-                span = sd->span;
                 group = find_idlest_group(sd, t, cpu);
                 if (!group) {
                         sd = sd->child;
                         continue;
                 }
 
-                new_cpu = find_idlest_cpu(group, t, cpu, &tmpmask);
+                new_cpu = find_idlest_cpu(group, t, cpu);
                 if (new_cpu == -1 || new_cpu == cpu) {
                         /* Now try balancing at a lower domain level of cpu */
                         sd = sd->child;
@@ -2196,10 +2223,10 @@ static int sched_balance_self(int cpu, int flag)
 
                 /* Now try balancing at a lower domain level of new_cpu */
                 cpu = new_cpu;
+                weight = cpumask_weight(sched_domain_span(sd));
                 sd = NULL;
-                weight = cpus_weight(span);
                 for_each_domain(cpu, tmp) {
-                        if (weight <= cpus_weight(tmp->span))
+                        if (weight <= cpumask_weight(sched_domain_span(tmp)))
                                 break;
                         if (tmp->flags & flag)
                                 sd = tmp;
@@ -2244,7 +2271,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
                 cpu = task_cpu(p);
 
                 for_each_domain(this_cpu, sd) {
-                        if (cpu_isset(cpu, sd->span)) {
+                        if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
                                 update_shares(sd);
                                 break;
                         }
@@ -2254,6 +2281,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
 
         smp_wmb();
         rq = task_rq_lock(p, &flags);
+        update_rq_clock(rq);
         old_state = p->state;
         if (!(old_state & state))
                 goto out;
@@ -2292,7 +2320,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
         else {
                 struct sched_domain *sd;
                 for_each_domain(this_cpu, sd) {
-                        if (cpu_isset(cpu, sd->span)) {
+                        if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
                                 schedstat_inc(sd, ttwu_wake_remote);
                                 break;
                         }
@@ -2311,12 +2339,11 @@ out_activate:
                 schedstat_inc(p, se.nr_wakeups_local);
         else
                 schedstat_inc(p, se.nr_wakeups_remote);
-        update_rq_clock(rq);
         activate_task(rq, p, 1);
         success = 1;
 
 out_running:
-        trace_sched_wakeup(rq, p);
+        trace_sched_wakeup(rq, p, success);
         check_preempt_curr(rq, p, sync);
 
         p->state = TASK_RUNNING;
@@ -2449,7 +2476,7 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
                 p->sched_class->task_new(rq, p);
                 inc_nr_running(rq);
         }
-        trace_sched_wakeup_new(rq, p);
+        trace_sched_wakeup_new(rq, p, 1);
         check_preempt_curr(rq, p, 0);
 #ifdef CONFIG_SMP
         if (p->sched_class->task_wake_up)
@@ -2812,40 +2839,6 @@ static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
 }
 
 /*
- * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
- */
-static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
-        __releases(this_rq->lock)
-        __acquires(busiest->lock)
-        __acquires(this_rq->lock)
-{
-        int ret = 0;
-
-        if (unlikely(!irqs_disabled())) {
-                /* printk() doesn't work good under rq->lock */
-                spin_unlock(&this_rq->lock);
-                BUG_ON(1);
-        }
-        if (unlikely(!spin_trylock(&busiest->lock))) {
-                if (busiest < this_rq) {
-                        spin_unlock(&this_rq->lock);
-                        spin_lock(&busiest->lock);
-                        spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
-                        ret = 1;
-                } else
-                        spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
-        }
-        return ret;
-}
-
-static void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
-        __releases(busiest->lock)
-{
-        spin_unlock(&busiest->lock);
-        lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
-}
-
-/*
  * If dest_cpu is allowed for this process, migrate the task to it.
  * This is accomplished by forcing the cpu_allowed mask to only
  * allow dest_cpu, which will force the cpu onto dest_cpu. Then
@@ -2858,11 +2851,10 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu)
         struct rq *rq;
 
         rq = task_rq_lock(p, &flags);
-        if (!cpu_isset(dest_cpu, p->cpus_allowed)
+        if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
             || unlikely(!cpu_active(dest_cpu)))
                 goto out;
 
-        trace_sched_migrate_task(rq, p, dest_cpu);
         /* force the process onto the specified CPU */
         if (migrate_task(p, dest_cpu, &req)) {
                 /* Need to wait for migration thread (might exit: take ref). */
@@ -2924,7 +2916,7 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
          * 2) cannot be migrated to this CPU due to cpus_allowed, or
          * 3) are cache-hot on their current CPU.
          */
-        if (!cpu_isset(this_cpu, p->cpus_allowed)) {
+        if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
                 schedstat_inc(p, se.nr_failed_migrations_affine);
                 return 0;
         }
@@ -3099,7 +3091,7 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
 static struct sched_group *
 find_busiest_group(struct sched_domain *sd, int this_cpu,
                    unsigned long *imbalance, enum cpu_idle_type idle,
-                   int *sd_idle, const cpumask_t *cpus, int *balance)
+                   int *sd_idle, const struct cpumask *cpus, int *balance)
 {
         struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
         unsigned long max_load, avg_load, total_load, this_load, total_pwr;
@@ -3135,10 +3127,11 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                 unsigned long sum_avg_load_per_task;
                 unsigned long avg_load_per_task;
 
-                local_group = cpu_isset(this_cpu, group->cpumask);
+                local_group = cpumask_test_cpu(this_cpu,
+                                               sched_group_cpus(group));
 
                 if (local_group)
-                        balance_cpu = first_cpu(group->cpumask);
+                        balance_cpu = cpumask_first(sched_group_cpus(group));
 
                 /* Tally up the load of all CPUs in the group */
                 sum_weighted_load = sum_nr_running = avg_load = 0;
@@ -3147,13 +3140,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                 max_cpu_load = 0;
                 min_cpu_load = ~0UL;
 
-                for_each_cpu_mask_nr(i, group->cpumask) {
-                        struct rq *rq;
-
-                        if (!cpu_isset(i, *cpus))
-                                continue;
-
-                        rq = cpu_rq(i);
+                for_each_cpu_and(i, sched_group_cpus(group), cpus) {
+                        struct rq *rq = cpu_rq(i);
 
                         if (*sd_idle && rq->nr_running)
                                 *sd_idle = 0;
@@ -3264,8 +3252,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                  */
                 if ((sum_nr_running < min_nr_running) ||
                     (sum_nr_running == min_nr_running &&
-                     first_cpu(group->cpumask) <
-                     first_cpu(group_min->cpumask))) {
+                     cpumask_first(sched_group_cpus(group)) >
+                     cpumask_first(sched_group_cpus(group_min)))) {
                         group_min = group;
                         min_nr_running = sum_nr_running;
                         min_load_per_task = sum_weighted_load /
@@ -3280,8 +3268,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
                 if (sum_nr_running <= group_capacity - 1) {
                         if (sum_nr_running > leader_nr_running ||
                             (sum_nr_running == leader_nr_running &&
-                             first_cpu(group->cpumask) >
-                              first_cpu(group_leader->cpumask))) {
+                             cpumask_first(sched_group_cpus(group)) <
+                             cpumask_first(sched_group_cpus(group_leader)))) {
                                 group_leader = group;
                                 leader_nr_running = sum_nr_running;
                         }
@@ -3407,6 +3395,10 @@ out_balanced:
 
         if (this == group_leader && group_leader != group_min) {
                 *imbalance = min_load_per_task;
+                if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP) {
+                        cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu =
+                                cpumask_first(sched_group_cpus(group_leader));
+                }
                 return group_min;
         }
 #endif
@@ -3420,16 +3412,16 @@ ret:
  */
 static struct rq *
 find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
-                   unsigned long imbalance, const cpumask_t *cpus)
+                   unsigned long imbalance, const struct cpumask *cpus)
 {
         struct rq *busiest = NULL, *rq;
         unsigned long max_load = 0;
         int i;
 
-        for_each_cpu_mask_nr(i, group->cpumask) {
+        for_each_cpu(i, sched_group_cpus(group)) {
                 unsigned long wl;
 
-                if (!cpu_isset(i, *cpus))
+                if (!cpumask_test_cpu(i, cpus))
                         continue;
 
                 rq = cpu_rq(i);
@@ -3459,7 +3451,7 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
  */
 static int load_balance(int this_cpu, struct rq *this_rq,
                         struct sched_domain *sd, enum cpu_idle_type idle,
-                        int *balance, cpumask_t *cpus)
+                        int *balance, struct cpumask *cpus)
 {
         int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
         struct sched_group *group;
@@ -3467,7 +3459,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
         struct rq *busiest;
         unsigned long flags;
 
-        cpus_setall(*cpus);
+        cpumask_setall(cpus);
 
         /*
          * When power savings policy is enabled for the parent domain, idle
@@ -3527,8 +3519,8 @@ redo:
 
                 /* All tasks on this runqueue were pinned by CPU affinity */
                 if (unlikely(all_pinned)) {
-                        cpu_clear(cpu_of(busiest), *cpus);
-                        if (!cpus_empty(*cpus))
+                        cpumask_clear_cpu(cpu_of(busiest), cpus);
+                        if (!cpumask_empty(cpus))
                                 goto redo;
                         goto out_balanced;
                 }
@@ -3545,7 +3537,8 @@ redo:
                         /* don't kick the migration_thread, if the curr
                          * task on busiest cpu can't be moved to this_cpu
                          */
-                        if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) {
+                        if (!cpumask_test_cpu(this_cpu,
+                                              &busiest->curr->cpus_allowed)) {
                                 spin_unlock_irqrestore(&busiest->lock, flags);
                                 all_pinned = 1;
                                 goto out_one_pinned;
@@ -3620,7 +3613,7 @@ out:
  */
 static int
 load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
-                        cpumask_t *cpus)
+                        struct cpumask *cpus)
 {
         struct sched_group *group;
         struct rq *busiest = NULL;
@@ -3629,7 +3622,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
         int sd_idle = 0;
         int all_pinned = 0;
 
-        cpus_setall(*cpus);
+        cpumask_setall(cpus);
 
         /*
          * When power savings policy is enabled for the parent domain, idle
@@ -3673,17 +3666,76 @@ redo:
                 double_unlock_balance(this_rq, busiest);
 
                 if (unlikely(all_pinned)) {
-                        cpu_clear(cpu_of(busiest), *cpus);
-                        if (!cpus_empty(*cpus))
+                        cpumask_clear_cpu(cpu_of(busiest), cpus);
+                        if (!cpumask_empty(cpus))
                                 goto redo;
                 }
         }
 
         if (!ld_moved) {
+                int active_balance = 0;
+
                 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
                 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
                     !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
                         return -1;
+
+                if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
+                        return -1;
+
+                if (sd->nr_balance_failed++ < 2)
+                        return -1;
+
+                /*
+                 * The only task running in a non-idle cpu can be moved to this
+                 * cpu in an attempt to completely freeup the other CPU
+                 * package. The same method used to move task in load_balance()
+                 * have been extended for load_balance_newidle() to speedup
+                 * consolidation at sched_mc=POWERSAVINGS_BALANCE_WAKEUP (2)
+                 *
+                 * The package power saving logic comes from
+                 * find_busiest_group().  If there are no imbalance, then
+                 * f_b_g() will return NULL.  However when sched_mc={1,2} then
+                 * f_b_g() will select a group from which a running task may be
+                 * pulled to this cpu in order to make the other package idle.
+                 * If there is no opportunity to make a package idle and if
+                 * there are no imbalance, then f_b_g() will return NULL and no
+                 * action will be taken in load_balance_newidle().
+                 *
+                 * Under normal task pull operation due to imbalance, there
+                 * will be more than one task in the source run queue and
+                 * move_tasks() will succeed.  ld_moved will be true and this
+                 * active balance code will not be triggered.
+                 */
+
+                /* Lock busiest in correct order while this_rq is held */
+                double_lock_balance(this_rq, busiest);
+
+                /*
+                 * don't kick the migration_thread, if the curr
+                 * task on busiest cpu can't be moved to this_cpu
+                 */
+                if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) {
+                        double_unlock_balance(this_rq, busiest);
+                        all_pinned = 1;
+                        return ld_moved;
+                }
+
+                if (!busiest->active_balance) {
+                        busiest->active_balance = 1;
+                        busiest->push_cpu = this_cpu;
+                        active_balance = 1;
+                }
+
+                double_unlock_balance(this_rq, busiest);
+                /*
+                 * Should not call ttwu while holding a rq->lock
+                 */
+                spin_unlock(&this_rq->lock);
+                if (active_balance)
+                        wake_up_process(busiest->migration_thread);
+                spin_lock(&this_rq->lock);
+
         } else
                 sd->nr_balance_failed = 0;
 
@@ -3707,9 +3759,12 @@ out_balanced:
 static void idle_balance(int this_cpu, struct rq *this_rq)
 {
         struct sched_domain *sd;
-        int pulled_task = -1;
+        int pulled_task = 0;
         unsigned long next_balance = jiffies + HZ;
-        cpumask_t tmpmask;
+        cpumask_var_t tmpmask;
+
+        if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
+                return;
 
         for_each_domain(this_cpu, sd) {
                 unsigned long interval;
@@ -3720,7 +3775,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
                 if (sd->flags & SD_BALANCE_NEWIDLE)
                         /* If we've pulled tasks over stop searching: */
                         pulled_task = load_balance_newidle(this_cpu, this_rq,
-                                                           sd, &tmpmask);
+                                                           sd, tmpmask);
 
                 interval = msecs_to_jiffies(sd->balance_interval);
                 if (time_after(next_balance, sd->last_balance + interval))
@@ -3735,6 +3790,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
                  */
                 this_rq->next_balance = next_balance;
         }
+        free_cpumask_var(tmpmask);
 }
 
 /*
@@ -3772,7 +3828,7 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
         /* Search for an sd spanning us and the target CPU. */
         for_each_domain(target_cpu, sd) {
                 if ((sd->flags & SD_LOAD_BALANCE) &&
-                    cpu_isset(busiest_cpu, sd->span))
+                    cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
                                 break;
         }
 
@@ -3791,10 +3847,9 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
 #ifdef CONFIG_NO_HZ
 static struct {
         atomic_t load_balancer;
-        cpumask_t cpu_mask;
+        cpumask_var_t cpu_mask;
 } nohz ____cacheline_aligned = {
         .load_balancer = ATOMIC_INIT(-1),
-        .cpu_mask = CPU_MASK_NONE,
 };
 
 /*
@@ -3822,7 +3877,7 @@ int select_nohz_load_balancer(int stop_tick)
         int cpu = smp_processor_id();
 
         if (stop_tick) {
-                cpu_set(cpu, nohz.cpu_mask);
+                cpumask_set_cpu(cpu, nohz.cpu_mask);
                 cpu_rq(cpu)->in_nohz_recently = 1;
 
                 /*
@@ -3836,7 +3891,7 @@ int select_nohz_load_balancer(int stop_tick)
                 }
 
                 /* time for ilb owner also to sleep */
-                if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) {
+                if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
                         if (atomic_read(&nohz.load_balancer) == cpu)
                                 atomic_set(&nohz.load_balancer, -1);
                         return 0;
@@ -3849,10 +3904,10 @@ int select_nohz_load_balancer(int stop_tick)
                 } else if (atomic_read(&nohz.load_balancer) == cpu)
                         return 1;
         } else {
-                if (!cpu_isset(cpu, nohz.cpu_mask))
+                if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
                         return 0;
 
-                cpu_clear(cpu, nohz.cpu_mask);
+                cpumask_clear_cpu(cpu, nohz.cpu_mask);
 
                 if (atomic_read(&nohz.load_balancer) == cpu)
                         if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
@@ -3880,7 +3935,11 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
         unsigned long next_balance = jiffies + 60*HZ;
         int update_next_balance = 0;
         int need_serialize;
-        cpumask_t tmp;
+        cpumask_var_t tmp;
+
+        /* Fails alloc?  Rebalancing probably not a priority right now. */
+        if (!alloc_cpumask_var(&tmp, GFP_ATOMIC))
+                return;
 
         for_each_domain(cpu, sd) {
                 if (!(sd->flags & SD_LOAD_BALANCE))
@@ -3905,7 +3964,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
                 }
 
                 if (time_after_eq(jiffies, sd->last_balance + interval)) {
-                        if (load_balance(cpu, rq, sd, idle, &balance, &tmp)) {
+                        if (load_balance(cpu, rq, sd, idle, &balance, tmp)) {
                                 /*
                                  * We've pulled tasks over so either we're no
                                  * longer idle, or one of our SMT siblings is
@@ -3939,6 +3998,8 @@ out:
          */
         if (likely(update_next_balance))
                 rq->next_balance = next_balance;
+
+        free_cpumask_var(tmp);
 }
 
 /*
@@ -3963,12 +4024,13 @@ static void run_rebalance_domains(struct softirq_action *h)
          */
         if (this_rq->idle_at_tick &&
             atomic_read(&nohz.load_balancer) == this_cpu) {
-                cpumask_t cpus = nohz.cpu_mask;
                 struct rq *rq;
                 int balance_cpu;
 
-                cpu_clear(this_cpu, cpus);
-                for_each_cpu_mask_nr(balance_cpu, cpus) {
+                for_each_cpu(balance_cpu, nohz.cpu_mask) {
+                        if (balance_cpu == this_cpu)
+                                continue;
+
                         /*
                          * If this cpu gets work to do, stop the load balancing
                          * work being done for other cpus. Next load
@@ -4006,7 +4068,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
                 rq->in_nohz_recently = 0;
 
                 if (atomic_read(&nohz.load_balancer) == cpu) {
-                        cpu_clear(cpu, nohz.cpu_mask);
+                        cpumask_clear_cpu(cpu, nohz.cpu_mask);
                         atomic_set(&nohz.load_balancer, -1);
                 }
 
@@ -4019,7 +4081,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
                          * TBD: Traverse the sched domains and nominate
                          * the nearest cpu in the nohz.cpu_mask.
                          */
-                        int ilb = first_cpu(nohz.cpu_mask);
+                        int ilb = cpumask_first(nohz.cpu_mask);
 
                         if (ilb < nr_cpu_ids)
                                 resched_cpu(ilb);
@@ -4031,7 +4093,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
          * cpus with ticks stopped, is it time for that to stop?
          */
         if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
-            cpus_weight(nohz.cpu_mask) == num_online_cpus()) {
+            cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
                 resched_cpu(cpu);
                 return;
         }
@@ -4041,7 +4103,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
          * someone else, then no need raise the SCHED_SOFTIRQ
          */
         if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
-            cpu_isset(cpu, nohz.cpu_mask))
+            cpumask_test_cpu(cpu, nohz.cpu_mask))
                 return;
 #endif
         if (time_after_eq(jiffies, rq->next_balance))
@@ -4093,13 +4155,17 @@ unsigned long long task_delta_exec(struct task_struct *p)
  * Account user cpu time to a process.
  * @p: the process that the cpu time gets accounted to
  * @cputime: the cpu time spent in user space since the last update
+ * @cputime_scaled: cputime scaled by cpu frequency
  */
-void account_user_time(struct task_struct *p, cputime_t cputime)
+void account_user_time(struct task_struct *p, cputime_t cputime,
+                       cputime_t cputime_scaled)
 {
         struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
         cputime64_t tmp;
 
+        /* Add user time to process. */
         p->utime = cputime_add(p->utime, cputime);
+        p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
         account_group_user_time(p, cputime);
 
         /* Add user time to cpustat. */
@@ -4116,51 +4182,48 @@ void account_user_time(struct task_struct *p, cputime_t cputime)
  * Account guest cpu time to a process.
  * @p: the process that the cpu time gets accounted to
  * @cputime: the cpu time spent in virtual machine since the last update
+ * @cputime_scaled: cputime scaled by cpu frequency
  */
-static void account_guest_time(struct task_struct *p, cputime_t cputime)
+static void account_guest_time(struct task_struct *p, cputime_t cputime,
+                               cputime_t cputime_scaled)
 {
         cputime64_t tmp;
         struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
 
         tmp = cputime_to_cputime64(cputime);
 
+        /* Add guest time to process. */
         p->utime = cputime_add(p->utime, cputime);
+        p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
         account_group_user_time(p, cputime);
         p->gtime = cputime_add(p->gtime, cputime);
 
+        /* Add guest time to cpustat. */
         cpustat->user = cputime64_add(cpustat->user, tmp);
         cpustat->guest = cputime64_add(cpustat->guest, tmp);
 }
 
 /*
- * Account scaled user cpu time to a process.
- * @p: the process that the cpu time gets accounted to
- * @cputime: the cpu time spent in user space since the last update
- */
-void account_user_time_scaled(struct task_struct *p, cputime_t cputime)
-{
-        p->utimescaled = cputime_add(p->utimescaled, cputime);
-}
-
-/*
  * Account system cpu time to a process.
  * @p: the process that the cpu time gets accounted to
  * @hardirq_offset: the offset to subtract from hardirq_count()
  * @cputime: the cpu time spent in kernel space since the last update
+ * @cputime_scaled: cputime scaled by cpu frequency
  */
 void account_system_time(struct task_struct *p, int hardirq_offset,
-                         cputime_t cputime)
+                         cputime_t cputime, cputime_t cputime_scaled)
 {
         struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
-        struct rq *rq = this_rq();
         cputime64_t tmp;
 
         if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
-                account_guest_time(p, cputime);
+                account_guest_time(p, cputime, cputime_scaled);
                 return;
         }
 
+        /* Add system time to process. */
         p->stime = cputime_add(p->stime, cputime);
+        p->stimescaled = cputime_add(p->stimescaled, cputime_scaled);
         account_group_system_time(p, cputime);
 
         /* Add system time to cpustat. */
@@ -4169,50 +4232,85 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
                 cpustat->irq = cputime64_add(cpustat->irq, tmp);
         else if (softirq_count())
                 cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
-        else if (p != rq->idle)
-                cpustat->system = cputime64_add(cpustat->system, tmp);
-        else if (atomic_read(&rq->nr_iowait) > 0)
-                cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
         else
-                cpustat->idle = cputime64_add(cpustat->idle, tmp);
+                cpustat->system = cputime64_add(cpustat->system, tmp);
+
         /* Account for system time used */
         acct_update_integrals(p);
 }
 
 /*
- * Account scaled system cpu time to a process.
- * @p: the process that the cpu time gets accounted to
- * @hardirq_offset: the offset to subtract from hardirq_count()
- * @cputime: the cpu time spent in kernel space since the last update
+ * Account for involuntary wait time.
+ * @steal: the cpu time spent in involuntary wait
  */
-void account_system_time_scaled(struct task_struct *p, cputime_t cputime)
+void account_steal_time(cputime_t cputime)
 {
-        p->stimescaled = cputime_add(p->stimescaled, cputime);
+        struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
+        cputime64_t cputime64 = cputime_to_cputime64(cputime);
+
+        cpustat->steal = cputime64_add(cpustat->steal, cputime64);
 }
 
 /*
- * Account for involuntary wait time.
- * @p: the process from which the cpu time has been stolen
- * @steal: the cpu time spent in involuntary wait
+ * Account for idle time.
+ * @cputime: the cpu time spent in idle wait
  */
-void account_steal_time(struct task_struct *p, cputime_t steal)
+void account_idle_time(cputime_t cputime)
 {
         struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
-        cputime64_t tmp = cputime_to_cputime64(steal);
+        cputime64_t cputime64 = cputime_to_cputime64(cputime);
         struct rq *rq = this_rq();
 
-        if (p == rq->idle) {
-                p->stime = cputime_add(p->stime, steal);
-                account_group_system_time(p, steal);
-                if (atomic_read(&rq->nr_iowait) > 0)
-                        cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
-                else
-                        cpustat->idle = cputime64_add(cpustat->idle, tmp);
-        } else
-                cpustat->steal = cputime64_add(cpustat->steal, tmp);
+        if (atomic_read(&rq->nr_iowait) > 0)
+                cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
+        else
+                cpustat->idle = cputime64_add(cpustat->idle, cputime64);
+}
+
+#ifndef CONFIG_VIRT_CPU_ACCOUNTING
+
+/*
+ * Account a single tick of cpu time.
+ * @p: the process that the cpu time gets accounted to
+ * @user_tick: indicates if the tick is a user or a system tick
+ */
+void account_process_tick(struct task_struct *p, int user_tick)
+{
+        cputime_t one_jiffy = jiffies_to_cputime(1);
+        cputime_t one_jiffy_scaled = cputime_to_scaled(one_jiffy);
+        struct rq *rq = this_rq();
+
+        if (user_tick)
+                account_user_time(p, one_jiffy, one_jiffy_scaled);
+        else if (p != rq->idle)
+                account_system_time(p, HARDIRQ_OFFSET, one_jiffy,
+                                    one_jiffy_scaled);
+        else
+                account_idle_time(one_jiffy);
+}
+
+/*
+ * Account multiple ticks of steal time.
+ * @p: the process from which the cpu time has been stolen
+ * @ticks: number of stolen ticks
+ */
+void account_steal_ticks(unsigned long ticks)
+{
+        account_steal_time(jiffies_to_cputime(ticks));
 }
 
 /*
+ * Account multiple ticks of idle time.
+ * @ticks: number of stolen ticks
+ */
+void account_idle_ticks(unsigned long ticks)
+{
+        account_idle_time(jiffies_to_cputime(ticks));
+}
+
+#endif
+
+/*
  * Use precise platform statistics if available:
  */
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
@@ -4339,7 +4437,7 @@ void __kprobes sub_preempt_count(int val)
         /*
          * Underflow?
          */
-        if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
+       if (DEBUG_LOCKS_WARN_ON(val > preempt_count() - (!!kernel_locked())))
                 return;
         /*
          * Is the spinlock portion underflowing?
@@ -5134,6 +5232,22 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
         set_load_weight(p);
 }
 
+/*
+ * check the target process has a UID that matches the current process's
+ */
+static bool check_same_owner(struct task_struct *p)
+{
+        const struct cred *cred = current_cred(), *pcred;
+        bool match;
+
+        rcu_read_lock();
+        pcred = __task_cred(p);
+        match = (cred->euid == pcred->euid ||
+                 cred->euid == pcred->uid);
+        rcu_read_unlock();
+        return match;
+}
+
 static int __sched_setscheduler(struct task_struct *p, int policy,
                                 struct sched_param *param, bool user)
 {
@@ -5193,8 +5307,7 @@ recheck:
                         return -EPERM;
 
                 /* can't change other user's priorities */
-                if ((current->euid != p->euid) &&
-                    (current->euid != p->uid))
+                if (!check_same_owner(p))
                         return -EPERM;
         }
 
@@ -5400,10 +5513,9 @@ out_unlock:
         return retval;
 }
 
-long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
+long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
 {
-        cpumask_t cpus_allowed;
-        cpumask_t new_mask = *in_mask;
+        cpumask_var_t cpus_allowed, new_mask;
         struct task_struct *p;
         int retval;
 
@@ -5425,46 +5537,57 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
         get_task_struct(p);
         read_unlock(&tasklist_lock);
 
+        if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
+                retval = -ENOMEM;
+                goto out_put_task;
+        }
+        if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
+                retval = -ENOMEM;
+                goto out_free_cpus_allowed;
+        }
         retval = -EPERM;
-        if ((current->euid != p->euid) && (current->euid != p->uid) &&
-                        !capable(CAP_SYS_NICE))
+        if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
                 goto out_unlock;
 
         retval = security_task_setscheduler(p, 0, NULL);
         if (retval)
                 goto out_unlock;
 
-        cpuset_cpus_allowed(p, &cpus_allowed);
-        cpus_and(new_mask, new_mask, cpus_allowed);
+        cpuset_cpus_allowed(p, cpus_allowed);
+        cpumask_and(new_mask, in_mask, cpus_allowed);
  again:
-        retval = set_cpus_allowed_ptr(p, &new_mask);
+        retval = set_cpus_allowed_ptr(p, new_mask);
 
         if (!retval) {
-                cpuset_cpus_allowed(p, &cpus_allowed);
-                if (!cpus_subset(new_mask, cpus_allowed)) {
+                cpuset_cpus_allowed(p, cpus_allowed);
+                if (!cpumask_subset(new_mask, cpus_allowed)) {
                         /*
                          * We must have raced with a concurrent cpuset
                          * update. Just reset the cpus_allowed to the
                          * cpuset's cpus_allowed
                          */
-                        new_mask = cpus_allowed;
+                        cpumask_copy(new_mask, cpus_allowed);
                         goto again;
                 }
         }
 out_unlock:
+        free_cpumask_var(new_mask);
+out_free_cpus_allowed:
+        free_cpumask_var(cpus_allowed);
+out_put_task:
         put_task_struct(p);
         put_online_cpus();
         return retval;
 }
 
 static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
-                             cpumask_t *new_mask)
+                             struct cpumask *new_mask)
 {
-        if (len < sizeof(cpumask_t)) {
-                memset(new_mask, 0, sizeof(cpumask_t));
-        } else if (len > sizeof(cpumask_t)) {
-                len = sizeof(cpumask_t);
-        }
+        if (len < cpumask_size())
+                cpumask_clear(new_mask);
+        else if (len > cpumask_size())
+                len = cpumask_size();
+
         return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
 }
 
@@ -5477,17 +5600,20 @@ static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
 asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
                                       unsigned long __user *user_mask_ptr)
 {
-        cpumask_t new_mask;
+        cpumask_var_t new_mask;
         int retval;
 
-        retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask);
-        if (retval)
-                return retval;
+        if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
+                return -ENOMEM;
 
-        return sched_setaffinity(pid, &new_mask);
+        retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
+        if (retval == 0)
+                retval = sched_setaffinity(pid, new_mask);
+        free_cpumask_var(new_mask);
+        return retval;
 }
 
-long sched_getaffinity(pid_t pid, cpumask_t *mask)
+long sched_getaffinity(pid_t pid, struct cpumask *mask)
 {
         struct task_struct *p;
         int retval;
@@ -5504,7 +5630,7 @@ long sched_getaffinity(pid_t pid, cpumask_t *mask)
         if (retval)
                 goto out_unlock;
 
-        cpus_and(*mask, p->cpus_allowed, cpu_online_map);
+        cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
 
 out_unlock:
         read_unlock(&tasklist_lock);
@@ -5523,19 +5649,24 @@ asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
                                       unsigned long __user *user_mask_ptr)
 {
         int ret;
-        cpumask_t mask;
+        cpumask_var_t mask;
 
-        if (len < sizeof(cpumask_t))
+        if (len < cpumask_size())
                 return -EINVAL;
 
-        ret = sched_getaffinity(pid, &mask);
-        if (ret < 0)
-                return ret;
+        if (!alloc_cpumask_var(&mask, GFP_KERNEL))
+                return -ENOMEM;
 
-        if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t)))
-                return -EFAULT;
+        ret = sched_getaffinity(pid, mask);
+        if (ret == 0) {
+                if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
+                        ret = -EFAULT;
+                else
+                        ret = cpumask_size();
+        }
+        free_cpumask_var(mask);
 
-        return sizeof(cpumask_t);
+        return ret;
 }
 
 /**
@@ -5877,7 +6008,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
         idle->se.exec_start = sched_clock();
 
         idle->prio = idle->normal_prio = MAX_PRIO;
-        idle->cpus_allowed = cpumask_of_cpu(cpu);
+        cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
         __set_task_cpu(idle, cpu);
 
         rq->curr = rq->idle = idle;
@@ -5896,6 +6027,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
          * The idle tasks have their own, simple scheduling class:
          */
         idle->sched_class = &idle_sched_class;
+        ftrace_graph_init_task(idle);
 }
 
 /*
@@ -5903,9 +6035,9 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
  * indicates which cpus entered this state. This is used
  * in the rcu update to wait only for active cpus. For system
  * which do not switch off the HZ timer nohz_cpu_mask should
- * always be CPU_MASK_NONE.
+ * always be CPU_BITS_NONE.
  */
-cpumask_t nohz_cpu_mask = CPU_MASK_NONE;
+cpumask_var_t nohz_cpu_mask;
 
 /*
  * Increase the granularity value when there are more CPUs,
@@ -5960,7 +6092,7 @@ static inline void sched_init_granularity(void)
  * task must not exit() & deallocate itself prematurely. The
  * call is not atomic; no spinlocks may be held.
  */
-int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
+int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
 {
         struct migration_req req;
         unsigned long flags;
@@ -5968,13 +6100,13 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
         int ret = 0;
 
         rq = task_rq_lock(p, &flags);
-        if (!cpus_intersects(*new_mask, cpu_online_map)) {
+        if (!cpumask_intersects(new_mask, cpu_online_mask)) {
                 ret = -EINVAL;
                 goto out;
         }
 
         if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
-                     !cpus_equal(p->cpus_allowed, *new_mask))) {
+                     !cpumask_equal(&p->cpus_allowed, new_mask))) {
                 ret = -EINVAL;
                 goto out;
         }
@@ -5982,15 +6114,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
         if (p->sched_class->set_cpus_allowed)
                 p->sched_class->set_cpus_allowed(p, new_mask);
         else {
-                p->cpus_allowed = *new_mask;
-                p->rt.nr_cpus_allowed = cpus_weight(*new_mask);
+                cpumask_copy(&p->cpus_allowed, new_mask);
+                p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
         }
 
         /* Can the task run on the task's current CPU? If so, we're done */
-        if (cpu_isset(task_cpu(p), *new_mask))
+        if (cpumask_test_cpu(task_cpu(p), new_mask))
                 goto out;
 
-        if (migrate_task(p, any_online_cpu(*new_mask), &req)) {
+        if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
                 /* Need help from migration thread: drop lock and wait. */
                 task_rq_unlock(rq, &flags);
                 wake_up_process(rq->migration_thread);
@@ -6032,7 +6164,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
         if (task_cpu(p) != src_cpu)
                 goto done;
         /* Affinity changed (again). */
-        if (!cpu_isset(dest_cpu, p->cpus_allowed))
+        if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
                 goto fail;
 
         on_rq = p->se.on_rq;
@@ -6126,54 +6258,44 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
 
 /*
  * Figure out where task on dead CPU should go, use force if necessary.
- * NOTE: interrupts should be disabled by the caller
  */
 static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
 {
-        unsigned long flags;
-        cpumask_t mask;
-        struct rq *rq;
         int dest_cpu;
+        const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(dead_cpu));
 
-        do {
-                /* On same node? */
-                mask = node_to_cpumask(cpu_to_node(dead_cpu));
-                cpus_and(mask, mask, p->cpus_allowed);
-                dest_cpu = any_online_cpu(mask);
+again:
+        /* Look for allowed, online CPU in same node. */
+        for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
+                if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
+                        goto move;
 
-                /* On any allowed CPU? */
-                if (dest_cpu >= nr_cpu_ids)
-                        dest_cpu = any_online_cpu(p->cpus_allowed);
+        /* Any allowed, online CPU? */
+        dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
+        if (dest_cpu < nr_cpu_ids)
+                goto move;
 
-                /* No more Mr. Nice Guy. */
-                if (dest_cpu >= nr_cpu_ids) {
-                        cpumask_t cpus_allowed;
+        /* No more Mr. Nice Guy. */
+        if (dest_cpu >= nr_cpu_ids) {
+                cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
+                dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
 
-                        cpuset_cpus_allowed_locked(p, &cpus_allowed);
-                        /*
-                         * Try to stay on the same cpuset, where the
-                         * current cpuset may be a subset of all cpus.
-                         * The cpuset_cpus_allowed_locked() variant of
-                         * cpuset_cpus_allowed() will not block. It must be
-                         * called within calls to cpuset_lock/cpuset_unlock.
-                         */
-                        rq = task_rq_lock(p, &flags);
-                        p->cpus_allowed = cpus_allowed;
-                        dest_cpu = any_online_cpu(p->cpus_allowed);
-                        task_rq_unlock(rq, &flags);
-
-                        /*
-                         * Don't tell them about moving exiting tasks or
-                         * kernel threads (both mm NULL), since they never
-                         * leave kernel.
-                         */
-                        if (p->mm && printk_ratelimit()) {
-                                printk(KERN_INFO "process %d (%s) no "
-                                       "longer affine to cpu%d\n",
-                                        task_pid_nr(p), p->comm, dead_cpu);
-                        }
+                /*
+                 * Don't tell them about moving exiting tasks or
+                 * kernel threads (both mm NULL), since they never
+                 * leave kernel.
+                 */
+                if (p->mm && printk_ratelimit()) {
+                        printk(KERN_INFO "process %d (%s) no "
+                               "longer affine to cpu%d\n",
+                               task_pid_nr(p), p->comm, dead_cpu);
                 }
-        } while (!__migrate_task_irq(p, dead_cpu, dest_cpu));
+        }
+
+move:
+        /* It can have affinity changed while we were choosing. */
+        if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
+                goto again;
 }
 
 /*
@@ -6185,7 +6307,7 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
  */
 static void migrate_nr_uninterruptible(struct rq *rq_src)
 {
-        struct rq *rq_dest = cpu_rq(any_online_cpu(*CPU_MASK_ALL_PTR));
+        struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
         unsigned long flags;
 
         local_irq_save(flags);
@@ -6475,7 +6597,7 @@ static void set_rq_online(struct rq *rq)
         if (!rq->online) {
                 const struct sched_class *class;
 
-                cpu_set(rq->cpu, rq->rd->online);
+                cpumask_set_cpu(rq->cpu, rq->rd->online);
                 rq->online = 1;
 
                 for_each_class(class) {
@@ -6495,7 +6617,7 @@ static void set_rq_offline(struct rq *rq)
                                 class->rq_offline(rq);
                 }
 
-                cpu_clear(rq->cpu, rq->rd->online);
+                cpumask_clear_cpu(rq->cpu, rq->rd->online);
                 rq->online = 0;
         }
 }
@@ -6536,7 +6658,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 rq = cpu_rq(cpu);
                 spin_lock_irqsave(&rq->lock, flags);
                 if (rq->rd) {
-                        BUG_ON(!cpu_isset(cpu, rq->rd->span));
+                        BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
 
                         set_rq_online(rq);
                 }
@@ -6550,7 +6672,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                         break;
                 /* Unbind it from offline cpu so it can run. Fall thru. */
                 kthread_bind(cpu_rq(cpu)->migration_thread,
-                             any_online_cpu(cpu_online_map));
+                             cpumask_any(cpu_online_mask));
                 kthread_stop(cpu_rq(cpu)->migration_thread);
                 cpu_rq(cpu)->migration_thread = NULL;
                 break;
@@ -6600,7 +6722,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
                 rq = cpu_rq(cpu);
                 spin_lock_irqsave(&rq->lock, flags);
                 if (rq->rd) {
-                        BUG_ON(!cpu_isset(cpu, rq->rd->span));
+                        BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
                         set_rq_offline(rq);
                 }
                 spin_unlock_irqrestore(&rq->lock, flags);
@@ -6638,36 +6760,14 @@ early_initcall(migration_init);
 
 #ifdef CONFIG_SCHED_DEBUG
 
-static inline const char *sd_level_to_string(enum sched_domain_level lvl)
-{
-        switch (lvl) {
-        case SD_LV_NONE:
-                        return "NONE";
-        case SD_LV_SIBLING:
-                        return "SIBLING";
-        case SD_LV_MC:
-                        return "MC";
-        case SD_LV_CPU:
-                        return "CPU";
-        case SD_LV_NODE:
-                        return "NODE";
-        case SD_LV_ALLNODES:
-                        return "ALLNODES";
-        case SD_LV_MAX:
-                        return "MAX";
-
-        }
-        return "MAX";
-}
-
 static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
-                                  cpumask_t *groupmask)
+                                  struct cpumask *groupmask)
 {
         struct sched_group *group = sd->groups;
         char str[256];
 
-        cpulist_scnprintf(str, sizeof(str), sd->span);
-        cpus_clear(*groupmask);
+        cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
+        cpumask_clear(groupmask);
 
         printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
 
@@ -6679,14 +6779,13 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                 return -1;
         }
 
-        printk(KERN_CONT "span %s level %s\n",
-                str, sd_level_to_string(sd->level));
+        printk(KERN_CONT "span %s level %s\n", str, sd->name);
 
-        if (!cpu_isset(cpu, sd->span)) {
+        if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
                 printk(KERN_ERR "ERROR: domain->span does not contain "
                                 "CPU%d\n", cpu);
         }
-        if (!cpu_isset(cpu, group->cpumask)) {
+        if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
                 printk(KERN_ERR "ERROR: domain->groups does not contain"
                                 " CPU%d\n", cpu);
         }
@@ -6706,31 +6805,32 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                         break;
                 }
 
-                if (!cpus_weight(group->cpumask)) {
+                if (!cpumask_weight(sched_group_cpus(group))) {
                         printk(KERN_CONT "\n");
                         printk(KERN_ERR "ERROR: empty group\n");
                         break;
                 }
 
-                if (cpus_intersects(*groupmask, group->cpumask)) {
+                if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
                         printk(KERN_CONT "\n");
                         printk(KERN_ERR "ERROR: repeated CPUs\n");
                         break;
                 }
 
-                cpus_or(*groupmask, *groupmask, group->cpumask);
+                cpumask_or(groupmask, groupmask, sched_group_cpus(group));
 
-                cpulist_scnprintf(str, sizeof(str), group->cpumask);
+                cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
                 printk(KERN_CONT " %s", str);
 
                 group = group->next;
         } while (group != sd->groups);
         printk(KERN_CONT "\n");
 
-        if (!cpus_equal(sd->span, *groupmask))
+        if (!cpumask_equal(sched_domain_span(sd), groupmask))
                 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
 
-        if (sd->parent && !cpus_subset(*groupmask, sd->parent->span))
+        if (sd->parent &&
+            !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
                 printk(KERN_ERR "ERROR: parent span is not a superset "
                         "of domain->span\n");
         return 0;
@@ -6738,7 +6838,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
 
 static void sched_domain_debug(struct sched_domain *sd, int cpu)
 {
-        cpumask_t *groupmask;
+        cpumask_var_t groupmask;
         int level = 0;
 
         if (!sd) {
@@ -6748,8 +6848,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
 
         printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
 
-        groupmask = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
-        if (!groupmask) {
+        if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
                 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
                 return;
         }
@@ -6762,7 +6861,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
                 if (!sd)
                         break;
         }
-        kfree(groupmask);
+        free_cpumask_var(groupmask);
 }
 #else /* !CONFIG_SCHED_DEBUG */
 # define sched_domain_debug(sd, cpu) do { } while (0)
@@ -6770,7 +6869,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
 
 static int sd_degenerate(struct sched_domain *sd)
 {
-        if (cpus_weight(sd->span) == 1)
+        if (cpumask_weight(sched_domain_span(sd)) == 1)
                 return 1;
 
         /* Following flags need at least 2 groups */
@@ -6801,7 +6900,7 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
         if (sd_degenerate(parent))
                 return 1;
 
-        if (!cpus_equal(sd->span, parent->span))
+        if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
                 return 0;
 
         /* Does parent contain flags not in child? */
@@ -6816,6 +6915,8 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
                                 SD_BALANCE_EXEC |
                                 SD_SHARE_CPUPOWER |
                                 SD_SHARE_PKG_RESOURCES);
+                if (nr_node_ids == 1)
+                        pflags &= ~SD_SERIALIZE;
         }
         if (~cflags & pflags)
                 return 0;
@@ -6823,6 +6924,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
         return 1;
 }
 
+static void free_rootdomain(struct root_domain *rd)
+{
+        cpupri_cleanup(&rd->cpupri);
+
+        free_cpumask_var(rd->rto_mask);
+        free_cpumask_var(rd->online);
+        free_cpumask_var(rd->span);
+        kfree(rd);
+}
+
 static void rq_attach_root(struct rq *rq, struct root_domain *rd)
 {
         unsigned long flags;
@@ -6832,38 +6943,62 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
         if (rq->rd) {
                 struct root_domain *old_rd = rq->rd;
 
-                if (cpu_isset(rq->cpu, old_rd->online))
+                if (cpumask_test_cpu(rq->cpu, old_rd->online))
                         set_rq_offline(rq);
 
-                cpu_clear(rq->cpu, old_rd->span);
+                cpumask_clear_cpu(rq->cpu, old_rd->span);
 
                 if (atomic_dec_and_test(&old_rd->refcount))
-                        kfree(old_rd);
+                        free_rootdomain(old_rd);
         }
 
         atomic_inc(&rd->refcount);
         rq->rd = rd;
 
-        cpu_set(rq->cpu, rd->span);
-        if (cpu_isset(rq->cpu, cpu_online_map))
+        cpumask_set_cpu(rq->cpu, rd->span);
+        if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
                 set_rq_online(rq);
 
         spin_unlock_irqrestore(&rq->lock, flags);
 }
 
-static void init_rootdomain(struct root_domain *rd)
+static int __init_refok init_rootdomain(struct root_domain *rd, bool bootmem)
 {
         memset(rd, 0, sizeof(*rd));
 
-        cpus_clear(rd->span);
-        cpus_clear(rd->online);
+        if (bootmem) {
+                alloc_bootmem_cpumask_var(&def_root_domain.span);
+                alloc_bootmem_cpumask_var(&def_root_domain.online);
+                alloc_bootmem_cpumask_var(&def_root_domain.rto_mask);
+                cpupri_init(&rd->cpupri, true);
+                return 0;
+        }
+
+        if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
+                goto out;
+        if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
+                goto free_span;
+        if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
+                goto free_online;
 
-        cpupri_init(&rd->cpupri);
+        if (cpupri_init(&rd->cpupri, false) != 0)
+                goto free_rto_mask;
+        return 0;
+
+free_rto_mask:
+        free_cpumask_var(rd->rto_mask);
+free_online:
+        free_cpumask_var(rd->online);
+free_span:
+        free_cpumask_var(rd->span);
+out:
+        return -ENOMEM;
 }
 
 static void init_defrootdomain(void)
 {
-        init_rootdomain(&def_root_domain);
+        init_rootdomain(&def_root_domain, true);
+
         atomic_set(&def_root_domain.refcount, 1);
 }
 
@@ -6875,7 +7010,10 @@ static struct root_domain *alloc_rootdomain(void)
         if (!rd)
                 return NULL;
 
-        init_rootdomain(rd);
+        if (init_rootdomain(rd, false) != 0) {
+                kfree(rd);
+                return NULL;
+        }
 
         return rd;
 }
@@ -6917,19 +7055,12 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
 }
 
 /* cpus with isolated domains */
-static cpumask_t cpu_isolated_map = CPU_MASK_NONE;
+static cpumask_var_t cpu_isolated_map;
 
 /* Setup the mask of cpus configured for isolated domains */
 static int __init isolated_cpu_setup(char *str)
 {
-        static int __initdata ints[NR_CPUS];
-        int i;
-
-        str = get_options(str, ARRAY_SIZE(ints), ints);
-        cpus_clear(cpu_isolated_map);
-        for (i = 1; i <= ints[0]; i++)
-                if (ints[i] < NR_CPUS)
-                        cpu_set(ints[i], cpu_isolated_map);
+        cpulist_parse(str, cpu_isolated_map);
         return 1;
 }
 
@@ -6938,42 +7069,43 @@ __setup("isolcpus=", isolated_cpu_setup);
 /*
  * init_sched_build_groups takes the cpumask we wish to span, and a pointer
  * to a function which identifies what group(along with sched group) a CPU
- * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS
- * (due to the fact that we keep track of groups covered with a cpumask_t).
+ * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
+ * (due to the fact that we keep track of groups covered with a struct cpumask).
  *
  * init_sched_build_groups will build a circular linked list of the groups
  * covered by the given span, and will set each group's ->cpumask correctly,
  * and ->cpu_power to 0.
  */
 static void
-init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map,
-                        int (*group_fn)(int cpu, const cpumask_t *cpu_map,
+init_sched_build_groups(const struct cpumask *span,
+                        const struct cpumask *cpu_map,
+                        int (*group_fn)(int cpu, const struct cpumask *cpu_map,
                                         struct sched_group **sg,
-                                        cpumask_t *tmpmask),
-                        cpumask_t *covered, cpumask_t *tmpmask)
+                                        struct cpumask *tmpmask),
+                        struct cpumask *covered, struct cpumask *tmpmask)
 {
         struct sched_group *first = NULL, *last = NULL;
         int i;
 
-        cpus_clear(*covered);
+        cpumask_clear(covered);
 
-        for_each_cpu_mask_nr(i, *span) {
+        for_each_cpu(i, span) {
                 struct sched_group *sg;
                 int group = group_fn(i, cpu_map, &sg, tmpmask);
                 int j;
 
-                if (cpu_isset(i, *covered))
+                if (cpumask_test_cpu(i, covered))
                         continue;
 
-                cpus_clear(sg->cpumask);
+                cpumask_clear(sched_group_cpus(sg));
                 sg->__cpu_power = 0;
 
-                for_each_cpu_mask_nr(j, *span) {
+                for_each_cpu(j, span) {
                         if (group_fn(j, cpu_map, NULL, tmpmask) != group)
                                 continue;
 
-                        cpu_set(j, *covered);
-                        cpu_set(j, sg->cpumask);
+                        cpumask_set_cpu(j, covered);
+                        cpumask_set_cpu(j, sched_group_cpus(sg));
                 }
                 if (!first)
                         first = sg;
@@ -7037,23 +7169,21 @@ static int find_next_best_node(int node, nodemask_t *used_nodes)
  * should be one that prevents unnecessary balancing, but also spreads tasks
  * out optimally.
  */
-static void sched_domain_node_span(int node, cpumask_t *span)
+static void sched_domain_node_span(int node, struct cpumask *span)
 {
         nodemask_t used_nodes;
-        node_to_cpumask_ptr(nodemask, node);
         int i;
 
-        cpus_clear(*span);
+        cpumask_clear(span);
         nodes_clear(used_nodes);
 
-        cpus_or(*span, *span, *nodemask);
+        cpumask_or(span, span, cpumask_of_node(node));
         node_set(node, used_nodes);
 
         for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
                 int next_node = find_next_best_node(node, &used_nodes);
 
-                node_to_cpumask_ptr_next(nodemask, next_node);
-                cpus_or(*span, *span, *nodemask);
+                cpumask_or(span, span, cpumask_of_node(next_node));
         }
 }
 #endif /* CONFIG_NUMA */
@@ -7061,18 +7191,33 @@ static void sched_domain_node_span(int node, cpumask_t *span)
 int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
 
 /*
+ * The cpus mask in sched_group and sched_domain hangs off the end.
+ * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space
+ * for nr_cpu_ids < CONFIG_NR_CPUS.
+ */
+struct static_sched_group {
+        struct sched_group sg;
+        DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
+};
+
+struct static_sched_domain {
+        struct sched_domain sd;
+        DECLARE_BITMAP(span, CONFIG_NR_CPUS);
+};
+
+/*
  * SMT sched-domains:
  */
 #ifdef CONFIG_SCHED_SMT
-static DEFINE_PER_CPU(struct sched_domain, cpu_domains);
-static DEFINE_PER_CPU(struct sched_group, sched_group_cpus);
+static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
+static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
 
 static int
-cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
-                 cpumask_t *unused)
+cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
+                 struct sched_group **sg, struct cpumask *unused)
 {
         if (sg)
-                *sg = &per_cpu(sched_group_cpus, cpu);
+                *sg = &per_cpu(sched_group_cpus, cpu).sg;
         return cpu;
 }
 #endif /* CONFIG_SCHED_SMT */
@@ -7081,56 +7226,53 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
  * multi-core sched-domains:
  */
 #ifdef CONFIG_SCHED_MC
-static DEFINE_PER_CPU(struct sched_domain, core_domains);
-static DEFINE_PER_CPU(struct sched_group, sched_group_core);
+static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
+static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
 #endif /* CONFIG_SCHED_MC */
 
 #if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
 static int
-cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
-                  cpumask_t *mask)
+cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
+                  struct sched_group **sg, struct cpumask *mask)
 {
         int group;
 
-        *mask = per_cpu(cpu_sibling_map, cpu);
-        cpus_and(*mask, *mask, *cpu_map);
-        group = first_cpu(*mask);
+        cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
+        group = cpumask_first(mask);
         if (sg)
-                *sg = &per_cpu(sched_group_core, group);
+                *sg = &per_cpu(sched_group_core, group).sg;
         return group;
 }
 #elif defined(CONFIG_SCHED_MC)
 static int
-cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
-                  cpumask_t *unused)
+cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
+                  struct sched_group **sg, struct cpumask *unused)
 {
         if (sg)
-                *sg = &per_cpu(sched_group_core, cpu);
+                *sg = &per_cpu(sched_group_core, cpu).sg;
         return cpu;
 }
 #endif
 
-static DEFINE_PER_CPU(struct sched_domain, phys_domains);
-static DEFINE_PER_CPU(struct sched_group, sched_group_phys);
+static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
+static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
 
 static int
-cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
-                  cpumask_t *mask)
+cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
+                  struct sched_group **sg, struct cpumask *mask)
 {
         int group;
 #ifdef CONFIG_SCHED_MC
-        *mask = cpu_coregroup_map(cpu);
-        cpus_and(*mask, *mask, *cpu_map);
-        group = first_cpu(*mask);
+        cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
+        group = cpumask_first(mask);
 #elif defined(CONFIG_SCHED_SMT)
-        *mask = per_cpu(cpu_sibling_map, cpu);
-        cpus_and(*mask, *mask, *cpu_map);
-        group = first_cpu(*mask);
+        cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
+        group = cpumask_first(mask);
 #else
         group = cpu;
 #endif
         if (sg)
-                *sg = &per_cpu(sched_group_phys, group);
+                *sg = &per_cpu(sched_group_phys, group).sg;
         return group;
 }
 
@@ -7144,19 +7286,19 @@ static DEFINE_PER_CPU(struct sched_domain, node_domains);
 static struct sched_group ***sched_group_nodes_bycpu;
 
 static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
-static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes);
+static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
 
-static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map,
-                                 struct sched_group **sg, cpumask_t *nodemask)
+static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
+                                 struct sched_group **sg,
+                                 struct cpumask *nodemask)
 {
         int group;
 
-        *nodemask = node_to_cpumask(cpu_to_node(cpu));
-        cpus_and(*nodemask, *nodemask, *cpu_map);
-        group = first_cpu(*nodemask);
+        cpumask_and(nodemask, cpumask_of_node(cpu_to_node(cpu)), cpu_map);
+        group = cpumask_first(nodemask);
 
         if (sg)
-                *sg = &per_cpu(sched_group_allnodes, group);
+                *sg = &per_cpu(sched_group_allnodes, group).sg;
         return group;
 }
 
@@ -7168,11 +7310,11 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
         if (!sg)
                 return;
         do {
-                for_each_cpu_mask_nr(j, sg->cpumask) {
+                for_each_cpu(j, sched_group_cpus(sg)) {
                         struct sched_domain *sd;
 
-                        sd = &per_cpu(phys_domains, j);
-                        if (j != first_cpu(sd->groups->cpumask)) {
+                        sd = &per_cpu(phys_domains, j).sd;
+                        if (j != cpumask_first(sched_group_cpus(sd->groups))) {
                                 /*
                                  * Only add "power" once for each
                                  * physical package.
@@ -7189,11 +7331,12 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
 
 #ifdef CONFIG_NUMA
 /* Free memory allocated for various sched_group structures */
-static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
+static void free_sched_groups(const struct cpumask *cpu_map,
+                              struct cpumask *nodemask)
 {
         int cpu, i;
 
-        for_each_cpu_mask_nr(cpu, *cpu_map) {
+        for_each_cpu(cpu, cpu_map) {
                 struct sched_group **sched_group_nodes
                         = sched_group_nodes_bycpu[cpu];
 
@@ -7203,9 +7346,8 @@ static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
                 for (i = 0; i < nr_node_ids; i++) {
                         struct sched_group *oldsg, *sg = sched_group_nodes[i];
 
-                        *nodemask = node_to_cpumask(i);
-                        cpus_and(*nodemask, *nodemask, *cpu_map);
-                        if (cpus_empty(*nodemask))
+                        cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
+                        if (cpumask_empty(nodemask))
                                 continue;
 
                         if (sg == NULL)
@@ -7223,7 +7365,8 @@ next_sg:
         }
 }
 #else /* !CONFIG_NUMA */
-static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
+static void free_sched_groups(const struct cpumask *cpu_map,
+                              struct cpumask *nodemask)
 {
 }
 #endif /* CONFIG_NUMA */
@@ -7249,7 +7392,7 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
 
         WARN_ON(!sd || !sd->groups);
 
-        if (cpu != first_cpu(sd->groups->cpumask))
+        if (cpu != cpumask_first(sched_group_cpus(sd->groups)))
                 return;
 
         child = sd->child;
@@ -7314,40 +7457,6 @@ SD_INIT_FUNC(CPU)
  SD_INIT_FUNC(MC)
 #endif
 
-/*
- * To minimize stack usage kmalloc room for cpumasks and share the
- * space as the usage in build_sched_domains() dictates.  Used only
- * if the amount of space is significant.
- */
-struct allmasks {
-        cpumask_t tmpmask;                      /* make this one first */
-        union {
-                cpumask_t nodemask;
-                cpumask_t this_sibling_map;
-                cpumask_t this_core_map;
-        };
-        cpumask_t send_covered;
-
-#ifdef CONFIG_NUMA
-        cpumask_t domainspan;
-        cpumask_t covered;
-        cpumask_t notcovered;
-#endif
-};
-
-#if     NR_CPUS > 128
-#define SCHED_CPUMASK_ALLOC             1
-#define SCHED_CPUMASK_FREE(v)           kfree(v)
-#define SCHED_CPUMASK_DECLARE(v)        struct allmasks *v
-#else
-#define SCHED_CPUMASK_ALLOC             0
-#define SCHED_CPUMASK_FREE(v)
-#define SCHED_CPUMASK_DECLARE(v)        struct allmasks _v, *v = &_v
-#endif
-
-#define SCHED_CPUMASK_VAR(v, a)         cpumask_t *v = (cpumask_t *) \
-                        ((unsigned long)(a) + offsetof(struct allmasks, v))
-
 static int default_relax_domain_level = -1;
 
 static int __init setup_relax_domain_level(char *str)
@@ -7387,17 +7496,38 @@ static void set_domain_attribute(struct sched_domain *sd,
  * Build sched domains for a given set of cpus and attach the sched domains
  * to the individual cpus
  */
-static int __build_sched_domains(const cpumask_t *cpu_map,
+static int __build_sched_domains(const struct cpumask *cpu_map,
                                  struct sched_domain_attr *attr)
 {
-        int i;
+        int i, err = -ENOMEM;
         struct root_domain *rd;
-        SCHED_CPUMASK_DECLARE(allmasks);
-        cpumask_t *tmpmask;
+        cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
+                tmpmask;
 #ifdef CONFIG_NUMA
+        cpumask_var_t domainspan, covered, notcovered;
         struct sched_group **sched_group_nodes = NULL;
         int sd_allnodes = 0;
 
+        if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
+                goto out;
+        if (!alloc_cpumask_var(&covered, GFP_KERNEL))
+                goto free_domainspan;
+        if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
+                goto free_covered;
+#endif
+
+        if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
+                goto free_notcovered;
+        if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
+                goto free_nodemask;
+        if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
+                goto free_this_sibling_map;
+        if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
+                goto free_this_core_map;
+        if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
+                goto free_send_covered;
+
+#ifdef CONFIG_NUMA
         /*
          * Allocate the per-node list of sched groups
          */
@@ -7405,55 +7535,35 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
                                     GFP_KERNEL);
         if (!sched_group_nodes) {
                 printk(KERN_WARNING "Can not alloc sched group node list\n");
-                return -ENOMEM;
+                goto free_tmpmask;
         }
 #endif
 
         rd = alloc_rootdomain();
         if (!rd) {
                 printk(KERN_WARNING "Cannot alloc root domain\n");
-#ifdef CONFIG_NUMA
-                kfree(sched_group_nodes);
-#endif
-                return -ENOMEM;
+                goto free_sched_groups;
         }
 
-#if SCHED_CPUMASK_ALLOC
-        /* get space for all scratch cpumask variables */
-        allmasks = kmalloc(sizeof(*allmasks), GFP_KERNEL);
-        if (!allmasks) {
-                printk(KERN_WARNING "Cannot alloc cpumask array\n");
-                kfree(rd);
 #ifdef CONFIG_NUMA
-                kfree(sched_group_nodes);
-#endif
-                return -ENOMEM;
-        }
-#endif
-        tmpmask = (cpumask_t *)allmasks;
-
-
-#ifdef CONFIG_NUMA
-        sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes;
+        sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
 #endif
 
         /*
          * Set up domains for cpus specified by the cpu_map.
          */
-        for_each_cpu_mask_nr(i, *cpu_map) {
+        for_each_cpu(i, cpu_map) {
                 struct sched_domain *sd = NULL, *p;
-                SCHED_CPUMASK_VAR(nodemask, allmasks);
 
-                *nodemask = node_to_cpumask(cpu_to_node(i));
-                cpus_and(*nodemask, *nodemask, *cpu_map);
+                cpumask_and(nodemask, cpumask_of_node(cpu_to_node(i)), cpu_map);
 
 #ifdef CONFIG_NUMA
-                if (cpus_weight(*cpu_map) >
-                                SD_NODES_PER_DOMAIN*cpus_weight(*nodemask)) {
+                if (cpumask_weight(cpu_map) >
+                                SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
                         sd = &per_cpu(allnodes_domains, i);
                         SD_INIT(sd, ALLNODES);
                         set_domain_attribute(sd, attr);
-                        sd->span = *cpu_map;
+                        cpumask_copy(sched_domain_span(sd), cpu_map);
                         cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
                         p = sd;
                         sd_allnodes = 1;
@@ -7463,18 +7573,19 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
                 sd = &per_cpu(node_domains, i);
                 SD_INIT(sd, NODE);
                 set_domain_attribute(sd, attr);
-                sched_domain_node_span(cpu_to_node(i), &sd->span);
+                sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
                 sd->parent = p;
                 if (p)
                         p->child = sd;
-                cpus_and(sd->span, sd->span, *cpu_map);
+                cpumask_and(sched_domain_span(sd),
+                            sched_domain_span(sd), cpu_map);
 #endif
 
                 p = sd;
-                sd = &per_cpu(phys_domains, i);
+                sd = &per_cpu(phys_domains, i).sd;
                 SD_INIT(sd, CPU);
                 set_domain_attribute(sd, attr);
-                sd->span = *nodemask;
+                cpumask_copy(sched_domain_span(sd), nodemask);
                 sd->parent = p;
                 if (p)
                         p->child = sd;
@@ -7482,11 +7593,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
 
 #ifdef CONFIG_SCHED_MC
                 p = sd;
-                sd = &per_cpu(core_domains, i);
+                sd = &per_cpu(core_domains, i).sd;
                 SD_INIT(sd, MC);
                 set_domain_attribute(sd, attr);
-                sd->span = cpu_coregroup_map(i);
-                cpus_and(sd->span, sd->span, *cpu_map);
+                cpumask_and(sched_domain_span(sd), cpu_map,
+                                                   cpu_coregroup_mask(i));
                 sd->parent = p;
                 p->child = sd;
                 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7494,11 +7605,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
 
 #ifdef CONFIG_SCHED_SMT
                 p = sd;
-                sd = &per_cpu(cpu_domains, i);
+                sd = &per_cpu(cpu_domains, i).sd;
                 SD_INIT(sd, SIBLING);
                 set_domain_attribute(sd, attr);
-                sd->span = per_cpu(cpu_sibling_map, i);
-                cpus_and(sd->span, sd->span, *cpu_map);
+                cpumask_and(sched_domain_span(sd),
+                            &per_cpu(cpu_sibling_map, i), cpu_map);
                 sd->parent = p;
                 p->child = sd;
                 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7507,13 +7618,10 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
 
 #ifdef CONFIG_SCHED_SMT
         /* Set up CPU (sibling) groups */
-        for_each_cpu_mask_nr(i, *cpu_map) {
-                SCHED_CPUMASK_VAR(this_sibling_map, allmasks);
-                SCHED_CPUMASK_VAR(send_covered, allmasks);
-
-                *this_sibling_map = per_cpu(cpu_sibling_map, i);
-                cpus_and(*this_sibling_map, *this_sibling_map, *cpu_map);
-                if (i != first_cpu(*this_sibling_map))
+        for_each_cpu(i, cpu_map) {
+                cpumask_and(this_sibling_map,
+                            &per_cpu(cpu_sibling_map, i), cpu_map);
+                if (i != cpumask_first(this_sibling_map))
                         continue;
 
                 init_sched_build_groups(this_sibling_map, cpu_map,
@@ -7524,13 +7632,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
 
 #ifdef CONFIG_SCHED_MC
         /* Set up multi-core groups */
-        for_each_cpu_mask_nr(i, *cpu_map) {
-                SCHED_CPUMASK_VAR(this_core_map, allmasks);
-                SCHED_CPUMASK_VAR(send_covered, allmasks);
-
-                *this_core_map = cpu_coregroup_map(i);
-                cpus_and(*this_core_map, *this_core_map, *cpu_map);
-                if (i != first_cpu(*this_core_map))
+        for_each_cpu(i, cpu_map) {
+                cpumask_and(this_core_map, cpu_coregroup_mask(i), cpu_map);
+                if (i != cpumask_first(this_core_map))
                         continue;
 
                 init_sched_build_groups(this_core_map, cpu_map,
@@ -7541,12 +7645,8 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
 
         /* Set up physical groups */
         for (i = 0; i < nr_node_ids; i++) {
-                SCHED_CPUMASK_VAR(nodemask, allmasks);
-                SCHED_CPUMASK_VAR(send_covered, allmasks);
-
-                *nodemask = node_to_cpumask(i);
-                cpus_and(*nodemask, *nodemask, *cpu_map);
-                if (cpus_empty(*nodemask))
+                cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
+                if (cpumask_empty(nodemask))
                         continue;
 
                 init_sched_build_groups(nodemask, cpu_map,
@@ -7557,8 +7657,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
 #ifdef CONFIG_NUMA
         /* Set up node groups */
         if (sd_allnodes) {
-                SCHED_CPUMASK_VAR(send_covered, allmasks);
-
                 init_sched_build_groups(cpu_map, cpu_map,
                                         &cpu_to_allnodes_group,
                                         send_covered, tmpmask);
@@ -7567,58 +7665,53 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
         for (i = 0; i < nr_node_ids; i++) {
                 /* Set up node groups */
                 struct sched_group *sg, *prev;
-                SCHED_CPUMASK_VAR(nodemask, allmasks);
-                SCHED_CPUMASK_VAR(domainspan, allmasks);
-                SCHED_CPUMASK_VAR(covered, allmasks);
                 int j;
 
-                *nodemask = node_to_cpumask(i);
-                cpus_clear(*covered);
-
-                cpus_and(*nodemask, *nodemask, *cpu_map);
-                if (cpus_empty(*nodemask)) {
+                cpumask_clear(covered);
+                cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
+                if (cpumask_empty(nodemask)) {
                         sched_group_nodes[i] = NULL;
                         continue;
                 }
 
                 sched_domain_node_span(i, domainspan);
-                cpus_and(*domainspan, *domainspan, *cpu_map);
+                cpumask_and(domainspan, domainspan, cpu_map);
 
-                sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i);
+                sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
+                                  GFP_KERNEL, i);
                 if (!sg) {
                         printk(KERN_WARNING "Can not alloc domain group for "
                                 "node %d\n", i);
                         goto error;
                 }
                 sched_group_nodes[i] = sg;
-                for_each_cpu_mask_nr(j, *nodemask) {
+                for_each_cpu(j, nodemask) {
                         struct sched_domain *sd;
 
                         sd = &per_cpu(node_domains, j);
                         sd->groups = sg;
                 }
                 sg->__cpu_power = 0;
-                sg->cpumask = *nodemask;
+                cpumask_copy(sched_group_cpus(sg), nodemask);
                 sg->next = sg;
-                cpus_or(*covered, *covered, *nodemask);
+                cpumask_or(covered, covered, nodemask);
                 prev = sg;
 
                 for (j = 0; j < nr_node_ids; j++) {
-                        SCHED_CPUMASK_VAR(notcovered, allmasks);
                         int n = (i + j) % nr_node_ids;
-                        node_to_cpumask_ptr(pnodemask, n);
 
-                        cpus_complement(*notcovered, *covered);
-                        cpus_and(*tmpmask, *notcovered, *cpu_map);
-                        cpus_and(*tmpmask, *tmpmask, *domainspan);
-                        if (cpus_empty(*tmpmask))
+                        cpumask_complement(notcovered, covered);
+                        cpumask_and(tmpmask, notcovered, cpu_map);
+                        cpumask_and(tmpmask, tmpmask, domainspan);
+                        if (cpumask_empty(tmpmask))
                                 break;
 
-                        cpus_and(*tmpmask, *tmpmask, *pnodemask);
-                        if (cpus_empty(*tmpmask))
+                        cpumask_and(tmpmask, tmpmask, cpumask_of_node(n));
+                        if (cpumask_empty(tmpmask))
                                 continue;
 
-                        sg = kmalloc_node(sizeof(struct sched_group),
+                        sg = kmalloc_node(sizeof(struct sched_group) +
+                                          cpumask_size(),
                                           GFP_KERNEL, i);
                         if (!sg) {
                                 printk(KERN_WARNING
@@ -7626,9 +7719,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
                                 goto error;
                         }
                         sg->__cpu_power = 0;
-                        sg->cpumask = *tmpmask;
+                        cpumask_copy(sched_group_cpus(sg), tmpmask);
                         sg->next = prev->next;
-                        cpus_or(*covered, *covered, *tmpmask);
+                        cpumask_or(covered, covered, tmpmask);
                         prev->next = sg;
                         prev = sg;
                 }
@@ -7637,22 +7730,22 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
 
         /* Calculate CPU power for physical packages and nodes */
 #ifdef CONFIG_SCHED_SMT
-        for_each_cpu_mask_nr(i, *cpu_map) {
-                struct sched_domain *sd = &per_cpu(cpu_domains, i);
+        for_each_cpu(i, cpu_map) {
+                struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
 
                 init_sched_groups_power(i, sd);
         }
 #endif
 #ifdef CONFIG_SCHED_MC
-        for_each_cpu_mask_nr(i, *cpu_map) {
-                struct sched_domain *sd = &per_cpu(core_domains, i);
+        for_each_cpu(i, cpu_map) {
+                struct sched_domain *sd = &per_cpu(core_domains, i).sd;
 
                 init_sched_groups_power(i, sd);
         }
 #endif
 
-        for_each_cpu_mask_nr(i, *cpu_map) {
-                struct sched_domain *sd = &per_cpu(phys_domains, i);
+        for_each_cpu(i, cpu_map) {
+                struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
 
                 init_sched_groups_power(i, sd);
         }
@@ -7664,56 +7757,87 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
         if (sd_allnodes) {
                 struct sched_group *sg;
 
-                cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg,
+                cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
                                                                 tmpmask);
                 init_numa_sched_groups_power(sg);
         }
 #endif
 
         /* Attach the domains */
-        for_each_cpu_mask_nr(i, *cpu_map) {
+        for_each_cpu(i, cpu_map) {
                 struct sched_domain *sd;
 #ifdef CONFIG_SCHED_SMT
-                sd = &per_cpu(cpu_domains, i);
+                sd = &per_cpu(cpu_domains, i).sd;
 #elif defined(CONFIG_SCHED_MC)
-                sd = &per_cpu(core_domains, i);
+                sd = &per_cpu(core_domains, i).sd;
 #else
-                sd = &per_cpu(phys_domains, i);
+                sd = &per_cpu(phys_domains, i).sd;
 #endif
                 cpu_attach_domain(sd, rd, i);
         }
 
-        SCHED_CPUMASK_FREE((void *)allmasks);
-        return 0;
+        err = 0;
+
+free_tmpmask:
+        free_cpumask_var(tmpmask);
+free_send_covered:
+        free_cpumask_var(send_covered);
+free_this_core_map:
+        free_cpumask_var(this_core_map);
+free_this_sibling_map:
+        free_cpumask_var(this_sibling_map);
+free_nodemask:
+        free_cpumask_var(nodemask);
+free_notcovered:
+#ifdef CONFIG_NUMA
+        free_cpumask_var(notcovered);
+free_covered:
+        free_cpumask_var(covered);
+free_domainspan:
+        free_cpumask_var(domainspan);
+out:
+#endif
+        return err;
+
+free_sched_groups:
+#ifdef CONFIG_NUMA
+        kfree(sched_group_nodes);
+#endif
+        goto free_tmpmask;
 
 #ifdef CONFIG_NUMA
 error:
         free_sched_groups(cpu_map, tmpmask);
-        SCHED_CPUMASK_FREE((void *)allmasks);
-        kfree(rd);
-        return -ENOMEM;
+        free_rootdomain(rd);
+        goto free_tmpmask;
 #endif
 }
 
-static int build_sched_domains(const cpumask_t *cpu_map)
+static int build_sched_domains(const struct cpumask *cpu_map)
 {
         return __build_sched_domains(cpu_map, NULL);
 }
 
-static cpumask_t *doms_cur;     /* current sched domains */
+static struct cpumask *doms_cur;        /* current sched domains */
 static int ndoms_cur;           /* number of sched domains in 'doms_cur' */
 static struct sched_domain_attr *dattr_cur;
                                 /* attribues of custom domains in 'doms_cur' */
 
 /*
  * Special case: If a kmalloc of a doms_cur partition (array of
- * cpumask_t) fails, then fallback to a single sched domain,
- * as determined by the single cpumask_t fallback_doms.
+ * cpumask) fails, then fallback to a single sched domain,
+ * as determined by the single cpumask fallback_doms.
  */
-static cpumask_t fallback_doms;
+static cpumask_var_t fallback_doms;
 
-void __attribute__((weak)) arch_update_cpu_topology(void)
+/*
+ * arch_update_cpu_topology lets virtualized architectures update the
+ * cpu core maps. It is supposed to return 1 if the topology changed
+ * or 0 if it stayed the same.
+ */
+int __attribute__((weak)) arch_update_cpu_topology(void)
 {
+        return 0;
 }
 
 /*
@@ -7721,16 +7845,16 @@ void __attribute__((weak)) arch_update_cpu_topology(void)
  * For now this just excludes isolated cpus, but could be used to
  * exclude other special cases in the future.
  */
-static int arch_init_sched_domains(const cpumask_t *cpu_map)
+static int arch_init_sched_domains(const struct cpumask *cpu_map)
 {
         int err;
 
         arch_update_cpu_topology();
         ndoms_cur = 1;
-        doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL);
+        doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
         if (!doms_cur)
-                doms_cur = &fallback_doms;
-        cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map);
+                doms_cur = fallback_doms;
+        cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
         dattr_cur = NULL;
         err = build_sched_domains(doms_cur);
         register_sched_domain_sysctl();
@@ -7738,8 +7862,8 @@ static int arch_init_sched_domains(const cpumask_t *cpu_map)
         return err;
 }
 
-static void arch_destroy_sched_domains(const cpumask_t *cpu_map,
-                                       cpumask_t *tmpmask)
+static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
+                                       struct cpumask *tmpmask)
 {
         free_sched_groups(cpu_map, tmpmask);
 }
@@ -7748,17 +7872,16 @@ static void arch_destroy_sched_domains(const cpumask_t *cpu_map,
  * Detach sched domains from a group of cpus specified in cpu_map
  * These cpus will now be attached to the NULL domain
  */
-static void detach_destroy_domains(const cpumask_t *cpu_map)
+static void detach_destroy_domains(const struct cpumask *cpu_map)
 {
-        cpumask_t tmpmask;
+        /* Save because hotplug lock held. */
+        static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
         int i;
 
-        unregister_sched_domain_sysctl();
-
-        for_each_cpu_mask_nr(i, *cpu_map)
+        for_each_cpu(i, cpu_map)
                 cpu_attach_domain(NULL, &def_root_domain, i);
         synchronize_sched();
-        arch_destroy_sched_domains(cpu_map, &tmpmask);
+        arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
 }
 
 /* handle null as "default" */
@@ -7783,7 +7906,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
  * doms_new[] to the current sched domain partitioning, doms_cur[].
  * It destroys each deleted domain and builds each new domain.
  *
- * 'doms_new' is an array of cpumask_t's of length 'ndoms_new'.
+ * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
  * The masks don't intersect (don't overlap.) We should setup one
  * sched domain for each mask. CPUs not in any of the cpumasks will
  * not be load balanced. If the same cpumask appears both in the
@@ -7797,28 +7920,33 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
  * the single partition 'fallback_doms', it also forces the domains
  * to be rebuilt.
  *
- * If doms_new == NULL it will be replaced with cpu_online_map.
+ * If doms_new == NULL it will be replaced with cpu_online_mask.
  * ndoms_new == 0 is a special case for destroying existing domains,
  * and it will not create the default domain.
  *
  * Call with hotplug lock held
  */
-void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
+/* FIXME: Change to struct cpumask *doms_new[] */
+void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
                              struct sched_domain_attr *dattr_new)
 {
         int i, j, n;
+        int new_topology;
 
         mutex_lock(&sched_domains_mutex);
 
         /* always unregister in case we don't destroy any domains */
         unregister_sched_domain_sysctl();
 
+        /* Let architecture update cpu core mappings. */
+        new_topology = arch_update_cpu_topology();
+
         n = doms_new ? ndoms_new : 0;
 
         /* Destroy deleted domains */
         for (i = 0; i < ndoms_cur; i++) {
-                for (j = 0; j < n; j++) {
-                        if (cpus_equal(doms_cur[i], doms_new[j])
+                for (j = 0; j < n && !new_topology; j++) {
+                        if (cpumask_equal(&doms_cur[i], &doms_new[j])
                             && dattrs_equal(dattr_cur, i, dattr_new, j))
                                 goto match1;
                 }
@@ -7830,15 +7958,15 @@ match1:
 
         if (doms_new == NULL) {
                 ndoms_cur = 0;
-                doms_new = &fallback_doms;
-                cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map);
-                dattr_new = NULL;
+                doms_new = fallback_doms;
+                cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
+                WARN_ON_ONCE(dattr_new);
         }
 
         /* Build new domains */
         for (i = 0; i < ndoms_new; i++) {
-                for (j = 0; j < ndoms_cur; j++) {
-                        if (cpus_equal(doms_new[i], doms_cur[j])
+                for (j = 0; j < ndoms_cur && !new_topology; j++) {
+                        if (cpumask_equal(&doms_new[i], &doms_cur[j])
                             && dattrs_equal(dattr_new, i, dattr_cur, j))
                                 goto match2;
                 }
@@ -7850,7 +7978,7 @@ match2:
         }
 
         /* Remember the new sched domains */
-        if (doms_cur != &fallback_doms)
+        if (doms_cur != fallback_doms)
                 kfree(doms_cur);
         kfree(dattr_cur);       /* kfree(NULL) is safe */
         doms_cur = doms_new;
@@ -7863,7 +7991,7 @@ match2:
 }
 
 #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
-int arch_reinit_sched_domains(void)
+static void arch_reinit_sched_domains(void)
 {
         get_online_cpus();
 
@@ -7872,25 +8000,33 @@ int arch_reinit_sched_domains(void)
 
         rebuild_sched_domains();
         put_online_cpus();
-
-        return 0;
 }
 
 static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
 {
-        int ret;
+        unsigned int level = 0;
 
-        if (buf[0] != '0' && buf[0] != '1')
+        if (sscanf(buf, "%u", &level) != 1)
+                return -EINVAL;
+
+        /*
+         * level is always be positive so don't check for
+         * level < POWERSAVINGS_BALANCE_NONE which is 0
+         * What happens on 0 or 1 byte write,
+         * need to check for count as well?
+         */
+
+        if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
                 return -EINVAL;
 
         if (smt)
-                sched_smt_power_savings = (buf[0] == '1');
+                sched_smt_power_savings = level;
         else
-                sched_mc_power_savings = (buf[0] == '1');
+                sched_mc_power_savings = level;
 
-        ret = arch_reinit_sched_domains();
+        arch_reinit_sched_domains();
 
-        return ret ? ret : count;
+        return count;
 }
 
 #ifdef CONFIG_SCHED_MC
@@ -7925,7 +8061,7 @@ static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
                    sched_smt_power_savings_store);
 #endif
 
-int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
+int __init sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
 {
         int err = 0;
 
@@ -7990,7 +8126,9 @@ static int update_runtime(struct notifier_block *nfb,
 
 void __init sched_init_smp(void)
 {
-        cpumask_t non_isolated_cpus;
+        cpumask_var_t non_isolated_cpus;
+
+        alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
 
 #if defined(CONFIG_NUMA)
         sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
@@ -7999,10 +8137,10 @@ void __init sched_init_smp(void)
 #endif
         get_online_cpus();
         mutex_lock(&sched_domains_mutex);
-        arch_init_sched_domains(&cpu_online_map);
-        cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map);
-        if (cpus_empty(non_isolated_cpus))
-                cpu_set(smp_processor_id(), non_isolated_cpus);
+        arch_init_sched_domains(cpu_online_mask);
+        cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
+        if (cpumask_empty(non_isolated_cpus))
+                cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
         mutex_unlock(&sched_domains_mutex);
         put_online_cpus();
 
@@ -8017,9 +8155,13 @@ void __init sched_init_smp(void)
         init_hrtick();
 
         /* Move init over to a non-isolated CPU */
-        if (set_cpus_allowed_ptr(current, &non_isolated_cpus) < 0)
+        if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
                 BUG();
         sched_init_granularity();
+        free_cpumask_var(non_isolated_cpus);
+
+        alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
+        init_sched_rt_class();
 }
 #else
 void __init sched_init_smp(void)
@@ -8334,6 +8476,15 @@ void __init sched_init(void)
          */
         current->sched_class = &fair_sched_class;
 
+        /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
+        alloc_bootmem_cpumask_var(&nohz_cpu_mask);
+#ifdef CONFIG_SMP
+#ifdef CONFIG_NO_HZ
+        alloc_bootmem_cpumask_var(&nohz.cpu_mask);
+#endif
+        alloc_bootmem_cpumask_var(&cpu_isolated_map);
+#endif /* SMP */
+
         scheduler_running = 1;
 }
 
@@ -8492,7 +8643,7 @@ static
 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
 {
         struct cfs_rq *cfs_rq;
-        struct sched_entity *se, *parent_se;
+        struct sched_entity *se;
         struct rq *rq;
         int i;
 
@@ -8508,18 +8659,17 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
         for_each_possible_cpu(i) {
                 rq = cpu_rq(i);
 
-                cfs_rq = kmalloc_node(sizeof(struct cfs_rq),
-                                GFP_KERNEL|__GFP_ZERO, cpu_to_node(i));
+                cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
+                                      GFP_KERNEL, cpu_to_node(i));
                 if (!cfs_rq)
                         goto err;
 
-                se = kmalloc_node(sizeof(struct sched_entity),
-                                GFP_KERNEL|__GFP_ZERO, cpu_to_node(i));
+                se = kzalloc_node(sizeof(struct sched_entity),
+                                  GFP_KERNEL, cpu_to_node(i));
                 if (!se)
                         goto err;
 
-                parent_se = parent ? parent->se[i] : NULL;
-                init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent_se);
+                init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
         }
 
         return 1;
@@ -8580,7 +8730,7 @@ static
 int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
 {
         struct rt_rq *rt_rq;
-        struct sched_rt_entity *rt_se, *parent_se;
+        struct sched_rt_entity *rt_se;
         struct rq *rq;
         int i;
 
@@ -8597,18 +8747,17 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
         for_each_possible_cpu(i) {
                 rq = cpu_rq(i);
 
-                rt_rq = kmalloc_node(sizeof(struct rt_rq),
-                                GFP_KERNEL|__GFP_ZERO, cpu_to_node(i));
+                rt_rq = kzalloc_node(sizeof(struct rt_rq),
+                                     GFP_KERNEL, cpu_to_node(i));
                 if (!rt_rq)
                         goto err;
 
-                rt_se = kmalloc_node(sizeof(struct sched_rt_entity),
-                                GFP_KERNEL|__GFP_ZERO, cpu_to_node(i));
+                rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
+                                     GFP_KERNEL, cpu_to_node(i));
                 if (!rt_se)
                         goto err;
 
-                parent_se = parent ? parent->rt_se[i] : NULL;
-                init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent_se);
+                init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
         }
 
         return 1;
@@ -9251,11 +9400,12 @@ struct cgroup_subsys cpu_cgroup_subsys = {
  * (balbir@in.ibm.com).
  */
 
-/* track cpu usage of a group of tasks */
+/* track cpu usage of a group of tasks and its child groups */
 struct cpuacct {
         struct cgroup_subsys_state css;
         /* cpuusage holds pointer to a u64-type object on every cpu */
         u64 *cpuusage;
+        struct cpuacct *parent;
 };
 
 struct cgroup_subsys cpuacct_subsys;
@@ -9289,6 +9439,9 @@ static struct cgroup_subsys_state *cpuacct_create(
                 return ERR_PTR(-ENOMEM);
         }
 
+        if (cgrp->parent)
+                ca->parent = cgroup_ca(cgrp->parent);
+
         return &ca->css;
 }
 
@@ -9302,6 +9455,41 @@ cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
         kfree(ca);
 }
 
+static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
+{
+        u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
+        u64 data;
+
+#ifndef CONFIG_64BIT
+        /*
+         * Take rq->lock to make 64-bit read safe on 32-bit platforms.
+         */
+        spin_lock_irq(&cpu_rq(cpu)->lock);
+        data = *cpuusage;
+        spin_unlock_irq(&cpu_rq(cpu)->lock);
+#else
+        data = *cpuusage;
+#endif
+
+        return data;
+}
+
+static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
+{
+        u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
+
+#ifndef CONFIG_64BIT
+        /*
+         * Take rq->lock to make 64-bit write safe on 32-bit platforms.
+         */
+        spin_lock_irq(&cpu_rq(cpu)->lock);
+        *cpuusage = val;
+        spin_unlock_irq(&cpu_rq(cpu)->lock);
+#else
+        *cpuusage = val;
+#endif
+}
+
 /* return total cpu usage (in nanoseconds) of a group */
 static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
 {
@@ -9309,17 +9497,8 @@ static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
         u64 totalcpuusage = 0;
         int i;
 
-        for_each_possible_cpu(i) {
-                u64 *cpuusage = percpu_ptr(ca->cpuusage, i);
-
-                /*
-                 * Take rq->lock to make 64-bit addition safe on 32-bit
-                 * platforms.
-                 */
-                spin_lock_irq(&cpu_rq(i)->lock);
-                totalcpuusage += *cpuusage;
-                spin_unlock_irq(&cpu_rq(i)->lock);
-        }
+        for_each_present_cpu(i)
+                totalcpuusage += cpuacct_cpuusage_read(ca, i);
 
         return totalcpuusage;
 }
@@ -9336,23 +9515,39 @@ static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
                 goto out;
         }
 
-        for_each_possible_cpu(i) {
-                u64 *cpuusage = percpu_ptr(ca->cpuusage, i);
+        for_each_present_cpu(i)
+                cpuacct_cpuusage_write(ca, i, 0);
 
-                spin_lock_irq(&cpu_rq(i)->lock);
-                *cpuusage = 0;
-                spin_unlock_irq(&cpu_rq(i)->lock);
-        }
 out:
         return err;
 }
 
+static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
+                                   struct seq_file *m)
+{
+        struct cpuacct *ca = cgroup_ca(cgroup);
+        u64 percpu;
+        int i;
+
+        for_each_present_cpu(i) {
+                percpu = cpuacct_cpuusage_read(ca, i);
+                seq_printf(m, "%llu ", (unsigned long long) percpu);
+        }
+        seq_printf(m, "\n");
+        return 0;
+}
+
 static struct cftype files[] = {
         {
                 .name = "usage",
                 .read_u64 = cpuusage_read,
                 .write_u64 = cpuusage_write,
         },
+        {
+                .name = "usage_percpu",
+                .read_seq_string = cpuacct_percpu_seq_read,
+        },
+
 };
 
 static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
@@ -9368,14 +9563,16 @@ static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
 static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
 {
         struct cpuacct *ca;
+        int cpu;
 
         if (!cpuacct_subsys.active)
                 return;
 
+        cpu = task_cpu(tsk);
         ca = task_ca(tsk);
-        if (ca) {
-                u64 *cpuusage = percpu_ptr(ca->cpuusage, task_cpu(tsk));
 
+        for (; ca; ca = ca->parent) {
+                u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
                 *cpuusage += cputime;
         }
 }