1 files changed, 187 insertions, 44 deletions

diff --git a/kernel/sched.c b/kernel/sched.c
index 73513f4e19df..26efa475bdc1 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -231,13 +231,20 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
 
         spin_lock(&rt_b->rt_runtime_lock);
         for (;;) {
+                unsigned long delta;
+                ktime_t soft, hard;
+
                 if (hrtimer_active(&rt_b->rt_period_timer))
                         break;
 
                 now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
                 hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
-                hrtimer_start_expires(&rt_b->rt_period_timer,
-                                HRTIMER_MODE_ABS);
+
+                soft = hrtimer_get_softexpires(&rt_b->rt_period_timer);
+                hard = hrtimer_get_expires(&rt_b->rt_period_timer);
+                delta = ktime_to_ns(ktime_sub(hard, soft));
+                __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta,
+                                HRTIMER_MODE_ABS, 0);
         }
         spin_unlock(&rt_b->rt_runtime_lock);
 }
@@ -1110,7 +1117,7 @@ static void hrtick_start(struct rq *rq, u64 delay)
         if (rq == this_rq()) {
                 hrtimer_restart(timer);
         } else if (!rq->hrtick_csd_pending) {
-                __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd);
+                __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0);
                 rq->hrtick_csd_pending = 1;
         }
 }
@@ -1146,7 +1153,8 @@ static __init void init_hrtick(void)
  */
 static void hrtick_start(struct rq *rq, u64 delay)
 {
-        hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL);
+        __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
+                        HRTIMER_MODE_REL, 0);
 }
 
 static inline void init_hrtick(void)
@@ -1410,10 +1418,22 @@ iter_move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
                    struct rq_iterator *iterator);
 #endif
 
+/* Time spent by the tasks of the cpu accounting group executing in ... */
+enum cpuacct_stat_index {
+        CPUACCT_STAT_USER,      /* ... user mode */
+        CPUACCT_STAT_SYSTEM,    /* ... kernel mode */
+
+        CPUACCT_STAT_NSTATS,
+};
+
 #ifdef CONFIG_CGROUP_CPUACCT
 static void cpuacct_charge(struct task_struct *tsk, u64 cputime);
+static void cpuacct_update_stats(struct task_struct *tsk,
+                enum cpuacct_stat_index idx, cputime_t val);
 #else
 static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
+static inline void cpuacct_update_stats(struct task_struct *tsk,
+                enum cpuacct_stat_index idx, cputime_t val) {}
 #endif
 
 static inline void inc_cpu_load(struct rq *rq, unsigned long load)
@@ -3818,19 +3838,23 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
  */
 #define MAX_PINNED_INTERVAL     512
 
+/* Working cpumask for load_balance and load_balance_newidle. */
+static DEFINE_PER_CPU(cpumask_var_t, load_balance_tmpmask);
+
 /*
  * Check this_cpu to ensure it is balanced within domain. Attempt to move
  * tasks if there is an imbalance.
  */
 static int load_balance(int this_cpu, struct rq *this_rq,
                         struct sched_domain *sd, enum cpu_idle_type idle,
-                        int *balance, struct cpumask *cpus)
+                        int *balance)
 {
         int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
         struct sched_group *group;
         unsigned long imbalance;
         struct rq *busiest;
         unsigned long flags;
+        struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
 
         cpumask_setall(cpus);
 
@@ -3985,8 +4009,7 @@ out:
  * this_rq is locked.
  */
 static int
-load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
-                        struct cpumask *cpus)
+load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
 {
         struct sched_group *group;
         struct rq *busiest = NULL;
@@ -3994,6 +4017,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
         int ld_moved = 0;
         int sd_idle = 0;
         int all_pinned = 0;
+        struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
 
         cpumask_setall(cpus);
 
@@ -4134,10 +4158,6 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
         struct sched_domain *sd;
         int pulled_task = 0;
         unsigned long next_balance = jiffies + HZ;
-        cpumask_var_t tmpmask;
-
-        if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
-                return;
 
         for_each_domain(this_cpu, sd) {
                 unsigned long interval;
@@ -4148,7 +4168,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
                 if (sd->flags & SD_BALANCE_NEWIDLE)
                         /* If we've pulled tasks over stop searching: */
                         pulled_task = load_balance_newidle(this_cpu, this_rq,
-                                                           sd, tmpmask);
+                                                           sd);
 
                 interval = msecs_to_jiffies(sd->balance_interval);
                 if (time_after(next_balance, sd->last_balance + interval))
@@ -4163,7 +4183,6 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
                  */
                 this_rq->next_balance = next_balance;
         }
-        free_cpumask_var(tmpmask);
 }
 
 /*
@@ -4313,11 +4332,6 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
         unsigned long next_balance = jiffies + 60*HZ;
         int update_next_balance = 0;
         int need_serialize;
-        cpumask_var_t tmp;
-
-        /* Fails alloc?  Rebalancing probably not a priority right now. */
-        if (!alloc_cpumask_var(&tmp, GFP_ATOMIC))
-                return;
 
         for_each_domain(cpu, sd) {
                 if (!(sd->flags & SD_LOAD_BALANCE))
@@ -4342,7 +4356,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
                 }
 
                 if (time_after_eq(jiffies, sd->last_balance + interval)) {
-                        if (load_balance(cpu, rq, sd, idle, &balance, tmp)) {
+                        if (load_balance(cpu, rq, sd, idle, &balance)) {
                                 /*
                                  * We've pulled tasks over so either we're no
                                  * longer idle, or one of our SMT siblings is
@@ -4376,8 +4390,6 @@ out:
          */
         if (likely(update_next_balance))
                 rq->next_balance = next_balance;
-
-        free_cpumask_var(tmp);
 }
 
 /*
@@ -4511,9 +4523,25 @@ DEFINE_PER_CPU(struct kernel_stat, kstat);
 EXPORT_PER_CPU_SYMBOL(kstat);
 
 /*
- * Return any ns on the sched_clock that have not yet been banked in
+ * Return any ns on the sched_clock that have not yet been accounted in
  * @p in case that task is currently running.
+ *
+ * Called with task_rq_lock() held on @rq.
  */
+static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
+{
+        u64 ns = 0;
+
+        if (task_current(rq, p)) {
+                update_rq_clock(rq);
+                ns = rq->clock - p->se.exec_start;
+                if ((s64)ns < 0)
+                        ns = 0;
+        }
+
+        return ns;
+}
+
 unsigned long long task_delta_exec(struct task_struct *p)
 {
         unsigned long flags;
@@ -4521,16 +4549,49 @@ unsigned long long task_delta_exec(struct task_struct *p)
         u64 ns = 0;
 
         rq = task_rq_lock(p, &flags);
+        ns = do_task_delta_exec(p, rq);
+        task_rq_unlock(rq, &flags);
 
-        if (task_current(rq, p)) {
-                u64 delta_exec;
+        return ns;
+}
 
-                update_rq_clock(rq);
-                delta_exec = rq->clock - p->se.exec_start;
-                if ((s64)delta_exec > 0)
-                        ns = delta_exec;
-        }
+/*
+ * Return accounted runtime for the task.
+ * In case the task is currently running, return the runtime plus current's
+ * pending runtime that have not been accounted yet.
+ */
+unsigned long long task_sched_runtime(struct task_struct *p)
+{
+        unsigned long flags;
+        struct rq *rq;
+        u64 ns = 0;
+
+        rq = task_rq_lock(p, &flags);
+        ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq);
+        task_rq_unlock(rq, &flags);
+
+        return ns;
+}
 
+/*
+ * Return sum_exec_runtime for the thread group.
+ * In case the task is currently running, return the sum plus current's
+ * pending runtime that have not been accounted yet.
+ *
+ * Note that the thread group might have other running tasks as well,
+ * so the return value not includes other pending runtime that other
+ * running tasks might have.
+ */
+unsigned long long thread_group_sched_runtime(struct task_struct *p)
+{
+        struct task_cputime totals;
+        unsigned long flags;
+        struct rq *rq;
+        u64 ns;
+
+        rq = task_rq_lock(p, &flags);
+        thread_group_cputime(p, &totals);
+        ns = totals.sum_exec_runtime + do_task_delta_exec(p, rq);
         task_rq_unlock(rq, &flags);
 
         return ns;
@@ -4559,6 +4620,8 @@ void account_user_time(struct task_struct *p, cputime_t cputime,
                 cpustat->nice = cputime64_add(cpustat->nice, tmp);
         else
                 cpustat->user = cputime64_add(cpustat->user, tmp);
+
+        cpuacct_update_stats(p, CPUACCT_STAT_USER, cputime);
         /* Account for user time used */
         acct_update_integrals(p);
 }
@@ -4620,6 +4683,8 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
         else
                 cpustat->system = cputime64_add(cpustat->system, tmp);
 
+        cpuacct_update_stats(p, CPUACCT_STAT_SYSTEM, cputime);
+
         /* Account for system time used */
         acct_update_integrals(p);
 }
@@ -4667,7 +4732,7 @@ void account_process_tick(struct task_struct *p, int user_tick)
 
         if (user_tick)
                 account_user_time(p, one_jiffy, one_jiffy_scaled);
-        else if (p != rq->idle)
+        else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET))
                 account_system_time(p, HARDIRQ_OFFSET, one_jiffy,
                                     one_jiffy_scaled);
         else
@@ -4781,10 +4846,7 @@ void scheduler_tick(void)
 #endif
 }
 
-#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
-                                defined(CONFIG_PREEMPT_TRACER))
-
-static inline unsigned long get_parent_ip(unsigned long addr)
+notrace unsigned long get_parent_ip(unsigned long addr)
 {
         if (in_lock_functions(addr)) {
                 addr = CALLER_ADDR2;
@@ -4794,6 +4856,9 @@ static inline unsigned long get_parent_ip(unsigned long addr)
         return addr;
 }
 
+#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
+                                defined(CONFIG_PREEMPT_TRACER))
+
 void __kprobes add_preempt_count(int val)
 {
 #ifdef CONFIG_DEBUG_PREEMPT
@@ -7302,7 +7367,12 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
                 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
 
                 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
+
                 printk(KERN_CONT " %s", str);
+                if (group->__cpu_power != SCHED_LOAD_SCALE) {
+                        printk(KERN_CONT " (__cpu_power = %d)",
+                                group->__cpu_power);
+                }
 
                 group = group->next;
         } while (group != sd->groups);
@@ -7728,7 +7798,7 @@ cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
 {
         int group;
 
-        cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
+        cpumask_and(mask, topology_thread_cpumask(cpu), cpu_map);
         group = cpumask_first(mask);
         if (sg)
                 *sg = &per_cpu(sched_group_core, group).sg;
@@ -7757,7 +7827,7 @@ cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
         cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
         group = cpumask_first(mask);
 #elif defined(CONFIG_SCHED_SMT)
-        cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
+        cpumask_and(mask, topology_thread_cpumask(cpu), cpu_map);
         group = cpumask_first(mask);
 #else
         group = cpu;
@@ -8100,7 +8170,7 @@ static int __build_sched_domains(const struct cpumask *cpu_map,
                 SD_INIT(sd, SIBLING);
                 set_domain_attribute(sd, attr);
                 cpumask_and(sched_domain_span(sd),
-                            &per_cpu(cpu_sibling_map, i), cpu_map);
+                            topology_thread_cpumask(i), cpu_map);
                 sd->parent = p;
                 p->child = sd;
                 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
@@ -8111,7 +8181,7 @@ static int __build_sched_domains(const struct cpumask *cpu_map,
         /* Set up CPU (sibling) groups */
         for_each_cpu(i, cpu_map) {
                 cpumask_and(this_sibling_map,
-                            &per_cpu(cpu_sibling_map, i), cpu_map);
+                            topology_thread_cpumask(i), cpu_map);
                 if (i != cpumask_first(this_sibling_map))
                         continue;
 
@@ -8787,6 +8857,9 @@ void __init sched_init(void)
 #ifdef CONFIG_USER_SCHED
         alloc_size *= 2;
 #endif
+#ifdef CONFIG_CPUMASK_OFFSTACK
+        alloc_size += num_possible_cpus() * cpumask_size();
+#endif
         /*
          * As sched_init() is called before page_alloc is setup,
          * we use alloc_bootmem().
@@ -8824,6 +8897,12 @@ void __init sched_init(void)
                 ptr += nr_cpu_ids * sizeof(void **);
 #endif /* CONFIG_USER_SCHED */
 #endif /* CONFIG_RT_GROUP_SCHED */
+#ifdef CONFIG_CPUMASK_OFFSTACK
+                for_each_possible_cpu(i) {
+                        per_cpu(load_balance_tmpmask, i) = (void *)ptr;
+                        ptr += cpumask_size();
+                }
+#endif /* CONFIG_CPUMASK_OFFSTACK */
         }
 
 #ifdef CONFIG_SMP
@@ -9916,6 +9995,7 @@ struct cpuacct {
         struct cgroup_subsys_state css;
         /* cpuusage holds pointer to a u64-type object on every cpu */
         u64 *cpuusage;
+        struct percpu_counter cpustat[CPUACCT_STAT_NSTATS];
         struct cpuacct *parent;
 };
 
@@ -9940,20 +10020,32 @@ static struct cgroup_subsys_state *cpuacct_create(
         struct cgroup_subsys *ss, struct cgroup *cgrp)
 {
         struct cpuacct *ca = kzalloc(sizeof(*ca), GFP_KERNEL);
+        int i;
 
         if (!ca)
-                return ERR_PTR(-ENOMEM);
+                goto out;
 
         ca->cpuusage = alloc_percpu(u64);
-        if (!ca->cpuusage) {
-                kfree(ca);
-                return ERR_PTR(-ENOMEM);
-        }
+        if (!ca->cpuusage)
+                goto out_free_ca;
+
+        for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
+                if (percpu_counter_init(&ca->cpustat[i], 0))
+                        goto out_free_counters;
 
         if (cgrp->parent)
                 ca->parent = cgroup_ca(cgrp->parent);
 
         return &ca->css;
+
+out_free_counters:
+        while (--i >= 0)
+                percpu_counter_destroy(&ca->cpustat[i]);
+        free_percpu(ca->cpuusage);
+out_free_ca:
+        kfree(ca);
+out:
+        return ERR_PTR(-ENOMEM);
 }
 
 /* destroy an existing cpu accounting group */
@@ -9961,7 +10053,10 @@ static void
 cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
 {
         struct cpuacct *ca = cgroup_ca(cgrp);
+        int i;
 
+        for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
+                percpu_counter_destroy(&ca->cpustat[i]);
         free_percpu(ca->cpuusage);
         kfree(ca);
 }
@@ -10048,6 +10143,25 @@ static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
         return 0;
 }
 
+static const char *cpuacct_stat_desc[] = {
+        [CPUACCT_STAT_USER] = "user",
+        [CPUACCT_STAT_SYSTEM] = "system",
+};
+
+static int cpuacct_stats_show(struct cgroup *cgrp, struct cftype *cft,
+                struct cgroup_map_cb *cb)
+{
+        struct cpuacct *ca = cgroup_ca(cgrp);
+        int i;
+
+        for (i = 0; i < CPUACCT_STAT_NSTATS; i++) {
+                s64 val = percpu_counter_read(&ca->cpustat[i]);
+                val = cputime64_to_clock_t(val);
+                cb->fill(cb, cpuacct_stat_desc[i], val);
+        }
+        return 0;
+}
+
 static struct cftype files[] = {
         {
                 .name = "usage",
@@ -10058,7 +10172,10 @@ static struct cftype files[] = {
                 .name = "usage_percpu",
                 .read_seq_string = cpuacct_percpu_seq_read,
         },
-
+        {
+                .name = "stat",
+                .read_map = cpuacct_stats_show,
+        },
 };
 
 static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
@@ -10080,12 +10197,38 @@ static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
                 return;
 
         cpu = task_cpu(tsk);
+
+        rcu_read_lock();
+
         ca = task_ca(tsk);
 
         for (; ca; ca = ca->parent) {
                 u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
                 *cpuusage += cputime;
         }
+
+        rcu_read_unlock();
+}
+
+/*
+ * Charge the system/user time to the task's accounting group.
+ */
+static void cpuacct_update_stats(struct task_struct *tsk,
+                enum cpuacct_stat_index idx, cputime_t val)
+{
+        struct cpuacct *ca;
+
+        if (unlikely(!cpuacct_subsys.active))
+                return;
+
+        rcu_read_lock();
+        ca = task_ca(tsk);
+
+        do {
+                percpu_counter_add(&ca->cpustat[idx], val);
+                ca = ca->parent;
+        } while (ca);
+        rcu_read_unlock();
 }
 
 struct cgroup_subsys cpuacct_subsys = {