10 files changed, 114 insertions, 54 deletions

diff --git a/arch/ia64/kernel/time.c b/arch/ia64/kernel/time.c
index 4ee367817049..f0ebb342409d 100644
--- a/arch/ia64/kernel/time.c
+++ b/arch/ia64/kernel/time.c
@@ -93,7 +93,10 @@ void ia64_account_on_switch(struct task_struct *prev, struct task_struct *next)
         now = ia64_get_itc();
 
         delta_stime = cycle_to_cputime(pi->ac_stime + (now - pi->ac_stamp));
-        account_system_time(prev, 0, delta_stime, delta_stime);
+        if (idle_task(smp_processor_id()) != prev)
+                account_system_time(prev, 0, delta_stime, delta_stime);
+        else
+                account_idle_time(delta_stime);
 
         if (pi->ac_utime) {
                 delta_utime = cycle_to_cputime(pi->ac_utime);
@@ -120,7 +123,10 @@ void account_system_vtime(struct task_struct *tsk)
         now = ia64_get_itc();
 
         delta_stime = cycle_to_cputime(ti->ac_stime + (now - ti->ac_stamp));
-        account_system_time(tsk, 0, delta_stime, delta_stime);
+        if (irq_count() || idle_task(smp_processor_id()) != tsk)
+                account_system_time(tsk, 0, delta_stime, delta_stime);
+        else
+                account_idle_time(delta_stime);
         ti->ac_stime = 0;
 
         ti->ac_stamp = now;
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 51b201ddf9a1..fb7049c054c0 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -33,6 +33,7 @@
 #include <linux/mqueue.h>
 #include <linux/hardirq.h>
 #include <linux/utsname.h>
+#include <linux/kernel_stat.h>
 
 #include <asm/pgtable.h>
 #include <asm/uaccess.h>
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 92650ccad2e1..3be355c1cfa7 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -256,7 +256,10 @@ void account_system_vtime(struct task_struct *tsk)
                 delta += sys_time;
                 get_paca()->system_time = 0;
         }
-        account_system_time(tsk, 0, delta, deltascaled);
+        if (in_irq() || idle_task(smp_processor_id()) != tsk)
+                account_system_time(tsk, 0, delta, deltascaled);
+        else
+                account_idle_time(delta);
         per_cpu(cputime_last_delta, smp_processor_id()) = delta;
         per_cpu(cputime_scaled_last_delta, smp_processor_id()) = deltascaled;
         local_irq_restore(flags);
@@ -335,8 +338,12 @@ void calculate_steal_time(void)
         tb = mftb();
         purr = mfspr(SPRN_PURR);
         stolen = (tb - pme->tb) - (purr - pme->purr);
-        if (stolen > 0)
-                account_steal_time(current, stolen);
+        if (stolen > 0) {
+                if (idle_task(smp_processor_id()) != current)
+                        account_steal_time(stolen);
+                else
+                        account_idle_time(stolen);
+        }
         pme->tb = tb;
         pme->purr = purr;
 }
diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c
index 07283aea2e56..4a4a34caec55 100644
--- a/arch/s390/kernel/vtime.c
+++ b/arch/s390/kernel/vtime.c
@@ -55,13 +55,19 @@ void account_process_tick(struct task_struct *tsk, int user_tick)
         cputime =  S390_lowcore.system_timer >> 12;
         S390_lowcore.system_timer -= cputime << 12;
         S390_lowcore.steal_clock -= cputime << 12;
-        account_system_time(tsk, HARDIRQ_OFFSET, cputime, cputime);
+        if (idle_task(smp_processor_id()) != current)
+                account_system_time(tsk, HARDIRQ_OFFSET, cputime, cputime);
+        else
+                account_idle_time(cputime);
 
         cputime = S390_lowcore.steal_clock;
         if ((__s64) cputime > 0) {
                 cputime >>= 12;
                 S390_lowcore.steal_clock -= cputime << 12;
-                account_steal_time(tsk, cputime);
+                if (idle_task(smp_processor_id()) != current)
+                        account_steal_time(cputime);
+                else
+                        account_idle_time(cputime);
         }
 }
 
@@ -87,7 +93,10 @@ void account_vtime(struct task_struct *tsk)
         cputime =  S390_lowcore.system_timer >> 12;
         S390_lowcore.system_timer -= cputime << 12;
         S390_lowcore.steal_clock -= cputime << 12;
-        account_system_time(tsk, 0, cputime, cputime);
+        if (idle_task(smp_processor_id()) != current)
+                account_system_time(tsk, 0, cputime, cputime);
+        else
+                account_idle_time(cputime);
 }
 
 /*
@@ -107,7 +116,10 @@ void account_system_vtime(struct task_struct *tsk)
         cputime =  S390_lowcore.system_timer >> 12;
         S390_lowcore.system_timer -= cputime << 12;
         S390_lowcore.steal_clock -= cputime << 12;
-        account_system_time(tsk, 0, cputime, cputime);
+        if (in_irq() || idle_task(smp_processor_id()) != current)
+                account_system_time(tsk, 0, cputime, cputime);
+        else
+                account_idle_time(cputime);
 }
 EXPORT_SYMBOL_GPL(account_system_vtime);
 
diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c
index c9f7cda48ed7..732e52dc991a 100644
--- a/arch/x86/xen/time.c
+++ b/arch/x86/xen/time.c
@@ -132,8 +132,7 @@ static void do_stolen_accounting(void)
         *snap = state;
 
         /* Add the appropriate number of ticks of stolen time,
-           including any left-overs from last time.  Passing NULL to
-           account_steal_time accounts the time as stolen. */
+           including any left-overs from last time. */
         stolen = runnable + offline + __get_cpu_var(residual_stolen);
 
         if (stolen < 0)
@@ -141,11 +140,10 @@ static void do_stolen_accounting(void)
 
         ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen);
         __get_cpu_var(residual_stolen) = stolen;
-        account_steal_time(NULL, ticks);
+        account_steal_ticks(ticks);
 
         /* Add the appropriate number of ticks of blocked time,
-           including any left-overs from last time.  Passing idle to
-           account_steal_time accounts the time as idle/wait. */
+           including any left-overs from last time. */
         blocked += __get_cpu_var(residual_blocked);
 
         if (blocked < 0)
@@ -153,7 +151,7 @@ static void do_stolen_accounting(void)
 
         ticks = iter_div_u64_rem(blocked, NS_PER_TICK, &blocked);
         __get_cpu_var(residual_blocked) = blocked;
-        account_steal_time(idle_task(smp_processor_id()), ticks);
+        account_idle_ticks(ticks);
 }
 
 /*
diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h
index c78a459662a6..570d20413119 100644
--- a/include/linux/kernel_stat.h
+++ b/include/linux/kernel_stat.h
@@ -81,6 +81,11 @@ static inline unsigned int kstat_irqs(unsigned int irq)
 extern unsigned long long task_delta_exec(struct task_struct *);
 extern void account_user_time(struct task_struct *, cputime_t, cputime_t);
 extern void account_system_time(struct task_struct *, int, cputime_t, cputime_t);
-extern void account_steal_time(struct task_struct *, cputime_t);
+extern void account_steal_time(cputime_t);
+extern void account_idle_time(cputime_t);
+
+extern void account_process_tick(struct task_struct *, int user);
+extern void account_steal_ticks(unsigned long ticks);
+extern void account_idle_ticks(unsigned long ticks);
 
 #endif /* _LINUX_KERNEL_STAT_H */
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 8395e715809d..b475d4db8053 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -284,7 +284,6 @@ long io_schedule_timeout(long timeout);
 
 extern void cpu_init (void);
 extern void trap_init(void);
-extern void account_process_tick(struct task_struct *task, int user);
 extern void update_process_times(int user);
 extern void scheduler_tick(void);
 
diff --git a/kernel/sched.c b/kernel/sched.c
index 5b03679ff712..635eaffe1e4c 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -4139,7 +4139,6 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
                          cputime_t cputime, cputime_t cputime_scaled)
 {
         struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
-        struct rq *rq = this_rq();
         cputime64_t tmp;
 
         if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
@@ -4158,37 +4157,84 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
                 cpustat->irq = cputime64_add(cpustat->irq, tmp);
         else if (softirq_count())
                 cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
-        else if (p != rq->idle)
-                cpustat->system = cputime64_add(cpustat->system, tmp);
-        else if (atomic_read(&rq->nr_iowait) > 0)
-                cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
         else
-                cpustat->idle = cputime64_add(cpustat->idle, tmp);
+                cpustat->system = cputime64_add(cpustat->system, tmp);
+
         /* Account for system time used */
         acct_update_integrals(p);
 }
 
 /*
  * Account for involuntary wait time.
- * @p: the process from which the cpu time has been stolen
  * @steal: the cpu time spent in involuntary wait
  */
-void account_steal_time(struct task_struct *p, cputime_t steal)
+void account_steal_time(cputime_t cputime)
+{
+        struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
+        cputime64_t cputime64 = cputime_to_cputime64(cputime);
+
+        cpustat->steal = cputime64_add(cpustat->steal, cputime64);
+}
+
+/*
+ * Account for idle time.
+ * @cputime: the cpu time spent in idle wait
+ */
+void account_idle_time(cputime_t cputime)
 {
         struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
-        cputime64_t tmp = cputime_to_cputime64(steal);
+        cputime64_t cputime64 = cputime_to_cputime64(cputime);
         struct rq *rq = this_rq();
 
-        if (p == rq->idle) {
-                p->stime = cputime_add(p->stime, steal);
-                if (atomic_read(&rq->nr_iowait) > 0)
-                        cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
-                else
-                        cpustat->idle = cputime64_add(cpustat->idle, tmp);
-        } else
-                cpustat->steal = cputime64_add(cpustat->steal, tmp);
+        if (atomic_read(&rq->nr_iowait) > 0)
+                cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
+        else
+                cpustat->idle = cputime64_add(cpustat->idle, cputime64);
+}
+
+#ifndef CONFIG_VIRT_CPU_ACCOUNTING
+
+/*
+ * Account a single tick of cpu time.
+ * @p: the process that the cpu time gets accounted to
+ * @user_tick: indicates if the tick is a user or a system tick
+ */
+void account_process_tick(struct task_struct *p, int user_tick)
+{
+        cputime_t one_jiffy = jiffies_to_cputime(1);
+        cputime_t one_jiffy_scaled = cputime_to_scaled(one_jiffy);
+        struct rq *rq = this_rq();
+
+        if (user_tick)
+                account_user_time(p, one_jiffy, one_jiffy_scaled);
+        else if (p != rq->idle)
+                account_system_time(p, HARDIRQ_OFFSET, one_jiffy,
+                                    one_jiffy_scaled);
+        else
+                account_idle_time(one_jiffy);
+}
+
+/*
+ * Account multiple ticks of steal time.
+ * @p: the process from which the cpu time has been stolen
+ * @ticks: number of stolen ticks
+ */
+void account_steal_ticks(unsigned long ticks)
+{
+        account_steal_time(jiffies_to_cputime(ticks));
+}
+
+/*
+ * Account multiple ticks of idle time.
+ * @ticks: number of stolen ticks
+ */
+void account_idle_ticks(unsigned long ticks)
+{
+        account_idle_time(jiffies_to_cputime(ticks));
 }
 
+#endif
+
 /*
  * Use precise platform statistics if available:
  */
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 1f2fce2479fe..611fa4c0baab 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -419,8 +419,9 @@ void tick_nohz_restart_sched_tick(void)
 {
         int cpu = smp_processor_id();
         struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
+#ifndef CONFIG_VIRT_CPU_ACCOUNTING
         unsigned long ticks;
-        cputime_t cputime;
+#endif
         ktime_t now;
 
         local_irq_disable();
@@ -442,6 +443,7 @@ void tick_nohz_restart_sched_tick(void)
         tick_do_update_jiffies64(now);
         cpu_clear(cpu, nohz_cpu_mask);
 
+#ifndef CONFIG_VIRT_CPU_ACCOUNTING
         /*
          * We stopped the tick in idle. Update process times would miss the
          * time we slept as update_process_times does only a 1 tick
@@ -451,12 +453,9 @@ void tick_nohz_restart_sched_tick(void)
         /*
          * We might be one off. Do not randomly account a huge number of ticks!
          */
-        if (ticks && ticks < LONG_MAX) {
-                add_preempt_count(HARDIRQ_OFFSET);
-                cputime = jiffies_to_cputime(ticks);
-                account_system_time(current, HARDIRQ_OFFSET, cputime, cputime);
-                sub_preempt_count(HARDIRQ_OFFSET);
-        }
+        if (ticks && ticks < LONG_MAX)
+                account_idle_ticks(ticks);
+#endif
 
         touch_softlockup_watchdog();
         /*
diff --git a/kernel/timer.c b/kernel/timer.c
index b5efb528aa1d..dee3f641a7a7 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -1018,19 +1018,6 @@ unsigned long get_next_timer_interrupt(unsigned long now)
 }
 #endif
 
-#ifndef CONFIG_VIRT_CPU_ACCOUNTING
-void account_process_tick(struct task_struct *p, int user_tick)
-{
-        cputime_t one_jiffy = jiffies_to_cputime(1);
-
-        if (user_tick)
-                account_user_time(p, one_jiffy, cputime_to_scaled(one_jiffy));
-        else
-                account_system_time(p, HARDIRQ_OFFSET, one_jiffy,
-                                    cputime_to_scaled(one_jiffy));
-}
-#endif
-
 /*
  * Called from the timer interrupt handler to charge one tick to the current
  * process.  user_tick is 1 if the tick is user time, 0 for system.