8 files changed, 163 insertions, 81 deletions

diff --git a/kernel/fork.c b/kernel/fork.c
index 28a740151988..8ca1a14cdc8c 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1123,6 +1123,9 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         p->blocked_on = NULL; /* not blocked yet */
 #endif
 
+        /* Perform scheduler related setup. Assign this task to a CPU. */
+        sched_fork(p, clone_flags);
+
         if ((retval = security_task_alloc(p)))
                 goto bad_fork_cleanup_policy;
         if ((retval = audit_alloc(p)))
@@ -1212,9 +1215,6 @@ static struct task_struct *copy_process(unsigned long clone_flags,
         INIT_LIST_HEAD(&p->ptrace_children);
         INIT_LIST_HEAD(&p->ptrace_list);
 
-        /* Perform scheduler related setup. Assign this task to a CPU. */
-        sched_fork(p, clone_flags);
-
         /* Now that the task is set up, run cgroup callbacks if
          * necessary. We need to run them before the task is visible
          * on the tasklist. */
diff --git a/kernel/futex_compat.c b/kernel/futex_compat.c
index 00b572666cc7..0a43def6fee7 100644
--- a/kernel/futex_compat.c
+++ b/kernel/futex_compat.c
@@ -30,6 +30,15 @@ fetch_robust_entry(compat_uptr_t *uentry, struct robust_list __user **entry,
         return 0;
 }
 
+static void __user *futex_uaddr(struct robust_list *entry,
+                                compat_long_t futex_offset)
+{
+        compat_uptr_t base = ptr_to_compat(entry);
+        void __user *uaddr = compat_ptr(base + futex_offset);
+
+        return uaddr;
+}
+
 /*
  * Walk curr->robust_list (very carefully, it's a userspace list!)
  * and mark any locks found there dead, and notify any waiters.
@@ -76,11 +85,12 @@ void compat_exit_robust_list(struct task_struct *curr)
                  * A pending lock might already be on the list, so
                  * dont process it twice:
                  */
-                if (entry != pending)
-                        if (handle_futex_death((void __user *)entry + futex_offset,
-                                                curr, pi))
-                                return;
+                if (entry != pending) {
+                        void __user *uaddr = futex_uaddr(entry, futex_offset);
 
+                        if (handle_futex_death(uaddr, curr, pi))
+                                return;
+                }
                 if (rc)
                         return;
                 uentry = next_uentry;
@@ -94,9 +104,11 @@ void compat_exit_robust_list(struct task_struct *curr)
 
                 cond_resched();
         }
-        if (pending)
-                handle_futex_death((void __user *)pending + futex_offset,
-                                   curr, pip);
+        if (pending) {
+                void __user *uaddr = futex_uaddr(pending, futex_offset);
+
+                handle_futex_death(uaddr, curr, pip);
+        }
 }
 
 asmlinkage long
diff --git a/kernel/sched.c b/kernel/sched.c
index 3f6bd1112900..b18f231a4875 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -75,7 +75,7 @@
  */
 unsigned long long __attribute__((weak)) sched_clock(void)
 {
-        return (unsigned long long)jiffies * (1000000000 / HZ);
+        return (unsigned long long)jiffies * (NSEC_PER_SEC / HZ);
 }
 
 /*
@@ -99,8 +99,8 @@ unsigned long long __attribute__((weak)) sched_clock(void)
 /*
  * Some helpers for converting nanosecond timing to jiffy resolution
  */
-#define NS_TO_JIFFIES(TIME)     ((unsigned long)(TIME) / (1000000000 / HZ))
-#define JIFFIES_TO_NS(TIME)     ((TIME) * (1000000000 / HZ))
+#define NS_TO_JIFFIES(TIME)     ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
+#define JIFFIES_TO_NS(TIME)     ((TIME) * (NSEC_PER_SEC / HZ))
 
 #define NICE_0_LOAD             SCHED_LOAD_SCALE
 #define NICE_0_SHIFT            SCHED_LOAD_SHIFT
@@ -460,7 +460,6 @@ enum {
         SCHED_FEAT_TREE_AVG             = 4,
         SCHED_FEAT_APPROX_AVG           = 8,
         SCHED_FEAT_WAKEUP_PREEMPT       = 16,
-        SCHED_FEAT_PREEMPT_RESTRICT     = 32,
 };
 
 const_debug unsigned int sysctl_sched_features =
@@ -468,12 +467,17 @@ const_debug unsigned int sysctl_sched_features =
                 SCHED_FEAT_START_DEBIT          * 1 |
                 SCHED_FEAT_TREE_AVG             * 0 |
                 SCHED_FEAT_APPROX_AVG           * 0 |
-                SCHED_FEAT_WAKEUP_PREEMPT       * 1 |
-                SCHED_FEAT_PREEMPT_RESTRICT     * 1;
+                SCHED_FEAT_WAKEUP_PREEMPT       * 1;
 
 #define sched_feat(x) (sysctl_sched_features & SCHED_FEAT_##x)
 
 /*
+ * Number of tasks to iterate in a single balance run.
+ * Limited because this is done with IRQs disabled.
+ */
+const_debug unsigned int sysctl_sched_nr_migrate = 32;
+
+/*
  * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
  * clock constructed from sched_clock():
  */
@@ -2237,7 +2241,7 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
               enum cpu_idle_type idle, int *all_pinned,
               int *this_best_prio, struct rq_iterator *iterator)
 {
-        int pulled = 0, pinned = 0, skip_for_load;
+        int loops = 0, pulled = 0, pinned = 0, skip_for_load;
         struct task_struct *p;
         long rem_load_move = max_load_move;
 
@@ -2251,10 +2255,10 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
          */
         p = iterator->start(iterator->arg);
 next:
-        if (!p)
+        if (!p || loops++ > sysctl_sched_nr_migrate)
                 goto out;
         /*
-         * To help distribute high priority tasks accross CPUs we don't
+         * To help distribute high priority tasks across CPUs we don't
          * skip a task if it will be the highest priority task (i.e. smallest
          * prio value) on its new queue regardless of its load weight
          */
@@ -2271,8 +2275,7 @@ next:
         rem_load_move -= p->se.load.weight;
 
         /*
-         * We only want to steal up to the prescribed number of tasks
-         * and the prescribed amount of weighted load.
+         * We only want to steal up to the prescribed amount of weighted load.
          */
         if (rem_load_move > 0) {
                 if (p->prio < *this_best_prio)
@@ -4992,6 +4995,32 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
  */
 cpumask_t nohz_cpu_mask = CPU_MASK_NONE;
 
+/*
+ * Increase the granularity value when there are more CPUs,
+ * because with more CPUs the 'effective latency' as visible
+ * to users decreases. But the relationship is not linear,
+ * so pick a second-best guess by going with the log2 of the
+ * number of CPUs.
+ *
+ * This idea comes from the SD scheduler of Con Kolivas:
+ */
+static inline void sched_init_granularity(void)
+{
+        unsigned int factor = 1 + ilog2(num_online_cpus());
+        const unsigned long limit = 200000000;
+
+        sysctl_sched_min_granularity *= factor;
+        if (sysctl_sched_min_granularity > limit)
+                sysctl_sched_min_granularity = limit;
+
+        sysctl_sched_latency *= factor;
+        if (sysctl_sched_latency > limit)
+                sysctl_sched_latency = limit;
+
+        sysctl_sched_wakeup_granularity *= factor;
+        sysctl_sched_batch_wakeup_granularity *= factor;
+}
+
 #ifdef CONFIG_SMP
 /*
  * This is how migration works:
@@ -5621,7 +5650,7 @@ static struct notifier_block __cpuinitdata migration_notifier = {
         .priority = 10
 };
 
-int __init migration_init(void)
+void __init migration_init(void)
 {
         void *cpu = (void *)(long)smp_processor_id();
         int err;
@@ -5631,8 +5660,6 @@ int __init migration_init(void)
         BUG_ON(err == NOTIFY_BAD);
         migration_call(&migration_notifier, CPU_ONLINE, cpu);
         register_cpu_notifier(&migration_notifier);
-
-        return 0;
 }
 #endif
 
@@ -6688,10 +6715,12 @@ void __init sched_init_smp(void)
         /* Move init over to a non-isolated CPU */
         if (set_cpus_allowed(current, non_isolated_cpus) < 0)
                 BUG();
+        sched_init_granularity();
 }
 #else
 void __init sched_init_smp(void)
 {
+        sched_init_granularity();
 }
 #endif /* CONFIG_SMP */
 
@@ -7228,7 +7257,7 @@ static u64 cpu_usage_read(struct cgroup *cgrp, struct cftype *cft)
                 spin_unlock_irqrestore(&cpu_rq(i)->lock, flags);
         }
         /* Convert from ns to ms */
-        do_div(res, 1000000);
+        do_div(res, NSEC_PER_MSEC);
 
         return res;
 }
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c
index 415e5c385542..ca198a797bfa 100644
--- a/kernel/sched_debug.c
+++ b/kernel/sched_debug.c
@@ -211,7 +211,7 @@ static int sched_debug_show(struct seq_file *m, void *v)
 #define PN(x) \
         SEQ_printf(m, "  .%-40s: %Ld.%06ld\n", #x, SPLIT_NS(x))
         PN(sysctl_sched_latency);
-        PN(sysctl_sched_nr_latency);
+        PN(sysctl_sched_min_granularity);
         PN(sysctl_sched_wakeup_granularity);
         PN(sysctl_sched_batch_wakeup_granularity);
         PN(sysctl_sched_child_runs_first);
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 01859f662ab7..d3c03070872d 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -22,7 +22,7 @@
 
 /*
  * Targeted preemption latency for CPU-bound tasks:
- * (default: 20ms, units: nanoseconds)
+ * (default: 20ms * ilog(ncpus), units: nanoseconds)
  *
  * NOTE: this latency value is not the same as the concept of
  * 'timeslice length' - timeslices in CFS are of variable length
@@ -32,19 +32,24 @@
  * (to see the precise effective timeslice length of your workload,
  *  run vmstat and monitor the context-switches (cs) field)
  */
-const_debug unsigned int sysctl_sched_latency = 20000000ULL;
+unsigned int sysctl_sched_latency = 20000000ULL;
 
 /*
- * After fork, child runs first. (default) If set to 0 then
- * parent will (try to) run first.
+ * Minimal preemption granularity for CPU-bound tasks:
+ * (default: 1 msec * ilog(ncpus), units: nanoseconds)
  */
-const_debug unsigned int sysctl_sched_child_runs_first = 1;
+unsigned int sysctl_sched_min_granularity = 1000000ULL;
 
 /*
- * Minimal preemption granularity for CPU-bound tasks:
- * (default: 2 msec, units: nanoseconds)
+ * is kept at sysctl_sched_latency / sysctl_sched_min_granularity
+ */
+unsigned int sched_nr_latency = 20;
+
+/*
+ * After fork, child runs first. (default) If set to 0 then
+ * parent will (try to) run first.
  */
-const_debug unsigned int sysctl_sched_nr_latency = 20;
+const_debug unsigned int sysctl_sched_child_runs_first = 1;
 
 /*
  * sys_sched_yield() compat mode
@@ -56,23 +61,23 @@ unsigned int __read_mostly sysctl_sched_compat_yield;
 
 /*
  * SCHED_BATCH wake-up granularity.
- * (default: 10 msec, units: nanoseconds)
+ * (default: 10 msec * ilog(ncpus), units: nanoseconds)
  *
  * This option delays the preemption effects of decoupled workloads
  * and reduces their over-scheduling. Synchronous workloads will still
  * have immediate wakeup/sleep latencies.
  */
-const_debug unsigned int sysctl_sched_batch_wakeup_granularity = 10000000UL;
+unsigned int sysctl_sched_batch_wakeup_granularity = 10000000UL;
 
 /*
  * SCHED_OTHER wake-up granularity.
- * (default: 10 msec, units: nanoseconds)
+ * (default: 10 msec * ilog(ncpus), units: nanoseconds)
  *
  * This option delays the preemption effects of decoupled workloads
  * and reduces their over-scheduling. Synchronous workloads will still
  * have immediate wakeup/sleep latencies.
  */
-const_debug unsigned int sysctl_sched_wakeup_granularity = 10000000UL;
+unsigned int sysctl_sched_wakeup_granularity = 10000000UL;
 
 const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
 
@@ -212,6 +217,22 @@ static inline struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
  * Scheduling class statistics methods:
  */
 
+#ifdef CONFIG_SCHED_DEBUG
+int sched_nr_latency_handler(struct ctl_table *table, int write,
+                struct file *filp, void __user *buffer, size_t *lenp,
+                loff_t *ppos)
+{
+        int ret = proc_dointvec_minmax(table, write, filp, buffer, lenp, ppos);
+
+        if (ret || !write)
+                return ret;
+
+        sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency,
+                                        sysctl_sched_min_granularity);
+
+        return 0;
+}
+#endif
 
 /*
  * The idea is to set a period in which each task runs once.
@@ -224,7 +245,7 @@ static inline struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
 static u64 __sched_period(unsigned long nr_running)
 {
         u64 period = sysctl_sched_latency;
-        unsigned long nr_latency = sysctl_sched_nr_latency;
+        unsigned long nr_latency = sched_nr_latency;
 
         if (unlikely(nr_running > nr_latency)) {
                 period *= nr_running;
@@ -259,6 +280,7 @@ static u64 __sched_vslice(unsigned long rq_weight, unsigned long nr_running)
 {
         u64 vslice = __sched_period(nr_running);
 
+        vslice *= NICE_0_LOAD;
         do_div(vslice, rq_weight);
 
         return vslice;
@@ -472,19 +494,26 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
         } else if (sched_feat(APPROX_AVG) && cfs_rq->nr_running)
                 vruntime += sched_vslice(cfs_rq)/2;
 
+        /*
+         * The 'current' period is already promised to the current tasks,
+         * however the extra weight of the new task will slow them down a
+         * little, place the new task so that it fits in the slot that
+         * stays open at the end.
+         */
         if (initial && sched_feat(START_DEBIT))
                 vruntime += sched_vslice_add(cfs_rq, se);
 
         if (!initial) {
+                /* sleeps upto a single latency don't count. */
                 if (sched_feat(NEW_FAIR_SLEEPERS) && entity_is_task(se) &&
                                 task_of(se)->policy != SCHED_BATCH)
                         vruntime -= sysctl_sched_latency;
 
-                vruntime = max_t(s64, vruntime, se->vruntime);
+                /* ensure we never gain time by being placed backwards. */
+                vruntime = max_vruntime(se->vruntime, vruntime);
         }
 
         se->vruntime = vruntime;
-
 }
 
 static void
@@ -517,7 +546,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
 
         update_stats_dequeue(cfs_rq, se);
         if (sleep) {
-                se->peer_preempt = 0;
 #ifdef CONFIG_SCHEDSTATS
                 if (entity_is_task(se)) {
                         struct task_struct *tsk = task_of(se);
@@ -545,10 +573,8 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
 
         ideal_runtime = sched_slice(cfs_rq, curr);
         delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
-        if (delta_exec > ideal_runtime ||
-                        (sched_feat(PREEMPT_RESTRICT) && curr->peer_preempt))
+        if (delta_exec > ideal_runtime)
                 resched_task(rq_of(cfs_rq)->curr);
-        curr->peer_preempt = 0;
 }
 
 static void
@@ -811,7 +837,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
         struct task_struct *curr = rq->curr;
         struct cfs_rq *cfs_rq = task_cfs_rq(curr);
         struct sched_entity *se = &curr->se, *pse = &p->se;
-        s64 delta, gran;
+        unsigned long gran;
 
         if (unlikely(rt_prio(p->prio))) {
                 update_rq_clock(rq);
@@ -826,24 +852,20 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
         if (unlikely(p->policy == SCHED_BATCH))
                 return;
 
-        if (sched_feat(WAKEUP_PREEMPT)) {
-                while (!is_same_group(se, pse)) {
-                        se = parent_entity(se);
-                        pse = parent_entity(pse);
-                }
+        if (!sched_feat(WAKEUP_PREEMPT))
+                return;
 
-                delta = se->vruntime - pse->vruntime;
-                gran = sysctl_sched_wakeup_granularity;
-                if (unlikely(se->load.weight != NICE_0_LOAD))
-                        gran = calc_delta_fair(gran, &se->load);
+        while (!is_same_group(se, pse)) {
+                se = parent_entity(se);
+                pse = parent_entity(pse);
+        }
 
-                if (delta > gran) {
-                        int now = !sched_feat(PREEMPT_RESTRICT);
+        gran = sysctl_sched_wakeup_granularity;
+        if (unlikely(se->load.weight != NICE_0_LOAD))
+                gran = calc_delta_fair(gran, &se->load);
 
-                        if (now || p->prio < curr->prio || !se->peer_preempt++)
-                                resched_task(curr);
-                }
-        }
+        if (pse->vruntime + gran < se->vruntime)
+                resched_task(curr);
 }
 
 static struct task_struct *pick_next_task_fair(struct rq *rq)
@@ -1045,8 +1067,9 @@ static void task_new_fair(struct rq *rq, struct task_struct *p)
         update_curr(cfs_rq);
         place_entity(cfs_rq, se, 1);
 
+        /* 'curr' will be NULL if the child belongs to a different group */
         if (sysctl_sched_child_runs_first && this_cpu == task_cpu(p) &&
-                        curr->vruntime < se->vruntime) {
+                        curr && curr->vruntime < se->vruntime) {
                 /*
                  * Upon rescheduling, sched_class::put_prev_task() will place
                  * 'current' within the tree based on its new key value.
@@ -1054,7 +1077,6 @@ static void task_new_fair(struct rq *rq, struct task_struct *p)
                 swap(curr->vruntime, se->vruntime);
         }
 
-        se->peer_preempt = 0;
         enqueue_task_fair(rq, p, 0);
         resched_task(rq->curr);
 }
diff --git a/kernel/sched_stats.h b/kernel/sched_stats.h
index ef1a7df80ea2..630178e53bb6 100644
--- a/kernel/sched_stats.h
+++ b/kernel/sched_stats.h
@@ -127,7 +127,7 @@ rq_sched_info_depart(struct rq *rq, unsigned long long delta)
 # define schedstat_set(var, val)        do { } while (0)
 #endif
 
-#ifdef CONFIG_SCHEDSTATS
+#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
 /*
  * Called when a process is dequeued from the active array and given
  * the cpu.  We should note that with the exception of interactive
@@ -155,7 +155,7 @@ static inline void sched_info_dequeued(struct task_struct *t)
  */
 static void sched_info_arrive(struct task_struct *t)
 {
-        unsigned long long now = sched_clock(), delta = 0;
+        unsigned long long now = task_rq(t)->clock, delta = 0;
 
         if (t->sched_info.last_queued)
                 delta = now - t->sched_info.last_queued;
@@ -186,7 +186,7 @@ static inline void sched_info_queued(struct task_struct *t)
 {
         if (unlikely(sched_info_on()))
                 if (!t->sched_info.last_queued)
-                        t->sched_info.last_queued = sched_clock();
+                        t->sched_info.last_queued = task_rq(t)->clock;
 }
 
 /*
@@ -195,7 +195,8 @@ static inline void sched_info_queued(struct task_struct *t)
  */
 static inline void sched_info_depart(struct task_struct *t)
 {
-        unsigned long long delta = sched_clock() - t->sched_info.last_arrival;
+        unsigned long long delta = task_rq(t)->clock -
+                                        t->sched_info.last_arrival;
 
         t->sched_info.cpu_time += delta;
         rq_sched_info_depart(task_rq(t), delta);
@@ -231,5 +232,5 @@ sched_info_switch(struct task_struct *prev, struct task_struct *next)
 #else
 #define sched_info_queued(t)            do { } while (0)
 #define sched_info_switch(t, next)      do { } while (0)
-#endif /* CONFIG_SCHEDSTATS */
+#endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */
 
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 3b4efbe26445..3a1744fed2b6 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -226,20 +226,23 @@ static struct ctl_table root_table[] = {
 
 #ifdef CONFIG_SCHED_DEBUG
 static unsigned long min_sched_granularity_ns = 100000;         /* 100 usecs */
-static unsigned long max_sched_granularity_ns = 1000000000;     /* 1 second */
+static unsigned long max_sched_granularity_ns = NSEC_PER_SEC;   /* 1 second */
 static unsigned long min_wakeup_granularity_ns;                 /* 0 usecs */
-static unsigned long max_wakeup_granularity_ns = 1000000000;    /* 1 second */
+static unsigned long max_wakeup_granularity_ns = NSEC_PER_SEC;  /* 1 second */
 #endif
 
 static struct ctl_table kern_table[] = {
 #ifdef CONFIG_SCHED_DEBUG
         {
                 .ctl_name       = CTL_UNNUMBERED,
-                .procname       = "sched_nr_latency",
-                .data           = &sysctl_sched_nr_latency,
+                .procname       = "sched_min_granularity_ns",
+                .data           = &sysctl_sched_min_granularity,
                 .maxlen         = sizeof(unsigned int),
                 .mode           = 0644,
-                .proc_handler   = &proc_dointvec,
+                .proc_handler   = &sched_nr_latency_handler,
+                .strategy       = &sysctl_intvec,
+                .extra1         = &min_sched_granularity_ns,
+                .extra2         = &max_sched_granularity_ns,
         },
         {
                 .ctl_name       = CTL_UNNUMBERED,
@@ -247,7 +250,7 @@ static struct ctl_table kern_table[] = {
                 .data           = &sysctl_sched_latency,
                 .maxlen         = sizeof(unsigned int),
                 .mode           = 0644,
-                .proc_handler   = &proc_dointvec_minmax,
+                .proc_handler   = &sched_nr_latency_handler,
                 .strategy       = &sysctl_intvec,
                 .extra1         = &min_sched_granularity_ns,
                 .extra2         = &max_sched_granularity_ns,
@@ -298,6 +301,14 @@ static struct ctl_table kern_table[] = {
                 .mode           = 0644,
                 .proc_handler   = &proc_dointvec,
         },
+        {
+                .ctl_name       = CTL_UNNUMBERED,
+                .procname       = "sched_nr_migrate",
+                .data           = &sysctl_sched_nr_migrate,
+                .maxlen         = sizeof(unsigned int),
+                .mode           = 644,
+                .proc_handler   = &proc_dointvec,
+        },
 #endif
         {
                 .ctl_name       = CTL_UNNUMBERED,
diff --git a/kernel/timer.c b/kernel/timer.c
index 00e44e2afd67..a05817c021d6 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -817,6 +817,19 @@ unsigned long next_timer_interrupt(void)
 
 #endif
 
+#ifndef CONFIG_VIRT_CPU_ACCOUNTING
+void account_process_tick(struct task_struct *p, int user_tick)
+{
+        if (user_tick) {
+                account_user_time(p, jiffies_to_cputime(1));
+                account_user_time_scaled(p, jiffies_to_cputime(1));
+        } else {
+                account_system_time(p, HARDIRQ_OFFSET, jiffies_to_cputime(1));
+                account_system_time_scaled(p, jiffies_to_cputime(1));
+        }
+}
+#endif
+
 /*
  * Called from the timer interrupt handler to charge one tick to the current
  * process.  user_tick is 1 if the tick is user time, 0 for system.
@@ -827,13 +840,7 @@ void update_process_times(int user_tick)
         int cpu = smp_processor_id();
 
         /* Note: this timer irq context must be accounted for as well. */
-        if (user_tick) {
-                account_user_time(p, jiffies_to_cputime(1));
-                account_user_time_scaled(p, jiffies_to_cputime(1));
-        } else {
-                account_system_time(p, HARDIRQ_OFFSET, jiffies_to_cputime(1));
-                account_system_time_scaled(p, jiffies_to_cputime(1));
-        }
+        account_process_tick(p, user_tick);
         run_local_timers();
         if (rcu_pending(cpu))
                 rcu_check_callbacks(cpu, user_tick);