1 files changed, 118 insertions, 97 deletions

diff --git a/kernel/timer.c b/kernel/timer.c
index b4555568b4e4..5db5a8d26811 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -37,6 +37,8 @@
 #include <linux/delay.h>
 #include <linux/tick.h>
 #include <linux/kallsyms.h>
+#include <linux/perf_event.h>
+#include <linux/sched.h>
 
 #include <asm/uaccess.h>
 #include <asm/unistd.h>
@@ -44,6 +46,9 @@
 #include <asm/timex.h>
 #include <asm/io.h>
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/timer.h>
+
 u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES;
 
 EXPORT_SYMBOL(jiffies_64);
@@ -70,6 +75,7 @@ struct tvec_base {
         spinlock_t lock;
         struct timer_list *running_timer;
         unsigned long timer_jiffies;
+        unsigned long next_timer;
         struct tvec_root tv1;
         struct tvec tv2;
         struct tvec tv3;
@@ -378,6 +384,8 @@ static void timer_stats_account_timer(struct timer_list *timer)
 {
         unsigned int flag = 0;
 
+        if (likely(!timer->start_site))
+                return;
         if (unlikely(tbase_get_deferrable(timer->base)))
                 flag |= TIMER_STATS_FLAG_DEFERRABLE;
 
@@ -516,6 +524,25 @@ static inline void debug_timer_activate(struct timer_list *timer) { }
 static inline void debug_timer_deactivate(struct timer_list *timer) { }
 #endif
 
+static inline void debug_init(struct timer_list *timer)
+{
+        debug_timer_init(timer);
+        trace_timer_init(timer);
+}
+
+static inline void
+debug_activate(struct timer_list *timer, unsigned long expires)
+{
+        debug_timer_activate(timer);
+        trace_timer_start(timer, expires);
+}
+
+static inline void debug_deactivate(struct timer_list *timer)
+{
+        debug_timer_deactivate(timer);
+        trace_timer_cancel(timer);
+}
+
 static void __init_timer(struct timer_list *timer,
                          const char *name,
                          struct lock_class_key *key)
@@ -531,17 +558,20 @@ static void __init_timer(struct timer_list *timer,
 }
 
 /**
- * init_timer - initialize a timer.
+ * init_timer_key - initialize a timer
  * @timer: the timer to be initialized
+ * @name: name of the timer
+ * @key: lockdep class key of the fake lock used for tracking timer
+ *       sync lock dependencies
  *
- * init_timer() must be done to a timer prior calling *any* of the
+ * init_timer_key() must be done to a timer prior calling *any* of the
  * other timer functions.
  */
 void init_timer_key(struct timer_list *timer,
                     const char *name,
                     struct lock_class_key *key)
 {
-        debug_timer_init(timer);
+        debug_init(timer);
         __init_timer(timer, name, key);
 }
 EXPORT_SYMBOL(init_timer_key);
@@ -560,7 +590,7 @@ static inline void detach_timer(struct timer_list *timer,
 {
         struct list_head *entry = &timer->entry;
 
-        debug_timer_deactivate(timer);
+        debug_deactivate(timer);
 
         __list_del(entry->prev, entry->next);
         if (clear_pending)
@@ -601,13 +631,12 @@ static struct tvec_base *lock_timer_base(struct timer_list *timer,
 }
 
 static inline int
-__mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
+__mod_timer(struct timer_list *timer, unsigned long expires,
+                                                bool pending_only, int pinned)
 {
         struct tvec_base *base, *new_base;
         unsigned long flags;
-        int ret;
-
-        ret = 0;
+        int ret = 0 , cpu;
 
         timer_stats_timer_set_start_info(timer);
         BUG_ON(!timer->function);
@@ -616,16 +645,31 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
 
         if (timer_pending(timer)) {
                 detach_timer(timer, 0);
+                if (timer->expires == base->next_timer &&
+                    !tbase_get_deferrable(timer->base))
+                        base->next_timer = base->timer_jiffies;
                 ret = 1;
         } else {
                 if (pending_only)
                         goto out_unlock;
         }
 
-        debug_timer_activate(timer);
+        debug_activate(timer, expires);
 
         new_base = __get_cpu_var(tvec_bases);
 
+        cpu = smp_processor_id();
+
+#if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP)
+        if (!pinned && get_sysctl_timer_migration() && idle_cpu(cpu)) {
+                int preferred_cpu = get_nohz_load_balancer();
+
+                if (preferred_cpu >= 0)
+                        cpu = preferred_cpu;
+        }
+#endif
+        new_base = per_cpu(tvec_bases, cpu);
+
         if (base != new_base) {
                 /*
                  * We are trying to schedule the timer on the local CPU.
@@ -645,6 +689,9 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
         }
 
         timer->expires = expires;
+        if (time_before(timer->expires, base->next_timer) &&
+            !tbase_get_deferrable(timer->base))
+                base->next_timer = timer->expires;
         internal_add_timer(base, timer);
 
 out_unlock:
@@ -665,7 +712,7 @@ out_unlock:
  */
 int mod_timer_pending(struct timer_list *timer, unsigned long expires)
 {
-        return __mod_timer(timer, expires, true);
+        return __mod_timer(timer, expires, true, TIMER_NOT_PINNED);
 }
 EXPORT_SYMBOL(mod_timer_pending);
 
@@ -696,14 +743,36 @@ int mod_timer(struct timer_list *timer, unsigned long expires)
          * networking code - if the timer is re-modified
          * to be the same thing then just return:
          */
-        if (timer->expires == expires && timer_pending(timer))
+        if (timer_pending(timer) && timer->expires == expires)
                 return 1;
 
-        return __mod_timer(timer, expires, false);
+        return __mod_timer(timer, expires, false, TIMER_NOT_PINNED);
 }
 EXPORT_SYMBOL(mod_timer);
 
 /**
+ * mod_timer_pinned - modify a timer's timeout
+ * @timer: the timer to be modified
+ * @expires: new timeout in jiffies
+ *
+ * mod_timer_pinned() is a way to update the expire field of an
+ * active timer (if the timer is inactive it will be activated)
+ * and not allow the timer to be migrated to a different CPU.
+ *
+ * mod_timer_pinned(timer, expires) is equivalent to:
+ *
+ *     del_timer(timer); timer->expires = expires; add_timer(timer);
+ */
+int mod_timer_pinned(struct timer_list *timer, unsigned long expires)
+{
+        if (timer->expires == expires && timer_pending(timer))
+                return 1;
+
+        return __mod_timer(timer, expires, false, TIMER_PINNED);
+}
+EXPORT_SYMBOL(mod_timer_pinned);
+
+/**
  * add_timer - start a timer
  * @timer: the timer to be added
  *
@@ -740,7 +809,10 @@ void add_timer_on(struct timer_list *timer, int cpu)
         BUG_ON(timer_pending(timer) || !timer->function);
         spin_lock_irqsave(&base->lock, flags);
         timer_set_base(timer, base);
-        debug_timer_activate(timer);
+        debug_activate(timer, timer->expires);
+        if (time_before(timer->expires, base->next_timer) &&
+            !tbase_get_deferrable(timer->base))
+                base->next_timer = timer->expires;
         internal_add_timer(base, timer);
         /*
          * Check whether the other CPU is idle and needs to be
@@ -753,6 +825,7 @@ void add_timer_on(struct timer_list *timer, int cpu)
         wake_up_idle_cpu(cpu);
         spin_unlock_irqrestore(&base->lock, flags);
 }
+EXPORT_SYMBOL_GPL(add_timer_on);
 
 /**
  * del_timer - deactive a timer.
@@ -776,6 +849,9 @@ int del_timer(struct timer_list *timer)
                 base = lock_timer_base(timer, &flags);
                 if (timer_pending(timer)) {
                         detach_timer(timer, 1);
+                        if (timer->expires == base->next_timer &&
+                            !tbase_get_deferrable(timer->base))
+                                base->next_timer = base->timer_jiffies;
                         ret = 1;
                 }
                 spin_unlock_irqrestore(&base->lock, flags);
@@ -809,6 +885,9 @@ int try_to_del_timer_sync(struct timer_list *timer)
         ret = 0;
         if (timer_pending(timer)) {
                 detach_timer(timer, 1);
+                if (timer->expires == base->next_timer &&
+                    !tbase_get_deferrable(timer->base))
+                        base->next_timer = base->timer_jiffies;
                 ret = 1;
         }
 out:
@@ -943,7 +1022,9 @@ static inline void __run_timers(struct tvec_base *base)
                                  */
                                 lock_map_acquire(&lockdep_map);
 
+                                trace_timer_expire_entry(timer);
                                 fn(data);
+                                trace_timer_expire_exit(timer);
 
                                 lock_map_release(&lockdep_map);
 
@@ -966,8 +1047,8 @@ static inline void __run_timers(struct tvec_base *base)
 #ifdef CONFIG_NO_HZ
 /*
  * Find out when the next timer event is due to happen. This
- * is used on S/390 to stop all activity when a cpus is idle.
- * This functions needs to be called disabled.
+ * is used on S/390 to stop all activity when a CPU is idle.
+ * This function needs to be called with interrupts disabled.
  */
 static unsigned long __next_timer_interrupt(struct tvec_base *base)
 {
@@ -1012,6 +1093,9 @@ cascade:
                 index = slot = timer_jiffies & TVN_MASK;
                 do {
                         list_for_each_entry(nte, varp->vec + slot, entry) {
+                                if (tbase_get_deferrable(nte->base))
+                                        continue;
+
                                 found = 1;
                                 if (time_before(nte->expires, expires))
                                         expires = nte->expires;
@@ -1090,7 +1174,9 @@ unsigned long get_next_timer_interrupt(unsigned long now)
         unsigned long expires;
 
         spin_lock(&base->lock);
-        expires = __next_timer_interrupt(base);
+        if (time_before_eq(base->next_timer, base->timer_jiffies))
+                base->next_timer = __next_timer_interrupt(base);
+        expires = base->next_timer;
         spin_unlock(&base->lock);
 
         if (time_before_eq(expires, now))
@@ -1112,61 +1198,21 @@ void update_process_times(int user_tick)
         /* Note: this timer irq context must be accounted for as well. */
         account_process_tick(p, user_tick);
         run_local_timers();
-        if (rcu_pending(cpu))
-                rcu_check_callbacks(cpu, user_tick);
+        rcu_check_callbacks(cpu, user_tick);
         printk_tick();
         scheduler_tick();
         run_posix_cpu_timers(p);
 }
 
 /*
- * Nr of active tasks - counted in fixed-point numbers
- */
-static unsigned long count_active_tasks(void)
-{
-        return nr_active() * FIXED_1;
-}
-
-/*
- * Hmm.. Changed this, as the GNU make sources (load.c) seems to
- * imply that avenrun[] is the standard name for this kind of thing.
- * Nothing else seems to be standardized: the fractional size etc
- * all seem to differ on different machines.
- *
- * Requires xtime_lock to access.
- */
-unsigned long avenrun[3];
-
-EXPORT_SYMBOL(avenrun);
-
-/*
- * calc_load - given tick count, update the avenrun load estimates.
- * This is called while holding a write_lock on xtime_lock.
- */
-static inline void calc_load(unsigned long ticks)
-{
-        unsigned long active_tasks; /* fixed-point */
-        static int count = LOAD_FREQ;
-
-        count -= ticks;
-        if (unlikely(count < 0)) {
-                active_tasks = count_active_tasks();
-                do {
-                        CALC_LOAD(avenrun[0], EXP_1, active_tasks);
-                        CALC_LOAD(avenrun[1], EXP_5, active_tasks);
-                        CALC_LOAD(avenrun[2], EXP_15, active_tasks);
-                        count += LOAD_FREQ;
-                } while (count < 0);
-        }
-}
-
-/*
  * This function runs timers and the timer-tq in bottom half context.
  */
 static void run_timer_softirq(struct softirq_action *h)
 {
         struct tvec_base *base = __get_cpu_var(tvec_bases);
 
+        perf_event_do_pending();
+
         hrtimer_run_pending();
 
         if (time_after_eq(jiffies, base->timer_jiffies))
@@ -1184,16 +1230,6 @@ void run_local_timers(void)
 }
 
 /*
- * Called by the timer interrupt. xtime_lock must already be taken
- * by the timer IRQ!
- */
-static inline void update_times(unsigned long ticks)
-{
-        update_wall_time();
-        calc_load(ticks);
-}
-
-/*
  * The 64-bit jiffies value is not atomic - you MUST NOT read it
  * without sampling the sequence number in xtime_lock.
  * jiffies is defined in the linker script...
@@ -1202,7 +1238,8 @@ static inline void update_times(unsigned long ticks)
 void do_timer(unsigned long ticks)
 {
         jiffies_64 += ticks;
-        update_times(ticks);
+        update_wall_time();
+        calc_global_load();
 }
 
 #ifdef __ARCH_WANT_SYS_ALARM
@@ -1350,7 +1387,7 @@ signed long __sched schedule_timeout(signed long timeout)
         expire = timeout + jiffies;
 
         setup_timer_on_stack(&timer, process_timeout, (unsigned long)current);
-        __mod_timer(&timer, expire, false);
+        __mod_timer(&timer, expire, false, TIMER_NOT_PINNED);
         schedule();
         del_singleshot_timer_sync(&timer);
 
@@ -1403,37 +1440,17 @@ int do_sysinfo(struct sysinfo *info)
 {
         unsigned long mem_total, sav_total;
         unsigned int mem_unit, bitcount;
-        unsigned long seq;
+        struct timespec tp;
 
         memset(info, 0, sizeof(struct sysinfo));
 
-        do {
-                struct timespec tp;
-                seq = read_seqbegin(&xtime_lock);
-
-                /*
-                 * This is annoying.  The below is the same thing
-                 * posix_get_clock_monotonic() does, but it wants to
-                 * take the lock which we want to cover the loads stuff
-                 * too.
-                 */
-
-                getnstimeofday(&tp);
-                tp.tv_sec += wall_to_monotonic.tv_sec;
-                tp.tv_nsec += wall_to_monotonic.tv_nsec;
-                monotonic_to_bootbased(&tp);
-                if (tp.tv_nsec - NSEC_PER_SEC >= 0) {
-                        tp.tv_nsec = tp.tv_nsec - NSEC_PER_SEC;
-                        tp.tv_sec++;
-                }
-                info->uptime = tp.tv_sec + (tp.tv_nsec ? 1 : 0);
+        ktime_get_ts(&tp);
+        monotonic_to_bootbased(&tp);
+        info->uptime = tp.tv_sec + (tp.tv_nsec ? 1 : 0);
 
-                info->loads[0] = avenrun[0] << (SI_LOAD_SHIFT - FSHIFT);
-                info->loads[1] = avenrun[1] << (SI_LOAD_SHIFT - FSHIFT);
-                info->loads[2] = avenrun[2] << (SI_LOAD_SHIFT - FSHIFT);
+        get_avenrun(info->loads, 0, SI_LOAD_SHIFT - FSHIFT);
 
-                info->procs = nr_threads;
-        } while (read_seqretry(&xtime_lock, seq));
+        info->procs = nr_threads;
 
         si_meminfo(info);
         si_swapinfo(info);
@@ -1547,6 +1564,7 @@ static int __cpuinit init_timers_cpu(int cpu)
                 INIT_LIST_HEAD(base->tv1.vec + j);
 
         base->timer_jiffies = jiffies;
+        base->next_timer = base->timer_jiffies;
         return 0;
 }
 
@@ -1559,6 +1577,9 @@ static void migrate_timer_list(struct tvec_base *new_base, struct list_head *hea
                 timer = list_first_entry(head, struct timer_list, entry);
                 detach_timer(timer, 0);
                 timer_set_base(timer, new_base);
+                if (time_before(timer->expires, new_base->next_timer) &&
+                    !tbase_get_deferrable(timer->base))
+                        new_base->next_timer = timer->expires;
                 internal_add_timer(new_base, timer);
         }
 }