4 files changed, 149 insertions, 62 deletions

diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 4759917adc71..ff037f0b1b4e 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -91,7 +91,6 @@ enum hrtimer_restart {
  * @function:   timer expiry callback function
  * @base:       pointer to the timer base (per cpu and per clock)
  * @state:      state information (See bit values above)
- * @cb_entry:   list head to enqueue an expired timer into the callback list
  * @start_site: timer statistics field to store the site where the timer
  *              was started
  * @start_comm: timer statistics field to store the name of the process which
@@ -108,7 +107,6 @@ struct hrtimer {
         enum hrtimer_restart            (*function)(struct hrtimer *);
         struct hrtimer_clock_base       *base;
         unsigned long                   state;
-        struct list_head                cb_entry;
 #ifdef CONFIG_TIMER_STATS
         int                             start_pid;
         void                            *start_site;
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index 49da79ab8486..e2f91ecc01a8 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -48,37 +48,6 @@
 
 #include <asm/uaccess.h>
 
-/**
- * ktime_get - get the monotonic time in ktime_t format
- *
- * returns the time in ktime_t format
- */
-ktime_t ktime_get(void)
-{
-        struct timespec now;
-
-        ktime_get_ts(&now);
-
-        return timespec_to_ktime(now);
-}
-EXPORT_SYMBOL_GPL(ktime_get);
-
-/**
- * ktime_get_real - get the real (wall-) time in ktime_t format
- *
- * returns the time in ktime_t format
- */
-ktime_t ktime_get_real(void)
-{
-        struct timespec now;
-
-        getnstimeofday(&now);
-
-        return timespec_to_ktime(now);
-}
-
-EXPORT_SYMBOL_GPL(ktime_get_real);
-
 /*
  * The timer bases:
  *
@@ -106,31 +75,6 @@ DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
         }
 };
 
-/**
- * ktime_get_ts - get the monotonic clock in timespec format
- * @ts:         pointer to timespec variable
- *
- * The function calculates the monotonic clock from the realtime
- * clock and the wall_to_monotonic offset and stores the result
- * in normalized timespec format in the variable pointed to by @ts.
- */
-void ktime_get_ts(struct timespec *ts)
-{
-        struct timespec tomono;
-        unsigned long seq;
-
-        do {
-                seq = read_seqbegin(&xtime_lock);
-                getnstimeofday(ts);
-                tomono = wall_to_monotonic;
-
-        } while (read_seqretry(&xtime_lock, seq));
-
-        set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
-                                ts->tv_nsec + tomono.tv_nsec);
-}
-EXPORT_SYMBOL_GPL(ktime_get_ts);
-
 /*
  * Get the coarse grained time at the softirq based on xtime and
  * wall_to_monotonic.
@@ -1154,7 +1098,6 @@ static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
                 clock_id = CLOCK_MONOTONIC;
 
         timer->base = &cpu_base->clock_base[clock_id];
-        INIT_LIST_HEAD(&timer->cb_entry);
         hrtimer_init_timer_hres(timer);
 
 #ifdef CONFIG_TIMER_STATS
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index e8c77d9c633a..02c0b2c9c674 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -125,6 +125,75 @@ void getnstimeofday(struct timespec *ts)
 
 EXPORT_SYMBOL(getnstimeofday);
 
+ktime_t ktime_get(void)
+{
+        cycle_t cycle_now, cycle_delta;
+        unsigned int seq;
+        s64 secs, nsecs;
+
+        WARN_ON(timekeeping_suspended);
+
+        do {
+                seq = read_seqbegin(&xtime_lock);
+                secs = xtime.tv_sec + wall_to_monotonic.tv_sec;
+                nsecs = xtime.tv_nsec + wall_to_monotonic.tv_nsec;
+
+                /* read clocksource: */
+                cycle_now = clocksource_read(clock);
+
+                /* calculate the delta since the last update_wall_time: */
+                cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
+
+                /* convert to nanoseconds: */
+                nsecs += cyc2ns(clock, cycle_delta);
+
+        } while (read_seqretry(&xtime_lock, seq));
+        /*
+         * Use ktime_set/ktime_add_ns to create a proper ktime on
+         * 32-bit architectures without CONFIG_KTIME_SCALAR.
+         */
+        return ktime_add_ns(ktime_set(secs, 0), nsecs);
+}
+EXPORT_SYMBOL_GPL(ktime_get);
+
+/**
+ * ktime_get_ts - get the monotonic clock in timespec format
+ * @ts:         pointer to timespec variable
+ *
+ * The function calculates the monotonic clock from the realtime
+ * clock and the wall_to_monotonic offset and stores the result
+ * in normalized timespec format in the variable pointed to by @ts.
+ */
+void ktime_get_ts(struct timespec *ts)
+{
+        cycle_t cycle_now, cycle_delta;
+        struct timespec tomono;
+        unsigned int seq;
+        s64 nsecs;
+
+        WARN_ON(timekeeping_suspended);
+
+        do {
+                seq = read_seqbegin(&xtime_lock);
+                *ts = xtime;
+                tomono = wall_to_monotonic;
+
+                /* read clocksource: */
+                cycle_now = clocksource_read(clock);
+
+                /* calculate the delta since the last update_wall_time: */
+                cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
+
+                /* convert to nanoseconds: */
+                nsecs = cyc2ns(clock, cycle_delta);
+
+        } while (read_seqretry(&xtime_lock, seq));
+
+        set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
+                                ts->tv_nsec + tomono.tv_nsec + nsecs);
+}
+EXPORT_SYMBOL_GPL(ktime_get_ts);
+
 /**
  * do_gettimeofday - Returns the time of day in a timeval
  * @tv:         pointer to the timeval to be set
@@ -221,10 +290,65 @@ static void change_clocksource(void)
                clock->name);
          */
 }
-#else
+#else /* GENERIC_TIME */
 static inline void clocksource_forward_now(void) { }
 static inline void change_clocksource(void) { }
-#endif
+
+/**
+ * ktime_get - get the monotonic time in ktime_t format
+ *
+ * returns the time in ktime_t format
+ */
+ktime_t ktime_get(void)
+{
+        struct timespec now;
+
+        ktime_get_ts(&now);
+
+        return timespec_to_ktime(now);
+}
+EXPORT_SYMBOL_GPL(ktime_get);
+
+/**
+ * ktime_get_ts - get the monotonic clock in timespec format
+ * @ts:         pointer to timespec variable
+ *
+ * The function calculates the monotonic clock from the realtime
+ * clock and the wall_to_monotonic offset and stores the result
+ * in normalized timespec format in the variable pointed to by @ts.
+ */
+void ktime_get_ts(struct timespec *ts)
+{
+        struct timespec tomono;
+        unsigned long seq;
+
+        do {
+                seq = read_seqbegin(&xtime_lock);
+                getnstimeofday(ts);
+                tomono = wall_to_monotonic;
+
+        } while (read_seqretry(&xtime_lock, seq));
+
+        set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
+                                ts->tv_nsec + tomono.tv_nsec);
+}
+EXPORT_SYMBOL_GPL(ktime_get_ts);
+#endif /* !GENERIC_TIME */
+
+/**
+ * ktime_get_real - get the real (wall-) time in ktime_t format
+ *
+ * returns the time in ktime_t format
+ */
+ktime_t ktime_get_real(void)
+{
+        struct timespec now;
+
+        getnstimeofday(&now);
+
+        return timespec_to_ktime(now);
+}
+EXPORT_SYMBOL_GPL(ktime_get_real);
 
 /**
  * getrawmonotonic - Returns the raw monotonic time in a timespec
diff --git a/kernel/timer.c b/kernel/timer.c
index a7f07d5a6241..33fc9d175f40 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -72,6 +72,7 @@ struct tvec_base {
         spinlock_t lock;
         struct timer_list *running_timer;
         unsigned long timer_jiffies;
+        unsigned long next_timer;
         struct tvec_root tv1;
         struct tvec tv2;
         struct tvec tv3;
@@ -622,6 +623,9 @@ __mod_timer(struct timer_list *timer, unsigned long expires,
 
         if (timer_pending(timer)) {
                 detach_timer(timer, 0);
+                if (timer->expires == base->next_timer &&
+                    !tbase_get_deferrable(timer->base))
+                        base->next_timer = base->timer_jiffies;
                 ret = 1;
         } else {
                 if (pending_only)
@@ -663,6 +667,9 @@ __mod_timer(struct timer_list *timer, unsigned long expires,
         }
 
         timer->expires = expires;
+        if (time_before(timer->expires, base->next_timer) &&
+            !tbase_get_deferrable(timer->base))
+                base->next_timer = timer->expires;
         internal_add_timer(base, timer);
 
 out_unlock:
@@ -781,6 +788,9 @@ void add_timer_on(struct timer_list *timer, int cpu)
         spin_lock_irqsave(&base->lock, flags);
         timer_set_base(timer, base);
         debug_timer_activate(timer);
+        if (time_before(timer->expires, base->next_timer) &&
+            !tbase_get_deferrable(timer->base))
+                base->next_timer = timer->expires;
         internal_add_timer(base, timer);
         /*
          * Check whether the other CPU is idle and needs to be
@@ -817,6 +827,9 @@ int del_timer(struct timer_list *timer)
                 base = lock_timer_base(timer, &flags);
                 if (timer_pending(timer)) {
                         detach_timer(timer, 1);
+                        if (timer->expires == base->next_timer &&
+                            !tbase_get_deferrable(timer->base))
+                                base->next_timer = base->timer_jiffies;
                         ret = 1;
                 }
                 spin_unlock_irqrestore(&base->lock, flags);
@@ -850,6 +863,9 @@ int try_to_del_timer_sync(struct timer_list *timer)
         ret = 0;
         if (timer_pending(timer)) {
                 detach_timer(timer, 1);
+                if (timer->expires == base->next_timer &&
+                    !tbase_get_deferrable(timer->base))
+                        base->next_timer = base->timer_jiffies;
                 ret = 1;
         }
 out:
@@ -1134,7 +1150,9 @@ unsigned long get_next_timer_interrupt(unsigned long now)
         unsigned long expires;
 
         spin_lock(&base->lock);
-        expires = __next_timer_interrupt(base);
+        if (time_before_eq(base->next_timer, base->timer_jiffies))
+                base->next_timer = __next_timer_interrupt(base);
+        expires = base->next_timer;
         spin_unlock(&base->lock);
 
         if (time_before_eq(expires, now))
@@ -1523,6 +1541,7 @@ static int __cpuinit init_timers_cpu(int cpu)
                 INIT_LIST_HEAD(base->tv1.vec + j);
 
         base->timer_jiffies = jiffies;
+        base->next_timer = base->timer_jiffies;
         return 0;
 }
 
@@ -1535,6 +1554,9 @@ static void migrate_timer_list(struct tvec_base *new_base, struct list_head *hea
                 timer = list_first_entry(head, struct timer_list, entry);
                 detach_timer(timer, 0);
                 timer_set_base(timer, new_base);
+                if (time_before(timer->expires, new_base->next_timer) &&
+                    !tbase_get_deferrable(timer->base))
+                        new_base->next_timer = timer->expires;
                 internal_add_timer(new_base, timer);
         }
 }