aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/hrtimer.h31
-rw-r--r--kernel/hrtimer.c56
2 files changed, 82 insertions, 5 deletions
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 485a634fd6e2..28259c336679 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -112,6 +112,7 @@ enum hrtimer_cb_mode {
112struct hrtimer { 112struct hrtimer {
113 struct rb_node node; 113 struct rb_node node;
114 ktime_t _expires; 114 ktime_t _expires;
115 ktime_t _softexpires;
115 enum hrtimer_restart (*function)(struct hrtimer *); 116 enum hrtimer_restart (*function)(struct hrtimer *);
116 struct hrtimer_clock_base *base; 117 struct hrtimer_clock_base *base;
117 unsigned long state; 118 unsigned long state;
@@ -220,20 +221,37 @@ static inline int hrtimer_is_hres_active(struct hrtimer *timer)
220static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time) 221static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time)
221{ 222{
222 timer->_expires = time; 223 timer->_expires = time;
224 timer->_softexpires = time;
223} 225}
226
227static inline void hrtimer_set_expires_range(struct hrtimer *timer, ktime_t time, ktime_t delta)
228{
229 timer->_softexpires = time;
230 timer->_expires = ktime_add_safe(time, delta);
231}
232
233static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, unsigned long delta)
234{
235 timer->_softexpires = time;
236 timer->_expires = ktime_add_safe(time, ns_to_ktime(delta));
237}
238
224static inline void hrtimer_set_expires_tv64(struct hrtimer *timer, s64 tv64) 239static inline void hrtimer_set_expires_tv64(struct hrtimer *timer, s64 tv64)
225{ 240{
226 timer->_expires.tv64 = tv64; 241 timer->_expires.tv64 = tv64;
242 timer->_softexpires.tv64 = tv64;
227} 243}
228 244
229static inline void hrtimer_add_expires(struct hrtimer *timer, ktime_t time) 245static inline void hrtimer_add_expires(struct hrtimer *timer, ktime_t time)
230{ 246{
231 timer->_expires = ktime_add_safe(timer->_expires, time); 247 timer->_expires = ktime_add_safe(timer->_expires, time);
248 timer->_softexpires = ktime_add_safe(timer->_softexpires, time);
232} 249}
233 250
234static inline void hrtimer_add_expires_ns(struct hrtimer *timer, unsigned long ns) 251static inline void hrtimer_add_expires_ns(struct hrtimer *timer, unsigned long ns)
235{ 252{
236 timer->_expires = ktime_add_ns(timer->_expires, ns); 253 timer->_expires = ktime_add_ns(timer->_expires, ns);
254 timer->_softexpires = ktime_add_ns(timer->_softexpires, ns);
237} 255}
238 256
239static inline ktime_t hrtimer_get_expires(const struct hrtimer *timer) 257static inline ktime_t hrtimer_get_expires(const struct hrtimer *timer)
@@ -241,10 +259,19 @@ static inline ktime_t hrtimer_get_expires(const struct hrtimer *timer)
241 return timer->_expires; 259 return timer->_expires;
242} 260}
243 261
262static inline ktime_t hrtimer_get_softexpires(const struct hrtimer *timer)
263{
264 return timer->_softexpires;
265}
266
244static inline s64 hrtimer_get_expires_tv64(const struct hrtimer *timer) 267static inline s64 hrtimer_get_expires_tv64(const struct hrtimer *timer)
245{ 268{
246 return timer->_expires.tv64; 269 return timer->_expires.tv64;
247} 270}
271static inline s64 hrtimer_get_softexpires_tv64(const struct hrtimer *timer)
272{
273 return timer->_softexpires.tv64;
274}
248 275
249static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer) 276static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
250{ 277{
@@ -334,7 +361,7 @@ static inline int hrtimer_start_expires(struct hrtimer *timer,
334 361
335static inline int hrtimer_restart(struct hrtimer *timer) 362static inline int hrtimer_restart(struct hrtimer *timer)
336{ 363{
337 return hrtimer_start(timer, timer->_expires, HRTIMER_MODE_ABS); 364 return hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
338} 365}
339 366
340/* Query timers: */ 367/* Query timers: */
@@ -391,6 +418,8 @@ extern long hrtimer_nanosleep_restart(struct restart_block *restart_block);
391extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, 418extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
392 struct task_struct *tsk); 419 struct task_struct *tsk);
393 420
421extern int schedule_hrtimeout_range(ktime_t *expires, unsigned long delta,
422 const enum hrtimer_mode mode);
394extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode); 423extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);
395 424
396/* Soft interrupt function to run the hrtimer queues: */ 425/* Soft interrupt function to run the hrtimer queues: */
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index ae307feec74c..01483004183d 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -1309,7 +1309,20 @@ void hrtimer_interrupt(struct clock_event_device *dev)
1309 1309
1310 timer = rb_entry(node, struct hrtimer, node); 1310 timer = rb_entry(node, struct hrtimer, node);
1311 1311
1312 if (basenow.tv64 < hrtimer_get_expires_tv64(timer)) { 1312 /*
1313 * The immediate goal for using the softexpires is
1314 * minimizing wakeups, not running timers at the
1315 * earliest interrupt after their soft expiration.
1316 * This allows us to avoid using a Priority Search
1317 * Tree, which can answer a stabbing querry for
1318 * overlapping intervals and instead use the simple
1319 * BST we already have.
1320 * We don't add extra wakeups by delaying timers that
1321 * are right-of a not yet expired timer, because that
1322 * timer will have to trigger a wakeup anyway.
1323 */
1324
1325 if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer)) {
1313 ktime_t expires; 1326 ktime_t expires;
1314 1327
1315 expires = ktime_sub(hrtimer_get_expires(timer), 1328 expires = ktime_sub(hrtimer_get_expires(timer),
@@ -1681,14 +1694,20 @@ void __init hrtimers_init(void)
1681} 1694}
1682 1695
1683/** 1696/**
1684 * schedule_hrtimeout - sleep until timeout 1697 * schedule_hrtimeout_range - sleep until timeout
1685 * @expires: timeout value (ktime_t) 1698 * @expires: timeout value (ktime_t)
1699 * @delta: slack in expires timeout (ktime_t)
1686 * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL 1700 * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
1687 * 1701 *
1688 * Make the current task sleep until the given expiry time has 1702 * Make the current task sleep until the given expiry time has
1689 * elapsed. The routine will return immediately unless 1703 * elapsed. The routine will return immediately unless
1690 * the current task state has been set (see set_current_state()). 1704 * the current task state has been set (see set_current_state()).
1691 * 1705 *
1706 * The @delta argument gives the kernel the freedom to schedule the
1707 * actual wakeup to a time that is both power and performance friendly.
1708 * The kernel give the normal best effort behavior for "@expires+@delta",
1709 * but may decide to fire the timer earlier, but no earlier than @expires.
1710 *
1692 * You can set the task state as follows - 1711 * You can set the task state as follows -
1693 * 1712 *
1694 * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to 1713 * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
@@ -1702,7 +1721,7 @@ void __init hrtimers_init(void)
1702 * 1721 *
1703 * Returns 0 when the timer has expired otherwise -EINTR 1722 * Returns 0 when the timer has expired otherwise -EINTR
1704 */ 1723 */
1705int __sched schedule_hrtimeout(ktime_t *expires, 1724int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta,
1706 const enum hrtimer_mode mode) 1725 const enum hrtimer_mode mode)
1707{ 1726{
1708 struct hrtimer_sleeper t; 1727 struct hrtimer_sleeper t;
@@ -1726,7 +1745,7 @@ int __sched schedule_hrtimeout(ktime_t *expires,
1726 } 1745 }
1727 1746
1728 hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, mode); 1747 hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, mode);
1729 hrtimer_set_expires(&t.timer, *expires); 1748 hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
1730 1749
1731 hrtimer_init_sleeper(&t, current); 1750 hrtimer_init_sleeper(&t, current);
1732 1751
@@ -1744,4 +1763,33 @@ int __sched schedule_hrtimeout(ktime_t *expires,
1744 1763
1745 return !t.task ? 0 : -EINTR; 1764 return !t.task ? 0 : -EINTR;
1746} 1765}
1766EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
1767
1768/**
1769 * schedule_hrtimeout - sleep until timeout
1770 * @expires: timeout value (ktime_t)
1771 * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL
1772 *
1773 * Make the current task sleep until the given expiry time has
1774 * elapsed. The routine will return immediately unless
1775 * the current task state has been set (see set_current_state()).
1776 *
1777 * You can set the task state as follows -
1778 *
1779 * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to
1780 * pass before the routine returns.
1781 *
1782 * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
1783 * delivered to the current task.
1784 *
1785 * The current task state is guaranteed to be TASK_RUNNING when this
1786 * routine returns.
1787 *
1788 * Returns 0 when the timer has expired otherwise -EINTR
1789 */
1790int __sched schedule_hrtimeout(ktime_t *expires,
1791 const enum hrtimer_mode mode)
1792{
1793 return schedule_hrtimeout_range(expires, 0, mode);
1794}
1747EXPORT_SYMBOL_GPL(schedule_hrtimeout); 1795EXPORT_SYMBOL_GPL(schedule_hrtimeout);