diff options
Diffstat (limited to 'kernel')
-rw-r--r-- | kernel/fork.c | 2 | ||||
-rw-r--r-- | kernel/futex.c | 11 | ||||
-rw-r--r-- | kernel/hrtimer.c | 212 | ||||
-rw-r--r-- | kernel/posix-timers.c | 10 | ||||
-rw-r--r-- | kernel/rtmutex.c | 3 | ||||
-rw-r--r-- | kernel/sched.c | 7 | ||||
-rw-r--r-- | kernel/sys.c | 10 | ||||
-rw-r--r-- | kernel/time.c | 18 | ||||
-rw-r--r-- | kernel/time/ntp.c | 3 | ||||
-rw-r--r-- | kernel/time/tick-sched.c | 21 | ||||
-rw-r--r-- | kernel/time/timer_list.c | 8 |
11 files changed, 252 insertions, 53 deletions
diff --git a/kernel/fork.c b/kernel/fork.c index 30de644a40c4..37b3e150ae39 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
@@ -989,6 +989,8 @@ static struct task_struct *copy_process(unsigned long clone_flags, | |||
989 | p->prev_utime = cputime_zero; | 989 | p->prev_utime = cputime_zero; |
990 | p->prev_stime = cputime_zero; | 990 | p->prev_stime = cputime_zero; |
991 | 991 | ||
992 | p->default_timer_slack_ns = current->timer_slack_ns; | ||
993 | |||
992 | #ifdef CONFIG_DETECT_SOFTLOCKUP | 994 | #ifdef CONFIG_DETECT_SOFTLOCKUP |
993 | p->last_switch_count = 0; | 995 | p->last_switch_count = 0; |
994 | p->last_switch_timestamp = 0; | 996 | p->last_switch_timestamp = 0; |
diff --git a/kernel/futex.c b/kernel/futex.c index 7d1136e97c14..8af10027514b 100644 --- a/kernel/futex.c +++ b/kernel/futex.c | |||
@@ -1296,13 +1296,16 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared, | |||
1296 | if (!abs_time) | 1296 | if (!abs_time) |
1297 | schedule(); | 1297 | schedule(); |
1298 | else { | 1298 | else { |
1299 | unsigned long slack; | ||
1300 | slack = current->timer_slack_ns; | ||
1301 | if (rt_task(current)) | ||
1302 | slack = 0; | ||
1299 | hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, | 1303 | hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, |
1300 | HRTIMER_MODE_ABS); | 1304 | HRTIMER_MODE_ABS); |
1301 | hrtimer_init_sleeper(&t, current); | 1305 | hrtimer_init_sleeper(&t, current); |
1302 | t.timer.expires = *abs_time; | 1306 | hrtimer_set_expires_range_ns(&t.timer, *abs_time, slack); |
1303 | 1307 | ||
1304 | hrtimer_start(&t.timer, t.timer.expires, | 1308 | hrtimer_start_expires(&t.timer, HRTIMER_MODE_ABS); |
1305 | HRTIMER_MODE_ABS); | ||
1306 | if (!hrtimer_active(&t.timer)) | 1309 | if (!hrtimer_active(&t.timer)) |
1307 | t.task = NULL; | 1310 | t.task = NULL; |
1308 | 1311 | ||
@@ -1404,7 +1407,7 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared, | |||
1404 | hrtimer_init_on_stack(&to->timer, CLOCK_REALTIME, | 1407 | hrtimer_init_on_stack(&to->timer, CLOCK_REALTIME, |
1405 | HRTIMER_MODE_ABS); | 1408 | HRTIMER_MODE_ABS); |
1406 | hrtimer_init_sleeper(to, current); | 1409 | hrtimer_init_sleeper(to, current); |
1407 | to->timer.expires = *time; | 1410 | hrtimer_set_expires(&to->timer, *time); |
1408 | } | 1411 | } |
1409 | 1412 | ||
1410 | q.pi_state = NULL; | 1413 | q.pi_state = NULL; |
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index cdec83e722fa..51ee90bca2de 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
@@ -517,7 +517,7 @@ static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base) | |||
517 | if (!base->first) | 517 | if (!base->first) |
518 | continue; | 518 | continue; |
519 | timer = rb_entry(base->first, struct hrtimer, node); | 519 | timer = rb_entry(base->first, struct hrtimer, node); |
520 | expires = ktime_sub(timer->expires, base->offset); | 520 | expires = ktime_sub(hrtimer_get_expires(timer), base->offset); |
521 | if (expires.tv64 < cpu_base->expires_next.tv64) | 521 | if (expires.tv64 < cpu_base->expires_next.tv64) |
522 | cpu_base->expires_next = expires; | 522 | cpu_base->expires_next = expires; |
523 | } | 523 | } |
@@ -539,10 +539,10 @@ static int hrtimer_reprogram(struct hrtimer *timer, | |||
539 | struct hrtimer_clock_base *base) | 539 | struct hrtimer_clock_base *base) |
540 | { | 540 | { |
541 | ktime_t *expires_next = &__get_cpu_var(hrtimer_bases).expires_next; | 541 | ktime_t *expires_next = &__get_cpu_var(hrtimer_bases).expires_next; |
542 | ktime_t expires = ktime_sub(timer->expires, base->offset); | 542 | ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset); |
543 | int res; | 543 | int res; |
544 | 544 | ||
545 | WARN_ON_ONCE(timer->expires.tv64 < 0); | 545 | WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0); |
546 | 546 | ||
547 | /* | 547 | /* |
548 | * When the callback is running, we do not reprogram the clock event | 548 | * When the callback is running, we do not reprogram the clock event |
@@ -795,7 +795,7 @@ u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) | |||
795 | u64 orun = 1; | 795 | u64 orun = 1; |
796 | ktime_t delta; | 796 | ktime_t delta; |
797 | 797 | ||
798 | delta = ktime_sub(now, timer->expires); | 798 | delta = ktime_sub(now, hrtimer_get_expires(timer)); |
799 | 799 | ||
800 | if (delta.tv64 < 0) | 800 | if (delta.tv64 < 0) |
801 | return 0; | 801 | return 0; |
@@ -807,8 +807,8 @@ u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) | |||
807 | s64 incr = ktime_to_ns(interval); | 807 | s64 incr = ktime_to_ns(interval); |
808 | 808 | ||
809 | orun = ktime_divns(delta, incr); | 809 | orun = ktime_divns(delta, incr); |
810 | timer->expires = ktime_add_ns(timer->expires, incr * orun); | 810 | hrtimer_add_expires_ns(timer, incr * orun); |
811 | if (timer->expires.tv64 > now.tv64) | 811 | if (hrtimer_get_expires_tv64(timer) > now.tv64) |
812 | return orun; | 812 | return orun; |
813 | /* | 813 | /* |
814 | * This (and the ktime_add() below) is the | 814 | * This (and the ktime_add() below) is the |
@@ -816,7 +816,7 @@ u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) | |||
816 | */ | 816 | */ |
817 | orun++; | 817 | orun++; |
818 | } | 818 | } |
819 | timer->expires = ktime_add_safe(timer->expires, interval); | 819 | hrtimer_add_expires(timer, interval); |
820 | 820 | ||
821 | return orun; | 821 | return orun; |
822 | } | 822 | } |
@@ -848,7 +848,8 @@ static void enqueue_hrtimer(struct hrtimer *timer, | |||
848 | * We dont care about collisions. Nodes with | 848 | * We dont care about collisions. Nodes with |
849 | * the same expiry time stay together. | 849 | * the same expiry time stay together. |
850 | */ | 850 | */ |
851 | if (timer->expires.tv64 < entry->expires.tv64) { | 851 | if (hrtimer_get_expires_tv64(timer) < |
852 | hrtimer_get_expires_tv64(entry)) { | ||
852 | link = &(*link)->rb_left; | 853 | link = &(*link)->rb_left; |
853 | } else { | 854 | } else { |
854 | link = &(*link)->rb_right; | 855 | link = &(*link)->rb_right; |
@@ -945,9 +946,10 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base) | |||
945 | } | 946 | } |
946 | 947 | ||
947 | /** | 948 | /** |
948 | * hrtimer_start - (re)start an relative timer on the current CPU | 949 | * hrtimer_start_range_ns - (re)start an relative timer on the current CPU |
949 | * @timer: the timer to be added | 950 | * @timer: the timer to be added |
950 | * @tim: expiry time | 951 | * @tim: expiry time |
952 | * @delta_ns: "slack" range for the timer | ||
951 | * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL) | 953 | * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL) |
952 | * | 954 | * |
953 | * Returns: | 955 | * Returns: |
@@ -955,7 +957,8 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base) | |||
955 | * 1 when the timer was active | 957 | * 1 when the timer was active |
956 | */ | 958 | */ |
957 | int | 959 | int |
958 | hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) | 960 | hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_ns, |
961 | const enum hrtimer_mode mode) | ||
959 | { | 962 | { |
960 | struct hrtimer_clock_base *base, *new_base; | 963 | struct hrtimer_clock_base *base, *new_base; |
961 | unsigned long flags; | 964 | unsigned long flags; |
@@ -983,7 +986,7 @@ hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) | |||
983 | #endif | 986 | #endif |
984 | } | 987 | } |
985 | 988 | ||
986 | timer->expires = tim; | 989 | hrtimer_set_expires_range_ns(timer, tim, delta_ns); |
987 | 990 | ||
988 | timer_stats_hrtimer_set_start_info(timer); | 991 | timer_stats_hrtimer_set_start_info(timer); |
989 | 992 | ||
@@ -1016,8 +1019,26 @@ hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) | |||
1016 | 1019 | ||
1017 | return ret; | 1020 | return ret; |
1018 | } | 1021 | } |
1022 | EXPORT_SYMBOL_GPL(hrtimer_start_range_ns); | ||
1023 | |||
1024 | /** | ||
1025 | * hrtimer_start - (re)start an relative timer on the current CPU | ||
1026 | * @timer: the timer to be added | ||
1027 | * @tim: expiry time | ||
1028 | * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL) | ||
1029 | * | ||
1030 | * Returns: | ||
1031 | * 0 on success | ||
1032 | * 1 when the timer was active | ||
1033 | */ | ||
1034 | int | ||
1035 | hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) | ||
1036 | { | ||
1037 | return hrtimer_start_range_ns(timer, tim, 0, mode); | ||
1038 | } | ||
1019 | EXPORT_SYMBOL_GPL(hrtimer_start); | 1039 | EXPORT_SYMBOL_GPL(hrtimer_start); |
1020 | 1040 | ||
1041 | |||
1021 | /** | 1042 | /** |
1022 | * hrtimer_try_to_cancel - try to deactivate a timer | 1043 | * hrtimer_try_to_cancel - try to deactivate a timer |
1023 | * @timer: hrtimer to stop | 1044 | * @timer: hrtimer to stop |
@@ -1077,7 +1098,7 @@ ktime_t hrtimer_get_remaining(const struct hrtimer *timer) | |||
1077 | ktime_t rem; | 1098 | ktime_t rem; |
1078 | 1099 | ||
1079 | base = lock_hrtimer_base(timer, &flags); | 1100 | base = lock_hrtimer_base(timer, &flags); |
1080 | rem = ktime_sub(timer->expires, base->get_time()); | 1101 | rem = hrtimer_expires_remaining(timer); |
1081 | unlock_hrtimer_base(timer, &flags); | 1102 | unlock_hrtimer_base(timer, &flags); |
1082 | 1103 | ||
1083 | return rem; | 1104 | return rem; |
@@ -1109,7 +1130,7 @@ ktime_t hrtimer_get_next_event(void) | |||
1109 | continue; | 1130 | continue; |
1110 | 1131 | ||
1111 | timer = rb_entry(base->first, struct hrtimer, node); | 1132 | timer = rb_entry(base->first, struct hrtimer, node); |
1112 | delta.tv64 = timer->expires.tv64; | 1133 | delta.tv64 = hrtimer_get_expires_tv64(timer); |
1113 | delta = ktime_sub(delta, base->get_time()); | 1134 | delta = ktime_sub(delta, base->get_time()); |
1114 | if (delta.tv64 < mindelta.tv64) | 1135 | if (delta.tv64 < mindelta.tv64) |
1115 | mindelta.tv64 = delta.tv64; | 1136 | mindelta.tv64 = delta.tv64; |
@@ -1310,10 +1331,23 @@ void hrtimer_interrupt(struct clock_event_device *dev) | |||
1310 | 1331 | ||
1311 | timer = rb_entry(node, struct hrtimer, node); | 1332 | timer = rb_entry(node, struct hrtimer, node); |
1312 | 1333 | ||
1313 | if (basenow.tv64 < timer->expires.tv64) { | 1334 | /* |
1335 | * The immediate goal for using the softexpires is | ||
1336 | * minimizing wakeups, not running timers at the | ||
1337 | * earliest interrupt after their soft expiration. | ||
1338 | * This allows us to avoid using a Priority Search | ||
1339 | * Tree, which can answer a stabbing querry for | ||
1340 | * overlapping intervals and instead use the simple | ||
1341 | * BST we already have. | ||
1342 | * We don't add extra wakeups by delaying timers that | ||
1343 | * are right-of a not yet expired timer, because that | ||
1344 | * timer will have to trigger a wakeup anyway. | ||
1345 | */ | ||
1346 | |||
1347 | if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer)) { | ||
1314 | ktime_t expires; | 1348 | ktime_t expires; |
1315 | 1349 | ||
1316 | expires = ktime_sub(timer->expires, | 1350 | expires = ktime_sub(hrtimer_get_expires(timer), |
1317 | base->offset); | 1351 | base->offset); |
1318 | if (expires.tv64 < expires_next.tv64) | 1352 | if (expires.tv64 < expires_next.tv64) |
1319 | expires_next = expires; | 1353 | expires_next = expires; |
@@ -1349,6 +1383,36 @@ void hrtimer_interrupt(struct clock_event_device *dev) | |||
1349 | raise_softirq(HRTIMER_SOFTIRQ); | 1383 | raise_softirq(HRTIMER_SOFTIRQ); |
1350 | } | 1384 | } |
1351 | 1385 | ||
1386 | /** | ||
1387 | * hrtimer_peek_ahead_timers -- run soft-expired timers now | ||
1388 | * | ||
1389 | * hrtimer_peek_ahead_timers will peek at the timer queue of | ||
1390 | * the current cpu and check if there are any timers for which | ||
1391 | * the soft expires time has passed. If any such timers exist, | ||
1392 | * they are run immediately and then removed from the timer queue. | ||
1393 | * | ||
1394 | */ | ||
1395 | void hrtimer_peek_ahead_timers(void) | ||
1396 | { | ||
1397 | unsigned long flags; | ||
1398 | struct tick_device *td; | ||
1399 | struct clock_event_device *dev; | ||
1400 | |||
1401 | if (!hrtimer_hres_active()) | ||
1402 | return; | ||
1403 | |||
1404 | local_irq_save(flags); | ||
1405 | td = &__get_cpu_var(tick_cpu_device); | ||
1406 | if (!td) | ||
1407 | goto out; | ||
1408 | dev = td->evtdev; | ||
1409 | if (!dev) | ||
1410 | goto out; | ||
1411 | hrtimer_interrupt(dev); | ||
1412 | out: | ||
1413 | local_irq_restore(flags); | ||
1414 | } | ||
1415 | |||
1352 | static void run_hrtimer_softirq(struct softirq_action *h) | 1416 | static void run_hrtimer_softirq(struct softirq_action *h) |
1353 | { | 1417 | { |
1354 | run_hrtimer_pending(&__get_cpu_var(hrtimer_bases)); | 1418 | run_hrtimer_pending(&__get_cpu_var(hrtimer_bases)); |
@@ -1416,7 +1480,8 @@ void hrtimer_run_queues(void) | |||
1416 | struct hrtimer *timer; | 1480 | struct hrtimer *timer; |
1417 | 1481 | ||
1418 | timer = rb_entry(node, struct hrtimer, node); | 1482 | timer = rb_entry(node, struct hrtimer, node); |
1419 | if (base->softirq_time.tv64 <= timer->expires.tv64) | 1483 | if (base->softirq_time.tv64 <= |
1484 | hrtimer_get_expires_tv64(timer)) | ||
1420 | break; | 1485 | break; |
1421 | 1486 | ||
1422 | if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) { | 1487 | if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) { |
@@ -1464,7 +1529,7 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod | |||
1464 | 1529 | ||
1465 | do { | 1530 | do { |
1466 | set_current_state(TASK_INTERRUPTIBLE); | 1531 | set_current_state(TASK_INTERRUPTIBLE); |
1467 | hrtimer_start(&t->timer, t->timer.expires, mode); | 1532 | hrtimer_start_expires(&t->timer, mode); |
1468 | if (!hrtimer_active(&t->timer)) | 1533 | if (!hrtimer_active(&t->timer)) |
1469 | t->task = NULL; | 1534 | t->task = NULL; |
1470 | 1535 | ||
@@ -1486,7 +1551,7 @@ static int update_rmtp(struct hrtimer *timer, struct timespec __user *rmtp) | |||
1486 | struct timespec rmt; | 1551 | struct timespec rmt; |
1487 | ktime_t rem; | 1552 | ktime_t rem; |
1488 | 1553 | ||
1489 | rem = ktime_sub(timer->expires, timer->base->get_time()); | 1554 | rem = hrtimer_expires_remaining(timer); |
1490 | if (rem.tv64 <= 0) | 1555 | if (rem.tv64 <= 0) |
1491 | return 0; | 1556 | return 0; |
1492 | rmt = ktime_to_timespec(rem); | 1557 | rmt = ktime_to_timespec(rem); |
@@ -1505,7 +1570,7 @@ long __sched hrtimer_nanosleep_restart(struct restart_block *restart) | |||
1505 | 1570 | ||
1506 | hrtimer_init_on_stack(&t.timer, restart->nanosleep.index, | 1571 | hrtimer_init_on_stack(&t.timer, restart->nanosleep.index, |
1507 | HRTIMER_MODE_ABS); | 1572 | HRTIMER_MODE_ABS); |
1508 | t.timer.expires.tv64 = restart->nanosleep.expires; | 1573 | hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); |
1509 | 1574 | ||
1510 | if (do_nanosleep(&t, HRTIMER_MODE_ABS)) | 1575 | if (do_nanosleep(&t, HRTIMER_MODE_ABS)) |
1511 | goto out; | 1576 | goto out; |
@@ -1530,9 +1595,14 @@ long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, | |||
1530 | struct restart_block *restart; | 1595 | struct restart_block *restart; |
1531 | struct hrtimer_sleeper t; | 1596 | struct hrtimer_sleeper t; |
1532 | int ret = 0; | 1597 | int ret = 0; |
1598 | unsigned long slack; | ||
1599 | |||
1600 | slack = current->timer_slack_ns; | ||
1601 | if (rt_task(current)) | ||
1602 | slack = 0; | ||
1533 | 1603 | ||
1534 | hrtimer_init_on_stack(&t.timer, clockid, mode); | 1604 | hrtimer_init_on_stack(&t.timer, clockid, mode); |
1535 | t.timer.expires = timespec_to_ktime(*rqtp); | 1605 | hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack); |
1536 | if (do_nanosleep(&t, mode)) | 1606 | if (do_nanosleep(&t, mode)) |
1537 | goto out; | 1607 | goto out; |
1538 | 1608 | ||
@@ -1552,7 +1622,7 @@ long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, | |||
1552 | restart->fn = hrtimer_nanosleep_restart; | 1622 | restart->fn = hrtimer_nanosleep_restart; |
1553 | restart->nanosleep.index = t.timer.base->index; | 1623 | restart->nanosleep.index = t.timer.base->index; |
1554 | restart->nanosleep.rmtp = rmtp; | 1624 | restart->nanosleep.rmtp = rmtp; |
1555 | restart->nanosleep.expires = t.timer.expires.tv64; | 1625 | restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer); |
1556 | 1626 | ||
1557 | ret = -ERESTART_RESTARTBLOCK; | 1627 | ret = -ERESTART_RESTARTBLOCK; |
1558 | out: | 1628 | out: |
@@ -1753,3 +1823,103 @@ void __init hrtimers_init(void) | |||
1753 | #endif | 1823 | #endif |
1754 | } | 1824 | } |
1755 | 1825 | ||
1826 | /** | ||
1827 | * schedule_hrtimeout_range - sleep until timeout | ||
1828 | * @expires: timeout value (ktime_t) | ||
1829 | * @delta: slack in expires timeout (ktime_t) | ||
1830 | * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL | ||
1831 | * | ||
1832 | * Make the current task sleep until the given expiry time has | ||
1833 | * elapsed. The routine will return immediately unless | ||
1834 | * the current task state has been set (see set_current_state()). | ||
1835 | * | ||
1836 | * The @delta argument gives the kernel the freedom to schedule the | ||
1837 | * actual wakeup to a time that is both power and performance friendly. | ||
1838 | * The kernel give the normal best effort behavior for "@expires+@delta", | ||
1839 | * but may decide to fire the timer earlier, but no earlier than @expires. | ||
1840 | * | ||
1841 | * You can set the task state as follows - | ||
1842 | * | ||
1843 | * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to | ||
1844 | * pass before the routine returns. | ||
1845 | * | ||
1846 | * %TASK_INTERRUPTIBLE - the routine may return early if a signal is | ||
1847 | * delivered to the current task. | ||
1848 | * | ||
1849 | * The current task state is guaranteed to be TASK_RUNNING when this | ||
1850 | * routine returns. | ||
1851 | * | ||
1852 | * Returns 0 when the timer has expired otherwise -EINTR | ||
1853 | */ | ||
1854 | int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta, | ||
1855 | const enum hrtimer_mode mode) | ||
1856 | { | ||
1857 | struct hrtimer_sleeper t; | ||
1858 | |||
1859 | /* | ||
1860 | * Optimize when a zero timeout value is given. It does not | ||
1861 | * matter whether this is an absolute or a relative time. | ||
1862 | */ | ||
1863 | if (expires && !expires->tv64) { | ||
1864 | __set_current_state(TASK_RUNNING); | ||
1865 | return 0; | ||
1866 | } | ||
1867 | |||
1868 | /* | ||
1869 | * A NULL parameter means "inifinte" | ||
1870 | */ | ||
1871 | if (!expires) { | ||
1872 | schedule(); | ||
1873 | __set_current_state(TASK_RUNNING); | ||
1874 | return -EINTR; | ||
1875 | } | ||
1876 | |||
1877 | hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, mode); | ||
1878 | hrtimer_set_expires_range_ns(&t.timer, *expires, delta); | ||
1879 | |||
1880 | hrtimer_init_sleeper(&t, current); | ||
1881 | |||
1882 | hrtimer_start_expires(&t.timer, mode); | ||
1883 | if (!hrtimer_active(&t.timer)) | ||
1884 | t.task = NULL; | ||
1885 | |||
1886 | if (likely(t.task)) | ||
1887 | schedule(); | ||
1888 | |||
1889 | hrtimer_cancel(&t.timer); | ||
1890 | destroy_hrtimer_on_stack(&t.timer); | ||
1891 | |||
1892 | __set_current_state(TASK_RUNNING); | ||
1893 | |||
1894 | return !t.task ? 0 : -EINTR; | ||
1895 | } | ||
1896 | EXPORT_SYMBOL_GPL(schedule_hrtimeout_range); | ||
1897 | |||
1898 | /** | ||
1899 | * schedule_hrtimeout - sleep until timeout | ||
1900 | * @expires: timeout value (ktime_t) | ||
1901 | * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL | ||
1902 | * | ||
1903 | * Make the current task sleep until the given expiry time has | ||
1904 | * elapsed. The routine will return immediately unless | ||
1905 | * the current task state has been set (see set_current_state()). | ||
1906 | * | ||
1907 | * You can set the task state as follows - | ||
1908 | * | ||
1909 | * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to | ||
1910 | * pass before the routine returns. | ||
1911 | * | ||
1912 | * %TASK_INTERRUPTIBLE - the routine may return early if a signal is | ||
1913 | * delivered to the current task. | ||
1914 | * | ||
1915 | * The current task state is guaranteed to be TASK_RUNNING when this | ||
1916 | * routine returns. | ||
1917 | * | ||
1918 | * Returns 0 when the timer has expired otherwise -EINTR | ||
1919 | */ | ||
1920 | int __sched schedule_hrtimeout(ktime_t *expires, | ||
1921 | const enum hrtimer_mode mode) | ||
1922 | { | ||
1923 | return schedule_hrtimeout_range(expires, 0, mode); | ||
1924 | } | ||
1925 | EXPORT_SYMBOL_GPL(schedule_hrtimeout); | ||
diff --git a/kernel/posix-timers.c b/kernel/posix-timers.c index 5131e5471169..ee204586149a 100644 --- a/kernel/posix-timers.c +++ b/kernel/posix-timers.c | |||
@@ -668,7 +668,7 @@ common_timer_get(struct k_itimer *timr, struct itimerspec *cur_setting) | |||
668 | (timr->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE)) | 668 | (timr->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE)) |
669 | timr->it_overrun += (unsigned int) hrtimer_forward(timer, now, iv); | 669 | timr->it_overrun += (unsigned int) hrtimer_forward(timer, now, iv); |
670 | 670 | ||
671 | remaining = ktime_sub(timer->expires, now); | 671 | remaining = ktime_sub(hrtimer_get_expires(timer), now); |
672 | /* Return 0 only, when the timer is expired and not pending */ | 672 | /* Return 0 only, when the timer is expired and not pending */ |
673 | if (remaining.tv64 <= 0) { | 673 | if (remaining.tv64 <= 0) { |
674 | /* | 674 | /* |
@@ -762,7 +762,7 @@ common_timer_set(struct k_itimer *timr, int flags, | |||
762 | hrtimer_init(&timr->it.real.timer, timr->it_clock, mode); | 762 | hrtimer_init(&timr->it.real.timer, timr->it_clock, mode); |
763 | timr->it.real.timer.function = posix_timer_fn; | 763 | timr->it.real.timer.function = posix_timer_fn; |
764 | 764 | ||
765 | timer->expires = timespec_to_ktime(new_setting->it_value); | 765 | hrtimer_set_expires(timer, timespec_to_ktime(new_setting->it_value)); |
766 | 766 | ||
767 | /* Convert interval */ | 767 | /* Convert interval */ |
768 | timr->it.real.interval = timespec_to_ktime(new_setting->it_interval); | 768 | timr->it.real.interval = timespec_to_ktime(new_setting->it_interval); |
@@ -771,14 +771,12 @@ common_timer_set(struct k_itimer *timr, int flags, | |||
771 | if (((timr->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE)) { | 771 | if (((timr->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE)) { |
772 | /* Setup correct expiry time for relative timers */ | 772 | /* Setup correct expiry time for relative timers */ |
773 | if (mode == HRTIMER_MODE_REL) { | 773 | if (mode == HRTIMER_MODE_REL) { |
774 | timer->expires = | 774 | hrtimer_add_expires(timer, timer->base->get_time()); |
775 | ktime_add_safe(timer->expires, | ||
776 | timer->base->get_time()); | ||
777 | } | 775 | } |
778 | return 0; | 776 | return 0; |
779 | } | 777 | } |
780 | 778 | ||
781 | hrtimer_start(timer, timer->expires, mode); | 779 | hrtimer_start_expires(timer, mode); |
782 | return 0; | 780 | return 0; |
783 | } | 781 | } |
784 | 782 | ||
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c index 6522ae5b14a2..69d9cb921ffa 100644 --- a/kernel/rtmutex.c +++ b/kernel/rtmutex.c | |||
@@ -631,8 +631,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, | |||
631 | 631 | ||
632 | /* Setup the timer, when timeout != NULL */ | 632 | /* Setup the timer, when timeout != NULL */ |
633 | if (unlikely(timeout)) { | 633 | if (unlikely(timeout)) { |
634 | hrtimer_start(&timeout->timer, timeout->timer.expires, | 634 | hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS); |
635 | HRTIMER_MODE_ABS); | ||
636 | if (!hrtimer_active(&timeout->timer)) | 635 | if (!hrtimer_active(&timeout->timer)) |
637 | timeout->task = NULL; | 636 | timeout->task = NULL; |
638 | } | 637 | } |
diff --git a/kernel/sched.c b/kernel/sched.c index 6f230596bd0c..eb3c72953615 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -226,9 +226,8 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b) | |||
226 | 226 | ||
227 | now = hrtimer_cb_get_time(&rt_b->rt_period_timer); | 227 | now = hrtimer_cb_get_time(&rt_b->rt_period_timer); |
228 | hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period); | 228 | hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period); |
229 | hrtimer_start(&rt_b->rt_period_timer, | 229 | hrtimer_start_expires(&rt_b->rt_period_timer, |
230 | rt_b->rt_period_timer.expires, | 230 | HRTIMER_MODE_ABS); |
231 | HRTIMER_MODE_ABS); | ||
232 | } | 231 | } |
233 | spin_unlock(&rt_b->rt_runtime_lock); | 232 | spin_unlock(&rt_b->rt_runtime_lock); |
234 | } | 233 | } |
@@ -1063,7 +1062,7 @@ static void hrtick_start(struct rq *rq, u64 delay) | |||
1063 | struct hrtimer *timer = &rq->hrtick_timer; | 1062 | struct hrtimer *timer = &rq->hrtick_timer; |
1064 | ktime_t time = ktime_add_ns(timer->base->get_time(), delay); | 1063 | ktime_t time = ktime_add_ns(timer->base->get_time(), delay); |
1065 | 1064 | ||
1066 | timer->expires = time; | 1065 | hrtimer_set_expires(timer, time); |
1067 | 1066 | ||
1068 | if (rq == this_rq()) { | 1067 | if (rq == this_rq()) { |
1069 | hrtimer_restart(timer); | 1068 | hrtimer_restart(timer); |
diff --git a/kernel/sys.c b/kernel/sys.c index 0bc8fa3c2288..fc71f99fb469 100644 --- a/kernel/sys.c +++ b/kernel/sys.c | |||
@@ -1739,6 +1739,16 @@ asmlinkage long sys_prctl(int option, unsigned long arg2, unsigned long arg3, | |||
1739 | case PR_SET_TSC: | 1739 | case PR_SET_TSC: |
1740 | error = SET_TSC_CTL(arg2); | 1740 | error = SET_TSC_CTL(arg2); |
1741 | break; | 1741 | break; |
1742 | case PR_GET_TIMERSLACK: | ||
1743 | error = current->timer_slack_ns; | ||
1744 | break; | ||
1745 | case PR_SET_TIMERSLACK: | ||
1746 | if (arg2 <= 0) | ||
1747 | current->timer_slack_ns = | ||
1748 | current->default_timer_slack_ns; | ||
1749 | else | ||
1750 | current->timer_slack_ns = arg2; | ||
1751 | break; | ||
1742 | default: | 1752 | default: |
1743 | error = -EINVAL; | 1753 | error = -EINVAL; |
1744 | break; | 1754 | break; |
diff --git a/kernel/time.c b/kernel/time.c index 6a08660b4fac..d63a4336fad6 100644 --- a/kernel/time.c +++ b/kernel/time.c | |||
@@ -669,3 +669,21 @@ EXPORT_SYMBOL(get_jiffies_64); | |||
669 | #endif | 669 | #endif |
670 | 670 | ||
671 | EXPORT_SYMBOL(jiffies); | 671 | EXPORT_SYMBOL(jiffies); |
672 | |||
673 | /* | ||
674 | * Add two timespec values and do a safety check for overflow. | ||
675 | * It's assumed that both values are valid (>= 0) | ||
676 | */ | ||
677 | struct timespec timespec_add_safe(const struct timespec lhs, | ||
678 | const struct timespec rhs) | ||
679 | { | ||
680 | struct timespec res; | ||
681 | |||
682 | set_normalized_timespec(&res, lhs.tv_sec + rhs.tv_sec, | ||
683 | lhs.tv_nsec + rhs.tv_nsec); | ||
684 | |||
685 | if (res.tv_sec < lhs.tv_sec || res.tv_sec < rhs.tv_sec) | ||
686 | res.tv_sec = TIME_T_MAX; | ||
687 | |||
688 | return res; | ||
689 | } | ||
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index 1ad46f3df6e7..9c114b726ab3 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c | |||
@@ -142,8 +142,7 @@ static enum hrtimer_restart ntp_leap_second(struct hrtimer *timer) | |||
142 | time_state = TIME_OOP; | 142 | time_state = TIME_OOP; |
143 | printk(KERN_NOTICE "Clock: " | 143 | printk(KERN_NOTICE "Clock: " |
144 | "inserting leap second 23:59:60 UTC\n"); | 144 | "inserting leap second 23:59:60 UTC\n"); |
145 | leap_timer.expires = ktime_add_ns(leap_timer.expires, | 145 | hrtimer_add_expires_ns(&leap_timer, NSEC_PER_SEC); |
146 | NSEC_PER_SEC); | ||
147 | res = HRTIMER_RESTART; | 146 | res = HRTIMER_RESTART; |
148 | break; | 147 | break; |
149 | case TIME_DEL: | 148 | case TIME_DEL: |
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index b711ffcb106c..a547be11cf97 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
@@ -300,7 +300,7 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
300 | goto out; | 300 | goto out; |
301 | } | 301 | } |
302 | 302 | ||
303 | ts->idle_tick = ts->sched_timer.expires; | 303 | ts->idle_tick = hrtimer_get_expires(&ts->sched_timer); |
304 | ts->tick_stopped = 1; | 304 | ts->tick_stopped = 1; |
305 | ts->idle_jiffies = last_jiffies; | 305 | ts->idle_jiffies = last_jiffies; |
306 | rcu_enter_nohz(); | 306 | rcu_enter_nohz(); |
@@ -431,21 +431,21 @@ void tick_nohz_restart_sched_tick(void) | |||
431 | ts->tick_stopped = 0; | 431 | ts->tick_stopped = 0; |
432 | ts->idle_exittime = now; | 432 | ts->idle_exittime = now; |
433 | hrtimer_cancel(&ts->sched_timer); | 433 | hrtimer_cancel(&ts->sched_timer); |
434 | ts->sched_timer.expires = ts->idle_tick; | 434 | hrtimer_set_expires(&ts->sched_timer, ts->idle_tick); |
435 | 435 | ||
436 | while (1) { | 436 | while (1) { |
437 | /* Forward the time to expire in the future */ | 437 | /* Forward the time to expire in the future */ |
438 | hrtimer_forward(&ts->sched_timer, now, tick_period); | 438 | hrtimer_forward(&ts->sched_timer, now, tick_period); |
439 | 439 | ||
440 | if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { | 440 | if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { |
441 | hrtimer_start(&ts->sched_timer, | 441 | hrtimer_start_expires(&ts->sched_timer, |
442 | ts->sched_timer.expires, | ||
443 | HRTIMER_MODE_ABS); | 442 | HRTIMER_MODE_ABS); |
444 | /* Check, if the timer was already in the past */ | 443 | /* Check, if the timer was already in the past */ |
445 | if (hrtimer_active(&ts->sched_timer)) | 444 | if (hrtimer_active(&ts->sched_timer)) |
446 | break; | 445 | break; |
447 | } else { | 446 | } else { |
448 | if (!tick_program_event(ts->sched_timer.expires, 0)) | 447 | if (!tick_program_event( |
448 | hrtimer_get_expires(&ts->sched_timer), 0)) | ||
449 | break; | 449 | break; |
450 | } | 450 | } |
451 | /* Update jiffies and reread time */ | 451 | /* Update jiffies and reread time */ |
@@ -458,7 +458,7 @@ void tick_nohz_restart_sched_tick(void) | |||
458 | static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now) | 458 | static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now) |
459 | { | 459 | { |
460 | hrtimer_forward(&ts->sched_timer, now, tick_period); | 460 | hrtimer_forward(&ts->sched_timer, now, tick_period); |
461 | return tick_program_event(ts->sched_timer.expires, 0); | 461 | return tick_program_event(hrtimer_get_expires(&ts->sched_timer), 0); |
462 | } | 462 | } |
463 | 463 | ||
464 | /* | 464 | /* |
@@ -541,7 +541,7 @@ static void tick_nohz_switch_to_nohz(void) | |||
541 | next = tick_init_jiffy_update(); | 541 | next = tick_init_jiffy_update(); |
542 | 542 | ||
543 | for (;;) { | 543 | for (;;) { |
544 | ts->sched_timer.expires = next; | 544 | hrtimer_set_expires(&ts->sched_timer, next); |
545 | if (!tick_program_event(next, 0)) | 545 | if (!tick_program_event(next, 0)) |
546 | break; | 546 | break; |
547 | next = ktime_add(next, tick_period); | 547 | next = ktime_add(next, tick_period); |
@@ -637,16 +637,15 @@ void tick_setup_sched_timer(void) | |||
637 | ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU; | 637 | ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU; |
638 | 638 | ||
639 | /* Get the next period (per cpu) */ | 639 | /* Get the next period (per cpu) */ |
640 | ts->sched_timer.expires = tick_init_jiffy_update(); | 640 | hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); |
641 | offset = ktime_to_ns(tick_period) >> 1; | 641 | offset = ktime_to_ns(tick_period) >> 1; |
642 | do_div(offset, num_possible_cpus()); | 642 | do_div(offset, num_possible_cpus()); |
643 | offset *= smp_processor_id(); | 643 | offset *= smp_processor_id(); |
644 | ts->sched_timer.expires = ktime_add_ns(ts->sched_timer.expires, offset); | 644 | hrtimer_add_expires_ns(&ts->sched_timer, offset); |
645 | 645 | ||
646 | for (;;) { | 646 | for (;;) { |
647 | hrtimer_forward(&ts->sched_timer, now, tick_period); | 647 | hrtimer_forward(&ts->sched_timer, now, tick_period); |
648 | hrtimer_start(&ts->sched_timer, ts->sched_timer.expires, | 648 | hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS); |
649 | HRTIMER_MODE_ABS); | ||
650 | /* Check, if the timer was already in the past */ | 649 | /* Check, if the timer was already in the past */ |
651 | if (hrtimer_active(&ts->sched_timer)) | 650 | if (hrtimer_active(&ts->sched_timer)) |
652 | break; | 651 | break; |
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index a40e20fd0001..122ee751d2d1 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c | |||
@@ -65,9 +65,11 @@ print_timer(struct seq_file *m, struct hrtimer *timer, int idx, u64 now) | |||
65 | SEQ_printf(m, ", %s/%d", tmp, timer->start_pid); | 65 | SEQ_printf(m, ", %s/%d", tmp, timer->start_pid); |
66 | #endif | 66 | #endif |
67 | SEQ_printf(m, "\n"); | 67 | SEQ_printf(m, "\n"); |
68 | SEQ_printf(m, " # expires at %Lu nsecs [in %Ld nsecs]\n", | 68 | SEQ_printf(m, " # expires at %Lu-%Lu nsecs [in %Ld to %Ld nsecs]\n", |
69 | (unsigned long long)ktime_to_ns(timer->expires), | 69 | (unsigned long long)ktime_to_ns(hrtimer_get_softexpires(timer)), |
70 | (long long)(ktime_to_ns(timer->expires) - now)); | 70 | (unsigned long long)ktime_to_ns(hrtimer_get_expires(timer)), |
71 | (long long)(ktime_to_ns(hrtimer_get_softexpires(timer)) - now), | ||
72 | (long long)(ktime_to_ns(hrtimer_get_expires(timer)) - now)); | ||
71 | } | 73 | } |
72 | 74 | ||
73 | static void | 75 | static void |