aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/hrtimer.h15
-rw-r--r--kernel/futex.c17
-rw-r--r--kernel/hrtimer.c177
3 files changed, 186 insertions, 23 deletions
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 56f3236da829..31a4d653389f 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -266,6 +266,21 @@ extern ktime_t ktime_get_real(void);
266extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock, 266extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock,
267 enum hrtimer_mode mode); 267 enum hrtimer_mode mode);
268 268
269#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
270extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock,
271 enum hrtimer_mode mode);
272
273extern void destroy_hrtimer_on_stack(struct hrtimer *timer);
274#else
275static inline void hrtimer_init_on_stack(struct hrtimer *timer,
276 clockid_t which_clock,
277 enum hrtimer_mode mode)
278{
279 hrtimer_init(timer, which_clock, mode);
280}
281static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
282#endif
283
269/* Basic timer operations: */ 284/* Basic timer operations: */
270extern int hrtimer_start(struct hrtimer *timer, ktime_t tim, 285extern int hrtimer_start(struct hrtimer *timer, ktime_t tim,
271 const enum hrtimer_mode mode); 286 const enum hrtimer_mode mode);
diff --git a/kernel/futex.c b/kernel/futex.c
index e43945e995f5..98092c9817f4 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1266,11 +1266,13 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
1266 if (!abs_time) 1266 if (!abs_time)
1267 schedule(); 1267 schedule();
1268 else { 1268 else {
1269 hrtimer_init(&t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 1269 hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC,
1270 HRTIMER_MODE_ABS);
1270 hrtimer_init_sleeper(&t, current); 1271 hrtimer_init_sleeper(&t, current);
1271 t.timer.expires = *abs_time; 1272 t.timer.expires = *abs_time;
1272 1273
1273 hrtimer_start(&t.timer, t.timer.expires, HRTIMER_MODE_ABS); 1274 hrtimer_start(&t.timer, t.timer.expires,
1275 HRTIMER_MODE_ABS);
1274 if (!hrtimer_active(&t.timer)) 1276 if (!hrtimer_active(&t.timer))
1275 t.task = NULL; 1277 t.task = NULL;
1276 1278
@@ -1286,6 +1288,8 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
1286 1288
1287 /* Flag if a timeout occured */ 1289 /* Flag if a timeout occured */
1288 rem = (t.task == NULL); 1290 rem = (t.task == NULL);
1291
1292 destroy_hrtimer_on_stack(&t.timer);
1289 } 1293 }
1290 } 1294 }
1291 __set_current_state(TASK_RUNNING); 1295 __set_current_state(TASK_RUNNING);
@@ -1367,7 +1371,8 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
1367 1371
1368 if (time) { 1372 if (time) {
1369 to = &timeout; 1373 to = &timeout;
1370 hrtimer_init(&to->timer, CLOCK_REALTIME, HRTIMER_MODE_ABS); 1374 hrtimer_init_on_stack(&to->timer, CLOCK_REALTIME,
1375 HRTIMER_MODE_ABS);
1371 hrtimer_init_sleeper(to, current); 1376 hrtimer_init_sleeper(to, current);
1372 to->timer.expires = *time; 1377 to->timer.expires = *time;
1373 } 1378 }
@@ -1581,6 +1586,8 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
1581 unqueue_me_pi(&q); 1586 unqueue_me_pi(&q);
1582 futex_unlock_mm(fshared); 1587 futex_unlock_mm(fshared);
1583 1588
1589 if (to)
1590 destroy_hrtimer_on_stack(&to->timer);
1584 return ret != -EINTR ? ret : -ERESTARTNOINTR; 1591 return ret != -EINTR ? ret : -ERESTARTNOINTR;
1585 1592
1586 out_unlock_release_sem: 1593 out_unlock_release_sem:
@@ -1588,6 +1595,8 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
1588 1595
1589 out_release_sem: 1596 out_release_sem:
1590 futex_unlock_mm(fshared); 1597 futex_unlock_mm(fshared);
1598 if (to)
1599 destroy_hrtimer_on_stack(&to->timer);
1591 return ret; 1600 return ret;
1592 1601
1593 uaddr_faulted: 1602 uaddr_faulted:
@@ -1615,6 +1624,8 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
1615 if (!ret && (uval != -EFAULT)) 1624 if (!ret && (uval != -EFAULT))
1616 goto retry; 1625 goto retry;
1617 1626
1627 if (to)
1628 destroy_hrtimer_on_stack(&to->timer);
1618 return ret; 1629 return ret;
1619} 1630}
1620 1631
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index dea4c9124ac8..9af1d6a8095e 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -43,6 +43,7 @@
43#include <linux/tick.h> 43#include <linux/tick.h>
44#include <linux/seq_file.h> 44#include <linux/seq_file.h>
45#include <linux/err.h> 45#include <linux/err.h>
46#include <linux/debugobjects.h>
46 47
47#include <asm/uaccess.h> 48#include <asm/uaccess.h>
48 49
@@ -342,6 +343,115 @@ ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
342 return res; 343 return res;
343} 344}
344 345
346#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
347
348static struct debug_obj_descr hrtimer_debug_descr;
349
350/*
351 * fixup_init is called when:
352 * - an active object is initialized
353 */
354static int hrtimer_fixup_init(void *addr, enum debug_obj_state state)
355{
356 struct hrtimer *timer = addr;
357
358 switch (state) {
359 case ODEBUG_STATE_ACTIVE:
360 hrtimer_cancel(timer);
361 debug_object_init(timer, &hrtimer_debug_descr);
362 return 1;
363 default:
364 return 0;
365 }
366}
367
368/*
369 * fixup_activate is called when:
370 * - an active object is activated
371 * - an unknown object is activated (might be a statically initialized object)
372 */
373static int hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
374{
375 switch (state) {
376
377 case ODEBUG_STATE_NOTAVAILABLE:
378 WARN_ON_ONCE(1);
379 return 0;
380
381 case ODEBUG_STATE_ACTIVE:
382 WARN_ON(1);
383
384 default:
385 return 0;
386 }
387}
388
389/*
390 * fixup_free is called when:
391 * - an active object is freed
392 */
393static int hrtimer_fixup_free(void *addr, enum debug_obj_state state)
394{
395 struct hrtimer *timer = addr;
396
397 switch (state) {
398 case ODEBUG_STATE_ACTIVE:
399 hrtimer_cancel(timer);
400 debug_object_free(timer, &hrtimer_debug_descr);
401 return 1;
402 default:
403 return 0;
404 }
405}
406
407static struct debug_obj_descr hrtimer_debug_descr = {
408 .name = "hrtimer",
409 .fixup_init = hrtimer_fixup_init,
410 .fixup_activate = hrtimer_fixup_activate,
411 .fixup_free = hrtimer_fixup_free,
412};
413
414static inline void debug_hrtimer_init(struct hrtimer *timer)
415{
416 debug_object_init(timer, &hrtimer_debug_descr);
417}
418
419static inline void debug_hrtimer_activate(struct hrtimer *timer)
420{
421 debug_object_activate(timer, &hrtimer_debug_descr);
422}
423
424static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
425{
426 debug_object_deactivate(timer, &hrtimer_debug_descr);
427}
428
429static inline void debug_hrtimer_free(struct hrtimer *timer)
430{
431 debug_object_free(timer, &hrtimer_debug_descr);
432}
433
434static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
435 enum hrtimer_mode mode);
436
437void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
438 enum hrtimer_mode mode)
439{
440 debug_object_init_on_stack(timer, &hrtimer_debug_descr);
441 __hrtimer_init(timer, clock_id, mode);
442}
443
444void destroy_hrtimer_on_stack(struct hrtimer *timer)
445{
446 debug_object_free(timer, &hrtimer_debug_descr);
447}
448
449#else
450static inline void debug_hrtimer_init(struct hrtimer *timer) { }
451static inline void debug_hrtimer_activate(struct hrtimer *timer) { }
452static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
453#endif
454
345/* 455/*
346 * Check, whether the timer is on the callback pending list 456 * Check, whether the timer is on the callback pending list
347 */ 457 */
@@ -567,6 +677,7 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
567 /* Timer is expired, act upon the callback mode */ 677 /* Timer is expired, act upon the callback mode */
568 switch(timer->cb_mode) { 678 switch(timer->cb_mode) {
569 case HRTIMER_CB_IRQSAFE_NO_RESTART: 679 case HRTIMER_CB_IRQSAFE_NO_RESTART:
680 debug_hrtimer_deactivate(timer);
570 /* 681 /*
571 * We can call the callback from here. No restart 682 * We can call the callback from here. No restart
572 * happens, so no danger of recursion 683 * happens, so no danger of recursion
@@ -581,6 +692,7 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
581 * the tick timer in the softirq ! The calling site 692 * the tick timer in the softirq ! The calling site
582 * takes care of this. 693 * takes care of this.
583 */ 694 */
695 debug_hrtimer_deactivate(timer);
584 return 1; 696 return 1;
585 case HRTIMER_CB_IRQSAFE: 697 case HRTIMER_CB_IRQSAFE:
586 case HRTIMER_CB_SOFTIRQ: 698 case HRTIMER_CB_SOFTIRQ:
@@ -735,6 +847,8 @@ static void enqueue_hrtimer(struct hrtimer *timer,
735 struct hrtimer *entry; 847 struct hrtimer *entry;
736 int leftmost = 1; 848 int leftmost = 1;
737 849
850 debug_hrtimer_activate(timer);
851
738 /* 852 /*
739 * Find the right place in the rbtree: 853 * Find the right place in the rbtree:
740 */ 854 */
@@ -831,6 +945,7 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base)
831 * reprogramming happens in the interrupt handler. This is a 945 * reprogramming happens in the interrupt handler. This is a
832 * rare case and less expensive than a smp call. 946 * rare case and less expensive than a smp call.
833 */ 947 */
948 debug_hrtimer_deactivate(timer);
834 timer_stats_hrtimer_clear_start_info(timer); 949 timer_stats_hrtimer_clear_start_info(timer);
835 reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases); 950 reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases);
836 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 951 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE,
@@ -878,6 +993,7 @@ hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)
878 tim = ktime_add_safe(tim, base->resolution); 993 tim = ktime_add_safe(tim, base->resolution);
879#endif 994#endif
880 } 995 }
996
881 timer->expires = tim; 997 timer->expires = tim;
882 998
883 timer_stats_hrtimer_set_start_info(timer); 999 timer_stats_hrtimer_set_start_info(timer);
@@ -1011,14 +1127,8 @@ ktime_t hrtimer_get_next_event(void)
1011} 1127}
1012#endif 1128#endif
1013 1129
1014/** 1130static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1015 * hrtimer_init - initialize a timer to the given clock 1131 enum hrtimer_mode mode)
1016 * @timer: the timer to be initialized
1017 * @clock_id: the clock to be used
1018 * @mode: timer mode abs/rel
1019 */
1020void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1021 enum hrtimer_mode mode)
1022{ 1132{
1023 struct hrtimer_cpu_base *cpu_base; 1133 struct hrtimer_cpu_base *cpu_base;
1024 1134
@@ -1039,6 +1149,19 @@ void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1039 memset(timer->start_comm, 0, TASK_COMM_LEN); 1149 memset(timer->start_comm, 0, TASK_COMM_LEN);
1040#endif 1150#endif
1041} 1151}
1152
1153/**
1154 * hrtimer_init - initialize a timer to the given clock
1155 * @timer: the timer to be initialized
1156 * @clock_id: the clock to be used
1157 * @mode: timer mode abs/rel
1158 */
1159void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1160 enum hrtimer_mode mode)
1161{
1162 debug_hrtimer_init(timer);
1163 __hrtimer_init(timer, clock_id, mode);
1164}
1042EXPORT_SYMBOL_GPL(hrtimer_init); 1165EXPORT_SYMBOL_GPL(hrtimer_init);
1043 1166
1044/** 1167/**
@@ -1072,6 +1195,7 @@ static void run_hrtimer_pending(struct hrtimer_cpu_base *cpu_base)
1072 timer = list_entry(cpu_base->cb_pending.next, 1195 timer = list_entry(cpu_base->cb_pending.next,
1073 struct hrtimer, cb_entry); 1196 struct hrtimer, cb_entry);
1074 1197
1198 debug_hrtimer_deactivate(timer);
1075 timer_stats_account_hrtimer(timer); 1199 timer_stats_account_hrtimer(timer);
1076 1200
1077 fn = timer->function; 1201 fn = timer->function;
@@ -1120,6 +1244,7 @@ static void __run_hrtimer(struct hrtimer *timer)
1120 enum hrtimer_restart (*fn)(struct hrtimer *); 1244 enum hrtimer_restart (*fn)(struct hrtimer *);
1121 int restart; 1245 int restart;
1122 1246
1247 debug_hrtimer_deactivate(timer);
1123 __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); 1248 __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);
1124 timer_stats_account_hrtimer(timer); 1249 timer_stats_account_hrtimer(timer);
1125 1250
@@ -1378,22 +1503,27 @@ long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
1378{ 1503{
1379 struct hrtimer_sleeper t; 1504 struct hrtimer_sleeper t;
1380 struct timespec __user *rmtp; 1505 struct timespec __user *rmtp;
1506 int ret = 0;
1381 1507
1382 hrtimer_init(&t.timer, restart->nanosleep.index, HRTIMER_MODE_ABS); 1508 hrtimer_init_on_stack(&t.timer, restart->nanosleep.index,
1509 HRTIMER_MODE_ABS);
1383 t.timer.expires.tv64 = restart->nanosleep.expires; 1510 t.timer.expires.tv64 = restart->nanosleep.expires;
1384 1511
1385 if (do_nanosleep(&t, HRTIMER_MODE_ABS)) 1512 if (do_nanosleep(&t, HRTIMER_MODE_ABS))
1386 return 0; 1513 goto out;
1387 1514
1388 rmtp = restart->nanosleep.rmtp; 1515 rmtp = restart->nanosleep.rmtp;
1389 if (rmtp) { 1516 if (rmtp) {
1390 int ret = update_rmtp(&t.timer, rmtp); 1517 ret = update_rmtp(&t.timer, rmtp);
1391 if (ret <= 0) 1518 if (ret <= 0)
1392 return ret; 1519 goto out;
1393 } 1520 }
1394 1521
1395 /* The other values in restart are already filled in */ 1522 /* The other values in restart are already filled in */
1396 return -ERESTART_RESTARTBLOCK; 1523 ret = -ERESTART_RESTARTBLOCK;
1524out:
1525 destroy_hrtimer_on_stack(&t.timer);
1526 return ret;
1397} 1527}
1398 1528
1399long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, 1529long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
@@ -1401,20 +1531,23 @@ long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
1401{ 1531{
1402 struct restart_block *restart; 1532 struct restart_block *restart;
1403 struct hrtimer_sleeper t; 1533 struct hrtimer_sleeper t;
1534 int ret = 0;
1404 1535
1405 hrtimer_init(&t.timer, clockid, mode); 1536 hrtimer_init_on_stack(&t.timer, clockid, mode);
1406 t.timer.expires = timespec_to_ktime(*rqtp); 1537 t.timer.expires = timespec_to_ktime(*rqtp);
1407 if (do_nanosleep(&t, mode)) 1538 if (do_nanosleep(&t, mode))
1408 return 0; 1539 goto out;
1409 1540
1410 /* Absolute timers do not update the rmtp value and restart: */ 1541 /* Absolute timers do not update the rmtp value and restart: */
1411 if (mode == HRTIMER_MODE_ABS) 1542 if (mode == HRTIMER_MODE_ABS) {
1412 return -ERESTARTNOHAND; 1543 ret = -ERESTARTNOHAND;
1544 goto out;
1545 }
1413 1546
1414 if (rmtp) { 1547 if (rmtp) {
1415 int ret = update_rmtp(&t.timer, rmtp); 1548 ret = update_rmtp(&t.timer, rmtp);
1416 if (ret <= 0) 1549 if (ret <= 0)
1417 return ret; 1550 goto out;
1418 } 1551 }
1419 1552
1420 restart = &current_thread_info()->restart_block; 1553 restart = &current_thread_info()->restart_block;
@@ -1423,7 +1556,10 @@ long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp,
1423 restart->nanosleep.rmtp = rmtp; 1556 restart->nanosleep.rmtp = rmtp;
1424 restart->nanosleep.expires = t.timer.expires.tv64; 1557 restart->nanosleep.expires = t.timer.expires.tv64;
1425 1558
1426 return -ERESTART_RESTARTBLOCK; 1559 ret = -ERESTART_RESTARTBLOCK;
1560out:
1561 destroy_hrtimer_on_stack(&t.timer);
1562 return ret;
1427} 1563}
1428 1564
1429asmlinkage long 1565asmlinkage long
@@ -1468,6 +1604,7 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
1468 while ((node = rb_first(&old_base->active))) { 1604 while ((node = rb_first(&old_base->active))) {
1469 timer = rb_entry(node, struct hrtimer, node); 1605 timer = rb_entry(node, struct hrtimer, node);
1470 BUG_ON(hrtimer_callback_running(timer)); 1606 BUG_ON(hrtimer_callback_running(timer));
1607 debug_hrtimer_deactivate(timer);
1471 __remove_hrtimer(timer, old_base, HRTIMER_STATE_INACTIVE, 0); 1608 __remove_hrtimer(timer, old_base, HRTIMER_STATE_INACTIVE, 0);
1472 timer->base = new_base; 1609 timer->base = new_base;
1473 /* 1610 /*