diff options
Diffstat (limited to 'kernel/hrtimer.c')
| -rw-r--r-- | kernel/hrtimer.c | 186 |
1 files changed, 157 insertions, 29 deletions
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index dea4c9124ac8..421be5fe5cc7 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
| @@ -43,6 +43,7 @@ | |||
| 43 | #include <linux/tick.h> | 43 | #include <linux/tick.h> |
| 44 | #include <linux/seq_file.h> | 44 | #include <linux/seq_file.h> |
| 45 | #include <linux/err.h> | 45 | #include <linux/err.h> |
| 46 | #include <linux/debugobjects.h> | ||
| 46 | 47 | ||
| 47 | #include <asm/uaccess.h> | 48 | #include <asm/uaccess.h> |
| 48 | 49 | ||
| @@ -153,15 +154,6 @@ static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base) | |||
| 153 | } | 154 | } |
| 154 | 155 | ||
| 155 | /* | 156 | /* |
| 156 | * Helper function to check, whether the timer is running the callback | ||
| 157 | * function | ||
| 158 | */ | ||
| 159 | static inline int hrtimer_callback_running(struct hrtimer *timer) | ||
| 160 | { | ||
| 161 | return timer->state & HRTIMER_STATE_CALLBACK; | ||
| 162 | } | ||
| 163 | |||
| 164 | /* | ||
| 165 | * Functions and macros which are different for UP/SMP systems are kept in a | 157 | * Functions and macros which are different for UP/SMP systems are kept in a |
| 166 | * single place | 158 | * single place |
| 167 | */ | 159 | */ |
| @@ -342,6 +334,115 @@ ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs) | |||
| 342 | return res; | 334 | return res; |
| 343 | } | 335 | } |
| 344 | 336 | ||
| 337 | #ifdef CONFIG_DEBUG_OBJECTS_TIMERS | ||
| 338 | |||
| 339 | static struct debug_obj_descr hrtimer_debug_descr; | ||
| 340 | |||
| 341 | /* | ||
| 342 | * fixup_init is called when: | ||
| 343 | * - an active object is initialized | ||
| 344 | */ | ||
| 345 | static int hrtimer_fixup_init(void *addr, enum debug_obj_state state) | ||
| 346 | { | ||
| 347 | struct hrtimer *timer = addr; | ||
| 348 | |||
| 349 | switch (state) { | ||
| 350 | case ODEBUG_STATE_ACTIVE: | ||
| 351 | hrtimer_cancel(timer); | ||
| 352 | debug_object_init(timer, &hrtimer_debug_descr); | ||
| 353 | return 1; | ||
| 354 | default: | ||
| 355 | return 0; | ||
| 356 | } | ||
| 357 | } | ||
| 358 | |||
| 359 | /* | ||
| 360 | * fixup_activate is called when: | ||
| 361 | * - an active object is activated | ||
| 362 | * - an unknown object is activated (might be a statically initialized object) | ||
| 363 | */ | ||
| 364 | static int hrtimer_fixup_activate(void *addr, enum debug_obj_state state) | ||
| 365 | { | ||
| 366 | switch (state) { | ||
| 367 | |||
| 368 | case ODEBUG_STATE_NOTAVAILABLE: | ||
| 369 | WARN_ON_ONCE(1); | ||
| 370 | return 0; | ||
| 371 | |||
| 372 | case ODEBUG_STATE_ACTIVE: | ||
| 373 | WARN_ON(1); | ||
| 374 | |||
| 375 | default: | ||
| 376 | return 0; | ||
| 377 | } | ||
| 378 | } | ||
| 379 | |||
| 380 | /* | ||
| 381 | * fixup_free is called when: | ||
| 382 | * - an active object is freed | ||
| 383 | */ | ||
| 384 | static int hrtimer_fixup_free(void *addr, enum debug_obj_state state) | ||
| 385 | { | ||
| 386 | struct hrtimer *timer = addr; | ||
| 387 | |||
| 388 | switch (state) { | ||
| 389 | case ODEBUG_STATE_ACTIVE: | ||
| 390 | hrtimer_cancel(timer); | ||
| 391 | debug_object_free(timer, &hrtimer_debug_descr); | ||
| 392 | return 1; | ||
| 393 | default: | ||
| 394 | return 0; | ||
| 395 | } | ||
| 396 | } | ||
| 397 | |||
| 398 | static struct debug_obj_descr hrtimer_debug_descr = { | ||
| 399 | .name = "hrtimer", | ||
| 400 | .fixup_init = hrtimer_fixup_init, | ||
| 401 | .fixup_activate = hrtimer_fixup_activate, | ||
| 402 | .fixup_free = hrtimer_fixup_free, | ||
| 403 | }; | ||
| 404 | |||
| 405 | static inline void debug_hrtimer_init(struct hrtimer *timer) | ||
| 406 | { | ||
| 407 | debug_object_init(timer, &hrtimer_debug_descr); | ||
| 408 | } | ||
| 409 | |||
| 410 | static inline void debug_hrtimer_activate(struct hrtimer *timer) | ||
| 411 | { | ||
| 412 | debug_object_activate(timer, &hrtimer_debug_descr); | ||
| 413 | } | ||
| 414 | |||
| 415 | static inline void debug_hrtimer_deactivate(struct hrtimer *timer) | ||
| 416 | { | ||
| 417 | debug_object_deactivate(timer, &hrtimer_debug_descr); | ||
| 418 | } | ||
| 419 | |||
| 420 | static inline void debug_hrtimer_free(struct hrtimer *timer) | ||
| 421 | { | ||
| 422 | debug_object_free(timer, &hrtimer_debug_descr); | ||
| 423 | } | ||
| 424 | |||
| 425 | static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, | ||
| 426 | enum hrtimer_mode mode); | ||
| 427 | |||
| 428 | void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id, | ||
| 429 | enum hrtimer_mode mode) | ||
| 430 | { | ||
| 431 | debug_object_init_on_stack(timer, &hrtimer_debug_descr); | ||
| 432 | __hrtimer_init(timer, clock_id, mode); | ||
| 433 | } | ||
| 434 | |||
| 435 | void destroy_hrtimer_on_stack(struct hrtimer *timer) | ||
| 436 | { | ||
| 437 | debug_object_free(timer, &hrtimer_debug_descr); | ||
| 438 | } | ||
| 439 | |||
| 440 | #else | ||
| 441 | static inline void debug_hrtimer_init(struct hrtimer *timer) { } | ||
| 442 | static inline void debug_hrtimer_activate(struct hrtimer *timer) { } | ||
| 443 | static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } | ||
| 444 | #endif | ||
| 445 | |||
| 345 | /* | 446 | /* |
| 346 | * Check, whether the timer is on the callback pending list | 447 | * Check, whether the timer is on the callback pending list |
| 347 | */ | 448 | */ |
| @@ -567,6 +668,7 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | |||
| 567 | /* Timer is expired, act upon the callback mode */ | 668 | /* Timer is expired, act upon the callback mode */ |
| 568 | switch(timer->cb_mode) { | 669 | switch(timer->cb_mode) { |
| 569 | case HRTIMER_CB_IRQSAFE_NO_RESTART: | 670 | case HRTIMER_CB_IRQSAFE_NO_RESTART: |
| 671 | debug_hrtimer_deactivate(timer); | ||
| 570 | /* | 672 | /* |
| 571 | * We can call the callback from here. No restart | 673 | * We can call the callback from here. No restart |
| 572 | * happens, so no danger of recursion | 674 | * happens, so no danger of recursion |
| @@ -581,6 +683,7 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | |||
| 581 | * the tick timer in the softirq ! The calling site | 683 | * the tick timer in the softirq ! The calling site |
| 582 | * takes care of this. | 684 | * takes care of this. |
| 583 | */ | 685 | */ |
| 686 | debug_hrtimer_deactivate(timer); | ||
| 584 | return 1; | 687 | return 1; |
| 585 | case HRTIMER_CB_IRQSAFE: | 688 | case HRTIMER_CB_IRQSAFE: |
| 586 | case HRTIMER_CB_SOFTIRQ: | 689 | case HRTIMER_CB_SOFTIRQ: |
| @@ -735,6 +838,8 @@ static void enqueue_hrtimer(struct hrtimer *timer, | |||
| 735 | struct hrtimer *entry; | 838 | struct hrtimer *entry; |
| 736 | int leftmost = 1; | 839 | int leftmost = 1; |
| 737 | 840 | ||
| 841 | debug_hrtimer_activate(timer); | ||
| 842 | |||
| 738 | /* | 843 | /* |
| 739 | * Find the right place in the rbtree: | 844 | * Find the right place in the rbtree: |
| 740 | */ | 845 | */ |
| @@ -831,6 +936,7 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base) | |||
| 831 | * reprogramming happens in the interrupt handler. This is a | 936 | * reprogramming happens in the interrupt handler. This is a |
| 832 | * rare case and less expensive than a smp call. | 937 | * rare case and less expensive than a smp call. |
| 833 | */ | 938 | */ |
| 939 | debug_hrtimer_deactivate(timer); | ||
| 834 | timer_stats_hrtimer_clear_start_info(timer); | 940 | timer_stats_hrtimer_clear_start_info(timer); |
| 835 | reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases); | 941 | reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases); |
| 836 | __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, | 942 | __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, |
| @@ -878,6 +984,7 @@ hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) | |||
| 878 | tim = ktime_add_safe(tim, base->resolution); | 984 | tim = ktime_add_safe(tim, base->resolution); |
| 879 | #endif | 985 | #endif |
| 880 | } | 986 | } |
| 987 | |||
| 881 | timer->expires = tim; | 988 | timer->expires = tim; |
| 882 | 989 | ||
| 883 | timer_stats_hrtimer_set_start_info(timer); | 990 | timer_stats_hrtimer_set_start_info(timer); |
| @@ -1011,14 +1118,8 @@ ktime_t hrtimer_get_next_event(void) | |||
| 1011 | } | 1118 | } |
| 1012 | #endif | 1119 | #endif |
| 1013 | 1120 | ||
| 1014 | /** | 1121 | static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, |
| 1015 | * hrtimer_init - initialize a timer to the given clock | 1122 | enum hrtimer_mode mode) |
| 1016 | * @timer: the timer to be initialized | ||
| 1017 | * @clock_id: the clock to be used | ||
| 1018 | * @mode: timer mode abs/rel | ||
| 1019 | */ | ||
| 1020 | void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, | ||
| 1021 | enum hrtimer_mode mode) | ||
| 1022 | { | 1123 | { |
| 1023 | struct hrtimer_cpu_base *cpu_base; | 1124 | struct hrtimer_cpu_base *cpu_base; |
| 1024 | 1125 | ||
| @@ -1039,6 +1140,19 @@ void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, | |||
| 1039 | memset(timer->start_comm, 0, TASK_COMM_LEN); | 1140 | memset(timer->start_comm, 0, TASK_COMM_LEN); |
| 1040 | #endif | 1141 | #endif |
| 1041 | } | 1142 | } |
| 1143 | |||
| 1144 | /** | ||
| 1145 | * hrtimer_init - initialize a timer to the given clock | ||
| 1146 | * @timer: the timer to be initialized | ||
| 1147 | * @clock_id: the clock to be used | ||
| 1148 | * @mode: timer mode abs/rel | ||
| 1149 | */ | ||
| 1150 | void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, | ||
| 1151 | enum hrtimer_mode mode) | ||
| 1152 | { | ||
| 1153 | debug_hrtimer_init(timer); | ||
| 1154 | __hrtimer_init(timer, clock_id, mode); | ||
| 1155 | } | ||
| 1042 | EXPORT_SYMBOL_GPL(hrtimer_init); | 1156 | EXPORT_SYMBOL_GPL(hrtimer_init); |
| 1043 | 1157 | ||
| 1044 | /** | 1158 | /** |
| @@ -1072,6 +1186,7 @@ static void run_hrtimer_pending(struct hrtimer_cpu_base *cpu_base) | |||
| 1072 | timer = list_entry(cpu_base->cb_pending.next, | 1186 | timer = list_entry(cpu_base->cb_pending.next, |
| 1073 | struct hrtimer, cb_entry); | 1187 | struct hrtimer, cb_entry); |
| 1074 | 1188 | ||
| 1189 | debug_hrtimer_deactivate(timer); | ||
| 1075 | timer_stats_account_hrtimer(timer); | 1190 | timer_stats_account_hrtimer(timer); |
| 1076 | 1191 | ||
| 1077 | fn = timer->function; | 1192 | fn = timer->function; |
| @@ -1120,6 +1235,7 @@ static void __run_hrtimer(struct hrtimer *timer) | |||
| 1120 | enum hrtimer_restart (*fn)(struct hrtimer *); | 1235 | enum hrtimer_restart (*fn)(struct hrtimer *); |
| 1121 | int restart; | 1236 | int restart; |
| 1122 | 1237 | ||
| 1238 | debug_hrtimer_deactivate(timer); | ||
| 1123 | __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); | 1239 | __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); |
| 1124 | timer_stats_account_hrtimer(timer); | 1240 | timer_stats_account_hrtimer(timer); |
| 1125 | 1241 | ||
| @@ -1378,22 +1494,27 @@ long __sched hrtimer_nanosleep_restart(struct restart_block *restart) | |||
| 1378 | { | 1494 | { |
| 1379 | struct hrtimer_sleeper t; | 1495 | struct hrtimer_sleeper t; |
| 1380 | struct timespec __user *rmtp; | 1496 | struct timespec __user *rmtp; |
| 1497 | int ret = 0; | ||
| 1381 | 1498 | ||
| 1382 | hrtimer_init(&t.timer, restart->nanosleep.index, HRTIMER_MODE_ABS); | 1499 | hrtimer_init_on_stack(&t.timer, restart->nanosleep.index, |
| 1500 | HRTIMER_MODE_ABS); | ||
| 1383 | t.timer.expires.tv64 = restart->nanosleep.expires; | 1501 | t.timer.expires.tv64 = restart->nanosleep.expires; |
| 1384 | 1502 | ||
| 1385 | if (do_nanosleep(&t, HRTIMER_MODE_ABS)) | 1503 | if (do_nanosleep(&t, HRTIMER_MODE_ABS)) |
| 1386 | return 0; | 1504 | goto out; |
| 1387 | 1505 | ||
| 1388 | rmtp = restart->nanosleep.rmtp; | 1506 | rmtp = restart->nanosleep.rmtp; |
| 1389 | if (rmtp) { | 1507 | if (rmtp) { |
| 1390 | int ret = update_rmtp(&t.timer, rmtp); | 1508 | ret = update_rmtp(&t.timer, rmtp); |
| 1391 | if (ret <= 0) | 1509 | if (ret <= 0) |
| 1392 | return ret; | 1510 | goto out; |
| 1393 | } | 1511 | } |
| 1394 | 1512 | ||
| 1395 | /* The other values in restart are already filled in */ | 1513 | /* The other values in restart are already filled in */ |
| 1396 | return -ERESTART_RESTARTBLOCK; | 1514 | ret = -ERESTART_RESTARTBLOCK; |
| 1515 | out: | ||
| 1516 | destroy_hrtimer_on_stack(&t.timer); | ||
| 1517 | return ret; | ||
| 1397 | } | 1518 | } |
| 1398 | 1519 | ||
| 1399 | long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, | 1520 | long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, |
| @@ -1401,20 +1522,23 @@ long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, | |||
| 1401 | { | 1522 | { |
| 1402 | struct restart_block *restart; | 1523 | struct restart_block *restart; |
| 1403 | struct hrtimer_sleeper t; | 1524 | struct hrtimer_sleeper t; |
| 1525 | int ret = 0; | ||
| 1404 | 1526 | ||
| 1405 | hrtimer_init(&t.timer, clockid, mode); | 1527 | hrtimer_init_on_stack(&t.timer, clockid, mode); |
| 1406 | t.timer.expires = timespec_to_ktime(*rqtp); | 1528 | t.timer.expires = timespec_to_ktime(*rqtp); |
| 1407 | if (do_nanosleep(&t, mode)) | 1529 | if (do_nanosleep(&t, mode)) |
| 1408 | return 0; | 1530 | goto out; |
| 1409 | 1531 | ||
| 1410 | /* Absolute timers do not update the rmtp value and restart: */ | 1532 | /* Absolute timers do not update the rmtp value and restart: */ |
| 1411 | if (mode == HRTIMER_MODE_ABS) | 1533 | if (mode == HRTIMER_MODE_ABS) { |
| 1412 | return -ERESTARTNOHAND; | 1534 | ret = -ERESTARTNOHAND; |
| 1535 | goto out; | ||
| 1536 | } | ||
| 1413 | 1537 | ||
| 1414 | if (rmtp) { | 1538 | if (rmtp) { |
| 1415 | int ret = update_rmtp(&t.timer, rmtp); | 1539 | ret = update_rmtp(&t.timer, rmtp); |
| 1416 | if (ret <= 0) | 1540 | if (ret <= 0) |
| 1417 | return ret; | 1541 | goto out; |
| 1418 | } | 1542 | } |
| 1419 | 1543 | ||
| 1420 | restart = ¤t_thread_info()->restart_block; | 1544 | restart = ¤t_thread_info()->restart_block; |
| @@ -1423,7 +1547,10 @@ long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, | |||
| 1423 | restart->nanosleep.rmtp = rmtp; | 1547 | restart->nanosleep.rmtp = rmtp; |
| 1424 | restart->nanosleep.expires = t.timer.expires.tv64; | 1548 | restart->nanosleep.expires = t.timer.expires.tv64; |
| 1425 | 1549 | ||
| 1426 | return -ERESTART_RESTARTBLOCK; | 1550 | ret = -ERESTART_RESTARTBLOCK; |
| 1551 | out: | ||
| 1552 | destroy_hrtimer_on_stack(&t.timer); | ||
| 1553 | return ret; | ||
| 1427 | } | 1554 | } |
| 1428 | 1555 | ||
| 1429 | asmlinkage long | 1556 | asmlinkage long |
| @@ -1468,6 +1595,7 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, | |||
| 1468 | while ((node = rb_first(&old_base->active))) { | 1595 | while ((node = rb_first(&old_base->active))) { |
| 1469 | timer = rb_entry(node, struct hrtimer, node); | 1596 | timer = rb_entry(node, struct hrtimer, node); |
| 1470 | BUG_ON(hrtimer_callback_running(timer)); | 1597 | BUG_ON(hrtimer_callback_running(timer)); |
| 1598 | debug_hrtimer_deactivate(timer); | ||
| 1471 | __remove_hrtimer(timer, old_base, HRTIMER_STATE_INACTIVE, 0); | 1599 | __remove_hrtimer(timer, old_base, HRTIMER_STATE_INACTIVE, 0); |
| 1472 | timer->base = new_base; | 1600 | timer->base = new_base; |
| 1473 | /* | 1601 | /* |
