diff options
Diffstat (limited to 'kernel/timer.c')
-rw-r--r-- | kernel/timer.c | 110 |
1 files changed, 61 insertions, 49 deletions
diff --git a/kernel/timer.c b/kernel/timer.c index 6ec7e7e0db4..a61c09374eb 100644 --- a/kernel/timer.c +++ b/kernel/timer.c | |||
@@ -77,6 +77,7 @@ struct tvec_base { | |||
77 | struct timer_list *running_timer; | 77 | struct timer_list *running_timer; |
78 | unsigned long timer_jiffies; | 78 | unsigned long timer_jiffies; |
79 | unsigned long next_timer; | 79 | unsigned long next_timer; |
80 | unsigned long active_timers; | ||
80 | struct tvec_root tv1; | 81 | struct tvec_root tv1; |
81 | struct tvec tv2; | 82 | struct tvec tv2; |
82 | struct tvec tv3; | 83 | struct tvec tv3; |
@@ -330,7 +331,8 @@ void set_timer_slack(struct timer_list *timer, int slack_hz) | |||
330 | } | 331 | } |
331 | EXPORT_SYMBOL_GPL(set_timer_slack); | 332 | EXPORT_SYMBOL_GPL(set_timer_slack); |
332 | 333 | ||
333 | static void internal_add_timer(struct tvec_base *base, struct timer_list *timer) | 334 | static void |
335 | __internal_add_timer(struct tvec_base *base, struct timer_list *timer) | ||
334 | { | 336 | { |
335 | unsigned long expires = timer->expires; | 337 | unsigned long expires = timer->expires; |
336 | unsigned long idx = expires - base->timer_jiffies; | 338 | unsigned long idx = expires - base->timer_jiffies; |
@@ -372,6 +374,19 @@ static void internal_add_timer(struct tvec_base *base, struct timer_list *timer) | |||
372 | list_add_tail(&timer->entry, vec); | 374 | list_add_tail(&timer->entry, vec); |
373 | } | 375 | } |
374 | 376 | ||
377 | static void internal_add_timer(struct tvec_base *base, struct timer_list *timer) | ||
378 | { | ||
379 | __internal_add_timer(base, timer); | ||
380 | /* | ||
381 | * Update base->active_timers and base->next_timer | ||
382 | */ | ||
383 | if (!tbase_get_deferrable(timer->base)) { | ||
384 | if (time_before(timer->expires, base->next_timer)) | ||
385 | base->next_timer = timer->expires; | ||
386 | base->active_timers++; | ||
387 | } | ||
388 | } | ||
389 | |||
375 | #ifdef CONFIG_TIMER_STATS | 390 | #ifdef CONFIG_TIMER_STATS |
376 | void __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr) | 391 | void __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr) |
377 | { | 392 | { |
@@ -654,8 +669,7 @@ void init_timer_deferrable_key(struct timer_list *timer, | |||
654 | } | 669 | } |
655 | EXPORT_SYMBOL(init_timer_deferrable_key); | 670 | EXPORT_SYMBOL(init_timer_deferrable_key); |
656 | 671 | ||
657 | static inline void detach_timer(struct timer_list *timer, | 672 | static inline void detach_timer(struct timer_list *timer, bool clear_pending) |
658 | int clear_pending) | ||
659 | { | 673 | { |
660 | struct list_head *entry = &timer->entry; | 674 | struct list_head *entry = &timer->entry; |
661 | 675 | ||
@@ -667,6 +681,29 @@ static inline void detach_timer(struct timer_list *timer, | |||
667 | entry->prev = LIST_POISON2; | 681 | entry->prev = LIST_POISON2; |
668 | } | 682 | } |
669 | 683 | ||
684 | static inline void | ||
685 | detach_expired_timer(struct timer_list *timer, struct tvec_base *base) | ||
686 | { | ||
687 | detach_timer(timer, true); | ||
688 | if (!tbase_get_deferrable(timer->base)) | ||
689 | timer->base->active_timers--; | ||
690 | } | ||
691 | |||
692 | static int detach_if_pending(struct timer_list *timer, struct tvec_base *base, | ||
693 | bool clear_pending) | ||
694 | { | ||
695 | if (!timer_pending(timer)) | ||
696 | return 0; | ||
697 | |||
698 | detach_timer(timer, clear_pending); | ||
699 | if (!tbase_get_deferrable(timer->base)) { | ||
700 | timer->base->active_timers--; | ||
701 | if (timer->expires == base->next_timer) | ||
702 | base->next_timer = base->timer_jiffies; | ||
703 | } | ||
704 | return 1; | ||
705 | } | ||
706 | |||
670 | /* | 707 | /* |
671 | * We are using hashed locking: holding per_cpu(tvec_bases).lock | 708 | * We are using hashed locking: holding per_cpu(tvec_bases).lock |
672 | * means that all timers which are tied to this base via timer->base are | 709 | * means that all timers which are tied to this base via timer->base are |
@@ -712,16 +749,9 @@ __mod_timer(struct timer_list *timer, unsigned long expires, | |||
712 | 749 | ||
713 | base = lock_timer_base(timer, &flags); | 750 | base = lock_timer_base(timer, &flags); |
714 | 751 | ||
715 | if (timer_pending(timer)) { | 752 | ret = detach_if_pending(timer, base, false); |
716 | detach_timer(timer, 0); | 753 | if (!ret && pending_only) |
717 | if (timer->expires == base->next_timer && | 754 | goto out_unlock; |
718 | !tbase_get_deferrable(timer->base)) | ||
719 | base->next_timer = base->timer_jiffies; | ||
720 | ret = 1; | ||
721 | } else { | ||
722 | if (pending_only) | ||
723 | goto out_unlock; | ||
724 | } | ||
725 | 755 | ||
726 | debug_activate(timer, expires); | 756 | debug_activate(timer, expires); |
727 | 757 | ||
@@ -752,9 +782,6 @@ __mod_timer(struct timer_list *timer, unsigned long expires, | |||
752 | } | 782 | } |
753 | 783 | ||
754 | timer->expires = expires; | 784 | timer->expires = expires; |
755 | if (time_before(timer->expires, base->next_timer) && | ||
756 | !tbase_get_deferrable(timer->base)) | ||
757 | base->next_timer = timer->expires; | ||
758 | internal_add_timer(base, timer); | 785 | internal_add_timer(base, timer); |
759 | 786 | ||
760 | out_unlock: | 787 | out_unlock: |
@@ -920,9 +947,6 @@ void add_timer_on(struct timer_list *timer, int cpu) | |||
920 | spin_lock_irqsave(&base->lock, flags); | 947 | spin_lock_irqsave(&base->lock, flags); |
921 | timer_set_base(timer, base); | 948 | timer_set_base(timer, base); |
922 | debug_activate(timer, timer->expires); | 949 | debug_activate(timer, timer->expires); |
923 | if (time_before(timer->expires, base->next_timer) && | ||
924 | !tbase_get_deferrable(timer->base)) | ||
925 | base->next_timer = timer->expires; | ||
926 | internal_add_timer(base, timer); | 950 | internal_add_timer(base, timer); |
927 | /* | 951 | /* |
928 | * Check whether the other CPU is idle and needs to be | 952 | * Check whether the other CPU is idle and needs to be |
@@ -959,13 +983,7 @@ int del_timer(struct timer_list *timer) | |||
959 | timer_stats_timer_clear_start_info(timer); | 983 | timer_stats_timer_clear_start_info(timer); |
960 | if (timer_pending(timer)) { | 984 | if (timer_pending(timer)) { |
961 | base = lock_timer_base(timer, &flags); | 985 | base = lock_timer_base(timer, &flags); |
962 | if (timer_pending(timer)) { | 986 | ret = detach_if_pending(timer, base, true); |
963 | detach_timer(timer, 1); | ||
964 | if (timer->expires == base->next_timer && | ||
965 | !tbase_get_deferrable(timer->base)) | ||
966 | base->next_timer = base->timer_jiffies; | ||
967 | ret = 1; | ||
968 | } | ||
969 | spin_unlock_irqrestore(&base->lock, flags); | 987 | spin_unlock_irqrestore(&base->lock, flags); |
970 | } | 988 | } |
971 | 989 | ||
@@ -990,19 +1008,10 @@ int try_to_del_timer_sync(struct timer_list *timer) | |||
990 | 1008 | ||
991 | base = lock_timer_base(timer, &flags); | 1009 | base = lock_timer_base(timer, &flags); |
992 | 1010 | ||
993 | if (base->running_timer == timer) | 1011 | if (base->running_timer != timer) { |
994 | goto out; | 1012 | timer_stats_timer_clear_start_info(timer); |
995 | 1013 | ret = detach_if_pending(timer, base, true); | |
996 | timer_stats_timer_clear_start_info(timer); | ||
997 | ret = 0; | ||
998 | if (timer_pending(timer)) { | ||
999 | detach_timer(timer, 1); | ||
1000 | if (timer->expires == base->next_timer && | ||
1001 | !tbase_get_deferrable(timer->base)) | ||
1002 | base->next_timer = base->timer_jiffies; | ||
1003 | ret = 1; | ||
1004 | } | 1014 | } |
1005 | out: | ||
1006 | spin_unlock_irqrestore(&base->lock, flags); | 1015 | spin_unlock_irqrestore(&base->lock, flags); |
1007 | 1016 | ||
1008 | return ret; | 1017 | return ret; |
@@ -1089,7 +1098,8 @@ static int cascade(struct tvec_base *base, struct tvec *tv, int index) | |||
1089 | */ | 1098 | */ |
1090 | list_for_each_entry_safe(timer, tmp, &tv_list, entry) { | 1099 | list_for_each_entry_safe(timer, tmp, &tv_list, entry) { |
1091 | BUG_ON(tbase_get_base(timer->base) != base); | 1100 | BUG_ON(tbase_get_base(timer->base) != base); |
1092 | internal_add_timer(base, timer); | 1101 | /* No accounting, while moving them */ |
1102 | __internal_add_timer(base, timer); | ||
1093 | } | 1103 | } |
1094 | 1104 | ||
1095 | return index; | 1105 | return index; |
@@ -1178,7 +1188,7 @@ static inline void __run_timers(struct tvec_base *base) | |||
1178 | timer_stats_account_timer(timer); | 1188 | timer_stats_account_timer(timer); |
1179 | 1189 | ||
1180 | base->running_timer = timer; | 1190 | base->running_timer = timer; |
1181 | detach_timer(timer, 1); | 1191 | detach_expired_timer(timer, base); |
1182 | 1192 | ||
1183 | spin_unlock_irq(&base->lock); | 1193 | spin_unlock_irq(&base->lock); |
1184 | call_timer_fn(timer, fn, data); | 1194 | call_timer_fn(timer, fn, data); |
@@ -1316,18 +1326,21 @@ static unsigned long cmp_next_hrtimer_event(unsigned long now, | |||
1316 | unsigned long get_next_timer_interrupt(unsigned long now) | 1326 | unsigned long get_next_timer_interrupt(unsigned long now) |
1317 | { | 1327 | { |
1318 | struct tvec_base *base = __this_cpu_read(tvec_bases); | 1328 | struct tvec_base *base = __this_cpu_read(tvec_bases); |
1319 | unsigned long expires; | 1329 | unsigned long expires = now + NEXT_TIMER_MAX_DELTA; |
1320 | 1330 | ||
1321 | /* | 1331 | /* |
1322 | * Pretend that there is no timer pending if the cpu is offline. | 1332 | * Pretend that there is no timer pending if the cpu is offline. |
1323 | * Possible pending timers will be migrated later to an active cpu. | 1333 | * Possible pending timers will be migrated later to an active cpu. |
1324 | */ | 1334 | */ |
1325 | if (cpu_is_offline(smp_processor_id())) | 1335 | if (cpu_is_offline(smp_processor_id())) |
1326 | return now + NEXT_TIMER_MAX_DELTA; | 1336 | return expires; |
1337 | |||
1327 | spin_lock(&base->lock); | 1338 | spin_lock(&base->lock); |
1328 | if (time_before_eq(base->next_timer, base->timer_jiffies)) | 1339 | if (base->active_timers) { |
1329 | base->next_timer = __next_timer_interrupt(base); | 1340 | if (time_before_eq(base->next_timer, base->timer_jiffies)) |
1330 | expires = base->next_timer; | 1341 | base->next_timer = __next_timer_interrupt(base); |
1342 | expires = base->next_timer; | ||
1343 | } | ||
1331 | spin_unlock(&base->lock); | 1344 | spin_unlock(&base->lock); |
1332 | 1345 | ||
1333 | if (time_before_eq(expires, now)) | 1346 | if (time_before_eq(expires, now)) |
@@ -1704,6 +1717,7 @@ static int __cpuinit init_timers_cpu(int cpu) | |||
1704 | 1717 | ||
1705 | base->timer_jiffies = jiffies; | 1718 | base->timer_jiffies = jiffies; |
1706 | base->next_timer = base->timer_jiffies; | 1719 | base->next_timer = base->timer_jiffies; |
1720 | base->active_timers = 0; | ||
1707 | return 0; | 1721 | return 0; |
1708 | } | 1722 | } |
1709 | 1723 | ||
@@ -1714,11 +1728,9 @@ static void migrate_timer_list(struct tvec_base *new_base, struct list_head *hea | |||
1714 | 1728 | ||
1715 | while (!list_empty(head)) { | 1729 | while (!list_empty(head)) { |
1716 | timer = list_first_entry(head, struct timer_list, entry); | 1730 | timer = list_first_entry(head, struct timer_list, entry); |
1717 | detach_timer(timer, 0); | 1731 | /* We ignore the accounting on the dying cpu */ |
1732 | detach_timer(timer, false); | ||
1718 | timer_set_base(timer, new_base); | 1733 | timer_set_base(timer, new_base); |
1719 | if (time_before(timer->expires, new_base->next_timer) && | ||
1720 | !tbase_get_deferrable(timer->base)) | ||
1721 | new_base->next_timer = timer->expires; | ||
1722 | internal_add_timer(new_base, timer); | 1734 | internal_add_timer(new_base, timer); |
1723 | } | 1735 | } |
1724 | } | 1736 | } |