diff options
-rw-r--r-- | include/linux/hrtimer.h | 5 | ||||
-rw-r--r-- | include/linux/interrupt.h | 1 | ||||
-rw-r--r-- | kernel/hrtimer.c | 55 | ||||
-rw-r--r-- | kernel/sched.c | 14 | ||||
-rw-r--r-- | kernel/softirq.c | 2 |
5 files changed, 52 insertions, 25 deletions
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index bd37078c2d7d..0d2f7c8a33d6 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h | |||
@@ -336,6 +336,11 @@ extern int hrtimer_start(struct hrtimer *timer, ktime_t tim, | |||
336 | const enum hrtimer_mode mode); | 336 | const enum hrtimer_mode mode); |
337 | extern int hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, | 337 | extern int hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, |
338 | unsigned long range_ns, const enum hrtimer_mode mode); | 338 | unsigned long range_ns, const enum hrtimer_mode mode); |
339 | extern int | ||
340 | __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, | ||
341 | unsigned long delta_ns, | ||
342 | const enum hrtimer_mode mode, int wakeup); | ||
343 | |||
339 | extern int hrtimer_cancel(struct hrtimer *timer); | 344 | extern int hrtimer_cancel(struct hrtimer *timer); |
340 | extern int hrtimer_try_to_cancel(struct hrtimer *timer); | 345 | extern int hrtimer_try_to_cancel(struct hrtimer *timer); |
341 | 346 | ||
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index c68bffd182bb..4528bf70866a 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h | |||
@@ -294,6 +294,7 @@ extern void softirq_init(void); | |||
294 | #define __raise_softirq_irqoff(nr) do { or_softirq_pending(1UL << (nr)); } while (0) | 294 | #define __raise_softirq_irqoff(nr) do { or_softirq_pending(1UL << (nr)); } while (0) |
295 | extern void raise_softirq_irqoff(unsigned int nr); | 295 | extern void raise_softirq_irqoff(unsigned int nr); |
296 | extern void raise_softirq(unsigned int nr); | 296 | extern void raise_softirq(unsigned int nr); |
297 | extern void wakeup_softirqd(void); | ||
297 | 298 | ||
298 | /* This is the worklist that queues up per-cpu softirq work. | 299 | /* This is the worklist that queues up per-cpu softirq work. |
299 | * | 300 | * |
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index f394d2a42ca3..cb8a15c19583 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
@@ -651,14 +651,20 @@ static inline void hrtimer_init_timer_hres(struct hrtimer *timer) | |||
651 | * and expiry check is done in the hrtimer_interrupt or in the softirq. | 651 | * and expiry check is done in the hrtimer_interrupt or in the softirq. |
652 | */ | 652 | */ |
653 | static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | 653 | static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, |
654 | struct hrtimer_clock_base *base) | 654 | struct hrtimer_clock_base *base, |
655 | int wakeup) | ||
655 | { | 656 | { |
656 | if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { | 657 | if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { |
657 | spin_unlock(&base->cpu_base->lock); | 658 | if (wakeup) { |
658 | raise_softirq_irqoff(HRTIMER_SOFTIRQ); | 659 | spin_unlock(&base->cpu_base->lock); |
659 | spin_lock(&base->cpu_base->lock); | 660 | raise_softirq_irqoff(HRTIMER_SOFTIRQ); |
661 | spin_lock(&base->cpu_base->lock); | ||
662 | } else | ||
663 | __raise_softirq_irqoff(HRTIMER_SOFTIRQ); | ||
664 | |||
660 | return 1; | 665 | return 1; |
661 | } | 666 | } |
667 | |||
662 | return 0; | 668 | return 0; |
663 | } | 669 | } |
664 | 670 | ||
@@ -703,7 +709,8 @@ static inline int hrtimer_is_hres_enabled(void) { return 0; } | |||
703 | static inline int hrtimer_switch_to_hres(void) { return 0; } | 709 | static inline int hrtimer_switch_to_hres(void) { return 0; } |
704 | static inline void hrtimer_force_reprogram(struct hrtimer_cpu_base *base) { } | 710 | static inline void hrtimer_force_reprogram(struct hrtimer_cpu_base *base) { } |
705 | static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | 711 | static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, |
706 | struct hrtimer_clock_base *base) | 712 | struct hrtimer_clock_base *base, |
713 | int wakeup) | ||
707 | { | 714 | { |
708 | return 0; | 715 | return 0; |
709 | } | 716 | } |
@@ -886,20 +893,9 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base) | |||
886 | return 0; | 893 | return 0; |
887 | } | 894 | } |
888 | 895 | ||
889 | /** | 896 | int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, |
890 | * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU | 897 | unsigned long delta_ns, const enum hrtimer_mode mode, |
891 | * @timer: the timer to be added | 898 | int wakeup) |
892 | * @tim: expiry time | ||
893 | * @delta_ns: "slack" range for the timer | ||
894 | * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL) | ||
895 | * | ||
896 | * Returns: | ||
897 | * 0 on success | ||
898 | * 1 when the timer was active | ||
899 | */ | ||
900 | int | ||
901 | hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_ns, | ||
902 | const enum hrtimer_mode mode) | ||
903 | { | 899 | { |
904 | struct hrtimer_clock_base *base, *new_base; | 900 | struct hrtimer_clock_base *base, *new_base; |
905 | unsigned long flags; | 901 | unsigned long flags; |
@@ -940,12 +936,29 @@ hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_n | |||
940 | * XXX send_remote_softirq() ? | 936 | * XXX send_remote_softirq() ? |
941 | */ | 937 | */ |
942 | if (leftmost && new_base->cpu_base == &__get_cpu_var(hrtimer_bases)) | 938 | if (leftmost && new_base->cpu_base == &__get_cpu_var(hrtimer_bases)) |
943 | hrtimer_enqueue_reprogram(timer, new_base); | 939 | hrtimer_enqueue_reprogram(timer, new_base, wakeup); |
944 | 940 | ||
945 | unlock_hrtimer_base(timer, &flags); | 941 | unlock_hrtimer_base(timer, &flags); |
946 | 942 | ||
947 | return ret; | 943 | return ret; |
948 | } | 944 | } |
945 | |||
946 | /** | ||
947 | * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU | ||
948 | * @timer: the timer to be added | ||
949 | * @tim: expiry time | ||
950 | * @delta_ns: "slack" range for the timer | ||
951 | * @mode: expiry mode: absolute (HRTIMER_ABS) or relative (HRTIMER_REL) | ||
952 | * | ||
953 | * Returns: | ||
954 | * 0 on success | ||
955 | * 1 when the timer was active | ||
956 | */ | ||
957 | int hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, | ||
958 | unsigned long delta_ns, const enum hrtimer_mode mode) | ||
959 | { | ||
960 | return __hrtimer_start_range_ns(timer, tim, delta_ns, mode, 1); | ||
961 | } | ||
949 | EXPORT_SYMBOL_GPL(hrtimer_start_range_ns); | 962 | EXPORT_SYMBOL_GPL(hrtimer_start_range_ns); |
950 | 963 | ||
951 | /** | 964 | /** |
@@ -961,7 +974,7 @@ EXPORT_SYMBOL_GPL(hrtimer_start_range_ns); | |||
961 | int | 974 | int |
962 | hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) | 975 | hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) |
963 | { | 976 | { |
964 | return hrtimer_start_range_ns(timer, tim, 0, mode); | 977 | return __hrtimer_start_range_ns(timer, tim, 0, mode, 1); |
965 | } | 978 | } |
966 | EXPORT_SYMBOL_GPL(hrtimer_start); | 979 | EXPORT_SYMBOL_GPL(hrtimer_start); |
967 | 980 | ||
diff --git a/kernel/sched.c b/kernel/sched.c index 196d48babbef..63256e3ede2a 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -231,13 +231,20 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b) | |||
231 | 231 | ||
232 | spin_lock(&rt_b->rt_runtime_lock); | 232 | spin_lock(&rt_b->rt_runtime_lock); |
233 | for (;;) { | 233 | for (;;) { |
234 | unsigned long delta; | ||
235 | ktime_t soft, hard; | ||
236 | |||
234 | if (hrtimer_active(&rt_b->rt_period_timer)) | 237 | if (hrtimer_active(&rt_b->rt_period_timer)) |
235 | break; | 238 | break; |
236 | 239 | ||
237 | now = hrtimer_cb_get_time(&rt_b->rt_period_timer); | 240 | now = hrtimer_cb_get_time(&rt_b->rt_period_timer); |
238 | hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period); | 241 | hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period); |
239 | hrtimer_start_expires(&rt_b->rt_period_timer, | 242 | |
240 | HRTIMER_MODE_ABS); | 243 | soft = hrtimer_get_softexpires(&rt_b->rt_period_timer); |
244 | hard = hrtimer_get_expires(&rt_b->rt_period_timer); | ||
245 | delta = ktime_to_ns(ktime_sub(hard, soft)); | ||
246 | __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta, | ||
247 | HRTIMER_MODE_ABS, 0); | ||
241 | } | 248 | } |
242 | spin_unlock(&rt_b->rt_runtime_lock); | 249 | spin_unlock(&rt_b->rt_runtime_lock); |
243 | } | 250 | } |
@@ -1146,7 +1153,8 @@ static __init void init_hrtick(void) | |||
1146 | */ | 1153 | */ |
1147 | static void hrtick_start(struct rq *rq, u64 delay) | 1154 | static void hrtick_start(struct rq *rq, u64 delay) |
1148 | { | 1155 | { |
1149 | hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL); | 1156 | __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0, |
1157 | HRTIMER_MODE_REL, 0); | ||
1150 | } | 1158 | } |
1151 | 1159 | ||
1152 | static inline void init_hrtick(void) | 1160 | static inline void init_hrtick(void) |
diff --git a/kernel/softirq.c b/kernel/softirq.c index 487751604300..accc85197c49 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c | |||
@@ -58,7 +58,7 @@ static DEFINE_PER_CPU(struct task_struct *, ksoftirqd); | |||
58 | * to the pending events, so lets the scheduler to balance | 58 | * to the pending events, so lets the scheduler to balance |
59 | * the softirq load for us. | 59 | * the softirq load for us. |
60 | */ | 60 | */ |
61 | static inline void wakeup_softirqd(void) | 61 | void wakeup_softirqd(void) |
62 | { | 62 | { |
63 | /* Interrupts are disabled: no need to stop preemption */ | 63 | /* Interrupts are disabled: no need to stop preemption */ |
64 | struct task_struct *tsk = __get_cpu_var(ksoftirqd); | 64 | struct task_struct *tsk = __get_cpu_var(ksoftirqd); |