diff options
-rw-r--r-- | include/linux/hrtimer.h | 2 | ||||
-rw-r--r-- | kernel/hrtimer.c | 50 | ||||
-rw-r--r-- | kernel/time/timer_list.c | 6 | ||||
-rw-r--r-- | kernel/time/timer_stats.c | 17 |
4 files changed, 38 insertions, 37 deletions
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index af634e95871d..5d86fb2309d2 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h | |||
@@ -169,7 +169,7 @@ struct hrtimer_clock_base { | |||
169 | * @max_hang_time: Maximum time spent in hrtimer_interrupt | 169 | * @max_hang_time: Maximum time spent in hrtimer_interrupt |
170 | */ | 170 | */ |
171 | struct hrtimer_cpu_base { | 171 | struct hrtimer_cpu_base { |
172 | spinlock_t lock; | 172 | raw_spinlock_t lock; |
173 | struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; | 173 | struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; |
174 | #ifdef CONFIG_HIGH_RES_TIMERS | 174 | #ifdef CONFIG_HIGH_RES_TIMERS |
175 | ktime_t expires_next; | 175 | ktime_t expires_next; |
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index d2f9239dc6ba..0086628b6e97 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
@@ -127,11 +127,11 @@ struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, | |||
127 | for (;;) { | 127 | for (;;) { |
128 | base = timer->base; | 128 | base = timer->base; |
129 | if (likely(base != NULL)) { | 129 | if (likely(base != NULL)) { |
130 | spin_lock_irqsave(&base->cpu_base->lock, *flags); | 130 | raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); |
131 | if (likely(base == timer->base)) | 131 | if (likely(base == timer->base)) |
132 | return base; | 132 | return base; |
133 | /* The timer has migrated to another CPU: */ | 133 | /* The timer has migrated to another CPU: */ |
134 | spin_unlock_irqrestore(&base->cpu_base->lock, *flags); | 134 | raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags); |
135 | } | 135 | } |
136 | cpu_relax(); | 136 | cpu_relax(); |
137 | } | 137 | } |
@@ -208,13 +208,13 @@ again: | |||
208 | 208 | ||
209 | /* See the comment in lock_timer_base() */ | 209 | /* See the comment in lock_timer_base() */ |
210 | timer->base = NULL; | 210 | timer->base = NULL; |
211 | spin_unlock(&base->cpu_base->lock); | 211 | raw_spin_unlock(&base->cpu_base->lock); |
212 | spin_lock(&new_base->cpu_base->lock); | 212 | raw_spin_lock(&new_base->cpu_base->lock); |
213 | 213 | ||
214 | if (cpu != this_cpu && hrtimer_check_target(timer, new_base)) { | 214 | if (cpu != this_cpu && hrtimer_check_target(timer, new_base)) { |
215 | cpu = this_cpu; | 215 | cpu = this_cpu; |
216 | spin_unlock(&new_base->cpu_base->lock); | 216 | raw_spin_unlock(&new_base->cpu_base->lock); |
217 | spin_lock(&base->cpu_base->lock); | 217 | raw_spin_lock(&base->cpu_base->lock); |
218 | timer->base = base; | 218 | timer->base = base; |
219 | goto again; | 219 | goto again; |
220 | } | 220 | } |
@@ -230,7 +230,7 @@ lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) | |||
230 | { | 230 | { |
231 | struct hrtimer_clock_base *base = timer->base; | 231 | struct hrtimer_clock_base *base = timer->base; |
232 | 232 | ||
233 | spin_lock_irqsave(&base->cpu_base->lock, *flags); | 233 | raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); |
234 | 234 | ||
235 | return base; | 235 | return base; |
236 | } | 236 | } |
@@ -628,12 +628,12 @@ static void retrigger_next_event(void *arg) | |||
628 | base = &__get_cpu_var(hrtimer_bases); | 628 | base = &__get_cpu_var(hrtimer_bases); |
629 | 629 | ||
630 | /* Adjust CLOCK_REALTIME offset */ | 630 | /* Adjust CLOCK_REALTIME offset */ |
631 | spin_lock(&base->lock); | 631 | raw_spin_lock(&base->lock); |
632 | base->clock_base[CLOCK_REALTIME].offset = | 632 | base->clock_base[CLOCK_REALTIME].offset = |
633 | timespec_to_ktime(realtime_offset); | 633 | timespec_to_ktime(realtime_offset); |
634 | 634 | ||
635 | hrtimer_force_reprogram(base, 0); | 635 | hrtimer_force_reprogram(base, 0); |
636 | spin_unlock(&base->lock); | 636 | raw_spin_unlock(&base->lock); |
637 | } | 637 | } |
638 | 638 | ||
639 | /* | 639 | /* |
@@ -694,9 +694,9 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | |||
694 | { | 694 | { |
695 | if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { | 695 | if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { |
696 | if (wakeup) { | 696 | if (wakeup) { |
697 | spin_unlock(&base->cpu_base->lock); | 697 | raw_spin_unlock(&base->cpu_base->lock); |
698 | raise_softirq_irqoff(HRTIMER_SOFTIRQ); | 698 | raise_softirq_irqoff(HRTIMER_SOFTIRQ); |
699 | spin_lock(&base->cpu_base->lock); | 699 | raw_spin_lock(&base->cpu_base->lock); |
700 | } else | 700 | } else |
701 | __raise_softirq_irqoff(HRTIMER_SOFTIRQ); | 701 | __raise_softirq_irqoff(HRTIMER_SOFTIRQ); |
702 | 702 | ||
@@ -790,7 +790,7 @@ static inline void timer_stats_account_hrtimer(struct hrtimer *timer) | |||
790 | static inline | 790 | static inline |
791 | void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) | 791 | void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) |
792 | { | 792 | { |
793 | spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); | 793 | raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); |
794 | } | 794 | } |
795 | 795 | ||
796 | /** | 796 | /** |
@@ -1123,7 +1123,7 @@ ktime_t hrtimer_get_next_event(void) | |||
1123 | unsigned long flags; | 1123 | unsigned long flags; |
1124 | int i; | 1124 | int i; |
1125 | 1125 | ||
1126 | spin_lock_irqsave(&cpu_base->lock, flags); | 1126 | raw_spin_lock_irqsave(&cpu_base->lock, flags); |
1127 | 1127 | ||
1128 | if (!hrtimer_hres_active()) { | 1128 | if (!hrtimer_hres_active()) { |
1129 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { | 1129 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { |
@@ -1140,7 +1140,7 @@ ktime_t hrtimer_get_next_event(void) | |||
1140 | } | 1140 | } |
1141 | } | 1141 | } |
1142 | 1142 | ||
1143 | spin_unlock_irqrestore(&cpu_base->lock, flags); | 1143 | raw_spin_unlock_irqrestore(&cpu_base->lock, flags); |
1144 | 1144 | ||
1145 | if (mindelta.tv64 < 0) | 1145 | if (mindelta.tv64 < 0) |
1146 | mindelta.tv64 = 0; | 1146 | mindelta.tv64 = 0; |
@@ -1222,11 +1222,11 @@ static void __run_hrtimer(struct hrtimer *timer, ktime_t *now) | |||
1222 | * they get migrated to another cpu, therefore its safe to unlock | 1222 | * they get migrated to another cpu, therefore its safe to unlock |
1223 | * the timer base. | 1223 | * the timer base. |
1224 | */ | 1224 | */ |
1225 | spin_unlock(&cpu_base->lock); | 1225 | raw_spin_unlock(&cpu_base->lock); |
1226 | trace_hrtimer_expire_entry(timer, now); | 1226 | trace_hrtimer_expire_entry(timer, now); |
1227 | restart = fn(timer); | 1227 | restart = fn(timer); |
1228 | trace_hrtimer_expire_exit(timer); | 1228 | trace_hrtimer_expire_exit(timer); |
1229 | spin_lock(&cpu_base->lock); | 1229 | raw_spin_lock(&cpu_base->lock); |
1230 | 1230 | ||
1231 | /* | 1231 | /* |
1232 | * Note: We clear the CALLBACK bit after enqueue_hrtimer and | 1232 | * Note: We clear the CALLBACK bit after enqueue_hrtimer and |
@@ -1261,7 +1261,7 @@ void hrtimer_interrupt(struct clock_event_device *dev) | |||
1261 | retry: | 1261 | retry: |
1262 | expires_next.tv64 = KTIME_MAX; | 1262 | expires_next.tv64 = KTIME_MAX; |
1263 | 1263 | ||
1264 | spin_lock(&cpu_base->lock); | 1264 | raw_spin_lock(&cpu_base->lock); |
1265 | /* | 1265 | /* |
1266 | * We set expires_next to KTIME_MAX here with cpu_base->lock | 1266 | * We set expires_next to KTIME_MAX here with cpu_base->lock |
1267 | * held to prevent that a timer is enqueued in our queue via | 1267 | * held to prevent that a timer is enqueued in our queue via |
@@ -1317,7 +1317,7 @@ retry: | |||
1317 | * against it. | 1317 | * against it. |
1318 | */ | 1318 | */ |
1319 | cpu_base->expires_next = expires_next; | 1319 | cpu_base->expires_next = expires_next; |
1320 | spin_unlock(&cpu_base->lock); | 1320 | raw_spin_unlock(&cpu_base->lock); |
1321 | 1321 | ||
1322 | /* Reprogramming necessary ? */ | 1322 | /* Reprogramming necessary ? */ |
1323 | if (expires_next.tv64 == KTIME_MAX || | 1323 | if (expires_next.tv64 == KTIME_MAX || |
@@ -1457,7 +1457,7 @@ void hrtimer_run_queues(void) | |||
1457 | gettime = 0; | 1457 | gettime = 0; |
1458 | } | 1458 | } |
1459 | 1459 | ||
1460 | spin_lock(&cpu_base->lock); | 1460 | raw_spin_lock(&cpu_base->lock); |
1461 | 1461 | ||
1462 | while ((node = base->first)) { | 1462 | while ((node = base->first)) { |
1463 | struct hrtimer *timer; | 1463 | struct hrtimer *timer; |
@@ -1469,7 +1469,7 @@ void hrtimer_run_queues(void) | |||
1469 | 1469 | ||
1470 | __run_hrtimer(timer, &base->softirq_time); | 1470 | __run_hrtimer(timer, &base->softirq_time); |
1471 | } | 1471 | } |
1472 | spin_unlock(&cpu_base->lock); | 1472 | raw_spin_unlock(&cpu_base->lock); |
1473 | } | 1473 | } |
1474 | } | 1474 | } |
1475 | 1475 | ||
@@ -1625,7 +1625,7 @@ static void __cpuinit init_hrtimers_cpu(int cpu) | |||
1625 | struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); | 1625 | struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); |
1626 | int i; | 1626 | int i; |
1627 | 1627 | ||
1628 | spin_lock_init(&cpu_base->lock); | 1628 | raw_spin_lock_init(&cpu_base->lock); |
1629 | 1629 | ||
1630 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) | 1630 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) |
1631 | cpu_base->clock_base[i].cpu_base = cpu_base; | 1631 | cpu_base->clock_base[i].cpu_base = cpu_base; |
@@ -1683,16 +1683,16 @@ static void migrate_hrtimers(int scpu) | |||
1683 | * The caller is globally serialized and nobody else | 1683 | * The caller is globally serialized and nobody else |
1684 | * takes two locks at once, deadlock is not possible. | 1684 | * takes two locks at once, deadlock is not possible. |
1685 | */ | 1685 | */ |
1686 | spin_lock(&new_base->lock); | 1686 | raw_spin_lock(&new_base->lock); |
1687 | spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); | 1687 | raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); |
1688 | 1688 | ||
1689 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { | 1689 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { |
1690 | migrate_hrtimer_list(&old_base->clock_base[i], | 1690 | migrate_hrtimer_list(&old_base->clock_base[i], |
1691 | &new_base->clock_base[i]); | 1691 | &new_base->clock_base[i]); |
1692 | } | 1692 | } |
1693 | 1693 | ||
1694 | spin_unlock(&old_base->lock); | 1694 | raw_spin_unlock(&old_base->lock); |
1695 | spin_unlock(&new_base->lock); | 1695 | raw_spin_unlock(&new_base->lock); |
1696 | 1696 | ||
1697 | /* Check, if we got expired work to do */ | 1697 | /* Check, if we got expired work to do */ |
1698 | __hrtimer_peek_ahead_timers(); | 1698 | __hrtimer_peek_ahead_timers(); |
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index 9d80db4747d4..28265636b6c2 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c | |||
@@ -84,7 +84,7 @@ print_active_timers(struct seq_file *m, struct hrtimer_clock_base *base, | |||
84 | 84 | ||
85 | next_one: | 85 | next_one: |
86 | i = 0; | 86 | i = 0; |
87 | spin_lock_irqsave(&base->cpu_base->lock, flags); | 87 | raw_spin_lock_irqsave(&base->cpu_base->lock, flags); |
88 | 88 | ||
89 | curr = base->first; | 89 | curr = base->first; |
90 | /* | 90 | /* |
@@ -100,13 +100,13 @@ next_one: | |||
100 | 100 | ||
101 | timer = rb_entry(curr, struct hrtimer, node); | 101 | timer = rb_entry(curr, struct hrtimer, node); |
102 | tmp = *timer; | 102 | tmp = *timer; |
103 | spin_unlock_irqrestore(&base->cpu_base->lock, flags); | 103 | raw_spin_unlock_irqrestore(&base->cpu_base->lock, flags); |
104 | 104 | ||
105 | print_timer(m, timer, &tmp, i, now); | 105 | print_timer(m, timer, &tmp, i, now); |
106 | next++; | 106 | next++; |
107 | goto next_one; | 107 | goto next_one; |
108 | } | 108 | } |
109 | spin_unlock_irqrestore(&base->cpu_base->lock, flags); | 109 | raw_spin_unlock_irqrestore(&base->cpu_base->lock, flags); |
110 | } | 110 | } |
111 | 111 | ||
112 | static void | 112 | static void |
diff --git a/kernel/time/timer_stats.c b/kernel/time/timer_stats.c index 63b117e9eba1..2f3b585b8d7d 100644 --- a/kernel/time/timer_stats.c +++ b/kernel/time/timer_stats.c | |||
@@ -86,7 +86,7 @@ static DEFINE_SPINLOCK(table_lock); | |||
86 | /* | 86 | /* |
87 | * Per-CPU lookup locks for fast hash lookup: | 87 | * Per-CPU lookup locks for fast hash lookup: |
88 | */ | 88 | */ |
89 | static DEFINE_PER_CPU(spinlock_t, tstats_lookup_lock); | 89 | static DEFINE_PER_CPU(raw_spinlock_t, tstats_lookup_lock); |
90 | 90 | ||
91 | /* | 91 | /* |
92 | * Mutex to serialize state changes with show-stats activities: | 92 | * Mutex to serialize state changes with show-stats activities: |
@@ -238,7 +238,7 @@ void timer_stats_update_stats(void *timer, pid_t pid, void *startf, | |||
238 | /* | 238 | /* |
239 | * It doesnt matter which lock we take: | 239 | * It doesnt matter which lock we take: |
240 | */ | 240 | */ |
241 | spinlock_t *lock; | 241 | raw_spinlock_t *lock; |
242 | struct entry *entry, input; | 242 | struct entry *entry, input; |
243 | unsigned long flags; | 243 | unsigned long flags; |
244 | 244 | ||
@@ -253,7 +253,7 @@ void timer_stats_update_stats(void *timer, pid_t pid, void *startf, | |||
253 | input.pid = pid; | 253 | input.pid = pid; |
254 | input.timer_flag = timer_flag; | 254 | input.timer_flag = timer_flag; |
255 | 255 | ||
256 | spin_lock_irqsave(lock, flags); | 256 | raw_spin_lock_irqsave(lock, flags); |
257 | if (!timer_stats_active) | 257 | if (!timer_stats_active) |
258 | goto out_unlock; | 258 | goto out_unlock; |
259 | 259 | ||
@@ -264,7 +264,7 @@ void timer_stats_update_stats(void *timer, pid_t pid, void *startf, | |||
264 | atomic_inc(&overflow_count); | 264 | atomic_inc(&overflow_count); |
265 | 265 | ||
266 | out_unlock: | 266 | out_unlock: |
267 | spin_unlock_irqrestore(lock, flags); | 267 | raw_spin_unlock_irqrestore(lock, flags); |
268 | } | 268 | } |
269 | 269 | ||
270 | static void print_name_offset(struct seq_file *m, unsigned long addr) | 270 | static void print_name_offset(struct seq_file *m, unsigned long addr) |
@@ -348,10 +348,11 @@ static void sync_access(void) | |||
348 | int cpu; | 348 | int cpu; |
349 | 349 | ||
350 | for_each_online_cpu(cpu) { | 350 | for_each_online_cpu(cpu) { |
351 | spinlock_t *lock = &per_cpu(tstats_lookup_lock, cpu); | 351 | raw_spinlock_t *lock = &per_cpu(tstats_lookup_lock, cpu); |
352 | spin_lock_irqsave(lock, flags); | 352 | |
353 | raw_spin_lock_irqsave(lock, flags); | ||
353 | /* nothing */ | 354 | /* nothing */ |
354 | spin_unlock_irqrestore(lock, flags); | 355 | raw_spin_unlock_irqrestore(lock, flags); |
355 | } | 356 | } |
356 | } | 357 | } |
357 | 358 | ||
@@ -409,7 +410,7 @@ void __init init_timer_stats(void) | |||
409 | int cpu; | 410 | int cpu; |
410 | 411 | ||
411 | for_each_possible_cpu(cpu) | 412 | for_each_possible_cpu(cpu) |
412 | spin_lock_init(&per_cpu(tstats_lookup_lock, cpu)); | 413 | raw_spin_lock_init(&per_cpu(tstats_lookup_lock, cpu)); |
413 | } | 414 | } |
414 | 415 | ||
415 | static int __init init_tstats_procfs(void) | 416 | static int __init init_tstats_procfs(void) |