diff options
author | Thomas Gleixner <tglx@linutronix.de> | 2011-05-20 07:05:15 -0400 |
---|---|---|
committer | Thomas Gleixner <tglx@linutronix.de> | 2011-05-23 07:59:54 -0400 |
commit | ab8177bc53e8ae3a3ba6d200ce2c2dae263f7ee5 (patch) | |
tree | c8b370496497b4f96d6a17da906bdd9314e9a090 | |
parent | f24444b01bf6c51c300fd3ffc73423383d747882 (diff) |
hrtimers: Avoid touching inactive timer bases
Instead of iterating over all possible timer bases avoid it by marking
the active bases in the cpu base.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Peter Zijlstra <peterz@infradead.org>
-rw-r--r-- | include/linux/hrtimer.h | 7 | ||||
-rw-r--r-- | include/linux/thread_info.h | 2 | ||||
-rw-r--r-- | kernel/hrtimer.c | 29 | ||||
-rw-r--r-- | kernel/posix-cpu-timers.c | 4 | ||||
-rw-r--r-- | kernel/posix-timers.c | 2 | ||||
-rw-r--r-- | kernel/time/alarmtimer.c | 4 |
6 files changed, 29 insertions, 19 deletions
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index cc5f5f51db10..771c95802edc 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h | |||
@@ -143,7 +143,8 @@ struct hrtimer_sleeper { | |||
143 | */ | 143 | */ |
144 | struct hrtimer_clock_base { | 144 | struct hrtimer_clock_base { |
145 | struct hrtimer_cpu_base *cpu_base; | 145 | struct hrtimer_cpu_base *cpu_base; |
146 | clockid_t index; | 146 | int index; |
147 | clockid_t clockid; | ||
147 | struct timerqueue_head active; | 148 | struct timerqueue_head active; |
148 | ktime_t resolution; | 149 | ktime_t resolution; |
149 | ktime_t (*get_time)(void); | 150 | ktime_t (*get_time)(void); |
@@ -162,7 +163,7 @@ enum hrtimer_base_type { | |||
162 | * struct hrtimer_cpu_base - the per cpu clock bases | 163 | * struct hrtimer_cpu_base - the per cpu clock bases |
163 | * @lock: lock protecting the base and associated clock bases | 164 | * @lock: lock protecting the base and associated clock bases |
164 | * and timers | 165 | * and timers |
165 | * @clock_base: array of clock bases for this cpu | 166 | * @active_bases: Bitfield to mark bases with active timers |
166 | * @expires_next: absolute time of the next event which was scheduled | 167 | * @expires_next: absolute time of the next event which was scheduled |
167 | * via clock_set_next_event() | 168 | * via clock_set_next_event() |
168 | * @hres_active: State of high resolution mode | 169 | * @hres_active: State of high resolution mode |
@@ -171,9 +172,11 @@ enum hrtimer_base_type { | |||
171 | * @nr_retries: Total number of hrtimer interrupt retries | 172 | * @nr_retries: Total number of hrtimer interrupt retries |
172 | * @nr_hangs: Total number of hrtimer interrupt hangs | 173 | * @nr_hangs: Total number of hrtimer interrupt hangs |
173 | * @max_hang_time: Maximum time spent in hrtimer_interrupt | 174 | * @max_hang_time: Maximum time spent in hrtimer_interrupt |
175 | * @clock_base: array of clock bases for this cpu | ||
174 | */ | 176 | */ |
175 | struct hrtimer_cpu_base { | 177 | struct hrtimer_cpu_base { |
176 | raw_spinlock_t lock; | 178 | raw_spinlock_t lock; |
179 | unsigned long active_bases; | ||
177 | #ifdef CONFIG_HIGH_RES_TIMERS | 180 | #ifdef CONFIG_HIGH_RES_TIMERS |
178 | ktime_t expires_next; | 181 | ktime_t expires_next; |
179 | int hres_active; | 182 | int hres_active; |
diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h index 20fc303947d3..8d03f079688c 100644 --- a/include/linux/thread_info.h +++ b/include/linux/thread_info.h | |||
@@ -29,7 +29,7 @@ struct restart_block { | |||
29 | } futex; | 29 | } futex; |
30 | /* For nanosleep */ | 30 | /* For nanosleep */ |
31 | struct { | 31 | struct { |
32 | clockid_t index; | 32 | clockid_t clockid; |
33 | struct timespec __user *rmtp; | 33 | struct timespec __user *rmtp; |
34 | #ifdef CONFIG_COMPAT | 34 | #ifdef CONFIG_COMPAT |
35 | struct compat_timespec __user *compat_rmtp; | 35 | struct compat_timespec __user *compat_rmtp; |
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 26dd32f9f6b2..1b08f6d67f12 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
@@ -64,17 +64,20 @@ DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) = | |||
64 | .clock_base = | 64 | .clock_base = |
65 | { | 65 | { |
66 | { | 66 | { |
67 | .index = CLOCK_REALTIME, | 67 | .index = HRTIMER_BASE_REALTIME, |
68 | .clockid = CLOCK_REALTIME, | ||
68 | .get_time = &ktime_get_real, | 69 | .get_time = &ktime_get_real, |
69 | .resolution = KTIME_LOW_RES, | 70 | .resolution = KTIME_LOW_RES, |
70 | }, | 71 | }, |
71 | { | 72 | { |
72 | .index = CLOCK_MONOTONIC, | 73 | .index = HRTIMER_BASE_MONOTONIC, |
74 | .clockid = CLOCK_MONOTONIC, | ||
73 | .get_time = &ktime_get, | 75 | .get_time = &ktime_get, |
74 | .resolution = KTIME_LOW_RES, | 76 | .resolution = KTIME_LOW_RES, |
75 | }, | 77 | }, |
76 | { | 78 | { |
77 | .index = CLOCK_BOOTTIME, | 79 | .index = HRTIMER_BASE_BOOTTIME, |
80 | .clockid = CLOCK_BOOTTIME, | ||
78 | .get_time = &ktime_get_boottime, | 81 | .get_time = &ktime_get_boottime, |
79 | .resolution = KTIME_LOW_RES, | 82 | .resolution = KTIME_LOW_RES, |
80 | }, | 83 | }, |
@@ -196,7 +199,7 @@ switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, | |||
196 | struct hrtimer_cpu_base *new_cpu_base; | 199 | struct hrtimer_cpu_base *new_cpu_base; |
197 | int this_cpu = smp_processor_id(); | 200 | int this_cpu = smp_processor_id(); |
198 | int cpu = hrtimer_get_target(this_cpu, pinned); | 201 | int cpu = hrtimer_get_target(this_cpu, pinned); |
199 | int basenum = hrtimer_clockid_to_base(base->index); | 202 | int basenum = base->index; |
200 | 203 | ||
201 | again: | 204 | again: |
202 | new_cpu_base = &per_cpu(hrtimer_bases, cpu); | 205 | new_cpu_base = &per_cpu(hrtimer_bases, cpu); |
@@ -857,6 +860,7 @@ static int enqueue_hrtimer(struct hrtimer *timer, | |||
857 | debug_activate(timer); | 860 | debug_activate(timer); |
858 | 861 | ||
859 | timerqueue_add(&base->active, &timer->node); | 862 | timerqueue_add(&base->active, &timer->node); |
863 | base->cpu_base->active_bases |= 1 << base->index; | ||
860 | 864 | ||
861 | /* | 865 | /* |
862 | * HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the | 866 | * HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the |
@@ -898,6 +902,8 @@ static void __remove_hrtimer(struct hrtimer *timer, | |||
898 | #endif | 902 | #endif |
899 | } | 903 | } |
900 | timerqueue_del(&base->active, &timer->node); | 904 | timerqueue_del(&base->active, &timer->node); |
905 | if (!timerqueue_getnext(&base->active)) | ||
906 | base->cpu_base->active_bases &= ~(1 << base->index); | ||
901 | out: | 907 | out: |
902 | timer->state = newstate; | 908 | timer->state = newstate; |
903 | } | 909 | } |
@@ -1235,7 +1241,6 @@ static void __run_hrtimer(struct hrtimer *timer, ktime_t *now) | |||
1235 | void hrtimer_interrupt(struct clock_event_device *dev) | 1241 | void hrtimer_interrupt(struct clock_event_device *dev) |
1236 | { | 1242 | { |
1237 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); | 1243 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); |
1238 | struct hrtimer_clock_base *base; | ||
1239 | ktime_t expires_next, now, entry_time, delta; | 1244 | ktime_t expires_next, now, entry_time, delta; |
1240 | int i, retries = 0; | 1245 | int i, retries = 0; |
1241 | 1246 | ||
@@ -1257,12 +1262,15 @@ retry: | |||
1257 | */ | 1262 | */ |
1258 | cpu_base->expires_next.tv64 = KTIME_MAX; | 1263 | cpu_base->expires_next.tv64 = KTIME_MAX; |
1259 | 1264 | ||
1260 | base = cpu_base->clock_base; | ||
1261 | |||
1262 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { | 1265 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { |
1263 | ktime_t basenow; | 1266 | struct hrtimer_clock_base *base; |
1264 | struct timerqueue_node *node; | 1267 | struct timerqueue_node *node; |
1268 | ktime_t basenow; | ||
1269 | |||
1270 | if (!(cpu_base->active_bases & (1 << i))) | ||
1271 | continue; | ||
1265 | 1272 | ||
1273 | base = cpu_base->clock_base + i; | ||
1266 | basenow = ktime_add(now, base->offset); | 1274 | basenow = ktime_add(now, base->offset); |
1267 | 1275 | ||
1268 | while ((node = timerqueue_getnext(&base->active))) { | 1276 | while ((node = timerqueue_getnext(&base->active))) { |
@@ -1295,7 +1303,6 @@ retry: | |||
1295 | 1303 | ||
1296 | __run_hrtimer(timer, &basenow); | 1304 | __run_hrtimer(timer, &basenow); |
1297 | } | 1305 | } |
1298 | base++; | ||
1299 | } | 1306 | } |
1300 | 1307 | ||
1301 | /* | 1308 | /* |
@@ -1526,7 +1533,7 @@ long __sched hrtimer_nanosleep_restart(struct restart_block *restart) | |||
1526 | struct timespec __user *rmtp; | 1533 | struct timespec __user *rmtp; |
1527 | int ret = 0; | 1534 | int ret = 0; |
1528 | 1535 | ||
1529 | hrtimer_init_on_stack(&t.timer, restart->nanosleep.index, | 1536 | hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid, |
1530 | HRTIMER_MODE_ABS); | 1537 | HRTIMER_MODE_ABS); |
1531 | hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); | 1538 | hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); |
1532 | 1539 | ||
@@ -1578,7 +1585,7 @@ long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, | |||
1578 | 1585 | ||
1579 | restart = ¤t_thread_info()->restart_block; | 1586 | restart = ¤t_thread_info()->restart_block; |
1580 | restart->fn = hrtimer_nanosleep_restart; | 1587 | restart->fn = hrtimer_nanosleep_restart; |
1581 | restart->nanosleep.index = t.timer.base->index; | 1588 | restart->nanosleep.clockid = t.timer.base->clockid; |
1582 | restart->nanosleep.rmtp = rmtp; | 1589 | restart->nanosleep.rmtp = rmtp; |
1583 | restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer); | 1590 | restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer); |
1584 | 1591 | ||
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index 0791b13df7bf..58f405b581e7 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c | |||
@@ -1514,7 +1514,7 @@ static int posix_cpu_nsleep(const clockid_t which_clock, int flags, | |||
1514 | return -EFAULT; | 1514 | return -EFAULT; |
1515 | 1515 | ||
1516 | restart_block->fn = posix_cpu_nsleep_restart; | 1516 | restart_block->fn = posix_cpu_nsleep_restart; |
1517 | restart_block->nanosleep.index = which_clock; | 1517 | restart_block->nanosleep.clockid = which_clock; |
1518 | restart_block->nanosleep.rmtp = rmtp; | 1518 | restart_block->nanosleep.rmtp = rmtp; |
1519 | restart_block->nanosleep.expires = timespec_to_ns(rqtp); | 1519 | restart_block->nanosleep.expires = timespec_to_ns(rqtp); |
1520 | } | 1520 | } |
@@ -1523,7 +1523,7 @@ static int posix_cpu_nsleep(const clockid_t which_clock, int flags, | |||
1523 | 1523 | ||
1524 | static long posix_cpu_nsleep_restart(struct restart_block *restart_block) | 1524 | static long posix_cpu_nsleep_restart(struct restart_block *restart_block) |
1525 | { | 1525 | { |
1526 | clockid_t which_clock = restart_block->nanosleep.index; | 1526 | clockid_t which_clock = restart_block->nanosleep.clockid; |
1527 | struct timespec t; | 1527 | struct timespec t; |
1528 | struct itimerspec it; | 1528 | struct itimerspec it; |
1529 | int error; | 1529 | int error; |
diff --git a/kernel/posix-timers.c b/kernel/posix-timers.c index e5498d7405c3..a1b5edf1bf92 100644 --- a/kernel/posix-timers.c +++ b/kernel/posix-timers.c | |||
@@ -1056,7 +1056,7 @@ SYSCALL_DEFINE4(clock_nanosleep, const clockid_t, which_clock, int, flags, | |||
1056 | */ | 1056 | */ |
1057 | long clock_nanosleep_restart(struct restart_block *restart_block) | 1057 | long clock_nanosleep_restart(struct restart_block *restart_block) |
1058 | { | 1058 | { |
1059 | clockid_t which_clock = restart_block->nanosleep.index; | 1059 | clockid_t which_clock = restart_block->nanosleep.clockid; |
1060 | struct k_clock *kc = clockid_to_kclock(which_clock); | 1060 | struct k_clock *kc = clockid_to_kclock(which_clock); |
1061 | 1061 | ||
1062 | if (WARN_ON_ONCE(!kc || !kc->nsleep_restart)) | 1062 | if (WARN_ON_ONCE(!kc || !kc->nsleep_restart)) |
diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index c6027fe9a4e6..2d966244ea60 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c | |||
@@ -494,7 +494,7 @@ static int update_rmtp(ktime_t exp, enum alarmtimer_type type, | |||
494 | */ | 494 | */ |
495 | static long __sched alarm_timer_nsleep_restart(struct restart_block *restart) | 495 | static long __sched alarm_timer_nsleep_restart(struct restart_block *restart) |
496 | { | 496 | { |
497 | enum alarmtimer_type type = restart->nanosleep.index; | 497 | enum alarmtimer_type type = restart->nanosleep.clockid; |
498 | ktime_t exp; | 498 | ktime_t exp; |
499 | struct timespec __user *rmtp; | 499 | struct timespec __user *rmtp; |
500 | struct alarm alarm; | 500 | struct alarm alarm; |
@@ -573,7 +573,7 @@ static int alarm_timer_nsleep(const clockid_t which_clock, int flags, | |||
573 | 573 | ||
574 | restart = ¤t_thread_info()->restart_block; | 574 | restart = ¤t_thread_info()->restart_block; |
575 | restart->fn = alarm_timer_nsleep_restart; | 575 | restart->fn = alarm_timer_nsleep_restart; |
576 | restart->nanosleep.index = type; | 576 | restart->nanosleep.clockid = type; |
577 | restart->nanosleep.expires = exp.tv64; | 577 | restart->nanosleep.expires = exp.tv64; |
578 | restart->nanosleep.rmtp = rmtp; | 578 | restart->nanosleep.rmtp = rmtp; |
579 | ret = -ERESTART_RESTARTBLOCK; | 579 | ret = -ERESTART_RESTARTBLOCK; |