diff options
author | David Woodhouse <David.Woodhouse@intel.com> | 2009-01-05 04:50:33 -0500 |
---|---|---|
committer | David Woodhouse <David.Woodhouse@intel.com> | 2009-01-05 04:50:33 -0500 |
commit | 353816f43d1fb340ff2d9a911dd5d0799c09f6a5 (patch) | |
tree | 517290fd884d286fe2971137ac89f89e3567785a /kernel/time | |
parent | 160bbab3000dafccbe43688e48208cecf4deb879 (diff) | |
parent | fe0bdec68b77020281dc814805edfe594ae89e0f (diff) |
Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6
Conflicts:
arch/arm/mach-pxa/corgi.c
arch/arm/mach-pxa/poodle.c
arch/arm/mach-pxa/spitz.c
Diffstat (limited to 'kernel/time')
-rw-r--r-- | kernel/time/clockevents.c | 2 | ||||
-rw-r--r-- | kernel/time/clocksource.c | 9 | ||||
-rw-r--r-- | kernel/time/ntp.c | 4 | ||||
-rw-r--r-- | kernel/time/tick-broadcast.c | 113 | ||||
-rw-r--r-- | kernel/time/tick-common.c | 18 | ||||
-rw-r--r-- | kernel/time/tick-sched.c | 66 |
6 files changed, 112 insertions, 100 deletions
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index f8d968063cea..ea2f48af83cf 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c | |||
@@ -166,6 +166,8 @@ static void clockevents_notify_released(void) | |||
166 | void clockevents_register_device(struct clock_event_device *dev) | 166 | void clockevents_register_device(struct clock_event_device *dev) |
167 | { | 167 | { |
168 | BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED); | 168 | BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED); |
169 | BUG_ON(!dev->cpumask); | ||
170 | |||
169 | /* | 171 | /* |
170 | * A nsec2cyc multiplicator of 0 is invalid and we'd crash | 172 | * A nsec2cyc multiplicator of 0 is invalid and we'd crash |
171 | * on it, so fix it up and emit a warning: | 173 | * on it, so fix it up and emit a warning: |
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 9ed2eec97526..ca89e1593f08 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c | |||
@@ -145,10 +145,11 @@ static void clocksource_watchdog(unsigned long data) | |||
145 | * Cycle through CPUs to check if the CPUs stay | 145 | * Cycle through CPUs to check if the CPUs stay |
146 | * synchronized to each other. | 146 | * synchronized to each other. |
147 | */ | 147 | */ |
148 | int next_cpu = next_cpu_nr(raw_smp_processor_id(), cpu_online_map); | 148 | int next_cpu = cpumask_next(raw_smp_processor_id(), |
149 | cpu_online_mask); | ||
149 | 150 | ||
150 | if (next_cpu >= nr_cpu_ids) | 151 | if (next_cpu >= nr_cpu_ids) |
151 | next_cpu = first_cpu(cpu_online_map); | 152 | next_cpu = cpumask_first(cpu_online_mask); |
152 | watchdog_timer.expires += WATCHDOG_INTERVAL; | 153 | watchdog_timer.expires += WATCHDOG_INTERVAL; |
153 | add_timer_on(&watchdog_timer, next_cpu); | 154 | add_timer_on(&watchdog_timer, next_cpu); |
154 | } | 155 | } |
@@ -173,7 +174,7 @@ static void clocksource_check_watchdog(struct clocksource *cs) | |||
173 | watchdog_last = watchdog->read(); | 174 | watchdog_last = watchdog->read(); |
174 | watchdog_timer.expires = jiffies + WATCHDOG_INTERVAL; | 175 | watchdog_timer.expires = jiffies + WATCHDOG_INTERVAL; |
175 | add_timer_on(&watchdog_timer, | 176 | add_timer_on(&watchdog_timer, |
176 | first_cpu(cpu_online_map)); | 177 | cpumask_first(cpu_online_mask)); |
177 | } | 178 | } |
178 | } else { | 179 | } else { |
179 | if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) | 180 | if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) |
@@ -195,7 +196,7 @@ static void clocksource_check_watchdog(struct clocksource *cs) | |||
195 | watchdog_timer.expires = | 196 | watchdog_timer.expires = |
196 | jiffies + WATCHDOG_INTERVAL; | 197 | jiffies + WATCHDOG_INTERVAL; |
197 | add_timer_on(&watchdog_timer, | 198 | add_timer_on(&watchdog_timer, |
198 | first_cpu(cpu_online_map)); | 199 | cpumask_first(cpu_online_mask)); |
199 | } | 200 | } |
200 | } | 201 | } |
201 | } | 202 | } |
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index 8ff15e5d486b..f5f793d92415 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c | |||
@@ -131,7 +131,7 @@ static enum hrtimer_restart ntp_leap_second(struct hrtimer *timer) | |||
131 | { | 131 | { |
132 | enum hrtimer_restart res = HRTIMER_NORESTART; | 132 | enum hrtimer_restart res = HRTIMER_NORESTART; |
133 | 133 | ||
134 | write_seqlock_irq(&xtime_lock); | 134 | write_seqlock(&xtime_lock); |
135 | 135 | ||
136 | switch (time_state) { | 136 | switch (time_state) { |
137 | case TIME_OK: | 137 | case TIME_OK: |
@@ -164,7 +164,7 @@ static enum hrtimer_restart ntp_leap_second(struct hrtimer *timer) | |||
164 | } | 164 | } |
165 | update_vsyscall(&xtime, clock); | 165 | update_vsyscall(&xtime, clock); |
166 | 166 | ||
167 | write_sequnlock_irq(&xtime_lock); | 167 | write_sequnlock(&xtime_lock); |
168 | 168 | ||
169 | return res; | 169 | return res; |
170 | } | 170 | } |
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index f98a1b7b16e9..118a3b3b3f9a 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c | |||
@@ -28,7 +28,9 @@ | |||
28 | */ | 28 | */ |
29 | 29 | ||
30 | struct tick_device tick_broadcast_device; | 30 | struct tick_device tick_broadcast_device; |
31 | static cpumask_t tick_broadcast_mask; | 31 | /* FIXME: Use cpumask_var_t. */ |
32 | static DECLARE_BITMAP(tick_broadcast_mask, NR_CPUS); | ||
33 | static DECLARE_BITMAP(tmpmask, NR_CPUS); | ||
32 | static DEFINE_SPINLOCK(tick_broadcast_lock); | 34 | static DEFINE_SPINLOCK(tick_broadcast_lock); |
33 | static int tick_broadcast_force; | 35 | static int tick_broadcast_force; |
34 | 36 | ||
@@ -46,9 +48,9 @@ struct tick_device *tick_get_broadcast_device(void) | |||
46 | return &tick_broadcast_device; | 48 | return &tick_broadcast_device; |
47 | } | 49 | } |
48 | 50 | ||
49 | cpumask_t *tick_get_broadcast_mask(void) | 51 | struct cpumask *tick_get_broadcast_mask(void) |
50 | { | 52 | { |
51 | return &tick_broadcast_mask; | 53 | return to_cpumask(tick_broadcast_mask); |
52 | } | 54 | } |
53 | 55 | ||
54 | /* | 56 | /* |
@@ -72,7 +74,7 @@ int tick_check_broadcast_device(struct clock_event_device *dev) | |||
72 | 74 | ||
73 | clockevents_exchange_device(NULL, dev); | 75 | clockevents_exchange_device(NULL, dev); |
74 | tick_broadcast_device.evtdev = dev; | 76 | tick_broadcast_device.evtdev = dev; |
75 | if (!cpus_empty(tick_broadcast_mask)) | 77 | if (!cpumask_empty(tick_get_broadcast_mask())) |
76 | tick_broadcast_start_periodic(dev); | 78 | tick_broadcast_start_periodic(dev); |
77 | return 1; | 79 | return 1; |
78 | } | 80 | } |
@@ -104,7 +106,7 @@ int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) | |||
104 | */ | 106 | */ |
105 | if (!tick_device_is_functional(dev)) { | 107 | if (!tick_device_is_functional(dev)) { |
106 | dev->event_handler = tick_handle_periodic; | 108 | dev->event_handler = tick_handle_periodic; |
107 | cpu_set(cpu, tick_broadcast_mask); | 109 | cpumask_set_cpu(cpu, tick_get_broadcast_mask()); |
108 | tick_broadcast_start_periodic(tick_broadcast_device.evtdev); | 110 | tick_broadcast_start_periodic(tick_broadcast_device.evtdev); |
109 | ret = 1; | 111 | ret = 1; |
110 | } else { | 112 | } else { |
@@ -116,7 +118,7 @@ int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) | |||
116 | if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) { | 118 | if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) { |
117 | int cpu = smp_processor_id(); | 119 | int cpu = smp_processor_id(); |
118 | 120 | ||
119 | cpu_clear(cpu, tick_broadcast_mask); | 121 | cpumask_clear_cpu(cpu, tick_get_broadcast_mask()); |
120 | tick_broadcast_clear_oneshot(cpu); | 122 | tick_broadcast_clear_oneshot(cpu); |
121 | } | 123 | } |
122 | } | 124 | } |
@@ -125,9 +127,9 @@ int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) | |||
125 | } | 127 | } |
126 | 128 | ||
127 | /* | 129 | /* |
128 | * Broadcast the event to the cpus, which are set in the mask | 130 | * Broadcast the event to the cpus, which are set in the mask (mangled). |
129 | */ | 131 | */ |
130 | static void tick_do_broadcast(cpumask_t mask) | 132 | static void tick_do_broadcast(struct cpumask *mask) |
131 | { | 133 | { |
132 | int cpu = smp_processor_id(); | 134 | int cpu = smp_processor_id(); |
133 | struct tick_device *td; | 135 | struct tick_device *td; |
@@ -135,21 +137,20 @@ static void tick_do_broadcast(cpumask_t mask) | |||
135 | /* | 137 | /* |
136 | * Check, if the current cpu is in the mask | 138 | * Check, if the current cpu is in the mask |
137 | */ | 139 | */ |
138 | if (cpu_isset(cpu, mask)) { | 140 | if (cpumask_test_cpu(cpu, mask)) { |
139 | cpu_clear(cpu, mask); | 141 | cpumask_clear_cpu(cpu, mask); |
140 | td = &per_cpu(tick_cpu_device, cpu); | 142 | td = &per_cpu(tick_cpu_device, cpu); |
141 | td->evtdev->event_handler(td->evtdev); | 143 | td->evtdev->event_handler(td->evtdev); |
142 | } | 144 | } |
143 | 145 | ||
144 | if (!cpus_empty(mask)) { | 146 | if (!cpumask_empty(mask)) { |
145 | /* | 147 | /* |
146 | * It might be necessary to actually check whether the devices | 148 | * It might be necessary to actually check whether the devices |
147 | * have different broadcast functions. For now, just use the | 149 | * have different broadcast functions. For now, just use the |
148 | * one of the first device. This works as long as we have this | 150 | * one of the first device. This works as long as we have this |
149 | * misfeature only on x86 (lapic) | 151 | * misfeature only on x86 (lapic) |
150 | */ | 152 | */ |
151 | cpu = first_cpu(mask); | 153 | td = &per_cpu(tick_cpu_device, cpumask_first(mask)); |
152 | td = &per_cpu(tick_cpu_device, cpu); | ||
153 | td->evtdev->broadcast(mask); | 154 | td->evtdev->broadcast(mask); |
154 | } | 155 | } |
155 | } | 156 | } |
@@ -160,12 +161,11 @@ static void tick_do_broadcast(cpumask_t mask) | |||
160 | */ | 161 | */ |
161 | static void tick_do_periodic_broadcast(void) | 162 | static void tick_do_periodic_broadcast(void) |
162 | { | 163 | { |
163 | cpumask_t mask; | ||
164 | |||
165 | spin_lock(&tick_broadcast_lock); | 164 | spin_lock(&tick_broadcast_lock); |
166 | 165 | ||
167 | cpus_and(mask, cpu_online_map, tick_broadcast_mask); | 166 | cpumask_and(to_cpumask(tmpmask), |
168 | tick_do_broadcast(mask); | 167 | cpu_online_mask, tick_get_broadcast_mask()); |
168 | tick_do_broadcast(to_cpumask(tmpmask)); | ||
169 | 169 | ||
170 | spin_unlock(&tick_broadcast_lock); | 170 | spin_unlock(&tick_broadcast_lock); |
171 | } | 171 | } |
@@ -228,13 +228,13 @@ static void tick_do_broadcast_on_off(void *why) | |||
228 | if (!tick_device_is_functional(dev)) | 228 | if (!tick_device_is_functional(dev)) |
229 | goto out; | 229 | goto out; |
230 | 230 | ||
231 | bc_stopped = cpus_empty(tick_broadcast_mask); | 231 | bc_stopped = cpumask_empty(tick_get_broadcast_mask()); |
232 | 232 | ||
233 | switch (*reason) { | 233 | switch (*reason) { |
234 | case CLOCK_EVT_NOTIFY_BROADCAST_ON: | 234 | case CLOCK_EVT_NOTIFY_BROADCAST_ON: |
235 | case CLOCK_EVT_NOTIFY_BROADCAST_FORCE: | 235 | case CLOCK_EVT_NOTIFY_BROADCAST_FORCE: |
236 | if (!cpu_isset(cpu, tick_broadcast_mask)) { | 236 | if (!cpumask_test_cpu(cpu, tick_get_broadcast_mask())) { |
237 | cpu_set(cpu, tick_broadcast_mask); | 237 | cpumask_set_cpu(cpu, tick_get_broadcast_mask()); |
238 | if (tick_broadcast_device.mode == | 238 | if (tick_broadcast_device.mode == |
239 | TICKDEV_MODE_PERIODIC) | 239 | TICKDEV_MODE_PERIODIC) |
240 | clockevents_shutdown(dev); | 240 | clockevents_shutdown(dev); |
@@ -244,8 +244,8 @@ static void tick_do_broadcast_on_off(void *why) | |||
244 | break; | 244 | break; |
245 | case CLOCK_EVT_NOTIFY_BROADCAST_OFF: | 245 | case CLOCK_EVT_NOTIFY_BROADCAST_OFF: |
246 | if (!tick_broadcast_force && | 246 | if (!tick_broadcast_force && |
247 | cpu_isset(cpu, tick_broadcast_mask)) { | 247 | cpumask_test_cpu(cpu, tick_get_broadcast_mask())) { |
248 | cpu_clear(cpu, tick_broadcast_mask); | 248 | cpumask_clear_cpu(cpu, tick_get_broadcast_mask()); |
249 | if (tick_broadcast_device.mode == | 249 | if (tick_broadcast_device.mode == |
250 | TICKDEV_MODE_PERIODIC) | 250 | TICKDEV_MODE_PERIODIC) |
251 | tick_setup_periodic(dev, 0); | 251 | tick_setup_periodic(dev, 0); |
@@ -253,7 +253,7 @@ static void tick_do_broadcast_on_off(void *why) | |||
253 | break; | 253 | break; |
254 | } | 254 | } |
255 | 255 | ||
256 | if (cpus_empty(tick_broadcast_mask)) { | 256 | if (cpumask_empty(tick_get_broadcast_mask())) { |
257 | if (!bc_stopped) | 257 | if (!bc_stopped) |
258 | clockevents_shutdown(bc); | 258 | clockevents_shutdown(bc); |
259 | } else if (bc_stopped) { | 259 | } else if (bc_stopped) { |
@@ -272,7 +272,7 @@ out: | |||
272 | */ | 272 | */ |
273 | void tick_broadcast_on_off(unsigned long reason, int *oncpu) | 273 | void tick_broadcast_on_off(unsigned long reason, int *oncpu) |
274 | { | 274 | { |
275 | if (!cpu_isset(*oncpu, cpu_online_map)) | 275 | if (!cpumask_test_cpu(*oncpu, cpu_online_mask)) |
276 | printk(KERN_ERR "tick-broadcast: ignoring broadcast for " | 276 | printk(KERN_ERR "tick-broadcast: ignoring broadcast for " |
277 | "offline CPU #%d\n", *oncpu); | 277 | "offline CPU #%d\n", *oncpu); |
278 | else | 278 | else |
@@ -303,10 +303,10 @@ void tick_shutdown_broadcast(unsigned int *cpup) | |||
303 | spin_lock_irqsave(&tick_broadcast_lock, flags); | 303 | spin_lock_irqsave(&tick_broadcast_lock, flags); |
304 | 304 | ||
305 | bc = tick_broadcast_device.evtdev; | 305 | bc = tick_broadcast_device.evtdev; |
306 | cpu_clear(cpu, tick_broadcast_mask); | 306 | cpumask_clear_cpu(cpu, tick_get_broadcast_mask()); |
307 | 307 | ||
308 | if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) { | 308 | if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) { |
309 | if (bc && cpus_empty(tick_broadcast_mask)) | 309 | if (bc && cpumask_empty(tick_get_broadcast_mask())) |
310 | clockevents_shutdown(bc); | 310 | clockevents_shutdown(bc); |
311 | } | 311 | } |
312 | 312 | ||
@@ -342,10 +342,10 @@ int tick_resume_broadcast(void) | |||
342 | 342 | ||
343 | switch (tick_broadcast_device.mode) { | 343 | switch (tick_broadcast_device.mode) { |
344 | case TICKDEV_MODE_PERIODIC: | 344 | case TICKDEV_MODE_PERIODIC: |
345 | if(!cpus_empty(tick_broadcast_mask)) | 345 | if (!cpumask_empty(tick_get_broadcast_mask())) |
346 | tick_broadcast_start_periodic(bc); | 346 | tick_broadcast_start_periodic(bc); |
347 | broadcast = cpu_isset(smp_processor_id(), | 347 | broadcast = cpumask_test_cpu(smp_processor_id(), |
348 | tick_broadcast_mask); | 348 | tick_get_broadcast_mask()); |
349 | break; | 349 | break; |
350 | case TICKDEV_MODE_ONESHOT: | 350 | case TICKDEV_MODE_ONESHOT: |
351 | broadcast = tick_resume_broadcast_oneshot(bc); | 351 | broadcast = tick_resume_broadcast_oneshot(bc); |
@@ -360,14 +360,15 @@ int tick_resume_broadcast(void) | |||
360 | 360 | ||
361 | #ifdef CONFIG_TICK_ONESHOT | 361 | #ifdef CONFIG_TICK_ONESHOT |
362 | 362 | ||
363 | static cpumask_t tick_broadcast_oneshot_mask; | 363 | /* FIXME: use cpumask_var_t. */ |
364 | static DECLARE_BITMAP(tick_broadcast_oneshot_mask, NR_CPUS); | ||
364 | 365 | ||
365 | /* | 366 | /* |
366 | * Debugging: see timer_list.c | 367 | * Exposed for debugging: see timer_list.c |
367 | */ | 368 | */ |
368 | cpumask_t *tick_get_broadcast_oneshot_mask(void) | 369 | struct cpumask *tick_get_broadcast_oneshot_mask(void) |
369 | { | 370 | { |
370 | return &tick_broadcast_oneshot_mask; | 371 | return to_cpumask(tick_broadcast_oneshot_mask); |
371 | } | 372 | } |
372 | 373 | ||
373 | static int tick_broadcast_set_event(ktime_t expires, int force) | 374 | static int tick_broadcast_set_event(ktime_t expires, int force) |
@@ -389,7 +390,7 @@ int tick_resume_broadcast_oneshot(struct clock_event_device *bc) | |||
389 | */ | 390 | */ |
390 | void tick_check_oneshot_broadcast(int cpu) | 391 | void tick_check_oneshot_broadcast(int cpu) |
391 | { | 392 | { |
392 | if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) { | 393 | if (cpumask_test_cpu(cpu, to_cpumask(tick_broadcast_oneshot_mask))) { |
393 | struct tick_device *td = &per_cpu(tick_cpu_device, cpu); | 394 | struct tick_device *td = &per_cpu(tick_cpu_device, cpu); |
394 | 395 | ||
395 | clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT); | 396 | clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT); |
@@ -402,7 +403,6 @@ void tick_check_oneshot_broadcast(int cpu) | |||
402 | static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) | 403 | static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) |
403 | { | 404 | { |
404 | struct tick_device *td; | 405 | struct tick_device *td; |
405 | cpumask_t mask; | ||
406 | ktime_t now, next_event; | 406 | ktime_t now, next_event; |
407 | int cpu; | 407 | int cpu; |
408 | 408 | ||
@@ -410,13 +410,13 @@ static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) | |||
410 | again: | 410 | again: |
411 | dev->next_event.tv64 = KTIME_MAX; | 411 | dev->next_event.tv64 = KTIME_MAX; |
412 | next_event.tv64 = KTIME_MAX; | 412 | next_event.tv64 = KTIME_MAX; |
413 | mask = CPU_MASK_NONE; | 413 | cpumask_clear(to_cpumask(tmpmask)); |
414 | now = ktime_get(); | 414 | now = ktime_get(); |
415 | /* Find all expired events */ | 415 | /* Find all expired events */ |
416 | for_each_cpu_mask_nr(cpu, tick_broadcast_oneshot_mask) { | 416 | for_each_cpu(cpu, tick_get_broadcast_oneshot_mask()) { |
417 | td = &per_cpu(tick_cpu_device, cpu); | 417 | td = &per_cpu(tick_cpu_device, cpu); |
418 | if (td->evtdev->next_event.tv64 <= now.tv64) | 418 | if (td->evtdev->next_event.tv64 <= now.tv64) |
419 | cpu_set(cpu, mask); | 419 | cpumask_set_cpu(cpu, to_cpumask(tmpmask)); |
420 | else if (td->evtdev->next_event.tv64 < next_event.tv64) | 420 | else if (td->evtdev->next_event.tv64 < next_event.tv64) |
421 | next_event.tv64 = td->evtdev->next_event.tv64; | 421 | next_event.tv64 = td->evtdev->next_event.tv64; |
422 | } | 422 | } |
@@ -424,7 +424,7 @@ again: | |||
424 | /* | 424 | /* |
425 | * Wakeup the cpus which have an expired event. | 425 | * Wakeup the cpus which have an expired event. |
426 | */ | 426 | */ |
427 | tick_do_broadcast(mask); | 427 | tick_do_broadcast(to_cpumask(tmpmask)); |
428 | 428 | ||
429 | /* | 429 | /* |
430 | * Two reasons for reprogram: | 430 | * Two reasons for reprogram: |
@@ -476,15 +476,16 @@ void tick_broadcast_oneshot_control(unsigned long reason) | |||
476 | goto out; | 476 | goto out; |
477 | 477 | ||
478 | if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) { | 478 | if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) { |
479 | if (!cpu_isset(cpu, tick_broadcast_oneshot_mask)) { | 479 | if (!cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) { |
480 | cpu_set(cpu, tick_broadcast_oneshot_mask); | 480 | cpumask_set_cpu(cpu, tick_get_broadcast_oneshot_mask()); |
481 | clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); | 481 | clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); |
482 | if (dev->next_event.tv64 < bc->next_event.tv64) | 482 | if (dev->next_event.tv64 < bc->next_event.tv64) |
483 | tick_broadcast_set_event(dev->next_event, 1); | 483 | tick_broadcast_set_event(dev->next_event, 1); |
484 | } | 484 | } |
485 | } else { | 485 | } else { |
486 | if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) { | 486 | if (cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) { |
487 | cpu_clear(cpu, tick_broadcast_oneshot_mask); | 487 | cpumask_clear_cpu(cpu, |
488 | tick_get_broadcast_oneshot_mask()); | ||
488 | clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT); | 489 | clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT); |
489 | if (dev->next_event.tv64 != KTIME_MAX) | 490 | if (dev->next_event.tv64 != KTIME_MAX) |
490 | tick_program_event(dev->next_event, 1); | 491 | tick_program_event(dev->next_event, 1); |
@@ -502,15 +503,16 @@ out: | |||
502 | */ | 503 | */ |
503 | static void tick_broadcast_clear_oneshot(int cpu) | 504 | static void tick_broadcast_clear_oneshot(int cpu) |
504 | { | 505 | { |
505 | cpu_clear(cpu, tick_broadcast_oneshot_mask); | 506 | cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask()); |
506 | } | 507 | } |
507 | 508 | ||
508 | static void tick_broadcast_init_next_event(cpumask_t *mask, ktime_t expires) | 509 | static void tick_broadcast_init_next_event(struct cpumask *mask, |
510 | ktime_t expires) | ||
509 | { | 511 | { |
510 | struct tick_device *td; | 512 | struct tick_device *td; |
511 | int cpu; | 513 | int cpu; |
512 | 514 | ||
513 | for_each_cpu_mask_nr(cpu, *mask) { | 515 | for_each_cpu(cpu, mask) { |
514 | td = &per_cpu(tick_cpu_device, cpu); | 516 | td = &per_cpu(tick_cpu_device, cpu); |
515 | if (td->evtdev) | 517 | if (td->evtdev) |
516 | td->evtdev->next_event = expires; | 518 | td->evtdev->next_event = expires; |
@@ -526,7 +528,6 @@ void tick_broadcast_setup_oneshot(struct clock_event_device *bc) | |||
526 | if (bc->event_handler != tick_handle_oneshot_broadcast) { | 528 | if (bc->event_handler != tick_handle_oneshot_broadcast) { |
527 | int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC; | 529 | int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC; |
528 | int cpu = smp_processor_id(); | 530 | int cpu = smp_processor_id(); |
529 | cpumask_t mask; | ||
530 | 531 | ||
531 | bc->event_handler = tick_handle_oneshot_broadcast; | 532 | bc->event_handler = tick_handle_oneshot_broadcast; |
532 | clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); | 533 | clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); |
@@ -540,13 +541,15 @@ void tick_broadcast_setup_oneshot(struct clock_event_device *bc) | |||
540 | * oneshot_mask bits for those and program the | 541 | * oneshot_mask bits for those and program the |
541 | * broadcast device to fire. | 542 | * broadcast device to fire. |
542 | */ | 543 | */ |
543 | mask = tick_broadcast_mask; | 544 | cpumask_copy(to_cpumask(tmpmask), tick_get_broadcast_mask()); |
544 | cpu_clear(cpu, mask); | 545 | cpumask_clear_cpu(cpu, to_cpumask(tmpmask)); |
545 | cpus_or(tick_broadcast_oneshot_mask, | 546 | cpumask_or(tick_get_broadcast_oneshot_mask(), |
546 | tick_broadcast_oneshot_mask, mask); | 547 | tick_get_broadcast_oneshot_mask(), |
547 | 548 | to_cpumask(tmpmask)); | |
548 | if (was_periodic && !cpus_empty(mask)) { | 549 | |
549 | tick_broadcast_init_next_event(&mask, tick_next_period); | 550 | if (was_periodic && !cpumask_empty(to_cpumask(tmpmask))) { |
551 | tick_broadcast_init_next_event(to_cpumask(tmpmask), | ||
552 | tick_next_period); | ||
550 | tick_broadcast_set_event(tick_next_period, 1); | 553 | tick_broadcast_set_event(tick_next_period, 1); |
551 | } else | 554 | } else |
552 | bc->next_event.tv64 = KTIME_MAX; | 555 | bc->next_event.tv64 = KTIME_MAX; |
@@ -585,7 +588,7 @@ void tick_shutdown_broadcast_oneshot(unsigned int *cpup) | |||
585 | * Clear the broadcast mask flag for the dead cpu, but do not | 588 | * Clear the broadcast mask flag for the dead cpu, but do not |
586 | * stop the broadcast device! | 589 | * stop the broadcast device! |
587 | */ | 590 | */ |
588 | cpu_clear(cpu, tick_broadcast_oneshot_mask); | 591 | cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask()); |
589 | 592 | ||
590 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); | 593 | spin_unlock_irqrestore(&tick_broadcast_lock, flags); |
591 | } | 594 | } |
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index df12434b43ca..63e05d423a09 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c | |||
@@ -136,7 +136,7 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast) | |||
136 | */ | 136 | */ |
137 | static void tick_setup_device(struct tick_device *td, | 137 | static void tick_setup_device(struct tick_device *td, |
138 | struct clock_event_device *newdev, int cpu, | 138 | struct clock_event_device *newdev, int cpu, |
139 | const cpumask_t *cpumask) | 139 | const struct cpumask *cpumask) |
140 | { | 140 | { |
141 | ktime_t next_event; | 141 | ktime_t next_event; |
142 | void (*handler)(struct clock_event_device *) = NULL; | 142 | void (*handler)(struct clock_event_device *) = NULL; |
@@ -171,8 +171,8 @@ static void tick_setup_device(struct tick_device *td, | |||
171 | * When the device is not per cpu, pin the interrupt to the | 171 | * When the device is not per cpu, pin the interrupt to the |
172 | * current cpu: | 172 | * current cpu: |
173 | */ | 173 | */ |
174 | if (!cpus_equal(newdev->cpumask, *cpumask)) | 174 | if (!cpumask_equal(newdev->cpumask, cpumask)) |
175 | irq_set_affinity(newdev->irq, *cpumask); | 175 | irq_set_affinity(newdev->irq, cpumask); |
176 | 176 | ||
177 | /* | 177 | /* |
178 | * When global broadcasting is active, check if the current | 178 | * When global broadcasting is active, check if the current |
@@ -202,14 +202,14 @@ static int tick_check_new_device(struct clock_event_device *newdev) | |||
202 | spin_lock_irqsave(&tick_device_lock, flags); | 202 | spin_lock_irqsave(&tick_device_lock, flags); |
203 | 203 | ||
204 | cpu = smp_processor_id(); | 204 | cpu = smp_processor_id(); |
205 | if (!cpu_isset(cpu, newdev->cpumask)) | 205 | if (!cpumask_test_cpu(cpu, newdev->cpumask)) |
206 | goto out_bc; | 206 | goto out_bc; |
207 | 207 | ||
208 | td = &per_cpu(tick_cpu_device, cpu); | 208 | td = &per_cpu(tick_cpu_device, cpu); |
209 | curdev = td->evtdev; | 209 | curdev = td->evtdev; |
210 | 210 | ||
211 | /* cpu local device ? */ | 211 | /* cpu local device ? */ |
212 | if (!cpus_equal(newdev->cpumask, cpumask_of_cpu(cpu))) { | 212 | if (!cpumask_equal(newdev->cpumask, cpumask_of(cpu))) { |
213 | 213 | ||
214 | /* | 214 | /* |
215 | * If the cpu affinity of the device interrupt can not | 215 | * If the cpu affinity of the device interrupt can not |
@@ -222,7 +222,7 @@ static int tick_check_new_device(struct clock_event_device *newdev) | |||
222 | * If we have a cpu local device already, do not replace it | 222 | * If we have a cpu local device already, do not replace it |
223 | * by a non cpu local device | 223 | * by a non cpu local device |
224 | */ | 224 | */ |
225 | if (curdev && cpus_equal(curdev->cpumask, cpumask_of_cpu(cpu))) | 225 | if (curdev && cpumask_equal(curdev->cpumask, cpumask_of(cpu))) |
226 | goto out_bc; | 226 | goto out_bc; |
227 | } | 227 | } |
228 | 228 | ||
@@ -254,7 +254,7 @@ static int tick_check_new_device(struct clock_event_device *newdev) | |||
254 | curdev = NULL; | 254 | curdev = NULL; |
255 | } | 255 | } |
256 | clockevents_exchange_device(curdev, newdev); | 256 | clockevents_exchange_device(curdev, newdev); |
257 | tick_setup_device(td, newdev, cpu, &cpumask_of_cpu(cpu)); | 257 | tick_setup_device(td, newdev, cpu, cpumask_of(cpu)); |
258 | if (newdev->features & CLOCK_EVT_FEAT_ONESHOT) | 258 | if (newdev->features & CLOCK_EVT_FEAT_ONESHOT) |
259 | tick_oneshot_notify(); | 259 | tick_oneshot_notify(); |
260 | 260 | ||
@@ -299,9 +299,9 @@ static void tick_shutdown(unsigned int *cpup) | |||
299 | } | 299 | } |
300 | /* Transfer the do_timer job away from this cpu */ | 300 | /* Transfer the do_timer job away from this cpu */ |
301 | if (*cpup == tick_do_timer_cpu) { | 301 | if (*cpup == tick_do_timer_cpu) { |
302 | int cpu = first_cpu(cpu_online_map); | 302 | int cpu = cpumask_first(cpu_online_mask); |
303 | 303 | ||
304 | tick_do_timer_cpu = (cpu != NR_CPUS) ? cpu : | 304 | tick_do_timer_cpu = (cpu < nr_cpu_ids) ? cpu : |
305 | TICK_DO_TIMER_NONE; | 305 | TICK_DO_TIMER_NONE; |
306 | } | 306 | } |
307 | spin_unlock_irqrestore(&tick_device_lock, flags); | 307 | spin_unlock_irqrestore(&tick_device_lock, flags); |
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 342fc9ccab46..1b6c05bd0d0a 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
@@ -144,7 +144,7 @@ void tick_nohz_update_jiffies(void) | |||
144 | if (!ts->tick_stopped) | 144 | if (!ts->tick_stopped) |
145 | return; | 145 | return; |
146 | 146 | ||
147 | cpu_clear(cpu, nohz_cpu_mask); | 147 | cpumask_clear_cpu(cpu, nohz_cpu_mask); |
148 | now = ktime_get(); | 148 | now = ktime_get(); |
149 | ts->idle_waketime = now; | 149 | ts->idle_waketime = now; |
150 | 150 | ||
@@ -247,7 +247,7 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
247 | if (need_resched()) | 247 | if (need_resched()) |
248 | goto end; | 248 | goto end; |
249 | 249 | ||
250 | if (unlikely(local_softirq_pending())) { | 250 | if (unlikely(local_softirq_pending() && cpu_online(cpu))) { |
251 | static int ratelimit; | 251 | static int ratelimit; |
252 | 252 | ||
253 | if (ratelimit < 10) { | 253 | if (ratelimit < 10) { |
@@ -282,8 +282,31 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
282 | /* Schedule the tick, if we are at least one jiffie off */ | 282 | /* Schedule the tick, if we are at least one jiffie off */ |
283 | if ((long)delta_jiffies >= 1) { | 283 | if ((long)delta_jiffies >= 1) { |
284 | 284 | ||
285 | /* | ||
286 | * calculate the expiry time for the next timer wheel | ||
287 | * timer | ||
288 | */ | ||
289 | expires = ktime_add_ns(last_update, tick_period.tv64 * | ||
290 | delta_jiffies); | ||
291 | |||
292 | /* | ||
293 | * If this cpu is the one which updates jiffies, then | ||
294 | * give up the assignment and let it be taken by the | ||
295 | * cpu which runs the tick timer next, which might be | ||
296 | * this cpu as well. If we don't drop this here the | ||
297 | * jiffies might be stale and do_timer() never | ||
298 | * invoked. | ||
299 | */ | ||
300 | if (cpu == tick_do_timer_cpu) | ||
301 | tick_do_timer_cpu = TICK_DO_TIMER_NONE; | ||
302 | |||
285 | if (delta_jiffies > 1) | 303 | if (delta_jiffies > 1) |
286 | cpu_set(cpu, nohz_cpu_mask); | 304 | cpumask_set_cpu(cpu, nohz_cpu_mask); |
305 | |||
306 | /* Skip reprogram of event if its not changed */ | ||
307 | if (ts->tick_stopped && ktime_equal(expires, dev->next_event)) | ||
308 | goto out; | ||
309 | |||
287 | /* | 310 | /* |
288 | * nohz_stop_sched_tick can be called several times before | 311 | * nohz_stop_sched_tick can be called several times before |
289 | * the nohz_restart_sched_tick is called. This happens when | 312 | * the nohz_restart_sched_tick is called. This happens when |
@@ -296,7 +319,7 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
296 | /* | 319 | /* |
297 | * sched tick not stopped! | 320 | * sched tick not stopped! |
298 | */ | 321 | */ |
299 | cpu_clear(cpu, nohz_cpu_mask); | 322 | cpumask_clear_cpu(cpu, nohz_cpu_mask); |
300 | goto out; | 323 | goto out; |
301 | } | 324 | } |
302 | 325 | ||
@@ -306,17 +329,6 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
306 | rcu_enter_nohz(); | 329 | rcu_enter_nohz(); |
307 | } | 330 | } |
308 | 331 | ||
309 | /* | ||
310 | * If this cpu is the one which updates jiffies, then | ||
311 | * give up the assignment and let it be taken by the | ||
312 | * cpu which runs the tick timer next, which might be | ||
313 | * this cpu as well. If we don't drop this here the | ||
314 | * jiffies might be stale and do_timer() never | ||
315 | * invoked. | ||
316 | */ | ||
317 | if (cpu == tick_do_timer_cpu) | ||
318 | tick_do_timer_cpu = TICK_DO_TIMER_NONE; | ||
319 | |||
320 | ts->idle_sleeps++; | 332 | ts->idle_sleeps++; |
321 | 333 | ||
322 | /* | 334 | /* |
@@ -332,12 +344,7 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
332 | goto out; | 344 | goto out; |
333 | } | 345 | } |
334 | 346 | ||
335 | /* | 347 | /* Mark expiries */ |
336 | * calculate the expiry time for the next timer wheel | ||
337 | * timer | ||
338 | */ | ||
339 | expires = ktime_add_ns(last_update, tick_period.tv64 * | ||
340 | delta_jiffies); | ||
341 | ts->idle_expires = expires; | 348 | ts->idle_expires = expires; |
342 | 349 | ||
343 | if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { | 350 | if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { |
@@ -354,7 +361,7 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
354 | * softirq. | 361 | * softirq. |
355 | */ | 362 | */ |
356 | tick_do_update_jiffies64(ktime_get()); | 363 | tick_do_update_jiffies64(ktime_get()); |
357 | cpu_clear(cpu, nohz_cpu_mask); | 364 | cpumask_clear_cpu(cpu, nohz_cpu_mask); |
358 | } | 365 | } |
359 | raise_softirq_irqoff(TIMER_SOFTIRQ); | 366 | raise_softirq_irqoff(TIMER_SOFTIRQ); |
360 | out: | 367 | out: |
@@ -412,7 +419,9 @@ void tick_nohz_restart_sched_tick(void) | |||
412 | { | 419 | { |
413 | int cpu = smp_processor_id(); | 420 | int cpu = smp_processor_id(); |
414 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); | 421 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); |
422 | #ifndef CONFIG_VIRT_CPU_ACCOUNTING | ||
415 | unsigned long ticks; | 423 | unsigned long ticks; |
424 | #endif | ||
416 | ktime_t now; | 425 | ktime_t now; |
417 | 426 | ||
418 | local_irq_disable(); | 427 | local_irq_disable(); |
@@ -432,8 +441,9 @@ void tick_nohz_restart_sched_tick(void) | |||
432 | select_nohz_load_balancer(0); | 441 | select_nohz_load_balancer(0); |
433 | now = ktime_get(); | 442 | now = ktime_get(); |
434 | tick_do_update_jiffies64(now); | 443 | tick_do_update_jiffies64(now); |
435 | cpu_clear(cpu, nohz_cpu_mask); | 444 | cpumask_clear_cpu(cpu, nohz_cpu_mask); |
436 | 445 | ||
446 | #ifndef CONFIG_VIRT_CPU_ACCOUNTING | ||
437 | /* | 447 | /* |
438 | * We stopped the tick in idle. Update process times would miss the | 448 | * We stopped the tick in idle. Update process times would miss the |
439 | * time we slept as update_process_times does only a 1 tick | 449 | * time we slept as update_process_times does only a 1 tick |
@@ -443,12 +453,9 @@ void tick_nohz_restart_sched_tick(void) | |||
443 | /* | 453 | /* |
444 | * We might be one off. Do not randomly account a huge number of ticks! | 454 | * We might be one off. Do not randomly account a huge number of ticks! |
445 | */ | 455 | */ |
446 | if (ticks && ticks < LONG_MAX) { | 456 | if (ticks && ticks < LONG_MAX) |
447 | add_preempt_count(HARDIRQ_OFFSET); | 457 | account_idle_ticks(ticks); |
448 | account_system_time(current, HARDIRQ_OFFSET, | 458 | #endif |
449 | jiffies_to_cputime(ticks)); | ||
450 | sub_preempt_count(HARDIRQ_OFFSET); | ||
451 | } | ||
452 | 459 | ||
453 | touch_softlockup_watchdog(); | 460 | touch_softlockup_watchdog(); |
454 | /* | 461 | /* |
@@ -681,7 +688,6 @@ void tick_setup_sched_timer(void) | |||
681 | */ | 688 | */ |
682 | hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); | 689 | hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); |
683 | ts->sched_timer.function = tick_sched_timer; | 690 | ts->sched_timer.function = tick_sched_timer; |
684 | ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU; | ||
685 | 691 | ||
686 | /* Get the next period (per cpu) */ | 692 | /* Get the next period (per cpu) */ |
687 | hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); | 693 | hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); |