diff options
| -rw-r--r-- | drivers/clocksource/i8253.c | 2 | ||||
| -rw-r--r-- | include/linux/jiffies.h | 3 | ||||
| -rw-r--r-- | kernel/time/jiffies.c | 6 | ||||
| -rw-r--r-- | kernel/time/tick-common.c | 8 | ||||
| -rw-r--r-- | kernel/time/tick-internal.h | 1 | ||||
| -rw-r--r-- | kernel/time/tick-sched.c | 22 | ||||
| -rw-r--r-- | kernel/time/timekeeping.c | 14 |
7 files changed, 25 insertions, 31 deletions
diff --git a/drivers/clocksource/i8253.c b/drivers/clocksource/i8253.c index e7cab2da910f..14ee3efcc404 100644 --- a/drivers/clocksource/i8253.c +++ b/drivers/clocksource/i8253.c | |||
| @@ -35,7 +35,7 @@ static cycle_t i8253_read(struct clocksource *cs) | |||
| 35 | 35 | ||
| 36 | raw_spin_lock_irqsave(&i8253_lock, flags); | 36 | raw_spin_lock_irqsave(&i8253_lock, flags); |
| 37 | /* | 37 | /* |
| 38 | * Although our caller may have the read side of xtime_lock, | 38 | * Although our caller may have the read side of jiffies_lock, |
| 39 | * this is now a seqlock, and we are cheating in this routine | 39 | * this is now a seqlock, and we are cheating in this routine |
| 40 | * by having side effects on state that we cannot undo if | 40 | * by having side effects on state that we cannot undo if |
| 41 | * there is a collision on the seqlock and our caller has to | 41 | * there is a collision on the seqlock and our caller has to |
diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h index 6b87413da9d6..82ed068b1ebe 100644 --- a/include/linux/jiffies.h +++ b/include/linux/jiffies.h | |||
| @@ -70,11 +70,12 @@ extern int register_refined_jiffies(long clock_tick_rate); | |||
| 70 | 70 | ||
| 71 | /* | 71 | /* |
| 72 | * The 64-bit value is not atomic - you MUST NOT read it | 72 | * The 64-bit value is not atomic - you MUST NOT read it |
| 73 | * without sampling the sequence number in xtime_lock. | 73 | * without sampling the sequence number in jiffies_lock. |
| 74 | * get_jiffies_64() will do this for you as appropriate. | 74 | * get_jiffies_64() will do this for you as appropriate. |
| 75 | */ | 75 | */ |
| 76 | extern u64 __jiffy_data jiffies_64; | 76 | extern u64 __jiffy_data jiffies_64; |
| 77 | extern unsigned long volatile __jiffy_data jiffies; | 77 | extern unsigned long volatile __jiffy_data jiffies; |
| 78 | extern seqlock_t jiffies_lock; | ||
| 78 | 79 | ||
| 79 | #if (BITS_PER_LONG < 64) | 80 | #if (BITS_PER_LONG < 64) |
| 80 | u64 get_jiffies_64(void); | 81 | u64 get_jiffies_64(void); |
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index 25f5b2699d37..7a925ba456fb 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c | |||
| @@ -67,6 +67,8 @@ static struct clocksource clocksource_jiffies = { | |||
| 67 | .shift = JIFFIES_SHIFT, | 67 | .shift = JIFFIES_SHIFT, |
| 68 | }; | 68 | }; |
| 69 | 69 | ||
| 70 | __cacheline_aligned_in_smp DEFINE_SEQLOCK(jiffies_lock); | ||
| 71 | |||
| 70 | #if (BITS_PER_LONG < 64) | 72 | #if (BITS_PER_LONG < 64) |
| 71 | u64 get_jiffies_64(void) | 73 | u64 get_jiffies_64(void) |
| 72 | { | 74 | { |
| @@ -74,9 +76,9 @@ u64 get_jiffies_64(void) | |||
| 74 | u64 ret; | 76 | u64 ret; |
| 75 | 77 | ||
| 76 | do { | 78 | do { |
| 77 | seq = read_seqbegin(&xtime_lock); | 79 | seq = read_seqbegin(&jiffies_lock); |
| 78 | ret = jiffies_64; | 80 | ret = jiffies_64; |
| 79 | } while (read_seqretry(&xtime_lock, seq)); | 81 | } while (read_seqretry(&jiffies_lock, seq)); |
| 80 | return ret; | 82 | return ret; |
| 81 | } | 83 | } |
| 82 | EXPORT_SYMBOL(get_jiffies_64); | 84 | EXPORT_SYMBOL(get_jiffies_64); |
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index da6c9ecad4e4..b1600a6973f4 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c | |||
| @@ -63,13 +63,13 @@ int tick_is_oneshot_available(void) | |||
| 63 | static void tick_periodic(int cpu) | 63 | static void tick_periodic(int cpu) |
| 64 | { | 64 | { |
| 65 | if (tick_do_timer_cpu == cpu) { | 65 | if (tick_do_timer_cpu == cpu) { |
| 66 | write_seqlock(&xtime_lock); | 66 | write_seqlock(&jiffies_lock); |
| 67 | 67 | ||
| 68 | /* Keep track of the next tick event */ | 68 | /* Keep track of the next tick event */ |
| 69 | tick_next_period = ktime_add(tick_next_period, tick_period); | 69 | tick_next_period = ktime_add(tick_next_period, tick_period); |
| 70 | 70 | ||
| 71 | do_timer(1); | 71 | do_timer(1); |
| 72 | write_sequnlock(&xtime_lock); | 72 | write_sequnlock(&jiffies_lock); |
| 73 | } | 73 | } |
| 74 | 74 | ||
| 75 | update_process_times(user_mode(get_irq_regs())); | 75 | update_process_times(user_mode(get_irq_regs())); |
| @@ -130,9 +130,9 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast) | |||
| 130 | ktime_t next; | 130 | ktime_t next; |
| 131 | 131 | ||
| 132 | do { | 132 | do { |
| 133 | seq = read_seqbegin(&xtime_lock); | 133 | seq = read_seqbegin(&jiffies_lock); |
| 134 | next = tick_next_period; | 134 | next = tick_next_period; |
| 135 | } while (read_seqretry(&xtime_lock, seq)); | 135 | } while (read_seqretry(&jiffies_lock, seq)); |
| 136 | 136 | ||
| 137 | clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT); | 137 | clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT); |
| 138 | 138 | ||
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index 4e265b901fed..cf3e59ed6dc0 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h | |||
| @@ -141,4 +141,3 @@ static inline int tick_device_is_functional(struct clock_event_device *dev) | |||
| 141 | #endif | 141 | #endif |
| 142 | 142 | ||
| 143 | extern void do_timer(unsigned long ticks); | 143 | extern void do_timer(unsigned long ticks); |
| 144 | extern seqlock_t xtime_lock; | ||
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index a40260885265..a678046c3e5e 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
| @@ -31,7 +31,7 @@ | |||
| 31 | static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched); | 31 | static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched); |
| 32 | 32 | ||
| 33 | /* | 33 | /* |
| 34 | * The time, when the last jiffy update happened. Protected by xtime_lock. | 34 | * The time, when the last jiffy update happened. Protected by jiffies_lock. |
| 35 | */ | 35 | */ |
| 36 | static ktime_t last_jiffies_update; | 36 | static ktime_t last_jiffies_update; |
| 37 | 37 | ||
| @@ -49,14 +49,14 @@ static void tick_do_update_jiffies64(ktime_t now) | |||
| 49 | ktime_t delta; | 49 | ktime_t delta; |
| 50 | 50 | ||
| 51 | /* | 51 | /* |
| 52 | * Do a quick check without holding xtime_lock: | 52 | * Do a quick check without holding jiffies_lock: |
| 53 | */ | 53 | */ |
| 54 | delta = ktime_sub(now, last_jiffies_update); | 54 | delta = ktime_sub(now, last_jiffies_update); |
| 55 | if (delta.tv64 < tick_period.tv64) | 55 | if (delta.tv64 < tick_period.tv64) |
| 56 | return; | 56 | return; |
| 57 | 57 | ||
| 58 | /* Reevalute with xtime_lock held */ | 58 | /* Reevalute with jiffies_lock held */ |
| 59 | write_seqlock(&xtime_lock); | 59 | write_seqlock(&jiffies_lock); |
| 60 | 60 | ||
| 61 | delta = ktime_sub(now, last_jiffies_update); | 61 | delta = ktime_sub(now, last_jiffies_update); |
| 62 | if (delta.tv64 >= tick_period.tv64) { | 62 | if (delta.tv64 >= tick_period.tv64) { |
| @@ -79,7 +79,7 @@ static void tick_do_update_jiffies64(ktime_t now) | |||
| 79 | /* Keep the tick_next_period variable up to date */ | 79 | /* Keep the tick_next_period variable up to date */ |
| 80 | tick_next_period = ktime_add(last_jiffies_update, tick_period); | 80 | tick_next_period = ktime_add(last_jiffies_update, tick_period); |
| 81 | } | 81 | } |
| 82 | write_sequnlock(&xtime_lock); | 82 | write_sequnlock(&jiffies_lock); |
| 83 | } | 83 | } |
| 84 | 84 | ||
| 85 | /* | 85 | /* |
| @@ -89,12 +89,12 @@ static ktime_t tick_init_jiffy_update(void) | |||
| 89 | { | 89 | { |
| 90 | ktime_t period; | 90 | ktime_t period; |
| 91 | 91 | ||
| 92 | write_seqlock(&xtime_lock); | 92 | write_seqlock(&jiffies_lock); |
| 93 | /* Did we start the jiffies update yet ? */ | 93 | /* Did we start the jiffies update yet ? */ |
| 94 | if (last_jiffies_update.tv64 == 0) | 94 | if (last_jiffies_update.tv64 == 0) |
| 95 | last_jiffies_update = tick_next_period; | 95 | last_jiffies_update = tick_next_period; |
| 96 | period = last_jiffies_update; | 96 | period = last_jiffies_update; |
| 97 | write_sequnlock(&xtime_lock); | 97 | write_sequnlock(&jiffies_lock); |
| 98 | return period; | 98 | return period; |
| 99 | } | 99 | } |
| 100 | 100 | ||
| @@ -282,11 +282,11 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, | |||
| 282 | 282 | ||
| 283 | /* Read jiffies and the time when jiffies were updated last */ | 283 | /* Read jiffies and the time when jiffies were updated last */ |
| 284 | do { | 284 | do { |
| 285 | seq = read_seqbegin(&xtime_lock); | 285 | seq = read_seqbegin(&jiffies_lock); |
| 286 | last_update = last_jiffies_update; | 286 | last_update = last_jiffies_update; |
| 287 | last_jiffies = jiffies; | 287 | last_jiffies = jiffies; |
| 288 | time_delta = timekeeping_max_deferment(); | 288 | time_delta = timekeeping_max_deferment(); |
| 289 | } while (read_seqretry(&xtime_lock, seq)); | 289 | } while (read_seqretry(&jiffies_lock, seq)); |
| 290 | 290 | ||
| 291 | if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) || printk_needs_cpu(cpu) || | 291 | if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) || printk_needs_cpu(cpu) || |
| 292 | arch_needs_cpu(cpu)) { | 292 | arch_needs_cpu(cpu)) { |
| @@ -658,7 +658,7 @@ static void tick_nohz_handler(struct clock_event_device *dev) | |||
| 658 | * concurrency: This happens only when the cpu in charge went | 658 | * concurrency: This happens only when the cpu in charge went |
| 659 | * into a long sleep. If two cpus happen to assign themself to | 659 | * into a long sleep. If two cpus happen to assign themself to |
| 660 | * this duty, then the jiffies update is still serialized by | 660 | * this duty, then the jiffies update is still serialized by |
| 661 | * xtime_lock. | 661 | * jiffies_lock. |
| 662 | */ | 662 | */ |
| 663 | if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) | 663 | if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) |
| 664 | tick_do_timer_cpu = cpu; | 664 | tick_do_timer_cpu = cpu; |
| @@ -810,7 +810,7 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) | |||
| 810 | * concurrency: This happens only when the cpu in charge went | 810 | * concurrency: This happens only when the cpu in charge went |
| 811 | * into a long sleep. If two cpus happen to assign themself to | 811 | * into a long sleep. If two cpus happen to assign themself to |
| 812 | * this duty, then the jiffies update is still serialized by | 812 | * this duty, then the jiffies update is still serialized by |
| 813 | * xtime_lock. | 813 | * jiffies_lock. |
| 814 | */ | 814 | */ |
| 815 | if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) | 815 | if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) |
| 816 | tick_do_timer_cpu = cpu; | 816 | tick_do_timer_cpu = cpu; |
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index e424970bb562..4c7de02eacdc 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c | |||
| @@ -25,12 +25,6 @@ | |||
| 25 | 25 | ||
| 26 | static struct timekeeper timekeeper; | 26 | static struct timekeeper timekeeper; |
| 27 | 27 | ||
| 28 | /* | ||
| 29 | * This read-write spinlock protects us from races in SMP while | ||
| 30 | * playing with xtime. | ||
| 31 | */ | ||
| 32 | __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); | ||
| 33 | |||
| 34 | /* flag for if timekeeping is suspended */ | 28 | /* flag for if timekeeping is suspended */ |
| 35 | int __read_mostly timekeeping_suspended; | 29 | int __read_mostly timekeeping_suspended; |
| 36 | 30 | ||
| @@ -1299,9 +1293,7 @@ struct timespec get_monotonic_coarse(void) | |||
| 1299 | } | 1293 | } |
| 1300 | 1294 | ||
| 1301 | /* | 1295 | /* |
| 1302 | * The 64-bit jiffies value is not atomic - you MUST NOT read it | 1296 | * Must hold jiffies_lock |
| 1303 | * without sampling the sequence number in xtime_lock. | ||
| 1304 | * jiffies is defined in the linker script... | ||
| 1305 | */ | 1297 | */ |
| 1306 | void do_timer(unsigned long ticks) | 1298 | void do_timer(unsigned long ticks) |
| 1307 | { | 1299 | { |
| @@ -1389,7 +1381,7 @@ EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset); | |||
| 1389 | */ | 1381 | */ |
| 1390 | void xtime_update(unsigned long ticks) | 1382 | void xtime_update(unsigned long ticks) |
| 1391 | { | 1383 | { |
| 1392 | write_seqlock(&xtime_lock); | 1384 | write_seqlock(&jiffies_lock); |
| 1393 | do_timer(ticks); | 1385 | do_timer(ticks); |
| 1394 | write_sequnlock(&xtime_lock); | 1386 | write_sequnlock(&jiffies_lock); |
| 1395 | } | 1387 | } |
