diff options
author | Richard Cochran <richardcochran@gmail.com> | 2014-12-21 13:47:06 -0500 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2014-12-30 18:29:27 -0500 |
commit | 2eebdde6528a722fbf8e2cffcf7aa52cbb4c2de0 (patch) | |
tree | c4b176d1e40eecb2caa00952eae95ecb2b2a046b | |
parent | f25a30be359d0535fb1c7c1619cabb0ad17cfbf1 (diff) |
timecounter: keep track of accumulated fractional nanoseconds
The current timecounter implementation will drop a variable amount
of resolution, depending on the magnitude of the time delta. In
other words, reading the clock too often or too close to a time
stamp conversion will introduce errors into the time values. This
patch fixes the issue by introducing a fractional nanosecond field
that accumulates the low order bits.
Reported-by: Janusz Użycki <j.uzycki@elproma.com.pl>
Signed-off-by: Richard Cochran <richardcochran@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r-- | drivers/net/ethernet/mellanox/mlx4/en_clock.c | 4 | ||||
-rw-r--r-- | include/linux/timecounter.h | 19 | ||||
-rw-r--r-- | kernel/time/timecounter.c | 31 | ||||
-rw-r--r-- | virt/kvm/arm/arch_timer.c | 3 |
4 files changed, 40 insertions, 17 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_clock.c b/drivers/net/ethernet/mellanox/mlx4/en_clock.c index df35d0e1b899..e9cce4f72b24 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_clock.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_clock.c | |||
@@ -240,7 +240,7 @@ void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev) | |||
240 | { | 240 | { |
241 | struct mlx4_dev *dev = mdev->dev; | 241 | struct mlx4_dev *dev = mdev->dev; |
242 | unsigned long flags; | 242 | unsigned long flags; |
243 | u64 ns; | 243 | u64 ns, zero = 0; |
244 | 244 | ||
245 | rwlock_init(&mdev->clock_lock); | 245 | rwlock_init(&mdev->clock_lock); |
246 | 246 | ||
@@ -265,7 +265,7 @@ void mlx4_en_init_timestamp(struct mlx4_en_dev *mdev) | |||
265 | /* Calculate period in seconds to call the overflow watchdog - to make | 265 | /* Calculate period in seconds to call the overflow watchdog - to make |
266 | * sure counter is checked at least once every wrap around. | 266 | * sure counter is checked at least once every wrap around. |
267 | */ | 267 | */ |
268 | ns = cyclecounter_cyc2ns(&mdev->cycles, mdev->cycles.mask); | 268 | ns = cyclecounter_cyc2ns(&mdev->cycles, mdev->cycles.mask, zero, &zero); |
269 | do_div(ns, NSEC_PER_SEC / 2 / HZ); | 269 | do_div(ns, NSEC_PER_SEC / 2 / HZ); |
270 | mdev->overflow_period = ns; | 270 | mdev->overflow_period = ns; |
271 | 271 | ||
diff --git a/include/linux/timecounter.h b/include/linux/timecounter.h index af3dfa4e90f0..74f45496e6d1 100644 --- a/include/linux/timecounter.h +++ b/include/linux/timecounter.h | |||
@@ -55,27 +55,32 @@ struct cyclecounter { | |||
55 | * @cycle_last: most recent cycle counter value seen by | 55 | * @cycle_last: most recent cycle counter value seen by |
56 | * timecounter_read() | 56 | * timecounter_read() |
57 | * @nsec: continuously increasing count | 57 | * @nsec: continuously increasing count |
58 | * @mask: bit mask for maintaining the 'frac' field | ||
59 | * @frac: accumulated fractional nanoseconds | ||
58 | */ | 60 | */ |
59 | struct timecounter { | 61 | struct timecounter { |
60 | const struct cyclecounter *cc; | 62 | const struct cyclecounter *cc; |
61 | cycle_t cycle_last; | 63 | cycle_t cycle_last; |
62 | u64 nsec; | 64 | u64 nsec; |
65 | u64 mask; | ||
66 | u64 frac; | ||
63 | }; | 67 | }; |
64 | 68 | ||
65 | /** | 69 | /** |
66 | * cyclecounter_cyc2ns - converts cycle counter cycles to nanoseconds | 70 | * cyclecounter_cyc2ns - converts cycle counter cycles to nanoseconds |
67 | * @cc: Pointer to cycle counter. | 71 | * @cc: Pointer to cycle counter. |
68 | * @cycles: Cycles | 72 | * @cycles: Cycles |
69 | * | 73 | * @mask: bit mask for maintaining the 'frac' field |
70 | * XXX - This could use some mult_lxl_ll() asm optimization. Same code | 74 | * @frac: pointer to storage for the fractional nanoseconds. |
71 | * as in cyc2ns, but with unsigned result. | ||
72 | */ | 75 | */ |
73 | static inline u64 cyclecounter_cyc2ns(const struct cyclecounter *cc, | 76 | static inline u64 cyclecounter_cyc2ns(const struct cyclecounter *cc, |
74 | cycle_t cycles) | 77 | cycle_t cycles, u64 mask, u64 *frac) |
75 | { | 78 | { |
76 | u64 ret = (u64)cycles; | 79 | u64 ns = (u64) cycles; |
77 | ret = (ret * cc->mult) >> cc->shift; | 80 | |
78 | return ret; | 81 | ns = (ns * cc->mult) + *frac; |
82 | *frac = ns & mask; | ||
83 | return ns >> cc->shift; | ||
79 | } | 84 | } |
80 | 85 | ||
81 | /** | 86 | /** |
diff --git a/kernel/time/timecounter.c b/kernel/time/timecounter.c index 59a1ec3a57cb..4687b3104bae 100644 --- a/kernel/time/timecounter.c +++ b/kernel/time/timecounter.c | |||
@@ -25,6 +25,8 @@ void timecounter_init(struct timecounter *tc, | |||
25 | tc->cc = cc; | 25 | tc->cc = cc; |
26 | tc->cycle_last = cc->read(cc); | 26 | tc->cycle_last = cc->read(cc); |
27 | tc->nsec = start_tstamp; | 27 | tc->nsec = start_tstamp; |
28 | tc->mask = (1ULL << cc->shift) - 1; | ||
29 | tc->frac = 0; | ||
28 | } | 30 | } |
29 | EXPORT_SYMBOL_GPL(timecounter_init); | 31 | EXPORT_SYMBOL_GPL(timecounter_init); |
30 | 32 | ||
@@ -51,7 +53,8 @@ static u64 timecounter_read_delta(struct timecounter *tc) | |||
51 | cycle_delta = (cycle_now - tc->cycle_last) & tc->cc->mask; | 53 | cycle_delta = (cycle_now - tc->cycle_last) & tc->cc->mask; |
52 | 54 | ||
53 | /* convert to nanoseconds: */ | 55 | /* convert to nanoseconds: */ |
54 | ns_offset = cyclecounter_cyc2ns(tc->cc, cycle_delta); | 56 | ns_offset = cyclecounter_cyc2ns(tc->cc, cycle_delta, |
57 | tc->mask, &tc->frac); | ||
55 | 58 | ||
56 | /* update time stamp of timecounter_read_delta() call: */ | 59 | /* update time stamp of timecounter_read_delta() call: */ |
57 | tc->cycle_last = cycle_now; | 60 | tc->cycle_last = cycle_now; |
@@ -72,22 +75,36 @@ u64 timecounter_read(struct timecounter *tc) | |||
72 | } | 75 | } |
73 | EXPORT_SYMBOL_GPL(timecounter_read); | 76 | EXPORT_SYMBOL_GPL(timecounter_read); |
74 | 77 | ||
78 | /* | ||
79 | * This is like cyclecounter_cyc2ns(), but it is used for computing a | ||
80 | * time previous to the time stored in the cycle counter. | ||
81 | */ | ||
82 | static u64 cc_cyc2ns_backwards(const struct cyclecounter *cc, | ||
83 | cycle_t cycles, u64 mask, u64 frac) | ||
84 | { | ||
85 | u64 ns = (u64) cycles; | ||
86 | |||
87 | ns = ((ns * cc->mult) - frac) >> cc->shift; | ||
88 | |||
89 | return ns; | ||
90 | } | ||
91 | |||
75 | u64 timecounter_cyc2time(struct timecounter *tc, | 92 | u64 timecounter_cyc2time(struct timecounter *tc, |
76 | cycle_t cycle_tstamp) | 93 | cycle_t cycle_tstamp) |
77 | { | 94 | { |
78 | u64 cycle_delta = (cycle_tstamp - tc->cycle_last) & tc->cc->mask; | 95 | u64 delta = (cycle_tstamp - tc->cycle_last) & tc->cc->mask; |
79 | u64 nsec; | 96 | u64 nsec = tc->nsec, frac = tc->frac; |
80 | 97 | ||
81 | /* | 98 | /* |
82 | * Instead of always treating cycle_tstamp as more recent | 99 | * Instead of always treating cycle_tstamp as more recent |
83 | * than tc->cycle_last, detect when it is too far in the | 100 | * than tc->cycle_last, detect when it is too far in the |
84 | * future and treat it as old time stamp instead. | 101 | * future and treat it as old time stamp instead. |
85 | */ | 102 | */ |
86 | if (cycle_delta > tc->cc->mask / 2) { | 103 | if (delta > tc->cc->mask / 2) { |
87 | cycle_delta = (tc->cycle_last - cycle_tstamp) & tc->cc->mask; | 104 | delta = (tc->cycle_last - cycle_tstamp) & tc->cc->mask; |
88 | nsec = tc->nsec - cyclecounter_cyc2ns(tc->cc, cycle_delta); | 105 | nsec -= cc_cyc2ns_backwards(tc->cc, delta, tc->mask, frac); |
89 | } else { | 106 | } else { |
90 | nsec = cyclecounter_cyc2ns(tc->cc, cycle_delta) + tc->nsec; | 107 | nsec += cyclecounter_cyc2ns(tc->cc, delta, tc->mask, &frac); |
91 | } | 108 | } |
92 | 109 | ||
93 | return nsec; | 110 | return nsec; |
diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c index 1c0772b340d8..6e54f3542126 100644 --- a/virt/kvm/arm/arch_timer.c +++ b/virt/kvm/arm/arch_timer.c | |||
@@ -152,7 +152,8 @@ void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu) | |||
152 | return; | 152 | return; |
153 | } | 153 | } |
154 | 154 | ||
155 | ns = cyclecounter_cyc2ns(timecounter->cc, cval - now); | 155 | ns = cyclecounter_cyc2ns(timecounter->cc, cval - now, timecounter->mask, |
156 | &timecounter->frac); | ||
156 | timer_arm(timer, ns); | 157 | timer_arm(timer, ns); |
157 | } | 158 | } |
158 | 159 | ||