aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorThomas Gleixner <tglx@linutronix.de>2016-03-04 02:12:27 -0500
committerThomas Gleixner <tglx@linutronix.de>2016-03-04 02:12:27 -0500
commit69365272332a5cbef778c97b2337b4f79c509293 (patch)
treec03bd4d13c12ae3be7282d4624cf25b23e1c2d1c
parent82e88ff1ea948d83125a8aaa7c9809f03ccc500f (diff)
parent01d7ada57ee9c735bd71fbe44ec0bcb70847afd4 (diff)
Merge branch 'fortglx/4.6/time' of https://git.linaro.org/people/john.stultz/linux into timers/core
Pull the cross-timestamp infrastructure from John Stultz. Allows precise correlation of device timestamps with system time. Primary use cases being PTP and audio.
-rw-r--r--Documentation/ptp/testptp.c6
-rw-r--r--arch/x86/include/asm/cpufeature.h2
-rw-r--r--arch/x86/include/asm/tsc.h2
-rw-r--r--arch/x86/kernel/tsc.c59
-rw-r--r--drivers/net/ethernet/intel/Kconfig9
-rw-r--r--drivers/net/ethernet/intel/e1000e/defines.h5
-rw-r--r--drivers/net/ethernet/intel/e1000e/ptp.c85
-rw-r--r--drivers/net/ethernet/intel/e1000e/regs.h4
-rw-r--r--drivers/ptp/ptp_chardev.c27
-rw-r--r--include/linux/pps_kernel.h17
-rw-r--r--include/linux/ptp_clock_kernel.h8
-rw-r--r--include/linux/timekeeper_internal.h2
-rw-r--r--include/linux/timekeeping.h58
-rw-r--r--include/uapi/linux/ptp_clock.h13
-rw-r--r--kernel/time/timekeeping.c286
15 files changed, 543 insertions, 40 deletions
diff --git a/Documentation/ptp/testptp.c b/Documentation/ptp/testptp.c
index 6c6247aaa7b9..d99012f41602 100644
--- a/Documentation/ptp/testptp.c
+++ b/Documentation/ptp/testptp.c
@@ -277,13 +277,15 @@ int main(int argc, char *argv[])
277 " %d external time stamp channels\n" 277 " %d external time stamp channels\n"
278 " %d programmable periodic signals\n" 278 " %d programmable periodic signals\n"
279 " %d pulse per second\n" 279 " %d pulse per second\n"
280 " %d programmable pins\n", 280 " %d programmable pins\n"
281 " %d cross timestamping\n",
281 caps.max_adj, 282 caps.max_adj,
282 caps.n_alarm, 283 caps.n_alarm,
283 caps.n_ext_ts, 284 caps.n_ext_ts,
284 caps.n_per_out, 285 caps.n_per_out,
285 caps.pps, 286 caps.pps,
286 caps.n_pins); 287 caps.n_pins,
288 caps.cross_timestamping);
287 } 289 }
288 } 290 }
289 291
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 7ad8c9464297..ff557b49ab77 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -85,7 +85,7 @@
85#define X86_FEATURE_P4 ( 3*32+ 7) /* "" P4 */ 85#define X86_FEATURE_P4 ( 3*32+ 7) /* "" P4 */
86#define X86_FEATURE_CONSTANT_TSC ( 3*32+ 8) /* TSC ticks at a constant rate */ 86#define X86_FEATURE_CONSTANT_TSC ( 3*32+ 8) /* TSC ticks at a constant rate */
87#define X86_FEATURE_UP ( 3*32+ 9) /* smp kernel running on up */ 87#define X86_FEATURE_UP ( 3*32+ 9) /* smp kernel running on up */
88/* free, was #define X86_FEATURE_FXSAVE_LEAK ( 3*32+10) * "" FXSAVE leaks FOP/FIP/FOP */ 88#define X86_FEATURE_ART (3*32+10) /* Platform has always running timer (ART) */
89#define X86_FEATURE_ARCH_PERFMON ( 3*32+11) /* Intel Architectural PerfMon */ 89#define X86_FEATURE_ARCH_PERFMON ( 3*32+11) /* Intel Architectural PerfMon */
90#define X86_FEATURE_PEBS ( 3*32+12) /* Precise-Event Based Sampling */ 90#define X86_FEATURE_PEBS ( 3*32+12) /* Precise-Event Based Sampling */
91#define X86_FEATURE_BTS ( 3*32+13) /* Branch Trace Store */ 91#define X86_FEATURE_BTS ( 3*32+13) /* Branch Trace Store */
diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
index 6d7c5479bcea..174c4212780a 100644
--- a/arch/x86/include/asm/tsc.h
+++ b/arch/x86/include/asm/tsc.h
@@ -29,6 +29,8 @@ static inline cycles_t get_cycles(void)
29 return rdtsc(); 29 return rdtsc();
30} 30}
31 31
32extern struct system_counterval_t convert_art_to_tsc(cycle_t art);
33
32extern void tsc_init(void); 34extern void tsc_init(void);
33extern void mark_tsc_unstable(char *reason); 35extern void mark_tsc_unstable(char *reason);
34extern int unsynchronized_tsc(void); 36extern int unsynchronized_tsc(void);
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 3d743da828d3..80d761e420c5 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -43,6 +43,11 @@ static DEFINE_STATIC_KEY_FALSE(__use_tsc);
43 43
44int tsc_clocksource_reliable; 44int tsc_clocksource_reliable;
45 45
46static u32 art_to_tsc_numerator;
47static u32 art_to_tsc_denominator;
48static u64 art_to_tsc_offset;
49struct clocksource *art_related_clocksource;
50
46/* 51/*
47 * Use a ring-buffer like data structure, where a writer advances the head by 52 * Use a ring-buffer like data structure, where a writer advances the head by
48 * writing a new data entry and a reader advances the tail when it observes a 53 * writing a new data entry and a reader advances the tail when it observes a
@@ -964,6 +969,37 @@ core_initcall(cpufreq_tsc);
964 969
965#endif /* CONFIG_CPU_FREQ */ 970#endif /* CONFIG_CPU_FREQ */
966 971
972#define ART_CPUID_LEAF (0x15)
973#define ART_MIN_DENOMINATOR (1)
974
975
976/*
977 * If ART is present detect the numerator:denominator to convert to TSC
978 */
979static void detect_art(void)
980{
981 unsigned int unused[2];
982
983 if (boot_cpu_data.cpuid_level < ART_CPUID_LEAF)
984 return;
985
986 cpuid(ART_CPUID_LEAF, &art_to_tsc_denominator,
987 &art_to_tsc_numerator, unused, unused+1);
988
989 /* Don't enable ART in a VM, non-stop TSC required */
990 if (boot_cpu_has(X86_FEATURE_HYPERVISOR) ||
991 !boot_cpu_has(X86_FEATURE_NONSTOP_TSC) ||
992 art_to_tsc_denominator < ART_MIN_DENOMINATOR)
993 return;
994
995 if (rdmsrl_safe(MSR_IA32_TSC_ADJUST, &art_to_tsc_offset))
996 return;
997
998 /* Make this sticky over multiple CPU init calls */
999 setup_force_cpu_cap(X86_FEATURE_ART);
1000}
1001
1002
967/* clocksource code */ 1003/* clocksource code */
968 1004
969static struct clocksource clocksource_tsc; 1005static struct clocksource clocksource_tsc;
@@ -1071,6 +1107,25 @@ int unsynchronized_tsc(void)
1071 return 0; 1107 return 0;
1072} 1108}
1073 1109
1110/*
1111 * Convert ART to TSC given numerator/denominator found in detect_art()
1112 */
1113struct system_counterval_t convert_art_to_tsc(cycle_t art)
1114{
1115 u64 tmp, res, rem;
1116
1117 rem = do_div(art, art_to_tsc_denominator);
1118
1119 res = art * art_to_tsc_numerator;
1120 tmp = rem * art_to_tsc_numerator;
1121
1122 do_div(tmp, art_to_tsc_denominator);
1123 res += tmp + art_to_tsc_offset;
1124
1125 return (struct system_counterval_t) {.cs = art_related_clocksource,
1126 .cycles = res};
1127}
1128EXPORT_SYMBOL(convert_art_to_tsc);
1074 1129
1075static void tsc_refine_calibration_work(struct work_struct *work); 1130static void tsc_refine_calibration_work(struct work_struct *work);
1076static DECLARE_DELAYED_WORK(tsc_irqwork, tsc_refine_calibration_work); 1131static DECLARE_DELAYED_WORK(tsc_irqwork, tsc_refine_calibration_work);
@@ -1142,6 +1197,8 @@ static void tsc_refine_calibration_work(struct work_struct *work)
1142 (unsigned long)tsc_khz % 1000); 1197 (unsigned long)tsc_khz % 1000);
1143 1198
1144out: 1199out:
1200 if (boot_cpu_has(X86_FEATURE_ART))
1201 art_related_clocksource = &clocksource_tsc;
1145 clocksource_register_khz(&clocksource_tsc, tsc_khz); 1202 clocksource_register_khz(&clocksource_tsc, tsc_khz);
1146} 1203}
1147 1204
@@ -1235,6 +1292,8 @@ void __init tsc_init(void)
1235 mark_tsc_unstable("TSCs unsynchronized"); 1292 mark_tsc_unstable("TSCs unsynchronized");
1236 1293
1237 check_system_tsc_reliable(); 1294 check_system_tsc_reliable();
1295
1296 detect_art();
1238} 1297}
1239 1298
1240#ifdef CONFIG_SMP 1299#ifdef CONFIG_SMP
diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig
index fa593dd3efe1..3772f3ac956e 100644
--- a/drivers/net/ethernet/intel/Kconfig
+++ b/drivers/net/ethernet/intel/Kconfig
@@ -83,6 +83,15 @@ config E1000E
83 To compile this driver as a module, choose M here. The module 83 To compile this driver as a module, choose M here. The module
84 will be called e1000e. 84 will be called e1000e.
85 85
86config E1000E_HWTS
87 bool "Support HW cross-timestamp on PCH devices"
88 default y
89 depends on E1000E && X86
90 ---help---
91 Say Y to enable hardware supported cross-timestamping on PCH
92 devices. The cross-timestamp is available through the PTP clock
93 driver precise cross-timestamp ioctl (PTP_SYS_OFFSET_PRECISE).
94
86config IGB 95config IGB
87 tristate "Intel(R) 82575/82576 PCI-Express Gigabit Ethernet support" 96 tristate "Intel(R) 82575/82576 PCI-Express Gigabit Ethernet support"
88 depends on PCI 97 depends on PCI
diff --git a/drivers/net/ethernet/intel/e1000e/defines.h b/drivers/net/ethernet/intel/e1000e/defines.h
index f7c7804d79e5..0641c0098738 100644
--- a/drivers/net/ethernet/intel/e1000e/defines.h
+++ b/drivers/net/ethernet/intel/e1000e/defines.h
@@ -528,6 +528,11 @@
528#define E1000_RXCW_C 0x20000000 /* Receive config */ 528#define E1000_RXCW_C 0x20000000 /* Receive config */
529#define E1000_RXCW_SYNCH 0x40000000 /* Receive config synch */ 529#define E1000_RXCW_SYNCH 0x40000000 /* Receive config synch */
530 530
531/* HH Time Sync */
532#define E1000_TSYNCTXCTL_MAX_ALLOWED_DLY_MASK 0x0000F000 /* max delay */
533#define E1000_TSYNCTXCTL_SYNC_COMP 0x40000000 /* sync complete */
534#define E1000_TSYNCTXCTL_START_SYNC 0x80000000 /* initiate sync */
535
531#define E1000_TSYNCTXCTL_VALID 0x00000001 /* Tx timestamp valid */ 536#define E1000_TSYNCTXCTL_VALID 0x00000001 /* Tx timestamp valid */
532#define E1000_TSYNCTXCTL_ENABLED 0x00000010 /* enable Tx timestamping */ 537#define E1000_TSYNCTXCTL_ENABLED 0x00000010 /* enable Tx timestamping */
533 538
diff --git a/drivers/net/ethernet/intel/e1000e/ptp.c b/drivers/net/ethernet/intel/e1000e/ptp.c
index 25a0ad5102d6..e2ff3ef75d5d 100644
--- a/drivers/net/ethernet/intel/e1000e/ptp.c
+++ b/drivers/net/ethernet/intel/e1000e/ptp.c
@@ -26,6 +26,12 @@
26 26
27#include "e1000.h" 27#include "e1000.h"
28 28
29#ifdef CONFIG_E1000E_HWTS
30#include <linux/clocksource.h>
31#include <linux/ktime.h>
32#include <asm/tsc.h>
33#endif
34
29/** 35/**
30 * e1000e_phc_adjfreq - adjust the frequency of the hardware clock 36 * e1000e_phc_adjfreq - adjust the frequency of the hardware clock
31 * @ptp: ptp clock structure 37 * @ptp: ptp clock structure
@@ -98,6 +104,78 @@ static int e1000e_phc_adjtime(struct ptp_clock_info *ptp, s64 delta)
98 return 0; 104 return 0;
99} 105}
100 106
107#ifdef CONFIG_E1000E_HWTS
108#define MAX_HW_WAIT_COUNT (3)
109
110/**
111 * e1000e_phc_get_syncdevicetime - Callback given to timekeeping code reads system/device registers
112 * @device: current device time
113 * @system: system counter value read synchronously with device time
114 * @ctx: context provided by timekeeping code
115 *
116 * Read device and system (ART) clock simultaneously and return the corrected
117 * clock values in ns.
118 **/
119static int e1000e_phc_get_syncdevicetime(ktime_t *device,
120 struct system_counterval_t *system,
121 void *ctx)
122{
123 struct e1000_adapter *adapter = (struct e1000_adapter *)ctx;
124 struct e1000_hw *hw = &adapter->hw;
125 unsigned long flags;
126 int i;
127 u32 tsync_ctrl;
128 cycle_t dev_cycles;
129 cycle_t sys_cycles;
130
131 tsync_ctrl = er32(TSYNCTXCTL);
132 tsync_ctrl |= E1000_TSYNCTXCTL_START_SYNC |
133 E1000_TSYNCTXCTL_MAX_ALLOWED_DLY_MASK;
134 ew32(TSYNCTXCTL, tsync_ctrl);
135 for (i = 0; i < MAX_HW_WAIT_COUNT; ++i) {
136 udelay(1);
137 tsync_ctrl = er32(TSYNCTXCTL);
138 if (tsync_ctrl & E1000_TSYNCTXCTL_SYNC_COMP)
139 break;
140 }
141
142 if (i == MAX_HW_WAIT_COUNT)
143 return -ETIMEDOUT;
144
145 dev_cycles = er32(SYSSTMPH);
146 dev_cycles <<= 32;
147 dev_cycles |= er32(SYSSTMPL);
148 spin_lock_irqsave(&adapter->systim_lock, flags);
149 *device = ns_to_ktime(timecounter_cyc2time(&adapter->tc, dev_cycles));
150 spin_unlock_irqrestore(&adapter->systim_lock, flags);
151
152 sys_cycles = er32(PLTSTMPH);
153 sys_cycles <<= 32;
154 sys_cycles |= er32(PLTSTMPL);
155 *system = convert_art_to_tsc(sys_cycles);
156
157 return 0;
158}
159
160/**
161 * e1000e_phc_getsynctime - Reads the current system/device cross timestamp
162 * @ptp: ptp clock structure
163 * @cts: structure containing timestamp
164 *
165 * Read device and system (ART) clock simultaneously and return the scaled
166 * clock values in ns.
167 **/
168static int e1000e_phc_getcrosststamp(struct ptp_clock_info *ptp,
169 struct system_device_crosststamp *xtstamp)
170{
171 struct e1000_adapter *adapter = container_of(ptp, struct e1000_adapter,
172 ptp_clock_info);
173
174 return get_device_system_crosststamp(e1000e_phc_get_syncdevicetime,
175 adapter, NULL, xtstamp);
176}
177#endif/*CONFIG_E1000E_HWTS*/
178
101/** 179/**
102 * e1000e_phc_gettime - Reads the current time from the hardware clock 180 * e1000e_phc_gettime - Reads the current time from the hardware clock
103 * @ptp: ptp clock structure 181 * @ptp: ptp clock structure
@@ -236,6 +314,13 @@ void e1000e_ptp_init(struct e1000_adapter *adapter)
236 break; 314 break;
237 } 315 }
238 316
317#ifdef CONFIG_E1000E_HWTS
318 /* CPU must have ART and GBe must be from Sunrise Point or greater */
319 if (hw->mac.type >= e1000_pch_spt && boot_cpu_has(X86_FEATURE_ART))
320 adapter->ptp_clock_info.getcrosststamp =
321 e1000e_phc_getcrosststamp;
322#endif/*CONFIG_E1000E_HWTS*/
323
239 INIT_DELAYED_WORK(&adapter->systim_overflow_work, 324 INIT_DELAYED_WORK(&adapter->systim_overflow_work,
240 e1000e_systim_overflow_work); 325 e1000e_systim_overflow_work);
241 326
diff --git a/drivers/net/ethernet/intel/e1000e/regs.h b/drivers/net/ethernet/intel/e1000e/regs.h
index 1d5e0b77062a..0cb4d365e5ad 100644
--- a/drivers/net/ethernet/intel/e1000e/regs.h
+++ b/drivers/net/ethernet/intel/e1000e/regs.h
@@ -245,6 +245,10 @@
245#define E1000_SYSTIML 0x0B600 /* System time register Low - RO */ 245#define E1000_SYSTIML 0x0B600 /* System time register Low - RO */
246#define E1000_SYSTIMH 0x0B604 /* System time register High - RO */ 246#define E1000_SYSTIMH 0x0B604 /* System time register High - RO */
247#define E1000_TIMINCA 0x0B608 /* Increment attributes register - RW */ 247#define E1000_TIMINCA 0x0B608 /* Increment attributes register - RW */
248#define E1000_SYSSTMPL 0x0B648 /* HH Timesync system stamp low register */
249#define E1000_SYSSTMPH 0x0B64C /* HH Timesync system stamp hi register */
250#define E1000_PLTSTMPL 0x0B640 /* HH Timesync platform stamp low register */
251#define E1000_PLTSTMPH 0x0B644 /* HH Timesync platform stamp hi register */
248#define E1000_RXMTRL 0x0B634 /* Time sync Rx EtherType and Msg Type - RW */ 252#define E1000_RXMTRL 0x0B634 /* Time sync Rx EtherType and Msg Type - RW */
249#define E1000_RXUDP 0x0B638 /* Time Sync Rx UDP Port - RW */ 253#define E1000_RXUDP 0x0B638 /* Time Sync Rx UDP Port - RW */
250 254
diff --git a/drivers/ptp/ptp_chardev.c b/drivers/ptp/ptp_chardev.c
index da7bae991552..579fd65299a0 100644
--- a/drivers/ptp/ptp_chardev.c
+++ b/drivers/ptp/ptp_chardev.c
@@ -22,6 +22,7 @@
22#include <linux/poll.h> 22#include <linux/poll.h>
23#include <linux/sched.h> 23#include <linux/sched.h>
24#include <linux/slab.h> 24#include <linux/slab.h>
25#include <linux/timekeeping.h>
25 26
26#include "ptp_private.h" 27#include "ptp_private.h"
27 28
@@ -120,11 +121,13 @@ long ptp_ioctl(struct posix_clock *pc, unsigned int cmd, unsigned long arg)
120 struct ptp_clock_caps caps; 121 struct ptp_clock_caps caps;
121 struct ptp_clock_request req; 122 struct ptp_clock_request req;
122 struct ptp_sys_offset *sysoff = NULL; 123 struct ptp_sys_offset *sysoff = NULL;
124 struct ptp_sys_offset_precise precise_offset;
123 struct ptp_pin_desc pd; 125 struct ptp_pin_desc pd;
124 struct ptp_clock *ptp = container_of(pc, struct ptp_clock, clock); 126 struct ptp_clock *ptp = container_of(pc, struct ptp_clock, clock);
125 struct ptp_clock_info *ops = ptp->info; 127 struct ptp_clock_info *ops = ptp->info;
126 struct ptp_clock_time *pct; 128 struct ptp_clock_time *pct;
127 struct timespec64 ts; 129 struct timespec64 ts;
130 struct system_device_crosststamp xtstamp;
128 int enable, err = 0; 131 int enable, err = 0;
129 unsigned int i, pin_index; 132 unsigned int i, pin_index;
130 133
@@ -138,6 +141,7 @@ long ptp_ioctl(struct posix_clock *pc, unsigned int cmd, unsigned long arg)
138 caps.n_per_out = ptp->info->n_per_out; 141 caps.n_per_out = ptp->info->n_per_out;
139 caps.pps = ptp->info->pps; 142 caps.pps = ptp->info->pps;
140 caps.n_pins = ptp->info->n_pins; 143 caps.n_pins = ptp->info->n_pins;
144 caps.cross_timestamping = ptp->info->getcrosststamp != NULL;
141 if (copy_to_user((void __user *)arg, &caps, sizeof(caps))) 145 if (copy_to_user((void __user *)arg, &caps, sizeof(caps)))
142 err = -EFAULT; 146 err = -EFAULT;
143 break; 147 break;
@@ -180,6 +184,29 @@ long ptp_ioctl(struct posix_clock *pc, unsigned int cmd, unsigned long arg)
180 err = ops->enable(ops, &req, enable); 184 err = ops->enable(ops, &req, enable);
181 break; 185 break;
182 186
187 case PTP_SYS_OFFSET_PRECISE:
188 if (!ptp->info->getcrosststamp) {
189 err = -EOPNOTSUPP;
190 break;
191 }
192 err = ptp->info->getcrosststamp(ptp->info, &xtstamp);
193 if (err)
194 break;
195
196 ts = ktime_to_timespec64(xtstamp.device);
197 precise_offset.device.sec = ts.tv_sec;
198 precise_offset.device.nsec = ts.tv_nsec;
199 ts = ktime_to_timespec64(xtstamp.sys_realtime);
200 precise_offset.sys_realtime.sec = ts.tv_sec;
201 precise_offset.sys_realtime.nsec = ts.tv_nsec;
202 ts = ktime_to_timespec64(xtstamp.sys_monoraw);
203 precise_offset.sys_monoraw.sec = ts.tv_sec;
204 precise_offset.sys_monoraw.nsec = ts.tv_nsec;
205 if (copy_to_user((void __user *)arg, &precise_offset,
206 sizeof(precise_offset)))
207 err = -EFAULT;
208 break;
209
183 case PTP_SYS_OFFSET: 210 case PTP_SYS_OFFSET:
184 sysoff = kmalloc(sizeof(*sysoff), GFP_KERNEL); 211 sysoff = kmalloc(sizeof(*sysoff), GFP_KERNEL);
185 if (!sysoff) { 212 if (!sysoff) {
diff --git a/include/linux/pps_kernel.h b/include/linux/pps_kernel.h
index 54bf1484d41f..35ac903956c7 100644
--- a/include/linux/pps_kernel.h
+++ b/include/linux/pps_kernel.h
@@ -111,22 +111,17 @@ static inline void timespec_to_pps_ktime(struct pps_ktime *kt,
111 kt->nsec = ts.tv_nsec; 111 kt->nsec = ts.tv_nsec;
112} 112}
113 113
114#ifdef CONFIG_NTP_PPS
115
116static inline void pps_get_ts(struct pps_event_time *ts) 114static inline void pps_get_ts(struct pps_event_time *ts)
117{ 115{
118 ktime_get_raw_and_real_ts64(&ts->ts_raw, &ts->ts_real); 116 struct system_time_snapshot snap;
119}
120 117
121#else /* CONFIG_NTP_PPS */ 118 ktime_get_snapshot(&snap);
122 119 ts->ts_real = ktime_to_timespec64(snap.real);
123static inline void pps_get_ts(struct pps_event_time *ts) 120#ifdef CONFIG_NTP_PPS
124{ 121 ts->ts_raw = ktime_to_timespec64(snap.raw);
125 ktime_get_real_ts64(&ts->ts_real); 122#endif
126} 123}
127 124
128#endif /* CONFIG_NTP_PPS */
129
130/* Subtract known time delay from PPS event time(s) */ 125/* Subtract known time delay from PPS event time(s) */
131static inline void pps_sub_ts(struct pps_event_time *ts, struct timespec64 delta) 126static inline void pps_sub_ts(struct pps_event_time *ts, struct timespec64 delta)
132{ 127{
diff --git a/include/linux/ptp_clock_kernel.h b/include/linux/ptp_clock_kernel.h
index b8b73066d137..6b15e168148a 100644
--- a/include/linux/ptp_clock_kernel.h
+++ b/include/linux/ptp_clock_kernel.h
@@ -38,6 +38,7 @@ struct ptp_clock_request {
38 }; 38 };
39}; 39};
40 40
41struct system_device_crosststamp;
41/** 42/**
42 * struct ptp_clock_info - decribes a PTP hardware clock 43 * struct ptp_clock_info - decribes a PTP hardware clock
43 * 44 *
@@ -67,6 +68,11 @@ struct ptp_clock_request {
67 * @gettime64: Reads the current time from the hardware clock. 68 * @gettime64: Reads the current time from the hardware clock.
68 * parameter ts: Holds the result. 69 * parameter ts: Holds the result.
69 * 70 *
71 * @getcrosststamp: Reads the current time from the hardware clock and
72 * system clock simultaneously.
73 * parameter cts: Contains timestamp (device,system) pair,
74 * where system time is realtime and monotonic.
75 *
70 * @settime64: Set the current time on the hardware clock. 76 * @settime64: Set the current time on the hardware clock.
71 * parameter ts: Time value to set. 77 * parameter ts: Time value to set.
72 * 78 *
@@ -105,6 +111,8 @@ struct ptp_clock_info {
105 int (*adjfreq)(struct ptp_clock_info *ptp, s32 delta); 111 int (*adjfreq)(struct ptp_clock_info *ptp, s32 delta);
106 int (*adjtime)(struct ptp_clock_info *ptp, s64 delta); 112 int (*adjtime)(struct ptp_clock_info *ptp, s64 delta);
107 int (*gettime64)(struct ptp_clock_info *ptp, struct timespec64 *ts); 113 int (*gettime64)(struct ptp_clock_info *ptp, struct timespec64 *ts);
114 int (*getcrosststamp)(struct ptp_clock_info *ptp,
115 struct system_device_crosststamp *cts);
108 int (*settime64)(struct ptp_clock_info *p, const struct timespec64 *ts); 116 int (*settime64)(struct ptp_clock_info *p, const struct timespec64 *ts);
109 int (*enable)(struct ptp_clock_info *ptp, 117 int (*enable)(struct ptp_clock_info *ptp,
110 struct ptp_clock_request *request, int on); 118 struct ptp_clock_request *request, int on);
diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h
index 25247220b4b7..e88005459035 100644
--- a/include/linux/timekeeper_internal.h
+++ b/include/linux/timekeeper_internal.h
@@ -50,6 +50,7 @@ struct tk_read_base {
50 * @offs_tai: Offset clock monotonic -> clock tai 50 * @offs_tai: Offset clock monotonic -> clock tai
51 * @tai_offset: The current UTC to TAI offset in seconds 51 * @tai_offset: The current UTC to TAI offset in seconds
52 * @clock_was_set_seq: The sequence number of clock was set events 52 * @clock_was_set_seq: The sequence number of clock was set events
53 * @cs_was_changed_seq: The sequence number of clocksource change events
53 * @next_leap_ktime: CLOCK_MONOTONIC time value of a pending leap-second 54 * @next_leap_ktime: CLOCK_MONOTONIC time value of a pending leap-second
54 * @raw_time: Monotonic raw base time in timespec64 format 55 * @raw_time: Monotonic raw base time in timespec64 format
55 * @cycle_interval: Number of clock cycles in one NTP interval 56 * @cycle_interval: Number of clock cycles in one NTP interval
@@ -91,6 +92,7 @@ struct timekeeper {
91 ktime_t offs_tai; 92 ktime_t offs_tai;
92 s32 tai_offset; 93 s32 tai_offset;
93 unsigned int clock_was_set_seq; 94 unsigned int clock_was_set_seq;
95 u8 cs_was_changed_seq;
94 ktime_t next_leap_ktime; 96 ktime_t next_leap_ktime;
95 struct timespec64 raw_time; 97 struct timespec64 raw_time;
96 98
diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h
index ec89d846324c..96f37bee3bc1 100644
--- a/include/linux/timekeeping.h
+++ b/include/linux/timekeeping.h
@@ -267,6 +267,64 @@ extern void ktime_get_raw_and_real_ts64(struct timespec64 *ts_raw,
267 struct timespec64 *ts_real); 267 struct timespec64 *ts_real);
268 268
269/* 269/*
270 * struct system_time_snapshot - simultaneous raw/real time capture with
271 * counter value
272 * @cycles: Clocksource counter value to produce the system times
273 * @real: Realtime system time
274 * @raw: Monotonic raw system time
275 * @clock_was_set_seq: The sequence number of clock was set events
276 * @cs_was_changed_seq: The sequence number of clocksource change events
277 */
278struct system_time_snapshot {
279 cycle_t cycles;
280 ktime_t real;
281 ktime_t raw;
282 unsigned int clock_was_set_seq;
283 u8 cs_was_changed_seq;
284};
285
286/*
287 * struct system_device_crosststamp - system/device cross-timestamp
288 * (syncronized capture)
289 * @device: Device time
290 * @sys_realtime: Realtime simultaneous with device time
291 * @sys_monoraw: Monotonic raw simultaneous with device time
292 */
293struct system_device_crosststamp {
294 ktime_t device;
295 ktime_t sys_realtime;
296 ktime_t sys_monoraw;
297};
298
299/*
300 * struct system_counterval_t - system counter value with the pointer to the
301 * corresponding clocksource
302 * @cycles: System counter value
303 * @cs: Clocksource corresponding to system counter value. Used by
304 * timekeeping code to verify comparibility of two cycle values
305 */
306struct system_counterval_t {
307 cycle_t cycles;
308 struct clocksource *cs;
309};
310
311/*
312 * Get cross timestamp between system clock and device clock
313 */
314extern int get_device_system_crosststamp(
315 int (*get_time_fn)(ktime_t *device_time,
316 struct system_counterval_t *system_counterval,
317 void *ctx),
318 void *ctx,
319 struct system_time_snapshot *history,
320 struct system_device_crosststamp *xtstamp);
321
322/*
323 * Simultaneously snapshot realtime and monotonic raw clocks
324 */
325extern void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot);
326
327/*
270 * Persistent clock related interfaces 328 * Persistent clock related interfaces
271 */ 329 */
272extern int persistent_clock_is_local; 330extern int persistent_clock_is_local;
diff --git a/include/uapi/linux/ptp_clock.h b/include/uapi/linux/ptp_clock.h
index f0b7bfe5da92..ac6dded80ffa 100644
--- a/include/uapi/linux/ptp_clock.h
+++ b/include/uapi/linux/ptp_clock.h
@@ -51,7 +51,9 @@ struct ptp_clock_caps {
51 int n_per_out; /* Number of programmable periodic signals. */ 51 int n_per_out; /* Number of programmable periodic signals. */
52 int pps; /* Whether the clock supports a PPS callback. */ 52 int pps; /* Whether the clock supports a PPS callback. */
53 int n_pins; /* Number of input/output pins. */ 53 int n_pins; /* Number of input/output pins. */
54 int rsv[14]; /* Reserved for future use. */ 54 /* Whether the clock supports precise system-device cross timestamps */
55 int cross_timestamping;
56 int rsv[13]; /* Reserved for future use. */
55}; 57};
56 58
57struct ptp_extts_request { 59struct ptp_extts_request {
@@ -81,6 +83,13 @@ struct ptp_sys_offset {
81 struct ptp_clock_time ts[2 * PTP_MAX_SAMPLES + 1]; 83 struct ptp_clock_time ts[2 * PTP_MAX_SAMPLES + 1];
82}; 84};
83 85
86struct ptp_sys_offset_precise {
87 struct ptp_clock_time device;
88 struct ptp_clock_time sys_realtime;
89 struct ptp_clock_time sys_monoraw;
90 unsigned int rsv[4]; /* Reserved for future use. */
91};
92
84enum ptp_pin_function { 93enum ptp_pin_function {
85 PTP_PF_NONE, 94 PTP_PF_NONE,
86 PTP_PF_EXTTS, 95 PTP_PF_EXTTS,
@@ -124,6 +133,8 @@ struct ptp_pin_desc {
124#define PTP_SYS_OFFSET _IOW(PTP_CLK_MAGIC, 5, struct ptp_sys_offset) 133#define PTP_SYS_OFFSET _IOW(PTP_CLK_MAGIC, 5, struct ptp_sys_offset)
125#define PTP_PIN_GETFUNC _IOWR(PTP_CLK_MAGIC, 6, struct ptp_pin_desc) 134#define PTP_PIN_GETFUNC _IOWR(PTP_CLK_MAGIC, 6, struct ptp_pin_desc)
126#define PTP_PIN_SETFUNC _IOW(PTP_CLK_MAGIC, 7, struct ptp_pin_desc) 135#define PTP_PIN_SETFUNC _IOW(PTP_CLK_MAGIC, 7, struct ptp_pin_desc)
136#define PTP_SYS_OFFSET_PRECISE \
137 _IOWR(PTP_CLK_MAGIC, 8, struct ptp_sys_offset_precise)
127 138
128struct ptp_extts_event { 139struct ptp_extts_event {
129 struct ptp_clock_time t; /* Time event occured. */ 140 struct ptp_clock_time t; /* Time event occured. */
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index 34b4cedfa80d..931b0b1a71e9 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -233,6 +233,7 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
233 u64 tmp, ntpinterval; 233 u64 tmp, ntpinterval;
234 struct clocksource *old_clock; 234 struct clocksource *old_clock;
235 235
236 ++tk->cs_was_changed_seq;
236 old_clock = tk->tkr_mono.clock; 237 old_clock = tk->tkr_mono.clock;
237 tk->tkr_mono.clock = clock; 238 tk->tkr_mono.clock = clock;
238 tk->tkr_mono.read = clock->read; 239 tk->tkr_mono.read = clock->read;
@@ -298,17 +299,34 @@ u32 (*arch_gettimeoffset)(void) = default_arch_gettimeoffset;
298static inline u32 arch_gettimeoffset(void) { return 0; } 299static inline u32 arch_gettimeoffset(void) { return 0; }
299#endif 300#endif
300 301
302static inline s64 timekeeping_delta_to_ns(struct tk_read_base *tkr,
303 cycle_t delta)
304{
305 s64 nsec;
306
307 nsec = delta * tkr->mult + tkr->xtime_nsec;
308 nsec >>= tkr->shift;
309
310 /* If arch requires, add in get_arch_timeoffset() */
311 return nsec + arch_gettimeoffset();
312}
313
301static inline s64 timekeeping_get_ns(struct tk_read_base *tkr) 314static inline s64 timekeeping_get_ns(struct tk_read_base *tkr)
302{ 315{
303 cycle_t delta; 316 cycle_t delta;
304 s64 nsec;
305 317
306 delta = timekeeping_get_delta(tkr); 318 delta = timekeeping_get_delta(tkr);
319 return timekeeping_delta_to_ns(tkr, delta);
320}
307 321
308 nsec = (delta * tkr->mult + tkr->xtime_nsec) >> tkr->shift; 322static inline s64 timekeeping_cycles_to_ns(struct tk_read_base *tkr,
323 cycle_t cycles)
324{
325 cycle_t delta;
309 326
310 /* If arch requires, add in get_arch_timeoffset() */ 327 /* calculate the delta since the last update_wall_time */
311 return nsec + arch_gettimeoffset(); 328 delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
329 return timekeeping_delta_to_ns(tkr, delta);
312} 330}
313 331
314/** 332/**
@@ -857,44 +875,262 @@ time64_t __ktime_get_real_seconds(void)
857 return tk->xtime_sec; 875 return tk->xtime_sec;
858} 876}
859 877
878/**
879 * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
880 * @systime_snapshot: pointer to struct receiving the system time snapshot
881 */
882void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
883{
884 struct timekeeper *tk = &tk_core.timekeeper;
885 unsigned long seq;
886 ktime_t base_raw;
887 ktime_t base_real;
888 s64 nsec_raw;
889 s64 nsec_real;
890 cycle_t now;
860 891
861#ifdef CONFIG_NTP_PPS 892 WARN_ON_ONCE(timekeeping_suspended);
893
894 do {
895 seq = read_seqcount_begin(&tk_core.seq);
896
897 now = tk->tkr_mono.read(tk->tkr_mono.clock);
898 systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
899 systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
900 base_real = ktime_add(tk->tkr_mono.base,
901 tk_core.timekeeper.offs_real);
902 base_raw = tk->tkr_raw.base;
903 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
904 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
905 } while (read_seqcount_retry(&tk_core.seq, seq));
906
907 systime_snapshot->cycles = now;
908 systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
909 systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
910}
911EXPORT_SYMBOL_GPL(ktime_get_snapshot);
912
913/* Scale base by mult/div checking for overflow */
914static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
915{
916 u64 tmp, rem;
917
918 tmp = div64_u64_rem(*base, div, &rem);
919
920 if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||
921 ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))
922 return -EOVERFLOW;
923 tmp *= mult;
924 rem *= mult;
925
926 do_div(rem, div);
927 *base = tmp + rem;
928 return 0;
929}
862 930
863/** 931/**
864 * ktime_get_raw_and_real_ts64 - get day and raw monotonic time in timespec format 932 * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
865 * @ts_raw: pointer to the timespec to be set to raw monotonic time 933 * @history: Snapshot representing start of history
866 * @ts_real: pointer to the timespec to be set to the time of day 934 * @partial_history_cycles: Cycle offset into history (fractional part)
935 * @total_history_cycles: Total history length in cycles
936 * @discontinuity: True indicates clock was set on history period
937 * @ts: Cross timestamp that should be adjusted using
938 * partial/total ratio
867 * 939 *
868 * This function reads both the time of day and raw monotonic time at the 940 * Helper function used by get_device_system_crosststamp() to correct the
869 * same time atomically and stores the resulting timestamps in timespec 941 * crosstimestamp corresponding to the start of the current interval to the
870 * format. 942 * system counter value (timestamp point) provided by the driver. The
943 * total_history_* quantities are the total history starting at the provided
944 * reference point and ending at the start of the current interval. The cycle
945 * count between the driver timestamp point and the start of the current
946 * interval is partial_history_cycles.
871 */ 947 */
872void ktime_get_raw_and_real_ts64(struct timespec64 *ts_raw, struct timespec64 *ts_real) 948static int adjust_historical_crosststamp(struct system_time_snapshot *history,
949 cycle_t partial_history_cycles,
950 cycle_t total_history_cycles,
951 bool discontinuity,
952 struct system_device_crosststamp *ts)
873{ 953{
874 struct timekeeper *tk = &tk_core.timekeeper; 954 struct timekeeper *tk = &tk_core.timekeeper;
875 unsigned long seq; 955 u64 corr_raw, corr_real;
876 s64 nsecs_raw, nsecs_real; 956 bool interp_forward;
957 int ret;
877 958
878 WARN_ON_ONCE(timekeeping_suspended); 959 if (total_history_cycles == 0 || partial_history_cycles == 0)
960 return 0;
961
962 /* Interpolate shortest distance from beginning or end of history */
963 interp_forward = partial_history_cycles > total_history_cycles/2 ?
964 true : false;
965 partial_history_cycles = interp_forward ?
966 total_history_cycles - partial_history_cycles :
967 partial_history_cycles;
968
969 /*
970 * Scale the monotonic raw time delta by:
971 * partial_history_cycles / total_history_cycles
972 */
973 corr_raw = (u64)ktime_to_ns(
974 ktime_sub(ts->sys_monoraw, history->raw));
975 ret = scale64_check_overflow(partial_history_cycles,
976 total_history_cycles, &corr_raw);
977 if (ret)
978 return ret;
979
980 /*
981 * If there is a discontinuity in the history, scale monotonic raw
982 * correction by:
983 * mult(real)/mult(raw) yielding the realtime correction
984 * Otherwise, calculate the realtime correction similar to monotonic
985 * raw calculation
986 */
987 if (discontinuity) {
988 corr_real = mul_u64_u32_div
989 (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
990 } else {
991 corr_real = (u64)ktime_to_ns(
992 ktime_sub(ts->sys_realtime, history->real));
993 ret = scale64_check_overflow(partial_history_cycles,
994 total_history_cycles, &corr_real);
995 if (ret)
996 return ret;
997 }
998
999 /* Fixup monotonic raw and real time time values */
1000 if (interp_forward) {
1001 ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
1002 ts->sys_realtime = ktime_add_ns(history->real, corr_real);
1003 } else {
1004 ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
1005 ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
1006 }
1007
1008 return 0;
1009}
1010
1011/*
1012 * cycle_between - true if test occurs chronologically between before and after
1013 */
1014static bool cycle_between(cycle_t before, cycle_t test, cycle_t after)
1015{
1016 if (test > before && test < after)
1017 return true;
1018 if (test < before && before > after)
1019 return true;
1020 return false;
1021}
1022
1023/**
1024 * get_device_system_crosststamp - Synchronously capture system/device timestamp
1025 * @get_time_fn: Callback to get simultaneous device time and
1026 * system counter from the device driver
1027 * @ctx: Context passed to get_time_fn()
1028 * @history_begin: Historical reference point used to interpolate system
1029 * time when counter provided by the driver is before the current interval
1030 * @xtstamp: Receives simultaneously captured system and device time
1031 *
1032 * Reads a timestamp from a device and correlates it to system time
1033 */
1034int get_device_system_crosststamp(int (*get_time_fn)
1035 (ktime_t *device_time,
1036 struct system_counterval_t *sys_counterval,
1037 void *ctx),
1038 void *ctx,
1039 struct system_time_snapshot *history_begin,
1040 struct system_device_crosststamp *xtstamp)
1041{
1042 struct system_counterval_t system_counterval;
1043 struct timekeeper *tk = &tk_core.timekeeper;
1044 cycle_t cycles, now, interval_start;
1045 unsigned int clock_was_set_seq;
1046 ktime_t base_real, base_raw;
1047 s64 nsec_real, nsec_raw;
1048 u8 cs_was_changed_seq;
1049 unsigned long seq;
1050 bool do_interp;
1051 int ret;
879 1052
880 do { 1053 do {
881 seq = read_seqcount_begin(&tk_core.seq); 1054 seq = read_seqcount_begin(&tk_core.seq);
1055 /*
1056 * Try to synchronously capture device time and a system
1057 * counter value calling back into the device driver
1058 */
1059 ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
1060 if (ret)
1061 return ret;
1062
1063 /*
1064 * Verify that the clocksource associated with the captured
1065 * system counter value is the same as the currently installed
1066 * timekeeper clocksource
1067 */
1068 if (tk->tkr_mono.clock != system_counterval.cs)
1069 return -ENODEV;
1070 cycles = system_counterval.cycles;
882 1071
883 *ts_raw = tk->raw_time; 1072 /*
884 ts_real->tv_sec = tk->xtime_sec; 1073 * Check whether the system counter value provided by the
885 ts_real->tv_nsec = 0; 1074 * device driver is on the current timekeeping interval.
1075 */
1076 now = tk->tkr_mono.read(tk->tkr_mono.clock);
1077 interval_start = tk->tkr_mono.cycle_last;
1078 if (!cycle_between(interval_start, cycles, now)) {
1079 clock_was_set_seq = tk->clock_was_set_seq;
1080 cs_was_changed_seq = tk->cs_was_changed_seq;
1081 cycles = interval_start;
1082 do_interp = true;
1083 } else {
1084 do_interp = false;
1085 }
886 1086
887 nsecs_raw = timekeeping_get_ns(&tk->tkr_raw); 1087 base_real = ktime_add(tk->tkr_mono.base,
888 nsecs_real = timekeeping_get_ns(&tk->tkr_mono); 1088 tk_core.timekeeper.offs_real);
1089 base_raw = tk->tkr_raw.base;
889 1090
1091 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono,
1092 system_counterval.cycles);
1093 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw,
1094 system_counterval.cycles);
890 } while (read_seqcount_retry(&tk_core.seq, seq)); 1095 } while (read_seqcount_retry(&tk_core.seq, seq));
891 1096
892 timespec64_add_ns(ts_raw, nsecs_raw); 1097 xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
893 timespec64_add_ns(ts_real, nsecs_real); 1098 xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
894}
895EXPORT_SYMBOL(ktime_get_raw_and_real_ts64);
896 1099
897#endif /* CONFIG_NTP_PPS */ 1100 /*
1101 * Interpolate if necessary, adjusting back from the start of the
1102 * current interval
1103 */
1104 if (do_interp) {
1105 cycle_t partial_history_cycles, total_history_cycles;
1106 bool discontinuity;
1107
1108 /*
1109 * Check that the counter value occurs after the provided
1110 * history reference and that the history doesn't cross a
1111 * clocksource change
1112 */
1113 if (!history_begin ||
1114 !cycle_between(history_begin->cycles,
1115 system_counterval.cycles, cycles) ||
1116 history_begin->cs_was_changed_seq != cs_was_changed_seq)
1117 return -EINVAL;
1118 partial_history_cycles = cycles - system_counterval.cycles;
1119 total_history_cycles = cycles - history_begin->cycles;
1120 discontinuity =
1121 history_begin->clock_was_set_seq != clock_was_set_seq;
1122
1123 ret = adjust_historical_crosststamp(history_begin,
1124 partial_history_cycles,
1125 total_history_cycles,
1126 discontinuity, xtstamp);
1127 if (ret)
1128 return ret;
1129 }
1130
1131 return 0;
1132}
1133EXPORT_SYMBOL_GPL(get_device_system_crosststamp);
898 1134
899/** 1135/**
900 * do_gettimeofday - Returns the time of day in a timeval 1136 * do_gettimeofday - Returns the time of day in a timeval