1 files changed, 83 insertions, 67 deletions
diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h
index c1825eb436ed..95640dcd1899 100644
--- a/include/linux/timekeeper_internal.h
+++ b/include/linux/timekeeper_internal.h
@@ -10,77 +10,100 @@
 #include <linux/jiffies.h>
 #include <linux/time.h>
-/* Structure holding internal timekeeping values. */
+/**
-struct timekeeper {
+ * struct tk_read_base - base structure for timekeeping readout
-        /* Current clocksource used for timekeeping. */
+ * @clock:      Current clocksource used for timekeeping.
+ * @read:       Read function of @clock
+ * @mask:       Bitmask for two's complement subtraction of non 64bit clocks
+ * @cycle_last: @clock cycle value at last update
+ * @mult:       NTP adjusted multiplier for scaled math conversion
+ * @shift:      Shift value for scaled math conversion
+ * @xtime_nsec: Shifted (fractional) nano seconds offset for readout
+ * @base_mono:  ktime_t (nanoseconds) base time for readout
+ *
+ * This struct has size 56 byte on 64 bit. Together with a seqcount it
+ * occupies a single 64byte cache line.
+ *
+ * The struct is separate from struct timekeeper as it is also used
+ * for a fast NMI safe accessor to clock monotonic.
+ */
+struct tk_read_base {
        struct clocksource      *clock;
-        /* NTP adjusted clock multiplier */
+        cycle_t                 (*read)(struct clocksource *cs);
+        cycle_t                 mask;
+        cycle_t                 cycle_last;
        u32                     mult;
-        /* The shift value of the current clocksource. */
        u32                     shift;
-        /* Number of clock cycles in one NTP interval. */
+        u64                     xtime_nsec;
+        ktime_t                 base_mono;
+};
+/**
+ * struct timekeeper - Structure holding internal timekeeping values.
+ * @tkr:                The readout base structure
+ * @xtime_sec:          Current CLOCK_REALTIME time in seconds
+ * @wall_to_monotonic:  CLOCK_REALTIME to CLOCK_MONOTONIC offset
+ * @offs_real:          Offset clock monotonic -> clock realtime
+ * @offs_boot:          Offset clock monotonic -> clock boottime
+ * @offs_tai:           Offset clock monotonic -> clock tai
+ * @tai_offset:         The current UTC to TAI offset in seconds
+ * @base_raw:           Monotonic raw base time in ktime_t format
+ * @raw_time:           Monotonic raw base time in timespec64 format
+ * @cycle_interval:     Number of clock cycles in one NTP interval
+ * @xtime_interval:     Number of clock shifted nano seconds in one NTP
+ *                      interval.
+ * @xtime_remainder:    Shifted nano seconds left over when rounding
+ *                      @cycle_interval
+ * @raw_interval:       Raw nano seconds accumulated per NTP interval.
+ * @ntp_error:          Difference between accumulated time and NTP time in ntp
+ *                      shifted nano seconds.
+ * @ntp_error_shift:    Shift conversion between clock shifted nano seconds and
+ *                      ntp shifted nano seconds.
+ *
+ * Note: For timespec(64) based interfaces wall_to_monotonic is what
+ * we need to add to xtime (or xtime corrected for sub jiffie times)
+ * to get to monotonic time.  Monotonic is pegged at zero at system
+ * boot time, so wall_to_monotonic will be negative, however, we will
+ * ALWAYS keep the tv_nsec part positive so we can use the usual
+ * normalization.
+ *
+ * wall_to_monotonic is moved after resume from suspend for the
+ * monotonic time not to jump. We need to add total_sleep_time to
+ * wall_to_monotonic to get the real boot based time offset.
+ *
+ * wall_to_monotonic is no longer the boot time, getboottime must be
+ * used instead.
+ */
+struct timekeeper {
+        struct tk_read_base     tkr;
+        u64                     xtime_sec;
+        struct timespec64       wall_to_monotonic;
+        ktime_t                 offs_real;
+        ktime_t                 offs_boot;
+        ktime_t                 offs_tai;
+        s32                     tai_offset;
+        ktime_t                 base_raw;
+        struct timespec64       raw_time;
+        /* The following members are for timekeeping internal use */
        cycle_t                 cycle_interval;
-        /* Last cycle value (also stored in clock->cycle_last) */
-        cycle_t                 cycle_last;
-        /* Number of clock shifted nano seconds in one NTP interval. */
        u64                     xtime_interval;
-        /* shifted nano seconds left over when rounding cycle_interval */
        s64                     xtime_remainder;
-        /* Raw nano seconds accumulated per NTP interval. */
        u32                     raw_interval;
+        /* The ntp_tick_length() value currently being used.
-        /* Current CLOCK_REALTIME time in seconds */
+         * This cached copy ensures we consistently apply the tick
-        u64                     xtime_sec;
+         * length for an entire tick, as ntp_tick_length may change
-        /* Clock shifted nano seconds */
+         * mid-tick, and we don't want to apply that new value to
-        u64                     xtime_nsec;
+         * the tick in progress.
+         */
+        u64                     ntp_tick;
        /* Difference between accumulated time and NTP time in ntp
         * shifted nano seconds. */
        s64                     ntp_error;
-        /* Shift conversion between clock shifted nano seconds and
-         * ntp shifted nano seconds. */
        u32                     ntp_error_shift;
+        u32                     ntp_err_mult;
-        /*
-         * wall_to_monotonic is what we need to add to xtime (or xtime corrected
-         * for sub jiffie times) to get to monotonic time.  Monotonic is pegged
-         * at zero at system boot time, so wall_to_monotonic will be negative,
-         * however, we will ALWAYS keep the tv_nsec part positive so we can use
-         * the usual normalization.
-         *
-         * wall_to_monotonic is moved after resume from suspend for the
-         * monotonic time not to jump. We need to add total_sleep_time to
-         * wall_to_monotonic to get the real boot based time offset.
-         *
-         * - wall_to_monotonic is no longer the boot time, getboottime must be
-         * used instead.
-         */
-        struct timespec         wall_to_monotonic;
-        /* Offset clock monotonic -> clock realtime */
-        ktime_t                 offs_real;
-        /* time spent in suspend */
-        struct timespec         total_sleep_time;
-        /* Offset clock monotonic -> clock boottime */
-        ktime_t                 offs_boot;
-        /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */
-        struct timespec         raw_time;
-        /* The current UTC to TAI offset in seconds */
-        s32                     tai_offset;
-        /* Offset clock monotonic -> clock tai */
-        ktime_t                 offs_tai;
 };
-static inline struct timespec tk_xtime(struct timekeeper *tk)
-{
-        struct timespec ts;
-        ts.tv_sec = tk->xtime_sec;
-        ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift);
-        return ts;
-}
 #ifdef CONFIG_GENERIC_TIME_VSYSCALL
 extern void update_vsyscall(struct timekeeper *tk);
@@ -89,17 +112,10 @@ extern void update_vsyscall_tz(void);
 #elif defined(CONFIG_GENERIC_TIME_VSYSCALL_OLD)
 extern void update_vsyscall_old(struct timespec *ts, struct timespec *wtm,
-                                struct clocksource *c, u32 mult);
+                                struct clocksource *c, u32 mult,
+                                cycle_t cycle_last);
 extern void update_vsyscall_tz(void);
-static inline void update_vsyscall(struct timekeeper *tk)
-{
-        struct timespec xt;
-        xt = tk_xtime(tk);
-        update_vsyscall_old(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult);
-}
 #else
 static inline void update_vsyscall(struct timekeeper *tk)

diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index c1825eb436ed..95640dcd1899 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h
@@ -10,77 +10,100 @@
10	#include <linux/jiffies.h>	10	#include <linux/jiffies.h>
11	#include <linux/time.h>	11	#include <linux/time.h>
12		12
13	/* Structure holding internal timekeeping values. */	13	/**
14	struct timekeeper {	14	* struct tk_read_base - base structure for timekeeping readout
15	/* Current clocksource used for timekeeping. */	15	* @clock: Current clocksource used for timekeeping.
		16	* @read: Read function of @clock
		17	* @mask: Bitmask for two's complement subtraction of non 64bit clocks
		18	* @cycle_last: @clock cycle value at last update
		19	* @mult: NTP adjusted multiplier for scaled math conversion
		20	* @shift: Shift value for scaled math conversion
		21	* @xtime_nsec: Shifted (fractional) nano seconds offset for readout
		22	* @base_mono: ktime_t (nanoseconds) base time for readout
		23	*
		24	* This struct has size 56 byte on 64 bit. Together with a seqcount it
		25	* occupies a single 64byte cache line.
		26	*
		27	* The struct is separate from struct timekeeper as it is also used
		28	* for a fast NMI safe accessor to clock monotonic.
		29	*/
		30	struct tk_read_base {
16	struct clocksource *clock;	31	struct clocksource *clock;
17	/* NTP adjusted clock multiplier */	32	cycle_t (read)(struct clocksource cs);
		33	cycle_t mask;
		34	cycle_t cycle_last;
18	u32 mult;	35	u32 mult;
19	/* The shift value of the current clocksource. */
20	u32 shift;	36	u32 shift;
21	/* Number of clock cycles in one NTP interval. */	37	u64 xtime_nsec;
		38	ktime_t base_mono;
		39	};
		40
		41	/**
		42	* struct timekeeper - Structure holding internal timekeeping values.
		43	* @tkr: The readout base structure
		44	* @xtime_sec: Current CLOCK_REALTIME time in seconds
		45	* @wall_to_monotonic: CLOCK_REALTIME to CLOCK_MONOTONIC offset
		46	* @offs_real: Offset clock monotonic -> clock realtime
		47	* @offs_boot: Offset clock monotonic -> clock boottime
		48	* @offs_tai: Offset clock monotonic -> clock tai
		49	* @tai_offset: The current UTC to TAI offset in seconds
		50	* @base_raw: Monotonic raw base time in ktime_t format
		51	* @raw_time: Monotonic raw base time in timespec64 format
		52	* @cycle_interval: Number of clock cycles in one NTP interval
		53	* @xtime_interval: Number of clock shifted nano seconds in one NTP
		54	* interval.
		55	* @xtime_remainder: Shifted nano seconds left over when rounding
		56	* @cycle_interval
		57	* @raw_interval: Raw nano seconds accumulated per NTP interval.
		58	* @ntp_error: Difference between accumulated time and NTP time in ntp
		59	* shifted nano seconds.
		60	* @ntp_error_shift: Shift conversion between clock shifted nano seconds and
		61	* ntp shifted nano seconds.
		62	*
		63	* Note: For timespec(64) based interfaces wall_to_monotonic is what
		64	* we need to add to xtime (or xtime corrected for sub jiffie times)
		65	* to get to monotonic time. Monotonic is pegged at zero at system
		66	* boot time, so wall_to_monotonic will be negative, however, we will
		67	* ALWAYS keep the tv_nsec part positive so we can use the usual
		68	* normalization.
		69	*
		70	* wall_to_monotonic is moved after resume from suspend for the
		71	* monotonic time not to jump. We need to add total_sleep_time to
		72	* wall_to_monotonic to get the real boot based time offset.
		73	*
		74	* wall_to_monotonic is no longer the boot time, getboottime must be
		75	* used instead.
		76	*/
		77	struct timekeeper {
		78	struct tk_read_base tkr;
		79	u64 xtime_sec;
		80	struct timespec64 wall_to_monotonic;
		81	ktime_t offs_real;
		82	ktime_t offs_boot;
		83	ktime_t offs_tai;
		84	s32 tai_offset;
		85	ktime_t base_raw;
		86	struct timespec64 raw_time;
		87
		88	/* The following members are for timekeeping internal use */
22	cycle_t cycle_interval;	89	cycle_t cycle_interval;
23	/* Last cycle value (also stored in clock->cycle_last) */
24	cycle_t cycle_last;
25	/* Number of clock shifted nano seconds in one NTP interval. */
26	u64 xtime_interval;	90	u64 xtime_interval;
27	/* shifted nano seconds left over when rounding cycle_interval */
28	s64 xtime_remainder;	91	s64 xtime_remainder;
29	/* Raw nano seconds accumulated per NTP interval. */
30	u32 raw_interval;	92	u32 raw_interval;
31		93	/* The ntp_tick_length() value currently being used.
32	/* Current CLOCK_REALTIME time in seconds */	94	* This cached copy ensures we consistently apply the tick
33	u64 xtime_sec;	95	* length for an entire tick, as ntp_tick_length may change
34	/* Clock shifted nano seconds */	96	* mid-tick, and we don't want to apply that new value to
35	u64 xtime_nsec;	97	* the tick in progress.
36		98	*/
		99	u64 ntp_tick;
37	/* Difference between accumulated time and NTP time in ntp	100	/* Difference between accumulated time and NTP time in ntp
38	* shifted nano seconds. */	101	* shifted nano seconds. */
39	s64 ntp_error;	102	s64 ntp_error;
40	/* Shift conversion between clock shifted nano seconds and
41	* ntp shifted nano seconds. */
42	u32 ntp_error_shift;	103	u32 ntp_error_shift;
43		104	u32 ntp_err_mult;
44	/*
45	* wall_to_monotonic is what we need to add to xtime (or xtime corrected
46	* for sub jiffie times) to get to monotonic time. Monotonic is pegged
47	* at zero at system boot time, so wall_to_monotonic will be negative,
48	* however, we will ALWAYS keep the tv_nsec part positive so we can use
49	* the usual normalization.
50	*
51	* wall_to_monotonic is moved after resume from suspend for the
52	* monotonic time not to jump. We need to add total_sleep_time to
53	* wall_to_monotonic to get the real boot based time offset.
54	*
55	* - wall_to_monotonic is no longer the boot time, getboottime must be
56	* used instead.
57	*/
58	struct timespec wall_to_monotonic;
59	/* Offset clock monotonic -> clock realtime */
60	ktime_t offs_real;
61	/* time spent in suspend */
62	struct timespec total_sleep_time;
63	/* Offset clock monotonic -> clock boottime */
64	ktime_t offs_boot;
65	/* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */
66	struct timespec raw_time;
67	/* The current UTC to TAI offset in seconds */
68	s32 tai_offset;
69	/* Offset clock monotonic -> clock tai */
70	ktime_t offs_tai;
71
72	};	105	};
73		106
74	static inline struct timespec tk_xtime(struct timekeeper *tk)
75	{
76	struct timespec ts;
77
78	ts.tv_sec = tk->xtime_sec;
79	ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift);
80	return ts;
81	}
82
83
84	#ifdef CONFIG_GENERIC_TIME_VSYSCALL	107	#ifdef CONFIG_GENERIC_TIME_VSYSCALL
85		108
86	extern void update_vsyscall(struct timekeeper *tk);	109	extern void update_vsyscall(struct timekeeper *tk);
@@ -89,17 +112,10 @@ extern void update_vsyscall_tz(void);
89	#elif defined(CONFIG_GENERIC_TIME_VSYSCALL_OLD)	112	#elif defined(CONFIG_GENERIC_TIME_VSYSCALL_OLD)
90		113
91	extern void update_vsyscall_old(struct timespec ts, struct timespec wtm,	114	extern void update_vsyscall_old(struct timespec ts, struct timespec wtm,
92	struct clocksource *c, u32 mult);	115	struct clocksource *c, u32 mult,
		116	cycle_t cycle_last);
93	extern void update_vsyscall_tz(void);	117	extern void update_vsyscall_tz(void);
94		118
95	static inline void update_vsyscall(struct timekeeper *tk)
96	{
97	struct timespec xt;
98
99	xt = tk_xtime(tk);
100	update_vsyscall_old(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult);
101	}
102
103	#else	119	#else
104		120
105	static inline void update_vsyscall(struct timekeeper *tk)	121	static inline void update_vsyscall(struct timekeeper *tk)