aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--kernel/time/sched_clock.c107
1 files changed, 56 insertions, 51 deletions
diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c
index eeea1e950b72..a26036d37a38 100644
--- a/kernel/time/sched_clock.c
+++ b/kernel/time/sched_clock.c
@@ -1,5 +1,6 @@
1/* 1/*
2 * sched_clock.c: support for extending counters to full 64-bit ns counter 2 * sched_clock.c: Generic sched_clock() support, to extend low level
3 * hardware time counters to full 64-bit ns values.
3 * 4 *
4 * This program is free software; you can redistribute it and/or modify 5 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 6 * it under the terms of the GNU General Public License version 2 as
@@ -19,15 +20,15 @@
19#include <linux/bitops.h> 20#include <linux/bitops.h>
20 21
21/** 22/**
22 * struct clock_read_data - data required to read from sched_clock 23 * struct clock_read_data - data required to read from sched_clock()
23 * 24 *
24 * @epoch_ns: sched_clock value at last update 25 * @epoch_ns: sched_clock() value at last update
25 * @epoch_cyc: Clock cycle value at last update 26 * @epoch_cyc: Clock cycle value at last update.
26 * @sched_clock_mask: Bitmask for two's complement subtraction of non 64bit 27 * @sched_clock_mask: Bitmask for two's complement subtraction of non 64bit
27 * clocks 28 * clocks.
28 * @read_sched_clock: Current clock source (or dummy source when suspended) 29 * @read_sched_clock: Current clock source (or dummy source when suspended).
29 * @mult: Multipler for scaled math conversion 30 * @mult: Multipler for scaled math conversion.
30 * @shift: Shift value for scaled math conversion 31 * @shift: Shift value for scaled math conversion.
31 * 32 *
32 * Care must be taken when updating this structure; it is read by 33 * Care must be taken when updating this structure; it is read by
33 * some very hot code paths. It occupies <=40 bytes and, when combined 34 * some very hot code paths. It occupies <=40 bytes and, when combined
@@ -44,25 +45,26 @@ struct clock_read_data {
44}; 45};
45 46
46/** 47/**
47 * struct clock_data - all data needed for sched_clock (including 48 * struct clock_data - all data needed for sched_clock() (including
48 * registration of a new clock source) 49 * registration of a new clock source)
49 * 50 *
50 * @seq: Sequence counter for protecting updates. The lowest 51 * @seq: Sequence counter for protecting updates. The lowest
51 * bit is the index for @read_data. 52 * bit is the index for @read_data.
52 * @read_data: Data required to read from sched_clock. 53 * @read_data: Data required to read from sched_clock.
53 * @wrap_kt: Duration for which clock can run before wrapping 54 * @wrap_kt: Duration for which clock can run before wrapping.
54 * @rate: Tick rate of the registered clock 55 * @rate: Tick rate of the registered clock.
55 * @actual_read_sched_clock: Registered clock read function 56 * @actual_read_sched_clock: Registered hardware level clock read function.
56 * 57 *
57 * The ordering of this structure has been chosen to optimize cache 58 * The ordering of this structure has been chosen to optimize cache
58 * performance. In particular seq and read_data[0] (combined) should fit 59 * performance. In particular 'seq' and 'read_data[0]' (combined) should fit
59 * into a single 64 byte cache line. 60 * into a single 64-byte cache line.
60 */ 61 */
61struct clock_data { 62struct clock_data {
62 seqcount_t seq; 63 seqcount_t seq;
63 struct clock_read_data read_data[2]; 64 struct clock_read_data read_data[2];
64 ktime_t wrap_kt; 65 ktime_t wrap_kt;
65 unsigned long rate; 66 unsigned long rate;
67
66 u64 (*actual_read_sched_clock)(void); 68 u64 (*actual_read_sched_clock)(void);
67}; 69};
68 70
@@ -112,10 +114,10 @@ unsigned long long notrace sched_clock(void)
112/* 114/*
113 * Updating the data required to read the clock. 115 * Updating the data required to read the clock.
114 * 116 *
115 * sched_clock will never observe mis-matched data even if called from 117 * sched_clock() will never observe mis-matched data even if called from
116 * an NMI. We do this by maintaining an odd/even copy of the data and 118 * an NMI. We do this by maintaining an odd/even copy of the data and
117 * steering sched_clock to one or the other using a sequence counter. 119 * steering sched_clock() to one or the other using a sequence counter.
118 * In order to preserve the data cache profile of sched_clock as much 120 * In order to preserve the data cache profile of sched_clock() as much
119 * as possible the system reverts back to the even copy when the update 121 * as possible the system reverts back to the even copy when the update
120 * completes; the odd copy is used *only* during an update. 122 * completes; the odd copy is used *only* during an update.
121 */ 123 */
@@ -135,7 +137,7 @@ static void update_clock_read_data(struct clock_read_data *rd)
135} 137}
136 138
137/* 139/*
138 * Atomically update the sched_clock epoch. 140 * Atomically update the sched_clock() epoch.
139 */ 141 */
140static void update_sched_clock(void) 142static void update_sched_clock(void)
141{ 143{
@@ -146,9 +148,7 @@ static void update_sched_clock(void)
146 rd = cd.read_data[0]; 148 rd = cd.read_data[0];
147 149
148 cyc = cd.actual_read_sched_clock(); 150 cyc = cd.actual_read_sched_clock();
149 ns = rd.epoch_ns + 151 ns = rd.epoch_ns + cyc_to_ns((cyc - rd.epoch_cyc) & rd.sched_clock_mask, rd.mult, rd.shift);
150 cyc_to_ns((cyc - rd.epoch_cyc) & rd.sched_clock_mask,
151 rd.mult, rd.shift);
152 152
153 rd.epoch_ns = ns; 153 rd.epoch_ns = ns;
154 rd.epoch_cyc = cyc; 154 rd.epoch_cyc = cyc;
@@ -160,11 +160,12 @@ static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt)
160{ 160{
161 update_sched_clock(); 161 update_sched_clock();
162 hrtimer_forward_now(hrt, cd.wrap_kt); 162 hrtimer_forward_now(hrt, cd.wrap_kt);
163
163 return HRTIMER_RESTART; 164 return HRTIMER_RESTART;
164} 165}
165 166
166void __init sched_clock_register(u64 (*read)(void), int bits, 167void __init
167 unsigned long rate) 168sched_clock_register(u64 (*read)(void), int bits, unsigned long rate)
168{ 169{
169 u64 res, wrap, new_mask, new_epoch, cyc, ns; 170 u64 res, wrap, new_mask, new_epoch, cyc, ns;
170 u32 new_mult, new_shift; 171 u32 new_mult, new_shift;
@@ -177,51 +178,53 @@ void __init sched_clock_register(u64 (*read)(void), int bits,
177 178
178 WARN_ON(!irqs_disabled()); 179 WARN_ON(!irqs_disabled());
179 180
180 /* calculate the mult/shift to convert counter ticks to ns. */ 181 /* Calculate the mult/shift to convert counter ticks to ns. */
181 clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600); 182 clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600);
182 183
183 new_mask = CLOCKSOURCE_MASK(bits); 184 new_mask = CLOCKSOURCE_MASK(bits);
184 cd.rate = rate; 185 cd.rate = rate;
185 186
186 /* calculate how many nanosecs until we risk wrapping */ 187 /* Calculate how many nanosecs until we risk wrapping */
187 wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask, NULL); 188 wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask, NULL);
188 cd.wrap_kt = ns_to_ktime(wrap); 189 cd.wrap_kt = ns_to_ktime(wrap);
189 190
190 rd = cd.read_data[0]; 191 rd = cd.read_data[0];
191 192
192 /* update epoch for new counter and update epoch_ns from old counter*/ 193 /* Update epoch for new counter and update 'epoch_ns' from old counter*/
193 new_epoch = read(); 194 new_epoch = read();
194 cyc = cd.actual_read_sched_clock(); 195 cyc = cd.actual_read_sched_clock();
195 ns = rd.epoch_ns + 196 ns = rd.epoch_ns + cyc_to_ns((cyc - rd.epoch_cyc) & rd.sched_clock_mask, rd.mult, rd.shift);
196 cyc_to_ns((cyc - rd.epoch_cyc) & rd.sched_clock_mask,
197 rd.mult, rd.shift);
198 cd.actual_read_sched_clock = read; 197 cd.actual_read_sched_clock = read;
199 198
200 rd.read_sched_clock = read; 199 rd.read_sched_clock = read;
201 rd.sched_clock_mask = new_mask; 200 rd.sched_clock_mask = new_mask;
202 rd.mult = new_mult; 201 rd.mult = new_mult;
203 rd.shift = new_shift; 202 rd.shift = new_shift;
204 rd.epoch_cyc = new_epoch; 203 rd.epoch_cyc = new_epoch;
205 rd.epoch_ns = ns; 204 rd.epoch_ns = ns;
205
206 update_clock_read_data(&rd); 206 update_clock_read_data(&rd);
207 207
208 r = rate; 208 r = rate;
209 if (r >= 4000000) { 209 if (r >= 4000000) {
210 r /= 1000000; 210 r /= 1000000;
211 r_unit = 'M'; 211 r_unit = 'M';
212 } else if (r >= 1000) { 212 } else {
213 r /= 1000; 213 if (r >= 1000) {
214 r_unit = 'k'; 214 r /= 1000;
215 } else 215 r_unit = 'k';
216 r_unit = ' '; 216 } else {
217 217 r_unit = ' ';
218 /* calculate the ns resolution of this counter */ 218 }
219 }
220
221 /* Calculate the ns resolution of this counter */
219 res = cyc_to_ns(1ULL, new_mult, new_shift); 222 res = cyc_to_ns(1ULL, new_mult, new_shift);
220 223
221 pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n", 224 pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n",
222 bits, r, r_unit, res, wrap); 225 bits, r, r_unit, res, wrap);
223 226
224 /* Enable IRQ time accounting if we have a fast enough sched_clock */ 227 /* Enable IRQ time accounting if we have a fast enough sched_clock() */
225 if (irqtime > 0 || (irqtime == -1 && rate >= 1000000)) 228 if (irqtime > 0 || (irqtime == -1 && rate >= 1000000))
226 enable_sched_clock_irqtime(); 229 enable_sched_clock_irqtime();
227 230
@@ -231,7 +234,7 @@ void __init sched_clock_register(u64 (*read)(void), int bits,
231void __init sched_clock_postinit(void) 234void __init sched_clock_postinit(void)
232{ 235{
233 /* 236 /*
234 * If no sched_clock function has been provided at that point, 237 * If no sched_clock() function has been provided at that point,
235 * make it the final one one. 238 * make it the final one one.
236 */ 239 */
237 if (cd.actual_read_sched_clock == jiffy_sched_clock_read) 240 if (cd.actual_read_sched_clock == jiffy_sched_clock_read)
@@ -257,7 +260,7 @@ void __init sched_clock_postinit(void)
257 * This function must only be called from the critical 260 * This function must only be called from the critical
258 * section in sched_clock(). It relies on the read_seqcount_retry() 261 * section in sched_clock(). It relies on the read_seqcount_retry()
259 * at the end of the critical section to be sure we observe the 262 * at the end of the critical section to be sure we observe the
260 * correct copy of epoch_cyc. 263 * correct copy of 'epoch_cyc'.
261 */ 264 */
262static u64 notrace suspended_sched_clock_read(void) 265static u64 notrace suspended_sched_clock_read(void)
263{ 266{
@@ -273,6 +276,7 @@ static int sched_clock_suspend(void)
273 update_sched_clock(); 276 update_sched_clock();
274 hrtimer_cancel(&sched_clock_timer); 277 hrtimer_cancel(&sched_clock_timer);
275 rd->read_sched_clock = suspended_sched_clock_read; 278 rd->read_sched_clock = suspended_sched_clock_read;
279
276 return 0; 280 return 0;
277} 281}
278 282
@@ -286,13 +290,14 @@ static void sched_clock_resume(void)
286} 290}
287 291
288static struct syscore_ops sched_clock_ops = { 292static struct syscore_ops sched_clock_ops = {
289 .suspend = sched_clock_suspend, 293 .suspend = sched_clock_suspend,
290 .resume = sched_clock_resume, 294 .resume = sched_clock_resume,
291}; 295};
292 296
293static int __init sched_clock_syscore_init(void) 297static int __init sched_clock_syscore_init(void)
294{ 298{
295 register_syscore_ops(&sched_clock_ops); 299 register_syscore_ops(&sched_clock_ops);
300
296 return 0; 301 return 0;
297} 302}
298device_initcall(sched_clock_syscore_init); 303device_initcall(sched_clock_syscore_init);