diff options
Diffstat (limited to 'arch/i386/kernel/tsc.c')
-rw-r--r-- | arch/i386/kernel/tsc.c | 478 |
1 files changed, 478 insertions, 0 deletions
diff --git a/arch/i386/kernel/tsc.c b/arch/i386/kernel/tsc.c new file mode 100644 index 000000000000..7e0d8dab2075 --- /dev/null +++ b/arch/i386/kernel/tsc.c | |||
@@ -0,0 +1,478 @@ | |||
1 | /* | ||
2 | * This code largely moved from arch/i386/kernel/timer/timer_tsc.c | ||
3 | * which was originally moved from arch/i386/kernel/time.c. | ||
4 | * See comments there for proper credits. | ||
5 | */ | ||
6 | |||
7 | #include <linux/clocksource.h> | ||
8 | #include <linux/workqueue.h> | ||
9 | #include <linux/cpufreq.h> | ||
10 | #include <linux/jiffies.h> | ||
11 | #include <linux/init.h> | ||
12 | #include <linux/dmi.h> | ||
13 | |||
14 | #include <asm/delay.h> | ||
15 | #include <asm/tsc.h> | ||
16 | #include <asm/delay.h> | ||
17 | #include <asm/io.h> | ||
18 | |||
19 | #include "mach_timer.h" | ||
20 | |||
21 | /* | ||
22 | * On some systems the TSC frequency does not | ||
23 | * change with the cpu frequency. So we need | ||
24 | * an extra value to store the TSC freq | ||
25 | */ | ||
26 | unsigned int tsc_khz; | ||
27 | |||
28 | int tsc_disable __cpuinitdata = 0; | ||
29 | |||
30 | #ifdef CONFIG_X86_TSC | ||
31 | static int __init tsc_setup(char *str) | ||
32 | { | ||
33 | printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, " | ||
34 | "cannot disable TSC.\n"); | ||
35 | return 1; | ||
36 | } | ||
37 | #else | ||
38 | /* | ||
39 | * disable flag for tsc. Takes effect by clearing the TSC cpu flag | ||
40 | * in cpu/common.c | ||
41 | */ | ||
42 | static int __init tsc_setup(char *str) | ||
43 | { | ||
44 | tsc_disable = 1; | ||
45 | |||
46 | return 1; | ||
47 | } | ||
48 | #endif | ||
49 | |||
50 | __setup("notsc", tsc_setup); | ||
51 | |||
52 | /* | ||
53 | * code to mark and check if the TSC is unstable | ||
54 | * due to cpufreq or due to unsynced TSCs | ||
55 | */ | ||
56 | static int tsc_unstable; | ||
57 | |||
58 | static inline int check_tsc_unstable(void) | ||
59 | { | ||
60 | return tsc_unstable; | ||
61 | } | ||
62 | |||
63 | void mark_tsc_unstable(void) | ||
64 | { | ||
65 | tsc_unstable = 1; | ||
66 | } | ||
67 | EXPORT_SYMBOL_GPL(mark_tsc_unstable); | ||
68 | |||
69 | /* Accellerators for sched_clock() | ||
70 | * convert from cycles(64bits) => nanoseconds (64bits) | ||
71 | * basic equation: | ||
72 | * ns = cycles / (freq / ns_per_sec) | ||
73 | * ns = cycles * (ns_per_sec / freq) | ||
74 | * ns = cycles * (10^9 / (cpu_khz * 10^3)) | ||
75 | * ns = cycles * (10^6 / cpu_khz) | ||
76 | * | ||
77 | * Then we use scaling math (suggested by george@mvista.com) to get: | ||
78 | * ns = cycles * (10^6 * SC / cpu_khz) / SC | ||
79 | * ns = cycles * cyc2ns_scale / SC | ||
80 | * | ||
81 | * And since SC is a constant power of two, we can convert the div | ||
82 | * into a shift. | ||
83 | * | ||
84 | * We can use khz divisor instead of mhz to keep a better percision, since | ||
85 | * cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits. | ||
86 | * (mathieu.desnoyers@polymtl.ca) | ||
87 | * | ||
88 | * -johnstul@us.ibm.com "math is hard, lets go shopping!" | ||
89 | */ | ||
90 | static unsigned long cyc2ns_scale __read_mostly; | ||
91 | |||
92 | #define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */ | ||
93 | |||
94 | static inline void set_cyc2ns_scale(unsigned long cpu_khz) | ||
95 | { | ||
96 | cyc2ns_scale = (1000000 << CYC2NS_SCALE_FACTOR)/cpu_khz; | ||
97 | } | ||
98 | |||
99 | static inline unsigned long long cycles_2_ns(unsigned long long cyc) | ||
100 | { | ||
101 | return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR; | ||
102 | } | ||
103 | |||
104 | /* | ||
105 | * Scheduler clock - returns current time in nanosec units. | ||
106 | */ | ||
107 | unsigned long long sched_clock(void) | ||
108 | { | ||
109 | unsigned long long this_offset; | ||
110 | |||
111 | /* | ||
112 | * in the NUMA case we dont use the TSC as they are not | ||
113 | * synchronized across all CPUs. | ||
114 | */ | ||
115 | #ifndef CONFIG_NUMA | ||
116 | if (!cpu_khz || check_tsc_unstable()) | ||
117 | #endif | ||
118 | /* no locking but a rare wrong value is not a big deal */ | ||
119 | return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ); | ||
120 | |||
121 | /* read the Time Stamp Counter: */ | ||
122 | rdtscll(this_offset); | ||
123 | |||
124 | /* return the value in ns */ | ||
125 | return cycles_2_ns(this_offset); | ||
126 | } | ||
127 | |||
128 | static unsigned long calculate_cpu_khz(void) | ||
129 | { | ||
130 | unsigned long long start, end; | ||
131 | unsigned long count; | ||
132 | u64 delta64; | ||
133 | int i; | ||
134 | unsigned long flags; | ||
135 | |||
136 | local_irq_save(flags); | ||
137 | |||
138 | /* run 3 times to ensure the cache is warm */ | ||
139 | for (i = 0; i < 3; i++) { | ||
140 | mach_prepare_counter(); | ||
141 | rdtscll(start); | ||
142 | mach_countup(&count); | ||
143 | rdtscll(end); | ||
144 | } | ||
145 | /* | ||
146 | * Error: ECTCNEVERSET | ||
147 | * The CTC wasn't reliable: we got a hit on the very first read, | ||
148 | * or the CPU was so fast/slow that the quotient wouldn't fit in | ||
149 | * 32 bits.. | ||
150 | */ | ||
151 | if (count <= 1) | ||
152 | goto err; | ||
153 | |||
154 | delta64 = end - start; | ||
155 | |||
156 | /* cpu freq too fast: */ | ||
157 | if (delta64 > (1ULL<<32)) | ||
158 | goto err; | ||
159 | |||
160 | /* cpu freq too slow: */ | ||
161 | if (delta64 <= CALIBRATE_TIME_MSEC) | ||
162 | goto err; | ||
163 | |||
164 | delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */ | ||
165 | do_div(delta64,CALIBRATE_TIME_MSEC); | ||
166 | |||
167 | local_irq_restore(flags); | ||
168 | return (unsigned long)delta64; | ||
169 | err: | ||
170 | local_irq_restore(flags); | ||
171 | return 0; | ||
172 | } | ||
173 | |||
174 | int recalibrate_cpu_khz(void) | ||
175 | { | ||
176 | #ifndef CONFIG_SMP | ||
177 | unsigned long cpu_khz_old = cpu_khz; | ||
178 | |||
179 | if (cpu_has_tsc) { | ||
180 | cpu_khz = calculate_cpu_khz(); | ||
181 | tsc_khz = cpu_khz; | ||
182 | cpu_data[0].loops_per_jiffy = | ||
183 | cpufreq_scale(cpu_data[0].loops_per_jiffy, | ||
184 | cpu_khz_old, cpu_khz); | ||
185 | return 0; | ||
186 | } else | ||
187 | return -ENODEV; | ||
188 | #else | ||
189 | return -ENODEV; | ||
190 | #endif | ||
191 | } | ||
192 | |||
193 | EXPORT_SYMBOL(recalibrate_cpu_khz); | ||
194 | |||
195 | void tsc_init(void) | ||
196 | { | ||
197 | if (!cpu_has_tsc || tsc_disable) | ||
198 | return; | ||
199 | |||
200 | cpu_khz = calculate_cpu_khz(); | ||
201 | tsc_khz = cpu_khz; | ||
202 | |||
203 | if (!cpu_khz) | ||
204 | return; | ||
205 | |||
206 | printk("Detected %lu.%03lu MHz processor.\n", | ||
207 | (unsigned long)cpu_khz / 1000, | ||
208 | (unsigned long)cpu_khz % 1000); | ||
209 | |||
210 | set_cyc2ns_scale(cpu_khz); | ||
211 | use_tsc_delay(); | ||
212 | } | ||
213 | |||
214 | #ifdef CONFIG_CPU_FREQ | ||
215 | |||
216 | static unsigned int cpufreq_delayed_issched = 0; | ||
217 | static unsigned int cpufreq_init = 0; | ||
218 | static struct work_struct cpufreq_delayed_get_work; | ||
219 | |||
220 | static void handle_cpufreq_delayed_get(void *v) | ||
221 | { | ||
222 | unsigned int cpu; | ||
223 | |||
224 | for_each_online_cpu(cpu) | ||
225 | cpufreq_get(cpu); | ||
226 | |||
227 | cpufreq_delayed_issched = 0; | ||
228 | } | ||
229 | |||
230 | /* | ||
231 | * if we notice cpufreq oddness, schedule a call to cpufreq_get() as it tries | ||
232 | * to verify the CPU frequency the timing core thinks the CPU is running | ||
233 | * at is still correct. | ||
234 | */ | ||
235 | static inline void cpufreq_delayed_get(void) | ||
236 | { | ||
237 | if (cpufreq_init && !cpufreq_delayed_issched) { | ||
238 | cpufreq_delayed_issched = 1; | ||
239 | printk(KERN_DEBUG "Checking if CPU frequency changed.\n"); | ||
240 | schedule_work(&cpufreq_delayed_get_work); | ||
241 | } | ||
242 | } | ||
243 | |||
244 | /* | ||
245 | * if the CPU frequency is scaled, TSC-based delays will need a different | ||
246 | * loops_per_jiffy value to function properly. | ||
247 | */ | ||
248 | static unsigned int ref_freq = 0; | ||
249 | static unsigned long loops_per_jiffy_ref = 0; | ||
250 | static unsigned long cpu_khz_ref = 0; | ||
251 | |||
252 | static int | ||
253 | time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data) | ||
254 | { | ||
255 | struct cpufreq_freqs *freq = data; | ||
256 | |||
257 | if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE) | ||
258 | write_seqlock_irq(&xtime_lock); | ||
259 | |||
260 | if (!ref_freq) { | ||
261 | if (!freq->old){ | ||
262 | ref_freq = freq->new; | ||
263 | goto end; | ||
264 | } | ||
265 | ref_freq = freq->old; | ||
266 | loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy; | ||
267 | cpu_khz_ref = cpu_khz; | ||
268 | } | ||
269 | |||
270 | if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || | ||
271 | (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) || | ||
272 | (val == CPUFREQ_RESUMECHANGE)) { | ||
273 | if (!(freq->flags & CPUFREQ_CONST_LOOPS)) | ||
274 | cpu_data[freq->cpu].loops_per_jiffy = | ||
275 | cpufreq_scale(loops_per_jiffy_ref, | ||
276 | ref_freq, freq->new); | ||
277 | |||
278 | if (cpu_khz) { | ||
279 | |||
280 | if (num_online_cpus() == 1) | ||
281 | cpu_khz = cpufreq_scale(cpu_khz_ref, | ||
282 | ref_freq, freq->new); | ||
283 | if (!(freq->flags & CPUFREQ_CONST_LOOPS)) { | ||
284 | tsc_khz = cpu_khz; | ||
285 | set_cyc2ns_scale(cpu_khz); | ||
286 | /* | ||
287 | * TSC based sched_clock turns | ||
288 | * to junk w/ cpufreq | ||
289 | */ | ||
290 | mark_tsc_unstable(); | ||
291 | } | ||
292 | } | ||
293 | } | ||
294 | end: | ||
295 | if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE) | ||
296 | write_sequnlock_irq(&xtime_lock); | ||
297 | |||
298 | return 0; | ||
299 | } | ||
300 | |||
301 | static struct notifier_block time_cpufreq_notifier_block = { | ||
302 | .notifier_call = time_cpufreq_notifier | ||
303 | }; | ||
304 | |||
305 | static int __init cpufreq_tsc(void) | ||
306 | { | ||
307 | int ret; | ||
308 | |||
309 | INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get, NULL); | ||
310 | ret = cpufreq_register_notifier(&time_cpufreq_notifier_block, | ||
311 | CPUFREQ_TRANSITION_NOTIFIER); | ||
312 | if (!ret) | ||
313 | cpufreq_init = 1; | ||
314 | |||
315 | return ret; | ||
316 | } | ||
317 | |||
318 | core_initcall(cpufreq_tsc); | ||
319 | |||
320 | #endif | ||
321 | |||
322 | /* clock source code */ | ||
323 | |||
324 | static unsigned long current_tsc_khz = 0; | ||
325 | static int tsc_update_callback(void); | ||
326 | |||
327 | static cycle_t read_tsc(void) | ||
328 | { | ||
329 | cycle_t ret; | ||
330 | |||
331 | rdtscll(ret); | ||
332 | |||
333 | return ret; | ||
334 | } | ||
335 | |||
336 | static struct clocksource clocksource_tsc = { | ||
337 | .name = "tsc", | ||
338 | .rating = 300, | ||
339 | .read = read_tsc, | ||
340 | .mask = CLOCKSOURCE_MASK(64), | ||
341 | .mult = 0, /* to be set */ | ||
342 | .shift = 22, | ||
343 | .update_callback = tsc_update_callback, | ||
344 | .is_continuous = 1, | ||
345 | }; | ||
346 | |||
347 | static int tsc_update_callback(void) | ||
348 | { | ||
349 | int change = 0; | ||
350 | |||
351 | /* check to see if we should switch to the safe clocksource: */ | ||
352 | if (clocksource_tsc.rating != 50 && check_tsc_unstable()) { | ||
353 | clocksource_tsc.rating = 50; | ||
354 | clocksource_reselect(); | ||
355 | change = 1; | ||
356 | } | ||
357 | |||
358 | /* only update if tsc_khz has changed: */ | ||
359 | if (current_tsc_khz != tsc_khz) { | ||
360 | current_tsc_khz = tsc_khz; | ||
361 | clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz, | ||
362 | clocksource_tsc.shift); | ||
363 | change = 1; | ||
364 | } | ||
365 | |||
366 | return change; | ||
367 | } | ||
368 | |||
369 | static int __init dmi_mark_tsc_unstable(struct dmi_system_id *d) | ||
370 | { | ||
371 | printk(KERN_NOTICE "%s detected: marking TSC unstable.\n", | ||
372 | d->ident); | ||
373 | mark_tsc_unstable(); | ||
374 | return 0; | ||
375 | } | ||
376 | |||
377 | /* List of systems that have known TSC problems */ | ||
378 | static struct dmi_system_id __initdata bad_tsc_dmi_table[] = { | ||
379 | { | ||
380 | .callback = dmi_mark_tsc_unstable, | ||
381 | .ident = "IBM Thinkpad 380XD", | ||
382 | .matches = { | ||
383 | DMI_MATCH(DMI_BOARD_VENDOR, "IBM"), | ||
384 | DMI_MATCH(DMI_BOARD_NAME, "2635FA0"), | ||
385 | }, | ||
386 | }, | ||
387 | {} | ||
388 | }; | ||
389 | |||
390 | #define TSC_FREQ_CHECK_INTERVAL (10*MSEC_PER_SEC) /* 10sec in MS */ | ||
391 | static struct timer_list verify_tsc_freq_timer; | ||
392 | |||
393 | /* XXX - Probably should add locking */ | ||
394 | static void verify_tsc_freq(unsigned long unused) | ||
395 | { | ||
396 | static u64 last_tsc; | ||
397 | static unsigned long last_jiffies; | ||
398 | |||
399 | u64 now_tsc, interval_tsc; | ||
400 | unsigned long now_jiffies, interval_jiffies; | ||
401 | |||
402 | |||
403 | if (check_tsc_unstable()) | ||
404 | return; | ||
405 | |||
406 | rdtscll(now_tsc); | ||
407 | now_jiffies = jiffies; | ||
408 | |||
409 | if (!last_jiffies) { | ||
410 | goto out; | ||
411 | } | ||
412 | |||
413 | interval_jiffies = now_jiffies - last_jiffies; | ||
414 | interval_tsc = now_tsc - last_tsc; | ||
415 | interval_tsc *= HZ; | ||
416 | do_div(interval_tsc, cpu_khz*1000); | ||
417 | |||
418 | if (interval_tsc < (interval_jiffies * 3 / 4)) { | ||
419 | printk("TSC appears to be running slowly. " | ||
420 | "Marking it as unstable\n"); | ||
421 | mark_tsc_unstable(); | ||
422 | return; | ||
423 | } | ||
424 | |||
425 | out: | ||
426 | last_tsc = now_tsc; | ||
427 | last_jiffies = now_jiffies; | ||
428 | /* set us up to go off on the next interval: */ | ||
429 | mod_timer(&verify_tsc_freq_timer, | ||
430 | jiffies + msecs_to_jiffies(TSC_FREQ_CHECK_INTERVAL)); | ||
431 | } | ||
432 | |||
433 | /* | ||
434 | * Make an educated guess if the TSC is trustworthy and synchronized | ||
435 | * over all CPUs. | ||
436 | */ | ||
437 | static __init int unsynchronized_tsc(void) | ||
438 | { | ||
439 | /* | ||
440 | * Intel systems are normally all synchronized. | ||
441 | * Exceptions must mark TSC as unstable: | ||
442 | */ | ||
443 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) | ||
444 | return 0; | ||
445 | |||
446 | /* assume multi socket systems are not synchronized: */ | ||
447 | return num_possible_cpus() > 1; | ||
448 | } | ||
449 | |||
450 | static int __init init_tsc_clocksource(void) | ||
451 | { | ||
452 | |||
453 | if (cpu_has_tsc && tsc_khz && !tsc_disable) { | ||
454 | /* check blacklist */ | ||
455 | dmi_check_system(bad_tsc_dmi_table); | ||
456 | |||
457 | if (unsynchronized_tsc()) /* mark unstable if unsynced */ | ||
458 | mark_tsc_unstable(); | ||
459 | current_tsc_khz = tsc_khz; | ||
460 | clocksource_tsc.mult = clocksource_khz2mult(current_tsc_khz, | ||
461 | clocksource_tsc.shift); | ||
462 | /* lower the rating if we already know its unstable: */ | ||
463 | if (check_tsc_unstable()) | ||
464 | clocksource_tsc.rating = 50; | ||
465 | |||
466 | init_timer(&verify_tsc_freq_timer); | ||
467 | verify_tsc_freq_timer.function = verify_tsc_freq; | ||
468 | verify_tsc_freq_timer.expires = | ||
469 | jiffies + msecs_to_jiffies(TSC_FREQ_CHECK_INTERVAL); | ||
470 | add_timer(&verify_tsc_freq_timer); | ||
471 | |||
472 | return clocksource_register(&clocksource_tsc); | ||
473 | } | ||
474 | |||
475 | return 0; | ||
476 | } | ||
477 | |||
478 | module_init(init_tsc_clocksource); | ||