aboutsummaryrefslogtreecommitdiffstats
path: root/arch/i386/kernel/timers
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@ppc970.osdl.org>2005-04-16 18:20:36 -0400
committerLinus Torvalds <torvalds@ppc970.osdl.org>2005-04-16 18:20:36 -0400
commit1da177e4c3f41524e886b7f1b8a0c1fc7321cac2 (patch)
tree0bba044c4ce775e45a88a51686b5d9f90697ea9d /arch/i386/kernel/timers
Linux-2.6.12-rc2v2.6.12-rc2
Initial git repository build. I'm not bothering with the full history, even though we have it. We can create a separate "historical" git archive of that later if we want to, and in the meantime it's about 3.2GB when imported into git - space that would just make the early git days unnecessarily complicated, when we don't have a lot of good infrastructure for it. Let it rip!
Diffstat (limited to 'arch/i386/kernel/timers')
-rw-r--r--arch/i386/kernel/timers/Makefile9
-rw-r--r--arch/i386/kernel/timers/common.c160
-rw-r--r--arch/i386/kernel/timers/timer.c66
-rw-r--r--arch/i386/kernel/timers/timer_cyclone.c259
-rw-r--r--arch/i386/kernel/timers/timer_hpet.c191
-rw-r--r--arch/i386/kernel/timers/timer_none.c39
-rw-r--r--arch/i386/kernel/timers/timer_pit.c206
-rw-r--r--arch/i386/kernel/timers/timer_pm.c258
-rw-r--r--arch/i386/kernel/timers/timer_tsc.c560
9 files changed, 1748 insertions, 0 deletions
diff --git a/arch/i386/kernel/timers/Makefile b/arch/i386/kernel/timers/Makefile
new file mode 100644
index 000000000000..8fa12be658dd
--- /dev/null
+++ b/arch/i386/kernel/timers/Makefile
@@ -0,0 +1,9 @@
1#
2# Makefile for x86 timers
3#
4
5obj-y := timer.o timer_none.o timer_tsc.o timer_pit.o common.o
6
7obj-$(CONFIG_X86_CYCLONE_TIMER) += timer_cyclone.o
8obj-$(CONFIG_HPET_TIMER) += timer_hpet.o
9obj-$(CONFIG_X86_PM_TIMER) += timer_pm.o
diff --git a/arch/i386/kernel/timers/common.c b/arch/i386/kernel/timers/common.c
new file mode 100644
index 000000000000..f7f90005e22e
--- /dev/null
+++ b/arch/i386/kernel/timers/common.c
@@ -0,0 +1,160 @@
1/*
2 * Common functions used across the timers go here
3 */
4
5#include <linux/init.h>
6#include <linux/timex.h>
7#include <linux/errno.h>
8#include <linux/jiffies.h>
9
10#include <asm/io.h>
11#include <asm/timer.h>
12#include <asm/hpet.h>
13
14#include "mach_timer.h"
15
16/* ------ Calibrate the TSC -------
17 * Return 2^32 * (1 / (TSC clocks per usec)) for do_fast_gettimeoffset().
18 * Too much 64-bit arithmetic here to do this cleanly in C, and for
19 * accuracy's sake we want to keep the overhead on the CTC speaker (channel 2)
20 * output busy loop as low as possible. We avoid reading the CTC registers
21 * directly because of the awkward 8-bit access mechanism of the 82C54
22 * device.
23 */
24
25#define CALIBRATE_TIME (5 * 1000020/HZ)
26
27unsigned long __init calibrate_tsc(void)
28{
29 mach_prepare_counter();
30
31 {
32 unsigned long startlow, starthigh;
33 unsigned long endlow, endhigh;
34 unsigned long count;
35
36 rdtsc(startlow,starthigh);
37 mach_countup(&count);
38 rdtsc(endlow,endhigh);
39
40
41 /* Error: ECTCNEVERSET */
42 if (count <= 1)
43 goto bad_ctc;
44
45 /* 64-bit subtract - gcc just messes up with long longs */
46 __asm__("subl %2,%0\n\t"
47 "sbbl %3,%1"
48 :"=a" (endlow), "=d" (endhigh)
49 :"g" (startlow), "g" (starthigh),
50 "0" (endlow), "1" (endhigh));
51
52 /* Error: ECPUTOOFAST */
53 if (endhigh)
54 goto bad_ctc;
55
56 /* Error: ECPUTOOSLOW */
57 if (endlow <= CALIBRATE_TIME)
58 goto bad_ctc;
59
60 __asm__("divl %2"
61 :"=a" (endlow), "=d" (endhigh)
62 :"r" (endlow), "0" (0), "1" (CALIBRATE_TIME));
63
64 return endlow;
65 }
66
67 /*
68 * The CTC wasn't reliable: we got a hit on the very first read,
69 * or the CPU was so fast/slow that the quotient wouldn't fit in
70 * 32 bits..
71 */
72bad_ctc:
73 return 0;
74}
75
76#ifdef CONFIG_HPET_TIMER
77/* ------ Calibrate the TSC using HPET -------
78 * Return 2^32 * (1 / (TSC clocks per usec)) for getting the CPU freq.
79 * Second output is parameter 1 (when non NULL)
80 * Set 2^32 * (1 / (tsc per HPET clk)) for delay_hpet().
81 * calibrate_tsc() calibrates the processor TSC by comparing
82 * it to the HPET timer of known frequency.
83 * Too much 64-bit arithmetic here to do this cleanly in C
84 */
85#define CALIBRATE_CNT_HPET (5 * hpet_tick)
86#define CALIBRATE_TIME_HPET (5 * KERNEL_TICK_USEC)
87
88unsigned long __init calibrate_tsc_hpet(unsigned long *tsc_hpet_quotient_ptr)
89{
90 unsigned long tsc_startlow, tsc_starthigh;
91 unsigned long tsc_endlow, tsc_endhigh;
92 unsigned long hpet_start, hpet_end;
93 unsigned long result, remain;
94
95 hpet_start = hpet_readl(HPET_COUNTER);
96 rdtsc(tsc_startlow, tsc_starthigh);
97 do {
98 hpet_end = hpet_readl(HPET_COUNTER);
99 } while ((hpet_end - hpet_start) < CALIBRATE_CNT_HPET);
100 rdtsc(tsc_endlow, tsc_endhigh);
101
102 /* 64-bit subtract - gcc just messes up with long longs */
103 __asm__("subl %2,%0\n\t"
104 "sbbl %3,%1"
105 :"=a" (tsc_endlow), "=d" (tsc_endhigh)
106 :"g" (tsc_startlow), "g" (tsc_starthigh),
107 "0" (tsc_endlow), "1" (tsc_endhigh));
108
109 /* Error: ECPUTOOFAST */
110 if (tsc_endhigh)
111 goto bad_calibration;
112
113 /* Error: ECPUTOOSLOW */
114 if (tsc_endlow <= CALIBRATE_TIME_HPET)
115 goto bad_calibration;
116
117 ASM_DIV64_REG(result, remain, tsc_endlow, 0, CALIBRATE_TIME_HPET);
118 if (remain > (tsc_endlow >> 1))
119 result++; /* rounding the result */
120
121 if (tsc_hpet_quotient_ptr) {
122 unsigned long tsc_hpet_quotient;
123
124 ASM_DIV64_REG(tsc_hpet_quotient, remain, tsc_endlow, 0,
125 CALIBRATE_CNT_HPET);
126 if (remain > (tsc_endlow >> 1))
127 tsc_hpet_quotient++; /* rounding the result */
128 *tsc_hpet_quotient_ptr = tsc_hpet_quotient;
129 }
130
131 return result;
132bad_calibration:
133 /*
134 * the CPU was so fast/slow that the quotient wouldn't fit in
135 * 32 bits..
136 */
137 return 0;
138}
139#endif
140
141/* calculate cpu_khz */
142void __init init_cpu_khz(void)
143{
144 if (cpu_has_tsc) {
145 unsigned long tsc_quotient = calibrate_tsc();
146 if (tsc_quotient) {
147 /* report CPU clock rate in Hz.
148 * The formula is (10^6 * 2^32) / (2^32 * 1 / (clocks/us)) =
149 * clock/second. Our precision is about 100 ppm.
150 */
151 { unsigned long eax=0, edx=1000;
152 __asm__("divl %2"
153 :"=a" (cpu_khz), "=d" (edx)
154 :"r" (tsc_quotient),
155 "0" (eax), "1" (edx));
156 printk("Detected %lu.%03lu MHz processor.\n", cpu_khz / 1000, cpu_khz % 1000);
157 }
158 }
159 }
160}
diff --git a/arch/i386/kernel/timers/timer.c b/arch/i386/kernel/timers/timer.c
new file mode 100644
index 000000000000..a3d6a288088b
--- /dev/null
+++ b/arch/i386/kernel/timers/timer.c
@@ -0,0 +1,66 @@
1#include <linux/init.h>
2#include <linux/kernel.h>
3#include <linux/string.h>
4#include <asm/timer.h>
5
6#ifdef CONFIG_HPET_TIMER
7/*
8 * HPET memory read is slower than tsc reads, but is more dependable as it
9 * always runs at constant frequency and reduces complexity due to
10 * cpufreq. So, we prefer HPET timer to tsc based one. Also, we cannot use
11 * timer_pit when HPET is active. So, we default to timer_tsc.
12 */
13#endif
14/* list of timers, ordered by preference, NULL terminated */
15static struct init_timer_opts* __initdata timers[] = {
16#ifdef CONFIG_X86_CYCLONE_TIMER
17 &timer_cyclone_init,
18#endif
19#ifdef CONFIG_HPET_TIMER
20 &timer_hpet_init,
21#endif
22#ifdef CONFIG_X86_PM_TIMER
23 &timer_pmtmr_init,
24#endif
25 &timer_tsc_init,
26 &timer_pit_init,
27 NULL,
28};
29
30static char clock_override[10] __initdata;
31
32static int __init clock_setup(char* str)
33{
34 if (str)
35 strlcpy(clock_override, str, sizeof(clock_override));
36 return 1;
37}
38__setup("clock=", clock_setup);
39
40
41/* The chosen timesource has been found to be bad.
42 * Fall back to a known good timesource (the PIT)
43 */
44void clock_fallback(void)
45{
46 cur_timer = &timer_pit;
47}
48
49/* iterates through the list of timers, returning the first
50 * one that initializes successfully.
51 */
52struct timer_opts* __init select_timer(void)
53{
54 int i = 0;
55
56 /* find most preferred working timer */
57 while (timers[i]) {
58 if (timers[i]->init)
59 if (timers[i]->init(clock_override) == 0)
60 return timers[i]->opts;
61 ++i;
62 }
63
64 panic("select_timer: Cannot find a suitable timer\n");
65 return NULL;
66}
diff --git a/arch/i386/kernel/timers/timer_cyclone.c b/arch/i386/kernel/timers/timer_cyclone.c
new file mode 100644
index 000000000000..f6f1206a11bb
--- /dev/null
+++ b/arch/i386/kernel/timers/timer_cyclone.c
@@ -0,0 +1,259 @@
1/* Cyclone-timer:
2 * This code implements timer_ops for the cyclone counter found
3 * on IBM x440, x360, and other Summit based systems.
4 *
5 * Copyright (C) 2002 IBM, John Stultz (johnstul@us.ibm.com)
6 */
7
8
9#include <linux/spinlock.h>
10#include <linux/init.h>
11#include <linux/timex.h>
12#include <linux/errno.h>
13#include <linux/string.h>
14#include <linux/jiffies.h>
15
16#include <asm/timer.h>
17#include <asm/io.h>
18#include <asm/pgtable.h>
19#include <asm/fixmap.h>
20#include "io_ports.h"
21
22extern spinlock_t i8253_lock;
23
24/* Number of usecs that the last interrupt was delayed */
25static int delay_at_last_interrupt;
26
27#define CYCLONE_CBAR_ADDR 0xFEB00CD0
28#define CYCLONE_PMCC_OFFSET 0x51A0
29#define CYCLONE_MPMC_OFFSET 0x51D0
30#define CYCLONE_MPCS_OFFSET 0x51A8
31#define CYCLONE_TIMER_FREQ 100000000
32#define CYCLONE_TIMER_MASK (((u64)1<<40)-1) /* 40 bit mask */
33int use_cyclone = 0;
34
35static u32* volatile cyclone_timer; /* Cyclone MPMC0 register */
36static u32 last_cyclone_low;
37static u32 last_cyclone_high;
38static unsigned long long monotonic_base;
39static seqlock_t monotonic_lock = SEQLOCK_UNLOCKED;
40
41/* helper macro to atomically read both cyclone counter registers */
42#define read_cyclone_counter(low,high) \
43 do{ \
44 high = cyclone_timer[1]; low = cyclone_timer[0]; \
45 } while (high != cyclone_timer[1]);
46
47
48static void mark_offset_cyclone(void)
49{
50 unsigned long lost, delay;
51 unsigned long delta = last_cyclone_low;
52 int count;
53 unsigned long long this_offset, last_offset;
54
55 write_seqlock(&monotonic_lock);
56 last_offset = ((unsigned long long)last_cyclone_high<<32)|last_cyclone_low;
57
58 spin_lock(&i8253_lock);
59 read_cyclone_counter(last_cyclone_low,last_cyclone_high);
60
61 /* read values for delay_at_last_interrupt */
62 outb_p(0x00, 0x43); /* latch the count ASAP */
63
64 count = inb_p(0x40); /* read the latched count */
65 count |= inb(0x40) << 8;
66
67 /*
68 * VIA686a test code... reset the latch if count > max + 1
69 * from timer_pit.c - cjb
70 */
71 if (count > LATCH) {
72 outb_p(0x34, PIT_MODE);
73 outb_p(LATCH & 0xff, PIT_CH0);
74 outb(LATCH >> 8, PIT_CH0);
75 count = LATCH - 1;
76 }
77 spin_unlock(&i8253_lock);
78
79 /* lost tick compensation */
80 delta = last_cyclone_low - delta;
81 delta /= (CYCLONE_TIMER_FREQ/1000000);
82 delta += delay_at_last_interrupt;
83 lost = delta/(1000000/HZ);
84 delay = delta%(1000000/HZ);
85 if (lost >= 2)
86 jiffies_64 += lost-1;
87
88 /* update the monotonic base value */
89 this_offset = ((unsigned long long)last_cyclone_high<<32)|last_cyclone_low;
90 monotonic_base += (this_offset - last_offset) & CYCLONE_TIMER_MASK;
91 write_sequnlock(&monotonic_lock);
92
93 /* calculate delay_at_last_interrupt */
94 count = ((LATCH-1) - count) * TICK_SIZE;
95 delay_at_last_interrupt = (count + LATCH/2) / LATCH;
96
97
98 /* catch corner case where tick rollover occured
99 * between cyclone and pit reads (as noted when
100 * usec delta is > 90% # of usecs/tick)
101 */
102 if (lost && abs(delay - delay_at_last_interrupt) > (900000/HZ))
103 jiffies_64++;
104}
105
106static unsigned long get_offset_cyclone(void)
107{
108 u32 offset;
109
110 if(!cyclone_timer)
111 return delay_at_last_interrupt;
112
113 /* Read the cyclone timer */
114 offset = cyclone_timer[0];
115
116 /* .. relative to previous jiffy */
117 offset = offset - last_cyclone_low;
118
119 /* convert cyclone ticks to microseconds */
120 /* XXX slow, can we speed this up? */
121 offset = offset/(CYCLONE_TIMER_FREQ/1000000);
122
123 /* our adjusted time offset in microseconds */
124 return delay_at_last_interrupt + offset;
125}
126
127static unsigned long long monotonic_clock_cyclone(void)
128{
129 u32 now_low, now_high;
130 unsigned long long last_offset, this_offset, base;
131 unsigned long long ret;
132 unsigned seq;
133
134 /* atomically read monotonic base & last_offset */
135 do {
136 seq = read_seqbegin(&monotonic_lock);
137 last_offset = ((unsigned long long)last_cyclone_high<<32)|last_cyclone_low;
138 base = monotonic_base;
139 } while (read_seqretry(&monotonic_lock, seq));
140
141
142 /* Read the cyclone counter */
143 read_cyclone_counter(now_low,now_high);
144 this_offset = ((unsigned long long)now_high<<32)|now_low;
145
146 /* convert to nanoseconds */
147 ret = base + ((this_offset - last_offset)&CYCLONE_TIMER_MASK);
148 return ret * (1000000000 / CYCLONE_TIMER_FREQ);
149}
150
151static int __init init_cyclone(char* override)
152{
153 u32* reg;
154 u32 base; /* saved cyclone base address */
155 u32 pageaddr; /* page that contains cyclone_timer register */
156 u32 offset; /* offset from pageaddr to cyclone_timer register */
157 int i;
158
159 /* check clock override */
160 if (override[0] && strncmp(override,"cyclone",7))
161 return -ENODEV;
162
163 /*make sure we're on a summit box*/
164 if(!use_cyclone) return -ENODEV;
165
166 printk(KERN_INFO "Summit chipset: Starting Cyclone Counter.\n");
167
168 /* find base address */
169 pageaddr = (CYCLONE_CBAR_ADDR)&PAGE_MASK;
170 offset = (CYCLONE_CBAR_ADDR)&(~PAGE_MASK);
171 set_fixmap_nocache(FIX_CYCLONE_TIMER, pageaddr);
172 reg = (u32*)(fix_to_virt(FIX_CYCLONE_TIMER) + offset);
173 if(!reg){
174 printk(KERN_ERR "Summit chipset: Could not find valid CBAR register.\n");
175 return -ENODEV;
176 }
177 base = *reg;
178 if(!base){
179 printk(KERN_ERR "Summit chipset: Could not find valid CBAR value.\n");
180 return -ENODEV;
181 }
182
183 /* setup PMCC */
184 pageaddr = (base + CYCLONE_PMCC_OFFSET)&PAGE_MASK;
185 offset = (base + CYCLONE_PMCC_OFFSET)&(~PAGE_MASK);
186 set_fixmap_nocache(FIX_CYCLONE_TIMER, pageaddr);
187 reg = (u32*)(fix_to_virt(FIX_CYCLONE_TIMER) + offset);
188 if(!reg){
189 printk(KERN_ERR "Summit chipset: Could not find valid PMCC register.\n");
190 return -ENODEV;
191 }
192 reg[0] = 0x00000001;
193
194 /* setup MPCS */
195 pageaddr = (base + CYCLONE_MPCS_OFFSET)&PAGE_MASK;
196 offset = (base + CYCLONE_MPCS_OFFSET)&(~PAGE_MASK);
197 set_fixmap_nocache(FIX_CYCLONE_TIMER, pageaddr);
198 reg = (u32*)(fix_to_virt(FIX_CYCLONE_TIMER) + offset);
199 if(!reg){
200 printk(KERN_ERR "Summit chipset: Could not find valid MPCS register.\n");
201 return -ENODEV;
202 }
203 reg[0] = 0x00000001;
204
205 /* map in cyclone_timer */
206 pageaddr = (base + CYCLONE_MPMC_OFFSET)&PAGE_MASK;
207 offset = (base + CYCLONE_MPMC_OFFSET)&(~PAGE_MASK);
208 set_fixmap_nocache(FIX_CYCLONE_TIMER, pageaddr);
209 cyclone_timer = (u32*)(fix_to_virt(FIX_CYCLONE_TIMER) + offset);
210 if(!cyclone_timer){
211 printk(KERN_ERR "Summit chipset: Could not find valid MPMC register.\n");
212 return -ENODEV;
213 }
214
215 /*quick test to make sure its ticking*/
216 for(i=0; i<3; i++){
217 u32 old = cyclone_timer[0];
218 int stall = 100;
219 while(stall--) barrier();
220 if(cyclone_timer[0] == old){
221 printk(KERN_ERR "Summit chipset: Counter not counting! DISABLED\n");
222 cyclone_timer = 0;
223 return -ENODEV;
224 }
225 }
226
227 init_cpu_khz();
228
229 /* Everything looks good! */
230 return 0;
231}
232
233
234static void delay_cyclone(unsigned long loops)
235{
236 unsigned long bclock, now;
237 if(!cyclone_timer)
238 return;
239 bclock = cyclone_timer[0];
240 do {
241 rep_nop();
242 now = cyclone_timer[0];
243 } while ((now-bclock) < loops);
244}
245/************************************************************/
246
247/* cyclone timer_opts struct */
248static struct timer_opts timer_cyclone = {
249 .name = "cyclone",
250 .mark_offset = mark_offset_cyclone,
251 .get_offset = get_offset_cyclone,
252 .monotonic_clock = monotonic_clock_cyclone,
253 .delay = delay_cyclone,
254};
255
256struct init_timer_opts __initdata timer_cyclone_init = {
257 .init = init_cyclone,
258 .opts = &timer_cyclone,
259};
diff --git a/arch/i386/kernel/timers/timer_hpet.c b/arch/i386/kernel/timers/timer_hpet.c
new file mode 100644
index 000000000000..713134e71844
--- /dev/null
+++ b/arch/i386/kernel/timers/timer_hpet.c
@@ -0,0 +1,191 @@
1/*
2 * This code largely moved from arch/i386/kernel/time.c.
3 * See comments there for proper credits.
4 */
5
6#include <linux/spinlock.h>
7#include <linux/init.h>
8#include <linux/timex.h>
9#include <linux/errno.h>
10#include <linux/string.h>
11#include <linux/jiffies.h>
12
13#include <asm/timer.h>
14#include <asm/io.h>
15#include <asm/processor.h>
16
17#include "io_ports.h"
18#include "mach_timer.h"
19#include <asm/hpet.h>
20
21static unsigned long hpet_usec_quotient; /* convert hpet clks to usec */
22static unsigned long tsc_hpet_quotient; /* convert tsc to hpet clks */
23static unsigned long hpet_last; /* hpet counter value at last tick*/
24static unsigned long last_tsc_low; /* lsb 32 bits of Time Stamp Counter */
25static unsigned long last_tsc_high; /* msb 32 bits of Time Stamp Counter */
26static unsigned long long monotonic_base;
27static seqlock_t monotonic_lock = SEQLOCK_UNLOCKED;
28
29/* convert from cycles(64bits) => nanoseconds (64bits)
30 * basic equation:
31 * ns = cycles / (freq / ns_per_sec)
32 * ns = cycles * (ns_per_sec / freq)
33 * ns = cycles * (10^9 / (cpu_mhz * 10^6))
34 * ns = cycles * (10^3 / cpu_mhz)
35 *
36 * Then we use scaling math (suggested by george@mvista.com) to get:
37 * ns = cycles * (10^3 * SC / cpu_mhz) / SC
38 * ns = cycles * cyc2ns_scale / SC
39 *
40 * And since SC is a constant power of two, we can convert the div
41 * into a shift.
42 * -johnstul@us.ibm.com "math is hard, lets go shopping!"
43 */
44static unsigned long cyc2ns_scale;
45#define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
46
47static inline void set_cyc2ns_scale(unsigned long cpu_mhz)
48{
49 cyc2ns_scale = (1000 << CYC2NS_SCALE_FACTOR)/cpu_mhz;
50}
51
52static inline unsigned long long cycles_2_ns(unsigned long long cyc)
53{
54 return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
55}
56
57static unsigned long long monotonic_clock_hpet(void)
58{
59 unsigned long long last_offset, this_offset, base;
60 unsigned seq;
61
62 /* atomically read monotonic base & last_offset */
63 do {
64 seq = read_seqbegin(&monotonic_lock);
65 last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
66 base = monotonic_base;
67 } while (read_seqretry(&monotonic_lock, seq));
68
69 /* Read the Time Stamp Counter */
70 rdtscll(this_offset);
71
72 /* return the value in ns */
73 return base + cycles_2_ns(this_offset - last_offset);
74}
75
76static unsigned long get_offset_hpet(void)
77{
78 register unsigned long eax, edx;
79
80 eax = hpet_readl(HPET_COUNTER);
81 eax -= hpet_last; /* hpet delta */
82
83 /*
84 * Time offset = (hpet delta) * ( usecs per HPET clock )
85 * = (hpet delta) * ( usecs per tick / HPET clocks per tick)
86 * = (hpet delta) * ( hpet_usec_quotient ) / (2^32)
87 *
88 * Where,
89 * hpet_usec_quotient = (2^32 * usecs per tick)/HPET clocks per tick
90 *
91 * Using a mull instead of a divl saves some cycles in critical path.
92 */
93 ASM_MUL64_REG(eax, edx, hpet_usec_quotient, eax);
94
95 /* our adjusted time offset in microseconds */
96 return edx;
97}
98
99static void mark_offset_hpet(void)
100{
101 unsigned long long this_offset, last_offset;
102 unsigned long offset;
103
104 write_seqlock(&monotonic_lock);
105 last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
106 rdtsc(last_tsc_low, last_tsc_high);
107
108 offset = hpet_readl(HPET_T0_CMP) - hpet_tick;
109 if (unlikely(((offset - hpet_last) > hpet_tick) && (hpet_last != 0))) {
110 int lost_ticks = (offset - hpet_last) / hpet_tick;
111 jiffies_64 += lost_ticks;
112 }
113 hpet_last = offset;
114
115 /* update the monotonic base value */
116 this_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
117 monotonic_base += cycles_2_ns(this_offset - last_offset);
118 write_sequnlock(&monotonic_lock);
119}
120
121static void delay_hpet(unsigned long loops)
122{
123 unsigned long hpet_start, hpet_end;
124 unsigned long eax;
125
126 /* loops is the number of cpu cycles. Convert it to hpet clocks */
127 ASM_MUL64_REG(eax, loops, tsc_hpet_quotient, loops);
128
129 hpet_start = hpet_readl(HPET_COUNTER);
130 do {
131 rep_nop();
132 hpet_end = hpet_readl(HPET_COUNTER);
133 } while ((hpet_end - hpet_start) < (loops));
134}
135
136static int __init init_hpet(char* override)
137{
138 unsigned long result, remain;
139
140 /* check clock override */
141 if (override[0] && strncmp(override,"hpet",4))
142 return -ENODEV;
143
144 if (!is_hpet_enabled())
145 return -ENODEV;
146
147 printk("Using HPET for gettimeofday\n");
148 if (cpu_has_tsc) {
149 unsigned long tsc_quotient = calibrate_tsc_hpet(&tsc_hpet_quotient);
150 if (tsc_quotient) {
151 /* report CPU clock rate in Hz.
152 * The formula is (10^6 * 2^32) / (2^32 * 1 / (clocks/us)) =
153 * clock/second. Our precision is about 100 ppm.
154 */
155 { unsigned long eax=0, edx=1000;
156 ASM_DIV64_REG(cpu_khz, edx, tsc_quotient,
157 eax, edx);
158 printk("Detected %lu.%03lu MHz processor.\n",
159 cpu_khz / 1000, cpu_khz % 1000);
160 }
161 set_cyc2ns_scale(cpu_khz/1000);
162 }
163 }
164
165 /*
166 * Math to calculate hpet to usec multiplier
167 * Look for the comments at get_offset_hpet()
168 */
169 ASM_DIV64_REG(result, remain, hpet_tick, 0, KERNEL_TICK_USEC);
170 if (remain > (hpet_tick >> 1))
171 result++; /* rounding the result */
172 hpet_usec_quotient = result;
173
174 return 0;
175}
176
177/************************************************************/
178
179/* tsc timer_opts struct */
180static struct timer_opts timer_hpet = {
181 .name = "hpet",
182 .mark_offset = mark_offset_hpet,
183 .get_offset = get_offset_hpet,
184 .monotonic_clock = monotonic_clock_hpet,
185 .delay = delay_hpet,
186};
187
188struct init_timer_opts __initdata timer_hpet_init = {
189 .init = init_hpet,
190 .opts = &timer_hpet,
191};
diff --git a/arch/i386/kernel/timers/timer_none.c b/arch/i386/kernel/timers/timer_none.c
new file mode 100644
index 000000000000..4ea2f414dbbd
--- /dev/null
+++ b/arch/i386/kernel/timers/timer_none.c
@@ -0,0 +1,39 @@
1#include <linux/init.h>
2#include <asm/timer.h>
3
4static void mark_offset_none(void)
5{
6 /* nothing needed */
7}
8
9static unsigned long get_offset_none(void)
10{
11 return 0;
12}
13
14static unsigned long long monotonic_clock_none(void)
15{
16 return 0;
17}
18
19static void delay_none(unsigned long loops)
20{
21 int d0;
22 __asm__ __volatile__(
23 "\tjmp 1f\n"
24 ".align 16\n"
25 "1:\tjmp 2f\n"
26 ".align 16\n"
27 "2:\tdecl %0\n\tjns 2b"
28 :"=&a" (d0)
29 :"0" (loops));
30}
31
32/* none timer_opts struct */
33struct timer_opts timer_none = {
34 .name = "none",
35 .mark_offset = mark_offset_none,
36 .get_offset = get_offset_none,
37 .monotonic_clock = monotonic_clock_none,
38 .delay = delay_none,
39};
diff --git a/arch/i386/kernel/timers/timer_pit.c b/arch/i386/kernel/timers/timer_pit.c
new file mode 100644
index 000000000000..967d5453cd0e
--- /dev/null
+++ b/arch/i386/kernel/timers/timer_pit.c
@@ -0,0 +1,206 @@
1/*
2 * This code largely moved from arch/i386/kernel/time.c.
3 * See comments there for proper credits.
4 */
5
6#include <linux/spinlock.h>
7#include <linux/module.h>
8#include <linux/device.h>
9#include <linux/irq.h>
10#include <linux/sysdev.h>
11#include <linux/timex.h>
12#include <asm/delay.h>
13#include <asm/mpspec.h>
14#include <asm/timer.h>
15#include <asm/smp.h>
16#include <asm/io.h>
17#include <asm/arch_hooks.h>
18
19extern spinlock_t i8259A_lock;
20extern spinlock_t i8253_lock;
21#include "do_timer.h"
22#include "io_ports.h"
23
24static int count_p; /* counter in get_offset_pit() */
25
26static int __init init_pit(char* override)
27{
28 /* check clock override */
29 if (override[0] && strncmp(override,"pit",3))
30 printk(KERN_ERR "Warning: clock= override failed. Defaulting to PIT\n");
31
32 count_p = LATCH;
33 return 0;
34}
35
36static void mark_offset_pit(void)
37{
38 /* nothing needed */
39}
40
41static unsigned long long monotonic_clock_pit(void)
42{
43 return 0;
44}
45
46static void delay_pit(unsigned long loops)
47{
48 int d0;
49 __asm__ __volatile__(
50 "\tjmp 1f\n"
51 ".align 16\n"
52 "1:\tjmp 2f\n"
53 ".align 16\n"
54 "2:\tdecl %0\n\tjns 2b"
55 :"=&a" (d0)
56 :"0" (loops));
57}
58
59
60/* This function must be called with xtime_lock held.
61 * It was inspired by Steve McCanne's microtime-i386 for BSD. -- jrs
62 *
63 * However, the pc-audio speaker driver changes the divisor so that
64 * it gets interrupted rather more often - it loads 64 into the
65 * counter rather than 11932! This has an adverse impact on
66 * do_gettimeoffset() -- it stops working! What is also not
67 * good is that the interval that our timer function gets called
68 * is no longer 10.0002 ms, but 9.9767 ms. To get around this
69 * would require using a different timing source. Maybe someone
70 * could use the RTC - I know that this can interrupt at frequencies
71 * ranging from 8192Hz to 2Hz. If I had the energy, I'd somehow fix
72 * it so that at startup, the timer code in sched.c would select
73 * using either the RTC or the 8253 timer. The decision would be
74 * based on whether there was any other device around that needed
75 * to trample on the 8253. I'd set up the RTC to interrupt at 1024 Hz,
76 * and then do some jiggery to have a version of do_timer that
77 * advanced the clock by 1/1024 s. Every time that reached over 1/100
78 * of a second, then do all the old code. If the time was kept correct
79 * then do_gettimeoffset could just return 0 - there is no low order
80 * divider that can be accessed.
81 *
82 * Ideally, you would be able to use the RTC for the speaker driver,
83 * but it appears that the speaker driver really needs interrupt more
84 * often than every 120 us or so.
85 *
86 * Anyway, this needs more thought.... pjsg (1993-08-28)
87 *
88 * If you are really that interested, you should be reading
89 * comp.protocols.time.ntp!
90 */
91
92static unsigned long get_offset_pit(void)
93{
94 int count;
95 unsigned long flags;
96 static unsigned long jiffies_p = 0;
97
98 /*
99 * cache volatile jiffies temporarily; we have xtime_lock.
100 */
101 unsigned long jiffies_t;
102
103 spin_lock_irqsave(&i8253_lock, flags);
104 /* timer count may underflow right here */
105 outb_p(0x00, PIT_MODE); /* latch the count ASAP */
106
107 count = inb_p(PIT_CH0); /* read the latched count */
108
109 /*
110 * We do this guaranteed double memory access instead of a _p
111 * postfix in the previous port access. Wheee, hackady hack
112 */
113 jiffies_t = jiffies;
114
115 count |= inb_p(PIT_CH0) << 8;
116
117 /* VIA686a test code... reset the latch if count > max + 1 */
118 if (count > LATCH) {
119 outb_p(0x34, PIT_MODE);
120 outb_p(LATCH & 0xff, PIT_CH0);
121 outb(LATCH >> 8, PIT_CH0);
122 count = LATCH - 1;
123 }
124
125 /*
126 * avoiding timer inconsistencies (they are rare, but they happen)...
127 * there are two kinds of problems that must be avoided here:
128 * 1. the timer counter underflows
129 * 2. hardware problem with the timer, not giving us continuous time,
130 * the counter does small "jumps" upwards on some Pentium systems,
131 * (see c't 95/10 page 335 for Neptun bug.)
132 */
133
134 if( jiffies_t == jiffies_p ) {
135 if( count > count_p ) {
136 /* the nutcase */
137 count = do_timer_overflow(count);
138 }
139 } else
140 jiffies_p = jiffies_t;
141
142 count_p = count;
143
144 spin_unlock_irqrestore(&i8253_lock, flags);
145
146 count = ((LATCH-1) - count) * TICK_SIZE;
147 count = (count + LATCH/2) / LATCH;
148
149 return count;
150}
151
152
153/* tsc timer_opts struct */
154struct timer_opts timer_pit = {
155 .name = "pit",
156 .mark_offset = mark_offset_pit,
157 .get_offset = get_offset_pit,
158 .monotonic_clock = monotonic_clock_pit,
159 .delay = delay_pit,
160};
161
162struct init_timer_opts __initdata timer_pit_init = {
163 .init = init_pit,
164 .opts = &timer_pit,
165};
166
167void setup_pit_timer(void)
168{
169 extern spinlock_t i8253_lock;
170 unsigned long flags;
171
172 spin_lock_irqsave(&i8253_lock, flags);
173 outb_p(0x34,PIT_MODE); /* binary, mode 2, LSB/MSB, ch 0 */
174 udelay(10);
175 outb_p(LATCH & 0xff , PIT_CH0); /* LSB */
176 udelay(10);
177 outb(LATCH >> 8 , PIT_CH0); /* MSB */
178 spin_unlock_irqrestore(&i8253_lock, flags);
179}
180
181static int timer_resume(struct sys_device *dev)
182{
183 setup_pit_timer();
184 return 0;
185}
186
187static struct sysdev_class timer_sysclass = {
188 set_kset_name("timer_pit"),
189 .resume = timer_resume,
190};
191
192static struct sys_device device_timer = {
193 .id = 0,
194 .cls = &timer_sysclass,
195};
196
197static int __init init_timer_sysfs(void)
198{
199 int error = sysdev_class_register(&timer_sysclass);
200 if (!error)
201 error = sysdev_register(&device_timer);
202 return error;
203}
204
205device_initcall(init_timer_sysfs);
206
diff --git a/arch/i386/kernel/timers/timer_pm.c b/arch/i386/kernel/timers/timer_pm.c
new file mode 100644
index 000000000000..d77f22030fe6
--- /dev/null
+++ b/arch/i386/kernel/timers/timer_pm.c
@@ -0,0 +1,258 @@
1/*
2 * (C) Dominik Brodowski <linux@brodo.de> 2003
3 *
4 * Driver to use the Power Management Timer (PMTMR) available in some
5 * southbridges as primary timing source for the Linux kernel.
6 *
7 * Based on parts of linux/drivers/acpi/hardware/hwtimer.c, timer_pit.c,
8 * timer_hpet.c, and on Arjan van de Ven's implementation for 2.4.
9 *
10 * This file is licensed under the GPL v2.
11 */
12
13
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/device.h>
17#include <linux/init.h>
18#include <asm/types.h>
19#include <asm/timer.h>
20#include <asm/smp.h>
21#include <asm/io.h>
22#include <asm/arch_hooks.h>
23
24#include <linux/timex.h>
25#include "mach_timer.h"
26
27/* Number of PMTMR ticks expected during calibration run */
28#define PMTMR_TICKS_PER_SEC 3579545
29#define PMTMR_EXPECTED_RATE \
30 ((CALIBRATE_LATCH * (PMTMR_TICKS_PER_SEC >> 10)) / (CLOCK_TICK_RATE>>10))
31
32
33/* The I/O port the PMTMR resides at.
34 * The location is detected during setup_arch(),
35 * in arch/i386/acpi/boot.c */
36u32 pmtmr_ioport = 0;
37
38
39/* value of the Power timer at last timer interrupt */
40static u32 offset_tick;
41static u32 offset_delay;
42
43static unsigned long long monotonic_base;
44static seqlock_t monotonic_lock = SEQLOCK_UNLOCKED;
45
46#define ACPI_PM_MASK 0xFFFFFF /* limit it to 24 bits */
47
48/*helper function to safely read acpi pm timesource*/
49static inline u32 read_pmtmr(void)
50{
51 u32 v1=0,v2=0,v3=0;
52 /* It has been reported that because of various broken
53 * chipsets (ICH4, PIIX4 and PIIX4E) where the ACPI PM time
54 * source is not latched, so you must read it multiple
55 * times to insure a safe value is read.
56 */
57 do {
58 v1 = inl(pmtmr_ioport);
59 v2 = inl(pmtmr_ioport);
60 v3 = inl(pmtmr_ioport);
61 } while ((v1 > v2 && v1 < v3) || (v2 > v3 && v2 < v1)
62 || (v3 > v1 && v3 < v2));
63
64 /* mask the output to 24 bits */
65 return v2 & ACPI_PM_MASK;
66}
67
68
69/*
70 * Some boards have the PMTMR running way too fast. We check
71 * the PMTMR rate against PIT channel 2 to catch these cases.
72 */
73static int verify_pmtmr_rate(void)
74{
75 u32 value1, value2;
76 unsigned long count, delta;
77
78 mach_prepare_counter();
79 value1 = read_pmtmr();
80 mach_countup(&count);
81 value2 = read_pmtmr();
82 delta = (value2 - value1) & ACPI_PM_MASK;
83
84 /* Check that the PMTMR delta is within 5% of what we expect */
85 if (delta < (PMTMR_EXPECTED_RATE * 19) / 20 ||
86 delta > (PMTMR_EXPECTED_RATE * 21) / 20) {
87 printk(KERN_INFO "PM-Timer running at invalid rate: %lu%% of normal - aborting.\n", 100UL * delta / PMTMR_EXPECTED_RATE);
88 return -1;
89 }
90
91 return 0;
92}
93
94
95static int init_pmtmr(char* override)
96{
97 u32 value1, value2;
98 unsigned int i;
99
100 if (override[0] && strncmp(override,"pmtmr",5))
101 return -ENODEV;
102
103 if (!pmtmr_ioport)
104 return -ENODEV;
105
106 /* we use the TSC for delay_pmtmr, so make sure it exists */
107 if (!cpu_has_tsc)
108 return -ENODEV;
109
110 /* "verify" this timing source */
111 value1 = read_pmtmr();
112 for (i = 0; i < 10000; i++) {
113 value2 = read_pmtmr();
114 if (value2 == value1)
115 continue;
116 if (value2 > value1)
117 goto pm_good;
118 if ((value2 < value1) && ((value2) < 0xFFF))
119 goto pm_good;
120 printk(KERN_INFO "PM-Timer had inconsistent results: 0x%#x, 0x%#x - aborting.\n", value1, value2);
121 return -EINVAL;
122 }
123 printk(KERN_INFO "PM-Timer had no reasonable result: 0x%#x - aborting.\n", value1);
124 return -ENODEV;
125
126pm_good:
127 if (verify_pmtmr_rate() != 0)
128 return -ENODEV;
129
130 init_cpu_khz();
131 return 0;
132}
133
134static inline u32 cyc2us(u32 cycles)
135{
136 /* The Power Management Timer ticks at 3.579545 ticks per microsecond.
137 * 1 / PM_TIMER_FREQUENCY == 0.27936511 =~ 286/1024 [error: 0.024%]
138 *
139 * Even with HZ = 100, delta is at maximum 35796 ticks, so it can
140 * easily be multiplied with 286 (=0x11E) without having to fear
141 * u32 overflows.
142 */
143 cycles *= 286;
144 return (cycles >> 10);
145}
146
147/*
148 * this gets called during each timer interrupt
149 * - Called while holding the writer xtime_lock
150 */
151static void mark_offset_pmtmr(void)
152{
153 u32 lost, delta, last_offset;
154 static int first_run = 1;
155 last_offset = offset_tick;
156
157 write_seqlock(&monotonic_lock);
158
159 offset_tick = read_pmtmr();
160
161 /* calculate tick interval */
162 delta = (offset_tick - last_offset) & ACPI_PM_MASK;
163
164 /* convert to usecs */
165 delta = cyc2us(delta);
166
167 /* update the monotonic base value */
168 monotonic_base += delta * NSEC_PER_USEC;
169 write_sequnlock(&monotonic_lock);
170
171 /* convert to ticks */
172 delta += offset_delay;
173 lost = delta / (USEC_PER_SEC / HZ);
174 offset_delay = delta % (USEC_PER_SEC / HZ);
175
176
177 /* compensate for lost ticks */
178 if (lost >= 2)
179 jiffies_64 += lost - 1;
180
181 /* don't calculate delay for first run,
182 or if we've got less then a tick */
183 if (first_run || (lost < 1)) {
184 first_run = 0;
185 offset_delay = 0;
186 }
187}
188
189
190static unsigned long long monotonic_clock_pmtmr(void)
191{
192 u32 last_offset, this_offset;
193 unsigned long long base, ret;
194 unsigned seq;
195
196
197 /* atomically read monotonic base & last_offset */
198 do {
199 seq = read_seqbegin(&monotonic_lock);
200 last_offset = offset_tick;
201 base = monotonic_base;
202 } while (read_seqretry(&monotonic_lock, seq));
203
204 /* Read the pmtmr */
205 this_offset = read_pmtmr();
206
207 /* convert to nanoseconds */
208 ret = (this_offset - last_offset) & ACPI_PM_MASK;
209 ret = base + (cyc2us(ret) * NSEC_PER_USEC);
210 return ret;
211}
212
213static void delay_pmtmr(unsigned long loops)
214{
215 unsigned long bclock, now;
216
217 rdtscl(bclock);
218 do
219 {
220 rep_nop();
221 rdtscl(now);
222 } while ((now-bclock) < loops);
223}
224
225
226/*
227 * get the offset (in microseconds) from the last call to mark_offset()
228 * - Called holding a reader xtime_lock
229 */
230static unsigned long get_offset_pmtmr(void)
231{
232 u32 now, offset, delta = 0;
233
234 offset = offset_tick;
235 now = read_pmtmr();
236 delta = (now - offset)&ACPI_PM_MASK;
237
238 return (unsigned long) offset_delay + cyc2us(delta);
239}
240
241
242/* acpi timer_opts struct */
243static struct timer_opts timer_pmtmr = {
244 .name = "pmtmr",
245 .mark_offset = mark_offset_pmtmr,
246 .get_offset = get_offset_pmtmr,
247 .monotonic_clock = monotonic_clock_pmtmr,
248 .delay = delay_pmtmr,
249};
250
251struct init_timer_opts __initdata timer_pmtmr_init = {
252 .init = init_pmtmr,
253 .opts = &timer_pmtmr,
254};
255
256MODULE_LICENSE("GPL");
257MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>");
258MODULE_DESCRIPTION("Power Management Timer (PMTMR) as primary timing source for x86");
diff --git a/arch/i386/kernel/timers/timer_tsc.c b/arch/i386/kernel/timers/timer_tsc.c
new file mode 100644
index 000000000000..a685994e5c8e
--- /dev/null
+++ b/arch/i386/kernel/timers/timer_tsc.c
@@ -0,0 +1,560 @@
1/*
2 * This code largely moved from arch/i386/kernel/time.c.
3 * See comments there for proper credits.
4 *
5 * 2004-06-25 Jesper Juhl
6 * moved mark_offset_tsc below cpufreq_delayed_get to avoid gcc 3.4
7 * failing to inline.
8 */
9
10#include <linux/spinlock.h>
11#include <linux/init.h>
12#include <linux/timex.h>
13#include <linux/errno.h>
14#include <linux/cpufreq.h>
15#include <linux/string.h>
16#include <linux/jiffies.h>
17
18#include <asm/timer.h>
19#include <asm/io.h>
20/* processor.h for distable_tsc flag */
21#include <asm/processor.h>
22
23#include "io_ports.h"
24#include "mach_timer.h"
25
26#include <asm/hpet.h>
27
28#ifdef CONFIG_HPET_TIMER
29static unsigned long hpet_usec_quotient;
30static unsigned long hpet_last;
31static struct timer_opts timer_tsc;
32#endif
33
34static inline void cpufreq_delayed_get(void);
35
36int tsc_disable __initdata = 0;
37
38extern spinlock_t i8253_lock;
39
40static int use_tsc;
41/* Number of usecs that the last interrupt was delayed */
42static int delay_at_last_interrupt;
43
44static unsigned long last_tsc_low; /* lsb 32 bits of Time Stamp Counter */
45static unsigned long last_tsc_high; /* msb 32 bits of Time Stamp Counter */
46static unsigned long long monotonic_base;
47static seqlock_t monotonic_lock = SEQLOCK_UNLOCKED;
48
49/* convert from cycles(64bits) => nanoseconds (64bits)
50 * basic equation:
51 * ns = cycles / (freq / ns_per_sec)
52 * ns = cycles * (ns_per_sec / freq)
53 * ns = cycles * (10^9 / (cpu_mhz * 10^6))
54 * ns = cycles * (10^3 / cpu_mhz)
55 *
56 * Then we use scaling math (suggested by george@mvista.com) to get:
57 * ns = cycles * (10^3 * SC / cpu_mhz) / SC
58 * ns = cycles * cyc2ns_scale / SC
59 *
60 * And since SC is a constant power of two, we can convert the div
61 * into a shift.
62 * -johnstul@us.ibm.com "math is hard, lets go shopping!"
63 */
64static unsigned long cyc2ns_scale;
65#define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
66
67static inline void set_cyc2ns_scale(unsigned long cpu_mhz)
68{
69 cyc2ns_scale = (1000 << CYC2NS_SCALE_FACTOR)/cpu_mhz;
70}
71
72static inline unsigned long long cycles_2_ns(unsigned long long cyc)
73{
74 return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
75}
76
77static int count2; /* counter for mark_offset_tsc() */
78
79/* Cached *multiplier* to convert TSC counts to microseconds.
80 * (see the equation below).
81 * Equal to 2^32 * (1 / (clocks per usec) ).
82 * Initialized in time_init.
83 */
84static unsigned long fast_gettimeoffset_quotient;
85
86static unsigned long get_offset_tsc(void)
87{
88 register unsigned long eax, edx;
89
90 /* Read the Time Stamp Counter */
91
92 rdtsc(eax,edx);
93
94 /* .. relative to previous jiffy (32 bits is enough) */
95 eax -= last_tsc_low; /* tsc_low delta */
96
97 /*
98 * Time offset = (tsc_low delta) * fast_gettimeoffset_quotient
99 * = (tsc_low delta) * (usecs_per_clock)
100 * = (tsc_low delta) * (usecs_per_jiffy / clocks_per_jiffy)
101 *
102 * Using a mull instead of a divl saves up to 31 clock cycles
103 * in the critical path.
104 */
105
106 __asm__("mull %2"
107 :"=a" (eax), "=d" (edx)
108 :"rm" (fast_gettimeoffset_quotient),
109 "0" (eax));
110
111 /* our adjusted time offset in microseconds */
112 return delay_at_last_interrupt + edx;
113}
114
115static unsigned long long monotonic_clock_tsc(void)
116{
117 unsigned long long last_offset, this_offset, base;
118 unsigned seq;
119
120 /* atomically read monotonic base & last_offset */
121 do {
122 seq = read_seqbegin(&monotonic_lock);
123 last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
124 base = monotonic_base;
125 } while (read_seqretry(&monotonic_lock, seq));
126
127 /* Read the Time Stamp Counter */
128 rdtscll(this_offset);
129
130 /* return the value in ns */
131 return base + cycles_2_ns(this_offset - last_offset);
132}
133
134/*
135 * Scheduler clock - returns current time in nanosec units.
136 */
137unsigned long long sched_clock(void)
138{
139 unsigned long long this_offset;
140
141 /*
142 * In the NUMA case we dont use the TSC as they are not
143 * synchronized across all CPUs.
144 */
145#ifndef CONFIG_NUMA
146 if (!use_tsc)
147#endif
148 /* no locking but a rare wrong value is not a big deal */
149 return jiffies_64 * (1000000000 / HZ);
150
151 /* Read the Time Stamp Counter */
152 rdtscll(this_offset);
153
154 /* return the value in ns */
155 return cycles_2_ns(this_offset);
156}
157
158static void delay_tsc(unsigned long loops)
159{
160 unsigned long bclock, now;
161
162 rdtscl(bclock);
163 do
164 {
165 rep_nop();
166 rdtscl(now);
167 } while ((now-bclock) < loops);
168}
169
170#ifdef CONFIG_HPET_TIMER
171static void mark_offset_tsc_hpet(void)
172{
173 unsigned long long this_offset, last_offset;
174 unsigned long offset, temp, hpet_current;
175
176 write_seqlock(&monotonic_lock);
177 last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
178 /*
179 * It is important that these two operations happen almost at
180 * the same time. We do the RDTSC stuff first, since it's
181 * faster. To avoid any inconsistencies, we need interrupts
182 * disabled locally.
183 */
184 /*
185 * Interrupts are just disabled locally since the timer irq
186 * has the SA_INTERRUPT flag set. -arca
187 */
188 /* read Pentium cycle counter */
189
190 hpet_current = hpet_readl(HPET_COUNTER);
191 rdtsc(last_tsc_low, last_tsc_high);
192
193 /* lost tick compensation */
194 offset = hpet_readl(HPET_T0_CMP) - hpet_tick;
195 if (unlikely(((offset - hpet_last) > hpet_tick) && (hpet_last != 0))) {
196 int lost_ticks = (offset - hpet_last) / hpet_tick;
197 jiffies_64 += lost_ticks;
198 }
199 hpet_last = hpet_current;
200
201 /* update the monotonic base value */
202 this_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
203 monotonic_base += cycles_2_ns(this_offset - last_offset);
204 write_sequnlock(&monotonic_lock);
205
206 /* calculate delay_at_last_interrupt */
207 /*
208 * Time offset = (hpet delta) * ( usecs per HPET clock )
209 * = (hpet delta) * ( usecs per tick / HPET clocks per tick)
210 * = (hpet delta) * ( hpet_usec_quotient ) / (2^32)
211 * Where,
212 * hpet_usec_quotient = (2^32 * usecs per tick)/HPET clocks per tick
213 */
214 delay_at_last_interrupt = hpet_current - offset;
215 ASM_MUL64_REG(temp, delay_at_last_interrupt,
216 hpet_usec_quotient, delay_at_last_interrupt);
217}
218#endif
219
220
221#ifdef CONFIG_CPU_FREQ
222#include <linux/workqueue.h>
223
224static unsigned int cpufreq_delayed_issched = 0;
225static unsigned int cpufreq_init = 0;
226static struct work_struct cpufreq_delayed_get_work;
227
228static void handle_cpufreq_delayed_get(void *v)
229{
230 unsigned int cpu;
231 for_each_online_cpu(cpu) {
232 cpufreq_get(cpu);
233 }
234 cpufreq_delayed_issched = 0;
235}
236
237/* if we notice lost ticks, schedule a call to cpufreq_get() as it tries
238 * to verify the CPU frequency the timing core thinks the CPU is running
239 * at is still correct.
240 */
241static inline void cpufreq_delayed_get(void)
242{
243 if (cpufreq_init && !cpufreq_delayed_issched) {
244 cpufreq_delayed_issched = 1;
245 printk(KERN_DEBUG "Losing some ticks... checking if CPU frequency changed.\n");
246 schedule_work(&cpufreq_delayed_get_work);
247 }
248}
249
250/* If the CPU frequency is scaled, TSC-based delays will need a different
251 * loops_per_jiffy value to function properly.
252 */
253
254static unsigned int ref_freq = 0;
255static unsigned long loops_per_jiffy_ref = 0;
256
257#ifndef CONFIG_SMP
258static unsigned long fast_gettimeoffset_ref = 0;
259static unsigned long cpu_khz_ref = 0;
260#endif
261
262static int
263time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
264 void *data)
265{
266 struct cpufreq_freqs *freq = data;
267
268 if (val != CPUFREQ_RESUMECHANGE)
269 write_seqlock_irq(&xtime_lock);
270 if (!ref_freq) {
271 ref_freq = freq->old;
272 loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy;
273#ifndef CONFIG_SMP
274 fast_gettimeoffset_ref = fast_gettimeoffset_quotient;
275 cpu_khz_ref = cpu_khz;
276#endif
277 }
278
279 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) ||
280 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
281 (val == CPUFREQ_RESUMECHANGE)) {
282 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
283 cpu_data[freq->cpu].loops_per_jiffy = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
284#ifndef CONFIG_SMP
285 if (cpu_khz)
286 cpu_khz = cpufreq_scale(cpu_khz_ref, ref_freq, freq->new);
287 if (use_tsc) {
288 if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
289 fast_gettimeoffset_quotient = cpufreq_scale(fast_gettimeoffset_ref, freq->new, ref_freq);
290 set_cyc2ns_scale(cpu_khz/1000);
291 }
292 }
293#endif
294 }
295
296 if (val != CPUFREQ_RESUMECHANGE)
297 write_sequnlock_irq(&xtime_lock);
298
299 return 0;
300}
301
302static struct notifier_block time_cpufreq_notifier_block = {
303 .notifier_call = time_cpufreq_notifier
304};
305
306
307static int __init cpufreq_tsc(void)
308{
309 int ret;
310 INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get, NULL);
311 ret = cpufreq_register_notifier(&time_cpufreq_notifier_block,
312 CPUFREQ_TRANSITION_NOTIFIER);
313 if (!ret)
314 cpufreq_init = 1;
315 return ret;
316}
317core_initcall(cpufreq_tsc);
318
319#else /* CONFIG_CPU_FREQ */
320static inline void cpufreq_delayed_get(void) { return; }
321#endif
322
323static void mark_offset_tsc(void)
324{
325 unsigned long lost,delay;
326 unsigned long delta = last_tsc_low;
327 int count;
328 int countmp;
329 static int count1 = 0;
330 unsigned long long this_offset, last_offset;
331 static int lost_count = 0;
332
333 write_seqlock(&monotonic_lock);
334 last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
335 /*
336 * It is important that these two operations happen almost at
337 * the same time. We do the RDTSC stuff first, since it's
338 * faster. To avoid any inconsistencies, we need interrupts
339 * disabled locally.
340 */
341
342 /*
343 * Interrupts are just disabled locally since the timer irq
344 * has the SA_INTERRUPT flag set. -arca
345 */
346
347 /* read Pentium cycle counter */
348
349 rdtsc(last_tsc_low, last_tsc_high);
350
351 spin_lock(&i8253_lock);
352 outb_p(0x00, PIT_MODE); /* latch the count ASAP */
353
354 count = inb_p(PIT_CH0); /* read the latched count */
355 count |= inb(PIT_CH0) << 8;
356
357 /*
358 * VIA686a test code... reset the latch if count > max + 1
359 * from timer_pit.c - cjb
360 */
361 if (count > LATCH) {
362 outb_p(0x34, PIT_MODE);
363 outb_p(LATCH & 0xff, PIT_CH0);
364 outb(LATCH >> 8, PIT_CH0);
365 count = LATCH - 1;
366 }
367
368 spin_unlock(&i8253_lock);
369
370 if (pit_latch_buggy) {
371 /* get center value of last 3 time lutch */
372 if ((count2 >= count && count >= count1)
373 || (count1 >= count && count >= count2)) {
374 count2 = count1; count1 = count;
375 } else if ((count1 >= count2 && count2 >= count)
376 || (count >= count2 && count2 >= count1)) {
377 countmp = count;count = count2;
378 count2 = count1;count1 = countmp;
379 } else {
380 count2 = count1; count1 = count; count = count1;
381 }
382 }
383
384 /* lost tick compensation */
385 delta = last_tsc_low - delta;
386 {
387 register unsigned long eax, edx;
388 eax = delta;
389 __asm__("mull %2"
390 :"=a" (eax), "=d" (edx)
391 :"rm" (fast_gettimeoffset_quotient),
392 "0" (eax));
393 delta = edx;
394 }
395 delta += delay_at_last_interrupt;
396 lost = delta/(1000000/HZ);
397 delay = delta%(1000000/HZ);
398 if (lost >= 2) {
399 jiffies_64 += lost-1;
400
401 /* sanity check to ensure we're not always losing ticks */
402 if (lost_count++ > 100) {
403 printk(KERN_WARNING "Losing too many ticks!\n");
404 printk(KERN_WARNING "TSC cannot be used as a timesource. \n");
405 printk(KERN_WARNING "Possible reasons for this are:\n");
406 printk(KERN_WARNING " You're running with Speedstep,\n");
407 printk(KERN_WARNING " You don't have DMA enabled for your hard disk (see hdparm),\n");
408 printk(KERN_WARNING " Incorrect TSC synchronization on an SMP system (see dmesg).\n");
409 printk(KERN_WARNING "Falling back to a sane timesource now.\n");
410
411 clock_fallback();
412 }
413 /* ... but give the TSC a fair chance */
414 if (lost_count > 25)
415 cpufreq_delayed_get();
416 } else
417 lost_count = 0;
418 /* update the monotonic base value */
419 this_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
420 monotonic_base += cycles_2_ns(this_offset - last_offset);
421 write_sequnlock(&monotonic_lock);
422
423 /* calculate delay_at_last_interrupt */
424 count = ((LATCH-1) - count) * TICK_SIZE;
425 delay_at_last_interrupt = (count + LATCH/2) / LATCH;
426
427 /* catch corner case where tick rollover occured
428 * between tsc and pit reads (as noted when
429 * usec delta is > 90% # of usecs/tick)
430 */
431 if (lost && abs(delay - delay_at_last_interrupt) > (900000/HZ))
432 jiffies_64++;
433}
434
435static int __init init_tsc(char* override)
436{
437
438 /* check clock override */
439 if (override[0] && strncmp(override,"tsc",3)) {
440#ifdef CONFIG_HPET_TIMER
441 if (is_hpet_enabled()) {
442 printk(KERN_ERR "Warning: clock= override failed. Defaulting to tsc\n");
443 } else
444#endif
445 {
446 return -ENODEV;
447 }
448 }
449
450 /*
451 * If we have APM enabled or the CPU clock speed is variable
452 * (CPU stops clock on HLT or slows clock to save power)
453 * then the TSC timestamps may diverge by up to 1 jiffy from
454 * 'real time' but nothing will break.
455 * The most frequent case is that the CPU is "woken" from a halt
456 * state by the timer interrupt itself, so we get 0 error. In the
457 * rare cases where a driver would "wake" the CPU and request a
458 * timestamp, the maximum error is < 1 jiffy. But timestamps are
459 * still perfectly ordered.
460 * Note that the TSC counter will be reset if APM suspends
461 * to disk; this won't break the kernel, though, 'cuz we're
462 * smart. See arch/i386/kernel/apm.c.
463 */
464 /*
465 * Firstly we have to do a CPU check for chips with
466 * a potentially buggy TSC. At this point we haven't run
467 * the ident/bugs checks so we must run this hook as it
468 * may turn off the TSC flag.
469 *
470 * NOTE: this doesn't yet handle SMP 486 machines where only
471 * some CPU's have a TSC. Thats never worked and nobody has
472 * moaned if you have the only one in the world - you fix it!
473 */
474
475 count2 = LATCH; /* initialize counter for mark_offset_tsc() */
476
477 if (cpu_has_tsc) {
478 unsigned long tsc_quotient;
479#ifdef CONFIG_HPET_TIMER
480 if (is_hpet_enabled()){
481 unsigned long result, remain;
482 printk("Using TSC for gettimeofday\n");
483 tsc_quotient = calibrate_tsc_hpet(NULL);
484 timer_tsc.mark_offset = &mark_offset_tsc_hpet;
485 /*
486 * Math to calculate hpet to usec multiplier
487 * Look for the comments at get_offset_tsc_hpet()
488 */
489 ASM_DIV64_REG(result, remain, hpet_tick,
490 0, KERNEL_TICK_USEC);
491 if (remain > (hpet_tick >> 1))
492 result++; /* rounding the result */
493
494 hpet_usec_quotient = result;
495 } else
496#endif
497 {
498 tsc_quotient = calibrate_tsc();
499 }
500
501 if (tsc_quotient) {
502 fast_gettimeoffset_quotient = tsc_quotient;
503 use_tsc = 1;
504 /*
505 * We could be more selective here I suspect
506 * and just enable this for the next intel chips ?
507 */
508 /* report CPU clock rate in Hz.
509 * The formula is (10^6 * 2^32) / (2^32 * 1 / (clocks/us)) =
510 * clock/second. Our precision is about 100 ppm.
511 */
512 { unsigned long eax=0, edx=1000;
513 __asm__("divl %2"
514 :"=a" (cpu_khz), "=d" (edx)
515 :"r" (tsc_quotient),
516 "0" (eax), "1" (edx));
517 printk("Detected %lu.%03lu MHz processor.\n", cpu_khz / 1000, cpu_khz % 1000);
518 }
519 set_cyc2ns_scale(cpu_khz/1000);
520 return 0;
521 }
522 }
523 return -ENODEV;
524}
525
526#ifndef CONFIG_X86_TSC
527/* disable flag for tsc. Takes effect by clearing the TSC cpu flag
528 * in cpu/common.c */
529static int __init tsc_setup(char *str)
530{
531 tsc_disable = 1;
532 return 1;
533}
534#else
535static int __init tsc_setup(char *str)
536{
537 printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
538 "cannot disable TSC.\n");
539 return 1;
540}
541#endif
542__setup("notsc", tsc_setup);
543
544
545
546/************************************************************/
547
548/* tsc timer_opts struct */
549static struct timer_opts timer_tsc = {
550 .name = "tsc",
551 .mark_offset = mark_offset_tsc,
552 .get_offset = get_offset_tsc,
553 .monotonic_clock = monotonic_clock_tsc,
554 .delay = delay_tsc,
555};
556
557struct init_timer_opts __initdata timer_tsc_init = {
558 .init = init_tsc,
559 .opts = &timer_tsc,
560};
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-25 22:42:04 -0500