/*
* This code largely moved from arch/i386/kernel/time.c.
* See comments there for proper credits.
*
* 2004-06-25 Jesper Juhl
* moved mark_offset_tsc below cpufreq_delayed_get to avoid gcc 3.4
* failing to inline.
*/
#include <linux/spinlock.h>
#include <linux/init.h>
#include <linux/timex.h>
#include <linux/errno.h>
#include <linux/cpufreq.h>
#include <linux/string.h>
#include <linux/jiffies.h>
#include <asm/timer.h>
#include <asm/io.h>
/* processor.h for distable_tsc flag */
#include <asm/processor.h>
#include "io_ports.h"
#include "mach_timer.h"
#include <asm/hpet.h>
#include <asm/i8253.h>
#ifdef CONFIG_HPET_TIMER
static unsigned long hpet_usec_quotient;
static unsigned long hpet_last;
static struct timer_opts timer_tsc;
#endif
static inline void cpufreq_delayed_get(void);
int tsc_disable __devinitdata = 0;
static int use_tsc;
/* Number of usecs that the last interrupt was delayed */
static int delay_at_last_interrupt;
static unsigned long last_tsc_low; /* lsb 32 bits of Time Stamp Counter */
static unsigned long last_tsc_high; /* msb 32 bits of Time Stamp Counter */
static unsigned long long monotonic_base;
static seqlock_t monotonic_lock = SEQLOCK_UNLOCKED;
/* Avoid compensating for lost ticks before TSCs are synched */
static int detect_lost_ticks;
static int __init start_lost_tick_compensation(void)
{
detect_lost_ticks = 1;
return 0;
}
late_initcall(start_lost_tick_compensation);
/* convert from cycles(64bits) => nanoseconds (64bits)
* basic equation:
* ns = cycles / (freq / ns_per_sec)
* ns = cycles * (ns_per_sec / freq)
* ns = cycles * (10^9 / (cpu_khz * 10^3))
* ns = cycles * (10^6 / cpu_khz)
*
* Then we use scaling math (suggested by george@mvista.com) to get:
* ns = cycles * (10^6 * SC / cpu_khz) / SC
* ns = cycles * cyc2ns_scale / SC
*
* And since SC is a constant power of two, we can convert the div
* into a shift.
*
* We can use khz divisor instead of mhz to keep a better percision, since
* cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
* (mathieu.desnoyers@polymtl.ca)
*
* -johnstul@us.ibm.com "math is hard, lets go shopping!"
*/
static unsigned long cyc2ns_scale __read_mostly;
#define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
static inline void set_cyc2ns_scale(unsigned long cpu_khz)
{
cyc2ns_scale = (1000000 << CYC2NS_SCALE_FACTOR)/cpu_khz;
}
static inline unsigned long long cycles_2_ns(unsigned long long cyc)
{
return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
}
static int count2; /* counter for mark_offset_tsc() */
/* Cached *multiplier* to convert TSC counts to microseconds.
* (see the equation below).
* Equal to 2^32 * (1 / (clocks per usec) ).
* Initialized in time_init.
*/
static unsigned long fast_gettimeoffset_quotient;
static unsigned long get_offset_tsc(void)
{
register unsigned long eax, edx;
/* Read the Time Stamp Counter */
rdtsc(eax,edx);
/* .. relative to previous jiffy (32 bits is enough) */
eax -= last_tsc_low; /* tsc_low delta */
/*
* Time offset = (tsc_low delta) * fast_gettimeoffset_quotient
* = (tsc_low delta) * (usecs_per_clock)
* = (tsc_low delta) * (usecs_per_jiffy / clocks_per_jiffy)
*
* Using a mull instead of a divl saves up to 31 clock cycles
* in the critical path.
*/
__asm__("mull %2"
:"=a" (eax), "=d" (edx)
:"rm" (fast_gettimeoffset_quotient),
"0" (eax));
/* our adjusted time offset in microseconds */
return delay_at_last_interrupt + edx;
}
static unsigned long long monotonic_clock_tsc(void)
{
unsigned long long last_offset, this_offset, base;
unsigned seq;
/* atomically read monotonic base & last_offset */
do {
seq = read_seqbegin(&monotonic_lock);
last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
base = monotonic_base;
} while (read_seqretry(&monotonic_lock, seq));
/* Read the Time Stamp Counter */
rdtscll(this_offset);
/* return the value in ns */
return base + cycles_2_ns(this_offset - last_offset);
}
/*
* Scheduler clock - returns current time in nanosec units.
*/
unsigned long long sched_clock(void)
{
unsigned long long this_offset;
/*
* In the NUMA case we dont use the TSC as they are not
* synchronized across all CPUs.
*/
#ifndef CONFIG_NUMA
if (!use_tsc)
#endif
/* no locking but a rare wrong value is not a big deal */
return jiffies_64 * (1000000000 / HZ);
/* Read the Time Stamp Counter */
rdtscll(this_offset);
/* return the value in ns */
return cycles_2_ns(this_offset);
}
static void delay_tsc(unsigned long loops)
{
unsigned long bclock, now;
rdtscl(bclock);
do
{
rep_nop();
rdtscl(now);
} while ((now-bclock) < loops);
}
#ifdef CONFIG_HPET_TIMER
static void mark_offset_tsc_hpet(void)
{
unsigned long long this_offset, last_offset;
unsigned long offset, temp, hpet_current;
write_seqlock(&monotonic_lock);
last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
/*
* It is important that these two operations happen almost at
* the same time. We do the RDTSC stuff first, since it's
* faster. To avoid any inconsistencies, we need interrupts
* disabled locally.
*/
/*
* Interrupts are just disabled locally since the timer irq
* has the SA_INTERRUPT flag set. -arca
*/
/* read Pentium cycle counter */
hpet_current = hpet_readl(HPET_COUNTER);
rdtsc(last_tsc_low, last_tsc_high);
/* lost tick compensation */
offset = hpet_readl(HPET_T0_CMP) - hpet_tick;
if (unlikely(((offset - hpet_last) > hpet_tick) && (hpet_last != 0))
&& detect_lost_ticks) {
int lost_ticks = (offset - hpet_last) / hpet_tick;
jiffies_64 += lost_ticks;
}
hpet_last = hpet_current;
/* update the monotonic base value */
this_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
monotonic_base += cycles_2_ns(this_offset - last_offset);
write_sequnlock(&monotonic_lock);
/* calculate delay_at_last_interrupt */
/*
* Time offset = (hpet delta) * ( usecs per HPET clock )
* = (hpet delta) * ( usecs per tick / HPET clocks per tick)
* = (hpet delta) * ( hpet_usec_quotient ) / (2^32)
* Where,
* hpet_usec_quotient = (2^32 * usecs per tick)/HPET clocks per tick
*/
delay_at_last_interrupt = hpet_current - offset;
ASM_MUL64_REG(temp, delay_at_last_interrupt,
hpet_usec_quotient, delay_at_last_interrupt);
}
#endif
#ifdef CONFIG_CPU_FREQ
#include <linux/workqueue.h>
static unsigned int cpufreq_delayed_issched = 0;
static unsigned int cpufreq_init = 0;
static struct work_struct cpufreq_delayed_get_work;
static void handle_cpufreq_delayed_get(void *v)
{
unsigned int cpu;
for_each_online_cpu(cpu) {
cpufreq_get(cpu);
}
cpufreq_delayed_issched = 0;
}
/* if we notice lost ticks, schedule a call to cpufreq_get() as it tries
* to verify the CPU frequency the timing core thinks the CPU is running
* at is still correct.
*/
static inline void cpufreq_delayed_get(void)
{
if (cpufreq_init && !cpufreq_delayed_issched) {
cpufreq_delayed_issched = 1;
printk(KERN_DEBUG "Losing some ticks... checking if CPU frequency changed.\n");
schedule_work(&cpufreq_delayed_get_work);
}
}
/* If the CPU frequency is scaled, TSC-based delays will need a different
* loops_per_jiffy value to function properly.
*/
static unsigned int ref_freq = 0;
static unsigned long loops_per_jiffy_ref = 0;
#ifndef CONFIG_SMP
static unsigned long fast_gettimeoffset_ref = 0;
static unsigned int cpu_khz_ref = 0;
#endif
static int
time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
void *data)
{
struct cpufreq_freqs *freq = data;
if (val != CPUFREQ_RESUMECHANGE)
write_seqlock_irq(&xtime_lock);
if (!ref_freq) {
if (!freq->old){
ref_freq = freq->new;
goto end;
}
ref_freq = freq->old;
loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy;
#ifndef CONFIG_SMP
fast_gettimeoffset_ref = fast_gettimeoffset_quotient;
cpu_khz_ref = cpu_khz;
#endif
}
if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) ||
(val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
(val == CPUFREQ_RESUMECHANGE)) {
if (!(freq->flags & CPUFREQ_CONST_LOOPS))
cpu_data[freq->cpu].loops_per_jiffy = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
#ifndef CONFIG_SMP
if (cpu_khz)
cpu_khz = cpufreq_scale(cpu_khz_ref, ref_freq, freq->new);
if (use_tsc) {
if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
fast_gettimeoffset_quotient = cpufreq_scale(fast_gettimeoffset_ref, freq->new, ref_freq);
set_cyc2ns_scale(cpu_khz);
}
}
#endif
}
end:
if (val != CPUFREQ_RESUMECHANGE)
write_sequnlock_irq(&xtime_lock);
return 0;
}
static struct notifier_block time_cpufreq_notifier_block = {
.notifier_call = time_cpufreq_notifier
};
static int __init cpufreq_tsc(void)
{
int ret;
INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get, NULL);
ret = cpufreq_register_notifier(&time_cpufreq_notifier_block,
CPUFREQ_TRANSITION_NOTIFIER);
if (!ret)
cpufreq_init = 1;
return ret;
}
core_initcall(cpufreq_tsc);
#else /* CONFIG_CPU_FREQ */
static inline void cpufreq_delayed_get(void) { return; }
#endif
int recalibrate_cpu_khz(void)
{
#ifndef CONFIG_SMP
unsigned int cpu_khz_old = cpu_khz;
if (cpu_has_tsc) {
local_irq_disable();
init_cpu_khz();
local_irq_enable();
cpu_data[0].loops_per_jiffy =
cpufreq_scale(cpu_data[0].loops_per_jiffy,
cpu_khz_old,
cpu_khz);
return 0;
} else
return -ENODEV;
#else
return -ENODEV;
#endif
}
EXPORT_SYMBOL(recalibrate_cpu_khz);
static void mark_offset_tsc(void)
{
unsigned long lost,delay;
unsigned long delta = last_tsc_low;
int count;
int countmp;
static int count1 = 0;
unsigned long long this_offset, last_offset;
static int lost_count = 0;
write_seqlock(&monotonic_lock);
last_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
/*
* It is important that these two operations happen almost at
* the same time. We do the RDTSC stuff first, since it's
* faster. To avoid any inconsistencies, we need interrupts
* disabled locally.
*/
/*
* Interrupts are just disabled locally since the timer irq
* has the SA_INTERRUPT flag set. -arca
*/
/* read Pentium cycle counter */
rdtsc(last_tsc_low, last_tsc_high);
spin_lock(&i8253_lock);
outb_p(0x00, PIT_MODE); /* latch the count ASAP */
count = inb_p(PIT_CH0); /* read the latched count */
count |= inb(PIT_CH0) << 8;
/*
* VIA686a test code... reset the latch if count > max + 1
* from timer_pit.c - cjb
*/
if (count > LATCH) {
outb_p(0x34, PIT_MODE);
outb_p(LATCH & 0xff, PIT_CH0);
outb(LATCH >> 8, PIT_CH0);
count = LATCH - 1;
}
spin_unlock(&i8253_lock);
if (pit_latch_buggy) {
/* get center value of last 3 time lutch */
if ((count2 >= count && count >= count1)
|| (count1 >= count && count >= count2)) {
count2 = count1; count1 = count;
} else if ((count1 >= count2 && count2 >= count)
|| (count >= count2 && count2 >= count1)) {
countmp = count;count = count2;
count2 = count1;count1 = countmp;
} else {
count2 = count1; count1 = count; count = count1;
}
}
/* lost tick compensation */
delta = last_tsc_low - delta;
{
register unsigned long eax, edx;
eax = delta;
__asm__("mull %2"
:"=a" (eax), "=d" (edx)
:"rm" (fast_gettimeoffset_quotient),
"0" (eax));
delta = edx;
}
delta += delay_at_last_interrupt;
lost = delta/(1000000/HZ);
delay = delta%(1000000/HZ);
if (lost >= 2 && detect_lost_ticks) {
jiffies_64 += lost-1;
/* sanity check to ensure we're not always losing ticks */
if (lost_count++ > 100) {
printk(KERN_WARNING "Losing too many ticks!\n");
printk(KERN_WARNING "TSC cannot be used as a timesource. \n");
printk(KERN_WARNING "Possible reasons for this are:\n");
printk(KERN_WARNING " You're running with Speedstep,\n");
printk(KERN_WARNING " You don't have DMA enabled for your hard disk (see hdparm),\n");
printk(KERN_WARNING " Incorrect TSC synchronization on an SMP system (see dmesg).\n");
printk(KERN_WARNING "Falling back to a sane timesource now.\n");
clock_fallback();
}
/* ... but give the TSC a fair chance */
if (lost_count > 25)
cpufreq_delayed_get();
} else
lost_count = 0;
/* update the monotonic base value */
this_offset = ((unsigned long long)last_tsc_high<<32)|last_tsc_low;
monotonic_base += cycles_2_ns(this_offset - last_offset);
write_sequnlock(&monotonic_lock);
/* calculate delay_at_last_interrupt */
count = ((LATCH-1) - count) * TICK_SIZE;
delay_at_last_interrupt = (count + LATCH/2) / LATCH;
/* catch corner case where tick rollover occured
* between tsc and pit reads (as noted when
* usec delta is > 90% # of usecs/tick)
*/
if (lost && abs(delay - delay_at_last_interrupt) > (900000/HZ))
jiffies_64++;
}
static int __init init_tsc(char* override)
{
/* check clock override */
if (override[0] && strncmp(override,"tsc",3)) {
#ifdef CONFIG_HPET_TIMER
if (is_hpet_enabled()) {
printk(KERN_ERR "Warning: clock= override failed. Defaulting to tsc\n");
} else
#endif
{
return -ENODEV;
}
}
/*
* If we have APM enabled or the CPU clock speed is variable
* (CPU stops clock on HLT or slows clock to save power)
* then the TSC timestamps may diverge by up to 1 jiffy from
* 'real time' but nothing will break.
* The most frequent case is that the CPU is "woken" from a halt
* state by the timer interrupt itself, so we get 0 error. In the
* rare cases where a driver would "wake" the CPU and request a
* timestamp, the maximum error is < 1 jiffy. But timestamps are
* still perfectly ordered.
* Note that the TSC counter will be reset if APM suspends
* to disk; this won't break the kernel, though, 'cuz we're
* smart. See arch/i386/kernel/apm.c.
*/
/*
* Firstly we have to do a CPU check for chips with
* a potentially buggy TSC. At this point we haven't run
* the ident/bugs checks so we must run this hook as it
* may turn off the TSC flag.
*
* NOTE: this doesn't yet handle SMP 486 machines where only
* some CPU's have a TSC. Thats never worked and nobody has
* moaned if you have the only one in the world - you fix it!
*/
count2 = LATCH; /* initialize counter for mark_offset_tsc() */
if (cpu_has_tsc) {
unsigned long tsc_quotient;
#ifdef CONFIG_HPET_TIMER
if (is_hpet_enabled() && hpet_use_timer) {
unsigned long result, remain;
printk("Using TSC for gettimeofday\n");
tsc_quotient = calibrate_tsc_hpet(NULL);
timer_tsc.mark_offset = &mark_offset_tsc_hpet;
/*
* Math to calculate hpet to usec multiplier
* Look for the comments at get_offset_tsc_hpet()
*/
ASM_DIV64_REG(result, remain, hpet_tick,
0, KERNEL_TICK_USEC);
if (remain > (hpet_tick >> 1))
result++; /* rounding the result */
hpet_usec_quotient = result;
} else
#endif
{
tsc_quotient = calibrate_tsc();
}
if (tsc_quotient) {
fast_gettimeoffset_quotient = tsc_quotient;
use_tsc = 1;
/*
* We could be more selective here I suspect
* and just enable this for the next intel chips ?
*/
/* report CPU clock rate in Hz.
* The formula is (10^6 * 2^32) / (2^32 * 1 / (clocks/us)) =
* clock/second. Our precision is about 100 ppm.
*/
{ unsigned long eax=0, edx=1000;
__asm__("divl %2"
:"=a" (cpu_khz), "=d" (edx)
:"r" (tsc_quotient),
"0" (eax), "1" (edx));
printk("Detected %u.%03u MHz processor.\n",
cpu_khz / 1000, cpu_khz % 1000);
}
set_cyc2ns_scale(cpu_khz);
return 0;
}
}
return -ENODEV;
}
static int tsc_resume(void)
{
write_seqlock(&monotonic_lock);
/* Assume this is the last mark offset time */
rdtsc(last_tsc_low, last_tsc_high);
#ifdef CONFIG_HPET_TIMER
if (is_hpet_enabled() && hpet_use_timer)
hpet_last = hpet_readl(HPET_COUNTER);
#endif
write_sequnlock(&monotonic_lock);
return 0;
}
#ifndef CONFIG_X86_TSC
/* disable flag for tsc. Takes effect by clearing the TSC cpu flag
* in cpu/common.c */
static int __init tsc_setup(char *str)
{
tsc_disable = 1;
return 1;
}
#else
static int __init tsc_setup(char *str)
{
printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
"cannot disable TSC.\n");
return 1;
}
#endif
__setup("notsc", tsc_setup);
/************************************************************/
/* tsc timer_opts struct */
static struct timer_opts timer_tsc = {
.name = "tsc",
.mark_offset = mark_offset_tsc,
.get_offset = get_offset_tsc,
.monotonic_clock = monotonic_clock_tsc,
.delay = delay_tsc,
.read_timer = read_timer_tsc,
.resume = tsc_resume,
};
struct init_timer_opts __initdata timer_tsc_init = {
.init = init_tsc,
.opts = &timer_tsc,
};
