diff options
Diffstat (limited to 'arch/powerpc/kernel/time.c')
-rw-r--r-- | arch/powerpc/kernel/time.c | 523 |
1 files changed, 338 insertions, 185 deletions
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 1886045a2fd8..4f20a5f15d49 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c | |||
@@ -50,6 +50,8 @@ | |||
50 | #include <linux/security.h> | 50 | #include <linux/security.h> |
51 | #include <linux/percpu.h> | 51 | #include <linux/percpu.h> |
52 | #include <linux/rtc.h> | 52 | #include <linux/rtc.h> |
53 | #include <linux/jiffies.h> | ||
54 | #include <linux/posix-timers.h> | ||
53 | 55 | ||
54 | #include <asm/io.h> | 56 | #include <asm/io.h> |
55 | #include <asm/processor.h> | 57 | #include <asm/processor.h> |
@@ -97,9 +99,18 @@ unsigned long tb_ticks_per_jiffy; | |||
97 | unsigned long tb_ticks_per_usec = 100; /* sane default */ | 99 | unsigned long tb_ticks_per_usec = 100; /* sane default */ |
98 | EXPORT_SYMBOL(tb_ticks_per_usec); | 100 | EXPORT_SYMBOL(tb_ticks_per_usec); |
99 | unsigned long tb_ticks_per_sec; | 101 | unsigned long tb_ticks_per_sec; |
102 | EXPORT_SYMBOL(tb_ticks_per_sec); /* for cputime_t conversions */ | ||
100 | u64 tb_to_xs; | 103 | u64 tb_to_xs; |
101 | unsigned tb_to_us; | 104 | unsigned tb_to_us; |
102 | unsigned long processor_freq; | 105 | |
106 | #define TICKLEN_SCALE (SHIFT_SCALE - 10) | ||
107 | u64 last_tick_len; /* units are ns / 2^TICKLEN_SCALE */ | ||
108 | u64 ticklen_to_xs; /* 0.64 fraction */ | ||
109 | |||
110 | /* If last_tick_len corresponds to about 1/HZ seconds, then | ||
111 | last_tick_len << TICKLEN_SHIFT will be about 2^63. */ | ||
112 | #define TICKLEN_SHIFT (63 - 30 - TICKLEN_SCALE + SHIFT_HZ) | ||
113 | |||
103 | DEFINE_SPINLOCK(rtc_lock); | 114 | DEFINE_SPINLOCK(rtc_lock); |
104 | EXPORT_SYMBOL_GPL(rtc_lock); | 115 | EXPORT_SYMBOL_GPL(rtc_lock); |
105 | 116 | ||
@@ -113,10 +124,6 @@ extern unsigned long wall_jiffies; | |||
113 | extern struct timezone sys_tz; | 124 | extern struct timezone sys_tz; |
114 | static long timezone_offset; | 125 | static long timezone_offset; |
115 | 126 | ||
116 | void ppc_adjtimex(void); | ||
117 | |||
118 | static unsigned adjusting_time = 0; | ||
119 | |||
120 | unsigned long ppc_proc_freq; | 127 | unsigned long ppc_proc_freq; |
121 | unsigned long ppc_tb_freq; | 128 | unsigned long ppc_tb_freq; |
122 | 129 | ||
@@ -130,6 +137,224 @@ unsigned long tb_last_stamp; | |||
130 | */ | 137 | */ |
131 | DEFINE_PER_CPU(unsigned long, last_jiffy); | 138 | DEFINE_PER_CPU(unsigned long, last_jiffy); |
132 | 139 | ||
140 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING | ||
141 | /* | ||
142 | * Factors for converting from cputime_t (timebase ticks) to | ||
143 | * jiffies, milliseconds, seconds, and clock_t (1/USER_HZ seconds). | ||
144 | * These are all stored as 0.64 fixed-point binary fractions. | ||
145 | */ | ||
146 | u64 __cputime_jiffies_factor; | ||
147 | EXPORT_SYMBOL(__cputime_jiffies_factor); | ||
148 | u64 __cputime_msec_factor; | ||
149 | EXPORT_SYMBOL(__cputime_msec_factor); | ||
150 | u64 __cputime_sec_factor; | ||
151 | EXPORT_SYMBOL(__cputime_sec_factor); | ||
152 | u64 __cputime_clockt_factor; | ||
153 | EXPORT_SYMBOL(__cputime_clockt_factor); | ||
154 | |||
155 | static void calc_cputime_factors(void) | ||
156 | { | ||
157 | struct div_result res; | ||
158 | |||
159 | div128_by_32(HZ, 0, tb_ticks_per_sec, &res); | ||
160 | __cputime_jiffies_factor = res.result_low; | ||
161 | div128_by_32(1000, 0, tb_ticks_per_sec, &res); | ||
162 | __cputime_msec_factor = res.result_low; | ||
163 | div128_by_32(1, 0, tb_ticks_per_sec, &res); | ||
164 | __cputime_sec_factor = res.result_low; | ||
165 | div128_by_32(USER_HZ, 0, tb_ticks_per_sec, &res); | ||
166 | __cputime_clockt_factor = res.result_low; | ||
167 | } | ||
168 | |||
169 | /* | ||
170 | * Read the PURR on systems that have it, otherwise the timebase. | ||
171 | */ | ||
172 | static u64 read_purr(void) | ||
173 | { | ||
174 | if (cpu_has_feature(CPU_FTR_PURR)) | ||
175 | return mfspr(SPRN_PURR); | ||
176 | return mftb(); | ||
177 | } | ||
178 | |||
179 | /* | ||
180 | * Account time for a transition between system, hard irq | ||
181 | * or soft irq state. | ||
182 | */ | ||
183 | void account_system_vtime(struct task_struct *tsk) | ||
184 | { | ||
185 | u64 now, delta; | ||
186 | unsigned long flags; | ||
187 | |||
188 | local_irq_save(flags); | ||
189 | now = read_purr(); | ||
190 | delta = now - get_paca()->startpurr; | ||
191 | get_paca()->startpurr = now; | ||
192 | if (!in_interrupt()) { | ||
193 | delta += get_paca()->system_time; | ||
194 | get_paca()->system_time = 0; | ||
195 | } | ||
196 | account_system_time(tsk, 0, delta); | ||
197 | local_irq_restore(flags); | ||
198 | } | ||
199 | |||
200 | /* | ||
201 | * Transfer the user and system times accumulated in the paca | ||
202 | * by the exception entry and exit code to the generic process | ||
203 | * user and system time records. | ||
204 | * Must be called with interrupts disabled. | ||
205 | */ | ||
206 | void account_process_vtime(struct task_struct *tsk) | ||
207 | { | ||
208 | cputime_t utime; | ||
209 | |||
210 | utime = get_paca()->user_time; | ||
211 | get_paca()->user_time = 0; | ||
212 | account_user_time(tsk, utime); | ||
213 | } | ||
214 | |||
215 | static void account_process_time(struct pt_regs *regs) | ||
216 | { | ||
217 | int cpu = smp_processor_id(); | ||
218 | |||
219 | account_process_vtime(current); | ||
220 | run_local_timers(); | ||
221 | if (rcu_pending(cpu)) | ||
222 | rcu_check_callbacks(cpu, user_mode(regs)); | ||
223 | scheduler_tick(); | ||
224 | run_posix_cpu_timers(current); | ||
225 | } | ||
226 | |||
227 | #ifdef CONFIG_PPC_SPLPAR | ||
228 | /* | ||
229 | * Stuff for accounting stolen time. | ||
230 | */ | ||
231 | struct cpu_purr_data { | ||
232 | int initialized; /* thread is running */ | ||
233 | u64 tb0; /* timebase at origin time */ | ||
234 | u64 purr0; /* PURR at origin time */ | ||
235 | u64 tb; /* last TB value read */ | ||
236 | u64 purr; /* last PURR value read */ | ||
237 | u64 stolen; /* stolen time so far */ | ||
238 | spinlock_t lock; | ||
239 | }; | ||
240 | |||
241 | static DEFINE_PER_CPU(struct cpu_purr_data, cpu_purr_data); | ||
242 | |||
243 | static void snapshot_tb_and_purr(void *data) | ||
244 | { | ||
245 | struct cpu_purr_data *p = &__get_cpu_var(cpu_purr_data); | ||
246 | |||
247 | p->tb0 = mftb(); | ||
248 | p->purr0 = mfspr(SPRN_PURR); | ||
249 | p->tb = p->tb0; | ||
250 | p->purr = 0; | ||
251 | wmb(); | ||
252 | p->initialized = 1; | ||
253 | } | ||
254 | |||
255 | /* | ||
256 | * Called during boot when all cpus have come up. | ||
257 | */ | ||
258 | void snapshot_timebases(void) | ||
259 | { | ||
260 | int cpu; | ||
261 | |||
262 | if (!cpu_has_feature(CPU_FTR_PURR)) | ||
263 | return; | ||
264 | for_each_cpu(cpu) | ||
265 | spin_lock_init(&per_cpu(cpu_purr_data, cpu).lock); | ||
266 | on_each_cpu(snapshot_tb_and_purr, NULL, 0, 1); | ||
267 | } | ||
268 | |||
269 | void calculate_steal_time(void) | ||
270 | { | ||
271 | u64 tb, purr, t0; | ||
272 | s64 stolen; | ||
273 | struct cpu_purr_data *p0, *pme, *phim; | ||
274 | int cpu; | ||
275 | |||
276 | if (!cpu_has_feature(CPU_FTR_PURR)) | ||
277 | return; | ||
278 | cpu = smp_processor_id(); | ||
279 | pme = &per_cpu(cpu_purr_data, cpu); | ||
280 | if (!pme->initialized) | ||
281 | return; /* this can happen in early boot */ | ||
282 | p0 = &per_cpu(cpu_purr_data, cpu & ~1); | ||
283 | phim = &per_cpu(cpu_purr_data, cpu ^ 1); | ||
284 | spin_lock(&p0->lock); | ||
285 | tb = mftb(); | ||
286 | purr = mfspr(SPRN_PURR) - pme->purr0; | ||
287 | if (!phim->initialized || !cpu_online(cpu ^ 1)) { | ||
288 | stolen = (tb - pme->tb) - (purr - pme->purr); | ||
289 | } else { | ||
290 | t0 = pme->tb0; | ||
291 | if (phim->tb0 < t0) | ||
292 | t0 = phim->tb0; | ||
293 | stolen = phim->tb - t0 - phim->purr - purr - p0->stolen; | ||
294 | } | ||
295 | if (stolen > 0) { | ||
296 | account_steal_time(current, stolen); | ||
297 | p0->stolen += stolen; | ||
298 | } | ||
299 | pme->tb = tb; | ||
300 | pme->purr = purr; | ||
301 | spin_unlock(&p0->lock); | ||
302 | } | ||
303 | |||
304 | /* | ||
305 | * Must be called before the cpu is added to the online map when | ||
306 | * a cpu is being brought up at runtime. | ||
307 | */ | ||
308 | static void snapshot_purr(void) | ||
309 | { | ||
310 | int cpu; | ||
311 | u64 purr; | ||
312 | struct cpu_purr_data *p0, *pme, *phim; | ||
313 | unsigned long flags; | ||
314 | |||
315 | if (!cpu_has_feature(CPU_FTR_PURR)) | ||
316 | return; | ||
317 | cpu = smp_processor_id(); | ||
318 | pme = &per_cpu(cpu_purr_data, cpu); | ||
319 | p0 = &per_cpu(cpu_purr_data, cpu & ~1); | ||
320 | phim = &per_cpu(cpu_purr_data, cpu ^ 1); | ||
321 | spin_lock_irqsave(&p0->lock, flags); | ||
322 | pme->tb = pme->tb0 = mftb(); | ||
323 | purr = mfspr(SPRN_PURR); | ||
324 | if (!phim->initialized) { | ||
325 | pme->purr = 0; | ||
326 | pme->purr0 = purr; | ||
327 | } else { | ||
328 | /* set p->purr and p->purr0 for no change in p0->stolen */ | ||
329 | pme->purr = phim->tb - phim->tb0 - phim->purr - p0->stolen; | ||
330 | pme->purr0 = purr - pme->purr; | ||
331 | } | ||
332 | pme->initialized = 1; | ||
333 | spin_unlock_irqrestore(&p0->lock, flags); | ||
334 | } | ||
335 | |||
336 | #endif /* CONFIG_PPC_SPLPAR */ | ||
337 | |||
338 | #else /* ! CONFIG_VIRT_CPU_ACCOUNTING */ | ||
339 | #define calc_cputime_factors() | ||
340 | #define account_process_time(regs) update_process_times(user_mode(regs)) | ||
341 | #define calculate_steal_time() do { } while (0) | ||
342 | #endif | ||
343 | |||
344 | #if !(defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(CONFIG_PPC_SPLPAR)) | ||
345 | #define snapshot_purr() do { } while (0) | ||
346 | #endif | ||
347 | |||
348 | /* | ||
349 | * Called when a cpu comes up after the system has finished booting, | ||
350 | * i.e. as a result of a hotplug cpu action. | ||
351 | */ | ||
352 | void snapshot_timebase(void) | ||
353 | { | ||
354 | __get_cpu_var(last_jiffy) = get_tb(); | ||
355 | snapshot_purr(); | ||
356 | } | ||
357 | |||
133 | void __delay(unsigned long loops) | 358 | void __delay(unsigned long loops) |
134 | { | 359 | { |
135 | unsigned long start; | 360 | unsigned long start; |
@@ -178,8 +403,7 @@ static __inline__ void timer_check_rtc(void) | |||
178 | */ | 403 | */ |
179 | if (ppc_md.set_rtc_time && ntp_synced() && | 404 | if (ppc_md.set_rtc_time && ntp_synced() && |
180 | xtime.tv_sec - last_rtc_update >= 659 && | 405 | xtime.tv_sec - last_rtc_update >= 659 && |
181 | abs((xtime.tv_nsec/1000) - (1000000-1000000/HZ)) < 500000/HZ && | 406 | abs((xtime.tv_nsec/1000) - (1000000-1000000/HZ)) < 500000/HZ) { |
182 | jiffies - wall_jiffies == 1) { | ||
183 | struct rtc_time tm; | 407 | struct rtc_time tm; |
184 | to_tm(xtime.tv_sec + 1 + timezone_offset, &tm); | 408 | to_tm(xtime.tv_sec + 1 + timezone_offset, &tm); |
185 | tm.tm_year -= 1900; | 409 | tm.tm_year -= 1900; |
@@ -226,15 +450,14 @@ void do_gettimeofday(struct timeval *tv) | |||
226 | if (__USE_RTC()) { | 450 | if (__USE_RTC()) { |
227 | /* do this the old way */ | 451 | /* do this the old way */ |
228 | unsigned long flags, seq; | 452 | unsigned long flags, seq; |
229 | unsigned int sec, nsec, usec, lost; | 453 | unsigned int sec, nsec, usec; |
230 | 454 | ||
231 | do { | 455 | do { |
232 | seq = read_seqbegin_irqsave(&xtime_lock, flags); | 456 | seq = read_seqbegin_irqsave(&xtime_lock, flags); |
233 | sec = xtime.tv_sec; | 457 | sec = xtime.tv_sec; |
234 | nsec = xtime.tv_nsec + tb_ticks_since(tb_last_stamp); | 458 | nsec = xtime.tv_nsec + tb_ticks_since(tb_last_stamp); |
235 | lost = jiffies - wall_jiffies; | ||
236 | } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); | 459 | } while (read_seqretry_irqrestore(&xtime_lock, seq, flags)); |
237 | usec = nsec / 1000 + lost * (1000000 / HZ); | 460 | usec = nsec / 1000; |
238 | while (usec >= 1000000) { | 461 | while (usec >= 1000000) { |
239 | usec -= 1000000; | 462 | usec -= 1000000; |
240 | ++sec; | 463 | ++sec; |
@@ -248,23 +471,6 @@ void do_gettimeofday(struct timeval *tv) | |||
248 | 471 | ||
249 | EXPORT_SYMBOL(do_gettimeofday); | 472 | EXPORT_SYMBOL(do_gettimeofday); |
250 | 473 | ||
251 | /* Synchronize xtime with do_gettimeofday */ | ||
252 | |||
253 | static inline void timer_sync_xtime(unsigned long cur_tb) | ||
254 | { | ||
255 | #ifdef CONFIG_PPC64 | ||
256 | /* why do we do this? */ | ||
257 | struct timeval my_tv; | ||
258 | |||
259 | __do_gettimeofday(&my_tv, cur_tb); | ||
260 | |||
261 | if (xtime.tv_sec <= my_tv.tv_sec) { | ||
262 | xtime.tv_sec = my_tv.tv_sec; | ||
263 | xtime.tv_nsec = my_tv.tv_usec * 1000; | ||
264 | } | ||
265 | #endif | ||
266 | } | ||
267 | |||
268 | /* | 474 | /* |
269 | * There are two copies of tb_to_xs and stamp_xsec so that no | 475 | * There are two copies of tb_to_xs and stamp_xsec so that no |
270 | * lock is needed to access and use these values in | 476 | * lock is needed to access and use these values in |
@@ -323,15 +529,30 @@ static __inline__ void timer_recalc_offset(u64 cur_tb) | |||
323 | { | 529 | { |
324 | unsigned long offset; | 530 | unsigned long offset; |
325 | u64 new_stamp_xsec; | 531 | u64 new_stamp_xsec; |
532 | u64 tlen, t2x; | ||
326 | 533 | ||
327 | if (__USE_RTC()) | 534 | if (__USE_RTC()) |
328 | return; | 535 | return; |
536 | tlen = current_tick_length(); | ||
329 | offset = cur_tb - do_gtod.varp->tb_orig_stamp; | 537 | offset = cur_tb - do_gtod.varp->tb_orig_stamp; |
330 | if ((offset & 0x80000000u) == 0) | 538 | if (tlen == last_tick_len && offset < 0x80000000u) { |
331 | return; | 539 | /* check that we're still in sync; if not, resync */ |
332 | new_stamp_xsec = do_gtod.varp->stamp_xsec | 540 | struct timeval tv; |
333 | + mulhdu(offset, do_gtod.varp->tb_to_xs); | 541 | __do_gettimeofday(&tv, cur_tb); |
334 | update_gtod(cur_tb, new_stamp_xsec, do_gtod.varp->tb_to_xs); | 542 | if (tv.tv_sec <= xtime.tv_sec && |
543 | (tv.tv_sec < xtime.tv_sec || | ||
544 | tv.tv_usec * 1000 <= xtime.tv_nsec)) | ||
545 | return; | ||
546 | } | ||
547 | if (tlen != last_tick_len) { | ||
548 | t2x = mulhdu(tlen << TICKLEN_SHIFT, ticklen_to_xs); | ||
549 | last_tick_len = tlen; | ||
550 | } else | ||
551 | t2x = do_gtod.varp->tb_to_xs; | ||
552 | new_stamp_xsec = (u64) xtime.tv_nsec * XSEC_PER_SEC; | ||
553 | do_div(new_stamp_xsec, 1000000000); | ||
554 | new_stamp_xsec += (u64) xtime.tv_sec * XSEC_PER_SEC; | ||
555 | update_gtod(cur_tb, new_stamp_xsec, t2x); | ||
335 | } | 556 | } |
336 | 557 | ||
337 | #ifdef CONFIG_SMP | 558 | #ifdef CONFIG_SMP |
@@ -381,6 +602,7 @@ static void iSeries_tb_recal(void) | |||
381 | new_tb_ticks_per_jiffy, sign, tick_diff ); | 602 | new_tb_ticks_per_jiffy, sign, tick_diff ); |
382 | tb_ticks_per_jiffy = new_tb_ticks_per_jiffy; | 603 | tb_ticks_per_jiffy = new_tb_ticks_per_jiffy; |
383 | tb_ticks_per_sec = new_tb_ticks_per_sec; | 604 | tb_ticks_per_sec = new_tb_ticks_per_sec; |
605 | calc_cputime_factors(); | ||
384 | div128_by_32( XSEC_PER_SEC, 0, tb_ticks_per_sec, &divres ); | 606 | div128_by_32( XSEC_PER_SEC, 0, tb_ticks_per_sec, &divres ); |
385 | do_gtod.tb_ticks_per_sec = tb_ticks_per_sec; | 607 | do_gtod.tb_ticks_per_sec = tb_ticks_per_sec; |
386 | tb_to_xs = divres.result_low; | 608 | tb_to_xs = divres.result_low; |
@@ -429,6 +651,7 @@ void timer_interrupt(struct pt_regs * regs) | |||
429 | irq_enter(); | 651 | irq_enter(); |
430 | 652 | ||
431 | profile_tick(CPU_PROFILING, regs); | 653 | profile_tick(CPU_PROFILING, regs); |
654 | calculate_steal_time(); | ||
432 | 655 | ||
433 | #ifdef CONFIG_PPC_ISERIES | 656 | #ifdef CONFIG_PPC_ISERIES |
434 | get_lppaca()->int_dword.fields.decr_int = 0; | 657 | get_lppaca()->int_dword.fields.decr_int = 0; |
@@ -450,7 +673,7 @@ void timer_interrupt(struct pt_regs * regs) | |||
450 | * is the case. | 673 | * is the case. |
451 | */ | 674 | */ |
452 | if (!cpu_is_offline(cpu)) | 675 | if (!cpu_is_offline(cpu)) |
453 | update_process_times(user_mode(regs)); | 676 | account_process_time(regs); |
454 | 677 | ||
455 | /* | 678 | /* |
456 | * No need to check whether cpu is offline here; boot_cpuid | 679 | * No need to check whether cpu is offline here; boot_cpuid |
@@ -462,13 +685,10 @@ void timer_interrupt(struct pt_regs * regs) | |||
462 | write_seqlock(&xtime_lock); | 685 | write_seqlock(&xtime_lock); |
463 | tb_last_jiffy += tb_ticks_per_jiffy; | 686 | tb_last_jiffy += tb_ticks_per_jiffy; |
464 | tb_last_stamp = per_cpu(last_jiffy, cpu); | 687 | tb_last_stamp = per_cpu(last_jiffy, cpu); |
465 | timer_recalc_offset(tb_last_jiffy); | ||
466 | do_timer(regs); | 688 | do_timer(regs); |
467 | timer_sync_xtime(tb_last_jiffy); | 689 | timer_recalc_offset(tb_last_jiffy); |
468 | timer_check_rtc(); | 690 | timer_check_rtc(); |
469 | write_sequnlock(&xtime_lock); | 691 | write_sequnlock(&xtime_lock); |
470 | if (adjusting_time && (time_adjust == 0)) | ||
471 | ppc_adjtimex(); | ||
472 | } | 692 | } |
473 | 693 | ||
474 | next_dec = tb_ticks_per_jiffy - ticks; | 694 | next_dec = tb_ticks_per_jiffy - ticks; |
@@ -492,29 +712,45 @@ void timer_interrupt(struct pt_regs * regs) | |||
492 | 712 | ||
493 | void wakeup_decrementer(void) | 713 | void wakeup_decrementer(void) |
494 | { | 714 | { |
495 | int i; | 715 | unsigned long ticks; |
496 | 716 | ||
497 | set_dec(tb_ticks_per_jiffy); | ||
498 | /* | 717 | /* |
499 | * We don't expect this to be called on a machine with a 601, | 718 | * The timebase gets saved on sleep and restored on wakeup, |
500 | * so using get_tbl is fine. | 719 | * so all we need to do is to reset the decrementer. |
501 | */ | 720 | */ |
502 | tb_last_stamp = tb_last_jiffy = get_tb(); | 721 | ticks = tb_ticks_since(__get_cpu_var(last_jiffy)); |
503 | for_each_cpu(i) | 722 | if (ticks < tb_ticks_per_jiffy) |
504 | per_cpu(last_jiffy, i) = tb_last_stamp; | 723 | ticks = tb_ticks_per_jiffy - ticks; |
724 | else | ||
725 | ticks = 1; | ||
726 | set_dec(ticks); | ||
505 | } | 727 | } |
506 | 728 | ||
507 | #ifdef CONFIG_SMP | 729 | #ifdef CONFIG_SMP |
508 | void __init smp_space_timers(unsigned int max_cpus) | 730 | void __init smp_space_timers(unsigned int max_cpus) |
509 | { | 731 | { |
510 | int i; | 732 | int i; |
733 | unsigned long half = tb_ticks_per_jiffy / 2; | ||
511 | unsigned long offset = tb_ticks_per_jiffy / max_cpus; | 734 | unsigned long offset = tb_ticks_per_jiffy / max_cpus; |
512 | unsigned long previous_tb = per_cpu(last_jiffy, boot_cpuid); | 735 | unsigned long previous_tb = per_cpu(last_jiffy, boot_cpuid); |
513 | 736 | ||
514 | /* make sure tb > per_cpu(last_jiffy, cpu) for all cpus always */ | 737 | /* make sure tb > per_cpu(last_jiffy, cpu) for all cpus always */ |
515 | previous_tb -= tb_ticks_per_jiffy; | 738 | previous_tb -= tb_ticks_per_jiffy; |
739 | /* | ||
740 | * The stolen time calculation for POWER5 shared-processor LPAR | ||
741 | * systems works better if the two threads' timebase interrupts | ||
742 | * are staggered by half a jiffy with respect to each other. | ||
743 | */ | ||
516 | for_each_cpu(i) { | 744 | for_each_cpu(i) { |
517 | if (i != boot_cpuid) { | 745 | if (i == boot_cpuid) |
746 | continue; | ||
747 | if (i == (boot_cpuid ^ 1)) | ||
748 | per_cpu(last_jiffy, i) = | ||
749 | per_cpu(last_jiffy, boot_cpuid) - half; | ||
750 | else if (i & 1) | ||
751 | per_cpu(last_jiffy, i) = | ||
752 | per_cpu(last_jiffy, i ^ 1) + half; | ||
753 | else { | ||
518 | previous_tb += offset; | 754 | previous_tb += offset; |
519 | per_cpu(last_jiffy, i) = previous_tb; | 755 | per_cpu(last_jiffy, i) = previous_tb; |
520 | } | 756 | } |
@@ -541,8 +777,8 @@ int do_settimeofday(struct timespec *tv) | |||
541 | time_t wtm_sec, new_sec = tv->tv_sec; | 777 | time_t wtm_sec, new_sec = tv->tv_sec; |
542 | long wtm_nsec, new_nsec = tv->tv_nsec; | 778 | long wtm_nsec, new_nsec = tv->tv_nsec; |
543 | unsigned long flags; | 779 | unsigned long flags; |
544 | long int tb_delta; | 780 | u64 new_xsec; |
545 | u64 new_xsec, tb_delta_xs; | 781 | unsigned long tb_delta; |
546 | 782 | ||
547 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) | 783 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) |
548 | return -EINVAL; | 784 | return -EINVAL; |
@@ -563,9 +799,19 @@ int do_settimeofday(struct timespec *tv) | |||
563 | first_settimeofday = 0; | 799 | first_settimeofday = 0; |
564 | } | 800 | } |
565 | #endif | 801 | #endif |
802 | |||
803 | /* | ||
804 | * Subtract off the number of nanoseconds since the | ||
805 | * beginning of the last tick. | ||
806 | * Note that since we don't increment jiffies_64 anywhere other | ||
807 | * than in do_timer (since we don't have a lost tick problem), | ||
808 | * wall_jiffies will always be the same as jiffies, | ||
809 | * and therefore the (jiffies - wall_jiffies) computation | ||
810 | * has been removed. | ||
811 | */ | ||
566 | tb_delta = tb_ticks_since(tb_last_stamp); | 812 | tb_delta = tb_ticks_since(tb_last_stamp); |
567 | tb_delta += (jiffies - wall_jiffies) * tb_ticks_per_jiffy; | 813 | tb_delta = mulhdu(tb_delta, do_gtod.varp->tb_to_xs); /* in xsec */ |
568 | tb_delta_xs = mulhdu(tb_delta, do_gtod.varp->tb_to_xs); | 814 | new_nsec -= SCALE_XSEC(tb_delta, 1000000000); |
569 | 815 | ||
570 | wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - new_sec); | 816 | wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - new_sec); |
571 | wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - new_nsec); | 817 | wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - new_nsec); |
@@ -580,12 +826,12 @@ int do_settimeofday(struct timespec *tv) | |||
580 | 826 | ||
581 | ntp_clear(); | 827 | ntp_clear(); |
582 | 828 | ||
583 | new_xsec = 0; | 829 | new_xsec = xtime.tv_nsec; |
584 | if (new_nsec != 0) { | 830 | if (new_xsec != 0) { |
585 | new_xsec = (u64)new_nsec * XSEC_PER_SEC; | 831 | new_xsec *= XSEC_PER_SEC; |
586 | do_div(new_xsec, NSEC_PER_SEC); | 832 | do_div(new_xsec, NSEC_PER_SEC); |
587 | } | 833 | } |
588 | new_xsec += (u64)new_sec * XSEC_PER_SEC - tb_delta_xs; | 834 | new_xsec += (u64)xtime.tv_sec * XSEC_PER_SEC; |
589 | update_gtod(tb_last_jiffy, new_xsec, do_gtod.varp->tb_to_xs); | 835 | update_gtod(tb_last_jiffy, new_xsec, do_gtod.varp->tb_to_xs); |
590 | 836 | ||
591 | vdso_data->tz_minuteswest = sys_tz.tz_minuteswest; | 837 | vdso_data->tz_minuteswest = sys_tz.tz_minuteswest; |
@@ -671,7 +917,7 @@ void __init time_init(void) | |||
671 | unsigned long flags; | 917 | unsigned long flags; |
672 | unsigned long tm = 0; | 918 | unsigned long tm = 0; |
673 | struct div_result res; | 919 | struct div_result res; |
674 | u64 scale; | 920 | u64 scale, x; |
675 | unsigned shift; | 921 | unsigned shift; |
676 | 922 | ||
677 | if (ppc_md.time_init != NULL) | 923 | if (ppc_md.time_init != NULL) |
@@ -693,11 +939,37 @@ void __init time_init(void) | |||
693 | } | 939 | } |
694 | 940 | ||
695 | tb_ticks_per_jiffy = ppc_tb_freq / HZ; | 941 | tb_ticks_per_jiffy = ppc_tb_freq / HZ; |
696 | tb_ticks_per_sec = tb_ticks_per_jiffy * HZ; | 942 | tb_ticks_per_sec = ppc_tb_freq; |
697 | tb_ticks_per_usec = ppc_tb_freq / 1000000; | 943 | tb_ticks_per_usec = ppc_tb_freq / 1000000; |
698 | tb_to_us = mulhwu_scale_factor(ppc_tb_freq, 1000000); | 944 | tb_to_us = mulhwu_scale_factor(ppc_tb_freq, 1000000); |
699 | div128_by_32(1024*1024, 0, tb_ticks_per_sec, &res); | 945 | calc_cputime_factors(); |
700 | tb_to_xs = res.result_low; | 946 | |
947 | /* | ||
948 | * Calculate the length of each tick in ns. It will not be | ||
949 | * exactly 1e9/HZ unless ppc_tb_freq is divisible by HZ. | ||
950 | * We compute 1e9 * tb_ticks_per_jiffy / ppc_tb_freq, | ||
951 | * rounded up. | ||
952 | */ | ||
953 | x = (u64) NSEC_PER_SEC * tb_ticks_per_jiffy + ppc_tb_freq - 1; | ||
954 | do_div(x, ppc_tb_freq); | ||
955 | tick_nsec = x; | ||
956 | last_tick_len = x << TICKLEN_SCALE; | ||
957 | |||
958 | /* | ||
959 | * Compute ticklen_to_xs, which is a factor which gets multiplied | ||
960 | * by (last_tick_len << TICKLEN_SHIFT) to get a tb_to_xs value. | ||
961 | * It is computed as: | ||
962 | * ticklen_to_xs = 2^N / (tb_ticks_per_jiffy * 1e9) | ||
963 | * where N = 64 + 20 - TICKLEN_SCALE - TICKLEN_SHIFT | ||
964 | * so as to give the result as a 0.64 fixed-point fraction. | ||
965 | */ | ||
966 | div128_by_32(1ULL << (64 + 20 - TICKLEN_SCALE - TICKLEN_SHIFT), 0, | ||
967 | tb_ticks_per_jiffy, &res); | ||
968 | div128_by_32(res.result_high, res.result_low, NSEC_PER_SEC, &res); | ||
969 | ticklen_to_xs = res.result_low; | ||
970 | |||
971 | /* Compute tb_to_xs from tick_nsec */ | ||
972 | tb_to_xs = mulhdu(last_tick_len << TICKLEN_SHIFT, ticklen_to_xs); | ||
701 | 973 | ||
702 | /* | 974 | /* |
703 | * Compute scale factor for sched_clock. | 975 | * Compute scale factor for sched_clock. |
@@ -724,6 +996,14 @@ void __init time_init(void) | |||
724 | tm = get_boot_time(); | 996 | tm = get_boot_time(); |
725 | 997 | ||
726 | write_seqlock_irqsave(&xtime_lock, flags); | 998 | write_seqlock_irqsave(&xtime_lock, flags); |
999 | |||
1000 | /* If platform provided a timezone (pmac), we correct the time */ | ||
1001 | if (timezone_offset) { | ||
1002 | sys_tz.tz_minuteswest = -timezone_offset / 60; | ||
1003 | sys_tz.tz_dsttime = 0; | ||
1004 | tm -= timezone_offset; | ||
1005 | } | ||
1006 | |||
727 | xtime.tv_sec = tm; | 1007 | xtime.tv_sec = tm; |
728 | xtime.tv_nsec = 0; | 1008 | xtime.tv_nsec = 0; |
729 | do_gtod.varp = &do_gtod.vars[0]; | 1009 | do_gtod.varp = &do_gtod.vars[0]; |
@@ -738,18 +1018,11 @@ void __init time_init(void) | |||
738 | vdso_data->tb_orig_stamp = tb_last_jiffy; | 1018 | vdso_data->tb_orig_stamp = tb_last_jiffy; |
739 | vdso_data->tb_update_count = 0; | 1019 | vdso_data->tb_update_count = 0; |
740 | vdso_data->tb_ticks_per_sec = tb_ticks_per_sec; | 1020 | vdso_data->tb_ticks_per_sec = tb_ticks_per_sec; |
741 | vdso_data->stamp_xsec = xtime.tv_sec * XSEC_PER_SEC; | 1021 | vdso_data->stamp_xsec = (u64) xtime.tv_sec * XSEC_PER_SEC; |
742 | vdso_data->tb_to_xs = tb_to_xs; | 1022 | vdso_data->tb_to_xs = tb_to_xs; |
743 | 1023 | ||
744 | time_freq = 0; | 1024 | time_freq = 0; |
745 | 1025 | ||
746 | /* If platform provided a timezone (pmac), we correct the time */ | ||
747 | if (timezone_offset) { | ||
748 | sys_tz.tz_minuteswest = -timezone_offset / 60; | ||
749 | sys_tz.tz_dsttime = 0; | ||
750 | xtime.tv_sec -= timezone_offset; | ||
751 | } | ||
752 | |||
753 | last_rtc_update = xtime.tv_sec; | 1026 | last_rtc_update = xtime.tv_sec; |
754 | set_normalized_timespec(&wall_to_monotonic, | 1027 | set_normalized_timespec(&wall_to_monotonic, |
755 | -xtime.tv_sec, -xtime.tv_nsec); | 1028 | -xtime.tv_sec, -xtime.tv_nsec); |
@@ -759,126 +1032,6 @@ void __init time_init(void) | |||
759 | set_dec(tb_ticks_per_jiffy); | 1032 | set_dec(tb_ticks_per_jiffy); |
760 | } | 1033 | } |
761 | 1034 | ||
762 | /* | ||
763 | * After adjtimex is called, adjust the conversion of tb ticks | ||
764 | * to microseconds to keep do_gettimeofday synchronized | ||
765 | * with ntpd. | ||
766 | * | ||
767 | * Use the time_adjust, time_freq and time_offset computed by adjtimex to | ||
768 | * adjust the frequency. | ||
769 | */ | ||
770 | |||
771 | /* #define DEBUG_PPC_ADJTIMEX 1 */ | ||
772 | |||
773 | void ppc_adjtimex(void) | ||
774 | { | ||
775 | #ifdef CONFIG_PPC64 | ||
776 | unsigned long den, new_tb_ticks_per_sec, tb_ticks, old_xsec, | ||
777 | new_tb_to_xs, new_xsec, new_stamp_xsec; | ||
778 | unsigned long tb_ticks_per_sec_delta; | ||
779 | long delta_freq, ltemp; | ||
780 | struct div_result divres; | ||
781 | unsigned long flags; | ||
782 | long singleshot_ppm = 0; | ||
783 | |||
784 | /* | ||
785 | * Compute parts per million frequency adjustment to | ||
786 | * accomplish the time adjustment implied by time_offset to be | ||
787 | * applied over the elapsed time indicated by time_constant. | ||
788 | * Use SHIFT_USEC to get it into the same units as | ||
789 | * time_freq. | ||
790 | */ | ||
791 | if ( time_offset < 0 ) { | ||
792 | ltemp = -time_offset; | ||
793 | ltemp <<= SHIFT_USEC - SHIFT_UPDATE; | ||
794 | ltemp >>= SHIFT_KG + time_constant; | ||
795 | ltemp = -ltemp; | ||
796 | } else { | ||
797 | ltemp = time_offset; | ||
798 | ltemp <<= SHIFT_USEC - SHIFT_UPDATE; | ||
799 | ltemp >>= SHIFT_KG + time_constant; | ||
800 | } | ||
801 | |||
802 | /* If there is a single shot time adjustment in progress */ | ||
803 | if ( time_adjust ) { | ||
804 | #ifdef DEBUG_PPC_ADJTIMEX | ||
805 | printk("ppc_adjtimex: "); | ||
806 | if ( adjusting_time == 0 ) | ||
807 | printk("starting "); | ||
808 | printk("single shot time_adjust = %ld\n", time_adjust); | ||
809 | #endif | ||
810 | |||
811 | adjusting_time = 1; | ||
812 | |||
813 | /* | ||
814 | * Compute parts per million frequency adjustment | ||
815 | * to match time_adjust | ||
816 | */ | ||
817 | singleshot_ppm = tickadj * HZ; | ||
818 | /* | ||
819 | * The adjustment should be tickadj*HZ to match the code in | ||
820 | * linux/kernel/timer.c, but experiments show that this is too | ||
821 | * large. 3/4 of tickadj*HZ seems about right | ||
822 | */ | ||
823 | singleshot_ppm -= singleshot_ppm / 4; | ||
824 | /* Use SHIFT_USEC to get it into the same units as time_freq */ | ||
825 | singleshot_ppm <<= SHIFT_USEC; | ||
826 | if ( time_adjust < 0 ) | ||
827 | singleshot_ppm = -singleshot_ppm; | ||
828 | } | ||
829 | else { | ||
830 | #ifdef DEBUG_PPC_ADJTIMEX | ||
831 | if ( adjusting_time ) | ||
832 | printk("ppc_adjtimex: ending single shot time_adjust\n"); | ||
833 | #endif | ||
834 | adjusting_time = 0; | ||
835 | } | ||
836 | |||
837 | /* Add up all of the frequency adjustments */ | ||
838 | delta_freq = time_freq + ltemp + singleshot_ppm; | ||
839 | |||
840 | /* | ||
841 | * Compute a new value for tb_ticks_per_sec based on | ||
842 | * the frequency adjustment | ||
843 | */ | ||
844 | den = 1000000 * (1 << (SHIFT_USEC - 8)); | ||
845 | if ( delta_freq < 0 ) { | ||
846 | tb_ticks_per_sec_delta = ( tb_ticks_per_sec * ( (-delta_freq) >> (SHIFT_USEC - 8))) / den; | ||
847 | new_tb_ticks_per_sec = tb_ticks_per_sec + tb_ticks_per_sec_delta; | ||
848 | } | ||
849 | else { | ||
850 | tb_ticks_per_sec_delta = ( tb_ticks_per_sec * ( delta_freq >> (SHIFT_USEC - 8))) / den; | ||
851 | new_tb_ticks_per_sec = tb_ticks_per_sec - tb_ticks_per_sec_delta; | ||
852 | } | ||
853 | |||
854 | #ifdef DEBUG_PPC_ADJTIMEX | ||
855 | printk("ppc_adjtimex: ltemp = %ld, time_freq = %ld, singleshot_ppm = %ld\n", ltemp, time_freq, singleshot_ppm); | ||
856 | printk("ppc_adjtimex: tb_ticks_per_sec - base = %ld new = %ld\n", tb_ticks_per_sec, new_tb_ticks_per_sec); | ||
857 | #endif | ||
858 | |||
859 | /* | ||
860 | * Compute a new value of tb_to_xs (used to convert tb to | ||
861 | * microseconds) and a new value of stamp_xsec which is the | ||
862 | * time (in 1/2^20 second units) corresponding to | ||
863 | * tb_orig_stamp. This new value of stamp_xsec compensates | ||
864 | * for the change in frequency (implied by the new tb_to_xs) | ||
865 | * which guarantees that the current time remains the same. | ||
866 | */ | ||
867 | write_seqlock_irqsave( &xtime_lock, flags ); | ||
868 | tb_ticks = get_tb() - do_gtod.varp->tb_orig_stamp; | ||
869 | div128_by_32(1024*1024, 0, new_tb_ticks_per_sec, &divres); | ||
870 | new_tb_to_xs = divres.result_low; | ||
871 | new_xsec = mulhdu(tb_ticks, new_tb_to_xs); | ||
872 | |||
873 | old_xsec = mulhdu(tb_ticks, do_gtod.varp->tb_to_xs); | ||
874 | new_stamp_xsec = do_gtod.varp->stamp_xsec + old_xsec - new_xsec; | ||
875 | |||
876 | update_gtod(do_gtod.varp->tb_orig_stamp, new_stamp_xsec, new_tb_to_xs); | ||
877 | |||
878 | write_sequnlock_irqrestore( &xtime_lock, flags ); | ||
879 | #endif /* CONFIG_PPC64 */ | ||
880 | } | ||
881 | |||
882 | 1035 | ||
883 | #define FEBRUARY 2 | 1036 | #define FEBRUARY 2 |
884 | #define STARTOFTIME 1970 | 1037 | #define STARTOFTIME 1970 |