diff options
Diffstat (limited to 'kernel/timer.c')
-rw-r--r-- | kernel/timer.c | 400 |
1 files changed, 363 insertions, 37 deletions
diff --git a/kernel/timer.c b/kernel/timer.c index eb97371b87d8..5a8960253063 100644 --- a/kernel/timer.c +++ b/kernel/timer.c | |||
@@ -597,7 +597,6 @@ long time_tolerance = MAXFREQ; /* frequency tolerance (ppm) */ | |||
597 | long time_precision = 1; /* clock precision (us) */ | 597 | long time_precision = 1; /* clock precision (us) */ |
598 | long time_maxerror = NTP_PHASE_LIMIT; /* maximum error (us) */ | 598 | long time_maxerror = NTP_PHASE_LIMIT; /* maximum error (us) */ |
599 | long time_esterror = NTP_PHASE_LIMIT; /* estimated error (us) */ | 599 | long time_esterror = NTP_PHASE_LIMIT; /* estimated error (us) */ |
600 | static long time_phase; /* phase offset (scaled us) */ | ||
601 | long time_freq = (((NSEC_PER_SEC + HZ/2) % HZ - HZ/2) << SHIFT_USEC) / NSEC_PER_USEC; | 600 | long time_freq = (((NSEC_PER_SEC + HZ/2) % HZ - HZ/2) << SHIFT_USEC) / NSEC_PER_USEC; |
602 | /* frequency offset (scaled ppm)*/ | 601 | /* frequency offset (scaled ppm)*/ |
603 | static long time_adj; /* tick adjust (scaled 1 / HZ) */ | 602 | static long time_adj; /* tick adjust (scaled 1 / HZ) */ |
@@ -747,27 +746,14 @@ static long adjtime_adjustment(void) | |||
747 | } | 746 | } |
748 | 747 | ||
749 | /* in the NTP reference this is called "hardclock()" */ | 748 | /* in the NTP reference this is called "hardclock()" */ |
750 | static void update_wall_time_one_tick(void) | 749 | static void update_ntp_one_tick(void) |
751 | { | 750 | { |
752 | long time_adjust_step, delta_nsec; | 751 | long time_adjust_step; |
753 | 752 | ||
754 | time_adjust_step = adjtime_adjustment(); | 753 | time_adjust_step = adjtime_adjustment(); |
755 | if (time_adjust_step) | 754 | if (time_adjust_step) |
756 | /* Reduce by this step the amount of time left */ | 755 | /* Reduce by this step the amount of time left */ |
757 | time_adjust -= time_adjust_step; | 756 | time_adjust -= time_adjust_step; |
758 | delta_nsec = tick_nsec + time_adjust_step * 1000; | ||
759 | /* | ||
760 | * Advance the phase, once it gets to one microsecond, then | ||
761 | * advance the tick more. | ||
762 | */ | ||
763 | time_phase += time_adj; | ||
764 | if ((time_phase >= FINENSEC) || (time_phase <= -FINENSEC)) { | ||
765 | long ltemp = shift_right(time_phase, (SHIFT_SCALE - 10)); | ||
766 | time_phase -= ltemp << (SHIFT_SCALE - 10); | ||
767 | delta_nsec += ltemp; | ||
768 | } | ||
769 | xtime.tv_nsec += delta_nsec; | ||
770 | time_interpolator_update(delta_nsec); | ||
771 | 757 | ||
772 | /* Changes by adjtime() do not take effect till next tick. */ | 758 | /* Changes by adjtime() do not take effect till next tick. */ |
773 | if (time_next_adjust != 0) { | 759 | if (time_next_adjust != 0) { |
@@ -780,36 +766,378 @@ static void update_wall_time_one_tick(void) | |||
780 | * Return how long ticks are at the moment, that is, how much time | 766 | * Return how long ticks are at the moment, that is, how much time |
781 | * update_wall_time_one_tick will add to xtime next time we call it | 767 | * update_wall_time_one_tick will add to xtime next time we call it |
782 | * (assuming no calls to do_adjtimex in the meantime). | 768 | * (assuming no calls to do_adjtimex in the meantime). |
783 | * The return value is in fixed-point nanoseconds with SHIFT_SCALE-10 | 769 | * The return value is in fixed-point nanoseconds shifted by the |
784 | * bits to the right of the binary point. | 770 | * specified number of bits to the right of the binary point. |
785 | * This function has no side-effects. | 771 | * This function has no side-effects. |
786 | */ | 772 | */ |
787 | u64 current_tick_length(void) | 773 | u64 current_tick_length(void) |
788 | { | 774 | { |
789 | long delta_nsec; | 775 | long delta_nsec; |
776 | u64 ret; | ||
790 | 777 | ||
778 | /* calculate the finest interval NTP will allow. | ||
779 | * ie: nanosecond value shifted by (SHIFT_SCALE - 10) | ||
780 | */ | ||
791 | delta_nsec = tick_nsec + adjtime_adjustment() * 1000; | 781 | delta_nsec = tick_nsec + adjtime_adjustment() * 1000; |
792 | return ((u64) delta_nsec << (SHIFT_SCALE - 10)) + time_adj; | 782 | ret = (u64)delta_nsec << TICK_LENGTH_SHIFT; |
783 | ret += (s64)time_adj << (TICK_LENGTH_SHIFT - (SHIFT_SCALE - 10)); | ||
784 | |||
785 | return ret; | ||
793 | } | 786 | } |
794 | 787 | ||
795 | /* | 788 | /* XXX - all of this timekeeping code should be later moved to time.c */ |
796 | * Using a loop looks inefficient, but "ticks" is | 789 | #include <linux/clocksource.h> |
797 | * usually just one (we shouldn't be losing ticks, | 790 | static struct clocksource *clock; /* pointer to current clocksource */ |
798 | * we're doing this this way mainly for interrupt | 791 | |
799 | * latency reasons, not because we think we'll | 792 | #ifdef CONFIG_GENERIC_TIME |
800 | * have lots of lost timer ticks | 793 | /** |
794 | * __get_nsec_offset - Returns nanoseconds since last call to periodic_hook | ||
795 | * | ||
796 | * private function, must hold xtime_lock lock when being | ||
797 | * called. Returns the number of nanoseconds since the | ||
798 | * last call to update_wall_time() (adjusted by NTP scaling) | ||
799 | */ | ||
800 | static inline s64 __get_nsec_offset(void) | ||
801 | { | ||
802 | cycle_t cycle_now, cycle_delta; | ||
803 | s64 ns_offset; | ||
804 | |||
805 | /* read clocksource: */ | ||
806 | cycle_now = clocksource_read(clock); | ||
807 | |||
808 | /* calculate the delta since the last update_wall_time: */ | ||
809 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; | ||
810 | |||
811 | /* convert to nanoseconds: */ | ||
812 | ns_offset = cyc2ns(clock, cycle_delta); | ||
813 | |||
814 | return ns_offset; | ||
815 | } | ||
816 | |||
817 | /** | ||
818 | * __get_realtime_clock_ts - Returns the time of day in a timespec | ||
819 | * @ts: pointer to the timespec to be set | ||
820 | * | ||
821 | * Returns the time of day in a timespec. Used by | ||
822 | * do_gettimeofday() and get_realtime_clock_ts(). | ||
801 | */ | 823 | */ |
802 | static void update_wall_time(unsigned long ticks) | 824 | static inline void __get_realtime_clock_ts(struct timespec *ts) |
803 | { | 825 | { |
826 | unsigned long seq; | ||
827 | s64 nsecs; | ||
828 | |||
829 | do { | ||
830 | seq = read_seqbegin(&xtime_lock); | ||
831 | |||
832 | *ts = xtime; | ||
833 | nsecs = __get_nsec_offset(); | ||
834 | |||
835 | } while (read_seqretry(&xtime_lock, seq)); | ||
836 | |||
837 | timespec_add_ns(ts, nsecs); | ||
838 | } | ||
839 | |||
840 | /** | ||
841 | * getnstimeofday - Returns the time of day in a timespec | ||
842 | * @ts: pointer to the timespec to be set | ||
843 | * | ||
844 | * Returns the time of day in a timespec. | ||
845 | */ | ||
846 | void getnstimeofday(struct timespec *ts) | ||
847 | { | ||
848 | __get_realtime_clock_ts(ts); | ||
849 | } | ||
850 | |||
851 | EXPORT_SYMBOL(getnstimeofday); | ||
852 | |||
853 | /** | ||
854 | * do_gettimeofday - Returns the time of day in a timeval | ||
855 | * @tv: pointer to the timeval to be set | ||
856 | * | ||
857 | * NOTE: Users should be converted to using get_realtime_clock_ts() | ||
858 | */ | ||
859 | void do_gettimeofday(struct timeval *tv) | ||
860 | { | ||
861 | struct timespec now; | ||
862 | |||
863 | __get_realtime_clock_ts(&now); | ||
864 | tv->tv_sec = now.tv_sec; | ||
865 | tv->tv_usec = now.tv_nsec/1000; | ||
866 | } | ||
867 | |||
868 | EXPORT_SYMBOL(do_gettimeofday); | ||
869 | /** | ||
870 | * do_settimeofday - Sets the time of day | ||
871 | * @tv: pointer to the timespec variable containing the new time | ||
872 | * | ||
873 | * Sets the time of day to the new time and update NTP and notify hrtimers | ||
874 | */ | ||
875 | int do_settimeofday(struct timespec *tv) | ||
876 | { | ||
877 | unsigned long flags; | ||
878 | time_t wtm_sec, sec = tv->tv_sec; | ||
879 | long wtm_nsec, nsec = tv->tv_nsec; | ||
880 | |||
881 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) | ||
882 | return -EINVAL; | ||
883 | |||
884 | write_seqlock_irqsave(&xtime_lock, flags); | ||
885 | |||
886 | nsec -= __get_nsec_offset(); | ||
887 | |||
888 | wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - sec); | ||
889 | wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - nsec); | ||
890 | |||
891 | set_normalized_timespec(&xtime, sec, nsec); | ||
892 | set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec); | ||
893 | |||
894 | ntp_clear(); | ||
895 | |||
896 | write_sequnlock_irqrestore(&xtime_lock, flags); | ||
897 | |||
898 | /* signal hrtimers about time change */ | ||
899 | clock_was_set(); | ||
900 | |||
901 | return 0; | ||
902 | } | ||
903 | |||
904 | EXPORT_SYMBOL(do_settimeofday); | ||
905 | |||
906 | /** | ||
907 | * change_clocksource - Swaps clocksources if a new one is available | ||
908 | * | ||
909 | * Accumulates current time interval and initializes new clocksource | ||
910 | */ | ||
911 | static int change_clocksource(void) | ||
912 | { | ||
913 | struct clocksource *new; | ||
914 | cycle_t now; | ||
915 | u64 nsec; | ||
916 | new = clocksource_get_next(); | ||
917 | if (clock != new) { | ||
918 | now = clocksource_read(new); | ||
919 | nsec = __get_nsec_offset(); | ||
920 | timespec_add_ns(&xtime, nsec); | ||
921 | |||
922 | clock = new; | ||
923 | clock->cycle_last = now; | ||
924 | printk(KERN_INFO "Time: %s clocksource has been installed.\n", | ||
925 | clock->name); | ||
926 | return 1; | ||
927 | } else if (clock->update_callback) { | ||
928 | return clock->update_callback(); | ||
929 | } | ||
930 | return 0; | ||
931 | } | ||
932 | #else | ||
933 | #define change_clocksource() (0) | ||
934 | #endif | ||
935 | |||
936 | /** | ||
937 | * timeofday_is_continuous - check to see if timekeeping is free running | ||
938 | */ | ||
939 | int timekeeping_is_continuous(void) | ||
940 | { | ||
941 | unsigned long seq; | ||
942 | int ret; | ||
943 | |||
804 | do { | 944 | do { |
805 | ticks--; | 945 | seq = read_seqbegin(&xtime_lock); |
806 | update_wall_time_one_tick(); | 946 | |
807 | if (xtime.tv_nsec >= 1000000000) { | 947 | ret = clock->is_continuous; |
808 | xtime.tv_nsec -= 1000000000; | 948 | |
949 | } while (read_seqretry(&xtime_lock, seq)); | ||
950 | |||
951 | return ret; | ||
952 | } | ||
953 | |||
954 | /* | ||
955 | * timekeeping_init - Initializes the clocksource and common timekeeping values | ||
956 | */ | ||
957 | void __init timekeeping_init(void) | ||
958 | { | ||
959 | unsigned long flags; | ||
960 | |||
961 | write_seqlock_irqsave(&xtime_lock, flags); | ||
962 | clock = clocksource_get_next(); | ||
963 | clocksource_calculate_interval(clock, tick_nsec); | ||
964 | clock->cycle_last = clocksource_read(clock); | ||
965 | ntp_clear(); | ||
966 | write_sequnlock_irqrestore(&xtime_lock, flags); | ||
967 | } | ||
968 | |||
969 | |||
970 | /* | ||
971 | * timekeeping_resume - Resumes the generic timekeeping subsystem. | ||
972 | * @dev: unused | ||
973 | * | ||
974 | * This is for the generic clocksource timekeeping. | ||
975 | * xtime/wall_to_monotonic/jiffies/wall_jiffies/etc are | ||
976 | * still managed by arch specific suspend/resume code. | ||
977 | */ | ||
978 | static int timekeeping_resume(struct sys_device *dev) | ||
979 | { | ||
980 | unsigned long flags; | ||
981 | |||
982 | write_seqlock_irqsave(&xtime_lock, flags); | ||
983 | /* restart the last cycle value */ | ||
984 | clock->cycle_last = clocksource_read(clock); | ||
985 | write_sequnlock_irqrestore(&xtime_lock, flags); | ||
986 | return 0; | ||
987 | } | ||
988 | |||
989 | /* sysfs resume/suspend bits for timekeeping */ | ||
990 | static struct sysdev_class timekeeping_sysclass = { | ||
991 | .resume = timekeeping_resume, | ||
992 | set_kset_name("timekeeping"), | ||
993 | }; | ||
994 | |||
995 | static struct sys_device device_timer = { | ||
996 | .id = 0, | ||
997 | .cls = &timekeeping_sysclass, | ||
998 | }; | ||
999 | |||
1000 | static int __init timekeeping_init_device(void) | ||
1001 | { | ||
1002 | int error = sysdev_class_register(&timekeeping_sysclass); | ||
1003 | if (!error) | ||
1004 | error = sysdev_register(&device_timer); | ||
1005 | return error; | ||
1006 | } | ||
1007 | |||
1008 | device_initcall(timekeeping_init_device); | ||
1009 | |||
1010 | /* | ||
1011 | * If the error is already larger, we look ahead another tick, | ||
1012 | * to compensate for late or lost adjustments. | ||
1013 | */ | ||
1014 | static __always_inline int clocksource_bigadjust(int sign, s64 error, s64 *interval, s64 *offset) | ||
1015 | { | ||
1016 | int adj; | ||
1017 | |||
1018 | /* | ||
1019 | * As soon as the machine is synchronized to the external time | ||
1020 | * source this should be the common case. | ||
1021 | */ | ||
1022 | error >>= 2; | ||
1023 | if (likely(sign > 0 ? error <= *interval : error >= *interval)) | ||
1024 | return sign; | ||
1025 | |||
1026 | /* | ||
1027 | * An extra look ahead dampens the effect of the current error, | ||
1028 | * which can grow quite large with continously late updates, as | ||
1029 | * it would dominate the adjustment value and can lead to | ||
1030 | * oscillation. | ||
1031 | */ | ||
1032 | error += current_tick_length() >> (TICK_LENGTH_SHIFT - clock->shift + 1); | ||
1033 | error -= clock->xtime_interval >> 1; | ||
1034 | |||
1035 | adj = 0; | ||
1036 | while (1) { | ||
1037 | error >>= 1; | ||
1038 | if (sign > 0 ? error <= *interval : error >= *interval) | ||
1039 | break; | ||
1040 | adj++; | ||
1041 | } | ||
1042 | |||
1043 | /* | ||
1044 | * Add the current adjustments to the error and take the offset | ||
1045 | * into account, the latter can cause the error to be hardly | ||
1046 | * reduced at the next tick. Check the error again if there's | ||
1047 | * room for another adjustment, thus further reducing the error | ||
1048 | * which otherwise had to be corrected at the next update. | ||
1049 | */ | ||
1050 | error = (error << 1) - *interval + *offset; | ||
1051 | if (sign > 0 ? error > *interval : error < *interval) | ||
1052 | adj++; | ||
1053 | |||
1054 | *interval <<= adj; | ||
1055 | *offset <<= adj; | ||
1056 | return sign << adj; | ||
1057 | } | ||
1058 | |||
1059 | /* | ||
1060 | * Adjust the multiplier to reduce the error value, | ||
1061 | * this is optimized for the most common adjustments of -1,0,1, | ||
1062 | * for other values we can do a bit more work. | ||
1063 | */ | ||
1064 | static void clocksource_adjust(struct clocksource *clock, s64 offset) | ||
1065 | { | ||
1066 | s64 error, interval = clock->cycle_interval; | ||
1067 | int adj; | ||
1068 | |||
1069 | error = clock->error >> (TICK_LENGTH_SHIFT - clock->shift - 1); | ||
1070 | if (error > interval) { | ||
1071 | adj = clocksource_bigadjust(1, error, &interval, &offset); | ||
1072 | } else if (error < -interval) { | ||
1073 | interval = -interval; | ||
1074 | offset = -offset; | ||
1075 | adj = clocksource_bigadjust(-1, error, &interval, &offset); | ||
1076 | } else | ||
1077 | return; | ||
1078 | |||
1079 | clock->mult += adj; | ||
1080 | clock->xtime_interval += interval; | ||
1081 | clock->xtime_nsec -= offset; | ||
1082 | clock->error -= (interval - offset) << (TICK_LENGTH_SHIFT - clock->shift); | ||
1083 | } | ||
1084 | |||
1085 | /* | ||
1086 | * update_wall_time - Uses the current clocksource to increment the wall time | ||
1087 | * | ||
1088 | * Called from the timer interrupt, must hold a write on xtime_lock. | ||
1089 | */ | ||
1090 | static void update_wall_time(void) | ||
1091 | { | ||
1092 | cycle_t offset; | ||
1093 | |||
1094 | clock->xtime_nsec += (s64)xtime.tv_nsec << clock->shift; | ||
1095 | |||
1096 | #ifdef CONFIG_GENERIC_TIME | ||
1097 | offset = (clocksource_read(clock) - clock->cycle_last) & clock->mask; | ||
1098 | #else | ||
1099 | offset = clock->cycle_interval; | ||
1100 | #endif | ||
1101 | |||
1102 | /* normally this loop will run just once, however in the | ||
1103 | * case of lost or late ticks, it will accumulate correctly. | ||
1104 | */ | ||
1105 | while (offset >= clock->cycle_interval) { | ||
1106 | /* accumulate one interval */ | ||
1107 | clock->xtime_nsec += clock->xtime_interval; | ||
1108 | clock->cycle_last += clock->cycle_interval; | ||
1109 | offset -= clock->cycle_interval; | ||
1110 | |||
1111 | if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) { | ||
1112 | clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift; | ||
809 | xtime.tv_sec++; | 1113 | xtime.tv_sec++; |
810 | second_overflow(); | 1114 | second_overflow(); |
811 | } | 1115 | } |
812 | } while (ticks); | 1116 | |
1117 | /* interpolator bits */ | ||
1118 | time_interpolator_update(clock->xtime_interval | ||
1119 | >> clock->shift); | ||
1120 | /* increment the NTP state machine */ | ||
1121 | update_ntp_one_tick(); | ||
1122 | |||
1123 | /* accumulate error between NTP and clock interval */ | ||
1124 | clock->error += current_tick_length(); | ||
1125 | clock->error -= clock->xtime_interval << (TICK_LENGTH_SHIFT - clock->shift); | ||
1126 | } | ||
1127 | |||
1128 | /* correct the clock when NTP error is too big */ | ||
1129 | clocksource_adjust(clock, offset); | ||
1130 | |||
1131 | /* store full nanoseconds into xtime */ | ||
1132 | xtime.tv_nsec = clock->xtime_nsec >> clock->shift; | ||
1133 | clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift; | ||
1134 | |||
1135 | /* check to see if there is a new clocksource to use */ | ||
1136 | if (change_clocksource()) { | ||
1137 | clock->error = 0; | ||
1138 | clock->xtime_nsec = 0; | ||
1139 | clocksource_calculate_interval(clock, tick_nsec); | ||
1140 | } | ||
813 | } | 1141 | } |
814 | 1142 | ||
815 | /* | 1143 | /* |
@@ -915,10 +1243,8 @@ static inline void update_times(void) | |||
915 | unsigned long ticks; | 1243 | unsigned long ticks; |
916 | 1244 | ||
917 | ticks = jiffies - wall_jiffies; | 1245 | ticks = jiffies - wall_jiffies; |
918 | if (ticks) { | 1246 | wall_jiffies += ticks; |
919 | wall_jiffies += ticks; | 1247 | update_wall_time(); |
920 | update_wall_time(ticks); | ||
921 | } | ||
922 | calc_load(ticks); | 1248 | calc_load(ticks); |
923 | } | 1249 | } |
924 | 1250 | ||
@@ -1326,7 +1652,7 @@ static void __devinit migrate_timers(int cpu) | |||
1326 | } | 1652 | } |
1327 | #endif /* CONFIG_HOTPLUG_CPU */ | 1653 | #endif /* CONFIG_HOTPLUG_CPU */ |
1328 | 1654 | ||
1329 | static int timer_cpu_notify(struct notifier_block *self, | 1655 | static int __devinit timer_cpu_notify(struct notifier_block *self, |
1330 | unsigned long action, void *hcpu) | 1656 | unsigned long action, void *hcpu) |
1331 | { | 1657 | { |
1332 | long cpu = (long)hcpu; | 1658 | long cpu = (long)hcpu; |
@@ -1346,7 +1672,7 @@ static int timer_cpu_notify(struct notifier_block *self, | |||
1346 | return NOTIFY_OK; | 1672 | return NOTIFY_OK; |
1347 | } | 1673 | } |
1348 | 1674 | ||
1349 | static struct notifier_block timers_nb = { | 1675 | static struct notifier_block __devinitdata timers_nb = { |
1350 | .notifier_call = timer_cpu_notify, | 1676 | .notifier_call = timer_cpu_notify, |
1351 | }; | 1677 | }; |
1352 | 1678 | ||