aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched_clock.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched_clock.c')
-rw-r--r--kernel/sched_clock.c109
1 files changed, 13 insertions, 96 deletions
diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c
index 5a2dc7d8fd98..9a7844158ae8 100644
--- a/kernel/sched_clock.c
+++ b/kernel/sched_clock.c
@@ -44,11 +44,6 @@ unsigned long long __attribute__((weak)) sched_clock(void)
44 44
45#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 45#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
46 46
47#define MULTI_SHIFT 15
48/* Max is double, Min is 1/2 */
49#define MAX_MULTI (2LL << MULTI_SHIFT)
50#define MIN_MULTI (1LL << (MULTI_SHIFT-1))
51
52struct sched_clock_data { 47struct sched_clock_data {
53 /* 48 /*
54 * Raw spinlock - this is a special case: this might be called 49 * Raw spinlock - this is a special case: this might be called
@@ -62,10 +57,6 @@ struct sched_clock_data {
62 u64 tick_raw; 57 u64 tick_raw;
63 u64 tick_gtod; 58 u64 tick_gtod;
64 u64 clock; 59 u64 clock;
65 s64 multi;
66#ifdef CONFIG_NO_HZ
67 int check_max;
68#endif
69}; 60};
70 61
71static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); 62static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
@@ -97,53 +88,18 @@ void sched_clock_init(void)
97 scd->tick_raw = 0; 88 scd->tick_raw = 0;
98 scd->tick_gtod = ktime_now; 89 scd->tick_gtod = ktime_now;
99 scd->clock = ktime_now; 90 scd->clock = ktime_now;
100 scd->multi = 1 << MULTI_SHIFT;
101#ifdef CONFIG_NO_HZ
102 scd->check_max = 1;
103#endif
104 } 91 }
105 92
106 sched_clock_running = 1; 93 sched_clock_running = 1;
107} 94}
108 95
109#ifdef CONFIG_NO_HZ
110/*
111 * The dynamic ticks makes the delta jiffies inaccurate. This
112 * prevents us from checking the maximum time update.
113 * Disable the maximum check during stopped ticks.
114 */
115void sched_clock_tick_stop(int cpu)
116{
117 struct sched_clock_data *scd = cpu_sdc(cpu);
118
119 scd->check_max = 0;
120}
121
122void sched_clock_tick_start(int cpu)
123{
124 struct sched_clock_data *scd = cpu_sdc(cpu);
125
126 scd->check_max = 1;
127}
128
129static int check_max(struct sched_clock_data *scd)
130{
131 return scd->check_max;
132}
133#else
134static int check_max(struct sched_clock_data *scd)
135{
136 return 1;
137}
138#endif /* CONFIG_NO_HZ */
139
140/* 96/*
141 * update the percpu scd from the raw @now value 97 * update the percpu scd from the raw @now value
142 * 98 *
143 * - filter out backward motion 99 * - filter out backward motion
144 * - use jiffies to generate a min,max window to clip the raw values 100 * - use jiffies to generate a min,max window to clip the raw values
145 */ 101 */
146static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *time) 102static void __update_sched_clock(struct sched_clock_data *scd, u64 now)
147{ 103{
148 unsigned long now_jiffies = jiffies; 104 unsigned long now_jiffies = jiffies;
149 long delta_jiffies = now_jiffies - scd->tick_jiffies; 105 long delta_jiffies = now_jiffies - scd->tick_jiffies;
@@ -152,31 +108,16 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim
152 s64 delta = now - scd->prev_raw; 108 s64 delta = now - scd->prev_raw;
153 109
154 WARN_ON_ONCE(!irqs_disabled()); 110 WARN_ON_ONCE(!irqs_disabled());
155 111 min_clock = scd->tick_gtod + delta_jiffies * TICK_NSEC;
156 /*
157 * At schedule tick the clock can be just under the gtod. We don't
158 * want to push it too prematurely.
159 */
160 min_clock = scd->tick_gtod + (delta_jiffies * TICK_NSEC);
161 if (min_clock > TICK_NSEC)
162 min_clock -= TICK_NSEC / 2;
163 112
164 if (unlikely(delta < 0)) { 113 if (unlikely(delta < 0)) {
165 clock++; 114 clock++;
166 goto out; 115 goto out;
167 } 116 }
168 117
169 /* 118 max_clock = min_clock + TICK_NSEC;
170 * The clock must stay within a jiffie of the gtod.
171 * But since we may be at the start of a jiffy or the end of one
172 * we add another jiffy buffer.
173 */
174 max_clock = scd->tick_gtod + (2 + delta_jiffies) * TICK_NSEC;
175
176 delta *= scd->multi;
177 delta >>= MULTI_SHIFT;
178 119
179 if (unlikely(clock + delta > max_clock) && check_max(scd)) { 120 if (unlikely(clock + delta > max_clock)) {
180 if (clock < max_clock) 121 if (clock < max_clock)
181 clock = max_clock; 122 clock = max_clock;
182 else 123 else
@@ -189,12 +130,9 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim
189 if (unlikely(clock < min_clock)) 130 if (unlikely(clock < min_clock))
190 clock = min_clock; 131 clock = min_clock;
191 132
192 if (time) 133 scd->prev_raw = now;
193 *time = clock; 134 scd->tick_jiffies = now_jiffies;
194 else { 135 scd->clock = clock;
195 scd->prev_raw = now;
196 scd->clock = clock;
197 }
198} 136}
199 137
200static void lock_double_clock(struct sched_clock_data *data1, 138static void lock_double_clock(struct sched_clock_data *data1,
@@ -238,26 +176,21 @@ u64 sched_clock_cpu(int cpu)
238 now -= scd->tick_gtod; 176 now -= scd->tick_gtod;
239 177
240 __raw_spin_unlock(&my_scd->lock); 178 __raw_spin_unlock(&my_scd->lock);
241
242 __update_sched_clock(scd, now, &clock);
243
244 __raw_spin_unlock(&scd->lock);
245
246 } else { 179 } else {
247 __raw_spin_lock(&scd->lock); 180 __raw_spin_lock(&scd->lock);
248 __update_sched_clock(scd, now, NULL);
249 clock = scd->clock;
250 __raw_spin_unlock(&scd->lock);
251 } 181 }
252 182
183 __update_sched_clock(scd, now);
184 clock = scd->clock;
185
186 __raw_spin_unlock(&scd->lock);
187
253 return clock; 188 return clock;
254} 189}
255 190
256void sched_clock_tick(void) 191void sched_clock_tick(void)
257{ 192{
258 struct sched_clock_data *scd = this_scd(); 193 struct sched_clock_data *scd = this_scd();
259 unsigned long now_jiffies = jiffies;
260 s64 mult, delta_gtod, delta_raw;
261 u64 now, now_gtod; 194 u64 now, now_gtod;
262 195
263 if (unlikely(!sched_clock_running)) 196 if (unlikely(!sched_clock_running))
@@ -269,29 +202,14 @@ void sched_clock_tick(void)
269 now = sched_clock(); 202 now = sched_clock();
270 203
271 __raw_spin_lock(&scd->lock); 204 __raw_spin_lock(&scd->lock);
272 __update_sched_clock(scd, now, NULL); 205 __update_sched_clock(scd, now);
273 /* 206 /*
274 * update tick_gtod after __update_sched_clock() because that will 207 * update tick_gtod after __update_sched_clock() because that will
275 * already observe 1 new jiffy; adding a new tick_gtod to that would 208 * already observe 1 new jiffy; adding a new tick_gtod to that would
276 * increase the clock 2 jiffies. 209 * increase the clock 2 jiffies.
277 */ 210 */
278 delta_gtod = now_gtod - scd->tick_gtod;
279 delta_raw = now - scd->tick_raw;
280
281 if ((long)delta_raw > 0) {
282 mult = delta_gtod << MULTI_SHIFT;
283 do_div(mult, delta_raw);
284 scd->multi = mult;
285 if (scd->multi > MAX_MULTI)
286 scd->multi = MAX_MULTI;
287 else if (scd->multi < MIN_MULTI)
288 scd->multi = MIN_MULTI;
289 } else
290 scd->multi = 1 << MULTI_SHIFT;
291
292 scd->tick_raw = now; 211 scd->tick_raw = now;
293 scd->tick_gtod = now_gtod; 212 scd->tick_gtod = now_gtod;
294 scd->tick_jiffies = now_jiffies;
295 __raw_spin_unlock(&scd->lock); 213 __raw_spin_unlock(&scd->lock);
296} 214}
297 215
@@ -321,7 +239,6 @@ void sched_clock_idle_wakeup_event(u64 delta_ns)
321 __raw_spin_lock(&scd->lock); 239 __raw_spin_lock(&scd->lock);
322 scd->prev_raw = now; 240 scd->prev_raw = now;
323 scd->clock += delta_ns; 241 scd->clock += delta_ns;
324 scd->multi = 1 << MULTI_SHIFT;
325 __raw_spin_unlock(&scd->lock); 242 __raw_spin_unlock(&scd->lock);
326 243
327 touch_softlockup_watchdog(); 244 touch_softlockup_watchdog();