aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/sched.h17
-rw-r--r--kernel/sched_clock.c109
-rw-r--r--kernel/time/tick-sched.c2
3 files changed, 14 insertions, 114 deletions
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 5270d449ff9d..ea436bc1a0e2 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1572,28 +1572,13 @@ static inline void sched_clock_idle_sleep_event(void)
1572static inline void sched_clock_idle_wakeup_event(u64 delta_ns) 1572static inline void sched_clock_idle_wakeup_event(u64 delta_ns)
1573{ 1573{
1574} 1574}
1575 1575#else
1576#ifdef CONFIG_NO_HZ
1577static inline void sched_clock_tick_stop(int cpu)
1578{
1579}
1580
1581static inline void sched_clock_tick_start(int cpu)
1582{
1583}
1584#endif
1585
1586#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
1587extern void sched_clock_init(void); 1576extern void sched_clock_init(void);
1588extern u64 sched_clock_cpu(int cpu); 1577extern u64 sched_clock_cpu(int cpu);
1589extern void sched_clock_tick(void); 1578extern void sched_clock_tick(void);
1590extern void sched_clock_idle_sleep_event(void); 1579extern void sched_clock_idle_sleep_event(void);
1591extern void sched_clock_idle_wakeup_event(u64 delta_ns); 1580extern void sched_clock_idle_wakeup_event(u64 delta_ns);
1592#ifdef CONFIG_NO_HZ
1593extern void sched_clock_tick_stop(int cpu);
1594extern void sched_clock_tick_start(int cpu);
1595#endif 1581#endif
1596#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
1597 1582
1598/* 1583/*
1599 * For kernel-internal use: high-speed (but slightly incorrect) per-cpu 1584 * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c
index 5a2dc7d8fd98..9a7844158ae8 100644
--- a/kernel/sched_clock.c
+++ b/kernel/sched_clock.c
@@ -44,11 +44,6 @@ unsigned long long __attribute__((weak)) sched_clock(void)
44 44
45#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 45#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
46 46
47#define MULTI_SHIFT 15
48/* Max is double, Min is 1/2 */
49#define MAX_MULTI (2LL << MULTI_SHIFT)
50#define MIN_MULTI (1LL << (MULTI_SHIFT-1))
51
52struct sched_clock_data { 47struct sched_clock_data {
53 /* 48 /*
54 * Raw spinlock - this is a special case: this might be called 49 * Raw spinlock - this is a special case: this might be called
@@ -62,10 +57,6 @@ struct sched_clock_data {
62 u64 tick_raw; 57 u64 tick_raw;
63 u64 tick_gtod; 58 u64 tick_gtod;
64 u64 clock; 59 u64 clock;
65 s64 multi;
66#ifdef CONFIG_NO_HZ
67 int check_max;
68#endif
69}; 60};
70 61
71static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); 62static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
@@ -97,53 +88,18 @@ void sched_clock_init(void)
97 scd->tick_raw = 0; 88 scd->tick_raw = 0;
98 scd->tick_gtod = ktime_now; 89 scd->tick_gtod = ktime_now;
99 scd->clock = ktime_now; 90 scd->clock = ktime_now;
100 scd->multi = 1 << MULTI_SHIFT;
101#ifdef CONFIG_NO_HZ
102 scd->check_max = 1;
103#endif
104 } 91 }
105 92
106 sched_clock_running = 1; 93 sched_clock_running = 1;
107} 94}
108 95
109#ifdef CONFIG_NO_HZ
110/*
111 * The dynamic ticks makes the delta jiffies inaccurate. This
112 * prevents us from checking the maximum time update.
113 * Disable the maximum check during stopped ticks.
114 */
115void sched_clock_tick_stop(int cpu)
116{
117 struct sched_clock_data *scd = cpu_sdc(cpu);
118
119 scd->check_max = 0;
120}
121
122void sched_clock_tick_start(int cpu)
123{
124 struct sched_clock_data *scd = cpu_sdc(cpu);
125
126 scd->check_max = 1;
127}
128
129static int check_max(struct sched_clock_data *scd)
130{
131 return scd->check_max;
132}
133#else
134static int check_max(struct sched_clock_data *scd)
135{
136 return 1;
137}
138#endif /* CONFIG_NO_HZ */
139
140/* 96/*
141 * update the percpu scd from the raw @now value 97 * update the percpu scd from the raw @now value
142 * 98 *
143 * - filter out backward motion 99 * - filter out backward motion
144 * - use jiffies to generate a min,max window to clip the raw values 100 * - use jiffies to generate a min,max window to clip the raw values
145 */ 101 */
146static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *time) 102static void __update_sched_clock(struct sched_clock_data *scd, u64 now)
147{ 103{
148 unsigned long now_jiffies = jiffies; 104 unsigned long now_jiffies = jiffies;
149 long delta_jiffies = now_jiffies - scd->tick_jiffies; 105 long delta_jiffies = now_jiffies - scd->tick_jiffies;
@@ -152,31 +108,16 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim
152 s64 delta = now - scd->prev_raw; 108 s64 delta = now - scd->prev_raw;
153 109
154 WARN_ON_ONCE(!irqs_disabled()); 110 WARN_ON_ONCE(!irqs_disabled());
155 111 min_clock = scd->tick_gtod + delta_jiffies * TICK_NSEC;
156 /*
157 * At schedule tick the clock can be just under the gtod. We don't
158 * want to push it too prematurely.
159 */
160 min_clock = scd->tick_gtod + (delta_jiffies * TICK_NSEC);
161 if (min_clock > TICK_NSEC)
162 min_clock -= TICK_NSEC / 2;
163 112
164 if (unlikely(delta < 0)) { 113 if (unlikely(delta < 0)) {
165 clock++; 114 clock++;
166 goto out; 115 goto out;
167 } 116 }
168 117
169 /* 118 max_clock = min_clock + TICK_NSEC;
170 * The clock must stay within a jiffie of the gtod.
171 * But since we may be at the start of a jiffy or the end of one
172 * we add another jiffy buffer.
173 */
174 max_clock = scd->tick_gtod + (2 + delta_jiffies) * TICK_NSEC;
175
176 delta *= scd->multi;
177 delta >>= MULTI_SHIFT;
178 119
179 if (unlikely(clock + delta > max_clock) && check_max(scd)) { 120 if (unlikely(clock + delta > max_clock)) {
180 if (clock < max_clock) 121 if (clock < max_clock)
181 clock = max_clock; 122 clock = max_clock;
182 else 123 else
@@ -189,12 +130,9 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim
189 if (unlikely(clock < min_clock)) 130 if (unlikely(clock < min_clock))
190 clock = min_clock; 131 clock = min_clock;
191 132
192 if (time) 133 scd->prev_raw = now;
193 *time = clock; 134 scd->tick_jiffies = now_jiffies;
194 else { 135 scd->clock = clock;
195 scd->prev_raw = now;
196 scd->clock = clock;
197 }
198} 136}
199 137
200static void lock_double_clock(struct sched_clock_data *data1, 138static void lock_double_clock(struct sched_clock_data *data1,
@@ -238,26 +176,21 @@ u64 sched_clock_cpu(int cpu)
238 now -= scd->tick_gtod; 176 now -= scd->tick_gtod;
239 177
240 __raw_spin_unlock(&my_scd->lock); 178 __raw_spin_unlock(&my_scd->lock);
241
242 __update_sched_clock(scd, now, &clock);
243
244 __raw_spin_unlock(&scd->lock);
245
246 } else { 179 } else {
247 __raw_spin_lock(&scd->lock); 180 __raw_spin_lock(&scd->lock);
248 __update_sched_clock(scd, now, NULL);
249 clock = scd->clock;
250 __raw_spin_unlock(&scd->lock);
251 } 181 }
252 182
183 __update_sched_clock(scd, now);
184 clock = scd->clock;
185
186 __raw_spin_unlock(&scd->lock);
187
253 return clock; 188 return clock;
254} 189}
255 190
256void sched_clock_tick(void) 191void sched_clock_tick(void)
257{ 192{
258 struct sched_clock_data *scd = this_scd(); 193 struct sched_clock_data *scd = this_scd();
259 unsigned long now_jiffies = jiffies;
260 s64 mult, delta_gtod, delta_raw;
261 u64 now, now_gtod; 194 u64 now, now_gtod;
262 195
263 if (unlikely(!sched_clock_running)) 196 if (unlikely(!sched_clock_running))
@@ -269,29 +202,14 @@ void sched_clock_tick(void)
269 now = sched_clock(); 202 now = sched_clock();
270 203
271 __raw_spin_lock(&scd->lock); 204 __raw_spin_lock(&scd->lock);
272 __update_sched_clock(scd, now, NULL); 205 __update_sched_clock(scd, now);
273 /* 206 /*
274 * update tick_gtod after __update_sched_clock() because that will 207 * update tick_gtod after __update_sched_clock() because that will
275 * already observe 1 new jiffy; adding a new tick_gtod to that would 208 * already observe 1 new jiffy; adding a new tick_gtod to that would
276 * increase the clock 2 jiffies. 209 * increase the clock 2 jiffies.
277 */ 210 */
278 delta_gtod = now_gtod - scd->tick_gtod;
279 delta_raw = now - scd->tick_raw;
280
281 if ((long)delta_raw > 0) {
282 mult = delta_gtod << MULTI_SHIFT;
283 do_div(mult, delta_raw);
284 scd->multi = mult;
285 if (scd->multi > MAX_MULTI)
286 scd->multi = MAX_MULTI;
287 else if (scd->multi < MIN_MULTI)
288 scd->multi = MIN_MULTI;
289 } else
290 scd->multi = 1 << MULTI_SHIFT;
291
292 scd->tick_raw = now; 211 scd->tick_raw = now;
293 scd->tick_gtod = now_gtod; 212 scd->tick_gtod = now_gtod;
294 scd->tick_jiffies = now_jiffies;
295 __raw_spin_unlock(&scd->lock); 213 __raw_spin_unlock(&scd->lock);
296} 214}
297 215
@@ -321,7 +239,6 @@ void sched_clock_idle_wakeup_event(u64 delta_ns)
321 __raw_spin_lock(&scd->lock); 239 __raw_spin_lock(&scd->lock);
322 scd->prev_raw = now; 240 scd->prev_raw = now;
323 scd->clock += delta_ns; 241 scd->clock += delta_ns;
324 scd->multi = 1 << MULTI_SHIFT;
325 __raw_spin_unlock(&scd->lock); 242 __raw_spin_unlock(&scd->lock);
326 243
327 touch_softlockup_watchdog(); 244 touch_softlockup_watchdog();
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 825b4c00fe44..f5da526424a9 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -289,7 +289,6 @@ void tick_nohz_stop_sched_tick(int inidle)
289 ts->tick_stopped = 1; 289 ts->tick_stopped = 1;
290 ts->idle_jiffies = last_jiffies; 290 ts->idle_jiffies = last_jiffies;
291 rcu_enter_nohz(); 291 rcu_enter_nohz();
292 sched_clock_tick_stop(cpu);
293 } 292 }
294 293
295 /* 294 /*
@@ -392,7 +391,6 @@ void tick_nohz_restart_sched_tick(void)
392 select_nohz_load_balancer(0); 391 select_nohz_load_balancer(0);
393 now = ktime_get(); 392 now = ktime_get();
394 tick_do_update_jiffies64(now); 393 tick_do_update_jiffies64(now);
395 sched_clock_tick_start(cpu);
396 cpu_clear(cpu, nohz_cpu_mask); 394 cpu_clear(cpu, nohz_cpu_mask);
397 395
398 /* 396 /*