aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/sched.h31
-rw-r--r--kernel/Kconfig.hz2
-rw-r--r--kernel/cpu.c5
-rw-r--r--kernel/sched.c2
-rw-r--r--kernel/sched_clock.c178
-rw-r--r--kernel/sched_fair.c21
-rw-r--r--kernel/time/tick-sched.c2
7 files changed, 77 insertions, 164 deletions
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 5270d449ff9d..5850bfb968a8 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1551,16 +1551,10 @@ static inline int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask)
1551 1551
1552extern unsigned long long sched_clock(void); 1552extern unsigned long long sched_clock(void);
1553 1553
1554#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 1554extern void sched_clock_init(void);
1555static inline void sched_clock_init(void) 1555extern u64 sched_clock_cpu(int cpu);
1556{
1557}
1558
1559static inline u64 sched_clock_cpu(int cpu)
1560{
1561 return sched_clock();
1562}
1563 1556
1557#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
1564static inline void sched_clock_tick(void) 1558static inline void sched_clock_tick(void)
1565{ 1559{
1566} 1560}
@@ -1572,28 +1566,11 @@ static inline void sched_clock_idle_sleep_event(void)
1572static inline void sched_clock_idle_wakeup_event(u64 delta_ns) 1566static inline void sched_clock_idle_wakeup_event(u64 delta_ns)
1573{ 1567{
1574} 1568}
1575 1569#else
1576#ifdef CONFIG_NO_HZ
1577static inline void sched_clock_tick_stop(int cpu)
1578{
1579}
1580
1581static inline void sched_clock_tick_start(int cpu)
1582{
1583}
1584#endif
1585
1586#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
1587extern void sched_clock_init(void);
1588extern u64 sched_clock_cpu(int cpu);
1589extern void sched_clock_tick(void); 1570extern void sched_clock_tick(void);
1590extern void sched_clock_idle_sleep_event(void); 1571extern void sched_clock_idle_sleep_event(void);
1591extern void sched_clock_idle_wakeup_event(u64 delta_ns); 1572extern void sched_clock_idle_wakeup_event(u64 delta_ns);
1592#ifdef CONFIG_NO_HZ
1593extern void sched_clock_tick_stop(int cpu);
1594extern void sched_clock_tick_start(int cpu);
1595#endif 1573#endif
1596#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
1597 1574
1598/* 1575/*
1599 * For kernel-internal use: high-speed (but slightly incorrect) per-cpu 1576 * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz
index 382dd5a8b2d7..94fabd534b03 100644
--- a/kernel/Kconfig.hz
+++ b/kernel/Kconfig.hz
@@ -55,4 +55,4 @@ config HZ
55 default 1000 if HZ_1000 55 default 1000 if HZ_1000
56 56
57config SCHED_HRTICK 57config SCHED_HRTICK
58 def_bool HIGH_RES_TIMERS && USE_GENERIC_SMP_HELPERS 58 def_bool HIGH_RES_TIMERS && (!SMP || USE_GENERIC_SMP_HELPERS)
diff --git a/kernel/cpu.c b/kernel/cpu.c
index e202a68d1cc1..c977c339f559 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -349,6 +349,8 @@ static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
349 goto out_notify; 349 goto out_notify;
350 BUG_ON(!cpu_online(cpu)); 350 BUG_ON(!cpu_online(cpu));
351 351
352 cpu_set(cpu, cpu_active_map);
353
352 /* Now call notifier in preparation. */ 354 /* Now call notifier in preparation. */
353 raw_notifier_call_chain(&cpu_chain, CPU_ONLINE | mod, hcpu); 355 raw_notifier_call_chain(&cpu_chain, CPU_ONLINE | mod, hcpu);
354 356
@@ -383,9 +385,6 @@ int __cpuinit cpu_up(unsigned int cpu)
383 385
384 err = _cpu_up(cpu, 0); 386 err = _cpu_up(cpu, 0);
385 387
386 if (cpu_online(cpu))
387 cpu_set(cpu, cpu_active_map);
388
389out: 388out:
390 cpu_maps_update_done(); 389 cpu_maps_update_done();
391 return err; 390 return err;
diff --git a/kernel/sched.c b/kernel/sched.c
index ace566bdfc68..d601fb0406ca 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -833,7 +833,7 @@ static inline u64 global_rt_period(void)
833 833
834static inline u64 global_rt_runtime(void) 834static inline u64 global_rt_runtime(void)
835{ 835{
836 if (sysctl_sched_rt_period < 0) 836 if (sysctl_sched_rt_runtime < 0)
837 return RUNTIME_INF; 837 return RUNTIME_INF;
838 838
839 return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC; 839 return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c
index 22ed55d1167f..204991a0bfa7 100644
--- a/kernel/sched_clock.c
+++ b/kernel/sched_clock.c
@@ -32,13 +32,19 @@
32#include <linux/ktime.h> 32#include <linux/ktime.h>
33#include <linux/module.h> 33#include <linux/module.h>
34 34
35/*
36 * Scheduler clock - returns current time in nanosec units.
37 * This is default implementation.
38 * Architectures and sub-architectures can override this.
39 */
40unsigned long long __attribute__((weak)) sched_clock(void)
41{
42 return (unsigned long long)jiffies * (NSEC_PER_SEC / HZ);
43}
35 44
36#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 45static __read_mostly int sched_clock_running;
37 46
38#define MULTI_SHIFT 15 47#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
39/* Max is double, Min is 1/2 */
40#define MAX_MULTI (2LL << MULTI_SHIFT)
41#define MIN_MULTI (1LL << (MULTI_SHIFT-1))
42 48
43struct sched_clock_data { 49struct sched_clock_data {
44 /* 50 /*
@@ -49,14 +55,9 @@ struct sched_clock_data {
49 raw_spinlock_t lock; 55 raw_spinlock_t lock;
50 56
51 unsigned long tick_jiffies; 57 unsigned long tick_jiffies;
52 u64 prev_raw;
53 u64 tick_raw; 58 u64 tick_raw;
54 u64 tick_gtod; 59 u64 tick_gtod;
55 u64 clock; 60 u64 clock;
56 s64 multi;
57#ifdef CONFIG_NO_HZ
58 int check_max;
59#endif
60}; 61};
61 62
62static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); 63static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
@@ -71,8 +72,6 @@ static inline struct sched_clock_data *cpu_sdc(int cpu)
71 return &per_cpu(sched_clock_data, cpu); 72 return &per_cpu(sched_clock_data, cpu);
72} 73}
73 74
74static __read_mostly int sched_clock_running;
75
76void sched_clock_init(void) 75void sched_clock_init(void)
77{ 76{
78 u64 ktime_now = ktime_to_ns(ktime_get()); 77 u64 ktime_now = ktime_to_ns(ktime_get());
@@ -84,90 +83,39 @@ void sched_clock_init(void)
84 83
85 scd->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; 84 scd->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
86 scd->tick_jiffies = now_jiffies; 85 scd->tick_jiffies = now_jiffies;
87 scd->prev_raw = 0;
88 scd->tick_raw = 0; 86 scd->tick_raw = 0;
89 scd->tick_gtod = ktime_now; 87 scd->tick_gtod = ktime_now;
90 scd->clock = ktime_now; 88 scd->clock = ktime_now;
91 scd->multi = 1 << MULTI_SHIFT;
92#ifdef CONFIG_NO_HZ
93 scd->check_max = 1;
94#endif
95 } 89 }
96 90
97 sched_clock_running = 1; 91 sched_clock_running = 1;
98} 92}
99 93
100#ifdef CONFIG_NO_HZ
101/*
102 * The dynamic ticks makes the delta jiffies inaccurate. This
103 * prevents us from checking the maximum time update.
104 * Disable the maximum check during stopped ticks.
105 */
106void sched_clock_tick_stop(int cpu)
107{
108 struct sched_clock_data *scd = cpu_sdc(cpu);
109
110 scd->check_max = 0;
111}
112
113void sched_clock_tick_start(int cpu)
114{
115 struct sched_clock_data *scd = cpu_sdc(cpu);
116
117 scd->check_max = 1;
118}
119
120static int check_max(struct sched_clock_data *scd)
121{
122 return scd->check_max;
123}
124#else
125static int check_max(struct sched_clock_data *scd)
126{
127 return 1;
128}
129#endif /* CONFIG_NO_HZ */
130
131/* 94/*
132 * update the percpu scd from the raw @now value 95 * update the percpu scd from the raw @now value
133 * 96 *
134 * - filter out backward motion 97 * - filter out backward motion
135 * - use jiffies to generate a min,max window to clip the raw values 98 * - use jiffies to generate a min,max window to clip the raw values
136 */ 99 */
137static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *time) 100static u64 __update_sched_clock(struct sched_clock_data *scd, u64 now)
138{ 101{
139 unsigned long now_jiffies = jiffies; 102 unsigned long now_jiffies = jiffies;
140 long delta_jiffies = now_jiffies - scd->tick_jiffies; 103 long delta_jiffies = now_jiffies - scd->tick_jiffies;
141 u64 clock = scd->clock; 104 u64 clock = scd->clock;
142 u64 min_clock, max_clock; 105 u64 min_clock, max_clock;
143 s64 delta = now - scd->prev_raw; 106 s64 delta = now - scd->tick_raw;
144 107
145 WARN_ON_ONCE(!irqs_disabled()); 108 WARN_ON_ONCE(!irqs_disabled());
146 109 min_clock = scd->tick_gtod + delta_jiffies * TICK_NSEC;
147 /*
148 * At schedule tick the clock can be just under the gtod. We don't
149 * want to push it too prematurely.
150 */
151 min_clock = scd->tick_gtod + (delta_jiffies * TICK_NSEC);
152 if (min_clock > TICK_NSEC)
153 min_clock -= TICK_NSEC / 2;
154 110
155 if (unlikely(delta < 0)) { 111 if (unlikely(delta < 0)) {
156 clock++; 112 clock++;
157 goto out; 113 goto out;
158 } 114 }
159 115
160 /* 116 max_clock = min_clock + TICK_NSEC;
161 * The clock must stay within a jiffie of the gtod.
162 * But since we may be at the start of a jiffy or the end of one
163 * we add another jiffy buffer.
164 */
165 max_clock = scd->tick_gtod + (2 + delta_jiffies) * TICK_NSEC;
166
167 delta *= scd->multi;
168 delta >>= MULTI_SHIFT;
169 117
170 if (unlikely(clock + delta > max_clock) && check_max(scd)) { 118 if (unlikely(clock + delta > max_clock)) {
171 if (clock < max_clock) 119 if (clock < max_clock)
172 clock = max_clock; 120 clock = max_clock;
173 else 121 else
@@ -180,12 +128,10 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim
180 if (unlikely(clock < min_clock)) 128 if (unlikely(clock < min_clock))
181 clock = min_clock; 129 clock = min_clock;
182 130
183 if (time) 131 scd->tick_jiffies = now_jiffies;
184 *time = clock; 132 scd->clock = clock;
185 else { 133
186 scd->prev_raw = now; 134 return clock;
187 scd->clock = clock;
188 }
189} 135}
190 136
191static void lock_double_clock(struct sched_clock_data *data1, 137static void lock_double_clock(struct sched_clock_data *data1,
@@ -203,7 +149,7 @@ static void lock_double_clock(struct sched_clock_data *data1,
203u64 sched_clock_cpu(int cpu) 149u64 sched_clock_cpu(int cpu)
204{ 150{
205 struct sched_clock_data *scd = cpu_sdc(cpu); 151 struct sched_clock_data *scd = cpu_sdc(cpu);
206 u64 now, clock; 152 u64 now, clock, this_clock, remote_clock;
207 153
208 if (unlikely(!sched_clock_running)) 154 if (unlikely(!sched_clock_running))
209 return 0ull; 155 return 0ull;
@@ -212,43 +158,44 @@ u64 sched_clock_cpu(int cpu)
212 now = sched_clock(); 158 now = sched_clock();
213 159
214 if (cpu != raw_smp_processor_id()) { 160 if (cpu != raw_smp_processor_id()) {
215 /*
216 * in order to update a remote cpu's clock based on our
217 * unstable raw time rebase it against:
218 * tick_raw (offset between raw counters)
219 * tick_gotd (tick offset between cpus)
220 */
221 struct sched_clock_data *my_scd = this_scd(); 161 struct sched_clock_data *my_scd = this_scd();
222 162
223 lock_double_clock(scd, my_scd); 163 lock_double_clock(scd, my_scd);
224 164
225 now -= my_scd->tick_raw; 165 this_clock = __update_sched_clock(my_scd, now);
226 now += scd->tick_raw; 166 remote_clock = scd->clock;
227 167
228 now += my_scd->tick_gtod; 168 /*
229 now -= scd->tick_gtod; 169 * Use the opportunity that we have both locks
170 * taken to couple the two clocks: we take the
171 * larger time as the latest time for both
172 * runqueues. (this creates monotonic movement)
173 */
174 if (likely(remote_clock < this_clock)) {
175 clock = this_clock;
176 scd->clock = clock;
177 } else {
178 /*
179 * Should be rare, but possible:
180 */
181 clock = remote_clock;
182 my_scd->clock = remote_clock;
183 }
230 184
231 __raw_spin_unlock(&my_scd->lock); 185 __raw_spin_unlock(&my_scd->lock);
232
233 __update_sched_clock(scd, now, &clock);
234
235 __raw_spin_unlock(&scd->lock);
236
237 } else { 186 } else {
238 __raw_spin_lock(&scd->lock); 187 __raw_spin_lock(&scd->lock);
239 __update_sched_clock(scd, now, NULL); 188 clock = __update_sched_clock(scd, now);
240 clock = scd->clock;
241 __raw_spin_unlock(&scd->lock);
242 } 189 }
243 190
191 __raw_spin_unlock(&scd->lock);
192
244 return clock; 193 return clock;
245} 194}
246 195
247void sched_clock_tick(void) 196void sched_clock_tick(void)
248{ 197{
249 struct sched_clock_data *scd = this_scd(); 198 struct sched_clock_data *scd = this_scd();
250 unsigned long now_jiffies = jiffies;
251 s64 mult, delta_gtod, delta_raw;
252 u64 now, now_gtod; 199 u64 now, now_gtod;
253 200
254 if (unlikely(!sched_clock_running)) 201 if (unlikely(!sched_clock_running))
@@ -260,29 +207,14 @@ void sched_clock_tick(void)
260 now = sched_clock(); 207 now = sched_clock();
261 208
262 __raw_spin_lock(&scd->lock); 209 __raw_spin_lock(&scd->lock);
263 __update_sched_clock(scd, now, NULL); 210 __update_sched_clock(scd, now);
264 /* 211 /*
265 * update tick_gtod after __update_sched_clock() because that will 212 * update tick_gtod after __update_sched_clock() because that will
266 * already observe 1 new jiffy; adding a new tick_gtod to that would 213 * already observe 1 new jiffy; adding a new tick_gtod to that would
267 * increase the clock 2 jiffies. 214 * increase the clock 2 jiffies.
268 */ 215 */
269 delta_gtod = now_gtod - scd->tick_gtod;
270 delta_raw = now - scd->tick_raw;
271
272 if ((long)delta_raw > 0) {
273 mult = delta_gtod << MULTI_SHIFT;
274 do_div(mult, delta_raw);
275 scd->multi = mult;
276 if (scd->multi > MAX_MULTI)
277 scd->multi = MAX_MULTI;
278 else if (scd->multi < MIN_MULTI)
279 scd->multi = MIN_MULTI;
280 } else
281 scd->multi = 1 << MULTI_SHIFT;
282
283 scd->tick_raw = now; 216 scd->tick_raw = now;
284 scd->tick_gtod = now_gtod; 217 scd->tick_gtod = now_gtod;
285 scd->tick_jiffies = now_jiffies;
286 __raw_spin_unlock(&scd->lock); 218 __raw_spin_unlock(&scd->lock);
287} 219}
288 220
@@ -301,7 +233,6 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event);
301void sched_clock_idle_wakeup_event(u64 delta_ns) 233void sched_clock_idle_wakeup_event(u64 delta_ns)
302{ 234{
303 struct sched_clock_data *scd = this_scd(); 235 struct sched_clock_data *scd = this_scd();
304 u64 now = sched_clock();
305 236
306 /* 237 /*
307 * Override the previous timestamp and ignore all 238 * Override the previous timestamp and ignore all
@@ -310,27 +241,30 @@ void sched_clock_idle_wakeup_event(u64 delta_ns)
310 * rq clock: 241 * rq clock:
311 */ 242 */
312 __raw_spin_lock(&scd->lock); 243 __raw_spin_lock(&scd->lock);
313 scd->prev_raw = now;
314 scd->clock += delta_ns; 244 scd->clock += delta_ns;
315 scd->multi = 1 << MULTI_SHIFT;
316 __raw_spin_unlock(&scd->lock); 245 __raw_spin_unlock(&scd->lock);
317 246
318 touch_softlockup_watchdog(); 247 touch_softlockup_watchdog();
319} 248}
320EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); 249EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
321 250
322#endif 251#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
323 252
324/* 253void sched_clock_init(void)
325 * Scheduler clock - returns current time in nanosec units.
326 * This is default implementation.
327 * Architectures and sub-architectures can override this.
328 */
329unsigned long long __attribute__((weak)) sched_clock(void)
330{ 254{
331 return (unsigned long long)jiffies * (NSEC_PER_SEC / HZ); 255 sched_clock_running = 1;
332} 256}
333 257
258u64 sched_clock_cpu(int cpu)
259{
260 if (unlikely(!sched_clock_running))
261 return 0;
262
263 return sched_clock();
264}
265
266#endif
267
334unsigned long long cpu_clock(int cpu) 268unsigned long long cpu_clock(int cpu)
335{ 269{
336 unsigned long long clock; 270 unsigned long long clock;
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index cf2cd6ce4cb2..fb8994c6d4bb 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -899,7 +899,7 @@ static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
899 * doesn't make sense. Rely on vruntime for fairness. 899 * doesn't make sense. Rely on vruntime for fairness.
900 */ 900 */
901 if (rq->curr != p) 901 if (rq->curr != p)
902 delta = max(10000LL, delta); 902 delta = max_t(s64, 10000LL, delta);
903 903
904 hrtick_start(rq, delta); 904 hrtick_start(rq, delta);
905 } 905 }
@@ -1442,18 +1442,23 @@ __load_balance_iterator(struct cfs_rq *cfs_rq, struct list_head *next)
1442 struct task_struct *p = NULL; 1442 struct task_struct *p = NULL;
1443 struct sched_entity *se; 1443 struct sched_entity *se;
1444 1444
1445 while (next != &cfs_rq->tasks) { 1445 if (next == &cfs_rq->tasks)
1446 return NULL;
1447
1448 /* Skip over entities that are not tasks */
1449 do {
1446 se = list_entry(next, struct sched_entity, group_node); 1450 se = list_entry(next, struct sched_entity, group_node);
1447 next = next->next; 1451 next = next->next;
1452 } while (next != &cfs_rq->tasks && !entity_is_task(se));
1448 1453
1449 /* Skip over entities that are not tasks */ 1454 if (next == &cfs_rq->tasks)
1450 if (entity_is_task(se)) { 1455 return NULL;
1451 p = task_of(se);
1452 break;
1453 }
1454 }
1455 1456
1456 cfs_rq->balance_iterator = next; 1457 cfs_rq->balance_iterator = next;
1458
1459 if (entity_is_task(se))
1460 p = task_of(se);
1461
1457 return p; 1462 return p;
1458} 1463}
1459 1464
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 825b4c00fe44..f5da526424a9 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -289,7 +289,6 @@ void tick_nohz_stop_sched_tick(int inidle)
289 ts->tick_stopped = 1; 289 ts->tick_stopped = 1;
290 ts->idle_jiffies = last_jiffies; 290 ts->idle_jiffies = last_jiffies;
291 rcu_enter_nohz(); 291 rcu_enter_nohz();
292 sched_clock_tick_stop(cpu);
293 } 292 }
294 293
295 /* 294 /*
@@ -392,7 +391,6 @@ void tick_nohz_restart_sched_tick(void)
392 select_nohz_load_balancer(0); 391 select_nohz_load_balancer(0);
393 now = ktime_get(); 392 now = ktime_get();
394 tick_do_update_jiffies64(now); 393 tick_do_update_jiffies64(now);
395 sched_clock_tick_start(cpu);
396 cpu_clear(cpu, nohz_cpu_mask); 394 cpu_clear(cpu, nohz_cpu_mask);
397 395
398 /* 396 /*