diff options
-rw-r--r-- | include/linux/sched.h | 31 | ||||
-rw-r--r-- | kernel/Kconfig.hz | 2 | ||||
-rw-r--r-- | kernel/cpu.c | 5 | ||||
-rw-r--r-- | kernel/sched.c | 2 | ||||
-rw-r--r-- | kernel/sched_clock.c | 178 | ||||
-rw-r--r-- | kernel/sched_fair.c | 21 | ||||
-rw-r--r-- | kernel/time/tick-sched.c | 2 |
7 files changed, 77 insertions, 164 deletions
diff --git a/include/linux/sched.h b/include/linux/sched.h index 5270d449ff9d..5850bfb968a8 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h | |||
@@ -1551,16 +1551,10 @@ static inline int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask) | |||
1551 | 1551 | ||
1552 | extern unsigned long long sched_clock(void); | 1552 | extern unsigned long long sched_clock(void); |
1553 | 1553 | ||
1554 | #ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK | 1554 | extern void sched_clock_init(void); |
1555 | static inline void sched_clock_init(void) | 1555 | extern u64 sched_clock_cpu(int cpu); |
1556 | { | ||
1557 | } | ||
1558 | |||
1559 | static inline u64 sched_clock_cpu(int cpu) | ||
1560 | { | ||
1561 | return sched_clock(); | ||
1562 | } | ||
1563 | 1556 | ||
1557 | #ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK | ||
1564 | static inline void sched_clock_tick(void) | 1558 | static inline void sched_clock_tick(void) |
1565 | { | 1559 | { |
1566 | } | 1560 | } |
@@ -1572,28 +1566,11 @@ static inline void sched_clock_idle_sleep_event(void) | |||
1572 | static inline void sched_clock_idle_wakeup_event(u64 delta_ns) | 1566 | static inline void sched_clock_idle_wakeup_event(u64 delta_ns) |
1573 | { | 1567 | { |
1574 | } | 1568 | } |
1575 | 1569 | #else | |
1576 | #ifdef CONFIG_NO_HZ | ||
1577 | static inline void sched_clock_tick_stop(int cpu) | ||
1578 | { | ||
1579 | } | ||
1580 | |||
1581 | static inline void sched_clock_tick_start(int cpu) | ||
1582 | { | ||
1583 | } | ||
1584 | #endif | ||
1585 | |||
1586 | #else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ | ||
1587 | extern void sched_clock_init(void); | ||
1588 | extern u64 sched_clock_cpu(int cpu); | ||
1589 | extern void sched_clock_tick(void); | 1570 | extern void sched_clock_tick(void); |
1590 | extern void sched_clock_idle_sleep_event(void); | 1571 | extern void sched_clock_idle_sleep_event(void); |
1591 | extern void sched_clock_idle_wakeup_event(u64 delta_ns); | 1572 | extern void sched_clock_idle_wakeup_event(u64 delta_ns); |
1592 | #ifdef CONFIG_NO_HZ | ||
1593 | extern void sched_clock_tick_stop(int cpu); | ||
1594 | extern void sched_clock_tick_start(int cpu); | ||
1595 | #endif | 1573 | #endif |
1596 | #endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ | ||
1597 | 1574 | ||
1598 | /* | 1575 | /* |
1599 | * For kernel-internal use: high-speed (but slightly incorrect) per-cpu | 1576 | * For kernel-internal use: high-speed (but slightly incorrect) per-cpu |
diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz index 382dd5a8b2d7..94fabd534b03 100644 --- a/kernel/Kconfig.hz +++ b/kernel/Kconfig.hz | |||
@@ -55,4 +55,4 @@ config HZ | |||
55 | default 1000 if HZ_1000 | 55 | default 1000 if HZ_1000 |
56 | 56 | ||
57 | config SCHED_HRTICK | 57 | config SCHED_HRTICK |
58 | def_bool HIGH_RES_TIMERS && USE_GENERIC_SMP_HELPERS | 58 | def_bool HIGH_RES_TIMERS && (!SMP || USE_GENERIC_SMP_HELPERS) |
diff --git a/kernel/cpu.c b/kernel/cpu.c index e202a68d1cc1..c977c339f559 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c | |||
@@ -349,6 +349,8 @@ static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen) | |||
349 | goto out_notify; | 349 | goto out_notify; |
350 | BUG_ON(!cpu_online(cpu)); | 350 | BUG_ON(!cpu_online(cpu)); |
351 | 351 | ||
352 | cpu_set(cpu, cpu_active_map); | ||
353 | |||
352 | /* Now call notifier in preparation. */ | 354 | /* Now call notifier in preparation. */ |
353 | raw_notifier_call_chain(&cpu_chain, CPU_ONLINE | mod, hcpu); | 355 | raw_notifier_call_chain(&cpu_chain, CPU_ONLINE | mod, hcpu); |
354 | 356 | ||
@@ -383,9 +385,6 @@ int __cpuinit cpu_up(unsigned int cpu) | |||
383 | 385 | ||
384 | err = _cpu_up(cpu, 0); | 386 | err = _cpu_up(cpu, 0); |
385 | 387 | ||
386 | if (cpu_online(cpu)) | ||
387 | cpu_set(cpu, cpu_active_map); | ||
388 | |||
389 | out: | 388 | out: |
390 | cpu_maps_update_done(); | 389 | cpu_maps_update_done(); |
391 | return err; | 390 | return err; |
diff --git a/kernel/sched.c b/kernel/sched.c index ace566bdfc68..d601fb0406ca 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -833,7 +833,7 @@ static inline u64 global_rt_period(void) | |||
833 | 833 | ||
834 | static inline u64 global_rt_runtime(void) | 834 | static inline u64 global_rt_runtime(void) |
835 | { | 835 | { |
836 | if (sysctl_sched_rt_period < 0) | 836 | if (sysctl_sched_rt_runtime < 0) |
837 | return RUNTIME_INF; | 837 | return RUNTIME_INF; |
838 | 838 | ||
839 | return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC; | 839 | return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC; |
diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c index 22ed55d1167f..204991a0bfa7 100644 --- a/kernel/sched_clock.c +++ b/kernel/sched_clock.c | |||
@@ -32,13 +32,19 @@ | |||
32 | #include <linux/ktime.h> | 32 | #include <linux/ktime.h> |
33 | #include <linux/module.h> | 33 | #include <linux/module.h> |
34 | 34 | ||
35 | /* | ||
36 | * Scheduler clock - returns current time in nanosec units. | ||
37 | * This is default implementation. | ||
38 | * Architectures and sub-architectures can override this. | ||
39 | */ | ||
40 | unsigned long long __attribute__((weak)) sched_clock(void) | ||
41 | { | ||
42 | return (unsigned long long)jiffies * (NSEC_PER_SEC / HZ); | ||
43 | } | ||
35 | 44 | ||
36 | #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK | 45 | static __read_mostly int sched_clock_running; |
37 | 46 | ||
38 | #define MULTI_SHIFT 15 | 47 | #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK |
39 | /* Max is double, Min is 1/2 */ | ||
40 | #define MAX_MULTI (2LL << MULTI_SHIFT) | ||
41 | #define MIN_MULTI (1LL << (MULTI_SHIFT-1)) | ||
42 | 48 | ||
43 | struct sched_clock_data { | 49 | struct sched_clock_data { |
44 | /* | 50 | /* |
@@ -49,14 +55,9 @@ struct sched_clock_data { | |||
49 | raw_spinlock_t lock; | 55 | raw_spinlock_t lock; |
50 | 56 | ||
51 | unsigned long tick_jiffies; | 57 | unsigned long tick_jiffies; |
52 | u64 prev_raw; | ||
53 | u64 tick_raw; | 58 | u64 tick_raw; |
54 | u64 tick_gtod; | 59 | u64 tick_gtod; |
55 | u64 clock; | 60 | u64 clock; |
56 | s64 multi; | ||
57 | #ifdef CONFIG_NO_HZ | ||
58 | int check_max; | ||
59 | #endif | ||
60 | }; | 61 | }; |
61 | 62 | ||
62 | static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); | 63 | static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); |
@@ -71,8 +72,6 @@ static inline struct sched_clock_data *cpu_sdc(int cpu) | |||
71 | return &per_cpu(sched_clock_data, cpu); | 72 | return &per_cpu(sched_clock_data, cpu); |
72 | } | 73 | } |
73 | 74 | ||
74 | static __read_mostly int sched_clock_running; | ||
75 | |||
76 | void sched_clock_init(void) | 75 | void sched_clock_init(void) |
77 | { | 76 | { |
78 | u64 ktime_now = ktime_to_ns(ktime_get()); | 77 | u64 ktime_now = ktime_to_ns(ktime_get()); |
@@ -84,90 +83,39 @@ void sched_clock_init(void) | |||
84 | 83 | ||
85 | scd->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; | 84 | scd->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; |
86 | scd->tick_jiffies = now_jiffies; | 85 | scd->tick_jiffies = now_jiffies; |
87 | scd->prev_raw = 0; | ||
88 | scd->tick_raw = 0; | 86 | scd->tick_raw = 0; |
89 | scd->tick_gtod = ktime_now; | 87 | scd->tick_gtod = ktime_now; |
90 | scd->clock = ktime_now; | 88 | scd->clock = ktime_now; |
91 | scd->multi = 1 << MULTI_SHIFT; | ||
92 | #ifdef CONFIG_NO_HZ | ||
93 | scd->check_max = 1; | ||
94 | #endif | ||
95 | } | 89 | } |
96 | 90 | ||
97 | sched_clock_running = 1; | 91 | sched_clock_running = 1; |
98 | } | 92 | } |
99 | 93 | ||
100 | #ifdef CONFIG_NO_HZ | ||
101 | /* | ||
102 | * The dynamic ticks makes the delta jiffies inaccurate. This | ||
103 | * prevents us from checking the maximum time update. | ||
104 | * Disable the maximum check during stopped ticks. | ||
105 | */ | ||
106 | void sched_clock_tick_stop(int cpu) | ||
107 | { | ||
108 | struct sched_clock_data *scd = cpu_sdc(cpu); | ||
109 | |||
110 | scd->check_max = 0; | ||
111 | } | ||
112 | |||
113 | void sched_clock_tick_start(int cpu) | ||
114 | { | ||
115 | struct sched_clock_data *scd = cpu_sdc(cpu); | ||
116 | |||
117 | scd->check_max = 1; | ||
118 | } | ||
119 | |||
120 | static int check_max(struct sched_clock_data *scd) | ||
121 | { | ||
122 | return scd->check_max; | ||
123 | } | ||
124 | #else | ||
125 | static int check_max(struct sched_clock_data *scd) | ||
126 | { | ||
127 | return 1; | ||
128 | } | ||
129 | #endif /* CONFIG_NO_HZ */ | ||
130 | |||
131 | /* | 94 | /* |
132 | * update the percpu scd from the raw @now value | 95 | * update the percpu scd from the raw @now value |
133 | * | 96 | * |
134 | * - filter out backward motion | 97 | * - filter out backward motion |
135 | * - use jiffies to generate a min,max window to clip the raw values | 98 | * - use jiffies to generate a min,max window to clip the raw values |
136 | */ | 99 | */ |
137 | static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *time) | 100 | static u64 __update_sched_clock(struct sched_clock_data *scd, u64 now) |
138 | { | 101 | { |
139 | unsigned long now_jiffies = jiffies; | 102 | unsigned long now_jiffies = jiffies; |
140 | long delta_jiffies = now_jiffies - scd->tick_jiffies; | 103 | long delta_jiffies = now_jiffies - scd->tick_jiffies; |
141 | u64 clock = scd->clock; | 104 | u64 clock = scd->clock; |
142 | u64 min_clock, max_clock; | 105 | u64 min_clock, max_clock; |
143 | s64 delta = now - scd->prev_raw; | 106 | s64 delta = now - scd->tick_raw; |
144 | 107 | ||
145 | WARN_ON_ONCE(!irqs_disabled()); | 108 | WARN_ON_ONCE(!irqs_disabled()); |
146 | 109 | min_clock = scd->tick_gtod + delta_jiffies * TICK_NSEC; | |
147 | /* | ||
148 | * At schedule tick the clock can be just under the gtod. We don't | ||
149 | * want to push it too prematurely. | ||
150 | */ | ||
151 | min_clock = scd->tick_gtod + (delta_jiffies * TICK_NSEC); | ||
152 | if (min_clock > TICK_NSEC) | ||
153 | min_clock -= TICK_NSEC / 2; | ||
154 | 110 | ||
155 | if (unlikely(delta < 0)) { | 111 | if (unlikely(delta < 0)) { |
156 | clock++; | 112 | clock++; |
157 | goto out; | 113 | goto out; |
158 | } | 114 | } |
159 | 115 | ||
160 | /* | 116 | max_clock = min_clock + TICK_NSEC; |
161 | * The clock must stay within a jiffie of the gtod. | ||
162 | * But since we may be at the start of a jiffy or the end of one | ||
163 | * we add another jiffy buffer. | ||
164 | */ | ||
165 | max_clock = scd->tick_gtod + (2 + delta_jiffies) * TICK_NSEC; | ||
166 | |||
167 | delta *= scd->multi; | ||
168 | delta >>= MULTI_SHIFT; | ||
169 | 117 | ||
170 | if (unlikely(clock + delta > max_clock) && check_max(scd)) { | 118 | if (unlikely(clock + delta > max_clock)) { |
171 | if (clock < max_clock) | 119 | if (clock < max_clock) |
172 | clock = max_clock; | 120 | clock = max_clock; |
173 | else | 121 | else |
@@ -180,12 +128,10 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim | |||
180 | if (unlikely(clock < min_clock)) | 128 | if (unlikely(clock < min_clock)) |
181 | clock = min_clock; | 129 | clock = min_clock; |
182 | 130 | ||
183 | if (time) | 131 | scd->tick_jiffies = now_jiffies; |
184 | *time = clock; | 132 | scd->clock = clock; |
185 | else { | 133 | |
186 | scd->prev_raw = now; | 134 | return clock; |
187 | scd->clock = clock; | ||
188 | } | ||
189 | } | 135 | } |
190 | 136 | ||
191 | static void lock_double_clock(struct sched_clock_data *data1, | 137 | static void lock_double_clock(struct sched_clock_data *data1, |
@@ -203,7 +149,7 @@ static void lock_double_clock(struct sched_clock_data *data1, | |||
203 | u64 sched_clock_cpu(int cpu) | 149 | u64 sched_clock_cpu(int cpu) |
204 | { | 150 | { |
205 | struct sched_clock_data *scd = cpu_sdc(cpu); | 151 | struct sched_clock_data *scd = cpu_sdc(cpu); |
206 | u64 now, clock; | 152 | u64 now, clock, this_clock, remote_clock; |
207 | 153 | ||
208 | if (unlikely(!sched_clock_running)) | 154 | if (unlikely(!sched_clock_running)) |
209 | return 0ull; | 155 | return 0ull; |
@@ -212,43 +158,44 @@ u64 sched_clock_cpu(int cpu) | |||
212 | now = sched_clock(); | 158 | now = sched_clock(); |
213 | 159 | ||
214 | if (cpu != raw_smp_processor_id()) { | 160 | if (cpu != raw_smp_processor_id()) { |
215 | /* | ||
216 | * in order to update a remote cpu's clock based on our | ||
217 | * unstable raw time rebase it against: | ||
218 | * tick_raw (offset between raw counters) | ||
219 | * tick_gotd (tick offset between cpus) | ||
220 | */ | ||
221 | struct sched_clock_data *my_scd = this_scd(); | 161 | struct sched_clock_data *my_scd = this_scd(); |
222 | 162 | ||
223 | lock_double_clock(scd, my_scd); | 163 | lock_double_clock(scd, my_scd); |
224 | 164 | ||
225 | now -= my_scd->tick_raw; | 165 | this_clock = __update_sched_clock(my_scd, now); |
226 | now += scd->tick_raw; | 166 | remote_clock = scd->clock; |
227 | 167 | ||
228 | now += my_scd->tick_gtod; | 168 | /* |
229 | now -= scd->tick_gtod; | 169 | * Use the opportunity that we have both locks |
170 | * taken to couple the two clocks: we take the | ||
171 | * larger time as the latest time for both | ||
172 | * runqueues. (this creates monotonic movement) | ||
173 | */ | ||
174 | if (likely(remote_clock < this_clock)) { | ||
175 | clock = this_clock; | ||
176 | scd->clock = clock; | ||
177 | } else { | ||
178 | /* | ||
179 | * Should be rare, but possible: | ||
180 | */ | ||
181 | clock = remote_clock; | ||
182 | my_scd->clock = remote_clock; | ||
183 | } | ||
230 | 184 | ||
231 | __raw_spin_unlock(&my_scd->lock); | 185 | __raw_spin_unlock(&my_scd->lock); |
232 | |||
233 | __update_sched_clock(scd, now, &clock); | ||
234 | |||
235 | __raw_spin_unlock(&scd->lock); | ||
236 | |||
237 | } else { | 186 | } else { |
238 | __raw_spin_lock(&scd->lock); | 187 | __raw_spin_lock(&scd->lock); |
239 | __update_sched_clock(scd, now, NULL); | 188 | clock = __update_sched_clock(scd, now); |
240 | clock = scd->clock; | ||
241 | __raw_spin_unlock(&scd->lock); | ||
242 | } | 189 | } |
243 | 190 | ||
191 | __raw_spin_unlock(&scd->lock); | ||
192 | |||
244 | return clock; | 193 | return clock; |
245 | } | 194 | } |
246 | 195 | ||
247 | void sched_clock_tick(void) | 196 | void sched_clock_tick(void) |
248 | { | 197 | { |
249 | struct sched_clock_data *scd = this_scd(); | 198 | struct sched_clock_data *scd = this_scd(); |
250 | unsigned long now_jiffies = jiffies; | ||
251 | s64 mult, delta_gtod, delta_raw; | ||
252 | u64 now, now_gtod; | 199 | u64 now, now_gtod; |
253 | 200 | ||
254 | if (unlikely(!sched_clock_running)) | 201 | if (unlikely(!sched_clock_running)) |
@@ -260,29 +207,14 @@ void sched_clock_tick(void) | |||
260 | now = sched_clock(); | 207 | now = sched_clock(); |
261 | 208 | ||
262 | __raw_spin_lock(&scd->lock); | 209 | __raw_spin_lock(&scd->lock); |
263 | __update_sched_clock(scd, now, NULL); | 210 | __update_sched_clock(scd, now); |
264 | /* | 211 | /* |
265 | * update tick_gtod after __update_sched_clock() because that will | 212 | * update tick_gtod after __update_sched_clock() because that will |
266 | * already observe 1 new jiffy; adding a new tick_gtod to that would | 213 | * already observe 1 new jiffy; adding a new tick_gtod to that would |
267 | * increase the clock 2 jiffies. | 214 | * increase the clock 2 jiffies. |
268 | */ | 215 | */ |
269 | delta_gtod = now_gtod - scd->tick_gtod; | ||
270 | delta_raw = now - scd->tick_raw; | ||
271 | |||
272 | if ((long)delta_raw > 0) { | ||
273 | mult = delta_gtod << MULTI_SHIFT; | ||
274 | do_div(mult, delta_raw); | ||
275 | scd->multi = mult; | ||
276 | if (scd->multi > MAX_MULTI) | ||
277 | scd->multi = MAX_MULTI; | ||
278 | else if (scd->multi < MIN_MULTI) | ||
279 | scd->multi = MIN_MULTI; | ||
280 | } else | ||
281 | scd->multi = 1 << MULTI_SHIFT; | ||
282 | |||
283 | scd->tick_raw = now; | 216 | scd->tick_raw = now; |
284 | scd->tick_gtod = now_gtod; | 217 | scd->tick_gtod = now_gtod; |
285 | scd->tick_jiffies = now_jiffies; | ||
286 | __raw_spin_unlock(&scd->lock); | 218 | __raw_spin_unlock(&scd->lock); |
287 | } | 219 | } |
288 | 220 | ||
@@ -301,7 +233,6 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event); | |||
301 | void sched_clock_idle_wakeup_event(u64 delta_ns) | 233 | void sched_clock_idle_wakeup_event(u64 delta_ns) |
302 | { | 234 | { |
303 | struct sched_clock_data *scd = this_scd(); | 235 | struct sched_clock_data *scd = this_scd(); |
304 | u64 now = sched_clock(); | ||
305 | 236 | ||
306 | /* | 237 | /* |
307 | * Override the previous timestamp and ignore all | 238 | * Override the previous timestamp and ignore all |
@@ -310,27 +241,30 @@ void sched_clock_idle_wakeup_event(u64 delta_ns) | |||
310 | * rq clock: | 241 | * rq clock: |
311 | */ | 242 | */ |
312 | __raw_spin_lock(&scd->lock); | 243 | __raw_spin_lock(&scd->lock); |
313 | scd->prev_raw = now; | ||
314 | scd->clock += delta_ns; | 244 | scd->clock += delta_ns; |
315 | scd->multi = 1 << MULTI_SHIFT; | ||
316 | __raw_spin_unlock(&scd->lock); | 245 | __raw_spin_unlock(&scd->lock); |
317 | 246 | ||
318 | touch_softlockup_watchdog(); | 247 | touch_softlockup_watchdog(); |
319 | } | 248 | } |
320 | EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); | 249 | EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); |
321 | 250 | ||
322 | #endif | 251 | #else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ |
323 | 252 | ||
324 | /* | 253 | void sched_clock_init(void) |
325 | * Scheduler clock - returns current time in nanosec units. | ||
326 | * This is default implementation. | ||
327 | * Architectures and sub-architectures can override this. | ||
328 | */ | ||
329 | unsigned long long __attribute__((weak)) sched_clock(void) | ||
330 | { | 254 | { |
331 | return (unsigned long long)jiffies * (NSEC_PER_SEC / HZ); | 255 | sched_clock_running = 1; |
332 | } | 256 | } |
333 | 257 | ||
258 | u64 sched_clock_cpu(int cpu) | ||
259 | { | ||
260 | if (unlikely(!sched_clock_running)) | ||
261 | return 0; | ||
262 | |||
263 | return sched_clock(); | ||
264 | } | ||
265 | |||
266 | #endif | ||
267 | |||
334 | unsigned long long cpu_clock(int cpu) | 268 | unsigned long long cpu_clock(int cpu) |
335 | { | 269 | { |
336 | unsigned long long clock; | 270 | unsigned long long clock; |
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index cf2cd6ce4cb2..fb8994c6d4bb 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c | |||
@@ -899,7 +899,7 @@ static void hrtick_start_fair(struct rq *rq, struct task_struct *p) | |||
899 | * doesn't make sense. Rely on vruntime for fairness. | 899 | * doesn't make sense. Rely on vruntime for fairness. |
900 | */ | 900 | */ |
901 | if (rq->curr != p) | 901 | if (rq->curr != p) |
902 | delta = max(10000LL, delta); | 902 | delta = max_t(s64, 10000LL, delta); |
903 | 903 | ||
904 | hrtick_start(rq, delta); | 904 | hrtick_start(rq, delta); |
905 | } | 905 | } |
@@ -1442,18 +1442,23 @@ __load_balance_iterator(struct cfs_rq *cfs_rq, struct list_head *next) | |||
1442 | struct task_struct *p = NULL; | 1442 | struct task_struct *p = NULL; |
1443 | struct sched_entity *se; | 1443 | struct sched_entity *se; |
1444 | 1444 | ||
1445 | while (next != &cfs_rq->tasks) { | 1445 | if (next == &cfs_rq->tasks) |
1446 | return NULL; | ||
1447 | |||
1448 | /* Skip over entities that are not tasks */ | ||
1449 | do { | ||
1446 | se = list_entry(next, struct sched_entity, group_node); | 1450 | se = list_entry(next, struct sched_entity, group_node); |
1447 | next = next->next; | 1451 | next = next->next; |
1452 | } while (next != &cfs_rq->tasks && !entity_is_task(se)); | ||
1448 | 1453 | ||
1449 | /* Skip over entities that are not tasks */ | 1454 | if (next == &cfs_rq->tasks) |
1450 | if (entity_is_task(se)) { | 1455 | return NULL; |
1451 | p = task_of(se); | ||
1452 | break; | ||
1453 | } | ||
1454 | } | ||
1455 | 1456 | ||
1456 | cfs_rq->balance_iterator = next; | 1457 | cfs_rq->balance_iterator = next; |
1458 | |||
1459 | if (entity_is_task(se)) | ||
1460 | p = task_of(se); | ||
1461 | |||
1457 | return p; | 1462 | return p; |
1458 | } | 1463 | } |
1459 | 1464 | ||
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 825b4c00fe44..f5da526424a9 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
@@ -289,7 +289,6 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
289 | ts->tick_stopped = 1; | 289 | ts->tick_stopped = 1; |
290 | ts->idle_jiffies = last_jiffies; | 290 | ts->idle_jiffies = last_jiffies; |
291 | rcu_enter_nohz(); | 291 | rcu_enter_nohz(); |
292 | sched_clock_tick_stop(cpu); | ||
293 | } | 292 | } |
294 | 293 | ||
295 | /* | 294 | /* |
@@ -392,7 +391,6 @@ void tick_nohz_restart_sched_tick(void) | |||
392 | select_nohz_load_balancer(0); | 391 | select_nohz_load_balancer(0); |
393 | now = ktime_get(); | 392 | now = ktime_get(); |
394 | tick_do_update_jiffies64(now); | 393 | tick_do_update_jiffies64(now); |
395 | sched_clock_tick_start(cpu); | ||
396 | cpu_clear(cpu, nohz_cpu_mask); | 394 | cpu_clear(cpu, nohz_cpu_mask); |
397 | 395 | ||
398 | /* | 396 | /* |