diff options
Diffstat (limited to 'kernel/time/tick-sched.c')
| -rw-r--r-- | kernel/time/tick-sched.c | 26 |
1 files changed, 22 insertions, 4 deletions
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index b854a895591e..825b4c00fe44 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
| @@ -48,6 +48,13 @@ static void tick_do_update_jiffies64(ktime_t now) | |||
| 48 | unsigned long ticks = 0; | 48 | unsigned long ticks = 0; |
| 49 | ktime_t delta; | 49 | ktime_t delta; |
| 50 | 50 | ||
| 51 | /* | ||
| 52 | * Do a quick check without holding xtime_lock: | ||
| 53 | */ | ||
| 54 | delta = ktime_sub(now, last_jiffies_update); | ||
| 55 | if (delta.tv64 < tick_period.tv64) | ||
| 56 | return; | ||
| 57 | |||
| 51 | /* Reevalute with xtime_lock held */ | 58 | /* Reevalute with xtime_lock held */ |
| 52 | write_seqlock(&xtime_lock); | 59 | write_seqlock(&xtime_lock); |
| 53 | 60 | ||
| @@ -133,8 +140,6 @@ void tick_nohz_update_jiffies(void) | |||
| 133 | if (!ts->tick_stopped) | 140 | if (!ts->tick_stopped) |
| 134 | return; | 141 | return; |
| 135 | 142 | ||
| 136 | touch_softlockup_watchdog(); | ||
| 137 | |||
| 138 | cpu_clear(cpu, nohz_cpu_mask); | 143 | cpu_clear(cpu, nohz_cpu_mask); |
| 139 | now = ktime_get(); | 144 | now = ktime_get(); |
| 140 | ts->idle_waketime = now; | 145 | ts->idle_waketime = now; |
| @@ -142,6 +147,8 @@ void tick_nohz_update_jiffies(void) | |||
| 142 | local_irq_save(flags); | 147 | local_irq_save(flags); |
| 143 | tick_do_update_jiffies64(now); | 148 | tick_do_update_jiffies64(now); |
| 144 | local_irq_restore(flags); | 149 | local_irq_restore(flags); |
| 150 | |||
| 151 | touch_softlockup_watchdog(); | ||
| 145 | } | 152 | } |
| 146 | 153 | ||
| 147 | void tick_nohz_stop_idle(int cpu) | 154 | void tick_nohz_stop_idle(int cpu) |
| @@ -188,7 +195,7 @@ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) | |||
| 188 | * Called either from the idle loop or from irq_exit() when an idle period was | 195 | * Called either from the idle loop or from irq_exit() when an idle period was |
| 189 | * just interrupted by an interrupt which did not cause a reschedule. | 196 | * just interrupted by an interrupt which did not cause a reschedule. |
| 190 | */ | 197 | */ |
| 191 | void tick_nohz_stop_sched_tick(void) | 198 | void tick_nohz_stop_sched_tick(int inidle) |
| 192 | { | 199 | { |
| 193 | unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags; | 200 | unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags; |
| 194 | struct tick_sched *ts; | 201 | struct tick_sched *ts; |
| @@ -217,6 +224,11 @@ void tick_nohz_stop_sched_tick(void) | |||
| 217 | if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) | 224 | if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) |
| 218 | goto end; | 225 | goto end; |
| 219 | 226 | ||
| 227 | if (!inidle && !ts->inidle) | ||
| 228 | goto end; | ||
| 229 | |||
| 230 | ts->inidle = 1; | ||
| 231 | |||
| 220 | if (need_resched()) | 232 | if (need_resched()) |
| 221 | goto end; | 233 | goto end; |
| 222 | 234 | ||
| @@ -228,6 +240,7 @@ void tick_nohz_stop_sched_tick(void) | |||
| 228 | local_softirq_pending()); | 240 | local_softirq_pending()); |
| 229 | ratelimit++; | 241 | ratelimit++; |
| 230 | } | 242 | } |
| 243 | goto end; | ||
| 231 | } | 244 | } |
| 232 | 245 | ||
| 233 | ts->idle_calls++; | 246 | ts->idle_calls++; |
| @@ -276,6 +289,7 @@ void tick_nohz_stop_sched_tick(void) | |||
| 276 | ts->tick_stopped = 1; | 289 | ts->tick_stopped = 1; |
| 277 | ts->idle_jiffies = last_jiffies; | 290 | ts->idle_jiffies = last_jiffies; |
| 278 | rcu_enter_nohz(); | 291 | rcu_enter_nohz(); |
| 292 | sched_clock_tick_stop(cpu); | ||
| 279 | } | 293 | } |
| 280 | 294 | ||
| 281 | /* | 295 | /* |
| @@ -364,17 +378,21 @@ void tick_nohz_restart_sched_tick(void) | |||
| 364 | local_irq_disable(); | 378 | local_irq_disable(); |
| 365 | tick_nohz_stop_idle(cpu); | 379 | tick_nohz_stop_idle(cpu); |
| 366 | 380 | ||
| 367 | if (!ts->tick_stopped) { | 381 | if (!ts->inidle || !ts->tick_stopped) { |
| 382 | ts->inidle = 0; | ||
| 368 | local_irq_enable(); | 383 | local_irq_enable(); |
| 369 | return; | 384 | return; |
| 370 | } | 385 | } |
| 371 | 386 | ||
| 387 | ts->inidle = 0; | ||
| 388 | |||
| 372 | rcu_exit_nohz(); | 389 | rcu_exit_nohz(); |
| 373 | 390 | ||
| 374 | /* Update jiffies first */ | 391 | /* Update jiffies first */ |
| 375 | select_nohz_load_balancer(0); | 392 | select_nohz_load_balancer(0); |
| 376 | now = ktime_get(); | 393 | now = ktime_get(); |
| 377 | tick_do_update_jiffies64(now); | 394 | tick_do_update_jiffies64(now); |
| 395 | sched_clock_tick_start(cpu); | ||
| 378 | cpu_clear(cpu, nohz_cpu_mask); | 396 | cpu_clear(cpu, nohz_cpu_mask); |
| 379 | 397 | ||
| 380 | /* | 398 | /* |
