aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/arm/kernel/process.c2
-rw-r--r--arch/avr32/kernel/process.c2
-rw-r--r--arch/blackfin/kernel/process.c2
-rw-r--r--arch/mips/kernel/process.c2
-rw-r--r--arch/powerpc/kernel/idle.c2
-rw-r--r--arch/powerpc/platforms/iseries/setup.c4
-rw-r--r--arch/s390/kernel/process.c2
-rw-r--r--arch/sh/kernel/process_32.c2
-rw-r--r--arch/sparc64/kernel/process.c2
-rw-r--r--arch/um/kernel/process.c2
-rw-r--r--arch/x86/kernel/process_32.c2
-rw-r--r--arch/x86/kernel/process_64.c2
-rw-r--r--include/linux/tick.h5
-rw-r--r--kernel/softirq.c2
-rw-r--r--kernel/time/tick-sched.c12
15 files changed, 27 insertions, 18 deletions
diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index 199b3680118b..89bfded70a1f 100644
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -162,7 +162,7 @@ void cpu_idle(void)
162 if (!idle) 162 if (!idle)
163 idle = default_idle; 163 idle = default_idle;
164 leds_event(led_idle_start); 164 leds_event(led_idle_start);
165 tick_nohz_stop_sched_tick(); 165 tick_nohz_stop_sched_tick(1);
166 while (!need_resched()) 166 while (!need_resched())
167 idle(); 167 idle();
168 leds_event(led_idle_end); 168 leds_event(led_idle_end);
diff --git a/arch/avr32/kernel/process.c b/arch/avr32/kernel/process.c
index 6cf9df176274..ff820a9e743a 100644
--- a/arch/avr32/kernel/process.c
+++ b/arch/avr32/kernel/process.c
@@ -31,7 +31,7 @@ void cpu_idle(void)
31{ 31{
32 /* endless idle loop with no priority at all */ 32 /* endless idle loop with no priority at all */
33 while (1) { 33 while (1) {
34 tick_nohz_stop_sched_tick(); 34 tick_nohz_stop_sched_tick(1);
35 while (!need_resched()) 35 while (!need_resched())
36 cpu_idle_sleep(); 36 cpu_idle_sleep();
37 tick_nohz_restart_sched_tick(); 37 tick_nohz_restart_sched_tick();
diff --git a/arch/blackfin/kernel/process.c b/arch/blackfin/kernel/process.c
index 53c2cd255441..77800dd83e57 100644
--- a/arch/blackfin/kernel/process.c
+++ b/arch/blackfin/kernel/process.c
@@ -105,7 +105,7 @@ void cpu_idle(void)
105#endif 105#endif
106 if (!idle) 106 if (!idle)
107 idle = default_idle; 107 idle = default_idle;
108 tick_nohz_stop_sched_tick(); 108 tick_nohz_stop_sched_tick(1);
109 while (!need_resched()) 109 while (!need_resched())
110 idle(); 110 idle();
111 tick_nohz_restart_sched_tick(); 111 tick_nohz_restart_sched_tick();
diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c
index c06f5b5d764c..b16facd9ea8e 100644
--- a/arch/mips/kernel/process.c
+++ b/arch/mips/kernel/process.c
@@ -53,7 +53,7 @@ void __noreturn cpu_idle(void)
53{ 53{
54 /* endless idle loop with no priority at all */ 54 /* endless idle loop with no priority at all */
55 while (1) { 55 while (1) {
56 tick_nohz_stop_sched_tick(); 56 tick_nohz_stop_sched_tick(1);
57 while (!need_resched()) { 57 while (!need_resched()) {
58#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG 58#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
59 extern void smtc_idle_loop_hook(void); 59 extern void smtc_idle_loop_hook(void);
diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c
index c3cf0e8f3ac1..d308a9f70f1b 100644
--- a/arch/powerpc/kernel/idle.c
+++ b/arch/powerpc/kernel/idle.c
@@ -60,7 +60,7 @@ void cpu_idle(void)
60 60
61 set_thread_flag(TIF_POLLING_NRFLAG); 61 set_thread_flag(TIF_POLLING_NRFLAG);
62 while (1) { 62 while (1) {
63 tick_nohz_stop_sched_tick(); 63 tick_nohz_stop_sched_tick(1);
64 while (!need_resched() && !cpu_should_die()) { 64 while (!need_resched() && !cpu_should_die()) {
65 ppc64_runlatch_off(); 65 ppc64_runlatch_off();
66 66
diff --git a/arch/powerpc/platforms/iseries/setup.c b/arch/powerpc/platforms/iseries/setup.c
index b72120751bbe..70b688c1aefb 100644
--- a/arch/powerpc/platforms/iseries/setup.c
+++ b/arch/powerpc/platforms/iseries/setup.c
@@ -561,7 +561,7 @@ static void yield_shared_processor(void)
561static void iseries_shared_idle(void) 561static void iseries_shared_idle(void)
562{ 562{
563 while (1) { 563 while (1) {
564 tick_nohz_stop_sched_tick(); 564 tick_nohz_stop_sched_tick(1);
565 while (!need_resched() && !hvlpevent_is_pending()) { 565 while (!need_resched() && !hvlpevent_is_pending()) {
566 local_irq_disable(); 566 local_irq_disable();
567 ppc64_runlatch_off(); 567 ppc64_runlatch_off();
@@ -591,7 +591,7 @@ static void iseries_dedicated_idle(void)
591 set_thread_flag(TIF_POLLING_NRFLAG); 591 set_thread_flag(TIF_POLLING_NRFLAG);
592 592
593 while (1) { 593 while (1) {
594 tick_nohz_stop_sched_tick(); 594 tick_nohz_stop_sched_tick(1);
595 if (!need_resched()) { 595 if (!need_resched()) {
596 while (!need_resched()) { 596 while (!need_resched()) {
597 ppc64_runlatch_off(); 597 ppc64_runlatch_off();
diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c
index 85defd01d293..9839767d0842 100644
--- a/arch/s390/kernel/process.c
+++ b/arch/s390/kernel/process.c
@@ -142,7 +142,7 @@ static void default_idle(void)
142void cpu_idle(void) 142void cpu_idle(void)
143{ 143{
144 for (;;) { 144 for (;;) {
145 tick_nohz_stop_sched_tick(); 145 tick_nohz_stop_sched_tick(1);
146 while (!need_resched()) 146 while (!need_resched())
147 default_idle(); 147 default_idle();
148 tick_nohz_restart_sched_tick(); 148 tick_nohz_restart_sched_tick();
diff --git a/arch/sh/kernel/process_32.c b/arch/sh/kernel/process_32.c
index b98e37a1f54c..921892c351da 100644
--- a/arch/sh/kernel/process_32.c
+++ b/arch/sh/kernel/process_32.c
@@ -86,7 +86,7 @@ void cpu_idle(void)
86 if (!idle) 86 if (!idle)
87 idle = default_idle; 87 idle = default_idle;
88 88
89 tick_nohz_stop_sched_tick(); 89 tick_nohz_stop_sched_tick(1);
90 while (!need_resched()) 90 while (!need_resched())
91 idle(); 91 idle();
92 tick_nohz_restart_sched_tick(); 92 tick_nohz_restart_sched_tick();
diff --git a/arch/sparc64/kernel/process.c b/arch/sparc64/kernel/process.c
index 31ea752d307b..8a9cd3e165b9 100644
--- a/arch/sparc64/kernel/process.c
+++ b/arch/sparc64/kernel/process.c
@@ -96,7 +96,7 @@ void cpu_idle(void)
96 set_thread_flag(TIF_POLLING_NRFLAG); 96 set_thread_flag(TIF_POLLING_NRFLAG);
97 97
98 while(1) { 98 while(1) {
99 tick_nohz_stop_sched_tick(); 99 tick_nohz_stop_sched_tick(1);
100 100
101 while (!need_resched() && !cpu_is_offline(cpu)) 101 while (!need_resched() && !cpu_is_offline(cpu))
102 sparc64_yield(cpu); 102 sparc64_yield(cpu);
diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c
index 83603cfbde81..a1c6d07cac3e 100644
--- a/arch/um/kernel/process.c
+++ b/arch/um/kernel/process.c
@@ -243,7 +243,7 @@ void default_idle(void)
243 if (need_resched()) 243 if (need_resched())
244 schedule(); 244 schedule();
245 245
246 tick_nohz_stop_sched_tick(); 246 tick_nohz_stop_sched_tick(1);
247 nsecs = disable_timer(); 247 nsecs = disable_timer();
248 idle_sleep(nsecs); 248 idle_sleep(nsecs);
249 tick_nohz_restart_sched_tick(); 249 tick_nohz_restart_sched_tick();
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 0c3927accb00..53bc653ed5ca 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -128,7 +128,7 @@ void cpu_idle(void)
128 128
129 /* endless idle loop with no priority at all */ 129 /* endless idle loop with no priority at all */
130 while (1) { 130 while (1) {
131 tick_nohz_stop_sched_tick(); 131 tick_nohz_stop_sched_tick(1);
132 while (!need_resched()) { 132 while (!need_resched()) {
133 133
134 check_pgt_cache(); 134 check_pgt_cache();
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index e8a8e1b99817..3fb62a7d9a16 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -120,7 +120,7 @@ void cpu_idle(void)
120 current_thread_info()->status |= TS_POLLING; 120 current_thread_info()->status |= TS_POLLING;
121 /* endless idle loop with no priority at all */ 121 /* endless idle loop with no priority at all */
122 while (1) { 122 while (1) {
123 tick_nohz_stop_sched_tick(); 123 tick_nohz_stop_sched_tick(1);
124 while (!need_resched()) { 124 while (!need_resched()) {
125 125
126 rmb(); 126 rmb();
diff --git a/include/linux/tick.h b/include/linux/tick.h
index a881c652f7e9..d3c02695dc5d 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -49,6 +49,7 @@ struct tick_sched {
49 unsigned long check_clocks; 49 unsigned long check_clocks;
50 enum tick_nohz_mode nohz_mode; 50 enum tick_nohz_mode nohz_mode;
51 ktime_t idle_tick; 51 ktime_t idle_tick;
52 int inidle;
52 int tick_stopped; 53 int tick_stopped;
53 unsigned long idle_jiffies; 54 unsigned long idle_jiffies;
54 unsigned long idle_calls; 55 unsigned long idle_calls;
@@ -105,14 +106,14 @@ static inline int tick_check_oneshot_change(int allow_nohz) { return 0; }
105#endif /* !CONFIG_GENERIC_CLOCKEVENTS */ 106#endif /* !CONFIG_GENERIC_CLOCKEVENTS */
106 107
107# ifdef CONFIG_NO_HZ 108# ifdef CONFIG_NO_HZ
108extern void tick_nohz_stop_sched_tick(void); 109extern void tick_nohz_stop_sched_tick(int inidle);
109extern void tick_nohz_restart_sched_tick(void); 110extern void tick_nohz_restart_sched_tick(void);
110extern void tick_nohz_update_jiffies(void); 111extern void tick_nohz_update_jiffies(void);
111extern ktime_t tick_nohz_get_sleep_length(void); 112extern ktime_t tick_nohz_get_sleep_length(void);
112extern void tick_nohz_stop_idle(int cpu); 113extern void tick_nohz_stop_idle(int cpu);
113extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); 114extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time);
114# else 115# else
115static inline void tick_nohz_stop_sched_tick(void) { } 116static inline void tick_nohz_stop_sched_tick(int inidle) { }
116static inline void tick_nohz_restart_sched_tick(void) { } 117static inline void tick_nohz_restart_sched_tick(void) { }
117static inline void tick_nohz_update_jiffies(void) { } 118static inline void tick_nohz_update_jiffies(void) { }
118static inline ktime_t tick_nohz_get_sleep_length(void) 119static inline ktime_t tick_nohz_get_sleep_length(void)
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 81e2fe0f983a..f6b03d56c2bf 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -286,7 +286,7 @@ void irq_exit(void)
286#ifdef CONFIG_NO_HZ 286#ifdef CONFIG_NO_HZ
287 /* Make sure that timer wheel updates are propagated */ 287 /* Make sure that timer wheel updates are propagated */
288 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched()) 288 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
289 tick_nohz_stop_sched_tick(); 289 tick_nohz_stop_sched_tick(0);
290 rcu_irq_exit(); 290 rcu_irq_exit();
291#endif 291#endif
292 preempt_enable_no_resched(); 292 preempt_enable_no_resched();
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 942fc7c85283..825b4c00fe44 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -195,7 +195,7 @@ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
195 * Called either from the idle loop or from irq_exit() when an idle period was 195 * Called either from the idle loop or from irq_exit() when an idle period was
196 * just interrupted by an interrupt which did not cause a reschedule. 196 * just interrupted by an interrupt which did not cause a reschedule.
197 */ 197 */
198void tick_nohz_stop_sched_tick(void) 198void tick_nohz_stop_sched_tick(int inidle)
199{ 199{
200 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags; 200 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags;
201 struct tick_sched *ts; 201 struct tick_sched *ts;
@@ -224,6 +224,11 @@ void tick_nohz_stop_sched_tick(void)
224 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) 224 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
225 goto end; 225 goto end;
226 226
227 if (!inidle && !ts->inidle)
228 goto end;
229
230 ts->inidle = 1;
231
227 if (need_resched()) 232 if (need_resched())
228 goto end; 233 goto end;
229 234
@@ -373,11 +378,14 @@ void tick_nohz_restart_sched_tick(void)
373 local_irq_disable(); 378 local_irq_disable();
374 tick_nohz_stop_idle(cpu); 379 tick_nohz_stop_idle(cpu);
375 380
376 if (!ts->tick_stopped) { 381 if (!ts->inidle || !ts->tick_stopped) {
382 ts->inidle = 0;
377 local_irq_enable(); 383 local_irq_enable();
378 return; 384 return;
379 } 385 }
380 386
387 ts->inidle = 0;
388
381 rcu_exit_nohz(); 389 rcu_exit_nohz();
382 390
383 /* Update jiffies first */ 391 /* Update jiffies first */