aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm/kernel/process.c4
-rw-r--r--arch/avr32/kernel/process.c4
-rw-r--r--arch/blackfin/kernel/process.c4
-rw-r--r--arch/microblaze/kernel/process.c4
-rw-r--r--arch/mips/kernel/process.c4
-rw-r--r--arch/openrisc/kernel/idle.c4
-rw-r--r--arch/powerpc/kernel/idle.c4
-rw-r--r--arch/powerpc/platforms/iseries/setup.c8
-rw-r--r--arch/s390/kernel/process.c4
-rw-r--r--arch/sh/kernel/idle.c4
-rw-r--r--arch/sparc/kernel/process_64.c4
-rw-r--r--arch/tile/kernel/process.c4
-rw-r--r--arch/um/kernel/process.c4
-rw-r--r--arch/unicore32/kernel/process.c4
-rw-r--r--arch/x86/kernel/process_32.c4
-rw-r--r--arch/x86/kernel/process_64.c4
-rw-r--r--include/linux/tick.h13
-rw-r--r--kernel/softirq.c2
-rw-r--r--kernel/time/tick-sched.c93
19 files changed, 99 insertions, 77 deletions
diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index 3d0c6fb74ae4..3f1f8daf703c 100644
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -183,7 +183,7 @@ void cpu_idle(void)
183 183
184 /* endless idle loop with no priority at all */ 184 /* endless idle loop with no priority at all */
185 while (1) { 185 while (1) {
186 tick_nohz_stop_sched_tick(1); 186 tick_nohz_idle_enter();
187 leds_event(led_idle_start); 187 leds_event(led_idle_start);
188 while (!need_resched()) { 188 while (!need_resched()) {
189#ifdef CONFIG_HOTPLUG_CPU 189#ifdef CONFIG_HOTPLUG_CPU
@@ -213,7 +213,7 @@ void cpu_idle(void)
213 } 213 }
214 } 214 }
215 leds_event(led_idle_end); 215 leds_event(led_idle_end);
216 tick_nohz_restart_sched_tick(); 216 tick_nohz_idle_exit();
217 preempt_enable_no_resched(); 217 preempt_enable_no_resched();
218 schedule(); 218 schedule();
219 preempt_disable(); 219 preempt_disable();
diff --git a/arch/avr32/kernel/process.c b/arch/avr32/kernel/process.c
index ef5a2a08fcca..6ee7952248db 100644
--- a/arch/avr32/kernel/process.c
+++ b/arch/avr32/kernel/process.c
@@ -34,10 +34,10 @@ void cpu_idle(void)
34{ 34{
35 /* endless idle loop with no priority at all */ 35 /* endless idle loop with no priority at all */
36 while (1) { 36 while (1) {
37 tick_nohz_stop_sched_tick(1); 37 tick_nohz_idle_enter();
38 while (!need_resched()) 38 while (!need_resched())
39 cpu_idle_sleep(); 39 cpu_idle_sleep();
40 tick_nohz_restart_sched_tick(); 40 tick_nohz_idle_exit();
41 preempt_enable_no_resched(); 41 preempt_enable_no_resched();
42 schedule(); 42 schedule();
43 preempt_disable(); 43 preempt_disable();
diff --git a/arch/blackfin/kernel/process.c b/arch/blackfin/kernel/process.c
index 6a80a9e9fc4a..7b141b5c9e8d 100644
--- a/arch/blackfin/kernel/process.c
+++ b/arch/blackfin/kernel/process.c
@@ -88,10 +88,10 @@ void cpu_idle(void)
88#endif 88#endif
89 if (!idle) 89 if (!idle)
90 idle = default_idle; 90 idle = default_idle;
91 tick_nohz_stop_sched_tick(1); 91 tick_nohz_idle_enter();
92 while (!need_resched()) 92 while (!need_resched())
93 idle(); 93 idle();
94 tick_nohz_restart_sched_tick(); 94 tick_nohz_idle_exit();
95 preempt_enable_no_resched(); 95 preempt_enable_no_resched();
96 schedule(); 96 schedule();
97 preempt_disable(); 97 preempt_disable();
diff --git a/arch/microblaze/kernel/process.c b/arch/microblaze/kernel/process.c
index 95cc295976a7..5407f09b4be4 100644
--- a/arch/microblaze/kernel/process.c
+++ b/arch/microblaze/kernel/process.c
@@ -103,10 +103,10 @@ void cpu_idle(void)
103 if (!idle) 103 if (!idle)
104 idle = default_idle; 104 idle = default_idle;
105 105
106 tick_nohz_stop_sched_tick(1); 106 tick_nohz_idle_enter();
107 while (!need_resched()) 107 while (!need_resched())
108 idle(); 108 idle();
109 tick_nohz_restart_sched_tick(); 109 tick_nohz_idle_exit();
110 110
111 preempt_enable_no_resched(); 111 preempt_enable_no_resched();
112 schedule(); 112 schedule();
diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c
index c47f96e453c0..c11e5ca2a434 100644
--- a/arch/mips/kernel/process.c
+++ b/arch/mips/kernel/process.c
@@ -56,7 +56,7 @@ void __noreturn cpu_idle(void)
56 56
57 /* endless idle loop with no priority at all */ 57 /* endless idle loop with no priority at all */
58 while (1) { 58 while (1) {
59 tick_nohz_stop_sched_tick(1); 59 tick_nohz_idle_enter();
60 while (!need_resched() && cpu_online(cpu)) { 60 while (!need_resched() && cpu_online(cpu)) {
61#ifdef CONFIG_MIPS_MT_SMTC 61#ifdef CONFIG_MIPS_MT_SMTC
62 extern void smtc_idle_loop_hook(void); 62 extern void smtc_idle_loop_hook(void);
@@ -77,7 +77,7 @@ void __noreturn cpu_idle(void)
77 system_state == SYSTEM_BOOTING)) 77 system_state == SYSTEM_BOOTING))
78 play_dead(); 78 play_dead();
79#endif 79#endif
80 tick_nohz_restart_sched_tick(); 80 tick_nohz_idle_exit();
81 preempt_enable_no_resched(); 81 preempt_enable_no_resched();
82 schedule(); 82 schedule();
83 preempt_disable(); 83 preempt_disable();
diff --git a/arch/openrisc/kernel/idle.c b/arch/openrisc/kernel/idle.c
index d5bc5f813e89..fb6a9bf40006 100644
--- a/arch/openrisc/kernel/idle.c
+++ b/arch/openrisc/kernel/idle.c
@@ -51,7 +51,7 @@ void cpu_idle(void)
51 51
52 /* endless idle loop with no priority at all */ 52 /* endless idle loop with no priority at all */
53 while (1) { 53 while (1) {
54 tick_nohz_stop_sched_tick(1); 54 tick_nohz_idle_enter();
55 55
56 while (!need_resched()) { 56 while (!need_resched()) {
57 check_pgt_cache(); 57 check_pgt_cache();
@@ -69,7 +69,7 @@ void cpu_idle(void)
69 set_thread_flag(TIF_POLLING_NRFLAG); 69 set_thread_flag(TIF_POLLING_NRFLAG);
70 } 70 }
71 71
72 tick_nohz_restart_sched_tick(); 72 tick_nohz_idle_exit();
73 preempt_enable_no_resched(); 73 preempt_enable_no_resched();
74 schedule(); 74 schedule();
75 preempt_disable(); 75 preempt_disable();
diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c
index 39a2baa6ad58..878572f70ac5 100644
--- a/arch/powerpc/kernel/idle.c
+++ b/arch/powerpc/kernel/idle.c
@@ -56,7 +56,7 @@ void cpu_idle(void)
56 56
57 set_thread_flag(TIF_POLLING_NRFLAG); 57 set_thread_flag(TIF_POLLING_NRFLAG);
58 while (1) { 58 while (1) {
59 tick_nohz_stop_sched_tick(1); 59 tick_nohz_idle_enter();
60 while (!need_resched() && !cpu_should_die()) { 60 while (!need_resched() && !cpu_should_die()) {
61 ppc64_runlatch_off(); 61 ppc64_runlatch_off();
62 62
@@ -93,7 +93,7 @@ void cpu_idle(void)
93 93
94 HMT_medium(); 94 HMT_medium();
95 ppc64_runlatch_on(); 95 ppc64_runlatch_on();
96 tick_nohz_restart_sched_tick(); 96 tick_nohz_idle_exit();
97 preempt_enable_no_resched(); 97 preempt_enable_no_resched();
98 if (cpu_should_die()) 98 if (cpu_should_die())
99 cpu_die(); 99 cpu_die();
diff --git a/arch/powerpc/platforms/iseries/setup.c b/arch/powerpc/platforms/iseries/setup.c
index ea0acbd8966d..e83dfaf89f69 100644
--- a/arch/powerpc/platforms/iseries/setup.c
+++ b/arch/powerpc/platforms/iseries/setup.c
@@ -563,7 +563,7 @@ static void yield_shared_processor(void)
563static void iseries_shared_idle(void) 563static void iseries_shared_idle(void)
564{ 564{
565 while (1) { 565 while (1) {
566 tick_nohz_stop_sched_tick(1); 566 tick_nohz_idle_enter();
567 while (!need_resched() && !hvlpevent_is_pending()) { 567 while (!need_resched() && !hvlpevent_is_pending()) {
568 local_irq_disable(); 568 local_irq_disable();
569 ppc64_runlatch_off(); 569 ppc64_runlatch_off();
@@ -577,7 +577,7 @@ static void iseries_shared_idle(void)
577 } 577 }
578 578
579 ppc64_runlatch_on(); 579 ppc64_runlatch_on();
580 tick_nohz_restart_sched_tick(); 580 tick_nohz_idle_exit();
581 581
582 if (hvlpevent_is_pending()) 582 if (hvlpevent_is_pending())
583 process_iSeries_events(); 583 process_iSeries_events();
@@ -593,7 +593,7 @@ static void iseries_dedicated_idle(void)
593 set_thread_flag(TIF_POLLING_NRFLAG); 593 set_thread_flag(TIF_POLLING_NRFLAG);
594 594
595 while (1) { 595 while (1) {
596 tick_nohz_stop_sched_tick(1); 596 tick_nohz_idle_enter();
597 if (!need_resched()) { 597 if (!need_resched()) {
598 while (!need_resched()) { 598 while (!need_resched()) {
599 ppc64_runlatch_off(); 599 ppc64_runlatch_off();
@@ -610,7 +610,7 @@ static void iseries_dedicated_idle(void)
610 } 610 }
611 611
612 ppc64_runlatch_on(); 612 ppc64_runlatch_on();
613 tick_nohz_restart_sched_tick(); 613 tick_nohz_idle_exit();
614 preempt_enable_no_resched(); 614 preempt_enable_no_resched();
615 schedule(); 615 schedule();
616 preempt_disable(); 616 preempt_disable();
diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c
index 9451b210a1b4..6224f9dbbc1f 100644
--- a/arch/s390/kernel/process.c
+++ b/arch/s390/kernel/process.c
@@ -91,10 +91,10 @@ static void default_idle(void)
91void cpu_idle(void) 91void cpu_idle(void)
92{ 92{
93 for (;;) { 93 for (;;) {
94 tick_nohz_stop_sched_tick(1); 94 tick_nohz_idle_enter();
95 while (!need_resched()) 95 while (!need_resched())
96 default_idle(); 96 default_idle();
97 tick_nohz_restart_sched_tick(); 97 tick_nohz_idle_exit();
98 preempt_enable_no_resched(); 98 preempt_enable_no_resched();
99 schedule(); 99 schedule();
100 preempt_disable(); 100 preempt_disable();
diff --git a/arch/sh/kernel/idle.c b/arch/sh/kernel/idle.c
index db4ecd731a00..6015743020a0 100644
--- a/arch/sh/kernel/idle.c
+++ b/arch/sh/kernel/idle.c
@@ -89,7 +89,7 @@ void cpu_idle(void)
89 89
90 /* endless idle loop with no priority at all */ 90 /* endless idle loop with no priority at all */
91 while (1) { 91 while (1) {
92 tick_nohz_stop_sched_tick(1); 92 tick_nohz_idle_enter();
93 93
94 while (!need_resched()) { 94 while (!need_resched()) {
95 check_pgt_cache(); 95 check_pgt_cache();
@@ -111,7 +111,7 @@ void cpu_idle(void)
111 start_critical_timings(); 111 start_critical_timings();
112 } 112 }
113 113
114 tick_nohz_restart_sched_tick(); 114 tick_nohz_idle_exit();
115 preempt_enable_no_resched(); 115 preempt_enable_no_resched();
116 schedule(); 116 schedule();
117 preempt_disable(); 117 preempt_disable();
diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c
index 3739a06a76cb..9c2795ba2cfe 100644
--- a/arch/sparc/kernel/process_64.c
+++ b/arch/sparc/kernel/process_64.c
@@ -95,12 +95,12 @@ void cpu_idle(void)
95 set_thread_flag(TIF_POLLING_NRFLAG); 95 set_thread_flag(TIF_POLLING_NRFLAG);
96 96
97 while(1) { 97 while(1) {
98 tick_nohz_stop_sched_tick(1); 98 tick_nohz_idle_enter();
99 99
100 while (!need_resched() && !cpu_is_offline(cpu)) 100 while (!need_resched() && !cpu_is_offline(cpu))
101 sparc64_yield(cpu); 101 sparc64_yield(cpu);
102 102
103 tick_nohz_restart_sched_tick(); 103 tick_nohz_idle_exit();
104 104
105 preempt_enable_no_resched(); 105 preempt_enable_no_resched();
106 106
diff --git a/arch/tile/kernel/process.c b/arch/tile/kernel/process.c
index 9c45d8bbdf57..920e674aedb9 100644
--- a/arch/tile/kernel/process.c
+++ b/arch/tile/kernel/process.c
@@ -85,7 +85,7 @@ void cpu_idle(void)
85 85
86 /* endless idle loop with no priority at all */ 86 /* endless idle loop with no priority at all */
87 while (1) { 87 while (1) {
88 tick_nohz_stop_sched_tick(1); 88 tick_nohz_idle_enter();
89 while (!need_resched()) { 89 while (!need_resched()) {
90 if (cpu_is_offline(cpu)) 90 if (cpu_is_offline(cpu))
91 BUG(); /* no HOTPLUG_CPU */ 91 BUG(); /* no HOTPLUG_CPU */
@@ -105,7 +105,7 @@ void cpu_idle(void)
105 local_irq_enable(); 105 local_irq_enable();
106 current_thread_info()->status |= TS_POLLING; 106 current_thread_info()->status |= TS_POLLING;
107 } 107 }
108 tick_nohz_restart_sched_tick(); 108 tick_nohz_idle_exit();
109 preempt_enable_no_resched(); 109 preempt_enable_no_resched();
110 schedule(); 110 schedule();
111 preempt_disable(); 111 preempt_disable();
diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c
index c5338351aecd..cfb657e92849 100644
--- a/arch/um/kernel/process.c
+++ b/arch/um/kernel/process.c
@@ -246,10 +246,10 @@ void default_idle(void)
246 if (need_resched()) 246 if (need_resched())
247 schedule(); 247 schedule();
248 248
249 tick_nohz_stop_sched_tick(1); 249 tick_nohz_idle_enter();
250 nsecs = disable_timer(); 250 nsecs = disable_timer();
251 idle_sleep(nsecs); 251 idle_sleep(nsecs);
252 tick_nohz_restart_sched_tick(); 252 tick_nohz_idle_exit();
253 } 253 }
254} 254}
255 255
diff --git a/arch/unicore32/kernel/process.c b/arch/unicore32/kernel/process.c
index ba401df971ed..9999b9a84d46 100644
--- a/arch/unicore32/kernel/process.c
+++ b/arch/unicore32/kernel/process.c
@@ -55,7 +55,7 @@ void cpu_idle(void)
55{ 55{
56 /* endless idle loop with no priority at all */ 56 /* endless idle loop with no priority at all */
57 while (1) { 57 while (1) {
58 tick_nohz_stop_sched_tick(1); 58 tick_nohz_idle_enter();
59 while (!need_resched()) { 59 while (!need_resched()) {
60 local_irq_disable(); 60 local_irq_disable();
61 stop_critical_timings(); 61 stop_critical_timings();
@@ -63,7 +63,7 @@ void cpu_idle(void)
63 local_irq_enable(); 63 local_irq_enable();
64 start_critical_timings(); 64 start_critical_timings();
65 } 65 }
66 tick_nohz_restart_sched_tick(); 66 tick_nohz_idle_exit();
67 preempt_enable_no_resched(); 67 preempt_enable_no_resched();
68 schedule(); 68 schedule();
69 preempt_disable(); 69 preempt_disable();
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 795b79f984c2..6d9d4d52cac5 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -99,7 +99,7 @@ void cpu_idle(void)
99 99
100 /* endless idle loop with no priority at all */ 100 /* endless idle loop with no priority at all */
101 while (1) { 101 while (1) {
102 tick_nohz_stop_sched_tick(1); 102 tick_nohz_idle_enter();
103 while (!need_resched()) { 103 while (!need_resched()) {
104 104
105 check_pgt_cache(); 105 check_pgt_cache();
@@ -116,7 +116,7 @@ void cpu_idle(void)
116 pm_idle(); 116 pm_idle();
117 start_critical_timings(); 117 start_critical_timings();
118 } 118 }
119 tick_nohz_restart_sched_tick(); 119 tick_nohz_idle_exit();
120 preempt_enable_no_resched(); 120 preempt_enable_no_resched();
121 schedule(); 121 schedule();
122 preempt_disable(); 122 preempt_disable();
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 3bd7e6eebf31..b069e9d7875f 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -122,7 +122,7 @@ void cpu_idle(void)
122 122
123 /* endless idle loop with no priority at all */ 123 /* endless idle loop with no priority at all */
124 while (1) { 124 while (1) {
125 tick_nohz_stop_sched_tick(1); 125 tick_nohz_idle_enter();
126 while (!need_resched()) { 126 while (!need_resched()) {
127 127
128 rmb(); 128 rmb();
@@ -149,7 +149,7 @@ void cpu_idle(void)
149 __exit_idle(); 149 __exit_idle();
150 } 150 }
151 151
152 tick_nohz_restart_sched_tick(); 152 tick_nohz_idle_exit();
153 preempt_enable_no_resched(); 153 preempt_enable_no_resched();
154 schedule(); 154 schedule();
155 preempt_disable(); 155 preempt_disable();
diff --git a/include/linux/tick.h b/include/linux/tick.h
index ca40838fdfb7..0df1d50a408a 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -121,21 +121,22 @@ static inline int tick_oneshot_mode_active(void) { return 0; }
121#endif /* !CONFIG_GENERIC_CLOCKEVENTS */ 121#endif /* !CONFIG_GENERIC_CLOCKEVENTS */
122 122
123# ifdef CONFIG_NO_HZ 123# ifdef CONFIG_NO_HZ
124extern void tick_nohz_stop_sched_tick(int inidle); 124extern void tick_nohz_idle_enter(void);
125extern void tick_nohz_restart_sched_tick(void); 125extern void tick_nohz_idle_exit(void);
126extern void tick_nohz_irq_exit(void);
126extern ktime_t tick_nohz_get_sleep_length(void); 127extern ktime_t tick_nohz_get_sleep_length(void);
127extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); 128extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time);
128extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); 129extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time);
129# else 130# else
130static inline void tick_nohz_stop_sched_tick(int inidle) 131static inline void tick_nohz_idle_enter(void)
131{ 132{
132 if (inidle) 133 rcu_idle_enter();
133 rcu_idle_enter();
134} 134}
135static inline void tick_nohz_restart_sched_tick(void) 135static inline void tick_nohz_idle_exit(void)
136{ 136{
137 rcu_idle_exit(); 137 rcu_idle_exit();
138} 138}
139
139static inline ktime_t tick_nohz_get_sleep_length(void) 140static inline ktime_t tick_nohz_get_sleep_length(void)
140{ 141{
141 ktime_t len = { .tv64 = NSEC_PER_SEC/HZ }; 142 ktime_t len = { .tv64 = NSEC_PER_SEC/HZ };
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 2c71d91efff0..f9f2aa81ce53 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -351,7 +351,7 @@ void irq_exit(void)
351#ifdef CONFIG_NO_HZ 351#ifdef CONFIG_NO_HZ
352 /* Make sure that timer wheel updates are propagated */ 352 /* Make sure that timer wheel updates are propagated */
353 if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched()) 353 if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
354 tick_nohz_stop_sched_tick(0); 354 tick_nohz_irq_exit();
355#endif 355#endif
356 preempt_enable_no_resched(); 356 preempt_enable_no_resched();
357} 357}
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 5d9d23665f12..266c242dc354 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -275,42 +275,17 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
275} 275}
276EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); 276EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
277 277
278/** 278static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
279 * tick_nohz_stop_sched_tick - stop the idle tick from the idle task
280 *
281 * When the next event is more than a tick into the future, stop the idle tick
282 * Called either from the idle loop or from irq_exit() when an idle period was
283 * just interrupted by an interrupt which did not cause a reschedule.
284 */
285void tick_nohz_stop_sched_tick(int inidle)
286{ 279{
287 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags; 280 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies;
288 struct tick_sched *ts;
289 ktime_t last_update, expires, now; 281 ktime_t last_update, expires, now;
290 struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev; 282 struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
291 u64 time_delta; 283 u64 time_delta;
292 int cpu; 284 int cpu;
293 285
294 local_irq_save(flags);
295
296 cpu = smp_processor_id(); 286 cpu = smp_processor_id();
297 ts = &per_cpu(tick_cpu_sched, cpu); 287 ts = &per_cpu(tick_cpu_sched, cpu);
298 288
299 /*
300 * Call to tick_nohz_start_idle stops the last_update_time from being
301 * updated. Thus, it must not be called in the event we are called from
302 * irq_exit() with the prior state different than idle.
303 */
304 if (!inidle && !ts->inidle)
305 goto end;
306
307 /*
308 * Set ts->inidle unconditionally. Even if the system did not
309 * switch to NOHZ mode the cpu frequency governers rely on the
310 * update of the idle time accounting in tick_nohz_start_idle().
311 */
312 ts->inidle = 1;
313
314 now = tick_nohz_start_idle(cpu, ts); 289 now = tick_nohz_start_idle(cpu, ts);
315 290
316 /* 291 /*
@@ -326,10 +301,10 @@ void tick_nohz_stop_sched_tick(int inidle)
326 } 301 }
327 302
328 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) 303 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
329 goto end; 304 return;
330 305
331 if (need_resched()) 306 if (need_resched())
332 goto end; 307 return;
333 308
334 if (unlikely(local_softirq_pending() && cpu_online(cpu))) { 309 if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
335 static int ratelimit; 310 static int ratelimit;
@@ -339,7 +314,7 @@ void tick_nohz_stop_sched_tick(int inidle)
339 (unsigned int) local_softirq_pending()); 314 (unsigned int) local_softirq_pending());
340 ratelimit++; 315 ratelimit++;
341 } 316 }
342 goto end; 317 return;
343 } 318 }
344 319
345 ts->idle_calls++; 320 ts->idle_calls++;
@@ -471,10 +446,54 @@ out:
471 ts->next_jiffies = next_jiffies; 446 ts->next_jiffies = next_jiffies;
472 ts->last_jiffies = last_jiffies; 447 ts->last_jiffies = last_jiffies;
473 ts->sleep_length = ktime_sub(dev->next_event, now); 448 ts->sleep_length = ktime_sub(dev->next_event, now);
474end: 449}
475 if (inidle) 450
476 rcu_idle_enter(); 451/**
477 local_irq_restore(flags); 452 * tick_nohz_idle_enter - stop the idle tick from the idle task
453 *
454 * When the next event is more than a tick into the future, stop the idle tick
455 * Called when we start the idle loop.
456 * This also enters into RCU extended quiescent state so that this CPU doesn't
457 * need anymore to be part of any global grace period completion. This way
458 * the tick can be stopped safely as we don't need to report quiescent states.
459 */
460void tick_nohz_idle_enter(void)
461{
462 struct tick_sched *ts;
463
464 WARN_ON_ONCE(irqs_disabled());
465
466 local_irq_disable();
467
468 ts = &__get_cpu_var(tick_cpu_sched);
469 /*
470 * set ts->inidle unconditionally. even if the system did not
471 * switch to nohz mode the cpu frequency governers rely on the
472 * update of the idle time accounting in tick_nohz_start_idle().
473 */
474 ts->inidle = 1;
475 tick_nohz_stop_sched_tick(ts);
476 rcu_idle_enter();
477
478 local_irq_enable();
479}
480
481/**
482 * tick_nohz_irq_exit - update next tick event from interrupt exit
483 *
484 * When an interrupt fires while we are idle and it doesn't cause
485 * a reschedule, it may still add, modify or delete a timer, enqueue
486 * an RCU callback, etc...
487 * So we need to re-calculate and reprogram the next tick event.
488 */
489void tick_nohz_irq_exit(void)
490{
491 struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
492
493 if (!ts->inidle)
494 return;
495
496 tick_nohz_stop_sched_tick(ts);
478} 497}
479 498
480/** 499/**
@@ -516,11 +535,13 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
516} 535}
517 536
518/** 537/**
519 * tick_nohz_restart_sched_tick - restart the idle tick from the idle task 538 * tick_nohz_idle_exit - restart the idle tick from the idle task
520 * 539 *
521 * Restart the idle tick when the CPU is woken up from idle 540 * Restart the idle tick when the CPU is woken up from idle
541 * This also exit the RCU extended quiescent state. The CPU
542 * can use RCU again after this function is called.
522 */ 543 */
523void tick_nohz_restart_sched_tick(void) 544void tick_nohz_idle_exit(void)
524{ 545{
525 int cpu = smp_processor_id(); 546 int cpu = smp_processor_id();
526 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); 547 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-01 12:22:11 -0500