diff options
-rw-r--r-- | include/linux/tick.h | 20 | ||||
-rw-r--r-- | kernel/rcu/tree_plugin.h | 14 | ||||
-rw-r--r-- | kernel/time/tick-sched.c | 10 |
3 files changed, 39 insertions, 5 deletions
diff --git a/include/linux/tick.h b/include/linux/tick.h index b84773cb9f4c..06cc093ab7ad 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h | |||
@@ -12,6 +12,7 @@ | |||
12 | #include <linux/hrtimer.h> | 12 | #include <linux/hrtimer.h> |
13 | #include <linux/context_tracking_state.h> | 13 | #include <linux/context_tracking_state.h> |
14 | #include <linux/cpumask.h> | 14 | #include <linux/cpumask.h> |
15 | #include <linux/sched.h> | ||
15 | 16 | ||
16 | #ifdef CONFIG_GENERIC_CLOCKEVENTS | 17 | #ifdef CONFIG_GENERIC_CLOCKEVENTS |
17 | 18 | ||
@@ -162,6 +163,7 @@ static inline u64 get_cpu_iowait_time_us(int cpu, u64 *unused) { return -1; } | |||
162 | #ifdef CONFIG_NO_HZ_FULL | 163 | #ifdef CONFIG_NO_HZ_FULL |
163 | extern bool tick_nohz_full_running; | 164 | extern bool tick_nohz_full_running; |
164 | extern cpumask_var_t tick_nohz_full_mask; | 165 | extern cpumask_var_t tick_nohz_full_mask; |
166 | extern cpumask_var_t housekeeping_mask; | ||
165 | 167 | ||
166 | static inline bool tick_nohz_full_enabled(void) | 168 | static inline bool tick_nohz_full_enabled(void) |
167 | { | 169 | { |
@@ -194,6 +196,24 @@ static inline void tick_nohz_full_kick_all(void) { } | |||
194 | static inline void __tick_nohz_task_switch(struct task_struct *tsk) { } | 196 | static inline void __tick_nohz_task_switch(struct task_struct *tsk) { } |
195 | #endif | 197 | #endif |
196 | 198 | ||
199 | static inline bool is_housekeeping_cpu(int cpu) | ||
200 | { | ||
201 | #ifdef CONFIG_NO_HZ_FULL | ||
202 | if (tick_nohz_full_enabled()) | ||
203 | return cpumask_test_cpu(cpu, housekeeping_mask); | ||
204 | #endif | ||
205 | return true; | ||
206 | } | ||
207 | |||
208 | static inline void housekeeping_affine(struct task_struct *t) | ||
209 | { | ||
210 | #ifdef CONFIG_NO_HZ_FULL | ||
211 | if (tick_nohz_full_enabled()) | ||
212 | set_cpus_allowed_ptr(t, housekeeping_mask); | ||
213 | |||
214 | #endif | ||
215 | } | ||
216 | |||
197 | static inline void tick_nohz_full_check(void) | 217 | static inline void tick_nohz_full_check(void) |
198 | { | 218 | { |
199 | if (tick_nohz_full_enabled()) | 219 | if (tick_nohz_full_enabled()) |
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 719587af7b10..b39ba7239bd6 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h | |||
@@ -2846,12 +2846,16 @@ static bool rcu_nohz_full_cpu(struct rcu_state *rsp) | |||
2846 | */ | 2846 | */ |
2847 | static void rcu_bind_gp_kthread(void) | 2847 | static void rcu_bind_gp_kthread(void) |
2848 | { | 2848 | { |
2849 | #ifdef CONFIG_NO_HZ_FULL | 2849 | int __maybe_unused cpu; |
2850 | int cpu = tick_do_timer_cpu; | ||
2851 | 2850 | ||
2852 | if (cpu < 0 || cpu >= nr_cpu_ids) | 2851 | if (!tick_nohz_full_enabled()) |
2853 | return; | 2852 | return; |
2854 | if (raw_smp_processor_id() != cpu) | 2853 | #ifdef CONFIG_NO_HZ_FULL_SYSIDLE |
2854 | cpu = tick_do_timer_cpu; | ||
2855 | if (cpu >= 0 && cpu < nr_cpu_ids && raw_smp_processor_id() != cpu) | ||
2855 | set_cpus_allowed_ptr(current, cpumask_of(cpu)); | 2856 | set_cpus_allowed_ptr(current, cpumask_of(cpu)); |
2856 | #endif /* #ifdef CONFIG_NO_HZ_FULL */ | 2857 | #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */ |
2858 | if (!is_housekeeping_cpu(raw_smp_processor_id())) | ||
2859 | housekeeping_affine(current); | ||
2860 | #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */ | ||
2857 | } | 2861 | } |
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 6558b7ac112d..f784d83e29f1 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
@@ -154,6 +154,7 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) | |||
154 | 154 | ||
155 | #ifdef CONFIG_NO_HZ_FULL | 155 | #ifdef CONFIG_NO_HZ_FULL |
156 | cpumask_var_t tick_nohz_full_mask; | 156 | cpumask_var_t tick_nohz_full_mask; |
157 | cpumask_var_t housekeeping_mask; | ||
157 | bool tick_nohz_full_running; | 158 | bool tick_nohz_full_running; |
158 | 159 | ||
159 | static bool can_stop_full_tick(void) | 160 | static bool can_stop_full_tick(void) |
@@ -281,6 +282,7 @@ static int __init tick_nohz_full_setup(char *str) | |||
281 | int cpu; | 282 | int cpu; |
282 | 283 | ||
283 | alloc_bootmem_cpumask_var(&tick_nohz_full_mask); | 284 | alloc_bootmem_cpumask_var(&tick_nohz_full_mask); |
285 | alloc_bootmem_cpumask_var(&housekeeping_mask); | ||
284 | if (cpulist_parse(str, tick_nohz_full_mask) < 0) { | 286 | if (cpulist_parse(str, tick_nohz_full_mask) < 0) { |
285 | pr_warning("NOHZ: Incorrect nohz_full cpumask\n"); | 287 | pr_warning("NOHZ: Incorrect nohz_full cpumask\n"); |
286 | return 1; | 288 | return 1; |
@@ -291,6 +293,8 @@ static int __init tick_nohz_full_setup(char *str) | |||
291 | pr_warning("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", cpu); | 293 | pr_warning("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", cpu); |
292 | cpumask_clear_cpu(cpu, tick_nohz_full_mask); | 294 | cpumask_clear_cpu(cpu, tick_nohz_full_mask); |
293 | } | 295 | } |
296 | cpumask_andnot(housekeeping_mask, | ||
297 | cpu_possible_mask, tick_nohz_full_mask); | ||
294 | tick_nohz_full_running = true; | 298 | tick_nohz_full_running = true; |
295 | 299 | ||
296 | return 1; | 300 | return 1; |
@@ -332,9 +336,15 @@ static int tick_nohz_init_all(void) | |||
332 | pr_err("NO_HZ: Can't allocate full dynticks cpumask\n"); | 336 | pr_err("NO_HZ: Can't allocate full dynticks cpumask\n"); |
333 | return err; | 337 | return err; |
334 | } | 338 | } |
339 | if (!alloc_cpumask_var(&housekeeping_mask, GFP_KERNEL)) { | ||
340 | pr_err("NO_HZ: Can't allocate not-full dynticks cpumask\n"); | ||
341 | return err; | ||
342 | } | ||
335 | err = 0; | 343 | err = 0; |
336 | cpumask_setall(tick_nohz_full_mask); | 344 | cpumask_setall(tick_nohz_full_mask); |
337 | cpumask_clear_cpu(smp_processor_id(), tick_nohz_full_mask); | 345 | cpumask_clear_cpu(smp_processor_id(), tick_nohz_full_mask); |
346 | cpumask_clear(housekeeping_mask); | ||
347 | cpumask_set_cpu(smp_processor_id(), housekeeping_mask); | ||
338 | tick_nohz_full_running = true; | 348 | tick_nohz_full_running = true; |
339 | #endif | 349 | #endif |
340 | return err; | 350 | return err; |