aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/sched/isolation.h3
-rw-r--r--kernel/sched/isolation.c13
2 files changed, 11 insertions, 5 deletions
diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h
index ed935ffc6ffa..194c586fbb12 100644
--- a/include/linux/sched/isolation.h
+++ b/include/linux/sched/isolation.h
@@ -6,6 +6,7 @@
6#include <linux/tick.h> 6#include <linux/tick.h>
7 7
8#ifdef CONFIG_NO_HZ_FULL 8#ifdef CONFIG_NO_HZ_FULL
9DECLARE_STATIC_KEY_FALSE(housekeeping_overriden);
9extern int housekeeping_any_cpu(void); 10extern int housekeeping_any_cpu(void);
10extern const struct cpumask *housekeeping_cpumask(void); 11extern const struct cpumask *housekeeping_cpumask(void);
11extern void housekeeping_affine(struct task_struct *t); 12extern void housekeeping_affine(struct task_struct *t);
@@ -31,7 +32,7 @@ static inline void housekeeping_init(void) { }
31static inline bool is_housekeeping_cpu(int cpu) 32static inline bool is_housekeeping_cpu(int cpu)
32{ 33{
33#ifdef CONFIG_NO_HZ_FULL 34#ifdef CONFIG_NO_HZ_FULL
34 if (tick_nohz_full_enabled()) 35 if (static_branch_unlikely(&housekeeping_overriden))
35 return housekeeping_test_cpu(cpu); 36 return housekeeping_test_cpu(cpu);
36#endif 37#endif
37 return true; 38 return true;
diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c
index 16445097eb25..bb8ba19a0235 100644
--- a/kernel/sched/isolation.c
+++ b/kernel/sched/isolation.c
@@ -10,12 +10,15 @@
10#include <linux/tick.h> 10#include <linux/tick.h>
11#include <linux/init.h> 11#include <linux/init.h>
12#include <linux/kernel.h> 12#include <linux/kernel.h>
13#include <linux/static_key.h>
13 14
15DEFINE_STATIC_KEY_FALSE(housekeeping_overriden);
16EXPORT_SYMBOL_GPL(housekeeping_overriden);
14static cpumask_var_t housekeeping_mask; 17static cpumask_var_t housekeeping_mask;
15 18
16int housekeeping_any_cpu(void) 19int housekeeping_any_cpu(void)
17{ 20{
18 if (tick_nohz_full_enabled()) 21 if (static_branch_unlikely(&housekeeping_overriden))
19 return cpumask_any_and(housekeeping_mask, cpu_online_mask); 22 return cpumask_any_and(housekeeping_mask, cpu_online_mask);
20 23
21 return smp_processor_id(); 24 return smp_processor_id();
@@ -24,7 +27,7 @@ EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
24 27
25const struct cpumask *housekeeping_cpumask(void) 28const struct cpumask *housekeeping_cpumask(void)
26{ 29{
27 if (tick_nohz_full_enabled()) 30 if (static_branch_unlikely(&housekeeping_overriden))
28 return housekeeping_mask; 31 return housekeeping_mask;
29 32
30 return cpu_possible_mask; 33 return cpu_possible_mask;
@@ -33,14 +36,14 @@ EXPORT_SYMBOL_GPL(housekeeping_cpumask);
33 36
34void housekeeping_affine(struct task_struct *t) 37void housekeeping_affine(struct task_struct *t)
35{ 38{
36 if (tick_nohz_full_enabled()) 39 if (static_branch_unlikely(&housekeeping_overriden))
37 set_cpus_allowed_ptr(t, housekeeping_mask); 40 set_cpus_allowed_ptr(t, housekeeping_mask);
38} 41}
39EXPORT_SYMBOL_GPL(housekeeping_affine); 42EXPORT_SYMBOL_GPL(housekeeping_affine);
40 43
41bool housekeeping_test_cpu(int cpu) 44bool housekeeping_test_cpu(int cpu)
42{ 45{
43 if (tick_nohz_full_enabled()) 46 if (static_branch_unlikely(&housekeeping_overriden))
44 return cpumask_test_cpu(cpu, housekeeping_mask); 47 return cpumask_test_cpu(cpu, housekeeping_mask);
45 48
46 return true; 49 return true;
@@ -62,6 +65,8 @@ void __init housekeeping_init(void)
62 cpumask_andnot(housekeeping_mask, 65 cpumask_andnot(housekeeping_mask,
63 cpu_possible_mask, tick_nohz_full_mask); 66 cpu_possible_mask, tick_nohz_full_mask);
64 67
68 static_branch_enable(&housekeeping_overriden);
69
65 /* We need at least one CPU to handle housekeeping work */ 70 /* We need at least one CPU to handle housekeeping work */
66 WARN_ON_ONCE(cpumask_empty(housekeeping_mask)); 71 WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
67} 72}