diff options
| -rw-r--r-- | drivers/net/ethernet/tile/tilegx.c | 4 | ||||
| -rw-r--r-- | include/linux/sched/isolation.h | 26 | ||||
| -rw-r--r-- | kernel/rcu/tree_plugin.h | 2 | ||||
| -rw-r--r-- | kernel/rcu/update.c | 2 | ||||
| -rw-r--r-- | kernel/sched/core.c | 8 | ||||
| -rw-r--r-- | kernel/sched/fair.c | 2 | ||||
| -rw-r--r-- | kernel/sched/isolation.c | 26 | ||||
| -rw-r--r-- | kernel/watchdog.c | 3 |
8 files changed, 43 insertions, 30 deletions
diff --git a/drivers/net/ethernet/tile/tilegx.c b/drivers/net/ethernet/tile/tilegx.c index 27a3272dcd48..b3e5816a4678 100644 --- a/drivers/net/ethernet/tile/tilegx.c +++ b/drivers/net/ethernet/tile/tilegx.c | |||
| @@ -2270,8 +2270,8 @@ static int __init tile_net_init_module(void) | |||
| 2270 | tile_net_dev_init(name, mac); | 2270 | tile_net_dev_init(name, mac); |
| 2271 | 2271 | ||
| 2272 | if (!network_cpus_init()) | 2272 | if (!network_cpus_init()) |
| 2273 | cpumask_and(&network_cpus_map, housekeeping_cpumask(), | 2273 | cpumask_and(&network_cpus_map, |
| 2274 | cpu_online_mask); | 2274 | housekeeping_cpumask(HK_FLAG_MISC), cpu_online_mask); |
| 2275 | 2275 | ||
| 2276 | return 0; | 2276 | return 0; |
| 2277 | } | 2277 | } |
diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h index 93ac2367a520..9bb753eece3b 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h | |||
| @@ -5,35 +5,43 @@ | |||
| 5 | #include <linux/init.h> | 5 | #include <linux/init.h> |
| 6 | #include <linux/tick.h> | 6 | #include <linux/tick.h> |
| 7 | 7 | ||
| 8 | enum hk_flags { | ||
| 9 | HK_FLAG_TIMER = 1, | ||
| 10 | HK_FLAG_RCU = (1 << 1), | ||
| 11 | HK_FLAG_MISC = (1 << 2), | ||
| 12 | HK_FLAG_SCHED = (1 << 3), | ||
| 13 | }; | ||
| 14 | |||
| 8 | #ifdef CONFIG_CPU_ISOLATION | 15 | #ifdef CONFIG_CPU_ISOLATION |
| 9 | DECLARE_STATIC_KEY_FALSE(housekeeping_overriden); | 16 | DECLARE_STATIC_KEY_FALSE(housekeeping_overriden); |
| 10 | extern int housekeeping_any_cpu(void); | 17 | extern int housekeeping_any_cpu(enum hk_flags flags); |
| 11 | extern const struct cpumask *housekeeping_cpumask(void); | 18 | extern const struct cpumask *housekeeping_cpumask(enum hk_flags flags); |
| 12 | extern void housekeeping_affine(struct task_struct *t); | 19 | extern void housekeeping_affine(struct task_struct *t, enum hk_flags flags); |
| 13 | extern bool housekeeping_test_cpu(int cpu); | 20 | extern bool housekeeping_test_cpu(int cpu, enum hk_flags flags); |
| 14 | extern void __init housekeeping_init(void); | 21 | extern void __init housekeeping_init(void); |
| 15 | 22 | ||
| 16 | #else | 23 | #else |
| 17 | 24 | ||
| 18 | static inline int housekeeping_any_cpu(void) | 25 | static inline int housekeeping_any_cpu(enum hk_flags flags) |
| 19 | { | 26 | { |
| 20 | return smp_processor_id(); | 27 | return smp_processor_id(); |
| 21 | } | 28 | } |
| 22 | 29 | ||
| 23 | static inline const struct cpumask *housekeeping_cpumask(void) | 30 | static inline const struct cpumask *housekeeping_cpumask(enum hk_flags flags) |
| 24 | { | 31 | { |
| 25 | return cpu_possible_mask; | 32 | return cpu_possible_mask; |
| 26 | } | 33 | } |
| 27 | 34 | ||
| 28 | static inline void housekeeping_affine(struct task_struct *t) { } | 35 | static inline void housekeeping_affine(struct task_struct *t, |
| 36 | enum hk_flags flags) { } | ||
| 29 | static inline void housekeeping_init(void) { } | 37 | static inline void housekeeping_init(void) { } |
| 30 | #endif /* CONFIG_CPU_ISOLATION */ | 38 | #endif /* CONFIG_CPU_ISOLATION */ |
| 31 | 39 | ||
| 32 | static inline bool housekeeping_cpu(int cpu) | 40 | static inline bool housekeeping_cpu(int cpu, enum hk_flags flags) |
| 33 | { | 41 | { |
| 34 | #ifdef CONFIG_CPU_ISOLATION | 42 | #ifdef CONFIG_CPU_ISOLATION |
| 35 | if (static_branch_unlikely(&housekeeping_overriden)) | 43 | if (static_branch_unlikely(&housekeeping_overriden)) |
| 36 | return housekeeping_test_cpu(cpu); | 44 | return housekeeping_test_cpu(cpu, flags); |
| 37 | #endif | 45 | #endif |
| 38 | return true; | 46 | return true; |
| 39 | } | 47 | } |
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index c7632f51c5a0..34125d23e58a 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h | |||
| @@ -2584,7 +2584,7 @@ static void rcu_bind_gp_kthread(void) | |||
| 2584 | 2584 | ||
| 2585 | if (!tick_nohz_full_enabled()) | 2585 | if (!tick_nohz_full_enabled()) |
| 2586 | return; | 2586 | return; |
| 2587 | housekeeping_affine(current); | 2587 | housekeeping_affine(current, HK_FLAG_RCU); |
| 2588 | } | 2588 | } |
| 2589 | 2589 | ||
| 2590 | /* Record the current task on dyntick-idle entry. */ | 2590 | /* Record the current task on dyntick-idle entry. */ |
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index 79abeb0ca329..e3e60efaafee 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c | |||
| @@ -719,7 +719,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) | |||
| 719 | LIST_HEAD(rcu_tasks_holdouts); | 719 | LIST_HEAD(rcu_tasks_holdouts); |
| 720 | 720 | ||
| 721 | /* Run on housekeeping CPUs by default. Sysadm can move if desired. */ | 721 | /* Run on housekeeping CPUs by default. Sysadm can move if desired. */ |
| 722 | housekeeping_affine(current); | 722 | housekeeping_affine(current, HK_FLAG_RCU); |
| 723 | 723 | ||
| 724 | /* | 724 | /* |
| 725 | * Each pass through the following loop makes one check for | 725 | * Each pass through the following loop makes one check for |
diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d0fb448dd43a..2210c0203e51 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c | |||
| @@ -527,7 +527,7 @@ int get_nohz_timer_target(void) | |||
| 527 | int i, cpu = smp_processor_id(); | 527 | int i, cpu = smp_processor_id(); |
| 528 | struct sched_domain *sd; | 528 | struct sched_domain *sd; |
| 529 | 529 | ||
| 530 | if (!idle_cpu(cpu) && housekeeping_cpu(cpu)) | 530 | if (!idle_cpu(cpu) && housekeeping_cpu(cpu, HK_FLAG_TIMER)) |
| 531 | return cpu; | 531 | return cpu; |
| 532 | 532 | ||
| 533 | rcu_read_lock(); | 533 | rcu_read_lock(); |
| @@ -536,15 +536,15 @@ int get_nohz_timer_target(void) | |||
| 536 | if (cpu == i) | 536 | if (cpu == i) |
| 537 | continue; | 537 | continue; |
| 538 | 538 | ||
| 539 | if (!idle_cpu(i) && housekeeping_cpu(i)) { | 539 | if (!idle_cpu(i) && housekeeping_cpu(i, HK_FLAG_TIMER)) { |
| 540 | cpu = i; | 540 | cpu = i; |
| 541 | goto unlock; | 541 | goto unlock; |
| 542 | } | 542 | } |
| 543 | } | 543 | } |
| 544 | } | 544 | } |
| 545 | 545 | ||
| 546 | if (!housekeeping_cpu(cpu)) | 546 | if (!housekeeping_cpu(cpu, HK_FLAG_TIMER)) |
| 547 | cpu = housekeeping_any_cpu(); | 547 | cpu = housekeeping_any_cpu(HK_FLAG_TIMER); |
| 548 | unlock: | 548 | unlock: |
| 549 | rcu_read_unlock(); | 549 | rcu_read_unlock(); |
| 550 | return cpu; | 550 | return cpu; |
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index cdece8f967f0..f755de86a813 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c | |||
| @@ -9027,7 +9027,7 @@ void nohz_balance_enter_idle(int cpu) | |||
| 9027 | return; | 9027 | return; |
| 9028 | 9028 | ||
| 9029 | /* Spare idle load balancing on CPUs that don't want to be disturbed: */ | 9029 | /* Spare idle load balancing on CPUs that don't want to be disturbed: */ |
| 9030 | if (!housekeeping_cpu(cpu)) | 9030 | if (!housekeeping_cpu(cpu, HK_FLAG_SCHED)) |
| 9031 | return; | 9031 | return; |
| 9032 | 9032 | ||
| 9033 | if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu))) | 9033 | if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu))) |
diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index bb8ba19a0235..37a138a780ff 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c | |||
| @@ -15,37 +15,39 @@ | |||
| 15 | DEFINE_STATIC_KEY_FALSE(housekeeping_overriden); | 15 | DEFINE_STATIC_KEY_FALSE(housekeeping_overriden); |
| 16 | EXPORT_SYMBOL_GPL(housekeeping_overriden); | 16 | EXPORT_SYMBOL_GPL(housekeeping_overriden); |
| 17 | static cpumask_var_t housekeeping_mask; | 17 | static cpumask_var_t housekeeping_mask; |
| 18 | static unsigned int housekeeping_flags; | ||
| 18 | 19 | ||
| 19 | int housekeeping_any_cpu(void) | 20 | int housekeeping_any_cpu(enum hk_flags flags) |
| 20 | { | 21 | { |
| 21 | if (static_branch_unlikely(&housekeeping_overriden)) | 22 | if (static_branch_unlikely(&housekeeping_overriden)) |
| 22 | return cpumask_any_and(housekeeping_mask, cpu_online_mask); | 23 | if (housekeeping_flags & flags) |
| 23 | 24 | return cpumask_any_and(housekeeping_mask, cpu_online_mask); | |
| 24 | return smp_processor_id(); | 25 | return smp_processor_id(); |
| 25 | } | 26 | } |
| 26 | EXPORT_SYMBOL_GPL(housekeeping_any_cpu); | 27 | EXPORT_SYMBOL_GPL(housekeeping_any_cpu); |
| 27 | 28 | ||
| 28 | const struct cpumask *housekeeping_cpumask(void) | 29 | const struct cpumask *housekeeping_cpumask(enum hk_flags flags) |
| 29 | { | 30 | { |
| 30 | if (static_branch_unlikely(&housekeeping_overriden)) | 31 | if (static_branch_unlikely(&housekeeping_overriden)) |
| 31 | return housekeeping_mask; | 32 | if (housekeeping_flags & flags) |
| 32 | 33 | return housekeeping_mask; | |
| 33 | return cpu_possible_mask; | 34 | return cpu_possible_mask; |
| 34 | } | 35 | } |
| 35 | EXPORT_SYMBOL_GPL(housekeeping_cpumask); | 36 | EXPORT_SYMBOL_GPL(housekeeping_cpumask); |
| 36 | 37 | ||
| 37 | void housekeeping_affine(struct task_struct *t) | 38 | void housekeeping_affine(struct task_struct *t, enum hk_flags flags) |
| 38 | { | 39 | { |
| 39 | if (static_branch_unlikely(&housekeeping_overriden)) | 40 | if (static_branch_unlikely(&housekeeping_overriden)) |
| 40 | set_cpus_allowed_ptr(t, housekeeping_mask); | 41 | if (housekeeping_flags & flags) |
| 42 | set_cpus_allowed_ptr(t, housekeeping_mask); | ||
| 41 | } | 43 | } |
| 42 | EXPORT_SYMBOL_GPL(housekeeping_affine); | 44 | EXPORT_SYMBOL_GPL(housekeeping_affine); |
| 43 | 45 | ||
| 44 | bool housekeeping_test_cpu(int cpu) | 46 | bool housekeeping_test_cpu(int cpu, enum hk_flags flags) |
| 45 | { | 47 | { |
| 46 | if (static_branch_unlikely(&housekeeping_overriden)) | 48 | if (static_branch_unlikely(&housekeeping_overriden)) |
| 47 | return cpumask_test_cpu(cpu, housekeeping_mask); | 49 | if (housekeeping_flags & flags) |
| 48 | 50 | return cpumask_test_cpu(cpu, housekeeping_mask); | |
| 49 | return true; | 51 | return true; |
| 50 | } | 52 | } |
| 51 | EXPORT_SYMBOL_GPL(housekeeping_test_cpu); | 53 | EXPORT_SYMBOL_GPL(housekeeping_test_cpu); |
| @@ -65,6 +67,8 @@ void __init housekeeping_init(void) | |||
| 65 | cpumask_andnot(housekeeping_mask, | 67 | cpumask_andnot(housekeeping_mask, |
| 66 | cpu_possible_mask, tick_nohz_full_mask); | 68 | cpu_possible_mask, tick_nohz_full_mask); |
| 67 | 69 | ||
| 70 | housekeeping_flags = HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC; | ||
| 71 | |||
| 68 | static_branch_enable(&housekeeping_overriden); | 72 | static_branch_enable(&housekeeping_overriden); |
| 69 | 73 | ||
| 70 | /* We need at least one CPU to handle housekeeping work */ | 74 | /* We need at least one CPU to handle housekeeping work */ |
diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 562652c9c815..e9e2ebb3db29 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c | |||
| @@ -777,7 +777,8 @@ void __init lockup_detector_init(void) | |||
| 777 | if (tick_nohz_full_enabled()) | 777 | if (tick_nohz_full_enabled()) |
| 778 | pr_info("Disabling watchdog on nohz_full cores by default\n"); | 778 | pr_info("Disabling watchdog on nohz_full cores by default\n"); |
| 779 | 779 | ||
| 780 | cpumask_copy(&watchdog_cpumask, housekeeping_cpumask()); | 780 | cpumask_copy(&watchdog_cpumask, |
| 781 | housekeeping_cpumask(HK_FLAG_TIMER)); | ||
| 781 | 782 | ||
| 782 | if (!watchdog_nmi_probe()) | 783 | if (!watchdog_nmi_probe()) |
| 783 | nmi_watchdog_available = true; | 784 | nmi_watchdog_available = true; |
