Merge branch 'timers/nohz-reviewed' of git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks into timers/nohz

Pull full dynticks timekeeping and RCU improvements from Frederic Weisbecker.

Signed-off-by: Ingo Molnar <mingo@kernel.org>

2 files changed, 51 insertions, 36 deletions

diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index 358d601a4fec..99c3f13dd478 100644
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -128,6 +128,16 @@ config NO_HZ_FULL
 
 endchoice
 
+config NO_HZ_FULL_ALL
+       bool "Full dynticks system on all CPUs by default"
+       depends on NO_HZ_FULL
+       help
+         If the user doesn't pass the nohz_full boot option to
+         define the range of full dynticks CPUs, consider that all
+         CPUs in the system are full dynticks by default.
+         Note the boot CPU will still be kept outside the range to
+         handle the timekeeping duty.
+
 config NO_HZ
         bool "Old Idle dynticks config"
         depends on !ARCH_USES_GETTIMEOFFSET && GENERIC_CLOCKEVENTS
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 369b5769fc97..a76e09044f9f 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -158,11 +158,21 @@ int tick_nohz_full_cpu(int cpu)
 /* Parse the boot-time nohz CPU list from the kernel parameters. */
 static int __init tick_nohz_full_setup(char *str)
 {
+        int cpu;
+
         alloc_bootmem_cpumask_var(&nohz_full_mask);
-        if (cpulist_parse(str, nohz_full_mask) < 0)
+        if (cpulist_parse(str, nohz_full_mask) < 0) {
                 pr_warning("NOHZ: Incorrect nohz_full cpumask\n");
-        else
-                have_nohz_full_mask = true;
+                return 1;
+        }
+
+        cpu = smp_processor_id();
+        if (cpumask_test_cpu(cpu, nohz_full_mask)) {
+                pr_warning("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", cpu);
+                cpumask_clear_cpu(cpu, nohz_full_mask);
+        }
+        have_nohz_full_mask = true;
+
         return 1;
 }
 __setup("nohz_full=", tick_nohz_full_setup);
@@ -193,51 +203,46 @@ static int __cpuinit tick_nohz_cpu_down_callback(struct notifier_block *nfb,
  */
 static char __initdata nohz_full_buf[NR_CPUS + 1];
 
-static int __init init_tick_nohz_full(void)
+static int tick_nohz_init_all(void)
 {
-        cpumask_var_t online_nohz;
-        int cpu;
+        int err = -1;
 
-        if (!have_nohz_full_mask)
-                return 0;
+#ifdef CONFIG_NO_HZ_FULL_ALL
+        if (!alloc_cpumask_var(&nohz_full_mask, GFP_KERNEL)) {
+                pr_err("NO_HZ: Can't allocate full dynticks cpumask\n");
+                return err;
+        }
+        err = 0;
+        cpumask_setall(nohz_full_mask);
+        cpumask_clear_cpu(smp_processor_id(), nohz_full_mask);
+        have_nohz_full_mask = true;
+#endif
+        return err;
+}
 
-        cpu_notifier(tick_nohz_cpu_down_callback, 0);
+void __init tick_nohz_init(void)
+{
+        int cpu;
 
-        if (!zalloc_cpumask_var(&online_nohz, GFP_KERNEL)) {
-                pr_warning("NO_HZ: Not enough memory to check full nohz mask\n");
-                return -ENOMEM;
+        if (!have_nohz_full_mask) {
+                if (tick_nohz_init_all() < 0)
+                        return;
         }
 
-        /*
-         * CPUs can probably not be concurrently offlined on initcall time.
-         * But we are paranoid, aren't we?
-         */
-        get_online_cpus();
+        cpu_notifier(tick_nohz_cpu_down_callback, 0);
 
-        /* Ensure we keep a CPU outside the dynticks range for timekeeping */
-        cpumask_and(online_nohz, cpu_online_mask, nohz_full_mask);
-        if (cpumask_equal(online_nohz, cpu_online_mask)) {
-                pr_warning("NO_HZ: Must keep at least one online CPU "
-                           "out of nohz_full range\n");
-                /*
-                 * We know the current CPU doesn't have its tick stopped.
-                 * Let's use it for the timekeeping duty.
-                 */
-                preempt_disable();
-                cpu = smp_processor_id();
-                pr_warning("NO_HZ: Clearing %d from nohz_full range\n", cpu);
-                cpumask_clear_cpu(cpu, nohz_full_mask);
-                preempt_enable();
+        /* Make sure full dynticks CPU are also RCU nocbs */
+        for_each_cpu(cpu, nohz_full_mask) {
+                if (!rcu_is_nocb_cpu(cpu)) {
+                        pr_warning("NO_HZ: CPU %d is not RCU nocb: "
+                                   "cleared from nohz_full range", cpu);
+                        cpumask_clear_cpu(cpu, nohz_full_mask);
+                }
         }
-        put_online_cpus();
-        free_cpumask_var(online_nohz);
 
         cpulist_scnprintf(nohz_full_buf, sizeof(nohz_full_buf), nohz_full_mask);
         pr_info("NO_HZ: Full dynticks CPUs: %s.\n", nohz_full_buf);
-
-        return 0;
 }
-core_initcall(init_tick_nohz_full);
 #else
 #define have_nohz_full_mask (0)
 #endif