1 files changed, 243 insertions, 74 deletions

diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index 3174bf8e3538..2316f50b07a4 100644
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -24,8 +24,33 @@
 #include <linux/kvm_para.h>
 #include <linux/perf_event.h>
 
-int watchdog_user_enabled = 1;
+/*
+ * The run state of the lockup detectors is controlled by the content of the
+ * 'watchdog_enabled' variable. Each lockup detector has its dedicated bit -
+ * bit 0 for the hard lockup detector and bit 1 for the soft lockup detector.
+ *
+ * 'watchdog_user_enabled', 'nmi_watchdog_enabled' and 'soft_watchdog_enabled'
+ * are variables that are only used as an 'interface' between the parameters
+ * in /proc/sys/kernel and the internal state bits in 'watchdog_enabled'. The
+ * 'watchdog_thresh' variable is handled differently because its value is not
+ * boolean, and the lockup detectors are 'suspended' while 'watchdog_thresh'
+ * is equal zero.
+ */
+#define NMI_WATCHDOG_ENABLED_BIT   0
+#define SOFT_WATCHDOG_ENABLED_BIT  1
+#define NMI_WATCHDOG_ENABLED      (1 << NMI_WATCHDOG_ENABLED_BIT)
+#define SOFT_WATCHDOG_ENABLED     (1 << SOFT_WATCHDOG_ENABLED_BIT)
+
+#ifdef CONFIG_HARDLOCKUP_DETECTOR
+static unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED|NMI_WATCHDOG_ENABLED;
+#else
+static unsigned long __read_mostly watchdog_enabled = SOFT_WATCHDOG_ENABLED;
+#endif
+int __read_mostly nmi_watchdog_enabled;
+int __read_mostly soft_watchdog_enabled;
+int __read_mostly watchdog_user_enabled;
 int __read_mostly watchdog_thresh = 10;
+
 #ifdef CONFIG_SMP
 int __read_mostly sysctl_softlockup_all_cpu_backtrace;
 #else
@@ -58,8 +83,6 @@ static unsigned long soft_lockup_nmi_warn;
 #ifdef CONFIG_HARDLOCKUP_DETECTOR
 static int hardlockup_panic =
                         CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE;
-
-static bool hardlockup_detector_enabled = true;
 /*
  * We may not want to enable hard lockup detection by default in all cases,
  * for example when running the kernel as a guest on a hypervisor. In these
@@ -68,14 +91,9 @@ static bool hardlockup_detector_enabled = true;
  * kernel command line parameters are parsed, because otherwise it is not
  * possible to override this in hardlockup_panic_setup().
  */
-void watchdog_enable_hardlockup_detector(bool val)
-{
-        hardlockup_detector_enabled = val;
-}
-
-bool watchdog_hardlockup_detector_is_enabled(void)
+void hardlockup_detector_disable(void)
 {
-        return hardlockup_detector_enabled;
+        watchdog_enabled &= ~NMI_WATCHDOG_ENABLED;
 }
 
 static int __init hardlockup_panic_setup(char *str)
@@ -85,15 +103,9 @@ static int __init hardlockup_panic_setup(char *str)
         else if (!strncmp(str, "nopanic", 7))
                 hardlockup_panic = 0;
         else if (!strncmp(str, "0", 1))
-                watchdog_user_enabled = 0;
-        else if (!strncmp(str, "1", 1) || !strncmp(str, "2", 1)) {
-                /*
-                 * Setting 'nmi_watchdog=1' or 'nmi_watchdog=2' (legacy option)
-                 * has the same effect.
-                 */
-                watchdog_user_enabled = 1;
-                watchdog_enable_hardlockup_detector(true);
-        }
+                watchdog_enabled &= ~NMI_WATCHDOG_ENABLED;
+        else if (!strncmp(str, "1", 1))
+                watchdog_enabled |= NMI_WATCHDOG_ENABLED;
         return 1;
 }
 __setup("nmi_watchdog=", hardlockup_panic_setup);
@@ -112,19 +124,18 @@ __setup("softlockup_panic=", softlockup_panic_setup);
 
 static int __init nowatchdog_setup(char *str)
 {
-        watchdog_user_enabled = 0;
+        watchdog_enabled = 0;
         return 1;
 }
 __setup("nowatchdog", nowatchdog_setup);
 
-/* deprecated */
 static int __init nosoftlockup_setup(char *str)
 {
-        watchdog_user_enabled = 0;
+        watchdog_enabled &= ~SOFT_WATCHDOG_ENABLED;
         return 1;
 }
 __setup("nosoftlockup", nosoftlockup_setup);
-/*  */
+
 #ifdef CONFIG_SMP
 static int __init softlockup_all_cpu_backtrace_setup(char *str)
 {
@@ -239,10 +250,11 @@ static int is_softlockup(unsigned long touch_ts)
 {
         unsigned long now = get_timestamp();
 
-        /* Warn about unreasonable delays: */
-        if (time_after(now, touch_ts + get_softlockup_thresh()))
-                return now - touch_ts;
-
+        if (watchdog_enabled & SOFT_WATCHDOG_ENABLED) {
+                /* Warn about unreasonable delays. */
+                if (time_after(now, touch_ts + get_softlockup_thresh()))
+                        return now - touch_ts;
+        }
         return 0;
 }
 
@@ -477,6 +489,21 @@ static void watchdog(unsigned int cpu)
         __this_cpu_write(soft_lockup_hrtimer_cnt,
                          __this_cpu_read(hrtimer_interrupts));
         __touch_watchdog();
+
+        /*
+         * watchdog_nmi_enable() clears the NMI_WATCHDOG_ENABLED bit in the
+         * failure path. Check for failures that can occur asynchronously -
+         * for example, when CPUs are on-lined - and shut down the hardware
+         * perf event on each CPU accordingly.
+         *
+         * The only non-obvious place this bit can be cleared is through
+         * watchdog_nmi_enable(), so a pr_info() is placed there.  Placing a
+         * pr_info here would be too noisy as it would result in a message
+         * every few seconds if the hardlockup was disabled but the softlockup
+         * enabled.
+         */
+        if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED))
+                watchdog_nmi_disable(cpu);
 }
 
 #ifdef CONFIG_HARDLOCKUP_DETECTOR
@@ -492,14 +519,9 @@ static int watchdog_nmi_enable(unsigned int cpu)
         struct perf_event_attr *wd_attr;
         struct perf_event *event = per_cpu(watchdog_ev, cpu);
 
-        /*
-         * Some kernels need to default hard lockup detection to
-         * 'disabled', for example a guest on a hypervisor.
-         */
-        if (!watchdog_hardlockup_detector_is_enabled()) {
-                event = ERR_PTR(-ENOENT);
-                goto handle_err;
-        }
+        /* nothing to do if the hard lockup detector is disabled */
+        if (!(watchdog_enabled & NMI_WATCHDOG_ENABLED))
+                goto out;
 
         /* is it already setup and enabled? */
         if (event && event->state > PERF_EVENT_STATE_OFF)
@@ -515,7 +537,6 @@ static int watchdog_nmi_enable(unsigned int cpu)
         /* Try to register using hardware perf events */
         event = perf_event_create_kernel_counter(wd_attr, cpu, NULL, watchdog_overflow_callback, NULL);
 
-handle_err:
         /* save cpu0 error for future comparision */
         if (cpu == 0 && IS_ERR(event))
                 cpu0_err = PTR_ERR(event);
@@ -527,6 +548,18 @@ handle_err:
                 goto out_save;
         }
 
+        /*
+         * Disable the hard lockup detector if _any_ CPU fails to set up
+         * set up the hardware perf event. The watchdog() function checks
+         * the NMI_WATCHDOG_ENABLED bit periodically.
+         *
+         * The barriers are for syncing up watchdog_enabled across all the
+         * cpus, as clear_bit() does not use barriers.
+         */
+        smp_mb__before_atomic();
+        clear_bit(NMI_WATCHDOG_ENABLED_BIT, &watchdog_enabled);
+        smp_mb__after_atomic();
+
         /* skip displaying the same error again */
         if (cpu > 0 && (PTR_ERR(event) == cpu0_err))
                 return PTR_ERR(event);
@@ -540,6 +573,9 @@ handle_err:
         else
                 pr_err("disabled (cpu%i): unable to create perf event: %ld\n",
                         cpu, PTR_ERR(event));
+
+        pr_info("Shutting down hard lockup detector on all cpus\n");
+
         return PTR_ERR(event);
 
         /* success path */
@@ -567,9 +603,37 @@ static void watchdog_nmi_disable(unsigned int cpu)
                 cpu0_err = 0;
         }
 }
+
+void watchdog_nmi_enable_all(void)
+{
+        int cpu;
+
+        if (!watchdog_user_enabled)
+                return;
+
+        get_online_cpus();
+        for_each_online_cpu(cpu)
+                watchdog_nmi_enable(cpu);
+        put_online_cpus();
+}
+
+void watchdog_nmi_disable_all(void)
+{
+        int cpu;
+
+        if (!watchdog_running)
+                return;
+
+        get_online_cpus();
+        for_each_online_cpu(cpu)
+                watchdog_nmi_disable(cpu);
+        put_online_cpus();
+}
 #else
 static int watchdog_nmi_enable(unsigned int cpu) { return 0; }
 static void watchdog_nmi_disable(unsigned int cpu) { return; }
+void watchdog_nmi_enable_all(void) {}
+void watchdog_nmi_disable_all(void) {}
 #endif /* CONFIG_HARDLOCKUP_DETECTOR */
 
 static struct smp_hotplug_thread watchdog_threads = {
@@ -600,7 +664,7 @@ static void restart_watchdog_hrtimer(void *info)
                                 HRTIMER_MODE_REL_PINNED);
 }
 
-static void update_timers(int cpu)
+static void update_watchdog(int cpu)
 {
         /*
          * Make sure that perf event counter will adopt to a new
@@ -615,17 +679,17 @@ static void update_timers(int cpu)
         watchdog_nmi_enable(cpu);
 }
 
-static void update_timers_all_cpus(void)
+static void update_watchdog_all_cpus(void)
 {
         int cpu;
 
         get_online_cpus();
         for_each_online_cpu(cpu)
-                update_timers(cpu);
+                update_watchdog(cpu);
         put_online_cpus();
 }
 
-static int watchdog_enable_all_cpus(bool sample_period_changed)
+static int watchdog_enable_all_cpus(void)
 {
         int err = 0;
 
@@ -635,8 +699,12 @@ static int watchdog_enable_all_cpus(bool sample_period_changed)
                         pr_err("Failed to create watchdog threads, disabled\n");
                 else
                         watchdog_running = 1;
-        } else if (sample_period_changed) {
-                update_timers_all_cpus();
+        } else {
+                /*
+                 * Enable/disable the lockup detectors or
+                 * change the sample period 'on the fly'.
+                 */
+                update_watchdog_all_cpus();
         }
 
         return err;
@@ -654,58 +722,159 @@ static void watchdog_disable_all_cpus(void)
 }
 
 /*
- * proc handler for /proc/sys/kernel/nmi_watchdog,watchdog_thresh
+ * Update the run state of the lockup detectors.
  */
+static int proc_watchdog_update(void)
+{
+        int err = 0;
 
-int proc_dowatchdog(struct ctl_table *table, int write,
-                    void __user *buffer, size_t *lenp, loff_t *ppos)
+        /*
+         * Watchdog threads won't be started if they are already active.
+         * The 'watchdog_running' variable in watchdog_*_all_cpus() takes
+         * care of this. If those threads are already active, the sample
+         * period will be updated and the lockup detectors will be enabled
+         * or disabled 'on the fly'.
+         */
+        if (watchdog_enabled && watchdog_thresh)
+                err = watchdog_enable_all_cpus();
+        else
+                watchdog_disable_all_cpus();
+
+        return err;
+
+}
+
+static DEFINE_MUTEX(watchdog_proc_mutex);
+
+/*
+ * common function for watchdog, nmi_watchdog and soft_watchdog parameter
+ *
+ * caller             | table->data points to | 'which' contains the flag(s)
+ * -------------------|-----------------------|-----------------------------
+ * proc_watchdog      | watchdog_user_enabled | NMI_WATCHDOG_ENABLED or'ed
+ *                    |                       | with SOFT_WATCHDOG_ENABLED
+ * -------------------|-----------------------|-----------------------------
+ * proc_nmi_watchdog  | nmi_watchdog_enabled  | NMI_WATCHDOG_ENABLED
+ * -------------------|-----------------------|-----------------------------
+ * proc_soft_watchdog | soft_watchdog_enabled | SOFT_WATCHDOG_ENABLED
+ */
+static int proc_watchdog_common(int which, struct ctl_table *table, int write,
+                                void __user *buffer, size_t *lenp, loff_t *ppos)
 {
-        int err, old_thresh, old_enabled;
-        bool old_hardlockup;
-        static DEFINE_MUTEX(watchdog_proc_mutex);
+        int err, old, new;
+        int *watchdog_param = (int *)table->data;
 
         mutex_lock(&watchdog_proc_mutex);
-        old_thresh = ACCESS_ONCE(watchdog_thresh);
-        old_enabled = ACCESS_ONCE(watchdog_user_enabled);
-        old_hardlockup = watchdog_hardlockup_detector_is_enabled();
 
-        err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
-        if (err || !write)
-                goto out;
-
-        set_sample_period();
         /*
-         * Watchdog threads shouldn't be enabled if they are
-         * disabled. The 'watchdog_running' variable check in
-         * watchdog_*_all_cpus() function takes care of this.
+         * If the parameter is being read return the state of the corresponding
+         * bit(s) in 'watchdog_enabled', else update 'watchdog_enabled' and the
+         * run state of the lockup detectors.
          */
-        if (watchdog_user_enabled && watchdog_thresh) {
+        if (!write) {
+                *watchdog_param = (watchdog_enabled & which) != 0;
+                err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+        } else {
+                err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+                if (err)
+                        goto out;
+
                 /*
-                 * Prevent a change in watchdog_thresh accidentally overriding
-                 * the enablement of the hardlockup detector.
+                 * There is a race window between fetching the current value
+                 * from 'watchdog_enabled' and storing the new value. During
+                 * this race window, watchdog_nmi_enable() can sneak in and
+                 * clear the NMI_WATCHDOG_ENABLED bit in 'watchdog_enabled'.
+                 * The 'cmpxchg' detects this race and the loop retries.
                  */
-                if (watchdog_user_enabled != old_enabled)
-                        watchdog_enable_hardlockup_detector(true);
-                err = watchdog_enable_all_cpus(old_thresh != watchdog_thresh);
-        } else
-                watchdog_disable_all_cpus();
+                do {
+                        old = watchdog_enabled;
+                        /*
+                         * If the parameter value is not zero set the
+                         * corresponding bit(s), else clear it(them).
+                         */
+                        if (*watchdog_param)
+                                new = old | which;
+                        else
+                                new = old & ~which;
+                } while (cmpxchg(&watchdog_enabled, old, new) != old);
 
-        /* Restore old values on failure */
-        if (err) {
-                watchdog_thresh = old_thresh;
-                watchdog_user_enabled = old_enabled;
-                watchdog_enable_hardlockup_detector(old_hardlockup);
+                /*
+                 * Update the run state of the lockup detectors.
+                 * Restore 'watchdog_enabled' on failure.
+                 */
+                err = proc_watchdog_update();
+                if (err)
+                        watchdog_enabled = old;
         }
 out:
         mutex_unlock(&watchdog_proc_mutex);
         return err;
 }
+
+/*
+ * /proc/sys/kernel/watchdog
+ */
+int proc_watchdog(struct ctl_table *table, int write,
+                  void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+        return proc_watchdog_common(NMI_WATCHDOG_ENABLED|SOFT_WATCHDOG_ENABLED,
+                                    table, write, buffer, lenp, ppos);
+}
+
+/*
+ * /proc/sys/kernel/nmi_watchdog
+ */
+int proc_nmi_watchdog(struct ctl_table *table, int write,
+                      void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+        return proc_watchdog_common(NMI_WATCHDOG_ENABLED,
+                                    table, write, buffer, lenp, ppos);
+}
+
+/*
+ * /proc/sys/kernel/soft_watchdog
+ */
+int proc_soft_watchdog(struct ctl_table *table, int write,
+                        void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+        return proc_watchdog_common(SOFT_WATCHDOG_ENABLED,
+                                    table, write, buffer, lenp, ppos);
+}
+
+/*
+ * /proc/sys/kernel/watchdog_thresh
+ */
+int proc_watchdog_thresh(struct ctl_table *table, int write,
+                         void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+        int err, old;
+
+        mutex_lock(&watchdog_proc_mutex);
+
+        old = ACCESS_ONCE(watchdog_thresh);
+        err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+
+        if (err || !write)
+                goto out;
+
+        /*
+         * Update the sample period.
+         * Restore 'watchdog_thresh' on failure.
+         */
+        set_sample_period();
+        err = proc_watchdog_update();
+        if (err)
+                watchdog_thresh = old;
+out:
+        mutex_unlock(&watchdog_proc_mutex);
+        return err;
+}
 #endif /* CONFIG_SYSCTL */
 
 void __init lockup_detector_init(void)
 {
         set_sample_period();
 
-        if (watchdog_user_enabled)
-                watchdog_enable_all_cpus(false);
+        if (watchdog_enabled)
+                watchdog_enable_all_cpus();
 }