[PATCH] detect soft lockups

This patch adds a new kernel debug feature: CONFIG_DETECT_SOFTLOCKUP.

When enabled then per-CPU watchdog threads are started, which try to run
once per second.  If they get delayed for more than 10 seconds then a
callback from the timer interrupt detects this condition and prints out a
warning message and a stack dump (once per lockup incident).  The feature
is otherwise non-intrusive, it doesnt try to unlock the box in any way, it
only gets the debug info out, automatically, and on all CPUs affected by
the lockup.

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Nishanth Aravamudan <nacc@us.ibm.com>
Signed-Off-By: Matthias Urlichs <smurf@smurf.noris.de>
Signed-off-by: Richard Purdie <rpurdie@rpsys.net>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

12 files changed, 201 insertions, 0 deletions

diff --git a/arch/i386/kernel/nmi.c b/arch/i386/kernel/nmi.c
index 8bbdbda07a..0178457db7 100644
--- a/arch/i386/kernel/nmi.c
+++ b/arch/i386/kernel/nmi.c
@@ -478,6 +478,11 @@ void touch_nmi_watchdog (void)
          */
         for (i = 0; i < NR_CPUS; i++)
                 alert_counter[i] = 0;
+
+        /*
+         * Tickle the softlockup detector too:
+         */
+        touch_softlockup_watchdog();
 }
 
 extern void die_nmi(struct pt_regs *, const char *msg);
diff --git a/arch/i386/kernel/time.c b/arch/i386/kernel/time.c
index 6f794a78ee..b0c5ee2b34 100644
--- a/arch/i386/kernel/time.c
+++ b/arch/i386/kernel/time.c
@@ -422,6 +422,7 @@ static int timer_resume(struct sys_device *dev)
                 last_timer->resume();
         cur_timer = last_timer;
         last_timer = NULL;
+        touch_softlockup_watchdog();
         return 0;
 }
 
diff --git a/arch/x86_64/kernel/nmi.c b/arch/x86_64/kernel/nmi.c
index 64a8e05d58..84cae81fff 100644
--- a/arch/x86_64/kernel/nmi.c
+++ b/arch/x86_64/kernel/nmi.c
@@ -463,6 +463,8 @@ void touch_nmi_watchdog (void)
          */
         for (i = 0; i < NR_CPUS; i++)
                 per_cpu(nmi_touch, i) = 1;
+
+        touch_softlockup_watchdog();
 }
 
 void nmi_watchdog_tick (struct pt_regs * regs, unsigned reason)
diff --git a/arch/x86_64/kernel/time.c b/arch/x86_64/kernel/time.c
index 66bf6ddeb0..2b5d9da912 100644
--- a/arch/x86_64/kernel/time.c
+++ b/arch/x86_64/kernel/time.c
@@ -1041,6 +1041,7 @@ static int timer_resume(struct sys_device *dev)
         write_sequnlock_irqrestore(&xtime_lock,flags);
         jiffies += sleep_length;
         wall_jiffies += sleep_length;
+        touch_softlockup_watchdog();
         return 0;
 }
 
diff --git a/drivers/mtd/nand/nand_base.c b/drivers/mtd/nand/nand_base.c
index eee5115658..04e54318bc 100644
--- a/drivers/mtd/nand/nand_base.c
+++ b/drivers/mtd/nand/nand_base.c
@@ -526,6 +526,7 @@ static void nand_wait_ready(struct mtd_info *mtd)
         do {
                 if (this->dev_ready(mtd))
                         return;
+                touch_softlockup_watchdog();
         } while (time_before(jiffies, timeo));  
 }
 
diff --git a/include/linux/sched.h b/include/linux/sched.h
index dec5827c77..5fb31bede1 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -176,6 +176,23 @@ extern void trap_init(void);
 extern void update_process_times(int user);
 extern void scheduler_tick(void);
 
+#ifdef CONFIG_DETECT_SOFTLOCKUP
+extern void softlockup_tick(struct pt_regs *regs);
+extern void spawn_softlockup_task(void);
+extern void touch_softlockup_watchdog(void);
+#else
+static inline void softlockup_tick(struct pt_regs *regs)
+{
+}
+static inline void spawn_softlockup_task(void)
+{
+}
+static inline void touch_softlockup_watchdog(void)
+{
+}
+#endif
+
+
 /* Attach to any functions which should be ignored in wchan output. */
 #define __sched         __attribute__((__section__(".sched.text")))
 /* Is this address in the __sched functions? */
diff --git a/init/main.c b/init/main.c
index ff410063e4..a29fb2ac72 100644
--- a/init/main.c
+++ b/init/main.c
@@ -614,6 +614,7 @@ static void do_pre_smp_initcalls(void)
         migration_init();
 #endif
         spawn_ksoftirqd();
+        spawn_softlockup_task();
 }
 
 static void run_init_process(char *init_filename)
diff --git a/kernel/Makefile b/kernel/Makefile
index cb05cd05d2..8d57a2f122 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -27,6 +27,7 @@ obj-$(CONFIG_AUDIT) += audit.o
 obj-$(CONFIG_AUDITSYSCALL) += auditsc.o
 obj-$(CONFIG_KPROBES) += kprobes.o
 obj-$(CONFIG_SYSFS) += ksysfs.o
+obj-$(CONFIG_DETECT_SOFTLOCKUP) += softlockup.o
 obj-$(CONFIG_GENERIC_HARDIRQS) += irq/
 obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
 obj-$(CONFIG_SECCOMP) += seccomp.o
diff --git a/kernel/power/swsusp.c b/kernel/power/swsusp.c
index eaacd5cb58..d967e875ee 100644
--- a/kernel/power/swsusp.c
+++ b/kernel/power/swsusp.c
@@ -1059,6 +1059,7 @@ int swsusp_resume(void)
         BUG_ON(!error);
         restore_processor_state();
         restore_highmem();
+        touch_softlockup_watchdog();
         device_power_up();
         local_irq_enable();
         return error;
diff --git a/kernel/softlockup.c b/kernel/softlockup.c
new file mode 100644
index 0000000000..75976209ce
--- /dev/null
+++ b/kernel/softlockup.c
@@ -0,0 +1,151 @@
+/*
+ * Detect Soft Lockups
+ *
+ * started by Ingo Molnar, (C) 2005, Red Hat
+ *
+ * this code detects soft lockups: incidents in where on a CPU
+ * the kernel does not reschedule for 10 seconds or more.
+ */
+
+#include <linux/mm.h>
+#include <linux/cpu.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+#include <linux/kthread.h>
+#include <linux/notifier.h>
+#include <linux/module.h>
+
+static DEFINE_SPINLOCK(print_lock);
+
+static DEFINE_PER_CPU(unsigned long, timestamp) = 0;
+static DEFINE_PER_CPU(unsigned long, print_timestamp) = 0;
+static DEFINE_PER_CPU(struct task_struct *, watchdog_task);
+
+static int did_panic = 0;
+static int softlock_panic(struct notifier_block *this, unsigned long event,
+                                void *ptr)
+{
+        did_panic = 1;
+
+        return NOTIFY_DONE;
+}
+
+static struct notifier_block panic_block = {
+        .notifier_call = softlock_panic,
+};
+
+void touch_softlockup_watchdog(void)
+{
+        per_cpu(timestamp, raw_smp_processor_id()) = jiffies;
+}
+EXPORT_SYMBOL(touch_softlockup_watchdog);
+
+/*
+ * This callback runs from the timer interrupt, and checks
+ * whether the watchdog thread has hung or not:
+ */
+void softlockup_tick(struct pt_regs *regs)
+{
+        int this_cpu = smp_processor_id();
+        unsigned long timestamp = per_cpu(timestamp, this_cpu);
+
+        if (per_cpu(print_timestamp, this_cpu) == timestamp)
+                return;
+
+        /* Do not cause a second panic when there already was one */
+        if (did_panic)
+                return;
+
+        if (time_after(jiffies, timestamp + 10*HZ)) {
+                per_cpu(print_timestamp, this_cpu) = timestamp;
+
+                spin_lock(&print_lock);
+                printk(KERN_ERR "BUG: soft lockup detected on CPU#%d!\n",
+                        this_cpu);
+                show_regs(regs);
+                spin_unlock(&print_lock);
+        }
+}
+
+/*
+ * The watchdog thread - runs every second and touches the timestamp.
+ */