1 files changed, 83 insertions, 81 deletions

diff --git a/kernel/softirq.c b/kernel/softirq.c
index 11025ccc06dd..490fcbb1dc5b 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -8,6 +8,8 @@
  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/export.h>
 #include <linux/kernel_stat.h>
 #include <linux/interrupt.h>
@@ -54,7 +56,7 @@ static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp
 
 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
 
-char *softirq_to_name[NR_SOFTIRQS] = {
+const char * const softirq_to_name[NR_SOFTIRQS] = {
         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
         "TASKLET", "SCHED", "HRTIMER", "RCU"
 };
@@ -89,7 +91,7 @@ static void wakeup_softirqd(void)
  * where hardirqs are disabled legitimately:
  */
 #ifdef CONFIG_TRACE_IRQFLAGS
-static void __local_bh_disable(unsigned long ip, unsigned int cnt)
+void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
 {
         unsigned long flags;
 
@@ -107,33 +109,21 @@ static void __local_bh_disable(unsigned long ip, unsigned int cnt)
         /*
          * Were softirqs turned off above:
          */
-        if (softirq_count() == cnt)
+        if (softirq_count() == (cnt & SOFTIRQ_MASK))
                 trace_softirqs_off(ip);
         raw_local_irq_restore(flags);
 
         if (preempt_count() == cnt)
                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 }
-#else /* !CONFIG_TRACE_IRQFLAGS */
-static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
-{
-        preempt_count_add(cnt);
-        barrier();
-}
+EXPORT_SYMBOL(__local_bh_disable_ip);
 #endif /* CONFIG_TRACE_IRQFLAGS */
 
-void local_bh_disable(void)
-{
-        __local_bh_disable(_RET_IP_, SOFTIRQ_DISABLE_OFFSET);
-}
-
-EXPORT_SYMBOL(local_bh_disable);
-
 static void __local_bh_enable(unsigned int cnt)
 {
         WARN_ON_ONCE(!irqs_disabled());
 
-        if (softirq_count() == cnt)
+        if (softirq_count() == (cnt & SOFTIRQ_MASK))
                 trace_softirqs_on(_RET_IP_);
         preempt_count_sub(cnt);
 }
@@ -148,10 +138,9 @@ void _local_bh_enable(void)
         WARN_ON_ONCE(in_irq());
         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 }
-
 EXPORT_SYMBOL(_local_bh_enable);
 
-static inline void _local_bh_enable_ip(unsigned long ip)
+void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
 {
         WARN_ON_ONCE(in_irq() || irqs_disabled());
 #ifdef CONFIG_TRACE_IRQFLAGS
@@ -165,8 +154,8 @@ static inline void _local_bh_enable_ip(unsigned long ip)
         /*
          * Keep preemption disabled until we are done with
          * softirq processing:
-         */
-        preempt_count_sub(SOFTIRQ_DISABLE_OFFSET - 1);
+         */
+        preempt_count_sub(cnt - 1);
 
         if (unlikely(!in_interrupt() && local_softirq_pending())) {
                 /*
@@ -182,18 +171,7 @@ static inline void _local_bh_enable_ip(unsigned long ip)
 #endif
         preempt_check_resched();
 }
-
-void local_bh_enable(void)
-{
-        _local_bh_enable_ip(_RET_IP_);
-}
-EXPORT_SYMBOL(local_bh_enable);
-
-void local_bh_enable_ip(unsigned long ip)
-{
-        _local_bh_enable_ip(ip);
-}
-EXPORT_SYMBOL(local_bh_enable_ip);
+EXPORT_SYMBOL(__local_bh_enable_ip);
 
 /*
  * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
@@ -211,14 +189,49 @@ EXPORT_SYMBOL(local_bh_enable_ip);
 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 #define MAX_SOFTIRQ_RESTART 10
 
+#ifdef CONFIG_TRACE_IRQFLAGS
+/*
+ * When we run softirqs from irq_exit() and thus on the hardirq stack we need
+ * to keep the lockdep irq context tracking as tight as possible in order to
+ * not miss-qualify lock contexts and miss possible deadlocks.
+ */
+
+static inline bool lockdep_softirq_start(void)
+{
+        bool in_hardirq = false;
+
+        if (trace_hardirq_context(current)) {
+                in_hardirq = true;
+                trace_hardirq_exit();
+        }
+
+        lockdep_softirq_enter();
+
+        return in_hardirq;
+}
+
+static inline void lockdep_softirq_end(bool in_hardirq)
+{
+        lockdep_softirq_exit();
+
+        if (in_hardirq)
+                trace_hardirq_enter();
+}
+#else
+static inline bool lockdep_softirq_start(void) { return false; }
+static inline void lockdep_softirq_end(bool in_hardirq) { }
+#endif
+
 asmlinkage void __do_softirq(void)
 {
-        struct softirq_action *h;
-        __u32 pending;
         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
-        int cpu;
         unsigned long old_flags = current->flags;
         int max_restart = MAX_SOFTIRQ_RESTART;
+        struct softirq_action *h;
+        bool in_hardirq;
+        __u32 pending;
+        int softirq_bit;
+        int cpu;
 
         /*
          * Mask out PF_MEMALLOC s current task context is borrowed for the
@@ -230,8 +243,8 @@ asmlinkage void __do_softirq(void)
         pending = local_softirq_pending();
         account_irq_enter_time(current);
 
-        __local_bh_disable(_RET_IP_, SOFTIRQ_OFFSET);
-        lockdep_softirq_enter();
+        __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
+        in_hardirq = lockdep_softirq_start();
 
         cpu = smp_processor_id();
 restart:
@@ -242,30 +255,30 @@ restart:
 
         h = softirq_vec;
 
-        do {
-                if (pending & 1) {
-                        unsigned int vec_nr = h - softirq_vec;
-                        int prev_count = preempt_count();
-
-                        kstat_incr_softirqs_this_cpu(vec_nr);
-
-                        trace_softirq_entry(vec_nr);
-                        h->action(h);
-                        trace_softirq_exit(vec_nr);
-                        if (unlikely(prev_count != preempt_count())) {
-                                printk(KERN_ERR "huh, entered softirq %u %s %p"
-                                       "with preempt_count %08x,"
-                                       " exited with %08x?\n", vec_nr,
-                                       softirq_to_name[vec_nr], h->action,
-                                       prev_count, preempt_count());
-                                preempt_count_set(prev_count);
-                        }
+        while ((softirq_bit = ffs(pending))) {
+                unsigned int vec_nr;
+                int prev_count;
+
+                h += softirq_bit - 1;
+
+                vec_nr = h - softirq_vec;
+                prev_count = preempt_count();
+
+                kstat_incr_softirqs_this_cpu(vec_nr);
 
-                        rcu_bh_qs(cpu);
+                trace_softirq_entry(vec_nr);
+                h->action(h);
+                trace_softirq_exit(vec_nr);
+                if (unlikely(prev_count != preempt_count())) {
+                        pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
+                               vec_nr, softirq_to_name[vec_nr], h->action,
+                               prev_count, preempt_count());
+                        preempt_count_set(prev_count);
                 }
+                rcu_bh_qs(cpu);
                 h++;
-                pending >>= 1;
-        } while (pending);
+                pending >>= softirq_bit;
+        }
 
         local_irq_disable();
 
@@ -278,16 +291,13 @@ restart:
                 wakeup_softirqd();
         }
 
-        lockdep_softirq_exit();
-
+        lockdep_softirq_end(in_hardirq);
         account_irq_exit_time(current);
         __local_bh_enable(SOFTIRQ_OFFSET);
         WARN_ON_ONCE(in_interrupt());
         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 }
 
-
-
 asmlinkage void do_softirq(void)
 {
         __u32 pending;
@@ -311,8 +321,6 @@ asmlinkage void do_softirq(void)
  */
 void irq_enter(void)
 {
-        int cpu = smp_processor_id();
-
         rcu_irq_enter();
         if (is_idle_task(current) && !in_interrupt()) {
                 /*
@@ -320,7 +328,7 @@ void irq_enter(void)
                  * here, as softirq will be serviced on return from interrupt.
                  */
                 local_bh_disable();
-                tick_check_idle(cpu);
+                tick_irq_enter();
                 _local_bh_enable();
         }
 
@@ -375,13 +383,13 @@ void irq_exit(void)
 #endif
 
         account_irq_exit_time(current);
-        trace_hardirq_exit();
         preempt_count_sub(HARDIRQ_OFFSET);
         if (!in_interrupt() && local_softirq_pending())
                 invoke_softirq();
 
         tick_irq_exit();
         rcu_irq_exit();
+        trace_hardirq_exit(); /* must be last! */
 }
 
 /*
@@ -427,8 +435,7 @@ void open_softirq(int nr, void (*action)(struct softirq_action *))
 /*
  * Tasklets
  */
-struct tasklet_head
-{
+struct tasklet_head {
         struct tasklet_struct *head;
         struct tasklet_struct **tail;
 };
@@ -447,7 +454,6 @@ void __tasklet_schedule(struct tasklet_struct *t)
         raise_softirq_irqoff(TASKLET_SOFTIRQ);
         local_irq_restore(flags);
 }
-
 EXPORT_SYMBOL(__tasklet_schedule);
 
 void __tasklet_hi_schedule(struct tasklet_struct *t)
@@ -461,7 +467,6 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
         raise_softirq_irqoff(HI_SOFTIRQ);
         local_irq_restore(flags);
 }
-
 EXPORT_SYMBOL(__tasklet_hi_schedule);
 
 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
@@ -472,7 +477,6 @@ void __tasklet_hi_schedule_first(struct tasklet_struct *t)
         __this_cpu_write(tasklet_hi_vec.head, t);
         __raise_softirq_irqoff(HI_SOFTIRQ);
 }
-
 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 
 static void tasklet_action(struct softirq_action *a)
@@ -492,7 +496,8 @@ static void tasklet_action(struct softirq_action *a)
 
                 if (tasklet_trylock(t)) {
                         if (!atomic_read(&t->count)) {
-                                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
+                                if (!test_and_clear_bit(TASKLET_STATE_SCHED,
+                                                        &t->state))
                                         BUG();
                                 t->func(t->data);
                                 tasklet_unlock(t);
@@ -527,7 +532,8 @@ static void tasklet_hi_action(struct softirq_action *a)
 
                 if (tasklet_trylock(t)) {
                         if (!atomic_read(&t->count)) {
-                                if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
+                                if (!test_and_clear_bit(TASKLET_STATE_SCHED,
+                                                        &t->state))
                                         BUG();
                                 t->func(t->data);
                                 tasklet_unlock(t);
@@ -545,7 +551,6 @@ static void tasklet_hi_action(struct softirq_action *a)
         }
 }
 
-
 void tasklet_init(struct tasklet_struct *t,
                   void (*func)(unsigned long), unsigned long data)
 {
@@ -555,13 +560,12 @@ void tasklet_init(struct tasklet_struct *t,
         t->func = func;
         t->data = data;
 }
-
 EXPORT_SYMBOL(tasklet_init);
 
 void tasklet_kill(struct tasklet_struct *t)
 {
         if (in_interrupt())
-                printk("Attempt to kill tasklet from interrupt\n");
+                pr_notice("Attempt to kill tasklet from interrupt\n");
 
         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
                 do {
@@ -571,7 +575,6 @@ void tasklet_kill(struct tasklet_struct *t)
         tasklet_unlock_wait(t);
         clear_bit(TASKLET_STATE_SCHED, &t->state);
 }
-
 EXPORT_SYMBOL(tasklet_kill);
 
 /*
@@ -721,9 +724,8 @@ static void takeover_tasklets(unsigned int cpu)
 }
 #endif /* CONFIG_HOTPLUG_CPU */
 
-static int cpu_callback(struct notifier_block *nfb,
-                                  unsigned long action,
-                                  void *hcpu)
+static int cpu_callback(struct notifier_block *nfb, unsigned long action,
+                        void *hcpu)
 {
         switch (action) {
 #ifdef CONFIG_HOTPLUG_CPU