1 files changed, 122 insertions, 19 deletions

diff --git a/kernel/softirq.c b/kernel/softirq.c
index 8f03e3b89b55..215541e26c1a 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -62,6 +62,119 @@ static inline void wakeup_softirqd(void)
 }
 
 /*
+ * This one is for softirq.c-internal use,
+ * where hardirqs are disabled legitimately:
+ */
+static void __local_bh_disable(unsigned long ip)
+{
+        unsigned long flags;
+
+        WARN_ON_ONCE(in_irq());
+
+        raw_local_irq_save(flags);
+        add_preempt_count(SOFTIRQ_OFFSET);
+        /*
+         * Were softirqs turned off above:
+         */
+        if (softirq_count() == SOFTIRQ_OFFSET)
+                trace_softirqs_off(ip);
+        raw_local_irq_restore(flags);
+}
+
+void local_bh_disable(void)
+{
+        __local_bh_disable((unsigned long)__builtin_return_address(0));
+}
+
+EXPORT_SYMBOL(local_bh_disable);
+
+void __local_bh_enable(void)
+{
+        WARN_ON_ONCE(in_irq());
+
+        /*
+         * softirqs should never be enabled by __local_bh_enable(),
+         * it always nests inside local_bh_enable() sections:
+         */
+        WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
+
+        sub_preempt_count(SOFTIRQ_OFFSET);
+}
+EXPORT_SYMBOL_GPL(__local_bh_enable);
+
+/*
+ * Special-case - softirqs can safely be enabled in
+ * cond_resched_softirq(), or by __do_softirq(),
+ * without processing still-pending softirqs:
+ */
+void _local_bh_enable(void)
+{
+        WARN_ON_ONCE(in_irq());
+        WARN_ON_ONCE(!irqs_disabled());
+
+        if (softirq_count() == SOFTIRQ_OFFSET)
+                trace_softirqs_on((unsigned long)__builtin_return_address(0));
+        sub_preempt_count(SOFTIRQ_OFFSET);
+}
+
+EXPORT_SYMBOL(_local_bh_enable);
+
+void local_bh_enable(void)
+{
+        unsigned long flags;
+
+        WARN_ON_ONCE(in_irq());
+        WARN_ON_ONCE(irqs_disabled());
+
+        local_irq_save(flags);
+        /*
+         * Are softirqs going to be turned on now:
+         */
+        if (softirq_count() == SOFTIRQ_OFFSET)
+                trace_softirqs_on((unsigned long)__builtin_return_address(0));
+        /*
+         * Keep preemption disabled until we are done with
+         * softirq processing:
+         */
+        sub_preempt_count(SOFTIRQ_OFFSET - 1);
+
+        if (unlikely(!in_interrupt() && local_softirq_pending()))
+                do_softirq();
+
+        dec_preempt_count();
+        local_irq_restore(flags);
+        preempt_check_resched();
+}
+EXPORT_SYMBOL(local_bh_enable);
+
+void local_bh_enable_ip(unsigned long ip)
+{
+        unsigned long flags;
+
+        WARN_ON_ONCE(in_irq());
+
+        local_irq_save(flags);
+        /*
+         * Are softirqs going to be turned on now:
+         */
+        if (softirq_count() == SOFTIRQ_OFFSET)
+                trace_softirqs_on(ip);
+        /*
+         * Keep preemption disabled until we are done with
+         * softirq processing:
+         */
+        sub_preempt_count(SOFTIRQ_OFFSET - 1);
+
+        if (unlikely(!in_interrupt() && local_softirq_pending()))
+                do_softirq();
+
+        dec_preempt_count();
+        local_irq_restore(flags);
+        preempt_check_resched();
+}
+EXPORT_SYMBOL(local_bh_enable_ip);
+
+/*
  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
  * and we fall back to softirqd after that.
  *
@@ -80,8 +193,11 @@ asmlinkage void __do_softirq(void)
         int cpu;
 
         pending = local_softirq_pending();
+        account_system_vtime(current);
+
+        __local_bh_disable((unsigned long)__builtin_return_address(0));
+        trace_softirq_enter();
 
-        local_bh_disable();
         cpu = smp_processor_id();
 restart:
         /* Reset the pending bitmask before enabling irqs */
@@ -109,7 +225,10 @@ restart:
         if (pending)
                 wakeup_softirqd();
 
-        __local_bh_enable();
+        trace_softirq_exit();
+
+        account_system_vtime(current);
+        _local_bh_enable();
 }
 
 #ifndef __ARCH_HAS_DO_SOFTIRQ
@@ -136,23 +255,6 @@ EXPORT_SYMBOL(do_softirq);
 
 #endif
 
-void local_bh_enable(void)
-{
-        WARN_ON(irqs_disabled());
-        /*
-         * Keep preemption disabled until we are done with
-         * softirq processing:
-         */
-        sub_preempt_count(SOFTIRQ_OFFSET - 1);
-
-        if (unlikely(!in_interrupt() && local_softirq_pending()))
-                do_softirq();
-
-        dec_preempt_count();
-        preempt_check_resched();
-}
-EXPORT_SYMBOL(local_bh_enable);
-
 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 # define invoke_softirq()       __do_softirq()
 #else
@@ -165,6 +267,7 @@ EXPORT_SYMBOL(local_bh_enable);
 void irq_exit(void)
 {
         account_system_vtime(current);
+        trace_hardirq_exit();
         sub_preempt_count(IRQ_EXIT_OFFSET);
         if (!in_interrupt() && local_softirq_pending())
                 invoke_softirq();