1 files changed, 41 insertions, 46 deletions

diff --git a/kernel/softirq.c b/kernel/softirq.c
index d4d918a91881..56e5dec837f0 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -54,7 +54,7 @@ EXPORT_SYMBOL(irq_stat);
 
 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
 
-static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
+DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
 
 char *softirq_to_name[NR_SOFTIRQS] = {
         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
@@ -70,7 +70,7 @@ char *softirq_to_name[NR_SOFTIRQS] = {
 static void wakeup_softirqd(void)
 {
         /* Interrupts are disabled: no need to stop preemption */
-        struct task_struct *tsk = __get_cpu_var(ksoftirqd);
+        struct task_struct *tsk = __this_cpu_read(ksoftirqd);
 
         if (tsk && tsk->state != TASK_RUNNING)
                 wake_up_process(tsk);
@@ -311,9 +311,21 @@ void irq_enter(void)
 }
 
 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
-# define invoke_softirq()       __do_softirq()
+static inline void invoke_softirq(void)
+{
+        if (!force_irqthreads)
+                __do_softirq();
+        else
+                wakeup_softirqd();
+}
 #else
-# define invoke_softirq()       do_softirq()
+static inline void invoke_softirq(void)
+{
+        if (!force_irqthreads)
+                do_softirq();
+        else
+                wakeup_softirqd();
+}
 #endif
 
 /*
@@ -388,8 +400,8 @@ void __tasklet_schedule(struct tasklet_struct *t)
 
         local_irq_save(flags);
         t->next = NULL;
-        *__get_cpu_var(tasklet_vec).tail = t;
-        __get_cpu_var(tasklet_vec).tail = &(t->next);
+        *__this_cpu_read(tasklet_vec.tail) = t;
+        __this_cpu_write(tasklet_vec.tail, &(t->next));
         raise_softirq_irqoff(TASKLET_SOFTIRQ);
         local_irq_restore(flags);
 }
@@ -402,8 +414,8 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
 
         local_irq_save(flags);
         t->next = NULL;
-        *__get_cpu_var(tasklet_hi_vec).tail = t;
-        __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
+        *__this_cpu_read(tasklet_hi_vec.tail) = t;
+        __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
         raise_softirq_irqoff(HI_SOFTIRQ);
         local_irq_restore(flags);
 }
@@ -414,8 +426,8 @@ void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 {
         BUG_ON(!irqs_disabled());
 
-        t->next = __get_cpu_var(tasklet_hi_vec).head;
-        __get_cpu_var(tasklet_hi_vec).head = t;
+        t->next = __this_cpu_read(tasklet_hi_vec.head);
+        __this_cpu_write(tasklet_hi_vec.head, t);
         __raise_softirq_irqoff(HI_SOFTIRQ);
 }
 
@@ -426,9 +438,9 @@ static void tasklet_action(struct softirq_action *a)
         struct tasklet_struct *list;
 
         local_irq_disable();
-        list = __get_cpu_var(tasklet_vec).head;
-        __get_cpu_var(tasklet_vec).head = NULL;
-        __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
+        list = __this_cpu_read(tasklet_vec.head);
+        __this_cpu_write(tasklet_vec.head, NULL);
+        __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
         local_irq_enable();
 
         while (list) {
@@ -449,8 +461,8 @@ static void tasklet_action(struct softirq_action *a)
 
                 local_irq_disable();
                 t->next = NULL;
-                *__get_cpu_var(tasklet_vec).tail = t;
-                __get_cpu_var(tasklet_vec).tail = &(t->next);
+                *__this_cpu_read(tasklet_vec.tail) = t;
+                __this_cpu_write(tasklet_vec.tail, &(t->next));
                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
                 local_irq_enable();
         }
@@ -461,9 +473,9 @@ static void tasklet_hi_action(struct softirq_action *a)
         struct tasklet_struct *list;
 
         local_irq_disable();
-        list = __get_cpu_var(tasklet_hi_vec).head;
-        __get_cpu_var(tasklet_hi_vec).head = NULL;
-        __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
+        list = __this_cpu_read(tasklet_hi_vec.head);
+        __this_cpu_write(tasklet_hi_vec.head, NULL);
+        __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
         local_irq_enable();
 
         while (list) {
@@ -484,8 +496,8 @@ static void tasklet_hi_action(struct softirq_action *a)
 
                 local_irq_disable();
                 t->next = NULL;
-                *__get_cpu_var(tasklet_hi_vec).tail = t;
-                __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
+                *__this_cpu_read(tasklet_hi_vec.tail) = t;
+                __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
                 __raise_softirq_irqoff(HI_SOFTIRQ);
                 local_irq_enable();
         }
@@ -721,7 +733,6 @@ static int run_ksoftirqd(void * __bind_cpu)
 {
         set_current_state(TASK_INTERRUPTIBLE);
 
-        current->flags |= PF_KSOFTIRQD;
         while (!kthread_should_stop()) {
                 preempt_disable();
                 if (!local_softirq_pending()) {
@@ -738,7 +749,10 @@ static int run_ksoftirqd(void * __bind_cpu)
                            don't process */
                         if (cpu_is_offline((long)__bind_cpu))
                                 goto wait_to_die;
-                        do_softirq();
+                        local_irq_disable();
+                        if (local_softirq_pending())
+                                __do_softirq();
+                        local_irq_enable();
                         preempt_enable_no_resched();
                         cond_resched();
                         preempt_disable();
@@ -802,16 +816,16 @@ static void takeover_tasklets(unsigned int cpu)
 
         /* Find end, append list for that CPU. */
         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
-                *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
-                __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
+                *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
+                this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
                 per_cpu(tasklet_vec, cpu).head = NULL;
                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
         }
         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 
         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
-                *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
-                __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
+                *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
+                __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
         }
@@ -853,7 +867,7 @@ static int __cpuinit cpu_callback(struct notifier_block *nfb,
                              cpumask_any(cpu_online_mask));
         case CPU_DEAD:
         case CPU_DEAD_FROZEN: {
-                static struct sched_param param = {
+                static const struct sched_param param = {
                         .sched_priority = MAX_RT_PRIO-1
                 };
 
@@ -885,25 +899,6 @@ static __init int spawn_ksoftirqd(void)
 }
 early_initcall(spawn_ksoftirqd);
 
-#ifdef CONFIG_SMP
-/*
- * Call a function on all processors
- */
-int on_each_cpu(void (*func) (void *info), void *info, int wait)
-{
-        int ret = 0;
-
-        preempt_disable();
-        ret = smp_call_function(func, info, wait);
-        local_irq_disable();
-        func(info);
-        local_irq_enable();
-        preempt_enable();
-        return ret;
-}
-EXPORT_SYMBOL(on_each_cpu);
-#endif
-
 /*
  * [ These __weak aliases are kept in a separate compilation unit, so that
  *   GCC does not inline them incorrectly. ]