aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/softirq.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/softirq.c')
-rw-r--r--kernel/softirq.c72
1 files changed, 34 insertions, 38 deletions
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 8a1e6e104892..850967068aaf 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -8,6 +8,8 @@
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903) 8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 */ 9 */
10 10
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
11#include <linux/export.h> 13#include <linux/export.h>
12#include <linux/kernel_stat.h> 14#include <linux/kernel_stat.h>
13#include <linux/interrupt.h> 15#include <linux/interrupt.h>
@@ -54,7 +56,7 @@ static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp
54 56
55DEFINE_PER_CPU(struct task_struct *, ksoftirqd); 57DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
56 58
57char *softirq_to_name[NR_SOFTIRQS] = { 59const char * const softirq_to_name[NR_SOFTIRQS] = {
58 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL", 60 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
59 "TASKLET", "SCHED", "HRTIMER", "RCU" 61 "TASKLET", "SCHED", "HRTIMER", "RCU"
60}; 62};
@@ -136,7 +138,6 @@ void _local_bh_enable(void)
136 WARN_ON_ONCE(in_irq()); 138 WARN_ON_ONCE(in_irq());
137 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET); 139 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
138} 140}
139
140EXPORT_SYMBOL(_local_bh_enable); 141EXPORT_SYMBOL(_local_bh_enable);
141 142
142void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) 143void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
@@ -153,7 +154,7 @@ void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
153 /* 154 /*
154 * Keep preemption disabled until we are done with 155 * Keep preemption disabled until we are done with
155 * softirq processing: 156 * softirq processing:
156 */ 157 */
157 preempt_count_sub(cnt - 1); 158 preempt_count_sub(cnt - 1);
158 159
159 if (unlikely(!in_interrupt() && local_softirq_pending())) { 160 if (unlikely(!in_interrupt() && local_softirq_pending())) {
@@ -229,6 +230,7 @@ asmlinkage void __do_softirq(void)
229 struct softirq_action *h; 230 struct softirq_action *h;
230 bool in_hardirq; 231 bool in_hardirq;
231 __u32 pending; 232 __u32 pending;
233 int softirq_bit;
232 int cpu; 234 int cpu;
233 235
234 /* 236 /*
@@ -253,30 +255,30 @@ restart:
253 255
254 h = softirq_vec; 256 h = softirq_vec;
255 257
256 do { 258 while ((softirq_bit = ffs(pending))) {
257 if (pending & 1) { 259 unsigned int vec_nr;
258 unsigned int vec_nr = h - softirq_vec; 260 int prev_count;
259 int prev_count = preempt_count();
260
261 kstat_incr_softirqs_this_cpu(vec_nr);
262
263 trace_softirq_entry(vec_nr);
264 h->action(h);
265 trace_softirq_exit(vec_nr);
266 if (unlikely(prev_count != preempt_count())) {
267 printk(KERN_ERR "huh, entered softirq %u %s %p"
268 "with preempt_count %08x,"
269 " exited with %08x?\n", vec_nr,
270 softirq_to_name[vec_nr], h->action,
271 prev_count, preempt_count());
272 preempt_count_set(prev_count);
273 }
274 261
275 rcu_bh_qs(cpu); 262 h += softirq_bit - 1;
263
264 vec_nr = h - softirq_vec;
265 prev_count = preempt_count();
266
267 kstat_incr_softirqs_this_cpu(vec_nr);
268
269 trace_softirq_entry(vec_nr);
270 h->action(h);
271 trace_softirq_exit(vec_nr);
272 if (unlikely(prev_count != preempt_count())) {
273 pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
274 vec_nr, softirq_to_name[vec_nr], h->action,
275 prev_count, preempt_count());
276 preempt_count_set(prev_count);
276 } 277 }
278 rcu_bh_qs(cpu);
277 h++; 279 h++;
278 pending >>= 1; 280 pending >>= softirq_bit;
279 } while (pending); 281 }
280 282
281 local_irq_disable(); 283 local_irq_disable();
282 284
@@ -433,8 +435,7 @@ void open_softirq(int nr, void (*action)(struct softirq_action *))
433/* 435/*
434 * Tasklets 436 * Tasklets
435 */ 437 */
436struct tasklet_head 438struct tasklet_head {
437{
438 struct tasklet_struct *head; 439 struct tasklet_struct *head;
439 struct tasklet_struct **tail; 440 struct tasklet_struct **tail;
440}; 441};
@@ -453,7 +454,6 @@ void __tasklet_schedule(struct tasklet_struct *t)
453 raise_softirq_irqoff(TASKLET_SOFTIRQ); 454 raise_softirq_irqoff(TASKLET_SOFTIRQ);
454 local_irq_restore(flags); 455 local_irq_restore(flags);
455} 456}
456
457EXPORT_SYMBOL(__tasklet_schedule); 457EXPORT_SYMBOL(__tasklet_schedule);
458 458
459void __tasklet_hi_schedule(struct tasklet_struct *t) 459void __tasklet_hi_schedule(struct tasklet_struct *t)
@@ -467,7 +467,6 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
467 raise_softirq_irqoff(HI_SOFTIRQ); 467 raise_softirq_irqoff(HI_SOFTIRQ);
468 local_irq_restore(flags); 468 local_irq_restore(flags);
469} 469}
470
471EXPORT_SYMBOL(__tasklet_hi_schedule); 470EXPORT_SYMBOL(__tasklet_hi_schedule);
472 471
473void __tasklet_hi_schedule_first(struct tasklet_struct *t) 472void __tasklet_hi_schedule_first(struct tasklet_struct *t)
@@ -478,7 +477,6 @@ void __tasklet_hi_schedule_first(struct tasklet_struct *t)
478 __this_cpu_write(tasklet_hi_vec.head, t); 477 __this_cpu_write(tasklet_hi_vec.head, t);
479 __raise_softirq_irqoff(HI_SOFTIRQ); 478 __raise_softirq_irqoff(HI_SOFTIRQ);
480} 479}
481
482EXPORT_SYMBOL(__tasklet_hi_schedule_first); 480EXPORT_SYMBOL(__tasklet_hi_schedule_first);
483 481
484static void tasklet_action(struct softirq_action *a) 482static void tasklet_action(struct softirq_action *a)
@@ -498,7 +496,8 @@ static void tasklet_action(struct softirq_action *a)
498 496
499 if (tasklet_trylock(t)) { 497 if (tasklet_trylock(t)) {
500 if (!atomic_read(&t->count)) { 498 if (!atomic_read(&t->count)) {
501 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) 499 if (!test_and_clear_bit(TASKLET_STATE_SCHED,
500 &t->state))
502 BUG(); 501 BUG();
503 t->func(t->data); 502 t->func(t->data);
504 tasklet_unlock(t); 503 tasklet_unlock(t);
@@ -533,7 +532,8 @@ static void tasklet_hi_action(struct softirq_action *a)
533 532
534 if (tasklet_trylock(t)) { 533 if (tasklet_trylock(t)) {
535 if (!atomic_read(&t->count)) { 534 if (!atomic_read(&t->count)) {
536 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) 535 if (!test_and_clear_bit(TASKLET_STATE_SCHED,
536 &t->state))
537 BUG(); 537 BUG();
538 t->func(t->data); 538 t->func(t->data);
539 tasklet_unlock(t); 539 tasklet_unlock(t);
@@ -551,7 +551,6 @@ static void tasklet_hi_action(struct softirq_action *a)
551 } 551 }
552} 552}
553 553
554
555void tasklet_init(struct tasklet_struct *t, 554void tasklet_init(struct tasklet_struct *t,
556 void (*func)(unsigned long), unsigned long data) 555 void (*func)(unsigned long), unsigned long data)
557{ 556{
@@ -561,13 +560,12 @@ void tasklet_init(struct tasklet_struct *t,
561 t->func = func; 560 t->func = func;
562 t->data = data; 561 t->data = data;
563} 562}
564
565EXPORT_SYMBOL(tasklet_init); 563EXPORT_SYMBOL(tasklet_init);
566 564
567void tasklet_kill(struct tasklet_struct *t) 565void tasklet_kill(struct tasklet_struct *t)
568{ 566{
569 if (in_interrupt()) 567 if (in_interrupt())
570 printk("Attempt to kill tasklet from interrupt\n"); 568 pr_notice("Attempt to kill tasklet from interrupt\n");
571 569
572 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { 570 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
573 do { 571 do {
@@ -577,7 +575,6 @@ void tasklet_kill(struct tasklet_struct *t)
577 tasklet_unlock_wait(t); 575 tasklet_unlock_wait(t);
578 clear_bit(TASKLET_STATE_SCHED, &t->state); 576 clear_bit(TASKLET_STATE_SCHED, &t->state);
579} 577}
580
581EXPORT_SYMBOL(tasklet_kill); 578EXPORT_SYMBOL(tasklet_kill);
582 579
583/* 580/*
@@ -727,9 +724,8 @@ static void takeover_tasklets(unsigned int cpu)
727} 724}
728#endif /* CONFIG_HOTPLUG_CPU */ 725#endif /* CONFIG_HOTPLUG_CPU */
729 726
730static int cpu_callback(struct notifier_block *nfb, 727static int cpu_callback(struct notifier_block *nfb, unsigned long action,
731 unsigned long action, 728 void *hcpu)
732 void *hcpu)
733{ 729{
734 switch (action) { 730 switch (action) {
735#ifdef CONFIG_HOTPLUG_CPU 731#ifdef CONFIG_HOTPLUG_CPU