diff options
Diffstat (limited to 'kernel/softirq.c')
-rw-r--r-- | kernel/softirq.c | 72 |
1 files changed, 34 insertions, 38 deletions
diff --git a/kernel/softirq.c b/kernel/softirq.c index 8a1e6e104892..850967068aaf 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c | |||
@@ -8,6 +8,8 @@ | |||
8 | * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903) | 8 | * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903) |
9 | */ | 9 | */ |
10 | 10 | ||
11 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | ||
12 | |||
11 | #include <linux/export.h> | 13 | #include <linux/export.h> |
12 | #include <linux/kernel_stat.h> | 14 | #include <linux/kernel_stat.h> |
13 | #include <linux/interrupt.h> | 15 | #include <linux/interrupt.h> |
@@ -54,7 +56,7 @@ static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp | |||
54 | 56 | ||
55 | DEFINE_PER_CPU(struct task_struct *, ksoftirqd); | 57 | DEFINE_PER_CPU(struct task_struct *, ksoftirqd); |
56 | 58 | ||
57 | char *softirq_to_name[NR_SOFTIRQS] = { | 59 | const char * const softirq_to_name[NR_SOFTIRQS] = { |
58 | "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL", | 60 | "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL", |
59 | "TASKLET", "SCHED", "HRTIMER", "RCU" | 61 | "TASKLET", "SCHED", "HRTIMER", "RCU" |
60 | }; | 62 | }; |
@@ -136,7 +138,6 @@ void _local_bh_enable(void) | |||
136 | WARN_ON_ONCE(in_irq()); | 138 | WARN_ON_ONCE(in_irq()); |
137 | __local_bh_enable(SOFTIRQ_DISABLE_OFFSET); | 139 | __local_bh_enable(SOFTIRQ_DISABLE_OFFSET); |
138 | } | 140 | } |
139 | |||
140 | EXPORT_SYMBOL(_local_bh_enable); | 141 | EXPORT_SYMBOL(_local_bh_enable); |
141 | 142 | ||
142 | void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) | 143 | void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) |
@@ -153,7 +154,7 @@ void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) | |||
153 | /* | 154 | /* |
154 | * Keep preemption disabled until we are done with | 155 | * Keep preemption disabled until we are done with |
155 | * softirq processing: | 156 | * softirq processing: |
156 | */ | 157 | */ |
157 | preempt_count_sub(cnt - 1); | 158 | preempt_count_sub(cnt - 1); |
158 | 159 | ||
159 | if (unlikely(!in_interrupt() && local_softirq_pending())) { | 160 | if (unlikely(!in_interrupt() && local_softirq_pending())) { |
@@ -229,6 +230,7 @@ asmlinkage void __do_softirq(void) | |||
229 | struct softirq_action *h; | 230 | struct softirq_action *h; |
230 | bool in_hardirq; | 231 | bool in_hardirq; |
231 | __u32 pending; | 232 | __u32 pending; |
233 | int softirq_bit; | ||
232 | int cpu; | 234 | int cpu; |
233 | 235 | ||
234 | /* | 236 | /* |
@@ -253,30 +255,30 @@ restart: | |||
253 | 255 | ||
254 | h = softirq_vec; | 256 | h = softirq_vec; |
255 | 257 | ||
256 | do { | 258 | while ((softirq_bit = ffs(pending))) { |
257 | if (pending & 1) { | 259 | unsigned int vec_nr; |
258 | unsigned int vec_nr = h - softirq_vec; | 260 | int prev_count; |
259 | int prev_count = preempt_count(); | ||
260 | |||
261 | kstat_incr_softirqs_this_cpu(vec_nr); | ||
262 | |||
263 | trace_softirq_entry(vec_nr); | ||
264 | h->action(h); | ||
265 | trace_softirq_exit(vec_nr); | ||
266 | if (unlikely(prev_count != preempt_count())) { | ||
267 | printk(KERN_ERR "huh, entered softirq %u %s %p" | ||
268 | "with preempt_count %08x," | ||
269 | " exited with %08x?\n", vec_nr, | ||
270 | softirq_to_name[vec_nr], h->action, | ||
271 | prev_count, preempt_count()); | ||
272 | preempt_count_set(prev_count); | ||
273 | } | ||
274 | 261 | ||
275 | rcu_bh_qs(cpu); | 262 | h += softirq_bit - 1; |
263 | |||
264 | vec_nr = h - softirq_vec; | ||
265 | prev_count = preempt_count(); | ||
266 | |||
267 | kstat_incr_softirqs_this_cpu(vec_nr); | ||
268 | |||
269 | trace_softirq_entry(vec_nr); | ||
270 | h->action(h); | ||
271 | trace_softirq_exit(vec_nr); | ||
272 | if (unlikely(prev_count != preempt_count())) { | ||
273 | pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", | ||
274 | vec_nr, softirq_to_name[vec_nr], h->action, | ||
275 | prev_count, preempt_count()); | ||
276 | preempt_count_set(prev_count); | ||
276 | } | 277 | } |
278 | rcu_bh_qs(cpu); | ||
277 | h++; | 279 | h++; |
278 | pending >>= 1; | 280 | pending >>= softirq_bit; |
279 | } while (pending); | 281 | } |
280 | 282 | ||
281 | local_irq_disable(); | 283 | local_irq_disable(); |
282 | 284 | ||
@@ -433,8 +435,7 @@ void open_softirq(int nr, void (*action)(struct softirq_action *)) | |||
433 | /* | 435 | /* |
434 | * Tasklets | 436 | * Tasklets |
435 | */ | 437 | */ |
436 | struct tasklet_head | 438 | struct tasklet_head { |
437 | { | ||
438 | struct tasklet_struct *head; | 439 | struct tasklet_struct *head; |
439 | struct tasklet_struct **tail; | 440 | struct tasklet_struct **tail; |
440 | }; | 441 | }; |
@@ -453,7 +454,6 @@ void __tasklet_schedule(struct tasklet_struct *t) | |||
453 | raise_softirq_irqoff(TASKLET_SOFTIRQ); | 454 | raise_softirq_irqoff(TASKLET_SOFTIRQ); |
454 | local_irq_restore(flags); | 455 | local_irq_restore(flags); |
455 | } | 456 | } |
456 | |||
457 | EXPORT_SYMBOL(__tasklet_schedule); | 457 | EXPORT_SYMBOL(__tasklet_schedule); |
458 | 458 | ||
459 | void __tasklet_hi_schedule(struct tasklet_struct *t) | 459 | void __tasklet_hi_schedule(struct tasklet_struct *t) |
@@ -467,7 +467,6 @@ void __tasklet_hi_schedule(struct tasklet_struct *t) | |||
467 | raise_softirq_irqoff(HI_SOFTIRQ); | 467 | raise_softirq_irqoff(HI_SOFTIRQ); |
468 | local_irq_restore(flags); | 468 | local_irq_restore(flags); |
469 | } | 469 | } |
470 | |||
471 | EXPORT_SYMBOL(__tasklet_hi_schedule); | 470 | EXPORT_SYMBOL(__tasklet_hi_schedule); |
472 | 471 | ||
473 | void __tasklet_hi_schedule_first(struct tasklet_struct *t) | 472 | void __tasklet_hi_schedule_first(struct tasklet_struct *t) |
@@ -478,7 +477,6 @@ void __tasklet_hi_schedule_first(struct tasklet_struct *t) | |||
478 | __this_cpu_write(tasklet_hi_vec.head, t); | 477 | __this_cpu_write(tasklet_hi_vec.head, t); |
479 | __raise_softirq_irqoff(HI_SOFTIRQ); | 478 | __raise_softirq_irqoff(HI_SOFTIRQ); |
480 | } | 479 | } |
481 | |||
482 | EXPORT_SYMBOL(__tasklet_hi_schedule_first); | 480 | EXPORT_SYMBOL(__tasklet_hi_schedule_first); |
483 | 481 | ||
484 | static void tasklet_action(struct softirq_action *a) | 482 | static void tasklet_action(struct softirq_action *a) |
@@ -498,7 +496,8 @@ static void tasklet_action(struct softirq_action *a) | |||
498 | 496 | ||
499 | if (tasklet_trylock(t)) { | 497 | if (tasklet_trylock(t)) { |
500 | if (!atomic_read(&t->count)) { | 498 | if (!atomic_read(&t->count)) { |
501 | if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) | 499 | if (!test_and_clear_bit(TASKLET_STATE_SCHED, |
500 | &t->state)) | ||
502 | BUG(); | 501 | BUG(); |
503 | t->func(t->data); | 502 | t->func(t->data); |
504 | tasklet_unlock(t); | 503 | tasklet_unlock(t); |
@@ -533,7 +532,8 @@ static void tasklet_hi_action(struct softirq_action *a) | |||
533 | 532 | ||
534 | if (tasklet_trylock(t)) { | 533 | if (tasklet_trylock(t)) { |
535 | if (!atomic_read(&t->count)) { | 534 | if (!atomic_read(&t->count)) { |
536 | if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) | 535 | if (!test_and_clear_bit(TASKLET_STATE_SCHED, |
536 | &t->state)) | ||
537 | BUG(); | 537 | BUG(); |
538 | t->func(t->data); | 538 | t->func(t->data); |
539 | tasklet_unlock(t); | 539 | tasklet_unlock(t); |
@@ -551,7 +551,6 @@ static void tasklet_hi_action(struct softirq_action *a) | |||
551 | } | 551 | } |
552 | } | 552 | } |
553 | 553 | ||
554 | |||
555 | void tasklet_init(struct tasklet_struct *t, | 554 | void tasklet_init(struct tasklet_struct *t, |
556 | void (*func)(unsigned long), unsigned long data) | 555 | void (*func)(unsigned long), unsigned long data) |
557 | { | 556 | { |
@@ -561,13 +560,12 @@ void tasklet_init(struct tasklet_struct *t, | |||
561 | t->func = func; | 560 | t->func = func; |
562 | t->data = data; | 561 | t->data = data; |
563 | } | 562 | } |
564 | |||
565 | EXPORT_SYMBOL(tasklet_init); | 563 | EXPORT_SYMBOL(tasklet_init); |
566 | 564 | ||
567 | void tasklet_kill(struct tasklet_struct *t) | 565 | void tasklet_kill(struct tasklet_struct *t) |
568 | { | 566 | { |
569 | if (in_interrupt()) | 567 | if (in_interrupt()) |
570 | printk("Attempt to kill tasklet from interrupt\n"); | 568 | pr_notice("Attempt to kill tasklet from interrupt\n"); |
571 | 569 | ||
572 | while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { | 570 | while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { |
573 | do { | 571 | do { |
@@ -577,7 +575,6 @@ void tasklet_kill(struct tasklet_struct *t) | |||
577 | tasklet_unlock_wait(t); | 575 | tasklet_unlock_wait(t); |
578 | clear_bit(TASKLET_STATE_SCHED, &t->state); | 576 | clear_bit(TASKLET_STATE_SCHED, &t->state); |
579 | } | 577 | } |
580 | |||
581 | EXPORT_SYMBOL(tasklet_kill); | 578 | EXPORT_SYMBOL(tasklet_kill); |
582 | 579 | ||
583 | /* | 580 | /* |
@@ -727,9 +724,8 @@ static void takeover_tasklets(unsigned int cpu) | |||
727 | } | 724 | } |
728 | #endif /* CONFIG_HOTPLUG_CPU */ | 725 | #endif /* CONFIG_HOTPLUG_CPU */ |
729 | 726 | ||
730 | static int cpu_callback(struct notifier_block *nfb, | 727 | static int cpu_callback(struct notifier_block *nfb, unsigned long action, |
731 | unsigned long action, | 728 | void *hcpu) |
732 | void *hcpu) | ||
733 | { | 729 | { |
734 | switch (action) { | 730 | switch (action) { |
735 | #ifdef CONFIG_HOTPLUG_CPU | 731 | #ifdef CONFIG_HOTPLUG_CPU |