diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2014-10-09 06:30:57 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2014-10-09 06:30:57 -0400 |
commit | afa3536be88b435a057cb727b48fd3d760a497d2 (patch) | |
tree | 8562d3c8327286746ae835ef8eb39d4494a1054d | |
parent | 35a9ad8af0bb0fa3525e6d0d20e32551d226f38e (diff) | |
parent | 9b01f5bf3999a3db5b1bbd9fdfd80d8d304e94ee (diff) |
Merge branch 'timers-nohz-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull timer fixes from Ingo Molnar:
"Main changes:
- Fix the deadlock reported by Dave Jones et al
- Clean up and fix nohz_full interaction with arch abilities
- nohz init code consolidation/cleanup"
* 'timers-nohz-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
nohz: nohz full depends on irq work self IPI support
nohz: Consolidate nohz full init code
arm64: Tell irq work about self IPI support
arm: Tell irq work about self IPI support
x86: Tell irq work about self IPI support
irq_work: Force raised irq work to run on irq work interrupt
irq_work: Introduce arch_irq_work_has_interrupt()
nohz: Move nohz full init call to tick init
43 files changed, 133 insertions, 29 deletions
diff --git a/arch/alpha/include/asm/Kbuild b/arch/alpha/include/asm/Kbuild index e858aa0ad8af..a52cbf178c3a 100644 --- a/arch/alpha/include/asm/Kbuild +++ b/arch/alpha/include/asm/Kbuild | |||
@@ -4,6 +4,7 @@ generic-y += clkdev.h | |||
4 | generic-y += cputime.h | 4 | generic-y += cputime.h |
5 | generic-y += exec.h | 5 | generic-y += exec.h |
6 | generic-y += hash.h | 6 | generic-y += hash.h |
7 | generic-y += irq_work.h | ||
7 | generic-y += mcs_spinlock.h | 8 | generic-y += mcs_spinlock.h |
8 | generic-y += preempt.h | 9 | generic-y += preempt.h |
9 | generic-y += scatterlist.h | 10 | generic-y += scatterlist.h |
diff --git a/arch/arc/include/asm/Kbuild b/arch/arc/include/asm/Kbuild index e76fd79f32b0..b8fffc1a2ac2 100644 --- a/arch/arc/include/asm/Kbuild +++ b/arch/arc/include/asm/Kbuild | |||
@@ -18,6 +18,7 @@ generic-y += ioctl.h | |||
18 | generic-y += ioctls.h | 18 | generic-y += ioctls.h |
19 | generic-y += ipcbuf.h | 19 | generic-y += ipcbuf.h |
20 | generic-y += irq_regs.h | 20 | generic-y += irq_regs.h |
21 | generic-y += irq_work.h | ||
21 | generic-y += kmap_types.h | 22 | generic-y += kmap_types.h |
22 | generic-y += kvm_para.h | 23 | generic-y += kvm_para.h |
23 | generic-y += local.h | 24 | generic-y += local.h |
diff --git a/arch/arm/include/asm/irq_work.h b/arch/arm/include/asm/irq_work.h new file mode 100644 index 000000000000..712d03e5973a --- /dev/null +++ b/arch/arm/include/asm/irq_work.h | |||
@@ -0,0 +1,11 @@ | |||
1 | #ifndef __ASM_ARM_IRQ_WORK_H | ||
2 | #define __ASM_ARM_IRQ_WORK_H | ||
3 | |||
4 | #include <asm/smp_plat.h> | ||
5 | |||
6 | static inline bool arch_irq_work_has_interrupt(void) | ||
7 | { | ||
8 | return is_smp(); | ||
9 | } | ||
10 | |||
11 | #endif /* _ASM_ARM_IRQ_WORK_H */ | ||
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 39c74a2c3df9..13396d3d600e 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c | |||
@@ -499,7 +499,7 @@ void arch_send_call_function_single_ipi(int cpu) | |||
499 | #ifdef CONFIG_IRQ_WORK | 499 | #ifdef CONFIG_IRQ_WORK |
500 | void arch_irq_work_raise(void) | 500 | void arch_irq_work_raise(void) |
501 | { | 501 | { |
502 | if (is_smp()) | 502 | if (arch_irq_work_has_interrupt()) |
503 | smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); | 503 | smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); |
504 | } | 504 | } |
505 | #endif | 505 | #endif |
diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild index 0b3fcf86e6ba..c1968475cc4e 100644 --- a/arch/arm64/include/asm/Kbuild +++ b/arch/arm64/include/asm/Kbuild | |||
@@ -9,8 +9,8 @@ generic-y += current.h | |||
9 | generic-y += delay.h | 9 | generic-y += delay.h |
10 | generic-y += div64.h | 10 | generic-y += div64.h |
11 | generic-y += dma.h | 11 | generic-y += dma.h |
12 | generic-y += emergency-restart.h | ||
13 | generic-y += early_ioremap.h | 12 | generic-y += early_ioremap.h |
13 | generic-y += emergency-restart.h | ||
14 | generic-y += errno.h | 14 | generic-y += errno.h |
15 | generic-y += ftrace.h | 15 | generic-y += ftrace.h |
16 | generic-y += hash.h | 16 | generic-y += hash.h |
diff --git a/arch/arm64/include/asm/irq_work.h b/arch/arm64/include/asm/irq_work.h new file mode 100644 index 000000000000..8e24ef3f7c82 --- /dev/null +++ b/arch/arm64/include/asm/irq_work.h | |||
@@ -0,0 +1,11 @@ | |||
1 | #ifndef __ASM_IRQ_WORK_H | ||
2 | #define __ASM_IRQ_WORK_H | ||
3 | |||
4 | #include <asm/smp.h> | ||
5 | |||
6 | static inline bool arch_irq_work_has_interrupt(void) | ||
7 | { | ||
8 | return !!__smp_cross_call; | ||
9 | } | ||
10 | |||
11 | #endif /* __ASM_IRQ_WORK_H */ | ||
diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index a498f2cd2c2a..780f82c827b6 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h | |||
@@ -48,6 +48,8 @@ extern void smp_init_cpus(void); | |||
48 | */ | 48 | */ |
49 | extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); | 49 | extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); |
50 | 50 | ||
51 | extern void (*__smp_cross_call)(const struct cpumask *, unsigned int); | ||
52 | |||
51 | /* | 53 | /* |
52 | * Called from the secondary holding pen, this is the secondary CPU entry point. | 54 | * Called from the secondary holding pen, this is the secondary CPU entry point. |
53 | */ | 55 | */ |
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 474339718105..b06d1d90ee8c 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c | |||
@@ -470,7 +470,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) | |||
470 | } | 470 | } |
471 | } | 471 | } |
472 | 472 | ||
473 | static void (*__smp_cross_call)(const struct cpumask *, unsigned int); | 473 | void (*__smp_cross_call)(const struct cpumask *, unsigned int); |
474 | 474 | ||
475 | void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int)) | 475 | void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int)) |
476 | { | 476 | { |
diff --git a/arch/avr32/include/asm/Kbuild b/arch/avr32/include/asm/Kbuild index 00a0f3ccd6eb..2a71b1cb9848 100644 --- a/arch/avr32/include/asm/Kbuild +++ b/arch/avr32/include/asm/Kbuild | |||
@@ -9,6 +9,7 @@ generic-y += exec.h | |||
9 | generic-y += futex.h | 9 | generic-y += futex.h |
10 | generic-y += hash.h | 10 | generic-y += hash.h |
11 | generic-y += irq_regs.h | 11 | generic-y += irq_regs.h |
12 | generic-y += irq_work.h | ||
12 | generic-y += local.h | 13 | generic-y += local.h |
13 | generic-y += local64.h | 14 | generic-y += local64.h |
14 | generic-y += mcs_spinlock.h | 15 | generic-y += mcs_spinlock.h |
diff --git a/arch/blackfin/include/asm/Kbuild b/arch/blackfin/include/asm/Kbuild index 0d93b9a79ca9..46ed6bb9c679 100644 --- a/arch/blackfin/include/asm/Kbuild +++ b/arch/blackfin/include/asm/Kbuild | |||
@@ -15,6 +15,7 @@ generic-y += hw_irq.h | |||
15 | generic-y += ioctl.h | 15 | generic-y += ioctl.h |
16 | generic-y += ipcbuf.h | 16 | generic-y += ipcbuf.h |
17 | generic-y += irq_regs.h | 17 | generic-y += irq_regs.h |
18 | generic-y += irq_work.h | ||
18 | generic-y += kdebug.h | 19 | generic-y += kdebug.h |
19 | generic-y += kmap_types.h | 20 | generic-y += kmap_types.h |
20 | generic-y += kvm_para.h | 21 | generic-y += kvm_para.h |
diff --git a/arch/c6x/include/asm/Kbuild b/arch/c6x/include/asm/Kbuild index 8dbdce8421b0..e77e0c1dbe75 100644 --- a/arch/c6x/include/asm/Kbuild +++ b/arch/c6x/include/asm/Kbuild | |||
@@ -22,6 +22,7 @@ generic-y += ioctl.h | |||
22 | generic-y += ioctls.h | 22 | generic-y += ioctls.h |
23 | generic-y += ipcbuf.h | 23 | generic-y += ipcbuf.h |
24 | generic-y += irq_regs.h | 24 | generic-y += irq_regs.h |
25 | generic-y += irq_work.h | ||
25 | generic-y += kdebug.h | 26 | generic-y += kdebug.h |
26 | generic-y += kmap_types.h | 27 | generic-y += kmap_types.h |
27 | generic-y += local.h | 28 | generic-y += local.h |
diff --git a/arch/cris/include/asm/Kbuild b/arch/cris/include/asm/Kbuild index 31742dfadff9..802b94c4ca86 100644 --- a/arch/cris/include/asm/Kbuild +++ b/arch/cris/include/asm/Kbuild | |||
@@ -8,6 +8,7 @@ generic-y += clkdev.h | |||
8 | generic-y += cputime.h | 8 | generic-y += cputime.h |
9 | generic-y += exec.h | 9 | generic-y += exec.h |
10 | generic-y += hash.h | 10 | generic-y += hash.h |
11 | generic-y += irq_work.h | ||
11 | generic-y += kvm_para.h | 12 | generic-y += kvm_para.h |
12 | generic-y += linkage.h | 13 | generic-y += linkage.h |
13 | generic-y += mcs_spinlock.h | 14 | generic-y += mcs_spinlock.h |
diff --git a/arch/frv/include/asm/Kbuild b/arch/frv/include/asm/Kbuild index 5b73921b6e9d..3caf05cabfc5 100644 --- a/arch/frv/include/asm/Kbuild +++ b/arch/frv/include/asm/Kbuild | |||
@@ -3,6 +3,7 @@ generic-y += clkdev.h | |||
3 | generic-y += cputime.h | 3 | generic-y += cputime.h |
4 | generic-y += exec.h | 4 | generic-y += exec.h |
5 | generic-y += hash.h | 5 | generic-y += hash.h |
6 | generic-y += irq_work.h | ||
6 | generic-y += mcs_spinlock.h | 7 | generic-y += mcs_spinlock.h |
7 | generic-y += preempt.h | 8 | generic-y += preempt.h |
8 | generic-y += scatterlist.h | 9 | generic-y += scatterlist.h |
diff --git a/arch/hexagon/include/asm/Kbuild b/arch/hexagon/include/asm/Kbuild index 0e69796b58c7..5f234a5a2320 100644 --- a/arch/hexagon/include/asm/Kbuild +++ b/arch/hexagon/include/asm/Kbuild | |||
@@ -23,6 +23,7 @@ generic-y += ioctls.h | |||
23 | generic-y += iomap.h | 23 | generic-y += iomap.h |
24 | generic-y += ipcbuf.h | 24 | generic-y += ipcbuf.h |
25 | generic-y += irq_regs.h | 25 | generic-y += irq_regs.h |
26 | generic-y += irq_work.h | ||
26 | generic-y += kdebug.h | 27 | generic-y += kdebug.h |
27 | generic-y += kmap_types.h | 28 | generic-y += kmap_types.h |
28 | generic-y += local.h | 29 | generic-y += local.h |
diff --git a/arch/ia64/include/asm/Kbuild b/arch/ia64/include/asm/Kbuild index e8317d2d6c8d..747320be9d0e 100644 --- a/arch/ia64/include/asm/Kbuild +++ b/arch/ia64/include/asm/Kbuild | |||
@@ -2,6 +2,7 @@ | |||
2 | generic-y += clkdev.h | 2 | generic-y += clkdev.h |
3 | generic-y += exec.h | 3 | generic-y += exec.h |
4 | generic-y += hash.h | 4 | generic-y += hash.h |
5 | generic-y += irq_work.h | ||
5 | generic-y += kvm_para.h | 6 | generic-y += kvm_para.h |
6 | generic-y += mcs_spinlock.h | 7 | generic-y += mcs_spinlock.h |
7 | generic-y += preempt.h | 8 | generic-y += preempt.h |
diff --git a/arch/m32r/include/asm/Kbuild b/arch/m32r/include/asm/Kbuild index accc10a3dc78..e02448b0648b 100644 --- a/arch/m32r/include/asm/Kbuild +++ b/arch/m32r/include/asm/Kbuild | |||
@@ -3,6 +3,7 @@ generic-y += clkdev.h | |||
3 | generic-y += cputime.h | 3 | generic-y += cputime.h |
4 | generic-y += exec.h | 4 | generic-y += exec.h |
5 | generic-y += hash.h | 5 | generic-y += hash.h |
6 | generic-y += irq_work.h | ||
6 | generic-y += mcs_spinlock.h | 7 | generic-y += mcs_spinlock.h |
7 | generic-y += module.h | 8 | generic-y += module.h |
8 | generic-y += preempt.h | 9 | generic-y += preempt.h |
diff --git a/arch/m68k/include/asm/Kbuild b/arch/m68k/include/asm/Kbuild index c67c94a2d672..dbaf9f3065e8 100644 --- a/arch/m68k/include/asm/Kbuild +++ b/arch/m68k/include/asm/Kbuild | |||
@@ -11,6 +11,7 @@ generic-y += hw_irq.h | |||
11 | generic-y += ioctl.h | 11 | generic-y += ioctl.h |
12 | generic-y += ipcbuf.h | 12 | generic-y += ipcbuf.h |
13 | generic-y += irq_regs.h | 13 | generic-y += irq_regs.h |
14 | generic-y += irq_work.h | ||
14 | generic-y += kdebug.h | 15 | generic-y += kdebug.h |
15 | generic-y += kmap_types.h | 16 | generic-y += kmap_types.h |
16 | generic-y += kvm_para.h | 17 | generic-y += kvm_para.h |
diff --git a/arch/metag/include/asm/Kbuild b/arch/metag/include/asm/Kbuild index c29ead89a317..7b8111c8f937 100644 --- a/arch/metag/include/asm/Kbuild +++ b/arch/metag/include/asm/Kbuild | |||
@@ -19,6 +19,7 @@ generic-y += ioctl.h | |||
19 | generic-y += ioctls.h | 19 | generic-y += ioctls.h |
20 | generic-y += ipcbuf.h | 20 | generic-y += ipcbuf.h |
21 | generic-y += irq_regs.h | 21 | generic-y += irq_regs.h |
22 | generic-y += irq_work.h | ||
22 | generic-y += kdebug.h | 23 | generic-y += kdebug.h |
23 | generic-y += kmap_types.h | 24 | generic-y += kmap_types.h |
24 | generic-y += kvm_para.h | 25 | generic-y += kvm_para.h |
diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild index 27a3acda6c19..448143b8cabd 100644 --- a/arch/microblaze/include/asm/Kbuild +++ b/arch/microblaze/include/asm/Kbuild | |||
@@ -5,6 +5,7 @@ generic-y += cputime.h | |||
5 | generic-y += device.h | 5 | generic-y += device.h |
6 | generic-y += exec.h | 6 | generic-y += exec.h |
7 | generic-y += hash.h | 7 | generic-y += hash.h |
8 | generic-y += irq_work.h | ||
8 | generic-y += mcs_spinlock.h | 9 | generic-y += mcs_spinlock.h |
9 | generic-y += preempt.h | 10 | generic-y += preempt.h |
10 | generic-y += scatterlist.h | 11 | generic-y += scatterlist.h |
diff --git a/arch/mips/include/asm/Kbuild b/arch/mips/include/asm/Kbuild index 335e5290ec75..57012ef1f51e 100644 --- a/arch/mips/include/asm/Kbuild +++ b/arch/mips/include/asm/Kbuild | |||
@@ -3,6 +3,7 @@ generic-y += cputime.h | |||
3 | generic-y += current.h | 3 | generic-y += current.h |
4 | generic-y += emergency-restart.h | 4 | generic-y += emergency-restart.h |
5 | generic-y += hash.h | 5 | generic-y += hash.h |
6 | generic-y += irq_work.h | ||
6 | generic-y += local64.h | 7 | generic-y += local64.h |
7 | generic-y += mcs_spinlock.h | 8 | generic-y += mcs_spinlock.h |
8 | generic-y += mutex.h | 9 | generic-y += mutex.h |
diff --git a/arch/mn10300/include/asm/Kbuild b/arch/mn10300/include/asm/Kbuild index ecbd6676bd33..77eb1a68d13b 100644 --- a/arch/mn10300/include/asm/Kbuild +++ b/arch/mn10300/include/asm/Kbuild | |||
@@ -4,6 +4,7 @@ generic-y += clkdev.h | |||
4 | generic-y += cputime.h | 4 | generic-y += cputime.h |
5 | generic-y += exec.h | 5 | generic-y += exec.h |
6 | generic-y += hash.h | 6 | generic-y += hash.h |
7 | generic-y += irq_work.h | ||
7 | generic-y += mcs_spinlock.h | 8 | generic-y += mcs_spinlock.h |
8 | generic-y += preempt.h | 9 | generic-y += preempt.h |
9 | generic-y += scatterlist.h | 10 | generic-y += scatterlist.h |
diff --git a/arch/openrisc/include/asm/Kbuild b/arch/openrisc/include/asm/Kbuild index 480af0d9c2f5..89b61d7dc790 100644 --- a/arch/openrisc/include/asm/Kbuild +++ b/arch/openrisc/include/asm/Kbuild | |||
@@ -31,6 +31,7 @@ generic-y += ioctl.h | |||
31 | generic-y += ioctls.h | 31 | generic-y += ioctls.h |
32 | generic-y += ipcbuf.h | 32 | generic-y += ipcbuf.h |
33 | generic-y += irq_regs.h | 33 | generic-y += irq_regs.h |
34 | generic-y += irq_work.h | ||
34 | generic-y += kdebug.h | 35 | generic-y += kdebug.h |
35 | generic-y += kmap_types.h | 36 | generic-y += kmap_types.h |
36 | generic-y += kvm_para.h | 37 | generic-y += kvm_para.h |
diff --git a/arch/parisc/include/asm/Kbuild b/arch/parisc/include/asm/Kbuild index ecf25e6678ad..ffb024b8423f 100644 --- a/arch/parisc/include/asm/Kbuild +++ b/arch/parisc/include/asm/Kbuild | |||
@@ -10,6 +10,7 @@ generic-y += exec.h | |||
10 | generic-y += hash.h | 10 | generic-y += hash.h |
11 | generic-y += hw_irq.h | 11 | generic-y += hw_irq.h |
12 | generic-y += irq_regs.h | 12 | generic-y += irq_regs.h |
13 | generic-y += irq_work.h | ||
13 | generic-y += kdebug.h | 14 | generic-y += kdebug.h |
14 | generic-y += kvm_para.h | 15 | generic-y += kvm_para.h |
15 | generic-y += local.h | 16 | generic-y += local.h |
diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild index 7f23f162ce9c..31e8f59aff38 100644 --- a/arch/powerpc/include/asm/Kbuild +++ b/arch/powerpc/include/asm/Kbuild | |||
@@ -1,6 +1,7 @@ | |||
1 | 1 | ||
2 | generic-y += clkdev.h | 2 | generic-y += clkdev.h |
3 | generic-y += hash.h | 3 | generic-y += hash.h |
4 | generic-y += irq_work.h | ||
4 | generic-y += mcs_spinlock.h | 5 | generic-y += mcs_spinlock.h |
5 | generic-y += preempt.h | 6 | generic-y += preempt.h |
6 | generic-y += rwsem.h | 7 | generic-y += rwsem.h |
diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild index b3fea0722ff1..773f86676588 100644 --- a/arch/s390/include/asm/Kbuild +++ b/arch/s390/include/asm/Kbuild | |||
@@ -2,6 +2,7 @@ | |||
2 | 2 | ||
3 | generic-y += clkdev.h | 3 | generic-y += clkdev.h |
4 | generic-y += hash.h | 4 | generic-y += hash.h |
5 | generic-y += irq_work.h | ||
5 | generic-y += mcs_spinlock.h | 6 | generic-y += mcs_spinlock.h |
6 | generic-y += preempt.h | 7 | generic-y += preempt.h |
7 | generic-y += scatterlist.h | 8 | generic-y += scatterlist.h |
diff --git a/arch/score/include/asm/Kbuild b/arch/score/include/asm/Kbuild index d26c48fc93c9..3fe5681744f1 100644 --- a/arch/score/include/asm/Kbuild +++ b/arch/score/include/asm/Kbuild | |||
@@ -6,6 +6,7 @@ generic-y += barrier.h | |||
6 | generic-y += clkdev.h | 6 | generic-y += clkdev.h |
7 | generic-y += cputime.h | 7 | generic-y += cputime.h |
8 | generic-y += hash.h | 8 | generic-y += hash.h |
9 | generic-y += irq_work.h | ||
9 | generic-y += mcs_spinlock.h | 10 | generic-y += mcs_spinlock.h |
10 | generic-y += preempt.h | 11 | generic-y += preempt.h |
11 | generic-y += scatterlist.h | 12 | generic-y += scatterlist.h |
diff --git a/arch/sh/include/asm/Kbuild b/arch/sh/include/asm/Kbuild index c19e47dacb31..5a6c9acff0d2 100644 --- a/arch/sh/include/asm/Kbuild +++ b/arch/sh/include/asm/Kbuild | |||
@@ -12,6 +12,7 @@ generic-y += hash.h | |||
12 | generic-y += ioctl.h | 12 | generic-y += ioctl.h |
13 | generic-y += ipcbuf.h | 13 | generic-y += ipcbuf.h |
14 | generic-y += irq_regs.h | 14 | generic-y += irq_regs.h |
15 | generic-y += irq_work.h | ||
15 | generic-y += kvm_para.h | 16 | generic-y += kvm_para.h |
16 | generic-y += local.h | 17 | generic-y += local.h |
17 | generic-y += local64.h | 18 | generic-y += local64.h |
diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild index cdd1b447bb6c..f5f94ce1692c 100644 --- a/arch/sparc/include/asm/Kbuild +++ b/arch/sparc/include/asm/Kbuild | |||
@@ -8,6 +8,7 @@ generic-y += emergency-restart.h | |||
8 | generic-y += exec.h | 8 | generic-y += exec.h |
9 | generic-y += hash.h | 9 | generic-y += hash.h |
10 | generic-y += irq_regs.h | 10 | generic-y += irq_regs.h |
11 | generic-y += irq_work.h | ||
11 | generic-y += linkage.h | 12 | generic-y += linkage.h |
12 | generic-y += local.h | 13 | generic-y += local.h |
13 | generic-y += local64.h | 14 | generic-y += local64.h |
diff --git a/arch/tile/include/asm/Kbuild b/arch/tile/include/asm/Kbuild index 0aa5675e7025..e6462b8a6284 100644 --- a/arch/tile/include/asm/Kbuild +++ b/arch/tile/include/asm/Kbuild | |||
@@ -17,6 +17,7 @@ generic-y += ioctl.h | |||
17 | generic-y += ioctls.h | 17 | generic-y += ioctls.h |
18 | generic-y += ipcbuf.h | 18 | generic-y += ipcbuf.h |
19 | generic-y += irq_regs.h | 19 | generic-y += irq_regs.h |
20 | generic-y += irq_work.h | ||
20 | generic-y += local.h | 21 | generic-y += local.h |
21 | generic-y += local64.h | 22 | generic-y += local64.h |
22 | generic-y += mcs_spinlock.h | 23 | generic-y += mcs_spinlock.h |
diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index 7bd64aa2e94a..244b12c8cb39 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild | |||
@@ -14,6 +14,7 @@ generic-y += hash.h | |||
14 | generic-y += hw_irq.h | 14 | generic-y += hw_irq.h |
15 | generic-y += io.h | 15 | generic-y += io.h |
16 | generic-y += irq_regs.h | 16 | generic-y += irq_regs.h |
17 | generic-y += irq_work.h | ||
17 | generic-y += kdebug.h | 18 | generic-y += kdebug.h |
18 | generic-y += mcs_spinlock.h | 19 | generic-y += mcs_spinlock.h |
19 | generic-y += mutex.h | 20 | generic-y += mutex.h |
diff --git a/arch/unicore32/include/asm/Kbuild b/arch/unicore32/include/asm/Kbuild index 1e5fb872a4aa..5a2bb53faa42 100644 --- a/arch/unicore32/include/asm/Kbuild +++ b/arch/unicore32/include/asm/Kbuild | |||
@@ -22,6 +22,7 @@ generic-y += ioctl.h | |||
22 | generic-y += ioctls.h | 22 | generic-y += ioctls.h |
23 | generic-y += ipcbuf.h | 23 | generic-y += ipcbuf.h |
24 | generic-y += irq_regs.h | 24 | generic-y += irq_regs.h |
25 | generic-y += irq_work.h | ||
25 | generic-y += kdebug.h | 26 | generic-y += kdebug.h |
26 | generic-y += kmap_types.h | 27 | generic-y += kmap_types.h |
27 | generic-y += local.h | 28 | generic-y += local.h |
diff --git a/arch/x86/include/asm/irq_work.h b/arch/x86/include/asm/irq_work.h new file mode 100644 index 000000000000..78162f8e248b --- /dev/null +++ b/arch/x86/include/asm/irq_work.h | |||
@@ -0,0 +1,11 @@ | |||
1 | #ifndef _ASM_IRQ_WORK_H | ||
2 | #define _ASM_IRQ_WORK_H | ||
3 | |||
4 | #include <asm/processor.h> | ||
5 | |||
6 | static inline bool arch_irq_work_has_interrupt(void) | ||
7 | { | ||
8 | return cpu_has_apic; | ||
9 | } | ||
10 | |||
11 | #endif /* _ASM_IRQ_WORK_H */ | ||
diff --git a/arch/x86/kernel/irq_work.c b/arch/x86/kernel/irq_work.c index 1de84e3ab4e0..15d741ddfeeb 100644 --- a/arch/x86/kernel/irq_work.c +++ b/arch/x86/kernel/irq_work.c | |||
@@ -41,7 +41,7 @@ __visible void smp_trace_irq_work_interrupt(struct pt_regs *regs) | |||
41 | void arch_irq_work_raise(void) | 41 | void arch_irq_work_raise(void) |
42 | { | 42 | { |
43 | #ifdef CONFIG_X86_LOCAL_APIC | 43 | #ifdef CONFIG_X86_LOCAL_APIC |
44 | if (!cpu_has_apic) | 44 | if (!arch_irq_work_has_interrupt()) |
45 | return; | 45 | return; |
46 | 46 | ||
47 | apic->send_IPI_self(IRQ_WORK_VECTOR); | 47 | apic->send_IPI_self(IRQ_WORK_VECTOR); |
diff --git a/arch/xtensa/include/asm/Kbuild b/arch/xtensa/include/asm/Kbuild index c3d20ba6eb86..105d38922c44 100644 --- a/arch/xtensa/include/asm/Kbuild +++ b/arch/xtensa/include/asm/Kbuild | |||
@@ -12,6 +12,7 @@ generic-y += hardirq.h | |||
12 | generic-y += hash.h | 12 | generic-y += hash.h |
13 | generic-y += ioctl.h | 13 | generic-y += ioctl.h |
14 | generic-y += irq_regs.h | 14 | generic-y += irq_regs.h |
15 | generic-y += irq_work.h | ||
15 | generic-y += kdebug.h | 16 | generic-y += kdebug.h |
16 | generic-y += kmap_types.h | 17 | generic-y += kmap_types.h |
17 | generic-y += kvm_para.h | 18 | generic-y += kvm_para.h |
diff --git a/include/asm-generic/irq_work.h b/include/asm-generic/irq_work.h new file mode 100644 index 000000000000..a44f452c6590 --- /dev/null +++ b/include/asm-generic/irq_work.h | |||
@@ -0,0 +1,10 @@ | |||
1 | #ifndef __ASM_IRQ_WORK_H | ||
2 | #define __ASM_IRQ_WORK_H | ||
3 | |||
4 | static inline bool arch_irq_work_has_interrupt(void) | ||
5 | { | ||
6 | return false; | ||
7 | } | ||
8 | |||
9 | #endif /* __ASM_IRQ_WORK_H */ | ||
10 | |||
diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h index bf9422c3aefe..bf3fe719c7ce 100644 --- a/include/linux/irq_work.h +++ b/include/linux/irq_work.h | |||
@@ -39,9 +39,12 @@ bool irq_work_queue_on(struct irq_work *work, int cpu); | |||
39 | #endif | 39 | #endif |
40 | 40 | ||
41 | void irq_work_run(void); | 41 | void irq_work_run(void); |
42 | void irq_work_tick(void); | ||
42 | void irq_work_sync(struct irq_work *work); | 43 | void irq_work_sync(struct irq_work *work); |
43 | 44 | ||
44 | #ifdef CONFIG_IRQ_WORK | 45 | #ifdef CONFIG_IRQ_WORK |
46 | #include <asm/irq_work.h> | ||
47 | |||
45 | bool irq_work_needs_cpu(void); | 48 | bool irq_work_needs_cpu(void); |
46 | #else | 49 | #else |
47 | static inline bool irq_work_needs_cpu(void) { return false; } | 50 | static inline bool irq_work_needs_cpu(void) { return false; } |
diff --git a/include/linux/tick.h b/include/linux/tick.h index 9a82c7dc3fdd..595ee86f5e0d 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h | |||
@@ -181,14 +181,12 @@ static inline bool tick_nohz_full_cpu(int cpu) | |||
181 | return cpumask_test_cpu(cpu, tick_nohz_full_mask); | 181 | return cpumask_test_cpu(cpu, tick_nohz_full_mask); |
182 | } | 182 | } |
183 | 183 | ||
184 | extern void tick_nohz_init(void); | ||
185 | extern void __tick_nohz_full_check(void); | 184 | extern void __tick_nohz_full_check(void); |
186 | extern void tick_nohz_full_kick(void); | 185 | extern void tick_nohz_full_kick(void); |
187 | extern void tick_nohz_full_kick_cpu(int cpu); | 186 | extern void tick_nohz_full_kick_cpu(int cpu); |
188 | extern void tick_nohz_full_kick_all(void); | 187 | extern void tick_nohz_full_kick_all(void); |
189 | extern void __tick_nohz_task_switch(struct task_struct *tsk); | 188 | extern void __tick_nohz_task_switch(struct task_struct *tsk); |
190 | #else | 189 | #else |
191 | static inline void tick_nohz_init(void) { } | ||
192 | static inline bool tick_nohz_full_enabled(void) { return false; } | 190 | static inline bool tick_nohz_full_enabled(void) { return false; } |
193 | static inline bool tick_nohz_full_cpu(int cpu) { return false; } | 191 | static inline bool tick_nohz_full_cpu(int cpu) { return false; } |
194 | static inline void __tick_nohz_full_check(void) { } | 192 | static inline void __tick_nohz_full_check(void) { } |
diff --git a/init/main.c b/init/main.c index bb1aed928f21..8af2f1abfe38 100644 --- a/init/main.c +++ b/init/main.c | |||
@@ -577,7 +577,6 @@ asmlinkage __visible void __init start_kernel(void) | |||
577 | local_irq_disable(); | 577 | local_irq_disable(); |
578 | idr_init_cache(); | 578 | idr_init_cache(); |
579 | rcu_init(); | 579 | rcu_init(); |
580 | tick_nohz_init(); | ||
581 | context_tracking_init(); | 580 | context_tracking_init(); |
582 | radix_tree_init(); | 581 | radix_tree_init(); |
583 | /* init some links before init_ISA_irqs() */ | 582 | /* init some links before init_ISA_irqs() */ |
diff --git a/kernel/irq_work.c b/kernel/irq_work.c index e6bcbe756663..385b85aded19 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c | |||
@@ -115,8 +115,10 @@ bool irq_work_needs_cpu(void) | |||
115 | 115 | ||
116 | raised = &__get_cpu_var(raised_list); | 116 | raised = &__get_cpu_var(raised_list); |
117 | lazy = &__get_cpu_var(lazy_list); | 117 | lazy = &__get_cpu_var(lazy_list); |
118 | if (llist_empty(raised) && llist_empty(lazy)) | 118 | |
119 | return false; | 119 | if (llist_empty(raised) || arch_irq_work_has_interrupt()) |
120 | if (llist_empty(lazy)) | ||
121 | return false; | ||
120 | 122 | ||
121 | /* All work should have been flushed before going offline */ | 123 | /* All work should have been flushed before going offline */ |
122 | WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); | 124 | WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); |
@@ -171,6 +173,15 @@ void irq_work_run(void) | |||
171 | } | 173 | } |
172 | EXPORT_SYMBOL_GPL(irq_work_run); | 174 | EXPORT_SYMBOL_GPL(irq_work_run); |
173 | 175 | ||
176 | void irq_work_tick(void) | ||
177 | { | ||
178 | struct llist_head *raised = &__get_cpu_var(raised_list); | ||
179 | |||
180 | if (!llist_empty(raised) && !arch_irq_work_has_interrupt()) | ||
181 | irq_work_run_list(raised); | ||
182 | irq_work_run_list(&__get_cpu_var(lazy_list)); | ||
183 | } | ||
184 | |||
174 | /* | 185 | /* |
175 | * Synchronize against the irq_work @entry, ensures the entry is not | 186 | * Synchronize against the irq_work @entry, ensures the entry is not |
176 | * currently in use. | 187 | * currently in use. |
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index 0a0608edeb26..052b4b53c3d6 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c | |||
@@ -400,4 +400,5 @@ void tick_resume(void) | |||
400 | void __init tick_init(void) | 400 | void __init tick_init(void) |
401 | { | 401 | { |
402 | tick_broadcast_init(); | 402 | tick_broadcast_init(); |
403 | tick_nohz_init(); | ||
403 | } | 404 | } |
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index c19c1d84b6f3..366aeb4f2c66 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h | |||
@@ -99,6 +99,13 @@ static inline int tick_broadcast_oneshot_active(void) { return 0; } | |||
99 | static inline bool tick_broadcast_oneshot_available(void) { return false; } | 99 | static inline bool tick_broadcast_oneshot_available(void) { return false; } |
100 | #endif /* !TICK_ONESHOT */ | 100 | #endif /* !TICK_ONESHOT */ |
101 | 101 | ||
102 | /* NO_HZ_FULL internal */ | ||
103 | #ifdef CONFIG_NO_HZ_FULL | ||
104 | extern void tick_nohz_init(void); | ||
105 | # else | ||
106 | static inline void tick_nohz_init(void) { } | ||
107 | #endif | ||
108 | |||
102 | /* | 109 | /* |
103 | * Broadcasting support | 110 | * Broadcasting support |
104 | */ | 111 | */ |
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index f654a8a298fa..5a9ff243588c 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
@@ -295,22 +295,12 @@ out: | |||
295 | /* Parse the boot-time nohz CPU list from the kernel parameters. */ | 295 | /* Parse the boot-time nohz CPU list from the kernel parameters. */ |
296 | static int __init tick_nohz_full_setup(char *str) | 296 | static int __init tick_nohz_full_setup(char *str) |
297 | { | 297 | { |
298 | int cpu; | ||
299 | |||
300 | alloc_bootmem_cpumask_var(&tick_nohz_full_mask); | 298 | alloc_bootmem_cpumask_var(&tick_nohz_full_mask); |
301 | alloc_bootmem_cpumask_var(&housekeeping_mask); | ||
302 | if (cpulist_parse(str, tick_nohz_full_mask) < 0) { | 299 | if (cpulist_parse(str, tick_nohz_full_mask) < 0) { |
303 | pr_warning("NOHZ: Incorrect nohz_full cpumask\n"); | 300 | pr_warning("NOHZ: Incorrect nohz_full cpumask\n"); |
301 | free_bootmem_cpumask_var(tick_nohz_full_mask); | ||
304 | return 1; | 302 | return 1; |
305 | } | 303 | } |
306 | |||
307 | cpu = smp_processor_id(); | ||
308 | if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) { | ||
309 | pr_warning("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", cpu); | ||
310 | cpumask_clear_cpu(cpu, tick_nohz_full_mask); | ||
311 | } | ||
312 | cpumask_andnot(housekeeping_mask, | ||
313 | cpu_possible_mask, tick_nohz_full_mask); | ||
314 | tick_nohz_full_running = true; | 304 | tick_nohz_full_running = true; |
315 | 305 | ||
316 | return 1; | 306 | return 1; |
@@ -349,18 +339,11 @@ static int tick_nohz_init_all(void) | |||
349 | 339 | ||
350 | #ifdef CONFIG_NO_HZ_FULL_ALL | 340 | #ifdef CONFIG_NO_HZ_FULL_ALL |
351 | if (!alloc_cpumask_var(&tick_nohz_full_mask, GFP_KERNEL)) { | 341 | if (!alloc_cpumask_var(&tick_nohz_full_mask, GFP_KERNEL)) { |
352 | pr_err("NO_HZ: Can't allocate full dynticks cpumask\n"); | 342 | WARN(1, "NO_HZ: Can't allocate full dynticks cpumask\n"); |
353 | return err; | ||
354 | } | ||
355 | if (!alloc_cpumask_var(&housekeeping_mask, GFP_KERNEL)) { | ||
356 | pr_err("NO_HZ: Can't allocate not-full dynticks cpumask\n"); | ||
357 | return err; | 343 | return err; |
358 | } | 344 | } |
359 | err = 0; | 345 | err = 0; |
360 | cpumask_setall(tick_nohz_full_mask); | 346 | cpumask_setall(tick_nohz_full_mask); |
361 | cpumask_clear_cpu(smp_processor_id(), tick_nohz_full_mask); | ||
362 | cpumask_clear(housekeeping_mask); | ||
363 | cpumask_set_cpu(smp_processor_id(), housekeeping_mask); | ||
364 | tick_nohz_full_running = true; | 347 | tick_nohz_full_running = true; |
365 | #endif | 348 | #endif |
366 | return err; | 349 | return err; |
@@ -375,6 +358,37 @@ void __init tick_nohz_init(void) | |||
375 | return; | 358 | return; |
376 | } | 359 | } |
377 | 360 | ||
361 | if (!alloc_cpumask_var(&housekeeping_mask, GFP_KERNEL)) { | ||
362 | WARN(1, "NO_HZ: Can't allocate not-full dynticks cpumask\n"); | ||
363 | cpumask_clear(tick_nohz_full_mask); | ||
364 | tick_nohz_full_running = false; | ||
365 | return; | ||
366 | } | ||
367 | |||
368 | /* | ||
369 | * Full dynticks uses irq work to drive the tick rescheduling on safe | ||
370 | * locking contexts. But then we need irq work to raise its own | ||
371 | * interrupts to avoid circular dependency on the tick | ||
372 | */ | ||
373 | if (!arch_irq_work_has_interrupt()) { | ||
374 | pr_warning("NO_HZ: Can't run full dynticks because arch doesn't " | ||
375 | "support irq work self-IPIs\n"); | ||
376 | cpumask_clear(tick_nohz_full_mask); | ||
377 | cpumask_copy(housekeeping_mask, cpu_possible_mask); | ||
378 | tick_nohz_full_running = false; | ||
379 | return; | ||
380 | } | ||
381 | |||
382 | cpu = smp_processor_id(); | ||
383 | |||
384 | if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) { | ||
385 | pr_warning("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", cpu); | ||
386 | cpumask_clear_cpu(cpu, tick_nohz_full_mask); | ||
387 | } | ||
388 | |||
389 | cpumask_andnot(housekeeping_mask, | ||
390 | cpu_possible_mask, tick_nohz_full_mask); | ||
391 | |||
378 | for_each_cpu(cpu, tick_nohz_full_mask) | 392 | for_each_cpu(cpu, tick_nohz_full_mask) |
379 | context_tracking_cpu_set(cpu); | 393 | context_tracking_cpu_set(cpu); |
380 | 394 | ||
diff --git a/kernel/time/timer.c b/kernel/time/timer.c index aca5dfe2fa3d..9bbb8344ed3b 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c | |||
@@ -1385,7 +1385,7 @@ void update_process_times(int user_tick) | |||
1385 | rcu_check_callbacks(cpu, user_tick); | 1385 | rcu_check_callbacks(cpu, user_tick); |
1386 | #ifdef CONFIG_IRQ_WORK | 1386 | #ifdef CONFIG_IRQ_WORK |
1387 | if (in_irq()) | 1387 | if (in_irq()) |
1388 | irq_work_run(); | 1388 | irq_work_tick(); |
1389 | #endif | 1389 | #endif |
1390 | scheduler_tick(); | 1390 | scheduler_tick(); |
1391 | run_posix_cpu_timers(p); | 1391 | run_posix_cpu_timers(p); |