diff options
43 files changed, 133 insertions, 29 deletions
diff --git a/arch/alpha/include/asm/Kbuild b/arch/alpha/include/asm/Kbuild index e858aa0ad8af..a52cbf178c3a 100644 --- a/arch/alpha/include/asm/Kbuild +++ b/arch/alpha/include/asm/Kbuild | |||
| @@ -4,6 +4,7 @@ generic-y += clkdev.h | |||
| 4 | generic-y += cputime.h | 4 | generic-y += cputime.h |
| 5 | generic-y += exec.h | 5 | generic-y += exec.h |
| 6 | generic-y += hash.h | 6 | generic-y += hash.h |
| 7 | generic-y += irq_work.h | ||
| 7 | generic-y += mcs_spinlock.h | 8 | generic-y += mcs_spinlock.h |
| 8 | generic-y += preempt.h | 9 | generic-y += preempt.h |
| 9 | generic-y += scatterlist.h | 10 | generic-y += scatterlist.h |
diff --git a/arch/arc/include/asm/Kbuild b/arch/arc/include/asm/Kbuild index e76fd79f32b0..b8fffc1a2ac2 100644 --- a/arch/arc/include/asm/Kbuild +++ b/arch/arc/include/asm/Kbuild | |||
| @@ -18,6 +18,7 @@ generic-y += ioctl.h | |||
| 18 | generic-y += ioctls.h | 18 | generic-y += ioctls.h |
| 19 | generic-y += ipcbuf.h | 19 | generic-y += ipcbuf.h |
| 20 | generic-y += irq_regs.h | 20 | generic-y += irq_regs.h |
| 21 | generic-y += irq_work.h | ||
| 21 | generic-y += kmap_types.h | 22 | generic-y += kmap_types.h |
| 22 | generic-y += kvm_para.h | 23 | generic-y += kvm_para.h |
| 23 | generic-y += local.h | 24 | generic-y += local.h |
diff --git a/arch/arm/include/asm/irq_work.h b/arch/arm/include/asm/irq_work.h new file mode 100644 index 000000000000..712d03e5973a --- /dev/null +++ b/arch/arm/include/asm/irq_work.h | |||
| @@ -0,0 +1,11 @@ | |||
| 1 | #ifndef __ASM_ARM_IRQ_WORK_H | ||
| 2 | #define __ASM_ARM_IRQ_WORK_H | ||
| 3 | |||
| 4 | #include <asm/smp_plat.h> | ||
| 5 | |||
| 6 | static inline bool arch_irq_work_has_interrupt(void) | ||
| 7 | { | ||
| 8 | return is_smp(); | ||
| 9 | } | ||
| 10 | |||
| 11 | #endif /* _ASM_ARM_IRQ_WORK_H */ | ||
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 39c74a2c3df9..13396d3d600e 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c | |||
| @@ -499,7 +499,7 @@ void arch_send_call_function_single_ipi(int cpu) | |||
| 499 | #ifdef CONFIG_IRQ_WORK | 499 | #ifdef CONFIG_IRQ_WORK |
| 500 | void arch_irq_work_raise(void) | 500 | void arch_irq_work_raise(void) |
| 501 | { | 501 | { |
| 502 | if (is_smp()) | 502 | if (arch_irq_work_has_interrupt()) |
| 503 | smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); | 503 | smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK); |
| 504 | } | 504 | } |
| 505 | #endif | 505 | #endif |
diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild index 0b3fcf86e6ba..c1968475cc4e 100644 --- a/arch/arm64/include/asm/Kbuild +++ b/arch/arm64/include/asm/Kbuild | |||
| @@ -9,8 +9,8 @@ generic-y += current.h | |||
| 9 | generic-y += delay.h | 9 | generic-y += delay.h |
| 10 | generic-y += div64.h | 10 | generic-y += div64.h |
| 11 | generic-y += dma.h | 11 | generic-y += dma.h |
| 12 | generic-y += emergency-restart.h | ||
| 13 | generic-y += early_ioremap.h | 12 | generic-y += early_ioremap.h |
| 13 | generic-y += emergency-restart.h | ||
| 14 | generic-y += errno.h | 14 | generic-y += errno.h |
| 15 | generic-y += ftrace.h | 15 | generic-y += ftrace.h |
| 16 | generic-y += hash.h | 16 | generic-y += hash.h |
diff --git a/arch/arm64/include/asm/irq_work.h b/arch/arm64/include/asm/irq_work.h new file mode 100644 index 000000000000..8e24ef3f7c82 --- /dev/null +++ b/arch/arm64/include/asm/irq_work.h | |||
| @@ -0,0 +1,11 @@ | |||
| 1 | #ifndef __ASM_IRQ_WORK_H | ||
| 2 | #define __ASM_IRQ_WORK_H | ||
| 3 | |||
| 4 | #include <asm/smp.h> | ||
| 5 | |||
| 6 | static inline bool arch_irq_work_has_interrupt(void) | ||
| 7 | { | ||
| 8 | return !!__smp_cross_call; | ||
| 9 | } | ||
| 10 | |||
| 11 | #endif /* __ASM_IRQ_WORK_H */ | ||
diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index a498f2cd2c2a..780f82c827b6 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h | |||
| @@ -48,6 +48,8 @@ extern void smp_init_cpus(void); | |||
| 48 | */ | 48 | */ |
| 49 | extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); | 49 | extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); |
| 50 | 50 | ||
| 51 | extern void (*__smp_cross_call)(const struct cpumask *, unsigned int); | ||
| 52 | |||
| 51 | /* | 53 | /* |
| 52 | * Called from the secondary holding pen, this is the secondary CPU entry point. | 54 | * Called from the secondary holding pen, this is the secondary CPU entry point. |
| 53 | */ | 55 | */ |
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 474339718105..b06d1d90ee8c 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c | |||
| @@ -470,7 +470,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) | |||
| 470 | } | 470 | } |
| 471 | } | 471 | } |
| 472 | 472 | ||
| 473 | static void (*__smp_cross_call)(const struct cpumask *, unsigned int); | 473 | void (*__smp_cross_call)(const struct cpumask *, unsigned int); |
| 474 | 474 | ||
| 475 | void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int)) | 475 | void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int)) |
| 476 | { | 476 | { |
diff --git a/arch/avr32/include/asm/Kbuild b/arch/avr32/include/asm/Kbuild index 00a0f3ccd6eb..2a71b1cb9848 100644 --- a/arch/avr32/include/asm/Kbuild +++ b/arch/avr32/include/asm/Kbuild | |||
| @@ -9,6 +9,7 @@ generic-y += exec.h | |||
| 9 | generic-y += futex.h | 9 | generic-y += futex.h |
| 10 | generic-y += hash.h | 10 | generic-y += hash.h |
| 11 | generic-y += irq_regs.h | 11 | generic-y += irq_regs.h |
| 12 | generic-y += irq_work.h | ||
| 12 | generic-y += local.h | 13 | generic-y += local.h |
| 13 | generic-y += local64.h | 14 | generic-y += local64.h |
| 14 | generic-y += mcs_spinlock.h | 15 | generic-y += mcs_spinlock.h |
diff --git a/arch/blackfin/include/asm/Kbuild b/arch/blackfin/include/asm/Kbuild index 0d93b9a79ca9..46ed6bb9c679 100644 --- a/arch/blackfin/include/asm/Kbuild +++ b/arch/blackfin/include/asm/Kbuild | |||
| @@ -15,6 +15,7 @@ generic-y += hw_irq.h | |||
| 15 | generic-y += ioctl.h | 15 | generic-y += ioctl.h |
| 16 | generic-y += ipcbuf.h | 16 | generic-y += ipcbuf.h |
| 17 | generic-y += irq_regs.h | 17 | generic-y += irq_regs.h |
| 18 | generic-y += irq_work.h | ||
| 18 | generic-y += kdebug.h | 19 | generic-y += kdebug.h |
| 19 | generic-y += kmap_types.h | 20 | generic-y += kmap_types.h |
| 20 | generic-y += kvm_para.h | 21 | generic-y += kvm_para.h |
diff --git a/arch/c6x/include/asm/Kbuild b/arch/c6x/include/asm/Kbuild index 8dbdce8421b0..e77e0c1dbe75 100644 --- a/arch/c6x/include/asm/Kbuild +++ b/arch/c6x/include/asm/Kbuild | |||
| @@ -22,6 +22,7 @@ generic-y += ioctl.h | |||
| 22 | generic-y += ioctls.h | 22 | generic-y += ioctls.h |
| 23 | generic-y += ipcbuf.h | 23 | generic-y += ipcbuf.h |
| 24 | generic-y += irq_regs.h | 24 | generic-y += irq_regs.h |
| 25 | generic-y += irq_work.h | ||
| 25 | generic-y += kdebug.h | 26 | generic-y += kdebug.h |
| 26 | generic-y += kmap_types.h | 27 | generic-y += kmap_types.h |
| 27 | generic-y += local.h | 28 | generic-y += local.h |
diff --git a/arch/cris/include/asm/Kbuild b/arch/cris/include/asm/Kbuild index 31742dfadff9..802b94c4ca86 100644 --- a/arch/cris/include/asm/Kbuild +++ b/arch/cris/include/asm/Kbuild | |||
| @@ -8,6 +8,7 @@ generic-y += clkdev.h | |||
| 8 | generic-y += cputime.h | 8 | generic-y += cputime.h |
| 9 | generic-y += exec.h | 9 | generic-y += exec.h |
| 10 | generic-y += hash.h | 10 | generic-y += hash.h |
| 11 | generic-y += irq_work.h | ||
| 11 | generic-y += kvm_para.h | 12 | generic-y += kvm_para.h |
| 12 | generic-y += linkage.h | 13 | generic-y += linkage.h |
| 13 | generic-y += mcs_spinlock.h | 14 | generic-y += mcs_spinlock.h |
diff --git a/arch/frv/include/asm/Kbuild b/arch/frv/include/asm/Kbuild index 5b73921b6e9d..3caf05cabfc5 100644 --- a/arch/frv/include/asm/Kbuild +++ b/arch/frv/include/asm/Kbuild | |||
| @@ -3,6 +3,7 @@ generic-y += clkdev.h | |||
| 3 | generic-y += cputime.h | 3 | generic-y += cputime.h |
| 4 | generic-y += exec.h | 4 | generic-y += exec.h |
| 5 | generic-y += hash.h | 5 | generic-y += hash.h |
| 6 | generic-y += irq_work.h | ||
| 6 | generic-y += mcs_spinlock.h | 7 | generic-y += mcs_spinlock.h |
| 7 | generic-y += preempt.h | 8 | generic-y += preempt.h |
| 8 | generic-y += scatterlist.h | 9 | generic-y += scatterlist.h |
diff --git a/arch/hexagon/include/asm/Kbuild b/arch/hexagon/include/asm/Kbuild index 0e69796b58c7..5f234a5a2320 100644 --- a/arch/hexagon/include/asm/Kbuild +++ b/arch/hexagon/include/asm/Kbuild | |||
| @@ -23,6 +23,7 @@ generic-y += ioctls.h | |||
| 23 | generic-y += iomap.h | 23 | generic-y += iomap.h |
| 24 | generic-y += ipcbuf.h | 24 | generic-y += ipcbuf.h |
| 25 | generic-y += irq_regs.h | 25 | generic-y += irq_regs.h |
| 26 | generic-y += irq_work.h | ||
| 26 | generic-y += kdebug.h | 27 | generic-y += kdebug.h |
| 27 | generic-y += kmap_types.h | 28 | generic-y += kmap_types.h |
| 28 | generic-y += local.h | 29 | generic-y += local.h |
diff --git a/arch/ia64/include/asm/Kbuild b/arch/ia64/include/asm/Kbuild index e8317d2d6c8d..747320be9d0e 100644 --- a/arch/ia64/include/asm/Kbuild +++ b/arch/ia64/include/asm/Kbuild | |||
| @@ -2,6 +2,7 @@ | |||
| 2 | generic-y += clkdev.h | 2 | generic-y += clkdev.h |
| 3 | generic-y += exec.h | 3 | generic-y += exec.h |
| 4 | generic-y += hash.h | 4 | generic-y += hash.h |
| 5 | generic-y += irq_work.h | ||
| 5 | generic-y += kvm_para.h | 6 | generic-y += kvm_para.h |
| 6 | generic-y += mcs_spinlock.h | 7 | generic-y += mcs_spinlock.h |
| 7 | generic-y += preempt.h | 8 | generic-y += preempt.h |
diff --git a/arch/m32r/include/asm/Kbuild b/arch/m32r/include/asm/Kbuild index accc10a3dc78..e02448b0648b 100644 --- a/arch/m32r/include/asm/Kbuild +++ b/arch/m32r/include/asm/Kbuild | |||
| @@ -3,6 +3,7 @@ generic-y += clkdev.h | |||
| 3 | generic-y += cputime.h | 3 | generic-y += cputime.h |
| 4 | generic-y += exec.h | 4 | generic-y += exec.h |
| 5 | generic-y += hash.h | 5 | generic-y += hash.h |
| 6 | generic-y += irq_work.h | ||
| 6 | generic-y += mcs_spinlock.h | 7 | generic-y += mcs_spinlock.h |
| 7 | generic-y += module.h | 8 | generic-y += module.h |
| 8 | generic-y += preempt.h | 9 | generic-y += preempt.h |
diff --git a/arch/m68k/include/asm/Kbuild b/arch/m68k/include/asm/Kbuild index c67c94a2d672..dbaf9f3065e8 100644 --- a/arch/m68k/include/asm/Kbuild +++ b/arch/m68k/include/asm/Kbuild | |||
| @@ -11,6 +11,7 @@ generic-y += hw_irq.h | |||
| 11 | generic-y += ioctl.h | 11 | generic-y += ioctl.h |
| 12 | generic-y += ipcbuf.h | 12 | generic-y += ipcbuf.h |
| 13 | generic-y += irq_regs.h | 13 | generic-y += irq_regs.h |
| 14 | generic-y += irq_work.h | ||
| 14 | generic-y += kdebug.h | 15 | generic-y += kdebug.h |
| 15 | generic-y += kmap_types.h | 16 | generic-y += kmap_types.h |
| 16 | generic-y += kvm_para.h | 17 | generic-y += kvm_para.h |
diff --git a/arch/metag/include/asm/Kbuild b/arch/metag/include/asm/Kbuild index c29ead89a317..7b8111c8f937 100644 --- a/arch/metag/include/asm/Kbuild +++ b/arch/metag/include/asm/Kbuild | |||
| @@ -19,6 +19,7 @@ generic-y += ioctl.h | |||
| 19 | generic-y += ioctls.h | 19 | generic-y += ioctls.h |
| 20 | generic-y += ipcbuf.h | 20 | generic-y += ipcbuf.h |
| 21 | generic-y += irq_regs.h | 21 | generic-y += irq_regs.h |
| 22 | generic-y += irq_work.h | ||
| 22 | generic-y += kdebug.h | 23 | generic-y += kdebug.h |
| 23 | generic-y += kmap_types.h | 24 | generic-y += kmap_types.h |
| 24 | generic-y += kvm_para.h | 25 | generic-y += kvm_para.h |
diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild index 27a3acda6c19..448143b8cabd 100644 --- a/arch/microblaze/include/asm/Kbuild +++ b/arch/microblaze/include/asm/Kbuild | |||
| @@ -5,6 +5,7 @@ generic-y += cputime.h | |||
| 5 | generic-y += device.h | 5 | generic-y += device.h |
| 6 | generic-y += exec.h | 6 | generic-y += exec.h |
| 7 | generic-y += hash.h | 7 | generic-y += hash.h |
| 8 | generic-y += irq_work.h | ||
| 8 | generic-y += mcs_spinlock.h | 9 | generic-y += mcs_spinlock.h |
| 9 | generic-y += preempt.h | 10 | generic-y += preempt.h |
| 10 | generic-y += scatterlist.h | 11 | generic-y += scatterlist.h |
diff --git a/arch/mips/include/asm/Kbuild b/arch/mips/include/asm/Kbuild index 335e5290ec75..57012ef1f51e 100644 --- a/arch/mips/include/asm/Kbuild +++ b/arch/mips/include/asm/Kbuild | |||
| @@ -3,6 +3,7 @@ generic-y += cputime.h | |||
| 3 | generic-y += current.h | 3 | generic-y += current.h |
| 4 | generic-y += emergency-restart.h | 4 | generic-y += emergency-restart.h |
| 5 | generic-y += hash.h | 5 | generic-y += hash.h |
| 6 | generic-y += irq_work.h | ||
| 6 | generic-y += local64.h | 7 | generic-y += local64.h |
| 7 | generic-y += mcs_spinlock.h | 8 | generic-y += mcs_spinlock.h |
| 8 | generic-y += mutex.h | 9 | generic-y += mutex.h |
diff --git a/arch/mn10300/include/asm/Kbuild b/arch/mn10300/include/asm/Kbuild index ecbd6676bd33..77eb1a68d13b 100644 --- a/arch/mn10300/include/asm/Kbuild +++ b/arch/mn10300/include/asm/Kbuild | |||
| @@ -4,6 +4,7 @@ generic-y += clkdev.h | |||
| 4 | generic-y += cputime.h | 4 | generic-y += cputime.h |
| 5 | generic-y += exec.h | 5 | generic-y += exec.h |
| 6 | generic-y += hash.h | 6 | generic-y += hash.h |
| 7 | generic-y += irq_work.h | ||
| 7 | generic-y += mcs_spinlock.h | 8 | generic-y += mcs_spinlock.h |
| 8 | generic-y += preempt.h | 9 | generic-y += preempt.h |
| 9 | generic-y += scatterlist.h | 10 | generic-y += scatterlist.h |
diff --git a/arch/openrisc/include/asm/Kbuild b/arch/openrisc/include/asm/Kbuild index 480af0d9c2f5..89b61d7dc790 100644 --- a/arch/openrisc/include/asm/Kbuild +++ b/arch/openrisc/include/asm/Kbuild | |||
| @@ -31,6 +31,7 @@ generic-y += ioctl.h | |||
| 31 | generic-y += ioctls.h | 31 | generic-y += ioctls.h |
| 32 | generic-y += ipcbuf.h | 32 | generic-y += ipcbuf.h |
| 33 | generic-y += irq_regs.h | 33 | generic-y += irq_regs.h |
| 34 | generic-y += irq_work.h | ||
| 34 | generic-y += kdebug.h | 35 | generic-y += kdebug.h |
| 35 | generic-y += kmap_types.h | 36 | generic-y += kmap_types.h |
| 36 | generic-y += kvm_para.h | 37 | generic-y += kvm_para.h |
diff --git a/arch/parisc/include/asm/Kbuild b/arch/parisc/include/asm/Kbuild index ecf25e6678ad..ffb024b8423f 100644 --- a/arch/parisc/include/asm/Kbuild +++ b/arch/parisc/include/asm/Kbuild | |||
| @@ -10,6 +10,7 @@ generic-y += exec.h | |||
| 10 | generic-y += hash.h | 10 | generic-y += hash.h |
| 11 | generic-y += hw_irq.h | 11 | generic-y += hw_irq.h |
| 12 | generic-y += irq_regs.h | 12 | generic-y += irq_regs.h |
| 13 | generic-y += irq_work.h | ||
| 13 | generic-y += kdebug.h | 14 | generic-y += kdebug.h |
| 14 | generic-y += kvm_para.h | 15 | generic-y += kvm_para.h |
| 15 | generic-y += local.h | 16 | generic-y += local.h |
diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild index 7f23f162ce9c..31e8f59aff38 100644 --- a/arch/powerpc/include/asm/Kbuild +++ b/arch/powerpc/include/asm/Kbuild | |||
| @@ -1,6 +1,7 @@ | |||
| 1 | 1 | ||
| 2 | generic-y += clkdev.h | 2 | generic-y += clkdev.h |
| 3 | generic-y += hash.h | 3 | generic-y += hash.h |
| 4 | generic-y += irq_work.h | ||
| 4 | generic-y += mcs_spinlock.h | 5 | generic-y += mcs_spinlock.h |
| 5 | generic-y += preempt.h | 6 | generic-y += preempt.h |
| 6 | generic-y += rwsem.h | 7 | generic-y += rwsem.h |
diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild index b3fea0722ff1..773f86676588 100644 --- a/arch/s390/include/asm/Kbuild +++ b/arch/s390/include/asm/Kbuild | |||
| @@ -2,6 +2,7 @@ | |||
| 2 | 2 | ||
| 3 | generic-y += clkdev.h | 3 | generic-y += clkdev.h |
| 4 | generic-y += hash.h | 4 | generic-y += hash.h |
| 5 | generic-y += irq_work.h | ||
| 5 | generic-y += mcs_spinlock.h | 6 | generic-y += mcs_spinlock.h |
| 6 | generic-y += preempt.h | 7 | generic-y += preempt.h |
| 7 | generic-y += scatterlist.h | 8 | generic-y += scatterlist.h |
diff --git a/arch/score/include/asm/Kbuild b/arch/score/include/asm/Kbuild index d26c48fc93c9..3fe5681744f1 100644 --- a/arch/score/include/asm/Kbuild +++ b/arch/score/include/asm/Kbuild | |||
| @@ -6,6 +6,7 @@ generic-y += barrier.h | |||
| 6 | generic-y += clkdev.h | 6 | generic-y += clkdev.h |
| 7 | generic-y += cputime.h | 7 | generic-y += cputime.h |
| 8 | generic-y += hash.h | 8 | generic-y += hash.h |
| 9 | generic-y += irq_work.h | ||
| 9 | generic-y += mcs_spinlock.h | 10 | generic-y += mcs_spinlock.h |
| 10 | generic-y += preempt.h | 11 | generic-y += preempt.h |
| 11 | generic-y += scatterlist.h | 12 | generic-y += scatterlist.h |
diff --git a/arch/sh/include/asm/Kbuild b/arch/sh/include/asm/Kbuild index c19e47dacb31..5a6c9acff0d2 100644 --- a/arch/sh/include/asm/Kbuild +++ b/arch/sh/include/asm/Kbuild | |||
| @@ -12,6 +12,7 @@ generic-y += hash.h | |||
| 12 | generic-y += ioctl.h | 12 | generic-y += ioctl.h |
| 13 | generic-y += ipcbuf.h | 13 | generic-y += ipcbuf.h |
| 14 | generic-y += irq_regs.h | 14 | generic-y += irq_regs.h |
| 15 | generic-y += irq_work.h | ||
| 15 | generic-y += kvm_para.h | 16 | generic-y += kvm_para.h |
| 16 | generic-y += local.h | 17 | generic-y += local.h |
| 17 | generic-y += local64.h | 18 | generic-y += local64.h |
diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild index cdd1b447bb6c..f5f94ce1692c 100644 --- a/arch/sparc/include/asm/Kbuild +++ b/arch/sparc/include/asm/Kbuild | |||
| @@ -8,6 +8,7 @@ generic-y += emergency-restart.h | |||
| 8 | generic-y += exec.h | 8 | generic-y += exec.h |
| 9 | generic-y += hash.h | 9 | generic-y += hash.h |
| 10 | generic-y += irq_regs.h | 10 | generic-y += irq_regs.h |
| 11 | generic-y += irq_work.h | ||
| 11 | generic-y += linkage.h | 12 | generic-y += linkage.h |
| 12 | generic-y += local.h | 13 | generic-y += local.h |
| 13 | generic-y += local64.h | 14 | generic-y += local64.h |
diff --git a/arch/tile/include/asm/Kbuild b/arch/tile/include/asm/Kbuild index 0aa5675e7025..e6462b8a6284 100644 --- a/arch/tile/include/asm/Kbuild +++ b/arch/tile/include/asm/Kbuild | |||
| @@ -17,6 +17,7 @@ generic-y += ioctl.h | |||
| 17 | generic-y += ioctls.h | 17 | generic-y += ioctls.h |
| 18 | generic-y += ipcbuf.h | 18 | generic-y += ipcbuf.h |
| 19 | generic-y += irq_regs.h | 19 | generic-y += irq_regs.h |
| 20 | generic-y += irq_work.h | ||
| 20 | generic-y += local.h | 21 | generic-y += local.h |
| 21 | generic-y += local64.h | 22 | generic-y += local64.h |
| 22 | generic-y += mcs_spinlock.h | 23 | generic-y += mcs_spinlock.h |
diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index 7bd64aa2e94a..244b12c8cb39 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild | |||
| @@ -14,6 +14,7 @@ generic-y += hash.h | |||
| 14 | generic-y += hw_irq.h | 14 | generic-y += hw_irq.h |
| 15 | generic-y += io.h | 15 | generic-y += io.h |
| 16 | generic-y += irq_regs.h | 16 | generic-y += irq_regs.h |
| 17 | generic-y += irq_work.h | ||
| 17 | generic-y += kdebug.h | 18 | generic-y += kdebug.h |
| 18 | generic-y += mcs_spinlock.h | 19 | generic-y += mcs_spinlock.h |
| 19 | generic-y += mutex.h | 20 | generic-y += mutex.h |
diff --git a/arch/unicore32/include/asm/Kbuild b/arch/unicore32/include/asm/Kbuild index 1e5fb872a4aa..5a2bb53faa42 100644 --- a/arch/unicore32/include/asm/Kbuild +++ b/arch/unicore32/include/asm/Kbuild | |||
| @@ -22,6 +22,7 @@ generic-y += ioctl.h | |||
| 22 | generic-y += ioctls.h | 22 | generic-y += ioctls.h |
| 23 | generic-y += ipcbuf.h | 23 | generic-y += ipcbuf.h |
| 24 | generic-y += irq_regs.h | 24 | generic-y += irq_regs.h |
| 25 | generic-y += irq_work.h | ||
| 25 | generic-y += kdebug.h | 26 | generic-y += kdebug.h |
| 26 | generic-y += kmap_types.h | 27 | generic-y += kmap_types.h |
| 27 | generic-y += local.h | 28 | generic-y += local.h |
diff --git a/arch/x86/include/asm/irq_work.h b/arch/x86/include/asm/irq_work.h new file mode 100644 index 000000000000..78162f8e248b --- /dev/null +++ b/arch/x86/include/asm/irq_work.h | |||
| @@ -0,0 +1,11 @@ | |||
| 1 | #ifndef _ASM_IRQ_WORK_H | ||
| 2 | #define _ASM_IRQ_WORK_H | ||
| 3 | |||
| 4 | #include <asm/processor.h> | ||
| 5 | |||
| 6 | static inline bool arch_irq_work_has_interrupt(void) | ||
| 7 | { | ||
| 8 | return cpu_has_apic; | ||
| 9 | } | ||
| 10 | |||
| 11 | #endif /* _ASM_IRQ_WORK_H */ | ||
diff --git a/arch/x86/kernel/irq_work.c b/arch/x86/kernel/irq_work.c index 1de84e3ab4e0..15d741ddfeeb 100644 --- a/arch/x86/kernel/irq_work.c +++ b/arch/x86/kernel/irq_work.c | |||
| @@ -41,7 +41,7 @@ __visible void smp_trace_irq_work_interrupt(struct pt_regs *regs) | |||
| 41 | void arch_irq_work_raise(void) | 41 | void arch_irq_work_raise(void) |
| 42 | { | 42 | { |
| 43 | #ifdef CONFIG_X86_LOCAL_APIC | 43 | #ifdef CONFIG_X86_LOCAL_APIC |
| 44 | if (!cpu_has_apic) | 44 | if (!arch_irq_work_has_interrupt()) |
| 45 | return; | 45 | return; |
| 46 | 46 | ||
| 47 | apic->send_IPI_self(IRQ_WORK_VECTOR); | 47 | apic->send_IPI_self(IRQ_WORK_VECTOR); |
diff --git a/arch/xtensa/include/asm/Kbuild b/arch/xtensa/include/asm/Kbuild index c3d20ba6eb86..105d38922c44 100644 --- a/arch/xtensa/include/asm/Kbuild +++ b/arch/xtensa/include/asm/Kbuild | |||
| @@ -12,6 +12,7 @@ generic-y += hardirq.h | |||
| 12 | generic-y += hash.h | 12 | generic-y += hash.h |
| 13 | generic-y += ioctl.h | 13 | generic-y += ioctl.h |
| 14 | generic-y += irq_regs.h | 14 | generic-y += irq_regs.h |
| 15 | generic-y += irq_work.h | ||
| 15 | generic-y += kdebug.h | 16 | generic-y += kdebug.h |
| 16 | generic-y += kmap_types.h | 17 | generic-y += kmap_types.h |
| 17 | generic-y += kvm_para.h | 18 | generic-y += kvm_para.h |
diff --git a/include/asm-generic/irq_work.h b/include/asm-generic/irq_work.h new file mode 100644 index 000000000000..a44f452c6590 --- /dev/null +++ b/include/asm-generic/irq_work.h | |||
| @@ -0,0 +1,10 @@ | |||
| 1 | #ifndef __ASM_IRQ_WORK_H | ||
| 2 | #define __ASM_IRQ_WORK_H | ||
| 3 | |||
| 4 | static inline bool arch_irq_work_has_interrupt(void) | ||
| 5 | { | ||
| 6 | return false; | ||
| 7 | } | ||
| 8 | |||
| 9 | #endif /* __ASM_IRQ_WORK_H */ | ||
| 10 | |||
diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h index bf9422c3aefe..bf3fe719c7ce 100644 --- a/include/linux/irq_work.h +++ b/include/linux/irq_work.h | |||
| @@ -39,9 +39,12 @@ bool irq_work_queue_on(struct irq_work *work, int cpu); | |||
| 39 | #endif | 39 | #endif |
| 40 | 40 | ||
| 41 | void irq_work_run(void); | 41 | void irq_work_run(void); |
| 42 | void irq_work_tick(void); | ||
| 42 | void irq_work_sync(struct irq_work *work); | 43 | void irq_work_sync(struct irq_work *work); |
| 43 | 44 | ||
| 44 | #ifdef CONFIG_IRQ_WORK | 45 | #ifdef CONFIG_IRQ_WORK |
| 46 | #include <asm/irq_work.h> | ||
| 47 | |||
| 45 | bool irq_work_needs_cpu(void); | 48 | bool irq_work_needs_cpu(void); |
| 46 | #else | 49 | #else |
| 47 | static inline bool irq_work_needs_cpu(void) { return false; } | 50 | static inline bool irq_work_needs_cpu(void) { return false; } |
diff --git a/include/linux/tick.h b/include/linux/tick.h index 9a82c7dc3fdd..595ee86f5e0d 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h | |||
| @@ -181,14 +181,12 @@ static inline bool tick_nohz_full_cpu(int cpu) | |||
| 181 | return cpumask_test_cpu(cpu, tick_nohz_full_mask); | 181 | return cpumask_test_cpu(cpu, tick_nohz_full_mask); |
| 182 | } | 182 | } |
| 183 | 183 | ||
| 184 | extern void tick_nohz_init(void); | ||
| 185 | extern void __tick_nohz_full_check(void); | 184 | extern void __tick_nohz_full_check(void); |
| 186 | extern void tick_nohz_full_kick(void); | 185 | extern void tick_nohz_full_kick(void); |
| 187 | extern void tick_nohz_full_kick_cpu(int cpu); | 186 | extern void tick_nohz_full_kick_cpu(int cpu); |
| 188 | extern void tick_nohz_full_kick_all(void); | 187 | extern void tick_nohz_full_kick_all(void); |
| 189 | extern void __tick_nohz_task_switch(struct task_struct *tsk); | 188 | extern void __tick_nohz_task_switch(struct task_struct *tsk); |
| 190 | #else | 189 | #else |
| 191 | static inline void tick_nohz_init(void) { } | ||
| 192 | static inline bool tick_nohz_full_enabled(void) { return false; } | 190 | static inline bool tick_nohz_full_enabled(void) { return false; } |
| 193 | static inline bool tick_nohz_full_cpu(int cpu) { return false; } | 191 | static inline bool tick_nohz_full_cpu(int cpu) { return false; } |
| 194 | static inline void __tick_nohz_full_check(void) { } | 192 | static inline void __tick_nohz_full_check(void) { } |
diff --git a/init/main.c b/init/main.c index bb1aed928f21..8af2f1abfe38 100644 --- a/init/main.c +++ b/init/main.c | |||
| @@ -577,7 +577,6 @@ asmlinkage __visible void __init start_kernel(void) | |||
| 577 | local_irq_disable(); | 577 | local_irq_disable(); |
| 578 | idr_init_cache(); | 578 | idr_init_cache(); |
| 579 | rcu_init(); | 579 | rcu_init(); |
| 580 | tick_nohz_init(); | ||
| 581 | context_tracking_init(); | 580 | context_tracking_init(); |
| 582 | radix_tree_init(); | 581 | radix_tree_init(); |
| 583 | /* init some links before init_ISA_irqs() */ | 582 | /* init some links before init_ISA_irqs() */ |
diff --git a/kernel/irq_work.c b/kernel/irq_work.c index e6bcbe756663..385b85aded19 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c | |||
| @@ -115,8 +115,10 @@ bool irq_work_needs_cpu(void) | |||
| 115 | 115 | ||
| 116 | raised = &__get_cpu_var(raised_list); | 116 | raised = &__get_cpu_var(raised_list); |
| 117 | lazy = &__get_cpu_var(lazy_list); | 117 | lazy = &__get_cpu_var(lazy_list); |
| 118 | if (llist_empty(raised) && llist_empty(lazy)) | 118 | |
| 119 | return false; | 119 | if (llist_empty(raised) || arch_irq_work_has_interrupt()) |
| 120 | if (llist_empty(lazy)) | ||
| 121 | return false; | ||
| 120 | 122 | ||
| 121 | /* All work should have been flushed before going offline */ | 123 | /* All work should have been flushed before going offline */ |
| 122 | WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); | 124 | WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); |
| @@ -171,6 +173,15 @@ void irq_work_run(void) | |||
| 171 | } | 173 | } |
| 172 | EXPORT_SYMBOL_GPL(irq_work_run); | 174 | EXPORT_SYMBOL_GPL(irq_work_run); |
| 173 | 175 | ||
| 176 | void irq_work_tick(void) | ||
| 177 | { | ||
| 178 | struct llist_head *raised = &__get_cpu_var(raised_list); | ||
| 179 | |||
| 180 | if (!llist_empty(raised) && !arch_irq_work_has_interrupt()) | ||
| 181 | irq_work_run_list(raised); | ||
| 182 | irq_work_run_list(&__get_cpu_var(lazy_list)); | ||
| 183 | } | ||
| 184 | |||
| 174 | /* | 185 | /* |
| 175 | * Synchronize against the irq_work @entry, ensures the entry is not | 186 | * Synchronize against the irq_work @entry, ensures the entry is not |
| 176 | * currently in use. | 187 | * currently in use. |
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index 0a0608edeb26..052b4b53c3d6 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c | |||
| @@ -400,4 +400,5 @@ void tick_resume(void) | |||
| 400 | void __init tick_init(void) | 400 | void __init tick_init(void) |
| 401 | { | 401 | { |
| 402 | tick_broadcast_init(); | 402 | tick_broadcast_init(); |
| 403 | tick_nohz_init(); | ||
| 403 | } | 404 | } |
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index c19c1d84b6f3..366aeb4f2c66 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h | |||
| @@ -99,6 +99,13 @@ static inline int tick_broadcast_oneshot_active(void) { return 0; } | |||
| 99 | static inline bool tick_broadcast_oneshot_available(void) { return false; } | 99 | static inline bool tick_broadcast_oneshot_available(void) { return false; } |
| 100 | #endif /* !TICK_ONESHOT */ | 100 | #endif /* !TICK_ONESHOT */ |
| 101 | 101 | ||
| 102 | /* NO_HZ_FULL internal */ | ||
| 103 | #ifdef CONFIG_NO_HZ_FULL | ||
| 104 | extern void tick_nohz_init(void); | ||
| 105 | # else | ||
| 106 | static inline void tick_nohz_init(void) { } | ||
| 107 | #endif | ||
| 108 | |||
| 102 | /* | 109 | /* |
| 103 | * Broadcasting support | 110 | * Broadcasting support |
| 104 | */ | 111 | */ |
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index f654a8a298fa..5a9ff243588c 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
| @@ -295,22 +295,12 @@ out: | |||
| 295 | /* Parse the boot-time nohz CPU list from the kernel parameters. */ | 295 | /* Parse the boot-time nohz CPU list from the kernel parameters. */ |
| 296 | static int __init tick_nohz_full_setup(char *str) | 296 | static int __init tick_nohz_full_setup(char *str) |
| 297 | { | 297 | { |
| 298 | int cpu; | ||
| 299 | |||
| 300 | alloc_bootmem_cpumask_var(&tick_nohz_full_mask); | 298 | alloc_bootmem_cpumask_var(&tick_nohz_full_mask); |
| 301 | alloc_bootmem_cpumask_var(&housekeeping_mask); | ||
| 302 | if (cpulist_parse(str, tick_nohz_full_mask) < 0) { | 299 | if (cpulist_parse(str, tick_nohz_full_mask) < 0) { |
| 303 | pr_warning("NOHZ: Incorrect nohz_full cpumask\n"); | 300 | pr_warning("NOHZ: Incorrect nohz_full cpumask\n"); |
| 301 | free_bootmem_cpumask_var(tick_nohz_full_mask); | ||
| 304 | return 1; | 302 | return 1; |
| 305 | } | 303 | } |
| 306 | |||
| 307 | cpu = smp_processor_id(); | ||
| 308 | if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) { | ||
| 309 | pr_warning("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", cpu); | ||
| 310 | cpumask_clear_cpu(cpu, tick_nohz_full_mask); | ||
| 311 | } | ||
| 312 | cpumask_andnot(housekeeping_mask, | ||
| 313 | cpu_possible_mask, tick_nohz_full_mask); | ||
| 314 | tick_nohz_full_running = true; | 304 | tick_nohz_full_running = true; |
| 315 | 305 | ||
| 316 | return 1; | 306 | return 1; |
| @@ -349,18 +339,11 @@ static int tick_nohz_init_all(void) | |||
| 349 | 339 | ||
| 350 | #ifdef CONFIG_NO_HZ_FULL_ALL | 340 | #ifdef CONFIG_NO_HZ_FULL_ALL |
| 351 | if (!alloc_cpumask_var(&tick_nohz_full_mask, GFP_KERNEL)) { | 341 | if (!alloc_cpumask_var(&tick_nohz_full_mask, GFP_KERNEL)) { |
| 352 | pr_err("NO_HZ: Can't allocate full dynticks cpumask\n"); | 342 | WARN(1, "NO_HZ: Can't allocate full dynticks cpumask\n"); |
| 353 | return err; | ||
| 354 | } | ||
| 355 | if (!alloc_cpumask_var(&housekeeping_mask, GFP_KERNEL)) { | ||
| 356 | pr_err("NO_HZ: Can't allocate not-full dynticks cpumask\n"); | ||
| 357 | return err; | 343 | return err; |
| 358 | } | 344 | } |
| 359 | err = 0; | 345 | err = 0; |
| 360 | cpumask_setall(tick_nohz_full_mask); | 346 | cpumask_setall(tick_nohz_full_mask); |
| 361 | cpumask_clear_cpu(smp_processor_id(), tick_nohz_full_mask); | ||
| 362 | cpumask_clear(housekeeping_mask); | ||
| 363 | cpumask_set_cpu(smp_processor_id(), housekeeping_mask); | ||
| 364 | tick_nohz_full_running = true; | 347 | tick_nohz_full_running = true; |
| 365 | #endif | 348 | #endif |
| 366 | return err; | 349 | return err; |
| @@ -375,6 +358,37 @@ void __init tick_nohz_init(void) | |||
| 375 | return; | 358 | return; |
| 376 | } | 359 | } |
| 377 | 360 | ||
| 361 | if (!alloc_cpumask_var(&housekeeping_mask, GFP_KERNEL)) { | ||
| 362 | WARN(1, "NO_HZ: Can't allocate not-full dynticks cpumask\n"); | ||
| 363 | cpumask_clear(tick_nohz_full_mask); | ||
| 364 | tick_nohz_full_running = false; | ||
| 365 | return; | ||
| 366 | } | ||
| 367 | |||
| 368 | /* | ||
| 369 | * Full dynticks uses irq work to drive the tick rescheduling on safe | ||
| 370 | * locking contexts. But then we need irq work to raise its own | ||
| 371 | * interrupts to avoid circular dependency on the tick | ||
| 372 | */ | ||
| 373 | if (!arch_irq_work_has_interrupt()) { | ||
| 374 | pr_warning("NO_HZ: Can't run full dynticks because arch doesn't " | ||
| 375 | "support irq work self-IPIs\n"); | ||
| 376 | cpumask_clear(tick_nohz_full_mask); | ||
| 377 | cpumask_copy(housekeeping_mask, cpu_possible_mask); | ||
| 378 | tick_nohz_full_running = false; | ||
| 379 | return; | ||
| 380 | } | ||
| 381 | |||
| 382 | cpu = smp_processor_id(); | ||
| 383 | |||
| 384 | if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) { | ||
| 385 | pr_warning("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", cpu); | ||
| 386 | cpumask_clear_cpu(cpu, tick_nohz_full_mask); | ||
| 387 | } | ||
| 388 | |||
| 389 | cpumask_andnot(housekeeping_mask, | ||
| 390 | cpu_possible_mask, tick_nohz_full_mask); | ||
| 391 | |||
| 378 | for_each_cpu(cpu, tick_nohz_full_mask) | 392 | for_each_cpu(cpu, tick_nohz_full_mask) |
| 379 | context_tracking_cpu_set(cpu); | 393 | context_tracking_cpu_set(cpu); |
| 380 | 394 | ||
diff --git a/kernel/time/timer.c b/kernel/time/timer.c index aca5dfe2fa3d..9bbb8344ed3b 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c | |||
| @@ -1385,7 +1385,7 @@ void update_process_times(int user_tick) | |||
| 1385 | rcu_check_callbacks(cpu, user_tick); | 1385 | rcu_check_callbacks(cpu, user_tick); |
| 1386 | #ifdef CONFIG_IRQ_WORK | 1386 | #ifdef CONFIG_IRQ_WORK |
| 1387 | if (in_irq()) | 1387 | if (in_irq()) |
| 1388 | irq_work_run(); | 1388 | irq_work_tick(); |
| 1389 | #endif | 1389 | #endif |
| 1390 | scheduler_tick(); | 1390 | scheduler_tick(); |
| 1391 | run_posix_cpu_timers(p); | 1391 | run_posix_cpu_timers(p); |
