aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/alpha/Kconfig1
-rw-r--r--arch/arm/Kconfig1
-rw-r--r--arch/arm64/Kconfig1
-rw-r--r--arch/blackfin/Kconfig2
-rw-r--r--arch/frv/Kconfig1
-rw-r--r--arch/hexagon/Kconfig2
-rw-r--r--arch/ia64/Kconfig1
-rw-r--r--arch/mips/Kconfig2
-rw-r--r--arch/parisc/Kconfig2
-rw-r--r--arch/powerpc/Kconfig2
-rw-r--r--arch/s390/Kconfig1
-rw-r--r--arch/sh/Kconfig4
-rw-r--r--arch/sparc/Kconfig1
-rw-r--r--arch/x86/Kconfig1
-rw-r--r--drivers/staging/iio/trigger/Kconfig1
-rw-r--r--include/linux/irq_work.h22
-rw-r--r--include/linux/printk.h3
-rw-r--r--include/linux/tick.h17
-rw-r--r--init/Kconfig5
-rw-r--r--kernel/irq/manage.c2
-rw-r--r--kernel/irq/spurious.c7
-rw-r--r--kernel/irq_work.c150
-rw-r--r--kernel/printk.c36
-rw-r--r--kernel/time/tick-sched.c7
-rw-r--r--kernel/timer.c1
25 files changed, 171 insertions, 102 deletions
diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig
index 9d5904cc7712..9b504af2e966 100644
--- a/arch/alpha/Kconfig
+++ b/arch/alpha/Kconfig
@@ -5,7 +5,6 @@ config ALPHA
5 select HAVE_IDE 5 select HAVE_IDE
6 select HAVE_OPROFILE 6 select HAVE_OPROFILE
7 select HAVE_SYSCALL_WRAPPERS 7 select HAVE_SYSCALL_WRAPPERS
8 select HAVE_IRQ_WORK
9 select HAVE_PCSPKR_PLATFORM 8 select HAVE_PCSPKR_PLATFORM
10 select HAVE_PERF_EVENTS 9 select HAVE_PERF_EVENTS
11 select HAVE_DMA_ATTRS 10 select HAVE_DMA_ATTRS
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 67874b82a4ed..9bbe760f2352 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -36,7 +36,6 @@ config ARM
36 select HAVE_GENERIC_HARDIRQS 36 select HAVE_GENERIC_HARDIRQS
37 select HAVE_HW_BREAKPOINT if (PERF_EVENTS && (CPU_V6 || CPU_V6K || CPU_V7)) 37 select HAVE_HW_BREAKPOINT if (PERF_EVENTS && (CPU_V6 || CPU_V6K || CPU_V7))
38 select HAVE_IDE if PCI || ISA || PCMCIA 38 select HAVE_IDE if PCI || ISA || PCMCIA
39 select HAVE_IRQ_WORK
40 select HAVE_KERNEL_GZIP 39 select HAVE_KERNEL_GZIP
41 select HAVE_KERNEL_LZMA 40 select HAVE_KERNEL_LZMA
42 select HAVE_KERNEL_LZO 41 select HAVE_KERNEL_LZO
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index f8f362aafee9..75e915b72471 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -21,7 +21,6 @@ config ARM64
21 select HAVE_GENERIC_DMA_COHERENT 21 select HAVE_GENERIC_DMA_COHERENT
22 select HAVE_GENERIC_HARDIRQS 22 select HAVE_GENERIC_HARDIRQS
23 select HAVE_HW_BREAKPOINT if PERF_EVENTS 23 select HAVE_HW_BREAKPOINT if PERF_EVENTS
24 select HAVE_IRQ_WORK
25 select HAVE_MEMBLOCK 24 select HAVE_MEMBLOCK
26 select HAVE_PERF_EVENTS 25 select HAVE_PERF_EVENTS
27 select IRQ_DOMAIN 26 select IRQ_DOMAIN
diff --git a/arch/blackfin/Kconfig b/arch/blackfin/Kconfig
index b6f3ad5441c5..67e4aaad78f5 100644
--- a/arch/blackfin/Kconfig
+++ b/arch/blackfin/Kconfig
@@ -24,7 +24,6 @@ config BLACKFIN
24 select HAVE_FUNCTION_TRACER 24 select HAVE_FUNCTION_TRACER
25 select HAVE_FUNCTION_TRACE_MCOUNT_TEST 25 select HAVE_FUNCTION_TRACE_MCOUNT_TEST
26 select HAVE_IDE 26 select HAVE_IDE
27 select HAVE_IRQ_WORK
28 select HAVE_KERNEL_GZIP if RAMKERNEL 27 select HAVE_KERNEL_GZIP if RAMKERNEL
29 select HAVE_KERNEL_BZIP2 if RAMKERNEL 28 select HAVE_KERNEL_BZIP2 if RAMKERNEL
30 select HAVE_KERNEL_LZMA if RAMKERNEL 29 select HAVE_KERNEL_LZMA if RAMKERNEL
@@ -38,7 +37,6 @@ config BLACKFIN
38 select HAVE_GENERIC_HARDIRQS 37 select HAVE_GENERIC_HARDIRQS
39 select GENERIC_ATOMIC64 38 select GENERIC_ATOMIC64
40 select GENERIC_IRQ_PROBE 39 select GENERIC_IRQ_PROBE
41 select IRQ_PER_CPU if SMP
42 select USE_GENERIC_SMP_HELPERS if SMP 40 select USE_GENERIC_SMP_HELPERS if SMP
43 select HAVE_NMI_WATCHDOG if NMI_WATCHDOG 41 select HAVE_NMI_WATCHDOG if NMI_WATCHDOG
44 select GENERIC_SMP_IDLE_THREAD 42 select GENERIC_SMP_IDLE_THREAD
diff --git a/arch/frv/Kconfig b/arch/frv/Kconfig
index 9d262645f667..17df48fc8f44 100644
--- a/arch/frv/Kconfig
+++ b/arch/frv/Kconfig
@@ -3,7 +3,6 @@ config FRV
3 default y 3 default y
4 select HAVE_IDE 4 select HAVE_IDE
5 select HAVE_ARCH_TRACEHOOK 5 select HAVE_ARCH_TRACEHOOK
6 select HAVE_IRQ_WORK
7 select HAVE_PERF_EVENTS 6 select HAVE_PERF_EVENTS
8 select HAVE_UID16 7 select HAVE_UID16
9 select HAVE_GENERIC_HARDIRQS 8 select HAVE_GENERIC_HARDIRQS
diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig
index 0744f7d7b1fd..e4decc6b8947 100644
--- a/arch/hexagon/Kconfig
+++ b/arch/hexagon/Kconfig
@@ -12,9 +12,7 @@ config HEXAGON
12 # select ARCH_WANT_OPTIONAL_GPIOLIB 12 # select ARCH_WANT_OPTIONAL_GPIOLIB
13 # select ARCH_REQUIRE_GPIOLIB 13 # select ARCH_REQUIRE_GPIOLIB
14 # select HAVE_CLK 14 # select HAVE_CLK
15 # select IRQ_PER_CPU
16 # select GENERIC_PENDING_IRQ if SMP 15 # select GENERIC_PENDING_IRQ if SMP
17 select HAVE_IRQ_WORK
18 select GENERIC_ATOMIC64 16 select GENERIC_ATOMIC64
19 select HAVE_PERF_EVENTS 17 select HAVE_PERF_EVENTS
20 select HAVE_GENERIC_HARDIRQS 18 select HAVE_GENERIC_HARDIRQS
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 3279646120e3..00c2e88f7755 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -29,7 +29,6 @@ config IA64
29 select ARCH_DISCARD_MEMBLOCK 29 select ARCH_DISCARD_MEMBLOCK
30 select GENERIC_IRQ_PROBE 30 select GENERIC_IRQ_PROBE
31 select GENERIC_PENDING_IRQ if SMP 31 select GENERIC_PENDING_IRQ if SMP
32 select IRQ_PER_CPU
33 select GENERIC_IRQ_SHOW 32 select GENERIC_IRQ_SHOW
34 select ARCH_WANT_OPTIONAL_GPIOLIB 33 select ARCH_WANT_OPTIONAL_GPIOLIB
35 select ARCH_HAVE_NMI_SAFE_CMPXCHG 34 select ARCH_HAVE_NMI_SAFE_CMPXCHG
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index 2ac626ab9d43..9becc44d9d7a 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -4,7 +4,6 @@ config MIPS
4 select HAVE_GENERIC_DMA_COHERENT 4 select HAVE_GENERIC_DMA_COHERENT
5 select HAVE_IDE 5 select HAVE_IDE
6 select HAVE_OPROFILE 6 select HAVE_OPROFILE
7 select HAVE_IRQ_WORK
8 select HAVE_PERF_EVENTS 7 select HAVE_PERF_EVENTS
9 select PERF_USE_VMALLOC 8 select PERF_USE_VMALLOC
10 select HAVE_ARCH_KGDB 9 select HAVE_ARCH_KGDB
@@ -2161,7 +2160,6 @@ source "mm/Kconfig"
2161config SMP 2160config SMP
2162 bool "Multi-Processing support" 2161 bool "Multi-Processing support"
2163 depends on SYS_SUPPORTS_SMP 2162 depends on SYS_SUPPORTS_SMP
2164 select IRQ_PER_CPU
2165 select USE_GENERIC_SMP_HELPERS 2163 select USE_GENERIC_SMP_HELPERS
2166 help 2164 help
2167 This enables support for systems with more than one CPU. If you have 2165 This enables support for systems with more than one CPU. If you have
diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig
index b77feffbadea..a32e34ecda9e 100644
--- a/arch/parisc/Kconfig
+++ b/arch/parisc/Kconfig
@@ -9,14 +9,12 @@ config PARISC
9 select RTC_DRV_GENERIC 9 select RTC_DRV_GENERIC
10 select INIT_ALL_POSSIBLE 10 select INIT_ALL_POSSIBLE
11 select BUG 11 select BUG
12 select HAVE_IRQ_WORK
13 select HAVE_PERF_EVENTS 12 select HAVE_PERF_EVENTS
14 select GENERIC_ATOMIC64 if !64BIT 13 select GENERIC_ATOMIC64 if !64BIT
15 select HAVE_GENERIC_HARDIRQS 14 select HAVE_GENERIC_HARDIRQS
16 select BROKEN_RODATA 15 select BROKEN_RODATA
17 select GENERIC_IRQ_PROBE 16 select GENERIC_IRQ_PROBE
18 select GENERIC_PCI_IOMAP 17 select GENERIC_PCI_IOMAP
19 select IRQ_PER_CPU
20 select ARCH_HAVE_NMI_SAFE_CMPXCHG 18 select ARCH_HAVE_NMI_SAFE_CMPXCHG
21 select GENERIC_SMP_IDLE_THREAD 19 select GENERIC_SMP_IDLE_THREAD
22 select GENERIC_STRNCPY_FROM_USER 20 select GENERIC_STRNCPY_FROM_USER
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 17903f1f356b..561ccca7b1a7 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -118,14 +118,12 @@ config PPC
118 select HAVE_SYSCALL_WRAPPERS if PPC64 118 select HAVE_SYSCALL_WRAPPERS if PPC64
119 select GENERIC_ATOMIC64 if PPC32 119 select GENERIC_ATOMIC64 if PPC32
120 select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE 120 select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE
121 select HAVE_IRQ_WORK
122 select HAVE_PERF_EVENTS 121 select HAVE_PERF_EVENTS
123 select HAVE_REGS_AND_STACK_ACCESS_API 122 select HAVE_REGS_AND_STACK_ACCESS_API
124 select HAVE_HW_BREAKPOINT if PERF_EVENTS && PPC_BOOK3S_64 123 select HAVE_HW_BREAKPOINT if PERF_EVENTS && PPC_BOOK3S_64
125 select HAVE_GENERIC_HARDIRQS 124 select HAVE_GENERIC_HARDIRQS
126 select ARCH_WANT_IPC_PARSE_VERSION 125 select ARCH_WANT_IPC_PARSE_VERSION
127 select SPARSE_IRQ 126 select SPARSE_IRQ
128 select IRQ_PER_CPU
129 select IRQ_DOMAIN 127 select IRQ_DOMAIN
130 select GENERIC_IRQ_SHOW 128 select GENERIC_IRQ_SHOW
131 select GENERIC_IRQ_SHOW_LEVEL 129 select GENERIC_IRQ_SHOW_LEVEL
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index b5ea38c25647..c15ba7d1be64 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -78,7 +78,6 @@ config S390
78 select HAVE_KVM if 64BIT 78 select HAVE_KVM if 64BIT
79 select HAVE_ARCH_TRACEHOOK 79 select HAVE_ARCH_TRACEHOOK
80 select INIT_ALL_POSSIBLE 80 select INIT_ALL_POSSIBLE
81 select HAVE_IRQ_WORK
82 select HAVE_PERF_EVENTS 81 select HAVE_PERF_EVENTS
83 select ARCH_HAVE_NMI_SAFE_CMPXCHG 82 select ARCH_HAVE_NMI_SAFE_CMPXCHG
84 select HAVE_DEBUG_KMEMLEAK 83 select HAVE_DEBUG_KMEMLEAK
diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig
index babc2b826c5c..9c833c585871 100644
--- a/arch/sh/Kconfig
+++ b/arch/sh/Kconfig
@@ -11,7 +11,6 @@ config SUPERH
11 select HAVE_ARCH_TRACEHOOK 11 select HAVE_ARCH_TRACEHOOK
12 select HAVE_DMA_API_DEBUG 12 select HAVE_DMA_API_DEBUG
13 select HAVE_DMA_ATTRS 13 select HAVE_DMA_ATTRS
14 select HAVE_IRQ_WORK
15 select HAVE_PERF_EVENTS 14 select HAVE_PERF_EVENTS
16 select HAVE_DEBUG_BUGVERBOSE 15 select HAVE_DEBUG_BUGVERBOSE
17 select ARCH_HAVE_CUSTOM_GPIO_H 16 select ARCH_HAVE_CUSTOM_GPIO_H
@@ -91,9 +90,6 @@ config GENERIC_CSUM
91config GENERIC_HWEIGHT 90config GENERIC_HWEIGHT
92 def_bool y 91 def_bool y
93 92
94config IRQ_PER_CPU
95 def_bool y
96
97config GENERIC_GPIO 93config GENERIC_GPIO
98 def_bool n 94 def_bool n
99 95
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index cb9c333d74e8..9bff3db17c8c 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -23,7 +23,6 @@ config SPARC
23 select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE 23 select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE
24 select RTC_CLASS 24 select RTC_CLASS
25 select RTC_DRV_M48T59 25 select RTC_DRV_M48T59
26 select HAVE_IRQ_WORK
27 select HAVE_DMA_ATTRS 26 select HAVE_DMA_ATTRS
28 select HAVE_DMA_API_DEBUG 27 select HAVE_DMA_API_DEBUG
29 select HAVE_ARCH_JUMP_LABEL 28 select HAVE_ARCH_JUMP_LABEL
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 225543bf45a5..36b05ed0bb3c 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -28,7 +28,6 @@ config X86
28 select HAVE_OPROFILE 28 select HAVE_OPROFILE
29 select HAVE_PCSPKR_PLATFORM 29 select HAVE_PCSPKR_PLATFORM
30 select HAVE_PERF_EVENTS 30 select HAVE_PERF_EVENTS
31 select HAVE_IRQ_WORK
32 select HAVE_IOREMAP_PROT 31 select HAVE_IOREMAP_PROT
33 select HAVE_KPROBES 32 select HAVE_KPROBES
34 select HAVE_MEMBLOCK 33 select HAVE_MEMBLOCK
diff --git a/drivers/staging/iio/trigger/Kconfig b/drivers/staging/iio/trigger/Kconfig
index 7d3207559265..d44d3ad26fa5 100644
--- a/drivers/staging/iio/trigger/Kconfig
+++ b/drivers/staging/iio/trigger/Kconfig
@@ -21,7 +21,6 @@ config IIO_GPIO_TRIGGER
21config IIO_SYSFS_TRIGGER 21config IIO_SYSFS_TRIGGER
22 tristate "SYSFS trigger" 22 tristate "SYSFS trigger"
23 depends on SYSFS 23 depends on SYSFS
24 depends on HAVE_IRQ_WORK
25 select IRQ_WORK 24 select IRQ_WORK
26 help 25 help
27 Provides support for using SYSFS entry as IIO triggers. 26 Provides support for using SYSFS entry as IIO triggers.
diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h
index 6a9e8f5399e2..f5dbce50466e 100644
--- a/include/linux/irq_work.h
+++ b/include/linux/irq_work.h
@@ -3,6 +3,20 @@
3 3
4#include <linux/llist.h> 4#include <linux/llist.h>
5 5
6/*
7 * An entry can be in one of four states:
8 *
9 * free NULL, 0 -> {claimed} : free to be used
10 * claimed NULL, 3 -> {pending} : claimed to be enqueued
11 * pending next, 3 -> {busy} : queued, pending callback
12 * busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
13 */
14
15#define IRQ_WORK_PENDING 1UL
16#define IRQ_WORK_BUSY 2UL
17#define IRQ_WORK_FLAGS 3UL
18#define IRQ_WORK_LAZY 4UL /* Doesn't want IPI, wait for tick */
19
6struct irq_work { 20struct irq_work {
7 unsigned long flags; 21 unsigned long flags;
8 struct llist_node llnode; 22 struct llist_node llnode;
@@ -16,8 +30,14 @@ void init_irq_work(struct irq_work *work, void (*func)(struct irq_work *))
16 work->func = func; 30 work->func = func;
17} 31}
18 32
19bool irq_work_queue(struct irq_work *work); 33void irq_work_queue(struct irq_work *work);
20void irq_work_run(void); 34void irq_work_run(void);
21void irq_work_sync(struct irq_work *work); 35void irq_work_sync(struct irq_work *work);
22 36
37#ifdef CONFIG_IRQ_WORK
38bool irq_work_needs_cpu(void);
39#else
40static bool irq_work_needs_cpu(void) { return false; }
41#endif
42
23#endif /* _LINUX_IRQ_WORK_H */ 43#endif /* _LINUX_IRQ_WORK_H */
diff --git a/include/linux/printk.h b/include/linux/printk.h
index 9afc01e5a0a6..86c4b6294713 100644
--- a/include/linux/printk.h
+++ b/include/linux/printk.h
@@ -98,9 +98,6 @@ int no_printk(const char *fmt, ...)
98extern asmlinkage __printf(1, 2) 98extern asmlinkage __printf(1, 2)
99void early_printk(const char *fmt, ...); 99void early_printk(const char *fmt, ...);
100 100
101extern int printk_needs_cpu(int cpu);
102extern void printk_tick(void);
103
104#ifdef CONFIG_PRINTK 101#ifdef CONFIG_PRINTK
105asmlinkage __printf(5, 0) 102asmlinkage __printf(5, 0)
106int vprintk_emit(int facility, int level, 103int vprintk_emit(int facility, int level,
diff --git a/include/linux/tick.h b/include/linux/tick.h
index 1a6567b48492..553272e6af55 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -8,6 +8,8 @@
8 8
9#include <linux/clockchips.h> 9#include <linux/clockchips.h>
10#include <linux/irqflags.h> 10#include <linux/irqflags.h>
11#include <linux/percpu.h>
12#include <linux/hrtimer.h>
11 13
12#ifdef CONFIG_GENERIC_CLOCKEVENTS 14#ifdef CONFIG_GENERIC_CLOCKEVENTS
13 15
@@ -122,13 +124,26 @@ static inline int tick_oneshot_mode_active(void) { return 0; }
122#endif /* !CONFIG_GENERIC_CLOCKEVENTS */ 124#endif /* !CONFIG_GENERIC_CLOCKEVENTS */
123 125
124# ifdef CONFIG_NO_HZ 126# ifdef CONFIG_NO_HZ
127DECLARE_PER_CPU(struct tick_sched, tick_cpu_sched);
128
129static inline int tick_nohz_tick_stopped(void)
130{
131 return __this_cpu_read(tick_cpu_sched.tick_stopped);
132}
133
125extern void tick_nohz_idle_enter(void); 134extern void tick_nohz_idle_enter(void);
126extern void tick_nohz_idle_exit(void); 135extern void tick_nohz_idle_exit(void);
127extern void tick_nohz_irq_exit(void); 136extern void tick_nohz_irq_exit(void);
128extern ktime_t tick_nohz_get_sleep_length(void); 137extern ktime_t tick_nohz_get_sleep_length(void);
129extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); 138extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time);
130extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); 139extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time);
131# else 140
141# else /* !CONFIG_NO_HZ */
142static inline int tick_nohz_tick_stopped(void)
143{
144 return 0;
145}
146
132static inline void tick_nohz_idle_enter(void) { } 147static inline void tick_nohz_idle_enter(void) { }
133static inline void tick_nohz_idle_exit(void) { } 148static inline void tick_nohz_idle_exit(void) { }
134 149
diff --git a/init/Kconfig b/init/Kconfig
index b023334df142..dcb68ac42b78 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -20,12 +20,8 @@ config CONSTRUCTORS
20 bool 20 bool
21 depends on !UML 21 depends on !UML
22 22
23config HAVE_IRQ_WORK
24 bool
25
26config IRQ_WORK 23config IRQ_WORK
27 bool 24 bool
28 depends on HAVE_IRQ_WORK
29 25
30config BUILDTIME_EXTABLE_SORT 26config BUILDTIME_EXTABLE_SORT
31 bool 27 bool
@@ -1273,6 +1269,7 @@ config HOTPLUG
1273config PRINTK 1269config PRINTK
1274 default y 1270 default y
1275 bool "Enable support for printk" if EXPERT 1271 bool "Enable support for printk" if EXPERT
1272 select IRQ_WORK
1276 help 1273 help
1277 This option enables normal printk support. Removing it 1274 This option enables normal printk support. Removing it
1278 eliminates most of the message strings from the kernel image 1275 eliminates most of the message strings from the kernel image
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index e49a288fa479..88e7bed62711 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -1524,6 +1524,7 @@ void enable_percpu_irq(unsigned int irq, unsigned int type)
1524out: 1524out:
1525 irq_put_desc_unlock(desc, flags); 1525 irq_put_desc_unlock(desc, flags);
1526} 1526}
1527EXPORT_SYMBOL_GPL(enable_percpu_irq);
1527 1528
1528void disable_percpu_irq(unsigned int irq) 1529void disable_percpu_irq(unsigned int irq)
1529{ 1530{
@@ -1537,6 +1538,7 @@ void disable_percpu_irq(unsigned int irq)
1537 irq_percpu_disable(desc, cpu); 1538 irq_percpu_disable(desc, cpu);
1538 irq_put_desc_unlock(desc, flags); 1539 irq_put_desc_unlock(desc, flags);
1539} 1540}
1541EXPORT_SYMBOL_GPL(disable_percpu_irq);
1540 1542
1541/* 1543/*
1542 * Internal function to unregister a percpu irqaction. 1544 * Internal function to unregister a percpu irqaction.
diff --git a/kernel/irq/spurious.c b/kernel/irq/spurious.c
index 611cd6003c45..7b5f012bde9d 100644
--- a/kernel/irq/spurious.c
+++ b/kernel/irq/spurious.c
@@ -80,13 +80,11 @@ static int try_one_irq(int irq, struct irq_desc *desc, bool force)
80 80
81 /* 81 /*
82 * All handlers must agree on IRQF_SHARED, so we test just the 82 * All handlers must agree on IRQF_SHARED, so we test just the
83 * first. Check for action->next as well. 83 * first.
84 */ 84 */
85 action = desc->action; 85 action = desc->action;
86 if (!action || !(action->flags & IRQF_SHARED) || 86 if (!action || !(action->flags & IRQF_SHARED) ||
87 (action->flags & __IRQF_TIMER) || 87 (action->flags & __IRQF_TIMER))
88 (action->handler(irq, action->dev_id) == IRQ_HANDLED) ||
89 !action->next)
90 goto out; 88 goto out;
91 89
92 /* Already running on another processor */ 90 /* Already running on another processor */
@@ -104,6 +102,7 @@ static int try_one_irq(int irq, struct irq_desc *desc, bool force)
104 do { 102 do {
105 if (handle_irq_event(desc) == IRQ_HANDLED) 103 if (handle_irq_event(desc) == IRQ_HANDLED)
106 ret = IRQ_HANDLED; 104 ret = IRQ_HANDLED;
105 /* Make sure that there is still a valid action */
107 action = desc->action; 106 action = desc->action;
108 } while ((desc->istate & IRQS_PENDING) && action); 107 } while ((desc->istate & IRQS_PENDING) && action);
109 desc->istate &= ~IRQS_POLL_INPROGRESS; 108 desc->istate &= ~IRQS_POLL_INPROGRESS;
diff --git a/kernel/irq_work.c b/kernel/irq_work.c
index 1588e3b2871b..55fcce6065cf 100644
--- a/kernel/irq_work.c
+++ b/kernel/irq_work.c
@@ -12,37 +12,36 @@
12#include <linux/percpu.h> 12#include <linux/percpu.h>
13#include <linux/hardirq.h> 13#include <linux/hardirq.h>
14#include <linux/irqflags.h> 14#include <linux/irqflags.h>
15#include <linux/sched.h>
16#include <linux/tick.h>
17#include <linux/cpu.h>
18#include <linux/notifier.h>
15#include <asm/processor.h> 19#include <asm/processor.h>
16 20
17/*
18 * An entry can be in one of four states:
19 *
20 * free NULL, 0 -> {claimed} : free to be used
21 * claimed NULL, 3 -> {pending} : claimed to be enqueued
22 * pending next, 3 -> {busy} : queued, pending callback
23 * busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
24 */
25
26#define IRQ_WORK_PENDING 1UL
27#define IRQ_WORK_BUSY 2UL
28#define IRQ_WORK_FLAGS 3UL
29 21
30static DEFINE_PER_CPU(struct llist_head, irq_work_list); 22static DEFINE_PER_CPU(struct llist_head, irq_work_list);
23static DEFINE_PER_CPU(int, irq_work_raised);
31 24
32/* 25/*
33 * Claim the entry so that no one else will poke at it. 26 * Claim the entry so that no one else will poke at it.
34 */ 27 */
35static bool irq_work_claim(struct irq_work *work) 28static bool irq_work_claim(struct irq_work *work)
36{ 29{
37 unsigned long flags, nflags; 30 unsigned long flags, oflags, nflags;
38 31
32 /*
33 * Start with our best wish as a premise but only trust any
34 * flag value after cmpxchg() result.
35 */
36 flags = work->flags & ~IRQ_WORK_PENDING;
39 for (;;) { 37 for (;;) {
40 flags = work->flags;
41 if (flags & IRQ_WORK_PENDING)
42 return false;
43 nflags = flags | IRQ_WORK_FLAGS; 38 nflags = flags | IRQ_WORK_FLAGS;
44 if (cmpxchg(&work->flags, flags, nflags) == flags) 39 oflags = cmpxchg(&work->flags, flags, nflags);
40 if (oflags == flags)
45 break; 41 break;
42 if (oflags & IRQ_WORK_PENDING)
43 return false;
44 flags = oflags;
46 cpu_relax(); 45 cpu_relax();
47 } 46 }
48 47
@@ -57,57 +56,69 @@ void __weak arch_irq_work_raise(void)
57} 56}
58 57
59/* 58/*
60 * Queue the entry and raise the IPI if needed. 59 * Enqueue the irq_work @entry unless it's already pending
60 * somewhere.
61 *
62 * Can be re-enqueued while the callback is still in progress.
61 */ 63 */
62static void __irq_work_queue(struct irq_work *work) 64void irq_work_queue(struct irq_work *work)
63{ 65{
64 bool empty; 66 /* Only queue if not already pending */
67 if (!irq_work_claim(work))
68 return;
65 69
70 /* Queue the entry and raise the IPI if needed. */
66 preempt_disable(); 71 preempt_disable();
67 72
68 empty = llist_add(&work->llnode, &__get_cpu_var(irq_work_list)); 73 llist_add(&work->llnode, &__get_cpu_var(irq_work_list));
69 /* The list was empty, raise self-interrupt to start processing. */ 74
70 if (empty) 75 /*
71 arch_irq_work_raise(); 76 * If the work is not "lazy" or the tick is stopped, raise the irq
77 * work interrupt (if supported by the arch), otherwise, just wait
78 * for the next tick.
79 */
80 if (!(work->flags & IRQ_WORK_LAZY) || tick_nohz_tick_stopped()) {
81 if (!this_cpu_cmpxchg(irq_work_raised, 0, 1))
82 arch_irq_work_raise();
83 }
72 84
73 preempt_enable(); 85 preempt_enable();
74} 86}
87EXPORT_SYMBOL_GPL(irq_work_queue);
75 88
76/* 89bool irq_work_needs_cpu(void)
77 * Enqueue the irq_work @entry, returns true on success, failure when the
78 * @entry was already enqueued by someone else.
79 *
80 * Can be re-enqueued while the callback is still in progress.
81 */
82bool irq_work_queue(struct irq_work *work)
83{ 90{
84 if (!irq_work_claim(work)) { 91 struct llist_head *this_list;
85 /* 92
86 * Already enqueued, can't do! 93 this_list = &__get_cpu_var(irq_work_list);
87 */ 94 if (llist_empty(this_list))
88 return false; 95 return false;
89 }
90 96
91 __irq_work_queue(work); 97 /* All work should have been flushed before going offline */
98 WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
99
92 return true; 100 return true;
93} 101}
94EXPORT_SYMBOL_GPL(irq_work_queue);
95 102
96/* 103static void __irq_work_run(void)
97 * Run the irq_work entries on this cpu. Requires to be ran from hardirq
98 * context with local IRQs disabled.
99 */
100void irq_work_run(void)
101{ 104{
105 unsigned long flags;
102 struct irq_work *work; 106 struct irq_work *work;
103 struct llist_head *this_list; 107 struct llist_head *this_list;
104 struct llist_node *llnode; 108 struct llist_node *llnode;
105 109
110
111 /*
112 * Reset the "raised" state right before we check the list because
113 * an NMI may enqueue after we find the list empty from the runner.
114 */
115 __this_cpu_write(irq_work_raised, 0);
116 barrier();
117
106 this_list = &__get_cpu_var(irq_work_list); 118 this_list = &__get_cpu_var(irq_work_list);
107 if (llist_empty(this_list)) 119 if (llist_empty(this_list))
108 return; 120 return;
109 121
110 BUG_ON(!in_irq());
111 BUG_ON(!irqs_disabled()); 122 BUG_ON(!irqs_disabled());
112 123
113 llnode = llist_del_all(this_list); 124 llnode = llist_del_all(this_list);
@@ -119,16 +130,31 @@ void irq_work_run(void)
119 /* 130 /*
120 * Clear the PENDING bit, after this point the @work 131 * Clear the PENDING bit, after this point the @work
121 * can be re-used. 132 * can be re-used.
133 * Make it immediately visible so that other CPUs trying
134 * to claim that work don't rely on us to handle their data
135 * while we are in the middle of the func.
122 */ 136 */
123 work->flags = IRQ_WORK_BUSY; 137 flags = work->flags & ~IRQ_WORK_PENDING;
138 xchg(&work->flags, flags);
139
124 work->func(work); 140 work->func(work);
125 /* 141 /*
126 * Clear the BUSY bit and return to the free state if 142 * Clear the BUSY bit and return to the free state if
127 * no-one else claimed it meanwhile. 143 * no-one else claimed it meanwhile.
128 */ 144 */
129 (void)cmpxchg(&work->flags, IRQ_WORK_BUSY, 0); 145 (void)cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY);
130 } 146 }
131} 147}
148
149/*
150 * Run the irq_work entries on this cpu. Requires to be ran from hardirq
151 * context with local IRQs disabled.
152 */
153void irq_work_run(void)
154{
155 BUG_ON(!in_irq());
156 __irq_work_run();
157}
132EXPORT_SYMBOL_GPL(irq_work_run); 158EXPORT_SYMBOL_GPL(irq_work_run);
133 159
134/* 160/*
@@ -143,3 +169,35 @@ void irq_work_sync(struct irq_work *work)
143 cpu_relax(); 169 cpu_relax();
144} 170}
145EXPORT_SYMBOL_GPL(irq_work_sync); 171EXPORT_SYMBOL_GPL(irq_work_sync);
172
173#ifdef CONFIG_HOTPLUG_CPU
174static int irq_work_cpu_notify(struct notifier_block *self,
175 unsigned long action, void *hcpu)
176{
177 long cpu = (long)hcpu;
178
179 switch (action) {
180 case CPU_DYING:
181 /* Called from stop_machine */
182 if (WARN_ON_ONCE(cpu != smp_processor_id()))
183 break;
184 __irq_work_run();
185 break;
186 default:
187 break;
188 }
189 return NOTIFY_OK;
190}
191
192static struct notifier_block cpu_notify;
193
194static __init int irq_work_init_cpu_notifier(void)
195{
196 cpu_notify.notifier_call = irq_work_cpu_notify;
197 cpu_notify.priority = 0;
198 register_cpu_notifier(&cpu_notify);
199 return 0;
200}
201device_initcall(irq_work_init_cpu_notifier);
202
203#endif /* CONFIG_HOTPLUG_CPU */
diff --git a/kernel/printk.c b/kernel/printk.c
index 267ce780abe8..f24633afa46a 100644
--- a/kernel/printk.c
+++ b/kernel/printk.c
@@ -42,6 +42,7 @@
42#include <linux/notifier.h> 42#include <linux/notifier.h>
43#include <linux/rculist.h> 43#include <linux/rculist.h>
44#include <linux/poll.h> 44#include <linux/poll.h>
45#include <linux/irq_work.h>
45 46
46#include <asm/uaccess.h> 47#include <asm/uaccess.h>
47 48
@@ -1959,30 +1960,32 @@ int is_console_locked(void)
1959static DEFINE_PER_CPU(int, printk_pending); 1960static DEFINE_PER_CPU(int, printk_pending);
1960static DEFINE_PER_CPU(char [PRINTK_BUF_SIZE], printk_sched_buf); 1961static DEFINE_PER_CPU(char [PRINTK_BUF_SIZE], printk_sched_buf);
1961 1962
1962void printk_tick(void) 1963static void wake_up_klogd_work_func(struct irq_work *irq_work)
1963{ 1964{
1964 if (__this_cpu_read(printk_pending)) { 1965 int pending = __this_cpu_xchg(printk_pending, 0);
1965 int pending = __this_cpu_xchg(printk_pending, 0); 1966
1966 if (pending & PRINTK_PENDING_SCHED) { 1967 if (pending & PRINTK_PENDING_SCHED) {
1967 char *buf = __get_cpu_var(printk_sched_buf); 1968 char *buf = __get_cpu_var(printk_sched_buf);
1968 printk(KERN_WARNING "[sched_delayed] %s", buf); 1969 printk(KERN_WARNING "[sched_delayed] %s", buf);
1969 }
1970 if (pending & PRINTK_PENDING_WAKEUP)
1971 wake_up_interruptible(&log_wait);
1972 } 1970 }
1973}
1974 1971
1975int printk_needs_cpu(int cpu) 1972 if (pending & PRINTK_PENDING_WAKEUP)
1976{ 1973 wake_up_interruptible(&log_wait);
1977 if (cpu_is_offline(cpu))
1978 printk_tick();
1979 return __this_cpu_read(printk_pending);
1980} 1974}
1981 1975
1976static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) = {
1977 .func = wake_up_klogd_work_func,
1978 .flags = IRQ_WORK_LAZY,
1979};
1980
1982void wake_up_klogd(void) 1981void wake_up_klogd(void)
1983{ 1982{
1984 if (waitqueue_active(&log_wait)) 1983 preempt_disable();
1984 if (waitqueue_active(&log_wait)) {
1985 this_cpu_or(printk_pending, PRINTK_PENDING_WAKEUP); 1985 this_cpu_or(printk_pending, PRINTK_PENDING_WAKEUP);
1986 irq_work_queue(&__get_cpu_var(wake_up_klogd_work));
1987 }
1988 preempt_enable();
1986} 1989}
1987 1990
1988static void console_cont_flush(char *text, size_t size) 1991static void console_cont_flush(char *text, size_t size)
@@ -2462,6 +2465,7 @@ int printk_sched(const char *fmt, ...)
2462 va_end(args); 2465 va_end(args);
2463 2466
2464 __this_cpu_or(printk_pending, PRINTK_PENDING_SCHED); 2467 __this_cpu_or(printk_pending, PRINTK_PENDING_SCHED);
2468 irq_work_queue(&__get_cpu_var(wake_up_klogd_work));
2465 local_irq_restore(flags); 2469 local_irq_restore(flags);
2466 2470
2467 return r; 2471 return r;
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index d58e552d9fd1..fb8e5e469d1c 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -20,6 +20,7 @@
20#include <linux/profile.h> 20#include <linux/profile.h>
21#include <linux/sched.h> 21#include <linux/sched.h>
22#include <linux/module.h> 22#include <linux/module.h>
23#include <linux/irq_work.h>
23 24
24#include <asm/irq_regs.h> 25#include <asm/irq_regs.h>
25 26
@@ -28,7 +29,7 @@
28/* 29/*
29 * Per cpu nohz control structure 30 * Per cpu nohz control structure
30 */ 31 */
31static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched); 32DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
32 33
33/* 34/*
34 * The time, when the last jiffy update happened. Protected by jiffies_lock. 35 * The time, when the last jiffy update happened. Protected by jiffies_lock.
@@ -331,8 +332,8 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
331 time_delta = timekeeping_max_deferment(); 332 time_delta = timekeeping_max_deferment();
332 } while (read_seqretry(&jiffies_lock, seq)); 333 } while (read_seqretry(&jiffies_lock, seq));
333 334
334 if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) || printk_needs_cpu(cpu) || 335 if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) ||
335 arch_needs_cpu(cpu)) { 336 arch_needs_cpu(cpu) || irq_work_needs_cpu()) {
336 next_jiffies = last_jiffies + 1; 337 next_jiffies = last_jiffies + 1;
337 delta_jiffies = 1; 338 delta_jiffies = 1;
338 } else { 339 } else {
diff --git a/kernel/timer.c b/kernel/timer.c
index 367d00858482..ff3b5165737b 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -1351,7 +1351,6 @@ void update_process_times(int user_tick)
1351 account_process_tick(p, user_tick); 1351 account_process_tick(p, user_tick);
1352 run_local_timers(); 1352 run_local_timers();
1353 rcu_check_callbacks(cpu, user_tick); 1353 rcu_check_callbacks(cpu, user_tick);
1354 printk_tick();
1355#ifdef CONFIG_IRQ_WORK 1354#ifdef CONFIG_IRQ_WORK
1356 if (in_irq()) 1355 if (in_irq())
1357 irq_work_run(); 1356 irq_work_run();