aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Zijlstra <a.p.zijlstra@chello.nl>2010-10-14 02:01:34 -0400
committerIngo Molnar <mingo@elte.hu>2010-10-18 13:58:50 -0400
commite360adbe29241a0194e10e20595360dd7b98a2b3 (patch)
treeef5fa5f50a895096bfb25bc11b25949603158238
parent8e5fc1a7320baf6076391607515dceb61319b36a (diff)
irq_work: Add generic hardirq context callbacks
Provide a mechanism that allows running code in IRQ context. It is most useful for NMI code that needs to interact with the rest of the system -- like wakeup a task to drain buffers. Perf currently has such a mechanism, so extract that and provide it as a generic feature, independent of perf so that others may also benefit. The IRQ context callback is generated through self-IPIs where possible, or on architectures like powerpc the decrementer (the built-in timer facility) is set to generate an interrupt immediately. Architectures that don't have anything like this get to do with a callback from the timer tick. These architectures can call irq_work_run() at the tail of any IRQ handlers that might enqueue such work (like the perf IRQ handler) to avoid undue latencies in processing the work. Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Acked-by: Kyle McMartin <kyle@mcmartin.ca> Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com> [ various fixes ] Signed-off-by: Huang Ying <ying.huang@intel.com> LKML-Reference: <1287036094.7768.291.camel@yhuang-dev> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat
-rw-r--r--arch/alpha/Kconfig1
-rw-r--r--arch/alpha/include/asm/perf_event.h5
-rw-r--r--arch/alpha/kernel/time.c30
-rw-r--r--arch/arm/Kconfig1
-rw-r--r--arch/arm/include/asm/perf_event.h12
-rw-r--r--arch/arm/kernel/perf_event.c8
-rw-r--r--arch/frv/Kconfig1
-rw-r--r--arch/frv/lib/Makefile2
-rw-r--r--arch/frv/lib/perf_event.c19
-rw-r--r--arch/parisc/Kconfig1
-rw-r--r--arch/parisc/include/asm/perf_event.h3
-rw-r--r--arch/powerpc/Kconfig1
-rw-r--r--arch/powerpc/include/asm/paca.h2
-rw-r--r--arch/powerpc/kernel/time.c42
-rw-r--r--arch/s390/Kconfig1
-rw-r--r--arch/s390/include/asm/perf_event.h3
-rw-r--r--arch/sh/Kconfig1
-rw-r--r--arch/sh/include/asm/perf_event.h7
-rw-r--r--arch/sparc/Kconfig2
-rw-r--r--arch/sparc/include/asm/perf_event.h4
-rw-r--r--arch/sparc/kernel/pcr.c8
-rw-r--r--arch/x86/Kconfig1
-rw-r--r--arch/x86/include/asm/entry_arch.h4
-rw-r--r--arch/x86/include/asm/hardirq.h2
-rw-r--r--arch/x86/include/asm/hw_irq.h2
-rw-r--r--arch/x86/include/asm/irq_vectors.h4
-rw-r--r--arch/x86/kernel/Makefile1
-rw-r--r--arch/x86/kernel/cpu/perf_event.c19
-rw-r--r--arch/x86/kernel/entry_64.S6
-rw-r--r--arch/x86/kernel/irq.c8
-rw-r--r--arch/x86/kernel/irq_work.c30
-rw-r--r--arch/x86/kernel/irqinit.c6
-rw-r--r--include/linux/irq_work.h20
-rw-r--r--include/linux/perf_event.h11
-rw-r--r--init/Kconfig8
-rw-r--r--kernel/Makefile2
-rw-r--r--kernel/irq_work.c164
-rw-r--r--kernel/perf_event.c104
-rw-r--r--kernel/timer.c7
39 files changed, 311 insertions, 242 deletions
diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig
index b9647bb66d13..d04ccd73af45 100644
--- a/arch/alpha/Kconfig
+++ b/arch/alpha/Kconfig
@@ -9,6 +9,7 @@ config ALPHA
9 select HAVE_IDE 9 select HAVE_IDE
10 select HAVE_OPROFILE 10 select HAVE_OPROFILE
11 select HAVE_SYSCALL_WRAPPERS 11 select HAVE_SYSCALL_WRAPPERS
12 select HAVE_IRQ_WORK
12 select HAVE_PERF_EVENTS 13 select HAVE_PERF_EVENTS
13 select HAVE_DMA_ATTRS 14 select HAVE_DMA_ATTRS
14 help 15 help
diff --git a/arch/alpha/include/asm/perf_event.h b/arch/alpha/include/asm/perf_event.h
index 4157cd3c44a9..fe792ca818f6 100644
--- a/arch/alpha/include/asm/perf_event.h
+++ b/arch/alpha/include/asm/perf_event.h
@@ -1,11 +1,6 @@
1#ifndef __ASM_ALPHA_PERF_EVENT_H 1#ifndef __ASM_ALPHA_PERF_EVENT_H
2#define __ASM_ALPHA_PERF_EVENT_H 2#define __ASM_ALPHA_PERF_EVENT_H
3 3
4/* Alpha only supports software events through this interface. */
5extern void set_perf_event_pending(void);
6
7#define PERF_EVENT_INDEX_OFFSET 0
8
9#ifdef CONFIG_PERF_EVENTS 4#ifdef CONFIG_PERF_EVENTS
10extern void init_hw_perf_events(void); 5extern void init_hw_perf_events(void);
11#else 6#else
diff --git a/arch/alpha/kernel/time.c b/arch/alpha/kernel/time.c
index 396af1799ea4..0f1d8493cfca 100644
--- a/arch/alpha/kernel/time.c
+++ b/arch/alpha/kernel/time.c
@@ -41,7 +41,7 @@
41#include <linux/init.h> 41#include <linux/init.h>
42#include <linux/bcd.h> 42#include <linux/bcd.h>
43#include <linux/profile.h> 43#include <linux/profile.h>
44#include <linux/perf_event.h> 44#include <linux/irq_work.h>
45 45
46#include <asm/uaccess.h> 46#include <asm/uaccess.h>
47#include <asm/io.h> 47#include <asm/io.h>
@@ -83,25 +83,25 @@ static struct {
83 83
84unsigned long est_cycle_freq; 84unsigned long est_cycle_freq;
85 85
86#ifdef CONFIG_PERF_EVENTS 86#ifdef CONFIG_IRQ_WORK
87 87
88DEFINE_PER_CPU(u8, perf_event_pending); 88DEFINE_PER_CPU(u8, irq_work_pending);
89 89
90#define set_perf_event_pending_flag() __get_cpu_var(perf_event_pending) = 1 90#define set_irq_work_pending_flag() __get_cpu_var(irq_work_pending) = 1
91#define test_perf_event_pending() __get_cpu_var(perf_event_pending) 91#define test_irq_work_pending() __get_cpu_var(irq_work_pending)
92#define clear_perf_event_pending() __get_cpu_var(perf_event_pending) = 0 92#define clear_irq_work_pending() __get_cpu_var(irq_work_pending) = 0
93 93
94void set_perf_event_pending(void) 94void set_irq_work_pending(void)
95{ 95{
96 set_perf_event_pending_flag(); 96 set_irq_work_pending_flag();
97} 97}
98 98
99#else /* CONFIG_PERF_EVENTS */ 99#else /* CONFIG_IRQ_WORK */
100 100
101#define test_perf_event_pending() 0 101#define test_irq_work_pending() 0
102#define clear_perf_event_pending() 102#define clear_irq_work_pending()
103 103
104#endif /* CONFIG_PERF_EVENTS */ 104#endif /* CONFIG_IRQ_WORK */
105 105
106 106
107static inline __u32 rpcc(void) 107static inline __u32 rpcc(void)
@@ -191,9 +191,9 @@ irqreturn_t timer_interrupt(int irq, void *dev)
191 191
192 write_sequnlock(&xtime_lock); 192 write_sequnlock(&xtime_lock);
193 193
194 if (test_perf_event_pending()) { 194 if (test_irq_work_pending()) {
195 clear_perf_event_pending(); 195 clear_irq_work_pending();
196 perf_event_do_pending(); 196 irq_work_run();
197 } 197 }
198 198
199#ifndef CONFIG_SMP 199#ifndef CONFIG_SMP
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 88c97bc7a6f5..7c0dfccd05bd 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -23,6 +23,7 @@ config ARM
23 select HAVE_KERNEL_GZIP 23 select HAVE_KERNEL_GZIP
24 select HAVE_KERNEL_LZO 24 select HAVE_KERNEL_LZO
25 select HAVE_KERNEL_LZMA 25 select HAVE_KERNEL_LZMA
26 select HAVE_IRQ_WORK
26 select HAVE_PERF_EVENTS 27 select HAVE_PERF_EVENTS
27 select PERF_USE_VMALLOC 28 select PERF_USE_VMALLOC
28 select HAVE_REGS_AND_STACK_ACCESS_API 29 select HAVE_REGS_AND_STACK_ACCESS_API
diff --git a/arch/arm/include/asm/perf_event.h b/arch/arm/include/asm/perf_event.h
index b5799a3b7117..c4aa4e8c6af9 100644
--- a/arch/arm/include/asm/perf_event.h
+++ b/arch/arm/include/asm/perf_event.h
@@ -12,18 +12,6 @@
12#ifndef __ARM_PERF_EVENT_H__ 12#ifndef __ARM_PERF_EVENT_H__
13#define __ARM_PERF_EVENT_H__ 13#define __ARM_PERF_EVENT_H__
14 14
15/*
16 * NOP: on *most* (read: all supported) ARM platforms, the performance
17 * counter interrupts are regular interrupts and not an NMI. This
18 * means that when we receive the interrupt we can call
19 * perf_event_do_pending() that handles all of the work with
20 * interrupts disabled.
21 */
22static inline void
23set_perf_event_pending(void)
24{
25}
26
27/* ARM performance counters start from 1 (in the cp15 accesses) so use the 15/* ARM performance counters start from 1 (in the cp15 accesses) so use the
28 * same indexes here for consistency. */ 16 * same indexes here for consistency. */
29#define PERF_EVENT_INDEX_OFFSET 1 17#define PERF_EVENT_INDEX_OFFSET 1
diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c
index 6cc6521881aa..49643b1467e6 100644
--- a/arch/arm/kernel/perf_event.c
+++ b/arch/arm/kernel/perf_event.c
@@ -1092,7 +1092,7 @@ armv6pmu_handle_irq(int irq_num,
1092 * platforms that can have the PMU interrupts raised as an NMI, this 1092 * platforms that can have the PMU interrupts raised as an NMI, this
1093 * will not work. 1093 * will not work.
1094 */ 1094 */
1095 perf_event_do_pending(); 1095 irq_work_run();
1096 1096
1097 return IRQ_HANDLED; 1097 return IRQ_HANDLED;
1098} 1098}
@@ -2068,7 +2068,7 @@ static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
2068 * platforms that can have the PMU interrupts raised as an NMI, this 2068 * platforms that can have the PMU interrupts raised as an NMI, this
2069 * will not work. 2069 * will not work.
2070 */ 2070 */
2071 perf_event_do_pending(); 2071 irq_work_run();
2072 2072
2073 return IRQ_HANDLED; 2073 return IRQ_HANDLED;
2074} 2074}
@@ -2436,7 +2436,7 @@ xscale1pmu_handle_irq(int irq_num, void *dev)
2436 armpmu->disable(hwc, idx); 2436 armpmu->disable(hwc, idx);
2437 } 2437 }
2438 2438
2439 perf_event_do_pending(); 2439 irq_work_run();
2440 2440
2441 /* 2441 /*
2442 * Re-enable the PMU. 2442 * Re-enable the PMU.
@@ -2763,7 +2763,7 @@ xscale2pmu_handle_irq(int irq_num, void *dev)
2763 armpmu->disable(hwc, idx); 2763 armpmu->disable(hwc, idx);
2764 } 2764 }
2765 2765
2766 perf_event_do_pending(); 2766 irq_work_run();
2767 2767
2768 /* 2768 /*
2769 * Re-enable the PMU. 2769 * Re-enable the PMU.
diff --git a/arch/frv/Kconfig b/arch/frv/Kconfig
index 16399bd24993..0f2417df6323 100644
--- a/arch/frv/Kconfig
+++ b/arch/frv/Kconfig
@@ -7,6 +7,7 @@ config FRV
7 default y 7 default y
8 select HAVE_IDE 8 select HAVE_IDE
9 select HAVE_ARCH_TRACEHOOK 9 select HAVE_ARCH_TRACEHOOK
10 select HAVE_IRQ_WORK
10 select HAVE_PERF_EVENTS 11 select HAVE_PERF_EVENTS
11 12
12config ZONE_DMA 13config ZONE_DMA
diff --git a/arch/frv/lib/Makefile b/arch/frv/lib/Makefile
index f4709756d0d9..4ff2fb1e6b16 100644
--- a/arch/frv/lib/Makefile
+++ b/arch/frv/lib/Makefile
@@ -5,4 +5,4 @@
5lib-y := \ 5lib-y := \
6 __ashldi3.o __lshrdi3.o __muldi3.o __ashrdi3.o __negdi2.o __ucmpdi2.o \ 6 __ashldi3.o __lshrdi3.o __muldi3.o __ashrdi3.o __negdi2.o __ucmpdi2.o \
7 checksum.o memcpy.o memset.o atomic-ops.o atomic64-ops.o \ 7 checksum.o memcpy.o memset.o atomic-ops.o atomic64-ops.o \
8 outsl_ns.o outsl_sw.o insl_ns.o insl_sw.o cache.o perf_event.o 8 outsl_ns.o outsl_sw.o insl_ns.o insl_sw.o cache.o
diff --git a/arch/frv/lib/perf_event.c b/arch/frv/lib/perf_event.c
deleted file mode 100644
index 9ac5acfd2e91..000000000000
--- a/arch/frv/lib/perf_event.c
+++ /dev/null
@@ -1,19 +0,0 @@
1/* Performance event handling
2 *
3 * Copyright (C) 2009 Red Hat, Inc. All Rights Reserved.
4 * Written by David Howells (dhowells@redhat.com)
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public Licence
8 * as published by the Free Software Foundation; either version
9 * 2 of the Licence, or (at your option) any later version.
10 */
11
12#include <linux/perf_event.h>
13
14/*
15 * mark the performance event as pending
16 */
17void set_perf_event_pending(void)
18{
19}
diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig
index 907417d187e1..79a04a9394d5 100644
--- a/arch/parisc/Kconfig
+++ b/arch/parisc/Kconfig
@@ -16,6 +16,7 @@ config PARISC
16 select RTC_DRV_GENERIC 16 select RTC_DRV_GENERIC
17 select INIT_ALL_POSSIBLE 17 select INIT_ALL_POSSIBLE
18 select BUG 18 select BUG
19 select HAVE_IRQ_WORK
19 select HAVE_PERF_EVENTS 20 select HAVE_PERF_EVENTS
20 select GENERIC_ATOMIC64 if !64BIT 21 select GENERIC_ATOMIC64 if !64BIT
21 help 22 help
diff --git a/arch/parisc/include/asm/perf_event.h b/arch/parisc/include/asm/perf_event.h
index cc146427d8f9..1e0fd8ba6c03 100644
--- a/arch/parisc/include/asm/perf_event.h
+++ b/arch/parisc/include/asm/perf_event.h
@@ -1,7 +1,6 @@
1#ifndef __ASM_PARISC_PERF_EVENT_H 1#ifndef __ASM_PARISC_PERF_EVENT_H
2#define __ASM_PARISC_PERF_EVENT_H 2#define __ASM_PARISC_PERF_EVENT_H
3 3
4/* parisc only supports software events through this interface. */ 4/* Empty, just to avoid compiling error */
5static inline void set_perf_event_pending(void) { }
6 5
7#endif /* __ASM_PARISC_PERF_EVENT_H */ 6#endif /* __ASM_PARISC_PERF_EVENT_H */
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 631e5a0fb6ab..4b1e521d966f 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -138,6 +138,7 @@ config PPC
138 select HAVE_OPROFILE 138 select HAVE_OPROFILE
139 select HAVE_SYSCALL_WRAPPERS if PPC64 139 select HAVE_SYSCALL_WRAPPERS if PPC64
140 select GENERIC_ATOMIC64 if PPC32 140 select GENERIC_ATOMIC64 if PPC32
141 select HAVE_IRQ_WORK
141 select HAVE_PERF_EVENTS 142 select HAVE_PERF_EVENTS
142 select HAVE_REGS_AND_STACK_ACCESS_API 143 select HAVE_REGS_AND_STACK_ACCESS_API
143 select HAVE_HW_BREAKPOINT if PERF_EVENTS && PPC_BOOK3S_64 144 select HAVE_HW_BREAKPOINT if PERF_EVENTS && PPC_BOOK3S_64
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 1ff6662f7faf..9b287fdd8ea3 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -129,7 +129,7 @@ struct paca_struct {
129 u8 soft_enabled; /* irq soft-enable flag */ 129 u8 soft_enabled; /* irq soft-enable flag */
130 u8 hard_enabled; /* set if irqs are enabled in MSR */ 130 u8 hard_enabled; /* set if irqs are enabled in MSR */
131 u8 io_sync; /* writel() needs spin_unlock sync */ 131 u8 io_sync; /* writel() needs spin_unlock sync */
132 u8 perf_event_pending; /* PM interrupt while soft-disabled */ 132 u8 irq_work_pending; /* IRQ_WORK interrupt while soft-disable */
133 133
134 /* Stuff for accurate time accounting */ 134 /* Stuff for accurate time accounting */
135 u64 user_time; /* accumulated usermode TB ticks */ 135 u64 user_time; /* accumulated usermode TB ticks */
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 8533b3b83f5d..54888eb10c3b 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -53,7 +53,7 @@
53#include <linux/posix-timers.h> 53#include <linux/posix-timers.h>
54#include <linux/irq.h> 54#include <linux/irq.h>
55#include <linux/delay.h> 55#include <linux/delay.h>
56#include <linux/perf_event.h> 56#include <linux/irq_work.h>
57#include <asm/trace.h> 57#include <asm/trace.h>
58 58
59#include <asm/io.h> 59#include <asm/io.h>
@@ -493,60 +493,60 @@ void __init iSeries_time_init_early(void)
493} 493}
494#endif /* CONFIG_PPC_ISERIES */ 494#endif /* CONFIG_PPC_ISERIES */
495 495
496#ifdef CONFIG_PERF_EVENTS 496#ifdef CONFIG_IRQ_WORK
497 497
498/* 498/*
499 * 64-bit uses a byte in the PACA, 32-bit uses a per-cpu variable... 499 * 64-bit uses a byte in the PACA, 32-bit uses a per-cpu variable...
500 */ 500 */
501#ifdef CONFIG_PPC64 501#ifdef CONFIG_PPC64
502static inline unsigned long test_perf_event_pending(void) 502static inline unsigned long test_irq_work_pending(void)
503{ 503{
504 unsigned long x; 504 unsigned long x;
505 505
506 asm volatile("lbz %0,%1(13)" 506 asm volatile("lbz %0,%1(13)"
507 : "=r" (x) 507 : "=r" (x)
508 : "i" (offsetof(struct paca_struct, perf_event_pending))); 508 : "i" (offsetof(struct paca_struct, irq_work_pending)));
509 return x; 509 return x;
510} 510}
511 511
512static inline void set_perf_event_pending_flag(void) 512static inline void set_irq_work_pending_flag(void)
513{ 513{
514 asm volatile("stb %0,%1(13)" : : 514 asm volatile("stb %0,%1(13)" : :
515 "r" (1), 515 "r" (1),
516 "i" (offsetof(struct paca_struct, perf_event_pending))); 516 "i" (offsetof(struct paca_struct, irq_work_pending)));
517} 517}
518 518
519static inline void clear_perf_event_pending(void) 519static inline void clear_irq_work_pending(void)
520{ 520{
521 asm volatile("stb %0,%1(13)" : : 521 asm volatile("stb %0,%1(13)" : :
522 "r" (0), 522 "r" (0),
523 "i" (offsetof(struct paca_struct, perf_event_pending))); 523 "i" (offsetof(struct paca_struct, irq_work_pending)));
524} 524}
525 525
526#else /* 32-bit */ 526#else /* 32-bit */
527 527
528DEFINE_PER_CPU(u8, perf_event_pending); 528DEFINE_PER_CPU(u8, irq_work_pending);
529 529
530#define set_perf_event_pending_flag() __get_cpu_var(perf_event_pending) = 1 530#define set_irq_work_pending_flag() __get_cpu_var(irq_work_pending) = 1
531#define test_perf_event_pending() __get_cpu_var(perf_event_pending) 531#define test_irq_work_pending() __get_cpu_var(irq_work_pending)
532#define clear_perf_event_pending() __get_cpu_var(perf_event_pending) = 0 532#define clear_irq_work_pending() __get_cpu_var(irq_work_pending) = 0
533 533
534#endif /* 32 vs 64 bit */ 534#endif /* 32 vs 64 bit */
535 535
536void set_perf_event_pending(void) 536void set_irq_work_pending(void)
537{ 537{
538 preempt_disable(); 538 preempt_disable();
539 set_perf_event_pending_flag(); 539 set_irq_work_pending_flag();
540 set_dec(1); 540 set_dec(1);
541 preempt_enable(); 541 preempt_enable();
542} 542}
543 543
544#else /* CONFIG_PERF_EVENTS */ 544#else /* CONFIG_IRQ_WORK */
545 545
546#define test_perf_event_pending() 0 546#define test_irq_work_pending() 0
547#define clear_perf_event_pending() 547#define clear_irq_work_pending()
548 548
549#endif /* CONFIG_PERF_EVENTS */ 549#endif /* CONFIG_IRQ_WORK */
550 550
551/* 551/*
552 * For iSeries shared processors, we have to let the hypervisor 552 * For iSeries shared processors, we have to let the hypervisor
@@ -587,9 +587,9 @@ void timer_interrupt(struct pt_regs * regs)
587 587
588 calculate_steal_time(); 588 calculate_steal_time();
589 589
590 if (test_perf_event_pending()) { 590 if (test_irq_work_pending()) {
591 clear_perf_event_pending(); 591 clear_irq_work_pending();
592 perf_event_do_pending(); 592 irq_work_run();
593 } 593 }
594 594
595#ifdef CONFIG_PPC_ISERIES 595#ifdef CONFIG_PPC_ISERIES
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index f0777a47e3a5..958f0dadeadf 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -95,6 +95,7 @@ config S390
95 select HAVE_KVM if 64BIT 95 select HAVE_KVM if 64BIT
96 select HAVE_ARCH_TRACEHOOK 96 select HAVE_ARCH_TRACEHOOK
97 select INIT_ALL_POSSIBLE 97 select INIT_ALL_POSSIBLE
98 select HAVE_IRQ_WORK
98 select HAVE_PERF_EVENTS 99 select HAVE_PERF_EVENTS
99 select HAVE_KERNEL_GZIP 100 select HAVE_KERNEL_GZIP
100 select HAVE_KERNEL_BZIP2 101 select HAVE_KERNEL_BZIP2
diff --git a/arch/s390/include/asm/perf_event.h b/arch/s390/include/asm/perf_event.h
index 3840cbe77637..a75f168d2718 100644
--- a/arch/s390/include/asm/perf_event.h
+++ b/arch/s390/include/asm/perf_event.h
@@ -4,7 +4,6 @@
4 * Copyright 2009 Martin Schwidefsky, IBM Corporation. 4 * Copyright 2009 Martin Schwidefsky, IBM Corporation.
5 */ 5 */
6 6
7static inline void set_perf_event_pending(void) {} 7/* Empty, just to avoid compiling error */
8static inline void clear_perf_event_pending(void) {}
9 8
10#define PERF_EVENT_INDEX_OFFSET 0 9#define PERF_EVENT_INDEX_OFFSET 0
diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig
index 35b6c3f85173..35b6879628a0 100644
--- a/arch/sh/Kconfig
+++ b/arch/sh/Kconfig
@@ -16,6 +16,7 @@ config SUPERH
16 select HAVE_ARCH_TRACEHOOK 16 select HAVE_ARCH_TRACEHOOK
17 select HAVE_DMA_API_DEBUG 17 select HAVE_DMA_API_DEBUG
18 select HAVE_DMA_ATTRS 18 select HAVE_DMA_ATTRS
19 select HAVE_IRQ_WORK
19 select HAVE_PERF_EVENTS 20 select HAVE_PERF_EVENTS
20 select PERF_USE_VMALLOC 21 select PERF_USE_VMALLOC
21 select HAVE_KERNEL_GZIP 22 select HAVE_KERNEL_GZIP
diff --git a/arch/sh/include/asm/perf_event.h b/arch/sh/include/asm/perf_event.h
index 3d0c9f36d150..14308bed7ea5 100644
--- a/arch/sh/include/asm/perf_event.h
+++ b/arch/sh/include/asm/perf_event.h
@@ -26,11 +26,4 @@ extern int register_sh_pmu(struct sh_pmu *);
26extern int reserve_pmc_hardware(void); 26extern int reserve_pmc_hardware(void);
27extern void release_pmc_hardware(void); 27extern void release_pmc_hardware(void);
28 28
29static inline void set_perf_event_pending(void)
30{
31 /* Nothing to see here, move along. */
32}
33
34#define PERF_EVENT_INDEX_OFFSET 0
35
36#endif /* __ASM_SH_PERF_EVENT_H */ 29#endif /* __ASM_SH_PERF_EVENT_H */
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 9212cd42a832..3e9d31401fb2 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -26,6 +26,7 @@ config SPARC
26 select ARCH_WANT_OPTIONAL_GPIOLIB 26 select ARCH_WANT_OPTIONAL_GPIOLIB
27 select RTC_CLASS 27 select RTC_CLASS
28 select RTC_DRV_M48T59 28 select RTC_DRV_M48T59
29 select HAVE_IRQ_WORK
29 select HAVE_PERF_EVENTS 30 select HAVE_PERF_EVENTS
30 select PERF_USE_VMALLOC 31 select PERF_USE_VMALLOC
31 select HAVE_DMA_ATTRS 32 select HAVE_DMA_ATTRS
@@ -54,6 +55,7 @@ config SPARC64
54 select RTC_DRV_BQ4802 55 select RTC_DRV_BQ4802
55 select RTC_DRV_SUN4V 56 select RTC_DRV_SUN4V
56 select RTC_DRV_STARFIRE 57 select RTC_DRV_STARFIRE
58 select HAVE_IRQ_WORK
57 select HAVE_PERF_EVENTS 59 select HAVE_PERF_EVENTS
58 select PERF_USE_VMALLOC 60 select PERF_USE_VMALLOC
59 61
diff --git a/arch/sparc/include/asm/perf_event.h b/arch/sparc/include/asm/perf_event.h
index 727af70646cb..6e8bfa1786da 100644
--- a/arch/sparc/include/asm/perf_event.h
+++ b/arch/sparc/include/asm/perf_event.h
@@ -1,10 +1,6 @@
1#ifndef __ASM_SPARC_PERF_EVENT_H 1#ifndef __ASM_SPARC_PERF_EVENT_H
2#define __ASM_SPARC_PERF_EVENT_H 2#define __ASM_SPARC_PERF_EVENT_H
3 3
4extern void set_perf_event_pending(void);
5
6#define PERF_EVENT_INDEX_OFFSET 0
7
8#ifdef CONFIG_PERF_EVENTS 4#ifdef CONFIG_PERF_EVENTS
9#include <asm/ptrace.h> 5#include <asm/ptrace.h>
10 6
diff --git a/arch/sparc/kernel/pcr.c b/arch/sparc/kernel/pcr.c
index c4a6a50b4849..b87873c0e8ea 100644
--- a/arch/sparc/kernel/pcr.c
+++ b/arch/sparc/kernel/pcr.c
@@ -7,7 +7,7 @@
7#include <linux/init.h> 7#include <linux/init.h>
8#include <linux/irq.h> 8#include <linux/irq.h>
9 9
10#include <linux/perf_event.h> 10#include <linux/irq_work.h>
11#include <linux/ftrace.h> 11#include <linux/ftrace.h>
12 12
13#include <asm/pil.h> 13#include <asm/pil.h>
@@ -43,14 +43,14 @@ void __irq_entry deferred_pcr_work_irq(int irq, struct pt_regs *regs)
43 43
44 old_regs = set_irq_regs(regs); 44 old_regs = set_irq_regs(regs);
45 irq_enter(); 45 irq_enter();
46#ifdef CONFIG_PERF_EVENTS 46#ifdef CONFIG_IRQ_WORK
47 perf_event_do_pending(); 47 irq_work_run();
48#endif 48#endif
49 irq_exit(); 49 irq_exit();
50 set_irq_regs(old_regs); 50 set_irq_regs(old_regs);
51} 51}
52 52
53void set_perf_event_pending(void) 53void arch_irq_work_raise(void)
54{ 54{
55 set_softint(1 << PIL_DEFERRED_PCR_WORK); 55 set_softint(1 << PIL_DEFERRED_PCR_WORK);
56} 56}
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 9815221976a7..fd227d6b8d9c 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -25,6 +25,7 @@ config X86
25 select HAVE_IDE 25 select HAVE_IDE
26 select HAVE_OPROFILE 26 select HAVE_OPROFILE
27 select HAVE_PERF_EVENTS if (!M386 && !M486) 27 select HAVE_PERF_EVENTS if (!M386 && !M486)
28 select HAVE_IRQ_WORK
28 select HAVE_IOREMAP_PROT 29 select HAVE_IOREMAP_PROT
29 select HAVE_KPROBES 30 select HAVE_KPROBES
30 select ARCH_WANT_OPTIONAL_GPIOLIB 31 select ARCH_WANT_OPTIONAL_GPIOLIB
diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h
index 8e8ec663a98f..b8e96a18676b 100644
--- a/arch/x86/include/asm/entry_arch.h
+++ b/arch/x86/include/asm/entry_arch.h
@@ -49,8 +49,8 @@ BUILD_INTERRUPT(apic_timer_interrupt,LOCAL_TIMER_VECTOR)
49BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR) 49BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR)
50BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR) 50BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR)
51 51
52#ifdef CONFIG_PERF_EVENTS 52#ifdef CONFIG_IRQ_WORK
53BUILD_INTERRUPT(perf_pending_interrupt, LOCAL_PENDING_VECTOR) 53BUILD_INTERRUPT(irq_work_interrupt, IRQ_WORK_VECTOR)
54#endif 54#endif
55 55
56#ifdef CONFIG_X86_THERMAL_VECTOR 56#ifdef CONFIG_X86_THERMAL_VECTOR
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index aeab29aee617..55e4de613f0e 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -14,7 +14,7 @@ typedef struct {
14#endif 14#endif
15 unsigned int x86_platform_ipis; /* arch dependent */ 15 unsigned int x86_platform_ipis; /* arch dependent */
16 unsigned int apic_perf_irqs; 16 unsigned int apic_perf_irqs;
17 unsigned int apic_pending_irqs; 17 unsigned int apic_irq_work_irqs;
18#ifdef CONFIG_SMP 18#ifdef CONFIG_SMP
19 unsigned int irq_resched_count; 19 unsigned int irq_resched_count;
20 unsigned int irq_call_count; 20 unsigned int irq_call_count;
diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
index 46c0fe05f230..3a54a1ca1a02 100644
--- a/arch/x86/include/asm/hw_irq.h
+++ b/arch/x86/include/asm/hw_irq.h
@@ -29,7 +29,7 @@
29extern void apic_timer_interrupt(void); 29extern void apic_timer_interrupt(void);
30extern void x86_platform_ipi(void); 30extern void x86_platform_ipi(void);
31extern void error_interrupt(void); 31extern void error_interrupt(void);
32extern void perf_pending_interrupt(void); 32extern void irq_work_interrupt(void);
33 33
34extern void spurious_interrupt(void); 34extern void spurious_interrupt(void);
35extern void thermal_interrupt(void); 35extern void thermal_interrupt(void);
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index e2ca30092557..6af0894dafb4 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -114,9 +114,9 @@
114#define X86_PLATFORM_IPI_VECTOR 0xed 114#define X86_PLATFORM_IPI_VECTOR 0xed
115 115
116/* 116/*
117 * Performance monitoring pending work vector: 117 * IRQ work vector:
118 */ 118 */
119#define LOCAL_PENDING_VECTOR 0xec 119#define IRQ_WORK_VECTOR 0xec
120 120
121#define UV_BAU_MESSAGE 0xea 121#define UV_BAU_MESSAGE 0xea
122 122
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 9d3f485e5dd0..7490bf8d1459 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -35,6 +35,7 @@ obj-y := process_$(BITS).o signal.o entry_$(BITS).o
35obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o 35obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
36obj-y += time.o ioport.o ldt.o dumpstack.o 36obj-y += time.o ioport.o ldt.o dumpstack.o
37obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o 37obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o
38obj-$(CONFIG_IRQ_WORK) += irq_work.o
38obj-$(CONFIG_X86_VISWS) += visws_quirks.o 39obj-$(CONFIG_X86_VISWS) += visws_quirks.o
39obj-$(CONFIG_X86_32) += probe_roms_32.o 40obj-$(CONFIG_X86_32) += probe_roms_32.o
40obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o 41obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index e2513f26ba8b..fe73c1844a9a 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -1196,25 +1196,6 @@ static int x86_pmu_handle_irq(struct pt_regs *regs)
1196 return handled; 1196 return handled;
1197} 1197}
1198 1198
1199void smp_perf_pending_interrupt(struct pt_regs *regs)
1200{
1201 irq_enter();
1202 ack_APIC_irq();
1203 inc_irq_stat(apic_pending_irqs);
1204 perf_event_do_pending();
1205 irq_exit();
1206}
1207
1208void set_perf_event_pending(void)
1209{
1210#ifdef CONFIG_X86_LOCAL_APIC
1211 if (!x86_pmu.apic || !x86_pmu_initialized())
1212 return;
1213
1214 apic->send_IPI_self(LOCAL_PENDING_VECTOR);
1215#endif
1216}
1217
1218void perf_events_lapic_init(void) 1199void perf_events_lapic_init(void)
1219{ 1200{
1220 if (!x86_pmu.apic || !x86_pmu_initialized()) 1201 if (!x86_pmu.apic || !x86_pmu_initialized())
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 17be5ec7cbba..c375c79065f8 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1023,9 +1023,9 @@ apicinterrupt ERROR_APIC_VECTOR \
1023apicinterrupt SPURIOUS_APIC_VECTOR \ 1023apicinterrupt SPURIOUS_APIC_VECTOR \
1024 spurious_interrupt smp_spurious_interrupt 1024 spurious_interrupt smp_spurious_interrupt
1025 1025
1026#ifdef CONFIG_PERF_EVENTS 1026#ifdef CONFIG_IRQ_WORK
1027apicinterrupt LOCAL_PENDING_VECTOR \ 1027apicinterrupt IRQ_WORK_VECTOR \
1028 perf_pending_interrupt smp_perf_pending_interrupt 1028 irq_work_interrupt smp_irq_work_interrupt
1029#endif 1029#endif
1030 1030
1031/* 1031/*
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 91fd0c70a18a..44edb03fc9ec 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -67,10 +67,10 @@ static int show_other_interrupts(struct seq_file *p, int prec)
67 for_each_online_cpu(j) 67 for_each_online_cpu(j)
68 seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs); 68 seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
69 seq_printf(p, " Performance monitoring interrupts\n"); 69 seq_printf(p, " Performance monitoring interrupts\n");
70 seq_printf(p, "%*s: ", prec, "PND"); 70 seq_printf(p, "%*s: ", prec, "IWI");
71 for_each_online_cpu(j) 71 for_each_online_cpu(j)
72 seq_printf(p, "%10u ", irq_stats(j)->apic_pending_irqs); 72 seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
73 seq_printf(p, " Performance pending work\n"); 73 seq_printf(p, " IRQ work interrupts\n");
74#endif 74#endif
75 if (x86_platform_ipi_callback) { 75 if (x86_platform_ipi_callback) {
76 seq_printf(p, "%*s: ", prec, "PLT"); 76 seq_printf(p, "%*s: ", prec, "PLT");
@@ -185,7 +185,7 @@ u64 arch_irq_stat_cpu(unsigned int cpu)
185 sum += irq_stats(cpu)->apic_timer_irqs; 185 sum += irq_stats(cpu)->apic_timer_irqs;
186 sum += irq_stats(cpu)->irq_spurious_count; 186 sum += irq_stats(cpu)->irq_spurious_count;
187 sum += irq_stats(cpu)->apic_perf_irqs; 187 sum += irq_stats(cpu)->apic_perf_irqs;
188 sum += irq_stats(cpu)->apic_pending_irqs; 188 sum += irq_stats(cpu)->apic_irq_work_irqs;
189#endif 189#endif
190 if (x86_platform_ipi_callback) 190 if (x86_platform_ipi_callback)
191 sum += irq_stats(cpu)->x86_platform_ipis; 191 sum += irq_stats(cpu)->x86_platform_ipis;
diff --git a/arch/x86/kernel/irq_work.c b/arch/x86/kernel/irq_work.c
new file mode 100644
index 000000000000..ca8f703a1e70
--- /dev/null
+++ b/arch/x86/kernel/irq_work.c
@@ -0,0 +1,30 @@
1/*
2 * x86 specific code for irq_work
3 *
4 * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
5 */
6
7#include <linux/kernel.h>
8#include <linux/irq_work.h>
9#include <linux/hardirq.h>
10#include <asm/apic.h>
11
12void smp_irq_work_interrupt(struct pt_regs *regs)
13{
14 irq_enter();
15 ack_APIC_irq();
16 inc_irq_stat(apic_irq_work_irqs);
17 irq_work_run();
18 irq_exit();
19}
20
21void arch_irq_work_raise(void)
22{
23#ifdef CONFIG_X86_LOCAL_APIC
24 if (!cpu_has_apic)
25 return;
26
27 apic->send_IPI_self(IRQ_WORK_VECTOR);
28 apic_wait_icr_idle();
29#endif
30}
diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
index 990ae7cfc578..713969b9266b 100644
--- a/arch/x86/kernel/irqinit.c
+++ b/arch/x86/kernel/irqinit.c
@@ -224,9 +224,9 @@ static void __init apic_intr_init(void)
224 alloc_intr_gate(SPURIOUS_APIC_VECTOR, spurious_interrupt); 224 alloc_intr_gate(SPURIOUS_APIC_VECTOR, spurious_interrupt);
225 alloc_intr_gate(ERROR_APIC_VECTOR, error_interrupt); 225 alloc_intr_gate(ERROR_APIC_VECTOR, error_interrupt);
226 226
227 /* Performance monitoring interrupts: */ 227 /* IRQ work interrupts: */
228# ifdef CONFIG_PERF_EVENTS 228# ifdef CONFIG_IRQ_WORK
229 alloc_intr_gate(LOCAL_PENDING_VECTOR, perf_pending_interrupt); 229 alloc_intr_gate(IRQ_WORK_VECTOR, irq_work_interrupt);
230# endif 230# endif
231 231
232#endif 232#endif
diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h
new file mode 100644
index 000000000000..4fa09d4d0b71
--- /dev/null
+++ b/include/linux/irq_work.h
@@ -0,0 +1,20 @@
1#ifndef _LINUX_IRQ_WORK_H
2#define _LINUX_IRQ_WORK_H
3
4struct irq_work {
5 struct irq_work *next;
6 void (*func)(struct irq_work *);
7};
8
9static inline
10void init_irq_work(struct irq_work *entry, void (*func)(struct irq_work *))
11{
12 entry->next = NULL;
13 entry->func = func;
14}
15
16bool irq_work_queue(struct irq_work *entry);
17void irq_work_run(void);
18void irq_work_sync(struct irq_work *entry);
19
20#endif /* _LINUX_IRQ_WORK_H */
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index a9227e985207..2ebfc9ae4755 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -486,6 +486,7 @@ struct perf_guest_info_callbacks {
486#include <linux/workqueue.h> 486#include <linux/workqueue.h>
487#include <linux/ftrace.h> 487#include <linux/ftrace.h>
488#include <linux/cpu.h> 488#include <linux/cpu.h>
489#include <linux/irq_work.h>
489#include <asm/atomic.h> 490#include <asm/atomic.h>
490#include <asm/local.h> 491#include <asm/local.h>
491 492
@@ -672,11 +673,6 @@ struct perf_buffer {
672 void *data_pages[0]; 673 void *data_pages[0];
673}; 674};
674 675
675struct perf_pending_entry {
676 struct perf_pending_entry *next;
677 void (*func)(struct perf_pending_entry *);
678};
679
680struct perf_sample_data; 676struct perf_sample_data;
681 677
682typedef void (*perf_overflow_handler_t)(struct perf_event *, int, 678typedef void (*perf_overflow_handler_t)(struct perf_event *, int,
@@ -784,7 +780,7 @@ struct perf_event {
784 int pending_wakeup; 780 int pending_wakeup;
785 int pending_kill; 781 int pending_kill;
786 int pending_disable; 782 int pending_disable;
787 struct perf_pending_entry pending; 783 struct irq_work pending;
788 784
789 atomic_t event_limit; 785 atomic_t event_limit;
790 786
@@ -898,8 +894,6 @@ extern int perf_event_init_task(struct task_struct *child);
898extern void perf_event_exit_task(struct task_struct *child); 894extern void perf_event_exit_task(struct task_struct *child);
899extern void perf_event_free_task(struct task_struct *task); 895extern void perf_event_free_task(struct task_struct *task);
900extern void perf_event_delayed_put(struct task_struct *task); 896extern void perf_event_delayed_put(struct task_struct *task);
901extern void set_perf_event_pending(void);
902extern void perf_event_do_pending(void);
903extern void perf_event_print_debug(void); 897extern void perf_event_print_debug(void);
904extern void perf_pmu_disable(struct pmu *pmu); 898extern void perf_pmu_disable(struct pmu *pmu);
905extern void perf_pmu_enable(struct pmu *pmu); 899extern void perf_pmu_enable(struct pmu *pmu);
@@ -1078,7 +1072,6 @@ static inline int perf_event_init_task(struct task_struct *child) { return 0; }
1078static inline void perf_event_exit_task(struct task_struct *child) { } 1072static inline void perf_event_exit_task(struct task_struct *child) { }
1079static inline void perf_event_free_task(struct task_struct *task) { } 1073static inline void perf_event_free_task(struct task_struct *task) { }
1080static inline void perf_event_delayed_put(struct task_struct *task) { } 1074static inline void perf_event_delayed_put(struct task_struct *task) { }
1081static inline void perf_event_do_pending(void) { }
1082static inline void perf_event_print_debug(void) { } 1075static inline void perf_event_print_debug(void) { }
1083static inline int perf_event_task_disable(void) { return -EINVAL; } 1076static inline int perf_event_task_disable(void) { return -EINVAL; }
1084static inline int perf_event_task_enable(void) { return -EINVAL; } 1077static inline int perf_event_task_enable(void) { return -EINVAL; }
diff --git a/init/Kconfig b/init/Kconfig
index 2de5b1cbadd9..1ef0b439908e 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -21,6 +21,13 @@ config CONSTRUCTORS
21 depends on !UML 21 depends on !UML
22 default y 22 default y
23 23
24config HAVE_IRQ_WORK
25 bool
26
27config IRQ_WORK
28 bool
29 depends on HAVE_IRQ_WORK
30
24menu "General setup" 31menu "General setup"
25 32
26config EXPERIMENTAL 33config EXPERIMENTAL
@@ -987,6 +994,7 @@ config PERF_EVENTS
987 default y if (PROFILING || PERF_COUNTERS) 994 default y if (PROFILING || PERF_COUNTERS)
988 depends on HAVE_PERF_EVENTS 995 depends on HAVE_PERF_EVENTS
989 select ANON_INODES 996 select ANON_INODES
997 select IRQ_WORK
990 help 998 help
991 Enable kernel support for various performance events provided 999 Enable kernel support for various performance events provided
992 by software and hardware. 1000 by software and hardware.
diff --git a/kernel/Makefile b/kernel/Makefile
index d52b473c99a1..4d9bf5f8531f 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -23,6 +23,7 @@ CFLAGS_REMOVE_rtmutex-debug.o = -pg
23CFLAGS_REMOVE_cgroup-debug.o = -pg 23CFLAGS_REMOVE_cgroup-debug.o = -pg
24CFLAGS_REMOVE_sched_clock.o = -pg 24CFLAGS_REMOVE_sched_clock.o = -pg
25CFLAGS_REMOVE_perf_event.o = -pg 25CFLAGS_REMOVE_perf_event.o = -pg
26CFLAGS_REMOVE_irq_work.o = -pg
26endif 27endif
27 28
28obj-$(CONFIG_FREEZER) += freezer.o 29obj-$(CONFIG_FREEZER) += freezer.o
@@ -100,6 +101,7 @@ obj-$(CONFIG_TRACING) += trace/
100obj-$(CONFIG_X86_DS) += trace/ 101obj-$(CONFIG_X86_DS) += trace/
101obj-$(CONFIG_RING_BUFFER) += trace/ 102obj-$(CONFIG_RING_BUFFER) += trace/
102obj-$(CONFIG_SMP) += sched_cpupri.o 103obj-$(CONFIG_SMP) += sched_cpupri.o
104obj-$(CONFIG_IRQ_WORK) += irq_work.o
103obj-$(CONFIG_PERF_EVENTS) += perf_event.o 105obj-$(CONFIG_PERF_EVENTS) += perf_event.o
104obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o 106obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
105obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o 107obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
diff --git a/kernel/irq_work.c b/kernel/irq_work.c
new file mode 100644
index 000000000000..f16763ff8481
--- /dev/null
+++ b/kernel/irq_work.c
@@ -0,0 +1,164 @@
1/*
2 * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
3 *
4 * Provides a framework for enqueueing and running callbacks from hardirq
5 * context. The enqueueing is NMI-safe.
6 */
7
8#include <linux/kernel.h>
9#include <linux/module.h>
10#include <linux/irq_work.h>
11#include <linux/hardirq.h>
12
13/*
14 * An entry can be in one of four states:
15 *
16 * free NULL, 0 -> {claimed} : free to be used
17 * claimed NULL, 3 -> {pending} : claimed to be enqueued
18 * pending next, 3 -> {busy} : queued, pending callback
19 * busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
20 *
21 * We use the lower two bits of the next pointer to keep PENDING and BUSY
22 * flags.
23 */
24
25#define IRQ_WORK_PENDING 1UL
26#define IRQ_WORK_BUSY 2UL
27#define IRQ_WORK_FLAGS 3UL
28
29static inline bool irq_work_is_set(struct irq_work *entry, int flags)
30{
31 return (unsigned long)entry->next & flags;
32}
33
34static inline struct irq_work *irq_work_next(struct irq_work *entry)
35{
36 unsigned long next = (unsigned long)entry->next;
37 next &= ~IRQ_WORK_FLAGS;
38 return (struct irq_work *)next;
39}
40
41static inline struct irq_work *next_flags(struct irq_work *entry, int flags)
42{
43 unsigned long next = (unsigned long)entry;
44 next |= flags;
45 return (struct irq_work *)next;
46}
47
48static DEFINE_PER_CPU(struct irq_work *, irq_work_list);
49
50/*
51 * Claim the entry so that no one else will poke at it.
52 */
53static bool irq_work_claim(struct irq_work *entry)
54{
55 struct irq_work *next, *nflags;
56
57 do {
58 next = entry->next;
59 if ((unsigned long)next & IRQ_WORK_PENDING)
60 return false;
61 nflags = next_flags(next, IRQ_WORK_FLAGS);
62 } while (cmpxchg(&entry->next, next, nflags) != next);
63
64 return true;
65}
66
67
68void __weak arch_irq_work_raise(void)
69{
70 /*
71 * Lame architectures will get the timer tick callback
72 */
73}
74
75/*
76 * Queue the entry and raise the IPI if needed.
77 */
78static void __irq_work_queue(struct irq_work *entry)
79{
80 struct irq_work **head, *next;
81
82 head = &get_cpu_var(irq_work_list);
83
84 do {
85 next = *head;
86 /* Can assign non-atomic because we keep the flags set. */
87 entry->next = next_flags(next, IRQ_WORK_FLAGS);
88 } while (cmpxchg(head, next, entry) != next);
89
90 /* The list was empty, raise self-interrupt to start processing. */
91 if (!irq_work_next(entry))
92 arch_irq_work_raise();
93
94 put_cpu_var(irq_work_list);
95}
96
97/*
98 * Enqueue the irq_work @entry, returns true on success, failure when the
99 * @entry was already enqueued by someone else.
100 *
101 * Can be re-enqueued while the callback is still in progress.
102 */
103bool irq_work_queue(struct irq_work *entry)
104{
105 if (!irq_work_claim(entry)) {
106 /*
107 * Already enqueued, can't do!
108 */
109 return false;
110 }
111
112 __irq_work_queue(entry);
113 return true;
114}
115EXPORT_SYMBOL_GPL(irq_work_queue);
116
117/*
118 * Run the irq_work entries on this cpu. Requires to be ran from hardirq
119 * context with local IRQs disabled.
120 */
121void irq_work_run(void)
122{
123 struct irq_work *list, **head;
124
125 head = &__get_cpu_var(irq_work_list);
126 if (*head == NULL)
127 return;
128
129 BUG_ON(!in_irq());
130 BUG_ON(!irqs_disabled());
131
132 list = xchg(head, NULL);
133 while (list != NULL) {
134 struct irq_work *entry = list;
135
136 list = irq_work_next(list);
137
138 /*
139 * Clear the PENDING bit, after this point the @entry
140 * can be re-used.
141 */
142 entry->next = next_flags(NULL, IRQ_WORK_BUSY);
143 entry->func(entry);
144 /*
145 * Clear the BUSY bit and return to the free state if
146 * no-one else claimed it meanwhile.
147 */
148 cmpxchg(&entry->next, next_flags(NULL, IRQ_WORK_BUSY), NULL);
149 }
150}
151EXPORT_SYMBOL_GPL(irq_work_run);
152
153/*
154 * Synchronize against the irq_work @entry, ensures the entry is not
155 * currently in use.
156 */
157void irq_work_sync(struct irq_work *entry)
158{
159 WARN_ON_ONCE(irqs_disabled());
160
161 while (irq_work_is_set(entry, IRQ_WORK_BUSY))
162 cpu_relax();
163}
164EXPORT_SYMBOL_GPL(irq_work_sync);
diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index 634f86a4b2f9..99b9700e74d0 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -2206,12 +2206,11 @@ static void free_event_rcu(struct rcu_head *head)
2206 kfree(event); 2206 kfree(event);
2207} 2207}
2208 2208
2209static void perf_pending_sync(struct perf_event *event);
2210static void perf_buffer_put(struct perf_buffer *buffer); 2209static void perf_buffer_put(struct perf_buffer *buffer);
2211 2210
2212static void free_event(struct perf_event *event) 2211static void free_event(struct perf_event *event)
2213{ 2212{
2214 perf_pending_sync(event); 2213 irq_work_sync(&event->pending);
2215 2214
2216 if (!event->parent) { 2215 if (!event->parent) {
2217 atomic_dec(&nr_events); 2216 atomic_dec(&nr_events);
@@ -3162,16 +3161,7 @@ void perf_event_wakeup(struct perf_event *event)
3162 } 3161 }
3163} 3162}
3164 3163
3165/* 3164static void perf_pending_event(struct irq_work *entry)
3166 * Pending wakeups
3167 *
3168 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
3169 *
3170 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
3171 * single linked list and use cmpxchg() to add entries lockless.
3172 */
3173
3174static void perf_pending_event(struct perf_pending_entry *entry)
3175{ 3165{
3176 struct perf_event *event = container_of(entry, 3166 struct perf_event *event = container_of(entry,
3177 struct perf_event, pending); 3167 struct perf_event, pending);
@@ -3187,89 +3177,6 @@ static void perf_pending_event(struct perf_pending_entry *entry)
3187 } 3177 }
3188} 3178}
3189 3179
3190#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
3191
3192static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
3193 PENDING_TAIL,
3194};
3195
3196static void perf_pending_queue(struct perf_pending_entry *entry,
3197 void (*func)(struct perf_pending_entry *))
3198{
3199 struct perf_pending_entry **head;
3200
3201 if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
3202 return;
3203
3204 entry->func = func;
3205
3206 head = &get_cpu_var(perf_pending_head);
3207
3208 do {
3209 entry->next = *head;
3210 } while (cmpxchg(head, entry->next, entry) != entry->next);
3211
3212 set_perf_event_pending();
3213
3214 put_cpu_var(perf_pending_head);
3215}
3216
3217static int __perf_pending_run(void)
3218{
3219 struct perf_pending_entry *list;
3220 int nr = 0;
3221
3222 list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
3223 while (list != PENDING_TAIL) {
3224 void (*func)(struct perf_pending_entry *);
3225 struct perf_pending_entry *entry = list;
3226
3227 list = list->next;
3228
3229 func = entry->func;
3230 entry->next = NULL;
3231 /*
3232 * Ensure we observe the unqueue before we issue the wakeup,
3233 * so that we won't be waiting forever.
3234 * -- see perf_not_pending().
3235 */
3236 smp_wmb();
3237
3238 func(entry);
3239 nr++;
3240 }
3241
3242 return nr;
3243}
3244
3245static inline int perf_not_pending(struct perf_event *event)
3246{
3247 /*
3248 * If we flush on whatever cpu we run, there is a chance we don't
3249 * need to wait.
3250 */
3251 get_cpu();
3252 __perf_pending_run();
3253 put_cpu();
3254
3255 /*
3256 * Ensure we see the proper queue state before going to sleep
3257 * so that we do not miss the wakeup. -- see perf_pending_handle()
3258 */
3259 smp_rmb();
3260 return event->pending.next == NULL;
3261}
3262
3263static void perf_pending_sync(struct perf_event *event)
3264{
3265 wait_event(event->waitq, perf_not_pending(event));
3266}
3267
3268void perf_event_do_pending(void)
3269{
3270 __perf_pending_run();
3271}
3272
3273/* 3180/*
3274 * We assume there is only KVM supporting the callbacks. 3181 * We assume there is only KVM supporting the callbacks.
3275 * Later on, we might change it to a list if there is 3182 * Later on, we might change it to a list if there is
@@ -3319,8 +3226,7 @@ static void perf_output_wakeup(struct perf_output_handle *handle)
3319 3226
3320 if (handle->nmi) { 3227 if (handle->nmi) {
3321 handle->event->pending_wakeup = 1; 3228 handle->event->pending_wakeup = 1;
3322 perf_pending_queue(&handle->event->pending, 3229 irq_work_queue(&handle->event->pending);
3323 perf_pending_event);
3324 } else 3230 } else
3325 perf_event_wakeup(handle->event); 3231 perf_event_wakeup(handle->event);
3326} 3232}
@@ -4356,8 +4262,7 @@ static int __perf_event_overflow(struct perf_event *event, int nmi,
4356 event->pending_kill = POLL_HUP; 4262 event->pending_kill = POLL_HUP;
4357 if (nmi) { 4263 if (nmi) {
4358 event->pending_disable = 1; 4264 event->pending_disable = 1;
4359 perf_pending_queue(&event->pending, 4265 irq_work_queue(&event->pending);
4360 perf_pending_event);
4361 } else 4266 } else
4362 perf_event_disable(event); 4267 perf_event_disable(event);
4363 } 4268 }
@@ -5374,6 +5279,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
5374 INIT_LIST_HEAD(&event->event_entry); 5279 INIT_LIST_HEAD(&event->event_entry);
5375 INIT_LIST_HEAD(&event->sibling_list); 5280 INIT_LIST_HEAD(&event->sibling_list);
5376 init_waitqueue_head(&event->waitq); 5281 init_waitqueue_head(&event->waitq);
5282 init_irq_work(&event->pending, perf_pending_event);
5377 5283
5378 mutex_init(&event->mmap_mutex); 5284 mutex_init(&event->mmap_mutex);
5379 5285
diff --git a/kernel/timer.c b/kernel/timer.c
index 97bf05baade7..68a9ae7679b7 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -37,7 +37,7 @@
37#include <linux/delay.h> 37#include <linux/delay.h>
38#include <linux/tick.h> 38#include <linux/tick.h>
39#include <linux/kallsyms.h> 39#include <linux/kallsyms.h>
40#include <linux/perf_event.h> 40#include <linux/irq_work.h>
41#include <linux/sched.h> 41#include <linux/sched.h>
42#include <linux/slab.h> 42#include <linux/slab.h>
43 43
@@ -1279,7 +1279,10 @@ void update_process_times(int user_tick)
1279 run_local_timers(); 1279 run_local_timers();
1280 rcu_check_callbacks(cpu, user_tick); 1280 rcu_check_callbacks(cpu, user_tick);
1281 printk_tick(); 1281 printk_tick();
1282 perf_event_do_pending(); 1282#ifdef CONFIG_IRQ_WORK
1283 if (in_irq())
1284 irq_work_run();
1285#endif
1283 scheduler_tick(); 1286 scheduler_tick();
1284 run_posix_cpu_timers(p); 1287 run_posix_cpu_timers(p);
1285} 1288}
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-09 23:39:48 -0500