diff options
Diffstat (limited to 'arch/x86/kernel/cpu/perf_event.c')
-rw-r--r-- | arch/x86/kernel/cpu/perf_event.c | 442 |
1 files changed, 28 insertions, 414 deletions
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c index cfa62ec090ec..640891014b2a 100644 --- a/arch/x86/kernel/cpu/perf_event.c +++ b/arch/x86/kernel/cpu/perf_event.c | |||
@@ -32,6 +32,8 @@ | |||
32 | #include <asm/smp.h> | 32 | #include <asm/smp.h> |
33 | #include <asm/alternative.h> | 33 | #include <asm/alternative.h> |
34 | 34 | ||
35 | #include "perf_event.h" | ||
36 | |||
35 | #if 0 | 37 | #if 0 |
36 | #undef wrmsrl | 38 | #undef wrmsrl |
37 | #define wrmsrl(msr, val) \ | 39 | #define wrmsrl(msr, val) \ |
@@ -43,283 +45,17 @@ do { \ | |||
43 | } while (0) | 45 | } while (0) |
44 | #endif | 46 | #endif |
45 | 47 | ||
46 | /* | 48 | struct x86_pmu x86_pmu __read_mostly; |
47 | * | NHM/WSM | SNB | | ||
48 | * register ------------------------------- | ||
49 | * | HT | no HT | HT | no HT | | ||
50 | *----------------------------------------- | ||
51 | * offcore | core | core | cpu | core | | ||
52 | * lbr_sel | core | core | cpu | core | | ||
53 | * ld_lat | cpu | core | cpu | core | | ||
54 | *----------------------------------------- | ||
55 | * | ||
56 | * Given that there is a small number of shared regs, | ||
57 | * we can pre-allocate their slot in the per-cpu | ||
58 | * per-core reg tables. | ||
59 | */ | ||
60 | enum extra_reg_type { | ||
61 | EXTRA_REG_NONE = -1, /* not used */ | ||
62 | |||
63 | EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */ | ||
64 | EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */ | ||
65 | |||
66 | EXTRA_REG_MAX /* number of entries needed */ | ||
67 | }; | ||
68 | |||
69 | struct event_constraint { | ||
70 | union { | ||
71 | unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; | ||
72 | u64 idxmsk64; | ||
73 | }; | ||
74 | u64 code; | ||
75 | u64 cmask; | ||
76 | int weight; | ||
77 | }; | ||
78 | |||
79 | struct amd_nb { | ||
80 | int nb_id; /* NorthBridge id */ | ||
81 | int refcnt; /* reference count */ | ||
82 | struct perf_event *owners[X86_PMC_IDX_MAX]; | ||
83 | struct event_constraint event_constraints[X86_PMC_IDX_MAX]; | ||
84 | }; | ||
85 | |||
86 | struct intel_percore; | ||
87 | |||
88 | #define MAX_LBR_ENTRIES 16 | ||
89 | |||
90 | struct cpu_hw_events { | ||
91 | /* | ||
92 | * Generic x86 PMC bits | ||
93 | */ | ||
94 | struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */ | ||
95 | unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; | ||
96 | unsigned long running[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; | ||
97 | int enabled; | ||
98 | |||
99 | int n_events; | ||
100 | int n_added; | ||
101 | int n_txn; | ||
102 | int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */ | ||
103 | u64 tags[X86_PMC_IDX_MAX]; | ||
104 | struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */ | ||
105 | |||
106 | unsigned int group_flag; | ||
107 | |||
108 | /* | ||
109 | * Intel DebugStore bits | ||
110 | */ | ||
111 | struct debug_store *ds; | ||
112 | u64 pebs_enabled; | ||
113 | |||
114 | /* | ||
115 | * Intel LBR bits | ||
116 | */ | ||
117 | int lbr_users; | ||
118 | void *lbr_context; | ||
119 | struct perf_branch_stack lbr_stack; | ||
120 | struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES]; | ||
121 | |||
122 | /* | ||
123 | * manage shared (per-core, per-cpu) registers | ||
124 | * used on Intel NHM/WSM/SNB | ||
125 | */ | ||
126 | struct intel_shared_regs *shared_regs; | ||
127 | |||
128 | /* | ||
129 | * AMD specific bits | ||
130 | */ | ||
131 | struct amd_nb *amd_nb; | ||
132 | }; | ||
133 | |||
134 | #define __EVENT_CONSTRAINT(c, n, m, w) {\ | ||
135 | { .idxmsk64 = (n) }, \ | ||
136 | .code = (c), \ | ||
137 | .cmask = (m), \ | ||
138 | .weight = (w), \ | ||
139 | } | ||
140 | |||
141 | #define EVENT_CONSTRAINT(c, n, m) \ | ||
142 | __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n)) | ||
143 | |||
144 | /* | ||
145 | * Constraint on the Event code. | ||
146 | */ | ||
147 | #define INTEL_EVENT_CONSTRAINT(c, n) \ | ||
148 | EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT) | ||
149 | |||
150 | /* | ||
151 | * Constraint on the Event code + UMask + fixed-mask | ||
152 | * | ||
153 | * filter mask to validate fixed counter events. | ||
154 | * the following filters disqualify for fixed counters: | ||
155 | * - inv | ||
156 | * - edge | ||
157 | * - cnt-mask | ||
158 | * The other filters are supported by fixed counters. | ||
159 | * The any-thread option is supported starting with v3. | ||
160 | */ | ||
161 | #define FIXED_EVENT_CONSTRAINT(c, n) \ | ||
162 | EVENT_CONSTRAINT(c, (1ULL << (32+n)), X86_RAW_EVENT_MASK) | ||
163 | |||
164 | /* | ||
165 | * Constraint on the Event code + UMask | ||
166 | */ | ||
167 | #define INTEL_UEVENT_CONSTRAINT(c, n) \ | ||
168 | EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK) | ||
169 | |||
170 | #define EVENT_CONSTRAINT_END \ | ||
171 | EVENT_CONSTRAINT(0, 0, 0) | ||
172 | |||
173 | #define for_each_event_constraint(e, c) \ | ||
174 | for ((e) = (c); (e)->weight; (e)++) | ||
175 | |||
176 | /* | ||
177 | * Per register state. | ||
178 | */ | ||
179 | struct er_account { | ||
180 | raw_spinlock_t lock; /* per-core: protect structure */ | ||
181 | u64 config; /* extra MSR config */ | ||
182 | u64 reg; /* extra MSR number */ | ||
183 | atomic_t ref; /* reference count */ | ||
184 | }; | ||
185 | |||
186 | /* | ||
187 | * Extra registers for specific events. | ||
188 | * | ||
189 | * Some events need large masks and require external MSRs. | ||
190 | * Those extra MSRs end up being shared for all events on | ||
191 | * a PMU and sometimes between PMU of sibling HT threads. | ||
192 | * In either case, the kernel needs to handle conflicting | ||
193 | * accesses to those extra, shared, regs. The data structure | ||
194 | * to manage those registers is stored in cpu_hw_event. | ||
195 | */ | ||
196 | struct extra_reg { | ||
197 | unsigned int event; | ||
198 | unsigned int msr; | ||
199 | u64 config_mask; | ||
200 | u64 valid_mask; | ||
201 | int idx; /* per_xxx->regs[] reg index */ | ||
202 | }; | ||
203 | |||
204 | #define EVENT_EXTRA_REG(e, ms, m, vm, i) { \ | ||
205 | .event = (e), \ | ||
206 | .msr = (ms), \ | ||
207 | .config_mask = (m), \ | ||
208 | .valid_mask = (vm), \ | ||
209 | .idx = EXTRA_REG_##i \ | ||
210 | } | ||
211 | |||
212 | #define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx) \ | ||
213 | EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx) | ||
214 | |||
215 | #define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0) | ||
216 | |||
217 | union perf_capabilities { | ||
218 | struct { | ||
219 | u64 lbr_format : 6; | ||
220 | u64 pebs_trap : 1; | ||
221 | u64 pebs_arch_reg : 1; | ||
222 | u64 pebs_format : 4; | ||
223 | u64 smm_freeze : 1; | ||
224 | }; | ||
225 | u64 capabilities; | ||
226 | }; | ||
227 | |||
228 | /* | ||
229 | * struct x86_pmu - generic x86 pmu | ||
230 | */ | ||
231 | struct x86_pmu { | ||
232 | /* | ||
233 | * Generic x86 PMC bits | ||
234 | */ | ||
235 | const char *name; | ||
236 | int version; | ||
237 | int (*handle_irq)(struct pt_regs *); | ||
238 | void (*disable_all)(void); | ||
239 | void (*enable_all)(int added); | ||
240 | void (*enable)(struct perf_event *); | ||
241 | void (*disable)(struct perf_event *); | ||
242 | int (*hw_config)(struct perf_event *event); | ||
243 | int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign); | ||
244 | unsigned eventsel; | ||
245 | unsigned perfctr; | ||
246 | u64 (*event_map)(int); | ||
247 | int max_events; | ||
248 | int num_counters; | ||
249 | int num_counters_fixed; | ||
250 | int cntval_bits; | ||
251 | u64 cntval_mask; | ||
252 | int apic; | ||
253 | u64 max_period; | ||
254 | struct event_constraint * | ||
255 | (*get_event_constraints)(struct cpu_hw_events *cpuc, | ||
256 | struct perf_event *event); | ||
257 | |||
258 | void (*put_event_constraints)(struct cpu_hw_events *cpuc, | ||
259 | struct perf_event *event); | ||
260 | struct event_constraint *event_constraints; | ||
261 | void (*quirks)(void); | ||
262 | int perfctr_second_write; | ||
263 | |||
264 | int (*cpu_prepare)(int cpu); | ||
265 | void (*cpu_starting)(int cpu); | ||
266 | void (*cpu_dying)(int cpu); | ||
267 | void (*cpu_dead)(int cpu); | ||
268 | |||
269 | /* | ||
270 | * Intel Arch Perfmon v2+ | ||
271 | */ | ||
272 | u64 intel_ctrl; | ||
273 | union perf_capabilities intel_cap; | ||
274 | 49 | ||
275 | /* | 50 | DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { |
276 | * Intel DebugStore bits | ||
277 | */ | ||
278 | int bts, pebs; | ||
279 | int bts_active, pebs_active; | ||
280 | int pebs_record_size; | ||
281 | void (*drain_pebs)(struct pt_regs *regs); | ||
282 | struct event_constraint *pebs_constraints; | ||
283 | |||
284 | /* | ||
285 | * Intel LBR | ||
286 | */ | ||
287 | unsigned long lbr_tos, lbr_from, lbr_to; /* MSR base regs */ | ||
288 | int lbr_nr; /* hardware stack size */ | ||
289 | |||
290 | /* | ||
291 | * Extra registers for events | ||
292 | */ | ||
293 | struct extra_reg *extra_regs; | ||
294 | unsigned int er_flags; | ||
295 | }; | ||
296 | |||
297 | #define ERF_NO_HT_SHARING 1 | ||
298 | #define ERF_HAS_RSP_1 2 | ||
299 | |||
300 | static struct x86_pmu x86_pmu __read_mostly; | ||
301 | |||
302 | static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { | ||
303 | .enabled = 1, | 51 | .enabled = 1, |
304 | }; | 52 | }; |
305 | 53 | ||
306 | static int x86_perf_event_set_period(struct perf_event *event); | 54 | u64 __read_mostly hw_cache_event_ids |
307 | |||
308 | /* | ||
309 | * Generalized hw caching related hw_event table, filled | ||
310 | * in on a per model basis. A value of 0 means | ||
311 | * 'not supported', -1 means 'hw_event makes no sense on | ||
312 | * this CPU', any other value means the raw hw_event | ||
313 | * ID. | ||
314 | */ | ||
315 | |||
316 | #define C(x) PERF_COUNT_HW_CACHE_##x | ||
317 | |||
318 | static u64 __read_mostly hw_cache_event_ids | ||
319 | [PERF_COUNT_HW_CACHE_MAX] | 55 | [PERF_COUNT_HW_CACHE_MAX] |
320 | [PERF_COUNT_HW_CACHE_OP_MAX] | 56 | [PERF_COUNT_HW_CACHE_OP_MAX] |
321 | [PERF_COUNT_HW_CACHE_RESULT_MAX]; | 57 | [PERF_COUNT_HW_CACHE_RESULT_MAX]; |
322 | static u64 __read_mostly hw_cache_extra_regs | 58 | u64 __read_mostly hw_cache_extra_regs |
323 | [PERF_COUNT_HW_CACHE_MAX] | 59 | [PERF_COUNT_HW_CACHE_MAX] |
324 | [PERF_COUNT_HW_CACHE_OP_MAX] | 60 | [PERF_COUNT_HW_CACHE_OP_MAX] |
325 | [PERF_COUNT_HW_CACHE_RESULT_MAX]; | 61 | [PERF_COUNT_HW_CACHE_RESULT_MAX]; |
@@ -329,8 +65,7 @@ static u64 __read_mostly hw_cache_extra_regs | |||
329 | * Can only be executed on the CPU where the event is active. | 65 | * Can only be executed on the CPU where the event is active. |
330 | * Returns the delta events processed. | 66 | * Returns the delta events processed. |
331 | */ | 67 | */ |
332 | static u64 | 68 | u64 x86_perf_event_update(struct perf_event *event) |
333 | x86_perf_event_update(struct perf_event *event) | ||
334 | { | 69 | { |
335 | struct hw_perf_event *hwc = &event->hw; | 70 | struct hw_perf_event *hwc = &event->hw; |
336 | int shift = 64 - x86_pmu.cntval_bits; | 71 | int shift = 64 - x86_pmu.cntval_bits; |
@@ -373,30 +108,6 @@ again: | |||
373 | return new_raw_count; | 108 | return new_raw_count; |
374 | } | 109 | } |
375 | 110 | ||
376 | static inline int x86_pmu_addr_offset(int index) | ||
377 | { | ||
378 | int offset; | ||
379 | |||
380 | /* offset = X86_FEATURE_PERFCTR_CORE ? index << 1 : index */ | ||
381 | alternative_io(ASM_NOP2, | ||
382 | "shll $1, %%eax", | ||
383 | X86_FEATURE_PERFCTR_CORE, | ||
384 | "=a" (offset), | ||
385 | "a" (index)); | ||
386 | |||
387 | return offset; | ||
388 | } | ||
389 | |||
390 | static inline unsigned int x86_pmu_config_addr(int index) | ||
391 | { | ||
392 | return x86_pmu.eventsel + x86_pmu_addr_offset(index); | ||
393 | } | ||
394 | |||
395 | static inline unsigned int x86_pmu_event_addr(int index) | ||
396 | { | ||
397 | return x86_pmu.perfctr + x86_pmu_addr_offset(index); | ||
398 | } | ||
399 | |||
400 | /* | 111 | /* |
401 | * Find and validate any extra registers to set up. | 112 | * Find and validate any extra registers to set up. |
402 | */ | 113 | */ |
@@ -532,9 +243,6 @@ msr_fail: | |||
532 | return false; | 243 | return false; |
533 | } | 244 | } |
534 | 245 | ||
535 | static void reserve_ds_buffers(void); | ||
536 | static void release_ds_buffers(void); | ||
537 | |||
538 | static void hw_perf_event_destroy(struct perf_event *event) | 246 | static void hw_perf_event_destroy(struct perf_event *event) |
539 | { | 247 | { |
540 | if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) { | 248 | if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) { |
@@ -583,7 +291,7 @@ set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) | |||
583 | return x86_pmu_extra_regs(val, event); | 291 | return x86_pmu_extra_regs(val, event); |
584 | } | 292 | } |
585 | 293 | ||
586 | static int x86_setup_perfctr(struct perf_event *event) | 294 | int x86_setup_perfctr(struct perf_event *event) |
587 | { | 295 | { |
588 | struct perf_event_attr *attr = &event->attr; | 296 | struct perf_event_attr *attr = &event->attr; |
589 | struct hw_perf_event *hwc = &event->hw; | 297 | struct hw_perf_event *hwc = &event->hw; |
@@ -647,7 +355,7 @@ static int x86_setup_perfctr(struct perf_event *event) | |||
647 | return 0; | 355 | return 0; |
648 | } | 356 | } |
649 | 357 | ||
650 | static int x86_pmu_hw_config(struct perf_event *event) | 358 | int x86_pmu_hw_config(struct perf_event *event) |
651 | { | 359 | { |
652 | if (event->attr.precise_ip) { | 360 | if (event->attr.precise_ip) { |
653 | int precise = 0; | 361 | int precise = 0; |
@@ -723,7 +431,7 @@ static int __x86_pmu_event_init(struct perf_event *event) | |||
723 | return x86_pmu.hw_config(event); | 431 | return x86_pmu.hw_config(event); |
724 | } | 432 | } |
725 | 433 | ||
726 | static void x86_pmu_disable_all(void) | 434 | void x86_pmu_disable_all(void) |
727 | { | 435 | { |
728 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); | 436 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
729 | int idx; | 437 | int idx; |
@@ -758,15 +466,7 @@ static void x86_pmu_disable(struct pmu *pmu) | |||
758 | x86_pmu.disable_all(); | 466 | x86_pmu.disable_all(); |
759 | } | 467 | } |
760 | 468 | ||
761 | static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc, | 469 | void x86_pmu_enable_all(int added) |
762 | u64 enable_mask) | ||
763 | { | ||
764 | if (hwc->extra_reg.reg) | ||
765 | wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config); | ||
766 | wrmsrl(hwc->config_base, hwc->config | enable_mask); | ||
767 | } | ||
768 | |||
769 | static void x86_pmu_enable_all(int added) | ||
770 | { | 470 | { |
771 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); | 471 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
772 | int idx; | 472 | int idx; |
@@ -788,7 +488,7 @@ static inline int is_x86_event(struct perf_event *event) | |||
788 | return event->pmu == &pmu; | 488 | return event->pmu == &pmu; |
789 | } | 489 | } |
790 | 490 | ||
791 | static int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) | 491 | int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) |
792 | { | 492 | { |
793 | struct event_constraint *c, *constraints[X86_PMC_IDX_MAX]; | 493 | struct event_constraint *c, *constraints[X86_PMC_IDX_MAX]; |
794 | unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; | 494 | unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; |
@@ -959,7 +659,6 @@ static inline int match_prev_assignment(struct hw_perf_event *hwc, | |||
959 | } | 659 | } |
960 | 660 | ||
961 | static void x86_pmu_start(struct perf_event *event, int flags); | 661 | static void x86_pmu_start(struct perf_event *event, int flags); |
962 | static void x86_pmu_stop(struct perf_event *event, int flags); | ||
963 | 662 | ||
964 | static void x86_pmu_enable(struct pmu *pmu) | 663 | static void x86_pmu_enable(struct pmu *pmu) |
965 | { | 664 | { |
@@ -1031,21 +730,13 @@ static void x86_pmu_enable(struct pmu *pmu) | |||
1031 | x86_pmu.enable_all(added); | 730 | x86_pmu.enable_all(added); |
1032 | } | 731 | } |
1033 | 732 | ||
1034 | static inline void x86_pmu_disable_event(struct perf_event *event) | ||
1035 | { | ||
1036 | struct hw_perf_event *hwc = &event->hw; | ||
1037 | |||
1038 | wrmsrl(hwc->config_base, hwc->config); | ||
1039 | } | ||
1040 | |||
1041 | static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); | 733 | static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); |
1042 | 734 | ||
1043 | /* | 735 | /* |
1044 | * Set the next IRQ period, based on the hwc->period_left value. | 736 | * Set the next IRQ period, based on the hwc->period_left value. |
1045 | * To be called with the event disabled in hw: | 737 | * To be called with the event disabled in hw: |
1046 | */ | 738 | */ |
1047 | static int | 739 | int x86_perf_event_set_period(struct perf_event *event) |
1048 | x86_perf_event_set_period(struct perf_event *event) | ||
1049 | { | 740 | { |
1050 | struct hw_perf_event *hwc = &event->hw; | 741 | struct hw_perf_event *hwc = &event->hw; |
1051 | s64 left = local64_read(&hwc->period_left); | 742 | s64 left = local64_read(&hwc->period_left); |
@@ -1105,7 +796,7 @@ x86_perf_event_set_period(struct perf_event *event) | |||
1105 | return ret; | 796 | return ret; |
1106 | } | 797 | } |
1107 | 798 | ||
1108 | static void x86_pmu_enable_event(struct perf_event *event) | 799 | void x86_pmu_enable_event(struct perf_event *event) |
1109 | { | 800 | { |
1110 | if (__this_cpu_read(cpu_hw_events.enabled)) | 801 | if (__this_cpu_read(cpu_hw_events.enabled)) |
1111 | __x86_pmu_enable_event(&event->hw, | 802 | __x86_pmu_enable_event(&event->hw, |
@@ -1244,7 +935,7 @@ void perf_event_print_debug(void) | |||
1244 | local_irq_restore(flags); | 935 | local_irq_restore(flags); |
1245 | } | 936 | } |
1246 | 937 | ||
1247 | static void x86_pmu_stop(struct perf_event *event, int flags) | 938 | void x86_pmu_stop(struct perf_event *event, int flags) |
1248 | { | 939 | { |
1249 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); | 940 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); |
1250 | struct hw_perf_event *hwc = &event->hw; | 941 | struct hw_perf_event *hwc = &event->hw; |
@@ -1297,7 +988,7 @@ static void x86_pmu_del(struct perf_event *event, int flags) | |||
1297 | perf_event_update_userpage(event); | 988 | perf_event_update_userpage(event); |
1298 | } | 989 | } |
1299 | 990 | ||
1300 | static int x86_pmu_handle_irq(struct pt_regs *regs) | 991 | int x86_pmu_handle_irq(struct pt_regs *regs) |
1301 | { | 992 | { |
1302 | struct perf_sample_data data; | 993 | struct perf_sample_data data; |
1303 | struct cpu_hw_events *cpuc; | 994 | struct cpu_hw_events *cpuc; |
@@ -1367,109 +1058,28 @@ void perf_events_lapic_init(void) | |||
1367 | apic_write(APIC_LVTPC, APIC_DM_NMI); | 1058 | apic_write(APIC_LVTPC, APIC_DM_NMI); |
1368 | } | 1059 | } |
1369 | 1060 | ||
1370 | struct pmu_nmi_state { | ||
1371 | unsigned int marked; | ||
1372 | int handled; | ||
1373 | }; | ||
1374 | |||
1375 | static DEFINE_PER_CPU(struct pmu_nmi_state, pmu_nmi); | ||
1376 | |||
1377 | static int __kprobes | 1061 | static int __kprobes |
1378 | perf_event_nmi_handler(struct notifier_block *self, | 1062 | perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) |
1379 | unsigned long cmd, void *__args) | ||
1380 | { | 1063 | { |
1381 | struct die_args *args = __args; | ||
1382 | unsigned int this_nmi; | ||
1383 | int handled; | ||
1384 | |||
1385 | if (!atomic_read(&active_events)) | 1064 | if (!atomic_read(&active_events)) |
1386 | return NOTIFY_DONE; | 1065 | return NMI_DONE; |
1387 | |||
1388 | switch (cmd) { | ||
1389 | case DIE_NMI: | ||
1390 | break; | ||
1391 | case DIE_NMIUNKNOWN: | ||
1392 | this_nmi = percpu_read(irq_stat.__nmi_count); | ||
1393 | if (this_nmi != __this_cpu_read(pmu_nmi.marked)) | ||
1394 | /* let the kernel handle the unknown nmi */ | ||
1395 | return NOTIFY_DONE; | ||
1396 | /* | ||
1397 | * This one is a PMU back-to-back nmi. Two events | ||
1398 | * trigger 'simultaneously' raising two back-to-back | ||
1399 | * NMIs. If the first NMI handles both, the latter | ||
1400 | * will be empty and daze the CPU. So, we drop it to | ||
1401 | * avoid false-positive 'unknown nmi' messages. | ||
1402 | */ | ||
1403 | return NOTIFY_STOP; | ||
1404 | default: | ||
1405 | return NOTIFY_DONE; | ||
1406 | } | ||
1407 | |||
1408 | handled = x86_pmu.handle_irq(args->regs); | ||
1409 | if (!handled) | ||
1410 | return NOTIFY_DONE; | ||
1411 | |||
1412 | this_nmi = percpu_read(irq_stat.__nmi_count); | ||
1413 | if ((handled > 1) || | ||
1414 | /* the next nmi could be a back-to-back nmi */ | ||
1415 | ((__this_cpu_read(pmu_nmi.marked) == this_nmi) && | ||
1416 | (__this_cpu_read(pmu_nmi.handled) > 1))) { | ||
1417 | /* | ||
1418 | * We could have two subsequent back-to-back nmis: The | ||
1419 | * first handles more than one counter, the 2nd | ||
1420 | * handles only one counter and the 3rd handles no | ||
1421 | * counter. | ||
1422 | * | ||
1423 | * This is the 2nd nmi because the previous was | ||
1424 | * handling more than one counter. We will mark the | ||
1425 | * next (3rd) and then drop it if unhandled. | ||
1426 | */ | ||
1427 | __this_cpu_write(pmu_nmi.marked, this_nmi + 1); | ||
1428 | __this_cpu_write(pmu_nmi.handled, handled); | ||
1429 | } | ||
1430 | 1066 | ||
1431 | return NOTIFY_STOP; | 1067 | return x86_pmu.handle_irq(regs); |
1432 | } | 1068 | } |
1433 | 1069 | ||
1434 | static __read_mostly struct notifier_block perf_event_nmi_notifier = { | 1070 | struct event_constraint emptyconstraint; |
1435 | .notifier_call = perf_event_nmi_handler, | 1071 | struct event_constraint unconstrained; |
1436 | .next = NULL, | ||
1437 | .priority = NMI_LOCAL_LOW_PRIOR, | ||
1438 | }; | ||
1439 | |||
1440 | static struct event_constraint unconstrained; | ||
1441 | static struct event_constraint emptyconstraint; | ||
1442 | |||
1443 | static struct event_constraint * | ||
1444 | x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event) | ||
1445 | { | ||
1446 | struct event_constraint *c; | ||
1447 | |||
1448 | if (x86_pmu.event_constraints) { | ||
1449 | for_each_event_constraint(c, x86_pmu.event_constraints) { | ||
1450 | if ((event->hw.config & c->cmask) == c->code) | ||
1451 | return c; | ||
1452 | } | ||
1453 | } | ||
1454 | |||
1455 | return &unconstrained; | ||
1456 | } | ||
1457 | |||
1458 | #include "perf_event_amd.c" | ||
1459 | #include "perf_event_p6.c" | ||
1460 | #include "perf_event_p4.c" | ||
1461 | #include "perf_event_intel_lbr.c" | ||
1462 | #include "perf_event_intel_ds.c" | ||
1463 | #include "perf_event_intel.c" | ||
1464 | 1072 | ||
1465 | static int __cpuinit | 1073 | static int __cpuinit |
1466 | x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) | 1074 | x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) |
1467 | { | 1075 | { |
1468 | unsigned int cpu = (long)hcpu; | 1076 | unsigned int cpu = (long)hcpu; |
1077 | struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); | ||
1469 | int ret = NOTIFY_OK; | 1078 | int ret = NOTIFY_OK; |
1470 | 1079 | ||
1471 | switch (action & ~CPU_TASKS_FROZEN) { | 1080 | switch (action & ~CPU_TASKS_FROZEN) { |
1472 | case CPU_UP_PREPARE: | 1081 | case CPU_UP_PREPARE: |
1082 | cpuc->kfree_on_online = NULL; | ||
1473 | if (x86_pmu.cpu_prepare) | 1083 | if (x86_pmu.cpu_prepare) |
1474 | ret = x86_pmu.cpu_prepare(cpu); | 1084 | ret = x86_pmu.cpu_prepare(cpu); |
1475 | break; | 1085 | break; |
@@ -1479,6 +1089,10 @@ x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) | |||
1479 | x86_pmu.cpu_starting(cpu); | 1089 | x86_pmu.cpu_starting(cpu); |
1480 | break; | 1090 | break; |
1481 | 1091 | ||
1092 | case CPU_ONLINE: | ||
1093 | kfree(cpuc->kfree_on_online); | ||
1094 | break; | ||
1095 | |||
1482 | case CPU_DYING: | 1096 | case CPU_DYING: |
1483 | if (x86_pmu.cpu_dying) | 1097 | if (x86_pmu.cpu_dying) |
1484 | x86_pmu.cpu_dying(cpu); | 1098 | x86_pmu.cpu_dying(cpu); |
@@ -1557,7 +1171,7 @@ static int __init init_hw_perf_events(void) | |||
1557 | ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED; | 1171 | ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED; |
1558 | 1172 | ||
1559 | perf_events_lapic_init(); | 1173 | perf_events_lapic_init(); |
1560 | register_die_notifier(&perf_event_nmi_notifier); | 1174 | register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); |
1561 | 1175 | ||
1562 | unconstrained = (struct event_constraint) | 1176 | unconstrained = (struct event_constraint) |
1563 | __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, | 1177 | __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, |