aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/kernel/perf_counter.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/powerpc/kernel/perf_counter.c')
-rw-r--r--arch/powerpc/kernel/perf_counter.c1306
1 files changed, 1306 insertions, 0 deletions
diff --git a/arch/powerpc/kernel/perf_counter.c b/arch/powerpc/kernel/perf_counter.c
new file mode 100644
index 000000000000..809fdf94b95f
--- /dev/null
+++ b/arch/powerpc/kernel/perf_counter.c
@@ -0,0 +1,1306 @@
1/*
2 * Performance counter support - powerpc architecture code
3 *
4 * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11#include <linux/kernel.h>
12#include <linux/sched.h>
13#include <linux/perf_counter.h>
14#include <linux/percpu.h>
15#include <linux/hardirq.h>
16#include <asm/reg.h>
17#include <asm/pmc.h>
18#include <asm/machdep.h>
19#include <asm/firmware.h>
20#include <asm/ptrace.h>
21
22struct cpu_hw_counters {
23 int n_counters;
24 int n_percpu;
25 int disabled;
26 int n_added;
27 int n_limited;
28 u8 pmcs_enabled;
29 struct perf_counter *counter[MAX_HWCOUNTERS];
30 u64 events[MAX_HWCOUNTERS];
31 unsigned int flags[MAX_HWCOUNTERS];
32 unsigned long mmcr[3];
33 struct perf_counter *limited_counter[MAX_LIMITED_HWCOUNTERS];
34 u8 limited_hwidx[MAX_LIMITED_HWCOUNTERS];
35};
36DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters);
37
38struct power_pmu *ppmu;
39
40/*
41 * Normally, to ignore kernel events we set the FCS (freeze counters
42 * in supervisor mode) bit in MMCR0, but if the kernel runs with the
43 * hypervisor bit set in the MSR, or if we are running on a processor
44 * where the hypervisor bit is forced to 1 (as on Apple G5 processors),
45 * then we need to use the FCHV bit to ignore kernel events.
46 */
47static unsigned int freeze_counters_kernel = MMCR0_FCS;
48
49/*
50 * 32-bit doesn't have MMCRA but does have an MMCR2,
51 * and a few other names are different.
52 */
53#ifdef CONFIG_PPC32
54
55#define MMCR0_FCHV 0
56#define MMCR0_PMCjCE MMCR0_PMCnCE
57
58#define SPRN_MMCRA SPRN_MMCR2
59#define MMCRA_SAMPLE_ENABLE 0
60
61static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
62{
63 return 0;
64}
65static inline void perf_set_pmu_inuse(int inuse) { }
66static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp) { }
67static inline u32 perf_get_misc_flags(struct pt_regs *regs)
68{
69 return 0;
70}
71static inline void perf_read_regs(struct pt_regs *regs) { }
72static inline int perf_intr_is_nmi(struct pt_regs *regs)
73{
74 return 0;
75}
76
77#endif /* CONFIG_PPC32 */
78
79/*
80 * Things that are specific to 64-bit implementations.
81 */
82#ifdef CONFIG_PPC64
83
84static inline unsigned long perf_ip_adjust(struct pt_regs *regs)
85{
86 unsigned long mmcra = regs->dsisr;
87
88 if ((mmcra & MMCRA_SAMPLE_ENABLE) && !(ppmu->flags & PPMU_ALT_SIPR)) {
89 unsigned long slot = (mmcra & MMCRA_SLOT) >> MMCRA_SLOT_SHIFT;
90 if (slot > 1)
91 return 4 * (slot - 1);
92 }
93 return 0;
94}
95
96static inline void perf_set_pmu_inuse(int inuse)
97{
98 get_lppaca()->pmcregs_in_use = inuse;
99}
100
101/*
102 * The user wants a data address recorded.
103 * If we're not doing instruction sampling, give them the SDAR
104 * (sampled data address). If we are doing instruction sampling, then
105 * only give them the SDAR if it corresponds to the instruction
106 * pointed to by SIAR; this is indicated by the [POWER6_]MMCRA_SDSYNC
107 * bit in MMCRA.
108 */
109static inline void perf_get_data_addr(struct pt_regs *regs, u64 *addrp)
110{
111 unsigned long mmcra = regs->dsisr;
112 unsigned long sdsync = (ppmu->flags & PPMU_ALT_SIPR) ?
113 POWER6_MMCRA_SDSYNC : MMCRA_SDSYNC;
114
115 if (!(mmcra & MMCRA_SAMPLE_ENABLE) || (mmcra & sdsync))
116 *addrp = mfspr(SPRN_SDAR);
117}
118
119static inline u32 perf_get_misc_flags(struct pt_regs *regs)
120{
121 unsigned long mmcra = regs->dsisr;
122
123 if (TRAP(regs) != 0xf00)
124 return 0; /* not a PMU interrupt */
125
126 if (ppmu->flags & PPMU_ALT_SIPR) {
127 if (mmcra & POWER6_MMCRA_SIHV)
128 return PERF_EVENT_MISC_HYPERVISOR;
129 return (mmcra & POWER6_MMCRA_SIPR) ?
130 PERF_EVENT_MISC_USER : PERF_EVENT_MISC_KERNEL;
131 }
132 if (mmcra & MMCRA_SIHV)
133 return PERF_EVENT_MISC_HYPERVISOR;
134 return (mmcra & MMCRA_SIPR) ? PERF_EVENT_MISC_USER :
135 PERF_EVENT_MISC_KERNEL;
136}
137
138/*
139 * Overload regs->dsisr to store MMCRA so we only need to read it once
140 * on each interrupt.
141 */
142static inline void perf_read_regs(struct pt_regs *regs)
143{
144 regs->dsisr = mfspr(SPRN_MMCRA);
145}
146
147/*
148 * If interrupts were soft-disabled when a PMU interrupt occurs, treat
149 * it as an NMI.
150 */
151static inline int perf_intr_is_nmi(struct pt_regs *regs)
152{
153 return !regs->softe;
154}
155
156#endif /* CONFIG_PPC64 */
157
158static void perf_counter_interrupt(struct pt_regs *regs);
159
160void perf_counter_print_debug(void)
161{
162}
163
164/*
165 * Read one performance monitor counter (PMC).
166 */
167static unsigned long read_pmc(int idx)
168{
169 unsigned long val;
170
171 switch (idx) {
172 case 1:
173 val = mfspr(SPRN_PMC1);
174 break;
175 case 2:
176 val = mfspr(SPRN_PMC2);
177 break;
178 case 3:
179 val = mfspr(SPRN_PMC3);
180 break;
181 case 4:
182 val = mfspr(SPRN_PMC4);
183 break;
184 case 5:
185 val = mfspr(SPRN_PMC5);
186 break;
187 case 6:
188 val = mfspr(SPRN_PMC6);
189 break;
190#ifdef CONFIG_PPC64
191 case 7:
192 val = mfspr(SPRN_PMC7);
193 break;
194 case 8:
195 val = mfspr(SPRN_PMC8);
196 break;
197#endif /* CONFIG_PPC64 */
198 default:
199 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
200 val = 0;
201 }
202 return val;
203}
204
205/*
206 * Write one PMC.
207 */
208static void write_pmc(int idx, unsigned long val)
209{
210 switch (idx) {
211 case 1:
212 mtspr(SPRN_PMC1, val);
213 break;
214 case 2:
215 mtspr(SPRN_PMC2, val);
216 break;
217 case 3:
218 mtspr(SPRN_PMC3, val);
219 break;
220 case 4:
221 mtspr(SPRN_PMC4, val);
222 break;
223 case 5:
224 mtspr(SPRN_PMC5, val);
225 break;
226 case 6:
227 mtspr(SPRN_PMC6, val);
228 break;
229#ifdef CONFIG_PPC64
230 case 7:
231 mtspr(SPRN_PMC7, val);
232 break;
233 case 8:
234 mtspr(SPRN_PMC8, val);
235 break;
236#endif /* CONFIG_PPC64 */
237 default:
238 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
239 }
240}
241
242/*
243 * Check if a set of events can all go on the PMU at once.
244 * If they can't, this will look at alternative codes for the events
245 * and see if any combination of alternative codes is feasible.
246 * The feasible set is returned in event[].
247 */
248static int power_check_constraints(u64 event[], unsigned int cflags[],
249 int n_ev)
250{
251 unsigned long mask, value, nv;
252 u64 alternatives[MAX_HWCOUNTERS][MAX_EVENT_ALTERNATIVES];
253 unsigned long amasks[MAX_HWCOUNTERS][MAX_EVENT_ALTERNATIVES];
254 unsigned long avalues[MAX_HWCOUNTERS][MAX_EVENT_ALTERNATIVES];
255 unsigned long smasks[MAX_HWCOUNTERS], svalues[MAX_HWCOUNTERS];
256 int n_alt[MAX_HWCOUNTERS], choice[MAX_HWCOUNTERS];
257 int i, j;
258 unsigned long addf = ppmu->add_fields;
259 unsigned long tadd = ppmu->test_adder;
260
261 if (n_ev > ppmu->n_counter)
262 return -1;
263
264 /* First see if the events will go on as-is */
265 for (i = 0; i < n_ev; ++i) {
266 if ((cflags[i] & PPMU_LIMITED_PMC_REQD)
267 && !ppmu->limited_pmc_event(event[i])) {
268 ppmu->get_alternatives(event[i], cflags[i],
269 alternatives[i]);
270 event[i] = alternatives[i][0];
271 }
272 if (ppmu->get_constraint(event[i], &amasks[i][0],
273 &avalues[i][0]))
274 return -1;
275 }
276 value = mask = 0;
277 for (i = 0; i < n_ev; ++i) {
278 nv = (value | avalues[i][0]) + (value & avalues[i][0] & addf);
279 if ((((nv + tadd) ^ value) & mask) != 0 ||
280 (((nv + tadd) ^ avalues[i][0]) & amasks[i][0]) != 0)
281 break;
282 value = nv;
283 mask |= amasks[i][0];
284 }
285 if (i == n_ev)
286 return 0; /* all OK */
287
288 /* doesn't work, gather alternatives... */
289 if (!ppmu->get_alternatives)
290 return -1;
291 for (i = 0; i < n_ev; ++i) {
292 choice[i] = 0;
293 n_alt[i] = ppmu->get_alternatives(event[i], cflags[i],
294 alternatives[i]);
295 for (j = 1; j < n_alt[i]; ++j)
296 ppmu->get_constraint(alternatives[i][j],
297 &amasks[i][j], &avalues[i][j]);
298 }
299
300 /* enumerate all possibilities and see if any will work */
301 i = 0;
302 j = -1;
303 value = mask = nv = 0;
304 while (i < n_ev) {
305 if (j >= 0) {
306 /* we're backtracking, restore context */
307 value = svalues[i];
308 mask = smasks[i];
309 j = choice[i];
310 }
311 /*
312 * See if any alternative k for event i,
313 * where k > j, will satisfy the constraints.
314 */
315 while (++j < n_alt[i]) {
316 nv = (value | avalues[i][j]) +
317 (value & avalues[i][j] & addf);
318 if ((((nv + tadd) ^ value) & mask) == 0 &&
319 (((nv + tadd) ^ avalues[i][j])
320 & amasks[i][j]) == 0)
321 break;
322 }
323 if (j >= n_alt[i]) {
324 /*
325 * No feasible alternative, backtrack
326 * to event i-1 and continue enumerating its
327 * alternatives from where we got up to.
328 */
329 if (--i < 0)
330 return -1;
331 } else {
332 /*
333 * Found a feasible alternative for event i,
334 * remember where we got up to with this event,
335 * go on to the next event, and start with
336 * the first alternative for it.
337 */
338 choice[i] = j;
339 svalues[i] = value;
340 smasks[i] = mask;
341 value = nv;
342 mask |= amasks[i][j];
343 ++i;
344 j = -1;
345 }
346 }
347
348 /* OK, we have a feasible combination, tell the caller the solution */
349 for (i = 0; i < n_ev; ++i)
350 event[i] = alternatives[i][choice[i]];
351 return 0;
352}
353
354/*
355 * Check if newly-added counters have consistent settings for
356 * exclude_{user,kernel,hv} with each other and any previously
357 * added counters.
358 */
359static int check_excludes(struct perf_counter **ctrs, unsigned int cflags[],
360 int n_prev, int n_new)
361{
362 int eu = 0, ek = 0, eh = 0;
363 int i, n, first;
364 struct perf_counter *counter;
365
366 n = n_prev + n_new;
367 if (n <= 1)
368 return 0;
369
370 first = 1;
371 for (i = 0; i < n; ++i) {
372 if (cflags[i] & PPMU_LIMITED_PMC_OK) {
373 cflags[i] &= ~PPMU_LIMITED_PMC_REQD;
374 continue;
375 }
376 counter = ctrs[i];
377 if (first) {
378 eu = counter->attr.exclude_user;
379 ek = counter->attr.exclude_kernel;
380 eh = counter->attr.exclude_hv;
381 first = 0;
382 } else if (counter->attr.exclude_user != eu ||
383 counter->attr.exclude_kernel != ek ||
384 counter->attr.exclude_hv != eh) {
385 return -EAGAIN;
386 }
387 }
388
389 if (eu || ek || eh)
390 for (i = 0; i < n; ++i)
391 if (cflags[i] & PPMU_LIMITED_PMC_OK)
392 cflags[i] |= PPMU_LIMITED_PMC_REQD;
393
394 return 0;
395}
396
397static void power_pmu_read(struct perf_counter *counter)
398{
399 s64 val, delta, prev;
400
401 if (!counter->hw.idx)
402 return;
403 /*
404 * Performance monitor interrupts come even when interrupts
405 * are soft-disabled, as long as interrupts are hard-enabled.
406 * Therefore we treat them like NMIs.
407 */
408 do {
409 prev = atomic64_read(&counter->hw.prev_count);
410 barrier();
411 val = read_pmc(counter->hw.idx);
412 } while (atomic64_cmpxchg(&counter->hw.prev_count, prev, val) != prev);
413
414 /* The counters are only 32 bits wide */
415 delta = (val - prev) & 0xfffffffful;
416 atomic64_add(delta, &counter->count);
417 atomic64_sub(delta, &counter->hw.period_left);
418}
419
420/*
421 * On some machines, PMC5 and PMC6 can't be written, don't respect
422 * the freeze conditions, and don't generate interrupts. This tells
423 * us if `counter' is using such a PMC.
424 */
425static int is_limited_pmc(int pmcnum)
426{
427 return (ppmu->flags & PPMU_LIMITED_PMC5_6)
428 && (pmcnum == 5 || pmcnum == 6);
429}
430
431static void freeze_limited_counters(struct cpu_hw_counters *cpuhw,
432 unsigned long pmc5, unsigned long pmc6)
433{
434 struct perf_counter *counter;
435 u64 val, prev, delta;
436 int i;
437
438 for (i = 0; i < cpuhw->n_limited; ++i) {
439 counter = cpuhw->limited_counter[i];
440 if (!counter->hw.idx)
441 continue;
442 val = (counter->hw.idx == 5) ? pmc5 : pmc6;
443 prev = atomic64_read(&counter->hw.prev_count);
444 counter->hw.idx = 0;
445 delta = (val - prev) & 0xfffffffful;
446 atomic64_add(delta, &counter->count);
447 }
448}
449
450static void thaw_limited_counters(struct cpu_hw_counters *cpuhw,
451 unsigned long pmc5, unsigned long pmc6)
452{
453 struct perf_counter *counter;
454 u64 val;
455 int i;
456
457 for (i = 0; i < cpuhw->n_limited; ++i) {
458 counter = cpuhw->limited_counter[i];
459 counter->hw.idx = cpuhw->limited_hwidx[i];
460 val = (counter->hw.idx == 5) ? pmc5 : pmc6;
461 atomic64_set(&counter->hw.prev_count, val);
462 perf_counter_update_userpage(counter);
463 }
464}
465
466/*
467 * Since limited counters don't respect the freeze conditions, we
468 * have to read them immediately after freezing or unfreezing the
469 * other counters. We try to keep the values from the limited
470 * counters as consistent as possible by keeping the delay (in
471 * cycles and instructions) between freezing/unfreezing and reading
472 * the limited counters as small and consistent as possible.
473 * Therefore, if any limited counters are in use, we read them
474 * both, and always in the same order, to minimize variability,
475 * and do it inside the same asm that writes MMCR0.
476 */
477static void write_mmcr0(struct cpu_hw_counters *cpuhw, unsigned long mmcr0)
478{
479 unsigned long pmc5, pmc6;
480
481 if (!cpuhw->n_limited) {
482 mtspr(SPRN_MMCR0, mmcr0);
483 return;
484 }
485
486 /*
487 * Write MMCR0, then read PMC5 and PMC6 immediately.
488 * To ensure we don't get a performance monitor interrupt
489 * between writing MMCR0 and freezing/thawing the limited
490 * counters, we first write MMCR0 with the counter overflow
491 * interrupt enable bits turned off.
492 */
493 asm volatile("mtspr %3,%2; mfspr %0,%4; mfspr %1,%5"
494 : "=&r" (pmc5), "=&r" (pmc6)
495 : "r" (mmcr0 & ~(MMCR0_PMC1CE | MMCR0_PMCjCE)),
496 "i" (SPRN_MMCR0),
497 "i" (SPRN_PMC5), "i" (SPRN_PMC6));
498
499 if (mmcr0 & MMCR0_FC)
500 freeze_limited_counters(cpuhw, pmc5, pmc6);
501 else
502 thaw_limited_counters(cpuhw, pmc5, pmc6);
503
504 /*
505 * Write the full MMCR0 including the counter overflow interrupt
506 * enable bits, if necessary.
507 */
508 if (mmcr0 & (MMCR0_PMC1CE | MMCR0_PMCjCE))
509 mtspr(SPRN_MMCR0, mmcr0);
510}
511
512/*
513 * Disable all counters to prevent PMU interrupts and to allow
514 * counters to be added or removed.
515 */
516void hw_perf_disable(void)
517{
518 struct cpu_hw_counters *cpuhw;
519 unsigned long flags;
520
521 local_irq_save(flags);
522 cpuhw = &__get_cpu_var(cpu_hw_counters);
523
524 if (!cpuhw->disabled) {
525 cpuhw->disabled = 1;
526 cpuhw->n_added = 0;
527
528 /*
529 * Check if we ever enabled the PMU on this cpu.
530 */
531 if (!cpuhw->pmcs_enabled) {
532 if (ppc_md.enable_pmcs)
533 ppc_md.enable_pmcs();
534 cpuhw->pmcs_enabled = 1;
535 }
536
537 /*
538 * Disable instruction sampling if it was enabled
539 */
540 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
541 mtspr(SPRN_MMCRA,
542 cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
543 mb();
544 }
545
546 /*
547 * Set the 'freeze counters' bit.
548 * The barrier is to make sure the mtspr has been
549 * executed and the PMU has frozen the counters
550 * before we return.
551 */
552 write_mmcr0(cpuhw, mfspr(SPRN_MMCR0) | MMCR0_FC);
553 mb();
554 }
555 local_irq_restore(flags);
556}
557
558/*
559 * Re-enable all counters if disable == 0.
560 * If we were previously disabled and counters were added, then
561 * put the new config on the PMU.
562 */
563void hw_perf_enable(void)
564{
565 struct perf_counter *counter;
566 struct cpu_hw_counters *cpuhw;
567 unsigned long flags;
568 long i;
569 unsigned long val;
570 s64 left;
571 unsigned int hwc_index[MAX_HWCOUNTERS];
572 int n_lim;
573 int idx;
574
575 local_irq_save(flags);
576 cpuhw = &__get_cpu_var(cpu_hw_counters);
577 if (!cpuhw->disabled) {
578 local_irq_restore(flags);
579 return;
580 }
581 cpuhw->disabled = 0;
582
583 /*
584 * If we didn't change anything, or only removed counters,
585 * no need to recalculate MMCR* settings and reset the PMCs.
586 * Just reenable the PMU with the current MMCR* settings
587 * (possibly updated for removal of counters).
588 */
589 if (!cpuhw->n_added) {
590 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
591 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
592 if (cpuhw->n_counters == 0)
593 perf_set_pmu_inuse(0);
594 goto out_enable;
595 }
596
597 /*
598 * Compute MMCR* values for the new set of counters
599 */
600 if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_counters, hwc_index,
601 cpuhw->mmcr)) {
602 /* shouldn't ever get here */
603 printk(KERN_ERR "oops compute_mmcr failed\n");
604 goto out;
605 }
606
607 /*
608 * Add in MMCR0 freeze bits corresponding to the
609 * attr.exclude_* bits for the first counter.
610 * We have already checked that all counters have the
611 * same values for these bits as the first counter.
612 */
613 counter = cpuhw->counter[0];
614 if (counter->attr.exclude_user)
615 cpuhw->mmcr[0] |= MMCR0_FCP;
616 if (counter->attr.exclude_kernel)
617 cpuhw->mmcr[0] |= freeze_counters_kernel;
618 if (counter->attr.exclude_hv)
619 cpuhw->mmcr[0] |= MMCR0_FCHV;
620
621 /*
622 * Write the new configuration to MMCR* with the freeze
623 * bit set and set the hardware counters to their initial values.
624 * Then unfreeze the counters.
625 */
626 perf_set_pmu_inuse(1);
627 mtspr(SPRN_MMCRA, cpuhw->mmcr[2] & ~MMCRA_SAMPLE_ENABLE);
628 mtspr(SPRN_MMCR1, cpuhw->mmcr[1]);
629 mtspr(SPRN_MMCR0, (cpuhw->mmcr[0] & ~(MMCR0_PMC1CE | MMCR0_PMCjCE))
630 | MMCR0_FC);
631
632 /*
633 * Read off any pre-existing counters that need to move
634 * to another PMC.
635 */
636 for (i = 0; i < cpuhw->n_counters; ++i) {
637 counter = cpuhw->counter[i];
638 if (counter->hw.idx && counter->hw.idx != hwc_index[i] + 1) {
639 power_pmu_read(counter);
640 write_pmc(counter->hw.idx, 0);
641 counter->hw.idx = 0;
642 }
643 }
644
645 /*
646 * Initialize the PMCs for all the new and moved counters.
647 */
648 cpuhw->n_limited = n_lim = 0;
649 for (i = 0; i < cpuhw->n_counters; ++i) {
650 counter = cpuhw->counter[i];
651 if (counter->hw.idx)
652 continue;
653 idx = hwc_index[i] + 1;
654 if (is_limited_pmc(idx)) {
655 cpuhw->limited_counter[n_lim] = counter;
656 cpuhw->limited_hwidx[n_lim] = idx;
657 ++n_lim;
658 continue;
659 }
660 val = 0;
661 if (counter->hw.sample_period) {
662 left = atomic64_read(&counter->hw.period_left);
663 if (left < 0x80000000L)
664 val = 0x80000000L - left;
665 }
666 atomic64_set(&counter->hw.prev_count, val);
667 counter->hw.idx = idx;
668 write_pmc(idx, val);
669 perf_counter_update_userpage(counter);
670 }
671 cpuhw->n_limited = n_lim;
672 cpuhw->mmcr[0] |= MMCR0_PMXE | MMCR0_FCECE;
673
674 out_enable:
675 mb();
676 write_mmcr0(cpuhw, cpuhw->mmcr[0]);
677
678 /*
679 * Enable instruction sampling if necessary
680 */
681 if (cpuhw->mmcr[2] & MMCRA_SAMPLE_ENABLE) {
682 mb();
683 mtspr(SPRN_MMCRA, cpuhw->mmcr[2]);
684 }
685
686 out:
687 local_irq_restore(flags);
688}
689
690static int collect_events(struct perf_counter *group, int max_count,
691 struct perf_counter *ctrs[], u64 *events,
692 unsigned int *flags)
693{
694 int n = 0;
695 struct perf_counter *counter;
696
697 if (!is_software_counter(group)) {
698 if (n >= max_count)
699 return -1;
700 ctrs[n] = group;
701 flags[n] = group->hw.counter_base;
702 events[n++] = group->hw.config;
703 }
704 list_for_each_entry(counter, &group->sibling_list, list_entry) {
705 if (!is_software_counter(counter) &&
706 counter->state != PERF_COUNTER_STATE_OFF) {
707 if (n >= max_count)
708 return -1;
709 ctrs[n] = counter;
710 flags[n] = counter->hw.counter_base;
711 events[n++] = counter->hw.config;
712 }
713 }
714 return n;
715}
716
717static void counter_sched_in(struct perf_counter *counter, int cpu)
718{
719 counter->state = PERF_COUNTER_STATE_ACTIVE;
720 counter->oncpu = cpu;
721 counter->tstamp_running += counter->ctx->time - counter->tstamp_stopped;
722 if (is_software_counter(counter))
723 counter->pmu->enable(counter);
724}
725
726/*
727 * Called to enable a whole group of counters.
728 * Returns 1 if the group was enabled, or -EAGAIN if it could not be.
729 * Assumes the caller has disabled interrupts and has
730 * frozen the PMU with hw_perf_save_disable.
731 */
732int hw_perf_group_sched_in(struct perf_counter *group_leader,
733 struct perf_cpu_context *cpuctx,
734 struct perf_counter_context *ctx, int cpu)
735{
736 struct cpu_hw_counters *cpuhw;
737 long i, n, n0;
738 struct perf_counter *sub;
739
740 cpuhw = &__get_cpu_var(cpu_hw_counters);
741 n0 = cpuhw->n_counters;
742 n = collect_events(group_leader, ppmu->n_counter - n0,
743 &cpuhw->counter[n0], &cpuhw->events[n0],
744 &cpuhw->flags[n0]);
745 if (n < 0)
746 return -EAGAIN;
747 if (check_excludes(cpuhw->counter, cpuhw->flags, n0, n))
748 return -EAGAIN;
749 i = power_check_constraints(cpuhw->events, cpuhw->flags, n + n0);
750 if (i < 0)
751 return -EAGAIN;
752 cpuhw->n_counters = n0 + n;
753 cpuhw->n_added += n;
754
755 /*
756 * OK, this group can go on; update counter states etc.,
757 * and enable any software counters
758 */
759 for (i = n0; i < n0 + n; ++i)
760 cpuhw->counter[i]->hw.config = cpuhw->events[i];
761 cpuctx->active_oncpu += n;
762 n = 1;
763 counter_sched_in(group_leader, cpu);
764 list_for_each_entry(sub, &group_leader->sibling_list, list_entry) {
765 if (sub->state != PERF_COUNTER_STATE_OFF) {
766 counter_sched_in(sub, cpu);
767 ++n;
768 }
769 }
770 ctx->nr_active += n;
771
772 return 1;
773}
774
775/*
776 * Add a counter to the PMU.
777 * If all counters are not already frozen, then we disable and
778 * re-enable the PMU in order to get hw_perf_enable to do the
779 * actual work of reconfiguring the PMU.
780 */
781static int power_pmu_enable(struct perf_counter *counter)
782{
783 struct cpu_hw_counters *cpuhw;
784 unsigned long flags;
785 int n0;
786 int ret = -EAGAIN;
787
788 local_irq_save(flags);
789 perf_disable();
790
791 /*
792 * Add the counter to the list (if there is room)
793 * and check whether the total set is still feasible.
794 */
795 cpuhw = &__get_cpu_var(cpu_hw_counters);
796 n0 = cpuhw->n_counters;
797 if (n0 >= ppmu->n_counter)
798 goto out;
799 cpuhw->counter[n0] = counter;
800 cpuhw->events[n0] = counter->hw.config;
801 cpuhw->flags[n0] = counter->hw.counter_base;
802 if (check_excludes(cpuhw->counter, cpuhw->flags, n0, 1))
803 goto out;
804 if (power_check_constraints(cpuhw->events, cpuhw->flags, n0 + 1))
805 goto out;
806
807 counter->hw.config = cpuhw->events[n0];
808 ++cpuhw->n_counters;
809 ++cpuhw->n_added;
810
811 ret = 0;
812 out:
813 perf_enable();
814 local_irq_restore(flags);
815 return ret;
816}
817
818/*
819 * Remove a counter from the PMU.
820 */
821static void power_pmu_disable(struct perf_counter *counter)
822{
823 struct cpu_hw_counters *cpuhw;
824 long i;
825 unsigned long flags;
826
827 local_irq_save(flags);
828 perf_disable();
829
830 power_pmu_read(counter);
831
832 cpuhw = &__get_cpu_var(cpu_hw_counters);
833 for (i = 0; i < cpuhw->n_counters; ++i) {
834 if (counter == cpuhw->counter[i]) {
835 while (++i < cpuhw->n_counters)
836 cpuhw->counter[i-1] = cpuhw->counter[i];
837 --cpuhw->n_counters;
838 ppmu->disable_pmc(counter->hw.idx - 1, cpuhw->mmcr);
839 if (counter->hw.idx) {
840 write_pmc(counter->hw.idx, 0);
841 counter->hw.idx = 0;
842 }
843 perf_counter_update_userpage(counter);
844 break;
845 }
846 }
847 for (i = 0; i < cpuhw->n_limited; ++i)
848 if (counter == cpuhw->limited_counter[i])
849 break;
850 if (i < cpuhw->n_limited) {
851 while (++i < cpuhw->n_limited) {
852 cpuhw->limited_counter[i-1] = cpuhw->limited_counter[i];
853 cpuhw->limited_hwidx[i-1] = cpuhw->limited_hwidx[i];
854 }
855 --cpuhw->n_limited;
856 }
857 if (cpuhw->n_counters == 0) {
858 /* disable exceptions if no counters are running */
859 cpuhw->mmcr[0] &= ~(MMCR0_PMXE | MMCR0_FCECE);
860 }
861
862 perf_enable();
863 local_irq_restore(flags);
864}
865
866/*
867 * Re-enable interrupts on a counter after they were throttled
868 * because they were coming too fast.
869 */
870static void power_pmu_unthrottle(struct perf_counter *counter)
871{
872 s64 val, left;
873 unsigned long flags;
874
875 if (!counter->hw.idx || !counter->hw.sample_period)
876 return;
877 local_irq_save(flags);
878 perf_disable();
879 power_pmu_read(counter);
880 left = counter->hw.sample_period;
881 counter->hw.last_period = left;
882 val = 0;
883 if (left < 0x80000000L)
884 val = 0x80000000L - left;
885 write_pmc(counter->hw.idx, val);
886 atomic64_set(&counter->hw.prev_count, val);
887 atomic64_set(&counter->hw.period_left, left);
888 perf_counter_update_userpage(counter);
889 perf_enable();
890 local_irq_restore(flags);
891}
892
893struct pmu power_pmu = {
894 .enable = power_pmu_enable,
895 .disable = power_pmu_disable,
896 .read = power_pmu_read,
897 .unthrottle = power_pmu_unthrottle,
898};
899
900/*
901 * Return 1 if we might be able to put counter on a limited PMC,
902 * or 0 if not.
903 * A counter can only go on a limited PMC if it counts something
904 * that a limited PMC can count, doesn't require interrupts, and
905 * doesn't exclude any processor mode.
906 */
907static int can_go_on_limited_pmc(struct perf_counter *counter, u64 ev,
908 unsigned int flags)
909{
910 int n;
911 u64 alt[MAX_EVENT_ALTERNATIVES];
912
913 if (counter->attr.exclude_user
914 || counter->attr.exclude_kernel
915 || counter->attr.exclude_hv
916 || counter->attr.sample_period)
917 return 0;
918
919 if (ppmu->limited_pmc_event(ev))
920 return 1;
921
922 /*
923 * The requested event isn't on a limited PMC already;
924 * see if any alternative code goes on a limited PMC.
925 */
926 if (!ppmu->get_alternatives)
927 return 0;
928
929 flags |= PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD;
930 n = ppmu->get_alternatives(ev, flags, alt);
931
932 return n > 0;
933}
934
935/*
936 * Find an alternative event that goes on a normal PMC, if possible,
937 * and return the event code, or 0 if there is no such alternative.
938 * (Note: event code 0 is "don't count" on all machines.)
939 */
940static u64 normal_pmc_alternative(u64 ev, unsigned long flags)
941{
942 u64 alt[MAX_EVENT_ALTERNATIVES];
943 int n;
944
945 flags &= ~(PPMU_LIMITED_PMC_OK | PPMU_LIMITED_PMC_REQD);
946 n = ppmu->get_alternatives(ev, flags, alt);
947 if (!n)
948 return 0;
949 return alt[0];
950}
951
952/* Number of perf_counters counting hardware events */
953static atomic_t num_counters;
954/* Used to avoid races in calling reserve/release_pmc_hardware */
955static DEFINE_MUTEX(pmc_reserve_mutex);
956
957/*
958 * Release the PMU if this is the last perf_counter.
959 */
960static void hw_perf_counter_destroy(struct perf_counter *counter)
961{
962 if (!atomic_add_unless(&num_counters, -1, 1)) {
963 mutex_lock(&pmc_reserve_mutex);
964 if (atomic_dec_return(&num_counters) == 0)
965 release_pmc_hardware();
966 mutex_unlock(&pmc_reserve_mutex);
967 }
968}
969
970/*
971 * Translate a generic cache event config to a raw event code.
972 */
973static int hw_perf_cache_event(u64 config, u64 *eventp)
974{
975 unsigned long type, op, result;
976 int ev;
977
978 if (!ppmu->cache_events)
979 return -EINVAL;
980
981 /* unpack config */
982 type = config & 0xff;
983 op = (config >> 8) & 0xff;
984 result = (config >> 16) & 0xff;
985
986 if (type >= PERF_COUNT_HW_CACHE_MAX ||
987 op >= PERF_COUNT_HW_CACHE_OP_MAX ||
988 result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
989 return -EINVAL;
990
991 ev = (*ppmu->cache_events)[type][op][result];
992 if (ev == 0)
993 return -EOPNOTSUPP;
994 if (ev == -1)
995 return -EINVAL;
996 *eventp = ev;
997 return 0;
998}
999
1000const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
1001{
1002 u64 ev;
1003 unsigned long flags;
1004 struct perf_counter *ctrs[MAX_HWCOUNTERS];
1005 u64 events[MAX_HWCOUNTERS];
1006 unsigned int cflags[MAX_HWCOUNTERS];
1007 int n;
1008 int err;
1009
1010 if (!ppmu)
1011 return ERR_PTR(-ENXIO);
1012 switch (counter->attr.type) {
1013 case PERF_TYPE_HARDWARE:
1014 ev = counter->attr.config;
1015 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
1016 return ERR_PTR(-EOPNOTSUPP);
1017 ev = ppmu->generic_events[ev];
1018 break;
1019 case PERF_TYPE_HW_CACHE:
1020 err = hw_perf_cache_event(counter->attr.config, &ev);
1021 if (err)
1022 return ERR_PTR(err);
1023 break;
1024 case PERF_TYPE_RAW:
1025 ev = counter->attr.config;
1026 break;
1027 default:
1028 return ERR_PTR(-EINVAL);
1029 }
1030 counter->hw.config_base = ev;
1031 counter->hw.idx = 0;
1032
1033 /*
1034 * If we are not running on a hypervisor, force the
1035 * exclude_hv bit to 0 so that we don't care what
1036 * the user set it to.
1037 */
1038 if (!firmware_has_feature(FW_FEATURE_LPAR))
1039 counter->attr.exclude_hv = 0;
1040
1041 /*
1042 * If this is a per-task counter, then we can use
1043 * PM_RUN_* events interchangeably with their non RUN_*
1044 * equivalents, e.g. PM_RUN_CYC instead of PM_CYC.
1045 * XXX we should check if the task is an idle task.
1046 */
1047 flags = 0;
1048 if (counter->ctx->task)
1049 flags |= PPMU_ONLY_COUNT_RUN;
1050
1051 /*
1052 * If this machine has limited counters, check whether this
1053 * event could go on a limited counter.
1054 */
1055 if (ppmu->flags & PPMU_LIMITED_PMC5_6) {
1056 if (can_go_on_limited_pmc(counter, ev, flags)) {
1057 flags |= PPMU_LIMITED_PMC_OK;
1058 } else if (ppmu->limited_pmc_event(ev)) {
1059 /*
1060 * The requested event is on a limited PMC,
1061 * but we can't use a limited PMC; see if any
1062 * alternative goes on a normal PMC.
1063 */
1064 ev = normal_pmc_alternative(ev, flags);
1065 if (!ev)
1066 return ERR_PTR(-EINVAL);
1067 }
1068 }
1069
1070 /*
1071 * If this is in a group, check if it can go on with all the
1072 * other hardware counters in the group. We assume the counter
1073 * hasn't been linked into its leader's sibling list at this point.
1074 */
1075 n = 0;
1076 if (counter->group_leader != counter) {
1077 n = collect_events(counter->group_leader, ppmu->n_counter - 1,
1078 ctrs, events, cflags);
1079 if (n < 0)
1080 return ERR_PTR(-EINVAL);
1081 }
1082 events[n] = ev;
1083 ctrs[n] = counter;
1084 cflags[n] = flags;
1085 if (check_excludes(ctrs, cflags, n, 1))
1086 return ERR_PTR(-EINVAL);
1087 if (power_check_constraints(events, cflags, n + 1))
1088 return ERR_PTR(-EINVAL);
1089
1090 counter->hw.config = events[n];
1091 counter->hw.counter_base = cflags[n];
1092 counter->hw.last_period = counter->hw.sample_period;
1093 atomic64_set(&counter->hw.period_left, counter->hw.last_period);
1094
1095 /*
1096 * See if we need to reserve the PMU.
1097 * If no counters are currently in use, then we have to take a
1098 * mutex to ensure that we don't race with another task doing
1099 * reserve_pmc_hardware or release_pmc_hardware.
1100 */
1101 err = 0;
1102 if (!atomic_inc_not_zero(&num_counters)) {
1103 mutex_lock(&pmc_reserve_mutex);
1104 if (atomic_read(&num_counters) == 0 &&
1105 reserve_pmc_hardware(perf_counter_interrupt))
1106 err = -EBUSY;
1107 else
1108 atomic_inc(&num_counters);
1109 mutex_unlock(&pmc_reserve_mutex);
1110 }
1111 counter->destroy = hw_perf_counter_destroy;
1112
1113 if (err)
1114 return ERR_PTR(err);
1115 return &power_pmu;
1116}
1117
1118/*
1119 * A counter has overflowed; update its count and record
1120 * things if requested. Note that interrupts are hard-disabled
1121 * here so there is no possibility of being interrupted.
1122 */
1123static void record_and_restart(struct perf_counter *counter, unsigned long val,
1124 struct pt_regs *regs, int nmi)
1125{
1126 u64 period = counter->hw.sample_period;
1127 s64 prev, delta, left;
1128 int record = 0;
1129
1130 /* we don't have to worry about interrupts here */
1131 prev = atomic64_read(&counter->hw.prev_count);
1132 delta = (val - prev) & 0xfffffffful;
1133 atomic64_add(delta, &counter->count);
1134
1135 /*
1136 * See if the total period for this counter has expired,
1137 * and update for the next period.
1138 */
1139 val = 0;
1140 left = atomic64_read(&counter->hw.period_left) - delta;
1141 if (period) {
1142 if (left <= 0) {
1143 left += period;
1144 if (left <= 0)
1145 left = period;
1146 record = 1;
1147 }
1148 if (left < 0x80000000LL)
1149 val = 0x80000000LL - left;
1150 }
1151
1152 /*
1153 * Finally record data if requested.
1154 */
1155 if (record) {
1156 struct perf_sample_data data = {
1157 .regs = regs,
1158 .addr = 0,
1159 .period = counter->hw.last_period,
1160 };
1161
1162 if (counter->attr.sample_type & PERF_SAMPLE_ADDR)
1163 perf_get_data_addr(regs, &data.addr);
1164
1165 if (perf_counter_overflow(counter, nmi, &data)) {
1166 /*
1167 * Interrupts are coming too fast - throttle them
1168 * by setting the counter to 0, so it will be
1169 * at least 2^30 cycles until the next interrupt
1170 * (assuming each counter counts at most 2 counts
1171 * per cycle).
1172 */
1173 val = 0;
1174 left = ~0ULL >> 1;
1175 }
1176 }
1177
1178 write_pmc(counter->hw.idx, val);
1179 atomic64_set(&counter->hw.prev_count, val);
1180 atomic64_set(&counter->hw.period_left, left);
1181 perf_counter_update_userpage(counter);
1182}
1183
1184/*
1185 * Called from generic code to get the misc flags (i.e. processor mode)
1186 * for an event.
1187 */
1188unsigned long perf_misc_flags(struct pt_regs *regs)
1189{
1190 u32 flags = perf_get_misc_flags(regs);
1191
1192 if (flags)
1193 return flags;
1194 return user_mode(regs) ? PERF_EVENT_MISC_USER :
1195 PERF_EVENT_MISC_KERNEL;
1196}
1197
1198/*
1199 * Called from generic code to get the instruction pointer
1200 * for an event.
1201 */
1202unsigned long perf_instruction_pointer(struct pt_regs *regs)
1203{
1204 unsigned long ip;
1205
1206 if (TRAP(regs) != 0xf00)
1207 return regs->nip; /* not a PMU interrupt */
1208
1209 ip = mfspr(SPRN_SIAR) + perf_ip_adjust(regs);
1210 return ip;
1211}
1212
1213/*
1214 * Performance monitor interrupt stuff
1215 */
1216static void perf_counter_interrupt(struct pt_regs *regs)
1217{
1218 int i;
1219 struct cpu_hw_counters *cpuhw = &__get_cpu_var(cpu_hw_counters);
1220 struct perf_counter *counter;
1221 unsigned long val;
1222 int found = 0;
1223 int nmi;
1224
1225 if (cpuhw->n_limited)
1226 freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5),
1227 mfspr(SPRN_PMC6));
1228
1229 perf_read_regs(regs);
1230
1231 nmi = perf_intr_is_nmi(regs);
1232 if (nmi)
1233 nmi_enter();
1234 else
1235 irq_enter();
1236
1237 for (i = 0; i < cpuhw->n_counters; ++i) {
1238 counter = cpuhw->counter[i];
1239 if (!counter->hw.idx || is_limited_pmc(counter->hw.idx))
1240 continue;
1241 val = read_pmc(counter->hw.idx);
1242 if ((int)val < 0) {
1243 /* counter has overflowed */
1244 found = 1;
1245 record_and_restart(counter, val, regs, nmi);
1246 }
1247 }
1248
1249 /*
1250 * In case we didn't find and reset the counter that caused
1251 * the interrupt, scan all counters and reset any that are
1252 * negative, to avoid getting continual interrupts.
1253 * Any that we processed in the previous loop will not be negative.
1254 */
1255 if (!found) {
1256 for (i = 0; i < ppmu->n_counter; ++i) {
1257 if (is_limited_pmc(i + 1))
1258 continue;
1259 val = read_pmc(i + 1);
1260 if ((int)val < 0)
1261 write_pmc(i + 1, 0);
1262 }
1263 }
1264
1265 /*
1266 * Reset MMCR0 to its normal value. This will set PMXE and
1267 * clear FC (freeze counters) and PMAO (perf mon alert occurred)
1268 * and thus allow interrupts to occur again.
1269 * XXX might want to use MSR.PM to keep the counters frozen until
1270 * we get back out of this interrupt.
1271 */
1272 write_mmcr0(cpuhw, cpuhw->mmcr[0]);
1273
1274 if (nmi)
1275 nmi_exit();
1276 else
1277 irq_exit();
1278}
1279
1280void hw_perf_counter_setup(int cpu)
1281{
1282 struct cpu_hw_counters *cpuhw = &per_cpu(cpu_hw_counters, cpu);
1283
1284 memset(cpuhw, 0, sizeof(*cpuhw));
1285 cpuhw->mmcr[0] = MMCR0_FC;
1286}
1287
1288int register_power_pmu(struct power_pmu *pmu)
1289{
1290 if (ppmu)
1291 return -EBUSY; /* something's already registered */
1292
1293 ppmu = pmu;
1294 pr_info("%s performance monitor hardware support registered\n",
1295 pmu->name);
1296
1297#ifdef MSR_HV
1298 /*
1299 * Use FCHV to ignore kernel events if MSR.HV is set.
1300 */
1301 if (mfmsr() & MSR_HV)
1302 freeze_counters_kernel = MMCR0_FCHV;
1303#endif /* CONFIG_PPC64 */
1304
1305 return 0;
1306}
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-27 10:25:56 -0500