aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Zijlstra <a.p.zijlstra@chello.nl>2010-06-16 08:37:10 -0400
committerIngo Molnar <mingo@elte.hu>2010-09-09 14:46:30 -0400
commita4eaf7f14675cb512d69f0c928055e73d0c6d252 (patch)
treee8a0f631fc28d4bd9becd2e9e2c71743c64ee3ec
parentfa407f35e0298d841e4088f95a7f9cf6e725c6d5 (diff)
perf: Rework the PMU methods
Replace pmu::{enable,disable,start,stop,unthrottle} with pmu::{add,del,start,stop}, all of which take a flags argument. The new interface extends the capability to stop a counter while keeping it scheduled on the PMU. We replace the throttled state with the generic stopped state. This also allows us to efficiently stop/start counters over certain code paths (like IRQ handlers). It also allows scheduling a counter without it starting, allowing for a generic frozen state (useful for rotating stopped counters). The stopped state is implemented in two different ways, depending on how the architecture implemented the throttled state: 1) We disable the counter: a) the pmu has per-counter enable bits, we flip that b) we program a NOP event, preserving the counter state 2) We store the counter state and ignore all read/overflow events Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: paulus <paulus@samba.org> Cc: stephane eranian <eranian@googlemail.com> Cc: Robert Richter <robert.richter@amd.com> Cc: Will Deacon <will.deacon@arm.com> Cc: Paul Mundt <lethal@linux-sh.org> Cc: Frederic Weisbecker <fweisbec@gmail.com> Cc: Cyrill Gorcunov <gorcunov@gmail.com> Cc: Lin Ming <ming.m.lin@intel.com> Cc: Yanmin <yanmin_zhang@linux.intel.com> Cc: Deng-Cheng Zhu <dengcheng.zhu@gmail.com> Cc: David Miller <davem@davemloft.net> Cc: Michael Cree <mcree@orcon.net.nz> LKML-Reference: <new-submission> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat
-rw-r--r--arch/alpha/kernel/perf_event.c71
-rw-r--r--arch/arm/kernel/perf_event.c96
-rw-r--r--arch/powerpc/kernel/perf_event.c105
-rw-r--r--arch/powerpc/kernel/perf_event_fsl_emb.c107
-rw-r--r--arch/sh/kernel/perf_event.c75
-rw-r--r--arch/sparc/kernel/perf_event.c109
-rw-r--r--arch/x86/kernel/cpu/perf_event.c106
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel.c2
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_ds.c2
-rw-r--r--include/linux/ftrace_event.h4
-rw-r--r--include/linux/perf_event.h54
-rw-r--r--kernel/hw_breakpoint.c29
-rw-r--r--kernel/perf_event.c140
-rw-r--r--kernel/trace/trace_event_perf.c7
14 files changed, 576 insertions, 331 deletions
diff --git a/arch/alpha/kernel/perf_event.c b/arch/alpha/kernel/perf_event.c
index 3e260731f8e6..380ef02d557a 100644
--- a/arch/alpha/kernel/perf_event.c
+++ b/arch/alpha/kernel/perf_event.c
@@ -307,7 +307,7 @@ again:
307 new_raw_count) != prev_raw_count) 307 new_raw_count) != prev_raw_count)
308 goto again; 308 goto again;
309 309
310 delta = (new_raw_count - (prev_raw_count & alpha_pmu->pmc_count_mask[idx])) + ovf; 310 delta = (new_raw_count - (prev_raw_count & alpha_pmu->pmc_count_mask[idx])) + ovf;
311 311
312 /* It is possible on very rare occasions that the PMC has overflowed 312 /* It is possible on very rare occasions that the PMC has overflowed
313 * but the interrupt is yet to come. Detect and fix this situation. 313 * but the interrupt is yet to come. Detect and fix this situation.
@@ -402,14 +402,13 @@ static void maybe_change_configuration(struct cpu_hw_events *cpuc)
402 struct hw_perf_event *hwc = &pe->hw; 402 struct hw_perf_event *hwc = &pe->hw;
403 int idx = hwc->idx; 403 int idx = hwc->idx;
404 404
405 if (cpuc->current_idx[j] != PMC_NO_INDEX) { 405 if (cpuc->current_idx[j] == PMC_NO_INDEX) {
406 cpuc->idx_mask |= (1<<cpuc->current_idx[j]); 406 alpha_perf_event_set_period(pe, hwc, idx);
407 continue; 407 cpuc->current_idx[j] = idx;
408 } 408 }
409 409
410 alpha_perf_event_set_period(pe, hwc, idx); 410 if (!(hwc->state & PERF_HES_STOPPED))
411 cpuc->current_idx[j] = idx; 411 cpuc->idx_mask |= (1<<cpuc->current_idx[j]);
412 cpuc->idx_mask |= (1<<cpuc->current_idx[j]);
413 } 412 }
414 cpuc->config = cpuc->event[0]->hw.config_base; 413 cpuc->config = cpuc->event[0]->hw.config_base;
415} 414}
@@ -420,7 +419,7 @@ static void maybe_change_configuration(struct cpu_hw_events *cpuc)
420 * - this function is called from outside this module via the pmu struct 419 * - this function is called from outside this module via the pmu struct
421 * returned from perf event initialisation. 420 * returned from perf event initialisation.
422 */ 421 */
423static int alpha_pmu_enable(struct perf_event *event) 422static int alpha_pmu_add(struct perf_event *event, int flags)
424{ 423{
425 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 424 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
426 int n0; 425 int n0;
@@ -455,6 +454,10 @@ static int alpha_pmu_enable(struct perf_event *event)
455 } 454 }
456 } 455 }
457 456
457 hwc->state = PERF_HES_UPTODATE;
458 if (!(flags & PERF_EF_START))
459 hwc->state |= PERF_HES_STOPPED;
460
458 local_irq_restore(flags); 461 local_irq_restore(flags);
459 perf_pmu_enable(event->pmu); 462 perf_pmu_enable(event->pmu);
460 463
@@ -467,7 +470,7 @@ static int alpha_pmu_enable(struct perf_event *event)
467 * - this function is called from outside this module via the pmu struct 470 * - this function is called from outside this module via the pmu struct
468 * returned from perf event initialisation. 471 * returned from perf event initialisation.
469 */ 472 */
470static void alpha_pmu_disable(struct perf_event *event) 473static void alpha_pmu_del(struct perf_event *event, int flags)
471{ 474{
472 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 475 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
473 struct hw_perf_event *hwc = &event->hw; 476 struct hw_perf_event *hwc = &event->hw;
@@ -514,13 +517,44 @@ static void alpha_pmu_read(struct perf_event *event)
514} 517}
515 518
516 519
517static void alpha_pmu_unthrottle(struct perf_event *event) 520static void alpha_pmu_stop(struct perf_event *event, int flags)
518{ 521{
519 struct hw_perf_event *hwc = &event->hw; 522 struct hw_perf_event *hwc = &event->hw;
520 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 523 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
521 524
525 if (!(hwc->state & PERF_HES_STOPPED)) {
526 cpuc->idx_mask &= !(1UL<<hwc->idx);
527 hwc->state |= PERF_HES_STOPPED;
528 }
529
530 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
531 alpha_perf_event_update(event, hwc, hwc->idx, 0);
532 hwc->state |= PERF_HES_UPTODATE;
533 }
534
535 if (cpuc->enabled)
536 wrperfmon(PERFMON_CMD_ENABLE, (1UL<<hwc->idx));
537}
538
539
540static void alpha_pmu_start(struct perf_event *event, int flags)
541{
542 struct hw_perf_event *hwc = &event->hw;
543 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
544
545 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
546 return;
547
548 if (flags & PERF_EF_RELOAD) {
549 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
550 alpha_perf_event_set_period(event, hwc, hwc->idx);
551 }
552
553 hwc->state = 0;
554
522 cpuc->idx_mask |= 1UL<<hwc->idx; 555 cpuc->idx_mask |= 1UL<<hwc->idx;
523 wrperfmon(PERFMON_CMD_ENABLE, (1UL<<hwc->idx)); 556 if (cpuc->enabled)
557 wrperfmon(PERFMON_CMD_ENABLE, (1UL<<hwc->idx));
524} 558}
525 559
526 560
@@ -671,7 +705,7 @@ static int alpha_pmu_event_init(struct perf_event *event)
671/* 705/*
672 * Main entry point - enable HW performance counters. 706 * Main entry point - enable HW performance counters.
673 */ 707 */
674static void alpha_pmu_pmu_enable(struct pmu *pmu) 708static void alpha_pmu_enable(struct pmu *pmu)
675{ 709{
676 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 710 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
677 711
@@ -697,7 +731,7 @@ static void alpha_pmu_pmu_enable(struct pmu *pmu)
697 * Main entry point - disable HW performance counters. 731 * Main entry point - disable HW performance counters.
698 */ 732 */
699 733
700static void alpha_pmu_pmu_disable(struct pmu *pmu) 734static void alpha_pmu_disable(struct pmu *pmu)
701{ 735{
702 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 736 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
703 737
@@ -711,13 +745,14 @@ static void alpha_pmu_pmu_disable(struct pmu *pmu)
711} 745}
712 746
713static struct pmu pmu = { 747static struct pmu pmu = {
714 .pmu_enable = alpha_pmu_pmu_enable, 748 .pmu_enable = alpha_pmu_enable,
715 .pmu_disable = alpha_pmu_pmu_disable, 749 .pmu_disable = alpha_pmu_disable,
716 .event_init = alpha_pmu_event_init, 750 .event_init = alpha_pmu_event_init,
717 .enable = alpha_pmu_enable, 751 .add = alpha_pmu_add,
718 .disable = alpha_pmu_disable, 752 .del = alpha_pmu_del,
753 .start = alpha_pmu_start,
754 .stop = alpha_pmu_stop,
719 .read = alpha_pmu_read, 755 .read = alpha_pmu_read,
720 .unthrottle = alpha_pmu_unthrottle,
721}; 756};
722 757
723 758
diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c
index 3343f3f4b973..448cfa6b3ef0 100644
--- a/arch/arm/kernel/perf_event.c
+++ b/arch/arm/kernel/perf_event.c
@@ -221,46 +221,56 @@ again:
221} 221}
222 222
223static void 223static void
224armpmu_disable(struct perf_event *event) 224armpmu_read(struct perf_event *event)
225{ 225{
226 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
227 struct hw_perf_event *hwc = &event->hw; 226 struct hw_perf_event *hwc = &event->hw;
228 int idx = hwc->idx;
229
230 WARN_ON(idx < 0);
231
232 clear_bit(idx, cpuc->active_mask);
233 armpmu->disable(hwc, idx);
234
235 barrier();
236 227
237 armpmu_event_update(event, hwc, idx); 228 /* Don't read disabled counters! */
238 cpuc->events[idx] = NULL; 229 if (hwc->idx < 0)
239 clear_bit(idx, cpuc->used_mask); 230 return;
240 231
241 perf_event_update_userpage(event); 232 armpmu_event_update(event, hwc, hwc->idx);
242} 233}
243 234
244static void 235static void
245armpmu_read(struct perf_event *event) 236armpmu_stop(struct perf_event *event, int flags)
246{ 237{
247 struct hw_perf_event *hwc = &event->hw; 238 struct hw_perf_event *hwc = &event->hw;
248 239
249 /* Don't read disabled counters! */ 240 if (!armpmu)
250 if (hwc->idx < 0)
251 return; 241 return;
252 242
253 armpmu_event_update(event, hwc, hwc->idx); 243 /*
244 * ARM pmu always has to update the counter, so ignore
245 * PERF_EF_UPDATE, see comments in armpmu_start().
246 */
247 if (!(hwc->state & PERF_HES_STOPPED)) {
248 armpmu->disable(hwc, hwc->idx);
249 barrier(); /* why? */
250 armpmu_event_update(event, hwc, hwc->idx);
251 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
252 }
254} 253}
255 254
256static void 255static void
257armpmu_unthrottle(struct perf_event *event) 256armpmu_start(struct perf_event *event, int flags)
258{ 257{
259 struct hw_perf_event *hwc = &event->hw; 258 struct hw_perf_event *hwc = &event->hw;
260 259
260 if (!armpmu)
261 return;
262
263 /*
264 * ARM pmu always has to reprogram the period, so ignore
265 * PERF_EF_RELOAD, see the comment below.
266 */
267 if (flags & PERF_EF_RELOAD)
268 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
269
270 hwc->state = 0;
261 /* 271 /*
262 * Set the period again. Some counters can't be stopped, so when we 272 * Set the period again. Some counters can't be stopped, so when we
263 * were throttled we simply disabled the IRQ source and the counter 273 * were stopped we simply disabled the IRQ source and the counter
264 * may have been left counting. If we don't do this step then we may 274 * may have been left counting. If we don't do this step then we may
265 * get an interrupt too soon or *way* too late if the overflow has 275 * get an interrupt too soon or *way* too late if the overflow has
266 * happened since disabling. 276 * happened since disabling.
@@ -269,8 +279,25 @@ armpmu_unthrottle(struct perf_event *event)
269 armpmu->enable(hwc, hwc->idx); 279 armpmu->enable(hwc, hwc->idx);
270} 280}
271 281
282static void
283armpmu_del(struct perf_event *event, int flags)
284{
285 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
286 struct hw_perf_event *hwc = &event->hw;
287 int idx = hwc->idx;
288
289 WARN_ON(idx < 0);
290
291 clear_bit(idx, cpuc->active_mask);
292 armpmu_stop(event, PERF_EF_UPDATE);
293 cpuc->events[idx] = NULL;
294 clear_bit(idx, cpuc->used_mask);
295
296 perf_event_update_userpage(event);
297}
298
272static int 299static int
273armpmu_enable(struct perf_event *event) 300armpmu_add(struct perf_event *event, int flags)
274{ 301{
275 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 302 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
276 struct hw_perf_event *hwc = &event->hw; 303 struct hw_perf_event *hwc = &event->hw;
@@ -295,11 +322,9 @@ armpmu_enable(struct perf_event *event)
295 cpuc->events[idx] = event; 322 cpuc->events[idx] = event;
296 set_bit(idx, cpuc->active_mask); 323 set_bit(idx, cpuc->active_mask);
297 324
298 /* Set the period for the event. */ 325 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
299 armpmu_event_set_period(event, hwc, idx); 326 if (flags & PERF_EF_START)
300 327 armpmu_start(event, PERF_EF_RELOAD);
301 /* Enable the event. */
302 armpmu->enable(hwc, idx);
303 328
304 /* Propagate our changes to the userspace mapping. */ 329 /* Propagate our changes to the userspace mapping. */
305 perf_event_update_userpage(event); 330 perf_event_update_userpage(event);
@@ -534,7 +559,7 @@ static int armpmu_event_init(struct perf_event *event)
534 return err; 559 return err;
535} 560}
536 561
537static void armpmu_pmu_enable(struct pmu *pmu) 562static void armpmu_enable(struct pmu *pmu)
538{ 563{
539 /* Enable all of the perf events on hardware. */ 564 /* Enable all of the perf events on hardware. */
540 int idx; 565 int idx;
@@ -555,20 +580,21 @@ static void armpmu_pmu_enable(struct pmu *pmu)
555 armpmu->start(); 580 armpmu->start();
556} 581}
557 582
558static void armpmu_pmu_disable(struct pmu *pmu) 583static void armpmu_disable(struct pmu *pmu)
559{ 584{
560 if (armpmu) 585 if (armpmu)
561 armpmu->stop(); 586 armpmu->stop();
562} 587}
563 588
564static struct pmu pmu = { 589static struct pmu pmu = {
565 .pmu_enable = armpmu_pmu_enable, 590 .pmu_enable = armpmu_enable,
566 .pmu_disable= armpmu_pmu_disable, 591 .pmu_disable = armpmu_disable,
567 .event_init = armpmu_event_init, 592 .event_init = armpmu_event_init,
568 .enable = armpmu_enable, 593 .add = armpmu_add,
569 .disable = armpmu_disable, 594 .del = armpmu_del,
570 .unthrottle = armpmu_unthrottle, 595 .start = armpmu_start,
571 .read = armpmu_read, 596 .stop = armpmu_stop,
597 .read = armpmu_read,
572}; 598};
573 599
574/* 600/*
diff --git a/arch/powerpc/kernel/perf_event.c b/arch/powerpc/kernel/perf_event.c
index deb84bbcb0e6..9cb4924b6c07 100644
--- a/arch/powerpc/kernel/perf_event.c
+++ b/arch/powerpc/kernel/perf_event.c
@@ -402,6 +402,9 @@ static void power_pmu_read(struct perf_event *event)
402{ 402{
403 s64 val, delta, prev; 403 s64 val, delta, prev;
404 404
405 if (event->hw.state & PERF_HES_STOPPED)
406 return;
407
405 if (!event->hw.idx) 408 if (!event->hw.idx)
406 return; 409 return;
407 /* 410 /*
@@ -517,7 +520,7 @@ static void write_mmcr0(struct cpu_hw_events *cpuhw, unsigned long mmcr0)
517 * Disable all events to prevent PMU interrupts and to allow 520 * Disable all events to prevent PMU interrupts and to allow
518 * events to be added or removed. 521 * events to be added or removed.
519 */ 522 */
520static void power_pmu_pmu_disable(struct pmu *pmu) 523static void power_pmu_disable(struct pmu *pmu)
521{ 524{
522 struct cpu_hw_events *cpuhw; 525 struct cpu_hw_events *cpuhw;
523 unsigned long flags; 526 unsigned long flags;
@@ -565,7 +568,7 @@ static void power_pmu_pmu_disable(struct pmu *pmu)
565 * If we were previously disabled and events were added, then 568 * If we were previously disabled and events were added, then
566 * put the new config on the PMU. 569 * put the new config on the PMU.
567 */ 570 */
568static void power_pmu_pmu_enable(struct pmu *pmu) 571static void power_pmu_enable(struct pmu *pmu)
569{ 572{
570 struct perf_event *event; 573 struct perf_event *event;
571 struct cpu_hw_events *cpuhw; 574 struct cpu_hw_events *cpuhw;
@@ -672,6 +675,8 @@ static void power_pmu_pmu_enable(struct pmu *pmu)
672 } 675 }
673 local64_set(&event->hw.prev_count, val); 676 local64_set(&event->hw.prev_count, val);
674 event->hw.idx = idx; 677 event->hw.idx = idx;
678 if (event->hw.state & PERF_HES_STOPPED)
679 val = 0;
675 write_pmc(idx, val); 680 write_pmc(idx, val);
676 perf_event_update_userpage(event); 681 perf_event_update_userpage(event);
677 } 682 }
@@ -727,7 +732,7 @@ static int collect_events(struct perf_event *group, int max_count,
727 * re-enable the PMU in order to get hw_perf_enable to do the 732 * re-enable the PMU in order to get hw_perf_enable to do the
728 * actual work of reconfiguring the PMU. 733 * actual work of reconfiguring the PMU.
729 */ 734 */
730static int power_pmu_enable(struct perf_event *event) 735static int power_pmu_add(struct perf_event *event, int ef_flags)
731{ 736{
732 struct cpu_hw_events *cpuhw; 737 struct cpu_hw_events *cpuhw;
733 unsigned long flags; 738 unsigned long flags;
@@ -749,6 +754,9 @@ static int power_pmu_enable(struct perf_event *event)
749 cpuhw->events[n0] = event->hw.config; 754 cpuhw->events[n0] = event->hw.config;
750 cpuhw->flags[n0] = event->hw.event_base; 755 cpuhw->flags[n0] = event->hw.event_base;
751 756
757 if (!(ef_flags & PERF_EF_START))
758 event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
759
752 /* 760 /*
753 * If group events scheduling transaction was started, 761 * If group events scheduling transaction was started,
754 * skip the schedulability test here, it will be peformed 762 * skip the schedulability test here, it will be peformed
@@ -777,7 +785,7 @@ nocheck:
777/* 785/*
778 * Remove a event from the PMU. 786 * Remove a event from the PMU.
779 */ 787 */
780static void power_pmu_disable(struct perf_event *event) 788static void power_pmu_del(struct perf_event *event, int ef_flags)
781{ 789{
782 struct cpu_hw_events *cpuhw; 790 struct cpu_hw_events *cpuhw;
783 long i; 791 long i;
@@ -826,27 +834,53 @@ static void power_pmu_disable(struct perf_event *event)
826} 834}
827 835
828/* 836/*
829 * Re-enable interrupts on a event after they were throttled 837 * POWER-PMU does not support disabling individual counters, hence
830 * because they were coming too fast. 838 * program their cycle counter to their max value and ignore the interrupts.
831 */ 839 */
832static void power_pmu_unthrottle(struct perf_event *event) 840
841static void power_pmu_start(struct perf_event *event, int ef_flags)
833{ 842{
834 s64 val, left;
835 unsigned long flags; 843 unsigned long flags;
844 s64 left;
836 845
837 if (!event->hw.idx || !event->hw.sample_period) 846 if (!event->hw.idx || !event->hw.sample_period)
838 return; 847 return;
848
849 if (!(event->hw.state & PERF_HES_STOPPED))
850 return;
851
852 if (ef_flags & PERF_EF_RELOAD)
853 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
854
855 local_irq_save(flags);
856 perf_pmu_disable(event->pmu);
857
858 event->hw.state = 0;
859 left = local64_read(&event->hw.period_left);
860 write_pmc(event->hw.idx, left);
861
862 perf_event_update_userpage(event);
863 perf_pmu_enable(event->pmu);
864 local_irq_restore(flags);
865}
866
867static void power_pmu_stop(struct perf_event *event, int ef_flags)
868{
869 unsigned long flags;
870
871 if (!event->hw.idx || !event->hw.sample_period)
872 return;
873
874 if (event->hw.state & PERF_HES_STOPPED)
875 return;
876
839 local_irq_save(flags); 877 local_irq_save(flags);
840 perf_pmu_disable(event->pmu); 878 perf_pmu_disable(event->pmu);
879
841 power_pmu_read(event); 880 power_pmu_read(event);
842 left = event->hw.sample_period; 881 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
843 event->hw.last_period = left; 882 write_pmc(event->hw.idx, 0);
844 val = 0; 883
845 if (left < 0x80000000L)
846 val = 0x80000000L - left;
847 write_pmc(event->hw.idx, val);
848 local64_set(&event->hw.prev_count, val);
849 local64_set(&event->hw.period_left, left);
850 perf_event_update_userpage(event); 884 perf_event_update_userpage(event);
851 perf_pmu_enable(event->pmu); 885 perf_pmu_enable(event->pmu);
852 local_irq_restore(flags); 886 local_irq_restore(flags);
@@ -1131,13 +1165,14 @@ static int power_pmu_event_init(struct perf_event *event)
1131} 1165}
1132 1166
1133struct pmu power_pmu = { 1167struct pmu power_pmu = {
1134 .pmu_enable = power_pmu_pmu_enable, 1168 .pmu_enable = power_pmu_enable,
1135 .pmu_disable = power_pmu_pmu_disable, 1169 .pmu_disable = power_pmu_disable,
1136 .event_init = power_pmu_event_init, 1170 .event_init = power_pmu_event_init,
1137 .enable = power_pmu_enable, 1171 .add = power_pmu_add,
1138 .disable = power_pmu_disable, 1172 .del = power_pmu_del,
1173 .start = power_pmu_start,
1174 .stop = power_pmu_stop,
1139 .read = power_pmu_read, 1175 .read = power_pmu_read,
1140 .unthrottle = power_pmu_unthrottle,
1141 .start_txn = power_pmu_start_txn, 1176 .start_txn = power_pmu_start_txn,
1142 .cancel_txn = power_pmu_cancel_txn, 1177 .cancel_txn = power_pmu_cancel_txn,
1143 .commit_txn = power_pmu_commit_txn, 1178 .commit_txn = power_pmu_commit_txn,
@@ -1155,6 +1190,11 @@ static void record_and_restart(struct perf_event *event, unsigned long val,
1155 s64 prev, delta, left; 1190 s64 prev, delta, left;
1156 int record = 0; 1191 int record = 0;
1157 1192
1193 if (event->hw.state & PERF_HES_STOPPED) {
1194 write_pmc(event->hw.idx, 0);
1195 return;
1196 }
1197
1158 /* we don't have to worry about interrupts here */ 1198 /* we don't have to worry about interrupts here */
1159 prev = local64_read(&event->hw.prev_count); 1199 prev = local64_read(&event->hw.prev_count);
1160 delta = (val - prev) & 0xfffffffful; 1200 delta = (val - prev) & 0xfffffffful;
@@ -1177,6 +1217,11 @@ static void record_and_restart(struct perf_event *event, unsigned long val,
1177 val = 0x80000000LL - left; 1217 val = 0x80000000LL - left;
1178 } 1218 }
1179 1219
1220 write_pmc(event->hw.idx, val);
1221 local64_set(&event->hw.prev_count, val);
1222 local64_set(&event->hw.period_left, left);
1223 perf_event_update_userpage(event);
1224
1180 /* 1225 /*
1181 * Finally record data if requested. 1226 * Finally record data if requested.
1182 */ 1227 */
@@ -1189,23 +1234,9 @@ static void record_and_restart(struct perf_event *event, unsigned long val,
1189 if (event->attr.sample_type & PERF_SAMPLE_ADDR) 1234 if (event->attr.sample_type & PERF_SAMPLE_ADDR)
1190 perf_get_data_addr(regs, &data.addr); 1235 perf_get_data_addr(regs, &data.addr);
1191 1236
1192 if (perf_event_overflow(event, nmi, &data, regs)) { 1237 if (perf_event_overflow(event, nmi, &data, regs))
1193 /* 1238 power_pmu_stop(event, 0);
1194 * Interrupts are coming too fast - throttle them
1195 * by setting the event to 0, so it will be
1196 * at least 2^30 cycles until the next interrupt
1197 * (assuming each event counts at most 2 counts
1198 * per cycle).
1199 */
1200 val = 0;
1201 left = ~0ULL >> 1;
1202 }
1203 } 1239 }
1204
1205 write_pmc(event->hw.idx, val);
1206 local64_set(&event->hw.prev_count, val);
1207 local64_set(&event->hw.period_left, left);
1208 perf_event_update_userpage(event);
1209} 1240}
1210 1241
1211/* 1242/*
diff --git a/arch/powerpc/kernel/perf_event_fsl_emb.c b/arch/powerpc/kernel/perf_event_fsl_emb.c
index 84b1974c628f..7ecca59ddf77 100644
--- a/arch/powerpc/kernel/perf_event_fsl_emb.c
+++ b/arch/powerpc/kernel/perf_event_fsl_emb.c
@@ -156,6 +156,9 @@ static void fsl_emb_pmu_read(struct perf_event *event)
156{ 156{
157 s64 val, delta, prev; 157 s64 val, delta, prev;
158 158
159 if (event->hw.state & PERF_HES_STOPPED)
160 return;
161
159 /* 162 /*
160 * Performance monitor interrupts come even when interrupts 163 * Performance monitor interrupts come even when interrupts
161 * are soft-disabled, as long as interrupts are hard-enabled. 164 * are soft-disabled, as long as interrupts are hard-enabled.
@@ -177,7 +180,7 @@ static void fsl_emb_pmu_read(struct perf_event *event)
177 * Disable all events to prevent PMU interrupts and to allow 180 * Disable all events to prevent PMU interrupts and to allow
178 * events to be added or removed. 181 * events to be added or removed.
179 */ 182 */
180static void fsl_emb_pmu_pmu_disable(struct pmu *pmu) 183static void fsl_emb_pmu_disable(struct pmu *pmu)
181{ 184{
182 struct cpu_hw_events *cpuhw; 185 struct cpu_hw_events *cpuhw;
183 unsigned long flags; 186 unsigned long flags;
@@ -216,7 +219,7 @@ static void fsl_emb_pmu_pmu_disable(struct pmu *pmu)
216 * If we were previously disabled and events were added, then 219 * If we were previously disabled and events were added, then
217 * put the new config on the PMU. 220 * put the new config on the PMU.
218 */ 221 */
219static void fsl_emb_pmu_pmu_enable(struct pmu *pmu) 222static void fsl_emb_pmu_enable(struct pmu *pmu)
220{ 223{
221 struct cpu_hw_events *cpuhw; 224 struct cpu_hw_events *cpuhw;
222 unsigned long flags; 225 unsigned long flags;
@@ -263,7 +266,7 @@ static int collect_events(struct perf_event *group, int max_count,
263} 266}
264 267
265/* context locked on entry */ 268/* context locked on entry */
266static int fsl_emb_pmu_enable(struct perf_event *event) 269static int fsl_emb_pmu_add(struct perf_event *event, int flags)
267{ 270{
268 struct cpu_hw_events *cpuhw; 271 struct cpu_hw_events *cpuhw;
269 int ret = -EAGAIN; 272 int ret = -EAGAIN;
@@ -302,6 +305,12 @@ static int fsl_emb_pmu_enable(struct perf_event *event)
302 val = 0x80000000L - left; 305 val = 0x80000000L - left;
303 } 306 }
304 local64_set(&event->hw.prev_count, val); 307 local64_set(&event->hw.prev_count, val);
308
309 if (!(flags & PERF_EF_START)) {
310 event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
311 val = 0;
312 }
313
305 write_pmc(i, val); 314 write_pmc(i, val);
306 perf_event_update_userpage(event); 315 perf_event_update_userpage(event);
307 316
@@ -316,7 +325,7 @@ static int fsl_emb_pmu_enable(struct perf_event *event)
316} 325}
317 326
318/* context locked on entry */ 327/* context locked on entry */
319static void fsl_emb_pmu_disable(struct perf_event *event) 328static void fsl_emb_pmu_del(struct perf_event *event, int flags)
320{ 329{
321 struct cpu_hw_events *cpuhw; 330 struct cpu_hw_events *cpuhw;
322 int i = event->hw.idx; 331 int i = event->hw.idx;
@@ -353,30 +362,49 @@ static void fsl_emb_pmu_disable(struct perf_event *event)
353 put_cpu_var(cpu_hw_events); 362 put_cpu_var(cpu_hw_events);
354} 363}
355 364
356/* 365static void fsl_emb_pmu_start(struct perf_event *event, int ef_flags)
357 * Re-enable interrupts on a event after they were throttled 366{
358 * because they were coming too fast. 367 unsigned long flags;
359 * 368 s64 left;
360 * Context is locked on entry, but perf is not disabled. 369
361 */ 370 if (event->hw.idx < 0 || !event->hw.sample_period)
362static void fsl_emb_pmu_unthrottle(struct perf_event *event) 371 return;
372
373 if (!(event->hw.state & PERF_HES_STOPPED))
374 return;
375
376 if (ef_flags & PERF_EF_RELOAD)
377 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
378
379 local_irq_save(flags);
380 perf_pmu_disable(event->pmu);
381
382 event->hw.state = 0;
383 left = local64_read(&event->hw.period_left);
384 write_pmc(event->hw.idx, left);
385
386 perf_event_update_userpage(event);
387 perf_pmu_enable(event->pmu);
388 local_irq_restore(flags);
389}
390
391static void fsl_emb_pmu_stop(struct perf_event *event, int ef_flags)
363{ 392{
364 s64 val, left;
365 unsigned long flags; 393 unsigned long flags;
366 394
367 if (event->hw.idx < 0 || !event->hw.sample_period) 395 if (event->hw.idx < 0 || !event->hw.sample_period)
368 return; 396 return;
397
398 if (event->hw.state & PERF_HES_STOPPED)
399 return;
400
369 local_irq_save(flags); 401 local_irq_save(flags);
370 perf_pmu_disable(event->pmu); 402 perf_pmu_disable(event->pmu);
403
371 fsl_emb_pmu_read(event); 404 fsl_emb_pmu_read(event);
372 left = event->hw.sample_period; 405 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
373 event->hw.last_period = left; 406 write_pmc(event->hw.idx, 0);
374 val = 0; 407
375 if (left < 0x80000000L)
376 val = 0x80000000L - left;
377 write_pmc(event->hw.idx, val);
378 local64_set(&event->hw.prev_count, val);
379 local64_set(&event->hw.period_left, left);
380 perf_event_update_userpage(event); 408 perf_event_update_userpage(event);
381 perf_pmu_enable(event->pmu); 409 perf_pmu_enable(event->pmu);
382 local_irq_restore(flags); 410 local_irq_restore(flags);
@@ -524,13 +552,14 @@ static int fsl_emb_pmu_event_init(struct perf_event *event)
524} 552}
525 553
526static struct pmu fsl_emb_pmu = { 554static struct pmu fsl_emb_pmu = {
527 .pmu_enable = fsl_emb_pmu_pmu_enable, 555 .pmu_enable = fsl_emb_pmu_enable,
528 .pmu_disable = fsl_emb_pmu_pmu_disable, 556 .pmu_disable = fsl_emb_pmu_disable,
529 .event_init = fsl_emb_pmu_event_init, 557 .event_init = fsl_emb_pmu_event_init,
530 .enable = fsl_emb_pmu_enable, 558 .add = fsl_emb_pmu_add,
531 .disable = fsl_emb_pmu_disable, 559 .del = fsl_emb_pmu_del,
560 .start = fsl_emb_pmu_start,
561 .stop = fsl_emb_pmu_stop,
532 .read = fsl_emb_pmu_read, 562 .read = fsl_emb_pmu_read,
533 .unthrottle = fsl_emb_pmu_unthrottle,
534}; 563};
535 564
536/* 565/*
@@ -545,6 +574,11 @@ static void record_and_restart(struct perf_event *event, unsigned long val,
545 s64 prev, delta, left; 574 s64 prev, delta, left;
546 int record = 0; 575 int record = 0;
547 576
577 if (event->hw.state & PERF_HES_STOPPED) {
578 write_pmc(event->hw.idx, 0);
579 return;
580 }
581
548 /* we don't have to worry about interrupts here */ 582 /* we don't have to worry about interrupts here */
549 prev = local64_read(&event->hw.prev_count); 583 prev = local64_read(&event->hw.prev_count);
550 delta = (val - prev) & 0xfffffffful; 584 delta = (val - prev) & 0xfffffffful;
@@ -567,6 +601,11 @@ static void record_and_restart(struct perf_event *event, unsigned long val,
567 val = 0x80000000LL - left; 601 val = 0x80000000LL - left;
568 } 602 }
569 603
604 write_pmc(event->hw.idx, val);
605 local64_set(&event->hw.prev_count, val);
606 local64_set(&event->hw.period_left, left);
607 perf_event_update_userpage(event);
608
570 /* 609 /*
571 * Finally record data if requested. 610 * Finally record data if requested.
572 */ 611 */
@@ -576,23 +615,9 @@ static void record_and_restart(struct perf_event *event, unsigned long val,
576 perf_sample_data_init(&data, 0); 615 perf_sample_data_init(&data, 0);
577 data.period = event->hw.last_period; 616 data.period = event->hw.last_period;
578 617
579 if (perf_event_overflow(event, nmi, &data, regs)) { 618 if (perf_event_overflow(event, nmi, &data, regs))
580 /* 619 fsl_emb_pmu_stop(event, 0);
581 * Interrupts are coming too fast - throttle them
582 * by setting the event to 0, so it will be
583 * at least 2^30 cycles until the next interrupt
584 * (assuming each event counts at most 2 counts
585 * per cycle).
586 */
587 val = 0;
588 left = ~0ULL >> 1;
589 }
590 } 620 }
591
592 write_pmc(event->hw.idx, val);
593 local64_set(&event->hw.prev_count, val);
594 local64_set(&event->hw.period_left, left);
595 perf_event_update_userpage(event);
596} 621}
597 622
598static void perf_event_interrupt(struct pt_regs *regs) 623static void perf_event_interrupt(struct pt_regs *regs)
diff --git a/arch/sh/kernel/perf_event.c b/arch/sh/kernel/perf_event.c
index 4bbe19058a58..cf39c4873468 100644
--- a/arch/sh/kernel/perf_event.c
+++ b/arch/sh/kernel/perf_event.c
@@ -206,26 +206,52 @@ again:
206 local64_add(delta, &event->count); 206 local64_add(delta, &event->count);
207} 207}
208 208
209static void sh_pmu_disable(struct perf_event *event) 209static void sh_pmu_stop(struct perf_event *event, int flags)
210{ 210{
211 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 211 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
212 struct hw_perf_event *hwc = &event->hw; 212 struct hw_perf_event *hwc = &event->hw;
213 int idx = hwc->idx; 213 int idx = hwc->idx;
214 214
215 clear_bit(idx, cpuc->active_mask); 215 if (!(event->hw.state & PERF_HES_STOPPED)) {
216 sh_pmu->disable(hwc, idx); 216 sh_pmu->disable(hwc, idx);
217 cpuc->events[idx] = NULL;
218 event->hw.state |= PERF_HES_STOPPED;
219 }
217 220
218 barrier(); 221 if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) {
222 sh_perf_event_update(event, &event->hw, idx);
223 event->hw.state |= PERF_HES_UPTODATE;
224 }
225}
219 226
220 sh_perf_event_update(event, &event->hw, idx); 227static void sh_pmu_start(struct perf_event *event, int flags)
228{
229 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
230 struct hw_perf_event *hwc = &event->hw;
231 int idx = hwc->idx;
221 232
222 cpuc->events[idx] = NULL; 233 if (WARN_ON_ONCE(idx == -1))
223 clear_bit(idx, cpuc->used_mask); 234 return;
235
236 if (flags & PERF_EF_RELOAD)
237 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
238
239 cpuc->events[idx] = event;
240 event->hw.state = 0;
241 sh_pmu->enable(hwc, idx);
242}
243
244static void sh_pmu_del(struct perf_event *event, int flags)
245{
246 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
247
248 sh_pmu_stop(event, PERF_EF_UPDATE);
249 __clear_bit(event->hw.idx, cpuc->used_mask);
224 250
225 perf_event_update_userpage(event); 251 perf_event_update_userpage(event);
226} 252}
227 253
228static int sh_pmu_enable(struct perf_event *event) 254static int sh_pmu_add(struct perf_event *event, int flags)
229{ 255{
230 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 256 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
231 struct hw_perf_event *hwc = &event->hw; 257 struct hw_perf_event *hwc = &event->hw;
@@ -234,21 +260,20 @@ static int sh_pmu_enable(struct perf_event *event)
234 260
235 perf_pmu_disable(event->pmu); 261 perf_pmu_disable(event->pmu);
236 262
237 if (test_and_set_bit(idx, cpuc->used_mask)) { 263 if (__test_and_set_bit(idx, cpuc->used_mask)) {
238 idx = find_first_zero_bit(cpuc->used_mask, sh_pmu->num_events); 264 idx = find_first_zero_bit(cpuc->used_mask, sh_pmu->num_events);
239 if (idx == sh_pmu->num_events) 265 if (idx == sh_pmu->num_events)
240 goto out; 266 goto out;
241 267
242 set_bit(idx, cpuc->used_mask); 268 __set_bit(idx, cpuc->used_mask);
243 hwc->idx = idx; 269 hwc->idx = idx;
244 } 270 }
245 271
246 sh_pmu->disable(hwc, idx); 272 sh_pmu->disable(hwc, idx);
247 273
248 cpuc->events[idx] = event; 274 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
249 set_bit(idx, cpuc->active_mask); 275 if (flags & PERF_EF_START)
250 276 sh_pmu_start(event, PERF_EF_RELOAD);
251 sh_pmu->enable(hwc, idx);
252 277
253 perf_event_update_userpage(event); 278 perf_event_update_userpage(event);
254 ret = 0; 279 ret = 0;
@@ -285,7 +310,7 @@ static int sh_pmu_event_init(struct perf_event *event)
285 return err; 310 return err;
286} 311}
287 312
288static void sh_pmu_pmu_enable(struct pmu *pmu) 313static void sh_pmu_enable(struct pmu *pmu)
289{ 314{
290 if (!sh_pmu_initialized()) 315 if (!sh_pmu_initialized())
291 return; 316 return;
@@ -293,7 +318,7 @@ static void sh_pmu_pmu_enable(struct pmu *pmu)
293 sh_pmu->enable_all(); 318 sh_pmu->enable_all();
294} 319}
295 320
296static void sh_pmu_pmu_disable(struct pmu *pmu) 321static void sh_pmu_disable(struct pmu *pmu)
297{ 322{
298 if (!sh_pmu_initialized()) 323 if (!sh_pmu_initialized())
299 return; 324 return;
@@ -302,11 +327,13 @@ static void sh_pmu_pmu_disable(struct pmu *pmu)
302} 327}
303 328
304static struct pmu pmu = { 329static struct pmu pmu = {
305 .pmu_enable = sh_pmu_pmu_enable, 330 .pmu_enable = sh_pmu_enable,
306 .pmu_disable = sh_pmu_pmu_disable, 331 .pmu_disable = sh_pmu_disable,
307 .event_init = sh_pmu_event_init, 332 .event_init = sh_pmu_event_init,
308 .enable = sh_pmu_enable, 333 .add = sh_pmu_add,
309 .disable = sh_pmu_disable, 334 .del = sh_pmu_del,
335 .start = sh_pmu_start,
336 .stop = sh_pmu_stop,
310 .read = sh_pmu_read, 337 .read = sh_pmu_read,
311}; 338};
312 339
@@ -334,15 +361,15 @@ sh_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
334 return NOTIFY_OK; 361 return NOTIFY_OK;
335} 362}
336 363
337int __cpuinit register_sh_pmu(struct sh_pmu *pmu) 364int __cpuinit register_sh_pmu(struct sh_pmu *_pmu)
338{ 365{
339 if (sh_pmu) 366 if (sh_pmu)
340 return -EBUSY; 367 return -EBUSY;
341 sh_pmu = pmu; 368 sh_pmu = _pmu;
342 369
343 pr_info("Performance Events: %s support registered\n", pmu->name); 370 pr_info("Performance Events: %s support registered\n", _pmu->name);
344 371
345 WARN_ON(pmu->num_events > MAX_HWEVENTS); 372 WARN_ON(_pmu->num_events > MAX_HWEVENTS);
346 373
347 perf_pmu_register(&pmu); 374 perf_pmu_register(&pmu);
348 perf_cpu_notifier(sh_pmu_notifier); 375 perf_cpu_notifier(sh_pmu_notifier);
diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c
index 37cae676536c..516be2314b54 100644
--- a/arch/sparc/kernel/perf_event.c
+++ b/arch/sparc/kernel/perf_event.c
@@ -658,13 +658,16 @@ static u64 maybe_change_configuration(struct cpu_hw_events *cpuc, u64 pcr)
658 658
659 enc = perf_event_get_enc(cpuc->events[i]); 659 enc = perf_event_get_enc(cpuc->events[i]);
660 pcr &= ~mask_for_index(idx); 660 pcr &= ~mask_for_index(idx);
661 pcr |= event_encoding(enc, idx); 661 if (hwc->state & PERF_HES_STOPPED)
662 pcr |= nop_for_index(idx);
663 else
664 pcr |= event_encoding(enc, idx);
662 } 665 }
663out: 666out:
664 return pcr; 667 return pcr;
665} 668}
666 669
667static void sparc_pmu_pmu_enable(struct pmu *pmu) 670static void sparc_pmu_enable(struct pmu *pmu)
668{ 671{
669 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 672 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
670 u64 pcr; 673 u64 pcr;
@@ -691,7 +694,7 @@ static void sparc_pmu_pmu_enable(struct pmu *pmu)
691 pcr_ops->write(cpuc->pcr); 694 pcr_ops->write(cpuc->pcr);
692} 695}
693 696
694static void sparc_pmu_pmu_disable(struct pmu *pmu) 697static void sparc_pmu_disable(struct pmu *pmu)
695{ 698{
696 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 699 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
697 u64 val; 700 u64 val;
@@ -710,10 +713,53 @@ static void sparc_pmu_pmu_disable(struct pmu *pmu)
710 pcr_ops->write(cpuc->pcr); 713 pcr_ops->write(cpuc->pcr);
711} 714}
712 715
713static void sparc_pmu_disable(struct perf_event *event) 716static int active_event_index(struct cpu_hw_events *cpuc,
717 struct perf_event *event)
718{
719 int i;
720
721 for (i = 0; i < cpuc->n_events; i++) {
722 if (cpuc->event[i] == event)
723 break;
724 }
725 BUG_ON(i == cpuc->n_events);
726 return cpuc->current_idx[i];
727}
728
729static void sparc_pmu_start(struct perf_event *event, int flags)
730{
731 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
732 int idx = active_event_index(cpuc, event);
733
734 if (flags & PERF_EF_RELOAD) {
735 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
736 sparc_perf_event_set_period(event, &event->hw, idx);
737 }
738
739 event->hw.state = 0;
740
741 sparc_pmu_enable_event(cpuc, &event->hw, idx);
742}
743
744static void sparc_pmu_stop(struct perf_event *event, int flags)
745{
746 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
747 int idx = active_event_index(cpuc, event);
748
749 if (!(event->hw.state & PERF_HES_STOPPED)) {
750 sparc_pmu_disable_event(cpuc, &event->hw, idx);
751 event->hw.state |= PERF_HES_STOPPED;
752 }
753
754 if (!(event->hw.state & PERF_HES_UPTODATE) && (flags & PERF_EF_UPDATE)) {
755 sparc_perf_event_update(event, &event->hw, idx);
756 event->hw.state |= PERF_HES_UPTODATE;
757 }
758}
759
760static void sparc_pmu_del(struct perf_event *event, int _flags)
714{ 761{
715 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 762 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
716 struct hw_perf_event *hwc = &event->hw;
717 unsigned long flags; 763 unsigned long flags;
718 int i; 764 int i;
719 765
@@ -722,7 +768,10 @@ static void sparc_pmu_disable(struct perf_event *event)
722 768
723 for (i = 0; i < cpuc->n_events; i++) { 769 for (i = 0; i < cpuc->n_events; i++) {
724 if (event == cpuc->event[i]) { 770 if (event == cpuc->event[i]) {
725 int idx = cpuc->current_idx[i]; 771 /* Absorb the final count and turn off the
772 * event.
773 */
774 sparc_pmu_stop(event, PERF_EF_UPDATE);
726 775
727 /* Shift remaining entries down into 776 /* Shift remaining entries down into
728 * the existing slot. 777 * the existing slot.
@@ -734,13 +783,6 @@ static void sparc_pmu_disable(struct perf_event *event)
734 cpuc->current_idx[i]; 783 cpuc->current_idx[i];
735 } 784 }
736 785
737 /* Absorb the final count and turn off the
738 * event.
739 */
740 sparc_pmu_disable_event(cpuc, hwc, idx);
741 barrier();
742 sparc_perf_event_update(event, hwc, idx);
743
744 perf_event_update_userpage(event); 786 perf_event_update_userpage(event);
745 787
746 cpuc->n_events--; 788 cpuc->n_events--;
@@ -752,19 +794,6 @@ static void sparc_pmu_disable(struct perf_event *event)
752 local_irq_restore(flags); 794 local_irq_restore(flags);
753} 795}
754 796
755static int active_event_index(struct cpu_hw_events *cpuc,
756 struct perf_event *event)
757{
758 int i;
759
760 for (i = 0; i < cpuc->n_events; i++) {
761 if (cpuc->event[i] == event)
762 break;
763 }
764 BUG_ON(i == cpuc->n_events);
765 return cpuc->current_idx[i];
766}
767
768static void sparc_pmu_read(struct perf_event *event) 797static void sparc_pmu_read(struct perf_event *event)
769{ 798{
770 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 799 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
@@ -774,15 +803,6 @@ static void sparc_pmu_read(struct perf_event *event)
774 sparc_perf_event_update(event, hwc, idx); 803 sparc_perf_event_update(event, hwc, idx);
775} 804}
776 805
777static void sparc_pmu_unthrottle(struct perf_event *event)
778{
779 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
780 int idx = active_event_index(cpuc, event);
781 struct hw_perf_event *hwc = &event->hw;
782
783 sparc_pmu_enable_event(cpuc, hwc, idx);
784}
785
786static atomic_t active_events = ATOMIC_INIT(0); 806static atomic_t active_events = ATOMIC_INIT(0);
787static DEFINE_MUTEX(pmc_grab_mutex); 807static DEFINE_MUTEX(pmc_grab_mutex);
788 808
@@ -984,7 +1004,7 @@ static int collect_events(struct perf_event *group, int max_count,
984 return n; 1004 return n;
985} 1005}
986 1006
987static int sparc_pmu_enable(struct perf_event *event) 1007static int sparc_pmu_add(struct perf_event *event, int ef_flags)
988{ 1008{
989 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 1009 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
990 int n0, ret = -EAGAIN; 1010 int n0, ret = -EAGAIN;
@@ -1001,6 +1021,10 @@ static int sparc_pmu_enable(struct perf_event *event)
1001 cpuc->events[n0] = event->hw.event_base; 1021 cpuc->events[n0] = event->hw.event_base;
1002 cpuc->current_idx[n0] = PIC_NO_INDEX; 1022 cpuc->current_idx[n0] = PIC_NO_INDEX;
1003 1023
1024 event->hw.state = PERF_HES_UPTODATE;
1025 if (!(ef_flags & PERF_EF_START))
1026 event->hw.state |= PERF_HES_STOPPED;
1027
1004 /* 1028 /*
1005 * If group events scheduling transaction was started, 1029 * If group events scheduling transaction was started,
1006 * skip the schedulability test here, it will be peformed 1030 * skip the schedulability test here, it will be peformed
@@ -1156,13 +1180,14 @@ static int sparc_pmu_commit_txn(struct pmu *pmu)
1156} 1180}
1157 1181
1158static struct pmu pmu = { 1182static struct pmu pmu = {
1159 .pmu_enable = sparc_pmu_pmu_enable, 1183 .pmu_enable = sparc_pmu_enable,
1160 .pmu_disable = sparc_pmu_pmu_disable, 1184 .pmu_disable = sparc_pmu_disable,
1161 .event_init = sparc_pmu_event_init, 1185 .event_init = sparc_pmu_event_init,
1162 .enable = sparc_pmu_enable, 1186 .add = sparc_pmu_add,
1163 .disable = sparc_pmu_disable, 1187 .del = sparc_pmu_del,
1188 .start = sparc_pmu_start,
1189 .stop = sparc_pmu_stop,
1164 .read = sparc_pmu_read, 1190 .read = sparc_pmu_read,
1165 .unthrottle = sparc_pmu_unthrottle,
1166 .start_txn = sparc_pmu_start_txn, 1191 .start_txn = sparc_pmu_start_txn,
1167 .cancel_txn = sparc_pmu_cancel_txn, 1192 .cancel_txn = sparc_pmu_cancel_txn,
1168 .commit_txn = sparc_pmu_commit_txn, 1193 .commit_txn = sparc_pmu_commit_txn,
@@ -1243,7 +1268,7 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self,
1243 continue; 1268 continue;
1244 1269
1245 if (perf_event_overflow(event, 1, &data, regs)) 1270 if (perf_event_overflow(event, 1, &data, regs))
1246 sparc_pmu_disable_event(cpuc, hwc, idx); 1271 sparc_pmu_stop(event, 0);
1247 } 1272 }
1248 1273
1249 return NOTIFY_STOP; 1274 return NOTIFY_STOP;
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 79705ac45019..dd6fec710677 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -583,7 +583,7 @@ static void x86_pmu_disable_all(void)
583 } 583 }
584} 584}
585 585
586static void x86_pmu_pmu_disable(struct pmu *pmu) 586static void x86_pmu_disable(struct pmu *pmu)
587{ 587{
588 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 588 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
589 589
@@ -800,10 +800,10 @@ static inline int match_prev_assignment(struct hw_perf_event *hwc,
800 hwc->last_tag == cpuc->tags[i]; 800 hwc->last_tag == cpuc->tags[i];
801} 801}
802 802
803static int x86_pmu_start(struct perf_event *event); 803static void x86_pmu_start(struct perf_event *event, int flags);
804static void x86_pmu_stop(struct perf_event *event); 804static void x86_pmu_stop(struct perf_event *event, int flags);
805 805
806static void x86_pmu_pmu_enable(struct pmu *pmu) 806static void x86_pmu_enable(struct pmu *pmu)
807{ 807{
808 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 808 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
809 struct perf_event *event; 809 struct perf_event *event;
@@ -839,7 +839,14 @@ static void x86_pmu_pmu_enable(struct pmu *pmu)
839 match_prev_assignment(hwc, cpuc, i)) 839 match_prev_assignment(hwc, cpuc, i))
840 continue; 840 continue;
841 841
842 x86_pmu_stop(event); 842 /*
843 * Ensure we don't accidentally enable a stopped
844 * counter simply because we rescheduled.
845 */
846 if (hwc->state & PERF_HES_STOPPED)
847 hwc->state |= PERF_HES_ARCH;
848
849 x86_pmu_stop(event, PERF_EF_UPDATE);
843 } 850 }
844 851
845 for (i = 0; i < cpuc->n_events; i++) { 852 for (i = 0; i < cpuc->n_events; i++) {
@@ -851,7 +858,10 @@ static void x86_pmu_pmu_enable(struct pmu *pmu)
851 else if (i < n_running) 858 else if (i < n_running)
852 continue; 859 continue;
853 860
854 x86_pmu_start(event); 861 if (hwc->state & PERF_HES_ARCH)
862 continue;
863
864 x86_pmu_start(event, PERF_EF_RELOAD);
855 } 865 }
856 cpuc->n_added = 0; 866 cpuc->n_added = 0;
857 perf_events_lapic_init(); 867 perf_events_lapic_init();
@@ -952,15 +962,12 @@ static void x86_pmu_enable_event(struct perf_event *event)
952} 962}
953 963
954/* 964/*
955 * activate a single event 965 * Add a single event to the PMU.
956 * 966 *
957 * The event is added to the group of enabled events 967 * The event is added to the group of enabled events
958 * but only if it can be scehduled with existing events. 968 * but only if it can be scehduled with existing events.
959 *
960 * Called with PMU disabled. If successful and return value 1,
961 * then guaranteed to call perf_enable() and hw_perf_enable()
962 */ 969 */
963static int x86_pmu_enable(struct perf_event *event) 970static int x86_pmu_add(struct perf_event *event, int flags)
964{ 971{
965 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 972 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
966 struct hw_perf_event *hwc; 973 struct hw_perf_event *hwc;
@@ -975,10 +982,14 @@ static int x86_pmu_enable(struct perf_event *event)
975 if (ret < 0) 982 if (ret < 0)
976 goto out; 983 goto out;
977 984
985 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
986 if (!(flags & PERF_EF_START))
987 hwc->state |= PERF_HES_ARCH;
988
978 /* 989 /*
979 * If group events scheduling transaction was started, 990 * If group events scheduling transaction was started,
980 * skip the schedulability test here, it will be peformed 991 * skip the schedulability test here, it will be peformed
981 * at commit time(->commit_txn) as a whole 992 * at commit time (->commit_txn) as a whole
982 */ 993 */
983 if (cpuc->group_flag & PERF_EVENT_TXN) 994 if (cpuc->group_flag & PERF_EVENT_TXN)
984 goto done_collect; 995 goto done_collect;
@@ -1003,27 +1014,28 @@ out:
1003 return ret; 1014 return ret;
1004} 1015}
1005 1016
1006static int x86_pmu_start(struct perf_event *event) 1017static void x86_pmu_start(struct perf_event *event, int flags)
1007{ 1018{
1008 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 1019 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1009 int idx = event->hw.idx; 1020 int idx = event->hw.idx;
1010 1021
1011 if (idx == -1) 1022 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1012 return -EAGAIN; 1023 return;
1024
1025 if (WARN_ON_ONCE(idx == -1))
1026 return;
1027
1028 if (flags & PERF_EF_RELOAD) {
1029 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1030 x86_perf_event_set_period(event);
1031 }
1032
1033 event->hw.state = 0;
1013 1034
1014 x86_perf_event_set_period(event);
1015 cpuc->events[idx] = event; 1035 cpuc->events[idx] = event;
1016 __set_bit(idx, cpuc->active_mask); 1036 __set_bit(idx, cpuc->active_mask);
1017 x86_pmu.enable(event); 1037 x86_pmu.enable(event);
1018 perf_event_update_userpage(event); 1038 perf_event_update_userpage(event);
1019
1020 return 0;
1021}
1022
1023static void x86_pmu_unthrottle(struct perf_event *event)
1024{
1025 int ret = x86_pmu_start(event);
1026 WARN_ON_ONCE(ret);
1027} 1039}
1028 1040
1029void perf_event_print_debug(void) 1041void perf_event_print_debug(void)
@@ -1080,27 +1092,29 @@ void perf_event_print_debug(void)
1080 local_irq_restore(flags); 1092 local_irq_restore(flags);
1081} 1093}
1082 1094
1083static void x86_pmu_stop(struct perf_event *event) 1095static void x86_pmu_stop(struct perf_event *event, int flags)
1084{ 1096{
1085 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 1097 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1086 struct hw_perf_event *hwc = &event->hw; 1098 struct hw_perf_event *hwc = &event->hw;
1087 int idx = hwc->idx;
1088
1089 if (!__test_and_clear_bit(idx, cpuc->active_mask))
1090 return;
1091
1092 x86_pmu.disable(event);
1093 1099
1094 /* 1100 if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
1095 * Drain the remaining delta count out of a event 1101 x86_pmu.disable(event);
1096 * that we are disabling: 1102 cpuc->events[hwc->idx] = NULL;
1097 */ 1103 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1098 x86_perf_event_update(event); 1104 hwc->state |= PERF_HES_STOPPED;
1105 }
1099 1106
1100 cpuc->events[idx] = NULL; 1107 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1108 /*
1109 * Drain the remaining delta count out of a event
1110 * that we are disabling:
1111 */
1112 x86_perf_event_update(event);
1113 hwc->state |= PERF_HES_UPTODATE;
1114 }
1101} 1115}
1102 1116
1103static void x86_pmu_disable(struct perf_event *event) 1117static void x86_pmu_del(struct perf_event *event, int flags)
1104{ 1118{
1105 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 1119 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1106 int i; 1120 int i;
@@ -1113,7 +1127,7 @@ static void x86_pmu_disable(struct perf_event *event)
1113 if (cpuc->group_flag & PERF_EVENT_TXN) 1127 if (cpuc->group_flag & PERF_EVENT_TXN)
1114 return; 1128 return;
1115 1129
1116 x86_pmu_stop(event); 1130 x86_pmu_stop(event, PERF_EF_UPDATE);
1117 1131
1118 for (i = 0; i < cpuc->n_events; i++) { 1132 for (i = 0; i < cpuc->n_events; i++) {
1119 if (event == cpuc->event_list[i]) { 1133 if (event == cpuc->event_list[i]) {
@@ -1165,7 +1179,7 @@ static int x86_pmu_handle_irq(struct pt_regs *regs)
1165 continue; 1179 continue;
1166 1180
1167 if (perf_event_overflow(event, 1, &data, regs)) 1181 if (perf_event_overflow(event, 1, &data, regs))
1168 x86_pmu_stop(event); 1182 x86_pmu_stop(event, 0);
1169 } 1183 }
1170 1184
1171 if (handled) 1185 if (handled)
@@ -1605,15 +1619,17 @@ int x86_pmu_event_init(struct perf_event *event)
1605} 1619}
1606 1620
1607static struct pmu pmu = { 1621static struct pmu pmu = {
1608 .pmu_enable = x86_pmu_pmu_enable, 1622 .pmu_enable = x86_pmu_enable,
1609 .pmu_disable = x86_pmu_pmu_disable, 1623 .pmu_disable = x86_pmu_disable,
1624
1610 .event_init = x86_pmu_event_init, 1625 .event_init = x86_pmu_event_init,
1611 .enable = x86_pmu_enable, 1626
1612 .disable = x86_pmu_disable, 1627 .add = x86_pmu_add,
1628 .del = x86_pmu_del,
1613 .start = x86_pmu_start, 1629 .start = x86_pmu_start,
1614 .stop = x86_pmu_stop, 1630 .stop = x86_pmu_stop,
1615 .read = x86_pmu_read, 1631 .read = x86_pmu_read,
1616 .unthrottle = x86_pmu_unthrottle, 1632
1617 .start_txn = x86_pmu_start_txn, 1633 .start_txn = x86_pmu_start_txn,
1618 .cancel_txn = x86_pmu_cancel_txn, 1634 .cancel_txn = x86_pmu_cancel_txn,
1619 .commit_txn = x86_pmu_commit_txn, 1635 .commit_txn = x86_pmu_commit_txn,
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index ee05c90012d2..82395f2378ec 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -763,7 +763,7 @@ again:
763 data.period = event->hw.last_period; 763 data.period = event->hw.last_period;
764 764
765 if (perf_event_overflow(event, 1, &data, regs)) 765 if (perf_event_overflow(event, 1, &data, regs))
766 x86_pmu_stop(event); 766 x86_pmu_stop(event, 0);
767 } 767 }
768 768
769 /* 769 /*
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index 18018d1311cd..9893a2f77b7a 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -491,7 +491,7 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
491 regs.flags &= ~PERF_EFLAGS_EXACT; 491 regs.flags &= ~PERF_EFLAGS_EXACT;
492 492
493 if (perf_event_overflow(event, 1, &data, &regs)) 493 if (perf_event_overflow(event, 1, &data, &regs))
494 x86_pmu_stop(event); 494 x86_pmu_stop(event, 0);
495} 495}
496 496
497static void intel_pmu_drain_pebs_core(struct pt_regs *iregs) 497static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
diff --git a/include/linux/ftrace_event.h b/include/linux/ftrace_event.h
index 5f8ad7bec636..8beabb958f61 100644
--- a/include/linux/ftrace_event.h
+++ b/include/linux/ftrace_event.h
@@ -252,8 +252,8 @@ DECLARE_PER_CPU(struct pt_regs, perf_trace_regs);
252 252
253extern int perf_trace_init(struct perf_event *event); 253extern int perf_trace_init(struct perf_event *event);
254extern void perf_trace_destroy(struct perf_event *event); 254extern void perf_trace_destroy(struct perf_event *event);
255extern int perf_trace_enable(struct perf_event *event); 255extern int perf_trace_add(struct perf_event *event, int flags);
256extern void perf_trace_disable(struct perf_event *event); 256extern void perf_trace_del(struct perf_event *event, int flags);
257extern int ftrace_profile_set_filter(struct perf_event *event, int event_id, 257extern int ftrace_profile_set_filter(struct perf_event *event, int event_id,
258 char *filter_str); 258 char *filter_str);
259extern void ftrace_profile_free_filter(struct perf_event *event); 259extern void ftrace_profile_free_filter(struct perf_event *event);
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 8cafa15af60d..402073c61669 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -538,6 +538,7 @@ struct hw_perf_event {
538 }; 538 };
539#endif 539#endif
540 }; 540 };
541 int state;
541 local64_t prev_count; 542 local64_t prev_count;
542 u64 sample_period; 543 u64 sample_period;
543 u64 last_period; 544 u64 last_period;
@@ -549,6 +550,13 @@ struct hw_perf_event {
549#endif 550#endif
550}; 551};
551 552
553/*
554 * hw_perf_event::state flags
555 */
556#define PERF_HES_STOPPED 0x01 /* the counter is stopped */
557#define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */
558#define PERF_HES_ARCH 0x04
559
552struct perf_event; 560struct perf_event;
553 561
554/* 562/*
@@ -564,42 +572,62 @@ struct pmu {
564 572
565 int *pmu_disable_count; 573 int *pmu_disable_count;
566 574
575 /*
576 * Fully disable/enable this PMU, can be used to protect from the PMI
577 * as well as for lazy/batch writing of the MSRs.
578 */
567 void (*pmu_enable) (struct pmu *pmu); /* optional */ 579 void (*pmu_enable) (struct pmu *pmu); /* optional */
568 void (*pmu_disable) (struct pmu *pmu); /* optional */ 580 void (*pmu_disable) (struct pmu *pmu); /* optional */
569 581
570 /* 582 /*
583 * Try and initialize the event for this PMU.
571 * Should return -ENOENT when the @event doesn't match this PMU. 584 * Should return -ENOENT when the @event doesn't match this PMU.
572 */ 585 */
573 int (*event_init) (struct perf_event *event); 586 int (*event_init) (struct perf_event *event);
574 587
575 int (*enable) (struct perf_event *event); 588#define PERF_EF_START 0x01 /* start the counter when adding */
576 void (*disable) (struct perf_event *event); 589#define PERF_EF_RELOAD 0x02 /* reload the counter when starting */
577 int (*start) (struct perf_event *event); 590#define PERF_EF_UPDATE 0x04 /* update the counter when stopping */
578 void (*stop) (struct perf_event *event); 591
592 /*
593 * Adds/Removes a counter to/from the PMU, can be done inside
594 * a transaction, see the ->*_txn() methods.
595 */
596 int (*add) (struct perf_event *event, int flags);
597 void (*del) (struct perf_event *event, int flags);
598
599 /*
600 * Starts/Stops a counter present on the PMU. The PMI handler
601 * should stop the counter when perf_event_overflow() returns
602 * !0. ->start() will be used to continue.
603 */
604 void (*start) (struct perf_event *event, int flags);
605 void (*stop) (struct perf_event *event, int flags);
606
607 /*
608 * Updates the counter value of the event.
609 */
579 void (*read) (struct perf_event *event); 610 void (*read) (struct perf_event *event);
580 void (*unthrottle) (struct perf_event *event);
581 611
582 /* 612 /*
583 * Group events scheduling is treated as a transaction, add 613 * Group events scheduling is treated as a transaction, add
584 * group events as a whole and perform one schedulability test. 614 * group events as a whole and perform one schedulability test.
585 * If the test fails, roll back the whole group 615 * If the test fails, roll back the whole group
586 */ 616 *
587 617 * Start the transaction, after this ->add() doesn't need to
588 /*
589 * Start the transaction, after this ->enable() doesn't need to
590 * do schedulability tests. 618 * do schedulability tests.
591 */ 619 */
592 void (*start_txn) (struct pmu *pmu); /* optional */ 620 void (*start_txn) (struct pmu *pmu); /* optional */
593 /* 621 /*
594 * If ->start_txn() disabled the ->enable() schedulability test 622 * If ->start_txn() disabled the ->add() schedulability test
595 * then ->commit_txn() is required to perform one. On success 623 * then ->commit_txn() is required to perform one. On success
596 * the transaction is closed. On error the transaction is kept 624 * the transaction is closed. On error the transaction is kept
597 * open until ->cancel_txn() is called. 625 * open until ->cancel_txn() is called.
598 */ 626 */
599 int (*commit_txn) (struct pmu *pmu); /* optional */ 627 int (*commit_txn) (struct pmu *pmu); /* optional */
600 /* 628 /*
601 * Will cancel the transaction, assumes ->disable() is called 629 * Will cancel the transaction, assumes ->del() is called
602 * for each successfull ->enable() during the transaction. 630 * for each successfull ->add() during the transaction.
603 */ 631 */
604 void (*cancel_txn) (struct pmu *pmu); /* optional */ 632 void (*cancel_txn) (struct pmu *pmu); /* optional */
605}; 633};
@@ -680,7 +708,7 @@ struct perf_event {
680 int nr_siblings; 708 int nr_siblings;
681 int group_flags; 709 int group_flags;
682 struct perf_event *group_leader; 710 struct perf_event *group_leader;
683 struct pmu *pmu; 711 struct pmu *pmu;
684 712
685 enum perf_event_active_state state; 713 enum perf_event_active_state state;
686 unsigned int attach_state; 714 unsigned int attach_state;
diff --git a/kernel/hw_breakpoint.c b/kernel/hw_breakpoint.c
index e9c5cfa1fd20..6f150095cafe 100644
--- a/kernel/hw_breakpoint.c
+++ b/kernel/hw_breakpoint.c
@@ -586,10 +586,35 @@ static int hw_breakpoint_event_init(struct perf_event *bp)
586 return 0; 586 return 0;
587} 587}
588 588
589static int hw_breakpoint_add(struct perf_event *bp, int flags)
590{
591 if (!(flags & PERF_EF_START))
592 bp->hw.state = PERF_HES_STOPPED;
593
594 return arch_install_hw_breakpoint(bp);
595}
596
597static void hw_breakpoint_del(struct perf_event *bp, int flags)
598{
599 arch_uninstall_hw_breakpoint(bp);
600}
601
602static void hw_breakpoint_start(struct perf_event *bp, int flags)
603{
604 bp->hw.state = 0;
605}
606
607static void hw_breakpoint_stop(struct perf_event *bp, int flags)
608{
609 bp->hw.state = PERF_HES_STOPPED;
610}
611
589static struct pmu perf_breakpoint = { 612static struct pmu perf_breakpoint = {
590 .event_init = hw_breakpoint_event_init, 613 .event_init = hw_breakpoint_event_init,
591 .enable = arch_install_hw_breakpoint, 614 .add = hw_breakpoint_add,
592 .disable = arch_uninstall_hw_breakpoint, 615 .del = hw_breakpoint_del,
616 .start = hw_breakpoint_start,
617 .stop = hw_breakpoint_stop,
593 .read = hw_breakpoint_pmu_read, 618 .read = hw_breakpoint_pmu_read,
594}; 619};
595 620
diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index 1a6cdbf0d091..3bace4fd0355 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -424,7 +424,7 @@ event_sched_out(struct perf_event *event,
424 event->state = PERF_EVENT_STATE_OFF; 424 event->state = PERF_EVENT_STATE_OFF;
425 } 425 }
426 event->tstamp_stopped = ctx->time; 426 event->tstamp_stopped = ctx->time;
427 event->pmu->disable(event); 427 event->pmu->del(event, 0);
428 event->oncpu = -1; 428 event->oncpu = -1;
429 429
430 if (!is_software_event(event)) 430 if (!is_software_event(event))
@@ -649,7 +649,7 @@ event_sched_in(struct perf_event *event,
649 */ 649 */
650 smp_wmb(); 650 smp_wmb();
651 651
652 if (event->pmu->enable(event)) { 652 if (event->pmu->add(event, PERF_EF_START)) {
653 event->state = PERF_EVENT_STATE_INACTIVE; 653 event->state = PERF_EVENT_STATE_INACTIVE;
654 event->oncpu = -1; 654 event->oncpu = -1;
655 return -EAGAIN; 655 return -EAGAIN;
@@ -1482,22 +1482,6 @@ do { \
1482 return div64_u64(dividend, divisor); 1482 return div64_u64(dividend, divisor);
1483} 1483}
1484 1484
1485static void perf_event_stop(struct perf_event *event)
1486{
1487 if (!event->pmu->stop)
1488 return event->pmu->disable(event);
1489
1490 return event->pmu->stop(event);
1491}
1492
1493static int perf_event_start(struct perf_event *event)
1494{
1495 if (!event->pmu->start)
1496 return event->pmu->enable(event);
1497
1498 return event->pmu->start(event);
1499}
1500
1501static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count) 1485static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
1502{ 1486{
1503 struct hw_perf_event *hwc = &event->hw; 1487 struct hw_perf_event *hwc = &event->hw;
@@ -1517,9 +1501,9 @@ static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
1517 hwc->sample_period = sample_period; 1501 hwc->sample_period = sample_period;
1518 1502
1519 if (local64_read(&hwc->period_left) > 8*sample_period) { 1503 if (local64_read(&hwc->period_left) > 8*sample_period) {
1520 perf_event_stop(event); 1504 event->pmu->stop(event, PERF_EF_UPDATE);
1521 local64_set(&hwc->period_left, 0); 1505 local64_set(&hwc->period_left, 0);
1522 perf_event_start(event); 1506 event->pmu->start(event, PERF_EF_RELOAD);
1523 } 1507 }
1524} 1508}
1525 1509
@@ -1548,7 +1532,7 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1548 */ 1532 */
1549 if (interrupts == MAX_INTERRUPTS) { 1533 if (interrupts == MAX_INTERRUPTS) {
1550 perf_log_throttle(event, 1); 1534 perf_log_throttle(event, 1);
1551 event->pmu->unthrottle(event); 1535 event->pmu->start(event, 0);
1552 } 1536 }
1553 1537
1554 if (!event->attr.freq || !event->attr.sample_freq) 1538 if (!event->attr.freq || !event->attr.sample_freq)
@@ -2506,6 +2490,9 @@ int perf_event_task_disable(void)
2506 2490
2507static int perf_event_index(struct perf_event *event) 2491static int perf_event_index(struct perf_event *event)
2508{ 2492{
2493 if (event->hw.state & PERF_HES_STOPPED)
2494 return 0;
2495
2509 if (event->state != PERF_EVENT_STATE_ACTIVE) 2496 if (event->state != PERF_EVENT_STATE_ACTIVE)
2510 return 0; 2497 return 0;
2511 2498
@@ -4120,8 +4107,6 @@ static int __perf_event_overflow(struct perf_event *event, int nmi,
4120 struct hw_perf_event *hwc = &event->hw; 4107 struct hw_perf_event *hwc = &event->hw;
4121 int ret = 0; 4108 int ret = 0;
4122 4109
4123 throttle = (throttle && event->pmu->unthrottle != NULL);
4124
4125 if (!throttle) { 4110 if (!throttle) {
4126 hwc->interrupts++; 4111 hwc->interrupts++;
4127 } else { 4112 } else {
@@ -4246,7 +4231,7 @@ static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4246 } 4231 }
4247} 4232}
4248 4233
4249static void perf_swevent_add(struct perf_event *event, u64 nr, 4234static void perf_swevent_event(struct perf_event *event, u64 nr,
4250 int nmi, struct perf_sample_data *data, 4235 int nmi, struct perf_sample_data *data,
4251 struct pt_regs *regs) 4236 struct pt_regs *regs)
4252{ 4237{
@@ -4272,6 +4257,9 @@ static void perf_swevent_add(struct perf_event *event, u64 nr,
4272static int perf_exclude_event(struct perf_event *event, 4257static int perf_exclude_event(struct perf_event *event,
4273 struct pt_regs *regs) 4258 struct pt_regs *regs)
4274{ 4259{
4260 if (event->hw.state & PERF_HES_STOPPED)
4261 return 0;
4262
4275 if (regs) { 4263 if (regs) {
4276 if (event->attr.exclude_user && user_mode(regs)) 4264 if (event->attr.exclude_user && user_mode(regs))
4277 return 1; 4265 return 1;
@@ -4371,7 +4359,7 @@ static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4371 4359
4372 hlist_for_each_entry_rcu(event, node, head, hlist_entry) { 4360 hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4373 if (perf_swevent_match(event, type, event_id, data, regs)) 4361 if (perf_swevent_match(event, type, event_id, data, regs))
4374 perf_swevent_add(event, nr, nmi, data, regs); 4362 perf_swevent_event(event, nr, nmi, data, regs);
4375 } 4363 }
4376end: 4364end:
4377 rcu_read_unlock(); 4365 rcu_read_unlock();
@@ -4415,7 +4403,7 @@ static void perf_swevent_read(struct perf_event *event)
4415{ 4403{
4416} 4404}
4417 4405
4418static int perf_swevent_enable(struct perf_event *event) 4406static int perf_swevent_add(struct perf_event *event, int flags)
4419{ 4407{
4420 struct hw_perf_event *hwc = &event->hw; 4408 struct hw_perf_event *hwc = &event->hw;
4421 struct perf_cpu_context *cpuctx; 4409 struct perf_cpu_context *cpuctx;
@@ -4428,6 +4416,8 @@ static int perf_swevent_enable(struct perf_event *event)
4428 perf_swevent_set_period(event); 4416 perf_swevent_set_period(event);
4429 } 4417 }
4430 4418
4419 hwc->state = !(flags & PERF_EF_START);
4420
4431 head = find_swevent_head(cpuctx, event); 4421 head = find_swevent_head(cpuctx, event);
4432 if (WARN_ON_ONCE(!head)) 4422 if (WARN_ON_ONCE(!head))
4433 return -EINVAL; 4423 return -EINVAL;
@@ -4437,18 +4427,19 @@ static int perf_swevent_enable(struct perf_event *event)
4437 return 0; 4427 return 0;
4438} 4428}
4439 4429
4440static void perf_swevent_disable(struct perf_event *event) 4430static void perf_swevent_del(struct perf_event *event, int flags)
4441{ 4431{
4442 hlist_del_rcu(&event->hlist_entry); 4432 hlist_del_rcu(&event->hlist_entry);
4443} 4433}
4444 4434
4445static void perf_swevent_void(struct perf_event *event) 4435static void perf_swevent_start(struct perf_event *event, int flags)
4446{ 4436{
4437 event->hw.state = 0;
4447} 4438}
4448 4439
4449static int perf_swevent_int(struct perf_event *event) 4440static void perf_swevent_stop(struct perf_event *event, int flags)
4450{ 4441{
4451 return 0; 4442 event->hw.state = PERF_HES_STOPPED;
4452} 4443}
4453 4444
4454/* Deref the hlist from the update side */ 4445/* Deref the hlist from the update side */
@@ -4604,12 +4595,11 @@ static int perf_swevent_init(struct perf_event *event)
4604 4595
4605static struct pmu perf_swevent = { 4596static struct pmu perf_swevent = {
4606 .event_init = perf_swevent_init, 4597 .event_init = perf_swevent_init,
4607 .enable = perf_swevent_enable, 4598 .add = perf_swevent_add,
4608 .disable = perf_swevent_disable, 4599 .del = perf_swevent_del,
4609 .start = perf_swevent_int, 4600 .start = perf_swevent_start,
4610 .stop = perf_swevent_void, 4601 .stop = perf_swevent_stop,
4611 .read = perf_swevent_read, 4602 .read = perf_swevent_read,
4612 .unthrottle = perf_swevent_void, /* hwc->interrupts already reset */
4613}; 4603};
4614 4604
4615#ifdef CONFIG_EVENT_TRACING 4605#ifdef CONFIG_EVENT_TRACING
@@ -4657,7 +4647,7 @@ void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
4657 4647
4658 hlist_for_each_entry_rcu(event, node, head, hlist_entry) { 4648 hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4659 if (perf_tp_event_match(event, &data, regs)) 4649 if (perf_tp_event_match(event, &data, regs))
4660 perf_swevent_add(event, count, 1, &data, regs); 4650 perf_swevent_event(event, count, 1, &data, regs);
4661 } 4651 }
4662 4652
4663 perf_swevent_put_recursion_context(rctx); 4653 perf_swevent_put_recursion_context(rctx);
@@ -4696,12 +4686,11 @@ static int perf_tp_event_init(struct perf_event *event)
4696 4686
4697static struct pmu perf_tracepoint = { 4687static struct pmu perf_tracepoint = {
4698 .event_init = perf_tp_event_init, 4688 .event_init = perf_tp_event_init,
4699 .enable = perf_trace_enable, 4689 .add = perf_trace_add,
4700 .disable = perf_trace_disable, 4690 .del = perf_trace_del,
4701 .start = perf_swevent_int, 4691 .start = perf_swevent_start,
4702 .stop = perf_swevent_void, 4692 .stop = perf_swevent_stop,
4703 .read = perf_swevent_read, 4693 .read = perf_swevent_read,
4704 .unthrottle = perf_swevent_void,
4705}; 4694};
4706 4695
4707static inline void perf_tp_register(void) 4696static inline void perf_tp_register(void)
@@ -4757,8 +4746,8 @@ void perf_bp_event(struct perf_event *bp, void *data)
4757 4746
4758 perf_sample_data_init(&sample, bp->attr.bp_addr); 4747 perf_sample_data_init(&sample, bp->attr.bp_addr);
4759 4748
4760 if (!perf_exclude_event(bp, regs)) 4749 if (!bp->hw.state && !perf_exclude_event(bp, regs))
4761 perf_swevent_add(bp, 1, 1, &sample, regs); 4750 perf_swevent_event(bp, 1, 1, &sample, regs);
4762} 4751}
4763#endif 4752#endif
4764 4753
@@ -4834,32 +4823,39 @@ static void perf_swevent_cancel_hrtimer(struct perf_event *event)
4834 4823
4835static void cpu_clock_event_update(struct perf_event *event) 4824static void cpu_clock_event_update(struct perf_event *event)
4836{ 4825{
4837 int cpu = raw_smp_processor_id();
4838 s64 prev; 4826 s64 prev;
4839 u64 now; 4827 u64 now;
4840 4828
4841 now = cpu_clock(cpu); 4829 now = local_clock();
4842 prev = local64_xchg(&event->hw.prev_count, now); 4830 prev = local64_xchg(&event->hw.prev_count, now);
4843 local64_add(now - prev, &event->count); 4831 local64_add(now - prev, &event->count);
4844} 4832}
4845 4833
4846static int cpu_clock_event_enable(struct perf_event *event) 4834static void cpu_clock_event_start(struct perf_event *event, int flags)
4847{ 4835{
4848 struct hw_perf_event *hwc = &event->hw; 4836 local64_set(&event->hw.prev_count, local_clock());
4849 int cpu = raw_smp_processor_id();
4850
4851 local64_set(&hwc->prev_count, cpu_clock(cpu));
4852 perf_swevent_start_hrtimer(event); 4837 perf_swevent_start_hrtimer(event);
4853
4854 return 0;
4855} 4838}
4856 4839
4857static void cpu_clock_event_disable(struct perf_event *event) 4840static void cpu_clock_event_stop(struct perf_event *event, int flags)
4858{ 4841{
4859 perf_swevent_cancel_hrtimer(event); 4842 perf_swevent_cancel_hrtimer(event);
4860 cpu_clock_event_update(event); 4843 cpu_clock_event_update(event);
4861} 4844}
4862 4845
4846static int cpu_clock_event_add(struct perf_event *event, int flags)
4847{
4848 if (flags & PERF_EF_START)
4849 cpu_clock_event_start(event, flags);
4850
4851 return 0;
4852}
4853
4854static void cpu_clock_event_del(struct perf_event *event, int flags)
4855{
4856 cpu_clock_event_stop(event, flags);
4857}
4858
4863static void cpu_clock_event_read(struct perf_event *event) 4859static void cpu_clock_event_read(struct perf_event *event)
4864{ 4860{
4865 cpu_clock_event_update(event); 4861 cpu_clock_event_update(event);
@@ -4878,8 +4874,10 @@ static int cpu_clock_event_init(struct perf_event *event)
4878 4874
4879static struct pmu perf_cpu_clock = { 4875static struct pmu perf_cpu_clock = {
4880 .event_init = cpu_clock_event_init, 4876 .event_init = cpu_clock_event_init,
4881 .enable = cpu_clock_event_enable, 4877 .add = cpu_clock_event_add,
4882 .disable = cpu_clock_event_disable, 4878 .del = cpu_clock_event_del,
4879 .start = cpu_clock_event_start,
4880 .stop = cpu_clock_event_stop,
4883 .read = cpu_clock_event_read, 4881 .read = cpu_clock_event_read,
4884}; 4882};
4885 4883
@@ -4897,25 +4895,29 @@ static void task_clock_event_update(struct perf_event *event, u64 now)
4897 local64_add(delta, &event->count); 4895 local64_add(delta, &event->count);
4898} 4896}
4899 4897
4900static int task_clock_event_enable(struct perf_event *event) 4898static void task_clock_event_start(struct perf_event *event, int flags)
4901{ 4899{
4902 struct hw_perf_event *hwc = &event->hw; 4900 local64_set(&event->hw.prev_count, event->ctx->time);
4903 u64 now;
4904
4905 now = event->ctx->time;
4906
4907 local64_set(&hwc->prev_count, now);
4908
4909 perf_swevent_start_hrtimer(event); 4901 perf_swevent_start_hrtimer(event);
4910
4911 return 0;
4912} 4902}
4913 4903
4914static void task_clock_event_disable(struct perf_event *event) 4904static void task_clock_event_stop(struct perf_event *event, int flags)
4915{ 4905{
4916 perf_swevent_cancel_hrtimer(event); 4906 perf_swevent_cancel_hrtimer(event);
4917 task_clock_event_update(event, event->ctx->time); 4907 task_clock_event_update(event, event->ctx->time);
4908}
4909
4910static int task_clock_event_add(struct perf_event *event, int flags)
4911{
4912 if (flags & PERF_EF_START)
4913 task_clock_event_start(event, flags);
4918 4914
4915 return 0;
4916}
4917
4918static void task_clock_event_del(struct perf_event *event, int flags)
4919{
4920 task_clock_event_stop(event, PERF_EF_UPDATE);
4919} 4921}
4920 4922
4921static void task_clock_event_read(struct perf_event *event) 4923static void task_clock_event_read(struct perf_event *event)
@@ -4947,8 +4949,10 @@ static int task_clock_event_init(struct perf_event *event)
4947 4949
4948static struct pmu perf_task_clock = { 4950static struct pmu perf_task_clock = {
4949 .event_init = task_clock_event_init, 4951 .event_init = task_clock_event_init,
4950 .enable = task_clock_event_enable, 4952 .add = task_clock_event_add,
4951 .disable = task_clock_event_disable, 4953 .del = task_clock_event_del,
4954 .start = task_clock_event_start,
4955 .stop = task_clock_event_stop,
4952 .read = task_clock_event_read, 4956 .read = task_clock_event_read,
4953}; 4957};
4954 4958
diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c
index f3bbcd1c90c8..39c059ca670e 100644
--- a/kernel/trace/trace_event_perf.c
+++ b/kernel/trace/trace_event_perf.c
@@ -101,7 +101,7 @@ int perf_trace_init(struct perf_event *p_event)
101 return ret; 101 return ret;
102} 102}
103 103
104int perf_trace_enable(struct perf_event *p_event) 104int perf_trace_add(struct perf_event *p_event, int flags)
105{ 105{
106 struct ftrace_event_call *tp_event = p_event->tp_event; 106 struct ftrace_event_call *tp_event = p_event->tp_event;
107 struct hlist_head __percpu *pcpu_list; 107 struct hlist_head __percpu *pcpu_list;
@@ -111,13 +111,16 @@ int perf_trace_enable(struct perf_event *p_event)
111 if (WARN_ON_ONCE(!pcpu_list)) 111 if (WARN_ON_ONCE(!pcpu_list))
112 return -EINVAL; 112 return -EINVAL;
113 113
114 if (!(flags & PERF_EF_START))
115 p_event->hw.state = PERF_HES_STOPPED;
116
114 list = this_cpu_ptr(pcpu_list); 117 list = this_cpu_ptr(pcpu_list);
115 hlist_add_head_rcu(&p_event->hlist_entry, list); 118 hlist_add_head_rcu(&p_event->hlist_entry, list);
116 119
117 return 0; 120 return 0;
118} 121}
119 122
120void perf_trace_disable(struct perf_event *p_event) 123void perf_trace_del(struct perf_event *p_event, int flags)
121{ 124{
122 hlist_del_rcu(&p_event->hlist_entry); 125 hlist_del_rcu(&p_event->hlist_entry);
123} 126}
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-25 09:16:55 -0500