aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux/perf_event.h
diff options
context:
space:
mode:
Diffstat (limited to 'include/linux/perf_event.h')
-rw-r--r--include/linux/perf_event.h222
1 files changed, 148 insertions, 74 deletions
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 716f99b682c1..40150f345982 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -486,6 +486,8 @@ struct perf_guest_info_callbacks {
486#include <linux/workqueue.h> 486#include <linux/workqueue.h>
487#include <linux/ftrace.h> 487#include <linux/ftrace.h>
488#include <linux/cpu.h> 488#include <linux/cpu.h>
489#include <linux/irq_work.h>
490#include <linux/jump_label_ref.h>
489#include <asm/atomic.h> 491#include <asm/atomic.h>
490#include <asm/local.h> 492#include <asm/local.h>
491 493
@@ -529,16 +531,22 @@ struct hw_perf_event {
529 int last_cpu; 531 int last_cpu;
530 }; 532 };
531 struct { /* software */ 533 struct { /* software */
532 s64 remaining;
533 struct hrtimer hrtimer; 534 struct hrtimer hrtimer;
534 }; 535 };
535#ifdef CONFIG_HAVE_HW_BREAKPOINT 536#ifdef CONFIG_HAVE_HW_BREAKPOINT
536 struct { /* breakpoint */ 537 struct { /* breakpoint */
537 struct arch_hw_breakpoint info; 538 struct arch_hw_breakpoint info;
538 struct list_head bp_list; 539 struct list_head bp_list;
540 /*
541 * Crufty hack to avoid the chicken and egg
542 * problem hw_breakpoint has with context
543 * creation and event initalization.
544 */
545 struct task_struct *bp_target;
539 }; 546 };
540#endif 547#endif
541 }; 548 };
549 int state;
542 local64_t prev_count; 550 local64_t prev_count;
543 u64 sample_period; 551 u64 sample_period;
544 u64 last_period; 552 u64 last_period;
@@ -550,6 +558,13 @@ struct hw_perf_event {
550#endif 558#endif
551}; 559};
552 560
561/*
562 * hw_perf_event::state flags
563 */
564#define PERF_HES_STOPPED 0x01 /* the counter is stopped */
565#define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */
566#define PERF_HES_ARCH 0x04
567
553struct perf_event; 568struct perf_event;
554 569
555/* 570/*
@@ -561,36 +576,70 @@ struct perf_event;
561 * struct pmu - generic performance monitoring unit 576 * struct pmu - generic performance monitoring unit
562 */ 577 */
563struct pmu { 578struct pmu {
564 int (*enable) (struct perf_event *event); 579 struct list_head entry;
565 void (*disable) (struct perf_event *event); 580
566 int (*start) (struct perf_event *event); 581 int * __percpu pmu_disable_count;
567 void (*stop) (struct perf_event *event); 582 struct perf_cpu_context * __percpu pmu_cpu_context;
568 void (*read) (struct perf_event *event); 583 int task_ctx_nr;
569 void (*unthrottle) (struct perf_event *event);
570 584
571 /* 585 /*
572 * Group events scheduling is treated as a transaction, add group 586 * Fully disable/enable this PMU, can be used to protect from the PMI
573 * events as a whole and perform one schedulability test. If the test 587 * as well as for lazy/batch writing of the MSRs.
574 * fails, roll back the whole group
575 */ 588 */
589 void (*pmu_enable) (struct pmu *pmu); /* optional */
590 void (*pmu_disable) (struct pmu *pmu); /* optional */
576 591
577 /* 592 /*
578 * Start the transaction, after this ->enable() doesn't need 593 * Try and initialize the event for this PMU.
579 * to do schedulability tests. 594 * Should return -ENOENT when the @event doesn't match this PMU.
580 */ 595 */
581 void (*start_txn) (const struct pmu *pmu); 596 int (*event_init) (struct perf_event *event);
597
598#define PERF_EF_START 0x01 /* start the counter when adding */
599#define PERF_EF_RELOAD 0x02 /* reload the counter when starting */
600#define PERF_EF_UPDATE 0x04 /* update the counter when stopping */
601
582 /* 602 /*
583 * If ->start_txn() disabled the ->enable() schedulability test 603 * Adds/Removes a counter to/from the PMU, can be done inside
604 * a transaction, see the ->*_txn() methods.
605 */
606 int (*add) (struct perf_event *event, int flags);
607 void (*del) (struct perf_event *event, int flags);
608
609 /*
610 * Starts/Stops a counter present on the PMU. The PMI handler
611 * should stop the counter when perf_event_overflow() returns
612 * !0. ->start() will be used to continue.
613 */
614 void (*start) (struct perf_event *event, int flags);
615 void (*stop) (struct perf_event *event, int flags);
616
617 /*
618 * Updates the counter value of the event.
619 */
620 void (*read) (struct perf_event *event);
621
622 /*
623 * Group events scheduling is treated as a transaction, add
624 * group events as a whole and perform one schedulability test.
625 * If the test fails, roll back the whole group
626 *
627 * Start the transaction, after this ->add() doesn't need to
628 * do schedulability tests.
629 */
630 void (*start_txn) (struct pmu *pmu); /* optional */
631 /*
632 * If ->start_txn() disabled the ->add() schedulability test
584 * then ->commit_txn() is required to perform one. On success 633 * then ->commit_txn() is required to perform one. On success
585 * the transaction is closed. On error the transaction is kept 634 * the transaction is closed. On error the transaction is kept
586 * open until ->cancel_txn() is called. 635 * open until ->cancel_txn() is called.
587 */ 636 */
588 int (*commit_txn) (const struct pmu *pmu); 637 int (*commit_txn) (struct pmu *pmu); /* optional */
589 /* 638 /*
590 * Will cancel the transaction, assumes ->disable() is called for 639 * Will cancel the transaction, assumes ->del() is called
591 * each successfull ->enable() during the transaction. 640 * for each successfull ->add() during the transaction.
592 */ 641 */
593 void (*cancel_txn) (const struct pmu *pmu); 642 void (*cancel_txn) (struct pmu *pmu); /* optional */
594}; 643};
595 644
596/** 645/**
@@ -631,11 +680,6 @@ struct perf_buffer {
631 void *data_pages[0]; 680 void *data_pages[0];
632}; 681};
633 682
634struct perf_pending_entry {
635 struct perf_pending_entry *next;
636 void (*func)(struct perf_pending_entry *);
637};
638
639struct perf_sample_data; 683struct perf_sample_data;
640 684
641typedef void (*perf_overflow_handler_t)(struct perf_event *, int, 685typedef void (*perf_overflow_handler_t)(struct perf_event *, int,
@@ -656,6 +700,7 @@ struct swevent_hlist {
656 700
657#define PERF_ATTACH_CONTEXT 0x01 701#define PERF_ATTACH_CONTEXT 0x01
658#define PERF_ATTACH_GROUP 0x02 702#define PERF_ATTACH_GROUP 0x02
703#define PERF_ATTACH_TASK 0x04
659 704
660/** 705/**
661 * struct perf_event - performance event kernel representation: 706 * struct perf_event - performance event kernel representation:
@@ -669,7 +714,7 @@ struct perf_event {
669 int nr_siblings; 714 int nr_siblings;
670 int group_flags; 715 int group_flags;
671 struct perf_event *group_leader; 716 struct perf_event *group_leader;
672 const struct pmu *pmu; 717 struct pmu *pmu;
673 718
674 enum perf_event_active_state state; 719 enum perf_event_active_state state;
675 unsigned int attach_state; 720 unsigned int attach_state;
@@ -702,6 +747,16 @@ struct perf_event {
702 u64 tstamp_running; 747 u64 tstamp_running;
703 u64 tstamp_stopped; 748 u64 tstamp_stopped;
704 749
750 /*
751 * timestamp shadows the actual context timing but it can
752 * be safely used in NMI interrupt context. It reflects the
753 * context time as it was when the event was last scheduled in.
754 *
755 * ctx_time already accounts for ctx->timestamp. Therefore to
756 * compute ctx_time for a sample, simply add perf_clock().
757 */
758 u64 shadow_ctx_time;
759
705 struct perf_event_attr attr; 760 struct perf_event_attr attr;
706 struct hw_perf_event hw; 761 struct hw_perf_event hw;
707 762
@@ -743,7 +798,7 @@ struct perf_event {
743 int pending_wakeup; 798 int pending_wakeup;
744 int pending_kill; 799 int pending_kill;
745 int pending_disable; 800 int pending_disable;
746 struct perf_pending_entry pending; 801 struct irq_work pending;
747 802
748 atomic_t event_limit; 803 atomic_t event_limit;
749 804
@@ -763,12 +818,19 @@ struct perf_event {
763#endif /* CONFIG_PERF_EVENTS */ 818#endif /* CONFIG_PERF_EVENTS */
764}; 819};
765 820
821enum perf_event_context_type {
822 task_context,
823 cpu_context,
824};
825
766/** 826/**
767 * struct perf_event_context - event context structure 827 * struct perf_event_context - event context structure
768 * 828 *
769 * Used as a container for task events and CPU events as well: 829 * Used as a container for task events and CPU events as well:
770 */ 830 */
771struct perf_event_context { 831struct perf_event_context {
832 enum perf_event_context_type type;
833 struct pmu *pmu;
772 /* 834 /*
773 * Protect the states of the events in the list, 835 * Protect the states of the events in the list,
774 * nr_active, and the list: 836 * nr_active, and the list:
@@ -808,6 +870,12 @@ struct perf_event_context {
808 struct rcu_head rcu_head; 870 struct rcu_head rcu_head;
809}; 871};
810 872
873/*
874 * Number of contexts where an event can trigger:
875 * task, softirq, hardirq, nmi.
876 */
877#define PERF_NR_CONTEXTS 4
878
811/** 879/**
812 * struct perf_event_cpu_context - per cpu event context structure 880 * struct perf_event_cpu_context - per cpu event context structure
813 */ 881 */
@@ -815,18 +883,9 @@ struct perf_cpu_context {
815 struct perf_event_context ctx; 883 struct perf_event_context ctx;
816 struct perf_event_context *task_ctx; 884 struct perf_event_context *task_ctx;
817 int active_oncpu; 885 int active_oncpu;
818 int max_pertask;
819 int exclusive; 886 int exclusive;
820 struct swevent_hlist *swevent_hlist; 887 struct list_head rotation_list;
821 struct mutex hlist_mutex; 888 int jiffies_interval;
822 int hlist_refcount;
823
824 /*
825 * Recursion avoidance:
826 *
827 * task, softirq, irq, nmi context
828 */
829 int recursion[4];
830}; 889};
831 890
832struct perf_output_handle { 891struct perf_output_handle {
@@ -842,26 +901,34 @@ struct perf_output_handle {
842 901
843#ifdef CONFIG_PERF_EVENTS 902#ifdef CONFIG_PERF_EVENTS
844 903
845/* 904extern int perf_pmu_register(struct pmu *pmu);
846 * Set by architecture code: 905extern void perf_pmu_unregister(struct pmu *pmu);
847 */ 906
848extern int perf_max_events; 907extern int perf_num_counters(void);
908extern const char *perf_pmu_name(void);
909extern void __perf_event_task_sched_in(struct task_struct *task);
910extern void __perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
849 911
850extern const struct pmu *hw_perf_event_init(struct perf_event *event); 912extern atomic_t perf_task_events;
913
914static inline void perf_event_task_sched_in(struct task_struct *task)
915{
916 COND_STMT(&perf_task_events, __perf_event_task_sched_in(task));
917}
918
919static inline
920void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
921{
922 COND_STMT(&perf_task_events, __perf_event_task_sched_out(task, next));
923}
851 924
852extern void perf_event_task_sched_in(struct task_struct *task);
853extern void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next);
854extern void perf_event_task_tick(struct task_struct *task);
855extern int perf_event_init_task(struct task_struct *child); 925extern int perf_event_init_task(struct task_struct *child);
856extern void perf_event_exit_task(struct task_struct *child); 926extern void perf_event_exit_task(struct task_struct *child);
857extern void perf_event_free_task(struct task_struct *task); 927extern void perf_event_free_task(struct task_struct *task);
858extern void set_perf_event_pending(void); 928extern void perf_event_delayed_put(struct task_struct *task);
859extern void perf_event_do_pending(void);
860extern void perf_event_print_debug(void); 929extern void perf_event_print_debug(void);
861extern void __perf_disable(void); 930extern void perf_pmu_disable(struct pmu *pmu);
862extern bool __perf_enable(void); 931extern void perf_pmu_enable(struct pmu *pmu);
863extern void perf_disable(void);
864extern void perf_enable(void);
865extern int perf_event_task_disable(void); 932extern int perf_event_task_disable(void);
866extern int perf_event_task_enable(void); 933extern int perf_event_task_enable(void);
867extern void perf_event_update_userpage(struct perf_event *event); 934extern void perf_event_update_userpage(struct perf_event *event);
@@ -869,7 +936,7 @@ extern int perf_event_release_kernel(struct perf_event *event);
869extern struct perf_event * 936extern struct perf_event *
870perf_event_create_kernel_counter(struct perf_event_attr *attr, 937perf_event_create_kernel_counter(struct perf_event_attr *attr,
871 int cpu, 938 int cpu,
872 pid_t pid, 939 struct task_struct *task,
873 perf_overflow_handler_t callback); 940 perf_overflow_handler_t callback);
874extern u64 perf_event_read_value(struct perf_event *event, 941extern u64 perf_event_read_value(struct perf_event *event,
875 u64 *enabled, u64 *running); 942 u64 *enabled, u64 *running);
@@ -920,14 +987,7 @@ extern int perf_event_overflow(struct perf_event *event, int nmi,
920 */ 987 */
921static inline int is_software_event(struct perf_event *event) 988static inline int is_software_event(struct perf_event *event)
922{ 989{
923 switch (event->attr.type) { 990 return event->pmu->task_ctx_nr == perf_sw_context;
924 case PERF_TYPE_SOFTWARE:
925 case PERF_TYPE_TRACEPOINT:
926 /* for now the breakpoint stuff also works as software event */
927 case PERF_TYPE_BREAKPOINT:
928 return 1;
929 }
930 return 0;
931} 991}
932 992
933extern atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX]; 993extern atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
@@ -954,18 +1014,20 @@ static inline void perf_fetch_caller_regs(struct pt_regs *regs)
954 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0); 1014 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
955} 1015}
956 1016
957static inline void 1017static __always_inline void
958perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr) 1018perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
959{ 1019{
960 if (atomic_read(&perf_swevent_enabled[event_id])) { 1020 struct pt_regs hot_regs;
961 struct pt_regs hot_regs; 1021
962 1022 JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);
963 if (!regs) { 1023 return;
964 perf_fetch_caller_regs(&hot_regs); 1024
965 regs = &hot_regs; 1025have_event:
966 } 1026 if (!regs) {
967 __perf_sw_event(event_id, nr, nmi, regs, addr); 1027 perf_fetch_caller_regs(&hot_regs);
1028 regs = &hot_regs;
968 } 1029 }
1030 __perf_sw_event(event_id, nr, nmi, regs, addr);
969} 1031}
970 1032
971extern void perf_event_mmap(struct vm_area_struct *vma); 1033extern void perf_event_mmap(struct vm_area_struct *vma);
@@ -976,7 +1038,21 @@ extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks
976extern void perf_event_comm(struct task_struct *tsk); 1038extern void perf_event_comm(struct task_struct *tsk);
977extern void perf_event_fork(struct task_struct *tsk); 1039extern void perf_event_fork(struct task_struct *tsk);
978 1040
979extern struct perf_callchain_entry *perf_callchain(struct pt_regs *regs); 1041/* Callchains */
1042DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
1043
1044extern void perf_callchain_user(struct perf_callchain_entry *entry,
1045 struct pt_regs *regs);
1046extern void perf_callchain_kernel(struct perf_callchain_entry *entry,
1047 struct pt_regs *regs);
1048
1049
1050static inline void
1051perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
1052{
1053 if (entry->nr < PERF_MAX_STACK_DEPTH)
1054 entry->ip[entry->nr++] = ip;
1055}
980 1056
981extern int sysctl_perf_event_paranoid; 1057extern int sysctl_perf_event_paranoid;
982extern int sysctl_perf_event_mlock; 1058extern int sysctl_perf_event_mlock;
@@ -1019,21 +1095,18 @@ extern int perf_swevent_get_recursion_context(void);
1019extern void perf_swevent_put_recursion_context(int rctx); 1095extern void perf_swevent_put_recursion_context(int rctx);
1020extern void perf_event_enable(struct perf_event *event); 1096extern void perf_event_enable(struct perf_event *event);
1021extern void perf_event_disable(struct perf_event *event); 1097extern void perf_event_disable(struct perf_event *event);
1098extern void perf_event_task_tick(void);
1022#else 1099#else
1023static inline void 1100static inline void
1024perf_event_task_sched_in(struct task_struct *task) { } 1101perf_event_task_sched_in(struct task_struct *task) { }
1025static inline void 1102static inline void
1026perf_event_task_sched_out(struct task_struct *task, 1103perf_event_task_sched_out(struct task_struct *task,
1027 struct task_struct *next) { } 1104 struct task_struct *next) { }
1028static inline void
1029perf_event_task_tick(struct task_struct *task) { }
1030static inline int perf_event_init_task(struct task_struct *child) { return 0; } 1105static inline int perf_event_init_task(struct task_struct *child) { return 0; }
1031static inline void perf_event_exit_task(struct task_struct *child) { } 1106static inline void perf_event_exit_task(struct task_struct *child) { }
1032static inline void perf_event_free_task(struct task_struct *task) { } 1107static inline void perf_event_free_task(struct task_struct *task) { }
1033static inline void perf_event_do_pending(void) { } 1108static inline void perf_event_delayed_put(struct task_struct *task) { }
1034static inline void perf_event_print_debug(void) { } 1109static inline void perf_event_print_debug(void) { }
1035static inline void perf_disable(void) { }
1036static inline void perf_enable(void) { }
1037static inline int perf_event_task_disable(void) { return -EINVAL; } 1110static inline int perf_event_task_disable(void) { return -EINVAL; }
1038static inline int perf_event_task_enable(void) { return -EINVAL; } 1111static inline int perf_event_task_enable(void) { return -EINVAL; }
1039 1112
@@ -1056,6 +1129,7 @@ static inline int perf_swevent_get_recursion_context(void) { return -1; }
1056static inline void perf_swevent_put_recursion_context(int rctx) { } 1129static inline void perf_swevent_put_recursion_context(int rctx) { }
1057static inline void perf_event_enable(struct perf_event *event) { } 1130static inline void perf_event_enable(struct perf_event *event) { }
1058static inline void perf_event_disable(struct perf_event *event) { } 1131static inline void perf_event_disable(struct perf_event *event) { }
1132static inline void perf_event_task_tick(void) { }
1059#endif 1133#endif
1060 1134
1061#define perf_output_put(handle, x) \ 1135#define perf_output_put(handle, x) \