1 files changed, 139 insertions, 62 deletions

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index c8e375440403..716f99b682c1 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -203,8 +203,20 @@ struct perf_event_attr {
                                 enable_on_exec :  1, /* next exec enables     */
                                 task           :  1, /* trace fork/exit       */
                                 watermark      :  1, /* wakeup_watermark      */
-
-                                __reserved_1   : 49;
+                                /*
+                                 * precise_ip:
+                                 *
+                                 *  0 - SAMPLE_IP can have arbitrary skid
+                                 *  1 - SAMPLE_IP must have constant skid
+                                 *  2 - SAMPLE_IP requested to have 0 skid
+                                 *  3 - SAMPLE_IP must have 0 skid
+                                 *
+                                 *  See also PERF_RECORD_MISC_EXACT_IP
+                                 */
+                                precise_ip     :  2, /* skid constraint       */
+                                mmap_data      :  1, /* non-exec mmap data    */
+
+                                __reserved_1   : 46;
 
         union {
                 __u32           wakeup_events;    /* wakeup every n events */
@@ -287,11 +299,24 @@ struct perf_event_mmap_page {
         __u64   data_tail;              /* user-space written tail */
 };
 
-#define PERF_RECORD_MISC_CPUMODE_MASK           (3 << 0)
+#define PERF_RECORD_MISC_CPUMODE_MASK           (7 << 0)
 #define PERF_RECORD_MISC_CPUMODE_UNKNOWN        (0 << 0)
 #define PERF_RECORD_MISC_KERNEL                 (1 << 0)
 #define PERF_RECORD_MISC_USER                   (2 << 0)
 #define PERF_RECORD_MISC_HYPERVISOR             (3 << 0)
+#define PERF_RECORD_MISC_GUEST_KERNEL           (4 << 0)
+#define PERF_RECORD_MISC_GUEST_USER             (5 << 0)
+
+/*
+ * Indicates that the content of PERF_SAMPLE_IP points to
+ * the actual instruction that triggered the event. See also
+ * perf_event_attr::precise_ip.
+ */
+#define PERF_RECORD_MISC_EXACT_IP               (1 << 14)
+/*
+ * Reserve the last bit to indicate some extended misc field
+ */
+#define PERF_RECORD_MISC_EXT_RESERVED           (1 << 15)
 
 struct perf_event_header {
         __u32   type;
@@ -437,8 +462,15 @@ enum perf_callchain_context {
 
 #ifdef CONFIG_PERF_EVENTS
 # include <asm/perf_event.h>
+# include <asm/local64.h>
 #endif
 
+struct perf_guest_info_callbacks {
+        int (*is_in_guest) (void);
+        int (*is_user_mode) (void);
+        unsigned long (*get_guest_ip) (void);
+};
+
 #ifdef CONFIG_HAVE_HW_BREAKPOINT
 #include <asm/hw_breakpoint.h>
 #endif
@@ -455,6 +487,7 @@ enum perf_callchain_context {
 #include <linux/ftrace.h>
 #include <linux/cpu.h>
 #include <asm/atomic.h>
+#include <asm/local.h>
 
 #define PERF_MAX_STACK_DEPTH            255
 
@@ -468,6 +501,17 @@ struct perf_raw_record {
         void                            *data;
 };
 
+struct perf_branch_entry {
+        __u64                           from;
+        __u64                           to;
+        __u64                           flags;
+};
+
+struct perf_branch_stack {
+        __u64                           nr;
+        struct perf_branch_entry        entries[0];
+};
+
 struct task_struct;
 
 /**
@@ -489,14 +533,16 @@ struct hw_perf_event {
                         struct hrtimer  hrtimer;
                 };
 #ifdef CONFIG_HAVE_HW_BREAKPOINT
-                /* breakpoint */
-                struct arch_hw_breakpoint       info;
+                struct { /* breakpoint */
+                        struct arch_hw_breakpoint       info;
+                        struct list_head                bp_list;
+                };
 #endif
         };
-        atomic64_t                      prev_count;
+        local64_t                       prev_count;
         u64                             sample_period;
         u64                             last_period;
-        atomic64_t                      period_left;
+        local64_t                       period_left;
         u64                             interrupts;
 
         u64                             freq_time_stamp;
@@ -506,6 +552,11 @@ struct hw_perf_event {
 
 struct perf_event;
 
+/*
+ * Common implementation detail of pmu::{start,commit,cancel}_txn
+ */
+#define PERF_EVENT_TXN 0x1
+
 /**
  * struct pmu - generic performance monitoring unit
  */
@@ -516,6 +567,30 @@ struct pmu {
         void (*stop)                    (struct perf_event *event);
         void (*read)                    (struct perf_event *event);
         void (*unthrottle)              (struct perf_event *event);
+
+        /*
+         * Group events scheduling is treated as a transaction, add group
+         * events as a whole and perform one schedulability test. If the test
+         * fails, roll back the whole group
+         */
+
+        /*
+         * Start the transaction, after this ->enable() doesn't need
+         * to do schedulability tests.
+         */
+        void (*start_txn)       (const struct pmu *pmu);
+        /*
+         * If ->start_txn() disabled the ->enable() schedulability test
+         * then ->commit_txn() is required to perform one. On success
+         * the transaction is closed. On error the transaction is kept
+         * open until ->cancel_txn() is called.
+         */
+        int  (*commit_txn)      (const struct pmu *pmu);
+        /*
+         * Will cancel the transaction, assumes ->disable() is called for
+         * each successfull ->enable() during the transaction.
+         */
+        void (*cancel_txn)      (const struct pmu *pmu);
 };
 
 /**
@@ -530,25 +605,25 @@ enum perf_event_active_state {
 
 struct file;
 
-struct perf_mmap_data {
+#define PERF_BUFFER_WRITABLE            0x01
+
+struct perf_buffer {
+        atomic_t                        refcount;
         struct rcu_head                 rcu_head;
 #ifdef CONFIG_PERF_USE_VMALLOC
         struct work_struct              work;
+        int                             page_order;     /* allocation order  */
 #endif
-        int                             data_order;
         int                             nr_pages;       /* nr of data pages  */
         int                             writable;       /* are we writable   */
-        int                             nr_locked;      /* nr pages mlocked  */
 
         atomic_t                        poll;           /* POLL_ for wakeups */
-        atomic_t                        events;         /* event_id limit       */
-
-        atomic_long_t                   head;           /* write position    */
-        atomic_long_t                   done_head;      /* completed head    */
 
-        atomic_t                        lock;           /* concurrent writes */
-        atomic_t                        wakeup;         /* needs a wakeup    */
-        atomic_t                        lost;           /* nr records lost   */
+        local_t                         head;           /* write position    */
+        local_t                         nest;           /* nested writers    */
+        local_t                         events;         /* event limit       */
+        local_t                         wakeup;         /* wakeup stamp      */
+        local_t                         lost;           /* nr records lost   */
 
         long                            watermark;      /* wakeup watermark  */
 
@@ -571,6 +646,17 @@ enum perf_group_flag {
         PERF_GROUP_SOFTWARE = 0x1,
 };
 
+#define SWEVENT_HLIST_BITS      8
+#define SWEVENT_HLIST_SIZE      (1 << SWEVENT_HLIST_BITS)
+
+struct swevent_hlist {
+        struct hlist_head       heads[SWEVENT_HLIST_SIZE];
+        struct rcu_head         rcu_head;
+};
+
+#define PERF_ATTACH_CONTEXT     0x01
+#define PERF_ATTACH_GROUP       0x02
+
 /**
  * struct perf_event - performance event kernel representation:
  */
@@ -579,14 +665,16 @@ struct perf_event {
         struct list_head                group_entry;
         struct list_head                event_entry;
         struct list_head                sibling_list;
+        struct hlist_node               hlist_entry;
         int                             nr_siblings;
         int                             group_flags;
         struct perf_event               *group_leader;
-        struct perf_event               *output;
         const struct pmu                *pmu;
 
         enum perf_event_active_state    state;
-        atomic64_t                      count;
+        unsigned int                    attach_state;
+        local64_t                       count;
+        atomic64_t                      child_count;
 
         /*
          * These are the total time in nanoseconds that the event
@@ -643,7 +731,9 @@ struct perf_event {
         /* mmap bits */
         struct mutex                    mmap_mutex;
         atomic_t                        mmap_count;
-        struct perf_mmap_data           *data;
+        int                             mmap_locked;
+        struct user_struct              *mmap_user;
+        struct perf_buffer              *buffer;
 
         /* poll related */
         wait_queue_head_t               waitq;
@@ -666,6 +756,7 @@ struct perf_event {
         perf_overflow_handler_t         overflow_handler;
 
 #ifdef CONFIG_EVENT_TRACING
+        struct ftrace_event_call        *tp_event;
         struct event_filter             *filter;
 #endif
 
@@ -726,6 +817,9 @@ struct perf_cpu_context {
         int                             active_oncpu;
         int                             max_pertask;
         int                             exclusive;
+        struct swevent_hlist            *swevent_hlist;
+        struct mutex                    hlist_mutex;
+        int                             hlist_refcount;
 
         /*
          * Recursion avoidance:
@@ -737,12 +831,13 @@ struct perf_cpu_context {
 
 struct perf_output_handle {
         struct perf_event               *event;
-        struct perf_mmap_data           *data;
-        unsigned long                   head;
-        unsigned long                   offset;
+        struct perf_buffer              *buffer;
+        unsigned long                   wakeup;
+        unsigned long                   size;
+        void                            *addr;
+        int                             page;
         int                             nmi;
         int                             sample;
-        int                             locked;
 };
 
 #ifdef CONFIG_PERF_EVENTS
@@ -769,9 +864,6 @@ extern void perf_disable(void);
 extern void perf_enable(void);
 extern int perf_event_task_disable(void);
 extern int perf_event_task_enable(void);
-extern int hw_perf_group_sched_in(struct perf_event *group_leader,
-               struct perf_cpu_context *cpuctx,
-               struct perf_event_context *ctx);
 extern void perf_event_update_userpage(struct perf_event *event);
 extern int perf_event_release_kernel(struct perf_event *event);
 extern struct perf_event *
@@ -842,8 +934,10 @@ extern atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
 
 extern void __perf_sw_event(u32, u64, int, struct pt_regs *, u64);
 
-extern void
-perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int skip);
+#ifndef perf_arch_fetch_caller_regs
+static inline void
+perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
+#endif
 
 /*
  * Take a snapshot of the regs. Skip ip and frame pointer to
@@ -853,31 +947,11 @@ perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip, int skip);
  * - bp for callchains
  * - eflags, for future purposes, just in case
  */
-static inline void perf_fetch_caller_regs(struct pt_regs *regs, int skip)
+static inline void perf_fetch_caller_regs(struct pt_regs *regs)
 {
-        unsigned long ip;
-
         memset(regs, 0, sizeof(*regs));
 
-        switch (skip) {
-        case 1 :
-                ip = CALLER_ADDR0;
-                break;
-        case 2 :
-                ip = CALLER_ADDR1;
-                break;
-        case 3 :
-                ip = CALLER_ADDR2;
-                break;
-        case 4:
-                ip = CALLER_ADDR3;
-                break;
-        /* No need to support further for now */
-        default:
-                ip = 0;
-        }
-
-        return perf_arch_fetch_caller_regs(regs, ip, skip);
+        perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
 }
 
 static inline void
@@ -887,20 +961,17 @@ perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
                 struct pt_regs hot_regs;
 
                 if (!regs) {
-                        perf_fetch_caller_regs(&hot_regs, 1);
+                        perf_fetch_caller_regs(&hot_regs);
                         regs = &hot_regs;
                 }
                 __perf_sw_event(event_id, nr, nmi, regs, addr);
         }
 }
 
-extern void __perf_event_mmap(struct vm_area_struct *vma);
-
-static inline void perf_event_mmap(struct vm_area_struct *vma)
-{
-        if (vma->vm_flags & VM_EXEC)
-                __perf_event_mmap(vma);
-}
+extern void perf_event_mmap(struct vm_area_struct *vma);
+extern struct perf_guest_info_callbacks *perf_guest_cbs;
+extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
+extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
 
 extern void perf_event_comm(struct task_struct *tsk);
 extern void perf_event_fork(struct task_struct *tsk);
@@ -927,8 +998,9 @@ static inline bool perf_paranoid_kernel(void)
 }
 
 extern void perf_event_init(void);
-extern void perf_tp_event(int event_id, u64 addr, u64 count, void *record,
-                          int entry_size, struct pt_regs *regs);
+extern void perf_tp_event(u64 addr, u64 count, void *record,
+                          int entry_size, struct pt_regs *regs,
+                          struct hlist_head *head, int rctx);
 extern void perf_bp_event(struct perf_event *event, void *data);
 
 #ifndef perf_misc_flags
@@ -971,6 +1043,11 @@ perf_sw_event(u32 event_id, u64 nr, int nmi,
 static inline void
 perf_bp_event(struct perf_event *event, void *data)                     { }
 
+static inline int perf_register_guest_info_callbacks
+(struct perf_guest_info_callbacks *callbacks) { return 0; }
+static inline int perf_unregister_guest_info_callbacks
+(struct perf_guest_info_callbacks *callbacks) { return 0; }
+
 static inline void perf_event_mmap(struct vm_area_struct *vma)          { }
 static inline void perf_event_comm(struct task_struct *tsk)             { }
 static inline void perf_event_fork(struct task_struct *tsk)             { }
@@ -990,7 +1067,7 @@ static inline void perf_event_disable(struct perf_event *event)		{ }
 #define perf_cpu_notifier(fn)                                   \
 do {                                                            \
         static struct notifier_block fn##_nb __cpuinitdata =    \
-                { .notifier_call = fn, .priority = 20 };        \
+                { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
         fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE,             \
                 (void *)(unsigned long)smp_processor_id());     \
         fn(&fn##_nb, (unsigned long)CPU_STARTING,               \