4 files changed, 255 insertions, 15 deletions

diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 55e6d63d46d0..a52f2f4fe030 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -10,6 +10,7 @@
 #include <linux/kallsyms.h>
 #include <linux/linkage.h>
 #include <linux/bitops.h>
+#include <linux/ptrace.h>
 #include <linux/ktime.h>
 #include <linux/sched.h>
 #include <linux/types.h>
@@ -18,6 +19,28 @@
 
 #include <asm/ftrace.h>
 
+/*
+ * If the arch supports passing the variable contents of
+ * function_trace_op as the third parameter back from the
+ * mcount call, then the arch should define this as 1.
+ */
+#ifndef ARCH_SUPPORTS_FTRACE_OPS
+#define ARCH_SUPPORTS_FTRACE_OPS 0
+#endif
+
+/*
+ * If the arch's mcount caller does not support all of ftrace's
+ * features, then it must call an indirect function that
+ * does. Or at least does enough to prevent any unwelcomed side effects.
+ */
+#if !defined(CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST) || \
+        !ARCH_SUPPORTS_FTRACE_OPS
+# define FTRACE_FORCE_LIST_FUNC 1
+#else
+# define FTRACE_FORCE_LIST_FUNC 0
+#endif
+
+
 struct module;
 struct ftrace_hash;
 
@@ -29,7 +52,10 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
                      void __user *buffer, size_t *lenp,
                      loff_t *ppos);
 
-typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip);
+struct ftrace_ops;
+
+typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip,
+                              struct ftrace_ops *op, struct pt_regs *regs);
 
 /*
  * FTRACE_OPS_FL_* bits denote the state of ftrace_ops struct and are
@@ -45,12 +71,33 @@ typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip);
  *           could be controled by following calls:
  *             ftrace_function_local_enable
  *             ftrace_function_local_disable
+ * SAVE_REGS - The ftrace_ops wants regs saved at each function called
+ *            and passed to the callback. If this flag is set, but the
+ *            architecture does not support passing regs
+ *            (ARCH_SUPPORTS_FTRACE_SAVE_REGS is not defined), then the
+ *            ftrace_ops will fail to register, unless the next flag
+ *            is set.
+ * SAVE_REGS_IF_SUPPORTED - This is the same as SAVE_REGS, but if the
+ *            handler can handle an arch that does not save regs
+ *            (the handler tests if regs == NULL), then it can set
+ *            this flag instead. It will not fail registering the ftrace_ops
+ *            but, the regs field will be NULL if the arch does not support
+ *            passing regs to the handler.
+ *            Note, if this flag is set, the SAVE_REGS flag will automatically
+ *            get set upon registering the ftrace_ops, if the arch supports it.
+ * RECURSION_SAFE - The ftrace_ops can set this to tell the ftrace infrastructure
+ *            that the call back has its own recursion protection. If it does
+ *            not set this, then the ftrace infrastructure will add recursion
+ *            protection for the caller.
  */
 enum {
-        FTRACE_OPS_FL_ENABLED           = 1 << 0,
-        FTRACE_OPS_FL_GLOBAL            = 1 << 1,
-        FTRACE_OPS_FL_DYNAMIC           = 1 << 2,
-        FTRACE_OPS_FL_CONTROL           = 1 << 3,
+        FTRACE_OPS_FL_ENABLED                   = 1 << 0,
+        FTRACE_OPS_FL_GLOBAL                    = 1 << 1,
+        FTRACE_OPS_FL_DYNAMIC                   = 1 << 2,
+        FTRACE_OPS_FL_CONTROL                   = 1 << 3,
+        FTRACE_OPS_FL_SAVE_REGS                 = 1 << 4,
+        FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED    = 1 << 5,
+        FTRACE_OPS_FL_RECURSION_SAFE            = 1 << 6,
 };
 
 struct ftrace_ops {
@@ -163,7 +210,8 @@ static inline int ftrace_function_local_disabled(struct ftrace_ops *ops)
         return *this_cpu_ptr(ops->disabled);
 }
 
-extern void ftrace_stub(unsigned long a0, unsigned long a1);
+extern void ftrace_stub(unsigned long a0, unsigned long a1,
+                        struct ftrace_ops *op, struct pt_regs *regs);
 
 #else /* !CONFIG_FUNCTION_TRACER */
 /*
@@ -172,6 +220,10 @@ extern void ftrace_stub(unsigned long a0, unsigned long a1);
  */
 #define register_ftrace_function(ops) ({ 0; })
 #define unregister_ftrace_function(ops) ({ 0; })
+static inline int ftrace_nr_registered_ops(void)
+{
+        return 0;
+}
 static inline void clear_ftrace_function(void) { }
 static inline void ftrace_kill(void) { }
 static inline void ftrace_stop(void) { }
@@ -227,12 +279,33 @@ extern void unregister_ftrace_function_probe_all(char *glob);
 
 extern int ftrace_text_reserved(void *start, void *end);
 
+extern int ftrace_nr_registered_ops(void);
+
+/*
+ * The dyn_ftrace record's flags field is split into two parts.
+ * the first part which is '0-FTRACE_REF_MAX' is a counter of
+ * the number of callbacks that have registered the function that
+ * the dyn_ftrace descriptor represents.
+ *
+ * The second part is a mask:
+ *  ENABLED - the function is being traced
+ *  REGS    - the record wants the function to save regs
+ *  REGS_EN - the function is set up to save regs.
+ *
+ * When a new ftrace_ops is registered and wants a function to save
+ * pt_regs, the rec->flag REGS is set. When the function has been
+ * set up to save regs, the REG_EN flag is set. Once a function
+ * starts saving regs it will do so until all ftrace_ops are removed
+ * from tracing that function.
+ */
 enum {
-        FTRACE_FL_ENABLED       = (1 << 30),
+        FTRACE_FL_ENABLED       = (1UL << 29),
+        FTRACE_FL_REGS          = (1UL << 30),
+        FTRACE_FL_REGS_EN       = (1UL << 31)
 };
 
-#define FTRACE_FL_MASK          (0x3UL << 30)
-#define FTRACE_REF_MAX          ((1 << 30) - 1)
+#define FTRACE_FL_MASK          (0x7UL << 29)
+#define FTRACE_REF_MAX          ((1UL << 29) - 1)
 
 struct dyn_ftrace {
         union {
@@ -244,6 +317,8 @@ struct dyn_ftrace {
 };
 
 int ftrace_force_update(void);
+int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
+                         int remove, int reset);
 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
                        int len, int reset);
 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
@@ -263,9 +338,23 @@ enum {
         FTRACE_STOP_FUNC_RET            = (1 << 4),
 };
 
+/*
+ * The FTRACE_UPDATE_* enum is used to pass information back
+ * from the ftrace_update_record() and ftrace_test_record()
+ * functions. These are called by the code update routines
+ * to find out what is to be done for a given function.
+ *
+ *  IGNORE           - The function is already what we want it to be
+ *  MAKE_CALL        - Start tracing the function
+ *  MODIFY_CALL      - Stop saving regs for the function
+ *  MODIFY_CALL_REGS - Start saving regs for the function
+ *  MAKE_NOP         - Stop tracing the function
+ */
 enum {
         FTRACE_UPDATE_IGNORE,
         FTRACE_UPDATE_MAKE_CALL,
+        FTRACE_UPDATE_MODIFY_CALL,
+        FTRACE_UPDATE_MODIFY_CALL_REGS,
         FTRACE_UPDATE_MAKE_NOP,
 };
 
@@ -317,7 +406,9 @@ extern int ftrace_dyn_arch_init(void *data);
 extern void ftrace_replace_code(int enable);
 extern int ftrace_update_ftrace_func(ftrace_func_t func);
 extern void ftrace_caller(void);
+extern void ftrace_regs_caller(void);
 extern void ftrace_call(void);
+extern void ftrace_regs_call(void);
 extern void mcount_call(void);
 
 void ftrace_modify_all_code(int command);
@@ -325,6 +416,15 @@ void ftrace_modify_all_code(int command);
 #ifndef FTRACE_ADDR
 #define FTRACE_ADDR ((unsigned long)ftrace_caller)
 #endif
+
+#ifndef FTRACE_REGS_ADDR
+#ifdef ARCH_SUPPORTS_FTRACE_SAVE_REGS
+# define FTRACE_REGS_ADDR ((unsigned long)ftrace_regs_caller)
+#else
+# define FTRACE_REGS_ADDR FTRACE_ADDR
+#endif
+#endif
+
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 extern void ftrace_graph_caller(void);
 extern int ftrace_enable_ftrace_graph_caller(void);
@@ -380,6 +480,39 @@ extern int ftrace_make_nop(struct module *mod,
  */
 extern int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr);
 
+#ifdef ARCH_SUPPORTS_FTRACE_SAVE_REGS
+/**
+ * ftrace_modify_call - convert from one addr to another (no nop)
+ * @rec: the mcount call site record
+ * @old_addr: the address expected to be currently called to
+ * @addr: the address to change to
+ *
+ * This is a very sensitive operation and great care needs
+ * to be taken by the arch.  The operation should carefully
+ * read the location, check to see if what is read is indeed
+ * what we expect it to be, and then on success of the compare,
+ * it should write to the location.
+ *
+ * The code segment at @rec->ip should be a caller to @old_addr
+ *
+ * Return must be:
+ *  0 on success
+ *  -EFAULT on error reading the location
+ *  -EINVAL on a failed compare of the contents
+ *  -EPERM  on error writing to the location
+ * Any other value will be considered a failure.
+ */
+extern int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
+                              unsigned long addr);
+#else
+/* Should never be called */
+static inline int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
+                                     unsigned long addr)
+{
+        return -EINVAL;
+}
+#endif
+
 /* May be defined in arch */
 extern int ftrace_arch_read_dyn_info(char *buf, int size);
 
@@ -387,7 +520,7 @@ extern int skip_trace(unsigned long ip);
 
 extern void ftrace_disable_daemon(void);
 extern void ftrace_enable_daemon(void);
-#else
+#else /* CONFIG_DYNAMIC_FTRACE */
 static inline int skip_trace(unsigned long ip) { return 0; }
 static inline int ftrace_force_update(void) { return 0; }
 static inline void ftrace_disable_daemon(void) { }
@@ -405,6 +538,10 @@ static inline int ftrace_text_reserved(void *start, void *end)
 {
         return 0;
 }
+static inline unsigned long ftrace_location(unsigned long ip)
+{
+        return 0;
+}
 
 /*
  * Again users of functions that have ftrace_ops may not
@@ -413,6 +550,7 @@ static inline int ftrace_text_reserved(void *start, void *end)
  */
 #define ftrace_regex_open(ops, flag, inod, file) ({ -ENODEV; })
 #define ftrace_set_early_filter(ops, buf, enable) do { } while (0)
+#define ftrace_set_filter_ip(ops, ip, remove, reset) ({ -ENODEV; })
 #define ftrace_set_filter(ops, buf, len, reset) ({ -ENODEV; })
 #define ftrace_set_notrace(ops, buf, len, reset) ({ -ENODEV; })
 #define ftrace_free_filter(ops) do { } while (0)
diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h
index b6e1f8c00577..23755ba42abc 100644
--- a/include/linux/kprobes.h
+++ b/include/linux/kprobes.h
@@ -38,6 +38,7 @@
 #include <linux/spinlock.h>
 #include <linux/rcupdate.h>
 #include <linux/mutex.h>
+#include <linux/ftrace.h>
 
 #ifdef CONFIG_KPROBES
 #include <asm/kprobes.h>
@@ -48,14 +49,26 @@
 #define KPROBE_REENTER          0x00000004
 #define KPROBE_HIT_SSDONE       0x00000008
 
+/*
+ * If function tracer is enabled and the arch supports full
+ * passing of pt_regs to function tracing, then kprobes can
+ * optimize on top of function tracing.
+ */
+#if defined(CONFIG_FUNCTION_TRACER) && defined(ARCH_SUPPORTS_FTRACE_SAVE_REGS) \
+        && defined(ARCH_SUPPORTS_KPROBES_ON_FTRACE)
+# define KPROBES_CAN_USE_FTRACE
+#endif
+
 /* Attach to insert probes on any functions which should be ignored*/
 #define __kprobes       __attribute__((__section__(".kprobes.text")))
+
 #else /* CONFIG_KPROBES */
 typedef int kprobe_opcode_t;
 struct arch_specific_insn {
         int dummy;
 };
 #define __kprobes
+
 #endif /* CONFIG_KPROBES */
 
 struct kprobe;
@@ -128,6 +141,7 @@ struct kprobe {
                                    * NOTE:
                                    * this flag is only for optimized_kprobe.
                                    */
+#define KPROBE_FLAG_FTRACE      8 /* probe is using ftrace */
 
 /* Has this kprobe gone ? */
 static inline int kprobe_gone(struct kprobe *p)
@@ -146,6 +160,13 @@ static inline int kprobe_optimized(struct kprobe *p)
 {
         return p->flags & KPROBE_FLAG_OPTIMIZED;
 }
+
+/* Is this kprobe uses ftrace ? */
+static inline int kprobe_ftrace(struct kprobe *p)
+{
+        return p->flags & KPROBE_FLAG_FTRACE;
+}
+
 /*
  * Special probe type that uses setjmp-longjmp type tricks to resume
  * execution at a specified entry with a matching prototype corresponding
@@ -295,6 +316,12 @@ extern int proc_kprobes_optimization_handler(struct ctl_table *table,
 #endif
 
 #endif /* CONFIG_OPTPROBES */
+#ifdef KPROBES_CAN_USE_FTRACE
+extern void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
+                                  struct ftrace_ops *ops, struct pt_regs *regs);
+extern int arch_prepare_kprobe_ftrace(struct kprobe *p);
+#endif
+
 
 /* Get the kprobe at this addr (if any) - called with preemption disabled */
 struct kprobe *get_kprobe(void *addr);
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 7602ccb3f40e..28f9cee3fbc3 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -130,8 +130,10 @@ enum perf_event_sample_format {
         PERF_SAMPLE_STREAM_ID                   = 1U << 9,
         PERF_SAMPLE_RAW                         = 1U << 10,
         PERF_SAMPLE_BRANCH_STACK                = 1U << 11,
+        PERF_SAMPLE_REGS_USER                   = 1U << 12,
+        PERF_SAMPLE_STACK_USER                  = 1U << 13,
 
-        PERF_SAMPLE_MAX = 1U << 12,             /* non-ABI */
+        PERF_SAMPLE_MAX = 1U << 14,             /* non-ABI */
 };
 
 /*
@@ -163,6 +165,15 @@ enum perf_branch_sample_type {
          PERF_SAMPLE_BRANCH_HV)
 
 /*
+ * Values to determine ABI of the registers dump.
+ */
+enum perf_sample_regs_abi {
+        PERF_SAMPLE_REGS_ABI_NONE       = 0,
+        PERF_SAMPLE_REGS_ABI_32         = 1,
+        PERF_SAMPLE_REGS_ABI_64         = 2,
+};
+
+/*
  * The format of the data returned by read() on a perf event fd,
  * as specified by attr.read_format:
  *
@@ -194,6 +205,8 @@ enum perf_event_read_format {
 #define PERF_ATTR_SIZE_VER0     64      /* sizeof first published struct */
 #define PERF_ATTR_SIZE_VER1     72      /* add: config2 */
 #define PERF_ATTR_SIZE_VER2     80      /* add: branch_sample_type */
+#define PERF_ATTR_SIZE_VER3     96      /* add: sample_regs_user */
+                                        /* add: sample_stack_user */
 
 /*
  * Hardware event_id to monitor via a performance monitoring event:
@@ -255,7 +268,10 @@ struct perf_event_attr {
                                 exclude_host   :  1, /* don't count in host   */
                                 exclude_guest  :  1, /* don't count in guest  */
 
-                                __reserved_1   : 43;
+                                exclude_callchain_kernel : 1, /* exclude kernel callchains */
+                                exclude_callchain_user   : 1, /* exclude user callchains */
+
+                                __reserved_1   : 41;
 
         union {
                 __u32           wakeup_events;    /* wakeup every n events */
@@ -271,7 +287,21 @@ struct perf_event_attr {
                 __u64           bp_len;
                 __u64           config2; /* extension of config1 */
         };
-        __u64   branch_sample_type; /* enum branch_sample_type */
+        __u64   branch_sample_type; /* enum perf_branch_sample_type */
+
+        /*
+         * Defines set of user regs to dump on samples.
+         * See asm/perf_regs.h for details.
+         */
+        __u64   sample_regs_user;
+
+        /*
+         * Defines size of the user stack to dump on samples.
+         */
+        __u32   sample_stack_user;
+
+        /* Align to u64. */
+        __u32   __reserved_2;
 };
 
 /*
@@ -548,6 +578,13 @@ enum perf_event_type {
          *        char                  data[size];}&& PERF_SAMPLE_RAW
          *
          *      { u64 from, to, flags } lbr[nr];} && PERF_SAMPLE_BRANCH_STACK
+         *
+         *      { u64                   abi; # enum perf_sample_regs_abi
+         *        u64                   regs[weight(mask)]; } && PERF_SAMPLE_REGS_USER
+         *
+         *      { u64                   size;
+         *        char                  data[size];
+         *        u64                   dyn_size; } && PERF_SAMPLE_STACK_USER
          * };
          */
         PERF_RECORD_SAMPLE                      = 9,
@@ -609,6 +646,7 @@ struct perf_guest_info_callbacks {
 #include <linux/static_key.h>
 #include <linux/atomic.h>
 #include <linux/sysfs.h>
+#include <linux/perf_regs.h>
 #include <asm/local.h>
 
 struct perf_callchain_entry {
@@ -654,6 +692,11 @@ struct perf_branch_stack {
         struct perf_branch_entry        entries[0];
 };
 
+struct perf_regs_user {
+        __u64           abi;
+        struct pt_regs  *regs;
+};
+
 struct task_struct;
 
 /*
@@ -1133,6 +1176,8 @@ struct perf_sample_data {
         struct perf_callchain_entry     *callchain;
         struct perf_raw_record          *raw;
         struct perf_branch_stack        *br_stack;
+        struct perf_regs_user           regs_user;
+        u64                             stack_user_size;
 };
 
 static inline void perf_sample_data_init(struct perf_sample_data *data,
@@ -1142,7 +1187,10 @@ static inline void perf_sample_data_init(struct perf_sample_data *data,
         data->addr = addr;
         data->raw  = NULL;
         data->br_stack = NULL;
-        data->period    = period;
+        data->period = period;
+        data->regs_user.abi = PERF_SAMPLE_REGS_ABI_NONE;
+        data->regs_user.regs = NULL;
+        data->stack_user_size = 0;
 }
 
 extern void perf_output_sample(struct perf_output_handle *handle,
@@ -1290,8 +1338,10 @@ static inline bool has_branch_stack(struct perf_event *event)
 extern int perf_output_begin(struct perf_output_handle *handle,
                              struct perf_event *event, unsigned int size);
 extern void perf_output_end(struct perf_output_handle *handle);
-extern void perf_output_copy(struct perf_output_handle *handle,
+extern unsigned int perf_output_copy(struct perf_output_handle *handle,
                              const void *buf, unsigned int len);
+extern unsigned int perf_output_skip(struct perf_output_handle *handle,
+                                     unsigned int len);
 extern int perf_swevent_get_recursion_context(void);
 extern void perf_swevent_put_recursion_context(int rctx);
 extern void perf_event_enable(struct perf_event *event);
diff --git a/include/linux/perf_regs.h b/include/linux/perf_regs.h
new file mode 100644
index 000000000000..3c73d5fe18be
--- /dev/null
+++ b/include/linux/perf_regs.h
@@ -0,0 +1,25 @@
+#ifndef _LINUX_PERF_REGS_H
+#define _LINUX_PERF_REGS_H
+
+#ifdef CONFIG_HAVE_PERF_REGS
+#include <asm/perf_regs.h>
+u64 perf_reg_value(struct pt_regs *regs, int idx);
+int perf_reg_validate(u64 mask);
+u64 perf_reg_abi(struct task_struct *task);
+#else
+static inline u64 perf_reg_value(struct pt_regs *regs, int idx)
+{
+        return 0;
+}
+
+static inline int perf_reg_validate(u64 mask)
+{
+        return mask ? -ENOSYS : 0;
+}
+
+static inline u64 perf_reg_abi(struct task_struct *task)
+{
+        return PERF_SAMPLE_REGS_ABI_NONE;
+}
+#endif /* CONFIG_HAVE_PERF_REGS */
+#endif /* _LINUX_PERF_REGS_H */