ftrace/x86: Add separate function to save regs

Add a way to have different functions calling different trampolines.
If a ftrace_ops wants regs saved on the return, then have only the
functions with ops registered to save regs. Functions registered by
other ops would not be affected, unless the functions overlap.

If one ftrace_ops registered functions A, B and C and another ops
registered fucntions to save regs on A, and D, then only functions
A and D would be saving regs. Function B and C would work as normal.
Although A is registered by both ops: normal and saves regs; this is fine
as saving the regs is needed to satisfy one of the ops that calls it
but the regs are ignored by the other ops function.

x86_64 implements the full regs saving, and i386 just passes a NULL
for regs to satisfy the ftrace_ops passing. Where an arch must supply
both regs and ftrace_ops parameters, even if regs is just NULL.

It is OK for an arch to pass NULL regs. All function trace users that
require regs passing must add the flag FTRACE_OPS_FL_SAVE_REGS when
registering the ftrace_ops. If the arch does not support saving regs
then the ftrace_ops will fail to register. The flag
FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED may be set that will prevent the
ftrace_ops from failing to register. In this case, the handler may
either check if regs is not NULL or check if ARCH_SUPPORTS_FTRACE_SAVE_REGS.
If the arch supports passing regs it will set this macro and pass regs
for ops that request them. All other archs will just pass NULL.

Link: Link: http://lkml.kernel.org/r/20120711195745.107705970@goodmis.org

Cc: Alexander van Heukelum <heukelum@fastmail.fm>
Reviewed-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>

6 files changed, 373 insertions, 47 deletions

diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index b3bb1f3f2773..a8475014c4ad 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -3,27 +3,33 @@
 
 #ifdef __ASSEMBLY__
 
-        .macro MCOUNT_SAVE_FRAME
-        /* taken from glibc */
-        subq $0x38, %rsp
-        movq %rax, (%rsp)
-        movq %rcx, 8(%rsp)
-        movq %rdx, 16(%rsp)
-        movq %rsi, 24(%rsp)
-        movq %rdi, 32(%rsp)
-        movq %r8, 40(%rsp)
-        movq %r9, 48(%rsp)
+        /* skip is set if the stack was already partially adjusted */
+        .macro MCOUNT_SAVE_FRAME skip=0
+         /*
+          * We add enough stack to save all regs.
+          */
+        subq $(SS+8-\skip), %rsp
+        movq %rax, RAX(%rsp)
+        movq %rcx, RCX(%rsp)
+        movq %rdx, RDX(%rsp)
+        movq %rsi, RSI(%rsp)
+        movq %rdi, RDI(%rsp)
+        movq %r8, R8(%rsp)
+        movq %r9, R9(%rsp)
+         /* Move RIP to its proper location */
+        movq SS+8(%rsp), %rdx
+        movq %rdx, RIP(%rsp)
         .endm
 
-        .macro MCOUNT_RESTORE_FRAME
-        movq 48(%rsp), %r9
-        movq 40(%rsp), %r8
-        movq 32(%rsp), %rdi
-        movq 24(%rsp), %rsi
-        movq 16(%rsp), %rdx
-        movq 8(%rsp), %rcx
-        movq (%rsp), %rax
-        addq $0x38, %rsp
+        .macro MCOUNT_RESTORE_FRAME skip=0
+        movq R9(%rsp), %r9
+        movq R8(%rsp), %r8
+        movq RDI(%rsp), %rdi
+        movq RSI(%rsp), %rsi
+        movq RDX(%rsp), %rdx
+        movq RCX(%rsp), %rcx
+        movq RAX(%rsp), %rax
+        addq $(SS+8-\skip), %rsp
         .endm
 
 #endif
@@ -34,6 +40,9 @@
 
 #ifdef CONFIG_DYNAMIC_FTRACE
 #define ARCH_SUPPORTS_FTRACE_OPS 1
+#ifdef CONFIG_X86_64
+#define ARCH_SUPPORTS_FTRACE_SAVE_REGS
+#endif
 #endif
 
 #ifndef __ASSEMBLY__
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index e3e17a0b7ff8..5da11d1b85ae 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -1109,7 +1109,8 @@ ENTRY(ftrace_caller)
         pushl %eax
         pushl %ecx
         pushl %edx
-        movl 0xc(%esp), %eax
+        pushl $0        /* Pass NULL as regs pointer */
+        movl 4*4(%esp), %eax
         movl 0x4(%ebp), %edx
         leal function_trace_op, %ecx
         subl $MCOUNT_INSN_SIZE, %eax
@@ -1118,6 +1119,7 @@ ENTRY(ftrace_caller)
 ftrace_call:
         call ftrace_stub
 
+        addl $4,%esp    /* skip NULL pointer */
         popl %edx
         popl %ecx
         popl %eax
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 2b4f94c5dc60..52bda2e8f7aa 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -73,21 +73,34 @@ ENTRY(mcount)
         retq
 END(mcount)
 
+/* skip is set if stack has been adjusted */
+.macro ftrace_caller_setup skip=0
+        MCOUNT_SAVE_FRAME \skip
+
+        /* Load the ftrace_ops into the 3rd parameter */
+        leaq function_trace_op, %rdx
+
+        /* Load ip into the first parameter */
+        movq RIP(%rsp), %rdi
+        subq $MCOUNT_INSN_SIZE, %rdi
+        /* Load the parent_ip into the second parameter */
+        movq 8(%rbp), %rsi
+.endm
+
 ENTRY(ftrace_caller)
+        /* Check if tracing was disabled (quick check) */
         cmpl $0, function_trace_stop
         jne  ftrace_stub
 
-        MCOUNT_SAVE_FRAME
-
-        leaq function_trace_op, %rdx
-        movq 0x38(%rsp), %rdi
-        movq 8(%rbp), %rsi
-        subq $MCOUNT_INSN_SIZE, %rdi
+        ftrace_caller_setup
+        /* regs go into 4th parameter (but make it NULL) */
+        movq $0, %rcx
 
 GLOBAL(ftrace_call)
         call ftrace_stub
 
         MCOUNT_RESTORE_FRAME
+ftrace_return:
 
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 GLOBAL(ftrace_graph_call)
@@ -98,6 +111,71 @@ GLOBAL(ftrace_stub)
         retq
 END(ftrace_caller)
 
+ENTRY(ftrace_regs_caller)
+        /* Save the current flags before compare (in SS location)*/
+        pushfq
+
+        /* Check if tracing was disabled (quick check) */
+        cmpl $0, function_trace_stop
+        jne  ftrace_restore_flags
+
+        /* skip=8 to skip flags saved in SS */
+        ftrace_caller_setup 8
+
+        /* Save the rest of pt_regs */
+        movq %r15, R15(%rsp)
+        movq %r14, R14(%rsp)
+        movq %r13, R13(%rsp)
+        movq %r12, R12(%rsp)
+        movq %r11, R11(%rsp)
+        movq %r10, R10(%rsp)
+        movq %rbp, RBP(%rsp)
+        movq %rbx, RBX(%rsp)
+        /* Copy saved flags */
+        movq SS(%rsp), %rcx
+        movq %rcx, EFLAGS(%rsp)
+        /* Kernel segments */
+        movq $__KERNEL_DS, %rcx
+        movq %rcx, SS(%rsp)
+        movq $__KERNEL_CS, %rcx
+        movq %rcx, CS(%rsp)
+        /* Stack - skipping return address */
+        leaq SS+16(%rsp), %rcx
+        movq %rcx, RSP(%rsp)
+
+        /* regs go into 4th parameter */
+        leaq (%rsp), %rcx
+
+GLOBAL(ftrace_regs_call)
+        call ftrace_stub
+
+        /* Copy flags back to SS, to restore them */
+        movq EFLAGS(%rsp), %rax
+        movq %rax, SS(%rsp)
+
+        /* restore the rest of pt_regs */
+        movq R15(%rsp), %r15
+        movq R14(%rsp), %r14
+        movq R13(%rsp), %r13
+        movq R12(%rsp), %r12
+        movq R10(%rsp), %r10
+        movq RBP(%rsp), %rbp
+        movq RBX(%rsp), %rbx
+
+        /* skip=8 to skip flags saved in SS */
+        MCOUNT_RESTORE_FRAME 8
+
+        /* Restore flags */
+        popfq
+
+        jmp ftrace_return
+ftrace_restore_flags:
+        popfq
+        jmp  ftrace_stub
+
+END(ftrace_regs_caller)
+
+
 #else /* ! CONFIG_DYNAMIC_FTRACE */
 ENTRY(mcount)
         cmpl $0, function_trace_stop
@@ -120,7 +198,7 @@ GLOBAL(ftrace_stub)
 trace:
         MCOUNT_SAVE_FRAME
 
-        movq 0x38(%rsp), %rdi
+        movq RIP(%rsp), %rdi
         movq 8(%rbp), %rsi
         subq $MCOUNT_INSN_SIZE, %rdi
 
@@ -141,7 +219,7 @@ ENTRY(ftrace_graph_caller)
         MCOUNT_SAVE_FRAME
 
         leaq 8(%rbp), %rdi
-        movq 0x38(%rsp), %rsi
+        movq RIP(%rsp), %rsi
         movq (%rbp), %rdx
         subq $MCOUNT_INSN_SIZE, %rsi
 
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index c3a7cb4bf6e6..b90eb1a13071 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -206,6 +206,23 @@ static int
 ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
                    unsigned const char *new_code);
 
+#ifdef ARCH_SUPPORTS_FTRACE_SAVE_REGS
+/*
+ * Should never be called:
+ *  As it is only called by __ftrace_replace_code() which is called by
+ *  ftrace_replace_code() that x86 overrides, and by ftrace_update_code()
+ *  which is called to turn mcount into nops or nops into function calls
+ *  but not to convert a function from not using regs to one that uses
+ *  regs, which ftrace_modify_call() is for.
+ */
+int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
+                                 unsigned long addr)
+{
+        WARN_ON(1);
+        return -EINVAL;
+}
+#endif
+
 int ftrace_update_ftrace_func(ftrace_func_t func)
 {
         unsigned long ip = (unsigned long)(&ftrace_call);
@@ -220,6 +237,16 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
 
         ret = ftrace_modify_code(ip, old, new);
 
+#ifdef ARCH_SUPPORTS_FTRACE_SAVE_REGS
+        /* Also update the regs callback function */
+        if (!ret) {
+                ip = (unsigned long)(&ftrace_regs_call);
+                memcpy(old, &ftrace_regs_call, MCOUNT_INSN_SIZE);
+                new = ftrace_call_replace(ip, (unsigned long)func);
+                ret = ftrace_modify_code(ip, old, new);
+        }
+#endif
+
         atomic_dec(&modifying_ftrace_code);
 
         return ret;
@@ -299,6 +326,32 @@ static int add_brk_on_nop(struct dyn_ftrace *rec)
         return add_break(rec->ip, old);
 }
 
+/*
+ * If the record has the FTRACE_FL_REGS set, that means that it
+ * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
+ * is not not set, then it wants to convert to the normal callback.
+ */
+static unsigned long get_ftrace_addr(struct dyn_ftrace *rec)
+{
+        if (rec->flags & FTRACE_FL_REGS)
+                return (unsigned long)FTRACE_REGS_ADDR;
+        else
+                return (unsigned long)FTRACE_ADDR;
+}
+
+/*
+ * The FTRACE_FL_REGS_EN is set when the record already points to
+ * a function that saves all the regs. Basically the '_EN' version
+ * represents the current state of the function.
+ */
+static unsigned long get_ftrace_old_addr(struct dyn_ftrace *rec)
+{
+        if (rec->flags & FTRACE_FL_REGS_EN)
+                return (unsigned long)FTRACE_REGS_ADDR;
+        else
+                return (unsigned long)FTRACE_ADDR;
+}
+
 static int add_breakpoints(struct dyn_ftrace *rec, int enable)
 {
         unsigned long ftrace_addr;
@@ -306,7 +359,7 @@ static int add_breakpoints(struct dyn_ftrace *rec, int enable)
 
         ret = ftrace_test_record(rec, enable);
 
-        ftrace_addr = (unsigned long)FTRACE_ADDR;
+        ftrace_addr = get_ftrace_addr(rec);
 
         switch (ret) {
         case FTRACE_UPDATE_IGNORE:
@@ -316,6 +369,10 @@ static int add_breakpoints(struct dyn_ftrace *rec, int enable)
                 /* converting nop to call */
                 return add_brk_on_nop(rec);
 
+        case FTRACE_UPDATE_MODIFY_CALL_REGS:
+        case FTRACE_UPDATE_MODIFY_CALL:
+                ftrace_addr = get_ftrace_old_addr(rec);
+                /* fall through */
         case FTRACE_UPDATE_MAKE_NOP:
                 /* converting a call to a nop */
                 return add_brk_on_call(rec, ftrace_addr);
@@ -360,13 +417,21 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
                  * If not, don't touch the breakpoint, we make just create
                  * a disaster.
                  */
-                ftrace_addr = (unsigned long)FTRACE_ADDR;
+                ftrace_addr = get_ftrace_addr(rec);
+                nop = ftrace_call_replace(ip, ftrace_addr);
+
+                if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
+                        goto update;
+
+                /* Check both ftrace_addr and ftrace_old_addr */
+                ftrace_addr = get_ftrace_old_addr(rec);
                 nop = ftrace_call_replace(ip, ftrace_addr);
 
                 if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
                         return -EINVAL;
         }
 
+ update:
         return probe_kernel_write((void *)ip, &nop[0], 1);
 }
 
@@ -405,12 +470,14 @@ static int add_update(struct dyn_ftrace *rec, int enable)
 
         ret = ftrace_test_record(rec, enable);
 
-        ftrace_addr = (unsigned long)FTRACE_ADDR;
+        ftrace_addr  = get_ftrace_addr(rec);
 
         switch (ret) {
         case FTRACE_UPDATE_IGNORE:
                 return 0;
 
+        case FTRACE_UPDATE_MODIFY_CALL_REGS:
+        case FTRACE_UPDATE_MODIFY_CALL:
         case FTRACE_UPDATE_MAKE_CALL:
                 /* converting nop to call */
                 return add_update_call(rec, ftrace_addr);
@@ -455,12 +522,14 @@ static int finish_update(struct dyn_ftrace *rec, int enable)
 
         ret = ftrace_update_record(rec, enable);
 
-        ftrace_addr = (unsigned long)FTRACE_ADDR;
+        ftrace_addr = get_ftrace_addr(rec);
 
         switch (ret) {
         case FTRACE_UPDATE_IGNORE:
                 return 0;
 
+        case FTRACE_UPDATE_MODIFY_CALL_REGS:
+        case FTRACE_UPDATE_MODIFY_CALL:
         case FTRACE_UPDATE_MAKE_CALL:
                 /* converting nop to call */
                 return finish_update_call(rec, ftrace_addr);
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index e4202881fb00..ab39990cc43f 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -71,12 +71,28 @@ typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip,
  *           could be controled by following calls:
  *             ftrace_function_local_enable
  *             ftrace_function_local_disable
+ * SAVE_REGS - The ftrace_ops wants regs saved at each function called
+ *            and passed to the callback. If this flag is set, but the
+ *            architecture does not support passing regs
+ *            (ARCH_SUPPORTS_FTRACE_SAVE_REGS is not defined), then the
+ *            ftrace_ops will fail to register, unless the next flag
+ *            is set.
+ * SAVE_REGS_IF_SUPPORTED - This is the same as SAVE_REGS, but if the
+ *            handler can handle an arch that does not save regs
+ *            (the handler tests if regs == NULL), then it can set
+ *            this flag instead. It will not fail registering the ftrace_ops
+ *            but, the regs field will be NULL if the arch does not support
+ *            passing regs to the handler.
+ *            Note, if this flag is set, the SAVE_REGS flag will automatically
+ *            get set upon registering the ftrace_ops, if the arch supports it.
  */
 enum {
-        FTRACE_OPS_FL_ENABLED           = 1 << 0,
-        FTRACE_OPS_FL_GLOBAL            = 1 << 1,
-        FTRACE_OPS_FL_DYNAMIC           = 1 << 2,
-        FTRACE_OPS_FL_CONTROL           = 1 << 3,
+        FTRACE_OPS_FL_ENABLED                   = 1 << 0,
+        FTRACE_OPS_FL_GLOBAL                    = 1 << 1,
+        FTRACE_OPS_FL_DYNAMIC                   = 1 << 2,
+        FTRACE_OPS_FL_CONTROL                   = 1 << 3,
+        FTRACE_OPS_FL_SAVE_REGS                 = 1 << 4,
+        FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED    = 1 << 5,
 };
 
 struct ftrace_ops {
@@ -254,12 +270,31 @@ extern void unregister_ftrace_function_probe_all(char *glob);
 
 extern int ftrace_text_reserved(void *start, void *end);
 
+/*
+ * The dyn_ftrace record's flags field is split into two parts.
+ * the first part which is '0-FTRACE_REF_MAX' is a counter of
+ * the number of callbacks that have registered the function that
+ * the dyn_ftrace descriptor represents.
+ *
+ * The second part is a mask:
+ *  ENABLED - the function is being traced
+ *  REGS    - the record wants the function to save regs
+ *  REGS_EN - the function is set up to save regs.
+ *
+ * When a new ftrace_ops is registered and wants a function to save
+ * pt_regs, the rec->flag REGS is set. When the function has been
+ * set up to save regs, the REG_EN flag is set. Once a function
+ * starts saving regs it will do so until all ftrace_ops are removed
+ * from tracing that function.
+ */
 enum {
-        FTRACE_FL_ENABLED       = (1 << 30),
+        FTRACE_FL_ENABLED       = (1UL << 29),
+        FTRACE_FL_REGS          = (1UL << 30),
+        FTRACE_FL_REGS_EN       = (1UL << 31)
 };
 
-#define FTRACE_FL_MASK          (0x3UL << 30)
-#define FTRACE_REF_MAX          ((1 << 30) - 1)
+#define FTRACE_FL_MASK          (0x7UL << 29)
+#define FTRACE_REF_MAX          ((1UL << 29) - 1)
 
 struct dyn_ftrace {
         union {
@@ -290,9 +325,23 @@ enum {
         FTRACE_STOP_FUNC_RET            = (1 << 4),
 };
 
+/*
+ * The FTRACE_UPDATE_* enum is used to pass information back
+ * from the ftrace_update_record() and ftrace_test_record()
+ * functions. These are called by the code update routines
+ * to find out what is to be done for a given function.
+ *
+ *  IGNORE           - The function is already what we want it to be
+ *  MAKE_CALL        - Start tracing the function
+ *  MODIFY_CALL      - Stop saving regs for the function
+ *  MODIFY_CALL_REGS - Start saving regs for the function
+ *  MAKE_NOP         - Stop tracing the function
+ */
 enum {
         FTRACE_UPDATE_IGNORE,
         FTRACE_UPDATE_MAKE_CALL,
+        FTRACE_UPDATE_MODIFY_CALL,
+        FTRACE_UPDATE_MODIFY_CALL_REGS,
         FTRACE_UPDATE_MAKE_NOP,
 };
 
@@ -344,7 +393,9 @@ extern int ftrace_dyn_arch_init(void *data);
 extern void ftrace_replace_code(int enable);
 extern int ftrace_update_ftrace_func(ftrace_func_t func);
 extern void ftrace_caller(void);
+extern void ftrace_regs_caller(void);
 extern void ftrace_call(void);
+extern void ftrace_regs_call(void);
 extern void mcount_call(void);
 
 void ftrace_modify_all_code(int command);
@@ -352,6 +403,15 @@ void ftrace_modify_all_code(int command);
 #ifndef FTRACE_ADDR
 #define FTRACE_ADDR ((unsigned long)ftrace_caller)
 #endif
+
+#ifndef FTRACE_REGS_ADDR
+#ifdef ARCH_SUPPORTS_FTRACE_SAVE_REGS
+# define FTRACE_REGS_ADDR ((unsigned long)ftrace_regs_caller)
+#else
+# define FTRACE_REGS_ADDR FTRACE_ADDR
+#endif
+#endif
+
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 extern void ftrace_graph_caller(void);
 extern int ftrace_enable_ftrace_graph_caller(void);
@@ -407,6 +467,39 @@ extern int ftrace_make_nop(struct module *mod,
  */
 extern int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr);
 
+#ifdef ARCH_SUPPORTS_FTRACE_SAVE_REGS
+/**
+ * ftrace_modify_call - convert from one addr to another (no nop)
+ * @rec: the mcount call site record
+ * @old_addr: the address expected to be currently called to
+ * @addr: the address to change to
+ *
+ * This is a very sensitive operation and great care needs
+ * to be taken by the arch.  The operation should carefully
+ * read the location, check to see if what is read is indeed
+ * what we expect it to be, and then on success of the compare,
+ * it should write to the location.
+ *
+ * The code segment at @rec->ip should be a caller to @old_addr
+ *
+ * Return must be:
+ *  0 on success
+ *  -EFAULT on error reading the location
+ *  -EINVAL on a failed compare of the contents
+ *  -EPERM  on error writing to the location
+ * Any other value will be considered a failure.
+ */
+extern int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
+                              unsigned long addr);
+#else
+/* Should never be called */
+static inline int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
+                                     unsigned long addr)
+{
+        return -EINVAL;
+}
+#endif
+
 /* May be defined in arch */
 extern int ftrace_arch_read_dyn_info(char *buf, int size);
 
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 6ff07ad0ede3..c55f7e274613 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -314,6 +314,20 @@ static int __register_ftrace_function(struct ftrace_ops *ops)
         if ((ops->flags & FL_GLOBAL_CONTROL_MASK) == FL_GLOBAL_CONTROL_MASK)
                 return -EINVAL;
 
+#ifndef ARCH_SUPPORTS_FTRACE_SAVE_REGS
+        /*
+         * If the ftrace_ops specifies SAVE_REGS, then it only can be used
+         * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
+         * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
+         */
+        if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
+            !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
+                return -EINVAL;
+
+        if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
+                ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
+#endif
+
         if (!core_kernel_data((unsigned long)ops))
                 ops->flags |= FTRACE_OPS_FL_DYNAMIC;
 
@@ -1515,6 +1529,12 @@ static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
                         rec->flags++;
                         if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
                                 return;
+                        /*
+                         * If any ops wants regs saved for this function
+                         * then all ops will get saved regs.
+                         */
+                        if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
+                                rec->flags |= FTRACE_FL_REGS;
                 } else {
                         if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
                                 return;
@@ -1606,18 +1626,59 @@ static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
         if (enable && (rec->flags & ~FTRACE_FL_MASK))
                 flag = FTRACE_FL_ENABLED;
 
+        /*
+         * If enabling and the REGS flag does not match the REGS_EN, then
+         * do not ignore this record. Set flags to fail the compare against
+         * ENABLED.
+         */
+        if (flag &&
+            (!(rec->flags & FTRACE_FL_REGS) != !(rec->flags & FTRACE_FL_REGS_EN)))
+                flag |= FTRACE_FL_REGS;
+
         /* If the state of this record hasn't changed, then do nothing */
         if ((rec->flags & FTRACE_FL_ENABLED) == flag)
                 return FTRACE_UPDATE_IGNORE;
 
         if (flag) {
-                if (update)
+                /* Save off if rec is being enabled (for return value) */
+                flag ^= rec->flags & FTRACE_FL_ENABLED;
+
+                if (update) {
                         rec->flags |= FTRACE_FL_ENABLED;
-                return FTRACE_UPDATE_MAKE_CALL;
+                        if (flag & FTRACE_FL_REGS) {
+                                if (rec->flags & FTRACE_FL_REGS)
+                                        rec->flags |= FTRACE_FL_REGS_EN;
+                                else
+                                        rec->flags &= ~FTRACE_FL_REGS_EN;
+                        }
+                }
+
+                /*
+                 * If this record is being updated from a nop, then
+                 *   return UPDATE_MAKE_CALL.
+                 * Otherwise, if the EN flag is set, then return
+                 *   UPDATE_MODIFY_CALL_REGS to tell the caller to convert
+                 *   from the non-save regs, to a save regs function.
+                 * Otherwise,
+                 *   return UPDATE_MODIFY_CALL to tell the caller to convert
+                 *   from the save regs, to a non-save regs function.
+                 */
+                if (flag & FTRACE_FL_ENABLED)
+                        return FTRACE_UPDATE_MAKE_CALL;
+                else if (rec->flags & FTRACE_FL_REGS_EN)
+                        return FTRACE_UPDATE_MODIFY_CALL_REGS;
+                else
+                        return FTRACE_UPDATE_MODIFY_CALL;
         }
 
-        if (update)
-                rec->flags &= ~FTRACE_FL_ENABLED;
+        if (update) {
+                /* If there's no more users, clear all flags */
+                if (!(rec->flags & ~FTRACE_FL_MASK))
+                        rec->flags = 0;
+                else
+                        /* Just disable the record (keep REGS state) */
+                        rec->flags &= ~FTRACE_FL_ENABLED;
+        }
 
         return FTRACE_UPDATE_MAKE_NOP;
 }
@@ -1652,13 +1713,17 @@ int ftrace_test_record(struct dyn_ftrace *rec, int enable)
 static int
 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
 {
+        unsigned long ftrace_old_addr;
         unsigned long ftrace_addr;
         int ret;
 
-        ftrace_addr = (unsigned long)FTRACE_ADDR;
-
         ret = ftrace_update_record(rec, enable);
 
+        if (rec->flags & FTRACE_FL_REGS)
+                ftrace_addr = (unsigned long)FTRACE_REGS_ADDR;
+        else
+                ftrace_addr = (unsigned long)FTRACE_ADDR;
+
         switch (ret) {
         case FTRACE_UPDATE_IGNORE:
                 return 0;
@@ -1668,6 +1733,15 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
 
         case FTRACE_UPDATE_MAKE_NOP:
                 return ftrace_make_nop(NULL, rec, ftrace_addr);
+
+        case FTRACE_UPDATE_MODIFY_CALL_REGS:
+        case FTRACE_UPDATE_MODIFY_CALL:
+                if (rec->flags & FTRACE_FL_REGS)
+                        ftrace_old_addr = (unsigned long)FTRACE_ADDR;
+                else
+                        ftrace_old_addr = (unsigned long)FTRACE_REGS_ADDR;
+
+                return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
         }
 
         return -1; /* unknow ftrace bug */
@@ -2421,8 +2495,9 @@ static int t_show(struct seq_file *m, void *v)
 
         seq_printf(m, "%ps", (void *)rec->ip);
         if (iter->flags & FTRACE_ITER_ENABLED)
-                seq_printf(m, " (%ld)",
-                           rec->flags & ~FTRACE_FL_MASK);
+                seq_printf(m, " (%ld)%s",
+                           rec->flags & ~FTRACE_FL_MASK,
+                           rec->flags & FTRACE_FL_REGS ? " R" : "");
         seq_printf(m, "\n");
 
         return 0;