1 files changed, 47 insertions, 36 deletions

diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index d4bdd253fea7..e6253195a301 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -77,8 +77,7 @@ within(unsigned long addr, unsigned long start, unsigned long end)
         return addr >= start && addr < end;
 }
 
-static int
-do_ftrace_mod_code(unsigned long ip, const void *new_code)
+static unsigned long text_ip_addr(unsigned long ip)
 {
         /*
          * On x86_64, kernel text mappings are mapped read-only with
@@ -91,7 +90,7 @@ do_ftrace_mod_code(unsigned long ip, const void *new_code)
         if (within(ip, (unsigned long)_text, (unsigned long)_etext))
                 ip = (unsigned long)__va(__pa_symbol(ip));
 
-        return probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE);
+        return ip;
 }
 
 static const unsigned char *ftrace_nop_replace(void)
@@ -123,8 +122,10 @@ ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
         if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
                 return -EINVAL;
 
+        ip = text_ip_addr(ip);
+
         /* replace the text with the new text */
-        if (do_ftrace_mod_code(ip, new_code))
+        if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
                 return -EPERM;
 
         sync_core();
@@ -221,37 +222,51 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
         return -EINVAL;
 }
 
-int ftrace_update_ftrace_func(ftrace_func_t func)
+static unsigned long ftrace_update_func;
+
+static int update_ftrace_func(unsigned long ip, void *new)
 {
-        unsigned long ip = (unsigned long)(&ftrace_call);
-        unsigned char old[MCOUNT_INSN_SIZE], *new;
+        unsigned char old[MCOUNT_INSN_SIZE];
         int ret;
 
-        memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
-        new = ftrace_call_replace(ip, (unsigned long)func);
+        memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
+
+        ftrace_update_func = ip;
+        /* Make sure the breakpoints see the ftrace_update_func update */
+        smp_wmb();
 
         /* See comment above by declaration of modifying_ftrace_code */
         atomic_inc(&modifying_ftrace_code);
 
         ret = ftrace_modify_code(ip, old, new);
 
+        atomic_dec(&modifying_ftrace_code);
+
+        return ret;
+}
+
+int ftrace_update_ftrace_func(ftrace_func_t func)
+{
+        unsigned long ip = (unsigned long)(&ftrace_call);
+        unsigned char *new;
+        int ret;
+
+        new = ftrace_call_replace(ip, (unsigned long)func);
+        ret = update_ftrace_func(ip, new);
+
         /* Also update the regs callback function */
         if (!ret) {
                 ip = (unsigned long)(&ftrace_regs_call);
-                memcpy(old, &ftrace_regs_call, MCOUNT_INSN_SIZE);
                 new = ftrace_call_replace(ip, (unsigned long)func);
-                ret = ftrace_modify_code(ip, old, new);
+                ret = update_ftrace_func(ip, new);
         }
 
-        atomic_dec(&modifying_ftrace_code);
-
         return ret;
 }
 
 static int is_ftrace_caller(unsigned long ip)
 {
-        if (ip == (unsigned long)(&ftrace_call) ||
-                ip == (unsigned long)(&ftrace_regs_call))
+        if (ip == ftrace_update_func)
                 return 1;
 
         return 0;
@@ -677,45 +692,41 @@ int __init ftrace_dyn_arch_init(void *data)
 #ifdef CONFIG_DYNAMIC_FTRACE
 extern void ftrace_graph_call(void);
 
-static int ftrace_mod_jmp(unsigned long ip,
-                          int old_offset, int new_offset)
+static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
 {
-        unsigned char code[MCOUNT_INSN_SIZE];
+        static union ftrace_code_union calc;
 
-        if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE))
-                return -EFAULT;
+        /* Jmp not a call (ignore the .e8) */
+        calc.e8         = 0xe9;
+        calc.offset     = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
 
-        if (code[0] != 0xe9 || old_offset != *(int *)(&code[1]))
-                return -EINVAL;
+        /*
+         * ftrace external locks synchronize the access to the static variable.
+         */
+        return calc.code;
+}
 
-        *(int *)(&code[1]) = new_offset;
+static int ftrace_mod_jmp(unsigned long ip, void *func)
+{
+        unsigned char *new;
 
-        if (do_ftrace_mod_code(ip, &code))
-                return -EPERM;
+        new = ftrace_jmp_replace(ip, (unsigned long)func);
 
-        return 0;
+        return update_ftrace_func(ip, new);
 }
 
 int ftrace_enable_ftrace_graph_caller(void)
 {
         unsigned long ip = (unsigned long)(&ftrace_graph_call);
-        int old_offset, new_offset;
 
-        old_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
-        new_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
-
-        return ftrace_mod_jmp(ip, old_offset, new_offset);
+        return ftrace_mod_jmp(ip, &ftrace_graph_caller);
 }
 
 int ftrace_disable_ftrace_graph_caller(void)
 {
         unsigned long ip = (unsigned long)(&ftrace_graph_call);
-        int old_offset, new_offset;
-
-        old_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
-        new_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
 
-        return ftrace_mod_jmp(ip, old_offset, new_offset);
+        return ftrace_mod_jmp(ip, &ftrace_stub);
 }
 
 #endif /* !CONFIG_DYNAMIC_FTRACE */