aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAbhishek Sagar <sagar.abhishek@gmail.com>2008-06-21 14:17:27 -0400
committerIngo Molnar <mingo@elte.hu>2008-06-23 16:10:56 -0400
commit395a59d0f8e86bb39cd700c3d185d30c670bb958 (patch)
tree1558e635efcede901c5dbe9acd625d475db5b369
parentf34bfb1beef8a17ba3d46b60f8fa19ffedc1ed8d (diff)
ftrace: store mcount address in rec->ip
Record the address of the mcount call-site. Currently all archs except sparc64 record the address of the instruction following the mcount call-site. Some general cleanups are entailed. Storing mcount addresses in rec->ip enables looking them up in the kprobe hash table later on to check if they're kprobe'd. Signed-off-by: Abhishek Sagar <sagar.abhishek@gmail.com> Cc: davem@davemloft.net Cc: Steven Rostedt <rostedt@goodmis.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
-rw-r--r--arch/arm/kernel/armksyms.c10
-rw-r--r--arch/arm/kernel/entry-common.S4
-rw-r--r--arch/arm/kernel/ftrace.c16
-rw-r--r--arch/powerpc/kernel/entry_32.S4
-rw-r--r--arch/powerpc/kernel/entry_64.S5
-rw-r--r--arch/powerpc/kernel/ftrace.c21
-rw-r--r--arch/sparc64/kernel/ftrace.c10
-rw-r--r--arch/sparc64/kernel/sparc64_ksyms.c2
-rw-r--r--arch/x86/kernel/entry_32.S4
-rw-r--r--arch/x86/kernel/entry_64.S4
-rw-r--r--arch/x86/kernel/ftrace.c26
-rw-r--r--arch/x86/kernel/i386_ksyms_32.c2
-rw-r--r--arch/x86/kernel/x8664_ksyms_64.c2
-rw-r--r--include/asm-arm/ftrace.h14
-rw-r--r--include/asm-powerpc/ftrace.h8
-rw-r--r--include/asm-sparc64/ftrace.h14
-rw-r--r--include/asm-x86/ftrace.h14
-rw-r--r--include/linux/ftrace.h3
-rw-r--r--kernel/trace/ftrace.c3
19 files changed, 110 insertions, 56 deletions
diff --git a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c
index 3b132215cbf8..cc7b246e9652 100644
--- a/arch/arm/kernel/armksyms.c
+++ b/arch/arm/kernel/armksyms.c
@@ -18,6 +18,7 @@
18#include <asm/io.h> 18#include <asm/io.h>
19#include <asm/system.h> 19#include <asm/system.h>
20#include <asm/uaccess.h> 20#include <asm/uaccess.h>
21#include <asm/ftrace.h>
21 22
22/* 23/*
23 * libgcc functions - functions that are used internally by the 24 * libgcc functions - functions that are used internally by the
@@ -48,11 +49,6 @@ extern void __aeabi_ulcmp(void);
48extern void fpundefinstr(void); 49extern void fpundefinstr(void);
49extern void fp_enter(void); 50extern void fp_enter(void);
50 51
51#ifdef CONFIG_FTRACE
52extern void mcount(void);
53EXPORT_SYMBOL(mcount);
54#endif
55
56/* 52/*
57 * This has a special calling convention; it doesn't 53 * This has a special calling convention; it doesn't
58 * modify any of the usual registers, except for LR. 54 * modify any of the usual registers, except for LR.
@@ -186,3 +182,7 @@ EXPORT_SYMBOL(_find_next_bit_be);
186#endif 182#endif
187 183
188EXPORT_SYMBOL(copy_page); 184EXPORT_SYMBOL(copy_page);
185
186#ifdef CONFIG_FTRACE
187EXPORT_SYMBOL(mcount);
188#endif
diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S
index 8f79a4789ed4..84694e88b428 100644
--- a/arch/arm/kernel/entry-common.S
+++ b/arch/arm/kernel/entry-common.S
@@ -9,6 +9,7 @@
9 */ 9 */
10 10
11#include <asm/unistd.h> 11#include <asm/unistd.h>
12#include <asm/ftrace.h>
12#include <asm/arch/entry-macro.S> 13#include <asm/arch/entry-macro.S>
13 14
14#include "entry-header.S" 15#include "entry-header.S"
@@ -104,6 +105,7 @@ ENTRY(ret_from_fork)
104ENTRY(mcount) 105ENTRY(mcount)
105 stmdb sp!, {r0-r3, lr} 106 stmdb sp!, {r0-r3, lr}
106 mov r0, lr 107 mov r0, lr
108 sub r0, r0, #MCOUNT_INSN_SIZE
107 109
108 .globl mcount_call 110 .globl mcount_call
109mcount_call: 111mcount_call:
@@ -114,6 +116,7 @@ ENTRY(ftrace_caller)
114 stmdb sp!, {r0-r3, lr} 116 stmdb sp!, {r0-r3, lr}
115 ldr r1, [fp, #-4] 117 ldr r1, [fp, #-4]
116 mov r0, lr 118 mov r0, lr
119 sub r0, r0, #MCOUNT_INSN_SIZE
117 120
118 .globl ftrace_call 121 .globl ftrace_call
119ftrace_call: 122ftrace_call:
@@ -134,6 +137,7 @@ ENTRY(mcount)
134trace: 137trace:
135 ldr r1, [fp, #-4] 138 ldr r1, [fp, #-4]
136 mov r0, lr 139 mov r0, lr
140 sub r0, r0, #MCOUNT_INSN_SIZE
137 mov lr, pc 141 mov lr, pc
138 mov pc, r2 142 mov pc, r2
139 ldmia sp!, {r0-r3, pc} 143 ldmia sp!, {r0-r3, pc}
diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c
index 22f3d6e309f9..76d50e6091bc 100644
--- a/arch/arm/kernel/ftrace.c
+++ b/arch/arm/kernel/ftrace.c
@@ -12,9 +12,10 @@
12 */ 12 */
13 13
14#include <linux/ftrace.h> 14#include <linux/ftrace.h>
15
15#include <asm/cacheflush.h> 16#include <asm/cacheflush.h>
17#include <asm/ftrace.h>
16 18
17#define INSN_SIZE 4
18#define PC_OFFSET 8 19#define PC_OFFSET 8
19#define BL_OPCODE 0xeb000000 20#define BL_OPCODE 0xeb000000
20#define BL_OFFSET_MASK 0x00ffffff 21#define BL_OFFSET_MASK 0x00ffffff
@@ -32,10 +33,10 @@ unsigned char *ftrace_call_replace(unsigned long pc, unsigned long addr)
32{ 33{
33 long offset; 34 long offset;
34 35
35 offset = (long)addr - (long)(pc - INSN_SIZE + PC_OFFSET); 36 offset = (long)addr - (long)(pc + PC_OFFSET);
36 if (unlikely(offset < -33554432 || offset > 33554428)) { 37 if (unlikely(offset < -33554432 || offset > 33554428)) {
37 /* Can't generate branches that far (from ARM ARM). Ftrace 38 /* Can't generate branches that far (from ARM ARM). Ftrace
38 * doesn't generate branches outside of core kernel text. 39 * doesn't generate branches outside of kernel text.
39 */ 40 */
40 WARN_ON_ONCE(1); 41 WARN_ON_ONCE(1);
41 return NULL; 42 return NULL;
@@ -52,7 +53,6 @@ int ftrace_modify_code(unsigned long pc, unsigned char *old_code,
52 53
53 old = *(unsigned long *)old_code; 54 old = *(unsigned long *)old_code;
54 new = *(unsigned long *)new_code; 55 new = *(unsigned long *)new_code;
55 pc -= INSN_SIZE;
56 56
57 __asm__ __volatile__ ( 57 __asm__ __volatile__ (
58 "1: ldr %1, [%2] \n" 58 "1: ldr %1, [%2] \n"
@@ -77,7 +77,7 @@ int ftrace_modify_code(unsigned long pc, unsigned char *old_code,
77 : "memory"); 77 : "memory");
78 78
79 if (!err && (replaced == old)) 79 if (!err && (replaced == old))
80 flush_icache_range(pc, pc + INSN_SIZE); 80 flush_icache_range(pc, pc + MCOUNT_INSN_SIZE);
81 81
82 return err; 82 return err;
83} 83}
@@ -89,8 +89,7 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
89 unsigned char *new; 89 unsigned char *new;
90 90
91 pc = (unsigned long)&ftrace_call; 91 pc = (unsigned long)&ftrace_call;
92 pc += INSN_SIZE; 92 memcpy(&old, &ftrace_call, MCOUNT_INSN_SIZE);
93 memcpy(&old, &ftrace_call, INSN_SIZE);
94 new = ftrace_call_replace(pc, (unsigned long)func); 93 new = ftrace_call_replace(pc, (unsigned long)func);
95 ret = ftrace_modify_code(pc, (unsigned char *)&old, new); 94 ret = ftrace_modify_code(pc, (unsigned char *)&old, new);
96 return ret; 95 return ret;
@@ -103,8 +102,7 @@ int ftrace_mcount_set(unsigned long *data)
103 unsigned char *new; 102 unsigned char *new;
104 103
105 pc = (unsigned long)&mcount_call; 104 pc = (unsigned long)&mcount_call;
106 pc += INSN_SIZE; 105 memcpy(&old, &mcount_call, MCOUNT_INSN_SIZE);
107 memcpy(&old, &mcount_call, INSN_SIZE);
108 new = ftrace_call_replace(pc, *addr); 106 new = ftrace_call_replace(pc, *addr);
109 *addr = ftrace_modify_code(pc, (unsigned char *)&old, new); 107 *addr = ftrace_modify_code(pc, (unsigned char *)&old, new);
110 return 0; 108 return 0;
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index 3b1dd29d9f91..7231a708af0d 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -30,6 +30,7 @@
30#include <asm/ppc_asm.h> 30#include <asm/ppc_asm.h>
31#include <asm/asm-offsets.h> 31#include <asm/asm-offsets.h>
32#include <asm/unistd.h> 32#include <asm/unistd.h>
33#include <asm/ftrace.h>
33 34
34#undef SHOW_SYSCALLS 35#undef SHOW_SYSCALLS
35#undef SHOW_SYSCALLS_TASK 36#undef SHOW_SYSCALLS_TASK
@@ -1053,6 +1054,7 @@ _GLOBAL(_mcount)
1053 stw r10,40(r1) 1054 stw r10,40(r1)
1054 stw r3, 44(r1) 1055 stw r3, 44(r1)
1055 stw r5, 8(r1) 1056 stw r5, 8(r1)
1057 subi r3, r3, MCOUNT_INSN_SIZE
1056 .globl mcount_call 1058 .globl mcount_call
1057mcount_call: 1059mcount_call:
1058 bl ftrace_stub 1060 bl ftrace_stub
@@ -1090,6 +1092,7 @@ _GLOBAL(ftrace_caller)
1090 stw r10,40(r1) 1092 stw r10,40(r1)
1091 stw r3, 44(r1) 1093 stw r3, 44(r1)
1092 stw r5, 8(r1) 1094 stw r5, 8(r1)
1095 subi r3, r3, MCOUNT_INSN_SIZE
1093.globl ftrace_call 1096.globl ftrace_call
1094ftrace_call: 1097ftrace_call:
1095 bl ftrace_stub 1098 bl ftrace_stub
@@ -1128,6 +1131,7 @@ _GLOBAL(_mcount)
1128 stw r3, 44(r1) 1131 stw r3, 44(r1)
1129 stw r5, 8(r1) 1132 stw r5, 8(r1)
1130 1133
1134 subi r3, r3, MCOUNT_INSN_SIZE
1131 LOAD_REG_ADDR(r5, ftrace_trace_function) 1135 LOAD_REG_ADDR(r5, ftrace_trace_function)
1132 lwz r5,0(r5) 1136 lwz r5,0(r5)
1133 1137
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 2c4d9e056ead..2f511a969d2c 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -31,6 +31,7 @@
31#include <asm/bug.h> 31#include <asm/bug.h>
32#include <asm/ptrace.h> 32#include <asm/ptrace.h>
33#include <asm/irqflags.h> 33#include <asm/irqflags.h>
34#include <asm/ftrace.h>
34 35
35/* 36/*
36 * System calls. 37 * System calls.
@@ -879,6 +880,7 @@ _GLOBAL(_mcount)
879 mflr r3 880 mflr r3
880 stdu r1, -112(r1) 881 stdu r1, -112(r1)
881 std r3, 128(r1) 882 std r3, 128(r1)
883 subi r3, r3, MCOUNT_INSN_SIZE
882 .globl mcount_call 884 .globl mcount_call
883mcount_call: 885mcount_call:
884 bl ftrace_stub 886 bl ftrace_stub
@@ -895,6 +897,7 @@ _GLOBAL(ftrace_caller)
895 stdu r1, -112(r1) 897 stdu r1, -112(r1)
896 std r3, 128(r1) 898 std r3, 128(r1)
897 ld r4, 16(r11) 899 ld r4, 16(r11)
900 subi r3, r3, MCOUNT_INSN_SIZE
898.globl ftrace_call 901.globl ftrace_call
899ftrace_call: 902ftrace_call:
900 bl ftrace_stub 903 bl ftrace_stub
@@ -916,7 +919,7 @@ _GLOBAL(_mcount)
916 std r3, 128(r1) 919 std r3, 128(r1)
917 ld r4, 16(r11) 920 ld r4, 16(r11)
918 921
919 922 subi r3, r3, MCOUNT_INSN_SIZE
920 LOAD_REG_ADDR(r5,ftrace_trace_function) 923 LOAD_REG_ADDR(r5,ftrace_trace_function)
921 ld r5,0(r5) 924 ld r5,0(r5)
922 ld r5,0(r5) 925 ld r5,0(r5)
diff --git a/arch/powerpc/kernel/ftrace.c b/arch/powerpc/kernel/ftrace.c
index e12c593ab9ca..3855ceb937b0 100644
--- a/arch/powerpc/kernel/ftrace.c
+++ b/arch/powerpc/kernel/ftrace.c
@@ -15,8 +15,8 @@
15#include <linux/list.h> 15#include <linux/list.h>
16 16
17#include <asm/cacheflush.h> 17#include <asm/cacheflush.h>
18#include <asm/ftrace.h>
18 19
19#define CALL_BACK 4
20 20
21static unsigned int ftrace_nop = 0x60000000; 21static unsigned int ftrace_nop = 0x60000000;
22 22
@@ -27,9 +27,10 @@ static unsigned int ftrace_nop = 0x60000000;
27# define GET_ADDR(addr) *(unsigned long *)addr 27# define GET_ADDR(addr) *(unsigned long *)addr
28#endif 28#endif
29 29
30
30static unsigned int notrace ftrace_calc_offset(long ip, long addr) 31static unsigned int notrace ftrace_calc_offset(long ip, long addr)
31{ 32{
32 return (int)((addr + CALL_BACK) - ip); 33 return (int)(addr - ip);
33} 34}
34 35
35notrace unsigned char *ftrace_nop_replace(void) 36notrace unsigned char *ftrace_nop_replace(void)
@@ -76,9 +77,6 @@ ftrace_modify_code(unsigned long ip, unsigned char *old_code,
76 unsigned new = *(unsigned *)new_code; 77 unsigned new = *(unsigned *)new_code;
77 int faulted = 0; 78 int faulted = 0;
78 79
79 /* move the IP back to the start of the call */
80 ip -= CALL_BACK;
81
82 /* 80 /*
83 * Note: Due to modules and __init, code can 81 * Note: Due to modules and __init, code can
84 * disappear and change, we need to protect against faulting 82 * disappear and change, we need to protect against faulting
@@ -118,12 +116,10 @@ ftrace_modify_code(unsigned long ip, unsigned char *old_code,
118notrace int ftrace_update_ftrace_func(ftrace_func_t func) 116notrace int ftrace_update_ftrace_func(ftrace_func_t func)
119{ 117{
120 unsigned long ip = (unsigned long)(&ftrace_call); 118 unsigned long ip = (unsigned long)(&ftrace_call);
121 unsigned char old[4], *new; 119 unsigned char old[MCOUNT_INSN_SIZE], *new;
122 int ret; 120 int ret;
123 121
124 ip += CALL_BACK; 122 memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
125
126 memcpy(old, &ftrace_call, 4);
127 new = ftrace_call_replace(ip, (unsigned long)func); 123 new = ftrace_call_replace(ip, (unsigned long)func);
128 ret = ftrace_modify_code(ip, old, new); 124 ret = ftrace_modify_code(ip, old, new);
129 125
@@ -134,16 +130,13 @@ notrace int ftrace_mcount_set(unsigned long *data)
134{ 130{
135 unsigned long ip = (long)(&mcount_call); 131 unsigned long ip = (long)(&mcount_call);
136 unsigned long *addr = data; 132 unsigned long *addr = data;
137 unsigned char old[4], *new; 133 unsigned char old[MCOUNT_INSN_SIZE], *new;
138
139 /* ip is at the location, but modify code will subtact this */
140 ip += CALL_BACK;
141 134
142 /* 135 /*
143 * Replace the mcount stub with a pointer to the 136 * Replace the mcount stub with a pointer to the
144 * ip recorder function. 137 * ip recorder function.
145 */ 138 */
146 memcpy(old, &mcount_call, 4); 139 memcpy(old, &mcount_call, MCOUNT_INSN_SIZE);
147 new = ftrace_call_replace(ip, *addr); 140 new = ftrace_call_replace(ip, *addr);
148 *addr = ftrace_modify_code(ip, old, new); 141 *addr = ftrace_modify_code(ip, old, new);
149 142
diff --git a/arch/sparc64/kernel/ftrace.c b/arch/sparc64/kernel/ftrace.c
index c17373195b1e..4298d0aee713 100644
--- a/arch/sparc64/kernel/ftrace.c
+++ b/arch/sparc64/kernel/ftrace.c
@@ -5,6 +5,8 @@
5#include <linux/init.h> 5#include <linux/init.h>
6#include <linux/list.h> 6#include <linux/list.h>
7 7
8#include <asm/ftrace.h>
9
8static const u32 ftrace_nop = 0x01000000; 10static const u32 ftrace_nop = 0x01000000;
9 11
10notrace unsigned char *ftrace_nop_replace(void) 12notrace unsigned char *ftrace_nop_replace(void)
@@ -60,9 +62,9 @@ ftrace_modify_code(unsigned long ip, unsigned char *old_code,
60notrace int ftrace_update_ftrace_func(ftrace_func_t func) 62notrace int ftrace_update_ftrace_func(ftrace_func_t func)
61{ 63{
62 unsigned long ip = (unsigned long)(&ftrace_call); 64 unsigned long ip = (unsigned long)(&ftrace_call);
63 unsigned char old[4], *new; 65 unsigned char old[MCOUNT_INSN_SIZE], *new;
64 66
65 memcpy(old, &ftrace_call, 4); 67 memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
66 new = ftrace_call_replace(ip, (unsigned long)func); 68 new = ftrace_call_replace(ip, (unsigned long)func);
67 return ftrace_modify_code(ip, old, new); 69 return ftrace_modify_code(ip, old, new);
68} 70}
@@ -71,13 +73,13 @@ notrace int ftrace_mcount_set(unsigned long *data)
71{ 73{
72 unsigned long ip = (long)(&mcount_call); 74 unsigned long ip = (long)(&mcount_call);
73 unsigned long *addr = data; 75 unsigned long *addr = data;
74 unsigned char old[4], *new; 76 unsigned char old[MCOUNT_INSN_SIZE], *new;
75 77
76 /* 78 /*
77 * Replace the mcount stub with a pointer to the 79 * Replace the mcount stub with a pointer to the
78 * ip recorder function. 80 * ip recorder function.
79 */ 81 */
80 memcpy(old, &mcount_call, 4); 82 memcpy(old, &mcount_call, MCOUNT_INSN_SIZE);
81 new = ftrace_call_replace(ip, *addr); 83 new = ftrace_call_replace(ip, *addr);
82 *addr = ftrace_modify_code(ip, old, new); 84 *addr = ftrace_modify_code(ip, old, new);
83 85
diff --git a/arch/sparc64/kernel/sparc64_ksyms.c b/arch/sparc64/kernel/sparc64_ksyms.c
index 8ac0b99f2c55..b80d982a29c6 100644
--- a/arch/sparc64/kernel/sparc64_ksyms.c
+++ b/arch/sparc64/kernel/sparc64_ksyms.c
@@ -53,6 +53,7 @@
53#include <asm/ns87303.h> 53#include <asm/ns87303.h>
54#include <asm/timer.h> 54#include <asm/timer.h>
55#include <asm/cpudata.h> 55#include <asm/cpudata.h>
56#include <asm/ftrace.h>
56 57
57struct poll { 58struct poll {
58 int fd; 59 int fd;
@@ -112,7 +113,6 @@ EXPORT_SYMBOL(smp_call_function);
112#endif /* CONFIG_SMP */ 113#endif /* CONFIG_SMP */
113 114
114#if defined(CONFIG_MCOUNT) 115#if defined(CONFIG_MCOUNT)
115extern void _mcount(void);
116EXPORT_SYMBOL(_mcount); 116EXPORT_SYMBOL(_mcount);
117#endif 117#endif
118 118
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 04ea83ccb979..95e6bbe3665e 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -51,6 +51,7 @@
51#include <asm/percpu.h> 51#include <asm/percpu.h>
52#include <asm/dwarf2.h> 52#include <asm/dwarf2.h>
53#include <asm/processor-flags.h> 53#include <asm/processor-flags.h>
54#include <asm/ftrace.h>
54#include "irq_vectors.h" 55#include "irq_vectors.h"
55 56
56/* 57/*
@@ -1118,6 +1119,7 @@ ENTRY(mcount)
1118 pushl %ecx 1119 pushl %ecx
1119 pushl %edx 1120 pushl %edx
1120 movl 0xc(%esp), %eax 1121 movl 0xc(%esp), %eax
1122 subl $MCOUNT_INSN_SIZE, %eax
1121 1123
1122.globl mcount_call 1124.globl mcount_call
1123mcount_call: 1125mcount_call:
@@ -1136,6 +1138,7 @@ ENTRY(ftrace_caller)
1136 pushl %edx 1138 pushl %edx
1137 movl 0xc(%esp), %eax 1139 movl 0xc(%esp), %eax
1138 movl 0x4(%ebp), %edx 1140 movl 0x4(%ebp), %edx
1141 subl $MCOUNT_INSN_SIZE, %eax
1139 1142
1140.globl ftrace_call 1143.globl ftrace_call
1141ftrace_call: 1144ftrace_call:
@@ -1166,6 +1169,7 @@ trace:
1166 pushl %edx 1169 pushl %edx
1167 movl 0xc(%esp), %eax 1170 movl 0xc(%esp), %eax
1168 movl 0x4(%ebp), %edx 1171 movl 0x4(%ebp), %edx
1172 subl $MCOUNT_INSN_SIZE, %eax
1169 1173
1170 call *ftrace_trace_function 1174 call *ftrace_trace_function
1171 1175
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index fe25e5febca3..b0f7308f78a6 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -51,6 +51,7 @@
51#include <asm/page.h> 51#include <asm/page.h>
52#include <asm/irqflags.h> 52#include <asm/irqflags.h>
53#include <asm/paravirt.h> 53#include <asm/paravirt.h>
54#include <asm/ftrace.h>
54 55
55 .code64 56 .code64
56 57
@@ -68,6 +69,7 @@ ENTRY(mcount)
68 movq %r9, 48(%rsp) 69 movq %r9, 48(%rsp)
69 70
70 movq 0x38(%rsp), %rdi 71 movq 0x38(%rsp), %rdi
72 subq $MCOUNT_INSN_SIZE, %rdi
71 73
72.globl mcount_call 74.globl mcount_call
73mcount_call: 75mcount_call:
@@ -99,6 +101,7 @@ ENTRY(ftrace_caller)
99 101
100 movq 0x38(%rsp), %rdi 102 movq 0x38(%rsp), %rdi
101 movq 8(%rbp), %rsi 103 movq 8(%rbp), %rsi
104 subq $MCOUNT_INSN_SIZE, %rdi
102 105
103.globl ftrace_call 106.globl ftrace_call
104ftrace_call: 107ftrace_call:
@@ -139,6 +142,7 @@ trace:
139 142
140 movq 0x38(%rsp), %rdi 143 movq 0x38(%rsp), %rdi
141 movq 8(%rbp), %rsi 144 movq 8(%rbp), %rsi
145 subq $MCOUNT_INSN_SIZE, %rdi
142 146
143 call *ftrace_trace_function 147 call *ftrace_trace_function
144 148
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 55828149e01e..ab115cd15fdf 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -17,20 +17,21 @@
17#include <linux/list.h> 17#include <linux/list.h>
18 18
19#include <asm/alternative.h> 19#include <asm/alternative.h>
20#include <asm/ftrace.h>
20 21
21#define CALL_BACK 5
22 22
23/* Long is fine, even if it is only 4 bytes ;-) */ 23/* Long is fine, even if it is only 4 bytes ;-) */
24static long *ftrace_nop; 24static long *ftrace_nop;
25 25
26union ftrace_code_union { 26union ftrace_code_union {
27 char code[5]; 27 char code[MCOUNT_INSN_SIZE];
28 struct { 28 struct {
29 char e8; 29 char e8;
30 int offset; 30 int offset;
31 } __attribute__((packed)); 31 } __attribute__((packed));
32}; 32};
33 33
34
34static int notrace ftrace_calc_offset(long ip, long addr) 35static int notrace ftrace_calc_offset(long ip, long addr)
35{ 36{
36 return (int)(addr - ip); 37 return (int)(addr - ip);
@@ -46,7 +47,7 @@ notrace unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
46 static union ftrace_code_union calc; 47 static union ftrace_code_union calc;
47 48
48 calc.e8 = 0xe8; 49 calc.e8 = 0xe8;
49 calc.offset = ftrace_calc_offset(ip, addr); 50 calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
50 51
51 /* 52 /*
52 * No locking needed, this must be called via kstop_machine 53 * No locking needed, this must be called via kstop_machine
@@ -65,9 +66,6 @@ ftrace_modify_code(unsigned long ip, unsigned char *old_code,
65 unsigned char newch = new_code[4]; 66 unsigned char newch = new_code[4];
66 int faulted = 0; 67 int faulted = 0;
67 68
68 /* move the IP back to the start of the call */
69 ip -= CALL_BACK;
70
71 /* 69 /*
72 * Note: Due to modules and __init, code can 70 * Note: Due to modules and __init, code can
73 * disappear and change, we need to protect against faulting 71 * disappear and change, we need to protect against faulting
@@ -102,12 +100,10 @@ ftrace_modify_code(unsigned long ip, unsigned char *old_code,
102notrace int ftrace_update_ftrace_func(ftrace_func_t func) 100notrace int ftrace_update_ftrace_func(ftrace_func_t func)
103{ 101{
104 unsigned long ip = (unsigned long)(&ftrace_call); 102 unsigned long ip = (unsigned long)(&ftrace_call);
105 unsigned char old[5], *new; 103 unsigned char old[MCOUNT_INSN_SIZE], *new;
106 int ret; 104 int ret;
107 105
108 ip += CALL_BACK; 106 memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
109
110 memcpy(old, &ftrace_call, 5);
111 new = ftrace_call_replace(ip, (unsigned long)func); 107 new = ftrace_call_replace(ip, (unsigned long)func);
112 ret = ftrace_modify_code(ip, old, new); 108 ret = ftrace_modify_code(ip, old, new);
113 109
@@ -118,16 +114,13 @@ notrace int ftrace_mcount_set(unsigned long *data)
118{ 114{
119 unsigned long ip = (long)(&mcount_call); 115 unsigned long ip = (long)(&mcount_call);
120 unsigned long *addr = data; 116 unsigned long *addr = data;
121 unsigned char old[5], *new; 117 unsigned char old[MCOUNT_INSN_SIZE], *new;
122
123 /* ip is at the location, but modify code will subtact this */
124 ip += CALL_BACK;
125 118
126 /* 119 /*
127 * Replace the mcount stub with a pointer to the 120 * Replace the mcount stub with a pointer to the
128 * ip recorder function. 121 * ip recorder function.
129 */ 122 */
130 memcpy(old, &mcount_call, 5); 123 memcpy(old, &mcount_call, MCOUNT_INSN_SIZE);
131 new = ftrace_call_replace(ip, *addr); 124 new = ftrace_call_replace(ip, *addr);
132 *addr = ftrace_modify_code(ip, old, new); 125 *addr = ftrace_modify_code(ip, old, new);
133 126
@@ -142,8 +135,7 @@ int __init ftrace_dyn_arch_init(void *data)
142 135
143 ftrace_mcount_set(data); 136 ftrace_mcount_set(data);
144 137
145 ftrace_nop = (unsigned long *)noptable[CALL_BACK]; 138 ftrace_nop = (unsigned long *)noptable[MCOUNT_INSN_SIZE];
146 139
147 return 0; 140 return 0;
148} 141}
149
diff --git a/arch/x86/kernel/i386_ksyms_32.c b/arch/x86/kernel/i386_ksyms_32.c
index 29999dbb754c..dd7ebee446af 100644
--- a/arch/x86/kernel/i386_ksyms_32.c
+++ b/arch/x86/kernel/i386_ksyms_32.c
@@ -1,9 +1,9 @@
1#include <linux/ftrace.h>
2#include <linux/module.h> 1#include <linux/module.h>
3 2
4#include <asm/checksum.h> 3#include <asm/checksum.h>
5#include <asm/pgtable.h> 4#include <asm/pgtable.h>
6#include <asm/desc.h> 5#include <asm/desc.h>
6#include <asm/ftrace.h>
7 7
8#ifdef CONFIG_FTRACE 8#ifdef CONFIG_FTRACE
9/* mcount is defined in assembly */ 9/* mcount is defined in assembly */
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c
index 122885bc5f3b..16ff4bf418d9 100644
--- a/arch/x86/kernel/x8664_ksyms_64.c
+++ b/arch/x86/kernel/x8664_ksyms_64.c
@@ -1,7 +1,6 @@
1/* Exports for assembly files. 1/* Exports for assembly files.
2 All C exports should go in the respective C files. */ 2 All C exports should go in the respective C files. */
3 3
4#include <linux/ftrace.h>
5#include <linux/module.h> 4#include <linux/module.h>
6#include <linux/smp.h> 5#include <linux/smp.h>
7 6
@@ -11,6 +10,7 @@
11#include <asm/pgtable.h> 10#include <asm/pgtable.h>
12#include <asm/uaccess.h> 11#include <asm/uaccess.h>
13#include <asm/desc.h> 12#include <asm/desc.h>
13#include <asm/ftrace.h>
14 14
15#ifdef CONFIG_FTRACE 15#ifdef CONFIG_FTRACE
16/* mcount is defined in assembly */ 16/* mcount is defined in assembly */
diff --git a/include/asm-arm/ftrace.h b/include/asm-arm/ftrace.h
new file mode 100644
index 000000000000..584ef9a8e5a5
--- /dev/null
+++ b/include/asm-arm/ftrace.h
@@ -0,0 +1,14 @@
1#ifndef _ASM_ARM_FTRACE
2#define _ASM_ARM_FTRACE
3
4#ifdef CONFIG_FTRACE
5#define MCOUNT_ADDR ((long)(mcount))
6#define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */
7
8#ifndef __ASSEMBLY__
9extern void mcount(void);
10#endif
11
12#endif
13
14#endif /* _ASM_ARM_FTRACE */
diff --git a/include/asm-powerpc/ftrace.h b/include/asm-powerpc/ftrace.h
index b1bfa704b6e5..de921326cca8 100644
--- a/include/asm-powerpc/ftrace.h
+++ b/include/asm-powerpc/ftrace.h
@@ -1,6 +1,14 @@
1#ifndef _ASM_POWERPC_FTRACE 1#ifndef _ASM_POWERPC_FTRACE
2#define _ASM_POWERPC_FTRACE 2#define _ASM_POWERPC_FTRACE
3 3
4#ifdef CONFIG_FTRACE
5#define MCOUNT_ADDR ((long)(_mcount))
6#define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */
7
8#ifndef __ASSEMBLY__
4extern void _mcount(void); 9extern void _mcount(void);
10#endif
5 11
6#endif 12#endif
13
14#endif /* _ASM_POWERPC_FTRACE */
diff --git a/include/asm-sparc64/ftrace.h b/include/asm-sparc64/ftrace.h
new file mode 100644
index 000000000000..f76a40a338bb
--- /dev/null
+++ b/include/asm-sparc64/ftrace.h
@@ -0,0 +1,14 @@
1#ifndef _ASM_SPARC64_FTRACE
2#define _ASM_SPARC64_FTRACE
3
4#ifdef CONFIG_FTRACE
5#define MCOUNT_ADDR ((long)(_mcount))
6#define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */
7
8#ifndef __ASSEMBLY__
9extern void _mcount(void);
10#endif
11
12#endif
13
14#endif /* _ASM_SPARC64_FTRACE */
diff --git a/include/asm-x86/ftrace.h b/include/asm-x86/ftrace.h
new file mode 100644
index 000000000000..c184441133f2
--- /dev/null
+++ b/include/asm-x86/ftrace.h
@@ -0,0 +1,14 @@
1#ifndef _ASM_X86_FTRACE
2#define _ASM_SPARC64_FTRACE
3
4#ifdef CONFIG_FTRACE
5#define MCOUNT_ADDR ((long)(mcount))
6#define MCOUNT_INSN_SIZE 5 /* sizeof mcount call */
7
8#ifndef __ASSEMBLY__
9extern void mcount(void);
10#endif
11
12#endif /* CONFIG_FTRACE */
13
14#endif /* _ASM_X86_FTRACE */
diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index 20e14d0093c7..366098d591de 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -31,7 +31,6 @@ int unregister_ftrace_function(struct ftrace_ops *ops);
31void clear_ftrace_function(void); 31void clear_ftrace_function(void);
32 32
33extern void ftrace_stub(unsigned long a0, unsigned long a1); 33extern void ftrace_stub(unsigned long a0, unsigned long a1);
34extern void mcount(void);
35 34
36#else /* !CONFIG_FTRACE */ 35#else /* !CONFIG_FTRACE */
37# define register_ftrace_function(ops) do { } while (0) 36# define register_ftrace_function(ops) do { } while (0)
@@ -54,7 +53,7 @@ enum {
54 53
55struct dyn_ftrace { 54struct dyn_ftrace {
56 struct hlist_node node; 55 struct hlist_node node;
57 unsigned long ip; 56 unsigned long ip; /* address of mcount call-site */
58 unsigned long flags; 57 unsigned long flags;
59}; 58};
60 59
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 0d5bcf69952d..f1e9e5c74e64 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -27,6 +27,8 @@
27#include <linux/hash.h> 27#include <linux/hash.h>
28#include <linux/list.h> 28#include <linux/list.h>
29 29
30#include <asm/ftrace.h>
31
30#include "trace.h" 32#include "trace.h"
31 33
32/* ftrace_enabled is a method to turn ftrace on or off */ 34/* ftrace_enabled is a method to turn ftrace on or off */
@@ -329,7 +331,6 @@ ftrace_record_ip(unsigned long ip)
329} 331}
330 332
331#define FTRACE_ADDR ((long)(ftrace_caller)) 333#define FTRACE_ADDR ((long)(ftrace_caller))
332#define MCOUNT_ADDR ((long)(mcount))
333 334
334static int 335static int
335__ftrace_replace_code(struct dyn_ftrace *rec, 336__ftrace_replace_code(struct dyn_ftrace *rec,