aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSteven Rostedt (VMware) <rostedt@goodmis.org>2017-03-23 10:33:52 -0400
committerThomas Gleixner <tglx@linutronix.de>2017-03-24 05:14:07 -0400
commit644e0e8dc76b919976c44d3929164d42cbe656bc (patch)
treee11a16a3e85b3138629648d44c3dc08d6bfc18df
parentff04b440d2d645fd8a5b3385b1b2e4d19d3fe746 (diff)
x86/ftrace: Add -mfentry support to x86_32 with DYNAMIC_FTRACE set
x86_64 has had fentry support for some time. I did not add support to x86_32 as I was unsure if it will be used much in the future. It is still very much used, and there's issues with function graph tracing with gcc playing around with the mcount frames, causing function graph to panic. The fentry code does not have this issue, and is able to cope as there is no frame to mess up. Note, this only adds support for fentry when DYNAMIC_FTRACE is set. There's really no reason to not have that set, because the performance of the machine drops significantly when it's not enabled. Keep !DYNAMIC_FTRACE around to test it off, as there's still some archs that have FTRACE but not DYNAMIC_FTRACE. Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org> Reviewed-by: Masami Hiramatsu <mhiramat@kernel.org> Reviewed-by: Josh Poimboeuf <jpoimboe@redhat.com> Reviewed-by: Ingo Molnar <mingo@kernel.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Andy Lutomirski <luto@amacapital.net> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Link: http://lkml.kernel.org/r/20170323143446.052202377@goodmis.org Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
-rw-r--r--arch/x86/Kconfig2
-rw-r--r--arch/x86/kernel/ftrace_32.S82
2 files changed, 73 insertions, 11 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index cc98d5a294ee..8c17146427ca 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -127,7 +127,7 @@ config X86
127 select HAVE_EBPF_JIT if X86_64 127 select HAVE_EBPF_JIT if X86_64
128 select HAVE_EFFICIENT_UNALIGNED_ACCESS 128 select HAVE_EFFICIENT_UNALIGNED_ACCESS
129 select HAVE_EXIT_THREAD 129 select HAVE_EXIT_THREAD
130 select HAVE_FENTRY if X86_64 130 select HAVE_FENTRY if X86_64 || DYNAMIC_FTRACE
131 select HAVE_FTRACE_MCOUNT_RECORD 131 select HAVE_FTRACE_MCOUNT_RECORD
132 select HAVE_FUNCTION_GRAPH_TRACER 132 select HAVE_FUNCTION_GRAPH_TRACER
133 select HAVE_FUNCTION_TRACER 133 select HAVE_FUNCTION_TRACER
diff --git a/arch/x86/kernel/ftrace_32.S b/arch/x86/kernel/ftrace_32.S
index 93e26647c3f2..80518ec5b882 100644
--- a/arch/x86/kernel/ftrace_32.S
+++ b/arch/x86/kernel/ftrace_32.S
@@ -9,26 +9,68 @@
9#include <asm/ftrace.h> 9#include <asm/ftrace.h>
10 10
11#ifdef CONFIG_FUNCTION_TRACER 11#ifdef CONFIG_FUNCTION_TRACER
12
13#ifdef CC_USING_FENTRY
14# define function_hook __fentry__
15EXPORT_SYMBOL(__fentry__)
16#else
17# define function_hook mcount
18EXPORT_SYMBOL(mcount)
19#endif
20
12#ifdef CONFIG_DYNAMIC_FTRACE 21#ifdef CONFIG_DYNAMIC_FTRACE
13 22
14ENTRY(mcount) 23/* mcount uses a frame pointer even if CONFIG_FRAME_POINTER is not set */
24#if !defined(CC_USING_FENTRY) || defined(CONFIG_FRAME_POINTER)
25# define USING_FRAME_POINTER
26#endif
27
28#ifdef USING_FRAME_POINTER
29# define MCOUNT_FRAME 1 /* using frame = true */
30#else
31# define MCOUNT_FRAME 0 /* using frame = false */
32#endif
33
34ENTRY(function_hook)
15 ret 35 ret
16END(mcount) 36END(function_hook)
17 37
18ENTRY(ftrace_caller) 38ENTRY(ftrace_caller)
19 39
40#ifdef USING_FRAME_POINTER
41# ifdef CC_USING_FENTRY
42 /*
43 * Frame pointers are of ip followed by bp.
44 * Since fentry is an immediate jump, we are left with
45 * parent-ip, function-ip. We need to add a frame with
46 * parent-ip followed by ebp.
47 */
48 pushl 4(%esp) /* parent ip */
20 pushl %ebp 49 pushl %ebp
21 movl %esp, %ebp 50 movl %esp, %ebp
22 51 pushl 2*4(%esp) /* function ip */
52# endif
53 /* For mcount, the function ip is directly above */
54 pushl %ebp
55 movl %esp, %ebp
56#endif
23 pushl %eax 57 pushl %eax
24 pushl %ecx 58 pushl %ecx
25 pushl %edx 59 pushl %edx
26 pushl $0 /* Pass NULL as regs pointer */ 60 pushl $0 /* Pass NULL as regs pointer */
27 movl 5*4(%esp), %eax 61
28 /* Copy original ebp into %edx */ 62#ifdef USING_FRAME_POINTER
63 /* Load parent ebp into edx */
29 movl 4*4(%esp), %edx 64 movl 4*4(%esp), %edx
65#else
66 /* There's no frame pointer, load the appropriate stack addr instead */
67 lea 4*4(%esp), %edx
68#endif
69
70 movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */
30 /* Get the parent ip */ 71 /* Get the parent ip */
31 movl 0x4(%edx), %edx 72 movl 4(%edx), %edx /* edx has ebp */
73
32 movl function_trace_op, %ecx 74 movl function_trace_op, %ecx
33 subl $MCOUNT_INSN_SIZE, %eax 75 subl $MCOUNT_INSN_SIZE, %eax
34 76
@@ -40,7 +82,14 @@ ftrace_call:
40 popl %edx 82 popl %edx
41 popl %ecx 83 popl %ecx
42 popl %eax 84 popl %eax
85#ifdef USING_FRAME_POINTER
43 popl %ebp 86 popl %ebp
87# ifdef CC_USING_FENTRY
88 addl $4,%esp /* skip function ip */
89 popl %ebp /* this is the orig bp */
90 addl $4, %esp /* skip parent ip */
91# endif
92#endif
44.Lftrace_ret: 93.Lftrace_ret:
45#ifdef CONFIG_FUNCTION_GRAPH_TRACER 94#ifdef CONFIG_FUNCTION_GRAPH_TRACER
46.globl ftrace_graph_call 95.globl ftrace_graph_call
@@ -81,6 +130,10 @@ ENTRY(ftrace_regs_caller)
81 pushl %edx 130 pushl %edx
82 pushl %ecx 131 pushl %ecx
83 pushl %ebx 132 pushl %ebx
133#ifdef CC_USING_FENTRY
134 /* Load 4 off of the parent ip addr into ebp */
135 lea 14*4(%esp), %ebp
136#endif
84 137
85 movl 12*4(%esp), %eax /* Load ip (1st parameter) */ 138 movl 12*4(%esp), %eax /* Load ip (1st parameter) */
86 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ 139 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */
@@ -119,7 +172,7 @@ GLOBAL(ftrace_regs_call)
119 jmp .Lftrace_ret 172 jmp .Lftrace_ret
120#else /* ! CONFIG_DYNAMIC_FTRACE */ 173#else /* ! CONFIG_DYNAMIC_FTRACE */
121 174
122ENTRY(mcount) 175ENTRY(function_hook)
123 cmpl $__PAGE_OFFSET, %esp 176 cmpl $__PAGE_OFFSET, %esp
124 jb ftrace_stub /* Paging not enabled yet? */ 177 jb ftrace_stub /* Paging not enabled yet? */
125 178
@@ -151,9 +204,8 @@ ftrace_stub:
151 popl %ecx 204 popl %ecx
152 popl %eax 205 popl %eax
153 jmp ftrace_stub 206 jmp ftrace_stub
154END(mcount) 207END(function_hook)
155#endif /* CONFIG_DYNAMIC_FTRACE */ 208#endif /* CONFIG_DYNAMIC_FTRACE */
156EXPORT_SYMBOL(mcount)
157#endif /* CONFIG_FUNCTION_TRACER */ 209#endif /* CONFIG_FUNCTION_TRACER */
158 210
159#ifdef CONFIG_FUNCTION_GRAPH_TRACER 211#ifdef CONFIG_FUNCTION_GRAPH_TRACER
@@ -161,9 +213,15 @@ ENTRY(ftrace_graph_caller)
161 pushl %eax 213 pushl %eax
162 pushl %ecx 214 pushl %ecx
163 pushl %edx 215 pushl %edx
164 movl 0xc(%esp), %eax 216 movl 3*4(%esp), %eax
217 /* Even with frame pointers, fentry doesn't have one here */
218#ifdef CC_USING_FENTRY
219 lea 4*4(%esp), %edx
220 movl $0, %ecx
221#else
165 lea 0x4(%ebp), %edx 222 lea 0x4(%ebp), %edx
166 movl (%ebp), %ecx 223 movl (%ebp), %ecx
224#endif
167 subl $MCOUNT_INSN_SIZE, %eax 225 subl $MCOUNT_INSN_SIZE, %eax
168 call prepare_ftrace_return 226 call prepare_ftrace_return
169 popl %edx 227 popl %edx
@@ -176,7 +234,11 @@ END(ftrace_graph_caller)
176return_to_handler: 234return_to_handler:
177 pushl %eax 235 pushl %eax
178 pushl %edx 236 pushl %edx
237#ifdef CC_USING_FENTRY
238 movl $0, %eax
239#else
179 movl %ebp, %eax 240 movl %ebp, %eax
241#endif
180 call ftrace_return_to_handler 242 call ftrace_return_to_handler
181 movl %eax, %ecx 243 movl %eax, %ecx
182 popl %edx 244 popl %edx