aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/asm-generic/vmlinux.lds.h9
-rw-r--r--include/linux/compiler.h24
-rw-r--r--kernel/trace/Kconfig3
-rw-r--r--kernel/trace/trace_branch.c39
4 files changed, 22 insertions, 53 deletions
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 3b46ae464933..8bccb49981e5 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -46,12 +46,9 @@
46#endif 46#endif
47 47
48#ifdef CONFIG_TRACE_BRANCH_PROFILING 48#ifdef CONFIG_TRACE_BRANCH_PROFILING
49#define LIKELY_PROFILE() VMLINUX_SYMBOL(__start_likely_profile) = .; \ 49#define LIKELY_PROFILE() VMLINUX_SYMBOL(__start_annotated_branch_profile) = .; \
50 *(_ftrace_likely) \ 50 *(_ftrace_annotated_branch) \
51 VMLINUX_SYMBOL(__stop_likely_profile) = .; \ 51 VMLINUX_SYMBOL(__stop_annotated_branch_profile) = .;
52 VMLINUX_SYMBOL(__start_unlikely_profile) = .; \
53 *(_ftrace_unlikely) \
54 VMLINUX_SYMBOL(__stop_unlikely_profile) = .;
55#else 52#else
56#define LIKELY_PROFILE() 53#define LIKELY_PROFILE()
57#endif 54#endif
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index c25e525121f0..0628a2013fae 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -77,32 +77,18 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);
77#define likely_notrace(x) __builtin_expect(!!(x), 1) 77#define likely_notrace(x) __builtin_expect(!!(x), 1)
78#define unlikely_notrace(x) __builtin_expect(!!(x), 0) 78#define unlikely_notrace(x) __builtin_expect(!!(x), 0)
79 79
80#define likely_check(x) ({ \ 80#define __branch_check__(x, expect) ({ \
81 int ______r; \ 81 int ______r; \
82 static struct ftrace_branch_data \ 82 static struct ftrace_branch_data \
83 __attribute__((__aligned__(4))) \ 83 __attribute__((__aligned__(4))) \
84 __attribute__((section("_ftrace_likely"))) \ 84 __attribute__((section("_ftrace_annotated_branch"))) \
85 ______f = { \ 85 ______f = { \
86 .func = __func__, \ 86 .func = __func__, \
87 .file = __FILE__, \ 87 .file = __FILE__, \
88 .line = __LINE__, \ 88 .line = __LINE__, \
89 }; \ 89 }; \
90 ______r = likely_notrace(x); \ 90 ______r = likely_notrace(x); \
91 ftrace_likely_update(&______f, ______r, 1); \ 91 ftrace_likely_update(&______f, ______r, expect); \
92 ______r; \
93 })
94#define unlikely_check(x) ({ \
95 int ______r; \
96 static struct ftrace_branch_data \
97 __attribute__((__aligned__(4))) \
98 __attribute__((section("_ftrace_unlikely"))) \
99 ______f = { \
100 .func = __func__, \
101 .file = __FILE__, \
102 .line = __LINE__, \
103 }; \
104 ______r = unlikely_notrace(x); \
105 ftrace_likely_update(&______f, ______r, 0); \
106 ______r; \ 92 ______r; \
107 }) 93 })
108 94
@@ -112,10 +98,10 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);
112 * written by Daniel Walker. 98 * written by Daniel Walker.
113 */ 99 */
114# ifndef likely 100# ifndef likely
115# define likely(x) (__builtin_constant_p(x) ? !!(x) : likely_check(x)) 101# define likely(x) (__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 1))
116# endif 102# endif
117# ifndef unlikely 103# ifndef unlikely
118# define unlikely(x) (__builtin_constant_p(x) ? !!(x) : unlikely_check(x)) 104# define unlikely(x) (__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 0))
119# endif 105# endif
120#else 106#else
121# define likely(x) __builtin_expect(!!(x), 1) 107# define likely(x) __builtin_expect(!!(x), 1)
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index b8378fad29a3..7e3548705708 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -166,8 +166,7 @@ config TRACE_BRANCH_PROFILING
166 This tracer profiles all the the likely and unlikely macros 166 This tracer profiles all the the likely and unlikely macros
167 in the kernel. It will display the results in: 167 in the kernel. It will display the results in:
168 168
169 /debugfs/tracing/profile_likely 169 /debugfs/tracing/profile_annotated_branch
170 /debugfs/tracing/profile_unlikely
171 170
172 Note: this will add a significant overhead, only turn this 171 Note: this will add a significant overhead, only turn this
173 on if you need to profile the system's use of these macros. 172 on if you need to profile the system's use of these macros.
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c
index 23f9b02ce967..21dedc8b50a4 100644
--- a/kernel/trace/trace_branch.c
+++ b/kernel/trace/trace_branch.c
@@ -261,7 +261,7 @@ static struct seq_operations tracing_likely_seq_ops = {
261 .show = t_show, 261 .show = t_show,
262}; 262};
263 263
264static int tracing_likely_open(struct inode *inode, struct file *file) 264static int tracing_branch_open(struct inode *inode, struct file *file)
265{ 265{
266 int ret; 266 int ret;
267 267
@@ -274,25 +274,18 @@ static int tracing_likely_open(struct inode *inode, struct file *file)
274 return ret; 274 return ret;
275} 275}
276 276
277static struct file_operations tracing_likely_fops = { 277static const struct file_operations tracing_branch_fops = {
278 .open = tracing_likely_open, 278 .open = tracing_branch_open,
279 .read = seq_read, 279 .read = seq_read,
280 .llseek = seq_lseek, 280 .llseek = seq_lseek,
281}; 281};
282 282
283extern unsigned long __start_likely_profile[]; 283extern unsigned long __start_annotated_branch_profile[];
284extern unsigned long __stop_likely_profile[]; 284extern unsigned long __stop_annotated_branch_profile[];
285extern unsigned long __start_unlikely_profile[];
286extern unsigned long __stop_unlikely_profile[];
287 285
288static struct ftrace_pointer ftrace_likely_pos = { 286static const struct ftrace_pointer ftrace_annotated_branch_pos = {
289 .start = __start_likely_profile, 287 .start = __start_annotated_branch_profile,
290 .stop = __stop_likely_profile, 288 .stop = __stop_annotated_branch_profile,
291};
292
293static struct ftrace_pointer ftrace_unlikely_pos = {
294 .start = __start_unlikely_profile,
295 .stop = __stop_unlikely_profile,
296}; 289};
297 290
298static __init int ftrace_branch_init(void) 291static __init int ftrace_branch_init(void)
@@ -302,18 +295,12 @@ static __init int ftrace_branch_init(void)
302 295
303 d_tracer = tracing_init_dentry(); 296 d_tracer = tracing_init_dentry();
304 297
305 entry = debugfs_create_file("profile_likely", 0444, d_tracer, 298 entry = debugfs_create_file("profile_annotated_branch", 0444, d_tracer,
306 &ftrace_likely_pos, 299 &ftrace_annotated_branch_pos,
307 &tracing_likely_fops); 300 &tracing_branch_fops);
308 if (!entry)
309 pr_warning("Could not create debugfs 'profile_likely' entry\n");
310
311 entry = debugfs_create_file("profile_unlikely", 0444, d_tracer,
312 &ftrace_unlikely_pos,
313 &tracing_likely_fops);
314 if (!entry) 301 if (!entry)
315 pr_warning("Could not create debugfs" 302 pr_warning("Could not create debugfs "
316 " 'profile_unlikely' entry\n"); 303 "'profile_annotatet_branch' entry\n");
317 304
318 return 0; 305 return 0;
319} 306}