diff options
-rw-r--r-- | include/asm-generic/vmlinux.lds.h | 9 | ||||
-rw-r--r-- | include/linux/compiler.h | 24 | ||||
-rw-r--r-- | kernel/trace/Kconfig | 3 | ||||
-rw-r--r-- | kernel/trace/trace_branch.c | 39 |
4 files changed, 22 insertions, 53 deletions
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 3b46ae464933..8bccb49981e5 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h | |||
@@ -46,12 +46,9 @@ | |||
46 | #endif | 46 | #endif |
47 | 47 | ||
48 | #ifdef CONFIG_TRACE_BRANCH_PROFILING | 48 | #ifdef CONFIG_TRACE_BRANCH_PROFILING |
49 | #define LIKELY_PROFILE() VMLINUX_SYMBOL(__start_likely_profile) = .; \ | 49 | #define LIKELY_PROFILE() VMLINUX_SYMBOL(__start_annotated_branch_profile) = .; \ |
50 | *(_ftrace_likely) \ | 50 | *(_ftrace_annotated_branch) \ |
51 | VMLINUX_SYMBOL(__stop_likely_profile) = .; \ | 51 | VMLINUX_SYMBOL(__stop_annotated_branch_profile) = .; |
52 | VMLINUX_SYMBOL(__start_unlikely_profile) = .; \ | ||
53 | *(_ftrace_unlikely) \ | ||
54 | VMLINUX_SYMBOL(__stop_unlikely_profile) = .; | ||
55 | #else | 52 | #else |
56 | #define LIKELY_PROFILE() | 53 | #define LIKELY_PROFILE() |
57 | #endif | 54 | #endif |
diff --git a/include/linux/compiler.h b/include/linux/compiler.h index c25e525121f0..0628a2013fae 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h | |||
@@ -77,32 +77,18 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); | |||
77 | #define likely_notrace(x) __builtin_expect(!!(x), 1) | 77 | #define likely_notrace(x) __builtin_expect(!!(x), 1) |
78 | #define unlikely_notrace(x) __builtin_expect(!!(x), 0) | 78 | #define unlikely_notrace(x) __builtin_expect(!!(x), 0) |
79 | 79 | ||
80 | #define likely_check(x) ({ \ | 80 | #define __branch_check__(x, expect) ({ \ |
81 | int ______r; \ | 81 | int ______r; \ |
82 | static struct ftrace_branch_data \ | 82 | static struct ftrace_branch_data \ |
83 | __attribute__((__aligned__(4))) \ | 83 | __attribute__((__aligned__(4))) \ |
84 | __attribute__((section("_ftrace_likely"))) \ | 84 | __attribute__((section("_ftrace_annotated_branch"))) \ |
85 | ______f = { \ | 85 | ______f = { \ |
86 | .func = __func__, \ | 86 | .func = __func__, \ |
87 | .file = __FILE__, \ | 87 | .file = __FILE__, \ |
88 | .line = __LINE__, \ | 88 | .line = __LINE__, \ |
89 | }; \ | 89 | }; \ |
90 | ______r = likely_notrace(x); \ | 90 | ______r = likely_notrace(x); \ |
91 | ftrace_likely_update(&______f, ______r, 1); \ | 91 | ftrace_likely_update(&______f, ______r, expect); \ |
92 | ______r; \ | ||
93 | }) | ||
94 | #define unlikely_check(x) ({ \ | ||
95 | int ______r; \ | ||
96 | static struct ftrace_branch_data \ | ||
97 | __attribute__((__aligned__(4))) \ | ||
98 | __attribute__((section("_ftrace_unlikely"))) \ | ||
99 | ______f = { \ | ||
100 | .func = __func__, \ | ||
101 | .file = __FILE__, \ | ||
102 | .line = __LINE__, \ | ||
103 | }; \ | ||
104 | ______r = unlikely_notrace(x); \ | ||
105 | ftrace_likely_update(&______f, ______r, 0); \ | ||
106 | ______r; \ | 92 | ______r; \ |
107 | }) | 93 | }) |
108 | 94 | ||
@@ -112,10 +98,10 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); | |||
112 | * written by Daniel Walker. | 98 | * written by Daniel Walker. |
113 | */ | 99 | */ |
114 | # ifndef likely | 100 | # ifndef likely |
115 | # define likely(x) (__builtin_constant_p(x) ? !!(x) : likely_check(x)) | 101 | # define likely(x) (__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 1)) |
116 | # endif | 102 | # endif |
117 | # ifndef unlikely | 103 | # ifndef unlikely |
118 | # define unlikely(x) (__builtin_constant_p(x) ? !!(x) : unlikely_check(x)) | 104 | # define unlikely(x) (__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 0)) |
119 | # endif | 105 | # endif |
120 | #else | 106 | #else |
121 | # define likely(x) __builtin_expect(!!(x), 1) | 107 | # define likely(x) __builtin_expect(!!(x), 1) |
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index b8378fad29a3..7e3548705708 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig | |||
@@ -166,8 +166,7 @@ config TRACE_BRANCH_PROFILING | |||
166 | This tracer profiles all the the likely and unlikely macros | 166 | This tracer profiles all the the likely and unlikely macros |
167 | in the kernel. It will display the results in: | 167 | in the kernel. It will display the results in: |
168 | 168 | ||
169 | /debugfs/tracing/profile_likely | 169 | /debugfs/tracing/profile_annotated_branch |
170 | /debugfs/tracing/profile_unlikely | ||
171 | 170 | ||
172 | Note: this will add a significant overhead, only turn this | 171 | Note: this will add a significant overhead, only turn this |
173 | on if you need to profile the system's use of these macros. | 172 | on if you need to profile the system's use of these macros. |
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c index 23f9b02ce967..21dedc8b50a4 100644 --- a/kernel/trace/trace_branch.c +++ b/kernel/trace/trace_branch.c | |||
@@ -261,7 +261,7 @@ static struct seq_operations tracing_likely_seq_ops = { | |||
261 | .show = t_show, | 261 | .show = t_show, |
262 | }; | 262 | }; |
263 | 263 | ||
264 | static int tracing_likely_open(struct inode *inode, struct file *file) | 264 | static int tracing_branch_open(struct inode *inode, struct file *file) |
265 | { | 265 | { |
266 | int ret; | 266 | int ret; |
267 | 267 | ||
@@ -274,25 +274,18 @@ static int tracing_likely_open(struct inode *inode, struct file *file) | |||
274 | return ret; | 274 | return ret; |
275 | } | 275 | } |
276 | 276 | ||
277 | static struct file_operations tracing_likely_fops = { | 277 | static const struct file_operations tracing_branch_fops = { |
278 | .open = tracing_likely_open, | 278 | .open = tracing_branch_open, |
279 | .read = seq_read, | 279 | .read = seq_read, |
280 | .llseek = seq_lseek, | 280 | .llseek = seq_lseek, |
281 | }; | 281 | }; |
282 | 282 | ||
283 | extern unsigned long __start_likely_profile[]; | 283 | extern unsigned long __start_annotated_branch_profile[]; |
284 | extern unsigned long __stop_likely_profile[]; | 284 | extern unsigned long __stop_annotated_branch_profile[]; |
285 | extern unsigned long __start_unlikely_profile[]; | ||
286 | extern unsigned long __stop_unlikely_profile[]; | ||
287 | 285 | ||
288 | static struct ftrace_pointer ftrace_likely_pos = { | 286 | static const struct ftrace_pointer ftrace_annotated_branch_pos = { |
289 | .start = __start_likely_profile, | 287 | .start = __start_annotated_branch_profile, |
290 | .stop = __stop_likely_profile, | 288 | .stop = __stop_annotated_branch_profile, |
291 | }; | ||
292 | |||
293 | static struct ftrace_pointer ftrace_unlikely_pos = { | ||
294 | .start = __start_unlikely_profile, | ||
295 | .stop = __stop_unlikely_profile, | ||
296 | }; | 289 | }; |
297 | 290 | ||
298 | static __init int ftrace_branch_init(void) | 291 | static __init int ftrace_branch_init(void) |
@@ -302,18 +295,12 @@ static __init int ftrace_branch_init(void) | |||
302 | 295 | ||
303 | d_tracer = tracing_init_dentry(); | 296 | d_tracer = tracing_init_dentry(); |
304 | 297 | ||
305 | entry = debugfs_create_file("profile_likely", 0444, d_tracer, | 298 | entry = debugfs_create_file("profile_annotated_branch", 0444, d_tracer, |
306 | &ftrace_likely_pos, | 299 | &ftrace_annotated_branch_pos, |
307 | &tracing_likely_fops); | 300 | &tracing_branch_fops); |
308 | if (!entry) | ||
309 | pr_warning("Could not create debugfs 'profile_likely' entry\n"); | ||
310 | |||
311 | entry = debugfs_create_file("profile_unlikely", 0444, d_tracer, | ||
312 | &ftrace_unlikely_pos, | ||
313 | &tracing_likely_fops); | ||
314 | if (!entry) | 301 | if (!entry) |
315 | pr_warning("Could not create debugfs" | 302 | pr_warning("Could not create debugfs " |
316 | " 'profile_unlikely' entry\n"); | 303 | "'profile_annotatet_branch' entry\n"); |
317 | 304 | ||
318 | return 0; | 305 | return 0; |
319 | } | 306 | } |