aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/compiler.h14
-rw-r--r--kernel/trace/trace_branch.c6
2 files changed, 13 insertions, 7 deletions
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index cf0fa5d86059..bbbe1570de1c 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -107,12 +107,13 @@ struct ftrace_branch_data {
107 */ 107 */
108#if defined(CONFIG_TRACE_BRANCH_PROFILING) \ 108#if defined(CONFIG_TRACE_BRANCH_PROFILING) \
109 && !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__) 109 && !defined(DISABLE_BRANCH_PROFILING) && !defined(__CHECKER__)
110void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); 110void ftrace_likely_update(struct ftrace_branch_data *f, int val,
111 int expect, int is_constant);
111 112
112#define likely_notrace(x) __builtin_expect(!!(x), 1) 113#define likely_notrace(x) __builtin_expect(!!(x), 1)
113#define unlikely_notrace(x) __builtin_expect(!!(x), 0) 114#define unlikely_notrace(x) __builtin_expect(!!(x), 0)
114 115
115#define __branch_check__(x, expect) ({ \ 116#define __branch_check__(x, expect, is_constant) ({ \
116 int ______r; \ 117 int ______r; \
117 static struct ftrace_branch_data \ 118 static struct ftrace_branch_data \
118 __attribute__((__aligned__(4))) \ 119 __attribute__((__aligned__(4))) \
@@ -122,8 +123,9 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);
122 .file = __FILE__, \ 123 .file = __FILE__, \
123 .line = __LINE__, \ 124 .line = __LINE__, \
124 }; \ 125 }; \
125 ______r = likely_notrace(x); \ 126 ______r = __builtin_expect(!!(x), expect); \
126 ftrace_likely_update(&______f, ______r, expect); \ 127 ftrace_likely_update(&______f, ______r, \
128 expect, is_constant); \
127 ______r; \ 129 ______r; \
128 }) 130 })
129 131
@@ -133,10 +135,10 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);
133 * written by Daniel Walker. 135 * written by Daniel Walker.
134 */ 136 */
135# ifndef likely 137# ifndef likely
136# define likely(x) (__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 1)) 138# define likely(x) (__branch_check__(x, 1, __builtin_constant_p(x)))
137# endif 139# endif
138# ifndef unlikely 140# ifndef unlikely
139# define unlikely(x) (__builtin_constant_p(x) ? !!(x) : __branch_check__(x, 0)) 141# define unlikely(x) (__branch_check__(x, 0, __builtin_constant_p(x)))
140# endif 142# endif
141 143
142#ifdef CONFIG_PROFILE_ALL_BRANCHES 144#ifdef CONFIG_PROFILE_ALL_BRANCHES
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c
index 75489de546b6..7afe426ea528 100644
--- a/kernel/trace/trace_branch.c
+++ b/kernel/trace/trace_branch.c
@@ -200,8 +200,12 @@ void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
200} 200}
201#endif /* CONFIG_BRANCH_TRACER */ 201#endif /* CONFIG_BRANCH_TRACER */
202 202
203void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect) 203void ftrace_likely_update(struct ftrace_branch_data *f, int val,
204 int expect, int is_constant)
204{ 205{
206 /* A constant is always correct */
207 if (is_constant)
208 val = expect;
205 /* 209 /*
206 * I would love to have a trace point here instead, but the 210 * I would love to have a trace point here instead, but the
207 * trace point code is so inundated with unlikely and likely 211 * trace point code is so inundated with unlikely and likely