aboutsummaryrefslogtreecommitdiffstats
path: root/include
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2015-09-03 18:46:07 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2015-09-03 18:46:07 -0400
commitca520cab25e0e8da717c596ccaa2c2b3650cfa09 (patch)
tree883eb497642d98635817f9cf954ac98e043fb573 /include
parent4c12ab7e5e2e892fa94df500f96001837918a281 (diff)
parentd420acd816c07c7be31bd19d09cbcb16e5572fa6 (diff)
Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking and atomic updates from Ingo Molnar: "Main changes in this cycle are: - Extend atomic primitives with coherent logic op primitives (atomic_{or,and,xor}()) and deprecate the old partial APIs (atomic_{set,clear}_mask()) The old ops were incoherent with incompatible signatures across architectures and with incomplete support. Now every architecture supports the primitives consistently (by Peter Zijlstra) - Generic support for 'relaxed atomics': - _acquire/release/relaxed() flavours of xchg(), cmpxchg() and {add,sub}_return() - atomic_read_acquire() - atomic_set_release() This came out of porting qwrlock code to arm64 (by Will Deacon) - Clean up the fragile static_key APIs that were causing repeat bugs, by introducing a new one: DEFINE_STATIC_KEY_TRUE(name); DEFINE_STATIC_KEY_FALSE(name); which define a key of different types with an initial true/false value. Then allow: static_branch_likely() static_branch_unlikely() to take a key of either type and emit the right instruction for the case. To be able to know the 'type' of the static key we encode it in the jump entry (by Peter Zijlstra) - Static key self-tests (by Jason Baron) - qrwlock optimizations (by Waiman Long) - small futex enhancements (by Davidlohr Bueso) - ... and misc other changes" * 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (63 commits) jump_label/x86: Work around asm build bug on older/backported GCCs locking, ARM, atomics: Define our SMP atomics in terms of _relaxed() operations locking, include/llist: Use linux/atomic.h instead of asm/cmpxchg.h locking/qrwlock: Make use of _{acquire|release|relaxed}() atomics locking/qrwlock: Implement queue_write_unlock() using smp_store_release() locking/lockref: Remove homebrew cmpxchg64_relaxed() macro definition locking, asm-generic: Add _{relaxed|acquire|release}() variants for 'atomic_long_t' locking, asm-generic: Rework atomic-long.h to avoid bulk code duplication locking/atomics: Add _{acquire|release|relaxed}() variants of some atomic operations locking, compiler.h: Cast away attributes in the WRITE_ONCE() magic locking/static_keys: Make verify_keys() static jump label, locking/static_keys: Update docs locking/static_keys: Provide a selftest jump_label: Provide a self-test s390/uaccess, locking/static_keys: employ static_branch_likely() x86, tsc, locking/static_keys: Employ static_branch_likely() locking/static_keys: Add selftest locking/static_keys: Add a new static_key interface locking/static_keys: Rework update logic locking/static_keys: Add static_key_{en,dis}able() helpers ...
Diffstat (limited to 'include')
-rw-r--r--include/asm-generic/atomic-long.h263
-rw-r--r--include/asm-generic/atomic.h11
-rw-r--r--include/asm-generic/atomic64.h4
-rw-r--r--include/asm-generic/barrier.h4
-rw-r--r--include/asm-generic/qrwlock.h78
-rw-r--r--include/linux/atomic.h361
-rw-r--r--include/linux/compiler.h7
-rw-r--r--include/linux/jump_label.h261
-rw-r--r--include/linux/llist.h2
9 files changed, 705 insertions, 286 deletions
diff --git a/include/asm-generic/atomic-long.h b/include/asm-generic/atomic-long.h
index b7babf0206b8..a94cbebbc33d 100644
--- a/include/asm-generic/atomic-long.h
+++ b/include/asm-generic/atomic-long.h
@@ -23,236 +23,159 @@
23typedef atomic64_t atomic_long_t; 23typedef atomic64_t atomic_long_t;
24 24
25#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i) 25#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i)
26#define ATOMIC_LONG_PFX(x) atomic64 ## x
26 27
27static inline long atomic_long_read(atomic_long_t *l) 28#else
28{
29 atomic64_t *v = (atomic64_t *)l;
30
31 return (long)atomic64_read(v);
32}
33
34static inline void atomic_long_set(atomic_long_t *l, long i)
35{
36 atomic64_t *v = (atomic64_t *)l;
37
38 atomic64_set(v, i);
39}
40
41static inline void atomic_long_inc(atomic_long_t *l)
42{
43 atomic64_t *v = (atomic64_t *)l;
44
45 atomic64_inc(v);
46}
47
48static inline void atomic_long_dec(atomic_long_t *l)
49{
50 atomic64_t *v = (atomic64_t *)l;
51
52 atomic64_dec(v);
53}
54
55static inline void atomic_long_add(long i, atomic_long_t *l)
56{
57 atomic64_t *v = (atomic64_t *)l;
58
59 atomic64_add(i, v);
60}
61
62static inline void atomic_long_sub(long i, atomic_long_t *l)
63{
64 atomic64_t *v = (atomic64_t *)l;
65
66 atomic64_sub(i, v);
67}
68
69static inline int atomic_long_sub_and_test(long i, atomic_long_t *l)
70{
71 atomic64_t *v = (atomic64_t *)l;
72
73 return atomic64_sub_and_test(i, v);
74}
75
76static inline int atomic_long_dec_and_test(atomic_long_t *l)
77{
78 atomic64_t *v = (atomic64_t *)l;
79
80 return atomic64_dec_and_test(v);
81}
82
83static inline int atomic_long_inc_and_test(atomic_long_t *l)
84{
85 atomic64_t *v = (atomic64_t *)l;
86
87 return atomic64_inc_and_test(v);
88}
89
90static inline int atomic_long_add_negative(long i, atomic_long_t *l)
91{
92 atomic64_t *v = (atomic64_t *)l;
93
94 return atomic64_add_negative(i, v);
95}
96
97static inline long atomic_long_add_return(long i, atomic_long_t *l)
98{
99 atomic64_t *v = (atomic64_t *)l;
100
101 return (long)atomic64_add_return(i, v);
102}
103
104static inline long atomic_long_sub_return(long i, atomic_long_t *l)
105{
106 atomic64_t *v = (atomic64_t *)l;
107
108 return (long)atomic64_sub_return(i, v);
109}
110
111static inline long atomic_long_inc_return(atomic_long_t *l)
112{
113 atomic64_t *v = (atomic64_t *)l;
114
115 return (long)atomic64_inc_return(v);
116}
117
118static inline long atomic_long_dec_return(atomic_long_t *l)
119{
120 atomic64_t *v = (atomic64_t *)l;
121
122 return (long)atomic64_dec_return(v);
123}
124
125static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u)
126{
127 atomic64_t *v = (atomic64_t *)l;
128
129 return (long)atomic64_add_unless(v, a, u);
130}
131
132#define atomic_long_inc_not_zero(l) atomic64_inc_not_zero((atomic64_t *)(l))
133
134#define atomic_long_cmpxchg(l, old, new) \
135 (atomic64_cmpxchg((atomic64_t *)(l), (old), (new)))
136#define atomic_long_xchg(v, new) \
137 (atomic64_xchg((atomic64_t *)(v), (new)))
138
139#else /* BITS_PER_LONG == 64 */
140 29
141typedef atomic_t atomic_long_t; 30typedef atomic_t atomic_long_t;
142 31
143#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i) 32#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i)
144static inline long atomic_long_read(atomic_long_t *l) 33#define ATOMIC_LONG_PFX(x) atomic ## x
145{ 34
146 atomic_t *v = (atomic_t *)l; 35#endif
147 36
148 return (long)atomic_read(v); 37#define ATOMIC_LONG_READ_OP(mo) \
149} 38static inline long atomic_long_read##mo(atomic_long_t *l) \
150 39{ \
151static inline void atomic_long_set(atomic_long_t *l, long i) 40 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
152{ 41 \
153 atomic_t *v = (atomic_t *)l; 42 return (long)ATOMIC_LONG_PFX(_read##mo)(v); \
154 43}
155 atomic_set(v, i); 44ATOMIC_LONG_READ_OP()
156} 45ATOMIC_LONG_READ_OP(_acquire)
46
47#undef ATOMIC_LONG_READ_OP
48
49#define ATOMIC_LONG_SET_OP(mo) \
50static inline void atomic_long_set##mo(atomic_long_t *l, long i) \
51{ \
52 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
53 \
54 ATOMIC_LONG_PFX(_set##mo)(v, i); \
55}
56ATOMIC_LONG_SET_OP()
57ATOMIC_LONG_SET_OP(_release)
58
59#undef ATOMIC_LONG_SET_OP
60
61#define ATOMIC_LONG_ADD_SUB_OP(op, mo) \
62static inline long \
63atomic_long_##op##_return##mo(long i, atomic_long_t *l) \
64{ \
65 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
66 \
67 return (long)ATOMIC_LONG_PFX(_##op##_return##mo)(i, v); \
68}
69ATOMIC_LONG_ADD_SUB_OP(add,)
70ATOMIC_LONG_ADD_SUB_OP(add, _relaxed)
71ATOMIC_LONG_ADD_SUB_OP(add, _acquire)
72ATOMIC_LONG_ADD_SUB_OP(add, _release)
73ATOMIC_LONG_ADD_SUB_OP(sub,)
74ATOMIC_LONG_ADD_SUB_OP(sub, _relaxed)
75ATOMIC_LONG_ADD_SUB_OP(sub, _acquire)
76ATOMIC_LONG_ADD_SUB_OP(sub, _release)
77
78#undef ATOMIC_LONG_ADD_SUB_OP
79
80#define atomic_long_cmpxchg_relaxed(l, old, new) \
81 (ATOMIC_LONG_PFX(_cmpxchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(l), \
82 (old), (new)))
83#define atomic_long_cmpxchg_acquire(l, old, new) \
84 (ATOMIC_LONG_PFX(_cmpxchg_acquire)((ATOMIC_LONG_PFX(_t) *)(l), \
85 (old), (new)))
86#define atomic_long_cmpxchg_release(l, old, new) \
87 (ATOMIC_LONG_PFX(_cmpxchg_release)((ATOMIC_LONG_PFX(_t) *)(l), \
88 (old), (new)))
89#define atomic_long_cmpxchg(l, old, new) \
90 (ATOMIC_LONG_PFX(_cmpxchg)((ATOMIC_LONG_PFX(_t) *)(l), (old), (new)))
91
92#define atomic_long_xchg_relaxed(v, new) \
93 (ATOMIC_LONG_PFX(_xchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
94#define atomic_long_xchg_acquire(v, new) \
95 (ATOMIC_LONG_PFX(_xchg_acquire)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
96#define atomic_long_xchg_release(v, new) \
97 (ATOMIC_LONG_PFX(_xchg_release)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
98#define atomic_long_xchg(v, new) \
99 (ATOMIC_LONG_PFX(_xchg)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
157 100
158static inline void atomic_long_inc(atomic_long_t *l) 101static inline void atomic_long_inc(atomic_long_t *l)
159{ 102{
160 atomic_t *v = (atomic_t *)l; 103 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
161 104
162 atomic_inc(v); 105 ATOMIC_LONG_PFX(_inc)(v);
163} 106}
164 107
165static inline void atomic_long_dec(atomic_long_t *l) 108static inline void atomic_long_dec(atomic_long_t *l)
166{ 109{
167 atomic_t *v = (atomic_t *)l; 110 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
168 111
169 atomic_dec(v); 112 ATOMIC_LONG_PFX(_dec)(v);
170} 113}
171 114
172static inline void atomic_long_add(long i, atomic_long_t *l) 115static inline void atomic_long_add(long i, atomic_long_t *l)
173{ 116{
174 atomic_t *v = (atomic_t *)l; 117 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
175 118
176 atomic_add(i, v); 119 ATOMIC_LONG_PFX(_add)(i, v);
177} 120}
178 121
179static inline void atomic_long_sub(long i, atomic_long_t *l) 122static inline void atomic_long_sub(long i, atomic_long_t *l)
180{ 123{
181 atomic_t *v = (atomic_t *)l; 124 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
182 125
183 atomic_sub(i, v); 126 ATOMIC_LONG_PFX(_sub)(i, v);
184} 127}
185 128
186static inline int atomic_long_sub_and_test(long i, atomic_long_t *l) 129static inline int atomic_long_sub_and_test(long i, atomic_long_t *l)
187{ 130{
188 atomic_t *v = (atomic_t *)l; 131 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
189 132
190 return atomic_sub_and_test(i, v); 133 return ATOMIC_LONG_PFX(_sub_and_test)(i, v);
191} 134}
192 135
193static inline int atomic_long_dec_and_test(atomic_long_t *l) 136static inline int atomic_long_dec_and_test(atomic_long_t *l)
194{ 137{
195 atomic_t *v = (atomic_t *)l; 138 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
196 139
197 return atomic_dec_and_test(v); 140 return ATOMIC_LONG_PFX(_dec_and_test)(v);
198} 141}
199 142
200static inline int atomic_long_inc_and_test(atomic_long_t *l) 143static inline int atomic_long_inc_and_test(atomic_long_t *l)
201{ 144{
202 atomic_t *v = (atomic_t *)l; 145 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
203 146
204 return atomic_inc_and_test(v); 147 return ATOMIC_LONG_PFX(_inc_and_test)(v);
205} 148}
206 149
207static inline int atomic_long_add_negative(long i, atomic_long_t *l) 150static inline int atomic_long_add_negative(long i, atomic_long_t *l)
208{ 151{
209 atomic_t *v = (atomic_t *)l; 152 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
210 153
211 return atomic_add_negative(i, v); 154 return ATOMIC_LONG_PFX(_add_negative)(i, v);
212}
213
214static inline long atomic_long_add_return(long i, atomic_long_t *l)
215{
216 atomic_t *v = (atomic_t *)l;
217
218 return (long)atomic_add_return(i, v);
219}
220
221static inline long atomic_long_sub_return(long i, atomic_long_t *l)
222{
223 atomic_t *v = (atomic_t *)l;
224
225 return (long)atomic_sub_return(i, v);
226} 155}
227 156
228static inline long atomic_long_inc_return(atomic_long_t *l) 157static inline long atomic_long_inc_return(atomic_long_t *l)
229{ 158{
230 atomic_t *v = (atomic_t *)l; 159 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
231 160
232 return (long)atomic_inc_return(v); 161 return (long)ATOMIC_LONG_PFX(_inc_return)(v);
233} 162}
234 163
235static inline long atomic_long_dec_return(atomic_long_t *l) 164static inline long atomic_long_dec_return(atomic_long_t *l)
236{ 165{
237 atomic_t *v = (atomic_t *)l; 166 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
238 167
239 return (long)atomic_dec_return(v); 168 return (long)ATOMIC_LONG_PFX(_dec_return)(v);
240} 169}
241 170
242static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u) 171static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u)
243{ 172{
244 atomic_t *v = (atomic_t *)l; 173 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
245 174
246 return (long)atomic_add_unless(v, a, u); 175 return (long)ATOMIC_LONG_PFX(_add_unless)(v, a, u);
247} 176}
248 177
249#define atomic_long_inc_not_zero(l) atomic_inc_not_zero((atomic_t *)(l)) 178#define atomic_long_inc_not_zero(l) \
250 179 ATOMIC_LONG_PFX(_inc_not_zero)((ATOMIC_LONG_PFX(_t) *)(l))
251#define atomic_long_cmpxchg(l, old, new) \
252 (atomic_cmpxchg((atomic_t *)(l), (old), (new)))
253#define atomic_long_xchg(v, new) \
254 (atomic_xchg((atomic_t *)(v), (new)))
255
256#endif /* BITS_PER_LONG == 64 */
257 180
258#endif /* _ASM_GENERIC_ATOMIC_LONG_H */ 181#endif /* _ASM_GENERIC_ATOMIC_LONG_H */
diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h
index 1973ad2b13f4..d4d7e337fdcb 100644
--- a/include/asm-generic/atomic.h
+++ b/include/asm-generic/atomic.h
@@ -98,15 +98,16 @@ ATOMIC_OP_RETURN(add, +)
98ATOMIC_OP_RETURN(sub, -) 98ATOMIC_OP_RETURN(sub, -)
99#endif 99#endif
100 100
101#ifndef atomic_clear_mask 101#ifndef atomic_and
102ATOMIC_OP(and, &) 102ATOMIC_OP(and, &)
103#define atomic_clear_mask(i, v) atomic_and(~(i), (v))
104#endif 103#endif
105 104
106#ifndef atomic_set_mask 105#ifndef atomic_or
107#define CONFIG_ARCH_HAS_ATOMIC_OR
108ATOMIC_OP(or, |) 106ATOMIC_OP(or, |)
109#define atomic_set_mask(i, v) atomic_or((i), (v)) 107#endif
108
109#ifndef atomic_xor
110ATOMIC_OP(xor, ^)
110#endif 111#endif
111 112
112#undef ATOMIC_OP_RETURN 113#undef ATOMIC_OP_RETURN
diff --git a/include/asm-generic/atomic64.h b/include/asm-generic/atomic64.h
index 30ad9c86cebb..d48e78ccad3d 100644
--- a/include/asm-generic/atomic64.h
+++ b/include/asm-generic/atomic64.h
@@ -32,6 +32,10 @@ extern long long atomic64_##op##_return(long long a, atomic64_t *v);
32ATOMIC64_OPS(add) 32ATOMIC64_OPS(add)
33ATOMIC64_OPS(sub) 33ATOMIC64_OPS(sub)
34 34
35ATOMIC64_OP(and)
36ATOMIC64_OP(or)
37ATOMIC64_OP(xor)
38
35#undef ATOMIC64_OPS 39#undef ATOMIC64_OPS
36#undef ATOMIC64_OP_RETURN 40#undef ATOMIC64_OP_RETURN
37#undef ATOMIC64_OP 41#undef ATOMIC64_OP
diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h
index 55e3abc2d027..b42afada1280 100644
--- a/include/asm-generic/barrier.h
+++ b/include/asm-generic/barrier.h
@@ -108,12 +108,12 @@
108do { \ 108do { \
109 compiletime_assert_atomic_type(*p); \ 109 compiletime_assert_atomic_type(*p); \
110 smp_mb(); \ 110 smp_mb(); \
111 ACCESS_ONCE(*p) = (v); \ 111 WRITE_ONCE(*p, v); \
112} while (0) 112} while (0)
113 113
114#define smp_load_acquire(p) \ 114#define smp_load_acquire(p) \
115({ \ 115({ \
116 typeof(*p) ___p1 = ACCESS_ONCE(*p); \ 116 typeof(*p) ___p1 = READ_ONCE(*p); \
117 compiletime_assert_atomic_type(*p); \ 117 compiletime_assert_atomic_type(*p); \
118 smp_mb(); \ 118 smp_mb(); \
119 ___p1; \ 119 ___p1; \
diff --git a/include/asm-generic/qrwlock.h b/include/asm-generic/qrwlock.h
index 6383d54bf983..54a8e65e18b6 100644
--- a/include/asm-generic/qrwlock.h
+++ b/include/asm-generic/qrwlock.h
@@ -36,39 +36,39 @@
36/* 36/*
37 * External function declarations 37 * External function declarations
38 */ 38 */
39extern void queue_read_lock_slowpath(struct qrwlock *lock); 39extern void queued_read_lock_slowpath(struct qrwlock *lock, u32 cnts);
40extern void queue_write_lock_slowpath(struct qrwlock *lock); 40extern void queued_write_lock_slowpath(struct qrwlock *lock);
41 41
42/** 42/**
43 * queue_read_can_lock- would read_trylock() succeed? 43 * queued_read_can_lock- would read_trylock() succeed?
44 * @lock: Pointer to queue rwlock structure 44 * @lock: Pointer to queue rwlock structure
45 */ 45 */
46static inline int queue_read_can_lock(struct qrwlock *lock) 46static inline int queued_read_can_lock(struct qrwlock *lock)
47{ 47{
48 return !(atomic_read(&lock->cnts) & _QW_WMASK); 48 return !(atomic_read(&lock->cnts) & _QW_WMASK);
49} 49}
50 50
51/** 51/**
52 * queue_write_can_lock- would write_trylock() succeed? 52 * queued_write_can_lock- would write_trylock() succeed?
53 * @lock: Pointer to queue rwlock structure 53 * @lock: Pointer to queue rwlock structure
54 */ 54 */
55static inline int queue_write_can_lock(struct qrwlock *lock) 55static inline int queued_write_can_lock(struct qrwlock *lock)
56{ 56{
57 return !atomic_read(&lock->cnts); 57 return !atomic_read(&lock->cnts);
58} 58}
59 59
60/** 60/**
61 * queue_read_trylock - try to acquire read lock of a queue rwlock 61 * queued_read_trylock - try to acquire read lock of a queue rwlock
62 * @lock : Pointer to queue rwlock structure 62 * @lock : Pointer to queue rwlock structure
63 * Return: 1 if lock acquired, 0 if failed 63 * Return: 1 if lock acquired, 0 if failed
64 */ 64 */
65static inline int queue_read_trylock(struct qrwlock *lock) 65static inline int queued_read_trylock(struct qrwlock *lock)
66{ 66{
67 u32 cnts; 67 u32 cnts;
68 68
69 cnts = atomic_read(&lock->cnts); 69 cnts = atomic_read(&lock->cnts);
70 if (likely(!(cnts & _QW_WMASK))) { 70 if (likely(!(cnts & _QW_WMASK))) {
71 cnts = (u32)atomic_add_return(_QR_BIAS, &lock->cnts); 71 cnts = (u32)atomic_add_return_acquire(_QR_BIAS, &lock->cnts);
72 if (likely(!(cnts & _QW_WMASK))) 72 if (likely(!(cnts & _QW_WMASK)))
73 return 1; 73 return 1;
74 atomic_sub(_QR_BIAS, &lock->cnts); 74 atomic_sub(_QR_BIAS, &lock->cnts);
@@ -77,11 +77,11 @@ static inline int queue_read_trylock(struct qrwlock *lock)
77} 77}
78 78
79/** 79/**
80 * queue_write_trylock - try to acquire write lock of a queue rwlock 80 * queued_write_trylock - try to acquire write lock of a queue rwlock
81 * @lock : Pointer to queue rwlock structure 81 * @lock : Pointer to queue rwlock structure
82 * Return: 1 if lock acquired, 0 if failed 82 * Return: 1 if lock acquired, 0 if failed
83 */ 83 */
84static inline int queue_write_trylock(struct qrwlock *lock) 84static inline int queued_write_trylock(struct qrwlock *lock)
85{ 85{
86 u32 cnts; 86 u32 cnts;
87 87
@@ -89,78 +89,70 @@ static inline int queue_write_trylock(struct qrwlock *lock)
89 if (unlikely(cnts)) 89 if (unlikely(cnts))
90 return 0; 90 return 0;
91 91
92 return likely(atomic_cmpxchg(&lock->cnts, 92 return likely(atomic_cmpxchg_acquire(&lock->cnts,
93 cnts, cnts | _QW_LOCKED) == cnts); 93 cnts, cnts | _QW_LOCKED) == cnts);
94} 94}
95/** 95/**
96 * queue_read_lock - acquire read lock of a queue rwlock 96 * queued_read_lock - acquire read lock of a queue rwlock
97 * @lock: Pointer to queue rwlock structure 97 * @lock: Pointer to queue rwlock structure
98 */ 98 */
99static inline void queue_read_lock(struct qrwlock *lock) 99static inline void queued_read_lock(struct qrwlock *lock)
100{ 100{
101 u32 cnts; 101 u32 cnts;
102 102
103 cnts = atomic_add_return(_QR_BIAS, &lock->cnts); 103 cnts = atomic_add_return_acquire(_QR_BIAS, &lock->cnts);
104 if (likely(!(cnts & _QW_WMASK))) 104 if (likely(!(cnts & _QW_WMASK)))
105 return; 105 return;
106 106
107 /* The slowpath will decrement the reader count, if necessary. */ 107 /* The slowpath will decrement the reader count, if necessary. */
108 queue_read_lock_slowpath(lock); 108 queued_read_lock_slowpath(lock, cnts);
109} 109}
110 110
111/** 111/**
112 * queue_write_lock - acquire write lock of a queue rwlock 112 * queued_write_lock - acquire write lock of a queue rwlock
113 * @lock : Pointer to queue rwlock structure 113 * @lock : Pointer to queue rwlock structure
114 */ 114 */
115static inline void queue_write_lock(struct qrwlock *lock) 115static inline void queued_write_lock(struct qrwlock *lock)
116{ 116{
117 /* Optimize for the unfair lock case where the fair flag is 0. */ 117 /* Optimize for the unfair lock case where the fair flag is 0. */
118 if (atomic_cmpxchg(&lock->cnts, 0, _QW_LOCKED) == 0) 118 if (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0)
119 return; 119 return;
120 120
121 queue_write_lock_slowpath(lock); 121 queued_write_lock_slowpath(lock);
122} 122}
123 123
124/** 124/**
125 * queue_read_unlock - release read lock of a queue rwlock 125 * queued_read_unlock - release read lock of a queue rwlock
126 * @lock : Pointer to queue rwlock structure 126 * @lock : Pointer to queue rwlock structure
127 */ 127 */
128static inline void queue_read_unlock(struct qrwlock *lock) 128static inline void queued_read_unlock(struct qrwlock *lock)
129{ 129{
130 /* 130 /*
131 * Atomically decrement the reader count 131 * Atomically decrement the reader count
132 */ 132 */
133 smp_mb__before_atomic(); 133 (void)atomic_sub_return_release(_QR_BIAS, &lock->cnts);
134 atomic_sub(_QR_BIAS, &lock->cnts);
135} 134}
136 135
137#ifndef queue_write_unlock
138/** 136/**
139 * queue_write_unlock - release write lock of a queue rwlock 137 * queued_write_unlock - release write lock of a queue rwlock
140 * @lock : Pointer to queue rwlock structure 138 * @lock : Pointer to queue rwlock structure
141 */ 139 */
142static inline void queue_write_unlock(struct qrwlock *lock) 140static inline void queued_write_unlock(struct qrwlock *lock)
143{ 141{
144 /* 142 smp_store_release((u8 *)&lock->cnts, 0);
145 * If the writer field is atomic, it can be cleared directly.
146 * Otherwise, an atomic subtraction will be used to clear it.
147 */
148 smp_mb__before_atomic();
149 atomic_sub(_QW_LOCKED, &lock->cnts);
150} 143}
151#endif
152 144
153/* 145/*
154 * Remapping rwlock architecture specific functions to the corresponding 146 * Remapping rwlock architecture specific functions to the corresponding
155 * queue rwlock functions. 147 * queue rwlock functions.
156 */ 148 */
157#define arch_read_can_lock(l) queue_read_can_lock(l) 149#define arch_read_can_lock(l) queued_read_can_lock(l)
158#define arch_write_can_lock(l) queue_write_can_lock(l) 150#define arch_write_can_lock(l) queued_write_can_lock(l)
159#define arch_read_lock(l) queue_read_lock(l) 151#define arch_read_lock(l) queued_read_lock(l)
160#define arch_write_lock(l) queue_write_lock(l) 152#define arch_write_lock(l) queued_write_lock(l)
161#define arch_read_trylock(l) queue_read_trylock(l) 153#define arch_read_trylock(l) queued_read_trylock(l)
162#define arch_write_trylock(l) queue_write_trylock(l) 154#define arch_write_trylock(l) queued_write_trylock(l)
163#define arch_read_unlock(l) queue_read_unlock(l) 155#define arch_read_unlock(l) queued_read_unlock(l)
164#define arch_write_unlock(l) queue_write_unlock(l) 156#define arch_write_unlock(l) queued_write_unlock(l)
165 157
166#endif /* __ASM_GENERIC_QRWLOCK_H */ 158#endif /* __ASM_GENERIC_QRWLOCK_H */
diff --git a/include/linux/atomic.h b/include/linux/atomic.h
index 5b08a8540ecf..00a5763e850e 100644
--- a/include/linux/atomic.h
+++ b/include/linux/atomic.h
@@ -2,6 +2,329 @@
2#ifndef _LINUX_ATOMIC_H 2#ifndef _LINUX_ATOMIC_H
3#define _LINUX_ATOMIC_H 3#define _LINUX_ATOMIC_H
4#include <asm/atomic.h> 4#include <asm/atomic.h>
5#include <asm/barrier.h>
6
7/*
8 * Relaxed variants of xchg, cmpxchg and some atomic operations.
9 *
10 * We support four variants:
11 *
12 * - Fully ordered: The default implementation, no suffix required.
13 * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
14 * - Release: Provides RELEASE semantics, _release suffix.
15 * - Relaxed: No ordering guarantees, _relaxed suffix.
16 *
17 * For compound atomics performing both a load and a store, ACQUIRE
18 * semantics apply only to the load and RELEASE semantics only to the
19 * store portion of the operation. Note that a failed cmpxchg_acquire
20 * does -not- imply any memory ordering constraints.
21 *
22 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
23 */
24
25#ifndef atomic_read_acquire
26#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
27#endif
28
29#ifndef atomic_set_release
30#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
31#endif
32
33/*
34 * The idea here is to build acquire/release variants by adding explicit
35 * barriers on top of the relaxed variant. In the case where the relaxed
36 * variant is already fully ordered, no additional barriers are needed.
37 */
38#define __atomic_op_acquire(op, args...) \
39({ \
40 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
41 smp_mb__after_atomic(); \
42 __ret; \
43})
44
45#define __atomic_op_release(op, args...) \
46({ \
47 smp_mb__before_atomic(); \
48 op##_relaxed(args); \
49})
50
51#define __atomic_op_fence(op, args...) \
52({ \
53 typeof(op##_relaxed(args)) __ret; \
54 smp_mb__before_atomic(); \
55 __ret = op##_relaxed(args); \
56 smp_mb__after_atomic(); \
57 __ret; \
58})
59
60/* atomic_add_return_relaxed */
61#ifndef atomic_add_return_relaxed
62#define atomic_add_return_relaxed atomic_add_return
63#define atomic_add_return_acquire atomic_add_return
64#define atomic_add_return_release atomic_add_return
65
66#else /* atomic_add_return_relaxed */
67
68#ifndef atomic_add_return_acquire
69#define atomic_add_return_acquire(...) \
70 __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
71#endif
72
73#ifndef atomic_add_return_release
74#define atomic_add_return_release(...) \
75 __atomic_op_release(atomic_add_return, __VA_ARGS__)
76#endif
77
78#ifndef atomic_add_return
79#define atomic_add_return(...) \
80 __atomic_op_fence(atomic_add_return, __VA_ARGS__)
81#endif
82#endif /* atomic_add_return_relaxed */
83
84/* atomic_sub_return_relaxed */
85#ifndef atomic_sub_return_relaxed
86#define atomic_sub_return_relaxed atomic_sub_return
87#define atomic_sub_return_acquire atomic_sub_return
88#define atomic_sub_return_release atomic_sub_return
89
90#else /* atomic_sub_return_relaxed */
91
92#ifndef atomic_sub_return_acquire
93#define atomic_sub_return_acquire(...) \
94 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
95#endif
96
97#ifndef atomic_sub_return_release
98#define atomic_sub_return_release(...) \
99 __atomic_op_release(atomic_sub_return, __VA_ARGS__)
100#endif
101
102#ifndef atomic_sub_return
103#define atomic_sub_return(...) \
104 __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
105#endif
106#endif /* atomic_sub_return_relaxed */
107
108/* atomic_xchg_relaxed */
109#ifndef atomic_xchg_relaxed
110#define atomic_xchg_relaxed atomic_xchg
111#define atomic_xchg_acquire atomic_xchg
112#define atomic_xchg_release atomic_xchg
113
114#else /* atomic_xchg_relaxed */
115
116#ifndef atomic_xchg_acquire
117#define atomic_xchg_acquire(...) \
118 __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
119#endif
120
121#ifndef atomic_xchg_release
122#define atomic_xchg_release(...) \
123 __atomic_op_release(atomic_xchg, __VA_ARGS__)
124#endif
125
126#ifndef atomic_xchg
127#define atomic_xchg(...) \
128 __atomic_op_fence(atomic_xchg, __VA_ARGS__)
129#endif
130#endif /* atomic_xchg_relaxed */
131
132/* atomic_cmpxchg_relaxed */
133#ifndef atomic_cmpxchg_relaxed
134#define atomic_cmpxchg_relaxed atomic_cmpxchg
135#define atomic_cmpxchg_acquire atomic_cmpxchg
136#define atomic_cmpxchg_release atomic_cmpxchg
137
138#else /* atomic_cmpxchg_relaxed */
139
140#ifndef atomic_cmpxchg_acquire
141#define atomic_cmpxchg_acquire(...) \
142 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
143#endif
144
145#ifndef atomic_cmpxchg_release
146#define atomic_cmpxchg_release(...) \
147 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
148#endif
149
150#ifndef atomic_cmpxchg
151#define atomic_cmpxchg(...) \
152 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
153#endif
154#endif /* atomic_cmpxchg_relaxed */
155
156#ifndef atomic64_read_acquire
157#define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter)
158#endif
159
160#ifndef atomic64_set_release
161#define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i))
162#endif
163
164/* atomic64_add_return_relaxed */
165#ifndef atomic64_add_return_relaxed
166#define atomic64_add_return_relaxed atomic64_add_return
167#define atomic64_add_return_acquire atomic64_add_return
168#define atomic64_add_return_release atomic64_add_return
169
170#else /* atomic64_add_return_relaxed */
171
172#ifndef atomic64_add_return_acquire
173#define atomic64_add_return_acquire(...) \
174 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
175#endif
176
177#ifndef atomic64_add_return_release
178#define atomic64_add_return_release(...) \
179 __atomic_op_release(atomic64_add_return, __VA_ARGS__)
180#endif
181
182#ifndef atomic64_add_return
183#define atomic64_add_return(...) \
184 __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
185#endif
186#endif /* atomic64_add_return_relaxed */
187
188/* atomic64_sub_return_relaxed */
189#ifndef atomic64_sub_return_relaxed
190#define atomic64_sub_return_relaxed atomic64_sub_return
191#define atomic64_sub_return_acquire atomic64_sub_return
192#define atomic64_sub_return_release atomic64_sub_return
193
194#else /* atomic64_sub_return_relaxed */
195
196#ifndef atomic64_sub_return_acquire
197#define atomic64_sub_return_acquire(...) \
198 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
199#endif
200
201#ifndef atomic64_sub_return_release
202#define atomic64_sub_return_release(...) \
203 __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
204#endif
205
206#ifndef atomic64_sub_return
207#define atomic64_sub_return(...) \
208 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
209#endif
210#endif /* atomic64_sub_return_relaxed */
211
212/* atomic64_xchg_relaxed */
213#ifndef atomic64_xchg_relaxed
214#define atomic64_xchg_relaxed atomic64_xchg
215#define atomic64_xchg_acquire atomic64_xchg
216#define atomic64_xchg_release atomic64_xchg
217
218#else /* atomic64_xchg_relaxed */
219
220#ifndef atomic64_xchg_acquire
221#define atomic64_xchg_acquire(...) \
222 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
223#endif
224
225#ifndef atomic64_xchg_release
226#define atomic64_xchg_release(...) \
227 __atomic_op_release(atomic64_xchg, __VA_ARGS__)
228#endif
229
230#ifndef atomic64_xchg
231#define atomic64_xchg(...) \
232 __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
233#endif
234#endif /* atomic64_xchg_relaxed */
235
236/* atomic64_cmpxchg_relaxed */
237#ifndef atomic64_cmpxchg_relaxed
238#define atomic64_cmpxchg_relaxed atomic64_cmpxchg
239#define atomic64_cmpxchg_acquire atomic64_cmpxchg
240#define atomic64_cmpxchg_release atomic64_cmpxchg
241
242#else /* atomic64_cmpxchg_relaxed */
243
244#ifndef atomic64_cmpxchg_acquire
245#define atomic64_cmpxchg_acquire(...) \
246 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
247#endif
248
249#ifndef atomic64_cmpxchg_release
250#define atomic64_cmpxchg_release(...) \
251 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
252#endif
253
254#ifndef atomic64_cmpxchg
255#define atomic64_cmpxchg(...) \
256 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
257#endif
258#endif /* atomic64_cmpxchg_relaxed */
259
260/* cmpxchg_relaxed */
261#ifndef cmpxchg_relaxed
262#define cmpxchg_relaxed cmpxchg
263#define cmpxchg_acquire cmpxchg
264#define cmpxchg_release cmpxchg
265
266#else /* cmpxchg_relaxed */
267
268#ifndef cmpxchg_acquire
269#define cmpxchg_acquire(...) \
270 __atomic_op_acquire(cmpxchg, __VA_ARGS__)
271#endif
272
273#ifndef cmpxchg_release
274#define cmpxchg_release(...) \
275 __atomic_op_release(cmpxchg, __VA_ARGS__)
276#endif
277
278#ifndef cmpxchg
279#define cmpxchg(...) \
280 __atomic_op_fence(cmpxchg, __VA_ARGS__)
281#endif
282#endif /* cmpxchg_relaxed */
283
284/* cmpxchg64_relaxed */
285#ifndef cmpxchg64_relaxed
286#define cmpxchg64_relaxed cmpxchg64
287#define cmpxchg64_acquire cmpxchg64
288#define cmpxchg64_release cmpxchg64
289
290#else /* cmpxchg64_relaxed */
291
292#ifndef cmpxchg64_acquire
293#define cmpxchg64_acquire(...) \
294 __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
295#endif
296
297#ifndef cmpxchg64_release
298#define cmpxchg64_release(...) \
299 __atomic_op_release(cmpxchg64, __VA_ARGS__)
300#endif
301
302#ifndef cmpxchg64
303#define cmpxchg64(...) \
304 __atomic_op_fence(cmpxchg64, __VA_ARGS__)
305#endif
306#endif /* cmpxchg64_relaxed */
307
308/* xchg_relaxed */
309#ifndef xchg_relaxed
310#define xchg_relaxed xchg
311#define xchg_acquire xchg
312#define xchg_release xchg
313
314#else /* xchg_relaxed */
315
316#ifndef xchg_acquire
317#define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__)
318#endif
319
320#ifndef xchg_release
321#define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__)
322#endif
323
324#ifndef xchg
325#define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__)
326#endif
327#endif /* xchg_relaxed */
5 328
6/** 329/**
7 * atomic_add_unless - add unless the number is already a given value 330 * atomic_add_unless - add unless the number is already a given value
@@ -28,6 +351,23 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u)
28#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 351#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
29#endif 352#endif
30 353
354#ifndef atomic_andnot
355static inline void atomic_andnot(int i, atomic_t *v)
356{
357 atomic_and(~i, v);
358}
359#endif
360
361static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
362{
363 atomic_andnot(mask, v);
364}
365
366static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v)
367{
368 atomic_or(mask, v);
369}
370
31/** 371/**
32 * atomic_inc_not_zero_hint - increment if not null 372 * atomic_inc_not_zero_hint - increment if not null
33 * @v: pointer of type atomic_t 373 * @v: pointer of type atomic_t
@@ -111,21 +451,16 @@ static inline int atomic_dec_if_positive(atomic_t *v)
111} 451}
112#endif 452#endif
113 453
114#ifndef CONFIG_ARCH_HAS_ATOMIC_OR
115static inline void atomic_or(int i, atomic_t *v)
116{
117 int old;
118 int new;
119
120 do {
121 old = atomic_read(v);
122 new = old | i;
123 } while (atomic_cmpxchg(v, old, new) != old);
124}
125#endif /* #ifndef CONFIG_ARCH_HAS_ATOMIC_OR */
126
127#include <asm-generic/atomic-long.h> 454#include <asm-generic/atomic-long.h>
128#ifdef CONFIG_GENERIC_ATOMIC64 455#ifdef CONFIG_GENERIC_ATOMIC64
129#include <asm-generic/atomic64.h> 456#include <asm-generic/atomic64.h>
130#endif 457#endif
458
459#ifndef atomic64_andnot
460static inline void atomic64_andnot(long long i, atomic64_t *v)
461{
462 atomic64_and(~i, v);
463}
464#endif
465
131#endif /* _LINUX_ATOMIC_H */ 466#endif /* _LINUX_ATOMIC_H */
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index e08a6ae7c0a4..c836eb2dc44d 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -252,7 +252,12 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
252 ({ union { typeof(x) __val; char __c[1]; } __u; __read_once_size(&(x), __u.__c, sizeof(x)); __u.__val; }) 252 ({ union { typeof(x) __val; char __c[1]; } __u; __read_once_size(&(x), __u.__c, sizeof(x)); __u.__val; })
253 253
254#define WRITE_ONCE(x, val) \ 254#define WRITE_ONCE(x, val) \
255 ({ union { typeof(x) __val; char __c[1]; } __u = { .__val = (val) }; __write_once_size(&(x), __u.__c, sizeof(x)); __u.__val; }) 255({ \
256 union { typeof(x) __val; char __c[1]; } __u = \
257 { .__val = (__force typeof(x)) (val) }; \
258 __write_once_size(&(x), __u.__c, sizeof(x)); \
259 __u.__val; \
260})
256 261
257/** 262/**
258 * READ_ONCE_CTRL - Read a value heading a control dependency 263 * READ_ONCE_CTRL - Read a value heading a control dependency
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index f4de473f226b..7f653e8f6690 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -7,17 +7,52 @@
7 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com> 7 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
8 * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com> 8 * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
9 * 9 *
10 * DEPRECATED API:
11 *
12 * The use of 'struct static_key' directly, is now DEPRECATED. In addition
13 * static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following:
14 *
15 * struct static_key false = STATIC_KEY_INIT_FALSE;
16 * struct static_key true = STATIC_KEY_INIT_TRUE;
17 * static_key_true()
18 * static_key_false()
19 *
20 * The updated API replacements are:
21 *
22 * DEFINE_STATIC_KEY_TRUE(key);
23 * DEFINE_STATIC_KEY_FALSE(key);
24 * static_key_likely()
25 * statick_key_unlikely()
26 *
10 * Jump labels provide an interface to generate dynamic branches using 27 * Jump labels provide an interface to generate dynamic branches using
11 * self-modifying code. Assuming toolchain and architecture support, the result 28 * self-modifying code. Assuming toolchain and architecture support, if we
12 * of a "if (static_key_false(&key))" statement is an unconditional branch (which 29 * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)",
13 * defaults to false - and the true block is placed out of line). 30 * an "if (static_branch_unlikely(&key))" statement is an unconditional branch
31 * (which defaults to false - and the true block is placed out of line).
32 * Similarly, we can define an initially true key via
33 * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same
34 * "if (static_branch_unlikely(&key))", in which case we will generate an
35 * unconditional branch to the out-of-line true branch. Keys that are
36 * initially true or false can be using in both static_branch_unlikely()
37 * and static_branch_likely() statements.
38 *
39 * At runtime we can change the branch target by setting the key
40 * to true via a call to static_branch_enable(), or false using
41 * static_branch_disable(). If the direction of the branch is switched by
42 * these calls then we run-time modify the branch target via a
43 * no-op -> jump or jump -> no-op conversion. For example, for an
44 * initially false key that is used in an "if (static_branch_unlikely(&key))"
45 * statement, setting the key to true requires us to patch in a jump
46 * to the out-of-line of true branch.
14 * 47 *
15 * However at runtime we can change the branch target using 48 * In addtion to static_branch_{enable,disable}, we can also reference count
16 * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key 49 * the key or branch direction via static_branch_{inc,dec}. Thus,
17 * object, and for as long as there are references all branches referring to 50 * static_branch_inc() can be thought of as a 'make more true' and
18 * that particular key will point to the (out of line) true block. 51 * static_branch_dec() as a 'make more false'. The inc()/dec()
52 * interface is meant to be used exclusively from the inc()/dec() for a given
53 * key.
19 * 54 *
20 * Since this relies on modifying code, the static_key_slow_{inc,dec}() functions 55 * Since this relies on modifying code, the branch modifying functions
21 * must be considered absolute slow paths (machine wide synchronization etc.). 56 * must be considered absolute slow paths (machine wide synchronization etc.).
22 * OTOH, since the affected branches are unconditional, their runtime overhead 57 * OTOH, since the affected branches are unconditional, their runtime overhead
23 * will be absolutely minimal, esp. in the default (off) case where the total 58 * will be absolutely minimal, esp. in the default (off) case where the total
@@ -29,20 +64,10 @@
29 * cause significant performance degradation. Struct static_key_deferred and 64 * cause significant performance degradation. Struct static_key_deferred and
30 * static_key_slow_dec_deferred() provide for this. 65 * static_key_slow_dec_deferred() provide for this.
31 * 66 *
32 * Lacking toolchain and or architecture support, jump labels fall back to a simple 67 * Lacking toolchain and or architecture support, static keys fall back to a
33 * conditional branch. 68 * simple conditional branch.
34 *
35 * struct static_key my_key = STATIC_KEY_INIT_TRUE;
36 *
37 * if (static_key_true(&my_key)) {
38 * }
39 * 69 *
40 * will result in the true case being in-line and starts the key with a single 70 * Additional babbling in: Documentation/static-keys.txt
41 * reference. Mixing static_key_true() and static_key_false() on the same key is not
42 * allowed.
43 *
44 * Not initializing the key (static data is initialized to 0s anyway) is the
45 * same as using STATIC_KEY_INIT_FALSE.
46 */ 71 */
47 72
48#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) 73#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
@@ -86,8 +111,8 @@ struct static_key {
86#ifndef __ASSEMBLY__ 111#ifndef __ASSEMBLY__
87 112
88enum jump_label_type { 113enum jump_label_type {
89 JUMP_LABEL_DISABLE = 0, 114 JUMP_LABEL_NOP = 0,
90 JUMP_LABEL_ENABLE, 115 JUMP_LABEL_JMP,
91}; 116};
92 117
93struct module; 118struct module;
@@ -101,33 +126,18 @@ static inline int static_key_count(struct static_key *key)
101 126
102#ifdef HAVE_JUMP_LABEL 127#ifdef HAVE_JUMP_LABEL
103 128
104#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL 129#define JUMP_TYPE_FALSE 0UL
105#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL 130#define JUMP_TYPE_TRUE 1UL
106#define JUMP_LABEL_TYPE_MASK 1UL 131#define JUMP_TYPE_MASK 1UL
107
108static
109inline struct jump_entry *jump_label_get_entries(struct static_key *key)
110{
111 return (struct jump_entry *)((unsigned long)key->entries
112 & ~JUMP_LABEL_TYPE_MASK);
113}
114
115static inline bool jump_label_get_branch_default(struct static_key *key)
116{
117 if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) ==
118 JUMP_LABEL_TYPE_TRUE_BRANCH)
119 return true;
120 return false;
121}
122 132
123static __always_inline bool static_key_false(struct static_key *key) 133static __always_inline bool static_key_false(struct static_key *key)
124{ 134{
125 return arch_static_branch(key); 135 return arch_static_branch(key, false);
126} 136}
127 137
128static __always_inline bool static_key_true(struct static_key *key) 138static __always_inline bool static_key_true(struct static_key *key)
129{ 139{
130 return !static_key_false(key); 140 return !arch_static_branch(key, true);
131} 141}
132 142
133extern struct jump_entry __start___jump_table[]; 143extern struct jump_entry __start___jump_table[];
@@ -145,12 +155,12 @@ extern void static_key_slow_inc(struct static_key *key);
145extern void static_key_slow_dec(struct static_key *key); 155extern void static_key_slow_dec(struct static_key *key);
146extern void jump_label_apply_nops(struct module *mod); 156extern void jump_label_apply_nops(struct module *mod);
147 157
148#define STATIC_KEY_INIT_TRUE ((struct static_key) \ 158#define STATIC_KEY_INIT_TRUE \
149 { .enabled = ATOMIC_INIT(1), \ 159 { .enabled = ATOMIC_INIT(1), \
150 .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH }) 160 .entries = (void *)JUMP_TYPE_TRUE }
151#define STATIC_KEY_INIT_FALSE ((struct static_key) \ 161#define STATIC_KEY_INIT_FALSE \
152 { .enabled = ATOMIC_INIT(0), \ 162 { .enabled = ATOMIC_INIT(0), \
153 .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH }) 163 .entries = (void *)JUMP_TYPE_FALSE }
154 164
155#else /* !HAVE_JUMP_LABEL */ 165#else /* !HAVE_JUMP_LABEL */
156 166
@@ -198,10 +208,8 @@ static inline int jump_label_apply_nops(struct module *mod)
198 return 0; 208 return 0;
199} 209}
200 210
201#define STATIC_KEY_INIT_TRUE ((struct static_key) \ 211#define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) }
202 { .enabled = ATOMIC_INIT(1) }) 212#define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) }
203#define STATIC_KEY_INIT_FALSE ((struct static_key) \
204 { .enabled = ATOMIC_INIT(0) })
205 213
206#endif /* HAVE_JUMP_LABEL */ 214#endif /* HAVE_JUMP_LABEL */
207 215
@@ -213,6 +221,157 @@ static inline bool static_key_enabled(struct static_key *key)
213 return static_key_count(key) > 0; 221 return static_key_count(key) > 0;
214} 222}
215 223
224static inline void static_key_enable(struct static_key *key)
225{
226 int count = static_key_count(key);
227
228 WARN_ON_ONCE(count < 0 || count > 1);
229
230 if (!count)
231 static_key_slow_inc(key);
232}
233
234static inline void static_key_disable(struct static_key *key)
235{
236 int count = static_key_count(key);
237
238 WARN_ON_ONCE(count < 0 || count > 1);
239
240 if (count)
241 static_key_slow_dec(key);
242}
243
244/* -------------------------------------------------------------------------- */
245
246/*
247 * Two type wrappers around static_key, such that we can use compile time
248 * type differentiation to emit the right code.
249 *
250 * All the below code is macros in order to play type games.
251 */
252
253struct static_key_true {
254 struct static_key key;
255};
256
257struct static_key_false {
258 struct static_key key;
259};
260
261#define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, }
262#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
263
264#define DEFINE_STATIC_KEY_TRUE(name) \
265 struct static_key_true name = STATIC_KEY_TRUE_INIT
266
267#define DEFINE_STATIC_KEY_FALSE(name) \
268 struct static_key_false name = STATIC_KEY_FALSE_INIT
269
270#ifdef HAVE_JUMP_LABEL
271
272/*
273 * Combine the right initial value (type) with the right branch order
274 * to generate the desired result.
275 *
276 *
277 * type\branch| likely (1) | unlikely (0)
278 * -----------+-----------------------+------------------
279 * | |
280 * true (1) | ... | ...
281 * | NOP | JMP L
282 * | <br-stmts> | 1: ...
283 * | L: ... |
284 * | |
285 * | | L: <br-stmts>
286 * | | jmp 1b
287 * | |
288 * -----------+-----------------------+------------------
289 * | |
290 * false (0) | ... | ...
291 * | JMP L | NOP
292 * | <br-stmts> | 1: ...
293 * | L: ... |
294 * | |
295 * | | L: <br-stmts>
296 * | | jmp 1b
297 * | |
298 * -----------+-----------------------+------------------
299 *
300 * The initial value is encoded in the LSB of static_key::entries,
301 * type: 0 = false, 1 = true.
302 *
303 * The branch type is encoded in the LSB of jump_entry::key,
304 * branch: 0 = unlikely, 1 = likely.
305 *
306 * This gives the following logic table:
307 *
308 * enabled type branch instuction
309 * -----------------------------+-----------
310 * 0 0 0 | NOP
311 * 0 0 1 | JMP
312 * 0 1 0 | NOP
313 * 0 1 1 | JMP
314 *
315 * 1 0 0 | JMP
316 * 1 0 1 | NOP
317 * 1 1 0 | JMP
318 * 1 1 1 | NOP
319 *
320 * Which gives the following functions:
321 *
322 * dynamic: instruction = enabled ^ branch
323 * static: instruction = type ^ branch
324 *
325 * See jump_label_type() / jump_label_init_type().
326 */
327
328extern bool ____wrong_branch_error(void);
329
330#define static_branch_likely(x) \
331({ \
332 bool branch; \
333 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
334 branch = !arch_static_branch(&(x)->key, true); \
335 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
336 branch = !arch_static_branch_jump(&(x)->key, true); \
337 else \
338 branch = ____wrong_branch_error(); \
339 branch; \
340})
341
342#define static_branch_unlikely(x) \
343({ \
344 bool branch; \
345 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
346 branch = arch_static_branch_jump(&(x)->key, false); \
347 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
348 branch = arch_static_branch(&(x)->key, false); \
349 else \
350 branch = ____wrong_branch_error(); \
351 branch; \
352})
353
354#else /* !HAVE_JUMP_LABEL */
355
356#define static_branch_likely(x) likely(static_key_enabled(&(x)->key))
357#define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key))
358
359#endif /* HAVE_JUMP_LABEL */
360
361/*
362 * Advanced usage; refcount, branch is enabled when: count != 0
363 */
364
365#define static_branch_inc(x) static_key_slow_inc(&(x)->key)
366#define static_branch_dec(x) static_key_slow_dec(&(x)->key)
367
368/*
369 * Normal usage; boolean enable/disable.
370 */
371
372#define static_branch_enable(x) static_key_enable(&(x)->key)
373#define static_branch_disable(x) static_key_disable(&(x)->key)
374
216#endif /* _LINUX_JUMP_LABEL_H */ 375#endif /* _LINUX_JUMP_LABEL_H */
217 376
218#endif /* __ASSEMBLY__ */ 377#endif /* __ASSEMBLY__ */
diff --git a/include/linux/llist.h b/include/linux/llist.h
index fbf10a0bc095..fd4ca0b4fe0f 100644
--- a/include/linux/llist.h
+++ b/include/linux/llist.h
@@ -55,8 +55,8 @@
55 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 55 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
56 */ 56 */
57 57
58#include <linux/atomic.h>
58#include <linux/kernel.h> 59#include <linux/kernel.h>
59#include <asm/cmpxchg.h>
60 60
61struct llist_head { 61struct llist_head {
62 struct llist_node *first; 62 struct llist_node *first;
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-11 21:17:18 -0400