summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/asm-generic/atomic-long.h36
-rw-r--r--include/asm-generic/atomic.h49
-rw-r--r--include/asm-generic/atomic64.h15
-rw-r--r--include/linux/atomic.h336
-rw-r--r--lib/atomic64.c32
-rw-r--r--lib/atomic64_test.c34
6 files changed, 493 insertions, 9 deletions
diff --git a/include/asm-generic/atomic-long.h b/include/asm-generic/atomic-long.h
index 5e1f345b58dd..2d0d3cf791ab 100644
--- a/include/asm-generic/atomic-long.h
+++ b/include/asm-generic/atomic-long.h
@@ -112,6 +112,40 @@ static __always_inline void atomic_long_dec(atomic_long_t *l)
112 ATOMIC_LONG_PFX(_dec)(v); 112 ATOMIC_LONG_PFX(_dec)(v);
113} 113}
114 114
115#define ATOMIC_LONG_FETCH_OP(op, mo) \
116static inline long \
117atomic_long_fetch_##op##mo(long i, atomic_long_t *l) \
118{ \
119 ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
120 \
121 return (long)ATOMIC_LONG_PFX(_fetch_##op##mo)(i, v); \
122}
123
124ATOMIC_LONG_FETCH_OP(add, )
125ATOMIC_LONG_FETCH_OP(add, _relaxed)
126ATOMIC_LONG_FETCH_OP(add, _acquire)
127ATOMIC_LONG_FETCH_OP(add, _release)
128ATOMIC_LONG_FETCH_OP(sub, )
129ATOMIC_LONG_FETCH_OP(sub, _relaxed)
130ATOMIC_LONG_FETCH_OP(sub, _acquire)
131ATOMIC_LONG_FETCH_OP(sub, _release)
132ATOMIC_LONG_FETCH_OP(and, )
133ATOMIC_LONG_FETCH_OP(and, _relaxed)
134ATOMIC_LONG_FETCH_OP(and, _acquire)
135ATOMIC_LONG_FETCH_OP(and, _release)
136ATOMIC_LONG_FETCH_OP(andnot, )
137ATOMIC_LONG_FETCH_OP(andnot, _relaxed)
138ATOMIC_LONG_FETCH_OP(andnot, _acquire)
139ATOMIC_LONG_FETCH_OP(andnot, _release)
140ATOMIC_LONG_FETCH_OP(or, )
141ATOMIC_LONG_FETCH_OP(or, _relaxed)
142ATOMIC_LONG_FETCH_OP(or, _acquire)
143ATOMIC_LONG_FETCH_OP(or, _release)
144ATOMIC_LONG_FETCH_OP(xor, )
145ATOMIC_LONG_FETCH_OP(xor, _relaxed)
146ATOMIC_LONG_FETCH_OP(xor, _acquire)
147ATOMIC_LONG_FETCH_OP(xor, _release)
148
115#define ATOMIC_LONG_OP(op) \ 149#define ATOMIC_LONG_OP(op) \
116static __always_inline void \ 150static __always_inline void \
117atomic_long_##op(long i, atomic_long_t *l) \ 151atomic_long_##op(long i, atomic_long_t *l) \
@@ -124,9 +158,9 @@ atomic_long_##op(long i, atomic_long_t *l) \
124ATOMIC_LONG_OP(add) 158ATOMIC_LONG_OP(add)
125ATOMIC_LONG_OP(sub) 159ATOMIC_LONG_OP(sub)
126ATOMIC_LONG_OP(and) 160ATOMIC_LONG_OP(and)
161ATOMIC_LONG_OP(andnot)
127ATOMIC_LONG_OP(or) 162ATOMIC_LONG_OP(or)
128ATOMIC_LONG_OP(xor) 163ATOMIC_LONG_OP(xor)
129ATOMIC_LONG_OP(andnot)
130 164
131#undef ATOMIC_LONG_OP 165#undef ATOMIC_LONG_OP
132 166
diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h
index 74f1a3704d7a..a2304ccf4ed0 100644
--- a/include/asm-generic/atomic.h
+++ b/include/asm-generic/atomic.h
@@ -61,6 +61,18 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \
61 return c c_op i; \ 61 return c c_op i; \
62} 62}
63 63
64#define ATOMIC_FETCH_OP(op, c_op) \
65static inline int atomic_fetch_##op(int i, atomic_t *v) \
66{ \
67 int c, old; \
68 \
69 c = v->counter; \
70 while ((old = cmpxchg(&v->counter, c, c c_op i)) != c) \
71 c = old; \
72 \
73 return c; \
74}
75
64#else 76#else
65 77
66#include <linux/irqflags.h> 78#include <linux/irqflags.h>
@@ -88,6 +100,20 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \
88 return ret; \ 100 return ret; \
89} 101}
90 102
103#define ATOMIC_FETCH_OP(op, c_op) \
104static inline int atomic_fetch_##op(int i, atomic_t *v) \
105{ \
106 unsigned long flags; \
107 int ret; \
108 \
109 raw_local_irq_save(flags); \
110 ret = v->counter; \
111 v->counter = v->counter c_op i; \
112 raw_local_irq_restore(flags); \
113 \
114 return ret; \
115}
116
91#endif /* CONFIG_SMP */ 117#endif /* CONFIG_SMP */
92 118
93#ifndef atomic_add_return 119#ifndef atomic_add_return
@@ -98,6 +124,28 @@ ATOMIC_OP_RETURN(add, +)
98ATOMIC_OP_RETURN(sub, -) 124ATOMIC_OP_RETURN(sub, -)
99#endif 125#endif
100 126
127#ifndef atomic_fetch_add
128ATOMIC_FETCH_OP(add, +)
129#endif
130
131#ifndef atomic_fetch_sub
132ATOMIC_FETCH_OP(sub, -)
133#endif
134
135#ifndef atomic_fetch_and
136ATOMIC_FETCH_OP(and, &)
137#endif
138
139#ifndef atomic_fetch_or
140#define atomic_fetch_or atomic_fetch_or
141
142ATOMIC_FETCH_OP(or, |)
143#endif
144
145#ifndef atomic_fetch_xor
146ATOMIC_FETCH_OP(xor, ^)
147#endif
148
101#ifndef atomic_and 149#ifndef atomic_and
102ATOMIC_OP(and, &) 150ATOMIC_OP(and, &)
103#endif 151#endif
@@ -110,6 +158,7 @@ ATOMIC_OP(or, |)
110ATOMIC_OP(xor, ^) 158ATOMIC_OP(xor, ^)
111#endif 159#endif
112 160
161#undef ATOMIC_FETCH_OP
113#undef ATOMIC_OP_RETURN 162#undef ATOMIC_OP_RETURN
114#undef ATOMIC_OP 163#undef ATOMIC_OP
115 164
diff --git a/include/asm-generic/atomic64.h b/include/asm-generic/atomic64.h
index d48e78ccad3d..dad68bf46c77 100644
--- a/include/asm-generic/atomic64.h
+++ b/include/asm-generic/atomic64.h
@@ -27,16 +27,23 @@ extern void atomic64_##op(long long a, atomic64_t *v);
27#define ATOMIC64_OP_RETURN(op) \ 27#define ATOMIC64_OP_RETURN(op) \
28extern long long atomic64_##op##_return(long long a, atomic64_t *v); 28extern long long atomic64_##op##_return(long long a, atomic64_t *v);
29 29
30#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) 30#define ATOMIC64_FETCH_OP(op) \
31extern long long atomic64_fetch_##op(long long a, atomic64_t *v);
32
33#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) ATOMIC64_FETCH_OP(op)
31 34
32ATOMIC64_OPS(add) 35ATOMIC64_OPS(add)
33ATOMIC64_OPS(sub) 36ATOMIC64_OPS(sub)
34 37
35ATOMIC64_OP(and) 38#undef ATOMIC64_OPS
36ATOMIC64_OP(or) 39#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_FETCH_OP(op)
37ATOMIC64_OP(xor) 40
41ATOMIC64_OPS(and)
42ATOMIC64_OPS(or)
43ATOMIC64_OPS(xor)
38 44
39#undef ATOMIC64_OPS 45#undef ATOMIC64_OPS
46#undef ATOMIC64_FETCH_OP
40#undef ATOMIC64_OP_RETURN 47#undef ATOMIC64_OP_RETURN
41#undef ATOMIC64_OP 48#undef ATOMIC64_OP
42 49
diff --git a/include/linux/atomic.h b/include/linux/atomic.h
index 351f89e1d15c..2e6c013ac5a4 100644
--- a/include/linux/atomic.h
+++ b/include/linux/atomic.h
@@ -163,6 +163,154 @@
163#endif 163#endif
164#endif /* atomic_dec_return_relaxed */ 164#endif /* atomic_dec_return_relaxed */
165 165
166
167/* atomic_fetch_add_relaxed */
168#ifndef atomic_fetch_add_relaxed
169#define atomic_fetch_add_relaxed atomic_fetch_add
170#define atomic_fetch_add_acquire atomic_fetch_add
171#define atomic_fetch_add_release atomic_fetch_add
172
173#else /* atomic_fetch_add_relaxed */
174
175#ifndef atomic_fetch_add_acquire
176#define atomic_fetch_add_acquire(...) \
177 __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
178#endif
179
180#ifndef atomic_fetch_add_release
181#define atomic_fetch_add_release(...) \
182 __atomic_op_release(atomic_fetch_add, __VA_ARGS__)
183#endif
184
185#ifndef atomic_fetch_add
186#define atomic_fetch_add(...) \
187 __atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
188#endif
189#endif /* atomic_fetch_add_relaxed */
190
191/* atomic_fetch_sub_relaxed */
192#ifndef atomic_fetch_sub_relaxed
193#define atomic_fetch_sub_relaxed atomic_fetch_sub
194#define atomic_fetch_sub_acquire atomic_fetch_sub
195#define atomic_fetch_sub_release atomic_fetch_sub
196
197#else /* atomic_fetch_sub_relaxed */
198
199#ifndef atomic_fetch_sub_acquire
200#define atomic_fetch_sub_acquire(...) \
201 __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
202#endif
203
204#ifndef atomic_fetch_sub_release
205#define atomic_fetch_sub_release(...) \
206 __atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
207#endif
208
209#ifndef atomic_fetch_sub
210#define atomic_fetch_sub(...) \
211 __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
212#endif
213#endif /* atomic_fetch_sub_relaxed */
214
215/* atomic_fetch_or_relaxed */
216#ifndef atomic_fetch_or_relaxed
217#define atomic_fetch_or_relaxed atomic_fetch_or
218#define atomic_fetch_or_acquire atomic_fetch_or
219#define atomic_fetch_or_release atomic_fetch_or
220
221#else /* atomic_fetch_or_relaxed */
222
223#ifndef atomic_fetch_or_acquire
224#define atomic_fetch_or_acquire(...) \
225 __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
226#endif
227
228#ifndef atomic_fetch_or_release
229#define atomic_fetch_or_release(...) \
230 __atomic_op_release(atomic_fetch_or, __VA_ARGS__)
231#endif
232
233#ifndef atomic_fetch_or
234#define atomic_fetch_or(...) \
235 __atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
236#endif
237#endif /* atomic_fetch_or_relaxed */
238
239/* atomic_fetch_and_relaxed */
240#ifndef atomic_fetch_and_relaxed
241#define atomic_fetch_and_relaxed atomic_fetch_and
242#define atomic_fetch_and_acquire atomic_fetch_and
243#define atomic_fetch_and_release atomic_fetch_and
244
245#else /* atomic_fetch_and_relaxed */
246
247#ifndef atomic_fetch_and_acquire
248#define atomic_fetch_and_acquire(...) \
249 __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
250#endif
251
252#ifndef atomic_fetch_and_release
253#define atomic_fetch_and_release(...) \
254 __atomic_op_release(atomic_fetch_and, __VA_ARGS__)
255#endif
256
257#ifndef atomic_fetch_and
258#define atomic_fetch_and(...) \
259 __atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
260#endif
261#endif /* atomic_fetch_and_relaxed */
262
263#ifdef atomic_andnot
264/* atomic_fetch_andnot_relaxed */
265#ifndef atomic_fetch_andnot_relaxed
266#define atomic_fetch_andnot_relaxed atomic_fetch_andnot
267#define atomic_fetch_andnot_acquire atomic_fetch_andnot
268#define atomic_fetch_andnot_release atomic_fetch_andnot
269
270#else /* atomic_fetch_andnot_relaxed */
271
272#ifndef atomic_fetch_andnot_acquire
273#define atomic_fetch_andnot_acquire(...) \
274 __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
275#endif
276
277#ifndef atomic_fetch_andnot_release
278#define atomic_fetch_andnot_release(...) \
279 __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
280#endif
281
282#ifndef atomic_fetch_andnot
283#define atomic_fetch_andnot(...) \
284 __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
285#endif
286#endif /* atomic_fetch_andnot_relaxed */
287#endif /* atomic_andnot */
288
289/* atomic_fetch_xor_relaxed */
290#ifndef atomic_fetch_xor_relaxed
291#define atomic_fetch_xor_relaxed atomic_fetch_xor
292#define atomic_fetch_xor_acquire atomic_fetch_xor
293#define atomic_fetch_xor_release atomic_fetch_xor
294
295#else /* atomic_fetch_xor_relaxed */
296
297#ifndef atomic_fetch_xor_acquire
298#define atomic_fetch_xor_acquire(...) \
299 __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
300#endif
301
302#ifndef atomic_fetch_xor_release
303#define atomic_fetch_xor_release(...) \
304 __atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
305#endif
306
307#ifndef atomic_fetch_xor
308#define atomic_fetch_xor(...) \
309 __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
310#endif
311#endif /* atomic_fetch_xor_relaxed */
312
313
166/* atomic_xchg_relaxed */ 314/* atomic_xchg_relaxed */
167#ifndef atomic_xchg_relaxed 315#ifndef atomic_xchg_relaxed
168#define atomic_xchg_relaxed atomic_xchg 316#define atomic_xchg_relaxed atomic_xchg
@@ -310,6 +458,26 @@ static inline void atomic_andnot(int i, atomic_t *v)
310{ 458{
311 atomic_and(~i, v); 459 atomic_and(~i, v);
312} 460}
461
462static inline int atomic_fetch_andnot(int i, atomic_t *v)
463{
464 return atomic_fetch_and(~i, v);
465}
466
467static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v)
468{
469 return atomic_fetch_and_relaxed(~i, v);
470}
471
472static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v)
473{
474 return atomic_fetch_and_acquire(~i, v);
475}
476
477static inline int atomic_fetch_andnot_release(int i, atomic_t *v)
478{
479 return atomic_fetch_and_release(~i, v);
480}
313#endif 481#endif
314 482
315static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) 483static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
@@ -535,6 +703,154 @@ static inline int atomic_fetch_or(int mask, atomic_t *p)
535#endif 703#endif
536#endif /* atomic64_dec_return_relaxed */ 704#endif /* atomic64_dec_return_relaxed */
537 705
706
707/* atomic64_fetch_add_relaxed */
708#ifndef atomic64_fetch_add_relaxed
709#define atomic64_fetch_add_relaxed atomic64_fetch_add
710#define atomic64_fetch_add_acquire atomic64_fetch_add
711#define atomic64_fetch_add_release atomic64_fetch_add
712
713#else /* atomic64_fetch_add_relaxed */
714
715#ifndef atomic64_fetch_add_acquire
716#define atomic64_fetch_add_acquire(...) \
717 __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
718#endif
719
720#ifndef atomic64_fetch_add_release
721#define atomic64_fetch_add_release(...) \
722 __atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
723#endif
724
725#ifndef atomic64_fetch_add
726#define atomic64_fetch_add(...) \
727 __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
728#endif
729#endif /* atomic64_fetch_add_relaxed */
730
731/* atomic64_fetch_sub_relaxed */
732#ifndef atomic64_fetch_sub_relaxed
733#define atomic64_fetch_sub_relaxed atomic64_fetch_sub
734#define atomic64_fetch_sub_acquire atomic64_fetch_sub
735#define atomic64_fetch_sub_release atomic64_fetch_sub
736
737#else /* atomic64_fetch_sub_relaxed */
738
739#ifndef atomic64_fetch_sub_acquire
740#define atomic64_fetch_sub_acquire(...) \
741 __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
742#endif
743
744#ifndef atomic64_fetch_sub_release
745#define atomic64_fetch_sub_release(...) \
746 __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
747#endif
748
749#ifndef atomic64_fetch_sub
750#define atomic64_fetch_sub(...) \
751 __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
752#endif
753#endif /* atomic64_fetch_sub_relaxed */
754
755/* atomic64_fetch_or_relaxed */
756#ifndef atomic64_fetch_or_relaxed
757#define atomic64_fetch_or_relaxed atomic64_fetch_or
758#define atomic64_fetch_or_acquire atomic64_fetch_or
759#define atomic64_fetch_or_release atomic64_fetch_or
760
761#else /* atomic64_fetch_or_relaxed */
762
763#ifndef atomic64_fetch_or_acquire
764#define atomic64_fetch_or_acquire(...) \
765 __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
766#endif
767
768#ifndef atomic64_fetch_or_release
769#define atomic64_fetch_or_release(...) \
770 __atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
771#endif
772
773#ifndef atomic64_fetch_or
774#define atomic64_fetch_or(...) \
775 __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
776#endif
777#endif /* atomic64_fetch_or_relaxed */
778
779/* atomic64_fetch_and_relaxed */
780#ifndef atomic64_fetch_and_relaxed
781#define atomic64_fetch_and_relaxed atomic64_fetch_and
782#define atomic64_fetch_and_acquire atomic64_fetch_and
783#define atomic64_fetch_and_release atomic64_fetch_and
784
785#else /* atomic64_fetch_and_relaxed */
786
787#ifndef atomic64_fetch_and_acquire
788#define atomic64_fetch_and_acquire(...) \
789 __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
790#endif
791
792#ifndef atomic64_fetch_and_release
793#define atomic64_fetch_and_release(...) \
794 __atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
795#endif
796
797#ifndef atomic64_fetch_and
798#define atomic64_fetch_and(...) \
799 __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
800#endif
801#endif /* atomic64_fetch_and_relaxed */
802
803#ifdef atomic64_andnot
804/* atomic64_fetch_andnot_relaxed */
805#ifndef atomic64_fetch_andnot_relaxed
806#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot
807#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot
808#define atomic64_fetch_andnot_release atomic64_fetch_andnot
809
810#else /* atomic64_fetch_andnot_relaxed */
811
812#ifndef atomic64_fetch_andnot_acquire
813#define atomic64_fetch_andnot_acquire(...) \
814 __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
815#endif
816
817#ifndef atomic64_fetch_andnot_release
818#define atomic64_fetch_andnot_release(...) \
819 __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
820#endif
821
822#ifndef atomic64_fetch_andnot
823#define atomic64_fetch_andnot(...) \
824 __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
825#endif
826#endif /* atomic64_fetch_andnot_relaxed */
827#endif /* atomic64_andnot */
828
829/* atomic64_fetch_xor_relaxed */
830#ifndef atomic64_fetch_xor_relaxed
831#define atomic64_fetch_xor_relaxed atomic64_fetch_xor
832#define atomic64_fetch_xor_acquire atomic64_fetch_xor
833#define atomic64_fetch_xor_release atomic64_fetch_xor
834
835#else /* atomic64_fetch_xor_relaxed */
836
837#ifndef atomic64_fetch_xor_acquire
838#define atomic64_fetch_xor_acquire(...) \
839 __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
840#endif
841
842#ifndef atomic64_fetch_xor_release
843#define atomic64_fetch_xor_release(...) \
844 __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
845#endif
846
847#ifndef atomic64_fetch_xor
848#define atomic64_fetch_xor(...) \
849 __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
850#endif
851#endif /* atomic64_fetch_xor_relaxed */
852
853
538/* atomic64_xchg_relaxed */ 854/* atomic64_xchg_relaxed */
539#ifndef atomic64_xchg_relaxed 855#ifndef atomic64_xchg_relaxed
540#define atomic64_xchg_relaxed atomic64_xchg 856#define atomic64_xchg_relaxed atomic64_xchg
@@ -588,6 +904,26 @@ static inline void atomic64_andnot(long long i, atomic64_t *v)
588{ 904{
589 atomic64_and(~i, v); 905 atomic64_and(~i, v);
590} 906}
907
908static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v)
909{
910 return atomic64_fetch_and(~i, v);
911}
912
913static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v)
914{
915 return atomic64_fetch_and_relaxed(~i, v);
916}
917
918static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v)
919{
920 return atomic64_fetch_and_acquire(~i, v);
921}
922
923static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v)
924{
925 return atomic64_fetch_and_release(~i, v);
926}
591#endif 927#endif
592 928
593#include <asm-generic/atomic-long.h> 929#include <asm-generic/atomic-long.h>
diff --git a/lib/atomic64.c b/lib/atomic64.c
index 2886ebac6567..53c2d5edc826 100644
--- a/lib/atomic64.c
+++ b/lib/atomic64.c
@@ -96,17 +96,41 @@ long long atomic64_##op##_return(long long a, atomic64_t *v) \
96} \ 96} \
97EXPORT_SYMBOL(atomic64_##op##_return); 97EXPORT_SYMBOL(atomic64_##op##_return);
98 98
99#define ATOMIC64_FETCH_OP(op, c_op) \
100long long atomic64_fetch_##op(long long a, atomic64_t *v) \
101{ \
102 unsigned long flags; \
103 raw_spinlock_t *lock = lock_addr(v); \
104 long long val; \
105 \
106 raw_spin_lock_irqsave(lock, flags); \
107 val = v->counter; \
108 v->counter c_op a; \
109 raw_spin_unlock_irqrestore(lock, flags); \
110 return val; \
111} \
112EXPORT_SYMBOL(atomic64_fetch_##op);
113
99#define ATOMIC64_OPS(op, c_op) \ 114#define ATOMIC64_OPS(op, c_op) \
100 ATOMIC64_OP(op, c_op) \ 115 ATOMIC64_OP(op, c_op) \
101 ATOMIC64_OP_RETURN(op, c_op) 116 ATOMIC64_OP_RETURN(op, c_op) \
117 ATOMIC64_FETCH_OP(op, c_op)
102 118
103ATOMIC64_OPS(add, +=) 119ATOMIC64_OPS(add, +=)
104ATOMIC64_OPS(sub, -=) 120ATOMIC64_OPS(sub, -=)
105ATOMIC64_OP(and, &=)
106ATOMIC64_OP(or, |=)
107ATOMIC64_OP(xor, ^=)
108 121
109#undef ATOMIC64_OPS 122#undef ATOMIC64_OPS
123#define ATOMIC64_OPS(op, c_op) \
124 ATOMIC64_OP(op, c_op) \
125 ATOMIC64_OP_RETURN(op, c_op) \
126 ATOMIC64_FETCH_OP(op, c_op)
127
128ATOMIC64_OPS(and, &=)
129ATOMIC64_OPS(or, |=)
130ATOMIC64_OPS(xor, ^=)
131
132#undef ATOMIC64_OPS
133#undef ATOMIC64_FETCH_OP
110#undef ATOMIC64_OP_RETURN 134#undef ATOMIC64_OP_RETURN
111#undef ATOMIC64_OP 135#undef ATOMIC64_OP
112 136
diff --git a/lib/atomic64_test.c b/lib/atomic64_test.c
index 123481814320..dbb369145dda 100644
--- a/lib/atomic64_test.c
+++ b/lib/atomic64_test.c
@@ -53,11 +53,25 @@ do { \
53 BUG_ON(atomic##bit##_read(&v) != r); \ 53 BUG_ON(atomic##bit##_read(&v) != r); \
54} while (0) 54} while (0)
55 55
56#define TEST_FETCH(bit, op, c_op, val) \
57do { \
58 atomic##bit##_set(&v, v0); \
59 r = v0; \
60 r c_op val; \
61 BUG_ON(atomic##bit##_##op(val, &v) != v0); \
62 BUG_ON(atomic##bit##_read(&v) != r); \
63} while (0)
64
56#define RETURN_FAMILY_TEST(bit, op, c_op, val) \ 65#define RETURN_FAMILY_TEST(bit, op, c_op, val) \
57do { \ 66do { \
58 FAMILY_TEST(TEST_RETURN, bit, op, c_op, val); \ 67 FAMILY_TEST(TEST_RETURN, bit, op, c_op, val); \
59} while (0) 68} while (0)
60 69
70#define FETCH_FAMILY_TEST(bit, op, c_op, val) \
71do { \
72 FAMILY_TEST(TEST_FETCH, bit, op, c_op, val); \
73} while (0)
74
61#define TEST_ARGS(bit, op, init, ret, expect, args...) \ 75#define TEST_ARGS(bit, op, init, ret, expect, args...) \
62do { \ 76do { \
63 atomic##bit##_set(&v, init); \ 77 atomic##bit##_set(&v, init); \
@@ -114,6 +128,16 @@ static __init void test_atomic(void)
114 RETURN_FAMILY_TEST(, sub_return, -=, onestwos); 128 RETURN_FAMILY_TEST(, sub_return, -=, onestwos);
115 RETURN_FAMILY_TEST(, sub_return, -=, -one); 129 RETURN_FAMILY_TEST(, sub_return, -=, -one);
116 130
131 FETCH_FAMILY_TEST(, fetch_add, +=, onestwos);
132 FETCH_FAMILY_TEST(, fetch_add, +=, -one);
133 FETCH_FAMILY_TEST(, fetch_sub, -=, onestwos);
134 FETCH_FAMILY_TEST(, fetch_sub, -=, -one);
135
136 FETCH_FAMILY_TEST(, fetch_or, |=, v1);
137 FETCH_FAMILY_TEST(, fetch_and, &=, v1);
138 FETCH_FAMILY_TEST(, fetch_andnot, &= ~, v1);
139 FETCH_FAMILY_TEST(, fetch_xor, ^=, v1);
140
117 INC_RETURN_FAMILY_TEST(, v0); 141 INC_RETURN_FAMILY_TEST(, v0);
118 DEC_RETURN_FAMILY_TEST(, v0); 142 DEC_RETURN_FAMILY_TEST(, v0);
119 143
@@ -154,6 +178,16 @@ static __init void test_atomic64(void)
154 RETURN_FAMILY_TEST(64, sub_return, -=, onestwos); 178 RETURN_FAMILY_TEST(64, sub_return, -=, onestwos);
155 RETURN_FAMILY_TEST(64, sub_return, -=, -one); 179 RETURN_FAMILY_TEST(64, sub_return, -=, -one);
156 180
181 FETCH_FAMILY_TEST(64, fetch_add, +=, onestwos);
182 FETCH_FAMILY_TEST(64, fetch_add, +=, -one);
183 FETCH_FAMILY_TEST(64, fetch_sub, -=, onestwos);
184 FETCH_FAMILY_TEST(64, fetch_sub, -=, -one);
185
186 FETCH_FAMILY_TEST(64, fetch_or, |=, v1);
187 FETCH_FAMILY_TEST(64, fetch_and, &=, v1);
188 FETCH_FAMILY_TEST(64, fetch_andnot, &= ~, v1);
189 FETCH_FAMILY_TEST(64, fetch_xor, ^=, v1);
190
157 INIT(v0); 191 INIT(v0);
158 atomic64_inc(&v); 192 atomic64_inc(&v);
159 r += one; 193 r += one;
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-24 06:53:49 -0500