locking/atomic: Implement atomic{,64,_long}_fetch_{add,sub,and,andnot,or,xor}{,_relaxed,_acquire,_release}()

Now that all the architectures have implemented support for these new
atomic primitives add on the generic infrastructure to expose and use
it.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Borislav Petkov <bp@suse.de>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will.deacon@arm.com>
Cc: linux-arch@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

2 files changed, 62 insertions, 4 deletions

diff --git a/lib/atomic64.c b/lib/atomic64.c
index 2886ebac6567..53c2d5edc826 100644
--- a/lib/atomic64.c
+++ b/lib/atomic64.c
@@ -96,17 +96,41 @@ long long atomic64_##op##_return(long long a, atomic64_t *v)		\
 }                                                                       \
 EXPORT_SYMBOL(atomic64_##op##_return);
 
+#define ATOMIC64_FETCH_OP(op, c_op)                                     \
+long long atomic64_fetch_##op(long long a, atomic64_t *v)               \
+{                                                                       \
+        unsigned long flags;                                            \
+        raw_spinlock_t *lock = lock_addr(v);                            \
+        long long val;                                                  \
+                                                                        \
+        raw_spin_lock_irqsave(lock, flags);                             \
+        val = v->counter;                                               \
+        v->counter c_op a;                                              \
+        raw_spin_unlock_irqrestore(lock, flags);                        \
+        return val;                                                     \
+}                                                                       \
+EXPORT_SYMBOL(atomic64_fetch_##op);
+
 #define ATOMIC64_OPS(op, c_op)                                          \
         ATOMIC64_OP(op, c_op)                                           \
-        ATOMIC64_OP_RETURN(op, c_op)
+        ATOMIC64_OP_RETURN(op, c_op)                                    \
+        ATOMIC64_FETCH_OP(op, c_op)
 
 ATOMIC64_OPS(add, +=)
 ATOMIC64_OPS(sub, -=)
-ATOMIC64_OP(and, &=)
-ATOMIC64_OP(or, |=)
-ATOMIC64_OP(xor, ^=)
 
 #undef ATOMIC64_OPS
+#define ATOMIC64_OPS(op, c_op)                                          \
+        ATOMIC64_OP(op, c_op)                                           \
+        ATOMIC64_OP_RETURN(op, c_op)                                    \
+        ATOMIC64_FETCH_OP(op, c_op)
+
+ATOMIC64_OPS(and, &=)
+ATOMIC64_OPS(or, |=)
+ATOMIC64_OPS(xor, ^=)
+
+#undef ATOMIC64_OPS
+#undef ATOMIC64_FETCH_OP
 #undef ATOMIC64_OP_RETURN
 #undef ATOMIC64_OP
 
diff --git a/lib/atomic64_test.c b/lib/atomic64_test.c
index 123481814320..dbb369145dda 100644
--- a/lib/atomic64_test.c
+++ b/lib/atomic64_test.c
@@ -53,11 +53,25 @@ do {								\
         BUG_ON(atomic##bit##_read(&v) != r);                    \
 } while (0)
 
+#define TEST_FETCH(bit, op, c_op, val)                          \
+do {                                                            \
+        atomic##bit##_set(&v, v0);                              \
+        r = v0;                                                 \
+        r c_op val;                                             \
+        BUG_ON(atomic##bit##_##op(val, &v) != v0);              \
+        BUG_ON(atomic##bit##_read(&v) != r);                    \
+} while (0)
+
 #define RETURN_FAMILY_TEST(bit, op, c_op, val)                  \
 do {                                                            \
         FAMILY_TEST(TEST_RETURN, bit, op, c_op, val);           \
 } while (0)
 
+#define FETCH_FAMILY_TEST(bit, op, c_op, val)                   \
+do {                                                            \
+        FAMILY_TEST(TEST_FETCH, bit, op, c_op, val);            \
+} while (0)
+
 #define TEST_ARGS(bit, op, init, ret, expect, args...)          \
 do {                                                            \
         atomic##bit##_set(&v, init);                            \
@@ -114,6 +128,16 @@ static __init void test_atomic(void)
         RETURN_FAMILY_TEST(, sub_return, -=, onestwos);
         RETURN_FAMILY_TEST(, sub_return, -=, -one);
 
+        FETCH_FAMILY_TEST(, fetch_add, +=, onestwos);
+        FETCH_FAMILY_TEST(, fetch_add, +=, -one);
+        FETCH_FAMILY_TEST(, fetch_sub, -=, onestwos);
+        FETCH_FAMILY_TEST(, fetch_sub, -=, -one);
+
+        FETCH_FAMILY_TEST(, fetch_or,  |=, v1);
+        FETCH_FAMILY_TEST(, fetch_and, &=, v1);
+        FETCH_FAMILY_TEST(, fetch_andnot, &= ~, v1);
+        FETCH_FAMILY_TEST(, fetch_xor, ^=, v1);
+
         INC_RETURN_FAMILY_TEST(, v0);
         DEC_RETURN_FAMILY_TEST(, v0);
 
@@ -154,6 +178,16 @@ static __init void test_atomic64(void)
         RETURN_FAMILY_TEST(64, sub_return, -=, onestwos);
         RETURN_FAMILY_TEST(64, sub_return, -=, -one);
 
+        FETCH_FAMILY_TEST(64, fetch_add, +=, onestwos);
+        FETCH_FAMILY_TEST(64, fetch_add, +=, -one);
+        FETCH_FAMILY_TEST(64, fetch_sub, -=, onestwos);
+        FETCH_FAMILY_TEST(64, fetch_sub, -=, -one);
+
+        FETCH_FAMILY_TEST(64, fetch_or,  |=, v1);
+        FETCH_FAMILY_TEST(64, fetch_and, &=, v1);
+        FETCH_FAMILY_TEST(64, fetch_andnot, &= ~, v1);
+        FETCH_FAMILY_TEST(64, fetch_xor, ^=, v1);
+
         INIT(v0);
         atomic64_inc(&v);
         r += one;