aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/arm/include/asm/spinlock.h4
-rw-r--r--arch/arm64/include/asm/spinlock.h4
-rw-r--r--arch/mips/mm/gup.c2
-rw-r--r--arch/s390/kvm/gaccess.c18
-rw-r--r--arch/x86/include/asm/spinlock.h8
-rw-r--r--arch/x86/mm/gup.c2
-rw-r--r--include/linux/compiler.h74
-rw-r--r--mm/gup.c2
-rw-r--r--mm/memory.c11
-rw-r--r--mm/rmap.c3
10 files changed, 103 insertions, 25 deletions
diff --git a/arch/arm/include/asm/spinlock.h b/arch/arm/include/asm/spinlock.h
index ac4bfae26702..0fa418463f49 100644
--- a/arch/arm/include/asm/spinlock.h
+++ b/arch/arm/include/asm/spinlock.h
@@ -120,12 +120,12 @@ static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
120 120
121static inline int arch_spin_is_locked(arch_spinlock_t *lock) 121static inline int arch_spin_is_locked(arch_spinlock_t *lock)
122{ 122{
123 return !arch_spin_value_unlocked(ACCESS_ONCE(*lock)); 123 return !arch_spin_value_unlocked(READ_ONCE(*lock));
124} 124}
125 125
126static inline int arch_spin_is_contended(arch_spinlock_t *lock) 126static inline int arch_spin_is_contended(arch_spinlock_t *lock)
127{ 127{
128 struct __raw_tickets tickets = ACCESS_ONCE(lock->tickets); 128 struct __raw_tickets tickets = READ_ONCE(lock->tickets);
129 return (tickets.next - tickets.owner) > 1; 129 return (tickets.next - tickets.owner) > 1;
130} 130}
131#define arch_spin_is_contended arch_spin_is_contended 131#define arch_spin_is_contended arch_spin_is_contended
diff --git a/arch/arm64/include/asm/spinlock.h b/arch/arm64/include/asm/spinlock.h
index c45b7b1b7197..cee128732435 100644
--- a/arch/arm64/include/asm/spinlock.h
+++ b/arch/arm64/include/asm/spinlock.h
@@ -99,12 +99,12 @@ static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
99 99
100static inline int arch_spin_is_locked(arch_spinlock_t *lock) 100static inline int arch_spin_is_locked(arch_spinlock_t *lock)
101{ 101{
102 return !arch_spin_value_unlocked(ACCESS_ONCE(*lock)); 102 return !arch_spin_value_unlocked(READ_ONCE(*lock));
103} 103}
104 104
105static inline int arch_spin_is_contended(arch_spinlock_t *lock) 105static inline int arch_spin_is_contended(arch_spinlock_t *lock)
106{ 106{
107 arch_spinlock_t lockval = ACCESS_ONCE(*lock); 107 arch_spinlock_t lockval = READ_ONCE(*lock);
108 return (lockval.next - lockval.owner) > 1; 108 return (lockval.next - lockval.owner) > 1;
109} 109}
110#define arch_spin_is_contended arch_spin_is_contended 110#define arch_spin_is_contended arch_spin_is_contended
diff --git a/arch/mips/mm/gup.c b/arch/mips/mm/gup.c
index 7cba480568c8..70795a67a276 100644
--- a/arch/mips/mm/gup.c
+++ b/arch/mips/mm/gup.c
@@ -30,7 +30,7 @@ retry:
30 30
31 return pte; 31 return pte;
32#else 32#else
33 return ACCESS_ONCE(*ptep); 33 return READ_ONCE(*ptep);
34#endif 34#endif
35} 35}
36 36
diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
index 8b9ccf02a2c5..8a1be9017730 100644
--- a/arch/s390/kvm/gaccess.c
+++ b/arch/s390/kvm/gaccess.c
@@ -227,12 +227,10 @@ static void ipte_lock_simple(struct kvm_vcpu *vcpu)
227 goto out; 227 goto out;
228 ic = &vcpu->kvm->arch.sca->ipte_control; 228 ic = &vcpu->kvm->arch.sca->ipte_control;
229 do { 229 do {
230 old = *ic; 230 old = READ_ONCE(*ic);
231 barrier();
232 while (old.k) { 231 while (old.k) {
233 cond_resched(); 232 cond_resched();
234 old = *ic; 233 old = READ_ONCE(*ic);
235 barrier();
236 } 234 }
237 new = old; 235 new = old;
238 new.k = 1; 236 new.k = 1;
@@ -251,8 +249,7 @@ static void ipte_unlock_simple(struct kvm_vcpu *vcpu)
251 goto out; 249 goto out;
252 ic = &vcpu->kvm->arch.sca->ipte_control; 250 ic = &vcpu->kvm->arch.sca->ipte_control;
253 do { 251 do {
254 old = *ic; 252 old = READ_ONCE(*ic);
255 barrier();
256 new = old; 253 new = old;
257 new.k = 0; 254 new.k = 0;
258 } while (cmpxchg(&ic->val, old.val, new.val) != old.val); 255 } while (cmpxchg(&ic->val, old.val, new.val) != old.val);
@@ -267,12 +264,10 @@ static void ipte_lock_siif(struct kvm_vcpu *vcpu)
267 264
268 ic = &vcpu->kvm->arch.sca->ipte_control; 265 ic = &vcpu->kvm->arch.sca->ipte_control;
269 do { 266 do {
270 old = *ic; 267 old = READ_ONCE(*ic);
271 barrier();
272 while (old.kg) { 268 while (old.kg) {
273 cond_resched(); 269 cond_resched();
274 old = *ic; 270 old = READ_ONCE(*ic);
275 barrier();
276 } 271 }
277 new = old; 272 new = old;
278 new.k = 1; 273 new.k = 1;
@@ -286,8 +281,7 @@ static void ipte_unlock_siif(struct kvm_vcpu *vcpu)
286 281
287 ic = &vcpu->kvm->arch.sca->ipte_control; 282 ic = &vcpu->kvm->arch.sca->ipte_control;
288 do { 283 do {
289 old = *ic; 284 old = READ_ONCE(*ic);
290 barrier();
291 new = old; 285 new = old;
292 new.kh--; 286 new.kh--;
293 if (!new.kh) 287 if (!new.kh)
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index a4efe477ceab..625660f8a2fc 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -92,7 +92,7 @@ static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
92 unsigned count = SPIN_THRESHOLD; 92 unsigned count = SPIN_THRESHOLD;
93 93
94 do { 94 do {
95 if (ACCESS_ONCE(lock->tickets.head) == inc.tail) 95 if (READ_ONCE(lock->tickets.head) == inc.tail)
96 goto out; 96 goto out;
97 cpu_relax(); 97 cpu_relax();
98 } while (--count); 98 } while (--count);
@@ -105,7 +105,7 @@ static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)
105{ 105{
106 arch_spinlock_t old, new; 106 arch_spinlock_t old, new;
107 107
108 old.tickets = ACCESS_ONCE(lock->tickets); 108 old.tickets = READ_ONCE(lock->tickets);
109 if (old.tickets.head != (old.tickets.tail & ~TICKET_SLOWPATH_FLAG)) 109 if (old.tickets.head != (old.tickets.tail & ~TICKET_SLOWPATH_FLAG))
110 return 0; 110 return 0;
111 111
@@ -162,14 +162,14 @@ static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
162 162
163static inline int arch_spin_is_locked(arch_spinlock_t *lock) 163static inline int arch_spin_is_locked(arch_spinlock_t *lock)
164{ 164{
165 struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets); 165 struct __raw_tickets tmp = READ_ONCE(lock->tickets);
166 166
167 return tmp.tail != tmp.head; 167 return tmp.tail != tmp.head;
168} 168}
169 169
170static inline int arch_spin_is_contended(arch_spinlock_t *lock) 170static inline int arch_spin_is_contended(arch_spinlock_t *lock)
171{ 171{
172 struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets); 172 struct __raw_tickets tmp = READ_ONCE(lock->tickets);
173 173
174 return (__ticket_t)(tmp.tail - tmp.head) > TICKET_LOCK_INC; 174 return (__ticket_t)(tmp.tail - tmp.head) > TICKET_LOCK_INC;
175} 175}
diff --git a/arch/x86/mm/gup.c b/arch/x86/mm/gup.c
index 207d9aef662d..d7547824e763 100644
--- a/arch/x86/mm/gup.c
+++ b/arch/x86/mm/gup.c
@@ -15,7 +15,7 @@
15static inline pte_t gup_get_pte(pte_t *ptep) 15static inline pte_t gup_get_pte(pte_t *ptep)
16{ 16{
17#ifndef CONFIG_X86_PAE 17#ifndef CONFIG_X86_PAE
18 return ACCESS_ONCE(*ptep); 18 return READ_ONCE(*ptep);
19#else 19#else
20 /* 20 /*
21 * With get_user_pages_fast, we walk down the pagetables without taking 21 * With get_user_pages_fast, we walk down the pagetables without taking
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index d5ad7b1118fc..a1c81f80978e 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -186,6 +186,80 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);
186# define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __LINE__) 186# define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __LINE__)
187#endif 187#endif
188 188
189#include <uapi/linux/types.h>