diff options
| -rw-r--r-- | arch/arm/include/asm/spinlock.h | 4 | ||||
| -rw-r--r-- | arch/arm64/include/asm/spinlock.h | 4 | ||||
| -rw-r--r-- | arch/mips/mm/gup.c | 2 | ||||
| -rw-r--r-- | arch/s390/kvm/gaccess.c | 18 | ||||
| -rw-r--r-- | arch/x86/include/asm/spinlock.h | 8 | ||||
| -rw-r--r-- | arch/x86/mm/gup.c | 2 | ||||
| -rw-r--r-- | include/linux/compiler.h | 74 | ||||
| -rw-r--r-- | mm/gup.c | 2 | ||||
| -rw-r--r-- | mm/memory.c | 11 | ||||
| -rw-r--r-- | mm/rmap.c | 3 |
10 files changed, 103 insertions, 25 deletions
diff --git a/arch/arm/include/asm/spinlock.h b/arch/arm/include/asm/spinlock.h index ac4bfae26702..0fa418463f49 100644 --- a/arch/arm/include/asm/spinlock.h +++ b/arch/arm/include/asm/spinlock.h | |||
| @@ -120,12 +120,12 @@ static inline int arch_spin_value_unlocked(arch_spinlock_t lock) | |||
| 120 | 120 | ||
| 121 | static inline int arch_spin_is_locked(arch_spinlock_t *lock) | 121 | static inline int arch_spin_is_locked(arch_spinlock_t *lock) |
| 122 | { | 122 | { |
| 123 | return !arch_spin_value_unlocked(ACCESS_ONCE(*lock)); | 123 | return !arch_spin_value_unlocked(READ_ONCE(*lock)); |
| 124 | } | 124 | } |
| 125 | 125 | ||
| 126 | static inline int arch_spin_is_contended(arch_spinlock_t *lock) | 126 | static inline int arch_spin_is_contended(arch_spinlock_t *lock) |
| 127 | { | 127 | { |
| 128 | struct __raw_tickets tickets = ACCESS_ONCE(lock->tickets); | 128 | struct __raw_tickets tickets = READ_ONCE(lock->tickets); |
| 129 | return (tickets.next - tickets.owner) > 1; | 129 | return (tickets.next - tickets.owner) > 1; |
| 130 | } | 130 | } |
| 131 | #define arch_spin_is_contended arch_spin_is_contended | 131 | #define arch_spin_is_contended arch_spin_is_contended |
diff --git a/arch/arm64/include/asm/spinlock.h b/arch/arm64/include/asm/spinlock.h index c45b7b1b7197..cee128732435 100644 --- a/arch/arm64/include/asm/spinlock.h +++ b/arch/arm64/include/asm/spinlock.h | |||
| @@ -99,12 +99,12 @@ static inline int arch_spin_value_unlocked(arch_spinlock_t lock) | |||
| 99 | 99 | ||
| 100 | static inline int arch_spin_is_locked(arch_spinlock_t *lock) | 100 | static inline int arch_spin_is_locked(arch_spinlock_t *lock) |
| 101 | { | 101 | { |
| 102 | return !arch_spin_value_unlocked(ACCESS_ONCE(*lock)); | 102 | return !arch_spin_value_unlocked(READ_ONCE(*lock)); |
| 103 | } | 103 | } |
| 104 | 104 | ||
| 105 | static inline int arch_spin_is_contended(arch_spinlock_t *lock) | 105 | static inline int arch_spin_is_contended(arch_spinlock_t *lock) |
| 106 | { | 106 | { |
| 107 | arch_spinlock_t lockval = ACCESS_ONCE(*lock); | 107 | arch_spinlock_t lockval = READ_ONCE(*lock); |
| 108 | return (lockval.next - lockval.owner) > 1; | 108 | return (lockval.next - lockval.owner) > 1; |
| 109 | } | 109 | } |
| 110 | #define arch_spin_is_contended arch_spin_is_contended | 110 | #define arch_spin_is_contended arch_spin_is_contended |
diff --git a/arch/mips/mm/gup.c b/arch/mips/mm/gup.c index 7cba480568c8..70795a67a276 100644 --- a/arch/mips/mm/gup.c +++ b/arch/mips/mm/gup.c | |||
| @@ -30,7 +30,7 @@ retry: | |||
| 30 | 30 | ||
| 31 | return pte; | 31 | return pte; |
| 32 | #else | 32 | #else |
| 33 | return ACCESS_ONCE(*ptep); | 33 | return READ_ONCE(*ptep); |
| 34 | #endif | 34 | #endif |
| 35 | } | 35 | } |
| 36 | 36 | ||
diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 8b9ccf02a2c5..8a1be9017730 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c | |||
| @@ -227,12 +227,10 @@ static void ipte_lock_simple(struct kvm_vcpu *vcpu) | |||
| 227 | goto out; | 227 | goto out; |
| 228 | ic = &vcpu->kvm->arch.sca->ipte_control; | 228 | ic = &vcpu->kvm->arch.sca->ipte_control; |
| 229 | do { | 229 | do { |
| 230 | old = *ic; | 230 | old = READ_ONCE(*ic); |
| 231 | barrier(); | ||
| 232 | while (old.k) { | 231 | while (old.k) { |
| 233 | cond_resched(); | 232 | cond_resched(); |
| 234 | old = *ic; | 233 | old = READ_ONCE(*ic); |
| 235 | barrier(); | ||
| 236 | } | 234 | } |
| 237 | new = old; | 235 | new = old; |
| 238 | new.k = 1; | 236 | new.k = 1; |
| @@ -251,8 +249,7 @@ static void ipte_unlock_simple(struct kvm_vcpu *vcpu) | |||
| 251 | goto out; | 249 | goto out; |
| 252 | ic = &vcpu->kvm->arch.sca->ipte_control; | 250 | ic = &vcpu->kvm->arch.sca->ipte_control; |
| 253 | do { | 251 | do { |
| 254 | old = *ic; | 252 | old = READ_ONCE(*ic); |
| 255 | barrier(); | ||
| 256 | new = old; | 253 | new = old; |
| 257 | new.k = 0; | 254 | new.k = 0; |
| 258 | } while (cmpxchg(&ic->val, old.val, new.val) != old.val); | 255 | } while (cmpxchg(&ic->val, old.val, new.val) != old.val); |
| @@ -267,12 +264,10 @@ static void ipte_lock_siif(struct kvm_vcpu *vcpu) | |||
| 267 | 264 | ||
| 268 | ic = &vcpu->kvm->arch.sca->ipte_control; | 265 | ic = &vcpu->kvm->arch.sca->ipte_control; |
| 269 | do { | 266 | do { |
| 270 | old = *ic; | 267 | old = READ_ONCE(*ic); |
| 271 | barrier(); | ||
| 272 | while (old.kg) { | 268 | while (old.kg) { |
| 273 | cond_resched(); | 269 | cond_resched(); |
| 274 | old = *ic; | 270 | old = READ_ONCE(*ic); |
| 275 | barrier(); | ||
| 276 | } | 271 | } |
| 277 | new = old; | 272 | new = old; |
| 278 | new.k = 1; | 273 | new.k = 1; |
| @@ -286,8 +281,7 @@ static void ipte_unlock_siif(struct kvm_vcpu *vcpu) | |||
| 286 | 281 | ||
| 287 | ic = &vcpu->kvm->arch.sca->ipte_control; | 282 | ic = &vcpu->kvm->arch.sca->ipte_control; |
| 288 | do { | 283 | do { |
| 289 | old = *ic; | 284 | old = READ_ONCE(*ic); |
| 290 | barrier(); | ||
| 291 | new = old; | 285 | new = old; |
| 292 | new.kh--; | 286 | new.kh--; |
| 293 | if (!new.kh) | 287 | if (!new.kh) |
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h index a4efe477ceab..625660f8a2fc 100644 --- a/arch/x86/include/asm/spinlock.h +++ b/arch/x86/include/asm/spinlock.h | |||
| @@ -92,7 +92,7 @@ static __always_inline void arch_spin_lock(arch_spinlock_t *lock) | |||
| 92 | unsigned count = SPIN_THRESHOLD; | 92 | unsigned count = SPIN_THRESHOLD; |
| 93 | 93 | ||
| 94 | do { | 94 | do { |
| 95 | if (ACCESS_ONCE(lock->tickets.head) == inc.tail) | 95 | if (READ_ONCE(lock->tickets.head) == inc.tail) |
| 96 | goto out; | 96 | goto out; |
| 97 | cpu_relax(); | 97 | cpu_relax(); |
| 98 | } while (--count); | 98 | } while (--count); |
| @@ -105,7 +105,7 @@ static __always_inline int arch_spin_trylock(arch_spinlock_t *lock) | |||
| 105 | { | 105 | { |
| 106 | arch_spinlock_t old, new; | 106 | arch_spinlock_t old, new; |
| 107 | 107 | ||
| 108 | old.tickets = ACCESS_ONCE(lock->tickets); | 108 | old.tickets = READ_ONCE(lock->tickets); |
| 109 | if (old.tickets.head != (old.tickets.tail & ~TICKET_SLOWPATH_FLAG)) | 109 | if (old.tickets.head != (old.tickets.tail & ~TICKET_SLOWPATH_FLAG)) |
| 110 | return 0; | 110 | return 0; |
| 111 | 111 | ||
| @@ -162,14 +162,14 @@ static __always_inline void arch_spin_unlock(arch_spinlock_t *lock) | |||
| 162 | 162 | ||
| 163 | static inline int arch_spin_is_locked(arch_spinlock_t *lock) | 163 | static inline int arch_spin_is_locked(arch_spinlock_t *lock) |
| 164 | { | 164 | { |
| 165 | struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets); | 165 | struct __raw_tickets tmp = READ_ONCE(lock->tickets); |
| 166 | 166 | ||
| 167 | return tmp.tail != tmp.head; | 167 | return tmp.tail != tmp.head; |
| 168 | } | 168 | } |
| 169 | 169 | ||
| 170 | static inline int arch_spin_is_contended(arch_spinlock_t *lock) | 170 | static inline int arch_spin_is_contended(arch_spinlock_t *lock) |
| 171 | { | 171 | { |
| 172 | struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets); | 172 | struct __raw_tickets tmp = READ_ONCE(lock->tickets); |
| 173 | 173 | ||
| 174 | return (__ticket_t)(tmp.tail - tmp.head) > TICKET_LOCK_INC; | 174 | return (__ticket_t)(tmp.tail - tmp.head) > TICKET_LOCK_INC; |
| 175 | } | 175 | } |
diff --git a/arch/x86/mm/gup.c b/arch/x86/mm/gup.c index 207d9aef662d..d7547824e763 100644 --- a/arch/x86/mm/gup.c +++ b/arch/x86/mm/gup.c | |||
| @@ -15,7 +15,7 @@ | |||
| 15 | static inline pte_t gup_get_pte(pte_t *ptep) | 15 | static inline pte_t gup_get_pte(pte_t *ptep) |
| 16 | { | 16 | { |
| 17 | #ifndef CONFIG_X86_PAE | 17 | #ifndef CONFIG_X86_PAE |
| 18 | return ACCESS_ONCE(*ptep); | 18 | return READ_ONCE(*ptep); |
| 19 | #else | 19 | #else |
| 20 | /* | 20 | /* |
| 21 | * With get_user_pages_fast, we walk down the pagetables without taking | 21 | * With get_user_pages_fast, we walk down the pagetables without taking |
diff --git a/include/linux/compiler.h b/include/linux/compiler.h index d5ad7b1118fc..a1c81f80978e 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h | |||
| @@ -186,6 +186,80 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); | |||
| 186 | # define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __LINE__) | 186 | # define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __LINE__) |
| 187 | #endif | 187 | #endif |
| 188 | 188 | ||
| 189 | #include <uapi/linux/types.h> | ||
