diff options
Diffstat (limited to 'include/asm-x86/spinlock.h')
-rw-r--r-- | include/asm-x86/spinlock.h | 55 |
1 files changed, 40 insertions, 15 deletions
diff --git a/include/asm-x86/spinlock.h b/include/asm-x86/spinlock.h index 21e89bf92f1c..9726144cdaba 100644 --- a/include/asm-x86/spinlock.h +++ b/include/asm-x86/spinlock.h | |||
@@ -6,7 +6,7 @@ | |||
6 | #include <asm/page.h> | 6 | #include <asm/page.h> |
7 | #include <asm/processor.h> | 7 | #include <asm/processor.h> |
8 | #include <linux/compiler.h> | 8 | #include <linux/compiler.h> |
9 | 9 | #include <asm/paravirt.h> | |
10 | /* | 10 | /* |
11 | * Your basic SMP spinlocks, allowing only a single CPU anywhere | 11 | * Your basic SMP spinlocks, allowing only a single CPU anywhere |
12 | * | 12 | * |
@@ -54,21 +54,21 @@ | |||
54 | * much between them in performance though, especially as locks are out of line. | 54 | * much between them in performance though, especially as locks are out of line. |
55 | */ | 55 | */ |
56 | #if (NR_CPUS < 256) | 56 | #if (NR_CPUS < 256) |
57 | static inline int __raw_spin_is_locked(raw_spinlock_t *lock) | 57 | static inline int __ticket_spin_is_locked(raw_spinlock_t *lock) |
58 | { | 58 | { |
59 | int tmp = ACCESS_ONCE(lock->slock); | 59 | int tmp = ACCESS_ONCE(lock->slock); |
60 | 60 | ||
61 | return (((tmp >> 8) & 0xff) != (tmp & 0xff)); | 61 | return (((tmp >> 8) & 0xff) != (tmp & 0xff)); |
62 | } | 62 | } |
63 | 63 | ||
64 | static inline int __raw_spin_is_contended(raw_spinlock_t *lock) | 64 | static inline int __ticket_spin_is_contended(raw_spinlock_t *lock) |
65 | { | 65 | { |
66 | int tmp = ACCESS_ONCE(lock->slock); | 66 | int tmp = ACCESS_ONCE(lock->slock); |
67 | 67 | ||
68 | return (((tmp >> 8) & 0xff) - (tmp & 0xff)) > 1; | 68 | return (((tmp >> 8) & 0xff) - (tmp & 0xff)) > 1; |
69 | } | 69 | } |
70 | 70 | ||
71 | static __always_inline void __raw_spin_lock(raw_spinlock_t *lock) | 71 | static __always_inline void __ticket_spin_lock(raw_spinlock_t *lock) |
72 | { | 72 | { |
73 | short inc = 0x0100; | 73 | short inc = 0x0100; |
74 | 74 | ||
@@ -87,9 +87,7 @@ static __always_inline void __raw_spin_lock(raw_spinlock_t *lock) | |||
87 | : "memory", "cc"); | 87 | : "memory", "cc"); |
88 | } | 88 | } |
89 | 89 | ||
90 | #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock) | 90 | static __always_inline int __ticket_spin_trylock(raw_spinlock_t *lock) |
91 | |||
92 | static __always_inline int __raw_spin_trylock(raw_spinlock_t *lock) | ||
93 | { | 91 | { |
94 | int tmp; | 92 | int tmp; |
95 | short new; | 93 | short new; |
@@ -110,7 +108,7 @@ static __always_inline int __raw_spin_trylock(raw_spinlock_t *lock) | |||
110 | return tmp; | 108 | return tmp; |
111 | } | 109 | } |
112 | 110 | ||
113 | static __always_inline void __raw_spin_unlock(raw_spinlock_t *lock) | 111 | static __always_inline void __ticket_spin_unlock(raw_spinlock_t *lock) |
114 | { | 112 | { |
115 | asm volatile(UNLOCK_LOCK_PREFIX "incb %0" | 113 | asm volatile(UNLOCK_LOCK_PREFIX "incb %0" |
116 | : "+m" (lock->slock) | 114 | : "+m" (lock->slock) |
@@ -118,21 +116,21 @@ static __always_inline void __raw_spin_unlock(raw_spinlock_t *lock) | |||
118 | : "memory", "cc"); | 116 | : "memory", "cc"); |
119 | } | 117 | } |
120 | #else | 118 | #else |
121 | static inline int __raw_spin_is_locked(raw_spinlock_t *lock) | 119 | static inline int __ticket_spin_is_locked(raw_spinlock_t *lock) |
122 | { | 120 | { |
123 | int tmp = ACCESS_ONCE(lock->slock); | 121 | int tmp = ACCESS_ONCE(lock->slock); |
124 | 122 | ||
125 | return (((tmp >> 16) & 0xffff) != (tmp & 0xffff)); | 123 | return (((tmp >> 16) & 0xffff) != (tmp & 0xffff)); |
126 | } | 124 | } |
127 | 125 | ||
128 | static inline int __raw_spin_is_contended(raw_spinlock_t *lock) | 126 | static inline int __ticket_spin_is_contended(raw_spinlock_t *lock) |
129 | { | 127 | { |
130 | int tmp = ACCESS_ONCE(lock->slock); | 128 | int tmp = ACCESS_ONCE(lock->slock); |
131 | 129 | ||
132 | return (((tmp >> 16) & 0xffff) - (tmp & 0xffff)) > 1; | 130 | return (((tmp >> 16) & 0xffff) - (tmp & 0xffff)) > 1; |
133 | } | 131 | } |
134 | 132 | ||
135 | static __always_inline void __raw_spin_lock(raw_spinlock_t *lock) | 133 | static __always_inline void __ticket_spin_lock(raw_spinlock_t *lock) |
136 | { | 134 | { |
137 | int inc = 0x00010000; | 135 | int inc = 0x00010000; |
138 | int tmp; | 136 | int tmp; |
@@ -153,9 +151,7 @@ static __always_inline void __raw_spin_lock(raw_spinlock_t *lock) | |||
153 | : "memory", "cc"); | 151 | : "memory", "cc"); |
154 | } | 152 | } |
155 | 153 | ||
156 | #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock) | 154 | static __always_inline int __ticket_spin_trylock(raw_spinlock_t *lock) |
157 | |||
158 | static __always_inline int __raw_spin_trylock(raw_spinlock_t *lock) | ||
159 | { | 155 | { |
160 | int tmp; | 156 | int tmp; |
161 | int new; | 157 | int new; |
@@ -177,7 +173,7 @@ static __always_inline int __raw_spin_trylock(raw_spinlock_t *lock) | |||
177 | return tmp; | 173 | return tmp; |
178 | } | 174 | } |
179 | 175 | ||
180 | static __always_inline void __raw_spin_unlock(raw_spinlock_t *lock) | 176 | static __always_inline void __ticket_spin_unlock(raw_spinlock_t *lock) |
181 | { | 177 | { |
182 | asm volatile(UNLOCK_LOCK_PREFIX "incw %0" | 178 | asm volatile(UNLOCK_LOCK_PREFIX "incw %0" |
183 | : "+m" (lock->slock) | 179 | : "+m" (lock->slock) |
@@ -186,6 +182,35 @@ static __always_inline void __raw_spin_unlock(raw_spinlock_t *lock) | |||
186 | } | 182 | } |
187 | #endif | 183 | #endif |
188 | 184 | ||
185 | #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock) | ||
186 | |||
187 | #ifndef CONFIG_PARAVIRT | ||
188 | static inline int __raw_spin_is_locked(raw_spinlock_t *lock) | ||
189 | { | ||
190 | return __ticket_spin_is_locked(lock); | ||
191 | } | ||
192 | |||
193 | static inline int __raw_spin_is_contended(raw_spinlock_t *lock) | ||
194 | { | ||
195 | return __ticket_spin_is_contended(lock); | ||
196 | } | ||
197 | |||
198 | static __always_inline void __raw_spin_lock(raw_spinlock_t *lock) | ||
199 | { | ||
200 | __ticket_spin_lock(lock); | ||
201 | } | ||
202 | |||
203 | static __always_inline int __raw_spin_trylock(raw_spinlock_t *lock) | ||
204 | { | ||
205 | return __ticket_spin_trylock(lock); | ||
206 | } | ||
207 | |||
208 | static __always_inline void __raw_spin_unlock(raw_spinlock_t *lock) | ||
209 | { | ||
210 | __ticket_spin_unlock(lock); | ||
211 | } | ||
212 | #endif /* CONFIG_PARAVIRT */ | ||
213 | |||
189 | static inline void __raw_spin_unlock_wait(raw_spinlock_t *lock) | 214 | static inline void __raw_spin_unlock_wait(raw_spinlock_t *lock) |
190 | { | 215 | { |
191 | while (__raw_spin_is_locked(lock)) | 216 | while (__raw_spin_is_locked(lock)) |