aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/asm-x86/rwsem.h169
1 files changed, 86 insertions, 83 deletions
diff --git a/include/asm-x86/rwsem.h b/include/asm-x86/rwsem.h
index 520a379f4b80..750f2a3542b3 100644
--- a/include/asm-x86/rwsem.h
+++ b/include/asm-x86/rwsem.h
@@ -56,14 +56,16 @@ extern asmregparm struct rw_semaphore *
56/* 56/*
57 * the semaphore definition 57 * the semaphore definition
58 */ 58 */
59struct rw_semaphore { 59
60 signed long count;
61#define RWSEM_UNLOCKED_VALUE 0x00000000 60#define RWSEM_UNLOCKED_VALUE 0x00000000
62#define RWSEM_ACTIVE_BIAS 0x00000001 61#define RWSEM_ACTIVE_BIAS 0x00000001
63#define RWSEM_ACTIVE_MASK 0x0000ffff 62#define RWSEM_ACTIVE_MASK 0x0000ffff
64#define RWSEM_WAITING_BIAS (-0x00010000) 63#define RWSEM_WAITING_BIAS (-0x00010000)
65#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS 64#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
66#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) 65#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
66
67struct rw_semaphore {
68 signed long count;
67 spinlock_t wait_lock; 69 spinlock_t wait_lock;
68 struct list_head wait_list; 70 struct list_head wait_list;
69#ifdef CONFIG_DEBUG_LOCK_ALLOC 71#ifdef CONFIG_DEBUG_LOCK_ALLOC
@@ -78,11 +80,13 @@ struct rw_semaphore {
78#endif 80#endif
79 81
80 82
81#define __RWSEM_INITIALIZER(name) \ 83#define __RWSEM_INITIALIZER(name) \
82{ RWSEM_UNLOCKED_VALUE, __SPIN_LOCK_UNLOCKED((name).wait_lock), \ 84{ \
83 LIST_HEAD_INIT((name).wait_list) __RWSEM_DEP_MAP_INIT(name) } 85 RWSEM_UNLOCKED_VALUE, __SPIN_LOCK_UNLOCKED((name).wait_lock), \
86 LIST_HEAD_INIT((name).wait_list) __RWSEM_DEP_MAP_INIT(name) \
87}
84 88
85#define DECLARE_RWSEM(name) \ 89#define DECLARE_RWSEM(name) \
86 struct rw_semaphore name = __RWSEM_INITIALIZER(name) 90 struct rw_semaphore name = __RWSEM_INITIALIZER(name)
87 91
88extern void __init_rwsem(struct rw_semaphore *sem, const char *name, 92extern void __init_rwsem(struct rw_semaphore *sem, const char *name,
@@ -100,16 +104,16 @@ do { \
100 */ 104 */
101static inline void __down_read(struct rw_semaphore *sem) 105static inline void __down_read(struct rw_semaphore *sem)
102{ 106{
103 __asm__ __volatile__( 107 asm volatile("# beginning down_read\n\t"
104 "# beginning down_read\n\t" 108 LOCK_PREFIX " incl (%%eax)\n\t"
105LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */ 109 /* adds 0x00000001, returns the old value */
106 " jns 1f\n" 110 " jns 1f\n"
107 " call call_rwsem_down_read_failed\n" 111 " call call_rwsem_down_read_failed\n"
108 "1:\n\t" 112 "1:\n\t"
109 "# ending down_read\n\t" 113 "# ending down_read\n\t"
110 : "+m" (sem->count) 114 : "+m" (sem->count)
111 : "a" (sem) 115 : "a" (sem)
112 : "memory", "cc"); 116 : "memory", "cc");
113} 117}
114 118
115/* 119/*
@@ -118,21 +122,20 @@ LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value
118static inline int __down_read_trylock(struct rw_semaphore *sem) 122static inline int __down_read_trylock(struct rw_semaphore *sem)
119{ 123{
120 __s32 result, tmp; 124 __s32 result, tmp;
121 __asm__ __volatile__( 125 asm volatile("# beginning __down_read_trylock\n\t"
122 "# beginning __down_read_trylock\n\t" 126 " movl %0,%1\n\t"
123 " movl %0,%1\n\t" 127 "1:\n\t"
124 "1:\n\t" 128 " movl %1,%2\n\t"
125 " movl %1,%2\n\t" 129 " addl %3,%2\n\t"
126 " addl %3,%2\n\t" 130 " jle 2f\n\t"
127 " jle 2f\n\t" 131 LOCK_PREFIX " cmpxchgl %2,%0\n\t"
128LOCK_PREFIX " cmpxchgl %2,%0\n\t" 132 " jnz 1b\n\t"
129 " jnz 1b\n\t" 133 "2:\n\t"
130 "2:\n\t" 134 "# ending __down_read_trylock\n\t"
131 "# ending __down_read_trylock\n\t" 135 : "+m" (sem->count), "=&a" (result), "=&r" (tmp)
132 : "+m" (sem->count), "=&a" (result), "=&r" (tmp) 136 : "i" (RWSEM_ACTIVE_READ_BIAS)
133 : "i" (RWSEM_ACTIVE_READ_BIAS) 137 : "memory", "cc");
134 : "memory", "cc"); 138 return result >= 0 ? 1 : 0;
135 return result>=0 ? 1 : 0;
136} 139}
137 140
138/* 141/*
@@ -143,17 +146,18 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
143 int tmp; 146 int tmp;
144 147
145 tmp = RWSEM_ACTIVE_WRITE_BIAS; 148 tmp = RWSEM_ACTIVE_WRITE_BIAS;
146 __asm__ __volatile__( 149 asm volatile("# beginning down_write\n\t"
147 "# beginning down_write\n\t" 150 LOCK_PREFIX " xadd %%edx,(%%eax)\n\t"
148LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */ 151 /* subtract 0x0000ffff, returns the old value */
149 " testl %%edx,%%edx\n\t" /* was the count 0 before? */ 152 " testl %%edx,%%edx\n\t"
150 " jz 1f\n" 153 /* was the count 0 before? */
151 " call call_rwsem_down_write_failed\n" 154 " jz 1f\n"
152 "1:\n" 155 " call call_rwsem_down_write_failed\n"
153 "# ending down_write" 156 "1:\n"
154 : "+m" (sem->count), "=d" (tmp) 157 "# ending down_write"
155 : "a" (sem), "1" (tmp) 158 : "+m" (sem->count), "=d" (tmp)
156 : "memory", "cc"); 159 : "a" (sem), "1" (tmp)
160 : "memory", "cc");
157} 161}
158 162
159static inline void __down_write(struct rw_semaphore *sem) 163static inline void __down_write(struct rw_semaphore *sem)
@@ -167,7 +171,7 @@ static inline void __down_write(struct rw_semaphore *sem)
167static inline int __down_write_trylock(struct rw_semaphore *sem) 171static inline int __down_write_trylock(struct rw_semaphore *sem)
168{ 172{
169 signed long ret = cmpxchg(&sem->count, 173 signed long ret = cmpxchg(&sem->count,
170 RWSEM_UNLOCKED_VALUE, 174 RWSEM_UNLOCKED_VALUE,
171 RWSEM_ACTIVE_WRITE_BIAS); 175 RWSEM_ACTIVE_WRITE_BIAS);
172 if (ret == RWSEM_UNLOCKED_VALUE) 176 if (ret == RWSEM_UNLOCKED_VALUE)
173 return 1; 177 return 1;
@@ -180,16 +184,16 @@ static inline int __down_write_trylock(struct rw_semaphore *sem)
180static inline void __up_read(struct rw_semaphore *sem) 184static inline void __up_read(struct rw_semaphore *sem)
181{ 185{
182 __s32 tmp = -RWSEM_ACTIVE_READ_BIAS; 186 __s32 tmp = -RWSEM_ACTIVE_READ_BIAS;
183 __asm__ __volatile__( 187 asm volatile("# beginning __up_read\n\t"
184 "# beginning __up_read\n\t" 188 LOCK_PREFIX " xadd %%edx,(%%eax)\n\t"
185LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */ 189 /* subtracts 1, returns the old value */
186 " jns 1f\n\t" 190 " jns 1f\n\t"
187 " call call_rwsem_wake\n" 191 " call call_rwsem_wake\n"
188 "1:\n" 192 "1:\n"
189 "# ending __up_read\n" 193 "# ending __up_read\n"
190 : "+m" (sem->count), "=d" (tmp) 194 : "+m" (sem->count), "=d" (tmp)
191 : "a" (sem), "1" (tmp) 195 : "a" (sem), "1" (tmp)
192 : "memory", "cc"); 196 : "memory", "cc");
193} 197}
194 198
195/* 199/*
@@ -197,17 +201,18 @@ LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old valu
197 */ 201 */
198static inline void __up_write(struct rw_semaphore *sem) 202static inline void __up_write(struct rw_semaphore *sem)
199{ 203{
200 __asm__ __volatile__( 204 asm volatile("# beginning __up_write\n\t"
201 "# beginning __up_write\n\t" 205 " movl %2,%%edx\n\t"
202 " movl %2,%%edx\n\t" 206 LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t"
203LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */ 207 /* tries to transition
204 " jz 1f\n" 208 0xffff0001 -> 0x00000000 */
205 " call call_rwsem_wake\n" 209 " jz 1f\n"
206 "1:\n\t" 210 " call call_rwsem_wake\n"
207 "# ending __up_write\n" 211 "1:\n\t"
208 : "+m" (sem->count) 212 "# ending __up_write\n"
209 : "a" (sem), "i" (-RWSEM_ACTIVE_WRITE_BIAS) 213 : "+m" (sem->count)
210 : "memory", "cc", "edx"); 214 : "a" (sem), "i" (-RWSEM_ACTIVE_WRITE_BIAS)
215 : "memory", "cc", "edx");
211} 216}
212 217
213/* 218/*
@@ -215,16 +220,16 @@ LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 ->
215 */ 220 */
216static inline void __downgrade_write(struct rw_semaphore *sem) 221static inline void __downgrade_write(struct rw_semaphore *sem)
217{ 222{
218 __asm__ __volatile__( 223 asm volatile("# beginning __downgrade_write\n\t"
219 "# beginning __downgrade_write\n\t" 224 LOCK_PREFIX " addl %2,(%%eax)\n\t"
220LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */ 225 /* transitions 0xZZZZ0001 -> 0xYYYY0001 */
221 " jns 1f\n\t" 226 " jns 1f\n\t"
222 " call call_rwsem_downgrade_wake\n" 227 " call call_rwsem_downgrade_wake\n"
223 "1:\n\t" 228 "1:\n\t"
224 "# ending __downgrade_write\n" 229 "# ending __downgrade_write\n"
225 : "+m" (sem->count) 230 : "+m" (sem->count)
226 : "a" (sem), "i" (-RWSEM_WAITING_BIAS) 231 : "a" (sem), "i" (-RWSEM_WAITING_BIAS)
227 : "memory", "cc"); 232 : "memory", "cc");
228} 233}
229 234
230/* 235/*
@@ -232,10 +237,9 @@ LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001
232 */ 237 */
233static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem) 238static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
234{ 239{
235 __asm__ __volatile__( 240 asm volatile(LOCK_PREFIX "addl %1,%0"
236LOCK_PREFIX "addl %1,%0" 241 : "+m" (sem->count)
237 : "+m" (sem->count) 242 : "ir" (delta));
238 : "ir" (delta));
239} 243}
240 244
241/* 245/*
@@ -245,12 +249,11 @@ static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)
245{ 249{
246 int tmp = delta; 250 int tmp = delta;
247 251
248 __asm__ __volatile__( 252 asm volatile(LOCK_PREFIX "xadd %0,%1"
249LOCK_PREFIX "xadd %0,%1" 253 : "+r" (tmp), "+m" (sem->count)
250 : "+r" (tmp), "+m" (sem->count) 254 : : "memory");
251 : : "memory");
252 255
253 return tmp+delta; 256 return tmp + delta;
254} 257}
255 258
256static inline int rwsem_is_locked(struct rw_semaphore *sem) 259static inline int rwsem_is_locked(struct rw_semaphore *sem)