diff options
| author | Ralf Baechle <ralf@linux-mips.org> | 2006-09-27 20:45:21 -0400 |
|---|---|---|
| committer | Ralf Baechle <ralf@linux-mips.org> | 2007-02-13 17:40:50 -0500 |
| commit | f65e4fa8e0c6022ad58dc88d1b11b12589ed7f9f (patch) | |
| tree | 2405e012e079693e0fcfde9ff981c549d6c68a21 /include/asm-mips/spinlock.h | |
| parent | 509cb37e173d4e39cec47238397e91b718730794 (diff) | |
[MIPS] Improve branch prediction in ll/sc atomic operations.
Now that finally all supported versions of binutils have functioning
support for .subsection use .subsection to tweak the branch prediction
I did not modify the R10000 errata variants because it seems unclear if
this will invalidate the workaround which actually relies on the cheesy
prediction of branch likely to cause a misspredict if the sc was
successful.
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
Diffstat (limited to 'include/asm-mips/spinlock.h')
| -rw-r--r-- | include/asm-mips/spinlock.h | 56 |
1 files changed, 44 insertions, 12 deletions
diff --git a/include/asm-mips/spinlock.h b/include/asm-mips/spinlock.h index fc3217fc1118..f1755d28a36a 100644 --- a/include/asm-mips/spinlock.h +++ b/include/asm-mips/spinlock.h | |||
| @@ -3,7 +3,7 @@ | |||
| 3 | * License. See the file "COPYING" in the main directory of this archive | 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. | 4 | * for more details. |
| 5 | * | 5 | * |
| 6 | * Copyright (C) 1999, 2000, 06 by Ralf Baechle | 6 | * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org) |
| 7 | * Copyright (C) 1999, 2000 Silicon Graphics, Inc. | 7 | * Copyright (C) 1999, 2000 Silicon Graphics, Inc. |
| 8 | */ | 8 | */ |
| 9 | #ifndef _ASM_SPINLOCK_H | 9 | #ifndef _ASM_SPINLOCK_H |
| @@ -49,11 +49,18 @@ static inline void __raw_spin_lock(raw_spinlock_t *lock) | |||
| 49 | __asm__ __volatile__( | 49 | __asm__ __volatile__( |
| 50 | " .set noreorder # __raw_spin_lock \n" | 50 | " .set noreorder # __raw_spin_lock \n" |
| 51 | "1: ll %1, %2 \n" | 51 | "1: ll %1, %2 \n" |
| 52 | " bnez %1, 1b \n" | 52 | " bnez %1, 2f \n" |
| 53 | " li %1, 1 \n" | 53 | " li %1, 1 \n" |
| 54 | " sc %1, %0 \n" | 54 | " sc %1, %0 \n" |
| 55 | " beqz %1, 1b \n" | 55 | " beqz %1, 2f \n" |
| 56 | " nop \n" | 56 | " nop \n" |
| 57 | " .subsection 2 \n" | ||
| 58 | "2: ll %1, %2 \n" | ||
| 59 | " bnez %1, 2b \n" | ||
| 60 | " li %1, 1 \n" | ||
| 61 | " b 1b \n" | ||
| 62 | " nop \n" | ||
| 63 | " .previous \n" | ||
| 57 | " .set reorder \n" | 64 | " .set reorder \n" |
| 58 | : "=m" (lock->lock), "=&r" (tmp) | 65 | : "=m" (lock->lock), "=&r" (tmp) |
| 59 | : "m" (lock->lock) | 66 | : "m" (lock->lock) |
| @@ -99,8 +106,12 @@ static inline unsigned int __raw_spin_trylock(raw_spinlock_t *lock) | |||
| 99 | "1: ll %0, %3 \n" | 106 | "1: ll %0, %3 \n" |
| 100 | " ori %2, %0, 1 \n" | 107 | " ori %2, %0, 1 \n" |
| 101 | " sc %2, %1 \n" | 108 | " sc %2, %1 \n" |
| 102 | " beqz %2, 1b \n" | 109 | " beqz %2, 2f \n" |
| 103 | " andi %2, %0, 1 \n" | 110 | " andi %2, %0, 1 \n" |
| 111 | " .subsection 2 \n" | ||
| 112 | "2: b 1b \n" | ||
| 113 | " nop \n" | ||
| 114 | " .previous \n" | ||
| 104 | " .set reorder" | 115 | " .set reorder" |
| 105 | : "=&r" (temp), "=m" (lock->lock), "=&r" (res) | 116 | : "=&r" (temp), "=m" (lock->lock), "=&r" (res) |
| 106 | : "m" (lock->lock) | 117 | : "m" (lock->lock) |
| @@ -154,11 +165,18 @@ static inline void __raw_read_lock(raw_rwlock_t *rw) | |||
| 154 | __asm__ __volatile__( | 165 | __asm__ __volatile__( |
| 155 | " .set noreorder # __raw_read_lock \n" | 166 | " .set noreorder # __raw_read_lock \n" |
| 156 | "1: ll %1, %2 \n" | 167 | "1: ll %1, %2 \n" |
| 157 | " bltz %1, 1b \n" | 168 | " bltz %1, 2f \n" |
| 158 | " addu %1, 1 \n" | 169 | " addu %1, 1 \n" |
| 159 | " sc %1, %0 \n" | 170 | " sc %1, %0 \n" |
| 160 | " beqz %1, 1b \n" | 171 | " beqz %1, 1b \n" |
| 161 | " nop \n" | 172 | " nop \n" |
| 173 | " .subsection 2 \n" | ||
| 174 | "2: ll %1, %2 \n" | ||
| 175 | " bltz %1, 2b \n" | ||
| 176 | " addu %1, 1 \n" | ||
| 177 | " b 1b \n" | ||
| 178 | " nop \n" | ||
| 179 | " .previous \n" | ||
| 162 | " .set reorder \n" | 180 | " .set reorder \n" |
| 163 | : "=m" (rw->lock), "=&r" (tmp) | 181 | : "=m" (rw->lock), "=&r" (tmp) |
| 164 | : "m" (rw->lock) | 182 | : "m" (rw->lock) |
| @@ -192,8 +210,12 @@ static inline void __raw_read_unlock(raw_rwlock_t *rw) | |||
| 192 | "1: ll %1, %2 \n" | 210 | "1: ll %1, %2 \n" |
| 193 | " sub %1, 1 \n" | 211 | " sub %1, 1 \n" |
| 194 | " sc %1, %0 \n" | 212 | " sc %1, %0 \n" |
| 195 | " beqz %1, 1b \n" | 213 | " beqz %1, 2f \n" |
| 214 | " nop \n" | ||
| 215 | " .subsection 2 \n" | ||
| 216 | "2: b 1b \n" | ||
| 196 | " nop \n" | 217 | " nop \n" |
| 218 | " .previous \n" | ||
| 197 | " .set reorder \n" | 219 | " .set reorder \n" |
| 198 | : "=m" (rw->lock), "=&r" (tmp) | 220 | : "=m" (rw->lock), "=&r" (tmp) |
| 199 | : "m" (rw->lock) | 221 | : "m" (rw->lock) |
| @@ -222,11 +244,18 @@ static inline void __raw_write_lock(raw_rwlock_t *rw) | |||
| 222 | __asm__ __volatile__( | 244 | __asm__ __volatile__( |
| 223 | " .set noreorder # __raw_write_lock \n" | 245 | " .set noreorder # __raw_write_lock \n" |
| 224 | "1: ll %1, %2 \n" | 246 | "1: ll %1, %2 \n" |
| 225 | " bnez %1, 1b \n" | 247 | " bnez %1, 2f \n" |
| 226 | " lui %1, 0x8000 \n" | 248 | " lui %1, 0x8000 \n" |
| 227 | " sc %1, %0 \n" | 249 | " sc %1, %0 \n" |
| 228 | " beqz %1, 1b \n" | 250 | " beqz %1, 2f \n" |
| 251 | " nop \n" | ||
| 252 | " .subsection 2 \n" | ||
| 253 | "2: ll %1, %2 \n" | ||
| 254 | " bnez %1, 2b \n" | ||
| 255 | " lui %1, 0x8000 \n" | ||
| 256 | " b 1b \n" | ||
| 229 | " nop \n" | 257 | " nop \n" |
| 258 | " .previous \n" | ||
| 230 | " .set reorder \n" | 259 | " .set reorder \n" |
| 231 | : "=m" (rw->lock), "=&r" (tmp) | 260 | : "=m" (rw->lock), "=&r" (tmp) |
| 232 | : "m" (rw->lock) | 261 | : "m" (rw->lock) |
| @@ -322,12 +351,15 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw) | |||
| 322 | " bnez %1, 2f \n" | 351 | " bnez %1, 2f \n" |
| 323 | " lui %1, 0x8000 \n" | 352 | " lui %1, 0x8000 \n" |
| 324 | " sc %1, %0 \n" | 353 | " sc %1, %0 \n" |
| 325 | " beqz %1, 1b \n" | 354 | " beqz %1, 3f \n" |
| 326 | " nop \n" | 355 | " li %2, 1 \n" |
| 356 | "2: \n" | ||
| 327 | __WEAK_ORDERING_MB | 357 | __WEAK_ORDERING_MB |
| 328 | " li %2, 1 \n" | 358 | " .subsection 2 \n" |
| 359 | "3: b 1b \n" | ||
| 360 | " li %2, 0 \n" | ||
| 361 | " .previous \n" | ||
| 329 | " .set reorder \n" | 362 | " .set reorder \n" |
| 330 | "2: \n" | ||
| 331 | : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret) | 363 | : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret) |
| 332 | : "m" (rw->lock) | 364 | : "m" (rw->lock) |
| 333 | : "memory"); | 365 | : "memory"); |