aboutsummaryrefslogtreecommitdiffstats
path: root/arch/mips/include
diff options
context:
space:
mode:
authorRalf Baechle <ralf@linux-mips.org>2008-08-28 10:17:49 -0400
committerRalf Baechle <ralf@linux-mips.org>2008-10-11 11:18:54 -0400
commit2a31b03335e570dce5fdd082e0d71d48b2cb4290 (patch)
tree17c691e8f53874eb309fa8ed9ed14f65167247d2 /arch/mips/include
parentb65a75b8c91c0f05047399401407371678fe9549 (diff)
MIPS: Rewrite spinlocks to ticket locks.
Based on patch by Chad Reese of Cavium Networks. Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
Diffstat (limited to 'arch/mips/include')
-rw-r--r--arch/mips/include/asm/spinlock.h253
-rw-r--r--arch/mips/include/asm/spinlock_types.h7
2 files changed, 188 insertions, 72 deletions
diff --git a/arch/mips/include/asm/spinlock.h b/arch/mips/include/asm/spinlock.h
index bb897016c491..5d98a3cb85b7 100644
--- a/arch/mips/include/asm/spinlock.h
+++ b/arch/mips/include/asm/spinlock.h
@@ -9,62 +9,125 @@
9#ifndef _ASM_SPINLOCK_H 9#ifndef _ASM_SPINLOCK_H
10#define _ASM_SPINLOCK_H 10#define _ASM_SPINLOCK_H
11 11
12#include <linux/compiler.h>
13
12#include <asm/barrier.h> 14#include <asm/barrier.h>
13#include <asm/war.h> 15#include <asm/war.h>
14 16
15/* 17/*
16 * Your basic SMP spinlocks, allowing only a single CPU anywhere 18 * Your basic SMP spinlocks, allowing only a single CPU anywhere
19 *
20 * Simple spin lock operations. There are two variants, one clears IRQ's
21 * on the local processor, one does not.
22 *
23 * These are fair FIFO ticket locks
24 *
25 * (the type definitions are in asm/spinlock_types.h)
17 */ 26 */
18 27
19#define __raw_spin_is_locked(x) ((x)->lock != 0)
20#define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock)
21#define __raw_spin_unlock_wait(x) \
22 do { cpu_relax(); } while ((x)->lock)
23 28
24/* 29/*
25 * Simple spin lock operations. There are two variants, one clears IRQ's 30 * Ticket locks are conceptually two parts, one indicating the current head of
26 * on the local processor, one does not. 31 * the queue, and the other indicating the current tail. The lock is acquired
27 * 32 * by atomically noting the tail and incrementing it by one (thus adding
28 * We make no fairness assumptions. They have a cost. 33 * ourself to the queue and noting our position), then waiting until the head
34 * becomes equal to the the initial value of the tail.
29 */ 35 */
30 36
37static inline int __raw_spin_is_locked(raw_spinlock_t *lock)
38{
39 unsigned int counters = ACCESS_ONCE(lock->lock);
40
41 return ((counters >> 14) ^ counters) & 0x1fff;
42}
43
44#define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock)
45#define __raw_spin_unlock_wait(x) \
46 while (__raw_spin_is_locked(x)) { cpu_relax(); }
47
48static inline int __raw_spin_is_contended(raw_spinlock_t *lock)
49{
50 unsigned int counters = ACCESS_ONCE(lock->lock);
51
52 return (((counters >> 14) - counters) & 0x1fff) > 1;
53}
54
31static inline void __raw_spin_lock(raw_spinlock_t *lock) 55static inline void __raw_spin_lock(raw_spinlock_t *lock)
32{ 56{
33 unsigned int tmp; 57 int my_ticket;
58 int tmp;
34 59
35 if (R10000_LLSC_WAR) { 60 if (R10000_LLSC_WAR) {
36 __asm__ __volatile__( 61 __asm__ __volatile__ (
37 " .set noreorder # __raw_spin_lock \n" 62 " .set push # __raw_spin_lock \n"
38 "1: ll %1, %2 \n" 63 " .set noreorder \n"
39 " bnez %1, 1b \n" 64 " \n"
40 " li %1, 1 \n" 65 "1: ll %[ticket], %[ticket_ptr] \n"
41 " sc %1, %0 \n" 66 " addiu %[my_ticket], %[ticket], 0x4000 \n"
42 " beqzl %1, 1b \n" 67 " sc %[my_ticket], %[ticket_ptr] \n"
68 " beqzl %[my_ticket], 1b \n"
43 " nop \n" 69 " nop \n"
44 " .set reorder \n" 70 " srl %[my_ticket], %[ticket], 14 \n"
45 : "=m" (lock->lock), "=&r" (tmp) 71 " andi %[my_ticket], %[my_ticket], 0x1fff \n"
46 : "m" (lock->lock) 72 " andi %[ticket], %[ticket], 0x1fff \n"
47 : "memory"); 73 " bne %[ticket], %[my_ticket], 4f \n"
74 " subu %[ticket], %[my_ticket], %[ticket] \n"
75 "2: \n"
76 " .subsection 2 \n"
77 "4: andi %[ticket], %[ticket], 0x1fff \n"
78 "5: sll %[ticket], 5 \n"
79 " \n"
80 "6: bnez %[ticket], 6b \n"
81 " subu %[ticket], 1 \n"
82 " \n"
83 " lw %[ticket], %[ticket_ptr] \n"
84 " andi %[ticket], %[ticket], 0x1fff \n"
85 " beq %[ticket], %[my_ticket], 2b \n"
86 " subu %[ticket], %[my_ticket], %[ticket] \n"
87 " b 5b \n"
88 " subu %[ticket], %[ticket], 1 \n"
89 " .previous \n"
90 " .set pop \n"
91 : [ticket_ptr] "+m" (lock->lock),
92 [ticket] "=&r" (tmp),
93 [my_ticket] "=&r" (my_ticket));
48 } else { 94 } else {
49 __asm__ __volatile__( 95 __asm__ __volatile__ (
50 " .set noreorder # __raw_spin_lock \n" 96 " .set push # __raw_spin_lock \n"
51 "1: ll %1, %2 \n" 97 " .set noreorder \n"
52 " bnez %1, 2f \n" 98 " \n"
53 " li %1, 1 \n" 99 " ll %[ticket], %[ticket_ptr] \n"
54 " sc %1, %0 \n" 100 "1: addiu %[my_ticket], %[ticket], 0x4000 \n"
55 " beqz %1, 2f \n" 101 " sc %[my_ticket], %[ticket_ptr] \n"
102 " beqz %[my_ticket], 3f \n"
56 " nop \n" 103 " nop \n"
104 " srl %[my_ticket], %[ticket], 14 \n"
105 " andi %[my_ticket], %[my_ticket], 0x1fff \n"
106 " andi %[ticket], %[ticket], 0x1fff \n"
107 " bne %[ticket], %[my_ticket], 4f \n"
108 " subu %[ticket], %[my_ticket], %[ticket] \n"
109 "2: \n"
57 " .subsection 2 \n" 110 " .subsection 2 \n"
58 "2: ll %1, %2 \n" 111 "3: b 1b \n"
59 " bnez %1, 2b \n" 112 " ll %[ticket], %[ticket_ptr] \n"
60 " li %1, 1 \n" 113 " \n"
61 " b 1b \n" 114 "4: andi %[ticket], %[ticket], 0x1fff \n"
62 " nop \n" 115 "5: sll %[ticket], 5 \n"
116 " \n"
117 "6: bnez %[ticket], 6b \n"
118 " subu %[ticket], 1 \n"
119 " \n"
120 " lw %[ticket], %[ticket_ptr] \n"
121 " andi %[ticket], %[ticket], 0x1fff \n"
122 " beq %[ticket], %[my_ticket], 2b \n"
123 " subu %[ticket], %[my_ticket], %[ticket] \n"
124 " b 5b \n"
125 " subu %[ticket], %[ticket], 1 \n"
63 " .previous \n" 126 " .previous \n"
64 " .set reorder \n" 127 " .set pop \n"
65 : "=m" (lock->lock), "=&r" (tmp) 128 : [ticket_ptr] "+m" (lock->lock),
66 : "m" (lock->lock) 129 [ticket] "=&r" (tmp),
67 : "memory"); 130 [my_ticket] "=&r" (my_ticket));
68 } 131 }
69 132
70 smp_llsc_mb(); 133 smp_llsc_mb();
@@ -72,55 +135,103 @@ static inline void __raw_spin_lock(raw_spinlock_t *lock)
72 135
73static inline void __raw_spin_unlock(raw_spinlock_t *lock) 136static inline void __raw_spin_unlock(raw_spinlock_t *lock)
74{ 137{
75 smp_mb(); 138 int tmp;
76 139
77 __asm__ __volatile__( 140 smp_llsc_mb();
78 " .set noreorder # __raw_spin_unlock \n" 141
79 " sw $0, %0 \n" 142 if (R10000_LLSC_WAR) {
80 " .set\treorder \n" 143 __asm__ __volatile__ (
81 : "=m" (lock->lock) 144 " # __raw_spin_unlock \n"
82 : "m" (lock->lock) 145 "1: ll %[ticket], %[ticket_ptr] \n"
83 : "memory"); 146 " addiu %[ticket], %[ticket], 1 \n"
147 " ori %[ticket], %[ticket], 0x2000 \n"
148 " xori %[ticket], %[ticket], 0x2000 \n"
149 " sc %[ticket], %[ticket_ptr] \n"
150 " beqzl %[ticket], 2f \n"
151 : [ticket_ptr] "+m" (lock->lock),
152 [ticket] "=&r" (tmp));
153 } else {
154 __asm__ __volatile__ (
155 " .set push # __raw_spin_unlock \n"
156 " .set noreorder \n"
157 " \n"
158 " ll %[ticket], %[ticket_ptr] \n"
159 "1: addiu %[ticket], %[ticket], 1 \n"
160 " ori %[ticket], %[ticket], 0x2000 \n"
161 " xori %[ticket], %[ticket], 0x2000 \n"
162 " sc %[ticket], %[ticket_ptr] \n"
163 " beqz %[ticket], 2f \n"
164 " nop \n"
165 " \n"
166 " .subsection 2 \n"
167 "2: b 1b \n"
168 " ll %[ticket], %[ticket_ptr] \n"
169 " .previous \n"
170 " .set pop \n"
171 : [ticket_ptr] "+m" (lock->lock),
172 [ticket] "=&r" (tmp));
173 }
84} 174}
85 175
86static inline unsigned int __raw_spin_trylock(raw_spinlock_t *lock) 176static inline unsigned int __raw_spin_trylock(raw_spinlock_t *lock)
87{ 177{
88 unsigned int temp, res; 178 int tmp, tmp2, tmp3;
89 179
90 if (R10000_LLSC_WAR) { 180 if (R10000_LLSC_WAR) {
91 __asm__ __volatile__( 181 __asm__ __volatile__ (
92 " .set noreorder # __raw_spin_trylock \n" 182 " .set push # __raw_spin_trylock \n"
93 "1: ll %0, %3 \n" 183 " .set noreorder \n"
94 " ori %2, %0, 1 \n" 184 " \n"
95 " sc %2, %1 \n" 185 "1: ll %[ticket], %[ticket_ptr] \n"
96 " beqzl %2, 1b \n" 186 " srl %[my_ticket], %[ticket], 14 \n"
97 " nop \n" 187 " andi %[my_ticket], %[my_ticket], 0x1fff \n"
98 " andi %2, %0, 1 \n" 188 " andi %[now_serving], %[ticket], 0x1fff \n"
99 " .set reorder" 189 " bne %[my_ticket], %[now_serving], 3f \n"
100 : "=&r" (temp), "=m" (lock->lock), "=&r" (res) 190 " addiu %[ticket], %[ticket], 0x4000 \n"
101 : "m" (lock->lock) 191 " sc %[ticket], %[ticket_ptr] \n"
102 : "memory"); 192 " beqzl %[ticket], 1b \n"
193 " li %[ticket], 1 \n"
194 "2: \n"
195 " .subsection 2 \n"
196 "3: b 2b \n"
197 " li %[ticket], 0 \n"
198 " .previous \n"
199 " .set pop \n"
200 : [ticket_ptr] "+m" (lock->lock),
201 [ticket] "=&r" (tmp),
202 [my_ticket] "=&r" (tmp2),
203 [now_serving] "=&r" (tmp3));
103 } else { 204 } else {
104 __asm__ __volatile__( 205 __asm__ __volatile__ (
105 " .set noreorder # __raw_spin_trylock \n" 206 " .set push # __raw_spin_trylock \n"
106 "1: ll %0, %3 \n" 207 " .set noreorder \n"
107 " ori %2, %0, 1 \n" 208 " \n"
108 " sc %2, %1 \n" 209 " ll %[ticket], %[ticket_ptr] \n"
109 " beqz %2, 2f \n" 210 "1: srl %[my_ticket], %[ticket], 14 \n"
110 " andi %2, %0, 1 \n" 211 " andi %[my_ticket], %[my_ticket], 0x1fff \n"
212 " andi %[now_serving], %[ticket], 0x1fff \n"
213 " bne %[my_ticket], %[now_serving], 3f \n"
214 " addiu %[ticket], %[ticket], 0x4000 \n"
215 " sc %[ticket], %[ticket_ptr] \n"
216 " beqz %[ticket], 4f \n"
217 " li %[ticket], 1 \n"
218 "2: \n"
111 " .subsection 2 \n" 219 " .subsection 2 \n"
112 "2: b 1b \n" 220 "3: b 2b \n"
113 " nop \n" 221 " li %[ticket], 0 \n"
222 "4: b 1b \n"
223 " ll %[ticket], %[ticket_ptr] \n"
114 " .previous \n" 224 " .previous \n"
115 " .set reorder" 225 " .set pop \n"
116 : "=&r" (temp), "=m" (lock->lock), "=&r" (res) 226 : [ticket_ptr] "+m" (lock->lock),
117 : "m" (lock->lock) 227 [ticket] "=&r" (tmp),
118 : "memory"); 228 [my_ticket] "=&r" (tmp2),
229 [now_serving] "=&r" (tmp3));
119 } 230 }
120 231
121 smp_llsc_mb(); 232 smp_llsc_mb();
122 233
123 return res == 0; 234 return tmp;
124} 235}
125 236
126/* 237/*
diff --git a/arch/mips/include/asm/spinlock_types.h b/arch/mips/include/asm/spinlock_types.h
index ce26c5048b15..adeedaa116c1 100644
--- a/arch/mips/include/asm/spinlock_types.h
+++ b/arch/mips/include/asm/spinlock_types.h
@@ -6,7 +6,12 @@
6#endif 6#endif
7 7
8typedef struct { 8typedef struct {
9 volatile unsigned int lock; 9 /*
10 * bits 0..13: serving_now
11 * bits 14 : junk data
12 * bits 15..28: ticket
13 */
14 unsigned int lock;
10} raw_spinlock_t; 15} raw_spinlock_t;
11 16
12#define __RAW_SPIN_LOCK_UNLOCKED { 0 } 17#define __RAW_SPIN_LOCK_UNLOCKED { 0 }
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-08 01:43:04 -0400