aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2010-08-18 01:49:26 -0400
committerDavid S. Miller <davem@davemloft.net>2010-08-18 01:49:26 -0400
commit9b3bb86acabe0c05923cea1ed3b0bee9439fef4b (patch)
tree44095535e5f2634c319122b718c8998ee3b52511 /arch/sparc
parentb10f997bb0f4e5b34d447f498fb85834a40d3acb (diff)
sparc64: Make rwsems 64-bit.
Basically tip-off the powerpc code, use a 64-bit type and atomic64_t interfaces for the implementation. This gets us off of the by-hand asm code I wrote, which frankly I think probably ruins I-cache hit rates. The idea was the keep the call chains less deep, but anything taking the rw-semaphores probably is also calling other stuff and therefore already has allocated a stack-frame. So no real stack frame savings ever. Ben H. has posted patches to make powerpc use 64-bit too and with some abstractions we can probably use a shared header file somewhere. With suggestions from Sam Ravnborg. Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'arch/sparc')
-rw-r--r--arch/sparc/include/asm/rwsem-const.h12
-rw-r--r--arch/sparc/include/asm/rwsem.h120
-rw-r--r--arch/sparc/lib/Makefile2
-rw-r--r--arch/sparc/lib/rwsem_64.S163
4 files changed, 104 insertions, 193 deletions
diff --git a/arch/sparc/include/asm/rwsem-const.h b/arch/sparc/include/asm/rwsem-const.h
deleted file mode 100644
index e4c61a18bb28..000000000000
--- a/arch/sparc/include/asm/rwsem-const.h
+++ /dev/null
@@ -1,12 +0,0 @@
1/* rwsem-const.h: RW semaphore counter constants. */
2#ifndef _SPARC64_RWSEM_CONST_H
3#define _SPARC64_RWSEM_CONST_H
4
5#define RWSEM_UNLOCKED_VALUE 0x00000000
6#define RWSEM_ACTIVE_BIAS 0x00000001
7#define RWSEM_ACTIVE_MASK 0x0000ffff
8#define RWSEM_WAITING_BIAS (-0x00010000)
9#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
10#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
11
12#endif /* _SPARC64_RWSEM_CONST_H */
diff --git a/arch/sparc/include/asm/rwsem.h b/arch/sparc/include/asm/rwsem.h
index 6e5621006f85..a2b4302869bc 100644
--- a/arch/sparc/include/asm/rwsem.h
+++ b/arch/sparc/include/asm/rwsem.h
@@ -15,16 +15,21 @@
15 15
16#include <linux/list.h> 16#include <linux/list.h>
17#include <linux/spinlock.h> 17#include <linux/spinlock.h>
18#include <asm/rwsem-const.h>
19 18
20struct rwsem_waiter; 19struct rwsem_waiter;
21 20
22struct rw_semaphore { 21struct rw_semaphore {
23 signed int count; 22 signed long count;
24 spinlock_t wait_lock; 23#define RWSEM_UNLOCKED_VALUE 0x00000000L
25 struct list_head wait_list; 24#define RWSEM_ACTIVE_BIAS 0x00000001L
25#define RWSEM_ACTIVE_MASK 0xffffffffL
26#define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1)
27#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
28#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
29 spinlock_t wait_lock;
30 struct list_head wait_list;
26#ifdef CONFIG_DEBUG_LOCK_ALLOC 31#ifdef CONFIG_DEBUG_LOCK_ALLOC
27 struct lockdep_map dep_map; 32 struct lockdep_map dep_map;
28#endif 33#endif
29}; 34};
30 35
@@ -41,6 +46,11 @@ struct rw_semaphore {
41#define DECLARE_RWSEM(name) \ 46#define DECLARE_RWSEM(name) \
42 struct rw_semaphore name = __RWSEM_INITIALIZER(name) 47 struct rw_semaphore name = __RWSEM_INITIALIZER(name)
43 48
49extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);
50extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);
51extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem);
52extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem);
53
44extern void __init_rwsem(struct rw_semaphore *sem, const char *name, 54extern void __init_rwsem(struct rw_semaphore *sem, const char *name,
45 struct lock_class_key *key); 55 struct lock_class_key *key);
46 56
@@ -51,27 +61,103 @@ do { \
51 __init_rwsem((sem), #sem, &__key); \ 61 __init_rwsem((sem), #sem, &__key); \
52} while (0) 62} while (0)
53 63
54extern void __down_read(struct rw_semaphore *sem); 64/*
55extern int __down_read_trylock(struct rw_semaphore *sem); 65 * lock for reading
56extern void __down_write(struct rw_semaphore *sem); 66 */
57extern int __down_write_trylock(struct rw_semaphore *sem); 67static inline void __down_read(struct rw_semaphore *sem)
58extern void __up_read(struct rw_semaphore *sem); 68{
59extern void __up_write(struct rw_semaphore *sem); 69 if (unlikely(atomic64_inc_return((atomic64_t *)(&sem->count)) <= 0L))
60extern void __downgrade_write(struct rw_semaphore *sem); 70 rwsem_down_read_failed(sem);
71}
72
73static inline int __down_read_trylock(struct rw_semaphore *sem)
74{
75 long tmp;
76
77 while ((tmp = sem->count) >= 0L) {
78 if (tmp == cmpxchg(&sem->count, tmp,
79 tmp + RWSEM_ACTIVE_READ_BIAS)) {
80 return 1;
81 }
82 }
83 return 0;
84}
61 85
86/*
87 * lock for writing
88 */
62static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) 89static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
63{ 90{
64 __down_write(sem); 91 long tmp;
92
93 tmp = atomic64_add_return(RWSEM_ACTIVE_WRITE_BIAS,
94 (atomic64_t *)(&sem->count));
95 if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
96 rwsem_down_write_failed(sem);
65} 97}
66 98
67static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem) 99static inline void __down_write(struct rw_semaphore *sem)
68{ 100{
69 return atomic_add_return(delta, (atomic_t *)(&sem->count)); 101 __down_write_nested(sem, 0);
102}
103
104static inline int __down_write_trylock(struct rw_semaphore *sem)
105{
106 long tmp;
107
108 tmp = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
109 RWSEM_ACTIVE_WRITE_BIAS);
110 return tmp == RWSEM_UNLOCKED_VALUE;
70} 111}
71 112
72static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem) 113/*
114 * unlock after reading
115 */
116static inline void __up_read(struct rw_semaphore *sem)
117{
118 long tmp;
119
120 tmp = atomic64_dec_return((atomic64_t *)(&sem->count));
121 if (unlikely(tmp < -1L && (tmp & RWSEM_ACTIVE_MASK) == 0L))
122 rwsem_wake(sem);
123}
124
125/*
126 * unlock after writing
127 */
128static inline void __up_write(struct rw_semaphore *sem)
129{
130 if (unlikely(atomic64_sub_return(RWSEM_ACTIVE_WRITE_BIAS,
131 (atomic64_t *)(&sem->count)) < 0L))
132 rwsem_wake(sem);
133}
134
135/*
136 * implement atomic add functionality
137 */
138static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem)
139{
140 atomic64_add(delta, (atomic64_t *)(&sem->count));
141}
142
143/*
144 * downgrade write lock to read lock
145 */
146static inline void __downgrade_write(struct rw_semaphore *sem)
147{
148 long tmp;
149
150 tmp = atomic64_add_return(-RWSEM_WAITING_BIAS, (atomic64_t *)(&sem->count));
151 if (tmp < 0L)
152 rwsem_downgrade_wake(sem);
153}
154
155/*
156 * implement exchange and add functionality
157 */
158static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem)
73{ 159{
74 atomic_add(delta, (atomic_t *)(&sem->count)); 160 return atomic64_add_return(delta, (atomic64_t *)(&sem->count));
75} 161}
76 162
77static inline int rwsem_is_locked(struct rw_semaphore *sem) 163static inline int rwsem_is_locked(struct rw_semaphore *sem)
diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
index c4b5e03af115..846d1c4374ea 100644
--- a/arch/sparc/lib/Makefile
+++ b/arch/sparc/lib/Makefile
@@ -15,7 +15,7 @@ lib-$(CONFIG_SPARC32) += divdi3.o udivdi3.o
15lib-$(CONFIG_SPARC32) += copy_user.o locks.o 15lib-$(CONFIG_SPARC32) += copy_user.o locks.o
16lib-y += atomic_$(BITS).o 16lib-y += atomic_$(BITS).o
17lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o 17lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o
18lib-y += rwsem_$(BITS).o 18lib-$(CONFIG_SPARC32) += rwsem_32.o
19lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o 19lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o
20 20
21lib-$(CONFIG_SPARC64) += copy_page.o clear_page.o bzero.o 21lib-$(CONFIG_SPARC64) += copy_page.o clear_page.o bzero.o
diff --git a/arch/sparc/lib/rwsem_64.S b/arch/sparc/lib/rwsem_64.S
deleted file mode 100644
index 91a7d29a79d5..000000000000
--- a/arch/sparc/lib/rwsem_64.S
+++ /dev/null
@@ -1,163 +0,0 @@
1/* rwsem.S: RW semaphore assembler.
2 *
3 * Written by David S. Miller (davem@redhat.com), 2001.
4 * Derived from asm-i386/rwsem.h
5 */
6
7#include <asm/rwsem-const.h>
8
9 .section .sched.text, "ax"
10
11 .globl __down_read
12__down_read:
131: lduw [%o0], %g1
14 add %g1, 1, %g7
15 cas [%o0], %g1, %g7
16 cmp %g1, %g7
17 bne,pn %icc, 1b
18 add %g7, 1, %g7
19 cmp %g7, 0
20 bl,pn %icc, 3f
21 nop
222:
23 retl
24 nop
253:
26 save %sp, -192, %sp
27 call rwsem_down_read_failed
28 mov %i0, %o0
29 ret
30 restore
31 .size __down_read, .-__down_read
32
33 .globl __down_read_trylock
34__down_read_trylock:
351: lduw [%o0], %g1
36 add %g1, 1, %g7
37 cmp %g7, 0
38 bl,pn %icc, 2f
39 mov 0, %o1
40 cas [%o0], %g1, %g7
41 cmp %g1, %g7
42 bne,pn %icc, 1b
43 mov 1, %o1
442: retl
45 mov %o1, %o0
46 .size __down_read_trylock, .-__down_read_trylock
47
48 .globl __down_write
49__down_write:
50 sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
51 or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
521:
53 lduw [%o0], %g3
54 add %g3, %g1, %g7
55 cas [%o0], %g3, %g7
56 cmp %g3, %g7
57 bne,pn %icc, 1b
58 cmp %g7, 0
59 bne,pn %icc, 3f
60 nop
612: retl
62 nop
633:
64 save %sp, -192, %sp
65 call rwsem_down_write_failed
66 mov %i0, %o0
67 ret
68 restore
69 .size __down_write, .-__down_write
70
71 .globl __down_write_trylock
72__down_write_trylock:
73 sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
74 or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
751:
76 lduw [%o0], %g3
77 cmp %g3, 0
78 bne,pn %icc, 2f
79 mov 0, %o1
80 add %g3, %g1, %g7
81 cas [%o0], %g3, %g7
82 cmp %g3, %g7
83 bne,pn %icc, 1b
84 mov 1, %o1
852: retl
86 mov %o1, %o0
87 .size __down_write_trylock, .-__down_write_trylock
88
89 .globl __up_read
90__up_read:
911:
92 lduw [%o0], %g1
93 sub %g1, 1, %g7
94 cas [%o0], %g1, %g7
95 cmp %g1, %g7
96 bne,pn %icc, 1b
97 cmp %g7, 0
98 bl,pn %icc, 3f
99 nop
1002: retl
101 nop
1023: sethi %hi(RWSEM_ACTIVE_MASK), %g1
103 sub %g7, 1, %g7
104 or %g1, %lo(RWSEM_ACTIVE_MASK), %g1
105 andcc %g7, %g1, %g0
106 bne,pn %icc, 2b
107 nop
108 save %sp, -192, %sp
109 call rwsem_wake
110 mov %i0, %o0
111 ret
112 restore
113 .size __up_read, .-__up_read
114
115 .globl __up_write
116__up_write:
117 sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
118 or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
1191:
120 lduw [%o0], %g3
121 sub %g3, %g1, %g7
122 cas [%o0], %g3, %g7
123 cmp %g3, %g7
124 bne,pn %icc, 1b
125 sub %g7, %g1, %g7
126 cmp %g7, 0
127 bl,pn %icc, 3f
128 nop
1292:
130 retl
131 nop
1323:
133 save %sp, -192, %sp
134 call rwsem_wake
135 mov %i0, %o0
136 ret
137 restore
138 .size __up_write, .-__up_write
139
140 .globl __downgrade_write
141__downgrade_write:
142 sethi %hi(RWSEM_WAITING_BIAS), %g1
143 or %g1, %lo(RWSEM_WAITING_BIAS), %g1
1441:
145 lduw [%o0], %g3
146 sub %g3, %g1, %g7
147 cas [%o0], %g3, %g7
148 cmp %g3, %g7
149 bne,pn %icc, 1b
150 sub %g7, %g1, %g7
151 cmp %g7, 0
152 bl,pn %icc, 3f
153 nop
1542:
155 retl
156 nop
1573:
158 save %sp, -192, %sp
159 call rwsem_downgrade_wake
160 mov %i0, %o0
161 ret
162 restore
163 .size __downgrade_write, .-__downgrade_write
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-23 19:02:31 -0500