aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2016-05-16 16:41:02 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2016-05-16 16:41:02 -0400
commit3469d261eac65912927dca13ee8f77c744ad7aa2 (patch)
tree09b25f80c065d52ee6e158c189ba44bb28bb76fc
parent1c19b68a279c58d6da4379bf8b6d679a300a1daf (diff)
parent4544ba8c6b1743499cabb682897a469911845f15 (diff)
Merge branch 'locking-rwsem-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull support for killable rwsems from Ingo Molnar: "This, by Michal Hocko, implements down_write_killable(). The main usecase will be to update mm_sem usage sites to use this new API, to allow the mm-reaper introduced in commit aac453635549 ("mm, oom: introduce oom reaper") to tear down oom victim address spaces asynchronously with minimum latencies and without deadlock worries" [ The vfs will want it too as the inode lock is changed from a mutex to a rwsem due to the parallel lookup and readdir updates ] * 'locking-rwsem-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/rwsem: Fix comment on register clobbering locking/rwsem: Fix down_write_killable() locking/rwsem, x86: Add frame annotation for call_rwsem_down_write_failed_killable() locking/rwsem: Provide down_write_killable() locking/rwsem, x86: Provide __down_write_killable() locking/rwsem, s390: Provide __down_write_killable() locking/rwsem, ia64: Provide __down_write_killable() locking/rwsem, alpha: Provide __down_write_killable() locking/rwsem: Introduce basis for down_write_killable() locking/rwsem, sparc: Drop superfluous arch specific implementation locking/rwsem, sh: Drop superfluous arch specific implementation locking/rwsem, xtensa: Drop superfluous arch specific implementation locking/rwsem: Drop explicit memory barriers locking/rwsem: Get rid of __down_write_nested()
Diffstat
-rw-r--r--arch/alpha/include/asm/rwsem.h18
-rw-r--r--arch/ia64/include/asm/rwsem.h22
-rw-r--r--arch/s390/include/asm/rwsem.h18
-rw-r--r--arch/sh/include/asm/Kbuild1
-rw-r--r--arch/sh/include/asm/rwsem.h132
-rw-r--r--arch/sparc/include/asm/Kbuild1
-rw-r--r--arch/sparc/include/asm/rwsem.h124
-rw-r--r--arch/x86/include/asm/rwsem.h42
-rw-r--r--arch/x86/lib/rwsem.S16
-rw-r--r--arch/xtensa/include/asm/Kbuild1
-rw-r--r--arch/xtensa/include/asm/rwsem.h131
-rw-r--r--include/asm-generic/rwsem.h13
-rw-r--r--include/linux/lockdep.h15
-rw-r--r--include/linux/rwsem-spinlock.h2
-rw-r--r--include/linux/rwsem.h3
-rw-r--r--kernel/locking/rwsem-spinlock.c19
-rw-r--r--kernel/locking/rwsem-xadd.c38
-rw-r--r--kernel/locking/rwsem.c19
18 files changed, 189 insertions, 426 deletions
diff --git a/arch/alpha/include/asm/rwsem.h b/arch/alpha/include/asm/rwsem.h
index a83bbea62c67..0131a7058778 100644
--- a/arch/alpha/include/asm/rwsem.h
+++ b/arch/alpha/include/asm/rwsem.h
@@ -63,7 +63,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
63 return res >= 0 ? 1 : 0; 63 return res >= 0 ? 1 : 0;
64} 64}
65 65
66static inline void __down_write(struct rw_semaphore *sem) 66static inline long ___down_write(struct rw_semaphore *sem)
67{ 67{
68 long oldcount; 68 long oldcount;
69#ifndef CONFIG_SMP 69#ifndef CONFIG_SMP
@@ -83,10 +83,24 @@ static inline void __down_write(struct rw_semaphore *sem)
83 :"=&r" (oldcount), "=m" (sem->count), "=&r" (temp) 83 :"=&r" (oldcount), "=m" (sem->count), "=&r" (temp)
84 :"Ir" (RWSEM_ACTIVE_WRITE_BIAS), "m" (sem->count) : "memory"); 84 :"Ir" (RWSEM_ACTIVE_WRITE_BIAS), "m" (sem->count) : "memory");
85#endif 85#endif
86 if (unlikely(oldcount)) 86 return oldcount;
87}
88
89static inline void __down_write(struct rw_semaphore *sem)
90{
91 if (unlikely(___down_write(sem)))
87 rwsem_down_write_failed(sem); 92 rwsem_down_write_failed(sem);
88} 93}
89 94
95static inline int __down_write_killable(struct rw_semaphore *sem)
96{
97 if (unlikely(___down_write(sem)))
98 if (IS_ERR(rwsem_down_write_failed_killable(sem)))
99 return -EINTR;
100
101 return 0;
102}
103
90/* 104/*
91 * trylock for writing -- returns 1 if successful, 0 if contention 105 * trylock for writing -- returns 1 if successful, 0 if contention
92 */ 106 */
diff --git a/arch/ia64/include/asm/rwsem.h b/arch/ia64/include/asm/rwsem.h
index ce112472bdd6..8b23e070b844 100644
--- a/arch/ia64/include/asm/rwsem.h
+++ b/arch/ia64/include/asm/rwsem.h
@@ -49,8 +49,8 @@ __down_read (struct rw_semaphore *sem)
49/* 49/*
50 * lock for writing 50 * lock for writing
51 */ 51 */
52static inline void 52static inline long
53__down_write (struct rw_semaphore *sem) 53___down_write (struct rw_semaphore *sem)
54{ 54{
55 long old, new; 55 long old, new;
56 56
@@ -59,10 +59,26 @@ __down_write (struct rw_semaphore *sem)
59 new = old + RWSEM_ACTIVE_WRITE_BIAS; 59 new = old + RWSEM_ACTIVE_WRITE_BIAS;
60 } while (cmpxchg_acq(&sem->count, old, new) != old); 60 } while (cmpxchg_acq(&sem->count, old, new) != old);
61 61
62 if (old != 0) 62 return old;
63}
64
65static inline void
66__down_write (struct rw_semaphore *sem)
67{
68 if (___down_write(sem))
63 rwsem_down_write_failed(sem); 69 rwsem_down_write_failed(sem);
64} 70}
65 71
72static inline int
73__down_write_killable (struct rw_semaphore *sem)
74{
75 if (___down_write(sem))
76 if (IS_ERR(rwsem_down_write_failed_killable(sem)))
77 return -EINTR;
78
79 return 0;
80}
81
66/* 82/*
67 * unlock after reading 83 * unlock after reading
68 */ 84 */
diff --git a/arch/s390/include/asm/rwsem.h b/arch/s390/include/asm/rwsem.h
index fead491dfc28..c75e4471e618 100644
--- a/arch/s390/include/asm/rwsem.h
+++ b/arch/s390/include/asm/rwsem.h
@@ -90,7 +90,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
90/* 90/*
91 * lock for writing 91 * lock for writing
92 */ 92 */
93static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) 93static inline long ___down_write(struct rw_semaphore *sem)
94{ 94{
95 signed long old, new, tmp; 95 signed long old, new, tmp;
96 96
@@ -104,13 +104,23 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
104 : "=&d" (old), "=&d" (new), "=Q" (sem->count) 104 : "=&d" (old), "=&d" (new), "=Q" (sem->count)
105 : "Q" (sem->count), "m" (tmp) 105 : "Q" (sem->count), "m" (tmp)
106 : "cc", "memory"); 106 : "cc", "memory");
107 if (old != 0) 107
108 rwsem_down_write_failed(sem); 108 return old;
109} 109}
110 110
111static inline void __down_write(struct rw_semaphore *sem) 111static inline void __down_write(struct rw_semaphore *sem)
112{ 112{
113 __down_write_nested(sem, 0); 113 if (___down_write(sem))
114 rwsem_down_write_failed(sem);
115}
116
117static inline int __down_write_killable(struct rw_semaphore *sem)
118{
119 if (___down_write(sem))
120 if (IS_ERR(rwsem_down_write_failed_killable(sem)))
121 return -EINTR;
122
123 return 0;
114} 124}
115 125
116/* 126/*
diff --git a/arch/sh/include/asm/Kbuild b/arch/sh/include/asm/Kbuild
index a319745a7b63..751c3373a92c 100644
--- a/arch/sh/include/asm/Kbuild
+++ b/arch/sh/include/asm/Kbuild
@@ -26,6 +26,7 @@ generic-y += percpu.h
26generic-y += poll.h 26generic-y += poll.h
27generic-y += preempt.h 27generic-y += preempt.h
28generic-y += resource.h 28generic-y += resource.h
29generic-y += rwsem.h
29generic-y += sembuf.h 30generic-y += sembuf.h
30generic-y += serial.h 31generic-y += serial.h
31generic-y += shmbuf.h 32generic-y += shmbuf.h
diff --git a/arch/sh/include/asm/rwsem.h b/arch/sh/include/asm/rwsem.h
deleted file mode 100644
index edab57265293..000000000000
--- a/arch/sh/include/asm/rwsem.h
+++ /dev/null
@@ -1,132 +0,0 @@
1/*
2 * include/asm-sh/rwsem.h: R/W semaphores for SH using the stuff
3 * in lib/rwsem.c.
4 */
5
6#ifndef _ASM_SH_RWSEM_H
7#define _ASM_SH_RWSEM_H
8
9#ifndef _LINUX_RWSEM_H
10#error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead"
11#endif
12
13#ifdef __KERNEL__
14
15#define RWSEM_UNLOCKED_VALUE 0x00000000
16#define RWSEM_ACTIVE_BIAS 0x00000001
17#define RWSEM_ACTIVE_MASK 0x0000ffff
18#define RWSEM_WAITING_BIAS (-0x00010000)
19#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
20#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
21
22/*
23 * lock for reading
24 */
25static inline void __down_read(struct rw_semaphore *sem)
26{
27 if (atomic_inc_return((atomic_t *)(&sem->count)) > 0)
28 smp_wmb();
29 else
30 rwsem_down_read_failed(sem);
31}
32
33static inline int __down_read_trylock(struct rw_semaphore *sem)
34{
35 int tmp;
36
37 while ((tmp = sem->count) >= 0) {
38 if (tmp == cmpxchg(&sem->count, tmp,
39 tmp + RWSEM_ACTIVE_READ_BIAS)) {
40 smp_wmb();
41 return 1;
42 }
43 }
44 return 0;
45}
46
47/*
48 * lock for writing
49 */
50static inline void __down_write(struct rw_semaphore *sem)
51{
52 int tmp;
53
54 tmp = atomic_add_return(RWSEM_ACTIVE_WRITE_BIAS,
55 (atomic_t *)(&sem->count));
56 if (tmp == RWSEM_ACTIVE_WRITE_BIAS)
57 smp_wmb();
58 else
59 rwsem_down_write_failed(sem);
60}
61
62static inline int __down_write_trylock(struct rw_semaphore *sem)
63{
64 int tmp;
65
66 tmp = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
67 RWSEM_ACTIVE_WRITE_BIAS);
68 smp_wmb();
69 return tmp == RWSEM_UNLOCKED_VALUE;
70}
71
72/*
73 * unlock after reading
74 */
75static inline void __up_read(struct rw_semaphore *sem)
76{
77 int tmp;
78
79 smp_wmb();
80 tmp = atomic_dec_return((atomic_t *)(&sem->count));
81 if (tmp < -1 && (tmp & RWSEM_ACTIVE_MASK) == 0)
82 rwsem_wake(sem);
83}
84
85/*
86 * unlock after writing
87 */
88static inline void __up_write(struct rw_semaphore *sem)
89{
90 smp_wmb();
91 if (atomic_sub_return(RWSEM_ACTIVE_WRITE_BIAS,
92 (atomic_t *)(&sem->count)) < 0)
93 rwsem_wake(sem);
94}
95
96/*
97 * implement atomic add functionality
98 */
99static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
100{
101 atomic_add(delta, (atomic_t *)(&sem->count));
102}
103
104/*
105 * downgrade write lock to read lock
106 */
107static inline void __downgrade_write(struct rw_semaphore *sem)
108{
109 int tmp;
110
111 smp_wmb();
112 tmp = atomic_add_return(-RWSEM_WAITING_BIAS, (atomic_t *)(&sem->count));
113 if (tmp < 0)
114 rwsem_downgrade_wake(sem);
115}
116
117static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
118{
119 __down_write(sem);
120}
121
122/*
123 * implement exchange and add functionality
124 */
125static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)
126{
127 smp_mb();
128 return atomic_add_return(delta, (atomic_t *)(&sem->count));
129}
130
131#endif /* __KERNEL__ */
132#endif /* _ASM_SH_RWSEM_H */
diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild
index e928618838bc..6024c26c0585 100644
--- a/arch/sparc/include/asm/Kbuild
+++ b/arch/sparc/include/asm/Kbuild
@@ -16,6 +16,7 @@ generic-y += mm-arch-hooks.h
16generic-y += module.h 16generic-y += module.h
17generic-y += mutex.h 17generic-y += mutex.h
18generic-y += preempt.h 18generic-y += preempt.h
19generic-y += rwsem.h
19generic-y += serial.h 20generic-y += serial.h
20generic-y += trace_clock.h 21generic-y += trace_clock.h
21generic-y += types.h 22generic-y += types.h
diff --git a/arch/sparc/include/asm/rwsem.h b/arch/sparc/include/asm/rwsem.h
deleted file mode 100644
index 069bf4d663a1..000000000000
--- a/arch/sparc/include/asm/rwsem.h
+++ /dev/null
@@ -1,124 +0,0 @@
1/*
2 * rwsem.h: R/W semaphores implemented using CAS
3 *
4 * Written by David S. Miller (davem@redhat.com), 2001.
5 * Derived from asm-i386/rwsem.h
6 */
7#ifndef _SPARC64_RWSEM_H
8#define _SPARC64_RWSEM_H
9
10#ifndef _LINUX_RWSEM_H
11#error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead"
12#endif
13
14#ifdef __KERNEL__
15
16#define RWSEM_UNLOCKED_VALUE 0x00000000L
17#define RWSEM_ACTIVE_BIAS 0x00000001L
18#define RWSEM_ACTIVE_MASK 0xffffffffL
19#define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1)
20#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
21#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
22
23/*
24 * lock for reading
25 */
26static inline void __down_read(struct rw_semaphore *sem)
27{
28 if (unlikely(atomic64_inc_return((atomic64_t *)(&sem->count)) <= 0L))
29 rwsem_down_read_failed(sem);
30}
31
32static inline int __down_read_trylock(struct rw_semaphore *sem)
33{
34 long tmp;
35
36 while ((tmp = sem->count) >= 0L) {
37 if (tmp == cmpxchg(&sem->count, tmp,
38 tmp + RWSEM_ACTIVE_READ_BIAS)) {
39 return 1;
40 }
41 }
42 return 0;
43}
44
45/*
46 * lock for writing
47 */
48static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
49{
50 long tmp;
51
52 tmp = atomic64_add_return(RWSEM_ACTIVE_WRITE_BIAS,
53 (atomic64_t *)(&sem->count));
54 if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
55 rwsem_down_write_failed(sem);
56}
57
58static inline void __down_write(struct rw_semaphore *sem)
59{
60 __down_write_nested(sem, 0);
61}
62
63static inline int __down_write_trylock(struct rw_semaphore *sem)
64{
65 long tmp;
66
67 tmp = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
68 RWSEM_ACTIVE_WRITE_BIAS);
69 return tmp == RWSEM_UNLOCKED_VALUE;
70}
71
72/*
73 * unlock after reading
74 */
75static inline void __up_read(struct rw_semaphore *sem)
76{
77 long tmp;
78
79 tmp = atomic64_dec_return((atomic64_t *)(&sem->count));
80 if (unlikely(tmp < -1L && (tmp & RWSEM_ACTIVE_MASK) == 0L))
81 rwsem_wake(sem);
82}
83
84/*
85 * unlock after writing
86 */
87static inline void __up_write(struct rw_semaphore *sem)
88{
89 if (unlikely(atomic64_sub_return(RWSEM_ACTIVE_WRITE_BIAS,
90 (atomic64_t *)(&sem->count)) < 0L))
91 rwsem_wake(sem);
92}
93
94/*
95 * implement atomic add functionality
96 */
97static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem)
98{
99 atomic64_add(delta, (atomic64_t *)(&sem->count));
100}
101
102/*
103 * downgrade write lock to read lock
104 */
105static inline void __downgrade_write(struct rw_semaphore *sem)
106{
107 long tmp;
108
109 tmp = atomic64_add_return(-RWSEM_WAITING_BIAS, (atomic64_t *)(&sem->count));
110 if (tmp < 0L)
111 rwsem_downgrade_wake(sem);
112}
113
114/*
115 * implement exchange and add functionality
116 */
117static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem)
118{
119 return atomic64_add_return(delta, (atomic64_t *)(&sem->count));
120}
121
122#endif /* __KERNEL__ */
123
124#endif /* _SPARC64_RWSEM_H */
diff --git a/arch/x86/include/asm/rwsem.h b/arch/x86/include/asm/rwsem.h
index ceec86eb68e9..453744c1d347 100644
--- a/arch/x86/include/asm/rwsem.h
+++ b/arch/x86/include/asm/rwsem.h
@@ -99,26 +99,36 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
99/* 99/*
100 * lock for writing 100 * lock for writing
101 */ 101 */
102static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) 102#define ____down_write(sem, slow_path) \
103({ \
104 long tmp; \
105 struct rw_semaphore* ret; \
106 asm volatile("# beginning down_write\n\t" \
107 LOCK_PREFIX " xadd %1,(%3)\n\t" \
108 /* adds 0xffff0001, returns the old value */ \
109 " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t" \
110 /* was the active mask 0 before? */\
111 " jz 1f\n" \
112 " call " slow_path "\n" \
113 "1:\n" \
114 "# ending down_write" \
115 : "+m" (sem->count), "=d" (tmp), "=a" (ret) \
116 : "a" (sem), "1" (RWSEM_ACTIVE_WRITE_BIAS) \
117 : "memory", "cc"); \
118 ret; \
119})
120
121static inline void __down_write(struct rw_semaphore *sem)
103{ 122{
104 long tmp; 123 ____down_write(sem, "call_rwsem_down_write_failed");
105 asm volatile("# beginning down_write\n\t"
106 LOCK_PREFIX " xadd %1,(%2)\n\t"
107 /* adds 0xffff0001, returns the old value */
108 " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t"
109 /* was the active mask 0 before? */
110 " jz 1f\n"
111 " call call_rwsem_down_write_failed\n"
112 "1:\n"
113 "# ending down_write"
114 : "+m" (sem->count), "=d" (tmp)
115 : "a" (sem), "1" (RWSEM_ACTIVE_WRITE_BIAS)
116 : "memory", "cc");
117} 124}
118 125
119static inline void __down_write(struct rw_semaphore *sem) 126static inline int __down_write_killable(struct rw_semaphore *sem)
120{ 127{
121 __down_write_nested(sem, 0); 128 if (IS_ERR(____down_write(sem, "call_rwsem_down_write_failed_killable")))
129 return -EINTR;
130
131 return 0;
122} 132}
123 133
124/* 134/*
diff --git a/arch/x86/lib/rwsem.S b/arch/x86/lib/rwsem.S
index be110efa0096..bf2c6074efd2 100644
--- a/arch/x86/lib/rwsem.S
+++ b/arch/x86/lib/rwsem.S
@@ -29,8 +29,10 @@
29 * there is contention on the semaphore. 29 * there is contention on the semaphore.
30 * 30 *
31 * %eax contains the semaphore pointer on entry. Save the C-clobbered 31 * %eax contains the semaphore pointer on entry. Save the C-clobbered
32 * registers (%eax, %edx and %ecx) except %eax whish is either a return 32 * registers (%eax, %edx and %ecx) except %eax which is either a return
33 * value or just clobbered.. 33 * value or just gets clobbered. Same is true for %edx so make sure GCC
34 * reloads it after the slow path, by making it hold a temporary, for
35 * example see ____down_write().
34 */ 36 */
35 37
36#define save_common_regs \ 38#define save_common_regs \
@@ -106,6 +108,16 @@ ENTRY(call_rwsem_down_write_failed)
106 ret 108 ret
107ENDPROC(call_rwsem_down_write_failed) 109ENDPROC(call_rwsem_down_write_failed)
108 110
111ENTRY(call_rwsem_down_write_failed_killable)
112 FRAME_BEGIN
113 save_common_regs
114 movq %rax,%rdi
115 call rwsem_down_write_failed_killable
116 restore_common_regs
117 FRAME_END
118 ret
119ENDPROC(call_rwsem_down_write_failed_killable)
120
109ENTRY(call_rwsem_wake) 121ENTRY(call_rwsem_wake)
110 FRAME_BEGIN 122 FRAME_BEGIN
111 /* do nothing if still outstanding active readers */ 123 /* do nothing if still outstanding active readers */
diff --git a/arch/xtensa/include/asm/Kbuild b/arch/xtensa/include/asm/Kbuild
index b56855a1382a..28cf4c5d65ef 100644
--- a/arch/xtensa/include/asm/Kbuild
+++ b/arch/xtensa/include/asm/Kbuild
@@ -22,6 +22,7 @@ generic-y += mm-arch-hooks.h
22generic-y += percpu.h 22generic-y += percpu.h
23generic-y += preempt.h 23generic-y += preempt.h
24generic-y += resource.h 24generic-y += resource.h
25generic-y += rwsem.h
25generic-y += sections.h 26generic-y += sections.h
26generic-y += siginfo.h 27generic-y += siginfo.h
27generic-y += statfs.h 28generic-y += statfs.h
diff --git a/arch/xtensa/include/asm/rwsem.h b/arch/xtensa/include/asm/rwsem.h
deleted file mode 100644
index 249619e7e7f2..000000000000
--- a/arch/xtensa/include/asm/rwsem.h
+++ /dev/null
@@ -1,131 +0,0 @@
1/*
2 * include/asm-xtensa/rwsem.h
3 *
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
6 * for more details.
7 *
8 * Largely copied from include/asm-ppc/rwsem.h
9 *
10 * Copyright (C) 2001 - 2005 Tensilica Inc.
11 */
12
13#ifndef _XTENSA_RWSEM_H
14#define _XTENSA_RWSEM_H
15
16#ifndef _LINUX_RWSEM_H
17#error "Please don't include <asm/rwsem.h> directly, use <linux/rwsem.h> instead."
18#endif
19
20#define RWSEM_UNLOCKED_VALUE 0x00000000
21#define RWSEM_ACTIVE_BIAS 0x00000001
22#define RWSEM_ACTIVE_MASK 0x0000ffff
23#define RWSEM_WAITING_BIAS (-0x00010000)
24#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
25#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
26
27/*
28 * lock for reading
29 */
30static inline void __down_read(struct rw_semaphore *sem)
31{
32 if (atomic_add_return(1,(atomic_t *)(&sem->count)) > 0)
33 smp_wmb();
34 else
35 rwsem_down_read_failed(sem);
36}
37
38static inline int __down_read_trylock(struct rw_semaphore *sem)
39{
40 int tmp;
41
42 while ((tmp = sem->count) >= 0) {
43 if (tmp == cmpxchg(&sem->count, tmp,
44 tmp + RWSEM_ACTIVE_READ_BIAS)) {
45 smp_wmb();
46 return 1;
47 }
48 }
49 return 0;
50}
51
52/*
53 * lock for writing
54 */
55static inline void __down_write(struct rw_semaphore *sem)
56{
57 int tmp;
58
59 tmp = atomic_add_return(RWSEM_ACTIVE_WRITE_BIAS,
60 (atomic_t *)(&sem->count));
61 if (tmp == RWSEM_ACTIVE_WRITE_BIAS)
62 smp_wmb();
63 else
64 rwsem_down_write_failed(sem);
65}
66
67static inline int __down_write_trylock(struct rw_semaphore *sem)
68{
69 int tmp;
70
71 tmp = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
72 RWSEM_ACTIVE_WRITE_BIAS);
73 smp_wmb();
74 return tmp == RWSEM_UNLOCKED_VALUE;
75}
76
77/*
78 * unlock after reading
79 */
80static inline void __up_read(struct rw_semaphore *sem)
81{
82 int tmp;
83
84 smp_wmb();
85 tmp = atomic_sub_return(1,(atomic_t *)(&sem->count));
86 if (tmp < -1 && (tmp & RWSEM_ACTIVE_MASK) == 0)
87 rwsem_wake(sem);
88}
89
90/*
91 * unlock after writing
92 */
93static inline void __up_write(struct rw_semaphore *sem)
94{
95 smp_wmb();
96 if (atomic_sub_return(RWSEM_ACTIVE_WRITE_BIAS,
97 (atomic_t *)(&sem->count)) < 0)
98 rwsem_wake(sem);
99}
100
101/*
102 * implement atomic add functionality
103 */
104static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
105{
106 atomic_add(delta, (atomic_t *)(&sem->count));
107}
108
109/*
110 * downgrade write lock to read lock
111 */
112static inline void __downgrade_write(struct rw_semaphore *sem)
113{
114 int tmp;
115
116 smp_wmb();
117 tmp = atomic_add_return(-RWSEM_WAITING_BIAS, (atomic_t *)(&sem->count));
118 if (tmp < 0)
119 rwsem_downgrade_wake(sem);
120}
121
122/*
123 * implement exchange and add functionality
124 */
125static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)
126{
127 smp_mb();
128 return atomic_add_return(delta, (atomic_t *)(&sem->count));
129}
130
131#endif /* _XTENSA_RWSEM_H */
diff --git a/include/asm-generic/rwsem.h b/include/asm-generic/rwsem.h
index d6d5dc98d7da..3fc94a046bf5 100644
--- a/include/asm-generic/rwsem.h
+++ b/include/asm-generic/rwsem.h
@@ -53,7 +53,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
53/* 53/*
54 * lock for writing 54 * lock for writing
55 */ 55 */
56static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) 56static inline void __down_write(struct rw_semaphore *sem)
57{ 57{
58 long tmp; 58 long tmp;
59 59
@@ -63,9 +63,16 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
63 rwsem_down_write_failed(sem); 63 rwsem_down_write_failed(sem);
64} 64}
65 65
66static inline void __down_write(struct rw_semaphore *sem) 66static inline int __down_write_killable(struct rw_semaphore *sem)
67{ 67{
68 __down_write_nested(sem, 0); 68 long tmp;
69
70 tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS,
71 (atomic_long_t *)&sem->count);
72 if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
73 if (IS_ERR(rwsem_down_write_failed_killable(sem)))
74 return -EINTR;
75 return 0;
69} 76}
70 77
71static inline int __down_write_trylock(struct rw_semaphore *sem) 78static inline int __down_write_trylock(struct rw_semaphore *sem)
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
index d10ef06971b5..f75222ea7f16 100644
--- a/include/linux/lockdep.h
+++ b/include/linux/lockdep.h
@@ -446,6 +446,18 @@ do { \
446 lock_acquired(&(_lock)->dep_map, _RET_IP_); \ 446 lock_acquired(&(_lock)->dep_map, _RET_IP_); \
447} while (0) 447} while (0)
448 448
449#define LOCK_CONTENDED_RETURN(_lock, try, lock) \
450({ \
451 int ____err = 0; \
452 if (!try(_lock)) { \
453 lock_contended(&(_lock)->dep_map, _RET_IP_); \
454 ____err = lock(_lock); \
455 } \
456 if (!____err) \
457 lock_acquired(&(_lock)->dep_map, _RET_IP_); \
458 ____err; \
459})
460
449#else /* CONFIG_LOCK_STAT */ 461#else /* CONFIG_LOCK_STAT */
450 462
451#define lock_contended(lockdep_map, ip) do {} while (0) 463#define lock_contended(lockdep_map, ip) do {} while (0)
@@ -454,6 +466,9 @@ do { \
454#define LOCK_CONTENDED(_lock, try, lock) \ 466#define LOCK_CONTENDED(_lock, try, lock) \
455 lock(_lock) 467 lock(_lock)
456 468
469#define LOCK_CONTENDED_RETURN(_lock, try, lock) \
470 lock(_lock)
471
457#endif /* CONFIG_LOCK_STAT */ 472#endif /* CONFIG_LOCK_STAT */
458 473
459#ifdef CONFIG_LOCKDEP 474#ifdef CONFIG_LOCKDEP
diff --git a/include/linux/rwsem-spinlock.h b/include/linux/rwsem-spinlock.h
index 561e8615528d..ae0528b834cd 100644
--- a/include/linux/rwsem-spinlock.h
+++ b/include/linux/rwsem-spinlock.h
@@ -34,7 +34,7 @@ struct rw_semaphore {
34extern void __down_read(struct rw_semaphore *sem); 34extern void __down_read(struct rw_semaphore *sem);
35extern int __down_read_trylock(struct rw_semaphore *sem); 35extern int __down_read_trylock(struct rw_semaphore *sem);
36extern void __down_write(struct rw_semaphore *sem); 36extern void __down_write(struct rw_semaphore *sem);
37extern void __down_write_nested(struct rw_semaphore *sem, int subclass); 37extern int __must_check __down_write_killable(struct rw_semaphore *sem);
38extern int __down_write_trylock(struct rw_semaphore *sem); 38extern int __down_write_trylock(struct rw_semaphore *sem);
39extern void __up_read(struct rw_semaphore *sem); 39extern void __up_read(struct rw_semaphore *sem);
40extern void __up_write(struct rw_semaphore *sem); 40extern void __up_write(struct rw_semaphore *sem);
diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index 8f498cdde280..d1c12d160ace 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -14,6 +14,7 @@
14#include <linux/list.h> 14#include <linux/list.h>
15#include <linux/spinlock.h> 15#include <linux/spinlock.h>
16#include <linux/atomic.h> 16#include <linux/atomic.h>
17#include <linux/err.h>
17#ifdef CONFIG_RWSEM_SPIN_ON_OWNER 18#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
18#include <linux/osq_lock.h> 19#include <linux/osq_lock.h>
19#endif 20#endif
@@ -43,6 +44,7 @@ struct rw_semaphore {
43 44
44extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem); 45extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);
45extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem); 46extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);
47extern struct rw_semaphore *rwsem_down_write_failed_killable(struct rw_semaphore *sem);
46extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *); 48extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *);
47extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem); 49extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem);
48 50
@@ -116,6 +118,7 @@ extern int down_read_trylock(struct rw_semaphore *sem);
116 * lock for writing 118 * lock for writing
117 */ 119 */
118extern void down_write(struct rw_semaphore *sem); 120extern void down_write(struct rw_semaphore *sem);
121extern int __must_check down_write_killable(struct rw_semaphore *sem);
119 122
120/* 123/*
121 * trylock for writing -- returns 1 if successful, 0 if contention 124 * trylock for writing -- returns 1 if successful, 0 if contention
diff --git a/kernel/locking/rwsem-spinlock.c b/kernel/locking/rwsem-spinlock.c
index 3a5048572065..1591f6b3539f 100644
--- a/kernel/locking/rwsem-spinlock.c
+++ b/kernel/locking/rwsem-spinlock.c
@@ -191,11 +191,12 @@ int __down_read_trylock(struct rw_semaphore *sem)
191/* 191/*
192 * get a write lock on the semaphore 192 * get a write lock on the semaphore
193 */ 193 */
194void __sched __down_write_nested(struct rw_semaphore *sem, int subclass) 194int __sched __down_write_common(struct rw_semaphore *sem, int state)
195{ 195{
196 struct rwsem_waiter waiter; 196 struct rwsem_waiter waiter;
197 struct task_struct *tsk; 197 struct task_struct *tsk;
198 unsigned long flags; 198 unsigned long flags;
199 int ret = 0;
199 200
200 raw_spin_lock_irqsave(&sem->wait_lock, flags); 201 raw_spin_lock_irqsave(&sem->wait_lock, flags);
201 202
@@ -215,21 +216,33 @@ void __sched __down_write_nested(struct rw_semaphore *sem, int subclass)
215 */ 216 */
216 if (sem->count == 0) 217 if (sem->count == 0)
217 break; 218 break;
218 set_task_state(tsk, TASK_UNINTERRUPTIBLE); 219 if (signal_pending_state(state, current)) {
220 ret = -EINTR;
221 goto out;
222 }
223 set_task_state(tsk, state);
219 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); 224 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
220 schedule(); 225 schedule();
221 raw_spin_lock_irqsave(&sem->wait_lock, flags); 226 raw_spin_lock_irqsave(&sem->wait_lock, flags);
222 } 227 }
223 /* got the lock */ 228 /* got the lock */
224 sem->count = -1; 229 sem->count = -1;
230out:
225 list_del(&waiter.list); 231 list_del(&waiter.list);
226 232
227 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); 233 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
234
235 return ret;
228} 236}
229 237
230void __sched __down_write(struct rw_semaphore *sem) 238void __sched __down_write(struct rw_semaphore *sem)
231{ 239{
232 __down_write_nested(sem, 0); 240 __down_write_common(sem, TASK_UNINTERRUPTIBLE);
241}
242
243int __sched __down_write_killable(struct rw_semaphore *sem)
244{
245 return __down_write_common(sem, TASK_KILLABLE);
233} 246}
234 247
235/* 248/*
diff --git a/kernel/locking/rwsem-xadd.c b/kernel/locking/rwsem-xadd.c
index a4d4de05b2d1..09e30c6225e5 100644
--- a/kernel/locking/rwsem-xadd.c
+++ b/kernel/locking/rwsem-xadd.c
@@ -433,12 +433,13 @@ static inline bool rwsem_has_spinner(struct rw_semaphore *sem)
433/* 433/*
434 * Wait until we successfully acquire the write lock 434 * Wait until we successfully acquire the write lock
435 */ 435 */
436__visible 436static inline struct rw_semaphore *
437struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem) 437__rwsem_down_write_failed_common(struct rw_semaphore *sem, int state)
438{ 438{
439 long count; 439 long count;
440 bool waiting = true; /* any queued threads before us */ 440 bool waiting = true; /* any queued threads before us */
441 struct rwsem_waiter waiter; 441 struct rwsem_waiter waiter;
442 struct rw_semaphore *ret = sem;
442 443
443 /* undo write bias from down_write operation, stop active locking */ 444 /* undo write bias from down_write operation, stop active locking */
444 count = rwsem_atomic_update(-RWSEM_ACTIVE_WRITE_BIAS, sem); 445 count = rwsem_atomic_update(-RWSEM_ACTIVE_WRITE_BIAS, sem);
@@ -478,7 +479,7 @@ struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem)
478 count = rwsem_atomic_update(RWSEM_WAITING_BIAS, sem); 479 count = rwsem_atomic_update(RWSEM_WAITING_BIAS, sem);
479 480
480 /* wait until we successfully acquire the lock */ 481 /* wait until we successfully acquire the lock */
481 set_current_state(TASK_UNINTERRUPTIBLE); 482 set_current_state(state);
482 while (true) { 483 while (true) {
483 if (rwsem_try_write_lock(count, sem)) 484 if (rwsem_try_write_lock(count, sem))
484 break; 485 break;
@@ -486,21 +487,48 @@ struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem)
486 487
487 /* Block until there are no active lockers. */ 488 /* Block until there are no active lockers. */
488 do { 489 do {
490 if (signal_pending_state(state, current))
491 goto out_nolock;
492
489 schedule(); 493 schedule();
490 set_current_state(TASK_UNINTERRUPTIBLE); 494 set_current_state(state);
491 } while ((count = sem->count) & RWSEM_ACTIVE_MASK); 495 } while ((count = sem->count) & RWSEM_ACTIVE_MASK);
492 496
493 raw_spin_lock_irq(&sem->wait_lock); 497 raw_spin_lock_irq(&sem->wait_lock);
494 } 498 }
495 __set_current_state(TASK_RUNNING); 499 __set_current_state(TASK_RUNNING);
500 list_del(&waiter.list);
501 raw_spin_unlock_irq(&sem->wait_lock);
496 502
503 return ret;
504
505out_nolock:
506 __set_current_state(TASK_RUNNING);
507 raw_spin_lock_irq(&sem->wait_lock);
497 list_del(&waiter.list); 508 list_del(&waiter.list);
509 if (list_empty(&sem->wait_list))
510 rwsem_atomic_update(-RWSEM_WAITING_BIAS, sem);
511 else
512 __rwsem_do_wake(sem, RWSEM_WAKE_ANY);
498 raw_spin_unlock_irq(&sem->wait_lock); 513 raw_spin_unlock_irq(&sem->wait_lock);
499 514
500 return sem; 515 return ERR_PTR(-EINTR);
516}
517
518__visible struct rw_semaphore * __sched
519rwsem_down_write_failed(struct rw_semaphore *sem)
520{
521 return __rwsem_down_write_failed_common(sem, TASK_UNINTERRUPTIBLE);
501} 522}
502EXPORT_SYMBOL(rwsem_down_write_failed); 523EXPORT_SYMBOL(rwsem_down_write_failed);
503 524
525__visible struct rw_semaphore * __sched
526rwsem_down_write_failed_killable(struct rw_semaphore *sem)
527{
528 return __rwsem_down_write_failed_common(sem, TASK_KILLABLE);
529}
530EXPORT_SYMBOL(rwsem_down_write_failed_killable);
531
504/* 532/*
505 * handle waking up a waiter on the semaphore 533 * handle waking up a waiter on the semaphore
506 * - up_read/up_write has decremented the active part of count if we come here 534 * - up_read/up_write has decremented the active part of count if we come here
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index 205be0ce34de..c817216c1615 100644
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -55,6 +55,25 @@ void __sched down_write(struct rw_semaphore *sem)
55EXPORT_SYMBOL(down_write); 55EXPORT_SYMBOL(down_write);
56 56
57/* 57/*
58 * lock for writing
59 */
60int __sched down_write_killable(struct rw_semaphore *sem)
61{
62 might_sleep();
63 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
64
65 if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) {
66 rwsem_release(&sem->dep_map, 1, _RET_IP_);
67 return -EINTR;
68 }
69
70 rwsem_set_owner(sem);
71 return 0;
72}
73
74EXPORT_SYMBOL(down_write_killable);
75
76/*
58 * trylock for writing -- returns 1 if successful, 0 if contention 77 * trylock for writing -- returns 1 if successful, 0 if contention
59 */ 78 */
60int down_write_trylock(struct rw_semaphore *sem) 79int down_write_trylock(struct rw_semaphore *sem)
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-27 20:24:15 -0400