aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMichel Lespinasse <walken@google.com>2011-03-10 21:48:51 -0500
committerThomas Gleixner <tglx@linutronix.de>2011-03-11 06:23:08 -0500
commit37a9d912b24f96a0591773e6e6c3642991ae5a70 (patch)
tree5c4d5b9a52e2c533269e589115afbd25b6c8534b
parent522d7decc0370070448a8c28982c8dfd8970489e (diff)
futex: Sanitize cmpxchg_futex_value_locked API
The cmpxchg_futex_value_locked API was funny in that it returned either the original, user-exposed futex value OR an error code such as -EFAULT. This was confusing at best, and could be a source of livelocks in places that retry the cmpxchg_futex_value_locked after trying to fix the issue by running fault_in_user_writeable(). This change makes the cmpxchg_futex_value_locked API more similar to the get_futex_value_locked one, returning an error code and updating the original value through a reference argument. Signed-off-by: Michel Lespinasse <walken@google.com> Acked-by: Chris Metcalf <cmetcalf@tilera.com> [tile] Acked-by: Tony Luck <tony.luck@intel.com> [ia64] Acked-by: Thomas Gleixner <tglx@linutronix.de> Tested-by: Michal Simek <monstr@monstr.eu> [microblaze] Acked-by: David Howells <dhowells@redhat.com> [frv] Cc: Darren Hart <darren@dvhart.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Matt Turner <mattst88@gmail.com> Cc: Russell King <linux@arm.linux.org.uk> Cc: Ralf Baechle <ralf@linux-mips.org> Cc: "James E.J. Bottomley" <jejb@parisc-linux.org> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Paul Mundt <lethal@linux-sh.org> Cc: "David S. Miller" <davem@davemloft.net> Cc: Linus Torvalds <torvalds@linux-foundation.org> LKML-Reference: <20110311024851.GC26122@google.com> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Diffstat
-rw-r--r--arch/alpha/include/asm/futex.h22
-rw-r--r--arch/arm/include/asm/futex.h18
-rw-r--r--arch/frv/include/asm/futex.h3
-rw-r--r--arch/ia64/include/asm/futex.h9
-rw-r--r--arch/microblaze/include/asm/futex.h24
-rw-r--r--arch/mips/include/asm/futex.h32
-rw-r--r--arch/parisc/include/asm/futex.h18
-rw-r--r--arch/powerpc/include/asm/futex.h20
-rw-r--r--arch/s390/include/asm/futex.h4
-rw-r--r--arch/s390/include/asm/uaccess.h2
-rw-r--r--arch/s390/lib/uaccess.h4
-rw-r--r--arch/s390/lib/uaccess_pt.c13
-rw-r--r--arch/s390/lib/uaccess_std.c6
-rw-r--r--arch/sh/include/asm/futex-irq.h9
-rw-r--r--arch/sh/include/asm/futex.h5
-rw-r--r--arch/sparc/include/asm/futex_64.h16
-rw-r--r--arch/tile/include/asm/futex.h7
-rw-r--r--arch/x86/include/asm/futex.h16
-rw-r--r--include/asm-generic/futex.h3
-rw-r--r--kernel/futex.c45
20 files changed, 144 insertions, 132 deletions
diff --git a/arch/alpha/include/asm/futex.h b/arch/alpha/include/asm/futex.h
index 945de222ab91..c4e5c2850cce 100644
--- a/arch/alpha/include/asm/futex.h
+++ b/arch/alpha/include/asm/futex.h
@@ -81,21 +81,22 @@ static inline int futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
81} 81}
82 82
83static inline int 83static inline int
84futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) 84futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
85 int oldval, int newval)
85{ 86{
86 int prev, cmp; 87 int ret = 0, prev, cmp;
87 88
88 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) 89 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
89 return -EFAULT; 90 return -EFAULT;
90 91
91 __asm__ __volatile__ ( 92 __asm__ __volatile__ (
92 __ASM_SMP_MB 93 __ASM_SMP_MB
93 "1: ldl_l %0,0(%2)\n" 94 "1: ldl_l %1,0(%3)\n"
94 " cmpeq %0,%3,%1\n" 95 " cmpeq %1,%4,%2\n"
95 " beq %1,3f\n" 96 " beq %2,3f\n"
96 " mov %4,%1\n" 97 " mov %5,%2\n"
97 "2: stl_c %1,0(%2)\n" 98 "2: stl_c %2,0(%3)\n"
98 " beq %1,4f\n" 99 " beq %2,4f\n"
99 "3: .subsection 2\n" 100 "3: .subsection 2\n"
100 "4: br 1b\n" 101 "4: br 1b\n"
101 " .previous\n" 102 " .previous\n"
@@ -105,11 +106,12 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
105 " .long 2b-.\n" 106 " .long 2b-.\n"
106 " lda $31,3b-2b(%0)\n" 107 " lda $31,3b-2b(%0)\n"
107 " .previous\n" 108 " .previous\n"
108 : "=&r"(prev), "=&r"(cmp) 109 : "+r"(ret), "=&r"(prev), "=&r"(cmp)
109 : "r"(uaddr), "r"((long)oldval), "r"(newval) 110 : "r"(uaddr), "r"((long)oldval), "r"(newval)
110 : "memory"); 111 : "memory");
111 112
112 return prev; 113 *uval = prev;
114 return ret;
113} 115}
114 116
115#endif /* __KERNEL__ */ 117#endif /* __KERNEL__ */
diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h
index 7133a8620830..d20b78fce758 100644
--- a/arch/arm/include/asm/futex.h
+++ b/arch/arm/include/asm/futex.h
@@ -88,9 +88,10 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
88} 88}
89 89
90static inline int 90static inline int
91futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) 91futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
92 int oldval, int newval)
92{ 93{
93 int val; 94 int ret = 0, val;
94 95
95 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) 96 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
96 return -EFAULT; 97 return -EFAULT;
@@ -99,24 +100,25 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
99 * call sites. */ 100 * call sites. */
100 101
101 __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n" 102 __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n"
102 "1: " T(ldr) " %0, [%3]\n" 103 "1: " T(ldr) " %1, [%4]\n"
103 " teq %0, %1\n" 104 " teq %1, %2\n"
104 " it eq @ explicit IT needed for the 2b label\n" 105 " it eq @ explicit IT needed for the 2b label\n"
105 "2: " T(streq) " %2, [%3]\n" 106 "2: " T(streq) " %3, [%4]\n"
106 "3:\n" 107 "3:\n"
107 " .pushsection __ex_table,\"a\"\n" 108 " .pushsection __ex_table,\"a\"\n"
108 " .align 3\n" 109 " .align 3\n"
109 " .long 1b, 4f, 2b, 4f\n" 110 " .long 1b, 4f, 2b, 4f\n"
110 " .popsection\n" 111 " .popsection\n"
111 " .pushsection .fixup,\"ax\"\n" 112 " .pushsection .fixup,\"ax\"\n"
112 "4: mov %0, %4\n" 113 "4: mov %0, %5\n"
113 " b 3b\n" 114 " b 3b\n"
114 " .popsection" 115 " .popsection"
115 : "=&r" (val) 116 : "+r" (ret), "=&r" (val)
116 : "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT) 117 : "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT)
117 : "cc", "memory"); 118 : "cc", "memory");
118 119
119 return val; 120 *uval = val;
121 return ret;
120} 122}
121 123
122#endif /* !SMP */ 124#endif /* !SMP */
diff --git a/arch/frv/include/asm/futex.h b/arch/frv/include/asm/futex.h
index 08b3d1da3583..0548f8e4d11e 100644
--- a/arch/frv/include/asm/futex.h
+++ b/arch/frv/include/asm/futex.h
@@ -10,7 +10,8 @@
10extern int futex_atomic_op_inuser(int encoded_op, int __user *uaddr); 10extern int futex_atomic_op_inuser(int encoded_op, int __user *uaddr);
11 11
12static inline int 12static inline int
13futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) 13futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
14 int oldval, int newval)
14{ 15{
15 return -ENOSYS; 16 return -ENOSYS;
16} 17}
diff --git a/arch/ia64/include/asm/futex.h b/arch/ia64/include/asm/futex.h
index c7f0f062239c..b0728404dad0 100644
--- a/arch/ia64/include/asm/futex.h
+++ b/arch/ia64/include/asm/futex.h
@@ -100,23 +100,26 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
100} 100}
101 101
102static inline int 102static inline int
103futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) 103futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
104 int oldval, int newval)
104{ 105{
105 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) 106 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
106 return -EFAULT; 107 return -EFAULT;
107 108
108 { 109 {
109 register unsigned long r8 __asm ("r8"); 110 register unsigned long r8 __asm ("r8") = 0;
111 unsigned long prev;
110 __asm__ __volatile__( 112 __asm__ __volatile__(
111 " mf;; \n" 113 " mf;; \n"
112 " mov ar.ccv=%3;; \n" 114 " mov ar.ccv=%3;; \n"
113 "[1:] cmpxchg4.acq %0=[%1],%2,ar.ccv \n" 115 "[1:] cmpxchg4.acq %0=[%1],%2,ar.ccv \n"
114 " .xdata4 \"__ex_table\", 1b-., 2f-. \n" 116 " .xdata4 \"__ex_table\", 1b-., 2f-. \n"
115 "[2:]" 117 "[2:]"
116 : "=r" (r8) 118 : "=r" (prev)
117 : "r" (uaddr), "r" (newval), 119 : "r" (uaddr), "r" (newval),
118 "rO" ((long) (unsigned) oldval) 120 "rO" ((long) (unsigned) oldval)
119 : "memory"); 121 : "memory");
122 *uval = prev;
120 return r8; 123 return r8;
121 } 124 }
122} 125}
diff --git a/arch/microblaze/include/asm/futex.h b/arch/microblaze/include/asm/futex.h
index ad3fd61b2fe7..fa019ed65dfb 100644
--- a/arch/microblaze/include/asm/futex.h
+++ b/arch/microblaze/include/asm/futex.h
@@ -94,31 +94,33 @@ futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
94} 94}
95 95
96static inline int 96static inline int
97futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) 97futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
98 int oldval, int newval)
98{ 99{
99 int prev, cmp; 100 int ret = 0, prev, cmp;
100 101
101 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) 102 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
102 return -EFAULT; 103 return -EFAULT;
103 104
104 __asm__ __volatile__ ("1: lwx %0, %2, r0; \ 105 __asm__ __volatile__ ("1: lwx %1, %3, r0; \
105 cmp %1, %0, %3; \ 106 cmp %2, %1, %4; \
106 beqi %1, 3f; \ 107 beqi %2, 3f; \
107 2: swx %4, %2, r0; \ 108 2: swx %5, %3, r0; \
108 addic %1, r0, 0; \ 109 addic %2, r0, 0; \
109 bnei %1, 1b; \ 110 bnei %2, 1b; \
110 3: \ 111 3: \
111 .section .fixup,\"ax\"; \ 112 .section .fixup,\"ax\"; \
112 4: brid 3b; \ 113 4: brid 3b; \
113 addik %0, r0, %5; \ 114 addik %0, r0, %6; \
114 .previous; \ 115 .previous; \
115 .section __ex_table,\"a\"; \ 116 .section __ex_table,\"a\"; \
116 .word 1b,4b,2b,4b; \ 117 .word 1b,4b,2b,4b; \
117 .previous;" \ 118 .previous;" \
118 : "=&r" (prev), "=&r"(cmp) \ 119 : "+r" (ret), "=&r" (prev), "=&r"(cmp) \
119 : "r" (uaddr), "r" (oldval), "r" (newval), "i" (-EFAULT)); 120 : "r" (uaddr), "r" (oldval), "r" (newval), "i" (-EFAULT));
120 121
121 return prev; 122 *uval = prev;
123 return ret;
122} 124}
123 125
124#endif /* __KERNEL__ */ 126#endif /* __KERNEL__ */
diff --git a/arch/mips/include/asm/futex.h b/arch/mips/include/asm/futex.h
index b9cce90346cf..692a24bd83b7 100644
--- a/arch/mips/include/asm/futex.h
+++ b/arch/mips/include/asm/futex.h
@@ -132,9 +132,10 @@ futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
132} 132}
133 133
134static inline int 134static inline int
135futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) 135futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
136 int oldval, int newval)
136{ 137{
137 int retval; 138 int ret = 0, val;
138 139
139 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) 140 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
140 return -EFAULT; 141 return -EFAULT;
@@ -145,25 +146,25 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
145 " .set push \n" 146 " .set push \n"
146 " .set noat \n" 147 " .set noat \n"
147 " .set mips3 \n" 148 " .set mips3 \n"
148 "1: ll %0, %2 \n" 149 "1: ll %1, %3 \n"
149 " bne %0, %z3, 3f \n" 150 " bne %1, %z4, 3f \n"
150 " .set mips0 \n" 151 " .set mips0 \n"
151 " move $1, %z4 \n" 152 " move $1, %z5 \n"
152 " .set mips3 \n" 153 " .set mips3 \n"
153 "2: sc $1, %1 \n" 154 "2: sc $1, %2 \n"
154 " beqzl $1, 1b \n" 155 " beqzl $1, 1b \n"
155 __WEAK_LLSC_MB 156 __WEAK_LLSC_MB
156 "3: \n" 157 "3: \n"
157 " .set pop \n" 158 " .set pop \n"
158 " .section .fixup,\"ax\" \n" 159 " .section .fixup,\"ax\" \n"
159 "4: li %0, %5 \n" 160 "4: li %0, %6 \n"
160 " j 3b \n" 161 " j 3b \n"
161 " .previous \n" 162 " .previous \n"
162 " .section __ex_table,\"a\" \n" 163 " .section __ex_table,\"a\" \n"
163 " "__UA_ADDR "\t1b, 4b \n" 164 " "__UA_ADDR "\t1b, 4b \n"
164 " "__UA_ADDR "\t2b, 4b \n" 165 " "__UA_ADDR "\t2b, 4b \n"
165 " .previous \n" 166 " .previous \n"
166 : "=&r" (retval), "=R" (*uaddr) 167 : "+r" (ret), "=&r" (val), "=R" (*uaddr)
167 : "R" (*uaddr), "Jr" (oldval), "Jr" (newval), "i" (-EFAULT) 168 : "R" (*uaddr), "Jr" (oldval), "Jr" (newval), "i" (-EFAULT)
168 : "memory"); 169 : "memory");
169 } else if (cpu_has_llsc) { 170 } else if (cpu_has_llsc) {
@@ -172,31 +173,32 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
172 " .set push \n" 173 " .set push \n"
173 " .set noat \n" 174 " .set noat \n"
174 " .set mips3 \n" 175 " .set mips3 \n"
175 "1: ll %0, %2 \n" 176 "1: ll %1, %3 \n"
176 " bne %0, %z3, 3f \n" 177 " bne %1, %z4, 3f \n"
177 " .set mips0 \n" 178 " .set mips0 \n"
178 " move $1, %z4 \n" 179 " move $1, %z5 \n"
179 " .set mips3 \n" 180 " .set mips3 \n"
180 "2: sc $1, %1 \n" 181 "2: sc $1, %2 \n"
181 " beqz $1, 1b \n" 182 " beqz $1, 1b \n"
182 __WEAK_LLSC_MB 183 __WEAK_LLSC_MB
183 "3: \n" 184 "3: \n"
184 " .set pop \n" 185 " .set pop \n"
185 " .section .fixup,\"ax\" \n" 186 " .section .fixup,\"ax\" \n"
186 "4: li %0, %5 \n" 187 "4: li %0, %6 \n"
187 " j 3b \n" 188 " j 3b \n"
188 " .previous \n" 189 " .previous \n"
189 " .section __ex_table,\"a\" \n" 190 " .section __ex_table,\"a\" \n"
190 " "__UA_ADDR "\t1b, 4b \n" 191 " "__UA_ADDR "\t1b, 4b \n"
191 " "__UA_ADDR "\t2b, 4b \n" 192 " "__UA_ADDR "\t2b, 4b \n"
192 " .previous \n" 193 " .previous \n"
193 : "=&r" (retval), "=R" (*uaddr) 194 : "+r" (ret), "=&r" (val), "=R" (*uaddr)
194 : "R" (*uaddr), "Jr" (oldval), "Jr" (newval), "i" (-EFAULT) 195 : "R" (*uaddr), "Jr" (oldval), "Jr" (newval), "i" (-EFAULT)
195 : "memory"); 196 : "memory");
196 } else 197 } else
197 return -ENOSYS; 198 return -ENOSYS;
198 199
199 return retval; 200 *uval = val;
201 return ret;
200} 202}
201 203
202#endif 204#endif
diff --git a/arch/parisc/include/asm/futex.h b/arch/parisc/include/asm/futex.h
index 0c705c3a55ef..4c6d8672325b 100644
--- a/arch/parisc/include/asm/futex.h
+++ b/arch/parisc/include/asm/futex.h
@@ -51,10 +51,10 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
51 51
52/* Non-atomic version */ 52/* Non-atomic version */
53static inline int 53static inline int
54futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) 54futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
55 int oldval, int newval)
55{ 56{
56 int err = 0; 57 int val;
57 int uval;
58 58
59 /* futex.c wants to do a cmpxchg_inatomic on kernel NULL, which is 59 /* futex.c wants to do a cmpxchg_inatomic on kernel NULL, which is
60 * our gateway page, and causes no end of trouble... 60 * our gateway page, and causes no end of trouble...
@@ -65,12 +65,12 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)
65 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) 65 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
66 return -EFAULT; 66 return -EFAULT;
67 67
68 err = get_user(uval, uaddr); 68 if (get_user(val, uaddr))
69 if (err) return -EFAULT; 69 return -EFAULT;
70 if (uval == oldval) 70 if (val == oldval && put_user(newval, uaddr))
71 err = put_user(newval, uaddr); 71 return -EFAULT;
72 if (err) return -EFAULT; 72 *uval = val;
73 return uval; 73 return 0;
74} 74}
75 75
76#endif /*__KERNEL__*/ 76#endif /*__KERNEL__*/
diff --git a/arch/powerpc/include/asm/futex.h b/arch/powerpc/include/asm/futex.h
index 7c589ef81fb0..631e8da60064 100644
--- a/arch/powerpc/include/asm/futex.h
+++ b/arch/powerpc/include/asm/futex.h
@@ -82,35 +82,37 @@ static inline int futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
82} 82}
83 83
84static inline int 84static inline int
85futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) 85futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
86 int oldval, int newval)
86{ 87{
87 int prev; 88 int ret = 0, prev;
88 89
89 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) 90 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
90 return -EFAULT; 91 return -EFAULT;
91 92
92 __asm__ __volatile__ ( 93 __asm__ __volatile__ (
93 PPC_RELEASE_BARRIER 94 PPC_RELEASE_BARRIER
94"1: lwarx %0,0,%2 # futex_atomic_cmpxchg_inatomic\n\ 95"1: lwarx %1,0,%3 # futex_atomic_cmpxchg_inatomic\n\
95 cmpw 0,%0,%3\n\ 96 cmpw 0,%1,%4\n\
96 bne- 3f\n" 97 bne- 3f\n"
97 PPC405_ERR77(0,%2) 98 PPC405_ERR77(0,%3)
98"2: stwcx. %4,0,%2\n\ 99"2: stwcx. %5,0,%3\n\
99 bne- 1b\n" 100 bne- 1b\n"
100 PPC_ACQUIRE_BARRIER 101 PPC_ACQUIRE_BARRIER
101"3: .section .fixup,\"ax\"\n\ 102"3: .section .fixup,\"ax\"\n\
1024: li %0,%5\n\ 1034: li %0,%6\n\
103 b 3b\n\ 104 b 3b\n\
104 .previous\n\ 105 .previous\n\
105 .section __ex_table,\"a\"\n\ 106 .section __ex_table,\"a\"\n\
106 .align 3\n\ 107 .align 3\n\
107 " PPC_LONG "1b,4b,2b,4b\n\ 108 " PPC_LONG "1b,4b,2b,4b\n\
108 .previous" \ 109 .previous" \
109 : "=&r" (prev), "+m" (*uaddr) 110 : "+r" (ret), "=&r" (prev), "+m" (*uaddr)
110 : "r" (uaddr), "r" (oldval), "r" (newval), "i" (-EFAULT) 111 : "r" (uaddr), "r" (oldval), "r" (newval), "i" (-EFAULT)
111 : "cc", "memory"); 112 : "cc", "memory");
112 113
113 return prev; 114 *uval = prev;
115 return ret;
114} 116}
115 117
116#endif /* __KERNEL__ */ 118#endif /* __KERNEL__ */
diff --git a/arch/s390/include/asm/futex.h b/arch/s390/include/asm/futex.h
index 5c5d02de49e9..27ac515ef59c 100644
--- a/arch/s390/include/asm/futex.h
+++ b/arch/s390/include/asm/futex.h
@@ -39,13 +39,13 @@ static inline int futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
39 return ret; 39 return ret;
40} 40}
41 41
42static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr, 42static inline int futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
43 int oldval, int newval) 43 int oldval, int newval)
44{ 44{
45 if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int))) 45 if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
46 return -EFAULT; 46 return -EFAULT;
47 47
48 return uaccess.futex_atomic_cmpxchg(uaddr, oldval, newval); 48 return uaccess.futex_atomic_cmpxchg(uval, uaddr, oldval, newval);
49} 49}
50 50
51#endif /* __KERNEL__ */ 51#endif /* __KERNEL__ */
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index d6b1ed0ec52b..549adf6a9b8b 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -84,7 +84,7 @@ struct uaccess_ops {
84 size_t (*strnlen_user)(size_t, const char __user *); 84 size_t (*strnlen_user)(size_t, const char __user *);
85 size_t (*strncpy_from_user)(size_t, const char __user *, char *); 85 size_t (*strncpy_from_user)(size_t, const char __user *, char *);
86 int (*futex_atomic_op)(int op, int __user *, int oparg, int *old); 86 int (*futex_atomic_op)(int op, int __user *, int oparg, int *old);
87 int (*futex_atomic_cmpxchg)(int __user *, int old, int new); 87 int (*futex_atomic_cmpxchg)(int *, int __user *, int old, int new);
88}; 88};
89 89
90extern struct uaccess_ops uaccess; 90extern struct uaccess_ops uaccess;
diff --git a/arch/s390/lib/uaccess.h b/arch/s390/lib/uaccess.h
index 126011df14f1..89a80674e44b 100644
--- a/arch/s390/lib/uaccess.h
+++ b/arch/s390/lib/uaccess.h
@@ -12,12 +12,12 @@ extern size_t copy_from_user_std(size_t, const void __user *, void *);
12extern size_t copy_to_user_std(size_t, void __user *, const void *); 12extern size_t copy_to_user_std(size_t, void __user *, const void *);
13extern size_t strnlen_user_std(size_t, const char __user *); 13extern size_t strnlen_user_std(size_t, const char __user *);
14extern size_t strncpy_from_user_std(size_t, const char __user *, char *); 14extern size_t strncpy_from_user_std(size_t, const char __user *, char *);
15extern int futex_atomic_cmpxchg_std(int __user *, int, int); 15extern int futex_atomic_cmpxchg_std(int *, int __user *, int, int);
16extern int futex_atomic_op_std(int, int __user *, int, int *); 16extern int futex_atomic_op_std(int, int __user *, int, int *);
17 17
18extern size_t copy_from_user_pt(size_t, const void __user *, void *); 18extern size_t copy_from_user_pt(size_t, const void __user *, void *);
19extern size_t copy_to_user_pt(size_t, void __user *, const void *); 19extern size_t copy_to_user_pt(size_t, void __user *, const void *);
20extern int futex_atomic_op_pt(int, int __user *, int, int *); 20extern int futex_atomic_op_pt(int, int __user *, int, int *);
21extern int futex_atomic_cmpxchg_pt(int __user *, int, int); 21extern int futex_atomic_cmpxchg_pt(int *, int __user *, int, int);
22 22
23#endif /* __ARCH_S390_LIB_UACCESS_H */ 23#endif /* __ARCH_S390_LIB_UACCESS_H */
diff --git a/arch/s390/lib/uaccess_pt.c b/arch/s390/lib/uaccess_pt.c
index 404f2de296dc..b3cebcd52f5c 100644
--- a/arch/s390/lib/uaccess_pt.c
+++ b/arch/s390/lib/uaccess_pt.c
@@ -354,26 +354,29 @@ int futex_atomic_op_pt(int op, int __user *uaddr, int oparg, int *old)
354 return ret; 354 return ret;
355} 355}
356 356
357static int __futex_atomic_cmpxchg_pt(int __user *uaddr, int oldval, int newval) 357static int __futex_atomic_cmpxchg_pt(int *uval, int __user *uaddr,
358 int oldval, int newval)
358{ 359{
359 int ret; 360 int ret;
360 361
361 asm volatile("0: cs %1,%4,0(%5)\n" 362 asm volatile("0: cs %1,%4,0(%5)\n"
362 "1: lr %0,%1\n" 363 "1: la %0,0\n"
363 "2:\n" 364 "2:\n"
364 EX_TABLE(0b,2b) EX_TABLE(1b,2b) 365 EX_TABLE(0b,2b) EX_TABLE(1b,2b)
365 : "=d" (ret), "+d" (oldval), "=m" (*uaddr) 366 : "=d" (ret), "+d" (oldval), "=m" (*uaddr)
366 : "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr) 367 : "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr)
367 : "cc", "memory" ); 368 : "cc", "memory" );
369 *uval = oldval;
368 return ret; 370 return ret;
369} 371}
370 372
371int futex_atomic_cmpxchg_pt(int __user *uaddr, int oldval, int newval) 373int futex_atomic_cmpxchg_pt(int *uval, int __user *uaddr,
374 int oldval, int newval)
372{ 375{
373 int ret; 376 int ret;
374 377
375 if (segment_eq(get_fs(), KERNEL_DS)) 378 if (segment_eq(get_fs(), KERNEL_DS))
376 return __futex_atomic_cmpxchg_pt(uaddr, oldval, newval); 379 return __futex_atomic_cmpxchg_pt(uval, uaddr, oldval, newval);
377 spin_lock(&current->mm->page_table_lock); 380 spin_lock(&current->mm->page_table_lock);
378 uaddr = (int __user *) __dat_user_addr((unsigned long) uaddr); 381 uaddr = (int __user *) __dat_user_addr((unsigned long) uaddr);
379 if (!uaddr) { 382 if (!uaddr) {
@@ -382,7 +385,7 @@ int futex_atomic_cmpxchg_pt(int __user *uaddr, int oldval, int newval)
382 } 385 }
383 get_page(virt_to_page(uaddr)); 386 get_page(virt_to_page(uaddr));
384 spin_unlock(&current->mm->page_table_lock); 387 spin_unlock(&current->mm->page_table_lock);
385 ret = __futex_atomic_cmpxchg_pt(uaddr, oldval, newval); 388 ret = __futex_atomic_cmpxchg_pt(uval, uaddr, oldval, newval);
386 put_page(virt_to_page(uaddr)); 389 put_page(virt_to_page(uaddr));
387 return ret; 390 return ret;
388} 391}
diff --git a/arch/s390/lib/uaccess_std.c b/arch/s390/lib/uaccess_std.c
index a6c4f7ed24a4..1d6643c0b95f 100644
--- a/arch/s390/lib/uaccess_std.c
+++ b/arch/s390/lib/uaccess_std.c
@@ -287,19 +287,21 @@ int futex_atomic_op_std(int op, int __user *uaddr, int oparg, int *old)
287 return ret; 287 return ret;
288} 288}
289 289
290int futex_atomic_cmpxchg_std(int __user *uaddr, int oldval, int newval) 290int futex_atomic_cmpxchg_std(int *uval, int __user *uaddr,
291 int oldval, int newval)
291{ 292{
292 int ret; 293 int ret;
293 294
294 asm volatile( 295 asm volatile(
295 " sacf 256\n" 296 " sacf 256\n"
296 "0: cs %1,%4,0(%5)\n" 297 "0: cs %1,%4,0(%5)\n"
297 "1: lr %0,%1\n" 298 "1: la %0,0\n"
298 "2: sacf 0\n" 299 "2: sacf 0\n"
299 EX_TABLE(0b,2b) EX_TABLE(1b,2b) 300 EX_TABLE(0b,2b) EX_TABLE(1b,2b)
300 : "=d" (ret), "+d" (oldval), "=m" (*uaddr) 301 : "=d" (ret), "+d" (oldval), "=m" (*uaddr)
301 : "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr) 302 : "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr)
302 : "cc", "memory" ); 303 : "cc", "memory" );
304 *uval = oldval;
303 return ret; 305 return ret;
304} 306}
305 307
diff --git a/arch/sh/include/asm/futex-irq.h b/arch/sh/include/asm/futex-irq.h
index a9f16a7f9aea..7b701cbd1e84 100644
--- a/arch/sh/include/asm/futex-irq.h
+++ b/arch/sh/include/asm/futex-irq.h
@@ -88,7 +88,8 @@ static inline int atomic_futex_op_xchg_xor(int oparg, int __user *uaddr,
88 return ret; 88 return ret;
89} 89}
90 90
91static inline int atomic_futex_op_cmpxchg_inatomic(int __user *uaddr, 91static inline int atomic_futex_op_cmpxchg_inatomic(int *uval,
92 int __user *uaddr,
92 int oldval, int newval) 93 int oldval, int newval)
93{ 94{
94 unsigned long flags; 95 unsigned long flags;
@@ -102,10 +103,8 @@ static inline int atomic_futex_op_cmpxchg_inatomic(int __user *uaddr,
102 103
103 local_irq_restore(flags); 104 local_irq_restore(flags);
104 105
105 if (ret) 106 *uval = prev;
106 return ret; 107 return ret;
107
108 return prev;
109} 108}
110 109
111#endif /* __ASM_SH_FUTEX_IRQ_H */ 110#endif /* __ASM_SH_FUTEX_IRQ_H */
diff --git a/arch/sh/include/asm/futex.h b/arch/sh/include/asm/futex.h
index 68256ec5fa35..a8a5125dc9b4 100644
--- a/arch/sh/include/asm/futex.h
+++ b/arch/sh/include/asm/futex.h
@@ -65,12 +65,13 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
65} 65}
66 66
67static inline int 67static inline int
68futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) 68futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
69 int oldval, int newval)
69{ 70{
70 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) 71 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
71 return -EFAULT; 72 return -EFAULT;
72 73
73 return atomic_futex_op_cmpxchg_inatomic(uaddr, oldval, newval); 74 return atomic_futex_op_cmpxchg_inatomic(uval, uaddr, oldval, newval);
74} 75}
75 76
76#endif /* __KERNEL__ */ 77#endif /* __KERNEL__ */
diff --git a/arch/sparc/include/asm/futex_64.h b/arch/sparc/include/asm/futex_64.h
index 47f95839dc69..e0862200d6a1 100644
--- a/arch/sparc/include/asm/futex_64.h
+++ b/arch/sparc/include/asm/futex_64.h
@@ -85,26 +85,30 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
85} 85}
86 86
87static inline int 87static inline int
88futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) 88futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
89 int oldval, int newval)
89{ 90{
91 int ret = 0;
92
90 __asm__ __volatile__( 93 __asm__ __volatile__(
91 "\n1: casa [%3] %%asi, %2, %0\n" 94 "\n1: casa [%4] %%asi, %3, %1\n"
92 "2:\n" 95 "2:\n"
93 " .section .fixup,#alloc,#execinstr\n" 96 " .section .fixup,#alloc,#execinstr\n"
94 " .align 4\n" 97 " .align 4\n"
95 "3: sethi %%hi(2b), %0\n" 98 "3: sethi %%hi(2b), %0\n"
96 " jmpl %0 + %%lo(2b), %%g0\n" 99 " jmpl %0 + %%lo(2b), %%g0\n"
97 " mov %4, %0\n" 100 " mov %5, %0\n"
98 " .previous\n" 101 " .previous\n"
99 " .section __ex_table,\"a\"\n" 102 " .section __ex_table,\"a\"\n"
100 " .align 4\n" 103 " .align 4\n"
101 " .word 1b, 3b\n" 104 " .word 1b, 3b\n"
102 " .previous\n" 105 " .previous\n"
103 : "=r" (newval) 106 : "+r" (ret), "=r" (newval)
104 : "0" (newval), "r" (oldval), "r" (uaddr), "i" (-EFAULT) 107 : "1" (newval), "r" (oldval), "r" (uaddr), "i" (-EFAULT)
105 : "memory"); 108 : "memory");
106 109
107 return newval; 110 *uval = newval;
111 return ret;
108} 112}
109 113
110#endif /* !(_SPARC64_FUTEX_H) */ 114#endif /* !(_SPARC64_FUTEX_H) */
diff --git a/arch/tile/include/asm/futex.h b/arch/tile/include/asm/futex.h
index fe0d10dcae57..664b20aa2584 100644
--- a/arch/tile/include/asm/futex.h
+++ b/arch/tile/include/asm/futex.h
@@ -119,8 +119,8 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
119 return ret; 119 return ret;
120} 120}
121 121
122static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, 122static inline int futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
123 int newval) 123 int oldval, int newval)
124{ 124{
125 struct __get_user asm_ret; 125 struct __get_user asm_ret;
126 126
@@ -128,7 +128,8 @@ static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval,
128 return -EFAULT; 128 return -EFAULT;
129 129
130 asm_ret = futex_cmpxchg(uaddr, oldval, newval); 130 asm_ret = futex_cmpxchg(uaddr, oldval, newval);
131 return asm_ret.err ? asm_ret.err : asm_ret.val; 131 *uval = asm_ret.val;
132 return asm_ret.err;
132} 133}
133 134
134#ifndef __tilegx__ 135#ifndef __tilegx__
diff --git a/arch/x86/include/asm/futex.h b/arch/x86/include/asm/futex.h
index 1f11ce44e956..884c0b5676f4 100644
--- a/arch/x86/include/asm/futex.h
+++ b/arch/x86/include/asm/futex.h
@@ -109,9 +109,10 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
109 return ret; 109 return ret;
110} 110}
111 111
112static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, 112static inline int futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
113 int newval) 113 int oldval, int newval)
114{ 114{
115 int ret = 0;
115 116
116#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_BSWAP) 117#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_BSWAP)
117 /* Real i386 machines have no cmpxchg instruction */ 118 /* Real i386 machines have no cmpxchg instruction */
@@ -122,18 +123,19 @@ static inline int futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval,
122 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) 123 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
123 return -EFAULT; 124 return -EFAULT;
124 125
125 asm volatile("1:\t" LOCK_PREFIX "cmpxchgl %3, %1\n" 126 asm volatile("1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n"
126 "2:\t.section .fixup, \"ax\"\n" 127 "2:\t.section .fixup, \"ax\"\n"
127 "3:\tmov %2, %0\n" 128 "3:\tmov %3, %0\n"
128 "\tjmp 2b\n" 129 "\tjmp 2b\n"
129 "\t.previous\n" 130 "\t.previous\n"
130 _ASM_EXTABLE(1b, 3b) 131 _ASM_EXTABLE(1b, 3b)
131 : "=a" (oldval), "+m" (*uaddr) 132 : "+r" (ret), "=a" (oldval), "+m" (*uaddr)
132 : "i" (-EFAULT), "r" (newval), "0" (oldval) 133 : "i" (-EFAULT), "r" (newval), "1" (oldval)
133 : "memory" 134 : "memory"
134 ); 135 );
135 136
136 return oldval; 137 *uval = oldval;
138 return ret;
137} 139}
138 140
139#endif 141#endif
diff --git a/include/asm-generic/futex.h b/include/asm-generic/futex.h
index 3c2344f48136..132bf5227b44 100644
--- a/include/asm-generic/futex.h
+++ b/include/asm-generic/futex.h
@@ -48,7 +48,8 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
48} 48}
49 49
50static inline int 50static inline int
51futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) 51futex_atomic_cmpxchg_inatomic(int *uval, int __user *uaddr,
52 int oldval, int newval)
52{ 53{
53 return -ENOSYS; 54 return -ENOSYS;
54} 55}
diff --git a/kernel/futex.c b/kernel/futex.c
index 773815465bac..237f14bfc022 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -381,15 +381,16 @@ static struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb,
381 return NULL; 381 return NULL;
382} 382}
383 383
384static u32 cmpxchg_futex_value_locked(u32 __user *uaddr, u32 uval, u32 newval) 384static int cmpxchg_futex_value_locked(u32 *curval, u32 __user *uaddr,
385 u32 uval, u32 newval)
385{ 386{
386 u32 curval; 387 int ret;
387 388
388 pagefault_disable(); 389 pagefault_disable();
389 curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval); 390 ret = futex_atomic_cmpxchg_inatomic(curval, uaddr, uval, newval);
390 pagefault_enable(); 391 pagefault_enable();
391 392
392 return curval; 393 return ret;
393} 394}
394 395
395static int get_futex_value_locked(u32 *dest, u32 __user *from) 396static int get_futex_value_locked(u32 *dest, u32 __user *from)
@@ -688,9 +689,7 @@ retry:
688 if (set_waiters) 689 if (set_waiters)
689 newval |= FUTEX_WAITERS; 690 newval |= FUTEX_WAITERS;
690 691
691 curval = cmpxchg_futex_value_locked(uaddr, 0, newval); 692 if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, 0, newval)))
692
693 if (unlikely(curval == -EFAULT))
694 return -EFAULT; 693 return -EFAULT;
695 694
696 /* 695 /*
@@ -728,9 +727,7 @@ retry:
728 lock_taken = 1; 727 lock_taken = 1;
729 } 728 }
730 729
731 curval = cmpxchg_futex_value_locked(uaddr, uval, newval); 730 if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)))
732
733 if (unlikely(curval == -EFAULT))
734 return -EFAULT; 731 return -EFAULT;
735 if (unlikely(curval != uval)) 732 if (unlikely(curval != uval))
736 goto retry; 733 goto retry;
@@ -843,9 +840,7 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
843 840
844 newval = FUTEX_WAITERS | task_pid_vnr(new_owner); 841 newval = FUTEX_WAITERS | task_pid_vnr(new_owner);
845 842
846 curval = cmpxchg_futex_value_locked(uaddr, uval, newval); 843 if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval))
847
848 if (curval == -EFAULT)
849 ret = -EFAULT; 844 ret = -EFAULT;
850 else if (curval != uval) 845 else if (curval != uval)
851 ret = -EINVAL; 846 ret = -EINVAL;
@@ -880,10 +875,8 @@ static int unlock_futex_pi(u32 __user *uaddr, u32 uval)
880 * There is no waiter, so we unlock the futex. The owner died 875 * There is no waiter, so we unlock the futex. The owner died
881 * bit has not to be preserved here. We are the owner: 876 * bit has not to be preserved here. We are the owner:
882 */ 877 */
883 oldval = cmpxchg_futex_value_locked(uaddr, uval, 0); 878 if (cmpxchg_futex_value_locked(&oldval, uaddr, uval, 0))
884 879 return -EFAULT;
885 if (oldval == -EFAULT)
886 return oldval;
887 if (oldval != uval) 880 if (oldval != uval)
888 return -EAGAIN; 881 return -EAGAIN;
889 882
@@ -1578,9 +1571,7 @@ retry:
1578 while (1) { 1571 while (1) {
1579 newval = (uval & FUTEX_OWNER_DIED) | newtid; 1572 newval = (uval & FUTEX_OWNER_DIED) | newtid;
1580 1573
1581 curval = cmpxchg_futex_value_locked(uaddr, uval, newval); 1574 if (cmpxchg_futex_value_locked(&curval, uaddr, uval, newval))
1582
1583 if (curval == -EFAULT)
1584 goto handle_fault; 1575 goto handle_fault;
1585 if (curval == uval) 1576 if (curval == uval)
1586 break; 1577 break;
@@ -2073,11 +2064,8 @@ retry:
2073 * again. If it succeeds then we can return without waking 2064 * again. If it succeeds then we can return without waking
2074 * anyone else up: 2065 * anyone else up:
2075 */ 2066 */
2076 if (!(uval & FUTEX_OWNER_DIED)) 2067 if (!(uval & FUTEX_OWNER_DIED) &&
2077 uval = cmpxchg_futex_value_locked(uaddr, vpid, 0); 2068 cmpxchg_futex_value_locked(&uval, uaddr, vpid, 0))
2078
2079
2080 if (unlikely(uval == -EFAULT))
2081 goto pi_faulted; 2069 goto pi_faulted;
2082 /* 2070 /*
2083 * Rare case: we managed to release the lock atomically, 2071 * Rare case: we managed to release the lock atomically,
@@ -2464,9 +2452,7 @@ retry:
2464 * userspace. 2452 * userspace.
2465 */ 2453 */
2466 mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED; 2454 mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
2467 nval = futex_atomic_cmpxchg_inatomic(uaddr, uval, mval); 2455 if (futex_atomic_cmpxchg_inatomic(&nval, uaddr, uval, mval))
2468
2469 if (nval == -EFAULT)
2470 return -1; 2456 return -1;
2471 2457
2472 if (nval != uval) 2458 if (nval != uval)
@@ -2679,8 +2665,7 @@ static int __init futex_init(void)
2679 * implementation, the non-functional ones will return 2665 * implementation, the non-functional ones will return
2680 * -ENOSYS. 2666 * -ENOSYS.
2681 */ 2667 */
2682 curval = cmpxchg_futex_value_locked(NULL, 0, 0); 2668 if (cmpxchg_futex_value_locked(&curval, NULL, 0, 0) == -EFAULT)
2683 if (curval == -EFAULT)
2684 futex_cmpxchg_enabled = 1; 2669 futex_cmpxchg_enabled = 1;
2685 2670
2686 for (i = 0; i < ARRAY_SIZE(futex_queues); i++) { 2671 for (i = 0; i < ARRAY_SIZE(futex_queues); i++) {
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-24 15:12:50 -0500