aboutsummaryrefslogtreecommitdiffstats
path: root/arch/s390/lib/uaccess.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/s390/lib/uaccess.c')
-rw-r--r--arch/s390/lib/uaccess.c136
1 files changed, 61 insertions, 75 deletions
diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c
index 53dd5d7a0c96..4614d415bb58 100644
--- a/arch/s390/lib/uaccess.c
+++ b/arch/s390/lib/uaccess.c
@@ -15,20 +15,6 @@
15#include <asm/mmu_context.h> 15#include <asm/mmu_context.h>
16#include <asm/facility.h> 16#include <asm/facility.h>
17 17
18#ifndef CONFIG_64BIT
19#define AHI "ahi"
20#define ALR "alr"
21#define CLR "clr"
22#define LHI "lhi"
23#define SLR "slr"
24#else
25#define AHI "aghi"
26#define ALR "algr"
27#define CLR "clgr"
28#define LHI "lghi"
29#define SLR "slgr"
30#endif
31
32static struct static_key have_mvcos = STATIC_KEY_INIT_FALSE; 18static struct static_key have_mvcos = STATIC_KEY_INIT_FALSE;
33 19
34static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr, 20static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr,
@@ -41,29 +27,29 @@ static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr
41 asm volatile( 27 asm volatile(
42 "0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n" 28 "0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n"
43 "9: jz 7f\n" 29 "9: jz 7f\n"
44 "1:"ALR" %0,%3\n" 30 "1: algr %0,%3\n"
45 " "SLR" %1,%3\n" 31 " slgr %1,%3\n"
46 " "SLR" %2,%3\n" 32 " slgr %2,%3\n"
47 " j 0b\n" 33 " j 0b\n"
48 "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */ 34 "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */
49 " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */ 35 " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */
50 " "SLR" %4,%1\n" 36 " slgr %4,%1\n"
51 " "CLR" %0,%4\n" /* copy crosses next page boundary? */ 37 " clgr %0,%4\n" /* copy crosses next page boundary? */
52 " jnh 4f\n" 38 " jnh 4f\n"
53 "3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n" 39 "3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n"
54 "10:"SLR" %0,%4\n" 40 "10:slgr %0,%4\n"
55 " "ALR" %2,%4\n" 41 " algr %2,%4\n"
56 "4:"LHI" %4,-1\n" 42 "4: lghi %4,-1\n"
57 " "ALR" %4,%0\n" /* copy remaining size, subtract 1 */ 43 " algr %4,%0\n" /* copy remaining size, subtract 1 */
58 " bras %3,6f\n" /* memset loop */ 44 " bras %3,6f\n" /* memset loop */
59 " xc 0(1,%2),0(%2)\n" 45 " xc 0(1,%2),0(%2)\n"
60 "5: xc 0(256,%2),0(%2)\n" 46 "5: xc 0(256,%2),0(%2)\n"
61 " la %2,256(%2)\n" 47 " la %2,256(%2)\n"
62 "6:"AHI" %4,-256\n" 48 "6: aghi %4,-256\n"
63 " jnm 5b\n" 49 " jnm 5b\n"
64 " ex %4,0(%3)\n" 50 " ex %4,0(%3)\n"
65 " j 8f\n" 51 " j 8f\n"
66 "7:"SLR" %0,%0\n" 52 "7:slgr %0,%0\n"
67 "8:\n" 53 "8:\n"
68 EX_TABLE(0b,2b) EX_TABLE(3b,4b) EX_TABLE(9b,2b) EX_TABLE(10b,4b) 54 EX_TABLE(0b,2b) EX_TABLE(3b,4b) EX_TABLE(9b,2b) EX_TABLE(10b,4b)
69 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) 55 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
@@ -82,32 +68,32 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr,
82 " sacf 0\n" 68 " sacf 0\n"
83 "0: mvcp 0(%0,%2),0(%1),%3\n" 69 "0: mvcp 0(%0,%2),0(%1),%3\n"
84 "10:jz 8f\n" 70 "10:jz 8f\n"
85 "1:"ALR" %0,%3\n" 71 "1: algr %0,%3\n"
86 " la %1,256(%1)\n" 72 " la %1,256(%1)\n"
87 " la %2,256(%2)\n" 73 " la %2,256(%2)\n"
88 "2: mvcp 0(%0,%2),0(%1),%3\n" 74 "2: mvcp 0(%0,%2),0(%1),%3\n"
89 "11:jnz 1b\n" 75 "11:jnz 1b\n"
90 " j 8f\n" 76 " j 8f\n"
91 "3: la %4,255(%1)\n" /* %4 = ptr + 255 */ 77 "3: la %4,255(%1)\n" /* %4 = ptr + 255 */
92 " "LHI" %3,-4096\n" 78 " lghi %3,-4096\n"
93 " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */ 79 " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */
94 " "SLR" %4,%1\n" 80 " slgr %4,%1\n"
95 " "CLR" %0,%4\n" /* copy crosses next page boundary? */ 81 " clgr %0,%4\n" /* copy crosses next page boundary? */
96 " jnh 5f\n" 82 " jnh 5f\n"
97 "4: mvcp 0(%4,%2),0(%1),%3\n" 83 "4: mvcp 0(%4,%2),0(%1),%3\n"
98 "12:"SLR" %0,%4\n" 84 "12:slgr %0,%4\n"
99 " "ALR" %2,%4\n" 85 " algr %2,%4\n"
100 "5:"LHI" %4,-1\n" 86 "5: lghi %4,-1\n"
101 " "ALR" %4,%0\n" /* copy remaining size, subtract 1 */ 87 " algr %4,%0\n" /* copy remaining size, subtract 1 */
102 " bras %3,7f\n" /* memset loop */ 88 " bras %3,7f\n" /* memset loop */
103 " xc 0(1,%2),0(%2)\n" 89 " xc 0(1,%2),0(%2)\n"
104 "6: xc 0(256,%2),0(%2)\n" 90 "6: xc 0(256,%2),0(%2)\n"
105 " la %2,256(%2)\n" 91 " la %2,256(%2)\n"
106 "7:"AHI" %4,-256\n" 92 "7: aghi %4,-256\n"
107 " jnm 6b\n" 93 " jnm 6b\n"
108 " ex %4,0(%3)\n" 94 " ex %4,0(%3)\n"
109 " j 9f\n" 95 " j 9f\n"
110 "8:"SLR" %0,%0\n" 96 "8:slgr %0,%0\n"
111 "9: sacf 768\n" 97 "9: sacf 768\n"
112 EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,5b) 98 EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,5b)
113 EX_TABLE(10b,3b) EX_TABLE(11b,3b) EX_TABLE(12b,5b) 99 EX_TABLE(10b,3b) EX_TABLE(11b,3b) EX_TABLE(12b,5b)
@@ -134,19 +120,19 @@ static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
134 asm volatile( 120 asm volatile(
135 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n" 121 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
136 "6: jz 4f\n" 122 "6: jz 4f\n"
137 "1:"ALR" %0,%3\n" 123 "1: algr %0,%3\n"
138 " "SLR" %1,%3\n" 124 " slgr %1,%3\n"
139 " "SLR" %2,%3\n" 125 " slgr %2,%3\n"
140 " j 0b\n" 126 " j 0b\n"
141 "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */ 127 "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */
142 " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */ 128 " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */
143 " "SLR" %4,%1\n" 129 " slgr %4,%1\n"
144 " "CLR" %0,%4\n" /* copy crosses next page boundary? */ 130 " clgr %0,%4\n" /* copy crosses next page boundary? */
145 " jnh 5f\n" 131 " jnh 5f\n"
146 "3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n" 132 "3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n"
147 "7:"SLR" %0,%4\n" 133 "7: slgr %0,%4\n"
148 " j 5f\n" 134 " j 5f\n"
149 "4:"SLR" %0,%0\n" 135 "4: slgr %0,%0\n"
150 "5:\n" 136 "5:\n"
151 EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b) 137 EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
152 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) 138 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
@@ -165,22 +151,22 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x,
165 " sacf 0\n" 151 " sacf 0\n"
166 "0: mvcs 0(%0,%1),0(%2),%3\n" 152 "0: mvcs 0(%0,%1),0(%2),%3\n"
167 "7: jz 5f\n" 153 "7: jz 5f\n"
168 "1:"ALR" %0,%3\n" 154 "1: algr %0,%3\n"
169 " la %1,256(%1)\n" 155 " la %1,256(%1)\n"
170 " la %2,256(%2)\n" 156 " la %2,256(%2)\n"
171 "2: mvcs 0(%0,%1),0(%2),%3\n" 157 "2: mvcs 0(%0,%1),0(%2),%3\n"
172 "8: jnz 1b\n" 158 "8: jnz 1b\n"
173 " j 5f\n" 159 " j 5f\n"
174 "3: la %4,255(%1)\n" /* %4 = ptr + 255 */ 160 "3: la %4,255(%1)\n" /* %4 = ptr + 255 */
175 " "LHI" %3,-4096\n" 161 " lghi %3,-4096\n"
176 " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */ 162 " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */
177 " "SLR" %4,%1\n" 163 " slgr %4,%1\n"
178 " "CLR" %0,%4\n" /* copy crosses next page boundary? */ 164 " clgr %0,%4\n" /* copy crosses next page boundary? */
179 " jnh 6f\n" 165 " jnh 6f\n"
180 "4: mvcs 0(%4,%1),0(%2),%3\n" 166 "4: mvcs 0(%4,%1),0(%2),%3\n"
181 "9:"SLR" %0,%4\n" 167 "9: slgr %0,%4\n"
182 " j 6f\n" 168 " j 6f\n"
183 "5:"SLR" %0,%0\n" 169 "5: slgr %0,%0\n"
184 "6: sacf 768\n" 170 "6: sacf 768\n"
185 EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b) 171 EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
186 EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b) 172 EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
@@ -208,11 +194,11 @@ static inline unsigned long copy_in_user_mvcos(void __user *to, const void __use
208 asm volatile( 194 asm volatile(
209 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n" 195 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
210 " jz 2f\n" 196 " jz 2f\n"
211 "1:"ALR" %0,%3\n" 197 "1: algr %0,%3\n"
212 " "SLR" %1,%3\n" 198 " slgr %1,%3\n"
213 " "SLR" %2,%3\n" 199 " slgr %2,%3\n"
214 " j 0b\n" 200 " j 0b\n"
215 "2:"SLR" %0,%0\n" 201 "2:slgr %0,%0\n"
216 "3: \n" 202 "3: \n"
217 EX_TABLE(0b,3b) 203 EX_TABLE(0b,3b)
218 : "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2) 204 : "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2)
@@ -228,23 +214,23 @@ static inline unsigned long copy_in_user_mvc(void __user *to, const void __user
228 load_kernel_asce(); 214 load_kernel_asce();
229 asm volatile( 215 asm volatile(
230 " sacf 256\n" 216 " sacf 256\n"
231 " "AHI" %0,-1\n" 217 " aghi %0,-1\n"
232 " jo 5f\n" 218 " jo 5f\n"
233 " bras %3,3f\n" 219 " bras %3,3f\n"
234 "0:"AHI" %0,257\n" 220 "0: aghi %0,257\n"
235 "1: mvc 0(1,%1),0(%2)\n" 221 "1: mvc 0(1,%1),0(%2)\n"
236 " la %1,1(%1)\n" 222 " la %1,1(%1)\n"
237 " la %2,1(%2)\n" 223 " la %2,1(%2)\n"
238 " "AHI" %0,-1\n" 224 " aghi %0,-1\n"
239 " jnz 1b\n" 225 " jnz 1b\n"
240 " j 5f\n" 226 " j 5f\n"
241 "2: mvc 0(256,%1),0(%2)\n" 227 "2: mvc 0(256,%1),0(%2)\n"
242 " la %1,256(%1)\n" 228 " la %1,256(%1)\n"
243 " la %2,256(%2)\n" 229 " la %2,256(%2)\n"
244 "3:"AHI" %0,-256\n" 230 "3: aghi %0,-256\n"
245 " jnm 2b\n" 231 " jnm 2b\n"
246 "4: ex %0,1b-0b(%3)\n" 232 "4: ex %0,1b-0b(%3)\n"
247 "5: "SLR" %0,%0\n" 233 "5: slgr %0,%0\n"
248 "6: sacf 768\n" 234 "6: sacf 768\n"
249 EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) 235 EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
250 : "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1) 236 : "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1)
@@ -269,18 +255,18 @@ static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size
269 asm volatile( 255 asm volatile(
270 "0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n" 256 "0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n"
271 " jz 4f\n" 257 " jz 4f\n"
272 "1:"ALR" %0,%2\n" 258 "1: algr %0,%2\n"
273 " "SLR" %1,%2\n" 259 " slgr %1,%2\n"
274 " j 0b\n" 260 " j 0b\n"
275 "2: la %3,4095(%1)\n"/* %4 = to + 4095 */ 261 "2: la %3,4095(%1)\n"/* %4 = to + 4095 */
276 " nr %3,%2\n" /* %4 = (to + 4095) & -4096 */ 262 " nr %3,%2\n" /* %4 = (to + 4095) & -4096 */
277 " "SLR" %3,%1\n" 263 " slgr %3,%1\n"
278 " "CLR" %0,%3\n" /* copy crosses next page boundary? */ 264 " clgr %0,%3\n" /* copy crosses next page boundary? */
279 " jnh 5f\n" 265 " jnh 5f\n"
280 "3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n" 266 "3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n"
281 " "SLR" %0,%3\n" 267 " slgr %0,%3\n"
282 " j 5f\n" 268 " j 5f\n"
283 "4:"SLR" %0,%0\n" 269 "4:slgr %0,%0\n"
284 "5:\n" 270 "5:\n"
285 EX_TABLE(0b,2b) EX_TABLE(3b,5b) 271 EX_TABLE(0b,2b) EX_TABLE(3b,5b)
286 : "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2) 272 : "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2)
@@ -295,28 +281,28 @@ static inline unsigned long clear_user_xc(void __user *to, unsigned long size)
295 load_kernel_asce(); 281 load_kernel_asce();
296 asm volatile( 282 asm volatile(
297 " sacf 256\n" 283 " sacf 256\n"
298 " "AHI" %0,-1\n" 284 " aghi %0,-1\n"
299 " jo 5f\n" 285 " jo 5f\n"
300 " bras %3,3f\n" 286 " bras %3,3f\n"
301 " xc 0(1,%1),0(%1)\n" 287 " xc 0(1,%1),0(%1)\n"
302 "0:"AHI" %0,257\n" 288 "0: aghi %0,257\n"
303 " la %2,255(%1)\n" /* %2 = ptr + 255 */ 289 " la %2,255(%1)\n" /* %2 = ptr + 255 */
304 " srl %2,12\n" 290 " srl %2,12\n"
305 " sll %2,12\n" /* %2 = (ptr + 255) & -4096 */ 291 " sll %2,12\n" /* %2 = (ptr + 255) & -4096 */
306 " "SLR" %2,%1\n" 292 " slgr %2,%1\n"
307 " "CLR" %0,%2\n" /* clear crosses next page boundary? */ 293 " clgr %0,%2\n" /* clear crosses next page boundary? */
308 " jnh 5f\n" 294 " jnh 5f\n"
309 " "AHI" %2,-1\n" 295 " aghi %2,-1\n"
310 "1: ex %2,0(%3)\n" 296 "1: ex %2,0(%3)\n"
311 " "AHI" %2,1\n" 297 " aghi %2,1\n"
312 " "SLR" %0,%2\n" 298 " slgr %0,%2\n"
313 " j 5f\n" 299 " j 5f\n"
314 "2: xc 0(256,%1),0(%1)\n" 300 "2: xc 0(256,%1),0(%1)\n"
315 " la %1,256(%1)\n" 301 " la %1,256(%1)\n"
316 "3:"AHI" %0,-256\n" 302 "3: aghi %0,-256\n"
317 " jnm 2b\n" 303 " jnm 2b\n"
318 "4: ex %0,0(%3)\n" 304 "4: ex %0,0(%3)\n"
319 "5: "SLR" %0,%0\n" 305 "5: slgr %0,%0\n"
320 "6: sacf 768\n" 306 "6: sacf 768\n"
321 EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) 307 EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
322 : "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2) 308 : "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2)
@@ -341,12 +327,12 @@ static inline unsigned long strnlen_user_srst(const char __user *src,
341 asm volatile( 327 asm volatile(
342 " la %2,0(%1)\n" 328 " la %2,0(%1)\n"
343 " la %3,0(%0,%1)\n" 329 " la %3,0(%0,%1)\n"
344 " "SLR" %0,%0\n" 330 " slgr %0,%0\n"
345 " sacf 256\n" 331 " sacf 256\n"
346 "0: srst %3,%2\n" 332 "0: srst %3,%2\n"
347 " jo 0b\n" 333 " jo 0b\n"
348 " la %0,1(%3)\n" /* strnlen_user results includes \0 */ 334 " la %0,1(%3)\n" /* strnlen_user results includes \0 */
349 " "SLR" %0,%1\n" 335 " slgr %0,%1\n"
350 "1: sacf 768\n" 336 "1: sacf 768\n"
351 EX_TABLE(0b,1b) 337 EX_TABLE(0b,1b)
352 : "+a" (size), "+a" (src), "=a" (tmp1), "=a" (tmp2) 338 : "+a" (size), "+a" (src), "=a" (tmp1), "=a" (tmp2)
@@ -399,7 +385,7 @@ early_param("uaccess_primary", parse_uaccess_pt);
399 385
400static int __init uaccess_init(void) 386static int __init uaccess_init(void)
401{ 387{
402 if (IS_ENABLED(CONFIG_64BIT) && !uaccess_primary && test_facility(27)) 388 if (!uaccess_primary && test_facility(27))
403 static_key_slow_inc(&have_mvcos); 389 static_key_slow_inc(&have_mvcos);
404 return 0; 390 return 0;
405} 391}
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-27 23:21:26 -0500