aboutsummaryrefslogtreecommitdiffstats
path: root/include/asm-s390/system.h
diff options
context:
space:
mode:
authorMartin Schwidefsky <schwidefsky@de.ibm.com>2006-09-28 10:56:43 -0400
committerMartin Schwidefsky <schwidefsky@de.ibm.com>2006-09-28 10:56:43 -0400
commit94c12cc7d196bab34aaa98d38521549fa1e5ef76 (patch)
tree8e0cec0ed44445d74a2cb5160303d6b4dfb1bc31 /include/asm-s390/system.h
parent25d83cbfaa44e1b9170c0941c3ef52ca39f54ccc (diff)
[S390] Inline assembly cleanup.
Major cleanup of all s390 inline assemblies. They now have a common coding style. Quite a few have been shortened, mainly by using register asm variables. Use of the EX_TABLE macro helps as well. The atomic ops, bit ops and locking inlines new use the Q-constraint if a newer gcc is used. That results in slightly better code. Thanks to Christian Borntraeger for proof reading the changes. Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Diffstat (limited to 'include/asm-s390/system.h')
-rw-r--r--include/asm-s390/system.h342
1 files changed, 137 insertions, 205 deletions
diff --git a/include/asm-s390/system.h b/include/asm-s390/system.h
index 16040048cd1b..ccbafe4bf2cb 100644
--- a/include/asm-s390/system.h
+++ b/include/asm-s390/system.h
@@ -23,74 +23,68 @@ struct task_struct;
23 23
24extern struct task_struct *__switch_to(void *, void *); 24extern struct task_struct *__switch_to(void *, void *);
25 25
26#ifdef __s390x__
27#define __FLAG_SHIFT 56
28#else /* ! __s390x__ */
29#define __FLAG_SHIFT 24
30#endif /* ! __s390x__ */
31
32static inline void save_fp_regs(s390_fp_regs *fpregs) 26static inline void save_fp_regs(s390_fp_regs *fpregs)
33{ 27{
34 asm volatile ( 28 asm volatile(
35 " std 0,8(%1)\n" 29 " std 0,8(%1)\n"
36 " std 2,24(%1)\n" 30 " std 2,24(%1)\n"
37 " std 4,40(%1)\n" 31 " std 4,40(%1)\n"
38 " std 6,56(%1)" 32 " std 6,56(%1)"
39 : "=m" (*fpregs) : "a" (fpregs), "m" (*fpregs) : "memory" ); 33 : "=m" (*fpregs) : "a" (fpregs), "m" (*fpregs) : "memory");
40 if (!MACHINE_HAS_IEEE) 34 if (!MACHINE_HAS_IEEE)
41 return; 35 return;
42 asm volatile( 36 asm volatile(
43 " stfpc 0(%1)\n" 37 " stfpc 0(%1)\n"
44 " std 1,16(%1)\n" 38 " std 1,16(%1)\n"
45 " std 3,32(%1)\n" 39 " std 3,32(%1)\n"
46 " std 5,48(%1)\n" 40 " std 5,48(%1)\n"
47 " std 7,64(%1)\n" 41 " std 7,64(%1)\n"
48 " std 8,72(%1)\n" 42 " std 8,72(%1)\n"
49 " std 9,80(%1)\n" 43 " std 9,80(%1)\n"
50 " std 10,88(%1)\n" 44 " std 10,88(%1)\n"
51 " std 11,96(%1)\n" 45 " std 11,96(%1)\n"
52 " std 12,104(%1)\n" 46 " std 12,104(%1)\n"
53 " std 13,112(%1)\n" 47 " std 13,112(%1)\n"
54 " std 14,120(%1)\n" 48 " std 14,120(%1)\n"
55 " std 15,128(%1)\n" 49 " std 15,128(%1)\n"
56 : "=m" (*fpregs) : "a" (fpregs), "m" (*fpregs) : "memory" ); 50 : "=m" (*fpregs) : "a" (fpregs), "m" (*fpregs) : "memory");
57} 51}
58 52
59static inline void restore_fp_regs(s390_fp_regs *fpregs) 53static inline void restore_fp_regs(s390_fp_regs *fpregs)
60{ 54{
61 asm volatile ( 55 asm volatile(
62 " ld 0,8(%0)\n" 56 " ld 0,8(%0)\n"
63 " ld 2,24(%0)\n" 57 " ld 2,24(%0)\n"
64 " ld 4,40(%0)\n" 58 " ld 4,40(%0)\n"
65 " ld 6,56(%0)" 59 " ld 6,56(%0)"
66 : : "a" (fpregs), "m" (*fpregs) ); 60 : : "a" (fpregs), "m" (*fpregs));
67 if (!MACHINE_HAS_IEEE) 61 if (!MACHINE_HAS_IEEE)
68 return; 62 return;
69 asm volatile( 63 asm volatile(
70 " lfpc 0(%0)\n" 64 " lfpc 0(%0)\n"
71 " ld 1,16(%0)\n" 65 " ld 1,16(%0)\n"
72 " ld 3,32(%0)\n" 66 " ld 3,32(%0)\n"
73 " ld 5,48(%0)\n" 67 " ld 5,48(%0)\n"
74 " ld 7,64(%0)\n" 68 " ld 7,64(%0)\n"
75 " ld 8,72(%0)\n" 69 " ld 8,72(%0)\n"
76 " ld 9,80(%0)\n" 70 " ld 9,80(%0)\n"
77 " ld 10,88(%0)\n" 71 " ld 10,88(%0)\n"
78 " ld 11,96(%0)\n" 72 " ld 11,96(%0)\n"
79 " ld 12,104(%0)\n" 73 " ld 12,104(%0)\n"
80 " ld 13,112(%0)\n" 74 " ld 13,112(%0)\n"
81 " ld 14,120(%0)\n" 75 " ld 14,120(%0)\n"
82 " ld 15,128(%0)\n" 76 " ld 15,128(%0)\n"
83 : : "a" (fpregs), "m" (*fpregs) ); 77 : : "a" (fpregs), "m" (*fpregs));
84} 78}
85 79
86static inline void save_access_regs(unsigned int *acrs) 80static inline void save_access_regs(unsigned int *acrs)
87{ 81{
88 asm volatile ("stam 0,15,0(%0)" : : "a" (acrs) : "memory" ); 82 asm volatile("stam 0,15,0(%0)" : : "a" (acrs) : "memory");
89} 83}
90 84
91static inline void restore_access_regs(unsigned int *acrs) 85static inline void restore_access_regs(unsigned int *acrs)
92{ 86{
93 asm volatile ("lam 0,15,0(%0)" : : "a" (acrs) ); 87 asm volatile("lam 0,15,0(%0)" : : "a" (acrs));
94} 88}
95 89
96#define switch_to(prev,next,last) do { \ 90#define switch_to(prev,next,last) do { \
@@ -126,7 +120,7 @@ extern void account_system_vtime(struct task_struct *);
126 account_vtime(prev); \ 120 account_vtime(prev); \
127} while (0) 121} while (0)
128 122
129#define nop() __asm__ __volatile__ ("nop") 123#define nop() asm volatile("nop")
130 124
131#define xchg(ptr,x) \ 125#define xchg(ptr,x) \
132({ \ 126({ \
@@ -147,15 +141,15 @@ static inline unsigned long __xchg(unsigned long x, void * ptr, int size)
147 shift = (3 ^ (addr & 3)) << 3; 141 shift = (3 ^ (addr & 3)) << 3;
148 addr ^= addr & 3; 142 addr ^= addr & 3;
149 asm volatile( 143 asm volatile(
150 " l %0,0(%4)\n" 144 " l %0,0(%4)\n"
151 "0: lr 0,%0\n" 145 "0: lr 0,%0\n"
152 " nr 0,%3\n" 146 " nr 0,%3\n"
153 " or 0,%2\n" 147 " or 0,%2\n"
154 " cs %0,0,0(%4)\n" 148 " cs %0,0,0(%4)\n"
155 " jl 0b\n" 149 " jl 0b\n"
156 : "=&d" (old), "=m" (*(int *) addr) 150 : "=&d" (old), "=m" (*(int *) addr)
157 : "d" (x << shift), "d" (~(255 << shift)), "a" (addr), 151 : "d" (x << shift), "d" (~(255 << shift)), "a" (addr),
158 "m" (*(int *) addr) : "memory", "cc", "0" ); 152 "m" (*(int *) addr) : "memory", "cc", "0");
159 x = old >> shift; 153 x = old >> shift;
160 break; 154 break;
161 case 2: 155 case 2:
@@ -163,36 +157,36 @@ static inline unsigned long __xchg(unsigned long x, void * ptr, int size)
163 shift = (2 ^ (addr & 2)) << 3; 157 shift = (2 ^ (addr & 2)) << 3;
164 addr ^= addr & 2; 158 addr ^= addr & 2;
165 asm volatile( 159 asm volatile(
166 " l %0,0(%4)\n" 160 " l %0,0(%4)\n"
167 "0: lr 0,%0\n" 161 "0: lr 0,%0\n"
168 " nr 0,%3\n" 162 " nr 0,%3\n"
169 " or 0,%2\n" 163 " or 0,%2\n"
170 " cs %0,0,0(%4)\n" 164 " cs %0,0,0(%4)\n"
171 " jl 0b\n" 165 " jl 0b\n"
172 : "=&d" (old), "=m" (*(int *) addr) 166 : "=&d" (old), "=m" (*(int *) addr)
173 : "d" (x << shift), "d" (~(65535 << shift)), "a" (addr), 167 : "d" (x << shift), "d" (~(65535 << shift)), "a" (addr),
174 "m" (*(int *) addr) : "memory", "cc", "0" ); 168 "m" (*(int *) addr) : "memory", "cc", "0");
175 x = old >> shift; 169 x = old >> shift;
176 break; 170 break;
177 case 4: 171 case 4:
178 asm volatile ( 172 asm volatile(
179 " l %0,0(%3)\n" 173 " l %0,0(%3)\n"
180 "0: cs %0,%2,0(%3)\n" 174 "0: cs %0,%2,0(%3)\n"
181 " jl 0b\n" 175 " jl 0b\n"
182 : "=&d" (old), "=m" (*(int *) ptr) 176 : "=&d" (old), "=m" (*(int *) ptr)
183 : "d" (x), "a" (ptr), "m" (*(int *) ptr) 177 : "d" (x), "a" (ptr), "m" (*(int *) ptr)
184 : "memory", "cc" ); 178 : "memory", "cc");
185 x = old; 179 x = old;
186 break; 180 break;
187#ifdef __s390x__ 181#ifdef __s390x__
188 case 8: 182 case 8:
189 asm volatile ( 183 asm volatile(
190 " lg %0,0(%3)\n" 184 " lg %0,0(%3)\n"
191 "0: csg %0,%2,0(%3)\n" 185 "0: csg %0,%2,0(%3)\n"
192 " jl 0b\n" 186 " jl 0b\n"
193 : "=&d" (old), "=m" (*(long *) ptr) 187 : "=&d" (old), "=m" (*(long *) ptr)
194 : "d" (x), "a" (ptr), "m" (*(long *) ptr) 188 : "d" (x), "a" (ptr), "m" (*(long *) ptr)
195 : "memory", "cc" ); 189 : "memory", "cc");
196 x = old; 190 x = old;
197 break; 191 break;
198#endif /* __s390x__ */ 192#endif /* __s390x__ */
@@ -224,55 +218,55 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
224 shift = (3 ^ (addr & 3)) << 3; 218 shift = (3 ^ (addr & 3)) << 3;
225 addr ^= addr & 3; 219 addr ^= addr & 3;
226 asm volatile( 220 asm volatile(
227 " l %0,0(%4)\n" 221 " l %0,0(%4)\n"
228 "0: nr %0,%5\n" 222 "0: nr %0,%5\n"
229 " lr %1,%0\n" 223 " lr %1,%0\n"
230 " or %0,%2\n" 224 " or %0,%2\n"
231 " or %1,%3\n" 225 " or %1,%3\n"
232 " cs %0,%1,0(%4)\n" 226 " cs %0,%1,0(%4)\n"
233 " jnl 1f\n" 227 " jnl 1f\n"
234 " xr %1,%0\n" 228 " xr %1,%0\n"
235 " nr %1,%5\n" 229 " nr %1,%5\n"
236 " jnz 0b\n" 230 " jnz 0b\n"
237 "1:" 231 "1:"
238 : "=&d" (prev), "=&d" (tmp) 232 : "=&d" (prev), "=&d" (tmp)
239 : "d" (old << shift), "d" (new << shift), "a" (ptr), 233 : "d" (old << shift), "d" (new << shift), "a" (ptr),
240 "d" (~(255 << shift)) 234 "d" (~(255 << shift))
241 : "memory", "cc" ); 235 : "memory", "cc");
242 return prev >> shift; 236 return prev >> shift;
243 case 2: 237 case 2:
244 addr = (unsigned long) ptr; 238 addr = (unsigned long) ptr;
245 shift = (2 ^ (addr & 2)) << 3; 239 shift = (2 ^ (addr & 2)) << 3;
246 addr ^= addr & 2; 240 addr ^= addr & 2;
247 asm volatile( 241 asm volatile(
248 " l %0,0(%4)\n" 242 " l %0,0(%4)\n"
249 "0: nr %0,%5\n" 243 "0: nr %0,%5\n"
250 " lr %1,%0\n" 244 " lr %1,%0\n"
251 " or %0,%2\n" 245 " or %0,%2\n"
252 " or %1,%3\n" 246 " or %1,%3\n"
253 " cs %0,%1,0(%4)\n" 247 " cs %0,%1,0(%4)\n"
254 " jnl 1f\n" 248 " jnl 1f\n"
255 " xr %1,%0\n" 249 " xr %1,%0\n"
256 " nr %1,%5\n" 250 " nr %1,%5\n"
257 " jnz 0b\n" 251 " jnz 0b\n"
258 "1:" 252 "1:"
259 : "=&d" (prev), "=&d" (tmp) 253 : "=&d" (prev), "=&d" (tmp)
260 : "d" (old << shift), "d" (new << shift), "a" (ptr), 254 : "d" (old << shift), "d" (new << shift), "a" (ptr),
261 "d" (~(65535 << shift)) 255 "d" (~(65535 << shift))
262 : "memory", "cc" ); 256 : "memory", "cc");
263 return prev >> shift; 257 return prev >> shift;
264 case 4: 258 case 4:
265 asm volatile ( 259 asm volatile(
266 " cs %0,%2,0(%3)\n" 260 " cs %0,%2,0(%3)\n"
267 : "=&d" (prev) : "0" (old), "d" (new), "a" (ptr) 261 : "=&d" (prev) : "0" (old), "d" (new), "a" (ptr)
268 : "memory", "cc" ); 262 : "memory", "cc");
269 return prev; 263 return prev;
270#ifdef __s390x__ 264#ifdef __s390x__
271 case 8: 265 case 8:
272 asm volatile ( 266 asm volatile(
273 " csg %0,%2,0(%3)\n" 267 " csg %0,%2,0(%3)\n"
274 : "=&d" (prev) : "0" (old), "d" (new), "a" (ptr) 268 : "=&d" (prev) : "0" (old), "d" (new), "a" (ptr)
275 : "memory", "cc" ); 269 : "memory", "cc");
276 return prev; 270 return prev;
277#endif /* __s390x__ */ 271#endif /* __s390x__ */
278 } 272 }
@@ -289,8 +283,8 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
289 * all memory ops have completed wrt other CPU's ( see 7-15 POP DJB ). 283 * all memory ops have completed wrt other CPU's ( see 7-15 POP DJB ).
290 */ 284 */
291 285
292#define eieio() __asm__ __volatile__ ( "bcr 15,0" : : : "memory" ) 286#define eieio() asm volatile("bcr 15,0" : : : "memory")
293# define SYNC_OTHER_CORES(x) eieio() 287#define SYNC_OTHER_CORES(x) eieio()
294#define mb() eieio() 288#define mb() eieio()
295#define rmb() eieio() 289#define rmb() eieio()
296#define wmb() eieio() 290#define wmb() eieio()
@@ -307,117 +301,56 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
307 301
308#ifdef __s390x__ 302#ifdef __s390x__
309 303
310#define __ctl_load(array, low, high) ({ \ 304#define __ctl_load(array, low, high) ({ \
311 typedef struct { char _[sizeof(array)]; } addrtype; \ 305 typedef struct { char _[sizeof(array)]; } addrtype; \
312 __asm__ __volatile__ ( \ 306 asm volatile( \
313 " bras 1,0f\n" \ 307 " lctlg %1,%2,0(%0)\n" \
314 " lctlg 0,0,0(%0)\n" \ 308 : : "a" (&array), "i" (low), "i" (high), \
315 "0: ex %1,0(1)" \ 309 "m" (*(addrtype *)(array))); \
316 : : "a" (&array), "a" (((low)<<4)+(high)), \
317 "m" (*(addrtype *)(array)) : "1" ); \
318 }) 310 })
319 311
320#define __ctl_store(array, low, high) ({ \ 312#define __ctl_store(array, low, high) ({ \
321 typedef struct { char _[sizeof(array)]; } addrtype; \ 313 typedef struct { char _[sizeof(array)]; } addrtype; \
322 __asm__ __volatile__ ( \ 314 asm volatile( \
323 " bras 1,0f\n" \ 315 " stctg %2,%3,0(%1)\n" \
324 " stctg 0,0,0(%1)\n" \ 316 : "=m" (*(addrtype *)(array)) \
325 "0: ex %2,0(1)" \ 317 : "a" (&array), "i" (low), "i" (high)); \
326 : "=m" (*(addrtype *)(array)) \
327 : "a" (&array), "a" (((low)<<4)+(high)) : "1" ); \
328 }) 318 })
329 319
330#define __ctl_set_bit(cr, bit) ({ \
331 __u8 __dummy[24]; \
332 __asm__ __volatile__ ( \
333 " bras 1,0f\n" /* skip indirect insns */ \
334 " stctg 0,0,0(%1)\n" \
335 " lctlg 0,0,0(%1)\n" \
336 "0: ex %2,0(1)\n" /* execute stctl */ \
337 " lg 0,0(%1)\n" \
338 " ogr 0,%3\n" /* set the bit */ \
339 " stg 0,0(%1)\n" \
340 "1: ex %2,6(1)" /* execute lctl */ \
341 : "=m" (__dummy) \
342 : "a" ((((unsigned long) &__dummy) + 7) & ~7UL), \
343 "a" (cr*17), "a" (1L<<(bit)) \
344 : "cc", "0", "1" ); \
345 })
346
347#define __ctl_clear_bit(cr, bit) ({ \
348 __u8 __dummy[16]; \
349 __asm__ __volatile__ ( \
350 " bras 1,0f\n" /* skip indirect insns */ \
351 " stctg 0,0,0(%1)\n" \
352 " lctlg 0,0,0(%1)\n" \
353 "0: ex %2,0(1)\n" /* execute stctl */ \
354 " lg 0,0(%1)\n" \
355 " ngr 0,%3\n" /* set the bit */ \
356 " stg 0,0(%1)\n" \
357 "1: ex %2,6(1)" /* execute lctl */ \
358 : "=m" (__dummy) \
359 : "a" ((((unsigned long) &__dummy) + 7) & ~7UL), \
360 "a" (cr*17), "a" (~(1L<<(bit))) \
361 : "cc", "0", "1" ); \
362 })
363
364#else /* __s390x__ */ 320#else /* __s390x__ */
365 321
366#define __ctl_load(array, low, high) ({ \ 322#define __ctl_load(array, low, high) ({ \
367 typedef struct { char _[sizeof(array)]; } addrtype; \ 323 typedef struct { char _[sizeof(array)]; } addrtype; \
368 __asm__ __volatile__ ( \ 324 asm volatile( \
369 " bras 1,0f\n" \ 325 " lctl %1,%2,0(%0)\n" \
370 " lctl 0,0,0(%0)\n" \ 326 : : "a" (&array), "i" (low), "i" (high), \
371 "0: ex %1,0(1)" \ 327 "m" (*(addrtype *)(array))); \
372 : : "a" (&array), "a" (((low)<<4)+(high)), \ 328})
373 "m" (*(addrtype *)(array)) : "1" ); \
374 })
375 329
376#define __ctl_store(array, low, high) ({ \ 330#define __ctl_store(array, low, high) ({ \
377 typedef struct { char _[sizeof(array)]; } addrtype; \ 331 typedef struct { char _[sizeof(array)]; } addrtype; \
378 __asm__ __volatile__ ( \ 332 asm volatile( \
379 " bras 1,0f\n" \ 333 " stctl %2,%3,0(%1)\n" \
380 " stctl 0,0,0(%1)\n" \ 334 : "=m" (*(addrtype *)(array)) \
381 "0: ex %2,0(1)" \ 335 : "a" (&array), "i" (low), "i" (high)); \
382 : "=m" (*(addrtype *)(array)) \
383 : "a" (&array), "a" (((low)<<4)+(high)): "1" ); \
384 }) 336 })
385 337
386#define __ctl_set_bit(cr, bit) ({ \
387 __u8 __dummy[16]; \
388 __asm__ __volatile__ ( \
389 " bras 1,0f\n" /* skip indirect insns */ \
390 " stctl 0,0,0(%1)\n" \
391 " lctl 0,0,0(%1)\n" \
392 "0: ex %2,0(1)\n" /* execute stctl */ \
393 " l 0,0(%1)\n" \
394 " or 0,%3\n" /* set the bit */ \
395 " st 0,0(%1)\n" \
396 "1: ex %2,4(1)" /* execute lctl */ \
397 : "=m" (__dummy) \
398 : "a" ((((unsigned long) &__dummy) + 7) & ~7UL), \
399 "a" (cr*17), "a" (1<<(bit)) \
400 : "cc", "0", "1" ); \
401 })
402
403#define __ctl_clear_bit(cr, bit) ({ \
404 __u8 __dummy[16]; \
405 __asm__ __volatile__ ( \
406 " bras 1,0f\n" /* skip indirect insns */ \
407 " stctl 0,0,0(%1)\n" \
408 " lctl 0,0,0(%1)\n" \
409 "0: ex %2,0(1)\n" /* execute stctl */ \
410 " l 0,0(%1)\n" \
411 " nr 0,%3\n" /* set the bit */ \
412 " st 0,0(%1)\n" \
413 "1: ex %2,4(1)" /* execute lctl */ \
414 : "=m" (__dummy) \
415 : "a" ((((unsigned long) &__dummy) + 7) & ~7UL), \
416 "a" (cr*17), "a" (~(1<<(bit))) \
417 : "cc", "0", "1" ); \
418 })
419#endif /* __s390x__ */ 338#endif /* __s390x__ */
420 339
340#define __ctl_set_bit(cr, bit) ({ \
341 unsigned long __dummy; \
342 __ctl_store(__dummy, cr, cr); \
343 __dummy |= 1UL << (bit); \
344 __ctl_load(__dummy, cr, cr); \
345})
346
347#define __ctl_clear_bit(cr, bit) ({ \
348 unsigned long __dummy; \
349 __ctl_store(__dummy, cr, cr); \
350 __dummy &= ~(1UL << (bit)); \
351 __ctl_load(__dummy, cr, cr); \
352})
353
421#include <linux/irqflags.h> 354#include <linux/irqflags.h>
422 355
423/* 356/*
@@ -427,8 +360,7 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
427static inline void 360static inline void
428__set_psw_mask(unsigned long mask) 361__set_psw_mask(unsigned long mask)
429{ 362{
430 local_save_flags(mask); 363 __load_psw_mask(mask | (__raw_local_irq_stosm(0x00) & ~(-1UL >> 8)));
431 __load_psw_mask(mask);
432} 364}
433 365
434#define local_mcck_enable() __set_psw_mask(PSW_KERNEL_BITS) 366#define local_mcck_enable() __set_psw_mask(PSW_KERNEL_BITS)
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-22 19:18:33 -0500