s390: split TIF bits into CIF, PIF and TIF bits

The oi and ni instructions used in entry[64].S to set and clear bits
in the thread-flags are not guaranteed to be atomic in regard to other
CPUs. Split the TIF bits into CPU, pt_regs and thread-info specific
bits. Updates on the TIF bits are done with atomic instructions,
updates on CPU and pt_regs bits are done with non-atomic instructions.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

1 files changed, 42 insertions, 40 deletions

diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index d15e7bf6a863..c41f3f906720 100644
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -42,13 +42,11 @@ STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
 STACK_SIZE  = 1 << STACK_SHIFT
 STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
 
-_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
-                 _TIF_MCCK_PENDING | _TIF_PER_TRAP | _TIF_ASCE)
-_TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
-                 _TIF_MCCK_PENDING | _TIF_ASCE)
-_TIF_TRACE    = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
-                 _TIF_SYSCALL_TRACEPOINT)
-_TIF_TRANSFER = (_TIF_MCCK_PENDING | _TIF_ASCE)
+_TIF_WORK       = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED)
+_TIF_TRACE      = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
+                   _TIF_SYSCALL_TRACEPOINT)
+_CIF_WORK       = (_CIF_MCCK_PENDING | _CIF_ASCE)
+_PIF_WORK       = (_PIF_PER_TRAP)
 
 #define BASED(name) name-system_call(%r13)
 
@@ -190,13 +188,7 @@ ENTRY(__switch_to)
         lctl    %c4,%c4,__TASK_pid(%r3)         # load pid to control reg. 4
         mvc     __LC_CURRENT_PID+4(4,%r0),__TASK_pid(%r3) # store pid of next
         lg      %r15,__THREAD_ksp(%r3)          # load kernel stack of next
-        llill   %r6,_TIF_TRANSFER               # transfer TIF bits
-        ng      %r6,__TI_flags(%r4)             # isolate TIF bits
-        jz      0f
-        og      %r6,__TI_flags(%r5)             # set TIF bits of next
-        stg     %r6,__TI_flags(%r5)
-        ni      __TI_flags+7(%r4),255-_TIF_TRANSFER # clear TIF bits of prev
-0:      lmg     %r6,%r15,__SF_GPRS(%r15)        # load gprs of next task
+        lmg     %r6,%r15,__SF_GPRS(%r15)        # load gprs of next task
         br      %r14
 
 __critical_start:
@@ -211,6 +203,7 @@ sysc_stmg:
         stmg    %r8,%r15,__LC_SAVE_AREA_SYNC
         lg      %r10,__LC_LAST_BREAK
         lg      %r12,__LC_THREAD_INFO
+        lghi    %r14,_PIF_SYSCALL
 sysc_per:
         lg      %r15,__LC_KERNEL_STACK
         la      %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
@@ -221,8 +214,8 @@ sysc_vtime:
         mvc     __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
         mvc     __PT_PSW(16,%r11),__LC_SVC_OLD_PSW
         mvc     __PT_INT_CODE(4,%r11),__LC_SVC_ILC
+        stg     %r14,__PT_FLAGS(%r11)
 sysc_do_svc:
-        oi      __TI_flags+7(%r12),_TIF_SYSCALL
         lg      %r10,__TI_sysc_table(%r12)      # address of system call table
         llgh    %r8,__PT_INT_CODE+2(%r11)
         slag    %r8,%r8,2                       # shift and test for svc 0
@@ -238,7 +231,7 @@ sysc_nr_ok:
         stg     %r2,__PT_ORIG_GPR2(%r11)
         stg     %r7,STACK_FRAME_OVERHEAD(%r15)
         lgf     %r9,0(%r8,%r10)                 # get system call add.
-        tm      __TI_flags+6(%r12),_TIF_TRACE >> 8
+        tm      __TI_flags+7(%r12),_TIF_TRACE
         jnz     sysc_tracesys
         basr    %r14,%r9                        # call sys_xxxx
         stg     %r2,__PT_R2(%r11)               # store return value
@@ -248,9 +241,12 @@ sysc_return:
 sysc_tif:
         tm      __PT_PSW+1(%r11),0x01           # returning to user ?
         jno     sysc_restore
-        tm      __TI_flags+7(%r12),_TIF_WORK_SVC
+        tm      __PT_FLAGS+7(%r11),_PIF_WORK
+        jnz     sysc_work
+        tm      __TI_flags+7(%r12),_TIF_WORK
         jnz     sysc_work                       # check for work
-        ni      __TI_flags+7(%r12),255-_TIF_SYSCALL
+        tm      __LC_CPU_FLAGS+7,_CIF_WORK
+        jnz     sysc_work
 sysc_restore:
         lg      %r14,__LC_VDSO_PER_CPU
         lmg     %r0,%r10,__PT_R0(%r11)
@@ -265,17 +261,17 @@ sysc_done:
 # One of the work bits is on. Find out which one.
 #
 sysc_work:
-        tm      __TI_flags+7(%r12),_TIF_MCCK_PENDING
+        tm      __LC_CPU_FLAGS+7,_CIF_MCCK_PENDING
         jo      sysc_mcck_pending
         tm      __TI_flags+7(%r12),_TIF_NEED_RESCHED
         jo      sysc_reschedule
-        tm      __TI_flags+7(%r12),_TIF_PER_TRAP
+        tm      __PT_FLAGS+7(%r11),_PIF_PER_TRAP
         jo      sysc_singlestep
         tm      __TI_flags+7(%r12),_TIF_SIGPENDING
         jo      sysc_sigpending
         tm      __TI_flags+7(%r12),_TIF_NOTIFY_RESUME
         jo      sysc_notify_resume
-        tm      __TI_flags+7(%r12),_TIF_ASCE
+        tm      __LC_CPU_FLAGS+7,_CIF_ASCE
         jo      sysc_uaccess
         j       sysc_return             # beware of critical section cleanup
 
@@ -287,17 +283,17 @@ sysc_reschedule:
         jg      schedule
 
 #
-# _TIF_MCCK_PENDING is set, call handler
+# _CIF_MCCK_PENDING is set, call handler
 #
 sysc_mcck_pending:
         larl    %r14,sysc_return
         jg      s390_handle_mcck        # TIF bit will be cleared by handler
 
 #
-# _TIF_ASCE is set, load user space asce
+# _CIF_ASCE is set, load user space asce
 #
 sysc_uaccess:
-        ni      __TI_flags+7(%r12),255-_TIF_ASCE
+        ni      __LC_CPU_FLAGS+7,255-_CIF_ASCE
         lctlg   %c1,%c1,__LC_USER_ASCE          # load primary asce
         j       sysc_return
 
@@ -307,7 +303,7 @@ sysc_uaccess:
 sysc_sigpending:
         lgr     %r2,%r11                # pass pointer to pt_regs
         brasl   %r14,do_signal
-        tm      __TI_flags+7(%r12),_TIF_SYSCALL
+        tm      __PT_FLAGS+7(%r11),_PIF_SYSCALL
         jno     sysc_return
         lmg     %r2,%r7,__PT_R2(%r11)   # load svc arguments
         lg      %r10,__TI_sysc_table(%r12)      # address of system call table
@@ -327,10 +323,10 @@ sysc_notify_resume:
         jg      do_notify_resume
 
 #
-# _TIF_PER_TRAP is set, call do_per_trap
+# _PIF_PER_TRAP is set, call do_per_trap
 #
 sysc_singlestep:
-        ni      __TI_flags+7(%r12),255-_TIF_PER_TRAP
+        ni      __PT_FLAGS+7(%r11),255-_PIF_PER_TRAP
         lgr     %r2,%r11                # pass pointer to pt_regs
         larl    %r14,sysc_return
         jg      do_per_trap
@@ -357,7 +353,7 @@ sysc_tracego:
         basr    %r14,%r9                # call sys_xxx
         stg     %r2,__PT_R2(%r11)       # store return value
 sysc_tracenogo:
-        tm      __TI_flags+6(%r12),_TIF_TRACE >> 8
+        tm      __TI_flags+7(%r12),_TIF_TRACE
         jz      sysc_return
         lgr     %r2,%r11                # pass pointer to pt_regs
         larl    %r14,sysc_return
@@ -416,12 +412,13 @@ ENTRY(pgm_check_handler)
         stmg    %r8,%r9,__PT_PSW(%r11)
         mvc     __PT_INT_CODE(4,%r11),__LC_PGM_ILC
         mvc     __PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE
+        xc      __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
         stg     %r10,__PT_ARGS(%r11)
         tm      __LC_PGM_ILC+3,0x80     # check for per exception
         jz      0f
         tmhh    %r8,0x0001              # kernel per event ?
         jz      pgm_kprobe
-        oi      __TI_flags+7(%r12),_TIF_PER_TRAP
+        oi      __PT_FLAGS+7(%r11),_PIF_PER_TRAP
         mvc     __THREAD_per_address(8,%r14),__LC_PER_ADDRESS
         mvc     __THREAD_per_cause(2,%r14),__LC_PER_CAUSE
         mvc     __THREAD_per_paid(1,%r14),__LC_PER_PAID
@@ -451,10 +448,10 @@ pgm_kprobe:
 # single stepped system call
 #
 pgm_svcper:
-        oi      __TI_flags+7(%r12),_TIF_PER_TRAP
         mvc     __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
         larl    %r14,sysc_per
         stg     %r14,__LC_RETURN_PSW+8
+        lghi    %r14,_PIF_SYSCALL | _PIF_PER_TRAP
         lpswe   __LC_RETURN_PSW         # branch to sysc_per and enable irqs
 
 /*
@@ -479,6 +476,7 @@ io_skip:
         mvc     __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
         stmg    %r8,%r9,__PT_PSW(%r11)
         mvc     __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
+        xc      __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
         TRACE_IRQS_OFF
         xc      __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
 io_loop:
@@ -499,8 +497,10 @@ io_return:
         LOCKDEP_SYS_EXIT
         TRACE_IRQS_ON
 io_tif:
-        tm      __TI_flags+7(%r12),_TIF_WORK_INT
+        tm      __TI_flags+7(%r12),_TIF_WORK
         jnz     io_work                 # there is work to do (signals etc.)
+        tm      __LC_CPU_FLAGS+7,_CIF_WORK
+        jnz     io_work
 io_restore:
         lg      %r14,__LC_VDSO_PER_CPU
         lmg     %r0,%r10,__PT_R0(%r11)
@@ -513,7 +513,7 @@ io_done:
 
 #
 # There is work todo, find out in which context we have been interrupted:
-# 1) if we return to user space we can do all _TIF_WORK_INT work
+# 1) if we return to user space we can do all _TIF_WORK work
 # 2) if we return to kernel code and kvm is enabled check if we need to
 #    modify the psw to leave SIE
 # 3) if we return to kernel code and preemptive scheduling is enabled check
@@ -557,11 +557,9 @@ io_work_user:
 
 #
 # One of the work bits is on. Find out which one.
-# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
-#              and _TIF_MCCK_PENDING
 #
 io_work_tif:
-        tm      __TI_flags+7(%r12),_TIF_MCCK_PENDING
+        tm      __LC_CPU_FLAGS+7,_CIF_MCCK_PENDING
         jo      io_mcck_pending
         tm      __TI_flags+7(%r12),_TIF_NEED_RESCHED
         jo      io_reschedule
@@ -569,12 +567,12 @@ io_work_tif:
         jo      io_sigpending
         tm      __TI_flags+7(%r12),_TIF_NOTIFY_RESUME
         jo      io_notify_resume
-        tm      __TI_flags+7(%r12),_TIF_ASCE
+        tm      __LC_CPU_FLAGS+7,_CIF_ASCE
         jo      io_uaccess
         j       io_return               # beware of critical section cleanup
 
 #
-# _TIF_MCCK_PENDING is set, call handler
+# _CIF_MCCK_PENDING is set, call handler
 #
 io_mcck_pending:
         # TRACE_IRQS_ON already done at io_return
@@ -583,10 +581,10 @@ io_mcck_pending:
         j       io_return
 
 #
-# _TIF_ASCE is set, load user space asce
+# _CIF_ASCE is set, load user space asce
 #
 io_uaccess:
-        ni      __TI_flags+7(%r12),255-_TIF_ASCE
+        ni      __LC_CPU_FLAGS+7,255-_CIF_ASCE
         lctlg   %c1,%c1,__LC_USER_ASCE          # load primary asce
         j       io_return
 
@@ -650,6 +648,7 @@ ext_skip:
         mvc     __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR
         mvc     __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
         mvc     __PT_INT_PARM_LONG(8,%r11),0(%r1)
+        xc      __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
         TRACE_IRQS_OFF
         xc      __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
         lgr     %r2,%r11                # pass pointer to pt_regs
@@ -716,6 +715,7 @@ mcck_skip:
         stmg    %r0,%r7,__PT_R0(%r11)
         mvc     __PT_R8(64,%r11),0(%r14)
         stmg    %r8,%r9,__PT_PSW(%r11)
+        xc      __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
         xc      __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
         lgr     %r2,%r11                # pass pointer to pt_regs
         brasl   %r14,s390_do_machine_check
@@ -727,7 +727,7 @@ mcck_skip:
         la      %r11,STACK_FRAME_OVERHEAD(%r1)
         lgr     %r15,%r1
         ssm     __LC_PGM_NEW_PSW        # turn dat on, keep irqs off
-        tm      __TI_flags+7(%r12),_TIF_MCCK_PENDING
+        tm      __LC_CPU_FLAGS+7,_CIF_MCCK_PENDING
         jno     mcck_return
         TRACE_IRQS_OFF
         brasl   %r14,s390_handle_mcck
@@ -884,6 +884,8 @@ cleanup_system_call:
         stmg    %r0,%r7,__PT_R0(%r9)
         mvc     __PT_PSW(16,%r9),__LC_SVC_OLD_PSW
         mvc     __PT_INT_CODE(4,%r9),__LC_SVC_ILC
+        xc      __PT_FLAGS(8,%r9),__PT_FLAGS(%r9)
+        mvi     __PT_FLAGS+7(%r9),_PIF_SYSCALL
         # setup saved register r15
         stg     %r15,56(%r11)           # r15 stack pointer
         # set new psw address and exit