[PATCH] s390: machine check handler bugs

The new machine check handler still has a few bugs.

1) The system entry time has to be stored in the machine check handler,

2) the machine check return psw may not be stored at the usual place
   because it might overwrite the return psw of the interrupted context,

3) the return address for the call to s390_handle_mcck in the i/o interrupt
   handler is not correct,

4) the system call cleanup has to take the different save area of the
   machine check handler into account,

5) the machine check handler may not call UPDATE_VTIME before
   CREATE_STACK_FRAME, and

6) the io leave path needs a critical section cleanup to make sure that the
   TIF_MCCK_PENDING bit is really checked before switching back to user space.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

2 files changed, 174 insertions, 55 deletions

diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 1a271b16cb5c..cbe7d6a2d02c 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -138,14 +138,14 @@ STACK_SIZE  = 1 << STACK_SHIFT
         st      %r12,__SF_BACKCHAIN(%r15)       # clear back chain
         .endm
 
-        .macro  RESTORE_ALL sync
-        mvc     __LC_RETURN_PSW(8),SP_PSW(%r15) # move user PSW to lowcore
+        .macro  RESTORE_ALL psworg,sync
+        mvc     \psworg(8),SP_PSW(%r15) # move user PSW to lowcore
         .if !\sync
-        ni      __LC_RETURN_PSW+1,0xfd  # clear wait state bit
+        ni      \psworg+1,0xfd          # clear wait state bit
         .endif
         lm      %r0,%r15,SP_R0(%r15)    # load gprs 0-15 of user
         STORE_TIMER __LC_EXIT_TIMER
-        lpsw    __LC_RETURN_PSW         # back to caller
+        lpsw    \psworg                 # back to caller
         .endm
 
 /*
@@ -235,7 +235,7 @@ sysc_return:
         tm      __TI_flags+3(%r9),_TIF_WORK_SVC
         bnz     BASED(sysc_work)  # there is work to do (signals etc.)
 sysc_leave:
-        RESTORE_ALL 1
+        RESTORE_ALL __LC_RETURN_PSW,1
 
 #
 # recheck if there is more work to do
@@ -312,8 +312,6 @@ sysc_singlestep:
         la      %r14,BASED(sysc_return) # load adr. of system return
         br      %r1                     # branch to do_single_step
 
-__critical_end:
-
 #
 # call trace before and after sys_call
 #
@@ -571,7 +569,8 @@ io_return:
         tm      __TI_flags+3(%r9),_TIF_WORK_INT
         bnz     BASED(io_work)         # there is work to do (signals etc.)
 io_leave:
-        RESTORE_ALL 0
+        RESTORE_ALL __LC_RETURN_PSW,0
+io_done:
 
 #ifdef CONFIG_PREEMPT
 io_preempt:
@@ -621,7 +620,7 @@ io_work_loop:
 #
 io_mcck_pending:
         l       %r1,BASED(.Ls390_handle_mcck)
-        l       %r14,BASED(io_work_loop)
+        la      %r14,BASED(io_work_loop)
         br      %r1                    # TIF bit will be cleared by handler
 
 #
@@ -674,6 +673,8 @@ ext_no_vtime:
         basr    %r14,%r1
         b       BASED(io_return)
 
+__critical_end:
+
 /*
  * Machine check handler routines
  */
@@ -681,6 +682,7 @@ ext_no_vtime:
         .globl mcck_int_handler
 mcck_int_handler:
         spt     __LC_CPU_TIMER_SAVE_AREA        # revalidate cpu timer
+        mvc     __LC_ASYNC_ENTER_TIMER(8),__LC_CPU_TIMER_SAVE_AREA
         lm      %r0,%r15,__LC_GPREGS_SAVE_AREA  # revalidate gprs
         SAVE_ALL_BASE __LC_SAVE_AREA+32
         la      %r12,__LC_MCK_OLD_PSW
@@ -693,17 +695,8 @@ mcck_int_handler:
         mvc     __LC_ASYNC_ENTER_TIMER(8),__LC_LAST_UPDATE_TIMER
         mvc     __LC_SYNC_ENTER_TIMER(8),__LC_LAST_UPDATE_TIMER
         mvc     __LC_EXIT_TIMER(8),__LC_LAST_UPDATE_TIMER
-0:      tm      __LC_MCCK_CODE+2,0x08   # mwp of old psw valid?
-        bno     BASED(mcck_no_vtime)    # no -> skip cleanup critical
-        tm      __LC_MCK_OLD_PSW+1,0x01 # interrupting from user ?
-        bz      BASED(mcck_no_vtime)
-        UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
-        UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
-        mvc     __LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
-mcck_no_vtime:
 #endif
-0:
-        tm      __LC_MCCK_CODE+2,0x09   # mwp + ia of old psw valid?
+0:      tm      __LC_MCCK_CODE+2,0x09   # mwp + ia of old psw valid?
         bno     BASED(mcck_int_main)    # no -> skip cleanup critical
         tm      __LC_MCK_OLD_PSW+1,0x01 # test problem state bit
         bnz     BASED(mcck_int_main)    # from user -> load async stack
@@ -720,6 +713,16 @@ mcck_int_main:
         be      BASED(0f)
         l       %r15,__LC_PANIC_STACK   # load panic stack
 0:      CREATE_STACK_FRAME __LC_MCK_OLD_PSW,__LC_SAVE_AREA+32
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+        tm      __LC_MCCK_CODE+2,0x08   # mwp of old psw valid?
+        bno     BASED(mcck_no_vtime)    # no -> skip cleanup critical
+        tm      __LC_MCK_OLD_PSW+1,0x01 # interrupting from user ?
+        bz      BASED(mcck_no_vtime)
+        UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
+        UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
+        mvc     __LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
+mcck_no_vtime:
+#endif
         l       %r9,__LC_THREAD_INFO    # load pointer to thread_info struct
         la      %r2,SP_PTREGS(%r15)     # load pt_regs
         l       %r1,BASED(.Ls390_mcck)
@@ -737,7 +740,7 @@ mcck_int_main:
         l       %r1,BASED(.Ls390_handle_mcck)
         basr    %r14,%r1                # call machine check handler
 mcck_return:
-        RESTORE_ALL 0
+        RESTORE_ALL __LC_RETURN_MCCK_PSW,0
 
 #ifdef CONFIG_SMP
 /*
@@ -803,6 +806,10 @@ cleanup_table_sysc_leave:
         .long   sysc_leave + 0x80000000, sysc_work_loop + 0x80000000
 cleanup_table_sysc_work_loop:
         .long   sysc_work_loop + 0x80000000, sysc_reschedule + 0x80000000
+cleanup_table_io_leave:
+        .long   io_leave + 0x80000000, io_done + 0x80000000
+cleanup_table_io_work_loop:
+        .long   io_work_loop + 0x80000000, io_mcck_pending + 0x80000000
 
 cleanup_critical:
         clc     4(4,%r12),BASED(cleanup_table_system_call)
@@ -825,10 +832,26 @@ cleanup_critical:
         clc     4(4,%r12),BASED(cleanup_table_sysc_work_loop+4)
         bl      BASED(cleanup_sysc_return)
 0:
+        clc     4(4,%r12),BASED(cleanup_table_io_leave)
+        bl      BASED(0f)
+        clc     4(4,%r12),BASED(cleanup_table_io_leave+4)
+        bl      BASED(cleanup_io_leave)
+0:
+        clc     4(4,%r12),BASED(cleanup_table_io_work_loop)
+        bl      BASED(0f)
+        clc     4(4,%r12),BASED(cleanup_table_io_work_loop+4)
+        bl      BASED(cleanup_io_return)
+0:
         br      %r14
 
 cleanup_system_call:
         mvc     __LC_RETURN_PSW(8),0(%r12)
+        c       %r12,BASED(.Lmck_old_psw)
+        be      BASED(0f)
+        la      %r12,__LC_SAVE_AREA+16
+        b       BASED(1f)
+0:      la      %r12,__LC_SAVE_AREA+32
+1:
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
         clc     __LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn+4)
         bh      BASED(0f)
@@ -838,11 +861,13 @@ cleanup_system_call:
 #endif
         clc     __LC_RETURN_PSW+4(4),BASED(cleanup_system_call_insn)
         bh      BASED(0f)
-        mvc     __LC_SAVE_AREA(16),__LC_SAVE_AREA+16
-0:      st      %r13,__LC_SAVE_AREA+20
+        mvc     __LC_SAVE_AREA(16),0(%r12)
+0:      st      %r13,4(%r12)
+        st      %r12,__LC_SAVE_AREA+48  # argh
         SAVE_ALL __LC_SVC_OLD_PSW,__LC_SAVE_AREA,1
         CREATE_STACK_FRAME __LC_SVC_OLD_PSW,__LC_SAVE_AREA
-        st      %r15,__LC_SAVE_AREA+28
+        l       %r12,__LC_SAVE_AREA+48  # argh
+        st      %r15,12(%r12)
         lh      %r7,0x8a
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
 cleanup_vtime:
@@ -879,17 +904,21 @@ cleanup_sysc_return:
 
 cleanup_sysc_leave:
         clc     4(4,%r12),BASED(cleanup_sysc_leave_insn)
-        be      BASED(0f)
+        be      BASED(2f)
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
         mvc     __LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
         clc     4(4,%r12),BASED(cleanup_sysc_leave_insn+4)
-        be      BASED(0f)
+        be      BASED(2f)
 #endif
         mvc     __LC_RETURN_PSW(8),SP_PSW(%r15)
-        mvc     __LC_SAVE_AREA+16(16),SP_R12(%r15)
-        lm      %r0,%r11,SP_R0(%r15)
+        c       %r12,BASED(.Lmck_old_psw)
+        bne     BASED(0f)
+        mvc     __LC_SAVE_AREA+32(16),SP_R12(%r15)
+        b       BASED(1f)
+0:      mvc     __LC_SAVE_AREA+16(16),SP_R12(%r15)
+1:      lm      %r0,%r11,SP_R0(%r15)
         l       %r15,SP_R15(%r15)
-0:      la      %r12,__LC_RETURN_PSW
+2:      la      %r12,__LC_RETURN_PSW
         br      %r14
 cleanup_sysc_leave_insn:
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
@@ -897,6 +926,36 @@ cleanup_sysc_leave_insn:
 #endif
         .long   sysc_leave + 10 + 0x80000000
 
+cleanup_io_return:
+        mvc     __LC_RETURN_PSW(4),0(%r12)
+        mvc     __LC_RETURN_PSW+4(4),BASED(cleanup_table_io_work_loop)
+        la      %r12,__LC_RETURN_PSW
+        br      %r14
+
+cleanup_io_leave:
+        clc     4(4,%r12),BASED(cleanup_io_leave_insn)
+        be      BASED(2f)
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+        mvc     __LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
+        clc     4(4,%r12),BASED(cleanup_io_leave_insn+4)
+        be      BASED(2f)
+#endif
+        mvc     __LC_RETURN_PSW(8),SP_PSW(%r15)
+        c       %r12,BASED(.Lmck_old_psw)
+        bne     BASED(0f)
+        mvc     __LC_SAVE_AREA+32(16),SP_R12(%r15)
+        b       BASED(1f)
+0:      mvc     __LC_SAVE_AREA+16(16),SP_R12(%r15)
+1:      lm      %r0,%r11,SP_R0(%r15)
+        l       %r15,SP_R15(%r15)
+2:      la      %r12,__LC_RETURN_PSW
+        br      %r14
+cleanup_io_leave_insn:
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+        .long   io_leave + 18 + 0x80000000
+#endif
+        .long   io_leave + 14 + 0x80000000
+
 /*
  * Integer constants
  */
@@ -918,6 +977,7 @@ cleanup_sysc_leave_insn:
 .Ls390_mcck:   .long  s390_do_machine_check
 .Ls390_handle_mcck:
                .long  s390_handle_mcck
+.Lmck_old_psw: .long  __LC_MCK_OLD_PSW
 .Ldo_IRQ:      .long  do_IRQ
 .Ldo_extint:   .long  do_extint
 .Ldo_signal:   .long  do_signal
diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S
index d9f22915008c..5db5976ff594 100644
--- a/arch/s390/kernel/entry64.S
+++ b/arch/s390/kernel/entry64.S
@@ -131,14 +131,14 @@ _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NEED_RESCHED | _TIF_MCCK_PENDING)
         stg     %r12,__SF_BACKCHAIN(%r15)
         .endm
 
-        .macro  RESTORE_ALL sync
-        mvc     __LC_RETURN_PSW(16),SP_PSW(%r15) # move user PSW to lowcore
+        .macro  RESTORE_ALL psworg,sync
+        mvc     \psworg(16),SP_PSW(%r15) # move user PSW to lowcore
         .if !\sync
-        ni      __LC_RETURN_PSW+1,0xfd  # clear wait state bit
+        ni      \psworg+1,0xfd          # clear wait state bit
         .endif
         lmg     %r0,%r15,SP_R0(%r15)    # load gprs 0-15 of user
         STORE_TIMER __LC_EXIT_TIMER
-        lpswe   __LC_RETURN_PSW         # back to caller
+        lpswe   \psworg                 # back to caller
         .endm
 
 /*
@@ -233,7 +233,7 @@ sysc_return:
         tm      __TI_flags+7(%r9),_TIF_WORK_SVC
         jnz     sysc_work         # there is work to do (signals etc.)
 sysc_leave:
-        RESTORE_ALL 1
+        RESTORE_ALL __LC_RETURN_PSW,1
 
 #
 # recheck if there is more work to do
@@ -308,8 +308,6 @@ sysc_singlestep:
         jg      do_single_step          # branch to do_sigtrap
 
 
-__critical_end:
-
 #
 # call syscall_trace before and after system call
 # special linkage: %r12 contains the return address for trace_svc
@@ -612,7 +610,8 @@ io_return:
         tm      __TI_flags+7(%r9),_TIF_WORK_INT
         jnz     io_work                # there is work to do (signals etc.)
 io_leave:
-        RESTORE_ALL 0
+        RESTORE_ALL __LC_RETURN_PSW,0
+io_done:
 
 #ifdef CONFIG_PREEMPT
 io_preempt:
@@ -711,6 +710,8 @@ ext_no_vtime:
         brasl   %r14,do_extint
         j       io_return
 
+__critical_end:
+
 /*
  * Machine check handler routines
  */
@@ -718,6 +719,7 @@ ext_no_vtime:
 mcck_int_handler:
         la      %r1,4095                # revalidate r1
         spt     __LC_CPU_TIMER_SAVE_AREA-4095(%r1)      # revalidate cpu timer
+        mvc     __LC_ASYNC_ENTER_TIMER(8),__LC_CPU_TIMER_SAVE_AREA-4095(%r1)
         lmg     %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs
         SAVE_ALL_BASE __LC_SAVE_AREA+64
         la      %r12,__LC_MCK_OLD_PSW
@@ -730,17 +732,8 @@ mcck_int_handler:
         mvc     __LC_ASYNC_ENTER_TIMER(8),__LC_LAST_UPDATE_TIMER
         mvc     __LC_SYNC_ENTER_TIMER(8),__LC_LAST_UPDATE_TIMER
         mvc     __LC_EXIT_TIMER(8),__LC_LAST_UPDATE_TIMER
-0:      tm      __LC_MCCK_CODE+2,0x08   # mwp of old psw valid?
-        jno     mcck_no_vtime           # no -> no timer update
-        tm      __LC_MCK_OLD_PSW+1,0x01 # interrupting from user ?
-        jz      mcck_no_vtime
-        UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
-        UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
-        mvc     __LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
-mcck_no_vtime:
 #endif
-0:
-        tm      __LC_MCCK_CODE+2,0x09   # mwp + ia of old psw valid?
+0:      tm      __LC_MCCK_CODE+2,0x09   # mwp + ia of old psw valid?
         jno     mcck_int_main           # no -> skip cleanup critical
         tm      __LC_MCK_OLD_PSW+1,0x01 # test problem state bit
         jnz     mcck_int_main           # from user -> load kernel stack
@@ -756,6 +749,16 @@ mcck_int_main:
         jz      0f
         lg      %r15,__LC_PANIC_STACK   # load panic stack
 0:      CREATE_STACK_FRAME __LC_MCK_OLD_PSW,__LC_SAVE_AREA+64
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+        tm      __LC_MCCK_CODE+2,0x08   # mwp of old psw valid?
+        jno     mcck_no_vtime           # no -> no timer update
+        tm      __LC_MCK_OLD_PSW+1,0x01 # interrupting from user ?
+        jz      mcck_no_vtime
+        UPDATE_VTIME __LC_EXIT_TIMER,__LC_ASYNC_ENTER_TIMER,__LC_USER_TIMER
+        UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER
+        mvc     __LC_LAST_UPDATE_TIMER(8),__LC_ASYNC_ENTER_TIMER
+mcck_no_vtime:
+#endif
         lg      %r9,__LC_THREAD_INFO    # load pointer to thread_info struct
         la      %r2,SP_PTREGS(%r15)     # load pt_regs
         brasl   %r14,s390_do_machine_check
@@ -771,7 +774,7 @@ mcck_int_main:
         jno     mcck_return
         brasl   %r14,s390_handle_mcck
 mcck_return:
-        RESTORE_ALL 0
+        RESTORE_ALL __LC_RETURN_MCCK_PSW,0
 
 #ifdef CONFIG_SMP
 /*
@@ -833,6 +836,10 @@ cleanup_table_sysc_leave:
         .quad   sysc_leave, sysc_work_loop
 cleanup_table_sysc_work_loop:
         .quad   sysc_work_loop, sysc_reschedule
+cleanup_table_io_leave:
+        .quad   io_leave, io_done
+cleanup_table_io_work_loop:
+        .quad   io_work_loop, io_mcck_pending
 
 cleanup_critical:
         clc     8(8,%r12),BASED(cleanup_table_system_call)
@@ -855,10 +862,26 @@ cleanup_critical:
         clc     8(8,%r12),BASED(cleanup_table_sysc_work_loop+8)
         jl      cleanup_sysc_return
 0:
+        clc     8(8,%r12),BASED(cleanup_table_io_leave)
+        jl      0f
+        clc     8(8,%r12),BASED(cleanup_table_io_leave+8)
+        jl      cleanup_io_leave
+0:
+        clc     8(8,%r12),BASED(cleanup_table_io_work_loop)
+        jl      0f
+        clc     8(8,%r12),BASED(cleanup_table_io_work_loop+8)
+        jl      cleanup_io_return
+0:
         br      %r14
 
 cleanup_system_call:
         mvc     __LC_RETURN_PSW(16),0(%r12)
+        cghi    %r12,__LC_MCK_OLD_PSW
+        je      0f
+        la      %r12,__LC_SAVE_AREA+32
+        j       1f
+0:      la      %r12,__LC_SAVE_AREA+64
+1:
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
         clc     __LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn+8)
         jh      0f
@@ -868,11 +891,13 @@ cleanup_system_call:
 #endif
         clc     __LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn)
         jh      0f
-        mvc     __LC_SAVE_AREA(32),__LC_SAVE_AREA+32
-0:      stg     %r13,__LC_SAVE_AREA+40
+        mvc     __LC_SAVE_AREA(32),0(%r12)
+0:      stg     %r13,8(%r12)
+        stg     %r12,__LC_SAVE_AREA+96  # argh
         SAVE_ALL __LC_SVC_OLD_PSW,__LC_SAVE_AREA,1
         CREATE_STACK_FRAME __LC_SVC_OLD_PSW,__LC_SAVE_AREA
-        stg     %r15,__LC_SAVE_AREA+56
+        lg      %r12,__LC_SAVE_AREA+96  # argh
+        stg     %r15,24(%r12)
         llgh    %r7,__LC_SVC_INT_CODE
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
 cleanup_vtime:
@@ -909,17 +934,21 @@ cleanup_sysc_return:
 
 cleanup_sysc_leave:
         clc     8(8,%r12),BASED(cleanup_sysc_leave_insn)
-        je      0f
+        je      2f
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
         mvc     __LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
         clc     8(8,%r12),BASED(cleanup_sysc_leave_insn+8)
-        je      0f
+        je      2f
 #endif
         mvc     __LC_RETURN_PSW(16),SP_PSW(%r15)
-        mvc     __LC_SAVE_AREA+32(32),SP_R12(%r15)
-        lmg     %r0,%r11,SP_R0(%r15)
+        cghi    %r12,__LC_MCK_OLD_PSW
+        jne     0f
+        mvc     __LC_SAVE_AREA+64(32),SP_R12(%r15)
+        j       1f
+0:      mvc     __LC_SAVE_AREA+32(32),SP_R12(%r15)
+1:      lmg     %r0,%r11,SP_R0(%r15)
         lg      %r15,SP_R15(%r15)
-0:      la      %r12,__LC_RETURN_PSW
+2:      la      %r12,__LC_RETURN_PSW
         br      %r14
 cleanup_sysc_leave_insn:
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
@@ -927,6 +956,36 @@ cleanup_sysc_leave_insn:
 #endif
         .quad   sysc_leave + 12
 
+cleanup_io_return:
+        mvc     __LC_RETURN_PSW(8),0(%r12)
+        mvc     __LC_RETURN_PSW+8(8),BASED(cleanup_table_io_work_loop)
+        la      %r12,__LC_RETURN_PSW
+        br      %r14
+
+cleanup_io_leave:
+        clc     8(8,%r12),BASED(cleanup_io_leave_insn)
+        je      2f
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+        mvc     __LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
+        clc     8(8,%r12),BASED(cleanup_io_leave_insn+8)
+        je      2f
+#endif
+        mvc     __LC_RETURN_PSW(16),SP_PSW(%r15)
+        cghi    %r12,__LC_MCK_OLD_PSW
+        jne     0f
+        mvc     __LC_SAVE_AREA+64(32),SP_R12(%r15)
+        j       1f
+0:      mvc     __LC_SAVE_AREA+32(32),SP_R12(%r15)
+1:      lmg     %r0,%r11,SP_R0(%r15)
+        lg      %r15,SP_R15(%r15)
+2:      la      %r12,__LC_RETURN_PSW
+        br      %r14
+cleanup_io_leave_insn:
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+        .quad   io_leave + 20
+#endif
+        .quad   io_leave + 16
+
 /*
  * Integer constants
  */