diff options
Diffstat (limited to 'arch/s390/kernel/entry64.S')
-rw-r--r-- | arch/s390/kernel/entry64.S | 106 |
1 files changed, 73 insertions, 33 deletions
diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S index 05e26d1fdf40..e15c80efdd05 100644 --- a/arch/s390/kernel/entry64.S +++ b/arch/s390/kernel/entry64.S | |||
@@ -67,12 +67,28 @@ _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK | _TIF_NEED_RESCHED | \ | |||
67 | brasl %r14,trace_hardirqs_off | 67 | brasl %r14,trace_hardirqs_off |
68 | .endm | 68 | .endm |
69 | 69 | ||
70 | .macro LOCKDEP_SYS_EXIT | 70 | .macro TRACE_IRQS_CHECK |
71 | brasl %r14,lockdep_sys_exit | 71 | tm SP_PSW(%r15),0x03 # irqs enabled? |
72 | jz 0f | ||
73 | brasl %r14,trace_hardirqs_on | ||
74 | j 1f | ||
75 | 0: brasl %r14,trace_hardirqs_off | ||
76 | 1: | ||
72 | .endm | 77 | .endm |
73 | #else | 78 | #else |
74 | #define TRACE_IRQS_ON | 79 | #define TRACE_IRQS_ON |
75 | #define TRACE_IRQS_OFF | 80 | #define TRACE_IRQS_OFF |
81 | #define TRACE_IRQS_CHECK | ||
82 | #endif | ||
83 | |||
84 | #ifdef CONFIG_LOCKDEP | ||
85 | .macro LOCKDEP_SYS_EXIT | ||
86 | tm SP_PSW+1(%r15),0x01 # returning to user ? | ||
87 | jz 0f | ||
88 | brasl %r14,lockdep_sys_exit | ||
89 | 0: | ||
90 | .endm | ||
91 | #else | ||
76 | #define LOCKDEP_SYS_EXIT | 92 | #define LOCKDEP_SYS_EXIT |
77 | #endif | 93 | #endif |
78 | 94 | ||
@@ -222,8 +238,6 @@ sysc_saveall: | |||
222 | llgh %r7,__LC_SVC_INT_CODE # get svc number from lowcore | 238 | llgh %r7,__LC_SVC_INT_CODE # get svc number from lowcore |
223 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING | 239 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING |
224 | sysc_vtime: | 240 | sysc_vtime: |
225 | tm SP_PSW+1(%r15),0x01 # interrupting from user ? | ||
226 | jz sysc_do_svc | ||
227 | UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER | 241 | UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER |
228 | sysc_stime: | 242 | sysc_stime: |
229 | UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER | 243 | UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER |
@@ -257,19 +271,34 @@ sysc_noemu: | |||
257 | 271 | ||
258 | sysc_return: | 272 | sysc_return: |
259 | tm SP_PSW+1(%r15),0x01 # returning to user ? | 273 | tm SP_PSW+1(%r15),0x01 # returning to user ? |
260 | jno sysc_leave | 274 | jno sysc_restore |
261 | tm __TI_flags+7(%r9),_TIF_WORK_SVC | 275 | tm __TI_flags+7(%r9),_TIF_WORK_SVC |
262 | jnz sysc_work # there is work to do (signals etc.) | 276 | jnz sysc_work # there is work to do (signals etc.) |
277 | sysc_restore: | ||
278 | #ifdef CONFIG_TRACE_IRQFLAGS | ||
279 | larl %r1,sysc_restore_trace_psw | ||
280 | lpswe 0(%r1) | ||
281 | sysc_restore_trace: | ||
282 | TRACE_IRQS_CHECK | ||
263 | LOCKDEP_SYS_EXIT | 283 | LOCKDEP_SYS_EXIT |
284 | #endif | ||
264 | sysc_leave: | 285 | sysc_leave: |
265 | RESTORE_ALL __LC_RETURN_PSW,1 | 286 | RESTORE_ALL __LC_RETURN_PSW,1 |
287 | sysc_done: | ||
288 | |||
289 | #ifdef CONFIG_TRACE_IRQFLAGS | ||
290 | .align 8 | ||
291 | .globl sysc_restore_trace_psw | ||
292 | sysc_restore_trace_psw: | ||
293 | .quad 0, sysc_restore_trace | ||
294 | #endif | ||
266 | 295 | ||
267 | # | 296 | # |
268 | # recheck if there is more work to do | 297 | # recheck if there is more work to do |
269 | # | 298 | # |
270 | sysc_work_loop: | 299 | sysc_work_loop: |
271 | tm __TI_flags+7(%r9),_TIF_WORK_SVC | 300 | tm __TI_flags+7(%r9),_TIF_WORK_SVC |
272 | jz sysc_leave # there is no work to do | 301 | jz sysc_restore # there is no work to do |
273 | # | 302 | # |
274 | # One of the work bits is on. Find out which one. | 303 | # One of the work bits is on. Find out which one. |
275 | # | 304 | # |
@@ -284,8 +313,8 @@ sysc_work: | |||
284 | jo sysc_restart | 313 | jo sysc_restart |
285 | tm __TI_flags+7(%r9),_TIF_SINGLE_STEP | 314 | tm __TI_flags+7(%r9),_TIF_SINGLE_STEP |
286 | jo sysc_singlestep | 315 | jo sysc_singlestep |
287 | LOCKDEP_SYS_EXIT | 316 | j sysc_restore |
288 | j sysc_leave | 317 | sysc_work_done: |
289 | 318 | ||
290 | # | 319 | # |
291 | # _TIF_NEED_RESCHED is set, call schedule | 320 | # _TIF_NEED_RESCHED is set, call schedule |
@@ -445,6 +474,7 @@ pgm_check_handler: | |||
445 | pgm_no_vtime: | 474 | pgm_no_vtime: |
446 | #endif | 475 | #endif |
447 | lg %r9,__LC_THREAD_INFO # load pointer to thread_info struct | 476 | lg %r9,__LC_THREAD_INFO # load pointer to thread_info struct |
477 | TRACE_IRQS_OFF | ||
448 | lgf %r3,__LC_PGM_ILC # load program interruption code | 478 | lgf %r3,__LC_PGM_ILC # load program interruption code |
449 | lghi %r8,0x7f | 479 | lghi %r8,0x7f |
450 | ngr %r8,%r3 | 480 | ngr %r8,%r3 |
@@ -484,6 +514,7 @@ pgm_per_std: | |||
484 | pgm_no_vtime2: | 514 | pgm_no_vtime2: |
485 | #endif | 515 | #endif |
486 | lg %r9,__LC_THREAD_INFO # load pointer to thread_info struct | 516 | lg %r9,__LC_THREAD_INFO # load pointer to thread_info struct |
517 | TRACE_IRQS_OFF | ||
487 | lg %r1,__TI_task(%r9) | 518 | lg %r1,__TI_task(%r9) |
488 | tm SP_PSW+1(%r15),0x01 # kernel per event ? | 519 | tm SP_PSW+1(%r15),0x01 # kernel per event ? |
489 | jz kernel_per | 520 | jz kernel_per |
@@ -504,12 +535,9 @@ pgm_svcper: | |||
504 | SAVE_ALL_SYNC __LC_SVC_OLD_PSW,__LC_SAVE_AREA | 535 | SAVE_ALL_SYNC __LC_SVC_OLD_PSW,__LC_SAVE_AREA |
505 | CREATE_STACK_FRAME __LC_SVC_OLD_PSW,__LC_SAVE_AREA | 536 | CREATE_STACK_FRAME __LC_SVC_OLD_PSW,__LC_SAVE_AREA |
506 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING | 537 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING |
507 | tm SP_PSW+1(%r15),0x01 # interrupting from user ? | ||
508 | jz pgm_no_vtime3 | ||
509 | UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER | 538 | UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER |
510 | UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER | 539 | UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER |
511 | mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER | 540 | mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER |
512 | pgm_no_vtime3: | ||
513 | #endif | 541 | #endif |
514 | llgh %r7,__LC_SVC_INT_CODE # get svc number from lowcore | 542 | llgh %r7,__LC_SVC_INT_CODE # get svc number from lowcore |
515 | lg %r9,__LC_THREAD_INFO # load pointer to thread_info struct | 543 | lg %r9,__LC_THREAD_INFO # load pointer to thread_info struct |
@@ -529,7 +557,7 @@ kernel_per: | |||
529 | lhi %r0,__LC_PGM_OLD_PSW | 557 | lhi %r0,__LC_PGM_OLD_PSW |
530 | sth %r0,SP_TRAP(%r15) # set trap indication to pgm check | 558 | sth %r0,SP_TRAP(%r15) # set trap indication to pgm check |
531 | la %r2,SP_PTREGS(%r15) # address of register-save area | 559 | la %r2,SP_PTREGS(%r15) # address of register-save area |
532 | larl %r14,sysc_leave # load adr. of system ret, no work | 560 | larl %r14,sysc_restore # load adr. of system ret, no work |
533 | jg do_single_step # branch to do_single_step | 561 | jg do_single_step # branch to do_single_step |
534 | 562 | ||
535 | /* | 563 | /* |
@@ -554,26 +582,38 @@ io_no_vtime: | |||
554 | TRACE_IRQS_OFF | 582 | TRACE_IRQS_OFF |
555 | la %r2,SP_PTREGS(%r15) # address of register-save area | 583 | la %r2,SP_PTREGS(%r15) # address of register-save area |
556 | brasl %r14,do_IRQ # call standard irq handler | 584 | brasl %r14,do_IRQ # call standard irq handler |
557 | TRACE_IRQS_ON | ||
558 | |||
559 | io_return: | 585 | io_return: |
560 | tm SP_PSW+1(%r15),0x01 # returning to user ? | 586 | tm SP_PSW+1(%r15),0x01 # returning to user ? |
561 | #ifdef CONFIG_PREEMPT | 587 | #ifdef CONFIG_PREEMPT |
562 | jno io_preempt # no -> check for preemptive scheduling | 588 | jno io_preempt # no -> check for preemptive scheduling |
563 | #else | 589 | #else |
564 | jno io_leave # no-> skip resched & signal | 590 | jno io_restore # no-> skip resched & signal |
565 | #endif | 591 | #endif |
566 | tm __TI_flags+7(%r9),_TIF_WORK_INT | 592 | tm __TI_flags+7(%r9),_TIF_WORK_INT |
567 | jnz io_work # there is work to do (signals etc.) | 593 | jnz io_work # there is work to do (signals etc.) |
594 | io_restore: | ||
595 | #ifdef CONFIG_TRACE_IRQFLAGS | ||
596 | larl %r1,io_restore_trace_psw | ||
597 | lpswe 0(%r1) | ||
598 | io_restore_trace: | ||
599 | TRACE_IRQS_CHECK | ||
568 | LOCKDEP_SYS_EXIT | 600 | LOCKDEP_SYS_EXIT |
601 | #endif | ||
569 | io_leave: | 602 | io_leave: |
570 | RESTORE_ALL __LC_RETURN_PSW,0 | 603 | RESTORE_ALL __LC_RETURN_PSW,0 |
571 | io_done: | 604 | io_done: |
572 | 605 | ||
606 | #ifdef CONFIG_TRACE_IRQFLAGS | ||
607 | .align 8 | ||
608 | .globl io_restore_trace_psw | ||
609 | io_restore_trace_psw: | ||
610 | .quad 0, io_restore_trace | ||
611 | #endif | ||
612 | |||
573 | #ifdef CONFIG_PREEMPT | 613 | #ifdef CONFIG_PREEMPT |
574 | io_preempt: | 614 | io_preempt: |
575 | icm %r0,15,__TI_precount(%r9) | 615 | icm %r0,15,__TI_precount(%r9) |
576 | jnz io_leave | 616 | jnz io_restore |
577 | # switch to kernel stack | 617 | # switch to kernel stack |
578 | lg %r1,SP_R15(%r15) | 618 | lg %r1,SP_R15(%r15) |
579 | aghi %r1,-SP_SIZE | 619 | aghi %r1,-SP_SIZE |
@@ -582,12 +622,14 @@ io_preempt: | |||
582 | lgr %r15,%r1 | 622 | lgr %r15,%r1 |
583 | io_resume_loop: | 623 | io_resume_loop: |
584 | tm __TI_flags+7(%r9),_TIF_NEED_RESCHED | 624 | tm __TI_flags+7(%r9),_TIF_NEED_RESCHED |
585 | jno io_leave | 625 | jno io_restore |
586 | larl %r1,.Lc_pactive | 626 | larl %r1,.Lc_pactive |
587 | mvc __TI_precount(4,%r9),0(%r1) | 627 | mvc __TI_precount(4,%r9),0(%r1) |
628 | TRACE_IRQS_ON | ||
588 | stosm __SF_EMPTY(%r15),0x03 # reenable interrupts | 629 | stosm __SF_EMPTY(%r15),0x03 # reenable interrupts |
589 | brasl %r14,schedule # call schedule | 630 | brasl %r14,schedule # call schedule |
590 | stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts | 631 | stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts |
632 | TRACE_IRQS_OFF | ||
591 | xc __TI_precount(4,%r9),__TI_precount(%r9) | 633 | xc __TI_precount(4,%r9),__TI_precount(%r9) |
592 | j io_resume_loop | 634 | j io_resume_loop |
593 | #endif | 635 | #endif |
@@ -613,37 +655,39 @@ io_work_loop: | |||
613 | jo io_reschedule | 655 | jo io_reschedule |
614 | tm __TI_flags+7(%r9),(_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK) | 656 | tm __TI_flags+7(%r9),(_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK) |
615 | jnz io_sigpending | 657 | jnz io_sigpending |
616 | LOCKDEP_SYS_EXIT | 658 | j io_restore |
617 | j io_leave | 659 | io_work_done: |
618 | 660 | ||
619 | # | 661 | # |
620 | # _TIF_MCCK_PENDING is set, call handler | 662 | # _TIF_MCCK_PENDING is set, call handler |
621 | # | 663 | # |
622 | io_mcck_pending: | 664 | io_mcck_pending: |
623 | TRACE_IRQS_OFF | ||
624 | brasl %r14,s390_handle_mcck # TIF bit will be cleared by handler | 665 | brasl %r14,s390_handle_mcck # TIF bit will be cleared by handler |
625 | TRACE_IRQS_ON | ||
626 | j io_work_loop | 666 | j io_work_loop |
627 | 667 | ||
628 | # | 668 | # |
629 | # _TIF_NEED_RESCHED is set, call schedule | 669 | # _TIF_NEED_RESCHED is set, call schedule |
630 | # | 670 | # |
631 | io_reschedule: | 671 | io_reschedule: |
672 | TRACE_IRQS_ON | ||
632 | stosm __SF_EMPTY(%r15),0x03 # reenable interrupts | 673 | stosm __SF_EMPTY(%r15),0x03 # reenable interrupts |
633 | brasl %r14,schedule # call scheduler | 674 | brasl %r14,schedule # call scheduler |
634 | stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts | 675 | stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts |
676 | TRACE_IRQS_OFF | ||
635 | tm __TI_flags+7(%r9),_TIF_WORK_INT | 677 | tm __TI_flags+7(%r9),_TIF_WORK_INT |
636 | jz io_leave # there is no work to do | 678 | jz io_restore # there is no work to do |
637 | j io_work_loop | 679 | j io_work_loop |
638 | 680 | ||
639 | # | 681 | # |
640 | # _TIF_SIGPENDING or _TIF_RESTORE_SIGMASK is set, call do_signal | 682 | # _TIF_SIGPENDING or _TIF_RESTORE_SIGMASK is set, call do_signal |
641 | # | 683 | # |
642 | io_sigpending: | 684 | io_sigpending: |
685 | TRACE_IRQS_ON | ||
643 | stosm __SF_EMPTY(%r15),0x03 # reenable interrupts | 686 | stosm __SF_EMPTY(%r15),0x03 # reenable interrupts |
644 | la %r2,SP_PTREGS(%r15) # load pt_regs | 687 | la %r2,SP_PTREGS(%r15) # load pt_regs |
645 | brasl %r14,do_signal # call do_signal | 688 | brasl %r14,do_signal # call do_signal |
646 | stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts | 689 | stnsm __SF_EMPTY(%r15),0xfc # disable I/O and ext. interrupts |
690 | TRACE_IRQS_OFF | ||
647 | j io_work_loop | 691 | j io_work_loop |
648 | 692 | ||
649 | /* | 693 | /* |
@@ -669,7 +713,6 @@ ext_no_vtime: | |||
669 | la %r2,SP_PTREGS(%r15) # address of register-save area | 713 | la %r2,SP_PTREGS(%r15) # address of register-save area |
670 | llgh %r3,__LC_EXT_INT_CODE # get interruption code | 714 | llgh %r3,__LC_EXT_INT_CODE # get interruption code |
671 | brasl %r14,do_extint | 715 | brasl %r14,do_extint |
672 | TRACE_IRQS_ON | ||
673 | j io_return | 716 | j io_return |
674 | 717 | ||
675 | __critical_end: | 718 | __critical_end: |
@@ -824,15 +867,15 @@ cleanup_table_system_call: | |||
824 | cleanup_table_sysc_return: | 867 | cleanup_table_sysc_return: |
825 | .quad sysc_return, sysc_leave | 868 | .quad sysc_return, sysc_leave |
826 | cleanup_table_sysc_leave: | 869 | cleanup_table_sysc_leave: |
827 | .quad sysc_leave, sysc_work_loop | 870 | .quad sysc_leave, sysc_done |
828 | cleanup_table_sysc_work_loop: | 871 | cleanup_table_sysc_work_loop: |
829 | .quad sysc_work_loop, sysc_reschedule | 872 | .quad sysc_work_loop, sysc_work_done |
830 | cleanup_table_io_return: | 873 | cleanup_table_io_return: |
831 | .quad io_return, io_leave | 874 | .quad io_return, io_leave |
832 | cleanup_table_io_leave: | 875 | cleanup_table_io_leave: |
833 | .quad io_leave, io_done | 876 | .quad io_leave, io_done |
834 | cleanup_table_io_work_loop: | 877 | cleanup_table_io_work_loop: |
835 | .quad io_work_loop, io_mcck_pending | 878 | .quad io_work_loop, io_work_done |
836 | 879 | ||
837 | cleanup_critical: | 880 | cleanup_critical: |
838 | clc 8(8,%r12),BASED(cleanup_table_system_call) | 881 | clc 8(8,%r12),BASED(cleanup_table_system_call) |
@@ -901,8 +944,6 @@ cleanup_system_call: | |||
901 | cleanup_vtime: | 944 | cleanup_vtime: |
902 | clc __LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn+24) | 945 | clc __LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn+24) |
903 | jhe cleanup_stime | 946 | jhe cleanup_stime |
904 | tm SP_PSW+1(%r15),0x01 # interrupting from user ? | ||
905 | jz cleanup_novtime | ||
906 | UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER | 947 | UPDATE_VTIME __LC_EXIT_TIMER,__LC_SYNC_ENTER_TIMER,__LC_USER_TIMER |
907 | cleanup_stime: | 948 | cleanup_stime: |
908 | clc __LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn+32) | 949 | clc __LC_RETURN_PSW+8(8),BASED(cleanup_system_call_insn+32) |
@@ -910,7 +951,6 @@ cleanup_stime: | |||
910 | UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER | 951 | UPDATE_VTIME __LC_LAST_UPDATE_TIMER,__LC_EXIT_TIMER,__LC_SYSTEM_TIMER |
911 | cleanup_update: | 952 | cleanup_update: |
912 | mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER | 953 | mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER |
913 | cleanup_novtime: | ||
914 | #endif | 954 | #endif |
915 | mvc __LC_RETURN_PSW+8(8),BASED(cleanup_table_system_call+8) | 955 | mvc __LC_RETURN_PSW+8(8),BASED(cleanup_table_system_call+8) |
916 | la %r12,__LC_RETURN_PSW | 956 | la %r12,__LC_RETURN_PSW |
@@ -949,10 +989,10 @@ cleanup_sysc_leave: | |||
949 | 2: la %r12,__LC_RETURN_PSW | 989 | 2: la %r12,__LC_RETURN_PSW |
950 | br %r14 | 990 | br %r14 |
951 | cleanup_sysc_leave_insn: | 991 | cleanup_sysc_leave_insn: |
992 | .quad sysc_done - 4 | ||
952 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING | 993 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING |
953 | .quad sysc_leave + 16 | 994 | .quad sysc_done - 8 |
954 | #endif | 995 | #endif |
955 | .quad sysc_leave + 12 | ||
956 | 996 | ||
957 | cleanup_io_return: | 997 | cleanup_io_return: |
958 | mvc __LC_RETURN_PSW(8),0(%r12) | 998 | mvc __LC_RETURN_PSW(8),0(%r12) |
@@ -979,10 +1019,10 @@ cleanup_io_leave: | |||
979 | 2: la %r12,__LC_RETURN_PSW | 1019 | 2: la %r12,__LC_RETURN_PSW |
980 | br %r14 | 1020 | br %r14 |
981 | cleanup_io_leave_insn: | 1021 | cleanup_io_leave_insn: |
1022 | .quad io_done - 4 | ||
982 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING | 1023 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING |
983 | .quad io_leave + 20 | 1024 | .quad io_done - 8 |
984 | #endif | 1025 | #endif |
985 | .quad io_leave + 16 | ||
986 | 1026 | ||
987 | /* | 1027 | /* |
988 | * Integer constants | 1028 | * Integer constants |