diff options
Diffstat (limited to 'arch/ia64/kernel/ivt.S')
-rw-r--r-- | arch/ia64/kernel/ivt.S | 198 |
1 files changed, 128 insertions, 70 deletions
diff --git a/arch/ia64/kernel/ivt.S b/arch/ia64/kernel/ivt.S index 2bc085a73e30..3bb3a13c4047 100644 --- a/arch/ia64/kernel/ivt.S +++ b/arch/ia64/kernel/ivt.S | |||
@@ -1,7 +1,7 @@ | |||
1 | /* | 1 | /* |
2 | * arch/ia64/kernel/ivt.S | 2 | * arch/ia64/kernel/ivt.S |
3 | * | 3 | * |
4 | * Copyright (C) 1998-2001, 2003 Hewlett-Packard Co | 4 | * Copyright (C) 1998-2001, 2003, 2005 Hewlett-Packard Co |
5 | * Stephane Eranian <eranian@hpl.hp.com> | 5 | * Stephane Eranian <eranian@hpl.hp.com> |
6 | * David Mosberger <davidm@hpl.hp.com> | 6 | * David Mosberger <davidm@hpl.hp.com> |
7 | * Copyright (C) 2000, 2002-2003 Intel Co | 7 | * Copyright (C) 2000, 2002-2003 Intel Co |
@@ -692,82 +692,118 @@ ENTRY(break_fault) | |||
692 | * to prevent leaking bits from kernel to user level. | 692 | * to prevent leaking bits from kernel to user level. |
693 | */ | 693 | */ |
694 | DBG_FAULT(11) | 694 | DBG_FAULT(11) |
695 | mov r16=IA64_KR(CURRENT) // r16 = current task; 12 cycle read lat. | 695 | mov.m r16=IA64_KR(CURRENT) // M2 r16 <- current task (12 cyc) |
696 | mov r17=cr.iim | 696 | mov r29=cr.ipsr // M2 (12 cyc) |
697 | mov r18=__IA64_BREAK_SYSCALL | 697 | mov r31=pr // I0 (2 cyc) |
698 | mov r21=ar.fpsr | 698 | |
699 | mov r29=cr.ipsr | 699 | mov r17=cr.iim // M2 (2 cyc) |
700 | mov r19=b6 | 700 | mov.m r27=ar.rsc // M2 (12 cyc) |
701 | mov r25=ar.unat | 701 | mov r18=__IA64_BREAK_SYSCALL // A |
702 | mov r27=ar.rsc | 702 | |
703 | mov r26=ar.pfs | 703 | mov.m ar.rsc=0 // M2 |
704 | mov r28=cr.iip | 704 | mov.m r21=ar.fpsr // M2 (12 cyc) |
705 | mov r31=pr // prepare to save predicates | 705 | mov r19=b6 // I0 (2 cyc) |
706 | mov r20=r1 | 706 | ;; |
707 | ;; | 707 | mov.m r23=ar.bspstore // M2 (12 cyc) |
708 | mov.m r24=ar.rnat // M2 (5 cyc) | ||
709 | mov.i r26=ar.pfs // I0 (2 cyc) | ||
710 | |||
711 | invala // M0|1 | ||
712 | nop.m 0 // M | ||
713 | mov r20=r1 // A save r1 | ||
714 | |||
715 | nop.m 0 | ||
716 | movl r30=sys_call_table // X | ||
717 | |||
718 | mov r28=cr.iip // M2 (2 cyc) | ||
719 | cmp.eq p0,p7=r18,r17 // I0 is this a system call? | ||
720 | (p7) br.cond.spnt non_syscall // B no -> | ||
721 | // | ||
722 | // From this point on, we are definitely on the syscall-path | ||
723 | // and we can use (non-banked) scratch registers. | ||
724 | // | ||
725 | /////////////////////////////////////////////////////////////////////// | ||
726 | mov r1=r16 // A move task-pointer to "addl"-addressable reg | ||
727 | mov r2=r16 // A setup r2 for ia64_syscall_setup | ||
728 | add r9=TI_FLAGS+IA64_TASK_SIZE,r16 // A r9 = ¤t_thread_info()->flags | ||
729 | |||
708 | adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 | 730 | adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 |
709 | cmp.eq p0,p7=r18,r17 // is this a system call? (p7 <- false, if so) | 731 | adds r15=-1024,r15 // A subtract 1024 from syscall number |
710 | (p7) br.cond.spnt non_syscall | 732 | mov r3=NR_syscalls - 1 |
711 | ;; | 733 | ;; |
712 | ld1 r17=[r16] // load current->thread.on_ustack flag | 734 | ld1.bias r17=[r16] // M0|1 r17 = current->thread.on_ustack flag |
713 | st1 [r16]=r0 // clear current->thread.on_ustack flag | 735 | ld4 r9=[r9] // M0|1 r9 = current_thread_info()->flags |
714 | add r1=-IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 // set r1 for MINSTATE_START_SAVE_MIN_VIRT | 736 | extr.u r8=r29,41,2 // I0 extract ei field from cr.ipsr |
737 | |||
738 | shladd r30=r15,3,r30 // A r30 = sys_call_table + 8*(syscall-1024) | ||
739 | addl r22=IA64_RBS_OFFSET,r1 // A compute base of RBS | ||
740 | cmp.leu p6,p7=r15,r3 // A syscall number in range? | ||
715 | ;; | 741 | ;; |
716 | invala | ||
717 | 742 | ||
718 | /* adjust return address so we skip over the break instruction: */ | 743 | lfetch.fault.excl.nt1 [r22] // M0|1 prefetch RBS |
744 | (p6) ld8 r30=[r30] // M0|1 load address of syscall entry point | ||
745 | tnat.nz.or p7,p0=r15 // I0 is syscall nr a NaT? | ||
719 | 746 | ||
720 | extr.u r8=r29,41,2 // extract ei field from cr.ipsr | 747 | mov.m ar.bspstore=r22 // M2 switch to kernel RBS |
721 | ;; | 748 | cmp.eq p8,p9=2,r8 // A isr.ei==2? |
722 | cmp.eq p6,p7=2,r8 // isr.ei==2? | ||
723 | mov r2=r1 // setup r2 for ia64_syscall_setup | ||
724 | ;; | ||
725 | (p6) mov r8=0 // clear ei to 0 | ||
726 | (p6) adds r28=16,r28 // switch cr.iip to next bundle cr.ipsr.ei wrapped | ||
727 | (p7) adds r8=1,r8 // increment ei to next slot | ||
728 | ;; | ||
729 | cmp.eq pKStk,pUStk=r0,r17 // are we in kernel mode already? | ||
730 | dep r29=r8,r29,41,2 // insert new ei into cr.ipsr | ||
731 | ;; | 749 | ;; |
732 | 750 | ||
733 | // switch from user to kernel RBS: | 751 | (p8) mov r8=0 // A clear ei to 0 |
734 | MINSTATE_START_SAVE_MIN_VIRT | 752 | (p7) movl r30=sys_ni_syscall // X |
735 | br.call.sptk.many b7=ia64_syscall_setup | ||
736 | ;; | ||
737 | MINSTATE_END_SAVE_MIN_VIRT // switch to bank 1 | ||
738 | ssm psr.ic | PSR_DEFAULT_BITS | ||
739 | ;; | ||
740 | srlz.i // guarantee that interruption collection is on | ||
741 | mov r3=NR_syscalls - 1 | ||
742 | ;; | ||
743 | (p15) ssm psr.i // restore psr.i | ||
744 | // p10==true means out registers are more than 8 or r15's Nat is true | ||
745 | (p10) br.cond.spnt.many ia64_ret_from_syscall | ||
746 | ;; | ||
747 | movl r16=sys_call_table | ||
748 | 753 | ||
749 | adds r15=-1024,r15 // r15 contains the syscall number---subtract 1024 | 754 | (p8) adds r28=16,r28 // A switch cr.iip to next bundle |
750 | movl r2=ia64_ret_from_syscall | 755 | (p9) adds r8=1,r8 // A increment ei to next slot |
751 | ;; | 756 | nop.i 0 |
752 | shladd r20=r15,3,r16 // r20 = sys_call_table + 8*(syscall-1024) | ||
753 | cmp.leu p6,p7=r15,r3 // (syscall > 0 && syscall < 1024 + NR_syscalls) ? | ||
754 | mov rp=r2 // set the real return addr | ||
755 | ;; | 757 | ;; |
756 | (p6) ld8 r20=[r20] // load address of syscall entry point | ||
757 | (p7) movl r20=sys_ni_syscall | ||
758 | 758 | ||
759 | add r2=TI_FLAGS+IA64_TASK_SIZE,r13 | 759 | mov.m r25=ar.unat // M2 (5 cyc) |
760 | ;; | 760 | dep r29=r8,r29,41,2 // I0 insert new ei into cr.ipsr |
761 | ld4 r2=[r2] // r2 = current_thread_info()->flags | 761 | adds r15=1024,r15 // A restore original syscall number |
762 | ;; | 762 | // |
763 | and r2=_TIF_SYSCALL_TRACEAUDIT,r2 // mask trace or audit | 763 | // If any of the above loads miss in L1D, we'll stall here until |
764 | // the data arrives. | ||
765 | // | ||
766 | /////////////////////////////////////////////////////////////////////// | ||
767 | st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag | ||
768 | mov b6=r30 // I0 setup syscall handler branch reg early | ||
769 | cmp.eq pKStk,pUStk=r0,r17 // A were we on kernel stacks already? | ||
770 | |||
771 | and r9=_TIF_SYSCALL_TRACEAUDIT,r9 // A mask trace or audit | ||
772 | mov r18=ar.bsp // M2 (12 cyc) | ||
773 | (pKStk) br.cond.spnt .break_fixup // B we're already in kernel-mode -- fix up RBS | ||
774 | ;; | ||
775 | .back_from_break_fixup: | ||
776 | (pUStk) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1 // A compute base of memory stack | ||
777 | cmp.eq p14,p0=r9,r0 // A are syscalls being traced/audited? | ||
778 | br.call.sptk.many b7=ia64_syscall_setup // B | ||
779 | 1: | ||
780 | mov ar.rsc=0x3 // M2 set eager mode, pl 0, LE, loadrs=0 | ||
781 | nop 0 | ||
782 | bsw.1 // B (6 cyc) regs are saved, switch to bank 1 | ||
764 | ;; | 783 | ;; |
765 | cmp.eq p8,p0=r2,r0 | 784 | |
766 | mov b6=r20 | 785 | ssm psr.ic | PSR_DEFAULT_BITS // M2 now it's safe to re-enable intr.-collection |
786 | movl r3=ia64_ret_from_syscall // X | ||
767 | ;; | 787 | ;; |
768 | (p8) br.call.sptk.many b6=b6 // ignore this return addr | 788 | |
769 | br.cond.sptk ia64_trace_syscall | 789 | srlz.i // M0 ensure interruption collection is on |
790 | mov rp=r3 // I0 set the real return addr | ||
791 | (p10) br.cond.spnt.many ia64_ret_from_syscall // B return if bad call-frame or r15 is a NaT | ||
792 | |||
793 | (p15) ssm psr.i // M2 restore psr.i | ||
794 | (p14) br.call.sptk.many b6=b6 // B invoke syscall-handker (ignore return addr) | ||
795 | br.cond.spnt.many ia64_trace_syscall // B do syscall-tracing thingamagic | ||
770 | // NOT REACHED | 796 | // NOT REACHED |
797 | /////////////////////////////////////////////////////////////////////// | ||
798 | // On entry, we optimistically assumed that we're coming from user-space. | ||
799 | // For the rare cases where a system-call is done from within the kernel, | ||
800 | // we fix things up at this point: | ||
801 | .break_fixup: | ||
802 | add r1=-IA64_PT_REGS_SIZE,sp // A allocate space for pt_regs structure | ||
803 | mov ar.rnat=r24 // M2 restore kernel's AR.RNAT | ||
804 | ;; | ||
805 | mov ar.bspstore=r23 // M2 restore kernel's AR.BSPSTORE | ||
806 | br.cond.sptk .back_from_break_fixup | ||
771 | END(break_fault) | 807 | END(break_fault) |
772 | 808 | ||
773 | .org ia64_ivt+0x3000 | 809 | .org ia64_ivt+0x3000 |
@@ -842,8 +878,6 @@ END(interrupt) | |||
842 | * - r31: saved pr | 878 | * - r31: saved pr |
843 | * - b0: original contents (to be saved) | 879 | * - b0: original contents (to be saved) |
844 | * On exit: | 880 | * On exit: |
845 | * - executing on bank 1 registers | ||
846 | * - psr.ic enabled, interrupts restored | ||
847 | * - p10: TRUE if syscall is invoked with more than 8 out | 881 | * - p10: TRUE if syscall is invoked with more than 8 out |
848 | * registers or r15's Nat is true | 882 | * registers or r15's Nat is true |
849 | * - r1: kernel's gp | 883 | * - r1: kernel's gp |
@@ -851,8 +885,11 @@ END(interrupt) | |||
851 | * - r8: -EINVAL if p10 is true | 885 | * - r8: -EINVAL if p10 is true |
852 | * - r12: points to kernel stack | 886 | * - r12: points to kernel stack |
853 | * - r13: points to current task | 887 | * - r13: points to current task |
888 | * - r14: preserved (same as on entry) | ||
889 | * - p13: preserved | ||
854 | * - p15: TRUE if interrupts need to be re-enabled | 890 | * - p15: TRUE if interrupts need to be re-enabled |
855 | * - ar.fpsr: set to kernel settings | 891 | * - ar.fpsr: set to kernel settings |
892 | * - b6: preserved (same as on entry) | ||
856 | */ | 893 | */ |
857 | GLOBAL_ENTRY(ia64_syscall_setup) | 894 | GLOBAL_ENTRY(ia64_syscall_setup) |
858 | #if PT(B6) != 0 | 895 | #if PT(B6) != 0 |
@@ -920,10 +957,10 @@ GLOBAL_ENTRY(ia64_syscall_setup) | |||
920 | (p13) mov in5=-1 | 957 | (p13) mov in5=-1 |
921 | ;; | 958 | ;; |
922 | st8 [r16]=r21,PT(R8)-PT(AR_FPSR) // save ar.fpsr | 959 | st8 [r16]=r21,PT(R8)-PT(AR_FPSR) // save ar.fpsr |
923 | tnat.nz p14,p0=in6 | 960 | tnat.nz p13,p0=in6 |
924 | cmp.lt p10,p9=r11,r8 // frame size can't be more than local+8 | 961 | cmp.lt p10,p9=r11,r8 // frame size can't be more than local+8 |
925 | ;; | 962 | ;; |
926 | stf8 [r16]=f1 // ensure pt_regs.r8 != 0 (see handle_syscall_error) | 963 | mov r8=1 |
927 | (p9) tnat.nz p10,p0=r15 | 964 | (p9) tnat.nz p10,p0=r15 |
928 | adds r12=-16,r1 // switch to kernel memory stack (with 16 bytes of scratch) | 965 | adds r12=-16,r1 // switch to kernel memory stack (with 16 bytes of scratch) |
929 | 966 | ||
@@ -934,9 +971,9 @@ GLOBAL_ENTRY(ia64_syscall_setup) | |||
934 | mov r13=r2 // establish `current' | 971 | mov r13=r2 // establish `current' |
935 | movl r1=__gp // establish kernel global pointer | 972 | movl r1=__gp // establish kernel global pointer |
936 | ;; | 973 | ;; |
937 | (p14) mov in6=-1 | 974 | st8 [r16]=r8 // ensure pt_regs.r8 != 0 (see handle_syscall_error) |
975 | (p13) mov in6=-1 | ||
938 | (p8) mov in7=-1 | 976 | (p8) mov in7=-1 |
939 | nop.i 0 | ||
940 | 977 | ||
941 | cmp.eq pSys,pNonSys=r0,r0 // set pSys=1, pNonSys=0 | 978 | cmp.eq pSys,pNonSys=r0,r0 // set pSys=1, pNonSys=0 |
942 | movl r17=FPSR_DEFAULT | 979 | movl r17=FPSR_DEFAULT |
@@ -1007,6 +1044,8 @@ END(dispatch_illegal_op_fault) | |||
1007 | FAULT(17) | 1044 | FAULT(17) |
1008 | 1045 | ||
1009 | ENTRY(non_syscall) | 1046 | ENTRY(non_syscall) |
1047 | mov ar.rsc=r27 // restore ar.rsc before SAVE_MIN_WITH_COVER | ||
1048 | ;; | ||
1010 | SAVE_MIN_WITH_COVER | 1049 | SAVE_MIN_WITH_COVER |
1011 | 1050 | ||
1012 | // There is no particular reason for this code to be here, other than that | 1051 | // There is no particular reason for this code to be here, other than that |
@@ -1204,6 +1243,25 @@ END(disabled_fp_reg) | |||
1204 | // 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50) | 1243 | // 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50) |
1205 | ENTRY(nat_consumption) | 1244 | ENTRY(nat_consumption) |
1206 | DBG_FAULT(26) | 1245 | DBG_FAULT(26) |
1246 | |||
1247 | mov r16=cr.ipsr | ||
1248 | mov r17=cr.isr | ||
1249 | mov r31=pr // save PR | ||
1250 | ;; | ||
1251 | and r18=0xf,r17 // r18 = cr.ipsr.code{3:0} | ||
1252 | tbit.z p6,p0=r17,IA64_ISR_NA_BIT | ||
1253 | ;; | ||
1254 | cmp.ne.or p6,p0=IA64_ISR_CODE_LFETCH,r18 | ||
1255 | dep r16=-1,r16,IA64_PSR_ED_BIT,1 | ||
1256 | (p6) br.cond.spnt 1f // branch if (cr.ispr.na == 0 || cr.ipsr.code{3:0} != LFETCH) | ||
1257 | ;; | ||
1258 | mov cr.ipsr=r16 // set cr.ipsr.na | ||
1259 | mov pr=r31,-1 | ||
1260 | ;; | ||
1261 | rfi | ||
1262 | |||
1263 | 1: mov pr=r31,-1 | ||
1264 | ;; | ||
1207 | FAULT(26) | 1265 | FAULT(26) |
1208 | END(nat_consumption) | 1266 | END(nat_consumption) |
1209 | 1267 | ||