aboutsummaryrefslogtreecommitdiffstats
path: root/arch/ia64/kernel/ivt.S
diff options
context:
space:
mode:
Diffstat (limited to 'arch/ia64/kernel/ivt.S')
-rw-r--r--arch/ia64/kernel/ivt.S198
1 files changed, 128 insertions, 70 deletions
diff --git a/arch/ia64/kernel/ivt.S b/arch/ia64/kernel/ivt.S
index 2bc085a73e30..3bb3a13c4047 100644
--- a/arch/ia64/kernel/ivt.S
+++ b/arch/ia64/kernel/ivt.S
@@ -1,7 +1,7 @@
1/* 1/*
2 * arch/ia64/kernel/ivt.S 2 * arch/ia64/kernel/ivt.S
3 * 3 *
4 * Copyright (C) 1998-2001, 2003 Hewlett-Packard Co 4 * Copyright (C) 1998-2001, 2003, 2005 Hewlett-Packard Co
5 * Stephane Eranian <eranian@hpl.hp.com> 5 * Stephane Eranian <eranian@hpl.hp.com>
6 * David Mosberger <davidm@hpl.hp.com> 6 * David Mosberger <davidm@hpl.hp.com>
7 * Copyright (C) 2000, 2002-2003 Intel Co 7 * Copyright (C) 2000, 2002-2003 Intel Co
@@ -692,82 +692,118 @@ ENTRY(break_fault)
692 * to prevent leaking bits from kernel to user level. 692 * to prevent leaking bits from kernel to user level.
693 */ 693 */
694 DBG_FAULT(11) 694 DBG_FAULT(11)
695 mov r16=IA64_KR(CURRENT) // r16 = current task; 12 cycle read lat. 695 mov.m r16=IA64_KR(CURRENT) // M2 r16 <- current task (12 cyc)
696 mov r17=cr.iim 696 mov r29=cr.ipsr // M2 (12 cyc)
697 mov r18=__IA64_BREAK_SYSCALL 697 mov r31=pr // I0 (2 cyc)
698 mov r21=ar.fpsr 698
699 mov r29=cr.ipsr 699 mov r17=cr.iim // M2 (2 cyc)
700 mov r19=b6 700 mov.m r27=ar.rsc // M2 (12 cyc)
701 mov r25=ar.unat 701 mov r18=__IA64_BREAK_SYSCALL // A
702 mov r27=ar.rsc 702
703 mov r26=ar.pfs 703 mov.m ar.rsc=0 // M2
704 mov r28=cr.iip 704 mov.m r21=ar.fpsr // M2 (12 cyc)
705 mov r31=pr // prepare to save predicates 705 mov r19=b6 // I0 (2 cyc)
706 mov r20=r1 706 ;;
707 ;; 707 mov.m r23=ar.bspstore // M2 (12 cyc)
708 mov.m r24=ar.rnat // M2 (5 cyc)
709 mov.i r26=ar.pfs // I0 (2 cyc)
710
711 invala // M0|1
712 nop.m 0 // M
713 mov r20=r1 // A save r1
714
715 nop.m 0
716 movl r30=sys_call_table // X
717
718 mov r28=cr.iip // M2 (2 cyc)
719 cmp.eq p0,p7=r18,r17 // I0 is this a system call?
720(p7) br.cond.spnt non_syscall // B no ->
721 //
722 // From this point on, we are definitely on the syscall-path
723 // and we can use (non-banked) scratch registers.
724 //
725///////////////////////////////////////////////////////////////////////
726 mov r1=r16 // A move task-pointer to "addl"-addressable reg
727 mov r2=r16 // A setup r2 for ia64_syscall_setup
728 add r9=TI_FLAGS+IA64_TASK_SIZE,r16 // A r9 = &current_thread_info()->flags
729
708 adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 730 adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16
709 cmp.eq p0,p7=r18,r17 // is this a system call? (p7 <- false, if so) 731 adds r15=-1024,r15 // A subtract 1024 from syscall number
710(p7) br.cond.spnt non_syscall 732 mov r3=NR_syscalls - 1
711 ;; 733 ;;
712 ld1 r17=[r16] // load current->thread.on_ustack flag 734 ld1.bias r17=[r16] // M0|1 r17 = current->thread.on_ustack flag
713 st1 [r16]=r0 // clear current->thread.on_ustack flag 735 ld4 r9=[r9] // M0|1 r9 = current_thread_info()->flags
714 add r1=-IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 // set r1 for MINSTATE_START_SAVE_MIN_VIRT 736 extr.u r8=r29,41,2 // I0 extract ei field from cr.ipsr
737
738 shladd r30=r15,3,r30 // A r30 = sys_call_table + 8*(syscall-1024)
739 addl r22=IA64_RBS_OFFSET,r1 // A compute base of RBS
740 cmp.leu p6,p7=r15,r3 // A syscall number in range?
715 ;; 741 ;;
716 invala
717 742
718 /* adjust return address so we skip over the break instruction: */ 743 lfetch.fault.excl.nt1 [r22] // M0|1 prefetch RBS
744(p6) ld8 r30=[r30] // M0|1 load address of syscall entry point
745 tnat.nz.or p7,p0=r15 // I0 is syscall nr a NaT?
719 746
720 extr.u r8=r29,41,2 // extract ei field from cr.ipsr 747 mov.m ar.bspstore=r22 // M2 switch to kernel RBS
721 ;; 748 cmp.eq p8,p9=2,r8 // A isr.ei==2?
722 cmp.eq p6,p7=2,r8 // isr.ei==2?
723 mov r2=r1 // setup r2 for ia64_syscall_setup
724 ;;
725(p6) mov r8=0 // clear ei to 0
726(p6) adds r28=16,r28 // switch cr.iip to next bundle cr.ipsr.ei wrapped
727(p7) adds r8=1,r8 // increment ei to next slot
728 ;;
729 cmp.eq pKStk,pUStk=r0,r17 // are we in kernel mode already?
730 dep r29=r8,r29,41,2 // insert new ei into cr.ipsr
731 ;; 749 ;;
732 750
733 // switch from user to kernel RBS: 751(p8) mov r8=0 // A clear ei to 0
734 MINSTATE_START_SAVE_MIN_VIRT 752(p7) movl r30=sys_ni_syscall // X
735 br.call.sptk.many b7=ia64_syscall_setup
736 ;;
737 MINSTATE_END_SAVE_MIN_VIRT // switch to bank 1
738 ssm psr.ic | PSR_DEFAULT_BITS
739 ;;
740 srlz.i // guarantee that interruption collection is on
741 mov r3=NR_syscalls - 1
742 ;;
743(p15) ssm psr.i // restore psr.i
744 // p10==true means out registers are more than 8 or r15's Nat is true
745(p10) br.cond.spnt.many ia64_ret_from_syscall
746 ;;
747 movl r16=sys_call_table
748 753
749 adds r15=-1024,r15 // r15 contains the syscall number---subtract 1024 754(p8) adds r28=16,r28 // A switch cr.iip to next bundle
750 movl r2=ia64_ret_from_syscall 755(p9) adds r8=1,r8 // A increment ei to next slot
751 ;; 756 nop.i 0
752 shladd r20=r15,3,r16 // r20 = sys_call_table + 8*(syscall-1024)
753 cmp.leu p6,p7=r15,r3 // (syscall > 0 && syscall < 1024 + NR_syscalls) ?
754 mov rp=r2 // set the real return addr
755 ;; 757 ;;
756(p6) ld8 r20=[r20] // load address of syscall entry point
757(p7) movl r20=sys_ni_syscall
758 758
759 add r2=TI_FLAGS+IA64_TASK_SIZE,r13 759 mov.m r25=ar.unat // M2 (5 cyc)
760 ;; 760 dep r29=r8,r29,41,2 // I0 insert new ei into cr.ipsr
761 ld4 r2=[r2] // r2 = current_thread_info()->flags 761 adds r15=1024,r15 // A restore original syscall number
762 ;; 762 //
763 and r2=_TIF_SYSCALL_TRACEAUDIT,r2 // mask trace or audit 763 // If any of the above loads miss in L1D, we'll stall here until
764 // the data arrives.
765 //
766///////////////////////////////////////////////////////////////////////
767 st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag
768 mov b6=r30 // I0 setup syscall handler branch reg early
769 cmp.eq pKStk,pUStk=r0,r17 // A were we on kernel stacks already?
770
771 and r9=_TIF_SYSCALL_TRACEAUDIT,r9 // A mask trace or audit
772 mov r18=ar.bsp // M2 (12 cyc)
773(pKStk) br.cond.spnt .break_fixup // B we're already in kernel-mode -- fix up RBS
774 ;;
775.back_from_break_fixup:
776(pUStk) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1 // A compute base of memory stack
777 cmp.eq p14,p0=r9,r0 // A are syscalls being traced/audited?
778 br.call.sptk.many b7=ia64_syscall_setup // B
7791:
780 mov ar.rsc=0x3 // M2 set eager mode, pl 0, LE, loadrs=0
781 nop 0
782 bsw.1 // B (6 cyc) regs are saved, switch to bank 1
764 ;; 783 ;;
765 cmp.eq p8,p0=r2,r0 784
766 mov b6=r20 785 ssm psr.ic | PSR_DEFAULT_BITS // M2 now it's safe to re-enable intr.-collection
786 movl r3=ia64_ret_from_syscall // X
767 ;; 787 ;;
768(p8) br.call.sptk.many b6=b6 // ignore this return addr 788
769 br.cond.sptk ia64_trace_syscall 789 srlz.i // M0 ensure interruption collection is on
790 mov rp=r3 // I0 set the real return addr
791(p10) br.cond.spnt.many ia64_ret_from_syscall // B return if bad call-frame or r15 is a NaT
792
793(p15) ssm psr.i // M2 restore psr.i
794(p14) br.call.sptk.many b6=b6 // B invoke syscall-handker (ignore return addr)
795 br.cond.spnt.many ia64_trace_syscall // B do syscall-tracing thingamagic
770 // NOT REACHED 796 // NOT REACHED
797///////////////////////////////////////////////////////////////////////
798 // On entry, we optimistically assumed that we're coming from user-space.
799 // For the rare cases where a system-call is done from within the kernel,
800 // we fix things up at this point:
801.break_fixup:
802 add r1=-IA64_PT_REGS_SIZE,sp // A allocate space for pt_regs structure
803 mov ar.rnat=r24 // M2 restore kernel's AR.RNAT
804 ;;
805 mov ar.bspstore=r23 // M2 restore kernel's AR.BSPSTORE
806 br.cond.sptk .back_from_break_fixup
771END(break_fault) 807END(break_fault)
772 808
773 .org ia64_ivt+0x3000 809 .org ia64_ivt+0x3000
@@ -842,8 +878,6 @@ END(interrupt)
842 * - r31: saved pr 878 * - r31: saved pr
843 * - b0: original contents (to be saved) 879 * - b0: original contents (to be saved)
844 * On exit: 880 * On exit:
845 * - executing on bank 1 registers
846 * - psr.ic enabled, interrupts restored
847 * - p10: TRUE if syscall is invoked with more than 8 out 881 * - p10: TRUE if syscall is invoked with more than 8 out
848 * registers or r15's Nat is true 882 * registers or r15's Nat is true
849 * - r1: kernel's gp 883 * - r1: kernel's gp
@@ -851,8 +885,11 @@ END(interrupt)
851 * - r8: -EINVAL if p10 is true 885 * - r8: -EINVAL if p10 is true
852 * - r12: points to kernel stack 886 * - r12: points to kernel stack
853 * - r13: points to current task 887 * - r13: points to current task
888 * - r14: preserved (same as on entry)
889 * - p13: preserved
854 * - p15: TRUE if interrupts need to be re-enabled 890 * - p15: TRUE if interrupts need to be re-enabled
855 * - ar.fpsr: set to kernel settings 891 * - ar.fpsr: set to kernel settings
892 * - b6: preserved (same as on entry)
856 */ 893 */
857GLOBAL_ENTRY(ia64_syscall_setup) 894GLOBAL_ENTRY(ia64_syscall_setup)
858#if PT(B6) != 0 895#if PT(B6) != 0
@@ -920,10 +957,10 @@ GLOBAL_ENTRY(ia64_syscall_setup)
920(p13) mov in5=-1 957(p13) mov in5=-1
921 ;; 958 ;;
922 st8 [r16]=r21,PT(R8)-PT(AR_FPSR) // save ar.fpsr 959 st8 [r16]=r21,PT(R8)-PT(AR_FPSR) // save ar.fpsr
923 tnat.nz p14,p0=in6 960 tnat.nz p13,p0=in6
924 cmp.lt p10,p9=r11,r8 // frame size can't be more than local+8 961 cmp.lt p10,p9=r11,r8 // frame size can't be more than local+8
925 ;; 962 ;;
926 stf8 [r16]=f1 // ensure pt_regs.r8 != 0 (see handle_syscall_error) 963 mov r8=1
927(p9) tnat.nz p10,p0=r15 964(p9) tnat.nz p10,p0=r15
928 adds r12=-16,r1 // switch to kernel memory stack (with 16 bytes of scratch) 965 adds r12=-16,r1 // switch to kernel memory stack (with 16 bytes of scratch)
929 966
@@ -934,9 +971,9 @@ GLOBAL_ENTRY(ia64_syscall_setup)
934 mov r13=r2 // establish `current' 971 mov r13=r2 // establish `current'
935 movl r1=__gp // establish kernel global pointer 972 movl r1=__gp // establish kernel global pointer
936 ;; 973 ;;
937(p14) mov in6=-1 974 st8 [r16]=r8 // ensure pt_regs.r8 != 0 (see handle_syscall_error)
975(p13) mov in6=-1
938(p8) mov in7=-1 976(p8) mov in7=-1
939 nop.i 0
940 977
941 cmp.eq pSys,pNonSys=r0,r0 // set pSys=1, pNonSys=0 978 cmp.eq pSys,pNonSys=r0,r0 // set pSys=1, pNonSys=0
942 movl r17=FPSR_DEFAULT 979 movl r17=FPSR_DEFAULT
@@ -1007,6 +1044,8 @@ END(dispatch_illegal_op_fault)
1007 FAULT(17) 1044 FAULT(17)
1008 1045
1009ENTRY(non_syscall) 1046ENTRY(non_syscall)
1047 mov ar.rsc=r27 // restore ar.rsc before SAVE_MIN_WITH_COVER
1048 ;;
1010 SAVE_MIN_WITH_COVER 1049 SAVE_MIN_WITH_COVER
1011 1050
1012 // There is no particular reason for this code to be here, other than that 1051 // There is no particular reason for this code to be here, other than that
@@ -1204,6 +1243,25 @@ END(disabled_fp_reg)
1204// 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50) 1243// 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50)
1205ENTRY(nat_consumption) 1244ENTRY(nat_consumption)
1206 DBG_FAULT(26) 1245 DBG_FAULT(26)
1246
1247 mov r16=cr.ipsr
1248 mov r17=cr.isr
1249 mov r31=pr // save PR
1250 ;;
1251 and r18=0xf,r17 // r18 = cr.ipsr.code{3:0}
1252 tbit.z p6,p0=r17,IA64_ISR_NA_BIT
1253 ;;
1254 cmp.ne.or p6,p0=IA64_ISR_CODE_LFETCH,r18
1255 dep r16=-1,r16,IA64_PSR_ED_BIT,1
1256(p6) br.cond.spnt 1f // branch if (cr.ispr.na == 0 || cr.ipsr.code{3:0} != LFETCH)
1257 ;;
1258 mov cr.ipsr=r16 // set cr.ipsr.na
1259 mov pr=r31,-1
1260 ;;
1261 rfi
1262
12631: mov pr=r31,-1
1264 ;;
1207 FAULT(26) 1265 FAULT(26)
1208END(nat_consumption) 1266END(nat_consumption)
1209 1267