[IA64] pvops: paravirtualize entry.S

paravirtualize ia64_swtich_to, ia64_leave_syscall and ia64_leave_kernel.
They include sensitive or performance critical privileged instructions
so that they need paravirtualization.
To paravirtualize them by single source and multi compile
they are converted into indirect jump. And define each pv instances.

Cc: Keith Owens <kaos@ocs.com.au>
Cc: "Dong, Eddie" <eddie.dong@intel.com>
Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp>
Signed-off-by: Tony Luck <tony.luck@intel.com>

4 files changed, 152 insertions, 44 deletions

diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile
index 8b2524293eb4..cea91f17d44b 100644
--- a/arch/ia64/kernel/Makefile
+++ b/arch/ia64/kernel/Makefile
@@ -36,7 +36,7 @@ obj-$(CONFIG_PCI_MSI)		+= msi_ia64.o
 mca_recovery-y                  += mca_drv.o mca_drv_asm.o
 obj-$(CONFIG_IA64_MC_ERR_INJECT)+= err_inject.o
 
-obj-$(CONFIG_PARAVIRT)          += paravirt.o
+obj-$(CONFIG_PARAVIRT)          += paravirt.o paravirtentry.o
 
 obj-$(CONFIG_IA64_ESI)          += esi.o
 ifneq ($(CONFIG_IA64_ESI),)
diff --git a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S
index ca2bb95726de..56ab156c48ae 100644
--- a/arch/ia64/kernel/entry.S
+++ b/arch/ia64/kernel/entry.S
@@ -23,6 +23,11 @@
  * 11/07/2000
  */
 /*
+ * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
+ *                    VA Linux Systems Japan K.K.
+ *                    pv_ops.
+ */
+/*
  * Global (preserved) predicate usage on syscall entry/exit path:
  *
  *      pKStk:          See entry.h.
@@ -45,6 +50,7 @@
 
 #include "minstate.h"
 
+#ifdef __IA64_ASM_PARAVIRTUALIZED_NATIVE
         /*
          * execve() is special because in case of success, we need to
          * setup a null register window frame.
@@ -173,6 +179,7 @@ GLOBAL_ENTRY(sys_clone)
         mov rp=loc0
         br.ret.sptk.many rp
 END(sys_clone)
+#endif /* __IA64_ASM_PARAVIRTUALIZED_NATIVE */
 
 /*
  * prev_task <- ia64_switch_to(struct task_struct *next)
@@ -180,7 +187,7 @@ END(sys_clone)
  *      called.  The code starting at .map relies on this.  The rest of the code
  *      doesn't care about the interrupt masking status.
  */
-GLOBAL_ENTRY(ia64_switch_to)
+GLOBAL_ENTRY(__paravirt_switch_to)
         .prologue
         alloc r16=ar.pfs,1,0,0,0
         DO_SAVE_SWITCH_STACK
@@ -204,7 +211,7 @@ GLOBAL_ENTRY(ia64_switch_to)
         ;;
 .done:
         ld8 sp=[r21]                    // load kernel stack pointer of new task
-        mov IA64_KR(CURRENT)=in0        // update "current" application register
+        MOV_TO_KR(CURRENT, in0, r8, r9)         // update "current" application register
         mov r8=r13                      // return pointer to previously running task
         mov r13=in0                     // set "current" pointer
         ;;
@@ -216,26 +223,25 @@ GLOBAL_ENTRY(ia64_switch_to)
         br.ret.sptk.many rp             // boogie on out in new context
 
 .map:
-        rsm psr.ic                      // interrupts (psr.i) are already disabled here
+        RSM_PSR_IC(r25)                 // interrupts (psr.i) are already disabled here
         movl r25=PAGE_KERNEL
         ;;
         srlz.d
         or r23=r25,r20                  // construct PA | page properties
         mov r25=IA64_GRANULE_SHIFT<<2
         ;;
-        mov cr.itir=r25
-        mov cr.ifa=in0                  // VA of next task...
+        MOV_TO_ITIR(p0, r25, r8)
+        MOV_TO_IFA(in0, r8)             // VA of next task...
         ;;
         mov r25=IA64_TR_CURRENT_STACK
-        mov IA64_KR(CURRENT_STACK)=r26  // remember last page we mapped...
+        MOV_TO_KR(CURRENT_STACK, r26, r8, r9)   // remember last page we mapped...
         ;;
         itr.d dtr[r25]=r23              // wire in new mapping...
-        ssm psr.ic                      // reenable the psr.ic bit
-        ;;
-        srlz.d
+        SSM_PSR_IC_AND_SRLZ_D(r8, r9)   // reenable the psr.ic bit
         br.cond.sptk .done
-END(ia64_switch_to)
+END(__paravirt_switch_to)
 
+#ifdef __IA64_ASM_PARAVIRTUALIZED_NATIVE
 /*
  * Note that interrupts are enabled during save_switch_stack and load_switch_stack.  This
  * means that we may get an interrupt with "sp" pointing to the new kernel stack while
@@ -375,7 +381,7 @@ END(save_switch_stack)
  *      - b7 holds address to return to
  *      - must not touch r8-r11
  */
-ENTRY(load_switch_stack)
+GLOBAL_ENTRY(load_switch_stack)
         .prologue
         .altrp b7
 
@@ -571,7 +577,7 @@ GLOBAL_ENTRY(ia64_trace_syscall)
 .ret3:
 (pUStk) cmp.eq.unc p6,p0=r0,r0                  // p6 <- pUStk
 (pUStk) rsm psr.i                               // disable interrupts
-        br.cond.sptk .work_pending_syscall_end
+        br.cond.sptk ia64_work_pending_syscall_end
 
 strace_error:
         ld8 r3=[r2]                             // load pt_regs.r8
@@ -636,8 +642,17 @@ GLOBAL_ENTRY(ia64_ret_from_syscall)
         adds r2=PT(R8)+16,sp                    // r2 = &pt_regs.r8
         mov r10=r0                              // clear error indication in r10
 (p7)    br.cond.spnt handle_syscall_error       // handle potential syscall failure
+#ifdef CONFIG_PARAVIRT
+        ;;
+        br.cond.sptk.few ia64_leave_syscall
+        ;;
+#endif /* CONFIG_PARAVIRT */
 END(ia64_ret_from_syscall)
+#ifndef CONFIG_PARAVIRT
         // fall through
+#endif
+#endif /* __IA64_ASM_PARAVIRTUALIZED_NATIVE */
+
 /*
  * ia64_leave_syscall(): Same as ia64_leave_kernel, except that it doesn't
  *      need to switch to bank 0 and doesn't restore the scratch registers.
@@ -682,7 +697,7 @@ END(ia64_ret_from_syscall)
  *            ar.csd: cleared
  *            ar.ssd: cleared
  */
-ENTRY(ia64_leave_syscall)
+GLOBAL_ENTRY(__paravirt_leave_syscall)
         PT_REGS_UNWIND_INFO(0)
         /*
          * work.need_resched etc. mustn't get changed by this CPU before it returns to
@@ -692,11 +707,11 @@ ENTRY(ia64_leave_syscall)
          * extra work.  We always check for extra work when returning to user-level.
          * With CONFIG_PREEMPT, we also check for extra work when the preempt_count
          * is 0.  After extra work processing has been completed, execution
-         * resumes at .work_processed_syscall with p6 set to 1 if the extra-work-check
+         * resumes at ia64_work_processed_syscall with p6 set to 1 if the extra-work-check
          * needs to be redone.
          */
 #ifdef CONFIG_PREEMPT
-        rsm psr.i                               // disable interrupts
+        RSM_PSR_I(p0, r2, r18)                  // disable interrupts
         cmp.eq pLvSys,p0=r0,r0                  // pLvSys=1: leave from syscall
 (pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13
         ;;
@@ -706,11 +721,12 @@ ENTRY(ia64_leave_syscall)
         ;;
         cmp.eq p6,p0=r21,r0             // p6 <- pUStk || (preempt_count == 0)
 #else /* !CONFIG_PREEMPT */
-(pUStk) rsm psr.i
+        RSM_PSR_I(pUStk, r2, r18)
         cmp.eq pLvSys,p0=r0,r0          // pLvSys=1: leave from syscall
 (pUStk) cmp.eq.unc p6,p0=r0,r0          // p6 <- pUStk
 #endif
-.work_processed_syscall:
+.global __paravirt_work_processed_syscall;
+__paravirt_work_processed_syscall:
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
         adds r2=PT(LOADRS)+16,r12
 (pUStk) mov.m r22=ar.itc                        // fetch time at leave
@@ -744,7 +760,7 @@ ENTRY(ia64_leave_syscall)
 (pNonSys) break 0               //      bug check: we shouldn't be here if pNonSys is TRUE!
         ;;
         invala                  // M0|1 invalidate ALAT
-        rsm psr.i | psr.ic      // M2   turn off interrupts and interruption collection
+        RSM_PSR_I_IC(r28, r29, r30)     // M2   turn off interrupts and interruption collection
         cmp.eq p9,p0=r0,r0      // A    set p9 to indicate that we should restore cr.ifs
 
         ld8 r29=[r2],16         // M0|1 load cr.ipsr
@@ -765,7 +781,7 @@ ENTRY(ia64_leave_syscall)
         ;;
 #endif
         ld8 r26=[r2],PT(B0)-PT(AR_PFS)  // M0|1 load ar.pfs
-(pKStk) mov r22=psr                     // M2   read PSR now that interrupts are disabled
+        MOV_FROM_PSR(pKStk, r22, r21)   // M2   read PSR now that interrupts are disabled
         nop 0
         ;;
         ld8 r21=[r2],PT(AR_RNAT)-PT(B0) // M0|1 load b0
@@ -798,7 +814,7 @@ ENTRY(ia64_leave_syscall)
 
         srlz.d                          // M0   ensure interruption collection is off (for cover)
         shr.u r18=r19,16                // I0|1 get byte size of existing "dirty" partition
-        cover                           // B    add current frame into dirty partition & set cr.ifs
+        COVER                           // B    add current frame into dirty partition & set cr.ifs
         ;;
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
         mov r19=ar.bsp                  // M2   get new backing store pointer
@@ -823,8 +839,9 @@ ENTRY(ia64_leave_syscall)
         mov.m ar.ssd=r0                 // M2   clear ar.ssd
         mov f11=f0                      // F    clear f11
         br.cond.sptk.many rbs_switch    // B
-END(ia64_leave_syscall)
+END(__paravirt_leave_syscall)
 
+#ifdef __IA64_ASM_PARAVIRTUALIZED_NATIVE
 #ifdef CONFIG_IA32_SUPPORT
 GLOBAL_ENTRY(ia64_ret_from_ia32_execve)
         PT_REGS_UNWIND_INFO(0)
@@ -835,10 +852,20 @@ GLOBAL_ENTRY(ia64_ret_from_ia32_execve)
         st8.spill [r2]=r8       // store return value in slot for r8 and set unat bit
         .mem.offset 8,0
         st8.spill [r3]=r0       // clear error indication in slot for r10 and set unat bit
+#ifdef CONFIG_PARAVIRT
+        ;;
+        // don't fall through, ia64_leave_kernel may be #define'd
+        br.cond.sptk.few ia64_leave_kernel
+        ;;
+#endif /* CONFIG_PARAVIRT */
 END(ia64_ret_from_ia32_execve)
+#ifndef CONFIG_PARAVIRT
         // fall through
+#endif
 #endif /* CONFIG_IA32_SUPPORT */
-GLOBAL_ENTRY(ia64_leave_kernel)
+#endif /* __IA64_ASM_PARAVIRTUALIZED_NATIVE */
+
+GLOBAL_ENTRY(__paravirt_leave_kernel)
         PT_REGS_UNWIND_INFO(0)
         /*
          * work.need_resched etc. mustn't get changed by this CPU before it returns to
@@ -852,7 +879,7 @@ GLOBAL_ENTRY(ia64_leave_kernel)
          * needs to be redone.
          */
 #ifdef CONFIG_PREEMPT
-        rsm psr.i                               // disable interrupts
+        RSM_PSR_I(p0, r17, r31)                 // disable interrupts
         cmp.eq p0,pLvSys=r0,r0                  // pLvSys=0: leave from kernel
 (pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13
         ;;
@@ -862,7 +889,7 @@ GLOBAL_ENTRY(ia64_leave_kernel)
         ;;
         cmp.eq p6,p0=r21,r0             // p6 <- pUStk || (preempt_count == 0)
 #else
-(pUStk) rsm psr.i
+        RSM_PSR_I(pUStk, r17, r31)
         cmp.eq p0,pLvSys=r0,r0          // pLvSys=0: leave from kernel
 (pUStk) cmp.eq.unc p6,p0=r0,r0          // p6 <- pUStk
 #endif
@@ -910,7 +937,7 @@ GLOBAL_ENTRY(ia64_leave_kernel)
         mov ar.csd=r30
         mov ar.ssd=r31
         ;;
-        rsm psr.i | psr.ic      // initiate turning off of interrupt and interruption collection
+        RSM_PSR_I_IC(r23, r22, r25)     // initiate turning off of interrupt and interruption collection
         invala                  // invalidate ALAT
         ;;
         ld8.fill r22=[r2],24
@@ -942,7 +969,7 @@ GLOBAL_ENTRY(ia64_leave_kernel)
         mov ar.ccv=r15
         ;;
         ldf.fill f11=[r2]
-        bsw.0                   // switch back to bank 0 (no stop bit required beforehand...)
+        BSW_0(r2, r3, r15)      // switch back to bank 0 (no stop bit required beforehand...)
         ;;
 (pUStk) mov r18=IA64_KR(CURRENT)// M2 (12 cycle read latency)
         adds r16=PT(CR_IPSR)+16,r12
@@ -950,12 +977,12 @@ GLOBAL_ENTRY(ia64_leave_kernel)
 
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
         .pred.rel.mutex pUStk,pKStk
-(pKStk) mov r22=psr             // M2 read PSR now that interrupts are disabled
+        MOV_FROM_PSR(pKStk, r22, r29)   // M2 read PSR now that interrupts are disabled
 (pUStk) mov.m r22=ar.itc        // M  fetch time at leave
         nop.i 0
         ;;
 #else
-(pKStk) mov r22=psr             // M2 read PSR now that interrupts are disabled
+        MOV_FROM_PSR(pKStk, r22, r29)   // M2 read PSR now that interrupts are disabled
         nop.i 0
         nop.i 0
         ;;
@@ -1027,7 +1054,7 @@ GLOBAL_ENTRY(ia64_leave_kernel)
          * NOTE: alloc, loadrs, and cover can't be predicated.
          */
 (pNonSys) br.cond.dpnt dont_preserve_current_frame
-        cover                           // add current frame into dirty partition and set cr.ifs
+        COVER                           // add current frame into dirty partition and set cr.ifs
         ;;
         mov r19=ar.bsp                  // get new backing store pointer
 rbs_switch:
@@ -1130,16 +1157,16 @@ skip_rbs_switch:
 (pKStk) dep r29=r22,r29,21,1    // I0 update ipsr.pp with psr.pp
 (pLvSys)mov r16=r0              // A  clear r16 for leave_syscall, no-op otherwise
         ;;
-        mov cr.ipsr=r29         // M2
+        MOV_TO_IPSR(p0, r29, r25)       // M2
         mov ar.pfs=r26          // I0
 (pLvSys)mov r17=r0              // A  clear r17 for leave_syscall, no-op otherwise
 
-(p9)    mov cr.ifs=r30          // M2
+        MOV_TO_IFS(p9, r30, r25)// M2
         mov b0=r21              // I0
 (pLvSys)mov r18=r0              // A  clear r18 for leave_syscall, no-op otherwise
 
         mov ar.fpsr=r20         // M2
-        mov cr.iip=r28          // M2
+        MOV_TO_IIP(r28, r25)    // M2
         nop 0
         ;;
 (pUStk) mov ar.rnat=r24         // M2 must happen with RSE in lazy mode
@@ -1148,7 +1175,7 @@ skip_rbs_switch:
 
         mov ar.rsc=r27          // M2
         mov pr=r31,-1           // I0
-        rfi                     // B
+        RFI                     // B
 
         /*
          * On entry:
@@ -1174,35 +1201,36 @@ skip_rbs_switch:
         ;;
 (pKStk) st4 [r20]=r21
 #endif
-        ssm psr.i               // enable interrupts
+        SSM_PSR_I(p0, p6, r2)   // enable interrupts
         br.call.spnt.many rp=schedule
 .ret9:  cmp.eq p6,p0=r0,r0      // p6 <- 1 (re-check)
-        rsm psr.i               // disable interrupts
+        RSM_PSR_I(p0, r2, r20)  // disable interrupts
         ;;
 #ifdef CONFIG_PREEMPT
 (pKStk) adds r20=TI_PRE_COUNT+IA64_TASK_SIZE,r13
         ;;
 (pKStk) st4 [r20]=r0            // preempt_count() <- 0
 #endif
-(pLvSys)br.cond.sptk.few  .work_pending_syscall_end
+(pLvSys)br.cond.sptk.few  __paravirt_pending_syscall_end
         br.cond.sptk.many .work_processed_kernel
 
 .notify:
 (pUStk) br.call.spnt.many rp=notify_resume_user
 .ret10: cmp.ne p6,p0=r0,r0      // p6 <- 0 (don't re-check)
-(pLvSys)br.cond.sptk.few  .work_pending_syscall_end
+(pLvSys)br.cond.sptk.few  __paravirt_pending_syscall_end
         br.cond.sptk.many .work_processed_kernel
 
-.work_pending_syscall_end:
+.global __paravirt_pending_syscall_end;
+__paravirt_pending_syscall_end:
         adds r2=PT(R8)+16,r12
         adds r3=PT(R10)+16,r12
         ;;
         ld8 r8=[r2]
         ld8 r10=[r3]
-        br.cond.sptk.many .work_processed_syscall
-
-END(ia64_leave_kernel)
+        br.cond.sptk.many __paravirt_work_processed_syscall_target
+END(__paravirt_leave_kernel)
 
+#ifdef __IA64_ASM_PARAVIRTUALIZED_NATIVE
 ENTRY(handle_syscall_error)
         /*
          * Some system calls (e.g., ptrace, mmap) can return arbitrary values which could
@@ -1244,7 +1272,7 @@ END(ia64_invoke_schedule_tail)
          * We declare 8 input registers so the system call args get preserved,
          * in case we need to restart a system call.
          */
-ENTRY(notify_resume_user)
+GLOBAL_ENTRY(notify_resume_user)
         .prologue ASM_UNW_PRLG_RP|ASM_UNW_PRLG_PFS, ASM_UNW_PRLG_GRSAVE(8)
         alloc loc1=ar.pfs,8,2,3,0 // preserve all eight input regs in case of syscall restart!
         mov r9=ar.unat
@@ -1306,7 +1334,7 @@ ENTRY(sys_rt_sigreturn)
         adds sp=16,sp
         ;;
         ld8 r9=[sp]                             // load new ar.unat
-        mov.sptk b7=r8,ia64_leave_kernel
+        mov.sptk b7=r8,ia64_native_leave_kernel
         ;;
         mov ar.unat=r9
         br.many b7
@@ -1665,3 +1693,4 @@ sys_call_table:
         data8 sys_timerfd_gettime
 
         .org sys_call_table + 8*NR_syscalls     // guard against failures to increase NR_syscalls
+#endif /* __IA64_ASM_PARAVIRTUALIZED_NATIVE */
diff --git a/arch/ia64/kernel/paravirt.c b/arch/ia64/kernel/paravirt.c
index e5482bb6841e..7126ea8f7ecc 100644
--- a/arch/ia64/kernel/paravirt.c
+++ b/arch/ia64/kernel/paravirt.c
@@ -286,3 +286,22 @@ struct pv_cpu_ops pv_cpu_ops = {
                         = ia64_native_intrin_local_irq_restore_func,
 };
 EXPORT_SYMBOL(pv_cpu_ops);
+
+/******************************************************************************
+ * replacement of hand written assembly codes.
+ */
+
+void
+paravirt_cpu_asm_init(const struct pv_cpu_asm_switch *cpu_asm_switch)
+{
+        extern unsigned long paravirt_switch_to_targ;
+        extern unsigned long paravirt_leave_syscall_targ;
+        extern unsigned long paravirt_work_processed_syscall_targ;
+        extern unsigned long paravirt_leave_kernel_targ;
+
+        paravirt_switch_to_targ = cpu_asm_switch->switch_to;
+        paravirt_leave_syscall_targ = cpu_asm_switch->leave_syscall;
+        paravirt_work_processed_syscall_targ =
+                cpu_asm_switch->work_processed_syscall;
+        paravirt_leave_kernel_targ = cpu_asm_switch->leave_kernel;
+}
diff --git a/arch/ia64/kernel/paravirtentry.S b/arch/ia64/kernel/paravirtentry.S
new file mode 100644
index 000000000000..2f42fcb9776a
--- /dev/null
+++ b/arch/ia64/kernel/paravirtentry.S
@@ -0,0 +1,60 @@
+/******************************************************************************
+ * linux/arch/ia64/xen/paravirtentry.S
+ *
+ * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
+ *                    VA Linux Systems Japan K.K.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include <asm/asmmacro.h>
+#include <asm/asm-offsets.h>
+#include "entry.h"
+
+#define DATA8(sym, init_value)                  \
+        .pushsection .data.read_mostly ;        \
+        .align 8 ;                              \
+        .global sym ;                           \
+        sym: ;                                  \
+        data8 init_value ;                      \
+        .popsection
+
+#define BRANCH(targ, reg, breg)         \
+        movl reg=targ ;                 \
+        ;;                              \
+        ld8 reg=[reg] ;                 \
+        ;;                              \
+        mov breg=reg ;                  \
+        br.cond.sptk.many breg
+
+#define BRANCH_PROC(sym, reg, breg)                             \
+        DATA8(paravirt_ ## sym ## _targ, ia64_native_ ## sym) ; \
+        GLOBAL_ENTRY(paravirt_ ## sym) ;                        \
+                BRANCH(paravirt_ ## sym ## _targ, reg, breg) ;  \
+        END(paravirt_ ## sym)
+
+#define BRANCH_PROC_UNWINFO(sym, reg, breg)                     \
+        DATA8(paravirt_ ## sym ## _targ, ia64_native_ ## sym) ; \
+        GLOBAL_ENTRY(paravirt_ ## sym) ;                        \
+                PT_REGS_UNWIND_INFO(0) ;                        \
+                BRANCH(paravirt_ ## sym ## _targ, reg, breg) ;  \
+        END(paravirt_ ## sym)
+
+
+BRANCH_PROC(switch_to, r22, b7)
+BRANCH_PROC_UNWINFO(leave_syscall, r22, b7)
+BRANCH_PROC(work_processed_syscall, r2, b7)
+BRANCH_PROC_UNWINFO(leave_kernel, r22, b7)