ia64/pv_ops: implement binary patching optimization for native.

implement binary patching optimization for pv_cpu_ops. With this optimization, indirect call for pv_cpu_ops methods can be converted into inline execution or direct call. Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp> Signed-off-by: Tony Luck <tony.luck@intel.com>
author: Isaku Yamahata <yamahata@valinux.co.jp> 2009-03-04 07:06:52 -0500
committer: Tony Luck <tony.luck@intel.com> 2009-03-26 14:02:42 -0400
commit: 03f511dd02f1431ef652fb97a7f2fe7aef47e025 (patch)
tree: 87bbff26febaa0eb0e85d1c23886a1cc913af30a /arch/ia64/kernel/paravirt.c
parent: bf7ab02f620c1020c869fc71a2c855918b6a5375 (diff)
1 files changed, 519 insertions, 1 deletions
diff --git a/arch/ia64/kernel/paravirt.c b/arch/ia64/kernel/paravirt.c
index 6bc33a6db755..158d52414e96 100644
--- a/arch/ia64/kernel/paravirt.c
+++ b/arch/ia64/kernel/paravirt.c
@@ -46,13 +46,23 @@ struct pv_info pv_info = {
 * initialization hooks.
 */
-struct pv_init_ops pv_init_ops;
+static void __init
+ia64_native_patch_branch(unsigned long tag, unsigned long type);
+struct pv_init_ops pv_init_ops =
+{
+#ifdef ASM_SUPPORTED
+        .patch_bundle = ia64_native_patch_bundle,
+#endif
+        .patch_branch = ia64_native_patch_branch,
+};
 /***************************************************************************
 * pv_cpu_ops
 * intrinsics hooks.
 */
+#ifndef ASM_SUPPORTED
 /* ia64_native_xxx are macros so that we have to make them real functions */
 #define DEFINE_VOID_FUNC1(name)                                 \
@@ -274,6 +284,261 @@ ia64_native_setreg_func(int regnum, unsigned long val)
                break;
        }
 }
+#else
+#define __DEFINE_FUNC(name, code)                                       \
+        extern const char ia64_native_ ## name ## _direct_start[];      \
+        extern const char ia64_native_ ## name ## _direct_end[];        \
+        asm (".align 32\n"                                              \
+             ".proc ia64_native_" #name "_func\n"                       \
+             "ia64_native_" #name "_func:\n"                            \
+             "ia64_native_" #name "_direct_start:\n"                    \
+             code                                                       \
+             "ia64_native_" #name "_direct_end:\n"                      \
+             "br.cond.sptk.many b6\n"                                   \
+             ".endp ia64_native_" #name "_func\n")
+#define DEFINE_VOID_FUNC0(name, code)                           \
+        extern void                                             \
+        ia64_native_ ## name ## _func(void);                    \
+        __DEFINE_FUNC(name, code)
+#define DEFINE_VOID_FUNC1(name, code)                           \
+        extern void                                             \
+        ia64_native_ ## name ## _func(unsigned long arg);       \
+        __DEFINE_FUNC(name, code)
+#define DEFINE_VOID_FUNC2(name, code)                           \
+        extern void                                             \
+        ia64_native_ ## name ## _func(unsigned long arg0,       \
+                                      unsigned long arg1);      \
+        __DEFINE_FUNC(name, code)
+#define DEFINE_FUNC0(name, code)                \
+        extern unsigned long                    \
+        ia64_native_ ## name ## _func(void);    \
+        __DEFINE_FUNC(name, code)
+#define DEFINE_FUNC1(name, type, code)                  \
+        extern unsigned long                            \
+        ia64_native_ ## name ## _func(type arg);        \
+        __DEFINE_FUNC(name, code)
+DEFINE_VOID_FUNC1(fc,
+                  "fc r8\n");
+DEFINE_VOID_FUNC1(intrin_local_irq_restore,
+                  ";;\n"
+                  "     cmp.ne p6, p7 = r8, r0\n"
+                  ";;\n"
+                  "(p6) ssm psr.i\n"
+                  "(p7) rsm psr.i\n"
+                  ";;\n"
+                  "(p6) srlz.d\n");
+DEFINE_VOID_FUNC2(ptcga,
+                  "ptc.ga r8, r9\n");
+DEFINE_VOID_FUNC2(set_rr,
+                  "mov rr[r8] = r9\n");
+/* ia64_native_getreg(_IA64_REG_PSR) & IA64_PSR_I */
+DEFINE_FUNC0(get_psr_i,
+             "mov r2 = " __stringify(1 << IA64_PSR_I_BIT) "\n"
+             "mov r8 = psr\n"
+             ";;\n"
+             "and r8 = r2, r8\n");
+DEFINE_FUNC1(thash, unsigned long,
+             "thash r8 = r8\n");
+DEFINE_FUNC1(get_cpuid, int,
+             "mov r8 = cpuid[r8]\n");
+DEFINE_FUNC1(get_pmd, int,
+             "mov r8 = pmd[r8]\n");
+DEFINE_FUNC1(get_rr, unsigned long,
+             "mov r8 = rr[r8]\n");
+DEFINE_VOID_FUNC0(ssm_i,
+                  "ssm psr.i\n");
+DEFINE_VOID_FUNC0(rsm_i,
+                  "rsm psr.i\n");
+extern void
+ia64_native_set_rr0_to_rr4_func(unsigned long val0, unsigned long val1,
+                                unsigned long val2, unsigned long val3,
+                                unsigned long val4);
+__DEFINE_FUNC(set_rr0_to_rr4,
+              "mov rr[r0] = r8\n"
+              "movl r2 = 0x2000000000000000\n"
+              ";;\n"
+              "mov rr[r2] = r9\n"
+              "shl r3 = r2, 1\n"        /* movl r3 = 0x4000000000000000 */
+              ";;\n"
+              "add r2 = r2, r3\n"       /* movl r2 = 0x6000000000000000 */
+              "mov rr[r3] = r10\n"
+              ";;\n"
+              "mov rr[r2] = r11\n"
+              "shl r3 = r3, 1\n"        /* movl r3 = 0x8000000000000000 */
+              ";;\n"
+              "mov rr[r3] = r14\n");
+extern unsigned long ia64_native_getreg_func(int regnum);
+asm(".global ia64_native_getreg_func\n");
+#define __DEFINE_GET_REG(id, reg)                       \
+        "mov r2 = " __stringify(_IA64_REG_ ## id) "\n"  \
+        ";;\n"                                          \
+        "cmp.eq p6, p0 = r2, r8\n"                      \
+        ";;\n"                                          \
+        "(p6) mov r8 = " #reg "\n"                      \
+        "(p6) br.cond.sptk.many b6\n"                   \
+        ";;\n"
+#define __DEFINE_GET_AR(id, reg)        __DEFINE_GET_REG(AR_ ## id, ar.reg)
+#define __DEFINE_GET_CR(id, reg)        __DEFINE_GET_REG(CR_ ## id, cr.reg)
+__DEFINE_FUNC(getreg,
+              __DEFINE_GET_REG(GP, gp)
+              /*__DEFINE_GET_REG(IP, ip)*/ /* returned ip value shouldn't be constant */
+              __DEFINE_GET_REG(PSR, psr)
+              __DEFINE_GET_REG(TP, tp)
+              __DEFINE_GET_REG(SP, sp)
+              __DEFINE_GET_REG(AR_KR0, ar0)
+              __DEFINE_GET_REG(AR_KR1, ar1)
+              __DEFINE_GET_REG(AR_KR2, ar2)
+              __DEFINE_GET_REG(AR_KR3, ar3)
+              __DEFINE_GET_REG(AR_KR4, ar4)
+              __DEFINE_GET_REG(AR_KR5, ar5)
+              __DEFINE_GET_REG(AR_KR6, ar6)
+              __DEFINE_GET_REG(AR_KR7, ar7)
+              __DEFINE_GET_AR(RSC, rsc)
+              __DEFINE_GET_AR(BSP, bsp)
+              __DEFINE_GET_AR(BSPSTORE, bspstore)
+              __DEFINE_GET_AR(RNAT, rnat)
+              __DEFINE_GET_AR(FCR, fcr)
+              __DEFINE_GET_AR(EFLAG, eflag)
+              __DEFINE_GET_AR(CSD, csd)
+              __DEFINE_GET_AR(SSD, ssd)
+              __DEFINE_GET_REG(AR_CFLAG, ar27)
+              __DEFINE_GET_AR(FSR, fsr)
+              __DEFINE_GET_AR(FIR, fir)
+              __DEFINE_GET_AR(FDR, fdr)
+              __DEFINE_GET_AR(CCV, ccv)
+              __DEFINE_GET_AR(UNAT, unat)
+              __DEFINE_GET_AR(FPSR, fpsr)
+              __DEFINE_GET_AR(ITC, itc)
+              __DEFINE_GET_AR(PFS, pfs)
+              __DEFINE_GET_AR(LC, lc)
+              __DEFINE_GET_AR(EC, ec)
+              __DEFINE_GET_CR(DCR, dcr)
+              __DEFINE_GET_CR(ITM, itm)
+              __DEFINE_GET_CR(IVA, iva)
+              __DEFINE_GET_CR(PTA, pta)
+              __DEFINE_GET_CR(IPSR, ipsr)
+              __DEFINE_GET_CR(ISR, isr)
+              __DEFINE_GET_CR(IIP, iip)
+              __DEFINE_GET_CR(IFA, ifa)
+              __DEFINE_GET_CR(ITIR, itir)
+              __DEFINE_GET_CR(IIPA, iipa)
+              __DEFINE_GET_CR(IFS, ifs)
+              __DEFINE_GET_CR(IIM, iim)
+              __DEFINE_GET_CR(IHA, iha)
+              __DEFINE_GET_CR(LID, lid)
+              __DEFINE_GET_CR(IVR, ivr)
+              __DEFINE_GET_CR(TPR, tpr)
+              __DEFINE_GET_CR(EOI, eoi)
+              __DEFINE_GET_CR(IRR0, irr0)
+              __DEFINE_GET_CR(IRR1, irr1)
+              __DEFINE_GET_CR(IRR2, irr2)
+              __DEFINE_GET_CR(IRR3, irr3)
+              __DEFINE_GET_CR(ITV, itv)
+              __DEFINE_GET_CR(PMV, pmv)
+              __DEFINE_GET_CR(CMCV, cmcv)
+              __DEFINE_GET_CR(LRR0, lrr0)
+              __DEFINE_GET_CR(LRR1, lrr1)
+              "mov r8 = -1\n"   /* unsupported case */
+        );
+extern void ia64_native_setreg_func(int regnum, unsigned long val);
+asm(".global ia64_native_setreg_func\n");
+#define __DEFINE_SET_REG(id, reg)                       \
+        "mov r2 = " __stringify(_IA64_REG_ ## id) "\n"  \
+        ";;\n"                                          \
+        "cmp.eq p6, p0 = r2, r9\n"                      \
+        ";;\n"                                          \
+        "(p6) mov " #reg " = r8\n"                      \
+        "(p6) br.cond.sptk.many b6\n"                   \
+        ";;\n"
+#define __DEFINE_SET_AR(id, reg)        __DEFINE_SET_REG(AR_ ## id, ar.reg)
+#define __DEFINE_SET_CR(id, reg)        __DEFINE_SET_REG(CR_ ## id, cr.reg)
+__DEFINE_FUNC(setreg,
+              "mov r2 = " __stringify(_IA64_REG_PSR_L) "\n"
+              ";;\n"
+              "cmp.eq p6, p0 = r2, r9\n"
+              ";;\n"
+              "(p6) mov psr.l = r8\n"
+#ifdef HAVE_SERIALIZE_DIRECTIVE
+              ".serialize.data\n"
+#endif
+              "(p6) br.cond.sptk.many b6\n"
+              __DEFINE_SET_REG(GP, gp)
+              __DEFINE_SET_REG(SP, sp)
+              __DEFINE_SET_REG(AR_KR0, ar0)
+              __DEFINE_SET_REG(AR_KR1, ar1)
+              __DEFINE_SET_REG(AR_KR2, ar2)
+              __DEFINE_SET_REG(AR_KR3, ar3)
+              __DEFINE_SET_REG(AR_KR4, ar4)
+              __DEFINE_SET_REG(AR_KR5, ar5)
+              __DEFINE_SET_REG(AR_KR6, ar6)
+              __DEFINE_SET_REG(AR_KR7, ar7)
+              __DEFINE_SET_AR(RSC, rsc)
+              __DEFINE_SET_AR(BSP, bsp)
+              __DEFINE_SET_AR(BSPSTORE, bspstore)
+              __DEFINE_SET_AR(RNAT, rnat)
+              __DEFINE_SET_AR(FCR, fcr)
+              __DEFINE_SET_AR(EFLAG, eflag)
+              __DEFINE_SET_AR(CSD, csd)
+              __DEFINE_SET_AR(SSD, ssd)
+              __DEFINE_SET_REG(AR_CFLAG, ar27)
+              __DEFINE_SET_AR(FSR, fsr)
+              __DEFINE_SET_AR(FIR, fir)
+              __DEFINE_SET_AR(FDR, fdr)
+              __DEFINE_SET_AR(CCV, ccv)
+              __DEFINE_SET_AR(UNAT, unat)
+              __DEFINE_SET_AR(FPSR, fpsr)
+              __DEFINE_SET_AR(ITC, itc)
+              __DEFINE_SET_AR(PFS, pfs)
+              __DEFINE_SET_AR(LC, lc)
+              __DEFINE_SET_AR(EC, ec)
+              __DEFINE_SET_CR(DCR, dcr)
+              __DEFINE_SET_CR(ITM, itm)
+              __DEFINE_SET_CR(IVA, iva)
+              __DEFINE_SET_CR(PTA, pta)
+              __DEFINE_SET_CR(IPSR, ipsr)
+              __DEFINE_SET_CR(ISR, isr)
+              __DEFINE_SET_CR(IIP, iip)
+              __DEFINE_SET_CR(IFA, ifa)
+              __DEFINE_SET_CR(ITIR, itir)
+              __DEFINE_SET_CR(IIPA, iipa)
+              __DEFINE_SET_CR(IFS, ifs)
+              __DEFINE_SET_CR(IIM, iim)
+              __DEFINE_SET_CR(IHA, iha)
+              __DEFINE_SET_CR(LID, lid)
+              __DEFINE_SET_CR(IVR, ivr)
+              __DEFINE_SET_CR(TPR, tpr)
+              __DEFINE_SET_CR(EOI, eoi)
+              __DEFINE_SET_CR(IRR0, irr0)
+              __DEFINE_SET_CR(IRR1, irr1)
+              __DEFINE_SET_CR(IRR2, irr2)
+              __DEFINE_SET_CR(IRR3, irr3)
+              __DEFINE_SET_CR(ITV, itv)
+              __DEFINE_SET_CR(PMV, pmv)
+              __DEFINE_SET_CR(CMCV, cmcv)
+              __DEFINE_SET_CR(LRR0, lrr0)
+              __DEFINE_SET_CR(LRR1, lrr1)
+        );
+#endif
 struct pv_cpu_ops pv_cpu_ops = {
        .fc             = ia64_native_fc_func,
@@ -368,3 +633,256 @@ struct pv_time_ops pv_time_ops = {
        .do_steal_accounting = ia64_native_do_steal_accounting,
        .sched_clock = ia64_native_sched_clock,
 };
+/***************************************************************************
+ * binary pacthing
+ * pv_init_ops.patch_bundle
+ */
+#ifdef ASM_SUPPORTED
+#define IA64_NATIVE_PATCH_DEFINE_GET_REG(name, reg)     \
+        __DEFINE_FUNC(get_ ## name,                     \
+                      ";;\n"                            \
+                      "mov r8 = " #reg "\n"             \
+                      ";;\n")
+#define IA64_NATIVE_PATCH_DEFINE_SET_REG(name, reg)     \
+        __DEFINE_FUNC(set_ ## name,                     \
+                      ";;\n"                            \
+                      "mov " #reg " = r8\n"             \
+                      ";;\n")
+#define IA64_NATIVE_PATCH_DEFINE_REG(name, reg)         \
+        IA64_NATIVE_PATCH_DEFINE_GET_REG(name, reg);    \
+        IA64_NATIVE_PATCH_DEFINE_SET_REG(name, reg)     \
+#define IA64_NATIVE_PATCH_DEFINE_AR(name, reg)                  \
+        IA64_NATIVE_PATCH_DEFINE_REG(ar_ ## name, ar.reg)
+#define IA64_NATIVE_PATCH_DEFINE_CR(name, reg)                  \
+        IA64_NATIVE_PATCH_DEFINE_REG(cr_ ## name, cr.reg)
+IA64_NATIVE_PATCH_DEFINE_GET_REG(psr, psr);
+IA64_NATIVE_PATCH_DEFINE_GET_REG(tp, tp);
+/* IA64_NATIVE_PATCH_DEFINE_SET_REG(psr_l, psr.l); */
+__DEFINE_FUNC(set_psr_l,
+              ";;\n"
+              "mov psr.l = r8\n"
+#ifdef HAVE_SERIALIZE_DIRECTIVE
+              ".serialize.data\n"
+#endif
+              ";;\n");
+IA64_NATIVE_PATCH_DEFINE_REG(gp, gp);
+IA64_NATIVE_PATCH_DEFINE_REG(sp, sp);
+IA64_NATIVE_PATCH_DEFINE_REG(kr0, ar0);
+IA64_NATIVE_PATCH_DEFINE_REG(kr1, ar1);
+IA64_NATIVE_PATCH_DEFINE_REG(kr2, ar2);
+IA64_NATIVE_PATCH_DEFINE_REG(kr3, ar3);
+IA64_NATIVE_PATCH_DEFINE_REG(kr4, ar4);
+IA64_NATIVE_PATCH_DEFINE_REG(kr5, ar5);
+IA64_NATIVE_PATCH_DEFINE_REG(kr6, ar6);
+IA64_NATIVE_PATCH_DEFINE_REG(kr7, ar7);
+IA64_NATIVE_PATCH_DEFINE_AR(rsc, rsc);
+IA64_NATIVE_PATCH_DEFINE_AR(bsp, bsp);
+IA64_NATIVE_PATCH_DEFINE_AR(bspstore, bspstore);
+IA64_NATIVE_PATCH_DEFINE_AR(rnat, rnat);
+IA64_NATIVE_PATCH_DEFINE_AR(fcr, fcr);
+IA64_NATIVE_PATCH_DEFINE_AR(eflag, eflag);
+IA64_NATIVE_PATCH_DEFINE_AR(csd, csd);
+IA64_NATIVE_PATCH_DEFINE_AR(ssd, ssd);
+IA64_NATIVE_PATCH_DEFINE_REG(ar27, ar27);
+IA64_NATIVE_PATCH_DEFINE_AR(fsr, fsr);
+IA64_NATIVE_PATCH_DEFINE_AR(fir, fir);
+IA64_NATIVE_PATCH_DEFINE_AR(fdr, fdr);
+IA64_NATIVE_PATCH_DEFINE_AR(ccv, ccv);
+IA64_NATIVE_PATCH_DEFINE_AR(unat, unat);
+IA64_NATIVE_PATCH_DEFINE_AR(fpsr, fpsr);
+IA64_NATIVE_PATCH_DEFINE_AR(itc, itc);
+IA64_NATIVE_PATCH_DEFINE_AR(pfs, pfs);
+IA64_NATIVE_PATCH_DEFINE_AR(lc, lc);
+IA64_NATIVE_PATCH_DEFINE_AR(ec, ec);
+IA64_NATIVE_PATCH_DEFINE_CR(dcr, dcr);
+IA64_NATIVE_PATCH_DEFINE_CR(itm, itm);
+IA64_NATIVE_PATCH_DEFINE_CR(iva, iva);
+IA64_NATIVE_PATCH_DEFINE_CR(pta, pta);
+IA64_NATIVE_PATCH_DEFINE_CR(ipsr, ipsr);
+IA64_NATIVE_PATCH_DEFINE_CR(isr, isr);
+IA64_NATIVE_PATCH_DEFINE_CR(iip, iip);
+IA64_NATIVE_PATCH_DEFINE_CR(ifa, ifa);
+IA64_NATIVE_PATCH_DEFINE_CR(itir, itir);
+IA64_NATIVE_PATCH_DEFINE_CR(iipa, iipa);
+IA64_NATIVE_PATCH_DEFINE_CR(ifs, ifs);
+IA64_NATIVE_PATCH_DEFINE_CR(iim, iim);
+IA64_NATIVE_PATCH_DEFINE_CR(iha, iha);
+IA64_NATIVE_PATCH_DEFINE_CR(lid, lid);
+IA64_NATIVE_PATCH_DEFINE_CR(ivr, ivr);
+IA64_NATIVE_PATCH_DEFINE_CR(tpr, tpr);
+IA64_NATIVE_PATCH_DEFINE_CR(eoi, eoi);
+IA64_NATIVE_PATCH_DEFINE_CR(irr0, irr0);
+IA64_NATIVE_PATCH_DEFINE_CR(irr1, irr1);
+IA64_NATIVE_PATCH_DEFINE_CR(irr2, irr2);
+IA64_NATIVE_PATCH_DEFINE_CR(irr3, irr3);
+IA64_NATIVE_PATCH_DEFINE_CR(itv, itv);
+IA64_NATIVE_PATCH_DEFINE_CR(pmv, pmv);
+IA64_NATIVE_PATCH_DEFINE_CR(cmcv, cmcv);
+IA64_NATIVE_PATCH_DEFINE_CR(lrr0, lrr0);
+IA64_NATIVE_PATCH_DEFINE_CR(lrr1, lrr1);
+static const struct paravirt_patch_bundle_elem ia64_native_patch_bundle_elems[]
+__initdata_or_module =
+{
+#define IA64_NATIVE_PATCH_BUNDLE_ELEM(name, type)               \
+        {                                                       \
+                (void*)ia64_native_ ## name ## _direct_start,   \
+                (void*)ia64_native_ ## name ## _direct_end,     \
+                PARAVIRT_PATCH_TYPE_ ## type,                   \
+        }
+        IA64_NATIVE_PATCH_BUNDLE_ELEM(fc, FC),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM(thash, THASH),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM(get_cpuid, GET_CPUID),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM(get_pmd, GET_PMD),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM(ptcga, PTCGA),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM(get_rr, GET_RR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM(set_rr, SET_RR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM(set_rr0_to_rr4, SET_RR0_TO_RR4),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM(ssm_i, SSM_I),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM(rsm_i, RSM_I),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM(get_psr_i, GET_PSR_I),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM(intrin_local_irq_restore,
+                                      INTRIN_LOCAL_IRQ_RESTORE),
+#define IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(name, reg)                 \
+        {                                                               \
+                (void*)ia64_native_get_ ## name ## _direct_start,       \
+                (void*)ia64_native_get_ ## name ## _direct_end,         \
+                PARAVIRT_PATCH_TYPE_GETREG + _IA64_REG_ ## reg,         \
+        }
+#define IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(name, reg)                 \
+        {                                                               \
+                (void*)ia64_native_set_ ## name ## _direct_start,       \
+                (void*)ia64_native_set_ ## name ## _direct_end,         \
+                PARAVIRT_PATCH_TYPE_SETREG + _IA64_REG_ ## reg,         \
+        }
+#define IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(name, reg)            \
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(name, reg),        \
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(name, reg)         \
+#define IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(name, reg)             \
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(ar_ ## name, AR_ ## reg)
+#define IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(name, reg)             \
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(cr_ ## name, CR_ ## reg)
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(psr, PSR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(tp, TP),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(psr_l, PSR_L),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(gp, GP),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(sp, SP),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr0, AR_KR0),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr1, AR_KR1),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr2, AR_KR2),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr3, AR_KR3),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr4, AR_KR4),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr5, AR_KR5),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr6, AR_KR6),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr7, AR_KR7),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(rsc, RSC),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(bsp, BSP),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(bspstore, BSPSTORE),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(rnat, RNAT),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fcr, FCR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(eflag, EFLAG),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(csd, CSD),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ssd, SSD),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(ar27, AR_CFLAG),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fsr, FSR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fir, FIR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fdr, FDR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ccv, CCV),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(unat, UNAT),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fpsr, FPSR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(itc, ITC),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(pfs, PFS),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(lc, LC),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ec, EC),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(dcr, DCR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itm, ITM),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iva, IVA),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(pta, PTA),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ipsr, IPSR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(isr, ISR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iip, IIP),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ifa, IFA),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itir, ITIR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iipa, IIPA),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ifs, IFS),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iim, IIM),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iha, IHA),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lid, LID),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ivr, IVR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(tpr, TPR),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(eoi, EOI),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr0, IRR0),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr1, IRR1),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr2, IRR2),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr3, IRR3),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itv, ITV),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(pmv, PMV),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(cmcv, CMCV),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lrr0, LRR0),
+        IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lrr1, LRR1),
+};
+unsigned long __init_or_module
+ia64_native_patch_bundle(void *sbundle, void *ebundle, unsigned long type)
+{
+        const unsigned long nelems = sizeof(ia64_native_patch_bundle_elems) /
+                sizeof(ia64_native_patch_bundle_elems[0]);
+        return __paravirt_patch_apply_bundle(sbundle, ebundle, type,
+                                              ia64_native_patch_bundle_elems,
+                                              nelems, NULL);
+}
+#endif /* ASM_SUPPOTED */
+extern const char ia64_native_switch_to[];
+extern const char ia64_native_leave_syscall[];
+extern const char ia64_native_work_processed_syscall[];
+extern const char ia64_native_leave_kernel[];
+const struct paravirt_patch_branch_target ia64_native_branch_target[]
+__initconst = {
+#define PARAVIRT_BR_TARGET(name, type)                  \
+        {                                               \
+                ia64_native_ ## name,                   \
+                PARAVIRT_PATCH_TYPE_BR_ ## type,        \
+        }
+        PARAVIRT_BR_TARGET(switch_to, SWITCH_TO),
+        PARAVIRT_BR_TARGET(leave_syscall, LEAVE_SYSCALL),
+        PARAVIRT_BR_TARGET(work_processed_syscall, WORK_PROCESSED_SYSCALL),
+        PARAVIRT_BR_TARGET(leave_kernel, LEAVE_KERNEL),
+};
+static void __init
+ia64_native_patch_branch(unsigned long tag, unsigned long type)
+{
+        const unsigned long nelem =
+                sizeof(ia64_native_branch_target) /
+                sizeof(ia64_native_branch_target[0]);
+        __paravirt_patch_apply_branch(tag, type,
+                                      ia64_native_branch_target, nelem);
+}
author	Isaku Yamahata <yamahata@valinux.co.jp>	2009-03-04 07:06:52 -0500
committer	Tony Luck <tony.luck@intel.com>	2009-03-26 14:02:42 -0400
commit	03f511dd02f1431ef652fb97a7f2fe7aef47e025 (patch)
tree	87bbff26febaa0eb0e85d1c23886a1cc913af30a /arch/ia64/kernel/paravirt.c
parent	bf7ab02f620c1020c869fc71a2c855918b6a5375 (diff)

diff --git a/arch/ia64/kernel/paravirt.c b/arch/ia64/kernel/paravirt.c index 6bc33a6db755..158d52414e96 100644 --- a/arch/ia64/kernel/paravirt.c +++ b/arch/ia64/kernel/paravirt.c
@@ -46,13 +46,23 @@ struct pv_info pv_info = {
46	* initialization hooks.	46	* initialization hooks.
47	*/	47	*/
48		48
49	struct pv_init_ops pv_init_ops;	49	static void __init
		50	ia64_native_patch_branch(unsigned long tag, unsigned long type);
		51
		52	struct pv_init_ops pv_init_ops =
		53	{
		54	#ifdef ASM_SUPPORTED
		55	.patch_bundle = ia64_native_patch_bundle,
		56	#endif
		57	.patch_branch = ia64_native_patch_branch,
		58	};
50		59
51	/***************************************************************************	60	/***************************************************************************
52	* pv_cpu_ops	61	* pv_cpu_ops
53	* intrinsics hooks.	62	* intrinsics hooks.
54	*/	63	*/
55		64
		65	#ifndef ASM_SUPPORTED
56	/* ia64_native_xxx are macros so that we have to make them real functions */	66	/* ia64_native_xxx are macros so that we have to make them real functions */
57		67
58	#define DEFINE_VOID_FUNC1(name) \	68	#define DEFINE_VOID_FUNC1(name) \
@@ -274,6 +284,261 @@ ia64_native_setreg_func(int regnum, unsigned long val)
274	break;	284	break;
275	}	285	}
276	}	286	}
		287	#else
		288
		289	#define __DEFINE_FUNC(name, code) \
		290	extern const char ia64_native_ ## name ## _direct_start[]; \
		291	extern const char ia64_native_ ## name ## _direct_end[]; \
		292	asm (".align 32\n" \
		293	".proc ia64_native_" #name "_func\n" \
		294	"ia64_native_" #name "_func:\n" \
		295	"ia64_native_" #name "_direct_start:\n" \
		296	code \
		297	"ia64_native_" #name "_direct_end:\n" \
		298	"br.cond.sptk.many b6\n" \
		299	".endp ia64_native_" #name "_func\n")
		300
		301	#define DEFINE_VOID_FUNC0(name, code) \
		302	extern void \
		303	ia64_native_ ## name ## _func(void); \
		304	__DEFINE_FUNC(name, code)
		305
		306	#define DEFINE_VOID_FUNC1(name, code) \
		307	extern void \
		308	ia64_native_ ## name ## _func(unsigned long arg); \
		309	__DEFINE_FUNC(name, code)
		310
		311	#define DEFINE_VOID_FUNC2(name, code) \
		312	extern void \
		313	ia64_native_ ## name ## _func(unsigned long arg0, \
		314	unsigned long arg1); \
		315	__DEFINE_FUNC(name, code)
		316
		317	#define DEFINE_FUNC0(name, code) \
		318	extern unsigned long \
		319	ia64_native_ ## name ## _func(void); \
		320	__DEFINE_FUNC(name, code)
		321
		322	#define DEFINE_FUNC1(name, type, code) \
		323	extern unsigned long \
		324	ia64_native_ ## name ## _func(type arg); \
		325	__DEFINE_FUNC(name, code)
		326
		327	DEFINE_VOID_FUNC1(fc,
		328	"fc r8\n");
		329	DEFINE_VOID_FUNC1(intrin_local_irq_restore,
		330	";;\n"
		331	" cmp.ne p6, p7 = r8, r0\n"
		332	";;\n"
		333	"(p6) ssm psr.i\n"
		334	"(p7) rsm psr.i\n"
		335	";;\n"
		336	"(p6) srlz.d\n");
		337
		338	DEFINE_VOID_FUNC2(ptcga,
		339	"ptc.ga r8, r9\n");
		340	DEFINE_VOID_FUNC2(set_rr,
		341	"mov rr[r8] = r9\n");
		342
		343	/* ia64_native_getreg(_IA64_REG_PSR) & IA64_PSR_I */
		344	DEFINE_FUNC0(get_psr_i,
		345	"mov r2 = " __stringify(1 << IA64_PSR_I_BIT) "\n"
		346	"mov r8 = psr\n"
		347	";;\n"
		348	"and r8 = r2, r8\n");
		349
		350	DEFINE_FUNC1(thash, unsigned long,
		351	"thash r8 = r8\n");
		352	DEFINE_FUNC1(get_cpuid, int,
		353	"mov r8 = cpuid[r8]\n");
		354	DEFINE_FUNC1(get_pmd, int,
		355	"mov r8 = pmd[r8]\n");
		356	DEFINE_FUNC1(get_rr, unsigned long,
		357	"mov r8 = rr[r8]\n");
		358
		359	DEFINE_VOID_FUNC0(ssm_i,
		360	"ssm psr.i\n");
		361	DEFINE_VOID_FUNC0(rsm_i,
		362	"rsm psr.i\n");
		363
		364	extern void
		365	ia64_native_set_rr0_to_rr4_func(unsigned long val0, unsigned long val1,
		366	unsigned long val2, unsigned long val3,
		367	unsigned long val4);
		368	__DEFINE_FUNC(set_rr0_to_rr4,
		369	"mov rr[r0] = r8\n"
		370	"movl r2 = 0x2000000000000000\n"
		371	";;\n"
		372	"mov rr[r2] = r9\n"
		373	"shl r3 = r2, 1\n" /* movl r3 = 0x4000000000000000 */
		374	";;\n"
		375	"add r2 = r2, r3\n" /* movl r2 = 0x6000000000000000 */
		376	"mov rr[r3] = r10\n"
		377	";;\n"
		378	"mov rr[r2] = r11\n"
		379	"shl r3 = r3, 1\n" /* movl r3 = 0x8000000000000000 */
		380	";;\n"
		381	"mov rr[r3] = r14\n");
		382
		383	extern unsigned long ia64_native_getreg_func(int regnum);
		384	asm(".global ia64_native_getreg_func\n");
		385	#define __DEFINE_GET_REG(id, reg) \
		386	"mov r2 = " __stringify(_IA64_REG_ ## id) "\n" \
		387	";;\n" \
		388	"cmp.eq p6, p0 = r2, r8\n" \
		389	";;\n" \
		390	"(p6) mov r8 = " #reg "\n" \
		391	"(p6) br.cond.sptk.many b6\n" \
		392	";;\n"
		393	#define __DEFINE_GET_AR(id, reg) __DEFINE_GET_REG(AR_ ## id, ar.reg)
		394	#define __DEFINE_GET_CR(id, reg) __DEFINE_GET_REG(CR_ ## id, cr.reg)
		395
		396	__DEFINE_FUNC(getreg,
		397	__DEFINE_GET_REG(GP, gp)
		398	/__DEFINE_GET_REG(IP, ip)/ /* returned ip value shouldn't be constant */
		399	__DEFINE_GET_REG(PSR, psr)
		400	__DEFINE_GET_REG(TP, tp)
		401	__DEFINE_GET_REG(SP, sp)
		402
		403	__DEFINE_GET_REG(AR_KR0, ar0)
		404	__DEFINE_GET_REG(AR_KR1, ar1)
		405	__DEFINE_GET_REG(AR_KR2, ar2)
		406	__DEFINE_GET_REG(AR_KR3, ar3)
		407	__DEFINE_GET_REG(AR_KR4, ar4)
		408	__DEFINE_GET_REG(AR_KR5, ar5)
		409	__DEFINE_GET_REG(AR_KR6, ar6)
		410	__DEFINE_GET_REG(AR_KR7, ar7)
		411	__DEFINE_GET_AR(RSC, rsc)
		412	__DEFINE_GET_AR(BSP, bsp)
		413	__DEFINE_GET_AR(BSPSTORE, bspstore)
		414	__DEFINE_GET_AR(RNAT, rnat)
		415	__DEFINE_GET_AR(FCR, fcr)
		416	__DEFINE_GET_AR(EFLAG, eflag)
		417	__DEFINE_GET_AR(CSD, csd)
		418	__DEFINE_GET_AR(SSD, ssd)
		419	__DEFINE_GET_REG(AR_CFLAG, ar27)
		420	__DEFINE_GET_AR(FSR, fsr)
		421	__DEFINE_GET_AR(FIR, fir)
		422	__DEFINE_GET_AR(FDR, fdr)
		423	__DEFINE_GET_AR(CCV, ccv)
		424	__DEFINE_GET_AR(UNAT, unat)
		425	__DEFINE_GET_AR(FPSR, fpsr)
		426	__DEFINE_GET_AR(ITC, itc)
		427	__DEFINE_GET_AR(PFS, pfs)
		428	__DEFINE_GET_AR(LC, lc)
		429	__DEFINE_GET_AR(EC, ec)
		430
		431	__DEFINE_GET_CR(DCR, dcr)
		432	__DEFINE_GET_CR(ITM, itm)
		433	__DEFINE_GET_CR(IVA, iva)
		434	__DEFINE_GET_CR(PTA, pta)
		435	__DEFINE_GET_CR(IPSR, ipsr)
		436	__DEFINE_GET_CR(ISR, isr)
		437	__DEFINE_GET_CR(IIP, iip)
		438	__DEFINE_GET_CR(IFA, ifa)
		439	__DEFINE_GET_CR(ITIR, itir)
		440	__DEFINE_GET_CR(IIPA, iipa)
		441	__DEFINE_GET_CR(IFS, ifs)
		442	__DEFINE_GET_CR(IIM, iim)
		443	__DEFINE_GET_CR(IHA, iha)
		444	__DEFINE_GET_CR(LID, lid)
		445	__DEFINE_GET_CR(IVR, ivr)
		446	__DEFINE_GET_CR(TPR, tpr)
		447	__DEFINE_GET_CR(EOI, eoi)
		448	__DEFINE_GET_CR(IRR0, irr0)
		449	__DEFINE_GET_CR(IRR1, irr1)
		450	__DEFINE_GET_CR(IRR2, irr2)
		451	__DEFINE_GET_CR(IRR3, irr3)
		452	__DEFINE_GET_CR(ITV, itv)
		453	__DEFINE_GET_CR(PMV, pmv)
		454	__DEFINE_GET_CR(CMCV, cmcv)
		455	__DEFINE_GET_CR(LRR0, lrr0)
		456	__DEFINE_GET_CR(LRR1, lrr1)
		457
		458	"mov r8 = -1\n" /* unsupported case */
		459	);
		460
		461	extern void ia64_native_setreg_func(int regnum, unsigned long val);
		462	asm(".global ia64_native_setreg_func\n");
		463	#define __DEFINE_SET_REG(id, reg) \
		464	"mov r2 = " __stringify(_IA64_REG_ ## id) "\n" \
		465	";;\n" \
		466	"cmp.eq p6, p0 = r2, r9\n" \
		467	";;\n" \
		468	"(p6) mov " #reg " = r8\n" \
		469	"(p6) br.cond.sptk.many b6\n" \
		470	";;\n"
		471	#define __DEFINE_SET_AR(id, reg) __DEFINE_SET_REG(AR_ ## id, ar.reg)
		472	#define __DEFINE_SET_CR(id, reg) __DEFINE_SET_REG(CR_ ## id, cr.reg)
		473	__DEFINE_FUNC(setreg,
		474	"mov r2 = " __stringify(_IA64_REG_PSR_L) "\n"
		475	";;\n"
		476	"cmp.eq p6, p0 = r2, r9\n"
		477	";;\n"
		478	"(p6) mov psr.l = r8\n"
		479	#ifdef HAVE_SERIALIZE_DIRECTIVE
		480	".serialize.data\n"
		481	#endif
		482	"(p6) br.cond.sptk.many b6\n"
		483	__DEFINE_SET_REG(GP, gp)
		484	__DEFINE_SET_REG(SP, sp)
		485
		486	__DEFINE_SET_REG(AR_KR0, ar0)
		487	__DEFINE_SET_REG(AR_KR1, ar1)
		488	__DEFINE_SET_REG(AR_KR2, ar2)
		489	__DEFINE_SET_REG(AR_KR3, ar3)
		490	__DEFINE_SET_REG(AR_KR4, ar4)
		491	__DEFINE_SET_REG(AR_KR5, ar5)
		492	__DEFINE_SET_REG(AR_KR6, ar6)
		493	__DEFINE_SET_REG(AR_KR7, ar7)
		494	__DEFINE_SET_AR(RSC, rsc)
		495	__DEFINE_SET_AR(BSP, bsp)
		496	__DEFINE_SET_AR(BSPSTORE, bspstore)
		497	__DEFINE_SET_AR(RNAT, rnat)
		498	__DEFINE_SET_AR(FCR, fcr)
		499	__DEFINE_SET_AR(EFLAG, eflag)
		500	__DEFINE_SET_AR(CSD, csd)
		501	__DEFINE_SET_AR(SSD, ssd)
		502	__DEFINE_SET_REG(AR_CFLAG, ar27)
		503	__DEFINE_SET_AR(FSR, fsr)
		504	__DEFINE_SET_AR(FIR, fir)
		505	__DEFINE_SET_AR(FDR, fdr)
		506	__DEFINE_SET_AR(CCV, ccv)
		507	__DEFINE_SET_AR(UNAT, unat)
		508	__DEFINE_SET_AR(FPSR, fpsr)
		509	__DEFINE_SET_AR(ITC, itc)
		510	__DEFINE_SET_AR(PFS, pfs)
		511	__DEFINE_SET_AR(LC, lc)
		512	__DEFINE_SET_AR(EC, ec)
		513
		514	__DEFINE_SET_CR(DCR, dcr)
		515	__DEFINE_SET_CR(ITM, itm)
		516	__DEFINE_SET_CR(IVA, iva)
		517	__DEFINE_SET_CR(PTA, pta)
		518	__DEFINE_SET_CR(IPSR, ipsr)
		519	__DEFINE_SET_CR(ISR, isr)
		520	__DEFINE_SET_CR(IIP, iip)
		521	__DEFINE_SET_CR(IFA, ifa)
		522	__DEFINE_SET_CR(ITIR, itir)
		523	__DEFINE_SET_CR(IIPA, iipa)
		524	__DEFINE_SET_CR(IFS, ifs)
		525	__DEFINE_SET_CR(IIM, iim)
		526	__DEFINE_SET_CR(IHA, iha)
		527	__DEFINE_SET_CR(LID, lid)
		528	__DEFINE_SET_CR(IVR, ivr)
		529	__DEFINE_SET_CR(TPR, tpr)
		530	__DEFINE_SET_CR(EOI, eoi)
		531	__DEFINE_SET_CR(IRR0, irr0)
		532	__DEFINE_SET_CR(IRR1, irr1)
		533	__DEFINE_SET_CR(IRR2, irr2)
		534	__DEFINE_SET_CR(IRR3, irr3)
		535	__DEFINE_SET_CR(ITV, itv)
		536	__DEFINE_SET_CR(PMV, pmv)
		537	__DEFINE_SET_CR(CMCV, cmcv)
		538	__DEFINE_SET_CR(LRR0, lrr0)
		539	__DEFINE_SET_CR(LRR1, lrr1)
		540	);
		541	#endif
277		542
278	struct pv_cpu_ops pv_cpu_ops = {	543	struct pv_cpu_ops pv_cpu_ops = {
279	.fc = ia64_native_fc_func,	544	.fc = ia64_native_fc_func,
@@ -368,3 +633,256 @@ struct pv_time_ops pv_time_ops = {
368	.do_steal_accounting = ia64_native_do_steal_accounting,	633	.do_steal_accounting = ia64_native_do_steal_accounting,
369	.sched_clock = ia64_native_sched_clock,	634	.sched_clock = ia64_native_sched_clock,
370	};	635	};
		636
		637	/***************************************************************************
		638	* binary pacthing
		639	* pv_init_ops.patch_bundle
		640	*/
		641
		642	#ifdef ASM_SUPPORTED
		643	#define IA64_NATIVE_PATCH_DEFINE_GET_REG(name, reg) \
		644	__DEFINE_FUNC(get_ ## name, \
		645	";;\n" \
		646	"mov r8 = " #reg "\n" \
		647	";;\n")
		648
		649	#define IA64_NATIVE_PATCH_DEFINE_SET_REG(name, reg) \
		650	__DEFINE_FUNC(set_ ## name, \
		651	";;\n" \
		652	"mov " #reg " = r8\n" \
		653	";;\n")
		654
		655	#define IA64_NATIVE_PATCH_DEFINE_REG(name, reg) \
		656	IA64_NATIVE_PATCH_DEFINE_GET_REG(name, reg); \
		657	IA64_NATIVE_PATCH_DEFINE_SET_REG(name, reg) \
		658
		659	#define IA64_NATIVE_PATCH_DEFINE_AR(name, reg) \
		660	IA64_NATIVE_PATCH_DEFINE_REG(ar_ ## name, ar.reg)
		661
		662	#define IA64_NATIVE_PATCH_DEFINE_CR(name, reg) \
		663	IA64_NATIVE_PATCH_DEFINE_REG(cr_ ## name, cr.reg)
		664
		665
		666	IA64_NATIVE_PATCH_DEFINE_GET_REG(psr, psr);
		667	IA64_NATIVE_PATCH_DEFINE_GET_REG(tp, tp);
		668
		669	/* IA64_NATIVE_PATCH_DEFINE_SET_REG(psr_l, psr.l); */
		670	__DEFINE_FUNC(set_psr_l,
		671	";;\n"
		672	"mov psr.l = r8\n"
		673	#ifdef HAVE_SERIALIZE_DIRECTIVE
		674	".serialize.data\n"
		675	#endif
		676	";;\n");
		677
		678	IA64_NATIVE_PATCH_DEFINE_REG(gp, gp);
		679	IA64_NATIVE_PATCH_DEFINE_REG(sp, sp);
		680
		681	IA64_NATIVE_PATCH_DEFINE_REG(kr0, ar0);
		682	IA64_NATIVE_PATCH_DEFINE_REG(kr1, ar1);
		683	IA64_NATIVE_PATCH_DEFINE_REG(kr2, ar2);
		684	IA64_NATIVE_PATCH_DEFINE_REG(kr3, ar3);
		685	IA64_NATIVE_PATCH_DEFINE_REG(kr4, ar4);
		686	IA64_NATIVE_PATCH_DEFINE_REG(kr5, ar5);
		687	IA64_NATIVE_PATCH_DEFINE_REG(kr6, ar6);
		688	IA64_NATIVE_PATCH_DEFINE_REG(kr7, ar7);
		689
		690	IA64_NATIVE_PATCH_DEFINE_AR(rsc, rsc);
		691	IA64_NATIVE_PATCH_DEFINE_AR(bsp, bsp);
		692	IA64_NATIVE_PATCH_DEFINE_AR(bspstore, bspstore);
		693	IA64_NATIVE_PATCH_DEFINE_AR(rnat, rnat);
		694	IA64_NATIVE_PATCH_DEFINE_AR(fcr, fcr);
		695	IA64_NATIVE_PATCH_DEFINE_AR(eflag, eflag);
		696	IA64_NATIVE_PATCH_DEFINE_AR(csd, csd);
		697	IA64_NATIVE_PATCH_DEFINE_AR(ssd, ssd);
		698	IA64_NATIVE_PATCH_DEFINE_REG(ar27, ar27);
		699	IA64_NATIVE_PATCH_DEFINE_AR(fsr, fsr);
		700	IA64_NATIVE_PATCH_DEFINE_AR(fir, fir);
		701	IA64_NATIVE_PATCH_DEFINE_AR(fdr, fdr);
		702	IA64_NATIVE_PATCH_DEFINE_AR(ccv, ccv);
		703	IA64_NATIVE_PATCH_DEFINE_AR(unat, unat);
		704	IA64_NATIVE_PATCH_DEFINE_AR(fpsr, fpsr);
		705	IA64_NATIVE_PATCH_DEFINE_AR(itc, itc);
		706	IA64_NATIVE_PATCH_DEFINE_AR(pfs, pfs);
		707	IA64_NATIVE_PATCH_DEFINE_AR(lc, lc);
		708	IA64_NATIVE_PATCH_DEFINE_AR(ec, ec);
		709
		710	IA64_NATIVE_PATCH_DEFINE_CR(dcr, dcr);
		711	IA64_NATIVE_PATCH_DEFINE_CR(itm, itm);
		712	IA64_NATIVE_PATCH_DEFINE_CR(iva, iva);
		713	IA64_NATIVE_PATCH_DEFINE_CR(pta, pta);
		714	IA64_NATIVE_PATCH_DEFINE_CR(ipsr, ipsr);
		715	IA64_NATIVE_PATCH_DEFINE_CR(isr, isr);
		716	IA64_NATIVE_PATCH_DEFINE_CR(iip, iip);
		717	IA64_NATIVE_PATCH_DEFINE_CR(ifa, ifa);
		718	IA64_NATIVE_PATCH_DEFINE_CR(itir, itir);
		719	IA64_NATIVE_PATCH_DEFINE_CR(iipa, iipa);
		720	IA64_NATIVE_PATCH_DEFINE_CR(ifs, ifs);
		721	IA64_NATIVE_PATCH_DEFINE_CR(iim, iim);
		722	IA64_NATIVE_PATCH_DEFINE_CR(iha, iha);
		723	IA64_NATIVE_PATCH_DEFINE_CR(lid, lid);
		724	IA64_NATIVE_PATCH_DEFINE_CR(ivr, ivr);
		725	IA64_NATIVE_PATCH_DEFINE_CR(tpr, tpr);
		726	IA64_NATIVE_PATCH_DEFINE_CR(eoi, eoi);
		727	IA64_NATIVE_PATCH_DEFINE_CR(irr0, irr0);
		728	IA64_NATIVE_PATCH_DEFINE_CR(irr1, irr1);
		729	IA64_NATIVE_PATCH_DEFINE_CR(irr2, irr2);
		730	IA64_NATIVE_PATCH_DEFINE_CR(irr3, irr3);
		731	IA64_NATIVE_PATCH_DEFINE_CR(itv, itv);
		732	IA64_NATIVE_PATCH_DEFINE_CR(pmv, pmv);
		733	IA64_NATIVE_PATCH_DEFINE_CR(cmcv, cmcv);
		734	IA64_NATIVE_PATCH_DEFINE_CR(lrr0, lrr0);
		735	IA64_NATIVE_PATCH_DEFINE_CR(lrr1, lrr1);
		736
		737	static const struct paravirt_patch_bundle_elem ia64_native_patch_bundle_elems[]
		738	__initdata_or_module =
		739	{
		740	#define IA64_NATIVE_PATCH_BUNDLE_ELEM(name, type) \
		741	{ \
		742	(void*)ia64_native_ ## name ## _direct_start, \
		743	(void*)ia64_native_ ## name ## _direct_end, \
		744	PARAVIRT_PATCH_TYPE_ ## type, \
		745	}
		746
		747	IA64_NATIVE_PATCH_BUNDLE_ELEM(fc, FC),
		748	IA64_NATIVE_PATCH_BUNDLE_ELEM(thash, THASH),
		749	IA64_NATIVE_PATCH_BUNDLE_ELEM(get_cpuid, GET_CPUID),
		750	IA64_NATIVE_PATCH_BUNDLE_ELEM(get_pmd, GET_PMD),
		751	IA64_NATIVE_PATCH_BUNDLE_ELEM(ptcga, PTCGA),
		752	IA64_NATIVE_PATCH_BUNDLE_ELEM(get_rr, GET_RR),
		753	IA64_NATIVE_PATCH_BUNDLE_ELEM(set_rr, SET_RR),
		754	IA64_NATIVE_PATCH_BUNDLE_ELEM(set_rr0_to_rr4, SET_RR0_TO_RR4),
		755	IA64_NATIVE_PATCH_BUNDLE_ELEM(ssm_i, SSM_I),
		756	IA64_NATIVE_PATCH_BUNDLE_ELEM(rsm_i, RSM_I),
		757	IA64_NATIVE_PATCH_BUNDLE_ELEM(get_psr_i, GET_PSR_I),
		758	IA64_NATIVE_PATCH_BUNDLE_ELEM(intrin_local_irq_restore,
		759	INTRIN_LOCAL_IRQ_RESTORE),
		760
		761	#define IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(name, reg) \
		762	{ \
		763	(void*)ia64_native_get_ ## name ## _direct_start, \
		764	(void*)ia64_native_get_ ## name ## _direct_end, \
		765	PARAVIRT_PATCH_TYPE_GETREG + _IA64_REG_ ## reg, \
		766	}
		767
		768	#define IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(name, reg) \
		769	{ \
		770	(void*)ia64_native_set_ ## name ## _direct_start, \
		771	(void*)ia64_native_set_ ## name ## _direct_end, \
		772	PARAVIRT_PATCH_TYPE_SETREG + _IA64_REG_ ## reg, \
		773	}
		774
		775	#define IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(name, reg) \
		776	IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(name, reg), \
		777	IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(name, reg) \
		778
		779	#define IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(name, reg) \
		780	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(ar_ ## name, AR_ ## reg)
		781
		782	#define IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(name, reg) \
		783	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(cr_ ## name, CR_ ## reg)
		784
		785	IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(psr, PSR),
		786	IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(tp, TP),
		787
		788	IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(psr_l, PSR_L),
		789
		790	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(gp, GP),
		791	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(sp, SP),
		792
		793	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr0, AR_KR0),
		794	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr1, AR_KR1),
		795	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr2, AR_KR2),
		796	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr3, AR_KR3),
		797	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr4, AR_KR4),
		798	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr5, AR_KR5),
		799	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr6, AR_KR6),
		800	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr7, AR_KR7),
		801
		802	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(rsc, RSC),
		803	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(bsp, BSP),
		804	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(bspstore, BSPSTORE),
		805	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(rnat, RNAT),
		806	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fcr, FCR),
		807	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(eflag, EFLAG),
		808	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(csd, CSD),
		809	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ssd, SSD),
		810	IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(ar27, AR_CFLAG),
		811	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fsr, FSR),
		812	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fir, FIR),
		813	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fdr, FDR),
		814	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ccv, CCV),
		815	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(unat, UNAT),
		816	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fpsr, FPSR),
		817	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(itc, ITC),
		818	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(pfs, PFS),
		819	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(lc, LC),
		820	IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ec, EC),
		821
		822	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(dcr, DCR),
		823	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itm, ITM),
		824	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iva, IVA),
		825	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(pta, PTA),
		826	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ipsr, IPSR),
		827	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(isr, ISR),
		828	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iip, IIP),
		829	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ifa, IFA),
		830	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itir, ITIR),
		831	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iipa, IIPA),
		832	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ifs, IFS),
		833	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iim, IIM),
		834	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iha, IHA),
		835	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lid, LID),
		836	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ivr, IVR),
		837	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(tpr, TPR),
		838	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(eoi, EOI),
		839	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr0, IRR0),
		840	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr1, IRR1),
		841	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr2, IRR2),
		842	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr3, IRR3),
		843	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itv, ITV),
		844	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(pmv, PMV),
		845	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(cmcv, CMCV),
		846	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lrr0, LRR0),
		847	IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lrr1, LRR1),
		848	};
		849
		850	unsigned long __init_or_module
		851	ia64_native_patch_bundle(void sbundle, void ebundle, unsigned long type)
		852	{
		853	const unsigned long nelems = sizeof(ia64_native_patch_bundle_elems) /
		854	sizeof(ia64_native_patch_bundle_elems[0]);
		855
		856	return __paravirt_patch_apply_bundle(sbundle, ebundle, type,
		857	ia64_native_patch_bundle_elems,
		858	nelems, NULL);
		859	}
		860	#endif /* ASM_SUPPOTED */
		861
		862	extern const char ia64_native_switch_to[];
		863	extern const char ia64_native_leave_syscall[];
		864	extern const char ia64_native_work_processed_syscall[];
		865	extern const char ia64_native_leave_kernel[];
		866
		867	const struct paravirt_patch_branch_target ia64_native_branch_target[]
		868	__initconst = {
		869	#define PARAVIRT_BR_TARGET(name, type) \
		870	{ \
		871	ia64_native_ ## name, \
		872	PARAVIRT_PATCH_TYPE_BR_ ## type, \
		873	}
		874	PARAVIRT_BR_TARGET(switch_to, SWITCH_TO),
		875	PARAVIRT_BR_TARGET(leave_syscall, LEAVE_SYSCALL),
		876	PARAVIRT_BR_TARGET(work_processed_syscall, WORK_PROCESSED_SYSCALL),
		877	PARAVIRT_BR_TARGET(leave_kernel, LEAVE_KERNEL),
		878	};
		879
		880	static void __init
		881	ia64_native_patch_branch(unsigned long tag, unsigned long type)
		882	{
		883	const unsigned long nelem =
		884	sizeof(ia64_native_branch_target) /
		885	sizeof(ia64_native_branch_target[0]);
		886	__paravirt_patch_apply_branch(tag, type,
		887	ia64_native_branch_target, nelem);
		888	}