aboutsummaryrefslogtreecommitdiffstats
path: root/arch/ia64/xen
diff options
context:
space:
mode:
Diffstat (limited to 'arch/ia64/xen')
-rw-r--r--arch/ia64/xen/Makefile19
-rw-r--r--arch/ia64/xen/gate-data.S3
-rw-r--r--arch/ia64/xen/hypercall.S2
-rw-r--r--arch/ia64/xen/time.c48
-rw-r--r--arch/ia64/xen/xen_pv_ops.c800
5 files changed, 867 insertions, 5 deletions
diff --git a/arch/ia64/xen/Makefile b/arch/ia64/xen/Makefile
index 0ad0224693d9..e6f4a0a74228 100644
--- a/arch/ia64/xen/Makefile
+++ b/arch/ia64/xen/Makefile
@@ -3,14 +3,29 @@
3# 3#
4 4
5obj-y := hypercall.o xenivt.o xensetup.o xen_pv_ops.o irq_xen.o \ 5obj-y := hypercall.o xenivt.o xensetup.o xen_pv_ops.o irq_xen.o \
6 hypervisor.o xencomm.o xcom_hcall.o grant-table.o time.o suspend.o 6 hypervisor.o xencomm.o xcom_hcall.o grant-table.o time.o suspend.o \
7 gate-data.o
7 8
8obj-$(CONFIG_IA64_GENERIC) += machvec.o 9obj-$(CONFIG_IA64_GENERIC) += machvec.o
9 10
11# The gate DSO image is built using a special linker script.
12include $(srctree)/arch/ia64/kernel/Makefile.gate
13
14# tell compiled for xen
15CPPFLAGS_gate.lds += -D__IA64_GATE_PARAVIRTUALIZED_XEN
16AFLAGS_gate.o += -D__IA64_ASM_PARAVIRTUALIZED_XEN -D__IA64_GATE_PARAVIRTUALIZED_XEN
17
18# use same file of native.
19$(obj)/gate.o: $(src)/../kernel/gate.S FORCE
20 $(call if_changed_dep,as_o_S)
21$(obj)/gate.lds: $(src)/../kernel/gate.lds.S FORCE
22 $(call if_changed_dep,cpp_lds_S)
23
24
10AFLAGS_xenivt.o += -D__IA64_ASM_PARAVIRTUALIZED_XEN 25AFLAGS_xenivt.o += -D__IA64_ASM_PARAVIRTUALIZED_XEN
11 26
12# xen multi compile 27# xen multi compile
13ASM_PARAVIRT_MULTI_COMPILE_SRCS = ivt.S entry.S 28ASM_PARAVIRT_MULTI_COMPILE_SRCS = ivt.S entry.S fsys.S
14ASM_PARAVIRT_OBJS = $(addprefix xen-,$(ASM_PARAVIRT_MULTI_COMPILE_SRCS:.S=.o)) 29ASM_PARAVIRT_OBJS = $(addprefix xen-,$(ASM_PARAVIRT_MULTI_COMPILE_SRCS:.S=.o))
15obj-y += $(ASM_PARAVIRT_OBJS) 30obj-y += $(ASM_PARAVIRT_OBJS)
16define paravirtualized_xen 31define paravirtualized_xen
diff --git a/arch/ia64/xen/gate-data.S b/arch/ia64/xen/gate-data.S
new file mode 100644
index 000000000000..7d4830afc91d
--- /dev/null
+++ b/arch/ia64/xen/gate-data.S
@@ -0,0 +1,3 @@
1 .section .data.gate.xen, "aw"
2
3 .incbin "arch/ia64/xen/gate.so"
diff --git a/arch/ia64/xen/hypercall.S b/arch/ia64/xen/hypercall.S
index 45e02bb64a92..e32dae444dd6 100644
--- a/arch/ia64/xen/hypercall.S
+++ b/arch/ia64/xen/hypercall.S
@@ -9,6 +9,7 @@
9#include <asm/intrinsics.h> 9#include <asm/intrinsics.h>
10#include <asm/xen/privop.h> 10#include <asm/xen/privop.h>
11 11
12#ifdef __INTEL_COMPILER
12/* 13/*
13 * Hypercalls without parameter. 14 * Hypercalls without parameter.
14 */ 15 */
@@ -72,6 +73,7 @@ GLOBAL_ENTRY(xen_set_rr0_to_rr4)
72 br.ret.sptk.many rp 73 br.ret.sptk.many rp
73 ;; 74 ;;
74END(xen_set_rr0_to_rr4) 75END(xen_set_rr0_to_rr4)
76#endif
75 77
76GLOBAL_ENTRY(xen_send_ipi) 78GLOBAL_ENTRY(xen_send_ipi)
77 mov r14=r32 79 mov r14=r32
diff --git a/arch/ia64/xen/time.c b/arch/ia64/xen/time.c
index 68d6204c3f16..fb8332690179 100644
--- a/arch/ia64/xen/time.c
+++ b/arch/ia64/xen/time.c
@@ -175,10 +175,58 @@ static void xen_itc_jitter_data_reset(void)
175 } while (unlikely(ret != lcycle)); 175 } while (unlikely(ret != lcycle));
176} 176}
177 177
178/* based on xen_sched_clock() in arch/x86/xen/time.c. */
179/*
180 * This relies on HAVE_UNSTABLE_SCHED_CLOCK. If it can't be defined,
181 * something similar logic should be implemented here.
182 */
183/*
184 * Xen sched_clock implementation. Returns the number of unstolen
185 * nanoseconds, which is nanoseconds the VCPU spent in RUNNING+BLOCKED
186 * states.
187 */
188static unsigned long long xen_sched_clock(void)
189{
190 struct vcpu_runstate_info runstate;
191
192 unsigned long long now;
193 unsigned long long offset;
194 unsigned long long ret;
195
196 /*
197 * Ideally sched_clock should be called on a per-cpu basis
198 * anyway, so preempt should already be disabled, but that's
199 * not current practice at the moment.
200 */
201 preempt_disable();
202
203 /*
204 * both ia64_native_sched_clock() and xen's runstate are
205 * based on mAR.ITC. So difference of them makes sense.
206 */
207 now = ia64_native_sched_clock();
208
209 get_runstate_snapshot(&runstate);
210
211 WARN_ON(runstate.state != RUNSTATE_running);
212
213 offset = 0;
214 if (now > runstate.state_entry_time)
215 offset = now - runstate.state_entry_time;
216 ret = runstate.time[RUNSTATE_blocked] +
217 runstate.time[RUNSTATE_running] +
218 offset;
219
220 preempt_enable();
221
222 return ret;
223}
224
178struct pv_time_ops xen_time_ops __initdata = { 225struct pv_time_ops xen_time_ops __initdata = {
179 .init_missing_ticks_accounting = xen_init_missing_ticks_accounting, 226 .init_missing_ticks_accounting = xen_init_missing_ticks_accounting,
180 .do_steal_accounting = xen_do_steal_accounting, 227 .do_steal_accounting = xen_do_steal_accounting,
181 .clocksource_resume = xen_itc_jitter_data_reset, 228 .clocksource_resume = xen_itc_jitter_data_reset,
229 .sched_clock = xen_sched_clock,
182}; 230};
183 231
184/* Called after suspend, to resume time. */ 232/* Called after suspend, to resume time. */
diff --git a/arch/ia64/xen/xen_pv_ops.c b/arch/ia64/xen/xen_pv_ops.c
index 936cff3c96e0..5e2270a999fa 100644
--- a/arch/ia64/xen/xen_pv_ops.c
+++ b/arch/ia64/xen/xen_pv_ops.c
@@ -24,6 +24,7 @@
24#include <linux/irq.h> 24#include <linux/irq.h>
25#include <linux/kernel.h> 25#include <linux/kernel.h>
26#include <linux/pm.h> 26#include <linux/pm.h>
27#include <linux/unistd.h>
27 28
28#include <asm/xen/hypervisor.h> 29#include <asm/xen/hypervisor.h>
29#include <asm/xen/xencomm.h> 30#include <asm/xen/xencomm.h>
@@ -153,6 +154,13 @@ xen_post_smp_prepare_boot_cpu(void)
153 xen_setup_vcpu_info_placement(); 154 xen_setup_vcpu_info_placement();
154} 155}
155 156
157#ifdef ASM_SUPPORTED
158static unsigned long __init_or_module
159xen_patch_bundle(void *sbundle, void *ebundle, unsigned long type);
160#endif
161static void __init
162xen_patch_branch(unsigned long tag, unsigned long type);
163
156static const struct pv_init_ops xen_init_ops __initconst = { 164static const struct pv_init_ops xen_init_ops __initconst = {
157 .banner = xen_banner, 165 .banner = xen_banner,
158 166
@@ -163,6 +171,53 @@ static const struct pv_init_ops xen_init_ops __initconst = {
163 .arch_setup_nomca = xen_arch_setup_nomca, 171 .arch_setup_nomca = xen_arch_setup_nomca,
164 172
165 .post_smp_prepare_boot_cpu = xen_post_smp_prepare_boot_cpu, 173 .post_smp_prepare_boot_cpu = xen_post_smp_prepare_boot_cpu,
174#ifdef ASM_SUPPORTED
175 .patch_bundle = xen_patch_bundle,
176#endif
177 .patch_branch = xen_patch_branch,
178};
179
180/***************************************************************************
181 * pv_fsys_data
182 * addresses for fsys
183 */
184
185extern unsigned long xen_fsyscall_table[NR_syscalls];
186extern char xen_fsys_bubble_down[];
187struct pv_fsys_data xen_fsys_data __initdata = {
188 .fsyscall_table = (unsigned long *)xen_fsyscall_table,
189 .fsys_bubble_down = (void *)xen_fsys_bubble_down,
190};
191
192/***************************************************************************
193 * pv_patchdata
194 * patchdata addresses
195 */
196
197#define DECLARE(name) \
198 extern unsigned long __xen_start_gate_##name##_patchlist[]; \
199 extern unsigned long __xen_end_gate_##name##_patchlist[]
200
201DECLARE(fsyscall);
202DECLARE(brl_fsys_bubble_down);
203DECLARE(vtop);
204DECLARE(mckinley_e9);
205
206extern unsigned long __xen_start_gate_section[];
207
208#define ASSIGN(name) \
209 .start_##name##_patchlist = \
210 (unsigned long)__xen_start_gate_##name##_patchlist, \
211 .end_##name##_patchlist = \
212 (unsigned long)__xen_end_gate_##name##_patchlist
213
214static struct pv_patchdata xen_patchdata __initdata = {
215 ASSIGN(fsyscall),
216 ASSIGN(brl_fsys_bubble_down),
217 ASSIGN(vtop),
218 ASSIGN(mckinley_e9),
219
220 .gate_section = (void*)__xen_start_gate_section,
166}; 221};
167 222
168/*************************************************************************** 223/***************************************************************************
@@ -170,6 +225,76 @@ static const struct pv_init_ops xen_init_ops __initconst = {
170 * intrinsics hooks. 225 * intrinsics hooks.
171 */ 226 */
172 227
228#ifndef ASM_SUPPORTED
229static void
230xen_set_itm_with_offset(unsigned long val)
231{
232 /* ia64_cpu_local_tick() calls this with interrupt enabled. */
233 /* WARN_ON(!irqs_disabled()); */
234 xen_set_itm(val - XEN_MAPPEDREGS->itc_offset);
235}
236
237static unsigned long
238xen_get_itm_with_offset(void)
239{
240 /* unused at this moment */
241 printk(KERN_DEBUG "%s is called.\n", __func__);
242
243 WARN_ON(!irqs_disabled());
244 return ia64_native_getreg(_IA64_REG_CR_ITM) +
245 XEN_MAPPEDREGS->itc_offset;
246}
247
248/* ia64_set_itc() is only called by
249 * cpu_init() with ia64_set_itc(0) and ia64_sync_itc().
250 * So XEN_MAPPEDRESG->itc_offset cal be considered as almost constant.
251 */
252static void
253xen_set_itc(unsigned long val)
254{
255 unsigned long mitc;
256
257 WARN_ON(!irqs_disabled());
258 mitc = ia64_native_getreg(_IA64_REG_AR_ITC);
259 XEN_MAPPEDREGS->itc_offset = val - mitc;
260 XEN_MAPPEDREGS->itc_last = val;
261}
262
263static unsigned long
264xen_get_itc(void)
265{
266 unsigned long res;
267 unsigned long itc_offset;
268 unsigned long itc_last;
269 unsigned long ret_itc_last;
270
271 itc_offset = XEN_MAPPEDREGS->itc_offset;
272 do {
273 itc_last = XEN_MAPPEDREGS->itc_last;
274 res = ia64_native_getreg(_IA64_REG_AR_ITC);
275 res += itc_offset;
276 if (itc_last >= res)
277 res = itc_last + 1;
278 ret_itc_last = cmpxchg(&XEN_MAPPEDREGS->itc_last,
279 itc_last, res);
280 } while (unlikely(ret_itc_last != itc_last));
281 return res;
282
283#if 0
284 /* ia64_itc_udelay() calls ia64_get_itc() with interrupt enabled.
285 Should it be paravirtualized instead? */
286 WARN_ON(!irqs_disabled());
287 itc_offset = XEN_MAPPEDREGS->itc_offset;
288 itc_last = XEN_MAPPEDREGS->itc_last;
289 res = ia64_native_getreg(_IA64_REG_AR_ITC);
290 res += itc_offset;
291 if (itc_last >= res)
292 res = itc_last + 1;
293 XEN_MAPPEDREGS->itc_last = res;
294 return res;
295#endif
296}
297
173static void xen_setreg(int regnum, unsigned long val) 298static void xen_setreg(int regnum, unsigned long val)
174{ 299{
175 switch (regnum) { 300 switch (regnum) {
@@ -181,11 +306,14 @@ static void xen_setreg(int regnum, unsigned long val)
181 xen_set_eflag(val); 306 xen_set_eflag(val);
182 break; 307 break;
183#endif 308#endif
309 case _IA64_REG_AR_ITC:
310 xen_set_itc(val);
311 break;
184 case _IA64_REG_CR_TPR: 312 case _IA64_REG_CR_TPR:
185 xen_set_tpr(val); 313 xen_set_tpr(val);
186 break; 314 break;
187 case _IA64_REG_CR_ITM: 315 case _IA64_REG_CR_ITM:
188 xen_set_itm(val); 316 xen_set_itm_with_offset(val);
189 break; 317 break;
190 case _IA64_REG_CR_EOI: 318 case _IA64_REG_CR_EOI:
191 xen_eoi(val); 319 xen_eoi(val);
@@ -209,6 +337,12 @@ static unsigned long xen_getreg(int regnum)
209 res = xen_get_eflag(); 337 res = xen_get_eflag();
210 break; 338 break;
211#endif 339#endif
340 case _IA64_REG_AR_ITC:
341 res = xen_get_itc();
342 break;
343 case _IA64_REG_CR_ITM:
344 res = xen_get_itm_with_offset();
345 break;
212 case _IA64_REG_CR_IVR: 346 case _IA64_REG_CR_IVR:
213 res = xen_get_ivr(); 347 res = xen_get_ivr();
214 break; 348 break;
@@ -259,8 +393,417 @@ xen_intrin_local_irq_restore(unsigned long mask)
259 else 393 else
260 xen_rsm_i(); 394 xen_rsm_i();
261} 395}
396#else
397#define __DEFINE_FUNC(name, code) \
398 extern const char xen_ ## name ## _direct_start[]; \
399 extern const char xen_ ## name ## _direct_end[]; \
400 asm (".align 32\n" \
401 ".proc xen_" #name "\n" \
402 "xen_" #name ":\n" \
403 "xen_" #name "_direct_start:\n" \
404 code \
405 "xen_" #name "_direct_end:\n" \
406 "br.cond.sptk.many b6\n" \
407 ".endp xen_" #name "\n")
408
409#define DEFINE_VOID_FUNC0(name, code) \
410 extern void \
411 xen_ ## name (void); \
412 __DEFINE_FUNC(name, code)
413
414#define DEFINE_VOID_FUNC1(name, code) \
415 extern void \
416 xen_ ## name (unsigned long arg); \
417 __DEFINE_FUNC(name, code)
418
419#define DEFINE_VOID_FUNC1_VOID(name, code) \
420 extern void \
421 xen_ ## name (void *arg); \
422 __DEFINE_FUNC(name, code)
423
424#define DEFINE_VOID_FUNC2(name, code) \
425 extern void \
426 xen_ ## name (unsigned long arg0, \
427 unsigned long arg1); \
428 __DEFINE_FUNC(name, code)
262 429
263static const struct pv_cpu_ops xen_cpu_ops __initdata = { 430#define DEFINE_FUNC0(name, code) \
431 extern unsigned long \
432 xen_ ## name (void); \
433 __DEFINE_FUNC(name, code)
434
435#define DEFINE_FUNC1(name, type, code) \
436 extern unsigned long \
437 xen_ ## name (type arg); \
438 __DEFINE_FUNC(name, code)
439
440#define XEN_PSR_I_ADDR_ADDR (XSI_BASE + XSI_PSR_I_ADDR_OFS)
441
442/*
443 * static void xen_set_itm_with_offset(unsigned long val)
444 * xen_set_itm(val - XEN_MAPPEDREGS->itc_offset);
445 */
446/* 2 bundles */
447DEFINE_VOID_FUNC1(set_itm_with_offset,
448 "mov r2 = " __stringify(XSI_BASE) " + "
449 __stringify(XSI_ITC_OFFSET_OFS) "\n"
450 ";;\n"
451 "ld8 r3 = [r2]\n"
452 ";;\n"
453 "sub r8 = r8, r3\n"
454 "break " __stringify(HYPERPRIVOP_SET_ITM) "\n");
455
456/*
457 * static unsigned long xen_get_itm_with_offset(void)
458 * return ia64_native_getreg(_IA64_REG_CR_ITM) + XEN_MAPPEDREGS->itc_offset;
459 */
460/* 2 bundles */
461DEFINE_FUNC0(get_itm_with_offset,
462 "mov r2 = " __stringify(XSI_BASE) " + "
463 __stringify(XSI_ITC_OFFSET_OFS) "\n"
464 ";;\n"
465 "ld8 r3 = [r2]\n"
466 "mov r8 = cr.itm\n"
467 ";;\n"
468 "add r8 = r8, r2\n");
469
470/*
471 * static void xen_set_itc(unsigned long val)
472 * unsigned long mitc;
473 *
474 * WARN_ON(!irqs_disabled());
475 * mitc = ia64_native_getreg(_IA64_REG_AR_ITC);
476 * XEN_MAPPEDREGS->itc_offset = val - mitc;
477 * XEN_MAPPEDREGS->itc_last = val;
478 */
479/* 2 bundles */
480DEFINE_VOID_FUNC1(set_itc,
481 "mov r2 = " __stringify(XSI_BASE) " + "
482 __stringify(XSI_ITC_LAST_OFS) "\n"
483 "mov r3 = ar.itc\n"
484 ";;\n"
485 "sub r3 = r8, r3\n"
486 "st8 [r2] = r8, "
487 __stringify(XSI_ITC_LAST_OFS) " - "
488 __stringify(XSI_ITC_OFFSET_OFS) "\n"
489 ";;\n"
490 "st8 [r2] = r3\n");
491
492/*
493 * static unsigned long xen_get_itc(void)
494 * unsigned long res;
495 * unsigned long itc_offset;
496 * unsigned long itc_last;
497 * unsigned long ret_itc_last;
498 *
499 * itc_offset = XEN_MAPPEDREGS->itc_offset;
500 * do {
501 * itc_last = XEN_MAPPEDREGS->itc_last;
502 * res = ia64_native_getreg(_IA64_REG_AR_ITC);
503 * res += itc_offset;
504 * if (itc_last >= res)
505 * res = itc_last + 1;
506 * ret_itc_last = cmpxchg(&XEN_MAPPEDREGS->itc_last,
507 * itc_last, res);
508 * } while (unlikely(ret_itc_last != itc_last));
509 * return res;
510 */
511/* 5 bundles */
512DEFINE_FUNC0(get_itc,
513 "mov r2 = " __stringify(XSI_BASE) " + "
514 __stringify(XSI_ITC_OFFSET_OFS) "\n"
515 ";;\n"
516 "ld8 r9 = [r2], " __stringify(XSI_ITC_LAST_OFS) " - "
517 __stringify(XSI_ITC_OFFSET_OFS) "\n"
518 /* r9 = itc_offset */
519 /* r2 = XSI_ITC_OFFSET */
520 "888:\n"
521 "mov r8 = ar.itc\n" /* res = ar.itc */
522 ";;\n"
523 "ld8 r3 = [r2]\n" /* r3 = itc_last */
524 "add r8 = r8, r9\n" /* res = ar.itc + itc_offset */
525 ";;\n"
526 "cmp.gtu p6, p0 = r3, r8\n"
527 ";;\n"
528 "(p6) add r8 = 1, r3\n" /* if (itc_last > res) itc_last + 1 */
529 ";;\n"
530 "mov ar.ccv = r8\n"
531 ";;\n"
532 "cmpxchg8.acq r10 = [r2], r8, ar.ccv\n"
533 ";;\n"
534 "cmp.ne p6, p0 = r10, r3\n"
535 "(p6) hint @pause\n"
536 "(p6) br.cond.spnt 888b\n");
537
538DEFINE_VOID_FUNC1_VOID(fc,
539 "break " __stringify(HYPERPRIVOP_FC) "\n");
540
541/*
542 * psr_i_addr_addr = XEN_PSR_I_ADDR_ADDR
543 * masked_addr = *psr_i_addr_addr
544 * pending_intr_addr = masked_addr - 1
545 * if (val & IA64_PSR_I) {
546 * masked = *masked_addr
547 * *masked_addr = 0:xen_set_virtual_psr_i(1)
548 * compiler barrier
549 * if (masked) {
550 * uint8_t pending = *pending_intr_addr;
551 * if (pending)
552 * XEN_HYPER_SSM_I
553 * }
554 * } else {
555 * *masked_addr = 1:xen_set_virtual_psr_i(0)
556 * }
557 */
558/* 6 bundles */
559DEFINE_VOID_FUNC1(intrin_local_irq_restore,
560 /* r8 = input value: 0 or IA64_PSR_I
561 * p6 = (flags & IA64_PSR_I)
562 * = if clause
563 * p7 = !(flags & IA64_PSR_I)
564 * = else clause
565 */
566 "cmp.ne p6, p7 = r8, r0\n"
567 "mov r9 = " __stringify(XEN_PSR_I_ADDR_ADDR) "\n"
568 ";;\n"
569 /* r9 = XEN_PSR_I_ADDR */
570 "ld8 r9 = [r9]\n"
571 ";;\n"
572
573 /* r10 = masked previous value */
574 "(p6) ld1.acq r10 = [r9]\n"
575 ";;\n"
576
577 /* p8 = !masked interrupt masked previously? */
578 "(p6) cmp.ne.unc p8, p0 = r10, r0\n"
579
580 /* p7 = else clause */
581 "(p7) mov r11 = 1\n"
582 ";;\n"
583 /* masked = 1 */
584 "(p7) st1.rel [r9] = r11\n"
585
586 /* p6 = if clause */
587 /* masked = 0
588 * r9 = masked_addr - 1
589 * = pending_intr_addr
590 */
591 "(p8) st1.rel [r9] = r0, -1\n"
592 ";;\n"
593 /* r8 = pending_intr */
594 "(p8) ld1.acq r11 = [r9]\n"
595 ";;\n"
596 /* p9 = interrupt pending? */
597 "(p8) cmp.ne.unc p9, p10 = r11, r0\n"
598 ";;\n"
599 "(p10) mf\n"
600 /* issue hypercall to trigger interrupt */
601 "(p9) break " __stringify(HYPERPRIVOP_SSM_I) "\n");
602
603DEFINE_VOID_FUNC2(ptcga,
604 "break " __stringify(HYPERPRIVOP_PTC_GA) "\n");
605DEFINE_VOID_FUNC2(set_rr,
606 "break " __stringify(HYPERPRIVOP_SET_RR) "\n");
607
608/*
609 * tmp = XEN_MAPPEDREGS->interrupt_mask_addr = XEN_PSR_I_ADDR_ADDR;
610 * tmp = *tmp
611 * tmp = *tmp;
612 * psr_i = tmp? 0: IA64_PSR_I;
613 */
614/* 4 bundles */
615DEFINE_FUNC0(get_psr_i,
616 "mov r9 = " __stringify(XEN_PSR_I_ADDR_ADDR) "\n"
617 ";;\n"
618 "ld8 r9 = [r9]\n" /* r9 = XEN_PSR_I_ADDR */
619 "mov r8 = 0\n" /* psr_i = 0 */
620 ";;\n"
621 "ld1.acq r9 = [r9]\n" /* r9 = XEN_PSR_I */
622 ";;\n"
623 "cmp.eq.unc p6, p0 = r9, r0\n" /* p6 = (XEN_PSR_I != 0) */
624 ";;\n"
625 "(p6) mov r8 = " __stringify(1 << IA64_PSR_I_BIT) "\n");
626
627DEFINE_FUNC1(thash, unsigned long,
628 "break " __stringify(HYPERPRIVOP_THASH) "\n");
629DEFINE_FUNC1(get_cpuid, int,
630 "break " __stringify(HYPERPRIVOP_GET_CPUID) "\n");
631DEFINE_FUNC1(get_pmd, int,
632 "break " __stringify(HYPERPRIVOP_GET_PMD) "\n");
633DEFINE_FUNC1(get_rr, unsigned long,
634 "break " __stringify(HYPERPRIVOP_GET_RR) "\n");
635
636/*
637 * void xen_privop_ssm_i(void)
638 *
639 * int masked = !xen_get_virtual_psr_i();
640 * // masked = *(*XEN_MAPPEDREGS->interrupt_mask_addr)
641 * xen_set_virtual_psr_i(1)
642 * // *(*XEN_MAPPEDREGS->interrupt_mask_addr) = 0
643 * // compiler barrier
644 * if (masked) {
645 * uint8_t* pend_int_addr =
646 * (uint8_t*)(*XEN_MAPPEDREGS->interrupt_mask_addr) - 1;
647 * uint8_t pending = *pend_int_addr;
648 * if (pending)
649 * XEN_HYPER_SSM_I
650 * }
651 */
652/* 4 bundles */
653DEFINE_VOID_FUNC0(ssm_i,
654 "mov r8 = " __stringify(XEN_PSR_I_ADDR_ADDR) "\n"
655 ";;\n"
656 "ld8 r8 = [r8]\n" /* r8 = XEN_PSR_I_ADDR */
657 ";;\n"
658 "ld1.acq r9 = [r8]\n" /* r9 = XEN_PSR_I */
659 ";;\n"
660 "st1.rel [r8] = r0, -1\n" /* psr_i = 0. enable interrupt
661 * r8 = XEN_PSR_I_ADDR - 1
662 * = pend_int_addr
663 */
664 "cmp.eq.unc p0, p6 = r9, r0\n"/* p6 = !XEN_PSR_I
665 * previously interrupt
666 * masked?
667 */
668 ";;\n"
669 "(p6) ld1.acq r8 = [r8]\n" /* r8 = xen_pend_int */
670 ";;\n"
671 "(p6) cmp.eq.unc p6, p7 = r8, r0\n" /*interrupt pending?*/
672 ";;\n"
673 /* issue hypercall to get interrupt */
674 "(p7) break " __stringify(HYPERPRIVOP_SSM_I) "\n"
675 ";;\n");
676
677/*
678 * psr_i_addr_addr = XEN_MAPPEDREGS->interrupt_mask_addr
679 * = XEN_PSR_I_ADDR_ADDR;
680 * psr_i_addr = *psr_i_addr_addr;
681 * *psr_i_addr = 1;
682 */
683/* 2 bundles */
684DEFINE_VOID_FUNC0(rsm_i,
685 "mov r8 = " __stringify(XEN_PSR_I_ADDR_ADDR) "\n"
686 /* r8 = XEN_PSR_I_ADDR */
687 "mov r9 = 1\n"
688 ";;\n"
689 "ld8 r8 = [r8]\n" /* r8 = XEN_PSR_I */
690 ";;\n"
691 "st1.rel [r8] = r9\n"); /* XEN_PSR_I = 1 */
692
693extern void
694xen_set_rr0_to_rr4(unsigned long val0, unsigned long val1,
695 unsigned long val2, unsigned long val3,
696 unsigned long val4);
697__DEFINE_FUNC(set_rr0_to_rr4,
698 "break " __stringify(HYPERPRIVOP_SET_RR0_TO_RR4) "\n");
699
700
701extern unsigned long xen_getreg(int regnum);
702#define __DEFINE_GET_REG(id, privop) \
703 "mov r2 = " __stringify(_IA64_REG_ ## id) "\n" \
704 ";;\n" \
705 "cmp.eq p6, p0 = r2, r8\n" \
706 ";;\n" \
707 "(p6) break " __stringify(HYPERPRIVOP_GET_ ## privop) "\n" \
708 "(p6) br.cond.sptk.many b6\n" \
709 ";;\n"
710
711__DEFINE_FUNC(getreg,
712 __DEFINE_GET_REG(PSR, PSR)
713#ifdef CONFIG_IA32_SUPPORT
714 __DEFINE_GET_REG(AR_EFLAG, EFLAG)
715#endif
716
717 /* get_itc */
718 "mov r2 = " __stringify(_IA64_REG_AR_ITC) "\n"
719 ";;\n"
720 "cmp.eq p6, p0 = r2, r8\n"
721 ";;\n"
722 "(p6) br.cond.spnt xen_get_itc\n"
723 ";;\n"
724
725 /* get itm */
726 "mov r2 = " __stringify(_IA64_REG_CR_ITM) "\n"
727 ";;\n"
728 "cmp.eq p6, p0 = r2, r8\n"
729 ";;\n"
730 "(p6) br.cond.spnt xen_get_itm_with_offset\n"
731 ";;\n"
732
733 __DEFINE_GET_REG(CR_IVR, IVR)
734 __DEFINE_GET_REG(CR_TPR, TPR)
735
736 /* fall back */
737 "movl r2 = ia64_native_getreg_func\n"
738 ";;\n"
739 "mov b7 = r2\n"
740 ";;\n"
741 "br.cond.sptk.many b7\n");
742
743extern void xen_setreg(int regnum, unsigned long val);
744#define __DEFINE_SET_REG(id, privop) \
745 "mov r2 = " __stringify(_IA64_REG_ ## id) "\n" \
746 ";;\n" \
747 "cmp.eq p6, p0 = r2, r9\n" \
748 ";;\n" \
749 "(p6) break " __stringify(HYPERPRIVOP_ ## privop) "\n" \
750 "(p6) br.cond.sptk.many b6\n" \
751 ";;\n"
752
753__DEFINE_FUNC(setreg,
754 /* kr0 .. kr 7*/
755 /*
756 * if (_IA64_REG_AR_KR0 <= regnum &&
757 * regnum <= _IA64_REG_AR_KR7) {
758 * register __index asm ("r8") = regnum - _IA64_REG_AR_KR0
759 * register __val asm ("r9") = val
760 * "break HYPERPRIVOP_SET_KR"
761 * }
762 */
763 "mov r17 = r9\n"
764 "mov r2 = " __stringify(_IA64_REG_AR_KR0) "\n"
765 ";;\n"
766 "cmp.ge p6, p0 = r9, r2\n"
767 "sub r17 = r17, r2\n"
768 ";;\n"
769 "(p6) cmp.ge.unc p7, p0 = "
770 __stringify(_IA64_REG_AR_KR7) " - " __stringify(_IA64_REG_AR_KR0)
771 ", r17\n"
772 ";;\n"
773 "(p7) mov r9 = r8\n"
774 ";;\n"
775 "(p7) mov r8 = r17\n"
776 "(p7) break " __stringify(HYPERPRIVOP_SET_KR) "\n"
777
778 /* set itm */
779 "mov r2 = " __stringify(_IA64_REG_CR_ITM) "\n"
780 ";;\n"
781 "cmp.eq p6, p0 = r2, r8\n"
782 ";;\n"
783 "(p6) br.cond.spnt xen_set_itm_with_offset\n"
784
785 /* set itc */
786 "mov r2 = " __stringify(_IA64_REG_AR_ITC) "\n"
787 ";;\n"
788 "cmp.eq p6, p0 = r2, r8\n"
789 ";;\n"
790 "(p6) br.cond.spnt xen_set_itc\n"
791
792#ifdef CONFIG_IA32_SUPPORT
793 __DEFINE_SET_REG(AR_EFLAG, SET_EFLAG)
794#endif
795 __DEFINE_SET_REG(CR_TPR, SET_TPR)
796 __DEFINE_SET_REG(CR_EOI, EOI)
797
798 /* fall back */
799 "movl r2 = ia64_native_setreg_func\n"
800 ";;\n"
801 "mov b7 = r2\n"
802 ";;\n"
803 "br.cond.sptk.many b7\n");
804#endif
805
806static const struct pv_cpu_ops xen_cpu_ops __initconst = {
264 .fc = xen_fc, 807 .fc = xen_fc,
265 .thash = xen_thash, 808 .thash = xen_thash,
266 .get_cpuid = xen_get_cpuid, 809 .get_cpuid = xen_get_cpuid,
@@ -337,7 +880,7 @@ xen_iosapic_write(char __iomem *iosapic, unsigned int reg, u32 val)
337 HYPERVISOR_physdev_op(PHYSDEVOP_apic_write, &apic_op); 880 HYPERVISOR_physdev_op(PHYSDEVOP_apic_write, &apic_op);
338} 881}
339 882
340static const struct pv_iosapic_ops xen_iosapic_ops __initconst = { 883static struct pv_iosapic_ops xen_iosapic_ops __initdata = {
341 .pcat_compat_init = xen_pcat_compat_init, 884 .pcat_compat_init = xen_pcat_compat_init,
342 .__get_irq_chip = xen_iosapic_get_irq_chip, 885 .__get_irq_chip = xen_iosapic_get_irq_chip,
343 886
@@ -355,6 +898,8 @@ xen_setup_pv_ops(void)
355 xen_info_init(); 898 xen_info_init();
356 pv_info = xen_info; 899 pv_info = xen_info;
357 pv_init_ops = xen_init_ops; 900 pv_init_ops = xen_init_ops;
901 pv_fsys_data = xen_fsys_data;
902 pv_patchdata = xen_patchdata;
358 pv_cpu_ops = xen_cpu_ops; 903 pv_cpu_ops = xen_cpu_ops;
359 pv_iosapic_ops = xen_iosapic_ops; 904 pv_iosapic_ops = xen_iosapic_ops;
360 pv_irq_ops = xen_irq_ops; 905 pv_irq_ops = xen_irq_ops;
@@ -362,3 +907,252 @@ xen_setup_pv_ops(void)
362 907
363 paravirt_cpu_asm_init(&xen_cpu_asm_switch); 908 paravirt_cpu_asm_init(&xen_cpu_asm_switch);
364} 909}
910
911#ifdef ASM_SUPPORTED
912/***************************************************************************
913 * binary pacthing
914 * pv_init_ops.patch_bundle
915 */
916
917#define DEFINE_FUNC_GETREG(name, privop) \
918 DEFINE_FUNC0(get_ ## name, \
919 "break "__stringify(HYPERPRIVOP_GET_ ## privop) "\n")
920
921DEFINE_FUNC_GETREG(psr, PSR);
922DEFINE_FUNC_GETREG(eflag, EFLAG);
923DEFINE_FUNC_GETREG(ivr, IVR);
924DEFINE_FUNC_GETREG(tpr, TPR);
925
926#define DEFINE_FUNC_SET_KR(n) \
927 DEFINE_VOID_FUNC0(set_kr ## n, \
928 ";;\n" \
929 "mov r9 = r8\n" \
930 "mov r8 = " #n "\n" \
931 "break " __stringify(HYPERPRIVOP_SET_KR) "\n")
932
933DEFINE_FUNC_SET_KR(0);
934DEFINE_FUNC_SET_KR(1);
935DEFINE_FUNC_SET_KR(2);
936DEFINE_FUNC_SET_KR(3);
937DEFINE_FUNC_SET_KR(4);
938DEFINE_FUNC_SET_KR(5);
939DEFINE_FUNC_SET_KR(6);
940DEFINE_FUNC_SET_KR(7);
941
942#define __DEFINE_FUNC_SETREG(name, privop) \
943 DEFINE_VOID_FUNC0(name, \
944 "break "__stringify(HYPERPRIVOP_ ## privop) "\n")
945
946#define DEFINE_FUNC_SETREG(name, privop) \
947 __DEFINE_FUNC_SETREG(set_ ## name, SET_ ## privop)
948
949DEFINE_FUNC_SETREG(eflag, EFLAG);
950DEFINE_FUNC_SETREG(tpr, TPR);
951__DEFINE_FUNC_SETREG(eoi, EOI);
952
953extern const char xen_check_events[];
954extern const char __xen_intrin_local_irq_restore_direct_start[];
955extern const char __xen_intrin_local_irq_restore_direct_end[];
956extern const unsigned long __xen_intrin_local_irq_restore_direct_reloc;
957
958asm (
959 ".align 32\n"
960 ".proc xen_check_events\n"
961 "xen_check_events:\n"
962 /* masked = 0
963 * r9 = masked_addr - 1
964 * = pending_intr_addr
965 */
966 "st1.rel [r9] = r0, -1\n"
967 ";;\n"
968 /* r8 = pending_intr */
969 "ld1.acq r11 = [r9]\n"
970 ";;\n"
971 /* p9 = interrupt pending? */
972 "cmp.ne p9, p10 = r11, r0\n"
973 ";;\n"
974 "(p10) mf\n"
975 /* issue hypercall to trigger interrupt */
976 "(p9) break " __stringify(HYPERPRIVOP_SSM_I) "\n"
977 "br.cond.sptk.many b6\n"
978 ".endp xen_check_events\n"
979 "\n"
980 ".align 32\n"
981 ".proc __xen_intrin_local_irq_restore_direct\n"
982 "__xen_intrin_local_irq_restore_direct:\n"
983 "__xen_intrin_local_irq_restore_direct_start:\n"
984 "1:\n"
985 "{\n"
986 "cmp.ne p6, p7 = r8, r0\n"
987 "mov r17 = ip\n" /* get ip to calc return address */
988 "mov r9 = "__stringify(XEN_PSR_I_ADDR_ADDR) "\n"
989 ";;\n"
990 "}\n"
991 "{\n"
992 /* r9 = XEN_PSR_I_ADDR */
993 "ld8 r9 = [r9]\n"
994 ";;\n"
995 /* r10 = masked previous value */
996 "(p6) ld1.acq r10 = [r9]\n"
997 "adds r17 = 1f - 1b, r17\n" /* calculate return address */
998 ";;\n"
999 "}\n"
1000 "{\n"
1001 /* p8 = !masked interrupt masked previously? */
1002 "(p6) cmp.ne.unc p8, p0 = r10, r0\n"
1003 "\n"
1004 /* p7 = else clause */
1005 "(p7) mov r11 = 1\n"
1006 ";;\n"
1007 "(p8) mov b6 = r17\n" /* set return address */
1008 "}\n"
1009 "{\n"
1010 /* masked = 1 */
1011 "(p7) st1.rel [r9] = r11\n"
1012 "\n"
1013 "[99:]\n"
1014 "(p8) brl.cond.dptk.few xen_check_events\n"
1015 "}\n"
1016 /* pv calling stub is 5 bundles. fill nop to adjust return address */
1017 "{\n"
1018 "nop 0\n"
1019 "nop 0\n"
1020 "nop 0\n"
1021 "}\n"
1022 "1:\n"
1023 "__xen_intrin_local_irq_restore_direct_end:\n"
1024 ".endp __xen_intrin_local_irq_restore_direct\n"
1025 "\n"
1026 ".align 8\n"
1027 "__xen_intrin_local_irq_restore_direct_reloc:\n"
1028 "data8 99b\n"
1029);
1030
1031static struct paravirt_patch_bundle_elem xen_patch_bundle_elems[]
1032__initdata_or_module =
1033{
1034#define XEN_PATCH_BUNDLE_ELEM(name, type) \
1035 { \
1036 (void*)xen_ ## name ## _direct_start, \
1037 (void*)xen_ ## name ## _direct_end, \
1038 PARAVIRT_PATCH_TYPE_ ## type, \
1039 }
1040
1041 XEN_PATCH_BUNDLE_ELEM(fc, FC),
1042 XEN_PATCH_BUNDLE_ELEM(thash, THASH),
1043 XEN_PATCH_BUNDLE_ELEM(get_cpuid, GET_CPUID),
1044 XEN_PATCH_BUNDLE_ELEM(get_pmd, GET_PMD),
1045 XEN_PATCH_BUNDLE_ELEM(ptcga, PTCGA),
1046 XEN_PATCH_BUNDLE_ELEM(get_rr, GET_RR),
1047 XEN_PATCH_BUNDLE_ELEM(set_rr, SET_RR),
1048 XEN_PATCH_BUNDLE_ELEM(set_rr0_to_rr4, SET_RR0_TO_RR4),
1049 XEN_PATCH_BUNDLE_ELEM(ssm_i, SSM_I),
1050 XEN_PATCH_BUNDLE_ELEM(rsm_i, RSM_I),
1051 XEN_PATCH_BUNDLE_ELEM(get_psr_i, GET_PSR_I),
1052 {
1053 (void*)__xen_intrin_local_irq_restore_direct_start,
1054 (void*)__xen_intrin_local_irq_restore_direct_end,
1055 PARAVIRT_PATCH_TYPE_INTRIN_LOCAL_IRQ_RESTORE,
1056 },
1057
1058#define XEN_PATCH_BUNDLE_ELEM_GETREG(name, reg) \
1059 { \
1060 xen_get_ ## name ## _direct_start, \
1061 xen_get_ ## name ## _direct_end, \
1062 PARAVIRT_PATCH_TYPE_GETREG + _IA64_REG_ ## reg, \
1063 }
1064
1065 XEN_PATCH_BUNDLE_ELEM_GETREG(psr, PSR),
1066 XEN_PATCH_BUNDLE_ELEM_GETREG(eflag, AR_EFLAG),
1067
1068 XEN_PATCH_BUNDLE_ELEM_GETREG(ivr, CR_IVR),
1069 XEN_PATCH_BUNDLE_ELEM_GETREG(tpr, CR_TPR),
1070
1071 XEN_PATCH_BUNDLE_ELEM_GETREG(itc, AR_ITC),
1072 XEN_PATCH_BUNDLE_ELEM_GETREG(itm_with_offset, CR_ITM),
1073
1074
1075#define __XEN_PATCH_BUNDLE_ELEM_SETREG(name, reg) \
1076 { \
1077 xen_ ## name ## _direct_start, \
1078 xen_ ## name ## _direct_end, \
1079 PARAVIRT_PATCH_TYPE_SETREG + _IA64_REG_ ## reg, \
1080 }
1081
1082#define XEN_PATCH_BUNDLE_ELEM_SETREG(name, reg) \
1083 __XEN_PATCH_BUNDLE_ELEM_SETREG(set_ ## name, reg)
1084
1085 XEN_PATCH_BUNDLE_ELEM_SETREG(kr0, AR_KR0),
1086 XEN_PATCH_BUNDLE_ELEM_SETREG(kr1, AR_KR1),
1087 XEN_PATCH_BUNDLE_ELEM_SETREG(kr2, AR_KR2),
1088 XEN_PATCH_BUNDLE_ELEM_SETREG(kr3, AR_KR3),
1089 XEN_PATCH_BUNDLE_ELEM_SETREG(kr4, AR_KR4),
1090 XEN_PATCH_BUNDLE_ELEM_SETREG(kr5, AR_KR5),
1091 XEN_PATCH_BUNDLE_ELEM_SETREG(kr6, AR_KR6),
1092 XEN_PATCH_BUNDLE_ELEM_SETREG(kr7, AR_KR7),
1093
1094 XEN_PATCH_BUNDLE_ELEM_SETREG(eflag, AR_EFLAG),
1095 XEN_PATCH_BUNDLE_ELEM_SETREG(tpr, CR_TPR),
1096 __XEN_PATCH_BUNDLE_ELEM_SETREG(eoi, CR_EOI),
1097
1098 XEN_PATCH_BUNDLE_ELEM_SETREG(itc, AR_ITC),
1099 XEN_PATCH_BUNDLE_ELEM_SETREG(itm_with_offset, CR_ITM),
1100};
1101
1102static unsigned long __init_or_module
1103xen_patch_bundle(void *sbundle, void *ebundle, unsigned long type)
1104{
1105 const unsigned long nelems = sizeof(xen_patch_bundle_elems) /
1106 sizeof(xen_patch_bundle_elems[0]);
1107 unsigned long used;
1108 const struct paravirt_patch_bundle_elem *found;
1109
1110 used = __paravirt_patch_apply_bundle(sbundle, ebundle, type,
1111 xen_patch_bundle_elems, nelems,
1112 &found);
1113
1114 if (found == NULL)
1115 /* fallback */
1116 return ia64_native_patch_bundle(sbundle, ebundle, type);
1117 if (used == 0)
1118 return used;
1119
1120 /* relocation */
1121 switch (type) {
1122 case PARAVIRT_PATCH_TYPE_INTRIN_LOCAL_IRQ_RESTORE: {
1123 unsigned long reloc =
1124 __xen_intrin_local_irq_restore_direct_reloc;
1125 unsigned long reloc_offset = reloc - (unsigned long)
1126 __xen_intrin_local_irq_restore_direct_start;
1127 unsigned long tag = (unsigned long)sbundle + reloc_offset;
1128 paravirt_patch_reloc_brl(tag, xen_check_events);
1129 break;
1130 }
1131 default:
1132 /* nothing */
1133 break;
1134 }
1135 return used;
1136}
1137#endif /* ASM_SUPPOTED */
1138
1139const struct paravirt_patch_branch_target xen_branch_target[]
1140__initconst = {
1141#define PARAVIRT_BR_TARGET(name, type) \
1142 { \
1143 &xen_ ## name, \
1144 PARAVIRT_PATCH_TYPE_BR_ ## type, \
1145 }
1146 PARAVIRT_BR_TARGET(switch_to, SWITCH_TO),
1147 PARAVIRT_BR_TARGET(leave_syscall, LEAVE_SYSCALL),
1148 PARAVIRT_BR_TARGET(work_processed_syscall, WORK_PROCESSED_SYSCALL),
1149 PARAVIRT_BR_TARGET(leave_kernel, LEAVE_KERNEL),
1150};
1151
1152static void __init
1153xen_patch_branch(unsigned long tag, unsigned long type)
1154{
1155 const unsigned long nelem =
1156 sizeof(xen_branch_target) / sizeof(xen_branch_target[0]);
1157 __paravirt_patch_apply_branch(tag, type, xen_branch_target, nelem);
1158}
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-09 23:09:36 -0500