aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/admin-guide/kernel-parameters.txt5
-rw-r--r--arch/arm/include/asm/paravirt.h9
-rw-r--r--arch/arm/kernel/paravirt.c4
-rw-r--r--arch/arm/xen/enlighten.c34
-rw-r--r--arch/arm64/include/asm/paravirt.h9
-rw-r--r--arch/arm64/kernel/paravirt.c4
-rw-r--r--arch/x86/Kconfig4
-rw-r--r--arch/x86/boot/compressed/misc.h1
-rw-r--r--arch/x86/entry/entry_32.S8
-rw-r--r--arch/x86/entry/entry_64.S8
-rw-r--r--arch/x86/hyperv/Makefile4
-rw-r--r--arch/x86/hyperv/hv_spinlock.c88
-rw-r--r--arch/x86/hyperv/mmu.c4
-rw-r--r--arch/x86/include/asm/debugreg.h2
-rw-r--r--arch/x86/include/asm/desc.h4
-rw-r--r--arch/x86/include/asm/fixmap.h2
-rw-r--r--arch/x86/include/asm/hyperv-tlfs.h5
-rw-r--r--arch/x86/include/asm/irqflags.h16
-rw-r--r--arch/x86/include/asm/mmu_context.h4
-rw-r--r--arch/x86/include/asm/mshyperv.h2
-rw-r--r--arch/x86/include/asm/msr.h4
-rw-r--r--arch/x86/include/asm/paravirt.h415
-rw-r--r--arch/x86/include/asm/paravirt_types.h82
-rw-r--r--arch/x86/include/asm/pgalloc.h2
-rw-r--r--arch/x86/include/asm/pgtable-3level_types.h2
-rw-r--r--arch/x86/include/asm/pgtable.h7
-rw-r--r--arch/x86/include/asm/processor.h4
-rw-r--r--arch/x86/include/asm/ptrace.h2
-rw-r--r--arch/x86/include/asm/segment.h2
-rw-r--r--arch/x86/include/asm/special_insns.h4
-rw-r--r--arch/x86/kernel/alternative.c2
-rw-r--r--arch/x86/kernel/asm-offsets.c13
-rw-r--r--arch/x86/kernel/asm-offsets_64.c9
-rw-r--r--arch/x86/kernel/cpu/common.c4
-rw-r--r--arch/x86/kernel/cpu/mshyperv.c14
-rw-r--r--arch/x86/kernel/cpu/vmware.c4
-rw-r--r--arch/x86/kernel/head_64.S2
-rw-r--r--arch/x86/kernel/kvm.c19
-rw-r--r--arch/x86/kernel/kvmclock.c4
-rw-r--r--arch/x86/kernel/paravirt-spinlocks.c15
-rw-r--r--arch/x86/kernel/paravirt.c316
-rw-r--r--arch/x86/kernel/paravirt_patch_32.c87
-rw-r--r--arch/x86/kernel/paravirt_patch_64.c97
-rw-r--r--arch/x86/kernel/tsc.c2
-rw-r--r--arch/x86/kernel/vsmp_64.c26
-rw-r--r--arch/x86/mm/mem_encrypt_identity.c1
-rw-r--r--arch/x86/xen/Kconfig1
-rw-r--r--arch/x86/xen/Makefile41
-rw-r--r--arch/x86/xen/efi.c14
-rw-r--r--arch/x86/xen/enlighten.c2
-rw-r--r--arch/x86/xen/enlighten_hvm.c2
-rw-r--r--arch/x86/xen/enlighten_pv.c31
-rw-r--r--arch/x86/xen/grant-table.c25
-rw-r--r--arch/x86/xen/irq.c2
-rw-r--r--arch/x86/xen/mmu.c188
-rw-r--r--arch/x86/xen/mmu_hvm.c2
-rw-r--r--arch/x86/xen/mmu_pv.c168
-rw-r--r--arch/x86/xen/p2m.c2
-rw-r--r--arch/x86/xen/pci-swiotlb-xen.c2
-rw-r--r--arch/x86/xen/platform-pci-unplug.c18
-rw-r--r--arch/x86/xen/spinlock.c11
-rw-r--r--arch/x86/xen/time.c4
-rw-r--r--arch/x86/xen/vdso.h2
-rw-r--r--arch/x86/xen/xen-pvh.S15
-rw-r--r--drivers/xen/time.c2
-rw-r--r--include/xen/events.h2
-rw-r--r--include/xen/interface/memory.h6
-rw-r--r--include/xen/xen-ops.h133
68 files changed, 1023 insertions, 1006 deletions
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 4c74a2f4ddfc..6d380890e075 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -1390,6 +1390,11 @@
1390 hvc_iucv_allow= [S390] Comma-separated list of z/VM user IDs. 1390 hvc_iucv_allow= [S390] Comma-separated list of z/VM user IDs.
1391 If specified, z/VM IUCV HVC accepts connections 1391 If specified, z/VM IUCV HVC accepts connections
1392 from listed z/VM user IDs only. 1392 from listed z/VM user IDs only.
1393
1394 hv_nopvspin [X86,HYPER_V] Disables the paravirt spinlock optimizations
1395 which allow the hypervisor to 'idle' the
1396 guest on lock contention.
1397
1393 keep_bootcon [KNL] 1398 keep_bootcon [KNL]
1394 Do not unregister boot console at start. This is only 1399 Do not unregister boot console at start. This is only
1395 useful for debugging when something happens in the window 1400 useful for debugging when something happens in the window
diff --git a/arch/arm/include/asm/paravirt.h b/arch/arm/include/asm/paravirt.h
index d51e5cd31d01..cdbf02d9c1d4 100644
--- a/arch/arm/include/asm/paravirt.h
+++ b/arch/arm/include/asm/paravirt.h
@@ -10,11 +10,16 @@ extern struct static_key paravirt_steal_rq_enabled;
10struct pv_time_ops { 10struct pv_time_ops {
11 unsigned long long (*steal_clock)(int cpu); 11 unsigned long long (*steal_clock)(int cpu);
12}; 12};
13extern struct pv_time_ops pv_time_ops; 13
14struct paravirt_patch_template {
15 struct pv_time_ops time;
16};
17
18extern struct paravirt_patch_template pv_ops;
14 19
15static inline u64 paravirt_steal_clock(int cpu) 20static inline u64 paravirt_steal_clock(int cpu)
16{ 21{
17 return pv_time_ops.steal_clock(cpu); 22 return pv_ops.time.steal_clock(cpu);
18} 23}
19#endif 24#endif
20 25
diff --git a/arch/arm/kernel/paravirt.c b/arch/arm/kernel/paravirt.c
index 53f371ed4568..75c158b0353f 100644
--- a/arch/arm/kernel/paravirt.c
+++ b/arch/arm/kernel/paravirt.c
@@ -21,5 +21,5 @@
21struct static_key paravirt_steal_enabled; 21struct static_key paravirt_steal_enabled;
22struct static_key paravirt_steal_rq_enabled; 22struct static_key paravirt_steal_rq_enabled;
23 23
24struct pv_time_ops pv_time_ops; 24struct paravirt_patch_template pv_ops;
25EXPORT_SYMBOL_GPL(pv_time_ops); 25EXPORT_SYMBOL_GPL(pv_ops);
diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c
index 07060e5b5864..17e478928276 100644
--- a/arch/arm/xen/enlighten.c
+++ b/arch/arm/xen/enlighten.c
@@ -62,29 +62,6 @@ static __read_mostly unsigned int xen_events_irq;
62uint32_t xen_start_flags; 62uint32_t xen_start_flags;
63EXPORT_SYMBOL(xen_start_flags); 63EXPORT_SYMBOL(xen_start_flags);
64 64
65int xen_remap_domain_gfn_array(struct vm_area_struct *vma,
66 unsigned long addr,
67 xen_pfn_t *gfn, int nr,
68 int *err_ptr, pgprot_t prot,
69 unsigned domid,
70 struct page **pages)
71{
72 return xen_xlate_remap_gfn_array(vma, addr, gfn, nr, err_ptr,
73 prot, domid, pages);
74}
75EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_array);
76
77/* Not used by XENFEAT_auto_translated guests. */
78int xen_remap_domain_gfn_range(struct vm_area_struct *vma,
79 unsigned long addr,
80 xen_pfn_t gfn, int nr,
81 pgprot_t prot, unsigned domid,
82 struct page **pages)
83{
84 return -ENOSYS;
85}
86EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_range);
87
88int xen_unmap_domain_gfn_range(struct vm_area_struct *vma, 65int xen_unmap_domain_gfn_range(struct vm_area_struct *vma,
89 int nr, struct page **pages) 66 int nr, struct page **pages)
90{ 67{
@@ -92,17 +69,6 @@ int xen_unmap_domain_gfn_range(struct vm_area_struct *vma,
92} 69}
93EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range); 70EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range);
94 71
95/* Not used by XENFEAT_auto_translated guests. */
96int xen_remap_domain_mfn_array(struct vm_area_struct *vma,
97 unsigned long addr,
98 xen_pfn_t *mfn, int nr,
99 int *err_ptr, pgprot_t prot,
100 unsigned int domid, struct page **pages)
101{
102 return -ENOSYS;
103}
104EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_array);
105
106static void xen_read_wallclock(struct timespec64 *ts) 72static void xen_read_wallclock(struct timespec64 *ts)
107{ 73{
108 u32 version; 74 u32 version;
diff --git a/arch/arm64/include/asm/paravirt.h b/arch/arm64/include/asm/paravirt.h
index bb5dcea42003..799d9dd6f7cc 100644
--- a/arch/arm64/include/asm/paravirt.h
+++ b/arch/arm64/include/asm/paravirt.h
@@ -10,11 +10,16 @@ extern struct static_key paravirt_steal_rq_enabled;
10struct pv_time_ops { 10struct pv_time_ops {
11 unsigned long long (*steal_clock)(int cpu); 11 unsigned long long (*steal_clock)(int cpu);
12}; 12};
13extern struct pv_time_ops pv_time_ops; 13
14struct paravirt_patch_template {
15 struct pv_time_ops time;
16};
17
18extern struct paravirt_patch_template pv_ops;
14 19
15static inline u64 paravirt_steal_clock(int cpu) 20static inline u64 paravirt_steal_clock(int cpu)
16{ 21{
17 return pv_time_ops.steal_clock(cpu); 22 return pv_ops.time.steal_clock(cpu);
18} 23}
19#endif 24#endif
20 25
diff --git a/arch/arm64/kernel/paravirt.c b/arch/arm64/kernel/paravirt.c
index 53f371ed4568..75c158b0353f 100644
--- a/arch/arm64/kernel/paravirt.c
+++ b/arch/arm64/kernel/paravirt.c
@@ -21,5 +21,5 @@
21struct static_key paravirt_steal_enabled; 21struct static_key paravirt_steal_enabled;
22struct static_key paravirt_steal_rq_enabled; 22struct static_key paravirt_steal_rq_enabled;
23 23
24struct pv_time_ops pv_time_ops; 24struct paravirt_patch_template pv_ops;
25EXPORT_SYMBOL_GPL(pv_time_ops); 25EXPORT_SYMBOL_GPL(pv_ops);
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index ff425a2d286c..46d42af4501a 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -523,6 +523,7 @@ config X86_VSMP
523 bool "ScaleMP vSMP" 523 bool "ScaleMP vSMP"
524 select HYPERVISOR_GUEST 524 select HYPERVISOR_GUEST
525 select PARAVIRT 525 select PARAVIRT
526 select PARAVIRT_XXL
526 depends on X86_64 && PCI 527 depends on X86_64 && PCI
527 depends on X86_EXTENDED_PLATFORM 528 depends on X86_EXTENDED_PLATFORM
528 depends on SMP 529 depends on SMP
@@ -753,6 +754,9 @@ config PARAVIRT
753 over full virtualization. However, when run without a hypervisor 754 over full virtualization. However, when run without a hypervisor
754 the kernel is theoretically slower and slightly larger. 755 the kernel is theoretically slower and slightly larger.
755 756
757config PARAVIRT_XXL
758 bool
759
756config PARAVIRT_DEBUG 760config PARAVIRT_DEBUG
757 bool "paravirt-ops debugging" 761 bool "paravirt-ops debugging"
758 depends on PARAVIRT && DEBUG_KERNEL 762 depends on PARAVIRT && DEBUG_KERNEL
diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h
index a423bdb42686..a1d5918765f3 100644
--- a/arch/x86/boot/compressed/misc.h
+++ b/arch/x86/boot/compressed/misc.h
@@ -9,6 +9,7 @@
9 * paravirt and debugging variants are added.) 9 * paravirt and debugging variants are added.)
10 */ 10 */
11#undef CONFIG_PARAVIRT 11#undef CONFIG_PARAVIRT
12#undef CONFIG_PARAVIRT_XXL
12#undef CONFIG_PARAVIRT_SPINLOCKS 13#undef CONFIG_PARAVIRT_SPINLOCKS
13#undef CONFIG_KASAN 14#undef CONFIG_KASAN
14 15
diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index fbbf1ba57ec6..687e47f8a796 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -783,7 +783,7 @@ GLOBAL(__begin_SYSENTER_singlestep_region)
783 * will ignore all of the single-step traps generated in this range. 783 * will ignore all of the single-step traps generated in this range.
784 */ 784 */
785 785
786#ifdef CONFIG_XEN 786#ifdef CONFIG_XEN_PV
787/* 787/*
788 * Xen doesn't set %esp to be precisely what the normal SYSENTER 788 * Xen doesn't set %esp to be precisely what the normal SYSENTER
789 * entry point expects, so fix it up before using the normal path. 789 * entry point expects, so fix it up before using the normal path.
@@ -1241,7 +1241,7 @@ ENTRY(spurious_interrupt_bug)
1241 jmp common_exception 1241 jmp common_exception
1242END(spurious_interrupt_bug) 1242END(spurious_interrupt_bug)
1243 1243
1244#ifdef CONFIG_XEN 1244#ifdef CONFIG_XEN_PV
1245ENTRY(xen_hypervisor_callback) 1245ENTRY(xen_hypervisor_callback)
1246 pushl $-1 /* orig_ax = -1 => not a system call */ 1246 pushl $-1 /* orig_ax = -1 => not a system call */
1247 SAVE_ALL 1247 SAVE_ALL
@@ -1322,11 +1322,13 @@ ENTRY(xen_failsafe_callback)
1322 _ASM_EXTABLE(3b, 8b) 1322 _ASM_EXTABLE(3b, 8b)
1323 _ASM_EXTABLE(4b, 9b) 1323 _ASM_EXTABLE(4b, 9b)
1324ENDPROC(xen_failsafe_callback) 1324ENDPROC(xen_failsafe_callback)
1325#endif /* CONFIG_XEN_PV */
1325 1326
1327#ifdef CONFIG_XEN_PVHVM
1326BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 1328BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
1327 xen_evtchn_do_upcall) 1329 xen_evtchn_do_upcall)
1330#endif
1328 1331
1329#endif /* CONFIG_XEN */
1330 1332
1331#if IS_ENABLED(CONFIG_HYPERV) 1333#if IS_ENABLED(CONFIG_HYPERV)
1332 1334
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index f95dcb209fdf..7c5ce0a6c4d2 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -1050,7 +1050,7 @@ ENTRY(do_softirq_own_stack)
1050 ret 1050 ret
1051ENDPROC(do_softirq_own_stack) 1051ENDPROC(do_softirq_own_stack)
1052 1052
1053#ifdef CONFIG_XEN 1053#ifdef CONFIG_XEN_PV
1054idtentry hypervisor_callback xen_do_hypervisor_callback has_error_code=0 1054idtentry hypervisor_callback xen_do_hypervisor_callback has_error_code=0
1055 1055
1056/* 1056/*
@@ -1130,11 +1130,13 @@ ENTRY(xen_failsafe_callback)
1130 ENCODE_FRAME_POINTER 1130 ENCODE_FRAME_POINTER
1131 jmp error_exit 1131 jmp error_exit
1132END(xen_failsafe_callback) 1132END(xen_failsafe_callback)
1133#endif /* CONFIG_XEN_PV */
1133 1134
1135#ifdef CONFIG_XEN_PVHVM
1134apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ 1136apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1135 xen_hvm_callback_vector xen_evtchn_do_upcall 1137 xen_hvm_callback_vector xen_evtchn_do_upcall
1138#endif
1136 1139
1137#endif /* CONFIG_XEN */
1138 1140
1139#if IS_ENABLED(CONFIG_HYPERV) 1141#if IS_ENABLED(CONFIG_HYPERV)
1140apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ 1142apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
@@ -1151,7 +1153,7 @@ idtentry debug do_debug has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK
1151idtentry int3 do_int3 has_error_code=0 1153idtentry int3 do_int3 has_error_code=0
1152idtentry stack_segment do_stack_segment has_error_code=1 1154idtentry stack_segment do_stack_segment has_error_code=1
1153 1155
1154#ifdef CONFIG_XEN 1156#ifdef CONFIG_XEN_PV
1155idtentry xennmi do_nmi has_error_code=0 1157idtentry xennmi do_nmi has_error_code=0
1156idtentry xendebug do_debug has_error_code=0 1158idtentry xendebug do_debug has_error_code=0
1157idtentry xenint3 do_int3 has_error_code=0 1159idtentry xenint3 do_int3 has_error_code=0
diff --git a/arch/x86/hyperv/Makefile b/arch/x86/hyperv/Makefile
index b21ee65c4101..1c11f9420a82 100644
--- a/arch/x86/hyperv/Makefile
+++ b/arch/x86/hyperv/Makefile
@@ -1,2 +1,6 @@
1obj-y := hv_init.o mmu.o nested.o 1obj-y := hv_init.o mmu.o nested.o
2obj-$(CONFIG_X86_64) += hv_apic.o 2obj-$(CONFIG_X86_64) += hv_apic.o
3
4ifdef CONFIG_X86_64
5obj-$(CONFIG_PARAVIRT_SPINLOCKS) += hv_spinlock.o
6endif
diff --git a/arch/x86/hyperv/hv_spinlock.c b/arch/x86/hyperv/hv_spinlock.c
new file mode 100644
index 000000000000..a861b0456b1a
--- /dev/null
+++ b/arch/x86/hyperv/hv_spinlock.c
@@ -0,0 +1,88 @@
1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Hyper-V specific spinlock code.
5 *
6 * Copyright (C) 2018, Intel, Inc.
7 *
8 * Author : Yi Sun <yi.y.sun@intel.com>
9 */
10
11#define pr_fmt(fmt) "Hyper-V: " fmt
12
13#include <linux/spinlock.h>
14
15#include <asm/mshyperv.h>
16#include <asm/paravirt.h>
17#include <asm/apic.h>
18
19static bool __initdata hv_pvspin = true;
20
21static void hv_qlock_kick(int cpu)
22{
23 apic->send_IPI(cpu, X86_PLATFORM_IPI_VECTOR);
24}
25
26static void hv_qlock_wait(u8 *byte, u8 val)
27{
28 unsigned long msr_val;
29 unsigned long flags;
30
31 if (in_nmi())
32 return;
33
34 /*
35 * Reading HV_X64_MSR_GUEST_IDLE MSR tells the hypervisor that the
36 * vCPU can be put into 'idle' state. This 'idle' state is
37 * terminated by an IPI, usually from hv_qlock_kick(), even if
38 * interrupts are disabled on the vCPU.
39 *
40 * To prevent a race against the unlock path it is required to
41 * disable interrupts before accessing the HV_X64_MSR_GUEST_IDLE
42 * MSR. Otherwise, if the IPI from hv_qlock_kick() arrives between
43 * the lock value check and the rdmsrl() then the vCPU might be put
44 * into 'idle' state by the hypervisor and kept in that state for
45 * an unspecified amount of time.
46 */
47 local_irq_save(flags);
48 /*
49 * Only issue the rdmsrl() when the lock state has not changed.
50 */
51 if (READ_ONCE(*byte) == val)
52 rdmsrl(HV_X64_MSR_GUEST_IDLE, msr_val);
53 local_irq_restore(flags);
54}
55
56/*
57 * Hyper-V does not support this so far.
58 */
59bool hv_vcpu_is_preempted(int vcpu)
60{
61 return false;
62}
63PV_CALLEE_SAVE_REGS_THUNK(hv_vcpu_is_preempted);
64
65void __init hv_init_spinlocks(void)
66{
67 if (!hv_pvspin || !apic ||
68 !(ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) ||
69 !(ms_hyperv.features & HV_X64_MSR_GUEST_IDLE_AVAILABLE)) {
70 pr_info("PV spinlocks disabled\n");
71 return;
72 }
73 pr_info("PV spinlocks enabled\n");
74
75 __pv_init_lock_hash();
76 pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath;
77 pv_ops.lock.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock);
78 pv_ops.lock.wait = hv_qlock_wait;
79 pv_ops.lock.kick = hv_qlock_kick;
80 pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(hv_vcpu_is_preempted);
81}
82
83static __init int hv_parse_nopvspin(char *arg)
84{
85 hv_pvspin = false;
86 return 0;
87}
88early_param("hv_nopvspin", hv_parse_nopvspin);
diff --git a/arch/x86/hyperv/mmu.c b/arch/x86/hyperv/mmu.c
index ef5f29f913d7..e65d7fe6489f 100644
--- a/arch/x86/hyperv/mmu.c
+++ b/arch/x86/hyperv/mmu.c
@@ -231,6 +231,6 @@ void hyperv_setup_mmu_ops(void)
231 return; 231 return;
232 232
233 pr_info("Using hypercall for remote TLB flush\n"); 233 pr_info("Using hypercall for remote TLB flush\n");
234 pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others; 234 pv_ops.mmu.flush_tlb_others = hyperv_flush_tlb_others;
235 pv_mmu_ops.tlb_remove_table = tlb_remove_table; 235 pv_ops.mmu.tlb_remove_table = tlb_remove_table;
236} 236}
diff --git a/arch/x86/include/asm/debugreg.h b/arch/x86/include/asm/debugreg.h
index 4505ac2735ad..9e5ca30738e5 100644
--- a/arch/x86/include/asm/debugreg.h
+++ b/arch/x86/include/asm/debugreg.h
@@ -8,7 +8,7 @@
8 8
9DECLARE_PER_CPU(unsigned long, cpu_dr7); 9DECLARE_PER_CPU(unsigned long, cpu_dr7);
10 10
11#ifndef CONFIG_PARAVIRT 11#ifndef CONFIG_PARAVIRT_XXL
12/* 12/*
13 * These special macros can be used to get or set a debugging register 13 * These special macros can be used to get or set a debugging register
14 */ 14 */
diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h
index 13c5ee878a47..68a99d2a5f33 100644
--- a/arch/x86/include/asm/desc.h
+++ b/arch/x86/include/asm/desc.h
@@ -108,7 +108,7 @@ static inline int desc_empty(const void *ptr)
108 return !(desc[0] | desc[1]); 108 return !(desc[0] | desc[1]);
109} 109}
110 110
111#ifdef CONFIG_PARAVIRT 111#ifdef CONFIG_PARAVIRT_XXL
112#include <asm/paravirt.h> 112#include <asm/paravirt.h>
113#else 113#else
114#define load_TR_desc() native_load_tr_desc() 114#define load_TR_desc() native_load_tr_desc()
@@ -134,7 +134,7 @@ static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
134static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries) 134static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
135{ 135{
136} 136}
137#endif /* CONFIG_PARAVIRT */ 137#endif /* CONFIG_PARAVIRT_XXL */
138 138
139#define store_ldt(ldt) asm("sldt %0" : "=m"(ldt)) 139#define store_ldt(ldt) asm("sldt %0" : "=m"(ldt))
140 140
diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h
index 6390bd8c141b..50ba74a34a37 100644
--- a/arch/x86/include/asm/fixmap.h
+++ b/arch/x86/include/asm/fixmap.h
@@ -162,7 +162,7 @@ void __native_set_fixmap(enum fixed_addresses idx, pte_t pte);
162void native_set_fixmap(enum fixed_addresses idx, 162void native_set_fixmap(enum fixed_addresses idx,
163 phys_addr_t phys, pgprot_t flags); 163 phys_addr_t phys, pgprot_t flags);
164 164
165#ifndef CONFIG_PARAVIRT 165#ifndef CONFIG_PARAVIRT_XXL
166static inline void __set_fixmap(enum fixed_addresses idx, 166static inline void __set_fixmap(enum fixed_addresses idx,
167 phys_addr_t phys, pgprot_t flags) 167 phys_addr_t phys, pgprot_t flags)
168{ 168{
diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h
index 00e01d215f74..4139f7650fe5 100644
--- a/arch/x86/include/asm/hyperv-tlfs.h
+++ b/arch/x86/include/asm/hyperv-tlfs.h
@@ -38,6 +38,8 @@
38#define HV_MSR_TIME_REF_COUNT_AVAILABLE (1 << 1) 38#define HV_MSR_TIME_REF_COUNT_AVAILABLE (1 << 1)
39/* Partition reference TSC MSR is available */ 39/* Partition reference TSC MSR is available */
40#define HV_MSR_REFERENCE_TSC_AVAILABLE (1 << 9) 40#define HV_MSR_REFERENCE_TSC_AVAILABLE (1 << 9)
41/* Partition Guest IDLE MSR is available */
42#define HV_X64_MSR_GUEST_IDLE_AVAILABLE (1 << 10)
41 43
42/* A partition's reference time stamp counter (TSC) page */ 44/* A partition's reference time stamp counter (TSC) page */
43#define HV_X64_MSR_REFERENCE_TSC 0x40000021 45#define HV_X64_MSR_REFERENCE_TSC 0x40000021
@@ -246,6 +248,9 @@
246#define HV_X64_MSR_STIMER3_CONFIG 0x400000B6 248#define HV_X64_MSR_STIMER3_CONFIG 0x400000B6
247#define HV_X64_MSR_STIMER3_COUNT 0x400000B7 249#define HV_X64_MSR_STIMER3_COUNT 0x400000B7
248 250
251/* Hyper-V guest idle MSR */
252#define HV_X64_MSR_GUEST_IDLE 0x400000F0
253
249/* Hyper-V guest crash notification MSR's */ 254/* Hyper-V guest crash notification MSR's */
250#define HV_X64_MSR_CRASH_P0 0x40000100 255#define HV_X64_MSR_CRASH_P0 0x40000100
251#define HV_X64_MSR_CRASH_P1 0x40000101 256#define HV_X64_MSR_CRASH_P1 0x40000101
diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h
index 15450a675031..058e40fed167 100644
--- a/arch/x86/include/asm/irqflags.h
+++ b/arch/x86/include/asm/irqflags.h
@@ -64,7 +64,7 @@ static inline __cpuidle void native_halt(void)
64 64
65#endif 65#endif
66 66
67#ifdef CONFIG_PARAVIRT 67#ifdef CONFIG_PARAVIRT_XXL
68#include <asm/paravirt.h> 68#include <asm/paravirt.h>
69#else 69#else
70#ifndef __ASSEMBLY__ 70#ifndef __ASSEMBLY__
@@ -123,6 +123,10 @@ static inline notrace unsigned long arch_local_irq_save(void)
123#define DISABLE_INTERRUPTS(x) cli 123#define DISABLE_INTERRUPTS(x) cli
124 124
125#ifdef CONFIG_X86_64 125#ifdef CONFIG_X86_64
126#ifdef CONFIG_DEBUG_ENTRY
127#define SAVE_FLAGS(x) pushfq; popq %rax
128#endif
129
126#define SWAPGS swapgs 130#define SWAPGS swapgs
127/* 131/*
128 * Currently paravirt can't handle swapgs nicely when we 132 * Currently paravirt can't handle swapgs nicely when we
@@ -135,8 +139,6 @@ static inline notrace unsigned long arch_local_irq_save(void)
135 */ 139 */
136#define SWAPGS_UNSAFE_STACK swapgs 140#define SWAPGS_UNSAFE_STACK swapgs
137 141
138#define PARAVIRT_ADJUST_EXCEPTION_FRAME /* */
139
140#define INTERRUPT_RETURN jmp native_iret 142#define INTERRUPT_RETURN jmp native_iret
141#define USERGS_SYSRET64 \ 143#define USERGS_SYSRET64 \
142 swapgs; \ 144 swapgs; \
@@ -145,18 +147,12 @@ static inline notrace unsigned long arch_local_irq_save(void)
145 swapgs; \ 147 swapgs; \
146 sysretl 148 sysretl
147 149
148#ifdef CONFIG_DEBUG_ENTRY
149#define SAVE_FLAGS(x) pushfq; popq %rax
150#endif
151#else 150#else
152#define INTERRUPT_RETURN iret 151#define INTERRUPT_RETURN iret
153#define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit
154#define GET_CR0_INTO_EAX movl %cr0, %eax
155#endif 152#endif
156 153
157
158#endif /* __ASSEMBLY__ */ 154#endif /* __ASSEMBLY__ */
159#endif /* CONFIG_PARAVIRT */ 155#endif /* CONFIG_PARAVIRT_XXL */
160 156
161#ifndef __ASSEMBLY__ 157#ifndef __ASSEMBLY__
162static inline int arch_irqs_disabled_flags(unsigned long flags) 158static inline int arch_irqs_disabled_flags(unsigned long flags)
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index eeeb9289c764..0ca50611e8ce 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -16,12 +16,12 @@
16 16
17extern atomic64_t last_mm_ctx_id; 17extern atomic64_t last_mm_ctx_id;
18 18
19#ifndef CONFIG_PARAVIRT 19#ifndef CONFIG_PARAVIRT_XXL
20static inline void paravirt_activate_mm(struct mm_struct *prev, 20static inline void paravirt_activate_mm(struct mm_struct *prev,
21 struct mm_struct *next) 21 struct mm_struct *next)
22{ 22{
23} 23}
24#endif /* !CONFIG_PARAVIRT */ 24#endif /* !CONFIG_PARAVIRT_XXL */
25 25
26#ifdef CONFIG_PERF_EVENTS 26#ifdef CONFIG_PERF_EVENTS
27 27
diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index f37704497d8f..0d6271cce198 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -351,6 +351,8 @@ int hyperv_flush_guest_mapping(u64 as);
351 351
352#ifdef CONFIG_X86_64 352#ifdef CONFIG_X86_64
353void hv_apic_init(void); 353void hv_apic_init(void);
354void __init hv_init_spinlocks(void);
355bool hv_vcpu_is_preempted(int vcpu);
354#else 356#else
355static inline void hv_apic_init(void) {} 357static inline void hv_apic_init(void) {}
356#endif 358#endif
diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h
index 04addd6e0a4a..91e4cf189914 100644
--- a/arch/x86/include/asm/msr.h
+++ b/arch/x86/include/asm/msr.h
@@ -242,7 +242,7 @@ static inline unsigned long long native_read_pmc(int counter)
242 return EAX_EDX_VAL(val, low, high); 242 return EAX_EDX_VAL(val, low, high);
243} 243}
244 244
245#ifdef CONFIG_PARAVIRT 245#ifdef CONFIG_PARAVIRT_XXL
246#include <asm/paravirt.h> 246#include <asm/paravirt.h>
247#else 247#else
248#include <linux/errno.h> 248#include <linux/errno.h>
@@ -305,7 +305,7 @@ do { \
305 305
306#define rdpmcl(counter, val) ((val) = native_read_pmc(counter)) 306#define rdpmcl(counter, val) ((val) = native_read_pmc(counter))
307 307
308#endif /* !CONFIG_PARAVIRT */ 308#endif /* !CONFIG_PARAVIRT_XXL */
309 309
310/* 310/*
311 * 64-bit version of wrmsr_safe(): 311 * 64-bit version of wrmsr_safe():
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index e375d4266b53..4bf42f9e4eea 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -17,16 +17,73 @@
17#include <linux/cpumask.h> 17#include <linux/cpumask.h>
18#include <asm/frame.h> 18#include <asm/frame.h>
19 19
20static inline unsigned long long paravirt_sched_clock(void)
21{
22 return PVOP_CALL0(unsigned long long, time.sched_clock);
23}
24
25struct static_key;
26extern struct static_key paravirt_steal_enabled;
27extern struct static_key paravirt_steal_rq_enabled;
28
29static inline u64 paravirt_steal_clock(int cpu)
30{
31 return PVOP_CALL1(u64, time.steal_clock, cpu);
32}
33
34/* The paravirtualized I/O functions */
35static inline void slow_down_io(void)
36{
37 pv_ops.cpu.io_delay();
38#ifdef REALLY_SLOW_IO
39 pv_ops.cpu.io_delay();
40 pv_ops.cpu.io_delay();
41 pv_ops.cpu.io_delay();
42#endif
43}
44
45static inline void __flush_tlb(void)
46{
47 PVOP_VCALL0(mmu.flush_tlb_user);
48}
49
50static inline void __flush_tlb_global(void)
51{
52 PVOP_VCALL0(mmu.flush_tlb_kernel);
53}
54
55static inline void __flush_tlb_one_user(unsigned long addr)
56{
57 PVOP_VCALL1(mmu.flush_tlb_one_user, addr);
58}
59
60static inline void flush_tlb_others(const struct cpumask *cpumask,
61 const struct flush_tlb_info *info)
62{
63 PVOP_VCALL2(mmu.flush_tlb_others, cpumask, info);
64}
65
66static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table)
67{
68 PVOP_VCALL2(mmu.tlb_remove_table, tlb, table);
69}
70
71static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
72{
73 PVOP_VCALL1(mmu.exit_mmap, mm);
74}
75
76#ifdef CONFIG_PARAVIRT_XXL
20static inline void load_sp0(unsigned long sp0) 77static inline void load_sp0(unsigned long sp0)
21{ 78{
22 PVOP_VCALL1(pv_cpu_ops.load_sp0, sp0); 79 PVOP_VCALL1(cpu.load_sp0, sp0);
23} 80}
24 81
25/* The paravirtualized CPUID instruction. */ 82/* The paravirtualized CPUID instruction. */
26static inline void __cpuid(unsigned int *eax, unsigned int *ebx, 83static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
27 unsigned int *ecx, unsigned int *edx) 84 unsigned int *ecx, unsigned int *edx)
28{ 85{
29 PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx); 86 PVOP_VCALL4(cpu.cpuid, eax, ebx, ecx, edx);
30} 87}
31 88
32/* 89/*
@@ -34,98 +91,98 @@ static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
34 */ 91 */
35static inline unsigned long paravirt_get_debugreg(int reg) 92static inline unsigned long paravirt_get_debugreg(int reg)
36{ 93{
37 return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg); 94 return PVOP_CALL1(unsigned long, cpu.get_debugreg, reg);
38} 95}
39#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) 96#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
40static inline void set_debugreg(unsigned long val, int reg) 97static inline void set_debugreg(unsigned long val, int reg)
41{ 98{
42 PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val); 99 PVOP_VCALL2(cpu.set_debugreg, reg, val);
43} 100}
44 101
45static inline unsigned long read_cr0(void) 102static inline unsigned long read_cr0(void)
46{ 103{
47 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0); 104 return PVOP_CALL0(unsigned long, cpu.read_cr0);
48} 105}
49 106
50static inline void write_cr0(unsigned long x) 107static inline void write_cr0(unsigned long x)
51{ 108{
52 PVOP_VCALL1(pv_cpu_ops.write_cr0, x); 109 PVOP_VCALL1(cpu.write_cr0, x);
53} 110}
54 111
55static inline unsigned long read_cr2(void) 112static inline unsigned long read_cr2(void)
56{ 113{
57 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2); 114 return PVOP_CALL0(unsigned long, mmu.read_cr2);
58} 115}
59 116
60static inline void write_cr2(unsigned long x) 117static inline void write_cr2(unsigned long x)
61{ 118{
62 PVOP_VCALL1(pv_mmu_ops.write_cr2, x); 119 PVOP_VCALL1(mmu.write_cr2, x);
63} 120}
64 121
65static inline unsigned long __read_cr3(void) 122static inline unsigned long __read_cr3(void)
66{ 123{
67 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3); 124 return PVOP_CALL0(unsigned long, mmu.read_cr3);
68} 125}
69 126
70static inline void write_cr3(unsigned long x) 127static inline void write_cr3(unsigned long x)
71{ 128{
72 PVOP_VCALL1(pv_mmu_ops.write_cr3, x); 129 PVOP_VCALL1(mmu.write_cr3, x);
73} 130}
74 131
75static inline void __write_cr4(unsigned long x) 132static inline void __write_cr4(unsigned long x)
76{ 133{
77 PVOP_VCALL1(pv_cpu_ops.write_cr4, x); 134 PVOP_VCALL1(cpu.write_cr4, x);
78} 135}
79 136
80#ifdef CONFIG_X86_64 137#ifdef CONFIG_X86_64
81static inline unsigned long read_cr8(void) 138static inline unsigned long read_cr8(void)
82{ 139{
83 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8); 140 return PVOP_CALL0(unsigned long, cpu.read_cr8);
84} 141}
85 142
86static inline void write_cr8(unsigned long x) 143static inline void write_cr8(unsigned long x)
87{ 144{
88 PVOP_VCALL1(pv_cpu_ops.write_cr8, x); 145 PVOP_VCALL1(cpu.write_cr8, x);
89} 146}
90#endif 147#endif
91 148
92static inline void arch_safe_halt(void) 149static inline void arch_safe_halt(void)
93{ 150{
94 PVOP_VCALL0(pv_irq_ops.safe_halt); 151 PVOP_VCALL0(irq.safe_halt);
95} 152}
96 153
97static inline void halt(void) 154static inline void halt(void)
98{ 155{
99 PVOP_VCALL0(pv_irq_ops.halt); 156 PVOP_VCALL0(irq.halt);
100} 157}
101 158
102static inline void wbinvd(void) 159static inline void wbinvd(void)
103{ 160{
104 PVOP_VCALL0(pv_cpu_ops.wbinvd); 161 PVOP_VCALL0(cpu.wbinvd);
105} 162}
106 163
107#define get_kernel_rpl() (pv_info.kernel_rpl) 164#define get_kernel_rpl() (pv_info.kernel_rpl)
108 165
109static inline u64 paravirt_read_msr(unsigned msr) 166static inline u64 paravirt_read_msr(unsigned msr)
110{ 167{
111 return PVOP_CALL1(u64, pv_cpu_ops.read_msr, msr); 168 return PVOP_CALL1(u64, cpu.read_msr, msr);
112} 169}
113 170
114static inline void paravirt_write_msr(unsigned msr, 171static inline void paravirt_write_msr(unsigned msr,
115 unsigned low, unsigned high) 172 unsigned low, unsigned high)
116{ 173{
117 PVOP_VCALL3(pv_cpu_ops.write_msr, msr, low, high); 174 PVOP_VCALL3(cpu.write_msr, msr, low, high);
118} 175}
119 176
120static inline u64 paravirt_read_msr_safe(unsigned msr, int *err) 177static inline u64 paravirt_read_msr_safe(unsigned msr, int *err)
121{ 178{
122 return PVOP_CALL2(u64, pv_cpu_ops.read_msr_safe, msr, err); 179 return PVOP_CALL2(u64, cpu.read_msr_safe, msr, err);
123} 180}
124 181
125static inline int paravirt_write_msr_safe(unsigned msr, 182static inline int paravirt_write_msr_safe(unsigned msr,
126 unsigned low, unsigned high) 183 unsigned low, unsigned high)
127{ 184{
128 return PVOP_CALL3(int, pv_cpu_ops.write_msr_safe, msr, low, high); 185 return PVOP_CALL3(int, cpu.write_msr_safe, msr, low, high);
129} 186}
130 187
131#define rdmsr(msr, val1, val2) \ 188#define rdmsr(msr, val1, val2) \
@@ -170,23 +227,9 @@ static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
170 return err; 227 return err;
171} 228}
172 229
173static inline unsigned long long paravirt_sched_clock(void)
174{
175 return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
176}
177
178struct static_key;
179extern struct static_key paravirt_steal_enabled;
180extern struct static_key paravirt_steal_rq_enabled;
181
182static inline u64 paravirt_steal_clock(int cpu)
183{
184 return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu);
185}
186
187static inline unsigned long long paravirt_read_pmc(int counter) 230static inline unsigned long long paravirt_read_pmc(int counter)
188{ 231{
189 return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter); 232 return PVOP_CALL1(u64, cpu.read_pmc, counter);
190} 233}
191 234
192#define rdpmc(counter, low, high) \ 235#define rdpmc(counter, low, high) \
@@ -200,166 +243,127 @@ do { \
200 243
201static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) 244static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
202{ 245{
203 PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries); 246 PVOP_VCALL2(cpu.alloc_ldt, ldt, entries);
204} 247}
205 248
206static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries) 249static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
207{ 250{
208 PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries); 251 PVOP_VCALL2(cpu.free_ldt, ldt, entries);
209} 252}
210 253
211static inline void load_TR_desc(void) 254static inline void load_TR_desc(void)
212{ 255{
213 PVOP_VCALL0(pv_cpu_ops.load_tr_desc); 256 PVOP_VCALL0(cpu.load_tr_desc);
214} 257}
215static inline void load_gdt(const struct desc_ptr *dtr) 258static inline void load_gdt(const struct desc_ptr *dtr)
216{ 259{
217 PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr); 260 PVOP_VCALL1(cpu.load_gdt, dtr);
218} 261}
219static inline void load_idt(const struct desc_ptr *dtr) 262static inline void load_idt(const struct desc_ptr *dtr)
220{ 263{
221 PVOP_VCALL1(pv_cpu_ops.load_idt, dtr); 264 PVOP_VCALL1(cpu.load_idt, dtr);
222} 265}
223static inline void set_ldt(const void *addr, unsigned entries) 266static inline void set_ldt(const void *addr, unsigned entries)
224{ 267{
225 PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries); 268 PVOP_VCALL2(cpu.set_ldt, addr, entries);
226} 269}
227static inline unsigned long paravirt_store_tr(void) 270static inline unsigned long paravirt_store_tr(void)
228{ 271{
229 return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr); 272 return PVOP_CALL0(unsigned long, cpu.store_tr);
230} 273}
274
231#define store_tr(tr) ((tr) = paravirt_store_tr()) 275#define store_tr(tr) ((tr) = paravirt_store_tr())
232static inline void load_TLS(struct thread_struct *t, unsigned cpu) 276static inline void load_TLS(struct thread_struct *t, unsigned cpu)
233{ 277{
234 PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu); 278 PVOP_VCALL2(cpu.load_tls, t, cpu);
235} 279}
236 280
237#ifdef CONFIG_X86_64 281#ifdef CONFIG_X86_64
238static inline void load_gs_index(unsigned int gs) 282static inline void load_gs_index(unsigned int gs)
239{ 283{
240 PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs); 284 PVOP_VCALL1(cpu.load_gs_index, gs);
241} 285}
242#endif 286#endif
243 287
244static inline void write_ldt_entry(struct desc_struct *dt, int entry, 288static inline void write_ldt_entry(struct desc_struct *dt, int entry,
245 const void *desc) 289 const void *desc)
246{ 290{
247 PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc); 291 PVOP_VCALL3(cpu.write_ldt_entry, dt, entry, desc);
248} 292}
249 293
250static inline void write_gdt_entry(struct desc_struct *dt, int entry, 294static inline void write_gdt_entry(struct desc_struct *dt, int entry,
251 void *desc, int type) 295 void *desc, int type)
252{ 296{
253 PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type); 297 PVOP_VCALL4(cpu.write_gdt_entry, dt, entry, desc, type);
254} 298}
255 299
256static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g) 300static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
257{ 301{
258 PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g); 302 PVOP_VCALL3(cpu.write_idt_entry, dt, entry, g);
259} 303}
260static inline void set_iopl_mask(unsigned mask) 304static inline void set_iopl_mask(unsigned mask)
261{ 305{
262 PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask); 306 PVOP_VCALL1(cpu.set_iopl_mask, mask);
263}
264
265/* The paravirtualized I/O functions */
266static inline void slow_down_io(void)
267{
268 pv_cpu_ops.io_delay();
269#ifdef REALLY_SLOW_IO
270 pv_cpu_ops.io_delay();
271 pv_cpu_ops.io_delay();
272 pv_cpu_ops.io_delay();
273#endif
274} 307}
275 308
276static inline void paravirt_activate_mm(struct mm_struct *prev, 309static inline void paravirt_activate_mm(struct mm_struct *prev,
277 struct mm_struct *next) 310 struct mm_struct *next)
278{ 311{
279 PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next); 312 PVOP_VCALL2(mmu.activate_mm, prev, next);
280} 313}
281 314
282static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm, 315static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
283 struct mm_struct *mm) 316 struct mm_struct *mm)
284{ 317{
285 PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm); 318 PVOP_VCALL2(mmu.dup_mmap, oldmm, mm);
286}
287
288static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
289{
290 PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
291}
292
293static inline void __flush_tlb(void)
294{
295 PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
296}
297static inline void __flush_tlb_global(void)
298{
299 PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
300}
301static inline void __flush_tlb_one_user(unsigned long addr)
302{
303 PVOP_VCALL1(pv_mmu_ops.flush_tlb_one_user, addr);
304}
305
306static inline void flush_tlb_others(const struct cpumask *cpumask,
307 const struct flush_tlb_info *info)
308{
309 PVOP_VCALL2(pv_mmu_ops.flush_tlb_others, cpumask, info);
310}
311
312static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table)
313{
314 PVOP_VCALL2(pv_mmu_ops.tlb_remove_table, tlb, table);
315} 319}
316 320
317static inline int paravirt_pgd_alloc(struct mm_struct *mm) 321static inline int paravirt_pgd_alloc(struct mm_struct *mm)
318{ 322{
319 return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm); 323 return PVOP_CALL1(int, mmu.pgd_alloc, mm);
320} 324}
321 325
322static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) 326static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
323{ 327{
324 PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd); 328 PVOP_VCALL2(mmu.pgd_free, mm, pgd);
325} 329}
326 330
327static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) 331static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
328{ 332{
329 PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn); 333 PVOP_VCALL2(mmu.alloc_pte, mm, pfn);
330} 334}
331static inline void paravirt_release_pte(unsigned long pfn) 335static inline void paravirt_release_pte(unsigned long pfn)
332{ 336{
333 PVOP_VCALL1(pv_mmu_ops.release_pte, pfn); 337 PVOP_VCALL1(mmu.release_pte, pfn);
334} 338}
335 339
336static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) 340static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
337{ 341{
338 PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn); 342 PVOP_VCALL2(mmu.alloc_pmd, mm, pfn);
339} 343}
340 344
341static inline void paravirt_release_pmd(unsigned long pfn) 345static inline void paravirt_release_pmd(unsigned long pfn)
342{ 346{
343 PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn); 347 PVOP_VCALL1(mmu.release_pmd, pfn);
344} 348}
345 349
346static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) 350static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
347{ 351{
348 PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn); 352 PVOP_VCALL2(mmu.alloc_pud, mm, pfn);
349} 353}
350static inline void paravirt_release_pud(unsigned long pfn) 354static inline void paravirt_release_pud(unsigned long pfn)
351{ 355{
352 PVOP_VCALL1(pv_mmu_ops.release_pud, pfn); 356 PVOP_VCALL1(mmu.release_pud, pfn);
353} 357}
354 358
355static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn) 359static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn)
356{ 360{
357 PVOP_VCALL2(pv_mmu_ops.alloc_p4d, mm, pfn); 361 PVOP_VCALL2(mmu.alloc_p4d, mm, pfn);
358} 362}
359 363
360static inline void paravirt_release_p4d(unsigned long pfn) 364static inline void paravirt_release_p4d(unsigned long pfn)
361{ 365{
362 PVOP_VCALL1(pv_mmu_ops.release_p4d, pfn); 366 PVOP_VCALL1(mmu.release_p4d, pfn);
363} 367}
364 368
365static inline pte_t __pte(pteval_t val) 369static inline pte_t __pte(pteval_t val)
@@ -367,13 +371,9 @@ static inline pte_t __pte(pteval_t val)
367 pteval_t ret; 371 pteval_t ret;
368 372
369 if (sizeof(pteval_t) > sizeof(long)) 373 if (sizeof(pteval_t) > sizeof(long))
370 ret = PVOP_CALLEE2(pteval_t, 374 ret = PVOP_CALLEE2(pteval_t, mmu.make_pte, val, (u64)val >> 32);
371 pv_mmu_ops.make_pte,
372 val, (u64)val >> 32);
373 else 375 else
374 ret = PVOP_CALLEE1(pteval_t, 376 ret = PVOP_CALLEE1(pteval_t, mmu.make_pte, val);
375 pv_mmu_ops.make_pte,
376 val);
377 377
378 return (pte_t) { .pte = ret }; 378 return (pte_t) { .pte = ret };
379} 379}
@@ -383,11 +383,10 @@ static inline pteval_t pte_val(pte_t pte)
383 pteval_t ret; 383 pteval_t ret;
384 384
385 if (sizeof(pteval_t) > sizeof(long)) 385 if (sizeof(pteval_t) > sizeof(long))
386 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val, 386 ret = PVOP_CALLEE2(pteval_t, mmu.pte_val,
387 pte.pte, (u64)pte.pte >> 32); 387 pte.pte, (u64)pte.pte >> 32);
388 else 388 else
389 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val, 389 ret = PVOP_CALLEE1(pteval_t, mmu.pte_val, pte.pte);
390 pte.pte);
391 390
392 return ret; 391 return ret;
393} 392}
@@ -397,11 +396,9 @@ static inline pgd_t __pgd(pgdval_t val)
397 pgdval_t ret; 396 pgdval_t ret;
398 397
399 if (sizeof(pgdval_t) > sizeof(long)) 398 if (sizeof(pgdval_t) > sizeof(long))
400 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd, 399 ret = PVOP_CALLEE2(pgdval_t, mmu.make_pgd, val, (u64)val >> 32);
401 val, (u64)val >> 32);
402 else 400 else
403 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd, 401 ret = PVOP_CALLEE1(pgdval_t, mmu.make_pgd, val);
404 val);
405 402
406 return (pgd_t) { ret }; 403 return (pgd_t) { ret };
407} 404}
@@ -411,11 +408,10 @@ static inline pgdval_t pgd_val(pgd_t pgd)
411 pgdval_t ret; 408 pgdval_t ret;
412 409
413 if (sizeof(pgdval_t) > sizeof(long)) 410 if (sizeof(pgdval_t) > sizeof(long))
414 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val, 411 ret = PVOP_CALLEE2(pgdval_t, mmu.pgd_val,
415 pgd.pgd, (u64)pgd.pgd >> 32); 412 pgd.pgd, (u64)pgd.pgd >> 32);
416 else 413 else
417 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val, 414 ret = PVOP_CALLEE1(pgdval_t, mmu.pgd_val, pgd.pgd);
418 pgd.pgd);
419 415
420 return ret; 416 return ret;
421} 417}
@@ -426,8 +422,7 @@ static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long a
426{ 422{
427 pteval_t ret; 423 pteval_t ret;
428 424
429 ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start, 425 ret = PVOP_CALL3(pteval_t, mmu.ptep_modify_prot_start, mm, addr, ptep);
430 mm, addr, ptep);
431 426
432 return (pte_t) { .pte = ret }; 427 return (pte_t) { .pte = ret };
433} 428}
@@ -437,20 +432,18 @@ static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long a
437{ 432{
438 if (sizeof(pteval_t) > sizeof(long)) 433 if (sizeof(pteval_t) > sizeof(long))
439 /* 5 arg words */ 434 /* 5 arg words */
440 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte); 435 pv_ops.mmu.ptep_modify_prot_commit(mm, addr, ptep, pte);
441 else 436 else
442 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit, 437 PVOP_VCALL4(mmu.ptep_modify_prot_commit,
443 mm, addr, ptep, pte.pte); 438 mm, addr, ptep, pte.pte);
444} 439}
445 440
446static inline void set_pte(pte_t *ptep, pte_t pte) 441static inline void set_pte(pte_t *ptep, pte_t pte)
447{ 442{
448 if (sizeof(pteval_t) > sizeof(long)) 443 if (sizeof(pteval_t) > sizeof(long))
449 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep, 444 PVOP_VCALL3(mmu.set_pte, ptep, pte.pte, (u64)pte.pte >> 32);
450 pte.pte, (u64)pte.pte >> 32);
451 else 445 else
452 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep, 446 PVOP_VCALL2(mmu.set_pte, ptep, pte.pte);
453 pte.pte);
454} 447}
455 448
456static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, 449static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
@@ -458,9 +451,9 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
458{ 451{
459 if (sizeof(pteval_t) > sizeof(long)) 452 if (sizeof(pteval_t) > sizeof(long))
460 /* 5 arg words */ 453 /* 5 arg words */
461 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte); 454 pv_ops.mmu.set_pte_at(mm, addr, ptep, pte);
462 else 455 else
463 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte); 456 PVOP_VCALL4(mmu.set_pte_at, mm, addr, ptep, pte.pte);
464} 457}
465 458
466static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) 459static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
@@ -468,9 +461,9 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
468 pmdval_t val = native_pmd_val(pmd); 461 pmdval_t val = native_pmd_val(pmd);
469 462
470 if (sizeof(pmdval_t) > sizeof(long)) 463 if (sizeof(pmdval_t) > sizeof(long))
471 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32); 464 PVOP_VCALL3(mmu.set_pmd, pmdp, val, (u64)val >> 32);
472 else 465 else
473 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val); 466 PVOP_VCALL2(mmu.set_pmd, pmdp, val);
474} 467}
475 468
476#if CONFIG_PGTABLE_LEVELS >= 3 469#if CONFIG_PGTABLE_LEVELS >= 3
@@ -479,11 +472,9 @@ static inline pmd_t __pmd(pmdval_t val)
479 pmdval_t ret; 472 pmdval_t ret;
480 473
481 if (sizeof(pmdval_t) > sizeof(long)) 474 if (sizeof(pmdval_t) > sizeof(long))
482 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd, 475 ret = PVOP_CALLEE2(pmdval_t, mmu.make_pmd, val, (u64)val >> 32);
483 val, (u64)val >> 32);
484 else 476 else
485 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd, 477 ret = PVOP_CALLEE1(pmdval_t, mmu.make_pmd, val);
486 val);
487 478
488 return (pmd_t) { ret }; 479 return (pmd_t) { ret };
489} 480}
@@ -493,11 +484,10 @@ static inline pmdval_t pmd_val(pmd_t pmd)
493 pmdval_t ret; 484 pmdval_t ret;
494 485
495 if (sizeof(pmdval_t) > sizeof(long)) 486 if (sizeof(pmdval_t) > sizeof(long))
496 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val, 487 ret = PVOP_CALLEE2(pmdval_t, mmu.pmd_val,
497 pmd.pmd, (u64)pmd.pmd >> 32); 488 pmd.pmd, (u64)pmd.pmd >> 32);
498 else 489 else
499 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val, 490 ret = PVOP_CALLEE1(pmdval_t, mmu.pmd_val, pmd.pmd);
500 pmd.pmd);
501 491
502 return ret; 492 return ret;
503} 493}
@@ -507,39 +497,23 @@ static inline void set_pud(pud_t *pudp, pud_t pud)
507 pudval_t val = native_pud_val(pud); 497 pudval_t val = native_pud_val(pud);
508 498
509 if (sizeof(pudval_t) > sizeof(long)) 499 if (sizeof(pudval_t) > sizeof(long))
510 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp, 500 PVOP_VCALL3(mmu.set_pud, pudp, val, (u64)val >> 32);
511 val, (u64)val >> 32);
512 else 501 else
513 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp, 502 PVOP_VCALL2(mmu.set_pud, pudp, val);
514 val);
515} 503}
516#if CONFIG_PGTABLE_LEVELS >= 4 504#if CONFIG_PGTABLE_LEVELS >= 4
517static inline pud_t __pud(pudval_t val) 505static inline pud_t __pud(pudval_t val)
518{ 506{
519 pudval_t ret; 507 pudval_t ret;
520 508
521 if (sizeof(pudval_t) > sizeof(long)) 509 ret = PVOP_CALLEE1(pudval_t, mmu.make_pud, val);
522 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
523 val, (u64)val >> 32);
524 else
525 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
526 val);
527 510
528 return (pud_t) { ret }; 511 return (pud_t) { ret };
529} 512}
530 513
531static inline pudval_t pud_val(pud_t pud) 514static inline pudval_t pud_val(pud_t pud)
532{ 515{
533 pudval_t ret; 516 return PVOP_CALLEE1(pudval_t, mmu.pud_val, pud.pud);
534
535 if (sizeof(pudval_t) > sizeof(long))
536 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
537 pud.pud, (u64)pud.pud >> 32);
538 else
539 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
540 pud.pud);
541
542 return ret;
543} 517}
544 518
545static inline void pud_clear(pud_t *pudp) 519static inline void pud_clear(pud_t *pudp)
@@ -551,31 +525,26 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
551{ 525{
552 p4dval_t val = native_p4d_val(p4d); 526 p4dval_t val = native_p4d_val(p4d);
553 527
554 if (sizeof(p4dval_t) > sizeof(long)) 528 PVOP_VCALL2(mmu.set_p4d, p4dp, val);
555 PVOP_VCALL3(pv_mmu_ops.set_p4d, p4dp,
556 val, (u64)val >> 32);
557 else
558 PVOP_VCALL2(pv_mmu_ops.set_p4d, p4dp,
559 val);
560} 529}
561 530
562#if CONFIG_PGTABLE_LEVELS >= 5 531#if CONFIG_PGTABLE_LEVELS >= 5
563 532
564static inline p4d_t __p4d(p4dval_t val) 533static inline p4d_t __p4d(p4dval_t val)
565{ 534{
566 p4dval_t ret = PVOP_CALLEE1(p4dval_t, pv_mmu_ops.make_p4d, val); 535 p4dval_t ret = PVOP_CALLEE1(p4dval_t, mmu.make_p4d, val);
567 536
568 return (p4d_t) { ret }; 537 return (p4d_t) { ret };
569} 538}
570 539
571static inline p4dval_t p4d_val(p4d_t p4d) 540static inline p4dval_t p4d_val(p4d_t p4d)
572{ 541{
573 return PVOP_CALLEE1(p4dval_t, pv_mmu_ops.p4d_val, p4d.p4d); 542 return PVOP_CALLEE1(p4dval_t, mmu.p4d_val, p4d.p4d);
574} 543}
575 544
576static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd) 545static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd)
577{ 546{
578 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp, native_pgd_val(pgd)); 547 PVOP_VCALL2(mmu.set_pgd, pgdp, native_pgd_val(pgd));
579} 548}
580 549
581#define set_pgd(pgdp, pgdval) do { \ 550#define set_pgd(pgdp, pgdval) do { \
@@ -606,19 +575,18 @@ static inline void p4d_clear(p4d_t *p4dp)
606 64-bit pte atomically */ 575 64-bit pte atomically */
607static inline void set_pte_atomic(pte_t *ptep, pte_t pte) 576static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
608{ 577{
609 PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep, 578 PVOP_VCALL3(mmu.set_pte_atomic, ptep, pte.pte, pte.pte >> 32);
610 pte.pte, pte.pte >> 32);
611} 579}
612 580
613static inline void pte_clear(struct mm_struct *mm, unsigned long addr, 581static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
614 pte_t *ptep) 582 pte_t *ptep)
615{ 583{
616 PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep); 584 PVOP_VCALL3(mmu.pte_clear, mm, addr, ptep);
617} 585}
618 586
619static inline void pmd_clear(pmd_t *pmdp) 587static inline void pmd_clear(pmd_t *pmdp)
620{ 588{
621 PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp); 589 PVOP_VCALL1(mmu.pmd_clear, pmdp);
622} 590}
623#else /* !CONFIG_X86_PAE */ 591#else /* !CONFIG_X86_PAE */
624static inline void set_pte_atomic(pte_t *ptep, pte_t pte) 592static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
@@ -641,64 +609,68 @@ static inline void pmd_clear(pmd_t *pmdp)
641#define __HAVE_ARCH_START_CONTEXT_SWITCH 609#define __HAVE_ARCH_START_CONTEXT_SWITCH
642static inline void arch_start_context_switch(struct task_struct *prev) 610static inline void arch_start_context_switch(struct task_struct *prev)
643{ 611{
644 PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev); 612 PVOP_VCALL1(cpu.start_context_switch, prev);
645} 613}
646 614
647static inline void arch_end_context_switch(struct task_struct *next) 615static inline void arch_end_context_switch(struct task_struct *next)
648{ 616{
649 PVOP_VCALL1(pv_cpu_ops.end_context_switch, next); 617 PVOP_VCALL1(cpu.end_context_switch, next);
650} 618}
651 619
652#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE 620#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
653static inline void arch_enter_lazy_mmu_mode(void) 621static inline void arch_enter_lazy_mmu_mode(void)
654{ 622{
655 PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter); 623 PVOP_VCALL0(mmu.lazy_mode.enter);
656} 624}
657 625
658static inline void arch_leave_lazy_mmu_mode(void) 626static inline void arch_leave_lazy_mmu_mode(void)
659{ 627{
660 PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave); 628 PVOP_VCALL0(mmu.lazy_mode.leave);
661} 629}
662 630
663static inline void arch_flush_lazy_mmu_mode(void) 631static inline void arch_flush_lazy_mmu_mode(void)
664{ 632{
665 PVOP_VCALL0(pv_mmu_ops.lazy_mode.flush); 633 PVOP_VCALL0(mmu.lazy_mode.flush);
666} 634}
667 635
668static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, 636static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
669 phys_addr_t phys, pgprot_t flags) 637 phys_addr_t phys, pgprot_t flags)
670{ 638{
671 pv_mmu_ops.set_fixmap(idx, phys, flags); 639 pv_ops.mmu.set_fixmap(idx, phys, flags);
672} 640}
641#endif
673 642
674#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS) 643#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
675 644
676static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock, 645static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock,
677 u32 val) 646 u32 val)
678{ 647{
679 PVOP_VCALL2(pv_lock_ops.queued_spin_lock_slowpath, lock, val); 648 PVOP_VCALL2(lock.queued_spin_lock_slowpath, lock, val);
680} 649}
681 650
682static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock) 651static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock)
683{ 652{
684 PVOP_VCALLEE1(pv_lock_ops.queued_spin_unlock, lock); 653 PVOP_VCALLEE1(lock.queued_spin_unlock, lock);
685} 654}
686 655
687static __always_inline void pv_wait(u8 *ptr, u8 val) 656static __always_inline void pv_wait(u8 *ptr, u8 val)
688{ 657{
689 PVOP_VCALL2(pv_lock_ops.wait, ptr, val); 658 PVOP_VCALL2(lock.wait, ptr, val);
690} 659}
691 660
692static __always_inline void pv_kick(int cpu) 661static __always_inline void pv_kick(int cpu)
693{ 662{
694 PVOP_VCALL1(pv_lock_ops.kick, cpu); 663 PVOP_VCALL1(lock.kick, cpu);
695} 664}
696 665
697static __always_inline bool pv_vcpu_is_preempted(long cpu) 666static __always_inline bool pv_vcpu_is_preempted(long cpu)
698{ 667{
699 return PVOP_CALLEE1(bool, pv_lock_ops.vcpu_is_preempted, cpu); 668 return PVOP_CALLEE1(bool, lock.vcpu_is_preempted, cpu);
700} 669}
701 670
671void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock);
672bool __raw_callee_save___native_vcpu_is_preempted(long cpu);
673
702#endif /* SMP && PARAVIRT_SPINLOCKS */ 674#endif /* SMP && PARAVIRT_SPINLOCKS */
703 675
704#ifdef CONFIG_X86_32 676#ifdef CONFIG_X86_32
@@ -778,24 +750,25 @@ static __always_inline bool pv_vcpu_is_preempted(long cpu)
778#define __PV_IS_CALLEE_SAVE(func) \ 750#define __PV_IS_CALLEE_SAVE(func) \
779 ((struct paravirt_callee_save) { func }) 751 ((struct paravirt_callee_save) { func })
780 752
753#ifdef CONFIG_PARAVIRT_XXL
781static inline notrace unsigned long arch_local_save_flags(void) 754static inline notrace unsigned long arch_local_save_flags(void)
782{ 755{
783 return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl); 756 return PVOP_CALLEE0(unsigned long, irq.save_fl);
784} 757}
785 758
786static inline notrace void arch_local_irq_restore(unsigned long f) 759static inline notrace void arch_local_irq_restore(unsigned long f)
787{ 760{
788 PVOP_VCALLEE1(pv_irq_ops.restore_fl, f); 761 PVOP_VCALLEE1(irq.restore_fl, f);
789} 762}
790 763
791static inline notrace void arch_local_irq_disable(void) 764static inline notrace void arch_local_irq_disable(void)
792{ 765{
793 PVOP_VCALLEE0(pv_irq_ops.irq_disable); 766 PVOP_VCALLEE0(irq.irq_disable);
794} 767}
795 768
796static inline notrace void arch_local_irq_enable(void) 769static inline notrace void arch_local_irq_enable(void)
797{ 770{
798 PVOP_VCALLEE0(pv_irq_ops.irq_enable); 771 PVOP_VCALLEE0(irq.irq_enable);
799} 772}
800 773
801static inline notrace unsigned long arch_local_irq_save(void) 774static inline notrace unsigned long arch_local_irq_save(void)
@@ -806,6 +779,7 @@ static inline notrace unsigned long arch_local_irq_save(void)
806 arch_local_irq_disable(); 779 arch_local_irq_disable();
807 return f; 780 return f;
808} 781}
782#endif
809 783
810 784
811/* Make sure as little as possible of this mess escapes. */ 785/* Make sure as little as possible of this mess escapes. */
@@ -827,7 +801,7 @@ extern void default_banner(void);
827 801
828#else /* __ASSEMBLY__ */ 802#else /* __ASSEMBLY__ */
829 803
830#define _PVSITE(ptype, clobbers, ops, word, algn) \ 804#define _PVSITE(ptype, ops, word, algn) \
831771:; \ 805771:; \
832 ops; \ 806 ops; \
833772:; \ 807772:; \
@@ -836,7 +810,6 @@ extern void default_banner(void);
836 word 771b; \ 810 word 771b; \
837 .byte ptype; \ 811 .byte ptype; \
838 .byte 772b-771b; \ 812 .byte 772b-771b; \
839 .short clobbers; \
840 .popsection 813 .popsection
841 814
842 815
@@ -868,8 +841,8 @@ extern void default_banner(void);
868 COND_POP(set, CLBR_RCX, rcx); \ 841 COND_POP(set, CLBR_RCX, rcx); \
869 COND_POP(set, CLBR_RAX, rax) 842 COND_POP(set, CLBR_RAX, rax)
870 843
871#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8) 844#define PARA_PATCH(off) ((off) / 8)
872#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8) 845#define PARA_SITE(ptype, ops) _PVSITE(ptype, ops, .quad, 8)
873#define PARA_INDIRECT(addr) *addr(%rip) 846#define PARA_INDIRECT(addr) *addr(%rip)
874#else 847#else
875#define PV_SAVE_REGS(set) \ 848#define PV_SAVE_REGS(set) \
@@ -883,46 +856,41 @@ extern void default_banner(void);
883 COND_POP(set, CLBR_EDI, edi); \ 856 COND_POP(set, CLBR_EDI, edi); \
884 COND_POP(set, CLBR_EAX, eax) 857 COND_POP(set, CLBR_EAX, eax)
885 858
886#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4) 859#define PARA_PATCH(off) ((off) / 4)
887#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4) 860#define PARA_SITE(ptype, ops) _PVSITE(ptype, ops, .long, 4)
888#define PARA_INDIRECT(addr) *%cs:addr 861#define PARA_INDIRECT(addr) *%cs:addr
889#endif 862#endif
890 863
864#ifdef CONFIG_PARAVIRT_XXL
891#define INTERRUPT_RETURN \ 865#define INTERRUPT_RETURN \
892 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \ 866 PARA_SITE(PARA_PATCH(PV_CPU_iret), \
893 ANNOTATE_RETPOLINE_SAFE; \ 867 ANNOTATE_RETPOLINE_SAFE; \
894 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret);) 868 jmp PARA_INDIRECT(pv_ops+PV_CPU_iret);)
895 869
896#define DISABLE_INTERRUPTS(clobbers) \ 870#define DISABLE_INTERRUPTS(clobbers) \
897 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \ 871 PARA_SITE(PARA_PATCH(PV_IRQ_irq_disable), \
898 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \ 872 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
899 ANNOTATE_RETPOLINE_SAFE; \ 873 ANNOTATE_RETPOLINE_SAFE; \
900 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \ 874 call PARA_INDIRECT(pv_ops+PV_IRQ_irq_disable); \
901 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);) 875 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
902 876
903#define ENABLE_INTERRUPTS(clobbers) \ 877#define ENABLE_INTERRUPTS(clobbers) \
904 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \ 878 PARA_SITE(PARA_PATCH(PV_IRQ_irq_enable), \
905 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \ 879 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
906 ANNOTATE_RETPOLINE_SAFE; \ 880 ANNOTATE_RETPOLINE_SAFE; \
907 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \ 881 call PARA_INDIRECT(pv_ops+PV_IRQ_irq_enable); \
908 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);) 882 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
883#endif
909 884
910#ifdef CONFIG_X86_32 885#ifdef CONFIG_X86_64
911#define GET_CR0_INTO_EAX \ 886#ifdef CONFIG_PARAVIRT_XXL
912 push %ecx; push %edx; \
913 ANNOTATE_RETPOLINE_SAFE; \
914 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
915 pop %edx; pop %ecx
916#else /* !CONFIG_X86_32 */
917
918/* 887/*
919 * If swapgs is used while the userspace stack is still current, 888 * If swapgs is used while the userspace stack is still current,
920 * there's no way to call a pvop. The PV replacement *must* be 889 * there's no way to call a pvop. The PV replacement *must* be
921 * inlined, or the swapgs instruction must be trapped and emulated. 890 * inlined, or the swapgs instruction must be trapped and emulated.
922 */ 891 */
923#define SWAPGS_UNSAFE_STACK \ 892#define SWAPGS_UNSAFE_STACK \
924 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \ 893 PARA_SITE(PARA_PATCH(PV_CPU_swapgs), swapgs)
925 swapgs)
926 894
927/* 895/*
928 * Note: swapgs is very special, and in practise is either going to be 896 * Note: swapgs is very special, and in practise is either going to be
@@ -931,44 +899,51 @@ extern void default_banner(void);
931 * it. 899 * it.
932 */ 900 */
933#define SWAPGS \ 901#define SWAPGS \
934 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \ 902 PARA_SITE(PARA_PATCH(PV_CPU_swapgs), \
935 ANNOTATE_RETPOLINE_SAFE; \ 903 ANNOTATE_RETPOLINE_SAFE; \
936 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs); \ 904 call PARA_INDIRECT(pv_ops+PV_CPU_swapgs); \
937 ) 905 )
906#endif
938 907
939#define GET_CR2_INTO_RAX \ 908#define GET_CR2_INTO_RAX \
940 ANNOTATE_RETPOLINE_SAFE; \ 909 ANNOTATE_RETPOLINE_SAFE; \
941 call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2); 910 call PARA_INDIRECT(pv_ops+PV_MMU_read_cr2);
942 911
912#ifdef CONFIG_PARAVIRT_XXL
943#define USERGS_SYSRET64 \ 913#define USERGS_SYSRET64 \
944 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64), \ 914 PARA_SITE(PARA_PATCH(PV_CPU_usergs_sysret64), \
945 CLBR_NONE, \ 915 ANNOTATE_RETPOLINE_SAFE; \
946 ANNOTATE_RETPOLINE_SAFE; \ 916 jmp PARA_INDIRECT(pv_ops+PV_CPU_usergs_sysret64);)
947 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64);)
948 917
949#ifdef CONFIG_DEBUG_ENTRY 918#ifdef CONFIG_DEBUG_ENTRY
950#define SAVE_FLAGS(clobbers) \ 919#define SAVE_FLAGS(clobbers) \
951 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_save_fl), clobbers, \ 920 PARA_SITE(PARA_PATCH(PV_IRQ_save_fl), \
952 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \ 921 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
953 ANNOTATE_RETPOLINE_SAFE; \ 922 ANNOTATE_RETPOLINE_SAFE; \
954 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_save_fl); \ 923 call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl); \
955 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);) 924 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
956#endif 925#endif
926#endif
957 927
958#endif /* CONFIG_X86_32 */ 928#endif /* CONFIG_X86_32 */
959 929
960#endif /* __ASSEMBLY__ */ 930#endif /* __ASSEMBLY__ */
961#else /* CONFIG_PARAVIRT */ 931#else /* CONFIG_PARAVIRT */
962# define default_banner x86_init_noop 932# define default_banner x86_init_noop
933#endif /* !CONFIG_PARAVIRT */
934
963#ifndef __ASSEMBLY__ 935#ifndef __ASSEMBLY__
936#ifndef CONFIG_PARAVIRT_XXL
964static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm, 937static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
965 struct mm_struct *mm) 938 struct mm_struct *mm)
966{ 939{
967} 940}
941#endif
968 942
943#ifndef CONFIG_PARAVIRT
969static inline void paravirt_arch_exit_mmap(struct mm_struct *mm) 944static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
970{ 945{
971} 946}
947#endif
972#endif /* __ASSEMBLY__ */ 948#endif /* __ASSEMBLY__ */
973#endif /* !CONFIG_PARAVIRT */
974#endif /* _ASM_X86_PARAVIRT_H */ 949#endif /* _ASM_X86_PARAVIRT_H */
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 83ce282eed0a..fba54ca23b2a 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -66,12 +66,14 @@ struct paravirt_callee_save {
66 66
67/* general info */ 67/* general info */
68struct pv_info { 68struct pv_info {
69#ifdef CONFIG_PARAVIRT_XXL
69 unsigned int kernel_rpl; 70 unsigned int kernel_rpl;
70 int shared_kernel_pmd; 71 int shared_kernel_pmd;
71 72
72#ifdef CONFIG_X86_64 73#ifdef CONFIG_X86_64
73 u16 extra_user_64bit_cs; /* __USER_CS if none */ 74 u16 extra_user_64bit_cs; /* __USER_CS if none */
74#endif 75#endif
76#endif
75 77
76 const char *name; 78 const char *name;
77}; 79};
@@ -85,17 +87,18 @@ struct pv_init_ops {
85 * the number of bytes of code generated, as we nop pad the 87 * the number of bytes of code generated, as we nop pad the
86 * rest in generic code. 88 * rest in generic code.
87 */ 89 */
88 unsigned (*patch)(u8 type, u16 clobber, void *insnbuf, 90 unsigned (*patch)(u8 type, void *insnbuf,
89 unsigned long addr, unsigned len); 91 unsigned long addr, unsigned len);
90} __no_randomize_layout; 92} __no_randomize_layout;
91 93
92 94#ifdef CONFIG_PARAVIRT_XXL
93struct pv_lazy_ops { 95struct pv_lazy_ops {
94 /* Set deferred update mode, used for batching operations. */ 96 /* Set deferred update mode, used for batching operations. */
95 void (*enter)(void); 97 void (*enter)(void);
96 void (*leave)(void); 98 void (*leave)(void);
97 void (*flush)(void); 99 void (*flush)(void);
98} __no_randomize_layout; 100} __no_randomize_layout;
101#endif
99 102
100struct pv_time_ops { 103struct pv_time_ops {
101 unsigned long long (*sched_clock)(void); 104 unsigned long long (*sched_clock)(void);
@@ -104,6 +107,9 @@ struct pv_time_ops {
104 107
105struct pv_cpu_ops { 108struct pv_cpu_ops {
106 /* hooks for various privileged instructions */ 109 /* hooks for various privileged instructions */
110 void (*io_delay)(void);
111
112#ifdef CONFIG_PARAVIRT_XXL
107 unsigned long (*get_debugreg)(int regno); 113 unsigned long (*get_debugreg)(int regno);
108 void (*set_debugreg)(int regno, unsigned long value); 114 void (*set_debugreg)(int regno, unsigned long value);
109 115
@@ -141,7 +147,6 @@ struct pv_cpu_ops {
141 void (*set_iopl_mask)(unsigned mask); 147 void (*set_iopl_mask)(unsigned mask);
142 148
143 void (*wbinvd)(void); 149 void (*wbinvd)(void);
144 void (*io_delay)(void);
145 150
146 /* cpuid emulation, mostly so that caps bits can be disabled */ 151 /* cpuid emulation, mostly so that caps bits can be disabled */
147 void (*cpuid)(unsigned int *eax, unsigned int *ebx, 152 void (*cpuid)(unsigned int *eax, unsigned int *ebx,
@@ -176,9 +181,11 @@ struct pv_cpu_ops {
176 181
177 void (*start_context_switch)(struct task_struct *prev); 182 void (*start_context_switch)(struct task_struct *prev);
178 void (*end_context_switch)(struct task_struct *next); 183 void (*end_context_switch)(struct task_struct *next);
184#endif
179} __no_randomize_layout; 185} __no_randomize_layout;
180 186
181struct pv_irq_ops { 187struct pv_irq_ops {
188#ifdef CONFIG_PARAVIRT_XXL
182 /* 189 /*
183 * Get/set interrupt state. save_fl and restore_fl are only 190 * Get/set interrupt state. save_fl and restore_fl are only
184 * expected to use X86_EFLAGS_IF; all other bits 191 * expected to use X86_EFLAGS_IF; all other bits
@@ -195,35 +202,34 @@ struct pv_irq_ops {
195 202
196 void (*safe_halt)(void); 203 void (*safe_halt)(void);
197 void (*halt)(void); 204 void (*halt)(void);
198 205#endif
199} __no_randomize_layout; 206} __no_randomize_layout;
200 207
201struct pv_mmu_ops { 208struct pv_mmu_ops {
209 /* TLB operations */
210 void (*flush_tlb_user)(void);
211 void (*flush_tlb_kernel)(void);
212 void (*flush_tlb_one_user)(unsigned long addr);
213 void (*flush_tlb_others)(const struct cpumask *cpus,
214 const struct flush_tlb_info *info);
215
216 void (*tlb_remove_table)(struct mmu_gather *tlb, void *table);
217
218 /* Hook for intercepting the destruction of an mm_struct. */
219 void (*exit_mmap)(struct mm_struct *mm);
220
221#ifdef CONFIG_PARAVIRT_XXL
202 unsigned long (*read_cr2)(void); 222 unsigned long (*read_cr2)(void);
203 void (*write_cr2)(unsigned long); 223 void (*write_cr2)(unsigned long);
204 224
205 unsigned long (*read_cr3)(void); 225 unsigned long (*read_cr3)(void);
206 void (*write_cr3)(unsigned long); 226 void (*write_cr3)(unsigned long);
207 227
208 /* 228 /* Hooks for intercepting the creation/use of an mm_struct. */
209 * Hooks for intercepting the creation/use/destruction of an
210 * mm_struct.
211 */
212 void (*activate_mm)(struct mm_struct *prev, 229 void (*activate_mm)(struct mm_struct *prev,
213 struct mm_struct *next); 230 struct mm_struct *next);
214 void (*dup_mmap)(struct mm_struct *oldmm, 231 void (*dup_mmap)(struct mm_struct *oldmm,
215 struct mm_struct *mm); 232 struct mm_struct *mm);
216 void (*exit_mmap)(struct mm_struct *mm);
217
218
219 /* TLB operations */
220 void (*flush_tlb_user)(void);
221 void (*flush_tlb_kernel)(void);
222 void (*flush_tlb_one_user)(unsigned long addr);
223 void (*flush_tlb_others)(const struct cpumask *cpus,
224 const struct flush_tlb_info *info);
225
226 void (*tlb_remove_table)(struct mmu_gather *tlb, void *table);
227 233
228 /* Hooks for allocating and freeing a pagetable top-level */ 234 /* Hooks for allocating and freeing a pagetable top-level */
229 int (*pgd_alloc)(struct mm_struct *mm); 235 int (*pgd_alloc)(struct mm_struct *mm);
@@ -298,6 +304,7 @@ struct pv_mmu_ops {
298 an mfn. We can tell which is which from the index. */ 304 an mfn. We can tell which is which from the index. */
299 void (*set_fixmap)(unsigned /* enum fixed_addresses */ idx, 305 void (*set_fixmap)(unsigned /* enum fixed_addresses */ idx,
300 phys_addr_t phys, pgprot_t flags); 306 phys_addr_t phys, pgprot_t flags);
307#endif
301} __no_randomize_layout; 308} __no_randomize_layout;
302 309
303struct arch_spinlock; 310struct arch_spinlock;
@@ -321,28 +328,23 @@ struct pv_lock_ops {
321 * number for each function using the offset which we use to indicate 328 * number for each function using the offset which we use to indicate
322 * what to patch. */ 329 * what to patch. */
323struct paravirt_patch_template { 330struct paravirt_patch_template {
324 struct pv_init_ops pv_init_ops; 331 struct pv_init_ops init;
325 struct pv_time_ops pv_time_ops; 332 struct pv_time_ops time;
326 struct pv_cpu_ops pv_cpu_ops; 333 struct pv_cpu_ops cpu;
327 struct pv_irq_ops pv_irq_ops; 334 struct pv_irq_ops irq;
328 struct pv_mmu_ops pv_mmu_ops; 335 struct pv_mmu_ops mmu;
329 struct pv_lock_ops pv_lock_ops; 336 struct pv_lock_ops lock;
330} __no_randomize_layout; 337} __no_randomize_layout;
331 338
332extern struct pv_info pv_info; 339extern struct pv_info pv_info;
333extern struct pv_init_ops pv_init_ops; 340extern struct paravirt_patch_template pv_ops;
334extern struct pv_time_ops pv_time_ops;
335extern struct pv_cpu_ops pv_cpu_ops;
336extern struct pv_irq_ops pv_irq_ops;
337extern struct pv_mmu_ops pv_mmu_ops;
338extern struct pv_lock_ops pv_lock_ops;
339 341
340#define PARAVIRT_PATCH(x) \ 342#define PARAVIRT_PATCH(x) \
341 (offsetof(struct paravirt_patch_template, x) / sizeof(void *)) 343 (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
342 344
343#define paravirt_type(op) \ 345#define paravirt_type(op) \
344 [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \ 346 [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \
345 [paravirt_opptr] "i" (&(op)) 347 [paravirt_opptr] "i" (&(pv_ops.op))
346#define paravirt_clobber(clobber) \ 348#define paravirt_clobber(clobber) \
347 [paravirt_clobber] "i" (clobber) 349 [paravirt_clobber] "i" (clobber)
348 350
@@ -361,20 +363,13 @@ extern struct pv_lock_ops pv_lock_ops;
361 363
362unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len); 364unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len);
363unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len); 365unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len);
364unsigned paravirt_patch_call(void *insnbuf, 366unsigned paravirt_patch_default(u8 type, void *insnbuf,
365 const void *target, u16 tgt_clobbers,
366 unsigned long addr, u16 site_clobbers,
367 unsigned len);
368unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
369 unsigned long addr, unsigned len);
370unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
371 unsigned long addr, unsigned len); 367 unsigned long addr, unsigned len);
372 368
373unsigned paravirt_patch_insns(void *insnbuf, unsigned len, 369unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
374 const char *start, const char *end); 370 const char *start, const char *end);
375 371
376unsigned native_patch(u8 type, u16 clobbers, void *ibuf, 372unsigned native_patch(u8 type, void *ibuf, unsigned long addr, unsigned len);
377 unsigned long addr, unsigned len);
378 373
379int paravirt_disable_iospace(void); 374int paravirt_disable_iospace(void);
380 375
@@ -488,9 +483,9 @@ int paravirt_disable_iospace(void);
488#endif /* CONFIG_X86_32 */ 483#endif /* CONFIG_X86_32 */
489 484
490#ifdef CONFIG_PARAVIRT_DEBUG 485#ifdef CONFIG_PARAVIRT_DEBUG
491#define PVOP_TEST_NULL(op) BUG_ON(op == NULL) 486#define PVOP_TEST_NULL(op) BUG_ON(pv_ops.op == NULL)
492#else 487#else
493#define PVOP_TEST_NULL(op) ((void)op) 488#define PVOP_TEST_NULL(op) ((void)pv_ops.op)
494#endif 489#endif
495 490
496#define PVOP_RETMASK(rettype) \ 491#define PVOP_RETMASK(rettype) \
@@ -666,7 +661,6 @@ struct paravirt_patch_site {
666 u8 *instr; /* original instructions */ 661 u8 *instr; /* original instructions */
667 u8 instrtype; /* type of this instruction */ 662 u8 instrtype; /* type of this instruction */
668 u8 len; /* length of original instruction */ 663 u8 len; /* length of original instruction */
669 u16 clobbers; /* what registers you may clobber */
670}; 664};
671 665
672extern struct paravirt_patch_site __parainstructions[], 666extern struct paravirt_patch_site __parainstructions[],
diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h
index fbd578daa66e..ec7f43327033 100644
--- a/arch/x86/include/asm/pgalloc.h
+++ b/arch/x86/include/asm/pgalloc.h
@@ -8,7 +8,7 @@
8 8
9static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; } 9static inline int __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; }
10 10
11#ifdef CONFIG_PARAVIRT 11#ifdef CONFIG_PARAVIRT_XXL
12#include <asm/paravirt.h> 12#include <asm/paravirt.h>
13#else 13#else
14#define paravirt_pgd_alloc(mm) __paravirt_pgd_alloc(mm) 14#define paravirt_pgd_alloc(mm) __paravirt_pgd_alloc(mm)
diff --git a/arch/x86/include/asm/pgtable-3level_types.h b/arch/x86/include/asm/pgtable-3level_types.h
index 858358a82b14..33845d36897c 100644
--- a/arch/x86/include/asm/pgtable-3level_types.h
+++ b/arch/x86/include/asm/pgtable-3level_types.h
@@ -20,7 +20,7 @@ typedef union {
20} pte_t; 20} pte_t;
21#endif /* !__ASSEMBLY__ */ 21#endif /* !__ASSEMBLY__ */
22 22
23#ifdef CONFIG_PARAVIRT 23#ifdef CONFIG_PARAVIRT_XXL
24#define SHARED_KERNEL_PMD ((!static_cpu_has(X86_FEATURE_PTI) && \ 24#define SHARED_KERNEL_PMD ((!static_cpu_has(X86_FEATURE_PTI) && \
25 (pv_info.shared_kernel_pmd))) 25 (pv_info.shared_kernel_pmd)))
26#else 26#else
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 690c0307afed..40616e805292 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -55,9 +55,9 @@ extern struct mm_struct *pgd_page_get_mm(struct page *page);
55 55
56extern pmdval_t early_pmd_flags; 56extern pmdval_t early_pmd_flags;
57 57
58#ifdef CONFIG_PARAVIRT 58#ifdef CONFIG_PARAVIRT_XXL
59#include <asm/paravirt.h> 59#include <asm/paravirt.h>
60#else /* !CONFIG_PARAVIRT */ 60#else /* !CONFIG_PARAVIRT_XXL */
61#define set_pte(ptep, pte) native_set_pte(ptep, pte) 61#define set_pte(ptep, pte) native_set_pte(ptep, pte)
62#define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte) 62#define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte)
63 63
@@ -112,8 +112,7 @@ extern pmdval_t early_pmd_flags;
112#define __pte(x) native_make_pte(x) 112#define __pte(x) native_make_pte(x)
113 113
114#define arch_end_context_switch(prev) do {} while(0) 114#define arch_end_context_switch(prev) do {} while(0)
115 115#endif /* CONFIG_PARAVIRT_XXL */
116#endif /* CONFIG_PARAVIRT */
117 116
118/* 117/*
119 * The following only work if pte_present() is true. 118 * The following only work if pte_present() is true.
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index d4dfd02b740e..c7a4e2a174b9 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -579,7 +579,7 @@ static inline bool on_thread_stack(void)
579 current_stack_pointer) < THREAD_SIZE; 579 current_stack_pointer) < THREAD_SIZE;
580} 580}
581 581
582#ifdef CONFIG_PARAVIRT 582#ifdef CONFIG_PARAVIRT_XXL
583#include <asm/paravirt.h> 583#include <asm/paravirt.h>
584#else 584#else
585#define __cpuid native_cpuid 585#define __cpuid native_cpuid
@@ -590,7 +590,7 @@ static inline void load_sp0(unsigned long sp0)
590} 590}
591 591
592#define set_iopl_mask native_set_iopl_mask 592#define set_iopl_mask native_set_iopl_mask
593#endif /* CONFIG_PARAVIRT */ 593#endif /* CONFIG_PARAVIRT_XXL */
594 594
595/* Free all resources held by a thread. */ 595/* Free all resources held by a thread. */
596extern void release_thread(struct task_struct *); 596extern void release_thread(struct task_struct *);
diff --git a/arch/x86/include/asm/ptrace.h b/arch/x86/include/asm/ptrace.h
index 25f49af1b13c..f236bcd5485d 100644
--- a/arch/x86/include/asm/ptrace.h
+++ b/arch/x86/include/asm/ptrace.h
@@ -146,7 +146,7 @@ static inline int v8086_mode(struct pt_regs *regs)
146static inline bool user_64bit_mode(struct pt_regs *regs) 146static inline bool user_64bit_mode(struct pt_regs *regs)
147{ 147{
148#ifdef CONFIG_X86_64 148#ifdef CONFIG_X86_64
149#ifndef CONFIG_PARAVIRT 149#ifndef CONFIG_PARAVIRT_XXL
150 /* 150 /*
151 * On non-paravirt systems, this is the only long mode CPL 3 151 * On non-paravirt systems, this is the only long mode CPL 3
152 * selector. We do not allow long mode selectors in the LDT. 152 * selector. We do not allow long mode selectors in the LDT.
diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h
index a314087add07..ac3892920419 100644
--- a/arch/x86/include/asm/segment.h
+++ b/arch/x86/include/asm/segment.h
@@ -210,7 +210,7 @@
210 210
211#endif 211#endif
212 212
213#ifndef CONFIG_PARAVIRT 213#ifndef CONFIG_PARAVIRT_XXL
214# define get_kernel_rpl() 0 214# define get_kernel_rpl() 0
215#endif 215#endif
216 216
diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h
index 317fc59b512c..43c029cdc3fe 100644
--- a/arch/x86/include/asm/special_insns.h
+++ b/arch/x86/include/asm/special_insns.h
@@ -141,7 +141,7 @@ static inline unsigned long __read_cr4(void)
141 return native_read_cr4(); 141 return native_read_cr4();
142} 142}
143 143
144#ifdef CONFIG_PARAVIRT 144#ifdef CONFIG_PARAVIRT_XXL
145#include <asm/paravirt.h> 145#include <asm/paravirt.h>
146#else 146#else
147 147
@@ -208,7 +208,7 @@ static inline void load_gs_index(unsigned selector)
208 208
209#endif 209#endif
210 210
211#endif/* CONFIG_PARAVIRT */ 211#endif /* CONFIG_PARAVIRT_XXL */
212 212
213static inline void clflush(volatile void *__p) 213static inline void clflush(volatile void *__p)
214{ 214{
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 184e9a06b0ff..ebeac487a20c 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -598,7 +598,7 @@ void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
598 BUG_ON(p->len > MAX_PATCH_LEN); 598 BUG_ON(p->len > MAX_PATCH_LEN);
599 /* prep the buffer with the original instructions */ 599 /* prep the buffer with the original instructions */
600 memcpy(insnbuf, p->instr, p->len); 600 memcpy(insnbuf, p->instr, p->len);
601 used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf, 601 used = pv_ops.init.patch(p->instrtype, insnbuf,
602 (unsigned long)p->instr, p->len); 602 (unsigned long)p->instr, p->len);
603 603
604 BUG_ON(used > p->len); 604 BUG_ON(used > p->len);
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 01de31db300d..fc02c3cf238f 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -64,15 +64,12 @@ void common(void) {
64 OFFSET(IA32_RT_SIGFRAME_sigcontext, rt_sigframe_ia32, uc.uc_mcontext); 64 OFFSET(IA32_RT_SIGFRAME_sigcontext, rt_sigframe_ia32, uc.uc_mcontext);
65#endif 65#endif
66 66
67#ifdef CONFIG_PARAVIRT 67#ifdef CONFIG_PARAVIRT_XXL
68 BLANK(); 68 BLANK();
69 OFFSET(PARAVIRT_PATCH_pv_cpu_ops, paravirt_patch_template, pv_cpu_ops); 69 OFFSET(PV_IRQ_irq_disable, paravirt_patch_template, irq.irq_disable);
70 OFFSET(PARAVIRT_PATCH_pv_irq_ops, paravirt_patch_template, pv_irq_ops); 70 OFFSET(PV_IRQ_irq_enable, paravirt_patch_template, irq.irq_enable);
71 OFFSET(PV_IRQ_irq_disable, pv_irq_ops, irq_disable); 71 OFFSET(PV_CPU_iret, paravirt_patch_template, cpu.iret);
72 OFFSET(PV_IRQ_irq_enable, pv_irq_ops, irq_enable); 72 OFFSET(PV_MMU_read_cr2, paravirt_patch_template, mmu.read_cr2);
73 OFFSET(PV_CPU_iret, pv_cpu_ops, iret);
74 OFFSET(PV_CPU_read_cr0, pv_cpu_ops, read_cr0);
75 OFFSET(PV_MMU_read_cr2, pv_mmu_ops, read_cr2);
76#endif 73#endif
77 74
78#ifdef CONFIG_XEN 75#ifdef CONFIG_XEN
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index 3b9405e7ba2b..ddced33184b5 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -21,10 +21,13 @@ static char syscalls_ia32[] = {
21int main(void) 21int main(void)
22{ 22{
23#ifdef CONFIG_PARAVIRT 23#ifdef CONFIG_PARAVIRT
24 OFFSET(PV_CPU_usergs_sysret64, pv_cpu_ops, usergs_sysret64); 24#ifdef CONFIG_PARAVIRT_XXL
25 OFFSET(PV_CPU_swapgs, pv_cpu_ops, swapgs); 25 OFFSET(PV_CPU_usergs_sysret64, paravirt_patch_template,
26 cpu.usergs_sysret64);
27 OFFSET(PV_CPU_swapgs, paravirt_patch_template, cpu.swapgs);
26#ifdef CONFIG_DEBUG_ENTRY 28#ifdef CONFIG_DEBUG_ENTRY
27 OFFSET(PV_IRQ_save_fl, pv_irq_ops, save_fl); 29 OFFSET(PV_IRQ_save_fl, paravirt_patch_template, irq.save_fl);
30#endif
28#endif 31#endif
29 BLANK(); 32 BLANK();
30#endif 33#endif
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index c519a079b3d5..9315a1660668 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1243,10 +1243,10 @@ static void generic_identify(struct cpuinfo_x86 *c)
1243 * ESPFIX issue, we can change this. 1243 * ESPFIX issue, we can change this.
1244 */ 1244 */
1245#ifdef CONFIG_X86_32 1245#ifdef CONFIG_X86_32
1246# ifdef CONFIG_PARAVIRT 1246# ifdef CONFIG_PARAVIRT_XXL
1247 do { 1247 do {
1248 extern void native_iret(void); 1248 extern void native_iret(void);
1249 if (pv_cpu_ops.iret == native_iret) 1249 if (pv_ops.cpu.iret == native_iret)
1250 set_cpu_bug(c, X86_BUG_ESPFIX); 1250 set_cpu_bug(c, X86_BUG_ESPFIX);
1251 } while (0); 1251 } while (0);
1252# else 1252# else
diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index ad12733f6058..1c72f3819eb1 100644
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -199,6 +199,16 @@ static unsigned long hv_get_tsc_khz(void)
199 return freq / 1000; 199 return freq / 1000;
200} 200}
201 201
202#if defined(CONFIG_SMP) && IS_ENABLED(CONFIG_HYPERV)
203static void __init hv_smp_prepare_boot_cpu(void)
204{
205 native_smp_prepare_boot_cpu();
206#if defined(CONFIG_X86_64) && defined(CONFIG_PARAVIRT_SPINLOCKS)
207 hv_init_spinlocks();
208#endif
209}
210#endif
211
202static void __init ms_hyperv_init_platform(void) 212static void __init ms_hyperv_init_platform(void)
203{ 213{
204 int hv_host_info_eax; 214 int hv_host_info_eax;
@@ -303,6 +313,10 @@ static void __init ms_hyperv_init_platform(void)
303 if (ms_hyperv.misc_features & HV_STIMER_DIRECT_MODE_AVAILABLE) 313 if (ms_hyperv.misc_features & HV_STIMER_DIRECT_MODE_AVAILABLE)
304 alloc_intr_gate(HYPERV_STIMER0_VECTOR, 314 alloc_intr_gate(HYPERV_STIMER0_VECTOR,
305 hv_stimer0_callback_vector); 315 hv_stimer0_callback_vector);
316
317# ifdef CONFIG_SMP
318 smp_ops.smp_prepare_boot_cpu = hv_smp_prepare_boot_cpu;
319# endif
306#endif 320#endif
307} 321}
308 322
diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c
index 8e005329648b..d9ab49bed8af 100644
--- a/arch/x86/kernel/cpu/vmware.c
+++ b/arch/x86/kernel/cpu/vmware.c
@@ -97,14 +97,14 @@ static void __init vmware_sched_clock_setup(void)
97 d->cyc2ns_offset = mul_u64_u32_shr(tsc_now, d->cyc2ns_mul, 97 d->cyc2ns_offset = mul_u64_u32_shr(tsc_now, d->cyc2ns_mul,
98 d->cyc2ns_shift); 98 d->cyc2ns_shift);
99 99
100 pv_time_ops.sched_clock = vmware_sched_clock; 100 pv_ops.time.sched_clock = vmware_sched_clock;
101 pr_info("using sched offset of %llu ns\n", d->cyc2ns_offset); 101 pr_info("using sched offset of %llu ns\n", d->cyc2ns_offset);
102} 102}
103 103
104static void __init vmware_paravirt_ops_setup(void) 104static void __init vmware_paravirt_ops_setup(void)
105{ 105{
106 pv_info.name = "VMware hypervisor"; 106 pv_info.name = "VMware hypervisor";
107 pv_cpu_ops.io_delay = paravirt_nop; 107 pv_ops.cpu.io_delay = paravirt_nop;
108 108
109 if (vmware_tsc_khz && vmw_sched_clock) 109 if (vmware_tsc_khz && vmw_sched_clock)
110 vmware_sched_clock_setup(); 110 vmware_sched_clock_setup();
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index a3618cf04cf6..747c758f67b7 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -26,7 +26,7 @@
26#include <asm/nospec-branch.h> 26#include <asm/nospec-branch.h>
27#include <asm/fixmap.h> 27#include <asm/fixmap.h>
28 28
29#ifdef CONFIG_PARAVIRT 29#ifdef CONFIG_PARAVIRT_XXL
30#include <asm/asm-offsets.h> 30#include <asm/asm-offsets.h>
31#include <asm/paravirt.h> 31#include <asm/paravirt.h>
32#define GET_CR2_INTO(reg) GET_CR2_INTO_RAX ; movq %rax, reg 32#define GET_CR2_INTO(reg) GET_CR2_INTO_RAX ; movq %rax, reg
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index d9b71924c23c..ba4bfb7f6a36 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -283,7 +283,7 @@ static void __init paravirt_ops_setup(void)
283 pv_info.name = "KVM"; 283 pv_info.name = "KVM";
284 284
285 if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY)) 285 if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY))
286 pv_cpu_ops.io_delay = kvm_io_delay; 286 pv_ops.cpu.io_delay = kvm_io_delay;
287 287
288#ifdef CONFIG_X86_IO_APIC 288#ifdef CONFIG_X86_IO_APIC
289 no_timer_check = 1; 289 no_timer_check = 1;
@@ -632,14 +632,14 @@ static void __init kvm_guest_init(void)
632 632
633 if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) { 633 if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
634 has_steal_clock = 1; 634 has_steal_clock = 1;
635 pv_time_ops.steal_clock = kvm_steal_clock; 635 pv_ops.time.steal_clock = kvm_steal_clock;
636 } 636 }
637 637
638 if (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) && 638 if (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
639 !kvm_para_has_hint(KVM_HINTS_REALTIME) && 639 !kvm_para_has_hint(KVM_HINTS_REALTIME) &&
640 kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) { 640 kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
641 pv_mmu_ops.flush_tlb_others = kvm_flush_tlb_others; 641 pv_ops.mmu.flush_tlb_others = kvm_flush_tlb_others;
642 pv_mmu_ops.tlb_remove_table = tlb_remove_table; 642 pv_ops.mmu.tlb_remove_table = tlb_remove_table;
643 } 643 }
644 644
645 if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) 645 if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
@@ -850,13 +850,14 @@ void __init kvm_spinlock_init(void)
850 return; 850 return;
851 851
852 __pv_init_lock_hash(); 852 __pv_init_lock_hash();
853 pv_lock_ops.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; 853 pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath;
854 pv_lock_ops.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); 854 pv_ops.lock.queued_spin_unlock =
855 pv_lock_ops.wait = kvm_wait; 855 PV_CALLEE_SAVE(__pv_queued_spin_unlock);
856 pv_lock_ops.kick = kvm_kick_cpu; 856 pv_ops.lock.wait = kvm_wait;
857 pv_ops.lock.kick = kvm_kick_cpu;
857 858
858 if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) { 859 if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
859 pv_lock_ops.vcpu_is_preempted = 860 pv_ops.lock.vcpu_is_preempted =
860 PV_CALLEE_SAVE(__kvm_vcpu_is_preempted); 861 PV_CALLEE_SAVE(__kvm_vcpu_is_preempted);
861 } 862 }
862} 863}
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index 013fe3d21dbb..30084ecaa20f 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -118,13 +118,13 @@ static u64 kvm_sched_clock_read(void)
118static inline void kvm_sched_clock_init(bool stable) 118static inline void kvm_sched_clock_init(bool stable)
119{ 119{
120 if (!stable) { 120 if (!stable) {
121 pv_time_ops.sched_clock = kvm_clock_read; 121 pv_ops.time.sched_clock = kvm_clock_read;
122 clear_sched_clock_stable(); 122 clear_sched_clock_stable();
123 return; 123 return;
124 } 124 }
125 125
126 kvm_sched_clock_offset = kvm_clock_read(); 126 kvm_sched_clock_offset = kvm_clock_read();
127 pv_time_ops.sched_clock = kvm_sched_clock_read; 127 pv_ops.time.sched_clock = kvm_sched_clock_read;
128 128
129 pr_info("kvm-clock: using sched offset of %llu cycles", 129 pr_info("kvm-clock: using sched offset of %llu cycles",
130 kvm_sched_clock_offset); 130 kvm_sched_clock_offset);
diff --git a/arch/x86/kernel/paravirt-spinlocks.c b/arch/x86/kernel/paravirt-spinlocks.c
index 71f2d1125ec0..4f75d0cf6305 100644
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -17,7 +17,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__native_queued_spin_unlock);
17 17
18bool pv_is_native_spin_unlock(void) 18bool pv_is_native_spin_unlock(void)
19{ 19{
20 return pv_lock_ops.queued_spin_unlock.func == 20 return pv_ops.lock.queued_spin_unlock.func ==
21 __raw_callee_save___native_queued_spin_unlock; 21 __raw_callee_save___native_queued_spin_unlock;
22} 22}
23 23
@@ -29,17 +29,6 @@ PV_CALLEE_SAVE_REGS_THUNK(__native_vcpu_is_preempted);
29 29
30bool pv_is_native_vcpu_is_preempted(void) 30bool pv_is_native_vcpu_is_preempted(void)
31{ 31{
32 return pv_lock_ops.vcpu_is_preempted.func == 32 return pv_ops.lock.vcpu_is_preempted.func ==
33 __raw_callee_save___native_vcpu_is_preempted; 33 __raw_callee_save___native_vcpu_is_preempted;
34} 34}
35
36struct pv_lock_ops pv_lock_ops = {
37#ifdef CONFIG_SMP
38 .queued_spin_lock_slowpath = native_queued_spin_lock_slowpath,
39 .queued_spin_unlock = PV_CALLEE_SAVE(__native_queued_spin_unlock),
40 .wait = paravirt_nop,
41 .kick = paravirt_nop,
42 .vcpu_is_preempted = PV_CALLEE_SAVE(__native_vcpu_is_preempted),
43#endif /* SMP */
44};
45EXPORT_SYMBOL(pv_lock_ops);
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 8dc69d82567e..e4d4df37922a 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -81,10 +81,8 @@ struct branch {
81 u32 delta; 81 u32 delta;
82} __attribute__((packed)); 82} __attribute__((packed));
83 83
84unsigned paravirt_patch_call(void *insnbuf, 84static unsigned paravirt_patch_call(void *insnbuf, const void *target,
85 const void *target, u16 tgt_clobbers, 85 unsigned long addr, unsigned len)
86 unsigned long addr, u16 site_clobbers,
87 unsigned len)
88{ 86{
89 struct branch *b = insnbuf; 87 struct branch *b = insnbuf;
90 unsigned long delta = (unsigned long)target - (addr+5); 88 unsigned long delta = (unsigned long)target - (addr+5);
@@ -103,8 +101,9 @@ unsigned paravirt_patch_call(void *insnbuf,
103 return 5; 101 return 5;
104} 102}
105 103
106unsigned paravirt_patch_jmp(void *insnbuf, const void *target, 104#ifdef CONFIG_PARAVIRT_XXL
107 unsigned long addr, unsigned len) 105static unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
106 unsigned long addr, unsigned len)
108{ 107{
109 struct branch *b = insnbuf; 108 struct branch *b = insnbuf;
110 unsigned long delta = (unsigned long)target - (addr+5); 109 unsigned long delta = (unsigned long)target - (addr+5);
@@ -121,6 +120,7 @@ unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
121 120
122 return 5; 121 return 5;
123} 122}
123#endif
124 124
125DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key); 125DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key);
126 126
@@ -130,29 +130,14 @@ void __init native_pv_lock_init(void)
130 static_branch_disable(&virt_spin_lock_key); 130 static_branch_disable(&virt_spin_lock_key);
131} 131}
132 132
133/* 133unsigned paravirt_patch_default(u8 type, void *insnbuf,
134 * Neat trick to map patch type back to the call within the
135 * corresponding structure.
136 */
137static void *get_call_destination(u8 type)
138{
139 struct paravirt_patch_template tmpl = {
140 .pv_init_ops = pv_init_ops,
141 .pv_time_ops = pv_time_ops,
142 .pv_cpu_ops = pv_cpu_ops,
143 .pv_irq_ops = pv_irq_ops,
144 .pv_mmu_ops = pv_mmu_ops,
145#ifdef CONFIG_PARAVIRT_SPINLOCKS
146 .pv_lock_ops = pv_lock_ops,
147#endif
148 };
149 return *((void **)&tmpl + type);
150}
151
152unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
153 unsigned long addr, unsigned len) 134 unsigned long addr, unsigned len)
154{ 135{
155 void *opfunc = get_call_destination(type); 136 /*
137 * Neat trick to map patch type back to the call within the
138 * corresponding structure.
139 */
140 void *opfunc = *((void **)&pv_ops + type);
156 unsigned ret; 141 unsigned ret;
157 142
158 if (opfunc == NULL) 143 if (opfunc == NULL)
@@ -167,15 +152,15 @@ unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
167 else if (opfunc == _paravirt_ident_64) 152 else if (opfunc == _paravirt_ident_64)
168 ret = paravirt_patch_ident_64(insnbuf, len); 153 ret = paravirt_patch_ident_64(insnbuf, len);
169 154
170 else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) || 155#ifdef CONFIG_PARAVIRT_XXL
171 type == PARAVIRT_PATCH(pv_cpu_ops.usergs_sysret64)) 156 else if (type == PARAVIRT_PATCH(cpu.iret) ||
157 type == PARAVIRT_PATCH(cpu.usergs_sysret64))
172 /* If operation requires a jmp, then jmp */ 158 /* If operation requires a jmp, then jmp */
173 ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len); 159 ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len);
160#endif
174 else 161 else
175 /* Otherwise call the function; assume target could 162 /* Otherwise call the function. */
176 clobber any caller-save reg */ 163 ret = paravirt_patch_call(insnbuf, opfunc, addr, len);
177 ret = paravirt_patch_call(insnbuf, opfunc, CLBR_ANY,
178 addr, clobbers, len);
179 164
180 return ret; 165 return ret;
181} 166}
@@ -281,6 +266,7 @@ void paravirt_flush_lazy_mmu(void)
281 preempt_enable(); 266 preempt_enable();
282} 267}
283 268
269#ifdef CONFIG_PARAVIRT_XXL
284void paravirt_start_context_switch(struct task_struct *prev) 270void paravirt_start_context_switch(struct task_struct *prev)
285{ 271{
286 BUG_ON(preemptible()); 272 BUG_ON(preemptible());
@@ -301,6 +287,7 @@ void paravirt_end_context_switch(struct task_struct *next)
301 if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES)) 287 if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES))
302 arch_enter_lazy_mmu_mode(); 288 arch_enter_lazy_mmu_mode();
303} 289}
290#endif
304 291
305enum paravirt_lazy_mode paravirt_get_lazy_mode(void) 292enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
306{ 293{
@@ -312,85 +299,16 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
312 299
313struct pv_info pv_info = { 300struct pv_info pv_info = {
314 .name = "bare hardware", 301 .name = "bare hardware",
302#ifdef CONFIG_PARAVIRT_XXL
315 .kernel_rpl = 0, 303 .kernel_rpl = 0,
316 .shared_kernel_pmd = 1, /* Only used when CONFIG_X86_PAE is set */ 304 .shared_kernel_pmd = 1, /* Only used when CONFIG_X86_PAE is set */
317 305
318#ifdef CONFIG_X86_64 306#ifdef CONFIG_X86_64
319 .extra_user_64bit_cs = __USER_CS, 307 .extra_user_64bit_cs = __USER_CS,
320#endif 308#endif
321};
322
323struct pv_init_ops pv_init_ops = {
324 .patch = native_patch,
325};
326
327struct pv_time_ops pv_time_ops = {
328 .sched_clock = native_sched_clock,
329 .steal_clock = native_steal_clock,
330};
331
332__visible struct pv_irq_ops pv_irq_ops = {
333 .save_fl = __PV_IS_CALLEE_SAVE(native_save_fl),
334 .restore_fl = __PV_IS_CALLEE_SAVE(native_restore_fl),
335 .irq_disable = __PV_IS_CALLEE_SAVE(native_irq_disable),
336 .irq_enable = __PV_IS_CALLEE_SAVE(native_irq_enable),
337 .safe_halt = native_safe_halt,
338 .halt = native_halt,
339};
340
341__visible struct pv_cpu_ops pv_cpu_ops = {
342 .cpuid = native_cpuid,
343 .get_debugreg = native_get_debugreg,
344 .set_debugreg = native_set_debugreg,
345 .read_cr0 = native_read_cr0,
346 .write_cr0 = native_write_cr0,
347 .write_cr4 = native_write_cr4,
348#ifdef CONFIG_X86_64
349 .read_cr8 = native_read_cr8,
350 .write_cr8 = native_write_cr8,
351#endif 309#endif
352 .wbinvd = native_wbinvd,
353 .read_msr = native_read_msr,
354 .write_msr = native_write_msr,
355 .read_msr_safe = native_read_msr_safe,
356 .write_msr_safe = native_write_msr_safe,
357 .read_pmc = native_read_pmc,
358 .load_tr_desc = native_load_tr_desc,
359 .set_ldt = native_set_ldt,
360 .load_gdt = native_load_gdt,
361 .load_idt = native_load_idt,
362 .store_tr = native_store_tr,
363 .load_tls = native_load_tls,
364#ifdef CONFIG_X86_64
365 .load_gs_index = native_load_gs_index,
366#endif
367 .write_ldt_entry = native_write_ldt_entry,
368 .write_gdt_entry = native_write_gdt_entry,
369 .write_idt_entry = native_write_idt_entry,
370
371 .alloc_ldt = paravirt_nop,
372 .free_ldt = paravirt_nop,
373
374 .load_sp0 = native_load_sp0,
375
376#ifdef CONFIG_X86_64
377 .usergs_sysret64 = native_usergs_sysret64,
378#endif
379 .iret = native_iret,
380 .swapgs = native_swapgs,
381
382 .set_iopl_mask = native_set_iopl_mask,
383 .io_delay = native_io_delay,
384
385 .start_context_switch = paravirt_nop,
386 .end_context_switch = paravirt_nop,
387}; 310};
388 311
389/* At this point, native_get/set_debugreg has real function entries */
390NOKPROBE_SYMBOL(native_get_debugreg);
391NOKPROBE_SYMBOL(native_set_debugreg);
392NOKPROBE_SYMBOL(native_load_idt);
393
394#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE) 312#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
395/* 32-bit pagetable entries */ 313/* 32-bit pagetable entries */
396#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_32) 314#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_32)
@@ -399,85 +317,171 @@ NOKPROBE_SYMBOL(native_load_idt);
399#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64) 317#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
400#endif 318#endif
401 319
402struct pv_mmu_ops pv_mmu_ops __ro_after_init = { 320struct paravirt_patch_template pv_ops = {
321 /* Init ops. */
322 .init.patch = native_patch,
403 323
404 .read_cr2 = native_read_cr2, 324 /* Time ops. */
405 .write_cr2 = native_write_cr2, 325 .time.sched_clock = native_sched_clock,
406 .read_cr3 = __native_read_cr3, 326 .time.steal_clock = native_steal_clock,
407 .write_cr3 = native_write_cr3,
408 327
409 .flush_tlb_user = native_flush_tlb, 328 /* Cpu ops. */
410 .flush_tlb_kernel = native_flush_tlb_global, 329 .cpu.io_delay = native_io_delay,
411 .flush_tlb_one_user = native_flush_tlb_one_user,
412 .flush_tlb_others = native_flush_tlb_others,
413 .tlb_remove_table = (void (*)(struct mmu_gather *, void *))tlb_remove_page,
414 330
415 .pgd_alloc = __paravirt_pgd_alloc, 331#ifdef CONFIG_PARAVIRT_XXL
416 .pgd_free = paravirt_nop, 332 .cpu.cpuid = native_cpuid,
333 .cpu.get_debugreg = native_get_debugreg,
334 .cpu.set_debugreg = native_set_debugreg,
335 .cpu.read_cr0 = native_read_cr0,
336 .cpu.write_cr0 = native_write_cr0,
337 .cpu.write_cr4 = native_write_cr4,
338#ifdef CONFIG_X86_64
339 .cpu.read_cr8 = native_read_cr8,
340 .cpu.write_cr8 = native_write_cr8,
341#endif
342 .cpu.wbinvd = native_wbinvd,
343 .cpu.read_msr = native_read_msr,
344 .cpu.write_msr = native_write_msr,
345 .cpu.read_msr_safe = native_read_msr_safe,
346 .cpu.write_msr_safe = native_write_msr_safe,
347 .cpu.read_pmc = native_read_pmc,
348 .cpu.load_tr_desc = native_load_tr_desc,
349 .cpu.set_ldt = native_set_ldt,
350 .cpu.load_gdt = native_load_gdt,
351 .cpu.load_idt = native_load_idt,
352 .cpu.store_tr = native_store_tr,
353 .cpu.load_tls = native_load_tls,
354#ifdef CONFIG_X86_64
355 .cpu.load_gs_index = native_load_gs_index,
356#endif
357 .cpu.write_ldt_entry = native_write_ldt_entry,
358 .cpu.write_gdt_entry = native_write_gdt_entry,
359 .cpu.write_idt_entry = native_write_idt_entry,
417 360
418 .alloc_pte = paravirt_nop, 361 .cpu.alloc_ldt = paravirt_nop,
419 .alloc_pmd = paravirt_nop, 362 .cpu.free_ldt = paravirt_nop,
420 .alloc_pud = paravirt_nop,
421 .alloc_p4d = paravirt_nop,
422 .release_pte = paravirt_nop,
423 .release_pmd = paravirt_nop,
424 .release_pud = paravirt_nop,
425 .release_p4d = paravirt_nop,
426 363
427 .set_pte = native_set_pte, 364 .cpu.load_sp0 = native_load_sp0,
428 .set_pte_at = native_set_pte_at,
429 .set_pmd = native_set_pmd,
430 365
431 .ptep_modify_prot_start = __ptep_modify_prot_start, 366#ifdef CONFIG_X86_64
432 .ptep_modify_prot_commit = __ptep_modify_prot_commit, 367 .cpu.usergs_sysret64 = native_usergs_sysret64,
368#endif
369 .cpu.iret = native_iret,
370 .cpu.swapgs = native_swapgs,
371
372 .cpu.set_iopl_mask = native_set_iopl_mask,
373
374 .cpu.start_context_switch = paravirt_nop,
375 .cpu.end_context_switch = paravirt_nop,
376
377 /* Irq ops. */
378 .irq.save_fl = __PV_IS_CALLEE_SAVE(native_save_fl),
379 .irq.restore_fl = __PV_IS_CALLEE_SAVE(native_restore_fl),
380 .irq.irq_disable = __PV_IS_CALLEE_SAVE(native_irq_disable),
381 .irq.irq_enable = __PV_IS_CALLEE_SAVE(native_irq_enable),
382 .irq.safe_halt = native_safe_halt,
383 .irq.halt = native_halt,
384#endif /* CONFIG_PARAVIRT_XXL */
385
386 /* Mmu ops. */
387 .mmu.flush_tlb_user = native_flush_tlb,
388 .mmu.flush_tlb_kernel = native_flush_tlb_global,
389 .mmu.flush_tlb_one_user = native_flush_tlb_one_user,
390 .mmu.flush_tlb_others = native_flush_tlb_others,
391 .mmu.tlb_remove_table =
392 (void (*)(struct mmu_gather *, void *))tlb_remove_page,
393
394 .mmu.exit_mmap = paravirt_nop,
395
396#ifdef CONFIG_PARAVIRT_XXL
397 .mmu.read_cr2 = native_read_cr2,
398 .mmu.write_cr2 = native_write_cr2,
399 .mmu.read_cr3 = __native_read_cr3,
400 .mmu.write_cr3 = native_write_cr3,
401
402 .mmu.pgd_alloc = __paravirt_pgd_alloc,
403 .mmu.pgd_free = paravirt_nop,
404
405 .mmu.alloc_pte = paravirt_nop,
406 .mmu.alloc_pmd = paravirt_nop,
407 .mmu.alloc_pud = paravirt_nop,
408 .mmu.alloc_p4d = paravirt_nop,
409 .mmu.release_pte = paravirt_nop,
410 .mmu.release_pmd = paravirt_nop,
411 .mmu.release_pud = paravirt_nop,
412 .mmu.release_p4d = paravirt_nop,
413
414 .mmu.set_pte = native_set_pte,
415 .mmu.set_pte_at = native_set_pte_at,
416 .mmu.set_pmd = native_set_pmd,
417
418 .mmu.ptep_modify_prot_start = __ptep_modify_prot_start,
419 .mmu.ptep_modify_prot_commit = __ptep_modify_prot_commit,
433 420
434#if CONFIG_PGTABLE_LEVELS >= 3 421#if CONFIG_PGTABLE_LEVELS >= 3
435#ifdef CONFIG_X86_PAE 422#ifdef CONFIG_X86_PAE
436 .set_pte_atomic = native_set_pte_atomic, 423 .mmu.set_pte_atomic = native_set_pte_atomic,
437 .pte_clear = native_pte_clear, 424 .mmu.pte_clear = native_pte_clear,
438 .pmd_clear = native_pmd_clear, 425 .mmu.pmd_clear = native_pmd_clear,
439#endif 426#endif
440 .set_pud = native_set_pud, 427 .mmu.set_pud = native_set_pud,
441 428
442 .pmd_val = PTE_IDENT, 429 .mmu.pmd_val = PTE_IDENT,
443 .make_pmd = PTE_IDENT, 430 .mmu.make_pmd = PTE_IDENT,
444 431
445#if CONFIG_PGTABLE_LEVELS >= 4 432#if CONFIG_PGTABLE_LEVELS >= 4
446 .pud_val = PTE_IDENT, 433 .mmu.pud_val = PTE_IDENT,
447 .make_pud = PTE_IDENT, 434 .mmu.make_pud = PTE_IDENT,
448 435
449 .set_p4d = native_set_p4d, 436 .mmu.set_p4d = native_set_p4d,
450 437
451#if CONFIG_PGTABLE_LEVELS >= 5 438#if CONFIG_PGTABLE_LEVELS >= 5
452 .p4d_val = PTE_IDENT, 439 .mmu.p4d_val = PTE_IDENT,
453 .make_p4d = PTE_IDENT, 440 .mmu.make_p4d = PTE_IDENT,
454 441
455 .set_pgd = native_set_pgd, 442 .mmu.set_pgd = native_set_pgd,
456#endif /* CONFIG_PGTABLE_LEVELS >= 5 */ 443#endif /* CONFIG_PGTABLE_LEVELS >= 5 */
457#endif /* CONFIG_PGTABLE_LEVELS >= 4 */ 444#endif /* CONFIG_PGTABLE_LEVELS >= 4 */
458#endif /* CONFIG_PGTABLE_LEVELS >= 3 */ 445#endif /* CONFIG_PGTABLE_LEVELS >= 3 */
459 446
460 .pte_val = PTE_IDENT, 447 .mmu.pte_val = PTE_IDENT,
461 .pgd_val = PTE_IDENT, 448 .mmu.pgd_val = PTE_IDENT,
462 449
463 .make_pte = PTE_IDENT, 450 .mmu.make_pte = PTE_IDENT,
464 .make_pgd = PTE_IDENT, 451 .mmu.make_pgd = PTE_IDENT,
465 452
466 .dup_mmap = paravirt_nop, 453 .mmu.dup_mmap = paravirt_nop,
467 .exit_mmap = paravirt_nop, 454 .mmu.activate_mm = paravirt_nop,
468 .activate_mm = paravirt_nop,
469 455
470 .lazy_mode = { 456 .mmu.lazy_mode = {
471 .enter = paravirt_nop, 457 .enter = paravirt_nop,
472 .leave = paravirt_nop, 458 .leave = paravirt_nop,
473 .flush = paravirt_nop, 459 .flush = paravirt_nop,
474 }, 460 },
475 461
476 .set_fixmap = native_set_fixmap, 462 .mmu.set_fixmap = native_set_fixmap,
463#endif /* CONFIG_PARAVIRT_XXL */
464
465#if defined(CONFIG_PARAVIRT_SPINLOCKS)
466 /* Lock ops. */
467#ifdef CONFIG_SMP
468 .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath,
469 .lock.queued_spin_unlock =
470 PV_CALLEE_SAVE(__native_queued_spin_unlock),
471 .lock.wait = paravirt_nop,
472 .lock.kick = paravirt_nop,
473 .lock.vcpu_is_preempted =
474 PV_CALLEE_SAVE(__native_vcpu_is_preempted),
475#endif /* SMP */
476#endif
477}; 477};
478 478
479EXPORT_SYMBOL_GPL(pv_time_ops); 479#ifdef CONFIG_PARAVIRT_XXL
480EXPORT_SYMBOL (pv_cpu_ops); 480/* At this point, native_get/set_debugreg has real function entries */
481EXPORT_SYMBOL (pv_mmu_ops); 481NOKPROBE_SYMBOL(native_get_debugreg);
482NOKPROBE_SYMBOL(native_set_debugreg);
483NOKPROBE_SYMBOL(native_load_idt);
484#endif
485
486EXPORT_SYMBOL_GPL(pv_ops);
482EXPORT_SYMBOL_GPL(pv_info); 487EXPORT_SYMBOL_GPL(pv_info);
483EXPORT_SYMBOL (pv_irq_ops);
diff --git a/arch/x86/kernel/paravirt_patch_32.c b/arch/x86/kernel/paravirt_patch_32.c
index 758e69d72ebf..6368c22fa1fa 100644
--- a/arch/x86/kernel/paravirt_patch_32.c
+++ b/arch/x86/kernel/paravirt_patch_32.c
@@ -1,18 +1,20 @@
1// SPDX-License-Identifier: GPL-2.0 1// SPDX-License-Identifier: GPL-2.0
2#include <asm/paravirt.h> 2#include <asm/paravirt.h>
3 3
4DEF_NATIVE(pv_irq_ops, irq_disable, "cli"); 4#ifdef CONFIG_PARAVIRT_XXL
5DEF_NATIVE(pv_irq_ops, irq_enable, "sti"); 5DEF_NATIVE(irq, irq_disable, "cli");
6DEF_NATIVE(pv_irq_ops, restore_fl, "push %eax; popf"); 6DEF_NATIVE(irq, irq_enable, "sti");
7DEF_NATIVE(pv_irq_ops, save_fl, "pushf; pop %eax"); 7DEF_NATIVE(irq, restore_fl, "push %eax; popf");
8DEF_NATIVE(pv_cpu_ops, iret, "iret"); 8DEF_NATIVE(irq, save_fl, "pushf; pop %eax");
9DEF_NATIVE(pv_mmu_ops, read_cr2, "mov %cr2, %eax"); 9DEF_NATIVE(cpu, iret, "iret");
10DEF_NATIVE(pv_mmu_ops, write_cr3, "mov %eax, %cr3"); 10DEF_NATIVE(mmu, read_cr2, "mov %cr2, %eax");
11DEF_NATIVE(pv_mmu_ops, read_cr3, "mov %cr3, %eax"); 11DEF_NATIVE(mmu, write_cr3, "mov %eax, %cr3");
12DEF_NATIVE(mmu, read_cr3, "mov %cr3, %eax");
13#endif
12 14
13#if defined(CONFIG_PARAVIRT_SPINLOCKS) 15#if defined(CONFIG_PARAVIRT_SPINLOCKS)
14DEF_NATIVE(pv_lock_ops, queued_spin_unlock, "movb $0, (%eax)"); 16DEF_NATIVE(lock, queued_spin_unlock, "movb $0, (%eax)");
15DEF_NATIVE(pv_lock_ops, vcpu_is_preempted, "xor %eax, %eax"); 17DEF_NATIVE(lock, vcpu_is_preempted, "xor %eax, %eax");
16#endif 18#endif
17 19
18unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len) 20unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len)
@@ -30,53 +32,42 @@ unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len)
30extern bool pv_is_native_spin_unlock(void); 32extern bool pv_is_native_spin_unlock(void);
31extern bool pv_is_native_vcpu_is_preempted(void); 33extern bool pv_is_native_vcpu_is_preempted(void);
32 34
33unsigned native_patch(u8 type, u16 clobbers, void *ibuf, 35unsigned native_patch(u8 type, void *ibuf, unsigned long addr, unsigned len)
34 unsigned long addr, unsigned len)
35{ 36{
36 const unsigned char *start, *end;
37 unsigned ret;
38
39#define PATCH_SITE(ops, x) \ 37#define PATCH_SITE(ops, x) \
40 case PARAVIRT_PATCH(ops.x): \ 38 case PARAVIRT_PATCH(ops.x): \
41 start = start_##ops##_##x; \ 39 return paravirt_patch_insns(ibuf, len, start_##ops##_##x, end_##ops##_##x)
42 end = end_##ops##_##x; \ 40
43 goto patch_site
44 switch (type) { 41 switch (type) {
45 PATCH_SITE(pv_irq_ops, irq_disable); 42#ifdef CONFIG_PARAVIRT_XXL
46 PATCH_SITE(pv_irq_ops, irq_enable); 43 PATCH_SITE(irq, irq_disable);
47 PATCH_SITE(pv_irq_ops, restore_fl); 44 PATCH_SITE(irq, irq_enable);
48 PATCH_SITE(pv_irq_ops, save_fl); 45 PATCH_SITE(irq, restore_fl);
49 PATCH_SITE(pv_cpu_ops, iret); 46 PATCH_SITE(irq, save_fl);
50 PATCH_SITE(pv_mmu_ops, read_cr2); 47 PATCH_SITE(cpu, iret);
51 PATCH_SITE(pv_mmu_ops, read_cr3); 48 PATCH_SITE(mmu, read_cr2);
52 PATCH_SITE(pv_mmu_ops, write_cr3); 49 PATCH_SITE(mmu, read_cr3);
50 PATCH_SITE(mmu, write_cr3);
51#endif
53#if defined(CONFIG_PARAVIRT_SPINLOCKS) 52#if defined(CONFIG_PARAVIRT_SPINLOCKS)
54 case PARAVIRT_PATCH(pv_lock_ops.queued_spin_unlock): 53 case PARAVIRT_PATCH(lock.queued_spin_unlock):
55 if (pv_is_native_spin_unlock()) { 54 if (pv_is_native_spin_unlock())
56 start = start_pv_lock_ops_queued_spin_unlock; 55 return paravirt_patch_insns(ibuf, len,
57 end = end_pv_lock_ops_queued_spin_unlock; 56 start_lock_queued_spin_unlock,
58 goto patch_site; 57 end_lock_queued_spin_unlock);
59 } 58 break;
60 goto patch_default;
61 59
62 case PARAVIRT_PATCH(pv_lock_ops.vcpu_is_preempted): 60 case PARAVIRT_PATCH(lock.vcpu_is_preempted):
63 if (pv_is_native_vcpu_is_preempted()) { 61 if (pv_is_native_vcpu_is_preempted())
64 start = start_pv_lock_ops_vcpu_is_preempted; 62 return paravirt_patch_insns(ibuf, len,
65 end = end_pv_lock_ops_vcpu_is_preempted; 63 start_lock_vcpu_is_preempted,
66 goto patch_site; 64 end_lock_vcpu_is_preempted);
67 } 65 break;
68 goto patch_default;
69#endif 66#endif
70 67
71 default: 68 default:
72patch_default: __maybe_unused
73 ret = paravirt_patch_default(type, clobbers, ibuf, addr, len);
74 break;
75
76patch_site:
77 ret = paravirt_patch_insns(ibuf, len, start, end);
78 break; 69 break;
79 } 70 }
80#undef PATCH_SITE 71#undef PATCH_SITE
81 return ret; 72 return paravirt_patch_default(type, ibuf, addr, len);
82} 73}
diff --git a/arch/x86/kernel/paravirt_patch_64.c b/arch/x86/kernel/paravirt_patch_64.c
index 9cb98f7b07c9..7ca9cb726f4d 100644
--- a/arch/x86/kernel/paravirt_patch_64.c
+++ b/arch/x86/kernel/paravirt_patch_64.c
@@ -3,24 +3,26 @@
3#include <asm/asm-offsets.h> 3#include <asm/asm-offsets.h>
4#include <linux/stringify.h> 4#include <linux/stringify.h>
5 5
6DEF_NATIVE(pv_irq_ops, irq_disable, "cli"); 6#ifdef CONFIG_PARAVIRT_XXL
7DEF_NATIVE(pv_irq_ops, irq_enable, "sti"); 7DEF_NATIVE(irq, irq_disable, "cli");
8DEF_NATIVE(pv_irq_ops, restore_fl, "pushq %rdi; popfq"); 8DEF_NATIVE(irq, irq_enable, "sti");
9DEF_NATIVE(pv_irq_ops, save_fl, "pushfq; popq %rax"); 9DEF_NATIVE(irq, restore_fl, "pushq %rdi; popfq");
10DEF_NATIVE(pv_mmu_ops, read_cr2, "movq %cr2, %rax"); 10DEF_NATIVE(irq, save_fl, "pushfq; popq %rax");
11DEF_NATIVE(pv_mmu_ops, read_cr3, "movq %cr3, %rax"); 11DEF_NATIVE(mmu, read_cr2, "movq %cr2, %rax");
12DEF_NATIVE(pv_mmu_ops, write_cr3, "movq %rdi, %cr3"); 12DEF_NATIVE(mmu, read_cr3, "movq %cr3, %rax");
13DEF_NATIVE(pv_cpu_ops, wbinvd, "wbinvd"); 13DEF_NATIVE(mmu, write_cr3, "movq %rdi, %cr3");
14DEF_NATIVE(cpu, wbinvd, "wbinvd");
14 15
15DEF_NATIVE(pv_cpu_ops, usergs_sysret64, "swapgs; sysretq"); 16DEF_NATIVE(cpu, usergs_sysret64, "swapgs; sysretq");
16DEF_NATIVE(pv_cpu_ops, swapgs, "swapgs"); 17DEF_NATIVE(cpu, swapgs, "swapgs");
18#endif
17 19
18DEF_NATIVE(, mov32, "mov %edi, %eax"); 20DEF_NATIVE(, mov32, "mov %edi, %eax");
19DEF_NATIVE(, mov64, "mov %rdi, %rax"); 21DEF_NATIVE(, mov64, "mov %rdi, %rax");
20 22
21#if defined(CONFIG_PARAVIRT_SPINLOCKS) 23#if defined(CONFIG_PARAVIRT_SPINLOCKS)
22DEF_NATIVE(pv_lock_ops, queued_spin_unlock, "movb $0, (%rdi)"); 24DEF_NATIVE(lock, queued_spin_unlock, "movb $0, (%rdi)");
23DEF_NATIVE(pv_lock_ops, vcpu_is_preempted, "xor %eax, %eax"); 25DEF_NATIVE(lock, vcpu_is_preempted, "xor %eax, %eax");
24#endif 26#endif
25 27
26unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len) 28unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len)
@@ -38,55 +40,44 @@ unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len)
38extern bool pv_is_native_spin_unlock(void); 40extern bool pv_is_native_spin_unlock(void);
39extern bool pv_is_native_vcpu_is_preempted(void); 41extern bool pv_is_native_vcpu_is_preempted(void);
40 42
41unsigned native_patch(u8 type, u16 clobbers, void *ibuf, 43unsigned native_patch(u8 type, void *ibuf, unsigned long addr, unsigned len)
42 unsigned long addr, unsigned len)
43{ 44{
44 const unsigned char *start, *end;
45 unsigned ret;
46
47#define PATCH_SITE(ops, x) \ 45#define PATCH_SITE(ops, x) \
48 case PARAVIRT_PATCH(ops.x): \ 46 case PARAVIRT_PATCH(ops.x): \
49 start = start_##ops##_##x; \ 47 return paravirt_patch_insns(ibuf, len, start_##ops##_##x, end_##ops##_##x)
50 end = end_##ops##_##x; \
51 goto patch_site
52 switch(type) {
53 PATCH_SITE(pv_irq_ops, restore_fl);
54 PATCH_SITE(pv_irq_ops, save_fl);
55 PATCH_SITE(pv_irq_ops, irq_enable);
56 PATCH_SITE(pv_irq_ops, irq_disable);
57 PATCH_SITE(pv_cpu_ops, usergs_sysret64);
58 PATCH_SITE(pv_cpu_ops, swapgs);
59 PATCH_SITE(pv_mmu_ops, read_cr2);
60 PATCH_SITE(pv_mmu_ops, read_cr3);
61 PATCH_SITE(pv_mmu_ops, write_cr3);
62 PATCH_SITE(pv_cpu_ops, wbinvd);
63#if defined(CONFIG_PARAVIRT_SPINLOCKS)
64 case PARAVIRT_PATCH(pv_lock_ops.queued_spin_unlock):
65 if (pv_is_native_spin_unlock()) {
66 start = start_pv_lock_ops_queued_spin_unlock;
67 end = end_pv_lock_ops_queued_spin_unlock;
68 goto patch_site;
69 }
70 goto patch_default;
71 48
72 case PARAVIRT_PATCH(pv_lock_ops.vcpu_is_preempted): 49 switch (type) {
73 if (pv_is_native_vcpu_is_preempted()) { 50#ifdef CONFIG_PARAVIRT_XXL
74 start = start_pv_lock_ops_vcpu_is_preempted; 51 PATCH_SITE(irq, restore_fl);
75 end = end_pv_lock_ops_vcpu_is_preempted; 52 PATCH_SITE(irq, save_fl);
76 goto patch_site; 53 PATCH_SITE(irq, irq_enable);
77 } 54 PATCH_SITE(irq, irq_disable);
78 goto patch_default; 55 PATCH_SITE(cpu, usergs_sysret64);
56 PATCH_SITE(cpu, swapgs);
57 PATCH_SITE(cpu, wbinvd);
58 PATCH_SITE(mmu, read_cr2);
59 PATCH_SITE(mmu, read_cr3);
60 PATCH_SITE(mmu, write_cr3);
79#endif 61#endif
62#if defined(CONFIG_PARAVIRT_SPINLOCKS)
63 case PARAVIRT_PATCH(lock.queued_spin_unlock):
64 if (pv_is_native_spin_unlock())
65 return paravirt_patch_insns(ibuf, len,
66 start_lock_queued_spin_unlock,
67 end_lock_queued_spin_unlock);
68 break;
80 69
81 default: 70 case PARAVIRT_PATCH(lock.vcpu_is_preempted):
82patch_default: __maybe_unused 71 if (pv_is_native_vcpu_is_preempted())
83 ret = paravirt_patch_default(type, clobbers, ibuf, addr, len); 72 return paravirt_patch_insns(ibuf, len,
73 start_lock_vcpu_is_preempted,
74 end_lock_vcpu_is_preempted);
84 break; 75 break;
76#endif
85 77
86patch_site: 78 default:
87 ret = paravirt_patch_insns(ibuf, len, start, end);
88 break; 79 break;
89 } 80 }
90#undef PATCH_SITE 81#undef PATCH_SITE
91 return ret; 82 return paravirt_patch_default(type, ibuf, addr, len);
92} 83}
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 03b7529333a6..e9f777bfed40 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -247,7 +247,7 @@ unsigned long long sched_clock(void)
247 247
248bool using_native_sched_clock(void) 248bool using_native_sched_clock(void)
249{ 249{
250 return pv_time_ops.sched_clock == native_sched_clock; 250 return pv_ops.time.sched_clock == native_sched_clock;
251} 251}
252#else 252#else
253unsigned long long 253unsigned long long
diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index 44685fb2a192..1eae5af491c2 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -26,7 +26,7 @@
26 26
27#define TOPOLOGY_REGISTER_OFFSET 0x10 27#define TOPOLOGY_REGISTER_OFFSET 0x10
28 28
29#if defined CONFIG_PCI && defined CONFIG_PARAVIRT 29#if defined CONFIG_PCI && defined CONFIG_PARAVIRT_XXL
30/* 30/*
31 * Interrupt control on vSMPowered systems: 31 * Interrupt control on vSMPowered systems:
32 * ~AC is a shadow of IF. If IF is 'on' AC should be 'off' 32 * ~AC is a shadow of IF. If IF is 'on' AC should be 'off'
@@ -69,17 +69,17 @@ asmlinkage __visible void vsmp_irq_enable(void)
69} 69}
70PV_CALLEE_SAVE_REGS_THUNK(vsmp_irq_enable); 70PV_CALLEE_SAVE_REGS_THUNK(vsmp_irq_enable);
71 71
72static unsigned __init vsmp_patch(u8 type, u16 clobbers, void *ibuf, 72static unsigned __init vsmp_patch(u8 type, void *ibuf,
73 unsigned long addr, unsigned len) 73 unsigned long addr, unsigned len)
74{ 74{
75 switch (type) { 75 switch (type) {
76 case PARAVIRT_PATCH(pv_irq_ops.irq_enable): 76 case PARAVIRT_PATCH(irq.irq_enable):
77 case PARAVIRT_PATCH(pv_irq_ops.irq_disable): 77 case PARAVIRT_PATCH(irq.irq_disable):
78 case PARAVIRT_PATCH(pv_irq_ops.save_fl): 78 case PARAVIRT_PATCH(irq.save_fl):
79 case PARAVIRT_PATCH(pv_irq_ops.restore_fl): 79 case PARAVIRT_PATCH(irq.restore_fl):
80 return paravirt_patch_default(type, clobbers, ibuf, addr, len); 80 return paravirt_patch_default(type, ibuf, addr, len);
81 default: 81 default:
82 return native_patch(type, clobbers, ibuf, addr, len); 82 return native_patch(type, ibuf, addr, len);
83 } 83 }
84 84
85} 85}
@@ -111,11 +111,11 @@ static void __init set_vsmp_pv_ops(void)
111 111
112 if (cap & ctl & (1 << 4)) { 112 if (cap & ctl & (1 << 4)) {
113 /* Setup irq ops and turn on vSMP IRQ fastpath handling */ 113 /* Setup irq ops and turn on vSMP IRQ fastpath handling */
114 pv_irq_ops.irq_disable = PV_CALLEE_SAVE(vsmp_irq_disable); 114 pv_ops.irq.irq_disable = PV_CALLEE_SAVE(vsmp_irq_disable);
115 pv_irq_ops.irq_enable = PV_CALLEE_SAVE(vsmp_irq_enable); 115 pv_ops.irq.irq_enable = PV_CALLEE_SAVE(vsmp_irq_enable);
116 pv_irq_ops.save_fl = PV_CALLEE_SAVE(vsmp_save_fl); 116 pv_ops.irq.save_fl = PV_CALLEE_SAVE(vsmp_save_fl);
117 pv_irq_ops.restore_fl = PV_CALLEE_SAVE(vsmp_restore_fl); 117 pv_ops.irq.restore_fl = PV_CALLEE_SAVE(vsmp_restore_fl);
118 pv_init_ops.patch = vsmp_patch; 118 pv_ops.init.patch = vsmp_patch;
119 ctl &= ~(1 << 4); 119 ctl &= ~(1 << 4);
120 } 120 }
121 writel(ctl, address + 4); 121 writel(ctl, address + 4);
diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c
index 7ae36868aed2..a19ef1a416ff 100644
--- a/arch/x86/mm/mem_encrypt_identity.c
+++ b/arch/x86/mm/mem_encrypt_identity.c
@@ -27,6 +27,7 @@
27 * be extended when new paravirt and debugging variants are added.) 27 * be extended when new paravirt and debugging variants are added.)
28 */ 28 */
29#undef CONFIG_PARAVIRT 29#undef CONFIG_PARAVIRT
30#undef CONFIG_PARAVIRT_XXL
30#undef CONFIG_PARAVIRT_SPINLOCKS 31#undef CONFIG_PARAVIRT_SPINLOCKS
31 32
32#include <linux/kernel.h> 33#include <linux/kernel.h>
diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig
index 14be6a5935e1..1ef391aa184d 100644
--- a/arch/x86/xen/Kconfig
+++ b/arch/x86/xen/Kconfig
@@ -18,6 +18,7 @@ config XEN_PV
18 bool "Xen PV guest support" 18 bool "Xen PV guest support"
19 default y 19 default y
20 depends on XEN 20 depends on XEN
21 select PARAVIRT_XXL
21 select XEN_HAVE_PVMMU 22 select XEN_HAVE_PVMMU
22 select XEN_HAVE_VPMU 23 select XEN_HAVE_VPMU
23 help 24 help
diff --git a/arch/x86/xen/Makefile b/arch/x86/xen/Makefile
index d83cb5478f54..dd2550d33b38 100644
--- a/arch/x86/xen/Makefile
+++ b/arch/x86/xen/Makefile
@@ -12,25 +12,46 @@ endif
12# Make sure early boot has no stackprotector 12# Make sure early boot has no stackprotector
13nostackp := $(call cc-option, -fno-stack-protector) 13nostackp := $(call cc-option, -fno-stack-protector)
14CFLAGS_enlighten_pv.o := $(nostackp) 14CFLAGS_enlighten_pv.o := $(nostackp)
15CFLAGS_mmu_pv.o := $(nostackp) 15CFLAGS_mmu_pv.o := $(nostackp)
16 16
17obj-y := enlighten.o multicalls.o mmu.o irq.o \ 17obj-y += enlighten.o
18 time.o xen-asm.o xen-asm_$(BITS).o \ 18obj-y += mmu.o
19 grant-table.o suspend.o platform-pci-unplug.o 19obj-y += time.o
20obj-y += grant-table.o
21obj-y += suspend.o
20 22
21obj-$(CONFIG_XEN_PVHVM) += enlighten_hvm.o mmu_hvm.o suspend_hvm.o 23obj-$(CONFIG_XEN_PVHVM) += enlighten_hvm.o
22obj-$(CONFIG_XEN_PV) += setup.o apic.o pmu.o suspend_pv.o \ 24obj-$(CONFIG_XEN_PVHVM) += mmu_hvm.o
23 p2m.o enlighten_pv.o mmu_pv.o 25obj-$(CONFIG_XEN_PVHVM) += suspend_hvm.o
24obj-$(CONFIG_XEN_PVH) += enlighten_pvh.o 26obj-$(CONFIG_XEN_PVHVM) += platform-pci-unplug.o
25 27
26obj-$(CONFIG_EVENT_TRACING) += trace.o 28obj-$(CONFIG_XEN_PV) += setup.o
29obj-$(CONFIG_XEN_PV) += apic.o
30obj-$(CONFIG_XEN_PV) += pmu.o
31obj-$(CONFIG_XEN_PV) += suspend_pv.o
32obj-$(CONFIG_XEN_PV) += p2m.o
33obj-$(CONFIG_XEN_PV) += enlighten_pv.o
34obj-$(CONFIG_XEN_PV) += mmu_pv.o
35obj-$(CONFIG_XEN_PV) += irq.o
36obj-$(CONFIG_XEN_PV) += multicalls.o
37obj-$(CONFIG_XEN_PV) += xen-asm.o
38obj-$(CONFIG_XEN_PV) += xen-asm_$(BITS).o
39
40obj-$(CONFIG_XEN_PVH) += enlighten_pvh.o
41obj-$(CONFIG_XEN_PVH) += xen-pvh.o
42
43obj-$(CONFIG_EVENT_TRACING) += trace.o
27 44
28obj-$(CONFIG_SMP) += smp.o 45obj-$(CONFIG_SMP) += smp.o
29obj-$(CONFIG_XEN_PV_SMP) += smp_pv.o 46obj-$(CONFIG_XEN_PV_SMP) += smp_pv.o
30obj-$(CONFIG_XEN_PVHVM_SMP) += smp_hvm.o 47obj-$(CONFIG_XEN_PVHVM_SMP) += smp_hvm.o
48
31obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= spinlock.o 49obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= spinlock.o
50
32obj-$(CONFIG_XEN_DEBUG_FS) += debugfs.o 51obj-$(CONFIG_XEN_DEBUG_FS) += debugfs.o
52
33obj-$(CONFIG_XEN_DOM0) += vga.o 53obj-$(CONFIG_XEN_DOM0) += vga.o
54
34obj-$(CONFIG_SWIOTLB_XEN) += pci-swiotlb-xen.o 55obj-$(CONFIG_SWIOTLB_XEN) += pci-swiotlb-xen.o
56
35obj-$(CONFIG_XEN_EFI) += efi.o 57obj-$(CONFIG_XEN_EFI) += efi.o
36obj-$(CONFIG_XEN_PVH) += xen-pvh.o
diff --git a/arch/x86/xen/efi.c b/arch/x86/xen/efi.c
index 1804b27f9632..1fbb629a9d78 100644
--- a/arch/x86/xen/efi.c
+++ b/arch/x86/xen/efi.c
@@ -1,18 +1,6 @@
1// SPDX-License-Identifier: GPL-2.0
1/* 2/*
2 * Copyright (c) 2014 Oracle Co., Daniel Kiper 3 * Copyright (c) 2014 Oracle Co., Daniel Kiper
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License along
15 * with this program. If not, see <http://www.gnu.org/licenses/>.
16 */ 4 */
17 5
18#include <linux/bitops.h> 6#include <linux/bitops.h>
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 0ca46e03b830..67b2f31a1265 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -1,3 +1,5 @@
1// SPDX-License-Identifier: GPL-2.0
2
1#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG 3#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
2#include <linux/bootmem.h> 4#include <linux/bootmem.h>
3#endif 5#endif
diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c
index 19c1ff542387..0e75642d42a3 100644
--- a/arch/x86/xen/enlighten_hvm.c
+++ b/arch/x86/xen/enlighten_hvm.c
@@ -1,3 +1,5 @@
1// SPDX-License-Identifier: GPL-2.0
2
1#include <linux/acpi.h> 3#include <linux/acpi.h>
2#include <linux/cpu.h> 4#include <linux/cpu.h>
3#include <linux/kexec.h> 5#include <linux/kexec.h>
diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index 52a7c3faee0c..ec7a4209f310 100644
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -995,11 +995,14 @@ void __init xen_setup_vcpu_info_placement(void)
995 * percpu area for all cpus, so make use of it. 995 * percpu area for all cpus, so make use of it.
996 */ 996 */
997 if (xen_have_vcpu_info_placement) { 997 if (xen_have_vcpu_info_placement) {
998 pv_irq_ops.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct); 998 pv_ops.irq.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct);
999 pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(xen_restore_fl_direct); 999 pv_ops.irq.restore_fl =
1000 pv_irq_ops.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct); 1000 __PV_IS_CALLEE_SAVE(xen_restore_fl_direct);
1001 pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct); 1001 pv_ops.irq.irq_disable =
1002 pv_mmu_ops.read_cr2 = xen_read_cr2_direct; 1002 __PV_IS_CALLEE_SAVE(xen_irq_disable_direct);
1003 pv_ops.irq.irq_enable =
1004 __PV_IS_CALLEE_SAVE(xen_irq_enable_direct);
1005 pv_ops.mmu.read_cr2 = xen_read_cr2_direct;
1003 } 1006 }
1004} 1007}
1005 1008
@@ -1174,14 +1177,14 @@ static void __init xen_boot_params_init_edd(void)
1174 */ 1177 */
1175static void __init xen_setup_gdt(int cpu) 1178static void __init xen_setup_gdt(int cpu)
1176{ 1179{
1177 pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot; 1180 pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry_boot;
1178 pv_cpu_ops.load_gdt = xen_load_gdt_boot; 1181 pv_ops.cpu.load_gdt = xen_load_gdt_boot;
1179 1182
1180 setup_stack_canary_segment(cpu); 1183 setup_stack_canary_segment(cpu);
1181 switch_to_new_gdt(cpu); 1184 switch_to_new_gdt(cpu);
1182 1185
1183 pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry; 1186 pv_ops.cpu.write_gdt_entry = xen_write_gdt_entry;
1184 pv_cpu_ops.load_gdt = xen_load_gdt; 1187 pv_ops.cpu.load_gdt = xen_load_gdt;
1185} 1188}
1186 1189
1187static void __init xen_dom0_set_legacy_features(void) 1190static void __init xen_dom0_set_legacy_features(void)
@@ -1206,8 +1209,8 @@ asmlinkage __visible void __init xen_start_kernel(void)
1206 1209
1207 /* Install Xen paravirt ops */ 1210 /* Install Xen paravirt ops */
1208 pv_info = xen_info; 1211 pv_info = xen_info;
1209 pv_init_ops.patch = paravirt_patch_default; 1212 pv_ops.init.patch = paravirt_patch_default;
1210 pv_cpu_ops = xen_cpu_ops; 1213 pv_ops.cpu = xen_cpu_ops;
1211 xen_init_irq_ops(); 1214 xen_init_irq_ops();
1212 1215
1213 /* 1216 /*
@@ -1276,8 +1279,10 @@ asmlinkage __visible void __init xen_start_kernel(void)
1276#endif 1279#endif
1277 1280
1278 if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) { 1281 if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
1279 pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start; 1282 pv_ops.mmu.ptep_modify_prot_start =
1280 pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit; 1283 xen_ptep_modify_prot_start;
1284 pv_ops.mmu.ptep_modify_prot_commit =
1285 xen_ptep_modify_prot_commit;
1281 } 1286 }
1282 1287
1283 machine_ops = xen_machine_ops; 1288 machine_ops = xen_machine_ops;
diff --git a/arch/x86/xen/grant-table.c b/arch/x86/xen/grant-table.c
index 92ccc718152d..ecb0d5450334 100644
--- a/arch/x86/xen/grant-table.c
+++ b/arch/x86/xen/grant-table.c
@@ -1,3 +1,4 @@
1// SPDX-License-Identifier: GPL-2.0 OR MIT
1/****************************************************************************** 2/******************************************************************************
2 * grant_table.c 3 * grant_table.c
3 * x86 specific part 4 * x86 specific part
@@ -8,30 +9,6 @@
8 * Copyright (c) 2004-2005, K A Fraser 9 * Copyright (c) 2004-2005, K A Fraser
9 * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp> 10 * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
10 * VA Linux Systems Japan. Split out x86 specific part. 11 * VA Linux Systems Japan. Split out x86 specific part.
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License version 2
14 * as published by the Free Software Foundation; or, when distributed
15 * separately from the Linux kernel or incorporated into other
16 * software packages, subject to the following license:
17 *
18 * Permission is hereby granted, free of charge, to any person obtaining a copy
19 * of this source file (the "Software"), to deal in the Software without
20 * restriction, including without limitation the rights to use, copy, modify,
21 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
22 * and to permit persons to whom the Software is furnished to do so, subject to
23 * the following conditions:
24 *
25 * The above copyright notice and this permission notice shall be included in
26 * all copies or substantial portions of the Software.
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
29 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
30 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
31 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
32 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
33 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
34 * IN THE SOFTWARE.
35 */ 12 */
36 13
37#include <linux/sched.h> 14#include <linux/sched.h>
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index 7515a19fd324..850c93f346c7 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -128,6 +128,6 @@ static const struct pv_irq_ops xen_irq_ops __initconst = {
128 128
129void __init xen_init_irq_ops(void) 129void __init xen_init_irq_ops(void)
130{ 130{
131 pv_irq_ops = xen_irq_ops; 131 pv_ops.irq = xen_irq_ops;
132 x86_init.irqs.intr_init = xen_init_IRQ; 132 x86_init.irqs.intr_init = xen_init_IRQ;
133} 133}
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 96fc2f0fdbfe..60e9c37fd79f 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1,3 +1,5 @@
1// SPDX-License-Identifier: GPL-2.0
2
1#include <linux/pfn.h> 3#include <linux/pfn.h>
2#include <asm/xen/page.h> 4#include <asm/xen/page.h>
3#include <asm/xen/hypercall.h> 5#include <asm/xen/hypercall.h>
@@ -6,12 +8,6 @@
6#include "multicalls.h" 8#include "multicalls.h"
7#include "mmu.h" 9#include "mmu.h"
8 10
9/*
10 * Protects atomic reservation decrease/increase against concurrent increases.
11 * Also protects non-atomic updates of current_pages and balloon lists.
12 */
13DEFINE_SPINLOCK(xen_reservation_lock);
14
15unsigned long arbitrary_virt_to_mfn(void *vaddr) 11unsigned long arbitrary_virt_to_mfn(void *vaddr)
16{ 12{
17 xmaddr_t maddr = arbitrary_virt_to_machine(vaddr); 13 xmaddr_t maddr = arbitrary_virt_to_machine(vaddr);
@@ -42,186 +38,6 @@ xmaddr_t arbitrary_virt_to_machine(void *vaddr)
42} 38}
43EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine); 39EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine);
44 40
45static noinline void xen_flush_tlb_all(void)
46{
47 struct mmuext_op *op;
48 struct multicall_space mcs;
49
50 preempt_disable();
51
52 mcs = xen_mc_entry(sizeof(*op));
53
54 op = mcs.args;
55 op->cmd = MMUEXT_TLB_FLUSH_ALL;
56 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
57
58 xen_mc_issue(PARAVIRT_LAZY_MMU);
59
60 preempt_enable();
61}
62
63#define REMAP_BATCH_SIZE 16
64
65struct remap_data {
66 xen_pfn_t *pfn;
67 bool contiguous;
68 bool no_translate;
69 pgprot_t prot;
70 struct mmu_update *mmu_update;
71};
72
73static int remap_area_pfn_pte_fn(pte_t *ptep, pgtable_t token,
74 unsigned long addr, void *data)
75{
76 struct remap_data *rmd = data;
77 pte_t pte = pte_mkspecial(mfn_pte(*rmd->pfn, rmd->prot));
78
79 /*
80 * If we have a contiguous range, just update the pfn itself,
81 * else update pointer to be "next pfn".
82 */
83 if (rmd->contiguous)
84 (*rmd->pfn)++;
85 else
86 rmd->pfn++;
87
88 rmd->mmu_update->ptr = virt_to_machine(ptep).maddr;
89 rmd->mmu_update->ptr |= rmd->no_translate ?
90 MMU_PT_UPDATE_NO_TRANSLATE :
91 MMU_NORMAL_PT_UPDATE;
92 rmd->mmu_update->val = pte_val_ma(pte);
93 rmd->mmu_update++;
94
95 return 0;
96}
97
98static int do_remap_pfn(struct vm_area_struct *vma,
99 unsigned long addr,
100 xen_pfn_t *pfn, int nr,
101 int *err_ptr, pgprot_t prot,
102 unsigned int domid,
103 bool no_translate,
104 struct page **pages)
105{
106 int err = 0;
107 struct remap_data rmd;
108 struct mmu_update mmu_update[REMAP_BATCH_SIZE];
109 unsigned long range;
110 int mapped = 0;
111
112 BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO)));
113
114 rmd.pfn = pfn;
115 rmd.prot = prot;
116 /*
117 * We use the err_ptr to indicate if there we are doing a contiguous
118 * mapping or a discontigious mapping.
119 */
120 rmd.contiguous = !err_ptr;
121 rmd.no_translate = no_translate;
122
123 while (nr) {
124 int index = 0;
125 int done = 0;
126 int batch = min(REMAP_BATCH_SIZE, nr);
127 int batch_left = batch;
128 range = (unsigned long)batch << PAGE_SHIFT;
129
130 rmd.mmu_update = mmu_update;
131 err = apply_to_page_range(vma->vm_mm, addr, range,
132 remap_area_pfn_pte_fn, &rmd);
133 if (err)
134 goto out;
135
136 /* We record the error for each page that gives an error, but
137 * continue mapping until the whole set is done */
138 do {
139 int i;
140
141 err = HYPERVISOR_mmu_update(&mmu_update[index],
142 batch_left, &done, domid);
143
144 /*
145 * @err_ptr may be the same buffer as @gfn, so
146 * only clear it after each chunk of @gfn is
147 * used.
148 */
149 if (err_ptr) {
150 for (i = index; i < index + done; i++)
151 err_ptr[i] = 0;
152 }
153 if (err < 0) {
154 if (!err_ptr)
155 goto out;
156 err_ptr[i] = err;
157 done++; /* Skip failed frame. */
158 } else
159 mapped += done;
160 batch_left -= done;
161 index += done;
162 } while (batch_left);
163
164 nr -= batch;
165 addr += range;
166 if (err_ptr)
167 err_ptr += batch;
168 cond_resched();
169 }
170out:
171
172 xen_flush_tlb_all();
173
174 return err < 0 ? err : mapped;
175}
176
177int xen_remap_domain_gfn_range(struct vm_area_struct *vma,
178 unsigned long addr,
179 xen_pfn_t gfn, int nr,
180 pgprot_t prot, unsigned domid,
181 struct page **pages)
182{
183 if (xen_feature(XENFEAT_auto_translated_physmap))
184 return -EOPNOTSUPP;
185
186 return do_remap_pfn(vma, addr, &gfn, nr, NULL, prot, domid, false,
187 pages);
188}
189EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_range);
190
191int xen_remap_domain_gfn_array(struct vm_area_struct *vma,
192 unsigned long addr,
193 xen_pfn_t *gfn, int nr,
194 int *err_ptr, pgprot_t prot,
195 unsigned domid, struct page **pages)
196{
197 if (xen_feature(XENFEAT_auto_translated_physmap))
198 return xen_xlate_remap_gfn_array(vma, addr, gfn, nr, err_ptr,
199 prot, domid, pages);
200
201 /* We BUG_ON because it's a programmer error to pass a NULL err_ptr,
202 * and the consequences later is quite hard to detect what the actual
203 * cause of "wrong memory was mapped in".
204 */
205 BUG_ON(err_ptr == NULL);
206 return do_remap_pfn(vma, addr, gfn, nr, err_ptr, prot, domid,
207 false, pages);
208}
209EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_array);
210
211int xen_remap_domain_mfn_array(struct vm_area_struct *vma,
212 unsigned long addr,
213 xen_pfn_t *mfn, int nr,
214 int *err_ptr, pgprot_t prot,
215 unsigned int domid, struct page **pages)
216{
217 if (xen_feature(XENFEAT_auto_translated_physmap))
218 return -EOPNOTSUPP;
219
220 return do_remap_pfn(vma, addr, mfn, nr, err_ptr, prot, domid,
221 true, pages);
222}
223EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_array);
224
225/* Returns: 0 success */ 41/* Returns: 0 success */
226int xen_unmap_domain_gfn_range(struct vm_area_struct *vma, 42int xen_unmap_domain_gfn_range(struct vm_area_struct *vma,
227 int nr, struct page **pages) 43 int nr, struct page **pages)
diff --git a/arch/x86/xen/mmu_hvm.c b/arch/x86/xen/mmu_hvm.c
index dd2ad82eee80..57409373750f 100644
--- a/arch/x86/xen/mmu_hvm.c
+++ b/arch/x86/xen/mmu_hvm.c
@@ -73,7 +73,7 @@ static int is_pagetable_dying_supported(void)
73void __init xen_hvm_init_mmu_ops(void) 73void __init xen_hvm_init_mmu_ops(void)
74{ 74{
75 if (is_pagetable_dying_supported()) 75 if (is_pagetable_dying_supported())
76 pv_mmu_ops.exit_mmap = xen_hvm_exit_mmap; 76 pv_ops.mmu.exit_mmap = xen_hvm_exit_mmap;
77#ifdef CONFIG_PROC_VMCORE 77#ifdef CONFIG_PROC_VMCORE
78 WARN_ON(register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram)); 78 WARN_ON(register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram));
79#endif 79#endif
diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c
index dd461c0167ef..70ea598a37d2 100644
--- a/arch/x86/xen/mmu_pv.c
+++ b/arch/x86/xen/mmu_pv.c
@@ -1,3 +1,5 @@
1// SPDX-License-Identifier: GPL-2.0
2
1/* 3/*
2 * Xen mmu operations 4 * Xen mmu operations
3 * 5 *
@@ -99,6 +101,12 @@ static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss;
99#endif /* CONFIG_X86_64 */ 101#endif /* CONFIG_X86_64 */
100 102
101/* 103/*
104 * Protects atomic reservation decrease/increase against concurrent increases.
105 * Also protects non-atomic updates of current_pages and balloon lists.
106 */
107static DEFINE_SPINLOCK(xen_reservation_lock);
108
109/*
102 * Note about cr3 (pagetable base) values: 110 * Note about cr3 (pagetable base) values:
103 * 111 *
104 * xen_cr3 contains the current logical cr3 value; it contains the 112 * xen_cr3 contains the current logical cr3 value; it contains the
@@ -2209,7 +2217,7 @@ static void __init xen_write_cr3_init(unsigned long cr3)
2209 set_page_prot(initial_page_table, PAGE_KERNEL); 2217 set_page_prot(initial_page_table, PAGE_KERNEL);
2210 set_page_prot(initial_kernel_pmd, PAGE_KERNEL); 2218 set_page_prot(initial_kernel_pmd, PAGE_KERNEL);
2211 2219
2212 pv_mmu_ops.write_cr3 = &xen_write_cr3; 2220 pv_ops.mmu.write_cr3 = &xen_write_cr3;
2213} 2221}
2214 2222
2215/* 2223/*
@@ -2358,27 +2366,27 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
2358 2366
2359static void __init xen_post_allocator_init(void) 2367static void __init xen_post_allocator_init(void)
2360{ 2368{
2361 pv_mmu_ops.set_pte = xen_set_pte; 2369 pv_ops.mmu.set_pte = xen_set_pte;
2362 pv_mmu_ops.set_pmd = xen_set_pmd; 2370 pv_ops.mmu.set_pmd = xen_set_pmd;
2363 pv_mmu_ops.set_pud = xen_set_pud; 2371 pv_ops.mmu.set_pud = xen_set_pud;
2364#ifdef CONFIG_X86_64 2372#ifdef CONFIG_X86_64
2365 pv_mmu_ops.set_p4d = xen_set_p4d; 2373 pv_ops.mmu.set_p4d = xen_set_p4d;
2366#endif 2374#endif
2367 2375
2368 /* This will work as long as patching hasn't happened yet 2376 /* This will work as long as patching hasn't happened yet
2369 (which it hasn't) */ 2377 (which it hasn't) */
2370 pv_mmu_ops.alloc_pte = xen_alloc_pte; 2378 pv_ops.mmu.alloc_pte = xen_alloc_pte;
2371 pv_mmu_ops.alloc_pmd = xen_alloc_pmd; 2379 pv_ops.mmu.alloc_pmd = xen_alloc_pmd;
2372 pv_mmu_ops.release_pte = xen_release_pte; 2380 pv_ops.mmu.release_pte = xen_release_pte;
2373 pv_mmu_ops.release_pmd = xen_release_pmd; 2381 pv_ops.mmu.release_pmd = xen_release_pmd;
2374#ifdef CONFIG_X86_64 2382#ifdef CONFIG_X86_64
2375 pv_mmu_ops.alloc_pud = xen_alloc_pud; 2383 pv_ops.mmu.alloc_pud = xen_alloc_pud;
2376 pv_mmu_ops.release_pud = xen_release_pud; 2384 pv_ops.mmu.release_pud = xen_release_pud;
2377#endif 2385#endif
2378 pv_mmu_ops.make_pte = PV_CALLEE_SAVE(xen_make_pte); 2386 pv_ops.mmu.make_pte = PV_CALLEE_SAVE(xen_make_pte);
2379 2387
2380#ifdef CONFIG_X86_64 2388#ifdef CONFIG_X86_64
2381 pv_mmu_ops.write_cr3 = &xen_write_cr3; 2389 pv_ops.mmu.write_cr3 = &xen_write_cr3;
2382#endif 2390#endif
2383} 2391}
2384 2392
@@ -2466,7 +2474,7 @@ void __init xen_init_mmu_ops(void)
2466 x86_init.paging.pagetable_init = xen_pagetable_init; 2474 x86_init.paging.pagetable_init = xen_pagetable_init;
2467 x86_init.hyper.init_after_bootmem = xen_after_bootmem; 2475 x86_init.hyper.init_after_bootmem = xen_after_bootmem;
2468 2476
2469 pv_mmu_ops = xen_mmu_ops; 2477 pv_ops.mmu = xen_mmu_ops;
2470 2478
2471 memset(dummy_mapping, 0xff, PAGE_SIZE); 2479 memset(dummy_mapping, 0xff, PAGE_SIZE);
2472} 2480}
@@ -2666,6 +2674,138 @@ void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order)
2666} 2674}
2667EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region); 2675EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region);
2668 2676
2677static noinline void xen_flush_tlb_all(void)
2678{
2679 struct mmuext_op *op;
2680 struct multicall_space mcs;
2681
2682 preempt_disable();
2683
2684 mcs = xen_mc_entry(sizeof(*op));
2685
2686 op = mcs.args;
2687 op->cmd = MMUEXT_TLB_FLUSH_ALL;
2688 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
2689
2690 xen_mc_issue(PARAVIRT_LAZY_MMU);
2691
2692 preempt_enable();
2693}
2694
2695#define REMAP_BATCH_SIZE 16
2696
2697struct remap_data {
2698 xen_pfn_t *pfn;
2699 bool contiguous;
2700 bool no_translate;
2701 pgprot_t prot;
2702 struct mmu_update *mmu_update;
2703};
2704
2705static int remap_area_pfn_pte_fn(pte_t *ptep, pgtable_t token,
2706 unsigned long addr, void *data)
2707{
2708 struct remap_data *rmd = data;
2709 pte_t pte = pte_mkspecial(mfn_pte(*rmd->pfn, rmd->prot));
2710
2711 /*
2712 * If we have a contiguous range, just update the pfn itself,
2713 * else update pointer to be "next pfn".
2714 */
2715 if (rmd->contiguous)
2716 (*rmd->pfn)++;
2717 else
2718 rmd->pfn++;
2719
2720 rmd->mmu_update->ptr = virt_to_machine(ptep).maddr;
2721 rmd->mmu_update->ptr |= rmd->no_translate ?
2722 MMU_PT_UPDATE_NO_TRANSLATE :
2723 MMU_NORMAL_PT_UPDATE;
2724 rmd->mmu_update->val = pte_val_ma(pte);
2725 rmd->mmu_update++;
2726
2727 return 0;
2728}
2729
2730int xen_remap_pfn(struct vm_area_struct *vma, unsigned long addr,
2731 xen_pfn_t *pfn, int nr, int *err_ptr, pgprot_t prot,
2732 unsigned int domid, bool no_translate, struct page **pages)
2733{
2734 int err = 0;
2735 struct remap_data rmd;
2736 struct mmu_update mmu_update[REMAP_BATCH_SIZE];
2737 unsigned long range;
2738 int mapped = 0;
2739
2740 BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO)));
2741
2742 rmd.pfn = pfn;
2743 rmd.prot = prot;
2744 /*
2745 * We use the err_ptr to indicate if there we are doing a contiguous
2746 * mapping or a discontigious mapping.
2747 */
2748 rmd.contiguous = !err_ptr;
2749 rmd.no_translate = no_translate;
2750
2751 while (nr) {
2752 int index = 0;
2753 int done = 0;
2754 int batch = min(REMAP_BATCH_SIZE, nr);
2755 int batch_left = batch;
2756
2757 range = (unsigned long)batch << PAGE_SHIFT;
2758
2759 rmd.mmu_update = mmu_update;
2760 err = apply_to_page_range(vma->vm_mm, addr, range,
2761 remap_area_pfn_pte_fn, &rmd);
2762 if (err)
2763 goto out;
2764
2765 /*
2766 * We record the error for each page that gives an error, but
2767 * continue mapping until the whole set is done
2768 */
2769 do {
2770 int i;
2771
2772 err = HYPERVISOR_mmu_update(&mmu_update[index],
2773 batch_left, &done, domid);
2774
2775 /*
2776 * @err_ptr may be the same buffer as @gfn, so
2777 * only clear it after each chunk of @gfn is
2778 * used.
2779 */
2780 if (err_ptr) {
2781 for (i = index; i < index + done; i++)
2782 err_ptr[i] = 0;
2783 }
2784 if (err < 0) {
2785 if (!err_ptr)
2786 goto out;
2787 err_ptr[i] = err;
2788 done++; /* Skip failed frame. */
2789 } else
2790 mapped += done;
2791 batch_left -= done;
2792 index += done;
2793 } while (batch_left);
2794
2795 nr -= batch;
2796 addr += range;
2797 if (err_ptr)
2798 err_ptr += batch;
2799 cond_resched();
2800 }
2801out:
2802
2803 xen_flush_tlb_all();
2804
2805 return err < 0 ? err : mapped;
2806}
2807EXPORT_SYMBOL_GPL(xen_remap_pfn);
2808
2669#ifdef CONFIG_KEXEC_CORE 2809#ifdef CONFIG_KEXEC_CORE
2670phys_addr_t paddr_vmcoreinfo_note(void) 2810phys_addr_t paddr_vmcoreinfo_note(void)
2671{ 2811{
diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c
index 159a897151d6..d6d74efd8912 100644
--- a/arch/x86/xen/p2m.c
+++ b/arch/x86/xen/p2m.c
@@ -1,3 +1,5 @@
1// SPDX-License-Identifier: GPL-2.0
2
1/* 3/*
2 * Xen leaves the responsibility for maintaining p2m mappings to the 4 * Xen leaves the responsibility for maintaining p2m mappings to the
3 * guests themselves, but it must also access and update the p2m array 5 * guests themselves, but it must also access and update the p2m array
diff --git a/arch/x86/xen/pci-swiotlb-xen.c b/arch/x86/xen/pci-swiotlb-xen.c
index 37c6056a7bba..33293ce01d8d 100644
--- a/arch/x86/xen/pci-swiotlb-xen.c
+++ b/arch/x86/xen/pci-swiotlb-xen.c
@@ -1,3 +1,5 @@
1// SPDX-License-Identifier: GPL-2.0
2
1/* Glue code to lib/swiotlb-xen.c */ 3/* Glue code to lib/swiotlb-xen.c */
2 4
3#include <linux/dma-mapping.h> 5#include <linux/dma-mapping.h>
diff --git a/arch/x86/xen/platform-pci-unplug.c b/arch/x86/xen/platform-pci-unplug.c
index b99585034dd2..66ab96a4e2b3 100644
--- a/arch/x86/xen/platform-pci-unplug.c
+++ b/arch/x86/xen/platform-pci-unplug.c
@@ -1,22 +1,10 @@
1// SPDX-License-Identifier: GPL-2.0
2
1/****************************************************************************** 3/******************************************************************************
2 * platform-pci-unplug.c 4 * platform-pci-unplug.c
3 * 5 *
4 * Xen platform PCI device driver 6 * Xen platform PCI device driver
5 * Copyright (c) 2010, Citrix 7 * Copyright (c) 2010, Citrix
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
18 * Place - Suite 330, Boston, MA 02111-1307 USA.
19 *
20 */ 8 */
21 9
22#include <linux/init.h> 10#include <linux/init.h>
@@ -31,7 +19,6 @@
31#define XEN_PLATFORM_ERR_PROTOCOL -2 19#define XEN_PLATFORM_ERR_PROTOCOL -2
32#define XEN_PLATFORM_ERR_BLACKLIST -3 20#define XEN_PLATFORM_ERR_BLACKLIST -3
33 21
34#ifdef CONFIG_XEN_PVHVM
35/* store the value of xen_emul_unplug after the unplug is done */ 22/* store the value of xen_emul_unplug after the unplug is done */
36static int xen_platform_pci_unplug; 23static int xen_platform_pci_unplug;
37static int xen_emul_unplug; 24static int xen_emul_unplug;
@@ -215,4 +202,3 @@ static int __init parse_xen_emul_unplug(char *arg)
215 return 0; 202 return 0;
216} 203}
217early_param("xen_emul_unplug", parse_xen_emul_unplug); 204early_param("xen_emul_unplug", parse_xen_emul_unplug);
218#endif
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index 973f10e05211..23f6793af88a 100644
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -141,11 +141,12 @@ void __init xen_init_spinlocks(void)
141 printk(KERN_DEBUG "xen: PV spinlocks enabled\n"); 141 printk(KERN_DEBUG "xen: PV spinlocks enabled\n");
142 142
143 __pv_init_lock_hash(); 143 __pv_init_lock_hash();
144 pv_lock_ops.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath; 144 pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath;
145 pv_lock_ops.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock); 145 pv_ops.lock.queued_spin_unlock =
146 pv_lock_ops.wait = xen_qlock_wait; 146 PV_CALLEE_SAVE(__pv_queued_spin_unlock);
147 pv_lock_ops.kick = xen_qlock_kick; 147 pv_ops.lock.wait = xen_qlock_wait;
148 pv_lock_ops.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen); 148 pv_ops.lock.kick = xen_qlock_kick;
149 pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen);
149} 150}
150 151
151static __init int xen_parse_nopvspin(char *arg) 152static __init int xen_parse_nopvspin(char *arg)
diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c
index c84f1e039d84..72bf446c3fee 100644
--- a/arch/x86/xen/time.c
+++ b/arch/x86/xen/time.c
@@ -513,7 +513,7 @@ static void __init xen_time_init(void)
513void __init xen_init_time_ops(void) 513void __init xen_init_time_ops(void)
514{ 514{
515 xen_sched_clock_offset = xen_clocksource_read(); 515 xen_sched_clock_offset = xen_clocksource_read();
516 pv_time_ops = xen_time_ops; 516 pv_ops.time = xen_time_ops;
517 517
518 x86_init.timers.timer_init = xen_time_init; 518 x86_init.timers.timer_init = xen_time_init;
519 x86_init.timers.setup_percpu_clockev = x86_init_noop; 519 x86_init.timers.setup_percpu_clockev = x86_init_noop;
@@ -555,7 +555,7 @@ void __init xen_hvm_init_time_ops(void)
555 } 555 }
556 556
557 xen_sched_clock_offset = xen_clocksource_read(); 557 xen_sched_clock_offset = xen_clocksource_read();
558 pv_time_ops = xen_time_ops; 558 pv_ops.time = xen_time_ops;
559 x86_init.timers.setup_percpu_clockev = xen_time_init; 559 x86_init.timers.setup_percpu_clockev = xen_time_init;
560 x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents; 560 x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
561 561
diff --git a/arch/x86/xen/vdso.h b/arch/x86/xen/vdso.h
index 861fedfe5230..873c54c488fe 100644
--- a/arch/x86/xen/vdso.h
+++ b/arch/x86/xen/vdso.h
@@ -1,3 +1,5 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
1/* Bit used for the pseudo-hwcap for non-negative segments. We use 3/* Bit used for the pseudo-hwcap for non-negative segments. We use
2 bit 1 to avoid bugs in some versions of glibc when bit 0 is 4 bit 1 to avoid bugs in some versions of glibc when bit 0 is
3 used; the choice is otherwise arbitrary. */ 5 used; the choice is otherwise arbitrary. */
diff --git a/arch/x86/xen/xen-pvh.S b/arch/x86/xen/xen-pvh.S
index ca2d3b2bf2af..b0e471506cd8 100644
--- a/arch/x86/xen/xen-pvh.S
+++ b/arch/x86/xen/xen-pvh.S
@@ -1,18 +1,7 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
1/* 3/*
2 * Copyright C 2016, Oracle and/or its affiliates. All rights reserved. 4 * Copyright C 2016, Oracle and/or its affiliates. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License along
15 * with this program. If not, see <http://www.gnu.org/licenses/>.
16 */ 5 */
17 6
18 .code32 7 .code32
diff --git a/drivers/xen/time.c b/drivers/xen/time.c
index 3e741cd1409c..0968859c29d0 100644
--- a/drivers/xen/time.c
+++ b/drivers/xen/time.c
@@ -175,7 +175,7 @@ void __init xen_time_setup_guest(void)
175 xen_runstate_remote = !HYPERVISOR_vm_assist(VMASST_CMD_enable, 175 xen_runstate_remote = !HYPERVISOR_vm_assist(VMASST_CMD_enable,
176 VMASST_TYPE_runstate_update_flag); 176 VMASST_TYPE_runstate_update_flag);
177 177
178 pv_time_ops.steal_clock = xen_steal_clock; 178 pv_ops.time.steal_clock = xen_steal_clock;
179 179
180 static_key_slow_inc(&paravirt_steal_enabled); 180 static_key_slow_inc(&paravirt_steal_enabled);
181 if (xen_runstate_remote) 181 if (xen_runstate_remote)
diff --git a/include/xen/events.h b/include/xen/events.h
index c3e6bc643a7b..a48897199975 100644
--- a/include/xen/events.h
+++ b/include/xen/events.h
@@ -89,11 +89,13 @@ unsigned irq_from_evtchn(unsigned int evtchn);
89int irq_from_virq(unsigned int cpu, unsigned int virq); 89int irq_from_virq(unsigned int cpu, unsigned int virq);
90unsigned int evtchn_from_irq(unsigned irq); 90unsigned int evtchn_from_irq(unsigned irq);
91 91
92#ifdef CONFIG_XEN_PVHVM
92/* Xen HVM evtchn vector callback */ 93/* Xen HVM evtchn vector callback */
93void xen_hvm_callback_vector(void); 94void xen_hvm_callback_vector(void);
94#ifdef CONFIG_TRACING 95#ifdef CONFIG_TRACING
95#define trace_xen_hvm_callback_vector xen_hvm_callback_vector 96#define trace_xen_hvm_callback_vector xen_hvm_callback_vector
96#endif 97#endif
98#endif
97int xen_set_callback_via(uint64_t via); 99int xen_set_callback_via(uint64_t via);
98void xen_evtchn_do_upcall(struct pt_regs *regs); 100void xen_evtchn_do_upcall(struct pt_regs *regs);
99void xen_hvm_evtchn_do_upcall(void); 101void xen_hvm_evtchn_do_upcall(void);
diff --git a/include/xen/interface/memory.h b/include/xen/interface/memory.h
index 4c5751c26f87..447004861f00 100644
--- a/include/xen/interface/memory.h
+++ b/include/xen/interface/memory.h
@@ -245,12 +245,6 @@ DEFINE_GUEST_HANDLE_STRUCT(xen_memory_map);
245 245
246 246
247/* 247/*
248 * Prevent the balloon driver from changing the memory reservation
249 * during a driver critical region.
250 */
251extern spinlock_t xen_reservation_lock;
252
253/*
254 * Unmaps the page appearing at a particular GPFN from the specified guest's 248 * Unmaps the page appearing at a particular GPFN from the specified guest's
255 * pseudophysical address space. 249 * pseudophysical address space.
256 * arg == addr of xen_remove_from_physmap_t. 250 * arg == addr of xen_remove_from_physmap_t.
diff --git a/include/xen/xen-ops.h b/include/xen/xen-ops.h
index fd18c974a619..18803ff76e27 100644
--- a/include/xen/xen-ops.h
+++ b/include/xen/xen-ops.h
@@ -5,6 +5,7 @@
5#include <linux/percpu.h> 5#include <linux/percpu.h>
6#include <linux/notifier.h> 6#include <linux/notifier.h>
7#include <linux/efi.h> 7#include <linux/efi.h>
8#include <xen/features.h>
8#include <asm/xen/interface.h> 9#include <asm/xen/interface.h>
9#include <xen/interface/vcpu.h> 10#include <xen/interface/vcpu.h>
10 11
@@ -47,6 +48,10 @@ int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order,
47 dma_addr_t *dma_handle); 48 dma_addr_t *dma_handle);
48 49
49void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order); 50void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order);
51
52int xen_remap_pfn(struct vm_area_struct *vma, unsigned long addr,
53 xen_pfn_t *pfn, int nr, int *err_ptr, pgprot_t prot,
54 unsigned int domid, bool no_translate, struct page **pages);
50#else 55#else
51static inline int xen_create_contiguous_region(phys_addr_t pstart, 56static inline int xen_create_contiguous_region(phys_addr_t pstart,
52 unsigned int order, 57 unsigned int order,
@@ -58,10 +63,50 @@ static inline int xen_create_contiguous_region(phys_addr_t pstart,
58 63
59static inline void xen_destroy_contiguous_region(phys_addr_t pstart, 64static inline void xen_destroy_contiguous_region(phys_addr_t pstart,
60 unsigned int order) { } 65 unsigned int order) { }
66
67static inline int xen_remap_pfn(struct vm_area_struct *vma, unsigned long addr,
68 xen_pfn_t *pfn, int nr, int *err_ptr,
69 pgprot_t prot, unsigned int domid,
70 bool no_translate, struct page **pages)
71{
72 BUG();
73 return 0;
74}
61#endif 75#endif
62 76
63struct vm_area_struct; 77struct vm_area_struct;
64 78
79#ifdef CONFIG_XEN_AUTO_XLATE
80int xen_xlate_remap_gfn_array(struct vm_area_struct *vma,
81 unsigned long addr,
82 xen_pfn_t *gfn, int nr,
83 int *err_ptr, pgprot_t prot,
84 unsigned int domid,
85 struct page **pages);
86int xen_xlate_unmap_gfn_range(struct vm_area_struct *vma,
87 int nr, struct page **pages);
88#else
89/*
90 * These two functions are called from arch/x86/xen/mmu.c and so stubs
91 * are needed for a configuration not specifying CONFIG_XEN_AUTO_XLATE.
92 */
93static inline int xen_xlate_remap_gfn_array(struct vm_area_struct *vma,
94 unsigned long addr,
95 xen_pfn_t *gfn, int nr,
96 int *err_ptr, pgprot_t prot,
97 unsigned int domid,
98 struct page **pages)
99{
100 return -EOPNOTSUPP;
101}
102
103static inline int xen_xlate_unmap_gfn_range(struct vm_area_struct *vma,
104 int nr, struct page **pages)
105{
106 return -EOPNOTSUPP;
107}
108#endif
109
65/* 110/*
66 * xen_remap_domain_gfn_array() - map an array of foreign frames by gfn 111 * xen_remap_domain_gfn_array() - map an array of foreign frames by gfn
67 * @vma: VMA to map the pages into 112 * @vma: VMA to map the pages into
@@ -79,12 +124,25 @@ struct vm_area_struct;
79 * Returns the number of successfully mapped frames, or a -ve error 124 * Returns the number of successfully mapped frames, or a -ve error
80 * code. 125 * code.
81 */ 126 */
82int xen_remap_domain_gfn_array(struct vm_area_struct *vma, 127static inline int xen_remap_domain_gfn_array(struct vm_area_struct *vma,
83 unsigned long addr, 128 unsigned long addr,
84 xen_pfn_t *gfn, int nr, 129 xen_pfn_t *gfn, int nr,
85 int *err_ptr, pgprot_t prot, 130 int *err_ptr, pgprot_t prot,
86 unsigned domid, 131 unsigned int domid,
87 struct page **pages); 132 struct page **pages)
133{
134 if (xen_feature(XENFEAT_auto_translated_physmap))
135 return xen_xlate_remap_gfn_array(vma, addr, gfn, nr, err_ptr,
136 prot, domid, pages);
137
138 /* We BUG_ON because it's a programmer error to pass a NULL err_ptr,
139 * and the consequences later is quite hard to detect what the actual
140 * cause of "wrong memory was mapped in".
141 */
142 BUG_ON(err_ptr == NULL);
143 return xen_remap_pfn(vma, addr, gfn, nr, err_ptr, prot, domid,
144 false, pages);
145}
88 146
89/* 147/*
90 * xen_remap_domain_mfn_array() - map an array of foreign frames by mfn 148 * xen_remap_domain_mfn_array() - map an array of foreign frames by mfn
@@ -103,10 +161,18 @@ int xen_remap_domain_gfn_array(struct vm_area_struct *vma,
103 * Returns the number of successfully mapped frames, or a -ve error 161 * Returns the number of successfully mapped frames, or a -ve error
104 * code. 162 * code.
105 */ 163 */
106int xen_remap_domain_mfn_array(struct vm_area_struct *vma, 164static inline int xen_remap_domain_mfn_array(struct vm_area_struct *vma,
107 unsigned long addr, xen_pfn_t *mfn, int nr, 165 unsigned long addr, xen_pfn_t *mfn,
108 int *err_ptr, pgprot_t prot, 166 int nr, int *err_ptr,
109 unsigned int domid, struct page **pages); 167 pgprot_t prot, unsigned int domid,
168 struct page **pages)
169{
170 if (xen_feature(XENFEAT_auto_translated_physmap))
171 return -EOPNOTSUPP;
172
173 return xen_remap_pfn(vma, addr, mfn, nr, err_ptr, prot, domid,
174 true, pages);
175}
110 176
111/* xen_remap_domain_gfn_range() - map a range of foreign frames 177/* xen_remap_domain_gfn_range() - map a range of foreign frames
112 * @vma: VMA to map the pages into 178 * @vma: VMA to map the pages into
@@ -120,44 +186,21 @@ int xen_remap_domain_mfn_array(struct vm_area_struct *vma,
120 * Returns the number of successfully mapped frames, or a -ve error 186 * Returns the number of successfully mapped frames, or a -ve error
121 * code. 187 * code.
122 */ 188 */
123int xen_remap_domain_gfn_range(struct vm_area_struct *vma, 189static inline int xen_remap_domain_gfn_range(struct vm_area_struct *vma,
124 unsigned long addr, 190 unsigned long addr,
125 xen_pfn_t gfn, int nr, 191 xen_pfn_t gfn, int nr,
126 pgprot_t prot, unsigned domid, 192 pgprot_t prot, unsigned int domid,
127 struct page **pages); 193 struct page **pages)
128int xen_unmap_domain_gfn_range(struct vm_area_struct *vma,
129 int numpgs, struct page **pages);
130
131#ifdef CONFIG_XEN_AUTO_XLATE
132int xen_xlate_remap_gfn_array(struct vm_area_struct *vma,
133 unsigned long addr,
134 xen_pfn_t *gfn, int nr,
135 int *err_ptr, pgprot_t prot,
136 unsigned domid,
137 struct page **pages);
138int xen_xlate_unmap_gfn_range(struct vm_area_struct *vma,
139 int nr, struct page **pages);
140#else
141/*
142 * These two functions are called from arch/x86/xen/mmu.c and so stubs
143 * are needed for a configuration not specifying CONFIG_XEN_AUTO_XLATE.
144 */
145static inline int xen_xlate_remap_gfn_array(struct vm_area_struct *vma,
146 unsigned long addr,
147 xen_pfn_t *gfn, int nr,
148 int *err_ptr, pgprot_t prot,
149 unsigned int domid,
150 struct page **pages)
151{ 194{
152 return -EOPNOTSUPP; 195 if (xen_feature(XENFEAT_auto_translated_physmap))
153} 196 return -EOPNOTSUPP;
154 197
155static inline int xen_xlate_unmap_gfn_range(struct vm_area_struct *vma, 198 return xen_remap_pfn(vma, addr, &gfn, nr, NULL, prot, domid, false,
156 int nr, struct page **pages) 199 pages);
157{
158 return -EOPNOTSUPP;
159} 200}
160#endif 201
202int xen_unmap_domain_gfn_range(struct vm_area_struct *vma,
203 int numpgs, struct page **pages);
161 204
162int xen_xlate_map_ballooned_pages(xen_pfn_t **pfns, void **vaddr, 205int xen_xlate_map_ballooned_pages(xen_pfn_t **pfns, void **vaddr,
163 unsigned long nr_grant_frames); 206 unsigned long nr_grant_frames);
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-28 14:24:04 -0400