aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86
diff options
context:
space:
mode:
authorDave Airlie <airlied@redhat.com>2018-09-26 21:06:46 -0400
committerDave Airlie <airlied@redhat.com>2018-09-26 21:06:46 -0400
commitbf78296ab1cb215d0609ac6cff4e43e941e51265 (patch)
treea193615b327d9ee538e71ca5f13bbfb4f3db4e6b /arch/x86
parent18eb2f6e19d77900695987e3a2b775cccbe5b84e (diff)
parent6bf4ca7fbc85d80446ac01c0d1d77db4d91a6d84 (diff)
BackMerge v4.19-rc5 into drm-next
Sean Paul requested an -rc5 backmerge from some sun4i fixes. Signed-off-by: Dave Airlie <airlied@redhat.com>
Diffstat (limited to 'arch/x86')
-rw-r--r--arch/x86/crypto/aegis128-aesni-glue.c1
-rw-r--r--arch/x86/crypto/aegis128l-aesni-glue.c1
-rw-r--r--arch/x86/crypto/aegis256-aesni-glue.c1
-rw-r--r--arch/x86/crypto/morus1280-sse2-glue.c1
-rw-r--r--arch/x86/crypto/morus640-sse2-glue.c1
-rw-r--r--arch/x86/events/intel/lbr.c4
-rw-r--r--arch/x86/hyperv/hv_apic.c8
-rw-r--r--arch/x86/include/asm/atomic.h12
-rw-r--r--arch/x86/include/asm/atomic64_32.h8
-rw-r--r--arch/x86/include/asm/atomic64_64.h12
-rw-r--r--arch/x86/include/asm/fixmap.h10
-rw-r--r--arch/x86/include/asm/hyperv-tlfs.h16
-rw-r--r--arch/x86/include/asm/kdebug.h12
-rw-r--r--arch/x86/include/asm/kvm_host.h27
-rw-r--r--arch/x86/include/asm/mem_encrypt.h7
-rw-r--r--arch/x86/include/asm/pgtable-2level.h9
-rw-r--r--arch/x86/include/asm/pgtable.h2
-rw-r--r--arch/x86/include/asm/pgtable_64.h23
-rw-r--r--arch/x86/include/uapi/asm/kvm.h1
-rw-r--r--arch/x86/kernel/apic/vector.c2
-rw-r--r--arch/x86/kernel/apm_32.c2
-rw-r--r--arch/x86/kernel/cpu/intel_rdt.h17
-rw-r--r--arch/x86/kernel/cpu/intel_rdt_ctrlmondata.c27
-rw-r--r--arch/x86/kernel/cpu/intel_rdt_rdtgroup.c53
-rw-r--r--arch/x86/kernel/cpu/microcode/amd.c24
-rw-r--r--arch/x86/kernel/cpu/microcode/intel.c17
-rw-r--r--arch/x86/kernel/dumpstack.c11
-rw-r--r--arch/x86/kernel/eisa.c10
-rw-r--r--arch/x86/kernel/head64.c20
-rw-r--r--arch/x86/kernel/head_64.S16
-rw-r--r--arch/x86/kernel/kvmclock.c52
-rw-r--r--arch/x86/kernel/paravirt.c4
-rw-r--r--arch/x86/kernel/process_32.c4
-rw-r--r--arch/x86/kernel/process_64.c12
-rw-r--r--arch/x86/kernel/topology.c4
-rw-r--r--arch/x86/kernel/tsc.c2
-rw-r--r--arch/x86/kernel/vmlinux.lds.S19
-rw-r--r--arch/x86/kvm/lapic.c49
-rw-r--r--arch/x86/kvm/mmu.c35
-rw-r--r--arch/x86/kvm/svm.c26
-rw-r--r--arch/x86/kvm/vmx.c181
-rw-r--r--arch/x86/kvm/x86.c129
-rw-r--r--arch/x86/kvm/x86.h2
-rw-r--r--arch/x86/mm/init.c4
-rw-r--r--arch/x86/mm/mem_encrypt.c24
-rw-r--r--arch/x86/mm/pgtable.c17
-rw-r--r--arch/x86/platform/efi/efi_32.c3
-rw-r--r--arch/x86/xen/mmu_pv.c8
-rw-r--r--arch/x86/xen/pmu.c2
49 files changed, 661 insertions, 271 deletions
diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c
index acd11b3bf639..2a356b948720 100644
--- a/arch/x86/crypto/aegis128-aesni-glue.c
+++ b/arch/x86/crypto/aegis128-aesni-glue.c
@@ -379,7 +379,6 @@ static int __init crypto_aegis128_aesni_module_init(void)
379{ 379{
380 if (!boot_cpu_has(X86_FEATURE_XMM2) || 380 if (!boot_cpu_has(X86_FEATURE_XMM2) ||
381 !boot_cpu_has(X86_FEATURE_AES) || 381 !boot_cpu_has(X86_FEATURE_AES) ||
382 !boot_cpu_has(X86_FEATURE_OSXSAVE) ||
383 !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) 382 !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
384 return -ENODEV; 383 return -ENODEV;
385 384
diff --git a/arch/x86/crypto/aegis128l-aesni-glue.c b/arch/x86/crypto/aegis128l-aesni-glue.c
index 2071c3d1ae07..dbe8bb980da1 100644
--- a/arch/x86/crypto/aegis128l-aesni-glue.c
+++ b/arch/x86/crypto/aegis128l-aesni-glue.c
@@ -379,7 +379,6 @@ static int __init crypto_aegis128l_aesni_module_init(void)
379{ 379{
380 if (!boot_cpu_has(X86_FEATURE_XMM2) || 380 if (!boot_cpu_has(X86_FEATURE_XMM2) ||
381 !boot_cpu_has(X86_FEATURE_AES) || 381 !boot_cpu_has(X86_FEATURE_AES) ||
382 !boot_cpu_has(X86_FEATURE_OSXSAVE) ||
383 !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) 382 !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
384 return -ENODEV; 383 return -ENODEV;
385 384
diff --git a/arch/x86/crypto/aegis256-aesni-glue.c b/arch/x86/crypto/aegis256-aesni-glue.c
index b5f2a8fd5a71..8bebda2de92f 100644
--- a/arch/x86/crypto/aegis256-aesni-glue.c
+++ b/arch/x86/crypto/aegis256-aesni-glue.c
@@ -379,7 +379,6 @@ static int __init crypto_aegis256_aesni_module_init(void)
379{ 379{
380 if (!boot_cpu_has(X86_FEATURE_XMM2) || 380 if (!boot_cpu_has(X86_FEATURE_XMM2) ||
381 !boot_cpu_has(X86_FEATURE_AES) || 381 !boot_cpu_has(X86_FEATURE_AES) ||
382 !boot_cpu_has(X86_FEATURE_OSXSAVE) ||
383 !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) 382 !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
384 return -ENODEV; 383 return -ENODEV;
385 384
diff --git a/arch/x86/crypto/morus1280-sse2-glue.c b/arch/x86/crypto/morus1280-sse2-glue.c
index 95cf857d2cbb..f40244eaf14d 100644
--- a/arch/x86/crypto/morus1280-sse2-glue.c
+++ b/arch/x86/crypto/morus1280-sse2-glue.c
@@ -40,7 +40,6 @@ MORUS1280_DECLARE_ALGS(sse2, "morus1280-sse2", 350);
40static int __init crypto_morus1280_sse2_module_init(void) 40static int __init crypto_morus1280_sse2_module_init(void)
41{ 41{
42 if (!boot_cpu_has(X86_FEATURE_XMM2) || 42 if (!boot_cpu_has(X86_FEATURE_XMM2) ||
43 !boot_cpu_has(X86_FEATURE_OSXSAVE) ||
44 !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) 43 !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
45 return -ENODEV; 44 return -ENODEV;
46 45
diff --git a/arch/x86/crypto/morus640-sse2-glue.c b/arch/x86/crypto/morus640-sse2-glue.c
index 615fb7bc9a32..9afaf8f8565a 100644
--- a/arch/x86/crypto/morus640-sse2-glue.c
+++ b/arch/x86/crypto/morus640-sse2-glue.c
@@ -40,7 +40,6 @@ MORUS640_DECLARE_ALGS(sse2, "morus640-sse2", 400);
40static int __init crypto_morus640_sse2_module_init(void) 40static int __init crypto_morus640_sse2_module_init(void)
41{ 41{
42 if (!boot_cpu_has(X86_FEATURE_XMM2) || 42 if (!boot_cpu_has(X86_FEATURE_XMM2) ||
43 !boot_cpu_has(X86_FEATURE_OSXSAVE) ||
44 !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) 43 !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
45 return -ENODEV; 44 return -ENODEV;
46 45
diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index f3e006bed9a7..c88ed39582a1 100644
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -1272,4 +1272,8 @@ void intel_pmu_lbr_init_knl(void)
1272 1272
1273 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1273 x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
1274 x86_pmu.lbr_sel_map = snb_lbr_sel_map; 1274 x86_pmu.lbr_sel_map = snb_lbr_sel_map;
1275
1276 /* Knights Landing does have MISPREDICT bit */
1277 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_LIP)
1278 x86_pmu.intel_cap.lbr_format = LBR_FORMAT_EIP_FLAGS;
1275} 1279}
diff --git a/arch/x86/hyperv/hv_apic.c b/arch/x86/hyperv/hv_apic.c
index 5b0f613428c2..2c43e3055948 100644
--- a/arch/x86/hyperv/hv_apic.c
+++ b/arch/x86/hyperv/hv_apic.c
@@ -95,8 +95,8 @@ static void hv_apic_eoi_write(u32 reg, u32 val)
95 */ 95 */
96static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector) 96static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector)
97{ 97{
98 struct ipi_arg_ex **arg; 98 struct hv_send_ipi_ex **arg;
99 struct ipi_arg_ex *ipi_arg; 99 struct hv_send_ipi_ex *ipi_arg;
100 unsigned long flags; 100 unsigned long flags;
101 int nr_bank = 0; 101 int nr_bank = 0;
102 int ret = 1; 102 int ret = 1;
@@ -105,7 +105,7 @@ static bool __send_ipi_mask_ex(const struct cpumask *mask, int vector)
105 return false; 105 return false;
106 106
107 local_irq_save(flags); 107 local_irq_save(flags);
108 arg = (struct ipi_arg_ex **)this_cpu_ptr(hyperv_pcpu_input_arg); 108 arg = (struct hv_send_ipi_ex **)this_cpu_ptr(hyperv_pcpu_input_arg);
109 109
110 ipi_arg = *arg; 110 ipi_arg = *arg;
111 if (unlikely(!ipi_arg)) 111 if (unlikely(!ipi_arg))
@@ -135,7 +135,7 @@ ipi_mask_ex_done:
135static bool __send_ipi_mask(const struct cpumask *mask, int vector) 135static bool __send_ipi_mask(const struct cpumask *mask, int vector)
136{ 136{
137 int cur_cpu, vcpu; 137 int cur_cpu, vcpu;
138 struct ipi_arg_non_ex ipi_arg; 138 struct hv_send_ipi ipi_arg;
139 int ret = 1; 139 int ret = 1;
140 140
141 trace_hyperv_send_ipi_mask(mask, vector); 141 trace_hyperv_send_ipi_mask(mask, vector);
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index b143717b92b3..ce84388e540c 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -80,11 +80,11 @@ static __always_inline void arch_atomic_sub(int i, atomic_t *v)
80 * true if the result is zero, or false for all 80 * true if the result is zero, or false for all
81 * other cases. 81 * other cases.
82 */ 82 */
83#define arch_atomic_sub_and_test arch_atomic_sub_and_test
84static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v) 83static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v)
85{ 84{
86 GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, "er", i, "%0", e); 85 GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, "er", i, "%0", e);
87} 86}
87#define arch_atomic_sub_and_test arch_atomic_sub_and_test
88 88
89/** 89/**
90 * arch_atomic_inc - increment atomic variable 90 * arch_atomic_inc - increment atomic variable
@@ -92,12 +92,12 @@ static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v)
92 * 92 *
93 * Atomically increments @v by 1. 93 * Atomically increments @v by 1.
94 */ 94 */
95#define arch_atomic_inc arch_atomic_inc
96static __always_inline void arch_atomic_inc(atomic_t *v) 95static __always_inline void arch_atomic_inc(atomic_t *v)
97{ 96{
98 asm volatile(LOCK_PREFIX "incl %0" 97 asm volatile(LOCK_PREFIX "incl %0"
99 : "+m" (v->counter)); 98 : "+m" (v->counter));
100} 99}
100#define arch_atomic_inc arch_atomic_inc
101 101
102/** 102/**
103 * arch_atomic_dec - decrement atomic variable 103 * arch_atomic_dec - decrement atomic variable
@@ -105,12 +105,12 @@ static __always_inline void arch_atomic_inc(atomic_t *v)
105 * 105 *
106 * Atomically decrements @v by 1. 106 * Atomically decrements @v by 1.
107 */ 107 */
108#define arch_atomic_dec arch_atomic_dec
109static __always_inline void arch_atomic_dec(atomic_t *v) 108static __always_inline void arch_atomic_dec(atomic_t *v)
110{ 109{
111 asm volatile(LOCK_PREFIX "decl %0" 110 asm volatile(LOCK_PREFIX "decl %0"
112 : "+m" (v->counter)); 111 : "+m" (v->counter));
113} 112}
113#define arch_atomic_dec arch_atomic_dec
114 114
115/** 115/**
116 * arch_atomic_dec_and_test - decrement and test 116 * arch_atomic_dec_and_test - decrement and test
@@ -120,11 +120,11 @@ static __always_inline void arch_atomic_dec(atomic_t *v)
120 * returns true if the result is 0, or false for all other 120 * returns true if the result is 0, or false for all other
121 * cases. 121 * cases.
122 */ 122 */
123#define arch_atomic_dec_and_test arch_atomic_dec_and_test
124static __always_inline bool arch_atomic_dec_and_test(atomic_t *v) 123static __always_inline bool arch_atomic_dec_and_test(atomic_t *v)
125{ 124{
126 GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, "%0", e); 125 GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, "%0", e);
127} 126}
127#define arch_atomic_dec_and_test arch_atomic_dec_and_test
128 128
129/** 129/**
130 * arch_atomic_inc_and_test - increment and test 130 * arch_atomic_inc_and_test - increment and test
@@ -134,11 +134,11 @@ static __always_inline bool arch_atomic_dec_and_test(atomic_t *v)
134 * and returns true if the result is zero, or false for all 134 * and returns true if the result is zero, or false for all
135 * other cases. 135 * other cases.
136 */ 136 */
137#define arch_atomic_inc_and_test arch_atomic_inc_and_test
138static __always_inline bool arch_atomic_inc_and_test(atomic_t *v) 137static __always_inline bool arch_atomic_inc_and_test(atomic_t *v)
139{ 138{
140 GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, "%0", e); 139 GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, "%0", e);
141} 140}
141#define arch_atomic_inc_and_test arch_atomic_inc_and_test
142 142
143/** 143/**
144 * arch_atomic_add_negative - add and test if negative 144 * arch_atomic_add_negative - add and test if negative
@@ -149,11 +149,11 @@ static __always_inline bool arch_atomic_inc_and_test(atomic_t *v)
149 * if the result is negative, or false when 149 * if the result is negative, or false when
150 * result is greater than or equal to zero. 150 * result is greater than or equal to zero.
151 */ 151 */
152#define arch_atomic_add_negative arch_atomic_add_negative
153static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v) 152static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v)
154{ 153{
155 GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, "er", i, "%0", s); 154 GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, "er", i, "%0", s);
156} 155}
156#define arch_atomic_add_negative arch_atomic_add_negative
157 157
158/** 158/**
159 * arch_atomic_add_return - add integer and return 159 * arch_atomic_add_return - add integer and return
diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atomic64_32.h
index ef959f02d070..6a5b0ec460da 100644
--- a/arch/x86/include/asm/atomic64_32.h
+++ b/arch/x86/include/asm/atomic64_32.h
@@ -205,12 +205,12 @@ static inline long long arch_atomic64_sub(long long i, atomic64_t *v)
205 * 205 *
206 * Atomically increments @v by 1. 206 * Atomically increments @v by 1.
207 */ 207 */
208#define arch_atomic64_inc arch_atomic64_inc
209static inline void arch_atomic64_inc(atomic64_t *v) 208static inline void arch_atomic64_inc(atomic64_t *v)
210{ 209{
211 __alternative_atomic64(inc, inc_return, /* no output */, 210 __alternative_atomic64(inc, inc_return, /* no output */,
212 "S" (v) : "memory", "eax", "ecx", "edx"); 211 "S" (v) : "memory", "eax", "ecx", "edx");
213} 212}
213#define arch_atomic64_inc arch_atomic64_inc
214 214
215/** 215/**
216 * arch_atomic64_dec - decrement atomic64 variable 216 * arch_atomic64_dec - decrement atomic64 variable
@@ -218,12 +218,12 @@ static inline void arch_atomic64_inc(atomic64_t *v)
218 * 218 *
219 * Atomically decrements @v by 1. 219 * Atomically decrements @v by 1.
220 */ 220 */
221#define arch_atomic64_dec arch_atomic64_dec
222static inline void arch_atomic64_dec(atomic64_t *v) 221static inline void arch_atomic64_dec(atomic64_t *v)
223{ 222{
224 __alternative_atomic64(dec, dec_return, /* no output */, 223 __alternative_atomic64(dec, dec_return, /* no output */,
225 "S" (v) : "memory", "eax", "ecx", "edx"); 224 "S" (v) : "memory", "eax", "ecx", "edx");
226} 225}
226#define arch_atomic64_dec arch_atomic64_dec
227 227
228/** 228/**
229 * arch_atomic64_add_unless - add unless the number is a given value 229 * arch_atomic64_add_unless - add unless the number is a given value
@@ -245,7 +245,6 @@ static inline int arch_atomic64_add_unless(atomic64_t *v, long long a,
245 return (int)a; 245 return (int)a;
246} 246}
247 247
248#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
249static inline int arch_atomic64_inc_not_zero(atomic64_t *v) 248static inline int arch_atomic64_inc_not_zero(atomic64_t *v)
250{ 249{
251 int r; 250 int r;
@@ -253,8 +252,8 @@ static inline int arch_atomic64_inc_not_zero(atomic64_t *v)
253 "S" (v) : "ecx", "edx", "memory"); 252 "S" (v) : "ecx", "edx", "memory");
254 return r; 253 return r;
255} 254}
255#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
256 256
257#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
258static inline long long arch_atomic64_dec_if_positive(atomic64_t *v) 257static inline long long arch_atomic64_dec_if_positive(atomic64_t *v)
259{ 258{
260 long long r; 259 long long r;
@@ -262,6 +261,7 @@ static inline long long arch_atomic64_dec_if_positive(atomic64_t *v)
262 "S" (v) : "ecx", "memory"); 261 "S" (v) : "ecx", "memory");
263 return r; 262 return r;
264} 263}
264#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
265 265
266#undef alternative_atomic64 266#undef alternative_atomic64
267#undef __alternative_atomic64 267#undef __alternative_atomic64
diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h
index 4343d9b4f30e..5f851d92eecd 100644
--- a/arch/x86/include/asm/atomic64_64.h
+++ b/arch/x86/include/asm/atomic64_64.h
@@ -71,11 +71,11 @@ static inline void arch_atomic64_sub(long i, atomic64_t *v)
71 * true if the result is zero, or false for all 71 * true if the result is zero, or false for all
72 * other cases. 72 * other cases.
73 */ 73 */
74#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
75static inline bool arch_atomic64_sub_and_test(long i, atomic64_t *v) 74static inline bool arch_atomic64_sub_and_test(long i, atomic64_t *v)
76{ 75{
77 GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, "er", i, "%0", e); 76 GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, "er", i, "%0", e);
78} 77}
78#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
79 79
80/** 80/**
81 * arch_atomic64_inc - increment atomic64 variable 81 * arch_atomic64_inc - increment atomic64 variable
@@ -83,13 +83,13 @@ static inline bool arch_atomic64_sub_and_test(long i, atomic64_t *v)
83 * 83 *
84 * Atomically increments @v by 1. 84 * Atomically increments @v by 1.
85 */ 85 */
86#define arch_atomic64_inc arch_atomic64_inc
87static __always_inline void arch_atomic64_inc(atomic64_t *v) 86static __always_inline void arch_atomic64_inc(atomic64_t *v)
88{ 87{
89 asm volatile(LOCK_PREFIX "incq %0" 88 asm volatile(LOCK_PREFIX "incq %0"
90 : "=m" (v->counter) 89 : "=m" (v->counter)
91 : "m" (v->counter)); 90 : "m" (v->counter));
92} 91}
92#define arch_atomic64_inc arch_atomic64_inc
93 93
94/** 94/**
95 * arch_atomic64_dec - decrement atomic64 variable 95 * arch_atomic64_dec - decrement atomic64 variable
@@ -97,13 +97,13 @@ static __always_inline void arch_atomic64_inc(atomic64_t *v)
97 * 97 *
98 * Atomically decrements @v by 1. 98 * Atomically decrements @v by 1.
99 */ 99 */
100#define arch_atomic64_dec arch_atomic64_dec
101static __always_inline void arch_atomic64_dec(atomic64_t *v) 100static __always_inline void arch_atomic64_dec(atomic64_t *v)
102{ 101{
103 asm volatile(LOCK_PREFIX "decq %0" 102 asm volatile(LOCK_PREFIX "decq %0"
104 : "=m" (v->counter) 103 : "=m" (v->counter)
105 : "m" (v->counter)); 104 : "m" (v->counter));
106} 105}
106#define arch_atomic64_dec arch_atomic64_dec
107 107
108/** 108/**
109 * arch_atomic64_dec_and_test - decrement and test 109 * arch_atomic64_dec_and_test - decrement and test
@@ -113,11 +113,11 @@ static __always_inline void arch_atomic64_dec(atomic64_t *v)
113 * returns true if the result is 0, or false for all other 113 * returns true if the result is 0, or false for all other
114 * cases. 114 * cases.
115 */ 115 */
116#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
117static inline bool arch_atomic64_dec_and_test(atomic64_t *v) 116static inline bool arch_atomic64_dec_and_test(atomic64_t *v)
118{ 117{
119 GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, "%0", e); 118 GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, "%0", e);
120} 119}
120#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
121 121
122/** 122/**
123 * arch_atomic64_inc_and_test - increment and test 123 * arch_atomic64_inc_and_test - increment and test
@@ -127,11 +127,11 @@ static inline bool arch_atomic64_dec_and_test(atomic64_t *v)
127 * and returns true if the result is zero, or false for all 127 * and returns true if the result is zero, or false for all
128 * other cases. 128 * other cases.
129 */ 129 */
130#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
131static inline bool arch_atomic64_inc_and_test(atomic64_t *v) 130static inline bool arch_atomic64_inc_and_test(atomic64_t *v)
132{ 131{
133 GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, "%0", e); 132 GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, "%0", e);
134} 133}
134#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
135 135
136/** 136/**
137 * arch_atomic64_add_negative - add and test if negative 137 * arch_atomic64_add_negative - add and test if negative
@@ -142,11 +142,11 @@ static inline bool arch_atomic64_inc_and_test(atomic64_t *v)
142 * if the result is negative, or false when 142 * if the result is negative, or false when
143 * result is greater than or equal to zero. 143 * result is greater than or equal to zero.
144 */ 144 */
145#define arch_atomic64_add_negative arch_atomic64_add_negative
146static inline bool arch_atomic64_add_negative(long i, atomic64_t *v) 145static inline bool arch_atomic64_add_negative(long i, atomic64_t *v)
147{ 146{
148 GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, "er", i, "%0", s); 147 GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, "er", i, "%0", s);
149} 148}
149#define arch_atomic64_add_negative arch_atomic64_add_negative
150 150
151/** 151/**
152 * arch_atomic64_add_return - add and return 152 * arch_atomic64_add_return - add and return
diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h
index e203169931c7..6390bd8c141b 100644
--- a/arch/x86/include/asm/fixmap.h
+++ b/arch/x86/include/asm/fixmap.h
@@ -14,6 +14,16 @@
14#ifndef _ASM_X86_FIXMAP_H 14#ifndef _ASM_X86_FIXMAP_H
15#define _ASM_X86_FIXMAP_H 15#define _ASM_X86_FIXMAP_H
16 16
17/*
18 * Exposed to assembly code for setting up initial page tables. Cannot be
19 * calculated in assembly code (fixmap entries are an enum), but is sanity
20 * checked in the actual fixmap C code to make sure that the fixmap is
21 * covered fully.
22 */
23#define FIXMAP_PMD_NUM 2
24/* fixmap starts downwards from the 507th entry in level2_fixmap_pgt */
25#define FIXMAP_PMD_TOP 507
26
17#ifndef __ASSEMBLY__ 27#ifndef __ASSEMBLY__
18#include <linux/kernel.h> 28#include <linux/kernel.h>
19#include <asm/acpi.h> 29#include <asm/acpi.h>
diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h
index e977b6b3a538..00e01d215f74 100644
--- a/arch/x86/include/asm/hyperv-tlfs.h
+++ b/arch/x86/include/asm/hyperv-tlfs.h
@@ -726,19 +726,21 @@ struct hv_enlightened_vmcs {
726#define HV_STIMER_AUTOENABLE (1ULL << 3) 726#define HV_STIMER_AUTOENABLE (1ULL << 3)
727#define HV_STIMER_SINT(config) (__u8)(((config) >> 16) & 0x0F) 727#define HV_STIMER_SINT(config) (__u8)(((config) >> 16) & 0x0F)
728 728
729struct ipi_arg_non_ex {
730 u32 vector;
731 u32 reserved;
732 u64 cpu_mask;
733};
734
735struct hv_vpset { 729struct hv_vpset {
736 u64 format; 730 u64 format;
737 u64 valid_bank_mask; 731 u64 valid_bank_mask;
738 u64 bank_contents[]; 732 u64 bank_contents[];
739}; 733};
740 734
741struct ipi_arg_ex { 735/* HvCallSendSyntheticClusterIpi hypercall */
736struct hv_send_ipi {
737 u32 vector;
738 u32 reserved;
739 u64 cpu_mask;
740};
741
742/* HvCallSendSyntheticClusterIpiEx hypercall */
743struct hv_send_ipi_ex {
742 u32 vector; 744 u32 vector;
743 u32 reserved; 745 u32 reserved;
744 struct hv_vpset vp_set; 746 struct hv_vpset vp_set;
diff --git a/arch/x86/include/asm/kdebug.h b/arch/x86/include/asm/kdebug.h
index 395c9631e000..75f1e35e7c15 100644
--- a/arch/x86/include/asm/kdebug.h
+++ b/arch/x86/include/asm/kdebug.h
@@ -22,10 +22,20 @@ enum die_val {
22 DIE_NMIUNKNOWN, 22 DIE_NMIUNKNOWN,
23}; 23};
24 24
25enum show_regs_mode {
26 SHOW_REGS_SHORT,
27 /*
28 * For when userspace crashed, but we don't think it's our fault, and
29 * therefore don't print kernel registers.
30 */
31 SHOW_REGS_USER,
32 SHOW_REGS_ALL
33};
34
25extern void die(const char *, struct pt_regs *,long); 35extern void die(const char *, struct pt_regs *,long);
26extern int __must_check __die(const char *, struct pt_regs *, long); 36extern int __must_check __die(const char *, struct pt_regs *, long);
27extern void show_stack_regs(struct pt_regs *regs); 37extern void show_stack_regs(struct pt_regs *regs);
28extern void __show_regs(struct pt_regs *regs, int all); 38extern void __show_regs(struct pt_regs *regs, enum show_regs_mode);
29extern void show_iret_regs(struct pt_regs *regs); 39extern void show_iret_regs(struct pt_regs *regs);
30extern unsigned long oops_begin(void); 40extern unsigned long oops_begin(void);
31extern void oops_end(unsigned long, struct pt_regs *, int signr); 41extern void oops_end(unsigned long, struct pt_regs *, int signr);
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 00ddb0c9e612..09b2e3e2cf1b 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -869,6 +869,8 @@ struct kvm_arch {
869 869
870 bool x2apic_format; 870 bool x2apic_format;
871 bool x2apic_broadcast_quirk_disabled; 871 bool x2apic_broadcast_quirk_disabled;
872
873 bool guest_can_read_msr_platform_info;
872}; 874};
873 875
874struct kvm_vm_stat { 876struct kvm_vm_stat {
@@ -1022,6 +1024,7 @@ struct kvm_x86_ops {
1022 void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu); 1024 void (*refresh_apicv_exec_ctrl)(struct kvm_vcpu *vcpu);
1023 void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr); 1025 void (*hwapic_irr_update)(struct kvm_vcpu *vcpu, int max_irr);
1024 void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr); 1026 void (*hwapic_isr_update)(struct kvm_vcpu *vcpu, int isr);
1027 bool (*guest_apic_has_interrupt)(struct kvm_vcpu *vcpu);
1025 void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); 1028 void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
1026 void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu); 1029 void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu);
1027 void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu, hpa_t hpa); 1030 void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu, hpa_t hpa);
@@ -1055,6 +1058,7 @@ struct kvm_x86_ops {
1055 bool (*umip_emulated)(void); 1058 bool (*umip_emulated)(void);
1056 1059
1057 int (*check_nested_events)(struct kvm_vcpu *vcpu, bool external_intr); 1060 int (*check_nested_events)(struct kvm_vcpu *vcpu, bool external_intr);
1061 void (*request_immediate_exit)(struct kvm_vcpu *vcpu);
1058 1062
1059 void (*sched_in)(struct kvm_vcpu *kvm, int cpu); 1063 void (*sched_in)(struct kvm_vcpu *kvm, int cpu);
1060 1064
@@ -1237,19 +1241,12 @@ enum emulation_result {
1237#define EMULTYPE_NO_DECODE (1 << 0) 1241#define EMULTYPE_NO_DECODE (1 << 0)
1238#define EMULTYPE_TRAP_UD (1 << 1) 1242#define EMULTYPE_TRAP_UD (1 << 1)
1239#define EMULTYPE_SKIP (1 << 2) 1243#define EMULTYPE_SKIP (1 << 2)
1240#define EMULTYPE_RETRY (1 << 3) 1244#define EMULTYPE_ALLOW_RETRY (1 << 3)
1241#define EMULTYPE_NO_REEXECUTE (1 << 4) 1245#define EMULTYPE_NO_UD_ON_FAIL (1 << 4)
1242#define EMULTYPE_NO_UD_ON_FAIL (1 << 5) 1246#define EMULTYPE_VMWARE (1 << 5)
1243#define EMULTYPE_VMWARE (1 << 6) 1247int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type);
1244int x86_emulate_instruction(struct kvm_vcpu *vcpu, unsigned long cr2, 1248int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
1245 int emulation_type, void *insn, int insn_len); 1249 void *insn, int insn_len);
1246
1247static inline int emulate_instruction(struct kvm_vcpu *vcpu,
1248 int emulation_type)
1249{
1250 return x86_emulate_instruction(vcpu, 0,
1251 emulation_type | EMULTYPE_NO_REEXECUTE, NULL, 0);
1252}
1253 1250
1254void kvm_enable_efer_bits(u64); 1251void kvm_enable_efer_bits(u64);
1255bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer); 1252bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer);
@@ -1450,7 +1447,6 @@ asmlinkage void kvm_spurious_fault(void);
1450 ____kvm_handle_fault_on_reboot(insn, "") 1447 ____kvm_handle_fault_on_reboot(insn, "")
1451 1448
1452#define KVM_ARCH_WANT_MMU_NOTIFIER 1449#define KVM_ARCH_WANT_MMU_NOTIFIER
1453int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
1454int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end); 1450int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end);
1455int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); 1451int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
1456int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); 1452int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
@@ -1463,7 +1459,7 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event);
1463void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu); 1459void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu);
1464 1460
1465int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, 1461int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
1466 unsigned long ipi_bitmap_high, int min, 1462 unsigned long ipi_bitmap_high, u32 min,
1467 unsigned long icr, int op_64_bit); 1463 unsigned long icr, int op_64_bit);
1468 1464
1469u64 kvm_get_arch_capabilities(void); 1465u64 kvm_get_arch_capabilities(void);
@@ -1490,6 +1486,7 @@ extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn);
1490 1486
1491int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu); 1487int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu);
1492int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err); 1488int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err);
1489void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu);
1493 1490
1494int kvm_is_in_guest(void); 1491int kvm_is_in_guest(void);
1495 1492
diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h
index c0643831706e..616f8e637bc3 100644
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -48,10 +48,13 @@ int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size);
48 48
49/* Architecture __weak replacement functions */ 49/* Architecture __weak replacement functions */
50void __init mem_encrypt_init(void); 50void __init mem_encrypt_init(void);
51void __init mem_encrypt_free_decrypted_mem(void);
51 52
52bool sme_active(void); 53bool sme_active(void);
53bool sev_active(void); 54bool sev_active(void);
54 55
56#define __bss_decrypted __attribute__((__section__(".bss..decrypted")))
57
55#else /* !CONFIG_AMD_MEM_ENCRYPT */ 58#else /* !CONFIG_AMD_MEM_ENCRYPT */
56 59
57#define sme_me_mask 0ULL 60#define sme_me_mask 0ULL
@@ -77,6 +80,8 @@ early_set_memory_decrypted(unsigned long vaddr, unsigned long size) { return 0;
77static inline int __init 80static inline int __init
78early_set_memory_encrypted(unsigned long vaddr, unsigned long size) { return 0; } 81early_set_memory_encrypted(unsigned long vaddr, unsigned long size) { return 0; }
79 82
83#define __bss_decrypted
84
80#endif /* CONFIG_AMD_MEM_ENCRYPT */ 85#endif /* CONFIG_AMD_MEM_ENCRYPT */
81 86
82/* 87/*
@@ -88,6 +93,8 @@ early_set_memory_encrypted(unsigned long vaddr, unsigned long size) { return 0;
88#define __sme_pa(x) (__pa(x) | sme_me_mask) 93#define __sme_pa(x) (__pa(x) | sme_me_mask)
89#define __sme_pa_nodebug(x) (__pa_nodebug(x) | sme_me_mask) 94#define __sme_pa_nodebug(x) (__pa_nodebug(x) | sme_me_mask)
90 95
96extern char __start_bss_decrypted[], __end_bss_decrypted[], __start_bss_decrypted_unused[];
97
91#endif /* __ASSEMBLY__ */ 98#endif /* __ASSEMBLY__ */
92 99
93#endif /* __X86_MEM_ENCRYPT_H__ */ 100#endif /* __X86_MEM_ENCRYPT_H__ */
diff --git a/arch/x86/include/asm/pgtable-2level.h b/arch/x86/include/asm/pgtable-2level.h
index 24c6cf5f16b7..60d0f9015317 100644
--- a/arch/x86/include/asm/pgtable-2level.h
+++ b/arch/x86/include/asm/pgtable-2level.h
@@ -19,9 +19,6 @@ static inline void native_set_pte(pte_t *ptep , pte_t pte)
19 19
20static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) 20static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
21{ 21{
22#ifdef CONFIG_PAGE_TABLE_ISOLATION
23 pmd.pud.p4d.pgd = pti_set_user_pgtbl(&pmdp->pud.p4d.pgd, pmd.pud.p4d.pgd);
24#endif
25 *pmdp = pmd; 22 *pmdp = pmd;
26} 23}
27 24
@@ -61,9 +58,6 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp)
61#ifdef CONFIG_SMP 58#ifdef CONFIG_SMP
62static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) 59static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)
63{ 60{
64#ifdef CONFIG_PAGE_TABLE_ISOLATION
65 pti_set_user_pgtbl(&xp->pud.p4d.pgd, __pgd(0));
66#endif
67 return __pmd(xchg((pmdval_t *)xp, 0)); 61 return __pmd(xchg((pmdval_t *)xp, 0));
68} 62}
69#else 63#else
@@ -73,9 +67,6 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)
73#ifdef CONFIG_SMP 67#ifdef CONFIG_SMP
74static inline pud_t native_pudp_get_and_clear(pud_t *xp) 68static inline pud_t native_pudp_get_and_clear(pud_t *xp)
75{ 69{
76#ifdef CONFIG_PAGE_TABLE_ISOLATION
77 pti_set_user_pgtbl(&xp->p4d.pgd, __pgd(0));
78#endif
79 return __pud(xchg((pudval_t *)xp, 0)); 70 return __pud(xchg((pudval_t *)xp, 0));
80} 71}
81#else 72#else
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index e4ffa565a69f..690c0307afed 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -1195,7 +1195,7 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
1195 return xchg(pmdp, pmd); 1195 return xchg(pmdp, pmd);
1196 } else { 1196 } else {
1197 pmd_t old = *pmdp; 1197 pmd_t old = *pmdp;
1198 *pmdp = pmd; 1198 WRITE_ONCE(*pmdp, pmd);
1199 return old; 1199 return old;
1200 } 1200 }
1201} 1201}
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index f773d5e6c8cc..9c85b54bf03c 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -14,6 +14,7 @@
14#include <asm/processor.h> 14#include <asm/processor.h>
15#include <linux/bitops.h> 15#include <linux/bitops.h>
16#include <linux/threads.h> 16#include <linux/threads.h>
17#include <asm/fixmap.h>
17 18
18extern p4d_t level4_kernel_pgt[512]; 19extern p4d_t level4_kernel_pgt[512];
19extern p4d_t level4_ident_pgt[512]; 20extern p4d_t level4_ident_pgt[512];
@@ -22,7 +23,7 @@ extern pud_t level3_ident_pgt[512];
22extern pmd_t level2_kernel_pgt[512]; 23extern pmd_t level2_kernel_pgt[512];
23extern pmd_t level2_fixmap_pgt[512]; 24extern pmd_t level2_fixmap_pgt[512];
24extern pmd_t level2_ident_pgt[512]; 25extern pmd_t level2_ident_pgt[512];
25extern pte_t level1_fixmap_pgt[512]; 26extern pte_t level1_fixmap_pgt[512 * FIXMAP_PMD_NUM];
26extern pgd_t init_top_pgt[]; 27extern pgd_t init_top_pgt[];
27 28
28#define swapper_pg_dir init_top_pgt 29#define swapper_pg_dir init_top_pgt
@@ -55,15 +56,15 @@ struct mm_struct;
55void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte); 56void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte);
56void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte); 57void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte);
57 58
58static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, 59static inline void native_set_pte(pte_t *ptep, pte_t pte)
59 pte_t *ptep)
60{ 60{
61 *ptep = native_make_pte(0); 61 WRITE_ONCE(*ptep, pte);
62} 62}
63 63
64static inline void native_set_pte(pte_t *ptep, pte_t pte) 64static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
65 pte_t *ptep)
65{ 66{
66 *ptep = pte; 67 native_set_pte(ptep, native_make_pte(0));
67} 68}
68 69
69static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) 70static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
@@ -73,7 +74,7 @@ static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
73 74
74static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) 75static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
75{ 76{
76 *pmdp = pmd; 77 WRITE_ONCE(*pmdp, pmd);
77} 78}
78 79
79static inline void native_pmd_clear(pmd_t *pmd) 80static inline void native_pmd_clear(pmd_t *pmd)
@@ -109,7 +110,7 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)
109 110
110static inline void native_set_pud(pud_t *pudp, pud_t pud) 111static inline void native_set_pud(pud_t *pudp, pud_t pud)
111{ 112{
112 *pudp = pud; 113 WRITE_ONCE(*pudp, pud);
113} 114}
114 115
115static inline void native_pud_clear(pud_t *pud) 116static inline void native_pud_clear(pud_t *pud)
@@ -137,13 +138,13 @@ static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d)
137 pgd_t pgd; 138 pgd_t pgd;
138 139
139 if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) { 140 if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) {
140 *p4dp = p4d; 141 WRITE_ONCE(*p4dp, p4d);
141 return; 142 return;
142 } 143 }
143 144
144 pgd = native_make_pgd(native_p4d_val(p4d)); 145 pgd = native_make_pgd(native_p4d_val(p4d));
145 pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); 146 pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd);
146 *p4dp = native_make_p4d(native_pgd_val(pgd)); 147 WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd)));
147} 148}
148 149
149static inline void native_p4d_clear(p4d_t *p4d) 150static inline void native_p4d_clear(p4d_t *p4d)
@@ -153,7 +154,7 @@ static inline void native_p4d_clear(p4d_t *p4d)
153 154
154static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) 155static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd)
155{ 156{
156 *pgdp = pti_set_user_pgtbl(pgdp, pgd); 157 WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd));
157} 158}
158 159
159static inline void native_pgd_clear(pgd_t *pgd) 160static inline void native_pgd_clear(pgd_t *pgd)
diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h
index 86299efa804a..fd23d5778ea1 100644
--- a/arch/x86/include/uapi/asm/kvm.h
+++ b/arch/x86/include/uapi/asm/kvm.h
@@ -377,6 +377,7 @@ struct kvm_sync_regs {
377 377
378#define KVM_X86_QUIRK_LINT0_REENABLED (1 << 0) 378#define KVM_X86_QUIRK_LINT0_REENABLED (1 << 0)
379#define KVM_X86_QUIRK_CD_NW_CLEARED (1 << 1) 379#define KVM_X86_QUIRK_CD_NW_CLEARED (1 << 1)
380#define KVM_X86_QUIRK_LAPIC_MMIO_HOLE (1 << 2)
380 381
381#define KVM_STATE_NESTED_GUEST_MODE 0x00000001 382#define KVM_STATE_NESTED_GUEST_MODE 0x00000001
382#define KVM_STATE_NESTED_RUN_PENDING 0x00000002 383#define KVM_STATE_NESTED_RUN_PENDING 0x00000002
diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c
index 9f148e3d45b4..7654febd5102 100644
--- a/arch/x86/kernel/apic/vector.c
+++ b/arch/x86/kernel/apic/vector.c
@@ -413,7 +413,7 @@ static int activate_managed(struct irq_data *irqd)
413 if (WARN_ON_ONCE(cpumask_empty(vector_searchmask))) { 413 if (WARN_ON_ONCE(cpumask_empty(vector_searchmask))) {
414 /* Something in the core code broke! Survive gracefully */ 414 /* Something in the core code broke! Survive gracefully */
415 pr_err("Managed startup for irq %u, but no CPU\n", irqd->irq); 415 pr_err("Managed startup for irq %u, but no CPU\n", irqd->irq);
416 return EINVAL; 416 return -EINVAL;
417 } 417 }
418 418
419 ret = assign_managed_vector(irqd, vector_searchmask); 419 ret = assign_managed_vector(irqd, vector_searchmask);
diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c
index ec00d1ff5098..f7151cd03cb0 100644
--- a/arch/x86/kernel/apm_32.c
+++ b/arch/x86/kernel/apm_32.c
@@ -1640,6 +1640,7 @@ static int do_open(struct inode *inode, struct file *filp)
1640 return 0; 1640 return 0;
1641} 1641}
1642 1642
1643#ifdef CONFIG_PROC_FS
1643static int proc_apm_show(struct seq_file *m, void *v) 1644static int proc_apm_show(struct seq_file *m, void *v)
1644{ 1645{
1645 unsigned short bx; 1646 unsigned short bx;
@@ -1719,6 +1720,7 @@ static int proc_apm_show(struct seq_file *m, void *v)
1719 units); 1720 units);
1720 return 0; 1721 return 0;
1721} 1722}
1723#endif
1722 1724
1723static int apm(void *unused) 1725static int apm(void *unused)
1724{ 1726{
diff --git a/arch/x86/kernel/cpu/intel_rdt.h b/arch/x86/kernel/cpu/intel_rdt.h
index 4e588f36228f..285eb3ec4200 100644
--- a/arch/x86/kernel/cpu/intel_rdt.h
+++ b/arch/x86/kernel/cpu/intel_rdt.h
@@ -382,6 +382,11 @@ static inline bool is_mbm_event(int e)
382 e <= QOS_L3_MBM_LOCAL_EVENT_ID); 382 e <= QOS_L3_MBM_LOCAL_EVENT_ID);
383} 383}
384 384
385struct rdt_parse_data {
386 struct rdtgroup *rdtgrp;
387 char *buf;
388};
389
385/** 390/**
386 * struct rdt_resource - attributes of an RDT resource 391 * struct rdt_resource - attributes of an RDT resource
387 * @rid: The index of the resource 392 * @rid: The index of the resource
@@ -423,16 +428,19 @@ struct rdt_resource {
423 struct rdt_cache cache; 428 struct rdt_cache cache;
424 struct rdt_membw membw; 429 struct rdt_membw membw;
425 const char *format_str; 430 const char *format_str;
426 int (*parse_ctrlval) (void *data, struct rdt_resource *r, 431 int (*parse_ctrlval)(struct rdt_parse_data *data,
427 struct rdt_domain *d); 432 struct rdt_resource *r,
433 struct rdt_domain *d);
428 struct list_head evt_list; 434 struct list_head evt_list;
429 int num_rmid; 435 int num_rmid;
430 unsigned int mon_scale; 436 unsigned int mon_scale;
431 unsigned long fflags; 437 unsigned long fflags;
432}; 438};
433 439
434int parse_cbm(void *_data, struct rdt_resource *r, struct rdt_domain *d); 440int parse_cbm(struct rdt_parse_data *data, struct rdt_resource *r,
435int parse_bw(void *_buf, struct rdt_resource *r, struct rdt_domain *d); 441 struct rdt_domain *d);
442int parse_bw(struct rdt_parse_data *data, struct rdt_resource *r,
443 struct rdt_domain *d);
436 444
437extern struct mutex rdtgroup_mutex; 445extern struct mutex rdtgroup_mutex;
438 446
@@ -536,6 +544,7 @@ int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp);
536void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp); 544void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp);
537struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r); 545struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r);
538int update_domains(struct rdt_resource *r, int closid); 546int update_domains(struct rdt_resource *r, int closid);
547int closids_supported(void);
539void closid_free(int closid); 548void closid_free(int closid);
540int alloc_rmid(void); 549int alloc_rmid(void);
541void free_rmid(u32 rmid); 550void free_rmid(u32 rmid);
diff --git a/arch/x86/kernel/cpu/intel_rdt_ctrlmondata.c b/arch/x86/kernel/cpu/intel_rdt_ctrlmondata.c
index af358ca05160..0f53049719cd 100644
--- a/arch/x86/kernel/cpu/intel_rdt_ctrlmondata.c
+++ b/arch/x86/kernel/cpu/intel_rdt_ctrlmondata.c
@@ -64,19 +64,19 @@ static bool bw_validate(char *buf, unsigned long *data, struct rdt_resource *r)
64 return true; 64 return true;
65} 65}
66 66
67int parse_bw(void *_buf, struct rdt_resource *r, struct rdt_domain *d) 67int parse_bw(struct rdt_parse_data *data, struct rdt_resource *r,
68 struct rdt_domain *d)
68{ 69{
69 unsigned long data; 70 unsigned long bw_val;
70 char *buf = _buf;
71 71
72 if (d->have_new_ctrl) { 72 if (d->have_new_ctrl) {
73 rdt_last_cmd_printf("duplicate domain %d\n", d->id); 73 rdt_last_cmd_printf("duplicate domain %d\n", d->id);
74 return -EINVAL; 74 return -EINVAL;
75 } 75 }
76 76
77 if (!bw_validate(buf, &data, r)) 77 if (!bw_validate(data->buf, &bw_val, r))
78 return -EINVAL; 78 return -EINVAL;
79 d->new_ctrl = data; 79 d->new_ctrl = bw_val;
80 d->have_new_ctrl = true; 80 d->have_new_ctrl = true;
81 81
82 return 0; 82 return 0;
@@ -123,18 +123,13 @@ static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r)
123 return true; 123 return true;
124} 124}
125 125
126struct rdt_cbm_parse_data {
127 struct rdtgroup *rdtgrp;
128 char *buf;
129};
130
131/* 126/*
132 * Read one cache bit mask (hex). Check that it is valid for the current 127 * Read one cache bit mask (hex). Check that it is valid for the current
133 * resource type. 128 * resource type.
134 */ 129 */
135int parse_cbm(void *_data, struct rdt_resource *r, struct rdt_domain *d) 130int parse_cbm(struct rdt_parse_data *data, struct rdt_resource *r,
131 struct rdt_domain *d)
136{ 132{
137 struct rdt_cbm_parse_data *data = _data;
138 struct rdtgroup *rdtgrp = data->rdtgrp; 133 struct rdtgroup *rdtgrp = data->rdtgrp;
139 u32 cbm_val; 134 u32 cbm_val;
140 135
@@ -195,11 +190,17 @@ int parse_cbm(void *_data, struct rdt_resource *r, struct rdt_domain *d)
195static int parse_line(char *line, struct rdt_resource *r, 190static int parse_line(char *line, struct rdt_resource *r,
196 struct rdtgroup *rdtgrp) 191 struct rdtgroup *rdtgrp)
197{ 192{
198 struct rdt_cbm_parse_data data; 193 struct rdt_parse_data data;
199 char *dom = NULL, *id; 194 char *dom = NULL, *id;
200 struct rdt_domain *d; 195 struct rdt_domain *d;
201 unsigned long dom_id; 196 unsigned long dom_id;
202 197
198 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP &&
199 r->rid == RDT_RESOURCE_MBA) {
200 rdt_last_cmd_puts("Cannot pseudo-lock MBA resource\n");
201 return -EINVAL;
202 }
203
203next: 204next:
204 if (!line || line[0] == '\0') 205 if (!line || line[0] == '\0')
205 return 0; 206 return 0;
diff --git a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
index b799c00bef09..1b8e86a5d5e1 100644
--- a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
+++ b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
@@ -97,6 +97,12 @@ void rdt_last_cmd_printf(const char *fmt, ...)
97 * limited as the number of resources grows. 97 * limited as the number of resources grows.
98 */ 98 */
99static int closid_free_map; 99static int closid_free_map;
100static int closid_free_map_len;
101
102int closids_supported(void)
103{
104 return closid_free_map_len;
105}
100 106
101static void closid_init(void) 107static void closid_init(void)
102{ 108{
@@ -111,6 +117,7 @@ static void closid_init(void)
111 117
112 /* CLOSID 0 is always reserved for the default group */ 118 /* CLOSID 0 is always reserved for the default group */
113 closid_free_map &= ~1; 119 closid_free_map &= ~1;
120 closid_free_map_len = rdt_min_closid;
114} 121}
115 122
116static int closid_alloc(void) 123static int closid_alloc(void)
@@ -802,7 +809,7 @@ static int rdt_bit_usage_show(struct kernfs_open_file *of,
802 sw_shareable = 0; 809 sw_shareable = 0;
803 exclusive = 0; 810 exclusive = 0;
804 seq_printf(seq, "%d=", dom->id); 811 seq_printf(seq, "%d=", dom->id);
805 for (i = 0; i < r->num_closid; i++, ctrl++) { 812 for (i = 0; i < closids_supported(); i++, ctrl++) {
806 if (!closid_allocated(i)) 813 if (!closid_allocated(i))
807 continue; 814 continue;
808 mode = rdtgroup_mode_by_closid(i); 815 mode = rdtgroup_mode_by_closid(i);
@@ -989,7 +996,7 @@ bool rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_domain *d,
989 996
990 /* Check for overlap with other resource groups */ 997 /* Check for overlap with other resource groups */
991 ctrl = d->ctrl_val; 998 ctrl = d->ctrl_val;
992 for (i = 0; i < r->num_closid; i++, ctrl++) { 999 for (i = 0; i < closids_supported(); i++, ctrl++) {
993 ctrl_b = (unsigned long *)ctrl; 1000 ctrl_b = (unsigned long *)ctrl;
994 mode = rdtgroup_mode_by_closid(i); 1001 mode = rdtgroup_mode_by_closid(i);
995 if (closid_allocated(i) && i != closid && 1002 if (closid_allocated(i) && i != closid &&
@@ -1024,16 +1031,27 @@ static bool rdtgroup_mode_test_exclusive(struct rdtgroup *rdtgrp)
1024{ 1031{
1025 int closid = rdtgrp->closid; 1032 int closid = rdtgrp->closid;
1026 struct rdt_resource *r; 1033 struct rdt_resource *r;
1034 bool has_cache = false;
1027 struct rdt_domain *d; 1035 struct rdt_domain *d;
1028 1036
1029 for_each_alloc_enabled_rdt_resource(r) { 1037 for_each_alloc_enabled_rdt_resource(r) {
1038 if (r->rid == RDT_RESOURCE_MBA)
1039 continue;
1040 has_cache = true;
1030 list_for_each_entry(d, &r->domains, list) { 1041 list_for_each_entry(d, &r->domains, list) {
1031 if (rdtgroup_cbm_overlaps(r, d, d->ctrl_val[closid], 1042 if (rdtgroup_cbm_overlaps(r, d, d->ctrl_val[closid],
1032 rdtgrp->closid, false)) 1043 rdtgrp->closid, false)) {
1044 rdt_last_cmd_puts("schemata overlaps\n");
1033 return false; 1045 return false;
1046 }
1034 } 1047 }
1035 } 1048 }
1036 1049
1050 if (!has_cache) {
1051 rdt_last_cmd_puts("cannot be exclusive without CAT/CDP\n");
1052 return false;
1053 }
1054
1037 return true; 1055 return true;
1038} 1056}
1039 1057
@@ -1085,7 +1103,6 @@ static ssize_t rdtgroup_mode_write(struct kernfs_open_file *of,
1085 rdtgrp->mode = RDT_MODE_SHAREABLE; 1103 rdtgrp->mode = RDT_MODE_SHAREABLE;
1086 } else if (!strcmp(buf, "exclusive")) { 1104 } else if (!strcmp(buf, "exclusive")) {
1087 if (!rdtgroup_mode_test_exclusive(rdtgrp)) { 1105 if (!rdtgroup_mode_test_exclusive(rdtgrp)) {
1088 rdt_last_cmd_printf("schemata overlaps\n");
1089 ret = -EINVAL; 1106 ret = -EINVAL;
1090 goto out; 1107 goto out;
1091 } 1108 }
@@ -1155,8 +1172,8 @@ static int rdtgroup_size_show(struct kernfs_open_file *of,
1155 struct rdt_resource *r; 1172 struct rdt_resource *r;
1156 struct rdt_domain *d; 1173 struct rdt_domain *d;
1157 unsigned int size; 1174 unsigned int size;
1158 bool sep = false; 1175 bool sep;
1159 u32 cbm; 1176 u32 ctrl;
1160 1177
1161 rdtgrp = rdtgroup_kn_lock_live(of->kn); 1178 rdtgrp = rdtgroup_kn_lock_live(of->kn);
1162 if (!rdtgrp) { 1179 if (!rdtgrp) {
@@ -1174,6 +1191,7 @@ static int rdtgroup_size_show(struct kernfs_open_file *of,
1174 } 1191 }
1175 1192
1176 for_each_alloc_enabled_rdt_resource(r) { 1193 for_each_alloc_enabled_rdt_resource(r) {
1194 sep = false;
1177 seq_printf(s, "%*s:", max_name_width, r->name); 1195 seq_printf(s, "%*s:", max_name_width, r->name);
1178 list_for_each_entry(d, &r->domains, list) { 1196 list_for_each_entry(d, &r->domains, list) {
1179 if (sep) 1197 if (sep)
@@ -1181,8 +1199,13 @@ static int rdtgroup_size_show(struct kernfs_open_file *of,
1181 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { 1199 if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
1182 size = 0; 1200 size = 0;
1183 } else { 1201 } else {
1184 cbm = d->ctrl_val[rdtgrp->closid]; 1202 ctrl = (!is_mba_sc(r) ?
1185 size = rdtgroup_cbm_to_size(r, d, cbm); 1203 d->ctrl_val[rdtgrp->closid] :
1204 d->mbps_val[rdtgrp->closid]);
1205 if (r->rid == RDT_RESOURCE_MBA)
1206 size = ctrl;
1207 else
1208 size = rdtgroup_cbm_to_size(r, d, ctrl);
1186 } 1209 }
1187 seq_printf(s, "%d=%u", d->id, size); 1210 seq_printf(s, "%d=%u", d->id, size);
1188 sep = true; 1211 sep = true;
@@ -2336,12 +2359,18 @@ static int rdtgroup_init_alloc(struct rdtgroup *rdtgrp)
2336 u32 *ctrl; 2359 u32 *ctrl;
2337 2360
2338 for_each_alloc_enabled_rdt_resource(r) { 2361 for_each_alloc_enabled_rdt_resource(r) {
2362 /*
2363 * Only initialize default allocations for CBM cache
2364 * resources
2365 */
2366 if (r->rid == RDT_RESOURCE_MBA)
2367 continue;
2339 list_for_each_entry(d, &r->domains, list) { 2368 list_for_each_entry(d, &r->domains, list) {
2340 d->have_new_ctrl = false; 2369 d->have_new_ctrl = false;
2341 d->new_ctrl = r->cache.shareable_bits; 2370 d->new_ctrl = r->cache.shareable_bits;
2342 used_b = r->cache.shareable_bits; 2371 used_b = r->cache.shareable_bits;
2343 ctrl = d->ctrl_val; 2372 ctrl = d->ctrl_val;
2344 for (i = 0; i < r->num_closid; i++, ctrl++) { 2373 for (i = 0; i < closids_supported(); i++, ctrl++) {
2345 if (closid_allocated(i) && i != closid) { 2374 if (closid_allocated(i) && i != closid) {
2346 mode = rdtgroup_mode_by_closid(i); 2375 mode = rdtgroup_mode_by_closid(i);
2347 if (mode == RDT_MODE_PSEUDO_LOCKSETUP) 2376 if (mode == RDT_MODE_PSEUDO_LOCKSETUP)
@@ -2373,6 +2402,12 @@ static int rdtgroup_init_alloc(struct rdtgroup *rdtgrp)
2373 } 2402 }
2374 2403
2375 for_each_alloc_enabled_rdt_resource(r) { 2404 for_each_alloc_enabled_rdt_resource(r) {
2405 /*
2406 * Only initialize default allocations for CBM cache
2407 * resources
2408 */
2409 if (r->rid == RDT_RESOURCE_MBA)
2410 continue;
2376 ret = update_domains(r, rdtgrp->closid); 2411 ret = update_domains(r, rdtgrp->closid);
2377 if (ret < 0) { 2412 if (ret < 0) {
2378 rdt_last_cmd_puts("failed to initialize allocations\n"); 2413 rdt_last_cmd_puts("failed to initialize allocations\n");
diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c
index 0624957aa068..07b5fc00b188 100644
--- a/arch/x86/kernel/cpu/microcode/amd.c
+++ b/arch/x86/kernel/cpu/microcode/amd.c
@@ -504,6 +504,7 @@ static enum ucode_state apply_microcode_amd(int cpu)
504 struct microcode_amd *mc_amd; 504 struct microcode_amd *mc_amd;
505 struct ucode_cpu_info *uci; 505 struct ucode_cpu_info *uci;
506 struct ucode_patch *p; 506 struct ucode_patch *p;
507 enum ucode_state ret;
507 u32 rev, dummy; 508 u32 rev, dummy;
508 509
509 BUG_ON(raw_smp_processor_id() != cpu); 510 BUG_ON(raw_smp_processor_id() != cpu);
@@ -521,9 +522,8 @@ static enum ucode_state apply_microcode_amd(int cpu)
521 522
522 /* need to apply patch? */ 523 /* need to apply patch? */
523 if (rev >= mc_amd->hdr.patch_id) { 524 if (rev >= mc_amd->hdr.patch_id) {
524 c->microcode = rev; 525 ret = UCODE_OK;
525 uci->cpu_sig.rev = rev; 526 goto out;
526 return UCODE_OK;
527 } 527 }
528 528
529 if (__apply_microcode_amd(mc_amd)) { 529 if (__apply_microcode_amd(mc_amd)) {
@@ -531,13 +531,21 @@ static enum ucode_state apply_microcode_amd(int cpu)
531 cpu, mc_amd->hdr.patch_id); 531 cpu, mc_amd->hdr.patch_id);
532 return UCODE_ERROR; 532 return UCODE_ERROR;
533 } 533 }
534 pr_info("CPU%d: new patch_level=0x%08x\n", cpu,
535 mc_amd->hdr.patch_id);
536 534
537 uci->cpu_sig.rev = mc_amd->hdr.patch_id; 535 rev = mc_amd->hdr.patch_id;
538 c->microcode = mc_amd->hdr.patch_id; 536 ret = UCODE_UPDATED;
537
538 pr_info("CPU%d: new patch_level=0x%08x\n", cpu, rev);
539 539
540 return UCODE_UPDATED; 540out:
541 uci->cpu_sig.rev = rev;
542 c->microcode = rev;
543
544 /* Update boot_cpu_data's revision too, if we're on the BSP: */
545 if (c->cpu_index == boot_cpu_data.cpu_index)
546 boot_cpu_data.microcode = rev;
547
548 return ret;
541} 549}
542 550
543static int install_equiv_cpu_table(const u8 *buf) 551static int install_equiv_cpu_table(const u8 *buf)
diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c
index 97ccf4c3b45b..16936a24795c 100644
--- a/arch/x86/kernel/cpu/microcode/intel.c
+++ b/arch/x86/kernel/cpu/microcode/intel.c
@@ -795,6 +795,7 @@ static enum ucode_state apply_microcode_intel(int cpu)
795 struct ucode_cpu_info *uci = ucode_cpu_info + cpu; 795 struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
796 struct cpuinfo_x86 *c = &cpu_data(cpu); 796 struct cpuinfo_x86 *c = &cpu_data(cpu);
797 struct microcode_intel *mc; 797 struct microcode_intel *mc;
798 enum ucode_state ret;
798 static int prev_rev; 799 static int prev_rev;
799 u32 rev; 800 u32 rev;
800 801
@@ -817,9 +818,8 @@ static enum ucode_state apply_microcode_intel(int cpu)
817 */ 818 */
818 rev = intel_get_microcode_revision(); 819 rev = intel_get_microcode_revision();
819 if (rev >= mc->hdr.rev) { 820 if (rev >= mc->hdr.rev) {
820 uci->cpu_sig.rev = rev; 821 ret = UCODE_OK;
821 c->microcode = rev; 822 goto out;
822 return UCODE_OK;
823 } 823 }
824 824
825 /* 825 /*
@@ -848,10 +848,17 @@ static enum ucode_state apply_microcode_intel(int cpu)
848 prev_rev = rev; 848 prev_rev = rev;
849 } 849 }
850 850
851 ret = UCODE_UPDATED;
852
853out:
851 uci->cpu_sig.rev = rev; 854 uci->cpu_sig.rev = rev;
852 c->microcode = rev; 855 c->microcode = rev;
856
857 /* Update boot_cpu_data's revision too, if we're on the BSP: */
858 if (c->cpu_index == boot_cpu_data.cpu_index)
859 boot_cpu_data.microcode = rev;
853 860
854 return UCODE_UPDATED; 861 return ret;
855} 862}
856 863
857static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size, 864static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index f56895106ccf..2b5886401e5f 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -146,7 +146,7 @@ static void show_regs_if_on_stack(struct stack_info *info, struct pt_regs *regs,
146 * they can be printed in the right context. 146 * they can be printed in the right context.
147 */ 147 */
148 if (!partial && on_stack(info, regs, sizeof(*regs))) { 148 if (!partial && on_stack(info, regs, sizeof(*regs))) {
149 __show_regs(regs, 0); 149 __show_regs(regs, SHOW_REGS_SHORT);
150 150
151 } else if (partial && on_stack(info, (void *)regs + IRET_FRAME_OFFSET, 151 } else if (partial && on_stack(info, (void *)regs + IRET_FRAME_OFFSET,
152 IRET_FRAME_SIZE)) { 152 IRET_FRAME_SIZE)) {
@@ -344,7 +344,7 @@ void oops_end(unsigned long flags, struct pt_regs *regs, int signr)
344 oops_exit(); 344 oops_exit();
345 345
346 /* Executive summary in case the oops scrolled away */ 346 /* Executive summary in case the oops scrolled away */
347 __show_regs(&exec_summary_regs, true); 347 __show_regs(&exec_summary_regs, SHOW_REGS_ALL);
348 348
349 if (!signr) 349 if (!signr)
350 return; 350 return;
@@ -407,14 +407,9 @@ void die(const char *str, struct pt_regs *regs, long err)
407 407
408void show_regs(struct pt_regs *regs) 408void show_regs(struct pt_regs *regs)
409{ 409{
410 bool all = true;
411
412 show_regs_print_info(KERN_DEFAULT); 410 show_regs_print_info(KERN_DEFAULT);
413 411
414 if (IS_ENABLED(CONFIG_X86_32)) 412 __show_regs(regs, user_mode(regs) ? SHOW_REGS_USER : SHOW_REGS_ALL);
415 all = !user_mode(regs);
416
417 __show_regs(regs, all);
418 413
419 /* 414 /*
420 * When in-kernel, we also print out the stack at the time of the fault.. 415 * When in-kernel, we also print out the stack at the time of the fault..
diff --git a/arch/x86/kernel/eisa.c b/arch/x86/kernel/eisa.c
index f260e452e4f8..e8c8c5d78dbd 100644
--- a/arch/x86/kernel/eisa.c
+++ b/arch/x86/kernel/eisa.c
@@ -7,11 +7,17 @@
7#include <linux/eisa.h> 7#include <linux/eisa.h>
8#include <linux/io.h> 8#include <linux/io.h>
9 9
10#include <xen/xen.h>
11
10static __init int eisa_bus_probe(void) 12static __init int eisa_bus_probe(void)
11{ 13{
12 void __iomem *p = ioremap(0x0FFFD9, 4); 14 void __iomem *p;
15
16 if (xen_pv_domain() && !xen_initial_domain())
17 return 0;
13 18
14 if (readl(p) == 'E' + ('I'<<8) + ('S'<<16) + ('A'<<24)) 19 p = ioremap(0x0FFFD9, 4);
20 if (p && readl(p) == 'E' + ('I' << 8) + ('S' << 16) + ('A' << 24))
15 EISA_bus = 1; 21 EISA_bus = 1;
16 iounmap(p); 22 iounmap(p);
17 return 0; 23 return 0;
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 8047379e575a..ddee1f0870c4 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -35,6 +35,7 @@
35#include <asm/bootparam_utils.h> 35#include <asm/bootparam_utils.h>
36#include <asm/microcode.h> 36#include <asm/microcode.h>
37#include <asm/kasan.h> 37#include <asm/kasan.h>
38#include <asm/fixmap.h>
38 39
39/* 40/*
40 * Manage page tables very early on. 41 * Manage page tables very early on.
@@ -112,6 +113,7 @@ static bool __head check_la57_support(unsigned long physaddr)
112unsigned long __head __startup_64(unsigned long physaddr, 113unsigned long __head __startup_64(unsigned long physaddr,
113 struct boot_params *bp) 114 struct boot_params *bp)
114{ 115{
116 unsigned long vaddr, vaddr_end;
115 unsigned long load_delta, *p; 117 unsigned long load_delta, *p;
116 unsigned long pgtable_flags; 118 unsigned long pgtable_flags;
117 pgdval_t *pgd; 119 pgdval_t *pgd;
@@ -165,7 +167,8 @@ unsigned long __head __startup_64(unsigned long physaddr,
165 pud[511] += load_delta; 167 pud[511] += load_delta;
166 168
167 pmd = fixup_pointer(level2_fixmap_pgt, physaddr); 169 pmd = fixup_pointer(level2_fixmap_pgt, physaddr);
168 pmd[506] += load_delta; 170 for (i = FIXMAP_PMD_TOP; i > FIXMAP_PMD_TOP - FIXMAP_PMD_NUM; i--)
171 pmd[i] += load_delta;
169 172
170 /* 173 /*
171 * Set up the identity mapping for the switchover. These 174 * Set up the identity mapping for the switchover. These
@@ -235,6 +238,21 @@ unsigned long __head __startup_64(unsigned long physaddr,
235 sme_encrypt_kernel(bp); 238 sme_encrypt_kernel(bp);
236 239
237 /* 240 /*
241 * Clear the memory encryption mask from the .bss..decrypted section.
242 * The bss section will be memset to zero later in the initialization so
243 * there is no need to zero it after changing the memory encryption
244 * attribute.
245 */
246 if (mem_encrypt_active()) {
247 vaddr = (unsigned long)__start_bss_decrypted;
248 vaddr_end = (unsigned long)__end_bss_decrypted;
249 for (; vaddr < vaddr_end; vaddr += PMD_SIZE) {
250 i = pmd_index(vaddr);
251 pmd[i] -= sme_get_me_mask();
252 }
253 }
254
255 /*
238 * Return the SME encryption mask (if SME is active) to be used as a 256 * Return the SME encryption mask (if SME is active) to be used as a
239 * modifier for the initial pgdir entry programmed into CR3. 257 * modifier for the initial pgdir entry programmed into CR3.
240 */ 258 */
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 15ebc2fc166e..a3618cf04cf6 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -24,6 +24,7 @@
24#include "../entry/calling.h" 24#include "../entry/calling.h"
25#include <asm/export.h> 25#include <asm/export.h>
26#include <asm/nospec-branch.h> 26#include <asm/nospec-branch.h>
27#include <asm/fixmap.h>
27 28
28#ifdef CONFIG_PARAVIRT 29#ifdef CONFIG_PARAVIRT
29#include <asm/asm-offsets.h> 30#include <asm/asm-offsets.h>
@@ -445,13 +446,20 @@ NEXT_PAGE(level2_kernel_pgt)
445 KERNEL_IMAGE_SIZE/PMD_SIZE) 446 KERNEL_IMAGE_SIZE/PMD_SIZE)
446 447
447NEXT_PAGE(level2_fixmap_pgt) 448NEXT_PAGE(level2_fixmap_pgt)
448 .fill 506,8,0 449 .fill (512 - 4 - FIXMAP_PMD_NUM),8,0
449 .quad level1_fixmap_pgt - __START_KERNEL_map + _PAGE_TABLE_NOENC 450 pgtno = 0
450 /* 8MB reserved for vsyscalls + a 2MB hole = 4 + 1 entries */ 451 .rept (FIXMAP_PMD_NUM)
451 .fill 5,8,0 452 .quad level1_fixmap_pgt + (pgtno << PAGE_SHIFT) - __START_KERNEL_map \
453 + _PAGE_TABLE_NOENC;
454 pgtno = pgtno + 1
455 .endr
456 /* 6 MB reserved space + a 2MB hole */
457 .fill 4,8,0
452 458
453NEXT_PAGE(level1_fixmap_pgt) 459NEXT_PAGE(level1_fixmap_pgt)
460 .rept (FIXMAP_PMD_NUM)
454 .fill 512,8,0 461 .fill 512,8,0
462 .endr
455 463
456#undef PMDS 464#undef PMDS
457 465
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index 1e6764648af3..013fe3d21dbb 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -28,6 +28,7 @@
28#include <linux/sched/clock.h> 28#include <linux/sched/clock.h>
29#include <linux/mm.h> 29#include <linux/mm.h>
30#include <linux/slab.h> 30#include <linux/slab.h>
31#include <linux/set_memory.h>
31 32
32#include <asm/hypervisor.h> 33#include <asm/hypervisor.h>
33#include <asm/mem_encrypt.h> 34#include <asm/mem_encrypt.h>
@@ -61,9 +62,10 @@ early_param("no-kvmclock-vsyscall", parse_no_kvmclock_vsyscall);
61 (PAGE_SIZE / sizeof(struct pvclock_vsyscall_time_info)) 62 (PAGE_SIZE / sizeof(struct pvclock_vsyscall_time_info))
62 63
63static struct pvclock_vsyscall_time_info 64static struct pvclock_vsyscall_time_info
64 hv_clock_boot[HVC_BOOT_ARRAY_SIZE] __aligned(PAGE_SIZE); 65 hv_clock_boot[HVC_BOOT_ARRAY_SIZE] __bss_decrypted __aligned(PAGE_SIZE);
65static struct pvclock_wall_clock wall_clock; 66static struct pvclock_wall_clock wall_clock __bss_decrypted;
66static DEFINE_PER_CPU(struct pvclock_vsyscall_time_info *, hv_clock_per_cpu); 67static DEFINE_PER_CPU(struct pvclock_vsyscall_time_info *, hv_clock_per_cpu);
68static struct pvclock_vsyscall_time_info *hvclock_mem;
67 69
68static inline struct pvclock_vcpu_time_info *this_cpu_pvti(void) 70static inline struct pvclock_vcpu_time_info *this_cpu_pvti(void)
69{ 71{
@@ -236,6 +238,45 @@ static void kvm_shutdown(void)
236 native_machine_shutdown(); 238 native_machine_shutdown();
237} 239}
238 240
241static void __init kvmclock_init_mem(void)
242{
243 unsigned long ncpus;
244 unsigned int order;
245 struct page *p;
246 int r;
247
248 if (HVC_BOOT_ARRAY_SIZE >= num_possible_cpus())
249 return;
250
251 ncpus = num_possible_cpus() - HVC_BOOT_ARRAY_SIZE;
252 order = get_order(ncpus * sizeof(*hvclock_mem));
253
254 p = alloc_pages(GFP_KERNEL, order);
255 if (!p) {
256 pr_warn("%s: failed to alloc %d pages", __func__, (1U << order));
257 return;
258 }
259
260 hvclock_mem = page_address(p);
261
262 /*
263 * hvclock is shared between the guest and the hypervisor, must
264 * be mapped decrypted.
265 */
266 if (sev_active()) {
267 r = set_memory_decrypted((unsigned long) hvclock_mem,
268 1UL << order);
269 if (r) {
270 __free_pages(p, order);
271 hvclock_mem = NULL;
272 pr_warn("kvmclock: set_memory_decrypted() failed. Disabling\n");
273 return;
274 }
275 }
276
277 memset(hvclock_mem, 0, PAGE_SIZE << order);
278}
279
239static int __init kvm_setup_vsyscall_timeinfo(void) 280static int __init kvm_setup_vsyscall_timeinfo(void)
240{ 281{
241#ifdef CONFIG_X86_64 282#ifdef CONFIG_X86_64
@@ -250,6 +291,9 @@ static int __init kvm_setup_vsyscall_timeinfo(void)
250 291
251 kvm_clock.archdata.vclock_mode = VCLOCK_PVCLOCK; 292 kvm_clock.archdata.vclock_mode = VCLOCK_PVCLOCK;
252#endif 293#endif
294
295 kvmclock_init_mem();
296
253 return 0; 297 return 0;
254} 298}
255early_initcall(kvm_setup_vsyscall_timeinfo); 299early_initcall(kvm_setup_vsyscall_timeinfo);
@@ -269,8 +313,10 @@ static int kvmclock_setup_percpu(unsigned int cpu)
269 /* Use the static page for the first CPUs, allocate otherwise */ 313 /* Use the static page for the first CPUs, allocate otherwise */
270 if (cpu < HVC_BOOT_ARRAY_SIZE) 314 if (cpu < HVC_BOOT_ARRAY_SIZE)
271 p = &hv_clock_boot[cpu]; 315 p = &hv_clock_boot[cpu];
316 else if (hvclock_mem)
317 p = hvclock_mem + cpu - HVC_BOOT_ARRAY_SIZE;
272 else 318 else
273 p = kzalloc(sizeof(*p), GFP_KERNEL); 319 return -ENOMEM;
274 320
275 per_cpu(hv_clock_per_cpu, cpu) = p; 321 per_cpu(hv_clock_per_cpu, cpu) = p;
276 return p ? 0 : -ENOMEM; 322 return p ? 0 : -ENOMEM;
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index afdb303285f8..8dc69d82567e 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -91,7 +91,7 @@ unsigned paravirt_patch_call(void *insnbuf,
91 91
92 if (len < 5) { 92 if (len < 5) {
93#ifdef CONFIG_RETPOLINE 93#ifdef CONFIG_RETPOLINE
94 WARN_ONCE("Failing to patch indirect CALL in %ps\n", (void *)addr); 94 WARN_ONCE(1, "Failing to patch indirect CALL in %ps\n", (void *)addr);
95#endif 95#endif
96 return len; /* call too long for patch site */ 96 return len; /* call too long for patch site */
97 } 97 }
@@ -111,7 +111,7 @@ unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
111 111
112 if (len < 5) { 112 if (len < 5) {
113#ifdef CONFIG_RETPOLINE 113#ifdef CONFIG_RETPOLINE
114 WARN_ONCE("Failing to patch indirect JMP in %ps\n", (void *)addr); 114 WARN_ONCE(1, "Failing to patch indirect JMP in %ps\n", (void *)addr);
115#endif 115#endif
116 return len; /* call too long for patch site */ 116 return len; /* call too long for patch site */
117 } 117 }
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 2924fd447e61..5046a3c9dec2 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -59,7 +59,7 @@
59#include <asm/intel_rdt_sched.h> 59#include <asm/intel_rdt_sched.h>
60#include <asm/proto.h> 60#include <asm/proto.h>
61 61
62void __show_regs(struct pt_regs *regs, int all) 62void __show_regs(struct pt_regs *regs, enum show_regs_mode mode)
63{ 63{
64 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L; 64 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L;
65 unsigned long d0, d1, d2, d3, d6, d7; 65 unsigned long d0, d1, d2, d3, d6, d7;
@@ -85,7 +85,7 @@ void __show_regs(struct pt_regs *regs, int all)
85 printk(KERN_DEFAULT "DS: %04x ES: %04x FS: %04x GS: %04x SS: %04x EFLAGS: %08lx\n", 85 printk(KERN_DEFAULT "DS: %04x ES: %04x FS: %04x GS: %04x SS: %04x EFLAGS: %08lx\n",
86 (u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, ss, regs->flags); 86 (u16)regs->ds, (u16)regs->es, (u16)regs->fs, gs, ss, regs->flags);
87 87
88 if (!all) 88 if (mode != SHOW_REGS_ALL)
89 return; 89 return;
90 90
91 cr0 = read_cr0(); 91 cr0 = read_cr0();
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index a451bc374b9b..ea5ea850348d 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -62,7 +62,7 @@
62__visible DEFINE_PER_CPU(unsigned long, rsp_scratch); 62__visible DEFINE_PER_CPU(unsigned long, rsp_scratch);
63 63
64/* Prints also some state that isn't saved in the pt_regs */ 64/* Prints also some state that isn't saved in the pt_regs */
65void __show_regs(struct pt_regs *regs, int all) 65void __show_regs(struct pt_regs *regs, enum show_regs_mode mode)
66{ 66{
67 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs; 67 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
68 unsigned long d0, d1, d2, d3, d6, d7; 68 unsigned long d0, d1, d2, d3, d6, d7;
@@ -87,9 +87,17 @@ void __show_regs(struct pt_regs *regs, int all)
87 printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n", 87 printk(KERN_DEFAULT "R13: %016lx R14: %016lx R15: %016lx\n",
88 regs->r13, regs->r14, regs->r15); 88 regs->r13, regs->r14, regs->r15);
89 89
90 if (!all) 90 if (mode == SHOW_REGS_SHORT)
91 return; 91 return;
92 92
93 if (mode == SHOW_REGS_USER) {
94 rdmsrl(MSR_FS_BASE, fs);
95 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
96 printk(KERN_DEFAULT "FS: %016lx GS: %016lx\n",
97 fs, shadowgs);
98 return;
99 }
100
93 asm("movl %%ds,%0" : "=r" (ds)); 101 asm("movl %%ds,%0" : "=r" (ds));
94 asm("movl %%cs,%0" : "=r" (cs)); 102 asm("movl %%cs,%0" : "=r" (cs));
95 asm("movl %%es,%0" : "=r" (es)); 103 asm("movl %%es,%0" : "=r" (es));
diff --git a/arch/x86/kernel/topology.c b/arch/x86/kernel/topology.c
index 12cbe2b88c0f..738bf42b0218 100644
--- a/arch/x86/kernel/topology.c
+++ b/arch/x86/kernel/topology.c
@@ -111,8 +111,10 @@ int arch_register_cpu(int num)
111 /* 111 /*
112 * Currently CPU0 is only hotpluggable on Intel platforms. Other 112 * Currently CPU0 is only hotpluggable on Intel platforms. Other
113 * vendors can add hotplug support later. 113 * vendors can add hotplug support later.
114 * Xen PV guests don't support CPU0 hotplug at all.
114 */ 115 */
115 if (c->x86_vendor != X86_VENDOR_INTEL) 116 if (c->x86_vendor != X86_VENDOR_INTEL ||
117 boot_cpu_has(X86_FEATURE_XENPV))
116 cpu0_hotpluggable = 0; 118 cpu0_hotpluggable = 0;
117 119
118 /* 120 /*
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 1463468ba9a0..6490f618e096 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -1415,7 +1415,7 @@ static bool __init determine_cpu_tsc_frequencies(bool early)
1415 1415
1416static unsigned long __init get_loops_per_jiffy(void) 1416static unsigned long __init get_loops_per_jiffy(void)
1417{ 1417{
1418 unsigned long lpj = tsc_khz * KHZ; 1418 u64 lpj = (u64)tsc_khz * KHZ;
1419 1419
1420 do_div(lpj, HZ); 1420 do_div(lpj, HZ);
1421 return lpj; 1421 return lpj;
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 8bde0a419f86..5dd3317d761f 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -65,6 +65,23 @@ jiffies_64 = jiffies;
65#define ALIGN_ENTRY_TEXT_BEGIN . = ALIGN(PMD_SIZE); 65#define ALIGN_ENTRY_TEXT_BEGIN . = ALIGN(PMD_SIZE);
66#define ALIGN_ENTRY_TEXT_END . = ALIGN(PMD_SIZE); 66#define ALIGN_ENTRY_TEXT_END . = ALIGN(PMD_SIZE);
67 67
68/*
69 * This section contains data which will be mapped as decrypted. Memory
70 * encryption operates on a page basis. Make this section PMD-aligned
71 * to avoid splitting the pages while mapping the section early.
72 *
73 * Note: We use a separate section so that only this section gets
74 * decrypted to avoid exposing more than we wish.
75 */
76#define BSS_DECRYPTED \
77 . = ALIGN(PMD_SIZE); \
78 __start_bss_decrypted = .; \
79 *(.bss..decrypted); \
80 . = ALIGN(PAGE_SIZE); \
81 __start_bss_decrypted_unused = .; \
82 . = ALIGN(PMD_SIZE); \
83 __end_bss_decrypted = .; \
84
68#else 85#else
69 86
70#define X86_ALIGN_RODATA_BEGIN 87#define X86_ALIGN_RODATA_BEGIN
@@ -74,6 +91,7 @@ jiffies_64 = jiffies;
74 91
75#define ALIGN_ENTRY_TEXT_BEGIN 92#define ALIGN_ENTRY_TEXT_BEGIN
76#define ALIGN_ENTRY_TEXT_END 93#define ALIGN_ENTRY_TEXT_END
94#define BSS_DECRYPTED
77 95
78#endif 96#endif
79 97
@@ -355,6 +373,7 @@ SECTIONS
355 __bss_start = .; 373 __bss_start = .;
356 *(.bss..page_aligned) 374 *(.bss..page_aligned)
357 *(.bss) 375 *(.bss)
376 BSS_DECRYPTED
358 . = ALIGN(PAGE_SIZE); 377 . = ALIGN(PAGE_SIZE);
359 __bss_stop = .; 378 __bss_stop = .;
360 } 379 }
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 0cefba28c864..fbb0e6df121b 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -548,7 +548,7 @@ int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
548} 548}
549 549
550int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low, 550int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
551 unsigned long ipi_bitmap_high, int min, 551 unsigned long ipi_bitmap_high, u32 min,
552 unsigned long icr, int op_64_bit) 552 unsigned long icr, int op_64_bit)
553{ 553{
554 int i; 554 int i;
@@ -571,18 +571,31 @@ int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
571 rcu_read_lock(); 571 rcu_read_lock();
572 map = rcu_dereference(kvm->arch.apic_map); 572 map = rcu_dereference(kvm->arch.apic_map);
573 573
574 if (min > map->max_apic_id)
575 goto out;
574 /* Bits above cluster_size are masked in the caller. */ 576 /* Bits above cluster_size are masked in the caller. */
575 for_each_set_bit(i, &ipi_bitmap_low, BITS_PER_LONG) { 577 for_each_set_bit(i, &ipi_bitmap_low,
576 vcpu = map->phys_map[min + i]->vcpu; 578 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
577 count += kvm_apic_set_irq(vcpu, &irq, NULL); 579 if (map->phys_map[min + i]) {
580 vcpu = map->phys_map[min + i]->vcpu;
581 count += kvm_apic_set_irq(vcpu, &irq, NULL);
582 }
578 } 583 }
579 584
580 min += cluster_size; 585 min += cluster_size;
581 for_each_set_bit(i, &ipi_bitmap_high, BITS_PER_LONG) { 586
582 vcpu = map->phys_map[min + i]->vcpu; 587 if (min > map->max_apic_id)
583 count += kvm_apic_set_irq(vcpu, &irq, NULL); 588 goto out;
589
590 for_each_set_bit(i, &ipi_bitmap_high,
591 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
592 if (map->phys_map[min + i]) {
593 vcpu = map->phys_map[min + i]->vcpu;
594 count += kvm_apic_set_irq(vcpu, &irq, NULL);
595 }
584 } 596 }
585 597
598out:
586 rcu_read_unlock(); 599 rcu_read_unlock();
587 return count; 600 return count;
588} 601}
@@ -1331,9 +1344,8 @@ EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
1331 1344
1332static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) 1345static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1333{ 1346{
1334 return kvm_apic_hw_enabled(apic) && 1347 return addr >= apic->base_address &&
1335 addr >= apic->base_address && 1348 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1336 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1337} 1349}
1338 1350
1339static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this, 1351static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
@@ -1345,6 +1357,15 @@ static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1345 if (!apic_mmio_in_range(apic, address)) 1357 if (!apic_mmio_in_range(apic, address))
1346 return -EOPNOTSUPP; 1358 return -EOPNOTSUPP;
1347 1359
1360 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
1361 if (!kvm_check_has_quirk(vcpu->kvm,
1362 KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
1363 return -EOPNOTSUPP;
1364
1365 memset(data, 0xff, len);
1366 return 0;
1367 }
1368
1348 kvm_lapic_reg_read(apic, offset, len, data); 1369 kvm_lapic_reg_read(apic, offset, len, data);
1349 1370
1350 return 0; 1371 return 0;
@@ -1904,6 +1925,14 @@ static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1904 if (!apic_mmio_in_range(apic, address)) 1925 if (!apic_mmio_in_range(apic, address))
1905 return -EOPNOTSUPP; 1926 return -EOPNOTSUPP;
1906 1927
1928 if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
1929 if (!kvm_check_has_quirk(vcpu->kvm,
1930 KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
1931 return -EOPNOTSUPP;
1932
1933 return 0;
1934 }
1935
1907 /* 1936 /*
1908 * APIC register must be aligned on 128-bits boundary. 1937 * APIC register must be aligned on 128-bits boundary.
1909 * 32/64/128 bits registers must be accessed thru 32 bits. 1938 * 32/64/128 bits registers must be accessed thru 32 bits.
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index a282321329b5..d7e9bce6ff61 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -899,7 +899,7 @@ static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
899{ 899{
900 /* 900 /*
901 * Make sure the write to vcpu->mode is not reordered in front of 901 * Make sure the write to vcpu->mode is not reordered in front of
902 * reads to sptes. If it does, kvm_commit_zap_page() can see us 902 * reads to sptes. If it does, kvm_mmu_commit_zap_page() can see us
903 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table. 903 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
904 */ 904 */
905 smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE); 905 smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
@@ -1853,11 +1853,6 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
1853 return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler); 1853 return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
1854} 1854}
1855 1855
1856int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
1857{
1858 return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
1859}
1860
1861int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end) 1856int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
1862{ 1857{
1863 return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp); 1858 return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
@@ -5217,7 +5212,7 @@ static int make_mmu_pages_available(struct kvm_vcpu *vcpu)
5217int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u64 error_code, 5212int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u64 error_code,
5218 void *insn, int insn_len) 5213 void *insn, int insn_len)
5219{ 5214{
5220 int r, emulation_type = EMULTYPE_RETRY; 5215 int r, emulation_type = 0;
5221 enum emulation_result er; 5216 enum emulation_result er;
5222 bool direct = vcpu->arch.mmu.direct_map; 5217 bool direct = vcpu->arch.mmu.direct_map;
5223 5218
@@ -5230,10 +5225,8 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u64 error_code,
5230 r = RET_PF_INVALID; 5225 r = RET_PF_INVALID;
5231 if (unlikely(error_code & PFERR_RSVD_MASK)) { 5226 if (unlikely(error_code & PFERR_RSVD_MASK)) {
5232 r = handle_mmio_page_fault(vcpu, cr2, direct); 5227 r = handle_mmio_page_fault(vcpu, cr2, direct);
5233 if (r == RET_PF_EMULATE) { 5228 if (r == RET_PF_EMULATE)
5234 emulation_type = 0;
5235 goto emulate; 5229 goto emulate;
5236 }
5237 } 5230 }
5238 5231
5239 if (r == RET_PF_INVALID) { 5232 if (r == RET_PF_INVALID) {
@@ -5260,8 +5253,19 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u64 error_code,
5260 return 1; 5253 return 1;
5261 } 5254 }
5262 5255
5263 if (mmio_info_in_cache(vcpu, cr2, direct)) 5256 /*
5264 emulation_type = 0; 5257 * vcpu->arch.mmu.page_fault returned RET_PF_EMULATE, but we can still
5258 * optimistically try to just unprotect the page and let the processor
5259 * re-execute the instruction that caused the page fault. Do not allow
5260 * retrying MMIO emulation, as it's not only pointless but could also
5261 * cause us to enter an infinite loop because the processor will keep
5262 * faulting on the non-existent MMIO address. Retrying an instruction
5263 * from a nested guest is also pointless and dangerous as we are only
5264 * explicitly shadowing L1's page tables, i.e. unprotecting something
5265 * for L1 isn't going to magically fix whatever issue cause L2 to fail.
5266 */
5267 if (!mmio_info_in_cache(vcpu, cr2, direct) && !is_guest_mode(vcpu))
5268 emulation_type = EMULTYPE_ALLOW_RETRY;
5265emulate: 5269emulate:
5266 /* 5270 /*
5267 * On AMD platforms, under certain conditions insn_len may be zero on #NPF. 5271 * On AMD platforms, under certain conditions insn_len may be zero on #NPF.
@@ -5413,7 +5417,12 @@ void kvm_mmu_setup(struct kvm_vcpu *vcpu)
5413{ 5417{
5414 MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa)); 5418 MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu.root_hpa));
5415 5419
5416 kvm_init_mmu(vcpu, true); 5420 /*
5421 * kvm_mmu_setup() is called only on vCPU initialization.
5422 * Therefore, no need to reset mmu roots as they are not yet
5423 * initialized.
5424 */
5425 kvm_init_mmu(vcpu, false);
5417} 5426}
5418 5427
5419static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm, 5428static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm,
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 6276140044d0..d96092b35936 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -776,7 +776,7 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
776 } 776 }
777 777
778 if (!svm->next_rip) { 778 if (!svm->next_rip) {
779 if (emulate_instruction(vcpu, EMULTYPE_SKIP) != 779 if (kvm_emulate_instruction(vcpu, EMULTYPE_SKIP) !=
780 EMULATE_DONE) 780 EMULATE_DONE)
781 printk(KERN_DEBUG "%s: NOP\n", __func__); 781 printk(KERN_DEBUG "%s: NOP\n", __func__);
782 return; 782 return;
@@ -1226,8 +1226,7 @@ static __init int sev_hardware_setup(void)
1226 min_sev_asid = cpuid_edx(0x8000001F); 1226 min_sev_asid = cpuid_edx(0x8000001F);
1227 1227
1228 /* Initialize SEV ASID bitmap */ 1228 /* Initialize SEV ASID bitmap */
1229 sev_asid_bitmap = kcalloc(BITS_TO_LONGS(max_sev_asid), 1229 sev_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
1230 sizeof(unsigned long), GFP_KERNEL);
1231 if (!sev_asid_bitmap) 1230 if (!sev_asid_bitmap)
1232 return 1; 1231 return 1;
1233 1232
@@ -1405,7 +1404,7 @@ static __exit void svm_hardware_unsetup(void)
1405 int cpu; 1404 int cpu;
1406 1405
1407 if (svm_sev_enabled()) 1406 if (svm_sev_enabled())
1408 kfree(sev_asid_bitmap); 1407 bitmap_free(sev_asid_bitmap);
1409 1408
1410 for_each_possible_cpu(cpu) 1409 for_each_possible_cpu(cpu)
1411 svm_cpu_uninit(cpu); 1410 svm_cpu_uninit(cpu);
@@ -2715,7 +2714,7 @@ static int gp_interception(struct vcpu_svm *svm)
2715 2714
2716 WARN_ON_ONCE(!enable_vmware_backdoor); 2715 WARN_ON_ONCE(!enable_vmware_backdoor);
2717 2716
2718 er = emulate_instruction(vcpu, 2717 er = kvm_emulate_instruction(vcpu,
2719 EMULTYPE_VMWARE | EMULTYPE_NO_UD_ON_FAIL); 2718 EMULTYPE_VMWARE | EMULTYPE_NO_UD_ON_FAIL);
2720 if (er == EMULATE_USER_EXIT) 2719 if (er == EMULATE_USER_EXIT)
2721 return 0; 2720 return 0;
@@ -2819,7 +2818,7 @@ static int io_interception(struct vcpu_svm *svm)
2819 string = (io_info & SVM_IOIO_STR_MASK) != 0; 2818 string = (io_info & SVM_IOIO_STR_MASK) != 0;
2820 in = (io_info & SVM_IOIO_TYPE_MASK) != 0; 2819 in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2821 if (string) 2820 if (string)
2822 return emulate_instruction(vcpu, 0) == EMULATE_DONE; 2821 return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
2823 2822
2824 port = io_info >> 16; 2823 port = io_info >> 16;
2825 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; 2824 size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
@@ -3861,7 +3860,7 @@ static int iret_interception(struct vcpu_svm *svm)
3861static int invlpg_interception(struct vcpu_svm *svm) 3860static int invlpg_interception(struct vcpu_svm *svm)
3862{ 3861{
3863 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS)) 3862 if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3864 return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE; 3863 return kvm_emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
3865 3864
3866 kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1); 3865 kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
3867 return kvm_skip_emulated_instruction(&svm->vcpu); 3866 return kvm_skip_emulated_instruction(&svm->vcpu);
@@ -3869,13 +3868,13 @@ static int invlpg_interception(struct vcpu_svm *svm)
3869 3868
3870static int emulate_on_interception(struct vcpu_svm *svm) 3869static int emulate_on_interception(struct vcpu_svm *svm)
3871{ 3870{
3872 return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE; 3871 return kvm_emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
3873} 3872}
3874 3873
3875static int rsm_interception(struct vcpu_svm *svm) 3874static int rsm_interception(struct vcpu_svm *svm)
3876{ 3875{
3877 return x86_emulate_instruction(&svm->vcpu, 0, 0, 3876 return kvm_emulate_instruction_from_buffer(&svm->vcpu,
3878 rsm_ins_bytes, 2) == EMULATE_DONE; 3877 rsm_ins_bytes, 2) == EMULATE_DONE;
3879} 3878}
3880 3879
3881static int rdpmc_interception(struct vcpu_svm *svm) 3880static int rdpmc_interception(struct vcpu_svm *svm)
@@ -4700,7 +4699,7 @@ static int avic_unaccelerated_access_interception(struct vcpu_svm *svm)
4700 ret = avic_unaccel_trap_write(svm); 4699 ret = avic_unaccel_trap_write(svm);
4701 } else { 4700 } else {
4702 /* Handling Fault */ 4701 /* Handling Fault */
4703 ret = (emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE); 4702 ret = (kvm_emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE);
4704 } 4703 }
4705 4704
4706 return ret; 4705 return ret;
@@ -6747,7 +6746,7 @@ e_free:
6747static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) 6746static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
6748{ 6747{
6749 unsigned long vaddr, vaddr_end, next_vaddr; 6748 unsigned long vaddr, vaddr_end, next_vaddr;
6750 unsigned long dst_vaddr, dst_vaddr_end; 6749 unsigned long dst_vaddr;
6751 struct page **src_p, **dst_p; 6750 struct page **src_p, **dst_p;
6752 struct kvm_sev_dbg debug; 6751 struct kvm_sev_dbg debug;
6753 unsigned long n; 6752 unsigned long n;
@@ -6763,7 +6762,6 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
6763 size = debug.len; 6762 size = debug.len;
6764 vaddr_end = vaddr + size; 6763 vaddr_end = vaddr + size;
6765 dst_vaddr = debug.dst_uaddr; 6764 dst_vaddr = debug.dst_uaddr;
6766 dst_vaddr_end = dst_vaddr + size;
6767 6765
6768 for (; vaddr < vaddr_end; vaddr = next_vaddr) { 6766 for (; vaddr < vaddr_end; vaddr = next_vaddr) {
6769 int len, s_off, d_off; 6767 int len, s_off, d_off;
@@ -7150,6 +7148,8 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
7150 .check_intercept = svm_check_intercept, 7148 .check_intercept = svm_check_intercept,
7151 .handle_external_intr = svm_handle_external_intr, 7149 .handle_external_intr = svm_handle_external_intr,
7152 7150
7151 .request_immediate_exit = __kvm_request_immediate_exit,
7152
7153 .sched_in = svm_sched_in, 7153 .sched_in = svm_sched_in,
7154 7154
7155 .pmu_ops = &amd_pmu_ops, 7155 .pmu_ops = &amd_pmu_ops,
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 1d26f3c4985b..06412ba46aa3 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -397,6 +397,7 @@ struct loaded_vmcs {
397 int cpu; 397 int cpu;
398 bool launched; 398 bool launched;
399 bool nmi_known_unmasked; 399 bool nmi_known_unmasked;
400 bool hv_timer_armed;
400 /* Support for vnmi-less CPUs */ 401 /* Support for vnmi-less CPUs */
401 int soft_vnmi_blocked; 402 int soft_vnmi_blocked;
402 ktime_t entry_time; 403 ktime_t entry_time;
@@ -1019,6 +1020,8 @@ struct vcpu_vmx {
1019 int ple_window; 1020 int ple_window;
1020 bool ple_window_dirty; 1021 bool ple_window_dirty;
1021 1022
1023 bool req_immediate_exit;
1024
1022 /* Support for PML */ 1025 /* Support for PML */
1023#define PML_ENTITY_NUM 512 1026#define PML_ENTITY_NUM 512
1024 struct page *pml_pg; 1027 struct page *pml_pg;
@@ -2864,6 +2867,8 @@ static void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
2864 u16 fs_sel, gs_sel; 2867 u16 fs_sel, gs_sel;
2865 int i; 2868 int i;
2866 2869
2870 vmx->req_immediate_exit = false;
2871
2867 if (vmx->loaded_cpu_state) 2872 if (vmx->loaded_cpu_state)
2868 return; 2873 return;
2869 2874
@@ -5393,9 +5398,10 @@ static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
5393 * To use VMXON (and later other VMX instructions), a guest 5398 * To use VMXON (and later other VMX instructions), a guest
5394 * must first be able to turn on cr4.VMXE (see handle_vmon()). 5399 * must first be able to turn on cr4.VMXE (see handle_vmon()).
5395 * So basically the check on whether to allow nested VMX 5400 * So basically the check on whether to allow nested VMX
5396 * is here. 5401 * is here. We operate under the default treatment of SMM,
5402 * so VMX cannot be enabled under SMM.
5397 */ 5403 */
5398 if (!nested_vmx_allowed(vcpu)) 5404 if (!nested_vmx_allowed(vcpu) || is_smm(vcpu))
5399 return 1; 5405 return 1;
5400 } 5406 }
5401 5407
@@ -6183,6 +6189,27 @@ static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
6183 nested_mark_vmcs12_pages_dirty(vcpu); 6189 nested_mark_vmcs12_pages_dirty(vcpu);
6184} 6190}
6185 6191
6192static bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
6193{
6194 struct vcpu_vmx *vmx = to_vmx(vcpu);
6195 void *vapic_page;
6196 u32 vppr;
6197 int rvi;
6198
6199 if (WARN_ON_ONCE(!is_guest_mode(vcpu)) ||
6200 !nested_cpu_has_vid(get_vmcs12(vcpu)) ||
6201 WARN_ON_ONCE(!vmx->nested.virtual_apic_page))
6202 return false;
6203
6204 rvi = vmcs_read16(GUEST_INTR_STATUS) & 0xff;
6205
6206 vapic_page = kmap(vmx->nested.virtual_apic_page);
6207 vppr = *((u32 *)(vapic_page + APIC_PROCPRI));
6208 kunmap(vmx->nested.virtual_apic_page);
6209
6210 return ((rvi & 0xf0) > (vppr & 0xf0));
6211}
6212
6186static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, 6213static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
6187 bool nested) 6214 bool nested)
6188{ 6215{
@@ -6983,7 +7010,7 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu,
6983 * Cause the #SS fault with 0 error code in VM86 mode. 7010 * Cause the #SS fault with 0 error code in VM86 mode.
6984 */ 7011 */
6985 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) { 7012 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
6986 if (emulate_instruction(vcpu, 0) == EMULATE_DONE) { 7013 if (kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE) {
6987 if (vcpu->arch.halt_request) { 7014 if (vcpu->arch.halt_request) {
6988 vcpu->arch.halt_request = 0; 7015 vcpu->arch.halt_request = 0;
6989 return kvm_vcpu_halt(vcpu); 7016 return kvm_vcpu_halt(vcpu);
@@ -7054,7 +7081,7 @@ static int handle_exception(struct kvm_vcpu *vcpu)
7054 7081
7055 if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) { 7082 if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) {
7056 WARN_ON_ONCE(!enable_vmware_backdoor); 7083 WARN_ON_ONCE(!enable_vmware_backdoor);
7057 er = emulate_instruction(vcpu, 7084 er = kvm_emulate_instruction(vcpu,
7058 EMULTYPE_VMWARE | EMULTYPE_NO_UD_ON_FAIL); 7085 EMULTYPE_VMWARE | EMULTYPE_NO_UD_ON_FAIL);
7059 if (er == EMULATE_USER_EXIT) 7086 if (er == EMULATE_USER_EXIT)
7060 return 0; 7087 return 0;
@@ -7157,7 +7184,7 @@ static int handle_io(struct kvm_vcpu *vcpu)
7157 ++vcpu->stat.io_exits; 7184 ++vcpu->stat.io_exits;
7158 7185
7159 if (string) 7186 if (string)
7160 return emulate_instruction(vcpu, 0) == EMULATE_DONE; 7187 return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
7161 7188
7162 port = exit_qualification >> 16; 7189 port = exit_qualification >> 16;
7163 size = (exit_qualification & 7) + 1; 7190 size = (exit_qualification & 7) + 1;
@@ -7231,7 +7258,7 @@ static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
7231static int handle_desc(struct kvm_vcpu *vcpu) 7258static int handle_desc(struct kvm_vcpu *vcpu)
7232{ 7259{
7233 WARN_ON(!(vcpu->arch.cr4 & X86_CR4_UMIP)); 7260 WARN_ON(!(vcpu->arch.cr4 & X86_CR4_UMIP));
7234 return emulate_instruction(vcpu, 0) == EMULATE_DONE; 7261 return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
7235} 7262}
7236 7263
7237static int handle_cr(struct kvm_vcpu *vcpu) 7264static int handle_cr(struct kvm_vcpu *vcpu)
@@ -7480,7 +7507,7 @@ static int handle_vmcall(struct kvm_vcpu *vcpu)
7480 7507
7481static int handle_invd(struct kvm_vcpu *vcpu) 7508static int handle_invd(struct kvm_vcpu *vcpu)
7482{ 7509{
7483 return emulate_instruction(vcpu, 0) == EMULATE_DONE; 7510 return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
7484} 7511}
7485 7512
7486static int handle_invlpg(struct kvm_vcpu *vcpu) 7513static int handle_invlpg(struct kvm_vcpu *vcpu)
@@ -7547,7 +7574,7 @@ static int handle_apic_access(struct kvm_vcpu *vcpu)
7547 return kvm_skip_emulated_instruction(vcpu); 7574 return kvm_skip_emulated_instruction(vcpu);
7548 } 7575 }
7549 } 7576 }
7550 return emulate_instruction(vcpu, 0) == EMULATE_DONE; 7577 return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
7551} 7578}
7552 7579
7553static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) 7580static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
@@ -7704,8 +7731,8 @@ static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
7704 if (!static_cpu_has(X86_FEATURE_HYPERVISOR)) 7731 if (!static_cpu_has(X86_FEATURE_HYPERVISOR))
7705 return kvm_skip_emulated_instruction(vcpu); 7732 return kvm_skip_emulated_instruction(vcpu);
7706 else 7733 else
7707 return x86_emulate_instruction(vcpu, gpa, EMULTYPE_SKIP, 7734 return kvm_emulate_instruction(vcpu, EMULTYPE_SKIP) ==
7708 NULL, 0) == EMULATE_DONE; 7735 EMULATE_DONE;
7709 } 7736 }
7710 7737
7711 return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0); 7738 return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0);
@@ -7748,7 +7775,7 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
7748 if (kvm_test_request(KVM_REQ_EVENT, vcpu)) 7775 if (kvm_test_request(KVM_REQ_EVENT, vcpu))
7749 return 1; 7776 return 1;
7750 7777
7751 err = emulate_instruction(vcpu, 0); 7778 err = kvm_emulate_instruction(vcpu, 0);
7752 7779
7753 if (err == EMULATE_USER_EXIT) { 7780 if (err == EMULATE_USER_EXIT) {
7754 ++vcpu->stat.mmio_exits; 7781 ++vcpu->stat.mmio_exits;
@@ -7966,6 +7993,9 @@ static __init int hardware_setup(void)
7966 kvm_x86_ops->enable_log_dirty_pt_masked = NULL; 7993 kvm_x86_ops->enable_log_dirty_pt_masked = NULL;
7967 } 7994 }
7968 7995
7996 if (!cpu_has_vmx_preemption_timer())
7997 kvm_x86_ops->request_immediate_exit = __kvm_request_immediate_exit;
7998
7969 if (cpu_has_vmx_preemption_timer() && enable_preemption_timer) { 7999 if (cpu_has_vmx_preemption_timer() && enable_preemption_timer) {
7970 u64 vmx_msr; 8000 u64 vmx_msr;
7971 8001
@@ -9208,7 +9238,8 @@ static int handle_pml_full(struct kvm_vcpu *vcpu)
9208 9238
9209static int handle_preemption_timer(struct kvm_vcpu *vcpu) 9239static int handle_preemption_timer(struct kvm_vcpu *vcpu)
9210{ 9240{
9211 kvm_lapic_expired_hv_timer(vcpu); 9241 if (!to_vmx(vcpu)->req_immediate_exit)
9242 kvm_lapic_expired_hv_timer(vcpu);
9212 return 1; 9243 return 1;
9213} 9244}
9214 9245
@@ -10595,24 +10626,43 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
10595 msrs[i].host, false); 10626 msrs[i].host, false);
10596} 10627}
10597 10628
10598static void vmx_arm_hv_timer(struct kvm_vcpu *vcpu) 10629static void vmx_arm_hv_timer(struct vcpu_vmx *vmx, u32 val)
10630{
10631 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, val);
10632 if (!vmx->loaded_vmcs->hv_timer_armed)
10633 vmcs_set_bits(PIN_BASED_VM_EXEC_CONTROL,
10634 PIN_BASED_VMX_PREEMPTION_TIMER);
10635 vmx->loaded_vmcs->hv_timer_armed = true;
10636}
10637
10638static void vmx_update_hv_timer(struct kvm_vcpu *vcpu)
10599{ 10639{
10600 struct vcpu_vmx *vmx = to_vmx(vcpu); 10640 struct vcpu_vmx *vmx = to_vmx(vcpu);
10601 u64 tscl; 10641 u64 tscl;
10602 u32 delta_tsc; 10642 u32 delta_tsc;
10603 10643
10604 if (vmx->hv_deadline_tsc == -1) 10644 if (vmx->req_immediate_exit) {
10645 vmx_arm_hv_timer(vmx, 0);
10605 return; 10646 return;
10647 }
10606 10648
10607 tscl = rdtsc(); 10649 if (vmx->hv_deadline_tsc != -1) {
10608 if (vmx->hv_deadline_tsc > tscl) 10650 tscl = rdtsc();
10609 /* sure to be 32 bit only because checked on set_hv_timer */ 10651 if (vmx->hv_deadline_tsc > tscl)
10610 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >> 10652 /* set_hv_timer ensures the delta fits in 32-bits */
10611 cpu_preemption_timer_multi); 10653 delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >>
10612 else 10654 cpu_preemption_timer_multi);
10613 delta_tsc = 0; 10655 else
10656 delta_tsc = 0;
10657
10658 vmx_arm_hv_timer(vmx, delta_tsc);
10659 return;
10660 }
10614 10661
10615 vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, delta_tsc); 10662 if (vmx->loaded_vmcs->hv_timer_armed)
10663 vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL,
10664 PIN_BASED_VMX_PREEMPTION_TIMER);
10665 vmx->loaded_vmcs->hv_timer_armed = false;
10616} 10666}
10617 10667
10618static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) 10668static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
@@ -10672,7 +10722,7 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
10672 10722
10673 atomic_switch_perf_msrs(vmx); 10723 atomic_switch_perf_msrs(vmx);
10674 10724
10675 vmx_arm_hv_timer(vcpu); 10725 vmx_update_hv_timer(vcpu);
10676 10726
10677 /* 10727 /*
10678 * If this vCPU has touched SPEC_CTRL, restore the guest's value if 10728 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
@@ -11427,16 +11477,18 @@ static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu)
11427 u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value; 11477 u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value;
11428 struct vcpu_vmx *vmx = to_vmx(vcpu); 11478 struct vcpu_vmx *vmx = to_vmx(vcpu);
11429 11479
11430 if (vcpu->arch.virtual_tsc_khz == 0) 11480 /*
11431 return; 11481 * A timer value of zero is architecturally guaranteed to cause
11432 11482 * a VMExit prior to executing any instructions in the guest.
11433 /* Make sure short timeouts reliably trigger an immediate vmexit. 11483 */
11434 * hrtimer_start does not guarantee this. */ 11484 if (preemption_timeout == 0) {
11435 if (preemption_timeout <= 1) {
11436 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); 11485 vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
11437 return; 11486 return;
11438 } 11487 }
11439 11488
11489 if (vcpu->arch.virtual_tsc_khz == 0)
11490 return;
11491
11440 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 11492 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
11441 preemption_timeout *= 1000000; 11493 preemption_timeout *= 1000000;
11442 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); 11494 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
@@ -11646,11 +11698,15 @@ static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
11646 * bits 15:8 should be zero in posted_intr_nv, 11698 * bits 15:8 should be zero in posted_intr_nv,
11647 * the descriptor address has been already checked 11699 * the descriptor address has been already checked
11648 * in nested_get_vmcs12_pages. 11700 * in nested_get_vmcs12_pages.
11701 *
11702 * bits 5:0 of posted_intr_desc_addr should be zero.
11649 */ 11703 */
11650 if (nested_cpu_has_posted_intr(vmcs12) && 11704 if (nested_cpu_has_posted_intr(vmcs12) &&
11651 (!nested_cpu_has_vid(vmcs12) || 11705 (!nested_cpu_has_vid(vmcs12) ||
11652 !nested_exit_intr_ack_set(vcpu) || 11706 !nested_exit_intr_ack_set(vcpu) ||
11653 vmcs12->posted_intr_nv & 0xff00)) 11707 (vmcs12->posted_intr_nv & 0xff00) ||
11708 (vmcs12->posted_intr_desc_addr & 0x3f) ||
11709 (!page_address_valid(vcpu, vmcs12->posted_intr_desc_addr))))
11654 return -EINVAL; 11710 return -EINVAL;
11655 11711
11656 /* tpr shadow is needed by all apicv features. */ 11712 /* tpr shadow is needed by all apicv features. */
@@ -12076,11 +12132,10 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
12076 12132
12077 exec_control = vmcs12->pin_based_vm_exec_control; 12133 exec_control = vmcs12->pin_based_vm_exec_control;
12078 12134
12079 /* Preemption timer setting is only taken from vmcs01. */ 12135 /* Preemption timer setting is computed directly in vmx_vcpu_run. */
12080 exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
12081 exec_control |= vmcs_config.pin_based_exec_ctrl; 12136 exec_control |= vmcs_config.pin_based_exec_ctrl;
12082 if (vmx->hv_deadline_tsc == -1) 12137 exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
12083 exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 12138 vmx->loaded_vmcs->hv_timer_armed = false;
12084 12139
12085 /* Posted interrupts setting is only taken from vmcs12. */ 12140 /* Posted interrupts setting is only taken from vmcs12. */
12086 if (nested_cpu_has_posted_intr(vmcs12)) { 12141 if (nested_cpu_has_posted_intr(vmcs12)) {
@@ -12318,6 +12373,9 @@ static int check_vmentry_prereqs(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
12318 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT) 12373 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT)
12319 return VMXERR_ENTRY_INVALID_CONTROL_FIELD; 12374 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12320 12375
12376 if (nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)
12377 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12378
12321 if (nested_vmx_check_io_bitmap_controls(vcpu, vmcs12)) 12379 if (nested_vmx_check_io_bitmap_controls(vcpu, vmcs12))
12322 return VMXERR_ENTRY_INVALID_CONTROL_FIELD; 12380 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12323 12381
@@ -12537,8 +12595,11 @@ static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu, u32 *exit_qual)
12537 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 12595 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
12538 bool from_vmentry = !!exit_qual; 12596 bool from_vmentry = !!exit_qual;
12539 u32 dummy_exit_qual; 12597 u32 dummy_exit_qual;
12598 u32 vmcs01_cpu_exec_ctrl;
12540 int r = 0; 12599 int r = 0;
12541 12600
12601 vmcs01_cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
12602
12542 enter_guest_mode(vcpu); 12603 enter_guest_mode(vcpu);
12543 12604
12544 if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) 12605 if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
@@ -12575,6 +12636,25 @@ static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu, u32 *exit_qual)
12575 } 12636 }
12576 12637
12577 /* 12638 /*
12639 * If L1 had a pending IRQ/NMI until it executed
12640 * VMLAUNCH/VMRESUME which wasn't delivered because it was
12641 * disallowed (e.g. interrupts disabled), L0 needs to
12642 * evaluate if this pending event should cause an exit from L2
12643 * to L1 or delivered directly to L2 (e.g. In case L1 don't
12644 * intercept EXTERNAL_INTERRUPT).
12645 *
12646 * Usually this would be handled by L0 requesting a
12647 * IRQ/NMI window by setting VMCS accordingly. However,
12648 * this setting was done on VMCS01 and now VMCS02 is active
12649 * instead. Thus, we force L0 to perform pending event
12650 * evaluation by requesting a KVM_REQ_EVENT.
12651 */
12652 if (vmcs01_cpu_exec_ctrl &
12653 (CPU_BASED_VIRTUAL_INTR_PENDING | CPU_BASED_VIRTUAL_NMI_PENDING)) {
12654 kvm_make_request(KVM_REQ_EVENT, vcpu);
12655 }
12656
12657 /*
12578 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point 12658 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
12579 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet 12659 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
12580 * returned as far as L1 is concerned. It will only return (and set 12660 * returned as far as L1 is concerned. It will only return (and set
@@ -12841,6 +12921,11 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr)
12841 return 0; 12921 return 0;
12842} 12922}
12843 12923
12924static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu)
12925{
12926 to_vmx(vcpu)->req_immediate_exit = true;
12927}
12928
12844static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) 12929static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
12845{ 12930{
12846 ktime_t remaining = 12931 ktime_t remaining =
@@ -13231,12 +13316,7 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
13231 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 13316 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
13232 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 13317 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
13233 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 13318 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
13234 if (vmx->hv_deadline_tsc == -1) 13319
13235 vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL,
13236 PIN_BASED_VMX_PREEMPTION_TIMER);
13237 else
13238 vmcs_set_bits(PIN_BASED_VM_EXEC_CONTROL,
13239 PIN_BASED_VMX_PREEMPTION_TIMER);
13240 if (kvm_has_tsc_control) 13320 if (kvm_has_tsc_control)
13241 decache_tsc_multiplier(vmx); 13321 decache_tsc_multiplier(vmx);
13242 13322
@@ -13440,18 +13520,12 @@ static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc)
13440 return -ERANGE; 13520 return -ERANGE;
13441 13521
13442 vmx->hv_deadline_tsc = tscl + delta_tsc; 13522 vmx->hv_deadline_tsc = tscl + delta_tsc;
13443 vmcs_set_bits(PIN_BASED_VM_EXEC_CONTROL,
13444 PIN_BASED_VMX_PREEMPTION_TIMER);
13445
13446 return delta_tsc == 0; 13523 return delta_tsc == 0;
13447} 13524}
13448 13525
13449static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu) 13526static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu)
13450{ 13527{
13451 struct vcpu_vmx *vmx = to_vmx(vcpu); 13528 to_vmx(vcpu)->hv_deadline_tsc = -1;
13452 vmx->hv_deadline_tsc = -1;
13453 vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL,
13454 PIN_BASED_VMX_PREEMPTION_TIMER);
13455} 13529}
13456#endif 13530#endif
13457 13531
@@ -13932,6 +14006,14 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu,
13932 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON)) 14006 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON))
13933 return -EINVAL; 14007 return -EINVAL;
13934 14008
14009 /*
14010 * SMM temporarily disables VMX, so we cannot be in guest mode,
14011 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags
14012 * must be zero.
14013 */
14014 if (is_smm(vcpu) ? kvm_state->flags : kvm_state->vmx.smm.flags)
14015 return -EINVAL;
14016
13935 if ((kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 14017 if ((kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
13936 !(kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON)) 14018 !(kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON))
13937 return -EINVAL; 14019 return -EINVAL;
@@ -13988,9 +14070,6 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu,
13988 check_vmentry_postreqs(vcpu, vmcs12, &exit_qual)) 14070 check_vmentry_postreqs(vcpu, vmcs12, &exit_qual))
13989 return -EINVAL; 14071 return -EINVAL;
13990 14072
13991 if (kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING)
13992 vmx->nested.nested_run_pending = 1;
13993
13994 vmx->nested.dirty_vmcs12 = true; 14073 vmx->nested.dirty_vmcs12 = true;
13995 ret = enter_vmx_non_root_mode(vcpu, NULL); 14074 ret = enter_vmx_non_root_mode(vcpu, NULL);
13996 if (ret) 14075 if (ret)
@@ -14078,6 +14157,7 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
14078 .apicv_post_state_restore = vmx_apicv_post_state_restore, 14157 .apicv_post_state_restore = vmx_apicv_post_state_restore,
14079 .hwapic_irr_update = vmx_hwapic_irr_update, 14158 .hwapic_irr_update = vmx_hwapic_irr_update,
14080 .hwapic_isr_update = vmx_hwapic_isr_update, 14159 .hwapic_isr_update = vmx_hwapic_isr_update,
14160 .guest_apic_has_interrupt = vmx_guest_apic_has_interrupt,
14081 .sync_pir_to_irr = vmx_sync_pir_to_irr, 14161 .sync_pir_to_irr = vmx_sync_pir_to_irr,
14082 .deliver_posted_interrupt = vmx_deliver_posted_interrupt, 14162 .deliver_posted_interrupt = vmx_deliver_posted_interrupt,
14083 14163
@@ -14111,6 +14191,7 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
14111 .umip_emulated = vmx_umip_emulated, 14191 .umip_emulated = vmx_umip_emulated,
14112 14192
14113 .check_nested_events = vmx_check_nested_events, 14193 .check_nested_events = vmx_check_nested_events,
14194 .request_immediate_exit = vmx_request_immediate_exit,
14114 14195
14115 .sched_in = vmx_sched_in, 14196 .sched_in = vmx_sched_in,
14116 14197
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 506bd2b4b8bb..edbf00ec56b3 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -628,7 +628,7 @@ bool pdptrs_changed(struct kvm_vcpu *vcpu)
628 gfn_t gfn; 628 gfn_t gfn;
629 int r; 629 int r;
630 630
631 if (is_long_mode(vcpu) || !is_pae(vcpu)) 631 if (is_long_mode(vcpu) || !is_pae(vcpu) || !is_paging(vcpu))
632 return false; 632 return false;
633 633
634 if (!test_bit(VCPU_EXREG_PDPTR, 634 if (!test_bit(VCPU_EXREG_PDPTR,
@@ -2537,7 +2537,6 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2537 break; 2537 break;
2538 case MSR_PLATFORM_INFO: 2538 case MSR_PLATFORM_INFO:
2539 if (!msr_info->host_initiated || 2539 if (!msr_info->host_initiated ||
2540 data & ~MSR_PLATFORM_INFO_CPUID_FAULT ||
2541 (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) && 2540 (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) &&
2542 cpuid_fault_enabled(vcpu))) 2541 cpuid_fault_enabled(vcpu)))
2543 return 1; 2542 return 1;
@@ -2780,6 +2779,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2780 msr_info->data = vcpu->arch.osvw.status; 2779 msr_info->data = vcpu->arch.osvw.status;
2781 break; 2780 break;
2782 case MSR_PLATFORM_INFO: 2781 case MSR_PLATFORM_INFO:
2782 if (!msr_info->host_initiated &&
2783 !vcpu->kvm->arch.guest_can_read_msr_platform_info)
2784 return 1;
2783 msr_info->data = vcpu->arch.msr_platform_info; 2785 msr_info->data = vcpu->arch.msr_platform_info;
2784 break; 2786 break;
2785 case MSR_MISC_FEATURES_ENABLES: 2787 case MSR_MISC_FEATURES_ENABLES:
@@ -2927,6 +2929,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
2927 case KVM_CAP_SPLIT_IRQCHIP: 2929 case KVM_CAP_SPLIT_IRQCHIP:
2928 case KVM_CAP_IMMEDIATE_EXIT: 2930 case KVM_CAP_IMMEDIATE_EXIT:
2929 case KVM_CAP_GET_MSR_FEATURES: 2931 case KVM_CAP_GET_MSR_FEATURES:
2932 case KVM_CAP_MSR_PLATFORM_INFO:
2930 r = 1; 2933 r = 1;
2931 break; 2934 break;
2932 case KVM_CAP_SYNC_REGS: 2935 case KVM_CAP_SYNC_REGS:
@@ -4007,19 +4010,23 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
4007 break; 4010 break;
4008 4011
4009 BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size)); 4012 BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
4013 r = -EFAULT;
4010 if (get_user(user_data_size, &user_kvm_nested_state->size)) 4014 if (get_user(user_data_size, &user_kvm_nested_state->size))
4011 return -EFAULT; 4015 break;
4012 4016
4013 r = kvm_x86_ops->get_nested_state(vcpu, user_kvm_nested_state, 4017 r = kvm_x86_ops->get_nested_state(vcpu, user_kvm_nested_state,
4014 user_data_size); 4018 user_data_size);
4015 if (r < 0) 4019 if (r < 0)
4016 return r; 4020 break;
4017 4021
4018 if (r > user_data_size) { 4022 if (r > user_data_size) {
4019 if (put_user(r, &user_kvm_nested_state->size)) 4023 if (put_user(r, &user_kvm_nested_state->size))
4020 return -EFAULT; 4024 r = -EFAULT;
4021 return -E2BIG; 4025 else
4026 r = -E2BIG;
4027 break;
4022 } 4028 }
4029
4023 r = 0; 4030 r = 0;
4024 break; 4031 break;
4025 } 4032 }
@@ -4031,19 +4038,21 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
4031 if (!kvm_x86_ops->set_nested_state) 4038 if (!kvm_x86_ops->set_nested_state)
4032 break; 4039 break;
4033 4040
4041 r = -EFAULT;
4034 if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state))) 4042 if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
4035 return -EFAULT; 4043 break;
4036 4044
4045 r = -EINVAL;
4037 if (kvm_state.size < sizeof(kvm_state)) 4046 if (kvm_state.size < sizeof(kvm_state))
4038 return -EINVAL; 4047 break;
4039 4048
4040 if (kvm_state.flags & 4049 if (kvm_state.flags &
4041 ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE)) 4050 ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE))
4042 return -EINVAL; 4051 break;
4043 4052
4044 /* nested_run_pending implies guest_mode. */ 4053 /* nested_run_pending implies guest_mode. */
4045 if (kvm_state.flags == KVM_STATE_NESTED_RUN_PENDING) 4054 if (kvm_state.flags == KVM_STATE_NESTED_RUN_PENDING)
4046 return -EINVAL; 4055 break;
4047 4056
4048 r = kvm_x86_ops->set_nested_state(vcpu, user_kvm_nested_state, &kvm_state); 4057 r = kvm_x86_ops->set_nested_state(vcpu, user_kvm_nested_state, &kvm_state);
4049 break; 4058 break;
@@ -4350,6 +4359,10 @@ split_irqchip_unlock:
4350 kvm->arch.pause_in_guest = true; 4359 kvm->arch.pause_in_guest = true;
4351 r = 0; 4360 r = 0;
4352 break; 4361 break;
4362 case KVM_CAP_MSR_PLATFORM_INFO:
4363 kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
4364 r = 0;
4365 break;
4353 default: 4366 default:
4354 r = -EINVAL; 4367 r = -EINVAL;
4355 break; 4368 break;
@@ -4987,7 +5000,7 @@ int handle_ud(struct kvm_vcpu *vcpu)
4987 emul_type = 0; 5000 emul_type = 0;
4988 } 5001 }
4989 5002
4990 er = emulate_instruction(vcpu, emul_type); 5003 er = kvm_emulate_instruction(vcpu, emul_type);
4991 if (er == EMULATE_USER_EXIT) 5004 if (er == EMULATE_USER_EXIT)
4992 return 0; 5005 return 0;
4993 if (er != EMULATE_DONE) 5006 if (er != EMULATE_DONE)
@@ -5870,7 +5883,10 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t cr2,
5870 gpa_t gpa = cr2; 5883 gpa_t gpa = cr2;
5871 kvm_pfn_t pfn; 5884 kvm_pfn_t pfn;
5872 5885
5873 if (emulation_type & EMULTYPE_NO_REEXECUTE) 5886 if (!(emulation_type & EMULTYPE_ALLOW_RETRY))
5887 return false;
5888
5889 if (WARN_ON_ONCE(is_guest_mode(vcpu)))
5874 return false; 5890 return false;
5875 5891
5876 if (!vcpu->arch.mmu.direct_map) { 5892 if (!vcpu->arch.mmu.direct_map) {
@@ -5958,7 +5974,10 @@ static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
5958 */ 5974 */
5959 vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0; 5975 vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;
5960 5976
5961 if (!(emulation_type & EMULTYPE_RETRY)) 5977 if (!(emulation_type & EMULTYPE_ALLOW_RETRY))
5978 return false;
5979
5980 if (WARN_ON_ONCE(is_guest_mode(vcpu)))
5962 return false; 5981 return false;
5963 5982
5964 if (x86_page_table_writing_insn(ctxt)) 5983 if (x86_page_table_writing_insn(ctxt))
@@ -6276,7 +6295,19 @@ restart:
6276 6295
6277 return r; 6296 return r;
6278} 6297}
6279EXPORT_SYMBOL_GPL(x86_emulate_instruction); 6298
6299int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
6300{
6301 return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
6302}
6303EXPORT_SYMBOL_GPL(kvm_emulate_instruction);
6304
6305int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
6306 void *insn, int insn_len)
6307{
6308 return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
6309}
6310EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer);
6280 6311
6281static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, 6312static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
6282 unsigned short port) 6313 unsigned short port)
@@ -7343,6 +7374,12 @@ void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
7343} 7374}
7344EXPORT_SYMBOL_GPL(kvm_vcpu_reload_apic_access_page); 7375EXPORT_SYMBOL_GPL(kvm_vcpu_reload_apic_access_page);
7345 7376
7377void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu)
7378{
7379 smp_send_reschedule(vcpu->cpu);
7380}
7381EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit);
7382
7346/* 7383/*
7347 * Returns 1 to let vcpu_run() continue the guest execution loop without 7384 * Returns 1 to let vcpu_run() continue the guest execution loop without
7348 * exiting to the userspace. Otherwise, the value will be returned to the 7385 * exiting to the userspace. Otherwise, the value will be returned to the
@@ -7547,7 +7584,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
7547 7584
7548 if (req_immediate_exit) { 7585 if (req_immediate_exit) {
7549 kvm_make_request(KVM_REQ_EVENT, vcpu); 7586 kvm_make_request(KVM_REQ_EVENT, vcpu);
7550 smp_send_reschedule(vcpu->cpu); 7587 kvm_x86_ops->request_immediate_exit(vcpu);
7551 } 7588 }
7552 7589
7553 trace_kvm_entry(vcpu->vcpu_id); 7590 trace_kvm_entry(vcpu->vcpu_id);
@@ -7734,7 +7771,7 @@ static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
7734{ 7771{
7735 int r; 7772 int r;
7736 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 7773 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
7737 r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE); 7774 r = kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
7738 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 7775 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
7739 if (r != EMULATE_DONE) 7776 if (r != EMULATE_DONE)
7740 return 0; 7777 return 0;
@@ -7811,6 +7848,29 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
7811 return 0; 7848 return 0;
7812} 7849}
7813 7850
7851/* Swap (qemu) user FPU context for the guest FPU context. */
7852static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
7853{
7854 preempt_disable();
7855 copy_fpregs_to_fpstate(&vcpu->arch.user_fpu);
7856 /* PKRU is separately restored in kvm_x86_ops->run. */
7857 __copy_kernel_to_fpregs(&vcpu->arch.guest_fpu.state,
7858 ~XFEATURE_MASK_PKRU);
7859 preempt_enable();
7860 trace_kvm_fpu(1);
7861}
7862
7863/* When vcpu_run ends, restore user space FPU context. */
7864static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
7865{
7866 preempt_disable();
7867 copy_fpregs_to_fpstate(&vcpu->arch.guest_fpu);
7868 copy_kernel_to_fpregs(&vcpu->arch.user_fpu.state);
7869 preempt_enable();
7870 ++vcpu->stat.fpu_reload;
7871 trace_kvm_fpu(0);
7872}
7873
7814int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 7874int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
7815{ 7875{
7816 int r; 7876 int r;
@@ -8159,7 +8219,7 @@ static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
8159 kvm_update_cpuid(vcpu); 8219 kvm_update_cpuid(vcpu);
8160 8220
8161 idx = srcu_read_lock(&vcpu->kvm->srcu); 8221 idx = srcu_read_lock(&vcpu->kvm->srcu);
8162 if (!is_long_mode(vcpu) && is_pae(vcpu)) { 8222 if (!is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu)) {
8163 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); 8223 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
8164 mmu_reset_needed = 1; 8224 mmu_reset_needed = 1;
8165 } 8225 }
@@ -8388,29 +8448,6 @@ static void fx_init(struct kvm_vcpu *vcpu)
8388 vcpu->arch.cr0 |= X86_CR0_ET; 8448 vcpu->arch.cr0 |= X86_CR0_ET;
8389} 8449}
8390 8450
8391/* Swap (qemu) user FPU context for the guest FPU context. */
8392void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
8393{
8394 preempt_disable();
8395 copy_fpregs_to_fpstate(&vcpu->arch.user_fpu);
8396 /* PKRU is separately restored in kvm_x86_ops->run. */
8397 __copy_kernel_to_fpregs(&vcpu->arch.guest_fpu.state,
8398 ~XFEATURE_MASK_PKRU);
8399 preempt_enable();
8400 trace_kvm_fpu(1);
8401}
8402
8403/* When vcpu_run ends, restore user space FPU context. */
8404void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
8405{
8406 preempt_disable();
8407 copy_fpregs_to_fpstate(&vcpu->arch.guest_fpu);
8408 copy_kernel_to_fpregs(&vcpu->arch.user_fpu.state);
8409 preempt_enable();
8410 ++vcpu->stat.fpu_reload;
8411 trace_kvm_fpu(0);
8412}
8413
8414void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) 8451void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
8415{ 8452{
8416 void *wbinvd_dirty_mask = vcpu->arch.wbinvd_dirty_mask; 8453 void *wbinvd_dirty_mask = vcpu->arch.wbinvd_dirty_mask;
@@ -8834,6 +8871,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
8834 kvm->arch.kvmclock_offset = -ktime_get_boot_ns(); 8871 kvm->arch.kvmclock_offset = -ktime_get_boot_ns();
8835 pvclock_update_vm_gtod_copy(kvm); 8872 pvclock_update_vm_gtod_copy(kvm);
8836 8873
8874 kvm->arch.guest_can_read_msr_platform_info = true;
8875
8837 INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn); 8876 INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
8838 INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn); 8877 INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
8839 8878
@@ -9182,6 +9221,13 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
9182 kvm_page_track_flush_slot(kvm, slot); 9221 kvm_page_track_flush_slot(kvm, slot);
9183} 9222}
9184 9223
9224static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
9225{
9226 return (is_guest_mode(vcpu) &&
9227 kvm_x86_ops->guest_apic_has_interrupt &&
9228 kvm_x86_ops->guest_apic_has_interrupt(vcpu));
9229}
9230
9185static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) 9231static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
9186{ 9232{
9187 if (!list_empty_careful(&vcpu->async_pf.done)) 9233 if (!list_empty_careful(&vcpu->async_pf.done))
@@ -9206,7 +9252,8 @@ static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
9206 return true; 9252 return true;
9207 9253
9208 if (kvm_arch_interrupt_allowed(vcpu) && 9254 if (kvm_arch_interrupt_allowed(vcpu) &&
9209 kvm_cpu_has_interrupt(vcpu)) 9255 (kvm_cpu_has_interrupt(vcpu) ||
9256 kvm_guest_apic_has_interrupt(vcpu)))
9210 return true; 9257 return true;
9211 9258
9212 if (kvm_hv_has_stimer_pending(vcpu)) 9259 if (kvm_hv_has_stimer_pending(vcpu))
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index 257f27620bc2..67b9568613f3 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -274,6 +274,8 @@ int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata);
274bool kvm_mtrr_check_gfn_range_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, 274bool kvm_mtrr_check_gfn_range_consistency(struct kvm_vcpu *vcpu, gfn_t gfn,
275 int page_num); 275 int page_num);
276bool kvm_vector_hashing_enabled(void); 276bool kvm_vector_hashing_enabled(void);
277int x86_emulate_instruction(struct kvm_vcpu *vcpu, unsigned long cr2,
278 int emulation_type, void *insn, int insn_len);
277 279
278#define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \ 280#define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \
279 | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \ 281 | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \
diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index 7a8fc26c1115..faca978ebf9d 100644
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -815,10 +815,14 @@ void free_kernel_image_pages(void *begin, void *end)
815 set_memory_np_noalias(begin_ul, len_pages); 815 set_memory_np_noalias(begin_ul, len_pages);
816} 816}
817 817
818void __weak mem_encrypt_free_decrypted_mem(void) { }
819
818void __ref free_initmem(void) 820void __ref free_initmem(void)
819{ 821{
820 e820__reallocate_tables(); 822 e820__reallocate_tables();
821 823
824 mem_encrypt_free_decrypted_mem();
825
822 free_kernel_image_pages(&__init_begin, &__init_end); 826 free_kernel_image_pages(&__init_begin, &__init_end);
823} 827}
824 828
diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
index b2de398d1fd3..006f373f54ab 100644
--- a/arch/x86/mm/mem_encrypt.c
+++ b/arch/x86/mm/mem_encrypt.c
@@ -348,6 +348,30 @@ bool sev_active(void)
348EXPORT_SYMBOL(sev_active); 348EXPORT_SYMBOL(sev_active);
349 349
350/* Architecture __weak replacement functions */ 350/* Architecture __weak replacement functions */
351void __init mem_encrypt_free_decrypted_mem(void)
352{
353 unsigned long vaddr, vaddr_end, npages;
354 int r;
355
356 vaddr = (unsigned long)__start_bss_decrypted_unused;
357 vaddr_end = (unsigned long)__end_bss_decrypted;
358 npages = (vaddr_end - vaddr) >> PAGE_SHIFT;
359
360 /*
361 * The unused memory range was mapped decrypted, change the encryption
362 * attribute from decrypted to encrypted before freeing it.
363 */
364 if (mem_encrypt_active()) {
365 r = set_memory_encrypted(vaddr, npages);
366 if (r) {
367 pr_warn("failed to free unused decrypted pages\n");
368 return;
369 }
370 }
371
372 free_init_pages("unused decrypted", vaddr, vaddr_end);
373}
374
351void __init mem_encrypt_init(void) 375void __init mem_encrypt_init(void)
352{ 376{
353 if (!sme_me_mask) 377 if (!sme_me_mask)
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index e848a4811785..089e78c4effd 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -269,7 +269,7 @@ static void mop_up_one_pmd(struct mm_struct *mm, pgd_t *pgdp)
269 if (pgd_val(pgd) != 0) { 269 if (pgd_val(pgd) != 0) {
270 pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd); 270 pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd);
271 271
272 *pgdp = native_make_pgd(0); 272 pgd_clear(pgdp);
273 273
274 paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT); 274 paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);
275 pmd_free(mm, pmd); 275 pmd_free(mm, pmd);
@@ -494,7 +494,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma,
494 int changed = !pte_same(*ptep, entry); 494 int changed = !pte_same(*ptep, entry);
495 495
496 if (changed && dirty) 496 if (changed && dirty)
497 *ptep = entry; 497 set_pte(ptep, entry);
498 498
499 return changed; 499 return changed;
500} 500}
@@ -509,7 +509,7 @@ int pmdp_set_access_flags(struct vm_area_struct *vma,
509 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 509 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
510 510
511 if (changed && dirty) { 511 if (changed && dirty) {
512 *pmdp = entry; 512 set_pmd(pmdp, entry);
513 /* 513 /*
514 * We had a write-protection fault here and changed the pmd 514 * We had a write-protection fault here and changed the pmd
515 * to to more permissive. No need to flush the TLB for that, 515 * to to more permissive. No need to flush the TLB for that,
@@ -529,7 +529,7 @@ int pudp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
529 VM_BUG_ON(address & ~HPAGE_PUD_MASK); 529 VM_BUG_ON(address & ~HPAGE_PUD_MASK);
530 530
531 if (changed && dirty) { 531 if (changed && dirty) {
532 *pudp = entry; 532 set_pud(pudp, entry);
533 /* 533 /*
534 * We had a write-protection fault here and changed the pud 534 * We had a write-protection fault here and changed the pud
535 * to to more permissive. No need to flush the TLB for that, 535 * to to more permissive. No need to flush the TLB for that,
@@ -637,6 +637,15 @@ void __native_set_fixmap(enum fixed_addresses idx, pte_t pte)
637{ 637{
638 unsigned long address = __fix_to_virt(idx); 638 unsigned long address = __fix_to_virt(idx);
639 639
640#ifdef CONFIG_X86_64
641 /*
642 * Ensure that the static initial page tables are covering the
643 * fixmap completely.
644 */
645 BUILD_BUG_ON(__end_of_permanent_fixed_addresses >
646 (FIXMAP_PMD_NUM * PTRS_PER_PTE));
647#endif
648
640 if (idx >= __end_of_fixed_addresses) { 649 if (idx >= __end_of_fixed_addresses) {
641 BUG(); 650 BUG();
642 return; 651 return;
diff --git a/arch/x86/platform/efi/efi_32.c b/arch/x86/platform/efi/efi_32.c
index 05ca14222463..9959657127f4 100644
--- a/arch/x86/platform/efi/efi_32.c
+++ b/arch/x86/platform/efi/efi_32.c
@@ -85,10 +85,9 @@ pgd_t * __init efi_call_phys_prolog(void)
85 85
86void __init efi_call_phys_epilog(pgd_t *save_pgd) 86void __init efi_call_phys_epilog(pgd_t *save_pgd)
87{ 87{
88 load_fixmap_gdt(0);
88 load_cr3(save_pgd); 89 load_cr3(save_pgd);
89 __flush_tlb_all(); 90 __flush_tlb_all();
90
91 load_fixmap_gdt(0);
92} 91}
93 92
94void __init efi_runtime_update_mappings(void) 93void __init efi_runtime_update_mappings(void)
diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c
index 2fe5c9b1816b..dd461c0167ef 100644
--- a/arch/x86/xen/mmu_pv.c
+++ b/arch/x86/xen/mmu_pv.c
@@ -1907,7 +1907,7 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
1907 /* L3_k[511] -> level2_fixmap_pgt */ 1907 /* L3_k[511] -> level2_fixmap_pgt */
1908 convert_pfn_mfn(level3_kernel_pgt); 1908 convert_pfn_mfn(level3_kernel_pgt);
1909 1909
1910 /* L3_k[511][506] -> level1_fixmap_pgt */ 1910 /* L3_k[511][508-FIXMAP_PMD_NUM ... 507] -> level1_fixmap_pgt */
1911 convert_pfn_mfn(level2_fixmap_pgt); 1911 convert_pfn_mfn(level2_fixmap_pgt);
1912 1912
1913 /* We get [511][511] and have Xen's version of level2_kernel_pgt */ 1913 /* We get [511][511] and have Xen's version of level2_kernel_pgt */
@@ -1952,7 +1952,11 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
1952 set_page_prot(level2_ident_pgt, PAGE_KERNEL_RO); 1952 set_page_prot(level2_ident_pgt, PAGE_KERNEL_RO);
1953 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO); 1953 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
1954 set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO); 1954 set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
1955 set_page_prot(level1_fixmap_pgt, PAGE_KERNEL_RO); 1955
1956 for (i = 0; i < FIXMAP_PMD_NUM; i++) {
1957 set_page_prot(level1_fixmap_pgt + i * PTRS_PER_PTE,
1958 PAGE_KERNEL_RO);
1959 }
1956 1960
1957 /* Pin down new L4 */ 1961 /* Pin down new L4 */
1958 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE, 1962 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c
index 7d00d4ad44d4..95997e6c0696 100644
--- a/arch/x86/xen/pmu.c
+++ b/arch/x86/xen/pmu.c
@@ -478,7 +478,7 @@ static void xen_convert_regs(const struct xen_pmu_regs *xen_regs,
478irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id) 478irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
479{ 479{
480 int err, ret = IRQ_NONE; 480 int err, ret = IRQ_NONE;
481 struct pt_regs regs; 481 struct pt_regs regs = {0};
482 const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); 482 const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
483 uint8_t xenpmu_flags = get_xenpmu_flags(); 483 uint8_t xenpmu_flags = get_xenpmu_flags();
484 484
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-27 00:42:08 -0400