aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/kvm/mmu.c
diff options
context:
space:
mode:
authorAvi Kivity <avi@qumranet.com>2007-12-16 04:02:48 -0500
committerAvi Kivity <avi@qumranet.com>2008-01-30 11:01:18 -0500
commitedf884172e9828c6234b254208af04655855038d (patch)
treef5e5d1eecaed9737eced6ba60d09fe93149751c1 /drivers/kvm/mmu.c
parent9584bf2c93f56656dba0de8f6c75b54ca7995143 (diff)
KVM: Move arch dependent files to new directory arch/x86/kvm/
This paves the way for multiple architecture support. Note that while ioapic.c could potentially be shared with ia64, it is also moved. Signed-off-by: Avi Kivity <avi@qumranet.com>
Diffstat (limited to 'drivers/kvm/mmu.c')
-rw-r--r--drivers/kvm/mmu.c1806
1 files changed, 0 insertions, 1806 deletions
diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c
deleted file mode 100644
index c26d83f86a3a..000000000000
--- a/drivers/kvm/mmu.c
+++ /dev/null
@@ -1,1806 +0,0 @@
1/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 *
11 * Authors:
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
17 *
18 */
19
20#include "vmx.h"
21#include "kvm.h"
22#include "x86.h"
23#include "mmu.h"
24
25#include <linux/types.h>
26#include <linux/string.h>
27#include <linux/mm.h>
28#include <linux/highmem.h>
29#include <linux/module.h>
30#include <linux/swap.h>
31
32#include <asm/page.h>
33#include <asm/cmpxchg.h>
34#include <asm/io.h>
35
36#undef MMU_DEBUG
37
38#undef AUDIT
39
40#ifdef AUDIT
41static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
42#else
43static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
44#endif
45
46#ifdef MMU_DEBUG
47
48#define pgprintk(x...) do { if (dbg) printk(x); } while (0)
49#define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
50
51#else
52
53#define pgprintk(x...) do { } while (0)
54#define rmap_printk(x...) do { } while (0)
55
56#endif
57
58#if defined(MMU_DEBUG) || defined(AUDIT)
59static int dbg = 1;
60#endif
61
62#ifndef MMU_DEBUG
63#define ASSERT(x) do { } while (0)
64#else
65#define ASSERT(x) \
66 if (!(x)) { \
67 printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
68 __FILE__, __LINE__, #x); \
69 }
70#endif
71
72#define PT64_PT_BITS 9
73#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
74#define PT32_PT_BITS 10
75#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
76
77#define PT_WRITABLE_SHIFT 1
78
79#define PT_PRESENT_MASK (1ULL << 0)
80#define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
81#define PT_USER_MASK (1ULL << 2)
82#define PT_PWT_MASK (1ULL << 3)
83#define PT_PCD_MASK (1ULL << 4)
84#define PT_ACCESSED_MASK (1ULL << 5)
85#define PT_DIRTY_MASK (1ULL << 6)
86#define PT_PAGE_SIZE_MASK (1ULL << 7)
87#define PT_PAT_MASK (1ULL << 7)
88#define PT_GLOBAL_MASK (1ULL << 8)
89#define PT64_NX_SHIFT 63
90#define PT64_NX_MASK (1ULL << PT64_NX_SHIFT)
91
92#define PT_PAT_SHIFT 7
93#define PT_DIR_PAT_SHIFT 12
94#define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
95
96#define PT32_DIR_PSE36_SIZE 4
97#define PT32_DIR_PSE36_SHIFT 13
98#define PT32_DIR_PSE36_MASK \
99 (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
100
101
102#define PT_FIRST_AVAIL_BITS_SHIFT 9
103#define PT64_SECOND_AVAIL_BITS_SHIFT 52
104
105#define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
106
107#define VALID_PAGE(x) ((x) != INVALID_PAGE)
108
109#define PT64_LEVEL_BITS 9
110
111#define PT64_LEVEL_SHIFT(level) \
112 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
113
114#define PT64_LEVEL_MASK(level) \
115 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
116
117#define PT64_INDEX(address, level)\
118 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
119
120
121#define PT32_LEVEL_BITS 10
122
123#define PT32_LEVEL_SHIFT(level) \
124 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
125
126#define PT32_LEVEL_MASK(level) \
127 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
128
129#define PT32_INDEX(address, level)\
130 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
131
132
133#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
134#define PT64_DIR_BASE_ADDR_MASK \
135 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
136
137#define PT32_BASE_ADDR_MASK PAGE_MASK
138#define PT32_DIR_BASE_ADDR_MASK \
139 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
140
141#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
142 | PT64_NX_MASK)
143
144#define PFERR_PRESENT_MASK (1U << 0)
145#define PFERR_WRITE_MASK (1U << 1)
146#define PFERR_USER_MASK (1U << 2)
147#define PFERR_FETCH_MASK (1U << 4)
148
149#define PT64_ROOT_LEVEL 4
150#define PT32_ROOT_LEVEL 2
151#define PT32E_ROOT_LEVEL 3
152
153#define PT_DIRECTORY_LEVEL 2
154#define PT_PAGE_TABLE_LEVEL 1
155
156#define RMAP_EXT 4
157
158#define ACC_EXEC_MASK 1
159#define ACC_WRITE_MASK PT_WRITABLE_MASK
160#define ACC_USER_MASK PT_USER_MASK
161#define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
162
163struct kvm_rmap_desc {
164 u64 *shadow_ptes[RMAP_EXT];
165 struct kvm_rmap_desc *more;
166};
167
168static struct kmem_cache *pte_chain_cache;
169static struct kmem_cache *rmap_desc_cache;
170static struct kmem_cache *mmu_page_header_cache;
171
172static u64 __read_mostly shadow_trap_nonpresent_pte;
173static u64 __read_mostly shadow_notrap_nonpresent_pte;
174
175void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
176{
177 shadow_trap_nonpresent_pte = trap_pte;
178 shadow_notrap_nonpresent_pte = notrap_pte;
179}
180EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
181
182static int is_write_protection(struct kvm_vcpu *vcpu)
183{
184 return vcpu->arch.cr0 & X86_CR0_WP;
185}
186
187static int is_cpuid_PSE36(void)
188{
189 return 1;
190}
191
192static int is_nx(struct kvm_vcpu *vcpu)
193{
194 return vcpu->arch.shadow_efer & EFER_NX;
195}
196
197static int is_present_pte(unsigned long pte)
198{
199 return pte & PT_PRESENT_MASK;
200}
201
202static int is_shadow_present_pte(u64 pte)
203{
204 pte &= ~PT_SHADOW_IO_MARK;
205 return pte != shadow_trap_nonpresent_pte
206 && pte != shadow_notrap_nonpresent_pte;
207}
208
209static int is_writeble_pte(unsigned long pte)
210{
211 return pte & PT_WRITABLE_MASK;
212}
213
214static int is_dirty_pte(unsigned long pte)
215{
216 return pte & PT_DIRTY_MASK;
217}
218
219static int is_io_pte(unsigned long pte)
220{
221 return pte & PT_SHADOW_IO_MARK;
222}
223
224static int is_rmap_pte(u64 pte)
225{
226 return pte != shadow_trap_nonpresent_pte
227 && pte != shadow_notrap_nonpresent_pte;
228}
229
230static gfn_t pse36_gfn_delta(u32 gpte)
231{
232 int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
233
234 return (gpte & PT32_DIR_PSE36_MASK) << shift;
235}
236
237static void set_shadow_pte(u64 *sptep, u64 spte)
238{
239#ifdef CONFIG_X86_64
240 set_64bit((unsigned long *)sptep, spte);
241#else
242 set_64bit((unsigned long long *)sptep, spte);
243#endif
244}
245
246static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
247 struct kmem_cache *base_cache, int min)
248{
249 void *obj;
250
251 if (cache->nobjs >= min)
252 return 0;
253 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
254 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
255 if (!obj)
256 return -ENOMEM;
257 cache->objects[cache->nobjs++] = obj;
258 }
259 return 0;
260}
261
262static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
263{
264 while (mc->nobjs)
265 kfree(mc->objects[--mc->nobjs]);
266}
267
268static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
269 int min)
270{
271 struct page *page;
272
273 if (cache->nobjs >= min)
274 return 0;
275 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
276 page = alloc_page(GFP_KERNEL);
277 if (!page)
278 return -ENOMEM;
279 set_page_private(page, 0);
280 cache->objects[cache->nobjs++] = page_address(page);
281 }
282 return 0;
283}
284
285static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
286{
287 while (mc->nobjs)
288 free_page((unsigned long)mc->objects[--mc->nobjs]);
289}
290
291static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
292{
293 int r;
294
295 kvm_mmu_free_some_pages(vcpu);
296 r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
297 pte_chain_cache, 4);
298 if (r)
299 goto out;
300 r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
301 rmap_desc_cache, 1);
302 if (r)
303 goto out;
304 r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
305 if (r)
306 goto out;
307 r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
308 mmu_page_header_cache, 4);
309out:
310 return r;
311}
312
313static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
314{
315 mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache);
316 mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache);
317 mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
318 mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
319}
320
321static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
322 size_t size)
323{
324 void *p;
325
326 BUG_ON(!mc->nobjs);
327 p = mc->objects[--mc->nobjs];
328 memset(p, 0, size);
329 return p;
330}
331
332static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
333{
334 return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
335 sizeof(struct kvm_pte_chain));
336}
337
338static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
339{
340 kfree(pc);
341}
342
343static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
344{
345 return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache,
346 sizeof(struct kvm_rmap_desc));
347}
348
349static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
350{
351 kfree(rd);
352}
353
354/*
355 * Take gfn and return the reverse mapping to it.
356 * Note: gfn must be unaliased before this function get called
357 */
358
359static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn)
360{
361 struct kvm_memory_slot *slot;
362
363 slot = gfn_to_memslot(kvm, gfn);
364 return &slot->rmap[gfn - slot->base_gfn];
365}
366
367/*
368 * Reverse mapping data structures:
369 *
370 * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
371 * that points to page_address(page).
372 *
373 * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
374 * containing more mappings.
375 */
376static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
377{
378 struct kvm_mmu_page *sp;
379 struct kvm_rmap_desc *desc;
380 unsigned long *rmapp;
381 int i;
382
383 if (!is_rmap_pte(*spte))
384 return;
385 gfn = unalias_gfn(vcpu->kvm, gfn);
386 sp = page_header(__pa(spte));
387 sp->gfns[spte - sp->spt] = gfn;
388 rmapp = gfn_to_rmap(vcpu->kvm, gfn);
389 if (!*rmapp) {
390 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
391 *rmapp = (unsigned long)spte;
392 } else if (!(*rmapp & 1)) {
393 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
394 desc = mmu_alloc_rmap_desc(vcpu);
395 desc->shadow_ptes[0] = (u64 *)*rmapp;
396 desc->shadow_ptes[1] = spte;
397 *rmapp = (unsigned long)desc | 1;
398 } else {
399 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
400 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
401 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
402 desc = desc->more;
403 if (desc->shadow_ptes[RMAP_EXT-1]) {
404 desc->more = mmu_alloc_rmap_desc(vcpu);
405 desc = desc->more;
406 }
407 for (i = 0; desc->shadow_ptes[i]; ++i)
408 ;
409 desc->shadow_ptes[i] = spte;
410 }
411}
412
413static void rmap_desc_remove_entry(unsigned long *rmapp,
414 struct kvm_rmap_desc *desc,
415 int i,
416 struct kvm_rmap_desc *prev_desc)
417{
418 int j;
419
420 for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
421 ;
422 desc->shadow_ptes[i] = desc->shadow_ptes[j];
423 desc->shadow_ptes[j] = NULL;
424 if (j != 0)
425 return;
426 if (!prev_desc && !desc->more)
427 *rmapp = (unsigned long)desc->shadow_ptes[0];
428 else
429 if (prev_desc)
430 prev_desc->more = desc->more;
431 else
432 *rmapp = (unsigned long)desc->more | 1;
433 mmu_free_rmap_desc(desc);
434}
435
436static void rmap_remove(struct kvm *kvm, u64 *spte)
437{
438 struct kvm_rmap_desc *desc;
439 struct kvm_rmap_desc *prev_desc;
440 struct kvm_mmu_page *sp;
441 struct page *page;
442 unsigned long *rmapp;
443 int i;
444
445 if (!is_rmap_pte(*spte))
446 return;
447 sp = page_header(__pa(spte));
448 page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
449 mark_page_accessed(page);
450 if (is_writeble_pte(*spte))
451 kvm_release_page_dirty(page);
452 else
453 kvm_release_page_clean(page);
454 rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt]);
455 if (!*rmapp) {
456 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
457 BUG();
458 } else if (!(*rmapp & 1)) {
459 rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
460 if ((u64 *)*rmapp != spte) {
461 printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
462 spte, *spte);
463 BUG();
464 }
465 *rmapp = 0;
466 } else {
467 rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
468 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
469 prev_desc = NULL;
470 while (desc) {
471 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
472 if (desc->shadow_ptes[i] == spte) {
473 rmap_desc_remove_entry(rmapp,
474 desc, i,
475 prev_desc);
476 return;
477 }
478 prev_desc = desc;
479 desc = desc->more;
480 }
481 BUG();
482 }
483}
484
485static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
486{
487 struct kvm_rmap_desc *desc;
488 struct kvm_rmap_desc *prev_desc;
489 u64 *prev_spte;
490 int i;
491
492 if (!*rmapp)
493 return NULL;
494 else if (!(*rmapp & 1)) {
495 if (!spte)
496 return (u64 *)*rmapp;
497 return NULL;
498 }
499 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
500 prev_desc = NULL;
501 prev_spte = NULL;
502 while (desc) {
503 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) {
504 if (prev_spte == spte)
505 return desc->shadow_ptes[i];
506 prev_spte = desc->shadow_ptes[i];
507 }
508 desc = desc->more;
509 }
510 return NULL;
511}
512
513static void rmap_write_protect(struct kvm *kvm, u64 gfn)
514{
515 unsigned long *rmapp;
516 u64 *spte;
517
518 gfn = unalias_gfn(kvm, gfn);
519 rmapp = gfn_to_rmap(kvm, gfn);
520
521 spte = rmap_next(kvm, rmapp, NULL);
522 while (spte) {
523 BUG_ON(!spte);
524 BUG_ON(!(*spte & PT_PRESENT_MASK));
525 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
526 if (is_writeble_pte(*spte))
527 set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
528 kvm_flush_remote_tlbs(kvm);
529 spte = rmap_next(kvm, rmapp, spte);
530 }
531}
532
533#ifdef MMU_DEBUG
534static int is_empty_shadow_page(u64 *spt)
535{
536 u64 *pos;
537 u64 *end;
538
539 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
540 if ((*pos & ~PT_SHADOW_IO_MARK) != shadow_trap_nonpresent_pte) {
541 printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__,
542 pos, *pos);
543 return 0;
544 }
545 return 1;
546}
547#endif
548
549static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
550{
551 ASSERT(is_empty_shadow_page(sp->spt));
552 list_del(&sp->link);
553 __free_page(virt_to_page(sp->spt));
554 __free_page(virt_to_page(sp->gfns));
555 kfree(sp);
556 ++kvm->arch.n_free_mmu_pages;
557}
558
559static unsigned kvm_page_table_hashfn(gfn_t gfn)
560{
561 return gfn;
562}
563
564static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
565 u64 *parent_pte)
566{
567 struct kvm_mmu_page *sp;
568
569 if (!vcpu->kvm->arch.n_free_mmu_pages)
570 return NULL;
571
572 sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
573 sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
574 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
575 set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
576 list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
577 ASSERT(is_empty_shadow_page(sp->spt));
578 sp->slot_bitmap = 0;
579 sp->multimapped = 0;
580 sp->parent_pte = parent_pte;
581 --vcpu->kvm->arch.n_free_mmu_pages;
582 return sp;
583}
584
585static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
586 struct kvm_mmu_page *sp, u64 *parent_pte)
587{
588 struct kvm_pte_chain *pte_chain;
589 struct hlist_node *node;
590 int i;
591
592 if (!parent_pte)
593 return;
594 if (!sp->multimapped) {
595 u64 *old = sp->parent_pte;
596
597 if (!old) {
598 sp->parent_pte = parent_pte;
599 return;
600 }
601 sp->multimapped = 1;
602 pte_chain = mmu_alloc_pte_chain(vcpu);
603 INIT_HLIST_HEAD(&sp->parent_ptes);
604 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
605 pte_chain->parent_ptes[0] = old;
606 }
607 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
608 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
609 continue;
610 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
611 if (!pte_chain->parent_ptes[i]) {
612 pte_chain->parent_ptes[i] = parent_pte;
613 return;
614 }
615 }
616 pte_chain = mmu_alloc_pte_chain(vcpu);
617 BUG_ON(!pte_chain);
618 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
619 pte_chain->parent_ptes[0] = parent_pte;
620}
621
622static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
623 u64 *parent_pte)
624{
625 struct kvm_pte_chain *pte_chain;
626 struct hlist_node *node;
627 int i;
628
629 if (!sp->multimapped) {
630 BUG_ON(sp->parent_pte != parent_pte);
631 sp->parent_pte = NULL;
632 return;
633 }
634 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
635 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
636 if (!pte_chain->parent_ptes[i])
637 break;
638 if (pte_chain->parent_ptes[i] != parent_pte)
639 continue;
640 while (i + 1 < NR_PTE_CHAIN_ENTRIES
641 && pte_chain->parent_ptes[i + 1]) {
642 pte_chain->parent_ptes[i]
643 = pte_chain->parent_ptes[i + 1];
644 ++i;
645 }
646 pte_chain->parent_ptes[i] = NULL;
647 if (i == 0) {
648 hlist_del(&pte_chain->link);
649 mmu_free_pte_chain(pte_chain);
650 if (hlist_empty(&sp->parent_ptes)) {
651 sp->multimapped = 0;
652 sp->parent_pte = NULL;
653 }
654 }
655 return;
656 }
657 BUG();
658}
659
660static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn)
661{
662 unsigned index;
663 struct hlist_head *bucket;
664 struct kvm_mmu_page *sp;
665 struct hlist_node *node;
666
667 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
668 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
669 bucket = &kvm->arch.mmu_page_hash[index];
670 hlist_for_each_entry(sp, node, bucket, hash_link)
671 if (sp->gfn == gfn && !sp->role.metaphysical) {
672 pgprintk("%s: found role %x\n",
673 __FUNCTION__, sp->role.word);
674 return sp;
675 }
676 return NULL;
677}
678
679static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
680 gfn_t gfn,
681 gva_t gaddr,
682 unsigned level,
683 int metaphysical,
684 unsigned access,
685 u64 *parent_pte,
686 bool *new_page)
687{
688 union kvm_mmu_page_role role;
689 unsigned index;
690 unsigned quadrant;
691 struct hlist_head *bucket;
692 struct kvm_mmu_page *sp;
693 struct hlist_node *node;
694
695 role.word = 0;
696 role.glevels = vcpu->arch.mmu.root_level;
697 role.level = level;
698 role.metaphysical = metaphysical;
699 role.access = access;
700 if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
701 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
702 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
703 role.quadrant = quadrant;
704 }
705 pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__,
706 gfn, role.word);
707 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
708 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
709 hlist_for_each_entry(sp, node, bucket, hash_link)
710 if (sp->gfn == gfn && sp->role.word == role.word) {
711 mmu_page_add_parent_pte(vcpu, sp, parent_pte);
712 pgprintk("%s: found\n", __FUNCTION__);
713 return sp;
714 }
715 sp = kvm_mmu_alloc_page(vcpu, parent_pte);
716 if (!sp)
717 return sp;
718 pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word);
719 sp->gfn = gfn;
720 sp->role = role;
721 hlist_add_head(&sp->hash_link, bucket);
722 vcpu->arch.mmu.prefetch_page(vcpu, sp);
723 if (!metaphysical)
724 rmap_write_protect(vcpu->kvm, gfn);
725 if (new_page)
726 *new_page = 1;
727 return sp;
728}
729
730static void kvm_mmu_page_unlink_children(struct kvm *kvm,
731 struct kvm_mmu_page *sp)
732{
733 unsigned i;
734 u64 *pt;
735 u64 ent;
736
737 pt = sp->spt;
738
739 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
740 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
741 if (is_shadow_present_pte(pt[i]))
742 rmap_remove(kvm, &pt[i]);
743 pt[i] = shadow_trap_nonpresent_pte;
744 }
745 kvm_flush_remote_tlbs(kvm);
746 return;
747 }
748
749 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
750 ent = pt[i];
751
752 pt[i] = shadow_trap_nonpresent_pte;
753 if (!is_shadow_present_pte(ent))
754 continue;
755 ent &= PT64_BASE_ADDR_MASK;
756 mmu_page_remove_parent_pte(page_header(ent), &pt[i]);
757 }
758 kvm_flush_remote_tlbs(kvm);
759}
760
761static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
762{
763 mmu_page_remove_parent_pte(sp, parent_pte);
764}
765
766static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
767{
768 int i;
769
770 for (i = 0; i < KVM_MAX_VCPUS; ++i)
771 if (kvm->vcpus[i])
772 kvm->vcpus[i]->arch.last_pte_updated = NULL;
773}
774
775static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp)
776{
777 u64 *parent_pte;
778
779 ++kvm->stat.mmu_shadow_zapped;
780 while (sp->multimapped || sp->parent_pte) {
781 if (!sp->multimapped)
782 parent_pte = sp->parent_pte;
783 else {
784 struct kvm_pte_chain *chain;
785
786 chain = container_of(sp->parent_ptes.first,
787 struct kvm_pte_chain, link);
788 parent_pte = chain->parent_ptes[0];
789 }
790 BUG_ON(!parent_pte);
791 kvm_mmu_put_page(sp, parent_pte);
792 set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte);
793 }
794 kvm_mmu_page_unlink_children(kvm, sp);
795 if (!sp->root_count) {
796 hlist_del(&sp->hash_link);
797 kvm_mmu_free_page(kvm, sp);
798 } else
799 list_move(&sp->link, &kvm->arch.active_mmu_pages);
800 kvm_mmu_reset_last_pte_updated(kvm);
801}
802
803/*
804 * Changing the number of mmu pages allocated to the vm
805 * Note: if kvm_nr_mmu_pages is too small, you will get dead lock
806 */
807void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages)
808{
809 /*
810 * If we set the number of mmu pages to be smaller be than the
811 * number of actived pages , we must to free some mmu pages before we
812 * change the value
813 */
814
815 if ((kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages) >
816 kvm_nr_mmu_pages) {
817 int n_used_mmu_pages = kvm->arch.n_alloc_mmu_pages
818 - kvm->arch.n_free_mmu_pages;
819
820 while (n_used_mmu_pages > kvm_nr_mmu_pages) {
821 struct kvm_mmu_page *page;
822
823 page = container_of(kvm->arch.active_mmu_pages.prev,
824 struct kvm_mmu_page, link);
825 kvm_mmu_zap_page(kvm, page);
826 n_used_mmu_pages--;
827 }
828 kvm->arch.n_free_mmu_pages = 0;
829 }
830 else
831 kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages
832 - kvm->arch.n_alloc_mmu_pages;
833
834 kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages;
835}
836
837static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
838{
839 unsigned index;
840 struct hlist_head *bucket;
841 struct kvm_mmu_page *sp;
842 struct hlist_node *node, *n;
843 int r;
844
845 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
846 r = 0;
847 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
848 bucket = &kvm->arch.mmu_page_hash[index];
849 hlist_for_each_entry_safe(sp, node, n, bucket, hash_link)
850 if (sp->gfn == gfn && !sp->role.metaphysical) {
851 pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn,
852 sp->role.word);
853 kvm_mmu_zap_page(kvm, sp);
854 r = 1;
855 }
856 return r;
857}
858
859static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
860{
861 struct kvm_mmu_page *sp;
862
863 while ((sp = kvm_mmu_lookup_page(kvm, gfn)) != NULL) {
864 pgprintk("%s: zap %lx %x\n", __FUNCTION__, gfn, sp->role.word);
865 kvm_mmu_zap_page(kvm, sp);
866 }
867}
868
869static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
870{
871 int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn));
872 struct kvm_mmu_page *sp = page_header(__pa(pte));
873
874 __set_bit(slot, &sp->slot_bitmap);
875}
876
877struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
878{
879 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
880
881 if (gpa == UNMAPPED_GVA)
882 return NULL;
883 return gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
884}
885
886static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
887 unsigned pt_access, unsigned pte_access,
888 int user_fault, int write_fault, int dirty,
889 int *ptwrite, gfn_t gfn)
890{
891 u64 spte;
892 int was_rmapped = is_rmap_pte(*shadow_pte);
893 struct page *page;
894
895 pgprintk("%s: spte %llx access %x write_fault %d"
896 " user_fault %d gfn %lx\n",
897 __FUNCTION__, *shadow_pte, pt_access,
898 write_fault, user_fault, gfn);
899
900 /*
901 * We don't set the accessed bit, since we sometimes want to see
902 * whether the guest actually used the pte (in order to detect
903 * demand paging).
904 */
905 spte = PT_PRESENT_MASK | PT_DIRTY_MASK;
906 if (!dirty)
907 pte_access &= ~ACC_WRITE_MASK;
908 if (!(pte_access & ACC_EXEC_MASK))
909 spte |= PT64_NX_MASK;
910
911 page = gfn_to_page(vcpu->kvm, gfn);
912
913 spte |= PT_PRESENT_MASK;
914 if (pte_access & ACC_USER_MASK)
915 spte |= PT_USER_MASK;
916
917 if (is_error_page(page)) {
918 set_shadow_pte(shadow_pte,
919 shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK);
920 kvm_release_page_clean(page);
921 return;
922 }
923
924 spte |= page_to_phys(page);
925
926 if ((pte_access & ACC_WRITE_MASK)
927 || (write_fault && !is_write_protection(vcpu) && !user_fault)) {
928 struct kvm_mmu_page *shadow;
929
930 spte |= PT_WRITABLE_MASK;
931 if (user_fault) {
932 mmu_unshadow(vcpu->kvm, gfn);
933 goto unshadowed;
934 }
935
936 shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn);
937 if (shadow) {
938 pgprintk("%s: found shadow page for %lx, marking ro\n",
939 __FUNCTION__, gfn);
940 pte_access &= ~ACC_WRITE_MASK;
941 if (is_writeble_pte(spte)) {
942 spte &= ~PT_WRITABLE_MASK;
943 kvm_x86_ops->tlb_flush(vcpu);
944 }
945 if (write_fault)
946 *ptwrite = 1;
947 }
948 }
949
950unshadowed:
951
952 if (pte_access & ACC_WRITE_MASK)
953 mark_page_dirty(vcpu->kvm, gfn);
954
955 pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte);
956 set_shadow_pte(shadow_pte, spte);
957 page_header_update_slot(vcpu->kvm, shadow_pte, gfn);
958 if (!was_rmapped) {
959 rmap_add(vcpu, shadow_pte, gfn);
960 if (!is_rmap_pte(*shadow_pte))
961 kvm_release_page_clean(page);
962 }
963 else
964 kvm_release_page_clean(page);
965 if (!ptwrite || !*ptwrite)
966 vcpu->arch.last_pte_updated = shadow_pte;
967}
968
969static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
970{
971}
972
973static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
974{
975 int level = PT32E_ROOT_LEVEL;
976 hpa_t table_addr = vcpu->arch.mmu.root_hpa;
977 int pt_write = 0;
978
979 for (; ; level--) {
980 u32 index = PT64_INDEX(v, level);
981 u64 *table;
982
983 ASSERT(VALID_PAGE(table_addr));
984 table = __va(table_addr);
985
986 if (level == 1) {
987 mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL,
988 0, write, 1, &pt_write, gfn);
989 return pt_write || is_io_pte(table[index]);
990 }
991
992 if (table[index] == shadow_trap_nonpresent_pte) {
993 struct kvm_mmu_page *new_table;
994 gfn_t pseudo_gfn;
995
996 pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
997 >> PAGE_SHIFT;
998 new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
999 v, level - 1,
1000 1, ACC_ALL, &table[index],
1001 NULL);
1002 if (!new_table) {
1003 pgprintk("nonpaging_map: ENOMEM\n");
1004 return -ENOMEM;
1005 }
1006
1007 table[index] = __pa(new_table->spt) | PT_PRESENT_MASK
1008 | PT_WRITABLE_MASK | PT_USER_MASK;
1009 }
1010 table_addr = table[index] & PT64_BASE_ADDR_MASK;
1011 }
1012}
1013
1014static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
1015 struct kvm_mmu_page *sp)
1016{
1017 int i;
1018
1019 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1020 sp->spt[i] = shadow_trap_nonpresent_pte;
1021}
1022
1023static void mmu_free_roots(struct kvm_vcpu *vcpu)
1024{
1025 int i;
1026 struct kvm_mmu_page *sp;
1027
1028 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
1029 return;
1030#ifdef CONFIG_X86_64
1031 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1032 hpa_t root = vcpu->arch.mmu.root_hpa;
1033
1034 sp = page_header(root);
1035 --sp->root_count;
1036 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1037 return;
1038 }
1039#endif
1040 for (i = 0; i < 4; ++i) {
1041 hpa_t root = vcpu->arch.mmu.pae_root[i];
1042
1043 if (root) {
1044 root &= PT64_BASE_ADDR_MASK;
1045 sp = page_header(root);
1046 --sp->root_count;
1047 }
1048 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
1049 }
1050 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1051}
1052
1053static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
1054{
1055 int i;
1056 gfn_t root_gfn;
1057 struct kvm_mmu_page *sp;
1058
1059 root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT;
1060
1061#ifdef CONFIG_X86_64
1062 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1063 hpa_t root = vcpu->arch.mmu.root_hpa;
1064
1065 ASSERT(!VALID_PAGE(root));
1066 sp = kvm_mmu_get_page(vcpu, root_gfn, 0,
1067 PT64_ROOT_LEVEL, 0, ACC_ALL, NULL, NULL);
1068 root = __pa(sp->spt);
1069 ++sp->root_count;
1070 vcpu->arch.mmu.root_hpa = root;
1071 return;
1072 }
1073#endif
1074 for (i = 0; i < 4; ++i) {
1075 hpa_t root = vcpu->arch.mmu.pae_root[i];
1076
1077 ASSERT(!VALID_PAGE(root));
1078 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
1079 if (!is_present_pte(vcpu->arch.pdptrs[i])) {
1080 vcpu->arch.mmu.pae_root[i] = 0;
1081 continue;
1082 }
1083 root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT;
1084 } else if (vcpu->arch.mmu.root_level == 0)
1085 root_gfn = 0;
1086 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
1087 PT32_ROOT_LEVEL, !is_paging(vcpu),
1088 ACC_ALL, NULL, NULL);
1089 root = __pa(sp->spt);
1090 ++sp->root_count;
1091 vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
1092 }
1093 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
1094}
1095
1096static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
1097{
1098 return vaddr;
1099}
1100
1101static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
1102 u32 error_code)
1103{
1104 gfn_t gfn;
1105 int r;
1106
1107 pgprintk("%s: gva %lx error %x\n", __FUNCTION__, gva, error_code);
1108 r = mmu_topup_memory_caches(vcpu);
1109 if (r)
1110 return r;
1111
1112 ASSERT(vcpu);
1113 ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
1114
1115 gfn = gva >> PAGE_SHIFT;
1116
1117 return nonpaging_map(vcpu, gva & PAGE_MASK,
1118 error_code & PFERR_WRITE_MASK, gfn);
1119}
1120
1121static void nonpaging_free(struct kvm_vcpu *vcpu)
1122{
1123 mmu_free_roots(vcpu);
1124}
1125
1126static int nonpaging_init_context(struct kvm_vcpu *vcpu)
1127{
1128 struct kvm_mmu *context = &vcpu->arch.mmu;
1129
1130 context->new_cr3 = nonpaging_new_cr3;
1131 context->page_fault = nonpaging_page_fault;
1132 context->gva_to_gpa = nonpaging_gva_to_gpa;
1133 context->free = nonpaging_free;
1134 context->prefetch_page = nonpaging_prefetch_page;
1135 context->root_level = 0;
1136 context->shadow_root_level = PT32E_ROOT_LEVEL;
1137 context->root_hpa = INVALID_PAGE;
1138 return 0;
1139}
1140
1141void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
1142{
1143 ++vcpu->stat.tlb_flush;
1144 kvm_x86_ops->tlb_flush(vcpu);
1145}
1146
1147static void paging_new_cr3(struct kvm_vcpu *vcpu)
1148{
1149 pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
1150 mmu_free_roots(vcpu);
1151}
1152
1153static void inject_page_fault(struct kvm_vcpu *vcpu,
1154 u64 addr,
1155 u32 err_code)
1156{
1157 kvm_inject_page_fault(vcpu, addr, err_code);
1158}
1159
1160static void paging_free(struct kvm_vcpu *vcpu)
1161{
1162 nonpaging_free(vcpu);
1163}
1164
1165#define PTTYPE 64
1166#include "paging_tmpl.h"
1167#undef PTTYPE
1168
1169#define PTTYPE 32
1170#include "paging_tmpl.h"
1171#undef PTTYPE
1172
1173static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
1174{
1175 struct kvm_mmu *context = &vcpu->arch.mmu;
1176
1177 ASSERT(is_pae(vcpu));
1178 context->new_cr3 = paging_new_cr3;
1179 context->page_fault = paging64_page_fault;
1180 context->gva_to_gpa = paging64_gva_to_gpa;
1181 context->prefetch_page = paging64_prefetch_page;
1182 context->free = paging_free;
1183 context->root_level = level;
1184 context->shadow_root_level = level;
1185 context->root_hpa = INVALID_PAGE;
1186 return 0;
1187}
1188
1189static int paging64_init_context(struct kvm_vcpu *vcpu)
1190{
1191 return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
1192}
1193
1194static int paging32_init_context(struct kvm_vcpu *vcpu)
1195{
1196 struct kvm_mmu *context = &vcpu->arch.mmu;
1197
1198 context->new_cr3 = paging_new_cr3;
1199 context->page_fault = paging32_page_fault;
1200 context->gva_to_gpa = paging32_gva_to_gpa;
1201 context->free = paging_free;
1202 context->prefetch_page = paging32_prefetch_page;
1203 context->root_level = PT32_ROOT_LEVEL;
1204 context->shadow_root_level = PT32E_ROOT_LEVEL;
1205 context->root_hpa = INVALID_PAGE;
1206 return 0;
1207}
1208
1209static int paging32E_init_context(struct kvm_vcpu *vcpu)
1210{
1211 return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
1212}
1213
1214static int init_kvm_mmu(struct kvm_vcpu *vcpu)
1215{
1216 ASSERT(vcpu);
1217 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
1218
1219 if (!is_paging(vcpu))
1220 return nonpaging_init_context(vcpu);
1221 else if (is_long_mode(vcpu))
1222 return paging64_init_context(vcpu);
1223 else if (is_pae(vcpu))
1224 return paging32E_init_context(vcpu);
1225 else
1226 return paging32_init_context(vcpu);
1227}
1228
1229static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
1230{
1231 ASSERT(vcpu);
1232 if (VALID_PAGE(vcpu->arch.mmu.root_hpa)) {
1233 vcpu->arch.mmu.free(vcpu);
1234 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1235 }
1236}
1237
1238int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
1239{
1240 destroy_kvm_mmu(vcpu);
1241 return init_kvm_mmu(vcpu);
1242}
1243EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
1244
1245int kvm_mmu_load(struct kvm_vcpu *vcpu)
1246{
1247 int r;
1248
1249 mutex_lock(&vcpu->kvm->lock);
1250 r = mmu_topup_memory_caches(vcpu);
1251 if (r)
1252 goto out;
1253 mmu_alloc_roots(vcpu);
1254 kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
1255 kvm_mmu_flush_tlb(vcpu);
1256out:
1257 mutex_unlock(&vcpu->kvm->lock);
1258 return r;
1259}
1260EXPORT_SYMBOL_GPL(kvm_mmu_load);
1261
1262void kvm_mmu_unload(struct kvm_vcpu *vcpu)
1263{
1264 mmu_free_roots(vcpu);
1265}
1266
1267static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
1268 struct kvm_mmu_page *sp,
1269 u64 *spte)
1270{
1271 u64 pte;
1272 struct kvm_mmu_page *child;
1273
1274 pte = *spte;
1275 if (is_shadow_present_pte(pte)) {
1276 if (sp->role.level == PT_PAGE_TABLE_LEVEL)
1277 rmap_remove(vcpu->kvm, spte);
1278 else {
1279 child = page_header(pte & PT64_BASE_ADDR_MASK);
1280 mmu_page_remove_parent_pte(child, spte);
1281 }
1282 }
1283 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
1284}
1285
1286static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
1287 struct kvm_mmu_page *sp,
1288 u64 *spte,
1289 const void *new, int bytes,
1290 int offset_in_pte)
1291{
1292 if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
1293 ++vcpu->kvm->stat.mmu_pde_zapped;
1294 return;
1295 }
1296
1297 ++vcpu->kvm->stat.mmu_pte_updated;
1298 if (sp->role.glevels == PT32_ROOT_LEVEL)
1299 paging32_update_pte(vcpu, sp, spte, new, bytes, offset_in_pte);
1300 else
1301 paging64_update_pte(vcpu, sp, spte, new, bytes, offset_in_pte);
1302}
1303
1304static bool need_remote_flush(u64 old, u64 new)
1305{
1306 if (!is_shadow_present_pte(old))
1307 return false;
1308 if (!is_shadow_present_pte(new))
1309 return true;
1310 if ((old ^ new) & PT64_BASE_ADDR_MASK)
1311 return true;
1312 old ^= PT64_NX_MASK;
1313 new ^= PT64_NX_MASK;
1314 return (old & ~new & PT64_PERM_MASK) != 0;
1315}
1316
1317static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new)
1318{
1319 if (need_remote_flush(old, new))
1320 kvm_flush_remote_tlbs(vcpu->kvm);
1321 else
1322 kvm_mmu_flush_tlb(vcpu);
1323}
1324
1325static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
1326{
1327 u64 *spte = vcpu->arch.last_pte_updated;
1328
1329 return !!(spte && (*spte & PT_ACCESSED_MASK));
1330}
1331
1332void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1333 const u8 *new, int bytes)
1334{
1335 gfn_t gfn = gpa >> PAGE_SHIFT;
1336 struct kvm_mmu_page *sp;
1337 struct hlist_node *node, *n;
1338 struct hlist_head *bucket;
1339 unsigned index;
1340 u64 entry;
1341 u64 *spte;
1342 unsigned offset = offset_in_page(gpa);
1343 unsigned pte_size;
1344 unsigned page_offset;
1345 unsigned misaligned;
1346 unsigned quadrant;
1347 int level;
1348 int flooded = 0;
1349 int npte;
1350
1351 pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes);
1352 ++vcpu->kvm->stat.mmu_pte_write;
1353 kvm_mmu_audit(vcpu, "pre pte write");
1354 if (gfn == vcpu->arch.last_pt_write_gfn
1355 && !last_updated_pte_accessed(vcpu)) {
1356 ++vcpu->arch.last_pt_write_count;
1357 if (vcpu->arch.last_pt_write_count >= 3)
1358 flooded = 1;
1359 } else {
1360 vcpu->arch.last_pt_write_gfn = gfn;
1361 vcpu->arch.last_pt_write_count = 1;
1362 vcpu->arch.last_pte_updated = NULL;
1363 }
1364 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
1365 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
1366 hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) {
1367 if (sp->gfn != gfn || sp->role.metaphysical)
1368 continue;
1369 pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
1370 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
1371 misaligned |= bytes < 4;
1372 if (misaligned || flooded) {
1373 /*
1374 * Misaligned accesses are too much trouble to fix
1375 * up; also, they usually indicate a page is not used
1376 * as a page table.
1377 *
1378 * If we're seeing too many writes to a page,
1379 * it may no longer be a page table, or we may be
1380 * forking, in which case it is better to unmap the
1381 * page.
1382 */
1383 pgprintk("misaligned: gpa %llx bytes %d role %x\n",
1384 gpa, bytes, sp->role.word);
1385 kvm_mmu_zap_page(vcpu->kvm, sp);
1386 ++vcpu->kvm->stat.mmu_flooded;
1387 continue;
1388 }
1389 page_offset = offset;
1390 level = sp->role.level;
1391 npte = 1;
1392 if (sp->role.glevels == PT32_ROOT_LEVEL) {
1393 page_offset <<= 1; /* 32->64 */
1394 /*
1395 * A 32-bit pde maps 4MB while the shadow pdes map
1396 * only 2MB. So we need to double the offset again
1397 * and zap two pdes instead of one.
1398 */
1399 if (level == PT32_ROOT_LEVEL) {
1400 page_offset &= ~7; /* kill rounding error */
1401 page_offset <<= 1;
1402 npte = 2;
1403 }
1404 quadrant = page_offset >> PAGE_SHIFT;
1405 page_offset &= ~PAGE_MASK;
1406 if (quadrant != sp->role.quadrant)
1407 continue;
1408 }
1409 spte = &sp->spt[page_offset / sizeof(*spte)];
1410 while (npte--) {
1411 entry = *spte;
1412 mmu_pte_write_zap_pte(vcpu, sp, spte);
1413 mmu_pte_write_new_pte(vcpu, sp, spte, new, bytes,
1414 page_offset & (pte_size - 1));
1415 mmu_pte_write_flush_tlb(vcpu, entry, *spte);
1416 ++spte;
1417 }
1418 }
1419 kvm_mmu_audit(vcpu, "post pte write");
1420}
1421
1422int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
1423{
1424 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
1425
1426 return kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1427}
1428
1429void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
1430{
1431 while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) {
1432 struct kvm_mmu_page *sp;
1433
1434 sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
1435 struct kvm_mmu_page, link);
1436 kvm_mmu_zap_page(vcpu->kvm, sp);
1437 ++vcpu->kvm->stat.mmu_recycled;
1438 }
1439}
1440
1441int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
1442{
1443 int r;
1444 enum emulation_result er;
1445
1446 mutex_lock(&vcpu->kvm->lock);
1447 r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
1448 if (r < 0)
1449 goto out;
1450
1451 if (!r) {
1452 r = 1;
1453 goto out;
1454 }
1455
1456 r = mmu_topup_memory_caches(vcpu);
1457 if (r)
1458 goto out;
1459
1460 er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0);
1461 mutex_unlock(&vcpu->kvm->lock);
1462
1463 switch (er) {
1464 case EMULATE_DONE:
1465 return 1;
1466 case EMULATE_DO_MMIO:
1467 ++vcpu->stat.mmio_exits;
1468 return 0;
1469 case EMULATE_FAIL:
1470 kvm_report_emulation_failure(vcpu, "pagetable");
1471 return 1;
1472 default:
1473 BUG();
1474 }
1475out:
1476 mutex_unlock(&vcpu->kvm->lock);
1477 return r;
1478}
1479EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
1480
1481static void free_mmu_pages(struct kvm_vcpu *vcpu)
1482{
1483 struct kvm_mmu_page *sp;
1484
1485 while (!list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
1486 sp = container_of(vcpu->kvm->arch.active_mmu_pages.next,
1487 struct kvm_mmu_page, link);
1488 kvm_mmu_zap_page(vcpu->kvm, sp);
1489 }
1490 free_page((unsigned long)vcpu->arch.mmu.pae_root);
1491}
1492
1493static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
1494{
1495 struct page *page;
1496 int i;
1497
1498 ASSERT(vcpu);
1499
1500 if (vcpu->kvm->arch.n_requested_mmu_pages)
1501 vcpu->kvm->arch.n_free_mmu_pages =
1502 vcpu->kvm->arch.n_requested_mmu_pages;
1503 else
1504 vcpu->kvm->arch.n_free_mmu_pages =
1505 vcpu->kvm->arch.n_alloc_mmu_pages;
1506 /*
1507 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
1508 * Therefore we need to allocate shadow page tables in the first
1509 * 4GB of memory, which happens to fit the DMA32 zone.
1510 */
1511 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
1512 if (!page)
1513 goto error_1;
1514 vcpu->arch.mmu.pae_root = page_address(page);
1515 for (i = 0; i < 4; ++i)
1516 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
1517
1518 return 0;
1519
1520error_1:
1521 free_mmu_pages(vcpu);
1522 return -ENOMEM;
1523}
1524
1525int kvm_mmu_create(struct kvm_vcpu *vcpu)
1526{
1527 ASSERT(vcpu);
1528 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
1529
1530 return alloc_mmu_pages(vcpu);
1531}
1532
1533int kvm_mmu_setup(struct kvm_vcpu *vcpu)
1534{
1535 ASSERT(vcpu);
1536 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
1537
1538 return init_kvm_mmu(vcpu);
1539}
1540
1541void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
1542{
1543 ASSERT(vcpu);
1544
1545 destroy_kvm_mmu(vcpu);
1546 free_mmu_pages(vcpu);
1547 mmu_free_memory_caches(vcpu);
1548}
1549
1550void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
1551{
1552 struct kvm_mmu_page *sp;
1553
1554 list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
1555 int i;
1556 u64 *pt;
1557
1558 if (!test_bit(slot, &sp->slot_bitmap))
1559 continue;
1560
1561 pt = sp->spt;
1562 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1563 /* avoid RMW */
1564 if (pt[i] & PT_WRITABLE_MASK)
1565 pt[i] &= ~PT_WRITABLE_MASK;
1566 }
1567}
1568
1569void kvm_mmu_zap_all(struct kvm *kvm)
1570{
1571 struct kvm_mmu_page *sp, *node;
1572
1573 list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
1574 kvm_mmu_zap_page(kvm, sp);
1575
1576 kvm_flush_remote_tlbs(kvm);
1577}
1578
1579void kvm_mmu_module_exit(void)
1580{
1581 if (pte_chain_cache)
1582 kmem_cache_destroy(pte_chain_cache);
1583 if (rmap_desc_cache)
1584 kmem_cache_destroy(rmap_desc_cache);
1585 if (mmu_page_header_cache)
1586 kmem_cache_destroy(mmu_page_header_cache);
1587}
1588
1589int kvm_mmu_module_init(void)
1590{
1591 pte_chain_cache = kmem_cache_create("kvm_pte_chain",
1592 sizeof(struct kvm_pte_chain),
1593 0, 0, NULL);
1594 if (!pte_chain_cache)
1595 goto nomem;
1596 rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
1597 sizeof(struct kvm_rmap_desc),
1598 0, 0, NULL);
1599 if (!rmap_desc_cache)
1600 goto nomem;
1601
1602 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
1603 sizeof(struct kvm_mmu_page),
1604 0, 0, NULL);
1605 if (!mmu_page_header_cache)
1606 goto nomem;
1607
1608 return 0;
1609
1610nomem:
1611 kvm_mmu_module_exit();
1612 return -ENOMEM;
1613}
1614
1615/*
1616 * Caculate mmu pages needed for kvm.
1617 */
1618unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
1619{
1620 int i;
1621 unsigned int nr_mmu_pages;
1622 unsigned int nr_pages = 0;
1623
1624 for (i = 0; i < kvm->nmemslots; i++)
1625 nr_pages += kvm->memslots[i].npages;
1626
1627 nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
1628 nr_mmu_pages = max(nr_mmu_pages,
1629 (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
1630
1631 return nr_mmu_pages;
1632}
1633
1634#ifdef AUDIT
1635
1636static const char *audit_msg;
1637
1638static gva_t canonicalize(gva_t gva)
1639{
1640#ifdef CONFIG_X86_64
1641 gva = (long long)(gva << 16) >> 16;
1642#endif
1643 return gva;
1644}
1645
1646static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
1647 gva_t va, int level)
1648{
1649 u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
1650 int i;
1651 gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
1652
1653 for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
1654 u64 ent = pt[i];
1655
1656 if (ent == shadow_trap_nonpresent_pte)
1657 continue;
1658
1659 va = canonicalize(va);
1660 if (level > 1) {
1661 if (ent == shadow_notrap_nonpresent_pte)
1662 printk(KERN_ERR "audit: (%s) nontrapping pte"
1663 " in nonleaf level: levels %d gva %lx"
1664 " level %d pte %llx\n", audit_msg,
1665 vcpu->arch.mmu.root_level, va, level, ent);
1666
1667 audit_mappings_page(vcpu, ent, va, level - 1);
1668 } else {
1669 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va);
1670 struct page *page = gpa_to_page(vcpu, gpa);
1671 hpa_t hpa = page_to_phys(page);
1672
1673 if (is_shadow_present_pte(ent)
1674 && (ent & PT64_BASE_ADDR_MASK) != hpa)
1675 printk(KERN_ERR "xx audit error: (%s) levels %d"
1676 " gva %lx gpa %llx hpa %llx ent %llx %d\n",
1677 audit_msg, vcpu->arch.mmu.root_level,
1678 va, gpa, hpa, ent,
1679 is_shadow_present_pte(ent));
1680 else if (ent == shadow_notrap_nonpresent_pte
1681 && !is_error_hpa(hpa))
1682 printk(KERN_ERR "audit: (%s) notrap shadow,"
1683 " valid guest gva %lx\n", audit_msg, va);
1684 kvm_release_page_clean(page);
1685
1686 }
1687 }
1688}
1689
1690static void audit_mappings(struct kvm_vcpu *vcpu)
1691{
1692 unsigned i;
1693
1694 if (vcpu->arch.mmu.root_level == 4)
1695 audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4);
1696 else
1697 for (i = 0; i < 4; ++i)
1698 if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK)
1699 audit_mappings_page(vcpu,
1700 vcpu->arch.mmu.pae_root[i],
1701 i << 30,
1702 2);
1703}
1704
1705static int count_rmaps(struct kvm_vcpu *vcpu)
1706{
1707 int nmaps = 0;
1708 int i, j, k;
1709
1710 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
1711 struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
1712 struct kvm_rmap_desc *d;
1713
1714 for (j = 0; j < m->npages; ++j) {
1715 unsigned long *rmapp = &m->rmap[j];
1716
1717 if (!*rmapp)
1718 continue;
1719 if (!(*rmapp & 1)) {
1720 ++nmaps;
1721 continue;
1722 }
1723 d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
1724 while (d) {
1725 for (k = 0; k < RMAP_EXT; ++k)
1726 if (d->shadow_ptes[k])
1727 ++nmaps;
1728 else
1729 break;
1730 d = d->more;
1731 }
1732 }
1733 }
1734 return nmaps;
1735}
1736
1737static int count_writable_mappings(struct kvm_vcpu *vcpu)
1738{
1739 int nmaps = 0;
1740 struct kvm_mmu_page *sp;
1741 int i;
1742
1743 list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
1744 u64 *pt = sp->spt;
1745
1746 if (sp->role.level != PT_PAGE_TABLE_LEVEL)
1747 continue;
1748
1749 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1750 u64 ent = pt[i];
1751
1752 if (!(ent & PT_PRESENT_MASK))
1753 continue;
1754 if (!(ent & PT_WRITABLE_MASK))
1755 continue;
1756 ++nmaps;
1757 }
1758 }
1759 return nmaps;
1760}
1761
1762static void audit_rmap(struct kvm_vcpu *vcpu)
1763{
1764 int n_rmap = count_rmaps(vcpu);
1765 int n_actual = count_writable_mappings(vcpu);
1766
1767 if (n_rmap != n_actual)
1768 printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
1769 __FUNCTION__, audit_msg, n_rmap, n_actual);
1770}
1771
1772static void audit_write_protection(struct kvm_vcpu *vcpu)
1773{
1774 struct kvm_mmu_page *sp;
1775 struct kvm_memory_slot *slot;
1776 unsigned long *rmapp;
1777 gfn_t gfn;
1778
1779 list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
1780 if (sp->role.metaphysical)
1781 continue;
1782
1783 slot = gfn_to_memslot(vcpu->kvm, sp->gfn);
1784 gfn = unalias_gfn(vcpu->kvm, sp->gfn);
1785 rmapp = &slot->rmap[gfn - slot->base_gfn];
1786 if (*rmapp)
1787 printk(KERN_ERR "%s: (%s) shadow page has writable"
1788 " mappings: gfn %lx role %x\n",
1789 __FUNCTION__, audit_msg, sp->gfn,
1790 sp->role.word);
1791 }
1792}
1793
1794static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
1795{
1796 int olddbg = dbg;
1797
1798 dbg = 0;
1799 audit_msg = msg;
1800 audit_rmap(vcpu);
1801 audit_write_protection(vcpu);
1802 audit_mappings(vcpu);
1803 dbg = olddbg;
1804}
1805
1806#endif
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-22 01:24:46 -0500