litmus-rt.git - The LITMUS^RT kernel.

diff options

author	Christoph Lameter <clameter@sgi.com>	2008-04-28 05:12:50 -0400
committer	Linus Torvalds <torvalds@linux-foundation.org>	2008-04-28 11:58:22 -0400
commit	6a1e7f777f613bf0df99c7772fa2123d01ce2f7d (patch)
tree	144629e467c8dcb7b5c8d23adc2538047bfa5b08
parent	f94a62e910840b3552c7adb7c57e0f8b3b345f6e (diff)

pageflags: convert to the use of new macros

Replace explicit definitions of page flags through the use of macros. Significantly reduces the size of the definitions and removes a lot of opportunity for errors. Additonal page flags can typically be generated with a single line. Signed-off-by: Christoph Lameter <clameter@sgi.com> Cc: Andy Whitcroft <apw@shadowen.org> Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Cc: Rik van Riel <riel@redhat.com> Cc: Mel Gorman <mel@csn.ul.ie> Cc: Jeremy Fitzhardinge <jeremy@goop.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Diffstat

-rw-r--r--

include/linux/page-flags.h

195

1 files changed, 68 insertions, 127 deletions


diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index e5bddbfcf7ae..ed7659adfaaf 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h
@@ -149,28 +149,58 @@ static inline int TestClearPage##uname(struct page *page) \
149	#define TESTSCFLAG(uname, lname) \	149	#define TESTSCFLAG(uname, lname) \
150	TESTSETFLAG(uname, lname) TESTCLEARFLAG(uname, lname)	150	TESTSETFLAG(uname, lname) TESTCLEARFLAG(uname, lname)
151		151
		152	struct page; /* forward declaration */
		153
		154	PAGEFLAG(Locked, locked) TESTSCFLAG(Locked, locked)
		155	PAGEFLAG(Error, error)
		156	PAGEFLAG(Referenced, referenced) TESTCLEARFLAG(Referenced, referenced)
		157	PAGEFLAG(Dirty, dirty) TESTSCFLAG(Dirty, dirty) __CLEARPAGEFLAG(Dirty, dirty)
		158	PAGEFLAG(LRU, lru) __CLEARPAGEFLAG(LRU, lru)
		159	PAGEFLAG(Active, active) __CLEARPAGEFLAG(Active, active)
		160	__PAGEFLAG(Slab, slab)
		161	PAGEFLAG(Checked, checked) /* Used by some filesystems */
		162	PAGEFLAG(Pinned, pinned) /* Xen pinned pagetable */
		163	PAGEFLAG(Reserved, reserved) __CLEARPAGEFLAG(Reserved, reserved)
		164	PAGEFLAG(Private, private) __CLEARPAGEFLAG(Private, private)
		165	__SETPAGEFLAG(Private, private)
		166
		167	/*
		168	* Only test-and-set exist for PG_writeback. The unconditional operators are
		169	* risky: they bypass page accounting.
		170	*/
		171	TESTPAGEFLAG(Writeback, writeback) TESTSCFLAG(Writeback, writeback)
		172	__PAGEFLAG(Buddy, buddy)
		173	PAGEFLAG(MappedToDisk, mappedtodisk)
		174
		175	/* PG_readahead is only used for file reads; PG_reclaim is only for writes */
		176	PAGEFLAG(Reclaim, reclaim) TESTCLEARFLAG(Reclaim, reclaim)
		177	PAGEFLAG(Readahead, readahead) /* Reminder to do async read-ahead */
		178
		179	#ifdef CONFIG_HIGHMEM
152	/*	180	/*
153	* Manipulation of page state flags	181	* Must use a macro here due to header dependency issues. page_zone() is not
		182	* available at this point.
154	*/	183	*/
155	#define PageLocked(page) \	184	#define PageHighMem(__p) is_highmem(page_zone(page))
156	test_bit(PG_locked, &(page)->flags)	185	#else
157	#define SetPageLocked(page) \	186	static inline int PageHighMem(struct page *page)
158	set_bit(PG_locked, &(page)->flags)	187	{
159	#define TestSetPageLocked(page) \	188	return 0;
160	test_and_set_bit(PG_locked, &(page)->flags)	189	}
161	#define ClearPageLocked(page) \	190	#endif
162	clear_bit(PG_locked, &(page)->flags)	191
163	#define TestClearPageLocked(page) \	192	#ifdef CONFIG_SWAP
164	test_and_clear_bit(PG_locked, &(page)->flags)	193	PAGEFLAG(SwapCache, swapcache)
165		194	#else
166	#define PageError(page) test_bit(PG_error, &(page)->flags)	195	static inline int PageSwapCache(struct page *page)
167	#define SetPageError(page) set_bit(PG_error, &(page)->flags)	196	{
168	#define ClearPageError(page) clear_bit(PG_error, &(page)->flags)	197	return 0;
169		198	}
170	#define PageReferenced(page) test_bit(PG_referenced, &(page)->flags)	199	#endif
171	#define SetPageReferenced(page) set_bit(PG_referenced, &(page)->flags)	200
172	#define ClearPageReferenced(page) clear_bit(PG_referenced, &(page)->flags)	201	#if (BITS_PER_LONG > 32)
173	#define TestClearPageReferenced(page) test_and_clear_bit(PG_referenced, &(page)->flags)	202	PAGEFLAG(Uncached, uncached)
		203	#endif
174		204
175	static inline int PageUptodate(struct page *page)	205	static inline int PageUptodate(struct page *page)
176	{	206	{
@@ -218,97 +248,37 @@ static inline void SetPageUptodate(struct page *page)
218	#endif	248	#endif
219	}	249	}
220		250
221	#define ClearPageUptodate(page) clear_bit(PG_uptodate, &(page)->flags)	251	CLEARPAGEFLAG(Uptodate, uptodate)
222
223	#define PageDirty(page) test_bit(PG_dirty, &(page)->flags)
224	#define SetPageDirty(page) set_bit(PG_dirty, &(page)->flags)
225	#define TestSetPageDirty(page) test_and_set_bit(PG_dirty, &(page)->flags)
226	#define ClearPageDirty(page) clear_bit(PG_dirty, &(page)->flags)
227	#define __ClearPageDirty(page) __clear_bit(PG_dirty, &(page)->flags)
228	#define TestClearPageDirty(page) test_and_clear_bit(PG_dirty, &(page)->flags)
229
230	#define PageLRU(page) test_bit(PG_lru, &(page)->flags)
231	#define SetPageLRU(page) set_bit(PG_lru, &(page)->flags)
232	#define ClearPageLRU(page) clear_bit(PG_lru, &(page)->flags)
233	#define __ClearPageLRU(page) __clear_bit(PG_lru, &(page)->flags)
234
235	#define PageActive(page) test_bit(PG_active, &(page)->flags)
236	#define SetPageActive(page) set_bit(PG_active, &(page)->flags)
237	#define ClearPageActive(page) clear_bit(PG_active, &(page)->flags)
238	#define __ClearPageActive(page) __clear_bit(PG_active, &(page)->flags)
239
240	#define PageSlab(page) test_bit(PG_slab, &(page)->flags)
241	#define __SetPageSlab(page) __set_bit(PG_slab, &(page)->flags)
242	#define __ClearPageSlab(page) __clear_bit(PG_slab, &(page)->flags)
243
244	#ifdef CONFIG_HIGHMEM
245	#define PageHighMem(page) is_highmem(page_zone(page))
246	#else
247	#define PageHighMem(page) 0 /* needed to optimize away at compile time */
248	#endif
249
250	#define PageChecked(page) test_bit(PG_checked, &(page)->flags)
251	#define SetPageChecked(page) set_bit(PG_checked, &(page)->flags)
252	#define ClearPageChecked(page) clear_bit(PG_checked, &(page)->flags)
253
254	#define PagePinned(page) test_bit(PG_pinned, &(page)->flags)
255	#define SetPagePinned(page) set_bit(PG_pinned, &(page)->flags)
256	#define ClearPagePinned(page) clear_bit(PG_pinned, &(page)->flags)
257
258	#define PageReserved(page) test_bit(PG_reserved, &(page)->flags)
259	#define SetPageReserved(page) set_bit(PG_reserved, &(page)->flags)
260	#define ClearPageReserved(page) clear_bit(PG_reserved, &(page)->flags)
261	#define __ClearPageReserved(page) __clear_bit(PG_reserved, &(page)->flags)
262
263	#define SetPagePrivate(page) set_bit(PG_private, &(page)->flags)
264	#define ClearPagePrivate(page) clear_bit(PG_private, &(page)->flags)
265	#define PagePrivate(page) test_bit(PG_private, &(page)->flags)
266	#define __SetPagePrivate(page) __set_bit(PG_private, &(page)->flags)
267	#define __ClearPagePrivate(page) __clear_bit(PG_private, &(page)->flags)
268		252
269	/*	253	extern void cancel_dirty_page(struct page *page, unsigned int account_size);
270	* Only test-and-set exist for PG_writeback. The unconditional operators are
271	* risky: they bypass page accounting.
272	*/
273	#define PageWriteback(page) test_bit(PG_writeback, &(page)->flags)
274	#define TestSetPageWriteback(page) test_and_set_bit(PG_writeback, \
275	&(page)->flags)
276	#define TestClearPageWriteback(page) test_and_clear_bit(PG_writeback, \
277	&(page)->flags)
278
279	#define PageBuddy(page) test_bit(PG_buddy, &(page)->flags)
280	#define __SetPageBuddy(page) __set_bit(PG_buddy, &(page)->flags)
281	#define __ClearPageBuddy(page) __clear_bit(PG_buddy, &(page)->flags)
282
283	#define PageMappedToDisk(page) test_bit(PG_mappedtodisk, &(page)->flags)
284	#define SetPageMappedToDisk(page) set_bit(PG_mappedtodisk, &(page)->flags)
285	#define ClearPageMappedToDisk(page) clear_bit(PG_mappedtodisk, &(page)->flags)
286		254
287	#define PageReadahead(page) test_bit(PG_readahead, &(page)->flags)	255	int test_clear_page_writeback(struct page *page);
288	#define SetPageReadahead(page) set_bit(PG_readahead, &(page)->flags)	256	int test_set_page_writeback(struct page *page);
289	#define ClearPageReadahead(page) clear_bit(PG_readahead, &(page)->flags)
290		257
291	#define PageReclaim(page) test_bit(PG_reclaim, &(page)->flags)	258	static inline void set_page_writeback(struct page *page)
292	#define SetPageReclaim(page) set_bit(PG_reclaim, &(page)->flags)	259	{
293	#define ClearPageReclaim(page) clear_bit(PG_reclaim, &(page)->flags)	260	test_set_page_writeback(page);
294	#define TestClearPageReclaim(page) test_and_clear_bit(PG_reclaim, &(page)->flags)	261	}
295		262
296	#define PageCompound(page) test_bit(PG_compound, &(page)->flags)	263	TESTPAGEFLAG(Compound, compound)
297	#define __SetPageCompound(page) __set_bit(PG_compound, &(page)->flags)	264	__PAGEFLAG(Head, compound)
298	#define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags)
299		265
300	/*	266	/*
301	* PG_reclaim is used in combination with PG_compound to mark the	267	* PG_reclaim is used in combination with PG_compound to mark the
302	* head and tail of a compound page	268	* head and tail of a compound page. This saves one page flag
		269	* but makes it impossible to use compound pages for the page cache.
		270	* The PG_reclaim bit would have to be used for reclaim or readahead
		271	* if compound pages enter the page cache.
303	*	272	*
304	* PG_compound & PG_reclaim => Tail page	273	* PG_compound & PG_reclaim => Tail page
305	* PG_compound & ~PG_reclaim => Head page	274	* PG_compound & ~PG_reclaim => Head page
306	*/	275	*/
307
308	#define PG_head_tail_mask ((1L << PG_compound) \| (1L << PG_reclaim))	276	#define PG_head_tail_mask ((1L << PG_compound) \| (1L << PG_reclaim))
309		277
310	#define PageTail(page) (((page)->flags & PG_head_tail_mask) \	278	static inline int PageTail(struct page *page)
311	== PG_head_tail_mask)	279	{
		280	return ((page->flags & PG_head_tail_mask) == PG_head_tail_mask);
		281	}
312		282
313	static inline void __SetPageTail(struct page *page)	283	static inline void __SetPageTail(struct page *page)
314	{	284	{
@@ -320,34 +290,5 @@ static inline void __ClearPageTail(struct page *page)
320	page->flags &= ~PG_head_tail_mask;	290	page->flags &= ~PG_head_tail_mask;
321	}	291	}
322		292
323	#define PageHead(page) (((page)->flags & PG_head_tail_mask) \
324	== (1L << PG_compound))
325	#define __SetPageHead(page) __SetPageCompound(page)
326	#define __ClearPageHead(page) __ClearPageCompound(page)
327
328	#ifdef CONFIG_SWAP
329	#define PageSwapCache(page) test_bit(PG_swapcache, &(page)->flags)
330	#define SetPageSwapCache(page) set_bit(PG_swapcache, &(page)->flags)
331	#define ClearPageSwapCache(page) clear_bit(PG_swapcache, &(page)->flags)
332	#else
333	#define PageSwapCache(page) 0
334	#endif
335
336	#define PageUncached(page) test_bit(PG_uncached, &(page)->flags)
337	#define SetPageUncached(page) set_bit(PG_uncached, &(page)->flags)
338	#define ClearPageUncached(page) clear_bit(PG_uncached, &(page)->flags)
339
340	struct page; /* forward declaration */
341
342	extern void cancel_dirty_page(struct page *page, unsigned int account_size);
343
344	int test_clear_page_writeback(struct page *page);
345	int test_set_page_writeback(struct page *page);
346
347	static inline void set_page_writeback(struct page *page)
348	{
349	test_set_page_writeback(page);
350	}
351
352	#endif /* !__GENERATING_BOUNDS_H */	293	#endif /* !__GENERATING_BOUNDS_H */
353	#endif /* PAGE_FLAGS_H */	294	#endif /* PAGE_FLAGS_H */

/* * Kernel-based Virtual Machine driver for Linux * * AMD SVM support * * Copyright (C) 2006 Qumranet, Inc. * * Authors: * Yaniv Kamay <yaniv@qumranet.com> * Avi Kivity <avi@qumranet.com> * * This work is licensed under the terms of the GNU GPL, version 2. See * the COPYING file in the top-level directory. * */ #include <linux/kvm_host.h> #include "irq.h" #include "mmu.h" #include "kvm_cache_regs.h" #include "x86.h" #include <linux/module.h> #include <linux/kernel.h> #include <linux/vmalloc.h> #include <linux/highmem.h> #include <linux/sched.h> #include <linux/ftrace_event.h> #include <asm/desc.h> #include <asm/virtext.h> #include "trace.h" #define __ex(x) __kvm_handle_fault_on_reboot(x) MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); #define IOPM_ALLOC_ORDER 2 #define MSRPM_ALLOC_ORDER 1 #define SEG_TYPE_LDT 2 #define SEG_TYPE_BUSY_TSS16 3 #define SVM_FEATURE_NPT (1 << 0) #define SVM_FEATURE_LBRV (1 << 1) #define SVM_FEATURE_SVML (1 << 2) #define SVM_FEATURE_PAUSE_FILTER (1 << 10) #define NESTED_EXIT_HOST 0 /* Exit handled on host level */ #define NESTED_EXIT_DONE 1 /* Exit caused nested vmexit */ #define NESTED_EXIT_CONTINUE 2 /* Further checks needed */ #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) static const u32 host_save_user_msrs[] = { #ifdef CONFIG_X86_64 MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE, MSR_FS_BASE, #endif MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, }; #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs) struct kvm_vcpu; struct nested_state { struct vmcb *hsave; u64 hsave_msr; u64 vmcb; /* These are the merged vectors */ u32 *msrpm; /* gpa pointers to the real vectors */ u64 vmcb_msrpm; /* A VMEXIT is required but not yet emulated */ bool exit_required; /* cache for intercepts of the guest */ u16 intercept_cr_read; u16 intercept_cr_write; u16 intercept_dr_read; u16 intercept_dr_write; u32 intercept_exceptions; u64 intercept; }; struct vcpu_svm { struct kvm_vcpu vcpu; struct vmcb *vmcb; unsigned long vmcb_pa; struct svm_cpu_data *svm_data; uint64_t asid_generation; uint64_t sysenter_esp; uint64_t sysenter_eip; u64 next_rip; u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS]; u64 host_gs_base; u32 *msrpm; struct nested_state nested; bool nmi_singlestep; }; /* enable NPT for AMD64 and X86 with PAE */ #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) static bool npt_enabled = true; #else static bool npt_enabled = false; #endif static int npt = 1; module_param(npt, int, S_IRUGO); static int nested = 1; module_param(nested, int, S_IRUGO); static void svm_flush_tlb(struct kvm_vcpu *vcpu); static void svm_complete_interrupts(struct vcpu_svm *svm); static int nested_svm_exit_handled(struct vcpu_svm *svm); static int nested_svm_vmexit(struct vcpu_svm *svm); static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, bool has_error_code, u32 error_code); static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu) { return container_of(vcpu, struct vcpu_svm, vcpu); } static inline bool is_nested(struct vcpu_svm *svm) { return svm->nested.vmcb; } static inline void enable_gif(struct vcpu_svm *svm) { svm->vcpu.arch.hflags |= HF_GIF_MASK; } static inline void disable_gif(struct vcpu_svm *svm) { svm->vcpu.arch.hflags &= ~HF_GIF_MASK; } static inline bool gif_set(struct vcpu_svm *svm) { return !!(svm->vcpu.arch.hflags & HF_GIF_MASK); } static unsigned long iopm_base; struct kvm_ldttss_desc { u16 limit0; u16 base0; unsigned base1 : 8, type : 5, dpl : 2, p : 1; unsigned limit1 : 4, zero0 : 3, g : 1, base2 : 8; u32 base3; u32 zero1; } __attribute__((packed)); struct svm_cpu_data { int cpu; u64 asid_generation; u32 max_asid; u32 next_asid; struct kvm_ldttss_desc *tss_desc; struct page *save_area; }; static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data); static uint32_t svm_features; struct svm_init_data { int cpu; int r; }; static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges) #define MSRS_RANGE_SIZE 2048 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2) #define MAX_INST_SIZE 15 static inline u32 svm_has(u32 feat) { return svm_features & feat; } static inline void clgi(void) { asm volatile (__ex(SVM_CLGI)); } static inline void stgi(void) { asm volatile (__ex(SVM_STGI)); } static inline void invlpga(unsigned long addr, u32 asid) { asm volatile (__ex(SVM_INVLPGA) :: "a"(addr), "c"(asid)); } static inline void force_new_asid(struct kvm_vcpu *vcpu) { to_svm(vcpu)->asid_generation--; } static inline void flush_guest_tlb(struct kvm_vcpu *vcpu) { force_new_asid(vcpu); } static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) { if (!npt_enabled && !(efer & EFER_LMA)) efer &= ~EFER_LME; to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME; vcpu->arch.shadow_efer = efer; } static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, bool has_error_code, u32 error_code) { struct vcpu_svm *svm = to_svm(vcpu); /* If we are within a nested VM we'd better #VMEXIT and let the guest handle the exception */ if (nested_svm_check_exception(svm, nr, has_error_code, error_code)) return; svm->vmcb->control.event_inj = nr | SVM_EVTINJ_VALID | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0) | SVM_EVTINJ_TYPE_EXEPT; svm->vmcb->control.event_inj_err = error_code; } static int is_external_interrupt(u32 info) { info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR); } static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) { struct vcpu_svm *svm = to_svm(vcpu); u32 ret = 0; if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) ret |= X86_SHADOW_INT_STI | X86_SHADOW_INT_MOV_SS; return ret & mask; } static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask) { struct vcpu_svm *svm = to_svm(vcpu); if (mask == 0) svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; else svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK; } static void skip_emulated_instruction(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); if (!svm->next_rip) { if (emulate_instruction(vcpu, 0, 0, EMULTYPE_SKIP) != EMULATE_DONE) printk(KERN_DEBUG "%s: NOP\n", __func__); return; } if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE) printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n", __func__, kvm_rip_read(vcpu), svm->next_rip); kvm_rip_write(vcpu, svm->next_rip); svm_set_interrupt_shadow(vcpu, 0); } static int has_svm(void) { const char *msg; if (!cpu_has_svm(&msg)) { printk(KERN_INFO "has_svm: %s\n", msg); return 0; } return 1; } static void svm_hardware_disable(void *garbage) { cpu_svm_disable(); } static int svm_hardware_enable(void *garbage) { struct svm_cpu_data *sd; uint64_t efer; struct descriptor_table gdt_descr; struct desc_struct *gdt; int me = raw_smp_processor_id(); rdmsrl(MSR_EFER, efer); if (efer & EFER_SVME) return -EBUSY; if (!has_svm()) { printk(KERN_ERR "svm_hardware_enable: err EOPNOTSUPP on %d\n", me); return -EINVAL; } sd = per_cpu(svm_data, me); if (!sd) { printk(KERN_ERR "svm_hardware_enable: svm_data is NULL on %d\n", me); return -EINVAL; } sd->asid_generation = 1; sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; sd->next_asid = sd->max_asid + 1; kvm_get_gdt(&gdt_descr); gdt = (struct desc_struct *)gdt_descr.base; sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); wrmsrl(MSR_EFER, efer | EFER_SVME); wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT); return 0; } static void svm_cpu_uninit(int cpu) { struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id()); if (!sd) return; per_cpu(svm_data, raw_smp_processor_id()) = NULL; __free_page(sd->save_area); kfree(sd); } static int svm_cpu_init(int cpu) { struct svm_cpu_data *sd; int r; sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL); if (!sd) return -ENOMEM; sd->cpu = cpu; sd->save_area = alloc_page(GFP_KERNEL); r = -ENOMEM; if (!sd->save_area) goto err_1; per_cpu(svm_data, cpu) = sd; return 0; err_1: kfree(sd); return r; } static void set_msr_interception(u32 *msrpm, unsigned msr, int read, int write) { int i; for (i = 0; i < NUM_MSR_MAPS; i++) { if (msr >= msrpm_ranges[i] && msr < msrpm_ranges[i] + MSRS_IN_RANGE) { u32 msr_offset = (i * MSRS_IN_RANGE + msr - msrpm_ranges[i]) * 2; u32 *base = msrpm + (msr_offset / 32); u32 msr_shift = msr_offset % 32; u32 mask = ((write) ? 0 : 2) | ((read) ? 0 : 1); *base = (*base & ~(0x3 << msr_shift)) | (mask << msr_shift); return; } } BUG(); } static void svm_vcpu_init_msrpm(u32 *msrpm) { memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER)); #ifdef CONFIG_X86_64 set_msr_interception(msrpm, MSR_GS_BASE, 1, 1); set_msr_interception(msrpm, MSR_FS_BASE, 1, 1); set_msr_interception(msrpm, MSR_KERNEL_GS_BASE, 1, 1); set_msr_interception(msrpm, MSR_LSTAR, 1, 1); set_msr_interception(msrpm, MSR_CSTAR, 1, 1); set_msr_interception(msrpm, MSR_SYSCALL_MASK, 1, 1); #endif set_msr_interception(msrpm, MSR_K6_STAR, 1, 1); set_msr_interception(msrpm, MSR_IA32_SYSENTER_CS, 1, 1); } static void svm_enable_lbrv(struct vcpu_svm *svm) { u32 *msrpm = svm->msrpm; svm->vmcb->control.lbr_ctl = 1; set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1); set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1); set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1); set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1); } static void svm_disable_lbrv(struct vcpu_svm *svm) { u32 *msrpm = svm->msrpm; svm->vmcb->control.lbr_ctl = 0; set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0); set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0); set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0); set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0); } static __init int svm_hardware_setup(void) { int cpu; struct page *iopm_pages; void *iopm_va; int r; iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER); if (!iopm_pages) return -ENOMEM; iopm_va = page_address(iopm_pages); memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER)); iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT; if (boot_cpu_has(X86_FEATURE_NX)) kvm_enable_efer_bits(EFER_NX); if (boot_cpu_has(X86_FEATURE_FXSR_OPT)) kvm_enable_efer_bits(EFER_FFXSR); if (nested) { printk(KERN_INFO "kvm: Nested Virtualization enabled\n"); kvm_enable_efer_bits(EFER_SVME); } for_each_possible_cpu(cpu) { r = svm_cpu_init(cpu); if (r) goto err; } svm_features = cpuid_edx(SVM_CPUID_FUNC); if (!svm_has(SVM_FEATURE_NPT)) npt_enabled = false; if (npt_enabled && !npt) { printk(KERN_INFO "kvm: Nested Paging disabled\n"); npt_enabled = false; } if (npt_enabled) { printk(KERN_INFO "kvm: Nested Paging enabled\n"); kvm_enable_tdp(); } else kvm_disable_tdp(); return 0; err: __free_pages(iopm_pages, IOPM_ALLOC_ORDER); iopm_base = 0; return r; } static __exit void svm_hardware_unsetup(void) { int cpu; for_each_possible_cpu(cpu) svm_cpu_uninit(cpu); __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER); iopm_base = 0; } static void init_seg(struct vmcb_seg *seg) { seg->selector = 0; seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK | SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */ seg->limit = 0xffff; seg->base = 0; } static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) { seg->selector = 0; seg->attrib = SVM_SELECTOR_P_MASK | type; seg->limit = 0xffff; seg->base = 0; } static void init_vmcb(struct vcpu_svm *svm) { struct vmcb_control_area *control = &svm->vmcb->control; struct vmcb_save_area *save = &svm->vmcb->save; svm->vcpu.fpu_active = 1; control->intercept_cr_read = INTERCEPT_CR0_MASK | INTERCEPT_CR3_MASK | INTERCEPT_CR4_MASK; control->intercept_cr_write = INTERCEPT_CR0_MASK | INTERCEPT_CR3_MASK | INTERCEPT_CR4_MASK | INTERCEPT_CR8_MASK; control->intercept_dr_read = INTERCEPT_DR0_MASK | INTERCEPT_DR1_MASK | INTERCEPT_DR2_MASK | INTERCEPT_DR3_MASK; control->intercept_dr_write = INTERCEPT_DR0_MASK | INTERCEPT_DR1_MASK | INTERCEPT_DR2_MASK | INTERCEPT_DR3_MASK | INTERCEPT_DR5_MASK | INTERCEPT_DR7_MASK; control->intercept_exceptions = (1 << PF_VECTOR) | (1 << UD_VECTOR) | (1 << MC_VECTOR); control->intercept = (1ULL << INTERCEPT_INTR) | (1ULL << INTERCEPT_NMI) | (1ULL << INTERCEPT_SMI) | (1ULL << INTERCEPT_CPUID) | (1ULL << INTERCEPT_INVD) | (1ULL << INTERCEPT_HLT) | (1ULL << INTERCEPT_INVLPG) | (1ULL << INTERCEPT_INVLPGA) | (1ULL << INTERCEPT_IOIO_PROT) | (1ULL << INTERCEPT_MSR_PROT) | (1ULL << INTERCEPT_TASK_SWITCH) | (1ULL << INTERCEPT_SHUTDOWN) | (1ULL << INTERCEPT_VMRUN) | (1ULL << INTERCEPT_VMMCALL) | (1ULL << INTERCEPT_VMLOAD) | (1ULL << INTERCEPT_VMSAVE) | (1ULL << INTERCEPT_STGI) | (1ULL << INTERCEPT_CLGI) | (1ULL << INTERCEPT_SKINIT) | (1ULL << INTERCEPT_WBINVD) | (1ULL << INTERCEPT_MONITOR) | (1ULL << INTERCEPT_MWAIT); control->iopm_base_pa = iopm_base; control->msrpm_base_pa = __pa(svm->msrpm); control->tsc_offset = 0; control->int_ctl = V_INTR_MASKING_MASK; init_seg(&save->es); init_seg(&save->ss); init_seg(&save->ds); init_seg(&save->fs); init_seg(&save->gs); save->cs.selector = 0xf000; /* Executable/Readable Code Segment */ save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK; save->cs.limit = 0xffff; /* * cs.base should really be 0xffff0000, but vmx can't handle that, so * be consistent with it. * * Replace when we have real mode working for vmx. */ save->cs.base = 0xf0000; save->gdtr.limit = 0xffff; save->idtr.limit = 0xffff; init_sys_seg(&save->ldtr, SEG_TYPE_LDT); init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); save->efer = EFER_SVME; save->dr6 = 0xffff0ff0; save->dr7 = 0x400; save->rflags = 2; save->rip = 0x0000fff0; svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip; /* This is the guest-visible cr0 value. * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0. */ svm->vcpu.arch.cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET; kvm_set_cr0(&svm->vcpu, svm->vcpu.arch.cr0); save->cr4 = X86_CR4_PAE; /* rdx = ?? */ if (npt_enabled) { /* Setup VMCB for Nested Paging */ control->nested_ctl = 1; control->intercept &= ~((1ULL << INTERCEPT_TASK_SWITCH) | (1ULL << INTERCEPT_INVLPG)); control->intercept_exceptions &= ~(1 << PF_VECTOR); control->intercept_cr_read &= ~INTERCEPT_CR3_MASK; control->intercept_cr_write &= ~INTERCEPT_CR3_MASK; save->g_pat = 0x0007040600070406ULL; save->cr3 = 0; save->cr4 = 0; } force_new_asid(&svm->vcpu); svm->nested.vmcb = 0; svm->vcpu.arch.hflags = 0; if (svm_has(SVM_FEATURE_PAUSE_FILTER)) { control->pause_filter_count = 3000; control->intercept |= (1ULL << INTERCEPT_PAUSE); } enable_gif(svm); } static int svm_vcpu_reset(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); init_vmcb(svm); if (!kvm_vcpu_is_bsp(vcpu)) { kvm_rip_write(vcpu, 0); svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12; svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8; } vcpu->arch.regs_avail = ~0; vcpu->arch.regs_dirty = ~0; return 0; } static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) { struct vcpu_svm *svm; struct page *page; struct page *msrpm_pages; struct page *hsave_page; struct page *nested_msrpm_pages; int err; svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); if (!svm) { err = -ENOMEM; goto out; } err = kvm_vcpu_init(&svm->vcpu, kvm, id); if (err) goto free_svm; page = alloc_page(GFP_KERNEL); if (!page) { err = -ENOMEM; goto uninit; } err = -ENOMEM; msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER); if (!msrpm_pages) goto uninit; nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER); if (!nested_msrpm_pages) goto uninit; svm->msrpm = page_address(msrpm_pages); svm_vcpu_init_msrpm(svm->msrpm); hsave_page = alloc_page(GFP_KERNEL); if (!hsave_page) goto uninit; svm->nested.hsave = page_address(hsave_page); svm->nested.msrpm = page_address(nested_msrpm_pages); svm->vmcb = page_address(page); clear_page(svm->vmcb); svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT; svm->asid_generation = 0; init_vmcb(svm); fx_init(&svm->vcpu); svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; if (kvm_vcpu_is_bsp(&svm->vcpu)) svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP; return &svm->vcpu; uninit: kvm_vcpu_uninit(&svm->vcpu); free_svm: kmem_cache_free(kvm_vcpu_cache, svm); out: return ERR_PTR(err); } static void svm_free_vcpu(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT)); __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER); __free_page(virt_to_page(svm->nested.hsave)); __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER); kvm_vcpu_uninit(vcpu); kmem_cache_free(kvm_vcpu_cache, svm); } static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { struct vcpu_svm *svm = to_svm(vcpu); int i; if (unlikely(cpu != vcpu->cpu)) { u64 delta; if (check_tsc_unstable()) { /* * Make sure that the guest sees a monotonically * increasing TSC. */ delta = vcpu->arch.host_tsc - native_read_tsc(); svm->vmcb->control.tsc_offset += delta; if (is_nested(svm)) svm->nested.hsave->control.tsc_offset += delta; } vcpu->cpu = cpu; kvm_migrate_timers(vcpu); svm->asid_generation = 0; } for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); } static void svm_vcpu_put(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); int i; ++vcpu->stat.host_state_reload; for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); vcpu->arch.host_tsc = native_read_tsc(); } static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) { return to_svm(vcpu)->vmcb->save.rflags; } static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) { to_svm(vcpu)->vmcb->save.rflags = rflags; } static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) { switch (reg) { case VCPU_EXREG_PDPTR: BUG_ON(!npt_enabled); load_pdptrs(vcpu, vcpu->arch.cr3); break; default: BUG(); } } static void svm_set_vintr(struct vcpu_svm *svm) { svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR; } static void svm_clear_vintr(struct vcpu_svm *svm) { svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR); } static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) { struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; switch (seg) { case VCPU_SREG_CS: return &save->cs; case VCPU_SREG_DS: return &save->ds; case VCPU_SREG_ES: return &save->es; case VCPU_SREG_FS: return &save->fs; case VCPU_SREG_GS: return &save->gs; case VCPU_SREG_SS: return &save->ss; case VCPU_SREG_TR: return &save->tr; case VCPU_SREG_LDTR: return &save->ldtr; } BUG(); return NULL; } static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg) { struct vmcb_seg *s = svm_seg(vcpu, seg); return s->base; } static void svm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) { struct vmcb_seg *s = svm_seg(vcpu, seg); var->base = s->base; var->limit = s->limit; var->selector = s->selector; var->type = s->attrib & SVM_SELECTOR_TYPE_MASK; var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1; var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1; var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1; var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1; var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1; /* AMD's VMCB does not have an explicit unusable field, so emulate it * for cross vendor migration purposes by "not present" */ var->unusable = !var->present || (var->type == 0); switch (seg) { case VCPU_SREG_CS: /* * SVM always stores 0 for the 'G' bit in the CS selector in * the VMCB on a VMEXIT. This hurts cross-vendor migration: * Intel's VMENTRY has a check on the 'G' bit. */ var->g = s->limit > 0xfffff; break; case VCPU_SREG_TR: /* * Work around a bug where the busy flag in the tr selector * isn't exposed */ var->type |= 0x2; break; case VCPU_SREG_DS: case VCPU_SREG_ES: case VCPU_SREG_FS: case VCPU_SREG_GS: /* * The accessed bit must always be set in the segment * descriptor cache, although it can be cleared in the * descriptor, the cached bit always remains at 1. Since * Intel has a check on this, set it here to support * cross-vendor migration. */ if (!var->unusable) var->type |= 0x1; break; case VCPU_SREG_SS: /* On AMD CPUs sometimes the DB bit in the segment * descriptor is left as 1, although the whole segment has * been made unusable. Clear it here to pass an Intel VMX * entry check when cross vendor migrating. */ if (var->unusable) var->db = 0; break; } } static int svm_get_cpl(struct kvm_vcpu *vcpu) { struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; return save->cpl; } static void svm_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) { struct vcpu_svm *svm = to_svm(vcpu); dt->limit = svm->vmcb->save.idtr.limit; dt->base = svm->vmcb->save.idtr.base; } static void svm_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) { struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->save.idtr.limit = dt->limit; svm->vmcb->save.idtr.base = dt->base ; } static void svm_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) { struct vcpu_svm *svm = to_svm(vcpu); dt->limit = svm->vmcb->save.gdtr.limit; dt->base = svm->vmcb->save.gdtr.base; } static void svm_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) { struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->save.gdtr.limit = dt->limit; svm->vmcb->save.gdtr.base = dt->base ; } static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) { } static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) { } static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { struct vcpu_svm *svm = to_svm(vcpu); #ifdef CONFIG_X86_64 if (vcpu->arch.shadow_efer & EFER_LME) { if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { vcpu->arch.shadow_efer |= EFER_LMA; svm->vmcb->save.efer |= EFER_LMA | EFER_LME; } if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { vcpu->arch.shadow_efer &= ~EFER_LMA; svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); } } #endif vcpu->arch.cr0 = cr0; if (!npt_enabled) cr0 |= X86_CR0_PG | X86_CR0_WP; if (!vcpu->fpu_active) cr0 |= X86_CR0_TS; /* * re-enable caching here because the QEMU bios * does not do it - this results in some delay at * reboot */ cr0 &= ~(X86_CR0_CD | X86_CR0_NW); svm->vmcb->save.cr0 = cr0; } static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) { unsigned long host_cr4_mce = read_cr4() & X86_CR4_MCE; unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4; if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE)) force_new_asid(vcpu); vcpu->arch.cr4 = cr4; if (!npt_enabled) cr4 |= X86_CR4_PAE; cr4 |= host_cr4_mce; to_svm(vcpu)->vmcb->save.cr4 = cr4; } static void svm_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) { struct vcpu_svm *svm = to_svm(vcpu); struct vmcb_seg *s = svm_seg(vcpu, seg); s->base = var->base; s->limit = var->limit; s->selector = var->selector; if (var->unusable) s->attrib = 0; else { s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK); s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT; s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT; s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT; s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT; s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT; s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT; s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT; } if (seg == VCPU_SREG_CS) svm->vmcb->save.cpl = (svm->vmcb->save.cs.attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; } static void update_db_intercept(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->control.intercept_exceptions &= ~((1 << DB_VECTOR) | (1 << BP_VECTOR)); if (svm->nmi_singlestep) svm->vmcb->control.intercept_exceptions |= (1 << DB_VECTOR); if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { if (vcpu->guest_debug & (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) svm->vmcb->control.intercept_exceptions |= 1 << DB_VECTOR; if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) svm->vmcb->control.intercept_exceptions |= 1 << BP_VECTOR; } else vcpu->guest_debug = 0; } static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) { struct vcpu_svm *svm = to_svm(vcpu); if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) svm->vmcb->save.dr7 = dbg->arch.debugreg[7]; else svm->vmcb->save.dr7 = vcpu->arch.dr7; update_db_intercept(vcpu); } static void load_host_msrs(struct kvm_vcpu *vcpu) { #ifdef CONFIG_X86_64 wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base); #endif } static void save_host_msrs(struct kvm_vcpu *vcpu) { #ifdef CONFIG_X86_64 rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base); #endif } static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) { if (sd->next_asid > sd->max_asid) { ++sd->asid_generation; sd->next_asid = 1; svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; } svm->asid_generation = sd->asid_generation; svm->vmcb->control.asid = sd->next_asid++; } static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr) { struct vcpu_svm *svm = to_svm(vcpu); unsigned long val; switch (dr) { case 0 ... 3: val = vcpu->arch.db[dr]; break; case 6: if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) val = vcpu->arch.dr6; else val = svm->vmcb->save.dr6; break; case 7: if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) val = vcpu->arch.dr7; else val = svm->vmcb->save.dr7; break; default: val = 0; } return val; } static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value, int *exception) { struct vcpu_svm *svm = to_svm(vcpu); *exception = 0; switch (dr) { case 0 ... 3: vcpu->arch.db[dr] = value; if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) vcpu->arch.eff_db[dr] = value; return; case 4 ... 5: if (vcpu->arch.cr4 & X86_CR4_DE) *exception = UD_VECTOR; return; case 6: if (value & 0xffffffff00000000ULL) { *exception = GP_VECTOR; return; } vcpu->arch.dr6 = (value & DR6_VOLATILE) | DR6_FIXED_1; return; case 7: if (value & 0xffffffff00000000ULL) { *exception = GP_VECTOR; return; } vcpu->arch.dr7 = (value & DR7_VOLATILE) | DR7_FIXED_1; if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { svm->vmcb->save.dr7 = vcpu->arch.dr7; vcpu->arch.switch_db_regs = (value & DR7_BP_EN_MASK); } return; default: /* FIXME: Possible case? */ printk(KERN_DEBUG "%s: unexpected dr %u\n", __func__, dr); *exception = UD_VECTOR; return; } } static int pf_interception(struct vcpu_svm *svm) { u64 fault_address; u32 error_code; fault_address = svm->vmcb->control.exit_info_2; error_code = svm->vmcb->control.exit_info_1; trace_kvm_page_fault(fault_address, error_code); if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu)) kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address); return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); } static int db_interception(struct vcpu_svm *svm) { struct kvm_run *kvm_run = svm->vcpu.run; if (!(svm->vcpu.guest_debug & (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) && !svm->nmi_singlestep) { kvm_queue_exception(&svm->vcpu, DB_VECTOR); return 1; } if (svm->nmi_singlestep) { svm->nmi_singlestep = false; if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) svm->vmcb->save.rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); update_db_intercept(&svm->vcpu); } if (svm->vcpu.guest_debug & (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)){ kvm_run->exit_reason = KVM_EXIT_DEBUG; kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; kvm_run->debug.arch.exception = DB_VECTOR; return 0; } return 1; } static int bp_interception(struct vcpu_svm *svm) { struct kvm_run *kvm_run = svm->vcpu.run; kvm_run->exit_reason = KVM_EXIT_DEBUG; kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; kvm_run->debug.arch.exception = BP_VECTOR; return 0; } static int ud_interception(struct vcpu_svm *svm) { int er; er = emulate_instruction(&svm->vcpu, 0, 0, EMULTYPE_TRAP_UD); if (er != EMULATE_DONE) kvm_queue_exception(&svm->vcpu, UD_VECTOR); return 1; } static int nm_interception(struct vcpu_svm *svm) { svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); if (!kvm_read_cr0_bits(&svm->vcpu, X86_CR0_TS)) svm->vmcb->save.cr0 &= ~X86_CR0_TS; else svm->vmcb->save.cr0 |= X86_CR0_TS; svm->vcpu.fpu_active = 1; return 1; } static int mc_interception(struct vcpu_svm *svm) { /* * On an #MC intercept the MCE handler is not called automatically in * the host. So do it by hand here. */ asm volatile ( "int $0x12\n"); /* not sure if we ever come back to this point */ return 1; } static int shutdown_interception(struct vcpu_svm *svm) { struct kvm_run *kvm_run = svm->vcpu.run; /* * VMCB is undefined after a SHUTDOWN intercept * so reinitialize it. */ clear_page(svm->vmcb); init_vmcb(svm); kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; return 0; } static int io_interception(struct vcpu_svm *svm) { u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ int size, in, string; unsigned port; ++svm->vcpu.stat.io_exits; svm->next_rip = svm->vmcb->control.exit_info_2; string = (io_info & SVM_IOIO_STR_MASK) != 0; if (string) { if (emulate_instruction(&svm->vcpu, 0, 0, 0) == EMULATE_DO_MMIO) return 0; return 1; } in = (io_info & SVM_IOIO_TYPE_MASK) != 0; port = io_info >> 16; size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; skip_emulated_instruction(&svm->vcpu); return kvm_emulate_pio(&svm->vcpu, in, size, port); } static int nmi_interception(struct vcpu_svm *svm) { return 1; } static int intr_interception(struct vcpu_svm *svm) { ++svm->vcpu.stat.irq_exits; return 1; } static int nop_on_interception(struct vcpu_svm *svm) { return 1; } static int halt_interception(struct vcpu_svm *svm) { svm->next_rip = kvm_rip_read(&svm->vcpu) + 1; skip_emulated_instruction(&svm->vcpu); return kvm_emulate_halt(&svm->vcpu); } static int vmmcall_interception(struct vcpu_svm *svm) { svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; skip_emulated_instruction(&svm->vcpu); kvm_emulate_hypercall(&svm->vcpu); return 1; } static int nested_svm_check_permissions(struct vcpu_svm *svm) { if (!(svm->vcpu.arch.shadow_efer & EFER_SVME) || !is_paging(&svm->vcpu)) { kvm_queue_exception(&svm->vcpu, UD_VECTOR); return 1; } if (svm->vmcb->save.cpl) { kvm_inject_gp(&svm->vcpu, 0); return 1; } return 0; } static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, bool has_error_code, u32 error_code) { if (!is_nested(svm)) return 0;


context:
space:
mode: