aboutsummaryrefslogtreecommitdiffstats
path: root/include/asm-i386/paravirt.h
diff options
context:
space:
mode:
Diffstat (limited to 'include/asm-i386/paravirt.h')
-rw-r--r--include/asm-i386/paravirt.h957
1 files changed, 735 insertions, 222 deletions
diff --git a/include/asm-i386/paravirt.h b/include/asm-i386/paravirt.h
index e63f1e444fcf..e2e7f98723c5 100644
--- a/include/asm-i386/paravirt.h
+++ b/include/asm-i386/paravirt.h
@@ -2,20 +2,9 @@
2#define __ASM_PARAVIRT_H 2#define __ASM_PARAVIRT_H
3/* Various instructions on x86 need to be replaced for 3/* Various instructions on x86 need to be replaced for
4 * para-virtualization: those hooks are defined here. */ 4 * para-virtualization: those hooks are defined here. */
5#include <linux/linkage.h>
6#include <linux/stringify.h>
7#include <asm/page.h>
8 5
9#ifdef CONFIG_PARAVIRT 6#ifdef CONFIG_PARAVIRT
10/* These are the most performance critical ops, so we want to be able to patch 7#include <asm/page.h>
11 * callers */
12#define PARAVIRT_IRQ_DISABLE 0
13#define PARAVIRT_IRQ_ENABLE 1
14#define PARAVIRT_RESTORE_FLAGS 2
15#define PARAVIRT_SAVE_FLAGS 3
16#define PARAVIRT_SAVE_FLAGS_IRQ_DISABLE 4
17#define PARAVIRT_INTERRUPT_RETURN 5
18#define PARAVIRT_STI_SYSEXIT 6
19 8
20/* Bitmask of what can be clobbered: usually at least eax. */ 9/* Bitmask of what can be clobbered: usually at least eax. */
21#define CLBR_NONE 0x0 10#define CLBR_NONE 0x0
@@ -25,13 +14,29 @@
25#define CLBR_ANY 0x7 14#define CLBR_ANY 0x7
26 15
27#ifndef __ASSEMBLY__ 16#ifndef __ASSEMBLY__
17#include <linux/types.h>
18#include <linux/cpumask.h>
19#include <asm/kmap_types.h>
20
21struct page;
28struct thread_struct; 22struct thread_struct;
29struct Xgt_desc_struct; 23struct Xgt_desc_struct;
30struct tss_struct; 24struct tss_struct;
31struct mm_struct; 25struct mm_struct;
26struct desc_struct;
27
28/* Lazy mode for batching updates / context switch */
29enum paravirt_lazy_mode {
30 PARAVIRT_LAZY_NONE = 0,
31 PARAVIRT_LAZY_MMU = 1,
32 PARAVIRT_LAZY_CPU = 2,
33 PARAVIRT_LAZY_FLUSH = 3,
34};
35
32struct paravirt_ops 36struct paravirt_ops
33{ 37{
34 unsigned int kernel_rpl; 38 unsigned int kernel_rpl;
39 int shared_kernel_pmd;
35 int paravirt_enabled; 40 int paravirt_enabled;
36 const char *name; 41 const char *name;
37 42
@@ -44,24 +49,33 @@ struct paravirt_ops
44 */ 49 */
45 unsigned (*patch)(u8 type, u16 clobber, void *firstinsn, unsigned len); 50 unsigned (*patch)(u8 type, u16 clobber, void *firstinsn, unsigned len);
46 51
52 /* Basic arch-specific setup */
47 void (*arch_setup)(void); 53 void (*arch_setup)(void);
48 char *(*memory_setup)(void); 54 char *(*memory_setup)(void);
49 void (*init_IRQ)(void); 55 void (*init_IRQ)(void);
56 void (*time_init)(void);
50 57
58 /*
59 * Called before/after init_mm pagetable setup. setup_start
60 * may reset %cr3, and may pre-install parts of the pagetable;
61 * pagetable setup is expected to preserve any existing
62 * mapping.
63 */
64 void (*pagetable_setup_start)(pgd_t *pgd_base);
65 void (*pagetable_setup_done)(pgd_t *pgd_base);
66
67 /* Print a banner to identify the environment */
51 void (*banner)(void); 68 void (*banner)(void);
52 69
70 /* Set and set time of day */
53 unsigned long (*get_wallclock)(void); 71 unsigned long (*get_wallclock)(void);
54 int (*set_wallclock)(unsigned long); 72 int (*set_wallclock)(unsigned long);
55 void (*time_init)(void);
56
57 /* All the function pointers here are declared as "fastcall"
58 so that we get a specific register-based calling
59 convention. This makes it easier to implement inline
60 assembler replacements. */
61 73
74 /* cpuid emulation, mostly so that caps bits can be disabled */
62 void (*cpuid)(unsigned int *eax, unsigned int *ebx, 75 void (*cpuid)(unsigned int *eax, unsigned int *ebx,
63 unsigned int *ecx, unsigned int *edx); 76 unsigned int *ecx, unsigned int *edx);
64 77
78 /* hooks for various privileged instructions */
65 unsigned long (*get_debugreg)(int regno); 79 unsigned long (*get_debugreg)(int regno);
66 void (*set_debugreg)(int regno, unsigned long value); 80 void (*set_debugreg)(int regno, unsigned long value);
67 81
@@ -80,15 +94,23 @@ struct paravirt_ops
80 unsigned long (*read_cr4)(void); 94 unsigned long (*read_cr4)(void);
81 void (*write_cr4)(unsigned long); 95 void (*write_cr4)(unsigned long);
82 96
97 /*
98 * Get/set interrupt state. save_fl and restore_fl are only
99 * expected to use X86_EFLAGS_IF; all other bits
100 * returned from save_fl are undefined, and may be ignored by
101 * restore_fl.
102 */
83 unsigned long (*save_fl)(void); 103 unsigned long (*save_fl)(void);
84 void (*restore_fl)(unsigned long); 104 void (*restore_fl)(unsigned long);
85 void (*irq_disable)(void); 105 void (*irq_disable)(void);
86 void (*irq_enable)(void); 106 void (*irq_enable)(void);
87 void (*safe_halt)(void); 107 void (*safe_halt)(void);
88 void (*halt)(void); 108 void (*halt)(void);
109
89 void (*wbinvd)(void); 110 void (*wbinvd)(void);
90 111
91 /* err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */ 112 /* MSR, PMC and TSR operations.
113 err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */
92 u64 (*read_msr)(unsigned int msr, int *err); 114 u64 (*read_msr)(unsigned int msr, int *err);
93 int (*write_msr)(unsigned int msr, u64 val); 115 int (*write_msr)(unsigned int msr, u64 val);
94 116
@@ -97,6 +119,7 @@ struct paravirt_ops
97 u64 (*get_scheduled_cycles)(void); 119 u64 (*get_scheduled_cycles)(void);
98 unsigned long (*get_cpu_khz)(void); 120 unsigned long (*get_cpu_khz)(void);
99 121
122 /* Segment descriptor handling */
100 void (*load_tr_desc)(void); 123 void (*load_tr_desc)(void);
101 void (*load_gdt)(const struct Xgt_desc_struct *); 124 void (*load_gdt)(const struct Xgt_desc_struct *);
102 void (*load_idt)(const struct Xgt_desc_struct *); 125 void (*load_idt)(const struct Xgt_desc_struct *);
@@ -105,59 +128,98 @@ struct paravirt_ops
105 void (*set_ldt)(const void *desc, unsigned entries); 128 void (*set_ldt)(const void *desc, unsigned entries);
106 unsigned long (*store_tr)(void); 129 unsigned long (*store_tr)(void);
107 void (*load_tls)(struct thread_struct *t, unsigned int cpu); 130 void (*load_tls)(struct thread_struct *t, unsigned int cpu);
108 void (*write_ldt_entry)(void *dt, int entrynum, 131 void (*write_ldt_entry)(struct desc_struct *,
109 u32 low, u32 high); 132 int entrynum, u32 low, u32 high);
110 void (*write_gdt_entry)(void *dt, int entrynum, 133 void (*write_gdt_entry)(struct desc_struct *,
111 u32 low, u32 high); 134 int entrynum, u32 low, u32 high);
112 void (*write_idt_entry)(void *dt, int entrynum, 135 void (*write_idt_entry)(struct desc_struct *,
113 u32 low, u32 high); 136 int entrynum, u32 low, u32 high);
114 void (*load_esp0)(struct tss_struct *tss, 137 void (*load_esp0)(struct tss_struct *tss, struct thread_struct *t);
115 struct thread_struct *thread);
116 138
117 void (*set_iopl_mask)(unsigned mask); 139 void (*set_iopl_mask)(unsigned mask);
118
119 void (*io_delay)(void); 140 void (*io_delay)(void);
120 141
142 /*
143 * Hooks for intercepting the creation/use/destruction of an
144 * mm_struct.
145 */
146 void (*activate_mm)(struct mm_struct *prev,
147 struct mm_struct *next);
148 void (*dup_mmap)(struct mm_struct *oldmm,
149 struct mm_struct *mm);
150 void (*exit_mmap)(struct mm_struct *mm);
151
121#ifdef CONFIG_X86_LOCAL_APIC 152#ifdef CONFIG_X86_LOCAL_APIC
153 /*
154 * Direct APIC operations, principally for VMI. Ideally
155 * these shouldn't be in this interface.
156 */
122 void (*apic_write)(unsigned long reg, unsigned long v); 157 void (*apic_write)(unsigned long reg, unsigned long v);
123 void (*apic_write_atomic)(unsigned long reg, unsigned long v); 158 void (*apic_write_atomic)(unsigned long reg, unsigned long v);
124 unsigned long (*apic_read)(unsigned long reg); 159 unsigned long (*apic_read)(unsigned long reg);
125 void (*setup_boot_clock)(void); 160 void (*setup_boot_clock)(void);
126 void (*setup_secondary_clock)(void); 161 void (*setup_secondary_clock)(void);
162
163 void (*startup_ipi_hook)(int phys_apicid,
164 unsigned long start_eip,
165 unsigned long start_esp);
127#endif 166#endif
128 167
168 /* TLB operations */
129 void (*flush_tlb_user)(void); 169 void (*flush_tlb_user)(void);
130 void (*flush_tlb_kernel)(void); 170 void (*flush_tlb_kernel)(void);
131 void (*flush_tlb_single)(u32 addr); 171 void (*flush_tlb_single)(unsigned long addr);
132 172 void (*flush_tlb_others)(const cpumask_t *cpus, struct mm_struct *mm,
133 void (*map_pt_hook)(int type, pte_t *va, u32 pfn); 173 unsigned long va);
134 174
175 /* Hooks for allocating/releasing pagetable pages */
135 void (*alloc_pt)(u32 pfn); 176 void (*alloc_pt)(u32 pfn);
136 void (*alloc_pd)(u32 pfn); 177 void (*alloc_pd)(u32 pfn);
137 void (*alloc_pd_clone)(u32 pfn, u32 clonepfn, u32 start, u32 count); 178 void (*alloc_pd_clone)(u32 pfn, u32 clonepfn, u32 start, u32 count);
138 void (*release_pt)(u32 pfn); 179 void (*release_pt)(u32 pfn);
139 void (*release_pd)(u32 pfn); 180 void (*release_pd)(u32 pfn);
140 181
182 /* Pagetable manipulation functions */
141 void (*set_pte)(pte_t *ptep, pte_t pteval); 183 void (*set_pte)(pte_t *ptep, pte_t pteval);
142 void (*set_pte_at)(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pteval); 184 void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
185 pte_t *ptep, pte_t pteval);
143 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval); 186 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
144 void (*pte_update)(struct mm_struct *mm, u32 addr, pte_t *ptep); 187 void (*pte_update)(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
145 void (*pte_update_defer)(struct mm_struct *mm, u32 addr, pte_t *ptep); 188 void (*pte_update_defer)(struct mm_struct *mm,
189 unsigned long addr, pte_t *ptep);
190
191#ifdef CONFIG_HIGHPTE
192 void *(*kmap_atomic_pte)(struct page *page, enum km_type type);
193#endif
194
146#ifdef CONFIG_X86_PAE 195#ifdef CONFIG_X86_PAE
147 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval); 196 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
148 void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); 197 void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte);
149 void (*set_pud)(pud_t *pudp, pud_t pudval); 198 void (*set_pud)(pud_t *pudp, pud_t pudval);
150 void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); 199 void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
151 void (*pmd_clear)(pmd_t *pmdp); 200 void (*pmd_clear)(pmd_t *pmdp);
201
202 unsigned long long (*pte_val)(pte_t);
203 unsigned long long (*pmd_val)(pmd_t);
204 unsigned long long (*pgd_val)(pgd_t);
205
206 pte_t (*make_pte)(unsigned long long pte);
207 pmd_t (*make_pmd)(unsigned long long pmd);
208 pgd_t (*make_pgd)(unsigned long long pgd);
209#else
210 unsigned long (*pte_val)(pte_t);
211 unsigned long (*pgd_val)(pgd_t);
212
213 pte_t (*make_pte)(unsigned long pte);
214 pgd_t (*make_pgd)(unsigned long pgd);
152#endif 215#endif
153 216
154 void (*set_lazy_mode)(int mode); 217 /* Set deferred update mode, used for batching operations. */
218 void (*set_lazy_mode)(enum paravirt_lazy_mode mode);
155 219
156 /* These two are jmp to, not actually called. */ 220 /* These two are jmp to, not actually called. */
157 void (*irq_enable_sysexit)(void); 221 void (*irq_enable_sysexit)(void);
158 void (*iret)(void); 222 void (*iret)(void);
159
160 void (*startup_ipi_hook)(int phys_apicid, unsigned long start_eip, unsigned long start_esp);
161}; 223};
162 224
163/* Mark a paravirt probe function. */ 225/* Mark a paravirt probe function. */
@@ -167,23 +229,202 @@ struct paravirt_ops
167 229
168extern struct paravirt_ops paravirt_ops; 230extern struct paravirt_ops paravirt_ops;
169 231
170#define paravirt_enabled() (paravirt_ops.paravirt_enabled) 232#define PARAVIRT_PATCH(x) \
233 (offsetof(struct paravirt_ops, x) / sizeof(void *))
234
235#define paravirt_type(type) \
236 [paravirt_typenum] "i" (PARAVIRT_PATCH(type))
237#define paravirt_clobber(clobber) \
238 [paravirt_clobber] "i" (clobber)
239
240/*
241 * Generate some code, and mark it as patchable by the
242 * apply_paravirt() alternate instruction patcher.
243 */
244#define _paravirt_alt(insn_string, type, clobber) \
245 "771:\n\t" insn_string "\n" "772:\n" \
246 ".pushsection .parainstructions,\"a\"\n" \
247 " .long 771b\n" \
248 " .byte " type "\n" \
249 " .byte 772b-771b\n" \
250 " .short " clobber "\n" \
251 ".popsection\n"
252
253/* Generate patchable code, with the default asm parameters. */
254#define paravirt_alt(insn_string) \
255 _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
256
257unsigned paravirt_patch_nop(void);
258unsigned paravirt_patch_ignore(unsigned len);
259unsigned paravirt_patch_call(void *target, u16 tgt_clobbers,
260 void *site, u16 site_clobbers,
261 unsigned len);
262unsigned paravirt_patch_jmp(void *target, void *site, unsigned len);
263unsigned paravirt_patch_default(u8 type, u16 clobbers, void *site, unsigned len);
264
265unsigned paravirt_patch_insns(void *site, unsigned len,
266 const char *start, const char *end);
267
268
269/*
270 * This generates an indirect call based on the operation type number.
271 * The type number, computed in PARAVIRT_PATCH, is derived from the
272 * offset into the paravirt_ops structure, and can therefore be freely
273 * converted back into a structure offset.
274 */
275#define PARAVIRT_CALL "call *(paravirt_ops+%c[paravirt_typenum]*4);"
276
277/*
278 * These macros are intended to wrap calls into a paravirt_ops
279 * operation, so that they can be later identified and patched at
280 * runtime.
281 *
282 * Normally, a call to a pv_op function is a simple indirect call:
283 * (paravirt_ops.operations)(args...).
284 *
285 * Unfortunately, this is a relatively slow operation for modern CPUs,
286 * because it cannot necessarily determine what the destination
287 * address is. In this case, the address is a runtime constant, so at
288 * the very least we can patch the call to e a simple direct call, or
289 * ideally, patch an inline implementation into the callsite. (Direct
290 * calls are essentially free, because the call and return addresses
291 * are completely predictable.)
292 *
293 * These macros rely on the standard gcc "regparm(3)" calling
294 * convention, in which the first three arguments are placed in %eax,
295 * %edx, %ecx (in that order), and the remaining arguments are placed
296 * on the stack. All caller-save registers (eax,edx,ecx) are expected
297 * to be modified (either clobbered or used for return values).
298 *
299 * The call instruction itself is marked by placing its start address
300 * and size into the .parainstructions section, so that
301 * apply_paravirt() in arch/i386/kernel/alternative.c can do the
302 * appropriate patching under the control of the backend paravirt_ops
303 * implementation.
304 *
305 * Unfortunately there's no way to get gcc to generate the args setup
306 * for the call, and then allow the call itself to be generated by an
307 * inline asm. Because of this, we must do the complete arg setup and
308 * return value handling from within these macros. This is fairly
309 * cumbersome.
310 *
311 * There are 5 sets of PVOP_* macros for dealing with 0-4 arguments.
312 * It could be extended to more arguments, but there would be little
313 * to be gained from that. For each number of arguments, there are
314 * the two VCALL and CALL variants for void and non-void functions.
315 *
316 * When there is a return value, the invoker of the macro must specify
317 * the return type. The macro then uses sizeof() on that type to
318 * determine whether its a 32 or 64 bit value, and places the return
319 * in the right register(s) (just %eax for 32-bit, and %edx:%eax for
320 * 64-bit).
321 *
322 * 64-bit arguments are passed as a pair of adjacent 32-bit arguments
323 * in low,high order.
324 *
325 * Small structures are passed and returned in registers. The macro
326 * calling convention can't directly deal with this, so the wrapper
327 * functions must do this.
328 *
329 * These PVOP_* macros are only defined within this header. This
330 * means that all uses must be wrapped in inline functions. This also
331 * makes sure the incoming and outgoing types are always correct.
332 */
333#define __PVOP_CALL(rettype, op, pre, post, ...) \
334 ({ \
335 rettype __ret; \
336 unsigned long __eax, __edx, __ecx; \
337 if (sizeof(rettype) > sizeof(unsigned long)) { \
338 asm volatile(pre \
339 paravirt_alt(PARAVIRT_CALL) \
340 post \
341 : "=a" (__eax), "=d" (__edx), \
342 "=c" (__ecx) \
343 : paravirt_type(op), \
344 paravirt_clobber(CLBR_ANY), \
345 ##__VA_ARGS__ \
346 : "memory", "cc"); \
347 __ret = (rettype)((((u64)__edx) << 32) | __eax); \
348 } else { \
349 asm volatile(pre \
350 paravirt_alt(PARAVIRT_CALL) \
351 post \
352 : "=a" (__eax), "=d" (__edx), \
353 "=c" (__ecx) \
354 : paravirt_type(op), \
355 paravirt_clobber(CLBR_ANY), \
356 ##__VA_ARGS__ \
357 : "memory", "cc"); \
358 __ret = (rettype)__eax; \
359 } \
360 __ret; \
361 })
362#define __PVOP_VCALL(op, pre, post, ...) \
363 ({ \
364 unsigned long __eax, __edx, __ecx; \
365 asm volatile(pre \
366 paravirt_alt(PARAVIRT_CALL) \
367 post \
368 : "=a" (__eax), "=d" (__edx), "=c" (__ecx) \
369 : paravirt_type(op), \
370 paravirt_clobber(CLBR_ANY), \
371 ##__VA_ARGS__ \
372 : "memory", "cc"); \
373 })
374
375#define PVOP_CALL0(rettype, op) \
376 __PVOP_CALL(rettype, op, "", "")
377#define PVOP_VCALL0(op) \
378 __PVOP_VCALL(op, "", "")
379
380#define PVOP_CALL1(rettype, op, arg1) \
381 __PVOP_CALL(rettype, op, "", "", "0" ((u32)(arg1)))
382#define PVOP_VCALL1(op, arg1) \
383 __PVOP_VCALL(op, "", "", "0" ((u32)(arg1)))
384
385#define PVOP_CALL2(rettype, op, arg1, arg2) \
386 __PVOP_CALL(rettype, op, "", "", "0" ((u32)(arg1)), "1" ((u32)(arg2)))
387#define PVOP_VCALL2(op, arg1, arg2) \
388 __PVOP_VCALL(op, "", "", "0" ((u32)(arg1)), "1" ((u32)(arg2)))
389
390#define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \
391 __PVOP_CALL(rettype, op, "", "", "0" ((u32)(arg1)), \
392 "1"((u32)(arg2)), "2"((u32)(arg3)))
393#define PVOP_VCALL3(op, arg1, arg2, arg3) \
394 __PVOP_VCALL(op, "", "", "0" ((u32)(arg1)), "1"((u32)(arg2)), \
395 "2"((u32)(arg3)))
396
397#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
398 __PVOP_CALL(rettype, op, \
399 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
400 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \
401 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
402#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
403 __PVOP_VCALL(op, \
404 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
405 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \
406 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
407
408static inline int paravirt_enabled(void)
409{
410 return paravirt_ops.paravirt_enabled;
411}
171 412
172static inline void load_esp0(struct tss_struct *tss, 413static inline void load_esp0(struct tss_struct *tss,
173 struct thread_struct *thread) 414 struct thread_struct *thread)
174{ 415{
175 paravirt_ops.load_esp0(tss, thread); 416 PVOP_VCALL2(load_esp0, tss, thread);
176} 417}
177 418
178#define ARCH_SETUP paravirt_ops.arch_setup(); 419#define ARCH_SETUP paravirt_ops.arch_setup();
179static inline unsigned long get_wallclock(void) 420static inline unsigned long get_wallclock(void)
180{ 421{
181 return paravirt_ops.get_wallclock(); 422 return PVOP_CALL0(unsigned long, get_wallclock);
182} 423}
183 424
184static inline int set_wallclock(unsigned long nowtime) 425static inline int set_wallclock(unsigned long nowtime)
185{ 426{
186 return paravirt_ops.set_wallclock(nowtime); 427 return PVOP_CALL1(int, set_wallclock, nowtime);
187} 428}
188 429
189static inline void (*choose_time_init(void))(void) 430static inline void (*choose_time_init(void))(void)
@@ -195,113 +436,208 @@ static inline void (*choose_time_init(void))(void)
195static inline void __cpuid(unsigned int *eax, unsigned int *ebx, 436static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
196 unsigned int *ecx, unsigned int *edx) 437 unsigned int *ecx, unsigned int *edx)
197{ 438{
198 paravirt_ops.cpuid(eax, ebx, ecx, edx); 439 PVOP_VCALL4(cpuid, eax, ebx, ecx, edx);
199} 440}
200 441
201/* 442/*
202 * These special macros can be used to get or set a debugging register 443 * These special macros can be used to get or set a debugging register
203 */ 444 */
204#define get_debugreg(var, reg) var = paravirt_ops.get_debugreg(reg) 445static inline unsigned long paravirt_get_debugreg(int reg)
205#define set_debugreg(val, reg) paravirt_ops.set_debugreg(reg, val) 446{
447 return PVOP_CALL1(unsigned long, get_debugreg, reg);
448}
449#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
450static inline void set_debugreg(unsigned long val, int reg)
451{
452 PVOP_VCALL2(set_debugreg, reg, val);
453}
454
455static inline void clts(void)
456{
457 PVOP_VCALL0(clts);
458}
206 459
207#define clts() paravirt_ops.clts() 460static inline unsigned long read_cr0(void)
461{
462 return PVOP_CALL0(unsigned long, read_cr0);
463}
208 464
209#define read_cr0() paravirt_ops.read_cr0() 465static inline void write_cr0(unsigned long x)
210#define write_cr0(x) paravirt_ops.write_cr0(x) 466{
467 PVOP_VCALL1(write_cr0, x);
468}
211 469
212#define read_cr2() paravirt_ops.read_cr2() 470static inline unsigned long read_cr2(void)
213#define write_cr2(x) paravirt_ops.write_cr2(x) 471{
472 return PVOP_CALL0(unsigned long, read_cr2);
473}
214 474
215#define read_cr3() paravirt_ops.read_cr3() 475static inline void write_cr2(unsigned long x)
216#define write_cr3(x) paravirt_ops.write_cr3(x) 476{
477 PVOP_VCALL1(write_cr2, x);
478}
217 479
218#define read_cr4() paravirt_ops.read_cr4() 480static inline unsigned long read_cr3(void)
219#define read_cr4_safe(x) paravirt_ops.read_cr4_safe() 481{
220#define write_cr4(x) paravirt_ops.write_cr4(x) 482 return PVOP_CALL0(unsigned long, read_cr3);
483}
484
485static inline void write_cr3(unsigned long x)
486{
487 PVOP_VCALL1(write_cr3, x);
488}
489
490static inline unsigned long read_cr4(void)
491{
492 return PVOP_CALL0(unsigned long, read_cr4);
493}
494static inline unsigned long read_cr4_safe(void)
495{
496 return PVOP_CALL0(unsigned long, read_cr4_safe);
497}
498
499static inline void write_cr4(unsigned long x)
500{
501 PVOP_VCALL1(write_cr4, x);
502}
221 503
222static inline void raw_safe_halt(void) 504static inline void raw_safe_halt(void)
223{ 505{
224 paravirt_ops.safe_halt(); 506 PVOP_VCALL0(safe_halt);
225} 507}
226 508
227static inline void halt(void) 509static inline void halt(void)
228{ 510{
229 paravirt_ops.safe_halt(); 511 PVOP_VCALL0(safe_halt);
512}
513
514static inline void wbinvd(void)
515{
516 PVOP_VCALL0(wbinvd);
230} 517}
231#define wbinvd() paravirt_ops.wbinvd()
232 518
233#define get_kernel_rpl() (paravirt_ops.kernel_rpl) 519#define get_kernel_rpl() (paravirt_ops.kernel_rpl)
234 520
235#define rdmsr(msr,val1,val2) do { \ 521static inline u64 paravirt_read_msr(unsigned msr, int *err)
236 int _err; \ 522{
237 u64 _l = paravirt_ops.read_msr(msr,&_err); \ 523 return PVOP_CALL2(u64, read_msr, msr, err);
238 val1 = (u32)_l; \ 524}
239 val2 = _l >> 32; \ 525static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
526{
527 return PVOP_CALL3(int, write_msr, msr, low, high);
528}
529
530/* These should all do BUG_ON(_err), but our headers are too tangled. */
531#define rdmsr(msr,val1,val2) do { \
532 int _err; \
533 u64 _l = paravirt_read_msr(msr, &_err); \
534 val1 = (u32)_l; \
535 val2 = _l >> 32; \
240} while(0) 536} while(0)
241 537
242#define wrmsr(msr,val1,val2) do { \ 538#define wrmsr(msr,val1,val2) do { \
243 u64 _l = ((u64)(val2) << 32) | (val1); \ 539 paravirt_write_msr(msr, val1, val2); \
244 paravirt_ops.write_msr((msr), _l); \
245} while(0) 540} while(0)
246 541
247#define rdmsrl(msr,val) do { \ 542#define rdmsrl(msr,val) do { \
248 int _err; \ 543 int _err; \
249 val = paravirt_ops.read_msr((msr),&_err); \ 544 val = paravirt_read_msr(msr, &_err); \
250} while(0) 545} while(0)
251 546
252#define wrmsrl(msr,val) (paravirt_ops.write_msr((msr),(val))) 547#define wrmsrl(msr,val) ((void)paravirt_write_msr(msr, val, 0))
253#define wrmsr_safe(msr,a,b) ({ \ 548#define wrmsr_safe(msr,a,b) paravirt_write_msr(msr, a, b)
254 u64 _l = ((u64)(b) << 32) | (a); \
255 paravirt_ops.write_msr((msr),_l); \
256})
257 549
258/* rdmsr with exception handling */ 550/* rdmsr with exception handling */
259#define rdmsr_safe(msr,a,b) ({ \ 551#define rdmsr_safe(msr,a,b) ({ \
260 int _err; \ 552 int _err; \
261 u64 _l = paravirt_ops.read_msr(msr,&_err); \ 553 u64 _l = paravirt_read_msr(msr, &_err); \
262 (*a) = (u32)_l; \ 554 (*a) = (u32)_l; \
263 (*b) = _l >> 32; \ 555 (*b) = _l >> 32; \
264 _err; }) 556 _err; })
265 557
266#define rdtsc(low,high) do { \ 558
267 u64 _l = paravirt_ops.read_tsc(); \ 559static inline u64 paravirt_read_tsc(void)
268 low = (u32)_l; \ 560{
269 high = _l >> 32; \ 561 return PVOP_CALL0(u64, read_tsc);
562}
563#define rdtsc(low,high) do { \
564 u64 _l = paravirt_read_tsc(); \
565 low = (u32)_l; \
566 high = _l >> 32; \
270} while(0) 567} while(0)
271 568
272#define rdtscl(low) do { \ 569#define rdtscl(low) do { \
273 u64 _l = paravirt_ops.read_tsc(); \ 570 u64 _l = paravirt_read_tsc(); \
274 low = (int)_l; \ 571 low = (int)_l; \
275} while(0) 572} while(0)
276 573
277#define rdtscll(val) (val = paravirt_ops.read_tsc()) 574#define rdtscll(val) (val = paravirt_read_tsc())
278 575
279#define get_scheduled_cycles(val) (val = paravirt_ops.get_scheduled_cycles()) 576#define get_scheduled_cycles(val) (val = paravirt_ops.get_scheduled_cycles())
280#define calculate_cpu_khz() (paravirt_ops.get_cpu_khz()) 577#define calculate_cpu_khz() (paravirt_ops.get_cpu_khz())
281 578
282#define write_tsc(val1,val2) wrmsr(0x10, val1, val2) 579#define write_tsc(val1,val2) wrmsr(0x10, val1, val2)
283 580
284#define rdpmc(counter,low,high) do { \ 581static inline unsigned long long paravirt_read_pmc(int counter)
285 u64 _l = paravirt_ops.read_pmc(); \ 582{
286 low = (u32)_l; \ 583 return PVOP_CALL1(u64, read_pmc, counter);
287 high = _l >> 32; \ 584}
585
586#define rdpmc(counter,low,high) do { \
587 u64 _l = paravirt_read_pmc(counter); \
588 low = (u32)_l; \
589 high = _l >> 32; \
288} while(0) 590} while(0)
289 591
290#define load_TR_desc() (paravirt_ops.load_tr_desc()) 592static inline void load_TR_desc(void)
291#define load_gdt(dtr) (paravirt_ops.load_gdt(dtr)) 593{
292#define load_idt(dtr) (paravirt_ops.load_idt(dtr)) 594 PVOP_VCALL0(load_tr_desc);
293#define set_ldt(addr, entries) (paravirt_ops.set_ldt((addr), (entries))) 595}
294#define store_gdt(dtr) (paravirt_ops.store_gdt(dtr)) 596static inline void load_gdt(const struct Xgt_desc_struct *dtr)
295#define store_idt(dtr) (paravirt_ops.store_idt(dtr)) 597{
296#define store_tr(tr) ((tr) = paravirt_ops.store_tr()) 598 PVOP_VCALL1(load_gdt, dtr);
297#define load_TLS(t,cpu) (paravirt_ops.load_tls((t),(cpu))) 599}
298#define write_ldt_entry(dt, entry, low, high) \ 600static inline void load_idt(const struct Xgt_desc_struct *dtr)
299 (paravirt_ops.write_ldt_entry((dt), (entry), (low), (high))) 601{
300#define write_gdt_entry(dt, entry, low, high) \ 602 PVOP_VCALL1(load_idt, dtr);
301 (paravirt_ops.write_gdt_entry((dt), (entry), (low), (high))) 603}
302#define write_idt_entry(dt, entry, low, high) \ 604static inline void set_ldt(const void *addr, unsigned entries)
303 (paravirt_ops.write_idt_entry((dt), (entry), (low), (high))) 605{
304#define set_iopl_mask(mask) (paravirt_ops.set_iopl_mask(mask)) 606 PVOP_VCALL2(set_ldt, addr, entries);
607}
608static inline void store_gdt(struct Xgt_desc_struct *dtr)
609{
610 PVOP_VCALL1(store_gdt, dtr);
611}
612static inline void store_idt(struct Xgt_desc_struct *dtr)
613{
614 PVOP_VCALL1(store_idt, dtr);
615}
616static inline unsigned long paravirt_store_tr(void)
617{
618 return PVOP_CALL0(unsigned long, store_tr);
619}
620#define store_tr(tr) ((tr) = paravirt_store_tr())
621static inline void load_TLS(struct thread_struct *t, unsigned cpu)
622{
623 PVOP_VCALL2(load_tls, t, cpu);
624}
625static inline void write_ldt_entry(void *dt, int entry, u32 low, u32 high)
626{
627 PVOP_VCALL4(write_ldt_entry, dt, entry, low, high);
628}
629static inline void write_gdt_entry(void *dt, int entry, u32 low, u32 high)
630{
631 PVOP_VCALL4(write_gdt_entry, dt, entry, low, high);
632}
633static inline void write_idt_entry(void *dt, int entry, u32 low, u32 high)
634{
635 PVOP_VCALL4(write_idt_entry, dt, entry, low, high);
636}
637static inline void set_iopl_mask(unsigned mask)
638{
639 PVOP_VCALL1(set_iopl_mask, mask);
640}
305 641
306/* The paravirtualized I/O functions */ 642/* The paravirtualized I/O functions */
307static inline void slow_down_io(void) { 643static inline void slow_down_io(void) {
@@ -319,215 +655,390 @@ static inline void slow_down_io(void) {
319 */ 655 */
320static inline void apic_write(unsigned long reg, unsigned long v) 656static inline void apic_write(unsigned long reg, unsigned long v)
321{ 657{
322 paravirt_ops.apic_write(reg,v); 658 PVOP_VCALL2(apic_write, reg, v);
323} 659}
324 660
325static inline void apic_write_atomic(unsigned long reg, unsigned long v) 661static inline void apic_write_atomic(unsigned long reg, unsigned long v)
326{ 662{
327 paravirt_ops.apic_write_atomic(reg,v); 663 PVOP_VCALL2(apic_write_atomic, reg, v);
328} 664}
329 665
330static inline unsigned long apic_read(unsigned long reg) 666static inline unsigned long apic_read(unsigned long reg)
331{ 667{
332 return paravirt_ops.apic_read(reg); 668 return PVOP_CALL1(unsigned long, apic_read, reg);
333} 669}
334 670
335static inline void setup_boot_clock(void) 671static inline void setup_boot_clock(void)
336{ 672{
337 paravirt_ops.setup_boot_clock(); 673 PVOP_VCALL0(setup_boot_clock);
338} 674}
339 675
340static inline void setup_secondary_clock(void) 676static inline void setup_secondary_clock(void)
341{ 677{
342 paravirt_ops.setup_secondary_clock(); 678 PVOP_VCALL0(setup_secondary_clock);
343} 679}
344#endif 680#endif
345 681
682static inline void paravirt_pagetable_setup_start(pgd_t *base)
683{
684 if (paravirt_ops.pagetable_setup_start)
685 (*paravirt_ops.pagetable_setup_start)(base);
686}
687
688static inline void paravirt_pagetable_setup_done(pgd_t *base)
689{
690 if (paravirt_ops.pagetable_setup_done)
691 (*paravirt_ops.pagetable_setup_done)(base);
692}
693
346#ifdef CONFIG_SMP 694#ifdef CONFIG_SMP
347static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip, 695static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
348 unsigned long start_esp) 696 unsigned long start_esp)
349{ 697{
350 return paravirt_ops.startup_ipi_hook(phys_apicid, start_eip, start_esp); 698 PVOP_VCALL3(startup_ipi_hook, phys_apicid, start_eip, start_esp);
351} 699}
352#endif 700#endif
353 701
354#define __flush_tlb() paravirt_ops.flush_tlb_user() 702static inline void paravirt_activate_mm(struct mm_struct *prev,
355#define __flush_tlb_global() paravirt_ops.flush_tlb_kernel() 703 struct mm_struct *next)
356#define __flush_tlb_single(addr) paravirt_ops.flush_tlb_single(addr) 704{
705 PVOP_VCALL2(activate_mm, prev, next);
706}
357 707
358#define paravirt_map_pt_hook(type, va, pfn) paravirt_ops.map_pt_hook(type, va, pfn) 708static inline void arch_dup_mmap(struct mm_struct *oldmm,
709 struct mm_struct *mm)
710{
711 PVOP_VCALL2(dup_mmap, oldmm, mm);
712}
359 713
360#define paravirt_alloc_pt(pfn) paravirt_ops.alloc_pt(pfn) 714static inline void arch_exit_mmap(struct mm_struct *mm)
361#define paravirt_release_pt(pfn) paravirt_ops.release_pt(pfn) 715{
716 PVOP_VCALL1(exit_mmap, mm);
717}
362 718
363#define paravirt_alloc_pd(pfn) paravirt_ops.alloc_pd(pfn) 719static inline void __flush_tlb(void)
364#define paravirt_alloc_pd_clone(pfn, clonepfn, start, count) \ 720{
365 paravirt_ops.alloc_pd_clone(pfn, clonepfn, start, count) 721 PVOP_VCALL0(flush_tlb_user);
366#define paravirt_release_pd(pfn) paravirt_ops.release_pd(pfn) 722}
723static inline void __flush_tlb_global(void)
724{
725 PVOP_VCALL0(flush_tlb_kernel);
726}
727static inline void __flush_tlb_single(unsigned long addr)
728{
729 PVOP_VCALL1(flush_tlb_single, addr);
730}
367 731
368static inline void set_pte(pte_t *ptep, pte_t pteval) 732static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
733 unsigned long va)
369{ 734{
370 paravirt_ops.set_pte(ptep, pteval); 735 PVOP_VCALL3(flush_tlb_others, &cpumask, mm, va);
371} 736}
372 737
373static inline void set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pteval) 738static inline void paravirt_alloc_pt(unsigned pfn)
374{ 739{
375 paravirt_ops.set_pte_at(mm, addr, ptep, pteval); 740 PVOP_VCALL1(alloc_pt, pfn);
741}
742static inline void paravirt_release_pt(unsigned pfn)
743{
744 PVOP_VCALL1(release_pt, pfn);
376} 745}
377 746
378static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) 747static inline void paravirt_alloc_pd(unsigned pfn)
379{ 748{
380 paravirt_ops.set_pmd(pmdp, pmdval); 749 PVOP_VCALL1(alloc_pd, pfn);
381} 750}
382 751
383static inline void pte_update(struct mm_struct *mm, u32 addr, pte_t *ptep) 752static inline void paravirt_alloc_pd_clone(unsigned pfn, unsigned clonepfn,
753 unsigned start, unsigned count)
384{ 754{
385 paravirt_ops.pte_update(mm, addr, ptep); 755 PVOP_VCALL4(alloc_pd_clone, pfn, clonepfn, start, count);
756}
757static inline void paravirt_release_pd(unsigned pfn)
758{
759 PVOP_VCALL1(release_pd, pfn);
386} 760}
387 761
388static inline void pte_update_defer(struct mm_struct *mm, u32 addr, pte_t *ptep) 762#ifdef CONFIG_HIGHPTE
763static inline void *kmap_atomic_pte(struct page *page, enum km_type type)
389{ 764{
390 paravirt_ops.pte_update_defer(mm, addr, ptep); 765 unsigned long ret;
766 ret = PVOP_CALL2(unsigned long, kmap_atomic_pte, page, type);
767 return (void *)ret;
768}
769#endif
770
771static inline void pte_update(struct mm_struct *mm, unsigned long addr,
772 pte_t *ptep)
773{
774 PVOP_VCALL3(pte_update, mm, addr, ptep);
775}
776
777static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
778 pte_t *ptep)
779{
780 PVOP_VCALL3(pte_update_defer, mm, addr, ptep);
391} 781}
392 782
393#ifdef CONFIG_X86_PAE 783#ifdef CONFIG_X86_PAE
784static inline pte_t __pte(unsigned long long val)
785{
786 unsigned long long ret = PVOP_CALL2(unsigned long long, make_pte,
787 val, val >> 32);
788 return (pte_t) { ret, ret >> 32 };
789}
790
791static inline pmd_t __pmd(unsigned long long val)
792{
793 return (pmd_t) { PVOP_CALL2(unsigned long long, make_pmd, val, val >> 32) };
794}
795
796static inline pgd_t __pgd(unsigned long long val)
797{
798 return (pgd_t) { PVOP_CALL2(unsigned long long, make_pgd, val, val >> 32) };
799}
800
801static inline unsigned long long pte_val(pte_t x)
802{
803 return PVOP_CALL2(unsigned long long, pte_val, x.pte_low, x.pte_high);
804}
805
806static inline unsigned long long pmd_val(pmd_t x)
807{
808 return PVOP_CALL2(unsigned long long, pmd_val, x.pmd, x.pmd >> 32);
809}
810
811static inline unsigned long long pgd_val(pgd_t x)
812{
813 return PVOP_CALL2(unsigned long long, pgd_val, x.pgd, x.pgd >> 32);
814}
815
816static inline void set_pte(pte_t *ptep, pte_t pteval)
817{
818 PVOP_VCALL3(set_pte, ptep, pteval.pte_low, pteval.pte_high);
819}
820
821static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
822 pte_t *ptep, pte_t pteval)
823{
824 /* 5 arg words */
825 paravirt_ops.set_pte_at(mm, addr, ptep, pteval);
826}
827
394static inline void set_pte_atomic(pte_t *ptep, pte_t pteval) 828static inline void set_pte_atomic(pte_t *ptep, pte_t pteval)
395{ 829{
396 paravirt_ops.set_pte_atomic(ptep, pteval); 830 PVOP_VCALL3(set_pte_atomic, ptep, pteval.pte_low, pteval.pte_high);
397} 831}
398 832
399static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) 833static inline void set_pte_present(struct mm_struct *mm, unsigned long addr,
834 pte_t *ptep, pte_t pte)
400{ 835{
836 /* 5 arg words */
401 paravirt_ops.set_pte_present(mm, addr, ptep, pte); 837 paravirt_ops.set_pte_present(mm, addr, ptep, pte);
402} 838}
403 839
840static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval)
841{
842 PVOP_VCALL3(set_pmd, pmdp, pmdval.pmd, pmdval.pmd >> 32);
843}
844
404static inline void set_pud(pud_t *pudp, pud_t pudval) 845static inline void set_pud(pud_t *pudp, pud_t pudval)
405{ 846{
406 paravirt_ops.set_pud(pudp, pudval); 847 PVOP_VCALL3(set_pud, pudp, pudval.pgd.pgd, pudval.pgd.pgd >> 32);
407} 848}
408 849
409static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) 850static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
410{ 851{
411 paravirt_ops.pte_clear(mm, addr, ptep); 852 PVOP_VCALL3(pte_clear, mm, addr, ptep);
412} 853}
413 854
414static inline void pmd_clear(pmd_t *pmdp) 855static inline void pmd_clear(pmd_t *pmdp)
415{ 856{
416 paravirt_ops.pmd_clear(pmdp); 857 PVOP_VCALL1(pmd_clear, pmdp);
417} 858}
418#endif
419 859
420/* Lazy mode for batching updates / context switch */ 860#else /* !CONFIG_X86_PAE */
421#define PARAVIRT_LAZY_NONE 0 861
422#define PARAVIRT_LAZY_MMU 1 862static inline pte_t __pte(unsigned long val)
423#define PARAVIRT_LAZY_CPU 2 863{
424#define PARAVIRT_LAZY_FLUSH 3 864 return (pte_t) { PVOP_CALL1(unsigned long, make_pte, val) };
865}
866
867static inline pgd_t __pgd(unsigned long val)
868{
869 return (pgd_t) { PVOP_CALL1(unsigned long, make_pgd, val) };
870}
871
872static inline unsigned long pte_val(pte_t x)
873{
874 return PVOP_CALL1(unsigned long, pte_val, x.pte_low);
875}
876
877static inline unsigned long pgd_val(pgd_t x)
878{
879 return PVOP_CALL1(unsigned long, pgd_val, x.pgd);
880}
881
882static inline void set_pte(pte_t *ptep, pte_t pteval)
883{
884 PVOP_VCALL2(set_pte, ptep, pteval.pte_low);
885}
886
887static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
888 pte_t *ptep, pte_t pteval)
889{
890 PVOP_VCALL4(set_pte_at, mm, addr, ptep, pteval.pte_low);
891}
892
893static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval)
894{
895 PVOP_VCALL2(set_pmd, pmdp, pmdval.pud.pgd.pgd);
896}
897#endif /* CONFIG_X86_PAE */
425 898
426#define __HAVE_ARCH_ENTER_LAZY_CPU_MODE 899#define __HAVE_ARCH_ENTER_LAZY_CPU_MODE
427#define arch_enter_lazy_cpu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_CPU) 900static inline void arch_enter_lazy_cpu_mode(void)
428#define arch_leave_lazy_cpu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_NONE) 901{
429#define arch_flush_lazy_cpu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_FLUSH) 902 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_CPU);
903}
904
905static inline void arch_leave_lazy_cpu_mode(void)
906{
907 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_NONE);
908}
909
910static inline void arch_flush_lazy_cpu_mode(void)
911{
912 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_FLUSH);
913}
914
430 915
431#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE 916#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
432#define arch_enter_lazy_mmu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_MMU) 917static inline void arch_enter_lazy_mmu_mode(void)
433#define arch_leave_lazy_mmu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_NONE) 918{
434#define arch_flush_lazy_mmu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_FLUSH) 919 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_MMU);
920}
921
922static inline void arch_leave_lazy_mmu_mode(void)
923{
924 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_NONE);
925}
926
927static inline void arch_flush_lazy_mmu_mode(void)
928{
929 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_FLUSH);
930}
931
932void _paravirt_nop(void);
933#define paravirt_nop ((void *)_paravirt_nop)
435 934
436/* These all sit in the .parainstructions section to tell us what to patch. */ 935/* These all sit in the .parainstructions section to tell us what to patch. */
437struct paravirt_patch { 936struct paravirt_patch_site {
438 u8 *instr; /* original instructions */ 937 u8 *instr; /* original instructions */
439 u8 instrtype; /* type of this instruction */ 938 u8 instrtype; /* type of this instruction */
440 u8 len; /* length of original instruction */ 939 u8 len; /* length of original instruction */
441 u16 clobbers; /* what registers you may clobber */ 940 u16 clobbers; /* what registers you may clobber */
442}; 941};
443 942
444#define paravirt_alt(insn_string, typenum, clobber) \ 943extern struct paravirt_patch_site __parainstructions[],
445 "771:\n\t" insn_string "\n" "772:\n" \ 944 __parainstructions_end[];
446 ".pushsection .parainstructions,\"a\"\n" \
447 " .long 771b\n" \
448 " .byte " __stringify(typenum) "\n" \
449 " .byte 772b-771b\n" \
450 " .short " __stringify(clobber) "\n" \
451 ".popsection"
452 945
453static inline unsigned long __raw_local_save_flags(void) 946static inline unsigned long __raw_local_save_flags(void)
454{ 947{
455 unsigned long f; 948 unsigned long f;
456 949
457 __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" 950 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
458 "call *%1;" 951 PARAVIRT_CALL
459 "popl %%edx; popl %%ecx", 952 "popl %%edx; popl %%ecx")
460 PARAVIRT_SAVE_FLAGS, CLBR_NONE) 953 : "=a"(f)
461 : "=a"(f): "m"(paravirt_ops.save_fl) 954 : paravirt_type(save_fl),
462 : "memory", "cc"); 955 paravirt_clobber(CLBR_EAX)
956 : "memory", "cc");
463 return f; 957 return f;
464} 958}
465 959
466static inline void raw_local_irq_restore(unsigned long f) 960static inline void raw_local_irq_restore(unsigned long f)
467{ 961{
468 __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" 962 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
469 "call *%1;" 963 PARAVIRT_CALL
470 "popl %%edx; popl %%ecx", 964 "popl %%edx; popl %%ecx")
471 PARAVIRT_RESTORE_FLAGS, CLBR_EAX) 965 : "=a"(f)
472 : "=a"(f) : "m" (paravirt_ops.restore_fl), "0"(f) 966 : "0"(f),
473 : "memory", "cc"); 967 paravirt_type(restore_fl),
968 paravirt_clobber(CLBR_EAX)
969 : "memory", "cc");
474} 970}
475 971
476static inline void raw_local_irq_disable(void) 972static inline void raw_local_irq_disable(void)
477{ 973{
478 __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" 974 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
479 "call *%0;" 975 PARAVIRT_CALL
480 "popl %%edx; popl %%ecx", 976 "popl %%edx; popl %%ecx")
481 PARAVIRT_IRQ_DISABLE, CLBR_EAX) 977 :
482 : : "m" (paravirt_ops.irq_disable) 978 : paravirt_type(irq_disable),
483 : "memory", "eax", "cc"); 979 paravirt_clobber(CLBR_EAX)
980 : "memory", "eax", "cc");
484} 981}
485 982
486static inline void raw_local_irq_enable(void) 983static inline void raw_local_irq_enable(void)
487{ 984{
488 __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" 985 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
489 "call *%0;" 986 PARAVIRT_CALL
490 "popl %%edx; popl %%ecx", 987 "popl %%edx; popl %%ecx")
491 PARAVIRT_IRQ_ENABLE, CLBR_EAX) 988 :
492 : : "m" (paravirt_ops.irq_enable) 989 : paravirt_type(irq_enable),
493 : "memory", "eax", "cc"); 990 paravirt_clobber(CLBR_EAX)
991 : "memory", "eax", "cc");
494} 992}
495 993
496static inline unsigned long __raw_local_irq_save(void) 994static inline unsigned long __raw_local_irq_save(void)
497{ 995{
498 unsigned long f; 996 unsigned long f;
499 997
500 __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" 998 f = __raw_local_save_flags();
501 "call *%1; pushl %%eax;" 999 raw_local_irq_disable();
502 "call *%2; popl %%eax;"
503 "popl %%edx; popl %%ecx",
504 PARAVIRT_SAVE_FLAGS_IRQ_DISABLE,
505 CLBR_NONE)
506 : "=a"(f)
507 : "m" (paravirt_ops.save_fl),
508 "m" (paravirt_ops.irq_disable)
509 : "memory", "cc");
510 return f; 1000 return f;
511} 1001}
512 1002
513#define CLI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \ 1003#define CLI_STRING \
514 "call *paravirt_ops+%c[irq_disable];" \ 1004 _paravirt_alt("pushl %%ecx; pushl %%edx;" \
515 "popl %%edx; popl %%ecx", \ 1005 "call *paravirt_ops+%c[paravirt_cli_type]*4;" \
516 PARAVIRT_IRQ_DISABLE, CLBR_EAX) 1006 "popl %%edx; popl %%ecx", \
1007 "%c[paravirt_cli_type]", "%c[paravirt_clobber]")
1008
1009#define STI_STRING \
1010 _paravirt_alt("pushl %%ecx; pushl %%edx;" \
1011 "call *paravirt_ops+%c[paravirt_sti_type]*4;" \
1012 "popl %%edx; popl %%ecx", \
1013 "%c[paravirt_sti_type]", "%c[paravirt_clobber]")
517 1014
518#define STI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \
519 "call *paravirt_ops+%c[irq_enable];" \
520 "popl %%edx; popl %%ecx", \
521 PARAVIRT_IRQ_ENABLE, CLBR_EAX)
522#define CLI_STI_CLOBBERS , "%eax" 1015#define CLI_STI_CLOBBERS , "%eax"
523#define CLI_STI_INPUT_ARGS \ 1016#define CLI_STI_INPUT_ARGS \
524 , \ 1017 , \
525 [irq_disable] "i" (offsetof(struct paravirt_ops, irq_disable)), \ 1018 [paravirt_cli_type] "i" (PARAVIRT_PATCH(irq_disable)), \
526 [irq_enable] "i" (offsetof(struct paravirt_ops, irq_enable)) 1019 [paravirt_sti_type] "i" (PARAVIRT_PATCH(irq_enable)), \
1020 paravirt_clobber(CLBR_EAX)
1021
1022/* Make sure as little as possible of this mess escapes. */
1023#undef PARAVIRT_CALL
1024#undef __PVOP_CALL
1025#undef __PVOP_VCALL
1026#undef PVOP_VCALL0
1027#undef PVOP_CALL0
1028#undef PVOP_VCALL1
1029#undef PVOP_CALL1
1030#undef PVOP_VCALL2
1031#undef PVOP_CALL2
1032#undef PVOP_VCALL3
1033#undef PVOP_CALL3
1034#undef PVOP_VCALL4
1035#undef PVOP_CALL4
527 1036
528#else /* __ASSEMBLY__ */ 1037#else /* __ASSEMBLY__ */
529 1038
530#define PARA_PATCH(ptype, clobbers, ops) \ 1039#define PARA_PATCH(off) ((off) / 4)
1040
1041#define PARA_SITE(ptype, clobbers, ops) \
531771:; \ 1042771:; \
532 ops; \ 1043 ops; \
533772:; \ 1044772:; \
@@ -538,28 +1049,30 @@ static inline unsigned long __raw_local_irq_save(void)
538 .short clobbers; \ 1049 .short clobbers; \
539 .popsection 1050 .popsection
540 1051
541#define INTERRUPT_RETURN \ 1052#define INTERRUPT_RETURN \
542 PARA_PATCH(PARAVIRT_INTERRUPT_RETURN, CLBR_ANY, \ 1053 PARA_SITE(PARA_PATCH(PARAVIRT_iret), CLBR_NONE, \
543 jmp *%cs:paravirt_ops+PARAVIRT_iret) 1054 jmp *%cs:paravirt_ops+PARAVIRT_iret)
544 1055
545#define DISABLE_INTERRUPTS(clobbers) \ 1056#define DISABLE_INTERRUPTS(clobbers) \
546 PARA_PATCH(PARAVIRT_IRQ_DISABLE, clobbers, \ 1057 PARA_SITE(PARA_PATCH(PARAVIRT_irq_disable), clobbers, \
547 pushl %ecx; pushl %edx; \ 1058 pushl %eax; pushl %ecx; pushl %edx; \
548 call *paravirt_ops+PARAVIRT_irq_disable; \ 1059 call *%cs:paravirt_ops+PARAVIRT_irq_disable; \
549 popl %edx; popl %ecx) \ 1060 popl %edx; popl %ecx; popl %eax) \
550 1061
551#define ENABLE_INTERRUPTS(clobbers) \ 1062#define ENABLE_INTERRUPTS(clobbers) \
552 PARA_PATCH(PARAVIRT_IRQ_ENABLE, clobbers, \ 1063 PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable), clobbers, \
553 pushl %ecx; pushl %edx; \ 1064 pushl %eax; pushl %ecx; pushl %edx; \
554 call *%cs:paravirt_ops+PARAVIRT_irq_enable; \ 1065 call *%cs:paravirt_ops+PARAVIRT_irq_enable; \
555 popl %edx; popl %ecx) 1066 popl %edx; popl %ecx; popl %eax)
556 1067
557#define ENABLE_INTERRUPTS_SYSEXIT \ 1068#define ENABLE_INTERRUPTS_SYSEXIT \
558 PARA_PATCH(PARAVIRT_STI_SYSEXIT, CLBR_ANY, \ 1069 PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable_sysexit), CLBR_NONE, \
559 jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit) 1070 jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit)
560 1071
561#define GET_CR0_INTO_EAX \ 1072#define GET_CR0_INTO_EAX \
562 call *paravirt_ops+PARAVIRT_read_cr0 1073 push %ecx; push %edx; \
1074 call *paravirt_ops+PARAVIRT_read_cr0; \
1075 pop %edx; pop %ecx
563 1076
564#endif /* __ASSEMBLY__ */ 1077#endif /* __ASSEMBLY__ */
565#endif /* CONFIG_PARAVIRT */ 1078#endif /* CONFIG_PARAVIRT */
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-24 08:22:48 -0400