aboutsummaryrefslogtreecommitdiffstats
path: root/include
diff options
context:
space:
mode:
authorJeremy Fitzhardinge <jeremy@xensource.com>2007-10-16 14:51:29 -0400
committerJeremy Fitzhardinge <jeremy@goop.org>2007-10-16 14:51:29 -0400
commit93b1eab3d29e7ea32ee583de3362da84db06ded8 (patch)
tree8dc7eb61d4c65a48f9ce21a49e392f4967185cfd /include
parentab9c232286c2b77be78441c2d8396500b045777e (diff)
paravirt: refactor struct paravirt_ops into smaller pv_*_ops
This patch refactors the paravirt_ops structure into groups of functionally related ops: pv_info - random info, rather than function entrypoints pv_init_ops - functions used at boot time (some for module_init too) pv_misc_ops - lazy mode, which didn't fit well anywhere else pv_time_ops - time-related functions pv_cpu_ops - various privileged instruction ops pv_irq_ops - operations for managing interrupt state pv_apic_ops - APIC operations pv_mmu_ops - operations for managing pagetables There are several motivations for this: 1. Some of these ops will be general to all x86, and some will be i386/x86-64 specific. This makes it easier to share common stuff while allowing separate implementations where needed. 2. At the moment we must export all of paravirt_ops, but modules only need selected parts of it. This allows us to export on a case by case basis (and also choose which export license we want to apply). 3. Functional groupings make things a bit more readable. Struct paravirt_ops is now only used as a template to generate patch-site identifiers, and to extract function pointers for inserting into jmp/calls when patching. It is only instantiated when needed. Signed-off-by: Jeremy Fitzhardinge <jeremy@xensource.com> Signed-off-by: Rusty Russell <rusty@rustcorp.com.au> Cc: Andi Kleen <ak@suse.de> Cc: Zach Amsden <zach@vmware.com> Cc: Avi Kivity <avi@qumranet.com> Cc: Anthony Liguory <aliguori@us.ibm.com> Cc: "Glauber de Oliveira Costa" <glommer@gmail.com> Cc: Jun Nakajima <jun.nakajima@intel.com>
Diffstat (limited to 'include')
-rw-r--r--include/asm-x86/paravirt.h457
-rw-r--r--include/asm-x86/pgtable-3level-defs.h2
2 files changed, 256 insertions, 203 deletions
diff --git a/include/asm-x86/paravirt.h b/include/asm-x86/paravirt.h
index 9fa3fa9e62d1..19726e12051e 100644
--- a/include/asm-x86/paravirt.h
+++ b/include/asm-x86/paravirt.h
@@ -33,19 +33,23 @@ enum paravirt_lazy_mode {
33 PARAVIRT_LAZY_FLUSH = 3, 33 PARAVIRT_LAZY_FLUSH = 3,
34}; 34};
35 35
36struct paravirt_ops 36
37{ 37/* general info */
38struct pv_info {
38 unsigned int kernel_rpl; 39 unsigned int kernel_rpl;
39 int shared_kernel_pmd; 40 int shared_kernel_pmd;
40 int paravirt_enabled; 41 int paravirt_enabled;
41 const char *name; 42 const char *name;
43};
42 44
45struct pv_init_ops {
43 /* 46 /*
44 * Patch may replace one of the defined code sequences with arbitrary 47 * Patch may replace one of the defined code sequences with
45 * code, subject to the same register constraints. This generally 48 * arbitrary code, subject to the same register constraints.
46 * means the code is not free to clobber any registers other than EAX. 49 * This generally means the code is not free to clobber any
47 * The patch function should return the number of bytes of code 50 * registers other than EAX. The patch function should return
48 * generated, as we nop pad the rest in generic code. 51 * the number of bytes of code generated, as we nop pad the
52 * rest in generic code.
49 */ 53 */
50 unsigned (*patch)(u8 type, u16 clobber, void *insnbuf, 54 unsigned (*patch)(u8 type, u16 clobber, void *insnbuf,
51 unsigned long addr, unsigned len); 55 unsigned long addr, unsigned len);
@@ -55,29 +59,28 @@ struct paravirt_ops
55 char *(*memory_setup)(void); 59 char *(*memory_setup)(void);
56 void (*post_allocator_init)(void); 60 void (*post_allocator_init)(void);
57 61
58 void (*init_IRQ)(void);
59 void (*time_init)(void);
60
61 /*
62 * Called before/after init_mm pagetable setup. setup_start
63 * may reset %cr3, and may pre-install parts of the pagetable;
64 * pagetable setup is expected to preserve any existing
65 * mapping.
66 */
67 void (*pagetable_setup_start)(pgd_t *pgd_base);
68 void (*pagetable_setup_done)(pgd_t *pgd_base);
69
70 /* Print a banner to identify the environment */ 62 /* Print a banner to identify the environment */
71 void (*banner)(void); 63 void (*banner)(void);
64};
65
66
67struct pv_misc_ops {
68 /* Set deferred update mode, used for batching operations. */
69 void (*set_lazy_mode)(enum paravirt_lazy_mode mode);
70};
71
72struct pv_time_ops {
73 void (*time_init)(void);
72 74
73 /* Set and set time of day */ 75 /* Set and set time of day */
74 unsigned long (*get_wallclock)(void); 76 unsigned long (*get_wallclock)(void);
75 int (*set_wallclock)(unsigned long); 77 int (*set_wallclock)(unsigned long);
76 78
77 /* cpuid emulation, mostly so that caps bits can be disabled */ 79 unsigned long long (*sched_clock)(void);
78 void (*cpuid)(unsigned int *eax, unsigned int *ebx, 80 unsigned long (*get_cpu_khz)(void);
79 unsigned int *ecx, unsigned int *edx); 81};
80 82
83struct pv_cpu_ops {
81 /* hooks for various privileged instructions */ 84 /* hooks for various privileged instructions */
82 unsigned long (*get_debugreg)(int regno); 85 unsigned long (*get_debugreg)(int regno);
83 void (*set_debugreg)(int regno, unsigned long value); 86 void (*set_debugreg)(int regno, unsigned long value);
@@ -87,41 +90,10 @@ struct paravirt_ops
87 unsigned long (*read_cr0)(void); 90 unsigned long (*read_cr0)(void);
88 void (*write_cr0)(unsigned long); 91 void (*write_cr0)(unsigned long);
89 92
90 unsigned long (*read_cr2)(void);
91 void (*write_cr2)(unsigned long);
92
93 unsigned long (*read_cr3)(void);
94 void (*write_cr3)(unsigned long);
95
96 unsigned long (*read_cr4_safe)(void); 93 unsigned long (*read_cr4_safe)(void);
97 unsigned long (*read_cr4)(void); 94 unsigned long (*read_cr4)(void);
98 void (*write_cr4)(unsigned long); 95 void (*write_cr4)(unsigned long);
99 96
100 /*
101 * Get/set interrupt state. save_fl and restore_fl are only
102 * expected to use X86_EFLAGS_IF; all other bits
103 * returned from save_fl are undefined, and may be ignored by
104 * restore_fl.
105 */
106 unsigned long (*save_fl)(void);
107 void (*restore_fl)(unsigned long);
108 void (*irq_disable)(void);
109 void (*irq_enable)(void);
110 void (*safe_halt)(void);
111 void (*halt)(void);
112
113 void (*wbinvd)(void);
114
115 /* MSR, PMC and TSR operations.
116 err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */
117 u64 (*read_msr)(unsigned int msr, int *err);
118 int (*write_msr)(unsigned int msr, u64 val);
119
120 u64 (*read_tsc)(void);
121 u64 (*read_pmc)(void);
122 unsigned long long (*sched_clock)(void);
123 unsigned long (*get_cpu_khz)(void);
124
125 /* Segment descriptor handling */ 97 /* Segment descriptor handling */
126 void (*load_tr_desc)(void); 98 void (*load_tr_desc)(void);
127 void (*load_gdt)(const struct Xgt_desc_struct *); 99 void (*load_gdt)(const struct Xgt_desc_struct *);
@@ -140,18 +112,45 @@ struct paravirt_ops
140 void (*load_esp0)(struct tss_struct *tss, struct thread_struct *t); 112 void (*load_esp0)(struct tss_struct *tss, struct thread_struct *t);
141 113
142 void (*set_iopl_mask)(unsigned mask); 114 void (*set_iopl_mask)(unsigned mask);
115
116 void (*wbinvd)(void);
143 void (*io_delay)(void); 117 void (*io_delay)(void);
144 118
119 /* cpuid emulation, mostly so that caps bits can be disabled */
120 void (*cpuid)(unsigned int *eax, unsigned int *ebx,
121 unsigned int *ecx, unsigned int *edx);
122
123 /* MSR, PMC and TSR operations.
124 err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */
125 u64 (*read_msr)(unsigned int msr, int *err);
126 int (*write_msr)(unsigned int msr, u64 val);
127
128 u64 (*read_tsc)(void);
129 u64 (*read_pmc)(void);
130
131 /* These two are jmp to, not actually called. */
132 void (*irq_enable_sysexit)(void);
133 void (*iret)(void);
134};
135
136struct pv_irq_ops {
137 void (*init_IRQ)(void);
138
145 /* 139 /*
146 * Hooks for intercepting the creation/use/destruction of an 140 * Get/set interrupt state. save_fl and restore_fl are only
147 * mm_struct. 141 * expected to use X86_EFLAGS_IF; all other bits
142 * returned from save_fl are undefined, and may be ignored by
143 * restore_fl.
148 */ 144 */
149 void (*activate_mm)(struct mm_struct *prev, 145 unsigned long (*save_fl)(void);
150 struct mm_struct *next); 146 void (*restore_fl)(unsigned long);
151 void (*dup_mmap)(struct mm_struct *oldmm, 147 void (*irq_disable)(void);
152 struct mm_struct *mm); 148 void (*irq_enable)(void);
153 void (*exit_mmap)(struct mm_struct *mm); 149 void (*safe_halt)(void);
150 void (*halt)(void);
151};
154 152
153struct pv_apic_ops {
155#ifdef CONFIG_X86_LOCAL_APIC 154#ifdef CONFIG_X86_LOCAL_APIC
156 /* 155 /*
157 * Direct APIC operations, principally for VMI. Ideally 156 * Direct APIC operations, principally for VMI. Ideally
@@ -167,6 +166,34 @@ struct paravirt_ops
167 unsigned long start_eip, 166 unsigned long start_eip,
168 unsigned long start_esp); 167 unsigned long start_esp);
169#endif 168#endif
169};
170
171struct pv_mmu_ops {
172 /*
173 * Called before/after init_mm pagetable setup. setup_start
174 * may reset %cr3, and may pre-install parts of the pagetable;
175 * pagetable setup is expected to preserve any existing
176 * mapping.
177 */
178 void (*pagetable_setup_start)(pgd_t *pgd_base);
179 void (*pagetable_setup_done)(pgd_t *pgd_base);
180
181 unsigned long (*read_cr2)(void);
182 void (*write_cr2)(unsigned long);
183
184 unsigned long (*read_cr3)(void);
185 void (*write_cr3)(unsigned long);
186
187 /*
188 * Hooks for intercepting the creation/use/destruction of an
189 * mm_struct.
190 */
191 void (*activate_mm)(struct mm_struct *prev,
192 struct mm_struct *next);
193 void (*dup_mmap)(struct mm_struct *oldmm,
194 struct mm_struct *mm);
195 void (*exit_mmap)(struct mm_struct *mm);
196
170 197
171 /* TLB operations */ 198 /* TLB operations */
172 void (*flush_tlb_user)(void); 199 void (*flush_tlb_user)(void);
@@ -191,15 +218,12 @@ struct paravirt_ops
191 void (*pte_update_defer)(struct mm_struct *mm, 218 void (*pte_update_defer)(struct mm_struct *mm,
192 unsigned long addr, pte_t *ptep); 219 unsigned long addr, pte_t *ptep);
193 220
194#ifdef CONFIG_HIGHPTE
195 void *(*kmap_atomic_pte)(struct page *page, enum km_type type);
196#endif
197
198#ifdef CONFIG_X86_PAE 221#ifdef CONFIG_X86_PAE
199 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval); 222 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
200 void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); 223 void (*set_pte_present)(struct mm_struct *mm, unsigned long addr,
224 pte_t *ptep, pte_t pte);
201 void (*set_pud)(pud_t *pudp, pud_t pudval); 225 void (*set_pud)(pud_t *pudp, pud_t pudval);
202 void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); 226 void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
203 void (*pmd_clear)(pmd_t *pmdp); 227 void (*pmd_clear)(pmd_t *pmdp);
204 228
205 unsigned long long (*pte_val)(pte_t); 229 unsigned long long (*pte_val)(pte_t);
@@ -217,21 +241,40 @@ struct paravirt_ops
217 pgd_t (*make_pgd)(unsigned long pgd); 241 pgd_t (*make_pgd)(unsigned long pgd);
218#endif 242#endif
219 243
220 /* Set deferred update mode, used for batching operations. */ 244#ifdef CONFIG_HIGHPTE
221 void (*set_lazy_mode)(enum paravirt_lazy_mode mode); 245 void *(*kmap_atomic_pte)(struct page *page, enum km_type type);
246#endif
247};
222 248
223 /* These two are jmp to, not actually called. */ 249/* This contains all the paravirt structures: we get a convenient
224 void (*irq_enable_sysexit)(void); 250 * number for each function using the offset which we use to indicate
225 void (*iret)(void); 251 * what to patch. */
252struct paravirt_patch_template
253{
254 struct pv_init_ops pv_init_ops;
255 struct pv_misc_ops pv_misc_ops;
256 struct pv_time_ops pv_time_ops;
257 struct pv_cpu_ops pv_cpu_ops;
258 struct pv_irq_ops pv_irq_ops;
259 struct pv_apic_ops pv_apic_ops;
260 struct pv_mmu_ops pv_mmu_ops;
226}; 261};
227 262
228extern struct paravirt_ops paravirt_ops; 263extern struct pv_info pv_info;
264extern struct pv_init_ops pv_init_ops;
265extern struct pv_misc_ops pv_misc_ops;
266extern struct pv_time_ops pv_time_ops;
267extern struct pv_cpu_ops pv_cpu_ops;
268extern struct pv_irq_ops pv_irq_ops;
269extern struct pv_apic_ops pv_apic_ops;
270extern struct pv_mmu_ops pv_mmu_ops;
229 271
230#define PARAVIRT_PATCH(x) \ 272#define PARAVIRT_PATCH(x) \
231 (offsetof(struct paravirt_ops, x) / sizeof(void *)) 273 (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
232 274
233#define paravirt_type(type) \ 275#define paravirt_type(op) \
234 [paravirt_typenum] "i" (PARAVIRT_PATCH(type)) 276 [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \
277 [paravirt_opptr] "m" (op)
235#define paravirt_clobber(clobber) \ 278#define paravirt_clobber(clobber) \
236 [paravirt_clobber] "i" (clobber) 279 [paravirt_clobber] "i" (clobber)
237 280
@@ -258,7 +301,7 @@ unsigned paravirt_patch_call(void *insnbuf,
258 const void *target, u16 tgt_clobbers, 301 const void *target, u16 tgt_clobbers,
259 unsigned long addr, u16 site_clobbers, 302 unsigned long addr, u16 site_clobbers,
260 unsigned len); 303 unsigned len);
261unsigned paravirt_patch_jmp(const void *target, void *insnbuf, 304unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
262 unsigned long addr, unsigned len); 305 unsigned long addr, unsigned len);
263unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf, 306unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
264 unsigned long addr, unsigned len); 307 unsigned long addr, unsigned len);
@@ -271,14 +314,14 @@ int paravirt_disable_iospace(void);
271/* 314/*
272 * This generates an indirect call based on the operation type number. 315 * This generates an indirect call based on the operation type number.
273 * The type number, computed in PARAVIRT_PATCH, is derived from the 316 * The type number, computed in PARAVIRT_PATCH, is derived from the
274 * offset into the paravirt_ops structure, and can therefore be freely 317 * offset into the paravirt_patch_template structure, and can therefore be
275 * converted back into a structure offset. 318 * freely converted back into a structure offset.
276 */ 319 */
277#define PARAVIRT_CALL "call *(paravirt_ops+%c[paravirt_typenum]*4);" 320#define PARAVIRT_CALL "call *%[paravirt_opptr];"
278 321
279/* 322/*
280 * These macros are intended to wrap calls into a paravirt_ops 323 * These macros are intended to wrap calls through one of the paravirt
281 * operation, so that they can be later identified and patched at 324 * ops structs, so that they can be later identified and patched at
282 * runtime. 325 * runtime.
283 * 326 *
284 * Normally, a call to a pv_op function is a simple indirect call: 327 * Normally, a call to a pv_op function is a simple indirect call:
@@ -301,7 +344,7 @@ int paravirt_disable_iospace(void);
301 * The call instruction itself is marked by placing its start address 344 * The call instruction itself is marked by placing its start address
302 * and size into the .parainstructions section, so that 345 * and size into the .parainstructions section, so that
303 * apply_paravirt() in arch/i386/kernel/alternative.c can do the 346 * apply_paravirt() in arch/i386/kernel/alternative.c can do the
304 * appropriate patching under the control of the backend paravirt_ops 347 * appropriate patching under the control of the backend pv_init_ops
305 * implementation. 348 * implementation.
306 * 349 *
307 * Unfortunately there's no way to get gcc to generate the args setup 350 * Unfortunately there's no way to get gcc to generate the args setup
@@ -409,36 +452,36 @@ int paravirt_disable_iospace(void);
409 452
410static inline int paravirt_enabled(void) 453static inline int paravirt_enabled(void)
411{ 454{
412 return paravirt_ops.paravirt_enabled; 455 return pv_info.paravirt_enabled;
413} 456}
414 457
415static inline void load_esp0(struct tss_struct *tss, 458static inline void load_esp0(struct tss_struct *tss,
416 struct thread_struct *thread) 459 struct thread_struct *thread)
417{ 460{
418 PVOP_VCALL2(load_esp0, tss, thread); 461 PVOP_VCALL2(pv_cpu_ops.load_esp0, tss, thread);
419} 462}
420 463
421#define ARCH_SETUP paravirt_ops.arch_setup(); 464#define ARCH_SETUP pv_init_ops.arch_setup();
422static inline unsigned long get_wallclock(void) 465static inline unsigned long get_wallclock(void)
423{ 466{
424 return PVOP_CALL0(unsigned long, get_wallclock); 467 return PVOP_CALL0(unsigned long, pv_time_ops.get_wallclock);
425} 468}
426 469
427static inline int set_wallclock(unsigned long nowtime) 470static inline int set_wallclock(unsigned long nowtime)
428{ 471{
429 return PVOP_CALL1(int, set_wallclock, nowtime); 472 return PVOP_CALL1(int, pv_time_ops.set_wallclock, nowtime);
430} 473}
431 474
432static inline void (*choose_time_init(void))(void) 475static inline void (*choose_time_init(void))(void)
433{ 476{
434 return paravirt_ops.time_init; 477 return pv_time_ops.time_init;
435} 478}
436 479
437/* The paravirtualized CPUID instruction. */ 480/* The paravirtualized CPUID instruction. */
438static inline void __cpuid(unsigned int *eax, unsigned int *ebx, 481static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
439 unsigned int *ecx, unsigned int *edx) 482 unsigned int *ecx, unsigned int *edx)
440{ 483{
441 PVOP_VCALL4(cpuid, eax, ebx, ecx, edx); 484 PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
442} 485}
443 486
444/* 487/*
@@ -446,87 +489,87 @@ static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
446 */ 489 */
447static inline unsigned long paravirt_get_debugreg(int reg) 490static inline unsigned long paravirt_get_debugreg(int reg)
448{ 491{
449 return PVOP_CALL1(unsigned long, get_debugreg, reg); 492 return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
450} 493}
451#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) 494#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
452static inline void set_debugreg(unsigned long val, int reg) 495static inline void set_debugreg(unsigned long val, int reg)
453{ 496{
454 PVOP_VCALL2(set_debugreg, reg, val); 497 PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
455} 498}
456 499
457static inline void clts(void) 500static inline void clts(void)
458{ 501{
459 PVOP_VCALL0(clts); 502 PVOP_VCALL0(pv_cpu_ops.clts);
460} 503}
461 504
462static inline unsigned long read_cr0(void) 505static inline unsigned long read_cr0(void)
463{ 506{
464 return PVOP_CALL0(unsigned long, read_cr0); 507 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
465} 508}
466 509
467static inline void write_cr0(unsigned long x) 510static inline void write_cr0(unsigned long x)
468{ 511{
469 PVOP_VCALL1(write_cr0, x); 512 PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
470} 513}
471 514
472static inline unsigned long read_cr2(void) 515static inline unsigned long read_cr2(void)
473{ 516{
474 return PVOP_CALL0(unsigned long, read_cr2); 517 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
475} 518}
476 519
477static inline void write_cr2(unsigned long x) 520static inline void write_cr2(unsigned long x)
478{ 521{
479 PVOP_VCALL1(write_cr2, x); 522 PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
480} 523}
481 524
482static inline unsigned long read_cr3(void) 525static inline unsigned long read_cr3(void)
483{ 526{
484 return PVOP_CALL0(unsigned long, read_cr3); 527 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
485} 528}
486 529
487static inline void write_cr3(unsigned long x) 530static inline void write_cr3(unsigned long x)
488{ 531{
489 PVOP_VCALL1(write_cr3, x); 532 PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
490} 533}
491 534
492static inline unsigned long read_cr4(void) 535static inline unsigned long read_cr4(void)
493{ 536{
494 return PVOP_CALL0(unsigned long, read_cr4); 537 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
495} 538}
496static inline unsigned long read_cr4_safe(void) 539static inline unsigned long read_cr4_safe(void)
497{ 540{
498 return PVOP_CALL0(unsigned long, read_cr4_safe); 541 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
499} 542}
500 543
501static inline void write_cr4(unsigned long x) 544static inline void write_cr4(unsigned long x)
502{ 545{
503 PVOP_VCALL1(write_cr4, x); 546 PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
504} 547}
505 548
506static inline void raw_safe_halt(void) 549static inline void raw_safe_halt(void)
507{ 550{
508 PVOP_VCALL0(safe_halt); 551 PVOP_VCALL0(pv_irq_ops.safe_halt);
509} 552}
510 553
511static inline void halt(void) 554static inline void halt(void)
512{ 555{
513 PVOP_VCALL0(safe_halt); 556 PVOP_VCALL0(pv_irq_ops.safe_halt);
514} 557}
515 558
516static inline void wbinvd(void) 559static inline void wbinvd(void)
517{ 560{
518 PVOP_VCALL0(wbinvd); 561 PVOP_VCALL0(pv_cpu_ops.wbinvd);
519} 562}
520 563
521#define get_kernel_rpl() (paravirt_ops.kernel_rpl) 564#define get_kernel_rpl() (pv_info.kernel_rpl)
522 565
523static inline u64 paravirt_read_msr(unsigned msr, int *err) 566static inline u64 paravirt_read_msr(unsigned msr, int *err)
524{ 567{
525 return PVOP_CALL2(u64, read_msr, msr, err); 568 return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
526} 569}
527static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) 570static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
528{ 571{
529 return PVOP_CALL3(int, write_msr, msr, low, high); 572 return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
530} 573}
531 574
532/* These should all do BUG_ON(_err), but our headers are too tangled. */ 575/* These should all do BUG_ON(_err), but our headers are too tangled. */
@@ -560,7 +603,7 @@ static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
560 603
561static inline u64 paravirt_read_tsc(void) 604static inline u64 paravirt_read_tsc(void)
562{ 605{
563 return PVOP_CALL0(u64, read_tsc); 606 return PVOP_CALL0(u64, pv_cpu_ops.read_tsc);
564} 607}
565 608
566#define rdtscl(low) do { \ 609#define rdtscl(low) do { \
@@ -572,15 +615,15 @@ static inline u64 paravirt_read_tsc(void)
572 615
573static inline unsigned long long paravirt_sched_clock(void) 616static inline unsigned long long paravirt_sched_clock(void)
574{ 617{
575 return PVOP_CALL0(unsigned long long, sched_clock); 618 return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
576} 619}
577#define calculate_cpu_khz() (paravirt_ops.get_cpu_khz()) 620#define calculate_cpu_khz() (pv_time_ops.get_cpu_khz())
578 621
579#define write_tsc(val1,val2) wrmsr(0x10, val1, val2) 622#define write_tsc(val1,val2) wrmsr(0x10, val1, val2)
580 623
581static inline unsigned long long paravirt_read_pmc(int counter) 624static inline unsigned long long paravirt_read_pmc(int counter)
582{ 625{
583 return PVOP_CALL1(u64, read_pmc, counter); 626 return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
584} 627}
585 628
586#define rdpmc(counter,low,high) do { \ 629#define rdpmc(counter,low,high) do { \
@@ -591,61 +634,61 @@ static inline unsigned long long paravirt_read_pmc(int counter)
591 634
592static inline void load_TR_desc(void) 635static inline void load_TR_desc(void)
593{ 636{
594 PVOP_VCALL0(load_tr_desc); 637 PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
595} 638}
596static inline void load_gdt(const struct Xgt_desc_struct *dtr) 639static inline void load_gdt(const struct Xgt_desc_struct *dtr)
597{ 640{
598 PVOP_VCALL1(load_gdt, dtr); 641 PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
599} 642}
600static inline void load_idt(const struct Xgt_desc_struct *dtr) 643static inline void load_idt(const struct Xgt_desc_struct *dtr)
601{ 644{
602 PVOP_VCALL1(load_idt, dtr); 645 PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
603} 646}
604static inline void set_ldt(const void *addr, unsigned entries) 647static inline void set_ldt(const void *addr, unsigned entries)
605{ 648{
606 PVOP_VCALL2(set_ldt, addr, entries); 649 PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
607} 650}
608static inline void store_gdt(struct Xgt_desc_struct *dtr) 651static inline void store_gdt(struct Xgt_desc_struct *dtr)
609{ 652{
610 PVOP_VCALL1(store_gdt, dtr); 653 PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr);
611} 654}
612static inline void store_idt(struct Xgt_desc_struct *dtr) 655static inline void store_idt(struct Xgt_desc_struct *dtr)
613{ 656{
614 PVOP_VCALL1(store_idt, dtr); 657 PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
615} 658}
616static inline unsigned long paravirt_store_tr(void) 659static inline unsigned long paravirt_store_tr(void)
617{ 660{
618 return PVOP_CALL0(unsigned long, store_tr); 661 return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
619} 662}
620#define store_tr(tr) ((tr) = paravirt_store_tr()) 663#define store_tr(tr) ((tr) = paravirt_store_tr())
621static inline void load_TLS(struct thread_struct *t, unsigned cpu) 664static inline void load_TLS(struct thread_struct *t, unsigned cpu)
622{ 665{
623 PVOP_VCALL2(load_tls, t, cpu); 666 PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
624} 667}
625static inline void write_ldt_entry(void *dt, int entry, u32 low, u32 high) 668static inline void write_ldt_entry(void *dt, int entry, u32 low, u32 high)
626{ 669{
627 PVOP_VCALL4(write_ldt_entry, dt, entry, low, high); 670 PVOP_VCALL4(pv_cpu_ops.write_ldt_entry, dt, entry, low, high);
628} 671}
629static inline void write_gdt_entry(void *dt, int entry, u32 low, u32 high) 672static inline void write_gdt_entry(void *dt, int entry, u32 low, u32 high)
630{ 673{
631 PVOP_VCALL4(write_gdt_entry, dt, entry, low, high); 674 PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, low, high);
632} 675}
633static inline void write_idt_entry(void *dt, int entry, u32 low, u32 high) 676static inline void write_idt_entry(void *dt, int entry, u32 low, u32 high)
634{ 677{
635 PVOP_VCALL4(write_idt_entry, dt, entry, low, high); 678 PVOP_VCALL4(pv_cpu_ops.write_idt_entry, dt, entry, low, high);
636} 679}
637static inline void set_iopl_mask(unsigned mask) 680static inline void set_iopl_mask(unsigned mask)
638{ 681{
639 PVOP_VCALL1(set_iopl_mask, mask); 682 PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
640} 683}
641 684
642/* The paravirtualized I/O functions */ 685/* The paravirtualized I/O functions */
643static inline void slow_down_io(void) { 686static inline void slow_down_io(void) {
644 paravirt_ops.io_delay(); 687 pv_cpu_ops.io_delay();
645#ifdef REALLY_SLOW_IO 688#ifdef REALLY_SLOW_IO
646 paravirt_ops.io_delay(); 689 pv_cpu_ops.io_delay();
647 paravirt_ops.io_delay(); 690 pv_cpu_ops.io_delay();
648 paravirt_ops.io_delay(); 691 pv_cpu_ops.io_delay();
649#endif 692#endif
650} 693}
651 694
@@ -655,121 +698,120 @@ static inline void slow_down_io(void) {
655 */ 698 */
656static inline void apic_write(unsigned long reg, unsigned long v) 699static inline void apic_write(unsigned long reg, unsigned long v)
657{ 700{
658 PVOP_VCALL2(apic_write, reg, v); 701 PVOP_VCALL2(pv_apic_ops.apic_write, reg, v);
659} 702}
660 703
661static inline void apic_write_atomic(unsigned long reg, unsigned long v) 704static inline void apic_write_atomic(unsigned long reg, unsigned long v)
662{ 705{
663 PVOP_VCALL2(apic_write_atomic, reg, v); 706 PVOP_VCALL2(pv_apic_ops.apic_write_atomic, reg, v);
664} 707}
665 708
666static inline unsigned long apic_read(unsigned long reg) 709static inline unsigned long apic_read(unsigned long reg)
667{ 710{
668 return PVOP_CALL1(unsigned long, apic_read, reg); 711 return PVOP_CALL1(unsigned long, pv_apic_ops.apic_read, reg);
669} 712}
670 713
671static inline void setup_boot_clock(void) 714static inline void setup_boot_clock(void)
672{ 715{
673 PVOP_VCALL0(setup_boot_clock); 716 PVOP_VCALL0(pv_apic_ops.setup_boot_clock);
674} 717}
675 718
676static inline void setup_secondary_clock(void) 719static inline void setup_secondary_clock(void)
677{ 720{
678 PVOP_VCALL0(setup_secondary_clock); 721 PVOP_VCALL0(pv_apic_ops.setup_secondary_clock);
679} 722}
680#endif 723#endif
681 724
682static inline void paravirt_post_allocator_init(void) 725static inline void paravirt_post_allocator_init(void)
683{ 726{
684 if (paravirt_ops.post_allocator_init) 727 if (pv_init_ops.post_allocator_init)
685 (*paravirt_ops.post_allocator_init)(); 728 (*pv_init_ops.post_allocator_init)();
686} 729}
687 730
688static inline void paravirt_pagetable_setup_start(pgd_t *base) 731static inline void paravirt_pagetable_setup_start(pgd_t *base)
689{ 732{
690 if (paravirt_ops.pagetable_setup_start) 733 (*pv_mmu_ops.pagetable_setup_start)(base);
691 (*paravirt_ops.pagetable_setup_start)(base);
692} 734}
693 735
694static inline void paravirt_pagetable_setup_done(pgd_t *base) 736static inline void paravirt_pagetable_setup_done(pgd_t *base)
695{ 737{
696 if (paravirt_ops.pagetable_setup_done) 738 (*pv_mmu_ops.pagetable_setup_done)(base);
697 (*paravirt_ops.pagetable_setup_done)(base);
698} 739}
699 740
700#ifdef CONFIG_SMP 741#ifdef CONFIG_SMP
701static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip, 742static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
702 unsigned long start_esp) 743 unsigned long start_esp)
703{ 744{
704 PVOP_VCALL3(startup_ipi_hook, phys_apicid, start_eip, start_esp); 745 PVOP_VCALL3(pv_apic_ops.startup_ipi_hook,
746 phys_apicid, start_eip, start_esp);
705} 747}
706#endif 748#endif
707 749
708static inline void paravirt_activate_mm(struct mm_struct *prev, 750static inline void paravirt_activate_mm(struct mm_struct *prev,
709 struct mm_struct *next) 751 struct mm_struct *next)
710{ 752{
711 PVOP_VCALL2(activate_mm, prev, next); 753 PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
712} 754}
713 755
714static inline void arch_dup_mmap(struct mm_struct *oldmm, 756static inline void arch_dup_mmap(struct mm_struct *oldmm,
715 struct mm_struct *mm) 757 struct mm_struct *mm)
716{ 758{
717 PVOP_VCALL2(dup_mmap, oldmm, mm); 759 PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
718} 760}
719 761
720static inline void arch_exit_mmap(struct mm_struct *mm) 762static inline void arch_exit_mmap(struct mm_struct *mm)
721{ 763{
722 PVOP_VCALL1(exit_mmap, mm); 764 PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
723} 765}
724 766
725static inline void __flush_tlb(void) 767static inline void __flush_tlb(void)
726{ 768{
727 PVOP_VCALL0(flush_tlb_user); 769 PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
728} 770}
729static inline void __flush_tlb_global(void) 771static inline void __flush_tlb_global(void)
730{ 772{
731 PVOP_VCALL0(flush_tlb_kernel); 773 PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
732} 774}
733static inline void __flush_tlb_single(unsigned long addr) 775static inline void __flush_tlb_single(unsigned long addr)
734{ 776{
735 PVOP_VCALL1(flush_tlb_single, addr); 777 PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
736} 778}
737 779
738static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, 780static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
739 unsigned long va) 781 unsigned long va)
740{ 782{
741 PVOP_VCALL3(flush_tlb_others, &cpumask, mm, va); 783 PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, &cpumask, mm, va);
742} 784}
743 785
744static inline void paravirt_alloc_pt(struct mm_struct *mm, unsigned pfn) 786static inline void paravirt_alloc_pt(struct mm_struct *mm, unsigned pfn)
745{ 787{
746 PVOP_VCALL2(alloc_pt, mm, pfn); 788 PVOP_VCALL2(pv_mmu_ops.alloc_pt, mm, pfn);
747} 789}
748static inline void paravirt_release_pt(unsigned pfn) 790static inline void paravirt_release_pt(unsigned pfn)
749{ 791{
750 PVOP_VCALL1(release_pt, pfn); 792 PVOP_VCALL1(pv_mmu_ops.release_pt, pfn);
751} 793}
752 794
753static inline void paravirt_alloc_pd(unsigned pfn) 795static inline void paravirt_alloc_pd(unsigned pfn)
754{ 796{
755 PVOP_VCALL1(alloc_pd, pfn); 797 PVOP_VCALL1(pv_mmu_ops.alloc_pd, pfn);
756} 798}
757 799
758static inline void paravirt_alloc_pd_clone(unsigned pfn, unsigned clonepfn, 800static inline void paravirt_alloc_pd_clone(unsigned pfn, unsigned clonepfn,
759 unsigned start, unsigned count) 801 unsigned start, unsigned count)
760{ 802{
761 PVOP_VCALL4(alloc_pd_clone, pfn, clonepfn, start, count); 803 PVOP_VCALL4(pv_mmu_ops.alloc_pd_clone, pfn, clonepfn, start, count);
762} 804}
763static inline void paravirt_release_pd(unsigned pfn) 805static inline void paravirt_release_pd(unsigned pfn)
764{ 806{
765 PVOP_VCALL1(release_pd, pfn); 807 PVOP_VCALL1(pv_mmu_ops.release_pd, pfn);
766} 808}
767 809
768#ifdef CONFIG_HIGHPTE 810#ifdef CONFIG_HIGHPTE
769static inline void *kmap_atomic_pte(struct page *page, enum km_type type) 811static inline void *kmap_atomic_pte(struct page *page, enum km_type type)
770{ 812{
771 unsigned long ret; 813 unsigned long ret;
772 ret = PVOP_CALL2(unsigned long, kmap_atomic_pte, page, type); 814 ret = PVOP_CALL2(unsigned long, pv_mmu_ops.kmap_atomic_pte, page, type);
773 return (void *)ret; 815 return (void *)ret;
774} 816}
775#endif 817#endif
@@ -777,162 +819,171 @@ static inline void *kmap_atomic_pte(struct page *page, enum km_type type)
777static inline void pte_update(struct mm_struct *mm, unsigned long addr, 819static inline void pte_update(struct mm_struct *mm, unsigned long addr,
778 pte_t *ptep) 820 pte_t *ptep)
779{ 821{
780 PVOP_VCALL3(pte_update, mm, addr, ptep); 822 PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
781} 823}
782 824
783static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr, 825static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
784 pte_t *ptep) 826 pte_t *ptep)
785{ 827{
786 PVOP_VCALL3(pte_update_defer, mm, addr, ptep); 828 PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
787} 829}
788 830
789#ifdef CONFIG_X86_PAE 831#ifdef CONFIG_X86_PAE
790static inline pte_t __pte(unsigned long long val) 832static inline pte_t __pte(unsigned long long val)
791{ 833{
792 unsigned long long ret = PVOP_CALL2(unsigned long long, make_pte, 834 unsigned long long ret = PVOP_CALL2(unsigned long long,
835 pv_mmu_ops.make_pte,
793 val, val >> 32); 836 val, val >> 32);
794 return (pte_t) { ret, ret >> 32 }; 837 return (pte_t) { ret, ret >> 32 };
795} 838}
796 839
797static inline pmd_t __pmd(unsigned long long val) 840static inline pmd_t __pmd(unsigned long long val)
798{ 841{
799 return (pmd_t) { PVOP_CALL2(unsigned long long, make_pmd, val, val >> 32) }; 842 return (pmd_t) { PVOP_CALL2(unsigned long long, pv_mmu_ops.make_pmd,
843 val, val >> 32) };
800} 844}
801 845
802static inline pgd_t __pgd(unsigned long long val) 846static inline pgd_t __pgd(unsigned long long val)
803{ 847{
804 return (pgd_t) { PVOP_CALL2(unsigned long long, make_pgd, val, val >> 32) }; 848 return (pgd_t) { PVOP_CALL2(unsigned long long, pv_mmu_ops.make_pgd,
849 val, val >> 32) };
805} 850}
806 851
807static inline unsigned long long pte_val(pte_t x) 852static inline unsigned long long pte_val(pte_t x)
808{ 853{
809 return PVOP_CALL2(unsigned long long, pte_val, x.pte_low, x.pte_high); 854 return PVOP_CALL2(unsigned long long, pv_mmu_ops.pte_val,
855 x.pte_low, x.pte_high);
810} 856}
811 857
812static inline unsigned long long pmd_val(pmd_t x) 858static inline unsigned long long pmd_val(pmd_t x)
813{ 859{
814 return PVOP_CALL2(unsigned long long, pmd_val, x.pmd, x.pmd >> 32); 860 return PVOP_CALL2(unsigned long long, pv_mmu_ops.pmd_val,
861 x.pmd, x.pmd >> 32);
815} 862}
816 863
817static inline unsigned long long pgd_val(pgd_t x) 864static inline unsigned long long pgd_val(pgd_t x)
818{ 865{
819 return PVOP_CALL2(unsigned long long, pgd_val, x.pgd, x.pgd >> 32); 866 return PVOP_CALL2(unsigned long long, pv_mmu_ops.pgd_val,
867 x.pgd, x.pgd >> 32);
820} 868}
821 869
822static inline void set_pte(pte_t *ptep, pte_t pteval) 870static inline void set_pte(pte_t *ptep, pte_t pteval)
823{ 871{
824 PVOP_VCALL3(set_pte, ptep, pteval.pte_low, pteval.pte_high); 872 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep, pteval.pte_low, pteval.pte_high);
825} 873}
826 874
827static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, 875static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
828 pte_t *ptep, pte_t pteval) 876 pte_t *ptep, pte_t pteval)
829{ 877{
830 /* 5 arg words */ 878 /* 5 arg words */
831 paravirt_ops.set_pte_at(mm, addr, ptep, pteval); 879 pv_mmu_ops.set_pte_at(mm, addr, ptep, pteval);
832} 880}
833 881
834static inline void set_pte_atomic(pte_t *ptep, pte_t pteval) 882static inline void set_pte_atomic(pte_t *ptep, pte_t pteval)
835{ 883{
836 PVOP_VCALL3(set_pte_atomic, ptep, pteval.pte_low, pteval.pte_high); 884 PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
885 pteval.pte_low, pteval.pte_high);
837} 886}
838 887
839static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, 888static inline void set_pte_present(struct mm_struct *mm, unsigned long addr,
840 pte_t *ptep, pte_t pte) 889 pte_t *ptep, pte_t pte)
841{ 890{
842 /* 5 arg words */ 891 /* 5 arg words */
843 paravirt_ops.set_pte_present(mm, addr, ptep, pte); 892 pv_mmu_ops.set_pte_present(mm, addr, ptep, pte);
844} 893}
845 894
846static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) 895static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval)
847{ 896{
848 PVOP_VCALL3(set_pmd, pmdp, pmdval.pmd, pmdval.pmd >> 32); 897 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp,
898 pmdval.pmd, pmdval.pmd >> 32);
849} 899}
850 900
851static inline void set_pud(pud_t *pudp, pud_t pudval) 901static inline void set_pud(pud_t *pudp, pud_t pudval)
852{ 902{
853 PVOP_VCALL3(set_pud, pudp, pudval.pgd.pgd, pudval.pgd.pgd >> 32); 903 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
904 pudval.pgd.pgd, pudval.pgd.pgd >> 32);
854} 905}
855 906
856static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) 907static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
857{ 908{
858 PVOP_VCALL3(pte_clear, mm, addr, ptep); 909 PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
859} 910}
860 911
861static inline void pmd_clear(pmd_t *pmdp) 912static inline void pmd_clear(pmd_t *pmdp)
862{ 913{
863 PVOP_VCALL1(pmd_clear, pmdp); 914 PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
864} 915}
865 916
866#else /* !CONFIG_X86_PAE */ 917#else /* !CONFIG_X86_PAE */
867 918
868static inline pte_t __pte(unsigned long val) 919static inline pte_t __pte(unsigned long val)
869{ 920{
870 return (pte_t) { PVOP_CALL1(unsigned long, make_pte, val) }; 921 return (pte_t) { PVOP_CALL1(unsigned long, pv_mmu_ops.make_pte, val) };
871} 922}
872 923
873static inline pgd_t __pgd(unsigned long val) 924static inline pgd_t __pgd(unsigned long val)
874{ 925{
875 return (pgd_t) { PVOP_CALL1(unsigned long, make_pgd, val) }; 926 return (pgd_t) { PVOP_CALL1(unsigned long, pv_mmu_ops.make_pgd, val) };
876} 927}
877 928
878static inline unsigned long pte_val(pte_t x) 929static inline unsigned long pte_val(pte_t x)
879{ 930{
880 return PVOP_CALL1(unsigned long, pte_val, x.pte_low); 931 return PVOP_CALL1(unsigned long, pv_mmu_ops.pte_val, x.pte_low);
881} 932}
882 933
883static inline unsigned long pgd_val(pgd_t x) 934static inline unsigned long pgd_val(pgd_t x)
884{ 935{
885 return PVOP_CALL1(unsigned long, pgd_val, x.pgd); 936 return PVOP_CALL1(unsigned long, pv_mmu_ops.pgd_val, x.pgd);
886} 937}
887 938
888static inline void set_pte(pte_t *ptep, pte_t pteval) 939static inline void set_pte(pte_t *ptep, pte_t pteval)
889{ 940{
890 PVOP_VCALL2(set_pte, ptep, pteval.pte_low); 941 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep, pteval.pte_low);
891} 942}
892 943
893static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, 944static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
894 pte_t *ptep, pte_t pteval) 945 pte_t *ptep, pte_t pteval)
895{ 946{
896 PVOP_VCALL4(set_pte_at, mm, addr, ptep, pteval.pte_low); 947 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pteval.pte_low);
897} 948}
898 949
899static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) 950static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval)
900{ 951{
901 PVOP_VCALL2(set_pmd, pmdp, pmdval.pud.pgd.pgd); 952 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, pmdval.pud.pgd.pgd);
902} 953}
903#endif /* CONFIG_X86_PAE */ 954#endif /* CONFIG_X86_PAE */
904 955
905#define __HAVE_ARCH_ENTER_LAZY_CPU_MODE 956#define __HAVE_ARCH_ENTER_LAZY_CPU_MODE
906static inline void arch_enter_lazy_cpu_mode(void) 957static inline void arch_enter_lazy_cpu_mode(void)
907{ 958{
908 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_CPU); 959 PVOP_VCALL1(pv_misc_ops.set_lazy_mode, PARAVIRT_LAZY_CPU);
909} 960}
910 961
911static inline void arch_leave_lazy_cpu_mode(void) 962static inline void arch_leave_lazy_cpu_mode(void)
912{ 963{
913 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_NONE); 964 PVOP_VCALL1(pv_misc_ops.set_lazy_mode, PARAVIRT_LAZY_NONE);
914} 965}
915 966
916static inline void arch_flush_lazy_cpu_mode(void) 967static inline void arch_flush_lazy_cpu_mode(void)
917{ 968{
918 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_FLUSH); 969 PVOP_VCALL1(pv_misc_ops.set_lazy_mode, PARAVIRT_LAZY_FLUSH);
919} 970}
920 971
921 972
922#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE 973#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
923static inline void arch_enter_lazy_mmu_mode(void) 974static inline void arch_enter_lazy_mmu_mode(void)
924{ 975{
925 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_MMU); 976 PVOP_VCALL1(pv_misc_ops.set_lazy_mode, PARAVIRT_LAZY_MMU);
926} 977}
927 978
928static inline void arch_leave_lazy_mmu_mode(void) 979static inline void arch_leave_lazy_mmu_mode(void)
929{ 980{
930 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_NONE); 981 PVOP_VCALL1(pv_misc_ops.set_lazy_mode, PARAVIRT_LAZY_NONE);
931} 982}
932 983
933static inline void arch_flush_lazy_mmu_mode(void) 984static inline void arch_flush_lazy_mmu_mode(void)
934{ 985{
935 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_FLUSH); 986 PVOP_VCALL1(pv_misc_ops.set_lazy_mode, PARAVIRT_LAZY_FLUSH);
936} 987}
937 988
938void _paravirt_nop(void); 989void _paravirt_nop(void);
@@ -957,7 +1008,7 @@ static inline unsigned long __raw_local_save_flags(void)
957 PARAVIRT_CALL 1008 PARAVIRT_CALL
958 "popl %%edx; popl %%ecx") 1009 "popl %%edx; popl %%ecx")
959 : "=a"(f) 1010 : "=a"(f)
960 : paravirt_type(save_fl), 1011 : paravirt_type(pv_irq_ops.save_fl),
961 paravirt_clobber(CLBR_EAX) 1012 paravirt_clobber(CLBR_EAX)
962 : "memory", "cc"); 1013 : "memory", "cc");
963 return f; 1014 return f;
@@ -970,7 +1021,7 @@ static inline void raw_local_irq_restore(unsigned long f)
970 "popl %%edx; popl %%ecx") 1021 "popl %%edx; popl %%ecx")
971 : "=a"(f) 1022 : "=a"(f)
972 : "0"(f), 1023 : "0"(f),
973 paravirt_type(restore_fl), 1024 paravirt_type(pv_irq_ops.restore_fl),
974 paravirt_clobber(CLBR_EAX) 1025 paravirt_clobber(CLBR_EAX)
975 : "memory", "cc"); 1026 : "memory", "cc");
976} 1027}
@@ -981,7 +1032,7 @@ static inline void raw_local_irq_disable(void)
981 PARAVIRT_CALL 1032 PARAVIRT_CALL
982 "popl %%edx; popl %%ecx") 1033 "popl %%edx; popl %%ecx")
983 : 1034 :
984 : paravirt_type(irq_disable), 1035 : paravirt_type(pv_irq_ops.irq_disable),
985 paravirt_clobber(CLBR_EAX) 1036 paravirt_clobber(CLBR_EAX)
986 : "memory", "eax", "cc"); 1037 : "memory", "eax", "cc");
987} 1038}
@@ -992,7 +1043,7 @@ static inline void raw_local_irq_enable(void)
992 PARAVIRT_CALL 1043 PARAVIRT_CALL
993 "popl %%edx; popl %%ecx") 1044 "popl %%edx; popl %%ecx")
994 : 1045 :
995 : paravirt_type(irq_enable), 1046 : paravirt_type(pv_irq_ops.irq_enable),
996 paravirt_clobber(CLBR_EAX) 1047 paravirt_clobber(CLBR_EAX)
997 : "memory", "eax", "cc"); 1048 : "memory", "eax", "cc");
998} 1049}
@@ -1008,21 +1059,23 @@ static inline unsigned long __raw_local_irq_save(void)
1008 1059
1009#define CLI_STRING \ 1060#define CLI_STRING \
1010 _paravirt_alt("pushl %%ecx; pushl %%edx;" \ 1061 _paravirt_alt("pushl %%ecx; pushl %%edx;" \
1011 "call *paravirt_ops+%c[paravirt_cli_type]*4;" \ 1062 "call *%[paravirt_cli_opptr];" \
1012 "popl %%edx; popl %%ecx", \ 1063 "popl %%edx; popl %%ecx", \
1013 "%c[paravirt_cli_type]", "%c[paravirt_clobber]") 1064 "%c[paravirt_cli_type]", "%c[paravirt_clobber]")
1014 1065
1015#define STI_STRING \ 1066#define STI_STRING \
1016 _paravirt_alt("pushl %%ecx; pushl %%edx;" \ 1067 _paravirt_alt("pushl %%ecx; pushl %%edx;" \
1017 "call *paravirt_ops+%c[paravirt_sti_type]*4;" \ 1068 "call *%[paravirt_sti_opptr];" \
1018 "popl %%edx; popl %%ecx", \ 1069 "popl %%edx; popl %%ecx", \
1019 "%c[paravirt_sti_type]", "%c[paravirt_clobber]") 1070 "%c[paravirt_sti_type]", "%c[paravirt_clobber]")
1020 1071
1021#define CLI_STI_CLOBBERS , "%eax" 1072#define CLI_STI_CLOBBERS , "%eax"
1022#define CLI_STI_INPUT_ARGS \ 1073#define CLI_STI_INPUT_ARGS \
1023 , \ 1074 , \
1024 [paravirt_cli_type] "i" (PARAVIRT_PATCH(irq_disable)), \ 1075 [paravirt_cli_type] "i" (PARAVIRT_PATCH(pv_irq_ops.irq_disable)), \
1025 [paravirt_sti_type] "i" (PARAVIRT_PATCH(irq_enable)), \ 1076 [paravirt_cli_opptr] "m" (pv_irq_ops.irq_disable), \
1077 [paravirt_sti_type] "i" (PARAVIRT_PATCH(pv_irq_ops.irq_enable)), \
1078 [paravirt_sti_opptr] "m" (pv_irq_ops.irq_enable), \
1026 paravirt_clobber(CLBR_EAX) 1079 paravirt_clobber(CLBR_EAX)
1027 1080
1028/* Make sure as little as possible of this mess escapes. */ 1081/* Make sure as little as possible of this mess escapes. */
@@ -1042,7 +1095,7 @@ static inline unsigned long __raw_local_irq_save(void)
1042 1095
1043#else /* __ASSEMBLY__ */ 1096#else /* __ASSEMBLY__ */
1044 1097
1045#define PARA_PATCH(off) ((off) / 4) 1098#define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4)
1046 1099
1047#define PARA_SITE(ptype, clobbers, ops) \ 1100#define PARA_SITE(ptype, clobbers, ops) \
1048771:; \ 1101771:; \
@@ -1055,29 +1108,29 @@ static inline unsigned long __raw_local_irq_save(void)
1055 .short clobbers; \ 1108 .short clobbers; \
1056 .popsection 1109 .popsection
1057 1110
1058#define INTERRUPT_RETURN \ 1111#define INTERRUPT_RETURN \
1059 PARA_SITE(PARA_PATCH(PARAVIRT_iret), CLBR_NONE, \ 1112 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \
1060 jmp *%cs:paravirt_ops+PARAVIRT_iret) 1113 jmp *%cs:pv_cpu_ops+PV_CPU_iret)
1061 1114
1062#define DISABLE_INTERRUPTS(clobbers) \ 1115#define DISABLE_INTERRUPTS(clobbers) \
1063 PARA_SITE(PARA_PATCH(PARAVIRT_irq_disable), clobbers, \ 1116 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
1064 pushl %eax; pushl %ecx; pushl %edx; \ 1117 pushl %eax; pushl %ecx; pushl %edx; \
1065 call *%cs:paravirt_ops+PARAVIRT_irq_disable; \ 1118 call *%cs:pv_irq_ops+PV_IRQ_irq_disable; \
1066 popl %edx; popl %ecx; popl %eax) \ 1119 popl %edx; popl %ecx; popl %eax) \
1067 1120
1068#define ENABLE_INTERRUPTS(clobbers) \ 1121#define ENABLE_INTERRUPTS(clobbers) \
1069 PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable), clobbers, \ 1122 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \
1070 pushl %eax; pushl %ecx; pushl %edx; \ 1123 pushl %eax; pushl %ecx; pushl %edx; \
1071 call *%cs:paravirt_ops+PARAVIRT_irq_enable; \ 1124 call *%cs:pv_irq_ops+PV_IRQ_irq_enable; \
1072 popl %edx; popl %ecx; popl %eax) 1125 popl %edx; popl %ecx; popl %eax)
1073 1126
1074#define ENABLE_INTERRUPTS_SYSEXIT \ 1127#define ENABLE_INTERRUPTS_SYSEXIT \
1075 PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable_sysexit), CLBR_NONE, \ 1128 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), CLBR_NONE,\
1076 jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit) 1129 jmp *%cs:pv_cpu_ops+PV_CPU_irq_enable_sysexit)
1077 1130
1078#define GET_CR0_INTO_EAX \ 1131#define GET_CR0_INTO_EAX \
1079 push %ecx; push %edx; \ 1132 push %ecx; push %edx; \
1080 call *paravirt_ops+PARAVIRT_read_cr0; \ 1133 call *pv_cpu_ops+PV_CPU_read_cr0; \
1081 pop %edx; pop %ecx 1134 pop %edx; pop %ecx
1082 1135
1083#endif /* __ASSEMBLY__ */ 1136#endif /* __ASSEMBLY__ */
diff --git a/include/asm-x86/pgtable-3level-defs.h b/include/asm-x86/pgtable-3level-defs.h
index c0df89f66e8b..448ac9516314 100644
--- a/include/asm-x86/pgtable-3level-defs.h
+++ b/include/asm-x86/pgtable-3level-defs.h
@@ -2,7 +2,7 @@
2#define _I386_PGTABLE_3LEVEL_DEFS_H 2#define _I386_PGTABLE_3LEVEL_DEFS_H
3 3
4#ifdef CONFIG_PARAVIRT 4#ifdef CONFIG_PARAVIRT
5#define SHARED_KERNEL_PMD (paravirt_ops.shared_kernel_pmd) 5#define SHARED_KERNEL_PMD (pv_info.shared_kernel_pmd)
6#else 6#else
7#define SHARED_KERNEL_PMD 1 7#define SHARED_KERNEL_PMD 1
8#endif 8#endif