diff options
author | Jeremy Fitzhardinge <jeremy@xensource.com> | 2007-10-16 14:51:29 -0400 |
---|---|---|
committer | Jeremy Fitzhardinge <jeremy@goop.org> | 2007-10-16 14:51:29 -0400 |
commit | 93b1eab3d29e7ea32ee583de3362da84db06ded8 (patch) | |
tree | 8dc7eb61d4c65a48f9ce21a49e392f4967185cfd /include | |
parent | ab9c232286c2b77be78441c2d8396500b045777e (diff) |
paravirt: refactor struct paravirt_ops into smaller pv_*_ops
This patch refactors the paravirt_ops structure into groups of
functionally related ops:
pv_info - random info, rather than function entrypoints
pv_init_ops - functions used at boot time (some for module_init too)
pv_misc_ops - lazy mode, which didn't fit well anywhere else
pv_time_ops - time-related functions
pv_cpu_ops - various privileged instruction ops
pv_irq_ops - operations for managing interrupt state
pv_apic_ops - APIC operations
pv_mmu_ops - operations for managing pagetables
There are several motivations for this:
1. Some of these ops will be general to all x86, and some will be
i386/x86-64 specific. This makes it easier to share common stuff
while allowing separate implementations where needed.
2. At the moment we must export all of paravirt_ops, but modules only
need selected parts of it. This allows us to export on a case by case
basis (and also choose which export license we want to apply).
3. Functional groupings make things a bit more readable.
Struct paravirt_ops is now only used as a template to generate
patch-site identifiers, and to extract function pointers for inserting
into jmp/calls when patching. It is only instantiated when needed.
Signed-off-by: Jeremy Fitzhardinge <jeremy@xensource.com>
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Cc: Andi Kleen <ak@suse.de>
Cc: Zach Amsden <zach@vmware.com>
Cc: Avi Kivity <avi@qumranet.com>
Cc: Anthony Liguory <aliguori@us.ibm.com>
Cc: "Glauber de Oliveira Costa" <glommer@gmail.com>
Cc: Jun Nakajima <jun.nakajima@intel.com>
Diffstat (limited to 'include')
-rw-r--r-- | include/asm-x86/paravirt.h | 457 | ||||
-rw-r--r-- | include/asm-x86/pgtable-3level-defs.h | 2 |
2 files changed, 256 insertions, 203 deletions
diff --git a/include/asm-x86/paravirt.h b/include/asm-x86/paravirt.h index 9fa3fa9e62d1..19726e12051e 100644 --- a/include/asm-x86/paravirt.h +++ b/include/asm-x86/paravirt.h | |||
@@ -33,19 +33,23 @@ enum paravirt_lazy_mode { | |||
33 | PARAVIRT_LAZY_FLUSH = 3, | 33 | PARAVIRT_LAZY_FLUSH = 3, |
34 | }; | 34 | }; |
35 | 35 | ||
36 | struct paravirt_ops | 36 | |
37 | { | 37 | /* general info */ |
38 | struct pv_info { | ||
38 | unsigned int kernel_rpl; | 39 | unsigned int kernel_rpl; |
39 | int shared_kernel_pmd; | 40 | int shared_kernel_pmd; |
40 | int paravirt_enabled; | 41 | int paravirt_enabled; |
41 | const char *name; | 42 | const char *name; |
43 | }; | ||
42 | 44 | ||
45 | struct pv_init_ops { | ||
43 | /* | 46 | /* |
44 | * Patch may replace one of the defined code sequences with arbitrary | 47 | * Patch may replace one of the defined code sequences with |
45 | * code, subject to the same register constraints. This generally | 48 | * arbitrary code, subject to the same register constraints. |
46 | * means the code is not free to clobber any registers other than EAX. | 49 | * This generally means the code is not free to clobber any |
47 | * The patch function should return the number of bytes of code | 50 | * registers other than EAX. The patch function should return |
48 | * generated, as we nop pad the rest in generic code. | 51 | * the number of bytes of code generated, as we nop pad the |
52 | * rest in generic code. | ||
49 | */ | 53 | */ |
50 | unsigned (*patch)(u8 type, u16 clobber, void *insnbuf, | 54 | unsigned (*patch)(u8 type, u16 clobber, void *insnbuf, |
51 | unsigned long addr, unsigned len); | 55 | unsigned long addr, unsigned len); |
@@ -55,29 +59,28 @@ struct paravirt_ops | |||
55 | char *(*memory_setup)(void); | 59 | char *(*memory_setup)(void); |
56 | void (*post_allocator_init)(void); | 60 | void (*post_allocator_init)(void); |
57 | 61 | ||
58 | void (*init_IRQ)(void); | ||
59 | void (*time_init)(void); | ||
60 | |||
61 | /* | ||
62 | * Called before/after init_mm pagetable setup. setup_start | ||
63 | * may reset %cr3, and may pre-install parts of the pagetable; | ||
64 | * pagetable setup is expected to preserve any existing | ||
65 | * mapping. | ||
66 | */ | ||
67 | void (*pagetable_setup_start)(pgd_t *pgd_base); | ||
68 | void (*pagetable_setup_done)(pgd_t *pgd_base); | ||
69 | |||
70 | /* Print a banner to identify the environment */ | 62 | /* Print a banner to identify the environment */ |
71 | void (*banner)(void); | 63 | void (*banner)(void); |
64 | }; | ||
65 | |||
66 | |||
67 | struct pv_misc_ops { | ||
68 | /* Set deferred update mode, used for batching operations. */ | ||
69 | void (*set_lazy_mode)(enum paravirt_lazy_mode mode); | ||
70 | }; | ||
71 | |||
72 | struct pv_time_ops { | ||
73 | void (*time_init)(void); | ||
72 | 74 | ||
73 | /* Set and set time of day */ | 75 | /* Set and set time of day */ |
74 | unsigned long (*get_wallclock)(void); | 76 | unsigned long (*get_wallclock)(void); |
75 | int (*set_wallclock)(unsigned long); | 77 | int (*set_wallclock)(unsigned long); |
76 | 78 | ||
77 | /* cpuid emulation, mostly so that caps bits can be disabled */ | 79 | unsigned long long (*sched_clock)(void); |
78 | void (*cpuid)(unsigned int *eax, unsigned int *ebx, | 80 | unsigned long (*get_cpu_khz)(void); |
79 | unsigned int *ecx, unsigned int *edx); | 81 | }; |
80 | 82 | ||
83 | struct pv_cpu_ops { | ||
81 | /* hooks for various privileged instructions */ | 84 | /* hooks for various privileged instructions */ |
82 | unsigned long (*get_debugreg)(int regno); | 85 | unsigned long (*get_debugreg)(int regno); |
83 | void (*set_debugreg)(int regno, unsigned long value); | 86 | void (*set_debugreg)(int regno, unsigned long value); |
@@ -87,41 +90,10 @@ struct paravirt_ops | |||
87 | unsigned long (*read_cr0)(void); | 90 | unsigned long (*read_cr0)(void); |
88 | void (*write_cr0)(unsigned long); | 91 | void (*write_cr0)(unsigned long); |
89 | 92 | ||
90 | unsigned long (*read_cr2)(void); | ||
91 | void (*write_cr2)(unsigned long); | ||
92 | |||
93 | unsigned long (*read_cr3)(void); | ||
94 | void (*write_cr3)(unsigned long); | ||
95 | |||
96 | unsigned long (*read_cr4_safe)(void); | 93 | unsigned long (*read_cr4_safe)(void); |
97 | unsigned long (*read_cr4)(void); | 94 | unsigned long (*read_cr4)(void); |
98 | void (*write_cr4)(unsigned long); | 95 | void (*write_cr4)(unsigned long); |
99 | 96 | ||
100 | /* | ||
101 | * Get/set interrupt state. save_fl and restore_fl are only | ||
102 | * expected to use X86_EFLAGS_IF; all other bits | ||
103 | * returned from save_fl are undefined, and may be ignored by | ||
104 | * restore_fl. | ||
105 | */ | ||
106 | unsigned long (*save_fl)(void); | ||
107 | void (*restore_fl)(unsigned long); | ||
108 | void (*irq_disable)(void); | ||
109 | void (*irq_enable)(void); | ||
110 | void (*safe_halt)(void); | ||
111 | void (*halt)(void); | ||
112 | |||
113 | void (*wbinvd)(void); | ||
114 | |||
115 | /* MSR, PMC and TSR operations. | ||
116 | err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */ | ||
117 | u64 (*read_msr)(unsigned int msr, int *err); | ||
118 | int (*write_msr)(unsigned int msr, u64 val); | ||
119 | |||
120 | u64 (*read_tsc)(void); | ||
121 | u64 (*read_pmc)(void); | ||
122 | unsigned long long (*sched_clock)(void); | ||
123 | unsigned long (*get_cpu_khz)(void); | ||
124 | |||
125 | /* Segment descriptor handling */ | 97 | /* Segment descriptor handling */ |
126 | void (*load_tr_desc)(void); | 98 | void (*load_tr_desc)(void); |
127 | void (*load_gdt)(const struct Xgt_desc_struct *); | 99 | void (*load_gdt)(const struct Xgt_desc_struct *); |
@@ -140,18 +112,45 @@ struct paravirt_ops | |||
140 | void (*load_esp0)(struct tss_struct *tss, struct thread_struct *t); | 112 | void (*load_esp0)(struct tss_struct *tss, struct thread_struct *t); |
141 | 113 | ||
142 | void (*set_iopl_mask)(unsigned mask); | 114 | void (*set_iopl_mask)(unsigned mask); |
115 | |||
116 | void (*wbinvd)(void); | ||
143 | void (*io_delay)(void); | 117 | void (*io_delay)(void); |
144 | 118 | ||
119 | /* cpuid emulation, mostly so that caps bits can be disabled */ | ||
120 | void (*cpuid)(unsigned int *eax, unsigned int *ebx, | ||
121 | unsigned int *ecx, unsigned int *edx); | ||
122 | |||
123 | /* MSR, PMC and TSR operations. | ||
124 | err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */ | ||
125 | u64 (*read_msr)(unsigned int msr, int *err); | ||
126 | int (*write_msr)(unsigned int msr, u64 val); | ||
127 | |||
128 | u64 (*read_tsc)(void); | ||
129 | u64 (*read_pmc)(void); | ||
130 | |||
131 | /* These two are jmp to, not actually called. */ | ||
132 | void (*irq_enable_sysexit)(void); | ||
133 | void (*iret)(void); | ||
134 | }; | ||
135 | |||
136 | struct pv_irq_ops { | ||
137 | void (*init_IRQ)(void); | ||
138 | |||
145 | /* | 139 | /* |
146 | * Hooks for intercepting the creation/use/destruction of an | 140 | * Get/set interrupt state. save_fl and restore_fl are only |
147 | * mm_struct. | 141 | * expected to use X86_EFLAGS_IF; all other bits |
142 | * returned from save_fl are undefined, and may be ignored by | ||
143 | * restore_fl. | ||
148 | */ | 144 | */ |
149 | void (*activate_mm)(struct mm_struct *prev, | 145 | unsigned long (*save_fl)(void); |
150 | struct mm_struct *next); | 146 | void (*restore_fl)(unsigned long); |
151 | void (*dup_mmap)(struct mm_struct *oldmm, | 147 | void (*irq_disable)(void); |
152 | struct mm_struct *mm); | 148 | void (*irq_enable)(void); |
153 | void (*exit_mmap)(struct mm_struct *mm); | 149 | void (*safe_halt)(void); |
150 | void (*halt)(void); | ||
151 | }; | ||
154 | 152 | ||
153 | struct pv_apic_ops { | ||
155 | #ifdef CONFIG_X86_LOCAL_APIC | 154 | #ifdef CONFIG_X86_LOCAL_APIC |
156 | /* | 155 | /* |
157 | * Direct APIC operations, principally for VMI. Ideally | 156 | * Direct APIC operations, principally for VMI. Ideally |
@@ -167,6 +166,34 @@ struct paravirt_ops | |||
167 | unsigned long start_eip, | 166 | unsigned long start_eip, |
168 | unsigned long start_esp); | 167 | unsigned long start_esp); |
169 | #endif | 168 | #endif |
169 | }; | ||
170 | |||
171 | struct pv_mmu_ops { | ||
172 | /* | ||
173 | * Called before/after init_mm pagetable setup. setup_start | ||
174 | * may reset %cr3, and may pre-install parts of the pagetable; | ||
175 | * pagetable setup is expected to preserve any existing | ||
176 | * mapping. | ||
177 | */ | ||
178 | void (*pagetable_setup_start)(pgd_t *pgd_base); | ||
179 | void (*pagetable_setup_done)(pgd_t *pgd_base); | ||
180 | |||
181 | unsigned long (*read_cr2)(void); | ||
182 | void (*write_cr2)(unsigned long); | ||
183 | |||
184 | unsigned long (*read_cr3)(void); | ||
185 | void (*write_cr3)(unsigned long); | ||
186 | |||
187 | /* | ||
188 | * Hooks for intercepting the creation/use/destruction of an | ||
189 | * mm_struct. | ||
190 | */ | ||
191 | void (*activate_mm)(struct mm_struct *prev, | ||
192 | struct mm_struct *next); | ||
193 | void (*dup_mmap)(struct mm_struct *oldmm, | ||
194 | struct mm_struct *mm); | ||
195 | void (*exit_mmap)(struct mm_struct *mm); | ||
196 | |||
170 | 197 | ||
171 | /* TLB operations */ | 198 | /* TLB operations */ |
172 | void (*flush_tlb_user)(void); | 199 | void (*flush_tlb_user)(void); |
@@ -191,15 +218,12 @@ struct paravirt_ops | |||
191 | void (*pte_update_defer)(struct mm_struct *mm, | 218 | void (*pte_update_defer)(struct mm_struct *mm, |
192 | unsigned long addr, pte_t *ptep); | 219 | unsigned long addr, pte_t *ptep); |
193 | 220 | ||
194 | #ifdef CONFIG_HIGHPTE | ||
195 | void *(*kmap_atomic_pte)(struct page *page, enum km_type type); | ||
196 | #endif | ||
197 | |||
198 | #ifdef CONFIG_X86_PAE | 221 | #ifdef CONFIG_X86_PAE |
199 | void (*set_pte_atomic)(pte_t *ptep, pte_t pteval); | 222 | void (*set_pte_atomic)(pte_t *ptep, pte_t pteval); |
200 | void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); | 223 | void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, |
224 | pte_t *ptep, pte_t pte); | ||
201 | void (*set_pud)(pud_t *pudp, pud_t pudval); | 225 | void (*set_pud)(pud_t *pudp, pud_t pudval); |
202 | void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); | 226 | void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); |
203 | void (*pmd_clear)(pmd_t *pmdp); | 227 | void (*pmd_clear)(pmd_t *pmdp); |
204 | 228 | ||
205 | unsigned long long (*pte_val)(pte_t); | 229 | unsigned long long (*pte_val)(pte_t); |
@@ -217,21 +241,40 @@ struct paravirt_ops | |||
217 | pgd_t (*make_pgd)(unsigned long pgd); | 241 | pgd_t (*make_pgd)(unsigned long pgd); |
218 | #endif | 242 | #endif |
219 | 243 | ||
220 | /* Set deferred update mode, used for batching operations. */ | 244 | #ifdef CONFIG_HIGHPTE |
221 | void (*set_lazy_mode)(enum paravirt_lazy_mode mode); | 245 | void *(*kmap_atomic_pte)(struct page *page, enum km_type type); |
246 | #endif | ||
247 | }; | ||
222 | 248 | ||
223 | /* These two are jmp to, not actually called. */ | 249 | /* This contains all the paravirt structures: we get a convenient |
224 | void (*irq_enable_sysexit)(void); | 250 | * number for each function using the offset which we use to indicate |
225 | void (*iret)(void); | 251 | * what to patch. */ |
252 | struct paravirt_patch_template | ||
253 | { | ||
254 | struct pv_init_ops pv_init_ops; | ||
255 | struct pv_misc_ops pv_misc_ops; | ||
256 | struct pv_time_ops pv_time_ops; | ||
257 | struct pv_cpu_ops pv_cpu_ops; | ||
258 | struct pv_irq_ops pv_irq_ops; | ||
259 | struct pv_apic_ops pv_apic_ops; | ||
260 | struct pv_mmu_ops pv_mmu_ops; | ||
226 | }; | 261 | }; |
227 | 262 | ||
228 | extern struct paravirt_ops paravirt_ops; | 263 | extern struct pv_info pv_info; |
264 | extern struct pv_init_ops pv_init_ops; | ||
265 | extern struct pv_misc_ops pv_misc_ops; | ||
266 | extern struct pv_time_ops pv_time_ops; | ||
267 | extern struct pv_cpu_ops pv_cpu_ops; | ||
268 | extern struct pv_irq_ops pv_irq_ops; | ||
269 | extern struct pv_apic_ops pv_apic_ops; | ||
270 | extern struct pv_mmu_ops pv_mmu_ops; | ||
229 | 271 | ||
230 | #define PARAVIRT_PATCH(x) \ | 272 | #define PARAVIRT_PATCH(x) \ |
231 | (offsetof(struct paravirt_ops, x) / sizeof(void *)) | 273 | (offsetof(struct paravirt_patch_template, x) / sizeof(void *)) |
232 | 274 | ||
233 | #define paravirt_type(type) \ | 275 | #define paravirt_type(op) \ |
234 | [paravirt_typenum] "i" (PARAVIRT_PATCH(type)) | 276 | [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \ |
277 | [paravirt_opptr] "m" (op) | ||
235 | #define paravirt_clobber(clobber) \ | 278 | #define paravirt_clobber(clobber) \ |
236 | [paravirt_clobber] "i" (clobber) | 279 | [paravirt_clobber] "i" (clobber) |
237 | 280 | ||
@@ -258,7 +301,7 @@ unsigned paravirt_patch_call(void *insnbuf, | |||
258 | const void *target, u16 tgt_clobbers, | 301 | const void *target, u16 tgt_clobbers, |
259 | unsigned long addr, u16 site_clobbers, | 302 | unsigned long addr, u16 site_clobbers, |
260 | unsigned len); | 303 | unsigned len); |
261 | unsigned paravirt_patch_jmp(const void *target, void *insnbuf, | 304 | unsigned paravirt_patch_jmp(void *insnbuf, const void *target, |
262 | unsigned long addr, unsigned len); | 305 | unsigned long addr, unsigned len); |
263 | unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf, | 306 | unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf, |
264 | unsigned long addr, unsigned len); | 307 | unsigned long addr, unsigned len); |
@@ -271,14 +314,14 @@ int paravirt_disable_iospace(void); | |||
271 | /* | 314 | /* |
272 | * This generates an indirect call based on the operation type number. | 315 | * This generates an indirect call based on the operation type number. |
273 | * The type number, computed in PARAVIRT_PATCH, is derived from the | 316 | * The type number, computed in PARAVIRT_PATCH, is derived from the |
274 | * offset into the paravirt_ops structure, and can therefore be freely | 317 | * offset into the paravirt_patch_template structure, and can therefore be |
275 | * converted back into a structure offset. | 318 | * freely converted back into a structure offset. |
276 | */ | 319 | */ |
277 | #define PARAVIRT_CALL "call *(paravirt_ops+%c[paravirt_typenum]*4);" | 320 | #define PARAVIRT_CALL "call *%[paravirt_opptr];" |
278 | 321 | ||
279 | /* | 322 | /* |
280 | * These macros are intended to wrap calls into a paravirt_ops | 323 | * These macros are intended to wrap calls through one of the paravirt |
281 | * operation, so that they can be later identified and patched at | 324 | * ops structs, so that they can be later identified and patched at |
282 | * runtime. | 325 | * runtime. |
283 | * | 326 | * |
284 | * Normally, a call to a pv_op function is a simple indirect call: | 327 | * Normally, a call to a pv_op function is a simple indirect call: |
@@ -301,7 +344,7 @@ int paravirt_disable_iospace(void); | |||
301 | * The call instruction itself is marked by placing its start address | 344 | * The call instruction itself is marked by placing its start address |
302 | * and size into the .parainstructions section, so that | 345 | * and size into the .parainstructions section, so that |
303 | * apply_paravirt() in arch/i386/kernel/alternative.c can do the | 346 | * apply_paravirt() in arch/i386/kernel/alternative.c can do the |
304 | * appropriate patching under the control of the backend paravirt_ops | 347 | * appropriate patching under the control of the backend pv_init_ops |
305 | * implementation. | 348 | * implementation. |
306 | * | 349 | * |
307 | * Unfortunately there's no way to get gcc to generate the args setup | 350 | * Unfortunately there's no way to get gcc to generate the args setup |
@@ -409,36 +452,36 @@ int paravirt_disable_iospace(void); | |||
409 | 452 | ||
410 | static inline int paravirt_enabled(void) | 453 | static inline int paravirt_enabled(void) |
411 | { | 454 | { |
412 | return paravirt_ops.paravirt_enabled; | 455 | return pv_info.paravirt_enabled; |
413 | } | 456 | } |
414 | 457 | ||
415 | static inline void load_esp0(struct tss_struct *tss, | 458 | static inline void load_esp0(struct tss_struct *tss, |
416 | struct thread_struct *thread) | 459 | struct thread_struct *thread) |
417 | { | 460 | { |
418 | PVOP_VCALL2(load_esp0, tss, thread); | 461 | PVOP_VCALL2(pv_cpu_ops.load_esp0, tss, thread); |
419 | } | 462 | } |
420 | 463 | ||
421 | #define ARCH_SETUP paravirt_ops.arch_setup(); | 464 | #define ARCH_SETUP pv_init_ops.arch_setup(); |
422 | static inline unsigned long get_wallclock(void) | 465 | static inline unsigned long get_wallclock(void) |
423 | { | 466 | { |
424 | return PVOP_CALL0(unsigned long, get_wallclock); | 467 | return PVOP_CALL0(unsigned long, pv_time_ops.get_wallclock); |
425 | } | 468 | } |
426 | 469 | ||
427 | static inline int set_wallclock(unsigned long nowtime) | 470 | static inline int set_wallclock(unsigned long nowtime) |
428 | { | 471 | { |
429 | return PVOP_CALL1(int, set_wallclock, nowtime); | 472 | return PVOP_CALL1(int, pv_time_ops.set_wallclock, nowtime); |
430 | } | 473 | } |
431 | 474 | ||
432 | static inline void (*choose_time_init(void))(void) | 475 | static inline void (*choose_time_init(void))(void) |
433 | { | 476 | { |
434 | return paravirt_ops.time_init; | 477 | return pv_time_ops.time_init; |
435 | } | 478 | } |
436 | 479 | ||
437 | /* The paravirtualized CPUID instruction. */ | 480 | /* The paravirtualized CPUID instruction. */ |
438 | static inline void __cpuid(unsigned int *eax, unsigned int *ebx, | 481 | static inline void __cpuid(unsigned int *eax, unsigned int *ebx, |
439 | unsigned int *ecx, unsigned int *edx) | 482 | unsigned int *ecx, unsigned int *edx) |
440 | { | 483 | { |
441 | PVOP_VCALL4(cpuid, eax, ebx, ecx, edx); | 484 | PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx); |
442 | } | 485 | } |
443 | 486 | ||
444 | /* | 487 | /* |
@@ -446,87 +489,87 @@ static inline void __cpuid(unsigned int *eax, unsigned int *ebx, | |||
446 | */ | 489 | */ |
447 | static inline unsigned long paravirt_get_debugreg(int reg) | 490 | static inline unsigned long paravirt_get_debugreg(int reg) |
448 | { | 491 | { |
449 | return PVOP_CALL1(unsigned long, get_debugreg, reg); | 492 | return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg); |
450 | } | 493 | } |
451 | #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) | 494 | #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) |
452 | static inline void set_debugreg(unsigned long val, int reg) | 495 | static inline void set_debugreg(unsigned long val, int reg) |
453 | { | 496 | { |
454 | PVOP_VCALL2(set_debugreg, reg, val); | 497 | PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val); |
455 | } | 498 | } |
456 | 499 | ||
457 | static inline void clts(void) | 500 | static inline void clts(void) |
458 | { | 501 | { |
459 | PVOP_VCALL0(clts); | 502 | PVOP_VCALL0(pv_cpu_ops.clts); |
460 | } | 503 | } |
461 | 504 | ||
462 | static inline unsigned long read_cr0(void) | 505 | static inline unsigned long read_cr0(void) |
463 | { | 506 | { |
464 | return PVOP_CALL0(unsigned long, read_cr0); | 507 | return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0); |
465 | } | 508 | } |
466 | 509 | ||
467 | static inline void write_cr0(unsigned long x) | 510 | static inline void write_cr0(unsigned long x) |
468 | { | 511 | { |
469 | PVOP_VCALL1(write_cr0, x); | 512 | PVOP_VCALL1(pv_cpu_ops.write_cr0, x); |
470 | } | 513 | } |
471 | 514 | ||
472 | static inline unsigned long read_cr2(void) | 515 | static inline unsigned long read_cr2(void) |
473 | { | 516 | { |
474 | return PVOP_CALL0(unsigned long, read_cr2); | 517 | return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2); |
475 | } | 518 | } |
476 | 519 | ||
477 | static inline void write_cr2(unsigned long x) | 520 | static inline void write_cr2(unsigned long x) |
478 | { | 521 | { |
479 | PVOP_VCALL1(write_cr2, x); | 522 | PVOP_VCALL1(pv_mmu_ops.write_cr2, x); |
480 | } | 523 | } |
481 | 524 | ||
482 | static inline unsigned long read_cr3(void) | 525 | static inline unsigned long read_cr3(void) |
483 | { | 526 | { |
484 | return PVOP_CALL0(unsigned long, read_cr3); | 527 | return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3); |
485 | } | 528 | } |
486 | 529 | ||
487 | static inline void write_cr3(unsigned long x) | 530 | static inline void write_cr3(unsigned long x) |
488 | { | 531 | { |
489 | PVOP_VCALL1(write_cr3, x); | 532 | PVOP_VCALL1(pv_mmu_ops.write_cr3, x); |
490 | } | 533 | } |
491 | 534 | ||
492 | static inline unsigned long read_cr4(void) | 535 | static inline unsigned long read_cr4(void) |
493 | { | 536 | { |
494 | return PVOP_CALL0(unsigned long, read_cr4); | 537 | return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4); |
495 | } | 538 | } |
496 | static inline unsigned long read_cr4_safe(void) | 539 | static inline unsigned long read_cr4_safe(void) |
497 | { | 540 | { |
498 | return PVOP_CALL0(unsigned long, read_cr4_safe); | 541 | return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe); |
499 | } | 542 | } |
500 | 543 | ||
501 | static inline void write_cr4(unsigned long x) | 544 | static inline void write_cr4(unsigned long x) |
502 | { | 545 | { |
503 | PVOP_VCALL1(write_cr4, x); | 546 | PVOP_VCALL1(pv_cpu_ops.write_cr4, x); |
504 | } | 547 | } |
505 | 548 | ||
506 | static inline void raw_safe_halt(void) | 549 | static inline void raw_safe_halt(void) |
507 | { | 550 | { |
508 | PVOP_VCALL0(safe_halt); | 551 | PVOP_VCALL0(pv_irq_ops.safe_halt); |
509 | } | 552 | } |
510 | 553 | ||
511 | static inline void halt(void) | 554 | static inline void halt(void) |
512 | { | 555 | { |
513 | PVOP_VCALL0(safe_halt); | 556 | PVOP_VCALL0(pv_irq_ops.safe_halt); |
514 | } | 557 | } |
515 | 558 | ||
516 | static inline void wbinvd(void) | 559 | static inline void wbinvd(void) |
517 | { | 560 | { |
518 | PVOP_VCALL0(wbinvd); | 561 | PVOP_VCALL0(pv_cpu_ops.wbinvd); |
519 | } | 562 | } |
520 | 563 | ||
521 | #define get_kernel_rpl() (paravirt_ops.kernel_rpl) | 564 | #define get_kernel_rpl() (pv_info.kernel_rpl) |
522 | 565 | ||
523 | static inline u64 paravirt_read_msr(unsigned msr, int *err) | 566 | static inline u64 paravirt_read_msr(unsigned msr, int *err) |
524 | { | 567 | { |
525 | return PVOP_CALL2(u64, read_msr, msr, err); | 568 | return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err); |
526 | } | 569 | } |
527 | static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) | 570 | static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) |
528 | { | 571 | { |
529 | return PVOP_CALL3(int, write_msr, msr, low, high); | 572 | return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high); |
530 | } | 573 | } |
531 | 574 | ||
532 | /* These should all do BUG_ON(_err), but our headers are too tangled. */ | 575 | /* These should all do BUG_ON(_err), but our headers are too tangled. */ |
@@ -560,7 +603,7 @@ static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) | |||
560 | 603 | ||
561 | static inline u64 paravirt_read_tsc(void) | 604 | static inline u64 paravirt_read_tsc(void) |
562 | { | 605 | { |
563 | return PVOP_CALL0(u64, read_tsc); | 606 | return PVOP_CALL0(u64, pv_cpu_ops.read_tsc); |
564 | } | 607 | } |
565 | 608 | ||
566 | #define rdtscl(low) do { \ | 609 | #define rdtscl(low) do { \ |
@@ -572,15 +615,15 @@ static inline u64 paravirt_read_tsc(void) | |||
572 | 615 | ||
573 | static inline unsigned long long paravirt_sched_clock(void) | 616 | static inline unsigned long long paravirt_sched_clock(void) |
574 | { | 617 | { |
575 | return PVOP_CALL0(unsigned long long, sched_clock); | 618 | return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock); |
576 | } | 619 | } |
577 | #define calculate_cpu_khz() (paravirt_ops.get_cpu_khz()) | 620 | #define calculate_cpu_khz() (pv_time_ops.get_cpu_khz()) |
578 | 621 | ||
579 | #define write_tsc(val1,val2) wrmsr(0x10, val1, val2) | 622 | #define write_tsc(val1,val2) wrmsr(0x10, val1, val2) |
580 | 623 | ||
581 | static inline unsigned long long paravirt_read_pmc(int counter) | 624 | static inline unsigned long long paravirt_read_pmc(int counter) |
582 | { | 625 | { |
583 | return PVOP_CALL1(u64, read_pmc, counter); | 626 | return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter); |
584 | } | 627 | } |
585 | 628 | ||
586 | #define rdpmc(counter,low,high) do { \ | 629 | #define rdpmc(counter,low,high) do { \ |
@@ -591,61 +634,61 @@ static inline unsigned long long paravirt_read_pmc(int counter) | |||
591 | 634 | ||
592 | static inline void load_TR_desc(void) | 635 | static inline void load_TR_desc(void) |
593 | { | 636 | { |
594 | PVOP_VCALL0(load_tr_desc); | 637 | PVOP_VCALL0(pv_cpu_ops.load_tr_desc); |
595 | } | 638 | } |
596 | static inline void load_gdt(const struct Xgt_desc_struct *dtr) | 639 | static inline void load_gdt(const struct Xgt_desc_struct *dtr) |
597 | { | 640 | { |
598 | PVOP_VCALL1(load_gdt, dtr); | 641 | PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr); |
599 | } | 642 | } |
600 | static inline void load_idt(const struct Xgt_desc_struct *dtr) | 643 | static inline void load_idt(const struct Xgt_desc_struct *dtr) |
601 | { | 644 | { |
602 | PVOP_VCALL1(load_idt, dtr); | 645 | PVOP_VCALL1(pv_cpu_ops.load_idt, dtr); |
603 | } | 646 | } |
604 | static inline void set_ldt(const void *addr, unsigned entries) | 647 | static inline void set_ldt(const void *addr, unsigned entries) |
605 | { | 648 | { |
606 | PVOP_VCALL2(set_ldt, addr, entries); | 649 | PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries); |
607 | } | 650 | } |
608 | static inline void store_gdt(struct Xgt_desc_struct *dtr) | 651 | static inline void store_gdt(struct Xgt_desc_struct *dtr) |
609 | { | 652 | { |
610 | PVOP_VCALL1(store_gdt, dtr); | 653 | PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr); |
611 | } | 654 | } |
612 | static inline void store_idt(struct Xgt_desc_struct *dtr) | 655 | static inline void store_idt(struct Xgt_desc_struct *dtr) |
613 | { | 656 | { |
614 | PVOP_VCALL1(store_idt, dtr); | 657 | PVOP_VCALL1(pv_cpu_ops.store_idt, dtr); |
615 | } | 658 | } |
616 | static inline unsigned long paravirt_store_tr(void) | 659 | static inline unsigned long paravirt_store_tr(void) |
617 | { | 660 | { |
618 | return PVOP_CALL0(unsigned long, store_tr); | 661 | return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr); |
619 | } | 662 | } |
620 | #define store_tr(tr) ((tr) = paravirt_store_tr()) | 663 | #define store_tr(tr) ((tr) = paravirt_store_tr()) |
621 | static inline void load_TLS(struct thread_struct *t, unsigned cpu) | 664 | static inline void load_TLS(struct thread_struct *t, unsigned cpu) |
622 | { | 665 | { |
623 | PVOP_VCALL2(load_tls, t, cpu); | 666 | PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu); |
624 | } | 667 | } |
625 | static inline void write_ldt_entry(void *dt, int entry, u32 low, u32 high) | 668 | static inline void write_ldt_entry(void *dt, int entry, u32 low, u32 high) |
626 | { | 669 | { |
627 | PVOP_VCALL4(write_ldt_entry, dt, entry, low, high); | 670 | PVOP_VCALL4(pv_cpu_ops.write_ldt_entry, dt, entry, low, high); |
628 | } | 671 | } |
629 | static inline void write_gdt_entry(void *dt, int entry, u32 low, u32 high) | 672 | static inline void write_gdt_entry(void *dt, int entry, u32 low, u32 high) |
630 | { | 673 | { |
631 | PVOP_VCALL4(write_gdt_entry, dt, entry, low, high); | 674 | PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, low, high); |
632 | } | 675 | } |
633 | static inline void write_idt_entry(void *dt, int entry, u32 low, u32 high) | 676 | static inline void write_idt_entry(void *dt, int entry, u32 low, u32 high) |
634 | { | 677 | { |
635 | PVOP_VCALL4(write_idt_entry, dt, entry, low, high); | 678 | PVOP_VCALL4(pv_cpu_ops.write_idt_entry, dt, entry, low, high); |
636 | } | 679 | } |
637 | static inline void set_iopl_mask(unsigned mask) | 680 | static inline void set_iopl_mask(unsigned mask) |
638 | { | 681 | { |
639 | PVOP_VCALL1(set_iopl_mask, mask); | 682 | PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask); |
640 | } | 683 | } |
641 | 684 | ||
642 | /* The paravirtualized I/O functions */ | 685 | /* The paravirtualized I/O functions */ |
643 | static inline void slow_down_io(void) { | 686 | static inline void slow_down_io(void) { |
644 | paravirt_ops.io_delay(); | 687 | pv_cpu_ops.io_delay(); |
645 | #ifdef REALLY_SLOW_IO | 688 | #ifdef REALLY_SLOW_IO |
646 | paravirt_ops.io_delay(); | 689 | pv_cpu_ops.io_delay(); |
647 | paravirt_ops.io_delay(); | 690 | pv_cpu_ops.io_delay(); |
648 | paravirt_ops.io_delay(); | 691 | pv_cpu_ops.io_delay(); |
649 | #endif | 692 | #endif |
650 | } | 693 | } |
651 | 694 | ||
@@ -655,121 +698,120 @@ static inline void slow_down_io(void) { | |||
655 | */ | 698 | */ |
656 | static inline void apic_write(unsigned long reg, unsigned long v) | 699 | static inline void apic_write(unsigned long reg, unsigned long v) |
657 | { | 700 | { |
658 | PVOP_VCALL2(apic_write, reg, v); | 701 | PVOP_VCALL2(pv_apic_ops.apic_write, reg, v); |
659 | } | 702 | } |
660 | 703 | ||
661 | static inline void apic_write_atomic(unsigned long reg, unsigned long v) | 704 | static inline void apic_write_atomic(unsigned long reg, unsigned long v) |
662 | { | 705 | { |
663 | PVOP_VCALL2(apic_write_atomic, reg, v); | 706 | PVOP_VCALL2(pv_apic_ops.apic_write_atomic, reg, v); |
664 | } | 707 | } |
665 | 708 | ||
666 | static inline unsigned long apic_read(unsigned long reg) | 709 | static inline unsigned long apic_read(unsigned long reg) |
667 | { | 710 | { |
668 | return PVOP_CALL1(unsigned long, apic_read, reg); | 711 | return PVOP_CALL1(unsigned long, pv_apic_ops.apic_read, reg); |
669 | } | 712 | } |
670 | 713 | ||
671 | static inline void setup_boot_clock(void) | 714 | static inline void setup_boot_clock(void) |
672 | { | 715 | { |
673 | PVOP_VCALL0(setup_boot_clock); | 716 | PVOP_VCALL0(pv_apic_ops.setup_boot_clock); |
674 | } | 717 | } |
675 | 718 | ||
676 | static inline void setup_secondary_clock(void) | 719 | static inline void setup_secondary_clock(void) |
677 | { | 720 | { |
678 | PVOP_VCALL0(setup_secondary_clock); | 721 | PVOP_VCALL0(pv_apic_ops.setup_secondary_clock); |
679 | } | 722 | } |
680 | #endif | 723 | #endif |
681 | 724 | ||
682 | static inline void paravirt_post_allocator_init(void) | 725 | static inline void paravirt_post_allocator_init(void) |
683 | { | 726 | { |
684 | if (paravirt_ops.post_allocator_init) | 727 | if (pv_init_ops.post_allocator_init) |
685 | (*paravirt_ops.post_allocator_init)(); | 728 | (*pv_init_ops.post_allocator_init)(); |
686 | } | 729 | } |
687 | 730 | ||
688 | static inline void paravirt_pagetable_setup_start(pgd_t *base) | 731 | static inline void paravirt_pagetable_setup_start(pgd_t *base) |
689 | { | 732 | { |
690 | if (paravirt_ops.pagetable_setup_start) | 733 | (*pv_mmu_ops.pagetable_setup_start)(base); |
691 | (*paravirt_ops.pagetable_setup_start)(base); | ||
692 | } | 734 | } |
693 | 735 | ||
694 | static inline void paravirt_pagetable_setup_done(pgd_t *base) | 736 | static inline void paravirt_pagetable_setup_done(pgd_t *base) |
695 | { | 737 | { |
696 | if (paravirt_ops.pagetable_setup_done) | 738 | (*pv_mmu_ops.pagetable_setup_done)(base); |
697 | (*paravirt_ops.pagetable_setup_done)(base); | ||
698 | } | 739 | } |
699 | 740 | ||
700 | #ifdef CONFIG_SMP | 741 | #ifdef CONFIG_SMP |
701 | static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip, | 742 | static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip, |
702 | unsigned long start_esp) | 743 | unsigned long start_esp) |
703 | { | 744 | { |
704 | PVOP_VCALL3(startup_ipi_hook, phys_apicid, start_eip, start_esp); | 745 | PVOP_VCALL3(pv_apic_ops.startup_ipi_hook, |
746 | phys_apicid, start_eip, start_esp); | ||
705 | } | 747 | } |
706 | #endif | 748 | #endif |
707 | 749 | ||
708 | static inline void paravirt_activate_mm(struct mm_struct *prev, | 750 | static inline void paravirt_activate_mm(struct mm_struct *prev, |
709 | struct mm_struct *next) | 751 | struct mm_struct *next) |
710 | { | 752 | { |
711 | PVOP_VCALL2(activate_mm, prev, next); | 753 | PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next); |
712 | } | 754 | } |
713 | 755 | ||
714 | static inline void arch_dup_mmap(struct mm_struct *oldmm, | 756 | static inline void arch_dup_mmap(struct mm_struct *oldmm, |
715 | struct mm_struct *mm) | 757 | struct mm_struct *mm) |
716 | { | 758 | { |
717 | PVOP_VCALL2(dup_mmap, oldmm, mm); | 759 | PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm); |
718 | } | 760 | } |
719 | 761 | ||
720 | static inline void arch_exit_mmap(struct mm_struct *mm) | 762 | static inline void arch_exit_mmap(struct mm_struct *mm) |
721 | { | 763 | { |
722 | PVOP_VCALL1(exit_mmap, mm); | 764 | PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm); |
723 | } | 765 | } |
724 | 766 | ||
725 | static inline void __flush_tlb(void) | 767 | static inline void __flush_tlb(void) |
726 | { | 768 | { |
727 | PVOP_VCALL0(flush_tlb_user); | 769 | PVOP_VCALL0(pv_mmu_ops.flush_tlb_user); |
728 | } | 770 | } |
729 | static inline void __flush_tlb_global(void) | 771 | static inline void __flush_tlb_global(void) |
730 | { | 772 | { |
731 | PVOP_VCALL0(flush_tlb_kernel); | 773 | PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel); |
732 | } | 774 | } |
733 | static inline void __flush_tlb_single(unsigned long addr) | 775 | static inline void __flush_tlb_single(unsigned long addr) |
734 | { | 776 | { |
735 | PVOP_VCALL1(flush_tlb_single, addr); | 777 | PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr); |
736 | } | 778 | } |
737 | 779 | ||
738 | static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, | 780 | static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, |
739 | unsigned long va) | 781 | unsigned long va) |
740 | { | 782 | { |
741 | PVOP_VCALL3(flush_tlb_others, &cpumask, mm, va); | 783 | PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, &cpumask, mm, va); |
742 | } | 784 | } |
743 | 785 | ||
744 | static inline void paravirt_alloc_pt(struct mm_struct *mm, unsigned pfn) | 786 | static inline void paravirt_alloc_pt(struct mm_struct *mm, unsigned pfn) |
745 | { | 787 | { |
746 | PVOP_VCALL2(alloc_pt, mm, pfn); | 788 | PVOP_VCALL2(pv_mmu_ops.alloc_pt, mm, pfn); |
747 | } | 789 | } |
748 | static inline void paravirt_release_pt(unsigned pfn) | 790 | static inline void paravirt_release_pt(unsigned pfn) |
749 | { | 791 | { |
750 | PVOP_VCALL1(release_pt, pfn); | 792 | PVOP_VCALL1(pv_mmu_ops.release_pt, pfn); |
751 | } | 793 | } |
752 | 794 | ||
753 | static inline void paravirt_alloc_pd(unsigned pfn) | 795 | static inline void paravirt_alloc_pd(unsigned pfn) |
754 | { | 796 | { |
755 | PVOP_VCALL1(alloc_pd, pfn); | 797 | PVOP_VCALL1(pv_mmu_ops.alloc_pd, pfn); |
756 | } | 798 | } |
757 | 799 | ||
758 | static inline void paravirt_alloc_pd_clone(unsigned pfn, unsigned clonepfn, | 800 | static inline void paravirt_alloc_pd_clone(unsigned pfn, unsigned clonepfn, |
759 | unsigned start, unsigned count) | 801 | unsigned start, unsigned count) |
760 | { | 802 | { |
761 | PVOP_VCALL4(alloc_pd_clone, pfn, clonepfn, start, count); | 803 | PVOP_VCALL4(pv_mmu_ops.alloc_pd_clone, pfn, clonepfn, start, count); |
762 | } | 804 | } |
763 | static inline void paravirt_release_pd(unsigned pfn) | 805 | static inline void paravirt_release_pd(unsigned pfn) |
764 | { | 806 | { |
765 | PVOP_VCALL1(release_pd, pfn); | 807 | PVOP_VCALL1(pv_mmu_ops.release_pd, pfn); |
766 | } | 808 | } |
767 | 809 | ||
768 | #ifdef CONFIG_HIGHPTE | 810 | #ifdef CONFIG_HIGHPTE |
769 | static inline void *kmap_atomic_pte(struct page *page, enum km_type type) | 811 | static inline void *kmap_atomic_pte(struct page *page, enum km_type type) |
770 | { | 812 | { |
771 | unsigned long ret; | 813 | unsigned long ret; |
772 | ret = PVOP_CALL2(unsigned long, kmap_atomic_pte, page, type); | 814 | ret = PVOP_CALL2(unsigned long, pv_mmu_ops.kmap_atomic_pte, page, type); |
773 | return (void *)ret; | 815 | return (void *)ret; |
774 | } | 816 | } |
775 | #endif | 817 | #endif |
@@ -777,162 +819,171 @@ static inline void *kmap_atomic_pte(struct page *page, enum km_type type) | |||
777 | static inline void pte_update(struct mm_struct *mm, unsigned long addr, | 819 | static inline void pte_update(struct mm_struct *mm, unsigned long addr, |
778 | pte_t *ptep) | 820 | pte_t *ptep) |
779 | { | 821 | { |
780 | PVOP_VCALL3(pte_update, mm, addr, ptep); | 822 | PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep); |
781 | } | 823 | } |
782 | 824 | ||
783 | static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr, | 825 | static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr, |
784 | pte_t *ptep) | 826 | pte_t *ptep) |
785 | { | 827 | { |
786 | PVOP_VCALL3(pte_update_defer, mm, addr, ptep); | 828 | PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep); |
787 | } | 829 | } |
788 | 830 | ||
789 | #ifdef CONFIG_X86_PAE | 831 | #ifdef CONFIG_X86_PAE |
790 | static inline pte_t __pte(unsigned long long val) | 832 | static inline pte_t __pte(unsigned long long val) |
791 | { | 833 | { |
792 | unsigned long long ret = PVOP_CALL2(unsigned long long, make_pte, | 834 | unsigned long long ret = PVOP_CALL2(unsigned long long, |
835 | pv_mmu_ops.make_pte, | ||
793 | val, val >> 32); | 836 | val, val >> 32); |
794 | return (pte_t) { ret, ret >> 32 }; | 837 | return (pte_t) { ret, ret >> 32 }; |
795 | } | 838 | } |
796 | 839 | ||
797 | static inline pmd_t __pmd(unsigned long long val) | 840 | static inline pmd_t __pmd(unsigned long long val) |
798 | { | 841 | { |
799 | return (pmd_t) { PVOP_CALL2(unsigned long long, make_pmd, val, val >> 32) }; | 842 | return (pmd_t) { PVOP_CALL2(unsigned long long, pv_mmu_ops.make_pmd, |
843 | val, val >> 32) }; | ||
800 | } | 844 | } |
801 | 845 | ||
802 | static inline pgd_t __pgd(unsigned long long val) | 846 | static inline pgd_t __pgd(unsigned long long val) |
803 | { | 847 | { |
804 | return (pgd_t) { PVOP_CALL2(unsigned long long, make_pgd, val, val >> 32) }; | 848 | return (pgd_t) { PVOP_CALL2(unsigned long long, pv_mmu_ops.make_pgd, |
849 | val, val >> 32) }; | ||
805 | } | 850 | } |
806 | 851 | ||
807 | static inline unsigned long long pte_val(pte_t x) | 852 | static inline unsigned long long pte_val(pte_t x) |
808 | { | 853 | { |
809 | return PVOP_CALL2(unsigned long long, pte_val, x.pte_low, x.pte_high); | 854 | return PVOP_CALL2(unsigned long long, pv_mmu_ops.pte_val, |
855 | x.pte_low, x.pte_high); | ||
810 | } | 856 | } |
811 | 857 | ||
812 | static inline unsigned long long pmd_val(pmd_t x) | 858 | static inline unsigned long long pmd_val(pmd_t x) |
813 | { | 859 | { |
814 | return PVOP_CALL2(unsigned long long, pmd_val, x.pmd, x.pmd >> 32); | 860 | return PVOP_CALL2(unsigned long long, pv_mmu_ops.pmd_val, |
861 | x.pmd, x.pmd >> 32); | ||
815 | } | 862 | } |
816 | 863 | ||
817 | static inline unsigned long long pgd_val(pgd_t x) | 864 | static inline unsigned long long pgd_val(pgd_t x) |
818 | { | 865 | { |
819 | return PVOP_CALL2(unsigned long long, pgd_val, x.pgd, x.pgd >> 32); | 866 | return PVOP_CALL2(unsigned long long, pv_mmu_ops.pgd_val, |
867 | x.pgd, x.pgd >> 32); | ||
820 | } | 868 | } |
821 | 869 | ||
822 | static inline void set_pte(pte_t *ptep, pte_t pteval) | 870 | static inline void set_pte(pte_t *ptep, pte_t pteval) |
823 | { | 871 | { |
824 | PVOP_VCALL3(set_pte, ptep, pteval.pte_low, pteval.pte_high); | 872 | PVOP_VCALL3(pv_mmu_ops.set_pte, ptep, pteval.pte_low, pteval.pte_high); |
825 | } | 873 | } |
826 | 874 | ||
827 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, | 875 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, |
828 | pte_t *ptep, pte_t pteval) | 876 | pte_t *ptep, pte_t pteval) |
829 | { | 877 | { |
830 | /* 5 arg words */ | 878 | /* 5 arg words */ |
831 | paravirt_ops.set_pte_at(mm, addr, ptep, pteval); | 879 | pv_mmu_ops.set_pte_at(mm, addr, ptep, pteval); |
832 | } | 880 | } |
833 | 881 | ||
834 | static inline void set_pte_atomic(pte_t *ptep, pte_t pteval) | 882 | static inline void set_pte_atomic(pte_t *ptep, pte_t pteval) |
835 | { | 883 | { |
836 | PVOP_VCALL3(set_pte_atomic, ptep, pteval.pte_low, pteval.pte_high); | 884 | PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep, |
885 | pteval.pte_low, pteval.pte_high); | ||
837 | } | 886 | } |
838 | 887 | ||
839 | static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, | 888 | static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, |
840 | pte_t *ptep, pte_t pte) | 889 | pte_t *ptep, pte_t pte) |
841 | { | 890 | { |
842 | /* 5 arg words */ | 891 | /* 5 arg words */ |
843 | paravirt_ops.set_pte_present(mm, addr, ptep, pte); | 892 | pv_mmu_ops.set_pte_present(mm, addr, ptep, pte); |
844 | } | 893 | } |
845 | 894 | ||
846 | static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) | 895 | static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) |
847 | { | 896 | { |
848 | PVOP_VCALL3(set_pmd, pmdp, pmdval.pmd, pmdval.pmd >> 32); | 897 | PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, |
898 | pmdval.pmd, pmdval.pmd >> 32); | ||
849 | } | 899 | } |
850 | 900 | ||
851 | static inline void set_pud(pud_t *pudp, pud_t pudval) | 901 | static inline void set_pud(pud_t *pudp, pud_t pudval) |
852 | { | 902 | { |
853 | PVOP_VCALL3(set_pud, pudp, pudval.pgd.pgd, pudval.pgd.pgd >> 32); | 903 | PVOP_VCALL3(pv_mmu_ops.set_pud, pudp, |
904 | pudval.pgd.pgd, pudval.pgd.pgd >> 32); | ||
854 | } | 905 | } |
855 | 906 | ||
856 | static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | 907 | static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) |
857 | { | 908 | { |
858 | PVOP_VCALL3(pte_clear, mm, addr, ptep); | 909 | PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep); |
859 | } | 910 | } |
860 | 911 | ||
861 | static inline void pmd_clear(pmd_t *pmdp) | 912 | static inline void pmd_clear(pmd_t *pmdp) |
862 | { | 913 | { |
863 | PVOP_VCALL1(pmd_clear, pmdp); | 914 | PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp); |
864 | } | 915 | } |
865 | 916 | ||
866 | #else /* !CONFIG_X86_PAE */ | 917 | #else /* !CONFIG_X86_PAE */ |
867 | 918 | ||
868 | static inline pte_t __pte(unsigned long val) | 919 | static inline pte_t __pte(unsigned long val) |
869 | { | 920 | { |
870 | return (pte_t) { PVOP_CALL1(unsigned long, make_pte, val) }; | 921 | return (pte_t) { PVOP_CALL1(unsigned long, pv_mmu_ops.make_pte, val) }; |
871 | } | 922 | } |
872 | 923 | ||
873 | static inline pgd_t __pgd(unsigned long val) | 924 | static inline pgd_t __pgd(unsigned long val) |
874 | { | 925 | { |
875 | return (pgd_t) { PVOP_CALL1(unsigned long, make_pgd, val) }; | 926 | return (pgd_t) { PVOP_CALL1(unsigned long, pv_mmu_ops.make_pgd, val) }; |
876 | } | 927 | } |
877 | 928 | ||
878 | static inline unsigned long pte_val(pte_t x) | 929 | static inline unsigned long pte_val(pte_t x) |
879 | { | 930 | { |
880 | return PVOP_CALL1(unsigned long, pte_val, x.pte_low); | 931 | return PVOP_CALL1(unsigned long, pv_mmu_ops.pte_val, x.pte_low); |
881 | } | 932 | } |
882 | 933 | ||
883 | static inline unsigned long pgd_val(pgd_t x) | 934 | static inline unsigned long pgd_val(pgd_t x) |
884 | { | 935 | { |
885 | return PVOP_CALL1(unsigned long, pgd_val, x.pgd); | 936 | return PVOP_CALL1(unsigned long, pv_mmu_ops.pgd_val, x.pgd); |
886 | } | 937 | } |
887 | 938 | ||
888 | static inline void set_pte(pte_t *ptep, pte_t pteval) | 939 | static inline void set_pte(pte_t *ptep, pte_t pteval) |
889 | { | 940 | { |
890 | PVOP_VCALL2(set_pte, ptep, pteval.pte_low); | 941 | PVOP_VCALL2(pv_mmu_ops.set_pte, ptep, pteval.pte_low); |
891 | } | 942 | } |
892 | 943 | ||
893 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, | 944 | static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, |
894 | pte_t *ptep, pte_t pteval) | 945 | pte_t *ptep, pte_t pteval) |
895 | { | 946 | { |
896 | PVOP_VCALL4(set_pte_at, mm, addr, ptep, pteval.pte_low); | 947 | PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pteval.pte_low); |
897 | } | 948 | } |
898 | 949 | ||
899 | static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) | 950 | static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) |
900 | { | 951 | { |
901 | PVOP_VCALL2(set_pmd, pmdp, pmdval.pud.pgd.pgd); | 952 | PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, pmdval.pud.pgd.pgd); |
902 | } | 953 | } |
903 | #endif /* CONFIG_X86_PAE */ | 954 | #endif /* CONFIG_X86_PAE */ |
904 | 955 | ||
905 | #define __HAVE_ARCH_ENTER_LAZY_CPU_MODE | 956 | #define __HAVE_ARCH_ENTER_LAZY_CPU_MODE |
906 | static inline void arch_enter_lazy_cpu_mode(void) | 957 | static inline void arch_enter_lazy_cpu_mode(void) |
907 | { | 958 | { |
908 | PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_CPU); | 959 | PVOP_VCALL1(pv_misc_ops.set_lazy_mode, PARAVIRT_LAZY_CPU); |
909 | } | 960 | } |
910 | 961 | ||
911 | static inline void arch_leave_lazy_cpu_mode(void) | 962 | static inline void arch_leave_lazy_cpu_mode(void) |
912 | { | 963 | { |
913 | PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_NONE); | 964 | PVOP_VCALL1(pv_misc_ops.set_lazy_mode, PARAVIRT_LAZY_NONE); |
914 | } | 965 | } |
915 | 966 | ||
916 | static inline void arch_flush_lazy_cpu_mode(void) | 967 | static inline void arch_flush_lazy_cpu_mode(void) |
917 | { | 968 | { |
918 | PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_FLUSH); | 969 | PVOP_VCALL1(pv_misc_ops.set_lazy_mode, PARAVIRT_LAZY_FLUSH); |
919 | } | 970 | } |
920 | 971 | ||
921 | 972 | ||
922 | #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE | 973 | #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE |
923 | static inline void arch_enter_lazy_mmu_mode(void) | 974 | static inline void arch_enter_lazy_mmu_mode(void) |
924 | { | 975 | { |
925 | PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_MMU); | 976 | PVOP_VCALL1(pv_misc_ops.set_lazy_mode, PARAVIRT_LAZY_MMU); |
926 | } | 977 | } |
927 | 978 | ||
928 | static inline void arch_leave_lazy_mmu_mode(void) | 979 | static inline void arch_leave_lazy_mmu_mode(void) |
929 | { | 980 | { |
930 | PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_NONE); | 981 | PVOP_VCALL1(pv_misc_ops.set_lazy_mode, PARAVIRT_LAZY_NONE); |
931 | } | 982 | } |
932 | 983 | ||
933 | static inline void arch_flush_lazy_mmu_mode(void) | 984 | static inline void arch_flush_lazy_mmu_mode(void) |
934 | { | 985 | { |
935 | PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_FLUSH); | 986 | PVOP_VCALL1(pv_misc_ops.set_lazy_mode, PARAVIRT_LAZY_FLUSH); |
936 | } | 987 | } |
937 | 988 | ||
938 | void _paravirt_nop(void); | 989 | void _paravirt_nop(void); |
@@ -957,7 +1008,7 @@ static inline unsigned long __raw_local_save_flags(void) | |||
957 | PARAVIRT_CALL | 1008 | PARAVIRT_CALL |
958 | "popl %%edx; popl %%ecx") | 1009 | "popl %%edx; popl %%ecx") |
959 | : "=a"(f) | 1010 | : "=a"(f) |
960 | : paravirt_type(save_fl), | 1011 | : paravirt_type(pv_irq_ops.save_fl), |
961 | paravirt_clobber(CLBR_EAX) | 1012 | paravirt_clobber(CLBR_EAX) |
962 | : "memory", "cc"); | 1013 | : "memory", "cc"); |
963 | return f; | 1014 | return f; |
@@ -970,7 +1021,7 @@ static inline void raw_local_irq_restore(unsigned long f) | |||
970 | "popl %%edx; popl %%ecx") | 1021 | "popl %%edx; popl %%ecx") |
971 | : "=a"(f) | 1022 | : "=a"(f) |
972 | : "0"(f), | 1023 | : "0"(f), |
973 | paravirt_type(restore_fl), | 1024 | paravirt_type(pv_irq_ops.restore_fl), |
974 | paravirt_clobber(CLBR_EAX) | 1025 | paravirt_clobber(CLBR_EAX) |
975 | : "memory", "cc"); | 1026 | : "memory", "cc"); |
976 | } | 1027 | } |
@@ -981,7 +1032,7 @@ static inline void raw_local_irq_disable(void) | |||
981 | PARAVIRT_CALL | 1032 | PARAVIRT_CALL |
982 | "popl %%edx; popl %%ecx") | 1033 | "popl %%edx; popl %%ecx") |
983 | : | 1034 | : |
984 | : paravirt_type(irq_disable), | 1035 | : paravirt_type(pv_irq_ops.irq_disable), |
985 | paravirt_clobber(CLBR_EAX) | 1036 | paravirt_clobber(CLBR_EAX) |
986 | : "memory", "eax", "cc"); | 1037 | : "memory", "eax", "cc"); |
987 | } | 1038 | } |
@@ -992,7 +1043,7 @@ static inline void raw_local_irq_enable(void) | |||
992 | PARAVIRT_CALL | 1043 | PARAVIRT_CALL |
993 | "popl %%edx; popl %%ecx") | 1044 | "popl %%edx; popl %%ecx") |
994 | : | 1045 | : |
995 | : paravirt_type(irq_enable), | 1046 | : paravirt_type(pv_irq_ops.irq_enable), |
996 | paravirt_clobber(CLBR_EAX) | 1047 | paravirt_clobber(CLBR_EAX) |
997 | : "memory", "eax", "cc"); | 1048 | : "memory", "eax", "cc"); |
998 | } | 1049 | } |
@@ -1008,21 +1059,23 @@ static inline unsigned long __raw_local_irq_save(void) | |||
1008 | 1059 | ||
1009 | #define CLI_STRING \ | 1060 | #define CLI_STRING \ |
1010 | _paravirt_alt("pushl %%ecx; pushl %%edx;" \ | 1061 | _paravirt_alt("pushl %%ecx; pushl %%edx;" \ |
1011 | "call *paravirt_ops+%c[paravirt_cli_type]*4;" \ | 1062 | "call *%[paravirt_cli_opptr];" \ |
1012 | "popl %%edx; popl %%ecx", \ | 1063 | "popl %%edx; popl %%ecx", \ |
1013 | "%c[paravirt_cli_type]", "%c[paravirt_clobber]") | 1064 | "%c[paravirt_cli_type]", "%c[paravirt_clobber]") |
1014 | 1065 | ||
1015 | #define STI_STRING \ | 1066 | #define STI_STRING \ |
1016 | _paravirt_alt("pushl %%ecx; pushl %%edx;" \ | 1067 | _paravirt_alt("pushl %%ecx; pushl %%edx;" \ |
1017 | "call *paravirt_ops+%c[paravirt_sti_type]*4;" \ | 1068 | "call *%[paravirt_sti_opptr];" \ |
1018 | "popl %%edx; popl %%ecx", \ | 1069 | "popl %%edx; popl %%ecx", \ |
1019 | "%c[paravirt_sti_type]", "%c[paravirt_clobber]") | 1070 | "%c[paravirt_sti_type]", "%c[paravirt_clobber]") |
1020 | 1071 | ||
1021 | #define CLI_STI_CLOBBERS , "%eax" | 1072 | #define CLI_STI_CLOBBERS , "%eax" |
1022 | #define CLI_STI_INPUT_ARGS \ | 1073 | #define CLI_STI_INPUT_ARGS \ |
1023 | , \ | 1074 | , \ |
1024 | [paravirt_cli_type] "i" (PARAVIRT_PATCH(irq_disable)), \ | 1075 | [paravirt_cli_type] "i" (PARAVIRT_PATCH(pv_irq_ops.irq_disable)), \ |
1025 | [paravirt_sti_type] "i" (PARAVIRT_PATCH(irq_enable)), \ | 1076 | [paravirt_cli_opptr] "m" (pv_irq_ops.irq_disable), \ |
1077 | [paravirt_sti_type] "i" (PARAVIRT_PATCH(pv_irq_ops.irq_enable)), \ | ||
1078 | [paravirt_sti_opptr] "m" (pv_irq_ops.irq_enable), \ | ||
1026 | paravirt_clobber(CLBR_EAX) | 1079 | paravirt_clobber(CLBR_EAX) |
1027 | 1080 | ||
1028 | /* Make sure as little as possible of this mess escapes. */ | 1081 | /* Make sure as little as possible of this mess escapes. */ |
@@ -1042,7 +1095,7 @@ static inline unsigned long __raw_local_irq_save(void) | |||
1042 | 1095 | ||
1043 | #else /* __ASSEMBLY__ */ | 1096 | #else /* __ASSEMBLY__ */ |
1044 | 1097 | ||
1045 | #define PARA_PATCH(off) ((off) / 4) | 1098 | #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4) |
1046 | 1099 | ||
1047 | #define PARA_SITE(ptype, clobbers, ops) \ | 1100 | #define PARA_SITE(ptype, clobbers, ops) \ |
1048 | 771:; \ | 1101 | 771:; \ |
@@ -1055,29 +1108,29 @@ static inline unsigned long __raw_local_irq_save(void) | |||
1055 | .short clobbers; \ | 1108 | .short clobbers; \ |
1056 | .popsection | 1109 | .popsection |
1057 | 1110 | ||
1058 | #define INTERRUPT_RETURN \ | 1111 | #define INTERRUPT_RETURN \ |
1059 | PARA_SITE(PARA_PATCH(PARAVIRT_iret), CLBR_NONE, \ | 1112 | PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \ |
1060 | jmp *%cs:paravirt_ops+PARAVIRT_iret) | 1113 | jmp *%cs:pv_cpu_ops+PV_CPU_iret) |
1061 | 1114 | ||
1062 | #define DISABLE_INTERRUPTS(clobbers) \ | 1115 | #define DISABLE_INTERRUPTS(clobbers) \ |
1063 | PARA_SITE(PARA_PATCH(PARAVIRT_irq_disable), clobbers, \ | 1116 | PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \ |
1064 | pushl %eax; pushl %ecx; pushl %edx; \ | 1117 | pushl %eax; pushl %ecx; pushl %edx; \ |
1065 | call *%cs:paravirt_ops+PARAVIRT_irq_disable; \ | 1118 | call *%cs:pv_irq_ops+PV_IRQ_irq_disable; \ |
1066 | popl %edx; popl %ecx; popl %eax) \ | 1119 | popl %edx; popl %ecx; popl %eax) \ |
1067 | 1120 | ||
1068 | #define ENABLE_INTERRUPTS(clobbers) \ | 1121 | #define ENABLE_INTERRUPTS(clobbers) \ |
1069 | PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable), clobbers, \ | 1122 | PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \ |
1070 | pushl %eax; pushl %ecx; pushl %edx; \ | 1123 | pushl %eax; pushl %ecx; pushl %edx; \ |
1071 | call *%cs:paravirt_ops+PARAVIRT_irq_enable; \ | 1124 | call *%cs:pv_irq_ops+PV_IRQ_irq_enable; \ |
1072 | popl %edx; popl %ecx; popl %eax) | 1125 | popl %edx; popl %ecx; popl %eax) |
1073 | 1126 | ||
1074 | #define ENABLE_INTERRUPTS_SYSEXIT \ | 1127 | #define ENABLE_INTERRUPTS_SYSEXIT \ |
1075 | PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable_sysexit), CLBR_NONE, \ | 1128 | PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), CLBR_NONE,\ |
1076 | jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit) | 1129 | jmp *%cs:pv_cpu_ops+PV_CPU_irq_enable_sysexit) |
1077 | 1130 | ||
1078 | #define GET_CR0_INTO_EAX \ | 1131 | #define GET_CR0_INTO_EAX \ |
1079 | push %ecx; push %edx; \ | 1132 | push %ecx; push %edx; \ |
1080 | call *paravirt_ops+PARAVIRT_read_cr0; \ | 1133 | call *pv_cpu_ops+PV_CPU_read_cr0; \ |
1081 | pop %edx; pop %ecx | 1134 | pop %edx; pop %ecx |
1082 | 1135 | ||
1083 | #endif /* __ASSEMBLY__ */ | 1136 | #endif /* __ASSEMBLY__ */ |
diff --git a/include/asm-x86/pgtable-3level-defs.h b/include/asm-x86/pgtable-3level-defs.h index c0df89f66e8b..448ac9516314 100644 --- a/include/asm-x86/pgtable-3level-defs.h +++ b/include/asm-x86/pgtable-3level-defs.h | |||
@@ -2,7 +2,7 @@ | |||
2 | #define _I386_PGTABLE_3LEVEL_DEFS_H | 2 | #define _I386_PGTABLE_3LEVEL_DEFS_H |
3 | 3 | ||
4 | #ifdef CONFIG_PARAVIRT | 4 | #ifdef CONFIG_PARAVIRT |
5 | #define SHARED_KERNEL_PMD (paravirt_ops.shared_kernel_pmd) | 5 | #define SHARED_KERNEL_PMD (pv_info.shared_kernel_pmd) |
6 | #else | 6 | #else |
7 | #define SHARED_KERNEL_PMD 1 | 7 | #define SHARED_KERNEL_PMD 1 |
8 | #endif | 8 | #endif |