aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/i386/kernel/paravirt.c14
-rw-r--r--arch/i386/kernel/vmi.c39
-rw-r--r--include/asm-i386/paravirt.h177
3 files changed, 104 insertions, 126 deletions
diff --git a/arch/i386/kernel/paravirt.c b/arch/i386/kernel/paravirt.c
index 54cc14d99740..f2982832d3b9 100644
--- a/arch/i386/kernel/paravirt.c
+++ b/arch/i386/kernel/paravirt.c
@@ -58,7 +58,6 @@ DEF_NATIVE(cli, "cli");
58DEF_NATIVE(sti, "sti"); 58DEF_NATIVE(sti, "sti");
59DEF_NATIVE(popf, "push %eax; popf"); 59DEF_NATIVE(popf, "push %eax; popf");
60DEF_NATIVE(pushf, "pushf; pop %eax"); 60DEF_NATIVE(pushf, "pushf; pop %eax");
61DEF_NATIVE(pushf_cli, "pushf; pop %eax; cli");
62DEF_NATIVE(iret, "iret"); 61DEF_NATIVE(iret, "iret");
63DEF_NATIVE(sti_sysexit, "sti; sysexit"); 62DEF_NATIVE(sti_sysexit, "sti; sysexit");
64 63
@@ -66,13 +65,12 @@ static const struct native_insns
66{ 65{
67 const char *start, *end; 66 const char *start, *end;
68} native_insns[] = { 67} native_insns[] = {
69 [PARAVIRT_IRQ_DISABLE] = { start_cli, end_cli }, 68 [PARAVIRT_PATCH(irq_disable)] = { start_cli, end_cli },
70 [PARAVIRT_IRQ_ENABLE] = { start_sti, end_sti }, 69 [PARAVIRT_PATCH(irq_enable)] = { start_sti, end_sti },
71 [PARAVIRT_RESTORE_FLAGS] = { start_popf, end_popf }, 70 [PARAVIRT_PATCH(restore_fl)] = { start_popf, end_popf },
72 [PARAVIRT_SAVE_FLAGS] = { start_pushf, end_pushf }, 71 [PARAVIRT_PATCH(save_fl)] = { start_pushf, end_pushf },
73 [PARAVIRT_SAVE_FLAGS_IRQ_DISABLE] = { start_pushf_cli, end_pushf_cli }, 72 [PARAVIRT_PATCH(iret)] = { start_iret, end_iret },
74 [PARAVIRT_INTERRUPT_RETURN] = { start_iret, end_iret }, 73 [PARAVIRT_PATCH(irq_enable_sysexit)] = { start_sti_sysexit, end_sti_sysexit },
75 [PARAVIRT_STI_SYSEXIT] = { start_sti_sysexit, end_sti_sysexit },
76}; 74};
77 75
78static unsigned native_patch(u8 type, u16 clobbers, void *insns, unsigned len) 76static unsigned native_patch(u8 type, u16 clobbers, void *insns, unsigned len)
diff --git a/arch/i386/kernel/vmi.c b/arch/i386/kernel/vmi.c
index ea77d93f59dd..b8d01c3cbff4 100644
--- a/arch/i386/kernel/vmi.c
+++ b/arch/i386/kernel/vmi.c
@@ -83,11 +83,6 @@ extern struct paravirt_patch __start_parainstructions[],
83#define MNEM_JMP 0xe9 83#define MNEM_JMP 0xe9
84#define MNEM_RET 0xc3 84#define MNEM_RET 0xc3
85 85
86static char irq_save_disable_callout[] = {
87 MNEM_CALL, 0, 0, 0, 0,
88 MNEM_CALL, 0, 0, 0, 0,
89 MNEM_RET
90};
91#define IRQ_PATCH_INT_MASK 0 86#define IRQ_PATCH_INT_MASK 0
92#define IRQ_PATCH_DISABLE 5 87#define IRQ_PATCH_DISABLE 5
93 88
@@ -135,33 +130,17 @@ static unsigned patch_internal(int call, unsigned len, void *insns)
135static unsigned vmi_patch(u8 type, u16 clobbers, void *insns, unsigned len) 130static unsigned vmi_patch(u8 type, u16 clobbers, void *insns, unsigned len)
136{ 131{
137 switch (type) { 132 switch (type) {
138 case PARAVIRT_IRQ_DISABLE: 133 case PARAVIRT_PATCH(irq_disable):
139 return patch_internal(VMI_CALL_DisableInterrupts, len, insns); 134 return patch_internal(VMI_CALL_DisableInterrupts, len, insns);
140 case PARAVIRT_IRQ_ENABLE: 135 case PARAVIRT_PATCH(irq_enable):
141 return patch_internal(VMI_CALL_EnableInterrupts, len, insns); 136 return patch_internal(VMI_CALL_EnableInterrupts, len, insns);
142 case PARAVIRT_RESTORE_FLAGS: 137 case PARAVIRT_PATCH(restore_fl):
143 return patch_internal(VMI_CALL_SetInterruptMask, len, insns); 138 return patch_internal(VMI_CALL_SetInterruptMask, len, insns);
144 case PARAVIRT_SAVE_FLAGS: 139 case PARAVIRT_PATCH(save_fl):
145 return patch_internal(VMI_CALL_GetInterruptMask, len, insns); 140 return patch_internal(VMI_CALL_GetInterruptMask, len, insns);
146 case PARAVIRT_SAVE_FLAGS_IRQ_DISABLE: 141 case PARAVIRT_PATCH(iret):
147 if (len >= 10) {
148 patch_internal(VMI_CALL_GetInterruptMask, len, insns);
149 patch_internal(VMI_CALL_DisableInterrupts, len-5, insns+5);
150 return 10;
151 } else {
152 /*
153 * You bastards didn't leave enough room to
154 * patch save_flags_irq_disable inline. Patch
155 * to a helper
156 */
157 BUG_ON(len < 5);
158 *(char *)insns = MNEM_CALL;
159 patch_offset(insns, irq_save_disable_callout);
160 return 5;
161 }
162 case PARAVIRT_INTERRUPT_RETURN:
163 return patch_internal(VMI_CALL_IRET, len, insns); 142 return patch_internal(VMI_CALL_IRET, len, insns);
164 case PARAVIRT_STI_SYSEXIT: 143 case PARAVIRT_PATCH(irq_enable_sysexit):
165 return patch_internal(VMI_CALL_SYSEXIT, len, insns); 144 return patch_internal(VMI_CALL_SYSEXIT, len, insns);
166 default: 145 default:
167 break; 146 break;
@@ -796,12 +775,6 @@ static inline int __init activate_vmi(void)
796 para_fill(irq_disable, DisableInterrupts); 775 para_fill(irq_disable, DisableInterrupts);
797 para_fill(irq_enable, EnableInterrupts); 776 para_fill(irq_enable, EnableInterrupts);
798 777
799 /* irq_save_disable !!! sheer pain */
800 patch_offset(&irq_save_disable_callout[IRQ_PATCH_INT_MASK],
801 (char *)paravirt_ops.save_fl);
802 patch_offset(&irq_save_disable_callout[IRQ_PATCH_DISABLE],
803 (char *)paravirt_ops.irq_disable);
804
805 para_fill(wbinvd, WBINVD); 778 para_fill(wbinvd, WBINVD);
806 para_fill(read_tsc, RDTSC); 779 para_fill(read_tsc, RDTSC);
807 780
diff --git a/include/asm-i386/paravirt.h b/include/asm-i386/paravirt.h
index b4cc2fc4031e..1dbc01f4ed4d 100644
--- a/include/asm-i386/paravirt.h
+++ b/include/asm-i386/paravirt.h
@@ -4,19 +4,8 @@
4 * para-virtualization: those hooks are defined here. */ 4 * para-virtualization: those hooks are defined here. */
5 5
6#ifdef CONFIG_PARAVIRT 6#ifdef CONFIG_PARAVIRT
7#include <linux/stringify.h>
8#include <asm/page.h> 7#include <asm/page.h>
9 8
10/* These are the most performance critical ops, so we want to be able to patch
11 * callers */
12#define PARAVIRT_IRQ_DISABLE 0
13#define PARAVIRT_IRQ_ENABLE 1
14#define PARAVIRT_RESTORE_FLAGS 2
15#define PARAVIRT_SAVE_FLAGS 3
16#define PARAVIRT_SAVE_FLAGS_IRQ_DISABLE 4
17#define PARAVIRT_INTERRUPT_RETURN 5
18#define PARAVIRT_STI_SYSEXIT 6
19
20/* Bitmask of what can be clobbered: usually at least eax. */ 9/* Bitmask of what can be clobbered: usually at least eax. */
21#define CLBR_NONE 0x0 10#define CLBR_NONE 0x0
22#define CLBR_EAX 0x1 11#define CLBR_EAX 0x1
@@ -191,6 +180,28 @@ struct paravirt_ops
191 180
192extern struct paravirt_ops paravirt_ops; 181extern struct paravirt_ops paravirt_ops;
193 182
183#define PARAVIRT_PATCH(x) \
184 (offsetof(struct paravirt_ops, x) / sizeof(void *))
185
186#define paravirt_type(type) \
187 [paravirt_typenum] "i" (PARAVIRT_PATCH(type))
188#define paravirt_clobber(clobber) \
189 [paravirt_clobber] "i" (clobber)
190
191#define PARAVIRT_CALL "call *paravirt_ops+%c[paravirt_typenum]*4;"
192
193#define _paravirt_alt(insn_string, type, clobber) \
194 "771:\n\t" insn_string "\n" "772:\n" \
195 ".pushsection .parainstructions,\"a\"\n" \
196 " .long 771b\n" \
197 " .byte " type "\n" \
198 " .byte 772b-771b\n" \
199 " .short " clobber "\n" \
200 ".popsection\n"
201
202#define paravirt_alt(insn_string) \
203 _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
204
194#define paravirt_enabled() (paravirt_ops.paravirt_enabled) 205#define paravirt_enabled() (paravirt_ops.paravirt_enabled)
195 206
196static inline void load_esp0(struct tss_struct *tss, 207static inline void load_esp0(struct tss_struct *tss,
@@ -515,93 +526,89 @@ struct paravirt_patch_site {
515extern struct paravirt_patch_site __parainstructions[], 526extern struct paravirt_patch_site __parainstructions[],
516 __parainstructions_end[]; 527 __parainstructions_end[];
517 528
518#define paravirt_alt(insn_string, typenum, clobber) \
519 "771:\n\t" insn_string "\n" "772:\n" \
520 ".pushsection .parainstructions,\"a\"\n" \
521 " .long 771b\n" \
522 " .byte " __stringify(typenum) "\n" \
523 " .byte 772b-771b\n" \
524 " .short " __stringify(clobber) "\n" \
525 ".popsection"
526
527static inline unsigned long __raw_local_save_flags(void) 529static inline unsigned long __raw_local_save_flags(void)
528{ 530{
529 unsigned long f; 531 unsigned long f;
530 532
531 __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" 533 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
532 "call *%1;" 534 PARAVIRT_CALL
533 "popl %%edx; popl %%ecx", 535 "popl %%edx; popl %%ecx")
534 PARAVIRT_SAVE_FLAGS, CLBR_NONE) 536 : "=a"(f)
535 : "=a"(f): "m"(paravirt_ops.save_fl) 537 : paravirt_type(save_fl),
536 : "memory", "cc"); 538 paravirt_clobber(CLBR_NONE)
539 : "memory", "cc");
537 return f; 540 return f;
538} 541}
539 542
540static inline void raw_local_irq_restore(unsigned long f) 543static inline void raw_local_irq_restore(unsigned long f)
541{ 544{
542 __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" 545 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
543 "call *%1;" 546 PARAVIRT_CALL
544 "popl %%edx; popl %%ecx", 547 "popl %%edx; popl %%ecx")
545 PARAVIRT_RESTORE_FLAGS, CLBR_EAX) 548 : "=a"(f)
546 : "=a"(f) : "m" (paravirt_ops.restore_fl), "0"(f) 549 : "0"(f),
547 : "memory", "cc"); 550 paravirt_type(restore_fl),
551 paravirt_clobber(CLBR_EAX)
552 : "memory", "cc");
548} 553}
549 554
550static inline void raw_local_irq_disable(void) 555static inline void raw_local_irq_disable(void)
551{ 556{
552 __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" 557 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
553 "call *%0;" 558 PARAVIRT_CALL
554 "popl %%edx; popl %%ecx", 559 "popl %%edx; popl %%ecx")
555 PARAVIRT_IRQ_DISABLE, CLBR_EAX) 560 :
556 : : "m" (paravirt_ops.irq_disable) 561 : paravirt_type(irq_disable),
557 : "memory", "eax", "cc"); 562 paravirt_clobber(CLBR_EAX)
563 : "memory", "eax", "cc");
558} 564}
559 565
560static inline void raw_local_irq_enable(void) 566static inline void raw_local_irq_enable(void)
561{ 567{
562 __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" 568 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;"
563 "call *%0;" 569 PARAVIRT_CALL
564 "popl %%edx; popl %%ecx", 570 "popl %%edx; popl %%ecx")
565 PARAVIRT_IRQ_ENABLE, CLBR_EAX) 571 :
566 : : "m" (paravirt_ops.irq_enable) 572 : paravirt_type(irq_enable),
567 : "memory", "eax", "cc"); 573 paravirt_clobber(CLBR_EAX)
574 : "memory", "eax", "cc");
568} 575}
569 576
570static inline unsigned long __raw_local_irq_save(void) 577static inline unsigned long __raw_local_irq_save(void)
571{ 578{
572 unsigned long f; 579 unsigned long f;
573 580
574 __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" 581 f = __raw_local_save_flags();
575 "call *%1; pushl %%eax;" 582 raw_local_irq_disable();
576 "call *%2; popl %%eax;"
577 "popl %%edx; popl %%ecx",
578 PARAVIRT_SAVE_FLAGS_IRQ_DISABLE,
579 CLBR_NONE)
580 : "=a"(f)
581 : "m" (paravirt_ops.save_fl),
582 "m" (paravirt_ops.irq_disable)
583 : "memory", "cc");
584 return f; 583 return f;
585} 584}
586 585
587#define CLI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \ 586#define CLI_STRING \
588 "call *paravirt_ops+%c[irq_disable];" \ 587 _paravirt_alt("pushl %%ecx; pushl %%edx;" \
589 "popl %%edx; popl %%ecx", \ 588 "call *paravirt_ops+%c[paravirt_cli_type]*4;" \
590 PARAVIRT_IRQ_DISABLE, CLBR_EAX) 589 "popl %%edx; popl %%ecx", \
590 "%c[paravirt_cli_type]", "%c[paravirt_clobber]")
591
592#define STI_STRING \
593 _paravirt_alt("pushl %%ecx; pushl %%edx;" \
594 "call *paravirt_ops+%c[paravirt_sti_type]*4;" \
595 "popl %%edx; popl %%ecx", \
596 "%c[paravirt_sti_type]", "%c[paravirt_clobber]")
591 597
592#define STI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \
593 "call *paravirt_ops+%c[irq_enable];" \
594 "popl %%edx; popl %%ecx", \
595 PARAVIRT_IRQ_ENABLE, CLBR_EAX)
596#define CLI_STI_CLOBBERS , "%eax" 598#define CLI_STI_CLOBBERS , "%eax"
597#define CLI_STI_INPUT_ARGS \ 599#define CLI_STI_INPUT_ARGS \
598 , \ 600 , \
599 [irq_disable] "i" (offsetof(struct paravirt_ops, irq_disable)), \ 601 [paravirt_cli_type] "i" (PARAVIRT_PATCH(irq_disable)), \
600 [irq_enable] "i" (offsetof(struct paravirt_ops, irq_enable)) 602 [paravirt_sti_type] "i" (PARAVIRT_PATCH(irq_enable)), \
603 paravirt_clobber(CLBR_EAX)
604
605#undef PARAVIRT_CALL
601 606
602#else /* __ASSEMBLY__ */ 607#else /* __ASSEMBLY__ */
603 608
604#define PARA_PATCH(ptype, clobbers, ops) \ 609#define PARA_PATCH(off) ((off) / 4)
610
611#define PARA_SITE(ptype, clobbers, ops) \
605771:; \ 612771:; \
606 ops; \ 613 ops; \
607772:; \ 614772:; \
@@ -612,25 +619,25 @@ static inline unsigned long __raw_local_irq_save(void)
612 .short clobbers; \ 619 .short clobbers; \
613 .popsection 620 .popsection
614 621
615#define INTERRUPT_RETURN \ 622#define INTERRUPT_RETURN \
616 PARA_PATCH(PARAVIRT_INTERRUPT_RETURN, CLBR_ANY, \ 623 PARA_SITE(PARA_PATCH(PARAVIRT_iret), CLBR_ANY, \
617 jmp *%cs:paravirt_ops+PARAVIRT_iret) 624 jmp *%cs:paravirt_ops+PARAVIRT_iret)
618 625
619#define DISABLE_INTERRUPTS(clobbers) \ 626#define DISABLE_INTERRUPTS(clobbers) \
620 PARA_PATCH(PARAVIRT_IRQ_DISABLE, clobbers, \ 627 PARA_SITE(PARA_PATCH(PARAVIRT_irq_disable), clobbers, \
621 pushl %ecx; pushl %edx; \ 628 pushl %ecx; pushl %edx; \
622 call *paravirt_ops+PARAVIRT_irq_disable; \ 629 call *%cs:paravirt_ops+PARAVIRT_irq_disable; \
623 popl %edx; popl %ecx) \ 630 popl %edx; popl %ecx) \
624 631
625#define ENABLE_INTERRUPTS(clobbers) \ 632#define ENABLE_INTERRUPTS(clobbers) \
626 PARA_PATCH(PARAVIRT_IRQ_ENABLE, clobbers, \ 633 PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable), clobbers, \
627 pushl %ecx; pushl %edx; \ 634 pushl %ecx; pushl %edx; \
628 call *%cs:paravirt_ops+PARAVIRT_irq_enable; \ 635 call *%cs:paravirt_ops+PARAVIRT_irq_enable; \
629 popl %edx; popl %ecx) 636 popl %edx; popl %ecx)
630 637
631#define ENABLE_INTERRUPTS_SYSEXIT \ 638#define ENABLE_INTERRUPTS_SYSEXIT \
632 PARA_PATCH(PARAVIRT_STI_SYSEXIT, CLBR_ANY, \ 639 PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable_sysexit), CLBR_ANY, \
633 jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit) 640 jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit)
634 641
635#define GET_CR0_INTO_EAX \ 642#define GET_CR0_INTO_EAX \
636 call *paravirt_ops+PARAVIRT_read_cr0 643 call *paravirt_ops+PARAVIRT_read_cr0
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-07 18:28:09 -0500