diff options
Diffstat (limited to 'arch/powerpc/kvm/book3s_hv_rmhandlers.S')
-rw-r--r-- | arch/powerpc/kvm/book3s_hv_rmhandlers.S | 228 |
1 files changed, 161 insertions, 67 deletions
diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index e33d11f1b977..b02f91e4c70d 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S | |||
@@ -79,10 +79,6 @@ _GLOBAL(kvmppc_hv_entry_trampoline) | |||
79 | * * | 79 | * * |
80 | *****************************************************************************/ | 80 | *****************************************************************************/ |
81 | 81 | ||
82 | #define XICS_XIRR 4 | ||
83 | #define XICS_QIRR 0xc | ||
84 | #define XICS_IPI 2 /* interrupt source # for IPIs */ | ||
85 | |||
86 | /* | 82 | /* |
87 | * We come in here when wakened from nap mode on a secondary hw thread. | 83 | * We come in here when wakened from nap mode on a secondary hw thread. |
88 | * Relocation is off and most register values are lost. | 84 | * Relocation is off and most register values are lost. |
@@ -101,50 +97,51 @@ kvm_start_guest: | |||
101 | li r0,1 | 97 | li r0,1 |
102 | stb r0,PACA_NAPSTATELOST(r13) | 98 | stb r0,PACA_NAPSTATELOST(r13) |
103 | 99 | ||
104 | /* get vcpu pointer, NULL if we have no vcpu to run */ | 100 | /* were we napping due to cede? */ |
105 | ld r4,HSTATE_KVM_VCPU(r13) | 101 | lbz r0,HSTATE_NAPPING(r13) |
106 | cmpdi cr1,r4,0 | 102 | cmpwi r0,0 |
103 | bne kvm_end_cede | ||
104 | |||
105 | /* | ||
106 | * We weren't napping due to cede, so this must be a secondary | ||
107 | * thread being woken up to run a guest, or being woken up due | ||
108 | * to a stray IPI. (Or due to some machine check or hypervisor | ||
109 | * maintenance interrupt while the core is in KVM.) | ||
110 | */ | ||
107 | 111 | ||
108 | /* Check the wake reason in SRR1 to see why we got here */ | 112 | /* Check the wake reason in SRR1 to see why we got here */ |
109 | mfspr r3,SPRN_SRR1 | 113 | mfspr r3,SPRN_SRR1 |
110 | rlwinm r3,r3,44-31,0x7 /* extract wake reason field */ | 114 | rlwinm r3,r3,44-31,0x7 /* extract wake reason field */ |
111 | cmpwi r3,4 /* was it an external interrupt? */ | 115 | cmpwi r3,4 /* was it an external interrupt? */ |
112 | bne 27f | 116 | bne 27f /* if not */ |
113 | 117 | ld r5,HSTATE_XICS_PHYS(r13) | |
114 | /* | 118 | li r7,XICS_XIRR /* if it was an external interrupt, */ |
115 | * External interrupt - for now assume it is an IPI, since we | ||
116 | * should never get any other interrupts sent to offline threads. | ||
117 | * Only do this for secondary threads. | ||
118 | */ | ||
119 | beq cr1,25f | ||
120 | lwz r3,VCPU_PTID(r4) | ||
121 | cmpwi r3,0 | ||
122 | beq 27f | ||
123 | 25: ld r5,HSTATE_XICS_PHYS(r13) | ||
124 | li r0,0xff | ||
125 | li r6,XICS_QIRR | ||
126 | li r7,XICS_XIRR | ||
127 | lwzcix r8,r5,r7 /* get and ack the interrupt */ | 119 | lwzcix r8,r5,r7 /* get and ack the interrupt */ |
128 | sync | 120 | sync |
129 | clrldi. r9,r8,40 /* get interrupt source ID. */ | 121 | clrldi. r9,r8,40 /* get interrupt source ID. */ |
130 | beq 27f /* none there? */ | 122 | beq 28f /* none there? */ |
131 | cmpwi r9,XICS_IPI | 123 | cmpwi r9,XICS_IPI /* was it an IPI? */ |
132 | bne 26f | 124 | bne 29f |
125 | li r0,0xff | ||
126 | li r6,XICS_MFRR | ||
133 | stbcix r0,r5,r6 /* clear IPI */ | 127 | stbcix r0,r5,r6 /* clear IPI */ |
134 | 26: stwcix r8,r5,r7 /* EOI the interrupt */ | 128 | stwcix r8,r5,r7 /* EOI the interrupt */ |
135 | 129 | sync /* order loading of vcpu after that */ | |
136 | 27: /* XXX should handle hypervisor maintenance interrupts etc. here */ | ||
137 | 130 | ||
138 | /* reload vcpu pointer after clearing the IPI */ | 131 | /* get vcpu pointer, NULL if we have no vcpu to run */ |
139 | ld r4,HSTATE_KVM_VCPU(r13) | 132 | ld r4,HSTATE_KVM_VCPU(r13) |
140 | cmpdi r4,0 | 133 | cmpdi r4,0 |
141 | /* if we have no vcpu to run, go back to sleep */ | 134 | /* if we have no vcpu to run, go back to sleep */ |
142 | beq kvm_no_guest | 135 | beq kvm_no_guest |
136 | b kvmppc_hv_entry | ||
143 | 137 | ||
144 | /* were we napping due to cede? */ | 138 | 27: /* XXX should handle hypervisor maintenance interrupts etc. here */ |
145 | lbz r0,HSTATE_NAPPING(r13) | 139 | b kvm_no_guest |
146 | cmpwi r0,0 | 140 | 28: /* SRR1 said external but ICP said nope?? */ |
147 | bne kvm_end_cede | 141 | b kvm_no_guest |
142 | 29: /* External non-IPI interrupt to offline secondary thread? help?? */ | ||
143 | stw r8,HSTATE_SAVED_XIRR(r13) | ||
144 | b kvm_no_guest | ||
148 | 145 | ||
149 | .global kvmppc_hv_entry | 146 | .global kvmppc_hv_entry |
150 | kvmppc_hv_entry: | 147 | kvmppc_hv_entry: |
@@ -260,6 +257,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206) | |||
260 | lwz r5, LPPACA_YIELDCOUNT(r3) | 257 | lwz r5, LPPACA_YIELDCOUNT(r3) |
261 | addi r5, r5, 1 | 258 | addi r5, r5, 1 |
262 | stw r5, LPPACA_YIELDCOUNT(r3) | 259 | stw r5, LPPACA_YIELDCOUNT(r3) |
260 | li r6, 1 | ||
261 | stb r6, VCPU_VPA_DIRTY(r4) | ||
263 | 25: | 262 | 25: |
264 | /* Load up DAR and DSISR */ | 263 | /* Load up DAR and DSISR */ |
265 | ld r5, VCPU_DAR(r4) | 264 | ld r5, VCPU_DAR(r4) |
@@ -485,20 +484,20 @@ toc_tlbie_lock: | |||
485 | mtctr r6 | 484 | mtctr r6 |
486 | mtxer r7 | 485 | mtxer r7 |
487 | 486 | ||
487 | ld r10, VCPU_PC(r4) | ||
488 | ld r11, VCPU_MSR(r4) | ||
488 | kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */ | 489 | kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */ |
489 | ld r6, VCPU_SRR0(r4) | 490 | ld r6, VCPU_SRR0(r4) |
490 | ld r7, VCPU_SRR1(r4) | 491 | ld r7, VCPU_SRR1(r4) |
491 | ld r10, VCPU_PC(r4) | ||
492 | ld r11, VCPU_MSR(r4) /* r11 = vcpu->arch.msr & ~MSR_HV */ | ||
493 | 492 | ||
493 | /* r11 = vcpu->arch.msr & ~MSR_HV */ | ||
494 | rldicl r11, r11, 63 - MSR_HV_LG, 1 | 494 | rldicl r11, r11, 63 - MSR_HV_LG, 1 |
495 | rotldi r11, r11, 1 + MSR_HV_LG | 495 | rotldi r11, r11, 1 + MSR_HV_LG |
496 | ori r11, r11, MSR_ME | 496 | ori r11, r11, MSR_ME |
497 | 497 | ||
498 | /* Check if we can deliver an external or decrementer interrupt now */ | 498 | /* Check if we can deliver an external or decrementer interrupt now */ |
499 | ld r0,VCPU_PENDING_EXC(r4) | 499 | ld r0,VCPU_PENDING_EXC(r4) |
500 | li r8,(1 << BOOK3S_IRQPRIO_EXTERNAL) | 500 | lis r8,(1 << BOOK3S_IRQPRIO_EXTERNAL_LEVEL)@h |
501 | oris r8,r8,(1 << BOOK3S_IRQPRIO_EXTERNAL_LEVEL)@h | ||
502 | and r0,r0,r8 | 501 | and r0,r0,r8 |
503 | cmpdi cr1,r0,0 | 502 | cmpdi cr1,r0,0 |
504 | andi. r0,r11,MSR_EE | 503 | andi. r0,r11,MSR_EE |
@@ -526,10 +525,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206) | |||
526 | /* Move SRR0 and SRR1 into the respective regs */ | 525 | /* Move SRR0 and SRR1 into the respective regs */ |
527 | 5: mtspr SPRN_SRR0, r6 | 526 | 5: mtspr SPRN_SRR0, r6 |
528 | mtspr SPRN_SRR1, r7 | 527 | mtspr SPRN_SRR1, r7 |
529 | li r0,0 | ||
530 | stb r0,VCPU_CEDED(r4) /* cancel cede */ | ||
531 | 528 | ||
532 | fast_guest_return: | 529 | fast_guest_return: |
530 | li r0,0 | ||
531 | stb r0,VCPU_CEDED(r4) /* cancel cede */ | ||
533 | mtspr SPRN_HSRR0,r10 | 532 | mtspr SPRN_HSRR0,r10 |
534 | mtspr SPRN_HSRR1,r11 | 533 | mtspr SPRN_HSRR1,r11 |
535 | 534 | ||
@@ -676,17 +675,99 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206) | |||
676 | cmpwi r12,BOOK3S_INTERRUPT_SYSCALL | 675 | cmpwi r12,BOOK3S_INTERRUPT_SYSCALL |
677 | beq hcall_try_real_mode | 676 | beq hcall_try_real_mode |
678 | 677 | ||
679 | /* Check for mediated interrupts (could be done earlier really ...) */ | 678 | /* Only handle external interrupts here on arch 206 and later */ |
680 | BEGIN_FTR_SECTION | 679 | BEGIN_FTR_SECTION |
681 | cmpwi r12,BOOK3S_INTERRUPT_EXTERNAL | 680 | b ext_interrupt_to_host |
682 | bne+ 1f | 681 | END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206) |
683 | andi. r0,r11,MSR_EE | 682 | |
684 | beq 1f | 683 | /* External interrupt ? */ |
685 | mfspr r5,SPRN_LPCR | 684 | cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL |
686 | andi. r0,r5,LPCR_MER | 685 | bne+ ext_interrupt_to_host |
687 | bne bounce_ext_interrupt | 686 | |
688 | 1: | 687 | /* External interrupt, first check for host_ipi. If this is |
689 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206) | 688 | * set, we know the host wants us out so let's do it now |
689 | */ | ||
690 | do_ext_interrupt: | ||
691 | lbz r0, HSTATE_HOST_IPI(r13) | ||
692 | cmpwi r0, 0 | ||
693 | bne ext_interrupt_to_host | ||
694 | |||
695 | /* Now read the interrupt from the ICP */ | ||
696 | ld r5, HSTATE_XICS_PHYS(r13) | ||
697 | li r7, XICS_XIRR | ||
698 | cmpdi r5, 0 | ||
699 | beq- ext_interrupt_to_host | ||
700 | lwzcix r3, r5, r7 | ||
701 | rlwinm. r0, r3, 0, 0xffffff | ||
702 | sync | ||
703 | beq 3f /* if nothing pending in the ICP */ | ||
704 | |||
705 | /* We found something in the ICP... | ||
706 | * | ||
707 | * If it's not an IPI, stash it in the PACA and return to | ||
708 | * the host, we don't (yet) handle directing real external | ||
709 | * interrupts directly to the guest | ||
710 | */ | ||
711 | cmpwi r0, XICS_IPI | ||
712 | bne ext_stash_for_host | ||
713 | |||
714 | /* It's an IPI, clear the MFRR and EOI it */ | ||
715 | li r0, 0xff | ||
716 | li r6, XICS_MFRR | ||
717 | stbcix r0, r5, r6 /* clear the IPI */ | ||
718 | stwcix r3, r5, r7 /* EOI it */ | ||
719 | sync | ||
720 | |||
721 | /* We need to re-check host IPI now in case it got set in the | ||
722 | * meantime. If it's clear, we bounce the interrupt to the | ||
723 | * guest | ||
724 | */ | ||
725 | lbz r0, HSTATE_HOST_IPI(r13) | ||
726 | cmpwi r0, 0 | ||
727 | bne- 1f | ||
728 | |||
729 | /* Allright, looks like an IPI for the guest, we need to set MER */ | ||
730 | 3: | ||
731 | /* Check if any CPU is heading out to the host, if so head out too */ | ||
732 | ld r5, HSTATE_KVM_VCORE(r13) | ||
733 | lwz r0, VCORE_ENTRY_EXIT(r5) | ||
734 | cmpwi r0, 0x100 | ||
735 | bge ext_interrupt_to_host | ||
736 | |||
737 | /* See if there is a pending interrupt for the guest */ | ||
738 | mfspr r8, SPRN_LPCR | ||
739 | ld r0, VCPU_PENDING_EXC(r9) | ||
740 | /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */ | ||
741 | rldicl. r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63 | ||
742 | rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH | ||
743 | beq 2f | ||
744 | |||
745 | /* And if the guest EE is set, we can deliver immediately, else | ||
746 | * we return to the guest with MER set | ||
747 | */ | ||
748 | andi. r0, r11, MSR_EE | ||
749 | beq 2f | ||
750 | mtspr SPRN_SRR0, r10 | ||
751 | mtspr SPRN_SRR1, r11 | ||
752 | li r10, BOOK3S_INTERRUPT_EXTERNAL | ||
753 | li r11, (MSR_ME << 1) | 1 /* synthesize MSR_SF | MSR_ME */ | ||
754 | rotldi r11, r11, 63 | ||
755 | 2: mr r4, r9 | ||
756 | mtspr SPRN_LPCR, r8 | ||
757 | b fast_guest_return | ||
758 | |||
759 | /* We raced with the host, we need to resend that IPI, bummer */ | ||
760 | 1: li r0, IPI_PRIORITY | ||
761 | stbcix r0, r5, r6 /* set the IPI */ | ||
762 | sync | ||
763 | b ext_interrupt_to_host | ||
764 | |||
765 | ext_stash_for_host: | ||
766 | /* It's not an IPI and it's for the host, stash it in the PACA | ||
767 | * before exit, it will be picked up by the host ICP driver | ||
768 | */ | ||
769 | stw r3, HSTATE_SAVED_XIRR(r13) | ||
770 | ext_interrupt_to_host: | ||
690 | 771 | ||
691 | guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ | 772 | guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ |
692 | /* Save DEC */ | 773 | /* Save DEC */ |
@@ -829,7 +910,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201) | |||
829 | beq 44f | 910 | beq 44f |
830 | ld r8,HSTATE_XICS_PHYS(r6) /* get thread's XICS reg addr */ | 911 | ld r8,HSTATE_XICS_PHYS(r6) /* get thread's XICS reg addr */ |
831 | li r0,IPI_PRIORITY | 912 | li r0,IPI_PRIORITY |
832 | li r7,XICS_QIRR | 913 | li r7,XICS_MFRR |
833 | stbcix r0,r7,r8 /* trigger the IPI */ | 914 | stbcix r0,r7,r8 /* trigger the IPI */ |
834 | 44: srdi. r3,r3,1 | 915 | 44: srdi. r3,r3,1 |
835 | addi r6,r6,PACA_SIZE | 916 | addi r6,r6,PACA_SIZE |
@@ -1018,6 +1099,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206) | |||
1018 | lwz r3, LPPACA_YIELDCOUNT(r8) | 1099 | lwz r3, LPPACA_YIELDCOUNT(r8) |
1019 | addi r3, r3, 1 | 1100 | addi r3, r3, 1 |
1020 | stw r3, LPPACA_YIELDCOUNT(r8) | 1101 | stw r3, LPPACA_YIELDCOUNT(r8) |
1102 | li r3, 1 | ||
1103 | stb r3, VCPU_VPA_DIRTY(r9) | ||
1021 | 25: | 1104 | 25: |
1022 | /* Save PMU registers if requested */ | 1105 | /* Save PMU registers if requested */ |
1023 | /* r8 and cr0.eq are live here */ | 1106 | /* r8 and cr0.eq are live here */ |
@@ -1350,11 +1433,19 @@ hcall_real_table: | |||
1350 | .long 0 /* 0x58 */ | 1433 | .long 0 /* 0x58 */ |
1351 | .long 0 /* 0x5c */ | 1434 | .long 0 /* 0x5c */ |
1352 | .long 0 /* 0x60 */ | 1435 | .long 0 /* 0x60 */ |
1353 | .long 0 /* 0x64 */ | 1436 | #ifdef CONFIG_KVM_XICS |
1354 | .long 0 /* 0x68 */ | 1437 | .long .kvmppc_rm_h_eoi - hcall_real_table |
1355 | .long 0 /* 0x6c */ | 1438 | .long .kvmppc_rm_h_cppr - hcall_real_table |
1356 | .long 0 /* 0x70 */ | 1439 | .long .kvmppc_rm_h_ipi - hcall_real_table |
1357 | .long 0 /* 0x74 */ | 1440 | .long 0 /* 0x70 - H_IPOLL */ |
1441 | .long .kvmppc_rm_h_xirr - hcall_real_table | ||
1442 | #else | ||
1443 | .long 0 /* 0x64 - H_EOI */ | ||
1444 | .long 0 /* 0x68 - H_CPPR */ | ||
1445 | .long 0 /* 0x6c - H_IPI */ | ||
1446 | .long 0 /* 0x70 - H_IPOLL */ | ||
1447 | .long 0 /* 0x74 - H_XIRR */ | ||
1448 | #endif | ||
1358 | .long 0 /* 0x78 */ | 1449 | .long 0 /* 0x78 */ |
1359 | .long 0 /* 0x7c */ | 1450 | .long 0 /* 0x7c */ |
1360 | .long 0 /* 0x80 */ | 1451 | .long 0 /* 0x80 */ |
@@ -1405,15 +1496,6 @@ ignore_hdec: | |||
1405 | mr r4,r9 | 1496 | mr r4,r9 |
1406 | b fast_guest_return | 1497 | b fast_guest_return |
1407 | 1498 | ||
1408 | bounce_ext_interrupt: | ||
1409 | mr r4,r9 | ||
1410 | mtspr SPRN_SRR0,r10 | ||
1411 | mtspr SPRN_SRR1,r11 | ||
1412 | li r10,BOOK3S_INTERRUPT_EXTERNAL | ||
1413 | li r11,(MSR_ME << 1) | 1 /* synthesize MSR_SF | MSR_ME */ | ||
1414 | rotldi r11,r11,63 | ||
1415 | b fast_guest_return | ||
1416 | |||
1417 | _GLOBAL(kvmppc_h_set_dabr) | 1499 | _GLOBAL(kvmppc_h_set_dabr) |
1418 | std r4,VCPU_DABR(r3) | 1500 | std r4,VCPU_DABR(r3) |
1419 | /* Work around P7 bug where DABR can get corrupted on mtspr */ | 1501 | /* Work around P7 bug where DABR can get corrupted on mtspr */ |
@@ -1519,6 +1601,9 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_206) | |||
1519 | b . | 1601 | b . |
1520 | 1602 | ||
1521 | kvm_end_cede: | 1603 | kvm_end_cede: |
1604 | /* get vcpu pointer */ | ||
1605 | ld r4, HSTATE_KVM_VCPU(r13) | ||
1606 | |||
1522 | /* Woken by external or decrementer interrupt */ | 1607 | /* Woken by external or decrementer interrupt */ |
1523 | ld r1, HSTATE_HOST_R1(r13) | 1608 | ld r1, HSTATE_HOST_R1(r13) |
1524 | 1609 | ||
@@ -1558,6 +1643,16 @@ kvm_end_cede: | |||
1558 | li r0,0 | 1643 | li r0,0 |
1559 | stb r0,HSTATE_NAPPING(r13) | 1644 | stb r0,HSTATE_NAPPING(r13) |
1560 | 1645 | ||
1646 | /* Check the wake reason in SRR1 to see why we got here */ | ||
1647 | mfspr r3, SPRN_SRR1 | ||
1648 | rlwinm r3, r3, 44-31, 0x7 /* extract wake reason field */ | ||
1649 | cmpwi r3, 4 /* was it an external interrupt? */ | ||
1650 | li r12, BOOK3S_INTERRUPT_EXTERNAL | ||
1651 | mr r9, r4 | ||
1652 | ld r10, VCPU_PC(r9) | ||
1653 | ld r11, VCPU_MSR(r9) | ||
1654 | beq do_ext_interrupt /* if so */ | ||
1655 | |||
1561 | /* see if any other thread is already exiting */ | 1656 | /* see if any other thread is already exiting */ |
1562 | lwz r0,VCORE_ENTRY_EXIT(r5) | 1657 | lwz r0,VCORE_ENTRY_EXIT(r5) |
1563 | cmpwi r0,0x100 | 1658 | cmpwi r0,0x100 |
@@ -1577,8 +1672,7 @@ kvm_cede_prodded: | |||
1577 | 1672 | ||
1578 | /* we've ceded but we want to give control to the host */ | 1673 | /* we've ceded but we want to give control to the host */ |
1579 | kvm_cede_exit: | 1674 | kvm_cede_exit: |
1580 | li r3,H_TOO_HARD | 1675 | b hcall_real_fallback |
1581 | blr | ||
1582 | 1676 | ||
1583 | /* Try to handle a machine check in real mode */ | 1677 | /* Try to handle a machine check in real mode */ |
1584 | machine_check_realmode: | 1678 | machine_check_realmode: |
@@ -1626,7 +1720,7 @@ secondary_nap: | |||
1626 | beq 37f | 1720 | beq 37f |
1627 | sync | 1721 | sync |
1628 | li r0, 0xff | 1722 | li r0, 0xff |
1629 | li r6, XICS_QIRR | 1723 | li r6, XICS_MFRR |
1630 | stbcix r0, r5, r6 /* clear the IPI */ | 1724 | stbcix r0, r5, r6 /* clear the IPI */ |
1631 | stwcix r3, r5, r7 /* EOI it */ | 1725 | stwcix r3, r5, r7 /* EOI it */ |
1632 | 37: sync | 1726 | 37: sync |