Merge branch 'kvm-updates/3.1' of git://git.kernel.org/pub/scm/virt/kvm/kvm

* 'kvm-updates/3.1' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (143 commits)
  KVM: IOMMU: Disable device assignment without interrupt remapping
  KVM: MMU: trace mmio page fault
  KVM: MMU: mmio page fault support
  KVM: MMU: reorganize struct kvm_shadow_walk_iterator
  KVM: MMU: lockless walking shadow page table
  KVM: MMU: do not need atomicly to set/clear spte
  KVM: MMU: introduce the rules to modify shadow page table
  KVM: MMU: abstract some functions to handle fault pfn
  KVM: MMU: filter out the mmio pfn from the fault pfn
  KVM: MMU: remove bypass_guest_pf
  KVM: MMU: split kvm_mmu_free_page
  KVM: MMU: count used shadow pages on prepareing path
  KVM: MMU: rename 'pt_write' to 'emulate'
  KVM: MMU: cleanup for FNAME(fetch)
  KVM: MMU: optimize to handle dirty bit
  KVM: MMU: cache mmio info on page fault path
  KVM: x86: introduce vcpu_mmio_gva_to_gpa to cleanup the code
  KVM: MMU: do not update slot bitmap if spte is nonpresent
  KVM: MMU: fix walking shadow page table
  KVM guest: KVM Steal time registration
  ...

17 files changed, 589 insertions, 137 deletions

diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h
index c0d842cfd012..e30442c539ce 100644
--- a/arch/powerpc/include/asm/cputable.h
+++ b/arch/powerpc/include/asm/cputable.h
@@ -179,8 +179,9 @@ extern const char *powerpc_base_platform;
 #define LONG_ASM_CONST(x)               0
 #endif
 
-
-#define CPU_FTR_HVMODE_206              LONG_ASM_CONST(0x0000000800000000)
+#define CPU_FTR_HVMODE                  LONG_ASM_CONST(0x0000000200000000)
+#define CPU_FTR_ARCH_201                LONG_ASM_CONST(0x0000000400000000)
+#define CPU_FTR_ARCH_206                LONG_ASM_CONST(0x0000000800000000)
 #define CPU_FTR_CFAR                    LONG_ASM_CONST(0x0000001000000000)
 #define CPU_FTR_IABR                    LONG_ASM_CONST(0x0000002000000000)
 #define CPU_FTR_MMCRA                   LONG_ASM_CONST(0x0000004000000000)
@@ -401,9 +402,10 @@ extern const char *powerpc_base_platform;
             CPU_FTR_MMCRA | CPU_FTR_CP_USE_DCBTZ | \
             CPU_FTR_STCX_CHECKS_ADDRESS)
 #define CPU_FTRS_PPC970 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \
-            CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \
+            CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_201 | \
             CPU_FTR_ALTIVEC_COMP | CPU_FTR_CAN_NAP | CPU_FTR_MMCRA | \
-            CPU_FTR_CP_USE_DCBTZ | CPU_FTR_STCX_CHECKS_ADDRESS)
+            CPU_FTR_CP_USE_DCBTZ | CPU_FTR_STCX_CHECKS_ADDRESS | \
+            CPU_FTR_HVMODE)
 #define CPU_FTRS_POWER5 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \
             CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \
             CPU_FTR_MMCRA | CPU_FTR_SMT | \
@@ -417,13 +419,13 @@ extern const char *powerpc_base_platform;
             CPU_FTR_DSCR | CPU_FTR_UNALIGNED_LD_STD | \
             CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_CFAR)
 #define CPU_FTRS_POWER7 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \
-            CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_HVMODE_206 |\
+            CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_206 |\
             CPU_FTR_MMCRA | CPU_FTR_SMT | \
             CPU_FTR_COHERENT_ICACHE | \
             CPU_FTR_PURR | CPU_FTR_SPURR | CPU_FTR_REAL_LE | \
             CPU_FTR_DSCR | CPU_FTR_SAO  | CPU_FTR_ASYM_SMT | \
             CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
-            CPU_FTR_ICSWX | CPU_FTR_CFAR)
+            CPU_FTR_ICSWX | CPU_FTR_CFAR | CPU_FTR_HVMODE)
 #define CPU_FTRS_CELL   (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \
             CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \
             CPU_FTR_ALTIVEC_COMP | CPU_FTR_MMCRA | CPU_FTR_SMT | \
diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index f5dfe3411f64..8057f4f6980f 100644
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -61,19 +61,22 @@
 #define EXC_HV  H
 #define EXC_STD
 
-#define EXCEPTION_PROLOG_1(area)                                        \
+#define __EXCEPTION_PROLOG_1(area, extra, vec)                          \
         GET_PACA(r13);                                                  \
         std     r9,area+EX_R9(r13);     /* save r9 - r12 */             \
         std     r10,area+EX_R10(r13);                                   \
-        std     r11,area+EX_R11(r13);                                   \
-        std     r12,area+EX_R12(r13);                                   \
         BEGIN_FTR_SECTION_NESTED(66);                                   \
         mfspr   r10,SPRN_CFAR;                                          \
         std     r10,area+EX_CFAR(r13);                                  \
         END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66);         \
-        GET_SCRATCH0(r9);                                               \
-        std     r9,area+EX_R13(r13);                                    \
-        mfcr    r9
+        mfcr    r9;                                                     \
+        extra(vec);                                                     \
+        std     r11,area+EX_R11(r13);                                   \
+        std     r12,area+EX_R12(r13);                                   \
+        GET_SCRATCH0(r10);                                              \
+        std     r10,area+EX_R13(r13)
+#define EXCEPTION_PROLOG_1(area, extra, vec)                            \
+        __EXCEPTION_PROLOG_1(area, extra, vec)
 
 #define __EXCEPTION_PROLOG_PSERIES_1(label, h)                          \
         ld      r12,PACAKBASE(r13);     /* get high part of &label */   \
@@ -85,13 +88,65 @@
         mtspr   SPRN_##h##SRR1,r10;                                     \
         h##rfid;                                                        \
         b       .       /* prevent speculative execution */
-#define EXCEPTION_PROLOG_PSERIES_1(label, h) \
+#define EXCEPTION_PROLOG_PSERIES_1(label, h)                            \
         __EXCEPTION_PROLOG_PSERIES_1(label, h)
 
-#define EXCEPTION_PROLOG_PSERIES(area, label, h)                        \
-        EXCEPTION_PROLOG_1(area);                                       \
+#define EXCEPTION_PROLOG_PSERIES(area, label, h, extra, vec)            \
+        EXCEPTION_PROLOG_1(area, extra, vec);                           \
         EXCEPTION_PROLOG_PSERIES_1(label, h);
 
+#define __KVMTEST(n)                                                    \
+        lbz     r10,HSTATE_IN_GUEST(r13);                       \
+        cmpwi   r10,0;                                                  \
+        bne     do_kvm_##n
+
+#define __KVM_HANDLER(area, h, n)                                       \
+do_kvm_##n:                                                             \
+        ld      r10,area+EX_R10(r13);                                   \
+        stw     r9,HSTATE_SCRATCH1(r13);                        \
+        ld      r9,area+EX_R9(r13);                                     \
+        std     r12,HSTATE_SCRATCH0(r13);                       \
+        li      r12,n;                                                  \
+        b       kvmppc_interrupt
+
+#define __KVM_HANDLER_SKIP(area, h, n)                                  \
+do_kvm_##n:                                                             \
+        cmpwi   r10,KVM_GUEST_MODE_SKIP;                                \
+        ld      r10,area+EX_R10(r13);                                   \
+        beq     89f;                                                    \
+        stw     r9,HSTATE_SCRATCH1(r13);                        \
+        ld      r9,area+EX_R9(r13);                                     \
+        std     r12,HSTATE_SCRATCH0(r13);                       \
+        li      r12,n;                                                  \
+        b       kvmppc_interrupt;                                       \
+89:     mtocrf  0x80,r9;                                                \
+        ld      r9,area+EX_R9(r13);                                     \
+        b       kvmppc_skip_##h##interrupt
+
+#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
+#define KVMTEST(n)                      __KVMTEST(n)
+#define KVM_HANDLER(area, h, n)         __KVM_HANDLER(area, h, n)
+#define KVM_HANDLER_SKIP(area, h, n)    __KVM_HANDLER_SKIP(area, h, n)
+
+#else
+#define KVMTEST(n)
+#define KVM_HANDLER(area, h, n)
+#define KVM_HANDLER_SKIP(area, h, n)
+#endif
+
+#ifdef CONFIG_KVM_BOOK3S_PR
+#define KVMTEST_PR(n)                   __KVMTEST(n)
+#define KVM_HANDLER_PR(area, h, n)      __KVM_HANDLER(area, h, n)
+#define KVM_HANDLER_PR_SKIP(area, h, n) __KVM_HANDLER_SKIP(area, h, n)
+
+#else
+#define KVMTEST_PR(n)
+#define KVM_HANDLER_PR(area, h, n)
+#define KVM_HANDLER_PR_SKIP(area, h, n)
+#endif
+
+#define NOTEST(n)
+
 /*
  * The common exception prolog is used for all except a few exceptions
  * such as a segment miss on a kernel address.  We have to be prepared
@@ -164,57 +219,58 @@
         .globl label##_pSeries;                         \
 label##_pSeries:                                        \
         HMT_MEDIUM;                                     \
-        DO_KVM  vec;                                    \
         SET_SCRATCH0(r13);              /* save r13 */          \
-        EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, EXC_STD)
+        EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common,    \
+                                 EXC_STD, KVMTEST_PR, vec)
 
 #define STD_EXCEPTION_HV(loc, vec, label)               \
         . = loc;                                        \
         .globl label##_hv;                              \
 label##_hv:                                             \
         HMT_MEDIUM;                                     \
-        DO_KVM  vec;                                    \
-        SET_SCRATCH0(r13);      /* save r13 */          \
-        EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, EXC_HV)
+        SET_SCRATCH0(r13);      /* save r13 */                  \
+        EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common,    \
+                                 EXC_HV, KVMTEST, vec)
 
-#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h)                     \
-        HMT_MEDIUM;                                                     \
-        DO_KVM  vec;                                                    \
-        SET_SCRATCH0(r13);    /* save r13 */                            \
-        GET_PACA(r13);                                                  \
-        std     r9,PACA_EXGEN+EX_R9(r13);       /* save r9, r10 */      \
-        std     r10,PACA_EXGEN+EX_R10(r13);                             \
+#define __SOFTEN_TEST(h)                                                \
         lbz     r10,PACASOFTIRQEN(r13);                                 \
-        mfcr    r9;                                                     \
         cmpwi   r10,0;                                                  \
-        beq     masked_##h##interrupt;                                  \
-        GET_SCRATCH0(r10);                                              \
-        std     r10,PACA_EXGEN+EX_R13(r13);                             \
-        std     r11,PACA_EXGEN+EX_R11(r13);                             \
-        std     r12,PACA_EXGEN+EX_R12(r13);                             \
-        ld      r12,PACAKBASE(r13);     /* get high part of &label */   \
-        ld      r10,PACAKMSR(r13);      /* get MSR value for kernel */  \
-        mfspr   r11,SPRN_##h##SRR0;     /* save SRR0 */                 \
-        LOAD_HANDLER(r12,label##_common)                                \
-        mtspr   SPRN_##h##SRR0,r12;                                     \
-        mfspr   r12,SPRN_##h##SRR1;     /* and SRR1 */                  \
-        mtspr   SPRN_##h##SRR1,r10;                                     \
-        h##rfid;                                                        \
-        b       .       /* prevent speculative execution */
-#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h)                      \
-        __MASKABLE_EXCEPTION_PSERIES(vec, label, h)
+        beq     masked_##h##interrupt
+#define _SOFTEN_TEST(h) __SOFTEN_TEST(h)
+
+#define SOFTEN_TEST_PR(vec)                                             \
+        KVMTEST_PR(vec);                                                \
+        _SOFTEN_TEST(EXC_STD)
+
+#define SOFTEN_TEST_HV(vec)                                             \
+        KVMTEST(vec);                                                   \
+        _SOFTEN_TEST(EXC_HV)
+
+#define SOFTEN_TEST_HV_201(vec)                                         \
+        KVMTEST(vec);                                                   \
+        _SOFTEN_TEST(EXC_STD)
+
+#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)              \
+        HMT_MEDIUM;                                                     \
+        SET_SCRATCH0(r13);    /* save r13 */                            \
+        __EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec);           \
+        EXCEPTION_PROLOG_PSERIES_1(label##_common, h);
+#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)               \
+        __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)
 
 #define MASKABLE_EXCEPTION_PSERIES(loc, vec, label)                     \
         . = loc;                                                        \
         .globl label##_pSeries;                                         \
 label##_pSeries:                                                        \
-        _MASKABLE_EXCEPTION_PSERIES(vec, label, EXC_STD)
+        _MASKABLE_EXCEPTION_PSERIES(vec, label,                         \
+                                    EXC_STD, SOFTEN_TEST_PR)
 
 #define MASKABLE_EXCEPTION_HV(loc, vec, label)                          \
         . = loc;                                                        \
         .globl label##_hv;                                              \
 label##_hv:                                                             \
-        _MASKABLE_EXCEPTION_PSERIES(vec, label, EXC_HV)
+        _MASKABLE_EXCEPTION_PSERIES(vec, label,                         \
+                                    EXC_HV, SOFTEN_TEST_HV)
 
 #ifdef CONFIG_PPC_ISERIES
 #define DISABLE_INTS                            \
diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h
index fd8201dddd4b..1c324ff55ea8 100644
--- a/arch/powerpc/include/asm/hvcall.h
+++ b/arch/powerpc/include/asm/hvcall.h
@@ -29,6 +29,10 @@
 #define H_LONG_BUSY_ORDER_100_SEC       9905  /* Long busy, hint that 100sec \
                                                  is a good time to retry */
 #define H_LONG_BUSY_END_RANGE           9905  /* End of long busy range */
+
+/* Internal value used in book3s_hv kvm support; not returned to guests */
+#define H_TOO_HARD      9999
+
 #define H_HARDWARE      -1      /* Hardware error */
 #define H_FUNCTION      -2      /* Function not supported */
 #define H_PRIVILEGE     -3      /* Caller not privileged */
@@ -100,6 +104,7 @@
 #define H_PAGE_SET_ACTIVE       H_PAGE_STATE_CHANGE
 #define H_AVPN                  (1UL<<(63-32))  /* An avpn is provided as a sanity test */
 #define H_ANDCOND               (1UL<<(63-33))
+#define H_LOCAL                 (1UL<<(63-35))
 #define H_ICACHE_INVALIDATE     (1UL<<(63-40))  /* icbi, etc.  (ignored for IO pages) */
 #define H_ICACHE_SYNCHRONIZE    (1UL<<(63-41))  /* dcbst, icbi, etc (ignored for IO pages */
 #define H_COALESCE_CAND (1UL<<(63-42))  /* page is a good candidate for coalescing */
diff --git a/arch/powerpc/include/asm/kvm.h b/arch/powerpc/include/asm/kvm.h
index d2ca5ed3877b..a4f6c85431f8 100644
--- a/arch/powerpc/include/asm/kvm.h
+++ b/arch/powerpc/include/asm/kvm.h
@@ -22,6 +22,10 @@
 
 #include <linux/types.h>
 
+/* Select powerpc specific features in <linux/kvm.h> */
+#define __KVM_HAVE_SPAPR_TCE
+#define __KVM_HAVE_PPC_SMT
+
 struct kvm_regs {
         __u64 pc;
         __u64 cr;
@@ -272,4 +276,15 @@ struct kvm_guest_debug_arch {
 #define KVM_INTERRUPT_UNSET     -2U
 #define KVM_INTERRUPT_SET_LEVEL -3U
 
+/* for KVM_CAP_SPAPR_TCE */
+struct kvm_create_spapr_tce {
+        __u64 liobn;
+        __u32 window_size;
+};
+
+/* for KVM_ALLOCATE_RMA */
+struct kvm_allocate_rma {
+        __u64 rma_size;
+};
+
 #endif /* __LINUX_KVM_POWERPC_H */
diff --git a/arch/powerpc/include/asm/kvm_asm.h b/arch/powerpc/include/asm/kvm_asm.h
index 0951b17f4eb5..7b1f0e0fc653 100644
--- a/arch/powerpc/include/asm/kvm_asm.h
+++ b/arch/powerpc/include/asm/kvm_asm.h
@@ -64,8 +64,12 @@
 #define BOOK3S_INTERRUPT_PROGRAM        0x700
 #define BOOK3S_INTERRUPT_FP_UNAVAIL     0x800
 #define BOOK3S_INTERRUPT_DECREMENTER    0x900
+#define BOOK3S_INTERRUPT_HV_DECREMENTER 0x980
 #define BOOK3S_INTERRUPT_SYSCALL        0xc00
 #define BOOK3S_INTERRUPT_TRACE          0xd00
+#define BOOK3S_INTERRUPT_H_DATA_STORAGE 0xe00
+#define BOOK3S_INTERRUPT_H_INST_STORAGE 0xe20
+#define BOOK3S_INTERRUPT_H_EMUL_ASSIST  0xe40
 #define BOOK3S_INTERRUPT_PERFMON        0xf00
 #define BOOK3S_INTERRUPT_ALTIVEC        0xf20
 #define BOOK3S_INTERRUPT_VSX            0xf40
diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h
index d62e703f1214..98da010252a3 100644
--- a/arch/powerpc/include/asm/kvm_book3s.h
+++ b/arch/powerpc/include/asm/kvm_book3s.h
@@ -24,20 +24,6 @@
 #include <linux/kvm_host.h>
 #include <asm/kvm_book3s_asm.h>
 
-struct kvmppc_slb {
-        u64 esid;
-        u64 vsid;
-        u64 orige;
-        u64 origv;
-        bool valid      : 1;
-        bool Ks         : 1;
-        bool Kp         : 1;
-        bool nx         : 1;
-        bool large      : 1;    /* PTEs are 16MB */
-        bool tb         : 1;    /* 1TB segment */
-        bool class      : 1;
-};
-
 struct kvmppc_bat {
         u64 raw;
         u32 bepi;
@@ -67,11 +53,22 @@ struct kvmppc_sid_map {
 #define VSID_POOL_SIZE  (SID_CONTEXTS * 16)
 #endif
 
+struct hpte_cache {
+        struct hlist_node list_pte;
+        struct hlist_node list_pte_long;
+        struct hlist_node list_vpte;
+        struct hlist_node list_vpte_long;
+        struct rcu_head rcu_head;
+        u64 host_va;
+        u64 pfn;
+        ulong slot;
+        struct kvmppc_pte pte;
+};
+
 struct kvmppc_vcpu_book3s {
         struct kvm_vcpu vcpu;
         struct kvmppc_book3s_shadow_vcpu *shadow_vcpu;
         struct kvmppc_sid_map sid_map[SID_MAP_NUM];
-        struct kvmppc_slb slb[64];
         struct {
                 u64 esid;
                 u64 vsid;
@@ -81,7 +78,6 @@ struct kvmppc_vcpu_book3s {
         struct kvmppc_bat dbat[8];
         u64 hid[6];
         u64 gqr[8];
-        int slb_nr;
         u64 sdr1;
         u64 hior;
         u64 msr_mask;
@@ -93,7 +89,13 @@ struct kvmppc_vcpu_book3s {
         u64 vsid_max;
 #endif
         int context_id[SID_CONTEXTS];
-        ulong prog_flags; /* flags to inject when giving a 700 trap */
+
+        struct hlist_head hpte_hash_pte[HPTEG_HASH_NUM_PTE];
+        struct hlist_head hpte_hash_pte_long[HPTEG_HASH_NUM_PTE_LONG];
+        struct hlist_head hpte_hash_vpte[HPTEG_HASH_NUM_VPTE];
+        struct hlist_head hpte_hash_vpte_long[HPTEG_HASH_NUM_VPTE_LONG];
+        int hpte_cache_count;
+        spinlock_t mmu_lock;
 };
 
 #define CONTEXT_HOST            0
@@ -110,8 +112,10 @@ extern void kvmppc_mmu_pte_flush(struct kvm_vcpu *vcpu, ulong ea, ulong ea_mask)
 extern void kvmppc_mmu_pte_vflush(struct kvm_vcpu *vcpu, u64 vp, u64 vp_mask);
 extern void kvmppc_mmu_pte_pflush(struct kvm_vcpu *vcpu, ulong pa_start, ulong pa_end);
 extern void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 new_msr);
+extern void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr);
 extern void kvmppc_mmu_book3s_64_init(struct kvm_vcpu *vcpu);
 extern void kvmppc_mmu_book3s_32_init(struct kvm_vcpu *vcpu);
+extern void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu);
 extern int kvmppc_mmu_map_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte);
 extern int kvmppc_mmu_map_segment(struct kvm_vcpu *vcpu, ulong eaddr);
 extern void kvmppc_mmu_flush_segments(struct kvm_vcpu *vcpu);
@@ -123,19 +127,22 @@ extern int kvmppc_mmu_hpte_init(struct kvm_vcpu *vcpu);
 extern void kvmppc_mmu_invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte);
 extern int kvmppc_mmu_hpte_sysinit(void);
 extern void kvmppc_mmu_hpte_sysexit(void);
+extern int kvmppc_mmu_hv_init(void);
 
 extern int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, bool data);
 extern int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, bool data);
 extern void kvmppc_book3s_queue_irqprio(struct kvm_vcpu *vcpu, unsigned int vec);
+extern void kvmppc_inject_interrupt(struct kvm_vcpu *vcpu, int vec, u64 flags);
 extern void kvmppc_set_bat(struct kvm_vcpu *vcpu, struct kvmppc_bat *bat,
                            bool upper, u32 val);
 extern void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr);
 extern int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu);
 extern pfn_t kvmppc_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn);
 
-extern ulong kvmppc_trampoline_lowmem;
-extern ulong kvmppc_trampoline_enter;
+extern void kvmppc_handler_lowmem_trampoline(void);
+extern void kvmppc_handler_trampoline_enter(void);
 extern void kvmppc_rmcall(ulong srr0, ulong srr1);
+extern void kvmppc_hv_entry_trampoline(void);
 extern void kvmppc_load_up_fpu(void);
 extern void kvmppc_load_up_altivec(void);
 extern void kvmppc_load_up_vsx(void);
@@ -147,15 +154,32 @@ static inline struct kvmppc_vcpu_book3s *to_book3s(struct kvm_vcpu *vcpu)
         return container_of(vcpu, struct kvmppc_vcpu_book3s, vcpu);
 }
 
-static inline ulong dsisr(void)
+extern void kvm_return_point(void);
+
+/* Also add subarch specific defines */
+
+#ifdef CONFIG_KVM_BOOK3S_32_HANDLER
+#include <asm/kvm_book3s_32.h>
+#endif
+#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
+#include <asm/kvm_book3s_64.h>
+#endif
+
+#ifdef CONFIG_KVM_BOOK3S_PR
+
+static inline unsigned long kvmppc_interrupt_offset(struct kvm_vcpu *vcpu)
 {
-        ulong r;
-        asm ( "mfdsisr %0 " : "=r" (r) );
-        return r;
+        return to_book3s(vcpu)->hior;
 }
 
-extern void kvm_return_point(void);
-static inline struct kvmppc_book3s_shadow_vcpu *to_svcpu(struct kvm_vcpu *vcpu);
+static inline void kvmppc_update_int_pending(struct kvm_vcpu *vcpu,
+                        unsigned long pending_now, unsigned long old_pending)
+{
+        if (pending_now)
+                vcpu->arch.shared->int_pending = 1;
+        else if (old_pending)
+                vcpu->arch.shared->int_pending = 0;
+}
 
 static inline void kvmppc_set_gpr(struct kvm_vcpu *vcpu, int num, ulong val)
 {
@@ -244,6 +268,120 @@ static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu)
         return to_svcpu(vcpu)->fault_dar;
 }
 
+static inline bool kvmppc_critical_section(struct kvm_vcpu *vcpu)
+{
+        ulong crit_raw = vcpu->arch.shared->critical;
+        ulong crit_r1 = kvmppc_get_gpr(vcpu, 1);
+        bool crit;
+
+        /* Truncate crit indicators in 32 bit mode */
+        if (!(vcpu->arch.shared->msr & MSR_SF)) {
+                crit_raw &= 0xffffffff;
+                crit_r1 &= 0xffffffff;
+        }
+
+        /* Critical section when crit == r1 */
+        crit = (crit_raw == crit_r1);
+        /* ... and we're in supervisor mode */
+        crit = crit && !(vcpu->arch.shared->msr & MSR_PR);
+
+        return crit;
+}
+#else /* CONFIG_KVM_BOOK3S_PR */
+
+static inline unsigned long kvmppc_interrupt_offset(struct kvm_vcpu *vcpu)
+{
+        return 0;
+}
+
+static inline void kvmppc_update_int_pending(struct kvm_vcpu *vcpu,
+                        unsigned long pending_now, unsigned long old_pending)
+{
+}
+
+static inline void kvmppc_set_gpr(struct kvm_vcpu *vcpu, int num, ulong val)
+{
+        vcpu->arch.gpr[num] = val;
+}
+
+static inline ulong kvmppc_get_gpr(struct kvm_vcpu *vcpu, int num)
+{
+        return vcpu->arch.gpr[num];
+}
+
+static inline void kvmppc_set_cr(struct kvm_vcpu *vcpu, u32 val)
+{
+        vcpu->arch.cr = val;
+}
+
+static inline u32 kvmppc_get_cr(struct kvm_vcpu *vcpu)
+{
+        return vcpu->arch.cr;
+}
+
+static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, u32 val)
+{
+        vcpu->arch.xer = val;
+}
+
+static inline u32 kvmppc_get_xer(struct kvm_vcpu *vcpu)
+{
+        return vcpu->arch.xer;
+}
+
+static inline void kvmppc_set_ctr(struct kvm_vcpu *vcpu, ulong val)
+{
+        vcpu->arch.ctr = val;
+}
+
+static inline ulong kvmppc_get_ctr(struct kvm_vcpu *vcpu)
+{
+        return vcpu->arch.ctr;
+}
+
+static inline void kvmppc_set_lr(struct kvm_vcpu *vcpu, ulong val)
+{
+        vcpu->arch.lr = val;
+}
+
+static inline ulong kvmppc_get_lr(struct kvm_vcpu *vcpu)
+{
+        return vcpu->arch.lr;
+}
+
+static inline void kvmppc_set_pc(struct kvm_vcpu *vcpu, ulong val)
+{
+        vcpu->arch.pc = val;
+}
+
+static inline ulong kvmppc_get_pc(struct kvm_vcpu *vcpu)
+{
+        return vcpu->arch.pc;
+}
+
+static inline u32 kvmppc_get_last_inst(struct kvm_vcpu *vcpu)
+{
+        ulong pc = kvmppc_get_pc(vcpu);
+
+        /* Load the instruction manually if it failed to do so in the
+         * exit path */
+        if (vcpu->arch.last_inst == KVM_INST_FETCH_FAILED)
+                kvmppc_ld(vcpu, &pc, sizeof(u32), &vcpu->arch.last_inst, false);
+
+        return vcpu->arch.last_inst;
+}
+
+static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu)
+{
+        return vcpu->arch.fault_dar;
+}
+
+static inline bool kvmppc_critical_section(struct kvm_vcpu *vcpu)
+{
+        return false;
+}
+#endif
+
 /* Magic register values loaded into r3 and r4 before the 'sc' assembly
  * instruction for the OSI hypercalls */
 #define OSI_SC_MAGIC_R3                 0x113724FA
@@ -251,12 +389,4 @@ static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu)
 
 #define INS_DCBZ                        0x7c0007ec
 
-/* Also add subarch specific defines */
-
-#ifdef CONFIG_PPC_BOOK3S_32
-#include <asm/kvm_book3s_32.h>
-#else
-#include <asm/kvm_book3s_64.h>
-#endif
-
 #endif /* __ASM_KVM_BOOK3S_H__ */
diff --git a/arch/powerpc/include/asm/kvm_book3s_64.h b/arch/powerpc/include/asm/kvm_book3s_64.h
index 4cadd612d575..e43fe42b9875 100644
--- a/arch/powerpc/include/asm/kvm_book3s_64.h
+++ b/arch/powerpc/include/asm/kvm_book3s_64.h
@@ -20,9 +20,13 @@
 #ifndef __ASM_KVM_BOOK3S_64_H__
 #define __ASM_KVM_BOOK3S_64_H__
 
+#ifdef CONFIG_KVM_BOOK3S_PR
 static inline struct kvmppc_book3s_shadow_vcpu *to_svcpu(struct kvm_vcpu *vcpu)
 {
         return &get_paca()->shadow_vcpu;
 }
+#endif
+
+#define SPAPR_TCE_SHIFT         12
 
 #endif /* __ASM_KVM_BOOK3S_64_H__ */
diff --git a/arch/powerpc/include/asm/kvm_book3s_asm.h b/arch/powerpc/include/asm/kvm_book3s_asm.h
index d5a8a3861635..ef7b3688c3b6 100644
--- a/arch/powerpc/include/asm/kvm_book3s_asm.h
+++ b/arch/powerpc/include/asm/kvm_book3s_asm.h
@@ -60,6 +60,36 @@ kvmppc_resume_\intno:
 
 #else  /*__ASSEMBLY__ */
 
+/*
+ * This struct goes in the PACA on 64-bit processors.  It is used
+ * to store host state that needs to be saved when we enter a guest
+ * and restored when we exit, but isn't specific to any particular
+ * guest or vcpu.  It also has some scratch fields used by the guest
+ * exit code.
+ */
+struct kvmppc_host_state {
+        ulong host_r1;
+        ulong host_r2;
+        ulong host_msr;
+        ulong vmhandler;
+        ulong scratch0;
+        ulong scratch1;
+        u8 in_guest;
+
+#ifdef CONFIG_KVM_BOOK3S_64_HV
+        struct kvm_vcpu *kvm_vcpu;
+        struct kvmppc_vcore *kvm_vcore;
+        unsigned long xics_phys;
+        u64 dabr;
+        u64 host_mmcr[3];
+        u32 host_pmc[8];
+        u64 host_purr;
+        u64 host_spurr;
+        u64 host_dscr;
+        u64 dec_expires;
+#endif
+};
+
 struct kvmppc_book3s_shadow_vcpu {
         ulong gpr[14];
         u32 cr;
@@ -73,17 +103,12 @@ struct kvmppc_book3s_shadow_vcpu {
         ulong shadow_srr1;
         ulong fault_dar;
 
-        ulong host_r1;
-        ulong host_r2;
-        ulong handler;
-        ulong scratch0;
-        ulong scratch1;
-        ulong vmhandler;
-        u8 in_guest;
-
 #ifdef CONFIG_PPC_BOOK3S_32
         u32     sr[16];                 /* Guest SRs */
+
+        struct kvmppc_host_state hstate;
 #endif
+
 #ifdef CONFIG_PPC_BOOK3S_64
         u8 slb_max;                     /* highest used guest slb entry */
         struct  {
diff --git a/arch/powerpc/include/asm/kvm_booke.h b/arch/powerpc/include/asm/kvm_booke.h
index 9c9ba3d59b1b..a90e09188777 100644
--- a/arch/powerpc/include/asm/kvm_booke.h
+++ b/arch/powerpc/include/asm/kvm_booke.h
@@ -93,4 +93,8 @@ static inline ulong kvmppc_get_fault_dar(struct kvm_vcpu *vcpu)
         return vcpu->arch.fault_dear;
 }
 
+static inline ulong kvmppc_get_msr(struct kvm_vcpu *vcpu)
+{
+        return vcpu->arch.shared->msr;
+}
 #endif /* __ASM_KVM_BOOKE_H__ */
diff --git a/arch/powerpc/include/asm/kvm_e500.h b/arch/powerpc/include/asm/kvm_e500.h
index 7a2a565f88c4..adbfca9dd100 100644
--- a/arch/powerpc/include/asm/kvm_e500.h
+++ b/arch/powerpc/include/asm/kvm_e500.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2008 Freescale Semiconductor, Inc. All rights reserved.
+ * Copyright (C) 2008-2011 Freescale Semiconductor, Inc. All rights reserved.
  *
  * Author: Yu Liu, <yu.liu@freescale.com>
  *
@@ -29,17 +29,25 @@ struct tlbe{
         u32 mas7;
 };
 
+#define E500_TLB_VALID 1
+#define E500_TLB_DIRTY 2
+
+struct tlbe_priv {
+        pfn_t pfn;
+        unsigned int flags; /* E500_TLB_* */
+};
+
+struct vcpu_id_table;
+
 struct kvmppc_vcpu_e500 {
         /* Unmodified copy of the guest's TLB. */
-        struct tlbe *guest_tlb[E500_TLB_NUM];
-        /* TLB that's actually used when the guest is running. */
-        struct tlbe *shadow_tlb[E500_TLB_NUM];
-        /* Pages which are referenced in the shadow TLB. */
-        struct page **shadow_pages[E500_TLB_NUM];
+        struct tlbe *gtlb_arch[E500_TLB_NUM];
 
-        unsigned int guest_tlb_size[E500_TLB_NUM];
-        unsigned int shadow_tlb_size[E500_TLB_NUM];
-        unsigned int guest_tlb_nv[E500_TLB_NUM];
+        /* KVM internal information associated with each guest TLB entry */
+        struct tlbe_priv *gtlb_priv[E500_TLB_NUM];
+
+        unsigned int gtlb_size[E500_TLB_NUM];
+        unsigned int gtlb_nv[E500_TLB_NUM];
 
         u32 host_pid[E500_PID_NUM];
         u32 pid[E500_PID_NUM];
@@ -53,6 +61,10 @@ struct kvmppc_vcpu_e500 {
         u32 mas5;
         u32 mas6;
         u32 mas7;
+
+        /* vcpu id table */
+        struct vcpu_id_table *idt;
+
         u32 l1csr0;
         u32 l1csr1;
         u32 hid0;
diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index 186f150b9b89..cc22b282d755 100644
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -25,15 +25,23 @@
 #include <linux/interrupt.h>
 #include <linux/types.h>
 #include <linux/kvm_types.h>
+#include <linux/threads.h>
+#include <linux/spinlock.h>
 #include <linux/kvm_para.h>
+#include <linux/list.h>
+#include <linux/atomic.h>
 #include <asm/kvm_asm.h>
+#include <asm/processor.h>
 
-#define KVM_MAX_VCPUS 1
+#define KVM_MAX_VCPUS           NR_CPUS
+#define KVM_MAX_VCORES          NR_CPUS
 #define KVM_MEMORY_SLOTS 32
 /* memory slots that does not exposed to userspace */
 #define KVM_PRIVATE_MEM_SLOTS 4
 
+#ifdef CONFIG_KVM_MMIO
 #define KVM_COALESCED_MMIO_PAGE_OFFSET 1
+#endif
 
 /* We don't currently support large pages. */
 #define KVM_HPAGE_GFN_SHIFT(x)  0
@@ -57,6 +65,10 @@ struct kvm;
 struct kvm_run;
 struct kvm_vcpu;
 
+struct lppaca;
+struct slb_shadow;
+struct dtl;
+
 struct kvm_vm_stat {
         u32 remote_tlb_flush;
 };
@@ -133,9 +145,74 @@ struct kvmppc_exit_timing {
         };
 };
 
+struct kvmppc_pginfo {
+        unsigned long pfn;
+        atomic_t refcnt;
+};
+
+struct kvmppc_spapr_tce_table {
+        struct list_head list;
+        struct kvm *kvm;
+        u64 liobn;
+        u32 window_size;
+        struct page *pages[0];
+};
+
+struct kvmppc_rma_info {
+        void            *base_virt;
+        unsigned long    base_pfn;
+        unsigned long    npages;
+        struct list_head list;
+        atomic_t         use_count;
+};
+
 struct kvm_arch {
+#ifdef CONFIG_KVM_BOOK3S_64_HV
+        unsigned long hpt_virt;
+        unsigned long ram_npages;
+        unsigned long ram_psize;
+        unsigned long ram_porder;
+        struct kvmppc_pginfo *ram_pginfo;
+        unsigned int lpid;
+        unsigned int host_lpid;
+        unsigned long host_lpcr;
+        unsigned long sdr1;
+        unsigned long host_sdr1;
+        int tlbie_lock;
+        int n_rma_pages;
+        unsigned long lpcr;
+        unsigned long rmor;
+        struct kvmppc_rma_info *rma;
+        struct list_head spapr_tce_tables;
+        unsigned short last_vcpu[NR_CPUS];
+        struct kvmppc_vcore *vcores[KVM_MAX_VCORES];
+#endif /* CONFIG_KVM_BOOK3S_64_HV */
 };
 
+/*
+ * Struct for a virtual core.
+ * Note: entry_exit_count combines an entry count in the bottom 8 bits
+ * and an exit count in the next 8 bits.  This is so that we can
+ * atomically increment the entry count iff the exit count is 0
+ * without taking the lock.
+ */
+struct kvmppc_vcore {
+        int n_runnable;
+        int n_blocked;
+        int num_threads;
+        int entry_exit_count;
+        int n_woken;
+        int nap_count;
+        u16 pcpu;
+        u8 vcore_running;
+        u8 in_guest;
+        struct list_head runnable_threads;
+        spinlock_t lock;
+};
+
+#define VCORE_ENTRY_COUNT(vc)   ((vc)->entry_exit_count & 0xff)
+#define VCORE_EXIT_COUNT(vc)    ((vc)->entry_exit_count >> 8)
+
 struct kvmppc_pte {
         ulong eaddr;
         u64 vpage;
@@ -163,16 +240,18 @@ struct kvmppc_mmu {
         bool (*is_dcbz32)(struct kvm_vcpu *vcpu);
 };
 
-struct hpte_cache {
-        struct hlist_node list_pte;
-        struct hlist_node list_pte_long;
-        struct hlist_node list_vpte;
-        struct hlist_node list_vpte_long;
-        struct rcu_head rcu_head;
-        u64 host_va;
-        u64 pfn;
-        ulong slot;
-        struct kvmppc_pte pte;
+struct kvmppc_slb {
+        u64 esid;
+        u64 vsid;
+        u64 orige;
+        u64 origv;
+        bool valid      : 1;
+        bool Ks         : 1;
+        bool Kp         : 1;
+        bool nx         : 1;
+        bool large      : 1;    /* PTEs are 16MB */
+        bool tb         : 1;    /* 1TB segment */
+        bool class      : 1;
 };
 
 struct kvm_vcpu_arch {
@@ -187,6 +266,9 @@ struct kvm_vcpu_arch {
         ulong highmem_handler;
         ulong rmcall;
         ulong host_paca_phys;
+        struct kvmppc_slb slb[64];
+        int slb_max;            /* 1 + index of last valid entry in slb[] */
+        int slb_nr;             /* total number of entries in SLB */
         struct kvmppc_mmu mmu;
 #endif
 
@@ -195,13 +277,19 @@ struct kvm_vcpu_arch {
         u64 fpr[32];
         u64 fpscr;
 
+#ifdef CONFIG_SPE
+        ulong evr[32];
+        ulong spefscr;
+        ulong host_spefscr;
+        u64 acc;
+#endif
 #ifdef CONFIG_ALTIVEC
         vector128 vr[32];
         vector128 vscr;
 #endif
 
 #ifdef CONFIG_VSX
-        u64 vsr[32];
+        u64 vsr[64];
 #endif
 
 #ifdef CONFIG_PPC_BOOK3S
@@ -209,22 +297,27 @@ struct kvm_vcpu_arch {
         u32 qpr[32];
 #endif
 
-#ifdef CONFIG_BOOKE
         ulong pc;
         ulong ctr;
         ulong lr;
 
         ulong xer;
         u32 cr;
-#endif
 
 #ifdef CONFIG_PPC_BOOK3S
-        ulong shadow_msr;
         ulong hflags;
         ulong guest_owned_ext;
+        ulong purr;
+        ulong spurr;
+        ulong dscr;
+        ulong amr;
+        ulong uamor;
+        u32 ctrl;
+        ulong dabr;
 #endif
         u32 vrsave; /* also USPRG0 */
         u32 mmucr;
+        ulong shadow_msr;
         ulong sprg4;
         ulong sprg5;
         ulong sprg6;
@@ -249,6 +342,7 @@ struct kvm_vcpu_arch {
         u32 pvr;
 
         u32 shadow_pid;
+        u32 shadow_pid1;
         u32 pid;
         u32 swap_pid;
 
@@ -258,6 +352,9 @@ struct kvm_vcpu_arch {
         u32 dbcr1;
         u32 dbsr;
 
+        u64 mmcr[3];
+        u32 pmc[8];
+
 #ifdef CONFIG_KVM_EXIT_TIMING
         struct mutex exit_timing_lock;
         struct kvmppc_exit_timing timing_exit;
@@ -272,8 +369,12 @@ struct kvm_vcpu_arch {
         struct dentry *debugfs_exit_timing;
 #endif
 
+#ifdef CONFIG_PPC_BOOK3S
+        ulong fault_dar;
+        u32 fault_dsisr;
+#endif
+
 #ifdef CONFIG_BOOKE
-        u32 last_inst;
         ulong fault_dear;
         ulong fault_esr;
         ulong queued_dear;
@@ -288,25 +389,47 @@ struct kvm_vcpu_arch {
         u8 dcr_is_write;
         u8 osi_needed;
         u8 osi_enabled;
+        u8 hcall_needed;
 
         u32 cpr0_cfgaddr; /* holds the last set cpr0_cfgaddr */
 
         struct hrtimer dec_timer;
         struct tasklet_struct tasklet;
         u64 dec_jiffies;
+        u64 dec_expires;
         unsigned long pending_exceptions;
+        u16 last_cpu;
+        u8 ceded;
+        u8 prodded;
+        u32 last_inst;
+
+        struct lppaca *vpa;
+        struct slb_shadow *slb_shadow;
+        struct dtl *dtl;
+        struct dtl *dtl_end;
+
+        struct kvmppc_vcore *vcore;
+        int ret;
+        int trap;
+        int state;
+        int ptid;
+        wait_queue_head_t cpu_run;
+
         struct kvm_vcpu_arch_shared *shared;
         unsigned long magic_page_pa; /* phys addr to map the magic page to */
         unsigned long magic_page_ea; /* effect. addr to map the magic page to */
 
-#ifdef CONFIG_PPC_BOOK3S
-        struct hlist_head hpte_hash_pte[HPTEG_HASH_NUM_PTE];
-        struct hlist_head hpte_hash_pte_long[HPTEG_HASH_NUM_PTE_LONG];
-        struct hlist_head hpte_hash_vpte[HPTEG_HASH_NUM_VPTE];
-        struct hlist_head hpte_hash_vpte_long[HPTEG_HASH_NUM_VPTE_LONG];
-        int hpte_cache_count;
-        spinlock_t mmu_lock;
+#ifdef CONFIG_KVM_BOOK3S_64_HV
+        struct kvm_vcpu_arch_shared shregs;
+
+        struct list_head run_list;
+        struct task_struct *run_task;
+        struct kvm_run *kvm_run;
 #endif
 };
 
+#define KVMPPC_VCPU_BUSY_IN_HOST        0
+#define KVMPPC_VCPU_BLOCKED             1
+#define KVMPPC_VCPU_RUNNABLE            2
+
 #endif /* __POWERPC_KVM_HOST_H__ */
diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h
index 9345238edecf..d121f49d62b8 100644
--- a/arch/powerpc/include/asm/kvm_ppc.h
+++ b/arch/powerpc/include/asm/kvm_ppc.h
@@ -33,6 +33,9 @@
 #else
 #include <asm/kvm_booke.h>
 #endif
+#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
+#include <asm/paca.h>
+#endif
 
 enum emulation_result {
         EMULATE_DONE,         /* no further processing */
@@ -42,6 +45,7 @@ enum emulation_result {
         EMULATE_AGAIN,        /* something went wrong. go again */
 };
 
+extern int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
 extern int __kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
 extern char kvmppc_handlers_start[];
 extern unsigned long kvmppc_handler_len;
@@ -109,6 +113,27 @@ extern void kvmppc_booke_exit(void);
 
 extern void kvmppc_core_destroy_mmu(struct kvm_vcpu *vcpu);
 extern int kvmppc_kvm_pv(struct kvm_vcpu *vcpu);
+extern void kvmppc_map_magic(struct kvm_vcpu *vcpu);
+
+extern long kvmppc_alloc_hpt(struct kvm *kvm);
+extern void kvmppc_free_hpt(struct kvm *kvm);
+extern long kvmppc_prepare_vrma(struct kvm *kvm,
+                                struct kvm_userspace_memory_region *mem);
+extern void kvmppc_map_vrma(struct kvm *kvm,
+                            struct kvm_userspace_memory_region *mem);
+extern int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu);
+extern long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
+                                struct kvm_create_spapr_tce *args);
+extern long kvm_vm_ioctl_allocate_rma(struct kvm *kvm,
+                                struct kvm_allocate_rma *rma);
+extern struct kvmppc_rma_info *kvm_alloc_rma(void);
+extern void kvm_release_rma(struct kvmppc_rma_info *ri);
+extern int kvmppc_core_init_vm(struct kvm *kvm);
+extern void kvmppc_core_destroy_vm(struct kvm *kvm);
+extern int kvmppc_core_prepare_memory_region(struct kvm *kvm,
+                                struct kvm_userspace_memory_region *mem);
+extern void kvmppc_core_commit_memory_region(struct kvm *kvm,
+                                struct kvm_userspace_memory_region *mem);
 
 /*
  * Cuts out inst bits with ordering according to spec.
@@ -151,4 +176,20 @@ int kvmppc_set_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs);
 
 void kvmppc_set_pid(struct kvm_vcpu *vcpu, u32 pid);
 
+#ifdef CONFIG_KVM_BOOK3S_64_HV
+static inline void kvmppc_set_xics_phys(int cpu, unsigned long addr)
+{
+        paca[cpu].kvm_hstate.xics_phys = addr;
+}
+
+extern void kvm_rma_init(void);
+
+#else
+static inline void kvmppc_set_xics_phys(int cpu, unsigned long addr)
+{}
+
+static inline void kvm_rma_init(void)
+{}
+#endif
+
 #endif /* __POWERPC_KVM_PPC_H__ */
diff --git a/arch/powerpc/include/asm/mmu-hash64.h b/arch/powerpc/include/asm/mmu-hash64.h
index d865bd909c7d..b445e0af4c2b 100644
--- a/arch/powerpc/include/asm/mmu-hash64.h
+++ b/arch/powerpc/include/asm/mmu-hash64.h
@@ -90,13 +90,19 @@ extern char initial_stab[];
 
 #define HPTE_R_PP0              ASM_CONST(0x8000000000000000)
 #define HPTE_R_TS               ASM_CONST(0x4000000000000000)
+#define HPTE_R_KEY_HI           ASM_CONST(0x3000000000000000)
 #define HPTE_R_RPN_SHIFT        12
-#define HPTE_R_RPN              ASM_CONST(0x3ffffffffffff000)
-#define HPTE_R_FLAGS            ASM_CONST(0x00000000000003ff)
+#define HPTE_R_RPN              ASM_CONST(0x0ffffffffffff000)
 #define HPTE_R_PP               ASM_CONST(0x0000000000000003)
 #define HPTE_R_N                ASM_CONST(0x0000000000000004)
+#define HPTE_R_G                ASM_CONST(0x0000000000000008)
+#define HPTE_R_M                ASM_CONST(0x0000000000000010)
+#define HPTE_R_I                ASM_CONST(0x0000000000000020)
+#define HPTE_R_W                ASM_CONST(0x0000000000000040)
+#define HPTE_R_WIMG             ASM_CONST(0x0000000000000078)
 #define HPTE_R_C                ASM_CONST(0x0000000000000080)
 #define HPTE_R_R                ASM_CONST(0x0000000000000100)
+#define HPTE_R_KEY_LO           ASM_CONST(0x0000000000000e00)
 
 #define HPTE_V_1TB_SEG          ASM_CONST(0x4000000000000000)
 #define HPTE_V_VRMA_MASK        ASM_CONST(0x4001ffffff000000)
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 74126765106a..a6da12859959 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -147,9 +147,12 @@ struct paca_struct {
         struct dtl_entry *dtl_curr;     /* pointer corresponding to dtl_ridx */
 
 #ifdef CONFIG_KVM_BOOK3S_HANDLER
+#ifdef CONFIG_KVM_BOOK3S_PR
         /* We use this to store guest state in */
         struct kvmppc_book3s_shadow_vcpu shadow_vcpu;
 #endif
+        struct kvmppc_host_state kvm_hstate;
+#endif
 };
 
 extern struct paca_struct *paca;
diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h
index 1b422381fc16..368f72f79808 100644
--- a/arch/powerpc/include/asm/ppc_asm.h
+++ b/arch/powerpc/include/asm/ppc_asm.h
@@ -150,18 +150,22 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
 #define REST_16VSRSU(n,b,base)  REST_8VSRSU(n,b,base); REST_8VSRSU(n+8,b,base)
 #define REST_32VSRSU(n,b,base)  REST_16VSRSU(n,b,base); REST_16VSRSU(n+16,b,base)
 
-#define SAVE_EVR(n,s,base)      evmergehi s,s,n; stw s,THREAD_EVR0+4*(n)(base)
-#define SAVE_2EVRS(n,s,base)    SAVE_EVR(n,s,base); SAVE_EVR(n+1,s,base)
-#define SAVE_4EVRS(n,s,base)    SAVE_2EVRS(n,s,base); SAVE_2EVRS(n+2,s,base)
-#define SAVE_8EVRS(n,s,base)    SAVE_4EVRS(n,s,base); SAVE_4EVRS(n+4,s,base)
-#define SAVE_16EVRS(n,s,base)   SAVE_8EVRS(n,s,base); SAVE_8EVRS(n+8,s,base)
-#define SAVE_32EVRS(n,s,base)   SAVE_16EVRS(n,s,base); SAVE_16EVRS(n+16,s,base)
-#define REST_EVR(n,s,base)      lwz s,THREAD_EVR0+4*(n)(base); evmergelo n,s,n
-#define REST_2EVRS(n,s,base)    REST_EVR(n,s,base); REST_EVR(n+1,s,base)
-#define REST_4EVRS(n,s,base)    REST_2EVRS(n,s,base); REST_2EVRS(n+2,s,base)
-#define REST_8EVRS(n,s,base)    REST_4EVRS(n,s,base); REST_4EVRS(n+4,s,base)
-#define REST_16EVRS(n,s,base)   REST_8EVRS(n,s,base); REST_8EVRS(n+8,s,base)
-#define REST_32EVRS(n,s,base)   REST_16EVRS(n,s,base); REST_16EVRS(n+16,s,base)
+/*
+ * b = base register for addressing, o = base offset from register of 1st EVR
+ * n = first EVR, s = scratch
+ */
+#define SAVE_EVR(n,s,b,o)       evmergehi s,s,n; stw s,o+4*(n)(b)
+#define SAVE_2EVRS(n,s,b,o)     SAVE_EVR(n,s,b,o); SAVE_EVR(n+1,s,b,o)
+#define SAVE_4EVRS(n,s,b,o)     SAVE_2EVRS(n,s,b,o); SAVE_2EVRS(n+2,s,b,o)
+#define SAVE_8EVRS(n,s,b,o)     SAVE_4EVRS(n,s,b,o); SAVE_4EVRS(n+4,s,b,o)
+#define SAVE_16EVRS(n,s,b,o)    SAVE_8EVRS(n,s,b,o); SAVE_8EVRS(n+8,s,b,o)
+#define SAVE_32EVRS(n,s,b,o)    SAVE_16EVRS(n,s,b,o); SAVE_16EVRS(n+16,s,b,o)
+#define REST_EVR(n,s,b,o)       lwz s,o+4*(n)(b); evmergelo n,s,n
+#define REST_2EVRS(n,s,b,o)     REST_EVR(n,s,b,o); REST_EVR(n+1,s,b,o)
+#define REST_4EVRS(n,s,b,o)     REST_2EVRS(n,s,b,o); REST_2EVRS(n+2,s,b,o)
+#define REST_8EVRS(n,s,b,o)     REST_4EVRS(n,s,b,o); REST_4EVRS(n+4,s,b,o)
+#define REST_16EVRS(n,s,b,o)    REST_8EVRS(n,s,b,o); REST_8EVRS(n+8,s,b,o)
+#define REST_32EVRS(n,s,b,o)    REST_16EVRS(n,s,b,o); REST_16EVRS(n+16,s,b,o)
 
 /* Macros to adjust thread priority for hardware multithreading */
 #define HMT_VERY_LOW    or      31,31,31        # very low priority
diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h
index c5cae0dd176c..ddbe57ae8584 100644
--- a/arch/powerpc/include/asm/reg.h
+++ b/arch/powerpc/include/asm/reg.h
@@ -189,6 +189,9 @@
 #define SPRN_CTR        0x009   /* Count Register */
 #define SPRN_DSCR       0x11
 #define SPRN_CFAR       0x1c    /* Come From Address Register */
+#define SPRN_AMR        0x1d    /* Authority Mask Register */
+#define SPRN_UAMOR      0x9d    /* User Authority Mask Override Register */
+#define SPRN_AMOR       0x15d   /* Authority Mask Override Register */
 #define SPRN_ACOP       0x1F    /* Available Coprocessor Register */
 #define SPRN_CTRLF      0x088
 #define SPRN_CTRLT      0x098
@@ -232,22 +235,28 @@
 #define   LPCR_VPM0     (1ul << (63-0))
 #define   LPCR_VPM1     (1ul << (63-1))
 #define   LPCR_ISL      (1ul << (63-2))
+#define   LPCR_VC_SH    (63-2)
 #define   LPCR_DPFD_SH  (63-11)
 #define   LPCR_VRMA_L   (1ul << (63-12))
 #define   LPCR_VRMA_LP0 (1ul << (63-15))
 #define   LPCR_VRMA_LP1 (1ul << (63-16))
+#define   LPCR_VRMASD_SH (63-16)
 #define   LPCR_RMLS    0x1C000000      /* impl dependent rmo limit sel */
+#define   LPCR_RMLS_SH  (63-37)
 #define   LPCR_ILE     0x02000000      /* !HV irqs set MSR:LE */
 #define   LPCR_PECE     0x00007000      /* powersave exit cause enable */
 #define     LPCR_PECE0  0x00004000      /* ext. exceptions can cause exit */
 #define     LPCR_PECE1  0x00002000      /* decrementer can cause exit */
 #define     LPCR_PECE2  0x00001000      /* machine check etc can cause exit */
 #define   LPCR_MER      0x00000800      /* Mediated External Exception */
+#define   LPCR_LPES    0x0000000c
 #define   LPCR_LPES0   0x00000008      /* LPAR Env selector 0 */
 #define   LPCR_LPES1   0x00000004      /* LPAR Env selector 1 */
+#define   LPCR_LPES_SH  2
 #define   LPCR_RMI     0x00000002      /* real mode is cache inhibit */
 #define   LPCR_HDICE   0x00000001      /* Hyp Decr enable (HV,PR,EE) */
 #define SPRN_LPID       0x13F   /* Logical Partition Identifier */
+#define   LPID_RSVD     0x3ff           /* Reserved LPID for partn switching */
 #define SPRN_HMER       0x150   /* Hardware m? error recovery */
 #define SPRN_HMEER      0x151   /* Hardware m? enable error recovery */
 #define SPRN_HEIR       0x153   /* Hypervisor Emulated Instruction Register */
@@ -298,6 +307,7 @@
 #define SPRN_HASH1      0x3D2           /* Primary Hash Address Register */
 #define SPRN_HASH2      0x3D3           /* Secondary Hash Address Resgister */
 #define SPRN_HID0       0x3F0           /* Hardware Implementation Register 0 */
+#define HID0_HDICE_SH   (63 - 23)       /* 970 HDEC interrupt enable */
 #define HID0_EMCP       (1<<31)         /* Enable Machine Check pin */
 #define HID0_EBA        (1<<29)         /* Enable Bus Address Parity */
 #define HID0_EBD        (1<<28)         /* Enable Bus Data Parity */
@@ -353,6 +363,13 @@
 #define SPRN_IABR2      0x3FA           /* 83xx */
 #define SPRN_IBCR       0x135           /* 83xx Insn Breakpoint Control Reg */
 #define SPRN_HID4       0x3F4           /* 970 HID4 */
+#define  HID4_LPES0      (1ul << (63-0)) /* LPAR env. sel. bit 0 */
+#define  HID4_RMLS2_SH   (63 - 2)       /* Real mode limit bottom 2 bits */
+#define  HID4_LPID5_SH   (63 - 6)       /* partition ID bottom 4 bits */
+#define  HID4_RMOR_SH    (63 - 22)      /* real mode offset (16 bits) */
+#define  HID4_LPES1      (1 << (63-57)) /* LPAR env. sel. bit 1 */
+#define  HID4_RMLS0_SH   (63 - 58)      /* Real mode limit top bit */
+#define  HID4_LPID1_SH   0              /* partition ID top 2 bits */
 #define SPRN_HID4_GEKKO 0x3F3           /* Gekko HID4 */
 #define SPRN_HID5       0x3F6           /* 970 HID5 */
 #define SPRN_HID6       0x3F9   /* BE HID 6 */
@@ -802,28 +819,28 @@
         mfspr   rX,SPRN_SPRG_PACA;                      \
         FTR_SECTION_ELSE_NESTED(66);                    \
         mfspr   rX,SPRN_SPRG_HPACA;                     \
-        ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE_206, 66)
+        ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE, 66)
 
 #define SET_PACA(rX)                                    \
         BEGIN_FTR_SECTION_NESTED(66);                   \
         mtspr   SPRN_SPRG_PACA,rX;                      \
         FTR_SECTION_ELSE_NESTED(66);                    \
         mtspr   SPRN_SPRG_HPACA,rX;                     \
-        ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE_206, 66)
+        ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE, 66)
 
 #define GET_SCRATCH0(rX)                                \
         BEGIN_FTR_SECTION_NESTED(66);                   \
         mfspr   rX,SPRN_SPRG_SCRATCH0;                  \
         FTR_SECTION_ELSE_NESTED(66);                    \
         mfspr   rX,SPRN_SPRG_HSCRATCH0;                 \
-        ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE_206, 66)
+        ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE, 66)
 
 #define SET_SCRATCH0(rX)                                \
         BEGIN_FTR_SECTION_NESTED(66);                   \
         mtspr   SPRN_SPRG_SCRATCH0,rX;                  \
         FTR_SECTION_ELSE_NESTED(66);                    \
         mtspr   SPRN_SPRG_HSCRATCH0,rX;                 \
-        ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE_206, 66)
+        ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_HVMODE, 66)
 
 #else /* CONFIG_PPC_BOOK3S_64 */
 #define GET_SCRATCH0(rX)        mfspr   rX,SPRN_SPRG_SCRATCH0
diff --git a/arch/powerpc/include/asm/reg_booke.h b/arch/powerpc/include/asm/reg_booke.h
index 0f0ad9fa01c1..9ec0b39f9ddc 100644
--- a/arch/powerpc/include/asm/reg_booke.h
+++ b/arch/powerpc/include/asm/reg_booke.h
@@ -318,6 +318,7 @@
 #define ESR_ILK         0x00100000      /* Instr. Cache Locking */
 #define ESR_PUO         0x00040000      /* Unimplemented Operation exception */
 #define ESR_BO          0x00020000      /* Byte Ordering */
+#define ESR_SPV         0x00000080      /* Signal Processing operation */
 
 /* Bit definitions related to the DBCR0. */
 #if defined(CONFIG_40x)