1 files changed, 196 insertions, 33 deletions

diff --git a/include/asm-x86/paravirt.h b/include/asm-x86/paravirt.h
index 0f13b945e240..695ce9383f52 100644
--- a/include/asm-x86/paravirt.h
+++ b/include/asm-x86/paravirt.h
@@ -84,7 +84,7 @@ struct pv_time_ops {
         int (*set_wallclock)(unsigned long);
 
         unsigned long long (*sched_clock)(void);
-        unsigned long (*get_cpu_khz)(void);
+        unsigned long (*get_tsc_khz)(void);
 };
 
 struct pv_cpu_ops {
@@ -115,6 +115,9 @@ struct pv_cpu_ops {
         void (*set_ldt)(const void *desc, unsigned entries);
         unsigned long (*store_tr)(void);
         void (*load_tls)(struct thread_struct *t, unsigned int cpu);
+#ifdef CONFIG_X86_64
+        void (*load_gs_index)(unsigned int idx);
+#endif
         void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum,
                                 const void *desc);
         void (*write_gdt_entry)(struct desc_struct *,
@@ -141,8 +144,32 @@ struct pv_cpu_ops {
         u64 (*read_pmc)(int counter);
         unsigned long long (*read_tscp)(unsigned int *aux);
 
-        /* These two are jmp to, not actually called. */
-        void (*irq_enable_syscall_ret)(void);
+        /*
+         * Atomically enable interrupts and return to userspace.  This
+         * is only ever used to return to 32-bit processes; in a
+         * 64-bit kernel, it's used for 32-on-64 compat processes, but
+         * never native 64-bit processes.  (Jump, not call.)
+         */
+        void (*irq_enable_sysexit)(void);
+
+        /*
+         * Switch to usermode gs and return to 64-bit usermode using
+         * sysret.  Only used in 64-bit kernels to return to 64-bit
+         * processes.  Usermode register state, including %rsp, must
+         * already be restored.
+         */
+        void (*usergs_sysret64)(void);
+
+        /*
+         * Switch to usermode gs and return to 32-bit usermode using
+         * sysret.  Used to return to 32-on-64 compat processes.
+         * Other usermode register state, including %esp, must already
+         * be restored.
+         */
+        void (*usergs_sysret32)(void);
+
+        /* Normal iret.  Jump to this with the standard iret stack
+           frame set up. */
         void (*iret)(void);
 
         void (*swapgs)(void);
@@ -165,6 +192,10 @@ struct pv_irq_ops {
         void (*irq_enable)(void);
         void (*safe_halt)(void);
         void (*halt)(void);
+
+#ifdef CONFIG_X86_64
+        void (*adjust_exception_frame)(void);
+#endif
 };
 
 struct pv_apic_ops {
@@ -174,7 +205,6 @@ struct pv_apic_ops {
          * these shouldn't be in this interface.
          */
         void (*apic_write)(unsigned long reg, u32 v);
-        void (*apic_write_atomic)(unsigned long reg, u32 v);
         u32 (*apic_read)(unsigned long reg);
         void (*setup_boot_clock)(void);
         void (*setup_secondary_clock)(void);
@@ -219,7 +249,14 @@ struct pv_mmu_ops {
         void (*flush_tlb_others)(const cpumask_t *cpus, struct mm_struct *mm,
                                  unsigned long va);
 
-        /* Hooks for allocating/releasing pagetable pages */
+        /* Hooks for allocating and freeing a pagetable top-level */
+        int  (*pgd_alloc)(struct mm_struct *mm);
+        void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
+
+        /*
+         * Hooks for allocating/releasing pagetable pages when they're
+         * attached to a pagetable
+         */
         void (*alloc_pte)(struct mm_struct *mm, u32 pfn);
         void (*alloc_pmd)(struct mm_struct *mm, u32 pfn);
         void (*alloc_pmd_clone)(u32 pfn, u32 clonepfn, u32 start, u32 count);
@@ -238,7 +275,13 @@ struct pv_mmu_ops {
         void (*pte_update_defer)(struct mm_struct *mm,
                                  unsigned long addr, pte_t *ptep);
 
+        pte_t (*ptep_modify_prot_start)(struct mm_struct *mm, unsigned long addr,
+                                        pte_t *ptep);
+        void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr,
+                                        pte_t *ptep, pte_t pte);
+
         pteval_t (*pte_val)(pte_t);
+        pteval_t (*pte_flags)(pte_t);
         pte_t (*make_pte)(pteval_t pte);
 
         pgdval_t (*pgd_val)(pgd_t);
@@ -273,6 +316,13 @@ struct pv_mmu_ops {
 #endif
 
         struct pv_lazy_ops lazy_mode;
+
+        /* dom0 ops */
+
+        /* Sometimes the physical address is a pfn, and sometimes its
+           an mfn.  We can tell which is which from the index. */
+        void (*set_fixmap)(unsigned /* enum fixed_addresses */ idx,
+                           unsigned long phys, pgprot_t flags);
 };
 
 /* This contains all the paravirt structures: we get a convenient
@@ -439,10 +489,17 @@ int paravirt_disable_iospace(void);
 #define VEXTRA_CLOBBERS  , "rax", "r8", "r9", "r10", "r11"
 #endif
 
+#ifdef CONFIG_PARAVIRT_DEBUG
+#define PVOP_TEST_NULL(op)      BUG_ON(op == NULL)
+#else
+#define PVOP_TEST_NULL(op)      ((void)op)
+#endif
+
 #define __PVOP_CALL(rettype, op, pre, post, ...)                        \
         ({                                                              \
                 rettype __ret;                                          \
                 PVOP_CALL_ARGS;                                 \
+                PVOP_TEST_NULL(op);                                     \
                 /* This is 32-bit specific, but is okay in 64-bit */    \
                 /* since this condition will never hold */              \
                 if (sizeof(rettype) > sizeof(unsigned long)) {          \
@@ -471,6 +528,7 @@ int paravirt_disable_iospace(void);
 #define __PVOP_VCALL(op, pre, post, ...)                                \
         ({                                                              \
                 PVOP_VCALL_ARGS;                                        \
+                PVOP_TEST_NULL(op);                                     \
                 asm volatile(pre                                        \
                              paravirt_alt(PARAVIRT_CALL)                \
                              post                                       \
@@ -720,7 +778,7 @@ static inline unsigned long long paravirt_sched_clock(void)
 {
         return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
 }
-#define calculate_cpu_khz() (pv_time_ops.get_cpu_khz())
+#define calibrate_tsc() (pv_time_ops.get_tsc_khz())
 
 static inline unsigned long long paravirt_read_pmc(int counter)
 {
@@ -789,6 +847,13 @@ static inline void load_TLS(struct thread_struct *t, unsigned cpu)
         PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
 }
 
+#ifdef CONFIG_X86_64
+static inline void load_gs_index(unsigned int gs)
+{
+        PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
+}
+#endif
+
 static inline void write_ldt_entry(struct desc_struct *dt, int entry,
                                    const void *desc)
 {
@@ -830,11 +895,6 @@ static inline void apic_write(unsigned long reg, u32 v)
         PVOP_VCALL2(pv_apic_ops.apic_write, reg, v);
 }
 
-static inline void apic_write_atomic(unsigned long reg, u32 v)
-{
-        PVOP_VCALL2(pv_apic_ops.apic_write_atomic, reg, v);
-}
-
 static inline u32 apic_read(unsigned long reg)
 {
         return PVOP_CALL1(unsigned long, pv_apic_ops.apic_read, reg);
@@ -912,6 +972,16 @@ static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
         PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, &cpumask, mm, va);
 }
 
+static inline int paravirt_pgd_alloc(struct mm_struct *mm)
+{
+        return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
+}
+
+static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
+{
+        PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
+}
+
 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned pfn)
 {
         PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
@@ -996,6 +1066,20 @@ static inline pteval_t pte_val(pte_t pte)
         return ret;
 }
 
+static inline pteval_t pte_flags(pte_t pte)
+{
+        pteval_t ret;
+
+        if (sizeof(pteval_t) > sizeof(long))
+                ret = PVOP_CALL2(pteval_t, pv_mmu_ops.pte_flags,
+                                 pte.pte, (u64)pte.pte >> 32);
+        else
+                ret = PVOP_CALL1(pteval_t, pv_mmu_ops.pte_flags,
+                                 pte.pte);
+
+        return ret;
+}
+
 static inline pgd_t __pgd(pgdval_t val)
 {
         pgdval_t ret;
@@ -1024,6 +1108,29 @@ static inline pgdval_t pgd_val(pgd_t pgd)
         return ret;
 }
 
+#define  __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
+static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
+                                           pte_t *ptep)
+{
+        pteval_t ret;
+
+        ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
+                         mm, addr, ptep);
+
+        return (pte_t) { .pte = ret };
+}
+
+static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
+                                           pte_t *ptep, pte_t pte)
+{
+        if (sizeof(pteval_t) > sizeof(long))
+                /* 5 arg words */
+                pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
+        else
+                PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
+                            mm, addr, ptep, pte.pte);
+}
+
 static inline void set_pte(pte_t *ptep, pte_t pte)
 {
         if (sizeof(pteval_t) > sizeof(long))
@@ -1252,6 +1359,12 @@ static inline void arch_flush_lazy_mmu_mode(void)
         }
 }
 
+static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
+                                unsigned long phys, pgprot_t flags)
+{
+        pv_mmu_ops.set_fixmap(idx, phys, flags);
+}
+
 void _paravirt_nop(void);
 #define paravirt_nop    ((void *)_paravirt_nop)
 
@@ -1277,8 +1390,8 @@ extern struct paravirt_patch_site __parainstructions[],
  * caller saved registers but the argument parameter */
 #define PV_SAVE_REGS "pushq %%rdi;"
 #define PV_RESTORE_REGS "popq %%rdi;"
-#define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx"
-#define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx"
+#define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
+#define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
 #define PV_FLAGS_ARG "D"
 #endif
 
@@ -1370,58 +1483,108 @@ static inline unsigned long __raw_local_irq_save(void)
 
 
 #ifdef CONFIG_X86_64
-#define PV_SAVE_REGS   pushq %rax; pushq %rdi; pushq %rcx; pushq %rdx
-#define PV_RESTORE_REGS popq %rdx; popq %rcx; popq %rdi; popq %rax
+#define PV_SAVE_REGS                            \
+        push %rax;                              \
+        push %rcx;                              \
+        push %rdx;                              \
+        push %rsi;                              \
+        push %rdi;                              \
+        push %r8;                               \
+        push %r9;                               \
+        push %r10;                              \
+        push %r11
+#define PV_RESTORE_REGS                         \
+        pop %r11;                               \
+        pop %r10;                               \
+        pop %r9;                                \
+        pop %r8;                                \
+        pop %rdi;                               \
+        pop %rsi;                               \
+        pop %rdx;                               \
+        pop %rcx;                               \
+        pop %rax
 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 8)
 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
+#define PARA_INDIRECT(addr)     *addr(%rip)
 #else
 #define PV_SAVE_REGS   pushl %eax; pushl %edi; pushl %ecx; pushl %edx
 #define PV_RESTORE_REGS popl %edx; popl %ecx; popl %edi; popl %eax
 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 4)
 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
+#define PARA_INDIRECT(addr)     *%cs:addr
 #endif
 
 #define INTERRUPT_RETURN                                                \
         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE,       \
-                  jmp *%cs:pv_cpu_ops+PV_CPU_iret)
+                  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
 
 #define DISABLE_INTERRUPTS(clobbers)                                    \
         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
-                  PV_SAVE_REGS;                 \
-                  call *%cs:pv_irq_ops+PV_IRQ_irq_disable;              \
+                  PV_SAVE_REGS;                                         \
+                  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable);    \
                   PV_RESTORE_REGS;)                     \
 
 #define ENABLE_INTERRUPTS(clobbers)                                     \
         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers,  \
-                  PV_SAVE_REGS;                 \
-                  call *%cs:pv_irq_ops+PV_IRQ_irq_enable;               \
+                  PV_SAVE_REGS;                                         \
+                  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable);     \
                   PV_RESTORE_REGS;)
 
-#define ENABLE_INTERRUPTS_SYSCALL_RET                                   \
-        PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_syscall_ret),\
+#define USERGS_SYSRET32                                                 \
+        PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32),       \
                   CLBR_NONE,                                            \
-                  jmp *%cs:pv_cpu_ops+PV_CPU_irq_enable_syscall_ret)
-
+                  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret32))
 
 #ifdef CONFIG_X86_32
-#define GET_CR0_INTO_EAX                        \
-        push %ecx; push %edx;                   \
-        call *pv_cpu_ops+PV_CPU_read_cr0;       \
+#define GET_CR0_INTO_EAX                                \
+        push %ecx; push %edx;                           \
+        call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
         pop %edx; pop %ecx
-#else
+
+#define ENABLE_INTERRUPTS_SYSEXIT                                       \
+        PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit),    \
+                  CLBR_NONE,                                            \
+                  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
+
+
+#else   /* !CONFIG_X86_32 */
+
+/*
+ * If swapgs is used while the userspace stack is still current,
+ * there's no way to call a pvop.  The PV replacement *must* be
+ * inlined, or the swapgs instruction must be trapped and emulated.
+ */
+#define SWAPGS_UNSAFE_STACK                                             \
+        PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,     \
+                  swapgs)
+
 #define SWAPGS                                                          \
         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,     \
                   PV_SAVE_REGS;                                         \
-                  call *pv_cpu_ops+PV_CPU_swapgs;                       \
+                  call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs);         \
                   PV_RESTORE_REGS                                       \
                  )
 
-#define GET_CR2_INTO_RCX                        \
-        call *pv_mmu_ops+PV_MMU_read_cr2;       \
-        movq %rax, %rcx;                        \
+#define GET_CR2_INTO_RCX                                \
+        call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2); \
+        movq %rax, %rcx;                                \
         xorq %rax, %rax;
 
-#endif
+#define PARAVIRT_ADJUST_EXCEPTION_FRAME                                 \
+        PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
+                  CLBR_NONE,                                            \
+                  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame))
+
+#define USERGS_SYSRET64                                                 \
+        PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64),       \
+                  CLBR_NONE,                                            \
+                  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
+
+#define ENABLE_INTERRUPTS_SYSEXIT32                                     \
+        PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit),    \
+                  CLBR_NONE,                                            \
+                  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
+#endif  /* CONFIG_X86_32 */
 
 #endif /* __ASSEMBLY__ */
 #endif /* CONFIG_PARAVIRT */