xen: implement ptep_modify_prot_start/commit

Xen has a pte update function which will update a pte while preserving
its accessed and dirty bits.  This means that ptep_modify_prot_start() can be
implemented as a simple read of the pte value.  The hardware may
update the pte in the meantime, but ptep_modify_prot_commit() updates it while
preserving any changes that may have happened in the meantime.

The updates in ptep_modify_prot_commit() are batched if we're currently in lazy
mmu mode.

The mmu_update hypercall can take a batch of updates to perform, but
this code doesn't make particular use of that feature, in favour of
using generic multicall batching to get them all into the hypervisor.

The net effect of this is that each mprotect pte update turns from two
expensive trap-and-emulate faults into they hypervisor into a single
hypercall whose cost is amortized in a batched multicall.

Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Acked-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

5 files changed, 45 insertions, 5 deletions

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 0b7553cbc529..bd74229081c3 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -168,7 +168,9 @@ static void __init xen_banner(void)
 {
         printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
                pv_info.name);
-        printk(KERN_INFO "Hypervisor signature: %s\n", xen_start_info->magic);
+        printk(KERN_INFO "Hypervisor signature: %s%s\n",
+               xen_start_info->magic,
+               xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
 }
 
 static void xen_cpuid(unsigned int *ax, unsigned int *bx,
@@ -1243,6 +1245,8 @@ asmlinkage void __init xen_start_kernel(void)
 
         BUG_ON(memcmp(xen_start_info->magic, "xen-3", 5) != 0);
 
+        xen_setup_features();
+
         /* Install Xen paravirt ops */
         pv_info = xen_info;
         pv_init_ops = xen_init_ops;
@@ -1252,14 +1256,17 @@ asmlinkage void __init xen_start_kernel(void)
         pv_apic_ops = xen_apic_ops;
         pv_mmu_ops = xen_mmu_ops;
 
+        if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
+                pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
+                pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
+        }
+
         machine_ops = xen_machine_ops;
 
 #ifdef CONFIG_SMP
         smp_ops = xen_smp_ops;
 #endif
 
-        xen_setup_features();
-
         /* Get mfn list */
         if (!xen_feature(XENFEAT_auto_translated_physmap))
                 xen_build_dynamic_phys_to_machine();
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 8132aa8c5d49..846dad7d54a5 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -323,6 +323,27 @@ out:
                 preempt_enable();
 }
 
+pte_t xen_ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
+{
+        /* Just return the pte as-is.  We preserve the bits on commit */
+        return *ptep;
+}
+
+void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
+                                 pte_t *ptep, pte_t pte)
+{
+        struct multicall_space mcs;
+        struct mmu_update *u;
+
+        mcs = xen_mc_entry(sizeof(*u));
+        u = mcs.args;
+        u->ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
+        u->val = pte_val_ma(pte);
+        MULTI_mmu_update(mcs.mc, u, 1, NULL, DOMID_SELF);
+
+        xen_mc_issue(PARAVIRT_LAZY_MMU);
+}
+
 /* Assume pteval_t is equivalent to all the other *val_t types. */
 static pteval_t pte_mfn_to_pfn(pteval_t val)
 {
diff --git a/arch/x86/xen/mmu.h b/arch/x86/xen/mmu.h
index e3dd09e25c63..297bf9f5b8bc 100644
--- a/arch/x86/xen/mmu.h
+++ b/arch/x86/xen/mmu.h
@@ -52,4 +52,8 @@ void xen_set_pud_hyper(pud_t *ptr, pud_t val);
 void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
 void xen_pmd_clear(pmd_t *pmdp);
 
+pte_t xen_ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
+void  xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
+                                  pte_t *ptep, pte_t pte);
+
 #endif  /* _XEN_MMU_H */
diff --git a/include/xen/interface/features.h b/include/xen/interface/features.h
index d73228d16488..f51b6413b054 100644
--- a/include/xen/interface/features.h
+++ b/include/xen/interface/features.h
@@ -38,6 +38,9 @@
  */
 #define XENFEAT_pae_pgdir_above_4gb        4
 
+/* x86: Does this Xen host support the MMU_PT_UPDATE_PRESERVE_AD hypercall? */
+#define XENFEAT_mmu_pt_update_preserve_ad  5
+
 #define XENFEAT_NR_SUBMAPS 1
 
 #endif /* __XEN_PUBLIC_FEATURES_H__ */
diff --git a/include/xen/interface/xen.h b/include/xen/interface/xen.h
index 819a0331cda9..2befa3e2f1bc 100644
--- a/include/xen/interface/xen.h
+++ b/include/xen/interface/xen.h
@@ -114,9 +114,14 @@
  * ptr[:2]  -- Machine address within the frame whose mapping to modify.
  *             The frame must belong to the FD, if one is specified.
  * val      -- Value to write into the mapping entry.
+ *
+ * ptr[1:0] == MMU_PT_UPDATE_PRESERVE_AD:
+ * As MMU_NORMAL_PT_UPDATE above, but A/D bits currently in the PTE are ORed
+ * with those in @val.
  */
-#define MMU_NORMAL_PT_UPDATE     0 /* checked '*ptr = val'. ptr is MA.       */
-#define MMU_MACHPHYS_UPDATE      1 /* ptr = MA of frame to modify entry for  */
+#define MMU_NORMAL_PT_UPDATE      0 /* checked '*ptr = val'. ptr is MA.       */
+#define MMU_MACHPHYS_UPDATE       1 /* ptr = MA of frame to modify entry for  */
+#define MMU_PT_UPDATE_PRESERVE_AD 2 /* atomically: *ptr = val | (*ptr&(A|D)) */
 
 /*
  * MMU EXTENDED OPERATIONS