x86: cpa, preserve large pages if possible

When CPA is called on a range which fits into a large page mapping,
avoid to split the page when:

1) There is no change of attributes
2) The range to change is a complete large mapping

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 130 insertions, 12 deletions

diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 79a9f1b42ddd..40b7ac58e671 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -18,12 +18,17 @@
 
 struct cpa_data {
         unsigned long   vaddr;
-        int             numpages;
         pgprot_t        mask_set;
         pgprot_t        mask_clr;
+        int             numpages;
         int             flushtlb;
 };
 
+enum {
+        CPA_NO_SPLIT = 0,
+        CPA_SPLIT,
+};
+
 static inline int
 within(unsigned long addr, unsigned long start, unsigned long end)
 {
@@ -230,6 +235,86 @@ static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
 #endif
 }
 
+static int try_preserve_large_page(pte_t *kpte, unsigned long address,
+                                   struct cpa_data *cpa)
+{
+        unsigned long nextpage_addr, numpages, pmask, psize, flags;
+        pte_t new_pte, old_pte, *tmp;
+        pgprot_t old_prot, new_prot;
+        int level, res = CPA_SPLIT;
+
+        spin_lock_irqsave(&pgd_lock, flags);
+        /*
+         * Check for races, another CPU might have split this page
+         * up already:
+         */
+        tmp = lookup_address(address, &level);
+        if (tmp != kpte)
+                goto out_unlock;
+
+        switch (level) {
+        case PG_LEVEL_2M:
+                psize = LARGE_PAGE_SIZE;
+                pmask = LARGE_PAGE_MASK;
+                break;
+        case PG_LEVEL_1G:
+        default:
+                res = -EINVAL;
+                goto out_unlock;
+        }
+
+        /*
+         * Calculate the number of pages, which fit into this large
+         * page starting at address:
+         */
+        nextpage_addr = (address + psize) & pmask;
+        numpages = (nextpage_addr - address) >> PAGE_SHIFT;
+        if (numpages < cpa->numpages)
+                cpa->numpages = numpages;
+
+        /*
+         * We are safe now. Check whether the new pgprot is the same:
+         */
+        old_pte = *kpte;
+        old_prot = new_prot = pte_pgprot(old_pte);
+
+        pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
+        pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
+        new_prot = static_protections(new_prot, address);
+
+        /*
+         * If there are no changes, return. maxpages has been updated
+         * above:
+         */
+        if (pgprot_val(new_prot) == pgprot_val(old_prot)) {
+                res = CPA_NO_SPLIT;
+                goto out_unlock;
+        }
+
+        /*
+         * We need to change the attributes. Check, whether we can
+         * change the large page in one go. We request a split, when
+         * the address is not aligned and the number of pages is
+         * smaller than the number of pages in the large page. Note
+         * that we limited the number of possible pages already to
+         * the number of pages in the large page.
+         */
+        if (address == (nextpage_addr - psize) && cpa->numpages == numpages) {
+                /*
+                 * The address is aligned and the number of pages
+                 * covers the full page.
+                 */
+                new_pte = pfn_pte(pte_pfn(old_pte), canon_pgprot(new_prot));
+                __set_pmd_pte(kpte, address, new_pte);
+                cpa->flushtlb = 1;
+                res = CPA_NO_SPLIT;
+        }
+
+out_unlock:
+        spin_unlock_irqrestore(&pgd_lock, flags);
+        return res;
+}
+
 static int split_large_page(pte_t *kpte, unsigned long address)
 {
         pgprot_t ref_prot = pte_pgprot(pte_clrhuge(*kpte));
@@ -295,7 +380,7 @@ out_unlock:
 static int __change_page_attr(unsigned long address, struct cpa_data *cpa)
 {
         struct page *kpte_page;
-        int level, err = 0;
+        int level, res;
         pte_t *kpte;
 
 repeat:
@@ -338,13 +423,34 @@ repeat:
                         set_pte_atomic(kpte, new_pte);
                         cpa->flushtlb = 1;
                 }
-        } else {
-                err = split_large_page(kpte, address);
-                if (!err)
-                        goto repeat;
-                cpa->flushtlb = 1;
+                cpa->numpages = 1;
+                return 0;
         }
-        return err;
+
+        /*
+         * Check, whether we can keep the large page intact
+         * and just change the pte:
+         */
+        res = try_preserve_large_page(kpte, address, cpa);
+        if (res < 0)
+                return res;
+
+        /*
+         * When the range fits into the existing large page,
+         * return. cp->numpages and cpa->tlbflush have been updated in
+         * try_large_page:
+         */
+        if (res == CPA_NO_SPLIT)
+                return 0;
+
+        /*
+         * We have to split the large page:
+         */
+        res = split_large_page(kpte, address);
+        if (res)
+                return res;
+        cpa->flushtlb = 1;
+        goto repeat;
 }
 
 /**
@@ -410,15 +516,27 @@ static int change_page_attr_addr(struct cpa_data *cpa)
 
 static int __change_page_attr_set_clr(struct cpa_data *cpa)
 {
-        unsigned int i;
-        int ret;
+        int ret, numpages = cpa->numpages;
 
-        for (i = 0; i < cpa->numpages ; i++, cpa->vaddr += PAGE_SIZE) {
+        while (numpages) {
+                /*
+                 * Store the remaining nr of pages for the large page
+                 * preservation check.
+                 */
+                cpa->numpages = numpages;
                 ret = change_page_attr_addr(cpa);
                 if (ret)
                         return ret;
-        }
 
+                /*
+                 * Adjust the number of pages with the result of the
+                 * CPA operation. Either a large page has been
+                 * preserved or a single page update happened.
+                 */
+                BUG_ON(cpa->numpages > numpages);
+                numpages -= cpa->numpages;
+                cpa->vaddr += cpa->numpages * PAGE_SIZE;
+        }
         return 0;
 }