1 files changed, 88 insertions, 16 deletions

diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 5c4ee422590e..8573b83a63d0 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -87,7 +87,19 @@ static inline void pgd_list_del(pgd_t *pgd)
 #define UNSHARED_PTRS_PER_PGD                           \
         (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
 
-static void pgd_ctor(pgd_t *pgd)
+
+static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)
+{
+        BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));
+        virt_to_page(pgd)->index = (pgoff_t)mm;
+}
+
+struct mm_struct *pgd_page_get_mm(struct page *page)
+{
+        return (struct mm_struct *)page->index;
+}
+
+static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)
 {
         /* If the pgd points to a shared pagetable level (either the
            ptes in non-PAE, or shared PMD in PAE), then just copy the
@@ -98,27 +110,23 @@ static void pgd_ctor(pgd_t *pgd)
                 clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY,
                                 swapper_pg_dir + KERNEL_PGD_BOUNDARY,
                                 KERNEL_PGD_PTRS);
-                paravirt_alloc_pmd_clone(__pa(pgd) >> PAGE_SHIFT,
-                                         __pa(swapper_pg_dir) >> PAGE_SHIFT,
-                                         KERNEL_PGD_BOUNDARY,
-                                         KERNEL_PGD_PTRS);
         }
 
         /* list required to sync kernel mapping updates */
-        if (!SHARED_KERNEL_PMD)
+        if (!SHARED_KERNEL_PMD) {
+                pgd_set_mm(pgd, mm);
                 pgd_list_add(pgd);
+        }
 }
 
 static void pgd_dtor(pgd_t *pgd)
 {
-        unsigned long flags; /* can be called from interrupt context */
-
         if (SHARED_KERNEL_PMD)
                 return;
 
-        spin_lock_irqsave(&pgd_lock, flags);
+        spin_lock(&pgd_lock);
         pgd_list_del(pgd);
-        spin_unlock_irqrestore(&pgd_lock, flags);
+        spin_unlock(&pgd_lock);
 }
 
 /*
@@ -160,8 +168,7 @@ void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
          * section 8.1: in PAE mode we explicitly have to flush the
          * TLB via cr3 if the top-level pgd is changed...
          */
-        if (mm == current->active_mm)
-                write_cr3(read_cr3());
+        flush_tlb_mm(mm);
 }
 #else  /* !CONFIG_X86_PAE */
 
@@ -250,7 +257,6 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
 {
         pgd_t *pgd;
         pmd_t *pmds[PREALLOCATED_PMDS];
-        unsigned long flags;
 
         pgd = (pgd_t *)__get_free_page(PGALLOC_GFP);
 
@@ -270,12 +276,12 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
          * respect to anything walking the pgd_list, so that they
          * never see a partially populated pgd.
          */
-        spin_lock_irqsave(&pgd_lock, flags);
+        spin_lock(&pgd_lock);
 
-        pgd_ctor(pgd);
+        pgd_ctor(mm, pgd);
         pgd_prepopulate_pmd(mm, pgd, pmds);
 
-        spin_unlock_irqrestore(&pgd_lock, flags);
+        spin_unlock(&pgd_lock);
 
         return pgd;
 
@@ -310,6 +316,25 @@ int ptep_set_access_flags(struct vm_area_struct *vma,
         return changed;
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+int pmdp_set_access_flags(struct vm_area_struct *vma,
+                          unsigned long address, pmd_t *pmdp,
+                          pmd_t entry, int dirty)
+{
+        int changed = !pmd_same(*pmdp, entry);
+
+        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+
+        if (changed && dirty) {
+                *pmdp = entry;
+                pmd_update_defer(vma->vm_mm, address, pmdp);
+                flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+        }
+
+        return changed;
+}
+#endif
+
 int ptep_test_and_clear_young(struct vm_area_struct *vma,
                               unsigned long addr, pte_t *ptep)
 {
@@ -325,6 +350,23 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma,
         return ret;
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+int pmdp_test_and_clear_young(struct vm_area_struct *vma,
+                              unsigned long addr, pmd_t *pmdp)
+{
+        int ret = 0;
+
+        if (pmd_young(*pmdp))
+                ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
+                                         (unsigned long *)pmdp);
+
+        if (ret)
+                pmd_update(vma->vm_mm, addr, pmdp);
+
+        return ret;
+}
+#endif
+
 int ptep_clear_flush_young(struct vm_area_struct *vma,
                            unsigned long address, pte_t *ptep)
 {
@@ -337,6 +379,36 @@ int ptep_clear_flush_young(struct vm_area_struct *vma,
         return young;
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+int pmdp_clear_flush_young(struct vm_area_struct *vma,
+                           unsigned long address, pmd_t *pmdp)
+{
+        int young;
+
+        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+
+        young = pmdp_test_and_clear_young(vma, address, pmdp);
+        if (young)
+                flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+
+        return young;
+}
+
+void pmdp_splitting_flush(struct vm_area_struct *vma,
+                          unsigned long address, pmd_t *pmdp)
+{
+        int set;
+        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+        set = !test_and_set_bit(_PAGE_BIT_SPLITTING,
+                                (unsigned long *)pmdp);
+        if (set) {
+                pmd_update(vma->vm_mm, address, pmdp);
+                /* need tlb flush only to serialize against gup-fast */
+                flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+        }
+}
+#endif
+
 /**
  * reserve_top_address - reserves a hole in the top of kernel address space
  * @reserve - size of hole to reserve