1 files changed, 79 insertions, 11 deletions

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 3c21775f196b..5c390f5a5207 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3326,8 +3326,8 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
         struct page *page;
         struct hstate *h = hstate_vma(vma);
         unsigned long sz = huge_page_size(h);
-        const unsigned long mmun_start = start; /* For mmu_notifiers */
-        const unsigned long mmun_end   = end;   /* For mmu_notifiers */
+        unsigned long mmun_start = start;       /* For mmu_notifiers */
+        unsigned long mmun_end   = end;         /* For mmu_notifiers */
 
         WARN_ON(!is_vm_hugetlb_page(vma));
         BUG_ON(start & ~huge_page_mask(h));
@@ -3339,6 +3339,11 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
          */
         tlb_remove_check_page_size_change(tlb, sz);
         tlb_start_vma(tlb, vma);
+
+        /*
+         * If sharing possible, alert mmu notifiers of worst case.
+         */
+        adjust_range_if_pmd_sharing_possible(vma, &mmun_start, &mmun_end);
         mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
         address = start;
         for (; address < end; address += sz) {
@@ -3349,6 +3354,10 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
                 ptl = huge_pte_lock(h, mm, ptep);
                 if (huge_pmd_unshare(mm, &address, ptep)) {
                         spin_unlock(ptl);
+                        /*
+                         * We just unmapped a page of PMDs by clearing a PUD.
+                         * The caller's TLB flush range should cover this area.
+                         */
                         continue;
                 }
 
@@ -3431,12 +3440,23 @@ void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
 {
         struct mm_struct *mm;
         struct mmu_gather tlb;
+        unsigned long tlb_start = start;
+        unsigned long tlb_end = end;
+
+        /*
+         * If shared PMDs were possibly used within this vma range, adjust
+         * start/end for worst case tlb flushing.
+         * Note that we can not be sure if PMDs are shared until we try to
+         * unmap pages.  However, we want to make sure TLB flushing covers
+         * the largest possible range.
+         */
+        adjust_range_if_pmd_sharing_possible(vma, &tlb_start, &tlb_end);
 
         mm = vma->vm_mm;
 
-        tlb_gather_mmu(&tlb, mm, start, end);
+        tlb_gather_mmu(&tlb, mm, tlb_start, tlb_end);
         __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
-        tlb_finish_mmu(&tlb, start, end);
+        tlb_finish_mmu(&tlb, tlb_start, tlb_end);
 }
 
 /*
@@ -4298,11 +4318,21 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
         pte_t pte;
         struct hstate *h = hstate_vma(vma);
         unsigned long pages = 0;
+        unsigned long f_start = start;
+        unsigned long f_end = end;
+        bool shared_pmd = false;
+
+        /*
+         * In the case of shared PMDs, the area to flush could be beyond
+         * start/end.  Set f_start/f_end to cover the maximum possible
+         * range if PMD sharing is possible.
+         */
+        adjust_range_if_pmd_sharing_possible(vma, &f_start, &f_end);
 
         BUG_ON(address >= end);
-        flush_cache_range(vma, address, end);
+        flush_cache_range(vma, f_start, f_end);
 
-        mmu_notifier_invalidate_range_start(mm, start, end);
+        mmu_notifier_invalidate_range_start(mm, f_start, f_end);
         i_mmap_lock_write(vma->vm_file->f_mapping);
         for (; address < end; address += huge_page_size(h)) {
                 spinlock_t *ptl;
@@ -4313,6 +4343,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
                 if (huge_pmd_unshare(mm, &address, ptep)) {
                         pages++;
                         spin_unlock(ptl);
+                        shared_pmd = true;
                         continue;
                 }
                 pte = huge_ptep_get(ptep);
@@ -4348,9 +4379,13 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
          * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
          * may have cleared our pud entry and done put_page on the page table:
          * once we release i_mmap_rwsem, another task can do the final put_page
-         * and that page table be reused and filled with junk.
+         * and that page table be reused and filled with junk.  If we actually
+         * did unshare a page of pmds, flush the range corresponding to the pud.
          */
-        flush_hugetlb_tlb_range(vma, start, end);
+        if (shared_pmd)
+                flush_hugetlb_tlb_range(vma, f_start, f_end);
+        else
+                flush_hugetlb_tlb_range(vma, start, end);
         /*
          * No need to call mmu_notifier_invalidate_range() we are downgrading
          * page table protection not changing it to point to a new page.
@@ -4358,7 +4393,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
          * See Documentation/vm/mmu_notifier.rst
          */
         i_mmap_unlock_write(vma->vm_file->f_mapping);
-        mmu_notifier_invalidate_range_end(mm, start, end);
+        mmu_notifier_invalidate_range_end(mm, f_start, f_end);
 
         return pages << h->order;
 }
@@ -4545,13 +4580,41 @@ static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
         /*
          * check on proper vm_flags and page table alignment
          */
-        if (vma->vm_flags & VM_MAYSHARE &&
-            vma->vm_start <= base && end <= vma->vm_end)
+        if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
                 return true;
         return false;
 }
 
 /*
+ * Determine if start,end range within vma could be mapped by shared pmd.
+ * If yes, adjust start and end to cover range associated with possible
+ * shared pmd mappings.
+ */
+void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
+                                unsigned long *start, unsigned long *end)
+{
+        unsigned long check_addr = *start;
+
+        if (!(vma->vm_flags & VM_MAYSHARE))
+                return;
+
+        for (check_addr = *start; check_addr < *end; check_addr += PUD_SIZE) {
+                unsigned long a_start = check_addr & PUD_MASK;
+                unsigned long a_end = a_start + PUD_SIZE;
+
+                /*
+                 * If sharing is possible, adjust start/end if necessary.
+                 */
+                if (range_in_vma(vma, a_start, a_end)) {
+                        if (a_start < *start)
+                                *start = a_start;
+                        if (a_end > *end)
+                                *end = a_end;
+                }
+        }
+}
+
+/*
  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
  * and returns the corresponding pte. While this is not necessary for the
  * !shared pmd case because we can allocate the pmd later as well, it makes the
@@ -4648,6 +4711,11 @@ int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
 {
         return 0;
 }
+
+void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
+                                unsigned long *start, unsigned long *end)
+{
+}
 #define want_pmd_share()        (0)
 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */