mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared

For VMAs that don't want write notifications, PTEs created for read faults
have their write bit set.  If the read fault happens after VM_SOFTDIRTY is
cleared, then the PTE's softdirty bit will remain clear after subsequent
writes.

Here's a simple code snippet to demonstrate the bug:

  char* m = mmap(NULL, getpagesize(), PROT_READ | PROT_WRITE,
                 MAP_ANONYMOUS | MAP_SHARED, -1, 0);
  system("echo 4 > /proc/$PPID/clear_refs"); /* clear VM_SOFTDIRTY */
  assert(*m == '\0');     /* new PTE allows write access */
  assert(!soft_dirty(x));
  *m = 'x';               /* should dirty the page */
  assert(soft_dirty(x));  /* fails */

With this patch, write notifications are enabled when VM_SOFTDIRTY is
cleared.  Furthermore, to avoid unnecessary faults, write notifications
are disabled when VM_SOFTDIRTY is set.

As a side effect of enabling and disabling write notifications with
care, this patch fixes a bug in mprotect where vm_page_prot bits set by
drivers were zapped on mprotect.  An analogous bug was fixed in mmap by
commit c9d0bf241451 ("mm: uncached vma support with writenotify").

Signed-off-by: Peter Feiner <pfeiner@google.com>
Reported-by: Peter Feiner <pfeiner@google.com>
Suggested-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Cyrill Gorcunov <gorcunov@openvz.org>
Cc: Pavel Emelyanov <xemul@parallels.com>
Cc: Jamie Liu <jamieliu@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

3 files changed, 35 insertions, 33 deletions

diff --git a/mm/memory.c b/mm/memory.c
index e229970e4223..1cc6bfbd872e 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2053,7 +2053,8 @@ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
         old_page = vm_normal_page(vma, address, orig_pte);
         if (!old_page) {
                 /*
-                 * VM_MIXEDMAP !pfn_valid() case
+                 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
+                 * VM_PFNMAP VMA.
                  *
                  * We should not cow pages in a shared writeable mapping.
                  * Just mark the pages writable as we can't do any dirty
diff --git a/mm/mmap.c b/mm/mmap.c
index 93d28c7e5420..7f855206e7fb 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -89,6 +89,25 @@ pgprot_t vm_get_page_prot(unsigned long vm_flags)
 }
 EXPORT_SYMBOL(vm_get_page_prot);
 
+static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
+{
+        return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
+}
+
+/* Update vma->vm_page_prot to reflect vma->vm_flags. */
+void vma_set_page_prot(struct vm_area_struct *vma)
+{
+        unsigned long vm_flags = vma->vm_flags;
+
+        vma->vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
+        if (vma_wants_writenotify(vma)) {
+                vm_flags &= ~VM_SHARED;
+                vma->vm_page_prot = vm_pgprot_modify(vma->vm_page_prot,
+                                                     vm_flags);
+        }
+}
+
+
 int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;  /* heuristic overcommit */
 int sysctl_overcommit_ratio __read_mostly = 50; /* default is 50% */
 unsigned long sysctl_overcommit_kbytes __read_mostly;
@@ -1475,11 +1494,16 @@ int vma_wants_writenotify(struct vm_area_struct *vma)
         if (vma->vm_ops && vma->vm_ops->page_mkwrite)
                 return 1;
 
-        /* The open routine did something to the protections already? */
+        /* The open routine did something to the protections that pgprot_modify
+         * won't preserve? */
         if (pgprot_val(vma->vm_page_prot) !=
-            pgprot_val(vm_get_page_prot(vm_flags)))
+            pgprot_val(vm_pgprot_modify(vma->vm_page_prot, vm_flags)))
                 return 0;
 
+        /* Do we need to track softdirty? */
+        if (IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) && !(vm_flags & VM_SOFTDIRTY))
+                return 1;
+
         /* Specialty mapping? */
         if (vm_flags & VM_PFNMAP)
                 return 0;
@@ -1615,21 +1639,6 @@ munmap_back:
                         goto free_vma;
         }
 
-        if (vma_wants_writenotify(vma)) {
-                pgprot_t pprot = vma->vm_page_prot;
-
-                /* Can vma->vm_page_prot have changed??
-                 *
-                 * Answer: Yes, drivers may have changed it in their
-                 *         f_op->mmap method.
-                 *
-                 * Ensures that vmas marked as uncached stay that way.
-                 */
-                vma->vm_page_prot = vm_get_page_prot(vm_flags & ~VM_SHARED);
-                if (pgprot_val(pprot) == pgprot_val(pgprot_noncached(pprot)))
-                        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
-        }
-
         vma_link(mm, vma, prev, rb_link, rb_parent);
         /* Once vma denies write, undo our temporary denial count */
         if (file) {
@@ -1663,6 +1672,8 @@ out:
          */
         vma->vm_flags |= VM_SOFTDIRTY;
 
+        vma_set_page_prot(vma);
+
         return addr;
 
 unmap_and_free_vma:
diff --git a/mm/mprotect.c b/mm/mprotect.c
index c43d557941f8..ace93454ce8e 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -29,13 +29,6 @@
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
 
-#ifndef pgprot_modify
-static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
-{
-        return newprot;
-}
-#endif
-
 /*
  * For a prot_numa update we only hold mmap_sem for read so there is a
  * potential race with faulting where a pmd was temporarily none. This
@@ -93,7 +86,9 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
                                  * Avoid taking write faults for pages we
                                  * know to be dirty.
                                  */
-                                if (dirty_accountable && pte_dirty(ptent))
+                                if (dirty_accountable && pte_dirty(ptent) &&
+                                    (pte_soft_dirty(ptent) ||
+                                     !(vma->vm_flags & VM_SOFTDIRTY)))
                                         ptent = pte_mkwrite(ptent);
                                 ptep_modify_prot_commit(mm, addr, pte, ptent);
                                 updated = true;
@@ -320,13 +315,8 @@ success:
          * held in write mode.
          */
         vma->vm_flags = newflags;
-        vma->vm_page_prot = pgprot_modify(vma->vm_page_prot,
-                                          vm_get_page_prot(newflags));
-
-        if (vma_wants_writenotify(vma)) {
-                vma->vm_page_prot = vm_get_page_prot(newflags & ~VM_SHARED);
-                dirty_accountable = 1;
-        }
+        dirty_accountable = vma_wants_writenotify(vma);
+        vma_set_page_prot(vma);
 
         change_protection(vma, start, end, vma->vm_page_prot,
                           dirty_accountable, 0);