1 files changed, 133 insertions, 35 deletions

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 3e636d864d56..107d026f5d6e 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -11,6 +11,7 @@
 #include <linux/rmap.h>
 #include <linux/swap.h>
 #include <linux/swapops.h>
+#include <linux/mmu_notifier.h>
 
 #include <asm/elf.h>
 #include <asm/uaccess.h>
@@ -688,10 +689,66 @@ const struct file_operations proc_tid_smaps_operations = {
         .release        = seq_release_private,
 };
 
+/*
+ * We do not want to have constant page-shift bits sitting in
+ * pagemap entries and are about to reuse them some time soon.
+ *
+ * Here's the "migration strategy":
+ * 1. when the system boots these bits remain what they are,
+ *    but a warning about future change is printed in log;
+ * 2. once anyone clears soft-dirty bits via clear_refs file,
+ *    these flag is set to denote, that user is aware of the
+ *    new API and those page-shift bits change their meaning.
+ *    The respective warning is printed in dmesg;
+ * 3. In a couple of releases we will remove all the mentions
+ *    of page-shift in pagemap entries.
+ */
+
+static bool soft_dirty_cleared __read_mostly;
+
+enum clear_refs_types {
+        CLEAR_REFS_ALL = 1,
+        CLEAR_REFS_ANON,
+        CLEAR_REFS_MAPPED,
+        CLEAR_REFS_SOFT_DIRTY,
+        CLEAR_REFS_LAST,
+};
+
+struct clear_refs_private {
+        struct vm_area_struct *vma;
+        enum clear_refs_types type;
+};
+
+static inline void clear_soft_dirty(struct vm_area_struct *vma,
+                unsigned long addr, pte_t *pte)
+{
+#ifdef CONFIG_MEM_SOFT_DIRTY
+        /*
+         * The soft-dirty tracker uses #PF-s to catch writes
+         * to pages, so write-protect the pte as well. See the
+         * Documentation/vm/soft-dirty.txt for full description
+         * of how soft-dirty works.
+         */
+        pte_t ptent = *pte;
+
+        if (pte_present(ptent)) {
+                ptent = pte_wrprotect(ptent);
+                ptent = pte_clear_flags(ptent, _PAGE_SOFT_DIRTY);
+        } else if (is_swap_pte(ptent)) {
+                ptent = pte_swp_clear_soft_dirty(ptent);
+        } else if (pte_file(ptent)) {
+                ptent = pte_file_clear_soft_dirty(ptent);
+        }
+
+        set_pte_at(vma->vm_mm, addr, pte, ptent);
+#endif
+}
+
 static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
                                 unsigned long end, struct mm_walk *walk)
 {
-        struct vm_area_struct *vma = walk->private;
+        struct clear_refs_private *cp = walk->private;
+        struct vm_area_struct *vma = cp->vma;
         pte_t *pte, ptent;
         spinlock_t *ptl;
         struct page *page;
@@ -703,6 +760,12 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
         for (; addr != end; pte++, addr += PAGE_SIZE) {
                 ptent = *pte;
+
+                if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
+                        clear_soft_dirty(vma, addr, pte);
+                        continue;
+                }
+
                 if (!pte_present(ptent))
                         continue;
 
@@ -719,10 +782,6 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
         return 0;
 }
 
-#define CLEAR_REFS_ALL 1
-#define CLEAR_REFS_ANON 2
-#define CLEAR_REFS_MAPPED 3
-
 static ssize_t clear_refs_write(struct file *file, const char __user *buf,
                                 size_t count, loff_t *ppos)
 {
@@ -730,7 +789,8 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
         char buffer[PROC_NUMBUF];
         struct mm_struct *mm;
         struct vm_area_struct *vma;
-        int type;
+        enum clear_refs_types type;
+        int itype;
         int rv;
 
         memset(buffer, 0, sizeof(buffer));
@@ -738,23 +798,37 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
                 count = sizeof(buffer) - 1;
         if (copy_from_user(buffer, buf, count))
                 return -EFAULT;
-        rv = kstrtoint(strstrip(buffer), 10, &type);
+        rv = kstrtoint(strstrip(buffer), 10, &itype);
         if (rv < 0)
                 return rv;
-        if (type < CLEAR_REFS_ALL || type > CLEAR_REFS_MAPPED)
+        type = (enum clear_refs_types)itype;
+        if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
                 return -EINVAL;
+
+        if (type == CLEAR_REFS_SOFT_DIRTY) {
+                soft_dirty_cleared = true;
+                pr_warn_once("The pagemap bits 55-60 has changed their meaning! "
+                                "See the linux/Documentation/vm/pagemap.txt for details.\n");
+        }
+
         task = get_proc_task(file_inode(file));
         if (!task)
                 return -ESRCH;
         mm = get_task_mm(task);
         if (mm) {
+                struct clear_refs_private cp = {
+                        .type = type,
+                };
                 struct mm_walk clear_refs_walk = {
                         .pmd_entry = clear_refs_pte_range,
                         .mm = mm,
+                        .private = &cp,
                 };
                 down_read(&mm->mmap_sem);
+                if (type == CLEAR_REFS_SOFT_DIRTY)
+                        mmu_notifier_invalidate_range_start(mm, 0, -1);
                 for (vma = mm->mmap; vma; vma = vma->vm_next) {
-                        clear_refs_walk.private = vma;
+                        cp.vma = vma;
                         if (is_vm_hugetlb_page(vma))
                                 continue;
                         /*
@@ -773,6 +847,8 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
                         walk_page_range(vma->vm_start, vma->vm_end,
                                         &clear_refs_walk);
                 }
+                if (type == CLEAR_REFS_SOFT_DIRTY)
+                        mmu_notifier_invalidate_range_end(mm, 0, -1);
                 flush_tlb_mm(mm);
                 up_read(&mm->mmap_sem);
                 mmput(mm);
@@ -792,14 +868,15 @@ typedef struct {
 } pagemap_entry_t;
 
 struct pagemapread {
-        int pos, len;
+        int pos, len;           /* units: PM_ENTRY_BYTES, not bytes */
         pagemap_entry_t *buffer;
+        bool v2;
 };
 
 #define PAGEMAP_WALK_SIZE       (PMD_SIZE)
 #define PAGEMAP_WALK_MASK       (PMD_MASK)
 
-#define PM_ENTRY_BYTES      sizeof(u64)
+#define PM_ENTRY_BYTES      sizeof(pagemap_entry_t)
 #define PM_STATUS_BITS      3
 #define PM_STATUS_OFFSET    (64 - PM_STATUS_BITS)
 #define PM_STATUS_MASK      (((1LL << PM_STATUS_BITS) - 1) << PM_STATUS_OFFSET)
@@ -807,14 +884,17 @@ struct pagemapread {
 #define PM_PSHIFT_BITS      6
 #define PM_PSHIFT_OFFSET    (PM_STATUS_OFFSET - PM_PSHIFT_BITS)
 #define PM_PSHIFT_MASK      (((1LL << PM_PSHIFT_BITS) - 1) << PM_PSHIFT_OFFSET)
-#define PM_PSHIFT(x)        (((u64) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)
+#define __PM_PSHIFT(x)      (((u64) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)
 #define PM_PFRAME_MASK      ((1LL << PM_PSHIFT_OFFSET) - 1)
 #define PM_PFRAME(x)        ((x) & PM_PFRAME_MASK)
+/* in "new" pagemap pshift bits are occupied with more status bits */
+#define PM_STATUS2(v2, x)   (__PM_PSHIFT(v2 ? x : PAGE_SHIFT))
 
+#define __PM_SOFT_DIRTY      (1LL)
 #define PM_PRESENT          PM_STATUS(4LL)
 #define PM_SWAP             PM_STATUS(2LL)
 #define PM_FILE             PM_STATUS(1LL)
-#define PM_NOT_PRESENT      PM_PSHIFT(PAGE_SHIFT)
+#define PM_NOT_PRESENT(v2)  PM_STATUS2(v2, 0)
 #define PM_END_OF_BUFFER    1
 
 static inline pagemap_entry_t make_pme(u64 val)
@@ -837,7 +917,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end,
         struct pagemapread *pm = walk->private;
         unsigned long addr;
         int err = 0;
-        pagemap_entry_t pme = make_pme(PM_NOT_PRESENT);
+        pagemap_entry_t pme = make_pme(PM_NOT_PRESENT(pm->v2));
 
         for (addr = start; addr < end; addr += PAGE_SIZE) {
                 err = add_to_pagemap(addr, &pme, pm);
@@ -847,38 +927,43 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end,
         return err;
 }
 
-static void pte_to_pagemap_entry(pagemap_entry_t *pme,
+static void pte_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,
                 struct vm_area_struct *vma, unsigned long addr, pte_t pte)
 {
         u64 frame, flags;
         struct page *page = NULL;
+        int flags2 = 0;
 
         if (pte_present(pte)) {
                 frame = pte_pfn(pte);
                 flags = PM_PRESENT;
                 page = vm_normal_page(vma, addr, pte);
         } else if (is_swap_pte(pte)) {
-                swp_entry_t entry = pte_to_swp_entry(pte);
-
+                swp_entry_t entry;
+                if (pte_swp_soft_dirty(pte))
+                        flags2 |= __PM_SOFT_DIRTY;
+                entry = pte_to_swp_entry(pte);
                 frame = swp_type(entry) |
                         (swp_offset(entry) << MAX_SWAPFILES_SHIFT);
                 flags = PM_SWAP;
                 if (is_migration_entry(entry))
                         page = migration_entry_to_page(entry);
         } else {
-                *pme = make_pme(PM_NOT_PRESENT);
+                *pme = make_pme(PM_NOT_PRESENT(pm->v2));
                 return;
         }
 
         if (page && !PageAnon(page))
                 flags |= PM_FILE;
+        if (pte_soft_dirty(pte))
+                flags2 |= __PM_SOFT_DIRTY;
 
-        *pme = make_pme(PM_PFRAME(frame) | PM_PSHIFT(PAGE_SHIFT) | flags);
+        *pme = make_pme(PM_PFRAME(frame) | PM_STATUS2(pm->v2, flags2) | flags);
 }
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-static void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme,
-                                        pmd_t pmd, int offset)
+static void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,
+                pmd_t pmd, int offset, int pmd_flags2)
 {
         /*
          * Currently pmd for thp is always present because thp can not be
@@ -887,13 +972,13 @@ static void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme,
          */
         if (pmd_present(pmd))
                 *pme = make_pme(PM_PFRAME(pmd_pfn(pmd) + offset)
-                                | PM_PSHIFT(PAGE_SHIFT) | PM_PRESENT);
+                                | PM_STATUS2(pm->v2, pmd_flags2) | PM_PRESENT);
         else
-                *pme = make_pme(PM_NOT_PRESENT);
+                *pme = make_pme(PM_NOT_PRESENT(pm->v2));
 }
 #else
-static inline void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme,
-                                                pmd_t pmd, int offset)
+static inline void thp_pmd_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,
+                pmd_t pmd, int offset, int pmd_flags2)
 {
 }
 #endif
@@ -905,17 +990,20 @@ static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
         struct pagemapread *pm = walk->private;
         pte_t *pte;
         int err = 0;
-        pagemap_entry_t pme = make_pme(PM_NOT_PRESENT);
+        pagemap_entry_t pme = make_pme(PM_NOT_PRESENT(pm->v2));
 
         /* find the first VMA at or above 'addr' */
         vma = find_vma(walk->mm, addr);
         if (vma && pmd_trans_huge_lock(pmd, vma) == 1) {
+                int pmd_flags2;
+
+                pmd_flags2 = (pmd_soft_dirty(*pmd) ? __PM_SOFT_DIRTY : 0);
                 for (; addr != end; addr += PAGE_SIZE) {
                         unsigned long offset;
 
                         offset = (addr & ~PAGEMAP_WALK_MASK) >>
                                         PAGE_SHIFT;
-                        thp_pmd_to_pagemap_entry(&pme, *pmd, offset);
+                        thp_pmd_to_pagemap_entry(&pme, pm, *pmd, offset, pmd_flags2);
                         err = add_to_pagemap(addr, &pme, pm);
                         if (err)
                                 break;
@@ -932,7 +1020,7 @@ static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
                  * and need a new, higher one */
                 if (vma && (addr >= vma->vm_end)) {
                         vma = find_vma(walk->mm, addr);
-                        pme = make_pme(PM_NOT_PRESENT);
+                        pme = make_pme(PM_NOT_PRESENT(pm->v2));
                 }
 
                 /* check that 'vma' actually covers this address,
@@ -940,7 +1028,7 @@ static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
                 if (vma && (vma->vm_start <= addr) &&
                     !is_vm_hugetlb_page(vma)) {
                         pte = pte_offset_map(pmd, addr);
-                        pte_to_pagemap_entry(&pme, vma, addr, *pte);
+                        pte_to_pagemap_entry(&pme, pm, vma, addr, *pte);
                         /* unmap before userspace copy */
                         pte_unmap(pte);
                 }
@@ -955,14 +1043,14 @@ static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
 }
 
 #ifdef CONFIG_HUGETLB_PAGE
-static void huge_pte_to_pagemap_entry(pagemap_entry_t *pme,
+static void huge_pte_to_pagemap_entry(pagemap_entry_t *pme, struct pagemapread *pm,
                                         pte_t pte, int offset)
 {
         if (pte_present(pte))
                 *pme = make_pme(PM_PFRAME(pte_pfn(pte) + offset)
-                                | PM_PSHIFT(PAGE_SHIFT) | PM_PRESENT);
+                                | PM_STATUS2(pm->v2, 0) | PM_PRESENT);
         else
-                *pme = make_pme(PM_NOT_PRESENT);
+                *pme = make_pme(PM_NOT_PRESENT(pm->v2));
 }
 
 /* This function walks within one hugetlb entry in the single call */
@@ -976,7 +1064,7 @@ static int pagemap_hugetlb_range(pte_t *pte, unsigned long hmask,
 
         for (; addr != end; addr += PAGE_SIZE) {
                 int offset = (addr & ~hmask) >> PAGE_SHIFT;
-                huge_pte_to_pagemap_entry(&pme, *pte, offset);
+                huge_pte_to_pagemap_entry(&pme, pm, *pte, offset);
                 err = add_to_pagemap(addr, &pme, pm);
                 if (err)
                         return err;
@@ -1038,8 +1126,9 @@ static ssize_t pagemap_read(struct file *file, char __user *buf,
         if (!count)
                 goto out_task;
 
-        pm.len = PM_ENTRY_BYTES * (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
-        pm.buffer = kmalloc(pm.len, GFP_TEMPORARY);
+        pm.v2 = soft_dirty_cleared;
+        pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
+        pm.buffer = kmalloc(pm.len * PM_ENTRY_BYTES, GFP_TEMPORARY);
         ret = -ENOMEM;
         if (!pm.buffer)
                 goto out_task;
@@ -1110,9 +1199,18 @@ out:
         return ret;
 }
 
+static int pagemap_open(struct inode *inode, struct file *file)
+{
+        pr_warn_once("Bits 55-60 of /proc/PID/pagemap entries are about "
+                        "to stop being page-shift some time soon. See the "
+                        "linux/Documentation/vm/pagemap.txt for details.\n");
+        return 0;
+}
+
 const struct file_operations proc_pagemap_operations = {
         .llseek         = mem_lseek, /* borrow this */
         .read           = pagemap_read,
+        .open           = pagemap_open,
 };
 #endif /* CONFIG_PROC_PAGE_MONITOR */