thp: khugepaged: make khugepaged aware about madvise

MADV_HUGEPAGE and MADV_NOHUGEPAGE were fully effective only if run after
mmap and before touching the memory.  While this is enough for most
usages, it's little effort to make madvise more dynamic at runtime on an
existing mapping by making khugepaged aware about madvise.

MADV_HUGEPAGE: register in khugepaged immediately without waiting a page
fault (that may not ever happen if all pages are already mapped and the
"enabled" knob was set to madvise during the initial page faults).

MADV_NOHUGEPAGE: skip vmas marked VM_NOHUGEPAGE in khugepaged to stop
collapsing pages where not needed.

[akpm@linux-foundation.org: tweak comment]
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

2 files changed, 20 insertions, 5 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index fce667c0281d..004c9c2aac78 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1389,7 +1389,8 @@ out:
         return ret;
 }
 
-int hugepage_madvise(unsigned long *vm_flags, int advice)
+int hugepage_madvise(struct vm_area_struct *vma,
+                     unsigned long *vm_flags, int advice)
 {
         switch (advice) {
         case MADV_HUGEPAGE:
@@ -1404,6 +1405,13 @@ int hugepage_madvise(unsigned long *vm_flags, int advice)
                         return -EINVAL;
                 *vm_flags &= ~VM_NOHUGEPAGE;
                 *vm_flags |= VM_HUGEPAGE;
+                /*
+                 * If the vma become good for khugepaged to scan,
+                 * register it here without waiting a page fault that
+                 * may not happen any time soon.
+                 */
+                if (unlikely(khugepaged_enter_vma_merge(vma)))
+                        return -ENOMEM;
                 break;
         case MADV_NOHUGEPAGE:
                 /*
@@ -1417,6 +1425,11 @@ int hugepage_madvise(unsigned long *vm_flags, int advice)
                         return -EINVAL;
                 *vm_flags &= ~VM_HUGEPAGE;
                 *vm_flags |= VM_NOHUGEPAGE;
+                /*
+                 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
+                 * this vma even if we leave the mm registered in khugepaged if
+                 * it got registered before VM_NOHUGEPAGE was set.
+                 */
                 break;
         }
 
@@ -1784,7 +1797,8 @@ static void collapse_huge_page(struct mm_struct *mm,
         if (address < hstart || address + HPAGE_PMD_SIZE > hend)
                 goto out;
 
-        if (!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always())
+        if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
+            (vma->vm_flags & VM_NOHUGEPAGE))
                 goto out;
 
         /* VM_PFNMAP vmas may have vm_ops null but vm_file set */
@@ -2007,8 +2021,9 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
                         break;
                 }
 
-                if (!(vma->vm_flags & VM_HUGEPAGE) &&
-                    !khugepaged_always()) {
+                if ((!(vma->vm_flags & VM_HUGEPAGE) &&
+                     !khugepaged_always()) ||
+                    (vma->vm_flags & VM_NOHUGEPAGE)) {
                         progress++;
                         continue;
                 }
diff --git a/mm/madvise.c b/mm/madvise.c
index bbac126e03ed..2221491ed503 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -73,7 +73,7 @@ static long madvise_behavior(struct vm_area_struct * vma,
                 break;
         case MADV_HUGEPAGE:
         case MADV_NOHUGEPAGE:
-                error = hugepage_madvise(&new_flags, behavior);
+                error = hugepage_madvise(vma, &new_flags, behavior);
                 if (error)
                         goto out;
                 break;