8 files changed, 251 insertions, 0 deletions

diff --git a/arch/alpha/include/uapi/asm/mman.h b/arch/alpha/include/uapi/asm/mman.h
index f3258fbf03d0..a18ec7f63888 100644
--- a/arch/alpha/include/uapi/asm/mman.h
+++ b/arch/alpha/include/uapi/asm/mman.h
@@ -69,6 +69,7 @@
 #define MADV_KEEPONFORK 19              /* Undo MADV_WIPEONFORK */
 
 #define MADV_COLD       20              /* deactivate these pages */
+#define MADV_PAGEOUT    21              /* reclaim these pages */
 
 /* compatibility flags */
 #define MAP_FILE        0
diff --git a/arch/mips/include/uapi/asm/mman.h b/arch/mips/include/uapi/asm/mman.h
index 00ad09fc5eb1..57dc2ac4f8bd 100644
--- a/arch/mips/include/uapi/asm/mman.h
+++ b/arch/mips/include/uapi/asm/mman.h
@@ -96,6 +96,7 @@
 #define MADV_KEEPONFORK 19              /* Undo MADV_WIPEONFORK */
 
 #define MADV_COLD       20              /* deactivate these pages */
+#define MADV_PAGEOUT    21              /* reclaim these pages */
 
 /* compatibility flags */
 #define MAP_FILE        0
diff --git a/arch/parisc/include/uapi/asm/mman.h b/arch/parisc/include/uapi/asm/mman.h
index eb14e3a7b8f3..6fd8871e4081 100644
--- a/arch/parisc/include/uapi/asm/mman.h
+++ b/arch/parisc/include/uapi/asm/mman.h
@@ -49,6 +49,7 @@
 #define MADV_DOFORK     11              /* do inherit across fork */
 
 #define MADV_COLD       20              /* deactivate these pages */
+#define MADV_PAGEOUT    21              /* reclaim these pages */
 
 #define MADV_MERGEABLE   65             /* KSM may merge identical pages */
 #define MADV_UNMERGEABLE 66             /* KSM may not merge identical pages */
diff --git a/arch/xtensa/include/uapi/asm/mman.h b/arch/xtensa/include/uapi/asm/mman.h
index f926b00ff11f..e5e643752947 100644
--- a/arch/xtensa/include/uapi/asm/mman.h
+++ b/arch/xtensa/include/uapi/asm/mman.h
@@ -104,6 +104,7 @@
 #define MADV_KEEPONFORK 19              /* Undo MADV_WIPEONFORK */
 
 #define MADV_COLD       20              /* deactivate these pages */
+#define MADV_PAGEOUT    21              /* reclaim these pages */
 
 /* compatibility flags */
 #define MAP_FILE        0
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 0ce997edb8bb..063c0c1e112b 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -365,6 +365,7 @@ extern int vm_swappiness;
 extern int remove_mapping(struct address_space *mapping, struct page *page);
 extern unsigned long vm_total_pages;
 
+extern unsigned long reclaim_pages(struct list_head *page_list);
 #ifdef CONFIG_NUMA
 extern int node_reclaim_mode;
 extern int sysctl_min_unmapped_ratio;
diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h
index 23431faf0eb6..c160a5354eb6 100644
--- a/include/uapi/asm-generic/mman-common.h
+++ b/include/uapi/asm-generic/mman-common.h
@@ -68,6 +68,7 @@
 #define MADV_KEEPONFORK 19              /* Undo MADV_WIPEONFORK */
 
 #define MADV_COLD       20              /* deactivate these pages */
+#define MADV_PAGEOUT    21              /* reclaim these pages */
 
 /* compatibility flags */
 #define MAP_FILE        0
diff --git a/mm/madvise.c b/mm/madvise.c
index e1aee62967c3..54c5639774b6 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -44,6 +44,7 @@ static int madvise_need_mmap_write(int behavior)
         case MADV_WILLNEED:
         case MADV_DONTNEED:
         case MADV_COLD:
+        case MADV_PAGEOUT:
         case MADV_FREE:
                 return 0;
         default:
@@ -461,6 +462,191 @@ static long madvise_cold(struct vm_area_struct *vma,
         return 0;
 }
 
+static int madvise_pageout_pte_range(pmd_t *pmd, unsigned long addr,
+                                unsigned long end, struct mm_walk *walk)
+{
+        struct mmu_gather *tlb = walk->private;
+        struct mm_struct *mm = tlb->mm;
+        struct vm_area_struct *vma = walk->vma;
+        pte_t *orig_pte, *pte, ptent;
+        spinlock_t *ptl;
+        LIST_HEAD(page_list);
+        struct page *page;
+
+        if (fatal_signal_pending(current))
+                return -EINTR;
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+        if (pmd_trans_huge(*pmd)) {
+                pmd_t orig_pmd;
+                unsigned long next = pmd_addr_end(addr, end);
+
+                tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
+                ptl = pmd_trans_huge_lock(pmd, vma);
+                if (!ptl)
+                        return 0;
+
+                orig_pmd = *pmd;
+                if (is_huge_zero_pmd(orig_pmd))
+                        goto huge_unlock;
+
+                if (unlikely(!pmd_present(orig_pmd))) {
+                        VM_BUG_ON(thp_migration_supported() &&
+                                        !is_pmd_migration_entry(orig_pmd));
+                        goto huge_unlock;
+                }
+
+                page = pmd_page(orig_pmd);
+                if (next - addr != HPAGE_PMD_SIZE) {
+                        int err;
+
+                        if (page_mapcount(page) != 1)
+                                goto huge_unlock;
+                        get_page(page);
+                        spin_unlock(ptl);
+                        lock_page(page);
+                        err = split_huge_page(page);
+                        unlock_page(page);
+                        put_page(page);
+                        if (!err)
+                                goto regular_page;
+                        return 0;
+                }
+
+                if (pmd_young(orig_pmd)) {
+                        pmdp_invalidate(vma, addr, pmd);
+                        orig_pmd = pmd_mkold(orig_pmd);
+
+                        set_pmd_at(mm, addr, pmd, orig_pmd);
+                        tlb_remove_tlb_entry(tlb, pmd, addr);
+                }
+
+                ClearPageReferenced(page);
+                test_and_clear_page_young(page);
+
+                if (!isolate_lru_page(page))
+                        list_add(&page->lru, &page_list);
+huge_unlock:
+                spin_unlock(ptl);
+                reclaim_pages(&page_list);
+                return 0;
+        }
+
+        if (pmd_trans_unstable(pmd))
+                return 0;
+regular_page:
+#endif
+        tlb_change_page_size(tlb, PAGE_SIZE);
+        orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
+        flush_tlb_batched_pending(mm);
+        arch_enter_lazy_mmu_mode();
+        for (; addr < end; pte++, addr += PAGE_SIZE) {
+                ptent = *pte;
+                if (!pte_present(ptent))
+                        continue;
+
+                page = vm_normal_page(vma, addr, ptent);
+                if (!page)
+                        continue;
+
+                /*
+                 * creating a THP page is expensive so split it only if we
+                 * are sure it's worth. Split it if we are only owner.
+                 */
+                if (PageTransCompound(page)) {
+                        if (page_mapcount(page) != 1)
+                                break;
+                        get_page(page);
+                        if (!trylock_page(page)) {
+                                put_page(page);
+                                break;
+                        }
+                        pte_unmap_unlock(orig_pte, ptl);
+                        if (split_huge_page(page)) {
+                                unlock_page(page);
+                                put_page(page);
+                                pte_offset_map_lock(mm, pmd, addr, &ptl);
+                                break;
+                        }
+                        unlock_page(page);
+                        put_page(page);
+                        pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
+                        pte--;
+                        addr -= PAGE_SIZE;
+                        continue;
+                }
+
+                VM_BUG_ON_PAGE(PageTransCompound(page), page);
+
+                if (pte_young(ptent)) {
+                        ptent = ptep_get_and_clear_full(mm, addr, pte,
+                                                        tlb->fullmm);
+                        ptent = pte_mkold(ptent);
+                        set_pte_at(mm, addr, pte, ptent);
+                        tlb_remove_tlb_entry(tlb, pte, addr);
+                }
+                ClearPageReferenced(page);
+                test_and_clear_page_young(page);
+
+                if (!isolate_lru_page(page))
+                        list_add(&page->lru, &page_list);
+        }
+
+        arch_leave_lazy_mmu_mode();
+        pte_unmap_unlock(orig_pte, ptl);
+        reclaim_pages(&page_list);
+        cond_resched();
+
+        return 0;
+}
+
+static void madvise_pageout_page_range(struct mmu_gather *tlb,
+                             struct vm_area_struct *vma,
+                             unsigned long addr, unsigned long end)
+{
+        tlb_start_vma(tlb, vma);
+        walk_page_range(vma->vm_mm, addr, end, &cold_walk_ops, NULL);
+        tlb_end_vma(tlb, vma);
+}
+
+static inline bool can_do_pageout(struct vm_area_struct *vma)
+{
+        if (vma_is_anonymous(vma))
+                return true;
+        if (!vma->vm_file)
+                return false;
+        /*
+         * paging out pagecache only for non-anonymous mappings that correspond
+         * to the files the calling process could (if tried) open for writing;
+         * otherwise we'd be including shared non-exclusive mappings, which
+         * opens a side channel.
+         */
+        return inode_owner_or_capable(file_inode(vma->vm_file)) ||
+                inode_permission(file_inode(vma->vm_file), MAY_WRITE) == 0;
+}
+
+static long madvise_pageout(struct vm_area_struct *vma,
+                        struct vm_area_struct **prev,
+                        unsigned long start_addr, unsigned long end_addr)
+{
+        struct mm_struct *mm = vma->vm_mm;
+        struct mmu_gather tlb;
+
+        *prev = vma;
+        if (!can_madv_lru_vma(vma))
+                return -EINVAL;
+
+        if (!can_do_pageout(vma))
+                return 0;
+
+        lru_add_drain();
+        tlb_gather_mmu(&tlb, mm, start_addr, end_addr);
+        madvise_pageout_page_range(&tlb, vma, start_addr, end_addr);
+        tlb_finish_mmu(&tlb, start_addr, end_addr);
+
+        return 0;
+}
+
 static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
                                 unsigned long end, struct mm_walk *walk)
 
@@ -843,6 +1029,8 @@ madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
                 return madvise_willneed(vma, prev, start, end);
         case MADV_COLD:
                 return madvise_cold(vma, prev, start, end);
+        case MADV_PAGEOUT:
+                return madvise_pageout(vma, prev, start, end);
         case MADV_FREE:
         case MADV_DONTNEED:
                 return madvise_dontneed_free(vma, prev, start, end, behavior);
@@ -865,6 +1053,7 @@ madvise_behavior_valid(int behavior)
         case MADV_DONTNEED:
         case MADV_FREE:
         case MADV_COLD:
+        case MADV_PAGEOUT:
 #ifdef CONFIG_KSM
         case MADV_MERGEABLE:
         case MADV_UNMERGEABLE:
diff --git a/mm/vmscan.c b/mm/vmscan.c
index d8bbaf068c35..e5d52d6a24af 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2145,6 +2145,62 @@ static void shrink_active_list(unsigned long nr_to_scan,
                         nr_deactivate, nr_rotated, sc->priority, file);
 }
 
+unsigned long reclaim_pages(struct list_head *page_list)
+{
+        int nid = -1;
+        unsigned long nr_reclaimed = 0;
+        LIST_HEAD(node_page_list);
+        struct reclaim_stat dummy_stat;
+        struct page *page;
+        struct scan_control sc = {
+                .gfp_mask = GFP_KERNEL,
+                .priority = DEF_PRIORITY,
+                .may_writepage = 1,
+                .may_unmap = 1,
+                .may_swap = 1,
+        };
+
+        while (!list_empty(page_list)) {
+                page = lru_to_page(page_list);
+                if (nid == -1) {
+                        nid = page_to_nid(page);
+                        INIT_LIST_HEAD(&node_page_list);
+                }
+
+                if (nid == page_to_nid(page)) {
+                        ClearPageActive(page);
+                        list_move(&page->lru, &node_page_list);
+                        continue;
+                }
+
+                nr_reclaimed += shrink_page_list(&node_page_list,
+                                                NODE_DATA(nid),
+                                                &sc, 0,
+                                                &dummy_stat, false);
+                while (!list_empty(&node_page_list)) {
+                        page = lru_to_page(&node_page_list);
+                        list_del(&page->lru);
+                        putback_lru_page(page);
+                }
+
+                nid = -1;
+        }
+
+        if (!list_empty(&node_page_list)) {
+                nr_reclaimed += shrink_page_list(&node_page_list,
+                                                NODE_DATA(nid),
+                                                &sc, 0,
+                                                &dummy_stat, false);
+                while (!list_empty(&node_page_list)) {
+                        page = lru_to_page(&node_page_list);
+                        list_del(&page->lru);
+                        putback_lru_page(page);
+                }
+        }
+
+        return nr_reclaimed;
+}
+
 /*
  * The inactive anon list should be small enough that the VM never has
  * to do too much work.