4 files changed, 54 insertions, 95 deletions

diff --git a/include/linux/swap.h b/include/linux/swap.h
index 7b6a59f722a3..a1a3f4ed94ce 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -337,8 +337,6 @@ extern void deactivate_file_page(struct page *page);
 extern void mark_page_lazyfree(struct page *page);
 extern void swap_setup(void);
 
-extern void add_page_to_unevictable_list(struct page *page);
-
 extern void lru_cache_add_active_or_unevictable(struct page *page,
                                                 struct vm_area_struct *vma);
 
diff --git a/mm/mlock.c b/mm/mlock.c
index 79398200e423..74e5a6547c3d 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -64,6 +64,12 @@ void clear_page_mlock(struct page *page)
         mod_zone_page_state(page_zone(page), NR_MLOCK,
                             -hpage_nr_pages(page));
         count_vm_event(UNEVICTABLE_PGCLEARED);
+        /*
+         * The previous TestClearPageMlocked() corresponds to the smp_mb()
+         * in __pagevec_lru_add_fn().
+         *
+         * See __pagevec_lru_add_fn for more explanation.
+         */
         if (!isolate_lru_page(page)) {
                 putback_lru_page(page);
         } else {
diff --git a/mm/swap.c b/mm/swap.c
index 567a7b96e41d..2d337710218f 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -446,30 +446,6 @@ void lru_cache_add(struct page *page)
 }
 
 /**
- * add_page_to_unevictable_list - add a page to the unevictable list
- * @page:  the page to be added to the unevictable list
- *
- * Add page directly to its zone's unevictable list.  To avoid races with
- * tasks that might be making the page evictable, through eg. munlock,
- * munmap or exit, while it's not on the lru, we want to add the page
- * while it's locked or otherwise "invisible" to other tasks.  This is
- * difficult to do when using the pagevec cache, so bypass that.
- */
-void add_page_to_unevictable_list(struct page *page)
-{
-        struct pglist_data *pgdat = page_pgdat(page);
-        struct lruvec *lruvec;
-
-        spin_lock_irq(&pgdat->lru_lock);
-        lruvec = mem_cgroup_page_lruvec(page, pgdat);
-        ClearPageActive(page);
-        SetPageUnevictable(page);
-        SetPageLRU(page);
-        add_page_to_lru_list(page, lruvec, LRU_UNEVICTABLE);
-        spin_unlock_irq(&pgdat->lru_lock);
-}
-
-/**
  * lru_cache_add_active_or_unevictable
  * @page:  the page to be added to LRU
  * @vma:   vma in which page is mapped for determining reclaimability
@@ -484,13 +460,9 @@ void lru_cache_add_active_or_unevictable(struct page *page,
 {
         VM_BUG_ON_PAGE(PageLRU(page), page);
 
-        if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED)) {
+        if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED))
                 SetPageActive(page);
-                lru_cache_add(page);
-                return;
-        }
-
-        if (!TestSetPageMlocked(page)) {
+        else if (!TestSetPageMlocked(page)) {
                 /*
                  * We use the irq-unsafe __mod_zone_page_stat because this
                  * counter is not modified from interrupt context, and the pte
@@ -500,7 +472,7 @@ void lru_cache_add_active_or_unevictable(struct page *page,
                                     hpage_nr_pages(page));
                 count_vm_event(UNEVICTABLE_PGMLOCKED);
         }
-        add_page_to_unevictable_list(page);
+        lru_cache_add(page);
 }
 
 /*
@@ -886,15 +858,55 @@ void lru_add_page_tail(struct page *page, struct page *page_tail,
 static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
                                  void *arg)
 {
-        int file = page_is_file_cache(page);
-        int active = PageActive(page);
-        enum lru_list lru = page_lru(page);
+        enum lru_list lru;
+        int was_unevictable = TestClearPageUnevictable(page);
 
         VM_BUG_ON_PAGE(PageLRU(page), page);
 
         SetPageLRU(page);
+        /*
+         * Page becomes evictable in two ways:
+         * 1) Within LRU lock [munlock_vma_pages() and __munlock_pagevec()].
+         * 2) Before acquiring LRU lock to put the page to correct LRU and then
+         *   a) do PageLRU check with lock [check_move_unevictable_pages]
+         *   b) do PageLRU check before lock [clear_page_mlock]
+         *
+         * (1) & (2a) are ok as LRU lock will serialize them. For (2b), we need
+         * following strict ordering:
+         *
+         * #0: __pagevec_lru_add_fn             #1: clear_page_mlock
+         *
+         * SetPageLRU()                         TestClearPageMlocked()
+         * smp_mb() // explicit ordering        // above provides strict
+         *                                      // ordering
+         * PageMlocked()                        PageLRU()
+         *
+         *
+         * if '#1' does not observe setting of PG_lru by '#0' and fails
+         * isolation, the explicit barrier will make sure that page_evictable
+         * check will put the page in correct LRU. Without smp_mb(), SetPageLRU
+         * can be reordered after PageMlocked check and can make '#1' to fail
+         * the isolation of the page whose Mlocked bit is cleared (#0 is also
+         * looking at the same page) and the evictable page will be stranded
+         * in an unevictable LRU.
+         */
+        smp_mb();
+
+        if (page_evictable(page)) {
+                lru = page_lru(page);
+                update_page_reclaim_stat(lruvec, page_is_file_cache(page),
+                                         PageActive(page));
+                if (was_unevictable)
+                        count_vm_event(UNEVICTABLE_PGRESCUED);
+        } else {
+                lru = LRU_UNEVICTABLE;
+                ClearPageActive(page);
+                SetPageUnevictable(page);
+                if (!was_unevictable)
+                        count_vm_event(UNEVICTABLE_PGCULLED);
+        }
+
         add_page_to_lru_list(page, lruvec, lru);
-        update_page_reclaim_stat(lruvec, file, active);
         trace_mm_lru_insertion(page, lru);
 }
 
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 444749669187..bee53495a829 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -769,64 +769,7 @@ int remove_mapping(struct address_space *mapping, struct page *page)
  */
 void putback_lru_page(struct page *page)
 {
-        bool is_unevictable;
-        int was_unevictable = PageUnevictable(page);
-
-        VM_BUG_ON_PAGE(PageLRU(page), page);
-
-redo:
-        ClearPageUnevictable(page);
-
-        if (page_evictable(page)) {
-                /*
-                 * For evictable pages, we can use the cache.
-                 * In event of a race, worst case is we end up with an
-                 * unevictable page on [in]active list.
-                 * We know how to handle that.
-                 */
-                is_unevictable = false;
-                lru_cache_add(page);
-        } else {
-                /*
-                 * Put unevictable pages directly on zone's unevictable
-                 * list.
-                 */
-                is_unevictable = true;
-                add_page_to_unevictable_list(page);
-                /*
-                 * When racing with an mlock or AS_UNEVICTABLE clearing
-                 * (page is unlocked) make sure that if the other thread
-                 * does not observe our setting of PG_lru and fails
-                 * isolation/check_move_unevictable_pages,
-                 * we see PG_mlocked/AS_UNEVICTABLE cleared below and move
-                 * the page back to the evictable list.
-                 *
-                 * The other side is TestClearPageMlocked() or shmem_lock().
-                 */
-                smp_mb();
-        }
-
-        /*
-         * page's status can change while we move it among lru. If an evictable
-         * page is on unevictable list, it never be freed. To avoid that,
-         * check after we added it to the list, again.
-         */
-        if (is_unevictable && page_evictable(page)) {
-                if (!isolate_lru_page(page)) {
-                        put_page(page);
-                        goto redo;
-                }
-                /* This means someone else dropped this page from LRU
-                 * So, it will be freed or putback to LRU again. There is
-                 * nothing to do here.
-                 */
-        }
-
-        if (was_unevictable && !is_unevictable)
-                count_vm_event(UNEVICTABLE_PGRESCUED);
-        else if (!was_unevictable && is_unevictable)
-                count_vm_event(UNEVICTABLE_PGCULLED);
-
+        lru_cache_add(page);
         put_page(page);         /* drop ref from isolate */
 }