1 files changed, 68 insertions, 38 deletions

diff --git a/mm/swap.c b/mm/swap.c
index dfd7d71d6841..4a1d0d2c52fa 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -34,10 +34,13 @@
 
 #include "internal.h"
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/pagemap.h>
+
 /* How many pages do we try to swap or page in/out together? */
 int page_cluster;
 
-static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
+static DEFINE_PER_CPU(struct pagevec, lru_add_pvec);
 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
 
@@ -384,6 +387,7 @@ static void __activate_page(struct page *page, struct lruvec *lruvec,
                 SetPageActive(page);
                 lru += LRU_ACTIVE;
                 add_page_to_lru_list(page, lruvec, lru);
+                trace_mm_lru_activate(page, page_to_pfn(page));
 
                 __count_vm_event(PGACTIVATE);
                 update_page_reclaim_stat(lruvec, file, 1);
@@ -428,6 +432,33 @@ void activate_page(struct page *page)
 }
 #endif
 
+static void __lru_cache_activate_page(struct page *page)
+{
+        struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
+        int i;
+
+        /*
+         * Search backwards on the optimistic assumption that the page being
+         * activated has just been added to this pagevec. Note that only
+         * the local pagevec is examined as a !PageLRU page could be in the
+         * process of being released, reclaimed, migrated or on a remote
+         * pagevec that is currently being drained. Furthermore, marking
+         * a remote pagevec's page PageActive potentially hits a race where
+         * a page is marked PageActive just after it is added to the inactive
+         * list causing accounting errors and BUG_ON checks to trigger.
+         */
+        for (i = pagevec_count(pvec) - 1; i >= 0; i--) {
+                struct page *pagevec_page = pvec->pages[i];
+
+                if (pagevec_page == page) {
+                        SetPageActive(page);
+                        break;
+                }
+        }
+
+        put_cpu_var(lru_add_pvec);
+}
+
 /*
  * Mark a page as having seen activity.
  *
@@ -438,8 +469,18 @@ void activate_page(struct page *page)
 void mark_page_accessed(struct page *page)
 {
         if (!PageActive(page) && !PageUnevictable(page) &&
-                        PageReferenced(page) && PageLRU(page)) {
-                activate_page(page);
+                        PageReferenced(page)) {
+
+                /*
+                 * If the page is on the LRU, queue it for activation via
+                 * activate_page_pvecs. Otherwise, assume the page is on a
+                 * pagevec, mark it active and it'll be moved to the active
+                 * LRU on the next drain.
+                 */
+                if (PageLRU(page))
+                        activate_page(page);
+                else
+                        __lru_cache_activate_page(page);
                 ClearPageReferenced(page);
         } else if (!PageReferenced(page)) {
                 SetPageReferenced(page);
@@ -448,42 +489,37 @@ void mark_page_accessed(struct page *page)
 EXPORT_SYMBOL(mark_page_accessed);
 
 /*
- * Order of operations is important: flush the pagevec when it's already
- * full, not when adding the last page, to make sure that last page is
- * not added to the LRU directly when passed to this function. Because
- * mark_page_accessed() (called after this when writing) only activates
- * pages that are on the LRU, linear writes in subpage chunks would see
- * every PAGEVEC_SIZE page activated, which is unexpected.
+ * Queue the page for addition to the LRU via pagevec. The decision on whether
+ * to add the page to the [in]active [file|anon] list is deferred until the
+ * pagevec is drained. This gives a chance for the caller of __lru_cache_add()
+ * have the page added to the active list using mark_page_accessed().
  */
-void __lru_cache_add(struct page *page, enum lru_list lru)
+void __lru_cache_add(struct page *page)
 {
-        struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru];
+        struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
 
         page_cache_get(page);
         if (!pagevec_space(pvec))
-                __pagevec_lru_add(pvec, lru);
+                __pagevec_lru_add(pvec);
         pagevec_add(pvec, page);
-        put_cpu_var(lru_add_pvecs);
+        put_cpu_var(lru_add_pvec);
 }
 EXPORT_SYMBOL(__lru_cache_add);
 
 /**
- * lru_cache_add_lru - add a page to a page list
+ * lru_cache_add - add a page to a page list
  * @page: the page to be added to the LRU.
- * @lru: the LRU list to which the page is added.
  */
-void lru_cache_add_lru(struct page *page, enum lru_list lru)
+void lru_cache_add(struct page *page)
 {
         if (PageActive(page)) {
                 VM_BUG_ON(PageUnevictable(page));
-                ClearPageActive(page);
         } else if (PageUnevictable(page)) {
                 VM_BUG_ON(PageActive(page));
-                ClearPageUnevictable(page);
         }
 
-        VM_BUG_ON(PageLRU(page) || PageActive(page) || PageUnevictable(page));
-        __lru_cache_add(page, lru);
+        VM_BUG_ON(PageLRU(page));
+        __lru_cache_add(page);
 }
 
 /**
@@ -583,15 +619,10 @@ static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
  */
 void lru_add_drain_cpu(int cpu)
 {
-        struct pagevec *pvecs = per_cpu(lru_add_pvecs, cpu);
-        struct pagevec *pvec;
-        int lru;
+        struct pagevec *pvec = &per_cpu(lru_add_pvec, cpu);
 
-        for_each_lru(lru) {
-                pvec = &pvecs[lru - LRU_BASE];
-                if (pagevec_count(pvec))
-                        __pagevec_lru_add(pvec, lru);
-        }
+        if (pagevec_count(pvec))
+                __pagevec_lru_add(pvec);
 
         pvec = &per_cpu(lru_rotate_pvecs, cpu);
         if (pagevec_count(pvec)) {
@@ -708,6 +739,9 @@ void release_pages(struct page **pages, int nr, int cold)
                         del_page_from_lru_list(page, lruvec, page_off_lru(page));
                 }
 
+                /* Clear Active bit in case of parallel mark_page_accessed */
+                ClearPageActive(page);
+
                 list_add(&page->lru, &pages_to_free);
         }
         if (zone)
@@ -795,30 +829,26 @@ void lru_add_page_tail(struct page *page, struct page *page_tail,
 static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
                                  void *arg)
 {
-        enum lru_list lru = (enum lru_list)arg;
-        int file = is_file_lru(lru);
-        int active = is_active_lru(lru);
+        int file = page_is_file_cache(page);
+        int active = PageActive(page);
+        enum lru_list lru = page_lru(page);
 
-        VM_BUG_ON(PageActive(page));
         VM_BUG_ON(PageUnevictable(page));
         VM_BUG_ON(PageLRU(page));
 
         SetPageLRU(page);
-        if (active)
-                SetPageActive(page);
         add_page_to_lru_list(page, lruvec, lru);
         update_page_reclaim_stat(lruvec, file, active);
+        trace_mm_lru_insertion(page, page_to_pfn(page), lru, trace_pagemap_flags(page));
 }
 
 /*
  * Add the passed pages to the LRU, then drop the caller's refcount
  * on them.  Reinitialises the caller's pagevec.
  */
-void __pagevec_lru_add(struct pagevec *pvec, enum lru_list lru)
+void __pagevec_lru_add(struct pagevec *pvec)
 {
-        VM_BUG_ON(is_unevictable_lru(lru));
-
-        pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, (void *)lru);
+        pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL);
 }
 EXPORT_SYMBOL(__pagevec_lru_add);