mm + fs: prepare for non-page entries in page cache radix trees

shmem mappings already contain exceptional entries where swap slot
information is remembered.

To be able to store eviction information for regular page cache, prepare
every site dealing with the radix trees directly to handle entries other
than pages.

The common lookup functions will filter out non-page entries and return
NULL for page cache holes, just as before.  But provide a raw version of
the API which returns non-page entries as well, and switch shmem over to
use it.

Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Minchan Kim <minchan@kernel.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Bob Liu <bob.liu@oracle.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jan Kara <jack@suse.cz>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Luigi Semenzato <semenzato@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Metin Doslu <metin@citusdata.com>
Cc: Michel Lespinasse <walken@google.com>
Cc: Ozgun Erdogan <ozgun@citusdata.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Roman Gushchin <klamm@yandex-team.ru>
Cc: Ryan Mallon <rmallon@gmail.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 61 insertions, 13 deletions

diff --git a/mm/truncate.c b/mm/truncate.c
index 353b683afd6e..2e84fe59190b 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -22,6 +22,22 @@
 #include <linux/cleancache.h>
 #include "internal.h"
 
+static void clear_exceptional_entry(struct address_space *mapping,
+                                    pgoff_t index, void *entry)
+{
+        /* Handled by shmem itself */
+        if (shmem_mapping(mapping))
+                return;
+
+        spin_lock_irq(&mapping->tree_lock);
+        /*
+         * Regular page slots are stabilized by the page lock even
+         * without the tree itself locked.  These unlocked entries
+         * need verification under the tree lock.
+         */
+        radix_tree_delete_item(&mapping->page_tree, index, entry);
+        spin_unlock_irq(&mapping->tree_lock);
+}
 
 /**
  * do_invalidatepage - invalidate part or all of a page
@@ -208,6 +224,7 @@ void truncate_inode_pages_range(struct address_space *mapping,
         unsigned int    partial_start;  /* inclusive */
         unsigned int    partial_end;    /* exclusive */
         struct pagevec  pvec;
+        pgoff_t         indices[PAGEVEC_SIZE];
         pgoff_t         index;
         int             i;
 
@@ -238,17 +255,23 @@ void truncate_inode_pages_range(struct address_space *mapping,
 
         pagevec_init(&pvec, 0);
         index = start;
-        while (index < end && pagevec_lookup(&pvec, mapping, index,
-                        min(end - index, (pgoff_t)PAGEVEC_SIZE))) {
+        while (index < end && pagevec_lookup_entries(&pvec, mapping, index,
+                        min(end - index, (pgoff_t)PAGEVEC_SIZE),
+                        indices)) {
                 mem_cgroup_uncharge_start();
                 for (i = 0; i < pagevec_count(&pvec); i++) {
                         struct page *page = pvec.pages[i];
 
                         /* We rely upon deletion not changing page->index */
-                        index = page->index;
+                        index = indices[i];
                         if (index >= end)
                                 break;
 
+                        if (radix_tree_exceptional_entry(page)) {
+                                clear_exceptional_entry(mapping, index, page);
+                                continue;
+                        }
+
                         if (!trylock_page(page))
                                 continue;
                         WARN_ON(page->index != index);
@@ -259,6 +282,7 @@ void truncate_inode_pages_range(struct address_space *mapping,
                         truncate_inode_page(mapping, page);
                         unlock_page(page);
                 }
+                pagevec_remove_exceptionals(&pvec);
                 pagevec_release(&pvec);
                 mem_cgroup_uncharge_end();
                 cond_resched();
@@ -307,14 +331,16 @@ void truncate_inode_pages_range(struct address_space *mapping,
         index = start;
         for ( ; ; ) {
                 cond_resched();
-                if (!pagevec_lookup(&pvec, mapping, index,
-                        min(end - index, (pgoff_t)PAGEVEC_SIZE))) {
+                if (!pagevec_lookup_entries(&pvec, mapping, index,
+                        min(end - index, (pgoff_t)PAGEVEC_SIZE),
+                        indices)) {
                         if (index == start)
                                 break;
                         index = start;
                         continue;
                 }
-                if (index == start && pvec.pages[0]->index >= end) {
+                if (index == start && indices[0] >= end) {
+                        pagevec_remove_exceptionals(&pvec);
                         pagevec_release(&pvec);
                         break;
                 }
@@ -323,16 +349,22 @@ void truncate_inode_pages_range(struct address_space *mapping,
                         struct page *page = pvec.pages[i];
 
                         /* We rely upon deletion not changing page->index */
-                        index = page->index;
+                        index = indices[i];
                         if (index >= end)
                                 break;
 
+                        if (radix_tree_exceptional_entry(page)) {
+                                clear_exceptional_entry(mapping, index, page);
+                                continue;
+                        }
+
                         lock_page(page);
                         WARN_ON(page->index != index);
                         wait_on_page_writeback(page);
                         truncate_inode_page(mapping, page);
                         unlock_page(page);
                 }
+                pagevec_remove_exceptionals(&pvec);
                 pagevec_release(&pvec);
                 mem_cgroup_uncharge_end();
                 index++;
@@ -375,6 +407,7 @@ EXPORT_SYMBOL(truncate_inode_pages);
 unsigned long invalidate_mapping_pages(struct address_space *mapping,
                 pgoff_t start, pgoff_t end)
 {
+        pgoff_t indices[PAGEVEC_SIZE];
         struct pagevec pvec;
         pgoff_t index = start;
         unsigned long ret;
@@ -390,17 +423,23 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping,
          */
 
         pagevec_init(&pvec, 0);
-        while (index <= end && pagevec_lookup(&pvec, mapping, index,
-                        min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
+        while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
+                        min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
+                        indices)) {
                 mem_cgroup_uncharge_start();
                 for (i = 0; i < pagevec_count(&pvec); i++) {
                         struct page *page = pvec.pages[i];
 
                         /* We rely upon deletion not changing page->index */
-                        index = page->index;
+                        index = indices[i];
                         if (index > end)
                                 break;
 
+                        if (radix_tree_exceptional_entry(page)) {
+                                clear_exceptional_entry(mapping, index, page);
+                                continue;
+                        }
+
                         if (!trylock_page(page))
                                 continue;
                         WARN_ON(page->index != index);
@@ -414,6 +453,7 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping,
                                 deactivate_page(page);
                         count += ret;
                 }
+                pagevec_remove_exceptionals(&pvec);
                 pagevec_release(&pvec);
                 mem_cgroup_uncharge_end();
                 cond_resched();
@@ -481,6 +521,7 @@ static int do_launder_page(struct address_space *mapping, struct page *page)
 int invalidate_inode_pages2_range(struct address_space *mapping,
                                   pgoff_t start, pgoff_t end)
 {
+        pgoff_t indices[PAGEVEC_SIZE];
         struct pagevec pvec;
         pgoff_t index;
         int i;
@@ -491,17 +532,23 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
         cleancache_invalidate_inode(mapping);
         pagevec_init(&pvec, 0);
         index = start;
-        while (index <= end && pagevec_lookup(&pvec, mapping, index,
-                        min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
+        while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
+                        min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
+                        indices)) {
                 mem_cgroup_uncharge_start();
                 for (i = 0; i < pagevec_count(&pvec); i++) {
                         struct page *page = pvec.pages[i];
 
                         /* We rely upon deletion not changing page->index */
-                        index = page->index;
+                        index = indices[i];
                         if (index > end)
                                 break;
 
+                        if (radix_tree_exceptional_entry(page)) {
+                                clear_exceptional_entry(mapping, index, page);
+                                continue;
+                        }
+
                         lock_page(page);
                         WARN_ON(page->index != index);
                         if (page->mapping != mapping) {
@@ -539,6 +586,7 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
                                 ret = ret2;
                         unlock_page(page);
                 }
+                pagevec_remove_exceptionals(&pvec);
                 pagevec_release(&pvec);
                 mem_cgroup_uncharge_end();
                 cond_resched();