mm + fs: prepare for non-page entries in page cache radix trees

shmem mappings already contain exceptional entries where swap slot
information is remembered.

To be able to store eviction information for regular page cache, prepare
every site dealing with the radix trees directly to handle entries other
than pages.

The common lookup functions will filter out non-page entries and return
NULL for page cache holes, just as before.  But provide a raw version of
the API which returns non-page entries as well, and switch shmem over to
use it.

Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Reviewed-by: Rik van Riel <riel@redhat.com>
Reviewed-by: Minchan Kim <minchan@kernel.org>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Bob Liu <bob.liu@oracle.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Jan Kara <jack@suse.cz>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Luigi Semenzato <semenzato@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Metin Doslu <metin@citusdata.com>
Cc: Michel Lespinasse <walken@google.com>
Cc: Ozgun Erdogan <ozgun@citusdata.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Roman Gushchin <klamm@yandex-team.ru>
Cc: Ryan Mallon <rmallon@gmail.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 178 insertions, 24 deletions

diff --git a/mm/filemap.c b/mm/filemap.c
index 40115c6c0791..efc63876477f 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -446,6 +446,29 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
 }
 EXPORT_SYMBOL_GPL(replace_page_cache_page);
 
+static int page_cache_tree_insert(struct address_space *mapping,
+                                  struct page *page)
+{
+        void **slot;
+        int error;
+
+        slot = radix_tree_lookup_slot(&mapping->page_tree, page->index);
+        if (slot) {
+                void *p;
+
+                p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
+                if (!radix_tree_exceptional_entry(p))
+                        return -EEXIST;
+                radix_tree_replace_slot(slot, page);
+                mapping->nrpages++;
+                return 0;
+        }
+        error = radix_tree_insert(&mapping->page_tree, page->index, page);
+        if (!error)
+                mapping->nrpages++;
+        return error;
+}
+
 /**
  * add_to_page_cache_locked - add a locked page to the pagecache
  * @page:       page to add
@@ -480,11 +503,10 @@ int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
         page->index = offset;
 
         spin_lock_irq(&mapping->tree_lock);
-        error = radix_tree_insert(&mapping->page_tree, offset, page);
+        error = page_cache_tree_insert(mapping, page);
         radix_tree_preload_end();
         if (unlikely(error))
                 goto err_insert;
-        mapping->nrpages++;
         __inc_zone_page_state(page, NR_FILE_PAGES);
         spin_unlock_irq(&mapping->tree_lock);
         trace_mm_filemap_add_to_page_cache(page);
@@ -712,7 +734,10 @@ pgoff_t page_cache_next_hole(struct address_space *mapping,
         unsigned long i;
 
         for (i = 0; i < max_scan; i++) {
-                if (!radix_tree_lookup(&mapping->page_tree, index))
+                struct page *page;
+
+                page = radix_tree_lookup(&mapping->page_tree, index);
+                if (!page || radix_tree_exceptional_entry(page))
                         break;
                 index++;
                 if (index == 0)
@@ -750,7 +775,10 @@ pgoff_t page_cache_prev_hole(struct address_space *mapping,
         unsigned long i;
 
         for (i = 0; i < max_scan; i++) {
-                if (!radix_tree_lookup(&mapping->page_tree, index))
+                struct page *page;
+
+                page = radix_tree_lookup(&mapping->page_tree, index);
+                if (!page || radix_tree_exceptional_entry(page))
                         break;
                 index--;
                 if (index == ULONG_MAX)
@@ -762,14 +790,19 @@ pgoff_t page_cache_prev_hole(struct address_space *mapping,
 EXPORT_SYMBOL(page_cache_prev_hole);
 
 /**
- * find_get_page - find and get a page reference
+ * find_get_entry - find and get a page cache entry
  * @mapping: the address_space to search
- * @offset: the page index
+ * @offset: the page cache index
+ *
+ * Looks up the page cache slot at @mapping & @offset.  If there is a
+ * page cache page, it is returned with an increased refcount.
  *
- * Is there a pagecache struct page at the given (mapping, offset) tuple?
- * If yes, increment its refcount and return it; if no, return NULL.
+ * If the slot holds a shadow entry of a previously evicted page, it
+ * is returned.
+ *
+ * Otherwise, %NULL is returned.
  */
-struct page *find_get_page(struct address_space *mapping, pgoff_t offset)
+struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
 {
         void **pagep;
         struct page *page;
@@ -810,24 +843,50 @@ out:
 
         return page;
 }
-EXPORT_SYMBOL(find_get_page);
+EXPORT_SYMBOL(find_get_entry);
 
 /**
- * find_lock_page - locate, pin and lock a pagecache page
+ * find_get_page - find and get a page reference
  * @mapping: the address_space to search
  * @offset: the page index
  *
- * Locates the desired pagecache page, locks it, increments its reference
- * count and returns its address.
+ * Looks up the page cache slot at @mapping & @offset.  If there is a
+ * page cache page, it is returned with an increased refcount.
  *
- * Returns zero if the page was not present. find_lock_page() may sleep.
+ * Otherwise, %NULL is returned.
  */
-struct page *find_lock_page(struct address_space *mapping, pgoff_t offset)
+struct page *find_get_page(struct address_space *mapping, pgoff_t offset)
+{
+        struct page *page = find_get_entry(mapping, offset);
+
+        if (radix_tree_exceptional_entry(page))
+                page = NULL;
+        return page;
+}
+EXPORT_SYMBOL(find_get_page);
+
+/**
+ * find_lock_entry - locate, pin and lock a page cache entry
+ * @mapping: the address_space to search
+ * @offset: the page cache index
+ *
+ * Looks up the page cache slot at @mapping & @offset.  If there is a
+ * page cache page, it is returned locked and with an increased
+ * refcount.
+ *
+ * If the slot holds a shadow entry of a previously evicted page, it
+ * is returned.
+ *
+ * Otherwise, %NULL is returned.
+ *
+ * find_lock_entry() may sleep.
+ */
+struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)
 {
         struct page *page;
 
 repeat:
-        page = find_get_page(mapping, offset);
+        page = find_get_entry(mapping, offset);
         if (page && !radix_tree_exception(page)) {
                 lock_page(page);
                 /* Has the page been truncated? */
@@ -840,6 +899,29 @@ repeat:
         }
         return page;
 }
+EXPORT_SYMBOL(find_lock_entry);
+
+/**
+ * find_lock_page - locate, pin and lock a pagecache page
+ * @mapping: the address_space to search
+ * @offset: the page index
+ *
+ * Looks up the page cache slot at @mapping & @offset.  If there is a
+ * page cache page, it is returned locked and with an increased
+ * refcount.
+ *
+ * Otherwise, %NULL is returned.
+ *
+ * find_lock_page() may sleep.
+ */
+struct page *find_lock_page(struct address_space *mapping, pgoff_t offset)
+{
+        struct page *page = find_lock_entry(mapping, offset);
+
+        if (radix_tree_exceptional_entry(page))
+                page = NULL;
+        return page;
+}
 EXPORT_SYMBOL(find_lock_page);
 
 /**
@@ -848,16 +930,18 @@ EXPORT_SYMBOL(find_lock_page);
  * @index: the page's index into the mapping
  * @gfp_mask: page allocation mode
  *
- * Locates a page in the pagecache.  If the page is not present, a new page
- * is allocated using @gfp_mask and is added to the pagecache and to the VM's
- * LRU list.  The returned page is locked and has its reference count
- * incremented.
+ * Looks up the page cache slot at @mapping & @offset.  If there is a
+ * page cache page, it is returned locked and with an increased
+ * refcount.
+ *
+ * If the page is not present, a new page is allocated using @gfp_mask
+ * and added to the page cache and the VM's LRU list.  The page is
+ * returned locked and with an increased refcount.
  *
- * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic
- * allocation!
+ * On memory exhaustion, %NULL is returned.
  *
- * find_or_create_page() returns the desired page's address, or zero on
- * memory exhaustion.
+ * find_or_create_page() may sleep, even if @gfp_flags specifies an
+ * atomic allocation!
  */
 struct page *find_or_create_page(struct address_space *mapping,
                 pgoff_t index, gfp_t gfp_mask)
@@ -890,6 +974,76 @@ repeat:
 EXPORT_SYMBOL(find_or_create_page);
 
 /**
+ * find_get_entries - gang pagecache lookup
+ * @mapping:    The address_space to search
+ * @start:      The starting page cache index
+ * @nr_entries: The maximum number of entries
+ * @entries:    Where the resulting entries are placed
+ * @indices:    The cache indices corresponding to the entries in @entries
+ *
+ * find_get_entries() will search for and return a group of up to
+ * @nr_entries entries in the mapping.  The entries are placed at
+ * @entries.  find_get_entries() takes a reference against any actual
+ * pages it returns.
+ *
+ * The search returns a group of mapping-contiguous page cache entries
+ * with ascending indexes.  There may be holes in the indices due to
+ * not-present pages.
+ *
+ * Any shadow entries of evicted pages are included in the returned
+ * array.
+ *
+ * find_get_entries() returns the number of pages and shadow entries
+ * which were found.
+ */
+unsigned find_get_entries(struct address_space *mapping,
+                          pgoff_t start, unsigned int nr_entries,
+                          struct page **entries, pgoff_t *indices)
+{
+        void **slot;
+        unsigned int ret = 0;
+        struct radix_tree_iter iter;
+
+        if (!nr_entries)
+                return 0;
+
+        rcu_read_lock();
+restart:
+        radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
+                struct page *page;
+repeat:
+                page = radix_tree_deref_slot(slot);
+                if (unlikely(!page))
+                        continue;
+                if (radix_tree_exception(page)) {
+                        if (radix_tree_deref_retry(page))
+                                goto restart;
+                        /*
+                         * Otherwise, we must be storing a swap entry
+                         * here as an exceptional entry: so return it
+                         * without attempting to raise page count.
+                         */
+                        goto export;
+                }
+                if (!page_cache_get_speculative(page))
+                        goto repeat;
+
+                /* Has the page moved? */
+                if (unlikely(page != *slot)) {
+                        page_cache_release(page);
+                        goto repeat;
+                }
+export:
+                indices[ret] = iter.index;
+                entries[ret] = page;
+                if (++ret == nr_entries)
+                        break;
+        }
+        rcu_read_unlock();
+        return ret;
+}
+
+/**
  * find_get_pages - gang pagecache lookup
  * @mapping:    The address_space to search
  * @start:      The starting page index