radix-tree: use iterators in find_get_pages* functions

Replace radix_tree_gang_lookup_slot() and
radix_tree_gang_lookup_tag_slot() in page-cache lookup functions with
brand-new radix-tree direct iterating.  This avoids the double-scanning
and pointer copying.

Iterator don't stop after nr_pages page-get fails in a row, it continue
lookup till the radix-tree end.  Thus we can safely remove these restart
conditions.

Unfortunately, old implementation didn't forbid nr_pages == 0, this corner
case does not fit into new code, so the patch adds an extra check at the
beginning.

Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org>
Tested-by: Hugh Dickins <hughd@google.com>
Cc: Christoph Hellwig <hch@lst.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 38 insertions, 48 deletions

diff --git a/mm/filemap.c b/mm/filemap.c
index c3811bc6b9e..79c4b2b0b14 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -813,20 +813,19 @@ EXPORT_SYMBOL(find_or_create_page);
 unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
                             unsigned int nr_pages, struct page **pages)
 {
-        unsigned int i;
-        unsigned int ret;
-        unsigned int nr_found, nr_skip;
+        struct radix_tree_iter iter;
+        void **slot;
+        unsigned ret = 0;
+
+        if (unlikely(!nr_pages))
+                return 0;
 
         rcu_read_lock();
 restart:
-        nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
-                                (void ***)pages, NULL, start, nr_pages);
-        ret = 0;
-        nr_skip = 0;
-        for (i = 0; i < nr_found; i++) {
+        radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
                 struct page *page;
 repeat:
-                page = radix_tree_deref_slot((void **)pages[i]);
+                page = radix_tree_deref_slot(slot);
                 if (unlikely(!page))
                         continue;
 
@@ -837,7 +836,7 @@ repeat:
                                  * when entry at index 0 moves out of or back
                                  * to root: none yet gotten, safe to restart.
                                  */
-                                WARN_ON(start | i);
+                                WARN_ON(iter.index);
                                 goto restart;
                         }
                         /*
@@ -845,7 +844,6 @@ repeat:
                          * here as an exceptional entry: so skip over it -
                          * we only reach this from invalidate_mapping_pages().
                          */
-                        nr_skip++;
                         continue;
                 }
 
@@ -853,21 +851,16 @@ repeat:
                         goto repeat;
 
                 /* Has the page moved? */
-                if (unlikely(page != *((void **)pages[i]))) {
+                if (unlikely(page != *slot)) {
                         page_cache_release(page);
                         goto repeat;
                 }
 
                 pages[ret] = page;
-                ret++;
+                if (++ret == nr_pages)
+                        break;
         }
 
-        /*
-         * If all entries were removed before we could secure them,
-         * try again, because callers stop trying once 0 is returned.
-         */
-        if (unlikely(!ret && nr_found > nr_skip))
-                goto restart;
         rcu_read_unlock();
         return ret;
 }
@@ -887,21 +880,22 @@ repeat:
 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
                                unsigned int nr_pages, struct page **pages)
 {
-        unsigned int i;
-        unsigned int ret;
-        unsigned int nr_found;
+        struct radix_tree_iter iter;
+        void **slot;
+        unsigned int ret = 0;
+
+        if (unlikely(!nr_pages))
+                return 0;
 
         rcu_read_lock();
 restart:
-        nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
-                                (void ***)pages, NULL, index, nr_pages);
-        ret = 0;
-        for (i = 0; i < nr_found; i++) {
+        radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
                 struct page *page;
 repeat:
-                page = radix_tree_deref_slot((void **)pages[i]);
+                page = radix_tree_deref_slot(slot);
+                /* The hole, there no reason to continue */
                 if (unlikely(!page))
-                        continue;
+                        break;
 
                 if (radix_tree_exception(page)) {
                         if (radix_tree_deref_retry(page)) {
@@ -924,7 +918,7 @@ repeat:
                         goto repeat;
 
                 /* Has the page moved? */
-                if (unlikely(page != *((void **)pages[i]))) {
+                if (unlikely(page != *slot)) {
                         page_cache_release(page);
                         goto repeat;
                 }
@@ -934,14 +928,14 @@ repeat:
                  * otherwise we can get both false positives and false
                  * negatives, which is just confusing to the caller.
                  */
-                if (page->mapping == NULL || page->index != index) {
+                if (page->mapping == NULL || page->index != iter.index) {
                         page_cache_release(page);
                         break;
                 }
 
                 pages[ret] = page;
-                ret++;
-                index++;
+                if (++ret == nr_pages)
+                        break;
         }
         rcu_read_unlock();
         return ret;
@@ -962,19 +956,20 @@ EXPORT_SYMBOL(find_get_pages_contig);
 unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
                         int tag, unsigned int nr_pages, struct page **pages)
 {
-        unsigned int i;
-        unsigned int ret;
-        unsigned int nr_found;
+        struct radix_tree_iter iter;
+        void **slot;
+        unsigned ret = 0;
+
+        if (unlikely(!nr_pages))
+                return 0;
 
         rcu_read_lock();
 restart:
-        nr_found = radix_tree_gang_lookup_tag_slot(&mapping->page_tree,
-                                (void ***)pages, *index, nr_pages, tag);
-        ret = 0;
-        for (i = 0; i < nr_found; i++) {
+        radix_tree_for_each_tagged(slot, &mapping->page_tree,
+                                   &iter, *index, tag) {
                 struct page *page;
 repeat:
-                page = radix_tree_deref_slot((void **)pages[i]);
+                page = radix_tree_deref_slot(slot);
                 if (unlikely(!page))
                         continue;
 
@@ -998,21 +993,16 @@ repeat:
                         goto repeat;
 
                 /* Has the page moved? */
-                if (unlikely(page != *((void **)pages[i]))) {
+                if (unlikely(page != *slot)) {
                         page_cache_release(page);
                         goto repeat;
                 }
 
                 pages[ret] = page;
-                ret++;
+                if (++ret == nr_pages)
+                        break;
         }
 
-        /*
-         * If all entries were removed before we could secure them,
-         * try again, because callers stop trying once 0 is returned.
-         */
-        if (unlikely(!ret && nr_found))
-                goto restart;
         rcu_read_unlock();
 
         if (ret)