1 files changed, 161 insertions, 67 deletions

diff --git a/mm/filemap.c b/mm/filemap.c
index 2d3ec1ffc66e..5de7633e1dbe 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -109,7 +109,7 @@
 /*
  * Remove a page from the page cache and free it. Caller has to make
  * sure the page is locked and that nobody else uses it - or that usage
- * is safe.  The caller must hold a write_lock on the mapping's tree_lock.
+ * is safe.  The caller must hold the mapping's tree_lock.
  */
 void __remove_from_page_cache(struct page *page)
 {
@@ -141,9 +141,9 @@ void remove_from_page_cache(struct page *page)
 
         BUG_ON(!PageLocked(page));
 
-        write_lock_irq(&mapping->tree_lock);
+        spin_lock_irq(&mapping->tree_lock);
         __remove_from_page_cache(page);
-        write_unlock_irq(&mapping->tree_lock);
+        spin_unlock_irq(&mapping->tree_lock);
 }
 
 static int sync_page(void *word)
@@ -442,48 +442,52 @@ int filemap_write_and_wait_range(struct address_space *mapping,
 }
 
 /**
- * add_to_page_cache - add newly allocated pagecache pages
+ * add_to_page_cache_locked - add a locked page to the pagecache
  * @page:       page to add
  * @mapping:    the page's address_space
  * @offset:     page index
  * @gfp_mask:   page allocation mode
  *
- * This function is used to add newly allocated pagecache pages;
- * the page is new, so we can just run SetPageLocked() against it.
- * The other page state flags were set by rmqueue().
- *
+ * This function is used to add a page to the pagecache. It must be locked.
  * This function does not add the page to the LRU.  The caller must do that.
  */
-int add_to_page_cache(struct page *page, struct address_space *mapping,
+int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
                 pgoff_t offset, gfp_t gfp_mask)
 {
-        int error = mem_cgroup_cache_charge(page, current->mm,
+        int error;
+
+        VM_BUG_ON(!PageLocked(page));
+
+        error = mem_cgroup_cache_charge(page, current->mm,
                                         gfp_mask & ~__GFP_HIGHMEM);
         if (error)
                 goto out;
 
         error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
         if (error == 0) {
-                write_lock_irq(&mapping->tree_lock);
+                page_cache_get(page);
+                page->mapping = mapping;
+                page->index = offset;
+
+                spin_lock_irq(&mapping->tree_lock);
                 error = radix_tree_insert(&mapping->page_tree, offset, page);
-                if (!error) {
-                        page_cache_get(page);
-                        SetPageLocked(page);
-                        page->mapping = mapping;
-                        page->index = offset;
+                if (likely(!error)) {
                         mapping->nrpages++;
                         __inc_zone_page_state(page, NR_FILE_PAGES);
-                } else
+                } else {
+                        page->mapping = NULL;
                         mem_cgroup_uncharge_cache_page(page);
+                        page_cache_release(page);
+                }
 
-                write_unlock_irq(&mapping->tree_lock);
+                spin_unlock_irq(&mapping->tree_lock);
                 radix_tree_preload_end();
         } else
                 mem_cgroup_uncharge_cache_page(page);
 out:
         return error;
 }
-EXPORT_SYMBOL(add_to_page_cache);
+EXPORT_SYMBOL(add_to_page_cache_locked);
 
 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
                                 pgoff_t offset, gfp_t gfp_mask)
@@ -633,15 +637,35 @@ void __lock_page_nosync(struct page *page)
  * Is there a pagecache struct page at the given (mapping, offset) tuple?
  * If yes, increment its refcount and return it; if no, return NULL.
  */
-struct page * find_get_page(struct address_space *mapping, pgoff_t offset)
+struct page *find_get_page(struct address_space *mapping, pgoff_t offset)
 {
+        void **pagep;
         struct page *page;
 
-        read_lock_irq(&mapping->tree_lock);
-        page = radix_tree_lookup(&mapping->page_tree, offset);
-        if (page)
-                page_cache_get(page);
-        read_unlock_irq(&mapping->tree_lock);
+        rcu_read_lock();
+repeat:
+        page = NULL;
+        pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
+        if (pagep) {
+                page = radix_tree_deref_slot(pagep);
+                if (unlikely(!page || page == RADIX_TREE_RETRY))
+                        goto repeat;
+
+                if (!page_cache_get_speculative(page))
+                        goto repeat;
+
+                /*
+                 * Has the page moved?
+                 * This is part of the lockless pagecache protocol. See
+                 * include/linux/pagemap.h for details.
+                 */
+                if (unlikely(page != *pagep)) {
+                        page_cache_release(page);
+                        goto repeat;
+                }
+        }
+        rcu_read_unlock();
+
         return page;
 }
 EXPORT_SYMBOL(find_get_page);
@@ -656,32 +680,22 @@ EXPORT_SYMBOL(find_get_page);
  *
  * Returns zero if the page was not present. find_lock_page() may sleep.
  */
-struct page *find_lock_page(struct address_space *mapping,
-                                pgoff_t offset)
+struct page *find_lock_page(struct address_space *mapping, pgoff_t offset)
 {
         struct page *page;
 
 repeat:
-        read_lock_irq(&mapping->tree_lock);
-        page = radix_tree_lookup(&mapping->page_tree, offset);
+        page = find_get_page(mapping, offset);
         if (page) {
-                page_cache_get(page);
-                if (TestSetPageLocked(page)) {
-                        read_unlock_irq(&mapping->tree_lock);
-                        __lock_page(page);
-
-                        /* Has the page been truncated while we slept? */
-                        if (unlikely(page->mapping != mapping)) {
-                                unlock_page(page);
-                                page_cache_release(page);
-                                goto repeat;
-                        }
-                        VM_BUG_ON(page->index != offset);
-                        goto out;
+                lock_page(page);
+                /* Has the page been truncated? */
+                if (unlikely(page->mapping != mapping)) {
+                        unlock_page(page);
+                        page_cache_release(page);
+                        goto repeat;
                 }
+                VM_BUG_ON(page->index != offset);
         }
-        read_unlock_irq(&mapping->tree_lock);
-out:
         return page;
 }
 EXPORT_SYMBOL(find_lock_page);
@@ -747,13 +761,39 @@ unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
 {
         unsigned int i;
         unsigned int ret;
+        unsigned int nr_found;
+
+        rcu_read_lock();
+restart:
+        nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
+                                (void ***)pages, start, nr_pages);
+        ret = 0;
+        for (i = 0; i < nr_found; i++) {
+                struct page *page;
+repeat:
+                page = radix_tree_deref_slot((void **)pages[i]);
+                if (unlikely(!page))
+                        continue;
+                /*
+                 * this can only trigger if nr_found == 1, making livelock
+                 * a non issue.
+                 */
+                if (unlikely(page == RADIX_TREE_RETRY))
+                        goto restart;
 
-        read_lock_irq(&mapping->tree_lock);
-        ret = radix_tree_gang_lookup(&mapping->page_tree,
-                                (void **)pages, start, nr_pages);
-        for (i = 0; i < ret; i++)
-                page_cache_get(pages[i]);
-        read_unlock_irq(&mapping->tree_lock);
+                if (!page_cache_get_speculative(page))
+                        goto repeat;
+
+                /* Has the page moved? */
+                if (unlikely(page != *((void **)pages[i]))) {
+                        page_cache_release(page);
+                        goto repeat;
+                }
+
+                pages[ret] = page;
+                ret++;
+        }
+        rcu_read_unlock();
         return ret;
 }
 
@@ -774,19 +814,44 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
 {
         unsigned int i;
         unsigned int ret;
+        unsigned int nr_found;
+
+        rcu_read_lock();
+restart:
+        nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
+                                (void ***)pages, index, nr_pages);
+        ret = 0;
+        for (i = 0; i < nr_found; i++) {
+                struct page *page;
+repeat:
+                page = radix_tree_deref_slot((void **)pages[i]);
+                if (unlikely(!page))
+                        continue;
+                /*
+                 * this can only trigger if nr_found == 1, making livelock
+                 * a non issue.
+                 */
+                if (unlikely(page == RADIX_TREE_RETRY))
+                        goto restart;
 
-        read_lock_irq(&mapping->tree_lock);
-        ret = radix_tree_gang_lookup(&mapping->page_tree,
-                                (void **)pages, index, nr_pages);
-        for (i = 0; i < ret; i++) {
-                if (pages[i]->mapping == NULL || pages[i]->index != index)
+                if (page->mapping == NULL || page->index != index)
                         break;
 
-                page_cache_get(pages[i]);
+                if (!page_cache_get_speculative(page))
+                        goto repeat;
+
+                /* Has the page moved? */
+                if (unlikely(page != *((void **)pages[i]))) {
+                        page_cache_release(page);
+                        goto repeat;
+                }
+
+                pages[ret] = page;
+                ret++;
                 index++;
         }
-        read_unlock_irq(&mapping->tree_lock);
-        return i;
+        rcu_read_unlock();
+        return ret;
 }
 EXPORT_SYMBOL(find_get_pages_contig);
 
@@ -806,15 +871,43 @@ unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
 {
         unsigned int i;
         unsigned int ret;
+        unsigned int nr_found;
+
+        rcu_read_lock();
+restart:
+        nr_found = radix_tree_gang_lookup_tag_slot(&mapping->page_tree,
+                                (void ***)pages, *index, nr_pages, tag);
+        ret = 0;
+        for (i = 0; i < nr_found; i++) {
+                struct page *page;
+repeat:
+                page = radix_tree_deref_slot((void **)pages[i]);
+                if (unlikely(!page))
+                        continue;
+                /*
+                 * this can only trigger if nr_found == 1, making livelock
+                 * a non issue.
+                 */
+                if (unlikely(page == RADIX_TREE_RETRY))
+                        goto restart;
+
+                if (!page_cache_get_speculative(page))
+                        goto repeat;
+
+                /* Has the page moved? */
+                if (unlikely(page != *((void **)pages[i]))) {
+                        page_cache_release(page);
+                        goto repeat;
+                }
+
+                pages[ret] = page;
+                ret++;
+        }
+        rcu_read_unlock();
 
-        read_lock_irq(&mapping->tree_lock);
-        ret = radix_tree_gang_lookup_tag(&mapping->page_tree,
-                                (void **)pages, *index, nr_pages, tag);
-        for (i = 0; i < ret; i++)
-                page_cache_get(pages[i]);
         if (ret)
                 *index = pages[ret - 1]->index + 1;
-        read_unlock_irq(&mapping->tree_lock);
+
         return ret;
 }
 EXPORT_SYMBOL(find_get_pages_tag);
@@ -1665,8 +1758,9 @@ static int __remove_suid(struct dentry *dentry, int kill)
         return notify_change(dentry, &newattrs);
 }
 
-int remove_suid(struct dentry *dentry)
+int file_remove_suid(struct file *file)
 {
+        struct dentry *dentry = file->f_path.dentry;
         int killsuid = should_remove_suid(dentry);
         int killpriv = security_inode_need_killpriv(dentry);
         int error = 0;
@@ -1680,7 +1774,7 @@ int remove_suid(struct dentry *dentry)
 
         return error;
 }
-EXPORT_SYMBOL(remove_suid);
+EXPORT_SYMBOL(file_remove_suid);
 
 static size_t __iovec_copy_from_user_inatomic(char *vaddr,
                         const struct iovec *iov, size_t base, size_t bytes)
@@ -2436,7 +2530,7 @@ __generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
         if (count == 0)
                 goto out;
 
-        err = remove_suid(file->f_path.dentry);
+        err = file_remove_suid(file);
         if (err)
                 goto out;