mm: let swap use exceptional entries

If swap entries are to be stored along with struct page pointers in a
radix tree, they need to be distinguished as exceptional entries.

Most of the handling of swap entries in radix tree will be contained in
shmem.c, but a few functions in filemap.c's common code need to check
for their appearance: find_get_page(), find_lock_page(),
find_get_pages() and find_get_pages_contig().

So as not to slow their fast paths, tuck those checks inside the
existing checks for unlikely radix_tree_deref_slot(); except for
find_lock_page(), where it is an added test.  And make it a BUG in
find_get_pages_tag(), which is not applied to tmpfs files.

A part of the reason for eliminating shmem_readpage() earlier, was to
minimize the places where common code would need to allow for swap
entries.

The swp_entry_t known to swapfile.c must be massaged into a slightly
different form when stored in the radix tree, just as it gets massaged
into a pte_t when stored in page tables.

In an i386 kernel this limits its information (type and page offset) to
30 bits: given 32 "types" of swapfile and 4kB pagesize, that's a maximum
swapfile size of 128GB.  Which is less than the 512GB we previously
allowed with X86_PAE (where the swap entry can occupy the entire upper
32 bits of a pte_t), but not a new limitation on 32-bit without PAE; and
there's not a new limitation on 64-bit (where swap filesize is already
limited to 16TB by a 32-bit page offset).  Thirty areas of 128GB is
probably still enough swap for a 64GB 32-bit machine.

Provide swp_to_radix_entry() and radix_to_swp_entry() conversions, and
enforce filesize limit in read_swap_header(), just as for ptes.

Signed-off-by: Hugh Dickins <hughd@google.com>
Acked-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 31 insertions, 18 deletions

diff --git a/mm/filemap.c b/mm/filemap.c
index b83aebfd0a00..76bfb6460f57 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -714,9 +714,12 @@ repeat:
                 page = radix_tree_deref_slot(pagep);
                 if (unlikely(!page))
                         goto out;
-                if (radix_tree_deref_retry(page))
+                if (radix_tree_exception(page)) {
+                        if (radix_tree_exceptional_entry(page))
+                                goto out;
+                        /* radix_tree_deref_retry(page) */
                         goto repeat;
-
+                }
                 if (!page_cache_get_speculative(page))
                         goto repeat;
 
@@ -753,7 +756,7 @@ struct page *find_lock_page(struct address_space *mapping, pgoff_t offset)
 
 repeat:
         page = find_get_page(mapping, offset);
-        if (page) {
+        if (page && !radix_tree_exception(page)) {
                 lock_page(page);
                 /* Has the page been truncated? */
                 if (unlikely(page->mapping != mapping)) {
@@ -849,11 +852,14 @@ repeat:
                 if (unlikely(!page))
                         continue;
 
-                /*
-                 * This can only trigger when the entry at index 0 moves out
-                 * of or back to the root: none yet gotten, safe to restart.
-                 */
-                if (radix_tree_deref_retry(page)) {
+                if (radix_tree_exception(page)) {
+                        if (radix_tree_exceptional_entry(page))
+                                continue;
+                        /*
+                         * radix_tree_deref_retry(page):
+                         * can only trigger when entry at index 0 moves out of
+                         * or back to root: none yet gotten, safe to restart.
+                         */
                         WARN_ON(start | i);
                         goto restart;
                 }
@@ -912,12 +918,16 @@ repeat:
                 if (unlikely(!page))
                         continue;
 
-                /*
-                 * This can only trigger when the entry at index 0 moves out
-                 * of or back to the root: none yet gotten, safe to restart.
-                 */
-                if (radix_tree_deref_retry(page))
+                if (radix_tree_exception(page)) {
+                        if (radix_tree_exceptional_entry(page))
+                                break;
+                        /*
+                         * radix_tree_deref_retry(page):
+                         * can only trigger when entry at index 0 moves out of
+                         * or back to root: none yet gotten, safe to restart.
+                         */
                         goto restart;
+                }
 
                 if (!page_cache_get_speculative(page))
                         goto repeat;
@@ -977,12 +987,15 @@ repeat:
                 if (unlikely(!page))
                         continue;
 
-                /*
-                 * This can only trigger when the entry at index 0 moves out
-                 * of or back to the root: none yet gotten, safe to restart.
-                 */
-                if (radix_tree_deref_retry(page))
+                if (radix_tree_exception(page)) {
+                        BUG_ON(radix_tree_exceptional_entry(page));
+                        /*
+                         * radix_tree_deref_retry(page):
+                         * can only trigger when entry at index 0 moves out of
+                         * or back to root: none yet gotten, safe to restart.
+                         */
                         goto restart;
+                }
 
                 if (!page_cache_get_speculative(page))
                         goto repeat;