1 files changed, 55 insertions, 35 deletions

diff --git a/mm/swapfile.c b/mm/swapfile.c
index bd1bb5920306..90cb67a5417c 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -33,17 +33,18 @@
 #include <asm/tlbflush.h>
 #include <linux/swapops.h>
 
-DEFINE_SPINLOCK(swap_lock);
-unsigned int nr_swapfiles;
+static DEFINE_SPINLOCK(swap_lock);
+static unsigned int nr_swapfiles;
 long total_swap_pages;
 static int swap_overflow;
+static int least_priority;
 
 static const char Bad_file[] = "Bad swap file entry ";
 static const char Unused_file[] = "Unused swap file entry ";
 static const char Bad_offset[] = "Bad swap offset entry ";
 static const char Unused_offset[] = "Unused swap offset entry ";
 
-struct swap_list_t swap_list = {-1, -1};
+static struct swap_list_t swap_list = {-1, -1};
 
 static struct swap_info_struct swap_info[MAX_SWAPFILES];
 
@@ -343,7 +344,7 @@ int can_share_swap_page(struct page *page)
  * Work out if there are any other processes sharing this
  * swap cache page. Free it if you can. Return success.
  */
-int remove_exclusive_swap_page(struct page *page)
+static int remove_exclusive_swap_page_count(struct page *page, int count)
 {
         int retval;
         struct swap_info_struct * p;
@@ -356,7 +357,7 @@ int remove_exclusive_swap_page(struct page *page)
                 return 0;
         if (PageWriteback(page))
                 return 0;
-        if (page_count(page) != 2) /* 2: us + cache */
+        if (page_count(page) != count) /* us + cache + ptes */
                 return 0;
 
         entry.val = page_private(page);
@@ -368,13 +369,13 @@ int remove_exclusive_swap_page(struct page *page)
         retval = 0;
         if (p->swap_map[swp_offset(entry)] == 1) {
                 /* Recheck the page count with the swapcache lock held.. */
-                write_lock_irq(&swapper_space.tree_lock);
-                if ((page_count(page) == 2) && !PageWriteback(page)) {
+                spin_lock_irq(&swapper_space.tree_lock);
+                if ((page_count(page) == count) && !PageWriteback(page)) {
                         __delete_from_swap_cache(page);
                         SetPageDirty(page);
                         retval = 1;
                 }
-                write_unlock_irq(&swapper_space.tree_lock);
+                spin_unlock_irq(&swapper_space.tree_lock);
         }
         spin_unlock(&swap_lock);
 
@@ -387,6 +388,25 @@ int remove_exclusive_swap_page(struct page *page)
 }
 
 /*
+ * Most of the time the page should have two references: one for the
+ * process and one for the swap cache.
+ */
+int remove_exclusive_swap_page(struct page *page)
+{
+        return remove_exclusive_swap_page_count(page, 2);
+}
+
+/*
+ * The pageout code holds an extra reference to the page.  That raises
+ * the reference count to test for to 2 for a page that is only in the
+ * swap cache plus 1 for each process that maps the page.
+ */
+int remove_exclusive_swap_page_ref(struct page *page)
+{
+        return remove_exclusive_swap_page_count(page, 2 + page_mapcount(page));
+}
+
+/*
  * Free the swap entry like above, but also try to
  * free the page cache entry if it is the last user.
  */
@@ -402,7 +422,7 @@ void free_swap_and_cache(swp_entry_t entry)
         if (p) {
                 if (swap_entry_free(p, swp_offset(entry)) == 1) {
                         page = find_get_page(&swapper_space, entry.val);
-                        if (page && unlikely(TestSetPageLocked(page))) {
+                        if (page && !trylock_page(page)) {
                                 page_cache_release(page);
                                 page = NULL;
                         }
@@ -655,8 +675,8 @@ static int unuse_mm(struct mm_struct *mm,
 
         if (!down_read_trylock(&mm->mmap_sem)) {
                 /*
-                 * Activate page so shrink_cache is unlikely to unmap its
-                 * ptes while lock is dropped, so swapoff can make progress.
+                 * Activate page so shrink_inactive_list is unlikely to unmap
+                 * its ptes while lock is dropped, so swapoff can make progress.
                  */
                 activate_page(page);
                 unlock_page(page);
@@ -1260,6 +1280,11 @@ asmlinkage long sys_swapoff(const char __user * specialfile)
                 /* just pick something that's safe... */
                 swap_list.next = swap_list.head;
         }
+        if (p->prio < 0) {
+                for (i = p->next; i >= 0; i = swap_info[i].next)
+                        swap_info[i].prio = p->prio--;
+                least_priority++;
+        }
         nr_swap_pages -= p->pages;
         total_swap_pages -= p->pages;
         p->flags &= ~SWP_WRITEOK;
@@ -1272,9 +1297,14 @@ asmlinkage long sys_swapoff(const char __user * specialfile)
         if (err) {
                 /* re-insert swap space back into swap_list */
                 spin_lock(&swap_lock);
-                for (prev = -1, i = swap_list.head; i >= 0; prev = i, i = swap_info[i].next)
+                if (p->prio < 0)
+                        p->prio = --least_priority;
+                prev = -1;
+                for (i = swap_list.head; i >= 0; i = swap_info[i].next) {
                         if (p->prio >= swap_info[i].prio)
                                 break;
+                        prev = i;
+                }
                 p->next = i;
                 if (prev < 0)
                         swap_list.head = swap_list.next = p - swap_info;
@@ -1447,7 +1477,6 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags)
         unsigned int type;
         int i, prev;
         int error;
-        static int least_priority;
         union swap_header *swap_header = NULL;
         int swap_header_version;
         unsigned int nr_good_pages = 0;
@@ -1455,7 +1484,7 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags)
         sector_t span;
         unsigned long maxpages = 1;
         int swapfilesize;
-        unsigned short *swap_map;
+        unsigned short *swap_map = NULL;
         struct page *page = NULL;
         struct inode *inode = NULL;
         int did_down = 0;
@@ -1474,22 +1503,10 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags)
         }
         if (type >= nr_swapfiles)
                 nr_swapfiles = type+1;
+        memset(p, 0, sizeof(*p));
         INIT_LIST_HEAD(&p->extent_list);
         p->flags = SWP_USED;
-        p->swap_file = NULL;
-        p->old_block_size = 0;
-        p->swap_map = NULL;
-        p->lowest_bit = 0;
-        p->highest_bit = 0;
-        p->cluster_nr = 0;
-        p->inuse_pages = 0;
         p->next = -1;
-        if (swap_flags & SWAP_FLAG_PREFER) {
-                p->prio =
-                  (swap_flags & SWAP_FLAG_PRIO_MASK)>>SWAP_FLAG_PRIO_SHIFT;
-        } else {
-                p->prio = --least_priority;
-        }
         spin_unlock(&swap_lock);
         name = getname(specialfile);
         error = PTR_ERR(name);
@@ -1632,19 +1649,20 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags)
                         goto bad_swap;
 
                 /* OK, set up the swap map and apply the bad block list */
-                if (!(p->swap_map = vmalloc(maxpages * sizeof(short)))) {
+                swap_map = vmalloc(maxpages * sizeof(short));
+                if (!swap_map) {
                         error = -ENOMEM;
                         goto bad_swap;
                 }
 
                 error = 0;
-                memset(p->swap_map, 0, maxpages * sizeof(short));
+                memset(swap_map, 0, maxpages * sizeof(short));
                 for (i = 0; i < swap_header->info.nr_badpages; i++) {
                         int page_nr = swap_header->info.badpages[i];
                         if (page_nr <= 0 || page_nr >= swap_header->info.last_page)
                                 error = -EINVAL;
                         else
-                                p->swap_map[page_nr] = SWAP_MAP_BAD;
+                                swap_map[page_nr] = SWAP_MAP_BAD;
                 }
                 nr_good_pages = swap_header->info.last_page -
                                 swap_header->info.nr_badpages -
@@ -1654,7 +1672,7 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags)
         }
 
         if (nr_good_pages) {
-                p->swap_map[0] = SWAP_MAP_BAD;
+                swap_map[0] = SWAP_MAP_BAD;
                 p->max = maxpages;
                 p->pages = nr_good_pages;
                 nr_extents = setup_swap_extents(p, &span);
@@ -1672,6 +1690,12 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags)
 
         mutex_lock(&swapon_mutex);
         spin_lock(&swap_lock);
+        if (swap_flags & SWAP_FLAG_PREFER)
+                p->prio =
+                  (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
+        else
+                p->prio = --least_priority;
+        p->swap_map = swap_map;
         p->flags = SWP_ACTIVE;
         nr_swap_pages += nr_good_pages;
         total_swap_pages += nr_good_pages;
@@ -1707,12 +1731,8 @@ bad_swap:
         destroy_swap_extents(p);
 bad_swap_2:
         spin_lock(&swap_lock);
-        swap_map = p->swap_map;
         p->swap_file = NULL;
-        p->swap_map = NULL;
         p->flags = 0;
-        if (!(swap_flags & SWAP_FLAG_PREFER))
-                ++least_priority;
         spin_unlock(&swap_lock);
         vfree(swap_map);
         if (swap_file)