1 files changed, 180 insertions, 90 deletions

diff --git a/kernel/power/swap.c b/kernel/power/swap.c
index f1dd146bd64d..9b2ee5344dee 100644
--- a/kernel/power/swap.c
+++ b/kernel/power/swap.c
@@ -22,6 +22,7 @@
 #include <linux/device.h>
 #include <linux/buffer_head.h>
 #include <linux/bio.h>
+#include <linux/blkdev.h>
 #include <linux/swap.h>
 #include <linux/swapops.h>
 #include <linux/pm.h>
@@ -49,18 +50,16 @@ static int mark_swapfiles(swp_entry_t start)
 {
         int error;
 
-        rw_swap_page_sync(READ,
-                          swp_entry(root_swap, 0),
-                          virt_to_page((unsigned long)&swsusp_header));
+        rw_swap_page_sync(READ, swp_entry(root_swap, 0),
+                          virt_to_page((unsigned long)&swsusp_header), NULL);
         if (!memcmp("SWAP-SPACE",swsusp_header.sig, 10) ||
             !memcmp("SWAPSPACE2",swsusp_header.sig, 10)) {
                 memcpy(swsusp_header.orig_sig,swsusp_header.sig, 10);
                 memcpy(swsusp_header.sig,SWSUSP_SIG, 10);
                 swsusp_header.image = start;
-                error = rw_swap_page_sync(WRITE,
-                                          swp_entry(root_swap, 0),
-                                          virt_to_page((unsigned long)
-                                                       &swsusp_header));
+                error = rw_swap_page_sync(WRITE, swp_entry(root_swap, 0),
+                                virt_to_page((unsigned long)&swsusp_header),
+                                NULL);
         } else {
                 pr_debug("swsusp: Partition is not swap space.\n");
                 error = -ENODEV;
@@ -88,16 +87,37 @@ static int swsusp_swap_check(void) /* This is called before saving image */
  *      write_page - Write one page to given swap location.
  *      @buf:           Address we're writing.
  *      @offset:        Offset of the swap page we're writing to.
+ *      @bio_chain:     Link the next write BIO here
  */
 
-static int write_page(void *buf, unsigned long offset)
+static int write_page(void *buf, unsigned long offset, struct bio **bio_chain)
 {
         swp_entry_t entry;
         int error = -ENOSPC;
 
         if (offset) {
+                struct page *page = virt_to_page(buf);
+
+                if (bio_chain) {
+                        /*
+                         * Whether or not we successfully allocated a copy page,
+                         * we take a ref on the page here.  It gets undone in
+                         * wait_on_bio_chain().
+                         */
+                        struct page *page_copy;
+                        page_copy = alloc_page(GFP_ATOMIC);
+                        if (page_copy == NULL) {
+                                WARN_ON_ONCE(1);
+                                bio_chain = NULL;       /* Go synchronous */
+                                get_page(page);
+                        } else {
+                                memcpy(page_address(page_copy),
+                                        page_address(page), PAGE_SIZE);
+                                page = page_copy;
+                        }
+                }
                 entry = swp_entry(root_swap, offset);
-                error = rw_swap_page_sync(WRITE, entry, virt_to_page(buf));
+                error = rw_swap_page_sync(WRITE, entry, page, bio_chain);
         }
         return error;
 }
@@ -146,6 +166,26 @@ static void release_swap_writer(struct swap_map_handle *handle)
         handle->bitmap = NULL;
 }
 
+static void show_speed(struct timeval *start, struct timeval *stop,
+                        unsigned nr_pages, char *msg)
+{
+        s64 elapsed_centisecs64;
+        int centisecs;
+        int k;
+        int kps;
+
+        elapsed_centisecs64 = timeval_to_ns(stop) - timeval_to_ns(start);
+        do_div(elapsed_centisecs64, NSEC_PER_SEC / 100);
+        centisecs = elapsed_centisecs64;
+        if (centisecs == 0)
+                centisecs = 1;  /* avoid div-by-zero */
+        k = nr_pages * (PAGE_SIZE / 1024);
+        kps = (k * 100) / centisecs;
+        printk("%s %d kbytes in %d.%02d seconds (%d.%02d MB/s)\n", msg, k,
+                        centisecs / 100, centisecs % 100,
+                        kps / 1000, (kps % 1000) / 10);
+}
+
 static int get_swap_writer(struct swap_map_handle *handle)
 {
         handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_KERNEL);
@@ -165,37 +205,70 @@ static int get_swap_writer(struct swap_map_handle *handle)
         return 0;
 }
 
-static int swap_write_page(struct swap_map_handle *handle, void *buf)
+static int wait_on_bio_chain(struct bio **bio_chain)
 {
-        int error;
+        struct bio *bio;
+        struct bio *next_bio;
+        int ret = 0;
+
+        if (bio_chain == NULL)
+                return 0;
+
+        bio = *bio_chain;
+        if (bio == NULL)
+                return 0;
+        while (bio) {
+                struct page *page;
+
+                next_bio = bio->bi_private;
+                page = bio->bi_io_vec[0].bv_page;
+                wait_on_page_locked(page);
+                if (!PageUptodate(page) || PageError(page))
+                        ret = -EIO;
+                put_page(page);
+                bio_put(bio);
+                bio = next_bio;
+        }
+        *bio_chain = NULL;
+        return ret;
+}
+
+static int swap_write_page(struct swap_map_handle *handle, void *buf,
+                                struct bio **bio_chain)
+{
+        int error = 0;
         unsigned long offset;
 
         if (!handle->cur)
                 return -EINVAL;
         offset = alloc_swap_page(root_swap, handle->bitmap);
-        error = write_page(buf, offset);
+        error = write_page(buf, offset, bio_chain);
         if (error)
                 return error;
         handle->cur->entries[handle->k++] = offset;
         if (handle->k >= MAP_PAGE_ENTRIES) {
+                error = wait_on_bio_chain(bio_chain);
+                if (error)
+                        goto out;
                 offset = alloc_swap_page(root_swap, handle->bitmap);
                 if (!offset)
                         return -ENOSPC;
                 handle->cur->next_swap = offset;
-                error = write_page(handle->cur, handle->cur_swap);
+                error = write_page(handle->cur, handle->cur_swap, NULL);
                 if (error)
-                        return error;
+                        goto out;
                 memset(handle->cur, 0, PAGE_SIZE);
                 handle->cur_swap = offset;
                 handle->k = 0;
         }
-        return 0;
+out:
+        return error;
 }
 
 static int flush_swap_writer(struct swap_map_handle *handle)
 {
         if (handle->cur && handle->cur_swap)
-                return write_page(handle->cur, handle->cur_swap);
+                return write_page(handle->cur, handle->cur_swap, NULL);
         else
                 return -EINVAL;
 }
@@ -206,21 +279,29 @@ static int flush_swap_writer(struct swap_map_handle *handle)
 
 static int save_image(struct swap_map_handle *handle,
                       struct snapshot_handle *snapshot,
-                      unsigned int nr_pages)
+                      unsigned int nr_to_write)
 {
         unsigned int m;
         int ret;
         int error = 0;
+        int nr_pages;
+        int err2;
+        struct bio *bio;
+        struct timeval start;
+        struct timeval stop;
 
-        printk("Saving image data pages (%u pages) ...     ", nr_pages);
-        m = nr_pages / 100;
+        printk("Saving image data pages (%u pages) ...     ", nr_to_write);
+        m = nr_to_write / 100;
         if (!m)
                 m = 1;
         nr_pages = 0;
+        bio = NULL;
+        do_gettimeofday(&start);
         do {
                 ret = snapshot_read_next(snapshot, PAGE_SIZE);
                 if (ret > 0) {
-                        error = swap_write_page(handle, data_of(*snapshot));
+                        error = swap_write_page(handle, data_of(*snapshot),
+                                                &bio);
                         if (error)
                                 break;
                         if (!(nr_pages % m))
@@ -228,8 +309,13 @@ static int save_image(struct swap_map_handle *handle,
                         nr_pages++;
                 }
         } while (ret > 0);
+        err2 = wait_on_bio_chain(&bio);
+        do_gettimeofday(&stop);
+        if (!error)
+                error = err2;
         if (!error)
                 printk("\b\b\b\bdone\n");
+        show_speed(&start, &stop, nr_to_write, "Wrote");
         return error;
 }
 
@@ -245,8 +331,7 @@ static int enough_swap(unsigned int nr_pages)
         unsigned int free_swap = count_swap_pages(root_swap, 1);
 
         pr_debug("swsusp: free swap pages: %u\n", free_swap);
-        return free_swap > (nr_pages + PAGES_FOR_IO +
-                (nr_pages + PBES_PER_PAGE - 1) / PBES_PER_PAGE);
+        return free_swap > nr_pages + PAGES_FOR_IO;
 }
 
 /**
@@ -266,7 +351,8 @@ int swsusp_write(void)
         int error;
 
         if ((error = swsusp_swap_check())) {
-                printk(KERN_ERR "swsusp: Cannot find swap device, try swapon -a.\n");
+                printk(KERN_ERR "swsusp: Cannot find swap device, try "
+                                "swapon -a.\n");
                 return error;
         }
         memset(&snapshot, 0, sizeof(struct snapshot_handle));
@@ -281,7 +367,7 @@ int swsusp_write(void)
         error = get_swap_writer(&handle);
         if (!error) {
                 unsigned long start = handle.cur_swap;
-                error = swap_write_page(&handle, header);
+                error = swap_write_page(&handle, header, NULL);
                 if (!error)
                         error = save_image(&handle, &snapshot,
                                         header->pages - 1);
@@ -298,27 +384,6 @@ int swsusp_write(void)
         return error;
 }
 
-/*
- *      Using bio to read from swap.
- *      This code requires a bit more work than just using buffer heads
- *      but, it is the recommended way for 2.5/2.6.
- *      The following are to signal the beginning and end of I/O. Bios
- *      finish asynchronously, while we want them to happen synchronously.
- *      A simple atomic_t, and a wait loop take care of this problem.
- */
-
-static atomic_t io_done = ATOMIC_INIT(0);
-
-static int end_io(struct bio *bio, unsigned int num, int err)
-{
-        if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) {
-                printk(KERN_ERR "I/O error reading swsusp image.\n");
-                return -EIO;
-        }
-        atomic_set(&io_done, 0);
-        return 0;
-}
-
 static struct block_device *resume_bdev;
 
 /**
@@ -326,15 +391,15 @@ static struct block_device *resume_bdev;
  *      @rw:    READ or WRITE.
  *      @off    physical offset of page.
  *      @page:  page we're reading or writing.
+ *      @bio_chain: list of pending biod (for async reading)
  *
  *      Straight from the textbook - allocate and initialize the bio.
- *      If we're writing, make sure the page is marked as dirty.
- *      Then submit it and wait.
+ *      If we're reading, make sure the page is marked as dirty.
+ *      Then submit it and, if @bio_chain == NULL, wait.
  */
-
-static int submit(int rw, pgoff_t page_off, void *page)
+static int submit(int rw, pgoff_t page_off, struct page *page,
+                        struct bio **bio_chain)
 {
-        int error = 0;
         struct bio *bio;
 
         bio = bio_alloc(GFP_ATOMIC, 1);
@@ -342,33 +407,40 @@ static int submit(int rw, pgoff_t page_off, void *page)
                 return -ENOMEM;
         bio->bi_sector = page_off * (PAGE_SIZE >> 9);
         bio->bi_bdev = resume_bdev;
-        bio->bi_end_io = end_io;
+        bio->bi_end_io = end_swap_bio_read;
 
-        if (bio_add_page(bio, virt_to_page(page), PAGE_SIZE, 0) < PAGE_SIZE) {
-                printk("swsusp: ERROR: adding page to bio at %ld\n",page_off);
-                error = -EFAULT;
-                goto Done;
+        if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
+                printk("swsusp: ERROR: adding page to bio at %ld\n", page_off);
+                bio_put(bio);
+                return -EFAULT;
         }
 
-        atomic_set(&io_done, 1);
-        submit_bio(rw | (1 << BIO_RW_SYNC), bio);
-        while (atomic_read(&io_done))
-                yield();
-        if (rw == READ)
-                bio_set_pages_dirty(bio);
- Done:
-        bio_put(bio);
-        return error;
+        lock_page(page);
+        bio_get(bio);
+
+        if (bio_chain == NULL) {
+                submit_bio(rw | (1 << BIO_RW_SYNC), bio);
+                wait_on_page_locked(page);
+                if (rw == READ)
+                        bio_set_pages_dirty(bio);
+                bio_put(bio);
+        } else {
+                get_page(page);
+                bio->bi_private = *bio_chain;
+                *bio_chain = bio;
+                submit_bio(rw | (1 << BIO_RW_SYNC), bio);
+        }
+        return 0;
 }
 
-static int bio_read_page(pgoff_t page_off, void *page)
+static int bio_read_page(pgoff_t page_off, void *addr, struct bio **bio_chain)
 {
-        return submit(READ, page_off, page);
+        return submit(READ, page_off, virt_to_page(addr), bio_chain);
 }
 
-static int bio_write_page(pgoff_t page_off, void *page)
+static int bio_write_page(pgoff_t page_off, void *addr)
 {
-        return submit(WRITE, page_off, page);
+        return submit(WRITE, page_off, virt_to_page(addr), NULL);
 }
 
 /**
@@ -393,7 +465,7 @@ static int get_swap_reader(struct swap_map_handle *handle,
         handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_ATOMIC);
         if (!handle->cur)
                 return -ENOMEM;
-        error = bio_read_page(swp_offset(start), handle->cur);
+        error = bio_read_page(swp_offset(start), handle->cur, NULL);
         if (error) {
                 release_swap_reader(handle);
                 return error;
@@ -402,7 +474,8 @@ static int get_swap_reader(struct swap_map_handle *handle,
         return 0;
 }
 
-static int swap_read_page(struct swap_map_handle *handle, void *buf)
+static int swap_read_page(struct swap_map_handle *handle, void *buf,
+                                struct bio **bio_chain)
 {
         unsigned long offset;
         int error;
@@ -412,16 +485,17 @@ static int swap_read_page(struct swap_map_handle *handle, void *buf)
         offset = handle->cur->entries[handle->k];
         if (!offset)
                 return -EFAULT;
-        error = bio_read_page(offset, buf);
+        error = bio_read_page(offset, buf, bio_chain);
         if (error)
                 return error;
         if (++handle->k >= MAP_PAGE_ENTRIES) {
+                error = wait_on_bio_chain(bio_chain);
                 handle->k = 0;
                 offset = handle->cur->next_swap;
                 if (!offset)
                         release_swap_reader(handle);
-                else
-                        error = bio_read_page(offset, handle->cur);
+                else if (!error)
+                        error = bio_read_page(offset, handle->cur, NULL);
         }
         return error;
 }
@@ -434,33 +508,49 @@ static int swap_read_page(struct swap_map_handle *handle, void *buf)
 
 static int load_image(struct swap_map_handle *handle,
                       struct snapshot_handle *snapshot,
-                      unsigned int nr_pages)
+                      unsigned int nr_to_read)
 {
         unsigned int m;
-        int ret;
         int error = 0;
+        struct timeval start;
+        struct timeval stop;
+        struct bio *bio;
+        int err2;
+        unsigned nr_pages;
 
-        printk("Loading image data pages (%u pages) ...     ", nr_pages);
-        m = nr_pages / 100;
+        printk("Loading image data pages (%u pages) ...     ", nr_to_read);
+        m = nr_to_read / 100;
         if (!m)
                 m = 1;
         nr_pages = 0;
-        do {
-                ret = snapshot_write_next(snapshot, PAGE_SIZE);
-                if (ret > 0) {
-                        error = swap_read_page(handle, data_of(*snapshot));
-                        if (error)
-                                break;
-                        if (!(nr_pages % m))
-                                printk("\b\b\b\b%3d%%", nr_pages / m);
-                        nr_pages++;
-                }
-        } while (ret > 0);
+        bio = NULL;
+        do_gettimeofday(&start);
+        for ( ; ; ) {
+                error = snapshot_write_next(snapshot, PAGE_SIZE);
+                if (error <= 0)
+                        break;
+                error = swap_read_page(handle, data_of(*snapshot), &bio);
+                if (error)
+                        break;
+                if (snapshot->sync_read)
+                        error = wait_on_bio_chain(&bio);
+                if (error)
+                        break;
+                if (!(nr_pages % m))
+                        printk("\b\b\b\b%3d%%", nr_pages / m);
+                nr_pages++;
+        }
+        err2 = wait_on_bio_chain(&bio);
+        do_gettimeofday(&stop);
+        if (!error)
+                error = err2;
         if (!error) {
                 printk("\b\b\b\bdone\n");
+                snapshot_free_unused_memory(snapshot);
                 if (!snapshot_image_loaded(snapshot))
                         error = -ENODATA;
         }
+        show_speed(&start, &stop, nr_to_read, "Read");
         return error;
 }
 
@@ -483,7 +573,7 @@ int swsusp_read(void)
         header = (struct swsusp_info *)data_of(snapshot);
         error = get_swap_reader(&handle, swsusp_header.image);
         if (!error)
-                error = swap_read_page(&handle, header);
+                error = swap_read_page(&handle, header, NULL);
         if (!error)
                 error = load_image(&handle, &snapshot, header->pages - 1);
         release_swap_reader(&handle);
@@ -509,7 +599,7 @@ int swsusp_check(void)
         if (!IS_ERR(resume_bdev)) {
                 set_blocksize(resume_bdev, PAGE_SIZE);
                 memset(&swsusp_header, 0, sizeof(swsusp_header));
-                if ((error = bio_read_page(0, &swsusp_header)))
+                if ((error = bio_read_page(0, &swsusp_header, NULL)))
                         return error;
                 if (!memcmp(SWSUSP_SIG, swsusp_header.sig, 10)) {
                         memcpy(swsusp_header.sig, swsusp_header.orig_sig, 10);