3 files changed, 98 insertions, 25 deletions

diff --git a/include/linux/swap.h b/include/linux/swap.h
index a2f5ad7c2d2e..3d434cbffe26 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -12,6 +12,8 @@
 
 struct notifier_block;
 
+struct bio;
+
 #define SWAP_FLAG_PREFER        0x8000  /* set if swap priority specified */
 #define SWAP_FLAG_PRIO_MASK     0x7fff
 #define SWAP_FLAG_PRIO_SHIFT    0
@@ -216,7 +218,8 @@ extern void swap_unplug_io_fn(struct backing_dev_info *, struct page *);
 /* linux/mm/page_io.c */
 extern int swap_readpage(struct file *, struct page *);
 extern int swap_writepage(struct page *page, struct writeback_control *wbc);
-extern int rw_swap_page_sync(int, swp_entry_t, struct page *);
+extern int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page,
+                                struct bio **bio_chain);
 
 /* linux/mm/swap_state.c */
 extern struct address_space swapper_space;
diff --git a/kernel/power/swap.c b/kernel/power/swap.c
index 79b66e734bdb..2dc883d361d5 100644
--- a/kernel/power/swap.c
+++ b/kernel/power/swap.c
@@ -49,18 +49,16 @@ static int mark_swapfiles(swp_entry_t start)
 {
         int error;
 
-        rw_swap_page_sync(READ,
-                          swp_entry(root_swap, 0),
-                          virt_to_page((unsigned long)&swsusp_header));
+        rw_swap_page_sync(READ, swp_entry(root_swap, 0),
+                          virt_to_page((unsigned long)&swsusp_header), NULL);
         if (!memcmp("SWAP-SPACE",swsusp_header.sig, 10) ||
             !memcmp("SWAPSPACE2",swsusp_header.sig, 10)) {
                 memcpy(swsusp_header.orig_sig,swsusp_header.sig, 10);
                 memcpy(swsusp_header.sig,SWSUSP_SIG, 10);
                 swsusp_header.image = start;
-                error = rw_swap_page_sync(WRITE,
-                                          swp_entry(root_swap, 0),
-                                          virt_to_page((unsigned long)
-                                                       &swsusp_header));
+                error = rw_swap_page_sync(WRITE, swp_entry(root_swap, 0),
+                                virt_to_page((unsigned long)&swsusp_header),
+                                NULL);
         } else {
                 pr_debug("swsusp: Partition is not swap space.\n");
                 error = -ENODEV;
@@ -88,16 +86,37 @@ static int swsusp_swap_check(void) /* This is called before saving image */
  *      write_page - Write one page to given swap location.
  *      @buf:           Address we're writing.
  *      @offset:        Offset of the swap page we're writing to.
+ *      @bio_chain:     Link the next write BIO here
  */
 
-static int write_page(void *buf, unsigned long offset)
+static int write_page(void *buf, unsigned long offset, struct bio **bio_chain)
 {
         swp_entry_t entry;
         int error = -ENOSPC;
 
         if (offset) {
+                struct page *page = virt_to_page(buf);
+
+                if (bio_chain) {
+                        /*
+                         * Whether or not we successfully allocated a copy page,
+                         * we take a ref on the page here.  It gets undone in
+                         * wait_on_bio_chain().
+                         */
+                        struct page *page_copy;
+                        page_copy = alloc_page(GFP_ATOMIC);
+                        if (page_copy == NULL) {
+                                WARN_ON_ONCE(1);
+                                bio_chain = NULL;       /* Go synchronous */
+                                get_page(page);
+                        } else {
+                                memcpy(page_address(page_copy),
+                                        page_address(page), PAGE_SIZE);
+                                page = page_copy;
+                        }
+                }
                 entry = swp_entry(root_swap, offset);
-                error = rw_swap_page_sync(WRITE, entry, virt_to_page(buf));
+                error = rw_swap_page_sync(WRITE, entry, page, bio_chain);
         }
         return error;
 }
@@ -185,37 +204,68 @@ static int get_swap_writer(struct swap_map_handle *handle)
         return 0;
 }
 
-static int swap_write_page(struct swap_map_handle *handle, void *buf)
+static int wait_on_bio_chain(struct bio **bio_chain)
 {
-        int error;
+        struct bio *bio;
+        struct bio *next_bio;
+        int ret = 0;
+
+        if (bio_chain == NULL)
+                return 0;
+
+        bio = *bio_chain;
+        while (bio) {
+                struct page *page;
+
+                next_bio = bio->bi_private;
+                page = bio->bi_io_vec[0].bv_page;
+                wait_on_page_locked(page);
+                if (!PageUptodate(page) || PageError(page))
+                        ret = -EIO;
+                put_page(page);
+                bio_put(bio);
+                bio = next_bio;
+        }
+        *bio_chain = NULL;
+        return ret;
+}
+
+static int swap_write_page(struct swap_map_handle *handle, void *buf,
+                                struct bio **bio_chain)
+{
+        int error = 0;
         unsigned long offset;
 
         if (!handle->cur)
                 return -EINVAL;
         offset = alloc_swap_page(root_swap, handle->bitmap);
-        error = write_page(buf, offset);
+        error = write_page(buf, offset, bio_chain);
         if (error)
                 return error;
         handle->cur->entries[handle->k++] = offset;
         if (handle->k >= MAP_PAGE_ENTRIES) {
+                error = wait_on_bio_chain(bio_chain);
+                if (error)
+                        goto out;
                 offset = alloc_swap_page(root_swap, handle->bitmap);
                 if (!offset)
                         return -ENOSPC;
                 handle->cur->next_swap = offset;
-                error = write_page(handle->cur, handle->cur_swap);
+                error = write_page(handle->cur, handle->cur_swap, NULL);
                 if (error)
-                        return error;
+                        goto out;
                 memset(handle->cur, 0, PAGE_SIZE);
                 handle->cur_swap = offset;
                 handle->k = 0;
         }
-        return 0;
+out:
+        return error;
 }
 
 static int flush_swap_writer(struct swap_map_handle *handle)
 {
         if (handle->cur && handle->cur_swap)
-                return write_page(handle->cur, handle->cur_swap);
+                return write_page(handle->cur, handle->cur_swap, NULL);
         else
                 return -EINVAL;
 }
@@ -232,6 +282,8 @@ static int save_image(struct swap_map_handle *handle,
         int ret;
         int error = 0;
         int nr_pages;
+        int err2;
+        struct bio *bio;
         struct timeval start;
         struct timeval stop;
 
@@ -240,11 +292,13 @@ static int save_image(struct swap_map_handle *handle,
         if (!m)
                 m = 1;
         nr_pages = 0;
+        bio = NULL;
         do_gettimeofday(&start);
         do {
                 ret = snapshot_read_next(snapshot, PAGE_SIZE);
                 if (ret > 0) {
-                        error = swap_write_page(handle, data_of(*snapshot));
+                        error = swap_write_page(handle, data_of(*snapshot),
+                                                &bio);
                         if (error)
                                 break;
                         if (!(nr_pages % m))
@@ -252,8 +306,11 @@ static int save_image(struct swap_map_handle *handle,
                         nr_pages++;
                 }
         } while (ret > 0);
+        err2 = wait_on_bio_chain(&bio);
         do_gettimeofday(&stop);
         if (!error)
+                error = err2;
+        if (!error)
                 printk("\b\b\b\bdone\n");
         show_speed(&start, &stop, nr_to_write, "Wrote");
         return error;
@@ -307,7 +364,7 @@ int swsusp_write(void)
         error = get_swap_writer(&handle);
         if (!error) {
                 unsigned long start = handle.cur_swap;
-                error = swap_write_page(&handle, header);
+                error = swap_write_page(&handle, header, NULL);
                 if (!error)
                         error = save_image(&handle, &snapshot,
                                         header->pages - 1);
diff --git a/mm/page_io.c b/mm/page_io.c
index d2f0a5783370..f46a9862b7ef 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -156,10 +156,12 @@ out:
  * We use end_swap_bio_read() even for writes, because it happens to do what
  * we want.
  */
-int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page)
+int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page,
+                        struct bio **bio_chain)
 {
         struct bio *bio;
         int ret = 0;
+        int bio_rw;
 
         lock_page(page);
 
@@ -170,11 +172,22 @@ int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page)
                 goto out;
         }
 
-        submit_bio(rw | (1 << BIO_RW_SYNC), bio);
-        wait_on_page_locked(page);
-
-        if (!PageUptodate(page) || PageError(page))
-                ret = -EIO;
+        bio_rw = rw;
+        if (!bio_chain)
+                bio_rw |= (1 << BIO_RW_SYNC);
+        if (bio_chain)
+                bio_get(bio);
+        submit_bio(bio_rw, bio);
+        if (bio_chain == NULL) {
+                wait_on_page_locked(page);
+
+                if (!PageUptodate(page) || PageError(page))
+                        ret = -EIO;
+        }
+        if (bio_chain) {
+                bio->bi_private = *bio_chain;
+                *bio_chain = bio;
+        }
 out:
         return ret;
 }