6 files changed, 79 insertions, 65 deletions

diff --git a/mm/filemap.c b/mm/filemap.c
index 606432f71b3a..8332c77b1bd1 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -1181,8 +1181,6 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
                 if (pos < size) {
                         retval = generic_file_direct_IO(READ, iocb,
                                                 iov, pos, nr_segs);
-                        if (retval > 0 && !is_sync_kiocb(iocb))
-                                retval = -EIOCBQUEUED;
                         if (retval > 0)
                                 *ppos = pos + retval;
                 }
@@ -2047,15 +2045,14 @@ generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
          * Sync the fs metadata but not the minor inode changes and
          * of course not the data as we did direct DMA for the IO.
          * i_mutex is held, which protects generic_osync_inode() from
-         * livelocking.
+         * livelocking.  AIO O_DIRECT ops attempt to sync metadata here.
          */
-        if (written >= 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
+        if ((written >= 0 || written == -EIOCBQUEUED) &&
+            ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
                 int err = generic_osync_inode(inode, mapping, OSYNC_METADATA);
                 if (err < 0)
                         written = err;
         }
-        if (written == count && !is_sync_kiocb(iocb))
-                written = -EIOCBQUEUED;
         return written;
 }
 EXPORT_SYMBOL(generic_file_direct_write);
diff --git a/mm/memory.c b/mm/memory.c
index 4198df0dff1c..bf6100236e62 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1110,23 +1110,29 @@ static int zeromap_pte_range(struct mm_struct *mm, pmd_t *pmd,
 {
         pte_t *pte;
         spinlock_t *ptl;
+        int err = 0;
 
         pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
         if (!pte)
-                return -ENOMEM;
+                return -EAGAIN;
         arch_enter_lazy_mmu_mode();
         do {
                 struct page *page = ZERO_PAGE(addr);
                 pte_t zero_pte = pte_wrprotect(mk_pte(page, prot));
+
+                if (unlikely(!pte_none(*pte))) {
+                        err = -EEXIST;
+                        pte++;
+                        break;
+                }
                 page_cache_get(page);
                 page_add_file_rmap(page);
                 inc_mm_counter(mm, file_rss);
-                BUG_ON(!pte_none(*pte));
                 set_pte_at(mm, addr, pte, zero_pte);
         } while (pte++, addr += PAGE_SIZE, addr != end);
         arch_leave_lazy_mmu_mode();
         pte_unmap_unlock(pte - 1, ptl);
-        return 0;
+        return err;
 }
 
 static inline int zeromap_pmd_range(struct mm_struct *mm, pud_t *pud,
@@ -1134,16 +1140,18 @@ static inline int zeromap_pmd_range(struct mm_struct *mm, pud_t *pud,
 {
         pmd_t *pmd;
         unsigned long next;
+        int err;
 
         pmd = pmd_alloc(mm, pud, addr);
         if (!pmd)
-                return -ENOMEM;
+                return -EAGAIN;
         do {
                 next = pmd_addr_end(addr, end);
-                if (zeromap_pte_range(mm, pmd, addr, next, prot))
-                        return -ENOMEM;
+                err = zeromap_pte_range(mm, pmd, addr, next, prot);
+                if (err)
+                        break;
         } while (pmd++, addr = next, addr != end);
-        return 0;
+        return err;
 }
 
 static inline int zeromap_pud_range(struct mm_struct *mm, pgd_t *pgd,
@@ -1151,16 +1159,18 @@ static inline int zeromap_pud_range(struct mm_struct *mm, pgd_t *pgd,
 {
         pud_t *pud;
         unsigned long next;
+        int err;
 
         pud = pud_alloc(mm, pgd, addr);
         if (!pud)
-                return -ENOMEM;
+                return -EAGAIN;
         do {
                 next = pud_addr_end(addr, end);
-                if (zeromap_pmd_range(mm, pud, addr, next, prot))
-                        return -ENOMEM;
+                err = zeromap_pmd_range(mm, pud, addr, next, prot);
+                if (err)
+                        break;
         } while (pud++, addr = next, addr != end);
-        return 0;
+        return err;
 }
 
 int zeromap_page_range(struct vm_area_struct *vma,
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 8d9b19f239c3..237107c1b084 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -21,6 +21,7 @@
 #include <linux/writeback.h>
 #include <linux/init.h>
 #include <linux/backing-dev.h>
+#include <linux/task_io_accounting_ops.h>
 #include <linux/blkdev.h>
 #include <linux/mpage.h>
 #include <linux/rmap.h>
@@ -761,23 +762,24 @@ int __set_page_dirty_nobuffers(struct page *page)
                 struct address_space *mapping = page_mapping(page);
                 struct address_space *mapping2;
 
-                if (mapping) {
-                        write_lock_irq(&mapping->tree_lock);
-                        mapping2 = page_mapping(page);
-                        if (mapping2) { /* Race with truncate? */
-                                BUG_ON(mapping2 != mapping);
-                                if (mapping_cap_account_dirty(mapping))
-                                        __inc_zone_page_state(page,
-                                                                NR_FILE_DIRTY);
-                                radix_tree_tag_set(&mapping->page_tree,
-                                        page_index(page), PAGECACHE_TAG_DIRTY);
-                        }
-                        write_unlock_irq(&mapping->tree_lock);
-                        if (mapping->host) {
-                                /* !PageAnon && !swapper_space */
-                                __mark_inode_dirty(mapping->host,
-                                                        I_DIRTY_PAGES);
+                if (!mapping)
+                        return 1;
+
+                write_lock_irq(&mapping->tree_lock);
+                mapping2 = page_mapping(page);
+                if (mapping2) { /* Race with truncate? */
+                        BUG_ON(mapping2 != mapping);
+                        if (mapping_cap_account_dirty(mapping)) {
+                                __inc_zone_page_state(page, NR_FILE_DIRTY);
+                                task_io_account_write(PAGE_CACHE_SIZE);
                         }
+                        radix_tree_tag_set(&mapping->page_tree,
+                                page_index(page), PAGECACHE_TAG_DIRTY);
+                }
+                write_unlock_irq(&mapping->tree_lock);
+                if (mapping->host) {
+                        /* !PageAnon && !swapper_space */
+                        __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
                 }
                 return 1;
         }
@@ -851,27 +853,26 @@ int test_clear_page_dirty(struct page *page)
         struct address_space *mapping = page_mapping(page);
         unsigned long flags;
 
-        if (mapping) {
-                write_lock_irqsave(&mapping->tree_lock, flags);
-                if (TestClearPageDirty(page)) {
-                        radix_tree_tag_clear(&mapping->page_tree,
-                                                page_index(page),
-                                                PAGECACHE_TAG_DIRTY);
-                        write_unlock_irqrestore(&mapping->tree_lock, flags);
-                        /*
-                         * We can continue to use `mapping' here because the
-                         * page is locked, which pins the address_space
-                         */
-                        if (mapping_cap_account_dirty(mapping)) {
-                                page_mkclean(page);
-                                dec_zone_page_state(page, NR_FILE_DIRTY);
-                        }
-                        return 1;
-                }
+        if (!mapping)
+                return TestClearPageDirty(page);
+
+        write_lock_irqsave(&mapping->tree_lock, flags);
+        if (TestClearPageDirty(page)) {
+                radix_tree_tag_clear(&mapping->page_tree,
+                                page_index(page), PAGECACHE_TAG_DIRTY);
                 write_unlock_irqrestore(&mapping->tree_lock, flags);
-                return 0;
+                /*
+                 * We can continue to use `mapping' here because the
+                 * page is locked, which pins the address_space
+                 */
+                if (mapping_cap_account_dirty(mapping)) {
+                        page_mkclean(page);
+                        dec_zone_page_state(page, NR_FILE_DIRTY);
+                }
+                return 1;
         }
-        return TestClearPageDirty(page);
+        write_unlock_irqrestore(&mapping->tree_lock, flags);
+        return 0;
 }
 EXPORT_SYMBOL(test_clear_page_dirty);
 
@@ -893,17 +894,17 @@ int clear_page_dirty_for_io(struct page *page)
 {
         struct address_space *mapping = page_mapping(page);
 
-        if (mapping) {
-                if (TestClearPageDirty(page)) {
-                        if (mapping_cap_account_dirty(mapping)) {
-                                page_mkclean(page);
-                                dec_zone_page_state(page, NR_FILE_DIRTY);
-                        }
-                        return 1;
+        if (!mapping)
+                return TestClearPageDirty(page);
+
+        if (TestClearPageDirty(page)) {
+                if (mapping_cap_account_dirty(mapping)) {
+                        page_mkclean(page);
+                        dec_zone_page_state(page, NR_FILE_DIRTY);
                 }
-                return 0;
+                return 1;
         }
-        return TestClearPageDirty(page);
+        return 0;
 }
 EXPORT_SYMBOL(clear_page_dirty_for_io);
 
diff --git a/mm/readahead.c b/mm/readahead.c
index c0df5ed05f62..0f539e8e827a 100644
--- a/mm/readahead.c
+++ b/mm/readahead.c
@@ -13,6 +13,7 @@
 #include <linux/module.h>
 #include <linux/blkdev.h>
 #include <linux/backing-dev.h>
+#include <linux/task_io_accounting_ops.h>
 #include <linux/pagevec.h>
 
 void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page)
@@ -151,6 +152,7 @@ int read_cache_pages(struct address_space *mapping, struct list_head *pages,
                         put_pages_list(pages);
                         break;
                 }
+                task_io_account_read(PAGE_CACHE_SIZE);
         }
         pagevec_lru_add(&lru_pvec);
         return ret;
diff --git a/mm/slab.c b/mm/slab.c
index 56af694c9e6a..2c655532f5ef 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -946,7 +946,8 @@ static void __devinit start_cpu_timer(int cpu)
         if (keventd_up() && reap_work->work.func == NULL) {
                 init_reap_node(cpu);
                 INIT_DELAYED_WORK(reap_work, cache_reap);
-                schedule_delayed_work_on(cpu, reap_work, HZ + 3 * cpu);
+                schedule_delayed_work_on(cpu, reap_work,
+                                        __round_jiffies_relative(HZ, cpu));
         }
 }
 
@@ -4006,7 +4007,7 @@ static void cache_reap(struct work_struct *unused)
         if (!mutex_trylock(&cache_chain_mutex)) {
                 /* Give up. Setup the next iteration. */
                 schedule_delayed_work(&__get_cpu_var(reap_work),
-                                      REAPTIMEOUT_CPUC);
+                                      round_jiffies_relative(REAPTIMEOUT_CPUC));
                 return;
         }
 
@@ -4052,7 +4053,8 @@ next:
         next_reap_node();
         refresh_cpu_vm_stats(smp_processor_id());
         /* Set up the next iteration */
-        schedule_delayed_work(&__get_cpu_var(reap_work), REAPTIMEOUT_CPUC);
+        schedule_delayed_work(&__get_cpu_var(reap_work),
+                round_jiffies_relative(REAPTIMEOUT_CPUC));
 }
 
 #ifdef CONFIG_PROC_FS
diff --git a/mm/truncate.c b/mm/truncate.c
index e07b1e682c38..9bfb8e853860 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -13,6 +13,7 @@
 #include <linux/module.h>
 #include <linux/pagemap.h>
 #include <linux/pagevec.h>
+#include <linux/task_io_accounting_ops.h>
 #include <linux/buffer_head.h>  /* grr. try_to_release_page,
                                    do_invalidatepage */
 
@@ -69,7 +70,8 @@ truncate_complete_page(struct address_space *mapping, struct page *page)
         if (PagePrivate(page))
                 do_invalidatepage(page, 0);
 
-        clear_page_dirty(page);
+        if (test_clear_page_dirty(page))
+                task_io_account_cancelled_write(PAGE_CACHE_SIZE);
         ClearPageUptodate(page);
         ClearPageMappedToDisk(page);
         remove_from_page_cache(page);