2 files changed, 39 insertions, 7 deletions

diff --git a/include/linux/highmem.h b/include/linux/highmem.h
index 61a5e5eb27f0..7dcbc82f3b7b 100644
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -68,8 +68,6 @@ static inline void clear_user_highpage(struct page *page, unsigned long vaddr)
         void *addr = kmap_atomic(page, KM_USER0);
         clear_user_page(addr, vaddr, page);
         kunmap_atomic(addr, KM_USER0);
-        /* Make sure this page is cleared on other CPU's too before using it */
-        smp_wmb();
 }
 
 #ifndef __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
@@ -172,8 +170,6 @@ static inline void copy_user_highpage(struct page *to, struct page *from,
         copy_user_page(vto, vfrom, vaddr, to);
         kunmap_atomic(vfrom, KM_USER0);
         kunmap_atomic(vto, KM_USER1);
-        /* Make sure this page is cleared on other CPU's too before using it */
-        smp_wmb();
 }
 
 #endif
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 209d3a47f50f..bbad43fb8181 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -131,16 +131,52 @@
 #define ClearPageReferenced(page)       clear_bit(PG_referenced, &(page)->flags)
 #define TestClearPageReferenced(page) test_and_clear_bit(PG_referenced, &(page)->flags)
 
-#define PageUptodate(page)      test_bit(PG_uptodate, &(page)->flags)
+static inline int PageUptodate(struct page *page)
+{
+        int ret = test_bit(PG_uptodate, &(page)->flags);
+
+        /*
+         * Must ensure that the data we read out of the page is loaded
+         * _after_ we've loaded page->flags to check for PageUptodate.
+         * We can skip the barrier if the page is not uptodate, because
+         * we wouldn't be reading anything from it.
+         *
+         * See SetPageUptodate() for the other side of the story.
+         */
+        if (ret)
+                smp_rmb();
+
+        return ret;
+}
+
+static inline void __SetPageUptodate(struct page *page)
+{
+        smp_wmb();
+        __set_bit(PG_uptodate, &(page)->flags);
 #ifdef CONFIG_S390
+        page_clear_dirty(page);
+#endif
+}
+
 static inline void SetPageUptodate(struct page *page)
 {
+#ifdef CONFIG_S390
         if (!test_and_set_bit(PG_uptodate, &page->flags))
                 page_clear_dirty(page);
-}
 #else
-#define SetPageUptodate(page)   set_bit(PG_uptodate, &(page)->flags)
+        /*
+         * Memory barrier must be issued before setting the PG_uptodate bit,
+         * so that all previous stores issued in order to bring the page
+         * uptodate are actually visible before PageUptodate becomes true.
+         *
+         * s390 doesn't need an explicit smp_wmb here because the test and
+         * set bit already provides full barriers.
+         */
+        smp_wmb();
+        set_bit(PG_uptodate, &(page)->flags);
 #endif
+}
+
 #define ClearPageUptodate(page) clear_bit(PG_uptodate, &(page)->flags)
 
 #define PageDirty(page)         test_bit(PG_dirty, &(page)->flags)