mm: fix slab->page flags corruption

Transparent huge pages can change page->flags (PG_compound_lock) without
taking Slab lock.  Since THP can not break slab pages we can safely access
compound page without taking compound lock.

Specifically this patch fixes a race between compound_unlock() and slab
functions which perform page-flags updates.  This can occur when
get_page()/put_page() is called on a page from slab.

[akpm@linux-foundation.org: tweak comment text, fix comment layout, fix label indenting]
Reported-by: Amey Bhide <abhide@nicira.com>
Signed-off-by: Pravin B Shelar <pshelar@nicira.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Acked-by: Andrea Arcangeli <aarcange@redhat.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 35 insertions, 2 deletions

diff --git a/mm/swap.c b/mm/swap.c
index 5c13f1338972..6fdd72ec15b0 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -82,6 +82,25 @@ static void put_compound_page(struct page *page)
                 if (likely(page != page_head &&
                            get_page_unless_zero(page_head))) {
                         unsigned long flags;
+
+                        /*
+                         * THP can not break up slab pages so avoid taking
+                         * compound_lock().  Slab performs non-atomic bit ops
+                         * on page->flags for better performance.  In particular
+                         * slab_unlock() in slub used to be a hot path.  It is
+                         * still hot on arches that do not support
+                         * this_cpu_cmpxchg_double().
+                         */
+                        if (PageSlab(page_head)) {
+                                if (PageTail(page)) {
+                                        if (put_page_testzero(page_head))
+                                                VM_BUG_ON(1);
+
+                                        atomic_dec(&page->_mapcount);
+                                        goto skip_lock_tail;
+                                } else
+                                        goto skip_lock;
+                        }
                         /*
                          * page_head wasn't a dangling pointer but it
                          * may not be a head page anymore by the time
@@ -92,10 +111,10 @@ static void put_compound_page(struct page *page)
                         if (unlikely(!PageTail(page))) {
                                 /* __split_huge_page_refcount run before us */
                                 compound_unlock_irqrestore(page_head, flags);
-                                VM_BUG_ON(PageHead(page_head));
+skip_lock:
                                 if (put_page_testzero(page_head))
                                         __put_single_page(page_head);
-                        out_put_single:
+out_put_single:
                                 if (put_page_testzero(page))
                                         __put_single_page(page);
                                 return;
@@ -115,6 +134,8 @@ static void put_compound_page(struct page *page)
                         VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
                         VM_BUG_ON(atomic_read(&page->_count) != 0);
                         compound_unlock_irqrestore(page_head, flags);
+
+skip_lock_tail:
                         if (put_page_testzero(page_head)) {
                                 if (PageHead(page_head))
                                         __put_compound_page(page_head);
@@ -162,6 +183,18 @@ bool __get_page_tail(struct page *page)
         struct page *page_head = compound_trans_head(page);
 
         if (likely(page != page_head && get_page_unless_zero(page_head))) {
+
+                /* Ref to put_compound_page() comment. */
+                if (PageSlab(page_head)) {
+                        if (likely(PageTail(page))) {
+                                __get_page_tail_foll(page, false);
+                                return true;
+                        } else {
+                                put_page(page_head);
+                                return false;
+                        }
+                }
+
                 /*
                  * page_head wasn't a dangling pointer but it
                  * may not be a head page anymore by the time