9 files changed, 72 insertions, 37 deletions

diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c
index 4f36987eea72..2da841110727 100644
--- a/arch/ia64/mm/init.c
+++ b/arch/ia64/mm/init.c
@@ -121,7 +121,7 @@ lazy_mmu_prot_update (pte_t pte)
                 return;                         /* i-cache is already coherent with d-cache */
 
         if (PageCompound(page)) {
-                order = (unsigned long) (page[1].lru.prev);
+                order = compound_order(page);
                 flush_icache_range(addr, addr + (1UL << order << PAGE_SHIFT));
         }
         else
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 9ba71b252f3e..8e1b7825e2f3 100644
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -450,7 +450,7 @@ static int hugetlbfs_symlink(struct inode *dir,
  */
 static int hugetlbfs_set_page_dirty(struct page *page)
 {
-        struct page *head = (struct page *)page_private(page);
+        struct page *head = compound_head(page);
 
         SetPageDirty(head);
         return 0;
diff --git a/include/linux/mm.h b/include/linux/mm.h
index c95d96ebd5ad..8c149fa4491d 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -267,17 +267,28 @@ static inline int get_page_unless_zero(struct page *page)
         return atomic_inc_not_zero(&page->_count);
 }
 
+static inline struct page *compound_head(struct page *page)
+{
+        /*
+         * We could avoid the PageCompound(page) check if
+         * we would not overload PageTail().
+         *
+         * This check has to be done in several performance critical
+         * paths of the slab etc. IMHO PageTail deserves its own flag.
+         */
+        if (unlikely(PageCompound(page) && PageTail(page)))
+                return page->first_page;
+        return page;
+}
+
 static inline int page_count(struct page *page)
 {
-        if (unlikely(PageCompound(page)))
-                page = (struct page *)page_private(page);
-        return atomic_read(&page->_count);
+        return atomic_read(&compound_head(page)->_count);
 }
 
 static inline void get_page(struct page *page)
 {
-        if (unlikely(PageCompound(page)))
-                page = (struct page *)page_private(page);
+        page = compound_head(page);
         VM_BUG_ON(atomic_read(&page->_count) == 0);
         atomic_inc(&page->_count);
 }
@@ -314,6 +325,18 @@ static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
         return (compound_page_dtor *)page[1].lru.next;
 }
 
+static inline int compound_order(struct page *page)
+{
+        if (!PageCompound(page) || PageTail(page))
+                return 0;
+        return (unsigned long)page[1].lru.prev;
+}
+
+static inline void set_compound_order(struct page *page, unsigned long order)
+{
+        page[1].lru.prev = (void *)order;
+}
+
 /*
  * Multiple processes may "see" the same page. E.g. for untouched
  * mappings of /dev/null, all processes see the same page full of
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 96326594e55d..a1e143634946 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -94,6 +94,12 @@
 /* PG_owner_priv_1 users should have descriptive aliases */
 #define PG_checked              PG_owner_priv_1 /* Used by some filesystems */
 
+/*
+ * Marks tail portion of a compound page. We currently do not reclaim
+ * compound pages so we can reuse a flag only used for reclaim here.
+ */
+#define PG_tail                 PG_reclaim
+
 #if (BITS_PER_LONG > 32)
 /*
  * 64-bit-only flags build down from bit 31
@@ -241,6 +247,14 @@ static inline void SetPageUptodate(struct page *page)
 #define __SetPageCompound(page) __set_bit(PG_compound, &(page)->flags)
 #define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags)
 
+/*
+ * Note: PG_tail is an alias of another page flag. The result of PageTail()
+ * is only valid if PageCompound(page) is true.
+ */
+#define PageTail(page)  test_bit(PG_tail, &(page)->flags)
+#define __SetPageTail(page)     __set_bit(PG_tail, &(page)->flags)
+#define __ClearPageTail(page)   __clear_bit(PG_tail, &(page)->flags)
+
 #ifdef CONFIG_SWAP
 #define PageSwapCache(page)     test_bit(PG_swapcache, &(page)->flags)
 #define SetPageSwapCache(page)  set_bit(PG_swapcache, &(page)->flags)
diff --git a/mm/internal.h b/mm/internal.h
index d527b80b292f..a3110c02aea7 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -24,7 +24,7 @@ static inline void set_page_count(struct page *page, int v)
  */
 static inline void set_page_refcounted(struct page *page)
 {
-        VM_BUG_ON(PageCompound(page) && page_private(page) != (unsigned long)page);
+        VM_BUG_ON(PageCompound(page) && PageTail(page));
         VM_BUG_ON(atomic_read(&page->_count));
         set_page_count(page, 1);
 }
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 542fc088ff50..fc241fe295ab 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -225,7 +225,7 @@ static void bad_page(struct page *page)
 
 static void free_compound_page(struct page *page)
 {
-        __free_pages_ok(page, (unsigned long)page[1].lru.prev);
+        __free_pages_ok(page, compound_order(page));
 }
 
 static void prep_compound_page(struct page *page, unsigned long order)
@@ -234,12 +234,14 @@ static void prep_compound_page(struct page *page, unsigned long order)
         int nr_pages = 1 << order;
 
         set_compound_page_dtor(page, free_compound_page);
-        page[1].lru.prev = (void *)order;
-        for (i = 0; i < nr_pages; i++) {
+        set_compound_order(page, order);
+        __SetPageCompound(page);
+        for (i = 1; i < nr_pages; i++) {
                 struct page *p = page + i;
 
+                __SetPageTail(p);
                 __SetPageCompound(p);
-                set_page_private(p, (unsigned long)page);
+                p->first_page = page;
         }
 }
 
@@ -248,15 +250,19 @@ static void destroy_compound_page(struct page *page, unsigned long order)
         int i;
         int nr_pages = 1 << order;
 
-        if (unlikely((unsigned long)page[1].lru.prev != order))
+        if (unlikely(compound_order(page) != order))
                 bad_page(page);
 
-        for (i = 0; i < nr_pages; i++) {
+        if (unlikely(!PageCompound(page)))
+                        bad_page(page);
+        __ClearPageCompound(page);
+        for (i = 1; i < nr_pages; i++) {
                 struct page *p = page + i;
 
-                if (unlikely(!PageCompound(p) |
-                                (page_private(p) != (unsigned long)page)))
+                if (unlikely(!PageCompound(p) | !PageTail(p) |
+                                (p->first_page != page)))
                         bad_page(page);
+                __ClearPageTail(p);
                 __ClearPageCompound(p);
         }
 }
@@ -429,13 +435,18 @@ static inline int free_pages_check(struct page *page)
                         1 << PG_private |
                         1 << PG_locked  |
                         1 << PG_active  |
-                        1 << PG_reclaim |
                         1 << PG_slab    |
                         1 << PG_swapcache |
                         1 << PG_writeback |
                         1 << PG_reserved |
                         1 << PG_buddy ))))
                 bad_page(page);
+        /*
+         * PageReclaim == PageTail. It is only an error
+         * for PageReclaim to be set if PageCompound is clear.
+         */
+        if (unlikely(!PageCompound(page) && PageReclaim(page)))
+                bad_page(page);
         if (PageDirty(page))
                 __ClearPageDirty(page);
         /*
diff --git a/mm/slab.c b/mm/slab.c
index 9cd01fa60004..f4b2e22b5c61 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -602,8 +602,7 @@ static inline void page_set_cache(struct page *page, struct kmem_cache *cache)
 
 static inline struct kmem_cache *page_get_cache(struct page *page)
 {
-        if (unlikely(PageCompound(page)))
-                page = (struct page *)page_private(page);
+        page = compound_head(page);
         BUG_ON(!PageSlab(page));
         return (struct kmem_cache *)page->lru.next;
 }
@@ -615,8 +614,7 @@ static inline void page_set_slab(struct page *page, struct slab *slab)
 
 static inline struct slab *page_get_slab(struct page *page)
 {
-        if (unlikely(PageCompound(page)))
-                page = (struct page *)page_private(page);
+        page = compound_head(page);
         BUG_ON(!PageSlab(page));
         return (struct slab *)page->lru.prev;
 }
diff --git a/mm/slub.c b/mm/slub.c
index 9d52cce7c999..8fa1c6e937f5 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1325,9 +1325,7 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
 
         page = virt_to_page(x);
 
-        if (unlikely(PageCompound(page)))
-                page = page->first_page;
-
+        page = compound_head(page);
 
         if (unlikely(PageError(page) && (s->flags & SLAB_STORE_USER)))
                 set_tracking(s, x, TRACK_FREE);
@@ -1338,10 +1336,7 @@ EXPORT_SYMBOL(kmem_cache_free);
 /* Figure out on which slab object the object resides */
 static struct page *get_object_page(const void *x)
 {
-        struct page *page = virt_to_page(x);
-
-        if (unlikely(PageCompound(page)))
-                page = page->first_page;
+        struct page *page = compound_head(virt_to_page(x));
 
         if (!PageSlab(page))
                 return NULL;
@@ -2081,10 +2076,7 @@ void kfree(const void *x)
         if (!x)
                 return;
 
-        page = virt_to_page(x);
-
-        if (unlikely(PageCompound(page)))
-                page = page->first_page;
+        page = compound_head(virt_to_page(x));
 
         s = page->slab;
 
@@ -2120,10 +2112,7 @@ void *krealloc(const void *p, size_t new_size, gfp_t flags)
                 return NULL;
         }
 
-        page = virt_to_page(p);
-
-        if (unlikely(PageCompound(page)))
-                page = page->first_page;
+        page = compound_head(virt_to_page(p));
 
         new_cache = get_slab(new_size, flags);
 
diff --git a/mm/swap.c b/mm/swap.c
index 2ed7be39795e..218c52a24a21 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -55,7 +55,7 @@ static void fastcall __page_cache_release(struct page *page)
 
 static void put_compound_page(struct page *page)
 {
-        page = (struct page *)page_private(page);
+        page = compound_head(page);
         if (put_page_testzero(page)) {
                 compound_page_dtor *dtor;