aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/mm.h11
-rw-r--r--include/linux/page-flags.h37
-rw-r--r--mm/page_alloc.c10
3 files changed, 32 insertions, 26 deletions
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 8c149fa4491d..695b90437bbc 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -269,14 +269,7 @@ static inline int get_page_unless_zero(struct page *page)
269 269
270static inline struct page *compound_head(struct page *page) 270static inline struct page *compound_head(struct page *page)
271{ 271{
272 /* 272 if (unlikely(PageTail(page)))
273 * We could avoid the PageCompound(page) check if
274 * we would not overload PageTail().
275 *
276 * This check has to be done in several performance critical
277 * paths of the slab etc. IMHO PageTail deserves its own flag.
278 */
279 if (unlikely(PageCompound(page) && PageTail(page)))
280 return page->first_page; 273 return page->first_page;
281 return page; 274 return page;
282} 275}
@@ -327,7 +320,7 @@ static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
327 320
328static inline int compound_order(struct page *page) 321static inline int compound_order(struct page *page)
329{ 322{
330 if (!PageCompound(page) || PageTail(page)) 323 if (!PageHead(page))
331 return 0; 324 return 0;
332 return (unsigned long)page[1].lru.prev; 325 return (unsigned long)page[1].lru.prev;
333} 326}
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index a1e143634946..a3c8b60a9c3a 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -6,6 +6,7 @@
6#define PAGE_FLAGS_H 6#define PAGE_FLAGS_H
7 7
8#include <linux/types.h> 8#include <linux/types.h>
9#include <linux/mm_types.h>
9 10
10/* 11/*
11 * Various page->flags bits: 12 * Various page->flags bits:
@@ -94,12 +95,6 @@
94/* PG_owner_priv_1 users should have descriptive aliases */ 95/* PG_owner_priv_1 users should have descriptive aliases */
95#define PG_checked PG_owner_priv_1 /* Used by some filesystems */ 96#define PG_checked PG_owner_priv_1 /* Used by some filesystems */
96 97
97/*
98 * Marks tail portion of a compound page. We currently do not reclaim
99 * compound pages so we can reuse a flag only used for reclaim here.
100 */
101#define PG_tail PG_reclaim
102
103#if (BITS_PER_LONG > 32) 98#if (BITS_PER_LONG > 32)
104/* 99/*
105 * 64-bit-only flags build down from bit 31 100 * 64-bit-only flags build down from bit 31
@@ -248,12 +243,32 @@ static inline void SetPageUptodate(struct page *page)
248#define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags) 243#define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags)
249 244
250/* 245/*
251 * Note: PG_tail is an alias of another page flag. The result of PageTail() 246 * PG_reclaim is used in combination with PG_compound to mark the
252 * is only valid if PageCompound(page) is true. 247 * head and tail of a compound page
248 *
249 * PG_compound & PG_reclaim => Tail page
250 * PG_compound & ~PG_reclaim => Head page
253 */ 251 */
254#define PageTail(page) test_bit(PG_tail, &(page)->flags) 252
255#define __SetPageTail(page) __set_bit(PG_tail, &(page)->flags) 253#define PG_head_tail_mask ((1L << PG_compound) | (1L << PG_reclaim))
256#define __ClearPageTail(page) __clear_bit(PG_tail, &(page)->flags) 254
255#define PageTail(page) ((page->flags & PG_head_tail_mask) \
256 == PG_head_tail_mask)
257
258static inline void __SetPageTail(struct page *page)
259{
260 page->flags |= PG_head_tail_mask;
261}
262
263static inline void __ClearPageTail(struct page *page)
264{
265 page->flags &= ~PG_head_tail_mask;
266}
267
268#define PageHead(page) ((page->flags & PG_head_tail_mask) \
269 == (1L << PG_compound))
270#define __SetPageHead(page) __SetPageCompound(page)
271#define __ClearPageHead(page) __ClearPageCompound(page)
257 272
258#ifdef CONFIG_SWAP 273#ifdef CONFIG_SWAP
259#define PageSwapCache(page) test_bit(PG_swapcache, &(page)->flags) 274#define PageSwapCache(page) test_bit(PG_swapcache, &(page)->flags)
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index fc241fe295ab..36d713e216e8 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -235,12 +235,11 @@ static void prep_compound_page(struct page *page, unsigned long order)
235 235
236 set_compound_page_dtor(page, free_compound_page); 236 set_compound_page_dtor(page, free_compound_page);
237 set_compound_order(page, order); 237 set_compound_order(page, order);
238 __SetPageCompound(page); 238 __SetPageHead(page);
239 for (i = 1; i < nr_pages; i++) { 239 for (i = 1; i < nr_pages; i++) {
240 struct page *p = page + i; 240 struct page *p = page + i;
241 241
242 __SetPageTail(p); 242 __SetPageTail(p);
243 __SetPageCompound(p);
244 p->first_page = page; 243 p->first_page = page;
245 } 244 }
246} 245}
@@ -253,17 +252,16 @@ static void destroy_compound_page(struct page *page, unsigned long order)
253 if (unlikely(compound_order(page) != order)) 252 if (unlikely(compound_order(page) != order))
254 bad_page(page); 253 bad_page(page);
255 254
256 if (unlikely(!PageCompound(page))) 255 if (unlikely(!PageHead(page)))
257 bad_page(page); 256 bad_page(page);
258 __ClearPageCompound(page); 257 __ClearPageHead(page);
259 for (i = 1; i < nr_pages; i++) { 258 for (i = 1; i < nr_pages; i++) {
260 struct page *p = page + i; 259 struct page *p = page + i;
261 260
262 if (unlikely(!PageCompound(p) | !PageTail(p) | 261 if (unlikely(!PageTail(p) |
263 (p->first_page != page))) 262 (p->first_page != page)))
264 bad_page(page); 263 bad_page(page);
265 __ClearPageTail(p); 264 __ClearPageTail(p);
266 __ClearPageCompound(p);
267 } 265 }
268} 266}
269 267