diff options
-rw-r--r-- | include/linux/mm.h | 11 | ||||
-rw-r--r-- | include/linux/page-flags.h | 37 | ||||
-rw-r--r-- | mm/page_alloc.c | 10 |
3 files changed, 32 insertions, 26 deletions
diff --git a/include/linux/mm.h b/include/linux/mm.h index 8c149fa4491d..695b90437bbc 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h | |||
@@ -269,14 +269,7 @@ static inline int get_page_unless_zero(struct page *page) | |||
269 | 269 | ||
270 | static inline struct page *compound_head(struct page *page) | 270 | static inline struct page *compound_head(struct page *page) |
271 | { | 271 | { |
272 | /* | 272 | if (unlikely(PageTail(page))) |
273 | * We could avoid the PageCompound(page) check if | ||
274 | * we would not overload PageTail(). | ||
275 | * | ||
276 | * This check has to be done in several performance critical | ||
277 | * paths of the slab etc. IMHO PageTail deserves its own flag. | ||
278 | */ | ||
279 | if (unlikely(PageCompound(page) && PageTail(page))) | ||
280 | return page->first_page; | 273 | return page->first_page; |
281 | return page; | 274 | return page; |
282 | } | 275 | } |
@@ -327,7 +320,7 @@ static inline compound_page_dtor *get_compound_page_dtor(struct page *page) | |||
327 | 320 | ||
328 | static inline int compound_order(struct page *page) | 321 | static inline int compound_order(struct page *page) |
329 | { | 322 | { |
330 | if (!PageCompound(page) || PageTail(page)) | 323 | if (!PageHead(page)) |
331 | return 0; | 324 | return 0; |
332 | return (unsigned long)page[1].lru.prev; | 325 | return (unsigned long)page[1].lru.prev; |
333 | } | 326 | } |
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index a1e143634946..a3c8b60a9c3a 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h | |||
@@ -6,6 +6,7 @@ | |||
6 | #define PAGE_FLAGS_H | 6 | #define PAGE_FLAGS_H |
7 | 7 | ||
8 | #include <linux/types.h> | 8 | #include <linux/types.h> |
9 | #include <linux/mm_types.h> | ||
9 | 10 | ||
10 | /* | 11 | /* |
11 | * Various page->flags bits: | 12 | * Various page->flags bits: |
@@ -94,12 +95,6 @@ | |||
94 | /* PG_owner_priv_1 users should have descriptive aliases */ | 95 | /* PG_owner_priv_1 users should have descriptive aliases */ |
95 | #define PG_checked PG_owner_priv_1 /* Used by some filesystems */ | 96 | #define PG_checked PG_owner_priv_1 /* Used by some filesystems */ |
96 | 97 | ||
97 | /* | ||
98 | * Marks tail portion of a compound page. We currently do not reclaim | ||
99 | * compound pages so we can reuse a flag only used for reclaim here. | ||
100 | */ | ||
101 | #define PG_tail PG_reclaim | ||
102 | |||
103 | #if (BITS_PER_LONG > 32) | 98 | #if (BITS_PER_LONG > 32) |
104 | /* | 99 | /* |
105 | * 64-bit-only flags build down from bit 31 | 100 | * 64-bit-only flags build down from bit 31 |
@@ -248,12 +243,32 @@ static inline void SetPageUptodate(struct page *page) | |||
248 | #define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags) | 243 | #define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags) |
249 | 244 | ||
250 | /* | 245 | /* |
251 | * Note: PG_tail is an alias of another page flag. The result of PageTail() | 246 | * PG_reclaim is used in combination with PG_compound to mark the |
252 | * is only valid if PageCompound(page) is true. | 247 | * head and tail of a compound page |
248 | * | ||
249 | * PG_compound & PG_reclaim => Tail page | ||
250 | * PG_compound & ~PG_reclaim => Head page | ||
253 | */ | 251 | */ |
254 | #define PageTail(page) test_bit(PG_tail, &(page)->flags) | 252 | |
255 | #define __SetPageTail(page) __set_bit(PG_tail, &(page)->flags) | 253 | #define PG_head_tail_mask ((1L << PG_compound) | (1L << PG_reclaim)) |
256 | #define __ClearPageTail(page) __clear_bit(PG_tail, &(page)->flags) | 254 | |
255 | #define PageTail(page) ((page->flags & PG_head_tail_mask) \ | ||
256 | == PG_head_tail_mask) | ||
257 | |||
258 | static inline void __SetPageTail(struct page *page) | ||
259 | { | ||
260 | page->flags |= PG_head_tail_mask; | ||
261 | } | ||
262 | |||
263 | static inline void __ClearPageTail(struct page *page) | ||
264 | { | ||
265 | page->flags &= ~PG_head_tail_mask; | ||
266 | } | ||
267 | |||
268 | #define PageHead(page) ((page->flags & PG_head_tail_mask) \ | ||
269 | == (1L << PG_compound)) | ||
270 | #define __SetPageHead(page) __SetPageCompound(page) | ||
271 | #define __ClearPageHead(page) __ClearPageCompound(page) | ||
257 | 272 | ||
258 | #ifdef CONFIG_SWAP | 273 | #ifdef CONFIG_SWAP |
259 | #define PageSwapCache(page) test_bit(PG_swapcache, &(page)->flags) | 274 | #define PageSwapCache(page) test_bit(PG_swapcache, &(page)->flags) |
diff --git a/mm/page_alloc.c b/mm/page_alloc.c index fc241fe295ab..36d713e216e8 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c | |||
@@ -235,12 +235,11 @@ static void prep_compound_page(struct page *page, unsigned long order) | |||
235 | 235 | ||
236 | set_compound_page_dtor(page, free_compound_page); | 236 | set_compound_page_dtor(page, free_compound_page); |
237 | set_compound_order(page, order); | 237 | set_compound_order(page, order); |
238 | __SetPageCompound(page); | 238 | __SetPageHead(page); |
239 | for (i = 1; i < nr_pages; i++) { | 239 | for (i = 1; i < nr_pages; i++) { |
240 | struct page *p = page + i; | 240 | struct page *p = page + i; |
241 | 241 | ||
242 | __SetPageTail(p); | 242 | __SetPageTail(p); |
243 | __SetPageCompound(p); | ||
244 | p->first_page = page; | 243 | p->first_page = page; |
245 | } | 244 | } |
246 | } | 245 | } |
@@ -253,17 +252,16 @@ static void destroy_compound_page(struct page *page, unsigned long order) | |||
253 | if (unlikely(compound_order(page) != order)) | 252 | if (unlikely(compound_order(page) != order)) |
254 | bad_page(page); | 253 | bad_page(page); |
255 | 254 | ||
256 | if (unlikely(!PageCompound(page))) | 255 | if (unlikely(!PageHead(page))) |
257 | bad_page(page); | 256 | bad_page(page); |
258 | __ClearPageCompound(page); | 257 | __ClearPageHead(page); |
259 | for (i = 1; i < nr_pages; i++) { | 258 | for (i = 1; i < nr_pages; i++) { |
260 | struct page *p = page + i; | 259 | struct page *p = page + i; |
261 | 260 | ||
262 | if (unlikely(!PageCompound(p) | !PageTail(p) | | 261 | if (unlikely(!PageTail(p) | |
263 | (p->first_page != page))) | 262 | (p->first_page != page))) |
264 | bad_page(page); | 263 | bad_page(page); |
265 | __ClearPageTail(p); | 264 | __ClearPageTail(p); |
266 | __ClearPageCompound(p); | ||
267 | } | 265 | } |
268 | } | 266 | } |
269 | 267 | ||