aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChristoph Lameter <clameter@sgi.com>2008-04-28 05:12:55 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2008-04-28 11:58:22 -0400
commite20b8cca760ed2a6abcfe37ef56f2306790db648 (patch)
tree85e5610364f73da193ee781be61710fddfbe045f
parent97965478a66fbdf0f4ad5e4ecc4828f0cb548a45 (diff)
PAGEFLAGS_EXTENDED and separate page flags for Head and Tail
Having separate page flags for the head and the tail of a compound page allows the compiler to use bitops instead of operations on a word to check for a tail page. That is f.e. important for virt_to_head_page() which is used in various critical code paths (kfree for example): Code for PageTail(page) Before: mov (%rdi),%rdx page->flags mov %rdx,%rax 3 bytes and $0x12000,%eax 5 bytes cmp $0x12000,%rax 6 bytes je 897 <kfree+0xa7> After: mov (%rdi),%rax test $0x40,%ah (3 bytes) jne 887 <kfree+0x97> So we go from 14 bytes to 3 bytes and from 3 instructions to one. From the use of 2 registers we go to none. We can only use page flags for this if we have page flags available. This patch introduces CONFIG_PAGEFLAGS_EXTENDED that is set if pageflags are not scarce due to SPARSEMEM using page flags for its sectionid on 32 bit NUMA platforms. Additional page flag definitions can be added to the CONFIG_PAGEFLAGS_EXTENDED section in page-flags.h if the functionality depends on PAGEFLAGS_EXTENDED or if more page flag overlapping tricks are used for the !PAGEFLAGS_EXTENDED fallback (the upcoming virtual compound patch may hook in here and Rik's/Lee's additional page flags to solve the reclaim issues could also be added there [hint... hint... where are these patchsets?]). Avoiding the overlaying of Pg_reclaim also clears the way for possible use of compound pages for the pagecache or on the LRU. Signed-off-by: Christoph Lameter <clameter@sgi.com> Cc: Nick Piggin <nickpiggin@yahoo.com.au> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
-rw-r--r--include/linux/page-flags.h28
-rw-r--r--mm/Kconfig12
2 files changed, 40 insertions, 0 deletions
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index d16efa9066d9..590cff32415d 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -83,7 +83,12 @@ enum pageflags {
83 PG_reserved, 83 PG_reserved,
84 PG_private, /* If pagecache, has fs-private data */ 84 PG_private, /* If pagecache, has fs-private data */
85 PG_writeback, /* Page is under writeback */ 85 PG_writeback, /* Page is under writeback */
86#ifdef CONFIG_PAGEFLAGS_EXTENDED
87 PG_head, /* A head page */
88 PG_tail, /* A tail page */
89#else
86 PG_compound, /* A compound page */ 90 PG_compound, /* A compound page */
91#endif
87 PG_swapcache, /* Swap page: swp_entry_t in private */ 92 PG_swapcache, /* Swap page: swp_entry_t in private */
88 PG_mappedtodisk, /* Has blocks allocated on-disk */ 93 PG_mappedtodisk, /* Has blocks allocated on-disk */
89 PG_reclaim, /* To be reclaimed asap */ 94 PG_reclaim, /* To be reclaimed asap */
@@ -248,6 +253,28 @@ static inline void set_page_writeback(struct page *page)
248 test_set_page_writeback(page); 253 test_set_page_writeback(page);
249} 254}
250 255
256#ifdef CONFIG_PAGEFLAGS_EXTENDED
257/*
258 * System with lots of page flags available. This allows separate
259 * flags for PageHead() and PageTail() checks of compound pages so that bit
260 * tests can be used in performance sensitive paths. PageCompound is
261 * generally not used in hot code paths.
262 */
263__PAGEFLAG(Head, head)
264__PAGEFLAG(Tail, tail)
265
266static inline int PageCompound(struct page *page)
267{
268 return page->flags & ((1L << PG_head) | (1L << PG_tail));
269
270}
271#else
272/*
273 * Reduce page flag use as much as possible by overlapping
274 * compound page flags with the flags used for page cache pages. Possible
275 * because PageCompound is always set for compound pages and not for
276 * pages on the LRU and/or pagecache.
277 */
251TESTPAGEFLAG(Compound, compound) 278TESTPAGEFLAG(Compound, compound)
252__PAGEFLAG(Head, compound) 279__PAGEFLAG(Head, compound)
253 280
@@ -278,5 +305,6 @@ static inline void __ClearPageTail(struct page *page)
278 page->flags &= ~PG_head_tail_mask; 305 page->flags &= ~PG_head_tail_mask;
279} 306}
280 307
308#endif /* !PAGEFLAGS_EXTENDED */
281#endif /* !__GENERATING_BOUNDS_H */ 309#endif /* !__GENERATING_BOUNDS_H */
282#endif /* PAGE_FLAGS_H */ 310#endif /* PAGE_FLAGS_H */
diff --git a/mm/Kconfig b/mm/Kconfig
index 0016ebd4dcba..3aa819d628c1 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -143,6 +143,18 @@ config MEMORY_HOTREMOVE
143 depends on MEMORY_HOTPLUG && ARCH_ENABLE_MEMORY_HOTREMOVE 143 depends on MEMORY_HOTPLUG && ARCH_ENABLE_MEMORY_HOTREMOVE
144 depends on MIGRATION 144 depends on MIGRATION
145 145
146#
147# If we have space for more page flags then we can enable additional
148# optimizations and functionality.
149#
150# Regular Sparsemem takes page flag bits for the sectionid if it does not
151# use a virtual memmap. Disable extended page flags for 32 bit platforms
152# that require the use of a sectionid in the page flags.
153#
154config PAGEFLAGS_EXTENDED
155 def_bool y
156 depends on 64BIT || SPARSEMEM_VMEMMAP || !NUMA || !SPARSEMEM
157
146# Heavily threaded applications may benefit from splitting the mm-wide 158# Heavily threaded applications may benefit from splitting the mm-wide
147# page_table_lock, so that faults on different parts of the user address 159# page_table_lock, so that faults on different parts of the user address
148# space can be handled with less contention: split it at this NR_CPUS. 160# space can be handled with less contention: split it at this NR_CPUS.