From eada35efcb2773cf49aa26277e056122e1a3405c Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Mon, 11 Feb 2008 22:47:46 +0200 Subject: slub: kmalloc page allocator pass-through cleanup This adds a proper function for kmalloc page allocator pass-through. While it simplifies any code that does slab tracing code a lot, I think it's a worthwhile cleanup in itself. Signed-off-by: Pekka Enberg Signed-off-by: Christoph Lameter --- include/linux/slub_def.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 5e6d3d634d5b..a849c472b845 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -188,12 +188,16 @@ static __always_inline struct kmem_cache *kmalloc_slab(size_t size) void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *__kmalloc(size_t size, gfp_t flags); +static __always_inline void *kmalloc_large(size_t size, gfp_t flags) +{ + return (void *)__get_free_pages(flags | __GFP_COMP, get_order(size)); +} + static __always_inline void *kmalloc(size_t size, gfp_t flags) { if (__builtin_constant_p(size)) { if (size > PAGE_SIZE / 2) - return (void *)__get_free_pages(flags | __GFP_COMP, - get_order(size)); + return kmalloc_large(size, flags); if (!(flags & SLUB_DMA)) { struct kmem_cache *s = kmalloc_slab(size); -- cgit v1.2.2 From b7a49f0d4c34166ae84089d9f145cfaae1b0eec5 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 14 Feb 2008 14:21:32 -0800 Subject: slub: Determine gfpflags once and not every time a slab is allocated Currently we determine the gfp flags to pass to the page allocator each time a slab is being allocated. Determine the bits to be set at the time the slab is created. Store in a new allocflags field and add the flags in allocate_slab(). Acked-by: Mel Gorman Reviewed-by: Pekka Enberg Signed-off-by: Christoph Lameter --- include/linux/slub_def.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index a849c472b845..98be113cf935 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -71,6 +71,7 @@ struct kmem_cache { /* Allocation and freeing of slabs */ int objects; /* Number of objects in slab */ + gfp_t allocflags; /* gfp flags to use on each alloc */ int refcount; /* Refcount for slab cache destroy */ void (*ctor)(struct kmem_cache *, void *); int inuse; /* Offset to metadata */ -- cgit v1.2.2 From 331dc558fa020451ff773973cee855fd721aa88e Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 14 Feb 2008 14:28:09 -0800 Subject: slub: Support 4k kmallocs again to compensate for page allocator slowness Currently we hand off PAGE_SIZEd kmallocs to the page allocator in the mistaken belief that the page allocator can handle these allocations effectively. However, measurements indicate a minimum slowdown by the factor of 8 (and that is only SMP, NUMA is much worse) vs the slub fastpath which causes regressions in tbench. Increase the number of kmalloc caches by one so that we again handle 4k kmallocs directly from slub. 4k page buffering for the page allocator will be performed by slub like done by slab. At some point the page allocator fastpath should be fixed. A lot of the kernel would benefit from a faster ability to allocate a single page. If that is done then the 4k allocs may again be forwarded to the page allocator and this patch could be reverted. Reviewed-by: Pekka Enberg Acked-by: Mel Gorman Signed-off-by: Christoph Lameter --- include/linux/slub_def.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 98be113cf935..57deecc79d52 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -111,7 +111,7 @@ struct kmem_cache { * We keep the general caches in an array of slab caches that are used for * 2^x bytes of allocations. */ -extern struct kmem_cache kmalloc_caches[PAGE_SHIFT]; +extern struct kmem_cache kmalloc_caches[PAGE_SHIFT + 1]; /* * Sorry that the following has to be that ugly but some versions of GCC @@ -197,7 +197,7 @@ static __always_inline void *kmalloc_large(size_t size, gfp_t flags) static __always_inline void *kmalloc(size_t size, gfp_t flags) { if (__builtin_constant_p(size)) { - if (size > PAGE_SIZE / 2) + if (size > PAGE_SIZE) return kmalloc_large(size, flags); if (!(flags & SLUB_DMA)) { @@ -219,7 +219,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) { if (__builtin_constant_p(size) && - size <= PAGE_SIZE / 2 && !(flags & SLUB_DMA)) { + size <= PAGE_SIZE && !(flags & SLUB_DMA)) { struct kmem_cache *s = kmalloc_slab(size); if (!s) -- cgit v1.2.2