aboutsummaryrefslogtreecommitdiffstats
path: root/mm/slub.c
diff options
context:
space:
mode:
authorMarcin Slusarz <marcin.slusarz@gmail.com>2011-06-26 15:39:18 -0400
committerPekka Enberg <penberg@kernel.org>2011-07-07 15:44:45 -0400
commitc4089f98e943ff445665dea49c190657b34ccffe (patch)
tree8238e65e7eaf575afc52c44179f428c3069584cd /mm/slub.c
parentd18a90dd85f8243ed20cdadb6d8a37d595df456d (diff)
slub: reduce overhead of slub_debug
slub checks for poison one byte by one, which is highly inefficient and shows up frequently as a highest cpu-eater in perf top. Joining reads gives nice speedup: (Compiling some project with different options) make -j12 make clean slub_debug disabled: 1m 27s 1.2 s slub_debug enabled: 1m 46s 7.6 s slub_debug enabled + this patch: 1m 33s 3.2 s check_bytes still shows up high, but not always at the top. Signed-off-by: Marcin Slusarz <marcin.slusarz@gmail.com> Cc: Christoph Lameter <cl@linux-foundation.org> Cc: Pekka Enberg <penberg@kernel.org> Cc: Matt Mackall <mpm@selenic.com> Cc: linux-mm@kvack.org Signed-off-by: Pekka Enberg <penberg@kernel.org>
Diffstat (limited to 'mm/slub.c')
-rw-r--r--mm/slub.c36
1 files changed, 34 insertions, 2 deletions
diff --git a/mm/slub.c b/mm/slub.c
index 0e4f4f8245bc..e3403b30159e 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -589,10 +589,10 @@ static void init_object(struct kmem_cache *s, void *object, u8 val)
589 memset(p + s->objsize, val, s->inuse - s->objsize); 589 memset(p + s->objsize, val, s->inuse - s->objsize);
590} 590}
591 591
592static u8 *check_bytes(u8 *start, unsigned int value, unsigned int bytes) 592static u8 *check_bytes8(u8 *start, u8 value, unsigned int bytes)
593{ 593{
594 while (bytes) { 594 while (bytes) {
595 if (*start != (u8)value) 595 if (*start != value)
596 return start; 596 return start;
597 start++; 597 start++;
598 bytes--; 598 bytes--;
@@ -600,6 +600,38 @@ static u8 *check_bytes(u8 *start, unsigned int value, unsigned int bytes)
600 return NULL; 600 return NULL;
601} 601}
602 602
603static u8 *check_bytes(u8 *start, u8 value, unsigned int bytes)
604{
605 u64 value64;
606 unsigned int words, prefix;
607
608 if (bytes <= 16)
609 return check_bytes8(start, value, bytes);
610
611 value64 = value | value << 8 | value << 16 | value << 24;
612 value64 = value64 | value64 << 32;
613 prefix = 8 - ((unsigned long)start) % 8;
614
615 if (prefix) {
616 u8 *r = check_bytes8(start, value, prefix);
617 if (r)
618 return r;
619 start += prefix;
620 bytes -= prefix;
621 }
622
623 words = bytes / 8;
624
625 while (words) {
626 if (*(u64 *)start != value64)
627 return check_bytes8(start, value, 8);
628 start += 8;
629 words--;
630 }
631
632 return check_bytes8(start, value, bytes % 8);
633}
634
603static void restore_bytes(struct kmem_cache *s, char *message, u8 data, 635static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
604 void *from, void *to) 636 void *from, void *to)
605{ 637{