aboutsummaryrefslogtreecommitdiffstats
path: root/lib/hweight.c
diff options
context:
space:
mode:
authorBorislav Petkov <borislav.petkov@amd.com>2010-03-05 11:34:46 -0500
committerH. Peter Anvin <hpa@zytor.com>2010-04-06 18:52:11 -0400
commitd61931d89be506372d01a90d1755f6d0a9fafe2d (patch)
tree652c34238edcb6c558163abc3cd9d6ce7c5f91a5 /lib/hweight.c
parent1527bc8b928dd1399c3d3467dd47d9ede210978a (diff)
x86: Add optimized popcnt variants
Add support for the hardware version of the Hamming weight function, popcnt, present in CPUs which advertize it under CPUID, Function 0x0000_0001_ECX[23]. On CPUs which don't support it, we fallback to the default lib/hweight.c sw versions. A synthetic benchmark comparing popcnt with __sw_hweight64 showed almost a 3x speedup on a F10h machine. Signed-off-by: Borislav Petkov <borislav.petkov@amd.com> LKML-Reference: <20100318112015.GC11152@aftab> Signed-off-by: H. Peter Anvin <hpa@zytor.com>
Diffstat (limited to 'lib/hweight.c')
-rw-r--r--lib/hweight.c20
1 files changed, 10 insertions, 10 deletions
diff --git a/lib/hweight.c b/lib/hweight.c
index a6927e76840f..3c79d50814cf 100644
--- a/lib/hweight.c
+++ b/lib/hweight.c
@@ -9,7 +9,7 @@
9 * The Hamming Weight of a number is the total number of bits set in it. 9 * The Hamming Weight of a number is the total number of bits set in it.
10 */ 10 */
11 11
12unsigned int __arch_hweight32(unsigned int w) 12unsigned int __sw_hweight32(unsigned int w)
13{ 13{
14#ifdef ARCH_HAS_FAST_MULTIPLIER 14#ifdef ARCH_HAS_FAST_MULTIPLIER
15 w -= (w >> 1) & 0x55555555; 15 w -= (w >> 1) & 0x55555555;
@@ -24,30 +24,30 @@ unsigned int __arch_hweight32(unsigned int w)
24 return (res + (res >> 16)) & 0x000000FF; 24 return (res + (res >> 16)) & 0x000000FF;
25#endif 25#endif
26} 26}
27EXPORT_SYMBOL(__arch_hweight32); 27EXPORT_SYMBOL(__sw_hweight32);
28 28
29unsigned int __arch_hweight16(unsigned int w) 29unsigned int __sw_hweight16(unsigned int w)
30{ 30{
31 unsigned int res = w - ((w >> 1) & 0x5555); 31 unsigned int res = w - ((w >> 1) & 0x5555);
32 res = (res & 0x3333) + ((res >> 2) & 0x3333); 32 res = (res & 0x3333) + ((res >> 2) & 0x3333);
33 res = (res + (res >> 4)) & 0x0F0F; 33 res = (res + (res >> 4)) & 0x0F0F;
34 return (res + (res >> 8)) & 0x00FF; 34 return (res + (res >> 8)) & 0x00FF;
35} 35}
36EXPORT_SYMBOL(__arch_hweight16); 36EXPORT_SYMBOL(__sw_hweight16);
37 37
38unsigned int __arch_hweight8(unsigned int w) 38unsigned int __sw_hweight8(unsigned int w)
39{ 39{
40 unsigned int res = w - ((w >> 1) & 0x55); 40 unsigned int res = w - ((w >> 1) & 0x55);
41 res = (res & 0x33) + ((res >> 2) & 0x33); 41 res = (res & 0x33) + ((res >> 2) & 0x33);
42 return (res + (res >> 4)) & 0x0F; 42 return (res + (res >> 4)) & 0x0F;
43} 43}
44EXPORT_SYMBOL(__arch_hweight8); 44EXPORT_SYMBOL(__sw_hweight8);
45 45
46unsigned long __arch_hweight64(__u64 w) 46unsigned long __sw_hweight64(__u64 w)
47{ 47{
48#if BITS_PER_LONG == 32 48#if BITS_PER_LONG == 32
49 return __arch_hweight32((unsigned int)(w >> 32)) + 49 return __sw_hweight32((unsigned int)(w >> 32)) +
50 __arch_hweight32((unsigned int)w); 50 __sw_hweight32((unsigned int)w);
51#elif BITS_PER_LONG == 64 51#elif BITS_PER_LONG == 64
52#ifdef ARCH_HAS_FAST_MULTIPLIER 52#ifdef ARCH_HAS_FAST_MULTIPLIER
53 w -= (w >> 1) & 0x5555555555555555ul; 53 w -= (w >> 1) & 0x5555555555555555ul;
@@ -64,4 +64,4 @@ unsigned long __arch_hweight64(__u64 w)
64#endif 64#endif
65#endif 65#endif
66} 66}
67EXPORT_SYMBOL(__arch_hweight64); 67EXPORT_SYMBOL(__sw_hweight64);