aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNick Piggin <npiggin@kernel.dk>2011-01-07 01:50:09 -0500
committerNick Piggin <npiggin@kernel.dk>2011-01-07 01:50:32 -0500
commit9d55c369bb5e695e629bc35cba2ef607755b3bee (patch)
treed97cd498a67f1f7987d4403ef90e399516a3b8c9
parente1bb57826381199cc79fbf44e9dfeee58fc7b339 (diff)
fs: implement faster dentry memcmp
The standard memcmp function on a Westmere system shows up hot in profiles in the `git diff` workload (both parallel and single threaded), and it is likely due to the costs associated with trapping into microcode, and little opportunity to improve memory access (dentry name is not likely to take up more than a cacheline). So replace it with an open-coded byte comparison. This increases code size by 8 bytes in the critical __d_lookup_rcu function, but the speedup is huge, averaging 10 runs of each: git diff st user sys elapsed CPU before 1.15 2.57 3.82 97.1 after 1.14 2.35 3.61 96.8 git diff mt user sys elapsed CPU before 1.27 3.85 1.46 349 after 1.26 3.54 1.43 333 Elapsed time for single threaded git diff at 95.0% confidence: -0.21 +/- 0.01 -5.45% +/- 0.24% It's -0.66% +/- 0.06% elapsed time on my Opteron, so rep cmp costs on the fam10h seem to be relatively smaller, but there is still a win. Signed-off-by: Nick Piggin <npiggin@kernel.dk>
-rw-r--r--fs/dcache.c12
-rw-r--r--include/linux/dcache.h21
2 files changed, 24 insertions, 9 deletions
diff --git a/fs/dcache.c b/fs/dcache.c
index 2a4ce7dc2306..5699d4c027cb 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -1454,9 +1454,7 @@ static struct dentry *__d_instantiate_unique(struct dentry *entry,
1454 continue; 1454 continue;
1455 if (alias->d_parent != entry->d_parent) 1455 if (alias->d_parent != entry->d_parent)
1456 continue; 1456 continue;
1457 if (qstr->len != len) 1457 if (dentry_cmp(qstr->name, qstr->len, name, len))
1458 continue;
1459 if (memcmp(qstr->name, name, len))
1460 continue; 1458 continue;
1461 __dget(alias); 1459 __dget(alias);
1462 return alias; 1460 return alias;
@@ -1810,9 +1808,7 @@ seqretry:
1810 tlen, tname, name)) 1808 tlen, tname, name))
1811 continue; 1809 continue;
1812 } else { 1810 } else {
1813 if (tlen != len) 1811 if (dentry_cmp(tname, tlen, str, len))
1814 continue;
1815 if (memcmp(tname, str, tlen))
1816 continue; 1812 continue;
1817 } 1813 }
1818 /* 1814 /*
@@ -1925,9 +1921,7 @@ struct dentry *__d_lookup(struct dentry *parent, struct qstr *name)
1925 tlen, tname, name)) 1921 tlen, tname, name))
1926 goto next; 1922 goto next;
1927 } else { 1923 } else {
1928 if (tlen != len) 1924 if (dentry_cmp(tname, tlen, str, len))
1929 goto next;
1930 if (memcmp(tname, str, tlen))
1931 goto next; 1925 goto next;
1932 } 1926 }
1933 1927
diff --git a/include/linux/dcache.h b/include/linux/dcache.h
index c0a2ca97c72f..bd07758943e0 100644
--- a/include/linux/dcache.h
+++ b/include/linux/dcache.h
@@ -47,6 +47,27 @@ struct dentry_stat_t {
47}; 47};
48extern struct dentry_stat_t dentry_stat; 48extern struct dentry_stat_t dentry_stat;
49 49
50/*
51 * Compare 2 name strings, return 0 if they match, otherwise non-zero.
52 * The strings are both count bytes long, and count is non-zero.
53 */
54static inline int dentry_cmp(const unsigned char *cs, size_t scount,
55 const unsigned char *ct, size_t tcount)
56{
57 int ret;
58 if (scount != tcount)
59 return 1;
60 do {
61 ret = (*cs != *ct);
62 if (ret)
63 break;
64 cs++;
65 ct++;
66 tcount--;
67 } while (tcount);
68 return ret;
69}
70
50/* Name hashing routines. Initial hash value */ 71/* Name hashing routines. Initial hash value */
51/* Hash courtesy of the R5 hash in reiserfs modulo sign bits */ 72/* Hash courtesy of the R5 hash in reiserfs modulo sign bits */
52#define init_name_hash() 0 73#define init_name_hash() 0