net dst: use a percpu_counter to track entries

struct dst_ops tracks number of allocated dst in an atomic_t field,
subject to high cache line contention in stress workload.

Switch to a percpu_counter, to reduce number of time we need to dirty a
central location. Place it on a separate cache line to avoid dirtying
read only fields.

Stress test :

(Sending 160.000.000 UDP frames,
IP route cache disabled, dual E5540 @2.53GHz,
32bit kernel, FIB_TRIE, SLUB/NUMA)

Before:

real    0m51.179s
user    0m15.329s
sys     10m15.942s

After:

real	0m45.570s
user	0m15.525s
sys	9m56.669s

With a small reordering of struct neighbour fields, subject of a
following patch, (to separate refcnt from other read mostly fields)

real	0m41.841s
user	0m15.261s
sys	8m45.949s

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 9 insertions, 2 deletions

diff --git a/net/bridge/br_netfilter.c b/net/bridge/br_netfilter.c
index 77f7b5fda45a..7f9ce9600ef3 100644
--- a/net/bridge/br_netfilter.c
+++ b/net/bridge/br_netfilter.c
@@ -106,7 +106,6 @@ static struct dst_ops fake_dst_ops = {
         .family =               AF_INET,
         .protocol =             cpu_to_be16(ETH_P_IP),
         .update_pmtu =          fake_update_pmtu,
-        .entries =              ATOMIC_INIT(0),
 };
 
 /*
@@ -1003,15 +1002,22 @@ int __init br_netfilter_init(void)
 {
         int ret;
 
-        ret = nf_register_hooks(br_nf_ops, ARRAY_SIZE(br_nf_ops));
+        ret = dst_entries_init(&fake_dst_ops);
         if (ret < 0)
                 return ret;
+
+        ret = nf_register_hooks(br_nf_ops, ARRAY_SIZE(br_nf_ops));
+        if (ret < 0) {
+                dst_entries_destroy(&fake_dst_ops);
+                return ret;
+        }
 #ifdef CONFIG_SYSCTL
         brnf_sysctl_header = register_sysctl_paths(brnf_path, brnf_table);
         if (brnf_sysctl_header == NULL) {
                 printk(KERN_WARNING
                        "br_netfilter: can't register to sysctl.\n");
                 nf_unregister_hooks(br_nf_ops, ARRAY_SIZE(br_nf_ops));
+                dst_entries_destroy(&fake_dst_ops);
                 return -ENOMEM;
         }
 #endif
@@ -1025,4 +1031,5 @@ void br_netfilter_fini(void)
 #ifdef CONFIG_SYSCTL
         unregister_sysctl_table(brnf_sysctl_header);
 #endif
+        dst_entries_destroy(&fake_dst_ops);
 }