aboutsummaryrefslogtreecommitdiffstats
path: root/net/ipv4/fib_semantics.c
diff options
context:
space:
mode:
authorEric Dumazet <edumazet@google.com>2012-07-31 01:45:30 -0400
committerDavid S. Miller <davem@davemloft.net>2012-07-31 17:41:39 -0400
commitd26b3a7c4b3b26319f18bb645de93eba8f4bdcd5 (patch)
treeca86c03450fafdc89dac98ce403b1906fcaa025d /net/ipv4/fib_semantics.c
parent54764bb647b2e847c512acf8d443df965da35000 (diff)
ipv4: percpu nh_rth_output cache
Input path is mostly run under RCU and doesnt touch dst refcnt But output path on forwarding or UDP workloads hits badly dst refcount, and we have lot of false sharing, for example in ipv4_mtu() when reading rt->rt_pmtu Using a percpu cache for nh_rth_output gives a nice performance increase at a small cost. 24 udpflood test on my 24 cpu machine (dummy0 output device) (each process sends 1.000.000 udp frames, 24 processes are started) before : 5.24 s after : 2.06 s For reference, time on linux-3.5 : 6.60 s Signed-off-by: Eric Dumazet <edumazet@google.com> Tested-by: Alexander Duyck <alexander.h.duyck@intel.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'net/ipv4/fib_semantics.c')
-rw-r--r--net/ipv4/fib_semantics.c20
1 files changed, 19 insertions, 1 deletions
diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c
index 625cf185c489..fe2ca02a1979 100644
--- a/net/ipv4/fib_semantics.c
+++ b/net/ipv4/fib_semantics.c
@@ -176,6 +176,23 @@ static void rt_nexthop_free(struct rtable __rcu **rtp)
176 dst_free(&rt->dst); 176 dst_free(&rt->dst);
177} 177}
178 178
179static void rt_nexthop_free_cpus(struct rtable __rcu * __percpu *rtp)
180{
181 int cpu;
182
183 if (!rtp)
184 return;
185
186 for_each_possible_cpu(cpu) {
187 struct rtable *rt;
188
189 rt = rcu_dereference_protected(*per_cpu_ptr(rtp, cpu), 1);
190 if (rt)
191 dst_free(&rt->dst);
192 }
193 free_percpu(rtp);
194}
195
179/* Release a nexthop info record */ 196/* Release a nexthop info record */
180static void free_fib_info_rcu(struct rcu_head *head) 197static void free_fib_info_rcu(struct rcu_head *head)
181{ 198{
@@ -186,7 +203,7 @@ static void free_fib_info_rcu(struct rcu_head *head)
186 dev_put(nexthop_nh->nh_dev); 203 dev_put(nexthop_nh->nh_dev);
187 if (nexthop_nh->nh_exceptions) 204 if (nexthop_nh->nh_exceptions)
188 free_nh_exceptions(nexthop_nh); 205 free_nh_exceptions(nexthop_nh);
189 rt_nexthop_free(&nexthop_nh->nh_rth_output); 206 rt_nexthop_free_cpus(nexthop_nh->nh_pcpu_rth_output);
190 rt_nexthop_free(&nexthop_nh->nh_rth_input); 207 rt_nexthop_free(&nexthop_nh->nh_rth_input);
191 } endfor_nexthops(fi); 208 } endfor_nexthops(fi);
192 209
@@ -817,6 +834,7 @@ struct fib_info *fib_create_info(struct fib_config *cfg)
817 fi->fib_nhs = nhs; 834 fi->fib_nhs = nhs;
818 change_nexthops(fi) { 835 change_nexthops(fi) {
819 nexthop_nh->nh_parent = fi; 836 nexthop_nh->nh_parent = fi;
837 nexthop_nh->nh_pcpu_rth_output = alloc_percpu(struct rtable __rcu *);
820 } endfor_nexthops(fi) 838 } endfor_nexthops(fi)
821 839
822 if (cfg->fc_mx) { 840 if (cfg->fc_mx) {