aboutsummaryrefslogtreecommitdiffstats
path: root/net/core/neighbour.c
diff options
context:
space:
mode:
authorEric Dumazet <eric.dumazet@gmail.com>2010-10-11 12:16:57 -0400
committerDavid S. Miller <davem@davemloft.net>2010-10-11 12:16:57 -0400
commit34d101dd6204bd100fc2e6f7b5f9a10f959ce2c9 (patch)
tree77b714a4de7273ec53dac80396b94a2f896cfa95 /net/core/neighbour.c
parent37f9fc452d138dfc4da2ee1ce5ae85094efc3606 (diff)
neigh: speedup neigh_hh_init()
When a new dst is used to send a frame, neigh_resolve_output() tries to associate an struct hh_cache to this dst, calling neigh_hh_init() with the neigh rwlock write locked. Most of the time, hh_cache is already known and linked into neighbour, so we find it and increment its refcount. This patch changes the logic so that we call neigh_hh_init() with neighbour lock read locked only, so that fast path can be run in parallel by concurrent cpus. This brings part of the speedup we got with commit c7d4426a98a5f (introduce DST_NOCACHE flag) for non cached dsts, even for cached ones, removing one of the contention point that routers hit on multiqueue enabled machines. Further improvements would need to use a seqlock instead of an rwlock to protect neigh->ha[], to not dirty neigh too often and remove two atomic ops. Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'net/core/neighbour.c')
-rw-r--r--net/core/neighbour.c99
1 files changed, 61 insertions, 38 deletions
diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index 3ffafaa0414c..2044906ecd1a 100644
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -709,8 +709,7 @@ void neigh_destroy(struct neighbour *neigh)
709 write_seqlock_bh(&hh->hh_lock); 709 write_seqlock_bh(&hh->hh_lock);
710 hh->hh_output = neigh_blackhole; 710 hh->hh_output = neigh_blackhole;
711 write_sequnlock_bh(&hh->hh_lock); 711 write_sequnlock_bh(&hh->hh_lock);
712 if (atomic_dec_and_test(&hh->hh_refcnt)) 712 hh_cache_put(hh);
713 kfree(hh);
714 } 713 }
715 714
716 skb_queue_purge(&neigh->arp_queue); 715 skb_queue_purge(&neigh->arp_queue);
@@ -1210,39 +1209,67 @@ struct neighbour *neigh_event_ns(struct neigh_table *tbl,
1210} 1209}
1211EXPORT_SYMBOL(neigh_event_ns); 1210EXPORT_SYMBOL(neigh_event_ns);
1212 1211
1212static inline bool neigh_hh_lookup(struct neighbour *n, struct dst_entry *dst,
1213 __be16 protocol)
1214{
1215 struct hh_cache *hh;
1216
1217 for (hh = n->hh; hh; hh = hh->hh_next) {
1218 if (hh->hh_type == protocol) {
1219 atomic_inc(&hh->hh_refcnt);
1220 if (unlikely(cmpxchg(&dst->hh, NULL, hh) != NULL))
1221 hh_cache_put(hh);
1222 return true;
1223 }
1224 }
1225 return false;
1226}
1227
1228/* called with read_lock_bh(&n->lock); */
1213static void neigh_hh_init(struct neighbour *n, struct dst_entry *dst, 1229static void neigh_hh_init(struct neighbour *n, struct dst_entry *dst,
1214 __be16 protocol) 1230 __be16 protocol)
1215{ 1231{
1216 struct hh_cache *hh; 1232 struct hh_cache *hh;
1217 struct net_device *dev = dst->dev; 1233 struct net_device *dev = dst->dev;
1218 1234
1219 for (hh = n->hh; hh; hh = hh->hh_next) 1235 if (likely(neigh_hh_lookup(n, dst, protocol)))
1220 if (hh->hh_type == protocol) 1236 return;
1221 break;
1222 1237
1223 if (!hh && (hh = kzalloc(sizeof(*hh), GFP_ATOMIC)) != NULL) { 1238 /* slow path */
1224 seqlock_init(&hh->hh_lock); 1239 hh = kzalloc(sizeof(*hh), GFP_ATOMIC);
1225 hh->hh_type = protocol; 1240 if (!hh)
1226 atomic_set(&hh->hh_refcnt, 0); 1241 return;
1227 hh->hh_next = NULL;
1228 1242
1229 if (dev->header_ops->cache(n, hh)) { 1243 seqlock_init(&hh->hh_lock);
1230 kfree(hh); 1244 hh->hh_type = protocol;
1231 hh = NULL; 1245 atomic_set(&hh->hh_refcnt, 2);
1232 } else { 1246
1233 atomic_inc(&hh->hh_refcnt); 1247 if (dev->header_ops->cache(n, hh)) {
1234 hh->hh_next = n->hh; 1248 kfree(hh);
1235 n->hh = hh; 1249 return;
1236 if (n->nud_state & NUD_CONNECTED)
1237 hh->hh_output = n->ops->hh_output;
1238 else
1239 hh->hh_output = n->ops->output;
1240 }
1241 } 1250 }
1242 if (hh) { 1251 read_unlock(&n->lock);
1243 atomic_inc(&hh->hh_refcnt); 1252 write_lock(&n->lock);
1244 dst->hh = hh; 1253
1254 /* must check if another thread already did the insert */
1255 if (neigh_hh_lookup(n, dst, protocol)) {
1256 kfree(hh);
1257 goto end;
1245 } 1258 }
1259
1260 if (n->nud_state & NUD_CONNECTED)
1261 hh->hh_output = n->ops->hh_output;
1262 else
1263 hh->hh_output = n->ops->output;
1264
1265 hh->hh_next = n->hh;
1266 n->hh = hh;
1267
1268 if (unlikely(cmpxchg(&dst->hh, NULL, hh) != NULL))
1269 hh_cache_put(hh);
1270end:
1271 write_unlock(&n->lock);
1272 read_lock(&n->lock);
1246} 1273}
1247 1274
1248/* This function can be used in contexts, where only old dev_queue_xmit 1275/* This function can be used in contexts, where only old dev_queue_xmit
@@ -1281,21 +1308,17 @@ int neigh_resolve_output(struct sk_buff *skb)
1281 if (!neigh_event_send(neigh, skb)) { 1308 if (!neigh_event_send(neigh, skb)) {
1282 int err; 1309 int err;
1283 struct net_device *dev = neigh->dev; 1310 struct net_device *dev = neigh->dev;
1311
1312 read_lock_bh(&neigh->lock);
1284 if (dev->header_ops->cache && 1313 if (dev->header_ops->cache &&
1285 !dst->hh && 1314 !dst->hh &&
1286 !(dst->flags & DST_NOCACHE)) { 1315 !(dst->flags & DST_NOCACHE))
1287 write_lock_bh(&neigh->lock); 1316 neigh_hh_init(neigh, dst, dst->ops->protocol);
1288 if (!dst->hh) 1317
1289 neigh_hh_init(neigh, dst, dst->ops->protocol); 1318 err = dev_hard_header(skb, dev, ntohs(skb->protocol),
1290 err = dev_hard_header(skb, dev, ntohs(skb->protocol), 1319 neigh->ha, NULL, skb->len);
1291 neigh->ha, NULL, skb->len); 1320 read_unlock_bh(&neigh->lock);
1292 write_unlock_bh(&neigh->lock); 1321
1293 } else {
1294 read_lock_bh(&neigh->lock);
1295 err = dev_hard_header(skb, dev, ntohs(skb->protocol),
1296 neigh->ha, NULL, skb->len);
1297 read_unlock_bh(&neigh->lock);
1298 }
1299 if (err >= 0) 1322 if (err >= 0)
1300 rc = neigh->ops->queue_xmit(skb); 1323 rc = neigh->ops->queue_xmit(skb);
1301 else 1324 else