aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorOctavian Purdila <opurdila@ixiacom.com>2009-12-26 06:51:02 -0500
committerDavid S. Miller <davem@davemloft.net>2009-12-26 23:41:43 -0500
commitb76f5a8427ac2928c07fa4ff2144bb8db072c240 (patch)
tree542d64a8bd28baf69f87b199a0156f865a06551d
parentabf9d537fea225af60762640361af7fb233b3103 (diff)
llc: convert the socket list to RCU locking
For the reclamation phase we use the SLAB_DESTROY_BY_RCU mechanism, which require some extra checks in the lookup code: a) If the current socket was released, reallocated & inserted in another list it will short circuit the iteration for the current list, thus we need to restart the lookup. b) If the current socket was released, reallocated & inserted in the same list we just need to recheck it matches the look-up criteria and if not we can skip to the next element. In this case there is no need to restart the lookup, since sockets are inserted at the start of the list and the worst that will happen is that we will iterate throught some of the list elements more then once. Note that the /proc and multicast delivery was not yet converted to RCU, it still uses spinlocks for protection. Signed-off-by: Octavian Purdila <opurdila@ixiacom.com> Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r--include/net/llc.h7
-rw-r--r--net/llc/af_llc.c1
-rw-r--r--net/llc/llc_conn.c93
-rw-r--r--net/llc/llc_core.c5
-rw-r--r--net/llc/llc_proc.c22
-rw-r--r--net/llc/llc_sap.c66
6 files changed, 123 insertions, 71 deletions
diff --git a/include/net/llc.h b/include/net/llc.h
index 7940da1606e7..1559cf10e874 100644
--- a/include/net/llc.h
+++ b/include/net/llc.h
@@ -16,6 +16,7 @@
16#include <linux/if_ether.h> 16#include <linux/if_ether.h>
17#include <linux/list.h> 17#include <linux/list.h>
18#include <linux/spinlock.h> 18#include <linux/spinlock.h>
19#include <linux/rculist_nulls.h>
19 20
20#include <asm/atomic.h> 21#include <asm/atomic.h>
21 22
@@ -53,10 +54,8 @@ struct llc_sap {
53 struct net_device *orig_dev); 54 struct net_device *orig_dev);
54 struct llc_addr laddr; 55 struct llc_addr laddr;
55 struct list_head node; 56 struct list_head node;
56 struct { 57 spinlock_t sk_lock;
57 rwlock_t lock; 58 struct hlist_nulls_head sk_list;
58 struct hlist_head list;
59 } sk_list;
60}; 59};
61 60
62#define LLC_DEST_INVALID 0 /* Invalid LLC PDU type */ 61#define LLC_DEST_INVALID 0 /* Invalid LLC PDU type */
diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c
index c4d1a1da813c..f49f3dd6fbd3 100644
--- a/net/llc/af_llc.c
+++ b/net/llc/af_llc.c
@@ -140,6 +140,7 @@ static struct proto llc_proto = {
140 .name = "LLC", 140 .name = "LLC",
141 .owner = THIS_MODULE, 141 .owner = THIS_MODULE,
142 .obj_size = sizeof(struct llc_sock), 142 .obj_size = sizeof(struct llc_sock),
143 .slab_flags = SLAB_DESTROY_BY_RCU,
143}; 144};
144 145
145/** 146/**
diff --git a/net/llc/llc_conn.c b/net/llc/llc_conn.c
index c6bab39b018e..77bb3816655e 100644
--- a/net/llc/llc_conn.c
+++ b/net/llc/llc_conn.c
@@ -468,6 +468,19 @@ static int llc_exec_conn_trans_actions(struct sock *sk,
468 return rc; 468 return rc;
469} 469}
470 470
471static inline bool llc_estab_match(const struct llc_sap *sap,
472 const struct llc_addr *daddr,
473 const struct llc_addr *laddr,
474 const struct sock *sk)
475{
476 struct llc_sock *llc = llc_sk(sk);
477
478 return llc->laddr.lsap == laddr->lsap &&
479 llc->daddr.lsap == daddr->lsap &&
480 llc_mac_match(llc->laddr.mac, laddr->mac) &&
481 llc_mac_match(llc->daddr.mac, daddr->mac);
482}
483
471/** 484/**
472 * __llc_lookup_established - Finds connection for the remote/local sap/mac 485 * __llc_lookup_established - Finds connection for the remote/local sap/mac
473 * @sap: SAP 486 * @sap: SAP
@@ -484,23 +497,26 @@ static struct sock *__llc_lookup_established(struct llc_sap *sap,
484 struct llc_addr *laddr) 497 struct llc_addr *laddr)
485{ 498{
486 struct sock *rc; 499 struct sock *rc;
487 struct hlist_node *node; 500 struct hlist_nulls_node *node;
488 501
489 read_lock(&sap->sk_list.lock); 502 rcu_read_lock();
490 sk_for_each(rc, node, &sap->sk_list.list) { 503again:
491 struct llc_sock *llc = llc_sk(rc); 504 sk_nulls_for_each_rcu(rc, node, &sap->sk_list) {
492 505 if (llc_estab_match(sap, daddr, laddr, rc)) {
493 if (llc->laddr.lsap == laddr->lsap && 506 /* Extra checks required by SLAB_DESTROY_BY_RCU */
494 llc->daddr.lsap == daddr->lsap && 507 if (unlikely(!atomic_inc_not_zero(&rc->sk_refcnt)))
495 llc_mac_match(llc->laddr.mac, laddr->mac) && 508 goto again;
496 llc_mac_match(llc->daddr.mac, daddr->mac)) { 509 if (unlikely(llc_sk(rc)->sap != sap ||
497 sock_hold(rc); 510 !llc_estab_match(sap, daddr, laddr, rc))) {
511 sock_put(rc);
512 continue;
513 }
498 goto found; 514 goto found;
499 } 515 }
500 } 516 }
501 rc = NULL; 517 rc = NULL;
502found: 518found:
503 read_unlock(&sap->sk_list.lock); 519 rcu_read_unlock();
504 return rc; 520 return rc;
505} 521}
506 522
@@ -516,6 +532,18 @@ struct sock *llc_lookup_established(struct llc_sap *sap,
516 return sk; 532 return sk;
517} 533}
518 534
535static inline bool llc_listener_match(const struct llc_sap *sap,
536 const struct llc_addr *laddr,
537 const struct sock *sk)
538{
539 struct llc_sock *llc = llc_sk(sk);
540
541 return sk->sk_type == SOCK_STREAM && sk->sk_state == TCP_LISTEN &&
542 llc->laddr.lsap == laddr->lsap &&
543 (llc_mac_match(llc->laddr.mac, laddr->mac) ||
544 llc_mac_null(llc->laddr.mac));
545}
546
519/** 547/**
520 * llc_lookup_listener - Finds listener for local MAC + SAP 548 * llc_lookup_listener - Finds listener for local MAC + SAP
521 * @sap: SAP 549 * @sap: SAP
@@ -530,23 +558,26 @@ static struct sock *llc_lookup_listener(struct llc_sap *sap,
530 struct llc_addr *laddr) 558 struct llc_addr *laddr)
531{ 559{
532 struct sock *rc; 560 struct sock *rc;
533 struct hlist_node *node; 561 struct hlist_nulls_node *node;
534 562
535 read_lock(&sap->sk_list.lock); 563 rcu_read_lock();
536 sk_for_each(rc, node, &sap->sk_list.list) { 564again:
537 struct llc_sock *llc = llc_sk(rc); 565 sk_nulls_for_each_rcu(rc, node, &sap->sk_list) {
538 566 if (llc_listener_match(sap, laddr, rc)) {
539 if (rc->sk_type == SOCK_STREAM && rc->sk_state == TCP_LISTEN && 567 /* Extra checks required by SLAB_DESTROY_BY_RCU */
540 llc->laddr.lsap == laddr->lsap && 568 if (unlikely(!atomic_inc_not_zero(&rc->sk_refcnt)))
541 (llc_mac_match(llc->laddr.mac, laddr->mac) || 569 goto again;
542 llc_mac_null(llc->laddr.mac))) { 570 if (unlikely(llc_sk(rc)->sap != sap ||
543 sock_hold(rc); 571 !llc_listener_match(sap, laddr, rc))) {
572 sock_put(rc);
573 continue;
574 }
544 goto found; 575 goto found;
545 } 576 }
546 } 577 }
547 rc = NULL; 578 rc = NULL;
548found: 579found:
549 read_unlock(&sap->sk_list.lock); 580 rcu_read_unlock();
550 return rc; 581 return rc;
551} 582}
552 583
@@ -652,10 +683,10 @@ static int llc_find_offset(int state, int ev_type)
652void llc_sap_add_socket(struct llc_sap *sap, struct sock *sk) 683void llc_sap_add_socket(struct llc_sap *sap, struct sock *sk)
653{ 684{
654 llc_sap_hold(sap); 685 llc_sap_hold(sap);
655 write_lock_bh(&sap->sk_list.lock); 686 spin_lock_bh(&sap->sk_lock);
656 llc_sk(sk)->sap = sap; 687 llc_sk(sk)->sap = sap;
657 sk_add_node(sk, &sap->sk_list.list); 688 sk_nulls_add_node_rcu(sk, &sap->sk_list);
658 write_unlock_bh(&sap->sk_list.lock); 689 spin_unlock_bh(&sap->sk_lock);
659} 690}
660 691
661/** 692/**
@@ -663,14 +694,14 @@ void llc_sap_add_socket(struct llc_sap *sap, struct sock *sk)
663 * @sap: SAP 694 * @sap: SAP
664 * @sk: socket 695 * @sk: socket
665 * 696 *
666 * This function removes a connection from sk_list.list of a SAP if 697 * This function removes a connection from sk_list of a SAP if
667 * the connection was in this list. 698 * the connection was in this list.
668 */ 699 */
669void llc_sap_remove_socket(struct llc_sap *sap, struct sock *sk) 700void llc_sap_remove_socket(struct llc_sap *sap, struct sock *sk)
670{ 701{
671 write_lock_bh(&sap->sk_list.lock); 702 spin_lock_bh(&sap->sk_lock);
672 sk_del_node_init(sk); 703 sk_nulls_del_node_init_rcu(sk);
673 write_unlock_bh(&sap->sk_list.lock); 704 spin_unlock_bh(&sap->sk_lock);
674 llc_sap_put(sap); 705 llc_sap_put(sap);
675} 706}
676 707
diff --git a/net/llc/llc_core.c b/net/llc/llc_core.c
index ff4c0ab96a69..5276b9722077 100644
--- a/net/llc/llc_core.c
+++ b/net/llc/llc_core.c
@@ -37,7 +37,8 @@ static struct llc_sap *llc_sap_alloc(void)
37 if (sap) { 37 if (sap) {
38 /* sap->laddr.mac - leave as a null, it's filled by bind */ 38 /* sap->laddr.mac - leave as a null, it's filled by bind */
39 sap->state = LLC_SAP_STATE_ACTIVE; 39 sap->state = LLC_SAP_STATE_ACTIVE;
40 rwlock_init(&sap->sk_list.lock); 40 spin_lock_init(&sap->sk_lock);
41 INIT_HLIST_NULLS_HEAD(&sap->sk_list, 0);
41 atomic_set(&sap->refcnt, 1); 42 atomic_set(&sap->refcnt, 1);
42 } 43 }
43 return sap; 44 return sap;
@@ -142,7 +143,7 @@ out:
142 */ 143 */
143void llc_sap_close(struct llc_sap *sap) 144void llc_sap_close(struct llc_sap *sap)
144{ 145{
145 WARN_ON(!hlist_empty(&sap->sk_list.list)); 146 WARN_ON(!hlist_nulls_empty(&sap->sk_list));
146 llc_del_sap(sap); 147 llc_del_sap(sap);
147 kfree(sap); 148 kfree(sap);
148} 149}
diff --git a/net/llc/llc_proc.c b/net/llc/llc_proc.c
index be47ac427f6b..6b3d033b3236 100644
--- a/net/llc/llc_proc.c
+++ b/net/llc/llc_proc.c
@@ -34,19 +34,19 @@ static struct sock *llc_get_sk_idx(loff_t pos)
34{ 34{
35 struct list_head *sap_entry; 35 struct list_head *sap_entry;
36 struct llc_sap *sap; 36 struct llc_sap *sap;
37 struct hlist_node *node; 37 struct hlist_nulls_node *node;
38 struct sock *sk = NULL; 38 struct sock *sk = NULL;
39 39
40 list_for_each(sap_entry, &llc_sap_list) { 40 list_for_each(sap_entry, &llc_sap_list) {
41 sap = list_entry(sap_entry, struct llc_sap, node); 41 sap = list_entry(sap_entry, struct llc_sap, node);
42 42
43 read_lock_bh(&sap->sk_list.lock); 43 spin_lock_bh(&sap->sk_lock);
44 sk_for_each(sk, node, &sap->sk_list.list) { 44 sk_nulls_for_each(sk, node, &sap->sk_list) {
45 if (!pos) 45 if (!pos)
46 goto found; 46 goto found;
47 --pos; 47 --pos;
48 } 48 }
49 read_unlock_bh(&sap->sk_list.lock); 49 spin_unlock_bh(&sap->sk_lock);
50 } 50 }
51 sk = NULL; 51 sk = NULL;
52found: 52found:
@@ -73,25 +73,25 @@ static void *llc_seq_next(struct seq_file *seq, void *v, loff_t *pos)
73 goto out; 73 goto out;
74 } 74 }
75 sk = v; 75 sk = v;
76 next = sk_next(sk); 76 next = sk_nulls_next(sk);
77 if (next) { 77 if (next) {
78 sk = next; 78 sk = next;
79 goto out; 79 goto out;
80 } 80 }
81 llc = llc_sk(sk); 81 llc = llc_sk(sk);
82 sap = llc->sap; 82 sap = llc->sap;
83 read_unlock_bh(&sap->sk_list.lock); 83 spin_unlock_bh(&sap->sk_lock);
84 sk = NULL; 84 sk = NULL;
85 for (;;) { 85 for (;;) {
86 if (sap->node.next == &llc_sap_list) 86 if (sap->node.next == &llc_sap_list)
87 break; 87 break;
88 sap = list_entry(sap->node.next, struct llc_sap, node); 88 sap = list_entry(sap->node.next, struct llc_sap, node);
89 read_lock_bh(&sap->sk_list.lock); 89 spin_lock_bh(&sap->sk_lock);
90 if (!hlist_empty(&sap->sk_list.list)) { 90 if (!hlist_nulls_empty(&sap->sk_list)) {
91 sk = sk_head(&sap->sk_list.list); 91 sk = sk_nulls_head(&sap->sk_list);
92 break; 92 break;
93 } 93 }
94 read_unlock_bh(&sap->sk_list.lock); 94 spin_unlock_bh(&sap->sk_lock);
95 } 95 }
96out: 96out:
97 return sk; 97 return sk;
@@ -104,7 +104,7 @@ static void llc_seq_stop(struct seq_file *seq, void *v)
104 struct llc_sock *llc = llc_sk(sk); 104 struct llc_sock *llc = llc_sk(sk);
105 struct llc_sap *sap = llc->sap; 105 struct llc_sap *sap = llc->sap;
106 106
107 read_unlock_bh(&sap->sk_list.lock); 107 spin_unlock_bh(&sap->sk_lock);
108 } 108 }
109 read_unlock_bh(&llc_sap_list_lock); 109 read_unlock_bh(&llc_sap_list_lock);
110} 110}
diff --git a/net/llc/llc_sap.c b/net/llc/llc_sap.c
index 008de1fc42ca..39760d013ce2 100644
--- a/net/llc/llc_sap.c
+++ b/net/llc/llc_sap.c
@@ -297,6 +297,17 @@ static void llc_sap_rcv(struct llc_sap *sap, struct sk_buff *skb,
297 llc_sap_state_process(sap, skb); 297 llc_sap_state_process(sap, skb);
298} 298}
299 299
300static inline bool llc_dgram_match(const struct llc_sap *sap,
301 const struct llc_addr *laddr,
302 const struct sock *sk)
303{
304 struct llc_sock *llc = llc_sk(sk);
305
306 return sk->sk_type == SOCK_DGRAM &&
307 llc->laddr.lsap == laddr->lsap &&
308 llc_mac_match(llc->laddr.mac, laddr->mac);
309}
310
300/** 311/**
301 * llc_lookup_dgram - Finds dgram socket for the local sap/mac 312 * llc_lookup_dgram - Finds dgram socket for the local sap/mac
302 * @sap: SAP 313 * @sap: SAP
@@ -309,25 +320,41 @@ static struct sock *llc_lookup_dgram(struct llc_sap *sap,
309 const struct llc_addr *laddr) 320 const struct llc_addr *laddr)
310{ 321{
311 struct sock *rc; 322 struct sock *rc;
312 struct hlist_node *node; 323 struct hlist_nulls_node *node;
313 324
314 read_lock_bh(&sap->sk_list.lock); 325 rcu_read_lock_bh();
315 sk_for_each(rc, node, &sap->sk_list.list) { 326again:
316 struct llc_sock *llc = llc_sk(rc); 327 sk_nulls_for_each_rcu(rc, node, &sap->sk_list) {
317 328 if (llc_dgram_match(sap, laddr, rc)) {
318 if (rc->sk_type == SOCK_DGRAM && 329 /* Extra checks required by SLAB_DESTROY_BY_RCU */
319 llc->laddr.lsap == laddr->lsap && 330 if (unlikely(!atomic_inc_not_zero(&rc->sk_refcnt)))
320 llc_mac_match(llc->laddr.mac, laddr->mac)) { 331 goto again;
321 sock_hold(rc); 332 if (unlikely(llc_sk(rc)->sap != sap ||
333 !llc_dgram_match(sap, laddr, rc))) {
334 sock_put(rc);
335 continue;
336 }
322 goto found; 337 goto found;
323 } 338 }
324 } 339 }
325 rc = NULL; 340 rc = NULL;
326found: 341found:
327 read_unlock_bh(&sap->sk_list.lock); 342 rcu_read_unlock_bh();
328 return rc; 343 return rc;
329} 344}
330 345
346static inline bool llc_mcast_match(const struct llc_sap *sap,
347 const struct llc_addr *laddr,
348 const struct sk_buff *skb,
349 const struct sock *sk)
350{
351 struct llc_sock *llc = llc_sk(sk);
352
353 return sk->sk_type == SOCK_DGRAM &&
354 llc->laddr.lsap == laddr->lsap &&
355 llc->dev == skb->dev;
356}
357
331/** 358/**
332 * llc_sap_mcast - Deliver multicast PDU's to all matching datagram sockets. 359 * llc_sap_mcast - Deliver multicast PDU's to all matching datagram sockets.
333 * @sap: SAP 360 * @sap: SAP
@@ -341,20 +368,13 @@ static void llc_sap_mcast(struct llc_sap *sap,
341 struct sk_buff *skb) 368 struct sk_buff *skb)
342{ 369{
343 struct sock *sk; 370 struct sock *sk;
344 struct hlist_node *node; 371 struct hlist_nulls_node *node;
345 372
346 read_lock_bh(&sap->sk_list.lock); 373 spin_lock_bh(&sap->sk_lock);
347 sk_for_each(sk, node, &sap->sk_list.list) { 374 sk_nulls_for_each_rcu(sk, node, &sap->sk_list) {
348 struct llc_sock *llc = llc_sk(sk);
349 struct sk_buff *skb1; 375 struct sk_buff *skb1;
350 376
351 if (sk->sk_type != SOCK_DGRAM) 377 if (!llc_mcast_match(sap, laddr, skb, sk))
352 continue;
353
354 if (llc->laddr.lsap != laddr->lsap)
355 continue;
356
357 if (llc->dev != skb->dev)
358 continue; 378 continue;
359 379
360 skb1 = skb_clone(skb, GFP_ATOMIC); 380 skb1 = skb_clone(skb, GFP_ATOMIC);
@@ -365,7 +385,7 @@ static void llc_sap_mcast(struct llc_sap *sap,
365 llc_sap_rcv(sap, skb1, sk); 385 llc_sap_rcv(sap, skb1, sk);
366 sock_put(sk); 386 sock_put(sk);
367 } 387 }
368 read_unlock_bh(&sap->sk_list.lock); 388 spin_unlock_bh(&sap->sk_lock);
369} 389}
370 390
371 391