diff options
author | Octavian Purdila <opurdila@ixiacom.com> | 2009-12-26 06:51:02 -0500 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2009-12-26 23:41:43 -0500 |
commit | b76f5a8427ac2928c07fa4ff2144bb8db072c240 (patch) | |
tree | 542d64a8bd28baf69f87b199a0156f865a06551d | |
parent | abf9d537fea225af60762640361af7fb233b3103 (diff) |
llc: convert the socket list to RCU locking
For the reclamation phase we use the SLAB_DESTROY_BY_RCU mechanism,
which require some extra checks in the lookup code:
a) If the current socket was released, reallocated & inserted in
another list it will short circuit the iteration for the current list,
thus we need to restart the lookup.
b) If the current socket was released, reallocated & inserted in the
same list we just need to recheck it matches the look-up criteria and
if not we can skip to the next element.
In this case there is no need to restart the lookup, since sockets are
inserted at the start of the list and the worst that will happen is
that we will iterate throught some of the list elements more then
once.
Note that the /proc and multicast delivery was not yet converted to
RCU, it still uses spinlocks for protection.
Signed-off-by: Octavian Purdila <opurdila@ixiacom.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r-- | include/net/llc.h | 7 | ||||
-rw-r--r-- | net/llc/af_llc.c | 1 | ||||
-rw-r--r-- | net/llc/llc_conn.c | 93 | ||||
-rw-r--r-- | net/llc/llc_core.c | 5 | ||||
-rw-r--r-- | net/llc/llc_proc.c | 22 | ||||
-rw-r--r-- | net/llc/llc_sap.c | 66 |
6 files changed, 123 insertions, 71 deletions
diff --git a/include/net/llc.h b/include/net/llc.h index 7940da1606e7..1559cf10e874 100644 --- a/include/net/llc.h +++ b/include/net/llc.h | |||
@@ -16,6 +16,7 @@ | |||
16 | #include <linux/if_ether.h> | 16 | #include <linux/if_ether.h> |
17 | #include <linux/list.h> | 17 | #include <linux/list.h> |
18 | #include <linux/spinlock.h> | 18 | #include <linux/spinlock.h> |
19 | #include <linux/rculist_nulls.h> | ||
19 | 20 | ||
20 | #include <asm/atomic.h> | 21 | #include <asm/atomic.h> |
21 | 22 | ||
@@ -53,10 +54,8 @@ struct llc_sap { | |||
53 | struct net_device *orig_dev); | 54 | struct net_device *orig_dev); |
54 | struct llc_addr laddr; | 55 | struct llc_addr laddr; |
55 | struct list_head node; | 56 | struct list_head node; |
56 | struct { | 57 | spinlock_t sk_lock; |
57 | rwlock_t lock; | 58 | struct hlist_nulls_head sk_list; |
58 | struct hlist_head list; | ||
59 | } sk_list; | ||
60 | }; | 59 | }; |
61 | 60 | ||
62 | #define LLC_DEST_INVALID 0 /* Invalid LLC PDU type */ | 61 | #define LLC_DEST_INVALID 0 /* Invalid LLC PDU type */ |
diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c index c4d1a1da813c..f49f3dd6fbd3 100644 --- a/net/llc/af_llc.c +++ b/net/llc/af_llc.c | |||
@@ -140,6 +140,7 @@ static struct proto llc_proto = { | |||
140 | .name = "LLC", | 140 | .name = "LLC", |
141 | .owner = THIS_MODULE, | 141 | .owner = THIS_MODULE, |
142 | .obj_size = sizeof(struct llc_sock), | 142 | .obj_size = sizeof(struct llc_sock), |
143 | .slab_flags = SLAB_DESTROY_BY_RCU, | ||
143 | }; | 144 | }; |
144 | 145 | ||
145 | /** | 146 | /** |
diff --git a/net/llc/llc_conn.c b/net/llc/llc_conn.c index c6bab39b018e..77bb3816655e 100644 --- a/net/llc/llc_conn.c +++ b/net/llc/llc_conn.c | |||
@@ -468,6 +468,19 @@ static int llc_exec_conn_trans_actions(struct sock *sk, | |||
468 | return rc; | 468 | return rc; |
469 | } | 469 | } |
470 | 470 | ||
471 | static inline bool llc_estab_match(const struct llc_sap *sap, | ||
472 | const struct llc_addr *daddr, | ||
473 | const struct llc_addr *laddr, | ||
474 | const struct sock *sk) | ||
475 | { | ||
476 | struct llc_sock *llc = llc_sk(sk); | ||
477 | |||
478 | return llc->laddr.lsap == laddr->lsap && | ||
479 | llc->daddr.lsap == daddr->lsap && | ||
480 | llc_mac_match(llc->laddr.mac, laddr->mac) && | ||
481 | llc_mac_match(llc->daddr.mac, daddr->mac); | ||
482 | } | ||
483 | |||
471 | /** | 484 | /** |
472 | * __llc_lookup_established - Finds connection for the remote/local sap/mac | 485 | * __llc_lookup_established - Finds connection for the remote/local sap/mac |
473 | * @sap: SAP | 486 | * @sap: SAP |
@@ -484,23 +497,26 @@ static struct sock *__llc_lookup_established(struct llc_sap *sap, | |||
484 | struct llc_addr *laddr) | 497 | struct llc_addr *laddr) |
485 | { | 498 | { |
486 | struct sock *rc; | 499 | struct sock *rc; |
487 | struct hlist_node *node; | 500 | struct hlist_nulls_node *node; |
488 | 501 | ||
489 | read_lock(&sap->sk_list.lock); | 502 | rcu_read_lock(); |
490 | sk_for_each(rc, node, &sap->sk_list.list) { | 503 | again: |
491 | struct llc_sock *llc = llc_sk(rc); | 504 | sk_nulls_for_each_rcu(rc, node, &sap->sk_list) { |
492 | 505 | if (llc_estab_match(sap, daddr, laddr, rc)) { | |
493 | if (llc->laddr.lsap == laddr->lsap && | 506 | /* Extra checks required by SLAB_DESTROY_BY_RCU */ |
494 | llc->daddr.lsap == daddr->lsap && | 507 | if (unlikely(!atomic_inc_not_zero(&rc->sk_refcnt))) |
495 | llc_mac_match(llc->laddr.mac, laddr->mac) && | 508 | goto again; |
496 | llc_mac_match(llc->daddr.mac, daddr->mac)) { | 509 | if (unlikely(llc_sk(rc)->sap != sap || |
497 | sock_hold(rc); | 510 | !llc_estab_match(sap, daddr, laddr, rc))) { |
511 | sock_put(rc); | ||
512 | continue; | ||
513 | } | ||
498 | goto found; | 514 | goto found; |
499 | } | 515 | } |
500 | } | 516 | } |
501 | rc = NULL; | 517 | rc = NULL; |
502 | found: | 518 | found: |
503 | read_unlock(&sap->sk_list.lock); | 519 | rcu_read_unlock(); |
504 | return rc; | 520 | return rc; |
505 | } | 521 | } |
506 | 522 | ||
@@ -516,6 +532,18 @@ struct sock *llc_lookup_established(struct llc_sap *sap, | |||
516 | return sk; | 532 | return sk; |
517 | } | 533 | } |
518 | 534 | ||
535 | static inline bool llc_listener_match(const struct llc_sap *sap, | ||
536 | const struct llc_addr *laddr, | ||
537 | const struct sock *sk) | ||
538 | { | ||
539 | struct llc_sock *llc = llc_sk(sk); | ||
540 | |||
541 | return sk->sk_type == SOCK_STREAM && sk->sk_state == TCP_LISTEN && | ||
542 | llc->laddr.lsap == laddr->lsap && | ||
543 | (llc_mac_match(llc->laddr.mac, laddr->mac) || | ||
544 | llc_mac_null(llc->laddr.mac)); | ||
545 | } | ||
546 | |||
519 | /** | 547 | /** |
520 | * llc_lookup_listener - Finds listener for local MAC + SAP | 548 | * llc_lookup_listener - Finds listener for local MAC + SAP |
521 | * @sap: SAP | 549 | * @sap: SAP |
@@ -530,23 +558,26 @@ static struct sock *llc_lookup_listener(struct llc_sap *sap, | |||
530 | struct llc_addr *laddr) | 558 | struct llc_addr *laddr) |
531 | { | 559 | { |
532 | struct sock *rc; | 560 | struct sock *rc; |
533 | struct hlist_node *node; | 561 | struct hlist_nulls_node *node; |
534 | 562 | ||
535 | read_lock(&sap->sk_list.lock); | 563 | rcu_read_lock(); |
536 | sk_for_each(rc, node, &sap->sk_list.list) { | 564 | again: |
537 | struct llc_sock *llc = llc_sk(rc); | 565 | sk_nulls_for_each_rcu(rc, node, &sap->sk_list) { |
538 | 566 | if (llc_listener_match(sap, laddr, rc)) { | |
539 | if (rc->sk_type == SOCK_STREAM && rc->sk_state == TCP_LISTEN && | 567 | /* Extra checks required by SLAB_DESTROY_BY_RCU */ |
540 | llc->laddr.lsap == laddr->lsap && | 568 | if (unlikely(!atomic_inc_not_zero(&rc->sk_refcnt))) |
541 | (llc_mac_match(llc->laddr.mac, laddr->mac) || | 569 | goto again; |
542 | llc_mac_null(llc->laddr.mac))) { | 570 | if (unlikely(llc_sk(rc)->sap != sap || |
543 | sock_hold(rc); | 571 | !llc_listener_match(sap, laddr, rc))) { |
572 | sock_put(rc); | ||
573 | continue; | ||
574 | } | ||
544 | goto found; | 575 | goto found; |
545 | } | 576 | } |
546 | } | 577 | } |
547 | rc = NULL; | 578 | rc = NULL; |
548 | found: | 579 | found: |
549 | read_unlock(&sap->sk_list.lock); | 580 | rcu_read_unlock(); |
550 | return rc; | 581 | return rc; |
551 | } | 582 | } |
552 | 583 | ||
@@ -652,10 +683,10 @@ static int llc_find_offset(int state, int ev_type) | |||
652 | void llc_sap_add_socket(struct llc_sap *sap, struct sock *sk) | 683 | void llc_sap_add_socket(struct llc_sap *sap, struct sock *sk) |
653 | { | 684 | { |
654 | llc_sap_hold(sap); | 685 | llc_sap_hold(sap); |
655 | write_lock_bh(&sap->sk_list.lock); | 686 | spin_lock_bh(&sap->sk_lock); |
656 | llc_sk(sk)->sap = sap; | 687 | llc_sk(sk)->sap = sap; |
657 | sk_add_node(sk, &sap->sk_list.list); | 688 | sk_nulls_add_node_rcu(sk, &sap->sk_list); |
658 | write_unlock_bh(&sap->sk_list.lock); | 689 | spin_unlock_bh(&sap->sk_lock); |
659 | } | 690 | } |
660 | 691 | ||
661 | /** | 692 | /** |
@@ -663,14 +694,14 @@ void llc_sap_add_socket(struct llc_sap *sap, struct sock *sk) | |||
663 | * @sap: SAP | 694 | * @sap: SAP |
664 | * @sk: socket | 695 | * @sk: socket |
665 | * | 696 | * |
666 | * This function removes a connection from sk_list.list of a SAP if | 697 | * This function removes a connection from sk_list of a SAP if |
667 | * the connection was in this list. | 698 | * the connection was in this list. |
668 | */ | 699 | */ |
669 | void llc_sap_remove_socket(struct llc_sap *sap, struct sock *sk) | 700 | void llc_sap_remove_socket(struct llc_sap *sap, struct sock *sk) |
670 | { | 701 | { |
671 | write_lock_bh(&sap->sk_list.lock); | 702 | spin_lock_bh(&sap->sk_lock); |
672 | sk_del_node_init(sk); | 703 | sk_nulls_del_node_init_rcu(sk); |
673 | write_unlock_bh(&sap->sk_list.lock); | 704 | spin_unlock_bh(&sap->sk_lock); |
674 | llc_sap_put(sap); | 705 | llc_sap_put(sap); |
675 | } | 706 | } |
676 | 707 | ||
diff --git a/net/llc/llc_core.c b/net/llc/llc_core.c index ff4c0ab96a69..5276b9722077 100644 --- a/net/llc/llc_core.c +++ b/net/llc/llc_core.c | |||
@@ -37,7 +37,8 @@ static struct llc_sap *llc_sap_alloc(void) | |||
37 | if (sap) { | 37 | if (sap) { |
38 | /* sap->laddr.mac - leave as a null, it's filled by bind */ | 38 | /* sap->laddr.mac - leave as a null, it's filled by bind */ |
39 | sap->state = LLC_SAP_STATE_ACTIVE; | 39 | sap->state = LLC_SAP_STATE_ACTIVE; |
40 | rwlock_init(&sap->sk_list.lock); | 40 | spin_lock_init(&sap->sk_lock); |
41 | INIT_HLIST_NULLS_HEAD(&sap->sk_list, 0); | ||
41 | atomic_set(&sap->refcnt, 1); | 42 | atomic_set(&sap->refcnt, 1); |
42 | } | 43 | } |
43 | return sap; | 44 | return sap; |
@@ -142,7 +143,7 @@ out: | |||
142 | */ | 143 | */ |
143 | void llc_sap_close(struct llc_sap *sap) | 144 | void llc_sap_close(struct llc_sap *sap) |
144 | { | 145 | { |
145 | WARN_ON(!hlist_empty(&sap->sk_list.list)); | 146 | WARN_ON(!hlist_nulls_empty(&sap->sk_list)); |
146 | llc_del_sap(sap); | 147 | llc_del_sap(sap); |
147 | kfree(sap); | 148 | kfree(sap); |
148 | } | 149 | } |
diff --git a/net/llc/llc_proc.c b/net/llc/llc_proc.c index be47ac427f6b..6b3d033b3236 100644 --- a/net/llc/llc_proc.c +++ b/net/llc/llc_proc.c | |||
@@ -34,19 +34,19 @@ static struct sock *llc_get_sk_idx(loff_t pos) | |||
34 | { | 34 | { |
35 | struct list_head *sap_entry; | 35 | struct list_head *sap_entry; |
36 | struct llc_sap *sap; | 36 | struct llc_sap *sap; |
37 | struct hlist_node *node; | 37 | struct hlist_nulls_node *node; |
38 | struct sock *sk = NULL; | 38 | struct sock *sk = NULL; |
39 | 39 | ||
40 | list_for_each(sap_entry, &llc_sap_list) { | 40 | list_for_each(sap_entry, &llc_sap_list) { |
41 | sap = list_entry(sap_entry, struct llc_sap, node); | 41 | sap = list_entry(sap_entry, struct llc_sap, node); |
42 | 42 | ||
43 | read_lock_bh(&sap->sk_list.lock); | 43 | spin_lock_bh(&sap->sk_lock); |
44 | sk_for_each(sk, node, &sap->sk_list.list) { | 44 | sk_nulls_for_each(sk, node, &sap->sk_list) { |
45 | if (!pos) | 45 | if (!pos) |
46 | goto found; | 46 | goto found; |
47 | --pos; | 47 | --pos; |
48 | } | 48 | } |
49 | read_unlock_bh(&sap->sk_list.lock); | 49 | spin_unlock_bh(&sap->sk_lock); |
50 | } | 50 | } |
51 | sk = NULL; | 51 | sk = NULL; |
52 | found: | 52 | found: |
@@ -73,25 +73,25 @@ static void *llc_seq_next(struct seq_file *seq, void *v, loff_t *pos) | |||
73 | goto out; | 73 | goto out; |
74 | } | 74 | } |
75 | sk = v; | 75 | sk = v; |
76 | next = sk_next(sk); | 76 | next = sk_nulls_next(sk); |
77 | if (next) { | 77 | if (next) { |
78 | sk = next; | 78 | sk = next; |
79 | goto out; | 79 | goto out; |
80 | } | 80 | } |
81 | llc = llc_sk(sk); | 81 | llc = llc_sk(sk); |
82 | sap = llc->sap; | 82 | sap = llc->sap; |
83 | read_unlock_bh(&sap->sk_list.lock); | 83 | spin_unlock_bh(&sap->sk_lock); |
84 | sk = NULL; | 84 | sk = NULL; |
85 | for (;;) { | 85 | for (;;) { |
86 | if (sap->node.next == &llc_sap_list) | 86 | if (sap->node.next == &llc_sap_list) |
87 | break; | 87 | break; |
88 | sap = list_entry(sap->node.next, struct llc_sap, node); | 88 | sap = list_entry(sap->node.next, struct llc_sap, node); |
89 | read_lock_bh(&sap->sk_list.lock); | 89 | spin_lock_bh(&sap->sk_lock); |
90 | if (!hlist_empty(&sap->sk_list.list)) { | 90 | if (!hlist_nulls_empty(&sap->sk_list)) { |
91 | sk = sk_head(&sap->sk_list.list); | 91 | sk = sk_nulls_head(&sap->sk_list); |
92 | break; | 92 | break; |
93 | } | 93 | } |
94 | read_unlock_bh(&sap->sk_list.lock); | 94 | spin_unlock_bh(&sap->sk_lock); |
95 | } | 95 | } |
96 | out: | 96 | out: |
97 | return sk; | 97 | return sk; |
@@ -104,7 +104,7 @@ static void llc_seq_stop(struct seq_file *seq, void *v) | |||
104 | struct llc_sock *llc = llc_sk(sk); | 104 | struct llc_sock *llc = llc_sk(sk); |
105 | struct llc_sap *sap = llc->sap; | 105 | struct llc_sap *sap = llc->sap; |
106 | 106 | ||
107 | read_unlock_bh(&sap->sk_list.lock); | 107 | spin_unlock_bh(&sap->sk_lock); |
108 | } | 108 | } |
109 | read_unlock_bh(&llc_sap_list_lock); | 109 | read_unlock_bh(&llc_sap_list_lock); |
110 | } | 110 | } |
diff --git a/net/llc/llc_sap.c b/net/llc/llc_sap.c index 008de1fc42ca..39760d013ce2 100644 --- a/net/llc/llc_sap.c +++ b/net/llc/llc_sap.c | |||
@@ -297,6 +297,17 @@ static void llc_sap_rcv(struct llc_sap *sap, struct sk_buff *skb, | |||
297 | llc_sap_state_process(sap, skb); | 297 | llc_sap_state_process(sap, skb); |
298 | } | 298 | } |
299 | 299 | ||
300 | static inline bool llc_dgram_match(const struct llc_sap *sap, | ||
301 | const struct llc_addr *laddr, | ||
302 | const struct sock *sk) | ||
303 | { | ||
304 | struct llc_sock *llc = llc_sk(sk); | ||
305 | |||
306 | return sk->sk_type == SOCK_DGRAM && | ||
307 | llc->laddr.lsap == laddr->lsap && | ||
308 | llc_mac_match(llc->laddr.mac, laddr->mac); | ||
309 | } | ||
310 | |||
300 | /** | 311 | /** |
301 | * llc_lookup_dgram - Finds dgram socket for the local sap/mac | 312 | * llc_lookup_dgram - Finds dgram socket for the local sap/mac |
302 | * @sap: SAP | 313 | * @sap: SAP |
@@ -309,25 +320,41 @@ static struct sock *llc_lookup_dgram(struct llc_sap *sap, | |||
309 | const struct llc_addr *laddr) | 320 | const struct llc_addr *laddr) |
310 | { | 321 | { |
311 | struct sock *rc; | 322 | struct sock *rc; |
312 | struct hlist_node *node; | 323 | struct hlist_nulls_node *node; |
313 | 324 | ||
314 | read_lock_bh(&sap->sk_list.lock); | 325 | rcu_read_lock_bh(); |
315 | sk_for_each(rc, node, &sap->sk_list.list) { | 326 | again: |
316 | struct llc_sock *llc = llc_sk(rc); | 327 | sk_nulls_for_each_rcu(rc, node, &sap->sk_list) { |
317 | 328 | if (llc_dgram_match(sap, laddr, rc)) { | |
318 | if (rc->sk_type == SOCK_DGRAM && | 329 | /* Extra checks required by SLAB_DESTROY_BY_RCU */ |
319 | llc->laddr.lsap == laddr->lsap && | 330 | if (unlikely(!atomic_inc_not_zero(&rc->sk_refcnt))) |
320 | llc_mac_match(llc->laddr.mac, laddr->mac)) { | 331 | goto again; |
321 | sock_hold(rc); | 332 | if (unlikely(llc_sk(rc)->sap != sap || |
333 | !llc_dgram_match(sap, laddr, rc))) { | ||
334 | sock_put(rc); | ||
335 | continue; | ||
336 | } | ||
322 | goto found; | 337 | goto found; |
323 | } | 338 | } |
324 | } | 339 | } |
325 | rc = NULL; | 340 | rc = NULL; |
326 | found: | 341 | found: |
327 | read_unlock_bh(&sap->sk_list.lock); | 342 | rcu_read_unlock_bh(); |
328 | return rc; | 343 | return rc; |
329 | } | 344 | } |
330 | 345 | ||
346 | static inline bool llc_mcast_match(const struct llc_sap *sap, | ||
347 | const struct llc_addr *laddr, | ||
348 | const struct sk_buff *skb, | ||
349 | const struct sock *sk) | ||
350 | { | ||
351 | struct llc_sock *llc = llc_sk(sk); | ||
352 | |||
353 | return sk->sk_type == SOCK_DGRAM && | ||
354 | llc->laddr.lsap == laddr->lsap && | ||
355 | llc->dev == skb->dev; | ||
356 | } | ||
357 | |||
331 | /** | 358 | /** |
332 | * llc_sap_mcast - Deliver multicast PDU's to all matching datagram sockets. | 359 | * llc_sap_mcast - Deliver multicast PDU's to all matching datagram sockets. |
333 | * @sap: SAP | 360 | * @sap: SAP |
@@ -341,20 +368,13 @@ static void llc_sap_mcast(struct llc_sap *sap, | |||
341 | struct sk_buff *skb) | 368 | struct sk_buff *skb) |
342 | { | 369 | { |
343 | struct sock *sk; | 370 | struct sock *sk; |
344 | struct hlist_node *node; | 371 | struct hlist_nulls_node *node; |
345 | 372 | ||
346 | read_lock_bh(&sap->sk_list.lock); | 373 | spin_lock_bh(&sap->sk_lock); |
347 | sk_for_each(sk, node, &sap->sk_list.list) { | 374 | sk_nulls_for_each_rcu(sk, node, &sap->sk_list) { |
348 | struct llc_sock *llc = llc_sk(sk); | ||
349 | struct sk_buff *skb1; | 375 | struct sk_buff *skb1; |
350 | 376 | ||
351 | if (sk->sk_type != SOCK_DGRAM) | 377 | if (!llc_mcast_match(sap, laddr, skb, sk)) |
352 | continue; | ||
353 | |||
354 | if (llc->laddr.lsap != laddr->lsap) | ||
355 | continue; | ||
356 | |||
357 | if (llc->dev != skb->dev) | ||
358 | continue; | 378 | continue; |
359 | 379 | ||
360 | skb1 = skb_clone(skb, GFP_ATOMIC); | 380 | skb1 = skb_clone(skb, GFP_ATOMIC); |
@@ -365,7 +385,7 @@ static void llc_sap_mcast(struct llc_sap *sap, | |||
365 | llc_sap_rcv(sap, skb1, sk); | 385 | llc_sap_rcv(sap, skb1, sk); |
366 | sock_put(sk); | 386 | sock_put(sk); |
367 | } | 387 | } |
368 | read_unlock_bh(&sap->sk_list.lock); | 388 | spin_unlock_bh(&sap->sk_lock); |
369 | } | 389 | } |
370 | 390 | ||
371 | 391 | ||