From 1d7138de878d1d4210727c1200193e69596f93b3 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Fri, 12 Nov 2010 05:46:50 +0000 Subject: igmp: RCU conversion of in_dev->mc_list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit in_dev->mc_list is protected by one rwlock (in_dev->mc_list_lock). This can easily be converted to a RCU protection. Writers hold RTNL, so mc_list_lock is removed, not replaced by a spinlock. Signed-off-by: Eric Dumazet Cc: Cypher Wu Cc: Américo Wang Signed-off-by: David S. Miller --- include/net/inet_sock.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/net/inet_sock.h') diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index 1989cfd7405f..8945f9fb192a 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -141,7 +141,7 @@ struct inet_sock { nodefrag:1; int mc_index; __be32 mc_addr; - struct ip_mc_socklist *mc_list; + struct ip_mc_socklist __rcu *mc_list; struct { unsigned int flags; unsigned int fragsize; -- cgit v1.2.2 From 68835aba4d9b74e2f94106d13b6a4bddc447c4c8 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 30 Nov 2010 19:04:07 +0000 Subject: net: optimize INET input path further Followup of commit b178bb3dfc30 (net: reorder struct sock fields) Optimize INET input path a bit further, by : 1) moving sk_refcnt close to sk_lock. This reduces number of dirtied cache lines by one on 64bit arches (and 64 bytes cache line size). 2) moving inet_daddr & inet_rcv_saddr at the beginning of sk (same cache line than hash / family / bound_dev_if / nulls_node) This reduces number of accessed cache lines in lookups by one, and dont increase size of inet and timewait socks. inet and tw sockets now share same place-holder for these fields. Before patch : offsetof(struct sock, sk_refcnt) = 0x10 offsetof(struct sock, sk_lock) = 0x40 offsetof(struct sock, sk_receive_queue) = 0x60 offsetof(struct inet_sock, inet_daddr) = 0x270 offsetof(struct inet_sock, inet_rcv_saddr) = 0x274 After patch : offsetof(struct sock, sk_refcnt) = 0x44 offsetof(struct sock, sk_lock) = 0x48 offsetof(struct sock, sk_receive_queue) = 0x68 offsetof(struct inet_sock, inet_daddr) = 0x0 offsetof(struct inet_sock, inet_rcv_saddr) = 0x4 compute_score() (udp or tcp) now use a single cache line per ignored item, instead of two. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/inet_sock.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include/net/inet_sock.h') diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index 8945f9fb192a..8181498fa96c 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -116,8 +116,9 @@ struct inet_sock { struct ipv6_pinfo *pinet6; #endif /* Socket demultiplex comparisons on incoming packets. */ - __be32 inet_daddr; - __be32 inet_rcv_saddr; +#define inet_daddr sk.__sk_common.skc_daddr +#define inet_rcv_saddr sk.__sk_common.skc_rcv_saddr + __be16 inet_dport; __u16 inet_num; __be32 inet_saddr; -- cgit v1.2.2 From a4daad6b0923030fbd3b00a01f570e4c3eef446b Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Thu, 27 Jan 2011 22:01:53 -0800 Subject: net: Pre-COW metrics for TCP. TCP is going to record metrics for the connection, so pre-COW the route metrics at route cache entry creation time. This avoids several atomic operations that have to occur if we COW the metrics after the entry reaches global visibility. Signed-off-by: David S. Miller --- include/net/inet_sock.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'include/net/inet_sock.h') diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index 8181498fa96c..6e6dfd757682 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -219,7 +219,13 @@ static inline struct request_sock *inet_reqsk_alloc(struct request_sock_ops *ops static inline __u8 inet_sk_flowi_flags(const struct sock *sk) { - return inet_sk(sk)->transparent ? FLOWI_FLAG_ANYSRC : 0; + __u8 flags = 0; + + if (inet_sk(sk)->transparent) + flags |= FLOWI_FLAG_ANYSRC; + if (sk->sk_protocol == IPPROTO_TCP) + flags |= FLOWI_FLAG_PRECOW_METRICS; + return flags; } #endif /* _INET_SOCK_H */ -- cgit v1.2.2 From 1470ddf7f8cecf776921e5ccee72e3d2b3d60cbc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 1 Mar 2011 02:36:47 +0000 Subject: inet: Remove explicit write references to sk/inet in ip_append_data In order to allow simultaneous calls to ip_append_data on the same socket, it must not modify any shared state in sk or inet (other than those that are designed to allow that such as atomic counters). This patch abstracts out write references to sk and inet_sk in ip_append_data and its friends so that we may use the underlying code in parallel. Signed-off-by: Herbert Xu Acked-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/inet_sock.h | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'include/net/inet_sock.h') diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index 6e6dfd757682..7a37369f8ea3 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -86,6 +86,19 @@ static inline struct inet_request_sock *inet_rsk(const struct request_sock *sk) return (struct inet_request_sock *)sk; } +struct inet_cork { + unsigned int flags; + unsigned int fragsize; + struct ip_options *opt; + struct dst_entry *dst; + int length; /* Total length of all frames */ + __be32 addr; + struct flowi fl; + struct page *page; + u32 off; + u8 tx_flags; +}; + struct ip_mc_socklist; struct ipv6_pinfo; struct rtable; @@ -143,15 +156,7 @@ struct inet_sock { int mc_index; __be32 mc_addr; struct ip_mc_socklist __rcu *mc_list; - struct { - unsigned int flags; - unsigned int fragsize; - struct ip_options *opt; - struct dst_entry *dst; - int length; /* Total length of all frames */ - __be32 addr; - struct flowi fl; - } cork; + struct inet_cork cork; }; #define IPCORK_OPT 1 /* ip-options has been held in ipcork.opt */ -- cgit v1.2.2 From f6d8bd051c391c1c0458a30b2a7abcd939329259 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 21 Apr 2011 09:45:37 +0000 Subject: inet: add RCU protection to inet->opt We lack proper synchronization to manipulate inet->opt ip_options Problem is ip_make_skb() calls ip_setup_cork() and ip_setup_cork() possibly makes a copy of ipc->opt (struct ip_options), without any protection against another thread manipulating inet->opt. Another thread can change inet->opt pointer and free old one under us. Use RCU to protect inet->opt (changed to inet->inet_opt). Instead of handling atomic refcounts, just copy ip_options when necessary, to avoid cache line dirtying. We cant insert an rcu_head in struct ip_options since its included in skb->cb[], so this patch is large because I had to introduce a new ip_options_rcu structure. Signed-off-by: Eric Dumazet Cc: Herbert Xu Signed-off-by: David S. Miller --- include/net/inet_sock.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'include/net/inet_sock.h') diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index 7a37369f8ea3..ed2ba6eca724 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -57,7 +57,15 @@ struct ip_options { unsigned char __data[0]; }; -#define optlength(opt) (sizeof(struct ip_options) + opt->optlen) +struct ip_options_rcu { + struct rcu_head rcu; + struct ip_options opt; +}; + +struct ip_options_data { + struct ip_options_rcu opt; + char data[40]; +}; struct inet_request_sock { struct request_sock req; @@ -78,7 +86,7 @@ struct inet_request_sock { acked : 1, no_srccheck: 1; kmemcheck_bitfield_end(flags); - struct ip_options *opt; + struct ip_options_rcu *opt; }; static inline struct inet_request_sock *inet_rsk(const struct request_sock *sk) @@ -140,7 +148,7 @@ struct inet_sock { __be16 inet_sport; __u16 inet_id; - struct ip_options *opt; + struct ip_options_rcu __rcu *inet_opt; __u8 tos; __u8 min_ttl; __u8 mc_ttl; -- cgit v1.2.2 From bdc712b4c2baf9515887de3a52e7ecd89fafc0c7 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Fri, 6 May 2011 15:02:07 -0700 Subject: inet: Decrease overhead of on-stack inet_cork. When we fast path datagram sends to avoid locking by putting the inet_cork on the stack we use up lots of space that isn't necessary. This is because inet_cork contains a "struct flowi" which isn't used in these code paths. Split inet_cork to two parts, "inet_cork" and "inet_cork_full". Only the latter of which has the "struct flowi" and is what is stored in inet_sock. Signed-off-by: David S. Miller Acked-by: Eric Dumazet --- include/net/inet_sock.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'include/net/inet_sock.h') diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index ed2ba6eca724..caaff5f5f39f 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -96,17 +96,21 @@ static inline struct inet_request_sock *inet_rsk(const struct request_sock *sk) struct inet_cork { unsigned int flags; - unsigned int fragsize; + __be32 addr; struct ip_options *opt; + unsigned int fragsize; struct dst_entry *dst; int length; /* Total length of all frames */ - __be32 addr; - struct flowi fl; struct page *page; u32 off; u8 tx_flags; }; +struct inet_cork_full { + struct inet_cork base; + struct flowi fl; +}; + struct ip_mc_socklist; struct ipv6_pinfo; struct rtable; @@ -164,7 +168,7 @@ struct inet_sock { int mc_index; __be32 mc_addr; struct ip_mc_socklist __rcu *mc_list; - struct inet_cork cork; + struct inet_cork_full cork; }; #define IPCORK_OPT 1 /* ip-options has been held in ipcork.opt */ -- cgit v1.2.2