aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/net/inet_common.h5
-rw-r--r--include/net/inet_hashtables.h122
-rw-r--r--include/net/tcp.h120
-rw-r--r--net/ipv4/Makefile2
-rw-r--r--net/ipv4/inet_hashtables.c51
-rw-r--r--net/ipv4/tcp_ipv4.c26
6 files changed, 181 insertions, 145 deletions
diff --git a/include/net/inet_common.h b/include/net/inet_common.h
index 1fbd94d8a316..f943306ce5ff 100644
--- a/include/net/inet_common.h
+++ b/include/net/inet_common.h
@@ -8,6 +8,11 @@ extern struct proto_ops inet_dgram_ops;
8 * INET4 prototypes used by INET6 8 * INET4 prototypes used by INET6
9 */ 9 */
10 10
11struct msghdr;
12struct sock;
13struct sockaddr;
14struct socket;
15
11extern void inet_remove_sock(struct sock *sk1); 16extern void inet_remove_sock(struct sock *sk1);
12extern void inet_put_sock(unsigned short num, 17extern void inet_put_sock(unsigned short num,
13 struct sock *sk); 18 struct sock *sk);
diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h
index c4c9e39f4505..3a6c11ca421d 100644
--- a/include/net/inet_hashtables.h
+++ b/include/net/inet_hashtables.h
@@ -14,8 +14,107 @@
14#ifndef _INET_HASHTABLES_H 14#ifndef _INET_HASHTABLES_H
15#define _INET_HASHTABLES_H 15#define _INET_HASHTABLES_H
16 16
17#include <linux/ip.h>
18#include <linux/list.h>
19#include <linux/slab.h>
20#include <linux/spinlock.h>
17#include <linux/types.h> 21#include <linux/types.h>
18 22
23/* This is for all connections with a full identity, no wildcards.
24 * New scheme, half the table is for TIME_WAIT, the other half is
25 * for the rest. I'll experiment with dynamic table growth later.
26 */
27struct inet_ehash_bucket {
28 rwlock_t lock;
29 struct hlist_head chain;
30} __attribute__((__aligned__(8)));
31
32/* There are a few simple rules, which allow for local port reuse by
33 * an application. In essence:
34 *
35 * 1) Sockets bound to different interfaces may share a local port.
36 * Failing that, goto test 2.
37 * 2) If all sockets have sk->sk_reuse set, and none of them are in
38 * TCP_LISTEN state, the port may be shared.
39 * Failing that, goto test 3.
40 * 3) If all sockets are bound to a specific inet_sk(sk)->rcv_saddr local
41 * address, and none of them are the same, the port may be
42 * shared.
43 * Failing this, the port cannot be shared.
44 *
45 * The interesting point, is test #2. This is what an FTP server does
46 * all day. To optimize this case we use a specific flag bit defined
47 * below. As we add sockets to a bind bucket list, we perform a
48 * check of: (newsk->sk_reuse && (newsk->sk_state != TCP_LISTEN))
49 * As long as all sockets added to a bind bucket pass this test,
50 * the flag bit will be set.
51 * The resulting situation is that tcp_v[46]_verify_bind() can just check
52 * for this flag bit, if it is set and the socket trying to bind has
53 * sk->sk_reuse set, we don't even have to walk the owners list at all,
54 * we return that it is ok to bind this socket to the requested local port.
55 *
56 * Sounds like a lot of work, but it is worth it. In a more naive
57 * implementation (ie. current FreeBSD etc.) the entire list of ports
58 * must be walked for each data port opened by an ftp server. Needless
59 * to say, this does not scale at all. With a couple thousand FTP
60 * users logged onto your box, isn't it nice to know that new data
61 * ports are created in O(1) time? I thought so. ;-) -DaveM
62 */
63struct inet_bind_bucket {
64 unsigned short port;
65 signed short fastreuse;
66 struct hlist_node node;
67 struct hlist_head owners;
68};
69
70#define inet_bind_bucket_for_each(tb, node, head) \
71 hlist_for_each_entry(tb, node, head, node)
72
73struct inet_bind_hashbucket {
74 spinlock_t lock;
75 struct hlist_head chain;
76};
77
78/* This is for listening sockets, thus all sockets which possess wildcards. */
79#define INET_LHTABLE_SIZE 32 /* Yes, really, this is all you need. */
80
81struct inet_hashinfo {
82 /* This is for sockets with full identity only. Sockets here will
83 * always be without wildcards and will have the following invariant:
84 *
85 * TCP_ESTABLISHED <= sk->sk_state < TCP_CLOSE
86 *
87 * First half of the table is for sockets not in TIME_WAIT, second half
88 * is for TIME_WAIT sockets only.
89 */
90 struct inet_ehash_bucket *ehash;
91
92 /* Ok, let's try this, I give up, we do need a local binding
93 * TCP hash as well as the others for fast bind/connect.
94 */
95 struct inet_bind_hashbucket *bhash;
96
97 int bhash_size;
98 int ehash_size;
99
100 /* All sockets in TCP_LISTEN state will be in here. This is the only
101 * table where wildcard'd TCP sockets can exist. Hash function here
102 * is just local port number.
103 */
104 struct hlist_head listening_hash[INET_LHTABLE_SIZE];
105
106 /* All the above members are written once at bootup and
107 * never written again _or_ are predominantly read-access.
108 *
109 * Now align to a new cache line as all the following members
110 * are often dirty.
111 */
112 rwlock_t lhash_lock ____cacheline_aligned;
113 atomic_t lhash_users;
114 wait_queue_head_t lhash_wait;
115 spinlock_t portalloc_lock;
116};
117
19static inline int inet_ehashfn(const __u32 laddr, const __u16 lport, 118static inline int inet_ehashfn(const __u32 laddr, const __u16 lport,
20 const __u32 faddr, const __u16 fport, 119 const __u32 faddr, const __u16 fport,
21 const int ehash_size) 120 const int ehash_size)
@@ -37,4 +136,27 @@ static inline int inet_sk_ehashfn(const struct sock *sk, const int ehash_size)
37 return inet_ehashfn(laddr, lport, faddr, fport, ehash_size); 136 return inet_ehashfn(laddr, lport, faddr, fport, ehash_size);
38} 137}
39 138
139extern struct inet_bind_bucket *
140 inet_bind_bucket_create(kmem_cache_t *cachep,
141 struct inet_bind_hashbucket *head,
142 const unsigned short snum);
143extern void inet_bind_bucket_destroy(kmem_cache_t *cachep,
144 struct inet_bind_bucket *tb);
145
146static inline int inet_bhashfn(const __u16 lport, const int bhash_size)
147{
148 return lport & (bhash_size - 1);
149}
150
151/* These can have wildcards, don't try too hard. */
152static inline int inet_lhashfn(const unsigned short num)
153{
154 return num & (INET_LHTABLE_SIZE - 1);
155}
156
157static inline int inet_sk_listen_hashfn(const struct sock *sk)
158{
159 return inet_lhashfn(inet_sk(sk)->num);
160}
161
40#endif /* _INET_HASHTABLES_H */ 162#endif /* _INET_HASHTABLES_H */
diff --git a/include/net/tcp.h b/include/net/tcp.h
index 6c9f6f7cab5c..ff5d30ac2b06 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -30,6 +30,7 @@
30#include <linux/slab.h> 30#include <linux/slab.h>
31#include <linux/cache.h> 31#include <linux/cache.h>
32#include <linux/percpu.h> 32#include <linux/percpu.h>
33#include <net/inet_hashtables.h>
33#include <net/checksum.h> 34#include <net/checksum.h>
34#include <net/request_sock.h> 35#include <net/request_sock.h>
35#include <net/sock.h> 36#include <net/sock.h>
@@ -40,101 +41,6 @@
40#endif 41#endif
41#include <linux/seq_file.h> 42#include <linux/seq_file.h>
42 43
43/* This is for all connections with a full identity, no wildcards.
44 * New scheme, half the table is for TIME_WAIT, the other half is
45 * for the rest. I'll experiment with dynamic table growth later.
46 */
47struct inet_ehash_bucket {
48 rwlock_t lock;
49 struct hlist_head chain;
50} __attribute__((__aligned__(8)));
51
52/* This is for listening sockets, thus all sockets which possess wildcards. */
53#define INET_LHTABLE_SIZE 32 /* Yes, really, this is all you need. */
54
55/* There are a few simple rules, which allow for local port reuse by
56 * an application. In essence:
57 *
58 * 1) Sockets bound to different interfaces may share a local port.
59 * Failing that, goto test 2.
60 * 2) If all sockets have sk->sk_reuse set, and none of them are in
61 * TCP_LISTEN state, the port may be shared.
62 * Failing that, goto test 3.
63 * 3) If all sockets are bound to a specific inet_sk(sk)->rcv_saddr local
64 * address, and none of them are the same, the port may be
65 * shared.
66 * Failing this, the port cannot be shared.
67 *
68 * The interesting point, is test #2. This is what an FTP server does
69 * all day. To optimize this case we use a specific flag bit defined
70 * below. As we add sockets to a bind bucket list, we perform a
71 * check of: (newsk->sk_reuse && (newsk->sk_state != TCP_LISTEN))
72 * As long as all sockets added to a bind bucket pass this test,
73 * the flag bit will be set.
74 * The resulting situation is that tcp_v[46]_verify_bind() can just check
75 * for this flag bit, if it is set and the socket trying to bind has
76 * sk->sk_reuse set, we don't even have to walk the owners list at all,
77 * we return that it is ok to bind this socket to the requested local port.
78 *
79 * Sounds like a lot of work, but it is worth it. In a more naive
80 * implementation (ie. current FreeBSD etc.) the entire list of ports
81 * must be walked for each data port opened by an ftp server. Needless
82 * to say, this does not scale at all. With a couple thousand FTP
83 * users logged onto your box, isn't it nice to know that new data
84 * ports are created in O(1) time? I thought so. ;-) -DaveM
85 */
86struct inet_bind_bucket {
87 unsigned short port;
88 signed short fastreuse;
89 struct hlist_node node;
90 struct hlist_head owners;
91};
92
93#define inet_bind_bucket_for_each(tb, node, head) \
94 hlist_for_each_entry(tb, node, head, node)
95
96struct inet_bind_hashbucket {
97 spinlock_t lock;
98 struct hlist_head chain;
99};
100
101struct inet_hashinfo {
102 /* This is for sockets with full identity only. Sockets here will
103 * always be without wildcards and will have the following invariant:
104 *
105 * TCP_ESTABLISHED <= sk->sk_state < TCP_CLOSE
106 *
107 * First half of the table is for sockets not in TIME_WAIT, second half
108 * is for TIME_WAIT sockets only.
109 */
110 struct inet_ehash_bucket *ehash;
111
112 /* Ok, let's try this, I give up, we do need a local binding
113 * TCP hash as well as the others for fast bind/connect.
114 */
115 struct inet_bind_hashbucket *bhash;
116
117 int bhash_size;
118 int ehash_size;
119
120 /* All sockets in TCP_LISTEN state will be in here. This is the only
121 * table where wildcard'd TCP sockets can exist. Hash function here
122 * is just local port number.
123 */
124 struct hlist_head listening_hash[INET_LHTABLE_SIZE];
125
126 /* All the above members are written once at bootup and
127 * never written again _or_ are predominantly read-access.
128 *
129 * Now align to a new cache line as all the following members
130 * are often dirty.
131 */
132 rwlock_t lhash_lock ____cacheline_aligned;
133 atomic_t lhash_users;
134 wait_queue_head_t lhash_wait;
135 spinlock_t portalloc_lock;
136};
137
138extern struct inet_hashinfo tcp_hashinfo; 44extern struct inet_hashinfo tcp_hashinfo;
139#define tcp_ehash (tcp_hashinfo.ehash) 45#define tcp_ehash (tcp_hashinfo.ehash)
140#define tcp_bhash (tcp_hashinfo.bhash) 46#define tcp_bhash (tcp_hashinfo.bhash)
@@ -147,19 +53,8 @@ extern struct inet_hashinfo tcp_hashinfo;
147#define tcp_portalloc_lock (tcp_hashinfo.portalloc_lock) 53#define tcp_portalloc_lock (tcp_hashinfo.portalloc_lock)
148 54
149extern kmem_cache_t *tcp_bucket_cachep; 55extern kmem_cache_t *tcp_bucket_cachep;
150extern struct inet_bind_bucket *
151 inet_bind_bucket_create(kmem_cache_t *cachep,
152 struct inet_bind_hashbucket *head,
153 const unsigned short snum);
154extern void inet_bind_bucket_destroy(kmem_cache_t *cachep,
155 struct inet_bind_bucket *tb);
156extern int tcp_port_rover;
157 56
158/* These are AF independent. */ 57extern int tcp_port_rover;
159static inline int inet_bhashfn(const __u16 lport, const int bhash_size)
160{
161 return lport & (bhash_size - 1);
162}
163 58
164extern void tcp_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, 59extern void tcp_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
165 unsigned short snum); 60 unsigned short snum);
@@ -359,17 +254,6 @@ extern void tcp_tw_deschedule(struct tcp_tw_bucket *tw);
359 ipv6_addr_equal(&inet6_sk(__sk)->rcv_saddr, (__daddr)) && \ 254 ipv6_addr_equal(&inet6_sk(__sk)->rcv_saddr, (__daddr)) && \
360 (!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif)))) 255 (!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
361 256
362/* These can have wildcards, don't try too hard. */
363static inline int inet_lhashfn(const unsigned short num)
364{
365 return num & (INET_LHTABLE_SIZE - 1);
366}
367
368static inline int inet_sk_listen_hashfn(const struct sock *sk)
369{
370 return inet_lhashfn(inet_sk(sk)->num);
371}
372
373#define MAX_TCP_HEADER (128 + MAX_HEADER) 257#define MAX_TCP_HEADER (128 + MAX_HEADER)
374 258
375/* 259/*
diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile
index 61c7386bcd2e..2d8d30e83eb0 100644
--- a/net/ipv4/Makefile
+++ b/net/ipv4/Makefile
@@ -4,7 +4,7 @@
4 4
5obj-y := route.o inetpeer.o protocol.o \ 5obj-y := route.o inetpeer.o protocol.o \
6 ip_input.o ip_fragment.o ip_forward.o ip_options.o \ 6 ip_input.o ip_fragment.o ip_forward.o ip_options.o \
7 ip_output.o ip_sockglue.o \ 7 ip_output.o ip_sockglue.o inet_hashtables.o \
8 tcp.o tcp_input.o tcp_output.o tcp_timer.o tcp_ipv4.o \ 8 tcp.o tcp_input.o tcp_output.o tcp_timer.o tcp_ipv4.o \
9 tcp_minisocks.o tcp_cong.o \ 9 tcp_minisocks.o tcp_cong.o \
10 datagram.o raw.o udp.o arp.o icmp.o devinet.o af_inet.o igmp.o \ 10 datagram.o raw.o udp.o arp.o icmp.o devinet.o af_inet.o igmp.o \
diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
new file mode 100644
index 000000000000..343a890bd617
--- /dev/null
+++ b/net/ipv4/inet_hashtables.c
@@ -0,0 +1,51 @@
1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic INET transport hashtables
7 *
8 * Authors: Lotsa people, from code originally in tcp
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
14 */
15
16#include <linux/config.h>
17#include <linux/slab.h>
18
19#include <net/inet_hashtables.h>
20
21/*
22 * Allocate and initialize a new local port bind bucket.
23 * The bindhash mutex for snum's hash chain must be held here.
24 */
25struct inet_bind_bucket *inet_bind_bucket_create(kmem_cache_t *cachep,
26 struct inet_bind_hashbucket *head,
27 const unsigned short snum)
28{
29 struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, SLAB_ATOMIC);
30
31 if (tb != NULL) {
32 tb->port = snum;
33 tb->fastreuse = 0;
34 INIT_HLIST_HEAD(&tb->owners);
35 hlist_add_head(&tb->node, &head->chain);
36 }
37 return tb;
38}
39
40EXPORT_SYMBOL(inet_bind_bucket_create);
41
42/*
43 * Caller must hold hashbucket lock for this tb with local BH disabled
44 */
45void inet_bind_bucket_destroy(kmem_cache_t *cachep, struct inet_bind_bucket *tb)
46{
47 if (hlist_empty(&tb->owners)) {
48 __hlist_del(&tb->node);
49 kmem_cache_free(cachep, tb);
50 }
51}
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 4138630556e3..58e36ed88f25 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -104,32 +104,6 @@ struct inet_hashinfo __cacheline_aligned tcp_hashinfo = {
104int sysctl_local_port_range[2] = { 1024, 4999 }; 104int sysctl_local_port_range[2] = { 1024, 4999 };
105int tcp_port_rover = 1024 - 1; 105int tcp_port_rover = 1024 - 1;
106 106
107/* Allocate and initialize a new local port bind bucket.
108 * The bindhash mutex for snum's hash chain must be held here.
109 */
110struct inet_bind_bucket *inet_bind_bucket_create(kmem_cache_t *cachep,
111 struct inet_bind_hashbucket *head,
112 const unsigned short snum)
113{
114 struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, SLAB_ATOMIC);
115 if (tb) {
116 tb->port = snum;
117 tb->fastreuse = 0;
118 INIT_HLIST_HEAD(&tb->owners);
119 hlist_add_head(&tb->node, &head->chain);
120 }
121 return tb;
122}
123
124/* Caller must hold hashbucket lock for this tb with local BH disabled */
125void inet_bind_bucket_destroy(kmem_cache_t *cachep, struct inet_bind_bucket *tb)
126{
127 if (hlist_empty(&tb->owners)) {
128 __hlist_del(&tb->node);
129 kmem_cache_free(cachep, tb);
130 }
131}
132
133/* Caller must disable local BH processing. */ 107/* Caller must disable local BH processing. */
134static __inline__ void __tcp_inherit_port(struct sock *sk, struct sock *child) 108static __inline__ void __tcp_inherit_port(struct sock *sk, struct sock *child)
135{ 109{
generated by cgit v1.2.2 (git 2.25.0) at 2025-11-11 06:11:29 -0500