diff options
Diffstat (limited to 'net/ipv4/udp.c')
-rw-r--r-- | net/ipv4/udp.c | 93 |
1 files changed, 67 insertions, 26 deletions
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index d0d436d6216c..ee61b3fc4cae 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c | |||
@@ -106,7 +106,7 @@ | |||
106 | #include <net/xfrm.h> | 106 | #include <net/xfrm.h> |
107 | #include "udp_impl.h" | 107 | #include "udp_impl.h" |
108 | 108 | ||
109 | struct udp_table udp_table; | 109 | struct udp_table udp_table __read_mostly; |
110 | EXPORT_SYMBOL(udp_table); | 110 | EXPORT_SYMBOL(udp_table); |
111 | 111 | ||
112 | int sysctl_udp_mem[3] __read_mostly; | 112 | int sysctl_udp_mem[3] __read_mostly; |
@@ -121,14 +121,16 @@ EXPORT_SYMBOL(sysctl_udp_wmem_min); | |||
121 | atomic_t udp_memory_allocated; | 121 | atomic_t udp_memory_allocated; |
122 | EXPORT_SYMBOL(udp_memory_allocated); | 122 | EXPORT_SYMBOL(udp_memory_allocated); |
123 | 123 | ||
124 | #define PORTS_PER_CHAIN (65536 / UDP_HTABLE_SIZE) | 124 | #define MAX_UDP_PORTS 65536 |
125 | #define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN) | ||
125 | 126 | ||
126 | static int udp_lib_lport_inuse(struct net *net, __u16 num, | 127 | static int udp_lib_lport_inuse(struct net *net, __u16 num, |
127 | const struct udp_hslot *hslot, | 128 | const struct udp_hslot *hslot, |
128 | unsigned long *bitmap, | 129 | unsigned long *bitmap, |
129 | struct sock *sk, | 130 | struct sock *sk, |
130 | int (*saddr_comp)(const struct sock *sk1, | 131 | int (*saddr_comp)(const struct sock *sk1, |
131 | const struct sock *sk2)) | 132 | const struct sock *sk2), |
133 | unsigned int log) | ||
132 | { | 134 | { |
133 | struct sock *sk2; | 135 | struct sock *sk2; |
134 | struct hlist_nulls_node *node; | 136 | struct hlist_nulls_node *node; |
@@ -142,8 +144,7 @@ static int udp_lib_lport_inuse(struct net *net, __u16 num, | |||
142 | || sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && | 144 | || sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && |
143 | (*saddr_comp)(sk, sk2)) { | 145 | (*saddr_comp)(sk, sk2)) { |
144 | if (bitmap) | 146 | if (bitmap) |
145 | __set_bit(sk2->sk_hash / UDP_HTABLE_SIZE, | 147 | __set_bit(sk2->sk_hash >> log, bitmap); |
146 | bitmap); | ||
147 | else | 148 | else |
148 | return 1; | 149 | return 1; |
149 | } | 150 | } |
@@ -180,13 +181,15 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, | |||
180 | /* | 181 | /* |
181 | * force rand to be an odd multiple of UDP_HTABLE_SIZE | 182 | * force rand to be an odd multiple of UDP_HTABLE_SIZE |
182 | */ | 183 | */ |
183 | rand = (rand | 1) * UDP_HTABLE_SIZE; | 184 | rand = (rand | 1) * (udptable->mask + 1); |
184 | for (last = first + UDP_HTABLE_SIZE; first != last; first++) { | 185 | for (last = first + udptable->mask + 1; |
185 | hslot = &udptable->hash[udp_hashfn(net, first)]; | 186 | first != last; |
187 | first++) { | ||
188 | hslot = udp_hashslot(udptable, net, first); | ||
186 | bitmap_zero(bitmap, PORTS_PER_CHAIN); | 189 | bitmap_zero(bitmap, PORTS_PER_CHAIN); |
187 | spin_lock_bh(&hslot->lock); | 190 | spin_lock_bh(&hslot->lock); |
188 | udp_lib_lport_inuse(net, snum, hslot, bitmap, sk, | 191 | udp_lib_lport_inuse(net, snum, hslot, bitmap, sk, |
189 | saddr_comp); | 192 | saddr_comp, udptable->log); |
190 | 193 | ||
191 | snum = first; | 194 | snum = first; |
192 | /* | 195 | /* |
@@ -196,7 +199,7 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, | |||
196 | */ | 199 | */ |
197 | do { | 200 | do { |
198 | if (low <= snum && snum <= high && | 201 | if (low <= snum && snum <= high && |
199 | !test_bit(snum / UDP_HTABLE_SIZE, bitmap)) | 202 | !test_bit(snum >> udptable->log, bitmap)) |
200 | goto found; | 203 | goto found; |
201 | snum += rand; | 204 | snum += rand; |
202 | } while (snum != first); | 205 | } while (snum != first); |
@@ -204,9 +207,10 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, | |||
204 | } | 207 | } |
205 | goto fail; | 208 | goto fail; |
206 | } else { | 209 | } else { |
207 | hslot = &udptable->hash[udp_hashfn(net, snum)]; | 210 | hslot = udp_hashslot(udptable, net, snum); |
208 | spin_lock_bh(&hslot->lock); | 211 | spin_lock_bh(&hslot->lock); |
209 | if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, saddr_comp)) | 212 | if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, |
213 | saddr_comp, 0)) | ||
210 | goto fail_unlock; | 214 | goto fail_unlock; |
211 | } | 215 | } |
212 | found: | 216 | found: |
@@ -283,7 +287,7 @@ static struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, | |||
283 | struct sock *sk, *result; | 287 | struct sock *sk, *result; |
284 | struct hlist_nulls_node *node; | 288 | struct hlist_nulls_node *node; |
285 | unsigned short hnum = ntohs(dport); | 289 | unsigned short hnum = ntohs(dport); |
286 | unsigned int hash = udp_hashfn(net, hnum); | 290 | unsigned int hash = udp_hashfn(net, hnum, udptable->mask); |
287 | struct udp_hslot *hslot = &udptable->hash[hash]; | 291 | struct udp_hslot *hslot = &udptable->hash[hash]; |
288 | int score, badness; | 292 | int score, badness; |
289 | 293 | ||
@@ -982,7 +986,7 @@ try_again: | |||
982 | UDP_INC_STATS_USER(sock_net(sk), | 986 | UDP_INC_STATS_USER(sock_net(sk), |
983 | UDP_MIB_INDATAGRAMS, is_udplite); | 987 | UDP_MIB_INDATAGRAMS, is_udplite); |
984 | 988 | ||
985 | sock_recv_timestamp(msg, sk, skb); | 989 | sock_recv_ts_and_drops(msg, sk, skb); |
986 | 990 | ||
987 | /* Copy the address. */ | 991 | /* Copy the address. */ |
988 | if (sin) { | 992 | if (sin) { |
@@ -1044,8 +1048,8 @@ void udp_lib_unhash(struct sock *sk) | |||
1044 | { | 1048 | { |
1045 | if (sk_hashed(sk)) { | 1049 | if (sk_hashed(sk)) { |
1046 | struct udp_table *udptable = sk->sk_prot->h.udp_table; | 1050 | struct udp_table *udptable = sk->sk_prot->h.udp_table; |
1047 | unsigned int hash = udp_hashfn(sock_net(sk), sk->sk_hash); | 1051 | struct udp_hslot *hslot = udp_hashslot(udptable, sock_net(sk), |
1048 | struct udp_hslot *hslot = &udptable->hash[hash]; | 1052 | sk->sk_hash); |
1049 | 1053 | ||
1050 | spin_lock_bh(&hslot->lock); | 1054 | spin_lock_bh(&hslot->lock); |
1051 | if (sk_nulls_del_node_init_rcu(sk)) { | 1055 | if (sk_nulls_del_node_init_rcu(sk)) { |
@@ -1200,7 +1204,7 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb, | |||
1200 | struct udp_table *udptable) | 1204 | struct udp_table *udptable) |
1201 | { | 1205 | { |
1202 | struct sock *sk; | 1206 | struct sock *sk; |
1203 | struct udp_hslot *hslot = &udptable->hash[udp_hashfn(net, ntohs(uh->dest))]; | 1207 | struct udp_hslot *hslot = udp_hashslot(udptable, net, ntohs(uh->dest)); |
1204 | int dif; | 1208 | int dif; |
1205 | 1209 | ||
1206 | spin_lock(&hslot->lock); | 1210 | spin_lock(&hslot->lock); |
@@ -1622,9 +1626,14 @@ static struct sock *udp_get_first(struct seq_file *seq, int start) | |||
1622 | struct udp_iter_state *state = seq->private; | 1626 | struct udp_iter_state *state = seq->private; |
1623 | struct net *net = seq_file_net(seq); | 1627 | struct net *net = seq_file_net(seq); |
1624 | 1628 | ||
1625 | for (state->bucket = start; state->bucket < UDP_HTABLE_SIZE; ++state->bucket) { | 1629 | for (state->bucket = start; state->bucket <= state->udp_table->mask; |
1630 | ++state->bucket) { | ||
1626 | struct hlist_nulls_node *node; | 1631 | struct hlist_nulls_node *node; |
1627 | struct udp_hslot *hslot = &state->udp_table->hash[state->bucket]; | 1632 | struct udp_hslot *hslot = &state->udp_table->hash[state->bucket]; |
1633 | |||
1634 | if (hlist_nulls_empty(&hslot->head)) | ||
1635 | continue; | ||
1636 | |||
1628 | spin_lock_bh(&hslot->lock); | 1637 | spin_lock_bh(&hslot->lock); |
1629 | sk_nulls_for_each(sk, node, &hslot->head) { | 1638 | sk_nulls_for_each(sk, node, &hslot->head) { |
1630 | if (!net_eq(sock_net(sk), net)) | 1639 | if (!net_eq(sock_net(sk), net)) |
@@ -1649,7 +1658,7 @@ static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk) | |||
1649 | } while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != state->family)); | 1658 | } while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != state->family)); |
1650 | 1659 | ||
1651 | if (!sk) { | 1660 | if (!sk) { |
1652 | if (state->bucket < UDP_HTABLE_SIZE) | 1661 | if (state->bucket <= state->udp_table->mask) |
1653 | spin_unlock_bh(&state->udp_table->hash[state->bucket].lock); | 1662 | spin_unlock_bh(&state->udp_table->hash[state->bucket].lock); |
1654 | return udp_get_first(seq, state->bucket + 1); | 1663 | return udp_get_first(seq, state->bucket + 1); |
1655 | } | 1664 | } |
@@ -1669,7 +1678,7 @@ static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos) | |||
1669 | static void *udp_seq_start(struct seq_file *seq, loff_t *pos) | 1678 | static void *udp_seq_start(struct seq_file *seq, loff_t *pos) |
1670 | { | 1679 | { |
1671 | struct udp_iter_state *state = seq->private; | 1680 | struct udp_iter_state *state = seq->private; |
1672 | state->bucket = UDP_HTABLE_SIZE; | 1681 | state->bucket = MAX_UDP_PORTS; |
1673 | 1682 | ||
1674 | return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN; | 1683 | return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN; |
1675 | } | 1684 | } |
@@ -1691,7 +1700,7 @@ static void udp_seq_stop(struct seq_file *seq, void *v) | |||
1691 | { | 1700 | { |
1692 | struct udp_iter_state *state = seq->private; | 1701 | struct udp_iter_state *state = seq->private; |
1693 | 1702 | ||
1694 | if (state->bucket < UDP_HTABLE_SIZE) | 1703 | if (state->bucket <= state->udp_table->mask) |
1695 | spin_unlock_bh(&state->udp_table->hash[state->bucket].lock); | 1704 | spin_unlock_bh(&state->udp_table->hash[state->bucket].lock); |
1696 | } | 1705 | } |
1697 | 1706 | ||
@@ -1751,7 +1760,7 @@ static void udp4_format_sock(struct sock *sp, struct seq_file *f, | |||
1751 | __u16 destp = ntohs(inet->dport); | 1760 | __u16 destp = ntohs(inet->dport); |
1752 | __u16 srcp = ntohs(inet->sport); | 1761 | __u16 srcp = ntohs(inet->sport); |
1753 | 1762 | ||
1754 | seq_printf(f, "%4d: %08X:%04X %08X:%04X" | 1763 | seq_printf(f, "%5d: %08X:%04X %08X:%04X" |
1755 | " %02X %08X:%08X %02X:%08lX %08X %5d %8d %lu %d %p %d%n", | 1764 | " %02X %08X:%08X %02X:%08lX %08X %5d %8d %lu %d %p %d%n", |
1756 | bucket, src, srcp, dest, destp, sp->sk_state, | 1765 | bucket, src, srcp, dest, destp, sp->sk_state, |
1757 | sk_wmem_alloc_get(sp), | 1766 | sk_wmem_alloc_get(sp), |
@@ -1817,11 +1826,43 @@ void udp4_proc_exit(void) | |||
1817 | } | 1826 | } |
1818 | #endif /* CONFIG_PROC_FS */ | 1827 | #endif /* CONFIG_PROC_FS */ |
1819 | 1828 | ||
1820 | void __init udp_table_init(struct udp_table *table) | 1829 | static __initdata unsigned long uhash_entries; |
1830 | static int __init set_uhash_entries(char *str) | ||
1821 | { | 1831 | { |
1822 | int i; | 1832 | if (!str) |
1833 | return 0; | ||
1834 | uhash_entries = simple_strtoul(str, &str, 0); | ||
1835 | if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN) | ||
1836 | uhash_entries = UDP_HTABLE_SIZE_MIN; | ||
1837 | return 1; | ||
1838 | } | ||
1839 | __setup("uhash_entries=", set_uhash_entries); | ||
1823 | 1840 | ||
1824 | for (i = 0; i < UDP_HTABLE_SIZE; i++) { | 1841 | void __init udp_table_init(struct udp_table *table, const char *name) |
1842 | { | ||
1843 | unsigned int i; | ||
1844 | |||
1845 | if (!CONFIG_BASE_SMALL) | ||
1846 | table->hash = alloc_large_system_hash(name, | ||
1847 | sizeof(struct udp_hslot), | ||
1848 | uhash_entries, | ||
1849 | 21, /* one slot per 2 MB */ | ||
1850 | 0, | ||
1851 | &table->log, | ||
1852 | &table->mask, | ||
1853 | 64 * 1024); | ||
1854 | /* | ||
1855 | * Make sure hash table has the minimum size | ||
1856 | */ | ||
1857 | if (CONFIG_BASE_SMALL || table->mask < UDP_HTABLE_SIZE_MIN - 1) { | ||
1858 | table->hash = kmalloc(UDP_HTABLE_SIZE_MIN * | ||
1859 | sizeof(struct udp_hslot), GFP_KERNEL); | ||
1860 | if (!table->hash) | ||
1861 | panic(name); | ||
1862 | table->log = ilog2(UDP_HTABLE_SIZE_MIN); | ||
1863 | table->mask = UDP_HTABLE_SIZE_MIN - 1; | ||
1864 | } | ||
1865 | for (i = 0; i <= table->mask; i++) { | ||
1825 | INIT_HLIST_NULLS_HEAD(&table->hash[i].head, i); | 1866 | INIT_HLIST_NULLS_HEAD(&table->hash[i].head, i); |
1826 | spin_lock_init(&table->hash[i].lock); | 1867 | spin_lock_init(&table->hash[i].lock); |
1827 | } | 1868 | } |
@@ -1831,7 +1872,7 @@ void __init udp_init(void) | |||
1831 | { | 1872 | { |
1832 | unsigned long nr_pages, limit; | 1873 | unsigned long nr_pages, limit; |
1833 | 1874 | ||
1834 | udp_table_init(&udp_table); | 1875 | udp_table_init(&udp_table, "UDP"); |
1835 | /* Set the pressure threshold up by the same strategy of TCP. It is a | 1876 | /* Set the pressure threshold up by the same strategy of TCP. It is a |
1836 | * fraction of global memory that is up to 1/2 at 256 MB, decreasing | 1877 | * fraction of global memory that is up to 1/2 at 256 MB, decreasing |
1837 | * toward zero with the amount of memory, with a floor of 128 pages. | 1878 | * toward zero with the amount of memory, with a floor of 128 pages. |