1 files changed, 35 insertions, 121 deletions

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 7d5a8661df76..f57c0e4c2326 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -594,27 +594,6 @@ static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk,
         return true;
 }
 
-static inline struct sock *udp_v4_mcast_next(struct net *net, struct sock *sk,
-                                             __be16 loc_port, __be32 loc_addr,
-                                             __be16 rmt_port, __be32 rmt_addr,
-                                             int dif)
-{
-        struct hlist_nulls_node *node;
-        struct sock *s = sk;
-        unsigned short hnum = ntohs(loc_port);
-
-        sk_nulls_for_each_from(s, node) {
-                if (__udp_is_mcast_sock(net, s,
-                                        loc_port, loc_addr,
-                                        rmt_port, rmt_addr,
-                                        dif, hnum))
-                        goto found;
-        }
-        s = NULL;
-found:
-        return s;
-}
-
 /*
  * This routine is called by the ICMP module when it gets some
  * sort of error condition.  If err < 0 then the socket should
@@ -1588,7 +1567,7 @@ int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
                 goto csum_error;
 
 
-        if (sk_rcvqueues_full(sk, skb, sk->sk_rcvbuf)) {
+        if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
                 UDP_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,
                                  is_udplite);
                 goto drop;
@@ -1640,6 +1619,8 @@ static void flush_stack(struct sock **stack, unsigned int count,
 
                 if (skb1 && udp_queue_rcv_skb(sk, skb1) <= 0)
                         skb1 = NULL;
+
+                sock_put(sk);
         }
         if (unlikely(skb1))
                 kfree_skb(skb1);
@@ -1668,41 +1649,50 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
                                     struct udp_table *udptable)
 {
         struct sock *sk, *stack[256 / sizeof(struct sock *)];
-        struct udp_hslot *hslot = udp_hashslot(udptable, net, ntohs(uh->dest));
-        int dif;
-        unsigned int i, count = 0;
+        struct hlist_nulls_node *node;
+        unsigned short hnum = ntohs(uh->dest);
+        struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
+        int dif = skb->dev->ifindex;
+        unsigned int count = 0, offset = offsetof(typeof(*sk), sk_nulls_node);
+        unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
+
+        if (use_hash2) {
+                hash2_any = udp4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
+                            udp_table.mask;
+                hash2 = udp4_portaddr_hash(net, daddr, hnum) & udp_table.mask;
+start_lookup:
+                hslot = &udp_table.hash2[hash2];
+                offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
+        }
 
         spin_lock(&hslot->lock);
-        sk = sk_nulls_head(&hslot->head);
-        dif = skb->dev->ifindex;
-        sk = udp_v4_mcast_next(net, sk, uh->dest, daddr, uh->source, saddr, dif);
-        while (sk) {
-                stack[count++] = sk;
-                sk = udp_v4_mcast_next(net, sk_nulls_next(sk), uh->dest,
-                                       daddr, uh->source, saddr, dif);
-                if (unlikely(count == ARRAY_SIZE(stack))) {
-                        if (!sk)
-                                break;
-                        flush_stack(stack, count, skb, ~0);
-                        count = 0;
+        sk_nulls_for_each_entry_offset(sk, node, &hslot->head, offset) {
+                if (__udp_is_mcast_sock(net, sk,
+                                        uh->dest, daddr,
+                                        uh->source, saddr,
+                                        dif, hnum)) {
+                        if (unlikely(count == ARRAY_SIZE(stack))) {
+                                flush_stack(stack, count, skb, ~0);
+                                count = 0;
+                        }
+                        stack[count++] = sk;
+                        sock_hold(sk);
                 }
         }
-        /*
-         * before releasing chain lock, we must take a reference on sockets
-         */
-        for (i = 0; i < count; i++)
-                sock_hold(stack[i]);
 
         spin_unlock(&hslot->lock);
 
+        /* Also lookup *:port if we are using hash2 and haven't done so yet. */
+        if (use_hash2 && hash2 != hash2_any) {
+                hash2 = hash2_any;
+                goto start_lookup;
+        }
+
         /*
          * do the slow work with no lock held
          */
         if (count) {
                 flush_stack(stack, count, skb, count - 1);
-
-                for (i = 0; i < count; i++)
-                        sock_put(stack[i]);
         } else {
                 kfree_skb(skb);
         }
@@ -2526,79 +2516,3 @@ void __init udp_init(void)
         sysctl_udp_rmem_min = SK_MEM_QUANTUM;
         sysctl_udp_wmem_min = SK_MEM_QUANTUM;
 }
-
-struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,
-                                       netdev_features_t features)
-{
-        struct sk_buff *segs = ERR_PTR(-EINVAL);
-        u16 mac_offset = skb->mac_header;
-        int mac_len = skb->mac_len;
-        int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);
-        __be16 protocol = skb->protocol;
-        netdev_features_t enc_features;
-        int udp_offset, outer_hlen;
-        unsigned int oldlen;
-        bool need_csum;
-
-        oldlen = (u16)~skb->len;
-
-        if (unlikely(!pskb_may_pull(skb, tnl_hlen)))
-                goto out;
-
-        skb->encapsulation = 0;
-        __skb_pull(skb, tnl_hlen);
-        skb_reset_mac_header(skb);
-        skb_set_network_header(skb, skb_inner_network_offset(skb));
-        skb->mac_len = skb_inner_network_offset(skb);
-        skb->protocol = htons(ETH_P_TEB);
-
-        need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);
-        if (need_csum)
-                skb->encap_hdr_csum = 1;
-
-        /* segment inner packet. */
-        enc_features = skb->dev->hw_enc_features & netif_skb_features(skb);
-        segs = skb_mac_gso_segment(skb, enc_features);
-        if (!segs || IS_ERR(segs)) {
-                skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,
-                                     mac_len);
-                goto out;
-        }
-
-        outer_hlen = skb_tnl_header_len(skb);
-        udp_offset = outer_hlen - tnl_hlen;
-        skb = segs;
-        do {
-                struct udphdr *uh;
-                int len;
-
-                skb_reset_inner_headers(skb);
-                skb->encapsulation = 1;
-
-                skb->mac_len = mac_len;
-
-                skb_push(skb, outer_hlen);
-                skb_reset_mac_header(skb);
-                skb_set_network_header(skb, mac_len);
-                skb_set_transport_header(skb, udp_offset);
-                len = skb->len - udp_offset;
-                uh = udp_hdr(skb);
-                uh->len = htons(len);
-
-                if (need_csum) {
-                        __be32 delta = htonl(oldlen + len);
-
-                        uh->check = ~csum_fold((__force __wsum)
-                                               ((__force u32)uh->check +
-                                                (__force u32)delta));
-                        uh->check = gso_make_checksum(skb, ~uh->check);
-
-                        if (uh->check == 0)
-                                uh->check = CSUM_MANGLED_0;
-                }
-
-                skb->protocol = protocol;
-        } while ((skb = skb->next));
-out:
-        return segs;
-}