1 files changed, 89 insertions, 52 deletions

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 8157b17959ee..f87a8eb76f3b 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -189,7 +189,7 @@ static int udp_lib_lport_inuse2(struct net *net, __u16 num,
  *  @sk:          socket struct in question
  *  @snum:        port number to look up
  *  @saddr_comp:  AF-dependent comparison of bound local IP addresses
- *  @hash2_nulladdr: AF-dependant hash value in secondary hash chains,
+ *  @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
  *                   with NULL address
  */
 int udp_lib_get_port(struct sock *sk, unsigned short snum,
@@ -663,75 +663,72 @@ void udp_flush_pending_frames(struct sock *sk)
 EXPORT_SYMBOL(udp_flush_pending_frames);
 
 /**
- *      udp4_hwcsum_outgoing  -  handle outgoing HW checksumming
- *      @sk:    socket we are sending on
+ *      udp4_hwcsum  -  handle outgoing HW checksumming
  *      @skb:   sk_buff containing the filled-in UDP header
  *              (checksum field must be zeroed out)
+ *      @src:   source IP address
+ *      @dst:   destination IP address
  */
-static void udp4_hwcsum_outgoing(struct sock *sk, struct sk_buff *skb,
-                                 __be32 src, __be32 dst, int len)
+static void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
 {
-        unsigned int offset;
         struct udphdr *uh = udp_hdr(skb);
+        struct sk_buff *frags = skb_shinfo(skb)->frag_list;
+        int offset = skb_transport_offset(skb);
+        int len = skb->len - offset;
+        int hlen = len;
         __wsum csum = 0;
 
-        if (skb_queue_len(&sk->sk_write_queue) == 1) {
+        if (!frags) {
                 /*
                  * Only one fragment on the socket.
                  */
                 skb->csum_start = skb_transport_header(skb) - skb->head;
                 skb->csum_offset = offsetof(struct udphdr, check);
-                uh->check = ~csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, 0);
+                uh->check = ~csum_tcpudp_magic(src, dst, len,
+                                               IPPROTO_UDP, 0);
         } else {
                 /*
                  * HW-checksum won't work as there are two or more
                  * fragments on the socket so that all csums of sk_buffs
                  * should be together
                  */
-                offset = skb_transport_offset(skb);
-                skb->csum = skb_checksum(skb, offset, skb->len - offset, 0);
+                do {
+                        csum = csum_add(csum, frags->csum);
+                        hlen -= frags->len;
+                } while ((frags = frags->next));
 
+                csum = skb_checksum(skb, offset, hlen, csum);
                 skb->ip_summed = CHECKSUM_NONE;
 
-                skb_queue_walk(&sk->sk_write_queue, skb) {
-                        csum = csum_add(csum, skb->csum);
-                }
-
                 uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
                 if (uh->check == 0)
                         uh->check = CSUM_MANGLED_0;
         }
 }
 
-/*
- * Push out all pending data as one UDP datagram. Socket is locked.
- */
-static int udp_push_pending_frames(struct sock *sk)
+static int udp_send_skb(struct sk_buff *skb, __be32 daddr, __be32 dport)
 {
-        struct udp_sock  *up = udp_sk(sk);
+        struct sock *sk = skb->sk;
         struct inet_sock *inet = inet_sk(sk);
-        struct flowi *fl = &inet->cork.fl;
-        struct sk_buff *skb;
         struct udphdr *uh;
+        struct rtable *rt = (struct rtable *)skb_dst(skb);
         int err = 0;
         int is_udplite = IS_UDPLITE(sk);
+        int offset = skb_transport_offset(skb);
+        int len = skb->len - offset;
         __wsum csum = 0;
 
-        /* Grab the skbuff where UDP header space exists. */
-        if ((skb = skb_peek(&sk->sk_write_queue)) == NULL)
-                goto out;
-
         /*
          * Create a UDP header
          */
         uh = udp_hdr(skb);
-        uh->source = fl->fl_ip_sport;
-        uh->dest = fl->fl_ip_dport;
-        uh->len = htons(up->len);
+        uh->source = inet->inet_sport;
+        uh->dest = dport;
+        uh->len = htons(len);
         uh->check = 0;
 
         if (is_udplite)                                  /*     UDP-Lite      */
-                csum  = udplite_csum_outgoing(sk, skb);
+                csum = udplite_csum(skb);
 
         else if (sk->sk_no_check == UDP_CSUM_NOXMIT) {   /* UDP csum disabled */
 
@@ -740,20 +737,20 @@ static int udp_push_pending_frames(struct sock *sk)
 
         } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
 
-                udp4_hwcsum_outgoing(sk, skb, fl->fl4_src, fl->fl4_dst, up->len);
+                udp4_hwcsum(skb, rt->rt_src, daddr);
                 goto send;
 
-        } else                                           /*   `normal' UDP    */
-                csum = udp_csum_outgoing(sk, skb);
+        } else
+                csum = udp_csum(skb);
 
         /* add protocol-dependent pseudo-header */
-        uh->check = csum_tcpudp_magic(fl->fl4_src, fl->fl4_dst, up->len,
+        uh->check = csum_tcpudp_magic(rt->rt_src, daddr, len,
                                       sk->sk_protocol, csum);
         if (uh->check == 0)
                 uh->check = CSUM_MANGLED_0;
 
 send:
-        err = ip_push_pending_frames(sk);
+        err = ip_send_skb(skb);
         if (err) {
                 if (err == -ENOBUFS && !inet->recverr) {
                         UDP_INC_STATS_USER(sock_net(sk),
@@ -763,6 +760,26 @@ send:
         } else
                 UDP_INC_STATS_USER(sock_net(sk),
                                    UDP_MIB_OUTDATAGRAMS, is_udplite);
+        return err;
+}
+
+/*
+ * Push out all pending data as one UDP datagram. Socket is locked.
+ */
+static int udp_push_pending_frames(struct sock *sk)
+{
+        struct udp_sock  *up = udp_sk(sk);
+        struct inet_sock *inet = inet_sk(sk);
+        struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
+        struct sk_buff *skb;
+        int err = 0;
+
+        skb = ip_finish_skb(sk);
+        if (!skb)
+                goto out;
+
+        err = udp_send_skb(skb, fl4->daddr, fl4->fl4_dport);
+
 out:
         up->len = 0;
         up->pending = 0;
@@ -774,6 +791,7 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
 {
         struct inet_sock *inet = inet_sk(sk);
         struct udp_sock *up = udp_sk(sk);
+        struct flowi4 *fl4;
         int ulen = len;
         struct ipcm_cookie ipc;
         struct rtable *rt = NULL;
@@ -785,6 +803,7 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
         int err, is_udplite = IS_UDPLITE(sk);
         int corkreq = up->corkflag || msg->msg_flags&MSG_MORE;
         int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
+        struct sk_buff *skb;
 
         if (len > 0xFFFF)
                 return -EMSGSIZE;
@@ -799,6 +818,8 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
         ipc.opt = NULL;
         ipc.tx_flags = 0;
 
+        getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
+
         if (up->pending) {
                 /*
                  * There are pending frames.
@@ -888,20 +909,25 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
                 rt = (struct rtable *)sk_dst_check(sk, 0);
 
         if (rt == NULL) {
-                struct flowi fl = { .oif = ipc.oif,
-                                    .mark = sk->sk_mark,
-                                    .fl4_dst = faddr,
-                                    .fl4_src = saddr,
-                                    .fl4_tos = tos,
-                                    .proto = sk->sk_protocol,
-                                    .flags = inet_sk_flowi_flags(sk),
-                                    .fl_ip_sport = inet->inet_sport,
-                                    .fl_ip_dport = dport };
+                struct flowi4 fl4 = {
+                        .flowi4_oif = ipc.oif,
+                        .flowi4_mark = sk->sk_mark,
+                        .daddr = faddr,
+                        .saddr = saddr,
+                        .flowi4_tos = tos,
+                        .flowi4_proto = sk->sk_protocol,
+                        .flowi4_flags = (inet_sk_flowi_flags(sk) |
+                                         FLOWI_FLAG_CAN_SLEEP),
+                        .fl4_sport = inet->inet_sport,
+                        .fl4_dport = dport,
+                };
                 struct net *net = sock_net(sk);
 
-                security_sk_classify_flow(sk, &fl);
-                err = ip_route_output_flow(net, &rt, &fl, sk, 1);
-                if (err) {
+                security_sk_classify_flow(sk, flowi4_to_flowi(&fl4));
+                rt = ip_route_output_flow(net, &fl4, sk);
+                if (IS_ERR(rt)) {
+                        err = PTR_ERR(rt);
+                        rt = NULL;
                         if (err == -ENETUNREACH)
                                 IP_INC_STATS_BH(net, IPSTATS_MIB_OUTNOROUTES);
                         goto out;
@@ -923,6 +949,17 @@ back_from_confirm:
         if (!ipc.addr)
                 daddr = ipc.addr = rt->rt_dst;
 
+        /* Lockless fast path for the non-corking case. */
+        if (!corkreq) {
+                skb = ip_make_skb(sk, getfrag, msg->msg_iov, ulen,
+                                  sizeof(struct udphdr), &ipc, &rt,
+                                  msg->msg_flags);
+                err = PTR_ERR(skb);
+                if (skb && !IS_ERR(skb))
+                        err = udp_send_skb(skb, daddr, dport);
+                goto out;
+        }
+
         lock_sock(sk);
         if (unlikely(up->pending)) {
                 /* The socket is already corked while preparing it. */
@@ -936,15 +973,15 @@ back_from_confirm:
         /*
          *      Now cork the socket to pend data.
          */
-        inet->cork.fl.fl4_dst = daddr;
-        inet->cork.fl.fl_ip_dport = dport;
-        inet->cork.fl.fl4_src = saddr;
-        inet->cork.fl.fl_ip_sport = inet->inet_sport;
+        fl4 = &inet->cork.fl.u.ip4;
+        fl4->daddr = daddr;
+        fl4->saddr = saddr;
+        fl4->fl4_dport = dport;
+        fl4->fl4_sport = inet->inet_sport;
         up->pending = AF_INET;
 
 do_append_data:
         up->len += ulen;
-        getfrag  =  is_udplite ?  udplite_getfrag : ip_generic_getfrag;
         err = ip_append_data(sk, getfrag, msg->msg_iov, ulen,
                         sizeof(struct udphdr), &ipc, &rt,
                         corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
@@ -2199,7 +2236,7 @@ int udp4_ufo_send_check(struct sk_buff *skb)
         return 0;
 }
 
-struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, int features)
+struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, u32 features)
 {
         struct sk_buff *segs = ERR_PTR(-EINVAL);
         unsigned int mss;