aboutsummaryrefslogtreecommitdiffstats
path: root/net
diff options
context:
space:
mode:
authorHerbert Xu <herbert@gondor.apana.org.au>2011-02-28 21:36:48 -0500
committerDavid S. Miller <davem@davemloft.net>2011-03-01 15:35:42 -0500
commit903ab86d195cca295379699299c5fc10beba31c7 (patch)
tree2b73fe727bf843bf0a0629dce044da61dc9cb166 /net
parentf6b9664f8b711cf4fd53e70aa0d21f72d5bf806c (diff)
udp: Add lockless transmit path
The UDP transmit path has been running under the socket lock for a long time because of the corking feature. This means that transmitting to the same socket in multiple threads does not scale at all. However, as most users don't actually use corking, the locking can be removed in the common case. This patch creates a lockless fast path where corking is not used. Please note that this does create a slight inaccuracy in the enforcement of socket send buffer limits. In particular, we may exceed the socket limit by up to (number of CPUs) * (packet size) because of the way the limit is computed. As the primary purpose of socket buffers is to indicate congestion, this should not be a great problem for now. Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> Acked-by: Eric Dumazet <eric.dumazet@gmail.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'net')
-rw-r--r--net/ipv4/udp.c15
1 files changed, 14 insertions, 1 deletions
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 61c22ee7d4ba..8155d6eda376 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -802,6 +802,7 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
802 int err, is_udplite = IS_UDPLITE(sk); 802 int err, is_udplite = IS_UDPLITE(sk);
803 int corkreq = up->corkflag || msg->msg_flags&MSG_MORE; 803 int corkreq = up->corkflag || msg->msg_flags&MSG_MORE;
804 int (*getfrag)(void *, char *, int, int, int, struct sk_buff *); 804 int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
805 struct sk_buff *skb;
805 806
806 if (len > 0xFFFF) 807 if (len > 0xFFFF)
807 return -EMSGSIZE; 808 return -EMSGSIZE;
@@ -816,6 +817,8 @@ int udp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
816 ipc.opt = NULL; 817 ipc.opt = NULL;
817 ipc.tx_flags = 0; 818 ipc.tx_flags = 0;
818 819
820 getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
821
819 if (up->pending) { 822 if (up->pending) {
820 /* 823 /*
821 * There are pending frames. 824 * There are pending frames.
@@ -940,6 +943,17 @@ back_from_confirm:
940 if (!ipc.addr) 943 if (!ipc.addr)
941 daddr = ipc.addr = rt->rt_dst; 944 daddr = ipc.addr = rt->rt_dst;
942 945
946 /* Lockless fast path for the non-corking case. */
947 if (!corkreq) {
948 skb = ip_make_skb(sk, getfrag, msg->msg_iov, ulen,
949 sizeof(struct udphdr), &ipc, &rt,
950 msg->msg_flags);
951 err = PTR_ERR(skb);
952 if (skb && !IS_ERR(skb))
953 err = udp_send_skb(skb, daddr, dport);
954 goto out;
955 }
956
943 lock_sock(sk); 957 lock_sock(sk);
944 if (unlikely(up->pending)) { 958 if (unlikely(up->pending)) {
945 /* The socket is already corked while preparing it. */ 959 /* The socket is already corked while preparing it. */
@@ -961,7 +975,6 @@ back_from_confirm:
961 975
962do_append_data: 976do_append_data:
963 up->len += ulen; 977 up->len += ulen;
964 getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
965 err = ip_append_data(sk, getfrag, msg->msg_iov, ulen, 978 err = ip_append_data(sk, getfrag, msg->msg_iov, ulen,
966 sizeof(struct udphdr), &ipc, &rt, 979 sizeof(struct udphdr), &ipc, &rt,
967 corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags); 980 corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);