qdisc: validate skb without holding lock

Validation of skb can be pretty expensive :

GSO segmentation and/or checksum computations.

We can do this without holding qdisc lock, so that other cpus
can queue additional packets.

Trick is that requeued packets were already validated, so we carry
a boolean so that sch_direct_xmit() can validate a fresh skb list,
or directly use an old one.

Tested on 40Gb NIC (8 TX queues) and 200 concurrent flows, 48 threads
host.

Turning TSO on or off had no effect on throughput, only few more cpu
cycles. Lock contention on qdisc lock disappeared.

Same if disabling TX checksum offload.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 26 insertions, 3 deletions

diff --git a/net/core/dev.c b/net/core/dev.c
index e55c546717d4..1a90530f83ff 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2655,7 +2655,7 @@ struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, netdev_features_t featur
         return skb;
 }
 
-struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
+static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
 {
         netdev_features_t features;
 
@@ -2720,6 +2720,30 @@ out_null:
         return NULL;
 }
 
+struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
+{
+        struct sk_buff *next, *head = NULL, *tail;
+
+        while (skb) {
+                next = skb->next;
+                skb->next = NULL;
+                skb = validate_xmit_skb(skb, dev);
+                if (skb) {
+                        struct sk_buff *end = skb;
+
+                        while (end->next)
+                                end = end->next;
+                        if (!head)
+                                head = skb;
+                        else
+                                tail->next = skb;
+                        tail = end;
+                }
+                skb = next;
+        }
+        return head;
+}
+
 static void qdisc_pkt_len_init(struct sk_buff *skb)
 {
         const struct skb_shared_info *shinfo = skb_shinfo(skb);
@@ -2786,8 +2810,7 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
 
                 qdisc_bstats_update(q, skb);
 
-                skb = validate_xmit_skb(skb, dev);
-                if (skb && sch_direct_xmit(skb, q, dev, txq, root_lock)) {
+                if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
                         if (unlikely(contended)) {
                                 spin_unlock(&q->busylock);
                                 contended = false;