net_sched: RCU conversion of stab

This patch converts stab qdisc management to RCU, so that we can perform
the qdisc_calculate_pkt_len() call before getting qdisc lock.

This shortens the lock's held time in __dev_xmit_skb().

This permits more qdiscs to get TCQ_F_CAN_BYPASS status, avoiding lot of
cache misses and so reducing latencies.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
CC: Patrick McHardy <kaber@trash.net>
CC: Jesper Dangaard Brouer <hawk@diku.dk>
CC: Jarek Poplawski <jarkao2@gmail.com>
CC: Jamal Hadi Salim <hadi@cyberus.ca>
CC: Stephen Hemminger <shemminger@vyatta.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 15 insertions, 6 deletions

diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index f6345f55041c..d531baa2506a 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -36,6 +36,7 @@ enum qdisc___state_t {
 };
 
 struct qdisc_size_table {
+        struct rcu_head         rcu;
         struct list_head        list;
         struct tc_sizespec      szopts;
         int                     refcnt;
@@ -53,7 +54,7 @@ struct Qdisc {
 #define TCQ_F_WARN_NONWC        (1 << 16)
         int                     padded;
         struct Qdisc_ops        *ops;
-        struct qdisc_size_table *stab;
+        struct qdisc_size_table __rcu *stab;
         struct list_head        list;
         u32                     handle;
         u32                     parent;
@@ -349,8 +350,8 @@ extern struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
                                  struct Qdisc_ops *ops);
 extern struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
                                        struct Qdisc_ops *ops, u32 parentid);
-extern void qdisc_calculate_pkt_len(struct sk_buff *skb,
-                                   struct qdisc_size_table *stab);
+extern void __qdisc_calculate_pkt_len(struct sk_buff *skb,
+                                      const struct qdisc_size_table *stab);
 extern void tcf_destroy(struct tcf_proto *tp);
 extern void tcf_destroy_chain(struct tcf_proto **fl);
 
@@ -429,12 +430,20 @@ enum net_xmit_qdisc_t {
 #define net_xmit_drop_count(e)  (1)
 #endif
 
-static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+static inline void qdisc_calculate_pkt_len(struct sk_buff *skb,
+                                           const struct Qdisc *sch)
 {
 #ifdef CONFIG_NET_SCHED
-        if (sch->stab)
-                qdisc_calculate_pkt_len(skb, sch->stab);
+        struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab);
+
+        if (stab)
+                __qdisc_calculate_pkt_len(skb, stab);
 #endif
+}
+
+static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+{
+        qdisc_calculate_pkt_len(skb, sch);
         return sch->enqueue(skb, sch);
 }