net: core: introduce mini_Qdisc and eliminate usage of tp->q for clsact fastpath

In sch_handle_egress and sch_handle_ingress tp->q is used only in order
to update stats. So stats and filter list are the only things that are
needed in clsact qdisc fastpath processing. Introduce new mini_Qdisc
struct to hold those items. Also, introduce a helper to swap the
mini_Qdisc structures in case filter list head changes.

This removes need for tp->q usage without added overhead.

Signed-off-by: Jiri Pirko <jiri@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

5 files changed, 109 insertions, 18 deletions

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 5e02f79b2110..7de7656550c2 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1559,6 +1559,8 @@ enum netdev_priv_flags {
  *
  *      @rx_handler:            handler for received packets
  *      @rx_handler_data:       XXX: need comments on this one
+ *      @miniq_ingress:         ingress/clsact qdisc specific data for
+ *                              ingress processing
  *      @ingress_queue:         XXX: need comments on this one
  *      @broadcast:             hw bcast address
  *
@@ -1576,7 +1578,8 @@ enum netdev_priv_flags {
  *      @tx_global_lock:        XXX: need comments on this one
  *
  *      @xps_maps:      XXX: need comments on this one
- *
+ *      @miniq_egress:          clsact qdisc specific data for
+ *                              egress processing
  *      @watchdog_timeo:        Represents the timeout that is used by
  *                              the watchdog (see dev_watchdog())
  *      @watchdog_timer:        List of timers
@@ -1795,7 +1798,7 @@ struct net_device {
         void __rcu              *rx_handler_data;
 
 #ifdef CONFIG_NET_CLS_ACT
-        struct tcf_proto __rcu  *ingress_cl_list;
+        struct mini_Qdisc __rcu *miniq_ingress;
 #endif
         struct netdev_queue __rcu *ingress_queue;
 #ifdef CONFIG_NETFILTER_INGRESS
@@ -1826,7 +1829,7 @@ struct net_device {
         struct xps_dev_maps __rcu *xps_maps;
 #endif
 #ifdef CONFIG_NET_CLS_ACT
-        struct tcf_proto __rcu  *egress_cl_list;
+        struct mini_Qdisc __rcu *miniq_egress;
 #endif
 
         /* These may be needed for future network-power-down code. */
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index f230269e0bfb..c64e62c9450a 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -904,4 +904,36 @@ static inline void psched_ratecfg_getrate(struct tc_ratespec *res,
         res->linklayer = (r->linklayer & TC_LINKLAYER_MASK);
 }
 
+/* Mini Qdisc serves for specific needs of ingress/clsact Qdisc.
+ * The fast path only needs to access filter list and to update stats
+ */
+struct mini_Qdisc {
+        struct tcf_proto *filter_list;
+        struct gnet_stats_basic_cpu __percpu *cpu_bstats;
+        struct gnet_stats_queue __percpu *cpu_qstats;
+        struct rcu_head rcu;
+};
+
+static inline void mini_qdisc_bstats_cpu_update(struct mini_Qdisc *miniq,
+                                                const struct sk_buff *skb)
+{
+        bstats_cpu_update(this_cpu_ptr(miniq->cpu_bstats), skb);
+}
+
+static inline void mini_qdisc_qstats_cpu_drop(struct mini_Qdisc *miniq)
+{
+        this_cpu_inc(miniq->cpu_qstats->drops);
+}
+
+struct mini_Qdisc_pair {
+        struct mini_Qdisc miniq1;
+        struct mini_Qdisc miniq2;
+        struct mini_Qdisc __rcu **p_miniq;
+};
+
+void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
+                          struct tcf_proto *tp_head);
+void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
+                          struct mini_Qdisc __rcu **p_miniq);
+
 #endif
diff --git a/net/core/dev.c b/net/core/dev.c
index 24ac9083bc13..1423cf4d695c 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3274,22 +3274,22 @@ EXPORT_SYMBOL(dev_loopback_xmit);
 static struct sk_buff *
 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
 {
-        struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
+        struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
         struct tcf_result cl_res;
 
-        if (!cl)
+        if (!miniq)
                 return skb;
 
         /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
-        qdisc_bstats_cpu_update(cl->q, skb);
+        mini_qdisc_bstats_cpu_update(miniq, skb);
 
-        switch (tcf_classify(skb, cl, &cl_res, false)) {
+        switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
         case TC_ACT_OK:
         case TC_ACT_RECLASSIFY:
                 skb->tc_index = TC_H_MIN(cl_res.classid);
                 break;
         case TC_ACT_SHOT:
-                qdisc_qstats_cpu_drop(cl->q);
+                mini_qdisc_qstats_cpu_drop(miniq);
                 *ret = NET_XMIT_DROP;
                 kfree_skb(skb);
                 return NULL;
@@ -4189,7 +4189,7 @@ sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
                    struct net_device *orig_dev)
 {
 #ifdef CONFIG_NET_CLS_ACT
-        struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
+        struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
         struct tcf_result cl_res;
 
         /* If there's at least one ingress present somewhere (so
@@ -4197,8 +4197,9 @@ sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
          * that are not configured with an ingress qdisc will bail
          * out here.
          */
-        if (!cl)
+        if (!miniq)
                 return skb;
+
         if (*pt_prev) {
                 *ret = deliver_skb(skb, *pt_prev, orig_dev);
                 *pt_prev = NULL;
@@ -4206,15 +4207,15 @@ sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
 
         qdisc_skb_cb(skb)->pkt_len = skb->len;
         skb->tc_at_ingress = 1;
-        qdisc_bstats_cpu_update(cl->q, skb);
+        mini_qdisc_bstats_cpu_update(miniq, skb);
 
-        switch (tcf_classify(skb, cl, &cl_res, false)) {
+        switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
         case TC_ACT_OK:
         case TC_ACT_RECLASSIFY:
                 skb->tc_index = TC_H_MIN(cl_res.classid);
                 break;
         case TC_ACT_SHOT:
-                qdisc_qstats_cpu_drop(cl->q);
+                mini_qdisc_qstats_cpu_drop(miniq);
                 kfree_skb(skb);
                 return NULL;
         case TC_ACT_STOLEN:
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index aa74aa42b5d7..3839cbbdc32b 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -1024,3 +1024,49 @@ void psched_ratecfg_precompute(struct psched_ratecfg *r,
         }
 }
 EXPORT_SYMBOL(psched_ratecfg_precompute);
+
+static void mini_qdisc_rcu_func(struct rcu_head *head)
+{
+}
+
+void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
+                          struct tcf_proto *tp_head)
+{
+        struct mini_Qdisc *miniq_old = rtnl_dereference(*miniqp->p_miniq);
+        struct mini_Qdisc *miniq;
+
+        if (!tp_head) {
+                RCU_INIT_POINTER(*miniqp->p_miniq, NULL);
+                return;
+        }
+
+        miniq = !miniq_old || miniq_old == &miniqp->miniq2 ?
+                &miniqp->miniq1 : &miniqp->miniq2;
+