net_sched: gen_estimator: complete rewrite of rate estimators

1) Old code was hard to maintain, due to complex lock chains.
   (We probably will be able to remove some kfree_rcu() in callers)

2) Using a single timer to update all estimators does not scale.

3) Code was buggy on 32bit kernel (WRITE_ONCE() on 64bit quantity
   is not supposed to work well)

In this rewrite :

- I removed the RB tree that had to be scanned in
  gen_estimator_active(). qdisc dumps should be much faster.

- Each estimator has its own timer.

- Estimations are maintained in net_rate_estimator structure,
  instead of dirtying the qdisc. Minor, but part of the simplification.

- Reading the estimator uses RCU and a seqcount to provide proper
  support for 32bit kernels.

- We reduce memory need when estimators are not used, since
  we store a pointer, instead of the bytes/packets counters.

- xt_rateest_mt() no longer has to grab a spinlock.
  (In the future, xt_rateest_tg() could be switched to per cpu counters)

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

4 files changed, 18 insertions, 13 deletions

diff --git a/include/net/act_api.h b/include/net/act_api.h
index 9dddf77a69cc..1d716449209e 100644
--- a/include/net/act_api.h
+++ b/include/net/act_api.h
@@ -36,7 +36,7 @@ struct tc_action {
         struct tcf_t                    tcfa_tm;
         struct gnet_stats_basic_packed  tcfa_bstats;
         struct gnet_stats_queue         tcfa_qstats;
-        struct gnet_stats_rate_est64    tcfa_rate_est;
+        struct net_rate_estimator __rcu *tcfa_rate_est;
         spinlock_t                      tcfa_lock;
         struct rcu_head                 tcfa_rcu;
         struct gnet_stats_basic_cpu __percpu *cpu_bstats;
diff --git a/include/net/gen_stats.h b/include/net/gen_stats.h
index 231e121cc7d9..8b7aa370e7a4 100644
--- a/include/net/gen_stats.h
+++ b/include/net/gen_stats.h
@@ -11,6 +11,8 @@ struct gnet_stats_basic_cpu {
         struct u64_stats_sync syncp;
 };
 
+struct net_rate_estimator;
+
 struct gnet_dump {
         spinlock_t *      lock;
         struct sk_buff *  skb;
@@ -42,8 +44,7 @@ void __gnet_stats_copy_basic(const seqcount_t *running,
                              struct gnet_stats_basic_cpu __percpu *cpu,
                              struct gnet_stats_basic_packed *b);
 int gnet_stats_copy_rate_est(struct gnet_dump *d,
-                             const struct gnet_stats_basic_packed *b,
-                             struct gnet_stats_rate_est64 *r);
+                             struct net_rate_estimator __rcu **ptr);
 int gnet_stats_copy_queue(struct gnet_dump *d,
                           struct gnet_stats_queue __percpu *cpu_q,
                           struct gnet_stats_queue *q, __u32 qlen);
@@ -53,16 +54,16 @@ int gnet_stats_finish_copy(struct gnet_dump *d);
 
 int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
                       struct gnet_stats_basic_cpu __percpu *cpu_bstats,
-                      struct gnet_stats_rate_est64 *rate_est,
+                      struct net_rate_estimator __rcu **rate_est,
                       spinlock_t *stats_lock,
                       seqcount_t *running, struct nlattr *opt);
-void gen_kill_estimator(struct gnet_stats_basic_packed *bstats,
-                        struct gnet_stats_rate_est64 *rate_est);
+void gen_kill_estimator(struct net_rate_estimator __rcu **ptr);
 int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
                           struct gnet_stats_basic_cpu __percpu *cpu_bstats,
-                          struct gnet_stats_rate_est64 *rate_est,
+                          struct net_rate_estimator __rcu **ptr,
                           spinlock_t *stats_lock,
                           seqcount_t *running, struct nlattr *opt);
-bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats,
-                          const struct gnet_stats_rate_est64 *rate_est);
+bool gen_estimator_active(struct net_rate_estimator __rcu **ptr);
+bool gen_estimator_read(struct net_rate_estimator __rcu **ptr,
+                        struct gnet_stats_rate_est64 *sample);
 #endif
diff --git a/include/net/netfilter/xt_rateest.h b/include/net/netfilter/xt_rateest.h
index 79f45e19f31e..130e58361f99 100644
--- a/include/net/netfilter/xt_rateest.h
+++ b/include/net/netfilter/xt_rateest.h
@@ -1,19 +1,23 @@
 #ifndef _XT_RATEEST_H
 #define _XT_RATEEST_H
 
+#include <net/gen_stats.h>
+
 struct xt_rateest {
         /* keep lock and bstats on same cache line to speedup xt_rateest_tg() */
         struct gnet_stats_basic_packed  bstats;
         spinlock_t                      lock;
-        /* keep rstats and lock on same cache line to speedup xt_rateest_mt() */
-        struct gnet_stats_rate_est64    rstats;
+
 
         /* following fields not accessed in hot path */
+        unsigned int                    refcnt;
         struct hlist_node               list;
         char                            name[IFNAMSIZ];
-        unsigned int                    refcnt;
         struct gnet_estimator           params;
         struct rcu_head                 rcu;
+
+        /* keep this field far away to speedup xt_rateest_mt() */
+        struct net_rate_estimator __rcu *rate_est;
 };
 
 struct xt_rateest *xt_rateest_lookup(const char *name);
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index e6aa0a249672..498f81b229a4 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -76,7 +76,7 @@ struct Qdisc {
 
         struct netdev_queue     *dev_queue;
 
-        struct gnet_stats_rate_est64    rate_est;
+        struct net_rate_estimator __rcu *rate_est;
         struct gnet_stats_basic_cpu __percpu *cpu_bstats;
         struct gnet_stats_queue __percpu *cpu_qstats;