net_sched: add 64bit rate estimators

struct gnet_stats_rate_est contains u32 fields, so the bytes per second
field can wrap at 34360Mbit.

Add a new gnet_stats_rate_est64 structure to get 64bit bps/pps fields,
and switch the kernel to use this structure natively.

This structure is dumped to user space as a new attribute :

TCA_STATS_RATE_EST64

Old tc command will now display the capped bps (to 34360Mbit), instead
of wrapped values, and updated tc command will display correct
information.

Old tc command output, after patch :

eric:~# tc -s -d qd sh dev lo
qdisc pfifo 8001: root refcnt 2 limit 1000p
 Sent 80868245400 bytes 1978837 pkt (dropped 0, overlimits 0 requeues 0)
 rate 34360Mbit 189696pps backlog 0b 0p requeues 0

This patch carefully reorganizes "struct Qdisc" layout to get optimal
performance on SMP.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Ben Hutchings <bhutchings@solarflare.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

5 files changed, 25 insertions, 13 deletions

diff --git a/include/net/act_api.h b/include/net/act_api.h
index 06ef7e926a66..b8ffac7b6bab 100644
--- a/include/net/act_api.h
+++ b/include/net/act_api.h
@@ -18,7 +18,7 @@ struct tcf_common {
         struct tcf_t                    tcfc_tm;
         struct gnet_stats_basic_packed  tcfc_bstats;
         struct gnet_stats_queue         tcfc_qstats;
-        struct gnet_stats_rate_est      tcfc_rate_est;
+        struct gnet_stats_rate_est64    tcfc_rate_est;
         spinlock_t                      tcfc_lock;
         struct rcu_head                 tcfc_rcu;
 };
diff --git a/include/net/gen_stats.h b/include/net/gen_stats.h
index a79b6cfb02a8..cf8439ba4d11 100644
--- a/include/net/gen_stats.h
+++ b/include/net/gen_stats.h
@@ -30,7 +30,7 @@ extern int gnet_stats_copy_basic(struct gnet_dump *d,
                                  struct gnet_stats_basic_packed *b);
 extern int gnet_stats_copy_rate_est(struct gnet_dump *d,
                                     const struct gnet_stats_basic_packed *b,
-                                    struct gnet_stats_rate_est *r);
+                                    struct gnet_stats_rate_est64 *r);
 extern int gnet_stats_copy_queue(struct gnet_dump *d,
                                  struct gnet_stats_queue *q);
 extern int gnet_stats_copy_app(struct gnet_dump *d, void *st, int len);
@@ -38,13 +38,13 @@ extern int gnet_stats_copy_app(struct gnet_dump *d, void *st, int len);
 extern int gnet_stats_finish_copy(struct gnet_dump *d);
 
 extern int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
-                             struct gnet_stats_rate_est *rate_est,
+                             struct gnet_stats_rate_est64 *rate_est,
                              spinlock_t *stats_lock, struct nlattr *opt);
 extern void gen_kill_estimator(struct gnet_stats_basic_packed *bstats,
-                               struct gnet_stats_rate_est *rate_est);
+                               struct gnet_stats_rate_est64 *rate_est);
 extern int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
-                                 struct gnet_stats_rate_est *rate_est,
+                                 struct gnet_stats_rate_est64 *rate_est,
                                  spinlock_t *stats_lock, struct nlattr *opt);
 extern bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats,
-                                 const struct gnet_stats_rate_est *rate_est);
+                                 const struct gnet_stats_rate_est64 *rate_est);
 #endif
diff --git a/include/net/netfilter/xt_rateest.h b/include/net/netfilter/xt_rateest.h
index 5a2978d1cb22..495c71f66e7e 100644
--- a/include/net/netfilter/xt_rateest.h
+++ b/include/net/netfilter/xt_rateest.h
@@ -6,7 +6,7 @@ struct xt_rateest {
         struct gnet_stats_basic_packed  bstats;
         spinlock_t                      lock;
         /* keep rstats and lock on same cache line to speedup xt_rateest_mt() */
-        struct gnet_stats_rate_est      rstats;
+        struct gnet_stats_rate_est64    rstats;
 
         /* following fields not accessed in hot path */
         struct hlist_node               list;
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index e7f4e21cc3e1..df5676029827 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -58,14 +58,12 @@ struct Qdisc {
                                       * multiqueue device.
                                       */
 #define TCQ_F_WARN_NONWC        (1 << 16)
-        int                     padded;
+        u32                     limit;
         const struct Qdisc_ops  *ops;
         struct qdisc_size_table __rcu *stab;
         struct list_head        list;
         u32                     handle;
         u32                     parent;
-        atomic_t                refcnt;
-        struct gnet_stats_rate_est      rate_est;
         int                     (*reshape_fail)(struct sk_buff *skb,
                                         struct Qdisc *q);
 
@@ -76,8 +74,9 @@ struct Qdisc {
          */
         struct Qdisc            *__parent;
         struct netdev_queue     *dev_queue;
-        struct Qdisc            *next_sched;
 
+        struct gnet_stats_rate_est64    rate_est;
+        struct Qdisc            *next_sched;
         struct sk_buff          *gso_skb;
         /*
          * For performance sake on SMP, we put highly modified fields at the end
@@ -88,8 +87,10 @@ struct Qdisc {
         unsigned int            __state;
         struct gnet_stats_queue qstats;
         struct rcu_head         rcu_head;
-        spinlock_t              busylock;
-        u32                     limit;
+        int                     padded;
+        atomic_t                refcnt;
+
+        spinlock_t              busylock ____cacheline_aligned_in_smp;
 };
 
 static inline bool qdisc_is_running(const struct Qdisc *qdisc)
diff --git a/include/uapi/linux/gen_stats.h b/include/uapi/linux/gen_stats.h
index 552c8a0a12d1..6487317ea619 100644
--- a/include/uapi/linux/gen_stats.h
+++ b/include/uapi/linux/gen_stats.h
@@ -9,6 +9,7 @@ enum {
         TCA_STATS_RATE_EST,
         TCA_STATS_QUEUE,
         TCA_STATS_APP,
+        TCA_STATS_RATE_EST64,
         __TCA_STATS_MAX,
 };
 #define TCA_STATS_MAX (__TCA_STATS_MAX - 1)
@@ -38,6 +39,16 @@ struct gnet_stats_rate_est {
 };
 
 /**
+ * struct gnet_stats_rate_est64 - rate estimator
+ * @bps: current byte rate
+ * @pps: current packet rate
+ */
+struct gnet_stats_rate_est64 {
+        __u64   bps;
+        __u64   pps;
+};
+
+/**
  * struct gnet_stats_queue - queuing statistics
  * @qlen: queue length
  * @backlog: backlog size of queue