aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJohn Fastabend <john.fastabend@gmail.com>2014-09-28 14:52:56 -0400
committerDavid S. Miller <davem@davemloft.net>2014-09-30 01:02:26 -0400
commit22e0f8b9322cb1a48b1357e8f4ae6f5a9eca8cfa (patch)
tree2c9ef18dca9d9a441d92ea57cf7f7a292f4ceb3f
parent79cf79abce71eb7dbc40e2f3121048ca5405cb47 (diff)
net: sched: make bstats per cpu and estimator RCU safe
In order to run qdisc's without locking statistics and estimators need to be handled correctly. To resolve bstats make the statistics per cpu. And because this is only needed for qdiscs that are running without locks which is not the case for most qdiscs in the near future only create percpu stats when qdiscs set the TCQ_F_CPUSTATS flag. Next because estimators use the bstats to calculate packets per second and bytes per second the estimator code paths are updated to use the per cpu statistics. Signed-off-by: John Fastabend <john.r.fastabend@intel.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat
-rw-r--r--include/net/gen_stats.h11
-rw-r--r--include/net/sch_generic.h22
-rw-r--r--net/core/gen_estimator.c29
-rw-r--r--net/core/gen_stats.c53
-rw-r--r--net/netfilter/xt_RATEEST.c2
-rw-r--r--net/sched/act_api.c5
-rw-r--r--net/sched/act_police.c2
-rw-r--r--net/sched/sch_api.c29
-rw-r--r--net/sched/sch_atm.c2
-rw-r--r--net/sched/sch_cbq.c7
-rw-r--r--net/sched/sch_drr.c7
-rw-r--r--net/sched/sch_generic.c3
-rw-r--r--net/sched/sch_hfsc.c13
-rw-r--r--net/sched/sch_htb.c12
-rw-r--r--net/sched/sch_mq.c2
-rw-r--r--net/sched/sch_mqprio.c4
-rw-r--r--net/sched/sch_multiq.c2
-rw-r--r--net/sched/sch_prio.c2
-rw-r--r--net/sched/sch_qfq.c8
19 files changed, 164 insertions, 51 deletions
diff --git a/include/net/gen_stats.h b/include/net/gen_stats.h
index ea4271dceff0..ce3c1281f2a0 100644
--- a/include/net/gen_stats.h
+++ b/include/net/gen_stats.h
@@ -6,6 +6,11 @@
6#include <linux/rtnetlink.h> 6#include <linux/rtnetlink.h>
7#include <linux/pkt_sched.h> 7#include <linux/pkt_sched.h>
8 8
9struct gnet_stats_basic_cpu {
10 struct gnet_stats_basic_packed bstats;
11 struct u64_stats_sync syncp;
12};
13
9struct gnet_dump { 14struct gnet_dump {
10 spinlock_t * lock; 15 spinlock_t * lock;
11 struct sk_buff * skb; 16 struct sk_buff * skb;
@@ -27,7 +32,11 @@ int gnet_stats_start_copy_compat(struct sk_buff *skb, int type,
27 spinlock_t *lock, struct gnet_dump *d); 32 spinlock_t *lock, struct gnet_dump *d);
28 33
29int gnet_stats_copy_basic(struct gnet_dump *d, 34int gnet_stats_copy_basic(struct gnet_dump *d,
35 struct gnet_stats_basic_cpu __percpu *cpu,
30 struct gnet_stats_basic_packed *b); 36 struct gnet_stats_basic_packed *b);
37void __gnet_stats_copy_basic(struct gnet_stats_basic_packed *bstats,
38 struct gnet_stats_basic_cpu __percpu *cpu,
39 struct gnet_stats_basic_packed *b);
31int gnet_stats_copy_rate_est(struct gnet_dump *d, 40int gnet_stats_copy_rate_est(struct gnet_dump *d,
32 const struct gnet_stats_basic_packed *b, 41 const struct gnet_stats_basic_packed *b,
33 struct gnet_stats_rate_est64 *r); 42 struct gnet_stats_rate_est64 *r);
@@ -37,11 +46,13 @@ int gnet_stats_copy_app(struct gnet_dump *d, void *st, int len);
37int gnet_stats_finish_copy(struct gnet_dump *d); 46int gnet_stats_finish_copy(struct gnet_dump *d);
38 47
39int gen_new_estimator(struct gnet_stats_basic_packed *bstats, 48int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
49 struct gnet_stats_basic_cpu __percpu *cpu_bstats,
40 struct gnet_stats_rate_est64 *rate_est, 50 struct gnet_stats_rate_est64 *rate_est,
41 spinlock_t *stats_lock, struct nlattr *opt); 51 spinlock_t *stats_lock, struct nlattr *opt);
42void gen_kill_estimator(struct gnet_stats_basic_packed *bstats, 52void gen_kill_estimator(struct gnet_stats_basic_packed *bstats,
43 struct gnet_stats_rate_est64 *rate_est); 53 struct gnet_stats_rate_est64 *rate_est);
44int gen_replace_estimator(struct gnet_stats_basic_packed *bstats, 54int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
55 struct gnet_stats_basic_cpu __percpu *cpu_bstats,
45 struct gnet_stats_rate_est64 *rate_est, 56 struct gnet_stats_rate_est64 *rate_est,
46 spinlock_t *stats_lock, struct nlattr *opt); 57 spinlock_t *stats_lock, struct nlattr *opt);
47bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats, 58bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats,
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index e65b8e0752af..4b9351120fd8 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -6,6 +6,7 @@
6#include <linux/rcupdate.h> 6#include <linux/rcupdate.h>
7#include <linux/pkt_sched.h> 7#include <linux/pkt_sched.h>
8#include <linux/pkt_cls.h> 8#include <linux/pkt_cls.h>
9#include <linux/percpu.h>
9#include <net/gen_stats.h> 10#include <net/gen_stats.h>
10#include <net/rtnetlink.h> 11#include <net/rtnetlink.h>
11 12
@@ -58,6 +59,7 @@ struct Qdisc {
58 * multiqueue device. 59 * multiqueue device.
59 */ 60 */
60#define TCQ_F_WARN_NONWC (1 << 16) 61#define TCQ_F_WARN_NONWC (1 << 16)
62#define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */
61 u32 limit; 63 u32 limit;
62 const struct Qdisc_ops *ops; 64 const struct Qdisc_ops *ops;
63 struct qdisc_size_table __rcu *stab; 65 struct qdisc_size_table __rcu *stab;
@@ -83,7 +85,10 @@ struct Qdisc {
83 */ 85 */
84 unsigned long state; 86 unsigned long state;
85 struct sk_buff_head q; 87 struct sk_buff_head q;
86 struct gnet_stats_basic_packed bstats; 88 union {
89 struct gnet_stats_basic_packed bstats;
90 struct gnet_stats_basic_cpu __percpu *cpu_bstats;
91 } __packed;
87 unsigned int __state; 92 unsigned int __state;
88 struct gnet_stats_queue qstats; 93 struct gnet_stats_queue qstats;
89 struct rcu_head rcu_head; 94 struct rcu_head rcu_head;
@@ -487,6 +492,10 @@ static inline int qdisc_enqueue_root(struct sk_buff *skb, struct Qdisc *sch)
487 return qdisc_enqueue(skb, sch) & NET_XMIT_MASK; 492 return qdisc_enqueue(skb, sch) & NET_XMIT_MASK;
488} 493}
489 494
495static inline bool qdisc_is_percpu_stats(const struct Qdisc *q)
496{
497 return q->flags & TCQ_F_CPUSTATS;
498}
490 499
491static inline void bstats_update(struct gnet_stats_basic_packed *bstats, 500static inline void bstats_update(struct gnet_stats_basic_packed *bstats,
492 const struct sk_buff *skb) 501 const struct sk_buff *skb)
@@ -495,6 +504,17 @@ static inline void bstats_update(struct gnet_stats_basic_packed *bstats,
495 bstats->packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; 504 bstats->packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;
496} 505}
497 506
507static inline void qdisc_bstats_update_cpu(struct Qdisc *sch,
508 const struct sk_buff *skb)
509{
510 struct gnet_stats_basic_cpu *bstats =
511 this_cpu_ptr(sch->cpu_bstats);
512
513 u64_stats_update_begin(&bstats->syncp);
514 bstats_update(&bstats->bstats, skb);
515 u64_stats_update_end(&bstats->syncp);
516}
517
498static inline void qdisc_bstats_update(struct Qdisc *sch, 518static inline void qdisc_bstats_update(struct Qdisc *sch,
499 const struct sk_buff *skb) 519 const struct sk_buff *skb)
500{ 520{
diff --git a/net/core/gen_estimator.c b/net/core/gen_estimator.c
index 9d33dfffca19..9dfb88a933e7 100644
--- a/net/core/gen_estimator.c
+++ b/net/core/gen_estimator.c
@@ -91,6 +91,8 @@ struct gen_estimator
91 u32 avpps; 91 u32 avpps;
92 struct rcu_head e_rcu; 92 struct rcu_head e_rcu;
93 struct rb_node node; 93 struct rb_node node;
94 struct gnet_stats_basic_cpu __percpu *cpu_bstats;
95 struct rcu_head head;
94}; 96};
95 97
96struct gen_estimator_head 98struct gen_estimator_head
@@ -115,9 +117,8 @@ static void est_timer(unsigned long arg)
115 117
116 rcu_read_lock(); 118 rcu_read_lock();
117 list_for_each_entry_rcu(e, &elist[idx].list, list) { 119 list_for_each_entry_rcu(e, &elist[idx].list, list) {
118 u64 nbytes; 120 struct gnet_stats_basic_packed b = {0};
119 u64 brate; 121 u64 brate;
120 u32 npackets;
121 u32 rate; 122 u32 rate;
122 123
123 spin_lock(e->stats_lock); 124 spin_lock(e->stats_lock);
@@ -125,15 +126,15 @@ static void est_timer(unsigned long arg)
125 if (e->bstats == NULL) 126 if (e->bstats == NULL)
126 goto skip; 127 goto skip;
127 128
128 nbytes = e->bstats->bytes; 129 __gnet_stats_copy_basic(&b, e->cpu_bstats, e->bstats);
129 npackets = e->bstats->packets; 130
130 brate = (nbytes - e->last_bytes)<<(7 - idx); 131 brate = (b.bytes - e->last_bytes)<<(7 - idx);
131 e->last_bytes = nbytes; 132 e->last_bytes = b.bytes;
132 e->avbps += (brate >> e->ewma_log) - (e->avbps >> e->ewma_log); 133 e->avbps += (brate >> e->ewma_log) - (e->avbps >> e->ewma_log);
133 e->rate_est->bps = (e->avbps+0xF)>>5; 134 e->rate_est->bps = (e->avbps+0xF)>>5;
134 135
135 rate = (npackets - e->last_packets)<<(12 - idx); 136 rate = (b.packets - e->last_packets)<<(12 - idx);
136 e->last_packets = npackets; 137 e->last_packets = b.packets;
137 e->avpps += (rate >> e->ewma_log) - (e->avpps >> e->ewma_log); 138 e->avpps += (rate >> e->ewma_log) - (e->avpps >> e->ewma_log);
138 e->rate_est->pps = (e->avpps+0x1FF)>>10; 139 e->rate_est->pps = (e->avpps+0x1FF)>>10;
139skip: 140skip:
@@ -203,12 +204,14 @@ struct gen_estimator *gen_find_node(const struct gnet_stats_basic_packed *bstats
203 * 204 *
204 */ 205 */
205int gen_new_estimator(struct gnet_stats_basic_packed *bstats, 206int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
207 struct gnet_stats_basic_cpu __percpu *cpu_bstats,
206 struct gnet_stats_rate_est64 *rate_est, 208 struct gnet_stats_rate_est64 *rate_est,
207 spinlock_t *stats_lock, 209 spinlock_t *stats_lock,
208 struct nlattr *opt) 210 struct nlattr *opt)
209{ 211{
210 struct gen_estimator *est; 212 struct gen_estimator *est;
211 struct gnet_estimator *parm = nla_data(opt); 213 struct gnet_estimator *parm = nla_data(opt);
214 struct gnet_stats_basic_packed b = {0};
212 int idx; 215 int idx;
213 216
214 if (nla_len(opt) < sizeof(*parm)) 217 if (nla_len(opt) < sizeof(*parm))
@@ -221,15 +224,18 @@ int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
221 if (est == NULL) 224 if (est == NULL)
222 return -ENOBUFS; 225 return -ENOBUFS;
223 226
227 __gnet_stats_copy_basic(&b, cpu_bstats, bstats);
228
224 idx = parm->interval + 2; 229 idx = parm->interval + 2;
225 est->bstats = bstats; 230 est->bstats = bstats;
226 est->rate_est = rate_est; 231 est->rate_est = rate_est;
227 est->stats_lock = stats_lock; 232 est->stats_lock = stats_lock;
228 est->ewma_log = parm->ewma_log; 233 est->ewma_log = parm->ewma_log;
229 est->last_bytes = bstats->bytes; 234 est->last_bytes = b.bytes;
230 est->avbps = rate_est->bps<<5; 235 est->avbps = rate_est->bps<<5;
231 est->last_packets = bstats->packets; 236 est->last_packets = b.packets;
232 est->avpps = rate_est->pps<<10; 237 est->avpps = rate_est->pps<<10;
238 est->cpu_bstats = cpu_bstats;
233 239
234 spin_lock_bh(&est_tree_lock); 240 spin_lock_bh(&est_tree_lock);
235 if (!elist[idx].timer.function) { 241 if (!elist[idx].timer.function) {
@@ -290,11 +296,12 @@ EXPORT_SYMBOL(gen_kill_estimator);
290 * Returns 0 on success or a negative error code. 296 * Returns 0 on success or a negative error code.
291 */ 297 */
292int gen_replace_estimator(struct gnet_stats_basic_packed *bstats, 298int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
299 struct gnet_stats_basic_cpu __percpu *cpu_bstats,
293 struct gnet_stats_rate_est64 *rate_est, 300 struct gnet_stats_rate_est64 *rate_est,
294 spinlock_t *stats_lock, struct nlattr *opt) 301 spinlock_t *stats_lock, struct nlattr *opt)
295{ 302{
296 gen_kill_estimator(bstats, rate_est); 303 gen_kill_estimator(bstats, rate_est);
297 return gen_new_estimator(bstats, rate_est, stats_lock, opt); 304 return gen_new_estimator(bstats, cpu_bstats, rate_est, stats_lock, opt);
298} 305}
299EXPORT_SYMBOL(gen_replace_estimator); 306EXPORT_SYMBOL(gen_replace_estimator);
300 307
diff --git a/net/core/gen_stats.c b/net/core/gen_stats.c
index 2ddbce4cce14..5ff8e80fe0bb 100644
--- a/net/core/gen_stats.c
+++ b/net/core/gen_stats.c
@@ -97,6 +97,43 @@ gnet_stats_start_copy(struct sk_buff *skb, int type, spinlock_t *lock,
97} 97}
98EXPORT_SYMBOL(gnet_stats_start_copy); 98EXPORT_SYMBOL(gnet_stats_start_copy);
99 99
100static void
101__gnet_stats_copy_basic_cpu(struct gnet_stats_basic_packed *bstats,
102 struct gnet_stats_basic_cpu __percpu *cpu)
103{
104 int i;
105
106 for_each_possible_cpu(i) {
107 struct gnet_stats_basic_cpu *bcpu = per_cpu_ptr(cpu, i);
108 unsigned int start;
109 __u64 bytes;
110 __u32 packets;
111
112 do {
113 start = u64_stats_fetch_begin_irq(&bcpu->syncp);
114 bytes = bcpu->bstats.bytes;
115 packets = bcpu->bstats.packets;
116 } while (u64_stats_fetch_retry_irq(&bcpu->syncp, start));
117
118 bstats->bytes += bcpu->bstats.bytes;
119 bstats->packets += bcpu->bstats.packets;
120 }
121}
122
123void
124__gnet_stats_copy_basic(struct gnet_stats_basic_packed *bstats,
125 struct gnet_stats_basic_cpu __percpu *cpu,
126 struct gnet_stats_basic_packed *b)
127{
128 if (cpu) {
129 __gnet_stats_copy_basic_cpu(bstats, cpu);
130 } else {
131 bstats->bytes = b->bytes;
132 bstats->packets = b->packets;
133 }
134}
135EXPORT_SYMBOL(__gnet_stats_copy_basic);
136
100/** 137/**
101 * gnet_stats_copy_basic - copy basic statistics into statistic TLV 138 * gnet_stats_copy_basic - copy basic statistics into statistic TLV
102 * @d: dumping handle 139 * @d: dumping handle
@@ -109,19 +146,25 @@ EXPORT_SYMBOL(gnet_stats_start_copy);
109 * if the room in the socket buffer was not sufficient. 146 * if the room in the socket buffer was not sufficient.
110 */ 147 */
111int 148int
112gnet_stats_copy_basic(struct gnet_dump *d, struct gnet_stats_basic_packed *b) 149gnet_stats_copy_basic(struct gnet_dump *d,
150 struct gnet_stats_basic_cpu __percpu *cpu,
151 struct gnet_stats_basic_packed *b)
113{ 152{
153 struct gnet_stats_basic_packed bstats = {0};
154
155 __gnet_stats_copy_basic(&bstats, cpu, b);
156
114 if (d->compat_tc_stats) { 157 if (d->compat_tc_stats) {
115 d->tc_stats.bytes = b->bytes; 158 d->tc_stats.bytes = bstats.bytes;
116 d->tc_stats.packets = b->packets; 159 d->tc_stats.packets = bstats.packets;
117 } 160 }
118 161
119 if (d->tail) { 162 if (d->tail) {
120 struct gnet_stats_basic sb; 163 struct gnet_stats_basic sb;
121 164
122 memset(&sb, 0, sizeof(sb)); 165 memset(&sb, 0, sizeof(sb));
123 sb.bytes = b->bytes; 166 sb.bytes = bstats.bytes;
124 sb.packets = b->packets; 167 sb.packets = bstats.packets;
125 return gnet_stats_copy(d, TCA_STATS_BASIC, &sb, sizeof(sb)); 168 return gnet_stats_copy(d, TCA_STATS_BASIC, &sb, sizeof(sb));
126 } 169 }
127 return 0; 170 return 0;
diff --git a/net/netfilter/xt_RATEEST.c b/net/netfilter/xt_RATEEST.c
index 370adf622cef..604df6fae6fc 100644
--- a/net/netfilter/xt_RATEEST.c
+++ b/net/netfilter/xt_RATEEST.c
@@ -136,7 +136,7 @@ static int xt_rateest_tg_checkentry(const struct xt_tgchk_param *par)
136 cfg.est.interval = info->interval; 136 cfg.est.interval = info->interval;
137 cfg.est.ewma_log = info->ewma_log; 137 cfg.est.ewma_log = info->ewma_log;
138 138
139 ret = gen_new_estimator(&est->bstats, &est->rstats, 139 ret = gen_new_estimator(&est->bstats, NULL, &est->rstats,
140 &est->lock, &cfg.opt); 140 &est->lock, &cfg.opt);
141 if (ret < 0) 141 if (ret < 0)
142 goto err2; 142 goto err2;
diff --git a/net/sched/act_api.c b/net/sched/act_api.c
index 648778aef1a2..eca4cf9ece2f 100644
--- a/net/sched/act_api.c
+++ b/net/sched/act_api.c
@@ -252,7 +252,8 @@ int tcf_hash_create(u32 index, struct nlattr *est, struct tc_action *a,
252 p->tcfc_tm.install = jiffies; 252 p->tcfc_tm.install = jiffies;
253 p->tcfc_tm.lastuse = jiffies; 253 p->tcfc_tm.lastuse = jiffies;
254 if (est) { 254 if (est) {
255 int err = gen_new_estimator(&p->tcfc_bstats, &p->tcfc_rate_est, 255 int err = gen_new_estimator(&p->tcfc_bstats, NULL,
256 &p->tcfc_rate_est,
256 &p->tcfc_lock, est); 257 &p->tcfc_lock, est);
257 if (err) { 258 if (err) {
258 kfree(p); 259 kfree(p);
@@ -619,7 +620,7 @@ int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *a,
619 if (err < 0) 620 if (err < 0)
620 goto errout; 621 goto errout;
621 622
622 if (gnet_stats_copy_basic(&d, &p->tcfc_bstats) < 0 || 623 if (gnet_stats_copy_basic(&d, NULL, &p->tcfc_bstats) < 0 ||
623 gnet_stats_copy_rate_est(&d, &p->tcfc_bstats, 624 gnet_stats_copy_rate_est(&d, &p->tcfc_bstats,
624 &p->tcfc_rate_est) < 0 || 625 &p->tcfc_rate_est) < 0 ||
625 gnet_stats_copy_queue(&d, &p->tcfc_qstats) < 0) 626 gnet_stats_copy_queue(&d, &p->tcfc_qstats) < 0)
diff --git a/net/sched/act_police.c b/net/sched/act_police.c
index f32bcb094915..69791ca77a05 100644
--- a/net/sched/act_police.c
+++ b/net/sched/act_police.c
@@ -178,7 +178,7 @@ override:
178 178
179 spin_lock_bh(&police->tcf_lock); 179 spin_lock_bh(&police->tcf_lock);
180 if (est) { 180 if (est) {
181 err = gen_replace_estimator(&police->tcf_bstats, 181 err = gen_replace_estimator(&police->tcf_bstats, NULL,
182 &police->tcf_rate_est, 182 &police->tcf_rate_est,
183 &police->tcf_lock, est); 183 &police->tcf_lock, est);
184 if (err) 184 if (err)
diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
index 15e7beee266c..a95e3b48fa51 100644
--- a/net/sched/sch_api.c
+++ b/net/sched/sch_api.c
@@ -942,6 +942,13 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
942 sch->handle = handle; 942 sch->handle = handle;
943 943
944 if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) { 944 if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) {
945 if (qdisc_is_percpu_stats(sch)) {
946 sch->cpu_bstats =
947 alloc_percpu(struct gnet_stats_basic_cpu);
948 if (!sch->cpu_bstats)
949 goto err_out4;
950 }
951
945 if (tca[TCA_STAB]) { 952 if (tca[TCA_STAB]) {
946 stab = qdisc_get_stab(tca[TCA_STAB]); 953 stab = qdisc_get_stab(tca[TCA_STAB]);
947 if (IS_ERR(stab)) { 954 if (IS_ERR(stab)) {
@@ -964,8 +971,11 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
964 else 971 else
965 root_lock = qdisc_lock(sch); 972 root_lock = qdisc_lock(sch);
966 973
967 err = gen_new_estimator(&sch->bstats, &sch->rate_est, 974 err = gen_new_estimator(&sch->bstats,
968 root_lock, tca[TCA_RATE]); 975 sch->cpu_bstats,
976 &sch->rate_est,
977 root_lock,
978 tca[TCA_RATE]);
969 if (err) 979 if (err)
970 goto err_out4; 980 goto err_out4;
971 } 981 }
@@ -984,6 +994,7 @@ err_out:
984 return NULL; 994 return NULL;
985 995
986err_out4: 996err_out4:
997 free_percpu(sch->cpu_bstats);
987 /* 998 /*
988 * Any broken qdiscs that would require a ops->reset() here? 999 * Any broken qdiscs that would require a ops->reset() here?
989 * The qdisc was never in action so it shouldn't be necessary. 1000 * The qdisc was never in action so it shouldn't be necessary.
@@ -1022,9 +1033,11 @@ static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)
1022 because change can't be undone. */ 1033 because change can't be undone. */
1023 if (sch->flags & TCQ_F_MQROOT) 1034 if (sch->flags & TCQ_F_MQROOT)
1024 goto out; 1035 goto out;
1025 gen_replace_estimator(&sch->bstats, &sch->rate_est, 1036 gen_replace_estimator(&sch->bstats,
1026 qdisc_root_sleeping_lock(sch), 1037 sch->cpu_bstats,
1027 tca[TCA_RATE]); 1038 &sch->rate_est,
1039 qdisc_root_sleeping_lock(sch),
1040 tca[TCA_RATE]);
1028 } 1041 }
1029out: 1042out:
1030 return 0; 1043 return 0;
@@ -1299,6 +1312,7 @@ graft:
1299static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid, 1312static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
1300 u32 portid, u32 seq, u16 flags, int event) 1313 u32 portid, u32 seq, u16 flags, int event)
1301{ 1314{
1315 struct gnet_stats_basic_cpu __percpu *cpu_bstats = NULL;
1302 struct tcmsg *tcm; 1316 struct tcmsg *tcm;
1303 struct nlmsghdr *nlh; 1317 struct nlmsghdr *nlh;
1304 unsigned char *b = skb_tail_pointer(skb); 1318 unsigned char *b = skb_tail_pointer(skb);
@@ -1334,7 +1348,10 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
1334 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0) 1348 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
1335 goto nla_put_failure; 1349 goto nla_put_failure;
1336 1350
1337 if (gnet_stats_copy_basic(&d, &q->bstats) < 0 || 1351 if (qdisc_is_percpu_stats(q))
1352 cpu_bstats = q->cpu_bstats;
1353
1354 if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats) < 0 ||
1338 gnet_stats_copy_rate_est(&d, &q->bstats, &q->rate_est) < 0 || 1355 gnet_stats_copy_rate_est(&d, &q->bstats, &q->rate_est) < 0 ||
1339 gnet_stats_copy_queue(&d, &q->qstats) < 0) 1356 gnet_stats_copy_queue(&d, &q->qstats) < 0)
1340 goto nla_put_failure; 1357 goto nla_put_failure;
diff --git a/net/sched/sch_atm.c b/net/sched/sch_atm.c
index c398f9c3dbdd..01017663e5d8 100644
--- a/net/sched/sch_atm.c
+++ b/net/sched/sch_atm.c
@@ -639,7 +639,7 @@ atm_tc_dump_class_stats(struct Qdisc *sch, unsigned long arg,
639 639
640 flow->qstats.qlen = flow->q->q.qlen; 640 flow->qstats.qlen = flow->q->q.qlen;
641 641
642 if (gnet_stats_copy_basic(d, &flow->bstats) < 0 || 642 if (gnet_stats_copy_basic(d, NULL, &flow->bstats) < 0 ||
643 gnet_stats_copy_queue(d, &flow->qstats) < 0) 643 gnet_stats_copy_queue(d, &flow->qstats) < 0)
644 return -1; 644 return -1;
645 645
diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c
index d2cd981ba60d..22a3a029a911 100644
--- a/net/sched/sch_cbq.c
+++ b/net/sched/sch_cbq.c
@@ -1601,7 +1601,7 @@ cbq_dump_class_stats(struct Qdisc *sch, unsigned long arg,
1601 if (cl->undertime != PSCHED_PASTPERFECT) 1601 if (cl->undertime != PSCHED_PASTPERFECT)
1602 cl->xstats.undertime = cl->undertime - q->now; 1602 cl->xstats.undertime = cl->undertime - q->now;
1603 1603
1604 if (gnet_stats_copy_basic(d, &cl->bstats) < 0 || 1604 if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
1605 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 || 1605 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
1606 gnet_stats_copy_queue(d, &cl->qstats) < 0) 1606 gnet_stats_copy_queue(d, &cl->qstats) < 0)
1607 return -1; 1607 return -1;
@@ -1759,7 +1759,8 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t
1759 } 1759 }
1760 1760
1761 if (tca[TCA_RATE]) { 1761 if (tca[TCA_RATE]) {
1762 err = gen_replace_estimator(&cl->bstats, &cl->rate_est, 1762 err = gen_replace_estimator(&cl->bstats, NULL,
1763 &cl->rate_est,
1763 qdisc_root_sleeping_lock(sch), 1764 qdisc_root_sleeping_lock(sch),
1764 tca[TCA_RATE]); 1765 tca[TCA_RATE]);
1765 if (err) { 1766 if (err) {
@@ -1852,7 +1853,7 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t
1852 goto failure; 1853 goto failure;
1853 1854
1854 if (tca[TCA_RATE]) { 1855 if (tca[TCA_RATE]) {
1855 err = gen_new_estimator(&cl->bstats, &cl->rate_est, 1856 err = gen_new_estimator(&cl->bstats, NULL, &cl->rate_est,
1856 qdisc_root_sleeping_lock(sch), 1857 qdisc_root_sleeping_lock(sch),
1857 tca[TCA_RATE]); 1858 tca[TCA_RATE]);
1858 if (err) { 1859 if (err) {
diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c
index d8b5ccfd248a..7a6243c5d270 100644
--- a/net/sched/sch_drr.c
+++ b/net/sched/sch_drr.c
@@ -88,7 +88,8 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
88 88
89 if (cl != NULL) { 89 if (cl != NULL) {
90 if (tca[TCA_RATE]) { 90 if (tca[TCA_RATE]) {
91 err = gen_replace_estimator(&cl->bstats, &cl->rate_est, 91 err = gen_replace_estimator(&cl->bstats, NULL,
92 &cl->rate_est,
92 qdisc_root_sleeping_lock(sch), 93 qdisc_root_sleeping_lock(sch),
93 tca[TCA_RATE]); 94 tca[TCA_RATE]);
94 if (err) 95 if (err)
@@ -116,7 +117,7 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
116 cl->qdisc = &noop_qdisc; 117 cl->qdisc = &noop_qdisc;
117 118
118 if (tca[TCA_RATE]) { 119 if (tca[TCA_RATE]) {
119 err = gen_replace_estimator(&cl->bstats, &cl->rate_est, 120 err = gen_replace_estimator(&cl->bstats, NULL, &cl->rate_est,
120 qdisc_root_sleeping_lock(sch), 121 qdisc_root_sleeping_lock(sch),
121 tca[TCA_RATE]); 122 tca[TCA_RATE]);
122 if (err) { 123 if (err) {
@@ -282,7 +283,7 @@ static int drr_dump_class_stats(struct Qdisc *sch, unsigned long arg,
282 cl->qdisc->qstats.qlen = cl->qdisc->q.qlen; 283 cl->qdisc->qstats.qlen = cl->qdisc->q.qlen;
283 } 284 }
284 285
285 if (gnet_stats_copy_basic(d, &cl->bstats) < 0 || 286 if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
286 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 || 287 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
287 gnet_stats_copy_queue(d, &cl->qdisc->qstats) < 0) 288 gnet_stats_copy_queue(d, &cl->qdisc->qstats) < 0)
288 return -1; 289 return -1;
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index 11b28f651ad1..7c8e5d73d433 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -632,6 +632,9 @@ static void qdisc_rcu_free(struct rcu_head *head)
632{ 632{
633 struct Qdisc *qdisc = container_of(head, struct Qdisc, rcu_head); 633 struct Qdisc *qdisc = container_of(head, struct Qdisc, rcu_head);
634 634
635 if (qdisc_is_percpu_stats(qdisc))
636 free_percpu(qdisc->cpu_bstats);
637
635 kfree((char *) qdisc - qdisc->padded); 638 kfree((char *) qdisc - qdisc->padded);
636} 639}
637 640
diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c
index 04b0de4c68b5..209b966b2eed 100644
--- a/net/sched/sch_hfsc.c
+++ b/net/sched/sch_hfsc.c
@@ -1014,9 +1014,12 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
1014 cur_time = psched_get_time(); 1014 cur_time = psched_get_time();
1015 1015
1016 if (tca[TCA_RATE]) { 1016 if (tca[TCA_RATE]) {
1017 err = gen_replace_estimator(&cl->bstats, &cl->rate_est, 1017 spinlock_t *lock = qdisc_root_sleeping_lock(sch);
1018 qdisc_root_sleeping_lock(sch), 1018
1019 tca[TCA_RATE]); 1019 err = gen_replace_estimator(&cl->bstats, NULL,
1020 &cl->rate_est,
1021 lock,
1022 tca[TCA_RATE]);
1020 if (err) 1023 if (err)
1021 return err; 1024 return err;
1022 } 1025 }
@@ -1063,7 +1066,7 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
1063 return -ENOBUFS; 1066 return -ENOBUFS;
1064 1067
1065 if (tca[TCA_RATE]) { 1068 if (tca[TCA_RATE]) {
1066 err = gen_new_estimator(&cl->bstats, &cl->rate_est, 1069 err = gen_new_estimator(&cl->bstats, NULL, &cl->rate_est,
1067 qdisc_root_sleeping_lock(sch), 1070 qdisc_root_sleeping_lock(sch),
1068 tca[TCA_RATE]); 1071 tca[TCA_RATE]);
1069 if (err) { 1072 if (err) {
@@ -1374,7 +1377,7 @@ hfsc_dump_class_stats(struct Qdisc *sch, unsigned long arg,
1374 xstats.work = cl->cl_total; 1377 xstats.work = cl->cl_total;
1375 xstats.rtwork = cl->cl_cumul; 1378 xstats.rtwork = cl->cl_cumul;
1376 1379
1377 if (gnet_stats_copy_basic(d, &cl->bstats) < 0 || 1380 if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
1378 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 || 1381 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
1379 gnet_stats_copy_queue(d, &cl->qstats) < 0) 1382 gnet_stats_copy_queue(d, &cl->qstats) < 0)
1380 return -1; 1383 return -1;
diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c
index 063e953d9848..0256dee69bd6 100644
--- a/net/sched/sch_htb.c
+++ b/net/sched/sch_htb.c
@@ -1144,7 +1144,7 @@ htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d)
1144 cl->xstats.tokens = PSCHED_NS2TICKS(cl->tokens); 1144 cl->xstats.tokens = PSCHED_NS2TICKS(cl->tokens);
1145 cl->xstats.ctokens = PSCHED_NS2TICKS(cl->ctokens); 1145 cl->xstats.ctokens = PSCHED_NS2TICKS(cl->ctokens);
1146 1146
1147 if (gnet_stats_copy_basic(d, &cl->bstats) < 0 || 1147 if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
1148 gnet_stats_copy_rate_est(d, NULL, &cl->rate_est) < 0 || 1148 gnet_stats_copy_rate_est(d, NULL, &cl->rate_est) < 0 ||
1149 gnet_stats_copy_queue(d, &cl->qstats) < 0) 1149 gnet_stats_copy_queue(d, &cl->qstats) < 0)
1150 return -1; 1150 return -1;
@@ -1402,7 +1402,8 @@ static int htb_change_class(struct Qdisc *sch, u32 classid,
1402 goto failure; 1402 goto failure;
1403 1403
1404 if (htb_rate_est || tca[TCA_RATE]) { 1404 if (htb_rate_est || tca[TCA_RATE]) {
1405 err = gen_new_estimator(&cl->bstats, &cl->rate_est, 1405 err = gen_new_estimator(&cl->bstats, NULL,
1406 &cl->rate_est,
1406 qdisc_root_sleeping_lock(sch), 1407 qdisc_root_sleeping_lock(sch),
1407 tca[TCA_RATE] ? : &est.nla); 1408 tca[TCA_RATE] ? : &est.nla);
1408 if (err) { 1409 if (err) {
@@ -1464,8 +1465,11 @@ static int htb_change_class(struct Qdisc *sch, u32 classid,
1464 parent->children++; 1465 parent->children++;
1465 } else { 1466 } else {
1466 if (tca[TCA_RATE]) { 1467 if (tca[TCA_RATE]) {
1467 err = gen_replace_estimator(&cl->bstats, &cl->rate_est, 1468 spinlock_t *lock = qdisc_root_sleeping_lock(sch);
1468 qdisc_root_sleeping_lock(sch), 1469
1470 err = gen_replace_estimator(&cl->bstats, NULL,
1471 &cl->rate_est,
1472 lock,
1469 tca[TCA_RATE]); 1473 tca[TCA_RATE]);
1470 if (err) 1474 if (err)
1471 return err; 1475 return err;
diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c
index a8b2864a696b..d3a27fb607af 100644
--- a/net/sched/sch_mq.c
+++ b/net/sched/sch_mq.c
@@ -201,7 +201,7 @@ static int mq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
201 201
202 sch = dev_queue->qdisc_sleeping; 202 sch = dev_queue->qdisc_sleeping;
203 sch->qstats.qlen = sch->q.qlen; 203 sch->qstats.qlen = sch->q.qlen;
204 if (gnet_stats_copy_basic(d, &sch->bstats) < 0 || 204 if (gnet_stats_copy_basic(d, NULL, &sch->bstats) < 0 ||
205 gnet_stats_copy_queue(d, &sch->qstats) < 0) 205 gnet_stats_copy_queue(d, &sch->qstats) < 0)
206 return -1; 206 return -1;
207 return 0; 207 return 0;
diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c
index 37e7d25d21f1..8917372fddc6 100644
--- a/net/sched/sch_mqprio.c
+++ b/net/sched/sch_mqprio.c
@@ -355,7 +355,7 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
355 } 355 }
356 /* Reclaim root sleeping lock before completing stats */ 356 /* Reclaim root sleeping lock before completing stats */
357 spin_lock_bh(d->lock); 357 spin_lock_bh(d->lock);
358 if (gnet_stats_copy_basic(d, &bstats) < 0 || 358 if (gnet_stats_copy_basic(d, NULL, &bstats) < 0 ||
359 gnet_stats_copy_queue(d, &qstats) < 0) 359 gnet_stats_copy_queue(d, &qstats) < 0)
360 return -1; 360 return -1;
361 } else { 361 } else {
@@ -363,7 +363,7 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
363 363
364 sch = dev_queue->qdisc_sleeping; 364 sch = dev_queue->qdisc_sleeping;
365 sch->qstats.qlen = sch->q.qlen; 365 sch->qstats.qlen = sch->q.qlen;
366 if (gnet_stats_copy_basic(d, &sch->bstats) < 0 || 366 if (gnet_stats_copy_basic(d, NULL, &sch->bstats) < 0 ||
367 gnet_stats_copy_queue(d, &sch->qstats) < 0) 367 gnet_stats_copy_queue(d, &sch->qstats) < 0)
368 return -1; 368 return -1;
369 } 369 }
diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c
index c0466c1840f3..4adbf7fefc09 100644
--- a/net/sched/sch_multiq.c
+++ b/net/sched/sch_multiq.c
@@ -361,7 +361,7 @@ static int multiq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
361 361
362 cl_q = q->queues[cl - 1]; 362 cl_q = q->queues[cl - 1];
363 cl_q->qstats.qlen = cl_q->q.qlen; 363 cl_q->qstats.qlen = cl_q->q.qlen;
364 if (gnet_stats_copy_basic(d, &cl_q->bstats) < 0 || 364 if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats) < 0 ||
365 gnet_stats_copy_queue(d, &cl_q->qstats) < 0) 365 gnet_stats_copy_queue(d, &cl_q->qstats) < 0)
366 return -1; 366 return -1;
367 367
diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c
index 03ef99e52a5c..68a8f25e30c3 100644
--- a/net/sched/sch_prio.c
+++ b/net/sched/sch_prio.c
@@ -325,7 +325,7 @@ static int prio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
325 325
326 cl_q = q->queues[cl - 1]; 326 cl_q = q->queues[cl - 1];
327 cl_q->qstats.qlen = cl_q->q.qlen; 327 cl_q->qstats.qlen = cl_q->q.qlen;
328 if (gnet_stats_copy_basic(d, &cl_q->bstats) < 0 || 328 if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats) < 0 ||
329 gnet_stats_copy_queue(d, &cl_q->qstats) < 0) 329 gnet_stats_copy_queue(d, &cl_q->qstats) < 0)
330 return -1; 330 return -1;
331 331
diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c
index 602ea01a4ddd..d59f8574540a 100644
--- a/net/sched/sch_qfq.c
+++ b/net/sched/sch_qfq.c
@@ -459,7 +459,8 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
459 459
460 if (cl != NULL) { /* modify existing class */ 460 if (cl != NULL) { /* modify existing class */
461 if (tca[TCA_RATE]) { 461 if (tca[TCA_RATE]) {
462 err = gen_replace_estimator(&cl->bstats, &cl->rate_est, 462 err = gen_replace_estimator(&cl->bstats, NULL,
463 &cl->rate_est,
463 qdisc_root_sleeping_lock(sch), 464 qdisc_root_sleeping_lock(sch),
464 tca[TCA_RATE]); 465 tca[TCA_RATE]);
465 if (err) 466 if (err)
@@ -484,7 +485,8 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
484 cl->qdisc = &noop_qdisc; 485 cl->qdisc = &noop_qdisc;
485 486
486 if (tca[TCA_RATE]) { 487 if (tca[TCA_RATE]) {
487 err = gen_new_estimator(&cl->bstats, &cl->rate_est, 488 err = gen_new_estimator(&cl->bstats, NULL,
489 &cl->rate_est,
488 qdisc_root_sleeping_lock(sch), 490 qdisc_root_sleeping_lock(sch),
489 tca[TCA_RATE]); 491 tca[TCA_RATE]);
490 if (err) 492 if (err)
@@ -667,7 +669,7 @@ static int qfq_dump_class_stats(struct Qdisc *sch, unsigned long arg,
667 xstats.weight = cl->agg->class_weight; 669 xstats.weight = cl->agg->class_weight;
668 xstats.lmax = cl->agg->lmax; 670 xstats.lmax = cl->agg->lmax;
669 671
670 if (gnet_stats_copy_basic(d, &cl->bstats) < 0 || 672 if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
671 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 || 673 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
672 gnet_stats_copy_queue(d, &cl->qdisc->qstats) < 0) 674 gnet_stats_copy_queue(d, &cl->qdisc->qstats) < 0)
673 return -1; 675 return -1;
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-20 03:37:41 -0400