aboutsummaryrefslogtreecommitdiffstats
path: root/net
diff options
context:
space:
mode:
Diffstat (limited to 'net')
-rw-r--r--net/core/gen_estimator.c29
-rw-r--r--net/core/gen_stats.c53
-rw-r--r--net/netfilter/xt_RATEEST.c2
-rw-r--r--net/sched/act_api.c5
-rw-r--r--net/sched/act_police.c2
-rw-r--r--net/sched/sch_api.c29
-rw-r--r--net/sched/sch_atm.c2
-rw-r--r--net/sched/sch_cbq.c7
-rw-r--r--net/sched/sch_drr.c7
-rw-r--r--net/sched/sch_generic.c3
-rw-r--r--net/sched/sch_hfsc.c13
-rw-r--r--net/sched/sch_htb.c12
-rw-r--r--net/sched/sch_mq.c2
-rw-r--r--net/sched/sch_mqprio.c4
-rw-r--r--net/sched/sch_multiq.c2
-rw-r--r--net/sched/sch_prio.c2
-rw-r--r--net/sched/sch_qfq.c8
17 files changed, 132 insertions, 50 deletions
diff --git a/net/core/gen_estimator.c b/net/core/gen_estimator.c
index 9d33dfffca19..9dfb88a933e7 100644
--- a/net/core/gen_estimator.c
+++ b/net/core/gen_estimator.c
@@ -91,6 +91,8 @@ struct gen_estimator
91 u32 avpps; 91 u32 avpps;
92 struct rcu_head e_rcu; 92 struct rcu_head e_rcu;
93 struct rb_node node; 93 struct rb_node node;
94 struct gnet_stats_basic_cpu __percpu *cpu_bstats;
95 struct rcu_head head;
94}; 96};
95 97
96struct gen_estimator_head 98struct gen_estimator_head
@@ -115,9 +117,8 @@ static void est_timer(unsigned long arg)
115 117
116 rcu_read_lock(); 118 rcu_read_lock();
117 list_for_each_entry_rcu(e, &elist[idx].list, list) { 119 list_for_each_entry_rcu(e, &elist[idx].list, list) {
118 u64 nbytes; 120 struct gnet_stats_basic_packed b = {0};
119 u64 brate; 121 u64 brate;
120 u32 npackets;
121 u32 rate; 122 u32 rate;
122 123
123 spin_lock(e->stats_lock); 124 spin_lock(e->stats_lock);
@@ -125,15 +126,15 @@ static void est_timer(unsigned long arg)
125 if (e->bstats == NULL) 126 if (e->bstats == NULL)
126 goto skip; 127 goto skip;
127 128
128 nbytes = e->bstats->bytes; 129 __gnet_stats_copy_basic(&b, e->cpu_bstats, e->bstats);
129 npackets = e->bstats->packets; 130
130 brate = (nbytes - e->last_bytes)<<(7 - idx); 131 brate = (b.bytes - e->last_bytes)<<(7 - idx);
131 e->last_bytes = nbytes; 132 e->last_bytes = b.bytes;
132 e->avbps += (brate >> e->ewma_log) - (e->avbps >> e->ewma_log); 133 e->avbps += (brate >> e->ewma_log) - (e->avbps >> e->ewma_log);
133 e->rate_est->bps = (e->avbps+0xF)>>5; 134 e->rate_est->bps = (e->avbps+0xF)>>5;
134 135
135 rate = (npackets - e->last_packets)<<(12 - idx); 136 rate = (b.packets - e->last_packets)<<(12 - idx);
136 e->last_packets = npackets; 137 e->last_packets = b.packets;
137 e->avpps += (rate >> e->ewma_log) - (e->avpps >> e->ewma_log); 138 e->avpps += (rate >> e->ewma_log) - (e->avpps >> e->ewma_log);
138 e->rate_est->pps = (e->avpps+0x1FF)>>10; 139 e->rate_est->pps = (e->avpps+0x1FF)>>10;
139skip: 140skip:
@@ -203,12 +204,14 @@ struct gen_estimator *gen_find_node(const struct gnet_stats_basic_packed *bstats
203 * 204 *
204 */ 205 */
205int gen_new_estimator(struct gnet_stats_basic_packed *bstats, 206int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
207 struct gnet_stats_basic_cpu __percpu *cpu_bstats,
206 struct gnet_stats_rate_est64 *rate_est, 208 struct gnet_stats_rate_est64 *rate_est,
207 spinlock_t *stats_lock, 209 spinlock_t *stats_lock,
208 struct nlattr *opt) 210 struct nlattr *opt)
209{ 211{
210 struct gen_estimator *est; 212 struct gen_estimator *est;
211 struct gnet_estimator *parm = nla_data(opt); 213 struct gnet_estimator *parm = nla_data(opt);
214 struct gnet_stats_basic_packed b = {0};
212 int idx; 215 int idx;
213 216
214 if (nla_len(opt) < sizeof(*parm)) 217 if (nla_len(opt) < sizeof(*parm))
@@ -221,15 +224,18 @@ int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
221 if (est == NULL) 224 if (est == NULL)
222 return -ENOBUFS; 225 return -ENOBUFS;
223 226
227 __gnet_stats_copy_basic(&b, cpu_bstats, bstats);
228
224 idx = parm->interval + 2; 229 idx = parm->interval + 2;
225 est->bstats = bstats; 230 est->bstats = bstats;
226 est->rate_est = rate_est; 231 est->rate_est = rate_est;
227 est->stats_lock = stats_lock; 232 est->stats_lock = stats_lock;
228 est->ewma_log = parm->ewma_log; 233 est->ewma_log = parm->ewma_log;
229 est->last_bytes = bstats->bytes; 234 est->last_bytes = b.bytes;
230 est->avbps = rate_est->bps<<5; 235 est->avbps = rate_est->bps<<5;
231 est->last_packets = bstats->packets; 236 est->last_packets = b.packets;
232 est->avpps = rate_est->pps<<10; 237 est->avpps = rate_est->pps<<10;
238 est->cpu_bstats = cpu_bstats;
233 239
234 spin_lock_bh(&est_tree_lock); 240 spin_lock_bh(&est_tree_lock);
235 if (!elist[idx].timer.function) { 241 if (!elist[idx].timer.function) {
@@ -290,11 +296,12 @@ EXPORT_SYMBOL(gen_kill_estimator);
290 * Returns 0 on success or a negative error code. 296 * Returns 0 on success or a negative error code.
291 */ 297 */
292int gen_replace_estimator(struct gnet_stats_basic_packed *bstats, 298int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
299 struct gnet_stats_basic_cpu __percpu *cpu_bstats,
293 struct gnet_stats_rate_est64 *rate_est, 300 struct gnet_stats_rate_est64 *rate_est,
294 spinlock_t *stats_lock, struct nlattr *opt) 301 spinlock_t *stats_lock, struct nlattr *opt)
295{ 302{
296 gen_kill_estimator(bstats, rate_est); 303 gen_kill_estimator(bstats, rate_est);
297 return gen_new_estimator(bstats, rate_est, stats_lock, opt); 304 return gen_new_estimator(bstats, cpu_bstats, rate_est, stats_lock, opt);
298} 305}
299EXPORT_SYMBOL(gen_replace_estimator); 306EXPORT_SYMBOL(gen_replace_estimator);
300 307
diff --git a/net/core/gen_stats.c b/net/core/gen_stats.c
index 2ddbce4cce14..5ff8e80fe0bb 100644
--- a/net/core/gen_stats.c
+++ b/net/core/gen_stats.c
@@ -97,6 +97,43 @@ gnet_stats_start_copy(struct sk_buff *skb, int type, spinlock_t *lock,
97} 97}
98EXPORT_SYMBOL(gnet_stats_start_copy); 98EXPORT_SYMBOL(gnet_stats_start_copy);
99 99
100static void
101__gnet_stats_copy_basic_cpu(struct gnet_stats_basic_packed *bstats,
102 struct gnet_stats_basic_cpu __percpu *cpu)
103{
104 int i;
105
106 for_each_possible_cpu(i) {
107 struct gnet_stats_basic_cpu *bcpu = per_cpu_ptr(cpu, i);
108 unsigned int start;
109 __u64 bytes;
110 __u32 packets;
111
112 do {
113 start = u64_stats_fetch_begin_irq(&bcpu->syncp);
114 bytes = bcpu->bstats.bytes;
115 packets = bcpu->bstats.packets;
116 } while (u64_stats_fetch_retry_irq(&bcpu->syncp, start));
117
118 bstats->bytes += bcpu->bstats.bytes;
119 bstats->packets += bcpu->bstats.packets;
120 }
121}
122
123void
124__gnet_stats_copy_basic(struct gnet_stats_basic_packed *bstats,
125 struct gnet_stats_basic_cpu __percpu *cpu,
126 struct gnet_stats_basic_packed *b)
127{
128 if (cpu) {
129 __gnet_stats_copy_basic_cpu(bstats, cpu);
130 } else {
131 bstats->bytes = b->bytes;
132 bstats->packets = b->packets;
133 }
134}
135EXPORT_SYMBOL(__gnet_stats_copy_basic);
136
100/** 137/**
101 * gnet_stats_copy_basic - copy basic statistics into statistic TLV 138 * gnet_stats_copy_basic - copy basic statistics into statistic TLV
102 * @d: dumping handle 139 * @d: dumping handle
@@ -109,19 +146,25 @@ EXPORT_SYMBOL(gnet_stats_start_copy);
109 * if the room in the socket buffer was not sufficient. 146 * if the room in the socket buffer was not sufficient.
110 */ 147 */
111int 148int
112gnet_stats_copy_basic(struct gnet_dump *d, struct gnet_stats_basic_packed *b) 149gnet_stats_copy_basic(struct gnet_dump *d,
150 struct gnet_stats_basic_cpu __percpu *cpu,
151 struct gnet_stats_basic_packed *b)
113{ 152{
153 struct gnet_stats_basic_packed bstats = {0};
154
155 __gnet_stats_copy_basic(&bstats, cpu, b);
156
114 if (d->compat_tc_stats) { 157 if (d->compat_tc_stats) {
115 d->tc_stats.bytes = b->bytes; 158 d->tc_stats.bytes = bstats.bytes;
116 d->tc_stats.packets = b->packets; 159 d->tc_stats.packets = bstats.packets;
117 } 160 }
118 161
119 if (d->tail) { 162 if (d->tail) {
120 struct gnet_stats_basic sb; 163 struct gnet_stats_basic sb;
121 164
122 memset(&sb, 0, sizeof(sb)); 165 memset(&sb, 0, sizeof(sb));
123 sb.bytes = b->bytes; 166 sb.bytes = bstats.bytes;
124 sb.packets = b->packets; 167 sb.packets = bstats.packets;
125 return gnet_stats_copy(d, TCA_STATS_BASIC, &sb, sizeof(sb)); 168 return gnet_stats_copy(d, TCA_STATS_BASIC, &sb, sizeof(sb));
126 } 169 }
127 return 0; 170 return 0;
diff --git a/net/netfilter/xt_RATEEST.c b/net/netfilter/xt_RATEEST.c
index 370adf622cef..604df6fae6fc 100644
--- a/net/netfilter/xt_RATEEST.c
+++ b/net/netfilter/xt_RATEEST.c
@@ -136,7 +136,7 @@ static int xt_rateest_tg_checkentry(const struct xt_tgchk_param *par)
136 cfg.est.interval = info->interval; 136 cfg.est.interval = info->interval;
137 cfg.est.ewma_log = info->ewma_log; 137 cfg.est.ewma_log = info->ewma_log;
138 138
139 ret = gen_new_estimator(&est->bstats, &est->rstats, 139 ret = gen_new_estimator(&est->bstats, NULL, &est->rstats,
140 &est->lock, &cfg.opt); 140 &est->lock, &cfg.opt);
141 if (ret < 0) 141 if (ret < 0)
142 goto err2; 142 goto err2;
diff --git a/net/sched/act_api.c b/net/sched/act_api.c
index 648778aef1a2..eca4cf9ece2f 100644
--- a/net/sched/act_api.c
+++ b/net/sched/act_api.c
@@ -252,7 +252,8 @@ int tcf_hash_create(u32 index, struct nlattr *est, struct tc_action *a,
252 p->tcfc_tm.install = jiffies; 252 p->tcfc_tm.install = jiffies;
253 p->tcfc_tm.lastuse = jiffies; 253 p->tcfc_tm.lastuse = jiffies;
254 if (est) { 254 if (est) {
255 int err = gen_new_estimator(&p->tcfc_bstats, &p->tcfc_rate_est, 255 int err = gen_new_estimator(&p->tcfc_bstats, NULL,
256 &p->tcfc_rate_est,
256 &p->tcfc_lock, est); 257 &p->tcfc_lock, est);
257 if (err) { 258 if (err) {
258 kfree(p); 259 kfree(p);
@@ -619,7 +620,7 @@ int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *a,
619 if (err < 0) 620 if (err < 0)
620 goto errout; 621 goto errout;
621 622
622 if (gnet_stats_copy_basic(&d, &p->tcfc_bstats) < 0 || 623 if (gnet_stats_copy_basic(&d, NULL, &p->tcfc_bstats) < 0 ||
623 gnet_stats_copy_rate_est(&d, &p->tcfc_bstats, 624 gnet_stats_copy_rate_est(&d, &p->tcfc_bstats,
624 &p->tcfc_rate_est) < 0 || 625 &p->tcfc_rate_est) < 0 ||
625 gnet_stats_copy_queue(&d, &p->tcfc_qstats) < 0) 626 gnet_stats_copy_queue(&d, &p->tcfc_qstats) < 0)
diff --git a/net/sched/act_police.c b/net/sched/act_police.c
index f32bcb094915..69791ca77a05 100644
--- a/net/sched/act_police.c
+++ b/net/sched/act_police.c
@@ -178,7 +178,7 @@ override:
178 178
179 spin_lock_bh(&police->tcf_lock); 179 spin_lock_bh(&police->tcf_lock);
180 if (est) { 180 if (est) {
181 err = gen_replace_estimator(&police->tcf_bstats, 181 err = gen_replace_estimator(&police->tcf_bstats, NULL,
182 &police->tcf_rate_est, 182 &police->tcf_rate_est,
183 &police->tcf_lock, est); 183 &police->tcf_lock, est);
184 if (err) 184 if (err)
diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
index 15e7beee266c..a95e3b48fa51 100644
--- a/net/sched/sch_api.c
+++ b/net/sched/sch_api.c
@@ -942,6 +942,13 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
942 sch->handle = handle; 942 sch->handle = handle;
943 943
944 if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) { 944 if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) {
945 if (qdisc_is_percpu_stats(sch)) {
946 sch->cpu_bstats =
947 alloc_percpu(struct gnet_stats_basic_cpu);
948 if (!sch->cpu_bstats)
949 goto err_out4;
950 }
951
945 if (tca[TCA_STAB]) { 952 if (tca[TCA_STAB]) {
946 stab = qdisc_get_stab(tca[TCA_STAB]); 953 stab = qdisc_get_stab(tca[TCA_STAB]);
947 if (IS_ERR(stab)) { 954 if (IS_ERR(stab)) {
@@ -964,8 +971,11 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
964 else 971 else
965 root_lock = qdisc_lock(sch); 972 root_lock = qdisc_lock(sch);
966 973
967 err = gen_new_estimator(&sch->bstats, &sch->rate_est, 974 err = gen_new_estimator(&sch->bstats,
968 root_lock, tca[TCA_RATE]); 975 sch->cpu_bstats,
976 &sch->rate_est,
977 root_lock,
978 tca[TCA_RATE]);
969 if (err) 979 if (err)
970 goto err_out4; 980 goto err_out4;
971 } 981 }
@@ -984,6 +994,7 @@ err_out:
984 return NULL; 994 return NULL;
985 995
986err_out4: 996err_out4:
997 free_percpu(sch->cpu_bstats);
987 /* 998 /*
988 * Any broken qdiscs that would require a ops->reset() here? 999 * Any broken qdiscs that would require a ops->reset() here?
989 * The qdisc was never in action so it shouldn't be necessary. 1000 * The qdisc was never in action so it shouldn't be necessary.
@@ -1022,9 +1033,11 @@ static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)
1022 because change can't be undone. */ 1033 because change can't be undone. */
1023 if (sch->flags & TCQ_F_MQROOT) 1034 if (sch->flags & TCQ_F_MQROOT)
1024 goto out; 1035 goto out;
1025 gen_replace_estimator(&sch->bstats, &sch->rate_est, 1036 gen_replace_estimator(&sch->bstats,
1026 qdisc_root_sleeping_lock(sch), 1037 sch->cpu_bstats,
1027 tca[TCA_RATE]); 1038 &sch->rate_est,
1039 qdisc_root_sleeping_lock(sch),
1040 tca[TCA_RATE]);
1028 } 1041 }
1029out: 1042out:
1030 return 0; 1043 return 0;
@@ -1299,6 +1312,7 @@ graft:
1299static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid, 1312static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
1300 u32 portid, u32 seq, u16 flags, int event) 1313 u32 portid, u32 seq, u16 flags, int event)
1301{ 1314{
1315 struct gnet_stats_basic_cpu __percpu *cpu_bstats = NULL;
1302 struct tcmsg *tcm; 1316 struct tcmsg *tcm;
1303 struct nlmsghdr *nlh; 1317 struct nlmsghdr *nlh;
1304 unsigned char *b = skb_tail_pointer(skb); 1318 unsigned char *b = skb_tail_pointer(skb);
@@ -1334,7 +1348,10 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
1334 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0) 1348 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
1335 goto nla_put_failure; 1349 goto nla_put_failure;
1336 1350
1337 if (gnet_stats_copy_basic(&d, &q->bstats) < 0 || 1351 if (qdisc_is_percpu_stats(q))
1352 cpu_bstats = q->cpu_bstats;
1353
1354 if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats) < 0 ||
1338 gnet_stats_copy_rate_est(&d, &q->bstats, &q->rate_est) < 0 || 1355 gnet_stats_copy_rate_est(&d, &q->bstats, &q->rate_est) < 0 ||
1339 gnet_stats_copy_queue(&d, &q->qstats) < 0) 1356 gnet_stats_copy_queue(&d, &q->qstats) < 0)
1340 goto nla_put_failure; 1357 goto nla_put_failure;
diff --git a/net/sched/sch_atm.c b/net/sched/sch_atm.c
index c398f9c3dbdd..01017663e5d8 100644
--- a/net/sched/sch_atm.c
+++ b/net/sched/sch_atm.c
@@ -639,7 +639,7 @@ atm_tc_dump_class_stats(struct Qdisc *sch, unsigned long arg,
639 639
640 flow->qstats.qlen = flow->q->q.qlen; 640 flow->qstats.qlen = flow->q->q.qlen;
641 641
642 if (gnet_stats_copy_basic(d, &flow->bstats) < 0 || 642 if (gnet_stats_copy_basic(d, NULL, &flow->bstats) < 0 ||
643 gnet_stats_copy_queue(d, &flow->qstats) < 0) 643 gnet_stats_copy_queue(d, &flow->qstats) < 0)
644 return -1; 644 return -1;
645 645
diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c
index d2cd981ba60d..22a3a029a911 100644
--- a/net/sched/sch_cbq.c
+++ b/net/sched/sch_cbq.c
@@ -1601,7 +1601,7 @@ cbq_dump_class_stats(struct Qdisc *sch, unsigned long arg,
1601 if (cl->undertime != PSCHED_PASTPERFECT) 1601 if (cl->undertime != PSCHED_PASTPERFECT)
1602 cl->xstats.undertime = cl->undertime - q->now; 1602 cl->xstats.undertime = cl->undertime - q->now;
1603 1603
1604 if (gnet_stats_copy_basic(d, &cl->bstats) < 0 || 1604 if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
1605 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 || 1605 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
1606 gnet_stats_copy_queue(d, &cl->qstats) < 0) 1606 gnet_stats_copy_queue(d, &cl->qstats) < 0)
1607 return -1; 1607 return -1;
@@ -1759,7 +1759,8 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t
1759 } 1759 }
1760 1760
1761 if (tca[TCA_RATE]) { 1761 if (tca[TCA_RATE]) {
1762 err = gen_replace_estimator(&cl->bstats, &cl->rate_est, 1762 err = gen_replace_estimator(&cl->bstats, NULL,
1763 &cl->rate_est,
1763 qdisc_root_sleeping_lock(sch), 1764 qdisc_root_sleeping_lock(sch),
1764 tca[TCA_RATE]); 1765 tca[TCA_RATE]);
1765 if (err) { 1766 if (err) {
@@ -1852,7 +1853,7 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t
1852 goto failure; 1853 goto failure;
1853 1854
1854 if (tca[TCA_RATE]) { 1855 if (tca[TCA_RATE]) {
1855 err = gen_new_estimator(&cl->bstats, &cl->rate_est, 1856 err = gen_new_estimator(&cl->bstats, NULL, &cl->rate_est,
1856 qdisc_root_sleeping_lock(sch), 1857 qdisc_root_sleeping_lock(sch),
1857 tca[TCA_RATE]); 1858 tca[TCA_RATE]);
1858 if (err) { 1859 if (err) {
diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c
index d8b5ccfd248a..7a6243c5d270 100644
--- a/net/sched/sch_drr.c
+++ b/net/sched/sch_drr.c
@@ -88,7 +88,8 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
88 88
89 if (cl != NULL) { 89 if (cl != NULL) {
90 if (tca[TCA_RATE]) { 90 if (tca[TCA_RATE]) {
91 err = gen_replace_estimator(&cl->bstats, &cl->rate_est, 91 err = gen_replace_estimator(&cl->bstats, NULL,
92 &cl->rate_est,
92 qdisc_root_sleeping_lock(sch), 93 qdisc_root_sleeping_lock(sch),
93 tca[TCA_RATE]); 94 tca[TCA_RATE]);
94 if (err) 95 if (err)
@@ -116,7 +117,7 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
116 cl->qdisc = &noop_qdisc; 117 cl->qdisc = &noop_qdisc;
117 118
118 if (tca[TCA_RATE]) { 119 if (tca[TCA_RATE]) {
119 err = gen_replace_estimator(&cl->bstats, &cl->rate_est, 120 err = gen_replace_estimator(&cl->bstats, NULL, &cl->rate_est,
120 qdisc_root_sleeping_lock(sch), 121 qdisc_root_sleeping_lock(sch),
121 tca[TCA_RATE]); 122 tca[TCA_RATE]);
122 if (err) { 123 if (err) {
@@ -282,7 +283,7 @@ static int drr_dump_class_stats(struct Qdisc *sch, unsigned long arg,
282 cl->qdisc->qstats.qlen = cl->qdisc->q.qlen; 283 cl->qdisc->qstats.qlen = cl->qdisc->q.qlen;
283 } 284 }
284 285
285 if (gnet_stats_copy_basic(d, &cl->bstats) < 0 || 286 if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
286 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 || 287 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
287 gnet_stats_copy_queue(d, &cl->qdisc->qstats) < 0) 288 gnet_stats_copy_queue(d, &cl->qdisc->qstats) < 0)
288 return -1; 289 return -1;
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index 11b28f651ad1..7c8e5d73d433 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -632,6 +632,9 @@ static void qdisc_rcu_free(struct rcu_head *head)
632{ 632{
633 struct Qdisc *qdisc = container_of(head, struct Qdisc, rcu_head); 633 struct Qdisc *qdisc = container_of(head, struct Qdisc, rcu_head);
634 634
635 if (qdisc_is_percpu_stats(qdisc))
636 free_percpu(qdisc->cpu_bstats);
637
635 kfree((char *) qdisc - qdisc->padded); 638 kfree((char *) qdisc - qdisc->padded);
636} 639}
637 640
diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c
index 04b0de4c68b5..209b966b2eed 100644
--- a/net/sched/sch_hfsc.c
+++ b/net/sched/sch_hfsc.c
@@ -1014,9 +1014,12 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
1014 cur_time = psched_get_time(); 1014 cur_time = psched_get_time();
1015 1015
1016 if (tca[TCA_RATE]) { 1016 if (tca[TCA_RATE]) {
1017 err = gen_replace_estimator(&cl->bstats, &cl->rate_est, 1017 spinlock_t *lock = qdisc_root_sleeping_lock(sch);
1018 qdisc_root_sleeping_lock(sch), 1018
1019 tca[TCA_RATE]); 1019 err = gen_replace_estimator(&cl->bstats, NULL,
1020 &cl->rate_est,
1021 lock,
1022 tca[TCA_RATE]);
1020 if (err) 1023 if (err)
1021 return err; 1024 return err;
1022 } 1025 }
@@ -1063,7 +1066,7 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
1063 return -ENOBUFS; 1066 return -ENOBUFS;
1064 1067
1065 if (tca[TCA_RATE]) { 1068 if (tca[TCA_RATE]) {
1066 err = gen_new_estimator(&cl->bstats, &cl->rate_est, 1069 err = gen_new_estimator(&cl->bstats, NULL, &cl->rate_est,
1067 qdisc_root_sleeping_lock(sch), 1070 qdisc_root_sleeping_lock(sch),
1068 tca[TCA_RATE]); 1071 tca[TCA_RATE]);
1069 if (err) { 1072 if (err) {
@@ -1374,7 +1377,7 @@ hfsc_dump_class_stats(struct Qdisc *sch, unsigned long arg,
1374 xstats.work = cl->cl_total; 1377 xstats.work = cl->cl_total;
1375 xstats.rtwork = cl->cl_cumul; 1378 xstats.rtwork = cl->cl_cumul;
1376 1379
1377 if (gnet_stats_copy_basic(d, &cl->bstats) < 0 || 1380 if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
1378 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 || 1381 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
1379 gnet_stats_copy_queue(d, &cl->qstats) < 0) 1382 gnet_stats_copy_queue(d, &cl->qstats) < 0)
1380 return -1; 1383 return -1;
diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c
index 063e953d9848..0256dee69bd6 100644
--- a/net/sched/sch_htb.c
+++ b/net/sched/sch_htb.c
@@ -1144,7 +1144,7 @@ htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d)
1144 cl->xstats.tokens = PSCHED_NS2TICKS(cl->tokens); 1144 cl->xstats.tokens = PSCHED_NS2TICKS(cl->tokens);
1145 cl->xstats.ctokens = PSCHED_NS2TICKS(cl->ctokens); 1145 cl->xstats.ctokens = PSCHED_NS2TICKS(cl->ctokens);
1146 1146
1147 if (gnet_stats_copy_basic(d, &cl->bstats) < 0 || 1147 if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
1148 gnet_stats_copy_rate_est(d, NULL, &cl->rate_est) < 0 || 1148 gnet_stats_copy_rate_est(d, NULL, &cl->rate_est) < 0 ||
1149 gnet_stats_copy_queue(d, &cl->qstats) < 0) 1149 gnet_stats_copy_queue(d, &cl->qstats) < 0)
1150 return -1; 1150 return -1;
@@ -1402,7 +1402,8 @@ static int htb_change_class(struct Qdisc *sch, u32 classid,
1402 goto failure; 1402 goto failure;
1403 1403
1404 if (htb_rate_est || tca[TCA_RATE]) { 1404 if (htb_rate_est || tca[TCA_RATE]) {
1405 err = gen_new_estimator(&cl->bstats, &cl->rate_est, 1405 err = gen_new_estimator(&cl->bstats, NULL,
1406 &cl->rate_est,
1406 qdisc_root_sleeping_lock(sch), 1407 qdisc_root_sleeping_lock(sch),
1407 tca[TCA_RATE] ? : &est.nla); 1408 tca[TCA_RATE] ? : &est.nla);
1408 if (err) { 1409 if (err) {
@@ -1464,8 +1465,11 @@ static int htb_change_class(struct Qdisc *sch, u32 classid,
1464 parent->children++; 1465 parent->children++;
1465 } else { 1466 } else {
1466 if (tca[TCA_RATE]) { 1467 if (tca[TCA_RATE]) {
1467 err = gen_replace_estimator(&cl->bstats, &cl->rate_est, 1468 spinlock_t *lock = qdisc_root_sleeping_lock(sch);
1468 qdisc_root_sleeping_lock(sch), 1469
1470 err = gen_replace_estimator(&cl->bstats, NULL,
1471 &cl->rate_est,
1472 lock,
1469 tca[TCA_RATE]); 1473 tca[TCA_RATE]);
1470 if (err) 1474 if (err)
1471 return err; 1475 return err;
diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c
index a8b2864a696b..d3a27fb607af 100644
--- a/net/sched/sch_mq.c
+++ b/net/sched/sch_mq.c
@@ -201,7 +201,7 @@ static int mq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
201 201
202 sch = dev_queue->qdisc_sleeping; 202 sch = dev_queue->qdisc_sleeping;
203 sch->qstats.qlen = sch->q.qlen; 203 sch->qstats.qlen = sch->q.qlen;
204 if (gnet_stats_copy_basic(d, &sch->bstats) < 0 || 204 if (gnet_stats_copy_basic(d, NULL, &sch->bstats) < 0 ||
205 gnet_stats_copy_queue(d, &sch->qstats) < 0) 205 gnet_stats_copy_queue(d, &sch->qstats) < 0)
206 return -1; 206 return -1;
207 return 0; 207 return 0;
diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c
index 37e7d25d21f1..8917372fddc6 100644
--- a/net/sched/sch_mqprio.c
+++ b/net/sched/sch_mqprio.c
@@ -355,7 +355,7 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
355 } 355 }
356 /* Reclaim root sleeping lock before completing stats */ 356 /* Reclaim root sleeping lock before completing stats */
357 spin_lock_bh(d->lock); 357 spin_lock_bh(d->lock);
358 if (gnet_stats_copy_basic(d, &bstats) < 0 || 358 if (gnet_stats_copy_basic(d, NULL, &bstats) < 0 ||
359 gnet_stats_copy_queue(d, &qstats) < 0) 359 gnet_stats_copy_queue(d, &qstats) < 0)
360 return -1; 360 return -1;
361 } else { 361 } else {
@@ -363,7 +363,7 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
363 363
364 sch = dev_queue->qdisc_sleeping; 364 sch = dev_queue->qdisc_sleeping;
365 sch->qstats.qlen = sch->q.qlen; 365 sch->qstats.qlen = sch->q.qlen;
366 if (gnet_stats_copy_basic(d, &sch->bstats) < 0 || 366 if (gnet_stats_copy_basic(d, NULL, &sch->bstats) < 0 ||
367 gnet_stats_copy_queue(d, &sch->qstats) < 0) 367 gnet_stats_copy_queue(d, &sch->qstats) < 0)
368 return -1; 368 return -1;
369 } 369 }
diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c
index c0466c1840f3..4adbf7fefc09 100644
--- a/net/sched/sch_multiq.c
+++ b/net/sched/sch_multiq.c
@@ -361,7 +361,7 @@ static int multiq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
361 361
362 cl_q = q->queues[cl - 1]; 362 cl_q = q->queues[cl - 1];
363 cl_q->qstats.qlen = cl_q->q.qlen; 363 cl_q->qstats.qlen = cl_q->q.qlen;
364 if (gnet_stats_copy_basic(d, &cl_q->bstats) < 0 || 364 if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats) < 0 ||
365 gnet_stats_copy_queue(d, &cl_q->qstats) < 0) 365 gnet_stats_copy_queue(d, &cl_q->qstats) < 0)
366 return -1; 366 return -1;
367 367
diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c
index 03ef99e52a5c..68a8f25e30c3 100644
--- a/net/sched/sch_prio.c
+++ b/net/sched/sch_prio.c
@@ -325,7 +325,7 @@ static int prio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
325 325
326 cl_q = q->queues[cl - 1]; 326 cl_q = q->queues[cl - 1];
327 cl_q->qstats.qlen = cl_q->q.qlen; 327 cl_q->qstats.qlen = cl_q->q.qlen;
328 if (gnet_stats_copy_basic(d, &cl_q->bstats) < 0 || 328 if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats) < 0 ||
329 gnet_stats_copy_queue(d, &cl_q->qstats) < 0) 329 gnet_stats_copy_queue(d, &cl_q->qstats) < 0)
330 return -1; 330 return -1;
331 331
diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c
index 602ea01a4ddd..d59f8574540a 100644
--- a/net/sched/sch_qfq.c
+++ b/net/sched/sch_qfq.c
@@ -459,7 +459,8 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
459 459
460 if (cl != NULL) { /* modify existing class */ 460 if (cl != NULL) { /* modify existing class */
461 if (tca[TCA_RATE]) { 461 if (tca[TCA_RATE]) {
462 err = gen_replace_estimator(&cl->bstats, &cl->rate_est, 462 err = gen_replace_estimator(&cl->bstats, NULL,
463 &cl->rate_est,
463 qdisc_root_sleeping_lock(sch), 464 qdisc_root_sleeping_lock(sch),
464 tca[TCA_RATE]); 465 tca[TCA_RATE]);
465 if (err) 466 if (err)
@@ -484,7 +485,8 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
484 cl->qdisc = &noop_qdisc; 485 cl->qdisc = &noop_qdisc;
485 486
486 if (tca[TCA_RATE]) { 487 if (tca[TCA_RATE]) {
487 err = gen_new_estimator(&cl->bstats, &cl->rate_est, 488 err = gen_new_estimator(&cl->bstats, NULL,
489 &cl->rate_est,
488 qdisc_root_sleeping_lock(sch), 490 qdisc_root_sleeping_lock(sch),
489 tca[TCA_RATE]); 491 tca[TCA_RATE]);
490 if (err) 492 if (err)
@@ -667,7 +669,7 @@ static int qfq_dump_class_stats(struct Qdisc *sch, unsigned long arg,
667 xstats.weight = cl->agg->class_weight; 669 xstats.weight = cl->agg->class_weight;
668 xstats.lmax = cl->agg->lmax; 670 xstats.lmax = cl->agg->lmax;
669 671
670 if (gnet_stats_copy_basic(d, &cl->bstats) < 0 || 672 if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
671 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 || 673 gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
672 gnet_stats_copy_queue(d, &cl->qdisc->qstats) < 0) 674 gnet_stats_copy_queue(d, &cl->qdisc->qstats) < 0)
673 return -1; 675 return -1;
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-13 09:33:31 -0400