14 files changed, 65 insertions, 33 deletions

diff --git a/net/sched/act_api.c b/net/sched/act_api.c
index 648778aef1a2..eca4cf9ece2f 100644
--- a/net/sched/act_api.c
+++ b/net/sched/act_api.c
@@ -252,7 +252,8 @@ int tcf_hash_create(u32 index, struct nlattr *est, struct tc_action *a,
         p->tcfc_tm.install = jiffies;
         p->tcfc_tm.lastuse = jiffies;
         if (est) {
-                int err = gen_new_estimator(&p->tcfc_bstats, &p->tcfc_rate_est,
+                int err = gen_new_estimator(&p->tcfc_bstats, NULL,
+                                            &p->tcfc_rate_est,
                                             &p->tcfc_lock, est);
                 if (err) {
                         kfree(p);
@@ -619,7 +620,7 @@ int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *a,
         if (err < 0)
                 goto errout;
 
-        if (gnet_stats_copy_basic(&d, &p->tcfc_bstats) < 0 ||
+        if (gnet_stats_copy_basic(&d, NULL, &p->tcfc_bstats) < 0 ||
             gnet_stats_copy_rate_est(&d, &p->tcfc_bstats,
                                      &p->tcfc_rate_est) < 0 ||
             gnet_stats_copy_queue(&d, &p->tcfc_qstats) < 0)
diff --git a/net/sched/act_police.c b/net/sched/act_police.c
index f32bcb094915..69791ca77a05 100644
--- a/net/sched/act_police.c
+++ b/net/sched/act_police.c
@@ -178,7 +178,7 @@ override:
 
         spin_lock_bh(&police->tcf_lock);
         if (est) {
-                err = gen_replace_estimator(&police->tcf_bstats,
+                err = gen_replace_estimator(&police->tcf_bstats, NULL,
                                             &police->tcf_rate_est,
                                             &police->tcf_lock, est);
                 if (err)
diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
index 15e7beee266c..a95e3b48fa51 100644
--- a/net/sched/sch_api.c
+++ b/net/sched/sch_api.c
@@ -942,6 +942,13 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
         sch->handle = handle;
 
         if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) {
+                if (qdisc_is_percpu_stats(sch)) {
+                        sch->cpu_bstats =
+                                alloc_percpu(struct gnet_stats_basic_cpu);
+                        if (!sch->cpu_bstats)
+                                goto err_out4;
+                }
+
                 if (tca[TCA_STAB]) {
                         stab = qdisc_get_stab(tca[TCA_STAB]);
                         if (IS_ERR(stab)) {
@@ -964,8 +971,11 @@ qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
                         else
                                 root_lock = qdisc_lock(sch);
 
-                        err = gen_new_estimator(&sch->bstats, &sch->rate_est,
-                                                root_lock, tca[TCA_RATE]);
+                        err = gen_new_estimator(&sch->bstats,
+                                                sch->cpu_bstats,
+                                                &sch->rate_est,
+                                                root_lock,
+                                                tca[TCA_RATE]);
                         if (err)
                                 goto err_out4;
                 }
@@ -984,6 +994,7 @@ err_out:
         return NULL;
 
 err_out4:
+        free_percpu(sch->cpu_bstats);
         /*
          * Any broken qdiscs that would require a ops->reset() here?
          * The qdisc was never in action so it shouldn't be necessary.
@@ -1022,9 +1033,11 @@ static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)
                    because change can't be undone. */
                 if (sch->flags & TCQ_F_MQROOT)
                         goto out;
-                gen_replace_estimator(&sch->bstats, &sch->rate_est,
-                                            qdisc_root_sleeping_lock(sch),
-                                            tca[TCA_RATE]);
+                gen_replace_estimator(&sch->bstats,
+                                      sch->cpu_bstats,
+                                      &sch->rate_est,
+                                      qdisc_root_sleeping_lock(sch),
+                                      tca[TCA_RATE]);
         }
 out:
         return 0;
@@ -1299,6 +1312,7 @@ graft:
 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
                          u32 portid, u32 seq, u16 flags, int event)
 {
+        struct gnet_stats_basic_cpu __percpu *cpu_bstats = NULL;
         struct tcmsg *tcm;
         struct nlmsghdr  *nlh;
         unsigned char *b = skb_tail_pointer(skb);
@@ -1334,7 +1348,10 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
         if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
                 goto nla_put_failure;
 
-        if (gnet_stats_copy_basic(&d, &q->bstats) < 0 ||
+        if (qdisc_is_percpu_stats(q))
+                cpu_bstats = q->cpu_bstats;
+
+        if (gnet_stats_copy_basic(&d, cpu_bstats, &q->bstats) < 0 ||
             gnet_stats_copy_rate_est(&d, &q->bstats, &q->rate_est) < 0 ||
             gnet_stats_copy_queue(&d, &q->qstats) < 0)
                 goto nla_put_failure;
diff --git a/net/sched/sch_atm.c b/net/sched/sch_atm.c
index c398f9c3dbdd..01017663e5d8 100644
--- a/net/sched/sch_atm.c
+++ b/net/sched/sch_atm.c
@@ -639,7 +639,7 @@ atm_tc_dump_class_stats(struct Qdisc *sch, unsigned long arg,
 
         flow->qstats.qlen = flow->q->q.qlen;
 
-        if (gnet_stats_copy_basic(d, &flow->bstats) < 0 ||
+        if (gnet_stats_copy_basic(d, NULL, &flow->bstats) < 0 ||
             gnet_stats_copy_queue(d, &flow->qstats) < 0)
                 return -1;
 
diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c
index d2cd981ba60d..22a3a029a911 100644
--- a/net/sched/sch_cbq.c
+++ b/net/sched/sch_cbq.c
@@ -1601,7 +1601,7 @@ cbq_dump_class_stats(struct Qdisc *sch, unsigned long arg,
         if (cl->undertime != PSCHED_PASTPERFECT)
                 cl->xstats.undertime = cl->undertime - q->now;
 
-        if (gnet_stats_copy_basic(d, &cl->bstats) < 0 ||
+        if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
             gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
             gnet_stats_copy_queue(d, &cl->qstats) < 0)
                 return -1;
@@ -1759,7 +1759,8 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t
                 }
 
                 if (tca[TCA_RATE]) {
-                        err = gen_replace_estimator(&cl->bstats, &cl->rate_est,
+                        err = gen_replace_estimator(&cl->bstats, NULL,
+                                                    &cl->rate_est,
                                                     qdisc_root_sleeping_lock(sch),
                                                     tca[TCA_RATE]);
                         if (err) {
@@ -1852,7 +1853,7 @@ cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **t
                 goto failure;
 
         if (tca[TCA_RATE]) {
-                err = gen_new_estimator(&cl->bstats, &cl->rate_est,
+                err = gen_new_estimator(&cl->bstats, NULL, &cl->rate_est,
                                         qdisc_root_sleeping_lock(sch),
                                         tca[TCA_RATE]);
                 if (err) {
diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c
index d8b5ccfd248a..7a6243c5d270 100644
--- a/net/sched/sch_drr.c
+++ b/net/sched/sch_drr.c
@@ -88,7 +88,8 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
 
         if (cl != NULL) {
                 if (tca[TCA_RATE]) {
-                        err = gen_replace_estimator(&cl->bstats, &cl->rate_est,
+                        err = gen_replace_estimator(&cl->bstats, NULL,
+                                                    &cl->rate_est,
                                                     qdisc_root_sleeping_lock(sch),
                                                     tca[TCA_RATE]);
                         if (err)
@@ -116,7 +117,7 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
                 cl->qdisc = &noop_qdisc;
 
         if (tca[TCA_RATE]) {
-                err = gen_replace_estimator(&cl->bstats, &cl->rate_est,
+                err = gen_replace_estimator(&cl->bstats, NULL, &cl->rate_est,
                                             qdisc_root_sleeping_lock(sch),
                                             tca[TCA_RATE]);
                 if (err) {
@@ -282,7 +283,7 @@ static int drr_dump_class_stats(struct Qdisc *sch, unsigned long arg,
                 cl->qdisc->qstats.qlen = cl->qdisc->q.qlen;
         }
 
-        if (gnet_stats_copy_basic(d, &cl->bstats) < 0 ||
+        if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
             gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
             gnet_stats_copy_queue(d, &cl->qdisc->qstats) < 0)
                 return -1;
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index 11b28f651ad1..7c8e5d73d433 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -632,6 +632,9 @@ static void qdisc_rcu_free(struct rcu_head *head)
 {
         struct Qdisc *qdisc = container_of(head, struct Qdisc, rcu_head);
 
+        if (qdisc_is_percpu_stats(qdisc))
+                free_percpu(qdisc->cpu_bstats);
+
         kfree((char *) qdisc - qdisc->padded);
 }
 
diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c
index 04b0de4c68b5..209b966b2eed 100644
--- a/net/sched/sch_hfsc.c
+++ b/net/sched/sch_hfsc.c
@@ -1014,9 +1014,12 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
                 cur_time = psched_get_time();
 
                 if (tca[TCA_RATE]) {
-                        err = gen_replace_estimator(&cl->bstats, &cl->rate_est,
-                                              qdisc_root_sleeping_lock(sch),
-                                              tca[TCA_RATE]);
+                        spinlock_t *lock = qdisc_root_sleeping_lock(sch);
+
+                        err = gen_replace_estimator(&cl->bstats, NULL,
+                                                    &cl->rate_est,
+                                                    lock,
+                                                    tca[TCA_RATE]);
                         if (err)
                                 return err;
                 }
@@ -1063,7 +1066,7 @@ hfsc_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
                 return -ENOBUFS;
 
         if (tca[TCA_RATE]) {
-                err = gen_new_estimator(&cl->bstats, &cl->rate_est,
+                err = gen_new_estimator(&cl->bstats, NULL, &cl->rate_est,
                                         qdisc_root_sleeping_lock(sch),
                                         tca[TCA_RATE]);
                 if (err) {
@@ -1374,7 +1377,7 @@ hfsc_dump_class_stats(struct Qdisc *sch, unsigned long arg,
         xstats.work    = cl->cl_total;
         xstats.rtwork  = cl->cl_cumul;
 
-        if (gnet_stats_copy_basic(d, &cl->bstats) < 0 ||
+        if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
             gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
             gnet_stats_copy_queue(d, &cl->qstats) < 0)
                 return -1;
diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c
index 063e953d9848..0256dee69bd6 100644
--- a/net/sched/sch_htb.c
+++ b/net/sched/sch_htb.c
@@ -1144,7 +1144,7 @@ htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d)
         cl->xstats.tokens = PSCHED_NS2TICKS(cl->tokens);
         cl->xstats.ctokens = PSCHED_NS2TICKS(cl->ctokens);
 
-        if (gnet_stats_copy_basic(d, &cl->bstats) < 0 ||
+        if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
             gnet_stats_copy_rate_est(d, NULL, &cl->rate_est) < 0 ||
             gnet_stats_copy_queue(d, &cl->qstats) < 0)
                 return -1;
@@ -1402,7 +1402,8 @@ static int htb_change_class(struct Qdisc *sch, u32 classid,
                         goto failure;
 
                 if (htb_rate_est || tca[TCA_RATE]) {
-                        err = gen_new_estimator(&cl->bstats, &cl->rate_est,
+                        err = gen_new_estimator(&cl->bstats, NULL,
+                                                &cl->rate_est,
                                                 qdisc_root_sleeping_lock(sch),
                                                 tca[TCA_RATE] ? : &est.nla);
                         if (err) {
@@ -1464,8 +1465,11 @@ static int htb_change_class(struct Qdisc *sch, u32 classid,
                         parent->children++;
         } else {
                 if (tca[TCA_RATE]) {
-                        err = gen_replace_estimator(&cl->bstats, &cl->rate_est,
-                                                    qdisc_root_sleeping_lock(sch),
+                        spinlock_t *lock = qdisc_root_sleeping_lock(sch);
+
+                        err = gen_replace_estimator(&cl->bstats, NULL,
+                                                    &cl->rate_est,
+                                                    lock,
                                                     tca[TCA_RATE]);
                         if (err)
                                 return err;
diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c
index a8b2864a696b..d3a27fb607af 100644
--- a/net/sched/sch_mq.c
+++ b/net/sched/sch_mq.c
@@ -201,7 +201,7 @@ static int mq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
 
         sch = dev_queue->qdisc_sleeping;
         sch->qstats.qlen = sch->q.qlen;
-        if (gnet_stats_copy_basic(d, &sch->bstats) < 0 ||
+        if (gnet_stats_copy_basic(d, NULL, &sch->bstats) < 0 ||
             gnet_stats_copy_queue(d, &sch->qstats) < 0)
                 return -1;
         return 0;
diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c
index 37e7d25d21f1..8917372fddc6 100644
--- a/net/sched/sch_mqprio.c
+++ b/net/sched/sch_mqprio.c
@@ -355,7 +355,7 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
                 }
                 /* Reclaim root sleeping lock before completing stats */
                 spin_lock_bh(d->lock);
-                if (gnet_stats_copy_basic(d, &bstats) < 0 ||
+                if (gnet_stats_copy_basic(d, NULL, &bstats) < 0 ||
                     gnet_stats_copy_queue(d, &qstats) < 0)
                         return -1;
         } else {
@@ -363,7 +363,7 @@ static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
 
                 sch = dev_queue->qdisc_sleeping;
                 sch->qstats.qlen = sch->q.qlen;
-                if (gnet_stats_copy_basic(d, &sch->bstats) < 0 ||
+                if (gnet_stats_copy_basic(d, NULL, &sch->bstats) < 0 ||
                     gnet_stats_copy_queue(d, &sch->qstats) < 0)
                         return -1;
         }
diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c
index c0466c1840f3..4adbf7fefc09 100644
--- a/net/sched/sch_multiq.c
+++ b/net/sched/sch_multiq.c
@@ -361,7 +361,7 @@ static int multiq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
 
         cl_q = q->queues[cl - 1];
         cl_q->qstats.qlen = cl_q->q.qlen;
-        if (gnet_stats_copy_basic(d, &cl_q->bstats) < 0 ||
+        if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats) < 0 ||
             gnet_stats_copy_queue(d, &cl_q->qstats) < 0)
                 return -1;
 
diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c
index 03ef99e52a5c..68a8f25e30c3 100644
--- a/net/sched/sch_prio.c
+++ b/net/sched/sch_prio.c
@@ -325,7 +325,7 @@ static int prio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
 
         cl_q = q->queues[cl - 1];
         cl_q->qstats.qlen = cl_q->q.qlen;
-        if (gnet_stats_copy_basic(d, &cl_q->bstats) < 0 ||
+        if (gnet_stats_copy_basic(d, NULL, &cl_q->bstats) < 0 ||
             gnet_stats_copy_queue(d, &cl_q->qstats) < 0)
                 return -1;
 
diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c
index 602ea01a4ddd..d59f8574540a 100644
--- a/net/sched/sch_qfq.c
+++ b/net/sched/sch_qfq.c
@@ -459,7 +459,8 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
 
         if (cl != NULL) { /* modify existing class */
                 if (tca[TCA_RATE]) {
-                        err = gen_replace_estimator(&cl->bstats, &cl->rate_est,
+                        err = gen_replace_estimator(&cl->bstats, NULL,
+                                                    &cl->rate_est,
                                                     qdisc_root_sleeping_lock(sch),
                                                     tca[TCA_RATE]);
                         if (err)
@@ -484,7 +485,8 @@ static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
                 cl->qdisc = &noop_qdisc;
 
         if (tca[TCA_RATE]) {
-                err = gen_new_estimator(&cl->bstats, &cl->rate_est,
+                err = gen_new_estimator(&cl->bstats, NULL,
+                                        &cl->rate_est,
                                         qdisc_root_sleeping_lock(sch),
                                         tca[TCA_RATE]);
                 if (err)
@@ -667,7 +669,7 @@ static int qfq_dump_class_stats(struct Qdisc *sch, unsigned long arg,
         xstats.weight = cl->agg->class_weight;
         xstats.lmax = cl->agg->lmax;
 
-        if (gnet_stats_copy_basic(d, &cl->bstats) < 0 ||
+        if (gnet_stats_copy_basic(d, NULL, &cl->bstats) < 0 ||
             gnet_stats_copy_rate_est(d, &cl->bstats, &cl->rate_est) < 0 ||
             gnet_stats_copy_queue(d, &cl->qdisc->qstats) < 0)
                 return -1;