3 files changed, 19 insertions, 27 deletions

diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index 9481f2c142e2..e7eb4aa6ccc9 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -51,7 +51,10 @@ struct qdisc_size_table {
 struct qdisc_skb_head {
         struct sk_buff  *head;
         struct sk_buff  *tail;
-        __u32           qlen;
+        union {
+                u32             qlen;
+                atomic_t        atomic_qlen;
+        };
         spinlock_t      lock;
 };
 
@@ -408,27 +411,19 @@ static inline void qdisc_cb_private_validate(const struct sk_buff *skb, int sz)
         BUILD_BUG_ON(sizeof(qcb->data) < sz);
 }
 
-static inline int qdisc_qlen_cpu(const struct Qdisc *q)
-{
-        return this_cpu_ptr(q->cpu_qstats)->qlen;
-}
-
 static inline int qdisc_qlen(const struct Qdisc *q)
 {
         return q->q.qlen;
 }
 
-static inline int qdisc_qlen_sum(const struct Qdisc *q)
+static inline u32 qdisc_qlen_sum(const struct Qdisc *q)
 {
-        __u32 qlen = q->qstats.qlen;
-        int i;
+        u32 qlen = q->qstats.qlen;
 
-        if (q->flags & TCQ_F_NOLOCK) {
-                for_each_possible_cpu(i)
-                        qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen;
-        } else {
+        if (q->flags & TCQ_F_NOLOCK)
+                qlen += atomic_read(&q->q.atomic_qlen);
+        else
                 qlen += q->q.qlen;
-        }
 
         return qlen;
 }
@@ -825,14 +820,14 @@ static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch,
         this_cpu_add(sch->cpu_qstats->backlog, qdisc_pkt_len(skb));
 }
 
-static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch)
+static inline void qdisc_qstats_atomic_qlen_inc(struct Qdisc *sch)
 {
-        this_cpu_inc(sch->cpu_qstats->qlen);
+        atomic_inc(&sch->q.atomic_qlen);
 }
 
-static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch)
+static inline void qdisc_qstats_atomic_qlen_dec(struct Qdisc *sch)
 {
-        this_cpu_dec(sch->cpu_qstats->qlen);
+        atomic_dec(&sch->q.atomic_qlen);
 }
 
 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch)
diff --git a/net/core/gen_stats.c b/net/core/gen_stats.c
index 9bf1b9ad1780..ac679f74ba47 100644
--- a/net/core/gen_stats.c
+++ b/net/core/gen_stats.c
@@ -291,7 +291,6 @@ __gnet_stats_copy_queue_cpu(struct gnet_stats_queue *qstats,
         for_each_possible_cpu(i) {
                 const struct gnet_stats_queue *qcpu = per_cpu_ptr(q, i);
 
-                qstats->qlen = 0;
                 qstats->backlog += qcpu->backlog;
                 qstats->drops += qcpu->drops;
                 qstats->requeues += qcpu->requeues;
@@ -307,7 +306,6 @@ void __gnet_stats_copy_queue(struct gnet_stats_queue *qstats,
         if (cpu) {
                 __gnet_stats_copy_queue_cpu(qstats, cpu);
         } else {
-                qstats->qlen = q->qlen;
                 qstats->backlog = q->backlog;
                 qstats->drops = q->drops;
                 qstats->requeues = q->requeues;
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index 968a85fe4d4a..de31f2f3b973 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -68,7 +68,7 @@ static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q)
                         skb = __skb_dequeue(&q->skb_bad_txq);
                         if (qdisc_is_percpu_stats(q)) {
                                 qdisc_qstats_cpu_backlog_dec(q, skb);
-                                qdisc_qstats_cpu_qlen_dec(q);
+                                qdisc_qstats_atomic_qlen_dec(q);
                         } else {
                                 qdisc_qstats_backlog_dec(q, skb);
                                 q->q.qlen--;
@@ -108,7 +108,7 @@ static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q,
 
         if (qdisc_is_percpu_stats(q)) {
                 qdisc_qstats_cpu_backlog_inc(q, skb);
-                qdisc_qstats_cpu_qlen_inc(q);
+                qdisc_qstats_atomic_qlen_inc(q);
         } else {
                 qdisc_qstats_backlog_inc(q, skb);
                 q->q.qlen++;
@@ -147,7 +147,7 @@ static inline int dev_requeue_skb_locked(struct sk_buff *skb, struct Qdisc *q)
 
                 qdisc_qstats_cpu_requeues_inc(q);
                 qdisc_qstats_cpu_backlog_inc(q, skb);
-                qdisc_qstats_cpu_qlen_inc(q);
+                qdisc_qstats_atomic_qlen_inc(q);
 
                 skb = next;
         }
@@ -252,7 +252,7 @@ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
                         skb = __skb_dequeue(&q->gso_skb);
                         if (qdisc_is_percpu_stats(q)) {
                                 qdisc_qstats_cpu_backlog_dec(q, skb);
-                                qdisc_qstats_cpu_qlen_dec(q);
+                                qdisc_qstats_atomic_qlen_dec(q);
                         } else {
                                 qdisc_qstats_backlog_dec(q, skb);
                                 q->q.qlen--;
@@ -645,7 +645,7 @@ static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
         if (unlikely(err))
                 return qdisc_drop_cpu(skb, qdisc, to_free);
 
-        qdisc_qstats_cpu_qlen_inc(qdisc);
+        qdisc_qstats_atomic_qlen_inc(qdisc);
         /* Note: skb can not be used after skb_array_produce(),
          * so we better not use qdisc_qstats_cpu_backlog_inc()
          */
@@ -670,7 +670,7 @@ static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
         if (likely(skb)) {
                 qdisc_qstats_cpu_backlog_dec(qdisc, skb);
                 qdisc_bstats_cpu_update(qdisc, skb);
-                qdisc_qstats_cpu_qlen_dec(qdisc);
+                qdisc_qstats_atomic_qlen_dec(qdisc);
         }
 
         return skb;
@@ -714,7 +714,6 @@ static void pfifo_fast_reset(struct Qdisc *qdisc)
                 struct gnet_stats_queue *q = per_cpu_ptr(qdisc->cpu_qstats, i);
 
                 q->backlog = 0;
-                q->qlen = 0;
         }
 }