From a02cec2155fbea457eca8881870fd2de1a4c4c76 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 22 Sep 2010 20:43:57 +0000 Subject: net: return operator cleanup Change "return (EXPR);" to "return EXPR;" return is not a function, parentheses are not required. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/sch_generic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/net/sch_generic.h') diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 3c8728aaab4e..eda8808fdacd 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -601,7 +601,7 @@ static inline u32 qdisc_l2t(struct qdisc_rate_table* rtab, unsigned int pktlen) slot = 0; slot >>= rtab->rate.cell_log; if (slot > 255) - return (rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF]); + return rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF]; return rtab->data[slot]; } -- cgit v1.2.2 From 3511c9132f8b1e1b5634e41a3331c44b0c13be70 Mon Sep 17 00:00:00 2001 From: Changli Gao Date: Sat, 16 Oct 2010 13:04:08 +0000 Subject: net_sched: remove the unused parameter of qdisc_create_dflt() The first parameter dev isn't in use in qdisc_create_dflt(). Signed-off-by: Changli Gao Acked-by: Jamal Hadi Salim Signed-off-by: David S. Miller --- include/net/sch_generic.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include/net/sch_generic.h') diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index eda8808fdacd..ea1f8a83160d 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -328,8 +328,7 @@ extern void qdisc_destroy(struct Qdisc *qdisc); extern void qdisc_tree_decrease_qlen(struct Qdisc *qdisc, unsigned int n); extern struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, struct Qdisc_ops *ops); -extern struct Qdisc *qdisc_create_dflt(struct net_device *dev, - struct netdev_queue *dev_queue, +extern struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, struct Qdisc_ops *ops, u32 parentid); extern void qdisc_calculate_pkt_len(struct sk_buff *skb, struct qdisc_size_table *stab); -- cgit v1.2.2 From 443457242beb6716b43db4d62fe148eab5515505 Mon Sep 17 00:00:00 2001 From: Octavian Purdila Date: Mon, 13 Dec 2010 12:44:07 +0000 Subject: net: factorize sync-rcu call in unregister_netdevice_many Add dev_close_many and dev_deactivate_many to factorize another sync-rcu operation on the netdevice unregister path. $ modprobe dummy numdummies=10000 $ ip link set dev dummy* up $ time rmmod dummy Without the patch With the patch real 0m 24.63s real 0m 5.15s user 0m 0.00s user 0m 0.00s sys 0m 6.05s sys 0m 5.14s Signed-off-by: Octavian Purdila Signed-off-by: David S. Miller --- include/net/sch_generic.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/net/sch_generic.h') diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index ea1f8a83160d..786cc396cb4a 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -321,6 +321,7 @@ extern void dev_init_scheduler(struct net_device *dev); extern void dev_shutdown(struct net_device *dev); extern void dev_activate(struct net_device *dev); extern void dev_deactivate(struct net_device *dev); +extern void dev_deactivate_many(struct list_head *head); extern struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, struct Qdisc *qdisc); extern void qdisc_reset(struct Qdisc *qdisc); -- cgit v1.2.2 From 173021072e86a0a5b3d2271347493a3e0d5f68e8 Mon Sep 17 00:00:00 2001 From: Changli Gao Date: Mon, 20 Dec 2010 04:35:30 +0000 Subject: net_sched: always clone skbs Pawel reported a panic related to handling shared skbs in ixgbe incorrectly. So we need to revert my previous patch to work around this bug. Instead of reverting the patch completely, I just revert the essential lines, so we can add the previous optimization back more easily in future. commit 3511c9132f8b1e1b5634e41a3331c44b0c13be70 Author: Changli Gao Date: Sat Oct 16 13:04:08 2010 +0000 net_sched: remove the unused parameter of qdisc_create_dflt() Reported-by: Pawel Staszewski Signed-off-by: Changli Gao Acked-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/sch_generic.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'include/net/sch_generic.h') diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index ea1f8a83160d..79f34e2b752f 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -610,11 +610,7 @@ static inline struct sk_buff *skb_act_clone(struct sk_buff *skb, gfp_t gfp_mask, { struct sk_buff *n; - if ((action == TC_ACT_STOLEN || action == TC_ACT_QUEUED) && - !skb_shared(skb)) - n = skb_get(skb); - else - n = skb_clone(skb, gfp_mask); + n = skb_clone(skb, gfp_mask); if (n) { n->tc_verd = SET_TC_VERD(n->tc_verd, 0); -- cgit v1.2.2 From bfe0d0298f2a67d94d58c39ea904a999aeeb7c3c Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sun, 9 Jan 2011 08:30:54 +0000 Subject: net_sched: factorize qdisc stats handling HTB takes into account skb is segmented in stats updates. Generalize this to all schedulers. They should use qdisc_bstats_update() helper instead of manipulating bstats.bytes and bstats.packets Add bstats_update() helper too for classes that use gnet_stats_basic_packed fields. Note : Right now, TCQ_F_CAN_BYPASS shortcurt can be taken only if no stab is setup on qdisc. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/sch_generic.h | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'include/net/sch_generic.h') diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 0af57ebae762..e9eee99d8b1f 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -207,7 +207,7 @@ static inline int qdisc_qlen(struct Qdisc *q) return q->q.qlen; } -static inline struct qdisc_skb_cb *qdisc_skb_cb(struct sk_buff *skb) +static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb) { return (struct qdisc_skb_cb *)skb->cb; } @@ -394,7 +394,7 @@ static inline bool qdisc_tx_is_noop(const struct net_device *dev) return true; } -static inline unsigned int qdisc_pkt_len(struct sk_buff *skb) +static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb) { return qdisc_skb_cb(skb)->pkt_len; } @@ -426,10 +426,18 @@ static inline int qdisc_enqueue_root(struct sk_buff *skb, struct Qdisc *sch) return qdisc_enqueue(skb, sch) & NET_XMIT_MASK; } -static inline void __qdisc_update_bstats(struct Qdisc *sch, unsigned int len) + +static inline void bstats_update(struct gnet_stats_basic_packed *bstats, + const struct sk_buff *skb) +{ + bstats->bytes += qdisc_pkt_len(skb); + bstats->packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; +} + +static inline void qdisc_bstats_update(struct Qdisc *sch, + const struct sk_buff *skb) { - sch->bstats.bytes += len; - sch->bstats.packets++; + bstats_update(&sch->bstats, skb); } static inline int __qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch, @@ -437,7 +445,7 @@ static inline int __qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch, { __skb_queue_tail(list, skb); sch->qstats.backlog += qdisc_pkt_len(skb); - __qdisc_update_bstats(sch, qdisc_pkt_len(skb)); + qdisc_bstats_update(sch, skb); return NET_XMIT_SUCCESS; } -- cgit v1.2.2 From fd245a4adb5288eac37250875f237c40a20a1944 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 20 Jan 2011 05:27:16 +0000 Subject: net_sched: move TCQ_F_THROTTLED flag In commit 371121057607e (net: QDISC_STATE_RUNNING dont need atomic bit ops) I moved QDISC_STATE_RUNNING flag to __state container, located in the cache line containing qdisc lock and often dirtied fields. I now move TCQ_F_THROTTLED bit too, so that we let first cache line read mostly, and shared by all cpus. This should speedup HTB/CBQ for example. Not using test_bit()/__clear_bit()/__test_and_set_bit allows to use an "unsigned int" for __state container, reducing by 8 bytes Qdisc size. Introduce helpers to hide implementation details. Signed-off-by: Eric Dumazet CC: Patrick McHardy CC: Jesper Dangaard Brouer CC: Jarek Poplawski CC: Jamal Hadi Salim CC: Stephen Hemminger Signed-off-by: David S. Miller --- include/net/sch_generic.h | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) (limited to 'include/net/sch_generic.h') diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index e9eee99d8b1f..f6345f55041c 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -31,7 +31,8 @@ enum qdisc_state_t { * following bits are only changed while qdisc lock is held */ enum qdisc___state_t { - __QDISC___STATE_RUNNING, + __QDISC___STATE_RUNNING = 1, + __QDISC___STATE_THROTTLED = 2, }; struct qdisc_size_table { @@ -46,10 +47,9 @@ struct Qdisc { struct sk_buff * (*dequeue)(struct Qdisc *dev); unsigned flags; #define TCQ_F_BUILTIN 1 -#define TCQ_F_THROTTLED 2 -#define TCQ_F_INGRESS 4 -#define TCQ_F_CAN_BYPASS 8 -#define TCQ_F_MQROOT 16 +#define TCQ_F_INGRESS 2 +#define TCQ_F_CAN_BYPASS 4 +#define TCQ_F_MQROOT 8 #define TCQ_F_WARN_NONWC (1 << 16) int padded; struct Qdisc_ops *ops; @@ -78,25 +78,43 @@ struct Qdisc { unsigned long state; struct sk_buff_head q; struct gnet_stats_basic_packed bstats; - unsigned long __state; + unsigned int __state; struct gnet_stats_queue qstats; struct rcu_head rcu_head; spinlock_t busylock; }; -static inline bool qdisc_is_running(struct Qdisc *qdisc) +static inline bool qdisc_is_running(const struct Qdisc *qdisc) { - return test_bit(__QDISC___STATE_RUNNING, &qdisc->__state); + return (qdisc->__state & __QDISC___STATE_RUNNING) ? true : false; } static inline bool qdisc_run_begin(struct Qdisc *qdisc) { - return !__test_and_set_bit(__QDISC___STATE_RUNNING, &qdisc->__state); + if (qdisc_is_running(qdisc)) + return false; + qdisc->__state |= __QDISC___STATE_RUNNING; + return true; } static inline void qdisc_run_end(struct Qdisc *qdisc) { - __clear_bit(__QDISC___STATE_RUNNING, &qdisc->__state); + qdisc->__state &= ~__QDISC___STATE_RUNNING; +} + +static inline bool qdisc_is_throttled(const struct Qdisc *qdisc) +{ + return (qdisc->__state & __QDISC___STATE_THROTTLED) ? true : false; +} + +static inline void qdisc_throttled(struct Qdisc *qdisc) +{ + qdisc->__state |= __QDISC___STATE_THROTTLED; +} + +static inline void qdisc_unthrottled(struct Qdisc *qdisc) +{ + qdisc->__state &= ~__QDISC___STATE_THROTTLED; } struct Qdisc_class_ops { -- cgit v1.2.2 From a2da570d62fcb9e8816f6920e1ec02c706b289fa Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 20 Jan 2011 03:48:19 +0000 Subject: net_sched: RCU conversion of stab This patch converts stab qdisc management to RCU, so that we can perform the qdisc_calculate_pkt_len() call before getting qdisc lock. This shortens the lock's held time in __dev_xmit_skb(). This permits more qdiscs to get TCQ_F_CAN_BYPASS status, avoiding lot of cache misses and so reducing latencies. Signed-off-by: Eric Dumazet CC: Patrick McHardy CC: Jesper Dangaard Brouer CC: Jarek Poplawski CC: Jamal Hadi Salim CC: Stephen Hemminger Signed-off-by: David S. Miller --- include/net/sch_generic.h | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'include/net/sch_generic.h') diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index f6345f55041c..d531baa2506a 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -36,6 +36,7 @@ enum qdisc___state_t { }; struct qdisc_size_table { + struct rcu_head rcu; struct list_head list; struct tc_sizespec szopts; int refcnt; @@ -53,7 +54,7 @@ struct Qdisc { #define TCQ_F_WARN_NONWC (1 << 16) int padded; struct Qdisc_ops *ops; - struct qdisc_size_table *stab; + struct qdisc_size_table __rcu *stab; struct list_head list; u32 handle; u32 parent; @@ -349,8 +350,8 @@ extern struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, struct Qdisc_ops *ops); extern struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, struct Qdisc_ops *ops, u32 parentid); -extern void qdisc_calculate_pkt_len(struct sk_buff *skb, - struct qdisc_size_table *stab); +extern void __qdisc_calculate_pkt_len(struct sk_buff *skb, + const struct qdisc_size_table *stab); extern void tcf_destroy(struct tcf_proto *tp); extern void tcf_destroy_chain(struct tcf_proto **fl); @@ -429,12 +430,20 @@ enum net_xmit_qdisc_t { #define net_xmit_drop_count(e) (1) #endif -static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch) +static inline void qdisc_calculate_pkt_len(struct sk_buff *skb, + const struct Qdisc *sch) { #ifdef CONFIG_NET_SCHED - if (sch->stab) - qdisc_calculate_pkt_len(skb, sch->stab); + struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab); + + if (stab) + __qdisc_calculate_pkt_len(skb, stab); #endif +} + +static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch) +{ + qdisc_calculate_pkt_len(skb, sch); return sch->enqueue(skb, sch); } -- cgit v1.2.2 From 9190b3b3208d052d98cb601fcc192f3f71a5658b Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 20 Jan 2011 23:31:33 -0800 Subject: net_sched: accurate bytes/packets stats/rates In commit 44b8288308ac9d (net_sched: pfifo_head_drop problem), we fixed a problem with pfifo_head drops that incorrectly decreased sch->bstats.bytes and sch->bstats.packets Several qdiscs (CHOKe, SFQ, pfifo_head, ...) are able to drop a previously enqueued packet, and bstats cannot be changed, so bstats/rates are not accurate (over estimated) This patch changes the qdisc_bstats updates to be done at dequeue() time instead of enqueue() time. bstats counters no longer account for dropped frames, and rates are more correct, since enqueue() bursts dont have effect on dequeue() rate. Signed-off-by: Eric Dumazet Acked-by: Stephen Hemminger Signed-off-by: David S. Miller --- include/net/sch_generic.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'include/net/sch_generic.h') diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index e9eee99d8b1f..160a407c1963 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -445,7 +445,6 @@ static inline int __qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch, { __skb_queue_tail(list, skb); sch->qstats.backlog += qdisc_pkt_len(skb); - qdisc_bstats_update(sch, skb); return NET_XMIT_SUCCESS; } @@ -460,8 +459,10 @@ static inline struct sk_buff *__qdisc_dequeue_head(struct Qdisc *sch, { struct sk_buff *skb = __skb_dequeue(list); - if (likely(skb != NULL)) + if (likely(skb != NULL)) { sch->qstats.backlog -= qdisc_pkt_len(skb); + qdisc_bstats_update(sch, skb); + } return skb; } @@ -474,10 +475,11 @@ static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, struct sk_buff_head *list) { - struct sk_buff *skb = __qdisc_dequeue_head(sch, list); + struct sk_buff *skb = __skb_dequeue(list); if (likely(skb != NULL)) { unsigned int len = qdisc_pkt_len(skb); + sch->qstats.backlog -= len; kfree_skb(skb); return len; } -- cgit v1.2.2 From 9e924cf4078e377b84e2fd24ae11a61be7c6f5a3 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 23 Feb 2011 07:05:07 +0000 Subject: net_sched: long word align struct qdisc_skb_cb data netem_skb_cb() does : return (struct netem_skb_cb *)qdisc_skb_cb(skb)->data; Unfortunatly struct qdisc_skb_cb data is not long word aligned, so access to psched_time_t time_to_send uses a non aligned access. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/sch_generic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/net/sch_generic.h') diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 160a407c1963..04f8556313d5 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -199,7 +199,7 @@ struct tcf_proto { struct qdisc_skb_cb { unsigned int pkt_len; - char data[]; + long data[]; }; static inline int qdisc_qlen(struct Qdisc *q) -- cgit v1.2.2 From d276055c4e90a7278cd5167ba9755c9b214bcff7 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 3 Mar 2011 11:10:02 -0800 Subject: net_sched: reduce fifo qdisc size Because of various alignements [SLUB / qdisc], we use 512 bytes of memory for one {p|b}fifo qdisc, instead of 256 bytes on 64bit arches and 192 bytes on 32bit ones. Move the "u32 limit" inside "struct Qdisc" (no impact on other qdiscs) Change qdisc_alloc(), first trying a regular allocation before an oversized one. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/sch_generic.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/net/sch_generic.h') diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 16626a04cb03..1934634f8896 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -83,6 +83,7 @@ struct Qdisc { struct gnet_stats_queue qstats; struct rcu_head rcu_head; spinlock_t busylock; + u32 limit; }; static inline bool qdisc_is_running(const struct Qdisc *qdisc) -- cgit v1.2.2 From ef352e7cdf714596f51ad18809404edeaa50e8fd Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 24 Mar 2011 00:13:14 -0700 Subject: net_sched: fix THROTTLED/RUNNING race commit fd245a4adb52 (net_sched: move TCQ_F_THROTTLED flag) added a race. qdisc_watchdog() is run from softirq, so special care should be taken or we can lose one state transition (THROTTLED/RUNNING) Prior to fd245a4adb52, we were manipulating q->flags (qdisc->flags &= ~TCQ_F_THROTTLED;) and this manipulation could only race with qdisc_warn_nonwc(). Since we want to avoid atomic ops in qdisc fast path - it was the meaning of commit 371121057607e (QDISC_STATE_RUNNING dont need atomic bit ops) - fix is to move THROTTLE bit into 'state' field, this one being manipulated with SMP and IRQ safe operations. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/sch_generic.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/net/sch_generic.h') diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index a9505b6a18e3..b931f021d7ab 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -25,6 +25,7 @@ struct qdisc_rate_table { enum qdisc_state_t { __QDISC_STATE_SCHED, __QDISC_STATE_DEACTIVATED, + __QDISC_STATE_THROTTLED, }; /* @@ -32,7 +33,6 @@ enum qdisc_state_t { */ enum qdisc___state_t { __QDISC___STATE_RUNNING = 1, - __QDISC___STATE_THROTTLED = 2, }; struct qdisc_size_table { @@ -106,17 +106,17 @@ static inline void qdisc_run_end(struct Qdisc *qdisc) static inline bool qdisc_is_throttled(const struct Qdisc *qdisc) { - return (qdisc->__state & __QDISC___STATE_THROTTLED) ? true : false; + return test_bit(__QDISC_STATE_THROTTLED, &qdisc->state) ? true : false; } static inline void qdisc_throttled(struct Qdisc *qdisc) { - qdisc->__state |= __QDISC___STATE_THROTTLED; + set_bit(__QDISC_STATE_THROTTLED, &qdisc->state); } static inline void qdisc_unthrottled(struct Qdisc *qdisc) { - qdisc->__state &= ~__QDISC___STATE_THROTTLED; + clear_bit(__QDISC_STATE_THROTTLED, &qdisc->state); } struct Qdisc_class_ops { -- cgit v1.2.2