Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp

Conflicts:
	litmus/sched_cedf.c

1 files changed, 253 insertions, 156 deletions

diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c
index 201cbac2b32c..b6ea6afa55b0 100644
--- a/net/sched/sch_sfq.c
+++ b/net/sched/sch_sfq.c
@@ -21,6 +21,7 @@
 #include <linux/skbuff.h>
 #include <linux/jhash.h>
 #include <linux/slab.h>
+#include <linux/vmalloc.h>
 #include <net/ip.h>
 #include <net/netlink.h>
 #include <net/pkt_sched.h>
@@ -67,55 +68,81 @@
 
         IMPLEMENTATION:
         This implementation limits maximal queue length to 128;
-        maximal mtu to 2^15-1; number of hash buckets to 1024.
+        max mtu to 2^18-1; max 128 flows, number of hash buckets to 1024.
         The only goal of this restrictions was that all data
-        fit into one 4K page :-). Struct sfq_sched_data is
-        organized in anti-cache manner: all the data for a bucket
-        are scattered over different locations. This is not good,
-        but it allowed me to put it into 4K.
+        fit into one 4K page on 32bit arches.
 
         It is easy to increase these values, but not in flight.  */
 
-#define SFQ_DEPTH               128
-#define SFQ_HASH_DIVISOR        1024
+#define SFQ_DEPTH               128 /* max number of packets per flow */
+#define SFQ_SLOTS               128 /* max number of flows */
+#define SFQ_EMPTY_SLOT          255
+#define SFQ_DEFAULT_HASH_DIVISOR 1024
 
-/* This type should contain at least SFQ_DEPTH*2 values */
+/* We use 16 bits to store allot, and want to handle packets up to 64K
+ * Scale allot by 8 (1<<3) so that no overflow occurs.
+ */
+#define SFQ_ALLOT_SHIFT         3
+#define SFQ_ALLOT_SIZE(X)       DIV_ROUND_UP(X, 1 << SFQ_ALLOT_SHIFT)
+
+/* This type should contain at least SFQ_DEPTH + SFQ_SLOTS values */
 typedef unsigned char sfq_index;
 
-struct sfq_head
-{
+/*
+ * We dont use pointers to save space.
+ * Small indexes [0 ... SFQ_SLOTS - 1] are 'pointers' to slots[] array
+ * while following values [SFQ_SLOTS ... SFQ_SLOTS + SFQ_DEPTH - 1]
+ * are 'pointers' to dep[] array
+ */
+struct sfq_head {
         sfq_index       next;
         sfq_index       prev;
 };
 
-struct sfq_sched_data
-{
+struct sfq_slot {
+        struct sk_buff  *skblist_next;
+        struct sk_buff  *skblist_prev;
+        sfq_index       qlen; /* number of skbs in skblist */
+        sfq_index       next; /* next slot in sfq chain */
+        struct sfq_head dep; /* anchor in dep[] chains */
+        unsigned short  hash; /* hash value (index in ht[]) */
+        short           allot; /* credit for this slot */
+};
+
+struct sfq_sched_data {
 /* Parameters */
         int             perturb_period;
-        unsigned        quantum;        /* Allotment per round: MUST BE >= MTU */
+        unsigned int    quantum;        /* Allotment per round: MUST BE >= MTU */
         int             limit;
-
+        unsigned int    divisor;        /* number of slots in hash table */
 /* Variables */
         struct tcf_proto *filter_list;
         struct timer_list perturb_timer;
         u32             perturbation;
-        sfq_index       tail;           /* Index of current slot in round */
-        sfq_index       max_depth;      /* Maximal depth */
-
-        sfq_index       ht[SFQ_HASH_DIVISOR];   /* Hash table */
-        sfq_index       next[SFQ_DEPTH];        /* Active slots link */
-        short           allot[SFQ_DEPTH];       /* Current allotment per slot */
-        unsigned short  hash[SFQ_DEPTH];        /* Hash value indexed by slots */
-        struct sk_buff_head     qs[SFQ_DEPTH];          /* Slot queue */
-        struct sfq_head dep[SFQ_DEPTH*2];       /* Linked list of slots, indexed by depth */
+        sfq_index       cur_depth;      /* depth of longest slot */
+        unsigned short  scaled_quantum; /* SFQ_ALLOT_SIZE(quantum) */
+        struct sfq_slot *tail;          /* current slot in round */
+        sfq_index       *ht;            /* Hash table (divisor slots) */
+        struct sfq_slot slots[SFQ_SLOTS];
+        struct sfq_head dep[SFQ_DEPTH]; /* Linked list of slots, indexed by depth */
 };
 
-static __inline__ unsigned sfq_fold_hash(struct sfq_sched_data *q, u32 h, u32 h1)
+/*
+ * sfq_head are either in a sfq_slot or in dep[] array
+ */
+static inline struct sfq_head *sfq_dep_head(struct sfq_sched_data *q, sfq_index val)
 {
-        return jhash_2words(h, h1, q->perturbation) & (SFQ_HASH_DIVISOR - 1);
+        if (val < SFQ_SLOTS)
+                return &q->slots[val].dep;
+        return &q->dep[val - SFQ_SLOTS];
 }
 
-static unsigned sfq_hash(struct sfq_sched_data *q, struct sk_buff *skb)
+static unsigned int sfq_fold_hash(struct sfq_sched_data *q, u32 h, u32 h1)
+{
+        return jhash_2words(h, h1, q->perturbation) & (q->divisor - 1);
+}
+
+static unsigned int sfq_hash(struct sfq_sched_data *q, struct sk_buff *skb)
 {
         u32 h, h2;
 
@@ -123,40 +150,39 @@ static unsigned sfq_hash(struct sfq_sched_data *q, struct sk_buff *skb)
         case htons(ETH_P_IP):
         {
                 const struct iphdr *iph;
+                int poff;
 
                 if (!pskb_network_may_pull(skb, sizeof(*iph)))
                         goto err;
                 iph = ip_hdr(skb);
                 h = (__force u32)iph->daddr;
                 h2 = (__force u32)iph->saddr ^ iph->protocol;
-                if (!(iph->frag_off&htons(IP_MF|IP_OFFSET)) &&
-                    (iph->protocol == IPPROTO_TCP ||
-                     iph->protocol == IPPROTO_UDP ||
-                     iph->protocol == IPPROTO_UDPLITE ||
-                     iph->protocol == IPPROTO_SCTP ||
-                     iph->protocol == IPPROTO_DCCP ||
-                     iph->protocol == IPPROTO_ESP) &&
-                     pskb_network_may_pull(skb, iph->ihl * 4 + 4))
-                        h2 ^= *(((u32*)iph) + iph->ihl);
+                if (iph->frag_off & htons(IP_MF | IP_OFFSET))
+                        break;
+                poff = proto_ports_offset(iph->protocol);
+                if (poff >= 0 &&
+                    pskb_network_may_pull(skb, iph->ihl * 4 + 4 + poff)) {
+                        iph = ip_hdr(skb);
+                        h2 ^= *(u32 *)((void *)iph + iph->ihl * 4 + poff);
+                }
                 break;
         }
         case htons(ETH_P_IPV6):
         {
-                struct ipv6hdr *iph;
+                const struct ipv6hdr *iph;
+                int poff;
 
                 if (!pskb_network_may_pull(skb, sizeof(*iph)))
                         goto err;
                 iph = ipv6_hdr(skb);
                 h = (__force u32)iph->daddr.s6_addr32[3];
                 h2 = (__force u32)iph->saddr.s6_addr32[3] ^ iph->nexthdr;
-                if ((iph->nexthdr == IPPROTO_TCP ||
-                     iph->nexthdr == IPPROTO_UDP ||
-                     iph->nexthdr == IPPROTO_UDPLITE ||
-                     iph->nexthdr == IPPROTO_SCTP ||
-                     iph->nexthdr == IPPROTO_DCCP ||
-                     iph->nexthdr == IPPROTO_ESP) &&
-                    pskb_network_may_pull(skb, sizeof(*iph) + 4))
-                        h2 ^= *(u32*)&iph[1];
+                poff = proto_ports_offset(iph->nexthdr);
+                if (poff >= 0 &&
+                    pskb_network_may_pull(skb, sizeof(*iph) + 4 + poff)) {
+                        iph = ipv6_hdr(skb);
+                        h2 ^= *(u32 *)((void *)iph + sizeof(*iph) + poff);
+                }
                 break;
         }
         default:
@@ -177,7 +203,7 @@ static unsigned int sfq_classify(struct sk_buff *skb, struct Qdisc *sch,
 
         if (TC_H_MAJ(skb->priority) == sch->handle &&
             TC_H_MIN(skb->priority) > 0 &&
-            TC_H_MIN(skb->priority) <= SFQ_HASH_DIVISOR)
+            TC_H_MIN(skb->priority) <= q->divisor)
                 return TC_H_MIN(skb->priority);
 
         if (!q->filter_list)
@@ -195,36 +221,47 @@ static unsigned int sfq_classify(struct sk_buff *skb, struct Qdisc *sch,
                         return 0;
                 }
 #endif
-                if (TC_H_MIN(res.classid) <= SFQ_HASH_DIVISOR)
+                if (TC_H_MIN(res.classid) <= q->divisor)
                         return TC_H_MIN(res.classid);
         }
         return 0;
 }
 
+/*
+ * x : slot number [0 .. SFQ_SLOTS - 1]
+ */
 static inline void sfq_link(struct sfq_sched_data *q, sfq_index x)
 {
         sfq_index p, n;
-        int d = q->qs[x].qlen + SFQ_DEPTH;
+        int qlen = q->slots[x].qlen;
 
-        p = d;
-        n = q->dep[d].next;
-        q->dep[x].next = n;
-        q->dep[x].prev = p;
-        q->dep[p].next = q->dep[n].prev = x;
+        p = qlen + SFQ_SLOTS;
+        n = q->dep[qlen].next;
+
+        q->slots[x].dep.next = n;
+        q->slots[x].dep.prev = p;
+
+        q->dep[qlen].next = x;          /* sfq_dep_head(q, p)->next = x */
+        sfq_dep_head(q, n)->prev = x;
 }
 
+#define sfq_unlink(q, x, n, p)                  \
+        n = q->slots[x].dep.next;               \
+        p = q->slots[x].dep.prev;               \
+        sfq_dep_head(q, p)->next = n;           \
+        sfq_dep_head(q, n)->prev = p
+
+
 static inline void sfq_dec(struct sfq_sched_data *q, sfq_index x)
 {
         sfq_index p, n;
+        int d;
 
-        n = q->dep[x].next;
-        p = q->dep[x].prev;
-        q->dep[p].next = n;
-        q->dep[n].prev = p;
-
-        if (n == p && q->max_depth == q->qs[x].qlen + 1)
-                q->max_depth--;
+        sfq_unlink(q, x, n, p);
 
+        d = q->slots[x].qlen--;
+        if (n == p && q->cur_depth == d)
+                q->cur_depth--;
         sfq_link(q, x);
 }
 
@@ -233,34 +270,74 @@ static inline void sfq_inc(struct sfq_sched_data *q, sfq_index x)
         sfq_index p, n;
         int d;
 
-        n = q->dep[x].next;
-        p = q->dep[x].prev;
-        q->dep[p].next = n;
-        q->dep[n].prev = p;
-        d = q->qs[x].qlen;
-        if (q->max_depth < d)
-                q->max_depth = d;
+        sfq_unlink(q, x, n, p);
 
+        d = ++q->slots[x].qlen;
+        if (q->cur_depth < d)
+                q->cur_depth = d;
         sfq_link(q, x);
 }
 
+/* helper functions : might be changed when/if skb use a standard list_head */
+
+/* remove one skb from tail of slot queue */
+static inline struct sk_buff *slot_dequeue_tail(struct sfq_slot *slot)
+{
+        struct sk_buff *skb = slot->skblist_prev;
+
+        slot->skblist_prev = skb->prev;
+        skb->prev->next = (struct sk_buff *)slot;
+        skb->next = skb->prev = NULL;
+        return skb;
+}
+
+/* remove one skb from head of slot queue */
+static inline struct sk_buff *slot_dequeue_head(struct sfq_slot *slot)
+{
+        struct sk_buff *skb = slot->skblist_next;
+
+        slot->skblist_next = skb->next;
+        skb->next->prev = (struct sk_buff *)slot;
+        skb->next = skb->prev = NULL;
+        return skb;
+}
+
+static inline void slot_queue_init(struct sfq_slot *slot)
+{
+        slot->skblist_prev = slot->skblist_next = (struct sk_buff *)slot;
+}
+
+/* add skb to slot queue (tail add) */
+static inline void slot_queue_add(struct sfq_slot *slot, struct sk_buff *skb)
+{
+        skb->prev = slot->skblist_prev;
+        skb->next = (struct sk_buff *)slot;
+        slot->skblist_prev->next = skb;
+        slot->skblist_prev = skb;
+}
+
+#define slot_queue_walk(slot, skb)              \
+        for (skb = slot->skblist_next;          \
+             skb != (struct sk_buff *)slot;     \
+             skb = skb->next)
+
 static unsigned int sfq_drop(struct Qdisc *sch)
 {
         struct sfq_sched_data *q = qdisc_priv(sch);
-        sfq_index d = q->max_depth;
+        sfq_index x, d = q->cur_depth;
         struct sk_buff *skb;
         unsigned int len;
+        struct sfq_slot *slot;
 
-        /* Queue is full! Find the longest slot and
-           drop a packet from it */
-
+        /* Queue is full! Find the longest slot and drop tail packet from it */
         if (d > 1) {
-                sfq_index x = q->dep[d + SFQ_DEPTH].next;
-                skb = q->qs[x].prev;
+                x = q->dep[d].next;
+                slot = &q->slots[x];
+drop:
+                skb = slot_dequeue_tail(slot);
                 len = qdisc_pkt_len(skb);
-                __skb_unlink(skb, &q->qs[x]);
-                kfree_skb(skb);
                 sfq_dec(q, x);
+                kfree_skb(skb);
                 sch->q.qlen--;
                 sch->qstats.drops++;
                 sch->qstats.backlog -= len;
@@ -269,19 +346,11 @@ static unsigned int sfq_drop(struct Qdisc *sch)
 
         if (d == 1) {
                 /* It is difficult to believe, but ALL THE SLOTS HAVE LENGTH 1. */
-                d = q->next[q->tail];
-                q->next[q->tail] = q->next[d];
-                q->allot[q->next[d]] += q->quantum;
-                skb = q->qs[d].prev;
-                len = qdisc_pkt_len(skb);
-                __skb_unlink(skb, &q->qs[d]);
-                kfree_skb(skb);
-                sfq_dec(q, d);
-                sch->q.qlen--;
-                q->ht[q->hash[d]] = SFQ_DEPTH;
-                sch->qstats.drops++;
-                sch->qstats.backlog -= len;
-                return len;
+                x = q->tail->next;
+                slot = &q->slots[x];
+                q->tail->next = slot->next;
+                q->ht[slot->hash] = SFQ_EMPTY_SLOT;
+                goto drop;
         }
 
         return 0;
@@ -292,7 +361,8 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
 {
         struct sfq_sched_data *q = qdisc_priv(sch);
         unsigned int hash;
-        sfq_index x;
+        sfq_index x, qlen;
+        struct sfq_slot *slot;
         int uninitialized_var(ret);
 
         hash = sfq_classify(skb, sch, &ret);
@@ -305,54 +375,42 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
         hash--;
 
         x = q->ht[hash];
-        if (x == SFQ_DEPTH) {
-                q->ht[hash] = x = q->dep[SFQ_DEPTH].next;
-                q->hash[x] = hash;
+        slot = &q->slots[x];
+        if (x == SFQ_EMPTY_SLOT) {
+                x = q->dep[0].next; /* get a free slot */
+                q->ht[hash] = x;
+                slot = &q->slots[x];
+                slot->hash = hash;
         }
 
-        /* If selected queue has length q->limit, this means that
-         * all another queues are empty and that we do simple tail drop,
+        /* If selected queue has length q->limit, do simple tail drop,
          * i.e. drop _this_ packet.
          */
-        if (q->qs[x].qlen >= q->limit)
+        if (slot->qlen >= q->limit)
                 return qdisc_drop(skb, sch);
 
         sch->qstats.backlog += qdisc_pkt_len(skb);
-        __skb_queue_tail(&q->qs[x], skb);
+        slot_queue_add(slot, skb);
         sfq_inc(q, x);
-        if (q->qs[x].qlen == 1) {               /* The flow is new */
-                if (q->tail == SFQ_DEPTH) {     /* It is the first flow */
-                        q->tail = x;
-                        q->next[x] = x;
-                        q->allot[x] = q->quantum;
+        if (slot->qlen == 1) {          /* The flow is new */
+                if (q->tail == NULL) {  /* It is the first flow */
+                        slot->next = x;
                 } else {
-                        q->next[x] = q->next[q->tail];
-                        q->next[q->tail] = x;
-                        q->tail = x;
+                        slot->next = q->tail->next;
+                        q->tail->next = x;
                 }
+                q->tail = slot;
+                slot->allot = q->scaled_quantum;
         }
-        if (++sch->q.qlen <= q->limit) {
-                sch->bstats.bytes += qdisc_pkt_len(skb);
-                sch->bstats.packets++;
+        if (++sch->q.qlen <= q->limit)
                 return NET_XMIT_SUCCESS;
-        }
 
+        qlen = slot->qlen;
         sfq_drop(sch);
-        return NET_XMIT_CN;
-}
-
-static struct sk_buff *
-sfq_peek(struct Qdisc *sch)
-{
-        struct sfq_sched_data *q = qdisc_priv(sch);
-        sfq_index a;
-
-        /* No active slots */
-        if (q->tail == SFQ_DEPTH)
-                return NULL;
-
-        a = q->next[q->tail];
-        return skb_peek(&q->qs[a]);
+        /* Return Congestion Notification only if we dropped a packet
+         * from this flow.
+         */
+        return (qlen != slot->qlen) ? NET_XMIT_CN : NET_XMIT_SUCCESS;
 }
 
 static struct sk_buff *
@@ -360,34 +418,38 @@ sfq_dequeue(struct Qdisc *sch)
 {
         struct sfq_sched_data *q = qdisc_priv(sch);
         struct sk_buff *skb;
-        sfq_index a, old_a;
+        sfq_index a, next_a;
+        struct sfq_slot *slot;
 
         /* No active slots */
-        if (q->tail == SFQ_DEPTH)
+        if (q->tail == NULL)
                 return NULL;
 
-        a = old_a = q->next[q->tail];
-
-        /* Grab packet */
-        skb = __skb_dequeue(&q->qs[a]);
+next_slot:
+        a = q->tail->next;
+        slot = &q->slots[a];
+        if (slot->allot <= 0) {
+                q->tail = slot;
+                slot->allot += q->scaled_quantum;
+                goto next_slot;
+        }
+        skb = slot_dequeue_head(slot);
         sfq_dec(q, a);
+        qdisc_bstats_update(sch, skb);
         sch->q.qlen--;
         sch->qstats.backlog -= qdisc_pkt_len(skb);
 
         /* Is the slot empty? */
-        if (q->qs[a].qlen == 0) {
-                q->ht[q->hash[a]] = SFQ_DEPTH;
-                a = q->next[a];
-                if (a == old_a) {
-                        q->tail = SFQ_DEPTH;
+        if (slot->qlen == 0) {
+                q->ht[slot->hash] = SFQ_EMPTY_SLOT;
+                next_a = slot->next;
+                if (a == next_a) {
+                        q->tail = NULL; /* no more active slots */
                         return skb;
                 }
-                q->next[q->tail] = a;
-                q->allot[a] += q->quantum;
-        } else if ((q->allot[a] -= qdisc_pkt_len(skb)) <= 0) {
-                q->tail = a;
-                a = q->next[a];
-                q->allot[a] += q->quantum;
+                q->tail->next = next_a;
+        } else {
+                slot->allot -= SFQ_ALLOT_SIZE(qdisc_pkt_len(skb));
         }
         return skb;
 }
@@ -421,12 +483,18 @@ static int sfq_change(struct Qdisc *sch, struct nlattr *opt)
         if (opt->nla_len < nla_attr_size(sizeof(*ctl)))
                 return -EINVAL;
 
+        if (ctl->divisor &&
+            (!is_power_of_2(ctl->divisor) || ctl->divisor > 65536))
+                return -EINVAL;
+
         sch_tree_lock(sch);
         q->quantum = ctl->quantum ? : psched_mtu(qdisc_dev(sch));
+        q->scaled_quantum = SFQ_ALLOT_SIZE(q->quantum);
         q->perturb_period = ctl->perturb_period * HZ;
         if (ctl->limit)
                 q->limit = min_t(u32, ctl->limit, SFQ_DEPTH - 1);
-
+        if (ctl->divisor)
+                q->divisor = ctl->divisor;
         qlen = sch->q.qlen;
         while (sch->q.qlen > q->limit)
                 sfq_drop(sch);
@@ -444,26 +512,25 @@ static int sfq_change(struct Qdisc *sch, struct nlattr *opt)
 static int sfq_init(struct Qdisc *sch, struct nlattr *opt)
 {
         struct sfq_sched_data *q = qdisc_priv(sch);
+        size_t sz;
         int i;
 
         q->perturb_timer.function = sfq_perturbation;
         q->perturb_timer.data = (unsigned long)sch;
         init_timer_deferrable(&q->perturb_timer);
 
-        for (i = 0; i < SFQ_HASH_DIVISOR; i++)
-                q->ht[i] = SFQ_DEPTH;
-
         for (i = 0; i < SFQ_DEPTH; i++) {
-                skb_queue_head_init(&q->qs[i]);
-                q->dep[i + SFQ_DEPTH].next = i + SFQ_DEPTH;
-                q->dep[i + SFQ_DEPTH].prev = i + SFQ_DEPTH;
+                q->dep[i].next = i + SFQ_SLOTS;
+                q->dep[i].prev = i + SFQ_SLOTS;
         }
 
         q->limit = SFQ_DEPTH - 1;
-        q->max_depth = 0;
-        q->tail = SFQ_DEPTH;
+        q->cur_depth = 0;
+        q->tail = NULL;
+        q->divisor = SFQ_DEFAULT_HASH_DIVISOR;
         if (opt == NULL) {
                 q->quantum = psched_mtu(qdisc_dev(sch));
+                q->scaled_quantum = SFQ_ALLOT_SIZE(q->quantum);
                 q->perturb_period = 0;
                 q->perturbation = net_random();
         } else {
@@ -472,8 +539,23 @@ static int sfq_init(struct Qdisc *sch, struct nlattr *opt)
                         return err;
         }
 
-        for (i = 0; i < SFQ_DEPTH; i++)
+        sz = sizeof(q->ht[0]) * q->divisor;
+        q->ht = kmalloc(sz, GFP_KERNEL);
+        if (!q->ht && sz > PAGE_SIZE)
+                q->ht = vmalloc(sz);
+        if (!q->ht)
+                return -ENOMEM;
+        for (i = 0; i < q->divisor; i++)
+                q->ht[i] = SFQ_EMPTY_SLOT;
+
+        for (i = 0; i < SFQ_SLOTS; i++) {
+                slot_queue_init(&q->slots[i]);
                 sfq_link(q, i);
+        }
+        if (q->limit >= 1)
+                sch->flags |= TCQ_F_CAN_BYPASS;
+        else
+                sch->flags &= ~TCQ_F_CAN_BYPASS;
         return 0;
 }
 
@@ -484,6 +566,10 @@ static void sfq_destroy(struct Qdisc *sch)
         tcf_destroy_chain(&q->filter_list);
         q->perturb_period = 0;
         del_timer_sync(&q->perturb_timer);
+        if (is_vmalloc_addr(q->ht))
+                vfree(q->ht);
+        else
+                kfree(q->ht);
 }
 
 static int sfq_dump(struct Qdisc *sch, struct sk_buff *skb)
@@ -496,7 +582,7 @@ static int sfq_dump(struct Qdisc *sch, struct sk_buff *skb)
         opt.perturb_period = q->perturb_period / HZ;
 
         opt.limit = q->limit;
-        opt.divisor = SFQ_HASH_DIVISOR;
+        opt.divisor = q->divisor;
         opt.flows = q->limit;
 
         NLA_PUT(skb, TCA_OPTIONS, sizeof(opt), &opt);
@@ -521,6 +607,8 @@ static unsigned long sfq_get(struct Qdisc *sch, u32 classid)
 static unsigned long sfq_bind(struct Qdisc *sch, unsigned long parent,
                               u32 classid)
 {
+        /* we cannot bypass queue discipline anymore */
+        sch->flags &= ~TCQ_F_CAN_BYPASS;
         return 0;
 }
 
@@ -548,10 +636,19 @@ static int sfq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
                                 struct gnet_dump *d)
 {
         struct sfq_sched_data *q = qdisc_priv(sch);
-        sfq_index idx = q->ht[cl-1];
-        struct gnet_stats_queue qs = { .qlen = q->qs[idx].qlen };
-        struct tc_sfq_xstats xstats = { .allot = q->allot[idx] };
+        sfq_index idx = q->ht[cl - 1];
+        struct gnet_stats_queue qs = { 0 };
+        struct tc_sfq_xstats xstats = { 0 };
+        struct sk_buff *skb;
 
+        if (idx != SFQ_EMPTY_SLOT) {
+                const struct sfq_slot *slot = &q->slots[idx];
+
+                xstats.allot = slot->allot << SFQ_ALLOT_SHIFT;
+                qs.qlen = slot->qlen;
+                slot_queue_walk(slot, skb)
+                        qs.backlog += qdisc_pkt_len(skb);
+        }
         if (gnet_stats_copy_queue(d, &qs) < 0)
                 return -1;
         return gnet_stats_copy_app(d, &xstats, sizeof(xstats));
@@ -565,8 +662,8 @@ static void sfq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
         if (arg->stop)
                 return;
 
-        for (i = 0; i < SFQ_HASH_DIVISOR; i++) {
-                if (q->ht[i] == SFQ_DEPTH ||
+        for (i = 0; i < q->divisor; i++) {
+                if (q->ht[i] == SFQ_EMPTY_SLOT ||
                     arg->count < arg->skip) {
                         arg->count++;
                         continue;
@@ -597,7 +694,7 @@ static struct Qdisc_ops sfq_qdisc_ops __read_mostly = {
         .priv_size      =       sizeof(struct sfq_sched_data),
         .enqueue        =       sfq_enqueue,
         .dequeue        =       sfq_dequeue,
-        .peek           =       sfq_peek,
+        .peek           =       qdisc_peek_dequeued,
         .drop           =       sfq_drop,
         .init           =       sfq_init,
         .reset          =       sfq_reset,