packet: Add fanout support.

Fanouts allow packet capturing to be demuxed to a set of AF_PACKET
sockets.  Two fanout policies are implemented:

1) Hashing based upon skb->rxhash

2) Pure round-robin

An AF_PACKET socket must be fully bound before it tries to add itself
to a fanout.  All AF_PACKET sockets trying to join the same fanout
must all have the same bind settings.

Fanouts are identified (within a network namespace) by a 16-bit ID.
The first socket to try to add itself to a fanout with a particular
ID, creates that fanout.  When the last socket leaves the fanout
(which happens only when the socket is closed), that fanout is
destroyed.

Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 251 insertions, 5 deletions

diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
index bb281bf330aa..3350f1d3c9aa 100644
--- a/net/packet/af_packet.c
+++ b/net/packet/af_packet.c
@@ -187,9 +187,11 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
 
 static void packet_flush_mclist(struct sock *sk);
 
+struct packet_fanout;
 struct packet_sock {
         /* struct sock has to be the first member of packet_sock */
         struct sock             sk;
+        struct packet_fanout    *fanout;
         struct tpacket_stats    stats;
         struct packet_ring_buffer       rx_ring;
         struct packet_ring_buffer       tx_ring;
@@ -212,6 +214,24 @@ struct packet_sock {
         struct packet_type      prot_hook ____cacheline_aligned_in_smp;
 };
 
+#define PACKET_FANOUT_MAX       256
+
+struct packet_fanout {
+#ifdef CONFIG_NET_NS
+        struct net              *net;
+#endif
+        unsigned int            num_members;
+        u16                     id;
+        u8                      type;
+        u8                      pad;
+        atomic_t                rr_cur;
+        struct list_head        list;
+        struct sock             *arr[PACKET_FANOUT_MAX];
+        spinlock_t              lock;
+        atomic_t                sk_ref;
+        struct packet_type      prot_hook ____cacheline_aligned_in_smp;
+};
+
 struct packet_skb_cb {
         unsigned int origlen;
         union {
@@ -227,6 +247,9 @@ static inline struct packet_sock *pkt_sk(struct sock *sk)
         return (struct packet_sock *)sk;
 }
 
+static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
+static void __fanout_link(struct sock *sk, struct packet_sock *po);
+
 /* register_prot_hook must be invoked with the po->bind_lock held,
  * or from a context in which asynchronous accesses to the packet
  * socket is not possible (packet_create()).
@@ -235,7 +258,10 @@ static void register_prot_hook(struct sock *sk)
 {
         struct packet_sock *po = pkt_sk(sk);
         if (!po->running) {
-                dev_add_pack(&po->prot_hook);
+                if (po->fanout)
+                        __fanout_link(sk, po);
+                else
+                        dev_add_pack(&po->prot_hook);
                 sock_hold(sk);
                 po->running = 1;
         }
@@ -253,7 +279,10 @@ static void __unregister_prot_hook(struct sock *sk, bool sync)
         struct packet_sock *po = pkt_sk(sk);
 
         po->running = 0;
-        __dev_remove_pack(&po->prot_hook);
+        if (po->fanout)
+                __fanout_unlink(sk, po);
+        else
+                __dev_remove_pack(&po->prot_hook);
         __sock_put(sk);
 
         if (sync) {
@@ -388,6 +417,201 @@ static void packet_sock_destruct(struct sock *sk)
         sk_refcnt_debug_dec(sk);
 }
 
+static int fanout_rr_next(struct packet_fanout *f, unsigned int num)
+{
+        int x = atomic_read(&f->rr_cur) + 1;
+
+        if (x >= num)
+                x = 0;
+
+        return x;
+}
+
+static struct sock *fanout_demux_hash(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
+{
+        u32 idx, hash = skb->rxhash;
+
+        idx = ((u64)hash * num) >> 32;
+
+        return f->arr[idx];
+}
+
+static struct sock *fanout_demux_lb(struct packet_fanout *f, struct sk_buff *skb, unsigned int num)
+{
+        int cur, old;
+
+        cur = atomic_read(&f->rr_cur);
+        while ((old = atomic_cmpxchg(&f->rr_cur, cur,
+                                     fanout_rr_next(f, num))) != cur)
+                cur = old;
+        return f->arr[cur];
+}
+
+static int packet_rcv_fanout_hash(struct sk_buff *skb, struct net_device *dev,
+                                  struct packet_type *pt, struct net_device *orig_dev)
+{
+        struct packet_fanout *f = pt->af_packet_priv;
+        unsigned int num = f->num_members;
+        struct packet_sock *po;
+        struct sock *sk;
+
+        if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
+            !num) {
+                kfree_skb(skb);
+                return 0;
+        }
+
+        skb_get_rxhash(skb);
+
+        sk = fanout_demux_hash(f, skb, num);
+        po = pkt_sk(sk);
+
+        return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
+}
+
+static int packet_rcv_fanout_lb(struct sk_buff *skb, struct net_device *dev,
+                                struct packet_type *pt, struct net_device *orig_dev)
+{
+        struct packet_fanout *f = pt->af_packet_priv;
+        unsigned int num = f->num_members;
+        struct packet_sock *po;
+        struct sock *sk;
+
+        if (!net_eq(dev_net(dev), read_pnet(&f->net)) ||
+            !num) {
+                kfree_skb(skb);
+                return 0;
+        }
+
+        sk = fanout_demux_lb(f, skb, num);
+        po = pkt_sk(sk);
+
+        return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
+}
+
+static DEFINE_MUTEX(fanout_mutex);
+static LIST_HEAD(fanout_list);
+
+static void __fanout_link(struct sock *sk, struct packet_sock *po)
+{
+        struct packet_fanout *f = po->fanout;
+
+        spin_lock(&f->lock);
+        f->arr[f->num_members] = sk;
+        smp_wmb();
+        f->num_members++;
+        spin_unlock(&f->lock);
+}
+
+static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
+{
+        struct packet_fanout *f = po->fanout;
+        int i;
+
+        spin_lock(&f->lock);
+        for (i = 0; i < f->num_members; i++) {
+                if (f->arr[i] == sk)
+                        break;
+        }
+        BUG_ON(i >= f->num_members);
+        f->arr[i] = f->arr[f->num_members - 1];
+        f->num_members--;
+        spin_unlock(&f->lock);
+}
+
+static int fanout_add(struct sock *sk, u16 id, u8 type)
+{
+        struct packet_sock *po = pkt_sk(sk);
+        struct packet_fanout *f, *match;
+        int err;
+
+        switch