1 files changed, 152 insertions, 30 deletions

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index ae836fded530..77eb60d2b496 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -169,6 +169,7 @@ struct skb_shared_hwtstamps {
  * @software:           generate software time stamp
  * @in_progress:        device driver is going to provide
  *                      hardware time stamp
+ * @prevent_sk_orphan:  make sk reference available on driver level
  * @flags:              all shared_tx flags
  *
  * These flags are attached to packets as part of the
@@ -178,7 +179,8 @@ union skb_shared_tx {
         struct {
                 __u8    hardware:1,
                         software:1,
-                        in_progress:1;
+                        in_progress:1,
+                        prevent_sk_orphan:1;
         };
         __u8 flags;
 };
@@ -187,12 +189,8 @@ union skb_shared_tx {
  * the end of the header data, ie. at skb->end.
  */
 struct skb_shared_info {
-        atomic_t        dataref;
         unsigned short  nr_frags;
         unsigned short  gso_size;
-#ifdef CONFIG_HAS_DMA
-        dma_addr_t      dma_head;
-#endif
         /* Warning: this field is not always filled in (UFO)! */
         unsigned short  gso_segs;
         unsigned short  gso_type;
@@ -200,13 +198,17 @@ struct skb_shared_info {
         union skb_shared_tx tx_flags;
         struct sk_buff  *frag_list;
         struct skb_shared_hwtstamps hwtstamps;
-        skb_frag_t      frags[MAX_SKB_FRAGS];
-#ifdef CONFIG_HAS_DMA
-        dma_addr_t      dma_maps[MAX_SKB_FRAGS];
-#endif
+
+        /*
+         * Warning : all fields before dataref are cleared in __alloc_skb()
+         */
+        atomic_t        dataref;
+
         /* Intermediate layers must ensure that destructor_arg
          * remains valid until skb destructor */
         void *          destructor_arg;
+        /* must be last field, see pskb_expand_head() */
+        skb_frag_t      frags[MAX_SKB_FRAGS];
 };
 
 /* We divide dataref into two halves.  The higher 16 bits hold references
@@ -265,7 +267,7 @@ typedef unsigned char *sk_buff_data_t;
  *      @transport_header: Transport layer header
  *      @network_header: Network layer header
  *      @mac_header: Link layer header
- *      @_skb_dst: destination entry
+ *      @_skb_refdst: destination entry (with norefcount bit)
  *      @sp: the security path, used for xfrm
  *      @cb: Control buffer. Free for use by every layer. Put private vars here
  *      @len: Length of actual data
@@ -300,6 +302,7 @@ typedef unsigned char *sk_buff_data_t;
  *      @nfct_reasm: netfilter conntrack re-assembly pointer
  *      @nf_bridge: Saved data about a bridged frame - see br_netfilter.c
  *      @skb_iif: ifindex of device we arrived on
+ *      @rxhash: the packet hash computed on receive
  *      @queue_mapping: Queue mapping for multiqueue devices
  *      @tc_index: Traffic control index
  *      @tc_verd: traffic control verdict
@@ -315,22 +318,23 @@ struct sk_buff {
         struct sk_buff          *next;
         struct sk_buff          *prev;
 
-        struct sock             *sk;
         ktime_t                 tstamp;
+
+        struct sock             *sk;
         struct net_device       *dev;
 
-        unsigned long           _skb_dst;
-#ifdef CONFIG_XFRM
-        struct  sec_path        *sp;
-#endif
         /*
          * This is the control buffer. It is free to use for every
          * layer. Please put your private variables there. If you
          * want to keep them across layers you have to do a skb_clone()
          * first. This is owned by whoever has the skb queued ATM.
          */
-        char                    cb[48];
+        char                    cb[48] __aligned(8);
 
+        unsigned long           _skb_refdst;
+#ifdef CONFIG_XFRM
+        struct  sec_path        *sp;
+#endif
         unsigned int            len,
                                 data_len;
         __u16                   mac_len,
@@ -354,8 +358,8 @@ struct sk_buff {
                                 ipvs_property:1,
                                 peeked:1,
                                 nf_trace:1;
-        __be16                  protocol:16;
         kmemcheck_bitfield_end(flags1);
+        __be16                  protocol;
 
         void                    (*destructor)(struct sk_buff *skb);
 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
@@ -374,10 +378,15 @@ struct sk_buff {
 #endif
 #endif
 
+        __u32                   rxhash;
+
         kmemcheck_bitfield_begin(flags2);
         __u16                   queue_mapping:16;
 #ifdef CONFIG_IPV6_NDISC_NODETYPE
-        __u8                    ndisc_nodetype:2;
+        __u8                    ndisc_nodetype:2,
+                                deliver_no_wcard:1;
+#else
+        __u8                    deliver_no_wcard:1;
 #endif
         kmemcheck_bitfield_end(flags2);
 
@@ -416,14 +425,64 @@ struct sk_buff {
 
 #include <asm/system.h>
 
+/*
+ * skb might have a dst pointer attached, refcounted or not.
+ * _skb_refdst low order bit is set if refcount was _not_ taken
+ */
+#define SKB_DST_NOREF   1UL
+#define SKB_DST_PTRMASK ~(SKB_DST_NOREF)
+
+/**
+ * skb_dst - returns skb dst_entry
+ * @skb: buffer
+ *
+ * Returns skb dst_entry, regardless of reference taken or not.
+ */
 static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
 {
-        return (struct dst_entry *)skb->_skb_dst;
+        /* If refdst was not refcounted, check we still are in a 
+         * rcu_read_lock section
+         */
+        WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
+                !rcu_read_lock_held() &&
+                !rcu_read_lock_bh_held());
+        return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
 }
 
+/**
+ * skb_dst_set - sets skb dst
+ * @skb: buffer
+ * @dst: dst entry
+ *
+ * Sets skb dst, assuming a reference was taken on dst and should
+ * be released by skb_dst_drop()
+ */
 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
 {
-        skb->_skb_dst = (unsigned long)dst;
+        skb->_skb_refdst = (unsigned long)dst;
+}
+
+/**
+ * skb_dst_set_noref - sets skb dst, without a reference
+ * @skb: buffer
+ * @dst: dst entry
+ *
+ * Sets skb dst, assuming a reference was not taken on dst
+ * skb_dst_drop() should not dst_release() this dst
+ */
+static inline void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst)
+{
+        WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
+        skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF;
+}
+
+/**
+ * skb_dst_is_noref - Test if skb dst isnt refcounted
+ * @skb: buffer
+ */
+static inline bool skb_dst_is_noref(const struct sk_buff *skb)
+{
+        return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
 }
 
 static inline struct rtable *skb_rtable(const struct sk_buff *skb)
@@ -448,7 +507,7 @@ static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
         return __alloc_skb(size, priority, 1, -1);
 }
 
-extern int skb_recycle_check(struct sk_buff *skb, int skb_size);
+extern bool skb_recycle_check(struct sk_buff *skb, int skb_size);
 
 extern struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
 extern struct sk_buff *skb_clone(struct sk_buff *skb,
@@ -472,11 +531,6 @@ extern int	       skb_cow_data(struct sk_buff *skb, int tailbits,
                                     struct sk_buff **trailer);
 extern int             skb_pad(struct sk_buff *skb, int pad);
 #define dev_kfree_skb(a)        consume_skb(a)
-#define dev_consume_skb(a)      kfree_skb_clean(a)
-extern void           skb_over_panic(struct sk_buff *skb, int len,
-                                     void *here);
-extern void           skb_under_panic(struct sk_buff *skb, int len,
-                                      void *here);
 
 extern int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
                         int getfrag(void *from, char *to, int offset,
@@ -738,7 +792,7 @@ static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
 }
 
 /**
- *      skb_peek
+ *      skb_peek - peek at the head of an &sk_buff_head
  *      @list_: list to peek at
  *
  *      Peek an &sk_buff. Unlike most other operations you _MUST_
@@ -759,7 +813,7 @@ static inline struct sk_buff *skb_peek(struct sk_buff_head *list_)
 }
 
 /**
- *      skb_peek_tail
+ *      skb_peek_tail - peek at the tail of an &sk_buff_head
  *      @list_: list to peek at
  *
  *      Peek an &sk_buff. Unlike most other operations you _MUST_
@@ -1135,6 +1189,11 @@ static inline unsigned char *__skb_pull(struct sk_buff *skb, unsigned int len)
         return skb->data += len;
 }
 
+static inline unsigned char *skb_pull_inline(struct sk_buff *skb, unsigned int len)
+{
+        return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
+}
+
 extern unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta);
 
 static inline unsigned char *__pskb_pull(struct sk_buff *skb, unsigned int len)
@@ -1320,6 +1379,11 @@ static inline int skb_network_offset(const struct sk_buff *skb)
         return skb_network_header(skb) - skb->data;
 }
 
+static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
+{
+        return pskb_may_pull(skb, skb_network_offset(skb) + len);
+}
+
 /*
  * CPUs often take a performance hit when accessing unaligned memory
  * locations. The actual performance hit varies, it can be small if the
@@ -1358,9 +1422,14 @@ static inline int skb_network_offset(const struct sk_buff *skb)
  *
  * Various parts of the networking layer expect at least 32 bytes of
  * headroom, you should not reduce this.
+ *
+ * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
+ * to reduce average number of cache lines per packet.
+ * get_rps_cpus() for example only access one 64 bytes aligned block :
+ * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
  */
 #ifndef NET_SKB_PAD
-#define NET_SKB_PAD     32
+#define NET_SKB_PAD     max(32, L1_CACHE_BYTES)
 #endif
 
 extern int ___pskb_trim(struct sk_buff *skb, unsigned int len);
@@ -1872,6 +1941,36 @@ static inline ktime_t net_invalid_timestamp(void)
         return ktime_set(0, 0);
 }
 
+extern void skb_timestamping_init(void);
+
+#ifdef CONFIG_NETWORK_PHY_TIMESTAMPING
+
+extern void skb_clone_tx_timestamp(struct sk_buff *skb);
+extern bool skb_defer_rx_timestamp(struct sk_buff *skb);
+
+#else /* CONFIG_NETWORK_PHY_TIMESTAMPING */
+
+static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
+{
+}
+
+static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
+{
+        return false;
+}
+
+#endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */
+
+/**
+ * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
+ *
+ * @skb: clone of the the original outgoing packet
+ * @hwtstamps: hardware time stamps
+ *
+ */
+void skb_complete_tx_timestamp(struct sk_buff *skb,
+                               struct skb_shared_hwtstamps *hwtstamps);
+
 /**
  * skb_tstamp_tx - queue clone of skb with send time stamps
  * @orig_skb:   the original outgoing packet
@@ -1886,6 +1985,28 @@ static inline ktime_t net_invalid_timestamp(void)
 extern void skb_tstamp_tx(struct sk_buff *orig_skb,
                         struct skb_shared_hwtstamps *hwtstamps);
 
+static inline void sw_tx_timestamp(struct sk_buff *skb)
+{
+        union skb_shared_tx *shtx = skb_tx(skb);
+        if (shtx->software && !shtx->in_progress)
+                skb_tstamp_tx(skb, NULL);
+}
+
+/**
+ * skb_tx_timestamp() - Driver hook for transmit timestamping
+ *
+ * Ethernet MAC Drivers should call this function in their hard_xmit()
+ * function as soon as possible after giving the sk_buff to the MAC
+ * hardware, but before freeing the sk_buff.
+ *
+ * @skb: A socket buffer.
+ */
+static inline void skb_tx_timestamp(struct sk_buff *skb)
+{
+        skb_clone_tx_timestamp(skb);
+        sw_tx_timestamp(skb);
+}
+
 extern __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
 extern __sum16 __skb_checksum_complete(struct sk_buff *skb);
 
@@ -2073,7 +2194,8 @@ static inline bool skb_warn_if_lro(const struct sk_buff *skb)
         /* LRO sets gso_size but not gso_type, whereas if GSO is really
          * wanted then gso_type will be set. */
         struct skb_shared_info *shinfo = skb_shinfo(skb);
-        if (shinfo->gso_size != 0 && unlikely(shinfo->gso_type == 0)) {
+        if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
+            unlikely(shinfo->gso_type == 0)) {
                 __skb_warn_lro_forwarding(skb);
                 return true;
         }