1 files changed, 324 insertions, 88 deletions

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 4ff3940210d8..253a2b9be9d6 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -27,20 +27,24 @@
 #include <net/checksum.h>
 #include <linux/rcupdate.h>
 #include <linux/dmaengine.h>
+#include <linux/hrtimer.h>
 
 #define HAVE_ALLOC_SKB          /* For the drivers to know */
 #define HAVE_ALIGNABLE_SKB      /* Ditto 8)                */
 
+/* Don't change this without changing skb_csum_unnecessary! */
 #define CHECKSUM_NONE 0
-#define CHECKSUM_PARTIAL 1
-#define CHECKSUM_UNNECESSARY 2
-#define CHECKSUM_COMPLETE 3
+#define CHECKSUM_UNNECESSARY 1
+#define CHECKSUM_COMPLETE 2
+#define CHECKSUM_PARTIAL 3
 
 #define SKB_DATA_ALIGN(X)       (((X) + (SMP_CACHE_BYTES - 1)) & \
                                  ~(SMP_CACHE_BYTES - 1))
-#define SKB_MAX_ORDER(X, ORDER) (((PAGE_SIZE << (ORDER)) - (X) - \
-                                  sizeof(struct skb_shared_info)) & \
-                                  ~(SMP_CACHE_BYTES - 1))
+#define SKB_WITH_OVERHEAD(X)    \
+        (((X) - sizeof(struct skb_shared_info)) & \
+         ~(SMP_CACHE_BYTES - 1))
+#define SKB_MAX_ORDER(X, ORDER) \
+        SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
 #define SKB_MAX_HEAD(X)         (SKB_MAX_ORDER((X), 0))
 #define SKB_MAX_ALLOC           (SKB_MAX_ORDER(0, 2))
 
@@ -66,8 +70,8 @@
  *      NONE: skb is checksummed by protocol or csum is not required.
  *
  *      PARTIAL: device is required to csum packet as seen by hard_start_xmit
- *      from skb->h.raw to the end and to record the checksum
- *      at skb->h.raw+skb->csum.
+ *      from skb->transport_header to the end and to record the checksum
+ *      at skb->transport_header + skb->csum.
  *
  *      Device must show its capabilities in dev->features, set
  *      at device setup time.
@@ -83,12 +87,13 @@
  */
 
 struct net_device;
+struct scatterlist;
 
-#ifdef CONFIG_NETFILTER
+#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
 struct nf_conntrack {
         atomic_t use;
-        void (*destroy)(struct nf_conntrack *);
 };
+#endif
 
 #ifdef CONFIG_BRIDGE_NETFILTER
 struct nf_bridge_info {
@@ -103,8 +108,6 @@ struct nf_bridge_info {
 };
 #endif
 
-#endif
-
 struct sk_buff_head {
         /* These two members must be first. */
         struct sk_buff  *next;
@@ -156,11 +159,6 @@ struct skb_shared_info {
 #define SKB_DATAREF_SHIFT 16
 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
 
-struct skb_timeval {
-        u32     off_sec;
-        u32     off_usec;
-};
-
 
 enum {
         SKB_FCLONE_UNAVAILABLE,
@@ -181,6 +179,16 @@ enum {
         SKB_GSO_TCPV6 = 1 << 4,
 };
 
+#if BITS_PER_LONG > 32
+#define NET_SKBUFF_DATA_USES_OFFSET 1
+#endif
+
+#ifdef NET_SKBUFF_DATA_USES_OFFSET
+typedef unsigned int sk_buff_data_t;
+#else
+typedef unsigned char *sk_buff_data_t;
+#endif
+
 /** 
  *      struct sk_buff - socket buffer
  *      @next: Next buffer in list
@@ -188,17 +196,19 @@ enum {
  *      @sk: Socket we are owned by
  *      @tstamp: Time we arrived
  *      @dev: Device we arrived on/are leaving by
- *      @input_dev: Device we arrived on
+ *      @iif: ifindex of device we arrived on
  *      @h: Transport layer header
- *      @nh: Network layer header
- *      @mac: Link layer header
+ *      @network_header: Network layer header
+ *      @mac_header: Link layer header
  *      @dst: destination entry
  *      @sp: the security path, used for xfrm
  *      @cb: Control buffer. Free for use by every layer. Put private vars here
  *      @len: Length of actual data
  *      @data_len: Data length
  *      @mac_len: Length of link layer header
- *      @csum: Checksum
+ *      @csum: Checksum (must include start/offset pair)
+ *      @csum_start: Offset from skb->head where checksumming should start
+ *      @csum_offset: Offset from csum_start where checksum should be stored
  *      @local_df: allow local fragmentation
  *      @cloned: Head may be cloned (check refcnt to be sure)
  *      @nohdr: Payload reference only, must not modify header
@@ -233,30 +243,10 @@ struct sk_buff {
         struct sk_buff          *prev;
 
         struct sock             *sk;
-        struct skb_timeval      tstamp;
+        ktime_t                 tstamp;
         struct net_device       *dev;
-        struct net_device       *input_dev;
-
-        union {
-                struct tcphdr   *th;
-                struct udphdr   *uh;
-                struct icmphdr  *icmph;
-                struct igmphdr  *igmph;
-                struct iphdr    *ipiph;
-                struct ipv6hdr  *ipv6h;
-                unsigned char   *raw;
-        } h;
-
-        union {
-                struct iphdr    *iph;
-                struct ipv6hdr  *ipv6h;
-                struct arphdr   *arph;
-                unsigned char   *raw;
-        } nh;
-
-        union {
-                unsigned char   *raw;
-        } mac;
+        int                     iif;
+        /* 4 byte hole on 64 bit*/
 
         struct  dst_entry       *dst;
         struct  sec_path        *sp;
@@ -274,7 +264,10 @@ struct sk_buff {
                                 mac_len;
         union {
                 __wsum          csum;
-                __u32           csum_offset;
+                struct {
+                        __u16   csum_start;
+                        __u16   csum_offset;
+                };
         };
         __u32                   priority;
         __u8                    local_df:1,
@@ -288,15 +281,13 @@ struct sk_buff {
         __be16                  protocol;
 
         void                    (*destructor)(struct sk_buff *skb);
-#ifdef CONFIG_NETFILTER
-        struct nf_conntrack     *nfct;
 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
+        struct nf_conntrack     *nfct;
         struct sk_buff          *nfct_reasm;
 #endif
 #ifdef CONFIG_BRIDGE_NETFILTER
         struct nf_bridge_info   *nf_bridge;
 #endif
-#endif /* CONFIG_NETFILTER */
 #ifdef CONFIG_NET_SCHED
         __u16                   tc_index;       /* traffic control index */
 #ifdef CONFIG_NET_CLS_ACT
@@ -312,13 +303,16 @@ struct sk_buff {
 
         __u32                   mark;
 
+        sk_buff_data_t          transport_header;
+        sk_buff_data_t          network_header;
+        sk_buff_data_t          mac_header;
         /* These elements must be at the end, see alloc_skb() for details.  */
+        sk_buff_data_t          tail;
+        sk_buff_data_t          end;
+        unsigned char           *head,
+                                *data;
         unsigned int            truesize;
         atomic_t                users;
-        unsigned char           *head,
-                                *data,
-                                *tail,
-                                *end;
 };
 
 #ifdef __KERNEL__
@@ -345,9 +339,6 @@ static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
         return __alloc_skb(size, priority, 1, -1);
 }
 
-extern struct sk_buff *alloc_skb_from_cache(struct kmem_cache *cp,
-                                            unsigned int size,
-                                            gfp_t priority);
 extern void            kfree_skbmem(struct sk_buff *skb);
 extern struct sk_buff *skb_clone(struct sk_buff *skb,
                                  gfp_t priority);
@@ -363,6 +354,11 @@ extern struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
 extern struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
                                        int newheadroom, int newtailroom,
                                        gfp_t priority);
+extern int             skb_to_sgvec(struct sk_buff *skb,
+                                    struct scatterlist *sg, int offset,
+                                    int len);
+extern int             skb_cow_data(struct sk_buff *skb, int tailbits,
+                                    struct sk_buff **trailer);
 extern int             skb_pad(struct sk_buff *skb, int pad);
 #define dev_kfree_skb(a)        kfree_skb(a)
 extern void           skb_over_panic(struct sk_buff *skb, int len,
@@ -404,8 +400,20 @@ extern unsigned int   skb_find_text(struct sk_buff *skb, unsigned int from,
                                     unsigned int to, struct ts_config *config,
                                     struct ts_state *state);
 
+#ifdef NET_SKBUFF_DATA_USES_OFFSET
+static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
+{
+        return skb->head + skb->end;
+}
+#else
+static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
+{
+        return skb->end;
+}
+#endif
+
 /* Internal */
-#define skb_shinfo(SKB)         ((struct skb_shared_info *)((SKB)->end))
+#define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB)))
 
 /**
  *      skb_queue_empty - check if a queue is empty
@@ -621,6 +629,13 @@ static inline void skb_queue_head_init(struct sk_buff_head *list)
         list->qlen = 0;
 }
 
+static inline void skb_queue_head_init_class(struct sk_buff_head *list,
+                struct lock_class_key *class)
+{
+        skb_queue_head_init(list);
+        lockdep_set_class(&list->lock, class);
+}
+
 /*
  *      Insert an sk_buff at the start of a list.
  *
@@ -817,12 +832,46 @@ static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
 #define SKB_FRAG_ASSERT(skb)    BUG_ON(skb_shinfo(skb)->frag_list)
 #define SKB_LINEAR_ASSERT(skb)  BUG_ON(skb_is_nonlinear(skb))
 
+#ifdef NET_SKBUFF_DATA_USES_OFFSET
+static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
+{
+        return skb->head + skb->tail;
+}
+
+static inline void skb_reset_tail_pointer(struct sk_buff *skb)
+{
+        skb->tail = skb->data - skb->head;
+}
+
+static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
+{
+        skb_reset_tail_pointer(skb);
+        skb->tail += offset;
+}
+#else /* NET_SKBUFF_DATA_USES_OFFSET */
+static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
+{
+        return skb->tail;
+}
+
+static inline void skb_reset_tail_pointer(struct sk_buff *skb)
+{
+        skb->tail = skb->data;
+}
+
+static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
+{
+        skb->tail = skb->data + offset;
+}
+
+#endif /* NET_SKBUFF_DATA_USES_OFFSET */
+
 /*
  *      Add data to an sk_buff
  */
 static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len)
 {
-        unsigned char *tmp = skb->tail;
+        unsigned char *tmp = skb_tail_pointer(skb);
         SKB_LINEAR_ASSERT(skb);
         skb->tail += len;
         skb->len  += len;
@@ -840,11 +889,11 @@ static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len)
  */
 static inline unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
 {
-        unsigned char *tmp = skb->tail;
+        unsigned char *tmp = skb_tail_pointer(skb);
         SKB_LINEAR_ASSERT(skb);
         skb->tail += len;
         skb->len  += len;
-        if (unlikely(skb->tail>skb->end))
+        if (unlikely(skb->tail > skb->end))
                 skb_over_panic(skb, len, current_text_addr());
         return tmp;
 }
@@ -957,6 +1006,130 @@ static inline void skb_reserve(struct sk_buff *skb, int len)
         skb->tail += len;
 }
 
+#ifdef NET_SKBUFF_DATA_USES_OFFSET
+static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
+{
+        return skb->head + skb->transport_header;
+}
+
+static inline void skb_reset_transport_header(struct sk_buff *skb)
+{
+        skb->transport_header = skb->data - skb->head;
+}
+
+static inline void skb_set_transport_header(struct sk_buff *skb,
+                                            const int offset)
+{
+        skb_reset_transport_header(skb);
+        skb->transport_header += offset;
+}
+
+static inline unsigned char *skb_network_header(const struct sk_buff *skb)
+{
+        return skb->head + skb->network_header;
+}
+
+static inline void skb_reset_network_header(struct sk_buff *skb)
+{
+        skb->network_header = skb->data - skb->head;
+}
+
+static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
+{
+        skb_reset_network_header(skb);
+        skb->network_header += offset;
+}
+
+static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
+{
+        return skb->head + skb->mac_header;
+}
+
+static inline int skb_mac_header_was_set(const struct sk_buff *skb)
+{
+        return skb->mac_header != ~0U;
+}
+
+static inline void skb_reset_mac_header(struct sk_buff *skb)
+{
+        skb->mac_header = skb->data - skb->head;
+}
+
+static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
+{
+        skb_reset_mac_header(skb);
+        skb->mac_header += offset;
+}
+
+#else /* NET_SKBUFF_DATA_USES_OFFSET */
+
+static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
+{
+        return skb->transport_header;
+}
+
+static inline void skb_reset_transport_header(struct sk_buff *skb)
+{
+        skb->transport_header = skb->data;
+}
+
+static inline void skb_set_transport_header(struct sk_buff *skb,
+                                            const int offset)
+{
+        skb->transport_header = skb->data + offset;
+}
+
+static inline unsigned char *skb_network_header(const struct sk_buff *skb)
+{
+        return skb->network_header;
+}
+
+static inline void skb_reset_network_header(struct sk_buff *skb)
+{
+        skb->network_header = skb->data;
+}
+
+static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
+{
+        skb->network_header = skb->data + offset;
+}
+
+static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
+{
+        return skb->mac_header;
+}
+
+static inline int skb_mac_header_was_set(const struct sk_buff *skb)
+{
+        return skb->mac_header != NULL;
+}
+
+static inline void skb_reset_mac_header(struct sk_buff *skb)
+{
+        skb->mac_header = skb->data;
+}
+
+static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
+{
+        skb->mac_header = skb->data + offset;
+}
+#endif /* NET_SKBUFF_DATA_USES_OFFSET */
+
+static inline int skb_transport_offset(const struct sk_buff *skb)
+{
+        return skb_transport_header(skb) - skb->data;
+}
+
+static inline u32 skb_network_header_len(const struct sk_buff *skb)
+{
+        return skb->transport_header - skb->network_header;
+}
+
+static inline int skb_network_offset(const struct sk_buff *skb)
+{
+        return skb_network_header(skb) - skb->data;
+}
+
 /*
  * CPUs often take a performance hit when accessing unaligned memory
  * locations. The actual performance hit varies, it can be small if the
@@ -1008,8 +1181,8 @@ static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
                 WARN_ON(1);
                 return;
         }
-        skb->len  = len;
-        skb->tail = skb->data + len;
+        skb->len = len;
+        skb_set_tail_pointer(skb, len);
 }
 
 /**
@@ -1298,6 +1471,11 @@ static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
                      prefetch(skb->next), (skb != (struct sk_buff *)(queue));   \
                      skb = skb->next)
 
+#define skb_queue_walk_safe(queue, skb, tmp)                                    \
+                for (skb = (queue)->next, tmp = skb->next;                      \
+                     skb != (struct sk_buff *)(queue);                          \
+                     skb = tmp, tmp = skb->next)
+
 #define skb_queue_reverse_walk(queue, skb) \
                 for (skb = (queue)->prev;                                       \
                      prefetch(skb->prev), (skb != (struct sk_buff *)(queue));   \
@@ -1321,8 +1499,8 @@ extern __wsum	       skb_checksum(const struct sk_buff *skb, int offset,
                                     int len, __wsum csum);
 extern int             skb_copy_bits(const struct sk_buff *skb, int offset,
                                      void *to, int len);
-extern int             skb_store_bits(const struct sk_buff *skb, int offset,
-                                      void *from, int len);
+extern int             skb_store_bits(struct sk_buff *skb, int offset,
+                                      const void *from, int len);
 extern __wsum          skb_copy_and_csum_bits(const struct sk_buff *skb,
                                               int offset, u8 *to, int len,
                                               __wsum csum);
@@ -1346,8 +1524,36 @@ static inline void *skb_header_pointer(const struct sk_buff *skb, int offset,
         return buffer;
 }
 
+static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
+                                             void *to,
+                                             const unsigned int len)
+{
+        memcpy(to, skb->data, len);
+}
+
+static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
+                                                    const int offset, void *to,
+                                                    const unsigned int len)
+{
+        memcpy(to, skb->data + offset, len);
+}
+
+static inline void skb_copy_to_linear_data(struct sk_buff *skb,
+                                           const void *from,
+                                           const unsigned int len)
+{
+        memcpy(skb->data, from, len);
+}
+
+static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
+                                                  const int offset,
+                                                  const void *from,
+                                                  const unsigned int len)
+{
+        memcpy(skb->data + offset, from, len);
+}
+
 extern void skb_init(void);
-extern void skb_add_mtu(int mtu);
 
 /**
  *      skb_get_timestamp - get timestamp from a skb
@@ -1360,29 +1566,28 @@ extern void skb_add_mtu(int mtu);
  */
 static inline void skb_get_timestamp(const struct sk_buff *skb, struct timeval *stamp)
 {
-        stamp->tv_sec  = skb->tstamp.off_sec;
-        stamp->tv_usec = skb->tstamp.off_usec;
+        *stamp = ktime_to_timeval(skb->tstamp);
 }
 
-/**
- *      skb_set_timestamp - set timestamp of a skb
- *      @skb: skb to set stamp of
- *      @stamp: pointer to struct timeval to get stamp from
- *
- *      Timestamps are stored in the skb as offsets to a base timestamp.
- *      This function converts a struct timeval to an offset and stores
- *      it in the skb.
- */
-static inline void skb_set_timestamp(struct sk_buff *skb, const struct timeval *stamp)
+static inline void __net_timestamp(struct sk_buff *skb)
+{
+        skb->tstamp = ktime_get_real();
+}
+
+static inline ktime_t net_timedelta(ktime_t t)
 {
-        skb->tstamp.off_sec  = stamp->tv_sec;
-        skb->tstamp.off_usec = stamp->tv_usec;
+        return ktime_sub(ktime_get_real(), t);
 }
 
-extern void __net_timestamp(struct sk_buff *skb);
 
+extern __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
 extern __sum16 __skb_checksum_complete(struct sk_buff *skb);
 
+static inline int skb_csum_unnecessary(const struct sk_buff *skb)
+{
+        return skb->ip_summed & CHECKSUM_UNNECESSARY;
+}
+
 /**
  *      skb_checksum_complete - Calculate checksum of an entire packet
  *      @skb: packet to process
@@ -1401,22 +1606,22 @@ extern __sum16 __skb_checksum_complete(struct sk_buff *skb);
  */
 static inline unsigned int skb_checksum_complete(struct sk_buff *skb)
 {
-        return skb->ip_summed != CHECKSUM_UNNECESSARY &&
-                __skb_checksum_complete(skb);
+        return skb_csum_unnecessary(skb) ?
+               0 : __skb_checksum_complete(skb);
 }
 
-#ifdef CONFIG_NETFILTER
+#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
+extern void nf_conntrack_destroy(struct nf_conntrack *nfct);
 static inline void nf_conntrack_put(struct nf_conntrack *nfct)
 {
         if (nfct && atomic_dec_and_test(&nfct->use))
-                nfct->destroy(nfct);
+                nf_conntrack_destroy(nfct);
 }
 static inline void nf_conntrack_get(struct nf_conntrack *nfct)
 {
         if (nfct)
                 atomic_inc(&nfct->use);
 }
-#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
 static inline void nf_conntrack_get_reasm(struct sk_buff *skb)
 {
         if (skb)
@@ -1442,9 +1647,9 @@ static inline void nf_bridge_get(struct nf_bridge_info *nf_bridge)
 #endif /* CONFIG_BRIDGE_NETFILTER */
 static inline void nf_reset(struct sk_buff *skb)
 {
+#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
         nf_conntrack_put(skb->nfct);
         skb->nfct = NULL;
-#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
         nf_conntrack_put_reasm(skb->nfct_reasm);
         skb->nfct_reasm = NULL;
 #endif
@@ -1454,9 +1659,33 @@ static inline void nf_reset(struct sk_buff *skb)
 #endif
 }
 
-#else /* CONFIG_NETFILTER */
-static inline void nf_reset(struct sk_buff *skb) {}
-#endif /* CONFIG_NETFILTER */
+/* Note: This doesn't put any conntrack and bridge info in dst. */
+static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src)
+{
+#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
+        dst->nfct = src->nfct;
+        nf_conntrack_get(src->nfct);
+        dst->nfctinfo = src->nfctinfo;
+        dst->nfct_reasm = src->nfct_reasm;
+        nf_conntrack_get_reasm(src->nfct_reasm);
+#endif
+#ifdef CONFIG_BRIDGE_NETFILTER
+        dst->nf_bridge  = src->nf_bridge;
+        nf_bridge_get(src->nf_bridge);
+#endif
+}
+
+static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
+{
+#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
+        nf_conntrack_put(dst->nfct);
+        nf_conntrack_put_reasm(dst->nfct_reasm);
+#endif
+#ifdef CONFIG_BRIDGE_NETFILTER
+        nf_bridge_put(dst->nf_bridge);
+#endif
+        __nf_copy(dst, src);
+}
 
 #ifdef CONFIG_NETWORK_SECMARK
 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
@@ -1481,5 +1710,12 @@ static inline int skb_is_gso(const struct sk_buff *skb)
         return skb_shinfo(skb)->gso_size;
 }
 
+static inline void skb_forward_csum(struct sk_buff *skb)
+{
+        /* Unfortunately we don't support this one.  Any brave souls? */
+        if (skb->ip_summed == CHECKSUM_COMPLETE)
+                skb->ip_summed = CHECKSUM_NONE;
+}
+
 #endif  /* __KERNEL__ */
 #endif  /* _LINUX_SKBUFF_H */