1 files changed, 92 insertions, 49 deletions

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index d81cff119f73..8c5197fe55a4 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -476,6 +476,18 @@ void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
 }
 EXPORT_SYMBOL(skb_add_rx_frag);
 
+void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size,
+                          unsigned int truesize)
+{
+        skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+
+        skb_frag_size_add(frag, size);
+        skb->len += size;
+        skb->data_len += size;
+        skb->truesize += truesize;
+}
+EXPORT_SYMBOL(skb_coalesce_rx_frag);
+
 static void skb_drop_list(struct sk_buff **listp)
 {
         kfree_skb_list(*listp);
@@ -903,6 +915,9 @@ EXPORT_SYMBOL(skb_clone);
 
 static void skb_headers_offset_update(struct sk_buff *skb, int off)
 {
+        /* Only adjust this if it actually is csum_start rather than csum */
+        if (skb->ip_summed == CHECKSUM_PARTIAL)
+                skb->csum_start += off;
         /* {transport,network,mac}_header and tail are relative to skb->head */
         skb->transport_header += off;
         skb->network_header   += off;
@@ -1036,8 +1051,8 @@ EXPORT_SYMBOL(__pskb_copy);
  *      @ntail: room to add at tail
  *      @gfp_mask: allocation priority
  *
- *      Expands (or creates identical copy, if &nhead and &ntail are zero)
- *      header of skb. &sk_buff itself is not changed. &sk_buff MUST have
+ *      Expands (or creates identical copy, if @nhead and @ntail are zero)
+ *      header of @skb. &sk_buff itself is not changed. &sk_buff MUST have
  *      reference count of 1. Returns zero in the case of success or error,
  *      if expansion failed. In the last case, &sk_buff is not changed.
  *
@@ -1109,9 +1124,6 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
 #endif
         skb->tail             += off;
         skb_headers_offset_update(skb, nhead);
-        /* Only adjust this if it actually is csum_start rather than csum */
-        if (skb->ip_summed == CHECKSUM_PARTIAL)
-                skb->csum_start += nhead;
         skb->cloned   = 0;
         skb->hdr_len  = 0;
         skb->nohdr    = 0;
@@ -1176,7 +1188,6 @@ struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
                                         NUMA_NO_NODE);
         int oldheadroom = skb_headroom(skb);
         int head_copy_len, head_copy_off;
-        int off;
 
         if (!n)
                 return NULL;
@@ -1200,11 +1211,7 @@ struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
 
         copy_skb_header(n, skb);
 
-        off                  = newheadroom - oldheadroom;
-        if (n->ip_summed == CHECKSUM_PARTIAL)
-                n->csum_start += off;
-
-        skb_headers_offset_update(n, off);
+        skb_headers_offset_update(n, newheadroom - oldheadroom);
 
         return n;
 }
@@ -1257,6 +1264,29 @@ free_skb:
 EXPORT_SYMBOL(skb_pad);
 
 /**
+ *      pskb_put - add data to the tail of a potentially fragmented buffer
+ *      @skb: start of the buffer to use
+ *      @tail: tail fragment of the buffer to use
+ *      @len: amount of data to add
+ *
+ *      This function extends the used data area of the potentially
+ *      fragmented buffer. @tail must be the last fragment of @skb -- or
+ *      @skb itself. If this would exceed the total buffer size the kernel
+ *      will panic. A pointer to the first byte of the extra data is
+ *      returned.
+ */
+
+unsigned char *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len)
+{
+        if (tail != skb) {
+                skb->data_len += len;
+                skb->len += len;
+        }
+        return skb_put(tail, len);
+}
+EXPORT_SYMBOL_GPL(pskb_put);
+
+/**
  *      skb_put - add data to a buffer
  *      @skb: buffer to use
  *      @len: amount of data to add
@@ -1933,9 +1963,8 @@ fault:
 EXPORT_SYMBOL(skb_store_bits);
 
 /* Checksum skb data. */
-
-__wsum skb_checksum(const struct sk_buff *skb, int offset,
-                          int len, __wsum csum)
+__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len,
+                      __wsum csum, const struct skb_checksum_ops *ops)
 {
         int start = skb_headlen(skb);
         int i, copy = start - offset;
@@ -1946,7 +1975,7 @@ __wsum skb_checksum(const struct sk_buff *skb, int offset,
         if (copy > 0) {
                 if (copy > len)
                         copy = len;
-                csum = csum_partial(skb->data + offset, copy, csum);
+                csum = ops->update(skb->data + offset, copy, csum);
                 if ((len -= copy) == 0)
                         return csum;
                 offset += copy;
@@ -1967,10 +1996,10 @@ __wsum skb_checksum(const struct sk_buff *skb, int offset,
                         if (copy > len)
                                 copy = len;
                         vaddr = kmap_atomic(skb_frag_page(frag));
-                        csum2 = csum_partial(vaddr + frag->page_offset +
-                                             offset - start, copy, 0);
+                        csum2 = ops->update(vaddr + frag->page_offset +
+                                            offset - start, copy, 0);
                         kunmap_atomic(vaddr);
-                        csum = csum_block_add(csum, csum2, pos);
+                        csum = ops->combine(csum, csum2, pos, copy);
                         if (!(len -= copy))
                                 return csum;
                         offset += copy;
@@ -1989,9 +2018,9 @@ __wsum skb_checksum(const struct sk_buff *skb, int offset,
                         __wsum csum2;
                         if (copy > len)
                                 copy = len;
-                        csum2 = skb_checksum(frag_iter, offset - start,
-                                             copy, 0);
-                        csum = csum_block_add(csum, csum2, pos);
+                        csum2 = __skb_checksum(frag_iter, offset - start,
+                                               copy, 0, ops);
+                        csum = ops->combine(csum, csum2, pos, copy);
                         if ((len -= copy) == 0)
                                 return csum;
                         offset += copy;
@@ -2003,6 +2032,18 @@ __wsum skb_checksum(const struct sk_buff *skb, int offset,
 
         return csum;
 }
+EXPORT_SYMBOL(__skb_checksum);
+
+__wsum skb_checksum(const struct sk_buff *skb, int offset,
+                    int len, __wsum csum)
+{
+        const struct skb_checksum_ops ops = {
+                .update  = csum_partial_ext,
+                .combine = csum_block_add_ext,
+        };
+
+        return __skb_checksum(skb, offset, len, csum, &ops);
+}
 EXPORT_SYMBOL(skb_checksum);
 
 /* Both of above in one bottle. */
@@ -2522,14 +2563,14 @@ EXPORT_SYMBOL(skb_prepare_seq_read);
  * @data: destination pointer for data to be returned
  * @st: state variable
  *
- * Reads a block of skb data at &consumed relative to the
+ * Reads a block of skb data at @consumed relative to the
  * lower offset specified to skb_prepare_seq_read(). Assigns
- * the head of the data block to &data and returns the length
+ * the head of the data block to @data and returns the length
  * of the block or 0 if the end of the skb data or the upper
  * offset has been reached.
  *
  * The caller is not required to consume all of the data
- * returned, i.e. &consumed is typically set to the number
+ * returned, i.e. @consumed is typically set to the number
  * of bytes already consumed and the next call to
  * skb_seq_read() will return the remaining part of the block.
  *
@@ -2837,14 +2878,7 @@ struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features)
                 __copy_skb_header(nskb, skb);
                 nskb->mac_len = skb->mac_len;
 
-                /* nskb and skb might have different headroom */
-                if (nskb->ip_summed == CHECKSUM_PARTIAL)
-                        nskb->csum_start += skb_headroom(nskb) - headroom;
-
-                skb_reset_mac_header(nskb);
-                skb_set_network_header(nskb, skb->mac_len);
-                nskb->transport_header = (nskb->network_header +
-                                          skb_network_header_len(skb));
+                skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom);
 
                 skb_copy_from_linear_data_offset(skb, -tnl_hlen,
                                                  nskb->data - tnl_hlen,
@@ -2936,32 +2970,30 @@ EXPORT_SYMBOL_GPL(skb_segment);
 
 int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
 {
-        struct sk_buff *p = *head;
-        struct sk_buff *nskb;
-        struct skb_shared_info *skbinfo = skb_shinfo(skb);
-        struct skb_shared_info *pinfo = skb_shinfo(p);
-        unsigned int headroom;
-        unsigned int len = skb_gro_len(skb);
+        struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb);
         unsigned int offset = skb_gro_offset(skb);
         unsigned int headlen = skb_headlen(skb);
+        struct sk_buff *nskb, *lp, *p = *head;
+        unsigned int len = skb_gro_len(skb);
         unsigned int delta_truesize;
+        unsigned int headroom;
 
-        if (p->len + len >= 65536)
+        if (unlikely(p->len + len >= 65536))
                 return -E2BIG;
 
-        if (pinfo->frag_list)
-                goto merge;
-        else if (headlen <= offset) {
+        lp = NAPI_GRO_CB(p)->last ?: p;
+        pinfo = skb_shinfo(lp);
+
+        if (headlen <= offset) {
                 skb_frag_t *frag;
                 skb_frag_t *frag2;
                 int i = skbinfo->nr_frags;
                 int nr_frags = pinfo->nr_frags + i;
 
-                offset -= headlen;
-
                 if (nr_frags > MAX_SKB_FRAGS)
-                        return -E2BIG;
+                        goto merge;
 
+                offset -= headlen;
                 pinfo->nr_frags = nr_frags;
                 skbinfo->nr_frags = 0;
 
@@ -2992,7 +3024,7 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
                 unsigned int first_offset;
 
                 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
-                        return -E2BIG;
+                        goto merge;
 
                 first_offset = skb->data -
                                (unsigned char *)page_address(page) +
@@ -3010,7 +3042,10 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
                 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff));
                 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD;
                 goto done;
-        } else if (skb_gro_len(p) != pinfo->gso_size)
+        }
+        if (pinfo->frag_list)
+                goto merge;
+        if (skb_gro_len(p) != pinfo->gso_size)
                 return -E2BIG;
 
         headroom = skb_headroom(p);
@@ -3062,16 +3097,24 @@ merge:
 
         __skb_pull(skb, offset);
 
-        NAPI_GRO_CB(p)->last->next = skb;
+        if (!NAPI_GRO_CB(p)->last)
+                skb_shinfo(p)->frag_list = skb;
+        else
+                NAPI_GRO_CB(p)->last->next = skb;
         NAPI_GRO_CB(p)->last = skb;
         skb_header_release(skb);
+        lp = p;
 
 done:
         NAPI_GRO_CB(p)->count++;
         p->data_len += len;
         p->truesize += delta_truesize;
         p->len += len;
-
+        if (lp != p) {
+                lp->data_len += len;
+                lp->truesize += delta_truesize;
+                lp->len += len;
+        }
         NAPI_GRO_CB(skb)->same_flow = 1;
         return 0;
 }