1 files changed, 62 insertions, 95 deletions

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 3ab989b0de42..33245ef54c3b 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -104,47 +104,37 @@ static const struct pipe_buf_operations sock_pipe_buf_ops = {
         .get = sock_pipe_buf_get,
 };
 
-/*
- *      Keep out-of-line to prevent kernel bloat.
- *      __builtin_return_address is not used because it is not always
- *      reliable.
- */
-
 /**
- *      skb_over_panic  -       private function
- *      @skb: buffer
- *      @sz: size
- *      @here: address
- *
- *      Out of line support code for skb_put(). Not user callable.
+ *      skb_panic - private function for out-of-line support
+ *      @skb:   buffer
+ *      @sz:    size
+ *      @addr:  address
+ *      @msg:   skb_over_panic or skb_under_panic
+ *
+ *      Out-of-line support for skb_put() and skb_push().
+ *      Called via the wrapper skb_over_panic() or skb_under_panic().
+ *      Keep out of line to prevent kernel bloat.
+ *      __builtin_return_address is not used because it is not always reliable.
  */
-static void skb_over_panic(struct sk_buff *skb, int sz, void *here)
+static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr,
+                      const char msg[])
 {
         pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
-                 __func__, here, skb->len, sz, skb->head, skb->data,
+                 msg, addr, skb->len, sz, skb->head, skb->data,
                  (unsigned long)skb->tail, (unsigned long)skb->end,
                  skb->dev ? skb->dev->name : "<NULL>");
         BUG();
 }
 
-/**
- *      skb_under_panic -       private function
- *      @skb: buffer
- *      @sz: size
- *      @here: address
- *
- *      Out of line support code for skb_push(). Not user callable.
- */
-
-static void skb_under_panic(struct sk_buff *skb, int sz, void *here)
+static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr)
 {
-        pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
-                 __func__, here, skb->len, sz, skb->head, skb->data,
-                 (unsigned long)skb->tail, (unsigned long)skb->end,
-                 skb->dev ? skb->dev->name : "<NULL>");
-        BUG();
+        skb_panic(skb, sz, addr, __func__);
 }
 
+static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
+{
+        skb_panic(skb, sz, addr, __func__);
+}
 
 /*
  * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells
@@ -155,8 +145,9 @@ static void skb_under_panic(struct sk_buff *skb, int sz, void *here)
  */
 #define kmalloc_reserve(size, gfp, node, pfmemalloc) \
          __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc)
-void *__kmalloc_reserve(size_t size, gfp_t flags, int node, unsigned long ip,
-                         bool *pfmemalloc)
+
+static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
+                               unsigned long ip, bool *pfmemalloc)
 {
         void *obj;
         bool ret_pfmemalloc = false;
@@ -259,6 +250,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
         skb->end = skb->tail + size;
 #ifdef NET_SKBUFF_DATA_USES_OFFSET
         skb->mac_header = ~0U;
+        skb->transport_header = ~0U;
 #endif
 
         /* make sure we initialize shinfo sequentially */
@@ -327,6 +319,7 @@ struct sk_buff *build_skb(void *data, unsigned int frag_size)
         skb->end = skb->tail + size;
 #ifdef NET_SKBUFF_DATA_USES_OFFSET
         skb->mac_header = ~0U;
+        skb->transport_header = ~0U;
 #endif
 
         /* make sure we initialize shinfo sequentially */
@@ -348,10 +341,6 @@ struct netdev_alloc_cache {
 };
 static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache);
 
-#define NETDEV_FRAG_PAGE_MAX_ORDER get_order(32768)
-#define NETDEV_FRAG_PAGE_MAX_SIZE  (PAGE_SIZE << NETDEV_FRAG_PAGE_MAX_ORDER)
-#define NETDEV_PAGECNT_MAX_BIAS    NETDEV_FRAG_PAGE_MAX_SIZE
-
 static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
 {
         struct netdev_alloc_cache *nc;
@@ -683,7 +672,7 @@ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
         new->network_header     = old->network_header;
         new->mac_header         = old->mac_header;
         new->inner_transport_header = old->inner_transport_header;
-        new->inner_network_header = old->inner_transport_header;
+        new->inner_network_header = old->inner_network_header;
         skb_dst_copy(new, old);
         new->rxhash             = old->rxhash;
         new->ooo_okay           = old->ooo_okay;
@@ -1649,7 +1638,7 @@ static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
 
 static struct page *linear_to_page(struct page *page, unsigned int *len,
                                    unsigned int *offset,
-                                   struct sk_buff *skb, struct sock *sk)
+                                   struct sock *sk)
 {
         struct page_frag *pfrag = sk_page_frag(sk);
 
@@ -1682,14 +1671,14 @@ static bool spd_can_coalesce(const struct splice_pipe_desc *spd,
 static bool spd_fill_page(struct splice_pipe_desc *spd,
                           struct pipe_inode_info *pipe, struct page *page,
                           unsigned int *len, unsigned int offset,
-                          struct sk_buff *skb, bool linear,
+                          bool linear,
                           struct sock *sk)
 {
         if (unlikely(spd->nr_pages == MAX_SKB_FRAGS))
                 return true;
 
         if (linear) {
-                page = linear_to_page(page, len, &offset, skb, sk);
+                page = linear_to_page(page, len, &offset, sk);
                 if (!page)
                         return true;
         }
@@ -1706,23 +1695,9 @@ static bool spd_fill_page(struct splice_pipe_desc *spd,
         return false;
 }
 
-static inline void __segment_seek(struct page **page, unsigned int *poff,
-                                  unsigned int *plen, unsigned int off)
-{
-        unsigned long n;
-
-        *poff += off;
-        n = *poff / PAGE_SIZE;
-        if (n)
-                *page = nth_page(*page, n);
-
-        *poff = *poff % PAGE_SIZE;
-        *plen -= off;
-}
-
 static bool __splice_segment(struct page *page, unsigned int poff,
                              unsigned int plen, unsigned int *off,
-                             unsigned int *len, struct sk_buff *skb,
+                             unsigned int *len,
                              struct splice_pipe_desc *spd, bool linear,
                              struct sock *sk,
                              struct pipe_inode_info *pipe)
@@ -1737,23 +1712,19 @@ static bool __splice_segment(struct page *page, unsigned int poff,
         }
 
         /* ignore any bits we already processed */
-        if (*off) {
-                __segment_seek(&page, &poff, &plen, *off);
-                *off = 0;
-        }
+        poff += *off;
+        plen -= *off;
+        *off = 0;
 
         do {
                 unsigned int flen = min(*len, plen);
 
-                /* the linear region may spread across several pages  */
-                flen = min_t(unsigned int, flen, PAGE_SIZE - poff);
-
-                if (spd_fill_page(spd, pipe, page, &flen, poff, skb, linear, sk))
+                if (spd_fill_page(spd, pipe, page, &flen, poff,
+                                  linear, sk))
                         return true;
-
-                __segment_seek(&page, &poff, &plen, flen);
+                poff += flen;
+                plen -= flen;
                 *len -= flen;
-
         } while (*len && plen);
 
         return false;
@@ -1777,7 +1748,7 @@ static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
         if (__splice_segment(virt_to_page(skb->data),
                              (unsigned long) skb->data & (PAGE_SIZE - 1),
                              skb_headlen(skb),
-                             offset, len, skb, spd,
+                             offset, len, spd,
                              skb_head_is_locked(skb),
                              sk, pipe))
                 return true;
@@ -1790,7 +1761,7 @@ static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
 
                 if (__splice_segment(skb_frag_page(f),
                                      f->page_offset, skb_frag_size(f),
-                                     offset, len, skb, spd, false, sk, pipe))
+                                     offset, len, spd, false, sk, pipe))
                         return true;
         }
 
@@ -2355,6 +2326,7 @@ void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
 {
         int pos = skb_headlen(skb);
 
+        skb_shinfo(skb1)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
         if (len < pos)  /* Split line is inside header. */
                 skb_split_inside_header(skb, skb1, len, pos);
         else            /* Second chunk has no header, nothing to copy. */
@@ -2686,48 +2658,37 @@ int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
                                         int len, int odd, struct sk_buff *skb),
                         void *from, int length)
 {
-        int frg_cnt = 0;
-        skb_frag_t *frag = NULL;
-        struct page *page = NULL;
-        int copy, left;
+        int frg_cnt = skb_shinfo(skb)->nr_frags;
+        int copy;
         int offset = 0;
         int ret;
+        struct page_frag *pfrag = &current->task_frag;
 
         do {
                 /* Return error if we don't have space for new frag */
-                frg_cnt = skb_shinfo(skb)->nr_frags;
                 if (frg_cnt >= MAX_SKB_FRAGS)
-                        return -EFAULT;
+                        return -EMSGSIZE;
 
-                /* allocate a new page for next frag */
-                page = alloc_pages(sk->sk_allocation, 0);
-
-                /* If alloc_page fails just return failure and caller will
-                 * free previous allocated pages by doing kfree_skb()
-                 */
-                if (page == NULL)
+                if (!sk_page_frag_refill(sk, pfrag))
                         return -ENOMEM;
 
-                /* initialize the next frag */
-                skb_fill_page_desc(skb, frg_cnt, page, 0, 0);
-                skb->truesize += PAGE_SIZE;
-                atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc);
-
-                /* get the new initialized frag */
-                frg_cnt = skb_shinfo(skb)->nr_frags;
-                frag = &skb_shinfo(skb)->frags[frg_cnt - 1];
-
                 /* copy the user data to page */
-                left = PAGE_SIZE - frag->page_offset;
-                copy = (length > left)? left : length;
+                copy = min_t(int, length, pfrag->size - pfrag->offset);
 
-                ret = getfrag(from, skb_frag_address(frag) + skb_frag_size(frag),
-                            offset, copy, 0, skb);
+                ret = getfrag(from, page_address(pfrag->page) + pfrag->offset,
+                              offset, copy, 0, skb);
                 if (ret < 0)
                         return -EFAULT;
 
                 /* copy was successful so update the size parameters */
-                skb_frag_size_add(frag, copy);
+                skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset,
+                                   copy);
+                frg_cnt++;
+                pfrag->offset += copy;
+                get_page(pfrag->page);
+
+                skb->truesize += copy;
+                atomic_add(copy, &sk->sk_wmem_alloc);
                 skb->len += copy;
                 skb->data_len += copy;
                 offset += copy;
@@ -2777,6 +2738,7 @@ struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features)
         unsigned int mss = skb_shinfo(skb)->gso_size;
         unsigned int doffset = skb->data - skb_mac_header(skb);
         unsigned int offset = doffset;
+        unsigned int tnl_hlen = skb_tnl_header_len(skb);
         unsigned int headroom;
         unsigned int len;
         int sg = !!(features & NETIF_F_SG);
@@ -2853,7 +2815,10 @@ struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features)
                 skb_set_network_header(nskb, skb->mac_len);
                 nskb->transport_header = (nskb->network_header +
                                           skb_network_header_len(skb));
-                skb_copy_from_linear_data(skb, nskb->data, doffset);
+
+                skb_copy_from_linear_data_offset(skb, -tnl_hlen,
+                                                 nskb->data - tnl_hlen,
+                                                 doffset + tnl_hlen);
 
                 if (fskb != skb_shinfo(skb)->frag_list)
                         continue;
@@ -2871,6 +2836,8 @@ struct sk_buff *skb_segment(struct sk_buff *skb, netdev_features_t features)
                 skb_copy_from_linear_data_offset(skb, offset,
                                                  skb_put(nskb, hsize), hsize);
 
+                skb_shinfo(nskb)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG;
+
                 while (pos < offset + len && i < nfrags) {
                         *frag = skb_shinfo(skb)->frags[i];
                         __skb_frag_ref(frag);