3 files changed, 26 insertions, 5 deletions

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 7b996ed86d5b..8bd383caa363 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -524,6 +524,7 @@ static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
 extern bool skb_recycle_check(struct sk_buff *skb, int skb_size);
 
 extern struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
+extern int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask);
 extern struct sk_buff *skb_clone(struct sk_buff *skb,
                                  gfp_t priority);
 extern struct sk_buff *skb_copy(const struct sk_buff *skb,
diff --git a/net/core/dev.c b/net/core/dev.c
index 17d67b579beb..b10ff0a71855 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -1515,6 +1515,14 @@ static inline bool is_skb_forwardable(struct net_device *dev,
  */
 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
 {
+        if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
+                if (skb_copy_ubufs(skb, GFP_ATOMIC)) {
+                        atomic_long_inc(&dev->rx_dropped);
+                        kfree_skb(skb);
+                        return NET_RX_DROP;
+                }
+        }
+
         skb_orphan(skb);
         nf_reset(skb);
 
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 27002dffe7ed..387703f56fce 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -611,8 +611,21 @@ struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
 }
 EXPORT_SYMBOL_GPL(skb_morph);
 
-/* skb frags copy userspace buffers to kernel */
-static int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
+/*      skb_copy_ubufs  -       copy userspace skb frags buffers to kernel
+ *      @skb: the skb to modify
+ *      @gfp_mask: allocation priority
+ *
+ *      This must be called on SKBTX_DEV_ZEROCOPY skb.
+ *      It will copy all frags into kernel and drop the reference
+ *      to userspace pages.
+ *
+ *      If this function is called from an interrupt gfp_mask() must be
+ *      %GFP_ATOMIC.
+ *
+ *      Returns 0 on success or a negative error code on failure
+ *      to allocate kernel memory to copy to.
+ */
+int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
 {
         int i;
         int num_frags = skb_shinfo(skb)->nr_frags;
@@ -652,6 +665,8 @@ static int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
                 skb_shinfo(skb)->frags[i - 1].page = head;
                 head = (struct page *)head->private;
         }
+
+        skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
         return 0;
 }
 
@@ -677,7 +692,6 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
         if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
                 if (skb_copy_ubufs(skb, gfp_mask))
                         return NULL;
-                skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
         }
 
         n = skb + 1;
@@ -803,7 +817,6 @@ struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask)
                                 n = NULL;
                                 goto out;
                         }
-                        skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
                 }
                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
                         skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
@@ -896,7 +909,6 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
                 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) {
                         if (skb_copy_ubufs(skb, gfp_mask))
                                 goto nofrags;
-                        skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY;
                 }
                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
                         get_page(skb_shinfo(skb)->frags[i].page);