12 files changed, 223 insertions, 171 deletions

diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
index a8bb8f664d3d..b10d1744e5ae 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
@@ -662,7 +662,7 @@ static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 {
         if (fp->rx_frag_size)
-                put_page(virt_to_head_page(data));
+                skb_free_frag(data);
         else
                 kfree(data);
 }
diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c
index 069952fa5d64..73c934cf6c61 100644
--- a/drivers/net/ethernet/broadcom/tg3.c
+++ b/drivers/net/ethernet/broadcom/tg3.c
@@ -6618,7 +6618,7 @@ static void tg3_tx(struct tg3_napi *tnapi)
 static void tg3_frag_free(bool is_frag, void *data)
 {
         if (is_frag)
-                put_page(virt_to_head_page(data));
+                skb_free_frag(data);
         else
                 kfree(data);
 }
diff --git a/drivers/net/ethernet/hisilicon/hip04_eth.c b/drivers/net/ethernet/hisilicon/hip04_eth.c
index 3b39fdddeb57..d49bee38cd31 100644
--- a/drivers/net/ethernet/hisilicon/hip04_eth.c
+++ b/drivers/net/ethernet/hisilicon/hip04_eth.c
@@ -798,7 +798,7 @@ static void hip04_free_ring(struct net_device *ndev, struct device *d)
 
         for (i = 0; i < RX_DESC_NUM; i++)
                 if (priv->rx_buf[i])
-                        put_page(virt_to_head_page(priv->rx_buf[i]));
+                        skb_free_frag(priv->rx_buf[i]);
 
         for (i = 0; i < TX_DESC_NUM; i++)
                 if (priv->tx_skb[i])
diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c
index 983eb4e6f7aa..74dc15055971 100644
--- a/drivers/net/ethernet/intel/e1000/e1000_main.c
+++ b/drivers/net/ethernet/intel/e1000/e1000_main.c
@@ -2079,11 +2079,6 @@ static void *e1000_alloc_frag(const struct e1000_adapter *a)
         return data;
 }
 
-static void e1000_free_frag(const void *data)
-{
-        put_page(virt_to_head_page(data));
-}
-
 /**
  * e1000_clean_rx_ring - Free Rx Buffers per Queue
  * @adapter: board private structure
@@ -2107,7 +2102,7 @@ static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
                                                  adapter->rx_buffer_len,
                                                  DMA_FROM_DEVICE);
                         if (buffer_info->rxbuf.data) {
-                                e1000_free_frag(buffer_info->rxbuf.data);
+                                skb_free_frag(buffer_info->rxbuf.data);
                                 buffer_info->rxbuf.data = NULL;
                         }
                 } else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) {
@@ -4594,28 +4589,28 @@ static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
                         data = e1000_alloc_frag(adapter);
                         /* Failed allocation, critical failure */
                         if (!data) {
-                                e1000_free_frag(olddata);
+                                skb_free_frag(olddata);
                                 adapter->alloc_rx_buff_failed++;
                                 break;
                         }
 
                         if (!e1000_check_64k_bound(adapter, data, bufsz)) {
                                 /* give up */
-                                e1000_free_frag(data);
-                                e1000_free_frag(olddata);
+                                skb_free_frag(data);
+                                skb_free_frag(olddata);
                                 adapter->alloc_rx_buff_failed++;
                                 break;
                         }
 
                         /* Use new allocation */
-                        e1000_free_frag(olddata);
+                        skb_free_frag(olddata);
                 }
                 buffer_info->dma = dma_map_single(&pdev->dev,
                                                   data,
                                                   adapter->rx_buffer_len,
                                                   DMA_FROM_DEVICE);
                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
-                        e1000_free_frag(data);
+                        skb_free_frag(data);
                         buffer_info->dma = 0;
                         adapter->alloc_rx_buff_failed++;
                         break;
@@ -4637,7 +4632,7 @@ static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
                                          adapter->rx_buffer_len,
                                          DMA_FROM_DEVICE);
 
-                        e1000_free_frag(data);
+                        skb_free_frag(data);
                         buffer_info->rxbuf.data = NULL;
                         buffer_info->dma = 0;
 
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index e63664649029..8a45ed7506c5 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -4974,6 +4974,7 @@ netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
         struct igb_tx_buffer *first;
         int tso;
         u32 tx_flags = 0;
+        unsigned short f;
         u16 count = TXD_USE_COUNT(skb_headlen(skb));
         __be16 protocol = vlan_get_protocol(skb);
         u8 hdr_len = 0;
@@ -4984,14 +4985,8 @@ netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
          *       + 1 desc for context descriptor,
          * otherwise try next time
          */
-        if (NETDEV_FRAG_PAGE_MAX_SIZE > IGB_MAX_DATA_PER_TXD) {
-                unsigned short f;
-
-                for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
-                        count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
-        } else {
-                count += skb_shinfo(skb)->nr_frags;
-        }
+        for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
+                count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
 
         if (igb_maybe_stop_tx(tx_ring, count + 3)) {
                 /* this is a hard error */
diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c
index ce5f7f9cff06..ecce8261ce3b 100644
--- a/drivers/net/ethernet/marvell/mvneta.c
+++ b/drivers/net/ethernet/marvell/mvneta.c
@@ -1359,7 +1359,7 @@ static void *mvneta_frag_alloc(const struct mvneta_port *pp)
 static void mvneta_frag_free(const struct mvneta_port *pp, void *data)
 {
         if (likely(pp->frag_size <= PAGE_SIZE))
-                put_page(virt_to_head_page(data));
+                skb_free_frag(data);
         else
                 kfree(data);
 }
diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c
index 43efc3a0cda5..0a28c07361cf 100644
--- a/drivers/net/ethernet/ti/netcp_core.c
+++ b/drivers/net/ethernet/ti/netcp_core.c
@@ -537,7 +537,7 @@ int netcp_unregister_rxhook(struct netcp_intf *netcp_priv, int order,
 static void netcp_frag_free(bool is_frag, void *ptr)
 {
         if (is_frag)
-                put_page(virt_to_head_page(ptr));
+                skb_free_frag(ptr);
         else
                 kfree(ptr);
 }
diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index 97a9373e61e8..70a7fee1efb3 100644
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -366,6 +366,11 @@ extern void free_pages(unsigned long addr, unsigned int order);
 extern void free_hot_cold_page(struct page *page, bool cold);
 extern void free_hot_cold_page_list(struct list_head *list, bool cold);
 
+struct page_frag_cache;
+extern void *__alloc_page_frag(struct page_frag_cache *nc,
+                               unsigned int fragsz, gfp_t gfp_mask);
+extern void __free_page_frag(void *addr);
+
 extern void __free_kmem_pages(struct page *page, unsigned int order);
 extern void free_kmem_pages(unsigned long addr, unsigned int order);
 
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 8d37e26a1007..0038ac7466fd 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -226,6 +226,24 @@ struct page_frag {
 #endif
 };
 
+#define PAGE_FRAG_CACHE_MAX_SIZE        __ALIGN_MASK(32768, ~PAGE_MASK)
+#define PAGE_FRAG_CACHE_MAX_ORDER       get_order(PAGE_FRAG_CACHE_MAX_SIZE)
+
+struct page_frag_cache {
+        void * va;
+#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
+        __u16 offset;
+        __u16 size;
+#else
+        __u32 offset;
+#endif
+        /* we maintain a pagecount bias, so that we dont dirty cache line
+         * containing page->_count every time we allocate a fragment.
+         */
+        unsigned int            pagecnt_bias;
+        bool pfmemalloc;
+};
+
 typedef unsigned long __nocast vm_flags_t;
 
 /*
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 9c2f793573fa..c0b574a414e7 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -2128,10 +2128,6 @@ static inline void __skb_queue_purge(struct sk_buff_head *list)
                 kfree_skb(skb);
 }
 
-#define NETDEV_FRAG_PAGE_MAX_ORDER get_order(32768)
-#define NETDEV_FRAG_PAGE_MAX_SIZE  (PAGE_SIZE << NETDEV_FRAG_PAGE_MAX_ORDER)
-#define NETDEV_PAGECNT_MAX_BIAS    NETDEV_FRAG_PAGE_MAX_SIZE
-
 void *netdev_alloc_frag(unsigned int fragsz);
 
 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int length,
@@ -2186,6 +2182,11 @@ static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
         return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
 }
 
+static inline void skb_free_frag(void *addr)
+{
+        __free_page_frag(addr);
+}
+
 void *napi_alloc_frag(unsigned int fragsz);
 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,
                                  unsigned int length, gfp_t gfp_mask);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index ebffa0e4a9c0..2fd31aebef30 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2967,6 +2967,104 @@ void free_pages(unsigned long addr, unsigned int order)
 EXPORT_SYMBOL(free_pages);
 
 /*
+ * Page Fragment:
+ *  An arbitrary-length arbitrary-offset area of memory which resides
+ *  within a 0 or higher order page.  Multiple fragments within that page
+ *  are individually refcounted, in the page's reference counter.
+ *
+ * The page_frag functions below provide a simple allocation framework for
+ * page fragments.  This is used by the network stack and network device
+ * drivers to provide a backing region of memory for use as either an
+ * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
+ */
+static struct page *__page_frag_refill(struct page_frag_cache *nc,
+                                       gfp_t gfp_mask)
+{
+        struct page *page = NULL;
+        gfp_t gfp = gfp_mask;
+
+#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
+        gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
+                    __GFP_NOMEMALLOC;
+        page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
+                                PAGE_FRAG_CACHE_MAX_ORDER);
+        nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
+#endif
+        if (unlikely(!page))
+                page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
+
+        nc->va = page ? page_address(page) : NULL;
+
+        return page;
+}
+
+void *__alloc_page_frag(struct page_frag_cache *nc,
+                        unsigned int fragsz, gfp_t gfp_mask)
+{
+        unsigned int size = PAGE_SIZE;
+        struct page *page;
+        int offset;
+
+        if (unlikely(!nc->va)) {
+refill:
+                page = __page_frag_refill(nc, gfp_mask);
+                if (!page)
+                        return NULL;
+
+#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
+                /* if size can vary use size else just use PAGE_SIZE */
+                size = nc->size;
+#endif
+                /* Even if we own the page, we do not use atomic_set().
+                 * This would break get_page_unless_zero() users.
+                 */
+                atomic_add(size - 1, &page->_count);
+
+                /* reset page count bias and offset to start of new frag */
+                nc->pfmemalloc = page->pfmemalloc;
+                nc->pagecnt_bias = size;
+                nc->offset = size;
+        }
+
+        offset = nc->offset - fragsz;
+        if (unlikely(offset < 0)) {
+                page = virt_to_page(nc->va);
+
+                if (!atomic_sub_and_test(nc->pagecnt_bias, &page->_count))
+                        goto refill;
+
+#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
+                /* if size can vary use size else just use PAGE_SIZE */
+                size = nc->size;
+#endif
+                /* OK, page count is 0, we can safely set it */
+                atomic_set(&page->_count, size);
+
+                /* reset page count bias and offset to start of new frag */
+                nc->pagecnt_bias = size;
+                offset = size - fragsz;
+        }
+
+        nc->pagecnt_bias--;
+        nc->offset = offset;
+
+        return nc->va + offset;
+}
+EXPORT_SYMBOL(__alloc_page_frag);
+
+/*
+ * Frees a page fragment allocated out of either a compound or order 0 page.
+ */
+void __free_page_frag(void *addr)
+{
+        struct page *page = virt_to_head_page(addr);
+
+        if (unlikely(put_page_testzero(page)))
+                __free_pages_ok(page, compound_order(page));
+}
+EXPORT_SYMBOL(__free_page_frag);
+
+/*
  * alloc_kmem_pages charges newly allocated pages to the kmem resource counter
  * of the current memory cgroup.
  *
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index b9eb90b39ac7..d67e612bf0ef 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -347,94 +347,18 @@ struct sk_buff *build_skb(void *data, unsigned int frag_size)
 }
 EXPORT_SYMBOL(build_skb);
 
-struct netdev_alloc_cache {
-        struct page_frag        frag;
-        /* we maintain a pagecount bias, so that we dont dirty cache line
-         * containing page->_count every time we allocate a fragment.
-         */
-        unsigned int            pagecnt_bias;
-};
-static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache);
-static DEFINE_PER_CPU(struct netdev_alloc_cache, napi_alloc_cache);
-
-static struct page *__page_frag_refill(struct netdev_alloc_cache *nc,
-                                       gfp_t gfp_mask)
-{
-        const unsigned int order = NETDEV_FRAG_PAGE_MAX_ORDER;
-        struct page *page = NULL;
-        gfp_t gfp = gfp_mask;
-
-        if (order) {
-                gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
-                            __GFP_NOMEMALLOC;
-                page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
-                nc->frag.size = PAGE_SIZE << (page ? order : 0);
-        }
-
-        if (unlikely(!page))
-                page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
-
-        nc->frag.page = page;
-
-        return page;
-}
-
-static void *__alloc_page_frag(struct netdev_alloc_cache __percpu *cache,
-                               unsigned int fragsz, gfp_t gfp_mask)
-{
-        struct netdev_alloc_cache *nc = this_cpu_ptr(cache);
-        struct page *page = nc->frag.page;
-        unsigned int size;
-        int offset;
-
-        if (unlikely(!page)) {
-refill:
-                page = __page_frag_refill(nc, gfp_mask);
-                if (!page)
-                        return NULL;
-
-                /* if size can vary use frag.size else just use PAGE_SIZE */
-                size = NETDEV_FRAG_PAGE_MAX_ORDER ? nc->frag.size : PAGE_SIZE;
-
-                /* Even if we own the page, we do not use atomic_set().
-                 * This would break get_page_unless_zero() users.
-                 */
-                atomic_add(size - 1, &page->_count);
-
-                /* reset page count bias and offset to start of new frag */
-                nc->pagecnt_bias = size;
-                nc->frag.offset = size;
-        }
-
-        offset = nc->frag.offset - fragsz;
-        if (unlikely(offset < 0)) {
-                if (!atomic_sub_and_test(nc->pagecnt_bias, &page->_count))
-                        goto refill;
-
-                /* if size can vary use frag.size else just use PAGE_SIZE */
-                size = NETDEV_FRAG_PAGE_MAX_ORDER ? nc->frag.size : PAGE_SIZE;
-
-                /* OK, page count is 0, we can safely set it */
-                atomic_set(&page->_count, size);
-
-                /* reset page count bias and offset to start of new frag */
-                nc->pagecnt_bias = size;
-                offset = size - fragsz;
-        }
-
-        nc->pagecnt_bias--;
-        nc->frag.offset = offset;
-
-        return page_address(page) + offset;
-}
+static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
+static DEFINE_PER_CPU(struct page_frag_cache, napi_alloc_cache);
 
 static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
 {
+        struct page_frag_cache *nc;
         unsigned long flags;
         void *data;
 
         local_irq_save(flags);
-        data = __alloc_page_frag(&netdev_alloc_cache, fragsz, gfp_mask);
+        nc = this_cpu_ptr(&netdev_alloc_cache);
+        data = __alloc_page_frag(nc, fragsz, gfp_mask);
         local_irq_restore(flags);
         return data;
 }
@@ -454,7 +378,9 @@ EXPORT_SYMBOL(netdev_alloc_frag);
 
 static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
 {
-        return __alloc_page_frag(&napi_alloc_cache, fragsz, gfp_mask);
+        struct page_frag_cache *nc = this_cpu_ptr(&napi_alloc_cache);
+
+        return __alloc_page_frag(nc, fragsz, gfp_mask);
 }
 
 void *napi_alloc_frag(unsigned int fragsz)
@@ -464,76 +390,64 @@ void *napi_alloc_frag(unsigned int fragsz)
 EXPORT_SYMBOL(napi_alloc_frag);
 
 /**
- *      __alloc_rx_skb - allocate an skbuff for rx
+ *      __netdev_alloc_skb - allocate an skbuff for rx on a specific device
+ *      @dev: network device to receive on
  *      @length: length to allocate
  *      @gfp_mask: get_free_pages mask, passed to alloc_skb
- *      @flags: If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
- *              allocations in case we have to fallback to __alloc_skb()
- *              If SKB_ALLOC_NAPI is set, page fragment will be allocated
- *              from napi_cache instead of netdev_cache.
  *
  *      Allocate a new &sk_buff and assign it a usage count of one. The
- *      buffer has unspecified headroom built in. Users should allocate
+ *      buffer has NET_SKB_PAD headroom built in. Users should allocate
  *      the headroom they think they need without accounting for the
  *      built in space. The built in space is used for optimisations.
  *
  *      %NULL is returned if there is no free memory.
  */
-static struct sk_buff *__alloc_rx_skb(unsigned int length, gfp_t gfp_mask,
-                                      int flags)
+struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
+                                   gfp_t gfp_mask)
 {
-        struct sk_buff *skb = NULL;
-        unsigned int fragsz = SKB_DATA_ALIGN(length) +
-                              SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+        struct page_frag_cache *nc;
+        unsigned long flags;
+        struct sk_buff *skb;
+        bool pfmemalloc;
+        void *data;
 
-        if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) {
-                void *data;
+        len += NET_SKB_PAD;
 
-                if (sk_memalloc_socks())
-                        gfp_mask |= __GFP_MEMALLOC;
+        if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
+            (gfp_mask & (__GFP_WAIT | GFP_DMA)))
+                return __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
 
-                data = (flags & SKB_ALLOC_NAPI) ?
-                        __napi_alloc_frag(fragsz, gfp_mask) :
-                        __netdev_alloc_frag(fragsz, gfp_mask);
+        len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+        len = SKB_DATA_ALIGN(len);
 
-                if (likely(data)) {
-                        skb = build_skb(data, fragsz);
-                        if (unlikely(!skb))
-                                put_page(virt_to_head_page(data));
-                }
-        } else {
-                skb = __alloc_skb(length, gfp_mask,
-                                  SKB_ALLOC_RX, NUMA_NO_NODE);
-        }
-        return skb;
-}
+        if (sk_memalloc_socks())
+                gfp_mask |= __GFP_MEMALLOC;
 
-/**
- *      __netdev_alloc_skb - allocate an skbuff for rx on a specific device
- *      @dev: network device to receive on
- *      @length: length to allocate
- *      @gfp_mask: get_free_pages mask, passed to alloc_skb
- *
- *      Allocate a new &sk_buff and assign it a usage count of one. The
- *      buffer has NET_SKB_PAD headroom built in. Users should allocate
- *      the headroom they think they need without accounting for the
- *      built in space. The built in space is used for optimisations.
- *
- *      %NULL is returned if there is no free memory.
- */
-struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
-                                   unsigned int length, gfp_t gfp_mask)
-{
-        struct sk_buff *skb;
+        local_irq_save(flags);
 
-        length += NET_SKB_PAD;
-        skb = __alloc_rx_skb(length, gfp_mask, 0);
+        nc = this_cpu_ptr(&netdev_alloc_cache);
+        data = __alloc_page_frag(nc, len, gfp_mask);
+        pfmemalloc = nc->pfmemalloc;
 
-        if (likely(skb)) {
-                skb_reserve(skb, NET_SKB_PAD);
-                skb->dev = dev;
+        local_irq_restore(flags);
+
+        if (unlikely(!data))
+                return NULL;
+
+        skb = __build_skb(data, len);
+        if (unlikely(!skb)) {
+                skb_free_frag(data);
+                return NULL;
         }
 
+        /* use OR instead of assignment to avoid clearing of bits in mask */
+        if (pfmemalloc)
+                skb->pfmemalloc = 1;
+        skb->head_frag = 1;
+
+        skb_reserve(skb, NET_SKB_PAD);
+        skb->dev = dev;
+
         return skb;
 }
 EXPORT_SYMBOL(__netdev_alloc_skb);
@@ -551,19 +465,43 @@ EXPORT_SYMBOL(__netdev_alloc_skb);
  *
  *      %NULL is returned if there is no free memory.
  */
-struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,
-                                 unsigned int length, gfp_t gfp_mask)
+struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
+                                 gfp_t gfp_mask)
 {
+        struct page_frag_cache *nc = this_cpu_ptr(&napi_alloc_cache);
         struct sk_buff *skb;
+        void *data;
+
+        len += NET_SKB_PAD + NET_IP_ALIGN;
 
-        length += NET_SKB_PAD + NET_IP_ALIGN;
-        skb = __alloc_rx_skb(length, gfp_mask, SKB_ALLOC_NAPI);
+        if ((len > SKB_WITH_OVERHEAD(PAGE_SIZE)) ||
+            (gfp_mask & (__GFP_WAIT | GFP_DMA)))
+                return __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE);
 
-        if (likely(skb)) {
-                skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
-                skb->dev = napi->dev;
+        len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+        len = SKB_DATA_ALIGN(len);
+
+        if (sk_memalloc_socks())
+                gfp_mask |= __GFP_MEMALLOC;
+
+        data = __alloc_page_frag(nc, len, gfp_mask);
+        if (unlikely(!data))
+                return NULL;
+
+        skb = __build_skb(data, len);
+        if (unlikely(!skb)) {
+                skb_free_frag(data);
+                return NULL;
         }
 
+        /* use OR instead of assignment to avoid clearing of bits in mask */
+        if (nc->pfmemalloc)
+                skb->pfmemalloc = 1;
+        skb->head_frag = 1;
+
+        skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
+        skb->dev = napi->dev;
+
         return skb;
 }
 EXPORT_SYMBOL(__napi_alloc_skb);
@@ -611,10 +549,12 @@ static void skb_clone_fraglist(struct sk_buff *skb)
 
 static void skb_free_head(struct sk_buff *skb)
 {
+        unsigned char *head = skb->head;
+
         if (skb->head_frag)
-                put_page(virt_to_head_page(skb->head));
+                skb_free_frag(head);
         else
-                kfree(skb->head);
+                kfree(head);
 }
 
 static void skb_release_data(struct sk_buff *skb)