Merge branch 'napi_page_frags'

Alexander Duyck says:

====================
net: Alloc NAPI page frags from their own pool

This patch series implements a means of allocating page fragments without
the need for the local_irq_save/restore in __netdev_alloc_frag.  By doing
this I am able to decrease packet processing time by 11ns per packet in my
test environment.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>

2 files changed, 145 insertions, 48 deletions

diff --git a/net/core/dev.c b/net/core/dev.c
index 3f191da383f6..80f798da3d9f 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4172,7 +4172,7 @@ struct sk_buff *napi_get_frags(struct napi_struct *napi)
         struct sk_buff *skb = napi->skb;
 
         if (!skb) {
-                skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
+                skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
                 napi->skb = skb;
         }
         return skb;
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 7a338fb55cc4..ae13ef6b3ea7 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -336,59 +336,85 @@ struct netdev_alloc_cache {
         unsigned int            pagecnt_bias;
 };
 static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache);
+static DEFINE_PER_CPU(struct netdev_alloc_cache, napi_alloc_cache);
 
-static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
+static struct page *__page_frag_refill(struct netdev_alloc_cache *nc,
+                                       gfp_t gfp_mask)
 {
-        struct netdev_alloc_cache *nc;
-        void *data = NULL;
-        int order;
-        unsigned long flags;
+        const unsigned int order = NETDEV_FRAG_PAGE_MAX_ORDER;
+        struct page *page = NULL;
+        gfp_t gfp = gfp_mask;
+
+        if (order) {
+                gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY;
+                page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
+                nc->frag.size = PAGE_SIZE << (page ? order : 0);
+        }
 
-        local_irq_save(flags);
-        nc = this_cpu_ptr(&netdev_alloc_cache);
-        if (unlikely(!nc->frag.page)) {
+        if (unlikely(!page))
+                page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
+
+        nc->frag.page = page;
+
+        return page;
+}
+
+static void *__alloc_page_frag(struct netdev_alloc_cache __percpu *cache,
+                               unsigned int fragsz, gfp_t gfp_mask)
+{
+        struct netdev_alloc_cache *nc = this_cpu_ptr(cache);
+        struct page *page = nc->frag.page;
+        unsigned int size;
+        int offset;
+
+        if (unlikely(!page)) {
 refill:
-                for (order = NETDEV_FRAG_PAGE_MAX_ORDER; ;) {
-                        gfp_t gfp = gfp_mask;
+                page = __page_frag_refill(nc, gfp_mask);
+                if (!page)
+                        return NULL;
+
+                /* if size can vary use frag.size else just use PAGE_SIZE */
+                size = NETDEV_FRAG_PAGE_MAX_ORDER ? nc->frag.size : PAGE_SIZE;
 
-                        if (order)
-                                gfp |= __GFP_COMP | __GFP_NOWARN;
-                        nc->frag.page = alloc_pages(gfp, order);
-                        if (likely(nc->frag.page))
-                                break;
-                        if (--order < 0)
-                                goto end;
-                }
-                nc->frag.size = PAGE_SIZE << order;
                 /* Even if we own the page, we do not use atomic_set().
                  * This would break get_page_unless_zero() users.
                  */
-                atomic_add(NETDEV_PAGECNT_MAX_BIAS - 1,
-                           &nc->frag.page->_count);
-                nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
-                nc->frag.offset = 0;
+                atomic_add(size - 1, &page->_count);
+
+                /* reset page count bias and offset to start of new frag */
+                nc->pagecnt_bias = size;
+                nc->frag.offset = size;
         }
 
-        if (nc->frag.offset + fragsz > nc->frag.size) {
-                if (atomic_read(&nc->frag.page->_count) != nc->pagecnt_bias) {
-                        if (!atomic_sub_and_test(nc->pagecnt_bias,
-                                                 &nc->frag.page->_count))
-                                goto refill;
-                        /* OK, page count is 0, we can safely set it */
-                        atomic_set(&nc->frag.page->_count,
-                                   NETDEV_PAGECNT_MAX_BIAS);
-                } else {
-                        atomic_add(NETDEV_PAGECNT_MAX_BIAS - nc->pagecnt_bias,
-                                   &nc->frag.page->_count);
-                }
-                nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
-                nc->frag.offset = 0;
+        offset = nc->frag.offset - fragsz;
+        if (unlikely(offset < 0)) {
+                if (!atomic_sub_and_test(nc->pagecnt_bias, &page->_count))
+                        goto refill;
+
+                /* if size can vary use frag.size else just use PAGE_SIZE */
+                size = NETDEV_FRAG_PAGE_MAX_ORDER ? nc->frag.size : PAGE_SIZE;
+
+                /* OK, page count is 0, we can safely set it */
+                atomic_set(&page->_count, size);
+
+                /* reset page count bias and offset to start of new frag */
+                nc->pagecnt_bias = size;
+                offset = size - fragsz;
         }
 
-        data = page_address(nc->frag.page) + nc->frag.offset;
-        nc->frag.offset += fragsz;
         nc->pagecnt_bias--;
-end:
+        nc->frag.offset = offset;
+
+        return page_address(page) + offset;
+}
+
+static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
+{
+        unsigned long flags;
+        void *data;
+
+        local_irq_save(flags);
+        data = __alloc_page_frag(&netdev_alloc_cache, fragsz, gfp_mask);
         local_irq_restore(flags);
         return data;
 }
@@ -406,11 +432,25 @@ void *netdev_alloc_frag(unsigned int fragsz)
 }
 EXPORT_SYMBOL(netdev_alloc_frag);
 
+static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
+{
+        return __alloc_page_frag(&napi_alloc_cache, fragsz, gfp_mask);
+}
+
+void *napi_alloc_frag(unsigned int fragsz)
+{
+        return __napi_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD);
+}
+EXPORT_SYMBOL(napi_alloc_frag);
+
 /**
- *      __netdev_alloc_skb - allocate an skbuff for rx on a specific device
- *      @dev: network device to receive on
+ *      __alloc_rx_skb - allocate an skbuff for rx
  *      @length: length to allocate
  *      @gfp_mask: get_free_pages mask, passed to alloc_skb
+ *      @flags: If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for
+ *              allocations in case we have to fallback to __alloc_skb()
+ *              If SKB_ALLOC_NAPI is set, page fragment will be allocated
+ *              from napi_cache instead of netdev_cache.
  *
  *      Allocate a new &sk_buff and assign it a usage count of one. The
  *      buffer has unspecified headroom built in. Users should allocate
@@ -419,11 +459,11 @@ EXPORT_SYMBOL(netdev_alloc_frag);
  *
  *      %NULL is returned if there is no free memory.
  */
-struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
-                                   unsigned int length, gfp_t gfp_mask)
+static struct sk_buff *__alloc_rx_skb(unsigned int length, gfp_t gfp_mask,
+                                      int flags)
 {
         struct sk_buff *skb = NULL;
-        unsigned int fragsz = SKB_DATA_ALIGN(length + NET_SKB_PAD) +
+        unsigned int fragsz = SKB_DATA_ALIGN(length) +
                               SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 
         if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) {
@@ -432,7 +472,9 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
                 if (sk_memalloc_socks())
                         gfp_mask |= __GFP_MEMALLOC;
 
-                data = __netdev_alloc_frag(fragsz, gfp_mask);
+                data = (flags & SKB_ALLOC_NAPI) ?
+                        __napi_alloc_frag(fragsz, gfp_mask) :
+                        __netdev_alloc_frag(fragsz, gfp_mask);
 
                 if (likely(data)) {
                         skb = build_skb(data, fragsz);
@@ -440,17 +482,72 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
                                 put_page(virt_to_head_page(data));
                 }
         } else {
-                skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask,
+                skb = __alloc_skb(length, gfp_mask,
                                   SKB_ALLOC_RX, NUMA_NO_NODE);
         }
+        return skb;
+}
+
+/**
+ *      __netdev_alloc_skb - allocate an skbuff for rx on a specific device
+ *      @dev: network device to receive on
+ *      @length: length to allocate
+ *      @gfp_mask: get_free_pages mask, passed to alloc_skb
+ *
+ *      Allocate a new &sk_buff and assign it a usage count of one. The
+ *      buffer has NET_SKB_PAD headroom built in. Users should allocate
+ *      the headroom they think they need without accounting for the
+ *      built in space. The built in space is used for optimisations.
+ *
+ *      %NULL is returned if there is no free memory.
+ */
+struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
+                                   unsigned int length, gfp_t gfp_mask)
+{
+        struct sk_buff *skb;
+
+        length += NET_SKB_PAD;
+        skb = __alloc_rx_skb(length, gfp_mask, 0);
+
         if (likely(skb)) {
                 skb_reserve(skb, NET_SKB_PAD);
                 skb->dev = dev;
         }
+
         return skb;
 }
 EXPORT_SYMBOL(__netdev_alloc_skb);
 
+/**
+ *      __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance
+ *      @napi: napi instance this buffer was allocated for
+ *      @length: length to allocate
+ *      @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages
+ *
+ *      Allocate a new sk_buff for use in NAPI receive.  This buffer will
+ *      attempt to allocate the head from a special reserved region used
+ *      only for NAPI Rx allocation.  By doing this we can save several
+ *      CPU cycles by avoiding having to disable and re-enable IRQs.
+ *
+ *      %NULL is returned if there is no free memory.
+ */
+struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,
+                                 unsigned int length, gfp_t gfp_mask)
+{
+        struct sk_buff *skb;
+
+        length += NET_SKB_PAD + NET_IP_ALIGN;
+        skb = __alloc_rx_skb(length, gfp_mask, SKB_ALLOC_NAPI);
+
+        if (likely(skb)) {
+                skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
+                skb->dev = napi->dev;
+        }
+
+        return skb;
+}
+EXPORT_SYMBOL(__napi_alloc_skb);
+
 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
                      int size, unsigned int truesize)
 {