Merge branch 'net-drivers-pgcnt'

Eric Dumazet says:

====================
net: fix races accessing page->_count

This is illegal to use atomic_set(&page->_count, ...) even if we 'own'
the page. Other entities in the kernel need to use get_page_unless_zero()
to get a reference to the page before testing page properties, so we could
loose a refcount increment.

The only case it is valid is when page->_count is 0, we can use this in
__netdev_alloc_frag()

Note that I never seen crashes caused by these races, the issue was reported
by Andres Lagar-Cavilla and Hugh Dickins.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>

5 files changed, 30 insertions, 23 deletions

diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_main.c b/drivers/net/ethernet/intel/fm10k/fm10k_main.c
index 6c800a330d66..9d7118a0d67a 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_main.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_main.c
@@ -219,11 +219,10 @@ static bool fm10k_can_reuse_rx_page(struct fm10k_rx_buffer *rx_buffer,
         /* flip page offset to other buffer */
         rx_buffer->page_offset ^= FM10K_RX_BUFSZ;
 
-        /* since we are the only owner of the page and we need to
-         * increment it, just set the value to 2 in order to avoid
-         * an unnecessary locked operation
+        /* Even if we own the page, we are not allowed to use atomic_set()
+         * This would break get_page_unless_zero() users.
          */
-        atomic_set(&page->_count, 2);
+        atomic_inc(&page->_count);
 #else
         /* move offset up to the next cache line */
         rx_buffer->page_offset += truesize;
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index ae59c0b108c5..a21b14495ebd 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -6545,11 +6545,10 @@ static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
         /* flip page offset to other buffer */
         rx_buffer->page_offset ^= IGB_RX_BUFSZ;
 
-        /* since we are the only owner of the page and we need to
-         * increment it, just set the value to 2 in order to avoid
-         * an unnecessary locked operation
+        /* Even if we own the page, we are not allowed to use atomic_set()
+         * This would break get_page_unless_zero() users.
          */
-        atomic_set(&page->_count, 2);
+        atomic_inc(&page->_count);
 #else
         /* move offset up to the next cache line */
         rx_buffer->page_offset += truesize;
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index d677b5a23b58..fec5212d4337 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -1865,12 +1865,10 @@ static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
         /* flip page offset to other buffer */
         rx_buffer->page_offset ^= truesize;
 
-        /*
-         * since we are the only owner of the page and we need to
-         * increment it, just set the value to 2 in order to avoid
-         * an unecessary locked operation
+        /* Even if we own the page, we are not allowed to use atomic_set()
+         * This would break get_page_unless_zero() users.
          */
-        atomic_set(&page->_count, 2);
+        atomic_inc(&page->_count);
 #else
         /* move offset up to the next cache line */
         rx_buffer->page_offset += truesize;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index a33048ee9621..01660c595f5c 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -76,10 +76,10 @@ static int mlx4_alloc_pages(struct mlx4_en_priv *priv,
         page_alloc->dma = dma;
         page_alloc->page_offset = frag_info->frag_align;
         /* Not doing get_page() for each frag is a big win
-         * on asymetric workloads.
+         * on asymetric workloads. Note we can not use atomic_set().
          */
-        atomic_set(&page->_count,
-                   page_alloc->page_size / frag_info->frag_stride);
+        atomic_add(page_alloc->page_size / frag_info->frag_stride - 1,
+                   &page->_count);
         return 0;
 }
 
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index a30d750647e7..829d013745ab 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -360,18 +360,29 @@ refill:
                                 goto end;
                 }
                 nc->frag.size = PAGE_SIZE << order;
-recycle:
-                atomic_set(&nc->frag.page->_count, NETDEV_PAGECNT_MAX_BIAS);
+                /* Even if we own the page, we do not use atomic_set().
+                 * This would break get_page_unless_zero() users.
+                 */
+                atomic_add(NETDEV_PAGECNT_MAX_BIAS - 1,
+                           &nc->frag.page->_count);
                 nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
                 nc->frag.offset = 0;
         }
 
         if (nc->frag.offset + fragsz > nc->frag.size) {
-                /* avoid unnecessary locked operations if possible */
-                if ((atomic_read(&nc->frag.page->_count) == nc->pagecnt_bias) ||
-                    atomic_sub_and_test(nc->pagecnt_bias, &nc->frag.page->_count))
-                        goto recycle;
-                goto refill;
+                if (atomic_read(&nc->frag.page->_count) != nc->pagecnt_bias) {
+                        if (!atomic_sub_and_test(nc->pagecnt_bias,
+                                                 &nc->frag.page->_count))
+                                goto refill;
+                        /* OK, page count is 0, we can safely set it */
+                        atomic_set(&nc->frag.page->_count,
+                                   NETDEV_PAGECNT_MAX_BIAS);
+                } else {
+                        atomic_add(NETDEV_PAGECNT_MAX_BIAS - nc->pagecnt_bias,
+                                   &nc->frag.page->_count);
+                }
+                nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
+                nc->frag.offset = 0;
         }
 
         data = page_address(nc->frag.page) + nc->frag.offset;