1 files changed, 55 insertions, 6 deletions

diff --git a/net/core/dev.c b/net/core/dev.c
index 5156c0edebe8..a9775d676285 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3963,6 +3963,8 @@ int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
 int dev_rx_weight __read_mostly = 64;
 int dev_tx_weight __read_mostly = 64;
+/* Maximum number of GRO_NORMAL skbs to batch up for list-RX */
+int gro_normal_batch __read_mostly = 8;
 
 /* Called with irq disabled */
 static inline void ____napi_schedule(struct softnet_data *sd,
@@ -5486,7 +5488,7 @@ static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
         skb->data_len -= grow;
         skb->tail += grow;
 
-        pinfo->frags[0].page_offset += grow;
+        skb_frag_off_add(&pinfo->frags[0], grow);
         skb_frag_size_sub(&pinfo->frags[0], grow);
 
         if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
@@ -5747,6 +5749,26 @@ struct sk_buff *napi_get_frags(struct napi_struct *napi)
 }
 EXPORT_SYMBOL(napi_get_frags);
 
+/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
+static void gro_normal_list(struct napi_struct *napi)
+{
+        if (!napi->rx_count)
+                return;
+        netif_receive_skb_list_internal(&napi->rx_list);
+        INIT_LIST_HEAD(&napi->rx_list);
+        napi->rx_count = 0;
+}
+
+/* Queue one GRO_NORMAL SKB up for list processing.  If batch size exceeded,
+ * pass the whole batch up to the stack.
+ */
+static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb)
+{
+        list_add_tail(&skb->list, &napi->rx_list);
+        if (++napi->rx_count >= gro_normal_batch)
+                gro_normal_list(napi);
+}
+
 static gro_result_t napi_frags_finish(struct napi_struct *napi,
                                       struct sk_buff *skb,
                                       gro_result_t ret)
@@ -5756,8 +5778,8 @@ static gro_result_t napi_frags_finish(struct napi_struct *napi,
         case GRO_HELD:
                 __skb_push(skb, ETH_HLEN);
                 skb->protocol = eth_type_trans(skb, skb->dev);
-                if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
-                        ret = GRO_DROP;
+                if (ret == GRO_NORMAL)
+                        gro_normal_one(napi, skb);
                 break;
 
         case GRO_DROP:
@@ -6034,6 +6056,8 @@ bool napi_complete_done(struct napi_struct *n, int work_done)
                                  NAPIF_STATE_IN_BUSY_POLL)))
                 return false;
 
+        gro_normal_list(n);
+
         if (n->gro_bitmask) {
                 unsigned long timeout = 0;
 
@@ -6119,10 +6143,19 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock)
          * Ideally, a new ndo_busy_poll_stop() could avoid another round.
          */
         rc = napi->poll(napi, BUSY_POLL_BUDGET);
+        /* We can't gro_normal_list() here, because napi->poll() might have
+         * rearmed the napi (napi_complete_done()) in which case it could
+         * already be running on another CPU.
+         */
         trace_napi_poll(napi, rc, BUSY_POLL_BUDGET);
         netpoll_poll_unlock(have_poll_lock);
-        if (rc == BUSY_POLL_BUDGET)
+        if (rc == BUSY_POLL_BUDGET) {
+                /* As the whole budget was spent, we still own the napi so can
+                 * safely handle the rx_list.
+                 */
+                gro_normal_list(napi);
                 __napi_schedule(napi);
+        }
         local_bh_enable();
 }
 
@@ -6167,6 +6200,7 @@ restart:
                 }
                 work = napi_poll(napi, BUSY_POLL_BUDGET);
                 trace_napi_poll(napi, work, BUSY_POLL_BUDGET);
+                gro_normal_list(napi);
 count:
                 if (work > 0)
                         __NET_ADD_STATS(dev_net(napi->dev),
@@ -6272,6 +6306,8 @@ void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
         napi->timer.function = napi_watchdog;
         init_gro_hash(napi);
         napi->skb = NULL;
+        INIT_LIST_HEAD(&napi->rx_list);
+        napi->rx_count = 0;
         napi->poll = poll;
         if (weight > NAPI_POLL_WEIGHT)
                 netdev_err_once(dev, "%s() called with weight %d\n", __func__,
@@ -6368,6 +6404,8 @@ static int napi_poll(struct napi_struct *n, struct list_head *repoll)
                 goto out_unlock;
         }
 
+        gro_normal_list(n);
+
         if (n->gro_bitmask) {
                 /* flush too old packets
                  * If HZ < 1000, flush all packets.
@@ -8088,12 +8126,15 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
                 bpf_chk = generic_xdp_install;
 
         if (fd >= 0) {
+                u32 prog_id;
+
                 if (!offload && __dev_xdp_query(dev, bpf_chk, XDP_QUERY_PROG)) {
                         NL_SET_ERR_MSG(extack, "native and generic XDP can't be active at the same time");
                         return -EEXIST;
                 }
-                if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) &&
-                    __dev_xdp_query(dev, bpf_op, query)) {
+
+                prog_id = __dev_xdp_query(dev, bpf_op, query);
+                if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && prog_id) {
                         NL_SET_ERR_MSG(extack, "XDP program already attached");
                         return -EBUSY;
                 }
@@ -8108,6 +8149,14 @@ int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
                         bpf_prog_put(prog);
                         return -EINVAL;
                 }
+
+                if (prog->aux->id == prog_id) {
+                        bpf_prog_put(prog);
+                        return 0;
+                }
+        } else {
+                if (!__dev_xdp_query(dev, bpf_op, query))
+                        return 0;
         }
 
         err = dev_xdp_install(dev, bpf_op, extack, flags, prog);