gro: Avoid copying headers of unmerged packets

Unfortunately simplicity isn't always the best.  The fraginfo
interface turned out to be suboptimal.  The problem was quite
obvious.  For every packet, we have to copy the headers from
the frags structure into skb->head, even though for 99% of the
packets this part is immediately thrown away after the merge.

LRO didn't have this problem because it directly read the headers
from the frags structure.

This patch attempts to address this by creating an interface
that allows GRO to access the headers in the first frag without
having to copy it.  Because all drivers that use frags place the
headers in the first frag this optimisation should be enough.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: David S. Miller <davem@davemloft.net>

3 files changed, 14 insertions, 14 deletions

diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 743f5542d65a..d6770f295d5b 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1253,10 +1253,10 @@ static struct sk_buff **inet_gro_receive(struct sk_buff **head,
         int proto;
         int id;
 
-        if (unlikely(!pskb_may_pull(skb, sizeof(*iph))))
+        iph = skb_gro_header(skb, sizeof(*iph));
+        if (unlikely(!iph))
                 goto out;
 
-        iph = ip_hdr(skb);
         proto = iph->protocol & (MAX_INET_PROTOS - 1);
 
         rcu_read_lock();
@@ -1270,7 +1270,7 @@ static struct sk_buff **inet_gro_receive(struct sk_buff **head,
         if (unlikely(ip_fast_csum((u8 *)iph, iph->ihl)))
                 goto out_unlock;
 
-        flush = ntohs(iph->tot_len) != skb->len ||
+        flush = ntohs(iph->tot_len) != skb_gro_len(skb) ||
                 iph->frag_off != htons(IP_DF);
         id = ntohs(iph->id);
 
@@ -1298,8 +1298,8 @@ static struct sk_buff **inet_gro_receive(struct sk_buff **head,
         }
 
         NAPI_GRO_CB(skb)->flush |= flush;
-        __skb_pull(skb, sizeof(*iph));
-        skb_reset_transport_header(skb);
+        skb_gro_pull(skb, sizeof(*iph));
+        skb_set_transport_header(skb, skb_gro_offset(skb));
 
         pp = ops->gro_receive(head, skb);
 
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 0cd71b84e483..1cd608253940 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -2481,19 +2481,19 @@ struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb)
         unsigned int mss = 1;
         int flush = 1;
 
-        if (!pskb_may_pull(skb, sizeof(*th)))
+        th = skb_gro_header(skb, sizeof(*th));
+        if (unlikely(!th))
                 goto out;
 
-        th = tcp_hdr(skb);
         thlen = th->doff * 4;
         if (thlen < sizeof(*th))
                 goto out;
 
-        if (!pskb_may_pull(skb, thlen))
+        th = skb_gro_header(skb, thlen);
+        if (unlikely(!th))
                 goto out;
 
-        th = tcp_hdr(skb);
-        __skb_pull(skb, thlen);
+        skb_gro_pull(skb, thlen);
 
         flags = tcp_flag_word(th);
 
@@ -2521,10 +2521,10 @@ found:
         flush |= th->ack_seq != th2->ack_seq || th->window != th2->window;
         flush |= memcmp(th + 1, th2 + 1, thlen - sizeof(*th));
 
-        total = p->len;
+        total = skb_gro_len(p);
         mss = skb_shinfo(p)->gso_size;
 
-        flush |= skb->len > mss || skb->len <= 0;
+        flush |= skb_gro_len(skb) > mss || !skb_gro_len(skb);
         flush |= ntohl(th2->seq) + total != ntohl(th->seq);
 
         if (flush || skb_gro_receive(head, skb)) {
@@ -2537,7 +2537,7 @@ found:
         tcp_flag_word(th2) |= flags & (TCP_FLAG_FIN | TCP_FLAG_PSH);
 
 out_check_final:
-        flush = skb->len < mss;
+        flush = skb_gro_len(skb) < mss;
         flush |= flags & (TCP_FLAG_URG | TCP_FLAG_PSH | TCP_FLAG_RST |
                           TCP_FLAG_SYN | TCP_FLAG_FIN);
 
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 19d7b429a262..f6b962f56ab4 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -2355,7 +2355,7 @@ struct sk_buff **tcp4_gro_receive(struct sk_buff **head, struct sk_buff *skb)
 
         switch (skb->ip_summed) {
         case CHECKSUM_COMPLETE:
-                if (!tcp_v4_check(skb->len, iph->saddr, iph->daddr,
+                if (!tcp_v4_check(skb_gro_len(skb), iph->saddr, iph->daddr,
                                   skb->csum)) {
                         skb->ip_summed = CHECKSUM_UNNECESSARY;
                         break;