aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHerbert Xu <herbert@gondor.apana.org.au>2009-05-26 14:50:28 -0400
committerDavid S. Miller <davem@davemloft.net>2009-05-27 06:26:01 -0400
commita5b1cf288d4200506ab62fbb86cc81ace948a306 (patch)
treec95339866f4d67220d111811a92e5c6b79de6ab4
parent7489594cb249aeb178287c9a43a9e4f366044259 (diff)
gro: Avoid unnecessary comparison after skb_gro_header
For the overwhelming majority of cases, skb_gro_header's return value cannot be NULL. Yet we must check it because of its current form. This patch splits it up into multiple functions in order to avoid this. Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r--include/linux/netdevice.h23
-rw-r--r--net/core/dev.c17
-rw-r--r--net/ipv4/af_inet.c13
-rw-r--r--net/ipv4/tcp.c22
-rw-r--r--net/ipv6/af_inet6.c13
5 files changed, 62 insertions, 26 deletions
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 2e44a049be0f..371ece521e58 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1132,18 +1132,23 @@ static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
1132 NAPI_GRO_CB(skb)->data_offset += len; 1132 NAPI_GRO_CB(skb)->data_offset += len;
1133} 1133}
1134 1134
1135static inline void *skb_gro_header(struct sk_buff *skb, unsigned int hlen) 1135static inline void *skb_gro_header_fast(struct sk_buff *skb,
1136 unsigned int offset)
1136{ 1137{
1137 unsigned int offset = skb_gro_offset(skb); 1138 return NAPI_GRO_CB(skb)->frag0 + offset;
1139}
1138 1140
1139 hlen += offset; 1141static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen)
1140 if (NAPI_GRO_CB(skb)->frag0_len < hlen) { 1142{
1141 NAPI_GRO_CB(skb)->frag0 = NULL; 1143 return NAPI_GRO_CB(skb)->frag0_len < hlen;
1142 NAPI_GRO_CB(skb)->frag0_len = 0; 1144}
1143 return pskb_may_pull(skb, hlen) ? skb->data + offset : NULL;
1144 }
1145 1145
1146 return NAPI_GRO_CB(skb)->frag0 + offset; 1146static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
1147 unsigned int offset)
1148{
1149 NAPI_GRO_CB(skb)->frag0 = NULL;
1150 NAPI_GRO_CB(skb)->frag0_len = 0;
1151 return pskb_may_pull(skb, hlen) ? skb->data + offset : NULL;
1147} 1152}
1148 1153
1149static inline void *skb_gro_mac_header(struct sk_buff *skb) 1154static inline void *skb_gro_mac_header(struct sk_buff *skb)
diff --git a/net/core/dev.c b/net/core/dev.c
index b1722a2d1fbe..cd29e613bc5a 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2590,17 +2590,24 @@ struct sk_buff *napi_frags_skb(struct napi_struct *napi)
2590{ 2590{
2591 struct sk_buff *skb = napi->skb; 2591 struct sk_buff *skb = napi->skb;
2592 struct ethhdr *eth; 2592 struct ethhdr *eth;
2593 unsigned int hlen;
2594 unsigned int off;
2593 2595
2594 napi->skb = NULL; 2596 napi->skb = NULL;
2595 2597
2596 skb_reset_mac_header(skb); 2598 skb_reset_mac_header(skb);
2597 skb_gro_reset_offset(skb); 2599 skb_gro_reset_offset(skb);
2598 2600
2599 eth = skb_gro_header(skb, sizeof(*eth)); 2601 off = skb_gro_offset(skb);
2600 if (!eth) { 2602 hlen = off + sizeof(*eth);
2601 napi_reuse_skb(napi, skb); 2603 eth = skb_gro_header_fast(skb, off);
2602 skb = NULL; 2604 if (skb_gro_header_hard(skb, hlen)) {
2603 goto out; 2605 eth = skb_gro_header_slow(skb, hlen, off);
2606 if (unlikely(!eth)) {
2607 napi_reuse_skb(napi, skb);
2608 skb = NULL;
2609 goto out;
2610 }
2604 } 2611 }
2605 2612
2606 skb_gro_pull(skb, sizeof(*eth)); 2613 skb_gro_pull(skb, sizeof(*eth));
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 170689681aa2..644cc5535319 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1246,13 +1246,20 @@ static struct sk_buff **inet_gro_receive(struct sk_buff **head,
1246 struct sk_buff **pp = NULL; 1246 struct sk_buff **pp = NULL;
1247 struct sk_buff *p; 1247 struct sk_buff *p;
1248 struct iphdr *iph; 1248 struct iphdr *iph;
1249 unsigned int hlen;
1250 unsigned int off;
1249 int flush = 1; 1251 int flush = 1;
1250 int proto; 1252 int proto;
1251 int id; 1253 int id;
1252 1254
1253 iph = skb_gro_header(skb, sizeof(*iph)); 1255 off = skb_gro_offset(skb);
1254 if (unlikely(!iph)) 1256 hlen = off + sizeof(*iph);
1255 goto out; 1257 iph = skb_gro_header_fast(skb, off);
1258 if (skb_gro_header_hard(skb, hlen)) {
1259 iph = skb_gro_header_slow(skb, hlen, off);
1260 if (unlikely(!iph))
1261 goto out;
1262 }
1256 1263
1257 proto = iph->protocol & (MAX_INET_PROTOS - 1); 1264 proto = iph->protocol & (MAX_INET_PROTOS - 1);
1258 1265
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 68342d431896..c3dcec5efea5 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -2518,20 +2518,30 @@ struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb)
2518 unsigned int thlen; 2518 unsigned int thlen;
2519 unsigned int flags; 2519 unsigned int flags;
2520 unsigned int mss = 1; 2520 unsigned int mss = 1;
2521 unsigned int hlen;
2522 unsigned int off;
2521 int flush = 1; 2523 int flush = 1;
2522 int i; 2524 int i;
2523 2525
2524 th = skb_gro_header(skb, sizeof(*th)); 2526 off = skb_gro_offset(skb);
2525 if (unlikely(!th)) 2527 hlen = off + sizeof(*th);
2526 goto out; 2528 th = skb_gro_header_fast(skb, off);
2529 if (skb_gro_header_hard(skb, hlen)) {
2530 th = skb_gro_header_slow(skb, hlen, off);
2531 if (unlikely(!th))
2532 goto out;
2533 }
2527 2534
2528 thlen = th->doff * 4; 2535 thlen = th->doff * 4;
2529 if (thlen < sizeof(*th)) 2536 if (thlen < sizeof(*th))
2530 goto out; 2537 goto out;
2531 2538
2532 th = skb_gro_header(skb, thlen); 2539 hlen = off + thlen;
2533 if (unlikely(!th)) 2540 if (skb_gro_header_hard(skb, hlen)) {
2534 goto out; 2541 th = skb_gro_header_slow(skb, hlen, off);
2542 if (unlikely(!th))
2543 goto out;
2544 }
2535 2545
2536 skb_gro_pull(skb, thlen); 2546 skb_gro_pull(skb, thlen);
2537 2547
diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c
index 61f55386a236..b6215be0963f 100644
--- a/net/ipv6/af_inet6.c
+++ b/net/ipv6/af_inet6.c
@@ -817,13 +817,20 @@ static struct sk_buff **ipv6_gro_receive(struct sk_buff **head,
817 struct sk_buff *p; 817 struct sk_buff *p;
818 struct ipv6hdr *iph; 818 struct ipv6hdr *iph;
819 unsigned int nlen; 819 unsigned int nlen;
820 unsigned int hlen;
821 unsigned int off;
820 int flush = 1; 822 int flush = 1;
821 int proto; 823 int proto;
822 __wsum csum; 824 __wsum csum;
823 825
824 iph = skb_gro_header(skb, sizeof(*iph)); 826 off = skb_gro_offset(skb);
825 if (unlikely(!iph)) 827 hlen = off + sizeof(*iph);
826 goto out; 828 iph = skb_gro_header_fast(skb, off);
829 if (skb_gro_header_hard(skb, hlen)) {
830 iph = skb_gro_header_slow(skb, hlen, off);
831 if (unlikely(!iph))
832 goto out;
833 }
827 834
828 skb_gro_pull(skb, sizeof(*iph)); 835 skb_gro_pull(skb, sizeof(*iph));
829 skb_set_transport_header(skb, skb_gro_offset(skb)); 836 skb_set_transport_header(skb, skb_gro_offset(skb));