diff options
author | Eric Dumazet <eric.dumazet@gmail.com> | 2010-08-27 01:03:08 -0400 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2010-08-27 01:03:08 -0400 |
commit | 40d0802b3eb47d57e2d57a5244a18cbbe9632e13 (patch) | |
tree | fd45987286080e1f711c2448f6fbdc7d362a0241 | |
parent | 250ad8f55c06eb866cfb57f8d3ea6ff961a7d1d7 (diff) |
gro: __napi_gro_receive() optimizations
compare_ether_header() can have a special implementation on 64 bit
arches if CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS is defined.
__napi_gro_receive() and vlan_gro_common() can avoid a conditional
branch to perform device match.
On x86_64, __napi_gro_receive() has now 38 instructions instead of 53
As gcc-4.4.3 still choose to not inline it, add inline keyword to this
performance critical function.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
CC: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r-- | include/linux/etherdevice.h | 18 | ||||
-rw-r--r-- | net/8021q/vlan_core.c | 9 | ||||
-rw-r--r-- | net/core/dev.c | 10 |
3 files changed, 29 insertions, 8 deletions
diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h index 2308fbb4523a..fb6aa6070921 100644 --- a/include/linux/etherdevice.h +++ b/include/linux/etherdevice.h | |||
@@ -237,13 +237,29 @@ static inline bool is_etherdev_addr(const struct net_device *dev, | |||
237 | * entry points. | 237 | * entry points. |
238 | */ | 238 | */ |
239 | 239 | ||
240 | static inline int compare_ether_header(const void *a, const void *b) | 240 | static inline unsigned long compare_ether_header(const void *a, const void *b) |
241 | { | 241 | { |
242 | #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64 | ||
243 | unsigned long fold; | ||
244 | |||
245 | /* | ||
246 | * We want to compare 14 bytes: | ||
247 | * [a0 ... a13] ^ [b0 ... b13] | ||
248 | * Use two long XOR, ORed together, with an overlap of two bytes. | ||
249 | * [a0 a1 a2 a3 a4 a5 a6 a7 ] ^ [b0 b1 b2 b3 b4 b5 b6 b7 ] | | ||
250 | * [a6 a7 a8 a9 a10 a11 a12 a13] ^ [b6 b7 b8 b9 b10 b11 b12 b13] | ||
251 | * This means the [a6 a7] ^ [b6 b7] part is done two times. | ||
252 | */ | ||
253 | fold = *(unsigned long *)a ^ *(unsigned long *)b; | ||
254 | fold |= *(unsigned long *)(a + 6) ^ *(unsigned long *)(b + 6); | ||
255 | return fold; | ||
256 | #else | ||
242 | u32 *a32 = (u32 *)((u8 *)a + 2); | 257 | u32 *a32 = (u32 *)((u8 *)a + 2); |
243 | u32 *b32 = (u32 *)((u8 *)b + 2); | 258 | u32 *b32 = (u32 *)((u8 *)b + 2); |
244 | 259 | ||
245 | return (*(u16 *)a ^ *(u16 *)b) | (a32[0] ^ b32[0]) | | 260 | return (*(u16 *)a ^ *(u16 *)b) | (a32[0] ^ b32[0]) | |
246 | (a32[1] ^ b32[1]) | (a32[2] ^ b32[2]); | 261 | (a32[1] ^ b32[1]) | (a32[2] ^ b32[2]); |
262 | #endif | ||
247 | } | 263 | } |
248 | 264 | ||
249 | #endif /* _LINUX_ETHERDEVICE_H */ | 265 | #endif /* _LINUX_ETHERDEVICE_H */ |
diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c index 07eeb5b99dce..3438c01bbacf 100644 --- a/net/8021q/vlan_core.c +++ b/net/8021q/vlan_core.c | |||
@@ -105,9 +105,12 @@ vlan_gro_common(struct napi_struct *napi, struct vlan_group *grp, | |||
105 | goto drop; | 105 | goto drop; |
106 | 106 | ||
107 | for (p = napi->gro_list; p; p = p->next) { | 107 | for (p = napi->gro_list; p; p = p->next) { |
108 | NAPI_GRO_CB(p)->same_flow = | 108 | unsigned long diffs; |
109 | p->dev == skb->dev && !compare_ether_header( | 109 | |
110 | skb_mac_header(p), skb_gro_mac_header(skb)); | 110 | diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; |
111 | diffs |= compare_ether_header(skb_mac_header(p), | ||
112 | skb_gro_mac_header(skb)); | ||
113 | NAPI_GRO_CB(p)->same_flow = !diffs; | ||
111 | NAPI_GRO_CB(p)->flush = 0; | 114 | NAPI_GRO_CB(p)->flush = 0; |
112 | } | 115 | } |
113 | 116 | ||
diff --git a/net/core/dev.c b/net/core/dev.c index 859e30ff044a..63bd20a75929 100644 --- a/net/core/dev.c +++ b/net/core/dev.c | |||
@@ -3169,16 +3169,18 @@ normal: | |||
3169 | } | 3169 | } |
3170 | EXPORT_SYMBOL(dev_gro_receive); | 3170 | EXPORT_SYMBOL(dev_gro_receive); |
3171 | 3171 | ||
3172 | static gro_result_t | 3172 | static inline gro_result_t |
3173 | __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) | 3173 | __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) |
3174 | { | 3174 | { |
3175 | struct sk_buff *p; | 3175 | struct sk_buff *p; |
3176 | 3176 | ||
3177 | for (p = napi->gro_list; p; p = p->next) { | 3177 | for (p = napi->gro_list; p; p = p->next) { |
3178 | NAPI_GRO_CB(p)->same_flow = | 3178 | unsigned long diffs; |
3179 | (p->dev == skb->dev) && | 3179 | |
3180 | !compare_ether_header(skb_mac_header(p), | 3180 | diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; |
3181 | diffs |= compare_ether_header(skb_mac_header(p), | ||
3181 | skb_gro_mac_header(skb)); | 3182 | skb_gro_mac_header(skb)); |
3183 | NAPI_GRO_CB(p)->same_flow = !diffs; | ||
3182 | NAPI_GRO_CB(p)->flush = 0; | 3184 | NAPI_GRO_CB(p)->flush = 0; |
3183 | } | 3185 | } |
3184 | 3186 | ||