7 files changed, 115 insertions, 68 deletions

diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h
index 5e11905a4f01..12729e966dc9 100644
--- a/include/linux/ipv6.h
+++ b/include/linux/ipv6.h
@@ -364,20 +364,22 @@ static inline struct raw6_sock *raw6_sk(const struct sock *sk)
 #define inet_v6_ipv6only(__sk)          0
 #endif /* IS_ENABLED(CONFIG_IPV6) */
 
-#define INET6_MATCH(__sk, __net, __hash, __saddr, __daddr, __ports, __dif)\
-        (((__sk)->sk_hash == (__hash)) && sock_net((__sk)) == (__net)   && \
-         ((*((__portpair *)&(inet_sk(__sk)->inet_dport))) == (__ports)) && \
-         ((__sk)->sk_family             == AF_INET6)            && \
-         ipv6_addr_equal(&inet6_sk(__sk)->daddr, (__saddr))     && \
-         ipv6_addr_equal(&inet6_sk(__sk)->rcv_saddr, (__daddr)) && \
-         (!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
-
-#define INET6_TW_MATCH(__sk, __net, __hash, __saddr, __daddr, __ports, __dif) \
-        (((__sk)->sk_hash == (__hash)) && sock_net((__sk)) == (__net)   && \
-         (*((__portpair *)&(inet_twsk(__sk)->tw_dport)) == (__ports))   && \
-         ((__sk)->sk_family            == PF_INET6)                     && \
-         (ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_daddr, (__saddr)))   && \
-         (ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_rcv_saddr, (__daddr))) && \
-         (!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
+#define INET6_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif)      \
+        ((inet_sk(__sk)->inet_portpair == (__ports))            &&      \
+         ((__sk)->sk_family == AF_INET6)                        &&      \
+         ipv6_addr_equal(&inet6_sk(__sk)->daddr, (__saddr))     &&      \
+         ipv6_addr_equal(&inet6_sk(__sk)->rcv_saddr, (__daddr)) &&      \
+         (!(__sk)->sk_bound_dev_if      ||                              \
+           ((__sk)->sk_bound_dev_if == (__dif)))                &&      \
+         net_eq(sock_net(__sk), (__net)))
+
+#define INET6_TW_MATCH(__sk, __net, __saddr, __daddr, __ports, __dif)      \
+        ((inet_twsk(__sk)->tw_portpair == (__ports))                    && \
+         ((__sk)->sk_family == AF_INET6)                                && \
+         ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_daddr, (__saddr))     && \
+         ipv6_addr_equal(&inet6_twsk(__sk)->tw_v6_rcv_saddr, (__daddr)) && \
+         (!(__sk)->sk_bound_dev_if      ||                                 \
+          ((__sk)->sk_bound_dev_if == (__dif)))                         && \
+         net_eq(sock_net(__sk), (__net)))
 
 #endif /* _IPV6_H */
diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h
index 54be0287eb98..d1de4fbd45c2 100644
--- a/include/net/inet_hashtables.h
+++ b/include/net/inet_hashtables.h
@@ -299,30 +299,34 @@ typedef __u64 __bitwise __addrpair;
                                    (((__force __u64)(__be32)(__daddr)) << 32) | \
                                    ((__force __u64)(__be32)(__saddr)));
 #endif /* __BIG_ENDIAN */
-#define INET_MATCH(__sk, __net, __hash, __cookie, __saddr, __daddr, __ports, __dif)\
-        (((__sk)->sk_hash == (__hash)) && net_eq(sock_net(__sk), (__net)) &&    \
-         ((*((__addrpair *)&(inet_sk(__sk)->inet_daddr))) == (__cookie))  &&    \
-         ((*((__portpair *)&(inet_sk(__sk)->inet_dport))) == (__ports))   &&    \
-         (!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
-#define INET_TW_MATCH(__sk, __net, __hash, __cookie, __saddr, __daddr, __ports, __dif)\
-        (((__sk)->sk_hash == (__hash)) && net_eq(sock_net(__sk), (__net)) &&    \
-         ((*((__addrpair *)&(inet_twsk(__sk)->tw_daddr))) == (__cookie)) &&     \
-         ((*((__portpair *)&(inet_twsk(__sk)->tw_dport))) == (__ports)) &&      \
-         (!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
+#define INET_MATCH(__sk, __net, __cookie, __saddr, __daddr, __ports, __dif)     \
+        ((inet_sk(__sk)->inet_portpair == (__ports))            &&      \
+         (inet_sk(__sk)->inet_addrpair == (__cookie))           &&      \
+         (!(__sk)->sk_bound_dev_if      ||                              \
+           ((__sk)->sk_bound_dev_if == (__dif)))                &&      \
+         net_eq(sock_net(__sk), (__net)))
+#define INET_TW_MATCH(__sk, __net, __cookie, __saddr, __daddr, __ports, __dif)\
+        ((inet_twsk(__sk)->tw_portpair == (__ports))    &&              \
+         (inet_twsk(__sk)->tw_addrpair == (__cookie))   &&              \
+         (!(__sk)->sk_bound_dev_if      ||                              \
+           ((__sk)->sk_bound_dev_if == (__dif)))        &&              \
+         net_eq(sock_net(__sk), (__net)))
 #else /* 32-bit arch */
 #define INET_ADDR_COOKIE(__name, __saddr, __daddr)
-#define INET_MATCH(__sk, __net, __hash, __cookie, __saddr, __daddr, __ports, __dif)     \
-        (((__sk)->sk_hash == (__hash)) && net_eq(sock_net(__sk), (__net))       &&      \
-         (inet_sk(__sk)->inet_daddr     == (__saddr))           &&      \
-         (inet_sk(__sk)->inet_rcv_saddr == (__daddr))           &&      \
-         ((*((__portpair *)&(inet_sk(__sk)->inet_dport))) == (__ports)) &&      \
-         (!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
-#define INET_TW_MATCH(__sk, __net, __hash,__cookie, __saddr, __daddr, __ports, __dif)   \
-        (((__sk)->sk_hash == (__hash)) && net_eq(sock_net(__sk), (__net))       &&      \
-         (inet_twsk(__sk)->tw_daddr     == (__saddr))           &&      \
-         (inet_twsk(__sk)->tw_rcv_saddr == (__daddr))           &&      \
-         ((*((__portpair *)&(inet_twsk(__sk)->tw_dport))) == (__ports)) &&      \
-         (!((__sk)->sk_bound_dev_if) || ((__sk)->sk_bound_dev_if == (__dif))))
+#define INET_MATCH(__sk, __net, __cookie, __saddr, __daddr, __ports, __dif) \
+        ((inet_sk(__sk)->inet_portpair == (__ports))    &&              \
+         (inet_sk(__sk)->inet_daddr     == (__saddr))   &&              \
+         (inet_sk(__sk)->inet_rcv_saddr == (__daddr))   &&              \
+         (!(__sk)->sk_bound_dev_if      ||                              \
+           ((__sk)->sk_bound_dev_if == (__dif)))        &&              \
+         net_eq(sock_net(__sk), (__net)))
+#define INET_TW_MATCH(__sk, __net, __cookie, __saddr, __daddr, __ports, __dif) \
+        ((inet_twsk(__sk)->tw_portpair == (__ports))    &&              \
+         (inet_twsk(__sk)->tw_daddr     == (__saddr))   &&              \
+         (inet_twsk(__sk)->tw_rcv_saddr == (__daddr))   &&              \
+         (!(__sk)->sk_bound_dev_if      ||                              \
+           ((__sk)->sk_bound_dev_if == (__dif)))        &&              \
+         net_eq(sock_net(__sk), (__net)))
 #endif /* 64-bit arch */
 
 /*
diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h
index 256c1ed2d69a..a4196cbc84ec 100644
--- a/include/net/inet_sock.h
+++ b/include/net/inet_sock.h
@@ -144,9 +144,11 @@ struct inet_sock {
         /* Socket demultiplex comparisons on incoming packets. */
 #define inet_daddr              sk.__sk_common.skc_daddr
 #define inet_rcv_saddr          sk.__sk_common.skc_rcv_saddr
+#define inet_addrpair           sk.__sk_common.skc_addrpair
+#define inet_dport              sk.__sk_common.skc_dport
+#define inet_num                sk.__sk_common.skc_num
+#define inet_portpair           sk.__sk_common.skc_portpair
 
-        __be16                  inet_dport;
-        __u16                   inet_num;
         __be32                  inet_saddr;
         __s16                   uc_ttl;
         __u16                   cmsg_flags;
@@ -154,6 +156,7 @@ struct inet_sock {
         __u16                   inet_id;
 
         struct ip_options_rcu __rcu     *inet_opt;
+        int                     rx_dst_ifindex;
         __u8                    tos;
         __u8                    min_ttl;
         __u8                    mc_ttl;
@@ -170,7 +173,6 @@ struct inet_sock {
         int                     uc_index;
         int                     mc_index;
         __be32                  mc_addr;
-        int                     rx_dst_ifindex;
         struct ip_mc_socklist __rcu     *mc_list;
         struct inet_cork_full   cork;
 };
diff --git a/include/net/inet_timewait_sock.h b/include/net/inet_timewait_sock.h
index ba52c830a7a5..7d658d577368 100644
--- a/include/net/inet_timewait_sock.h
+++ b/include/net/inet_timewait_sock.h
@@ -112,6 +112,11 @@ struct inet_timewait_sock {
 #define tw_net                  __tw_common.skc_net
 #define tw_daddr                __tw_common.skc_daddr
 #define tw_rcv_saddr            __tw_common.skc_rcv_saddr
+#define tw_addrpair             __tw_common.skc_addrpair
+#define tw_dport                __tw_common.skc_dport
+#define tw_num                  __tw_common.skc_num
+#define tw_portpair             __tw_common.skc_portpair
+
         int                     tw_timeout;
         volatile unsigned char  tw_substate;
         unsigned char           tw_rcv_wscale;
@@ -119,8 +124,6 @@ struct inet_timewait_sock {
         /* Socket demultiplex comparisons on incoming packets. */
         /* these three are in inet_sock */
         __be16                  tw_sport;
-        __be16                  tw_dport;
-        __u16                   tw_num;
         kmemcheck_bitfield_begin(flags);
         /* And these are ours. */
         unsigned int            tw_ipv6only     : 1,
diff --git a/include/net/sock.h b/include/net/sock.h
index c945fba4f543..c4132c1b63a8 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -132,6 +132,8 @@ struct net;
  *      @skc_rcv_saddr: Bound local IPv4 addr
  *      @skc_hash: hash value used with various protocol lookup tables
  *      @skc_u16hashes: two u16 hash values used by UDP lookup tables
+ *      @skc_dport: placeholder for inet_dport/tw_dport
+ *      @skc_num: placeholder for inet_num/tw_num
  *      @skc_family: network address family
  *      @skc_state: Connection state
  *      @skc_reuse: %SO_REUSEADDR setting
@@ -149,16 +151,29 @@ struct net;
  *      for struct sock and struct inet_timewait_sock.
  */
 struct sock_common {
-        /* skc_daddr and skc_rcv_saddr must be grouped :
-         * cf INET_MATCH() and INET_TW_MATCH()
+        /* skc_daddr and skc_rcv_saddr must be grouped on a 8 bytes aligned
+         * address on 64bit arches : cf INET_MATCH() and INET_TW_MATCH()
          */
-        __be32                  skc_daddr;
-        __be32                  skc_rcv_saddr;
-
+        union {
+                unsigned long   skc_addrpair;
+                struct {
+                        __be32  skc_daddr;
+                        __be32  skc_rcv_saddr;
+                };
+        };
         union  {
                 unsigned int    skc_hash;
                 __u16           skc_u16hashes[2];
         };
+        /* skc_dport && skc_num must be grouped as well */
+        union {
+                u32             skc_portpair;
+                struct {
+                        __be16  skc_dport;
+                        __u16   skc_num;
+                };
+        };
+
         unsigned short          skc_family;
         volatile unsigned char  skc_state;
         unsigned char           skc_reuse;
diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
index 7880af970208..fa3ae8148710 100644
--- a/net/ipv4/inet_hashtables.c
+++ b/net/ipv4/inet_hashtables.c
@@ -237,12 +237,14 @@ struct sock *__inet_lookup_established(struct net *net,
         rcu_read_lock();
 begin:
         sk_nulls_for_each_rcu(sk, node, &head->chain) {
-                if (INET_MATCH(sk, net, hash, acookie,
-                                        saddr, daddr, ports, dif)) {
+                if (sk->sk_hash != hash)
+                        continue;
+                if (likely(INET_MATCH(sk, net, acookie,
+                                      saddr, daddr, ports, dif))) {
                         if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
                                 goto begintw;
-                        if (unlikely(!INET_MATCH(sk, net, hash, acookie,
-                                saddr, daddr, ports, dif))) {
+                        if (unlikely(!INET_MATCH(sk, net, acookie,
+                                                 saddr, daddr, ports, dif))) {
                                 sock_put(sk);
                                 goto begin;
                         }
@@ -260,14 +262,18 @@ begin:
 begintw:
         /* Must check for a TIME_WAIT'er before going to listener hash. */
         sk_nulls_for_each_rcu(sk, node, &head->twchain) {
-                if (INET_TW_MATCH(sk, net, hash, acookie,
-                                        saddr, daddr, ports, dif)) {
+                if (sk->sk_hash != hash)
+                        continue;
+                if (likely(INET_TW_MATCH(sk, net, acookie,
+                                         saddr, daddr, ports,
+                                         dif))) {
                         if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt))) {
                                 sk = NULL;
                                 goto out;
                         }
-                        if (unlikely(!INET_TW_MATCH(sk, net, hash, acookie,
-                                 saddr, daddr, ports, dif))) {
+                        if (unlikely(!INET_TW_MATCH(sk, net, acookie,
+                                                    saddr, daddr, ports,
+                                                    dif))) {
                                 sock_put(sk);
                                 goto begintw;
                         }
@@ -314,10 +320,12 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row,
 
         /* Check TIME-WAIT sockets first. */
         sk_nulls_for_each(sk2, node, &head->twchain) {
-                tw = inet_twsk(sk2);
+                if (sk2->sk_hash != hash)
+                        continue;
 
-                if (INET_TW_MATCH(sk2, net, hash, acookie,
-                                        saddr, daddr, ports, dif)) {
+                if (likely(INET_TW_MATCH(sk2, net, acookie,
+                                         saddr, daddr, ports, dif))) {
+                        tw = inet_twsk(sk2);
                         if (twsk_unique(sk, sk2, twp))
                                 goto unique;
                         else
@@ -328,8 +336,10 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row,
 
         /* And established part... */
         sk_nulls_for_each(sk2, node, &head->chain) {
-                if (INET_MATCH(sk2, net, hash, acookie,
-                                        saddr, daddr, ports, dif))
+                if (sk2->sk_hash != hash)
+                        continue;
+                if (likely(INET_MATCH(sk2, net, acookie,
+                                      saddr, daddr, ports, dif)))
                         goto not_unique;
         }
 
diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c
index 73f1a00a96af..dea17fd28e50 100644
--- a/net/ipv6/inet6_hashtables.c
+++ b/net/ipv6/inet6_hashtables.c
@@ -87,11 +87,13 @@ struct sock *__inet6_lookup_established(struct net *net,
         rcu_read_lock();
 begin:
         sk_nulls_for_each_rcu(sk, node, &head->chain) {
-                /* For IPV6 do the cheaper port and family tests first. */
-                if (INET6_MATCH(sk, net, hash, saddr, daddr, ports, dif)) {
+                if (sk->sk_hash != hash)
+                        continue;
+                if (likely(INET6_MATCH(sk, net, saddr, daddr, ports, dif))) {
                         if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))
                                 goto begintw;
-                        if (!INET6_MATCH(sk, net, hash, saddr, daddr, ports, dif)) {
+                        if (unlikely(!INET6_MATCH(sk, net, saddr, daddr,
+                                                  ports, dif))) {
                                 sock_put(sk);
                                 goto begin;
                         }
@@ -104,12 +106,16 @@ begin:
 begintw:
         /* Must check for a TIME_WAIT'er before going to listener hash. */
         sk_nulls_for_each_rcu(sk, node, &head->twchain) {
-                if (INET6_TW_MATCH(sk, net, hash, saddr, daddr, ports, dif)) {
+                if (sk->sk_hash != hash)
+                        continue;
+                if (likely(INET6_TW_MATCH(sk, net, saddr, daddr,
+                                          ports, dif))) {
                         if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt))) {
                                 sk = NULL;
                                 goto out;
                         }
-                        if (!INET6_TW_MATCH(sk, net, hash, saddr, daddr, ports, dif)) {
+                        if (unlikely(!INET6_TW_MATCH(sk, net, saddr, daddr,
+                                                     ports, dif))) {
                                 sock_put(sk);
                                 goto begintw;
                         }
@@ -236,9 +242,12 @@ static int __inet6_check_established(struct inet_timewait_death_row *death_row,
 
         /* Check TIME-WAIT sockets first. */
         sk_nulls_for_each(sk2, node, &head->twchain) {
-                tw = inet_twsk(sk2);
+                if (sk2->sk_hash != hash)
+                        continue;
 
-                if (INET6_TW_MATCH(sk2, net, hash, saddr, daddr, ports, dif)) {
+                if (likely(INET6_TW_MATCH(sk2, net, saddr, daddr,
+                                          ports, dif))) {
+                        tw = inet_twsk(sk2);
                         if (twsk_unique(sk, sk2, twp))
                                 goto unique;
                         else
@@ -249,7 +258,9 @@ static int __inet6_check_established(struct inet_timewait_death_row *death_row,
 
         /* And established part... */
         sk_nulls_for_each(sk2, node, &head->chain) {
-                if (INET6_MATCH(sk2, net, hash, saddr, daddr, ports, dif))
+                if (sk2->sk_hash != hash)
+                        continue;
+                if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports, dif)))
                         goto not_unique;
         }