netfilter: nf_conntrack: IPS_UNTRACKED bit

NOTRACK makes all cpus share a cache line on nf_conntrack_untracked
twice per packet. This is bad for performance.
__read_mostly annotation is also a bad choice.

This patch introduces IPS_UNTRACKED bit so that we can use later a
per_cpu untrack structure more easily.

A new helper, nf_ct_untracked_get() returns a pointer to
nf_conntrack_untracked.

Another one, nf_ct_untracked_status_or() is used by nf_nat_init() to add
IPS_NAT_DONE_MASK bits to untracked status.

nf_ct_is_untracked() prototype is changed to work on a nf_conn pointer.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: Patrick McHardy <kaber@trash.net>

9 files changed, 30 insertions, 22 deletions

diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c
index eeeb8bc73982..6c1da212380d 100644
--- a/net/netfilter/nf_conntrack_core.c
+++ b/net/netfilter/nf_conntrack_core.c
@@ -62,7 +62,7 @@ EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
 unsigned int nf_conntrack_max __read_mostly;
 EXPORT_SYMBOL_GPL(nf_conntrack_max);
 
-struct nf_conn nf_conntrack_untracked __read_mostly;
+struct nf_conn nf_conntrack_untracked;
 EXPORT_SYMBOL_GPL(nf_conntrack_untracked);
 
 static int nf_conntrack_hash_rnd_initted;
@@ -1321,6 +1321,12 @@ EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);
 module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
                   &nf_conntrack_htable_size, 0600);
 
+void nf_ct_untracked_status_or(unsigned long bits)
+{
+        nf_conntrack_untracked.status |= bits;
+}
+EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
+
 static int nf_conntrack_init_init_net(void)
 {
         int max_factor = 8;
@@ -1368,8 +1374,7 @@ static int nf_conntrack_init_init_net(void)
 #endif
         atomic_set(&nf_conntrack_untracked.ct_general.use, 1);
         /*  - and look it like as a confirmed connection */
-        set_bit(IPS_CONFIRMED_BIT, &nf_conntrack_untracked.status);
-
+        nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);
         return 0;
 
 #ifdef CONFIG_NF_CONNTRACK_ZONES
diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c
index c42ff6aa441d..5bae1cd15eea 100644
--- a/net/netfilter/nf_conntrack_netlink.c
+++ b/net/netfilter/nf_conntrack_netlink.c
@@ -480,7 +480,7 @@ ctnetlink_conntrack_event(unsigned int events, struct nf_ct_event *item)
         int err;
 
         /* ignore our fake conntrack entry */
-        if (ct == &nf_conntrack_untracked)
+        if (nf_ct_is_untracked(ct))
                 return 0;
 
         if (events & (1 << IPCT_DESTROY)) {
diff --git a/net/netfilter/xt_CT.c b/net/netfilter/xt_CT.c
index 562bf3266e04..0cb6053f02fd 100644
--- a/net/netfilter/xt_CT.c
+++ b/net/netfilter/xt_CT.c
@@ -67,7 +67,7 @@ static int xt_ct_tg_check(const struct xt_tgchk_param *par)
                 return -EINVAL;
 
         if (info->flags & XT_CT_NOTRACK) {
-                ct = &nf_conntrack_untracked;
+                ct = nf_ct_untracked_get();
                 atomic_inc(&ct->ct_general.use);
                 goto out;
         }
@@ -132,7 +132,7 @@ static void xt_ct_tg_destroy(const struct xt_tgdtor_param *par)
         struct nf_conn *ct = info->ct;
         struct nf_conn_help *help;
 
-        if (ct != &nf_conntrack_untracked) {
+        if (!nf_ct_is_untracked(ct)) {
                 help = nfct_help(ct);
                 if (help)
                         module_put(help->helper->me);
diff --git a/net/netfilter/xt_NOTRACK.c b/net/netfilter/xt_NOTRACK.c
index 512b9123252f..9d782181b6c8 100644
--- a/net/netfilter/xt_NOTRACK.c
+++ b/net/netfilter/xt_NOTRACK.c
@@ -23,7 +23,7 @@ notrack_tg(struct sk_buff *skb, const struct xt_action_param *par)
            If there is a real ct entry correspondig to this packet,
            it'll hang aroun till timing out. We don't deal with it
            for performance reasons. JK */
-        skb->nfct = &nf_conntrack_untracked.ct_general;
+        skb->nfct = &nf_ct_untracked_get()->ct_general;
         skb->nfctinfo = IP_CT_NEW;
         nf_conntrack_get(skb->nfct);
 
diff --git a/net/netfilter/xt_TEE.c b/net/netfilter/xt_TEE.c
index 859d9fd429c8..7a118267c4c4 100644
--- a/net/netfilter/xt_TEE.c
+++ b/net/netfilter/xt_TEE.c
@@ -104,7 +104,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
 #ifdef WITH_CONNTRACK
         /* Avoid counting cloned packets towards the original connection. */
         nf_conntrack_put(skb->nfct);
-        skb->nfct     = &nf_conntrack_untracked.ct_general;
+        skb->nfct     = &nf_ct_untracked_get()->ct_general;
         skb->nfctinfo = IP_CT_NEW;
         nf_conntrack_get(skb->nfct);
 #endif
@@ -177,7 +177,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
 
 #ifdef WITH_CONNTRACK
         nf_conntrack_put(skb->nfct);
-        skb->nfct     = &nf_conntrack_untracked.ct_general;
+        skb->nfct     = &nf_ct_untracked_get()->ct_general;
         skb->nfctinfo = IP_CT_NEW;
         nf_conntrack_get(skb->nfct);
 #endif
diff --git a/net/netfilter/xt_cluster.c b/net/netfilter/xt_cluster.c
index 30b95a1c1c89..f4af1bfafb1c 100644
--- a/net/netfilter/xt_cluster.c
+++ b/net/netfilter/xt_cluster.c
@@ -120,7 +120,7 @@ xt_cluster_mt(const struct sk_buff *skb, struct xt_action_param *par)
         if (ct == NULL)
                 return false;
 
-        if (ct == &nf_conntrack_untracked)
+        if (nf_ct_is_untracked(ct))
                 return false;
 
         if (ct->master)
diff --git a/net/netfilter/xt_conntrack.c b/net/netfilter/xt_conntrack.c
index 39681f10291c..e536710ad916 100644
--- a/net/netfilter/xt_conntrack.c
+++ b/net/netfilter/xt_conntrack.c
@@ -123,11 +123,12 @@ conntrack_mt(const struct sk_buff *skb, struct xt_action_param *par,
 
         ct = nf_ct_get(skb, &ctinfo);
 
-        if (ct == &nf_conntrack_untracked)
-                statebit = XT_CONNTRACK_STATE_UNTRACKED;
-        else if (ct != NULL)
-                statebit = XT_CONNTRACK_STATE_BIT(ctinfo);
-        else
+        if (ct) {
+                if (nf_ct_is_untracked(ct))
+                        statebit = XT_CONNTRACK_STATE_UNTRACKED;
+                else
+                        statebit = XT_CONNTRACK_STATE_BIT(ctinfo);
+        } else
                 statebit = XT_CONNTRACK_STATE_INVALID;
 
         if (info->match_flags & XT_CONNTRACK_STATE) {
diff --git a/net/netfilter/xt_socket.c b/net/netfilter/xt_socket.c
index 3d54c236a1ba..1ca89908cbad 100644
--- a/net/netfilter/xt_socket.c
+++ b/net/netfilter/xt_socket.c
@@ -127,7 +127,7 @@ socket_match(const struct sk_buff *skb, struct xt_action_param *par,
          * reply packet of an established SNAT-ted connection. */
 
         ct = nf_ct_get(skb, &ctinfo);
-        if (ct && (ct != &nf_conntrack_untracked) &&
+        if (ct && !nf_ct_is_untracked(ct) &&
             ((iph->protocol != IPPROTO_ICMP &&
               ctinfo == IP_CT_IS_REPLY + IP_CT_ESTABLISHED) ||
              (iph->protocol == IPPROTO_ICMP &&
diff --git a/net/netfilter/xt_state.c b/net/netfilter/xt_state.c
index e12e053d3782..a507922d80cd 100644
--- a/net/netfilter/xt_state.c
+++ b/net/netfilter/xt_state.c
@@ -26,14 +26,16 @@ state_mt(const struct sk_buff *skb, struct xt_action_param *par)
         const struct xt_state_info *sinfo = par->matchinfo;
         enum ip_conntrack_info ctinfo;
         unsigned int statebit;
+        struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
 
-        if (nf_ct_is_untracked(skb))
-                statebit = XT_STATE_UNTRACKED;
-        else if (!nf_ct_get(skb, &ctinfo))
+        if (!ct)
                 statebit = XT_STATE_INVALID;
-        else
-                statebit = XT_STATE_BIT(ctinfo);
-
+        else {
+                if (nf_ct_is_untracked(ct))
+                        statebit = XT_STATE_UNTRACKED;
+                else
+                        statebit = XT_STATE_BIT(ctinfo);
+        }
         return (sinfo->statemask & statebit);
 }