1 files changed, 50 insertions, 43 deletions

diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
index 451f97c42eb4..ba39a52d18c1 100644
--- a/net/ipv4/ip_output.c
+++ b/net/ipv4/ip_output.c
@@ -113,19 +113,6 @@ int ip_local_out(struct sk_buff *skb)
 }
 EXPORT_SYMBOL_GPL(ip_local_out);
 
-/* dev_loopback_xmit for use with netfilter. */
-static int ip_dev_loopback_xmit(struct sk_buff *newskb)
-{
-        skb_reset_mac_header(newskb);
-        __skb_pull(newskb, skb_network_offset(newskb));
-        newskb->pkt_type = PACKET_LOOPBACK;
-        newskb->ip_summed = CHECKSUM_UNNECESSARY;
-        WARN_ON(!skb_dst(newskb));
-        skb_dst_force(newskb);
-        netif_rx_ni(newskb);
-        return 0;
-}
-
 static inline int ip_select_ttl(struct inet_sock *inet, struct dst_entry *dst)
 {
         int ttl = inet->uc_ttl;
@@ -183,6 +170,7 @@ static inline int ip_finish_output2(struct sk_buff *skb)
         struct net_device *dev = dst->dev;
         unsigned int hh_len = LL_RESERVED_SPACE(dev);
         struct neighbour *neigh;
+        u32 nexthop;
 
         if (rt->rt_type == RTN_MULTICAST) {
                 IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTMCAST, skb->len);
@@ -200,19 +188,22 @@ static inline int ip_finish_output2(struct sk_buff *skb)
                 }
                 if (skb->sk)
                         skb_set_owner_w(skb2, skb->sk);
-                kfree_skb(skb);
+                consume_skb(skb);
                 skb = skb2;
         }
 
-        rcu_read_lock();
-        neigh = dst_get_neighbour_noref(dst);
+        rcu_read_lock_bh();
+        nexthop = rt->rt_gateway ? rt->rt_gateway : ip_hdr(skb)->daddr;
+        neigh = __ipv4_neigh_lookup_noref(dev, nexthop);
+        if (unlikely(!neigh))
+                neigh = __neigh_create(&arp_tbl, &nexthop, dev, false);
         if (neigh) {
-                int res = neigh_output(neigh, skb);
+                int res = dst_neigh_output(dst, neigh, skb);
 
-                rcu_read_unlock();
+                rcu_read_unlock_bh();
                 return res;
         }
-        rcu_read_unlock();
+        rcu_read_unlock_bh();
 
         net_dbg_ratelimited("%s: No header cache and no neighbour!\n",
                             __func__);
@@ -281,7 +272,7 @@ int ip_mc_output(struct sk_buff *skb)
                         if (newskb)
                                 NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING,
                                         newskb, NULL, newskb->dev,
-                                        ip_dev_loopback_xmit);
+                                        dev_loopback_xmit);
                 }
 
                 /* Multicasts with ttl 0 must not go beyond the host */
@@ -296,7 +287,7 @@ int ip_mc_output(struct sk_buff *skb)
                 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
                 if (newskb)
                         NF_HOOK(NFPROTO_IPV4, NF_INET_POST_ROUTING, newskb,
-                                NULL, newskb->dev, ip_dev_loopback_xmit);
+                                NULL, newskb->dev, dev_loopback_xmit);
         }
 
         return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, skb, NULL,
@@ -380,7 +371,7 @@ int ip_queue_xmit(struct sk_buff *skb, struct flowi *fl)
         skb_dst_set_noref(skb, &rt->dst);
 
 packet_routed:
-        if (inet_opt && inet_opt->opt.is_strictroute && fl4->daddr != rt->rt_gateway)
+        if (inet_opt && inet_opt->opt.is_strictroute && rt->rt_gateway)
                 goto no_route;
 
         /* OK, we know where to send it, allocate and build IP header. */
@@ -709,7 +700,7 @@ slow_path:
 
                 IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGCREATES);
         }
-        kfree_skb(skb);
+        consume_skb(skb);
         IP_INC_STATS(dev_net(dev), IPSTATS_MIB_FRAGOKS);
         return err;
 
@@ -1472,19 +1463,34 @@ static int ip_reply_glue_bits(void *dptr, char *to, int offset,
 
 /*
  *      Generic function to send a packet as reply to another packet.
- *      Used to send TCP resets so far. ICMP should use this function too.
+ *      Used to send some TCP resets/acks so far.
  *
- *      Should run single threaded per socket because it uses the sock
- *      structure to pass arguments.
+ *      Use a fake percpu inet socket to avoid false sharing and contention.
  */
-void ip_send_reply(struct sock *sk, struct sk_buff *skb, __be32 daddr,
-                   const struct ip_reply_arg *arg, unsigned int len)
+static DEFINE_PER_CPU(struct inet_sock, unicast_sock) = {
+        .sk = {
+                .__sk_common = {
+                        .skc_refcnt = ATOMIC_INIT(1),
+                },
+                .sk_wmem_alloc  = ATOMIC_INIT(1),
+                .sk_allocation  = GFP_ATOMIC,
+                .sk_flags       = (1UL << SOCK_USE_WRITE_QUEUE),
+        },
+        .pmtudisc       = IP_PMTUDISC_WANT,
+        .uc_ttl         = -1,
+};
+
+void ip_send_unicast_reply(struct net *net, struct sk_buff *skb, __be32 daddr,
+                           __be32 saddr, const struct ip_reply_arg *arg,
+                           unsigned int len)
 {
-        struct inet_sock *inet = inet_sk(sk);
         struct ip_options_data replyopts;
         struct ipcm_cookie ipc;
         struct flowi4 fl4;
         struct rtable *rt = skb_rtable(skb);
+        struct sk_buff *nskb;
+        struct sock *sk;
+        struct inet_sock *inet;
 
         if (ip_options_echo(&replyopts.opt.opt, skb))
                 return;
@@ -1502,38 +1508,39 @@ void ip_send_reply(struct sock *sk, struct sk_buff *skb, __be32 daddr,
 
         flowi4_init_output(&fl4, arg->bound_dev_if, 0,
                            RT_TOS(arg->tos),
-                           RT_SCOPE_UNIVERSE, sk->sk_protocol,
+                           RT_SCOPE_UNIVERSE, ip_hdr(skb)->protocol,
                            ip_reply_arg_flowi_flags(arg),
-                           daddr, rt->rt_spec_dst,
+                           daddr, saddr,
                            tcp_hdr(skb)->source, tcp_hdr(skb)->dest);
         security_skb_classify_flow(skb, flowi4_to_flowi(&fl4));
-        rt = ip_route_output_key(sock_net(sk), &fl4);
+        rt = ip_route_output_key(net, &fl4);
         if (IS_ERR(rt))
                 return;
 
-        /* And let IP do all the hard work.
+        inet = &get_cpu_var(unicast_sock);
 
-           This chunk is not reenterable, hence spinlock.
-           Note that it uses the fact, that this function is called
-           with locally disabled BH and that sk cannot be already spinlocked.
-         */
-        bh_lock_sock(sk);
         inet->tos = arg->tos;
+        sk = &inet->sk;
         sk->sk_priority = skb->priority;
         sk->sk_protocol = ip_hdr(skb)->protocol;
         sk->sk_bound_dev_if = arg->bound_dev_if;
+        sock_net_set(sk, net);
+        __skb_queue_head_init(&sk->sk_write_queue);
+        sk->sk_sndbuf = sysctl_wmem_default;
         ip_append_data(sk, &fl4, ip_reply_glue_bits, arg->iov->iov_base, len, 0,
                        &ipc, &rt, MSG_DONTWAIT);
-        if ((skb = skb_peek(&sk->sk_write_queue)) != NULL) {
+        nskb = skb_peek(&sk->sk_write_queue);
+        if (nskb) {
                 if (arg->csumoffset >= 0)
-                        *((__sum16 *)skb_transport_header(skb) +
-                          arg->csumoffset) = csum_fold(csum_add(skb->csum,
+                        *((__sum16 *)skb_transport_header(nskb) +
+                          arg->csumoffset) = csum_fold(csum_add(nskb->csum,
                                                                 arg->csum));
-                skb->ip_summed = CHECKSUM_NONE;
+                nskb->ip_summed = CHECKSUM_NONE;
+                skb_set_queue_mapping(nskb, skb_get_queue_mapping(skb));
                 ip_push_pending_frames(sk, &fl4);
         }
 
-        bh_unlock_sock(sk);
+        put_cpu_var(unicast_sock);
 
         ip_rt_put(rt);
 }