v4 GRE: Add TCP segmentation offload for GRE

Following patch adds GRE protocol offload handler so that
skb_gso_segment() can segment GRE packets.
SKB GSO CB is added to keep track of total header length so that
skb_segment can push entire header. e.g. in case of GRE, skb_segment
need to push inner and outer headers to every segment.
New NETIF_F_GRE_GSO feature is added for devices which support HW
GRE TSO offload. Currently none of devices support it therefore GRE GSO
always fall backs to software GSO.

[ Compute pkt_len before ip_local_out() invocation. -DaveM ]

Signed-off-by: Pravin B Shelar <pshelar@nicira.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

5 files changed, 197 insertions, 8 deletions

diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index e6e5d8506336..e225a4e5b572 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1287,6 +1287,7 @@ static struct sk_buff *inet_gso_segment(struct sk_buff *skb,
                        SKB_GSO_UDP |
                        SKB_GSO_DODGY |
                        SKB_GSO_TCP_ECN |
+                       SKB_GSO_GRE |
                        0)))
                 goto out;
 
diff --git a/net/ipv4/gre.c b/net/ipv4/gre.c
index 42a491055c76..7a4c710c4cdd 100644
--- a/net/ipv4/gre.c
+++ b/net/ipv4/gre.c
@@ -19,6 +19,7 @@
 #include <linux/in.h>
 #include <linux/ip.h>
 #include <linux/netdevice.h>
+#include <linux/if_tunnel.h>
 #include <linux/spinlock.h>
 #include <net/protocol.h>
 #include <net/gre.h>
@@ -26,6 +27,11 @@
 
 static const struct gre_protocol __rcu *gre_proto[GREPROTO_MAX] __read_mostly;
 static DEFINE_SPINLOCK(gre_proto_lock);
+struct gre_base_hdr {
+        __be16 flags;
+        __be16 protocol;
+};
+#define GRE_HEADER_SECTION 4
 
 int gre_add_protocol(const struct gre_protocol *proto, u8 version)
 {
@@ -112,12 +118,117 @@ static void gre_err(struct sk_buff *skb, u32 info)
         rcu_read_unlock();
 }
 
+static struct sk_buff *gre_gso_segment(struct sk_buff *skb,
+                                       netdev_features_t features)
+{
+        struct sk_buff *segs = ERR_PTR(-EINVAL);
+        netdev_features_t enc_features;
+        int ghl = GRE_HEADER_SECTION;
+        struct gre_base_hdr *greh;
+        int mac_len = skb->mac_len;
+        int tnl_hlen;
+        bool csum;
+
+        if (unlikely(skb_shinfo(skb)->gso_type &
+                                ~(SKB_GSO_TCPV4 |
+                                  SKB_GSO_TCPV6 |
+                                  SKB_GSO_UDP |
+                                  SKB_GSO_DODGY |
+                                  SKB_GSO_TCP_ECN |
+                                  SKB_GSO_GRE)))
+                goto out;
+
+        if (unlikely(!pskb_may_pull(skb, sizeof(*greh))))
+                goto out;
+
+        greh = (struct gre_base_hdr *)skb_transport_header(skb);
+
+        if (greh->flags & GRE_KEY)
+                ghl += GRE_HEADER_SECTION;
+        if (greh->flags & GRE_SEQ)
+                ghl += GRE_HEADER_SECTION;
+        if (greh->flags & GRE_CSUM) {
+                ghl += GRE_HEADER_SECTION;
+                csum = true;
+        } else
+                csum = false;
+
+        /* setup inner skb. */
+        if (greh->protocol == htons(ETH_P_TEB)) {
+                struct ethhdr *eth = eth_hdr(skb);
+                skb->protocol = eth->h_proto;
+        } else {
+                skb->protocol = greh->protocol;
+        }
+
+        skb->encapsulation = 0;
+
+        if (unlikely(!pskb_may_pull(skb, ghl)))
+                goto out;
+        __skb_pull(skb, ghl);
+        skb_reset_mac_header(skb);
+        skb_set_network_header(skb, skb_inner_network_offset(skb));
+        skb->mac_len = skb_inner_network_offset(skb);
+
+        /* segment inner packet. */
+        enc_features = skb->dev->hw_enc_features & netif_skb_features(skb);
+        segs = skb_mac_gso_segment(skb, enc_features);
+        if (!segs || IS_ERR(segs))
+                goto out;
+
+        skb = segs;
+        tnl_hlen = skb_tnl_header_len(skb);
+        do {
+                __skb_push(skb, ghl);
+                if (csum) {
+                        __be32 *pcsum;
+
+                        if (skb_has_shared_frag(skb)) {
+                                int err;
+
+                                err = __skb_linearize(skb);
+                                if (err) {
+                                        kfree_skb(segs);
+                                        segs = ERR_PTR(err);
+                                        goto out;
+                                }
+                        }
+
+                        greh = (struct gre_base_hdr *)(skb->data);
+                        pcsum = (__be32 *)(greh + 1);
+                        *pcsum = 0;
+                        *(__sum16 *)pcsum = csum_fold(skb_checksum(skb, 0, skb->len, 0));
+                }
+                __skb_push(skb, tnl_hlen - ghl);
+
+                skb_reset_mac_header(skb);
+                skb_set_network_header(skb, mac_len);
+                skb->mac_len = mac_len;
+        } while ((skb = skb->next));
+out:
+        return segs;
+}
+
+static int gre_gso_send_check(struct sk_buff *skb)
+{
+        if (!skb->encapsulation)
+                return -EINVAL;
+        return 0;
+}
+
 static const struct net_protocol net_gre_protocol = {
         .handler     = gre_rcv,
         .err_handler = gre_err,
         .netns_ok    = 1,
 };
 
+static const struct net_offload gre_offload = {
+        .callbacks = {
+                .gso_send_check =       gre_gso_send_check,
+                .gso_segment    =       gre_gso_segment,
+        },
+};
+
 static int __init gre_init(void)
 {
         pr_info("GRE over IPv4 demultiplexor driver\n");
@@ -127,11 +238,18 @@ static int __init gre_init(void)
                 return -EAGAIN;
         }
 
+        if (inet_add_offload(&gre_offload, IPPROTO_GRE)) {
+                pr_err("can't add protocol offload\n");
+                inet_del_protocol(&net_gre_protocol, IPPROTO_GRE);
+                return -EAGAIN;
+        }
+
         return 0;
 }
 
 static void __exit gre_exit(void)
 {
+        inet_del_offload(&gre_offload, IPPROTO_GRE);
         inet_del_protocol(&net_gre_protocol, IPPROTO_GRE);
 }
 
diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c
index 00a14b9864ea..a56f1182c176 100644
--- a/net/ipv4/ip_gre.c
+++ b/net/ipv4/ip_gre.c
@@ -735,8 +735,33 @@ drop:
         return 0;
 }
 
+static struct sk_buff *handle_offloads(struct sk_buff *skb)
+{
+        int err;
+
+        if (skb_is_gso(skb)) {
+                err = skb_unclone(skb, GFP_ATOMIC);
+                if (unlikely(err))
+                        goto error;
+                skb_shinfo(skb)->gso_type |= SKB_GSO_GRE;
+                return skb;
+        } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
+                err = skb_checksum_help(skb);
+                if (unlikely(err))
+                        goto error;
+        }
+        skb->ip_summed = CHECKSUM_NONE;
+
+        return skb;
+
+error:
+        kfree_skb(skb);
+        return ERR_PTR(err);
+}
+
 static netdev_tx_t ipgre_tunnel_xmit(struct sk_buff *skb, struct net_device *dev)
 {
+        struct pcpu_tstats *tstats = this_cpu_ptr(dev->tstats);
         struct ip_tunnel *tunnel = netdev_priv(dev);
         const struct iphdr  *old_iph;
         const struct iphdr  *tiph;
@@ -751,10 +776,19 @@ static netdev_tx_t ipgre_tunnel_xmit(struct sk_buff *skb, struct net_device *dev
         __be32 dst;
         int    mtu;
         u8     ttl;
+        int    err;
+        int    pkt_len;
 
-        if (skb->ip_summed == CHECKSUM_PARTIAL &&
-            skb_checksum_help(skb))
-                goto tx_error;
+        skb = handle_offloads(skb);
+        if (IS_ERR(skb)) {
+                dev->stats.tx_dropped++;
+                return NETDEV_TX_OK;
+        }
+
+        if (!skb->encapsulation) {
+                skb_reset_inner_headers(skb);
+                skb->encapsulation = 1;
+        }
 
         old_iph = ip_hdr(skb);
 
@@ -855,7 +889,8 @@ static netdev_tx_t ipgre_tunnel_xmit(struct sk_buff *skb, struct net_device *dev
         if (skb->protocol == htons(ETH_P_IP)) {
                 df |= (old_iph->frag_off&htons(IP_DF));
 
-                if ((old_iph->frag_off&htons(IP_DF)) &&
+                if (!skb_is_gso(skb) &&
+                    (old_iph->frag_off&htons(IP_DF)) &&
                     mtu < ntohs(old_iph->tot_len)) {
                         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu));
                         ip_rt_put(rt);
@@ -875,7 +910,9 @@ static netdev_tx_t ipgre_tunnel_xmit(struct sk_buff *skb, struct net_device *dev
                         }
                 }
 
-                if (mtu >= IPV6_MIN_MTU && mtu < skb->len - tunnel->hlen + gre_hlen) {
+                if (!skb_is_gso(skb) &&
+                    mtu >= IPV6_MIN_MTU &&
+                    mtu < skb->len - tunnel->hlen + gre_hlen) {
                         icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
                         ip_rt_put(rt);
                         goto tx_error;
@@ -936,6 +973,7 @@ static netdev_tx_t ipgre_tunnel_xmit(struct sk_buff *skb, struct net_device *dev
         iph->daddr              =       fl4.daddr;
         iph->saddr              =       fl4.saddr;
         iph->ttl                =       ttl;
+        iph->id                 =       0;
 
         if (ttl == 0) {
                 if (skb->protocol == htons(ETH_P_IP))
@@ -964,9 +1002,19 @@ static netdev_tx_t ipgre_tunnel_xmit(struct sk_buff *skb, struct net_device *dev
                         *ptr = tunnel->parms.o_key;
                         ptr--;
                 }
-                if (tunnel->parms.o_flags&GRE_CSUM) {
+                /* Skip GRE checksum if skb is getting offloaded. */
+                if (!(skb_shinfo(skb)->gso_type & SKB_GSO_GRE) &&
+                    (tunnel->parms.o_flags&GRE_CSUM)) {
                         int offset = skb_transport_offset(skb);
 
+                        if (skb_has_shared_frag(skb)) {
+                                err = __skb_linearize(skb);
+                                if (err) {
+                                        ip_rt_put(rt);
+                                        goto tx_error;
+                                }
+                        }
+
                         *ptr = 0;
                         *(__sum16 *)ptr = csum_fold(skb_checksum(skb, offset,
                                                                  skb->len - offset,
@@ -974,7 +1022,19 @@ static netdev_tx_t ipgre_tunnel_xmit(struct sk_buff *skb, struct net_device *dev
                 }
         }
 
-        iptunnel_xmit(skb, dev);
+        nf_reset(skb);
+
+        pkt_len = skb->len - skb_transport_offset(skb);
+        err = ip_local_out(skb);
+        if (likely(net_xmit_eval(err) == 0)) {
+                u64_stats_update_begin(&tstats->syncp);
+                tstats->tx_bytes += pkt_len;
+                tstats->tx_packets++;
+                u64_stats_update_end(&tstats->syncp);
+        } else {
+                dev->stats.tx_errors++;
+                dev->stats.tx_aborted_errors++;
+        }
         return NETDEV_TX_OK;
 
 #if IS_ENABLED(CONFIG_IPV6)
@@ -1044,6 +1104,11 @@ static int ipgre_tunnel_bind_dev(struct net_device *dev)
                 mtu = 68;
 
         tunnel->hlen = addend;
+        /* TCP offload with GRE SEQ is not supported. */
+        if (!(tunnel->parms.o_flags & GRE_SEQ)) {
+                dev->features           |= NETIF_F_GSO_SOFTWARE;
+                dev->hw_features        |= NETIF_F_GSO_SOFTWARE;
+        }
 
         return mtu;
 }
@@ -1593,6 +1658,9 @@ static void ipgre_tap_setup(struct net_device *dev)
 
         dev->iflink             = 0;
         dev->features           |= NETIF_F_NETNS_LOCAL;
+
+        dev->features           |= GRE_FEATURES;
+        dev->hw_features        |= GRE_FEATURES;
 }
 
 static int ipgre_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[],
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 1f0bedb8622f..7a5ba48c2cc9 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -3043,6 +3043,7 @@ struct sk_buff *tcp_tso_segment(struct sk_buff *skb,
                                SKB_GSO_DODGY |
                                SKB_GSO_TCP_ECN |
                                SKB_GSO_TCPV6 |
+                               SKB_GSO_GRE |
                                0) ||
                              !(type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))))
                         goto out;
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 6791aac06ea9..39a5e7a9a77f 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -2305,7 +2305,8 @@ struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,
                 /* Packet is from an untrusted source, reset gso_segs. */
                 int type = skb_shinfo(skb)->gso_type;
 
-                if (unlikely(type & ~(SKB_GSO_UDP | SKB_GSO_DODGY) ||
+                if (unlikely(type & ~(SKB_GSO_UDP | SKB_GSO_DODGY |
+                                      SKB_GSO_GRE) ||
                              !(type & (SKB_GSO_UDP))))
                         goto out;