aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHerbert Xu <herbert@gondor.apana.org.au>2006-06-22 06:02:40 -0400
committerDavid S. Miller <davem@sunset.davemloft.net>2006-06-23 05:07:33 -0400
commitf4c50d990dcf11a296679dc05de3873783236711 (patch)
treef4daf1c80fe591d45631e998b0b5d31d6fe76d85
parentf6a78bfcb141f963187464bac838d46a81c3882a (diff)
[NET]: Add software TSOv4
This patch adds the GSO implementation for IPv4 TCP. Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r--include/linux/skbuff.h1
-rw-r--r--include/net/protocol.h1
-rw-r--r--include/net/tcp.h2
-rw-r--r--net/core/skbuff.c126
-rw-r--r--net/ipv4/af_inet.c51
-rw-r--r--net/ipv4/tcp.c62
6 files changed, 243 insertions, 0 deletions
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 97b0d2d1a6b0..a45bba9b8cbd 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -1297,6 +1297,7 @@ extern void skb_split(struct sk_buff *skb,
1297 struct sk_buff *skb1, const u32 len); 1297 struct sk_buff *skb1, const u32 len);
1298 1298
1299extern void skb_release_data(struct sk_buff *skb); 1299extern void skb_release_data(struct sk_buff *skb);
1300extern struct sk_buff *skb_segment(struct sk_buff *skb, int sg);
1300 1301
1301static inline void *skb_header_pointer(const struct sk_buff *skb, int offset, 1302static inline void *skb_header_pointer(const struct sk_buff *skb, int offset,
1302 int len, void *buffer) 1303 int len, void *buffer)
diff --git a/include/net/protocol.h b/include/net/protocol.h
index bcaee39bd2ff..3b6dc15c68a5 100644
--- a/include/net/protocol.h
+++ b/include/net/protocol.h
@@ -36,6 +36,7 @@
36struct net_protocol { 36struct net_protocol {
37 int (*handler)(struct sk_buff *skb); 37 int (*handler)(struct sk_buff *skb);
38 void (*err_handler)(struct sk_buff *skb, u32 info); 38 void (*err_handler)(struct sk_buff *skb, u32 info);
39 struct sk_buff *(*gso_segment)(struct sk_buff *skb, int sg);
39 int no_policy; 40 int no_policy;
40}; 41};
41 42
diff --git a/include/net/tcp.h b/include/net/tcp.h
index b197a9e615c1..ca3d38dfc00b 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -1086,6 +1086,8 @@ extern struct request_sock_ops tcp_request_sock_ops;
1086 1086
1087extern int tcp_v4_destroy_sock(struct sock *sk); 1087extern int tcp_v4_destroy_sock(struct sock *sk);
1088 1088
1089extern struct sk_buff *tcp_tso_segment(struct sk_buff *skb, int sg);
1090
1089#ifdef CONFIG_PROC_FS 1091#ifdef CONFIG_PROC_FS
1090extern int tcp4_proc_init(void); 1092extern int tcp4_proc_init(void);
1091extern void tcp4_proc_exit(void); 1093extern void tcp4_proc_exit(void);
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 368d98578c14..8e5044ba3ab6 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -1842,6 +1842,132 @@ unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
1842 1842
1843EXPORT_SYMBOL_GPL(skb_pull_rcsum); 1843EXPORT_SYMBOL_GPL(skb_pull_rcsum);
1844 1844
1845/**
1846 * skb_segment - Perform protocol segmentation on skb.
1847 * @skb: buffer to segment
1848 * @sg: whether scatter-gather can be used for generated segments
1849 *
1850 * This function performs segmentation on the given skb. It returns
1851 * the segment at the given position. It returns NULL if there are
1852 * no more segments to generate, or when an error is encountered.
1853 */
1854struct sk_buff *skb_segment(struct sk_buff *skb, int sg)
1855{
1856 struct sk_buff *segs = NULL;
1857 struct sk_buff *tail = NULL;
1858 unsigned int mss = skb_shinfo(skb)->gso_size;
1859 unsigned int doffset = skb->data - skb->mac.raw;
1860 unsigned int offset = doffset;
1861 unsigned int headroom;
1862 unsigned int len;
1863 int nfrags = skb_shinfo(skb)->nr_frags;
1864 int err = -ENOMEM;
1865 int i = 0;
1866 int pos;
1867
1868 __skb_push(skb, doffset);
1869 headroom = skb_headroom(skb);
1870 pos = skb_headlen(skb);
1871
1872 do {
1873 struct sk_buff *nskb;
1874 skb_frag_t *frag;
1875 int hsize, nsize;
1876 int k;
1877 int size;
1878
1879 len = skb->len - offset;
1880 if (len > mss)
1881 len = mss;
1882
1883 hsize = skb_headlen(skb) - offset;
1884 if (hsize < 0)
1885 hsize = 0;
1886 nsize = hsize + doffset;
1887 if (nsize > len + doffset || !sg)
1888 nsize = len + doffset;
1889
1890 nskb = alloc_skb(nsize + headroom, GFP_ATOMIC);
1891 if (unlikely(!nskb))
1892 goto err;
1893
1894 if (segs)
1895 tail->next = nskb;
1896 else
1897 segs = nskb;
1898 tail = nskb;
1899
1900 nskb->dev = skb->dev;
1901 nskb->priority = skb->priority;
1902 nskb->protocol = skb->protocol;
1903 nskb->dst = dst_clone(skb->dst);
1904 memcpy(nskb->cb, skb->cb, sizeof(skb->cb));
1905 nskb->pkt_type = skb->pkt_type;
1906 nskb->mac_len = skb->mac_len;
1907
1908 skb_reserve(nskb, headroom);
1909 nskb->mac.raw = nskb->data;
1910 nskb->nh.raw = nskb->data + skb->mac_len;
1911 nskb->h.raw = nskb->nh.raw + (skb->h.raw - skb->nh.raw);
1912 memcpy(skb_put(nskb, doffset), skb->data, doffset);
1913
1914 if (!sg) {
1915 nskb->csum = skb_copy_and_csum_bits(skb, offset,
1916 skb_put(nskb, len),
1917 len, 0);
1918 continue;
1919 }
1920
1921 frag = skb_shinfo(nskb)->frags;
1922 k = 0;
1923
1924 nskb->ip_summed = CHECKSUM_HW;
1925 nskb->csum = skb->csum;
1926 memcpy(skb_put(nskb, hsize), skb->data + offset, hsize);
1927
1928 while (pos < offset + len) {
1929 BUG_ON(i >= nfrags);
1930
1931 *frag = skb_shinfo(skb)->frags[i];
1932 get_page(frag->page);
1933 size = frag->size;
1934
1935 if (pos < offset) {
1936 frag->page_offset += offset - pos;
1937 frag->size -= offset - pos;
1938 }
1939
1940 k++;
1941
1942 if (pos + size <= offset + len) {
1943 i++;
1944 pos += size;
1945 } else {
1946 frag->size -= pos + size - (offset + len);
1947 break;
1948 }
1949
1950 frag++;
1951 }
1952
1953 skb_shinfo(nskb)->nr_frags = k;
1954 nskb->data_len = len - hsize;
1955 nskb->len += nskb->data_len;
1956 nskb->truesize += nskb->data_len;
1957 } while ((offset += len) < skb->len);
1958
1959 return segs;
1960
1961err:
1962 while ((skb = segs)) {
1963 segs = skb->next;
1964 kfree(skb);
1965 }
1966 return ERR_PTR(err);
1967}
1968
1969EXPORT_SYMBOL_GPL(skb_segment);
1970
1845void __init skb_init(void) 1971void __init skb_init(void)
1846{ 1972{
1847 skbuff_head_cache = kmem_cache_create("skbuff_head_cache", 1973 skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 0a277453526b..461216b47948 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -68,6 +68,7 @@
68 */ 68 */
69 69
70#include <linux/config.h> 70#include <linux/config.h>
71#include <linux/err.h>
71#include <linux/errno.h> 72#include <linux/errno.h>
72#include <linux/types.h> 73#include <linux/types.h>
73#include <linux/socket.h> 74#include <linux/socket.h>
@@ -1096,6 +1097,54 @@ int inet_sk_rebuild_header(struct sock *sk)
1096 1097
1097EXPORT_SYMBOL(inet_sk_rebuild_header); 1098EXPORT_SYMBOL(inet_sk_rebuild_header);
1098 1099
1100static struct sk_buff *inet_gso_segment(struct sk_buff *skb, int sg)
1101{
1102 struct sk_buff *segs = ERR_PTR(-EINVAL);
1103 struct iphdr *iph;
1104 struct net_protocol *ops;
1105 int proto;
1106 int ihl;
1107 int id;
1108
1109 if (!pskb_may_pull(skb, sizeof(*iph)))
1110 goto out;
1111
1112 iph = skb->nh.iph;
1113 ihl = iph->ihl * 4;
1114 if (ihl < sizeof(*iph))
1115 goto out;
1116
1117 if (!pskb_may_pull(skb, ihl))
1118 goto out;
1119
1120 skb->h.raw = __skb_pull(skb, ihl);
1121 iph = skb->nh.iph;
1122 id = ntohs(iph->id);
1123 proto = iph->protocol & (MAX_INET_PROTOS - 1);
1124 segs = ERR_PTR(-EPROTONOSUPPORT);
1125
1126 rcu_read_lock();
1127 ops = rcu_dereference(inet_protos[proto]);
1128 if (ops && ops->gso_segment)
1129 segs = ops->gso_segment(skb, sg);
1130 rcu_read_unlock();
1131
1132 if (IS_ERR(segs))
1133 goto out;
1134
1135 skb = segs;
1136 do {
1137 iph = skb->nh.iph;
1138 iph->id = htons(id++);
1139 iph->tot_len = htons(skb->len - skb->mac_len);
1140 iph->check = 0;
1141 iph->check = ip_fast_csum(skb->nh.raw, iph->ihl);
1142 } while ((skb = skb->next));
1143
1144out:
1145 return segs;
1146}
1147
1099#ifdef CONFIG_IP_MULTICAST 1148#ifdef CONFIG_IP_MULTICAST
1100static struct net_protocol igmp_protocol = { 1149static struct net_protocol igmp_protocol = {
1101 .handler = igmp_rcv, 1150 .handler = igmp_rcv,
@@ -1105,6 +1154,7 @@ static struct net_protocol igmp_protocol = {
1105static struct net_protocol tcp_protocol = { 1154static struct net_protocol tcp_protocol = {
1106 .handler = tcp_v4_rcv, 1155 .handler = tcp_v4_rcv,
1107 .err_handler = tcp_v4_err, 1156 .err_handler = tcp_v4_err,
1157 .gso_segment = tcp_tso_segment,
1108 .no_policy = 1, 1158 .no_policy = 1,
1109}; 1159};
1110 1160
@@ -1150,6 +1200,7 @@ static int ipv4_proc_init(void);
1150static struct packet_type ip_packet_type = { 1200static struct packet_type ip_packet_type = {
1151 .type = __constant_htons(ETH_P_IP), 1201 .type = __constant_htons(ETH_P_IP),
1152 .func = ip_rcv, 1202 .func = ip_rcv,
1203 .gso_segment = inet_gso_segment,
1153}; 1204};
1154 1205
1155static int __init inet_init(void) 1206static int __init inet_init(void)
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 062dd1a0d8a8..0e029c4e2903 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -258,6 +258,7 @@
258#include <linux/random.h> 258#include <linux/random.h>
259#include <linux/bootmem.h> 259#include <linux/bootmem.h>
260#include <linux/cache.h> 260#include <linux/cache.h>
261#include <linux/err.h>
261 262
262#include <net/icmp.h> 263#include <net/icmp.h>
263#include <net/tcp.h> 264#include <net/tcp.h>
@@ -2144,6 +2145,67 @@ int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
2144EXPORT_SYMBOL(compat_tcp_getsockopt); 2145EXPORT_SYMBOL(compat_tcp_getsockopt);
2145#endif 2146#endif
2146 2147
2148struct sk_buff *tcp_tso_segment(struct sk_buff *skb, int sg)
2149{
2150 struct sk_buff *segs = ERR_PTR(-EINVAL);
2151 struct tcphdr *th;
2152 unsigned thlen;
2153 unsigned int seq;
2154 unsigned int delta;
2155 unsigned int oldlen;
2156 unsigned int len;
2157
2158 if (!pskb_may_pull(skb, sizeof(*th)))
2159 goto out;
2160
2161 th = skb->h.th;
2162 thlen = th->doff * 4;
2163 if (thlen < sizeof(*th))
2164 goto out;
2165
2166 if (!pskb_may_pull(skb, thlen))
2167 goto out;
2168
2169 oldlen = ~htonl(skb->len);
2170 __skb_pull(skb, thlen);
2171
2172 segs = skb_segment(skb, sg);
2173 if (IS_ERR(segs))
2174 goto out;
2175
2176 len = skb_shinfo(skb)->gso_size;
2177 delta = csum_add(oldlen, htonl(thlen + len));
2178
2179 skb = segs;
2180 th = skb->h.th;
2181 seq = ntohl(th->seq);
2182
2183 do {
2184 th->fin = th->psh = 0;
2185
2186 if (skb->ip_summed == CHECKSUM_NONE) {
2187 th->check = csum_fold(csum_partial(
2188 skb->h.raw, thlen, csum_add(skb->csum, delta)));
2189 }
2190
2191 seq += len;
2192 skb = skb->next;
2193 th = skb->h.th;
2194
2195 th->seq = htonl(seq);
2196 th->cwr = 0;
2197 } while (skb->next);
2198
2199 if (skb->ip_summed == CHECKSUM_NONE) {
2200 delta = csum_add(oldlen, htonl(skb->tail - skb->h.raw));
2201 th->check = csum_fold(csum_partial(
2202 skb->h.raw, thlen, csum_add(skb->csum, delta)));
2203 }
2204
2205out:
2206 return segs;
2207}
2208
2147extern void __skb_cb_too_small_for_tcp(int, int); 2209extern void __skb_cb_too_small_for_tcp(int, int);
2148extern struct tcp_congestion_ops tcp_reno; 2210extern struct tcp_congestion_ops tcp_reno;
2149 2211