22 files changed, 78 insertions, 99 deletions

diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c
index c67fe6f75653..7f7de1a04de6 100644
--- a/net/8021q/vlan_core.c
+++ b/net/8021q/vlan_core.c
@@ -114,9 +114,9 @@ int vlan_gro_receive(struct napi_struct *napi, struct vlan_group *grp,
 EXPORT_SYMBOL(vlan_gro_receive);
 
 int vlan_gro_frags(struct napi_struct *napi, struct vlan_group *grp,
-                   unsigned int vlan_tci, struct napi_gro_fraginfo *info)
+                   unsigned int vlan_tci)
 {
-        struct sk_buff *skb = napi_fraginfo_skb(napi, info);
+        struct sk_buff *skb = napi_frags_skb(napi);
 
         if (!skb)
                 return NET_RX_DROP;
diff --git a/net/core/dev.c b/net/core/dev.c
index 308a7d0c277f..e48c08af76ad 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2525,16 +2525,10 @@ void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
 }
 EXPORT_SYMBOL(napi_reuse_skb);
 
-struct sk_buff *napi_fraginfo_skb(struct napi_struct *napi,
-                                  struct napi_gro_fraginfo *info)
+struct sk_buff *napi_get_frags(struct napi_struct *napi)
 {
         struct net_device *dev = napi->dev;
         struct sk_buff *skb = napi->skb;
-        struct ethhdr *eth;
-        skb_frag_t *frag;
-        int i;
-
-        napi->skb = NULL;
 
         if (!skb) {
                 skb = netdev_alloc_skb(dev, GRO_MAX_HEAD + NET_IP_ALIGN);
@@ -2542,47 +2536,14 @@ struct sk_buff *napi_fraginfo_skb(struct napi_struct *napi,
                         goto out;
 
                 skb_reserve(skb, NET_IP_ALIGN);
-        }
-
-        BUG_ON(info->nr_frags > MAX_SKB_FRAGS);
-        frag = info->frags;
 
-        for (i = 0; i < info->nr_frags; i++) {
-                skb_fill_page_desc(skb, i, frag->page, frag->page_offset,
-                                   frag->size);
-                frag++;
+                napi->skb = skb;
         }
-        skb_shinfo(skb)->nr_frags = info->nr_frags;
-
-        skb->data_len = info->len;
-        skb->len += info->len;
-        skb->truesize += info->len;
-
-        skb_reset_mac_header(skb);
-        skb_gro_reset_offset(skb);
-
-        eth = skb_gro_header(skb, sizeof(*eth));
-        if (!eth) {
-                napi_reuse_skb(napi, skb);
-                skb = NULL;
-                goto out;
-        }
-
-        skb_gro_pull(skb, sizeof(*eth));
-
-        /*
-         * This works because the only protocols we care about don't require
-         * special handling.  We'll fix it up properly at the end.
-         */
-        skb->protocol = eth->h_proto;
-
-        skb->ip_summed = info->ip_summed;
-        skb->csum = info->csum;
 
 out:
         return skb;
 }
-EXPORT_SYMBOL(napi_fraginfo_skb);
+EXPORT_SYMBOL(napi_get_frags);
 
 int napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb, int ret)
 {
@@ -2612,9 +2573,39 @@ int napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb, int ret)
 }
 EXPORT_SYMBOL(napi_frags_finish);
 
-int napi_gro_frags(struct napi_struct *napi, struct napi_gro_fraginfo *info)
+struct sk_buff *napi_frags_skb(struct napi_struct *napi)
+{
+        struct sk_buff *skb = napi->skb;
+        struct ethhdr *eth;
+
+        napi->skb = NULL;
+
+        skb_reset_mac_header(skb);
+        skb_gro_reset_offset(skb);
+
+        eth = skb_gro_header(skb, sizeof(*eth));
+        if (!eth) {
+                napi_reuse_skb(napi, skb);
+                skb = NULL;
+                goto out;
+        }
+
+        skb_gro_pull(skb, sizeof(*eth));
+
+        /*
+         * This works because the only protocols we care about don't require
+         * special handling.  We'll fix it up properly at the end.
+         */
+        skb->protocol = eth->h_proto;
+
+out:
+        return skb;
+}
+EXPORT_SYMBOL(napi_frags_skb);
+
+int napi_gro_frags(struct napi_struct *napi)
 {
-        struct sk_buff *skb = napi_fraginfo_skb(napi, info);
+        struct sk_buff *skb = napi_frags_skb(napi);
 
         if (!skb)
                 return NET_RX_DROP;
@@ -2718,7 +2709,7 @@ void netif_napi_del(struct napi_struct *napi)
         struct sk_buff *skb, *next;
 
         list_del_init(&napi->dev_list);
-        kfree_skb(napi->skb);
+        napi_free_frags(napi);
 
         for (skb = napi->gro_list; skb; skb = next) {
                 next = skb->next;
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 7f03373b8c07..170689681aa2 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1003,8 +1003,6 @@ void inet_register_protosw(struct inet_protosw *p)
 out:
         spin_unlock_bh(&inetsw_lock);
 
-        synchronize_net();
-
         return;
 
 out_permanent:
diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c
index b35a950d2e06..cd2b97f1b6e1 100644
--- a/net/ipv4/syncookies.c
+++ b/net/ipv4/syncookies.c
@@ -161,13 +161,12 @@ static __u16 const msstab[] = {
  */
 __u32 cookie_v4_init_sequence(struct sock *sk, struct sk_buff *skb, __u16 *mssp)
 {
-        struct tcp_sock *tp = tcp_sk(sk);
         const struct iphdr *iph = ip_hdr(skb);
         const struct tcphdr *th = tcp_hdr(skb);
         int mssind;
         const __u16 mss = *mssp;
 
-        tp->last_synq_overflow = jiffies;
+        tcp_synq_overflow(sk);
 
         /* XXX sort msstab[] by probability?  Binary search? */
         for (mssind = 0; mss > msstab[mssind + 1]; mssind++)
@@ -268,7 +267,7 @@ struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb,
         if (!sysctl_tcp_syncookies || !th->ack)
                 goto out;
 
-        if (time_after(jiffies, tp->last_synq_overflow + TCP_TIMEOUT_INIT) ||
+        if (tcp_synq_no_recent_overflow(sk) ||
             (mss = cookie_check(skb, cookie)) == 0) {
                 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_SYNCOOKIESFAILED);
                 goto out;
diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c
index 711175e0571f..8c2513982b61 100644
--- a/net/ipv6/syncookies.c
+++ b/net/ipv6/syncookies.c
@@ -131,7 +131,7 @@ __u32 cookie_v6_init_sequence(struct sock *sk, struct sk_buff *skb, __u16 *mssp)
         int mssind;
         const __u16 mss = *mssp;
 
-        tcp_sk(sk)->last_synq_overflow = jiffies;
+        tcp_synq_overflow(sk);
 
         for (mssind = 0; mss > msstab[mssind + 1]; mssind++)
                 ;
@@ -175,7 +175,7 @@ struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb)
         if (!sysctl_tcp_syncookies || !th->ack)
                 goto out;
 
-        if (time_after(jiffies, tp->last_synq_overflow + TCP_TIMEOUT_INIT) ||
+        if (tcp_synq_no_recent_overflow(sk) ||
                 (mss = cookie_check(skb, cookie)) == 0) {
                 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_SYNCOOKIESFAILED);
                 goto out;
diff --git a/net/rds/af_rds.c b/net/rds/af_rds.c
index 20cf16fc572f..b11e7e527864 100644
--- a/net/rds/af_rds.c
+++ b/net/rds/af_rds.c
@@ -35,7 +35,6 @@
 #include <linux/kernel.h>
 #include <linux/in.h>
 #include <linux/poll.h>
-#include <linux/version.h>
 #include <net/sock.h>
 
 #include "rds.h"
diff --git a/net/rds/connection.c b/net/rds/connection.c
index 273f064930a8..d14445c48304 100644
--- a/net/rds/connection.c
+++ b/net/rds/connection.c
@@ -148,14 +148,12 @@ static struct rds_connection *__rds_conn_create(__be32 laddr, __be32 faddr,
         if (conn)
                 goto out;
 
-        conn = kmem_cache_alloc(rds_conn_slab, gfp);
+        conn = kmem_cache_zalloc(rds_conn_slab, gfp);
         if (conn == NULL) {
                 conn = ERR_PTR(-ENOMEM);
                 goto out;
         }
 
-        memset(conn, 0, sizeof(*conn));
-
         INIT_HLIST_NODE(&conn->c_hash_node);
         conn->c_version = RDS_PROTOCOL_3_0;
         conn->c_laddr = laddr;
diff --git a/net/rds/ib.c b/net/rds/ib.c
index 4933b380985e..b9bcd32431e1 100644
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -224,8 +224,8 @@ static int rds_ib_laddr_check(__be32 addr)
          * IB and iWARP capable NICs.
          */
         cm_id = rdma_create_id(NULL, NULL, RDMA_PS_TCP);
-        if (!cm_id)
-                return -EADDRNOTAVAIL;
+        if (IS_ERR(cm_id))
+                return PTR_ERR(cm_id);
 
         memset(&sin, 0, sizeof(sin));
         sin.sin_family = AF_INET;
diff --git a/net/rds/ib.h b/net/rds/ib.h
index 069206cae733..455ae73047fe 100644
--- a/net/rds/ib.h
+++ b/net/rds/ib.h
@@ -333,7 +333,7 @@ int rds_ib_xmit_rdma(struct rds_connection *conn, struct rds_rdma_op *op);
 void rds_ib_send_add_credits(struct rds_connection *conn, unsigned int credits);
 void rds_ib_advertise_credits(struct rds_connection *conn, unsigned int posted);
 int rds_ib_send_grab_credits(struct rds_ib_connection *ic, u32 wanted,
-                             u32 *adv_credits, int need_posted);
+                             u32 *adv_credits, int need_posted, int max_posted);
 
 /* ib_stats.c */
 DECLARE_PER_CPU(struct rds_ib_statistics, rds_ib_stats);
diff --git a/net/rds/ib_recv.c b/net/rds/ib_recv.c
index 36d931573ff4..5709bad28329 100644
--- a/net/rds/ib_recv.c
+++ b/net/rds/ib_recv.c
@@ -524,7 +524,7 @@ void rds_ib_attempt_ack(struct rds_ib_connection *ic)
         }
 
         /* Can we get a send credit? */
-        if (!rds_ib_send_grab_credits(ic, 1, &adv_credits, 0)) {
+        if (!rds_ib_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) {
                 rds_ib_stats_inc(s_ib_tx_throttle);
                 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
                 return;
diff --git a/net/rds/ib_ring.c b/net/rds/ib_ring.c
index 99a6ccae964c..ff97e8eda858 100644
--- a/net/rds/ib_ring.c
+++ b/net/rds/ib_ring.c
@@ -137,7 +137,7 @@ int rds_ib_ring_empty(struct rds_ib_work_ring *ring)
 
 int rds_ib_ring_low(struct rds_ib_work_ring *ring)
 {
-        return __rds_ib_ring_used(ring) <= (ring->w_nr >> 2);
+        return __rds_ib_ring_used(ring) <= (ring->w_nr >> 1);
 }
 
 /*
diff --git a/net/rds/ib_send.c b/net/rds/ib_send.c
index cb6c52cb1c4c..23bf830db2d5 100644
--- a/net/rds/ib_send.c
+++ b/net/rds/ib_send.c
@@ -311,7 +311,7 @@ void rds_ib_send_cq_comp_handler(struct ib_cq *cq, void *context)
  * and using atomic_cmpxchg when updating the two counters.
  */
 int rds_ib_send_grab_credits(struct rds_ib_connection *ic,
-                             u32 wanted, u32 *adv_credits, int need_posted)
+                             u32 wanted, u32 *adv_credits, int need_posted, int max_posted)
 {
         unsigned int avail, posted, got = 0, advertise;
         long oldval, newval;
@@ -351,7 +351,7 @@ try_again:
          * available.
          */
         if (posted && (got || need_posted)) {
-                advertise = min_t(unsigned int, posted, RDS_MAX_ADV_CREDIT);
+                advertise = min_t(unsigned int, posted, max_posted);
                 newval -= IB_SET_POST_CREDITS(advertise);
         }
 
@@ -498,7 +498,7 @@ int rds_ib_xmit(struct rds_connection *conn, struct rds_message *rm,
 
         credit_alloc = work_alloc;
         if (ic->i_flowctl) {
-                credit_alloc = rds_ib_send_grab_credits(ic, work_alloc, &posted, 0);
+                credit_alloc = rds_ib_send_grab_credits(ic, work_alloc, &posted, 0, RDS_MAX_ADV_CREDIT);
                 adv_credits += posted;
                 if (credit_alloc < work_alloc) {
                         rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - credit_alloc);
@@ -506,7 +506,7 @@ int rds_ib_xmit(struct rds_connection *conn, struct rds_message *rm,
                         flow_controlled++;
                 }
                 if (work_alloc == 0) {
-                        rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);
+                        set_bit(RDS_LL_SEND_FULL, &conn->c_flags);
                         rds_ib_stats_inc(s_ib_tx_throttle);
                         ret = -ENOMEM;
                         goto out;
@@ -571,7 +571,7 @@ int rds_ib_xmit(struct rds_connection *conn, struct rds_message *rm,
                 /*
                  * Update adv_credits since we reset the ACK_REQUIRED bit.
                  */
-                rds_ib_send_grab_credits(ic, 0, &posted, 1);
+                rds_ib_send_grab_credits(ic, 0, &posted, 1, RDS_MAX_ADV_CREDIT - adv_credits);
                 adv_credits += posted;
                 BUG_ON(adv_credits > 255);
         } else if (ic->i_rm != rm)
diff --git a/net/rds/info.c b/net/rds/info.c
index 1d885535214d..62aeef37aefe 100644
--- a/net/rds/info.c
+++ b/net/rds/info.c
@@ -188,10 +188,7 @@ int rds_info_getsockopt(struct socket *sock, int optname, char __user *optval,
                 ret = -ENOMEM;
                 goto out;
         }
-        down_read(&current->mm->mmap_sem);
-        ret = get_user_pages(current, current->mm, start, nr_pages, 1, 0,
-                             pages, NULL);
-        up_read(&current->mm->mmap_sem);
+        ret = get_user_pages_fast(start, nr_pages, 1, pages);
         if (ret != nr_pages) {
                 if (ret > 0)
                         nr_pages = ret;
diff --git a/net/rds/iw.c b/net/rds/iw.c
index b732efb5b634..d16e1cbc8e83 100644
--- a/net/rds/iw.c
+++ b/net/rds/iw.c
@@ -233,8 +233,8 @@ static int rds_iw_laddr_check(__be32 addr)
          * IB and iWARP capable NICs.
          */
         cm_id = rdma_create_id(NULL, NULL, RDMA_PS_TCP);
-        if (!cm_id)
-                return -EADDRNOTAVAIL;
+        if (IS_ERR(cm_id))
+                return PTR_ERR(cm_id);
 
         memset(&sin, 0, sizeof(sin));
         sin.sin_family = AF_INET;
diff --git a/net/rds/iw.h b/net/rds/iw.h
index b4fb27252895..0715dde323e7 100644
--- a/net/rds/iw.h
+++ b/net/rds/iw.h
@@ -361,7 +361,7 @@ int rds_iw_xmit_rdma(struct rds_connection *conn, struct rds_rdma_op *op);
 void rds_iw_send_add_credits(struct rds_connection *conn, unsigned int credits);
 void rds_iw_advertise_credits(struct rds_connection *conn, unsigned int posted);
 int rds_iw_send_grab_credits(struct rds_iw_connection *ic, u32 wanted,
-                             u32 *adv_credits, int need_posted);
+                             u32 *adv_credits, int need_posted, int max_posted);
 
 /* ib_stats.c */
 DECLARE_PER_CPU(struct rds_iw_statistics, rds_iw_stats);
diff --git a/net/rds/iw_recv.c b/net/rds/iw_recv.c
index fde470fa50d5..8683f5f66c4b 100644
--- a/net/rds/iw_recv.c
+++ b/net/rds/iw_recv.c
@@ -524,7 +524,7 @@ void rds_iw_attempt_ack(struct rds_iw_connection *ic)
         }
 
         /* Can we get a send credit? */
-        if (!rds_iw_send_grab_credits(ic, 1, &adv_credits, 0)) {
+        if (!rds_iw_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) {
                 rds_iw_stats_inc(s_iw_tx_throttle);
                 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
                 return;
diff --git a/net/rds/iw_ring.c b/net/rds/iw_ring.c
index d422d4b5deef..da8e3b63f663 100644
--- a/net/rds/iw_ring.c
+++ b/net/rds/iw_ring.c
@@ -137,7 +137,7 @@ int rds_iw_ring_empty(struct rds_iw_work_ring *ring)
 
 int rds_iw_ring_low(struct rds_iw_work_ring *ring)
 {
-        return __rds_iw_ring_used(ring) <= (ring->w_nr >> 2);
+        return __rds_iw_ring_used(ring) <= (ring->w_nr >> 1);
 }
 
 
diff --git a/net/rds/iw_send.c b/net/rds/iw_send.c
index 22dd38ffd608..44a6a0551f28 100644
--- a/net/rds/iw_send.c
+++ b/net/rds/iw_send.c
@@ -347,7 +347,7 @@ void rds_iw_send_cq_comp_handler(struct ib_cq *cq, void *context)
  * and using atomic_cmpxchg when updating the two counters.
  */
 int rds_iw_send_grab_credits(struct rds_iw_connection *ic,
-                             u32 wanted, u32 *adv_credits, int need_posted)
+                             u32 wanted, u32 *adv_credits, int need_posted, int max_posted)
 {
         unsigned int avail, posted, got = 0, advertise;
         long oldval, newval;
@@ -387,7 +387,7 @@ try_again:
          * available.
          */
         if (posted && (got || need_posted)) {
-                advertise = min_t(unsigned int, posted, RDS_MAX_ADV_CREDIT);
+                advertise = min_t(unsigned int, posted, max_posted);
                 newval -= IB_SET_POST_CREDITS(advertise);
         }
 
@@ -541,7 +541,7 @@ int rds_iw_xmit(struct rds_connection *conn, struct rds_message *rm,
 
         credit_alloc = work_alloc;
         if (ic->i_flowctl) {
-                credit_alloc = rds_iw_send_grab_credits(ic, work_alloc, &posted, 0);
+                credit_alloc = rds_iw_send_grab_credits(ic, work_alloc, &posted, 0, RDS_MAX_ADV_CREDIT);
                 adv_credits += posted;
                 if (credit_alloc < work_alloc) {
                         rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc - credit_alloc);
@@ -549,7 +549,7 @@ int rds_iw_xmit(struct rds_connection *conn, struct rds_message *rm,
                         flow_controlled++;
                 }
                 if (work_alloc == 0) {
-                        rds_iw_ring_unalloc(&ic->i_send_ring, work_alloc);
+                        set_bit(RDS_LL_SEND_FULL, &conn->c_flags);
                         rds_iw_stats_inc(s_iw_tx_throttle);
                         ret = -ENOMEM;
                         goto out;
@@ -614,7 +614,7 @@ int rds_iw_xmit(struct rds_connection *conn, struct rds_message *rm,
                 /*
                  * Update adv_credits since we reset the ACK_REQUIRED bit.
                  */
-                rds_iw_send_grab_credits(ic, 0, &posted, 1);
+                rds_iw_send_grab_credits(ic, 0, &posted, 1, RDS_MAX_ADV_CREDIT - adv_credits);
                 adv_credits += posted;
                 BUG_ON(adv_credits > 255);
         } else if (ic->i_rm != rm)
diff --git a/net/rds/rdma.c b/net/rds/rdma.c
index eaeeb91e1119..8dc83d2caa58 100644
--- a/net/rds/rdma.c
+++ b/net/rds/rdma.c
@@ -150,12 +150,9 @@ static int rds_pin_pages(unsigned long user_addr, unsigned int nr_pages,
 {
         int ret;
 
-        down_read(&current->mm->mmap_sem);
-        ret = get_user_pages(current, current->mm, user_addr,
-                             nr_pages, write, 0, pages, NULL);
-        up_read(&current->mm->mmap_sem);
+        ret = get_user_pages_fast(user_addr, nr_pages, write, pages);
 
-        if (0 <= ret && (unsigned) ret < nr_pages) {
+        if (ret >= 0 && ret < nr_pages) {
                 while (ret--)
                         put_page(pages[ret]);
                 ret = -EFAULT;
diff --git a/net/rds/rdma_transport.c b/net/rds/rdma_transport.c
index 7b19024f9706..7d0f901c93d5 100644
--- a/net/rds/rdma_transport.c
+++ b/net/rds/rdma_transport.c
@@ -34,7 +34,7 @@
 
 #include "rdma_transport.h"
 
-static struct rdma_cm_id *rds_iw_listen_id;
+static struct rdma_cm_id *rds_rdma_listen_id;
 
 int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id,
                               struct rdma_cm_event *event)
@@ -161,7 +161,7 @@ static int __init rds_rdma_listen_init(void)
 
         rdsdebug("cm %p listening on port %u\n", cm_id, RDS_PORT);
 
-        rds_iw_listen_id = cm_id;
+        rds_rdma_listen_id = cm_id;
         cm_id = NULL;
 out:
         if (cm_id)
@@ -171,10 +171,10 @@ out:
 
 static void rds_rdma_listen_stop(void)
 {
-        if (rds_iw_listen_id) {
-                rdsdebug("cm %p\n", rds_iw_listen_id);
-                rdma_destroy_id(rds_iw_listen_id);
-                rds_iw_listen_id = NULL;
+        if (rds_rdma_listen_id) {
+                rdsdebug("cm %p\n", rds_rdma_listen_id);
+                rdma_destroy_id(rds_rdma_listen_id);
+                rds_rdma_listen_id = NULL;
         }
 }
 
diff --git a/net/rds/rds.h b/net/rds/rds.h
index 619f0a30a4e5..1f82ec0d2066 100644
--- a/net/rds/rds.h
+++ b/net/rds/rds.h
@@ -132,7 +132,7 @@ struct rds_connection {
 #define RDS_FLAG_CONG_BITMAP    0x01
 #define RDS_FLAG_ACK_REQUIRED   0x02
 #define RDS_FLAG_RETRANSMITTED  0x04
-#define RDS_MAX_ADV_CREDIT      127
+#define RDS_MAX_ADV_CREDIT      255
 
 /*
  * Maximum space available for extension headers.
diff --git a/net/rds/send.c b/net/rds/send.c
index 104fe033203d..a4a7f428cd76 100644
--- a/net/rds/send.c
+++ b/net/rds/send.c
@@ -854,11 +854,6 @@ int rds_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
 
         rm->m_daddr = daddr;
 
-        /* Parse any control messages the user may have included. */
-        ret = rds_cmsg_send(rs, rm, msg, &allocated_mr);
-        if (ret)
-                goto out;
-
         /* rds_conn_create has a spinlock that runs with IRQ off.
          * Caching the conn in the socket helps a lot. */
         if (rs->rs_conn && rs->rs_conn->c_faddr == daddr)
@@ -874,6 +869,11 @@ int rds_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
                 rs->rs_conn = conn;
         }
 
+        /* Parse any control messages the user may have included. */
+        ret = rds_cmsg_send(rs, rm, msg, &allocated_mr);
+        if (ret)
+                goto out;
+
         if ((rm->m_rdma_cookie || rm->m_rdma_op)
          && conn->c_trans->xmit_rdma == NULL) {
                 if (printk_ratelimit())