aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/infiniband
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/infiniband')
-rw-r--r--drivers/infiniband/core/cma.c44
-rw-r--r--drivers/infiniband/core/uverbs.h1
-rw-r--r--drivers/infiniband/core/uverbs_cmd.c22
-rw-r--r--drivers/infiniband/core/verbs.c3
-rw-r--r--drivers/infiniband/hw/amso1100/c2_ae.c18
-rw-r--r--drivers/infiniband/hw/amso1100/c2_cm.c16
-rw-r--r--drivers/infiniband/hw/cxgb3/iwch_cm.c46
-rw-r--r--drivers/infiniband/hw/cxgb4/Kconfig2
-rw-r--r--drivers/infiniband/hw/cxgb4/cm.c860
-rw-r--r--drivers/infiniband/hw/cxgb4/cq.c329
-rw-r--r--drivers/infiniband/hw/cxgb4/device.c116
-rw-r--r--drivers/infiniband/hw/cxgb4/ev.c10
-rw-r--r--drivers/infiniband/hw/cxgb4/iw_cxgb4.h9
-rw-r--r--drivers/infiniband/hw/cxgb4/qp.c41
-rw-r--r--drivers/infiniband/hw/cxgb4/t4.h29
-rw-r--r--drivers/infiniband/hw/nes/nes_cm.c153
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma.h22
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_abi.h32
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_ah.c14
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_hw.c458
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_hw.h13
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_main.c7
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_sli.h210
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_verbs.c878
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_verbs.h6
-rw-r--r--drivers/infiniband/hw/qib/qib.h5
-rw-r--r--drivers/infiniband/hw/qib/qib_common.h32
-rw-r--r--drivers/infiniband/hw/qib/qib_file_ops.c2
-rw-r--r--drivers/infiniband/hw/qib/qib_init.c2
-rw-r--r--drivers/infiniband/hw/qib/qib_mad.h3
-rw-r--r--drivers/infiniband/hw/qib/qib_pcie.c10
-rw-r--r--drivers/infiniband/hw/qib/qib_sdma.c8
-rw-r--r--drivers/infiniband/hw/qib/qib_user_sdma.c909
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_cm.c3
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_main.c9
-rw-r--r--drivers/infiniband/ulp/iser/iscsi_iser.c19
-rw-r--r--drivers/infiniband/ulp/iser/iscsi_iser.h73
-rw-r--r--drivers/infiniband/ulp/iser/iser_initiator.c139
-rw-r--r--drivers/infiniband/ulp/iser/iser_memory.c231
-rw-r--r--drivers/infiniband/ulp/iser/iser_verbs.c292
40 files changed, 3540 insertions, 1536 deletions
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 7c0f9535fb7d..3a2c3c3bf723 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -1385,8 +1385,9 @@ static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1385{ 1385{
1386 struct rdma_id_private *id_priv = iw_id->context; 1386 struct rdma_id_private *id_priv = iw_id->context;
1387 struct rdma_cm_event event; 1387 struct rdma_cm_event event;
1388 struct sockaddr_in *sin;
1389 int ret = 0; 1388 int ret = 0;
1389 struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
1390 struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
1390 1391
1391 if (cma_disable_callback(id_priv, RDMA_CM_CONNECT)) 1392 if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
1392 return 0; 1393 return 0;
@@ -1397,10 +1398,10 @@ static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1397 event.event = RDMA_CM_EVENT_DISCONNECTED; 1398 event.event = RDMA_CM_EVENT_DISCONNECTED;
1398 break; 1399 break;
1399 case IW_CM_EVENT_CONNECT_REPLY: 1400 case IW_CM_EVENT_CONNECT_REPLY:
1400 sin = (struct sockaddr_in *) cma_src_addr(id_priv); 1401 memcpy(cma_src_addr(id_priv), laddr,
1401 *sin = iw_event->local_addr; 1402 rdma_addr_size(laddr));
1402 sin = (struct sockaddr_in *) cma_dst_addr(id_priv); 1403 memcpy(cma_dst_addr(id_priv), raddr,
1403 *sin = iw_event->remote_addr; 1404 rdma_addr_size(raddr));
1404 switch (iw_event->status) { 1405 switch (iw_event->status) {
1405 case 0: 1406 case 0:
1406 event.event = RDMA_CM_EVENT_ESTABLISHED; 1407 event.event = RDMA_CM_EVENT_ESTABLISHED;
@@ -1450,11 +1451,12 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1450{ 1451{
1451 struct rdma_cm_id *new_cm_id; 1452 struct rdma_cm_id *new_cm_id;
1452 struct rdma_id_private *listen_id, *conn_id; 1453 struct rdma_id_private *listen_id, *conn_id;
1453 struct sockaddr_in *sin;
1454 struct net_device *dev = NULL; 1454 struct net_device *dev = NULL;
1455 struct rdma_cm_event event; 1455 struct rdma_cm_event event;
1456 int ret; 1456 int ret;
1457 struct ib_device_attr attr; 1457 struct ib_device_attr attr;
1458 struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
1459 struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
1458 1460
1459 listen_id = cm_id->context; 1461 listen_id = cm_id->context;
1460 if (cma_disable_callback(listen_id, RDMA_CM_LISTEN)) 1462 if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
@@ -1472,14 +1474,7 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1472 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); 1474 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1473 conn_id->state = RDMA_CM_CONNECT; 1475 conn_id->state = RDMA_CM_CONNECT;
1474 1476
1475 dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr); 1477 ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr);
1476 if (!dev) {
1477 ret = -EADDRNOTAVAIL;
1478 mutex_unlock(&conn_id->handler_mutex);
1479 rdma_destroy_id(new_cm_id);
1480 goto out;
1481 }
1482 ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1483 if (ret) { 1478 if (ret) {
1484 mutex_unlock(&conn_id->handler_mutex); 1479 mutex_unlock(&conn_id->handler_mutex);
1485 rdma_destroy_id(new_cm_id); 1480 rdma_destroy_id(new_cm_id);
@@ -1497,10 +1492,8 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1497 cm_id->context = conn_id; 1492 cm_id->context = conn_id;
1498 cm_id->cm_handler = cma_iw_handler; 1493 cm_id->cm_handler = cma_iw_handler;
1499 1494
1500 sin = (struct sockaddr_in *) cma_src_addr(conn_id); 1495 memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr));
1501 *sin = iw_event->local_addr; 1496 memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr));
1502 sin = (struct sockaddr_in *) cma_dst_addr(conn_id);
1503 *sin = iw_event->remote_addr;
1504 1497
1505 ret = ib_query_device(conn_id->id.device, &attr); 1498 ret = ib_query_device(conn_id->id.device, &attr);
1506 if (ret) { 1499 if (ret) {
@@ -1576,7 +1569,6 @@ static int cma_ib_listen(struct rdma_id_private *id_priv)
1576static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog) 1569static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1577{ 1570{
1578 int ret; 1571 int ret;
1579 struct sockaddr_in *sin;
1580 struct iw_cm_id *id; 1572 struct iw_cm_id *id;
1581 1573
1582 id = iw_create_cm_id(id_priv->id.device, 1574 id = iw_create_cm_id(id_priv->id.device,
@@ -1587,8 +1579,8 @@ static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1587 1579
1588 id_priv->cm_id.iw = id; 1580 id_priv->cm_id.iw = id;
1589 1581
1590 sin = (struct sockaddr_in *) cma_src_addr(id_priv); 1582 memcpy(&id_priv->cm_id.iw->local_addr, cma_src_addr(id_priv),
1591 id_priv->cm_id.iw->local_addr = *sin; 1583 rdma_addr_size(cma_src_addr(id_priv)));
1592 1584
1593 ret = iw_cm_listen(id_priv->cm_id.iw, backlog); 1585 ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1594 1586
@@ -2803,7 +2795,6 @@ static int cma_connect_iw(struct rdma_id_private *id_priv,
2803 struct rdma_conn_param *conn_param) 2795 struct rdma_conn_param *conn_param)
2804{ 2796{
2805 struct iw_cm_id *cm_id; 2797 struct iw_cm_id *cm_id;
2806 struct sockaddr_in* sin;
2807 int ret; 2798 int ret;
2808 struct iw_cm_conn_param iw_param; 2799 struct iw_cm_conn_param iw_param;
2809 2800
@@ -2813,11 +2804,10 @@ static int cma_connect_iw(struct rdma_id_private *id_priv,
2813 2804
2814 id_priv->cm_id.iw = cm_id; 2805 id_priv->cm_id.iw = cm_id;
2815 2806
2816 sin = (struct sockaddr_in *) cma_src_addr(id_priv); 2807 memcpy(&cm_id->local_addr, cma_src_addr(id_priv),
2817 cm_id->local_addr = *sin; 2808 rdma_addr_size(cma_src_addr(id_priv)));
2818 2809 memcpy(&cm_id->remote_addr, cma_dst_addr(id_priv),
2819 sin = (struct sockaddr_in *) cma_dst_addr(id_priv); 2810 rdma_addr_size(cma_dst_addr(id_priv)));
2820 cm_id->remote_addr = *sin;
2821 2811
2822 ret = cma_modify_qp_rtr(id_priv, conn_param); 2812 ret = cma_modify_qp_rtr(id_priv, conn_param);
2823 if (ret) 2813 if (ret)
diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
index ad9d1028102d..d040b877475f 100644
--- a/drivers/infiniband/core/uverbs.h
+++ b/drivers/infiniband/core/uverbs.h
@@ -135,6 +135,7 @@ struct ib_usrq_object {
135struct ib_uqp_object { 135struct ib_uqp_object {
136 struct ib_uevent_object uevent; 136 struct ib_uevent_object uevent;
137 struct list_head mcast_list; 137 struct list_head mcast_list;
138 struct ib_uxrcd_object *uxrcd;
138}; 139};
139 140
140struct ib_ucq_object { 141struct ib_ucq_object {
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 9112410da11d..f2b81b9ee0d6 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -1528,7 +1528,7 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
1528 (unsigned long) cmd.response + sizeof resp, 1528 (unsigned long) cmd.response + sizeof resp,
1529 in_len - sizeof cmd, out_len - sizeof resp); 1529 in_len - sizeof cmd, out_len - sizeof resp);
1530 1530
1531 obj = kmalloc(sizeof *obj, GFP_KERNEL); 1531 obj = kzalloc(sizeof *obj, GFP_KERNEL);
1532 if (!obj) 1532 if (!obj)
1533 return -ENOMEM; 1533 return -ENOMEM;
1534 1534
@@ -1644,8 +1644,13 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
1644 goto err_copy; 1644 goto err_copy;
1645 } 1645 }
1646 1646
1647 if (xrcd) 1647 if (xrcd) {
1648 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object,
1649 uobject);
1650 atomic_inc(&obj->uxrcd->refcnt);
1648 put_xrcd_read(xrcd_uobj); 1651 put_xrcd_read(xrcd_uobj);
1652 }
1653
1649 if (pd) 1654 if (pd)
1650 put_pd_read(pd); 1655 put_pd_read(pd);
1651 if (scq) 1656 if (scq)
@@ -1755,6 +1760,8 @@ ssize_t ib_uverbs_open_qp(struct ib_uverbs_file *file,
1755 goto err_remove; 1760 goto err_remove;
1756 } 1761 }
1757 1762
1763 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject);
1764 atomic_inc(&obj->uxrcd->refcnt);
1758 put_xrcd_read(xrcd_uobj); 1765 put_xrcd_read(xrcd_uobj);
1759 1766
1760 mutex_lock(&file->mutex); 1767 mutex_lock(&file->mutex);
@@ -2021,6 +2028,9 @@ ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file,
2021 if (ret) 2028 if (ret)
2022 return ret; 2029 return ret;
2023 2030
2031 if (obj->uxrcd)
2032 atomic_dec(&obj->uxrcd->refcnt);
2033
2024 idr_remove_uobj(&ib_uverbs_qp_idr, uobj); 2034 idr_remove_uobj(&ib_uverbs_qp_idr, uobj);
2025 2035
2026 mutex_lock(&file->mutex); 2036 mutex_lock(&file->mutex);
@@ -3088,6 +3098,8 @@ ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file,
3088 struct ib_srq *srq; 3098 struct ib_srq *srq;
3089 struct ib_uevent_object *obj; 3099 struct ib_uevent_object *obj;
3090 int ret = -EINVAL; 3100 int ret = -EINVAL;
3101 struct ib_usrq_object *us;
3102 enum ib_srq_type srq_type;
3091 3103
3092 if (copy_from_user(&cmd, buf, sizeof cmd)) 3104 if (copy_from_user(&cmd, buf, sizeof cmd))
3093 return -EFAULT; 3105 return -EFAULT;
@@ -3097,6 +3109,7 @@ ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file,
3097 return -EINVAL; 3109 return -EINVAL;
3098 srq = uobj->object; 3110 srq = uobj->object;
3099 obj = container_of(uobj, struct ib_uevent_object, uobject); 3111 obj = container_of(uobj, struct ib_uevent_object, uobject);
3112 srq_type = srq->srq_type;
3100 3113
3101 ret = ib_destroy_srq(srq); 3114 ret = ib_destroy_srq(srq);
3102 if (!ret) 3115 if (!ret)
@@ -3107,6 +3120,11 @@ ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file,
3107 if (ret) 3120 if (ret)
3108 return ret; 3121 return ret;
3109 3122
3123 if (srq_type == IB_SRQT_XRC) {
3124 us = container_of(obj, struct ib_usrq_object, uevent);
3125 atomic_dec(&us->uxrcd->refcnt);
3126 }
3127
3110 idr_remove_uobj(&ib_uverbs_srq_idr, uobj); 3128 idr_remove_uobj(&ib_uverbs_srq_idr, uobj);
3111 3129
3112 mutex_lock(&file->mutex); 3130 mutex_lock(&file->mutex);
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 87a8102c9dbe..a321df28bab2 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -346,10 +346,13 @@ EXPORT_SYMBOL(ib_destroy_srq);
346static void __ib_shared_qp_event_handler(struct ib_event *event, void *context) 346static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
347{ 347{
348 struct ib_qp *qp = context; 348 struct ib_qp *qp = context;
349 unsigned long flags;
349 350
351 spin_lock_irqsave(&qp->device->event_handler_lock, flags);
350 list_for_each_entry(event->element.qp, &qp->open_list, open_list) 352 list_for_each_entry(event->element.qp, &qp->open_list, open_list)
351 if (event->element.qp->event_handler) 353 if (event->element.qp->event_handler)
352 event->element.qp->event_handler(event, event->element.qp->qp_context); 354 event->element.qp->event_handler(event, event->element.qp->qp_context);
355 spin_unlock_irqrestore(&qp->device->event_handler_lock, flags);
353} 356}
354 357
355static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp) 358static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
diff --git a/drivers/infiniband/hw/amso1100/c2_ae.c b/drivers/infiniband/hw/amso1100/c2_ae.c
index 706cf97cbe8f..d5d1929753e4 100644
--- a/drivers/infiniband/hw/amso1100/c2_ae.c
+++ b/drivers/infiniband/hw/amso1100/c2_ae.c
@@ -155,6 +155,8 @@ void c2_ae_event(struct c2_dev *c2dev, u32 mq_index)
155 enum c2_event_id event_id; 155 enum c2_event_id event_id;
156 unsigned long flags; 156 unsigned long flags;
157 int status; 157 int status;
158 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_event.local_addr;
159 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_event.remote_addr;
158 160
159 /* 161 /*
160 * retrieve the message 162 * retrieve the message
@@ -206,10 +208,10 @@ void c2_ae_event(struct c2_dev *c2dev, u32 mq_index)
206 case CCAE_ACTIVE_CONNECT_RESULTS: 208 case CCAE_ACTIVE_CONNECT_RESULTS:
207 res = &wr->ae.ae_active_connect_results; 209 res = &wr->ae.ae_active_connect_results;
208 cm_event.event = IW_CM_EVENT_CONNECT_REPLY; 210 cm_event.event = IW_CM_EVENT_CONNECT_REPLY;
209 cm_event.local_addr.sin_addr.s_addr = res->laddr; 211 laddr->sin_addr.s_addr = res->laddr;
210 cm_event.remote_addr.sin_addr.s_addr = res->raddr; 212 raddr->sin_addr.s_addr = res->raddr;
211 cm_event.local_addr.sin_port = res->lport; 213 laddr->sin_port = res->lport;
212 cm_event.remote_addr.sin_port = res->rport; 214 raddr->sin_port = res->rport;
213 if (status == 0) { 215 if (status == 0) {
214 cm_event.private_data_len = 216 cm_event.private_data_len =
215 be32_to_cpu(res->private_data_length); 217 be32_to_cpu(res->private_data_length);
@@ -281,10 +283,10 @@ void c2_ae_event(struct c2_dev *c2dev, u32 mq_index)
281 } 283 }
282 cm_event.event = IW_CM_EVENT_CONNECT_REQUEST; 284 cm_event.event = IW_CM_EVENT_CONNECT_REQUEST;
283 cm_event.provider_data = (void*)(unsigned long)req->cr_handle; 285 cm_event.provider_data = (void*)(unsigned long)req->cr_handle;
284 cm_event.local_addr.sin_addr.s_addr = req->laddr; 286 laddr->sin_addr.s_addr = req->laddr;
285 cm_event.remote_addr.sin_addr.s_addr = req->raddr; 287 raddr->sin_addr.s_addr = req->raddr;
286 cm_event.local_addr.sin_port = req->lport; 288 laddr->sin_port = req->lport;
287 cm_event.remote_addr.sin_port = req->rport; 289 raddr->sin_port = req->rport;
288 cm_event.private_data_len = 290 cm_event.private_data_len =
289 be32_to_cpu(req->private_data_length); 291 be32_to_cpu(req->private_data_length);
290 cm_event.private_data = req->private_data; 292 cm_event.private_data = req->private_data;
diff --git a/drivers/infiniband/hw/amso1100/c2_cm.c b/drivers/infiniband/hw/amso1100/c2_cm.c
index 95f58ab1e0b8..23bfa94fbd4e 100644
--- a/drivers/infiniband/hw/amso1100/c2_cm.c
+++ b/drivers/infiniband/hw/amso1100/c2_cm.c
@@ -46,6 +46,10 @@ int c2_llp_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param)
46 struct c2wr_qp_connect_req *wr; /* variable size needs a malloc. */ 46 struct c2wr_qp_connect_req *wr; /* variable size needs a malloc. */
47 struct c2_vq_req *vq_req; 47 struct c2_vq_req *vq_req;
48 int err; 48 int err;
49 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr;
50
51 if (cm_id->remote_addr.ss_family != AF_INET)
52 return -ENOSYS;
49 53
50 ibqp = c2_get_qp(cm_id->device, iw_param->qpn); 54 ibqp = c2_get_qp(cm_id->device, iw_param->qpn);
51 if (!ibqp) 55 if (!ibqp)
@@ -91,8 +95,8 @@ int c2_llp_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param)
91 wr->rnic_handle = c2dev->adapter_handle; 95 wr->rnic_handle = c2dev->adapter_handle;
92 wr->qp_handle = qp->adapter_handle; 96 wr->qp_handle = qp->adapter_handle;
93 97
94 wr->remote_addr = cm_id->remote_addr.sin_addr.s_addr; 98 wr->remote_addr = raddr->sin_addr.s_addr;
95 wr->remote_port = cm_id->remote_addr.sin_port; 99 wr->remote_port = raddr->sin_port;
96 100
97 /* 101 /*
98 * Move any private data from the callers's buf into 102 * Move any private data from the callers's buf into
@@ -135,6 +139,10 @@ int c2_llp_service_create(struct iw_cm_id *cm_id, int backlog)
135 struct c2wr_ep_listen_create_rep *reply; 139 struct c2wr_ep_listen_create_rep *reply;
136 struct c2_vq_req *vq_req; 140 struct c2_vq_req *vq_req;
137 int err; 141 int err;
142 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr;
143
144 if (cm_id->local_addr.ss_family != AF_INET)
145 return -ENOSYS;
138 146
139 c2dev = to_c2dev(cm_id->device); 147 c2dev = to_c2dev(cm_id->device);
140 if (c2dev == NULL) 148 if (c2dev == NULL)
@@ -153,8 +161,8 @@ int c2_llp_service_create(struct iw_cm_id *cm_id, int backlog)
153 c2_wr_set_id(&wr, CCWR_EP_LISTEN_CREATE); 161 c2_wr_set_id(&wr, CCWR_EP_LISTEN_CREATE);
154 wr.hdr.context = (u64) (unsigned long) vq_req; 162 wr.hdr.context = (u64) (unsigned long) vq_req;
155 wr.rnic_handle = c2dev->adapter_handle; 163 wr.rnic_handle = c2dev->adapter_handle;
156 wr.local_addr = cm_id->local_addr.sin_addr.s_addr; 164 wr.local_addr = laddr->sin_addr.s_addr;
157 wr.local_port = cm_id->local_addr.sin_port; 165 wr.local_port = laddr->sin_port;
158 wr.backlog = cpu_to_be32(backlog); 166 wr.backlog = cpu_to_be32(backlog);
159 wr.user_context = (u64) (unsigned long) cm_id; 167 wr.user_context = (u64) (unsigned long) cm_id;
160 168
diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c
index 3e094cd6a0e3..095bb046e2c8 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_cm.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_cm.c
@@ -721,8 +721,10 @@ static void connect_reply_upcall(struct iwch_ep *ep, int status)
721 memset(&event, 0, sizeof(event)); 721 memset(&event, 0, sizeof(event));
722 event.event = IW_CM_EVENT_CONNECT_REPLY; 722 event.event = IW_CM_EVENT_CONNECT_REPLY;
723 event.status = status; 723 event.status = status;
724 event.local_addr = ep->com.local_addr; 724 memcpy(&event.local_addr, &ep->com.local_addr,
725 event.remote_addr = ep->com.remote_addr; 725 sizeof(ep->com.local_addr));
726 memcpy(&event.remote_addr, &ep->com.remote_addr,
727 sizeof(ep->com.remote_addr));
726 728
727 if ((status == 0) || (status == -ECONNREFUSED)) { 729 if ((status == 0) || (status == -ECONNREFUSED)) {
728 event.private_data_len = ep->plen; 730 event.private_data_len = ep->plen;
@@ -747,8 +749,10 @@ static void connect_request_upcall(struct iwch_ep *ep)
747 PDBG("%s ep %p tid %d\n", __func__, ep, ep->hwtid); 749 PDBG("%s ep %p tid %d\n", __func__, ep, ep->hwtid);
748 memset(&event, 0, sizeof(event)); 750 memset(&event, 0, sizeof(event));
749 event.event = IW_CM_EVENT_CONNECT_REQUEST; 751 event.event = IW_CM_EVENT_CONNECT_REQUEST;
750 event.local_addr = ep->com.local_addr; 752 memcpy(&event.local_addr, &ep->com.local_addr,
751 event.remote_addr = ep->com.remote_addr; 753 sizeof(ep->com.local_addr));
754 memcpy(&event.remote_addr, &ep->com.remote_addr,
755 sizeof(ep->com.local_addr));
752 event.private_data_len = ep->plen; 756 event.private_data_len = ep->plen;
753 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 757 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
754 event.provider_data = ep; 758 event.provider_data = ep;
@@ -1872,8 +1876,9 @@ err:
1872static int is_loopback_dst(struct iw_cm_id *cm_id) 1876static int is_loopback_dst(struct iw_cm_id *cm_id)
1873{ 1877{
1874 struct net_device *dev; 1878 struct net_device *dev;
1879 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr;
1875 1880
1876 dev = ip_dev_find(&init_net, cm_id->remote_addr.sin_addr.s_addr); 1881 dev = ip_dev_find(&init_net, raddr->sin_addr.s_addr);
1877 if (!dev) 1882 if (!dev)
1878 return 0; 1883 return 0;
1879 dev_put(dev); 1884 dev_put(dev);
@@ -1886,6 +1891,13 @@ int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
1886 struct iwch_ep *ep; 1891 struct iwch_ep *ep;
1887 struct rtable *rt; 1892 struct rtable *rt;
1888 int err = 0; 1893 int err = 0;
1894 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr;
1895 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr;
1896
1897 if (cm_id->remote_addr.ss_family != PF_INET) {
1898 err = -ENOSYS;
1899 goto out;
1900 }
1889 1901
1890 if (is_loopback_dst(cm_id)) { 1902 if (is_loopback_dst(cm_id)) {
1891 err = -ENOSYS; 1903 err = -ENOSYS;
@@ -1929,11 +1941,9 @@ int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
1929 } 1941 }
1930 1942
1931 /* find a route */ 1943 /* find a route */
1932 rt = find_route(h->rdev.t3cdev_p, 1944 rt = find_route(h->rdev.t3cdev_p, laddr->sin_addr.s_addr,
1933 cm_id->local_addr.sin_addr.s_addr, 1945 raddr->sin_addr.s_addr, laddr->sin_port,
1934 cm_id->remote_addr.sin_addr.s_addr, 1946 raddr->sin_port, IPTOS_LOWDELAY);
1935 cm_id->local_addr.sin_port,
1936 cm_id->remote_addr.sin_port, IPTOS_LOWDELAY);
1937 if (!rt) { 1947 if (!rt) {
1938 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 1948 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__);
1939 err = -EHOSTUNREACH; 1949 err = -EHOSTUNREACH;
@@ -1941,7 +1951,7 @@ int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
1941 } 1951 }
1942 ep->dst = &rt->dst; 1952 ep->dst = &rt->dst;
1943 ep->l2t = t3_l2t_get(ep->com.tdev, ep->dst, NULL, 1953 ep->l2t = t3_l2t_get(ep->com.tdev, ep->dst, NULL,
1944 &cm_id->remote_addr.sin_addr.s_addr); 1954 &raddr->sin_addr.s_addr);
1945 if (!ep->l2t) { 1955 if (!ep->l2t) {
1946 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__); 1956 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__);
1947 err = -ENOMEM; 1957 err = -ENOMEM;
@@ -1950,8 +1960,10 @@ int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
1950 1960
1951 state_set(&ep->com, CONNECTING); 1961 state_set(&ep->com, CONNECTING);
1952 ep->tos = IPTOS_LOWDELAY; 1962 ep->tos = IPTOS_LOWDELAY;
1953 ep->com.local_addr = cm_id->local_addr; 1963 memcpy(&ep->com.local_addr, &cm_id->local_addr,
1954 ep->com.remote_addr = cm_id->remote_addr; 1964 sizeof(ep->com.local_addr));
1965 memcpy(&ep->com.remote_addr, &cm_id->remote_addr,
1966 sizeof(ep->com.remote_addr));
1955 1967
1956 /* send connect request to rnic */ 1968 /* send connect request to rnic */
1957 err = send_connect(ep); 1969 err = send_connect(ep);
@@ -1979,6 +1991,11 @@ int iwch_create_listen(struct iw_cm_id *cm_id, int backlog)
1979 1991
1980 might_sleep(); 1992 might_sleep();
1981 1993
1994 if (cm_id->local_addr.ss_family != PF_INET) {
1995 err = -ENOSYS;
1996 goto fail1;
1997 }
1998
1982 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 1999 ep = alloc_ep(sizeof(*ep), GFP_KERNEL);
1983 if (!ep) { 2000 if (!ep) {
1984 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2001 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__);
@@ -1990,7 +2007,8 @@ int iwch_create_listen(struct iw_cm_id *cm_id, int backlog)
1990 cm_id->add_ref(cm_id); 2007 cm_id->add_ref(cm_id);
1991 ep->com.cm_id = cm_id; 2008 ep->com.cm_id = cm_id;
1992 ep->backlog = backlog; 2009 ep->backlog = backlog;
1993 ep->com.local_addr = cm_id->local_addr; 2010 memcpy(&ep->com.local_addr, &cm_id->local_addr,
2011 sizeof(ep->com.local_addr));
1994 2012
1995 /* 2013 /*
1996 * Allocate a server TID. 2014 * Allocate a server TID.
diff --git a/drivers/infiniband/hw/cxgb4/Kconfig b/drivers/infiniband/hw/cxgb4/Kconfig
index 6b7e6c543534..d4e8983fba53 100644
--- a/drivers/infiniband/hw/cxgb4/Kconfig
+++ b/drivers/infiniband/hw/cxgb4/Kconfig
@@ -1,6 +1,6 @@
1config INFINIBAND_CXGB4 1config INFINIBAND_CXGB4
2 tristate "Chelsio T4 RDMA Driver" 2 tristate "Chelsio T4 RDMA Driver"
3 depends on CHELSIO_T4 && INET 3 depends on CHELSIO_T4 && INET && (IPV6 || IPV6=n)
4 select GENERIC_ALLOCATOR 4 select GENERIC_ALLOCATOR
5 ---help--- 5 ---help---
6 This is an iWARP/RDMA driver for the Chelsio T4 1GbE and 6 This is an iWARP/RDMA driver for the Chelsio T4 1GbE and
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index 65c30ea8c1a1..12fef76c791c 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -44,6 +44,8 @@
44#include <net/netevent.h> 44#include <net/netevent.h>
45#include <net/route.h> 45#include <net/route.h>
46#include <net/tcp.h> 46#include <net/tcp.h>
47#include <net/ip6_route.h>
48#include <net/addrconf.h>
47 49
48#include "iw_cxgb4.h" 50#include "iw_cxgb4.h"
49 51
@@ -330,22 +332,80 @@ static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp)
330 } else { 332 } else {
331 skb = alloc_skb(len, gfp); 333 skb = alloc_skb(len, gfp);
332 } 334 }
335 t4_set_arp_err_handler(skb, NULL, NULL);
333 return skb; 336 return skb;
334} 337}
335 338
336static struct rtable *find_route(struct c4iw_dev *dev, __be32 local_ip, 339static struct net_device *get_real_dev(struct net_device *egress_dev)
340{
341 struct net_device *phys_dev = egress_dev;
342 if (egress_dev->priv_flags & IFF_802_1Q_VLAN)
343 phys_dev = vlan_dev_real_dev(egress_dev);
344 return phys_dev;
345}
346
347static int our_interface(struct c4iw_dev *dev, struct net_device *egress_dev)
348{
349 int i;
350
351 egress_dev = get_real_dev(egress_dev);
352 for (i = 0; i < dev->rdev.lldi.nports; i++)
353 if (dev->rdev.lldi.ports[i] == egress_dev)
354 return 1;
355 return 0;
356}
357
358static struct dst_entry *find_route6(struct c4iw_dev *dev, __u8 *local_ip,
359 __u8 *peer_ip, __be16 local_port,
360 __be16 peer_port, u8 tos,
361 __u32 sin6_scope_id)
362{
363 struct dst_entry *dst = NULL;
364
365 if (IS_ENABLED(CONFIG_IPV6)) {
366 struct flowi6 fl6;
367
368 memset(&fl6, 0, sizeof(fl6));
369 memcpy(&fl6.daddr, peer_ip, 16);
370 memcpy(&fl6.saddr, local_ip, 16);
371 if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL)
372 fl6.flowi6_oif = sin6_scope_id;
373 dst = ip6_route_output(&init_net, NULL, &fl6);
374 if (!dst)
375 goto out;
376 if (!our_interface(dev, ip6_dst_idev(dst)->dev) &&
377 !(ip6_dst_idev(dst)->dev->flags & IFF_LOOPBACK)) {
378 dst_release(dst);
379 dst = NULL;
380 }
381 }
382
383out:
384 return dst;
385}
386
387static struct dst_entry *find_route(struct c4iw_dev *dev, __be32 local_ip,
337 __be32 peer_ip, __be16 local_port, 388 __be32 peer_ip, __be16 local_port,
338 __be16 peer_port, u8 tos) 389 __be16 peer_port, u8 tos)
339{ 390{
340 struct rtable *rt; 391 struct rtable *rt;
341 struct flowi4 fl4; 392 struct flowi4 fl4;
393 struct neighbour *n;
342 394
343 rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip, 395 rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip,
344 peer_port, local_port, IPPROTO_TCP, 396 peer_port, local_port, IPPROTO_TCP,
345 tos, 0); 397 tos, 0);
346 if (IS_ERR(rt)) 398 if (IS_ERR(rt))
347 return NULL; 399 return NULL;
348 return rt; 400 n = dst_neigh_lookup(&rt->dst, &peer_ip);
401 if (!n)
402 return NULL;
403 if (!our_interface(dev, n->dev)) {
404 dst_release(&rt->dst);
405 return NULL;
406 }
407 neigh_release(n);
408 return &rt->dst;
349} 409}
350 410
351static void arp_failure_discard(void *handle, struct sk_buff *skb) 411static void arp_failure_discard(void *handle, struct sk_buff *skb)
@@ -487,7 +547,7 @@ static unsigned int select_ntuple(struct c4iw_dev *dev, struct dst_entry *dst,
487 ntuple |= FILTER_SEL_VLAN_NONE << FILTER_SEL_WIDTH_P_FC; 547 ntuple |= FILTER_SEL_VLAN_NONE << FILTER_SEL_WIDTH_P_FC;
488 else { 548 else {
489 ntuple |= l2t->vlan << FILTER_SEL_WIDTH_P_FC; 549 ntuple |= l2t->vlan << FILTER_SEL_WIDTH_P_FC;
490 ntuple |= 1 << FILTER_SEL_WIDTH_VLD_TAG_P_FC; 550 ntuple |= 1 << FILTER_SEL_WIDTH_TAG_P_FC;
491 } 551 }
492 ntuple |= l2t->lport << S_PORT | IPPROTO_TCP << 552 ntuple |= l2t->lport << S_PORT | IPPROTO_TCP <<
493 FILTER_SEL_WIDTH_VLD_TAG_P_FC; 553 FILTER_SEL_WIDTH_VLD_TAG_P_FC;
@@ -512,15 +572,28 @@ static int send_connect(struct c4iw_ep *ep)
512{ 572{
513 struct cpl_act_open_req *req; 573 struct cpl_act_open_req *req;
514 struct cpl_t5_act_open_req *t5_req; 574 struct cpl_t5_act_open_req *t5_req;
575 struct cpl_act_open_req6 *req6;
576 struct cpl_t5_act_open_req6 *t5_req6;
515 struct sk_buff *skb; 577 struct sk_buff *skb;
516 u64 opt0; 578 u64 opt0;
517 u32 opt2; 579 u32 opt2;
518 unsigned int mtu_idx; 580 unsigned int mtu_idx;
519 int wscale; 581 int wscale;
520 int size = is_t4(ep->com.dev->rdev.lldi.adapter_type) ? 582 int wrlen;
521 sizeof(struct cpl_act_open_req) : 583 int sizev4 = is_t4(ep->com.dev->rdev.lldi.adapter_type) ?
522 sizeof(struct cpl_t5_act_open_req); 584 sizeof(struct cpl_act_open_req) :
523 int wrlen = roundup(size, 16); 585 sizeof(struct cpl_t5_act_open_req);
586 int sizev6 = is_t4(ep->com.dev->rdev.lldi.adapter_type) ?
587 sizeof(struct cpl_act_open_req6) :
588 sizeof(struct cpl_t5_act_open_req6);
589 struct sockaddr_in *la = (struct sockaddr_in *)&ep->com.local_addr;
590 struct sockaddr_in *ra = (struct sockaddr_in *)&ep->com.remote_addr;
591 struct sockaddr_in6 *la6 = (struct sockaddr_in6 *)&ep->com.local_addr;
592 struct sockaddr_in6 *ra6 = (struct sockaddr_in6 *)&ep->com.remote_addr;
593
594 wrlen = (ep->com.remote_addr.ss_family == AF_INET) ?
595 roundup(sizev4, 16) :
596 roundup(sizev6, 16);
524 597
525 PDBG("%s ep %p atid %u\n", __func__, ep, ep->atid); 598 PDBG("%s ep %p atid %u\n", __func__, ep, ep->atid);
526 599
@@ -557,33 +630,82 @@ static int send_connect(struct c4iw_ep *ep)
557 t4_set_arp_err_handler(skb, NULL, act_open_req_arp_failure); 630 t4_set_arp_err_handler(skb, NULL, act_open_req_arp_failure);
558 631
559 if (is_t4(ep->com.dev->rdev.lldi.adapter_type)) { 632 if (is_t4(ep->com.dev->rdev.lldi.adapter_type)) {
560 req = (struct cpl_act_open_req *) skb_put(skb, wrlen); 633 if (ep->com.remote_addr.ss_family == AF_INET) {
561 INIT_TP_WR(req, 0); 634 req = (struct cpl_act_open_req *) skb_put(skb, wrlen);
562 OPCODE_TID(req) = cpu_to_be32( 635 INIT_TP_WR(req, 0);
563 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 636 OPCODE_TID(req) = cpu_to_be32(
564 ((ep->rss_qid << 14) | ep->atid))); 637 MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
565 req->local_port = ep->com.local_addr.sin_port; 638 ((ep->rss_qid << 14) | ep->atid)));
566 req->peer_port = ep->com.remote_addr.sin_port; 639 req->local_port = la->sin_port;
567 req->local_ip = ep->com.local_addr.sin_addr.s_addr; 640 req->peer_port = ra->sin_port;
568 req->peer_ip = ep->com.remote_addr.sin_addr.s_addr; 641 req->local_ip = la->sin_addr.s_addr;
569 req->opt0 = cpu_to_be64(opt0); 642 req->peer_ip = ra->sin_addr.s_addr;
570 req->params = cpu_to_be32(select_ntuple(ep->com.dev, 643 req->opt0 = cpu_to_be64(opt0);
571 ep->dst, ep->l2t)); 644 req->params = cpu_to_be32(select_ntuple(ep->com.dev,
572 req->opt2 = cpu_to_be32(opt2); 645 ep->dst, ep->l2t));
646 req->opt2 = cpu_to_be32(opt2);
647 } else {
648 req6 = (struct cpl_act_open_req6 *)skb_put(skb, wrlen);
649
650 INIT_TP_WR(req6, 0);
651 OPCODE_TID(req6) = cpu_to_be32(
652 MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
653 ((ep->rss_qid<<14)|ep->atid)));
654 req6->local_port = la6->sin6_port;
655 req6->peer_port = ra6->sin6_port;
656 req6->local_ip_hi = *((__be64 *)
657 (la6->sin6_addr.s6_addr));
658 req6->local_ip_lo = *((__be64 *)
659 (la6->sin6_addr.s6_addr + 8));
660 req6->peer_ip_hi = *((__be64 *)
661 (ra6->sin6_addr.s6_addr));
662 req6->peer_ip_lo = *((__be64 *)
663 (ra6->sin6_addr.s6_addr + 8));
664 req6->opt0 = cpu_to_be64(opt0);
665 req6->params = cpu_to_be32(
666 select_ntuple(ep->com.dev, ep->dst,
667 ep->l2t));
668 req6->opt2 = cpu_to_be32(opt2);
669 }
573 } else { 670 } else {
574 t5_req = (struct cpl_t5_act_open_req *) skb_put(skb, wrlen); 671 if (ep->com.remote_addr.ss_family == AF_INET) {
575 INIT_TP_WR(t5_req, 0); 672 t5_req = (struct cpl_t5_act_open_req *)
576 OPCODE_TID(t5_req) = cpu_to_be32( 673 skb_put(skb, wrlen);
674 INIT_TP_WR(t5_req, 0);
675 OPCODE_TID(t5_req) = cpu_to_be32(
577 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 676 MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
578 ((ep->rss_qid << 14) | ep->atid))); 677 ((ep->rss_qid << 14) | ep->atid)));
579 t5_req->local_port = ep->com.local_addr.sin_port; 678 t5_req->local_port = la->sin_port;
580 t5_req->peer_port = ep->com.remote_addr.sin_port; 679 t5_req->peer_port = ra->sin_port;
581 t5_req->local_ip = ep->com.local_addr.sin_addr.s_addr; 680 t5_req->local_ip = la->sin_addr.s_addr;
582 t5_req->peer_ip = ep->com.remote_addr.sin_addr.s_addr; 681 t5_req->peer_ip = ra->sin_addr.s_addr;
583 t5_req->opt0 = cpu_to_be64(opt0); 682 t5_req->opt0 = cpu_to_be64(opt0);
584 t5_req->params = cpu_to_be64(V_FILTER_TUPLE( 683 t5_req->params = cpu_to_be64(V_FILTER_TUPLE(
585 select_ntuple(ep->com.dev, ep->dst, ep->l2t))); 684 select_ntuple(ep->com.dev,
586 t5_req->opt2 = cpu_to_be32(opt2); 685 ep->dst, ep->l2t)));
686 t5_req->opt2 = cpu_to_be32(opt2);
687 } else {
688 t5_req6 = (struct cpl_t5_act_open_req6 *)
689 skb_put(skb, wrlen);
690 INIT_TP_WR(t5_req6, 0);
691 OPCODE_TID(t5_req6) = cpu_to_be32(
692 MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
693 ((ep->rss_qid<<14)|ep->atid)));
694 t5_req6->local_port = la6->sin6_port;
695 t5_req6->peer_port = ra6->sin6_port;
696 t5_req6->local_ip_hi = *((__be64 *)
697 (la6->sin6_addr.s6_addr));
698 t5_req6->local_ip_lo = *((__be64 *)
699 (la6->sin6_addr.s6_addr + 8));
700 t5_req6->peer_ip_hi = *((__be64 *)
701 (ra6->sin6_addr.s6_addr));
702 t5_req6->peer_ip_lo = *((__be64 *)
703 (ra6->sin6_addr.s6_addr + 8));
704 t5_req6->opt0 = cpu_to_be64(opt0);
705 t5_req6->params = (__force __be64)cpu_to_be32(
706 select_ntuple(ep->com.dev, ep->dst, ep->l2t));
707 t5_req6->opt2 = cpu_to_be32(opt2);
708 }
587 } 709 }
588 710
589 set_bit(ACT_OPEN_REQ, &ep->com.history); 711 set_bit(ACT_OPEN_REQ, &ep->com.history);
@@ -952,8 +1074,10 @@ static void connect_reply_upcall(struct c4iw_ep *ep, int status)
952 memset(&event, 0, sizeof(event)); 1074 memset(&event, 0, sizeof(event));
953 event.event = IW_CM_EVENT_CONNECT_REPLY; 1075 event.event = IW_CM_EVENT_CONNECT_REPLY;
954 event.status = status; 1076 event.status = status;
955 event.local_addr = ep->com.local_addr; 1077 memcpy(&event.local_addr, &ep->com.local_addr,
956 event.remote_addr = ep->com.remote_addr; 1078 sizeof(ep->com.local_addr));
1079 memcpy(&event.remote_addr, &ep->com.remote_addr,
1080 sizeof(ep->com.remote_addr));
957 1081
958 if ((status == 0) || (status == -ECONNREFUSED)) { 1082 if ((status == 0) || (status == -ECONNREFUSED)) {
959 if (!ep->tried_with_mpa_v1) { 1083 if (!ep->tried_with_mpa_v1) {
@@ -989,8 +1113,10 @@ static void connect_request_upcall(struct c4iw_ep *ep)
989 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1113 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
990 memset(&event, 0, sizeof(event)); 1114 memset(&event, 0, sizeof(event));
991 event.event = IW_CM_EVENT_CONNECT_REQUEST; 1115 event.event = IW_CM_EVENT_CONNECT_REQUEST;
992 event.local_addr = ep->com.local_addr; 1116 memcpy(&event.local_addr, &ep->com.local_addr,
993 event.remote_addr = ep->com.remote_addr; 1117 sizeof(ep->com.local_addr));
1118 memcpy(&event.remote_addr, &ep->com.remote_addr,
1119 sizeof(ep->com.remote_addr));
994 event.provider_data = ep; 1120 event.provider_data = ep;
995 if (!ep->tried_with_mpa_v1) { 1121 if (!ep->tried_with_mpa_v1) {
996 /* this means MPA_v2 is used */ 1122 /* this means MPA_v2 is used */
@@ -1447,10 +1573,9 @@ static int rx_data(struct c4iw_dev *dev, struct sk_buff *skb)
1447 " qpid %u ep %p state %d tid %u status %d\n", 1573 " qpid %u ep %p state %d tid %u status %d\n",
1448 __func__, ep->com.qp->wq.sq.qid, ep, 1574 __func__, ep->com.qp->wq.sq.qid, ep,
1449 state_read(&ep->com), ep->hwtid, status); 1575 state_read(&ep->com), ep->hwtid, status);
1450 attrs.next_state = C4IW_QP_STATE_ERROR; 1576 attrs.next_state = C4IW_QP_STATE_TERMINATE;
1451 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1577 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
1452 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 1578 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0);
1453 c4iw_ep_disconnect(ep, 1, GFP_KERNEL);
1454 break; 1579 break;
1455 } 1580 }
1456 default: 1581 default:
@@ -1498,6 +1623,7 @@ static void send_fw_act_open_req(struct c4iw_ep *ep, unsigned int atid)
1498 struct fw_ofld_connection_wr *req; 1623 struct fw_ofld_connection_wr *req;
1499 unsigned int mtu_idx; 1624 unsigned int mtu_idx;
1500 int wscale; 1625 int wscale;
1626 struct sockaddr_in *sin;
1501 1627
1502 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 1628 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
1503 req = (struct fw_ofld_connection_wr *)__skb_put(skb, sizeof(*req)); 1629 req = (struct fw_ofld_connection_wr *)__skb_put(skb, sizeof(*req));
@@ -1506,10 +1632,12 @@ static void send_fw_act_open_req(struct c4iw_ep *ep, unsigned int atid)
1506 req->len16_pkd = htonl(FW_WR_LEN16(DIV_ROUND_UP(sizeof(*req), 16))); 1632 req->len16_pkd = htonl(FW_WR_LEN16(DIV_ROUND_UP(sizeof(*req), 16)));
1507 req->le.filter = cpu_to_be32(select_ntuple(ep->com.dev, ep->dst, 1633 req->le.filter = cpu_to_be32(select_ntuple(ep->com.dev, ep->dst,
1508 ep->l2t)); 1634 ep->l2t));
1509 req->le.lport = ep->com.local_addr.sin_port; 1635 sin = (struct sockaddr_in *)&ep->com.local_addr;
1510 req->le.pport = ep->com.remote_addr.sin_port; 1636 req->le.lport = sin->sin_port;
1511 req->le.u.ipv4.lip = ep->com.local_addr.sin_addr.s_addr; 1637 req->le.u.ipv4.lip = sin->sin_addr.s_addr;
1512 req->le.u.ipv4.pip = ep->com.remote_addr.sin_addr.s_addr; 1638 sin = (struct sockaddr_in *)&ep->com.remote_addr;
1639 req->le.pport = sin->sin_port;
1640 req->le.u.ipv4.pip = sin->sin_addr.s_addr;
1513 req->tcb.t_state_to_astid = 1641 req->tcb.t_state_to_astid =
1514 htonl(V_FW_OFLD_CONNECTION_WR_T_STATE(TCP_SYN_SENT) | 1642 htonl(V_FW_OFLD_CONNECTION_WR_T_STATE(TCP_SYN_SENT) |
1515 V_FW_OFLD_CONNECTION_WR_ASTID(atid)); 1643 V_FW_OFLD_CONNECTION_WR_ASTID(atid));
@@ -1560,14 +1688,98 @@ static inline int act_open_has_tid(int status)
1560 1688
1561#define ACT_OPEN_RETRY_COUNT 2 1689#define ACT_OPEN_RETRY_COUNT 2
1562 1690
1691static int import_ep(struct c4iw_ep *ep, int iptype, __u8 *peer_ip,
1692 struct dst_entry *dst, struct c4iw_dev *cdev,
1693 bool clear_mpa_v1)
1694{
1695 struct neighbour *n;
1696 int err, step;
1697 struct net_device *pdev;
1698
1699 n = dst_neigh_lookup(dst, peer_ip);
1700 if (!n)
1701 return -ENODEV;
1702
1703 rcu_read_lock();
1704 err = -ENOMEM;
1705 if (n->dev->flags & IFF_LOOPBACK) {
1706 if (iptype == 4)
1707 pdev = ip_dev_find(&init_net, *(__be32 *)peer_ip);
1708 else if (IS_ENABLED(CONFIG_IPV6))
1709 for_each_netdev(&init_net, pdev) {
1710 if (ipv6_chk_addr(&init_net,
1711 (struct in6_addr *)peer_ip,
1712 pdev, 1))
1713 break;
1714 }
1715 else
1716 pdev = NULL;
1717
1718 if (!pdev) {
1719 err = -ENODEV;
1720 goto out;
1721 }
1722 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t,
1723 n, pdev, 0);
1724 if (!ep->l2t)
1725 goto out;
1726 ep->mtu = pdev->mtu;
1727 ep->tx_chan = cxgb4_port_chan(pdev);
1728 ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1;
1729 step = cdev->rdev.lldi.ntxq /
1730 cdev->rdev.lldi.nchan;
1731 ep->txq_idx = cxgb4_port_idx(pdev) * step;
1732 step = cdev->rdev.lldi.nrxq /
1733 cdev->rdev.lldi.nchan;
1734 ep->ctrlq_idx = cxgb4_port_idx(pdev);
1735 ep->rss_qid = cdev->rdev.lldi.rxq_ids[
1736 cxgb4_port_idx(pdev) * step];
1737 dev_put(pdev);
1738 } else {
1739 pdev = get_real_dev(n->dev);
1740 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t,
1741 n, pdev, 0);
1742 if (!ep->l2t)
1743 goto out;
1744 ep->mtu = dst_mtu(dst);
1745 ep->tx_chan = cxgb4_port_chan(n->dev);
1746 ep->smac_idx = (cxgb4_port_viid(n->dev) & 0x7F) << 1;
1747 step = cdev->rdev.lldi.ntxq /
1748 cdev->rdev.lldi.nchan;
1749 ep->txq_idx = cxgb4_port_idx(n->dev) * step;
1750 ep->ctrlq_idx = cxgb4_port_idx(n->dev);
1751 step = cdev->rdev.lldi.nrxq /
1752 cdev->rdev.lldi.nchan;
1753 ep->rss_qid = cdev->rdev.lldi.rxq_ids[
1754 cxgb4_port_idx(n->dev) * step];
1755
1756 if (clear_mpa_v1) {
1757 ep->retry_with_mpa_v1 = 0;
1758 ep->tried_with_mpa_v1 = 0;
1759 }
1760 }
1761 err = 0;
1762out:
1763 rcu_read_unlock();
1764
1765 neigh_release(n);
1766
1767 return err;
1768}
1769
1563static int c4iw_reconnect(struct c4iw_ep *ep) 1770static int c4iw_reconnect(struct c4iw_ep *ep)
1564{ 1771{
1565 int err = 0; 1772 int err = 0;
1566 struct rtable *rt; 1773 struct sockaddr_in *laddr = (struct sockaddr_in *)
1567 struct port_info *pi; 1774 &ep->com.cm_id->local_addr;
1568 struct net_device *pdev; 1775 struct sockaddr_in *raddr = (struct sockaddr_in *)
1569 int step; 1776 &ep->com.cm_id->remote_addr;
1570 struct neighbour *neigh; 1777 struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)
1778 &ep->com.cm_id->local_addr;
1779 struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)
1780 &ep->com.cm_id->remote_addr;
1781 int iptype;
1782 __u8 *ra;
1571 1783
1572 PDBG("%s qp %p cm_id %p\n", __func__, ep->com.qp, ep->com.cm_id); 1784 PDBG("%s qp %p cm_id %p\n", __func__, ep->com.qp, ep->com.cm_id);
1573 init_timer(&ep->timer); 1785 init_timer(&ep->timer);
@@ -1584,57 +1796,28 @@ static int c4iw_reconnect(struct c4iw_ep *ep)
1584 insert_handle(ep->com.dev, &ep->com.dev->atid_idr, ep, ep->atid); 1796 insert_handle(ep->com.dev, &ep->com.dev->atid_idr, ep, ep->atid);
1585 1797
1586 /* find a route */ 1798 /* find a route */
1587 rt = find_route(ep->com.dev, 1799 if (ep->com.cm_id->local_addr.ss_family == AF_INET) {
1588 ep->com.cm_id->local_addr.sin_addr.s_addr, 1800 ep->dst = find_route(ep->com.dev, laddr->sin_addr.s_addr,
1589 ep->com.cm_id->remote_addr.sin_addr.s_addr, 1801 raddr->sin_addr.s_addr, laddr->sin_port,
1590 ep->com.cm_id->local_addr.sin_port, 1802 raddr->sin_port, 0);
1591 ep->com.cm_id->remote_addr.sin_port, 0); 1803 iptype = 4;
1592 if (!rt) { 1804 ra = (__u8 *)&raddr->sin_addr;
1805 } else {
1806 ep->dst = find_route6(ep->com.dev, laddr6->sin6_addr.s6_addr,
1807 raddr6->sin6_addr.s6_addr,
1808 laddr6->sin6_port, raddr6->sin6_port, 0,
1809 raddr6->sin6_scope_id);
1810 iptype = 6;
1811 ra = (__u8 *)&raddr6->sin6_addr;
1812 }
1813 if (!ep->dst) {
1593 pr_err("%s - cannot find route.\n", __func__); 1814 pr_err("%s - cannot find route.\n", __func__);
1594 err = -EHOSTUNREACH; 1815 err = -EHOSTUNREACH;
1595 goto fail3; 1816 goto fail3;
1596 } 1817 }
1597 ep->dst = &rt->dst; 1818 err = import_ep(ep, iptype, ra, ep->dst, ep->com.dev, false);
1598 1819 if (err) {
1599 neigh = dst_neigh_lookup(ep->dst,
1600 &ep->com.cm_id->remote_addr.sin_addr.s_addr);
1601 if (!neigh) {
1602 pr_err("%s - cannot alloc neigh.\n", __func__);
1603 err = -ENOMEM;
1604 goto fail4;
1605 }
1606
1607 /* get a l2t entry */
1608 if (neigh->dev->flags & IFF_LOOPBACK) {
1609 PDBG("%s LOOPBACK\n", __func__);
1610 pdev = ip_dev_find(&init_net,
1611 ep->com.cm_id->remote_addr.sin_addr.s_addr);
1612 ep->l2t = cxgb4_l2t_get(ep->com.dev->rdev.lldi.l2t,
1613 neigh, pdev, 0);
1614 pi = (struct port_info *)netdev_priv(pdev);
1615 ep->mtu = pdev->mtu;
1616 ep->tx_chan = cxgb4_port_chan(pdev);
1617 ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1;
1618 dev_put(pdev);
1619 } else {
1620 ep->l2t = cxgb4_l2t_get(ep->com.dev->rdev.lldi.l2t,
1621 neigh, neigh->dev, 0);
1622 pi = (struct port_info *)netdev_priv(neigh->dev);
1623 ep->mtu = dst_mtu(ep->dst);
1624 ep->tx_chan = cxgb4_port_chan(neigh->dev);
1625 ep->smac_idx = (cxgb4_port_viid(neigh->dev) &
1626 0x7F) << 1;
1627 }
1628
1629 step = ep->com.dev->rdev.lldi.ntxq / ep->com.dev->rdev.lldi.nchan;
1630 ep->txq_idx = pi->port_id * step;
1631 ep->ctrlq_idx = pi->port_id;
1632 step = ep->com.dev->rdev.lldi.nrxq / ep->com.dev->rdev.lldi.nchan;
1633 ep->rss_qid = ep->com.dev->rdev.lldi.rxq_ids[pi->port_id * step];
1634
1635 if (!ep->l2t) {
1636 pr_err("%s - cannot alloc l2e.\n", __func__); 1820 pr_err("%s - cannot alloc l2e.\n", __func__);
1637 err = -ENOMEM;
1638 goto fail4; 1821 goto fail4;
1639 } 1822 }
1640 1823
@@ -1677,8 +1860,16 @@ static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
1677 ntohl(rpl->atid_status))); 1860 ntohl(rpl->atid_status)));
1678 struct tid_info *t = dev->rdev.lldi.tids; 1861 struct tid_info *t = dev->rdev.lldi.tids;
1679 int status = GET_AOPEN_STATUS(ntohl(rpl->atid_status)); 1862 int status = GET_AOPEN_STATUS(ntohl(rpl->atid_status));
1863 struct sockaddr_in *la;
1864 struct sockaddr_in *ra;
1865 struct sockaddr_in6 *la6;
1866 struct sockaddr_in6 *ra6;
1680 1867
1681 ep = lookup_atid(t, atid); 1868 ep = lookup_atid(t, atid);
1869 la = (struct sockaddr_in *)&ep->com.local_addr;
1870 ra = (struct sockaddr_in *)&ep->com.remote_addr;
1871 la6 = (struct sockaddr_in6 *)&ep->com.local_addr;
1872 ra6 = (struct sockaddr_in6 *)&ep->com.remote_addr;
1682 1873
1683 PDBG("%s ep %p atid %u status %u errno %d\n", __func__, ep, atid, 1874 PDBG("%s ep %p atid %u status %u errno %d\n", __func__, ep, atid,
1684 status, status2errno(status)); 1875 status, status2errno(status));
@@ -1699,10 +1890,11 @@ static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
1699 case CPL_ERR_CONN_TIMEDOUT: 1890 case CPL_ERR_CONN_TIMEDOUT:
1700 break; 1891 break;
1701 case CPL_ERR_TCAM_FULL: 1892 case CPL_ERR_TCAM_FULL:
1893 mutex_lock(&dev->rdev.stats.lock);
1702 dev->rdev.stats.tcam_full++; 1894 dev->rdev.stats.tcam_full++;
1703 if (dev->rdev.lldi.enable_fw_ofld_conn) { 1895 mutex_unlock(&dev->rdev.stats.lock);
1704 mutex_lock(&dev->rdev.stats.lock); 1896 if (ep->com.local_addr.ss_family == AF_INET &&
1705 mutex_unlock(&dev->rdev.stats.lock); 1897 dev->rdev.lldi.enable_fw_ofld_conn) {
1706 send_fw_act_open_req(ep, 1898 send_fw_act_open_req(ep,
1707 GET_TID_TID(GET_AOPEN_ATID( 1899 GET_TID_TID(GET_AOPEN_ATID(
1708 ntohl(rpl->atid_status)))); 1900 ntohl(rpl->atid_status))));
@@ -1722,13 +1914,17 @@ static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
1722 } 1914 }
1723 break; 1915 break;
1724 default: 1916 default:
1725 printk(KERN_INFO MOD "Active open failure - " 1917 if (ep->com.local_addr.ss_family == AF_INET) {
1726 "atid %u status %u errno %d %pI4:%u->%pI4:%u\n", 1918 pr_info("Active open failure - atid %u status %u errno %d %pI4:%u->%pI4:%u\n",
1727 atid, status, status2errno(status), 1919 atid, status, status2errno(status),
1728 &ep->com.local_addr.sin_addr.s_addr, 1920 &la->sin_addr.s_addr, ntohs(la->sin_port),
1729 ntohs(ep->com.local_addr.sin_port), 1921 &ra->sin_addr.s_addr, ntohs(ra->sin_port));
1730 &ep->com.remote_addr.sin_addr.s_addr, 1922 } else {
1731 ntohs(ep->com.remote_addr.sin_port)); 1923 pr_info("Active open failure - atid %u status %u errno %d %pI6:%u->%pI6:%u\n",
1924 atid, status, status2errno(status),
1925 la6->sin6_addr.s6_addr, ntohs(la6->sin6_port),
1926 ra6->sin6_addr.s6_addr, ntohs(ra6->sin6_port));
1927 }
1732 break; 1928 break;
1733 } 1929 }
1734 1930
@@ -1766,27 +1962,6 @@ out:
1766 return 0; 1962 return 0;
1767} 1963}
1768 1964
1769static int listen_stop(struct c4iw_listen_ep *ep)
1770{
1771 struct sk_buff *skb;
1772 struct cpl_close_listsvr_req *req;
1773
1774 PDBG("%s ep %p\n", __func__, ep);
1775 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
1776 if (!skb) {
1777 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__);
1778 return -ENOMEM;
1779 }
1780 req = (struct cpl_close_listsvr_req *) skb_put(skb, sizeof(*req));
1781 INIT_TP_WR(req, 0);
1782 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ,
1783 ep->stid));
1784 req->reply_ctrl = cpu_to_be16(
1785 QUEUENO(ep->com.dev->rdev.lldi.rxq_ids[0]));
1786 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0);
1787 return c4iw_ofld_send(&ep->com.dev->rdev, skb);
1788}
1789
1790static int close_listsrv_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1965static int close_listsrv_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
1791{ 1966{
1792 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb); 1967 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb);
@@ -1799,7 +1974,7 @@ static int close_listsrv_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
1799 return 0; 1974 return 0;
1800} 1975}
1801 1976
1802static void accept_cr(struct c4iw_ep *ep, __be32 peer_ip, struct sk_buff *skb, 1977static void accept_cr(struct c4iw_ep *ep, struct sk_buff *skb,
1803 struct cpl_pass_accept_req *req) 1978 struct cpl_pass_accept_req *req)
1804{ 1979{
1805 struct cpl_pass_accept_rpl *rpl; 1980 struct cpl_pass_accept_rpl *rpl;
@@ -1851,16 +2026,15 @@ static void accept_cr(struct c4iw_ep *ep, __be32 peer_ip, struct sk_buff *skb,
1851 rpl->opt0 = cpu_to_be64(opt0); 2026 rpl->opt0 = cpu_to_be64(opt0);
1852 rpl->opt2 = cpu_to_be32(opt2); 2027 rpl->opt2 = cpu_to_be32(opt2);
1853 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 2028 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx);
2029 t4_set_arp_err_handler(skb, NULL, arp_failure_discard);
1854 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 2030 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t);
1855 2031
1856 return; 2032 return;
1857} 2033}
1858 2034
1859static void reject_cr(struct c4iw_dev *dev, u32 hwtid, __be32 peer_ip, 2035static void reject_cr(struct c4iw_dev *dev, u32 hwtid, struct sk_buff *skb)
1860 struct sk_buff *skb)
1861{ 2036{
1862 PDBG("%s c4iw_dev %p tid %u peer_ip %x\n", __func__, dev, hwtid, 2037 PDBG("%s c4iw_dev %p tid %u\n", __func__, dev, hwtid);
1863 peer_ip);
1864 BUG_ON(skb_cloned(skb)); 2038 BUG_ON(skb_cloned(skb));
1865 skb_trim(skb, sizeof(struct cpl_tid_release)); 2039 skb_trim(skb, sizeof(struct cpl_tid_release));
1866 skb_get(skb); 2040 skb_get(skb);
@@ -1868,95 +2042,38 @@ static void reject_cr(struct c4iw_dev *dev, u32 hwtid, __be32 peer_ip,
1868 return; 2042 return;
1869} 2043}
1870 2044
1871static void get_4tuple(struct cpl_pass_accept_req *req, 2045static void get_4tuple(struct cpl_pass_accept_req *req, int *iptype,
1872 __be32 *local_ip, __be32 *peer_ip, 2046 __u8 *local_ip, __u8 *peer_ip,
1873 __be16 *local_port, __be16 *peer_port) 2047 __be16 *local_port, __be16 *peer_port)
1874{ 2048{
1875 int eth_len = G_ETH_HDR_LEN(be32_to_cpu(req->hdr_len)); 2049 int eth_len = G_ETH_HDR_LEN(be32_to_cpu(req->hdr_len));
1876 int ip_len = G_IP_HDR_LEN(be32_to_cpu(req->hdr_len)); 2050 int ip_len = G_IP_HDR_LEN(be32_to_cpu(req->hdr_len));
1877 struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len); 2051 struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len);
2052 struct ipv6hdr *ip6 = (struct ipv6hdr *)((u8 *)(req + 1) + eth_len);
1878 struct tcphdr *tcp = (struct tcphdr *) 2053 struct tcphdr *tcp = (struct tcphdr *)
1879 ((u8 *)(req + 1) + eth_len + ip_len); 2054 ((u8 *)(req + 1) + eth_len + ip_len);
1880 2055
1881 PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__, 2056 if (ip->version == 4) {
1882 ntohl(ip->saddr), ntohl(ip->daddr), ntohs(tcp->source), 2057 PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__,
1883 ntohs(tcp->dest)); 2058 ntohl(ip->saddr), ntohl(ip->daddr), ntohs(tcp->source),
1884 2059 ntohs(tcp->dest));
1885 *peer_ip = ip->saddr; 2060 *iptype = 4;
1886 *local_ip = ip->daddr; 2061 memcpy(peer_ip, &ip->saddr, 4);
2062 memcpy(local_ip, &ip->daddr, 4);
2063 } else {
2064 PDBG("%s saddr %pI6 daddr %pI6 sport %u dport %u\n", __func__,
2065 ip6->saddr.s6_addr, ip6->daddr.s6_addr, ntohs(tcp->source),
2066 ntohs(tcp->dest));
2067 *iptype = 6;
2068 memcpy(peer_ip, ip6->saddr.s6_addr, 16);
2069 memcpy(local_ip, ip6->daddr.s6_addr, 16);
2070 }
1887 *peer_port = tcp->source; 2071 *peer_port = tcp->source;
1888 *local_port = tcp->dest; 2072 *local_port = tcp->dest;
1889 2073
1890 return; 2074 return;
1891} 2075}
1892 2076
1893static int import_ep(struct c4iw_ep *ep, __be32 peer_ip, struct dst_entry *dst,
1894 struct c4iw_dev *cdev, bool clear_mpa_v1)
1895{
1896 struct neighbour *n;
1897 int err, step;
1898
1899 n = dst_neigh_lookup(dst, &peer_ip);
1900 if (!n)
1901 return -ENODEV;
1902
1903 rcu_read_lock();
1904 err = -ENOMEM;
1905 if (n->dev->flags & IFF_LOOPBACK) {
1906 struct net_device *pdev;
1907
1908 pdev = ip_dev_find(&init_net, peer_ip);
1909 if (!pdev) {
1910 err = -ENODEV;
1911 goto out;
1912 }
1913 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t,
1914 n, pdev, 0);
1915 if (!ep->l2t)
1916 goto out;
1917 ep->mtu = pdev->mtu;
1918 ep->tx_chan = cxgb4_port_chan(pdev);
1919 ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1;
1920 step = cdev->rdev.lldi.ntxq /
1921 cdev->rdev.lldi.nchan;
1922 ep->txq_idx = cxgb4_port_idx(pdev) * step;
1923 step = cdev->rdev.lldi.nrxq /
1924 cdev->rdev.lldi.nchan;
1925 ep->ctrlq_idx = cxgb4_port_idx(pdev);
1926 ep->rss_qid = cdev->rdev.lldi.rxq_ids[
1927 cxgb4_port_idx(pdev) * step];
1928 dev_put(pdev);
1929 } else {
1930 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t,
1931 n, n->dev, 0);
1932 if (!ep->l2t)
1933 goto out;
1934 ep->mtu = dst_mtu(dst);
1935 ep->tx_chan = cxgb4_port_chan(n->dev);
1936 ep->smac_idx = (cxgb4_port_viid(n->dev) & 0x7F) << 1;
1937 step = cdev->rdev.lldi.ntxq /
1938 cdev->rdev.lldi.nchan;
1939 ep->txq_idx = cxgb4_port_idx(n->dev) * step;
1940 ep->ctrlq_idx = cxgb4_port_idx(n->dev);
1941 step = cdev->rdev.lldi.nrxq /
1942 cdev->rdev.lldi.nchan;
1943 ep->rss_qid = cdev->rdev.lldi.rxq_ids[
1944 cxgb4_port_idx(n->dev) * step];
1945
1946 if (clear_mpa_v1) {
1947 ep->retry_with_mpa_v1 = 0;
1948 ep->tried_with_mpa_v1 = 0;
1949 }
1950 }
1951 err = 0;
1952out:
1953 rcu_read_unlock();
1954
1955 neigh_release(n);
1956
1957 return err;
1958}
1959
1960static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) 2077static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
1961{ 2078{
1962 struct c4iw_ep *child_ep = NULL, *parent_ep; 2079 struct c4iw_ep *child_ep = NULL, *parent_ep;
@@ -1965,23 +2082,17 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
1965 struct tid_info *t = dev->rdev.lldi.tids; 2082 struct tid_info *t = dev->rdev.lldi.tids;
1966 unsigned int hwtid = GET_TID(req); 2083 unsigned int hwtid = GET_TID(req);
1967 struct dst_entry *dst; 2084 struct dst_entry *dst;
1968 struct rtable *rt; 2085 __u8 local_ip[16], peer_ip[16];
1969 __be32 local_ip, peer_ip = 0;
1970 __be16 local_port, peer_port; 2086 __be16 local_port, peer_port;
1971 int err; 2087 int err;
1972 u16 peer_mss = ntohs(req->tcpopt.mss); 2088 u16 peer_mss = ntohs(req->tcpopt.mss);
2089 int iptype;
1973 2090
1974 parent_ep = lookup_stid(t, stid); 2091 parent_ep = lookup_stid(t, stid);
1975 if (!parent_ep) { 2092 if (!parent_ep) {
1976 PDBG("%s connect request on invalid stid %d\n", __func__, stid); 2093 PDBG("%s connect request on invalid stid %d\n", __func__, stid);
1977 goto reject; 2094 goto reject;
1978 } 2095 }
1979 get_4tuple(req, &local_ip, &peer_ip, &local_port, &peer_port);
1980
1981 PDBG("%s parent ep %p hwtid %u laddr 0x%x raddr 0x%x lport %d " \
1982 "rport %d peer_mss %d\n", __func__, parent_ep, hwtid,
1983 ntohl(local_ip), ntohl(peer_ip), ntohs(local_port),
1984 ntohs(peer_port), peer_mss);
1985 2096
1986 if (state_read(&parent_ep->com) != LISTEN) { 2097 if (state_read(&parent_ep->com) != LISTEN) {
1987 printk(KERN_ERR "%s - listening ep not in LISTEN\n", 2098 printk(KERN_ERR "%s - listening ep not in LISTEN\n",
@@ -1989,15 +2100,32 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
1989 goto reject; 2100 goto reject;
1990 } 2101 }
1991 2102
2103 get_4tuple(req, &iptype, local_ip, peer_ip, &local_port, &peer_port);
2104
1992 /* Find output route */ 2105 /* Find output route */
1993 rt = find_route(dev, local_ip, peer_ip, local_port, peer_port, 2106 if (iptype == 4) {
1994 GET_POPEN_TOS(ntohl(req->tos_stid))); 2107 PDBG("%s parent ep %p hwtid %u laddr %pI4 raddr %pI4 lport %d rport %d peer_mss %d\n"
1995 if (!rt) { 2108 , __func__, parent_ep, hwtid,
2109 local_ip, peer_ip, ntohs(local_port),
2110 ntohs(peer_port), peer_mss);
2111 dst = find_route(dev, *(__be32 *)local_ip, *(__be32 *)peer_ip,
2112 local_port, peer_port,
2113 GET_POPEN_TOS(ntohl(req->tos_stid)));
2114 } else {
2115 PDBG("%s parent ep %p hwtid %u laddr %pI6 raddr %pI6 lport %d rport %d peer_mss %d\n"
2116 , __func__, parent_ep, hwtid,
2117 local_ip, peer_ip, ntohs(local_port),
2118 ntohs(peer_port), peer_mss);
2119 dst = find_route6(dev, local_ip, peer_ip, local_port, peer_port,
2120 PASS_OPEN_TOS(ntohl(req->tos_stid)),
2121 ((struct sockaddr_in6 *)
2122 &parent_ep->com.local_addr)->sin6_scope_id);
2123 }
2124 if (!dst) {
1996 printk(KERN_ERR MOD "%s - failed to find dst entry!\n", 2125 printk(KERN_ERR MOD "%s - failed to find dst entry!\n",
1997 __func__); 2126 __func__);
1998 goto reject; 2127 goto reject;
1999 } 2128 }
2000 dst = &rt->dst;
2001 2129
2002 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 2130 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL);
2003 if (!child_ep) { 2131 if (!child_ep) {
@@ -2007,7 +2135,7 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
2007 goto reject; 2135 goto reject;
2008 } 2136 }
2009 2137
2010 err = import_ep(child_ep, peer_ip, dst, dev, false); 2138 err = import_ep(child_ep, iptype, peer_ip, dst, dev, false);
2011 if (err) { 2139 if (err) {
2012 printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n", 2140 printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n",
2013 __func__); 2141 __func__);
@@ -2022,12 +2150,27 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
2022 state_set(&child_ep->com, CONNECTING); 2150 state_set(&child_ep->com, CONNECTING);
2023 child_ep->com.dev = dev; 2151 child_ep->com.dev = dev;
2024 child_ep->com.cm_id = NULL; 2152 child_ep->com.cm_id = NULL;
2025 child_ep->com.local_addr.sin_family = PF_INET; 2153 if (iptype == 4) {
2026 child_ep->com.local_addr.sin_port = local_port; 2154 struct sockaddr_in *sin = (struct sockaddr_in *)
2027 child_ep->com.local_addr.sin_addr.s_addr = local_ip; 2155 &child_ep->com.local_addr;
2028 child_ep->com.remote_addr.sin_family = PF_INET; 2156 sin->sin_family = PF_INET;
2029 child_ep->com.remote_addr.sin_port = peer_port; 2157 sin->sin_port = local_port;
2030 child_ep->com.remote_addr.sin_addr.s_addr = peer_ip; 2158 sin->sin_addr.s_addr = *(__be32 *)local_ip;
2159 sin = (struct sockaddr_in *)&child_ep->com.remote_addr;
2160 sin->sin_family = PF_INET;
2161 sin->sin_port = peer_port;
2162 sin->sin_addr.s_addr = *(__be32 *)peer_ip;
2163 } else {
2164 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)
2165 &child_ep->com.local_addr;
2166 sin6->sin6_family = PF_INET6;
2167 sin6->sin6_port = local_port;
2168 memcpy(sin6->sin6_addr.s6_addr, local_ip, 16);
2169 sin6 = (struct sockaddr_in6 *)&child_ep->com.remote_addr;
2170 sin6->sin6_family = PF_INET6;
2171 sin6->sin6_port = peer_port;
2172 memcpy(sin6->sin6_addr.s6_addr, peer_ip, 16);
2173 }
2031 c4iw_get_ep(&parent_ep->com); 2174 c4iw_get_ep(&parent_ep->com);
2032 child_ep->parent_ep = parent_ep; 2175 child_ep->parent_ep = parent_ep;
2033 child_ep->tos = GET_POPEN_TOS(ntohl(req->tos_stid)); 2176 child_ep->tos = GET_POPEN_TOS(ntohl(req->tos_stid));
@@ -2040,11 +2183,11 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb)
2040 init_timer(&child_ep->timer); 2183 init_timer(&child_ep->timer);
2041 cxgb4_insert_tid(t, child_ep, hwtid); 2184 cxgb4_insert_tid(t, child_ep, hwtid);
2042 insert_handle(dev, &dev->hwtid_idr, child_ep, child_ep->hwtid); 2185 insert_handle(dev, &dev->hwtid_idr, child_ep, child_ep->hwtid);
2043 accept_cr(child_ep, peer_ip, skb, req); 2186 accept_cr(child_ep, skb, req);
2044 set_bit(PASS_ACCEPT_REQ, &child_ep->com.history); 2187 set_bit(PASS_ACCEPT_REQ, &child_ep->com.history);
2045 goto out; 2188 goto out;
2046reject: 2189reject:
2047 reject_cr(dev, hwtid, peer_ip, skb); 2190 reject_cr(dev, hwtid, skb);
2048out: 2191out:
2049 return 0; 2192 return 0;
2050} 2193}
@@ -2512,12 +2655,79 @@ err:
2512 return err; 2655 return err;
2513} 2656}
2514 2657
2658static int pick_local_ipaddrs(struct c4iw_dev *dev, struct iw_cm_id *cm_id)
2659{
2660 struct in_device *ind;
2661 int found = 0;
2662 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr;
2663 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr;
2664
2665 ind = in_dev_get(dev->rdev.lldi.ports[0]);
2666 if (!ind)
2667 return -EADDRNOTAVAIL;
2668 for_primary_ifa(ind) {
2669 laddr->sin_addr.s_addr = ifa->ifa_address;
2670 raddr->sin_addr.s_addr = ifa->ifa_address;
2671 found = 1;
2672 break;
2673 }
2674 endfor_ifa(ind);
2675 in_dev_put(ind);
2676 return found ? 0 : -EADDRNOTAVAIL;
2677}
2678
2679static int get_lladdr(struct net_device *dev, struct in6_addr *addr,
2680 unsigned char banned_flags)
2681{
2682 struct inet6_dev *idev;
2683 int err = -EADDRNOTAVAIL;
2684
2685 rcu_read_lock();
2686 idev = __in6_dev_get(dev);
2687 if (idev != NULL) {
2688 struct inet6_ifaddr *ifp;
2689
2690 read_lock_bh(&idev->lock);
2691 list_for_each_entry(ifp, &idev->addr_list, if_list) {
2692 if (ifp->scope == IFA_LINK &&
2693 !(ifp->flags & banned_flags)) {
2694 memcpy(addr, &ifp->addr, 16);
2695 err = 0;
2696 break;
2697 }
2698 }
2699 read_unlock_bh(&idev->lock);
2700 }
2701 rcu_read_unlock();
2702 return err;
2703}
2704
2705static int pick_local_ip6addrs(struct c4iw_dev *dev, struct iw_cm_id *cm_id)
2706{
2707 struct in6_addr uninitialized_var(addr);
2708 struct sockaddr_in6 *la6 = (struct sockaddr_in6 *)&cm_id->local_addr;
2709 struct sockaddr_in6 *ra6 = (struct sockaddr_in6 *)&cm_id->remote_addr;
2710
2711 if (get_lladdr(dev->rdev.lldi.ports[0], &addr, IFA_F_TENTATIVE)) {
2712 memcpy(la6->sin6_addr.s6_addr, &addr, 16);
2713 memcpy(ra6->sin6_addr.s6_addr, &addr, 16);
2714 return 0;
2715 }
2716 return -EADDRNOTAVAIL;
2717}
2718
2515int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2719int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
2516{ 2720{
2517 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2721 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device);
2518 struct c4iw_ep *ep; 2722 struct c4iw_ep *ep;
2519 struct rtable *rt;
2520 int err = 0; 2723 int err = 0;
2724 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr;
2725 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr;
2726 struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&cm_id->local_addr;
2727 struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)
2728 &cm_id->remote_addr;
2729 __u8 *ra;
2730 int iptype;
2521 2731
2522 if ((conn_param->ord > c4iw_max_read_depth) || 2732 if ((conn_param->ord > c4iw_max_read_depth) ||
2523 (conn_param->ird > c4iw_max_read_depth)) { 2733 (conn_param->ird > c4iw_max_read_depth)) {
@@ -2545,7 +2755,11 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
2545 ep->com.dev = dev; 2755 ep->com.dev = dev;
2546 ep->com.cm_id = cm_id; 2756 ep->com.cm_id = cm_id;
2547 ep->com.qp = get_qhp(dev, conn_param->qpn); 2757 ep->com.qp = get_qhp(dev, conn_param->qpn);
2548 BUG_ON(!ep->com.qp); 2758 if (!ep->com.qp) {
2759 PDBG("%s qpn 0x%x not found!\n", __func__, conn_param->qpn);
2760 err = -EINVAL;
2761 goto fail2;
2762 }
2549 ref_qp(ep); 2763 ref_qp(ep);
2550 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn, 2764 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn,
2551 ep->com.qp, cm_id); 2765 ep->com.qp, cm_id);
@@ -2561,27 +2775,56 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
2561 } 2775 }
2562 insert_handle(dev, &dev->atid_idr, ep, ep->atid); 2776 insert_handle(dev, &dev->atid_idr, ep, ep->atid);
2563 2777
2564 PDBG("%s saddr 0x%x sport 0x%x raddr 0x%x rport 0x%x\n", __func__, 2778 if (cm_id->remote_addr.ss_family == AF_INET) {
2565 ntohl(cm_id->local_addr.sin_addr.s_addr), 2779 iptype = 4;
2566 ntohs(cm_id->local_addr.sin_port), 2780 ra = (__u8 *)&raddr->sin_addr;
2567 ntohl(cm_id->remote_addr.sin_addr.s_addr),
2568 ntohs(cm_id->remote_addr.sin_port));
2569 2781
2570 /* find a route */ 2782 /*
2571 rt = find_route(dev, 2783 * Handle loopback requests to INADDR_ANY.
2572 cm_id->local_addr.sin_addr.s_addr, 2784 */
2573 cm_id->remote_addr.sin_addr.s_addr, 2785 if ((__force int)raddr->sin_addr.s_addr == INADDR_ANY) {
2574 cm_id->local_addr.sin_port, 2786 err = pick_local_ipaddrs(dev, cm_id);
2575 cm_id->remote_addr.sin_port, 0); 2787 if (err)
2576 if (!rt) { 2788 goto fail2;
2789 }
2790
2791 /* find a route */
2792 PDBG("%s saddr %pI4 sport 0x%x raddr %pI4 rport 0x%x\n",
2793 __func__, &laddr->sin_addr, ntohs(laddr->sin_port),
2794 ra, ntohs(raddr->sin_port));
2795 ep->dst = find_route(dev, laddr->sin_addr.s_addr,
2796 raddr->sin_addr.s_addr, laddr->sin_port,
2797 raddr->sin_port, 0);
2798 } else {
2799 iptype = 6;
2800 ra = (__u8 *)&raddr6->sin6_addr;
2801
2802 /*
2803 * Handle loopback requests to INADDR_ANY.
2804 */
2805 if (ipv6_addr_type(&raddr6->sin6_addr) == IPV6_ADDR_ANY) {
2806 err = pick_local_ip6addrs(dev, cm_id);
2807 if (err)
2808 goto fail2;
2809 }
2810
2811 /* find a route */
2812 PDBG("%s saddr %pI6 sport 0x%x raddr %pI6 rport 0x%x\n",
2813 __func__, laddr6->sin6_addr.s6_addr,
2814 ntohs(laddr6->sin6_port),
2815 raddr6->sin6_addr.s6_addr, ntohs(raddr6->sin6_port));
2816 ep->dst = find_route6(dev, laddr6->sin6_addr.s6_addr,
2817 raddr6->sin6_addr.s6_addr,
2818 laddr6->sin6_port, raddr6->sin6_port, 0,
2819 raddr6->sin6_scope_id);
2820 }
2821 if (!ep->dst) {
2577 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2822 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__);
2578 err = -EHOSTUNREACH; 2823 err = -EHOSTUNREACH;
2579 goto fail3; 2824 goto fail3;
2580 } 2825 }
2581 ep->dst = &rt->dst;
2582 2826
2583 err = import_ep(ep, cm_id->remote_addr.sin_addr.s_addr, 2827 err = import_ep(ep, iptype, ra, ep->dst, ep->com.dev, true);
2584 ep->dst, ep->com.dev, true);
2585 if (err) { 2828 if (err) {
2586 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__); 2829 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__);
2587 goto fail4; 2830 goto fail4;
@@ -2593,8 +2836,10 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
2593 2836
2594 state_set(&ep->com, CONNECTING); 2837 state_set(&ep->com, CONNECTING);
2595 ep->tos = 0; 2838 ep->tos = 0;
2596 ep->com.local_addr = cm_id->local_addr; 2839 memcpy(&ep->com.local_addr, &cm_id->local_addr,
2597 ep->com.remote_addr = cm_id->remote_addr; 2840 sizeof(ep->com.local_addr));
2841 memcpy(&ep->com.remote_addr, &cm_id->remote_addr,
2842 sizeof(ep->com.remote_addr));
2598 2843
2599 /* send connect request to rnic */ 2844 /* send connect request to rnic */
2600 err = send_connect(ep); 2845 err = send_connect(ep);
@@ -2614,6 +2859,60 @@ out:
2614 return err; 2859 return err;
2615} 2860}
2616 2861
2862static int create_server6(struct c4iw_dev *dev, struct c4iw_listen_ep *ep)
2863{
2864 int err;
2865 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ep->com.local_addr;
2866
2867 c4iw_init_wr_wait(&ep->com.wr_wait);
2868 err = cxgb4_create_server6(ep->com.dev->rdev.lldi.ports[0],
2869 ep->stid, &sin6->sin6_addr,
2870 sin6->sin6_port,
2871 ep->com.dev->rdev.lldi.rxq_ids[0]);
2872 if (!err)
2873 err = c4iw_wait_for_reply(&ep->com.dev->rdev,
2874 &ep->com.wr_wait,
2875 0, 0, __func__);
2876 if (err)
2877 pr_err("cxgb4_create_server6/filter failed err %d stid %d laddr %pI6 lport %d\n",
2878 err, ep->stid,
2879 sin6->sin6_addr.s6_addr, ntohs(sin6->sin6_port));
2880 return err;
2881}
2882
2883static int create_server4(struct c4iw_dev *dev, struct c4iw_listen_ep *ep)
2884{
2885 int err;
2886 struct sockaddr_in *sin = (struct sockaddr_in *)&ep->com.local_addr;
2887
2888 if (dev->rdev.lldi.enable_fw_ofld_conn) {
2889 do {
2890 err = cxgb4_create_server_filter(
2891 ep->com.dev->rdev.lldi.ports[0], ep->stid,
2892 sin->sin_addr.s_addr, sin->sin_port, 0,
2893 ep->com.dev->rdev.lldi.rxq_ids[0], 0, 0);
2894 if (err == -EBUSY) {
2895 set_current_state(TASK_UNINTERRUPTIBLE);
2896 schedule_timeout(usecs_to_jiffies(100));
2897 }
2898 } while (err == -EBUSY);
2899 } else {
2900 c4iw_init_wr_wait(&ep->com.wr_wait);
2901 err = cxgb4_create_server(ep->com.dev->rdev.lldi.ports[0],
2902 ep->stid, sin->sin_addr.s_addr, sin->sin_port,
2903 0, ep->com.dev->rdev.lldi.rxq_ids[0]);
2904 if (!err)
2905 err = c4iw_wait_for_reply(&ep->com.dev->rdev,
2906 &ep->com.wr_wait,
2907 0, 0, __func__);
2908 }
2909 if (err)
2910 pr_err("cxgb4_create_server/filter failed err %d stid %d laddr %pI4 lport %d\n"
2911 , err, ep->stid,
2912 &sin->sin_addr, ntohs(sin->sin_port));
2913 return err;
2914}
2915
2617int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog) 2916int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog)
2618{ 2917{
2619 int err = 0; 2918 int err = 0;
@@ -2633,15 +2932,18 @@ int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog)
2633 ep->com.cm_id = cm_id; 2932 ep->com.cm_id = cm_id;
2634 ep->com.dev = dev; 2933 ep->com.dev = dev;
2635 ep->backlog = backlog; 2934 ep->backlog = backlog;
2636 ep->com.local_addr = cm_id->local_addr; 2935 memcpy(&ep->com.local_addr, &cm_id->local_addr,
2936 sizeof(ep->com.local_addr));
2637 2937
2638 /* 2938 /*
2639 * Allocate a server TID. 2939 * Allocate a server TID.
2640 */ 2940 */
2641 if (dev->rdev.lldi.enable_fw_ofld_conn) 2941 if (dev->rdev.lldi.enable_fw_ofld_conn)
2642 ep->stid = cxgb4_alloc_sftid(dev->rdev.lldi.tids, PF_INET, ep); 2942 ep->stid = cxgb4_alloc_sftid(dev->rdev.lldi.tids,
2943 cm_id->local_addr.ss_family, ep);
2643 else 2944 else
2644 ep->stid = cxgb4_alloc_stid(dev->rdev.lldi.tids, PF_INET, ep); 2945 ep->stid = cxgb4_alloc_stid(dev->rdev.lldi.tids,
2946 cm_id->local_addr.ss_family, ep);
2645 2947
2646 if (ep->stid == -1) { 2948 if (ep->stid == -1) {
2647 printk(KERN_ERR MOD "%s - cannot alloc stid.\n", __func__); 2949 printk(KERN_ERR MOD "%s - cannot alloc stid.\n", __func__);
@@ -2650,43 +2952,16 @@ int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog)
2650 } 2952 }
2651 insert_handle(dev, &dev->stid_idr, ep, ep->stid); 2953 insert_handle(dev, &dev->stid_idr, ep, ep->stid);
2652 state_set(&ep->com, LISTEN); 2954 state_set(&ep->com, LISTEN);
2653 if (dev->rdev.lldi.enable_fw_ofld_conn) { 2955 if (ep->com.local_addr.ss_family == AF_INET)
2654 do { 2956 err = create_server4(dev, ep);
2655 err = cxgb4_create_server_filter( 2957 else
2656 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2958 err = create_server6(dev, ep);
2657 ep->com.local_addr.sin_addr.s_addr,
2658 ep->com.local_addr.sin_port,
2659 0,
2660 ep->com.dev->rdev.lldi.rxq_ids[0],
2661 0,
2662 0);
2663 if (err == -EBUSY) {
2664 set_current_state(TASK_UNINTERRUPTIBLE);
2665 schedule_timeout(usecs_to_jiffies(100));
2666 }
2667 } while (err == -EBUSY);
2668 } else {
2669 c4iw_init_wr_wait(&ep->com.wr_wait);
2670 err = cxgb4_create_server(ep->com.dev->rdev.lldi.ports[0],
2671 ep->stid, ep->com.local_addr.sin_addr.s_addr,
2672 ep->com.local_addr.sin_port,
2673 0,
2674 ep->com.dev->rdev.lldi.rxq_ids[0]);
2675 if (!err)
2676 err = c4iw_wait_for_reply(&ep->com.dev->rdev,
2677 &ep->com.wr_wait,
2678 0, 0, __func__);
2679 }
2680 if (!err) { 2959 if (!err) {
2681 cm_id->provider_data = ep; 2960 cm_id->provider_data = ep;
2682 goto out; 2961 goto out;
2683 } 2962 }
2684 pr_err("%s cxgb4_create_server/filter failed err %d " \ 2963 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid,
2685 "stid %d laddr %08x lport %d\n", \ 2964 ep->com.local_addr.ss_family);
2686 __func__, err, ep->stid,
2687 ntohl(ep->com.local_addr.sin_addr.s_addr),
2688 ntohs(ep->com.local_addr.sin_port));
2689 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET);
2690fail2: 2965fail2:
2691 cm_id->rem_ref(cm_id); 2966 cm_id->rem_ref(cm_id);
2692 c4iw_put_ep(&ep->com); 2967 c4iw_put_ep(&ep->com);
@@ -2704,20 +2979,24 @@ int c4iw_destroy_listen(struct iw_cm_id *cm_id)
2704 2979
2705 might_sleep(); 2980 might_sleep();
2706 state_set(&ep->com, DEAD); 2981 state_set(&ep->com, DEAD);
2707 if (ep->com.dev->rdev.lldi.enable_fw_ofld_conn) { 2982 if (ep->com.dev->rdev.lldi.enable_fw_ofld_conn &&
2983 ep->com.local_addr.ss_family == AF_INET) {
2708 err = cxgb4_remove_server_filter( 2984 err = cxgb4_remove_server_filter(
2709 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2985 ep->com.dev->rdev.lldi.ports[0], ep->stid,
2710 ep->com.dev->rdev.lldi.rxq_ids[0], 0); 2986 ep->com.dev->rdev.lldi.rxq_ids[0], 0);
2711 } else { 2987 } else {
2712 c4iw_init_wr_wait(&ep->com.wr_wait); 2988 c4iw_init_wr_wait(&ep->com.wr_wait);
2713 err = listen_stop(ep); 2989 err = cxgb4_remove_server(
2990 ep->com.dev->rdev.lldi.ports[0], ep->stid,
2991 ep->com.dev->rdev.lldi.rxq_ids[0], 0);
2714 if (err) 2992 if (err)
2715 goto done; 2993 goto done;
2716 err = c4iw_wait_for_reply(&ep->com.dev->rdev, &ep->com.wr_wait, 2994 err = c4iw_wait_for_reply(&ep->com.dev->rdev, &ep->com.wr_wait,
2717 0, 0, __func__); 2995 0, 0, __func__);
2718 } 2996 }
2719 remove_handle(ep->com.dev, &ep->com.dev->stid_idr, ep->stid); 2997 remove_handle(ep->com.dev, &ep->com.dev->stid_idr, ep->stid);
2720 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET); 2998 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid,
2999 ep->com.local_addr.ss_family);
2721done: 3000done:
2722 cm_id->rem_ref(cm_id); 3001 cm_id->rem_ref(cm_id);
2723 c4iw_put_ep(&ep->com); 3002 c4iw_put_ep(&ep->com);
@@ -3021,7 +3300,6 @@ static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb)
3021 struct cpl_pass_accept_req *req = (void *)(rss + 1); 3300 struct cpl_pass_accept_req *req = (void *)(rss + 1);
3022 struct l2t_entry *e; 3301 struct l2t_entry *e;
3023 struct dst_entry *dst; 3302 struct dst_entry *dst;
3024 struct rtable *rt;
3025 struct c4iw_ep *lep; 3303 struct c4iw_ep *lep;
3026 u16 window; 3304 u16 window;
3027 struct port_info *pi; 3305 struct port_info *pi;
@@ -3079,14 +3357,13 @@ static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb)
3079 ntohl(iph->daddr), ntohs(tcph->dest), ntohl(iph->saddr), 3357 ntohl(iph->daddr), ntohs(tcph->dest), ntohl(iph->saddr),
3080 ntohs(tcph->source), iph->tos); 3358 ntohs(tcph->source), iph->tos);
3081 3359
3082 rt = find_route(dev, iph->daddr, iph->saddr, tcph->dest, tcph->source, 3360 dst = find_route(dev, iph->daddr, iph->saddr, tcph->dest, tcph->source,
3083 iph->tos); 3361 iph->tos);
3084 if (!rt) { 3362 if (!dst) {
3085 pr_err("%s - failed to find dst entry!\n", 3363 pr_err("%s - failed to find dst entry!\n",
3086 __func__); 3364 __func__);
3087 goto reject; 3365 goto reject;
3088 } 3366 }
3089 dst = &rt->dst;
3090 neigh = dst_neigh_lookup_skb(dst, skb); 3367 neigh = dst_neigh_lookup_skb(dst, skb);
3091 3368
3092 if (!neigh) { 3369 if (!neigh) {
@@ -3103,10 +3380,11 @@ static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb)
3103 tx_chan = cxgb4_port_chan(pdev); 3380 tx_chan = cxgb4_port_chan(pdev);
3104 dev_put(pdev); 3381 dev_put(pdev);
3105 } else { 3382 } else {
3383 pdev = get_real_dev(neigh->dev);
3106 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, 3384 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh,
3107 neigh->dev, 0); 3385 pdev, 0);
3108 pi = (struct port_info *)netdev_priv(neigh->dev); 3386 pi = (struct port_info *)netdev_priv(pdev);
3109 tx_chan = cxgb4_port_chan(neigh->dev); 3387 tx_chan = cxgb4_port_chan(pdev);
3110 } 3388 }
3111 if (!e) { 3389 if (!e) {
3112 pr_err("%s - failed to allocate l2t entry!\n", 3390 pr_err("%s - failed to allocate l2t entry!\n",
diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c
index 0f1607c8325a..88de3aa9c5b0 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -225,43 +225,186 @@ static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq,
225 t4_swcq_produce(cq); 225 t4_swcq_produce(cq);
226} 226}
227 227
228int c4iw_flush_sq(struct t4_wq *wq, struct t4_cq *cq, int count) 228static void advance_oldest_read(struct t4_wq *wq);
229
230int c4iw_flush_sq(struct c4iw_qp *qhp)
229{ 231{
230 int flushed = 0; 232 int flushed = 0;
231 struct t4_swsqe *swsqe = &wq->sq.sw_sq[wq->sq.cidx + count]; 233 struct t4_wq *wq = &qhp->wq;
232 int in_use = wq->sq.in_use - count; 234 struct c4iw_cq *chp = to_c4iw_cq(qhp->ibqp.send_cq);
233 235 struct t4_cq *cq = &chp->cq;
234 BUG_ON(in_use < 0); 236 int idx;
235 while (in_use--) { 237 struct t4_swsqe *swsqe;
236 swsqe->signaled = 0; 238 int error = (qhp->attr.state != C4IW_QP_STATE_CLOSING &&
237 insert_sq_cqe(wq, cq, swsqe); 239 qhp->attr.state != C4IW_QP_STATE_IDLE);
238 swsqe++; 240
239 if (swsqe == (wq->sq.sw_sq + wq->sq.size)) 241 if (wq->sq.flush_cidx == -1)
240 swsqe = wq->sq.sw_sq; 242 wq->sq.flush_cidx = wq->sq.cidx;
241 flushed++; 243 idx = wq->sq.flush_cidx;
244 BUG_ON(idx >= wq->sq.size);
245 while (idx != wq->sq.pidx) {
246 if (error) {
247 swsqe = &wq->sq.sw_sq[idx];
248 BUG_ON(swsqe->flushed);
249 swsqe->flushed = 1;
250 insert_sq_cqe(wq, cq, swsqe);
251 if (wq->sq.oldest_read == swsqe) {
252 BUG_ON(swsqe->opcode != FW_RI_READ_REQ);
253 advance_oldest_read(wq);
254 }
255 flushed++;
256 } else {
257 t4_sq_consume(wq);
258 }
259 if (++idx == wq->sq.size)
260 idx = 0;
242 } 261 }
262 wq->sq.flush_cidx += flushed;
263 if (wq->sq.flush_cidx >= wq->sq.size)
264 wq->sq.flush_cidx -= wq->sq.size;
243 return flushed; 265 return flushed;
244} 266}
245 267
268static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq)
269{
270 struct t4_swsqe *swsqe;
271 int cidx;
272
273 if (wq->sq.flush_cidx == -1)
274 wq->sq.flush_cidx = wq->sq.cidx;
275 cidx = wq->sq.flush_cidx;
276 BUG_ON(cidx > wq->sq.size);
277
278 while (cidx != wq->sq.pidx) {
279 swsqe = &wq->sq.sw_sq[cidx];
280 if (!swsqe->signaled) {
281 if (++cidx == wq->sq.size)
282 cidx = 0;
283 } else if (swsqe->complete) {
284
285 BUG_ON(swsqe->flushed);
286
287 /*
288 * Insert this completed cqe into the swcq.
289 */
290 PDBG("%s moving cqe into swcq sq idx %u cq idx %u\n",
291 __func__, cidx, cq->sw_pidx);
292 swsqe->cqe.header |= htonl(V_CQE_SWCQE(1));
293 cq->sw_queue[cq->sw_pidx] = swsqe->cqe;
294 t4_swcq_produce(cq);
295 swsqe->flushed = 1;
296 if (++cidx == wq->sq.size)
297 cidx = 0;
298 wq->sq.flush_cidx = cidx;
299 } else
300 break;
301 }
302}
303
304static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe,
305 struct t4_cqe *read_cqe)
306{
307 read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx;
308 read_cqe->len = htonl(wq->sq.oldest_read->read_len);
309 read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(hw_cqe)) |
310 V_CQE_SWCQE(SW_CQE(hw_cqe)) |
311 V_CQE_OPCODE(FW_RI_READ_REQ) |
312 V_CQE_TYPE(1));
313 read_cqe->bits_type_ts = hw_cqe->bits_type_ts;
314}
315
316static void advance_oldest_read(struct t4_wq *wq)
317{
318
319 u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1;
320
321 if (rptr == wq->sq.size)
322 rptr = 0;
323 while (rptr != wq->sq.pidx) {
324 wq->sq.oldest_read = &wq->sq.sw_sq[rptr];
325
326 if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ)
327 return;
328 if (++rptr == wq->sq.size)
329 rptr = 0;
330 }
331 wq->sq.oldest_read = NULL;
332}
333
246/* 334/*
247 * Move all CQEs from the HWCQ into the SWCQ. 335 * Move all CQEs from the HWCQ into the SWCQ.
336 * Deal with out-of-order and/or completions that complete
337 * prior unsignalled WRs.
248 */ 338 */
249void c4iw_flush_hw_cq(struct t4_cq *cq) 339void c4iw_flush_hw_cq(struct c4iw_cq *chp)
250{ 340{
251 struct t4_cqe *cqe = NULL, *swcqe; 341 struct t4_cqe *hw_cqe, *swcqe, read_cqe;
342 struct c4iw_qp *qhp;
343 struct t4_swsqe *swsqe;
252 int ret; 344 int ret;
253 345
254 PDBG("%s cq %p cqid 0x%x\n", __func__, cq, cq->cqid); 346 PDBG("%s cqid 0x%x\n", __func__, chp->cq.cqid);
255 ret = t4_next_hw_cqe(cq, &cqe); 347 ret = t4_next_hw_cqe(&chp->cq, &hw_cqe);
348
349 /*
350 * This logic is similar to poll_cq(), but not quite the same
351 * unfortunately. Need to move pertinent HW CQEs to the SW CQ but
352 * also do any translation magic that poll_cq() normally does.
353 */
256 while (!ret) { 354 while (!ret) {
257 PDBG("%s flushing hwcq cidx 0x%x swcq pidx 0x%x\n", 355 qhp = get_qhp(chp->rhp, CQE_QPID(hw_cqe));
258 __func__, cq->cidx, cq->sw_pidx); 356
259 swcqe = &cq->sw_queue[cq->sw_pidx]; 357 /*
260 *swcqe = *cqe; 358 * drop CQEs with no associated QP
261 swcqe->header |= cpu_to_be32(V_CQE_SWCQE(1)); 359 */
262 t4_swcq_produce(cq); 360 if (qhp == NULL)
263 t4_hwcq_consume(cq); 361 goto next_cqe;
264 ret = t4_next_hw_cqe(cq, &cqe); 362
363 if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE)
364 goto next_cqe;
365
366 if (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP) {
367
368 /*
369 * drop peer2peer RTR reads.
370 */
371 if (CQE_WRID_STAG(hw_cqe) == 1)
372 goto next_cqe;
373
374 /*
375 * Eat completions for unsignaled read WRs.
376 */
377 if (!qhp->wq.sq.oldest_read->signaled) {
378 advance_oldest_read(&qhp->wq);
379 goto next_cqe;
380 }
381
382 /*
383 * Don't write to the HWCQ, create a new read req CQE
384 * in local memory and move it into the swcq.
385 */
386 create_read_req_cqe(&qhp->wq, hw_cqe, &read_cqe);
387 hw_cqe = &read_cqe;
388 advance_oldest_read(&qhp->wq);
389 }
390
391 /* if its a SQ completion, then do the magic to move all the
392 * unsignaled and now in-order completions into the swcq.
393 */
394 if (SQ_TYPE(hw_cqe)) {
395 swsqe = &qhp->wq.sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)];
396 swsqe->cqe = *hw_cqe;
397 swsqe->complete = 1;
398 flush_completed_wrs(&qhp->wq, &chp->cq);
399 } else {
400 swcqe = &chp->cq.sw_queue[chp->cq.sw_pidx];
401 *swcqe = *hw_cqe;
402 swcqe->header |= cpu_to_be32(V_CQE_SWCQE(1));
403 t4_swcq_produce(&chp->cq);
404 }
405next_cqe:
406 t4_hwcq_consume(&chp->cq);
407 ret = t4_next_hw_cqe(&chp->cq, &hw_cqe);
265 } 408 }
266} 409}
267 410
@@ -281,25 +424,6 @@ static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq)
281 return 1; 424 return 1;
282} 425}
283 426
284void c4iw_count_scqes(struct t4_cq *cq, struct t4_wq *wq, int *count)
285{
286 struct t4_cqe *cqe;
287 u32 ptr;
288
289 *count = 0;
290 ptr = cq->sw_cidx;
291 while (ptr != cq->sw_pidx) {
292 cqe = &cq->sw_queue[ptr];
293 if ((SQ_TYPE(cqe) || ((CQE_OPCODE(cqe) == FW_RI_READ_RESP) &&
294 wq->sq.oldest_read)) &&
295 (CQE_QPID(cqe) == wq->sq.qid))
296 (*count)++;
297 if (++ptr == cq->size)
298 ptr = 0;
299 }
300 PDBG("%s cq %p count %d\n", __func__, cq, *count);
301}
302
303void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count) 427void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)
304{ 428{
305 struct t4_cqe *cqe; 429 struct t4_cqe *cqe;
@@ -319,70 +443,6 @@ void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)
319 PDBG("%s cq %p count %d\n", __func__, cq, *count); 443 PDBG("%s cq %p count %d\n", __func__, cq, *count);
320} 444}
321 445
322static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq)
323{
324 struct t4_swsqe *swsqe;
325 u16 ptr = wq->sq.cidx;
326 int count = wq->sq.in_use;
327 int unsignaled = 0;
328
329 swsqe = &wq->sq.sw_sq[ptr];
330 while (count--)
331 if (!swsqe->signaled) {
332 if (++ptr == wq->sq.size)
333 ptr = 0;
334 swsqe = &wq->sq.sw_sq[ptr];
335 unsignaled++;
336 } else if (swsqe->complete) {
337
338 /*
339 * Insert this completed cqe into the swcq.
340 */
341 PDBG("%s moving cqe into swcq sq idx %u cq idx %u\n",
342 __func__, ptr, cq->sw_pidx);
343 swsqe->cqe.header |= htonl(V_CQE_SWCQE(1));
344 cq->sw_queue[cq->sw_pidx] = swsqe->cqe;
345 t4_swcq_produce(cq);
346 swsqe->signaled = 0;
347 wq->sq.in_use -= unsignaled;
348 break;
349 } else
350 break;
351}
352
353static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe,
354 struct t4_cqe *read_cqe)
355{
356 read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx;
357 read_cqe->len = cpu_to_be32(wq->sq.oldest_read->read_len);
358 read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(hw_cqe)) |
359 V_CQE_SWCQE(SW_CQE(hw_cqe)) |
360 V_CQE_OPCODE(FW_RI_READ_REQ) |
361 V_CQE_TYPE(1));
362 read_cqe->bits_type_ts = hw_cqe->bits_type_ts;
363}
364
365/*
366 * Return a ptr to the next read wr in the SWSQ or NULL.
367 */
368static void advance_oldest_read(struct t4_wq *wq)
369{
370
371 u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1;
372
373 if (rptr == wq->sq.size)
374 rptr = 0;
375 while (rptr != wq->sq.pidx) {
376 wq->sq.oldest_read = &wq->sq.sw_sq[rptr];
377
378 if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ)
379 return;
380 if (++rptr == wq->sq.size)
381 rptr = 0;
382 }
383 wq->sq.oldest_read = NULL;
384}
385
386/* 446/*
387 * poll_cq 447 * poll_cq
388 * 448 *
@@ -427,6 +487,22 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
427 } 487 }
428 488
429 /* 489 /*
490 * skip hw cqe's if the wq is flushed.
491 */
492 if (wq->flushed && !SW_CQE(hw_cqe)) {
493 ret = -EAGAIN;
494 goto skip_cqe;
495 }
496
497 /*
498 * skip TERMINATE cqes...
499 */
500 if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) {
501 ret = -EAGAIN;
502 goto skip_cqe;
503 }
504
505 /*
430 * Gotta tweak READ completions: 506 * Gotta tweak READ completions:
431 * 1) the cqe doesn't contain the sq_wptr from the wr. 507 * 1) the cqe doesn't contain the sq_wptr from the wr.
432 * 2) opcode not reflected from the wr. 508 * 2) opcode not reflected from the wr.
@@ -440,7 +516,7 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
440 * was generated by the kernel driver as part of peer-2-peer 516 * was generated by the kernel driver as part of peer-2-peer
441 * connection setup. So ignore the completion. 517 * connection setup. So ignore the completion.
442 */ 518 */
443 if (!wq->sq.oldest_read) { 519 if (CQE_WRID_STAG(hw_cqe) == 1) {
444 if (CQE_STATUS(hw_cqe)) 520 if (CQE_STATUS(hw_cqe))
445 t4_set_wq_in_error(wq); 521 t4_set_wq_in_error(wq);
446 ret = -EAGAIN; 522 ret = -EAGAIN;
@@ -448,6 +524,15 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
448 } 524 }
449 525
450 /* 526 /*
527 * Eat completions for unsignaled read WRs.
528 */
529 if (!wq->sq.oldest_read->signaled) {
530 advance_oldest_read(wq);
531 ret = -EAGAIN;
532 goto skip_cqe;
533 }
534
535 /*
451 * Don't write to the HWCQ, so create a new read req CQE 536 * Don't write to the HWCQ, so create a new read req CQE
452 * in local memory. 537 * in local memory.
453 */ 538 */
@@ -457,14 +542,8 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
457 } 542 }
458 543
459 if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) { 544 if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) {
460 *cqe_flushed = t4_wq_in_error(wq); 545 *cqe_flushed = (CQE_STATUS(hw_cqe) == T4_ERR_SWFLUSH);
461 t4_set_wq_in_error(wq); 546 t4_set_wq_in_error(wq);
462 goto proc_cqe;
463 }
464
465 if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) {
466 ret = -EAGAIN;
467 goto skip_cqe;
468 } 547 }
469 548
470 /* 549 /*
@@ -523,7 +602,24 @@ proc_cqe:
523 * completion. 602 * completion.
524 */ 603 */
525 if (SQ_TYPE(hw_cqe)) { 604 if (SQ_TYPE(hw_cqe)) {
526 wq->sq.cidx = CQE_WRID_SQ_IDX(hw_cqe); 605 int idx = CQE_WRID_SQ_IDX(hw_cqe);
606 BUG_ON(idx > wq->sq.size);
607
608 /*
609 * Account for any unsignaled completions completed by
610 * this signaled completion. In this case, cidx points
611 * to the first unsignaled one, and idx points to the
612 * signaled one. So adjust in_use based on this delta.
613 * if this is not completing any unsigned wrs, then the
614 * delta will be 0. Handle wrapping also!
615 */
616 if (idx < wq->sq.cidx)
617 wq->sq.in_use -= wq->sq.size + idx - wq->sq.cidx;
618 else
619 wq->sq.in_use -= idx - wq->sq.cidx;
620 BUG_ON(wq->sq.in_use < 0 && wq->sq.in_use < wq->sq.size);
621
622 wq->sq.cidx = (uint16_t)idx;
527 PDBG("%s completing sq idx %u\n", __func__, wq->sq.cidx); 623 PDBG("%s completing sq idx %u\n", __func__, wq->sq.cidx);
528 *cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id; 624 *cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id;
529 t4_sq_consume(wq); 625 t4_sq_consume(wq);
@@ -532,6 +628,7 @@ proc_cqe:
532 *cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id; 628 *cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id;
533 BUG_ON(t4_rq_empty(wq)); 629 BUG_ON(t4_rq_empty(wq));
534 t4_rq_consume(wq); 630 t4_rq_consume(wq);
631 goto skip_cqe;
535 } 632 }
536 633
537flush_wq: 634flush_wq:
diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c
index ae656016e1ae..33d2cc6ab562 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -103,18 +103,43 @@ static int dump_qp(int id, void *p, void *data)
103 if (space == 0) 103 if (space == 0)
104 return 1; 104 return 1;
105 105
106 if (qp->ep) 106 if (qp->ep) {
107 cc = snprintf(qpd->buf + qpd->pos, space, 107 if (qp->ep->com.local_addr.ss_family == AF_INET) {
108 "qp sq id %u rq id %u state %u onchip %u " 108 struct sockaddr_in *lsin = (struct sockaddr_in *)
109 "ep tid %u state %u %pI4:%u->%pI4:%u\n", 109 &qp->ep->com.local_addr;
110 qp->wq.sq.qid, qp->wq.rq.qid, (int)qp->attr.state, 110 struct sockaddr_in *rsin = (struct sockaddr_in *)
111 qp->wq.sq.flags & T4_SQ_ONCHIP, 111 &qp->ep->com.remote_addr;
112 qp->ep->hwtid, (int)qp->ep->com.state, 112
113 &qp->ep->com.local_addr.sin_addr.s_addr, 113 cc = snprintf(qpd->buf + qpd->pos, space,
114 ntohs(qp->ep->com.local_addr.sin_port), 114 "rc qp sq id %u rq id %u state %u "
115 &qp->ep->com.remote_addr.sin_addr.s_addr, 115 "onchip %u ep tid %u state %u "
116 ntohs(qp->ep->com.remote_addr.sin_port)); 116 "%pI4:%u->%pI4:%u\n",
117 else 117 qp->wq.sq.qid, qp->wq.rq.qid,
118 (int)qp->attr.state,
119 qp->wq.sq.flags & T4_SQ_ONCHIP,
120 qp->ep->hwtid, (int)qp->ep->com.state,
121 &lsin->sin_addr, ntohs(lsin->sin_port),
122 &rsin->sin_addr, ntohs(rsin->sin_port));
123 } else {
124 struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *)
125 &qp->ep->com.local_addr;
126 struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *)
127 &qp->ep->com.remote_addr;
128
129 cc = snprintf(qpd->buf + qpd->pos, space,
130 "rc qp sq id %u rq id %u state %u "
131 "onchip %u ep tid %u state %u "
132 "%pI6:%u->%pI6:%u\n",
133 qp->wq.sq.qid, qp->wq.rq.qid,
134 (int)qp->attr.state,
135 qp->wq.sq.flags & T4_SQ_ONCHIP,
136 qp->ep->hwtid, (int)qp->ep->com.state,
137 &lsin6->sin6_addr,
138 ntohs(lsin6->sin6_port),
139 &rsin6->sin6_addr,
140 ntohs(rsin6->sin6_port));
141 }
142 } else
118 cc = snprintf(qpd->buf + qpd->pos, space, 143 cc = snprintf(qpd->buf + qpd->pos, space,
119 "qp sq id %u rq id %u state %u onchip %u\n", 144 "qp sq id %u rq id %u state %u onchip %u\n",
120 qp->wq.sq.qid, qp->wq.rq.qid, 145 qp->wq.sq.qid, qp->wq.rq.qid,
@@ -351,15 +376,37 @@ static int dump_ep(int id, void *p, void *data)
351 if (space == 0) 376 if (space == 0)
352 return 1; 377 return 1;
353 378
354 cc = snprintf(epd->buf + epd->pos, space, 379 if (ep->com.local_addr.ss_family == AF_INET) {
355 "ep %p cm_id %p qp %p state %d flags 0x%lx history 0x%lx " 380 struct sockaddr_in *lsin = (struct sockaddr_in *)
356 "hwtid %d atid %d %pI4:%d <-> %pI4:%d\n", 381 &ep->com.local_addr;
357 ep, ep->com.cm_id, ep->com.qp, (int)ep->com.state, 382 struct sockaddr_in *rsin = (struct sockaddr_in *)
358 ep->com.flags, ep->com.history, ep->hwtid, ep->atid, 383 &ep->com.remote_addr;
359 &ep->com.local_addr.sin_addr.s_addr, 384
360 ntohs(ep->com.local_addr.sin_port), 385 cc = snprintf(epd->buf + epd->pos, space,
361 &ep->com.remote_addr.sin_addr.s_addr, 386 "ep %p cm_id %p qp %p state %d flags 0x%lx "
362 ntohs(ep->com.remote_addr.sin_port)); 387 "history 0x%lx hwtid %d atid %d "
388 "%pI4:%d <-> %pI4:%d\n",
389 ep, ep->com.cm_id, ep->com.qp,
390 (int)ep->com.state, ep->com.flags,
391 ep->com.history, ep->hwtid, ep->atid,
392 &lsin->sin_addr, ntohs(lsin->sin_port),
393 &rsin->sin_addr, ntohs(rsin->sin_port));
394 } else {
395 struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *)
396 &ep->com.local_addr;
397 struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *)
398 &ep->com.remote_addr;
399
400 cc = snprintf(epd->buf + epd->pos, space,
401 "ep %p cm_id %p qp %p state %d flags 0x%lx "
402 "history 0x%lx hwtid %d atid %d "
403 "%pI6:%d <-> %pI6:%d\n",
404 ep, ep->com.cm_id, ep->com.qp,
405 (int)ep->com.state, ep->com.flags,
406 ep->com.history, ep->hwtid, ep->atid,
407 &lsin6->sin6_addr, ntohs(lsin6->sin6_port),
408 &rsin6->sin6_addr, ntohs(rsin6->sin6_port));
409 }
363 if (cc < space) 410 if (cc < space)
364 epd->pos += cc; 411 epd->pos += cc;
365 return 0; 412 return 0;
@@ -376,12 +423,27 @@ static int dump_listen_ep(int id, void *p, void *data)
376 if (space == 0) 423 if (space == 0)
377 return 1; 424 return 1;
378 425
379 cc = snprintf(epd->buf + epd->pos, space, 426 if (ep->com.local_addr.ss_family == AF_INET) {
380 "ep %p cm_id %p state %d flags 0x%lx stid %d backlog %d " 427 struct sockaddr_in *lsin = (struct sockaddr_in *)
381 "%pI4:%d\n", ep, ep->com.cm_id, (int)ep->com.state, 428 &ep->com.local_addr;
382 ep->com.flags, ep->stid, ep->backlog, 429
383 &ep->com.local_addr.sin_addr.s_addr, 430 cc = snprintf(epd->buf + epd->pos, space,
384 ntohs(ep->com.local_addr.sin_port)); 431 "ep %p cm_id %p state %d flags 0x%lx stid %d "
432 "backlog %d %pI4:%d\n",
433 ep, ep->com.cm_id, (int)ep->com.state,
434 ep->com.flags, ep->stid, ep->backlog,
435 &lsin->sin_addr, ntohs(lsin->sin_port));
436 } else {
437 struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *)
438 &ep->com.local_addr;
439
440 cc = snprintf(epd->buf + epd->pos, space,
441 "ep %p cm_id %p state %d flags 0x%lx stid %d "
442 "backlog %d %pI6:%d\n",
443 ep, ep->com.cm_id, (int)ep->com.state,
444 ep->com.flags, ep->stid, ep->backlog,
445 &lsin6->sin6_addr, ntohs(lsin6->sin6_port));
446 }
385 if (cc < space) 447 if (cc < space)
386 epd->pos += cc; 448 epd->pos += cc;
387 return 0; 449 return 0;
diff --git a/drivers/infiniband/hw/cxgb4/ev.c b/drivers/infiniband/hw/cxgb4/ev.c
index 1a840b2211dd..d61d0a18f784 100644
--- a/drivers/infiniband/hw/cxgb4/ev.c
+++ b/drivers/infiniband/hw/cxgb4/ev.c
@@ -44,16 +44,6 @@ static void post_qp_event(struct c4iw_dev *dev, struct c4iw_cq *chp,
44 struct c4iw_qp_attributes attrs; 44 struct c4iw_qp_attributes attrs;
45 unsigned long flag; 45 unsigned long flag;
46 46
47 if ((qhp->attr.state == C4IW_QP_STATE_ERROR) ||
48 (qhp->attr.state == C4IW_QP_STATE_TERMINATE)) {
49 pr_err("%s AE after RTS - qpid 0x%x opcode %d status 0x%x "\
50 "type %d wrid.hi 0x%x wrid.lo 0x%x\n",
51 __func__, CQE_QPID(err_cqe), CQE_OPCODE(err_cqe),
52 CQE_STATUS(err_cqe), CQE_TYPE(err_cqe),
53 CQE_WRID_HI(err_cqe), CQE_WRID_LOW(err_cqe));
54 return;
55 }
56
57 printk(KERN_ERR MOD "AE qpid 0x%x opcode %d status 0x%x " 47 printk(KERN_ERR MOD "AE qpid 0x%x opcode %d status 0x%x "
58 "type %d wrid.hi 0x%x wrid.lo 0x%x\n", 48 "type %d wrid.hi 0x%x wrid.lo 0x%x\n",
59 CQE_QPID(err_cqe), CQE_OPCODE(err_cqe), 49 CQE_QPID(err_cqe), CQE_OPCODE(err_cqe),
diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
index 485183ad34cd..23eaeabab93b 100644
--- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
+++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
@@ -752,8 +752,8 @@ struct c4iw_ep_common {
752 enum c4iw_ep_state state; 752 enum c4iw_ep_state state;
753 struct kref kref; 753 struct kref kref;
754 struct mutex mutex; 754 struct mutex mutex;
755 struct sockaddr_in local_addr; 755 struct sockaddr_storage local_addr;
756 struct sockaddr_in remote_addr; 756 struct sockaddr_storage remote_addr;
757 struct c4iw_wr_wait wr_wait; 757 struct c4iw_wr_wait wr_wait;
758 unsigned long flags; 758 unsigned long flags;
759 unsigned long history; 759 unsigned long history;
@@ -917,12 +917,11 @@ void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
917u32 c4iw_ocqp_pool_alloc(struct c4iw_rdev *rdev, int size); 917u32 c4iw_ocqp_pool_alloc(struct c4iw_rdev *rdev, int size);
918void c4iw_ocqp_pool_free(struct c4iw_rdev *rdev, u32 addr, int size); 918void c4iw_ocqp_pool_free(struct c4iw_rdev *rdev, u32 addr, int size);
919int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb); 919int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb);
920void c4iw_flush_hw_cq(struct t4_cq *cq); 920void c4iw_flush_hw_cq(struct c4iw_cq *chp);
921void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count); 921void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
922void c4iw_count_scqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
923int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp); 922int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);
924int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count); 923int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count);
925int c4iw_flush_sq(struct t4_wq *wq, struct t4_cq *cq, int count); 924int c4iw_flush_sq(struct c4iw_qp *qhp);
926int c4iw_ev_handler(struct c4iw_dev *rnicp, u32 qid); 925int c4iw_ev_handler(struct c4iw_dev *rnicp, u32 qid);
927u16 c4iw_rqes_posted(struct c4iw_qp *qhp); 926u16 c4iw_rqes_posted(struct c4iw_qp *qhp);
928int c4iw_post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe); 927int c4iw_post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe);
diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c
index a4975e1654a6..582936708e6e 100644
--- a/drivers/infiniband/hw/cxgb4/qp.c
+++ b/drivers/infiniband/hw/cxgb4/qp.c
@@ -737,6 +737,7 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
737 swsqe->idx = qhp->wq.sq.pidx; 737 swsqe->idx = qhp->wq.sq.pidx;
738 swsqe->complete = 0; 738 swsqe->complete = 0;
739 swsqe->signaled = (wr->send_flags & IB_SEND_SIGNALED); 739 swsqe->signaled = (wr->send_flags & IB_SEND_SIGNALED);
740 swsqe->flushed = 0;
740 swsqe->wr_id = wr->wr_id; 741 swsqe->wr_id = wr->wr_id;
741 742
742 init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16); 743 init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16);
@@ -1006,7 +1007,15 @@ static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp,
1006 /* locking hierarchy: cq lock first, then qp lock. */ 1007 /* locking hierarchy: cq lock first, then qp lock. */
1007 spin_lock_irqsave(&rchp->lock, flag); 1008 spin_lock_irqsave(&rchp->lock, flag);
1008 spin_lock(&qhp->lock); 1009 spin_lock(&qhp->lock);
1009 c4iw_flush_hw_cq(&rchp->cq); 1010
1011 if (qhp->wq.flushed) {
1012 spin_unlock(&qhp->lock);
1013 spin_unlock_irqrestore(&rchp->lock, flag);
1014 return;
1015 }
1016 qhp->wq.flushed = 1;
1017
1018 c4iw_flush_hw_cq(rchp);
1010 c4iw_count_rcqes(&rchp->cq, &qhp->wq, &count); 1019 c4iw_count_rcqes(&rchp->cq, &qhp->wq, &count);
1011 flushed = c4iw_flush_rq(&qhp->wq, &rchp->cq, count); 1020 flushed = c4iw_flush_rq(&qhp->wq, &rchp->cq, count);
1012 spin_unlock(&qhp->lock); 1021 spin_unlock(&qhp->lock);
@@ -1020,9 +1029,9 @@ static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp,
1020 /* locking hierarchy: cq lock first, then qp lock. */ 1029 /* locking hierarchy: cq lock first, then qp lock. */
1021 spin_lock_irqsave(&schp->lock, flag); 1030 spin_lock_irqsave(&schp->lock, flag);
1022 spin_lock(&qhp->lock); 1031 spin_lock(&qhp->lock);
1023 c4iw_flush_hw_cq(&schp->cq); 1032 if (schp != rchp)
1024 c4iw_count_scqes(&schp->cq, &qhp->wq, &count); 1033 c4iw_flush_hw_cq(schp);
1025 flushed = c4iw_flush_sq(&qhp->wq, &schp->cq, count); 1034 flushed = c4iw_flush_sq(qhp);
1026 spin_unlock(&qhp->lock); 1035 spin_unlock(&qhp->lock);
1027 spin_unlock_irqrestore(&schp->lock, flag); 1036 spin_unlock_irqrestore(&schp->lock, flag);
1028 if (flushed) { 1037 if (flushed) {
@@ -1037,11 +1046,11 @@ static void flush_qp(struct c4iw_qp *qhp)
1037 struct c4iw_cq *rchp, *schp; 1046 struct c4iw_cq *rchp, *schp;
1038 unsigned long flag; 1047 unsigned long flag;
1039 1048
1040 rchp = get_chp(qhp->rhp, qhp->attr.rcq); 1049 rchp = to_c4iw_cq(qhp->ibqp.recv_cq);
1041 schp = get_chp(qhp->rhp, qhp->attr.scq); 1050 schp = to_c4iw_cq(qhp->ibqp.send_cq);
1042 1051
1052 t4_set_wq_in_error(&qhp->wq);
1043 if (qhp->ibqp.uobject) { 1053 if (qhp->ibqp.uobject) {
1044 t4_set_wq_in_error(&qhp->wq);
1045 t4_set_cq_in_error(&rchp->cq); 1054 t4_set_cq_in_error(&rchp->cq);
1046 spin_lock_irqsave(&rchp->comp_handler_lock, flag); 1055 spin_lock_irqsave(&rchp->comp_handler_lock, flag);
1047 (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); 1056 (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context);
@@ -1330,8 +1339,7 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
1330 disconnect = 1; 1339 disconnect = 1;
1331 c4iw_get_ep(&qhp->ep->com); 1340 c4iw_get_ep(&qhp->ep->com);
1332 } 1341 }
1333 if (qhp->ibqp.uobject) 1342 t4_set_wq_in_error(&qhp->wq);
1334 t4_set_wq_in_error(&qhp->wq);
1335 ret = rdma_fini(rhp, qhp, ep); 1343 ret = rdma_fini(rhp, qhp, ep);
1336 if (ret) 1344 if (ret)
1337 goto err; 1345 goto err;
@@ -1340,18 +1348,21 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
1340 set_state(qhp, C4IW_QP_STATE_TERMINATE); 1348 set_state(qhp, C4IW_QP_STATE_TERMINATE);
1341 qhp->attr.layer_etype = attrs->layer_etype; 1349 qhp->attr.layer_etype = attrs->layer_etype;
1342 qhp->attr.ecode = attrs->ecode; 1350 qhp->attr.ecode = attrs->ecode;
1343 if (qhp->ibqp.uobject) 1351 t4_set_wq_in_error(&qhp->wq);
1344 t4_set_wq_in_error(&qhp->wq);
1345 ep = qhp->ep; 1352 ep = qhp->ep;
1353 disconnect = 1;
1346 if (!internal) 1354 if (!internal)
1347 terminate = 1; 1355 terminate = 1;
1348 disconnect = 1; 1356 else {
1357 ret = rdma_fini(rhp, qhp, ep);
1358 if (ret)
1359 goto err;
1360 }
1349 c4iw_get_ep(&qhp->ep->com); 1361 c4iw_get_ep(&qhp->ep->com);
1350 break; 1362 break;
1351 case C4IW_QP_STATE_ERROR: 1363 case C4IW_QP_STATE_ERROR:
1352 set_state(qhp, C4IW_QP_STATE_ERROR); 1364 set_state(qhp, C4IW_QP_STATE_ERROR);
1353 if (qhp->ibqp.uobject) 1365 t4_set_wq_in_error(&qhp->wq);
1354 t4_set_wq_in_error(&qhp->wq);
1355 if (!internal) { 1366 if (!internal) {
1356 abort = 1; 1367 abort = 1;
1357 disconnect = 1; 1368 disconnect = 1;
@@ -1552,12 +1563,12 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
1552 1563
1553 ucontext = pd->uobject ? to_c4iw_ucontext(pd->uobject->context) : NULL; 1564 ucontext = pd->uobject ? to_c4iw_ucontext(pd->uobject->context) : NULL;
1554 1565
1555
1556 qhp = kzalloc(sizeof(*qhp), GFP_KERNEL); 1566 qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
1557 if (!qhp) 1567 if (!qhp)
1558 return ERR_PTR(-ENOMEM); 1568 return ERR_PTR(-ENOMEM);
1559 qhp->wq.sq.size = sqsize; 1569 qhp->wq.sq.size = sqsize;
1560 qhp->wq.sq.memsize = (sqsize + 1) * sizeof *qhp->wq.sq.queue; 1570 qhp->wq.sq.memsize = (sqsize + 1) * sizeof *qhp->wq.sq.queue;
1571 qhp->wq.sq.flush_cidx = -1;
1561 qhp->wq.rq.size = rqsize; 1572 qhp->wq.rq.size = rqsize;
1562 qhp->wq.rq.memsize = (rqsize + 1) * sizeof *qhp->wq.rq.queue; 1573 qhp->wq.rq.memsize = (rqsize + 1) * sizeof *qhp->wq.rq.queue;
1563 1574
diff --git a/drivers/infiniband/hw/cxgb4/t4.h b/drivers/infiniband/hw/cxgb4/t4.h
index ebcb03bd1b72..e73ace739183 100644
--- a/drivers/infiniband/hw/cxgb4/t4.h
+++ b/drivers/infiniband/hw/cxgb4/t4.h
@@ -36,9 +36,9 @@
36#include "t4_msg.h" 36#include "t4_msg.h"
37#include "t4fw_ri_api.h" 37#include "t4fw_ri_api.h"
38 38
39#define T4_MAX_NUM_QP (1<<16) 39#define T4_MAX_NUM_QP 65536
40#define T4_MAX_NUM_CQ (1<<15) 40#define T4_MAX_NUM_CQ 65536
41#define T4_MAX_NUM_PD (1<<15) 41#define T4_MAX_NUM_PD 65536
42#define T4_EQ_STATUS_ENTRIES (L1_CACHE_BYTES > 64 ? 2 : 1) 42#define T4_EQ_STATUS_ENTRIES (L1_CACHE_BYTES > 64 ? 2 : 1)
43#define T4_MAX_EQ_SIZE (65520 - T4_EQ_STATUS_ENTRIES) 43#define T4_MAX_EQ_SIZE (65520 - T4_EQ_STATUS_ENTRIES)
44#define T4_MAX_IQ_SIZE (65520 - 1) 44#define T4_MAX_IQ_SIZE (65520 - 1)
@@ -47,7 +47,7 @@
47#define T4_MAX_QP_DEPTH (T4_MAX_RQ_SIZE - 1) 47#define T4_MAX_QP_DEPTH (T4_MAX_RQ_SIZE - 1)
48#define T4_MAX_CQ_DEPTH (T4_MAX_IQ_SIZE - 1) 48#define T4_MAX_CQ_DEPTH (T4_MAX_IQ_SIZE - 1)
49#define T4_MAX_NUM_STAG (1<<15) 49#define T4_MAX_NUM_STAG (1<<15)
50#define T4_MAX_MR_SIZE (~0ULL - 1) 50#define T4_MAX_MR_SIZE (~0ULL)
51#define T4_PAGESIZE_MASK 0xffff000 /* 4KB-128MB */ 51#define T4_PAGESIZE_MASK 0xffff000 /* 4KB-128MB */
52#define T4_STAG_UNSET 0xffffffff 52#define T4_STAG_UNSET 0xffffffff
53#define T4_FW_MAJ 0 53#define T4_FW_MAJ 0
@@ -269,6 +269,7 @@ struct t4_swsqe {
269 int complete; 269 int complete;
270 int signaled; 270 int signaled;
271 u16 idx; 271 u16 idx;
272 int flushed;
272}; 273};
273 274
274static inline pgprot_t t4_pgprot_wc(pgprot_t prot) 275static inline pgprot_t t4_pgprot_wc(pgprot_t prot)
@@ -300,6 +301,7 @@ struct t4_sq {
300 u16 pidx; 301 u16 pidx;
301 u16 wq_pidx; 302 u16 wq_pidx;
302 u16 flags; 303 u16 flags;
304 short flush_cidx;
303}; 305};
304 306
305struct t4_swrqe { 307struct t4_swrqe {
@@ -330,6 +332,7 @@ struct t4_wq {
330 void __iomem *db; 332 void __iomem *db;
331 void __iomem *gts; 333 void __iomem *gts;
332 struct c4iw_rdev *rdev; 334 struct c4iw_rdev *rdev;
335 int flushed;
333}; 336};
334 337
335static inline int t4_rqes_posted(struct t4_wq *wq) 338static inline int t4_rqes_posted(struct t4_wq *wq)
@@ -412,6 +415,9 @@ static inline void t4_sq_produce(struct t4_wq *wq, u8 len16)
412 415
413static inline void t4_sq_consume(struct t4_wq *wq) 416static inline void t4_sq_consume(struct t4_wq *wq)
414{ 417{
418 BUG_ON(wq->sq.in_use < 1);
419 if (wq->sq.cidx == wq->sq.flush_cidx)
420 wq->sq.flush_cidx = -1;
415 wq->sq.in_use--; 421 wq->sq.in_use--;
416 if (++wq->sq.cidx == wq->sq.size) 422 if (++wq->sq.cidx == wq->sq.size)
417 wq->sq.cidx = 0; 423 wq->sq.cidx = 0;
@@ -505,12 +511,18 @@ static inline int t4_arm_cq(struct t4_cq *cq, int se)
505static inline void t4_swcq_produce(struct t4_cq *cq) 511static inline void t4_swcq_produce(struct t4_cq *cq)
506{ 512{
507 cq->sw_in_use++; 513 cq->sw_in_use++;
514 if (cq->sw_in_use == cq->size) {
515 PDBG("%s cxgb4 sw cq overflow cqid %u\n", __func__, cq->cqid);
516 cq->error = 1;
517 BUG_ON(1);
518 }
508 if (++cq->sw_pidx == cq->size) 519 if (++cq->sw_pidx == cq->size)
509 cq->sw_pidx = 0; 520 cq->sw_pidx = 0;
510} 521}
511 522
512static inline void t4_swcq_consume(struct t4_cq *cq) 523static inline void t4_swcq_consume(struct t4_cq *cq)
513{ 524{
525 BUG_ON(cq->sw_in_use < 1);
514 cq->sw_in_use--; 526 cq->sw_in_use--;
515 if (++cq->sw_cidx == cq->size) 527 if (++cq->sw_cidx == cq->size)
516 cq->sw_cidx = 0; 528 cq->sw_cidx = 0;
@@ -519,7 +531,7 @@ static inline void t4_swcq_consume(struct t4_cq *cq)
519static inline void t4_hwcq_consume(struct t4_cq *cq) 531static inline void t4_hwcq_consume(struct t4_cq *cq)
520{ 532{
521 cq->bits_type_ts = cq->queue[cq->cidx].bits_type_ts; 533 cq->bits_type_ts = cq->queue[cq->cidx].bits_type_ts;
522 if (++cq->cidx_inc == (cq->size >> 4)) { 534 if (++cq->cidx_inc == (cq->size >> 4) || cq->cidx_inc == CIDXINC_MASK) {
523 u32 val; 535 u32 val;
524 536
525 val = SEINTARM(0) | CIDXINC(cq->cidx_inc) | TIMERREG(7) | 537 val = SEINTARM(0) | CIDXINC(cq->cidx_inc) | TIMERREG(7) |
@@ -552,6 +564,7 @@ static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
552 ret = -EOVERFLOW; 564 ret = -EOVERFLOW;
553 cq->error = 1; 565 cq->error = 1;
554 printk(KERN_ERR MOD "cq overflow cqid %u\n", cq->cqid); 566 printk(KERN_ERR MOD "cq overflow cqid %u\n", cq->cqid);
567 BUG_ON(1);
555 } else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) { 568 } else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) {
556 *cqe = &cq->queue[cq->cidx]; 569 *cqe = &cq->queue[cq->cidx];
557 ret = 0; 570 ret = 0;
@@ -562,6 +575,12 @@ static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
562 575
563static inline struct t4_cqe *t4_next_sw_cqe(struct t4_cq *cq) 576static inline struct t4_cqe *t4_next_sw_cqe(struct t4_cq *cq)
564{ 577{
578 if (cq->sw_in_use == cq->size) {
579 PDBG("%s cxgb4 sw cq overflow cqid %u\n", __func__, cq->cqid);
580 cq->error = 1;
581 BUG_ON(1);
582 return NULL;
583 }
565 if (cq->sw_in_use) 584 if (cq->sw_in_use)
566 return &cq->sw_queue[cq->sw_cidx]; 585 return &cq->sw_queue[cq->sw_cidx];
567 return NULL; 586 return NULL;
diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c
index 24b9f1a0107b..6b29249aa85a 100644
--- a/drivers/infiniband/hw/nes/nes_cm.c
+++ b/drivers/infiniband/hw/nes/nes_cm.c
@@ -2998,6 +2998,8 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
2998 u8 *start_ptr = &start_addr; 2998 u8 *start_ptr = &start_addr;
2999 u8 **start_buff = &start_ptr; 2999 u8 **start_buff = &start_ptr;
3000 u16 buff_len = 0; 3000 u16 buff_len = 0;
3001 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr;
3002 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr;
3001 3003
3002 ibqp = nes_get_qp(cm_id->device, conn_param->qpn); 3004 ibqp = nes_get_qp(cm_id->device, conn_param->qpn);
3003 if (!ibqp) 3005 if (!ibqp)
@@ -3062,8 +3064,7 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3062 /* setup our first outgoing iWarp send WQE (the IETF frame response) */ 3064 /* setup our first outgoing iWarp send WQE (the IETF frame response) */
3063 wqe = &nesqp->hwqp.sq_vbase[0]; 3065 wqe = &nesqp->hwqp.sq_vbase[0];
3064 3066
3065 if (cm_id->remote_addr.sin_addr.s_addr != 3067 if (raddr->sin_addr.s_addr != laddr->sin_addr.s_addr) {
3066 cm_id->local_addr.sin_addr.s_addr) {
3067 u64temp = (unsigned long)nesqp; 3068 u64temp = (unsigned long)nesqp;
3068 nesibdev = nesvnic->nesibdev; 3069 nesibdev = nesvnic->nesibdev;
3069 nespd = nesqp->nespd; 3070 nespd = nesqp->nespd;
@@ -3132,13 +3133,10 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3132 3133
3133 nes_cm_init_tsa_conn(nesqp, cm_node); 3134 nes_cm_init_tsa_conn(nesqp, cm_node);
3134 3135
3135 nesqp->nesqp_context->tcpPorts[0] = 3136 nesqp->nesqp_context->tcpPorts[0] = cpu_to_le16(ntohs(laddr->sin_port));
3136 cpu_to_le16(ntohs(cm_id->local_addr.sin_port)); 3137 nesqp->nesqp_context->tcpPorts[1] = cpu_to_le16(ntohs(raddr->sin_port));
3137 nesqp->nesqp_context->tcpPorts[1] =
3138 cpu_to_le16(ntohs(cm_id->remote_addr.sin_port));
3139 3138
3140 nesqp->nesqp_context->ip0 = 3139 nesqp->nesqp_context->ip0 = cpu_to_le32(ntohl(raddr->sin_addr.s_addr));
3141 cpu_to_le32(ntohl(cm_id->remote_addr.sin_addr.s_addr));
3142 3140
3143 nesqp->nesqp_context->misc2 |= cpu_to_le32( 3141 nesqp->nesqp_context->misc2 |= cpu_to_le32(
3144 (u32)PCI_FUNC(nesdev->pcidev->devfn) << 3142 (u32)PCI_FUNC(nesdev->pcidev->devfn) <<
@@ -3162,9 +3160,9 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3162 memset(&nes_quad, 0, sizeof(nes_quad)); 3160 memset(&nes_quad, 0, sizeof(nes_quad));
3163 nes_quad.DstIpAdrIndex = 3161 nes_quad.DstIpAdrIndex =
3164 cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24); 3162 cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24);
3165 nes_quad.SrcIpadr = cm_id->remote_addr.sin_addr.s_addr; 3163 nes_quad.SrcIpadr = raddr->sin_addr.s_addr;
3166 nes_quad.TcpPorts[0] = cm_id->remote_addr.sin_port; 3164 nes_quad.TcpPorts[0] = raddr->sin_port;
3167 nes_quad.TcpPorts[1] = cm_id->local_addr.sin_port; 3165 nes_quad.TcpPorts[1] = laddr->sin_port;
3168 3166
3169 /* Produce hash key */ 3167 /* Produce hash key */
3170 crc_value = get_crc_value(&nes_quad); 3168 crc_value = get_crc_value(&nes_quad);
@@ -3180,10 +3178,8 @@ int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3180 nes_debug(NES_DBG_CM, "QP%u, Destination IP = 0x%08X:0x%04X, local = " 3178 nes_debug(NES_DBG_CM, "QP%u, Destination IP = 0x%08X:0x%04X, local = "
3181 "0x%08X:0x%04X, rcv_nxt=0x%08X, snd_nxt=0x%08X, mpa + " 3179 "0x%08X:0x%04X, rcv_nxt=0x%08X, snd_nxt=0x%08X, mpa + "
3182 "private data length=%u.\n", nesqp->hwqp.qp_id, 3180 "private data length=%u.\n", nesqp->hwqp.qp_id,
3183 ntohl(cm_id->remote_addr.sin_addr.s_addr), 3181 ntohl(raddr->sin_addr.s_addr), ntohs(raddr->sin_port),
3184 ntohs(cm_id->remote_addr.sin_port), 3182 ntohl(laddr->sin_addr.s_addr), ntohs(laddr->sin_port),
3185 ntohl(cm_id->local_addr.sin_addr.s_addr),
3186 ntohs(cm_id->local_addr.sin_port),
3187 le32_to_cpu(nesqp->nesqp_context->rcv_nxt), 3183 le32_to_cpu(nesqp->nesqp_context->rcv_nxt),
3188 le32_to_cpu(nesqp->nesqp_context->snd_nxt), 3184 le32_to_cpu(nesqp->nesqp_context->snd_nxt),
3189 buff_len); 3185 buff_len);
@@ -3263,7 +3259,11 @@ int nes_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3263 struct nes_cm_node *cm_node; 3259 struct nes_cm_node *cm_node;
3264 struct nes_cm_info cm_info; 3260 struct nes_cm_info cm_info;
3265 int apbvt_set = 0; 3261 int apbvt_set = 0;
3262 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr;
3263 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr;
3266 3264
3265 if (cm_id->remote_addr.ss_family != AF_INET)
3266 return -ENOSYS;
3267 ibqp = nes_get_qp(cm_id->device, conn_param->qpn); 3267 ibqp = nes_get_qp(cm_id->device, conn_param->qpn);
3268 if (!ibqp) 3268 if (!ibqp)
3269 return -EINVAL; 3269 return -EINVAL;
@@ -3277,16 +3277,14 @@ int nes_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3277 if (!nesdev) 3277 if (!nesdev)
3278 return -EINVAL; 3278 return -EINVAL;
3279 3279
3280 if (!(cm_id->local_addr.sin_port) || !(cm_id->remote_addr.sin_port)) 3280 if (!laddr->sin_port || !raddr->sin_port)
3281 return -EINVAL; 3281 return -EINVAL;
3282 3282
3283 nes_debug(NES_DBG_CM, "QP%u, current IP = 0x%08X, Destination IP = " 3283 nes_debug(NES_DBG_CM, "QP%u, current IP = 0x%08X, Destination IP = "
3284 "0x%08X:0x%04X, local = 0x%08X:0x%04X.\n", nesqp->hwqp.qp_id, 3284 "0x%08X:0x%04X, local = 0x%08X:0x%04X.\n", nesqp->hwqp.qp_id,
3285 ntohl(nesvnic->local_ipaddr), 3285 ntohl(nesvnic->local_ipaddr), ntohl(raddr->sin_addr.s_addr),
3286 ntohl(cm_id->remote_addr.sin_addr.s_addr), 3286 ntohs(raddr->sin_port), ntohl(laddr->sin_addr.s_addr),
3287 ntohs(cm_id->remote_addr.sin_port), 3287 ntohs(laddr->sin_port));
3288 ntohl(cm_id->local_addr.sin_addr.s_addr),
3289 ntohs(cm_id->local_addr.sin_port));
3290 3288
3291 atomic_inc(&cm_connects); 3289 atomic_inc(&cm_connects);
3292 nesqp->active_conn = 1; 3290 nesqp->active_conn = 1;
@@ -3306,18 +3304,18 @@ int nes_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3306 nes_debug(NES_DBG_CM, "mpa private data len =%u\n", 3304 nes_debug(NES_DBG_CM, "mpa private data len =%u\n",
3307 conn_param->private_data_len); 3305 conn_param->private_data_len);
3308 3306
3309 if (cm_id->local_addr.sin_addr.s_addr != 3307 if (laddr->sin_addr.s_addr != raddr->sin_addr.s_addr) {
3310 cm_id->remote_addr.sin_addr.s_addr) { 3308 nes_manage_apbvt(nesvnic, ntohs(laddr->sin_port),
3311 nes_manage_apbvt(nesvnic, ntohs(cm_id->local_addr.sin_port), 3309 PCI_FUNC(nesdev->pcidev->devfn),
3312 PCI_FUNC(nesdev->pcidev->devfn), NES_MANAGE_APBVT_ADD); 3310 NES_MANAGE_APBVT_ADD);
3313 apbvt_set = 1; 3311 apbvt_set = 1;
3314 } 3312 }
3315 3313
3316 /* set up the connection params for the node */ 3314 /* set up the connection params for the node */
3317 cm_info.loc_addr = htonl(cm_id->local_addr.sin_addr.s_addr); 3315 cm_info.loc_addr = htonl(laddr->sin_addr.s_addr);
3318 cm_info.loc_port = htons(cm_id->local_addr.sin_port); 3316 cm_info.loc_port = htons(laddr->sin_port);
3319 cm_info.rem_addr = htonl(cm_id->remote_addr.sin_addr.s_addr); 3317 cm_info.rem_addr = htonl(raddr->sin_addr.s_addr);
3320 cm_info.rem_port = htons(cm_id->remote_addr.sin_port); 3318 cm_info.rem_port = htons(raddr->sin_port);
3321 cm_info.cm_id = cm_id; 3319 cm_info.cm_id = cm_id;
3322 cm_info.conn_type = NES_CM_IWARP_CONN_TYPE; 3320 cm_info.conn_type = NES_CM_IWARP_CONN_TYPE;
3323 3321
@@ -3329,7 +3327,7 @@ int nes_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3329 &cm_info); 3327 &cm_info);
3330 if (!cm_node) { 3328 if (!cm_node) {
3331 if (apbvt_set) 3329 if (apbvt_set)
3332 nes_manage_apbvt(nesvnic, ntohs(cm_id->local_addr.sin_port), 3330 nes_manage_apbvt(nesvnic, ntohs(laddr->sin_port),
3333 PCI_FUNC(nesdev->pcidev->devfn), 3331 PCI_FUNC(nesdev->pcidev->devfn),
3334 NES_MANAGE_APBVT_DEL); 3332 NES_MANAGE_APBVT_DEL);
3335 3333
@@ -3355,10 +3353,13 @@ int nes_create_listen(struct iw_cm_id *cm_id, int backlog)
3355 struct nes_cm_listener *cm_node; 3353 struct nes_cm_listener *cm_node;
3356 struct nes_cm_info cm_info; 3354 struct nes_cm_info cm_info;
3357 int err; 3355 int err;
3356 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr;
3358 3357
3359 nes_debug(NES_DBG_CM, "cm_id = %p, local port = 0x%04X.\n", 3358 nes_debug(NES_DBG_CM, "cm_id = %p, local port = 0x%04X.\n",
3360 cm_id, ntohs(cm_id->local_addr.sin_port)); 3359 cm_id, ntohs(laddr->sin_port));
3361 3360
3361 if (cm_id->local_addr.ss_family != AF_INET)
3362 return -ENOSYS;
3362 nesvnic = to_nesvnic(cm_id->device); 3363 nesvnic = to_nesvnic(cm_id->device);
3363 if (!nesvnic) 3364 if (!nesvnic)
3364 return -EINVAL; 3365 return -EINVAL;
@@ -3367,11 +3368,11 @@ int nes_create_listen(struct iw_cm_id *cm_id, int backlog)
3367 nesvnic, nesvnic->netdev, nesvnic->netdev->name); 3368 nesvnic, nesvnic->netdev, nesvnic->netdev->name);
3368 3369
3369 nes_debug(NES_DBG_CM, "nesvnic->local_ipaddr=0x%08x, sin_addr.s_addr=0x%08x\n", 3370 nes_debug(NES_DBG_CM, "nesvnic->local_ipaddr=0x%08x, sin_addr.s_addr=0x%08x\n",
3370 nesvnic->local_ipaddr, cm_id->local_addr.sin_addr.s_addr); 3371 nesvnic->local_ipaddr, laddr->sin_addr.s_addr);
3371 3372
3372 /* setup listen params in our api call struct */ 3373 /* setup listen params in our api call struct */
3373 cm_info.loc_addr = nesvnic->local_ipaddr; 3374 cm_info.loc_addr = nesvnic->local_ipaddr;
3374 cm_info.loc_port = cm_id->local_addr.sin_port; 3375 cm_info.loc_port = laddr->sin_port;
3375 cm_info.backlog = backlog; 3376 cm_info.backlog = backlog;
3376 cm_info.cm_id = cm_id; 3377 cm_info.cm_id = cm_id;
3377 3378
@@ -3388,8 +3389,7 @@ int nes_create_listen(struct iw_cm_id *cm_id, int backlog)
3388 cm_id->provider_data = cm_node; 3389 cm_id->provider_data = cm_node;
3389 3390
3390 if (!cm_node->reused_node) { 3391 if (!cm_node->reused_node) {
3391 err = nes_manage_apbvt(nesvnic, 3392 err = nes_manage_apbvt(nesvnic, ntohs(laddr->sin_port),
3392 ntohs(cm_id->local_addr.sin_port),
3393 PCI_FUNC(nesvnic->nesdev->pcidev->devfn), 3393 PCI_FUNC(nesvnic->nesdev->pcidev->devfn),
3394 NES_MANAGE_APBVT_ADD); 3394 NES_MANAGE_APBVT_ADD);
3395 if (err) { 3395 if (err) {
@@ -3487,6 +3487,9 @@ static void cm_event_connected(struct nes_cm_event *event)
3487 struct nes_v4_quad nes_quad; 3487 struct nes_v4_quad nes_quad;
3488 u32 crc_value; 3488 u32 crc_value;
3489 int ret; 3489 int ret;
3490 struct sockaddr_in *laddr;
3491 struct sockaddr_in *raddr;
3492 struct sockaddr_in *cm_event_laddr;
3490 3493
3491 /* get all our handles */ 3494 /* get all our handles */
3492 cm_node = event->cm_node; 3495 cm_node = event->cm_node;
@@ -3496,27 +3499,24 @@ static void cm_event_connected(struct nes_cm_event *event)
3496 nesvnic = to_nesvnic(nesqp->ibqp.device); 3499 nesvnic = to_nesvnic(nesqp->ibqp.device);
3497 nesdev = nesvnic->nesdev; 3500 nesdev = nesvnic->nesdev;
3498 nesadapter = nesdev->nesadapter; 3501 nesadapter = nesdev->nesadapter;
3502 laddr = (struct sockaddr_in *)&cm_id->local_addr;
3503 raddr = (struct sockaddr_in *)&cm_id->remote_addr;
3504 cm_event_laddr = (struct sockaddr_in *)&cm_event.local_addr;
3499 3505
3500 if (nesqp->destroyed) 3506 if (nesqp->destroyed)
3501 return; 3507 return;
3502 atomic_inc(&cm_connecteds); 3508 atomic_inc(&cm_connecteds);
3503 nes_debug(NES_DBG_CM, "QP%u attempting to connect to 0x%08X:0x%04X on" 3509 nes_debug(NES_DBG_CM, "QP%u attempting to connect to 0x%08X:0x%04X on"
3504 " local port 0x%04X. jiffies = %lu.\n", 3510 " local port 0x%04X. jiffies = %lu.\n",
3505 nesqp->hwqp.qp_id, 3511 nesqp->hwqp.qp_id, ntohl(raddr->sin_addr.s_addr),
3506 ntohl(cm_id->remote_addr.sin_addr.s_addr), 3512 ntohs(raddr->sin_port), ntohs(laddr->sin_port), jiffies);
3507 ntohs(cm_id->remote_addr.sin_port),
3508 ntohs(cm_id->local_addr.sin_port),
3509 jiffies);
3510 3513
3511 nes_cm_init_tsa_conn(nesqp, cm_node); 3514 nes_cm_init_tsa_conn(nesqp, cm_node);
3512 3515
3513 /* set the QP tsa context */ 3516 /* set the QP tsa context */
3514 nesqp->nesqp_context->tcpPorts[0] = 3517 nesqp->nesqp_context->tcpPorts[0] = cpu_to_le16(ntohs(laddr->sin_port));
3515 cpu_to_le16(ntohs(cm_id->local_addr.sin_port)); 3518 nesqp->nesqp_context->tcpPorts[1] = cpu_to_le16(ntohs(raddr->sin_port));
3516 nesqp->nesqp_context->tcpPorts[1] = 3519 nesqp->nesqp_context->ip0 = cpu_to_le32(ntohl(raddr->sin_addr.s_addr));
3517 cpu_to_le16(ntohs(cm_id->remote_addr.sin_port));
3518 nesqp->nesqp_context->ip0 =
3519 cpu_to_le32(ntohl(cm_id->remote_addr.sin_addr.s_addr));
3520 3520
3521 nesqp->nesqp_context->misc2 |= cpu_to_le32( 3521 nesqp->nesqp_context->misc2 |= cpu_to_le32(
3522 (u32)PCI_FUNC(nesdev->pcidev->devfn) << 3522 (u32)PCI_FUNC(nesdev->pcidev->devfn) <<
@@ -3544,9 +3544,9 @@ static void cm_event_connected(struct nes_cm_event *event)
3544 3544
3545 nes_quad.DstIpAdrIndex = 3545 nes_quad.DstIpAdrIndex =
3546 cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24); 3546 cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24);
3547 nes_quad.SrcIpadr = cm_id->remote_addr.sin_addr.s_addr; 3547 nes_quad.SrcIpadr = raddr->sin_addr.s_addr;
3548 nes_quad.TcpPorts[0] = cm_id->remote_addr.sin_port; 3548 nes_quad.TcpPorts[0] = raddr->sin_port;
3549 nes_quad.TcpPorts[1] = cm_id->local_addr.sin_port; 3549 nes_quad.TcpPorts[1] = laddr->sin_port;
3550 3550
3551 /* Produce hash key */ 3551 /* Produce hash key */
3552 crc_value = get_crc_value(&nes_quad); 3552 crc_value = get_crc_value(&nes_quad);
@@ -3565,8 +3565,8 @@ static void cm_event_connected(struct nes_cm_event *event)
3565 cm_event.event = IW_CM_EVENT_CONNECT_REPLY; 3565 cm_event.event = IW_CM_EVENT_CONNECT_REPLY;
3566 cm_event.status = 0; 3566 cm_event.status = 0;
3567 cm_event.provider_data = cm_id->provider_data; 3567 cm_event.provider_data = cm_id->provider_data;
3568 cm_event.local_addr.sin_family = AF_INET; 3568 cm_event_laddr->sin_family = AF_INET;
3569 cm_event.local_addr.sin_port = cm_id->local_addr.sin_port; 3569 cm_event_laddr->sin_port = laddr->sin_port;
3570 cm_event.remote_addr = cm_id->remote_addr; 3570 cm_event.remote_addr = cm_id->remote_addr;
3571 3571
3572 cm_event.private_data = (void *)event->cm_node->mpa_frame_buf; 3572 cm_event.private_data = (void *)event->cm_node->mpa_frame_buf;
@@ -3574,7 +3574,7 @@ static void cm_event_connected(struct nes_cm_event *event)
3574 cm_event.ird = cm_node->ird_size; 3574 cm_event.ird = cm_node->ird_size;
3575 cm_event.ord = cm_node->ord_size; 3575 cm_event.ord = cm_node->ord_size;
3576 3576
3577 cm_event.local_addr.sin_addr.s_addr = event->cm_info.rem_addr; 3577 cm_event_laddr->sin_addr.s_addr = event->cm_info.rem_addr;
3578 ret = cm_id->event_handler(cm_id, &cm_event); 3578 ret = cm_id->event_handler(cm_id, &cm_event);
3579 nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret); 3579 nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret);
3580 3580
@@ -3627,9 +3627,16 @@ static void cm_event_connect_error(struct nes_cm_event *event)
3627 cm_event.private_data = NULL; 3627 cm_event.private_data = NULL;
3628 cm_event.private_data_len = 0; 3628 cm_event.private_data_len = 0;
3629 3629
3630 nes_debug(NES_DBG_CM, "call CM_EVENT REJECTED, local_addr=%08x, " 3630#ifdef CONFIG_INFINIBAND_NES_DEBUG
3631 "remove_addr=%08x\n", cm_event.local_addr.sin_addr.s_addr, 3631 {
3632 cm_event.remote_addr.sin_addr.s_addr); 3632 struct sockaddr_in *cm_event_laddr = (struct sockaddr_in *)
3633 &cm_event.local_addr;
3634 struct sockaddr_in *cm_event_raddr = (struct sockaddr_in *)
3635 &cm_event.remote_addr;
3636 nes_debug(NES_DBG_CM, "call CM_EVENT REJECTED, local_addr=%08x, remote_addr=%08x\n",
3637 cm_event_laddr->sin_addr.s_addr, cm_event_raddr->sin_addr.s_addr);
3638 }
3639#endif
3633 3640
3634 ret = cm_id->event_handler(cm_id, &cm_event); 3641 ret = cm_id->event_handler(cm_id, &cm_event);
3635 nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret); 3642 nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret);
@@ -3709,6 +3716,10 @@ static void cm_event_mpa_req(struct nes_cm_event *event)
3709 struct iw_cm_event cm_event; 3716 struct iw_cm_event cm_event;
3710 int ret; 3717 int ret;
3711 struct nes_cm_node *cm_node; 3718 struct nes_cm_node *cm_node;
3719 struct sockaddr_in *cm_event_laddr = (struct sockaddr_in *)
3720 &cm_event.local_addr;
3721 struct sockaddr_in *cm_event_raddr = (struct sockaddr_in *)
3722 &cm_event.remote_addr;
3712 3723
3713 cm_node = event->cm_node; 3724 cm_node = event->cm_node;
3714 if (!cm_node) 3725 if (!cm_node)
@@ -3723,13 +3734,13 @@ static void cm_event_mpa_req(struct nes_cm_event *event)
3723 cm_event.status = 0; 3734 cm_event.status = 0;
3724 cm_event.provider_data = (void *)cm_node; 3735 cm_event.provider_data = (void *)cm_node;
3725 3736
3726 cm_event.local_addr.sin_family = AF_INET; 3737 cm_event_laddr->sin_family = AF_INET;
3727 cm_event.local_addr.sin_port = htons(event->cm_info.loc_port); 3738 cm_event_laddr->sin_port = htons(event->cm_info.loc_port);
3728 cm_event.local_addr.sin_addr.s_addr = htonl(event->cm_info.loc_addr); 3739 cm_event_laddr->sin_addr.s_addr = htonl(event->cm_info.loc_addr);
3729 3740
3730 cm_event.remote_addr.sin_family = AF_INET; 3741 cm_event_raddr->sin_family = AF_INET;
3731 cm_event.remote_addr.sin_port = htons(event->cm_info.rem_port); 3742 cm_event_raddr->sin_port = htons(event->cm_info.rem_port);
3732 cm_event.remote_addr.sin_addr.s_addr = htonl(event->cm_info.rem_addr); 3743 cm_event_raddr->sin_addr.s_addr = htonl(event->cm_info.rem_addr);
3733 cm_event.private_data = cm_node->mpa_frame_buf; 3744 cm_event.private_data = cm_node->mpa_frame_buf;
3734 cm_event.private_data_len = (u8)cm_node->mpa_frame_size; 3745 cm_event.private_data_len = (u8)cm_node->mpa_frame_size;
3735 cm_event.ird = cm_node->ird_size; 3746 cm_event.ird = cm_node->ird_size;
@@ -3749,6 +3760,10 @@ static void cm_event_mpa_reject(struct nes_cm_event *event)
3749 struct iw_cm_event cm_event; 3760 struct iw_cm_event cm_event;
3750 struct nes_cm_node *cm_node; 3761 struct nes_cm_node *cm_node;
3751 int ret; 3762 int ret;
3763 struct sockaddr_in *cm_event_laddr = (struct sockaddr_in *)
3764 &cm_event.local_addr;
3765 struct sockaddr_in *cm_event_raddr = (struct sockaddr_in *)
3766 &cm_event.remote_addr;
3752 3767
3753 cm_node = event->cm_node; 3768 cm_node = event->cm_node;
3754 if (!cm_node) 3769 if (!cm_node)
@@ -3763,21 +3778,21 @@ static void cm_event_mpa_reject(struct nes_cm_event *event)
3763 cm_event.status = -ECONNREFUSED; 3778 cm_event.status = -ECONNREFUSED;
3764 cm_event.provider_data = cm_id->provider_data; 3779 cm_event.provider_data = cm_id->provider_data;
3765 3780
3766 cm_event.local_addr.sin_family = AF_INET; 3781 cm_event_laddr->sin_family = AF_INET;
3767 cm_event.local_addr.sin_port = htons(event->cm_info.loc_port); 3782 cm_event_laddr->sin_port = htons(event->cm_info.loc_port);
3768 cm_event.local_addr.sin_addr.s_addr = htonl(event->cm_info.loc_addr); 3783 cm_event_laddr->sin_addr.s_addr = htonl(event->cm_info.loc_addr);
3769 3784
3770 cm_event.remote_addr.sin_family = AF_INET; 3785 cm_event_raddr->sin_family = AF_INET;
3771 cm_event.remote_addr.sin_port = htons(event->cm_info.rem_port); 3786 cm_event_raddr->sin_port = htons(event->cm_info.rem_port);
3772 cm_event.remote_addr.sin_addr.s_addr = htonl(event->cm_info.rem_addr); 3787 cm_event_raddr->sin_addr.s_addr = htonl(event->cm_info.rem_addr);
3773 3788
3774 cm_event.private_data = cm_node->mpa_frame_buf; 3789 cm_event.private_data = cm_node->mpa_frame_buf;
3775 cm_event.private_data_len = (u8)cm_node->mpa_frame_size; 3790 cm_event.private_data_len = (u8)cm_node->mpa_frame_size;
3776 3791
3777 nes_debug(NES_DBG_CM, "call CM_EVENT_MPA_REJECTED, local_addr=%08x, " 3792 nes_debug(NES_DBG_CM, "call CM_EVENT_MPA_REJECTED, local_addr=%08x, "
3778 "remove_addr=%08x\n", 3793 "remove_addr=%08x\n",
3779 cm_event.local_addr.sin_addr.s_addr, 3794 cm_event_laddr->sin_addr.s_addr,
3780 cm_event.remote_addr.sin_addr.s_addr); 3795 cm_event_raddr->sin_addr.s_addr);
3781 3796
3782 ret = cm_id->event_handler(cm_id, &cm_event); 3797 ret = cm_id->event_handler(cm_id, &cm_event);
3783 if (ret) 3798 if (ret)
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma.h b/drivers/infiniband/hw/ocrdma/ocrdma.h
index d540180a8e42..adc11d14f878 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma.h
@@ -56,10 +56,12 @@ struct ocrdma_dev_attr {
56 u16 max_qp; 56 u16 max_qp;
57 u16 max_wqe; 57 u16 max_wqe;
58 u16 max_rqe; 58 u16 max_rqe;
59 u16 max_srq;
59 u32 max_inline_data; 60 u32 max_inline_data;
60 int max_send_sge; 61 int max_send_sge;
61 int max_recv_sge; 62 int max_recv_sge;
62 int max_srq_sge; 63 int max_srq_sge;
64 int max_rdma_sge;
63 int max_mr; 65 int max_mr;
64 u64 max_mr_size; 66 u64 max_mr_size;
65 u32 max_num_mr_pbl; 67 u32 max_num_mr_pbl;
@@ -130,8 +132,7 @@ struct ocrdma_dev {
130 struct ocrdma_cq **cq_tbl; 132 struct ocrdma_cq **cq_tbl;
131 struct ocrdma_qp **qp_tbl; 133 struct ocrdma_qp **qp_tbl;
132 134
133 struct ocrdma_eq meq; 135 struct ocrdma_eq *eq_tbl;
134 struct ocrdma_eq *qp_eq_tbl;
135 int eq_cnt; 136 int eq_cnt;
136 u16 base_eqid; 137 u16 base_eqid;
137 u16 max_eq; 138 u16 max_eq;
@@ -168,11 +169,12 @@ struct ocrdma_dev {
168 struct list_head entry; 169 struct list_head entry;
169 struct rcu_head rcu; 170 struct rcu_head rcu;
170 int id; 171 int id;
172 u64 stag_arr[OCRDMA_MAX_STAG];
173 u16 pvid;
171}; 174};
172 175
173struct ocrdma_cq { 176struct ocrdma_cq {
174 struct ib_cq ibcq; 177 struct ib_cq ibcq;
175 struct ocrdma_dev *dev;
176 struct ocrdma_cqe *va; 178 struct ocrdma_cqe *va;
177 u32 phase; 179 u32 phase;
178 u32 getp; /* pointer to pending wrs to 180 u32 getp; /* pointer to pending wrs to
@@ -214,7 +216,6 @@ struct ocrdma_pd {
214 216
215struct ocrdma_ah { 217struct ocrdma_ah {
216 struct ib_ah ibah; 218 struct ib_ah ibah;
217 struct ocrdma_dev *dev;
218 struct ocrdma_av *av; 219 struct ocrdma_av *av;
219 u16 sgid_index; 220 u16 sgid_index;
220 u32 id; 221 u32 id;
@@ -234,7 +235,6 @@ struct ocrdma_qp_hwq_info {
234 235
235struct ocrdma_srq { 236struct ocrdma_srq {
236 struct ib_srq ibsrq; 237 struct ib_srq ibsrq;
237 struct ocrdma_dev *dev;
238 u8 __iomem *db; 238 u8 __iomem *db;
239 struct ocrdma_qp_hwq_info rq; 239 struct ocrdma_qp_hwq_info rq;
240 u64 *rqe_wr_id_tbl; 240 u64 *rqe_wr_id_tbl;
@@ -290,10 +290,11 @@ struct ocrdma_qp {
290 u32 qkey; 290 u32 qkey;
291 bool dpp_enabled; 291 bool dpp_enabled;
292 u8 *ird_q_va; 292 u8 *ird_q_va;
293 bool signaled;
294 u16 db_cache;
293}; 295};
294 296
295struct ocrdma_hw_mr { 297struct ocrdma_hw_mr {
296 struct ocrdma_dev *dev;
297 u32 lkey; 298 u32 lkey;
298 u8 fr_mr; 299 u8 fr_mr;
299 u8 remote_atomic; 300 u8 remote_atomic;
@@ -317,15 +318,16 @@ struct ocrdma_mr {
317 struct ib_mr ibmr; 318 struct ib_mr ibmr;
318 struct ib_umem *umem; 319 struct ib_umem *umem;
319 struct ocrdma_hw_mr hwmr; 320 struct ocrdma_hw_mr hwmr;
320 struct ocrdma_pd *pd;
321}; 321};
322 322
323struct ocrdma_ucontext { 323struct ocrdma_ucontext {
324 struct ib_ucontext ibucontext; 324 struct ib_ucontext ibucontext;
325 struct ocrdma_dev *dev;
326 325
327 struct list_head mm_head; 326 struct list_head mm_head;
328 struct mutex mm_list_lock; /* protects list entries of mm type */ 327 struct mutex mm_list_lock; /* protects list entries of mm type */
328 struct ocrdma_pd *cntxt_pd;
329 int pd_in_use;
330
329 struct { 331 struct {
330 u32 *va; 332 u32 *va;
331 dma_addr_t pa; 333 dma_addr_t pa;
@@ -386,14 +388,14 @@ static inline struct ocrdma_srq *get_ocrdma_srq(struct ib_srq *ibsrq)
386static inline int ocrdma_get_num_posted_shift(struct ocrdma_qp *qp) 388static inline int ocrdma_get_num_posted_shift(struct ocrdma_qp *qp)
387{ 389{
388 return ((qp->dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY && 390 return ((qp->dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY &&
389 qp->id < 64) ? 24 : 16); 391 qp->id < 128) ? 24 : 16);
390} 392}
391 393
392static inline int is_cqe_valid(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe) 394static inline int is_cqe_valid(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe)
393{ 395{
394 int cqe_valid; 396 int cqe_valid;
395 cqe_valid = le32_to_cpu(cqe->flags_status_srcqpn) & OCRDMA_CQE_VALID; 397 cqe_valid = le32_to_cpu(cqe->flags_status_srcqpn) & OCRDMA_CQE_VALID;
396 return ((cqe_valid == cq->phase) ? 1 : 0); 398 return (cqe_valid == cq->phase);
397} 399}
398 400
399static inline int is_cqe_for_sq(struct ocrdma_cqe *cqe) 401static inline int is_cqe_for_sq(struct ocrdma_cqe *cqe)
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_abi.h b/drivers/infiniband/hw/ocrdma/ocrdma_abi.h
index 517ab20b727c..fbac8eb44036 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_abi.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_abi.h
@@ -28,6 +28,9 @@
28#ifndef __OCRDMA_ABI_H__ 28#ifndef __OCRDMA_ABI_H__
29#define __OCRDMA_ABI_H__ 29#define __OCRDMA_ABI_H__
30 30
31#define OCRDMA_ABI_VERSION 1
32/* user kernel communication data structures. */
33
31struct ocrdma_alloc_ucontext_resp { 34struct ocrdma_alloc_ucontext_resp {
32 u32 dev_id; 35 u32 dev_id;
33 u32 wqe_size; 36 u32 wqe_size;
@@ -35,16 +38,16 @@ struct ocrdma_alloc_ucontext_resp {
35 u32 dpp_wqe_size; 38 u32 dpp_wqe_size;
36 u64 ah_tbl_page; 39 u64 ah_tbl_page;
37 u32 ah_tbl_len; 40 u32 ah_tbl_len;
38 u32 rsvd;
39 u8 fw_ver[32];
40 u32 rqe_size; 41 u32 rqe_size;
42 u8 fw_ver[32];
43 /* for future use/new features in progress */
41 u64 rsvd1; 44 u64 rsvd1;
42} __packed; 45 u64 rsvd2;
46};
43 47
44/* user kernel communication data structures. */
45struct ocrdma_alloc_pd_ureq { 48struct ocrdma_alloc_pd_ureq {
46 u64 rsvd1; 49 u64 rsvd1;
47} __packed; 50};
48 51
49struct ocrdma_alloc_pd_uresp { 52struct ocrdma_alloc_pd_uresp {
50 u32 id; 53 u32 id;
@@ -52,12 +55,12 @@ struct ocrdma_alloc_pd_uresp {
52 u32 dpp_page_addr_hi; 55 u32 dpp_page_addr_hi;
53 u32 dpp_page_addr_lo; 56 u32 dpp_page_addr_lo;
54 u64 rsvd1; 57 u64 rsvd1;
55} __packed; 58};
56 59
57struct ocrdma_create_cq_ureq { 60struct ocrdma_create_cq_ureq {
58 u32 dpp_cq; 61 u32 dpp_cq;
59 u32 rsvd; 62 u32 rsvd; /* pad */
60} __packed; 63};
61 64
62#define MAX_CQ_PAGES 8 65#define MAX_CQ_PAGES 8
63struct ocrdma_create_cq_uresp { 66struct ocrdma_create_cq_uresp {
@@ -69,9 +72,10 @@ struct ocrdma_create_cq_uresp {
69 u64 db_page_addr; 72 u64 db_page_addr;
70 u32 db_page_size; 73 u32 db_page_size;
71 u32 phase_change; 74 u32 phase_change;
75 /* for future use/new features in progress */
72 u64 rsvd1; 76 u64 rsvd1;
73 u64 rsvd2; 77 u64 rsvd2;
74} __packed; 78};
75 79
76#define MAX_QP_PAGES 8 80#define MAX_QP_PAGES 8
77#define MAX_UD_AV_PAGES 8 81#define MAX_UD_AV_PAGES 8
@@ -80,14 +84,14 @@ struct ocrdma_create_qp_ureq {
80 u8 enable_dpp_cq; 84 u8 enable_dpp_cq;
81 u8 rsvd; 85 u8 rsvd;
82 u16 dpp_cq_id; 86 u16 dpp_cq_id;
83 u32 rsvd1; 87 u32 rsvd1; /* pad */
84}; 88};
85 89
86struct ocrdma_create_qp_uresp { 90struct ocrdma_create_qp_uresp {
87 u16 qp_id; 91 u16 qp_id;
88 u16 sq_dbid; 92 u16 sq_dbid;
89 u16 rq_dbid; 93 u16 rq_dbid;
90 u16 resv0; 94 u16 resv0; /* pad */
91 u32 sq_page_size; 95 u32 sq_page_size;
92 u32 rq_page_size; 96 u32 rq_page_size;
93 u32 num_sq_pages; 97 u32 num_sq_pages;
@@ -98,19 +102,19 @@ struct ocrdma_create_qp_uresp {
98 u32 db_page_size; 102 u32 db_page_size;
99 u32 dpp_credit; 103 u32 dpp_credit;
100 u32 dpp_offset; 104 u32 dpp_offset;
101 u32 rsvd1;
102 u32 num_wqe_allocated; 105 u32 num_wqe_allocated;
103 u32 num_rqe_allocated; 106 u32 num_rqe_allocated;
104 u32 db_sq_offset; 107 u32 db_sq_offset;
105 u32 db_rq_offset; 108 u32 db_rq_offset;
106 u32 db_shift; 109 u32 db_shift;
110 u64 rsvd1;
107 u64 rsvd2; 111 u64 rsvd2;
108 u64 rsvd3; 112 u64 rsvd3;
109} __packed; 113} __packed;
110 114
111struct ocrdma_create_srq_uresp { 115struct ocrdma_create_srq_uresp {
112 u16 rq_dbid; 116 u16 rq_dbid;
113 u16 resv0; 117 u16 resv0; /* pad */
114 u32 resv1; 118 u32 resv1;
115 119
116 u32 rq_page_size; 120 u32 rq_page_size;
@@ -126,6 +130,6 @@ struct ocrdma_create_srq_uresp {
126 130
127 u64 rsvd2; 131 u64 rsvd2;
128 u64 rsvd3; 132 u64 rsvd3;
129} __packed; 133};
130 134
131#endif /* __OCRDMA_ABI_H__ */ 135#endif /* __OCRDMA_ABI_H__ */
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
index f4c587c68f64..ee499d942257 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
@@ -35,12 +35,11 @@
35#include "ocrdma_ah.h" 35#include "ocrdma_ah.h"
36#include "ocrdma_hw.h" 36#include "ocrdma_hw.h"
37 37
38static inline int set_av_attr(struct ocrdma_ah *ah, 38static inline int set_av_attr(struct ocrdma_dev *dev, struct ocrdma_ah *ah,
39 struct ib_ah_attr *attr, int pdid) 39 struct ib_ah_attr *attr, int pdid)
40{ 40{
41 int status = 0; 41 int status = 0;
42 u16 vlan_tag; bool vlan_enabled = false; 42 u16 vlan_tag; bool vlan_enabled = false;
43 struct ocrdma_dev *dev = ah->dev;
44 struct ocrdma_eth_vlan eth; 43 struct ocrdma_eth_vlan eth;
45 struct ocrdma_grh grh; 44 struct ocrdma_grh grh;
46 int eth_sz; 45 int eth_sz;
@@ -51,6 +50,8 @@ static inline int set_av_attr(struct ocrdma_ah *ah,
51 ah->sgid_index = attr->grh.sgid_index; 50 ah->sgid_index = attr->grh.sgid_index;
52 51
53 vlan_tag = rdma_get_vlan_id(&attr->grh.dgid); 52 vlan_tag = rdma_get_vlan_id(&attr->grh.dgid);
53 if (!vlan_tag || (vlan_tag > 0xFFF))
54 vlan_tag = dev->pvid;
54 if (vlan_tag && (vlan_tag < 0x1000)) { 55 if (vlan_tag && (vlan_tag < 0x1000)) {
55 eth.eth_type = cpu_to_be16(0x8100); 56 eth.eth_type = cpu_to_be16(0x8100);
56 eth.roce_eth_type = cpu_to_be16(OCRDMA_ROCE_ETH_TYPE); 57 eth.roce_eth_type = cpu_to_be16(OCRDMA_ROCE_ETH_TYPE);
@@ -92,7 +93,7 @@ struct ib_ah *ocrdma_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
92 int status; 93 int status;
93 struct ocrdma_ah *ah; 94 struct ocrdma_ah *ah;
94 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 95 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
95 struct ocrdma_dev *dev = pd->dev; 96 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
96 97
97 if (!(attr->ah_flags & IB_AH_GRH)) 98 if (!(attr->ah_flags & IB_AH_GRH))
98 return ERR_PTR(-EINVAL); 99 return ERR_PTR(-EINVAL);
@@ -100,12 +101,11 @@ struct ib_ah *ocrdma_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
100 ah = kzalloc(sizeof *ah, GFP_ATOMIC); 101 ah = kzalloc(sizeof *ah, GFP_ATOMIC);
101 if (!ah) 102 if (!ah)
102 return ERR_PTR(-ENOMEM); 103 return ERR_PTR(-ENOMEM);
103 ah->dev = pd->dev;
104 104
105 status = ocrdma_alloc_av(dev, ah); 105 status = ocrdma_alloc_av(dev, ah);
106 if (status) 106 if (status)
107 goto av_err; 107 goto av_err;
108 status = set_av_attr(ah, attr, pd->id); 108 status = set_av_attr(dev, ah, attr, pd->id);
109 if (status) 109 if (status)
110 goto av_conf_err; 110 goto av_conf_err;
111 111
@@ -126,7 +126,9 @@ av_err:
126int ocrdma_destroy_ah(struct ib_ah *ibah) 126int ocrdma_destroy_ah(struct ib_ah *ibah)
127{ 127{
128 struct ocrdma_ah *ah = get_ocrdma_ah(ibah); 128 struct ocrdma_ah *ah = get_ocrdma_ah(ibah);
129 ocrdma_free_av(ah->dev, ah); 129 struct ocrdma_dev *dev = get_ocrdma_dev(ibah->device);
130
131 ocrdma_free_av(dev, ah);
130 kfree(ah); 132 kfree(ah);
131 return 0; 133 return 0;
132} 134}
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
index 0965278dd2ed..4ed8235d2d36 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
@@ -94,7 +94,7 @@ enum cqe_status {
94 94
95static inline void *ocrdma_get_eqe(struct ocrdma_eq *eq) 95static inline void *ocrdma_get_eqe(struct ocrdma_eq *eq)
96{ 96{
97 return (u8 *)eq->q.va + (eq->q.tail * sizeof(struct ocrdma_eqe)); 97 return eq->q.va + (eq->q.tail * sizeof(struct ocrdma_eqe));
98} 98}
99 99
100static inline void ocrdma_eq_inc_tail(struct ocrdma_eq *eq) 100static inline void ocrdma_eq_inc_tail(struct ocrdma_eq *eq)
@@ -105,8 +105,7 @@ static inline void ocrdma_eq_inc_tail(struct ocrdma_eq *eq)
105static inline void *ocrdma_get_mcqe(struct ocrdma_dev *dev) 105static inline void *ocrdma_get_mcqe(struct ocrdma_dev *dev)
106{ 106{
107 struct ocrdma_mcqe *cqe = (struct ocrdma_mcqe *) 107 struct ocrdma_mcqe *cqe = (struct ocrdma_mcqe *)
108 ((u8 *) dev->mq.cq.va + 108 (dev->mq.cq.va + (dev->mq.cq.tail * sizeof(struct ocrdma_mcqe)));
109 (dev->mq.cq.tail * sizeof(struct ocrdma_mcqe)));
110 109
111 if (!(le32_to_cpu(cqe->valid_ae_cmpl_cons) & OCRDMA_MCQE_VALID_MASK)) 110 if (!(le32_to_cpu(cqe->valid_ae_cmpl_cons) & OCRDMA_MCQE_VALID_MASK))
112 return NULL; 111 return NULL;
@@ -120,9 +119,7 @@ static inline void ocrdma_mcq_inc_tail(struct ocrdma_dev *dev)
120 119
121static inline struct ocrdma_mqe *ocrdma_get_mqe(struct ocrdma_dev *dev) 120static inline struct ocrdma_mqe *ocrdma_get_mqe(struct ocrdma_dev *dev)
122{ 121{
123 return (struct ocrdma_mqe *)((u8 *) dev->mq.sq.va + 122 return dev->mq.sq.va + (dev->mq.sq.head * sizeof(struct ocrdma_mqe));
124 (dev->mq.sq.head *
125 sizeof(struct ocrdma_mqe)));
126} 123}
127 124
128static inline void ocrdma_mq_inc_head(struct ocrdma_dev *dev) 125static inline void ocrdma_mq_inc_head(struct ocrdma_dev *dev)
@@ -132,8 +129,7 @@ static inline void ocrdma_mq_inc_head(struct ocrdma_dev *dev)
132 129
133static inline void *ocrdma_get_mqe_rsp(struct ocrdma_dev *dev) 130static inline void *ocrdma_get_mqe_rsp(struct ocrdma_dev *dev)
134{ 131{
135 return (void *)((u8 *) dev->mq.sq.va + 132 return dev->mq.sq.va + (dev->mqe_ctx.tag * sizeof(struct ocrdma_mqe));
136 (dev->mqe_ctx.tag * sizeof(struct ocrdma_mqe)));
137} 133}
138 134
139enum ib_qp_state get_ibqp_state(enum ocrdma_qp_state qps) 135enum ib_qp_state get_ibqp_state(enum ocrdma_qp_state qps)
@@ -181,7 +177,7 @@ static enum ocrdma_qp_state get_ocrdma_qp_state(enum ib_qp_state qps)
181 177
182static int ocrdma_get_mbx_errno(u32 status) 178static int ocrdma_get_mbx_errno(u32 status)
183{ 179{
184 int err_num = -EFAULT; 180 int err_num;
185 u8 mbox_status = (status & OCRDMA_MBX_RSP_STATUS_MASK) >> 181 u8 mbox_status = (status & OCRDMA_MBX_RSP_STATUS_MASK) >>
186 OCRDMA_MBX_RSP_STATUS_SHIFT; 182 OCRDMA_MBX_RSP_STATUS_SHIFT;
187 u8 add_status = (status & OCRDMA_MBX_RSP_ASTATUS_MASK) >> 183 u8 add_status = (status & OCRDMA_MBX_RSP_ASTATUS_MASK) >>
@@ -260,10 +256,11 @@ static int ocrdma_get_mbx_cqe_errno(u16 cqe_status)
260 break; 256 break;
261 case OCRDMA_MBX_CQE_STATUS_INSUFFICIENT_RESOURCES: 257 case OCRDMA_MBX_CQE_STATUS_INSUFFICIENT_RESOURCES:
262 case OCRDMA_MBX_CQE_STATUS_QUEUE_FLUSHING: 258 case OCRDMA_MBX_CQE_STATUS_QUEUE_FLUSHING:
263 err_num = -EAGAIN; 259 err_num = -EINVAL;
264 break; 260 break;
265 case OCRDMA_MBX_CQE_STATUS_DMA_FAILED: 261 case OCRDMA_MBX_CQE_STATUS_DMA_FAILED:
266 err_num = -EIO; 262 default:
263 err_num = -EINVAL;
267 break; 264 break;
268 } 265 }
269 return err_num; 266 return err_num;
@@ -367,22 +364,6 @@ static void ocrdma_build_q_pages(struct ocrdma_pa *q_pa, int cnt,
367 } 364 }
368} 365}
369 366
370static void ocrdma_assign_eq_vect_gen2(struct ocrdma_dev *dev,
371 struct ocrdma_eq *eq)
372{
373 /* assign vector and update vector id for next EQ */
374 eq->vector = dev->nic_info.msix.start_vector;
375 dev->nic_info.msix.start_vector += 1;
376}
377
378static void ocrdma_free_eq_vect_gen2(struct ocrdma_dev *dev)
379{
380 /* this assumes that EQs are freed in exactly reverse order
381 * as its allocation.
382 */
383 dev->nic_info.msix.start_vector -= 1;
384}
385
386static int ocrdma_mbx_delete_q(struct ocrdma_dev *dev, struct ocrdma_queue_info *q, 367static int ocrdma_mbx_delete_q(struct ocrdma_dev *dev, struct ocrdma_queue_info *q,
387 int queue_type) 368 int queue_type)
388{ 369{
@@ -423,11 +404,8 @@ static int ocrdma_mbx_create_eq(struct ocrdma_dev *dev, struct ocrdma_eq *eq)
423 memset(cmd, 0, sizeof(*cmd)); 404 memset(cmd, 0, sizeof(*cmd));
424 ocrdma_init_mch(&cmd->req, OCRDMA_CMD_CREATE_EQ, OCRDMA_SUBSYS_COMMON, 405 ocrdma_init_mch(&cmd->req, OCRDMA_CMD_CREATE_EQ, OCRDMA_SUBSYS_COMMON,
425 sizeof(*cmd)); 406 sizeof(*cmd));
426 if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY)
427 cmd->req.rsvd_version = 0;
428 else
429 cmd->req.rsvd_version = 2;
430 407
408 cmd->req.rsvd_version = 2;
431 cmd->num_pages = 4; 409 cmd->num_pages = 4;
432 cmd->valid = OCRDMA_CREATE_EQ_VALID; 410 cmd->valid = OCRDMA_CREATE_EQ_VALID;
433 cmd->cnt = 4 << OCRDMA_CREATE_EQ_CNT_SHIFT; 411 cmd->cnt = 4 << OCRDMA_CREATE_EQ_CNT_SHIFT;
@@ -438,12 +416,7 @@ static int ocrdma_mbx_create_eq(struct ocrdma_dev *dev, struct ocrdma_eq *eq)
438 NULL); 416 NULL);
439 if (!status) { 417 if (!status) {
440 eq->q.id = rsp->vector_eqid & 0xffff; 418 eq->q.id = rsp->vector_eqid & 0xffff;
441 if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) 419 eq->vector = (rsp->vector_eqid >> 16) & 0xffff;
442 ocrdma_assign_eq_vect_gen2(dev, eq);
443 else {
444 eq->vector = (rsp->vector_eqid >> 16) & 0xffff;
445 dev->nic_info.msix.start_vector += 1;
446 }
447 eq->q.created = true; 420 eq->q.created = true;
448 } 421 }
449 return status; 422 return status;
@@ -486,8 +459,6 @@ static void _ocrdma_destroy_eq(struct ocrdma_dev *dev, struct ocrdma_eq *eq)
486{ 459{
487 if (eq->q.created) { 460 if (eq->q.created) {
488 ocrdma_mbx_delete_q(dev, &eq->q, QTYPE_EQ); 461 ocrdma_mbx_delete_q(dev, &eq->q, QTYPE_EQ);
489 if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY)
490 ocrdma_free_eq_vect_gen2(dev);
491 ocrdma_free_q(dev, &eq->q); 462 ocrdma_free_q(dev, &eq->q);
492 } 463 }
493} 464}
@@ -506,13 +477,12 @@ static void ocrdma_destroy_eq(struct ocrdma_dev *dev, struct ocrdma_eq *eq)
506 _ocrdma_destroy_eq(dev, eq); 477 _ocrdma_destroy_eq(dev, eq);
507} 478}
508 479
509static void ocrdma_destroy_qp_eqs(struct ocrdma_dev *dev) 480static void ocrdma_destroy_eqs(struct ocrdma_dev *dev)
510{ 481{
511 int i; 482 int i;
512 483
513 /* deallocate the data path eqs */
514 for (i = 0; i < dev->eq_cnt; i++) 484 for (i = 0; i < dev->eq_cnt; i++)
515 ocrdma_destroy_eq(dev, &dev->qp_eq_tbl[i]); 485 ocrdma_destroy_eq(dev, &dev->eq_tbl[i]);
516} 486}
517 487
518static int ocrdma_mbx_mq_cq_create(struct ocrdma_dev *dev, 488static int ocrdma_mbx_mq_cq_create(struct ocrdma_dev *dev,
@@ -527,16 +497,21 @@ static int ocrdma_mbx_mq_cq_create(struct ocrdma_dev *dev,
527 ocrdma_init_mch(&cmd->req, OCRDMA_CMD_CREATE_CQ, 497 ocrdma_init_mch(&cmd->req, OCRDMA_CMD_CREATE_CQ,
528 OCRDMA_SUBSYS_COMMON, sizeof(*cmd)); 498 OCRDMA_SUBSYS_COMMON, sizeof(*cmd));
529 499
530 cmd->pgsz_pgcnt = PAGES_4K_SPANNED(cq->va, cq->size); 500 cmd->req.rsvd_version = OCRDMA_CREATE_CQ_VER2;
501 cmd->pgsz_pgcnt = (cq->size / OCRDMA_MIN_Q_PAGE_SIZE) <<
502 OCRDMA_CREATE_CQ_PAGE_SIZE_SHIFT;
503 cmd->pgsz_pgcnt |= PAGES_4K_SPANNED(cq->va, cq->size);
504
531 cmd->ev_cnt_flags = OCRDMA_CREATE_CQ_DEF_FLAGS; 505 cmd->ev_cnt_flags = OCRDMA_CREATE_CQ_DEF_FLAGS;
532 cmd->eqn = (eq->id << OCRDMA_CREATE_CQ_EQID_SHIFT); 506 cmd->eqn = eq->id;
507 cmd->cqe_count = cq->size / sizeof(struct ocrdma_mcqe);
533 508
534 ocrdma_build_q_pages(&cmd->pa[0], cmd->pgsz_pgcnt, 509 ocrdma_build_q_pages(&cmd->pa[0], cq->size / OCRDMA_MIN_Q_PAGE_SIZE,
535 cq->dma, PAGE_SIZE_4K); 510 cq->dma, PAGE_SIZE_4K);
536 status = be_roce_mcc_cmd(dev->nic_info.netdev, 511 status = be_roce_mcc_cmd(dev->nic_info.netdev,
537 cmd, sizeof(*cmd), NULL, NULL); 512 cmd, sizeof(*cmd), NULL, NULL);
538 if (!status) { 513 if (!status) {
539 cq->id = (rsp->cq_id & OCRDMA_CREATE_CQ_RSP_CQ_ID_MASK); 514 cq->id = (u16) (rsp->cq_id & OCRDMA_CREATE_CQ_RSP_CQ_ID_MASK);
540 cq->created = true; 515 cq->created = true;
541 } 516 }
542 return status; 517 return status;
@@ -569,7 +544,10 @@ static int ocrdma_mbx_create_mq(struct ocrdma_dev *dev,
569 cmd->cqid_pages = num_pages; 544 cmd->cqid_pages = num_pages;
570 cmd->cqid_pages |= (cq->id << OCRDMA_CREATE_MQ_CQ_ID_SHIFT); 545 cmd->cqid_pages |= (cq->id << OCRDMA_CREATE_MQ_CQ_ID_SHIFT);
571 cmd->async_cqid_valid = OCRDMA_CREATE_MQ_ASYNC_CQ_VALID; 546 cmd->async_cqid_valid = OCRDMA_CREATE_MQ_ASYNC_CQ_VALID;
572 cmd->async_event_bitmap = Bit(20); 547
548 cmd->async_event_bitmap = Bit(OCRDMA_ASYNC_GRP5_EVE_CODE);
549 cmd->async_event_bitmap |= Bit(OCRDMA_ASYNC_RDMA_EVE_CODE);
550
573 cmd->async_cqid_ringsize = cq->id; 551 cmd->async_cqid_ringsize = cq->id;
574 cmd->async_cqid_ringsize |= (ocrdma_encoded_q_len(mq->len) << 552 cmd->async_cqid_ringsize |= (ocrdma_encoded_q_len(mq->len) <<
575 OCRDMA_CREATE_MQ_RING_SIZE_SHIFT); 553 OCRDMA_CREATE_MQ_RING_SIZE_SHIFT);
@@ -596,7 +574,7 @@ static int ocrdma_create_mq(struct ocrdma_dev *dev)
596 if (status) 574 if (status)
597 goto alloc_err; 575 goto alloc_err;
598 576
599 status = ocrdma_mbx_mq_cq_create(dev, &dev->mq.cq, &dev->meq.q); 577 status = ocrdma_mbx_mq_cq_create(dev, &dev->mq.cq, &dev->eq_tbl[0].q);
600 if (status) 578 if (status)
601 goto mbx_cq_free; 579 goto mbx_cq_free;
602 580
@@ -653,7 +631,7 @@ static void ocrdma_process_qpcat_error(struct ocrdma_dev *dev,
653 631
654 if (qp == NULL) 632 if (qp == NULL)
655 BUG(); 633 BUG();
656 ocrdma_qp_state_machine(qp, new_ib_qps, &old_ib_qps); 634 ocrdma_qp_state_change(qp, new_ib_qps, &old_ib_qps);
657} 635}
658 636
659static void ocrdma_dispatch_ibevent(struct ocrdma_dev *dev, 637static void ocrdma_dispatch_ibevent(struct ocrdma_dev *dev,
@@ -746,11 +724,35 @@ static void ocrdma_dispatch_ibevent(struct ocrdma_dev *dev,
746 qp->srq->ibsrq.event_handler(&ib_evt, 724 qp->srq->ibsrq.event_handler(&ib_evt,
747 qp->srq->ibsrq. 725 qp->srq->ibsrq.
748 srq_context); 726 srq_context);
749 } else if (dev_event) 727 } else if (dev_event) {
750 ib_dispatch_event(&ib_evt); 728 ib_dispatch_event(&ib_evt);
729 }
751 730
752} 731}
753 732
733static void ocrdma_process_grp5_aync(struct ocrdma_dev *dev,
734 struct ocrdma_ae_mcqe *cqe)
735{
736 struct ocrdma_ae_pvid_mcqe *evt;
737 int type = (cqe->valid_ae_event & OCRDMA_AE_MCQE_EVENT_TYPE_MASK) >>
738 OCRDMA_AE_MCQE_EVENT_TYPE_SHIFT;
739
740 switch (type) {
741 case OCRDMA_ASYNC_EVENT_PVID_STATE:
742 evt = (struct ocrdma_ae_pvid_mcqe *)cqe;
743 if ((evt->tag_enabled & OCRDMA_AE_PVID_MCQE_ENABLED_MASK) >>
744 OCRDMA_AE_PVID_MCQE_ENABLED_SHIFT)
745 dev->pvid = ((evt->tag_enabled &
746 OCRDMA_AE_PVID_MCQE_TAG_MASK) >>
747 OCRDMA_AE_PVID_MCQE_TAG_SHIFT);
748 break;
749 default:
750 /* Not interested evts. */
751 break;
752 }
753}
754
755
754static void ocrdma_process_acqe(struct ocrdma_dev *dev, void *ae_cqe) 756static void ocrdma_process_acqe(struct ocrdma_dev *dev, void *ae_cqe)
755{ 757{
756 /* async CQE processing */ 758 /* async CQE processing */
@@ -758,8 +760,10 @@ static void ocrdma_process_acqe(struct ocrdma_dev *dev, void *ae_cqe)
758 u32 evt_code = (cqe->valid_ae_event & OCRDMA_AE_MCQE_EVENT_CODE_MASK) >> 760 u32 evt_code = (cqe->valid_ae_event & OCRDMA_AE_MCQE_EVENT_CODE_MASK) >>
759 OCRDMA_AE_MCQE_EVENT_CODE_SHIFT; 761 OCRDMA_AE_MCQE_EVENT_CODE_SHIFT;
760 762
761 if (evt_code == OCRDMA_ASYNC_EVE_CODE) 763 if (evt_code == OCRDMA_ASYNC_RDMA_EVE_CODE)
762 ocrdma_dispatch_ibevent(dev, cqe); 764 ocrdma_dispatch_ibevent(dev, cqe);
765 else if (evt_code == OCRDMA_ASYNC_GRP5_EVE_CODE)
766 ocrdma_process_grp5_aync(dev, cqe);
763 else 767 else
764 pr_err("%s(%d) invalid evt code=0x%x\n", __func__, 768 pr_err("%s(%d) invalid evt code=0x%x\n", __func__,
765 dev->id, evt_code); 769 dev->id, evt_code);
@@ -957,9 +961,8 @@ static int ocrdma_mbx_cmd(struct ocrdma_dev *dev, struct ocrdma_mqe *mqe)
957 rsp = ocrdma_get_mqe_rsp(dev); 961 rsp = ocrdma_get_mqe_rsp(dev);
958 ocrdma_copy_le32_to_cpu(mqe, rsp, (sizeof(*mqe))); 962 ocrdma_copy_le32_to_cpu(mqe, rsp, (sizeof(*mqe)));
959 if (cqe_status || ext_status) { 963 if (cqe_status || ext_status) {
960 pr_err 964 pr_err("%s() opcode=0x%x, cqe_status=0x%x, ext_status=0x%x\n",
961 ("%s() opcode=0x%x, cqe_status=0x%x, ext_status=0x%x\n", 965 __func__,
962 __func__,
963 (rsp->u.rsp.subsys_op & OCRDMA_MBX_RSP_OPCODE_MASK) >> 966 (rsp->u.rsp.subsys_op & OCRDMA_MBX_RSP_OPCODE_MASK) >>
964 OCRDMA_MBX_RSP_OPCODE_SHIFT, cqe_status, ext_status); 967 OCRDMA_MBX_RSP_OPCODE_SHIFT, cqe_status, ext_status);
965 status = ocrdma_get_mbx_cqe_errno(cqe_status); 968 status = ocrdma_get_mbx_cqe_errno(cqe_status);
@@ -991,9 +994,15 @@ static void ocrdma_get_attr(struct ocrdma_dev *dev,
991 attr->max_srq_sge = (rsp->max_srq_rqe_sge & 994 attr->max_srq_sge = (rsp->max_srq_rqe_sge &
992 OCRDMA_MBX_QUERY_CFG_MAX_SRQ_SGE_MASK) >> 995 OCRDMA_MBX_QUERY_CFG_MAX_SRQ_SGE_MASK) >>
993 OCRDMA_MBX_QUERY_CFG_MAX_SRQ_SGE_OFFSET; 996 OCRDMA_MBX_QUERY_CFG_MAX_SRQ_SGE_OFFSET;
997 attr->max_rdma_sge = (rsp->max_write_send_sge &
998 OCRDMA_MBX_QUERY_CFG_MAX_WRITE_SGE_MASK) >>
999 OCRDMA_MBX_QUERY_CFG_MAX_WRITE_SGE_SHIFT;
994 attr->max_ord_per_qp = (rsp->max_ird_ord_per_qp & 1000 attr->max_ord_per_qp = (rsp->max_ird_ord_per_qp &
995 OCRDMA_MBX_QUERY_CFG_MAX_ORD_PER_QP_MASK) >> 1001 OCRDMA_MBX_QUERY_CFG_MAX_ORD_PER_QP_MASK) >>
996 OCRDMA_MBX_QUERY_CFG_MAX_ORD_PER_QP_SHIFT; 1002 OCRDMA_MBX_QUERY_CFG_MAX_ORD_PER_QP_SHIFT;
1003 attr->max_srq =
1004 (rsp->max_srq_rpir_qps & OCRDMA_MBX_QUERY_CFG_MAX_SRQ_MASK) >>
1005 OCRDMA_MBX_QUERY_CFG_MAX_SRQ_OFFSET;
997 attr->max_ird_per_qp = (rsp->max_ird_ord_per_qp & 1006 attr->max_ird_per_qp = (rsp->max_ird_ord_per_qp &
998 OCRDMA_MBX_QUERY_CFG_MAX_IRD_PER_QP_MASK) >> 1007 OCRDMA_MBX_QUERY_CFG_MAX_IRD_PER_QP_MASK) >>
999 OCRDMA_MBX_QUERY_CFG_MAX_IRD_PER_QP_SHIFT; 1008 OCRDMA_MBX_QUERY_CFG_MAX_IRD_PER_QP_SHIFT;
@@ -1013,6 +1022,9 @@ static void ocrdma_get_attr(struct ocrdma_dev *dev,
1013 attr->max_num_mr_pbl = rsp->max_num_mr_pbl; 1022 attr->max_num_mr_pbl = rsp->max_num_mr_pbl;
1014 attr->max_cqe = rsp->max_cq_cqes_per_cq & 1023 attr->max_cqe = rsp->max_cq_cqes_per_cq &
1015 OCRDMA_MBX_QUERY_CFG_MAX_CQES_PER_CQ_MASK; 1024 OCRDMA_MBX_QUERY_CFG_MAX_CQES_PER_CQ_MASK;
1025 attr->max_cq = (rsp->max_cq_cqes_per_cq &
1026 OCRDMA_MBX_QUERY_CFG_MAX_CQ_MASK) >>
1027 OCRDMA_MBX_QUERY_CFG_MAX_CQ_OFFSET;
1016 attr->wqe_size = ((rsp->wqe_rqe_stride_max_dpp_cqs & 1028 attr->wqe_size = ((rsp->wqe_rqe_stride_max_dpp_cqs &
1017 OCRDMA_MBX_QUERY_CFG_MAX_WQE_SIZE_MASK) >> 1029 OCRDMA_MBX_QUERY_CFG_MAX_WQE_SIZE_MASK) >>
1018 OCRDMA_MBX_QUERY_CFG_MAX_WQE_SIZE_OFFSET) * 1030 OCRDMA_MBX_QUERY_CFG_MAX_WQE_SIZE_OFFSET) *
@@ -1045,7 +1057,6 @@ static int ocrdma_check_fw_config(struct ocrdma_dev *dev,
1045 return -EINVAL; 1057 return -EINVAL;
1046 dev->base_eqid = conf->base_eqid; 1058 dev->base_eqid = conf->base_eqid;
1047 dev->max_eq = conf->max_eq; 1059 dev->max_eq = conf->max_eq;
1048 dev->attr.max_cq = OCRDMA_MAX_CQ - 1;
1049 return 0; 1060 return 0;
1050} 1061}
1051 1062
@@ -1118,6 +1129,34 @@ mbx_err:
1118 return status; 1129 return status;
1119} 1130}
1120 1131
1132int ocrdma_mbx_get_link_speed(struct ocrdma_dev *dev, u8 *lnk_speed)
1133{
1134 int status = -ENOMEM;
1135 struct ocrdma_get_link_speed_rsp *rsp;
1136 struct ocrdma_mqe *cmd;
1137
1138 cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_QUERY_NTWK_LINK_CONFIG_V1,
1139 sizeof(*cmd));
1140 if (!cmd)
1141 return status;
1142 ocrdma_init_mch((struct ocrdma_mbx_hdr *)&cmd->u.cmd[0],
1143 OCRDMA_CMD_QUERY_NTWK_LINK_CONFIG_V1,
1144 OCRDMA_SUBSYS_COMMON, sizeof(*cmd));
1145
1146 ((struct ocrdma_mbx_hdr *)cmd->u.cmd)->rsvd_version = 0x1;
1147
1148 status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
1149 if (status)
1150 goto mbx_err;
1151
1152 rsp = (struct ocrdma_get_link_speed_rsp *)cmd;
1153 *lnk_speed = rsp->phys_port_speed;
1154
1155mbx_err:
1156 kfree(cmd);
1157 return status;
1158}
1159
1121int ocrdma_mbx_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd) 1160int ocrdma_mbx_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd)
1122{ 1161{
1123 int status = -ENOMEM; 1162 int status = -ENOMEM;
@@ -1296,19 +1335,19 @@ static u16 ocrdma_bind_eq(struct ocrdma_dev *dev)
1296 u16 eq_id; 1335 u16 eq_id;
1297 1336
1298 mutex_lock(&dev->dev_lock); 1337 mutex_lock(&dev->dev_lock);
1299 cq_cnt = dev->qp_eq_tbl[0].cq_cnt; 1338 cq_cnt = dev->eq_tbl[0].cq_cnt;
1300 eq_id = dev->qp_eq_tbl[0].q.id; 1339 eq_id = dev->eq_tbl[0].q.id;
1301 /* find the EQ which is has the least number of 1340 /* find the EQ which is has the least number of
1302 * CQs associated with it. 1341 * CQs associated with it.
1303 */ 1342 */
1304 for (i = 0; i < dev->eq_cnt; i++) { 1343 for (i = 0; i < dev->eq_cnt; i++) {
1305 if (dev->qp_eq_tbl[i].cq_cnt < cq_cnt) { 1344 if (dev->eq_tbl[i].cq_cnt < cq_cnt) {
1306 cq_cnt = dev->qp_eq_tbl[i].cq_cnt; 1345 cq_cnt = dev->eq_tbl[i].cq_cnt;
1307 eq_id = dev->qp_eq_tbl[i].q.id; 1346 eq_id = dev->eq_tbl[i].q.id;
1308 selected_eq = i; 1347 selected_eq = i;
1309 } 1348 }
1310 } 1349 }
1311 dev->qp_eq_tbl[selected_eq].cq_cnt += 1; 1350 dev->eq_tbl[selected_eq].cq_cnt += 1;
1312 mutex_unlock(&dev->dev_lock); 1351 mutex_unlock(&dev->dev_lock);
1313 return eq_id; 1352 return eq_id;
1314} 1353}
@@ -1319,16 +1358,16 @@ static void ocrdma_unbind_eq(struct ocrdma_dev *dev, u16 eq_id)
1319 1358
1320 mutex_lock(&dev->dev_lock); 1359 mutex_lock(&dev->dev_lock);
1321 for (i = 0; i < dev->eq_cnt; i++) { 1360 for (i = 0; i < dev->eq_cnt; i++) {
1322 if (dev->qp_eq_tbl[i].q.id != eq_id) 1361 if (dev->eq_tbl[i].q.id != eq_id)
1323 continue; 1362 continue;
1324 dev->qp_eq_tbl[i].cq_cnt -= 1; 1363 dev->eq_tbl[i].cq_cnt -= 1;
1325 break; 1364 break;
1326 } 1365 }
1327 mutex_unlock(&dev->dev_lock); 1366 mutex_unlock(&dev->dev_lock);
1328} 1367}
1329 1368
1330int ocrdma_mbx_create_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq, 1369int ocrdma_mbx_create_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
1331 int entries, int dpp_cq) 1370 int entries, int dpp_cq, u16 pd_id)
1332{ 1371{
1333 int status = -ENOMEM; int max_hw_cqe; 1372 int status = -ENOMEM; int max_hw_cqe;
1334 struct pci_dev *pdev = dev->nic_info.pdev; 1373 struct pci_dev *pdev = dev->nic_info.pdev;
@@ -1336,8 +1375,6 @@ int ocrdma_mbx_create_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
1336 struct ocrdma_create_cq_rsp *rsp; 1375 struct ocrdma_create_cq_rsp *rsp;
1337 u32 hw_pages, cqe_size, page_size, cqe_count; 1376 u32 hw_pages, cqe_size, page_size, cqe_count;
1338 1377
1339 if (dpp_cq)
1340 return -EINVAL;
1341 if (entries > dev->attr.max_cqe) { 1378 if (entries > dev->attr.max_cqe) {
1342 pr_err("%s(%d) max_cqe=0x%x, requester_cqe=0x%x\n", 1379 pr_err("%s(%d) max_cqe=0x%x, requester_cqe=0x%x\n",
1343 __func__, dev->id, dev->attr.max_cqe, entries); 1380 __func__, dev->id, dev->attr.max_cqe, entries);
@@ -1377,15 +1414,13 @@ int ocrdma_mbx_create_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
1377 cmd->cmd.pgsz_pgcnt |= hw_pages; 1414 cmd->cmd.pgsz_pgcnt |= hw_pages;
1378 cmd->cmd.ev_cnt_flags = OCRDMA_CREATE_CQ_DEF_FLAGS; 1415 cmd->cmd.ev_cnt_flags = OCRDMA_CREATE_CQ_DEF_FLAGS;
1379 1416
1380 if (dev->eq_cnt < 0)
1381 goto eq_err;
1382 cq->eqn = ocrdma_bind_eq(dev); 1417 cq->eqn = ocrdma_bind_eq(dev);
1383 cmd->cmd.req.rsvd_version = OCRDMA_CREATE_CQ_VER2; 1418 cmd->cmd.req.rsvd_version = OCRDMA_CREATE_CQ_VER3;
1384 cqe_count = cq->len / cqe_size; 1419 cqe_count = cq->len / cqe_size;
1385 if (cqe_count > 1024) 1420 if (cqe_count > 1024) {
1386 /* Set cnt to 3 to indicate more than 1024 cq entries */ 1421 /* Set cnt to 3 to indicate more than 1024 cq entries */
1387 cmd->cmd.ev_cnt_flags |= (0x3 << OCRDMA_CREATE_CQ_CNT_SHIFT); 1422 cmd->cmd.ev_cnt_flags |= (0x3 << OCRDMA_CREATE_CQ_CNT_SHIFT);
1388 else { 1423 } else {
1389 u8 count = 0; 1424 u8 count = 0;
1390 switch (cqe_count) { 1425 switch (cqe_count) {
1391 case 256: 1426 case 256:
@@ -1416,6 +1451,7 @@ int ocrdma_mbx_create_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
1416 cq->phase_change = true; 1451 cq->phase_change = true;
1417 } 1452 }
1418 1453
1454 cmd->cmd.pd_id = pd_id; /* valid only for v3 */
1419 ocrdma_build_q_pages(&cmd->cmd.pa[0], hw_pages, cq->pa, page_size); 1455 ocrdma_build_q_pages(&cmd->cmd.pa[0], hw_pages, cq->pa, page_size);
1420 status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd); 1456 status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
1421 if (status) 1457 if (status)
@@ -1427,7 +1463,6 @@ int ocrdma_mbx_create_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
1427 return 0; 1463 return 0;
1428mbx_err: 1464mbx_err:
1429 ocrdma_unbind_eq(dev, cq->eqn); 1465 ocrdma_unbind_eq(dev, cq->eqn);
1430eq_err:
1431 dma_free_coherent(&pdev->dev, cq->len, cq->va, cq->pa); 1466 dma_free_coherent(&pdev->dev, cq->len, cq->va, cq->pa);
1432mem_err: 1467mem_err:
1433 kfree(cmd); 1468 kfree(cmd);
@@ -1524,6 +1559,7 @@ static int ocrdma_mbx_reg_mr(struct ocrdma_dev *dev, struct ocrdma_hw_mr *hwmr,
1524 return -ENOMEM; 1559 return -ENOMEM;
1525 cmd->num_pbl_pdid = 1560 cmd->num_pbl_pdid =
1526 pdid | (hwmr->num_pbls << OCRDMA_REG_NSMR_NUM_PBL_SHIFT); 1561 pdid | (hwmr->num_pbls << OCRDMA_REG_NSMR_NUM_PBL_SHIFT);
1562 cmd->fr_mr = hwmr->fr_mr;
1527 1563
1528 cmd->flags_hpage_pbe_sz |= (hwmr->remote_wr << 1564 cmd->flags_hpage_pbe_sz |= (hwmr->remote_wr <<
1529 OCRDMA_REG_NSMR_REMOTE_WR_SHIFT); 1565 OCRDMA_REG_NSMR_REMOTE_WR_SHIFT);
@@ -1678,8 +1714,16 @@ void ocrdma_flush_qp(struct ocrdma_qp *qp)
1678 spin_unlock_irqrestore(&qp->dev->flush_q_lock, flags); 1714 spin_unlock_irqrestore(&qp->dev->flush_q_lock, flags);
1679} 1715}
1680 1716
1681int ocrdma_qp_state_machine(struct ocrdma_qp *qp, enum ib_qp_state new_ib_state, 1717static void ocrdma_init_hwq_ptr(struct ocrdma_qp *qp)
1682 enum ib_qp_state *old_ib_state) 1718{
1719 qp->sq.head = 0;
1720 qp->sq.tail = 0;
1721 qp->rq.head = 0;
1722 qp->rq.tail = 0;
1723}
1724
1725int ocrdma_qp_state_change(struct ocrdma_qp *qp, enum ib_qp_state new_ib_state,
1726 enum ib_qp_state *old_ib_state)
1683{ 1727{
1684 unsigned long flags; 1728 unsigned long flags;
1685 int status = 0; 1729 int status = 0;
@@ -1696,96 +1740,15 @@ int ocrdma_qp_state_machine(struct ocrdma_qp *qp, enum ib_qp_state new_ib_state,
1696 return 1; 1740 return 1;
1697 } 1741 }
1698 1742
1699 switch (qp->state) { 1743
1700 case OCRDMA_QPS_RST: 1744 if (new_state == OCRDMA_QPS_INIT) {
1701 switch (new_state) { 1745 ocrdma_init_hwq_ptr(qp);
1702 case OCRDMA_QPS_RST: 1746 ocrdma_del_flush_qp(qp);
1703 case OCRDMA_QPS_INIT: 1747 } else if (new_state == OCRDMA_QPS_ERR) {
1704 break; 1748 ocrdma_flush_qp(qp);
1705 default: 1749 }
1706 status = -EINVAL; 1750
1707 break; 1751 qp->state = new_state;
1708 };
1709 break;
1710 case OCRDMA_QPS_INIT:
1711 /* qps: INIT->XXX */
1712 switch (new_state) {
1713 case OCRDMA_QPS_INIT:
1714 case OCRDMA_QPS_RTR:
1715 break;
1716 case OCRDMA_QPS_ERR:
1717 ocrdma_flush_qp(qp);
1718 break;
1719 default:
1720 status = -EINVAL;
1721 break;
1722 };
1723 break;
1724 case OCRDMA_QPS_RTR:
1725 /* qps: RTS->XXX */
1726 switch (new_state) {
1727 case OCRDMA_QPS_RTS:
1728 break;
1729 case OCRDMA_QPS_ERR:
1730 ocrdma_flush_qp(qp);
1731 break;
1732 default:
1733 status = -EINVAL;
1734 break;
1735 };
1736 break;
1737 case OCRDMA_QPS_RTS:
1738 /* qps: RTS->XXX */
1739 switch (new_state) {
1740 case OCRDMA_QPS_SQD:
1741 case OCRDMA_QPS_SQE:
1742 break;
1743 case OCRDMA_QPS_ERR:
1744 ocrdma_flush_qp(qp);
1745 break;
1746 default:
1747 status = -EINVAL;
1748 break;
1749 };
1750 break;
1751 case OCRDMA_QPS_SQD:
1752 /* qps: SQD->XXX */
1753 switch (new_state) {
1754 case OCRDMA_QPS_RTS:
1755 case OCRDMA_QPS_SQE:
1756 case OCRDMA_QPS_ERR:
1757 break;
1758 default:
1759 status = -EINVAL;
1760 break;
1761 };
1762 break;
1763 case OCRDMA_QPS_SQE:
1764 switch (new_state) {
1765 case OCRDMA_QPS_RTS:
1766 case OCRDMA_QPS_ERR:
1767 break;
1768 default:
1769 status = -EINVAL;
1770 break;
1771 };
1772 break;
1773 case OCRDMA_QPS_ERR:
1774 /* qps: ERR->XXX */
1775 switch (new_state) {
1776 case OCRDMA_QPS_RST:
1777 break;
1778 default:
1779 status = -EINVAL;
1780 break;
1781 };
1782 break;
1783 default:
1784 status = -EINVAL;
1785 break;
1786 };
1787 if (!status)
1788 qp->state = new_state;
1789 1752
1790 spin_unlock_irqrestore(&qp->q_lock, flags); 1753 spin_unlock_irqrestore(&qp->q_lock, flags);
1791 return status; 1754 return status;
@@ -1819,10 +1782,9 @@ static int ocrdma_set_create_qp_sq_cmd(struct ocrdma_create_qp_req *cmd,
1819 u32 max_wqe_allocated; 1782 u32 max_wqe_allocated;
1820 u32 max_sges = attrs->cap.max_send_sge; 1783 u32 max_sges = attrs->cap.max_send_sge;
1821 1784
1822 max_wqe_allocated = attrs->cap.max_send_wr; 1785 /* QP1 may exceed 127 */
1823 /* need to allocate one extra to for GEN1 family */ 1786 max_wqe_allocated = min_t(int, attrs->cap.max_send_wr + 1,
1824 if (dev->nic_info.dev_family != OCRDMA_GEN2_FAMILY) 1787 dev->attr.max_wqe);
1825 max_wqe_allocated += 1;
1826 1788
1827 status = ocrdma_build_q_conf(&max_wqe_allocated, 1789 status = ocrdma_build_q_conf(&max_wqe_allocated,
1828 dev->attr.wqe_size, &hw_pages, &hw_page_size); 1790 dev->attr.wqe_size, &hw_pages, &hw_page_size);
@@ -1934,6 +1896,8 @@ static int ocrdma_set_create_qp_ird_cmd(struct ocrdma_create_qp_req *cmd,
1934 dma_addr_t pa = 0; 1896 dma_addr_t pa = 0;
1935 int ird_page_size = dev->attr.ird_page_size; 1897 int ird_page_size = dev->attr.ird_page_size;
1936 int ird_q_len = dev->attr.num_ird_pages * ird_page_size; 1898 int ird_q_len = dev->attr.num_ird_pages * ird_page_size;
1899 struct ocrdma_hdr_wqe *rqe;
1900 int i = 0;
1937 1901
1938 if (dev->attr.ird == 0) 1902 if (dev->attr.ird == 0)
1939 return 0; 1903 return 0;
@@ -1945,6 +1909,15 @@ static int ocrdma_set_create_qp_ird_cmd(struct ocrdma_create_qp_req *cmd,
1945 memset(qp->ird_q_va, 0, ird_q_len); 1909 memset(qp->ird_q_va, 0, ird_q_len);
1946 ocrdma_build_q_pages(&cmd->ird_addr[0], dev->attr.num_ird_pages, 1910 ocrdma_build_q_pages(&cmd->ird_addr[0], dev->attr.num_ird_pages,
1947 pa, ird_page_size); 1911 pa, ird_page_size);
1912 for (; i < ird_q_len / dev->attr.rqe_size; i++) {
1913 rqe = (struct ocrdma_hdr_wqe *)(qp->ird_q_va +
1914 (i * dev->attr.rqe_size));
1915 rqe->cw = 0;
1916 rqe->cw |= 2;
1917 rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1918 rqe->cw |= (8 << OCRDMA_WQE_SIZE_SHIFT);
1919 rqe->cw |= (8 << OCRDMA_WQE_NXT_WQE_SIZE_SHIFT);
1920 }
1948 return 0; 1921 return 0;
1949} 1922}
1950 1923
@@ -2057,9 +2030,10 @@ int ocrdma_mbx_create_qp(struct ocrdma_qp *qp, struct ib_qp_init_attr *attrs,
2057 qp->rq_cq = cq; 2030 qp->rq_cq = cq;
2058 2031
2059 if (pd->dpp_enabled && attrs->cap.max_inline_data && pd->num_dpp_qp && 2032 if (pd->dpp_enabled && attrs->cap.max_inline_data && pd->num_dpp_qp &&
2060 (attrs->cap.max_inline_data <= dev->attr.max_inline_data)) 2033 (attrs->cap.max_inline_data <= dev->attr.max_inline_data)) {
2061 ocrdma_set_create_qp_dpp_cmd(cmd, pd, qp, enable_dpp_cq, 2034 ocrdma_set_create_qp_dpp_cmd(cmd, pd, qp, enable_dpp_cq,
2062 dpp_cq_id); 2035 dpp_cq_id);
2036 }
2063 2037
2064 status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd); 2038 status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
2065 if (status) 2039 if (status)
@@ -2108,38 +2082,48 @@ int ocrdma_resolve_dgid(struct ocrdma_dev *dev, union ib_gid *dgid,
2108 struct in6_addr in6; 2082 struct in6_addr in6;
2109 2083
2110 memcpy(&in6, dgid, sizeof in6); 2084 memcpy(&in6, dgid, sizeof in6);
2111 if (rdma_is_multicast_addr(&in6)) 2085 if (rdma_is_multicast_addr(&in6)) {
2112 rdma_get_mcast_mac(&in6, mac_addr); 2086 rdma_get_mcast_mac(&in6, mac_addr);
2113 else if (rdma_link_local_addr(&in6)) 2087 } else if (rdma_link_local_addr(&in6)) {
2114 rdma_get_ll_mac(&in6, mac_addr); 2088 rdma_get_ll_mac(&in6, mac_addr);
2115 else { 2089 } else {
2116 pr_err("%s() fail to resolve mac_addr.\n", __func__); 2090 pr_err("%s() fail to resolve mac_addr.\n", __func__);
2117 return -EINVAL; 2091 return -EINVAL;
2118 } 2092 }
2119 return 0; 2093 return 0;
2120} 2094}
2121 2095
2122static void ocrdma_set_av_params(struct ocrdma_qp *qp, 2096static int ocrdma_set_av_params(struct ocrdma_qp *qp,
2123 struct ocrdma_modify_qp *cmd, 2097 struct ocrdma_modify_qp *cmd,
2124 struct ib_qp_attr *attrs) 2098 struct ib_qp_attr *attrs)
2125{ 2099{
2100 int status;
2126 struct ib_ah_attr *ah_attr = &attrs->ah_attr; 2101 struct ib_ah_attr *ah_attr = &attrs->ah_attr;
2127 union ib_gid sgid; 2102 union ib_gid sgid, zgid;
2128 u32 vlan_id; 2103 u32 vlan_id;
2129 u8 mac_addr[6]; 2104 u8 mac_addr[6];
2105
2130 if ((ah_attr->ah_flags & IB_AH_GRH) == 0) 2106 if ((ah_attr->ah_flags & IB_AH_GRH) == 0)
2131 return; 2107 return -EINVAL;
2132 cmd->params.tclass_sq_psn |= 2108 cmd->params.tclass_sq_psn |=
2133 (ah_attr->grh.traffic_class << OCRDMA_QP_PARAMS_TCLASS_SHIFT); 2109 (ah_attr->grh.traffic_class << OCRDMA_QP_PARAMS_TCLASS_SHIFT);
2134 cmd->params.rnt_rc_sl_fl |= 2110 cmd->params.rnt_rc_sl_fl |=
2135 (ah_attr->grh.flow_label & OCRDMA_QP_PARAMS_FLOW_LABEL_MASK); 2111 (ah_attr->grh.flow_label & OCRDMA_QP_PARAMS_FLOW_LABEL_MASK);
2112 cmd->params.rnt_rc_sl_fl |= (ah_attr->sl << OCRDMA_QP_PARAMS_SL_SHIFT);
2136 cmd->params.hop_lmt_rq_psn |= 2113 cmd->params.hop_lmt_rq_psn |=
2137 (ah_attr->grh.hop_limit << OCRDMA_QP_PARAMS_HOP_LMT_SHIFT); 2114 (ah_attr->grh.hop_limit << OCRDMA_QP_PARAMS_HOP_LMT_SHIFT);
2138 cmd->flags |= OCRDMA_QP_PARA_FLOW_LBL_VALID; 2115 cmd->flags |= OCRDMA_QP_PARA_FLOW_LBL_VALID;
2139 memcpy(&cmd->params.dgid[0], &ah_attr->grh.dgid.raw[0], 2116 memcpy(&cmd->params.dgid[0], &ah_attr->grh.dgid.raw[0],
2140 sizeof(cmd->params.dgid)); 2117 sizeof(cmd->params.dgid));
2141 ocrdma_query_gid(&qp->dev->ibdev, 1, 2118 status = ocrdma_query_gid(&qp->dev->ibdev, 1,
2142 ah_attr->grh.sgid_index, &sgid); 2119 ah_attr->grh.sgid_index, &sgid);
2120 if (status)
2121 return status;
2122
2123 memset(&zgid, 0, sizeof(zgid));
2124 if (!memcmp(&sgid, &zgid, sizeof(zgid)))
2125 return -EINVAL;
2126
2143 qp->sgid_idx = ah_attr->grh.sgid_index; 2127 qp->sgid_idx = ah_attr->grh.sgid_index;
2144 memcpy(&cmd->params.sgid[0], &sgid.raw[0], sizeof(cmd->params.sgid)); 2128 memcpy(&cmd->params.sgid[0], &sgid.raw[0], sizeof(cmd->params.sgid));
2145 ocrdma_resolve_dgid(qp->dev, &ah_attr->grh.dgid, &mac_addr[0]); 2129 ocrdma_resolve_dgid(qp->dev, &ah_attr->grh.dgid, &mac_addr[0]);
@@ -2155,6 +2139,7 @@ static void ocrdma_set_av_params(struct ocrdma_qp *qp,
2155 vlan_id << OCRDMA_QP_PARAMS_VLAN_SHIFT; 2139 vlan_id << OCRDMA_QP_PARAMS_VLAN_SHIFT;
2156 cmd->flags |= OCRDMA_QP_PARA_VLAN_EN_VALID; 2140 cmd->flags |= OCRDMA_QP_PARA_VLAN_EN_VALID;
2157 } 2141 }
2142 return 0;
2158} 2143}
2159 2144
2160static int ocrdma_set_qp_params(struct ocrdma_qp *qp, 2145static int ocrdma_set_qp_params(struct ocrdma_qp *qp,
@@ -2163,8 +2148,6 @@ static int ocrdma_set_qp_params(struct ocrdma_qp *qp,
2163 enum ib_qp_state old_qps) 2148 enum ib_qp_state old_qps)
2164{ 2149{
2165 int status = 0; 2150 int status = 0;
2166 struct net_device *netdev = qp->dev->nic_info.netdev;
2167 int eth_mtu = iboe_get_mtu(netdev->mtu);
2168 2151
2169 if (attr_mask & IB_QP_PKEY_INDEX) { 2152 if (attr_mask & IB_QP_PKEY_INDEX) {
2170 cmd->params.path_mtu_pkey_indx |= (attrs->pkey_index & 2153 cmd->params.path_mtu_pkey_indx |= (attrs->pkey_index &
@@ -2176,9 +2159,11 @@ static int ocrdma_set_qp_params(struct ocrdma_qp *qp,
2176 cmd->params.qkey = attrs->qkey; 2159 cmd->params.qkey = attrs->qkey;
2177 cmd->flags |= OCRDMA_QP_PARA_QKEY_VALID; 2160 cmd->flags |= OCRDMA_QP_PARA_QKEY_VALID;
2178 } 2161 }
2179 if (attr_mask & IB_QP_AV) 2162 if (attr_mask & IB_QP_AV) {
2180 ocrdma_set_av_params(qp, cmd, attrs); 2163 status = ocrdma_set_av_params(qp, cmd, attrs);
2181 else if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_UD) { 2164 if (status)
2165 return status;
2166 } else if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_UD) {
2182 /* set the default mac address for UD, GSI QPs */ 2167 /* set the default mac address for UD, GSI QPs */
2183 cmd->params.dmac_b0_to_b3 = qp->dev->nic_info.mac_addr[0] | 2168 cmd->params.dmac_b0_to_b3 = qp->dev->nic_info.mac_addr[0] |
2184 (qp->dev->nic_info.mac_addr[1] << 8) | 2169 (qp->dev->nic_info.mac_addr[1] << 8) |
@@ -2199,8 +2184,8 @@ static int ocrdma_set_qp_params(struct ocrdma_qp *qp,
2199 cmd->flags |= OCRDMA_QP_PARA_DST_QPN_VALID; 2184 cmd->flags |= OCRDMA_QP_PARA_DST_QPN_VALID;
2200 } 2185 }
2201 if (attr_mask & IB_QP_PATH_MTU) { 2186 if (attr_mask & IB_QP_PATH_MTU) {
2202 if (ib_mtu_enum_to_int(eth_mtu) < 2187 if (attrs->path_mtu < IB_MTU_256 ||
2203 ib_mtu_enum_to_int(attrs->path_mtu)) { 2188 attrs->path_mtu > IB_MTU_4096) {
2204 status = -EINVAL; 2189 status = -EINVAL;
2205 goto pmtu_err; 2190 goto pmtu_err;
2206 } 2191 }
@@ -2283,10 +2268,12 @@ int ocrdma_mbx_modify_qp(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
2283 OCRDMA_QP_PARAMS_STATE_SHIFT) & 2268 OCRDMA_QP_PARAMS_STATE_SHIFT) &
2284 OCRDMA_QP_PARAMS_STATE_MASK; 2269 OCRDMA_QP_PARAMS_STATE_MASK;
2285 cmd->flags |= OCRDMA_QP_PARA_QPS_VALID; 2270 cmd->flags |= OCRDMA_QP_PARA_QPS_VALID;
2286 } else 2271 } else {
2287 cmd->params.max_sge_recv_flags |= 2272 cmd->params.max_sge_recv_flags |=
2288 (qp->state << OCRDMA_QP_PARAMS_STATE_SHIFT) & 2273 (qp->state << OCRDMA_QP_PARAMS_STATE_SHIFT) &
2289 OCRDMA_QP_PARAMS_STATE_MASK; 2274 OCRDMA_QP_PARAMS_STATE_MASK;
2275 }
2276
2290 status = ocrdma_set_qp_params(qp, cmd, attrs, attr_mask, old_qps); 2277 status = ocrdma_set_qp_params(qp, cmd, attrs, attr_mask, old_qps);
2291 if (status) 2278 if (status)
2292 goto mbx_err; 2279 goto mbx_err;
@@ -2324,7 +2311,7 @@ mbx_err:
2324 return status; 2311 return status;
2325} 2312}
2326 2313
2327int ocrdma_mbx_create_srq(struct ocrdma_srq *srq, 2314int ocrdma_mbx_create_srq(struct ocrdma_dev *dev, struct ocrdma_srq *srq,
2328 struct ib_srq_init_attr *srq_attr, 2315 struct ib_srq_init_attr *srq_attr,
2329 struct ocrdma_pd *pd) 2316 struct ocrdma_pd *pd)
2330{ 2317{
@@ -2334,7 +2321,6 @@ int ocrdma_mbx_create_srq(struct ocrdma_srq *srq,
2334 struct ocrdma_create_srq_rsp *rsp; 2321 struct ocrdma_create_srq_rsp *rsp;
2335 struct ocrdma_create_srq *cmd; 2322 struct ocrdma_create_srq *cmd;
2336 dma_addr_t pa; 2323 dma_addr_t pa;
2337 struct ocrdma_dev *dev = srq->dev;
2338 struct pci_dev *pdev = dev->nic_info.pdev; 2324 struct pci_dev *pdev = dev->nic_info.pdev;
2339 u32 max_rqe_allocated; 2325 u32 max_rqe_allocated;
2340 2326
@@ -2404,13 +2390,16 @@ int ocrdma_mbx_modify_srq(struct ocrdma_srq *srq, struct ib_srq_attr *srq_attr)
2404{ 2390{
2405 int status = -ENOMEM; 2391 int status = -ENOMEM;
2406 struct ocrdma_modify_srq *cmd; 2392 struct ocrdma_modify_srq *cmd;
2407 cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_CREATE_SRQ, sizeof(*cmd)); 2393 struct ocrdma_pd *pd = srq->pd;
2394 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);
2395
2396 cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_MODIFY_SRQ, sizeof(*cmd));
2408 if (!cmd) 2397 if (!cmd)
2409 return status; 2398 return status;
2410 cmd->id = srq->id; 2399 cmd->id = srq->id;
2411 cmd->limit_max_rqe |= srq_attr->srq_limit << 2400 cmd->limit_max_rqe |= srq_attr->srq_limit <<
2412 OCRDMA_MODIFY_SRQ_LIMIT_SHIFT; 2401 OCRDMA_MODIFY_SRQ_LIMIT_SHIFT;
2413 status = ocrdma_mbx_cmd(srq->dev, (struct ocrdma_mqe *)cmd); 2402 status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
2414 kfree(cmd); 2403 kfree(cmd);
2415 return status; 2404 return status;
2416} 2405}
@@ -2419,11 +2408,13 @@ int ocrdma_mbx_query_srq(struct ocrdma_srq *srq, struct ib_srq_attr *srq_attr)
2419{ 2408{
2420 int status = -ENOMEM; 2409 int status = -ENOMEM;
2421 struct ocrdma_query_srq *cmd; 2410 struct ocrdma_query_srq *cmd;
2422 cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_CREATE_SRQ, sizeof(*cmd)); 2411 struct ocrdma_dev *dev = get_ocrdma_dev(srq->ibsrq.device);
2412
2413 cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_QUERY_SRQ, sizeof(*cmd));
2423 if (!cmd) 2414 if (!cmd)
2424 return status; 2415 return status;
2425 cmd->id = srq->rq.dbid; 2416 cmd->id = srq->rq.dbid;
2426 status = ocrdma_mbx_cmd(srq->dev, (struct ocrdma_mqe *)cmd); 2417 status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
2427 if (status == 0) { 2418 if (status == 0) {
2428 struct ocrdma_query_srq_rsp *rsp = 2419 struct ocrdma_query_srq_rsp *rsp =
2429 (struct ocrdma_query_srq_rsp *)cmd; 2420 (struct ocrdma_query_srq_rsp *)cmd;
@@ -2448,7 +2439,7 @@ int ocrdma_mbx_destroy_srq(struct ocrdma_dev *dev, struct ocrdma_srq *srq)
2448 if (!cmd) 2439 if (!cmd)
2449 return status; 2440 return status;
2450 cmd->id = srq->id; 2441 cmd->id = srq->id;
2451 status = ocrdma_mbx_cmd(srq->dev, (struct ocrdma_mqe *)cmd); 2442 status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
2452 if (srq->rq.va) 2443 if (srq->rq.va)
2453 dma_free_coherent(&pdev->dev, srq->rq.len, 2444 dma_free_coherent(&pdev->dev, srq->rq.len,
2454 srq->rq.va, srq->rq.pa); 2445 srq->rq.va, srq->rq.pa);
@@ -2490,38 +2481,7 @@ int ocrdma_free_av(struct ocrdma_dev *dev, struct ocrdma_ah *ah)
2490 return 0; 2481 return 0;
2491} 2482}
2492 2483
2493static int ocrdma_create_mq_eq(struct ocrdma_dev *dev) 2484static int ocrdma_create_eqs(struct ocrdma_dev *dev)
2494{
2495 int status;
2496 int irq;
2497 unsigned long flags = 0;
2498 int num_eq = 0;
2499
2500 if (dev->nic_info.intr_mode == BE_INTERRUPT_MODE_INTX)
2501 flags = IRQF_SHARED;
2502 else {
2503 num_eq = dev->nic_info.msix.num_vectors -
2504 dev->nic_info.msix.start_vector;
2505 /* minimum two vectors/eq are required for rdma to work.
2506 * one for control path and one for data path.
2507 */
2508 if (num_eq < 2)
2509 return -EBUSY;
2510 }
2511
2512 status = ocrdma_create_eq(dev, &dev->meq, OCRDMA_EQ_LEN);
2513 if (status)
2514 return status;
2515 sprintf(dev->meq.irq_name, "ocrdma_mq%d", dev->id);
2516 irq = ocrdma_get_irq(dev, &dev->meq);
2517 status = request_irq(irq, ocrdma_irq_handler, flags, dev->meq.irq_name,
2518 &dev->meq);
2519 if (status)
2520 _ocrdma_destroy_eq(dev, &dev->meq);
2521 return status;
2522}
2523
2524static int ocrdma_create_qp_eqs(struct ocrdma_dev *dev)
2525{ 2485{
2526 int num_eq, i, status = 0; 2486 int num_eq, i, status = 0;
2527 int irq; 2487 int irq;
@@ -2532,49 +2492,47 @@ static int ocrdma_create_qp_eqs(struct ocrdma_dev *dev)
2532 if (dev->nic_info.intr_mode == BE_INTERRUPT_MODE_INTX) { 2492 if (dev->nic_info.intr_mode == BE_INTERRUPT_MODE_INTX) {
2533 num_eq = 1; 2493 num_eq = 1;
2534 flags = IRQF_SHARED; 2494 flags = IRQF_SHARED;
2535 } else 2495 } else {
2536 num_eq = min_t(u32, num_eq, num_online_cpus()); 2496 num_eq = min_t(u32, num_eq, num_online_cpus());
2537 dev->qp_eq_tbl = kzalloc(sizeof(struct ocrdma_eq) * num_eq, GFP_KERNEL); 2497 }
2538 if (!dev->qp_eq_tbl) 2498
2499 if (!num_eq)
2500 return -EINVAL;
2501
2502 dev->eq_tbl = kzalloc(sizeof(struct ocrdma_eq) * num_eq, GFP_KERNEL);
2503 if (!dev->eq_tbl)
2539 return -ENOMEM; 2504 return -ENOMEM;
2540 2505
2541 for (i = 0; i < num_eq; i++) { 2506 for (i = 0; i < num_eq; i++) {
2542 status = ocrdma_create_eq(dev, &dev->qp_eq_tbl[i], 2507 status = ocrdma_create_eq(dev, &dev->eq_tbl[i],
2543 OCRDMA_EQ_LEN); 2508 OCRDMA_EQ_LEN);
2544 if (status) { 2509 if (status) {
2545 status = -EINVAL; 2510 status = -EINVAL;
2546 break; 2511 break;
2547 } 2512 }
2548 sprintf(dev->qp_eq_tbl[i].irq_name, "ocrdma_qp%d-%d", 2513 sprintf(dev->eq_tbl[i].irq_name, "ocrdma%d-%d",
2549 dev->id, i); 2514 dev->id, i);
2550 irq = ocrdma_get_irq(dev, &dev->qp_eq_tbl[i]); 2515 irq = ocrdma_get_irq(dev, &dev->eq_tbl[i]);
2551 status = request_irq(irq, ocrdma_irq_handler, flags, 2516 status = request_irq(irq, ocrdma_irq_handler, flags,
2552 dev->qp_eq_tbl[i].irq_name, 2517 dev->eq_tbl[i].irq_name,
2553 &dev->qp_eq_tbl[i]); 2518 &dev->eq_tbl[i]);
2554 if (status) { 2519 if (status)
2555 _ocrdma_destroy_eq(dev, &dev->qp_eq_tbl[i]); 2520 goto done;
2556 status = -EINVAL;
2557 break;
2558 }
2559 dev->eq_cnt += 1; 2521 dev->eq_cnt += 1;
2560 } 2522 }
2561 /* one eq is sufficient for data path to work */ 2523 /* one eq is sufficient for data path to work */
2562 if (dev->eq_cnt >= 1) 2524 return 0;
2563 return 0; 2525done:
2564 if (status) 2526 ocrdma_destroy_eqs(dev);
2565 ocrdma_destroy_qp_eqs(dev);
2566 return status; 2527 return status;
2567} 2528}
2568 2529
2569int ocrdma_init_hw(struct ocrdma_dev *dev) 2530int ocrdma_init_hw(struct ocrdma_dev *dev)
2570{ 2531{
2571 int status; 2532 int status;
2572 /* set up control path eq */ 2533
2573 status = ocrdma_create_mq_eq(dev); 2534 /* create the eqs */
2574 if (status) 2535 status = ocrdma_create_eqs(dev);
2575 return status;
2576 /* set up data path eq */
2577 status = ocrdma_create_qp_eqs(dev);
2578 if (status) 2536 if (status)
2579 goto qpeq_err; 2537 goto qpeq_err;
2580 status = ocrdma_create_mq(dev); 2538 status = ocrdma_create_mq(dev);
@@ -2597,9 +2555,8 @@ int ocrdma_init_hw(struct ocrdma_dev *dev)
2597conf_err: 2555conf_err:
2598 ocrdma_destroy_mq(dev); 2556 ocrdma_destroy_mq(dev);
2599mq_err: 2557mq_err:
2600 ocrdma_destroy_qp_eqs(dev); 2558 ocrdma_destroy_eqs(dev);
2601qpeq_err: 2559qpeq_err:
2602 ocrdma_destroy_eq(dev, &dev->meq);
2603 pr_err("%s() status=%d\n", __func__, status); 2560 pr_err("%s() status=%d\n", __func__, status);
2604 return status; 2561 return status;
2605} 2562}
@@ -2608,10 +2565,9 @@ void ocrdma_cleanup_hw(struct ocrdma_dev *dev)
2608{ 2565{
2609 ocrdma_mbx_delete_ah_tbl(dev); 2566 ocrdma_mbx_delete_ah_tbl(dev);
2610 2567
2611 /* cleanup the data path eqs */ 2568 /* cleanup the eqs */
2612 ocrdma_destroy_qp_eqs(dev); 2569 ocrdma_destroy_eqs(dev);
2613 2570
2614 /* cleanup the control path */ 2571 /* cleanup the control path */
2615 ocrdma_destroy_mq(dev); 2572 ocrdma_destroy_mq(dev);
2616 ocrdma_destroy_eq(dev, &dev->meq);
2617} 2573}
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.h b/drivers/infiniband/hw/ocrdma/ocrdma_hw.h
index be5db77404db..f2a89d4cc7c4 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.h
@@ -78,6 +78,11 @@ static inline void ocrdma_copy_le32_to_cpu(void *dst, void *src, u32 len)
78#endif 78#endif
79} 79}
80 80
81static inline u64 ocrdma_get_db_addr(struct ocrdma_dev *dev, u32 pdid)
82{
83 return dev->nic_info.unmapped_db + (pdid * dev->nic_info.db_page_size);
84}
85
81int ocrdma_init_hw(struct ocrdma_dev *); 86int ocrdma_init_hw(struct ocrdma_dev *);
82void ocrdma_cleanup_hw(struct ocrdma_dev *); 87void ocrdma_cleanup_hw(struct ocrdma_dev *);
83 88
@@ -86,6 +91,7 @@ void ocrdma_ring_cq_db(struct ocrdma_dev *, u16 cq_id, bool armed,
86 bool solicited, u16 cqe_popped); 91 bool solicited, u16 cqe_popped);
87 92
88/* verbs specific mailbox commands */ 93/* verbs specific mailbox commands */
94int ocrdma_mbx_get_link_speed(struct ocrdma_dev *dev, u8 *lnk_speed);
89int ocrdma_query_config(struct ocrdma_dev *, 95int ocrdma_query_config(struct ocrdma_dev *,
90 struct ocrdma_mbx_query_config *config); 96 struct ocrdma_mbx_query_config *config);
91int ocrdma_resolve_dgid(struct ocrdma_dev *, union ib_gid *dgid, u8 *mac_addr); 97int ocrdma_resolve_dgid(struct ocrdma_dev *, union ib_gid *dgid, u8 *mac_addr);
@@ -100,7 +106,7 @@ int ocrdma_mbx_dealloc_lkey(struct ocrdma_dev *, int fmr, u32 lkey);
100int ocrdma_reg_mr(struct ocrdma_dev *, struct ocrdma_hw_mr *hwmr, 106int ocrdma_reg_mr(struct ocrdma_dev *, struct ocrdma_hw_mr *hwmr,
101 u32 pd_id, int acc); 107 u32 pd_id, int acc);
102int ocrdma_mbx_create_cq(struct ocrdma_dev *, struct ocrdma_cq *, 108int ocrdma_mbx_create_cq(struct ocrdma_dev *, struct ocrdma_cq *,
103 int entries, int dpp_cq); 109 int entries, int dpp_cq, u16 pd_id);
104int ocrdma_mbx_destroy_cq(struct ocrdma_dev *, struct ocrdma_cq *); 110int ocrdma_mbx_destroy_cq(struct ocrdma_dev *, struct ocrdma_cq *);
105 111
106int ocrdma_mbx_create_qp(struct ocrdma_qp *, struct ib_qp_init_attr *attrs, 112int ocrdma_mbx_create_qp(struct ocrdma_qp *, struct ib_qp_init_attr *attrs,
@@ -112,8 +118,7 @@ int ocrdma_mbx_modify_qp(struct ocrdma_dev *, struct ocrdma_qp *,
112int ocrdma_mbx_query_qp(struct ocrdma_dev *, struct ocrdma_qp *, 118int ocrdma_mbx_query_qp(struct ocrdma_dev *, struct ocrdma_qp *,
113 struct ocrdma_qp_params *param); 119 struct ocrdma_qp_params *param);
114int ocrdma_mbx_destroy_qp(struct ocrdma_dev *, struct ocrdma_qp *); 120int ocrdma_mbx_destroy_qp(struct ocrdma_dev *, struct ocrdma_qp *);
115 121int ocrdma_mbx_create_srq(struct ocrdma_dev *, struct ocrdma_srq *,
116int ocrdma_mbx_create_srq(struct ocrdma_srq *,
117 struct ib_srq_init_attr *, 122 struct ib_srq_init_attr *,
118 struct ocrdma_pd *); 123 struct ocrdma_pd *);
119int ocrdma_mbx_modify_srq(struct ocrdma_srq *, struct ib_srq_attr *); 124int ocrdma_mbx_modify_srq(struct ocrdma_srq *, struct ib_srq_attr *);
@@ -123,7 +128,7 @@ int ocrdma_mbx_destroy_srq(struct ocrdma_dev *, struct ocrdma_srq *);
123int ocrdma_alloc_av(struct ocrdma_dev *, struct ocrdma_ah *); 128int ocrdma_alloc_av(struct ocrdma_dev *, struct ocrdma_ah *);
124int ocrdma_free_av(struct ocrdma_dev *, struct ocrdma_ah *); 129int ocrdma_free_av(struct ocrdma_dev *, struct ocrdma_ah *);
125 130
126int ocrdma_qp_state_machine(struct ocrdma_qp *, enum ib_qp_state new_state, 131int ocrdma_qp_state_change(struct ocrdma_qp *, enum ib_qp_state new_state,
127 enum ib_qp_state *old_ib_state); 132 enum ib_qp_state *old_ib_state);
128bool ocrdma_is_qp_in_sq_flushlist(struct ocrdma_cq *, struct ocrdma_qp *); 133bool ocrdma_is_qp_in_sq_flushlist(struct ocrdma_cq *, struct ocrdma_qp *);
129bool ocrdma_is_qp_in_rq_flushlist(struct ocrdma_cq *, struct ocrdma_qp *); 134bool ocrdma_is_qp_in_rq_flushlist(struct ocrdma_cq *, struct ocrdma_qp *);
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
index ded416f1adea..56e004940f18 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
@@ -39,6 +39,7 @@
39#include "ocrdma_ah.h" 39#include "ocrdma_ah.h"
40#include "be_roce.h" 40#include "be_roce.h"
41#include "ocrdma_hw.h" 41#include "ocrdma_hw.h"
42#include "ocrdma_abi.h"
42 43
43MODULE_VERSION(OCRDMA_ROCE_DEV_VERSION); 44MODULE_VERSION(OCRDMA_ROCE_DEV_VERSION);
44MODULE_DESCRIPTION("Emulex RoCE HCA Driver"); 45MODULE_DESCRIPTION("Emulex RoCE HCA Driver");
@@ -265,6 +266,7 @@ static int ocrdma_register_device(struct ocrdma_dev *dev)
265 memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC, 266 memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC,
266 sizeof(OCRDMA_NODE_DESC)); 267 sizeof(OCRDMA_NODE_DESC));
267 dev->ibdev.owner = THIS_MODULE; 268 dev->ibdev.owner = THIS_MODULE;
269 dev->ibdev.uverbs_abi_ver = OCRDMA_ABI_VERSION;
268 dev->ibdev.uverbs_cmd_mask = 270 dev->ibdev.uverbs_cmd_mask =
269 OCRDMA_UVERBS(GET_CONTEXT) | 271 OCRDMA_UVERBS(GET_CONTEXT) |
270 OCRDMA_UVERBS(QUERY_DEVICE) | 272 OCRDMA_UVERBS(QUERY_DEVICE) |
@@ -326,9 +328,14 @@ static int ocrdma_register_device(struct ocrdma_dev *dev)
326 dev->ibdev.req_notify_cq = ocrdma_arm_cq; 328 dev->ibdev.req_notify_cq = ocrdma_arm_cq;
327 329
328 dev->ibdev.get_dma_mr = ocrdma_get_dma_mr; 330 dev->ibdev.get_dma_mr = ocrdma_get_dma_mr;
331 dev->ibdev.reg_phys_mr = ocrdma_reg_kernel_mr;
329 dev->ibdev.dereg_mr = ocrdma_dereg_mr; 332 dev->ibdev.dereg_mr = ocrdma_dereg_mr;
330 dev->ibdev.reg_user_mr = ocrdma_reg_user_mr; 333 dev->ibdev.reg_user_mr = ocrdma_reg_user_mr;
331 334
335 dev->ibdev.alloc_fast_reg_mr = ocrdma_alloc_frmr;
336 dev->ibdev.alloc_fast_reg_page_list = ocrdma_alloc_frmr_page_list;
337 dev->ibdev.free_fast_reg_page_list = ocrdma_free_frmr_page_list;
338
332 /* mandatory to support user space verbs consumer. */ 339 /* mandatory to support user space verbs consumer. */
333 dev->ibdev.alloc_ucontext = ocrdma_alloc_ucontext; 340 dev->ibdev.alloc_ucontext = ocrdma_alloc_ucontext;
334 dev->ibdev.dealloc_ucontext = ocrdma_dealloc_ucontext; 341 dev->ibdev.dealloc_ucontext = ocrdma_dealloc_ucontext;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_sli.h b/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
index 36b062da2aea..9f9570ec3c2e 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
@@ -70,6 +70,7 @@ enum {
70 70
71#define OCRDMA_SUBSYS_COMMON 1 71#define OCRDMA_SUBSYS_COMMON 1
72enum { 72enum {
73 OCRDMA_CMD_QUERY_NTWK_LINK_CONFIG_V1 = 5,
73 OCRDMA_CMD_CREATE_CQ = 12, 74 OCRDMA_CMD_CREATE_CQ = 12,
74 OCRDMA_CMD_CREATE_EQ = 13, 75 OCRDMA_CMD_CREATE_EQ = 13,
75 OCRDMA_CMD_CREATE_MQ = 21, 76 OCRDMA_CMD_CREATE_MQ = 21,
@@ -91,15 +92,15 @@ enum {
91 92
92#define OCRDMA_MAX_QP 2048 93#define OCRDMA_MAX_QP 2048
93#define OCRDMA_MAX_CQ 2048 94#define OCRDMA_MAX_CQ 2048
95#define OCRDMA_MAX_STAG 8192
94 96
95enum { 97enum {
96 OCRDMA_DB_RQ_OFFSET = 0xE0, 98 OCRDMA_DB_RQ_OFFSET = 0xE0,
97 OCRDMA_DB_GEN2_RQ1_OFFSET = 0x100, 99 OCRDMA_DB_GEN2_RQ_OFFSET = 0x100,
98 OCRDMA_DB_GEN2_RQ2_OFFSET = 0xC0,
99 OCRDMA_DB_SQ_OFFSET = 0x60, 100 OCRDMA_DB_SQ_OFFSET = 0x60,
100 OCRDMA_DB_GEN2_SQ_OFFSET = 0x1C0, 101 OCRDMA_DB_GEN2_SQ_OFFSET = 0x1C0,
101 OCRDMA_DB_SRQ_OFFSET = OCRDMA_DB_RQ_OFFSET, 102 OCRDMA_DB_SRQ_OFFSET = OCRDMA_DB_RQ_OFFSET,
102 OCRDMA_DB_GEN2_SRQ_OFFSET = OCRDMA_DB_GEN2_RQ1_OFFSET, 103 OCRDMA_DB_GEN2_SRQ_OFFSET = OCRDMA_DB_GEN2_RQ_OFFSET,
103 OCRDMA_DB_CQ_OFFSET = 0x120, 104 OCRDMA_DB_CQ_OFFSET = 0x120,
104 OCRDMA_DB_EQ_OFFSET = OCRDMA_DB_CQ_OFFSET, 105 OCRDMA_DB_EQ_OFFSET = OCRDMA_DB_CQ_OFFSET,
105 OCRDMA_DB_MQ_OFFSET = 0x140 106 OCRDMA_DB_MQ_OFFSET = 0x140
@@ -143,8 +144,11 @@ enum {
143# 2: 16K Bytes 144# 2: 16K Bytes
144# 3: 32K Bytes 145# 3: 32K Bytes
145# 4: 64K Bytes 146# 4: 64K Bytes
147# 5: 128K Bytes
148# 6: 256K Bytes
149# 7: 512K Bytes
146*/ 150*/
147#define OCRDMA_MAX_Q_PAGE_SIZE_CNT (5) 151#define OCRDMA_MAX_Q_PAGE_SIZE_CNT (8)
148#define OCRDMA_Q_PAGE_BASE_SIZE (OCRDMA_MIN_Q_PAGE_SIZE * OCRDMA_MAX_Q_PAGES) 152#define OCRDMA_Q_PAGE_BASE_SIZE (OCRDMA_MIN_Q_PAGE_SIZE * OCRDMA_MAX_Q_PAGES)
149 153
150#define MAX_OCRDMA_QP_PAGES (8) 154#define MAX_OCRDMA_QP_PAGES (8)
@@ -177,7 +181,7 @@ struct ocrdma_mbx_hdr {
177 u32 timeout; /* in seconds */ 181 u32 timeout; /* in seconds */
178 u32 cmd_len; 182 u32 cmd_len;
179 u32 rsvd_version; 183 u32 rsvd_version;
180} __packed; 184};
181 185
182enum { 186enum {
183 OCRDMA_MBX_RSP_OPCODE_SHIFT = 0, 187 OCRDMA_MBX_RSP_OPCODE_SHIFT = 0,
@@ -197,7 +201,7 @@ struct ocrdma_mbx_rsp {
197 u32 status; 201 u32 status;
198 u32 rsp_len; 202 u32 rsp_len;
199 u32 add_rsp_len; 203 u32 add_rsp_len;
200} __packed; 204};
201 205
202enum { 206enum {
203 OCRDMA_MQE_EMBEDDED = 1, 207 OCRDMA_MQE_EMBEDDED = 1,
@@ -208,7 +212,7 @@ struct ocrdma_mqe_sge {
208 u32 pa_lo; 212 u32 pa_lo;
209 u32 pa_hi; 213 u32 pa_hi;
210 u32 len; 214 u32 len;
211} __packed; 215};
212 216
213enum { 217enum {
214 OCRDMA_MQE_HDR_EMB_SHIFT = 0, 218 OCRDMA_MQE_HDR_EMB_SHIFT = 0,
@@ -225,12 +229,12 @@ struct ocrdma_mqe_hdr {
225 u32 tag_lo; 229 u32 tag_lo;
226 u32 tag_hi; 230 u32 tag_hi;
227 u32 rsvd3; 231 u32 rsvd3;
228} __packed; 232};
229 233
230struct ocrdma_mqe_emb_cmd { 234struct ocrdma_mqe_emb_cmd {
231 struct ocrdma_mbx_hdr mch; 235 struct ocrdma_mbx_hdr mch;
232 u8 pyld[220]; 236 u8 pyld[220];
233} __packed; 237};
234 238
235struct ocrdma_mqe { 239struct ocrdma_mqe {
236 struct ocrdma_mqe_hdr hdr; 240 struct ocrdma_mqe_hdr hdr;
@@ -242,7 +246,7 @@ struct ocrdma_mqe {
242 u8 cmd[236]; 246 u8 cmd[236];
243 struct ocrdma_mbx_rsp rsp; 247 struct ocrdma_mbx_rsp rsp;
244 } u; 248 } u;
245} __packed; 249};
246 250
247#define OCRDMA_EQ_LEN 4096 251#define OCRDMA_EQ_LEN 4096
248#define OCRDMA_MQ_CQ_LEN 256 252#define OCRDMA_MQ_CQ_LEN 256
@@ -259,12 +263,12 @@ struct ocrdma_mqe {
259struct ocrdma_delete_q_req { 263struct ocrdma_delete_q_req {
260 struct ocrdma_mbx_hdr req; 264 struct ocrdma_mbx_hdr req;
261 u32 id; 265 u32 id;
262} __packed; 266};
263 267
264struct ocrdma_pa { 268struct ocrdma_pa {
265 u32 lo; 269 u32 lo;
266 u32 hi; 270 u32 hi;
267} __packed; 271};
268 272
269#define MAX_OCRDMA_EQ_PAGES (8) 273#define MAX_OCRDMA_EQ_PAGES (8)
270struct ocrdma_create_eq_req { 274struct ocrdma_create_eq_req {
@@ -275,7 +279,7 @@ struct ocrdma_create_eq_req {
275 u32 delay; 279 u32 delay;
276 u32 rsvd; 280 u32 rsvd;
277 struct ocrdma_pa pa[MAX_OCRDMA_EQ_PAGES]; 281 struct ocrdma_pa pa[MAX_OCRDMA_EQ_PAGES];
278} __packed; 282};
279 283
280enum { 284enum {
281 OCRDMA_CREATE_EQ_VALID = Bit(29), 285 OCRDMA_CREATE_EQ_VALID = Bit(29),
@@ -310,7 +314,7 @@ struct ocrdma_mcqe {
310 u32 tag_lo; 314 u32 tag_lo;
311 u32 tag_hi; 315 u32 tag_hi;
312 u32 valid_ae_cmpl_cons; 316 u32 valid_ae_cmpl_cons;
313} __packed; 317};
314 318
315enum { 319enum {
316 OCRDMA_AE_MCQE_QPVALID = Bit(31), 320 OCRDMA_AE_MCQE_QPVALID = Bit(31),
@@ -332,7 +336,21 @@ struct ocrdma_ae_mcqe {
332 u32 cqvalid_cqid; 336 u32 cqvalid_cqid;
333 u32 evt_tag; 337 u32 evt_tag;
334 u32 valid_ae_event; 338 u32 valid_ae_event;
335} __packed; 339};
340
341enum {
342 OCRDMA_AE_PVID_MCQE_ENABLED_SHIFT = 0,
343 OCRDMA_AE_PVID_MCQE_ENABLED_MASK = 0xFF,
344 OCRDMA_AE_PVID_MCQE_TAG_SHIFT = 16,
345 OCRDMA_AE_PVID_MCQE_TAG_MASK = 0xFFFF << OCRDMA_AE_PVID_MCQE_TAG_SHIFT
346};
347
348struct ocrdma_ae_pvid_mcqe {
349 u32 tag_enabled;
350 u32 event_tag;
351 u32 rsvd1;
352 u32 rsvd2;
353};
336 354
337enum { 355enum {
338 OCRDMA_AE_MPA_MCQE_REQ_ID_SHIFT = 16, 356 OCRDMA_AE_MPA_MCQE_REQ_ID_SHIFT = 16,
@@ -356,7 +374,7 @@ struct ocrdma_ae_mpa_mcqe {
356 u32 w1; 374 u32 w1;
357 u32 w2; 375 u32 w2;
358 u32 valid_ae_event; 376 u32 valid_ae_event;
359} __packed; 377};
360 378
361enum { 379enum {
362 OCRDMA_AE_QP_MCQE_NEW_QP_STATE_SHIFT = 0, 380 OCRDMA_AE_QP_MCQE_NEW_QP_STATE_SHIFT = 0,
@@ -382,9 +400,11 @@ struct ocrdma_ae_qp_mcqe {
382 u32 w1; 400 u32 w1;
383 u32 w2; 401 u32 w2;
384 u32 valid_ae_event; 402 u32 valid_ae_event;
385} __packed; 403};
386 404
387#define OCRDMA_ASYNC_EVE_CODE 0x14 405#define OCRDMA_ASYNC_RDMA_EVE_CODE 0x14
406#define OCRDMA_ASYNC_GRP5_EVE_CODE 0x5
407#define OCRDMA_ASYNC_EVENT_PVID_STATE 0x3
388 408
389enum OCRDMA_ASYNC_EVENT_TYPE { 409enum OCRDMA_ASYNC_EVENT_TYPE {
390 OCRDMA_CQ_ERROR = 0x00, 410 OCRDMA_CQ_ERROR = 0x00,
@@ -487,7 +507,8 @@ struct ocrdma_mbx_query_config {
487 u32 max_ird_ord_per_qp; 507 u32 max_ird_ord_per_qp;
488 u32 max_shared_ird_ord; 508 u32 max_shared_ird_ord;
489 u32 max_mr; 509 u32 max_mr;
490 u64 max_mr_size; 510 u32 max_mr_size_lo;
511 u32 max_mr_size_hi;
491 u32 max_num_mr_pbl; 512 u32 max_num_mr_pbl;
492 u32 max_mw; 513 u32 max_mw;
493 u32 max_fmr; 514 u32 max_fmr;
@@ -502,14 +523,14 @@ struct ocrdma_mbx_query_config {
502 u32 max_wqes_rqes_per_q; 523 u32 max_wqes_rqes_per_q;
503 u32 max_cq_cqes_per_cq; 524 u32 max_cq_cqes_per_cq;
504 u32 max_srq_rqe_sge; 525 u32 max_srq_rqe_sge;
505} __packed; 526};
506 527
507struct ocrdma_fw_ver_rsp { 528struct ocrdma_fw_ver_rsp {
508 struct ocrdma_mqe_hdr hdr; 529 struct ocrdma_mqe_hdr hdr;
509 struct ocrdma_mbx_rsp rsp; 530 struct ocrdma_mbx_rsp rsp;
510 531
511 u8 running_ver[32]; 532 u8 running_ver[32];
512} __packed; 533};
513 534
514struct ocrdma_fw_conf_rsp { 535struct ocrdma_fw_conf_rsp {
515 struct ocrdma_mqe_hdr hdr; 536 struct ocrdma_mqe_hdr hdr;
@@ -535,14 +556,41 @@ struct ocrdma_fw_conf_rsp {
535 u32 base_eqid; 556 u32 base_eqid;
536 u32 max_eq; 557 u32 max_eq;
537 558
538} __packed; 559};
539 560
540enum { 561enum {
541 OCRDMA_FN_MODE_RDMA = 0x4 562 OCRDMA_FN_MODE_RDMA = 0x4
542}; 563};
543 564
565struct ocrdma_get_link_speed_rsp {
566 struct ocrdma_mqe_hdr hdr;
567 struct ocrdma_mbx_rsp rsp;
568
569 u8 pt_port_num;
570 u8 link_duplex;
571 u8 phys_port_speed;
572 u8 phys_port_fault;
573 u16 rsvd1;
574 u16 qos_lnk_speed;
575 u8 logical_lnk_status;
576 u8 rsvd2[3];
577};
578
579enum {
580 OCRDMA_PHYS_LINK_SPEED_ZERO = 0x0,
581 OCRDMA_PHYS_LINK_SPEED_10MBPS = 0x1,
582 OCRDMA_PHYS_LINK_SPEED_100MBPS = 0x2,
583 OCRDMA_PHYS_LINK_SPEED_1GBPS = 0x3,
584 OCRDMA_PHYS_LINK_SPEED_10GBPS = 0x4,
585 OCRDMA_PHYS_LINK_SPEED_20GBPS = 0x5,
586 OCRDMA_PHYS_LINK_SPEED_25GBPS = 0x6,
587 OCRDMA_PHYS_LINK_SPEED_40GBPS = 0x7,
588 OCRDMA_PHYS_LINK_SPEED_100GBPS = 0x8
589};
590
544enum { 591enum {
545 OCRDMA_CREATE_CQ_VER2 = 2, 592 OCRDMA_CREATE_CQ_VER2 = 2,
593 OCRDMA_CREATE_CQ_VER3 = 3,
546 594
547 OCRDMA_CREATE_CQ_PAGE_CNT_MASK = 0xFFFF, 595 OCRDMA_CREATE_CQ_PAGE_CNT_MASK = 0xFFFF,
548 OCRDMA_CREATE_CQ_PAGE_SIZE_SHIFT = 16, 596 OCRDMA_CREATE_CQ_PAGE_SIZE_SHIFT = 16,
@@ -576,7 +624,8 @@ struct ocrdma_create_cq_cmd {
576 u32 pgsz_pgcnt; 624 u32 pgsz_pgcnt;
577 u32 ev_cnt_flags; 625 u32 ev_cnt_flags;
578 u32 eqn; 626 u32 eqn;
579 u32 cqe_count; 627 u16 cqe_count;
628 u16 pd_id;
580 u32 rsvd6; 629 u32 rsvd6;
581 struct ocrdma_pa pa[OCRDMA_CREATE_CQ_MAX_PAGES]; 630 struct ocrdma_pa pa[OCRDMA_CREATE_CQ_MAX_PAGES];
582}; 631};
@@ -584,7 +633,7 @@ struct ocrdma_create_cq_cmd {
584struct ocrdma_create_cq { 633struct ocrdma_create_cq {
585 struct ocrdma_mqe_hdr hdr; 634 struct ocrdma_mqe_hdr hdr;
586 struct ocrdma_create_cq_cmd cmd; 635 struct ocrdma_create_cq_cmd cmd;
587} __packed; 636};
588 637
589enum { 638enum {
590 OCRDMA_CREATE_CQ_RSP_CQ_ID_MASK = 0xFFFF 639 OCRDMA_CREATE_CQ_RSP_CQ_ID_MASK = 0xFFFF
@@ -593,12 +642,12 @@ enum {
593struct ocrdma_create_cq_cmd_rsp { 642struct ocrdma_create_cq_cmd_rsp {
594 struct ocrdma_mbx_rsp rsp; 643 struct ocrdma_mbx_rsp rsp;
595 u32 cq_id; 644 u32 cq_id;
596} __packed; 645};
597 646
598struct ocrdma_create_cq_rsp { 647struct ocrdma_create_cq_rsp {
599 struct ocrdma_mqe_hdr hdr; 648 struct ocrdma_mqe_hdr hdr;
600 struct ocrdma_create_cq_cmd_rsp rsp; 649 struct ocrdma_create_cq_cmd_rsp rsp;
601} __packed; 650};
602 651
603enum { 652enum {
604 OCRDMA_CREATE_MQ_V0_CQ_ID_SHIFT = 22, 653 OCRDMA_CREATE_MQ_V0_CQ_ID_SHIFT = 22,
@@ -617,12 +666,12 @@ struct ocrdma_create_mq_req {
617 u32 async_cqid_valid; 666 u32 async_cqid_valid;
618 u32 rsvd; 667 u32 rsvd;
619 struct ocrdma_pa pa[8]; 668 struct ocrdma_pa pa[8];
620} __packed; 669};
621 670
622struct ocrdma_create_mq_rsp { 671struct ocrdma_create_mq_rsp {
623 struct ocrdma_mbx_rsp rsp; 672 struct ocrdma_mbx_rsp rsp;
624 u32 id; 673 u32 id;
625} __packed; 674};
626 675
627enum { 676enum {
628 OCRDMA_DESTROY_CQ_QID_SHIFT = 0, 677 OCRDMA_DESTROY_CQ_QID_SHIFT = 0,
@@ -637,12 +686,12 @@ struct ocrdma_destroy_cq {
637 struct ocrdma_mbx_hdr req; 686 struct ocrdma_mbx_hdr req;
638 687
639 u32 bypass_flush_qid; 688 u32 bypass_flush_qid;
640} __packed; 689};
641 690
642struct ocrdma_destroy_cq_rsp { 691struct ocrdma_destroy_cq_rsp {
643 struct ocrdma_mqe_hdr hdr; 692 struct ocrdma_mqe_hdr hdr;
644 struct ocrdma_mbx_rsp rsp; 693 struct ocrdma_mbx_rsp rsp;
645} __packed; 694};
646 695
647enum { 696enum {
648 OCRDMA_QPT_GSI = 1, 697 OCRDMA_QPT_GSI = 1,
@@ -766,7 +815,7 @@ struct ocrdma_create_qp_req {
766 u32 dpp_credits_cqid; 815 u32 dpp_credits_cqid;
767 u32 rpir_lkey; 816 u32 rpir_lkey;
768 struct ocrdma_pa ird_addr[MAX_OCRDMA_IRD_PAGES]; 817 struct ocrdma_pa ird_addr[MAX_OCRDMA_IRD_PAGES];
769} __packed; 818};
770 819
771enum { 820enum {
772 OCRDMA_CREATE_QP_RSP_QP_ID_SHIFT = 0, 821 OCRDMA_CREATE_QP_RSP_QP_ID_SHIFT = 0,
@@ -820,18 +869,18 @@ struct ocrdma_create_qp_rsp {
820 u32 max_ord_ird; 869 u32 max_ord_ird;
821 u32 sq_rq_id; 870 u32 sq_rq_id;
822 u32 dpp_response; 871 u32 dpp_response;
823} __packed; 872};
824 873
825struct ocrdma_destroy_qp { 874struct ocrdma_destroy_qp {
826 struct ocrdma_mqe_hdr hdr; 875 struct ocrdma_mqe_hdr hdr;
827 struct ocrdma_mbx_hdr req; 876 struct ocrdma_mbx_hdr req;
828 u32 qp_id; 877 u32 qp_id;
829} __packed; 878};
830 879
831struct ocrdma_destroy_qp_rsp { 880struct ocrdma_destroy_qp_rsp {
832 struct ocrdma_mqe_hdr hdr; 881 struct ocrdma_mqe_hdr hdr;
833 struct ocrdma_mbx_rsp rsp; 882 struct ocrdma_mbx_rsp rsp;
834} __packed; 883};
835 884
836enum { 885enum {
837 OCRDMA_MODIFY_QP_ID_SHIFT = 0, 886 OCRDMA_MODIFY_QP_ID_SHIFT = 0,
@@ -975,7 +1024,7 @@ struct ocrdma_qp_params {
975 u32 dmac_b0_to_b3; 1024 u32 dmac_b0_to_b3;
976 u32 vlan_dmac_b4_to_b5; 1025 u32 vlan_dmac_b4_to_b5;
977 u32 qkey; 1026 u32 qkey;
978} __packed; 1027};
979 1028
980 1029
981struct ocrdma_modify_qp { 1030struct ocrdma_modify_qp {
@@ -986,7 +1035,7 @@ struct ocrdma_modify_qp {
986 u32 flags; 1035 u32 flags;
987 u32 rdma_flags; 1036 u32 rdma_flags;
988 u32 num_outstanding_atomic_rd; 1037 u32 num_outstanding_atomic_rd;
989} __packed; 1038};
990 1039
991enum { 1040enum {
992 OCRDMA_MODIFY_QP_RSP_MAX_RQE_SHIFT = 0, 1041 OCRDMA_MODIFY_QP_RSP_MAX_RQE_SHIFT = 0,
@@ -1007,7 +1056,7 @@ struct ocrdma_modify_qp_rsp {
1007 1056
1008 u32 max_wqe_rqe; 1057 u32 max_wqe_rqe;
1009 u32 max_ord_ird; 1058 u32 max_ord_ird;
1010} __packed; 1059};
1011 1060
1012struct ocrdma_query_qp { 1061struct ocrdma_query_qp {
1013 struct ocrdma_mqe_hdr hdr; 1062 struct ocrdma_mqe_hdr hdr;
@@ -1016,13 +1065,13 @@ struct ocrdma_query_qp {
1016#define OCRDMA_QUERY_UP_QP_ID_SHIFT 0 1065#define OCRDMA_QUERY_UP_QP_ID_SHIFT 0
1017#define OCRDMA_QUERY_UP_QP_ID_MASK 0xFFFFFF 1066#define OCRDMA_QUERY_UP_QP_ID_MASK 0xFFFFFF
1018 u32 qp_id; 1067 u32 qp_id;
1019} __packed; 1068};
1020 1069
1021struct ocrdma_query_qp_rsp { 1070struct ocrdma_query_qp_rsp {
1022 struct ocrdma_mqe_hdr hdr; 1071 struct ocrdma_mqe_hdr hdr;
1023 struct ocrdma_mbx_rsp rsp; 1072 struct ocrdma_mbx_rsp rsp;
1024 struct ocrdma_qp_params params; 1073 struct ocrdma_qp_params params;
1025} __packed; 1074};
1026 1075
1027enum { 1076enum {
1028 OCRDMA_CREATE_SRQ_PD_ID_SHIFT = 0, 1077 OCRDMA_CREATE_SRQ_PD_ID_SHIFT = 0,
@@ -1051,7 +1100,7 @@ struct ocrdma_create_srq {
1051 u32 max_sge_rqe; 1100 u32 max_sge_rqe;
1052 u32 pages_rqe_sz; 1101 u32 pages_rqe_sz;
1053 struct ocrdma_pa rq_addr[MAX_OCRDMA_SRQ_PAGES]; 1102 struct ocrdma_pa rq_addr[MAX_OCRDMA_SRQ_PAGES];
1054} __packed; 1103};
1055 1104
1056enum { 1105enum {
1057 OCRDMA_CREATE_SRQ_RSP_SRQ_ID_SHIFT = 0, 1106 OCRDMA_CREATE_SRQ_RSP_SRQ_ID_SHIFT = 0,
@@ -1070,7 +1119,7 @@ struct ocrdma_create_srq_rsp {
1070 1119
1071 u32 id; 1120 u32 id;
1072 u32 max_sge_rqe_allocated; 1121 u32 max_sge_rqe_allocated;
1073} __packed; 1122};
1074 1123
1075enum { 1124enum {
1076 OCRDMA_MODIFY_SRQ_ID_SHIFT = 0, 1125 OCRDMA_MODIFY_SRQ_ID_SHIFT = 0,
@@ -1089,7 +1138,7 @@ struct ocrdma_modify_srq {
1089 1138
1090 u32 id; 1139 u32 id;
1091 u32 limit_max_rqe; 1140 u32 limit_max_rqe;
1092} __packed; 1141};
1093 1142
1094enum { 1143enum {
1095 OCRDMA_QUERY_SRQ_ID_SHIFT = 0, 1144 OCRDMA_QUERY_SRQ_ID_SHIFT = 0,
@@ -1101,7 +1150,7 @@ struct ocrdma_query_srq {
1101 struct ocrdma_mbx_rsp req; 1150 struct ocrdma_mbx_rsp req;
1102 1151
1103 u32 id; 1152 u32 id;
1104} __packed; 1153};
1105 1154
1106enum { 1155enum {
1107 OCRDMA_QUERY_SRQ_RSP_PD_ID_SHIFT = 0, 1156 OCRDMA_QUERY_SRQ_RSP_PD_ID_SHIFT = 0,
@@ -1123,7 +1172,7 @@ struct ocrdma_query_srq_rsp {
1123 1172
1124 u32 max_rqe_pdid; 1173 u32 max_rqe_pdid;
1125 u32 srq_lmt_max_sge; 1174 u32 srq_lmt_max_sge;
1126} __packed; 1175};
1127 1176
1128enum { 1177enum {
1129 OCRDMA_DESTROY_SRQ_ID_SHIFT = 0, 1178 OCRDMA_DESTROY_SRQ_ID_SHIFT = 0,
@@ -1135,7 +1184,7 @@ struct ocrdma_destroy_srq {
1135 struct ocrdma_mbx_rsp req; 1184 struct ocrdma_mbx_rsp req;
1136 1185
1137 u32 id; 1186 u32 id;
1138} __packed; 1187};
1139 1188
1140enum { 1189enum {
1141 OCRDMA_ALLOC_PD_ENABLE_DPP = BIT(16), 1190 OCRDMA_ALLOC_PD_ENABLE_DPP = BIT(16),
@@ -1147,7 +1196,7 @@ struct ocrdma_alloc_pd {
1147 struct ocrdma_mqe_hdr hdr; 1196 struct ocrdma_mqe_hdr hdr;
1148 struct ocrdma_mbx_hdr req; 1197 struct ocrdma_mbx_hdr req;
1149 u32 enable_dpp_rsvd; 1198 u32 enable_dpp_rsvd;
1150} __packed; 1199};
1151 1200
1152enum { 1201enum {
1153 OCRDMA_ALLOC_PD_RSP_DPP = Bit(16), 1202 OCRDMA_ALLOC_PD_RSP_DPP = Bit(16),
@@ -1159,18 +1208,18 @@ struct ocrdma_alloc_pd_rsp {
1159 struct ocrdma_mqe_hdr hdr; 1208 struct ocrdma_mqe_hdr hdr;
1160 struct ocrdma_mbx_rsp rsp; 1209 struct ocrdma_mbx_rsp rsp;
1161 u32 dpp_page_pdid; 1210 u32 dpp_page_pdid;
1162} __packed; 1211};
1163 1212
1164struct ocrdma_dealloc_pd { 1213struct ocrdma_dealloc_pd {
1165 struct ocrdma_mqe_hdr hdr; 1214 struct ocrdma_mqe_hdr hdr;
1166 struct ocrdma_mbx_hdr req; 1215 struct ocrdma_mbx_hdr req;
1167 u32 id; 1216 u32 id;
1168} __packed; 1217};
1169 1218
1170struct ocrdma_dealloc_pd_rsp { 1219struct ocrdma_dealloc_pd_rsp {
1171 struct ocrdma_mqe_hdr hdr; 1220 struct ocrdma_mqe_hdr hdr;
1172 struct ocrdma_mbx_rsp rsp; 1221 struct ocrdma_mbx_rsp rsp;
1173} __packed; 1222};
1174 1223
1175enum { 1224enum {
1176 OCRDMA_ADDR_CHECK_ENABLE = 1, 1225 OCRDMA_ADDR_CHECK_ENABLE = 1,
@@ -1206,7 +1255,7 @@ struct ocrdma_alloc_lkey {
1206 1255
1207 u32 pdid; 1256 u32 pdid;
1208 u32 pbl_sz_flags; 1257 u32 pbl_sz_flags;
1209} __packed; 1258};
1210 1259
1211struct ocrdma_alloc_lkey_rsp { 1260struct ocrdma_alloc_lkey_rsp {
1212 struct ocrdma_mqe_hdr hdr; 1261 struct ocrdma_mqe_hdr hdr;
@@ -1214,7 +1263,7 @@ struct ocrdma_alloc_lkey_rsp {
1214 1263
1215 u32 lrkey; 1264 u32 lrkey;
1216 u32 num_pbl_rsvd; 1265 u32 num_pbl_rsvd;
1217} __packed; 1266};
1218 1267
1219struct ocrdma_dealloc_lkey { 1268struct ocrdma_dealloc_lkey {
1220 struct ocrdma_mqe_hdr hdr; 1269 struct ocrdma_mqe_hdr hdr;
@@ -1222,12 +1271,12 @@ struct ocrdma_dealloc_lkey {
1222 1271
1223 u32 lkey; 1272 u32 lkey;
1224 u32 rsvd_frmr; 1273 u32 rsvd_frmr;
1225} __packed; 1274};
1226 1275
1227struct ocrdma_dealloc_lkey_rsp { 1276struct ocrdma_dealloc_lkey_rsp {
1228 struct ocrdma_mqe_hdr hdr; 1277 struct ocrdma_mqe_hdr hdr;
1229 struct ocrdma_mbx_rsp rsp; 1278 struct ocrdma_mbx_rsp rsp;
1230} __packed; 1279};
1231 1280
1232#define MAX_OCRDMA_NSMR_PBL (u32)22 1281#define MAX_OCRDMA_NSMR_PBL (u32)22
1233#define MAX_OCRDMA_PBL_SIZE 65536 1282#define MAX_OCRDMA_PBL_SIZE 65536
@@ -1273,7 +1322,7 @@ struct ocrdma_reg_nsmr {
1273 struct ocrdma_mqe_hdr hdr; 1322 struct ocrdma_mqe_hdr hdr;
1274 struct ocrdma_mbx_hdr cmd; 1323 struct ocrdma_mbx_hdr cmd;
1275 1324
1276 u32 lrkey_key_index; 1325 u32 fr_mr;
1277 u32 num_pbl_pdid; 1326 u32 num_pbl_pdid;
1278 u32 flags_hpage_pbe_sz; 1327 u32 flags_hpage_pbe_sz;
1279 u32 totlen_low; 1328 u32 totlen_low;
@@ -1283,7 +1332,7 @@ struct ocrdma_reg_nsmr {
1283 u32 va_loaddr; 1332 u32 va_loaddr;
1284 u32 va_hiaddr; 1333 u32 va_hiaddr;
1285 struct ocrdma_pa pbl[MAX_OCRDMA_NSMR_PBL]; 1334 struct ocrdma_pa pbl[MAX_OCRDMA_NSMR_PBL];
1286} __packed; 1335};
1287 1336
1288enum { 1337enum {
1289 OCRDMA_REG_NSMR_CONT_PBL_SHIFT = 0, 1338 OCRDMA_REG_NSMR_CONT_PBL_SHIFT = 0,
@@ -1305,12 +1354,12 @@ struct ocrdma_reg_nsmr_cont {
1305 u32 last; 1354 u32 last;
1306 1355
1307 struct ocrdma_pa pbl[MAX_OCRDMA_NSMR_PBL]; 1356 struct ocrdma_pa pbl[MAX_OCRDMA_NSMR_PBL];
1308} __packed; 1357};
1309 1358
1310struct ocrdma_pbe { 1359struct ocrdma_pbe {
1311 u32 pa_hi; 1360 u32 pa_hi;
1312 u32 pa_lo; 1361 u32 pa_lo;
1313} __packed; 1362};
1314 1363
1315enum { 1364enum {
1316 OCRDMA_REG_NSMR_RSP_NUM_PBL_SHIFT = 16, 1365 OCRDMA_REG_NSMR_RSP_NUM_PBL_SHIFT = 16,
@@ -1322,7 +1371,7 @@ struct ocrdma_reg_nsmr_rsp {
1322 1371
1323 u32 lrkey; 1372 u32 lrkey;
1324 u32 num_pbl; 1373 u32 num_pbl;
1325} __packed; 1374};
1326 1375
1327enum { 1376enum {
1328 OCRDMA_REG_NSMR_CONT_RSP_LRKEY_INDEX_SHIFT = 0, 1377 OCRDMA_REG_NSMR_CONT_RSP_LRKEY_INDEX_SHIFT = 0,
@@ -1342,7 +1391,7 @@ struct ocrdma_reg_nsmr_cont_rsp {
1342 1391
1343 u32 lrkey_key_index; 1392 u32 lrkey_key_index;
1344 u32 num_pbl; 1393 u32 num_pbl;
1345} __packed; 1394};
1346 1395
1347enum { 1396enum {
1348 OCRDMA_ALLOC_MW_PD_ID_SHIFT = 0, 1397 OCRDMA_ALLOC_MW_PD_ID_SHIFT = 0,
@@ -1354,7 +1403,7 @@ struct ocrdma_alloc_mw {
1354 struct ocrdma_mbx_hdr req; 1403 struct ocrdma_mbx_hdr req;
1355 1404
1356 u32 pdid; 1405 u32 pdid;
1357} __packed; 1406};
1358 1407
1359enum { 1408enum {
1360 OCRDMA_ALLOC_MW_RSP_LRKEY_INDEX_SHIFT = 0, 1409 OCRDMA_ALLOC_MW_RSP_LRKEY_INDEX_SHIFT = 0,
@@ -1366,7 +1415,7 @@ struct ocrdma_alloc_mw_rsp {
1366 struct ocrdma_mbx_rsp rsp; 1415 struct ocrdma_mbx_rsp rsp;
1367 1416
1368 u32 lrkey_index; 1417 u32 lrkey_index;
1369} __packed; 1418};
1370 1419
1371struct ocrdma_attach_mcast { 1420struct ocrdma_attach_mcast {
1372 struct ocrdma_mqe_hdr hdr; 1421 struct ocrdma_mqe_hdr hdr;
@@ -1375,12 +1424,12 @@ struct ocrdma_attach_mcast {
1375 u8 mgid[16]; 1424 u8 mgid[16];
1376 u32 mac_b0_to_b3; 1425 u32 mac_b0_to_b3;
1377 u32 vlan_mac_b4_to_b5; 1426 u32 vlan_mac_b4_to_b5;
1378} __packed; 1427};
1379 1428
1380struct ocrdma_attach_mcast_rsp { 1429struct ocrdma_attach_mcast_rsp {
1381 struct ocrdma_mqe_hdr hdr; 1430 struct ocrdma_mqe_hdr hdr;
1382 struct ocrdma_mbx_rsp rsp; 1431 struct ocrdma_mbx_rsp rsp;
1383} __packed; 1432};
1384 1433
1385struct ocrdma_detach_mcast { 1434struct ocrdma_detach_mcast {
1386 struct ocrdma_mqe_hdr hdr; 1435 struct ocrdma_mqe_hdr hdr;
@@ -1389,12 +1438,12 @@ struct ocrdma_detach_mcast {
1389 u8 mgid[16]; 1438 u8 mgid[16];
1390 u32 mac_b0_to_b3; 1439 u32 mac_b0_to_b3;
1391 u32 vlan_mac_b4_to_b5; 1440 u32 vlan_mac_b4_to_b5;
1392} __packed; 1441};
1393 1442
1394struct ocrdma_detach_mcast_rsp { 1443struct ocrdma_detach_mcast_rsp {
1395 struct ocrdma_mqe_hdr hdr; 1444 struct ocrdma_mqe_hdr hdr;
1396 struct ocrdma_mbx_rsp rsp; 1445 struct ocrdma_mbx_rsp rsp;
1397} __packed; 1446};
1398 1447
1399enum { 1448enum {
1400 OCRDMA_CREATE_AH_NUM_PAGES_SHIFT = 19, 1449 OCRDMA_CREATE_AH_NUM_PAGES_SHIFT = 19,
@@ -1418,24 +1467,24 @@ struct ocrdma_create_ah_tbl {
1418 1467
1419 u32 ah_conf; 1468 u32 ah_conf;
1420 struct ocrdma_pa tbl_addr[8]; 1469 struct ocrdma_pa tbl_addr[8];
1421} __packed; 1470};
1422 1471
1423struct ocrdma_create_ah_tbl_rsp { 1472struct ocrdma_create_ah_tbl_rsp {
1424 struct ocrdma_mqe_hdr hdr; 1473 struct ocrdma_mqe_hdr hdr;
1425 struct ocrdma_mbx_rsp rsp; 1474 struct ocrdma_mbx_rsp rsp;
1426 u32 ahid; 1475 u32 ahid;
1427} __packed; 1476};
1428 1477
1429struct ocrdma_delete_ah_tbl { 1478struct ocrdma_delete_ah_tbl {
1430 struct ocrdma_mqe_hdr hdr; 1479 struct ocrdma_mqe_hdr hdr;
1431 struct ocrdma_mbx_hdr req; 1480 struct ocrdma_mbx_hdr req;
1432 u32 ahid; 1481 u32 ahid;
1433} __packed; 1482};
1434 1483
1435struct ocrdma_delete_ah_tbl_rsp { 1484struct ocrdma_delete_ah_tbl_rsp {
1436 struct ocrdma_mqe_hdr hdr; 1485 struct ocrdma_mqe_hdr hdr;
1437 struct ocrdma_mbx_rsp rsp; 1486 struct ocrdma_mbx_rsp rsp;
1438} __packed; 1487};
1439 1488
1440enum { 1489enum {
1441 OCRDMA_EQE_VALID_SHIFT = 0, 1490 OCRDMA_EQE_VALID_SHIFT = 0,
@@ -1448,7 +1497,7 @@ enum {
1448 1497
1449struct ocrdma_eqe { 1498struct ocrdma_eqe {
1450 u32 id_valid; 1499 u32 id_valid;
1451} __packed; 1500};
1452 1501
1453enum OCRDMA_CQE_STATUS { 1502enum OCRDMA_CQE_STATUS {
1454 OCRDMA_CQE_SUCCESS = 0, 1503 OCRDMA_CQE_SUCCESS = 0,
@@ -1532,14 +1581,14 @@ struct ocrdma_cqe {
1532 } cmn; 1581 } cmn;
1533 }; 1582 };
1534 u32 flags_status_srcqpn; /* w3 */ 1583 u32 flags_status_srcqpn; /* w3 */
1535} __packed; 1584};
1536 1585
1537struct ocrdma_sge { 1586struct ocrdma_sge {
1538 u32 addr_hi; 1587 u32 addr_hi;
1539 u32 addr_lo; 1588 u32 addr_lo;
1540 u32 lrkey; 1589 u32 lrkey;
1541 u32 len; 1590 u32 len;
1542} __packed; 1591};
1543 1592
1544enum { 1593enum {
1545 OCRDMA_FLAG_SIG = 0x1, 1594 OCRDMA_FLAG_SIG = 0x1,
@@ -1563,6 +1612,7 @@ enum OCRDMA_WQE_OPCODE {
1563 OCRDMA_SEND = 0x00, 1612 OCRDMA_SEND = 0x00,
1564 OCRDMA_CMP_SWP = 0x14, 1613 OCRDMA_CMP_SWP = 0x14,
1565 OCRDMA_BIND_MW = 0x10, 1614 OCRDMA_BIND_MW = 0x10,
1615 OCRDMA_FR_MR = 0x11,
1566 OCRDMA_RESV1 = 0x0A, 1616 OCRDMA_RESV1 = 0x0A,
1567 OCRDMA_LKEY_INV = 0x15, 1617 OCRDMA_LKEY_INV = 0x15,
1568 OCRDMA_FETCH_ADD = 0x13, 1618 OCRDMA_FETCH_ADD = 0x13,
@@ -1600,14 +1650,26 @@ struct ocrdma_hdr_wqe {
1600 u32 lkey; 1650 u32 lkey;
1601 }; 1651 };
1602 u32 total_len; 1652 u32 total_len;
1603} __packed; 1653};
1604 1654
1605struct ocrdma_ewqe_ud_hdr { 1655struct ocrdma_ewqe_ud_hdr {
1606 u32 rsvd_dest_qpn; 1656 u32 rsvd_dest_qpn;
1607 u32 qkey; 1657 u32 qkey;
1608 u32 rsvd_ahid; 1658 u32 rsvd_ahid;
1609 u32 rsvd; 1659 u32 rsvd;
1610} __packed; 1660};
1661
1662/* extended wqe followed by hdr_wqe for Fast Memory register */
1663struct ocrdma_ewqe_fr {
1664 u32 va_hi;
1665 u32 va_lo;
1666 u32 fbo_hi;
1667 u32 fbo_lo;
1668 u32 size_sge;
1669 u32 num_sges;
1670 u32 rsvd;
1671 u32 rsvd2;
1672};
1611 1673
1612struct ocrdma_eth_basic { 1674struct ocrdma_eth_basic {
1613 u8 dmac[6]; 1675 u8 dmac[6];
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index f36630e4b6be..6e982bb43c31 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -75,14 +75,15 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr)
75 attr->vendor_part_id = dev->nic_info.pdev->device; 75 attr->vendor_part_id = dev->nic_info.pdev->device;
76 attr->hw_ver = 0; 76 attr->hw_ver = 0;
77 attr->max_qp = dev->attr.max_qp; 77 attr->max_qp = dev->attr.max_qp;
78 attr->max_ah = dev->attr.max_qp; 78 attr->max_ah = OCRDMA_MAX_AH;
79 attr->max_qp_wr = dev->attr.max_wqe; 79 attr->max_qp_wr = dev->attr.max_wqe;
80 80
81 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 81 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
82 IB_DEVICE_RC_RNR_NAK_GEN | 82 IB_DEVICE_RC_RNR_NAK_GEN |
83 IB_DEVICE_SHUTDOWN_PORT | 83 IB_DEVICE_SHUTDOWN_PORT |
84 IB_DEVICE_SYS_IMAGE_GUID | 84 IB_DEVICE_SYS_IMAGE_GUID |
85 IB_DEVICE_LOCAL_DMA_LKEY; 85 IB_DEVICE_LOCAL_DMA_LKEY |
86 IB_DEVICE_MEM_MGT_EXTENSIONS;
86 attr->max_sge = min(dev->attr.max_send_sge, dev->attr.max_srq_sge); 87 attr->max_sge = min(dev->attr.max_send_sge, dev->attr.max_srq_sge);
87 attr->max_sge_rd = 0; 88 attr->max_sge_rd = 0;
88 attr->max_cq = dev->attr.max_cq; 89 attr->max_cq = dev->attr.max_cq;
@@ -96,7 +97,7 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr)
96 attr->max_qp_rd_atom = 97 attr->max_qp_rd_atom =
97 min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp); 98 min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp);
98 attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp; 99 attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp;
99 attr->max_srq = (dev->attr.max_qp - 1); 100 attr->max_srq = dev->attr.max_srq;
100 attr->max_srq_sge = dev->attr.max_srq_sge; 101 attr->max_srq_sge = dev->attr.max_srq_sge;
101 attr->max_srq_wr = dev->attr.max_rqe; 102 attr->max_srq_wr = dev->attr.max_rqe;
102 attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay; 103 attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay;
@@ -105,6 +106,45 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr)
105 return 0; 106 return 0;
106} 107}
107 108
109static inline void get_link_speed_and_width(struct ocrdma_dev *dev,
110 u8 *ib_speed, u8 *ib_width)
111{
112 int status;
113 u8 speed;
114
115 status = ocrdma_mbx_get_link_speed(dev, &speed);
116 if (status)
117 speed = OCRDMA_PHYS_LINK_SPEED_ZERO;
118
119 switch (speed) {
120 case OCRDMA_PHYS_LINK_SPEED_1GBPS:
121 *ib_speed = IB_SPEED_SDR;
122 *ib_width = IB_WIDTH_1X;
123 break;
124
125 case OCRDMA_PHYS_LINK_SPEED_10GBPS:
126 *ib_speed = IB_SPEED_QDR;
127 *ib_width = IB_WIDTH_1X;
128 break;
129
130 case OCRDMA_PHYS_LINK_SPEED_20GBPS:
131 *ib_speed = IB_SPEED_DDR;
132 *ib_width = IB_WIDTH_4X;
133 break;
134
135 case OCRDMA_PHYS_LINK_SPEED_40GBPS:
136 *ib_speed = IB_SPEED_QDR;
137 *ib_width = IB_WIDTH_4X;
138 break;
139
140 default:
141 /* Unsupported */
142 *ib_speed = IB_SPEED_SDR;
143 *ib_width = IB_WIDTH_1X;
144 };
145}
146
147
108int ocrdma_query_port(struct ib_device *ibdev, 148int ocrdma_query_port(struct ib_device *ibdev,
109 u8 port, struct ib_port_attr *props) 149 u8 port, struct ib_port_attr *props)
110{ 150{
@@ -141,8 +181,8 @@ int ocrdma_query_port(struct ib_device *ibdev,
141 props->pkey_tbl_len = 1; 181 props->pkey_tbl_len = 1;
142 props->bad_pkey_cntr = 0; 182 props->bad_pkey_cntr = 0;
143 props->qkey_viol_cntr = 0; 183 props->qkey_viol_cntr = 0;
144 props->active_width = IB_WIDTH_1X; 184 get_link_speed_and_width(dev, &props->active_speed,
145 props->active_speed = 4; 185 &props->active_width);
146 props->max_msg_sz = 0x80000000; 186 props->max_msg_sz = 0x80000000;
147 props->max_vl_num = 4; 187 props->max_vl_num = 4;
148 return 0; 188 return 0;
@@ -186,7 +226,7 @@ static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
186 226
187 mutex_lock(&uctx->mm_list_lock); 227 mutex_lock(&uctx->mm_list_lock);
188 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { 228 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
189 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 229 if (len != mm->key.len && phy_addr != mm->key.phy_addr)
190 continue; 230 continue;
191 231
192 list_del(&mm->entry); 232 list_del(&mm->entry);
@@ -204,7 +244,7 @@ static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
204 244
205 mutex_lock(&uctx->mm_list_lock); 245 mutex_lock(&uctx->mm_list_lock);
206 list_for_each_entry(mm, &uctx->mm_head, entry) { 246 list_for_each_entry(mm, &uctx->mm_head, entry) {
207 if (len != mm->key.len || phy_addr != mm->key.phy_addr) 247 if (len != mm->key.len && phy_addr != mm->key.phy_addr)
208 continue; 248 continue;
209 249
210 found = true; 250 found = true;
@@ -214,6 +254,108 @@ static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
214 return found; 254 return found;
215} 255}
216 256
257static struct ocrdma_pd *_ocrdma_alloc_pd(struct ocrdma_dev *dev,
258 struct ocrdma_ucontext *uctx,
259 struct ib_udata *udata)
260{
261 struct ocrdma_pd *pd = NULL;
262 int status = 0;
263
264 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
265 if (!pd)
266 return ERR_PTR(-ENOMEM);
267
268 if (udata && uctx) {
269 pd->dpp_enabled =
270 dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY;
271 pd->num_dpp_qp =
272 pd->dpp_enabled ? OCRDMA_PD_MAX_DPP_ENABLED_QP : 0;
273 }
274
275retry:
276 status = ocrdma_mbx_alloc_pd(dev, pd);
277 if (status) {
278 if (pd->dpp_enabled) {
279 pd->dpp_enabled = false;
280 pd->num_dpp_qp = 0;
281 goto retry;
282 } else {
283 kfree(pd);
284 return ERR_PTR(status);
285 }
286 }
287
288 return pd;
289}
290
291static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx,
292 struct ocrdma_pd *pd)
293{
294 return (uctx->cntxt_pd == pd ? true : false);
295}
296
297static int _ocrdma_dealloc_pd(struct ocrdma_dev *dev,
298 struct ocrdma_pd *pd)
299{
300 int status = 0;
301
302 status = ocrdma_mbx_dealloc_pd(dev, pd);
303 kfree(pd);
304 return status;
305}
306
307static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev,
308 struct ocrdma_ucontext *uctx,
309 struct ib_udata *udata)
310{
311 int status = 0;
312
313 uctx->cntxt_pd = _ocrdma_alloc_pd(dev, uctx, udata);
314 if (IS_ERR(uctx->cntxt_pd)) {
315 status = PTR_ERR(uctx->cntxt_pd);
316 uctx->cntxt_pd = NULL;
317 goto err;
318 }
319
320 uctx->cntxt_pd->uctx = uctx;
321 uctx->cntxt_pd->ibpd.device = &dev->ibdev;
322err:
323 return status;
324}
325
326static int ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx)
327{
328 int status = 0;
329 struct ocrdma_pd *pd = uctx->cntxt_pd;
330 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);
331
332 BUG_ON(uctx->pd_in_use);
333 uctx->cntxt_pd = NULL;
334 status = _ocrdma_dealloc_pd(dev, pd);
335 return status;
336}
337
338static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx)
339{
340 struct ocrdma_pd *pd = NULL;
341
342 mutex_lock(&uctx->mm_list_lock);
343 if (!uctx->pd_in_use) {
344 uctx->pd_in_use = true;
345 pd = uctx->cntxt_pd;
346 }
347 mutex_unlock(&uctx->mm_list_lock);
348
349 return pd;
350}
351
352static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx)
353{
354 mutex_lock(&uctx->mm_list_lock);
355 uctx->pd_in_use = false;
356 mutex_unlock(&uctx->mm_list_lock);
357}
358
217struct ib_ucontext *ocrdma_alloc_ucontext(struct ib_device *ibdev, 359struct ib_ucontext *ocrdma_alloc_ucontext(struct ib_device *ibdev,
218 struct ib_udata *udata) 360 struct ib_udata *udata)
219{ 361{
@@ -229,7 +371,6 @@ struct ib_ucontext *ocrdma_alloc_ucontext(struct ib_device *ibdev,
229 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 371 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
230 if (!ctx) 372 if (!ctx)
231 return ERR_PTR(-ENOMEM); 373 return ERR_PTR(-ENOMEM);
232 ctx->dev = dev;
233 INIT_LIST_HEAD(&ctx->mm_head); 374 INIT_LIST_HEAD(&ctx->mm_head);
234 mutex_init(&ctx->mm_list_lock); 375 mutex_init(&ctx->mm_list_lock);
235 376
@@ -249,6 +390,11 @@ struct ib_ucontext *ocrdma_alloc_ucontext(struct ib_device *ibdev,
249 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len); 390 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len);
250 if (status) 391 if (status)
251 goto map_err; 392 goto map_err;
393
394 status = ocrdma_alloc_ucontext_pd(dev, ctx, udata);
395 if (status)
396 goto pd_err;
397
252 resp.dev_id = dev->id; 398 resp.dev_id = dev->id;
253 resp.max_inline_data = dev->attr.max_inline_data; 399 resp.max_inline_data = dev->attr.max_inline_data;
254 resp.wqe_size = dev->attr.wqe_size; 400 resp.wqe_size = dev->attr.wqe_size;
@@ -262,6 +408,7 @@ struct ib_ucontext *ocrdma_alloc_ucontext(struct ib_device *ibdev,
262 return &ctx->ibucontext; 408 return &ctx->ibucontext;
263 409
264cpy_err: 410cpy_err:
411pd_err:
265 ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len); 412 ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len);
266map_err: 413map_err:
267 dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va, 414 dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va,
@@ -272,9 +419,13 @@ map_err:
272 419
273int ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx) 420int ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx)
274{ 421{
422 int status = 0;
275 struct ocrdma_mm *mm, *tmp; 423 struct ocrdma_mm *mm, *tmp;
276 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx); 424 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx);
277 struct pci_dev *pdev = uctx->dev->nic_info.pdev; 425 struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device);
426 struct pci_dev *pdev = dev->nic_info.pdev;
427
428 status = ocrdma_dealloc_ucontext_pd(uctx);
278 429
279 ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len); 430 ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len);
280 dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va, 431 dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va,
@@ -285,13 +436,13 @@ int ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx)
285 kfree(mm); 436 kfree(mm);
286 } 437 }
287 kfree(uctx); 438 kfree(uctx);
288 return 0; 439 return status;
289} 440}
290 441
291int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 442int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
292{ 443{
293 struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context); 444 struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context);
294 struct ocrdma_dev *dev = ucontext->dev; 445 struct ocrdma_dev *dev = get_ocrdma_dev(context->device);
295 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; 446 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
296 u64 unmapped_db = (u64) dev->nic_info.unmapped_db; 447 u64 unmapped_db = (u64) dev->nic_info.unmapped_db;
297 unsigned long len = (vma->vm_end - vma->vm_start); 448 unsigned long len = (vma->vm_end - vma->vm_start);
@@ -307,7 +458,10 @@ int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
307 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + 458 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
308 dev->nic_info.db_total_size)) && 459 dev->nic_info.db_total_size)) &&
309 (len <= dev->nic_info.db_page_size)) { 460 (len <= dev->nic_info.db_page_size)) {
310 /* doorbell mapping */ 461 if (vma->vm_flags & VM_READ)
462 return -EPERM;
463
464 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
311 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 465 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
312 len, vma->vm_page_prot); 466 len, vma->vm_page_prot);
313 } else if (dev->nic_info.dpp_unmapped_len && 467 } else if (dev->nic_info.dpp_unmapped_len &&
@@ -315,19 +469,20 @@ int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
315 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr + 469 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr +
316 dev->nic_info.dpp_unmapped_len)) && 470 dev->nic_info.dpp_unmapped_len)) &&
317 (len <= dev->nic_info.dpp_unmapped_len)) { 471 (len <= dev->nic_info.dpp_unmapped_len)) {
318 /* dpp area mapping */ 472 if (vma->vm_flags & VM_READ)
473 return -EPERM;
474
319 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 475 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
320 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 476 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
321 len, vma->vm_page_prot); 477 len, vma->vm_page_prot);
322 } else { 478 } else {
323 /* queue memory mapping */
324 status = remap_pfn_range(vma, vma->vm_start, 479 status = remap_pfn_range(vma, vma->vm_start,
325 vma->vm_pgoff, len, vma->vm_page_prot); 480 vma->vm_pgoff, len, vma->vm_page_prot);
326 } 481 }
327 return status; 482 return status;
328} 483}
329 484
330static int ocrdma_copy_pd_uresp(struct ocrdma_pd *pd, 485static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
331 struct ib_ucontext *ib_ctx, 486 struct ib_ucontext *ib_ctx,
332 struct ib_udata *udata) 487 struct ib_udata *udata)
333{ 488{
@@ -341,19 +496,18 @@ static int ocrdma_copy_pd_uresp(struct ocrdma_pd *pd,
341 memset(&rsp, 0, sizeof(rsp)); 496 memset(&rsp, 0, sizeof(rsp));
342 rsp.id = pd->id; 497 rsp.id = pd->id;
343 rsp.dpp_enabled = pd->dpp_enabled; 498 rsp.dpp_enabled = pd->dpp_enabled;
344 db_page_addr = pd->dev->nic_info.unmapped_db + 499 db_page_addr = ocrdma_get_db_addr(dev, pd->id);
345 (pd->id * pd->dev->nic_info.db_page_size); 500 db_page_size = dev->nic_info.db_page_size;
346 db_page_size = pd->dev->nic_info.db_page_size;
347 501
348 status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size); 502 status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size);
349 if (status) 503 if (status)
350 return status; 504 return status;
351 505
352 if (pd->dpp_enabled) { 506 if (pd->dpp_enabled) {
353 dpp_page_addr = pd->dev->nic_info.dpp_unmapped_addr + 507 dpp_page_addr = dev->nic_info.dpp_unmapped_addr +
354 (pd->id * OCRDMA_DPP_PAGE_SIZE); 508 (pd->id * PAGE_SIZE);
355 status = ocrdma_add_mmap(uctx, dpp_page_addr, 509 status = ocrdma_add_mmap(uctx, dpp_page_addr,
356 OCRDMA_DPP_PAGE_SIZE); 510 PAGE_SIZE);
357 if (status) 511 if (status)
358 goto dpp_map_err; 512 goto dpp_map_err;
359 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr); 513 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr);
@@ -369,7 +523,7 @@ static int ocrdma_copy_pd_uresp(struct ocrdma_pd *pd,
369 523
370ucopy_err: 524ucopy_err:
371 if (pd->dpp_enabled) 525 if (pd->dpp_enabled)
372 ocrdma_del_mmap(pd->uctx, dpp_page_addr, OCRDMA_DPP_PAGE_SIZE); 526 ocrdma_del_mmap(pd->uctx, dpp_page_addr, PAGE_SIZE);
373dpp_map_err: 527dpp_map_err:
374 ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size); 528 ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size);
375 return status; 529 return status;
@@ -381,76 +535,75 @@ struct ib_pd *ocrdma_alloc_pd(struct ib_device *ibdev,
381{ 535{
382 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 536 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
383 struct ocrdma_pd *pd; 537 struct ocrdma_pd *pd;
538 struct ocrdma_ucontext *uctx = NULL;
384 int status; 539 int status;
540 u8 is_uctx_pd = false;
385 541
386 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
387 if (!pd)
388 return ERR_PTR(-ENOMEM);
389 pd->dev = dev;
390 if (udata && context) { 542 if (udata && context) {
391 pd->dpp_enabled = (dev->nic_info.dev_family == 543 uctx = get_ocrdma_ucontext(context);
392 OCRDMA_GEN2_FAMILY) ? true : false; 544 pd = ocrdma_get_ucontext_pd(uctx);
393 pd->num_dpp_qp = 545 if (pd) {
394 pd->dpp_enabled ? OCRDMA_PD_MAX_DPP_ENABLED_QP : 0; 546 is_uctx_pd = true;
547 goto pd_mapping;
548 }
395 } 549 }
396 status = ocrdma_mbx_alloc_pd(dev, pd); 550
397 if (status) { 551 pd = _ocrdma_alloc_pd(dev, uctx, udata);
398 kfree(pd); 552 if (IS_ERR(pd)) {
399 return ERR_PTR(status); 553 status = PTR_ERR(pd);
554 goto exit;
400 } 555 }
401 556
557pd_mapping:
402 if (udata && context) { 558 if (udata && context) {
403 status = ocrdma_copy_pd_uresp(pd, context, udata); 559 status = ocrdma_copy_pd_uresp(dev, pd, context, udata);
404 if (status) 560 if (status)
405 goto err; 561 goto err;
406 } 562 }
407 return &pd->ibpd; 563 return &pd->ibpd;
408 564
409err: 565err:
410 ocrdma_dealloc_pd(&pd->ibpd); 566 if (is_uctx_pd) {
567 ocrdma_release_ucontext_pd(uctx);
568 } else {
569 status = ocrdma_mbx_dealloc_pd(dev, pd);
570 kfree(pd);
571 }
572exit:
411 return ERR_PTR(status); 573 return ERR_PTR(status);
412} 574}
413 575
414int ocrdma_dealloc_pd(struct ib_pd *ibpd) 576int ocrdma_dealloc_pd(struct ib_pd *ibpd)
415{ 577{
416 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 578 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
417 struct ocrdma_dev *dev = pd->dev; 579 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
418 int status; 580 struct ocrdma_ucontext *uctx = NULL;
581 int status = 0;
419 u64 usr_db; 582 u64 usr_db;
420 583
421 status = ocrdma_mbx_dealloc_pd(dev, pd); 584 uctx = pd->uctx;
422 if (pd->uctx) { 585 if (uctx) {
423 u64 dpp_db = dev->nic_info.dpp_unmapped_addr + 586 u64 dpp_db = dev->nic_info.dpp_unmapped_addr +
424 (pd->id * OCRDMA_DPP_PAGE_SIZE); 587 (pd->id * PAGE_SIZE);
425 if (pd->dpp_enabled) 588 if (pd->dpp_enabled)
426 ocrdma_del_mmap(pd->uctx, dpp_db, OCRDMA_DPP_PAGE_SIZE); 589 ocrdma_del_mmap(pd->uctx, dpp_db, PAGE_SIZE);
427 usr_db = dev->nic_info.unmapped_db + 590 usr_db = ocrdma_get_db_addr(dev, pd->id);
428 (pd->id * dev->nic_info.db_page_size);
429 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size); 591 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size);
592
593 if (is_ucontext_pd(uctx, pd)) {
594 ocrdma_release_ucontext_pd(uctx);
595 return status;
596 }
430 } 597 }
431 kfree(pd); 598 status = _ocrdma_dealloc_pd(dev, pd);
432 return status; 599 return status;
433} 600}
434 601
435static struct ocrdma_mr *ocrdma_alloc_lkey(struct ib_pd *ibpd, 602static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
436 int acc, u32 num_pbls, 603 u32 pdid, int acc, u32 num_pbls, u32 addr_check)
437 u32 addr_check)
438{ 604{
439 int status; 605 int status;
440 struct ocrdma_mr *mr;
441 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
442 struct ocrdma_dev *dev = pd->dev;
443
444 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
445 pr_err("%s(%d) leaving err, invalid access rights\n",
446 __func__, dev->id);
447 return ERR_PTR(-EINVAL);
448 }
449 606
450 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
451 if (!mr)
452 return ERR_PTR(-ENOMEM);
453 mr->hwmr.dev = dev;
454 mr->hwmr.fr_mr = 0; 607 mr->hwmr.fr_mr = 0;
455 mr->hwmr.local_rd = 1; 608 mr->hwmr.local_rd = 1;
456 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 609 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
@@ -460,25 +613,38 @@ static struct ocrdma_mr *ocrdma_alloc_lkey(struct ib_pd *ibpd,
460 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 613 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
461 mr->hwmr.num_pbls = num_pbls; 614 mr->hwmr.num_pbls = num_pbls;
462 615
463 status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pd->id, addr_check); 616 status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check);
464 if (status) { 617 if (status)
465 kfree(mr); 618 return status;
466 return ERR_PTR(-ENOMEM); 619
467 }
468 mr->pd = pd;
469 mr->ibmr.lkey = mr->hwmr.lkey; 620 mr->ibmr.lkey = mr->hwmr.lkey;
470 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 621 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
471 mr->ibmr.rkey = mr->hwmr.lkey; 622 mr->ibmr.rkey = mr->hwmr.lkey;
472 return mr; 623 return 0;
473} 624}
474 625
475struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc) 626struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc)
476{ 627{
628 int status;
477 struct ocrdma_mr *mr; 629 struct ocrdma_mr *mr;
630 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
631 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
478 632
479 mr = ocrdma_alloc_lkey(ibpd, acc, 0, OCRDMA_ADDR_CHECK_DISABLE); 633 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
480 if (IS_ERR(mr)) 634 pr_err("%s err, invalid access rights\n", __func__);
481 return ERR_CAST(mr); 635 return ERR_PTR(-EINVAL);
636 }
637
638 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
639 if (!mr)
640 return ERR_PTR(-ENOMEM);
641
642 status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0,
643 OCRDMA_ADDR_CHECK_DISABLE);
644 if (status) {
645 kfree(mr);
646 return ERR_PTR(status);
647 }
482 648
483 return &mr->ibmr; 649 return &mr->ibmr;
484} 650}
@@ -502,7 +668,8 @@ static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev,
502 } 668 }
503} 669}
504 670
505static int ocrdma_get_pbl_info(struct ocrdma_mr *mr, u32 num_pbes) 671static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
672 u32 num_pbes)
506{ 673{
507 u32 num_pbls = 0; 674 u32 num_pbls = 0;
508 u32 idx = 0; 675 u32 idx = 0;
@@ -518,7 +685,7 @@ static int ocrdma_get_pbl_info(struct ocrdma_mr *mr, u32 num_pbes)
518 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64))); 685 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64)));
519 num_pbls = num_pbls / (pbl_size / sizeof(u64)); 686 num_pbls = num_pbls / (pbl_size / sizeof(u64));
520 idx++; 687 idx++;
521 } while (num_pbls >= mr->hwmr.dev->attr.max_num_mr_pbl); 688 } while (num_pbls >= dev->attr.max_num_mr_pbl);
522 689
523 mr->hwmr.num_pbes = num_pbes; 690 mr->hwmr.num_pbes = num_pbes;
524 mr->hwmr.num_pbls = num_pbls; 691 mr->hwmr.num_pbls = num_pbls;
@@ -613,13 +780,12 @@ struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
613 u64 usr_addr, int acc, struct ib_udata *udata) 780 u64 usr_addr, int acc, struct ib_udata *udata)
614{ 781{
615 int status = -ENOMEM; 782 int status = -ENOMEM;
616 struct ocrdma_dev *dev; 783 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
617 struct ocrdma_mr *mr; 784 struct ocrdma_mr *mr;
618 struct ocrdma_pd *pd; 785 struct ocrdma_pd *pd;
619 u32 num_pbes; 786 u32 num_pbes;
620 787
621 pd = get_ocrdma_pd(ibpd); 788 pd = get_ocrdma_pd(ibpd);
622 dev = pd->dev;
623 789
624 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 790 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
625 return ERR_PTR(-EINVAL); 791 return ERR_PTR(-EINVAL);
@@ -627,14 +793,13 @@ struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
627 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 793 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
628 if (!mr) 794 if (!mr)
629 return ERR_PTR(status); 795 return ERR_PTR(status);
630 mr->hwmr.dev = dev;
631 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); 796 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
632 if (IS_ERR(mr->umem)) { 797 if (IS_ERR(mr->umem)) {
633 status = -EFAULT; 798 status = -EFAULT;
634 goto umem_err; 799 goto umem_err;
635 } 800 }
636 num_pbes = ib_umem_page_count(mr->umem); 801 num_pbes = ib_umem_page_count(mr->umem);
637 status = ocrdma_get_pbl_info(mr, num_pbes); 802 status = ocrdma_get_pbl_info(dev, mr, num_pbes);
638 if (status) 803 if (status)
639 goto umem_err; 804 goto umem_err;
640 805
@@ -654,7 +819,6 @@ struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
654 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc); 819 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);
655 if (status) 820 if (status)
656 goto mbx_err; 821 goto mbx_err;
657 mr->pd = pd;
658 mr->ibmr.lkey = mr->hwmr.lkey; 822 mr->ibmr.lkey = mr->hwmr.lkey;
659 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd) 823 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
660 mr->ibmr.rkey = mr->hwmr.lkey; 824 mr->ibmr.rkey = mr->hwmr.lkey;
@@ -671,7 +835,7 @@ umem_err:
671int ocrdma_dereg_mr(struct ib_mr *ib_mr) 835int ocrdma_dereg_mr(struct ib_mr *ib_mr)
672{ 836{
673 struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr); 837 struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr);
674 struct ocrdma_dev *dev = mr->hwmr.dev; 838 struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device);
675 int status; 839 int status;
676 840
677 status = ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey); 841 status = ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey);
@@ -686,29 +850,29 @@ int ocrdma_dereg_mr(struct ib_mr *ib_mr)
686 return status; 850 return status;
687} 851}
688 852
689static int ocrdma_copy_cq_uresp(struct ocrdma_cq *cq, struct ib_udata *udata, 853static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
854 struct ib_udata *udata,
690 struct ib_ucontext *ib_ctx) 855 struct ib_ucontext *ib_ctx)
691{ 856{
692 int status; 857 int status;
693 struct ocrdma_ucontext *uctx; 858 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ib_ctx);
694 struct ocrdma_create_cq_uresp uresp; 859 struct ocrdma_create_cq_uresp uresp;
695 860
696 memset(&uresp, 0, sizeof(uresp)); 861 memset(&uresp, 0, sizeof(uresp));
697 uresp.cq_id = cq->id; 862 uresp.cq_id = cq->id;
698 uresp.page_size = cq->len; 863 uresp.page_size = PAGE_ALIGN(cq->len);
699 uresp.num_pages = 1; 864 uresp.num_pages = 1;
700 uresp.max_hw_cqe = cq->max_hw_cqe; 865 uresp.max_hw_cqe = cq->max_hw_cqe;
701 uresp.page_addr[0] = cq->pa; 866 uresp.page_addr[0] = cq->pa;
702 uresp.db_page_addr = cq->dev->nic_info.unmapped_db; 867 uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id);
703 uresp.db_page_size = cq->dev->nic_info.db_page_size; 868 uresp.db_page_size = dev->nic_info.db_page_size;
704 uresp.phase_change = cq->phase_change ? 1 : 0; 869 uresp.phase_change = cq->phase_change ? 1 : 0;
705 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 870 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
706 if (status) { 871 if (status) {
707 pr_err("%s(%d) copy error cqid=0x%x.\n", 872 pr_err("%s(%d) copy error cqid=0x%x.\n",
708 __func__, cq->dev->id, cq->id); 873 __func__, dev->id, cq->id);
709 goto err; 874 goto err;
710 } 875 }
711 uctx = get_ocrdma_ucontext(ib_ctx);
712 status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size); 876 status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
713 if (status) 877 if (status)
714 goto err; 878 goto err;
@@ -728,6 +892,8 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, int entries, int vector,
728{ 892{
729 struct ocrdma_cq *cq; 893 struct ocrdma_cq *cq;
730 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); 894 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
895 struct ocrdma_ucontext *uctx = NULL;
896 u16 pd_id = 0;
731 int status; 897 int status;
732 struct ocrdma_create_cq_ureq ureq; 898 struct ocrdma_create_cq_ureq ureq;
733 899
@@ -744,15 +910,19 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, int entries, int vector,
744 spin_lock_init(&cq->comp_handler_lock); 910 spin_lock_init(&cq->comp_handler_lock);
745 INIT_LIST_HEAD(&cq->sq_head); 911 INIT_LIST_HEAD(&cq->sq_head);
746 INIT_LIST_HEAD(&cq->rq_head); 912 INIT_LIST_HEAD(&cq->rq_head);
747 cq->dev = dev;
748 913
749 status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq); 914 if (ib_ctx) {
915 uctx = get_ocrdma_ucontext(ib_ctx);
916 pd_id = uctx->cntxt_pd->id;
917 }
918
919 status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id);
750 if (status) { 920 if (status) {
751 kfree(cq); 921 kfree(cq);
752 return ERR_PTR(status); 922 return ERR_PTR(status);
753 } 923 }
754 if (ib_ctx) { 924 if (ib_ctx) {
755 status = ocrdma_copy_cq_uresp(cq, udata, ib_ctx); 925 status = ocrdma_copy_cq_uresp(dev, cq, udata, ib_ctx);
756 if (status) 926 if (status)
757 goto ctx_err; 927 goto ctx_err;
758 } 928 }
@@ -786,13 +956,17 @@ int ocrdma_destroy_cq(struct ib_cq *ibcq)
786{ 956{
787 int status; 957 int status;
788 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); 958 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
789 struct ocrdma_dev *dev = cq->dev; 959 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
960 int pdid = 0;
790 961
791 status = ocrdma_mbx_destroy_cq(dev, cq); 962 status = ocrdma_mbx_destroy_cq(dev, cq);
792 963
793 if (cq->ucontext) { 964 if (cq->ucontext) {
794 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa, cq->len); 965 pdid = cq->ucontext->cntxt_pd->id;
795 ocrdma_del_mmap(cq->ucontext, dev->nic_info.unmapped_db, 966 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa,
967 PAGE_ALIGN(cq->len));
968 ocrdma_del_mmap(cq->ucontext,
969 ocrdma_get_db_addr(dev, pdid),
796 dev->nic_info.db_page_size); 970 dev->nic_info.db_page_size);
797 } 971 }
798 dev->cq_tbl[cq->id] = NULL; 972 dev->cq_tbl[cq->id] = NULL;
@@ -820,14 +994,17 @@ static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
820static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev, 994static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev,
821 struct ib_qp_init_attr *attrs) 995 struct ib_qp_init_attr *attrs)
822{ 996{
823 if (attrs->qp_type != IB_QPT_GSI && 997 if ((attrs->qp_type != IB_QPT_GSI) &&
824 attrs->qp_type != IB_QPT_RC && 998 (attrs->qp_type != IB_QPT_RC) &&
825 attrs->qp_type != IB_QPT_UD) { 999 (attrs->qp_type != IB_QPT_UC) &&
1000 (attrs->qp_type != IB_QPT_UD)) {
826 pr_err("%s(%d) unsupported qp type=0x%x requested\n", 1001 pr_err("%s(%d) unsupported qp type=0x%x requested\n",
827 __func__, dev->id, attrs->qp_type); 1002 __func__, dev->id, attrs->qp_type);
828 return -EINVAL; 1003 return -EINVAL;
829 } 1004 }
830 if (attrs->cap.max_send_wr > dev->attr.max_wqe) { 1005 /* Skip the check for QP1 to support CM size of 128 */
1006 if ((attrs->qp_type != IB_QPT_GSI) &&
1007 (attrs->cap.max_send_wr > dev->attr.max_wqe)) {
831 pr_err("%s(%d) unsupported send_wr=0x%x requested\n", 1008 pr_err("%s(%d) unsupported send_wr=0x%x requested\n",
832 __func__, dev->id, attrs->cap.max_send_wr); 1009 __func__, dev->id, attrs->cap.max_send_wr);
833 pr_err("%s(%d) supported send_wr=0x%x\n", 1010 pr_err("%s(%d) supported send_wr=0x%x\n",
@@ -878,11 +1055,9 @@ static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev,
878 /* verify consumer QPs are not trying to use GSI QP's CQ */ 1055 /* verify consumer QPs are not trying to use GSI QP's CQ */
879 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { 1056 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
880 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) || 1057 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) ||
881 (dev->gsi_sqcq == get_ocrdma_cq(attrs->recv_cq)) || 1058 (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) {
882 (dev->gsi_rqcq == get_ocrdma_cq(attrs->send_cq)) ||
883 (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) {
884 pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n", 1059 pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n",
885 __func__, dev->id); 1060 __func__, dev->id);
886 return -EINVAL; 1061 return -EINVAL;
887 } 1062 }
888 } 1063 }
@@ -905,13 +1080,13 @@ static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
905 uresp.qp_id = qp->id; 1080 uresp.qp_id = qp->id;
906 uresp.sq_dbid = qp->sq.dbid; 1081 uresp.sq_dbid = qp->sq.dbid;
907 uresp.num_sq_pages = 1; 1082 uresp.num_sq_pages = 1;
908 uresp.sq_page_size = qp->sq.len; 1083 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len);
909 uresp.sq_page_addr[0] = qp->sq.pa; 1084 uresp.sq_page_addr[0] = qp->sq.pa;
910 uresp.num_wqe_allocated = qp->sq.max_cnt; 1085 uresp.num_wqe_allocated = qp->sq.max_cnt;
911 if (!srq) { 1086 if (!srq) {
912 uresp.rq_dbid = qp->rq.dbid; 1087 uresp.rq_dbid = qp->rq.dbid;
913 uresp.num_rq_pages = 1; 1088 uresp.num_rq_pages = 1;
914 uresp.rq_page_size = qp->rq.len; 1089 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len);
915 uresp.rq_page_addr[0] = qp->rq.pa; 1090 uresp.rq_page_addr[0] = qp->rq.pa;
916 uresp.num_rqe_allocated = qp->rq.max_cnt; 1091 uresp.num_rqe_allocated = qp->rq.max_cnt;
917 } 1092 }
@@ -919,9 +1094,8 @@ static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
919 uresp.db_page_size = dev->nic_info.db_page_size; 1094 uresp.db_page_size = dev->nic_info.db_page_size;
920 if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) { 1095 if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
921 uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET; 1096 uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET;
922 uresp.db_rq_offset = ((qp->id & 0xFFFF) < 128) ? 1097 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;
923 OCRDMA_DB_GEN2_RQ1_OFFSET : OCRDMA_DB_GEN2_RQ2_OFFSET; 1098 uresp.db_shift = 24;
924 uresp.db_shift = (qp->id < 128) ? 24 : 16;
925 } else { 1099 } else {
926 uresp.db_sq_offset = OCRDMA_DB_SQ_OFFSET; 1100 uresp.db_sq_offset = OCRDMA_DB_SQ_OFFSET;
927 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET; 1101 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET;
@@ -964,8 +1138,7 @@ static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
964 OCRDMA_DB_GEN2_SQ_OFFSET; 1138 OCRDMA_DB_GEN2_SQ_OFFSET;
965 qp->rq_db = dev->nic_info.db + 1139 qp->rq_db = dev->nic_info.db +
966 (pd->id * dev->nic_info.db_page_size) + 1140 (pd->id * dev->nic_info.db_page_size) +
967 ((qp->id < 128) ? 1141 OCRDMA_DB_GEN2_RQ_OFFSET;
968 OCRDMA_DB_GEN2_RQ1_OFFSET : OCRDMA_DB_GEN2_RQ2_OFFSET);
969 } else { 1142 } else {
970 qp->sq_db = dev->nic_info.db + 1143 qp->sq_db = dev->nic_info.db +
971 (pd->id * dev->nic_info.db_page_size) + 1144 (pd->id * dev->nic_info.db_page_size) +
@@ -1006,6 +1179,7 @@ static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp,
1006 qp->sq.max_sges = attrs->cap.max_send_sge; 1179 qp->sq.max_sges = attrs->cap.max_send_sge;
1007 qp->rq.max_sges = attrs->cap.max_recv_sge; 1180 qp->rq.max_sges = attrs->cap.max_recv_sge;
1008 qp->state = OCRDMA_QPS_RST; 1181 qp->state = OCRDMA_QPS_RST;
1182 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1009} 1183}
1010 1184
1011 1185
@@ -1026,7 +1200,7 @@ struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd,
1026 int status; 1200 int status;
1027 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 1201 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
1028 struct ocrdma_qp *qp; 1202 struct ocrdma_qp *qp;
1029 struct ocrdma_dev *dev = pd->dev; 1203 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
1030 struct ocrdma_create_qp_ureq ureq; 1204 struct ocrdma_create_qp_ureq ureq;
1031 u16 dpp_credit_lmt, dpp_offset; 1205 u16 dpp_credit_lmt, dpp_offset;
1032 1206
@@ -1046,6 +1220,9 @@ struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd,
1046 } 1220 }
1047 qp->dev = dev; 1221 qp->dev = dev;
1048 ocrdma_set_qp_init_params(qp, pd, attrs); 1222 ocrdma_set_qp_init_params(qp, pd, attrs);
1223 if (udata == NULL)
1224 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 |
1225 OCRDMA_QP_FAST_REG);
1049 1226
1050 mutex_lock(&dev->dev_lock); 1227 mutex_lock(&dev->dev_lock);
1051 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq, 1228 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq,
@@ -1056,8 +1233,6 @@ struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd,
1056 1233
1057 /* user space QP's wr_id table are managed in library */ 1234 /* user space QP's wr_id table are managed in library */
1058 if (udata == NULL) { 1235 if (udata == NULL) {
1059 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 |
1060 OCRDMA_QP_FAST_REG);
1061 status = ocrdma_alloc_wr_id_tbl(qp); 1236 status = ocrdma_alloc_wr_id_tbl(qp);
1062 if (status) 1237 if (status)
1063 goto map_err; 1238 goto map_err;
@@ -1093,6 +1268,17 @@ gen_err:
1093 return ERR_PTR(status); 1268 return ERR_PTR(status);
1094} 1269}
1095 1270
1271
1272static void ocrdma_flush_rq_db(struct ocrdma_qp *qp)
1273{
1274 if (qp->db_cache) {
1275 u32 val = qp->rq.dbid | (qp->db_cache <<
1276 ocrdma_get_num_posted_shift(qp));
1277 iowrite32(val, qp->rq_db);
1278 qp->db_cache = 0;
1279 }
1280}
1281
1096int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1282int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1097 int attr_mask) 1283 int attr_mask)
1098{ 1284{
@@ -1104,13 +1290,16 @@ int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1104 qp = get_ocrdma_qp(ibqp); 1290 qp = get_ocrdma_qp(ibqp);
1105 dev = qp->dev; 1291 dev = qp->dev;
1106 if (attr_mask & IB_QP_STATE) 1292 if (attr_mask & IB_QP_STATE)
1107 status = ocrdma_qp_state_machine(qp, attr->qp_state, &old_qps); 1293 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps);
1108 /* if new and previous states are same hw doesn't need to 1294 /* if new and previous states are same hw doesn't need to
1109 * know about it. 1295 * know about it.
1110 */ 1296 */
1111 if (status < 0) 1297 if (status < 0)
1112 return status; 1298 return status;
1113 status = ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask, old_qps); 1299 status = ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask, old_qps);
1300 if (!status && attr_mask & IB_QP_STATE && attr->qp_state == IB_QPS_RTR)
1301 ocrdma_flush_rq_db(qp);
1302
1114 return status; 1303 return status;
1115} 1304}
1116 1305
@@ -1215,7 +1404,7 @@ int ocrdma_query_qp(struct ib_qp *ibqp,
1215 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1; 1404 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1;
1216 qp_attr->cap.max_send_sge = qp->sq.max_sges; 1405 qp_attr->cap.max_send_sge = qp->sq.max_sges;
1217 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 1406 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
1218 qp_attr->cap.max_inline_data = dev->attr.max_inline_data; 1407 qp_attr->cap.max_inline_data = qp->max_inline_data;
1219 qp_init_attr->cap = qp_attr->cap; 1408 qp_init_attr->cap = qp_attr->cap;
1220 memcpy(&qp_attr->ah_attr.grh.dgid, &params.dgid[0], 1409 memcpy(&qp_attr->ah_attr.grh.dgid, &params.dgid[0],
1221 sizeof(params.dgid)); 1410 sizeof(params.dgid));
@@ -1278,23 +1467,17 @@ static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, int idx)
1278 1467
1279static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q) 1468static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q)
1280{ 1469{
1281 int free_cnt; 1470 return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt;
1282 if (q->head >= q->tail)
1283 free_cnt = (q->max_cnt - q->head) + q->tail;
1284 else
1285 free_cnt = q->tail - q->head;
1286 return free_cnt;
1287} 1471}
1288 1472
1289static int is_hw_sq_empty(struct ocrdma_qp *qp) 1473static int is_hw_sq_empty(struct ocrdma_qp *qp)
1290{ 1474{
1291 return (qp->sq.tail == qp->sq.head && 1475 return (qp->sq.tail == qp->sq.head);
1292 ocrdma_hwq_free_cnt(&qp->sq) ? 1 : 0);
1293} 1476}
1294 1477
1295static int is_hw_rq_empty(struct ocrdma_qp *qp) 1478static int is_hw_rq_empty(struct ocrdma_qp *qp)
1296{ 1479{
1297 return (qp->rq.tail == qp->rq.head) ? 1 : 0; 1480 return (qp->rq.tail == qp->rq.head);
1298} 1481}
1299 1482
1300static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q) 1483static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q)
@@ -1360,17 +1543,18 @@ static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
1360 */ 1543 */
1361 discard_cnt += 1; 1544 discard_cnt += 1;
1362 cqe->cmn.qpn = 0; 1545 cqe->cmn.qpn = 0;
1363 if (is_cqe_for_sq(cqe)) 1546 if (is_cqe_for_sq(cqe)) {
1364 ocrdma_hwq_inc_tail(&qp->sq); 1547 ocrdma_hwq_inc_tail(&qp->sq);
1365 else { 1548 } else {
1366 if (qp->srq) { 1549 if (qp->srq) {
1367 spin_lock_irqsave(&qp->srq->q_lock, flags); 1550 spin_lock_irqsave(&qp->srq->q_lock, flags);
1368 ocrdma_hwq_inc_tail(&qp->srq->rq); 1551 ocrdma_hwq_inc_tail(&qp->srq->rq);
1369 ocrdma_srq_toggle_bit(qp->srq, cur_getp); 1552 ocrdma_srq_toggle_bit(qp->srq, cur_getp);
1370 spin_unlock_irqrestore(&qp->srq->q_lock, flags); 1553 spin_unlock_irqrestore(&qp->srq->q_lock, flags);
1371 1554
1372 } else 1555 } else {
1373 ocrdma_hwq_inc_tail(&qp->rq); 1556 ocrdma_hwq_inc_tail(&qp->rq);
1557 }
1374 } 1558 }
1375skip_cqe: 1559skip_cqe:
1376 cur_getp = (cur_getp + 1) % cq->max_hw_cqe; 1560 cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
@@ -1378,7 +1562,7 @@ skip_cqe:
1378 spin_unlock_irqrestore(&cq->cq_lock, cq_flags); 1562 spin_unlock_irqrestore(&cq->cq_lock, cq_flags);
1379} 1563}
1380 1564
1381static void ocrdma_del_flush_qp(struct ocrdma_qp *qp) 1565void ocrdma_del_flush_qp(struct ocrdma_qp *qp)
1382{ 1566{
1383 int found = false; 1567 int found = false;
1384 unsigned long flags; 1568 unsigned long flags;
@@ -1444,9 +1628,11 @@ int ocrdma_destroy_qp(struct ib_qp *ibqp)
1444 mutex_unlock(&dev->dev_lock); 1628 mutex_unlock(&dev->dev_lock);
1445 1629
1446 if (pd->uctx) { 1630 if (pd->uctx) {
1447 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa, qp->sq.len); 1631 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa,
1632 PAGE_ALIGN(qp->sq.len));
1448 if (!qp->srq) 1633 if (!qp->srq)
1449 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa, qp->rq.len); 1634 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa,
1635 PAGE_ALIGN(qp->rq.len));
1450 } 1636 }
1451 1637
1452 ocrdma_del_flush_qp(qp); 1638 ocrdma_del_flush_qp(qp);
@@ -1457,7 +1643,8 @@ int ocrdma_destroy_qp(struct ib_qp *ibqp)
1457 return status; 1643 return status;
1458} 1644}
1459 1645
1460static int ocrdma_copy_srq_uresp(struct ocrdma_srq *srq, struct ib_udata *udata) 1646static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq,
1647 struct ib_udata *udata)
1461{ 1648{
1462 int status; 1649 int status;
1463 struct ocrdma_create_srq_uresp uresp; 1650 struct ocrdma_create_srq_uresp uresp;
@@ -1467,12 +1654,12 @@ static int ocrdma_copy_srq_uresp(struct ocrdma_srq *srq, struct ib_udata *udata)
1467 uresp.num_rq_pages = 1; 1654 uresp.num_rq_pages = 1;
1468 uresp.rq_page_addr[0] = srq->rq.pa; 1655 uresp.rq_page_addr[0] = srq->rq.pa;
1469 uresp.rq_page_size = srq->rq.len; 1656 uresp.rq_page_size = srq->rq.len;
1470 uresp.db_page_addr = srq->dev->nic_info.unmapped_db + 1657 uresp.db_page_addr = dev->nic_info.unmapped_db +
1471 (srq->pd->id * srq->dev->nic_info.db_page_size); 1658 (srq->pd->id * dev->nic_info.db_page_size);
1472 uresp.db_page_size = srq->dev->nic_info.db_page_size; 1659 uresp.db_page_size = dev->nic_info.db_page_size;
1473 uresp.num_rqe_allocated = srq->rq.max_cnt; 1660 uresp.num_rqe_allocated = srq->rq.max_cnt;
1474 if (srq->dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) { 1661 if (dev->nic_info.dev_family == OCRDMA_GEN2_FAMILY) {
1475 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ1_OFFSET; 1662 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;
1476 uresp.db_shift = 24; 1663 uresp.db_shift = 24;
1477 } else { 1664 } else {
1478 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET; 1665 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET;
@@ -1495,7 +1682,7 @@ struct ib_srq *ocrdma_create_srq(struct ib_pd *ibpd,
1495{ 1682{
1496 int status = -ENOMEM; 1683 int status = -ENOMEM;
1497 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd); 1684 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
1498 struct ocrdma_dev *dev = pd->dev; 1685 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
1499 struct ocrdma_srq *srq; 1686 struct ocrdma_srq *srq;
1500 1687
1501 if (init_attr->attr.max_sge > dev->attr.max_recv_sge) 1688 if (init_attr->attr.max_sge > dev->attr.max_recv_sge)
@@ -1508,10 +1695,9 @@ struct ib_srq *ocrdma_create_srq(struct ib_pd *ibpd,
1508 return ERR_PTR(status); 1695 return ERR_PTR(status);
1509 1696
1510 spin_lock_init(&srq->q_lock); 1697 spin_lock_init(&srq->q_lock);
1511 srq->dev = dev;
1512 srq->pd = pd; 1698 srq->pd = pd;
1513 srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size); 1699 srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size);
1514 status = ocrdma_mbx_create_srq(srq, init_attr, pd); 1700 status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd);
1515 if (status) 1701 if (status)
1516 goto err; 1702 goto err;
1517 1703
@@ -1538,7 +1724,7 @@ struct ib_srq *ocrdma_create_srq(struct ib_pd *ibpd,
1538 } 1724 }
1539 1725
1540 if (udata) { 1726 if (udata) {
1541 status = ocrdma_copy_srq_uresp(srq, udata); 1727 status = ocrdma_copy_srq_uresp(dev, srq, udata);
1542 if (status) 1728 if (status)
1543 goto arm_err; 1729 goto arm_err;
1544 } 1730 }
@@ -1584,15 +1770,15 @@ int ocrdma_destroy_srq(struct ib_srq *ibsrq)
1584{ 1770{
1585 int status; 1771 int status;
1586 struct ocrdma_srq *srq; 1772 struct ocrdma_srq *srq;
1587 struct ocrdma_dev *dev; 1773 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);
1588 1774
1589 srq = get_ocrdma_srq(ibsrq); 1775 srq = get_ocrdma_srq(ibsrq);
1590 dev = srq->dev;
1591 1776
1592 status = ocrdma_mbx_destroy_srq(dev, srq); 1777 status = ocrdma_mbx_destroy_srq(dev, srq);
1593 1778
1594 if (srq->pd->uctx) 1779 if (srq->pd->uctx)
1595 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa, srq->rq.len); 1780 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa,
1781 PAGE_ALIGN(srq->rq.len));
1596 1782
1597 kfree(srq->idx_bit_fields); 1783 kfree(srq->idx_bit_fields);
1598 kfree(srq->rqe_wr_id_tbl); 1784 kfree(srq->rqe_wr_id_tbl);
@@ -1634,23 +1820,43 @@ static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr,
1634 memset(sge, 0, sizeof(*sge)); 1820 memset(sge, 0, sizeof(*sge));
1635} 1821}
1636 1822
1823static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge)
1824{
1825 uint32_t total_len = 0, i;
1826
1827 for (i = 0; i < num_sge; i++)
1828 total_len += sg_list[i].length;
1829 return total_len;
1830}
1831
1832
1637static int ocrdma_build_inline_sges(struct ocrdma_qp *qp, 1833static int ocrdma_build_inline_sges(struct ocrdma_qp *qp,
1638 struct ocrdma_hdr_wqe *hdr, 1834 struct ocrdma_hdr_wqe *hdr,
1639 struct ocrdma_sge *sge, 1835 struct ocrdma_sge *sge,
1640 struct ib_send_wr *wr, u32 wqe_size) 1836 struct ib_send_wr *wr, u32 wqe_size)
1641{ 1837{
1642 if (wr->send_flags & IB_SEND_INLINE) { 1838 int i;
1643 if (wr->sg_list[0].length > qp->max_inline_data) { 1839 char *dpp_addr;
1840
1841 if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) {
1842 hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge);
1843 if (unlikely(hdr->total_len > qp->max_inline_data)) {
1644 pr_err("%s() supported_len=0x%x,\n" 1844 pr_err("%s() supported_len=0x%x,\n"
1645 " unspported len req=0x%x\n", __func__, 1845 " unspported len req=0x%x\n", __func__,
1646 qp->max_inline_data, wr->sg_list[0].length); 1846 qp->max_inline_data, hdr->total_len);
1647 return -EINVAL; 1847 return -EINVAL;
1648 } 1848 }
1649 memcpy(sge, 1849 dpp_addr = (char *)sge;
1650 (void *)(unsigned long)wr->sg_list[0].addr, 1850 for (i = 0; i < wr->num_sge; i++) {
1651 wr->sg_list[0].length); 1851 memcpy(dpp_addr,
1652 hdr->total_len = wr->sg_list[0].length; 1852 (void *)(unsigned long)wr->sg_list[i].addr,
1853 wr->sg_list[i].length);
1854 dpp_addr += wr->sg_list[i].length;
1855 }
1856
1653 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES); 1857 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES);
1858 if (0 == hdr->total_len)
1859 wqe_size += sizeof(struct ocrdma_sge);
1654 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT); 1860 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT);
1655 } else { 1861 } else {
1656 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list); 1862 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
@@ -1675,8 +1881,9 @@ static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1675 ocrdma_build_ud_hdr(qp, hdr, wr); 1881 ocrdma_build_ud_hdr(qp, hdr, wr);
1676 sge = (struct ocrdma_sge *)(hdr + 2); 1882 sge = (struct ocrdma_sge *)(hdr + 2);
1677 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr); 1883 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr);
1678 } else 1884 } else {
1679 sge = (struct ocrdma_sge *)(hdr + 1); 1885 sge = (struct ocrdma_sge *)(hdr + 1);
1886 }
1680 1887
1681 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size); 1888 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1682 return status; 1889 return status;
@@ -1719,6 +1926,96 @@ static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1719 ext_rw->len = hdr->total_len; 1926 ext_rw->len = hdr->total_len;
1720} 1927}
1721 1928
1929static void build_frmr_pbes(struct ib_send_wr *wr, struct ocrdma_pbl *pbl_tbl,
1930 struct ocrdma_hw_mr *hwmr)
1931{
1932 int i;
1933 u64 buf_addr = 0;
1934 int num_pbes;
1935 struct ocrdma_pbe *pbe;
1936
1937 pbe = (struct ocrdma_pbe *)pbl_tbl->va;
1938 num_pbes = 0;
1939
1940 /* go through the OS phy regions & fill hw pbe entries into pbls. */
1941 for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) {
1942 /* number of pbes can be more for one OS buf, when
1943 * buffers are of different sizes.
1944 * split the ib_buf to one or more pbes.
1945 */
1946 buf_addr = wr->wr.fast_reg.page_list->page_list[i];
1947 pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK));
1948 pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr));
1949 num_pbes += 1;
1950 pbe++;
1951
1952 /* if the pbl is full storing the pbes,
1953 * move to next pbl.
1954 */
1955 if (num_pbes == (hwmr->pbl_size/sizeof(u64))) {
1956 pbl_tbl++;
1957 pbe = (struct ocrdma_pbe *)pbl_tbl->va;
1958 }
1959 }
1960 return;
1961}
1962
1963static int get_encoded_page_size(int pg_sz)
1964{
1965 /* Max size is 256M 4096 << 16 */
1966 int i = 0;
1967 for (; i < 17; i++)
1968 if (pg_sz == (4096 << i))
1969 break;
1970 return i;
1971}
1972
1973
1974static int ocrdma_build_fr(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1975 struct ib_send_wr *wr)
1976{
1977 u64 fbo;
1978 struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1);
1979 struct ocrdma_mr *mr;
1980 u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr);
1981
1982 wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES);
1983
1984 if ((wr->wr.fast_reg.page_list_len >
1985 qp->dev->attr.max_pages_per_frmr) ||
1986 (wr->wr.fast_reg.length > 0xffffffffULL))
1987 return -EINVAL;
1988
1989 hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT);
1990 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
1991
1992 if (wr->wr.fast_reg.page_list_len == 0)
1993 BUG();
1994 if (wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE)
1995 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR;
1996 if (wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE)
1997 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR;
1998 if (wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ)
1999 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD;
2000 hdr->lkey = wr->wr.fast_reg.rkey;
2001 hdr->total_len = wr->wr.fast_reg.length;
2002
2003 fbo = wr->wr.fast_reg.iova_start -
2004 (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK);
2005
2006 fast_reg->va_hi = upper_32_bits(wr->wr.fast_reg.iova_start);
2007 fast_reg->va_lo = (u32) (wr->wr.fast_reg.iova_start & 0xffffffff);
2008 fast_reg->fbo_hi = upper_32_bits(fbo);
2009 fast_reg->fbo_lo = (u32) fbo & 0xffffffff;
2010 fast_reg->num_sges = wr->wr.fast_reg.page_list_len;
2011 fast_reg->size_sge =
2012 get_encoded_page_size(1 << wr->wr.fast_reg.page_shift);
2013 mr = (struct ocrdma_mr *) (unsigned long) qp->dev->stag_arr[(hdr->lkey >> 8) &
2014 (OCRDMA_MAX_STAG - 1)];
2015 build_frmr_pbes(wr, mr->hwmr.pbl_table, &mr->hwmr);
2016 return 0;
2017}
2018
1722static void ocrdma_ring_sq_db(struct ocrdma_qp *qp) 2019static void ocrdma_ring_sq_db(struct ocrdma_qp *qp)
1723{ 2020{
1724 u32 val = qp->sq.dbid | (1 << 16); 2021 u32 val = qp->sq.dbid | (1 << 16);
@@ -1750,7 +2047,7 @@ int ocrdma_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
1750 } 2047 }
1751 hdr = ocrdma_hwq_head(&qp->sq); 2048 hdr = ocrdma_hwq_head(&qp->sq);
1752 hdr->cw = 0; 2049 hdr->cw = 0;
1753 if (wr->send_flags & IB_SEND_SIGNALED) 2050 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
1754 hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT); 2051 hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
1755 if (wr->send_flags & IB_SEND_FENCE) 2052 if (wr->send_flags & IB_SEND_FENCE)
1756 hdr->cw |= 2053 hdr->cw |=
@@ -1788,10 +2085,14 @@ int ocrdma_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
1788 case IB_WR_LOCAL_INV: 2085 case IB_WR_LOCAL_INV:
1789 hdr->cw |= 2086 hdr->cw |=
1790 (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT); 2087 (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT);
1791 hdr->cw |= (sizeof(struct ocrdma_hdr_wqe) / 2088 hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) +
2089 sizeof(struct ocrdma_sge)) /
1792 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT; 2090 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT;
1793 hdr->lkey = wr->ex.invalidate_rkey; 2091 hdr->lkey = wr->ex.invalidate_rkey;
1794 break; 2092 break;
2093 case IB_WR_FAST_REG_MR:
2094 status = ocrdma_build_fr(qp, hdr, wr);
2095 break;
1795 default: 2096 default:
1796 status = -EINVAL; 2097 status = -EINVAL;
1797 break; 2098 break;
@@ -1800,7 +2101,7 @@ int ocrdma_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
1800 *bad_wr = wr; 2101 *bad_wr = wr;
1801 break; 2102 break;
1802 } 2103 }
1803 if (wr->send_flags & IB_SEND_SIGNALED) 2104 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
1804 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1; 2105 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1;
1805 else 2106 else
1806 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0; 2107 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0;
@@ -1824,7 +2125,10 @@ static void ocrdma_ring_rq_db(struct ocrdma_qp *qp)
1824{ 2125{
1825 u32 val = qp->rq.dbid | (1 << ocrdma_get_num_posted_shift(qp)); 2126 u32 val = qp->rq.dbid | (1 << ocrdma_get_num_posted_shift(qp));
1826 2127
1827 iowrite32(val, qp->rq_db); 2128 if (qp->state != OCRDMA_QPS_INIT)
2129 iowrite32(val, qp->rq_db);
2130 else
2131 qp->db_cache++;
1828} 2132}
1829 2133
1830static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, struct ib_recv_wr *wr, 2134static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, struct ib_recv_wr *wr,
@@ -1958,7 +2262,7 @@ int ocrdma_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
1958 2262
1959static enum ib_wc_status ocrdma_to_ibwc_err(u16 status) 2263static enum ib_wc_status ocrdma_to_ibwc_err(u16 status)
1960{ 2264{
1961 enum ib_wc_status ibwc_status = IB_WC_GENERAL_ERR; 2265 enum ib_wc_status ibwc_status;
1962 2266
1963 switch (status) { 2267 switch (status) {
1964 case OCRDMA_CQE_GENERAL_ERR: 2268 case OCRDMA_CQE_GENERAL_ERR:
@@ -2055,6 +2359,9 @@ static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc,
2055 case OCRDMA_SEND: 2359 case OCRDMA_SEND:
2056 ibwc->opcode = IB_WC_SEND; 2360 ibwc->opcode = IB_WC_SEND;
2057 break; 2361 break;
2362 case OCRDMA_FR_MR:
2363 ibwc->opcode = IB_WC_FAST_REG_MR;
2364 break;
2058 case OCRDMA_LKEY_INV: 2365 case OCRDMA_LKEY_INV:
2059 ibwc->opcode = IB_WC_LOCAL_INV; 2366 ibwc->opcode = IB_WC_LOCAL_INV;
2060 break; 2367 break;
@@ -2108,7 +2415,7 @@ static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2108 ibwc->status = ocrdma_to_ibwc_err(status); 2415 ibwc->status = ocrdma_to_ibwc_err(status);
2109 2416
2110 ocrdma_flush_qp(qp); 2417 ocrdma_flush_qp(qp);
2111 ocrdma_qp_state_machine(qp, IB_QPS_ERR, NULL); 2418 ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL);
2112 2419
2113 /* if wqe/rqe pending for which cqe needs to be returned, 2420 /* if wqe/rqe pending for which cqe needs to be returned,
2114 * trigger inflating it. 2421 * trigger inflating it.
@@ -2193,7 +2500,8 @@ static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp,
2193 ocrdma_update_wc(qp, ibwc, tail); 2500 ocrdma_update_wc(qp, ibwc, tail);
2194 *polled = true; 2501 *polled = true;
2195 } 2502 }
2196 wqe_idx = le32_to_cpu(cqe->wq.wqeidx) & OCRDMA_CQE_WQEIDX_MASK; 2503 wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) &
2504 OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx;
2197 if (tail != wqe_idx) 2505 if (tail != wqe_idx)
2198 expand = true; /* Coalesced CQE can't be consumed yet */ 2506 expand = true; /* Coalesced CQE can't be consumed yet */
2199 2507
@@ -2242,7 +2550,8 @@ static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc,
2242 u32 wqe_idx; 2550 u32 wqe_idx;
2243 2551
2244 srq = get_ocrdma_srq(qp->ibqp.srq); 2552 srq = get_ocrdma_srq(qp->ibqp.srq);
2245 wqe_idx = le32_to_cpu(cqe->rq.buftag_qpn) >> OCRDMA_CQE_BUFTAG_SHIFT; 2553 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>
2554 OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx;
2246 ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx]; 2555 ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx];
2247 spin_lock_irqsave(&srq->q_lock, flags); 2556 spin_lock_irqsave(&srq->q_lock, flags);
2248 ocrdma_srq_toggle_bit(srq, wqe_idx); 2557 ocrdma_srq_toggle_bit(srq, wqe_idx);
@@ -2299,9 +2608,9 @@ static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp,
2299 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt); 2608 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt);
2300 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE; 2609 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE;
2301 } 2610 }
2302 if (qp->ibqp.srq) 2611 if (qp->ibqp.srq) {
2303 ocrdma_update_free_srq_cqe(ibwc, cqe, qp); 2612 ocrdma_update_free_srq_cqe(ibwc, cqe, qp);
2304 else { 2613 } else {
2305 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2614 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2306 ocrdma_hwq_inc_tail(&qp->rq); 2615 ocrdma_hwq_inc_tail(&qp->rq);
2307 } 2616 }
@@ -2314,13 +2623,14 @@ static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2314 bool expand = false; 2623 bool expand = false;
2315 2624
2316 ibwc->wc_flags = 0; 2625 ibwc->wc_flags = 0;
2317 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) 2626 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2318 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2627 status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2319 OCRDMA_CQE_UD_STATUS_MASK) >> 2628 OCRDMA_CQE_UD_STATUS_MASK) >>
2320 OCRDMA_CQE_UD_STATUS_SHIFT; 2629 OCRDMA_CQE_UD_STATUS_SHIFT;
2321 else 2630 } else {
2322 status = (le32_to_cpu(cqe->flags_status_srcqpn) & 2631 status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2323 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT; 2632 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2633 }
2324 2634
2325 if (status == OCRDMA_CQE_SUCCESS) { 2635 if (status == OCRDMA_CQE_SUCCESS) {
2326 *polled = true; 2636 *polled = true;
@@ -2338,9 +2648,10 @@ static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe,
2338 if (cq->phase_change) { 2648 if (cq->phase_change) {
2339 if (cur_getp == 0) 2649 if (cur_getp == 0)
2340 cq->phase = (~cq->phase & OCRDMA_CQE_VALID); 2650 cq->phase = (~cq->phase & OCRDMA_CQE_VALID);
2341 } else 2651 } else {
2342 /* clear valid bit */ 2652 /* clear valid bit */
2343 cqe->flags_status_srcqpn = 0; 2653 cqe->flags_status_srcqpn = 0;
2654 }
2344} 2655}
2345 2656
2346static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries, 2657static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries,
@@ -2351,7 +2662,7 @@ static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries,
2351 bool expand = false; 2662 bool expand = false;
2352 int polled_hw_cqes = 0; 2663 int polled_hw_cqes = 0;
2353 struct ocrdma_qp *qp = NULL; 2664 struct ocrdma_qp *qp = NULL;
2354 struct ocrdma_dev *dev = cq->dev; 2665 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);
2355 struct ocrdma_cqe *cqe; 2666 struct ocrdma_cqe *cqe;
2356 u16 cur_getp; bool polled = false; bool stop = false; 2667 u16 cur_getp; bool polled = false; bool stop = false;
2357 2668
@@ -2417,8 +2728,9 @@ static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries,
2417 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) { 2728 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) {
2418 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail]; 2729 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2419 ocrdma_hwq_inc_tail(&qp->rq); 2730 ocrdma_hwq_inc_tail(&qp->rq);
2420 } else 2731 } else {
2421 return err_cqes; 2732 return err_cqes;
2733 }
2422 ibwc->byte_len = 0; 2734 ibwc->byte_len = 0;
2423 ibwc->status = IB_WC_WR_FLUSH_ERR; 2735 ibwc->status = IB_WC_WR_FLUSH_ERR;
2424 ibwc = ibwc + 1; 2736 ibwc = ibwc + 1;
@@ -2431,14 +2743,11 @@ static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries,
2431int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 2743int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
2432{ 2744{
2433 int cqes_to_poll = num_entries; 2745 int cqes_to_poll = num_entries;
2434 struct ocrdma_cq *cq = NULL; 2746 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2435 unsigned long flags; 2747 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
2436 struct ocrdma_dev *dev;
2437 int num_os_cqe = 0, err_cqes = 0; 2748 int num_os_cqe = 0, err_cqes = 0;
2438 struct ocrdma_qp *qp; 2749 struct ocrdma_qp *qp;
2439 2750 unsigned long flags;
2440 cq = get_ocrdma_cq(ibcq);
2441 dev = cq->dev;
2442 2751
2443 /* poll cqes from adapter CQ */ 2752 /* poll cqes from adapter CQ */
2444 spin_lock_irqsave(&cq->cq_lock, flags); 2753 spin_lock_irqsave(&cq->cq_lock, flags);
@@ -2469,16 +2778,14 @@ int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
2469 2778
2470int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags) 2779int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags)
2471{ 2780{
2472 struct ocrdma_cq *cq; 2781 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2473 unsigned long flags; 2782 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
2474 struct ocrdma_dev *dev;
2475 u16 cq_id; 2783 u16 cq_id;
2476 u16 cur_getp; 2784 u16 cur_getp;
2477 struct ocrdma_cqe *cqe; 2785 struct ocrdma_cqe *cqe;
2786 unsigned long flags;
2478 2787
2479 cq = get_ocrdma_cq(ibcq);
2480 cq_id = cq->id; 2788 cq_id = cq->id;
2481 dev = cq->dev;
2482 2789
2483 spin_lock_irqsave(&cq->cq_lock, flags); 2790 spin_lock_irqsave(&cq->cq_lock, flags);
2484 if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED) 2791 if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED)
@@ -2500,3 +2807,226 @@ int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags)
2500 spin_unlock_irqrestore(&cq->cq_lock, flags); 2807 spin_unlock_irqrestore(&cq->cq_lock, flags);
2501 return 0; 2808 return 0;
2502} 2809}
2810
2811struct ib_mr *ocrdma_alloc_frmr(struct ib_pd *ibpd, int max_page_list_len)
2812{
2813 int status;
2814 struct ocrdma_mr *mr;
2815 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
2816 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
2817
2818 if (max_page_list_len > dev->attr.max_pages_per_frmr)
2819 return ERR_PTR(-EINVAL);
2820
2821 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2822 if (!mr)
2823 return ERR_PTR(-ENOMEM);
2824
2825 status = ocrdma_get_pbl_info(dev, mr, max_page_list_len);
2826 if (status)
2827 goto pbl_err;
2828 mr->hwmr.fr_mr = 1;
2829 mr->hwmr.remote_rd = 0;
2830 mr->hwmr.remote_wr = 0;
2831 mr->hwmr.local_rd = 0;
2832 mr->hwmr.local_wr = 0;
2833 mr->hwmr.mw_bind = 0;
2834 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
2835 if (status)
2836 goto pbl_err;
2837 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0);
2838 if (status)
2839 goto mbx_err;
2840 mr->ibmr.rkey = mr->hwmr.lkey;
2841 mr->ibmr.lkey = mr->hwmr.lkey;
2842 dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] = (unsigned long) mr;
2843 return &mr->ibmr;
2844mbx_err:
2845 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
2846pbl_err:
2847 kfree(mr);
2848 return ERR_PTR(-ENOMEM);
2849}
2850
2851struct ib_fast_reg_page_list *ocrdma_alloc_frmr_page_list(struct ib_device
2852 *ibdev,
2853 int page_list_len)
2854{
2855 struct ib_fast_reg_page_list *frmr_list;
2856 int size;
2857
2858 size = sizeof(*frmr_list) + (page_list_len * sizeof(u64));
2859 frmr_list = kzalloc(size, GFP_KERNEL);
2860 if (!frmr_list)
2861 return ERR_PTR(-ENOMEM);
2862 frmr_list->page_list = (u64 *)(frmr_list + 1);
2863 return frmr_list;
2864}
2865
2866void ocrdma_free_frmr_page_list(struct ib_fast_reg_page_list *page_list)
2867{
2868 kfree(page_list);
2869}
2870
2871#define MAX_KERNEL_PBE_SIZE 65536
2872static inline int count_kernel_pbes(struct ib_phys_buf *buf_list,
2873 int buf_cnt, u32 *pbe_size)
2874{
2875 u64 total_size = 0;
2876 u64 buf_size = 0;
2877 int i;
2878 *pbe_size = roundup(buf_list[0].size, PAGE_SIZE);
2879 *pbe_size = roundup_pow_of_two(*pbe_size);
2880
2881 /* find the smallest PBE size that we can have */
2882 for (i = 0; i < buf_cnt; i++) {
2883 /* first addr may not be page aligned, so ignore checking */
2884 if ((i != 0) && ((buf_list[i].addr & ~PAGE_MASK) ||
2885 (buf_list[i].size & ~PAGE_MASK))) {
2886 return 0;
2887 }
2888
2889 /* if configured PBE size is greater then the chosen one,
2890 * reduce the PBE size.
2891 */
2892 buf_size = roundup(buf_list[i].size, PAGE_SIZE);
2893 /* pbe_size has to be even multiple of 4K 1,2,4,8...*/
2894 buf_size = roundup_pow_of_two(buf_size);
2895 if (*pbe_size > buf_size)
2896 *pbe_size = buf_size;
2897
2898 total_size += buf_size;
2899 }
2900 *pbe_size = *pbe_size > MAX_KERNEL_PBE_SIZE ?
2901 (MAX_KERNEL_PBE_SIZE) : (*pbe_size);
2902
2903 /* num_pbes = total_size / (*pbe_size); this is implemented below. */
2904
2905 return total_size >> ilog2(*pbe_size);
2906}
2907
2908static void build_kernel_pbes(struct ib_phys_buf *buf_list, int ib_buf_cnt,
2909 u32 pbe_size, struct ocrdma_pbl *pbl_tbl,
2910 struct ocrdma_hw_mr *hwmr)
2911{
2912 int i;
2913 int idx;
2914 int pbes_per_buf = 0;
2915 u64 buf_addr = 0;
2916 int num_pbes;
2917 struct ocrdma_pbe *pbe;
2918 int total_num_pbes = 0;
2919
2920 if (!hwmr->num_pbes)
2921 return;
2922
2923 pbe = (struct ocrdma_pbe *)pbl_tbl->va;
2924 num_pbes = 0;
2925
2926 /* go through the OS phy regions & fill hw pbe entries into pbls. */
2927 for (i = 0; i < ib_buf_cnt; i++) {
2928 buf_addr = buf_list[i].addr;
2929 pbes_per_buf =
2930 roundup_pow_of_two(roundup(buf_list[i].size, PAGE_SIZE)) /
2931 pbe_size;
2932 hwmr->len += buf_list[i].size;
2933 /* number of pbes can be more for one OS buf, when
2934 * buffers are of different sizes.
2935 * split the ib_buf to one or more pbes.
2936 */
2937 for (idx = 0; idx < pbes_per_buf; idx++) {
2938 /* we program always page aligned addresses,
2939 * first unaligned address is taken care by fbo.
2940 */
2941 if (i == 0) {
2942 /* for non zero fbo, assign the
2943 * start of the page.
2944 */
2945 pbe->pa_lo =
2946 cpu_to_le32((u32) (buf_addr & PAGE_MASK));
2947 pbe->pa_hi =
2948 cpu_to_le32((u32) upper_32_bits(buf_addr));
2949 } else {
2950 pbe->pa_lo =
2951 cpu_to_le32((u32) (buf_addr & 0xffffffff));
2952 pbe->pa_hi =
2953 cpu_to_le32((u32) upper_32_bits(buf_addr));
2954 }
2955 buf_addr += pbe_size;
2956 num_pbes += 1;
2957 total_num_pbes += 1;
2958 pbe++;
2959
2960 if (total_num_pbes == hwmr->num_pbes)
2961 goto mr_tbl_done;
2962 /* if the pbl is full storing the pbes,
2963 * move to next pbl.
2964 */
2965 if (num_pbes == (hwmr->pbl_size/sizeof(u64))) {
2966 pbl_tbl++;
2967 pbe = (struct ocrdma_pbe *)pbl_tbl->va;
2968 num_pbes = 0;
2969 }
2970 }
2971 }
2972mr_tbl_done:
2973 return;
2974}
2975
2976struct ib_mr *ocrdma_reg_kernel_mr(struct ib_pd *ibpd,
2977 struct ib_phys_buf *buf_list,
2978 int buf_cnt, int acc, u64 *iova_start)
2979{
2980 int status = -ENOMEM;
2981 struct ocrdma_mr *mr;
2982 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
2983 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
2984 u32 num_pbes;
2985 u32 pbe_size = 0;
2986
2987 if ((acc & IB_ACCESS_REMOTE_WRITE) && !(acc & IB_ACCESS_LOCAL_WRITE))
2988 return ERR_PTR(-EINVAL);
2989
2990 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2991 if (!mr)
2992 return ERR_PTR(status);
2993
2994 num_pbes = count_kernel_pbes(buf_list, buf_cnt, &pbe_size);
2995 if (num_pbes == 0) {
2996 status = -EINVAL;
2997 goto pbl_err;
2998 }
2999 status = ocrdma_get_pbl_info(dev, mr, num_pbes);
3000 if (status)
3001 goto pbl_err;
3002
3003 mr->hwmr.pbe_size = pbe_size;
3004 mr->hwmr.fbo = *iova_start - (buf_list[0].addr & PAGE_MASK);
3005 mr->hwmr.va = *iova_start;
3006 mr->hwmr.local_rd = 1;
3007 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3008 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3009 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3010 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3011 mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;
3012
3013 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
3014 if (status)
3015 goto pbl_err;
3016 build_kernel_pbes(buf_list, buf_cnt, pbe_size, mr->hwmr.pbl_table,
3017 &mr->hwmr);
3018 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);
3019 if (status)
3020 goto mbx_err;
3021
3022 mr->ibmr.lkey = mr->hwmr.lkey;
3023 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
3024 mr->ibmr.rkey = mr->hwmr.lkey;
3025 return &mr->ibmr;
3026
3027mbx_err:
3028 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
3029pbl_err:
3030 kfree(mr);
3031 return ERR_PTR(status);
3032}
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
index 633f03d80274..b8f7853fd36c 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
@@ -72,6 +72,7 @@ int ocrdma_query_qp(struct ib_qp *,
72 struct ib_qp_attr *qp_attr, 72 struct ib_qp_attr *qp_attr,
73 int qp_attr_mask, struct ib_qp_init_attr *); 73 int qp_attr_mask, struct ib_qp_init_attr *);
74int ocrdma_destroy_qp(struct ib_qp *); 74int ocrdma_destroy_qp(struct ib_qp *);
75void ocrdma_del_flush_qp(struct ocrdma_qp *qp);
75 76
76struct ib_srq *ocrdma_create_srq(struct ib_pd *, struct ib_srq_init_attr *, 77struct ib_srq *ocrdma_create_srq(struct ib_pd *, struct ib_srq_init_attr *,
77 struct ib_udata *); 78 struct ib_udata *);
@@ -89,5 +90,10 @@ struct ib_mr *ocrdma_reg_kernel_mr(struct ib_pd *,
89 int num_phys_buf, int acc, u64 *iova_start); 90 int num_phys_buf, int acc, u64 *iova_start);
90struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *, u64 start, u64 length, 91struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *, u64 start, u64 length,
91 u64 virt, int acc, struct ib_udata *); 92 u64 virt, int acc, struct ib_udata *);
93struct ib_mr *ocrdma_alloc_frmr(struct ib_pd *pd, int max_page_list_len);
94struct ib_fast_reg_page_list *ocrdma_alloc_frmr_page_list(struct ib_device
95 *ibdev,
96 int page_list_len);
97void ocrdma_free_frmr_page_list(struct ib_fast_reg_page_list *page_list);
92 98
93#endif /* __OCRDMA_VERBS_H__ */ 99#endif /* __OCRDMA_VERBS_H__ */
diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h
index 4a9af795b88f..1946101419a3 100644
--- a/drivers/infiniband/hw/qib/qib.h
+++ b/drivers/infiniband/hw/qib/qib.h
@@ -89,7 +89,6 @@ struct qlogic_ib_stats {
89 89
90extern struct qlogic_ib_stats qib_stats; 90extern struct qlogic_ib_stats qib_stats;
91extern const struct pci_error_handlers qib_pci_err_handler; 91extern const struct pci_error_handlers qib_pci_err_handler;
92extern struct pci_driver qib_driver;
93 92
94#define QIB_CHIP_SWVERSION QIB_CHIP_VERS_MAJ 93#define QIB_CHIP_SWVERSION QIB_CHIP_VERS_MAJ
95/* 94/*
@@ -576,11 +575,13 @@ struct qib_pportdata {
576 /* read/write using lock */ 575 /* read/write using lock */
577 spinlock_t sdma_lock ____cacheline_aligned_in_smp; 576 spinlock_t sdma_lock ____cacheline_aligned_in_smp;
578 struct list_head sdma_activelist; 577 struct list_head sdma_activelist;
578 struct list_head sdma_userpending;
579 u64 sdma_descq_added; 579 u64 sdma_descq_added;
580 u64 sdma_descq_removed; 580 u64 sdma_descq_removed;
581 u16 sdma_descq_tail; 581 u16 sdma_descq_tail;
582 u16 sdma_descq_head; 582 u16 sdma_descq_head;
583 u8 sdma_generation; 583 u8 sdma_generation;
584 u8 sdma_intrequest;
584 585
585 struct tasklet_struct sdma_sw_clean_up_task 586 struct tasklet_struct sdma_sw_clean_up_task
586 ____cacheline_aligned_in_smp; 587 ____cacheline_aligned_in_smp;
@@ -1326,6 +1327,8 @@ int qib_setup_sdma(struct qib_pportdata *);
1326void qib_teardown_sdma(struct qib_pportdata *); 1327void qib_teardown_sdma(struct qib_pportdata *);
1327void __qib_sdma_intr(struct qib_pportdata *); 1328void __qib_sdma_intr(struct qib_pportdata *);
1328void qib_sdma_intr(struct qib_pportdata *); 1329void qib_sdma_intr(struct qib_pportdata *);
1330void qib_user_sdma_send_desc(struct qib_pportdata *dd,
1331 struct list_head *pktlist);
1329int qib_sdma_verbs_send(struct qib_pportdata *, struct qib_sge_state *, 1332int qib_sdma_verbs_send(struct qib_pportdata *, struct qib_sge_state *,
1330 u32, struct qib_verbs_txreq *); 1333 u32, struct qib_verbs_txreq *);
1331/* ppd->sdma_lock should be locked before calling this. */ 1334/* ppd->sdma_lock should be locked before calling this. */
diff --git a/drivers/infiniband/hw/qib/qib_common.h b/drivers/infiniband/hw/qib/qib_common.h
index 4f255b723ffd..5670ace27c63 100644
--- a/drivers/infiniband/hw/qib/qib_common.h
+++ b/drivers/infiniband/hw/qib/qib_common.h
@@ -279,7 +279,7 @@ struct qib_base_info {
279 * may not be implemented; the user code must deal with this if it 279 * may not be implemented; the user code must deal with this if it
280 * cares, or it must abort after initialization reports the difference. 280 * cares, or it must abort after initialization reports the difference.
281 */ 281 */
282#define QIB_USER_SWMINOR 12 282#define QIB_USER_SWMINOR 13
283 283
284#define QIB_USER_SWVERSION ((QIB_USER_SWMAJOR << 16) | QIB_USER_SWMINOR) 284#define QIB_USER_SWVERSION ((QIB_USER_SWMAJOR << 16) | QIB_USER_SWMINOR)
285 285
@@ -701,7 +701,37 @@ struct qib_message_header {
701 __be32 bth[3]; 701 __be32 bth[3];
702 /* fields below this point are in host byte order */ 702 /* fields below this point are in host byte order */
703 struct qib_header iph; 703 struct qib_header iph;
704 /* fields below are simplified, but should match PSM */
705 /* some are accessed by driver when packet spliting is needed */
704 __u8 sub_opcode; 706 __u8 sub_opcode;
707 __u8 flags;
708 __u16 commidx;
709 __u32 ack_seq_num;
710 __u8 flowid;
711 __u8 hdr_dlen;
712 __u16 mqhdr;
713 __u32 uwords[4];
714};
715
716/* sequence number bits for message */
717union qib_seqnum {
718 struct {
719 __u32 seq:11;
720 __u32 gen:8;
721 __u32 flow:5;
722 };
723 struct {
724 __u32 pkt:16;
725 __u32 msg:8;
726 };
727 __u32 val;
728};
729
730/* qib receiving-dma tid-session-member */
731struct qib_tid_session_member {
732 __u16 tid;
733 __u16 offset;
734 __u16 length;
705}; 735};
706 736
707/* IB - LRH header consts */ 737/* IB - LRH header consts */
diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c
index b51a51486cb8..275f247f9fca 100644
--- a/drivers/infiniband/hw/qib/qib_file_ops.c
+++ b/drivers/infiniband/hw/qib/qib_file_ops.c
@@ -1220,7 +1220,7 @@ static int qib_compatible_subctxts(int user_swmajor, int user_swminor)
1220 return user_swminor == 3; 1220 return user_swminor == 3;
1221 default: 1221 default:
1222 /* >= 4 are compatible (or are expected to be) */ 1222 /* >= 4 are compatible (or are expected to be) */
1223 return user_swminor >= 4; 1223 return user_swminor <= QIB_USER_SWMINOR;
1224 } 1224 }
1225 } 1225 }
1226 /* make no promises yet for future major versions */ 1226 /* make no promises yet for future major versions */
diff --git a/drivers/infiniband/hw/qib/qib_init.c b/drivers/infiniband/hw/qib/qib_init.c
index 36e048e0e1d9..24e802f4ea2f 100644
--- a/drivers/infiniband/hw/qib/qib_init.c
+++ b/drivers/infiniband/hw/qib/qib_init.c
@@ -1193,7 +1193,7 @@ static DEFINE_PCI_DEVICE_TABLE(qib_pci_tbl) = {
1193 1193
1194MODULE_DEVICE_TABLE(pci, qib_pci_tbl); 1194MODULE_DEVICE_TABLE(pci, qib_pci_tbl);
1195 1195
1196struct pci_driver qib_driver = { 1196static struct pci_driver qib_driver = {
1197 .name = QIB_DRV_NAME, 1197 .name = QIB_DRV_NAME,
1198 .probe = qib_init_one, 1198 .probe = qib_init_one,
1199 .remove = qib_remove_one, 1199 .remove = qib_remove_one,
diff --git a/drivers/infiniband/hw/qib/qib_mad.h b/drivers/infiniband/hw/qib/qib_mad.h
index 57bd3fa016bc..28874f8606f8 100644
--- a/drivers/infiniband/hw/qib/qib_mad.h
+++ b/drivers/infiniband/hw/qib/qib_mad.h
@@ -415,7 +415,6 @@ struct cc_table_shadow {
415 struct ib_cc_table_entry_shadow entries[CC_TABLE_SHADOW_MAX]; 415 struct ib_cc_table_entry_shadow entries[CC_TABLE_SHADOW_MAX];
416} __packed; 416} __packed;
417 417
418#endif /* _QIB_MAD_H */
419/* 418/*
420 * The PortSamplesControl.CounterMasks field is an array of 3 bit fields 419 * The PortSamplesControl.CounterMasks field is an array of 3 bit fields
421 * which specify the N'th counter's capabilities. See ch. 16.1.3.2. 420 * which specify the N'th counter's capabilities. See ch. 16.1.3.2.
@@ -428,3 +427,5 @@ struct cc_table_shadow {
428 COUNTER_MASK(1, 2) | \ 427 COUNTER_MASK(1, 2) | \
429 COUNTER_MASK(1, 3) | \ 428 COUNTER_MASK(1, 3) | \
430 COUNTER_MASK(1, 4)) 429 COUNTER_MASK(1, 4))
430
431#endif /* _QIB_MAD_H */
diff --git a/drivers/infiniband/hw/qib/qib_pcie.c b/drivers/infiniband/hw/qib/qib_pcie.c
index c574ec7c85e6..3f14009fb662 100644
--- a/drivers/infiniband/hw/qib/qib_pcie.c
+++ b/drivers/infiniband/hw/qib/qib_pcie.c
@@ -283,12 +283,12 @@ int qib_pcie_params(struct qib_devdata *dd, u32 minw, u32 *nent,
283 goto bail; 283 goto bail;
284 } 284 }
285 285
286 pos = pci_find_capability(dd->pcidev, PCI_CAP_ID_MSIX); 286 pos = dd->pcidev->msix_cap;
287 if (nent && *nent && pos) { 287 if (nent && *nent && pos) {
288 qib_msix_setup(dd, pos, nent, entry); 288 qib_msix_setup(dd, pos, nent, entry);
289 ret = 0; /* did it, either MSIx or INTx */ 289 ret = 0; /* did it, either MSIx or INTx */
290 } else { 290 } else {
291 pos = pci_find_capability(dd->pcidev, PCI_CAP_ID_MSI); 291 pos = dd->pcidev->msi_cap;
292 if (pos) 292 if (pos)
293 ret = qib_msi_setup(dd, pos); 293 ret = qib_msi_setup(dd, pos);
294 else 294 else
@@ -357,7 +357,7 @@ int qib_reinit_intr(struct qib_devdata *dd)
357 if (!dd->msi_lo) 357 if (!dd->msi_lo)
358 goto bail; 358 goto bail;
359 359
360 pos = pci_find_capability(dd->pcidev, PCI_CAP_ID_MSI); 360 pos = dd->pcidev->msi_cap;
361 if (!pos) { 361 if (!pos) {
362 qib_dev_err(dd, 362 qib_dev_err(dd,
363 "Can't find MSI capability, can't restore MSI settings\n"); 363 "Can't find MSI capability, can't restore MSI settings\n");
@@ -426,7 +426,7 @@ void qib_enable_intx(struct pci_dev *pdev)
426 if (new != cw) 426 if (new != cw)
427 pci_write_config_word(pdev, PCI_COMMAND, new); 427 pci_write_config_word(pdev, PCI_COMMAND, new);
428 428
429 pos = pci_find_capability(pdev, PCI_CAP_ID_MSI); 429 pos = pdev->msi_cap;
430 if (pos) { 430 if (pos) {
431 /* then turn off MSI */ 431 /* then turn off MSI */
432 pci_read_config_word(pdev, pos + PCI_MSI_FLAGS, &cw); 432 pci_read_config_word(pdev, pos + PCI_MSI_FLAGS, &cw);
@@ -434,7 +434,7 @@ void qib_enable_intx(struct pci_dev *pdev)
434 if (new != cw) 434 if (new != cw)
435 pci_write_config_word(pdev, pos + PCI_MSI_FLAGS, new); 435 pci_write_config_word(pdev, pos + PCI_MSI_FLAGS, new);
436 } 436 }
437 pos = pci_find_capability(pdev, PCI_CAP_ID_MSIX); 437 pos = pdev->msix_cap;
438 if (pos) { 438 if (pos) {
439 /* then turn off MSIx */ 439 /* then turn off MSIx */
440 pci_read_config_word(pdev, pos + PCI_MSIX_FLAGS, &cw); 440 pci_read_config_word(pdev, pos + PCI_MSIX_FLAGS, &cw);
diff --git a/drivers/infiniband/hw/qib/qib_sdma.c b/drivers/infiniband/hw/qib/qib_sdma.c
index 9b5322d8cd5a..c6d6a54d2e19 100644
--- a/drivers/infiniband/hw/qib/qib_sdma.c
+++ b/drivers/infiniband/hw/qib/qib_sdma.c
@@ -423,8 +423,11 @@ void qib_sdma_intr(struct qib_pportdata *ppd)
423 423
424void __qib_sdma_intr(struct qib_pportdata *ppd) 424void __qib_sdma_intr(struct qib_pportdata *ppd)
425{ 425{
426 if (__qib_sdma_running(ppd)) 426 if (__qib_sdma_running(ppd)) {
427 qib_sdma_make_progress(ppd); 427 qib_sdma_make_progress(ppd);
428 if (!list_empty(&ppd->sdma_userpending))
429 qib_user_sdma_send_desc(ppd, &ppd->sdma_userpending);
430 }
428} 431}
429 432
430int qib_setup_sdma(struct qib_pportdata *ppd) 433int qib_setup_sdma(struct qib_pportdata *ppd)
@@ -452,6 +455,9 @@ int qib_setup_sdma(struct qib_pportdata *ppd)
452 ppd->sdma_descq_removed = 0; 455 ppd->sdma_descq_removed = 0;
453 ppd->sdma_descq_added = 0; 456 ppd->sdma_descq_added = 0;
454 457
458 ppd->sdma_intrequest = 0;
459 INIT_LIST_HEAD(&ppd->sdma_userpending);
460
455 INIT_LIST_HEAD(&ppd->sdma_activelist); 461 INIT_LIST_HEAD(&ppd->sdma_activelist);
456 462
457 tasklet_init(&ppd->sdma_sw_clean_up_task, sdma_sw_clean_up_task, 463 tasklet_init(&ppd->sdma_sw_clean_up_task, sdma_sw_clean_up_task,
diff --git a/drivers/infiniband/hw/qib/qib_user_sdma.c b/drivers/infiniband/hw/qib/qib_user_sdma.c
index 82442085cbe6..d0a0ea0c14d6 100644
--- a/drivers/infiniband/hw/qib/qib_user_sdma.c
+++ b/drivers/infiniband/hw/qib/qib_user_sdma.c
@@ -53,20 +53,36 @@
53#define QIB_USER_SDMA_DRAIN_TIMEOUT 500 53#define QIB_USER_SDMA_DRAIN_TIMEOUT 500
54 54
55struct qib_user_sdma_pkt { 55struct qib_user_sdma_pkt {
56 u8 naddr; /* dimension of addr (1..3) ... */ 56 struct list_head list; /* list element */
57
58 u8 tiddma; /* if this is NEW tid-sdma */
59 u8 largepkt; /* this is large pkt from kmalloc */
60 u16 frag_size; /* frag size used by PSM */
61 u16 index; /* last header index or push index */
62 u16 naddr; /* dimension of addr (1..3) ... */
63 u16 addrlimit; /* addr array size */
64 u16 tidsmidx; /* current tidsm index */
65 u16 tidsmcount; /* tidsm array item count */
66 u16 payload_size; /* payload size so far for header */
67 u32 bytes_togo; /* bytes for processing */
57 u32 counter; /* sdma pkts queued counter for this entry */ 68 u32 counter; /* sdma pkts queued counter for this entry */
69 struct qib_tid_session_member *tidsm; /* tid session member array */
70 struct qib_user_sdma_queue *pq; /* which pq this pkt belongs to */
58 u64 added; /* global descq number of entries */ 71 u64 added; /* global descq number of entries */
59 72
60 struct { 73 struct {
61 u32 offset; /* offset for kvaddr, addr */ 74 u16 offset; /* offset for kvaddr, addr */
62 u32 length; /* length in page */ 75 u16 length; /* length in page */
63 u8 put_page; /* should we put_page? */ 76 u16 first_desc; /* first desc */
64 u8 dma_mapped; /* is page dma_mapped? */ 77 u16 last_desc; /* last desc */
78 u16 put_page; /* should we put_page? */
79 u16 dma_mapped; /* is page dma_mapped? */
80 u16 dma_length; /* for dma_unmap_page() */
81 u16 padding;
65 struct page *page; /* may be NULL (coherent mem) */ 82 struct page *page; /* may be NULL (coherent mem) */
66 void *kvaddr; /* FIXME: only for pio hack */ 83 void *kvaddr; /* FIXME: only for pio hack */
67 dma_addr_t addr; 84 dma_addr_t addr;
68 } addr[4]; /* max pages, any more and we coalesce */ 85 } addr[4]; /* max pages, any more and we coalesce */
69 struct list_head list; /* list element */
70}; 86};
71 87
72struct qib_user_sdma_queue { 88struct qib_user_sdma_queue {
@@ -77,6 +93,12 @@ struct qib_user_sdma_queue {
77 */ 93 */
78 struct list_head sent; 94 struct list_head sent;
79 95
96 /*
97 * Because above list will be accessed by both process and
98 * signal handler, we need a spinlock for it.
99 */
100 spinlock_t sent_lock ____cacheline_aligned_in_smp;
101
80 /* headers with expected length are allocated from here... */ 102 /* headers with expected length are allocated from here... */
81 char header_cache_name[64]; 103 char header_cache_name[64];
82 struct dma_pool *header_cache; 104 struct dma_pool *header_cache;
@@ -88,6 +110,12 @@ struct qib_user_sdma_queue {
88 /* as packets go on the queued queue, they are counted... */ 110 /* as packets go on the queued queue, they are counted... */
89 u32 counter; 111 u32 counter;
90 u32 sent_counter; 112 u32 sent_counter;
113 /* pending packets, not sending yet */
114 u32 num_pending;
115 /* sending packets, not complete yet */
116 u32 num_sending;
117 /* global descq number of entry of last sending packet */
118 u64 added;
91 119
92 /* dma page table */ 120 /* dma page table */
93 struct rb_root dma_pages_root; 121 struct rb_root dma_pages_root;
@@ -107,8 +135,12 @@ qib_user_sdma_queue_create(struct device *dev, int unit, int ctxt, int sctxt)
107 135
108 pq->counter = 0; 136 pq->counter = 0;
109 pq->sent_counter = 0; 137 pq->sent_counter = 0;
110 INIT_LIST_HEAD(&pq->sent); 138 pq->num_pending = 0;
139 pq->num_sending = 0;
140 pq->added = 0;
111 141
142 INIT_LIST_HEAD(&pq->sent);
143 spin_lock_init(&pq->sent_lock);
112 mutex_init(&pq->lock); 144 mutex_init(&pq->lock);
113 145
114 snprintf(pq->pkt_slab_name, sizeof(pq->pkt_slab_name), 146 snprintf(pq->pkt_slab_name, sizeof(pq->pkt_slab_name),
@@ -144,34 +176,310 @@ done:
144} 176}
145 177
146static void qib_user_sdma_init_frag(struct qib_user_sdma_pkt *pkt, 178static void qib_user_sdma_init_frag(struct qib_user_sdma_pkt *pkt,
147 int i, size_t offset, size_t len, 179 int i, u16 offset, u16 len,
148 int put_page, int dma_mapped, 180 u16 first_desc, u16 last_desc,
149 struct page *page, 181 u16 put_page, u16 dma_mapped,
150 void *kvaddr, dma_addr_t dma_addr) 182 struct page *page, void *kvaddr,
183 dma_addr_t dma_addr, u16 dma_length)
151{ 184{
152 pkt->addr[i].offset = offset; 185 pkt->addr[i].offset = offset;
153 pkt->addr[i].length = len; 186 pkt->addr[i].length = len;
187 pkt->addr[i].first_desc = first_desc;
188 pkt->addr[i].last_desc = last_desc;
154 pkt->addr[i].put_page = put_page; 189 pkt->addr[i].put_page = put_page;
155 pkt->addr[i].dma_mapped = dma_mapped; 190 pkt->addr[i].dma_mapped = dma_mapped;
156 pkt->addr[i].page = page; 191 pkt->addr[i].page = page;
157 pkt->addr[i].kvaddr = kvaddr; 192 pkt->addr[i].kvaddr = kvaddr;
158 pkt->addr[i].addr = dma_addr; 193 pkt->addr[i].addr = dma_addr;
194 pkt->addr[i].dma_length = dma_length;
159} 195}
160 196
161static void qib_user_sdma_init_header(struct qib_user_sdma_pkt *pkt, 197static void *qib_user_sdma_alloc_header(struct qib_user_sdma_queue *pq,
162 u32 counter, size_t offset, 198 size_t len, dma_addr_t *dma_addr)
163 size_t len, int dma_mapped,
164 struct page *page,
165 void *kvaddr, dma_addr_t dma_addr)
166{ 199{
167 pkt->naddr = 1; 200 void *hdr;
168 pkt->counter = counter; 201
169 qib_user_sdma_init_frag(pkt, 0, offset, len, 0, dma_mapped, page, 202 if (len == QIB_USER_SDMA_EXP_HEADER_LENGTH)
170 kvaddr, dma_addr); 203 hdr = dma_pool_alloc(pq->header_cache, GFP_KERNEL,
204 dma_addr);
205 else
206 hdr = NULL;
207
208 if (!hdr) {
209 hdr = kmalloc(len, GFP_KERNEL);
210 if (!hdr)
211 return NULL;
212
213 *dma_addr = 0;
214 }
215
216 return hdr;
217}
218
219static int qib_user_sdma_page_to_frags(const struct qib_devdata *dd,
220 struct qib_user_sdma_queue *pq,
221 struct qib_user_sdma_pkt *pkt,
222 struct page *page, u16 put,
223 u16 offset, u16 len, void *kvaddr)
224{
225 __le16 *pbc16;
226 void *pbcvaddr;
227 struct qib_message_header *hdr;
228 u16 newlen, pbclen, lastdesc, dma_mapped;
229 u32 vcto;
230 union qib_seqnum seqnum;
231 dma_addr_t pbcdaddr;
232 dma_addr_t dma_addr =
233 dma_map_page(&dd->pcidev->dev,
234 page, offset, len, DMA_TO_DEVICE);
235 int ret = 0;
236
237 if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) {
238 /*
239 * dma mapping error, pkt has not managed
240 * this page yet, return the page here so
241 * the caller can ignore this page.
242 */
243 if (put) {
244 put_page(page);
245 } else {
246 /* coalesce case */
247 kunmap(page);
248 __free_page(page);
249 }
250 ret = -ENOMEM;
251 goto done;
252 }
253 offset = 0;
254 dma_mapped = 1;
255
256
257next_fragment:
258
259 /*
260 * In tid-sdma, the transfer length is restricted by
261 * receiver side current tid page length.
262 */
263 if (pkt->tiddma && len > pkt->tidsm[pkt->tidsmidx].length)
264 newlen = pkt->tidsm[pkt->tidsmidx].length;
265 else
266 newlen = len;
267
268 /*
269 * Then the transfer length is restricted by MTU.
270 * the last descriptor flag is determined by:
271 * 1. the current packet is at frag size length.
272 * 2. the current tid page is done if tid-sdma.
273 * 3. there is no more byte togo if sdma.
274 */
275 lastdesc = 0;
276 if ((pkt->payload_size + newlen) >= pkt->frag_size) {
277 newlen = pkt->frag_size - pkt->payload_size;
278 lastdesc = 1;
279 } else if (pkt->tiddma) {
280 if (newlen == pkt->tidsm[pkt->tidsmidx].length)
281 lastdesc = 1;
282 } else {
283 if (newlen == pkt->bytes_togo)
284 lastdesc = 1;
285 }
286
287 /* fill the next fragment in this page */
288 qib_user_sdma_init_frag(pkt, pkt->naddr, /* index */
289 offset, newlen, /* offset, len */
290 0, lastdesc, /* first last desc */
291 put, dma_mapped, /* put page, dma mapped */
292 page, kvaddr, /* struct page, virt addr */
293 dma_addr, len); /* dma addr, dma length */
294 pkt->bytes_togo -= newlen;
295 pkt->payload_size += newlen;
296 pkt->naddr++;
297 if (pkt->naddr == pkt->addrlimit) {
298 ret = -EFAULT;
299 goto done;
300 }
301
302 /* If there is no more byte togo. (lastdesc==1) */
303 if (pkt->bytes_togo == 0) {
304 /* The packet is done, header is not dma mapped yet.
305 * it should be from kmalloc */
306 if (!pkt->addr[pkt->index].addr) {
307 pkt->addr[pkt->index].addr =
308 dma_map_single(&dd->pcidev->dev,
309 pkt->addr[pkt->index].kvaddr,
310 pkt->addr[pkt->index].dma_length,
311 DMA_TO_DEVICE);
312 if (dma_mapping_error(&dd->pcidev->dev,
313 pkt->addr[pkt->index].addr)) {
314 ret = -ENOMEM;
315 goto done;
316 }
317 pkt->addr[pkt->index].dma_mapped = 1;
318 }
319
320 goto done;
321 }
322
323 /* If tid-sdma, advance tid info. */
324 if (pkt->tiddma) {
325 pkt->tidsm[pkt->tidsmidx].length -= newlen;
326 if (pkt->tidsm[pkt->tidsmidx].length) {
327 pkt->tidsm[pkt->tidsmidx].offset += newlen;
328 } else {
329 pkt->tidsmidx++;
330 if (pkt->tidsmidx == pkt->tidsmcount) {
331 ret = -EFAULT;
332 goto done;
333 }
334 }
335 }
336
337 /*
338 * If this is NOT the last descriptor. (newlen==len)
339 * the current packet is not done yet, but the current
340 * send side page is done.
341 */
342 if (lastdesc == 0)
343 goto done;
344
345 /*
346 * If running this driver under PSM with message size
347 * fitting into one transfer unit, it is not possible
348 * to pass this line. otherwise, it is a buggggg.
349 */
350
351 /*
352 * Since the current packet is done, and there are more
353 * bytes togo, we need to create a new sdma header, copying
354 * from previous sdma header and modify both.
355 */
356 pbclen = pkt->addr[pkt->index].length;
357 pbcvaddr = qib_user_sdma_alloc_header(pq, pbclen, &pbcdaddr);
358 if (!pbcvaddr) {
359 ret = -ENOMEM;
360 goto done;
361 }
362 /* Copy the previous sdma header to new sdma header */
363 pbc16 = (__le16 *)pkt->addr[pkt->index].kvaddr;
364 memcpy(pbcvaddr, pbc16, pbclen);
365
366 /* Modify the previous sdma header */
367 hdr = (struct qib_message_header *)&pbc16[4];
368
369 /* New pbc length */
370 pbc16[0] = cpu_to_le16(le16_to_cpu(pbc16[0])-(pkt->bytes_togo>>2));
371
372 /* New packet length */
373 hdr->lrh[2] = cpu_to_be16(le16_to_cpu(pbc16[0]));
374
375 if (pkt->tiddma) {
376 /* turn on the header suppression */
377 hdr->iph.pkt_flags =
378 cpu_to_le16(le16_to_cpu(hdr->iph.pkt_flags)|0x2);
379 /* turn off ACK_REQ: 0x04 and EXPECTED_DONE: 0x20 */
380 hdr->flags &= ~(0x04|0x20);
381 } else {
382 /* turn off extra bytes: 20-21 bits */
383 hdr->bth[0] = cpu_to_be32(be32_to_cpu(hdr->bth[0])&0xFFCFFFFF);
384 /* turn off ACK_REQ: 0x04 */
385 hdr->flags &= ~(0x04);
386 }
387
388 /* New kdeth checksum */
389 vcto = le32_to_cpu(hdr->iph.ver_ctxt_tid_offset);
390 hdr->iph.chksum = cpu_to_le16(QIB_LRH_BTH +
391 be16_to_cpu(hdr->lrh[2]) -
392 ((vcto>>16)&0xFFFF) - (vcto&0xFFFF) -
393 le16_to_cpu(hdr->iph.pkt_flags));
394
395 /* The packet is done, header is not dma mapped yet.
396 * it should be from kmalloc */
397 if (!pkt->addr[pkt->index].addr) {
398 pkt->addr[pkt->index].addr =
399 dma_map_single(&dd->pcidev->dev,
400 pkt->addr[pkt->index].kvaddr,
401 pkt->addr[pkt->index].dma_length,
402 DMA_TO_DEVICE);
403 if (dma_mapping_error(&dd->pcidev->dev,
404 pkt->addr[pkt->index].addr)) {
405 ret = -ENOMEM;
406 goto done;
407 }
408 pkt->addr[pkt->index].dma_mapped = 1;
409 }
410
411 /* Modify the new sdma header */
412 pbc16 = (__le16 *)pbcvaddr;
413 hdr = (struct qib_message_header *)&pbc16[4];
414
415 /* New pbc length */
416 pbc16[0] = cpu_to_le16(le16_to_cpu(pbc16[0])-(pkt->payload_size>>2));
417
418 /* New packet length */
419 hdr->lrh[2] = cpu_to_be16(le16_to_cpu(pbc16[0]));
420
421 if (pkt->tiddma) {
422 /* Set new tid and offset for new sdma header */
423 hdr->iph.ver_ctxt_tid_offset = cpu_to_le32(
424 (le32_to_cpu(hdr->iph.ver_ctxt_tid_offset)&0xFF000000) +
425 (pkt->tidsm[pkt->tidsmidx].tid<<QLOGIC_IB_I_TID_SHIFT) +
426 (pkt->tidsm[pkt->tidsmidx].offset>>2));
427 } else {
428 /* Middle protocol new packet offset */
429 hdr->uwords[2] += pkt->payload_size;
430 }
431
432 /* New kdeth checksum */
433 vcto = le32_to_cpu(hdr->iph.ver_ctxt_tid_offset);
434 hdr->iph.chksum = cpu_to_le16(QIB_LRH_BTH +
435 be16_to_cpu(hdr->lrh[2]) -
436 ((vcto>>16)&0xFFFF) - (vcto&0xFFFF) -
437 le16_to_cpu(hdr->iph.pkt_flags));
438
439 /* Next sequence number in new sdma header */
440 seqnum.val = be32_to_cpu(hdr->bth[2]);
441 if (pkt->tiddma)
442 seqnum.seq++;
443 else
444 seqnum.pkt++;
445 hdr->bth[2] = cpu_to_be32(seqnum.val);
446
447 /* Init new sdma header. */
448 qib_user_sdma_init_frag(pkt, pkt->naddr, /* index */
449 0, pbclen, /* offset, len */
450 1, 0, /* first last desc */
451 0, 0, /* put page, dma mapped */
452 NULL, pbcvaddr, /* struct page, virt addr */
453 pbcdaddr, pbclen); /* dma addr, dma length */
454 pkt->index = pkt->naddr;
455 pkt->payload_size = 0;
456 pkt->naddr++;
457 if (pkt->naddr == pkt->addrlimit) {
458 ret = -EFAULT;
459 goto done;
460 }
461
462 /* Prepare for next fragment in this page */
463 if (newlen != len) {
464 if (dma_mapped) {
465 put = 0;
466 dma_mapped = 0;
467 page = NULL;
468 kvaddr = NULL;
469 }
470 len -= newlen;
471 offset += newlen;
472
473 goto next_fragment;
474 }
475
476done:
477 return ret;
171} 478}
172 479
173/* we've too many pages in the iovec, coalesce to a single page */ 480/* we've too many pages in the iovec, coalesce to a single page */
174static int qib_user_sdma_coalesce(const struct qib_devdata *dd, 481static int qib_user_sdma_coalesce(const struct qib_devdata *dd,
482 struct qib_user_sdma_queue *pq,
175 struct qib_user_sdma_pkt *pkt, 483 struct qib_user_sdma_pkt *pkt,
176 const struct iovec *iov, 484 const struct iovec *iov,
177 unsigned long niov) 485 unsigned long niov)
@@ -182,7 +490,6 @@ static int qib_user_sdma_coalesce(const struct qib_devdata *dd,
182 char *mpage; 490 char *mpage;
183 int i; 491 int i;
184 int len = 0; 492 int len = 0;
185 dma_addr_t dma_addr;
186 493
187 if (!page) { 494 if (!page) {
188 ret = -ENOMEM; 495 ret = -ENOMEM;
@@ -205,17 +512,8 @@ static int qib_user_sdma_coalesce(const struct qib_devdata *dd,
205 len += iov[i].iov_len; 512 len += iov[i].iov_len;
206 } 513 }
207 514
208 dma_addr = dma_map_page(&dd->pcidev->dev, page, 0, len, 515 ret = qib_user_sdma_page_to_frags(dd, pq, pkt,
209 DMA_TO_DEVICE); 516 page, 0, 0, len, mpage_save);
210 if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) {
211 ret = -ENOMEM;
212 goto free_unmap;
213 }
214
215 qib_user_sdma_init_frag(pkt, 1, 0, len, 0, 1, page, mpage_save,
216 dma_addr);
217 pkt->naddr = 2;
218
219 goto done; 517 goto done;
220 518
221free_unmap: 519free_unmap:
@@ -238,16 +536,6 @@ static int qib_user_sdma_num_pages(const struct iovec *iov)
238 return 1 + ((epage - spage) >> PAGE_SHIFT); 536 return 1 + ((epage - spage) >> PAGE_SHIFT);
239} 537}
240 538
241/*
242 * Truncate length to page boundary.
243 */
244static int qib_user_sdma_page_length(unsigned long addr, unsigned long len)
245{
246 const unsigned long offset = addr & ~PAGE_MASK;
247
248 return ((offset + len) > PAGE_SIZE) ? (PAGE_SIZE - offset) : len;
249}
250
251static void qib_user_sdma_free_pkt_frag(struct device *dev, 539static void qib_user_sdma_free_pkt_frag(struct device *dev,
252 struct qib_user_sdma_queue *pq, 540 struct qib_user_sdma_queue *pq,
253 struct qib_user_sdma_pkt *pkt, 541 struct qib_user_sdma_pkt *pkt,
@@ -256,10 +544,11 @@ static void qib_user_sdma_free_pkt_frag(struct device *dev,
256 const int i = frag; 544 const int i = frag;
257 545
258 if (pkt->addr[i].page) { 546 if (pkt->addr[i].page) {
547 /* only user data has page */
259 if (pkt->addr[i].dma_mapped) 548 if (pkt->addr[i].dma_mapped)
260 dma_unmap_page(dev, 549 dma_unmap_page(dev,
261 pkt->addr[i].addr, 550 pkt->addr[i].addr,
262 pkt->addr[i].length, 551 pkt->addr[i].dma_length,
263 DMA_TO_DEVICE); 552 DMA_TO_DEVICE);
264 553
265 if (pkt->addr[i].kvaddr) 554 if (pkt->addr[i].kvaddr)
@@ -269,55 +558,81 @@ static void qib_user_sdma_free_pkt_frag(struct device *dev,
269 put_page(pkt->addr[i].page); 558 put_page(pkt->addr[i].page);
270 else 559 else
271 __free_page(pkt->addr[i].page); 560 __free_page(pkt->addr[i].page);
272 } else if (pkt->addr[i].kvaddr) 561 } else if (pkt->addr[i].kvaddr) {
273 /* free coherent mem from cache... */ 562 /* for headers */
274 dma_pool_free(pq->header_cache, 563 if (pkt->addr[i].dma_mapped) {
564 /* from kmalloc & dma mapped */
565 dma_unmap_single(dev,
566 pkt->addr[i].addr,
567 pkt->addr[i].dma_length,
568 DMA_TO_DEVICE);
569 kfree(pkt->addr[i].kvaddr);
570 } else if (pkt->addr[i].addr) {
571 /* free coherent mem from cache... */
572 dma_pool_free(pq->header_cache,
275 pkt->addr[i].kvaddr, pkt->addr[i].addr); 573 pkt->addr[i].kvaddr, pkt->addr[i].addr);
574 } else {
575 /* from kmalloc but not dma mapped */
576 kfree(pkt->addr[i].kvaddr);
577 }
578 }
276} 579}
277 580
278/* return number of pages pinned... */ 581/* return number of pages pinned... */
279static int qib_user_sdma_pin_pages(const struct qib_devdata *dd, 582static int qib_user_sdma_pin_pages(const struct qib_devdata *dd,
583 struct qib_user_sdma_queue *pq,
280 struct qib_user_sdma_pkt *pkt, 584 struct qib_user_sdma_pkt *pkt,
281 unsigned long addr, int tlen, int npages) 585 unsigned long addr, int tlen, int npages)
282{ 586{
283 struct page *pages[2]; 587 struct page *pages[8];
284 int j; 588 int i, j;
285 int ret; 589 int ret = 0;
286
287 ret = get_user_pages(current, current->mm, addr,
288 npages, 0, 1, pages, NULL);
289
290 if (ret != npages) {
291 int i;
292
293 for (i = 0; i < ret; i++)
294 put_page(pages[i]);
295
296 ret = -ENOMEM;
297 goto done;
298 }
299 590
300 for (j = 0; j < npages; j++) { 591 while (npages) {
301 /* map the pages... */ 592 if (npages > 8)
302 const int flen = qib_user_sdma_page_length(addr, tlen); 593 j = 8;
303 dma_addr_t dma_addr = 594 else
304 dma_map_page(&dd->pcidev->dev, 595 j = npages;
305 pages[j], 0, flen, DMA_TO_DEVICE);
306 unsigned long fofs = addr & ~PAGE_MASK;
307 596
308 if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) { 597 ret = get_user_pages(current, current->mm, addr,
598 j, 0, 1, pages, NULL);
599 if (ret != j) {
600 i = 0;
601 j = ret;
309 ret = -ENOMEM; 602 ret = -ENOMEM;
310 goto done; 603 goto free_pages;
311 } 604 }
312 605
313 qib_user_sdma_init_frag(pkt, pkt->naddr, fofs, flen, 1, 1, 606 for (i = 0; i < j; i++) {
314 pages[j], kmap(pages[j]), dma_addr); 607 /* map the pages... */
608 unsigned long fofs = addr & ~PAGE_MASK;
609 int flen = ((fofs + tlen) > PAGE_SIZE) ?
610 (PAGE_SIZE - fofs) : tlen;
611
612 ret = qib_user_sdma_page_to_frags(dd, pq, pkt,
613 pages[i], 1, fofs, flen, NULL);
614 if (ret < 0) {
615 /* current page has beed taken
616 * care of inside above call.
617 */
618 i++;
619 goto free_pages;
620 }
315 621
316 pkt->naddr++; 622 addr += flen;
317 addr += flen; 623 tlen -= flen;
318 tlen -= flen; 624 }
625
626 npages -= j;
319 } 627 }
320 628
629 goto done;
630
631 /* if error, return all pages not managed by pkt */
632free_pages:
633 while (i < j)
634 put_page(pages[i++]);
635
321done: 636done:
322 return ret; 637 return ret;
323} 638}
@@ -335,7 +650,7 @@ static int qib_user_sdma_pin_pkt(const struct qib_devdata *dd,
335 const int npages = qib_user_sdma_num_pages(iov + idx); 650 const int npages = qib_user_sdma_num_pages(iov + idx);
336 const unsigned long addr = (unsigned long) iov[idx].iov_base; 651 const unsigned long addr = (unsigned long) iov[idx].iov_base;
337 652
338 ret = qib_user_sdma_pin_pages(dd, pkt, addr, 653 ret = qib_user_sdma_pin_pages(dd, pq, pkt, addr,
339 iov[idx].iov_len, npages); 654 iov[idx].iov_len, npages);
340 if (ret < 0) 655 if (ret < 0)
341 goto free_pkt; 656 goto free_pkt;
@@ -344,9 +659,22 @@ static int qib_user_sdma_pin_pkt(const struct qib_devdata *dd,
344 goto done; 659 goto done;
345 660
346free_pkt: 661free_pkt:
347 for (idx = 0; idx < pkt->naddr; idx++) 662 /* we need to ignore the first entry here */
663 for (idx = 1; idx < pkt->naddr; idx++)
348 qib_user_sdma_free_pkt_frag(&dd->pcidev->dev, pq, pkt, idx); 664 qib_user_sdma_free_pkt_frag(&dd->pcidev->dev, pq, pkt, idx);
349 665
666 /* need to dma unmap the first entry, this is to restore to
667 * the original state so that caller can free the memory in
668 * error condition. Caller does not know if dma mapped or not*/
669 if (pkt->addr[0].dma_mapped) {
670 dma_unmap_single(&dd->pcidev->dev,
671 pkt->addr[0].addr,
672 pkt->addr[0].dma_length,
673 DMA_TO_DEVICE);
674 pkt->addr[0].addr = 0;
675 pkt->addr[0].dma_mapped = 0;
676 }
677
350done: 678done:
351 return ret; 679 return ret;
352} 680}
@@ -359,8 +687,9 @@ static int qib_user_sdma_init_payload(const struct qib_devdata *dd,
359{ 687{
360 int ret = 0; 688 int ret = 0;
361 689
362 if (npages >= ARRAY_SIZE(pkt->addr)) 690 if (pkt->frag_size == pkt->bytes_togo &&
363 ret = qib_user_sdma_coalesce(dd, pkt, iov, niov); 691 npages >= ARRAY_SIZE(pkt->addr))
692 ret = qib_user_sdma_coalesce(dd, pq, pkt, iov, niov);
364 else 693 else
365 ret = qib_user_sdma_pin_pkt(dd, pq, pkt, iov, niov); 694 ret = qib_user_sdma_pin_pkt(dd, pq, pkt, iov, niov);
366 695
@@ -380,7 +709,10 @@ static void qib_user_sdma_free_pkt_list(struct device *dev,
380 for (i = 0; i < pkt->naddr; i++) 709 for (i = 0; i < pkt->naddr; i++)
381 qib_user_sdma_free_pkt_frag(dev, pq, pkt, i); 710 qib_user_sdma_free_pkt_frag(dev, pq, pkt, i);
382 711
383 kmem_cache_free(pq->pkt_slab, pkt); 712 if (pkt->largepkt)
713 kfree(pkt);
714 else
715 kmem_cache_free(pq->pkt_slab, pkt);
384 } 716 }
385 INIT_LIST_HEAD(list); 717 INIT_LIST_HEAD(list);
386} 718}
@@ -393,63 +725,48 @@ static void qib_user_sdma_free_pkt_list(struct device *dev,
393 * as, if there is an error we clean it... 725 * as, if there is an error we clean it...
394 */ 726 */
395static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd, 727static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd,
728 struct qib_pportdata *ppd,
396 struct qib_user_sdma_queue *pq, 729 struct qib_user_sdma_queue *pq,
397 struct list_head *list,
398 const struct iovec *iov, 730 const struct iovec *iov,
399 unsigned long niov, 731 unsigned long niov,
400 int maxpkts) 732 struct list_head *list,
733 int *maxpkts, int *ndesc)
401{ 734{
402 unsigned long idx = 0; 735 unsigned long idx = 0;
403 int ret = 0; 736 int ret = 0;
404 int npkts = 0; 737 int npkts = 0;
405 struct page *page = NULL;
406 __le32 *pbc; 738 __le32 *pbc;
407 dma_addr_t dma_addr; 739 dma_addr_t dma_addr;
408 struct qib_user_sdma_pkt *pkt = NULL; 740 struct qib_user_sdma_pkt *pkt = NULL;
409 size_t len; 741 size_t len;
410 size_t nw; 742 size_t nw;
411 u32 counter = pq->counter; 743 u32 counter = pq->counter;
412 int dma_mapped = 0; 744 u16 frag_size;
413 745
414 while (idx < niov && npkts < maxpkts) { 746 while (idx < niov && npkts < *maxpkts) {
415 const unsigned long addr = (unsigned long) iov[idx].iov_base; 747 const unsigned long addr = (unsigned long) iov[idx].iov_base;
416 const unsigned long idx_save = idx; 748 const unsigned long idx_save = idx;
417 unsigned pktnw; 749 unsigned pktnw;
418 unsigned pktnwc; 750 unsigned pktnwc;
419 int nfrags = 0; 751 int nfrags = 0;
420 int npages = 0; 752 int npages = 0;
753 int bytes_togo = 0;
754 int tiddma = 0;
421 int cfur; 755 int cfur;
422 756
423 dma_mapped = 0;
424 len = iov[idx].iov_len; 757 len = iov[idx].iov_len;
425 nw = len >> 2; 758 nw = len >> 2;
426 page = NULL;
427
428 pkt = kmem_cache_alloc(pq->pkt_slab, GFP_KERNEL);
429 if (!pkt) {
430 ret = -ENOMEM;
431 goto free_list;
432 }
433 759
434 if (len < QIB_USER_SDMA_MIN_HEADER_LENGTH || 760 if (len < QIB_USER_SDMA_MIN_HEADER_LENGTH ||
435 len > PAGE_SIZE || len & 3 || addr & 3) { 761 len > PAGE_SIZE || len & 3 || addr & 3) {
436 ret = -EINVAL; 762 ret = -EINVAL;
437 goto free_pkt; 763 goto free_list;
438 } 764 }
439 765
440 if (len == QIB_USER_SDMA_EXP_HEADER_LENGTH) 766 pbc = qib_user_sdma_alloc_header(pq, len, &dma_addr);
441 pbc = dma_pool_alloc(pq->header_cache, GFP_KERNEL,
442 &dma_addr);
443 else
444 pbc = NULL;
445
446 if (!pbc) { 767 if (!pbc) {
447 page = alloc_page(GFP_KERNEL); 768 ret = -ENOMEM;
448 if (!page) { 769 goto free_list;
449 ret = -ENOMEM;
450 goto free_pkt;
451 }
452 pbc = kmap(page);
453 } 770 }
454 771
455 cfur = copy_from_user(pbc, iov[idx].iov_base, len); 772 cfur = copy_from_user(pbc, iov[idx].iov_base, len);
@@ -474,8 +791,8 @@ static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd,
474 * we can verify that the packet is consistent with the 791 * we can verify that the packet is consistent with the
475 * iovec lengths. 792 * iovec lengths.
476 */ 793 */
477 pktnw = le32_to_cpu(*pbc) & QIB_PBC_LENGTH_MASK; 794 pktnw = le32_to_cpu(*pbc) & 0xFFFF;
478 if (pktnw < pktnwc || pktnw > pktnwc + (PAGE_SIZE >> 2)) { 795 if (pktnw < pktnwc) {
479 ret = -EINVAL; 796 ret = -EINVAL;
480 goto free_pbc; 797 goto free_pbc;
481 } 798 }
@@ -486,17 +803,14 @@ static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd,
486 const unsigned long faddr = 803 const unsigned long faddr =
487 (unsigned long) iov[idx].iov_base; 804 (unsigned long) iov[idx].iov_base;
488 805
489 if (slen & 3 || faddr & 3 || !slen || 806 if (slen & 3 || faddr & 3 || !slen) {
490 slen > PAGE_SIZE) {
491 ret = -EINVAL; 807 ret = -EINVAL;
492 goto free_pbc; 808 goto free_pbc;
493 } 809 }
494 810
495 npages++; 811 npages += qib_user_sdma_num_pages(&iov[idx]);
496 if ((faddr & PAGE_MASK) !=
497 ((faddr + slen - 1) & PAGE_MASK))
498 npages++;
499 812
813 bytes_togo += slen;
500 pktnwc += slen >> 2; 814 pktnwc += slen >> 2;
501 idx++; 815 idx++;
502 nfrags++; 816 nfrags++;
@@ -507,48 +821,139 @@ static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd,
507 goto free_pbc; 821 goto free_pbc;
508 } 822 }
509 823
510 if (page) { 824 frag_size = ((le32_to_cpu(*pbc))>>16) & 0xFFFF;
511 dma_addr = dma_map_page(&dd->pcidev->dev, 825 if (((frag_size ? frag_size : bytes_togo) + len) >
512 page, 0, len, DMA_TO_DEVICE); 826 ppd->ibmaxlen) {
513 if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) { 827 ret = -EINVAL;
828 goto free_pbc;
829 }
830
831 if (frag_size) {
832 int pktsize, tidsmsize, n;
833
834 n = npages*((2*PAGE_SIZE/frag_size)+1);
835 pktsize = sizeof(*pkt) + sizeof(pkt->addr[0])*n;
836
837 /*
838 * Determine if this is tid-sdma or just sdma.
839 */
840 tiddma = (((le32_to_cpu(pbc[7])>>
841 QLOGIC_IB_I_TID_SHIFT)&
842 QLOGIC_IB_I_TID_MASK) !=
843 QLOGIC_IB_I_TID_MASK);
844
845 if (tiddma)
846 tidsmsize = iov[idx].iov_len;
847 else
848 tidsmsize = 0;
849
850 pkt = kmalloc(pktsize+tidsmsize, GFP_KERNEL);
851 if (!pkt) {
514 ret = -ENOMEM; 852 ret = -ENOMEM;
515 goto free_pbc; 853 goto free_pbc;
516 } 854 }
855 pkt->largepkt = 1;
856 pkt->frag_size = frag_size;
857 pkt->addrlimit = n + ARRAY_SIZE(pkt->addr);
858
859 if (tiddma) {
860 char *tidsm = (char *)pkt + pktsize;
861 cfur = copy_from_user(tidsm,
862 iov[idx].iov_base, tidsmsize);
863 if (cfur) {
864 ret = -EFAULT;
865 goto free_pkt;
866 }
867 pkt->tidsm =
868 (struct qib_tid_session_member *)tidsm;
869 pkt->tidsmcount = tidsmsize/
870 sizeof(struct qib_tid_session_member);
871 pkt->tidsmidx = 0;
872 idx++;
873 }
517 874
518 dma_mapped = 1; 875 /*
876 * pbc 'fill1' field is borrowed to pass frag size,
877 * we need to clear it after picking frag size, the
878 * hardware requires this field to be zero.
879 */
880 *pbc = cpu_to_le32(le32_to_cpu(*pbc) & 0x0000FFFF);
881 } else {
882 pkt = kmem_cache_alloc(pq->pkt_slab, GFP_KERNEL);
883 if (!pkt) {
884 ret = -ENOMEM;
885 goto free_pbc;
886 }
887 pkt->largepkt = 0;
888 pkt->frag_size = bytes_togo;
889 pkt->addrlimit = ARRAY_SIZE(pkt->addr);
519 } 890 }
520 891 pkt->bytes_togo = bytes_togo;
521 qib_user_sdma_init_header(pkt, counter, 0, len, dma_mapped, 892 pkt->payload_size = 0;
522 page, pbc, dma_addr); 893 pkt->counter = counter;
894 pkt->tiddma = tiddma;
895
896 /* setup the first header */
897 qib_user_sdma_init_frag(pkt, 0, /* index */
898 0, len, /* offset, len */
899 1, 0, /* first last desc */
900 0, 0, /* put page, dma mapped */
901 NULL, pbc, /* struct page, virt addr */
902 dma_addr, len); /* dma addr, dma length */
903 pkt->index = 0;
904 pkt->naddr = 1;
523 905
524 if (nfrags) { 906 if (nfrags) {
525 ret = qib_user_sdma_init_payload(dd, pq, pkt, 907 ret = qib_user_sdma_init_payload(dd, pq, pkt,
526 iov + idx_save + 1, 908 iov + idx_save + 1,
527 nfrags, npages); 909 nfrags, npages);
528 if (ret < 0) 910 if (ret < 0)
529 goto free_pbc_dma; 911 goto free_pkt;
912 } else {
913 /* since there is no payload, mark the
914 * header as the last desc. */
915 pkt->addr[0].last_desc = 1;
916
917 if (dma_addr == 0) {
918 /*
919 * the header is not dma mapped yet.
920 * it should be from kmalloc.
921 */
922 dma_addr = dma_map_single(&dd->pcidev->dev,
923 pbc, len, DMA_TO_DEVICE);
924 if (dma_mapping_error(&dd->pcidev->dev,
925 dma_addr)) {
926 ret = -ENOMEM;
927 goto free_pkt;
928 }
929 pkt->addr[0].addr = dma_addr;
930 pkt->addr[0].dma_mapped = 1;
931 }
530 } 932 }
531 933
532 counter++; 934 counter++;
533 npkts++; 935 npkts++;
936 pkt->pq = pq;
937 pkt->index = 0; /* reset index for push on hw */
938 *ndesc += pkt->naddr;
534 939
535 list_add_tail(&pkt->list, list); 940 list_add_tail(&pkt->list, list);
536 } 941 }
537 942
943 *maxpkts = npkts;
538 ret = idx; 944 ret = idx;
539 goto done; 945 goto done;
540 946
541free_pbc_dma: 947free_pkt:
542 if (dma_mapped) 948 if (pkt->largepkt)
543 dma_unmap_page(&dd->pcidev->dev, dma_addr, len, DMA_TO_DEVICE); 949 kfree(pkt);
950 else
951 kmem_cache_free(pq->pkt_slab, pkt);
544free_pbc: 952free_pbc:
545 if (page) { 953 if (dma_addr)
546 kunmap(page);
547 __free_page(page);
548 } else
549 dma_pool_free(pq->header_cache, pbc, dma_addr); 954 dma_pool_free(pq->header_cache, pbc, dma_addr);
550free_pkt: 955 else
551 kmem_cache_free(pq->pkt_slab, pkt); 956 kfree(pbc);
552free_list: 957free_list:
553 qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, list); 958 qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, list);
554done: 959done:
@@ -569,10 +974,20 @@ static int qib_user_sdma_queue_clean(struct qib_pportdata *ppd,
569 struct list_head free_list; 974 struct list_head free_list;
570 struct qib_user_sdma_pkt *pkt; 975 struct qib_user_sdma_pkt *pkt;
571 struct qib_user_sdma_pkt *pkt_prev; 976 struct qib_user_sdma_pkt *pkt_prev;
977 unsigned long flags;
572 int ret = 0; 978 int ret = 0;
573 979
980 if (!pq->num_sending)
981 return 0;
982
574 INIT_LIST_HEAD(&free_list); 983 INIT_LIST_HEAD(&free_list);
575 984
985 /*
986 * We need this spin lock here because interrupt handler
987 * might modify this list in qib_user_sdma_send_desc(), also
988 * we can not get interrupted, otherwise it is a deadlock.
989 */
990 spin_lock_irqsave(&pq->sent_lock, flags);
576 list_for_each_entry_safe(pkt, pkt_prev, &pq->sent, list) { 991 list_for_each_entry_safe(pkt, pkt_prev, &pq->sent, list) {
577 s64 descd = ppd->sdma_descq_removed - pkt->added; 992 s64 descd = ppd->sdma_descq_removed - pkt->added;
578 993
@@ -583,7 +998,9 @@ static int qib_user_sdma_queue_clean(struct qib_pportdata *ppd,
583 998
584 /* one more packet cleaned */ 999 /* one more packet cleaned */
585 ret++; 1000 ret++;
1001 pq->num_sending--;
586 } 1002 }
1003 spin_unlock_irqrestore(&pq->sent_lock, flags);
587 1004
588 if (!list_empty(&free_list)) { 1005 if (!list_empty(&free_list)) {
589 u32 counter; 1006 u32 counter;
@@ -627,6 +1044,7 @@ void qib_user_sdma_queue_drain(struct qib_pportdata *ppd,
627 struct qib_user_sdma_queue *pq) 1044 struct qib_user_sdma_queue *pq)
628{ 1045{
629 struct qib_devdata *dd = ppd->dd; 1046 struct qib_devdata *dd = ppd->dd;
1047 unsigned long flags;
630 int i; 1048 int i;
631 1049
632 if (!pq) 1050 if (!pq)
@@ -634,7 +1052,7 @@ void qib_user_sdma_queue_drain(struct qib_pportdata *ppd,
634 1052
635 for (i = 0; i < QIB_USER_SDMA_DRAIN_TIMEOUT; i++) { 1053 for (i = 0; i < QIB_USER_SDMA_DRAIN_TIMEOUT; i++) {
636 mutex_lock(&pq->lock); 1054 mutex_lock(&pq->lock);
637 if (list_empty(&pq->sent)) { 1055 if (!pq->num_pending && !pq->num_sending) {
638 mutex_unlock(&pq->lock); 1056 mutex_unlock(&pq->lock);
639 break; 1057 break;
640 } 1058 }
@@ -644,29 +1062,44 @@ void qib_user_sdma_queue_drain(struct qib_pportdata *ppd,
644 msleep(10); 1062 msleep(10);
645 } 1063 }
646 1064
647 if (!list_empty(&pq->sent)) { 1065 if (pq->num_pending || pq->num_sending) {
1066 struct qib_user_sdma_pkt *pkt;
1067 struct qib_user_sdma_pkt *pkt_prev;
648 struct list_head free_list; 1068 struct list_head free_list;
649 1069
1070 mutex_lock(&pq->lock);
1071 spin_lock_irqsave(&ppd->sdma_lock, flags);
1072 /*
1073 * Since we hold sdma_lock, it is safe without sent_lock.
1074 */
1075 if (pq->num_pending) {
1076 list_for_each_entry_safe(pkt, pkt_prev,
1077 &ppd->sdma_userpending, list) {
1078 if (pkt->pq == pq) {
1079 list_move_tail(&pkt->list, &pq->sent);
1080 pq->num_pending--;
1081 pq->num_sending++;
1082 }
1083 }
1084 }
1085 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1086
650 qib_dev_err(dd, "user sdma lists not empty: forcing!\n"); 1087 qib_dev_err(dd, "user sdma lists not empty: forcing!\n");
651 INIT_LIST_HEAD(&free_list); 1088 INIT_LIST_HEAD(&free_list);
652 mutex_lock(&pq->lock);
653 list_splice_init(&pq->sent, &free_list); 1089 list_splice_init(&pq->sent, &free_list);
1090 pq->num_sending = 0;
654 qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list); 1091 qib_user_sdma_free_pkt_list(&dd->pcidev->dev, pq, &free_list);
655 mutex_unlock(&pq->lock); 1092 mutex_unlock(&pq->lock);
656 } 1093 }
657} 1094}
658 1095
659static inline __le64 qib_sdma_make_desc0(struct qib_pportdata *ppd, 1096static inline __le64 qib_sdma_make_desc0(u8 gen,
660 u64 addr, u64 dwlen, u64 dwoffset) 1097 u64 addr, u64 dwlen, u64 dwoffset)
661{ 1098{
662 u8 tmpgen;
663
664 tmpgen = ppd->sdma_generation;
665
666 return cpu_to_le64(/* SDmaPhyAddr[31:0] */ 1099 return cpu_to_le64(/* SDmaPhyAddr[31:0] */
667 ((addr & 0xfffffffcULL) << 32) | 1100 ((addr & 0xfffffffcULL) << 32) |
668 /* SDmaGeneration[1:0] */ 1101 /* SDmaGeneration[1:0] */
669 ((tmpgen & 3ULL) << 30) | 1102 ((gen & 3ULL) << 30) |
670 /* SDmaDwordCount[10:0] */ 1103 /* SDmaDwordCount[10:0] */
671 ((dwlen & 0x7ffULL) << 16) | 1104 ((dwlen & 0x7ffULL) << 16) |
672 /* SDmaBufOffset[12:2] */ 1105 /* SDmaBufOffset[12:2] */
@@ -692,7 +1125,7 @@ static inline __le64 qib_sdma_make_desc1(u64 addr)
692 1125
693static void qib_user_sdma_send_frag(struct qib_pportdata *ppd, 1126static void qib_user_sdma_send_frag(struct qib_pportdata *ppd,
694 struct qib_user_sdma_pkt *pkt, int idx, 1127 struct qib_user_sdma_pkt *pkt, int idx,
695 unsigned ofs, u16 tail) 1128 unsigned ofs, u16 tail, u8 gen)
696{ 1129{
697 const u64 addr = (u64) pkt->addr[idx].addr + 1130 const u64 addr = (u64) pkt->addr[idx].addr +
698 (u64) pkt->addr[idx].offset; 1131 (u64) pkt->addr[idx].offset;
@@ -702,104 +1135,132 @@ static void qib_user_sdma_send_frag(struct qib_pportdata *ppd,
702 1135
703 descqp = &ppd->sdma_descq[tail].qw[0]; 1136 descqp = &ppd->sdma_descq[tail].qw[0];
704 1137
705 descq0 = qib_sdma_make_desc0(ppd, addr, dwlen, ofs); 1138 descq0 = qib_sdma_make_desc0(gen, addr, dwlen, ofs);
706 if (idx == 0) 1139 if (pkt->addr[idx].first_desc)
707 descq0 = qib_sdma_make_first_desc0(descq0); 1140 descq0 = qib_sdma_make_first_desc0(descq0);
708 if (idx == pkt->naddr - 1) 1141 if (pkt->addr[idx].last_desc) {
709 descq0 = qib_sdma_make_last_desc0(descq0); 1142 descq0 = qib_sdma_make_last_desc0(descq0);
1143 if (ppd->sdma_intrequest) {
1144 descq0 |= cpu_to_le64(1ULL << 15);
1145 ppd->sdma_intrequest = 0;
1146 }
1147 }
710 1148
711 descqp[0] = descq0; 1149 descqp[0] = descq0;
712 descqp[1] = qib_sdma_make_desc1(addr); 1150 descqp[1] = qib_sdma_make_desc1(addr);
713} 1151}
714 1152
715/* pq->lock must be held, get packets on the wire... */ 1153void qib_user_sdma_send_desc(struct qib_pportdata *ppd,
716static int qib_user_sdma_push_pkts(struct qib_pportdata *ppd, 1154 struct list_head *pktlist)
717 struct qib_user_sdma_queue *pq,
718 struct list_head *pktlist)
719{ 1155{
720 struct qib_devdata *dd = ppd->dd; 1156 struct qib_devdata *dd = ppd->dd;
721 int ret = 0; 1157 u16 nfree, nsent;
722 unsigned long flags; 1158 u16 tail, tail_c;
723 u16 tail; 1159 u8 gen, gen_c;
724 u8 generation;
725 u64 descq_added;
726
727 if (list_empty(pktlist))
728 return 0;
729 1160
730 if (unlikely(!(ppd->lflags & QIBL_LINKACTIVE))) 1161 nfree = qib_sdma_descq_freecnt(ppd);
731 return -ECOMM; 1162 if (!nfree)
732 1163 return;
733 spin_lock_irqsave(&ppd->sdma_lock, flags);
734
735 /* keep a copy for restoring purposes in case of problems */
736 generation = ppd->sdma_generation;
737 descq_added = ppd->sdma_descq_added;
738
739 if (unlikely(!__qib_sdma_running(ppd))) {
740 ret = -ECOMM;
741 goto unlock;
742 }
743 1164
744 tail = ppd->sdma_descq_tail; 1165retry:
1166 nsent = 0;
1167 tail_c = tail = ppd->sdma_descq_tail;
1168 gen_c = gen = ppd->sdma_generation;
745 while (!list_empty(pktlist)) { 1169 while (!list_empty(pktlist)) {
746 struct qib_user_sdma_pkt *pkt = 1170 struct qib_user_sdma_pkt *pkt =
747 list_entry(pktlist->next, struct qib_user_sdma_pkt, 1171 list_entry(pktlist->next, struct qib_user_sdma_pkt,
748 list); 1172 list);
749 int i; 1173 int i, j, c = 0;
750 unsigned ofs = 0; 1174 unsigned ofs = 0;
751 u16 dtail = tail; 1175 u16 dtail = tail;
752 1176
753 if (pkt->naddr > qib_sdma_descq_freecnt(ppd)) 1177 for (i = pkt->index; i < pkt->naddr && nfree; i++) {
754 goto unlock_check_tail; 1178 qib_user_sdma_send_frag(ppd, pkt, i, ofs, tail, gen);
755
756 for (i = 0; i < pkt->naddr; i++) {
757 qib_user_sdma_send_frag(ppd, pkt, i, ofs, tail);
758 ofs += pkt->addr[i].length >> 2; 1179 ofs += pkt->addr[i].length >> 2;
759 1180
760 if (++tail == ppd->sdma_descq_cnt) { 1181 if (++tail == ppd->sdma_descq_cnt) {
761 tail = 0; 1182 tail = 0;
762 ++ppd->sdma_generation; 1183 ++gen;
1184 ppd->sdma_intrequest = 1;
1185 } else if (tail == (ppd->sdma_descq_cnt>>1)) {
1186 ppd->sdma_intrequest = 1;
763 } 1187 }
764 } 1188 nfree--;
1189 if (pkt->addr[i].last_desc == 0)
1190 continue;
765 1191
766 if ((ofs << 2) > ppd->ibmaxlen) { 1192 /*
767 ret = -EMSGSIZE; 1193 * If the packet is >= 2KB mtu equivalent, we
768 goto unlock; 1194 * have to use the large buffers, and have to
769 } 1195 * mark each descriptor as part of a large
770 1196 * buffer packet.
771 /* 1197 */
772 * If the packet is >= 2KB mtu equivalent, we have to use 1198 if (ofs > dd->piosize2kmax_dwords) {
773 * the large buffers, and have to mark each descriptor as 1199 for (j = pkt->index; j <= i; j++) {
774 * part of a large buffer packet. 1200 ppd->sdma_descq[dtail].qw[0] |=
775 */ 1201 cpu_to_le64(1ULL << 14);
776 if (ofs > dd->piosize2kmax_dwords) { 1202 if (++dtail == ppd->sdma_descq_cnt)
777 for (i = 0; i < pkt->naddr; i++) { 1203 dtail = 0;
778 ppd->sdma_descq[dtail].qw[0] |= 1204 }
779 cpu_to_le64(1ULL << 14);
780 if (++dtail == ppd->sdma_descq_cnt)
781 dtail = 0;
782 } 1205 }
1206 c += i + 1 - pkt->index;
1207 pkt->index = i + 1; /* index for next first */
1208 tail_c = dtail = tail;
1209 gen_c = gen;
1210 ofs = 0; /* reset for next packet */
783 } 1211 }
784 1212
785 ppd->sdma_descq_added += pkt->naddr; 1213 ppd->sdma_descq_added += c;
786 pkt->added = ppd->sdma_descq_added; 1214 nsent += c;
787 list_move_tail(&pkt->list, &pq->sent); 1215 if (pkt->index == pkt->naddr) {
788 ret++; 1216 pkt->added = ppd->sdma_descq_added;
1217 pkt->pq->added = pkt->added;
1218 pkt->pq->num_pending--;
1219 spin_lock(&pkt->pq->sent_lock);
1220 pkt->pq->num_sending++;
1221 list_move_tail(&pkt->list, &pkt->pq->sent);
1222 spin_unlock(&pkt->pq->sent_lock);
1223 }
1224 if (!nfree || (nsent<<2) > ppd->sdma_descq_cnt)
1225 break;
789 } 1226 }
790 1227
791unlock_check_tail:
792 /* advance the tail on the chip if necessary */ 1228 /* advance the tail on the chip if necessary */
793 if (ppd->sdma_descq_tail != tail) 1229 if (ppd->sdma_descq_tail != tail_c) {
794 dd->f_sdma_update_tail(ppd, tail); 1230 ppd->sdma_generation = gen_c;
1231 dd->f_sdma_update_tail(ppd, tail_c);
1232 }
795 1233
796unlock: 1234 if (nfree && !list_empty(pktlist))
797 if (unlikely(ret < 0)) { 1235 goto retry;
798 ppd->sdma_generation = generation; 1236
799 ppd->sdma_descq_added = descq_added; 1237 return;
1238}
1239
1240/* pq->lock must be held, get packets on the wire... */
1241static int qib_user_sdma_push_pkts(struct qib_pportdata *ppd,
1242 struct qib_user_sdma_queue *pq,
1243 struct list_head *pktlist, int count)
1244{
1245 int ret = 0;
1246 unsigned long flags;
1247
1248 if (unlikely(!(ppd->lflags & QIBL_LINKACTIVE)))
1249 return -ECOMM;
1250
1251 spin_lock_irqsave(&ppd->sdma_lock, flags);
1252
1253 if (unlikely(!__qib_sdma_running(ppd))) {
1254 ret = -ECOMM;
1255 goto unlock;
800 } 1256 }
801 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
802 1257
1258 pq->num_pending += count;
1259 list_splice_tail_init(pktlist, &ppd->sdma_userpending);
1260 qib_user_sdma_send_desc(ppd, &ppd->sdma_userpending);
1261
1262unlock:
1263 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
803 return ret; 1264 return ret;
804} 1265}
805 1266
@@ -822,19 +1283,23 @@ int qib_user_sdma_writev(struct qib_ctxtdata *rcd,
822 if (!qib_sdma_running(ppd)) 1283 if (!qib_sdma_running(ppd))
823 goto done_unlock; 1284 goto done_unlock;
824 1285
825 if (ppd->sdma_descq_added != ppd->sdma_descq_removed) { 1286 /* if I have packets not complete yet */
1287 if (pq->added > ppd->sdma_descq_removed)
826 qib_user_sdma_hwqueue_clean(ppd); 1288 qib_user_sdma_hwqueue_clean(ppd);
1289 /* if I have complete packets to be freed */
1290 if (pq->num_sending)
827 qib_user_sdma_queue_clean(ppd, pq); 1291 qib_user_sdma_queue_clean(ppd, pq);
828 }
829 1292
830 while (dim) { 1293 while (dim) {
831 const int mxp = 8; 1294 int mxp = 8;
1295 int ndesc = 0;
832 1296
833 down_write(&current->mm->mmap_sem); 1297 down_write(&current->mm->mmap_sem);
834 ret = qib_user_sdma_queue_pkts(dd, pq, &list, iov, dim, mxp); 1298 ret = qib_user_sdma_queue_pkts(dd, ppd, pq,
1299 iov, dim, &list, &mxp, &ndesc);
835 up_write(&current->mm->mmap_sem); 1300 up_write(&current->mm->mmap_sem);
836 1301
837 if (ret <= 0) 1302 if (ret < 0)
838 goto done_unlock; 1303 goto done_unlock;
839 else { 1304 else {
840 dim -= ret; 1305 dim -= ret;
@@ -844,24 +1309,20 @@ int qib_user_sdma_writev(struct qib_ctxtdata *rcd,
844 /* force packets onto the sdma hw queue... */ 1309 /* force packets onto the sdma hw queue... */
845 if (!list_empty(&list)) { 1310 if (!list_empty(&list)) {
846 /* 1311 /*
847 * Lazily clean hw queue. the 4 is a guess of about 1312 * Lazily clean hw queue.
848 * how many sdma descriptors a packet will take (it
849 * doesn't have to be perfect).
850 */ 1313 */
851 if (qib_sdma_descq_freecnt(ppd) < ret * 4) { 1314 if (qib_sdma_descq_freecnt(ppd) < ndesc) {
852 qib_user_sdma_hwqueue_clean(ppd); 1315 qib_user_sdma_hwqueue_clean(ppd);
853 qib_user_sdma_queue_clean(ppd, pq); 1316 if (pq->num_sending)
1317 qib_user_sdma_queue_clean(ppd, pq);
854 } 1318 }
855 1319
856 ret = qib_user_sdma_push_pkts(ppd, pq, &list); 1320 ret = qib_user_sdma_push_pkts(ppd, pq, &list, mxp);
857 if (ret < 0) 1321 if (ret < 0)
858 goto done_unlock; 1322 goto done_unlock;
859 else { 1323 else {
860 npkts += ret; 1324 npkts += mxp;
861 pq->counter += ret; 1325 pq->counter += mxp;
862
863 if (!list_empty(&list))
864 goto done_unlock;
865 } 1326 }
866 } 1327 }
867 } 1328 }
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
index 3eceb61e3532..7a3175400b2a 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
@@ -817,7 +817,6 @@ void ipoib_cm_handle_tx_wc(struct net_device *dev, struct ib_wc *wc)
817 817
818 if (neigh) { 818 if (neigh) {
819 neigh->cm = NULL; 819 neigh->cm = NULL;
820 list_del(&neigh->list);
821 ipoib_neigh_free(neigh); 820 ipoib_neigh_free(neigh);
822 821
823 tx->neigh = NULL; 822 tx->neigh = NULL;
@@ -1234,7 +1233,6 @@ static int ipoib_cm_tx_handler(struct ib_cm_id *cm_id,
1234 1233
1235 if (neigh) { 1234 if (neigh) {
1236 neigh->cm = NULL; 1235 neigh->cm = NULL;
1237 list_del(&neigh->list);
1238 ipoib_neigh_free(neigh); 1236 ipoib_neigh_free(neigh);
1239 1237
1240 tx->neigh = NULL; 1238 tx->neigh = NULL;
@@ -1325,7 +1323,6 @@ static void ipoib_cm_tx_start(struct work_struct *work)
1325 neigh = p->neigh; 1323 neigh = p->neigh;
1326 if (neigh) { 1324 if (neigh) {
1327 neigh->cm = NULL; 1325 neigh->cm = NULL;
1328 list_del(&neigh->list);
1329 ipoib_neigh_free(neigh); 1326 ipoib_neigh_free(neigh);
1330 } 1327 }
1331 list_del(&p->list); 1328 list_del(&p->list);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index c6f71a88c55c..82cec1af902c 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -493,7 +493,6 @@ static void path_rec_completion(int status,
493 path, 493 path,
494 neigh)); 494 neigh));
495 if (!ipoib_cm_get(neigh)) { 495 if (!ipoib_cm_get(neigh)) {
496 list_del(&neigh->list);
497 ipoib_neigh_free(neigh); 496 ipoib_neigh_free(neigh);
498 continue; 497 continue;
499 } 498 }
@@ -618,7 +617,6 @@ static void neigh_add_path(struct sk_buff *skb, u8 *daddr,
618 if (!ipoib_cm_get(neigh)) 617 if (!ipoib_cm_get(neigh))
619 ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh)); 618 ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh));
620 if (!ipoib_cm_get(neigh)) { 619 if (!ipoib_cm_get(neigh)) {
621 list_del(&neigh->list);
622 ipoib_neigh_free(neigh); 620 ipoib_neigh_free(neigh);
623 goto err_drop; 621 goto err_drop;
624 } 622 }
@@ -639,7 +637,7 @@ static void neigh_add_path(struct sk_buff *skb, u8 *daddr,
639 neigh->ah = NULL; 637 neigh->ah = NULL;
640 638
641 if (!path->query && path_rec_start(dev, path)) 639 if (!path->query && path_rec_start(dev, path))
642 goto err_list; 640 goto err_path;
643 641
644 __skb_queue_tail(&neigh->queue, skb); 642 __skb_queue_tail(&neigh->queue, skb);
645 } 643 }
@@ -648,9 +646,6 @@ static void neigh_add_path(struct sk_buff *skb, u8 *daddr,
648 ipoib_neigh_put(neigh); 646 ipoib_neigh_put(neigh);
649 return; 647 return;
650 648
651err_list:
652 list_del(&neigh->list);
653
654err_path: 649err_path:
655 ipoib_neigh_free(neigh); 650 ipoib_neigh_free(neigh);
656err_drop: 651err_drop:
@@ -1098,6 +1093,8 @@ void ipoib_neigh_free(struct ipoib_neigh *neigh)
1098 rcu_assign_pointer(*np, 1093 rcu_assign_pointer(*np,
1099 rcu_dereference_protected(neigh->hnext, 1094 rcu_dereference_protected(neigh->hnext,
1100 lockdep_is_held(&priv->lock))); 1095 lockdep_is_held(&priv->lock)));
1096 /* remove from parent list */
1097 list_del(&neigh->list);
1101 call_rcu(&neigh->rcu, ipoib_neigh_reclaim); 1098 call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
1102 return; 1099 return;
1103 } else { 1100 } else {
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index 2e84ef859c5b..705de7b40201 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -347,6 +347,7 @@ iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,
347{ 347{
348 struct iscsi_conn *conn = cls_conn->dd_data; 348 struct iscsi_conn *conn = cls_conn->dd_data;
349 struct iscsi_iser_conn *iser_conn; 349 struct iscsi_iser_conn *iser_conn;
350 struct iscsi_session *session;
350 struct iser_conn *ib_conn; 351 struct iser_conn *ib_conn;
351 struct iscsi_endpoint *ep; 352 struct iscsi_endpoint *ep;
352 int error; 353 int error;
@@ -365,7 +366,8 @@ iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,
365 } 366 }
366 ib_conn = ep->dd_data; 367 ib_conn = ep->dd_data;
367 368
368 if (iser_alloc_rx_descriptors(ib_conn)) 369 session = conn->session;
370 if (iser_alloc_rx_descriptors(ib_conn, session))
369 return -ENOMEM; 371 return -ENOMEM;
370 372
371 /* binds the iSER connection retrieved from the previously 373 /* binds the iSER connection retrieved from the previously
@@ -419,12 +421,13 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
419 struct iscsi_cls_session *cls_session; 421 struct iscsi_cls_session *cls_session;
420 struct iscsi_session *session; 422 struct iscsi_session *session;
421 struct Scsi_Host *shost; 423 struct Scsi_Host *shost;
422 struct iser_conn *ib_conn; 424 struct iser_conn *ib_conn = NULL;
423 425
424 shost = iscsi_host_alloc(&iscsi_iser_sht, 0, 0); 426 shost = iscsi_host_alloc(&iscsi_iser_sht, 0, 0);
425 if (!shost) 427 if (!shost)
426 return NULL; 428 return NULL;
427 shost->transportt = iscsi_iser_scsi_transport; 429 shost->transportt = iscsi_iser_scsi_transport;
430 shost->cmd_per_lun = qdepth;
428 shost->max_lun = iscsi_max_lun; 431 shost->max_lun = iscsi_max_lun;
429 shost->max_id = 0; 432 shost->max_id = 0;
430 shost->max_channel = 0; 433 shost->max_channel = 0;
@@ -441,12 +444,14 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
441 ep ? ib_conn->device->ib_device->dma_device : NULL)) 444 ep ? ib_conn->device->ib_device->dma_device : NULL))
442 goto free_host; 445 goto free_host;
443 446
444 /* 447 if (cmds_max > ISER_DEF_XMIT_CMDS_MAX) {
445 * we do not support setting can_queue cmd_per_lun from userspace yet 448 iser_info("cmds_max changed from %u to %u\n",
446 * because we preallocate so many resources 449 cmds_max, ISER_DEF_XMIT_CMDS_MAX);
447 */ 450 cmds_max = ISER_DEF_XMIT_CMDS_MAX;
451 }
452
448 cls_session = iscsi_session_setup(&iscsi_iser_transport, shost, 453 cls_session = iscsi_session_setup(&iscsi_iser_transport, shost,
449 ISCSI_DEF_XMIT_CMDS_MAX, 0, 454 cmds_max, 0,
450 sizeof(struct iscsi_iser_task), 455 sizeof(struct iscsi_iser_task),
451 initial_cmdsn, 0); 456 initial_cmdsn, 0);
452 if (!cls_session) 457 if (!cls_session)
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index 4f069c0d4c04..67914027c614 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -78,14 +78,14 @@
78 78
79#define iser_warn(fmt, arg...) \ 79#define iser_warn(fmt, arg...) \
80 do { \ 80 do { \
81 if (iser_debug_level > 1) \ 81 if (iser_debug_level > 0) \
82 pr_warn(PFX "%s:" fmt, \ 82 pr_warn(PFX "%s:" fmt, \
83 __func__ , ## arg); \ 83 __func__ , ## arg); \
84 } while (0) 84 } while (0)
85 85
86#define iser_info(fmt, arg...) \ 86#define iser_info(fmt, arg...) \
87 do { \ 87 do { \
88 if (iser_debug_level > 0) \ 88 if (iser_debug_level > 1) \
89 pr_info(PFX "%s:" fmt, \ 89 pr_info(PFX "%s:" fmt, \
90 __func__ , ## arg); \ 90 __func__ , ## arg); \
91 } while (0) 91 } while (0)
@@ -102,7 +102,13 @@
102 102
103 /* support up to 512KB in one RDMA */ 103 /* support up to 512KB in one RDMA */
104#define ISCSI_ISER_SG_TABLESIZE (0x80000 >> SHIFT_4K) 104#define ISCSI_ISER_SG_TABLESIZE (0x80000 >> SHIFT_4K)
105#define ISER_DEF_CMD_PER_LUN ISCSI_DEF_XMIT_CMDS_MAX 105#define ISER_DEF_XMIT_CMDS_DEFAULT 512
106#if ISCSI_DEF_XMIT_CMDS_MAX > ISER_DEF_XMIT_CMDS_DEFAULT
107 #define ISER_DEF_XMIT_CMDS_MAX ISCSI_DEF_XMIT_CMDS_MAX
108#else
109 #define ISER_DEF_XMIT_CMDS_MAX ISER_DEF_XMIT_CMDS_DEFAULT
110#endif
111#define ISER_DEF_CMD_PER_LUN ISER_DEF_XMIT_CMDS_MAX
106 112
107/* QP settings */ 113/* QP settings */
108/* Maximal bounds on received asynchronous PDUs */ 114/* Maximal bounds on received asynchronous PDUs */
@@ -111,9 +117,9 @@
111#define ISER_MAX_TX_MISC_PDUS 6 /* NOOP_OUT(2), TEXT(1), * 117#define ISER_MAX_TX_MISC_PDUS 6 /* NOOP_OUT(2), TEXT(1), *
112 * SCSI_TMFUNC(2), LOGOUT(1) */ 118 * SCSI_TMFUNC(2), LOGOUT(1) */
113 119
114#define ISER_QP_MAX_RECV_DTOS (ISCSI_DEF_XMIT_CMDS_MAX) 120#define ISER_QP_MAX_RECV_DTOS (ISER_DEF_XMIT_CMDS_MAX)
115 121
116#define ISER_MIN_POSTED_RX (ISCSI_DEF_XMIT_CMDS_MAX >> 2) 122#define ISER_MIN_POSTED_RX (ISER_DEF_XMIT_CMDS_MAX >> 2)
117 123
118/* the max TX (send) WR supported by the iSER QP is defined by * 124/* the max TX (send) WR supported by the iSER QP is defined by *
119 * max_send_wr = T * (1 + D) + C ; D is how many inflight dataouts we expect * 125 * max_send_wr = T * (1 + D) + C ; D is how many inflight dataouts we expect *
@@ -123,7 +129,7 @@
123 129
124#define ISER_INFLIGHT_DATAOUTS 8 130#define ISER_INFLIGHT_DATAOUTS 8
125 131
126#define ISER_QP_MAX_REQ_DTOS (ISCSI_DEF_XMIT_CMDS_MAX * \ 132#define ISER_QP_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
127 (1 + ISER_INFLIGHT_DATAOUTS) + \ 133 (1 + ISER_INFLIGHT_DATAOUTS) + \
128 ISER_MAX_TX_MISC_PDUS + \ 134 ISER_MAX_TX_MISC_PDUS + \
129 ISER_MAX_RX_MISC_PDUS) 135 ISER_MAX_RX_MISC_PDUS)
@@ -205,7 +211,7 @@ struct iser_mem_reg {
205 u64 va; 211 u64 va;
206 u64 len; 212 u64 len;
207 void *mem_h; 213 void *mem_h;
208 int is_fmr; 214 int is_mr;
209}; 215};
210 216
211struct iser_regd_buf { 217struct iser_regd_buf {
@@ -246,6 +252,9 @@ struct iser_rx_desc {
246 252
247#define ISER_MAX_CQ 4 253#define ISER_MAX_CQ 4
248 254
255struct iser_conn;
256struct iscsi_iser_task;
257
249struct iser_device { 258struct iser_device {
250 struct ib_device *ib_device; 259 struct ib_device *ib_device;
251 struct ib_pd *pd; 260 struct ib_pd *pd;
@@ -259,6 +268,22 @@ struct iser_device {
259 int cq_active_qps[ISER_MAX_CQ]; 268 int cq_active_qps[ISER_MAX_CQ];
260 int cqs_used; 269 int cqs_used;
261 struct iser_cq_desc *cq_desc; 270 struct iser_cq_desc *cq_desc;
271 int (*iser_alloc_rdma_reg_res)(struct iser_conn *ib_conn,
272 unsigned cmds_max);
273 void (*iser_free_rdma_reg_res)(struct iser_conn *ib_conn);
274 int (*iser_reg_rdma_mem)(struct iscsi_iser_task *iser_task,
275 enum iser_data_dir cmd_dir);
276 void (*iser_unreg_rdma_mem)(struct iscsi_iser_task *iser_task,
277 enum iser_data_dir cmd_dir);
278};
279
280struct fast_reg_descriptor {
281 struct list_head list;
282 /* For fast registration - FRWR */
283 struct ib_mr *data_mr;
284 struct ib_fast_reg_page_list *data_frpl;
285 /* Valid for fast registration flag */
286 bool valid;
262}; 287};
263 288
264struct iser_conn { 289struct iser_conn {
@@ -270,13 +295,13 @@ struct iser_conn {
270 struct iser_device *device; /* device context */ 295 struct iser_device *device; /* device context */
271 struct rdma_cm_id *cma_id; /* CMA ID */ 296 struct rdma_cm_id *cma_id; /* CMA ID */
272 struct ib_qp *qp; /* QP */ 297 struct ib_qp *qp; /* QP */
273 struct ib_fmr_pool *fmr_pool; /* pool of IB FMRs */
274 wait_queue_head_t wait; /* waitq for conn/disconn */ 298 wait_queue_head_t wait; /* waitq for conn/disconn */
299 unsigned qp_max_recv_dtos; /* num of rx buffers */
300 unsigned qp_max_recv_dtos_mask; /* above minus 1 */
301 unsigned min_posted_rx; /* qp_max_recv_dtos >> 2 */
275 int post_recv_buf_count; /* posted rx count */ 302 int post_recv_buf_count; /* posted rx count */
276 atomic_t post_send_buf_count; /* posted tx count */ 303 atomic_t post_send_buf_count; /* posted tx count */
277 char name[ISER_OBJECT_NAME_SIZE]; 304 char name[ISER_OBJECT_NAME_SIZE];
278 struct iser_page_vec *page_vec; /* represents SG to fmr maps*
279 * maps serialized as tx is*/
280 struct list_head conn_list; /* entry in ig conn list */ 305 struct list_head conn_list; /* entry in ig conn list */
281 306
282 char *login_buf; 307 char *login_buf;
@@ -285,6 +310,17 @@ struct iser_conn {
285 unsigned int rx_desc_head; 310 unsigned int rx_desc_head;
286 struct iser_rx_desc *rx_descs; 311 struct iser_rx_desc *rx_descs;
287 struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX]; 312 struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX];
313 union {
314 struct {
315 struct ib_fmr_pool *pool; /* pool of IB FMRs */
316 struct iser_page_vec *page_vec; /* represents SG to fmr maps*
317 * maps serialized as tx is*/
318 } fmr;
319 struct {
320 struct list_head pool;
321 int pool_size;
322 } frwr;
323 } fastreg;
288}; 324};
289 325
290struct iscsi_iser_conn { 326struct iscsi_iser_conn {
@@ -368,8 +404,10 @@ void iser_free_rx_descriptors(struct iser_conn *ib_conn);
368void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *task, 404void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *task,
369 enum iser_data_dir cmd_dir); 405 enum iser_data_dir cmd_dir);
370 406
371int iser_reg_rdma_mem(struct iscsi_iser_task *task, 407int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *task,
372 enum iser_data_dir cmd_dir); 408 enum iser_data_dir cmd_dir);
409int iser_reg_rdma_mem_frwr(struct iscsi_iser_task *task,
410 enum iser_data_dir cmd_dir);
373 411
374int iser_connect(struct iser_conn *ib_conn, 412int iser_connect(struct iser_conn *ib_conn,
375 struct sockaddr_in *src_addr, 413 struct sockaddr_in *src_addr,
@@ -380,7 +418,10 @@ int iser_reg_page_vec(struct iser_conn *ib_conn,
380 struct iser_page_vec *page_vec, 418 struct iser_page_vec *page_vec,
381 struct iser_mem_reg *mem_reg); 419 struct iser_mem_reg *mem_reg);
382 420
383void iser_unreg_mem(struct iser_mem_reg *mem_reg); 421void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
422 enum iser_data_dir cmd_dir);
423void iser_unreg_mem_frwr(struct iscsi_iser_task *iser_task,
424 enum iser_data_dir cmd_dir);
384 425
385int iser_post_recvl(struct iser_conn *ib_conn); 426int iser_post_recvl(struct iser_conn *ib_conn);
386int iser_post_recvm(struct iser_conn *ib_conn, int count); 427int iser_post_recvm(struct iser_conn *ib_conn, int count);
@@ -394,5 +435,9 @@ int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
394void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task); 435void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task);
395int iser_initialize_task_headers(struct iscsi_task *task, 436int iser_initialize_task_headers(struct iscsi_task *task,
396 struct iser_tx_desc *tx_desc); 437 struct iser_tx_desc *tx_desc);
397int iser_alloc_rx_descriptors(struct iser_conn *ib_conn); 438int iser_alloc_rx_descriptors(struct iser_conn *ib_conn, struct iscsi_session *session);
439int iser_create_fmr_pool(struct iser_conn *ib_conn, unsigned cmds_max);
440void iser_free_fmr_pool(struct iser_conn *ib_conn);
441int iser_create_frwr_pool(struct iser_conn *ib_conn, unsigned cmds_max);
442void iser_free_frwr_pool(struct iser_conn *ib_conn);
398#endif 443#endif
diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c
index b6d81a86c976..5f01da99ad66 100644
--- a/drivers/infiniband/ulp/iser/iser_initiator.c
+++ b/drivers/infiniband/ulp/iser/iser_initiator.c
@@ -49,6 +49,7 @@ static int iser_prepare_read_cmd(struct iscsi_task *task,
49 49
50{ 50{
51 struct iscsi_iser_task *iser_task = task->dd_data; 51 struct iscsi_iser_task *iser_task = task->dd_data;
52 struct iser_device *device = iser_task->iser_conn->ib_conn->device;
52 struct iser_regd_buf *regd_buf; 53 struct iser_regd_buf *regd_buf;
53 int err; 54 int err;
54 struct iser_hdr *hdr = &iser_task->desc.iser_header; 55 struct iser_hdr *hdr = &iser_task->desc.iser_header;
@@ -69,7 +70,7 @@ static int iser_prepare_read_cmd(struct iscsi_task *task,
69 return -EINVAL; 70 return -EINVAL;
70 } 71 }
71 72
72 err = iser_reg_rdma_mem(iser_task,ISER_DIR_IN); 73 err = device->iser_reg_rdma_mem(iser_task, ISER_DIR_IN);
73 if (err) { 74 if (err) {
74 iser_err("Failed to set up Data-IN RDMA\n"); 75 iser_err("Failed to set up Data-IN RDMA\n");
75 return err; 76 return err;
@@ -98,6 +99,7 @@ iser_prepare_write_cmd(struct iscsi_task *task,
98 unsigned int edtl) 99 unsigned int edtl)
99{ 100{
100 struct iscsi_iser_task *iser_task = task->dd_data; 101 struct iscsi_iser_task *iser_task = task->dd_data;
102 struct iser_device *device = iser_task->iser_conn->ib_conn->device;
101 struct iser_regd_buf *regd_buf; 103 struct iser_regd_buf *regd_buf;
102 int err; 104 int err;
103 struct iser_hdr *hdr = &iser_task->desc.iser_header; 105 struct iser_hdr *hdr = &iser_task->desc.iser_header;
@@ -119,7 +121,7 @@ iser_prepare_write_cmd(struct iscsi_task *task,
119 return -EINVAL; 121 return -EINVAL;
120 } 122 }
121 123
122 err = iser_reg_rdma_mem(iser_task,ISER_DIR_OUT); 124 err = device->iser_reg_rdma_mem(iser_task, ISER_DIR_OUT);
123 if (err != 0) { 125 if (err != 0) {
124 iser_err("Failed to register write cmd RDMA mem\n"); 126 iser_err("Failed to register write cmd RDMA mem\n");
125 return err; 127 return err;
@@ -170,8 +172,78 @@ static void iser_create_send_desc(struct iser_conn *ib_conn,
170 } 172 }
171} 173}
172 174
175static void iser_free_login_buf(struct iser_conn *ib_conn)
176{
177 if (!ib_conn->login_buf)
178 return;
179
180 if (ib_conn->login_req_dma)
181 ib_dma_unmap_single(ib_conn->device->ib_device,
182 ib_conn->login_req_dma,
183 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
184
185 if (ib_conn->login_resp_dma)
186 ib_dma_unmap_single(ib_conn->device->ib_device,
187 ib_conn->login_resp_dma,
188 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
189
190 kfree(ib_conn->login_buf);
191
192 /* make sure we never redo any unmapping */
193 ib_conn->login_req_dma = 0;
194 ib_conn->login_resp_dma = 0;
195 ib_conn->login_buf = NULL;
196}
197
198static int iser_alloc_login_buf(struct iser_conn *ib_conn)
199{
200 struct iser_device *device;
201 int req_err, resp_err;
202
203 BUG_ON(ib_conn->device == NULL);
204
205 device = ib_conn->device;
206
207 ib_conn->login_buf = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
208 ISER_RX_LOGIN_SIZE, GFP_KERNEL);
209 if (!ib_conn->login_buf)
210 goto out_err;
211
212 ib_conn->login_req_buf = ib_conn->login_buf;
213 ib_conn->login_resp_buf = ib_conn->login_buf +
214 ISCSI_DEF_MAX_RECV_SEG_LEN;
215
216 ib_conn->login_req_dma = ib_dma_map_single(ib_conn->device->ib_device,
217 (void *)ib_conn->login_req_buf,
218 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
219
220 ib_conn->login_resp_dma = ib_dma_map_single(ib_conn->device->ib_device,
221 (void *)ib_conn->login_resp_buf,
222 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
223
224 req_err = ib_dma_mapping_error(device->ib_device,
225 ib_conn->login_req_dma);
226 resp_err = ib_dma_mapping_error(device->ib_device,
227 ib_conn->login_resp_dma);
173 228
174int iser_alloc_rx_descriptors(struct iser_conn *ib_conn) 229 if (req_err || resp_err) {
230 if (req_err)
231 ib_conn->login_req_dma = 0;
232 if (resp_err)
233 ib_conn->login_resp_dma = 0;
234 goto free_login_buf;
235 }
236 return 0;
237
238free_login_buf:
239 iser_free_login_buf(ib_conn);
240
241out_err:
242 iser_err("unable to alloc or map login buf\n");
243 return -ENOMEM;
244}
245
246int iser_alloc_rx_descriptors(struct iser_conn *ib_conn, struct iscsi_session *session)
175{ 247{
176 int i, j; 248 int i, j;
177 u64 dma_addr; 249 u64 dma_addr;
@@ -179,14 +251,24 @@ int iser_alloc_rx_descriptors(struct iser_conn *ib_conn)
179 struct ib_sge *rx_sg; 251 struct ib_sge *rx_sg;
180 struct iser_device *device = ib_conn->device; 252 struct iser_device *device = ib_conn->device;
181 253
182 ib_conn->rx_descs = kmalloc(ISER_QP_MAX_RECV_DTOS * 254 ib_conn->qp_max_recv_dtos = session->cmds_max;
255 ib_conn->qp_max_recv_dtos_mask = session->cmds_max - 1; /* cmds_max is 2^N */
256 ib_conn->min_posted_rx = ib_conn->qp_max_recv_dtos >> 2;
257
258 if (device->iser_alloc_rdma_reg_res(ib_conn, session->scsi_cmds_max))
259 goto create_rdma_reg_res_failed;
260
261 if (iser_alloc_login_buf(ib_conn))
262 goto alloc_login_buf_fail;
263
264 ib_conn->rx_descs = kmalloc(session->cmds_max *
183 sizeof(struct iser_rx_desc), GFP_KERNEL); 265 sizeof(struct iser_rx_desc), GFP_KERNEL);
184 if (!ib_conn->rx_descs) 266 if (!ib_conn->rx_descs)
185 goto rx_desc_alloc_fail; 267 goto rx_desc_alloc_fail;
186 268
187 rx_desc = ib_conn->rx_descs; 269 rx_desc = ib_conn->rx_descs;
188 270
189 for (i = 0; i < ISER_QP_MAX_RECV_DTOS; i++, rx_desc++) { 271 for (i = 0; i < ib_conn->qp_max_recv_dtos; i++, rx_desc++) {
190 dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc, 272 dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc,
191 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 273 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
192 if (ib_dma_mapping_error(device->ib_device, dma_addr)) 274 if (ib_dma_mapping_error(device->ib_device, dma_addr))
@@ -207,10 +289,14 @@ rx_desc_dma_map_failed:
207 rx_desc = ib_conn->rx_descs; 289 rx_desc = ib_conn->rx_descs;
208 for (j = 0; j < i; j++, rx_desc++) 290 for (j = 0; j < i; j++, rx_desc++)
209 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 291 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
210 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 292 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
211 kfree(ib_conn->rx_descs); 293 kfree(ib_conn->rx_descs);
212 ib_conn->rx_descs = NULL; 294 ib_conn->rx_descs = NULL;
213rx_desc_alloc_fail: 295rx_desc_alloc_fail:
296 iser_free_login_buf(ib_conn);
297alloc_login_buf_fail:
298 device->iser_free_rdma_reg_res(ib_conn);
299create_rdma_reg_res_failed:
214 iser_err("failed allocating rx descriptors / data buffers\n"); 300 iser_err("failed allocating rx descriptors / data buffers\n");
215 return -ENOMEM; 301 return -ENOMEM;
216} 302}
@@ -222,13 +308,21 @@ void iser_free_rx_descriptors(struct iser_conn *ib_conn)
222 struct iser_device *device = ib_conn->device; 308 struct iser_device *device = ib_conn->device;
223 309
224 if (!ib_conn->rx_descs) 310 if (!ib_conn->rx_descs)
225 return; 311 goto free_login_buf;
312
313 if (device->iser_free_rdma_reg_res)
314 device->iser_free_rdma_reg_res(ib_conn);
226 315
227 rx_desc = ib_conn->rx_descs; 316 rx_desc = ib_conn->rx_descs;
228 for (i = 0; i < ISER_QP_MAX_RECV_DTOS; i++, rx_desc++) 317 for (i = 0; i < ib_conn->qp_max_recv_dtos; i++, rx_desc++)
229 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 318 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
230 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 319 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
231 kfree(ib_conn->rx_descs); 320 kfree(ib_conn->rx_descs);
321 /* make sure we never redo any unmapping */
322 ib_conn->rx_descs = NULL;
323
324free_login_buf:
325 iser_free_login_buf(ib_conn);
232} 326}
233 327
234static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req) 328static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req)
@@ -248,9 +342,10 @@ static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req)
248 WARN_ON(iser_conn->ib_conn->post_recv_buf_count != 1); 342 WARN_ON(iser_conn->ib_conn->post_recv_buf_count != 1);
249 WARN_ON(atomic_read(&iser_conn->ib_conn->post_send_buf_count) != 0); 343 WARN_ON(atomic_read(&iser_conn->ib_conn->post_send_buf_count) != 0);
250 344
251 iser_dbg("Initially post: %d\n", ISER_MIN_POSTED_RX); 345 iser_dbg("Initially post: %d\n", iser_conn->ib_conn->min_posted_rx);
252 /* Initial post receive buffers */ 346 /* Initial post receive buffers */
253 if (iser_post_recvm(iser_conn->ib_conn, ISER_MIN_POSTED_RX)) 347 if (iser_post_recvm(iser_conn->ib_conn,
348 iser_conn->ib_conn->min_posted_rx))
254 return -ENOMEM; 349 return -ENOMEM;
255 350
256 return 0; 351 return 0;
@@ -487,9 +582,9 @@ void iser_rcv_completion(struct iser_rx_desc *rx_desc,
487 return; 582 return;
488 583
489 outstanding = ib_conn->post_recv_buf_count; 584 outstanding = ib_conn->post_recv_buf_count;
490 if (outstanding + ISER_MIN_POSTED_RX <= ISER_QP_MAX_RECV_DTOS) { 585 if (outstanding + ib_conn->min_posted_rx <= ib_conn->qp_max_recv_dtos) {
491 count = min(ISER_QP_MAX_RECV_DTOS - outstanding, 586 count = min(ib_conn->qp_max_recv_dtos - outstanding,
492 ISER_MIN_POSTED_RX); 587 ib_conn->min_posted_rx);
493 err = iser_post_recvm(ib_conn, count); 588 err = iser_post_recvm(ib_conn, count);
494 if (err) 589 if (err)
495 iser_err("posting %d rx bufs err %d\n", count, err); 590 iser_err("posting %d rx bufs err %d\n", count, err);
@@ -538,8 +633,8 @@ void iser_task_rdma_init(struct iscsi_iser_task *iser_task)
538 633
539void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task) 634void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
540{ 635{
636 struct iser_device *device = iser_task->iser_conn->ib_conn->device;
541 int is_rdma_aligned = 1; 637 int is_rdma_aligned = 1;
542 struct iser_regd_buf *regd;
543 638
544 /* if we were reading, copy back to unaligned sglist, 639 /* if we were reading, copy back to unaligned sglist,
545 * anyway dma_unmap and free the copy 640 * anyway dma_unmap and free the copy
@@ -553,17 +648,11 @@ void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
553 iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_OUT); 648 iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_OUT);
554 } 649 }
555 650
556 if (iser_task->dir[ISER_DIR_IN]) { 651 if (iser_task->dir[ISER_DIR_IN])
557 regd = &iser_task->rdma_regd[ISER_DIR_IN]; 652 device->iser_unreg_rdma_mem(iser_task, ISER_DIR_IN);
558 if (regd->reg.is_fmr)
559 iser_unreg_mem(&regd->reg);
560 }
561 653
562 if (iser_task->dir[ISER_DIR_OUT]) { 654 if (iser_task->dir[ISER_DIR_OUT])
563 regd = &iser_task->rdma_regd[ISER_DIR_OUT]; 655 device->iser_unreg_rdma_mem(iser_task, ISER_DIR_OUT);
564 if (regd->reg.is_fmr)
565 iser_unreg_mem(&regd->reg);
566 }
567 656
568 /* if the data was unaligned, it was already unmapped and then copied */ 657 /* if the data was unaligned, it was already unmapped and then copied */
569 if (is_rdma_aligned) 658 if (is_rdma_aligned)
diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c
index 7827baf455a1..1ce0c97d2ccb 100644
--- a/drivers/infiniband/ulp/iser/iser_memory.c
+++ b/drivers/infiniband/ulp/iser/iser_memory.c
@@ -170,8 +170,8 @@ void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
170 */ 170 */
171 171
172static int iser_sg_to_page_vec(struct iser_data_buf *data, 172static int iser_sg_to_page_vec(struct iser_data_buf *data,
173 struct iser_page_vec *page_vec, 173 struct ib_device *ibdev, u64 *pages,
174 struct ib_device *ibdev) 174 int *offset, int *data_size)
175{ 175{
176 struct scatterlist *sg, *sgl = (struct scatterlist *)data->buf; 176 struct scatterlist *sg, *sgl = (struct scatterlist *)data->buf;
177 u64 start_addr, end_addr, page, chunk_start = 0; 177 u64 start_addr, end_addr, page, chunk_start = 0;
@@ -180,7 +180,7 @@ static int iser_sg_to_page_vec(struct iser_data_buf *data,
180 int i, new_chunk, cur_page, last_ent = data->dma_nents - 1; 180 int i, new_chunk, cur_page, last_ent = data->dma_nents - 1;
181 181
182 /* compute the offset of first element */ 182 /* compute the offset of first element */
183 page_vec->offset = (u64) sgl[0].offset & ~MASK_4K; 183 *offset = (u64) sgl[0].offset & ~MASK_4K;
184 184
185 new_chunk = 1; 185 new_chunk = 1;
186 cur_page = 0; 186 cur_page = 0;
@@ -204,13 +204,14 @@ static int iser_sg_to_page_vec(struct iser_data_buf *data,
204 which might be unaligned */ 204 which might be unaligned */
205 page = chunk_start & MASK_4K; 205 page = chunk_start & MASK_4K;
206 do { 206 do {
207 page_vec->pages[cur_page++] = page; 207 pages[cur_page++] = page;
208 page += SIZE_4K; 208 page += SIZE_4K;
209 } while (page < end_addr); 209 } while (page < end_addr);
210 } 210 }
211 211
212 page_vec->data_size = total_sz; 212 *data_size = total_sz;
213 iser_dbg("page_vec->data_size:%d cur_page %d\n", page_vec->data_size,cur_page); 213 iser_dbg("page_vec->data_size:%d cur_page %d\n",
214 *data_size, cur_page);
214 return cur_page; 215 return cur_page;
215} 216}
216 217
@@ -267,11 +268,8 @@ static void iser_data_buf_dump(struct iser_data_buf *data,
267 struct scatterlist *sg; 268 struct scatterlist *sg;
268 int i; 269 int i;
269 270
270 if (iser_debug_level == 0)
271 return;
272
273 for_each_sg(sgl, sg, data->dma_nents, i) 271 for_each_sg(sgl, sg, data->dma_nents, i)
274 iser_warn("sg[%d] dma_addr:0x%lX page:0x%p " 272 iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p "
275 "off:0x%x sz:0x%x dma_len:0x%x\n", 273 "off:0x%x sz:0x%x dma_len:0x%x\n",
276 i, (unsigned long)ib_sg_dma_address(ibdev, sg), 274 i, (unsigned long)ib_sg_dma_address(ibdev, sg),
277 sg_page(sg), sg->offset, 275 sg_page(sg), sg->offset,
@@ -298,8 +296,10 @@ static void iser_page_vec_build(struct iser_data_buf *data,
298 page_vec->offset = 0; 296 page_vec->offset = 0;
299 297
300 iser_dbg("Translating sg sz: %d\n", data->dma_nents); 298 iser_dbg("Translating sg sz: %d\n", data->dma_nents);
301 page_vec_len = iser_sg_to_page_vec(data, page_vec, ibdev); 299 page_vec_len = iser_sg_to_page_vec(data, ibdev, page_vec->pages,
302 iser_dbg("sg len %d page_vec_len %d\n", data->dma_nents,page_vec_len); 300 &page_vec->offset,
301 &page_vec->data_size);
302 iser_dbg("sg len %d page_vec_len %d\n", data->dma_nents, page_vec_len);
303 303
304 page_vec->length = page_vec_len; 304 page_vec->length = page_vec_len;
305 305
@@ -347,16 +347,41 @@ void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task)
347 } 347 }
348} 348}
349 349
350static int fall_to_bounce_buf(struct iscsi_iser_task *iser_task,
351 struct ib_device *ibdev,
352 enum iser_data_dir cmd_dir,
353 int aligned_len)
354{
355 struct iscsi_conn *iscsi_conn = iser_task->iser_conn->iscsi_conn;
356 struct iser_data_buf *mem = &iser_task->data[cmd_dir];
357
358 iscsi_conn->fmr_unalign_cnt++;
359 iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n",
360 aligned_len, mem->size);
361
362 if (iser_debug_level > 0)
363 iser_data_buf_dump(mem, ibdev);
364
365 /* unmap the command data before accessing it */
366 iser_dma_unmap_task_data(iser_task);
367
368 /* allocate copy buf, if we are writing, copy the */
369 /* unaligned scatterlist, dma map the copy */
370 if (iser_start_rdma_unaligned_sg(iser_task, cmd_dir) != 0)
371 return -ENOMEM;
372
373 return 0;
374}
375
350/** 376/**
351 * iser_reg_rdma_mem - Registers memory intended for RDMA, 377 * iser_reg_rdma_mem_fmr - Registers memory intended for RDMA,
352 * obtaining rkey and va 378 * using FMR (if possible) obtaining rkey and va
353 * 379 *
354 * returns 0 on success, errno code on failure 380 * returns 0 on success, errno code on failure
355 */ 381 */
356int iser_reg_rdma_mem(struct iscsi_iser_task *iser_task, 382int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task,
357 enum iser_data_dir cmd_dir) 383 enum iser_data_dir cmd_dir)
358{ 384{
359 struct iscsi_conn *iscsi_conn = iser_task->iser_conn->iscsi_conn;
360 struct iser_conn *ib_conn = iser_task->iser_conn->ib_conn; 385 struct iser_conn *ib_conn = iser_task->iser_conn->ib_conn;
361 struct iser_device *device = ib_conn->device; 386 struct iser_device *device = ib_conn->device;
362 struct ib_device *ibdev = device->ib_device; 387 struct ib_device *ibdev = device->ib_device;
@@ -370,20 +395,13 @@ int iser_reg_rdma_mem(struct iscsi_iser_task *iser_task,
370 regd_buf = &iser_task->rdma_regd[cmd_dir]; 395 regd_buf = &iser_task->rdma_regd[cmd_dir];
371 396
372 aligned_len = iser_data_buf_aligned_len(mem, ibdev); 397 aligned_len = iser_data_buf_aligned_len(mem, ibdev);
373 if (aligned_len != mem->dma_nents || 398 if (aligned_len != mem->dma_nents) {
374 (!ib_conn->fmr_pool && mem->dma_nents > 1)) { 399 err = fall_to_bounce_buf(iser_task, ibdev,
375 iscsi_conn->fmr_unalign_cnt++; 400 cmd_dir, aligned_len);
376 iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n", 401 if (err) {
377 aligned_len, mem->size); 402 iser_err("failed to allocate bounce buffer\n");
378 iser_data_buf_dump(mem, ibdev); 403 return err;
379 404 }
380 /* unmap the command data before accessing it */
381 iser_dma_unmap_task_data(iser_task);
382
383 /* allocate copy buf, if we are writing, copy the */
384 /* unaligned scatterlist, dma map the copy */
385 if (iser_start_rdma_unaligned_sg(iser_task, cmd_dir) != 0)
386 return -ENOMEM;
387 mem = &iser_task->data_copy[cmd_dir]; 405 mem = &iser_task->data_copy[cmd_dir];
388 } 406 }
389 407
@@ -395,7 +413,7 @@ int iser_reg_rdma_mem(struct iscsi_iser_task *iser_task,
395 regd_buf->reg.rkey = device->mr->rkey; 413 regd_buf->reg.rkey = device->mr->rkey;
396 regd_buf->reg.len = ib_sg_dma_len(ibdev, &sg[0]); 414 regd_buf->reg.len = ib_sg_dma_len(ibdev, &sg[0]);
397 regd_buf->reg.va = ib_sg_dma_address(ibdev, &sg[0]); 415 regd_buf->reg.va = ib_sg_dma_address(ibdev, &sg[0]);
398 regd_buf->reg.is_fmr = 0; 416 regd_buf->reg.is_mr = 0;
399 417
400 iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X " 418 iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X "
401 "va: 0x%08lX sz: %ld]\n", 419 "va: 0x%08lX sz: %ld]\n",
@@ -404,22 +422,159 @@ int iser_reg_rdma_mem(struct iscsi_iser_task *iser_task,
404 (unsigned long)regd_buf->reg.va, 422 (unsigned long)regd_buf->reg.va,
405 (unsigned long)regd_buf->reg.len); 423 (unsigned long)regd_buf->reg.len);
406 } else { /* use FMR for multiple dma entries */ 424 } else { /* use FMR for multiple dma entries */
407 iser_page_vec_build(mem, ib_conn->page_vec, ibdev); 425 iser_page_vec_build(mem, ib_conn->fastreg.fmr.page_vec, ibdev);
408 err = iser_reg_page_vec(ib_conn, ib_conn->page_vec, &regd_buf->reg); 426 err = iser_reg_page_vec(ib_conn, ib_conn->fastreg.fmr.page_vec,
427 &regd_buf->reg);
409 if (err && err != -EAGAIN) { 428 if (err && err != -EAGAIN) {
410 iser_data_buf_dump(mem, ibdev); 429 iser_data_buf_dump(mem, ibdev);
411 iser_err("mem->dma_nents = %d (dlength = 0x%x)\n", 430 iser_err("mem->dma_nents = %d (dlength = 0x%x)\n",
412 mem->dma_nents, 431 mem->dma_nents,
413 ntoh24(iser_task->desc.iscsi_header.dlength)); 432 ntoh24(iser_task->desc.iscsi_header.dlength));
414 iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n", 433 iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n",
415 ib_conn->page_vec->data_size, ib_conn->page_vec->length, 434 ib_conn->fastreg.fmr.page_vec->data_size,
416 ib_conn->page_vec->offset); 435 ib_conn->fastreg.fmr.page_vec->length,
417 for (i=0 ; i<ib_conn->page_vec->length ; i++) 436 ib_conn->fastreg.fmr.page_vec->offset);
437 for (i = 0; i < ib_conn->fastreg.fmr.page_vec->length; i++)
418 iser_err("page_vec[%d] = 0x%llx\n", i, 438 iser_err("page_vec[%d] = 0x%llx\n", i,
419 (unsigned long long) ib_conn->page_vec->pages[i]); 439 (unsigned long long) ib_conn->fastreg.fmr.page_vec->pages[i]);
420 } 440 }
421 if (err) 441 if (err)
422 return err; 442 return err;
423 } 443 }
424 return 0; 444 return 0;
425} 445}
446
447static int iser_fast_reg_mr(struct fast_reg_descriptor *desc,
448 struct iser_conn *ib_conn,
449 struct iser_regd_buf *regd_buf,
450 u32 offset, unsigned int data_size,
451 unsigned int page_list_len)
452{
453 struct ib_send_wr fastreg_wr, inv_wr;
454 struct ib_send_wr *bad_wr, *wr = NULL;
455 u8 key;
456 int ret;
457
458 if (!desc->valid) {
459 memset(&inv_wr, 0, sizeof(inv_wr));
460 inv_wr.opcode = IB_WR_LOCAL_INV;
461 inv_wr.send_flags = IB_SEND_SIGNALED;
462 inv_wr.ex.invalidate_rkey = desc->data_mr->rkey;
463 wr = &inv_wr;
464 /* Bump the key */
465 key = (u8)(desc->data_mr->rkey & 0x000000FF);
466 ib_update_fast_reg_key(desc->data_mr, ++key);
467 }
468
469 /* Prepare FASTREG WR */
470 memset(&fastreg_wr, 0, sizeof(fastreg_wr));
471 fastreg_wr.opcode = IB_WR_FAST_REG_MR;
472 fastreg_wr.send_flags = IB_SEND_SIGNALED;
473 fastreg_wr.wr.fast_reg.iova_start = desc->data_frpl->page_list[0] + offset;
474 fastreg_wr.wr.fast_reg.page_list = desc->data_frpl;
475 fastreg_wr.wr.fast_reg.page_list_len = page_list_len;
476 fastreg_wr.wr.fast_reg.page_shift = SHIFT_4K;
477 fastreg_wr.wr.fast_reg.length = data_size;
478 fastreg_wr.wr.fast_reg.rkey = desc->data_mr->rkey;
479 fastreg_wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE |
480 IB_ACCESS_REMOTE_WRITE |
481 IB_ACCESS_REMOTE_READ);
482
483 if (!wr) {
484 wr = &fastreg_wr;
485 atomic_inc(&ib_conn->post_send_buf_count);
486 } else {
487 wr->next = &fastreg_wr;
488 atomic_add(2, &ib_conn->post_send_buf_count);
489 }
490
491 ret = ib_post_send(ib_conn->qp, wr, &bad_wr);
492 if (ret) {
493 if (bad_wr->next)
494 atomic_sub(2, &ib_conn->post_send_buf_count);
495 else
496 atomic_dec(&ib_conn->post_send_buf_count);
497 iser_err("fast registration failed, ret:%d\n", ret);
498 return ret;
499 }
500 desc->valid = false;
501
502 regd_buf->reg.mem_h = desc;
503 regd_buf->reg.lkey = desc->data_mr->lkey;
504 regd_buf->reg.rkey = desc->data_mr->rkey;
505 regd_buf->reg.va = desc->data_frpl->page_list[0] + offset;
506 regd_buf->reg.len = data_size;
507 regd_buf->reg.is_mr = 1;
508
509 return ret;
510}
511
512/**
513 * iser_reg_rdma_mem_frwr - Registers memory intended for RDMA,
514 * using Fast Registration WR (if possible) obtaining rkey and va
515 *
516 * returns 0 on success, errno code on failure
517 */
518int iser_reg_rdma_mem_frwr(struct iscsi_iser_task *iser_task,
519 enum iser_data_dir cmd_dir)
520{
521 struct iser_conn *ib_conn = iser_task->iser_conn->ib_conn;
522 struct iser_device *device = ib_conn->device;
523 struct ib_device *ibdev = device->ib_device;
524 struct iser_data_buf *mem = &iser_task->data[cmd_dir];
525 struct iser_regd_buf *regd_buf = &iser_task->rdma_regd[cmd_dir];
526 struct fast_reg_descriptor *desc;
527 unsigned int data_size, page_list_len;
528 int err, aligned_len;
529 unsigned long flags;
530 u32 offset;
531
532 aligned_len = iser_data_buf_aligned_len(mem, ibdev);
533 if (aligned_len != mem->dma_nents) {
534 err = fall_to_bounce_buf(iser_task, ibdev,
535 cmd_dir, aligned_len);
536 if (err) {
537 iser_err("failed to allocate bounce buffer\n");
538 return err;
539 }
540 mem = &iser_task->data_copy[cmd_dir];
541 }
542
543 /* if there a single dma entry, dma mr suffices */
544 if (mem->dma_nents == 1) {
545 struct scatterlist *sg = (struct scatterlist *)mem->buf;
546
547 regd_buf->reg.lkey = device->mr->lkey;
548 regd_buf->reg.rkey = device->mr->rkey;
549 regd_buf->reg.len = ib_sg_dma_len(ibdev, &sg[0]);
550 regd_buf->reg.va = ib_sg_dma_address(ibdev, &sg[0]);
551 regd_buf->reg.is_mr = 0;
552 } else {
553 spin_lock_irqsave(&ib_conn->lock, flags);
554 desc = list_first_entry(&ib_conn->fastreg.frwr.pool,
555 struct fast_reg_descriptor, list);
556 list_del(&desc->list);
557 spin_unlock_irqrestore(&ib_conn->lock, flags);
558 page_list_len = iser_sg_to_page_vec(mem, device->ib_device,
559 desc->data_frpl->page_list,
560 &offset, &data_size);
561
562 if (page_list_len * SIZE_4K < data_size) {
563 iser_err("fast reg page_list too short to hold this SG\n");
564 err = -EINVAL;
565 goto err_reg;
566 }
567
568 err = iser_fast_reg_mr(desc, ib_conn, regd_buf,
569 offset, data_size, page_list_len);
570 if (err)
571 goto err_reg;
572 }
573
574 return 0;
575err_reg:
576 spin_lock_irqsave(&ib_conn->lock, flags);
577 list_add_tail(&desc->list, &ib_conn->fastreg.frwr.pool);
578 spin_unlock_irqrestore(&ib_conn->lock, flags);
579 return err;
580}
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 2c4941d0656b..afe95674008b 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -73,6 +73,36 @@ static int iser_create_device_ib_res(struct iser_device *device)
73{ 73{
74 int i, j; 74 int i, j;
75 struct iser_cq_desc *cq_desc; 75 struct iser_cq_desc *cq_desc;
76 struct ib_device_attr *dev_attr;
77
78 dev_attr = kmalloc(sizeof(*dev_attr), GFP_KERNEL);
79 if (!dev_attr)
80 return -ENOMEM;
81
82 if (ib_query_device(device->ib_device, dev_attr)) {
83 pr_warn("Query device failed for %s\n", device->ib_device->name);
84 goto dev_attr_err;
85 }
86
87 /* Assign function handles - based on FMR support */
88 if (device->ib_device->alloc_fmr && device->ib_device->dealloc_fmr &&
89 device->ib_device->map_phys_fmr && device->ib_device->unmap_fmr) {
90 iser_info("FMR supported, using FMR for registration\n");
91 device->iser_alloc_rdma_reg_res = iser_create_fmr_pool;
92 device->iser_free_rdma_reg_res = iser_free_fmr_pool;
93 device->iser_reg_rdma_mem = iser_reg_rdma_mem_fmr;
94 device->iser_unreg_rdma_mem = iser_unreg_mem_fmr;
95 } else
96 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) {
97 iser_info("FRWR supported, using FRWR for registration\n");
98 device->iser_alloc_rdma_reg_res = iser_create_frwr_pool;
99 device->iser_free_rdma_reg_res = iser_free_frwr_pool;
100 device->iser_reg_rdma_mem = iser_reg_rdma_mem_frwr;
101 device->iser_unreg_rdma_mem = iser_unreg_mem_frwr;
102 } else {
103 iser_err("IB device does not support FMRs nor FRWRs, can't register memory\n");
104 goto dev_attr_err;
105 }
76 106
77 device->cqs_used = min(ISER_MAX_CQ, device->ib_device->num_comp_vectors); 107 device->cqs_used = min(ISER_MAX_CQ, device->ib_device->num_comp_vectors);
78 iser_info("using %d CQs, device %s supports %d vectors\n", 108 iser_info("using %d CQs, device %s supports %d vectors\n",
@@ -128,6 +158,7 @@ static int iser_create_device_ib_res(struct iser_device *device)
128 if (ib_register_event_handler(&device->event_handler)) 158 if (ib_register_event_handler(&device->event_handler))
129 goto handler_err; 159 goto handler_err;
130 160
161 kfree(dev_attr);
131 return 0; 162 return 0;
132 163
133handler_err: 164handler_err:
@@ -147,6 +178,8 @@ pd_err:
147 kfree(device->cq_desc); 178 kfree(device->cq_desc);
148cq_desc_err: 179cq_desc_err:
149 iser_err("failed to allocate an IB resource\n"); 180 iser_err("failed to allocate an IB resource\n");
181dev_attr_err:
182 kfree(dev_attr);
150 return -1; 183 return -1;
151} 184}
152 185
@@ -178,56 +211,23 @@ static void iser_free_device_ib_res(struct iser_device *device)
178} 211}
179 212
180/** 213/**
181 * iser_create_ib_conn_res - Creates FMR pool and Queue-Pair (QP) 214 * iser_create_fmr_pool - Creates FMR pool and page_vector
182 * 215 *
183 * returns 0 on success, -1 on failure 216 * returns 0 on success, or errno code on failure
184 */ 217 */
185static int iser_create_ib_conn_res(struct iser_conn *ib_conn) 218int iser_create_fmr_pool(struct iser_conn *ib_conn, unsigned cmds_max)
186{ 219{
187 struct iser_device *device; 220 struct iser_device *device = ib_conn->device;
188 struct ib_qp_init_attr init_attr;
189 int req_err, resp_err, ret = -ENOMEM;
190 struct ib_fmr_pool_param params; 221 struct ib_fmr_pool_param params;
191 int index, min_index = 0; 222 int ret = -ENOMEM;
192
193 BUG_ON(ib_conn->device == NULL);
194
195 device = ib_conn->device;
196
197 ib_conn->login_buf = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
198 ISER_RX_LOGIN_SIZE, GFP_KERNEL);
199 if (!ib_conn->login_buf)
200 goto out_err;
201
202 ib_conn->login_req_buf = ib_conn->login_buf;
203 ib_conn->login_resp_buf = ib_conn->login_buf + ISCSI_DEF_MAX_RECV_SEG_LEN;
204
205 ib_conn->login_req_dma = ib_dma_map_single(ib_conn->device->ib_device,
206 (void *)ib_conn->login_req_buf,
207 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
208
209 ib_conn->login_resp_dma = ib_dma_map_single(ib_conn->device->ib_device,
210 (void *)ib_conn->login_resp_buf,
211 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
212
213 req_err = ib_dma_mapping_error(device->ib_device, ib_conn->login_req_dma);
214 resp_err = ib_dma_mapping_error(device->ib_device, ib_conn->login_resp_dma);
215
216 if (req_err || resp_err) {
217 if (req_err)
218 ib_conn->login_req_dma = 0;
219 if (resp_err)
220 ib_conn->login_resp_dma = 0;
221 goto out_err;
222 }
223 223
224 ib_conn->page_vec = kmalloc(sizeof(struct iser_page_vec) + 224 ib_conn->fastreg.fmr.page_vec = kmalloc(sizeof(struct iser_page_vec) +
225 (sizeof(u64) * (ISCSI_ISER_SG_TABLESIZE +1)), 225 (sizeof(u64)*(ISCSI_ISER_SG_TABLESIZE + 1)),
226 GFP_KERNEL); 226 GFP_KERNEL);
227 if (!ib_conn->page_vec) 227 if (!ib_conn->fastreg.fmr.page_vec)
228 goto out_err; 228 return ret;
229 229
230 ib_conn->page_vec->pages = (u64 *) (ib_conn->page_vec + 1); 230 ib_conn->fastreg.fmr.page_vec->pages = (u64 *)(ib_conn->fastreg.fmr.page_vec + 1);
231 231
232 params.page_shift = SHIFT_4K; 232 params.page_shift = SHIFT_4K;
233 /* when the first/last SG element are not start/end * 233 /* when the first/last SG element are not start/end *
@@ -235,24 +235,143 @@ static int iser_create_ib_conn_res(struct iser_conn *ib_conn)
235 params.max_pages_per_fmr = ISCSI_ISER_SG_TABLESIZE + 1; 235 params.max_pages_per_fmr = ISCSI_ISER_SG_TABLESIZE + 1;
236 /* make the pool size twice the max number of SCSI commands * 236 /* make the pool size twice the max number of SCSI commands *
237 * the ML is expected to queue, watermark for unmap at 50% */ 237 * the ML is expected to queue, watermark for unmap at 50% */
238 params.pool_size = ISCSI_DEF_XMIT_CMDS_MAX * 2; 238 params.pool_size = cmds_max * 2;
239 params.dirty_watermark = ISCSI_DEF_XMIT_CMDS_MAX; 239 params.dirty_watermark = cmds_max;
240 params.cache = 0; 240 params.cache = 0;
241 params.flush_function = NULL; 241 params.flush_function = NULL;
242 params.access = (IB_ACCESS_LOCAL_WRITE | 242 params.access = (IB_ACCESS_LOCAL_WRITE |
243 IB_ACCESS_REMOTE_WRITE | 243 IB_ACCESS_REMOTE_WRITE |
244 IB_ACCESS_REMOTE_READ); 244 IB_ACCESS_REMOTE_READ);
245 245
246 ib_conn->fmr_pool = ib_create_fmr_pool(device->pd, &params); 246 ib_conn->fastreg.fmr.pool = ib_create_fmr_pool(device->pd, &params);
247 ret = PTR_ERR(ib_conn->fmr_pool); 247 if (!IS_ERR(ib_conn->fastreg.fmr.pool))
248 if (IS_ERR(ib_conn->fmr_pool) && ret != -ENOSYS) { 248 return 0;
249 ib_conn->fmr_pool = NULL; 249
250 goto out_err; 250 /* no FMR => no need for page_vec */
251 } else if (ret == -ENOSYS) { 251 kfree(ib_conn->fastreg.fmr.page_vec);
252 ib_conn->fmr_pool = NULL; 252 ib_conn->fastreg.fmr.page_vec = NULL;
253
254 ret = PTR_ERR(ib_conn->fastreg.fmr.pool);
255 ib_conn->fastreg.fmr.pool = NULL;
256 if (ret != -ENOSYS) {
257 iser_err("FMR allocation failed, err %d\n", ret);
258 return ret;
259 } else {
253 iser_warn("FMRs are not supported, using unaligned mode\n"); 260 iser_warn("FMRs are not supported, using unaligned mode\n");
254 ret = 0; 261 return 0;
255 } 262 }
263}
264
265/**
266 * iser_free_fmr_pool - releases the FMR pool and page vec
267 */
268void iser_free_fmr_pool(struct iser_conn *ib_conn)
269{
270 iser_info("freeing conn %p fmr pool %p\n",
271 ib_conn, ib_conn->fastreg.fmr.pool);
272
273 if (ib_conn->fastreg.fmr.pool != NULL)
274 ib_destroy_fmr_pool(ib_conn->fastreg.fmr.pool);
275
276 ib_conn->fastreg.fmr.pool = NULL;
277
278 kfree(ib_conn->fastreg.fmr.page_vec);
279 ib_conn->fastreg.fmr.page_vec = NULL;
280}
281
282/**
283 * iser_create_frwr_pool - Creates pool of fast_reg descriptors
284 * for fast registration work requests.
285 * returns 0 on success, or errno code on failure
286 */
287int iser_create_frwr_pool(struct iser_conn *ib_conn, unsigned cmds_max)
288{
289 struct iser_device *device = ib_conn->device;
290 struct fast_reg_descriptor *desc;
291 int i, ret;
292
293 INIT_LIST_HEAD(&ib_conn->fastreg.frwr.pool);
294 ib_conn->fastreg.frwr.pool_size = 0;
295 for (i = 0; i < cmds_max; i++) {
296 desc = kmalloc(sizeof(*desc), GFP_KERNEL);
297 if (!desc) {
298 iser_err("Failed to allocate a new fast_reg descriptor\n");
299 ret = -ENOMEM;
300 goto err;
301 }
302
303 desc->data_frpl = ib_alloc_fast_reg_page_list(device->ib_device,
304 ISCSI_ISER_SG_TABLESIZE + 1);
305 if (IS_ERR(desc->data_frpl)) {
306 ret = PTR_ERR(desc->data_frpl);
307 iser_err("Failed to allocate ib_fast_reg_page_list err=%d\n", ret);
308 goto fast_reg_page_failure;
309 }
310
311 desc->data_mr = ib_alloc_fast_reg_mr(device->pd,
312 ISCSI_ISER_SG_TABLESIZE + 1);
313 if (IS_ERR(desc->data_mr)) {
314 ret = PTR_ERR(desc->data_mr);
315 iser_err("Failed to allocate ib_fast_reg_mr err=%d\n", ret);
316 goto fast_reg_mr_failure;
317 }
318 desc->valid = true;
319 list_add_tail(&desc->list, &ib_conn->fastreg.frwr.pool);
320 ib_conn->fastreg.frwr.pool_size++;
321 }
322
323 return 0;
324
325fast_reg_mr_failure:
326 ib_free_fast_reg_page_list(desc->data_frpl);
327fast_reg_page_failure:
328 kfree(desc);
329err:
330 iser_free_frwr_pool(ib_conn);
331 return ret;
332}
333
334/**
335 * iser_free_frwr_pool - releases the pool of fast_reg descriptors
336 */
337void iser_free_frwr_pool(struct iser_conn *ib_conn)
338{
339 struct fast_reg_descriptor *desc, *tmp;
340 int i = 0;
341
342 if (list_empty(&ib_conn->fastreg.frwr.pool))
343 return;
344
345 iser_info("freeing conn %p frwr pool\n", ib_conn);
346
347 list_for_each_entry_safe(desc, tmp, &ib_conn->fastreg.frwr.pool, list) {
348 list_del(&desc->list);
349 ib_free_fast_reg_page_list(desc->data_frpl);
350 ib_dereg_mr(desc->data_mr);
351 kfree(desc);
352 ++i;
353 }
354
355 if (i < ib_conn->fastreg.frwr.pool_size)
356 iser_warn("pool still has %d regions registered\n",
357 ib_conn->fastreg.frwr.pool_size - i);
358}
359
360/**
361 * iser_create_ib_conn_res - Queue-Pair (QP)
362 *
363 * returns 0 on success, -1 on failure
364 */
365static int iser_create_ib_conn_res(struct iser_conn *ib_conn)
366{
367 struct iser_device *device;
368 struct ib_qp_init_attr init_attr;
369 int ret = -ENOMEM;
370 int index, min_index = 0;
371
372 BUG_ON(ib_conn->device == NULL);
373
374 device = ib_conn->device;
256 375
257 memset(&init_attr, 0, sizeof init_attr); 376 memset(&init_attr, 0, sizeof init_attr);
258 377
@@ -282,9 +401,9 @@ static int iser_create_ib_conn_res(struct iser_conn *ib_conn)
282 goto out_err; 401 goto out_err;
283 402
284 ib_conn->qp = ib_conn->cma_id->qp; 403 ib_conn->qp = ib_conn->cma_id->qp;
285 iser_info("setting conn %p cma_id %p: fmr_pool %p qp %p\n", 404 iser_info("setting conn %p cma_id %p qp %p\n",
286 ib_conn, ib_conn->cma_id, 405 ib_conn, ib_conn->cma_id,
287 ib_conn->fmr_pool, ib_conn->cma_id->qp); 406 ib_conn->cma_id->qp);
288 return ret; 407 return ret;
289 408
290out_err: 409out_err:
@@ -293,7 +412,7 @@ out_err:
293} 412}
294 413
295/** 414/**
296 * releases the FMR pool and QP objects, returns 0 on success, 415 * releases the QP objects, returns 0 on success,
297 * -1 on failure 416 * -1 on failure
298 */ 417 */
299static int iser_free_ib_conn_res(struct iser_conn *ib_conn) 418static int iser_free_ib_conn_res(struct iser_conn *ib_conn)
@@ -301,13 +420,11 @@ static int iser_free_ib_conn_res(struct iser_conn *ib_conn)
301 int cq_index; 420 int cq_index;
302 BUG_ON(ib_conn == NULL); 421 BUG_ON(ib_conn == NULL);
303 422
304 iser_info("freeing conn %p cma_id %p fmr pool %p qp %p\n", 423 iser_info("freeing conn %p cma_id %p qp %p\n",
305 ib_conn, ib_conn->cma_id, 424 ib_conn, ib_conn->cma_id,
306 ib_conn->fmr_pool, ib_conn->qp); 425 ib_conn->qp);
307 426
308 /* qp is created only once both addr & route are resolved */ 427 /* qp is created only once both addr & route are resolved */
309 if (ib_conn->fmr_pool != NULL)
310 ib_destroy_fmr_pool(ib_conn->fmr_pool);
311 428
312 if (ib_conn->qp != NULL) { 429 if (ib_conn->qp != NULL) {
313 cq_index = ((struct iser_cq_desc *)ib_conn->qp->recv_cq->cq_context)->cq_index; 430 cq_index = ((struct iser_cq_desc *)ib_conn->qp->recv_cq->cq_context)->cq_index;
@@ -316,21 +433,7 @@ static int iser_free_ib_conn_res(struct iser_conn *ib_conn)
316 rdma_destroy_qp(ib_conn->cma_id); 433 rdma_destroy_qp(ib_conn->cma_id);
317 } 434 }
318 435
319 ib_conn->fmr_pool = NULL;
320 ib_conn->qp = NULL; 436 ib_conn->qp = NULL;
321 kfree(ib_conn->page_vec);
322
323 if (ib_conn->login_buf) {
324 if (ib_conn->login_req_dma)
325 ib_dma_unmap_single(ib_conn->device->ib_device,
326 ib_conn->login_req_dma,
327 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
328 if (ib_conn->login_resp_dma)
329 ib_dma_unmap_single(ib_conn->device->ib_device,
330 ib_conn->login_resp_dma,
331 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
332 kfree(ib_conn->login_buf);
333 }
334 437
335 return 0; 438 return 0;
336} 439}
@@ -694,7 +797,7 @@ int iser_reg_page_vec(struct iser_conn *ib_conn,
694 page_list = page_vec->pages; 797 page_list = page_vec->pages;
695 io_addr = page_list[0]; 798 io_addr = page_list[0];
696 799
697 mem = ib_fmr_pool_map_phys(ib_conn->fmr_pool, 800 mem = ib_fmr_pool_map_phys(ib_conn->fastreg.fmr.pool,
698 page_list, 801 page_list,
699 page_vec->length, 802 page_vec->length,
700 io_addr); 803 io_addr);
@@ -709,7 +812,7 @@ int iser_reg_page_vec(struct iser_conn *ib_conn,
709 mem_reg->rkey = mem->fmr->rkey; 812 mem_reg->rkey = mem->fmr->rkey;
710 mem_reg->len = page_vec->length * SIZE_4K; 813 mem_reg->len = page_vec->length * SIZE_4K;
711 mem_reg->va = io_addr; 814 mem_reg->va = io_addr;
712 mem_reg->is_fmr = 1; 815 mem_reg->is_mr = 1;
713 mem_reg->mem_h = (void *)mem; 816 mem_reg->mem_h = (void *)mem;
714 817
715 mem_reg->va += page_vec->offset; 818 mem_reg->va += page_vec->offset;
@@ -727,12 +830,18 @@ int iser_reg_page_vec(struct iser_conn *ib_conn,
727} 830}
728 831
729/** 832/**
730 * Unregister (previosuly registered) memory. 833 * Unregister (previosuly registered using FMR) memory.
834 * If memory is non-FMR does nothing.
731 */ 835 */
732void iser_unreg_mem(struct iser_mem_reg *reg) 836void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
837 enum iser_data_dir cmd_dir)
733{ 838{
839 struct iser_mem_reg *reg = &iser_task->rdma_regd[cmd_dir].reg;
734 int ret; 840 int ret;
735 841
842 if (!reg->is_mr)
843 return;
844
736 iser_dbg("PHYSICAL Mem.Unregister mem_h %p\n",reg->mem_h); 845 iser_dbg("PHYSICAL Mem.Unregister mem_h %p\n",reg->mem_h);
737 846
738 ret = ib_fmr_pool_unmap((struct ib_pool_fmr *)reg->mem_h); 847 ret = ib_fmr_pool_unmap((struct ib_pool_fmr *)reg->mem_h);
@@ -742,6 +851,23 @@ void iser_unreg_mem(struct iser_mem_reg *reg)
742 reg->mem_h = NULL; 851 reg->mem_h = NULL;
743} 852}
744 853
854void iser_unreg_mem_frwr(struct iscsi_iser_task *iser_task,
855 enum iser_data_dir cmd_dir)
856{
857 struct iser_mem_reg *reg = &iser_task->rdma_regd[cmd_dir].reg;
858 struct iser_conn *ib_conn = iser_task->iser_conn->ib_conn;
859 struct fast_reg_descriptor *desc = reg->mem_h;
860
861 if (!reg->is_mr)
862 return;
863
864 reg->mem_h = NULL;
865 reg->is_mr = 0;
866 spin_lock_bh(&ib_conn->lock);
867 list_add_tail(&desc->list, &ib_conn->fastreg.frwr.pool);
868 spin_unlock_bh(&ib_conn->lock);
869}
870
745int iser_post_recvl(struct iser_conn *ib_conn) 871int iser_post_recvl(struct iser_conn *ib_conn)
746{ 872{
747 struct ib_recv_wr rx_wr, *rx_wr_failed; 873 struct ib_recv_wr rx_wr, *rx_wr_failed;
@@ -779,7 +905,7 @@ int iser_post_recvm(struct iser_conn *ib_conn, int count)
779 rx_wr->sg_list = &rx_desc->rx_sg; 905 rx_wr->sg_list = &rx_desc->rx_sg;
780 rx_wr->num_sge = 1; 906 rx_wr->num_sge = 1;
781 rx_wr->next = rx_wr + 1; 907 rx_wr->next = rx_wr + 1;
782 my_rx_head = (my_rx_head + 1) & (ISER_QP_MAX_RECV_DTOS - 1); 908 my_rx_head = (my_rx_head + 1) & ib_conn->qp_max_recv_dtos_mask;
783 } 909 }
784 910
785 rx_wr--; 911 rx_wr--;
@@ -863,7 +989,11 @@ static int iser_drain_tx_cq(struct iser_device *device, int cq_index)
863 if (wc.status == IB_WC_SUCCESS) { 989 if (wc.status == IB_WC_SUCCESS) {
864 if (wc.opcode == IB_WC_SEND) 990 if (wc.opcode == IB_WC_SEND)
865 iser_snd_completion(tx_desc, ib_conn); 991 iser_snd_completion(tx_desc, ib_conn);
866 else 992 else if (wc.opcode == IB_WC_LOCAL_INV ||
993 wc.opcode == IB_WC_FAST_REG_MR) {
994 atomic_dec(&ib_conn->post_send_buf_count);
995 continue;
996 } else
867 iser_err("expected opcode %d got %d\n", 997 iser_err("expected opcode %d got %d\n",
868 IB_WC_SEND, wc.opcode); 998 IB_WC_SEND, wc.opcode);
869 } else { 999 } else {
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-04 23:05:47 -0500