aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arc/include/asm/barrier.h37
-rw-r--r--drivers/infiniband/hw/cxgb4/cm.c89
-rw-r--r--drivers/infiniband/hw/cxgb4/cq.c24
-rw-r--r--drivers/infiniband/hw/cxgb4/device.c41
-rw-r--r--drivers/infiniband/hw/cxgb4/iw_cxgb4.h2
-rw-r--r--drivers/infiniband/hw/cxgb4/mem.c6
-rw-r--r--drivers/infiniband/hw/cxgb4/provider.c2
-rw-r--r--drivers/infiniband/hw/cxgb4/qp.c70
-rw-r--r--drivers/infiniband/hw/cxgb4/resource.c10
-rw-r--r--drivers/infiniband/hw/cxgb4/t4.h72
-rw-r--r--drivers/infiniband/hw/mlx5/main.c2
-rw-r--r--drivers/infiniband/hw/mlx5/qp.c12
-rw-r--r--drivers/infiniband/hw/mthca/mthca_main.c8
-rw-r--r--drivers/infiniband/hw/qib/qib_pcie.c55
-rw-r--r--include/linux/mlx5/device.h1
-rw-r--r--include/linux/mlx5/qp.h1
16 files changed, 270 insertions, 162 deletions
diff --git a/arch/arc/include/asm/barrier.h b/arch/arc/include/asm/barrier.h
deleted file mode 100644
index c32245c3d1e9..000000000000
--- a/arch/arc/include/asm/barrier.h
+++ /dev/null
@@ -1,37 +0,0 @@
1/*
2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9#ifndef __ASM_BARRIER_H
10#define __ASM_BARRIER_H
11
12#ifndef __ASSEMBLY__
13
14/* TODO-vineetg: Need to see what this does, don't we need sync anywhere */
15#define mb() __asm__ __volatile__ ("" : : : "memory")
16#define rmb() mb()
17#define wmb() mb()
18#define set_mb(var, value) do { var = value; mb(); } while (0)
19#define set_wmb(var, value) do { var = value; wmb(); } while (0)
20#define read_barrier_depends() mb()
21
22/* TODO-vineetg verify the correctness of macros here */
23#ifdef CONFIG_SMP
24#define smp_mb() mb()
25#define smp_rmb() rmb()
26#define smp_wmb() wmb()
27#else
28#define smp_mb() barrier()
29#define smp_rmb() barrier()
30#define smp_wmb() barrier()
31#endif
32
33#define smp_read_barrier_depends() do { } while (0)
34
35#endif
36
37#endif
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index 02436d5d0dab..185452abf32c 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -173,12 +173,15 @@ static void start_ep_timer(struct c4iw_ep *ep)
173 add_timer(&ep->timer); 173 add_timer(&ep->timer);
174} 174}
175 175
176static void stop_ep_timer(struct c4iw_ep *ep) 176static int stop_ep_timer(struct c4iw_ep *ep)
177{ 177{
178 PDBG("%s ep %p stopping\n", __func__, ep); 178 PDBG("%s ep %p stopping\n", __func__, ep);
179 del_timer_sync(&ep->timer); 179 del_timer_sync(&ep->timer);
180 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) 180 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) {
181 c4iw_put_ep(&ep->com); 181 c4iw_put_ep(&ep->com);
182 return 0;
183 }
184 return 1;
182} 185}
183 186
184static int c4iw_l2t_send(struct c4iw_rdev *rdev, struct sk_buff *skb, 187static int c4iw_l2t_send(struct c4iw_rdev *rdev, struct sk_buff *skb,
@@ -1165,12 +1168,11 @@ static void process_mpa_reply(struct c4iw_ep *ep, struct sk_buff *skb)
1165 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1168 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1166 1169
1167 /* 1170 /*
1168 * Stop mpa timer. If it expired, then the state has 1171 * Stop mpa timer. If it expired, then
1169 * changed and we bail since ep_timeout already aborted 1172 * we ignore the MPA reply. process_timeout()
1170 * the connection. 1173 * will abort the connection.
1171 */ 1174 */
1172 stop_ep_timer(ep); 1175 if (stop_ep_timer(ep))
1173 if (ep->com.state != MPA_REQ_SENT)
1174 return; 1176 return;
1175 1177
1176 /* 1178 /*
@@ -1375,15 +1377,12 @@ static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb)
1375 1377
1376 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1378 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1377 1379
1378 if (ep->com.state != MPA_REQ_WAIT)
1379 return;
1380
1381 /* 1380 /*
1382 * If we get more than the supported amount of private data 1381 * If we get more than the supported amount of private data
1383 * then we must fail this connection. 1382 * then we must fail this connection.
1384 */ 1383 */
1385 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1384 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) {
1386 stop_ep_timer(ep); 1385 (void)stop_ep_timer(ep);
1387 abort_connection(ep, skb, GFP_KERNEL); 1386 abort_connection(ep, skb, GFP_KERNEL);
1388 return; 1387 return;
1389 } 1388 }
@@ -1413,13 +1412,13 @@ static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb)
1413 if (mpa->revision > mpa_rev) { 1412 if (mpa->revision > mpa_rev) {
1414 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1413 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d,"
1415 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1414 " Received = %d\n", __func__, mpa_rev, mpa->revision);
1416 stop_ep_timer(ep); 1415 (void)stop_ep_timer(ep);
1417 abort_connection(ep, skb, GFP_KERNEL); 1416 abort_connection(ep, skb, GFP_KERNEL);
1418 return; 1417 return;
1419 } 1418 }
1420 1419
1421 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) { 1420 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) {
1422 stop_ep_timer(ep); 1421 (void)stop_ep_timer(ep);
1423 abort_connection(ep, skb, GFP_KERNEL); 1422 abort_connection(ep, skb, GFP_KERNEL);
1424 return; 1423 return;
1425 } 1424 }
@@ -1430,7 +1429,7 @@ static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb)
1430 * Fail if there's too much private data. 1429 * Fail if there's too much private data.
1431 */ 1430 */
1432 if (plen > MPA_MAX_PRIVATE_DATA) { 1431 if (plen > MPA_MAX_PRIVATE_DATA) {
1433 stop_ep_timer(ep); 1432 (void)stop_ep_timer(ep);
1434 abort_connection(ep, skb, GFP_KERNEL); 1433 abort_connection(ep, skb, GFP_KERNEL);
1435 return; 1434 return;
1436 } 1435 }
@@ -1439,7 +1438,7 @@ static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb)
1439 * If plen does not account for pkt size 1438 * If plen does not account for pkt size
1440 */ 1439 */
1441 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1440 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
1442 stop_ep_timer(ep); 1441 (void)stop_ep_timer(ep);
1443 abort_connection(ep, skb, GFP_KERNEL); 1442 abort_connection(ep, skb, GFP_KERNEL);
1444 return; 1443 return;
1445 } 1444 }
@@ -1496,18 +1495,24 @@ static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb)
1496 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1495 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version,
1497 ep->mpa_attr.p2p_type); 1496 ep->mpa_attr.p2p_type);
1498 1497
1499 __state_set(&ep->com, MPA_REQ_RCVD); 1498 /*
1500 stop_ep_timer(ep); 1499 * If the endpoint timer already expired, then we ignore
1501 1500 * the start request. process_timeout() will abort
1502 /* drive upcall */ 1501 * the connection.
1503 mutex_lock(&ep->parent_ep->com.mutex); 1502 */
1504 if (ep->parent_ep->com.state != DEAD) { 1503 if (!stop_ep_timer(ep)) {
1505 if (connect_request_upcall(ep)) 1504 __state_set(&ep->com, MPA_REQ_RCVD);
1505
1506 /* drive upcall */
1507 mutex_lock(&ep->parent_ep->com.mutex);
1508 if (ep->parent_ep->com.state != DEAD) {
1509 if (connect_request_upcall(ep))
1510 abort_connection(ep, skb, GFP_KERNEL);
1511 } else {
1506 abort_connection(ep, skb, GFP_KERNEL); 1512 abort_connection(ep, skb, GFP_KERNEL);
1507 } else { 1513 }
1508 abort_connection(ep, skb, GFP_KERNEL); 1514 mutex_unlock(&ep->parent_ep->com.mutex);
1509 } 1515 }
1510 mutex_unlock(&ep->parent_ep->com.mutex);
1511 return; 1516 return;
1512} 1517}
1513 1518
@@ -2265,7 +2270,7 @@ static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb)
2265 disconnect = 0; 2270 disconnect = 0;
2266 break; 2271 break;
2267 case MORIBUND: 2272 case MORIBUND:
2268 stop_ep_timer(ep); 2273 (void)stop_ep_timer(ep);
2269 if (ep->com.cm_id && ep->com.qp) { 2274 if (ep->com.cm_id && ep->com.qp) {
2270 attrs.next_state = C4IW_QP_STATE_IDLE; 2275 attrs.next_state = C4IW_QP_STATE_IDLE;
2271 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2276 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp,
@@ -2325,10 +2330,10 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb)
2325 case CONNECTING: 2330 case CONNECTING:
2326 break; 2331 break;
2327 case MPA_REQ_WAIT: 2332 case MPA_REQ_WAIT:
2328 stop_ep_timer(ep); 2333 (void)stop_ep_timer(ep);
2329 break; 2334 break;
2330 case MPA_REQ_SENT: 2335 case MPA_REQ_SENT:
2331 stop_ep_timer(ep); 2336 (void)stop_ep_timer(ep);
2332 if (mpa_rev == 1 || (mpa_rev == 2 && ep->tried_with_mpa_v1)) 2337 if (mpa_rev == 1 || (mpa_rev == 2 && ep->tried_with_mpa_v1))
2333 connect_reply_upcall(ep, -ECONNRESET); 2338 connect_reply_upcall(ep, -ECONNRESET);
2334 else { 2339 else {
@@ -2433,7 +2438,7 @@ static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
2433 __state_set(&ep->com, MORIBUND); 2438 __state_set(&ep->com, MORIBUND);
2434 break; 2439 break;
2435 case MORIBUND: 2440 case MORIBUND:
2436 stop_ep_timer(ep); 2441 (void)stop_ep_timer(ep);
2437 if ((ep->com.cm_id) && (ep->com.qp)) { 2442 if ((ep->com.cm_id) && (ep->com.qp)) {
2438 attrs.next_state = C4IW_QP_STATE_IDLE; 2443 attrs.next_state = C4IW_QP_STATE_IDLE;
2439 c4iw_modify_qp(ep->com.qp->rhp, 2444 c4iw_modify_qp(ep->com.qp->rhp,
@@ -3028,7 +3033,7 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp)
3028 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 3033 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) {
3029 close = 1; 3034 close = 1;
3030 if (abrupt) { 3035 if (abrupt) {
3031 stop_ep_timer(ep); 3036 (void)stop_ep_timer(ep);
3032 ep->com.state = ABORTING; 3037 ep->com.state = ABORTING;
3033 } else 3038 } else
3034 ep->com.state = MORIBUND; 3039 ep->com.state = MORIBUND;
@@ -3462,6 +3467,16 @@ static void process_timeout(struct c4iw_ep *ep)
3462 __state_set(&ep->com, ABORTING); 3467 __state_set(&ep->com, ABORTING);
3463 close_complete_upcall(ep, -ETIMEDOUT); 3468 close_complete_upcall(ep, -ETIMEDOUT);
3464 break; 3469 break;
3470 case ABORTING:
3471 case DEAD:
3472
3473 /*
3474 * These states are expected if the ep timed out at the same
3475 * time as another thread was calling stop_ep_timer().
3476 * So we silently do nothing for these states.
3477 */
3478 abort = 0;
3479 break;
3465 default: 3480 default:
3466 WARN(1, "%s unexpected state ep %p tid %u state %u\n", 3481 WARN(1, "%s unexpected state ep %p tid %u state %u\n",
3467 __func__, ep, ep->hwtid, ep->com.state); 3482 __func__, ep, ep->hwtid, ep->com.state);
@@ -3483,6 +3498,8 @@ static void process_timedout_eps(void)
3483 3498
3484 tmp = timeout_list.next; 3499 tmp = timeout_list.next;
3485 list_del(tmp); 3500 list_del(tmp);
3501 tmp->next = NULL;
3502 tmp->prev = NULL;
3486 spin_unlock_irq(&timeout_lock); 3503 spin_unlock_irq(&timeout_lock);
3487 ep = list_entry(tmp, struct c4iw_ep, entry); 3504 ep = list_entry(tmp, struct c4iw_ep, entry);
3488 process_timeout(ep); 3505 process_timeout(ep);
@@ -3499,6 +3516,7 @@ static void process_work(struct work_struct *work)
3499 unsigned int opcode; 3516 unsigned int opcode;
3500 int ret; 3517 int ret;
3501 3518
3519 process_timedout_eps();
3502 while ((skb = skb_dequeue(&rxq))) { 3520 while ((skb = skb_dequeue(&rxq))) {
3503 rpl = cplhdr(skb); 3521 rpl = cplhdr(skb);
3504 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *))); 3522 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *)));
@@ -3508,8 +3526,8 @@ static void process_work(struct work_struct *work)
3508 ret = work_handlers[opcode](dev, skb); 3526 ret = work_handlers[opcode](dev, skb);
3509 if (!ret) 3527 if (!ret)
3510 kfree_skb(skb); 3528 kfree_skb(skb);
3529 process_timedout_eps();
3511 } 3530 }
3512 process_timedout_eps();
3513} 3531}
3514 3532
3515static DECLARE_WORK(skb_work, process_work); 3533static DECLARE_WORK(skb_work, process_work);
@@ -3521,8 +3539,13 @@ static void ep_timeout(unsigned long arg)
3521 3539
3522 spin_lock(&timeout_lock); 3540 spin_lock(&timeout_lock);
3523 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 3541 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) {
3524 list_add_tail(&ep->entry, &timeout_list); 3542 /*
3525 kickit = 1; 3543 * Only insert if it is not already on the list.
3544 */
3545 if (!ep->entry.next) {
3546 list_add_tail(&ep->entry, &timeout_list);
3547 kickit = 1;
3548 }
3526 } 3549 }
3527 spin_unlock(&timeout_lock); 3550 spin_unlock(&timeout_lock);
3528 if (kickit) 3551 if (kickit)
diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c
index ce468e542428..cfaa56ada189 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -235,27 +235,21 @@ int c4iw_flush_sq(struct c4iw_qp *qhp)
235 struct t4_cq *cq = &chp->cq; 235 struct t4_cq *cq = &chp->cq;
236 int idx; 236 int idx;
237 struct t4_swsqe *swsqe; 237 struct t4_swsqe *swsqe;
238 int error = (qhp->attr.state != C4IW_QP_STATE_CLOSING &&
239 qhp->attr.state != C4IW_QP_STATE_IDLE);
240 238
241 if (wq->sq.flush_cidx == -1) 239 if (wq->sq.flush_cidx == -1)
242 wq->sq.flush_cidx = wq->sq.cidx; 240 wq->sq.flush_cidx = wq->sq.cidx;
243 idx = wq->sq.flush_cidx; 241 idx = wq->sq.flush_cidx;
244 BUG_ON(idx >= wq->sq.size); 242 BUG_ON(idx >= wq->sq.size);
245 while (idx != wq->sq.pidx) { 243 while (idx != wq->sq.pidx) {
246 if (error) { 244 swsqe = &wq->sq.sw_sq[idx];
247 swsqe = &wq->sq.sw_sq[idx]; 245 BUG_ON(swsqe->flushed);
248 BUG_ON(swsqe->flushed); 246 swsqe->flushed = 1;
249 swsqe->flushed = 1; 247 insert_sq_cqe(wq, cq, swsqe);
250 insert_sq_cqe(wq, cq, swsqe); 248 if (wq->sq.oldest_read == swsqe) {
251 if (wq->sq.oldest_read == swsqe) { 249 BUG_ON(swsqe->opcode != FW_RI_READ_REQ);
252 BUG_ON(swsqe->opcode != FW_RI_READ_REQ); 250 advance_oldest_read(wq);
253 advance_oldest_read(wq);
254 }
255 flushed++;
256 } else {
257 t4_sq_consume(wq);
258 } 251 }
252 flushed++;
259 if (++idx == wq->sq.size) 253 if (++idx == wq->sq.size)
260 idx = 0; 254 idx = 0;
261 } 255 }
@@ -678,7 +672,7 @@ skip_cqe:
678static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc) 672static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
679{ 673{
680 struct c4iw_qp *qhp = NULL; 674 struct c4iw_qp *qhp = NULL;
681 struct t4_cqe cqe = {0, 0}, *rd_cqe; 675 struct t4_cqe uninitialized_var(cqe), *rd_cqe;
682 struct t4_wq *wq; 676 struct t4_wq *wq;
683 u32 credit = 0; 677 u32 credit = 0;
684 u8 cqe_flushed; 678 u8 cqe_flushed;
diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c
index 9489a388376c..f4fa50a609e2 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -682,7 +682,10 @@ static void c4iw_dealloc(struct uld_ctx *ctx)
682 idr_destroy(&ctx->dev->hwtid_idr); 682 idr_destroy(&ctx->dev->hwtid_idr);
683 idr_destroy(&ctx->dev->stid_idr); 683 idr_destroy(&ctx->dev->stid_idr);
684 idr_destroy(&ctx->dev->atid_idr); 684 idr_destroy(&ctx->dev->atid_idr);
685 iounmap(ctx->dev->rdev.oc_mw_kva); 685 if (ctx->dev->rdev.bar2_kva)
686 iounmap(ctx->dev->rdev.bar2_kva);
687 if (ctx->dev->rdev.oc_mw_kva)
688 iounmap(ctx->dev->rdev.oc_mw_kva);
686 ib_dealloc_device(&ctx->dev->ibdev); 689 ib_dealloc_device(&ctx->dev->ibdev);
687 ctx->dev = NULL; 690 ctx->dev = NULL;
688} 691}
@@ -722,11 +725,31 @@ static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop)
722 } 725 }
723 devp->rdev.lldi = *infop; 726 devp->rdev.lldi = *infop;
724 727
725 devp->rdev.oc_mw_pa = pci_resource_start(devp->rdev.lldi.pdev, 2) + 728 /*
726 (pci_resource_len(devp->rdev.lldi.pdev, 2) - 729 * For T5 devices, we map all of BAR2 with WC.
727 roundup_pow_of_two(devp->rdev.lldi.vr->ocq.size)); 730 * For T4 devices with onchip qp mem, we map only that part
728 devp->rdev.oc_mw_kva = ioremap_wc(devp->rdev.oc_mw_pa, 731 * of BAR2 with WC.
729 devp->rdev.lldi.vr->ocq.size); 732 */
733 devp->rdev.bar2_pa = pci_resource_start(devp->rdev.lldi.pdev, 2);
734 if (is_t5(devp->rdev.lldi.adapter_type)) {
735 devp->rdev.bar2_kva = ioremap_wc(devp->rdev.bar2_pa,
736 pci_resource_len(devp->rdev.lldi.pdev, 2));
737 if (!devp->rdev.bar2_kva) {
738 pr_err(MOD "Unable to ioremap BAR2\n");
739 return ERR_PTR(-EINVAL);
740 }
741 } else if (ocqp_supported(infop)) {
742 devp->rdev.oc_mw_pa =
743 pci_resource_start(devp->rdev.lldi.pdev, 2) +
744 pci_resource_len(devp->rdev.lldi.pdev, 2) -
745 roundup_pow_of_two(devp->rdev.lldi.vr->ocq.size);
746 devp->rdev.oc_mw_kva = ioremap_wc(devp->rdev.oc_mw_pa,
747 devp->rdev.lldi.vr->ocq.size);
748 if (!devp->rdev.oc_mw_kva) {
749 pr_err(MOD "Unable to ioremap onchip mem\n");
750 return ERR_PTR(-EINVAL);
751 }
752 }
730 753
731 PDBG(KERN_INFO MOD "ocq memory: " 754 PDBG(KERN_INFO MOD "ocq memory: "
732 "hw_start 0x%x size %u mw_pa 0x%lx mw_kva %p\n", 755 "hw_start 0x%x size %u mw_pa 0x%lx mw_kva %p\n",
@@ -1003,9 +1026,11 @@ static int enable_qp_db(int id, void *p, void *data)
1003static void resume_rc_qp(struct c4iw_qp *qp) 1026static void resume_rc_qp(struct c4iw_qp *qp)
1004{ 1027{
1005 spin_lock(&qp->lock); 1028 spin_lock(&qp->lock);
1006 t4_ring_sq_db(&qp->wq, qp->wq.sq.wq_pidx_inc); 1029 t4_ring_sq_db(&qp->wq, qp->wq.sq.wq_pidx_inc,
1030 is_t5(qp->rhp->rdev.lldi.adapter_type), NULL);
1007 qp->wq.sq.wq_pidx_inc = 0; 1031 qp->wq.sq.wq_pidx_inc = 0;
1008 t4_ring_rq_db(&qp->wq, qp->wq.rq.wq_pidx_inc); 1032 t4_ring_rq_db(&qp->wq, qp->wq.rq.wq_pidx_inc,
1033 is_t5(qp->rhp->rdev.lldi.adapter_type), NULL);
1009 qp->wq.rq.wq_pidx_inc = 0; 1034 qp->wq.rq.wq_pidx_inc = 0;
1010 spin_unlock(&qp->lock); 1035 spin_unlock(&qp->lock);
1011} 1036}
diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
index e872203c5424..7b8c5806a09d 100644
--- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
+++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
@@ -149,6 +149,8 @@ struct c4iw_rdev {
149 struct gen_pool *ocqp_pool; 149 struct gen_pool *ocqp_pool;
150 u32 flags; 150 u32 flags;
151 struct cxgb4_lld_info lldi; 151 struct cxgb4_lld_info lldi;
152 unsigned long bar2_pa;
153 void __iomem *bar2_kva;
152 unsigned long oc_mw_pa; 154 unsigned long oc_mw_pa;
153 void __iomem *oc_mw_kva; 155 void __iomem *oc_mw_kva;
154 struct c4iw_stats stats; 156 struct c4iw_stats stats;
diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c
index f9ca072a99ed..ec7a2988a703 100644
--- a/drivers/infiniband/hw/cxgb4/mem.c
+++ b/drivers/infiniband/hw/cxgb4/mem.c
@@ -259,8 +259,12 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,
259 259
260 if ((!reset_tpt_entry) && (*stag == T4_STAG_UNSET)) { 260 if ((!reset_tpt_entry) && (*stag == T4_STAG_UNSET)) {
261 stag_idx = c4iw_get_resource(&rdev->resource.tpt_table); 261 stag_idx = c4iw_get_resource(&rdev->resource.tpt_table);
262 if (!stag_idx) 262 if (!stag_idx) {
263 mutex_lock(&rdev->stats.lock);
264 rdev->stats.stag.fail++;
265 mutex_unlock(&rdev->stats.lock);
263 return -ENOMEM; 266 return -ENOMEM;
267 }
264 mutex_lock(&rdev->stats.lock); 268 mutex_lock(&rdev->stats.lock);
265 rdev->stats.stag.cur += 32; 269 rdev->stats.stag.cur += 32;
266 if (rdev->stats.stag.cur > rdev->stats.stag.max) 270 if (rdev->stats.stag.cur > rdev->stats.stag.max)
diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index 79429256023a..a94a3e12c349 100644
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -328,7 +328,7 @@ static int c4iw_query_device(struct ib_device *ibdev,
328 props->max_mr = c4iw_num_stags(&dev->rdev); 328 props->max_mr = c4iw_num_stags(&dev->rdev);
329 props->max_pd = T4_MAX_NUM_PD; 329 props->max_pd = T4_MAX_NUM_PD;
330 props->local_ca_ack_delay = 0; 330 props->local_ca_ack_delay = 0;
331 props->max_fast_reg_page_list_len = T4_MAX_FR_DEPTH; 331 props->max_fast_reg_page_list_len = t4_max_fr_depth(use_dsgl);
332 332
333 return 0; 333 return 0;
334} 334}
diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c
index cb76eb5eee1f..7b5114cb486f 100644
--- a/drivers/infiniband/hw/cxgb4/qp.c
+++ b/drivers/infiniband/hw/cxgb4/qp.c
@@ -212,13 +212,23 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
212 212
213 wq->db = rdev->lldi.db_reg; 213 wq->db = rdev->lldi.db_reg;
214 wq->gts = rdev->lldi.gts_reg; 214 wq->gts = rdev->lldi.gts_reg;
215 if (user) { 215 if (user || is_t5(rdev->lldi.adapter_type)) {
216 wq->sq.udb = (u64)pci_resource_start(rdev->lldi.pdev, 2) + 216 u32 off;
217 (wq->sq.qid << rdev->qpshift); 217
218 wq->sq.udb &= PAGE_MASK; 218 off = (wq->sq.qid << rdev->qpshift) & PAGE_MASK;
219 wq->rq.udb = (u64)pci_resource_start(rdev->lldi.pdev, 2) + 219 if (user) {
220 (wq->rq.qid << rdev->qpshift); 220 wq->sq.udb = (u64 __iomem *)(rdev->bar2_pa + off);
221 wq->rq.udb &= PAGE_MASK; 221 } else {
222 off += 128 * (wq->sq.qid & rdev->qpmask) + 8;
223 wq->sq.udb = (u64 __iomem *)(rdev->bar2_kva + off);
224 }
225 off = (wq->rq.qid << rdev->qpshift) & PAGE_MASK;
226 if (user) {
227 wq->rq.udb = (u64 __iomem *)(rdev->bar2_pa + off);
228 } else {
229 off += 128 * (wq->rq.qid & rdev->qpmask) + 8;
230 wq->rq.udb = (u64 __iomem *)(rdev->bar2_kva + off);
231 }
222 } 232 }
223 wq->rdev = rdev; 233 wq->rdev = rdev;
224 wq->rq.msn = 1; 234 wq->rq.msn = 1;
@@ -299,9 +309,10 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq,
299 if (ret) 309 if (ret)
300 goto free_dma; 310 goto free_dma;
301 311
302 PDBG("%s sqid 0x%x rqid 0x%x kdb 0x%p squdb 0x%llx rqudb 0x%llx\n", 312 PDBG("%s sqid 0x%x rqid 0x%x kdb 0x%p squdb 0x%lx rqudb 0x%lx\n",
303 __func__, wq->sq.qid, wq->rq.qid, wq->db, 313 __func__, wq->sq.qid, wq->rq.qid, wq->db,
304 (unsigned long long)wq->sq.udb, (unsigned long long)wq->rq.udb); 314 (__force unsigned long) wq->sq.udb,
315 (__force unsigned long) wq->rq.udb);
305 316
306 return 0; 317 return 0;
307free_dma: 318free_dma:
@@ -425,6 +436,8 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe,
425 default: 436 default:
426 return -EINVAL; 437 return -EINVAL;
427 } 438 }
439 wqe->send.r3 = 0;
440 wqe->send.r4 = 0;
428 441
429 plen = 0; 442 plen = 0;
430 if (wr->num_sge) { 443 if (wr->num_sge) {
@@ -555,7 +568,8 @@ static int build_fastreg(struct t4_sq *sq, union t4_wr *wqe,
555 int pbllen = roundup(wr->wr.fast_reg.page_list_len * sizeof(u64), 32); 568 int pbllen = roundup(wr->wr.fast_reg.page_list_len * sizeof(u64), 32);
556 int rem; 569 int rem;
557 570
558 if (wr->wr.fast_reg.page_list_len > T4_MAX_FR_DEPTH) 571 if (wr->wr.fast_reg.page_list_len >
572 t4_max_fr_depth(use_dsgl))
559 return -EINVAL; 573 return -EINVAL;
560 574
561 wqe->fr.qpbinde_to_dcacpu = 0; 575 wqe->fr.qpbinde_to_dcacpu = 0;
@@ -650,9 +664,10 @@ static int ring_kernel_sq_db(struct c4iw_qp *qhp, u16 inc)
650 664
651 spin_lock_irqsave(&qhp->rhp->lock, flags); 665 spin_lock_irqsave(&qhp->rhp->lock, flags);
652 spin_lock(&qhp->lock); 666 spin_lock(&qhp->lock);
653 if (qhp->rhp->db_state == NORMAL) { 667 if (qhp->rhp->db_state == NORMAL)
654 t4_ring_sq_db(&qhp->wq, inc); 668 t4_ring_sq_db(&qhp->wq, inc,
655 } else { 669 is_t5(qhp->rhp->rdev.lldi.adapter_type), NULL);
670 else {
656 add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry); 671 add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry);
657 qhp->wq.sq.wq_pidx_inc += inc; 672 qhp->wq.sq.wq_pidx_inc += inc;
658 } 673 }
@@ -667,9 +682,10 @@ static int ring_kernel_rq_db(struct c4iw_qp *qhp, u16 inc)
667 682
668 spin_lock_irqsave(&qhp->rhp->lock, flags); 683 spin_lock_irqsave(&qhp->rhp->lock, flags);
669 spin_lock(&qhp->lock); 684 spin_lock(&qhp->lock);
670 if (qhp->rhp->db_state == NORMAL) { 685 if (qhp->rhp->db_state == NORMAL)
671 t4_ring_rq_db(&qhp->wq, inc); 686 t4_ring_rq_db(&qhp->wq, inc,
672 } else { 687 is_t5(qhp->rhp->rdev.lldi.adapter_type), NULL);
688 else {
673 add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry); 689 add_to_fc_list(&qhp->rhp->db_fc_list, &qhp->db_fc_entry);
674 qhp->wq.rq.wq_pidx_inc += inc; 690 qhp->wq.rq.wq_pidx_inc += inc;
675 } 691 }
@@ -686,7 +702,7 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
686 enum fw_wr_opcodes fw_opcode = 0; 702 enum fw_wr_opcodes fw_opcode = 0;
687 enum fw_ri_wr_flags fw_flags; 703 enum fw_ri_wr_flags fw_flags;
688 struct c4iw_qp *qhp; 704 struct c4iw_qp *qhp;
689 union t4_wr *wqe; 705 union t4_wr *wqe = NULL;
690 u32 num_wrs; 706 u32 num_wrs;
691 struct t4_swsqe *swsqe; 707 struct t4_swsqe *swsqe;
692 unsigned long flag; 708 unsigned long flag;
@@ -792,7 +808,8 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
792 idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); 808 idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);
793 } 809 }
794 if (!qhp->rhp->rdev.status_page->db_off) { 810 if (!qhp->rhp->rdev.status_page->db_off) {
795 t4_ring_sq_db(&qhp->wq, idx); 811 t4_ring_sq_db(&qhp->wq, idx,
812 is_t5(qhp->rhp->rdev.lldi.adapter_type), wqe);
796 spin_unlock_irqrestore(&qhp->lock, flag); 813 spin_unlock_irqrestore(&qhp->lock, flag);
797 } else { 814 } else {
798 spin_unlock_irqrestore(&qhp->lock, flag); 815 spin_unlock_irqrestore(&qhp->lock, flag);
@@ -806,7 +823,7 @@ int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
806{ 823{
807 int err = 0; 824 int err = 0;
808 struct c4iw_qp *qhp; 825 struct c4iw_qp *qhp;
809 union t4_recv_wr *wqe; 826 union t4_recv_wr *wqe = NULL;
810 u32 num_wrs; 827 u32 num_wrs;
811 u8 len16 = 0; 828 u8 len16 = 0;
812 unsigned long flag; 829 unsigned long flag;
@@ -858,7 +875,8 @@ int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
858 num_wrs--; 875 num_wrs--;
859 } 876 }
860 if (!qhp->rhp->rdev.status_page->db_off) { 877 if (!qhp->rhp->rdev.status_page->db_off) {
861 t4_ring_rq_db(&qhp->wq, idx); 878 t4_ring_rq_db(&qhp->wq, idx,
879 is_t5(qhp->rhp->rdev.lldi.adapter_type), wqe);
862 spin_unlock_irqrestore(&qhp->lock, flag); 880 spin_unlock_irqrestore(&qhp->lock, flag);
863 } else { 881 } else {
864 spin_unlock_irqrestore(&qhp->lock, flag); 882 spin_unlock_irqrestore(&qhp->lock, flag);
@@ -1352,6 +1370,7 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
1352 switch (attrs->next_state) { 1370 switch (attrs->next_state) {
1353 case C4IW_QP_STATE_CLOSING: 1371 case C4IW_QP_STATE_CLOSING:
1354 BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); 1372 BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2);
1373 t4_set_wq_in_error(&qhp->wq);
1355 set_state(qhp, C4IW_QP_STATE_CLOSING); 1374 set_state(qhp, C4IW_QP_STATE_CLOSING);
1356 ep = qhp->ep; 1375 ep = qhp->ep;
1357 if (!internal) { 1376 if (!internal) {
@@ -1359,18 +1378,18 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
1359 disconnect = 1; 1378 disconnect = 1;
1360 c4iw_get_ep(&qhp->ep->com); 1379 c4iw_get_ep(&qhp->ep->com);
1361 } 1380 }
1362 t4_set_wq_in_error(&qhp->wq);
1363 ret = rdma_fini(rhp, qhp, ep); 1381 ret = rdma_fini(rhp, qhp, ep);
1364 if (ret) 1382 if (ret)
1365 goto err; 1383 goto err;
1366 break; 1384 break;
1367 case C4IW_QP_STATE_TERMINATE: 1385 case C4IW_QP_STATE_TERMINATE:
1386 t4_set_wq_in_error(&qhp->wq);
1368 set_state(qhp, C4IW_QP_STATE_TERMINATE); 1387 set_state(qhp, C4IW_QP_STATE_TERMINATE);
1369 qhp->attr.layer_etype = attrs->layer_etype; 1388 qhp->attr.layer_etype = attrs->layer_etype;
1370 qhp->attr.ecode = attrs->ecode; 1389 qhp->attr.ecode = attrs->ecode;
1371 t4_set_wq_in_error(&qhp->wq);
1372 ep = qhp->ep; 1390 ep = qhp->ep;
1373 disconnect = 1; 1391 disconnect = 1;
1392 c4iw_get_ep(&qhp->ep->com);
1374 if (!internal) 1393 if (!internal)
1375 terminate = 1; 1394 terminate = 1;
1376 else { 1395 else {
@@ -1378,11 +1397,10 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
1378 if (ret) 1397 if (ret)
1379 goto err; 1398 goto err;
1380 } 1399 }
1381 c4iw_get_ep(&qhp->ep->com);
1382 break; 1400 break;
1383 case C4IW_QP_STATE_ERROR: 1401 case C4IW_QP_STATE_ERROR:
1384 set_state(qhp, C4IW_QP_STATE_ERROR);
1385 t4_set_wq_in_error(&qhp->wq); 1402 t4_set_wq_in_error(&qhp->wq);
1403 set_state(qhp, C4IW_QP_STATE_ERROR);
1386 if (!internal) { 1404 if (!internal) {
1387 abort = 1; 1405 abort = 1;
1388 disconnect = 1; 1406 disconnect = 1;
@@ -1677,11 +1695,11 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
1677 mm2->len = PAGE_ALIGN(qhp->wq.rq.memsize); 1695 mm2->len = PAGE_ALIGN(qhp->wq.rq.memsize);
1678 insert_mmap(ucontext, mm2); 1696 insert_mmap(ucontext, mm2);
1679 mm3->key = uresp.sq_db_gts_key; 1697 mm3->key = uresp.sq_db_gts_key;
1680 mm3->addr = qhp->wq.sq.udb; 1698 mm3->addr = (__force unsigned long) qhp->wq.sq.udb;
1681 mm3->len = PAGE_SIZE; 1699 mm3->len = PAGE_SIZE;
1682 insert_mmap(ucontext, mm3); 1700 insert_mmap(ucontext, mm3);
1683 mm4->key = uresp.rq_db_gts_key; 1701 mm4->key = uresp.rq_db_gts_key;
1684 mm4->addr = qhp->wq.rq.udb; 1702 mm4->addr = (__force unsigned long) qhp->wq.rq.udb;
1685 mm4->len = PAGE_SIZE; 1703 mm4->len = PAGE_SIZE;
1686 insert_mmap(ucontext, mm4); 1704 insert_mmap(ucontext, mm4);
1687 if (mm5) { 1705 if (mm5) {
diff --git a/drivers/infiniband/hw/cxgb4/resource.c b/drivers/infiniband/hw/cxgb4/resource.c
index cdef4d7fb6d8..67df71a7012e 100644
--- a/drivers/infiniband/hw/cxgb4/resource.c
+++ b/drivers/infiniband/hw/cxgb4/resource.c
@@ -179,8 +179,12 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
179 kfree(entry); 179 kfree(entry);
180 } else { 180 } else {
181 qid = c4iw_get_resource(&rdev->resource.qid_table); 181 qid = c4iw_get_resource(&rdev->resource.qid_table);
182 if (!qid) 182 if (!qid) {
183 mutex_lock(&rdev->stats.lock);
184 rdev->stats.qid.fail++;
185 mutex_unlock(&rdev->stats.lock);
183 goto out; 186 goto out;
187 }
184 mutex_lock(&rdev->stats.lock); 188 mutex_lock(&rdev->stats.lock);
185 rdev->stats.qid.cur += rdev->qpmask + 1; 189 rdev->stats.qid.cur += rdev->qpmask + 1;
186 mutex_unlock(&rdev->stats.lock); 190 mutex_unlock(&rdev->stats.lock);
@@ -322,8 +326,8 @@ u32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size)
322 unsigned long addr = gen_pool_alloc(rdev->rqt_pool, size << 6); 326 unsigned long addr = gen_pool_alloc(rdev->rqt_pool, size << 6);
323 PDBG("%s addr 0x%x size %d\n", __func__, (u32)addr, size << 6); 327 PDBG("%s addr 0x%x size %d\n", __func__, (u32)addr, size << 6);
324 if (!addr) 328 if (!addr)
325 printk_ratelimited(KERN_WARNING MOD "%s: Out of RQT memory\n", 329 pr_warn_ratelimited(MOD "%s: Out of RQT memory\n",
326 pci_name(rdev->lldi.pdev)); 330 pci_name(rdev->lldi.pdev));
327 mutex_lock(&rdev->stats.lock); 331 mutex_lock(&rdev->stats.lock);
328 if (addr) { 332 if (addr) {
329 rdev->stats.rqt.cur += roundup(size << 6, 1 << MIN_RQT_SHIFT); 333 rdev->stats.rqt.cur += roundup(size << 6, 1 << MIN_RQT_SHIFT);
diff --git a/drivers/infiniband/hw/cxgb4/t4.h b/drivers/infiniband/hw/cxgb4/t4.h
index eeca8b1e6376..2178f3198410 100644
--- a/drivers/infiniband/hw/cxgb4/t4.h
+++ b/drivers/infiniband/hw/cxgb4/t4.h
@@ -84,7 +84,14 @@ struct t4_status_page {
84 sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge)) 84 sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge))
85#define T4_MAX_FR_IMMD ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_fr_nsmr_wr) - \ 85#define T4_MAX_FR_IMMD ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_fr_nsmr_wr) - \
86 sizeof(struct fw_ri_immd)) & ~31UL) 86 sizeof(struct fw_ri_immd)) & ~31UL)
87#define T4_MAX_FR_DEPTH (1024 / sizeof(u64)) 87#define T4_MAX_FR_IMMD_DEPTH (T4_MAX_FR_IMMD / sizeof(u64))
88#define T4_MAX_FR_DSGL 1024
89#define T4_MAX_FR_DSGL_DEPTH (T4_MAX_FR_DSGL / sizeof(u64))
90
91static inline int t4_max_fr_depth(int use_dsgl)
92{
93 return use_dsgl ? T4_MAX_FR_DSGL_DEPTH : T4_MAX_FR_IMMD_DEPTH;
94}
88 95
89#define T4_RQ_NUM_SLOTS 2 96#define T4_RQ_NUM_SLOTS 2
90#define T4_RQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_RQ_NUM_SLOTS) 97#define T4_RQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_RQ_NUM_SLOTS)
@@ -292,7 +299,7 @@ struct t4_sq {
292 unsigned long phys_addr; 299 unsigned long phys_addr;
293 struct t4_swsqe *sw_sq; 300 struct t4_swsqe *sw_sq;
294 struct t4_swsqe *oldest_read; 301 struct t4_swsqe *oldest_read;
295 u64 udb; 302 u64 __iomem *udb;
296 size_t memsize; 303 size_t memsize;
297 u32 qid; 304 u32 qid;
298 u16 in_use; 305 u16 in_use;
@@ -314,7 +321,7 @@ struct t4_rq {
314 dma_addr_t dma_addr; 321 dma_addr_t dma_addr;
315 DEFINE_DMA_UNMAP_ADDR(mapping); 322 DEFINE_DMA_UNMAP_ADDR(mapping);
316 struct t4_swrqe *sw_rq; 323 struct t4_swrqe *sw_rq;
317 u64 udb; 324 u64 __iomem *udb;
318 size_t memsize; 325 size_t memsize;
319 u32 qid; 326 u32 qid;
320 u32 msn; 327 u32 msn;
@@ -435,15 +442,67 @@ static inline u16 t4_sq_wq_size(struct t4_wq *wq)
435 return wq->sq.size * T4_SQ_NUM_SLOTS; 442 return wq->sq.size * T4_SQ_NUM_SLOTS;
436} 443}
437 444
438static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc) 445/* This function copies 64 byte coalesced work request to memory
446 * mapped BAR2 space. For coalesced WRs, the SGE fetches data
447 * from the FIFO instead of from Host.
448 */
449static inline void pio_copy(u64 __iomem *dst, u64 *src)
450{
451 int count = 8;
452
453 while (count) {
454 writeq(*src, dst);
455 src++;
456 dst++;
457 count--;
458 }
459}
460
461static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc, u8 t5,
462 union t4_wr *wqe)
439{ 463{
464
465 /* Flush host queue memory writes. */
440 wmb(); 466 wmb();
467 if (t5) {
468 if (inc == 1 && wqe) {
469 PDBG("%s: WC wq->sq.pidx = %d\n",
470 __func__, wq->sq.pidx);
471 pio_copy(wq->sq.udb + 7, (void *)wqe);
472 } else {
473 PDBG("%s: DB wq->sq.pidx = %d\n",
474 __func__, wq->sq.pidx);
475 writel(PIDX_T5(inc), wq->sq.udb);
476 }
477
478 /* Flush user doorbell area writes. */
479 wmb();
480 return;
481 }
441 writel(QID(wq->sq.qid) | PIDX(inc), wq->db); 482 writel(QID(wq->sq.qid) | PIDX(inc), wq->db);
442} 483}
443 484
444static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc) 485static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc, u8 t5,
486 union t4_recv_wr *wqe)
445{ 487{
488
489 /* Flush host queue memory writes. */
446 wmb(); 490 wmb();
491 if (t5) {
492 if (inc == 1 && wqe) {
493 PDBG("%s: WC wq->rq.pidx = %d\n",
494 __func__, wq->rq.pidx);
495 pio_copy(wq->rq.udb + 7, (void *)wqe);
496 } else {
497 PDBG("%s: DB wq->rq.pidx = %d\n",
498 __func__, wq->rq.pidx);
499 writel(PIDX_T5(inc), wq->rq.udb);
500 }
501
502 /* Flush user doorbell area writes. */
503 wmb();
504 return;
505 }
447 writel(QID(wq->rq.qid) | PIDX(inc), wq->db); 506 writel(QID(wq->rq.qid) | PIDX(inc), wq->db);
448} 507}
449 508
@@ -568,6 +627,9 @@ static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
568 printk(KERN_ERR MOD "cq overflow cqid %u\n", cq->cqid); 627 printk(KERN_ERR MOD "cq overflow cqid %u\n", cq->cqid);
569 BUG_ON(1); 628 BUG_ON(1);
570 } else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) { 629 } else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) {
630
631 /* Ensure CQE is flushed to memory */
632 rmb();
571 *cqe = &cq->queue[cq->cidx]; 633 *cqe = &cq->queue[cq->cidx];
572 ret = 0; 634 ret = 0;
573 } else 635 } else
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index fa6dc870adae..364d4b6937f5 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -282,6 +282,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
282 props->sig_guard_cap = IB_GUARD_T10DIF_CRC | 282 props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
283 IB_GUARD_T10DIF_CSUM; 283 IB_GUARD_T10DIF_CSUM;
284 } 284 }
285 if (flags & MLX5_DEV_CAP_FLAG_BLOCK_MCAST)
286 props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
285 287
286 props->vendor_id = be32_to_cpup((__be32 *)(out_mad->data + 36)) & 288 props->vendor_id = be32_to_cpup((__be32 *)(out_mad->data + 36)) &
287 0xffffff; 289 0xffffff;
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index ae788d27b93f..dc930ed21eca 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -807,6 +807,15 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
807 spin_lock_init(&qp->sq.lock); 807 spin_lock_init(&qp->sq.lock);
808 spin_lock_init(&qp->rq.lock); 808 spin_lock_init(&qp->rq.lock);
809 809
810 if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
811 if (!(dev->mdev.caps.flags & MLX5_DEV_CAP_FLAG_BLOCK_MCAST)) {
812 mlx5_ib_dbg(dev, "block multicast loopback isn't supported\n");
813 return -EINVAL;
814 } else {
815 qp->flags |= MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK;
816 }
817 }
818
810 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) 819 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
811 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE; 820 qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
812 821
@@ -878,6 +887,9 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
878 if (qp->wq_sig) 887 if (qp->wq_sig)
879 in->ctx.flags_pd |= cpu_to_be32(MLX5_QP_ENABLE_SIG); 888 in->ctx.flags_pd |= cpu_to_be32(MLX5_QP_ENABLE_SIG);
880 889
890 if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK)
891 in->ctx.flags_pd |= cpu_to_be32(MLX5_QP_BLOCK_MCAST);
892
881 if (qp->scat_cqe && is_connected(init_attr->qp_type)) { 893 if (qp->scat_cqe && is_connected(init_attr->qp_type)) {
882 int rcqe_sz; 894 int rcqe_sz;
883 int scqe_sz; 895 int scqe_sz;
diff --git a/drivers/infiniband/hw/mthca/mthca_main.c b/drivers/infiniband/hw/mthca/mthca_main.c
index 87897b95666d..ded76c101dde 100644
--- a/drivers/infiniband/hw/mthca/mthca_main.c
+++ b/drivers/infiniband/hw/mthca/mthca_main.c
@@ -858,13 +858,9 @@ static int mthca_enable_msi_x(struct mthca_dev *mdev)
858 entries[1].entry = 1; 858 entries[1].entry = 1;
859 entries[2].entry = 2; 859 entries[2].entry = 2;
860 860
861 err = pci_enable_msix(mdev->pdev, entries, ARRAY_SIZE(entries)); 861 err = pci_enable_msix_exact(mdev->pdev, entries, ARRAY_SIZE(entries));
862 if (err) { 862 if (err)
863 if (err > 0)
864 mthca_info(mdev, "Only %d MSI-X vectors available, "
865 "not using MSI-X\n", err);
866 return err; 863 return err;
867 }
868 864
869 mdev->eq_table.eq[MTHCA_EQ_COMP ].msi_x_vector = entries[0].vector; 865 mdev->eq_table.eq[MTHCA_EQ_COMP ].msi_x_vector = entries[0].vector;
870 mdev->eq_table.eq[MTHCA_EQ_ASYNC].msi_x_vector = entries[1].vector; 866 mdev->eq_table.eq[MTHCA_EQ_ASYNC].msi_x_vector = entries[1].vector;
diff --git a/drivers/infiniband/hw/qib/qib_pcie.c b/drivers/infiniband/hw/qib/qib_pcie.c
index c8d9c4ab142b..61a0046efb76 100644
--- a/drivers/infiniband/hw/qib/qib_pcie.c
+++ b/drivers/infiniband/hw/qib/qib_pcie.c
@@ -197,46 +197,47 @@ static void qib_msix_setup(struct qib_devdata *dd, int pos, u32 *msixcnt,
197 struct qib_msix_entry *qib_msix_entry) 197 struct qib_msix_entry *qib_msix_entry)
198{ 198{
199 int ret; 199 int ret;
200 u32 tabsize = 0; 200 int nvec = *msixcnt;
201 u16 msix_flags;
202 struct msix_entry *msix_entry; 201 struct msix_entry *msix_entry;
203 int i; 202 int i;
204 203
204 ret = pci_msix_vec_count(dd->pcidev);
205 if (ret < 0)
206 goto do_intx;
207
208 nvec = min(nvec, ret);
209
205 /* We can't pass qib_msix_entry array to qib_msix_setup 210 /* We can't pass qib_msix_entry array to qib_msix_setup
206 * so use a dummy msix_entry array and copy the allocated 211 * so use a dummy msix_entry array and copy the allocated
207 * irq back to the qib_msix_entry array. */ 212 * irq back to the qib_msix_entry array. */
208 msix_entry = kmalloc(*msixcnt * sizeof(*msix_entry), GFP_KERNEL); 213 msix_entry = kmalloc(nvec * sizeof(*msix_entry), GFP_KERNEL);
209 if (!msix_entry) { 214 if (!msix_entry)
210 ret = -ENOMEM;
211 goto do_intx; 215 goto do_intx;
212 } 216
213 for (i = 0; i < *msixcnt; i++) 217 for (i = 0; i < nvec; i++)
214 msix_entry[i] = qib_msix_entry[i].msix; 218 msix_entry[i] = qib_msix_entry[i].msix;
215 219
216 pci_read_config_word(dd->pcidev, pos + PCI_MSIX_FLAGS, &msix_flags); 220 ret = pci_enable_msix_range(dd->pcidev, msix_entry, 1, nvec);
217 tabsize = 1 + (msix_flags & PCI_MSIX_FLAGS_QSIZE); 221 if (ret < 0)
218 if (tabsize > *msixcnt) 222 goto free_msix_entry;
219 tabsize = *msixcnt; 223 else
220 ret = pci_enable_msix(dd->pcidev, msix_entry, tabsize); 224 nvec = ret;
221 if (ret > 0) { 225
222 tabsize = ret; 226 for (i = 0; i < nvec; i++)
223 ret = pci_enable_msix(dd->pcidev, msix_entry, tabsize);
224 }
225do_intx:
226 if (ret) {
227 qib_dev_err(dd,
228 "pci_enable_msix %d vectors failed: %d, falling back to INTx\n",
229 tabsize, ret);
230 tabsize = 0;
231 }
232 for (i = 0; i < tabsize; i++)
233 qib_msix_entry[i].msix = msix_entry[i]; 227 qib_msix_entry[i].msix = msix_entry[i];
228
234 kfree(msix_entry); 229 kfree(msix_entry);
235 *msixcnt = tabsize; 230 *msixcnt = nvec;
231 return;
236 232
237 if (ret) 233free_msix_entry:
238 qib_enable_intx(dd->pcidev); 234 kfree(msix_entry);
239 235
236do_intx:
237 qib_dev_err(dd, "pci_enable_msix_range %d vectors failed: %d, "
238 "falling back to INTx\n", nvec, ret);
239 *msixcnt = 0;
240 qib_enable_intx(dd->pcidev);
240} 241}
241 242
242/** 243/**
diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index 407bdb67fd4f..3406cfb1267a 100644
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -179,6 +179,7 @@ enum {
179 MLX5_DEV_CAP_FLAG_BAD_QKEY_CNTR = 1LL << 9, 179 MLX5_DEV_CAP_FLAG_BAD_QKEY_CNTR = 1LL << 9,
180 MLX5_DEV_CAP_FLAG_APM = 1LL << 17, 180 MLX5_DEV_CAP_FLAG_APM = 1LL << 17,
181 MLX5_DEV_CAP_FLAG_ATOMIC = 1LL << 18, 181 MLX5_DEV_CAP_FLAG_ATOMIC = 1LL << 18,
182 MLX5_DEV_CAP_FLAG_BLOCK_MCAST = 1LL << 23,
182 MLX5_DEV_CAP_FLAG_ON_DMND_PG = 1LL << 24, 183 MLX5_DEV_CAP_FLAG_ON_DMND_PG = 1LL << 24,
183 MLX5_DEV_CAP_FLAG_CQ_MODER = 1LL << 29, 184 MLX5_DEV_CAP_FLAG_CQ_MODER = 1LL << 29,
184 MLX5_DEV_CAP_FLAG_RESIZE_CQ = 1LL << 30, 185 MLX5_DEV_CAP_FLAG_RESIZE_CQ = 1LL << 30,
diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h
index f829ad80ff28..9709b30e2d69 100644
--- a/include/linux/mlx5/qp.h
+++ b/include/linux/mlx5/qp.h
@@ -146,6 +146,7 @@ enum {
146 146
147enum { 147enum {
148 MLX5_QP_LAT_SENSITIVE = 1 << 28, 148 MLX5_QP_LAT_SENSITIVE = 1 << 28,
149 MLX5_QP_BLOCK_MCAST = 1 << 30,
149 MLX5_QP_ENABLE_SIG = 1 << 31, 150 MLX5_QP_ENABLE_SIG = 1 << 31,
150}; 151};
151 152
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-22 18:37:04 -0400